2022-09-29T05:13:41.355 INFO:root:teuthology version: 1.1.1.dev427+g35ea38a9 2022-09-29T05:13:41.358 DEBUG:teuthology.run:Teuthology command: teuthology --owner scheduled_lflores@teuthology --description upgrade:telemetry-upgrade/pacific-x/{0-random-distro$/{centos_8.stream_container_tools} 0-start 1-tasks} --archive /home/teuthworker/archive/lflores-2022-09-29_05:03:04-upgrade:telemetry-upgrade-main-distro-default-smithi/7047304 --verbose --name lflores-2022-09-29_05:03:04-upgrade:telemetry-upgrade-main-distro-default-smithi -- /home/teuthworker/archive/lflores-2022-09-29_05:03:04-upgrade:telemetry-upgrade-main-distro-default-smithi/7047304/orig.config.yaml 2022-09-29T05:13:41.431 DEBUG:teuthology.report:Pushing job info to http://paddles.front.sepia.ceph.com/ 2022-09-29T05:13:41.520 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/lflores-2022-09-29_05:03:04-upgrade:telemetry-upgrade-main-distro-default-smithi/7047304 branch: main description: upgrade:telemetry-upgrade/pacific-x/{0-random-distro$/{centos_8.stream_container_tools} 0-start 1-tasks} email: null first_in_suite: false job_id: '7047304' kernel: kdb: true sha1: distro last_in_suite: false machine_type: smithi name: lflores-2022-09-29_05:03:04-upgrade:telemetry-upgrade-main-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 8.stream overrides: admin_socket: branch: main ceph: conf: mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: 652bf75409c45c34ae939b241b00adba20f12869 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: osd default pool size: 2 install: ceph: flavor: default sha1: 652bf75409c45c34ae939b241b00adba20f12869 selinux: whitelist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: wip-upgrade-telemetry sha1: dbaedf5bf154c5830301d16d6da9fb01a6c0c0c5 owner: scheduled_lflores@teuthology priority: 70 repo: git://git.ceph.com/ceph-ci.git roles: - - mon.a - mon.c - mgr.y - mds.a - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mds.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b sha1: 652bf75409c45c34ae939b241b00adba20f12869 sleep_before_teardown: 0 suite: upgrade:telemetry-upgrade suite_branch: wip-upgrade-telemetry suite_path: /home/teuthworker/src/github.com_ljflores_ceph_dbaedf5bf154c5830301d16d6da9fb01a6c0c0c5/qa suite_relpath: qa suite_repo: https://github.com/ljflores/ceph.git suite_sha1: dbaedf5bf154c5830301d16d6da9fb01a6c0c0c5 targets: smithi083.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDr/JhWmtHMEekWBV5bEXMmOwblnHnXCWHjG06+j5Chpx4UPoXAVHlbwqsRH7KkkeULX7+1p3Jzx0SBz3q3ONlw5vwO7o2wcwG0fTICEx5hqpP4r5NFcH2Muqx57f51Ddj5y8DryGBPbTSy8kEMGChegFc3N7blbqSvKby6nccdPMq+NtY1hlP8i+3xouplad6tcwMoNyndv5r+5M5u0FbP5XyxX1K5UtsQTTIQ137JD5SvwhjdTxIS5qPPoxWY0+bbP8kg49bjP/OOT7+gksHRVzSoYAgbZB4dTR7kkd5SHvSwpB1A3LJxVcd9YIHOy6Jn1o3qnK3kGkfsxGoPs6tw5DlhFPKj6EP2cFsJ+uPLHOun49IDnxbrhZ0UVj2TPhBcEOTeCyZfu9VTfCxbaJXfNIjL7G9OXZImphZfOUc1Q2jG+i+CS3NEtgsYY4ZpHyQkIs8Gkhs4mJSE5a/7tfdCzfPakBx5BHAco9wnIHV4h5eQfzB8XKAKnEO7aSp5pns= smithi153.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDuCDE7MiWnaxqnxBCgmaVlbjlAJWjWK+frWrSIni4eyA0GtFuaGewEAK8E3cD/4NxiW5Qr9Yjlp9lM6HUy8LUVaeBEkcV6d28Rt4opCrmBPTZRth14miAHFs3yNPYrPpgIPh50EgU+0pomANhRrwlwSrpnT43+TcRBwuQDKje2Xc9QCP2/HyhbV4khamABVGnpv6r3x2UBuZrWKaEVWKzcDsLmavjvWEjgSaGY8PMmId1hJ1VhMjkPQ7Kc6HXDrNLcg88r2gONCa51rvE6PEm/lWioALqa0Is4567gz4jWAQcoK6QrRghBDDC2CEP0+KVBrI8fkXhKzqPLJEBDme9zGOdsPhS396DCo0YjRtwDvJgRedtdiB1rA93qCoF219+a2J3NXEN4jxVZLIzRN1eW0WE6Oh/FsSEo/vuhWNSsupeWc8ima2fX4M2HSuZ3i0Z+ct3ynDUtw0TyWmhMC21Z9z17hI4aefklEVM/26iZ9XAzp7OaUvPA4q+EYG2XTIU= tasks: - pexec: all: - sudo cp /etc/containers/registries.conf /etc/containers/registries.conf.backup - sudo dnf -y module reset container-tools - sudo dnf -y module install container-tools - sudo cp /etc/containers/registries.conf.backup /etc/containers/registries.conf - install: branch: pacific exclude_packages: - ceph-volume - print: '**** done install task...' - print: '**** done start installing pacific cephadm ...' - cephadm: cephadm_branch: pacific cephadm_git_url: https://github.com/ceph/ceph conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.io/ceph/daemon-base:latest-pacific - print: '**** done end installing pacific cephadm ...' - print: '**** done start cephadm.shell ceph config set mgr...' - cephadm.shell: mon.a: - ceph config set mgr mgr/cephadm/use_repo_digest true --force - print: '**** done cephadm.shell ceph config set mgr...' - print: '**** done start telemetry pacific...' - workunit: clients: client.0: - test_telemetry_pacific.sh - print: '**** done end telemetry pacific...' - print: '**** done start upgrade sequence...' - sequential: - print: '**** done start upgrade, wait' - cephadm.shell: env: - sha1 mon.a: - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 - print: '**** done end upgrade, wait...' - print: '**** done start telemetry x...' - workunit: clients: client.0: - test_telemetry_pacific_x.sh - print: '**** done end telemetry x...' - print: '**** done end upgrade, wait' teuthology_branch: main teuthology_sha1: 35ea38a9840006713a3d42472a2c536a25e88c15 timestamp: 2022-09-29_05:03:04 tube: smithi user: lflores verbose: true worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.18616 2022-09-29T05:13:41.521 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/github.com_ljflores_ceph_dbaedf5bf154c5830301d16d6da9fb01a6c0c0c5/qa; will attempt to use it 2022-09-29T05:13:41.522 INFO:teuthology.run:Found tasks at /home/teuthworker/src/github.com_ljflores_ceph_dbaedf5bf154c5830301d16d6da9fb01a6c0c0c5/qa/tasks 2022-09-29T05:13:41.523 INFO:teuthology.run_tasks:Running task internal.check_packages... 2022-09-29T05:13:41.526 INFO:teuthology.task.internal:Checking packages... 2022-09-29T05:13:41.553 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash '652bf75409c45c34ae939b241b00adba20f12869' 2022-09-29T05:13:41.554 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2022-09-29T05:13:41.554 INFO:teuthology.packaging:ref: None 2022-09-29T05:13:41.555 INFO:teuthology.packaging:tag: None 2022-09-29T05:13:41.555 INFO:teuthology.packaging:branch: main 2022-09-29T05:13:41.555 INFO:teuthology.packaging:sha1: 652bf75409c45c34ae939b241b00adba20f12869 2022-09-29T05:13:41.556 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=main 2022-09-29T05:13:41.797 INFO:teuthology.task.internal:Found packages for ceph version 18.0.0-128.g652bf754 2022-09-29T05:13:41.798 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2022-09-29T05:13:41.809 INFO:teuthology.task.internal:no buildpackages task found 2022-09-29T05:13:41.809 INFO:teuthology.run_tasks:Running task internal.save_config... 2022-09-29T05:13:41.815 INFO:teuthology.task.internal:Saving configuration 2022-09-29T05:13:41.848 INFO:teuthology.run_tasks:Running task internal.check_lock... 2022-09-29T05:13:41.859 INFO:teuthology.task.internal.check_lock:Checking locks... 2022-09-29T05:13:41.890 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi083.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/lflores-2022-09-29_05:03:04-upgrade:telemetry-upgrade-main-distro-default-smithi/7047304', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '8', 'arch': 'x86_64', 'locked': True, 'locked_since': '2022-09-29 05:08:36.394692', 'locked_by': 'scheduled_lflores@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFh+/17Rx6JSH3MTH1Nf1RK7W726mw4MhJHVKDik7m/0kuAuUXWqYye/I+GPf7upwBrlRz/lybtjMjL1x9QErcM='} 2022-09-29T05:13:41.924 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi153.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/lflores-2022-09-29_05:03:04-upgrade:telemetry-upgrade-main-distro-default-smithi/7047304', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '8', 'arch': 'x86_64', 'locked': True, 'locked_since': '2022-09-29 05:08:36.391994', 'locked_by': 'scheduled_lflores@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBD6OAE7DJFjJ+1T+rc5kzclWcw1w3FkCX5RX7yo/Ed9eF8EwCc8Duej03r1H+MwK9zd6f+LIwElRVcHc8d2JjBg='} 2022-09-29T05:13:41.924 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2022-09-29T05:13:41.937 INFO:teuthology.task.internal:roles: ubuntu@smithi083.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'mds.a', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2022-09-29T05:13:41.937 INFO:teuthology.task.internal:roles: ubuntu@smithi153.front.sepia.ceph.com - ['mon.b', 'mds.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2022-09-29T05:13:41.938 INFO:teuthology.run_tasks:Running task console_log... 2022-09-29T05:13:42.038 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f121f399510>, signals=[15]) 2022-09-29T05:13:42.041 INFO:teuthology.run_tasks:Running task internal.connect... 2022-09-29T05:13:42.063 INFO:teuthology.task.internal:Opening connections... 2022-09-29T05:13:42.063 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi083.front.sepia.ceph.com 2022-09-29T05:13:42.066 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi083.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2022-09-29T05:13:42.177 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi153.front.sepia.ceph.com 2022-09-29T05:13:42.179 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi153.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2022-09-29T05:13:42.284 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2022-09-29T05:13:42.293 DEBUG:teuthology.orchestra.run.smithi083:> uname -m 2022-09-29T05:13:42.316 INFO:teuthology.orchestra.run.smithi083.stdout:x86_64 2022-09-29T05:13:42.317 DEBUG:teuthology.orchestra.run.smithi083:> cat /etc/os-release 2022-09-29T05:13:42.376 INFO:teuthology.orchestra.run.smithi083.stdout:NAME="CentOS Stream" 2022-09-29T05:13:42.378 INFO:teuthology.orchestra.run.smithi083.stdout:VERSION="8" 2022-09-29T05:13:42.378 INFO:teuthology.orchestra.run.smithi083.stdout:ID="centos" 2022-09-29T05:13:42.379 INFO:teuthology.orchestra.run.smithi083.stdout:ID_LIKE="rhel fedora" 2022-09-29T05:13:42.379 INFO:teuthology.orchestra.run.smithi083.stdout:VERSION_ID="8" 2022-09-29T05:13:42.381 INFO:teuthology.orchestra.run.smithi083.stdout:PLATFORM_ID="platform:el8" 2022-09-29T05:13:42.381 INFO:teuthology.orchestra.run.smithi083.stdout:PRETTY_NAME="CentOS Stream 8" 2022-09-29T05:13:42.382 INFO:teuthology.orchestra.run.smithi083.stdout:ANSI_COLOR="0;31" 2022-09-29T05:13:42.383 INFO:teuthology.orchestra.run.smithi083.stdout:CPE_NAME="cpe:/o:centos:centos:8" 2022-09-29T05:13:42.383 INFO:teuthology.orchestra.run.smithi083.stdout:HOME_URL="https://centos.org/" 2022-09-29T05:13:42.383 INFO:teuthology.orchestra.run.smithi083.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2022-09-29T05:13:42.384 INFO:teuthology.orchestra.run.smithi083.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 8" 2022-09-29T05:13:42.384 INFO:teuthology.orchestra.run.smithi083.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2022-09-29T05:13:42.385 INFO:teuthology.lock.ops:Updating smithi083.front.sepia.ceph.com on lock server 2022-09-29T05:13:42.406 DEBUG:teuthology.orchestra.run.smithi153:> uname -m 2022-09-29T05:13:42.430 INFO:teuthology.orchestra.run.smithi153.stdout:x86_64 2022-09-29T05:13:42.432 DEBUG:teuthology.orchestra.run.smithi153:> cat /etc/os-release 2022-09-29T05:13:42.491 INFO:teuthology.orchestra.run.smithi153.stdout:NAME="CentOS Stream" 2022-09-29T05:13:42.492 INFO:teuthology.orchestra.run.smithi153.stdout:VERSION="8" 2022-09-29T05:13:42.492 INFO:teuthology.orchestra.run.smithi153.stdout:ID="centos" 2022-09-29T05:13:42.494 INFO:teuthology.orchestra.run.smithi153.stdout:ID_LIKE="rhel fedora" 2022-09-29T05:13:42.494 INFO:teuthology.orchestra.run.smithi153.stdout:VERSION_ID="8" 2022-09-29T05:13:42.495 INFO:teuthology.orchestra.run.smithi153.stdout:PLATFORM_ID="platform:el8" 2022-09-29T05:13:42.495 INFO:teuthology.orchestra.run.smithi153.stdout:PRETTY_NAME="CentOS Stream 8" 2022-09-29T05:13:42.495 INFO:teuthology.orchestra.run.smithi153.stdout:ANSI_COLOR="0;31" 2022-09-29T05:13:42.496 INFO:teuthology.orchestra.run.smithi153.stdout:CPE_NAME="cpe:/o:centos:centos:8" 2022-09-29T05:13:42.497 INFO:teuthology.orchestra.run.smithi153.stdout:HOME_URL="https://centos.org/" 2022-09-29T05:13:42.497 INFO:teuthology.orchestra.run.smithi153.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2022-09-29T05:13:42.497 INFO:teuthology.orchestra.run.smithi153.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 8" 2022-09-29T05:13:42.498 INFO:teuthology.orchestra.run.smithi153.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2022-09-29T05:13:42.498 INFO:teuthology.lock.ops:Updating smithi153.front.sepia.ceph.com on lock server 2022-09-29T05:13:42.534 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2022-09-29T05:13:42.550 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2022-09-29T05:13:42.558 INFO:teuthology.task.internal:Checking for old test directory... 2022-09-29T05:13:42.558 DEBUG:teuthology.orchestra.run.smithi083:> test '!' -e /home/ubuntu/cephtest 2022-09-29T05:13:42.568 DEBUG:teuthology.orchestra.run.smithi153:> test '!' -e /home/ubuntu/cephtest 2022-09-29T05:13:42.586 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2022-09-29T05:13:42.594 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2022-09-29T05:13:42.595 DEBUG:teuthology.orchestra.run.smithi083:> test -z $(ls -A /var/lib/ceph) 2022-09-29T05:13:42.627 DEBUG:teuthology.orchestra.run.smithi153:> test -z $(ls -A /var/lib/ceph) 2022-09-29T05:13:42.661 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2022-09-29T05:13:42.735 INFO:teuthology.run_tasks:Running task kernel... 2022-09-29T05:13:42.755 INFO:teuthology.task.kernel:normalize config orig: {'kdb': True, 'sha1': 'distro'} 2022-09-29T05:13:42.757 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': True, 'sha1': 'distro'}, 'mon.b': {'kdb': True, 'sha1': 'distro'}}, timeout 300 2022-09-29T05:13:42.757 DEBUG:teuthology.orchestra.run.smithi083:> test -f /run/.containerenv -o -f /.dockerenv 2022-09-29T05:13:42.760 DEBUG:teuthology.orchestra.run.smithi153:> test -f /run/.containerenv -o -f /.dockerenv 2022-09-29T05:13:42.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2022-09-29T05:13:42.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2022-09-29T05:13:42.783 DEBUG:teuthology.orchestra.run.smithi153:> uname -r 2022-09-29T05:13:42.785 DEBUG:teuthology.orchestra.run.smithi083:> uname -r 2022-09-29T05:13:42.843 INFO:teuthology.orchestra.run.smithi153.stdout:4.18.0-408.el8.x86_64 2022-09-29T05:13:42.844 INFO:teuthology.task.kernel:Running kernel on smithi153: 4.18.0-408.el8.x86_64 2022-09-29T05:13:42.844 DEBUG:teuthology.orchestra.run.smithi153:> sudo yum install -y kernel 2022-09-29T05:13:42.847 INFO:teuthology.orchestra.run.smithi083.stdout:4.18.0-408.el8.x86_64 2022-09-29T05:13:42.848 INFO:teuthology.task.kernel:Running kernel on smithi083: 4.18.0-408.el8.x86_64 2022-09-29T05:13:42.848 DEBUG:teuthology.orchestra.run.smithi083:> sudo yum install -y kernel 2022-09-29T05:13:45.917 INFO:teuthology.orchestra.run.smithi153.stdout:CentOS-8 - AppStream 34 MB/s | 25 MB 00:00 2022-09-29T05:13:46.217 INFO:teuthology.orchestra.run.smithi083.stdout:CentOS-8 - AppStream 30 MB/s | 25 MB 00:00 2022-09-29T05:13:51.761 INFO:teuthology.orchestra.run.smithi153.stdout:CentOS-8 - BaseOS 18 MB/s | 25 MB 00:01 2022-09-29T05:13:51.797 INFO:teuthology.orchestra.run.smithi083.stdout:CentOS-8 - BaseOS 22 MB/s | 25 MB 00:01 2022-09-29T05:13:55.599 INFO:teuthology.orchestra.run.smithi153.stdout:CentOS-8 - Extras 126 kB/s | 18 kB 00:00 2022-09-29T05:13:55.654 INFO:teuthology.orchestra.run.smithi083.stdout:CentOS-8 - Extras 170 kB/s | 18 kB 00:00 2022-09-29T05:13:56.127 INFO:teuthology.orchestra.run.smithi153.stdout:CentOS-8 - PowerTools 12 MB/s | 5.0 MB 00:00 2022-09-29T05:13:56.157 INFO:teuthology.orchestra.run.smithi083.stdout:CentOS-8 - PowerTools 13 MB/s | 5.0 MB 00:00 2022-09-29T05:13:57.192 INFO:teuthology.orchestra.run.smithi153.stdout:Copr repo for python3-asyncssh owned by ceph 31 kB/s | 3.5 kB 00:00 2022-09-29T05:13:57.247 INFO:teuthology.orchestra.run.smithi083.stdout:Copr repo for python3-asyncssh owned by ceph 28 kB/s | 3.5 kB 00:00 2022-09-29T05:13:58.176 INFO:teuthology.orchestra.run.smithi153.stdout:Extra Packages for Enterprise Linux 15 MB/s | 13 MB 00:00 2022-09-29T05:13:58.248 INFO:teuthology.orchestra.run.smithi083.stdout:Extra Packages for Enterprise Linux 15 MB/s | 13 MB 00:00 2022-09-29T05:14:01.288 INFO:teuthology.orchestra.run.smithi153.stdout:lab-extras 461 kB/s | 24 kB 00:00 2022-09-29T05:14:01.298 INFO:teuthology.orchestra.run.smithi083.stdout:lab-extras 443 kB/s | 24 kB 00:00 2022-09-29T05:14:04.163 INFO:teuthology.orchestra.run.smithi083.stdout:Package kernel-4.18.0-408.el8.x86_64 is already installed. 2022-09-29T05:14:04.163 INFO:teuthology.orchestra.run.smithi083.stdout:Package kernel-4.18.0-305.3.1.el8.x86_64 is already installed. 2022-09-29T05:14:04.173 INFO:teuthology.orchestra.run.smithi153.stdout:Package kernel-4.18.0-408.el8.x86_64 is already installed. 2022-09-29T05:14:04.174 INFO:teuthology.orchestra.run.smithi153.stdout:Package kernel-4.18.0-305.3.1.el8.x86_64 is already installed. 2022-09-29T05:14:04.301 INFO:teuthology.orchestra.run.smithi083.stdout:Dependencies resolved. 2022-09-29T05:14:04.302 INFO:teuthology.orchestra.run.smithi083.stdout:Nothing to do. 2022-09-29T05:14:04.302 INFO:teuthology.orchestra.run.smithi083.stdout:Complete! 2022-09-29T05:14:04.329 INFO:teuthology.orchestra.run.smithi153.stdout:Dependencies resolved. 2022-09-29T05:14:04.330 INFO:teuthology.orchestra.run.smithi153.stdout:Nothing to do. 2022-09-29T05:14:04.331 INFO:teuthology.orchestra.run.smithi153.stdout:Complete! 2022-09-29T05:14:04.400 DEBUG:teuthology.orchestra.run.smithi083:> echo no | sudo yum reinstall kernel || true 2022-09-29T05:14:04.443 DEBUG:teuthology.orchestra.run.smithi153:> echo no | sudo yum reinstall kernel || true 2022-09-29T05:14:04.967 INFO:teuthology.orchestra.run.smithi083.stdout:Last metadata expiration check: 0:00:03 ago on Thu 29 Sep 2022 05:14:01 AM UTC. 2022-09-29T05:14:05.011 INFO:teuthology.orchestra.run.smithi153.stdout:Last metadata expiration check: 0:00:04 ago on Thu 29 Sep 2022 05:14:01 AM UTC. 2022-09-29T05:14:06.037 INFO:teuthology.orchestra.run.smithi083.stdout:Dependencies resolved. 2022-09-29T05:14:06.039 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2022-09-29T05:14:06.040 INFO:teuthology.orchestra.run.smithi083.stdout: Package Architecture Version Repository Size 2022-09-29T05:14:06.040 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2022-09-29T05:14:06.041 INFO:teuthology.orchestra.run.smithi083.stdout:Reinstalling: 2022-09-29T05:14:06.041 INFO:teuthology.orchestra.run.smithi083.stdout: kernel x86_64 4.18.0-408.el8 CentOS-BaseOS 8.5 M 2022-09-29T05:14:06.041 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:14:06.042 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction Summary 2022-09-29T05:14:06.042 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2022-09-29T05:14:06.043 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:14:06.043 INFO:teuthology.orchestra.run.smithi083.stdout:Total download size: 8.5 M 2022-09-29T05:14:06.044 INFO:teuthology.orchestra.run.smithi083.stdout:Installed size: 0 2022-09-29T05:14:06.044 INFO:teuthology.orchestra.run.smithi083.stderr:Operation aborted. 2022-09-29T05:14:06.061 INFO:teuthology.orchestra.run.smithi153.stdout:Dependencies resolved. 2022-09-29T05:14:06.063 INFO:teuthology.orchestra.run.smithi153.stdout:================================================================================ 2022-09-29T05:14:06.063 INFO:teuthology.orchestra.run.smithi153.stdout: Package Architecture Version Repository Size 2022-09-29T05:14:06.064 INFO:teuthology.orchestra.run.smithi153.stdout:================================================================================ 2022-09-29T05:14:06.064 INFO:teuthology.orchestra.run.smithi153.stdout:Reinstalling: 2022-09-29T05:14:06.065 INFO:teuthology.orchestra.run.smithi153.stdout: kernel x86_64 4.18.0-408.el8 CentOS-BaseOS 8.5 M 2022-09-29T05:14:06.065 INFO:teuthology.orchestra.run.smithi153.stdout: 2022-09-29T05:14:06.065 INFO:teuthology.orchestra.run.smithi153.stdout:Transaction Summary 2022-09-29T05:14:06.066 INFO:teuthology.orchestra.run.smithi153.stdout:================================================================================ 2022-09-29T05:14:06.066 INFO:teuthology.orchestra.run.smithi153.stdout: 2022-09-29T05:14:06.067 INFO:teuthology.orchestra.run.smithi153.stdout:Total download size: 8.5 M 2022-09-29T05:14:06.067 INFO:teuthology.orchestra.run.smithi153.stdout:Installed size: 0 2022-09-29T05:14:06.068 INFO:teuthology.orchestra.run.smithi153.stderr:Operation aborted. 2022-09-29T05:14:06.107 INFO:teuthology.orchestra.run.smithi083.stdout:Is this ok [y/N]: 2022-09-29T05:14:06.108 DEBUG:teuthology.orchestra.run.smithi083:> sudo yum reinstall -y kernel || true 2022-09-29T05:14:06.125 INFO:teuthology.orchestra.run.smithi153.stdout:Is this ok [y/N]: 2022-09-29T05:14:06.126 DEBUG:teuthology.orchestra.run.smithi153:> sudo yum reinstall -y kernel || true 2022-09-29T05:14:06.668 INFO:teuthology.orchestra.run.smithi083.stdout:Last metadata expiration check: 0:00:05 ago on Thu 29 Sep 2022 05:14:01 AM UTC. 2022-09-29T05:14:06.687 INFO:teuthology.orchestra.run.smithi153.stdout:Last metadata expiration check: 0:00:05 ago on Thu 29 Sep 2022 05:14:01 AM UTC. 2022-09-29T05:14:07.721 INFO:teuthology.orchestra.run.smithi083.stdout:Dependencies resolved. 2022-09-29T05:14:07.722 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2022-09-29T05:14:07.723 INFO:teuthology.orchestra.run.smithi083.stdout: Package Architecture Version Repository Size 2022-09-29T05:14:07.723 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2022-09-29T05:14:07.723 INFO:teuthology.orchestra.run.smithi083.stdout:Reinstalling: 2022-09-29T05:14:07.724 INFO:teuthology.orchestra.run.smithi083.stdout: kernel x86_64 4.18.0-408.el8 CentOS-BaseOS 8.5 M 2022-09-29T05:14:07.724 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:14:07.724 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction Summary 2022-09-29T05:14:07.724 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2022-09-29T05:14:07.725 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:14:07.725 INFO:teuthology.orchestra.run.smithi083.stdout:Total download size: 8.5 M 2022-09-29T05:14:07.725 INFO:teuthology.orchestra.run.smithi083.stdout:Installed size: 0 2022-09-29T05:14:07.725 INFO:teuthology.orchestra.run.smithi083.stdout:Downloading Packages: 2022-09-29T05:14:07.745 INFO:teuthology.orchestra.run.smithi153.stdout:Dependencies resolved. 2022-09-29T05:14:07.746 INFO:teuthology.orchestra.run.smithi153.stdout:================================================================================ 2022-09-29T05:14:07.746 INFO:teuthology.orchestra.run.smithi153.stdout: Package Architecture Version Repository Size 2022-09-29T05:14:07.746 INFO:teuthology.orchestra.run.smithi153.stdout:================================================================================ 2022-09-29T05:14:07.746 INFO:teuthology.orchestra.run.smithi153.stdout:Reinstalling: 2022-09-29T05:14:07.747 INFO:teuthology.orchestra.run.smithi153.stdout: kernel x86_64 4.18.0-408.el8 CentOS-BaseOS 8.5 M 2022-09-29T05:14:07.747 INFO:teuthology.orchestra.run.smithi153.stdout: 2022-09-29T05:14:07.747 INFO:teuthology.orchestra.run.smithi153.stdout:Transaction Summary 2022-09-29T05:14:07.747 INFO:teuthology.orchestra.run.smithi153.stdout:================================================================================ 2022-09-29T05:14:07.748 INFO:teuthology.orchestra.run.smithi153.stdout: 2022-09-29T05:14:07.748 INFO:teuthology.orchestra.run.smithi153.stdout:Total download size: 8.5 M 2022-09-29T05:14:07.748 INFO:teuthology.orchestra.run.smithi153.stdout:Installed size: 0 2022-09-29T05:14:07.748 INFO:teuthology.orchestra.run.smithi153.stdout:Downloading Packages: 2022-09-29T05:14:07.975 INFO:teuthology.orchestra.run.smithi083.stdout:kernel-4.18.0-408.el8.x86_64.rpm 34 MB/s | 8.5 MB 00:00 2022-09-29T05:14:07.976 INFO:teuthology.orchestra.run.smithi083.stdout:-------------------------------------------------------------------------------- 2022-09-29T05:14:07.976 INFO:teuthology.orchestra.run.smithi083.stdout:Total 34 MB/s | 8.5 MB 00:00 2022-09-29T05:14:08.060 INFO:teuthology.orchestra.run.smithi153.stdout:kernel-4.18.0-408.el8.x86_64.rpm 27 MB/s | 8.5 MB 00:00 2022-09-29T05:14:08.061 INFO:teuthology.orchestra.run.smithi153.stdout:-------------------------------------------------------------------------------- 2022-09-29T05:14:08.061 INFO:teuthology.orchestra.run.smithi153.stdout:Total 27 MB/s | 8.5 MB 00:00 2022-09-29T05:14:08.078 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction check 2022-09-29T05:14:08.159 INFO:teuthology.orchestra.run.smithi153.stdout:Running transaction check 2022-09-29T05:14:08.469 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction check succeeded. 2022-09-29T05:14:08.470 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction test 2022-09-29T05:14:08.542 INFO:teuthology.orchestra.run.smithi153.stdout:Transaction check succeeded. 2022-09-29T05:14:08.543 INFO:teuthology.orchestra.run.smithi153.stdout:Running transaction test 2022-09-29T05:14:08.620 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction test succeeded. 2022-09-29T05:14:08.647 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction 2022-09-29T05:14:08.695 INFO:teuthology.orchestra.run.smithi153.stdout:Transaction test succeeded. 2022-09-29T05:14:08.709 INFO:teuthology.orchestra.run.smithi153.stdout:Running transaction 2022-09-29T05:14:08.969 INFO:teuthology.orchestra.run.smithi083.stdout: Preparing : 1/1 2022-09-29T05:14:09.011 INFO:teuthology.orchestra.run.smithi153.stdout: Preparing : 1/1 2022-09-29T05:14:09.162 INFO:teuthology.orchestra.run.smithi083.stdout: Reinstalling : kernel-4.18.0-408.el8.x86_64 1/2 2022-09-29T05:14:09.163 INFO:teuthology.orchestra.run.smithi083.stdout: Cleanup : kernel-4.18.0-408.el8.x86_64 2/2 2022-09-29T05:14:09.211 INFO:teuthology.orchestra.run.smithi153.stdout: Reinstalling : kernel-4.18.0-408.el8.x86_64 1/2 2022-09-29T05:14:09.212 INFO:teuthology.orchestra.run.smithi153.stdout: Cleanup : kernel-4.18.0-408.el8.x86_64 2/2 2022-09-29T05:14:09.558 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: kernel-4.18.0-408.el8.x86_64 2/2 2022-09-29T05:14:09.558 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : kernel-4.18.0-408.el8.x86_64 1/2 2022-09-29T05:14:09.592 INFO:teuthology.orchestra.run.smithi153.stdout: Running scriptlet: kernel-4.18.0-408.el8.x86_64 2/2 2022-09-29T05:14:09.592 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : kernel-4.18.0-408.el8.x86_64 1/2 2022-09-29T05:14:10.079 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : kernel-4.18.0-408.el8.x86_64 2/2 2022-09-29T05:14:10.079 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:14:10.080 INFO:teuthology.orchestra.run.smithi083.stdout:Reinstalled: 2022-09-29T05:14:10.080 INFO:teuthology.orchestra.run.smithi083.stdout: kernel-4.18.0-408.el8.x86_64 2022-09-29T05:14:10.080 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:14:10.081 INFO:teuthology.orchestra.run.smithi083.stdout:Complete! 2022-09-29T05:14:10.106 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : kernel-4.18.0-408.el8.x86_64 2/2 2022-09-29T05:14:10.107 INFO:teuthology.orchestra.run.smithi153.stdout: 2022-09-29T05:14:10.107 INFO:teuthology.orchestra.run.smithi153.stdout:Reinstalled: 2022-09-29T05:14:10.107 INFO:teuthology.orchestra.run.smithi153.stdout: kernel-4.18.0-408.el8.x86_64 2022-09-29T05:14:10.107 INFO:teuthology.orchestra.run.smithi153.stdout: 2022-09-29T05:14:10.108 INFO:teuthology.orchestra.run.smithi153.stdout:Complete! 2022-09-29T05:14:10.175 DEBUG:teuthology.orchestra.run.smithi083:> rpm -q kernel --last | head -n 1 2022-09-29T05:14:10.189 DEBUG:teuthology.orchestra.run.smithi153:> rpm -q kernel --last | head -n 1 2022-09-29T05:14:10.315 INFO:teuthology.orchestra.run.smithi153.stdout:kernel-4.18.0-408.el8.x86_64 Thu 29 Sep 2022 05:14:09 AM UTC 2022-09-29T05:14:10.317 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-408.el8.x86_64 2022-09-29T05:14:10.318 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2022-09-29T05:14:10.319 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2022-09-29T05:14:10.320 DEBUG:teuthology.orchestra.run.smithi153:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2022-09-29T05:14:10.329 INFO:teuthology.orchestra.run.smithi083.stdout:kernel-4.18.0-408.el8.x86_64 Thu 29 Sep 2022 05:14:08 AM UTC 2022-09-29T05:14:10.330 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-408.el8.x86_64 2022-09-29T05:14:10.330 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2022-09-29T05:14:10.330 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2022-09-29T05:14:10.331 DEBUG:teuthology.orchestra.run.smithi083:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2022-09-29T05:14:10.363 INFO:teuthology.orchestra.run.smithi153.stdout:ttyS1 2022-09-29T05:14:10.364 INFO:teuthology.orchestra.run.smithi083.stdout:ttyS1 2022-09-29T05:14:10.375 DEBUG:teuthology.parallel:result is None 2022-09-29T05:14:10.377 DEBUG:teuthology.parallel:result is None 2022-09-29T05:14:10.377 INFO:teuthology.run_tasks:Running task internal.base... 2022-09-29T05:14:10.386 INFO:teuthology.task.internal:Creating test directory... 2022-09-29T05:14:10.387 DEBUG:teuthology.orchestra.run.smithi083:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2022-09-29T05:14:10.423 DEBUG:teuthology.orchestra.run.smithi153:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2022-09-29T05:14:10.446 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2022-09-29T05:14:10.460 INFO:teuthology.run_tasks:Running task internal.archive... 2022-09-29T05:14:10.473 INFO:teuthology.task.internal:Creating archive directory... 2022-09-29T05:14:10.474 DEBUG:teuthology.orchestra.run.smithi083:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2022-09-29T05:14:10.490 DEBUG:teuthology.orchestra.run.smithi153:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2022-09-29T05:14:10.520 INFO:teuthology.run_tasks:Running task internal.coredump... 2022-09-29T05:14:10.530 INFO:teuthology.task.internal:Enabling coredump saving... 2022-09-29T05:14:10.530 DEBUG:teuthology.orchestra.run.smithi083:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2022-09-29T05:14:10.552 DEBUG:teuthology.orchestra.run.smithi153:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2022-09-29T05:14:10.582 INFO:teuthology.orchestra.run.smithi083.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2022-09-29T05:14:10.592 INFO:teuthology.orchestra.run.smithi083.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2022-09-29T05:14:10.593 INFO:teuthology.orchestra.run.smithi153.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2022-09-29T05:14:10.607 INFO:teuthology.orchestra.run.smithi153.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2022-09-29T05:14:10.608 INFO:teuthology.run_tasks:Running task internal.sudo... 2022-09-29T05:14:10.624 INFO:teuthology.task.internal:Configuring sudo... 2022-09-29T05:14:10.625 DEBUG:teuthology.orchestra.run.smithi083:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2022-09-29T05:14:10.637 DEBUG:teuthology.orchestra.run.smithi153:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2022-09-29T05:14:10.675 INFO:teuthology.run_tasks:Running task internal.syslog... 2022-09-29T05:14:10.690 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2022-09-29T05:14:10.692 DEBUG:teuthology.orchestra.run.smithi083:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2022-09-29T05:14:10.704 DEBUG:teuthology.orchestra.run.smithi153:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2022-09-29T05:14:10.736 DEBUG:teuthology.orchestra.run.smithi083:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2022-09-29T05:14:10.806 DEBUG:teuthology.orchestra.run.smithi083:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2022-09-29T05:14:10.865 DEBUG:teuthology.orchestra.run.smithi083:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2022-09-29T05:14:10.955 DEBUG:teuthology.orchestra.run.smithi083:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2022-09-29T05:14:10.995 DEBUG:teuthology.orchestra.run.smithi083:> set -ex 2022-09-29T05:14:10.996 DEBUG:teuthology.orchestra.run.smithi083:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2022-09-29T05:14:11.063 DEBUG:teuthology.orchestra.run.smithi153:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2022-09-29T05:14:11.121 DEBUG:teuthology.orchestra.run.smithi153:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2022-09-29T05:14:11.157 DEBUG:teuthology.orchestra.run.smithi153:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2022-09-29T05:14:11.240 DEBUG:teuthology.orchestra.run.smithi153:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2022-09-29T05:14:11.286 DEBUG:teuthology.orchestra.run.smithi153:> set -ex 2022-09-29T05:14:11.287 DEBUG:teuthology.orchestra.run.smithi153:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2022-09-29T05:14:11.353 DEBUG:teuthology.orchestra.run.smithi083:> sudo service rsyslog restart 2022-09-29T05:14:11.360 DEBUG:teuthology.orchestra.run.smithi153:> sudo service rsyslog restart 2022-09-29T05:14:11.411 INFO:teuthology.orchestra.run.smithi083.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2022-09-29T05:14:11.448 INFO:teuthology.orchestra.run.smithi153.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2022-09-29T05:14:11.820 INFO:teuthology.run_tasks:Running task internal.timer... 2022-09-29T05:14:11.836 INFO:teuthology.task.internal:Starting timer... 2022-09-29T05:14:11.837 INFO:teuthology.run_tasks:Running task pcp... 2022-09-29T05:14:11.855 INFO:teuthology.run_tasks:Running task selinux... 2022-09-29T05:14:11.877 DEBUG:teuthology.task:Applying overrides for task selinux: {'whitelist': ['scontext=system_u:system_r:logrotate_t:s0']} 2022-09-29T05:14:11.878 DEBUG:teuthology.orchestra.run.smithi083:> sudo service auditd rotate 2022-09-29T05:14:11.938 INFO:teuthology.orchestra.run.smithi083.stdout:Rotating logs: 2022-09-29T05:14:11.940 DEBUG:teuthology.orchestra.run.smithi153:> sudo service auditd rotate 2022-09-29T05:14:12.039 INFO:teuthology.orchestra.run.smithi153.stdout:Rotating logs: 2022-09-29T05:14:12.041 DEBUG:teuthology.task.selinux:Getting current SELinux state 2022-09-29T05:14:12.042 DEBUG:teuthology.orchestra.run.smithi083:> /usr/sbin/getenforce 2022-09-29T05:14:12.073 INFO:teuthology.orchestra.run.smithi083.stdout:Permissive 2022-09-29T05:14:12.074 DEBUG:teuthology.orchestra.run.smithi153:> /usr/sbin/getenforce 2022-09-29T05:14:12.097 INFO:teuthology.orchestra.run.smithi153.stdout:Permissive 2022-09-29T05:14:12.097 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi083.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi153.front.sepia.ceph.com': 'permissive'} 2022-09-29T05:14:12.098 DEBUG:teuthology.orchestra.run.smithi083:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av '\(comm="dmidecode"\|chronyd.service\|name="cephtest"\|scontext=system_u:system_r:nrpe_t:s0\|scontext=system_u:system_r:pcp_pmlogger_t\|scontext=system_u:system_r:pcp_pmcd_t:s0\|comm="rhsmd"\|scontext=system_u:system_r:syslogd_t:s0\|tcontext=system_u:system_r:nrpe_t:s0\|comm="updatedb"\|comm="smartd"\|comm="rhsmcertd-worke"\|comm="setroubleshootd"\|comm="rpm"\|tcontext=system_u:object_r:container_runtime_exec_t:s0\|comm="ksmtuned"\|comm="sssd"\|comm="sss_cache"\|context=system_u:system_r:NetworkManager_dispatcher_t:s0\)' 2022-09-29T05:14:12.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2022-09-29T05:14:12.145 DEBUG:teuthology.orchestra.run.smithi153:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av '\(comm="dmidecode"\|chronyd.service\|name="cephtest"\|scontext=system_u:system_r:nrpe_t:s0\|scontext=system_u:system_r:pcp_pmlogger_t\|scontext=system_u:system_r:pcp_pmcd_t:s0\|comm="rhsmd"\|scontext=system_u:system_r:syslogd_t:s0\|tcontext=system_u:system_r:nrpe_t:s0\|comm="updatedb"\|comm="smartd"\|comm="rhsmcertd-worke"\|comm="setroubleshootd"\|comm="rpm"\|tcontext=system_u:object_r:container_runtime_exec_t:s0\|comm="ksmtuned"\|comm="sssd"\|comm="sss_cache"\|context=system_u:system_r:NetworkManager_dispatcher_t:s0\)' 2022-09-29T05:14:12.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2022-09-29T05:14:12.175 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2022-09-29T05:14:12.177 DEBUG:teuthology.orchestra.run.smithi083:> sudo /usr/sbin/setenforce permissive 2022-09-29T05:14:12.215 DEBUG:teuthology.orchestra.run.smithi153:> sudo /usr/sbin/setenforce permissive 2022-09-29T05:14:12.249 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2022-09-29T05:14:12.265 DEBUG:teuthology.repo_utils:Setting repo remote to git://git.ceph.com/git/ceph-cm-ansible.git 2022-09-29T05:14:12.279 INFO:teuthology.repo_utils:Fetching main from origin 2022-09-29T05:14:12.353 INFO:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_git_ceph-cm-ansible_main to origin/main 2022-09-29T05:14:12.387 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2022-09-29T05:14:12.389 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi083.front.sepia.ceph.com,smithi153.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_git_ceph-cm-ansible_main/cephlab.yml 2022-09-29T05:17:20.545 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi083.front.sepia.ceph.com'), Remote(name='ubuntu@smithi153.front.sepia.ceph.com')] 2022-09-29T05:17:20.549 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi083.front.sepia.ceph.com' 2022-09-29T05:17:20.552 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi083.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2022-09-29T05:17:20.641 DEBUG:teuthology.orchestra.run.smithi083:> true 2022-09-29T05:17:20.705 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi153.front.sepia.ceph.com' 2022-09-29T05:17:20.706 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi153.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2022-09-29T05:17:20.787 DEBUG:teuthology.orchestra.run.smithi153:> true 2022-09-29T05:17:20.860 INFO:teuthology.run_tasks:Running task clock... 2022-09-29T05:17:20.874 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2022-09-29T05:17:20.875 INFO:teuthology.orchestra.run:Running command with timeout 360 2022-09-29T05:17:20.879 DEBUG:teuthology.orchestra.run.smithi083:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2022-09-29T05:17:20.883 INFO:teuthology.orchestra.run:Running command with timeout 360 2022-09-29T05:17:20.884 DEBUG:teuthology.orchestra.run.smithi153:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2022-09-29T05:17:20.913 INFO:teuthology.orchestra.run.smithi083.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2022-09-29T05:17:20.926 INFO:teuthology.orchestra.run.smithi083.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2022-09-29T05:17:20.932 INFO:teuthology.orchestra.run.smithi153.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2022-09-29T05:17:20.946 INFO:teuthology.orchestra.run.smithi153.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2022-09-29T05:17:20.996 INFO:teuthology.orchestra.run.smithi083.stderr:sudo: ntpd: command not found 2022-09-29T05:17:21.007 INFO:teuthology.orchestra.run.smithi083.stdout:506 Cannot talk to daemon 2022-09-29T05:17:21.011 INFO:teuthology.orchestra.run.smithi153.stderr:sudo: ntpd: command not found 2022-09-29T05:17:21.020 INFO:teuthology.orchestra.run.smithi083.stderr:Failed to start ntp.service: Unit ntp.service not found. 2022-09-29T05:17:21.022 INFO:teuthology.orchestra.run.smithi153.stdout:506 Cannot talk to daemon 2022-09-29T05:17:21.034 INFO:teuthology.orchestra.run.smithi083.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2022-09-29T05:17:21.036 INFO:teuthology.orchestra.run.smithi153.stderr:Failed to start ntp.service: Unit ntp.service not found. 2022-09-29T05:17:21.050 INFO:teuthology.orchestra.run.smithi153.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2022-09-29T05:17:21.100 INFO:teuthology.orchestra.run.smithi083.stderr:bash: ntpq: command not found 2022-09-29T05:17:21.102 INFO:teuthology.orchestra.run.smithi083.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2022-09-29T05:17:21.103 INFO:teuthology.orchestra.run.smithi083.stdout:=============================================================================== 2022-09-29T05:17:21.103 INFO:teuthology.orchestra.run.smithi083.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2022-09-29T05:17:21.103 INFO:teuthology.orchestra.run.smithi083.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2022-09-29T05:17:21.103 INFO:teuthology.orchestra.run.smithi083.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2022-09-29T05:17:21.103 INFO:teuthology.orchestra.run.smithi083.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2022-09-29T05:17:21.115 INFO:teuthology.orchestra.run.smithi153.stderr:bash: ntpq: command not found 2022-09-29T05:17:21.119 INFO:teuthology.orchestra.run.smithi153.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2022-09-29T05:17:21.120 INFO:teuthology.orchestra.run.smithi153.stdout:=============================================================================== 2022-09-29T05:17:21.121 INFO:teuthology.orchestra.run.smithi153.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2022-09-29T05:17:21.121 INFO:teuthology.orchestra.run.smithi153.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2022-09-29T05:17:21.122 INFO:teuthology.orchestra.run.smithi153.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2022-09-29T05:17:21.122 INFO:teuthology.orchestra.run.smithi153.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2022-09-29T05:17:21.123 INFO:teuthology.run_tasks:Running task pexec... 2022-09-29T05:17:21.138 INFO:teuthology.task.pexec:Executing custom commands... 2022-09-29T05:17:21.139 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi083.front.sepia.ceph.com 2022-09-29T05:17:21.140 DEBUG:teuthology.orchestra.run.smithi083:> TESTDIR=/home/ubuntu/cephtest bash -s 2022-09-29T05:17:21.140 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi153.front.sepia.ceph.com 2022-09-29T05:17:21.141 DEBUG:teuthology.orchestra.run.smithi153:> TESTDIR=/home/ubuntu/cephtest bash -s 2022-09-29T05:17:21.743 INFO:teuthology.orchestra.run.smithi083.stdout:Last metadata expiration check: 0:01:14 ago on Thu 29 Sep 2022 05:16:07 AM UTC. 2022-09-29T05:17:21.759 INFO:teuthology.orchestra.run.smithi153.stdout:Last metadata expiration check: 0:01:14 ago on Thu 29 Sep 2022 05:16:07 AM UTC. 2022-09-29T05:17:23.103 INFO:teuthology.orchestra.run.smithi083.stdout:Dependencies resolved. 2022-09-29T05:17:23.104 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2022-09-29T05:17:23.104 INFO:teuthology.orchestra.run.smithi083.stdout: Package Architecture Version Repository Size 2022-09-29T05:17:23.105 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2022-09-29T05:17:23.105 INFO:teuthology.orchestra.run.smithi083.stdout:Resetting modules: 2022-09-29T05:17:23.106 INFO:teuthology.orchestra.run.smithi083.stdout: container-tools 2022-09-29T05:17:23.106 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:17:23.107 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction Summary 2022-09-29T05:17:23.107 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2022-09-29T05:17:23.108 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:17:23.108 INFO:teuthology.orchestra.run.smithi153.stdout:Dependencies resolved. 2022-09-29T05:17:23.109 INFO:teuthology.orchestra.run.smithi153.stdout:================================================================================ 2022-09-29T05:17:23.110 INFO:teuthology.orchestra.run.smithi153.stdout: Package Architecture Version Repository Size 2022-09-29T05:17:23.110 INFO:teuthology.orchestra.run.smithi153.stdout:================================================================================ 2022-09-29T05:17:23.110 INFO:teuthology.orchestra.run.smithi153.stdout:Resetting modules: 2022-09-29T05:17:23.110 INFO:teuthology.orchestra.run.smithi153.stdout: container-tools 2022-09-29T05:17:23.111 INFO:teuthology.orchestra.run.smithi153.stdout: 2022-09-29T05:17:23.111 INFO:teuthology.orchestra.run.smithi153.stdout:Transaction Summary 2022-09-29T05:17:23.111 INFO:teuthology.orchestra.run.smithi153.stdout:================================================================================ 2022-09-29T05:17:23.111 INFO:teuthology.orchestra.run.smithi153.stdout: 2022-09-29T05:17:23.319 INFO:teuthology.orchestra.run.smithi083.stdout:Complete! 2022-09-29T05:17:23.320 INFO:teuthology.orchestra.run.smithi153.stdout:Complete! 2022-09-29T05:17:23.947 INFO:teuthology.orchestra.run.smithi083.stdout:Last metadata expiration check: 0:01:16 ago on Thu 29 Sep 2022 05:16:07 AM UTC. 2022-09-29T05:17:23.981 INFO:teuthology.orchestra.run.smithi153.stdout:Last metadata expiration check: 0:01:16 ago on Thu 29 Sep 2022 05:16:07 AM UTC. 2022-09-29T05:17:25.282 INFO:teuthology.orchestra.run.smithi083.stdout:Dependencies resolved. 2022-09-29T05:17:25.283 INFO:teuthology.orchestra.run.smithi083.stdout:========================================================================================= 2022-09-29T05:17:25.283 INFO:teuthology.orchestra.run.smithi083.stdout: Package Arch Version Repository Size 2022-09-29T05:17:25.284 INFO:teuthology.orchestra.run.smithi083.stdout:========================================================================================= 2022-09-29T05:17:25.284 INFO:teuthology.orchestra.run.smithi083.stdout:Installing group/module packages: 2022-09-29T05:17:25.285 INFO:teuthology.orchestra.run.smithi083.stdout: aardvark-dns x86_64 2:1.1.0-4.module_el8.7.0+1216+b022c01d CentOS-AppStream 997 k 2022-09-29T05:17:25.285 INFO:teuthology.orchestra.run.smithi083.stdout: buildah x86_64 1:1.27.0-2.module_el8.7.0+1216+b022c01d CentOS-AppStream 8.1 M 2022-09-29T05:17:25.286 INFO:teuthology.orchestra.run.smithi083.stdout: cockpit-podman noarch 53-1.module_el8.7.0+1216+b022c01d CentOS-AppStream 547 k 2022-09-29T05:17:25.286 INFO:teuthology.orchestra.run.smithi083.stdout: crun x86_64 1.5-1.module_el8.7.0+1216+b022c01d CentOS-AppStream 213 k 2022-09-29T05:17:25.287 INFO:teuthology.orchestra.run.smithi083.stdout: netavark x86_64 2:1.1.0-6.module_el8.7.0+1216+b022c01d CentOS-AppStream 2.2 M 2022-09-29T05:17:25.287 INFO:teuthology.orchestra.run.smithi083.stdout: python3-podman noarch 4.2.0-1.module_el8.7.0+1216+b022c01d CentOS-AppStream 151 k 2022-09-29T05:17:25.288 INFO:teuthology.orchestra.run.smithi083.stdout: skopeo x86_64 2:1.9.2-1.module_el8.7.0+1216+b022c01d CentOS-AppStream 6.8 M 2022-09-29T05:17:25.288 INFO:teuthology.orchestra.run.smithi083.stdout: toolbox x86_64 0.0.99.3-1.module_el8.5.0+877+1c30e0c9 CentOS-AppStream 2.3 M 2022-09-29T05:17:25.289 INFO:teuthology.orchestra.run.smithi083.stdout: udica noarch 0.2.6-3.module_el8.7.0+1216+b022c01d CentOS-AppStream 49 k 2022-09-29T05:17:25.289 INFO:teuthology.orchestra.run.smithi083.stdout:Installing dependencies: 2022-09-29T05:17:25.290 INFO:teuthology.orchestra.run.smithi083.stdout: python3-pytoml noarch 0.1.14-5.git7dea353.el8 CentOS-AppStream 25 k 2022-09-29T05:17:25.291 INFO:teuthology.orchestra.run.smithi083.stdout: python3-pyxdg noarch 0.25-16.el8 CentOS-AppStream 94 k 2022-09-29T05:17:25.291 INFO:teuthology.orchestra.run.smithi083.stdout: yajl x86_64 2.1.0-11.el8 CentOS-AppStream 41 k 2022-09-29T05:17:25.292 INFO:teuthology.orchestra.run.smithi083.stdout:Installing module profiles: 2022-09-29T05:17:25.292 INFO:teuthology.orchestra.run.smithi083.stdout: container-tools/common 2022-09-29T05:17:25.293 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:17:25.293 INFO:teuthology.orchestra.run.smithi083.stdout:Enabling module streams: 2022-09-29T05:17:25.293 INFO:teuthology.orchestra.run.smithi083.stdout: container-tools rhel8 2022-09-29T05:17:25.294 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:17:25.294 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction Summary 2022-09-29T05:17:25.295 INFO:teuthology.orchestra.run.smithi083.stdout:========================================================================================= 2022-09-29T05:17:25.295 INFO:teuthology.orchestra.run.smithi083.stdout:Install 12 Packages 2022-09-29T05:17:25.296 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:17:25.298 INFO:teuthology.orchestra.run.smithi083.stdout:Total download size: 21 M 2022-09-29T05:17:25.298 INFO:teuthology.orchestra.run.smithi083.stdout:Installed size: 71 M 2022-09-29T05:17:25.299 INFO:teuthology.orchestra.run.smithi083.stdout:Downloading Packages: 2022-09-29T05:17:25.315 INFO:teuthology.orchestra.run.smithi153.stdout:Dependencies resolved. 2022-09-29T05:17:25.316 INFO:teuthology.orchestra.run.smithi153.stdout:========================================================================================= 2022-09-29T05:17:25.317 INFO:teuthology.orchestra.run.smithi153.stdout: Package Arch Version Repository Size 2022-09-29T05:17:25.317 INFO:teuthology.orchestra.run.smithi153.stdout:========================================================================================= 2022-09-29T05:17:25.318 INFO:teuthology.orchestra.run.smithi153.stdout:Installing group/module packages: 2022-09-29T05:17:25.318 INFO:teuthology.orchestra.run.smithi153.stdout: aardvark-dns x86_64 2:1.1.0-4.module_el8.7.0+1216+b022c01d CentOS-AppStream 997 k 2022-09-29T05:17:25.319 INFO:teuthology.orchestra.run.smithi153.stdout: buildah x86_64 1:1.27.0-2.module_el8.7.0+1216+b022c01d CentOS-AppStream 8.1 M 2022-09-29T05:17:25.319 INFO:teuthology.orchestra.run.smithi153.stdout: cockpit-podman noarch 53-1.module_el8.7.0+1216+b022c01d CentOS-AppStream 547 k 2022-09-29T05:17:25.319 INFO:teuthology.orchestra.run.smithi153.stdout: crun x86_64 1.5-1.module_el8.7.0+1216+b022c01d CentOS-AppStream 213 k 2022-09-29T05:17:25.320 INFO:teuthology.orchestra.run.smithi153.stdout: netavark x86_64 2:1.1.0-6.module_el8.7.0+1216+b022c01d CentOS-AppStream 2.2 M 2022-09-29T05:17:25.320 INFO:teuthology.orchestra.run.smithi153.stdout: python3-podman noarch 4.2.0-1.module_el8.7.0+1216+b022c01d CentOS-AppStream 151 k 2022-09-29T05:17:25.321 INFO:teuthology.orchestra.run.smithi153.stdout: skopeo x86_64 2:1.9.2-1.module_el8.7.0+1216+b022c01d CentOS-AppStream 6.8 M 2022-09-29T05:17:25.321 INFO:teuthology.orchestra.run.smithi153.stdout: toolbox x86_64 0.0.99.3-1.module_el8.5.0+877+1c30e0c9 CentOS-AppStream 2.3 M 2022-09-29T05:17:25.322 INFO:teuthology.orchestra.run.smithi153.stdout: udica noarch 0.2.6-3.module_el8.7.0+1216+b022c01d CentOS-AppStream 49 k 2022-09-29T05:17:25.322 INFO:teuthology.orchestra.run.smithi153.stdout:Installing dependencies: 2022-09-29T05:17:25.322 INFO:teuthology.orchestra.run.smithi153.stdout: python3-pytoml noarch 0.1.14-5.git7dea353.el8 CentOS-AppStream 25 k 2022-09-29T05:17:25.323 INFO:teuthology.orchestra.run.smithi153.stdout: python3-pyxdg noarch 0.25-16.el8 CentOS-AppStream 94 k 2022-09-29T05:17:25.323 INFO:teuthology.orchestra.run.smithi153.stdout: yajl x86_64 2.1.0-11.el8 CentOS-AppStream 41 k 2022-09-29T05:17:25.323 INFO:teuthology.orchestra.run.smithi153.stdout:Installing module profiles: 2022-09-29T05:17:25.323 INFO:teuthology.orchestra.run.smithi153.stdout: container-tools/common 2022-09-29T05:17:25.324 INFO:teuthology.orchestra.run.smithi153.stdout: 2022-09-29T05:17:25.324 INFO:teuthology.orchestra.run.smithi153.stdout:Enabling module streams: 2022-09-29T05:17:25.324 INFO:teuthology.orchestra.run.smithi153.stdout: container-tools rhel8 2022-09-29T05:17:25.324 INFO:teuthology.orchestra.run.smithi153.stdout: 2022-09-29T05:17:25.325 INFO:teuthology.orchestra.run.smithi153.stdout:Transaction Summary 2022-09-29T05:17:25.325 INFO:teuthology.orchestra.run.smithi153.stdout:========================================================================================= 2022-09-29T05:17:25.325 INFO:teuthology.orchestra.run.smithi153.stdout:Install 12 Packages 2022-09-29T05:17:25.325 INFO:teuthology.orchestra.run.smithi153.stdout: 2022-09-29T05:17:25.326 INFO:teuthology.orchestra.run.smithi153.stdout:Total download size: 21 M 2022-09-29T05:17:25.327 INFO:teuthology.orchestra.run.smithi153.stdout:Installed size: 71 M 2022-09-29T05:17:25.327 INFO:teuthology.orchestra.run.smithi153.stdout:Downloading Packages: 2022-09-29T05:17:25.509 INFO:teuthology.orchestra.run.smithi153.stdout:(1/12): aardvark-dns-1.1.0-4.module_el8.7.0+121 5.2 MB/s | 997 kB 00:00 2022-09-29T05:17:25.514 INFO:teuthology.orchestra.run.smithi083.stdout:(1/12): aardvark-dns-1.1.0-4.module_el8.7.0+121 4.3 MB/s | 997 kB 00:00 2022-09-29T05:17:25.550 INFO:teuthology.orchestra.run.smithi153.stdout:(2/12): cockpit-podman-53-1.module_el8.7.0+1216 2.3 MB/s | 547 kB 00:00 2022-09-29T05:17:25.556 INFO:teuthology.orchestra.run.smithi083.stdout:(2/12): cockpit-podman-53-1.module_el8.7.0+1216 2.0 MB/s | 547 kB 00:00 2022-09-29T05:17:25.584 INFO:teuthology.orchestra.run.smithi153.stdout:(3/12): crun-1.5-1.module_el8.7.0+1216+b022c01d 2.8 MB/s | 213 kB 00:00 2022-09-29T05:17:25.598 INFO:teuthology.orchestra.run.smithi083.stdout:(3/12): crun-1.5-1.module_el8.7.0+1216+b022c01d 2.5 MB/s | 213 kB 00:00 2022-09-29T05:17:25.651 INFO:teuthology.orchestra.run.smithi153.stdout:(4/12): python3-podman-4.2.0-1.module_el8.7.0+1 2.2 MB/s | 151 kB 00:00 2022-09-29T05:17:25.657 INFO:teuthology.orchestra.run.smithi083.stdout:(4/12): python3-podman-4.2.0-1.module_el8.7.0+1 2.5 MB/s | 151 kB 00:00 2022-09-29T05:17:25.699 INFO:teuthology.orchestra.run.smithi083.stdout:(5/12): python3-pytoml-0.1.14-5.git7dea353.el8. 595 kB/s | 25 kB 00:00 2022-09-29T05:17:25.760 INFO:teuthology.orchestra.run.smithi153.stdout:(5/12): buildah-1.27.0-2.module_el8.7.0+1216+b0 18 MB/s | 8.1 MB 00:00 2022-09-29T05:17:25.762 INFO:teuthology.orchestra.run.smithi083.stdout:(6/12): python3-pyxdg-0.25-16.el8.noarch.rpm 1.4 MB/s | 94 kB 00:00 2022-09-29T05:17:25.776 INFO:teuthology.orchestra.run.smithi153.stdout:(6/12): python3-pytoml-0.1.14-5.git7dea353.el8. 198 kB/s | 25 kB 00:00 2022-09-29T05:17:25.802 INFO:teuthology.orchestra.run.smithi153.stdout:(7/12): python3-pyxdg-0.25-16.el8.noarch.rpm 2.2 MB/s | 94 kB 00:00 2022-09-29T05:17:25.869 INFO:teuthology.orchestra.run.smithi153.stdout:(8/12): netavark-1.1.0-6.module_el8.7.0+1216+b0 6.8 MB/s | 2.2 MB 00:00 2022-09-29T05:17:25.883 INFO:teuthology.orchestra.run.smithi083.stdout:(7/12): buildah-1.27.0-2.module_el8.7.0+1216+b0 14 MB/s | 8.1 MB 00:00 2022-09-29T05:17:25.919 INFO:teuthology.orchestra.run.smithi153.stdout:(9/12): udica-0.2.6-3.module_el8.7.0+1216+b022c 988 kB/s | 49 kB 00:00 2022-09-29T05:17:25.933 INFO:teuthology.orchestra.run.smithi083.stdout:(8/12): netavark-1.1.0-6.module_el8.7.0+1216+b0 5.7 MB/s | 2.2 MB 00:00 2022-09-29T05:17:25.951 INFO:teuthology.orchestra.run.smithi083.stdout:(9/12): udica-0.2.6-3.module_el8.7.0+1216+b022c 2.9 MB/s | 49 kB 00:00 2022-09-29T05:17:25.967 INFO:teuthology.orchestra.run.smithi083.stdout:(10/12): yajl-2.1.0-11.el8.x86_64.rpm 2.4 MB/s | 41 kB 00:00 2022-09-29T05:17:25.969 INFO:teuthology.orchestra.run.smithi153.stdout:(10/12): toolbox-0.0.99.3-1.module_el8.5.0+877+ 13 MB/s | 2.3 MB 00:00 2022-09-29T05:17:25.994 INFO:teuthology.orchestra.run.smithi153.stdout:(11/12): yajl-2.1.0-11.el8.x86_64.rpm 542 kB/s | 41 kB 00:00 2022-09-29T05:17:26.017 INFO:teuthology.orchestra.run.smithi083.stdout:(11/12): toolbox-0.0.99.3-1.module_el8.5.0+877+ 17 MB/s | 2.3 MB 00:00 2022-09-29T05:17:26.192 INFO:teuthology.orchestra.run.smithi083.stdout:(12/12): skopeo-1.9.2-1.module_el8.7.0+1216+b02 16 MB/s | 6.8 MB 00:00 2022-09-29T05:17:26.195 INFO:teuthology.orchestra.run.smithi083.stdout:-------------------------------------------------------------------------------- 2022-09-29T05:17:26.195 INFO:teuthology.orchestra.run.smithi083.stdout:Total 23 MB/s | 21 MB 00:00 2022-09-29T05:17:26.203 INFO:teuthology.orchestra.run.smithi153.stdout:(12/12): skopeo-1.9.2-1.module_el8.7.0+1216+b02 16 MB/s | 6.8 MB 00:00 2022-09-29T05:17:26.205 INFO:teuthology.orchestra.run.smithi153.stdout:-------------------------------------------------------------------------------- 2022-09-29T05:17:26.206 INFO:teuthology.orchestra.run.smithi153.stdout:Total 24 MB/s | 21 MB 00:00 2022-09-29T05:17:26.421 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction check 2022-09-29T05:17:26.429 INFO:teuthology.orchestra.run.smithi153.stdout:Running transaction check 2022-09-29T05:17:26.440 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction check succeeded. 2022-09-29T05:17:26.440 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction test 2022-09-29T05:17:26.449 INFO:teuthology.orchestra.run.smithi153.stdout:Transaction check succeeded. 2022-09-29T05:17:26.449 INFO:teuthology.orchestra.run.smithi153.stdout:Running transaction test 2022-09-29T05:17:26.933 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction test succeeded. 2022-09-29T05:17:26.936 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction 2022-09-29T05:17:26.944 INFO:teuthology.orchestra.run.smithi153.stdout:Transaction test succeeded. 2022-09-29T05:17:26.947 INFO:teuthology.orchestra.run.smithi153.stdout:Running transaction 2022-09-29T05:17:27.494 INFO:teuthology.orchestra.run.smithi153.stdout: Preparing : 1/1 2022-09-29T05:17:27.558 INFO:teuthology.orchestra.run.smithi083.stdout: Preparing : 1/1 2022-09-29T05:17:27.708 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : yajl-2.1.0-11.el8.x86_64 1/12 2022-09-29T05:17:27.781 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : yajl-2.1.0-11.el8.x86_64 1/12 2022-09-29T05:17:27.869 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : python3-pyxdg-0.25-16.el8.noarch 2/12 2022-09-29T05:17:27.923 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-pyxdg-0.25-16.el8.noarch 2/12 2022-09-29T05:17:28.124 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : python3-pytoml-0.1.14-5.git7dea353.el8.noarch 3/12 2022-09-29T05:17:28.208 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-pytoml-0.1.14-5.git7dea353.el8.noarch 3/12 2022-09-29T05:17:28.506 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : aardvark-dns-2:1.1.0-4.module_el8.7.0+1216+b022c01 4/12 2022-09-29T05:17:28.606 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : aardvark-dns-2:1.1.0-4.module_el8.7.0+1216+b022c01 4/12 2022-09-29T05:17:29.411 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : netavark-2:1.1.0-6.module_el8.7.0+1216+b022c01d.x8 5/12 2022-09-29T05:17:29.536 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : netavark-2:1.1.0-6.module_el8.7.0+1216+b022c01d.x8 5/12 2022-09-29T05:17:29.642 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : buildah-1:1.27.0-2.module_el8.7.0+1216+b022c01d.x8 6/12 2022-09-29T05:17:29.759 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : buildah-1:1.27.0-2.module_el8.7.0+1216+b022c01d.x8 6/12 2022-09-29T05:17:29.896 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : python3-podman-4.2.0-1.module_el8.7.0+1216+b022c01 7/12 2022-09-29T05:17:29.992 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-podman-4.2.0-1.module_el8.7.0+1216+b022c01 7/12 2022-09-29T05:17:30.082 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : crun-1.5-1.module_el8.7.0+1216+b022c01d.x86_64 8/12 2022-09-29T05:17:30.176 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : crun-1.5-1.module_el8.7.0+1216+b022c01d.x86_64 8/12 2022-09-29T05:17:30.492 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : udica-0.2.6-3.module_el8.7.0+1216+b022c01d.noarch 9/12 2022-09-29T05:17:30.527 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : udica-0.2.6-3.module_el8.7.0+1216+b022c01d.noarch 9/12 2022-09-29T05:17:31.248 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : toolbox-0.0.99.3-1.module_el8.5.0+877+1c30e0c9.x86 10/12 2022-09-29T05:17:31.292 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : toolbox-0.0.99.3-1.module_el8.5.0+877+1c30e0c9.x86 10/12 2022-09-29T05:17:31.464 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : skopeo-2:1.9.2-1.module_el8.7.0+1216+b022c01d.x86_ 11/12 2022-09-29T05:17:31.492 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : skopeo-2:1.9.2-1.module_el8.7.0+1216+b022c01d.x86_ 11/12 2022-09-29T05:17:31.626 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : cockpit-podman-53-1.module_el8.7.0+1216+b022c01d.n 12/12 2022-09-29T05:17:31.688 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : cockpit-podman-53-1.module_el8.7.0+1216+b022c01d.n 12/12 2022-09-29T05:17:31.907 INFO:teuthology.orchestra.run.smithi153.stdout: Running scriptlet: cockpit-podman-53-1.module_el8.7.0+1216+b022c01d.n 12/12 2022-09-29T05:17:31.907 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : aardvark-dns-2:1.1.0-4.module_el8.7.0+1216+b022c01 1/12 2022-09-29T05:17:31.908 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : buildah-1:1.27.0-2.module_el8.7.0+1216+b022c01d.x8 2/12 2022-09-29T05:17:31.908 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : cockpit-podman-53-1.module_el8.7.0+1216+b022c01d.n 3/12 2022-09-29T05:17:31.908 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : crun-1.5-1.module_el8.7.0+1216+b022c01d.x86_64 4/12 2022-09-29T05:17:31.908 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : netavark-2:1.1.0-6.module_el8.7.0+1216+b022c01d.x8 5/12 2022-09-29T05:17:31.909 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : python3-podman-4.2.0-1.module_el8.7.0+1216+b022c01 6/12 2022-09-29T05:17:31.909 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : python3-pytoml-0.1.14-5.git7dea353.el8.noarch 7/12 2022-09-29T05:17:31.909 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : python3-pyxdg-0.25-16.el8.noarch 8/12 2022-09-29T05:17:31.909 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : skopeo-2:1.9.2-1.module_el8.7.0+1216+b022c01d.x86_ 9/12 2022-09-29T05:17:31.910 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : toolbox-0.0.99.3-1.module_el8.5.0+877+1c30e0c9.x86 10/12 2022-09-29T05:17:31.910 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : udica-0.2.6-3.module_el8.7.0+1216+b022c01d.noarch 11/12 2022-09-29T05:17:31.948 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: cockpit-podman-53-1.module_el8.7.0+1216+b022c01d.n 12/12 2022-09-29T05:17:31.949 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : aardvark-dns-2:1.1.0-4.module_el8.7.0+1216+b022c01 1/12 2022-09-29T05:17:31.950 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : buildah-1:1.27.0-2.module_el8.7.0+1216+b022c01d.x8 2/12 2022-09-29T05:17:31.950 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : cockpit-podman-53-1.module_el8.7.0+1216+b022c01d.n 3/12 2022-09-29T05:17:31.950 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : crun-1.5-1.module_el8.7.0+1216+b022c01d.x86_64 4/12 2022-09-29T05:17:31.951 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : netavark-2:1.1.0-6.module_el8.7.0+1216+b022c01d.x8 5/12 2022-09-29T05:17:31.951 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-podman-4.2.0-1.module_el8.7.0+1216+b022c01 6/12 2022-09-29T05:17:31.952 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-pytoml-0.1.14-5.git7dea353.el8.noarch 7/12 2022-09-29T05:17:31.953 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-pyxdg-0.25-16.el8.noarch 8/12 2022-09-29T05:17:31.953 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : skopeo-2:1.9.2-1.module_el8.7.0+1216+b022c01d.x86_ 9/12 2022-09-29T05:17:31.954 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : toolbox-0.0.99.3-1.module_el8.5.0+877+1c30e0c9.x86 10/12 2022-09-29T05:17:31.954 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : udica-0.2.6-3.module_el8.7.0+1216+b022c01d.noarch 11/12 2022-09-29T05:17:32.407 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : yajl-2.1.0-11.el8.x86_64 12/12 2022-09-29T05:17:32.408 INFO:teuthology.orchestra.run.smithi153.stdout: 2022-09-29T05:17:32.408 INFO:teuthology.orchestra.run.smithi153.stdout:Installed: 2022-09-29T05:17:32.408 INFO:teuthology.orchestra.run.smithi153.stdout: aardvark-dns-2:1.1.0-4.module_el8.7.0+1216+b022c01d.x86_64 2022-09-29T05:17:32.408 INFO:teuthology.orchestra.run.smithi153.stdout: buildah-1:1.27.0-2.module_el8.7.0+1216+b022c01d.x86_64 2022-09-29T05:17:32.409 INFO:teuthology.orchestra.run.smithi153.stdout: cockpit-podman-53-1.module_el8.7.0+1216+b022c01d.noarch 2022-09-29T05:17:32.409 INFO:teuthology.orchestra.run.smithi153.stdout: crun-1.5-1.module_el8.7.0+1216+b022c01d.x86_64 2022-09-29T05:17:32.409 INFO:teuthology.orchestra.run.smithi153.stdout: netavark-2:1.1.0-6.module_el8.7.0+1216+b022c01d.x86_64 2022-09-29T05:17:32.409 INFO:teuthology.orchestra.run.smithi153.stdout: python3-podman-4.2.0-1.module_el8.7.0+1216+b022c01d.noarch 2022-09-29T05:17:32.409 INFO:teuthology.orchestra.run.smithi153.stdout: python3-pytoml-0.1.14-5.git7dea353.el8.noarch 2022-09-29T05:17:32.410 INFO:teuthology.orchestra.run.smithi153.stdout: python3-pyxdg-0.25-16.el8.noarch 2022-09-29T05:17:32.410 INFO:teuthology.orchestra.run.smithi153.stdout: skopeo-2:1.9.2-1.module_el8.7.0+1216+b022c01d.x86_64 2022-09-29T05:17:32.410 INFO:teuthology.orchestra.run.smithi153.stdout: toolbox-0.0.99.3-1.module_el8.5.0+877+1c30e0c9.x86_64 2022-09-29T05:17:32.411 INFO:teuthology.orchestra.run.smithi153.stdout: udica-0.2.6-3.module_el8.7.0+1216+b022c01d.noarch 2022-09-29T05:17:32.411 INFO:teuthology.orchestra.run.smithi153.stdout: yajl-2.1.0-11.el8.x86_64 2022-09-29T05:17:32.411 INFO:teuthology.orchestra.run.smithi153.stdout: 2022-09-29T05:17:32.411 INFO:teuthology.orchestra.run.smithi153.stdout:Complete! 2022-09-29T05:17:32.412 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : yajl-2.1.0-11.el8.x86_64 12/12 2022-09-29T05:17:32.412 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:17:32.412 INFO:teuthology.orchestra.run.smithi083.stdout:Installed: 2022-09-29T05:17:32.413 INFO:teuthology.orchestra.run.smithi083.stdout: aardvark-dns-2:1.1.0-4.module_el8.7.0+1216+b022c01d.x86_64 2022-09-29T05:17:32.413 INFO:teuthology.orchestra.run.smithi083.stdout: buildah-1:1.27.0-2.module_el8.7.0+1216+b022c01d.x86_64 2022-09-29T05:17:32.413 INFO:teuthology.orchestra.run.smithi083.stdout: cockpit-podman-53-1.module_el8.7.0+1216+b022c01d.noarch 2022-09-29T05:17:32.413 INFO:teuthology.orchestra.run.smithi083.stdout: crun-1.5-1.module_el8.7.0+1216+b022c01d.x86_64 2022-09-29T05:17:32.414 INFO:teuthology.orchestra.run.smithi083.stdout: netavark-2:1.1.0-6.module_el8.7.0+1216+b022c01d.x86_64 2022-09-29T05:17:32.414 INFO:teuthology.orchestra.run.smithi083.stdout: python3-podman-4.2.0-1.module_el8.7.0+1216+b022c01d.noarch 2022-09-29T05:17:32.414 INFO:teuthology.orchestra.run.smithi083.stdout: python3-pytoml-0.1.14-5.git7dea353.el8.noarch 2022-09-29T05:17:32.414 INFO:teuthology.orchestra.run.smithi083.stdout: python3-pyxdg-0.25-16.el8.noarch 2022-09-29T05:17:32.415 INFO:teuthology.orchestra.run.smithi083.stdout: skopeo-2:1.9.2-1.module_el8.7.0+1216+b022c01d.x86_64 2022-09-29T05:17:32.415 INFO:teuthology.orchestra.run.smithi083.stdout: toolbox-0.0.99.3-1.module_el8.5.0+877+1c30e0c9.x86_64 2022-09-29T05:17:32.415 INFO:teuthology.orchestra.run.smithi083.stdout: udica-0.2.6-3.module_el8.7.0+1216+b022c01d.noarch 2022-09-29T05:17:32.415 INFO:teuthology.orchestra.run.smithi083.stdout: yajl-2.1.0-11.el8.x86_64 2022-09-29T05:17:32.416 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:17:32.416 INFO:teuthology.orchestra.run.smithi083.stdout:Complete! 2022-09-29T05:17:32.532 DEBUG:teuthology.parallel:result is None 2022-09-29T05:17:32.607 DEBUG:teuthology.parallel:result is None 2022-09-29T05:17:32.607 INFO:teuthology.run_tasks:Running task install... 2022-09-29T05:17:32.621 DEBUG:teuthology.task.install:project ceph 2022-09-29T05:17:32.622 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': '652bf75409c45c34ae939b241b00adba20f12869'}} 2022-09-29T05:17:32.623 DEBUG:teuthology.task.install:config {'branch': 'pacific', 'exclude_packages': ['ceph-volume'], 'flavor': 'default', 'sha1': '652bf75409c45c34ae939b241b00adba20f12869'} 2022-09-29T05:17:32.624 INFO:teuthology.task.install:Using flavor: default 2022-09-29T05:17:32.640 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2022-09-29T05:17:32.641 INFO:teuthology.task.install:extra packages: [] 2022-09-29T05:17:32.642 DEBUG:teuthology.orchestra.run.smithi083:> sudo lsb_release -is 2022-09-29T05:17:32.680 INFO:teuthology.orchestra.run.smithi083.stdout:CentOSStream 2022-09-29T05:17:32.681 DEBUG:teuthology.misc:System to be installed: CentOSStream 2022-09-29T05:17:32.682 DEBUG:teuthology.orchestra.run.smithi153:> sudo lsb_release -is 2022-09-29T05:17:32.682 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'pacific', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': '652bf75409c45c34ae939b241b00adba20f12869', 'tag': None, 'wait_for_package': False} 2022-09-29T05:17:32.683 DEBUG:teuthology.orchestra.run.smithi083:> sudo lsb_release -is 2022-09-29T05:17:32.719 INFO:teuthology.orchestra.run.smithi153.stdout:CentOSStream 2022-09-29T05:17:32.721 DEBUG:teuthology.misc:System to be installed: CentOSStream 2022-09-29T05:17:32.722 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'pacific', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': '652bf75409c45c34ae939b241b00adba20f12869', 'tag': None, 'wait_for_package': False} 2022-09-29T05:17:32.723 DEBUG:teuthology.orchestra.run.smithi153:> sudo lsb_release -is 2022-09-29T05:17:32.763 INFO:teuthology.orchestra.run.smithi083.stdout:CentOSStream 2022-09-29T05:17:32.767 DEBUG:teuthology.misc:System to be installed: CentOSStream 2022-09-29T05:17:32.767 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2022-09-29T05:17:32.768 INFO:teuthology.packaging:ref: None 2022-09-29T05:17:32.768 INFO:teuthology.packaging:tag: None 2022-09-29T05:17:32.769 INFO:teuthology.packaging:branch: pacific 2022-09-29T05:17:32.769 INFO:teuthology.packaging:sha1: 652bf75409c45c34ae939b241b00adba20f12869 2022-09-29T05:17:32.770 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=pacific 2022-09-29T05:17:32.823 INFO:teuthology.orchestra.run.smithi153.stdout:CentOSStream 2022-09-29T05:17:32.824 DEBUG:teuthology.misc:System to be installed: CentOSStream 2022-09-29T05:17:32.824 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2022-09-29T05:17:32.824 INFO:teuthology.packaging:ref: None 2022-09-29T05:17:32.825 INFO:teuthology.packaging:tag: None 2022-09-29T05:17:32.825 INFO:teuthology.packaging:branch: pacific 2022-09-29T05:17:32.825 INFO:teuthology.packaging:sha1: 652bf75409c45c34ae939b241b00adba20f12869 2022-09-29T05:17:32.825 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=pacific 2022-09-29T05:17:33.045 INFO:teuthology.task.install.rpm:Pulling from https://1.chacra.ceph.com/r/ceph/pacific/966bd07c7933dde97fbebe5b6735924d693c5fe3/centos/8/flavors/default/ 2022-09-29T05:17:33.046 INFO:teuthology.task.install.rpm:Package version is 16.2.10-932.g966bd07c 2022-09-29T05:17:33.049 INFO:teuthology.task.install.rpm:Pulling from https://1.chacra.ceph.com/r/ceph/pacific/966bd07c7933dde97fbebe5b6735924d693c5fe3/centos/8/flavors/default/ 2022-09-29T05:17:33.050 INFO:teuthology.task.install.rpm:Package version is 16.2.10-932.g966bd07c 2022-09-29T05:17:33.177 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://1.chacra.ceph.com/r/ceph/pacific/966bd07c7933dde97fbebe5b6735924d693c5fe3/centos/8/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://1.chacra.ceph.com/r/ceph/pacific/966bd07c7933dde97fbebe5b6735924d693c5fe3/centos/8/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://1.chacra.ceph.com/r/ceph/pacific/966bd07c7933dde97fbebe5b6735924d693c5fe3/centos/8/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2022-09-29T05:17:33.178 DEBUG:teuthology.orchestra.run.smithi083:> set -ex 2022-09-29T05:17:33.178 DEBUG:teuthology.orchestra.run.smithi083:> sudo dd of=/etc/yum.repos.d/ceph.repo 2022-09-29T05:17:33.189 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://1.chacra.ceph.com/r/ceph/pacific/966bd07c7933dde97fbebe5b6735924d693c5fe3/centos/8/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://1.chacra.ceph.com/r/ceph/pacific/966bd07c7933dde97fbebe5b6735924d693c5fe3/centos/8/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://1.chacra.ceph.com/r/ceph/pacific/966bd07c7933dde97fbebe5b6735924d693c5fe3/centos/8/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2022-09-29T05:17:33.189 DEBUG:teuthology.orchestra.run.smithi153:> set -ex 2022-09-29T05:17:33.190 DEBUG:teuthology.orchestra.run.smithi153:> sudo dd of=/etc/yum.repos.d/ceph.repo 2022-09-29T05:17:33.206 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2022-09-29T05:17:33.207 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2022-09-29T05:17:33.207 INFO:teuthology.packaging:ref: None 2022-09-29T05:17:33.208 INFO:teuthology.packaging:tag: None 2022-09-29T05:17:33.208 INFO:teuthology.packaging:branch: pacific 2022-09-29T05:17:33.208 INFO:teuthology.packaging:sha1: 652bf75409c45c34ae939b241b00adba20f12869 2022-09-29T05:17:33.209 DEBUG:teuthology.orchestra.run.smithi083:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/pacific/;g' /etc/yum.repos.d/ceph.repo ; fi 2022-09-29T05:17:33.217 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2022-09-29T05:17:33.218 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2022-09-29T05:17:33.218 INFO:teuthology.packaging:ref: None 2022-09-29T05:17:33.218 INFO:teuthology.packaging:tag: None 2022-09-29T05:17:33.219 INFO:teuthology.packaging:branch: pacific 2022-09-29T05:17:33.219 INFO:teuthology.packaging:sha1: 652bf75409c45c34ae939b241b00adba20f12869 2022-09-29T05:17:33.220 DEBUG:teuthology.orchestra.run.smithi153:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/pacific/;g' /etc/yum.repos.d/ceph.repo ; fi 2022-09-29T05:17:33.274 DEBUG:teuthology.orchestra.run.smithi083:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2022-09-29T05:17:33.285 DEBUG:teuthology.orchestra.run.smithi153:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2022-09-29T05:17:33.351 DEBUG:teuthology.orchestra.run.smithi083:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2022-09-29T05:17:33.364 DEBUG:teuthology.orchestra.run.smithi153:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2022-09-29T05:17:33.417 INFO:teuthology.orchestra.run.smithi083.stdout:check_obsoletes = 1 2022-09-29T05:17:33.419 DEBUG:teuthology.orchestra.run.smithi083:> sudo yum clean all 2022-09-29T05:17:33.436 INFO:teuthology.orchestra.run.smithi153.stdout:check_obsoletes = 1 2022-09-29T05:17:33.437 DEBUG:teuthology.orchestra.run.smithi153:> sudo yum clean all 2022-09-29T05:17:33.672 INFO:teuthology.orchestra.run.smithi083.stdout:55 files removed 2022-09-29T05:17:33.690 INFO:teuthology.orchestra.run.smithi153.stdout:55 files removed 2022-09-29T05:17:33.699 DEBUG:teuthology.orchestra.run.smithi083:> sudo yum -y install ceph-radosgw 2022-09-29T05:17:33.717 DEBUG:teuthology.orchestra.run.smithi153:> sudo yum -y install ceph-radosgw 2022-09-29T05:17:34.320 INFO:teuthology.orchestra.run.smithi083.stdout:ceph packages for x86_64 293 kB/s | 76 kB 00:00 2022-09-29T05:17:34.339 INFO:teuthology.orchestra.run.smithi153.stdout:ceph packages for x86_64 298 kB/s | 76 kB 00:00 2022-09-29T05:17:34.599 INFO:teuthology.orchestra.run.smithi083.stdout:ceph noarch packages 63 kB/s | 15 kB 00:00 2022-09-29T05:17:34.647 INFO:teuthology.orchestra.run.smithi153.stdout:ceph noarch packages 55 kB/s | 15 kB 00:00 2022-09-29T05:17:34.863 INFO:teuthology.orchestra.run.smithi083.stdout:ceph source packages 6.0 kB/s | 1.4 kB 00:00 2022-09-29T05:17:34.914 INFO:teuthology.orchestra.run.smithi153.stdout:ceph source packages 5.9 kB/s | 1.4 kB 00:00 2022-09-29T05:17:35.586 INFO:teuthology.orchestra.run.smithi153.stdout:CentOS-8 - AppStream 39 MB/s | 25 MB 00:00 2022-09-29T05:17:35.717 INFO:teuthology.orchestra.run.smithi083.stdout:CentOS-8 - AppStream 30 MB/s | 25 MB 00:00 2022-09-29T05:17:40.777 INFO:teuthology.orchestra.run.smithi153.stdout:CentOS-8 - BaseOS 36 MB/s | 25 MB 00:00 2022-09-29T05:17:41.094 INFO:teuthology.orchestra.run.smithi083.stdout:CentOS-8 - BaseOS 33 MB/s | 25 MB 00:00 2022-09-29T05:17:44.598 INFO:teuthology.orchestra.run.smithi153.stdout:CentOS-8 - Extras 172 kB/s | 18 kB 00:00 2022-09-29T05:17:44.962 INFO:teuthology.orchestra.run.smithi153.stdout:CentOS-8 - PowerTools 20 MB/s | 5.0 MB 00:00 2022-09-29T05:17:45.010 INFO:teuthology.orchestra.run.smithi083.stdout:CentOS-8 - Extras 132 kB/s | 18 kB 00:00 2022-09-29T05:17:45.428 INFO:teuthology.orchestra.run.smithi083.stdout:CentOS-8 - PowerTools 17 MB/s | 5.0 MB 00:00 2022-09-29T05:17:46.036 INFO:teuthology.orchestra.run.smithi153.stdout:Copr repo for python3-asyncssh owned by ceph 32 kB/s | 3.5 kB 00:00 2022-09-29T05:17:46.528 INFO:teuthology.orchestra.run.smithi083.stdout:Copr repo for python3-asyncssh owned by ceph 26 kB/s | 3.5 kB 00:00 2022-09-29T05:17:47.204 INFO:teuthology.orchestra.run.smithi153.stdout:Extra Packages for Enterprise Linux 12 MB/s | 13 MB 00:01 2022-09-29T05:17:47.678 INFO:teuthology.orchestra.run.smithi083.stdout:Extra Packages for Enterprise Linux 13 MB/s | 13 MB 00:01 2022-09-29T05:17:50.312 INFO:teuthology.orchestra.run.smithi153.stdout:lab-extras 425 kB/s | 24 kB 00:00 2022-09-29T05:17:50.795 INFO:teuthology.orchestra.run.smithi083.stdout:lab-extras 350 kB/s | 24 kB 00:00 2022-09-29T05:17:53.443 INFO:teuthology.orchestra.run.smithi153.stdout:Dependencies resolved. 2022-09-29T05:17:53.445 INFO:teuthology.orchestra.run.smithi153.stdout:================================================================================ 2022-09-29T05:17:53.446 INFO:teuthology.orchestra.run.smithi153.stdout: Package Arch Version Repository Size 2022-09-29T05:17:53.446 INFO:teuthology.orchestra.run.smithi153.stdout:================================================================================ 2022-09-29T05:17:53.446 INFO:teuthology.orchestra.run.smithi153.stdout:Installing: 2022-09-29T05:17:53.447 INFO:teuthology.orchestra.run.smithi153.stdout: ceph-radosgw x86_64 2:16.2.10-932.g966bd07c.el8 ceph 12 M 2022-09-29T05:17:53.447 INFO:teuthology.orchestra.run.smithi153.stdout:Upgrading: 2022-09-29T05:17:53.447 INFO:teuthology.orchestra.run.smithi153.stdout: librados2 x86_64 2:16.2.10-932.g966bd07c.el8 ceph 3.7 M 2022-09-29T05:17:53.447 INFO:teuthology.orchestra.run.smithi153.stdout: librbd1 x86_64 2:16.2.10-932.g966bd07c.el8 ceph 4.2 M 2022-09-29T05:17:53.447 INFO:teuthology.orchestra.run.smithi153.stdout:Installing dependencies: 2022-09-29T05:17:53.448 INFO:teuthology.orchestra.run.smithi153.stdout: ceph-base x86_64 2:16.2.10-932.g966bd07c.el8 ceph 6.1 M 2022-09-29T05:17:53.448 INFO:teuthology.orchestra.run.smithi153.stdout: ceph-common x86_64 2:16.2.10-932.g966bd07c.el8 ceph 24 M 2022-09-29T05:17:53.448 INFO:teuthology.orchestra.run.smithi153.stdout: ceph-selinux x86_64 2:16.2.10-932.g966bd07c.el8 ceph 26 k 2022-09-29T05:17:53.449 INFO:teuthology.orchestra.run.smithi153.stdout: gperftools-libs x86_64 1:2.7-9.el8 epel 306 k 2022-09-29T05:17:53.449 INFO:teuthology.orchestra.run.smithi153.stdout: leveldb x86_64 1.22-1.el8 epel 181 k 2022-09-29T05:17:53.449 INFO:teuthology.orchestra.run.smithi153.stdout: libcephfs2 x86_64 2:16.2.10-932.g966bd07c.el8 ceph 820 k 2022-09-29T05:17:53.449 INFO:teuthology.orchestra.run.smithi153.stdout: liboath x86_64 2.6.2-3.el8 epel 59 k 2022-09-29T05:17:53.449 INFO:teuthology.orchestra.run.smithi153.stdout: librabbitmq x86_64 0.9.0-3.el8 CentOS-BaseOS 47 k 2022-09-29T05:17:53.450 INFO:teuthology.orchestra.run.smithi153.stdout: libradosstriper1 x86_64 2:16.2.10-932.g966bd07c.el8 ceph 493 k 2022-09-29T05:17:53.450 INFO:teuthology.orchestra.run.smithi153.stdout: librdkafka x86_64 0.11.4-3.el8 CentOS-AppStream 354 k 2022-09-29T05:17:53.451 INFO:teuthology.orchestra.run.smithi153.stdout: librgw2 x86_64 2:16.2.10-932.g966bd07c.el8 ceph 3.8 M 2022-09-29T05:17:53.451 INFO:teuthology.orchestra.run.smithi153.stdout: libunwind x86_64 1.3.1-3.el8 epel 75 k 2022-09-29T05:17:53.452 INFO:teuthology.orchestra.run.smithi153.stdout: lttng-ust x86_64 2.8.1-11.el8 CentOS-AppStream 259 k 2022-09-29T05:17:53.452 INFO:teuthology.orchestra.run.smithi153.stdout: python3-ceph-argparse 2022-09-29T05:17:53.453 INFO:teuthology.orchestra.run.smithi153.stdout: x86_64 2:16.2.10-932.g966bd07c.el8 ceph 45 k 2022-09-29T05:17:53.453 INFO:teuthology.orchestra.run.smithi153.stdout: python3-ceph-common x86_64 2:16.2.10-932.g966bd07c.el8 ceph 96 k 2022-09-29T05:17:53.454 INFO:teuthology.orchestra.run.smithi153.stdout: python3-cephfs x86_64 2:16.2.10-932.g966bd07c.el8 ceph 215 k 2022-09-29T05:17:53.454 INFO:teuthology.orchestra.run.smithi153.stdout: python3-prettytable noarch 0.7.2-14.el8 CentOS-AppStream 44 k 2022-09-29T05:17:53.455 INFO:teuthology.orchestra.run.smithi153.stdout: python3-rados x86_64 2:16.2.10-932.g966bd07c.el8 ceph 387 k 2022-09-29T05:17:53.455 INFO:teuthology.orchestra.run.smithi153.stdout: python3-rbd x86_64 2:16.2.10-932.g966bd07c.el8 ceph 368 k 2022-09-29T05:17:53.455 INFO:teuthology.orchestra.run.smithi153.stdout: python3-rgw x86_64 2:16.2.10-932.g966bd07c.el8 ceph 114 k 2022-09-29T05:17:53.456 INFO:teuthology.orchestra.run.smithi153.stdout: 2022-09-29T05:17:53.456 INFO:teuthology.orchestra.run.smithi153.stdout:Transaction Summary 2022-09-29T05:17:53.457 INFO:teuthology.orchestra.run.smithi153.stdout:================================================================================ 2022-09-29T05:17:53.457 INFO:teuthology.orchestra.run.smithi153.stdout:Install 21 Packages 2022-09-29T05:17:53.458 INFO:teuthology.orchestra.run.smithi153.stdout:Upgrade 2 Packages 2022-09-29T05:17:53.458 INFO:teuthology.orchestra.run.smithi153.stdout: 2022-09-29T05:17:53.460 INFO:teuthology.orchestra.run.smithi153.stdout:Total download size: 57 M 2022-09-29T05:17:53.460 INFO:teuthology.orchestra.run.smithi153.stdout:Downloading Packages: 2022-09-29T05:17:54.013 INFO:teuthology.orchestra.run.smithi083.stdout:Dependencies resolved. 2022-09-29T05:17:54.016 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2022-09-29T05:17:54.016 INFO:teuthology.orchestra.run.smithi083.stdout: Package Arch Version Repository Size 2022-09-29T05:17:54.016 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2022-09-29T05:17:54.016 INFO:teuthology.orchestra.run.smithi083.stdout:Installing: 2022-09-29T05:17:54.017 INFO:teuthology.orchestra.run.smithi083.stdout: ceph-radosgw x86_64 2:16.2.10-932.g966bd07c.el8 ceph 12 M 2022-09-29T05:17:54.017 INFO:teuthology.orchestra.run.smithi083.stdout:Upgrading: 2022-09-29T05:17:54.017 INFO:teuthology.orchestra.run.smithi083.stdout: librados2 x86_64 2:16.2.10-932.g966bd07c.el8 ceph 3.7 M 2022-09-29T05:17:54.017 INFO:teuthology.orchestra.run.smithi083.stdout: librbd1 x86_64 2:16.2.10-932.g966bd07c.el8 ceph 4.2 M 2022-09-29T05:17:54.018 INFO:teuthology.orchestra.run.smithi083.stdout:Installing dependencies: 2022-09-29T05:17:54.018 INFO:teuthology.orchestra.run.smithi083.stdout: ceph-base x86_64 2:16.2.10-932.g966bd07c.el8 ceph 6.1 M 2022-09-29T05:17:54.018 INFO:teuthology.orchestra.run.smithi083.stdout: ceph-common x86_64 2:16.2.10-932.g966bd07c.el8 ceph 24 M 2022-09-29T05:17:54.018 INFO:teuthology.orchestra.run.smithi083.stdout: ceph-selinux x86_64 2:16.2.10-932.g966bd07c.el8 ceph 26 k 2022-09-29T05:17:54.019 INFO:teuthology.orchestra.run.smithi083.stdout: gperftools-libs x86_64 1:2.7-9.el8 epel 306 k 2022-09-29T05:17:54.019 INFO:teuthology.orchestra.run.smithi083.stdout: leveldb x86_64 1.22-1.el8 epel 181 k 2022-09-29T05:17:54.019 INFO:teuthology.orchestra.run.smithi083.stdout: libcephfs2 x86_64 2:16.2.10-932.g966bd07c.el8 ceph 820 k 2022-09-29T05:17:54.019 INFO:teuthology.orchestra.run.smithi083.stdout: liboath x86_64 2.6.2-3.el8 epel 59 k 2022-09-29T05:17:54.019 INFO:teuthology.orchestra.run.smithi083.stdout: librabbitmq x86_64 0.9.0-3.el8 CentOS-BaseOS 47 k 2022-09-29T05:17:54.020 INFO:teuthology.orchestra.run.smithi083.stdout: libradosstriper1 x86_64 2:16.2.10-932.g966bd07c.el8 ceph 493 k 2022-09-29T05:17:54.020 INFO:teuthology.orchestra.run.smithi083.stdout: librdkafka x86_64 0.11.4-3.el8 CentOS-AppStream 354 k 2022-09-29T05:17:54.020 INFO:teuthology.orchestra.run.smithi083.stdout: librgw2 x86_64 2:16.2.10-932.g966bd07c.el8 ceph 3.8 M 2022-09-29T05:17:54.020 INFO:teuthology.orchestra.run.smithi083.stdout: libunwind x86_64 1.3.1-3.el8 epel 75 k 2022-09-29T05:17:54.021 INFO:teuthology.orchestra.run.smithi083.stdout: lttng-ust x86_64 2.8.1-11.el8 CentOS-AppStream 259 k 2022-09-29T05:17:54.021 INFO:teuthology.orchestra.run.smithi083.stdout: python3-ceph-argparse 2022-09-29T05:17:54.021 INFO:teuthology.orchestra.run.smithi083.stdout: x86_64 2:16.2.10-932.g966bd07c.el8 ceph 45 k 2022-09-29T05:17:54.021 INFO:teuthology.orchestra.run.smithi083.stdout: python3-ceph-common x86_64 2:16.2.10-932.g966bd07c.el8 ceph 96 k 2022-09-29T05:17:54.021 INFO:teuthology.orchestra.run.smithi083.stdout: python3-cephfs x86_64 2:16.2.10-932.g966bd07c.el8 ceph 215 k 2022-09-29T05:17:54.022 INFO:teuthology.orchestra.run.smithi083.stdout: python3-prettytable noarch 0.7.2-14.el8 CentOS-AppStream 44 k 2022-09-29T05:17:54.022 INFO:teuthology.orchestra.run.smithi083.stdout: python3-rados x86_64 2:16.2.10-932.g966bd07c.el8 ceph 387 k 2022-09-29T05:17:54.023 INFO:teuthology.orchestra.run.smithi083.stdout: python3-rbd x86_64 2:16.2.10-932.g966bd07c.el8 ceph 368 k 2022-09-29T05:17:54.023 INFO:teuthology.orchestra.run.smithi083.stdout: python3-rgw x86_64 2:16.2.10-932.g966bd07c.el8 ceph 114 k 2022-09-29T05:17:54.023 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:17:54.023 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction Summary 2022-09-29T05:17:54.024 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2022-09-29T05:17:54.024 INFO:teuthology.orchestra.run.smithi083.stdout:Install 21 Packages 2022-09-29T05:17:54.024 INFO:teuthology.orchestra.run.smithi083.stdout:Upgrade 2 Packages 2022-09-29T05:17:54.025 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:17:54.026 INFO:teuthology.orchestra.run.smithi083.stdout:Total download size: 57 M 2022-09-29T05:17:54.026 INFO:teuthology.orchestra.run.smithi083.stdout:Downloading Packages: 2022-09-29T05:17:54.197 INFO:teuthology.orchestra.run.smithi153.stdout:(1/23): ceph-base-16.2.10-932.g966bd07c.el8.x86 8.1 MB/s | 6.1 MB 00:00 2022-09-29T05:17:54.247 INFO:teuthology.orchestra.run.smithi153.stdout:(2/23): ceph-selinux-16.2.10-932.g966bd07c.el8. 514 kB/s | 26 kB 00:00 2022-09-29T05:17:54.397 INFO:teuthology.orchestra.run.smithi153.stdout:(3/23): libcephfs2-16.2.10-932.g966bd07c.el8.x8 5.3 MB/s | 820 kB 00:00 2022-09-29T05:17:54.497 INFO:teuthology.orchestra.run.smithi153.stdout:(4/23): libradosstriper1-16.2.10-932.g966bd07c. 4.8 MB/s | 493 kB 00:00 2022-09-29T05:17:55.031 INFO:teuthology.orchestra.run.smithi153.stdout:(5/23): librgw2-16.2.10-932.g966bd07c.el8.x86_6 7.0 MB/s | 3.8 MB 00:00 2022-09-29T05:17:55.156 INFO:teuthology.orchestra.run.smithi153.stdout:(6/23): ceph-radosgw-16.2.10-932.g966bd07c.el8. 6.8 MB/s | 12 MB 00:01 2022-09-29T05:17:55.173 INFO:teuthology.orchestra.run.smithi153.stdout:(7/23): python3-ceph-argparse-16.2.10-932.g966b 316 kB/s | 45 kB 00:00 2022-09-29T05:17:55.223 INFO:teuthology.orchestra.run.smithi153.stdout:(8/23): python3-ceph-common-16.2.10-932.g966bd0 1.4 MB/s | 96 kB 00:00 2022-09-29T05:17:55.257 INFO:teuthology.orchestra.run.smithi153.stdout:(9/23): python3-cephfs-16.2.10-932.g966bd07c.el 2.5 MB/s | 215 kB 00:00 2022-09-29T05:17:55.332 INFO:teuthology.orchestra.run.smithi153.stdout:(10/23): python3-rados-16.2.10-932.g966bd07c.el 3.5 MB/s | 387 kB 00:00 2022-09-29T05:17:55.358 INFO:teuthology.orchestra.run.smithi153.stdout:(11/23): python3-rbd-16.2.10-932.g966bd07c.el8. 3.6 MB/s | 368 kB 00:00 2022-09-29T05:17:55.408 INFO:teuthology.orchestra.run.smithi153.stdout:(12/23): python3-rgw-16.2.10-932.g966bd07c.el8. 1.5 MB/s | 114 kB 00:00 2022-09-29T05:17:55.442 INFO:teuthology.orchestra.run.smithi153.stdout:(13/23): librdkafka-0.11.4-3.el8.x86_64.rpm 4.1 MB/s | 354 kB 00:00 2022-09-29T05:17:55.467 INFO:teuthology.orchestra.run.smithi153.stdout:(14/23): python3-prettytable-0.7.2-14.el8.noarc 1.8 MB/s | 44 kB 00:00 2022-09-29T05:17:55.492 INFO:teuthology.orchestra.run.smithi153.stdout:(15/23): librabbitmq-0.9.0-3.el8.x86_64.rpm 1.8 MB/s | 47 kB 00:00 2022-09-29T05:17:55.517 INFO:teuthology.orchestra.run.smithi153.stdout:(16/23): lttng-ust-2.8.1-11.el8.x86_64.rpm 2.3 MB/s | 259 kB 00:00 2022-09-29T05:17:55.718 INFO:teuthology.orchestra.run.smithi153.stdout:(17/23): gperftools-libs-2.7-9.el8.x86_64.rpm 1.3 MB/s | 306 kB 00:00 2022-09-29T05:17:55.897 INFO:teuthology.orchestra.run.smithi083.stdout:(1/23): ceph-base-16.2.10-932.g966bd07c.el8.x86 3.2 MB/s | 6.1 MB 00:01 2022-09-29T05:17:55.951 INFO:teuthology.orchestra.run.smithi153.stdout:(18/23): ceph-common-16.2.10-932.g966bd07c.el8. 9.6 MB/s | 24 MB 00:02 2022-09-29T05:17:55.972 INFO:teuthology.orchestra.run.smithi083.stdout:(2/23): ceph-selinux-16.2.10-932.g966bd07c.el8. 342 kB/s | 26 kB 00:00 2022-09-29T05:17:55.976 INFO:teuthology.orchestra.run.smithi153.stdout:(19/23): leveldb-1.22-1.el8.x86_64.rpm 394 kB/s | 181 kB 00:00 2022-09-29T05:17:56.002 INFO:teuthology.orchestra.run.smithi153.stdout:(20/23): liboath-2.6.2-3.el8.x86_64.rpm 207 kB/s | 59 kB 00:00 2022-09-29T05:17:56.052 INFO:teuthology.orchestra.run.smithi153.stdout:(21/23): libunwind-1.3.1-3.el8.x86_64.rpm 747 kB/s | 75 kB 00:00 2022-09-29T05:17:56.139 INFO:teuthology.orchestra.run.smithi083.stdout:(3/23): libcephfs2-16.2.10-932.g966bd07c.el8.x8 4.8 MB/s | 820 kB 00:00 2022-09-29T05:17:56.447 INFO:teuthology.orchestra.run.smithi083.stdout:(4/23): libradosstriper1-16.2.10-932.g966bd07c. 1.6 MB/s | 493 kB 00:00 2022-09-29T05:17:56.552 INFO:teuthology.orchestra.run.smithi153.stdout:(22/23): librbd1-16.2.10-932.g966bd07c.el8.x86_ 7.6 MB/s | 4.2 MB 00:00 2022-09-29T05:17:56.702 INFO:teuthology.orchestra.run.smithi153.stdout:(23/23): librados2-16.2.10-932.g966bd07c.el8.x8 5.2 MB/s | 3.7 MB 00:00 2022-09-29T05:17:56.706 INFO:teuthology.orchestra.run.smithi153.stdout:-------------------------------------------------------------------------------- 2022-09-29T05:17:56.706 INFO:teuthology.orchestra.run.smithi153.stdout:Total 18 MB/s | 57 MB 00:03 2022-09-29T05:17:56.746 INFO:teuthology.orchestra.run.smithi153.stdout:Running transaction check 2022-09-29T05:17:56.793 INFO:teuthology.orchestra.run.smithi153.stdout:Transaction check succeeded. 2022-09-29T05:17:56.794 INFO:teuthology.orchestra.run.smithi153.stdout:Running transaction test 2022-09-29T05:17:57.006 INFO:teuthology.orchestra.run.smithi083.stdout:(5/23): librgw2-16.2.10-932.g966bd07c.el8.x86_6 6.7 MB/s | 3.8 MB 00:00 2022-09-29T05:17:57.141 INFO:teuthology.orchestra.run.smithi153.stdout:Transaction test succeeded. 2022-09-29T05:17:57.144 INFO:teuthology.orchestra.run.smithi153.stdout:Running transaction 2022-09-29T05:17:57.148 INFO:teuthology.orchestra.run.smithi083.stdout:(6/23): ceph-radosgw-16.2.10-932.g966bd07c.el8. 3.7 MB/s | 12 MB 00:03 2022-09-29T05:17:57.166 INFO:teuthology.orchestra.run.smithi083.stdout:(7/23): python3-ceph-argparse-16.2.10-932.g966b 283 kB/s | 45 kB 00:00 2022-09-29T05:17:57.190 INFO:teuthology.orchestra.run.smithi083.stdout:(8/23): python3-ceph-common-16.2.10-932.g966bd0 2.2 MB/s | 96 kB 00:00 2022-09-29T05:17:57.224 INFO:teuthology.orchestra.run.smithi083.stdout:(9/23): python3-cephfs-16.2.10-932.g966bd07c.el 3.6 MB/s | 215 kB 00:00 2022-09-29T05:17:57.274 INFO:teuthology.orchestra.run.smithi083.stdout:(10/23): python3-rados-16.2.10-932.g966bd07c.el 4.5 MB/s | 387 kB 00:00 2022-09-29T05:17:57.324 INFO:teuthology.orchestra.run.smithi083.stdout:(11/23): python3-rbd-16.2.10-932.g966bd07c.el8. 3.6 MB/s | 368 kB 00:00 2022-09-29T05:17:57.349 INFO:teuthology.orchestra.run.smithi083.stdout:(12/23): python3-rgw-16.2.10-932.g966bd07c.el8. 1.5 MB/s | 114 kB 00:00 2022-09-29T05:17:57.442 INFO:teuthology.orchestra.run.smithi083.stdout:(13/23): librdkafka-0.11.4-3.el8.x86_64.rpm 3.0 MB/s | 354 kB 00:00 2022-09-29T05:17:57.475 INFO:teuthology.orchestra.run.smithi083.stdout:(14/23): lttng-ust-2.8.1-11.el8.x86_64.rpm 2.1 MB/s | 259 kB 00:00 2022-09-29T05:17:57.500 INFO:teuthology.orchestra.run.smithi083.stdout:(15/23): python3-prettytable-0.7.2-14.el8.noarc 751 kB/s | 44 kB 00:00 2022-09-29T05:17:57.526 INFO:teuthology.orchestra.run.smithi083.stdout:(16/23): librabbitmq-0.9.0-3.el8.x86_64.rpm 930 kB/s | 47 kB 00:00 2022-09-29T05:17:57.786 INFO:teuthology.orchestra.run.smithi083.stdout:(17/23): ceph-common-16.2.10-932.g966bd07c.el8. 6.4 MB/s | 24 MB 00:03 2022-09-29T05:17:58.053 INFO:teuthology.orchestra.run.smithi083.stdout:(18/23): leveldb-1.22-1.el8.x86_64.rpm 343 kB/s | 181 kB 00:00 2022-09-29T05:17:58.085 INFO:teuthology.orchestra.run.smithi083.stdout:(19/23): liboath-2.6.2-3.el8.x86_64.rpm 196 kB/s | 59 kB 00:00 2022-09-29T05:17:58.110 INFO:teuthology.orchestra.run.smithi083.stdout:(20/23): gperftools-libs-2.7-9.el8.x86_64.rpm 502 kB/s | 306 kB 00:00 2022-09-29T05:17:58.127 INFO:teuthology.orchestra.run.smithi083.stdout:(21/23): libunwind-1.3.1-3.el8.x86_64.rpm 1.0 MB/s | 75 kB 00:00 2022-09-29T05:17:58.191 INFO:teuthology.orchestra.run.smithi153.stdout: Preparing : 1/1 2022-09-29T05:17:58.452 INFO:teuthology.orchestra.run.smithi083.stdout:(22/23): librbd1-16.2.10-932.g966bd07c.el8.x86_ 12 MB/s | 4.2 MB 00:00 2022-09-29T05:17:58.471 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : liboath-2.6.2-3.el8.x86_64 1/25 2022-09-29T05:17:58.536 INFO:teuthology.orchestra.run.smithi083.stdout:(23/23): librados2-16.2.10-932.g966bd07c.el8.x8 8.3 MB/s | 3.7 MB 00:00 2022-09-29T05:17:58.540 INFO:teuthology.orchestra.run.smithi083.stdout:-------------------------------------------------------------------------------- 2022-09-29T05:17:58.540 INFO:teuthology.orchestra.run.smithi083.stdout:Total 13 MB/s | 57 MB 00:04 2022-09-29T05:17:58.582 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction check 2022-09-29T05:17:58.628 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction check succeeded. 2022-09-29T05:17:58.629 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction test 2022-09-29T05:17:58.657 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : lttng-ust-2.8.1-11.el8.x86_64 2/25 2022-09-29T05:17:59.038 INFO:teuthology.orchestra.run.smithi153.stdout: Running scriptlet: lttng-ust-2.8.1-11.el8.x86_64 2/25 2022-09-29T05:17:59.185 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction test succeeded. 2022-09-29T05:17:59.188 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction 2022-09-29T05:17:59.348 INFO:teuthology.orchestra.run.smithi153.stdout: Upgrading : librados2-2:16.2.10-932.g966bd07c.el8.x86_64 3/25 2022-09-29T05:17:59.452 INFO:teuthology.orchestra.run.smithi153.stdout: Running scriptlet: librados2-2:16.2.10-932.g966bd07c.el8.x86_64 3/25 2022-09-29T05:17:59.705 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : python3-rados-2:16.2.10-932.g966bd07c.el8.x86_64 4/25 2022-09-29T05:17:59.865 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : libcephfs2-2:16.2.10-932.g966bd07c.el8.x86_64 5/25 2022-09-29T05:18:00.276 INFO:teuthology.orchestra.run.smithi153.stdout: Running scriptlet: libcephfs2-2:16.2.10-932.g966bd07c.el8.x86_64 5/25 2022-09-29T05:18:00.343 INFO:teuthology.orchestra.run.smithi083.stdout: Preparing : 1/1 2022-09-29T05:18:00.465 INFO:teuthology.orchestra.run.smithi153.stdout: Upgrading : librbd1-2:16.2.10-932.g966bd07c.el8.x86_64 6/25 2022-09-29T05:18:00.517 INFO:teuthology.orchestra.run.smithi153.stdout: Running scriptlet: librbd1-2:16.2.10-932.g966bd07c.el8.x86_64 6/25 2022-09-29T05:18:00.623 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : liboath-2.6.2-3.el8.x86_64 1/25 2022-09-29T05:18:00.705 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : librabbitmq-0.9.0-3.el8.x86_64 7/25 2022-09-29T05:18:00.840 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : librdkafka-0.11.4-3.el8.x86_64 8/25 2022-09-29T05:18:00.846 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : lttng-ust-2.8.1-11.el8.x86_64 2/25 2022-09-29T05:18:01.225 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: lttng-ust-2.8.1-11.el8.x86_64 2/25 2022-09-29T05:18:01.226 INFO:teuthology.orchestra.run.smithi153.stdout: Running scriptlet: librdkafka-0.11.4-3.el8.x86_64 8/25 2022-09-29T05:18:01.423 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : librgw2-2:16.2.10-932.g966bd07c.el8.x86_64 9/25 2022-09-29T05:18:01.461 INFO:teuthology.orchestra.run.smithi083.stdout: Upgrading : librados2-2:16.2.10-932.g966bd07c.el8.x86_64 3/25 2022-09-29T05:18:01.482 INFO:teuthology.orchestra.run.smithi153.stdout: Running scriptlet: librgw2-2:16.2.10-932.g966bd07c.el8.x86_64 9/25 2022-09-29T05:18:01.562 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: librados2-2:16.2.10-932.g966bd07c.el8.x86_64 3/25 2022-09-29T05:18:01.679 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : leveldb-1.22-1.el8.x86_64 10/25 2022-09-29T05:18:01.835 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-rados-2:16.2.10-932.g966bd07c.el8.x86_64 4/25 2022-09-29T05:18:01.845 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : python3-ceph-argparse-2:16.2.10-932.g966bd07c.el8. 11/25 2022-09-29T05:18:02.028 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : libcephfs2-2:16.2.10-932.g966bd07c.el8.x86_64 5/25 2022-09-29T05:18:02.035 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : python3-cephfs-2:16.2.10-932.g966bd07c.el8.x86_64 12/25 2022-09-29T05:18:02.274 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : python3-rgw-2:16.2.10-932.g966bd07c.el8.x86_64 13/25 2022-09-29T05:18:02.442 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: libcephfs2-2:16.2.10-932.g966bd07c.el8.x86_64 5/25 2022-09-29T05:18:02.489 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : python3-rbd-2:16.2.10-932.g966bd07c.el8.x86_64 14/25 2022-09-29T05:18:02.636 INFO:teuthology.orchestra.run.smithi083.stdout: Upgrading : librbd1-2:16.2.10-932.g966bd07c.el8.x86_64 6/25 2022-09-29T05:18:02.692 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : libradosstriper1-2:16.2.10-932.g966bd07c.el8.x86_6 15/25 2022-09-29T05:18:02.701 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: librbd1-2:16.2.10-932.g966bd07c.el8.x86_64 6/25 2022-09-29T05:18:02.764 INFO:teuthology.orchestra.run.smithi153.stdout: Running scriptlet: libradosstriper1-2:16.2.10-932.g966bd07c.el8.x86_6 15/25 2022-09-29T05:18:02.859 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : librabbitmq-0.9.0-3.el8.x86_64 7/25 2022-09-29T05:18:03.012 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : libunwind-1.3.1-3.el8.x86_64 16/25 2022-09-29T05:18:03.013 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : librdkafka-0.11.4-3.el8.x86_64 8/25 2022-09-29T05:18:03.281 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : gperftools-libs-1:2.7-9.el8.x86_64 17/25 2022-09-29T05:18:03.390 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: librdkafka-0.11.4-3.el8.x86_64 8/25 2022-09-29T05:18:03.491 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : python3-prettytable-0.7.2-14.el8.noarch 18/25 2022-09-29T05:18:03.619 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : librgw2-2:16.2.10-932.g966bd07c.el8.x86_64 9/25 2022-09-29T05:18:03.673 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : python3-ceph-common-2:16.2.10-932.g966bd07c.el8.x8 19/25 2022-09-29T05:18:03.683 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: librgw2-2:16.2.10-932.g966bd07c.el8.x86_64 9/25 2022-09-29T05:18:03.892 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : leveldb-1.22-1.el8.x86_64 10/25 2022-09-29T05:18:04.024 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-ceph-argparse-2:16.2.10-932.g966bd07c.el8. 11/25 2022-09-29T05:18:04.256 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-cephfs-2:16.2.10-932.g966bd07c.el8.x86_64 12/25 2022-09-29T05:18:04.498 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-rgw-2:16.2.10-932.g966bd07c.el8.x86_64 13/25 2022-09-29T05:18:05.112 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-rbd-2:16.2.10-932.g966bd07c.el8.x86_64 14/25 2022-09-29T05:18:05.302 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : libradosstriper1-2:16.2.10-932.g966bd07c.el8.x86_6 15/25 2022-09-29T05:18:05.372 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: libradosstriper1-2:16.2.10-932.g966bd07c.el8.x86_6 15/25 2022-09-29T05:18:05.618 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : libunwind-1.3.1-3.el8.x86_64 16/25 2022-09-29T05:18:05.894 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : gperftools-libs-1:2.7-9.el8.x86_64 17/25 2022-09-29T05:18:06.080 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-prettytable-0.7.2-14.el8.noarch 18/25 2022-09-29T05:18:06.244 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-ceph-common-2:16.2.10-932.g966bd07c.el8.x8 19/25 2022-09-29T05:18:07.344 INFO:teuthology.orchestra.run.smithi153.stdout: Running scriptlet: ceph-common-2:16.2.10-932.g966bd07c.el8.x86_64 20/25 2022-09-29T05:18:07.548 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : ceph-common-2:16.2.10-932.g966bd07c.el8.x86_64 20/25 2022-09-29T05:18:08.235 INFO:teuthology.orchestra.run.smithi153.stdout: Running scriptlet: ceph-common-2:16.2.10-932.g966bd07c.el8.x86_64 20/25 2022-09-29T05:18:08.645 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : ceph-base-2:16.2.10-932.g966bd07c.el8.x86_64 21/25 2022-09-29T05:18:08.744 INFO:teuthology.orchestra.run.smithi153.stdout: Running scriptlet: ceph-base-2:16.2.10-932.g966bd07c.el8.x86_64 21/25 2022-09-29T05:18:08.947 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : ceph-selinux-2:16.2.10-932.g966bd07c.el8.x86_64 22/25 2022-09-29T05:18:09.870 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: ceph-common-2:16.2.10-932.g966bd07c.el8.x86_64 20/25 2022-09-29T05:18:10.077 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : ceph-common-2:16.2.10-932.g966bd07c.el8.x86_64 20/25 2022-09-29T05:18:10.767 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: ceph-common-2:16.2.10-932.g966bd07c.el8.x86_64 20/25 2022-09-29T05:18:11.056 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : ceph-base-2:16.2.10-932.g966bd07c.el8.x86_64 21/25 2022-09-29T05:18:11.164 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: ceph-base-2:16.2.10-932.g966bd07c.el8.x86_64 21/25 2022-09-29T05:18:11.335 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : ceph-selinux-2:16.2.10-932.g966bd07c.el8.x86_64 22/25 2022-09-29T05:18:28.025 INFO:teuthology.orchestra.run.smithi153.stdout: Running scriptlet: ceph-selinux-2:16.2.10-932.g966bd07c.el8.x86_64 22/25 2022-09-29T05:18:28.026 INFO:teuthology.orchestra.run.smithi153.stdout:skipping the directory /sys 2022-09-29T05:18:28.026 INFO:teuthology.orchestra.run.smithi153.stdout:skipping the directory /proc 2022-09-29T05:18:28.027 INFO:teuthology.orchestra.run.smithi153.stdout:skipping the directory /dev 2022-09-29T05:18:28.028 INFO:teuthology.orchestra.run.smithi153.stdout:skipping the directory /run 2022-09-29T05:18:28.028 INFO:teuthology.orchestra.run.smithi153.stdout:skipping the directory /mnt 2022-09-29T05:18:28.029 INFO:teuthology.orchestra.run.smithi153.stdout:skipping the directory /var/tmp 2022-09-29T05:18:28.029 INFO:teuthology.orchestra.run.smithi153.stdout:skipping the directory /home 2022-09-29T05:18:28.029 INFO:teuthology.orchestra.run.smithi153.stdout:skipping the directory /tmp 2022-09-29T05:18:28.033 INFO:teuthology.orchestra.run.smithi153.stdout:skipping the directory /dev 2022-09-29T05:18:28.034 INFO:teuthology.orchestra.run.smithi153.stdout: 2022-09-29T05:18:29.239 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : ceph-radosgw-2:16.2.10-932.g966bd07c.el8.x86_64 23/25 2022-09-29T05:18:29.306 INFO:teuthology.orchestra.run.smithi153.stdout: Running scriptlet: ceph-radosgw-2:16.2.10-932.g966bd07c.el8.x86_64 23/25 2022-09-29T05:18:29.307 INFO:teuthology.orchestra.run.smithi153.stdout: Cleanup : librbd1-1:12.2.7-9.el8.x86_64 24/25 2022-09-29T05:18:29.529 INFO:teuthology.orchestra.run.smithi153.stdout: Running scriptlet: librbd1-1:12.2.7-9.el8.x86_64 24/25 2022-09-29T05:18:29.529 INFO:teuthology.orchestra.run.smithi153.stdout: Cleanup : librados2-1:12.2.7-9.el8.x86_64 25/25 2022-09-29T05:18:30.174 INFO:teuthology.orchestra.run.smithi153.stdout: Running scriptlet: librados2-1:12.2.7-9.el8.x86_64 25/25 2022-09-29T05:18:30.175 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : ceph-base-2:16.2.10-932.g966bd07c.el8.x86_64 1/25 2022-09-29T05:18:30.175 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : ceph-common-2:16.2.10-932.g966bd07c.el8.x86_64 2/25 2022-09-29T05:18:30.176 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : ceph-radosgw-2:16.2.10-932.g966bd07c.el8.x86_64 3/25 2022-09-29T05:18:30.176 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : ceph-selinux-2:16.2.10-932.g966bd07c.el8.x86_64 4/25 2022-09-29T05:18:30.177 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : libcephfs2-2:16.2.10-932.g966bd07c.el8.x86_64 5/25 2022-09-29T05:18:30.177 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : libradosstriper1-2:16.2.10-932.g966bd07c.el8.x86_6 6/25 2022-09-29T05:18:30.186 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : librgw2-2:16.2.10-932.g966bd07c.el8.x86_64 7/25 2022-09-29T05:18:30.186 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : python3-ceph-argparse-2:16.2.10-932.g966bd07c.el8. 8/25 2022-09-29T05:18:30.187 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : python3-ceph-common-2:16.2.10-932.g966bd07c.el8.x8 9/25 2022-09-29T05:18:30.187 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : python3-cephfs-2:16.2.10-932.g966bd07c.el8.x86_64 10/25 2022-09-29T05:18:30.195 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : python3-rados-2:16.2.10-932.g966bd07c.el8.x86_64 11/25 2022-09-29T05:18:30.196 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : python3-rbd-2:16.2.10-932.g966bd07c.el8.x86_64 12/25 2022-09-29T05:18:30.196 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : python3-rgw-2:16.2.10-932.g966bd07c.el8.x86_64 13/25 2022-09-29T05:18:30.197 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : librdkafka-0.11.4-3.el8.x86_64 14/25 2022-09-29T05:18:30.197 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : lttng-ust-2.8.1-11.el8.x86_64 15/25 2022-09-29T05:18:30.198 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : python3-prettytable-0.7.2-14.el8.noarch 16/25 2022-09-29T05:18:30.198 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : librabbitmq-0.9.0-3.el8.x86_64 17/25 2022-09-29T05:18:30.202 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : gperftools-libs-1:2.7-9.el8.x86_64 18/25 2022-09-29T05:18:30.203 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : leveldb-1.22-1.el8.x86_64 19/25 2022-09-29T05:18:30.203 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : liboath-2.6.2-3.el8.x86_64 20/25 2022-09-29T05:18:30.203 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : libunwind-1.3.1-3.el8.x86_64 21/25 2022-09-29T05:18:30.204 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : librados2-2:16.2.10-932.g966bd07c.el8.x86_64 22/25 2022-09-29T05:18:30.204 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : librados2-1:12.2.7-9.el8.x86_64 23/25 2022-09-29T05:18:30.204 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : librbd1-2:16.2.10-932.g966bd07c.el8.x86_64 24/25 2022-09-29T05:18:30.606 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: ceph-selinux-2:16.2.10-932.g966bd07c.el8.x86_64 22/25 2022-09-29T05:18:30.607 INFO:teuthology.orchestra.run.smithi083.stdout:skipping the directory /sys 2022-09-29T05:18:30.607 INFO:teuthology.orchestra.run.smithi083.stdout:skipping the directory /proc 2022-09-29T05:18:30.607 INFO:teuthology.orchestra.run.smithi083.stdout:skipping the directory /dev 2022-09-29T05:18:30.607 INFO:teuthology.orchestra.run.smithi083.stdout:skipping the directory /run 2022-09-29T05:18:30.608 INFO:teuthology.orchestra.run.smithi083.stdout:skipping the directory /mnt 2022-09-29T05:18:30.608 INFO:teuthology.orchestra.run.smithi083.stdout:skipping the directory /var/tmp 2022-09-29T05:18:30.608 INFO:teuthology.orchestra.run.smithi083.stdout:skipping the directory /home 2022-09-29T05:18:30.608 INFO:teuthology.orchestra.run.smithi083.stdout:skipping the directory /tmp 2022-09-29T05:18:30.609 INFO:teuthology.orchestra.run.smithi083.stdout:skipping the directory /dev 2022-09-29T05:18:30.609 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:18:31.120 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : librbd1-1:12.2.7-9.el8.x86_64 25/25 2022-09-29T05:18:31.120 INFO:teuthology.orchestra.run.smithi153.stdout: 2022-09-29T05:18:31.120 INFO:teuthology.orchestra.run.smithi153.stdout:Upgraded: 2022-09-29T05:18:31.120 INFO:teuthology.orchestra.run.smithi153.stdout: librados2-2:16.2.10-932.g966bd07c.el8.x86_64 2022-09-29T05:18:31.121 INFO:teuthology.orchestra.run.smithi153.stdout: librbd1-2:16.2.10-932.g966bd07c.el8.x86_64 2022-09-29T05:18:31.121 INFO:teuthology.orchestra.run.smithi153.stdout:Installed: 2022-09-29T05:18:31.121 INFO:teuthology.orchestra.run.smithi153.stdout: ceph-base-2:16.2.10-932.g966bd07c.el8.x86_64 2022-09-29T05:18:31.121 INFO:teuthology.orchestra.run.smithi153.stdout: ceph-common-2:16.2.10-932.g966bd07c.el8.x86_64 2022-09-29T05:18:31.122 INFO:teuthology.orchestra.run.smithi153.stdout: ceph-radosgw-2:16.2.10-932.g966bd07c.el8.x86_64 2022-09-29T05:18:31.122 INFO:teuthology.orchestra.run.smithi153.stdout: ceph-selinux-2:16.2.10-932.g966bd07c.el8.x86_64 2022-09-29T05:18:31.122 INFO:teuthology.orchestra.run.smithi153.stdout: gperftools-libs-1:2.7-9.el8.x86_64 2022-09-29T05:18:31.123 INFO:teuthology.orchestra.run.smithi153.stdout: leveldb-1.22-1.el8.x86_64 2022-09-29T05:18:31.123 INFO:teuthology.orchestra.run.smithi153.stdout: libcephfs2-2:16.2.10-932.g966bd07c.el8.x86_64 2022-09-29T05:18:31.123 INFO:teuthology.orchestra.run.smithi153.stdout: liboath-2.6.2-3.el8.x86_64 2022-09-29T05:18:31.123 INFO:teuthology.orchestra.run.smithi153.stdout: librabbitmq-0.9.0-3.el8.x86_64 2022-09-29T05:18:31.124 INFO:teuthology.orchestra.run.smithi153.stdout: libradosstriper1-2:16.2.10-932.g966bd07c.el8.x86_64 2022-09-29T05:18:31.124 INFO:teuthology.orchestra.run.smithi153.stdout: librdkafka-0.11.4-3.el8.x86_64 2022-09-29T05:18:31.124 INFO:teuthology.orchestra.run.smithi153.stdout: librgw2-2:16.2.10-932.g966bd07c.el8.x86_64 2022-09-29T05:18:31.124 INFO:teuthology.orchestra.run.smithi153.stdout: libunwind-1.3.1-3.el8.x86_64 2022-09-29T05:18:31.124 INFO:teuthology.orchestra.run.smithi153.stdout: lttng-ust-2.8.1-11.el8.x86_64 2022-09-29T05:18:31.125 INFO:teuthology.orchestra.run.smithi153.stdout: python3-ceph-argparse-2:16.2.10-932.g966bd07c.el8.x86_64 2022-09-29T05:18:31.125 INFO:teuthology.orchestra.run.smithi153.stdout: python3-ceph-common-2:16.2.10-932.g966bd07c.el8.x86_64 2022-09-29T05:18:31.125 INFO:teuthology.orchestra.run.smithi153.stdout: python3-cephfs-2:16.2.10-932.g966bd07c.el8.x86_64 2022-09-29T05:18:31.125 INFO:teuthology.orchestra.run.smithi153.stdout: python3-prettytable-0.7.2-14.el8.noarch 2022-09-29T05:18:31.136 INFO:teuthology.orchestra.run.smithi153.stdout: python3-rados-2:16.2.10-932.g966bd07c.el8.x86_64 2022-09-29T05:18:31.147 INFO:teuthology.orchestra.run.smithi153.stdout: python3-rbd-2:16.2.10-932.g966bd07c.el8.x86_64 2022-09-29T05:18:31.147 INFO:teuthology.orchestra.run.smithi153.stdout: python3-rgw-2:16.2.10-932.g966bd07c.el8.x86_64 2022-09-29T05:18:31.147 INFO:teuthology.orchestra.run.smithi153.stdout: 2022-09-29T05:18:31.148 INFO:teuthology.orchestra.run.smithi153.stdout:Complete! 2022-09-29T05:18:31.392 DEBUG:teuthology.orchestra.run.smithi153:> sudo yum -y install ceph-test 2022-09-29T05:18:31.770 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : ceph-radosgw-2:16.2.10-932.g966bd07c.el8.x86_64 23/25 2022-09-29T05:18:31.830 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: ceph-radosgw-2:16.2.10-932.g966bd07c.el8.x86_64 23/25 2022-09-29T05:18:31.831 INFO:teuthology.orchestra.run.smithi083.stdout: Cleanup : librbd1-1:12.2.7-9.el8.x86_64 24/25 2022-09-29T05:18:31.970 INFO:teuthology.orchestra.run.smithi153.stdout:Last metadata expiration check: 0:00:41 ago on Thu 29 Sep 2022 05:17:50 AM UTC. 2022-09-29T05:18:32.043 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: librbd1-1:12.2.7-9.el8.x86_64 24/25 2022-09-29T05:18:32.043 INFO:teuthology.orchestra.run.smithi083.stdout: Cleanup : librados2-1:12.2.7-9.el8.x86_64 25/25 2022-09-29T05:18:32.700 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: librados2-1:12.2.7-9.el8.x86_64 25/25 2022-09-29T05:18:32.700 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : ceph-base-2:16.2.10-932.g966bd07c.el8.x86_64 1/25 2022-09-29T05:18:32.700 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : ceph-common-2:16.2.10-932.g966bd07c.el8.x86_64 2/25 2022-09-29T05:18:32.701 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : ceph-radosgw-2:16.2.10-932.g966bd07c.el8.x86_64 3/25 2022-09-29T05:18:32.701 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : ceph-selinux-2:16.2.10-932.g966bd07c.el8.x86_64 4/25 2022-09-29T05:18:32.701 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : libcephfs2-2:16.2.10-932.g966bd07c.el8.x86_64 5/25 2022-09-29T05:18:32.701 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : libradosstriper1-2:16.2.10-932.g966bd07c.el8.x86_6 6/25 2022-09-29T05:18:32.702 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : librgw2-2:16.2.10-932.g966bd07c.el8.x86_64 7/25 2022-09-29T05:18:32.702 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-ceph-argparse-2:16.2.10-932.g966bd07c.el8. 8/25 2022-09-29T05:18:32.702 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-ceph-common-2:16.2.10-932.g966bd07c.el8.x8 9/25 2022-09-29T05:18:32.702 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-cephfs-2:16.2.10-932.g966bd07c.el8.x86_64 10/25 2022-09-29T05:18:32.703 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-rados-2:16.2.10-932.g966bd07c.el8.x86_64 11/25 2022-09-29T05:18:32.703 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-rbd-2:16.2.10-932.g966bd07c.el8.x86_64 12/25 2022-09-29T05:18:32.703 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-rgw-2:16.2.10-932.g966bd07c.el8.x86_64 13/25 2022-09-29T05:18:32.703 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : librdkafka-0.11.4-3.el8.x86_64 14/25 2022-09-29T05:18:32.703 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : lttng-ust-2.8.1-11.el8.x86_64 15/25 2022-09-29T05:18:32.704 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-prettytable-0.7.2-14.el8.noarch 16/25 2022-09-29T05:18:32.704 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : librabbitmq-0.9.0-3.el8.x86_64 17/25 2022-09-29T05:18:32.704 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : gperftools-libs-1:2.7-9.el8.x86_64 18/25 2022-09-29T05:18:32.704 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : leveldb-1.22-1.el8.x86_64 19/25 2022-09-29T05:18:32.705 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : liboath-2.6.2-3.el8.x86_64 20/25 2022-09-29T05:18:32.705 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : libunwind-1.3.1-3.el8.x86_64 21/25 2022-09-29T05:18:32.705 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : librados2-2:16.2.10-932.g966bd07c.el8.x86_64 22/25 2022-09-29T05:18:32.705 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : librados2-1:12.2.7-9.el8.x86_64 23/25 2022-09-29T05:18:32.709 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : librbd1-2:16.2.10-932.g966bd07c.el8.x86_64 24/25 2022-09-29T05:18:33.051 INFO:teuthology.orchestra.run.smithi153.stdout:Dependencies resolved. 2022-09-29T05:18:33.052 INFO:teuthology.orchestra.run.smithi153.stdout:================================================================================ 2022-09-29T05:18:33.053 INFO:teuthology.orchestra.run.smithi153.stdout: Package Arch Version Repository Size 2022-09-29T05:18:33.058 INFO:teuthology.orchestra.run.smithi153.stdout:================================================================================ 2022-09-29T05:18:33.059 INFO:teuthology.orchestra.run.smithi153.stdout:Installing: 2022-09-29T05:18:33.059 INFO:teuthology.orchestra.run.smithi153.stdout: ceph-test x86_64 2:16.2.10-932.g966bd07c.el8 ceph 52 M 2022-09-29T05:18:33.059 INFO:teuthology.orchestra.run.smithi153.stdout:Installing dependencies: 2022-09-29T05:18:33.059 INFO:teuthology.orchestra.run.smithi153.stdout: jq x86_64 1.6-3.el8 CentOS-AppStream 202 k 2022-09-29T05:18:33.060 INFO:teuthology.orchestra.run.smithi153.stdout: libcephsqlite x86_64 2:16.2.10-932.g966bd07c.el8 ceph 173 k 2022-09-29T05:18:33.060 INFO:teuthology.orchestra.run.smithi153.stdout: oniguruma x86_64 6.8.2-2.el8 CentOS-AppStream 187 k 2022-09-29T05:18:33.060 INFO:teuthology.orchestra.run.smithi153.stdout: socat x86_64 1.7.4.1-1.el8 CentOS-AppStream 323 k 2022-09-29T05:18:33.060 INFO:teuthology.orchestra.run.smithi153.stdout: xmlstarlet x86_64 1.6.1-20.el8 CentOS-PowerTools 71 k 2022-09-29T05:18:33.060 INFO:teuthology.orchestra.run.smithi153.stdout: 2022-09-29T05:18:33.061 INFO:teuthology.orchestra.run.smithi153.stdout:Transaction Summary 2022-09-29T05:18:33.061 INFO:teuthology.orchestra.run.smithi153.stdout:================================================================================ 2022-09-29T05:18:33.061 INFO:teuthology.orchestra.run.smithi153.stdout:Install 6 Packages 2022-09-29T05:18:33.061 INFO:teuthology.orchestra.run.smithi153.stdout: 2022-09-29T05:18:33.062 INFO:teuthology.orchestra.run.smithi153.stdout:Total download size: 53 M 2022-09-29T05:18:33.062 INFO:teuthology.orchestra.run.smithi153.stdout:Installed size: 195 M 2022-09-29T05:18:33.062 INFO:teuthology.orchestra.run.smithi153.stdout:Downloading Packages: 2022-09-29T05:18:33.114 INFO:teuthology.orchestra.run.smithi153.stdout:(1/6): jq-1.6-3.el8.x86_64.rpm 3.1 MB/s | 202 kB 00:00 2022-09-29T05:18:33.138 INFO:teuthology.orchestra.run.smithi153.stdout:(2/6): oniguruma-6.8.2-2.el8.x86_64.rpm 7.6 MB/s | 187 kB 00:00 2022-09-29T05:18:33.183 INFO:teuthology.orchestra.run.smithi153.stdout:(3/6): socat-1.7.4.1-1.el8.x86_64.rpm 7.6 MB/s | 323 kB 00:00 2022-09-29T05:18:33.208 INFO:teuthology.orchestra.run.smithi153.stdout:(4/6): xmlstarlet-1.6.1-20.el8.x86_64.rpm 2.8 MB/s | 71 kB 00:00 2022-09-29T05:18:33.263 INFO:teuthology.orchestra.run.smithi153.stdout:(5/6): libcephsqlite-16.2.10-932.g966bd07c.el8. 878 kB/s | 173 kB 00:00 2022-09-29T05:18:33.622 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : librbd1-1:12.2.7-9.el8.x86_64 25/25 2022-09-29T05:18:33.623 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:18:33.623 INFO:teuthology.orchestra.run.smithi083.stdout:Upgraded: 2022-09-29T05:18:33.624 INFO:teuthology.orchestra.run.smithi083.stdout: librados2-2:16.2.10-932.g966bd07c.el8.x86_64 2022-09-29T05:18:33.624 INFO:teuthology.orchestra.run.smithi083.stdout: librbd1-2:16.2.10-932.g966bd07c.el8.x86_64 2022-09-29T05:18:33.624 INFO:teuthology.orchestra.run.smithi083.stdout:Installed: 2022-09-29T05:18:33.625 INFO:teuthology.orchestra.run.smithi083.stdout: ceph-base-2:16.2.10-932.g966bd07c.el8.x86_64 2022-09-29T05:18:33.625 INFO:teuthology.orchestra.run.smithi083.stdout: ceph-common-2:16.2.10-932.g966bd07c.el8.x86_64 2022-09-29T05:18:33.626 INFO:teuthology.orchestra.run.smithi083.stdout: ceph-radosgw-2:16.2.10-932.g966bd07c.el8.x86_64 2022-09-29T05:18:33.626 INFO:teuthology.orchestra.run.smithi083.stdout: ceph-selinux-2:16.2.10-932.g966bd07c.el8.x86_64 2022-09-29T05:18:33.627 INFO:teuthology.orchestra.run.smithi083.stdout: gperftools-libs-1:2.7-9.el8.x86_64 2022-09-29T05:18:33.627 INFO:teuthology.orchestra.run.smithi083.stdout: leveldb-1.22-1.el8.x86_64 2022-09-29T05:18:33.628 INFO:teuthology.orchestra.run.smithi083.stdout: libcephfs2-2:16.2.10-932.g966bd07c.el8.x86_64 2022-09-29T05:18:33.628 INFO:teuthology.orchestra.run.smithi083.stdout: liboath-2.6.2-3.el8.x86_64 2022-09-29T05:18:33.629 INFO:teuthology.orchestra.run.smithi083.stdout: librabbitmq-0.9.0-3.el8.x86_64 2022-09-29T05:18:33.629 INFO:teuthology.orchestra.run.smithi083.stdout: libradosstriper1-2:16.2.10-932.g966bd07c.el8.x86_64 2022-09-29T05:18:33.630 INFO:teuthology.orchestra.run.smithi083.stdout: librdkafka-0.11.4-3.el8.x86_64 2022-09-29T05:18:33.643 INFO:teuthology.orchestra.run.smithi083.stdout: librgw2-2:16.2.10-932.g966bd07c.el8.x86_64 2022-09-29T05:18:33.643 INFO:teuthology.orchestra.run.smithi083.stdout: libunwind-1.3.1-3.el8.x86_64 2022-09-29T05:18:33.644 INFO:teuthology.orchestra.run.smithi083.stdout: lttng-ust-2.8.1-11.el8.x86_64 2022-09-29T05:18:33.644 INFO:teuthology.orchestra.run.smithi083.stdout: python3-ceph-argparse-2:16.2.10-932.g966bd07c.el8.x86_64 2022-09-29T05:18:33.645 INFO:teuthology.orchestra.run.smithi083.stdout: python3-ceph-common-2:16.2.10-932.g966bd07c.el8.x86_64 2022-09-29T05:18:33.645 INFO:teuthology.orchestra.run.smithi083.stdout: python3-cephfs-2:16.2.10-932.g966bd07c.el8.x86_64 2022-09-29T05:18:33.645 INFO:teuthology.orchestra.run.smithi083.stdout: python3-prettytable-0.7.2-14.el8.noarch 2022-09-29T05:18:33.646 INFO:teuthology.orchestra.run.smithi083.stdout: python3-rados-2:16.2.10-932.g966bd07c.el8.x86_64 2022-09-29T05:18:33.647 INFO:teuthology.orchestra.run.smithi083.stdout: python3-rbd-2:16.2.10-932.g966bd07c.el8.x86_64 2022-09-29T05:18:33.647 INFO:teuthology.orchestra.run.smithi083.stdout: python3-rgw-2:16.2.10-932.g966bd07c.el8.x86_64 2022-09-29T05:18:33.647 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:18:33.648 INFO:teuthology.orchestra.run.smithi083.stdout:Complete! 2022-09-29T05:18:33.950 DEBUG:teuthology.orchestra.run.smithi083:> sudo yum -y install ceph-test 2022-09-29T05:18:34.528 INFO:teuthology.orchestra.run.smithi083.stdout:Last metadata expiration check: 0:00:44 ago on Thu 29 Sep 2022 05:17:50 AM UTC. 2022-09-29T05:18:35.572 INFO:teuthology.orchestra.run.smithi153.stdout:(6/6): ceph-test-16.2.10-932.g966bd07c.el8.x86_ 21 MB/s | 52 MB 00:02 2022-09-29T05:18:35.576 INFO:teuthology.orchestra.run.smithi153.stdout:-------------------------------------------------------------------------------- 2022-09-29T05:18:35.577 INFO:teuthology.orchestra.run.smithi153.stdout:Total 21 MB/s | 53 MB 00:02 2022-09-29T05:18:35.594 INFO:teuthology.orchestra.run.smithi083.stdout:Dependencies resolved. 2022-09-29T05:18:35.595 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2022-09-29T05:18:35.595 INFO:teuthology.orchestra.run.smithi083.stdout: Package Arch Version Repository Size 2022-09-29T05:18:35.596 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2022-09-29T05:18:35.596 INFO:teuthology.orchestra.run.smithi083.stdout:Installing: 2022-09-29T05:18:35.597 INFO:teuthology.orchestra.run.smithi083.stdout: ceph-test x86_64 2:16.2.10-932.g966bd07c.el8 ceph 52 M 2022-09-29T05:18:35.597 INFO:teuthology.orchestra.run.smithi083.stdout:Installing dependencies: 2022-09-29T05:18:35.598 INFO:teuthology.orchestra.run.smithi083.stdout: jq x86_64 1.6-3.el8 CentOS-AppStream 202 k 2022-09-29T05:18:35.603 INFO:teuthology.orchestra.run.smithi083.stdout: libcephsqlite x86_64 2:16.2.10-932.g966bd07c.el8 ceph 173 k 2022-09-29T05:18:35.604 INFO:teuthology.orchestra.run.smithi083.stdout: oniguruma x86_64 6.8.2-2.el8 CentOS-AppStream 187 k 2022-09-29T05:18:35.604 INFO:teuthology.orchestra.run.smithi083.stdout: socat x86_64 1.7.4.1-1.el8 CentOS-AppStream 323 k 2022-09-29T05:18:35.604 INFO:teuthology.orchestra.run.smithi083.stdout: xmlstarlet x86_64 1.6.1-20.el8 CentOS-PowerTools 71 k 2022-09-29T05:18:35.605 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:18:35.605 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction Summary 2022-09-29T05:18:35.606 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2022-09-29T05:18:35.606 INFO:teuthology.orchestra.run.smithi083.stdout:Install 6 Packages 2022-09-29T05:18:35.607 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:18:35.608 INFO:teuthology.orchestra.run.smithi083.stdout:Total download size: 53 M 2022-09-29T05:18:35.609 INFO:teuthology.orchestra.run.smithi083.stdout:Installed size: 195 M 2022-09-29T05:18:35.609 INFO:teuthology.orchestra.run.smithi083.stdout:Downloading Packages: 2022-09-29T05:18:35.619 INFO:teuthology.orchestra.run.smithi153.stdout:Running transaction check 2022-09-29T05:18:35.638 INFO:teuthology.orchestra.run.smithi153.stdout:Transaction check succeeded. 2022-09-29T05:18:35.639 INFO:teuthology.orchestra.run.smithi153.stdout:Running transaction test 2022-09-29T05:18:35.678 INFO:teuthology.orchestra.run.smithi083.stdout:(1/6): jq-1.6-3.el8.x86_64.rpm 2.6 MB/s | 202 kB 00:00 2022-09-29T05:18:35.693 INFO:teuthology.orchestra.run.smithi083.stdout:(2/6): oniguruma-6.8.2-2.el8.x86_64.rpm 12 MB/s | 187 kB 00:00 2022-09-29T05:18:35.725 INFO:teuthology.orchestra.run.smithi083.stdout:(3/6): socat-1.7.4.1-1.el8.x86_64.rpm 9.5 MB/s | 323 kB 00:00 2022-09-29T05:18:35.751 INFO:teuthology.orchestra.run.smithi083.stdout:(4/6): xmlstarlet-1.6.1-20.el8.x86_64.rpm 2.8 MB/s | 71 kB 00:00 2022-09-29T05:18:35.822 INFO:teuthology.orchestra.run.smithi083.stdout:(5/6): libcephsqlite-16.2.10-932.g966bd07c.el8. 791 kB/s | 173 kB 00:00 2022-09-29T05:18:35.967 INFO:teuthology.orchestra.run.smithi153.stdout:Transaction test succeeded. 2022-09-29T05:18:35.967 INFO:teuthology.orchestra.run.smithi153.stdout:Running transaction 2022-09-29T05:18:36.465 INFO:teuthology.orchestra.run.smithi153.stdout: Preparing : 1/1 2022-09-29T05:18:36.642 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : xmlstarlet-1.6.1-20.el8.x86_64 1/6 2022-09-29T05:18:36.847 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : socat-1.7.4.1-1.el8.x86_64 2/6 2022-09-29T05:18:37.001 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : oniguruma-6.8.2-2.el8.x86_64 3/6 2022-09-29T05:18:37.087 INFO:teuthology.orchestra.run.smithi153.stdout: Running scriptlet: oniguruma-6.8.2-2.el8.x86_64 3/6 2022-09-29T05:18:37.274 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : jq-1.6-3.el8.x86_64 4/6 2022-09-29T05:18:37.452 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : libcephsqlite-2:16.2.10-932.g966bd07c.el8.x86_64 5/6 2022-09-29T05:18:38.116 INFO:teuthology.orchestra.run.smithi083.stdout:(6/6): ceph-test-16.2.10-932.g966bd07c.el8.x86_ 21 MB/s | 52 MB 00:02 2022-09-29T05:18:38.120 INFO:teuthology.orchestra.run.smithi083.stdout:-------------------------------------------------------------------------------- 2022-09-29T05:18:38.120 INFO:teuthology.orchestra.run.smithi083.stdout:Total 21 MB/s | 53 MB 00:02 2022-09-29T05:18:38.162 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction check 2022-09-29T05:18:38.184 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction check succeeded. 2022-09-29T05:18:38.185 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction test 2022-09-29T05:18:38.502 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction test succeeded. 2022-09-29T05:18:38.505 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction 2022-09-29T05:18:39.029 INFO:teuthology.orchestra.run.smithi083.stdout: Preparing : 1/1 2022-09-29T05:18:39.251 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : xmlstarlet-1.6.1-20.el8.x86_64 1/6 2022-09-29T05:18:39.477 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : socat-1.7.4.1-1.el8.x86_64 2/6 2022-09-29T05:18:39.611 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : oniguruma-6.8.2-2.el8.x86_64 3/6 2022-09-29T05:18:39.696 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: oniguruma-6.8.2-2.el8.x86_64 3/6 2022-09-29T05:18:39.887 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : jq-1.6-3.el8.x86_64 4/6 2022-09-29T05:18:40.047 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : libcephsqlite-2:16.2.10-932.g966bd07c.el8.x86_64 5/6 2022-09-29T05:18:42.059 INFO:teuthology.orchestra.run.smithi153.stdout: Running scriptlet: libcephsqlite-2:16.2.10-932.g966bd07c.el8.x86_64 5/6 2022-09-29T05:18:42.289 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : ceph-test-2:16.2.10-932.g966bd07c.el8.x86_64 6/6 2022-09-29T05:18:42.539 INFO:teuthology.orchestra.run.smithi153.stdout: Running scriptlet: ceph-test-2:16.2.10-932.g966bd07c.el8.x86_64 6/6 2022-09-29T05:18:42.539 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : ceph-test-2:16.2.10-932.g966bd07c.el8.x86_64 1/6 2022-09-29T05:18:42.540 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : libcephsqlite-2:16.2.10-932.g966bd07c.el8.x86_64 2/6 2022-09-29T05:18:42.540 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : jq-1.6-3.el8.x86_64 3/6 2022-09-29T05:18:42.540 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : oniguruma-6.8.2-2.el8.x86_64 4/6 2022-09-29T05:18:42.541 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : socat-1.7.4.1-1.el8.x86_64 5/6 2022-09-29T05:18:43.021 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : xmlstarlet-1.6.1-20.el8.x86_64 6/6 2022-09-29T05:18:43.022 INFO:teuthology.orchestra.run.smithi153.stdout: 2022-09-29T05:18:43.022 INFO:teuthology.orchestra.run.smithi153.stdout:Installed: 2022-09-29T05:18:43.022 INFO:teuthology.orchestra.run.smithi153.stdout: ceph-test-2:16.2.10-932.g966bd07c.el8.x86_64 2022-09-29T05:18:43.023 INFO:teuthology.orchestra.run.smithi153.stdout: jq-1.6-3.el8.x86_64 2022-09-29T05:18:43.023 INFO:teuthology.orchestra.run.smithi153.stdout: libcephsqlite-2:16.2.10-932.g966bd07c.el8.x86_64 2022-09-29T05:18:43.023 INFO:teuthology.orchestra.run.smithi153.stdout: oniguruma-6.8.2-2.el8.x86_64 2022-09-29T05:18:43.023 INFO:teuthology.orchestra.run.smithi153.stdout: socat-1.7.4.1-1.el8.x86_64 2022-09-29T05:18:43.024 INFO:teuthology.orchestra.run.smithi153.stdout: xmlstarlet-1.6.1-20.el8.x86_64 2022-09-29T05:18:43.024 INFO:teuthology.orchestra.run.smithi153.stdout: 2022-09-29T05:18:43.024 INFO:teuthology.orchestra.run.smithi153.stdout:Complete! 2022-09-29T05:18:43.201 DEBUG:teuthology.orchestra.run.smithi153:> sudo yum -y install ceph 2022-09-29T05:18:43.820 INFO:teuthology.orchestra.run.smithi153.stdout:Last metadata expiration check: 0:00:53 ago on Thu 29 Sep 2022 05:17:50 AM UTC. 2022-09-29T05:18:44.627 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: libcephsqlite-2:16.2.10-932.g966bd07c.el8.x86_64 5/6 2022-09-29T05:18:44.881 INFO:teuthology.orchestra.run.smithi153.stdout:Dependencies resolved. 2022-09-29T05:18:44.884 INFO:teuthology.orchestra.run.smithi153.stdout:======================================================================================= 2022-09-29T05:18:44.884 INFO:teuthology.orchestra.run.smithi153.stdout: Package Arch Version Repository Size 2022-09-29T05:18:44.885 INFO:teuthology.orchestra.run.smithi153.stdout:======================================================================================= 2022-09-29T05:18:44.885 INFO:teuthology.orchestra.run.smithi153.stdout:Installing: 2022-09-29T05:18:44.885 INFO:teuthology.orchestra.run.smithi153.stdout: ceph x86_64 2:16.2.10-932.g966bd07c.el8 ceph 6.4 k 2022-09-29T05:18:44.885 INFO:teuthology.orchestra.run.smithi153.stdout:Installing dependencies: 2022-09-29T05:18:44.886 INFO:teuthology.orchestra.run.smithi153.stdout: ceph-mds x86_64 2:16.2.10-932.g966bd07c.el8 ceph 2.4 M 2022-09-29T05:18:44.886 INFO:teuthology.orchestra.run.smithi153.stdout: ceph-mgr x86_64 2:16.2.10-932.g966bd07c.el8 ceph 1.6 M 2022-09-29T05:18:44.886 INFO:teuthology.orchestra.run.smithi153.stdout: ceph-mgr-modules-core noarch 2:16.2.10-932.g966bd07c.el8 ceph-noarch 241 k 2022-09-29T05:18:44.886 INFO:teuthology.orchestra.run.smithi153.stdout: ceph-mon x86_64 2:16.2.10-932.g966bd07c.el8 ceph 5.0 M 2022-09-29T05:18:44.887 INFO:teuthology.orchestra.run.smithi153.stdout: ceph-osd x86_64 2:16.2.10-932.g966bd07c.el8 ceph 18 M 2022-09-29T05:18:44.887 INFO:teuthology.orchestra.run.smithi153.stdout: python3-bcrypt x86_64 3.1.6-2.el8.1 epel 44 k 2022-09-29T05:18:44.887 INFO:teuthology.orchestra.run.smithi153.stdout: python3-beautifulsoup4 noarch 4.6.3-2.el8.1 epel 185 k 2022-09-29T05:18:44.887 INFO:teuthology.orchestra.run.smithi153.stdout: python3-cffi x86_64 1.11.5-5.el8 CentOS-BaseOS 237 k 2022-09-29T05:18:44.888 INFO:teuthology.orchestra.run.smithi153.stdout: python3-cheroot noarch 8.5.2-1.el8 epel 173 k 2022-09-29T05:18:44.888 INFO:teuthology.orchestra.run.smithi153.stdout: python3-cherrypy noarch 18.4.0-1.el8 epel 384 k 2022-09-29T05:18:44.888 INFO:teuthology.orchestra.run.smithi153.stdout: python3-cryptography x86_64 3.2.1-5.el8 CentOS-BaseOS 559 k 2022-09-29T05:18:44.888 INFO:teuthology.orchestra.run.smithi153.stdout: python3-jaraco noarch 6.2-6.el8 epel 11 k 2022-09-29T05:18:44.889 INFO:teuthology.orchestra.run.smithi153.stdout: python3-jaraco-functools noarch 2.0-4.el8 epel 18 k 2022-09-29T05:18:44.889 INFO:teuthology.orchestra.run.smithi153.stdout: python3-logutils noarch 0.3.5-11.el8 epel 49 k 2022-09-29T05:18:44.889 INFO:teuthology.orchestra.run.smithi153.stdout: python3-mako noarch 1.0.6-13.el8 CentOS-AppStream 157 k 2022-09-29T05:18:44.889 INFO:teuthology.orchestra.run.smithi153.stdout: python3-markupsafe x86_64 0.23-19.el8 CentOS-AppStream 39 k 2022-09-29T05:18:44.889 INFO:teuthology.orchestra.run.smithi153.stdout: python3-more-itertools noarch 7.2.0-3.el8 epel 59 k 2022-09-29T05:18:44.890 INFO:teuthology.orchestra.run.smithi153.stdout: python3-pecan noarch 1.3.2-9.el8 epel 283 k 2022-09-29T05:18:44.890 INFO:teuthology.orchestra.run.smithi153.stdout: python3-portend noarch 2.6-1.el8 epel 16 k 2022-09-29T05:18:44.890 INFO:teuthology.orchestra.run.smithi153.stdout: python3-pyOpenSSL noarch 19.0.0-1.el8 CentOS-AppStream 103 k 2022-09-29T05:18:44.890 INFO:teuthology.orchestra.run.smithi153.stdout: python3-pycparser noarch 2.14-14.el8 CentOS-BaseOS 109 k 2022-09-29T05:18:44.891 INFO:teuthology.orchestra.run.smithi153.stdout: python3-pytz noarch 2017.2-9.el8 CentOS-AppStream 54 k 2022-09-29T05:18:44.891 INFO:teuthology.orchestra.run.smithi153.stdout: python3-simplegeneric noarch 0.8.1-17.el8 epel 19 k 2022-09-29T05:18:44.891 INFO:teuthology.orchestra.run.smithi153.stdout: python3-singledispatch noarch 3.4.0.3-18.el8 epel 24 k 2022-09-29T05:18:44.891 INFO:teuthology.orchestra.run.smithi153.stdout: python3-tempora noarch 1.14.1-5.el8 epel 29 k 2022-09-29T05:18:44.892 INFO:teuthology.orchestra.run.smithi153.stdout: python3-trustme noarch 0.6.0-4.el8 epel 27 k 2022-09-29T05:18:44.892 INFO:teuthology.orchestra.run.smithi153.stdout: python3-waitress noarch 1.2.1-2.el8.1 epel 241 k 2022-09-29T05:18:44.892 INFO:teuthology.orchestra.run.smithi153.stdout: python3-webob noarch 1.8.5-1.el8.1 epel 251 k 2022-09-29T05:18:44.892 INFO:teuthology.orchestra.run.smithi153.stdout: python3-webtest noarch 2.0.33-1.el8 epel 85 k 2022-09-29T05:18:44.892 INFO:teuthology.orchestra.run.smithi153.stdout: python3-werkzeug noarch 0.12.2-4.el8 CentOS-AppStream 457 k 2022-09-29T05:18:44.893 INFO:teuthology.orchestra.run.smithi153.stdout: python3-zc-lockfile noarch 2.0-2.el8 epel 23 k 2022-09-29T05:18:44.893 INFO:teuthology.orchestra.run.smithi153.stdout: 2022-09-29T05:18:44.893 INFO:teuthology.orchestra.run.smithi153.stdout:Transaction Summary 2022-09-29T05:18:44.893 INFO:teuthology.orchestra.run.smithi153.stdout:======================================================================================= 2022-09-29T05:18:44.894 INFO:teuthology.orchestra.run.smithi153.stdout:Install 32 Packages 2022-09-29T05:18:44.894 INFO:teuthology.orchestra.run.smithi153.stdout: 2022-09-29T05:18:44.895 INFO:teuthology.orchestra.run.smithi153.stdout:Total download size: 31 M 2022-09-29T05:18:44.895 INFO:teuthology.orchestra.run.smithi153.stdout:Installed size: 101 M 2022-09-29T05:18:44.895 INFO:teuthology.orchestra.run.smithi153.stdout:Downloading Packages: 2022-09-29T05:18:44.896 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : ceph-test-2:16.2.10-932.g966bd07c.el8.x86_64 6/6 2022-09-29T05:18:45.015 INFO:teuthology.orchestra.run.smithi153.stdout:(1/32): ceph-16.2.10-932.g966bd07c.el8.x86_64.r 51 kB/s | 6.4 kB 00:00 2022-09-29T05:18:45.169 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: ceph-test-2:16.2.10-932.g966bd07c.el8.x86_64 6/6 2022-09-29T05:18:45.170 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : ceph-test-2:16.2.10-932.g966bd07c.el8.x86_64 1/6 2022-09-29T05:18:45.170 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : libcephsqlite-2:16.2.10-932.g966bd07c.el8.x86_64 2/6 2022-09-29T05:18:45.170 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : jq-1.6-3.el8.x86_64 3/6 2022-09-29T05:18:45.171 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : oniguruma-6.8.2-2.el8.x86_64 4/6 2022-09-29T05:18:45.171 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : socat-1.7.4.1-1.el8.x86_64 5/6 2022-09-29T05:18:45.241 INFO:teuthology.orchestra.run.smithi153.stdout:(2/32): ceph-mds-16.2.10-932.g966bd07c.el8.x86_ 6.9 MB/s | 2.4 MB 00:00 2022-09-29T05:18:45.274 INFO:teuthology.orchestra.run.smithi153.stdout:(3/32): ceph-mgr-16.2.10-932.g966bd07c.el8.x86_ 4.2 MB/s | 1.6 MB 00:00 2022-09-29T05:18:45.358 INFO:teuthology.orchestra.run.smithi153.stdout:(4/32): ceph-mgr-modules-core-16.2.10-932.g966b 2.8 MB/s | 241 kB 00:00 2022-09-29T05:18:45.417 INFO:teuthology.orchestra.run.smithi153.stdout:(5/32): python3-mako-1.0.6-13.el8.noarch.rpm 2.6 MB/s | 157 kB 00:00 2022-09-29T05:18:45.442 INFO:teuthology.orchestra.run.smithi153.stdout:(6/32): python3-markupsafe-0.23-19.el8.x86_64.r 1.5 MB/s | 39 kB 00:00 2022-09-29T05:18:45.459 INFO:teuthology.orchestra.run.smithi153.stdout:(7/32): python3-pyOpenSSL-19.0.0-1.el8.noarch.r 6.0 MB/s | 103 kB 00:00 2022-09-29T05:18:45.484 INFO:teuthology.orchestra.run.smithi153.stdout:(8/32): python3-pytz-2017.2-9.el8.noarch.rpm 2.1 MB/s | 54 kB 00:00 2022-09-29T05:18:45.518 INFO:teuthology.orchestra.run.smithi153.stdout:(9/32): python3-werkzeug-0.12.2-4.el8.noarch.rp 13 MB/s | 457 kB 00:00 2022-09-29T05:18:45.543 INFO:teuthology.orchestra.run.smithi153.stdout:(10/32): python3-cffi-1.11.5-5.el8.x86_64.rpm 9.2 MB/s | 237 kB 00:00 2022-09-29T05:18:45.585 INFO:teuthology.orchestra.run.smithi153.stdout:(11/32): python3-cryptography-3.2.1-5.el8.x86_6 13 MB/s | 559 kB 00:00 2022-09-29T05:18:45.611 INFO:teuthology.orchestra.run.smithi153.stdout:(12/32): python3-pycparser-2.14-14.el8.noarch.r 4.2 MB/s | 109 kB 00:00 2022-09-29T05:18:45.691 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : xmlstarlet-1.6.1-20.el8.x86_64 6/6 2022-09-29T05:18:45.692 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:18:45.692 INFO:teuthology.orchestra.run.smithi083.stdout:Installed: 2022-09-29T05:18:45.693 INFO:teuthology.orchestra.run.smithi083.stdout: ceph-test-2:16.2.10-932.g966bd07c.el8.x86_64 2022-09-29T05:18:45.693 INFO:teuthology.orchestra.run.smithi083.stdout: jq-1.6-3.el8.x86_64 2022-09-29T05:18:45.693 INFO:teuthology.orchestra.run.smithi083.stdout: libcephsqlite-2:16.2.10-932.g966bd07c.el8.x86_64 2022-09-29T05:18:45.693 INFO:teuthology.orchestra.run.smithi083.stdout: oniguruma-6.8.2-2.el8.x86_64 2022-09-29T05:18:45.694 INFO:teuthology.orchestra.run.smithi083.stdout: socat-1.7.4.1-1.el8.x86_64 2022-09-29T05:18:45.694 INFO:teuthology.orchestra.run.smithi083.stdout: xmlstarlet-1.6.1-20.el8.x86_64 2022-09-29T05:18:45.694 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:18:45.694 INFO:teuthology.orchestra.run.smithi083.stdout:Complete! 2022-09-29T05:18:45.886 INFO:teuthology.orchestra.run.smithi153.stdout:(13/32): python3-bcrypt-3.1.6-2.el8.1.x86_64.rp 159 kB/s | 44 kB 00:00 2022-09-29T05:18:45.895 DEBUG:teuthology.orchestra.run.smithi083:> sudo yum -y install ceph 2022-09-29T05:18:46.178 INFO:teuthology.orchestra.run.smithi153.stdout:(14/32): ceph-osd-16.2.10-932.g966bd07c.el8.x86 19 MB/s | 18 MB 00:00 2022-09-29T05:18:46.203 INFO:teuthology.orchestra.run.smithi153.stdout:(15/32): python3-beautifulsoup4-4.6.3-2.el8.1.n 584 kB/s | 185 kB 00:00 2022-09-29T05:18:46.282 INFO:teuthology.orchestra.run.smithi153.stdout:(16/32): ceph-mon-16.2.10-932.g966bd07c.el8.x86 4.0 MB/s | 5.0 MB 00:01 2022-09-29T05:18:46.345 INFO:teuthology.orchestra.run.smithi153.stdout:(17/32): python3-cherrypy-18.4.0-1.el8.noarch.r 2.7 MB/s | 384 kB 00:00 2022-09-29T05:18:46.438 INFO:teuthology.orchestra.run.smithi153.stdout:(18/32): python3-jaraco-functools-2.0-4.el8.noa 191 kB/s | 18 kB 00:00 2022-09-29T05:18:46.454 INFO:teuthology.orchestra.run.smithi153.stdout:(19/32): python3-jaraco-6.2-6.el8.noarch.rpm 63 kB/s | 11 kB 00:00 2022-09-29T05:18:46.470 INFO:teuthology.orchestra.run.smithi083.stdout:Last metadata expiration check: 0:00:56 ago on Thu 29 Sep 2022 05:17:50 AM UTC. 2022-09-29T05:18:46.521 INFO:teuthology.orchestra.run.smithi153.stdout:(20/32): python3-logutils-0.3.5-11.el8.noarch.r 584 kB/s | 49 kB 00:00 2022-09-29T05:18:46.546 INFO:teuthology.orchestra.run.smithi153.stdout:(21/32): python3-cheroot-8.5.2-1.el8.noarch.rpm 469 kB/s | 173 kB 00:00 2022-09-29T05:18:46.589 INFO:teuthology.orchestra.run.smithi153.stdout:(22/32): python3-more-itertools-7.2.0-3.el8.noa 438 kB/s | 59 kB 00:00 2022-09-29T05:18:46.664 INFO:teuthology.orchestra.run.smithi153.stdout:(23/32): python3-portend-2.6-1.el8.noarch.rpm 133 kB/s | 16 kB 00:00 2022-09-29T05:18:46.681 INFO:teuthology.orchestra.run.smithi153.stdout:(24/32): python3-simplegeneric-0.8.1-17.el8.noa 204 kB/s | 19 kB 00:00 2022-09-29T05:18:46.714 INFO:teuthology.orchestra.run.smithi153.stdout:(25/32): python3-pecan-1.3.2-9.el8.noarch.rpm 1.4 MB/s | 283 kB 00:00 2022-09-29T05:18:46.773 INFO:teuthology.orchestra.run.smithi153.stdout:(26/32): python3-singledispatch-3.4.0.3-18.el8. 219 kB/s | 24 kB 00:00 2022-09-29T05:18:46.790 INFO:teuthology.orchestra.run.smithi153.stdout:(27/32): python3-tempora-1.14.1-5.el8.noarch.rp 268 kB/s | 29 kB 00:00 2022-09-29T05:18:46.807 INFO:teuthology.orchestra.run.smithi153.stdout:(28/32): python3-trustme-0.6.0-4.el8.noarch.rpm 298 kB/s | 27 kB 00:00 2022-09-29T05:18:46.907 INFO:teuthology.orchestra.run.smithi153.stdout:(29/32): python3-webtest-2.0.33-1.el8.noarch.rp 853 kB/s | 85 kB 00:00 2022-09-29T05:18:46.949 INFO:teuthology.orchestra.run.smithi153.stdout:(30/32): python3-waitress-1.2.1-2.el8.1.noarch. 1.3 MB/s | 241 kB 00:00 2022-09-29T05:18:46.991 INFO:teuthology.orchestra.run.smithi153.stdout:(31/32): python3-zc-lockfile-2.0-2.el8.noarch.r 275 kB/s | 23 kB 00:00 2022-09-29T05:18:47.016 INFO:teuthology.orchestra.run.smithi153.stdout:(32/32): python3-webob-1.8.5-1.el8.1.noarch.rpm 1.1 MB/s | 251 kB 00:00 2022-09-29T05:18:47.019 INFO:teuthology.orchestra.run.smithi153.stdout:-------------------------------------------------------------------------------- 2022-09-29T05:18:47.020 INFO:teuthology.orchestra.run.smithi153.stdout:Total 15 MB/s | 31 MB 00:02 2022-09-29T05:18:47.099 INFO:teuthology.orchestra.run.smithi153.stdout:Running transaction check 2022-09-29T05:18:47.137 INFO:teuthology.orchestra.run.smithi153.stdout:Transaction check succeeded. 2022-09-29T05:18:47.138 INFO:teuthology.orchestra.run.smithi153.stdout:Running transaction test 2022-09-29T05:18:47.444 INFO:teuthology.orchestra.run.smithi153.stdout:Transaction test succeeded. 2022-09-29T05:18:47.447 INFO:teuthology.orchestra.run.smithi153.stdout:Running transaction 2022-09-29T05:18:47.545 INFO:teuthology.orchestra.run.smithi083.stdout:Dependencies resolved. 2022-09-29T05:18:47.548 INFO:teuthology.orchestra.run.smithi083.stdout:======================================================================================= 2022-09-29T05:18:47.548 INFO:teuthology.orchestra.run.smithi083.stdout: Package Arch Version Repository Size 2022-09-29T05:18:47.549 INFO:teuthology.orchestra.run.smithi083.stdout:======================================================================================= 2022-09-29T05:18:47.549 INFO:teuthology.orchestra.run.smithi083.stdout:Installing: 2022-09-29T05:18:47.549 INFO:teuthology.orchestra.run.smithi083.stdout: ceph x86_64 2:16.2.10-932.g966bd07c.el8 ceph 6.4 k 2022-09-29T05:18:47.549 INFO:teuthology.orchestra.run.smithi083.stdout:Installing dependencies: 2022-09-29T05:18:47.550 INFO:teuthology.orchestra.run.smithi083.stdout: ceph-mds x86_64 2:16.2.10-932.g966bd07c.el8 ceph 2.4 M 2022-09-29T05:18:47.550 INFO:teuthology.orchestra.run.smithi083.stdout: ceph-mgr x86_64 2:16.2.10-932.g966bd07c.el8 ceph 1.6 M 2022-09-29T05:18:47.550 INFO:teuthology.orchestra.run.smithi083.stdout: ceph-mgr-modules-core noarch 2:16.2.10-932.g966bd07c.el8 ceph-noarch 241 k 2022-09-29T05:18:47.550 INFO:teuthology.orchestra.run.smithi083.stdout: ceph-mon x86_64 2:16.2.10-932.g966bd07c.el8 ceph 5.0 M 2022-09-29T05:18:47.551 INFO:teuthology.orchestra.run.smithi083.stdout: ceph-osd x86_64 2:16.2.10-932.g966bd07c.el8 ceph 18 M 2022-09-29T05:18:47.551 INFO:teuthology.orchestra.run.smithi083.stdout: python3-bcrypt x86_64 3.1.6-2.el8.1 epel 44 k 2022-09-29T05:18:47.551 INFO:teuthology.orchestra.run.smithi083.stdout: python3-beautifulsoup4 noarch 4.6.3-2.el8.1 epel 185 k 2022-09-29T05:18:47.551 INFO:teuthology.orchestra.run.smithi083.stdout: python3-cffi x86_64 1.11.5-5.el8 CentOS-BaseOS 237 k 2022-09-29T05:18:47.552 INFO:teuthology.orchestra.run.smithi083.stdout: python3-cheroot noarch 8.5.2-1.el8 epel 173 k 2022-09-29T05:18:47.552 INFO:teuthology.orchestra.run.smithi083.stdout: python3-cherrypy noarch 18.4.0-1.el8 epel 384 k 2022-09-29T05:18:47.552 INFO:teuthology.orchestra.run.smithi083.stdout: python3-cryptography x86_64 3.2.1-5.el8 CentOS-BaseOS 559 k 2022-09-29T05:18:47.552 INFO:teuthology.orchestra.run.smithi083.stdout: python3-jaraco noarch 6.2-6.el8 epel 11 k 2022-09-29T05:18:47.553 INFO:teuthology.orchestra.run.smithi083.stdout: python3-jaraco-functools noarch 2.0-4.el8 epel 18 k 2022-09-29T05:18:47.553 INFO:teuthology.orchestra.run.smithi083.stdout: python3-logutils noarch 0.3.5-11.el8 epel 49 k 2022-09-29T05:18:47.553 INFO:teuthology.orchestra.run.smithi083.stdout: python3-mako noarch 1.0.6-13.el8 CentOS-AppStream 157 k 2022-09-29T05:18:47.553 INFO:teuthology.orchestra.run.smithi083.stdout: python3-markupsafe x86_64 0.23-19.el8 CentOS-AppStream 39 k 2022-09-29T05:18:47.553 INFO:teuthology.orchestra.run.smithi083.stdout: python3-more-itertools noarch 7.2.0-3.el8 epel 59 k 2022-09-29T05:18:47.554 INFO:teuthology.orchestra.run.smithi083.stdout: python3-pecan noarch 1.3.2-9.el8 epel 283 k 2022-09-29T05:18:47.554 INFO:teuthology.orchestra.run.smithi083.stdout: python3-portend noarch 2.6-1.el8 epel 16 k 2022-09-29T05:18:47.554 INFO:teuthology.orchestra.run.smithi083.stdout: python3-pyOpenSSL noarch 19.0.0-1.el8 CentOS-AppStream 103 k 2022-09-29T05:18:47.555 INFO:teuthology.orchestra.run.smithi083.stdout: python3-pycparser noarch 2.14-14.el8 CentOS-BaseOS 109 k 2022-09-29T05:18:47.555 INFO:teuthology.orchestra.run.smithi083.stdout: python3-pytz noarch 2017.2-9.el8 CentOS-AppStream 54 k 2022-09-29T05:18:47.555 INFO:teuthology.orchestra.run.smithi083.stdout: python3-simplegeneric noarch 0.8.1-17.el8 epel 19 k 2022-09-29T05:18:47.556 INFO:teuthology.orchestra.run.smithi083.stdout: python3-singledispatch noarch 3.4.0.3-18.el8 epel 24 k 2022-09-29T05:18:47.556 INFO:teuthology.orchestra.run.smithi083.stdout: python3-tempora noarch 1.14.1-5.el8 epel 29 k 2022-09-29T05:18:47.557 INFO:teuthology.orchestra.run.smithi083.stdout: python3-trustme noarch 0.6.0-4.el8 epel 27 k 2022-09-29T05:18:47.557 INFO:teuthology.orchestra.run.smithi083.stdout: python3-waitress noarch 1.2.1-2.el8.1 epel 241 k 2022-09-29T05:18:47.557 INFO:teuthology.orchestra.run.smithi083.stdout: python3-webob noarch 1.8.5-1.el8.1 epel 251 k 2022-09-29T05:18:47.557 INFO:teuthology.orchestra.run.smithi083.stdout: python3-webtest noarch 2.0.33-1.el8 epel 85 k 2022-09-29T05:18:47.558 INFO:teuthology.orchestra.run.smithi083.stdout: python3-werkzeug noarch 0.12.2-4.el8 CentOS-AppStream 457 k 2022-09-29T05:18:47.558 INFO:teuthology.orchestra.run.smithi083.stdout: python3-zc-lockfile noarch 2.0-2.el8 epel 23 k 2022-09-29T05:18:47.558 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:18:47.558 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction Summary 2022-09-29T05:18:47.559 INFO:teuthology.orchestra.run.smithi083.stdout:======================================================================================= 2022-09-29T05:18:47.559 INFO:teuthology.orchestra.run.smithi083.stdout:Install 32 Packages 2022-09-29T05:18:47.559 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:18:47.561 INFO:teuthology.orchestra.run.smithi083.stdout:Total download size: 31 M 2022-09-29T05:18:47.562 INFO:teuthology.orchestra.run.smithi083.stdout:Installed size: 101 M 2022-09-29T05:18:47.562 INFO:teuthology.orchestra.run.smithi083.stdout:Downloading Packages: 2022-09-29T05:18:47.685 INFO:teuthology.orchestra.run.smithi083.stdout:(1/32): ceph-16.2.10-932.g966bd07c.el8.x86_64.r 49 kB/s | 6.4 kB 00:00 2022-09-29T05:18:47.918 INFO:teuthology.orchestra.run.smithi083.stdout:(2/32): ceph-mds-16.2.10-932.g966bd07c.el8.x86_ 6.7 MB/s | 2.4 MB 00:00 2022-09-29T05:18:47.952 INFO:teuthology.orchestra.run.smithi083.stdout:(3/32): ceph-mgr-16.2.10-932.g966bd07c.el8.x86_ 4.0 MB/s | 1.6 MB 00:00 2022-09-29T05:18:48.036 INFO:teuthology.orchestra.run.smithi083.stdout:(4/32): ceph-mgr-modules-core-16.2.10-932.g966b 2.8 MB/s | 241 kB 00:00 2022-09-29T05:18:48.086 INFO:teuthology.orchestra.run.smithi083.stdout:(5/32): python3-mako-1.0.6-13.el8.noarch.rpm 3.1 MB/s | 157 kB 00:00 2022-09-29T05:18:48.112 INFO:teuthology.orchestra.run.smithi083.stdout:(6/32): python3-markupsafe-0.23-19.el8.x86_64.r 1.5 MB/s | 39 kB 00:00 2022-09-29T05:18:48.136 INFO:teuthology.orchestra.run.smithi083.stdout:(7/32): python3-pyOpenSSL-19.0.0-1.el8.noarch.r 4.0 MB/s | 103 kB 00:00 2022-09-29T05:18:48.187 INFO:teuthology.orchestra.run.smithi083.stdout:(8/32): python3-pytz-2017.2-9.el8.noarch.rpm 1.1 MB/s | 54 kB 00:00 2022-09-29T05:18:48.221 INFO:teuthology.orchestra.run.smithi083.stdout:(9/32): python3-werkzeug-0.12.2-4.el8.noarch.rp 13 MB/s | 457 kB 00:00 2022-09-29T05:18:48.246 INFO:teuthology.orchestra.run.smithi083.stdout:(10/32): python3-cffi-1.11.5-5.el8.x86_64.rpm 9.3 MB/s | 237 kB 00:00 2022-09-29T05:18:48.280 INFO:teuthology.orchestra.run.smithi083.stdout:(11/32): python3-cryptography-3.2.1-5.el8.x86_6 16 MB/s | 559 kB 00:00 2022-09-29T05:18:48.306 INFO:teuthology.orchestra.run.smithi083.stdout:(12/32): python3-pycparser-2.14-14.el8.noarch.r 4.3 MB/s | 109 kB 00:00 2022-09-29T05:18:48.465 INFO:teuthology.orchestra.run.smithi083.stdout:(13/32): python3-bcrypt-3.1.6-2.el8.1.x86_64.rp 273 kB/s | 44 kB 00:00 2022-09-29T05:18:48.589 INFO:teuthology.orchestra.run.smithi083.stdout:(14/32): python3-beautifulsoup4-4.6.3-2.el8.1.n 1.5 MB/s | 185 kB 00:00 2022-09-29T05:18:48.632 INFO:teuthology.orchestra.run.smithi153.stdout: Preparing : 1/1 2022-09-29T05:18:48.639 INFO:teuthology.orchestra.run.smithi083.stdout:(15/32): python3-cheroot-8.5.2-1.el8.noarch.rpm 3.4 MB/s | 173 kB 00:00 2022-09-29T05:18:48.864 INFO:teuthology.orchestra.run.smithi083.stdout:(16/32): ceph-osd-16.2.10-932.g966bd07c.el8.x86 19 MB/s | 18 MB 00:00 2022-09-29T05:18:48.871 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : python3-more-itertools-7.2.0-3.el8.noarch 1/32 2022-09-29T05:18:48.890 INFO:teuthology.orchestra.run.smithi083.stdout:(17/32): python3-cherrypy-18.4.0-1.el8.noarch.r 1.5 MB/s | 384 kB 00:00 2022-09-29T05:18:48.981 INFO:teuthology.orchestra.run.smithi083.stdout:(18/32): ceph-mon-16.2.10-932.g966bd07c.el8.x86 3.9 MB/s | 5.0 MB 00:01 2022-09-29T05:18:49.008 INFO:teuthology.orchestra.run.smithi083.stdout:(19/32): python3-jaraco-6.2-6.el8.noarch.rpm 78 kB/s | 11 kB 00:00 2022-09-29T05:18:49.032 INFO:teuthology.orchestra.run.smithi083.stdout:(20/32): python3-jaraco-functools-2.0-4.el8.noa 123 kB/s | 18 kB 00:00 2022-09-29T05:18:49.064 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : python3-webob-1.8.5-1.el8.1.noarch 2/32 2022-09-29T05:18:49.090 INFO:teuthology.orchestra.run.smithi083.stdout:(21/32): python3-more-itertools-7.2.0-3.el8.noa 702 kB/s | 59 kB 00:00 2022-09-29T05:18:49.116 INFO:teuthology.orchestra.run.smithi083.stdout:(22/32): python3-logutils-0.3.5-11.el8.noarch.r 363 kB/s | 49 kB 00:00 2022-09-29T05:18:49.141 INFO:teuthology.orchestra.run.smithi083.stdout:(23/32): python3-portend-2.6-1.el8.noarch.rpm 310 kB/s | 16 kB 00:00 2022-09-29T05:18:49.166 INFO:teuthology.orchestra.run.smithi083.stdout:(24/32): python3-simplegeneric-0.8.1-17.el8.noa 372 kB/s | 19 kB 00:00 2022-09-29T05:18:49.191 INFO:teuthology.orchestra.run.smithi083.stdout:(25/32): python3-singledispatch-3.4.0.3-18.el8. 476 kB/s | 24 kB 00:00 2022-09-29T05:18:49.225 INFO:teuthology.orchestra.run.smithi083.stdout:(26/32): python3-tempora-1.14.1-5.el8.noarch.rp 495 kB/s | 29 kB 00:00 2022-09-29T05:18:49.243 INFO:teuthology.orchestra.run.smithi083.stdout:(27/32): python3-trustme-0.6.0-4.el8.noarch.rpm 543 kB/s | 27 kB 00:00 2022-09-29T05:18:49.276 INFO:teuthology.orchestra.run.smithi083.stdout:(28/32): python3-pecan-1.3.2-9.el8.noarch.rpm 1.1 MB/s | 283 kB 00:00 2022-09-29T05:18:49.322 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : python3-zc-lockfile-2.0-2.el8.noarch 3/32 2022-09-29T05:18:49.326 INFO:teuthology.orchestra.run.smithi083.stdout:(29/32): python3-webob-1.8.5-1.el8.1.noarch.rpm 2.9 MB/s | 251 kB 00:00 2022-09-29T05:18:49.368 INFO:teuthology.orchestra.run.smithi083.stdout:(30/32): python3-webtest-2.0.33-1.el8.noarch.rp 929 kB/s | 85 kB 00:00 2022-09-29T05:18:49.385 INFO:teuthology.orchestra.run.smithi083.stdout:(31/32): python3-zc-lockfile-2.0-2.el8.noarch.r 393 kB/s | 23 kB 00:00 2022-09-29T05:18:49.413 INFO:teuthology.orchestra.run.smithi083.stdout:(32/32): python3-waitress-1.2.1-2.el8.1.noarch. 1.3 MB/s | 241 kB 00:00 2022-09-29T05:18:49.414 INFO:teuthology.orchestra.run.smithi083.stdout:-------------------------------------------------------------------------------- 2022-09-29T05:18:49.414 INFO:teuthology.orchestra.run.smithi083.stdout:Total 17 MB/s | 31 MB 00:01 2022-09-29T05:18:49.490 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction check 2022-09-29T05:18:49.528 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction check succeeded. 2022-09-29T05:18:49.528 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction test 2022-09-29T05:18:49.568 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : python3-waitress-1.2.1-2.el8.1.noarch 4/32 2022-09-29T05:18:49.786 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : python3-singledispatch-3.4.0.3-18.el8.noarch 5/32 2022-09-29T05:18:49.857 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction test succeeded. 2022-09-29T05:18:49.858 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction 2022-09-29T05:18:49.975 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : python3-simplegeneric-0.8.1-17.el8.noarch 6/32 2022-09-29T05:18:50.159 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : python3-logutils-0.3.5-11.el8.noarch 7/32 2022-09-29T05:18:50.329 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : python3-jaraco-6.2-6.el8.noarch 8/32 2022-09-29T05:18:50.564 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : python3-jaraco-functools-2.0-4.el8.noarch 9/32 2022-09-29T05:18:50.825 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 10/32 2022-09-29T05:18:51.045 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : python3-webtest-2.0.33-1.el8.noarch 11/32 2022-09-29T05:18:51.162 INFO:teuthology.orchestra.run.smithi083.stdout: Preparing : 1/1 2022-09-29T05:18:51.269 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : python3-pycparser-2.14-14.el8.noarch 12/32 2022-09-29T05:18:51.385 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-more-itertools-7.2.0-3.el8.noarch 1/32 2022-09-29T05:18:51.604 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-webob-1.8.5-1.el8.1.noarch 2/32 2022-09-29T05:18:51.655 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : python3-cffi-1.11.5-5.el8.x86_64 13/32 2022-09-29T05:18:51.850 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-zc-lockfile-2.0-2.el8.noarch 3/32 2022-09-29T05:18:51.884 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : python3-cryptography-3.2.1-5.el8.x86_64 14/32 2022-09-29T05:18:52.080 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : python3-pyOpenSSL-19.0.0-1.el8.noarch 15/32 2022-09-29T05:18:52.095 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-waitress-1.2.1-2.el8.1.noarch 4/32 2022-09-29T05:18:52.267 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-singledispatch-3.4.0.3-18.el8.noarch 5/32 2022-09-29T05:18:52.297 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : python3-trustme-0.6.0-4.el8.noarch 16/32 2022-09-29T05:18:52.455 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-simplegeneric-0.8.1-17.el8.noarch 6/32 2022-09-29T05:18:52.525 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : python3-cheroot-8.5.2-1.el8.noarch 17/32 2022-09-29T05:18:52.659 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-logutils-0.3.5-11.el8.noarch 7/32 2022-09-29T05:18:52.802 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-jaraco-6.2-6.el8.noarch 8/32 2022-09-29T05:18:52.841 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : python3-bcrypt-3.1.6-2.el8.1.x86_64 18/32 2022-09-29T05:18:53.020 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-jaraco-functools-2.0-4.el8.noarch 9/32 2022-09-29T05:18:53.057 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : python3-werkzeug-0.12.2-4.el8.noarch 19/32 2022-09-29T05:18:53.225 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : python3-pytz-2017.2-9.el8.noarch 20/32 2022-09-29T05:18:53.274 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 10/32 2022-09-29T05:18:53.420 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : python3-tempora-1.14.1-5.el8.noarch 21/32 2022-09-29T05:18:53.520 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-webtest-2.0.33-1.el8.noarch 11/32 2022-09-29T05:18:53.689 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : python3-portend-2.6-1.el8.noarch 22/32 2022-09-29T05:18:53.729 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-pycparser-2.14-14.el8.noarch 12/32 2022-09-29T05:18:54.010 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : python3-cherrypy-18.4.0-1.el8.noarch 23/32 2022-09-29T05:18:54.097 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-cffi-1.11.5-5.el8.x86_64 13/32 2022-09-29T05:18:54.216 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : python3-markupsafe-0.23-19.el8.x86_64 24/32 2022-09-29T05:18:54.324 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-cryptography-3.2.1-5.el8.x86_64 14/32 2022-09-29T05:18:54.537 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-pyOpenSSL-19.0.0-1.el8.noarch 15/32 2022-09-29T05:18:54.586 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : python3-mako-1.0.6-13.el8.noarch 25/32 2022-09-29T05:18:54.779 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-trustme-0.6.0-4.el8.noarch 16/32 2022-09-29T05:18:54.918 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : python3-pecan-1.3.2-9.el8.noarch 26/32 2022-09-29T05:18:55.015 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-cheroot-8.5.2-1.el8.noarch 17/32 2022-09-29T05:18:55.263 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : ceph-mgr-modules-core-2:16.2.10-932.g966bd07c.el8. 27/32 2022-09-29T05:18:55.296 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-bcrypt-3.1.6-2.el8.1.x86_64 18/32 2022-09-29T05:18:55.429 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : ceph-mgr-2:16.2.10-932.g966bd07c.el8.x86_64 28/32 2022-09-29T05:18:55.494 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-werkzeug-0.12.2-4.el8.noarch 19/32 2022-09-29T05:18:55.672 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-pytz-2017.2-9.el8.noarch 20/32 2022-09-29T05:18:55.875 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-tempora-1.14.1-5.el8.noarch 21/32 2022-09-29T05:18:56.094 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-portend-2.6-1.el8.noarch 22/32 2022-09-29T05:18:56.357 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-cherrypy-18.4.0-1.el8.noarch 23/32 2022-09-29T05:18:56.596 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-markupsafe-0.23-19.el8.x86_64 24/32 2022-09-29T05:18:56.926 INFO:teuthology.orchestra.run.smithi153.stdout: Running scriptlet: ceph-mgr-2:16.2.10-932.g966bd07c.el8.x86_64 28/32 2022-09-29T05:18:56.982 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-mako-1.0.6-13.el8.noarch 25/32 2022-09-29T05:18:57.114 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : ceph-osd-2:16.2.10-932.g966bd07c.el8.x86_64 29/32 2022-09-29T05:18:57.322 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-pecan-1.3.2-9.el8.noarch 26/32 2022-09-29T05:18:57.542 INFO:teuthology.orchestra.run.smithi153.stdout: Running scriptlet: ceph-osd-2:16.2.10-932.g966bd07c.el8.x86_64 29/32 2022-09-29T05:18:57.692 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : ceph-mgr-modules-core-2:16.2.10-932.g966bd07c.el8. 27/32 2022-09-29T05:18:57.748 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : ceph-mon-2:16.2.10-932.g966bd07c.el8.x86_64 30/32 2022-09-29T05:18:57.962 INFO:teuthology.orchestra.run.smithi153.stdout: Running scriptlet: ceph-mon-2:16.2.10-932.g966bd07c.el8.x86_64 30/32 2022-09-29T05:18:57.966 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : ceph-mgr-2:16.2.10-932.g966bd07c.el8.x86_64 28/32 2022-09-29T05:18:58.156 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : ceph-mds-2:16.2.10-932.g966bd07c.el8.x86_64 31/32 2022-09-29T05:18:58.174 INFO:teuthology.orchestra.run.smithi153.stdout: Running scriptlet: ceph-mds-2:16.2.10-932.g966bd07c.el8.x86_64 31/32 2022-09-29T05:18:58.288 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : ceph-2:16.2.10-932.g966bd07c.el8.x86_64 32/32 2022-09-29T05:18:58.658 INFO:teuthology.orchestra.run.smithi153.stdout: Running scriptlet: ceph-2:16.2.10-932.g966bd07c.el8.x86_64 32/32 2022-09-29T05:18:58.658 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : ceph-2:16.2.10-932.g966bd07c.el8.x86_64 1/32 2022-09-29T05:18:58.658 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : ceph-mds-2:16.2.10-932.g966bd07c.el8.x86_64 2/32 2022-09-29T05:18:58.659 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : ceph-mgr-2:16.2.10-932.g966bd07c.el8.x86_64 3/32 2022-09-29T05:18:58.659 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : ceph-mon-2:16.2.10-932.g966bd07c.el8.x86_64 4/32 2022-09-29T05:18:58.659 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : ceph-osd-2:16.2.10-932.g966bd07c.el8.x86_64 5/32 2022-09-29T05:18:58.659 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : ceph-mgr-modules-core-2:16.2.10-932.g966bd07c.el8. 6/32 2022-09-29T05:18:58.660 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : python3-mako-1.0.6-13.el8.noarch 7/32 2022-09-29T05:18:58.660 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : python3-markupsafe-0.23-19.el8.x86_64 8/32 2022-09-29T05:18:58.660 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : python3-pyOpenSSL-19.0.0-1.el8.noarch 9/32 2022-09-29T05:18:58.660 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : python3-pytz-2017.2-9.el8.noarch 10/32 2022-09-29T05:18:58.660 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : python3-werkzeug-0.12.2-4.el8.noarch 11/32 2022-09-29T05:18:58.661 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : python3-cffi-1.11.5-5.el8.x86_64 12/32 2022-09-29T05:18:58.661 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : python3-cryptography-3.2.1-5.el8.x86_64 13/32 2022-09-29T05:18:58.661 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : python3-pycparser-2.14-14.el8.noarch 14/32 2022-09-29T05:18:58.661 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : python3-bcrypt-3.1.6-2.el8.1.x86_64 15/32 2022-09-29T05:18:58.662 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 16/32 2022-09-29T05:18:58.662 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : python3-cheroot-8.5.2-1.el8.noarch 17/32 2022-09-29T05:18:58.662 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : python3-cherrypy-18.4.0-1.el8.noarch 18/32 2022-09-29T05:18:58.662 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : python3-jaraco-6.2-6.el8.noarch 19/32 2022-09-29T05:18:58.663 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : python3-jaraco-functools-2.0-4.el8.noarch 20/32 2022-09-29T05:18:58.663 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : python3-logutils-0.3.5-11.el8.noarch 21/32 2022-09-29T05:18:58.663 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : python3-more-itertools-7.2.0-3.el8.noarch 22/32 2022-09-29T05:18:58.663 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : python3-pecan-1.3.2-9.el8.noarch 23/32 2022-09-29T05:18:58.664 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : python3-portend-2.6-1.el8.noarch 24/32 2022-09-29T05:18:58.664 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : python3-simplegeneric-0.8.1-17.el8.noarch 25/32 2022-09-29T05:18:58.664 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : python3-singledispatch-3.4.0.3-18.el8.noarch 26/32 2022-09-29T05:18:58.665 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : python3-tempora-1.14.1-5.el8.noarch 27/32 2022-09-29T05:18:58.666 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : python3-trustme-0.6.0-4.el8.noarch 28/32 2022-09-29T05:18:58.666 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : python3-waitress-1.2.1-2.el8.1.noarch 29/32 2022-09-29T05:18:58.666 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : python3-webob-1.8.5-1.el8.1.noarch 30/32 2022-09-29T05:18:58.666 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : python3-webtest-2.0.33-1.el8.noarch 31/32 2022-09-29T05:18:59.411 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : python3-zc-lockfile-2.0-2.el8.noarch 32/32 2022-09-29T05:18:59.412 INFO:teuthology.orchestra.run.smithi153.stdout: 2022-09-29T05:18:59.412 INFO:teuthology.orchestra.run.smithi153.stdout:Installed: 2022-09-29T05:18:59.412 INFO:teuthology.orchestra.run.smithi153.stdout: ceph-2:16.2.10-932.g966bd07c.el8.x86_64 2022-09-29T05:18:59.413 INFO:teuthology.orchestra.run.smithi153.stdout: ceph-mds-2:16.2.10-932.g966bd07c.el8.x86_64 2022-09-29T05:18:59.413 INFO:teuthology.orchestra.run.smithi153.stdout: ceph-mgr-2:16.2.10-932.g966bd07c.el8.x86_64 2022-09-29T05:18:59.414 INFO:teuthology.orchestra.run.smithi153.stdout: ceph-mgr-modules-core-2:16.2.10-932.g966bd07c.el8.noarch 2022-09-29T05:18:59.414 INFO:teuthology.orchestra.run.smithi153.stdout: ceph-mon-2:16.2.10-932.g966bd07c.el8.x86_64 2022-09-29T05:18:59.414 INFO:teuthology.orchestra.run.smithi153.stdout: ceph-osd-2:16.2.10-932.g966bd07c.el8.x86_64 2022-09-29T05:18:59.415 INFO:teuthology.orchestra.run.smithi153.stdout: python3-bcrypt-3.1.6-2.el8.1.x86_64 2022-09-29T05:18:59.415 INFO:teuthology.orchestra.run.smithi153.stdout: python3-beautifulsoup4-4.6.3-2.el8.1.noarch 2022-09-29T05:18:59.416 INFO:teuthology.orchestra.run.smithi153.stdout: python3-cffi-1.11.5-5.el8.x86_64 2022-09-29T05:18:59.416 INFO:teuthology.orchestra.run.smithi153.stdout: python3-cheroot-8.5.2-1.el8.noarch 2022-09-29T05:18:59.417 INFO:teuthology.orchestra.run.smithi153.stdout: python3-cherrypy-18.4.0-1.el8.noarch 2022-09-29T05:18:59.417 INFO:teuthology.orchestra.run.smithi153.stdout: python3-cryptography-3.2.1-5.el8.x86_64 2022-09-29T05:18:59.418 INFO:teuthology.orchestra.run.smithi153.stdout: python3-jaraco-6.2-6.el8.noarch 2022-09-29T05:18:59.418 INFO:teuthology.orchestra.run.smithi153.stdout: python3-jaraco-functools-2.0-4.el8.noarch 2022-09-29T05:18:59.419 INFO:teuthology.orchestra.run.smithi153.stdout: python3-logutils-0.3.5-11.el8.noarch 2022-09-29T05:18:59.419 INFO:teuthology.orchestra.run.smithi153.stdout: python3-mako-1.0.6-13.el8.noarch 2022-09-29T05:18:59.419 INFO:teuthology.orchestra.run.smithi153.stdout: python3-markupsafe-0.23-19.el8.x86_64 2022-09-29T05:18:59.420 INFO:teuthology.orchestra.run.smithi153.stdout: python3-more-itertools-7.2.0-3.el8.noarch 2022-09-29T05:18:59.420 INFO:teuthology.orchestra.run.smithi153.stdout: python3-pecan-1.3.2-9.el8.noarch 2022-09-29T05:18:59.420 INFO:teuthology.orchestra.run.smithi153.stdout: python3-portend-2.6-1.el8.noarch 2022-09-29T05:18:59.421 INFO:teuthology.orchestra.run.smithi153.stdout: python3-pyOpenSSL-19.0.0-1.el8.noarch 2022-09-29T05:18:59.421 INFO:teuthology.orchestra.run.smithi153.stdout: python3-pycparser-2.14-14.el8.noarch 2022-09-29T05:18:59.422 INFO:teuthology.orchestra.run.smithi153.stdout: python3-pytz-2017.2-9.el8.noarch 2022-09-29T05:18:59.422 INFO:teuthology.orchestra.run.smithi153.stdout: python3-simplegeneric-0.8.1-17.el8.noarch 2022-09-29T05:18:59.423 INFO:teuthology.orchestra.run.smithi153.stdout: python3-singledispatch-3.4.0.3-18.el8.noarch 2022-09-29T05:18:59.423 INFO:teuthology.orchestra.run.smithi153.stdout: python3-tempora-1.14.1-5.el8.noarch 2022-09-29T05:18:59.423 INFO:teuthology.orchestra.run.smithi153.stdout: python3-trustme-0.6.0-4.el8.noarch 2022-09-29T05:18:59.426 INFO:teuthology.orchestra.run.smithi153.stdout: python3-waitress-1.2.1-2.el8.1.noarch 2022-09-29T05:18:59.426 INFO:teuthology.orchestra.run.smithi153.stdout: python3-webob-1.8.5-1.el8.1.noarch 2022-09-29T05:18:59.427 INFO:teuthology.orchestra.run.smithi153.stdout: python3-webtest-2.0.33-1.el8.noarch 2022-09-29T05:18:59.427 INFO:teuthology.orchestra.run.smithi153.stdout: python3-werkzeug-0.12.2-4.el8.noarch 2022-09-29T05:18:59.427 INFO:teuthology.orchestra.run.smithi153.stdout: python3-zc-lockfile-2.0-2.el8.noarch 2022-09-29T05:18:59.427 INFO:teuthology.orchestra.run.smithi153.stdout: 2022-09-29T05:18:59.428 INFO:teuthology.orchestra.run.smithi153.stdout:Complete! 2022-09-29T05:18:59.479 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: ceph-mgr-2:16.2.10-932.g966bd07c.el8.x86_64 28/32 2022-09-29T05:18:59.625 DEBUG:teuthology.orchestra.run.smithi153:> sudo yum -y install ceph-base 2022-09-29T05:18:59.700 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : ceph-osd-2:16.2.10-932.g966bd07c.el8.x86_64 29/32 2022-09-29T05:19:00.122 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: ceph-osd-2:16.2.10-932.g966bd07c.el8.x86_64 29/32 2022-09-29T05:19:00.211 INFO:teuthology.orchestra.run.smithi153.stdout:Last metadata expiration check: 0:01:10 ago on Thu 29 Sep 2022 05:17:50 AM UTC. 2022-09-29T05:19:00.345 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : ceph-mon-2:16.2.10-932.g966bd07c.el8.x86_64 30/32 2022-09-29T05:19:00.561 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: ceph-mon-2:16.2.10-932.g966bd07c.el8.x86_64 30/32 2022-09-29T05:19:00.728 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : ceph-mds-2:16.2.10-932.g966bd07c.el8.x86_64 31/32 2022-09-29T05:19:00.746 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: ceph-mds-2:16.2.10-932.g966bd07c.el8.x86_64 31/32 2022-09-29T05:19:00.885 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : ceph-2:16.2.10-932.g966bd07c.el8.x86_64 32/32 2022-09-29T05:19:01.201 INFO:teuthology.orchestra.run.smithi153.stdout:Package ceph-base-2:16.2.10-932.g966bd07c.el8.x86_64 is already installed. 2022-09-29T05:19:01.261 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: ceph-2:16.2.10-932.g966bd07c.el8.x86_64 32/32 2022-09-29T05:19:01.262 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : ceph-2:16.2.10-932.g966bd07c.el8.x86_64 1/32 2022-09-29T05:19:01.262 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : ceph-mds-2:16.2.10-932.g966bd07c.el8.x86_64 2/32 2022-09-29T05:19:01.263 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : ceph-mgr-2:16.2.10-932.g966bd07c.el8.x86_64 3/32 2022-09-29T05:19:01.263 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : ceph-mon-2:16.2.10-932.g966bd07c.el8.x86_64 4/32 2022-09-29T05:19:01.264 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : ceph-osd-2:16.2.10-932.g966bd07c.el8.x86_64 5/32 2022-09-29T05:19:01.264 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : ceph-mgr-modules-core-2:16.2.10-932.g966bd07c.el8. 6/32 2022-09-29T05:19:01.265 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-mako-1.0.6-13.el8.noarch 7/32 2022-09-29T05:19:01.265 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-markupsafe-0.23-19.el8.x86_64 8/32 2022-09-29T05:19:01.266 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-pyOpenSSL-19.0.0-1.el8.noarch 9/32 2022-09-29T05:19:01.266 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-pytz-2017.2-9.el8.noarch 10/32 2022-09-29T05:19:01.267 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-werkzeug-0.12.2-4.el8.noarch 11/32 2022-09-29T05:19:01.267 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-cffi-1.11.5-5.el8.x86_64 12/32 2022-09-29T05:19:01.268 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-cryptography-3.2.1-5.el8.x86_64 13/32 2022-09-29T05:19:01.268 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-pycparser-2.14-14.el8.noarch 14/32 2022-09-29T05:19:01.269 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-bcrypt-3.1.6-2.el8.1.x86_64 15/32 2022-09-29T05:19:01.269 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 16/32 2022-09-29T05:19:01.270 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-cheroot-8.5.2-1.el8.noarch 17/32 2022-09-29T05:19:01.270 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-cherrypy-18.4.0-1.el8.noarch 18/32 2022-09-29T05:19:01.270 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-jaraco-6.2-6.el8.noarch 19/32 2022-09-29T05:19:01.271 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-jaraco-functools-2.0-4.el8.noarch 20/32 2022-09-29T05:19:01.271 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-logutils-0.3.5-11.el8.noarch 21/32 2022-09-29T05:19:01.272 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-more-itertools-7.2.0-3.el8.noarch 22/32 2022-09-29T05:19:01.272 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-pecan-1.3.2-9.el8.noarch 23/32 2022-09-29T05:19:01.273 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-portend-2.6-1.el8.noarch 24/32 2022-09-29T05:19:01.273 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-simplegeneric-0.8.1-17.el8.noarch 25/32 2022-09-29T05:19:01.274 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-singledispatch-3.4.0.3-18.el8.noarch 26/32 2022-09-29T05:19:01.274 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-tempora-1.14.1-5.el8.noarch 27/32 2022-09-29T05:19:01.275 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-trustme-0.6.0-4.el8.noarch 28/32 2022-09-29T05:19:01.275 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-waitress-1.2.1-2.el8.1.noarch 29/32 2022-09-29T05:19:01.275 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-webob-1.8.5-1.el8.1.noarch 30/32 2022-09-29T05:19:01.276 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-webtest-2.0.33-1.el8.noarch 31/32 2022-09-29T05:19:01.280 INFO:teuthology.orchestra.run.smithi153.stdout:Dependencies resolved. 2022-09-29T05:19:01.281 INFO:teuthology.orchestra.run.smithi153.stdout:Nothing to do. 2022-09-29T05:19:01.281 INFO:teuthology.orchestra.run.smithi153.stdout:Complete! 2022-09-29T05:19:01.351 DEBUG:teuthology.orchestra.run.smithi153:> sudo yum -y install cephadm 2022-09-29T05:19:01.923 INFO:teuthology.orchestra.run.smithi153.stdout:Last metadata expiration check: 0:01:11 ago on Thu 29 Sep 2022 05:17:50 AM UTC. 2022-09-29T05:19:02.064 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-zc-lockfile-2.0-2.el8.noarch 32/32 2022-09-29T05:19:02.064 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:19:02.065 INFO:teuthology.orchestra.run.smithi083.stdout:Installed: 2022-09-29T05:19:02.065 INFO:teuthology.orchestra.run.smithi083.stdout: ceph-2:16.2.10-932.g966bd07c.el8.x86_64 2022-09-29T05:19:02.065 INFO:teuthology.orchestra.run.smithi083.stdout: ceph-mds-2:16.2.10-932.g966bd07c.el8.x86_64 2022-09-29T05:19:02.065 INFO:teuthology.orchestra.run.smithi083.stdout: ceph-mgr-2:16.2.10-932.g966bd07c.el8.x86_64 2022-09-29T05:19:02.066 INFO:teuthology.orchestra.run.smithi083.stdout: ceph-mgr-modules-core-2:16.2.10-932.g966bd07c.el8.noarch 2022-09-29T05:19:02.066 INFO:teuthology.orchestra.run.smithi083.stdout: ceph-mon-2:16.2.10-932.g966bd07c.el8.x86_64 2022-09-29T05:19:02.066 INFO:teuthology.orchestra.run.smithi083.stdout: ceph-osd-2:16.2.10-932.g966bd07c.el8.x86_64 2022-09-29T05:19:02.066 INFO:teuthology.orchestra.run.smithi083.stdout: python3-bcrypt-3.1.6-2.el8.1.x86_64 2022-09-29T05:19:02.067 INFO:teuthology.orchestra.run.smithi083.stdout: python3-beautifulsoup4-4.6.3-2.el8.1.noarch 2022-09-29T05:19:02.067 INFO:teuthology.orchestra.run.smithi083.stdout: python3-cffi-1.11.5-5.el8.x86_64 2022-09-29T05:19:02.067 INFO:teuthology.orchestra.run.smithi083.stdout: python3-cheroot-8.5.2-1.el8.noarch 2022-09-29T05:19:02.067 INFO:teuthology.orchestra.run.smithi083.stdout: python3-cherrypy-18.4.0-1.el8.noarch 2022-09-29T05:19:02.068 INFO:teuthology.orchestra.run.smithi083.stdout: python3-cryptography-3.2.1-5.el8.x86_64 2022-09-29T05:19:02.068 INFO:teuthology.orchestra.run.smithi083.stdout: python3-jaraco-6.2-6.el8.noarch 2022-09-29T05:19:02.068 INFO:teuthology.orchestra.run.smithi083.stdout: python3-jaraco-functools-2.0-4.el8.noarch 2022-09-29T05:19:02.068 INFO:teuthology.orchestra.run.smithi083.stdout: python3-logutils-0.3.5-11.el8.noarch 2022-09-29T05:19:02.069 INFO:teuthology.orchestra.run.smithi083.stdout: python3-mako-1.0.6-13.el8.noarch 2022-09-29T05:19:02.069 INFO:teuthology.orchestra.run.smithi083.stdout: python3-markupsafe-0.23-19.el8.x86_64 2022-09-29T05:19:02.069 INFO:teuthology.orchestra.run.smithi083.stdout: python3-more-itertools-7.2.0-3.el8.noarch 2022-09-29T05:19:02.069 INFO:teuthology.orchestra.run.smithi083.stdout: python3-pecan-1.3.2-9.el8.noarch 2022-09-29T05:19:02.070 INFO:teuthology.orchestra.run.smithi083.stdout: python3-portend-2.6-1.el8.noarch 2022-09-29T05:19:02.070 INFO:teuthology.orchestra.run.smithi083.stdout: python3-pyOpenSSL-19.0.0-1.el8.noarch 2022-09-29T05:19:02.070 INFO:teuthology.orchestra.run.smithi083.stdout: python3-pycparser-2.14-14.el8.noarch 2022-09-29T05:19:02.071 INFO:teuthology.orchestra.run.smithi083.stdout: python3-pytz-2017.2-9.el8.noarch 2022-09-29T05:19:02.071 INFO:teuthology.orchestra.run.smithi083.stdout: python3-simplegeneric-0.8.1-17.el8.noarch 2022-09-29T05:19:02.071 INFO:teuthology.orchestra.run.smithi083.stdout: python3-singledispatch-3.4.0.3-18.el8.noarch 2022-09-29T05:19:02.071 INFO:teuthology.orchestra.run.smithi083.stdout: python3-tempora-1.14.1-5.el8.noarch 2022-09-29T05:19:02.072 INFO:teuthology.orchestra.run.smithi083.stdout: python3-trustme-0.6.0-4.el8.noarch 2022-09-29T05:19:02.072 INFO:teuthology.orchestra.run.smithi083.stdout: python3-waitress-1.2.1-2.el8.1.noarch 2022-09-29T05:19:02.072 INFO:teuthology.orchestra.run.smithi083.stdout: python3-webob-1.8.5-1.el8.1.noarch 2022-09-29T05:19:02.072 INFO:teuthology.orchestra.run.smithi083.stdout: python3-webtest-2.0.33-1.el8.noarch 2022-09-29T05:19:02.073 INFO:teuthology.orchestra.run.smithi083.stdout: python3-werkzeug-0.12.2-4.el8.noarch 2022-09-29T05:19:02.073 INFO:teuthology.orchestra.run.smithi083.stdout: python3-zc-lockfile-2.0-2.el8.noarch 2022-09-29T05:19:02.073 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:19:02.073 INFO:teuthology.orchestra.run.smithi083.stdout:Complete! 2022-09-29T05:19:02.264 DEBUG:teuthology.orchestra.run.smithi083:> sudo yum -y install ceph-base 2022-09-29T05:19:02.848 INFO:teuthology.orchestra.run.smithi083.stdout:Last metadata expiration check: 0:01:12 ago on Thu 29 Sep 2022 05:17:50 AM UTC. 2022-09-29T05:19:03.002 INFO:teuthology.orchestra.run.smithi153.stdout:Dependencies resolved. 2022-09-29T05:19:03.002 INFO:teuthology.orchestra.run.smithi153.stdout:================================================================================ 2022-09-29T05:19:03.003 INFO:teuthology.orchestra.run.smithi153.stdout: Package Arch Version Repository Size 2022-09-29T05:19:03.003 INFO:teuthology.orchestra.run.smithi153.stdout:================================================================================ 2022-09-29T05:19:03.004 INFO:teuthology.orchestra.run.smithi153.stdout:Installing: 2022-09-29T05:19:03.004 INFO:teuthology.orchestra.run.smithi153.stdout: cephadm noarch 2:16.2.10-932.g966bd07c.el8 ceph-noarch 82 k 2022-09-29T05:19:03.004 INFO:teuthology.orchestra.run.smithi153.stdout: 2022-09-29T05:19:03.005 INFO:teuthology.orchestra.run.smithi153.stdout:Transaction Summary 2022-09-29T05:19:03.005 INFO:teuthology.orchestra.run.smithi153.stdout:================================================================================ 2022-09-29T05:19:03.006 INFO:teuthology.orchestra.run.smithi153.stdout:Install 1 Package 2022-09-29T05:19:03.006 INFO:teuthology.orchestra.run.smithi153.stdout: 2022-09-29T05:19:03.006 INFO:teuthology.orchestra.run.smithi153.stdout:Total download size: 82 k 2022-09-29T05:19:03.007 INFO:teuthology.orchestra.run.smithi153.stdout:Installed size: 340 k 2022-09-29T05:19:03.007 INFO:teuthology.orchestra.run.smithi153.stdout:Downloading Packages: 2022-09-29T05:19:03.234 INFO:teuthology.orchestra.run.smithi153.stdout:cephadm-16.2.10-932.g966bd07c.el8.noarch.rpm 357 kB/s | 82 kB 00:00 2022-09-29T05:19:03.235 INFO:teuthology.orchestra.run.smithi153.stdout:-------------------------------------------------------------------------------- 2022-09-29T05:19:03.235 INFO:teuthology.orchestra.run.smithi153.stdout:Total 355 kB/s | 82 kB 00:00 2022-09-29T05:19:03.236 INFO:teuthology.orchestra.run.smithi153.stdout:Running transaction check 2022-09-29T05:19:03.245 INFO:teuthology.orchestra.run.smithi153.stdout:Transaction check succeeded. 2022-09-29T05:19:03.245 INFO:teuthology.orchestra.run.smithi153.stdout:Running transaction test 2022-09-29T05:19:03.251 INFO:teuthology.orchestra.run.smithi153.stdout:Transaction test succeeded. 2022-09-29T05:19:03.252 INFO:teuthology.orchestra.run.smithi153.stdout:Running transaction 2022-09-29T05:19:03.293 INFO:teuthology.orchestra.run.smithi153.stdout: Preparing : 1/1 2022-09-29T05:19:03.838 INFO:teuthology.orchestra.run.smithi083.stdout:Package ceph-base-2:16.2.10-932.g966bd07c.el8.x86_64 is already installed. 2022-09-29T05:19:03.917 INFO:teuthology.orchestra.run.smithi083.stdout:Dependencies resolved. 2022-09-29T05:19:03.918 INFO:teuthology.orchestra.run.smithi083.stdout:Nothing to do. 2022-09-29T05:19:03.919 INFO:teuthology.orchestra.run.smithi083.stdout:Complete! 2022-09-29T05:19:04.008 DEBUG:teuthology.orchestra.run.smithi083:> sudo yum -y install cephadm 2022-09-29T05:19:04.583 INFO:teuthology.orchestra.run.smithi083.stdout:Last metadata expiration check: 0:01:14 ago on Thu 29 Sep 2022 05:17:50 AM UTC. 2022-09-29T05:19:04.843 INFO:teuthology.orchestra.run.smithi153.stdout: Running scriptlet: cephadm-2:16.2.10-932.g966bd07c.el8.noarch 1/1 2022-09-29T05:19:05.023 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : cephadm-2:16.2.10-932.g966bd07c.el8.noarch 1/1 2022-09-29T05:19:05.211 INFO:teuthology.orchestra.run.smithi153.stdout: Running scriptlet: cephadm-2:16.2.10-932.g966bd07c.el8.noarch 1/1 2022-09-29T05:19:05.599 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : cephadm-2:16.2.10-932.g966bd07c.el8.noarch 1/1 2022-09-29T05:19:05.600 INFO:teuthology.orchestra.run.smithi153.stdout: 2022-09-29T05:19:05.600 INFO:teuthology.orchestra.run.smithi153.stdout:Installed: 2022-09-29T05:19:05.600 INFO:teuthology.orchestra.run.smithi153.stdout: cephadm-2:16.2.10-932.g966bd07c.el8.noarch 2022-09-29T05:19:05.601 INFO:teuthology.orchestra.run.smithi153.stdout: 2022-09-29T05:19:05.601 INFO:teuthology.orchestra.run.smithi153.stdout:Complete! 2022-09-29T05:19:05.659 INFO:teuthology.orchestra.run.smithi083.stdout:Dependencies resolved. 2022-09-29T05:19:05.660 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2022-09-29T05:19:05.660 INFO:teuthology.orchestra.run.smithi083.stdout: Package Arch Version Repository Size 2022-09-29T05:19:05.661 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2022-09-29T05:19:05.661 INFO:teuthology.orchestra.run.smithi083.stdout:Installing: 2022-09-29T05:19:05.661 INFO:teuthology.orchestra.run.smithi083.stdout: cephadm noarch 2:16.2.10-932.g966bd07c.el8 ceph-noarch 82 k 2022-09-29T05:19:05.661 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:19:05.662 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction Summary 2022-09-29T05:19:05.662 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2022-09-29T05:19:05.662 INFO:teuthology.orchestra.run.smithi083.stdout:Install 1 Package 2022-09-29T05:19:05.662 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:19:05.663 INFO:teuthology.orchestra.run.smithi083.stdout:Total download size: 82 k 2022-09-29T05:19:05.663 INFO:teuthology.orchestra.run.smithi083.stdout:Installed size: 340 k 2022-09-29T05:19:05.663 INFO:teuthology.orchestra.run.smithi083.stdout:Downloading Packages: 2022-09-29T05:19:05.783 DEBUG:teuthology.orchestra.run.smithi153:> sudo yum -y install ceph-immutable-object-cache 2022-09-29T05:19:05.834 INFO:teuthology.orchestra.run.smithi083.stdout:cephadm-16.2.10-932.g966bd07c.el8.noarch.rpm 484 kB/s | 82 kB 00:00 2022-09-29T05:19:05.835 INFO:teuthology.orchestra.run.smithi083.stdout:-------------------------------------------------------------------------------- 2022-09-29T05:19:05.835 INFO:teuthology.orchestra.run.smithi083.stdout:Total 480 kB/s | 82 kB 00:00 2022-09-29T05:19:05.836 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction check 2022-09-29T05:19:05.839 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction check succeeded. 2022-09-29T05:19:05.840 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction test 2022-09-29T05:19:05.846 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction test succeeded. 2022-09-29T05:19:05.846 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction 2022-09-29T05:19:05.912 INFO:teuthology.orchestra.run.smithi083.stdout: Preparing : 1/1 2022-09-29T05:19:06.372 INFO:teuthology.orchestra.run.smithi153.stdout:Last metadata expiration check: 0:01:16 ago on Thu 29 Sep 2022 05:17:50 AM UTC. 2022-09-29T05:19:07.432 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: cephadm-2:16.2.10-932.g966bd07c.el8.noarch 1/1 2022-09-29T05:19:07.444 INFO:teuthology.orchestra.run.smithi153.stdout:Dependencies resolved. 2022-09-29T05:19:07.445 INFO:teuthology.orchestra.run.smithi153.stdout:================================================================================ 2022-09-29T05:19:07.445 INFO:teuthology.orchestra.run.smithi153.stdout: Package Arch Version Repo Size 2022-09-29T05:19:07.445 INFO:teuthology.orchestra.run.smithi153.stdout:================================================================================ 2022-09-29T05:19:07.446 INFO:teuthology.orchestra.run.smithi153.stdout:Installing: 2022-09-29T05:19:07.446 INFO:teuthology.orchestra.run.smithi153.stdout: ceph-immutable-object-cache x86_64 2:16.2.10-932.g966bd07c.el8 ceph 161 k 2022-09-29T05:19:07.446 INFO:teuthology.orchestra.run.smithi153.stdout: 2022-09-29T05:19:07.447 INFO:teuthology.orchestra.run.smithi153.stdout:Transaction Summary 2022-09-29T05:19:07.447 INFO:teuthology.orchestra.run.smithi153.stdout:================================================================================ 2022-09-29T05:19:07.447 INFO:teuthology.orchestra.run.smithi153.stdout:Install 1 Package 2022-09-29T05:19:07.447 INFO:teuthology.orchestra.run.smithi153.stdout: 2022-09-29T05:19:07.447 INFO:teuthology.orchestra.run.smithi153.stdout:Total download size: 161 k 2022-09-29T05:19:07.448 INFO:teuthology.orchestra.run.smithi153.stdout:Installed size: 470 k 2022-09-29T05:19:07.448 INFO:teuthology.orchestra.run.smithi153.stdout:Downloading Packages: 2022-09-29T05:19:07.586 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : cephadm-2:16.2.10-932.g966bd07c.el8.noarch 1/1 2022-09-29T05:19:07.649 INFO:teuthology.orchestra.run.smithi153.stdout:ceph-immutable-object-cache-16.2.10-932.g966bd0 793 kB/s | 161 kB 00:00 2022-09-29T05:19:07.650 INFO:teuthology.orchestra.run.smithi153.stdout:-------------------------------------------------------------------------------- 2022-09-29T05:19:07.650 INFO:teuthology.orchestra.run.smithi153.stdout:Total 789 kB/s | 161 kB 00:00 2022-09-29T05:19:07.650 INFO:teuthology.orchestra.run.smithi153.stdout:Running transaction check 2022-09-29T05:19:07.661 INFO:teuthology.orchestra.run.smithi153.stdout:Transaction check succeeded. 2022-09-29T05:19:07.662 INFO:teuthology.orchestra.run.smithi153.stdout:Running transaction test 2022-09-29T05:19:07.707 INFO:teuthology.orchestra.run.smithi153.stdout:Transaction test succeeded. 2022-09-29T05:19:07.710 INFO:teuthology.orchestra.run.smithi153.stdout:Running transaction 2022-09-29T05:19:07.780 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: cephadm-2:16.2.10-932.g966bd07c.el8.noarch 1/1 2022-09-29T05:19:07.825 INFO:teuthology.orchestra.run.smithi153.stdout: Preparing : 1/1 2022-09-29T05:19:08.063 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : ceph-immutable-object-cache-2:16.2.10-932.g966bd07c. 1/1 2022-09-29T05:19:08.199 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : cephadm-2:16.2.10-932.g966bd07c.el8.noarch 1/1 2022-09-29T05:19:08.200 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:19:08.200 INFO:teuthology.orchestra.run.smithi083.stdout:Installed: 2022-09-29T05:19:08.200 INFO:teuthology.orchestra.run.smithi083.stdout: cephadm-2:16.2.10-932.g966bd07c.el8.noarch 2022-09-29T05:19:08.201 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:19:08.201 INFO:teuthology.orchestra.run.smithi083.stdout:Complete! 2022-09-29T05:19:08.391 DEBUG:teuthology.orchestra.run.smithi083:> sudo yum -y install ceph-immutable-object-cache 2022-09-29T05:19:08.446 INFO:teuthology.orchestra.run.smithi153.stdout: Running scriptlet: ceph-immutable-object-cache-2:16.2.10-932.g966bd07c. 1/1 2022-09-29T05:19:08.971 INFO:teuthology.orchestra.run.smithi083.stdout:Last metadata expiration check: 0:01:18 ago on Thu 29 Sep 2022 05:17:50 AM UTC. 2022-09-29T05:19:09.589 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : ceph-immutable-object-cache-2:16.2.10-932.g966bd07c. 1/1 2022-09-29T05:19:09.589 INFO:teuthology.orchestra.run.smithi153.stdout: 2022-09-29T05:19:09.590 INFO:teuthology.orchestra.run.smithi153.stdout:Installed: 2022-09-29T05:19:09.590 INFO:teuthology.orchestra.run.smithi153.stdout: ceph-immutable-object-cache-2:16.2.10-932.g966bd07c.el8.x86_64 2022-09-29T05:19:09.591 INFO:teuthology.orchestra.run.smithi153.stdout: 2022-09-29T05:19:09.591 INFO:teuthology.orchestra.run.smithi153.stdout:Complete! 2022-09-29T05:19:09.743 DEBUG:teuthology.orchestra.run.smithi153:> sudo yum -y install ceph-mgr 2022-09-29T05:19:10.045 INFO:teuthology.orchestra.run.smithi083.stdout:Dependencies resolved. 2022-09-29T05:19:10.046 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2022-09-29T05:19:10.046 INFO:teuthology.orchestra.run.smithi083.stdout: Package Arch Version Repo Size 2022-09-29T05:19:10.047 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2022-09-29T05:19:10.047 INFO:teuthology.orchestra.run.smithi083.stdout:Installing: 2022-09-29T05:19:10.048 INFO:teuthology.orchestra.run.smithi083.stdout: ceph-immutable-object-cache x86_64 2:16.2.10-932.g966bd07c.el8 ceph 161 k 2022-09-29T05:19:10.048 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:19:10.049 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction Summary 2022-09-29T05:19:10.049 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2022-09-29T05:19:10.049 INFO:teuthology.orchestra.run.smithi083.stdout:Install 1 Package 2022-09-29T05:19:10.050 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:19:10.050 INFO:teuthology.orchestra.run.smithi083.stdout:Total download size: 161 k 2022-09-29T05:19:10.051 INFO:teuthology.orchestra.run.smithi083.stdout:Installed size: 470 k 2022-09-29T05:19:10.051 INFO:teuthology.orchestra.run.smithi083.stdout:Downloading Packages: 2022-09-29T05:19:10.260 INFO:teuthology.orchestra.run.smithi083.stdout:ceph-immutable-object-cache-16.2.10-932.g966bd0 755 kB/s | 161 kB 00:00 2022-09-29T05:19:10.261 INFO:teuthology.orchestra.run.smithi083.stdout:-------------------------------------------------------------------------------- 2022-09-29T05:19:10.261 INFO:teuthology.orchestra.run.smithi083.stdout:Total 751 kB/s | 161 kB 00:00 2022-09-29T05:19:10.262 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction check 2022-09-29T05:19:10.271 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction check succeeded. 2022-09-29T05:19:10.272 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction test 2022-09-29T05:19:10.317 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction test succeeded. 2022-09-29T05:19:10.320 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction 2022-09-29T05:19:10.335 INFO:teuthology.orchestra.run.smithi153.stdout:Last metadata expiration check: 0:01:20 ago on Thu 29 Sep 2022 05:17:50 AM UTC. 2022-09-29T05:19:10.443 INFO:teuthology.orchestra.run.smithi083.stdout: Preparing : 1/1 2022-09-29T05:19:11.160 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : ceph-immutable-object-cache-2:16.2.10-932.g966bd07c. 1/1 2022-09-29T05:19:11.324 INFO:teuthology.orchestra.run.smithi153.stdout:Package ceph-mgr-2:16.2.10-932.g966bd07c.el8.x86_64 is already installed. 2022-09-29T05:19:11.402 INFO:teuthology.orchestra.run.smithi153.stdout:Dependencies resolved. 2022-09-29T05:19:11.404 INFO:teuthology.orchestra.run.smithi153.stdout:Nothing to do. 2022-09-29T05:19:11.404 INFO:teuthology.orchestra.run.smithi153.stdout:Complete! 2022-09-29T05:19:11.477 DEBUG:teuthology.orchestra.run.smithi153:> sudo yum -y install ceph-mgr-dashboard 2022-09-29T05:19:11.589 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: ceph-immutable-object-cache-2:16.2.10-932.g966bd07c. 1/1 2022-09-29T05:19:12.025 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : ceph-immutable-object-cache-2:16.2.10-932.g966bd07c. 1/1 2022-09-29T05:19:12.026 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:19:12.026 INFO:teuthology.orchestra.run.smithi083.stdout:Installed: 2022-09-29T05:19:12.027 INFO:teuthology.orchestra.run.smithi083.stdout: ceph-immutable-object-cache-2:16.2.10-932.g966bd07c.el8.x86_64 2022-09-29T05:19:12.027 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:19:12.027 INFO:teuthology.orchestra.run.smithi083.stdout:Complete! 2022-09-29T05:19:12.050 INFO:teuthology.orchestra.run.smithi153.stdout:Last metadata expiration check: 0:01:22 ago on Thu 29 Sep 2022 05:17:50 AM UTC. 2022-09-29T05:19:12.213 DEBUG:teuthology.orchestra.run.smithi083:> sudo yum -y install ceph-mgr 2022-09-29T05:19:12.801 INFO:teuthology.orchestra.run.smithi083.stdout:Last metadata expiration check: 0:01:22 ago on Thu 29 Sep 2022 05:17:50 AM UTC. 2022-09-29T05:19:13.115 INFO:teuthology.orchestra.run.smithi153.stdout:Dependencies resolved. 2022-09-29T05:19:13.117 INFO:teuthology.orchestra.run.smithi153.stdout:================================================================================ 2022-09-29T05:19:13.117 INFO:teuthology.orchestra.run.smithi153.stdout: Package Arch Version Repository Size 2022-09-29T05:19:13.118 INFO:teuthology.orchestra.run.smithi153.stdout:================================================================================ 2022-09-29T05:19:13.118 INFO:teuthology.orchestra.run.smithi153.stdout:Installing: 2022-09-29T05:19:13.119 INFO:teuthology.orchestra.run.smithi153.stdout: ceph-mgr-dashboard noarch 2:16.2.10-932.g966bd07c.el8 ceph-noarch 3.1 M 2022-09-29T05:19:13.119 INFO:teuthology.orchestra.run.smithi153.stdout:Installing dependencies: 2022-09-29T05:19:13.120 INFO:teuthology.orchestra.run.smithi153.stdout: ceph-grafana-dashboards noarch 2:16.2.10-932.g966bd07c.el8 ceph-noarch 24 k 2022-09-29T05:19:13.120 INFO:teuthology.orchestra.run.smithi153.stdout: ceph-prometheus-alerts noarch 2:16.2.10-932.g966bd07c.el8 ceph-noarch 15 k 2022-09-29T05:19:13.121 INFO:teuthology.orchestra.run.smithi153.stdout: python3-jwt noarch 1.6.1-2.el8 CentOS-BaseOS 43 k 2022-09-29T05:19:13.121 INFO:teuthology.orchestra.run.smithi153.stdout: python3-repoze-lru noarch 0.7-6.el8 epel 34 k 2022-09-29T05:19:13.122 INFO:teuthology.orchestra.run.smithi153.stdout: python3-routes noarch 2.4.1-12.el8 epel 196 k 2022-09-29T05:19:13.122 INFO:teuthology.orchestra.run.smithi153.stdout: 2022-09-29T05:19:13.122 INFO:teuthology.orchestra.run.smithi153.stdout:Transaction Summary 2022-09-29T05:19:13.123 INFO:teuthology.orchestra.run.smithi153.stdout:================================================================================ 2022-09-29T05:19:13.123 INFO:teuthology.orchestra.run.smithi153.stdout:Install 6 Packages 2022-09-29T05:19:13.124 INFO:teuthology.orchestra.run.smithi153.stdout: 2022-09-29T05:19:13.124 INFO:teuthology.orchestra.run.smithi153.stdout:Total download size: 3.4 M 2022-09-29T05:19:13.125 INFO:teuthology.orchestra.run.smithi153.stdout:Installed size: 21 M 2022-09-29T05:19:13.125 INFO:teuthology.orchestra.run.smithi153.stdout:Downloading Packages: 2022-09-29T05:19:13.261 INFO:teuthology.orchestra.run.smithi153.stdout:(1/6): ceph-grafana-dashboards-16.2.10-932.g966 170 kB/s | 24 kB 00:00 2022-09-29T05:19:13.277 INFO:teuthology.orchestra.run.smithi153.stdout:(2/6): ceph-prometheus-alerts-16.2.10-932.g966b 93 kB/s | 15 kB 00:00 2022-09-29T05:19:13.328 INFO:teuthology.orchestra.run.smithi153.stdout:(3/6): python3-jwt-1.6.1-2.el8.noarch.rpm 639 kB/s | 43 kB 00:00 2022-09-29T05:19:13.387 INFO:teuthology.orchestra.run.smithi153.stdout:(4/6): python3-repoze-lru-0.7-6.el8.noarch.rpm 308 kB/s | 34 kB 00:00 2022-09-29T05:19:13.462 INFO:teuthology.orchestra.run.smithi153.stdout:(5/6): ceph-mgr-dashboard-16.2.10-932.g966bd07c 9.0 MB/s | 3.1 MB 00:00 2022-09-29T05:19:13.512 INFO:teuthology.orchestra.run.smithi153.stdout:(6/6): python3-routes-2.4.1-12.el8.noarch.rpm 1.0 MB/s | 196 kB 00:00 2022-09-29T05:19:13.514 INFO:teuthology.orchestra.run.smithi153.stdout:-------------------------------------------------------------------------------- 2022-09-29T05:19:13.515 INFO:teuthology.orchestra.run.smithi153.stdout:Total 8.5 MB/s | 3.4 MB 00:00 2022-09-29T05:19:13.524 INFO:teuthology.orchestra.run.smithi153.stdout:Running transaction check 2022-09-29T05:19:13.534 INFO:teuthology.orchestra.run.smithi153.stdout:Transaction check succeeded. 2022-09-29T05:19:13.534 INFO:teuthology.orchestra.run.smithi153.stdout:Running transaction test 2022-09-29T05:19:13.699 INFO:teuthology.orchestra.run.smithi153.stdout:Transaction test succeeded. 2022-09-29T05:19:13.702 INFO:teuthology.orchestra.run.smithi153.stdout:Running transaction 2022-09-29T05:19:13.785 INFO:teuthology.orchestra.run.smithi083.stdout:Package ceph-mgr-2:16.2.10-932.g966bd07c.el8.x86_64 is already installed. 2022-09-29T05:19:13.863 INFO:teuthology.orchestra.run.smithi083.stdout:Dependencies resolved. 2022-09-29T05:19:13.864 INFO:teuthology.orchestra.run.smithi083.stdout:Nothing to do. 2022-09-29T05:19:13.865 INFO:teuthology.orchestra.run.smithi083.stdout:Complete! 2022-09-29T05:19:13.960 INFO:teuthology.orchestra.run.smithi153.stdout: Preparing : 1/1 2022-09-29T05:19:13.963 DEBUG:teuthology.orchestra.run.smithi083:> sudo yum -y install ceph-mgr-dashboard 2022-09-29T05:19:14.150 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : python3-repoze-lru-0.7-6.el8.noarch 1/6 2022-09-29T05:19:14.354 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : python3-routes-2.4.1-12.el8.noarch 2/6 2022-09-29T05:19:14.522 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : python3-jwt-1.6.1-2.el8.noarch 3/6 2022-09-29T05:19:14.532 INFO:teuthology.orchestra.run.smithi083.stdout:Last metadata expiration check: 0:01:24 ago on Thu 29 Sep 2022 05:17:50 AM UTC. 2022-09-29T05:19:15.599 INFO:teuthology.orchestra.run.smithi083.stdout:Dependencies resolved. 2022-09-29T05:19:15.600 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2022-09-29T05:19:15.601 INFO:teuthology.orchestra.run.smithi083.stdout: Package Arch Version Repository Size 2022-09-29T05:19:15.601 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2022-09-29T05:19:15.602 INFO:teuthology.orchestra.run.smithi083.stdout:Installing: 2022-09-29T05:19:15.602 INFO:teuthology.orchestra.run.smithi083.stdout: ceph-mgr-dashboard noarch 2:16.2.10-932.g966bd07c.el8 ceph-noarch 3.1 M 2022-09-29T05:19:15.603 INFO:teuthology.orchestra.run.smithi083.stdout:Installing dependencies: 2022-09-29T05:19:15.603 INFO:teuthology.orchestra.run.smithi083.stdout: ceph-grafana-dashboards noarch 2:16.2.10-932.g966bd07c.el8 ceph-noarch 24 k 2022-09-29T05:19:15.604 INFO:teuthology.orchestra.run.smithi083.stdout: ceph-prometheus-alerts noarch 2:16.2.10-932.g966bd07c.el8 ceph-noarch 15 k 2022-09-29T05:19:15.604 INFO:teuthology.orchestra.run.smithi083.stdout: python3-jwt noarch 1.6.1-2.el8 CentOS-BaseOS 43 k 2022-09-29T05:19:15.604 INFO:teuthology.orchestra.run.smithi083.stdout: python3-repoze-lru noarch 0.7-6.el8 epel 34 k 2022-09-29T05:19:15.605 INFO:teuthology.orchestra.run.smithi083.stdout: python3-routes noarch 2.4.1-12.el8 epel 196 k 2022-09-29T05:19:15.605 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:19:15.605 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction Summary 2022-09-29T05:19:15.605 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2022-09-29T05:19:15.605 INFO:teuthology.orchestra.run.smithi083.stdout:Install 6 Packages 2022-09-29T05:19:15.606 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:19:15.607 INFO:teuthology.orchestra.run.smithi083.stdout:Total download size: 3.4 M 2022-09-29T05:19:15.607 INFO:teuthology.orchestra.run.smithi083.stdout:Installed size: 21 M 2022-09-29T05:19:15.607 INFO:teuthology.orchestra.run.smithi083.stdout:Downloading Packages: 2022-09-29T05:19:15.609 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : ceph-prometheus-alerts-2:16.2.10-932.g966bd07c.el8.n 4/6 2022-09-29T05:19:15.758 INFO:teuthology.orchestra.run.smithi083.stdout:(1/6): ceph-grafana-dashboards-16.2.10-932.g966 155 kB/s | 24 kB 00:00 2022-09-29T05:19:15.783 INFO:teuthology.orchestra.run.smithi083.stdout:(2/6): ceph-prometheus-alerts-16.2.10-932.g966b 82 kB/s | 15 kB 00:00 2022-09-29T05:19:15.842 INFO:teuthology.orchestra.run.smithi083.stdout:(3/6): python3-jwt-1.6.1-2.el8.noarch.rpm 510 kB/s | 43 kB 00:00 2022-09-29T05:19:15.917 INFO:teuthology.orchestra.run.smithi083.stdout:(4/6): python3-repoze-lru-0.7-6.el8.noarch.rpm 251 kB/s | 34 kB 00:00 2022-09-29T05:19:15.976 INFO:teuthology.orchestra.run.smithi083.stdout:(5/6): ceph-mgr-dashboard-16.2.10-932.g966bd07c 8.3 MB/s | 3.1 MB 00:00 2022-09-29T05:19:16.051 INFO:teuthology.orchestra.run.smithi083.stdout:(6/6): python3-routes-2.4.1-12.el8.noarch.rpm 941 kB/s | 196 kB 00:00 2022-09-29T05:19:16.054 INFO:teuthology.orchestra.run.smithi083.stdout:-------------------------------------------------------------------------------- 2022-09-29T05:19:16.055 INFO:teuthology.orchestra.run.smithi083.stdout:Total 7.5 MB/s | 3.4 MB 00:00 2022-09-29T05:19:16.064 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction check 2022-09-29T05:19:16.073 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction check succeeded. 2022-09-29T05:19:16.074 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction test 2022-09-29T05:19:16.471 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction test succeeded. 2022-09-29T05:19:16.474 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction 2022-09-29T05:19:16.668 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : ceph-grafana-dashboards-2:16.2.10-932.g966bd07c.el8. 5/6 2022-09-29T05:19:16.900 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : ceph-mgr-dashboard-2:16.2.10-932.g966bd07c.el8.noarc 6/6 2022-09-29T05:19:17.156 INFO:teuthology.orchestra.run.smithi153.stdout: Running scriptlet: ceph-mgr-dashboard-2:16.2.10-932.g966bd07c.el8.noarc 6/6 2022-09-29T05:19:17.157 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : ceph-grafana-dashboards-2:16.2.10-932.g966bd07c.el8. 1/6 2022-09-29T05:19:17.157 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : ceph-mgr-dashboard-2:16.2.10-932.g966bd07c.el8.noarc 2/6 2022-09-29T05:19:17.157 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : ceph-prometheus-alerts-2:16.2.10-932.g966bd07c.el8.n 3/6 2022-09-29T05:19:17.157 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : python3-jwt-1.6.1-2.el8.noarch 4/6 2022-09-29T05:19:17.158 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : python3-repoze-lru-0.7-6.el8.noarch 5/6 2022-09-29T05:19:17.540 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : python3-routes-2.4.1-12.el8.noarch 6/6 2022-09-29T05:19:17.541 INFO:teuthology.orchestra.run.smithi153.stdout: 2022-09-29T05:19:17.541 INFO:teuthology.orchestra.run.smithi153.stdout:Installed: 2022-09-29T05:19:17.541 INFO:teuthology.orchestra.run.smithi153.stdout: ceph-grafana-dashboards-2:16.2.10-932.g966bd07c.el8.noarch 2022-09-29T05:19:17.541 INFO:teuthology.orchestra.run.smithi153.stdout: ceph-mgr-dashboard-2:16.2.10-932.g966bd07c.el8.noarch 2022-09-29T05:19:17.542 INFO:teuthology.orchestra.run.smithi153.stdout: ceph-prometheus-alerts-2:16.2.10-932.g966bd07c.el8.noarch 2022-09-29T05:19:17.542 INFO:teuthology.orchestra.run.smithi153.stdout: python3-jwt-1.6.1-2.el8.noarch 2022-09-29T05:19:17.542 INFO:teuthology.orchestra.run.smithi153.stdout: python3-repoze-lru-0.7-6.el8.noarch 2022-09-29T05:19:17.542 INFO:teuthology.orchestra.run.smithi153.stdout: python3-routes-2.4.1-12.el8.noarch 2022-09-29T05:19:17.542 INFO:teuthology.orchestra.run.smithi153.stdout: 2022-09-29T05:19:17.543 INFO:teuthology.orchestra.run.smithi153.stdout:Complete! 2022-09-29T05:19:17.643 DEBUG:teuthology.orchestra.run.smithi153:> sudo yum -y install ceph-mgr-diskprediction-local 2022-09-29T05:19:18.040 INFO:teuthology.orchestra.run.smithi083.stdout: Preparing : 1/1 2022-09-29T05:19:18.219 INFO:teuthology.orchestra.run.smithi153.stdout:Last metadata expiration check: 0:01:28 ago on Thu 29 Sep 2022 05:17:50 AM UTC. 2022-09-29T05:19:18.281 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-repoze-lru-0.7-6.el8.noarch 1/6 2022-09-29T05:19:18.510 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-routes-2.4.1-12.el8.noarch 2/6 2022-09-29T05:19:18.728 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-jwt-1.6.1-2.el8.noarch 3/6 2022-09-29T05:19:18.840 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : ceph-prometheus-alerts-2:16.2.10-932.g966bd07c.el8.n 4/6 2022-09-29T05:19:19.287 INFO:teuthology.orchestra.run.smithi153.stdout:Dependencies resolved. 2022-09-29T05:19:19.288 INFO:teuthology.orchestra.run.smithi153.stdout:===================================================================================================== 2022-09-29T05:19:19.288 INFO:teuthology.orchestra.run.smithi153.stdout: Package Arch Version Repository Size 2022-09-29T05:19:19.289 INFO:teuthology.orchestra.run.smithi153.stdout:===================================================================================================== 2022-09-29T05:19:19.289 INFO:teuthology.orchestra.run.smithi153.stdout:Installing: 2022-09-29T05:19:19.289 INFO:teuthology.orchestra.run.smithi153.stdout: ceph-mgr-diskprediction-local noarch 2:16.2.10-932.g966bd07c.el8 ceph-noarch 8.6 M 2022-09-29T05:19:19.290 INFO:teuthology.orchestra.run.smithi153.stdout:Installing dependencies: 2022-09-29T05:19:19.290 INFO:teuthology.orchestra.run.smithi153.stdout: libgfortran x86_64 8.5.0-15.el8 CentOS-BaseOS 644 k 2022-09-29T05:19:19.290 INFO:teuthology.orchestra.run.smithi153.stdout: libquadmath x86_64 8.5.0-15.el8 CentOS-BaseOS 172 k 2022-09-29T05:19:19.291 INFO:teuthology.orchestra.run.smithi153.stdout: openblas x86_64 0.3.15-4.el8 CentOS-AppStream 4.7 M 2022-09-29T05:19:19.291 INFO:teuthology.orchestra.run.smithi153.stdout: openblas-threads x86_64 0.3.15-4.el8 CentOS-AppStream 4.9 M 2022-09-29T05:19:19.292 INFO:teuthology.orchestra.run.smithi153.stdout: python3-numpy x86_64 1:1.14.3-10.el8 CentOS-AppStream 3.7 M 2022-09-29T05:19:19.293 INFO:teuthology.orchestra.run.smithi153.stdout: python3-numpy-f2py x86_64 1:1.14.3-10.el8 CentOS-AppStream 225 k 2022-09-29T05:19:19.293 INFO:teuthology.orchestra.run.smithi153.stdout: python3-scipy x86_64 1.0.0-21.module_el8.5.0+771+e5d9a225 CentOS-AppStream 14 M 2022-09-29T05:19:19.293 INFO:teuthology.orchestra.run.smithi153.stdout: 2022-09-29T05:19:19.294 INFO:teuthology.orchestra.run.smithi153.stdout:Transaction Summary 2022-09-29T05:19:19.294 INFO:teuthology.orchestra.run.smithi153.stdout:===================================================================================================== 2022-09-29T05:19:19.294 INFO:teuthology.orchestra.run.smithi153.stdout:Install 8 Packages 2022-09-29T05:19:19.295 INFO:teuthology.orchestra.run.smithi153.stdout: 2022-09-29T05:19:19.296 INFO:teuthology.orchestra.run.smithi153.stdout:Total download size: 37 M 2022-09-29T05:19:19.297 INFO:teuthology.orchestra.run.smithi153.stdout:Installed size: 210 M 2022-09-29T05:19:19.297 INFO:teuthology.orchestra.run.smithi153.stdout:Downloading Packages: 2022-09-29T05:19:19.482 INFO:teuthology.orchestra.run.smithi153.stdout:(1/8): openblas-0.3.15-4.el8.x86_64.rpm 25 MB/s | 4.7 MB 00:00 2022-09-29T05:19:19.608 INFO:teuthology.orchestra.run.smithi153.stdout:(2/8): openblas-threads-0.3.15-4.el8.x86_64.rpm 15 MB/s | 4.9 MB 00:00 2022-09-29T05:19:19.667 INFO:teuthology.orchestra.run.smithi153.stdout:(3/8): python3-numpy-1.14.3-10.el8.x86_64.rpm 20 MB/s | 3.7 MB 00:00 2022-09-29T05:19:19.692 INFO:teuthology.orchestra.run.smithi153.stdout:(4/8): python3-numpy-f2py-1.14.3-10.el8.x86_64. 2.6 MB/s | 225 kB 00:00 2022-09-29T05:19:19.734 INFO:teuthology.orchestra.run.smithi153.stdout:(5/8): libgfortran-8.5.0-15.el8.x86_64.rpm 15 MB/s | 644 kB 00:00 2022-09-29T05:19:19.767 INFO:teuthology.orchestra.run.smithi153.stdout:(6/8): libquadmath-8.5.0-15.el8.x86_64.rpm 5.0 MB/s | 172 kB 00:00 2022-09-29T05:19:19.900 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : ceph-grafana-dashboards-2:16.2.10-932.g966bd07c.el8. 5/6 2022-09-29T05:19:19.909 INFO:teuthology.orchestra.run.smithi153.stdout:(7/8): ceph-mgr-diskprediction-local-16.2.10-93 14 MB/s | 8.6 MB 00:00 2022-09-29T05:19:20.101 INFO:teuthology.orchestra.run.smithi153.stdout:(8/8): python3-scipy-1.0.0-21.module_el8.5.0+77 32 MB/s | 14 MB 00:00 2022-09-29T05:19:20.103 INFO:teuthology.orchestra.run.smithi153.stdout:-------------------------------------------------------------------------------- 2022-09-29T05:19:20.104 INFO:teuthology.orchestra.run.smithi153.stdout:Total 45 MB/s | 37 MB 00:00 2022-09-29T05:19:20.195 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : ceph-mgr-dashboard-2:16.2.10-932.g966bd07c.el8.noarc 6/6 2022-09-29T05:19:20.331 INFO:teuthology.orchestra.run.smithi153.stdout:Running transaction check 2022-09-29T05:19:20.350 INFO:teuthology.orchestra.run.smithi153.stdout:Transaction check succeeded. 2022-09-29T05:19:20.351 INFO:teuthology.orchestra.run.smithi153.stdout:Running transaction test 2022-09-29T05:19:20.429 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: ceph-mgr-dashboard-2:16.2.10-932.g966bd07c.el8.noarc 6/6 2022-09-29T05:19:20.429 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : ceph-grafana-dashboards-2:16.2.10-932.g966bd07c.el8. 1/6 2022-09-29T05:19:20.430 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : ceph-mgr-dashboard-2:16.2.10-932.g966bd07c.el8.noarc 2/6 2022-09-29T05:19:20.431 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : ceph-prometheus-alerts-2:16.2.10-932.g966bd07c.el8.n 3/6 2022-09-29T05:19:20.432 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-jwt-1.6.1-2.el8.noarch 4/6 2022-09-29T05:19:20.433 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-repoze-lru-0.7-6.el8.noarch 5/6 2022-09-29T05:19:20.669 INFO:teuthology.orchestra.run.smithi153.stdout:Transaction test succeeded. 2022-09-29T05:19:20.672 INFO:teuthology.orchestra.run.smithi153.stdout:Running transaction 2022-09-29T05:19:20.843 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-routes-2.4.1-12.el8.noarch 6/6 2022-09-29T05:19:20.843 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:19:20.844 INFO:teuthology.orchestra.run.smithi083.stdout:Installed: 2022-09-29T05:19:20.844 INFO:teuthology.orchestra.run.smithi083.stdout: ceph-grafana-dashboards-2:16.2.10-932.g966bd07c.el8.noarch 2022-09-29T05:19:20.844 INFO:teuthology.orchestra.run.smithi083.stdout: ceph-mgr-dashboard-2:16.2.10-932.g966bd07c.el8.noarch 2022-09-29T05:19:20.844 INFO:teuthology.orchestra.run.smithi083.stdout: ceph-prometheus-alerts-2:16.2.10-932.g966bd07c.el8.noarch 2022-09-29T05:19:20.845 INFO:teuthology.orchestra.run.smithi083.stdout: python3-jwt-1.6.1-2.el8.noarch 2022-09-29T05:19:20.845 INFO:teuthology.orchestra.run.smithi083.stdout: python3-repoze-lru-0.7-6.el8.noarch 2022-09-29T05:19:20.845 INFO:teuthology.orchestra.run.smithi083.stdout: python3-routes-2.4.1-12.el8.noarch 2022-09-29T05:19:20.845 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:19:20.845 INFO:teuthology.orchestra.run.smithi083.stdout:Complete! 2022-09-29T05:19:20.954 DEBUG:teuthology.orchestra.run.smithi083:> sudo yum -y install ceph-mgr-diskprediction-local 2022-09-29T05:19:21.339 INFO:teuthology.orchestra.run.smithi153.stdout: Preparing : 1/1 2022-09-29T05:19:21.485 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : libquadmath-8.5.0-15.el8.x86_64 1/8 2022-09-29T05:19:21.524 INFO:teuthology.orchestra.run.smithi083.stdout:Last metadata expiration check: 0:01:31 ago on Thu 29 Sep 2022 05:17:50 AM UTC. 2022-09-29T05:19:21.602 INFO:teuthology.orchestra.run.smithi153.stdout: Running scriptlet: libquadmath-8.5.0-15.el8.x86_64 1/8 2022-09-29T05:19:21.818 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : libgfortran-8.5.0-15.el8.x86_64 2/8 2022-09-29T05:19:22.416 INFO:teuthology.orchestra.run.smithi153.stdout: Running scriptlet: libgfortran-8.5.0-15.el8.x86_64 2/8 2022-09-29T05:19:22.585 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : openblas-0.3.15-4.el8.x86_64 3/8 2022-09-29T05:19:22.607 INFO:teuthology.orchestra.run.smithi083.stdout:Dependencies resolved. 2022-09-29T05:19:22.608 INFO:teuthology.orchestra.run.smithi083.stdout:===================================================================================================== 2022-09-29T05:19:22.608 INFO:teuthology.orchestra.run.smithi083.stdout: Package Arch Version Repository Size 2022-09-29T05:19:22.608 INFO:teuthology.orchestra.run.smithi083.stdout:===================================================================================================== 2022-09-29T05:19:22.609 INFO:teuthology.orchestra.run.smithi083.stdout:Installing: 2022-09-29T05:19:22.609 INFO:teuthology.orchestra.run.smithi083.stdout: ceph-mgr-diskprediction-local noarch 2:16.2.10-932.g966bd07c.el8 ceph-noarch 8.6 M 2022-09-29T05:19:22.609 INFO:teuthology.orchestra.run.smithi083.stdout:Installing dependencies: 2022-09-29T05:19:22.609 INFO:teuthology.orchestra.run.smithi083.stdout: libgfortran x86_64 8.5.0-15.el8 CentOS-BaseOS 644 k 2022-09-29T05:19:22.610 INFO:teuthology.orchestra.run.smithi083.stdout: libquadmath x86_64 8.5.0-15.el8 CentOS-BaseOS 172 k 2022-09-29T05:19:22.610 INFO:teuthology.orchestra.run.smithi083.stdout: openblas x86_64 0.3.15-4.el8 CentOS-AppStream 4.7 M 2022-09-29T05:19:22.610 INFO:teuthology.orchestra.run.smithi083.stdout: openblas-threads x86_64 0.3.15-4.el8 CentOS-AppStream 4.9 M 2022-09-29T05:19:22.610 INFO:teuthology.orchestra.run.smithi083.stdout: python3-numpy x86_64 1:1.14.3-10.el8 CentOS-AppStream 3.7 M 2022-09-29T05:19:22.611 INFO:teuthology.orchestra.run.smithi083.stdout: python3-numpy-f2py x86_64 1:1.14.3-10.el8 CentOS-AppStream 225 k 2022-09-29T05:19:22.611 INFO:teuthology.orchestra.run.smithi083.stdout: python3-scipy x86_64 1.0.0-21.module_el8.5.0+771+e5d9a225 CentOS-AppStream 14 M 2022-09-29T05:19:22.611 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:19:22.611 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction Summary 2022-09-29T05:19:22.612 INFO:teuthology.orchestra.run.smithi083.stdout:===================================================================================================== 2022-09-29T05:19:22.612 INFO:teuthology.orchestra.run.smithi083.stdout:Install 8 Packages 2022-09-29T05:19:22.612 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:19:22.613 INFO:teuthology.orchestra.run.smithi083.stdout:Total download size: 37 M 2022-09-29T05:19:22.613 INFO:teuthology.orchestra.run.smithi083.stdout:Installed size: 210 M 2022-09-29T05:19:22.613 INFO:teuthology.orchestra.run.smithi083.stdout:Downloading Packages: 2022-09-29T05:19:22.818 INFO:teuthology.orchestra.run.smithi083.stdout:(1/8): openblas-0.3.15-4.el8.x86_64.rpm 23 MB/s | 4.7 MB 00:00 2022-09-29T05:19:23.193 INFO:teuthology.orchestra.run.smithi083.stdout:(2/8): python3-numpy-1.14.3-10.el8.x86_64.rpm 10 MB/s | 3.7 MB 00:00 2022-09-29T05:19:23.212 INFO:teuthology.orchestra.run.smithi153.stdout: Running scriptlet: openblas-0.3.15-4.el8.x86_64 3/8 2022-09-29T05:19:23.300 INFO:teuthology.orchestra.run.smithi083.stdout:(3/8): python3-numpy-f2py-1.14.3-10.el8.x86_64. 2.1 MB/s | 225 kB 00:00 2022-09-29T05:19:23.375 INFO:teuthology.orchestra.run.smithi083.stdout:(4/8): openblas-threads-0.3.15-4.el8.x86_64.rpm 6.4 MB/s | 4.9 MB 00:00 2022-09-29T05:19:23.376 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : openblas-threads-0.3.15-4.el8.x86_64 4/8 2022-09-29T05:19:23.426 INFO:teuthology.orchestra.run.smithi083.stdout:(5/8): libgfortran-8.5.0-15.el8.x86_64.rpm 13 MB/s | 644 kB 00:00 2022-09-29T05:19:23.459 INFO:teuthology.orchestra.run.smithi083.stdout:(6/8): libquadmath-8.5.0-15.el8.x86_64.rpm 5.0 MB/s | 172 kB 00:00 2022-09-29T05:19:23.593 INFO:teuthology.orchestra.run.smithi083.stdout:(7/8): ceph-mgr-diskprediction-local-16.2.10-93 8.7 MB/s | 8.6 MB 00:00 2022-09-29T05:19:23.827 INFO:teuthology.orchestra.run.smithi083.stdout:(8/8): python3-scipy-1.0.0-21.module_el8.5.0+77 26 MB/s | 14 MB 00:00 2022-09-29T05:19:23.829 INFO:teuthology.orchestra.run.smithi083.stdout:-------------------------------------------------------------------------------- 2022-09-29T05:19:23.829 INFO:teuthology.orchestra.run.smithi083.stdout:Total 30 MB/s | 37 MB 00:01 2022-09-29T05:19:24.055 INFO:teuthology.orchestra.run.smithi153.stdout: Running scriptlet: openblas-threads-0.3.15-4.el8.x86_64 4/8 2022-09-29T05:19:24.059 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction check 2022-09-29T05:19:24.078 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction check succeeded. 2022-09-29T05:19:24.078 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction test 2022-09-29T05:19:24.370 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : python3-numpy-1:1.14.3-10.el8.x86_64 5/8 2022-09-29T05:19:24.397 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction test succeeded. 2022-09-29T05:19:24.400 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction 2022-09-29T05:19:24.610 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2022-09-29T05:19:24.950 INFO:teuthology.orchestra.run.smithi083.stdout: Preparing : 1/1 2022-09-29T05:19:25.108 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : libquadmath-8.5.0-15.el8.x86_64 1/8 2022-09-29T05:19:25.245 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: libquadmath-8.5.0-15.el8.x86_64 1/8 2022-09-29T05:19:25.441 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : libgfortran-8.5.0-15.el8.x86_64 2/8 2022-09-29T05:19:26.044 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: libgfortran-8.5.0-15.el8.x86_64 2/8 2022-09-29T05:19:26.249 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : openblas-0.3.15-4.el8.x86_64 3/8 2022-09-29T05:19:26.347 INFO:teuthology.orchestra.run.smithi153.stdout: Running scriptlet: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2022-09-29T05:19:26.865 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: openblas-0.3.15-4.el8.x86_64 3/8 2022-09-29T05:19:27.041 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : openblas-threads-0.3.15-4.el8.x86_64 4/8 2022-09-29T05:19:27.706 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: openblas-threads-0.3.15-4.el8.x86_64 4/8 2022-09-29T05:19:28.030 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : python3-scipy-1.0.0-21.module_el8.5.0+771+e5d9a225.x 7/8 2022-09-29T05:19:28.052 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-numpy-1:1.14.3-10.el8.x86_64 5/8 2022-09-29T05:19:28.200 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : ceph-mgr-diskprediction-local-2:16.2.10-932.g966bd07 8/8 2022-09-29T05:19:28.324 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2022-09-29T05:19:28.480 INFO:teuthology.orchestra.run.smithi153.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:16.2.10-932.g966bd07 8/8 2022-09-29T05:19:28.480 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : ceph-mgr-diskprediction-local-2:16.2.10-932.g966bd07 1/8 2022-09-29T05:19:28.481 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : openblas-0.3.15-4.el8.x86_64 2/8 2022-09-29T05:19:28.481 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : openblas-threads-0.3.15-4.el8.x86_64 3/8 2022-09-29T05:19:28.481 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : python3-numpy-1:1.14.3-10.el8.x86_64 4/8 2022-09-29T05:19:28.482 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 5/8 2022-09-29T05:19:28.483 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : python3-scipy-1.0.0-21.module_el8.5.0+771+e5d9a225.x 6/8 2022-09-29T05:19:28.483 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : libgfortran-8.5.0-15.el8.x86_64 7/8 2022-09-29T05:19:28.908 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : libquadmath-8.5.0-15.el8.x86_64 8/8 2022-09-29T05:19:28.908 INFO:teuthology.orchestra.run.smithi153.stdout: 2022-09-29T05:19:28.909 INFO:teuthology.orchestra.run.smithi153.stdout:Installed: 2022-09-29T05:19:28.909 INFO:teuthology.orchestra.run.smithi153.stdout: ceph-mgr-diskprediction-local-2:16.2.10-932.g966bd07c.el8.noarch 2022-09-29T05:19:28.909 INFO:teuthology.orchestra.run.smithi153.stdout: libgfortran-8.5.0-15.el8.x86_64 2022-09-29T05:19:28.909 INFO:teuthology.orchestra.run.smithi153.stdout: libquadmath-8.5.0-15.el8.x86_64 2022-09-29T05:19:28.909 INFO:teuthology.orchestra.run.smithi153.stdout: openblas-0.3.15-4.el8.x86_64 2022-09-29T05:19:28.910 INFO:teuthology.orchestra.run.smithi153.stdout: openblas-threads-0.3.15-4.el8.x86_64 2022-09-29T05:19:28.910 INFO:teuthology.orchestra.run.smithi153.stdout: python3-numpy-1:1.14.3-10.el8.x86_64 2022-09-29T05:19:28.910 INFO:teuthology.orchestra.run.smithi153.stdout: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 2022-09-29T05:19:28.910 INFO:teuthology.orchestra.run.smithi153.stdout: python3-scipy-1.0.0-21.module_el8.5.0+771+e5d9a225.x86_64 2022-09-29T05:19:28.911 INFO:teuthology.orchestra.run.smithi153.stdout: 2022-09-29T05:19:28.911 INFO:teuthology.orchestra.run.smithi153.stdout:Complete! 2022-09-29T05:19:29.009 DEBUG:teuthology.orchestra.run.smithi153:> sudo yum -y install ceph-mgr-rook 2022-09-29T05:19:29.579 INFO:teuthology.orchestra.run.smithi153.stdout:Last metadata expiration check: 0:01:39 ago on Thu 29 Sep 2022 05:17:50 AM UTC. 2022-09-29T05:19:30.057 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2022-09-29T05:19:30.657 INFO:teuthology.orchestra.run.smithi153.stdout:Dependencies resolved. 2022-09-29T05:19:30.659 INFO:teuthology.orchestra.run.smithi153.stdout:======================================================================================== 2022-09-29T05:19:30.659 INFO:teuthology.orchestra.run.smithi153.stdout: Package Arch Version Repository Size 2022-09-29T05:19:30.659 INFO:teuthology.orchestra.run.smithi153.stdout:======================================================================================== 2022-09-29T05:19:30.659 INFO:teuthology.orchestra.run.smithi153.stdout:Installing: 2022-09-29T05:19:30.660 INFO:teuthology.orchestra.run.smithi153.stdout: ceph-mgr-rook noarch 2:16.2.10-932.g966bd07c.el8 ceph-noarch 27 k 2022-09-29T05:19:30.660 INFO:teuthology.orchestra.run.smithi153.stdout:Installing dependencies: 2022-09-29T05:19:30.660 INFO:teuthology.orchestra.run.smithi153.stdout: python3-cachetools noarch 3.1.1-4.el8 epel 33 k 2022-09-29T05:19:30.660 INFO:teuthology.orchestra.run.smithi153.stdout: python3-certifi noarch 2018.10.15-7.el8 epel 16 k 2022-09-29T05:19:30.661 INFO:teuthology.orchestra.run.smithi153.stdout: python3-google-auth noarch 1:1.1.1-10.el8 epel 81 k 2022-09-29T05:19:30.661 INFO:teuthology.orchestra.run.smithi153.stdout: python3-jsonpatch noarch 1.21-2.el8 CentOS-AppStream 27 k 2022-09-29T05:19:30.661 INFO:teuthology.orchestra.run.smithi153.stdout: python3-jsonpointer noarch 1.10-11.el8 CentOS-AppStream 20 k 2022-09-29T05:19:30.662 INFO:teuthology.orchestra.run.smithi153.stdout: python3-kubernetes noarch 1:11.0.0-6.el8 epel 1.2 M 2022-09-29T05:19:30.662 INFO:teuthology.orchestra.run.smithi153.stdout: python3-oauthlib noarch 2.1.0-1.el8 CentOS-BaseOS 155 k 2022-09-29T05:19:30.662 INFO:teuthology.orchestra.run.smithi153.stdout: python3-pyasn1 noarch 0.3.7-6.el8 CentOS-AppStream 126 k 2022-09-29T05:19:30.663 INFO:teuthology.orchestra.run.smithi153.stdout: python3-pyasn1-modules noarch 0.3.7-6.el8 CentOS-AppStream 110 k 2022-09-29T05:19:30.663 INFO:teuthology.orchestra.run.smithi153.stdout: python3-requests-oauthlib noarch 1.0.0-1.el8 CentOS-BaseOS 43 k 2022-09-29T05:19:30.663 INFO:teuthology.orchestra.run.smithi153.stdout: python3-rsa noarch 4.8-1.el8 epel 61 k 2022-09-29T05:19:30.663 INFO:teuthology.orchestra.run.smithi153.stdout: python3-websocket-client noarch 0.56.0-5.el8 epel 61 k 2022-09-29T05:19:30.664 INFO:teuthology.orchestra.run.smithi153.stdout: 2022-09-29T05:19:30.664 INFO:teuthology.orchestra.run.smithi153.stdout:Transaction Summary 2022-09-29T05:19:30.664 INFO:teuthology.orchestra.run.smithi153.stdout:======================================================================================== 2022-09-29T05:19:30.665 INFO:teuthology.orchestra.run.smithi153.stdout:Install 13 Packages 2022-09-29T05:19:30.665 INFO:teuthology.orchestra.run.smithi153.stdout: 2022-09-29T05:19:30.666 INFO:teuthology.orchestra.run.smithi153.stdout:Total download size: 1.9 M 2022-09-29T05:19:30.666 INFO:teuthology.orchestra.run.smithi153.stdout:Installed size: 24 M 2022-09-29T05:19:30.666 INFO:teuthology.orchestra.run.smithi153.stdout:Downloading Packages: 2022-09-29T05:19:30.733 INFO:teuthology.orchestra.run.smithi153.stdout:(1/13): python3-jsonpatch-1.21-2.el8.noarch.rpm 376 kB/s | 27 kB 00:00 2022-09-29T05:19:30.759 INFO:teuthology.orchestra.run.smithi153.stdout:(2/13): python3-pyasn1-0.3.7-6.el8.noarch.rpm 5.0 MB/s | 126 kB 00:00 2022-09-29T05:19:30.784 INFO:teuthology.orchestra.run.smithi153.stdout:(3/13): python3-jsonpointer-1.10-11.el8.noarch. 164 kB/s | 20 kB 00:00 2022-09-29T05:19:30.809 INFO:teuthology.orchestra.run.smithi153.stdout:(4/13): python3-pyasn1-modules-0.3.7-6.el8.noar 2.1 MB/s | 110 kB 00:00 2022-09-29T05:19:30.835 INFO:teuthology.orchestra.run.smithi153.stdout:(5/13): ceph-mgr-rook-16.2.10-932.g966bd07c.el8 159 kB/s | 27 kB 00:00 2022-09-29T05:19:30.915 INFO:teuthology.orchestra.run.smithi153.stdout:(6/13): python3-requests-oauthlib-1.0.0-1.el8.n 846 kB/s | 43 kB 00:00 2022-09-29T05:19:30.916 INFO:teuthology.orchestra.run.smithi153.stdout:(7/13): python3-oauthlib-2.1.0-1.el8.noarch.rpm 1.5 MB/s | 155 kB 00:00 2022-09-29T05:19:31.035 INFO:teuthology.orchestra.run.smithi153.stdout:(8/13): python3-cachetools-3.1.1-4.el8.noarch.r 164 kB/s | 33 kB 00:00 2022-09-29T05:19:31.061 INFO:teuthology.orchestra.run.smithi153.stdout:(9/13): python3-certifi-2018.10.15-7.el8.noarch 81 kB/s | 16 kB 00:00 2022-09-29T05:19:31.077 INFO:teuthology.orchestra.run.smithi153.stdout:(10/13): python3-google-auth-1.1.1-10.el8.noarc 423 kB/s | 81 kB 00:00 2022-09-29T05:19:31.136 INFO:teuthology.orchestra.run.smithi153.stdout:(11/13): python3-websocket-client-0.56.0-5.el8. 1.0 MB/s | 61 kB 00:00 2022-09-29T05:19:31.153 INFO:teuthology.orchestra.run.smithi153.stdout:(12/13): python3-rsa-4.8-1.el8.noarch.rpm 667 kB/s | 61 kB 00:00 2022-09-29T05:19:31.253 INFO:teuthology.orchestra.run.smithi153.stdout:(13/13): python3-kubernetes-11.0.0-6.el8.noarch 5.4 MB/s | 1.2 MB 00:00 2022-09-29T05:19:31.255 INFO:teuthology.orchestra.run.smithi153.stdout:-------------------------------------------------------------------------------- 2022-09-29T05:19:31.255 INFO:teuthology.orchestra.run.smithi153.stdout:Total 3.2 MB/s | 1.9 MB 00:00 2022-09-29T05:19:31.306 INFO:teuthology.orchestra.run.smithi153.stdout:Running transaction check 2022-09-29T05:19:31.320 INFO:teuthology.orchestra.run.smithi153.stdout:Transaction check succeeded. 2022-09-29T05:19:31.321 INFO:teuthology.orchestra.run.smithi153.stdout:Running transaction test 2022-09-29T05:19:31.373 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-scipy-1.0.0-21.module_el8.5.0+771+e5d9a225.x 7/8 2022-09-29T05:19:31.400 INFO:teuthology.orchestra.run.smithi153.stdout:Transaction test succeeded. 2022-09-29T05:19:31.403 INFO:teuthology.orchestra.run.smithi153.stdout:Running transaction 2022-09-29T05:19:31.548 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : ceph-mgr-diskprediction-local-2:16.2.10-932.g966bd07 8/8 2022-09-29T05:19:31.844 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:16.2.10-932.g966bd07 8/8 2022-09-29T05:19:31.844 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : ceph-mgr-diskprediction-local-2:16.2.10-932.g966bd07 1/8 2022-09-29T05:19:31.845 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : openblas-0.3.15-4.el8.x86_64 2/8 2022-09-29T05:19:31.845 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : openblas-threads-0.3.15-4.el8.x86_64 3/8 2022-09-29T05:19:31.845 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-numpy-1:1.14.3-10.el8.x86_64 4/8 2022-09-29T05:19:31.845 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 5/8 2022-09-29T05:19:31.846 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-scipy-1.0.0-21.module_el8.5.0+771+e5d9a225.x 6/8 2022-09-29T05:19:31.846 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : libgfortran-8.5.0-15.el8.x86_64 7/8 2022-09-29T05:19:32.137 INFO:teuthology.orchestra.run.smithi153.stdout: Preparing : 1/1 2022-09-29T05:19:32.299 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : libquadmath-8.5.0-15.el8.x86_64 8/8 2022-09-29T05:19:32.299 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:19:32.300 INFO:teuthology.orchestra.run.smithi083.stdout:Installed: 2022-09-29T05:19:32.300 INFO:teuthology.orchestra.run.smithi083.stdout: ceph-mgr-diskprediction-local-2:16.2.10-932.g966bd07c.el8.noarch 2022-09-29T05:19:32.300 INFO:teuthology.orchestra.run.smithi083.stdout: libgfortran-8.5.0-15.el8.x86_64 2022-09-29T05:19:32.300 INFO:teuthology.orchestra.run.smithi083.stdout: libquadmath-8.5.0-15.el8.x86_64 2022-09-29T05:19:32.301 INFO:teuthology.orchestra.run.smithi083.stdout: openblas-0.3.15-4.el8.x86_64 2022-09-29T05:19:32.301 INFO:teuthology.orchestra.run.smithi083.stdout: openblas-threads-0.3.15-4.el8.x86_64 2022-09-29T05:19:32.301 INFO:teuthology.orchestra.run.smithi083.stdout: python3-numpy-1:1.14.3-10.el8.x86_64 2022-09-29T05:19:32.301 INFO:teuthology.orchestra.run.smithi083.stdout: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 2022-09-29T05:19:32.301 INFO:teuthology.orchestra.run.smithi083.stdout: python3-scipy-1.0.0-21.module_el8.5.0+771+e5d9a225.x86_64 2022-09-29T05:19:32.302 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:19:32.302 INFO:teuthology.orchestra.run.smithi083.stdout:Complete! 2022-09-29T05:19:32.369 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : python3-pyasn1-0.3.7-6.el8.noarch 1/13 2022-09-29T05:19:32.416 DEBUG:teuthology.orchestra.run.smithi083:> sudo yum -y install ceph-mgr-rook 2022-09-29T05:19:32.559 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : python3-pyasn1-modules-0.3.7-6.el8.noarch 2/13 2022-09-29T05:19:32.794 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : python3-rsa-4.8-1.el8.noarch 3/13 2022-09-29T05:19:32.987 INFO:teuthology.orchestra.run.smithi083.stdout:Last metadata expiration check: 0:01:42 ago on Thu 29 Sep 2022 05:17:50 AM UTC. 2022-09-29T05:19:33.040 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : python3-websocket-client-0.56.0-5.el8.noarch 4/13 2022-09-29T05:19:33.263 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : python3-certifi-2018.10.15-7.el8.noarch 5/13 2022-09-29T05:19:33.486 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : python3-cachetools-3.1.1-4.el8.noarch 6/13 2022-09-29T05:19:33.802 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : python3-google-auth-1:1.1.1-10.el8.noarch 7/13 2022-09-29T05:19:34.012 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : python3-oauthlib-2.1.0-1.el8.noarch 8/13 2022-09-29T05:19:34.051 INFO:teuthology.orchestra.run.smithi083.stdout:Dependencies resolved. 2022-09-29T05:19:34.052 INFO:teuthology.orchestra.run.smithi083.stdout:======================================================================================== 2022-09-29T05:19:34.053 INFO:teuthology.orchestra.run.smithi083.stdout: Package Arch Version Repository Size 2022-09-29T05:19:34.053 INFO:teuthology.orchestra.run.smithi083.stdout:======================================================================================== 2022-09-29T05:19:34.054 INFO:teuthology.orchestra.run.smithi083.stdout:Installing: 2022-09-29T05:19:34.054 INFO:teuthology.orchestra.run.smithi083.stdout: ceph-mgr-rook noarch 2:16.2.10-932.g966bd07c.el8 ceph-noarch 27 k 2022-09-29T05:19:34.055 INFO:teuthology.orchestra.run.smithi083.stdout:Installing dependencies: 2022-09-29T05:19:34.056 INFO:teuthology.orchestra.run.smithi083.stdout: python3-cachetools noarch 3.1.1-4.el8 epel 33 k 2022-09-29T05:19:34.056 INFO:teuthology.orchestra.run.smithi083.stdout: python3-certifi noarch 2018.10.15-7.el8 epel 16 k 2022-09-29T05:19:34.057 INFO:teuthology.orchestra.run.smithi083.stdout: python3-google-auth noarch 1:1.1.1-10.el8 epel 81 k 2022-09-29T05:19:34.057 INFO:teuthology.orchestra.run.smithi083.stdout: python3-jsonpatch noarch 1.21-2.el8 CentOS-AppStream 27 k 2022-09-29T05:19:34.057 INFO:teuthology.orchestra.run.smithi083.stdout: python3-jsonpointer noarch 1.10-11.el8 CentOS-AppStream 20 k 2022-09-29T05:19:34.058 INFO:teuthology.orchestra.run.smithi083.stdout: python3-kubernetes noarch 1:11.0.0-6.el8 epel 1.2 M 2022-09-29T05:19:34.058 INFO:teuthology.orchestra.run.smithi083.stdout: python3-oauthlib noarch 2.1.0-1.el8 CentOS-BaseOS 155 k 2022-09-29T05:19:34.059 INFO:teuthology.orchestra.run.smithi083.stdout: python3-pyasn1 noarch 0.3.7-6.el8 CentOS-AppStream 126 k 2022-09-29T05:19:34.059 INFO:teuthology.orchestra.run.smithi083.stdout: python3-pyasn1-modules noarch 0.3.7-6.el8 CentOS-AppStream 110 k 2022-09-29T05:19:34.060 INFO:teuthology.orchestra.run.smithi083.stdout: python3-requests-oauthlib noarch 1.0.0-1.el8 CentOS-BaseOS 43 k 2022-09-29T05:19:34.060 INFO:teuthology.orchestra.run.smithi083.stdout: python3-rsa noarch 4.8-1.el8 epel 61 k 2022-09-29T05:19:34.060 INFO:teuthology.orchestra.run.smithi083.stdout: python3-websocket-client noarch 0.56.0-5.el8 epel 61 k 2022-09-29T05:19:34.061 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:19:34.061 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction Summary 2022-09-29T05:19:34.062 INFO:teuthology.orchestra.run.smithi083.stdout:======================================================================================== 2022-09-29T05:19:34.062 INFO:teuthology.orchestra.run.smithi083.stdout:Install 13 Packages 2022-09-29T05:19:34.063 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:19:34.064 INFO:teuthology.orchestra.run.smithi083.stdout:Total download size: 1.9 M 2022-09-29T05:19:34.064 INFO:teuthology.orchestra.run.smithi083.stdout:Installed size: 24 M 2022-09-29T05:19:34.064 INFO:teuthology.orchestra.run.smithi083.stdout:Downloading Packages: 2022-09-29T05:19:34.346 INFO:teuthology.orchestra.run.smithi083.stdout:(1/13): python3-jsonpatch-1.21-2.el8.noarch.rpm 91 kB/s | 27 kB 00:00 2022-09-29T05:19:34.570 INFO:teuthology.orchestra.run.smithi083.stdout:(2/13): python3-pyasn1-0.3.7-6.el8.noarch.rpm 567 kB/s | 126 kB 00:00 2022-09-29T05:19:34.587 INFO:teuthology.orchestra.run.smithi083.stdout:(3/13): python3-jsonpointer-1.10-11.el8.noarch. 38 kB/s | 20 kB 00:00 2022-09-29T05:19:34.613 INFO:teuthology.orchestra.run.smithi083.stdout:(4/13): python3-pyasn1-modules-0.3.7-6.el8.noar 2.6 MB/s | 110 kB 00:00 2022-09-29T05:19:34.638 INFO:teuthology.orchestra.run.smithi083.stdout:(5/13): ceph-mgr-rook-16.2.10-932.g966bd07c.el8 47 kB/s | 27 kB 00:00 2022-09-29T05:19:34.695 INFO:teuthology.orchestra.run.smithi083.stdout:(6/13): python3-requests-oauthlib-1.0.0-1.el8.n 521 kB/s | 43 kB 00:00 2022-09-29T05:19:34.722 INFO:teuthology.orchestra.run.smithi083.stdout:(7/13): python3-oauthlib-2.1.0-1.el8.noarch.rpm 1.1 MB/s | 155 kB 00:00 2022-09-29T05:19:34.797 INFO:teuthology.orchestra.run.smithi083.stdout:(8/13): python3-certifi-2018.10.15-7.el8.noarch 160 kB/s | 16 kB 00:00 2022-09-29T05:19:34.822 INFO:teuthology.orchestra.run.smithi083.stdout:(9/13): python3-cachetools-3.1.1-4.el8.noarch.r 179 kB/s | 33 kB 00:00 2022-09-29T05:19:34.856 INFO:teuthology.orchestra.run.smithi083.stdout:(10/13): python3-google-auth-1.1.1-10.el8.noarc 607 kB/s | 81 kB 00:00 2022-09-29T05:19:34.898 INFO:teuthology.orchestra.run.smithi083.stdout:(11/13): python3-rsa-4.8-1.el8.noarch.rpm 817 kB/s | 61 kB 00:00 2022-09-29T05:19:34.923 INFO:teuthology.orchestra.run.smithi083.stdout:(12/13): python3-websocket-client-0.56.0-5.el8. 916 kB/s | 61 kB 00:00 2022-09-29T05:19:35.023 INFO:teuthology.orchestra.run.smithi083.stdout:(13/13): python3-kubernetes-11.0.0-6.el8.noarch 5.2 MB/s | 1.2 MB 00:00 2022-09-29T05:19:35.025 INFO:teuthology.orchestra.run.smithi083.stdout:-------------------------------------------------------------------------------- 2022-09-29T05:19:35.025 INFO:teuthology.orchestra.run.smithi083.stdout:Total 2.0 MB/s | 1.9 MB 00:00 2022-09-29T05:19:35.073 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : python3-requests-oauthlib-1.0.0-1.el8.noarch 9/13 2022-09-29T05:19:35.081 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction check 2022-09-29T05:19:35.098 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction check succeeded. 2022-09-29T05:19:35.098 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction test 2022-09-29T05:19:35.176 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction test succeeded. 2022-09-29T05:19:35.179 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction 2022-09-29T05:19:35.314 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : python3-kubernetes-1:11.0.0-6.el8.noarch 10/13 2022-09-29T05:19:35.531 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : python3-jsonpointer-1.10-11.el8.noarch 11/13 2022-09-29T05:19:35.699 INFO:teuthology.orchestra.run.smithi083.stdout: Preparing : 1/1 2022-09-29T05:19:35.724 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : python3-jsonpatch-1.21-2.el8.noarch 12/13 2022-09-29T05:19:35.893 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : ceph-mgr-rook-2:16.2.10-932.g966bd07c.el8.noarch 13/13 2022-09-29T05:19:35.975 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-pyasn1-0.3.7-6.el8.noarch 1/13 2022-09-29T05:19:36.133 INFO:teuthology.orchestra.run.smithi153.stdout: Running scriptlet: ceph-mgr-rook-2:16.2.10-932.g966bd07c.el8.noarch 13/13 2022-09-29T05:19:36.133 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : ceph-mgr-rook-2:16.2.10-932.g966bd07c.el8.noarch 1/13 2022-09-29T05:19:36.134 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : python3-jsonpatch-1.21-2.el8.noarch 2/13 2022-09-29T05:19:36.134 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : python3-jsonpointer-1.10-11.el8.noarch 3/13 2022-09-29T05:19:36.135 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : python3-pyasn1-0.3.7-6.el8.noarch 4/13 2022-09-29T05:19:36.136 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : python3-pyasn1-modules-0.3.7-6.el8.noarch 5/13 2022-09-29T05:19:36.136 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : python3-oauthlib-2.1.0-1.el8.noarch 6/13 2022-09-29T05:19:36.136 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : python3-requests-oauthlib-1.0.0-1.el8.noarch 7/13 2022-09-29T05:19:36.137 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : python3-cachetools-3.1.1-4.el8.noarch 8/13 2022-09-29T05:19:36.137 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : python3-certifi-2018.10.15-7.el8.noarch 9/13 2022-09-29T05:19:36.138 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : python3-google-auth-1:1.1.1-10.el8.noarch 10/13 2022-09-29T05:19:36.138 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : python3-kubernetes-1:11.0.0-6.el8.noarch 11/13 2022-09-29T05:19:36.139 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : python3-rsa-4.8-1.el8.noarch 12/13 2022-09-29T05:19:36.173 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-pyasn1-modules-0.3.7-6.el8.noarch 2/13 2022-09-29T05:19:36.409 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-rsa-4.8-1.el8.noarch 3/13 2022-09-29T05:19:36.581 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : python3-websocket-client-0.56.0-5.el8.noarch 13/13 2022-09-29T05:19:36.582 INFO:teuthology.orchestra.run.smithi153.stdout: 2022-09-29T05:19:36.582 INFO:teuthology.orchestra.run.smithi153.stdout:Installed: 2022-09-29T05:19:36.583 INFO:teuthology.orchestra.run.smithi153.stdout: ceph-mgr-rook-2:16.2.10-932.g966bd07c.el8.noarch 2022-09-29T05:19:36.583 INFO:teuthology.orchestra.run.smithi153.stdout: python3-cachetools-3.1.1-4.el8.noarch 2022-09-29T05:19:36.583 INFO:teuthology.orchestra.run.smithi153.stdout: python3-certifi-2018.10.15-7.el8.noarch 2022-09-29T05:19:36.584 INFO:teuthology.orchestra.run.smithi153.stdout: python3-google-auth-1:1.1.1-10.el8.noarch 2022-09-29T05:19:36.584 INFO:teuthology.orchestra.run.smithi153.stdout: python3-jsonpatch-1.21-2.el8.noarch 2022-09-29T05:19:36.584 INFO:teuthology.orchestra.run.smithi153.stdout: python3-jsonpointer-1.10-11.el8.noarch 2022-09-29T05:19:36.584 INFO:teuthology.orchestra.run.smithi153.stdout: python3-kubernetes-1:11.0.0-6.el8.noarch 2022-09-29T05:19:36.585 INFO:teuthology.orchestra.run.smithi153.stdout: python3-oauthlib-2.1.0-1.el8.noarch 2022-09-29T05:19:36.585 INFO:teuthology.orchestra.run.smithi153.stdout: python3-pyasn1-0.3.7-6.el8.noarch 2022-09-29T05:19:36.585 INFO:teuthology.orchestra.run.smithi153.stdout: python3-pyasn1-modules-0.3.7-6.el8.noarch 2022-09-29T05:19:36.585 INFO:teuthology.orchestra.run.smithi153.stdout: python3-requests-oauthlib-1.0.0-1.el8.noarch 2022-09-29T05:19:36.585 INFO:teuthology.orchestra.run.smithi153.stdout: python3-rsa-4.8-1.el8.noarch 2022-09-29T05:19:36.586 INFO:teuthology.orchestra.run.smithi153.stdout: python3-websocket-client-0.56.0-5.el8.noarch 2022-09-29T05:19:36.586 INFO:teuthology.orchestra.run.smithi153.stdout: 2022-09-29T05:19:36.586 INFO:teuthology.orchestra.run.smithi153.stdout:Complete! 2022-09-29T05:19:36.668 DEBUG:teuthology.orchestra.run.smithi153:> sudo yum -y install ceph-mgr-cephadm 2022-09-29T05:19:36.696 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-websocket-client-0.56.0-5.el8.noarch 4/13 2022-09-29T05:19:36.878 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-certifi-2018.10.15-7.el8.noarch 5/13 2022-09-29T05:19:37.092 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-cachetools-3.1.1-4.el8.noarch 6/13 2022-09-29T05:19:37.243 INFO:teuthology.orchestra.run.smithi153.stdout:Last metadata expiration check: 0:01:47 ago on Thu 29 Sep 2022 05:17:50 AM UTC. 2022-09-29T05:19:37.325 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-google-auth-1:1.1.1-10.el8.noarch 7/13 2022-09-29T05:19:37.510 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-oauthlib-2.1.0-1.el8.noarch 8/13 2022-09-29T05:19:38.308 INFO:teuthology.orchestra.run.smithi153.stdout:Dependencies resolved. 2022-09-29T05:19:38.309 INFO:teuthology.orchestra.run.smithi153.stdout:================================================================================ 2022-09-29T05:19:38.309 INFO:teuthology.orchestra.run.smithi153.stdout: Package Arch Version Repository Size 2022-09-29T05:19:38.309 INFO:teuthology.orchestra.run.smithi153.stdout:================================================================================ 2022-09-29T05:19:38.310 INFO:teuthology.orchestra.run.smithi153.stdout:Installing: 2022-09-29T05:19:38.310 INFO:teuthology.orchestra.run.smithi153.stdout: ceph-mgr-cephadm noarch 2:16.2.10-932.g966bd07c.el8 ceph-noarch 114 k 2022-09-29T05:19:38.310 INFO:teuthology.orchestra.run.smithi153.stdout:Installing dependencies: 2022-09-29T05:19:38.310 INFO:teuthology.orchestra.run.smithi153.stdout: python3-apipkg noarch 1.5-6.el8 epel 21 k 2022-09-29T05:19:38.310 INFO:teuthology.orchestra.run.smithi153.stdout: python3-babel noarch 2.5.1-7.el8 CentOS-AppStream 4.8 M 2022-09-29T05:19:38.311 INFO:teuthology.orchestra.run.smithi153.stdout: python3-execnet noarch 1.7.1-1.el8 epel 355 k 2022-09-29T05:19:38.311 INFO:teuthology.orchestra.run.smithi153.stdout: python3-jinja2 noarch 2.10.1-3.el8 CentOS-AppStream 538 k 2022-09-29T05:19:38.311 INFO:teuthology.orchestra.run.smithi153.stdout: python3-remoto noarch 1.2.1-1.el8 epel 49 k 2022-09-29T05:19:38.311 INFO:teuthology.orchestra.run.smithi153.stdout: 2022-09-29T05:19:38.312 INFO:teuthology.orchestra.run.smithi153.stdout:Transaction Summary 2022-09-29T05:19:38.312 INFO:teuthology.orchestra.run.smithi153.stdout:================================================================================ 2022-09-29T05:19:38.312 INFO:teuthology.orchestra.run.smithi153.stdout:Install 6 Packages 2022-09-29T05:19:38.312 INFO:teuthology.orchestra.run.smithi153.stdout: 2022-09-29T05:19:38.313 INFO:teuthology.orchestra.run.smithi153.stdout:Total download size: 5.8 M 2022-09-29T05:19:38.313 INFO:teuthology.orchestra.run.smithi153.stdout:Installed size: 24 M 2022-09-29T05:19:38.313 INFO:teuthology.orchestra.run.smithi153.stdout:Downloading Packages: 2022-09-29T05:19:38.427 INFO:teuthology.orchestra.run.smithi153.stdout:(1/6): python3-jinja2-2.10.1-3.el8.noarch.rpm 4.5 MB/s | 538 kB 00:00 2022-09-29T05:19:38.478 INFO:teuthology.orchestra.run.smithi153.stdout:(2/6): ceph-mgr-cephadm-16.2.10-932.g966bd07c.e 684 kB/s | 114 kB 00:00 2022-09-29T05:19:38.562 INFO:teuthology.orchestra.run.smithi153.stdout:(3/6): python3-babel-2.5.1-7.el8.noarch.rpm 19 MB/s | 4.8 MB 00:00 2022-09-29T05:19:38.586 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-requests-oauthlib-1.0.0-1.el8.noarch 9/13 2022-09-29T05:19:38.587 INFO:teuthology.orchestra.run.smithi153.stdout:(4/6): python3-apipkg-1.5-6.el8.noarch.rpm 133 kB/s | 21 kB 00:00 2022-09-29T05:19:38.704 INFO:teuthology.orchestra.run.smithi153.stdout:(5/6): python3-remoto-1.2.1-1.el8.noarch.rpm 344 kB/s | 49 kB 00:00 2022-09-29T05:19:38.787 INFO:teuthology.orchestra.run.smithi153.stdout:(6/6): python3-execnet-1.7.1-1.el8.noarch.rpm 1.1 MB/s | 355 kB 00:00 2022-09-29T05:19:38.789 INFO:teuthology.orchestra.run.smithi153.stdout:-------------------------------------------------------------------------------- 2022-09-29T05:19:38.789 INFO:teuthology.orchestra.run.smithi153.stdout:Total 12 MB/s | 5.8 MB 00:00 2022-09-29T05:19:38.839 INFO:teuthology.orchestra.run.smithi153.stdout:Running transaction check 2022-09-29T05:19:38.849 INFO:teuthology.orchestra.run.smithi153.stdout:Transaction check succeeded. 2022-09-29T05:19:38.850 INFO:teuthology.orchestra.run.smithi153.stdout:Running transaction test 2022-09-29T05:19:38.854 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-kubernetes-1:11.0.0-6.el8.noarch 10/13 2022-09-29T05:19:38.939 INFO:teuthology.orchestra.run.smithi153.stdout:Transaction test succeeded. 2022-09-29T05:19:38.942 INFO:teuthology.orchestra.run.smithi153.stdout:Running transaction 2022-09-29T05:19:39.071 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-jsonpointer-1.10-11.el8.noarch 11/13 2022-09-29T05:19:39.204 INFO:teuthology.orchestra.run.smithi153.stdout: Preparing : 1/1 2022-09-29T05:19:39.230 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-jsonpatch-1.21-2.el8.noarch 12/13 2022-09-29T05:19:39.398 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : ceph-mgr-rook-2:16.2.10-932.g966bd07c.el8.noarch 13/13 2022-09-29T05:19:39.465 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : python3-apipkg-1.5-6.el8.noarch 1/6 2022-09-29T05:19:39.643 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: ceph-mgr-rook-2:16.2.10-932.g966bd07c.el8.noarch 13/13 2022-09-29T05:19:39.643 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : ceph-mgr-rook-2:16.2.10-932.g966bd07c.el8.noarch 1/13 2022-09-29T05:19:39.643 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-jsonpatch-1.21-2.el8.noarch 2/13 2022-09-29T05:19:39.644 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-jsonpointer-1.10-11.el8.noarch 3/13 2022-09-29T05:19:39.644 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-pyasn1-0.3.7-6.el8.noarch 4/13 2022-09-29T05:19:39.644 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-pyasn1-modules-0.3.7-6.el8.noarch 5/13 2022-09-29T05:19:39.644 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-oauthlib-2.1.0-1.el8.noarch 6/13 2022-09-29T05:19:39.645 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-requests-oauthlib-1.0.0-1.el8.noarch 7/13 2022-09-29T05:19:39.645 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-cachetools-3.1.1-4.el8.noarch 8/13 2022-09-29T05:19:39.645 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-certifi-2018.10.15-7.el8.noarch 9/13 2022-09-29T05:19:39.645 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-google-auth-1:1.1.1-10.el8.noarch 10/13 2022-09-29T05:19:39.646 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-kubernetes-1:11.0.0-6.el8.noarch 11/13 2022-09-29T05:19:39.646 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-rsa-4.8-1.el8.noarch 12/13 2022-09-29T05:19:39.693 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : python3-execnet-1.7.1-1.el8.noarch 2/6 2022-09-29T05:19:40.542 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-websocket-client-0.56.0-5.el8.noarch 13/13 2022-09-29T05:19:40.542 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:19:40.542 INFO:teuthology.orchestra.run.smithi083.stdout:Installed: 2022-09-29T05:19:40.543 INFO:teuthology.orchestra.run.smithi083.stdout: ceph-mgr-rook-2:16.2.10-932.g966bd07c.el8.noarch 2022-09-29T05:19:40.543 INFO:teuthology.orchestra.run.smithi083.stdout: python3-cachetools-3.1.1-4.el8.noarch 2022-09-29T05:19:40.543 INFO:teuthology.orchestra.run.smithi083.stdout: python3-certifi-2018.10.15-7.el8.noarch 2022-09-29T05:19:40.543 INFO:teuthology.orchestra.run.smithi083.stdout: python3-google-auth-1:1.1.1-10.el8.noarch 2022-09-29T05:19:40.544 INFO:teuthology.orchestra.run.smithi083.stdout: python3-jsonpatch-1.21-2.el8.noarch 2022-09-29T05:19:40.544 INFO:teuthology.orchestra.run.smithi083.stdout: python3-jsonpointer-1.10-11.el8.noarch 2022-09-29T05:19:40.544 INFO:teuthology.orchestra.run.smithi083.stdout: python3-kubernetes-1:11.0.0-6.el8.noarch 2022-09-29T05:19:40.544 INFO:teuthology.orchestra.run.smithi083.stdout: python3-oauthlib-2.1.0-1.el8.noarch 2022-09-29T05:19:40.544 INFO:teuthology.orchestra.run.smithi083.stdout: python3-pyasn1-0.3.7-6.el8.noarch 2022-09-29T05:19:40.545 INFO:teuthology.orchestra.run.smithi083.stdout: python3-pyasn1-modules-0.3.7-6.el8.noarch 2022-09-29T05:19:40.545 INFO:teuthology.orchestra.run.smithi083.stdout: python3-requests-oauthlib-1.0.0-1.el8.noarch 2022-09-29T05:19:40.545 INFO:teuthology.orchestra.run.smithi083.stdout: python3-rsa-4.8-1.el8.noarch 2022-09-29T05:19:40.545 INFO:teuthology.orchestra.run.smithi083.stdout: python3-websocket-client-0.56.0-5.el8.noarch 2022-09-29T05:19:40.546 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:19:40.546 INFO:teuthology.orchestra.run.smithi083.stdout:Complete! 2022-09-29T05:19:40.607 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : python3-remoto-1.2.1-1.el8.noarch 3/6 2022-09-29T05:19:40.668 DEBUG:teuthology.orchestra.run.smithi083:> sudo yum -y install ceph-mgr-cephadm 2022-09-29T05:19:40.943 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : python3-babel-2.5.1-7.el8.noarch 4/6 2022-09-29T05:19:41.172 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : python3-jinja2-2.10.1-3.el8.noarch 5/6 2022-09-29T05:19:41.239 INFO:teuthology.orchestra.run.smithi083.stdout:Last metadata expiration check: 0:01:51 ago on Thu 29 Sep 2022 05:17:50 AM UTC. 2022-09-29T05:19:41.365 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : ceph-mgr-cephadm-2:16.2.10-932.g966bd07c.el8.noarch 6/6 2022-09-29T05:19:41.601 INFO:teuthology.orchestra.run.smithi153.stdout: Running scriptlet: ceph-mgr-cephadm-2:16.2.10-932.g966bd07c.el8.noarch 6/6 2022-09-29T05:19:41.601 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : ceph-mgr-cephadm-2:16.2.10-932.g966bd07c.el8.noarch 1/6 2022-09-29T05:19:41.602 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : python3-babel-2.5.1-7.el8.noarch 2/6 2022-09-29T05:19:41.602 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : python3-jinja2-2.10.1-3.el8.noarch 3/6 2022-09-29T05:19:41.602 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : python3-apipkg-1.5-6.el8.noarch 4/6 2022-09-29T05:19:41.602 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : python3-execnet-1.7.1-1.el8.noarch 5/6 2022-09-29T05:19:42.007 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : python3-remoto-1.2.1-1.el8.noarch 6/6 2022-09-29T05:19:42.007 INFO:teuthology.orchestra.run.smithi153.stdout: 2022-09-29T05:19:42.008 INFO:teuthology.orchestra.run.smithi153.stdout:Installed: 2022-09-29T05:19:42.008 INFO:teuthology.orchestra.run.smithi153.stdout: ceph-mgr-cephadm-2:16.2.10-932.g966bd07c.el8.noarch 2022-09-29T05:19:42.008 INFO:teuthology.orchestra.run.smithi153.stdout: python3-apipkg-1.5-6.el8.noarch 2022-09-29T05:19:42.008 INFO:teuthology.orchestra.run.smithi153.stdout: python3-babel-2.5.1-7.el8.noarch 2022-09-29T05:19:42.008 INFO:teuthology.orchestra.run.smithi153.stdout: python3-execnet-1.7.1-1.el8.noarch 2022-09-29T05:19:42.009 INFO:teuthology.orchestra.run.smithi153.stdout: python3-jinja2-2.10.1-3.el8.noarch 2022-09-29T05:19:42.009 INFO:teuthology.orchestra.run.smithi153.stdout: python3-remoto-1.2.1-1.el8.noarch 2022-09-29T05:19:42.009 INFO:teuthology.orchestra.run.smithi153.stdout: 2022-09-29T05:19:42.009 INFO:teuthology.orchestra.run.smithi153.stdout:Complete! 2022-09-29T05:19:42.098 DEBUG:teuthology.orchestra.run.smithi153:> sudo yum -y install ceph-fuse 2022-09-29T05:19:42.310 INFO:teuthology.orchestra.run.smithi083.stdout:Dependencies resolved. 2022-09-29T05:19:42.311 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2022-09-29T05:19:42.311 INFO:teuthology.orchestra.run.smithi083.stdout: Package Arch Version Repository Size 2022-09-29T05:19:42.311 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2022-09-29T05:19:42.312 INFO:teuthology.orchestra.run.smithi083.stdout:Installing: 2022-09-29T05:19:42.312 INFO:teuthology.orchestra.run.smithi083.stdout: ceph-mgr-cephadm noarch 2:16.2.10-932.g966bd07c.el8 ceph-noarch 114 k 2022-09-29T05:19:42.312 INFO:teuthology.orchestra.run.smithi083.stdout:Installing dependencies: 2022-09-29T05:19:42.312 INFO:teuthology.orchestra.run.smithi083.stdout: python3-apipkg noarch 1.5-6.el8 epel 21 k 2022-09-29T05:19:42.313 INFO:teuthology.orchestra.run.smithi083.stdout: python3-babel noarch 2.5.1-7.el8 CentOS-AppStream 4.8 M 2022-09-29T05:19:42.313 INFO:teuthology.orchestra.run.smithi083.stdout: python3-execnet noarch 1.7.1-1.el8 epel 355 k 2022-09-29T05:19:42.313 INFO:teuthology.orchestra.run.smithi083.stdout: python3-jinja2 noarch 2.10.1-3.el8 CentOS-AppStream 538 k 2022-09-29T05:19:42.313 INFO:teuthology.orchestra.run.smithi083.stdout: python3-remoto noarch 1.2.1-1.el8 epel 49 k 2022-09-29T05:19:42.313 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:19:42.314 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction Summary 2022-09-29T05:19:42.314 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2022-09-29T05:19:42.314 INFO:teuthology.orchestra.run.smithi083.stdout:Install 6 Packages 2022-09-29T05:19:42.314 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:19:42.315 INFO:teuthology.orchestra.run.smithi083.stdout:Total download size: 5.8 M 2022-09-29T05:19:42.315 INFO:teuthology.orchestra.run.smithi083.stdout:Installed size: 24 M 2022-09-29T05:19:42.316 INFO:teuthology.orchestra.run.smithi083.stdout:Downloading Packages: 2022-09-29T05:19:42.499 INFO:teuthology.orchestra.run.smithi083.stdout:(1/6): ceph-mgr-cephadm-16.2.10-932.g966bd07c.e 614 kB/s | 114 kB 00:00 2022-09-29T05:19:42.624 INFO:teuthology.orchestra.run.smithi083.stdout:(2/6): python3-apipkg-1.5-6.el8.noarch.rpm 169 kB/s | 21 kB 00:00 2022-09-29T05:19:42.678 INFO:teuthology.orchestra.run.smithi153.stdout:Last metadata expiration check: 0:01:52 ago on Thu 29 Sep 2022 05:17:50 AM UTC. 2022-09-29T05:19:42.758 INFO:teuthology.orchestra.run.smithi083.stdout:(3/6): python3-execnet-1.7.1-1.el8.noarch.rpm 2.6 MB/s | 355 kB 00:00 2022-09-29T05:19:42.808 INFO:teuthology.orchestra.run.smithi083.stdout:(4/6): python3-remoto-1.2.1-1.el8.noarch.rpm 976 kB/s | 49 kB 00:00 2022-09-29T05:19:43.133 INFO:teuthology.orchestra.run.smithi083.stdout:(5/6): python3-babel-2.5.1-7.el8.noarch.rpm 5.8 MB/s | 4.8 MB 00:00 2022-09-29T05:19:43.200 INFO:teuthology.orchestra.run.smithi083.stdout:(6/6): python3-jinja2-2.10.1-3.el8.noarch.rpm 606 kB/s | 538 kB 00:00 2022-09-29T05:19:43.203 INFO:teuthology.orchestra.run.smithi083.stdout:-------------------------------------------------------------------------------- 2022-09-29T05:19:43.203 INFO:teuthology.orchestra.run.smithi083.stdout:Total 6.6 MB/s | 5.8 MB 00:00 2022-09-29T05:19:43.254 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction check 2022-09-29T05:19:43.263 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction check succeeded. 2022-09-29T05:19:43.264 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction test 2022-09-29T05:19:43.361 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction test succeeded. 2022-09-29T05:19:43.364 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction 2022-09-29T05:19:43.660 INFO:teuthology.orchestra.run.smithi083.stdout: Preparing : 1/1 2022-09-29T05:19:43.755 INFO:teuthology.orchestra.run.smithi153.stdout:Dependencies resolved. 2022-09-29T05:19:43.755 INFO:teuthology.orchestra.run.smithi153.stdout:================================================================================ 2022-09-29T05:19:43.756 INFO:teuthology.orchestra.run.smithi153.stdout: Package Arch Version Repository Size 2022-09-29T05:19:43.756 INFO:teuthology.orchestra.run.smithi153.stdout:================================================================================ 2022-09-29T05:19:43.756 INFO:teuthology.orchestra.run.smithi153.stdout:Installing: 2022-09-29T05:19:43.756 INFO:teuthology.orchestra.run.smithi153.stdout: ceph-fuse x86_64 2:16.2.10-932.g966bd07c.el8 ceph 906 k 2022-09-29T05:19:43.757 INFO:teuthology.orchestra.run.smithi153.stdout:Installing dependencies: 2022-09-29T05:19:43.757 INFO:teuthology.orchestra.run.smithi153.stdout: fuse x86_64 2.9.7-16.el8 CentOS-BaseOS 83 k 2022-09-29T05:19:43.757 INFO:teuthology.orchestra.run.smithi153.stdout: 2022-09-29T05:19:43.757 INFO:teuthology.orchestra.run.smithi153.stdout:Transaction Summary 2022-09-29T05:19:43.757 INFO:teuthology.orchestra.run.smithi153.stdout:================================================================================ 2022-09-29T05:19:43.758 INFO:teuthology.orchestra.run.smithi153.stdout:Install 2 Packages 2022-09-29T05:19:43.758 INFO:teuthology.orchestra.run.smithi153.stdout: 2022-09-29T05:19:43.759 INFO:teuthology.orchestra.run.smithi153.stdout:Total download size: 989 k 2022-09-29T05:19:43.759 INFO:teuthology.orchestra.run.smithi153.stdout:Installed size: 2.8 M 2022-09-29T05:19:43.759 INFO:teuthology.orchestra.run.smithi153.stdout:Downloading Packages: 2022-09-29T05:19:43.829 INFO:teuthology.orchestra.run.smithi153.stdout:(1/2): fuse-2.9.7-16.el8.x86_64.rpm 1.1 MB/s | 83 kB 00:00 2022-09-29T05:19:43.921 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-apipkg-1.5-6.el8.noarch 1/6 2022-09-29T05:19:44.029 INFO:teuthology.orchestra.run.smithi153.stdout:(2/2): ceph-fuse-16.2.10-932.g966bd07c.el8.x86_ 3.3 MB/s | 906 kB 00:00 2022-09-29T05:19:44.029 INFO:teuthology.orchestra.run.smithi153.stdout:-------------------------------------------------------------------------------- 2022-09-29T05:19:44.030 INFO:teuthology.orchestra.run.smithi153.stdout:Total 3.5 MB/s | 989 kB 00:00 2022-09-29T05:19:44.039 INFO:teuthology.orchestra.run.smithi153.stdout:Running transaction check 2022-09-29T05:19:44.060 INFO:teuthology.orchestra.run.smithi153.stdout:Transaction check succeeded. 2022-09-29T05:19:44.060 INFO:teuthology.orchestra.run.smithi153.stdout:Running transaction test 2022-09-29T05:19:44.135 INFO:teuthology.orchestra.run.smithi153.stdout:Transaction test succeeded. 2022-09-29T05:19:44.138 INFO:teuthology.orchestra.run.smithi153.stdout:Running transaction 2022-09-29T05:19:44.175 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-execnet-1.7.1-1.el8.noarch 2/6 2022-09-29T05:19:44.307 INFO:teuthology.orchestra.run.smithi153.stdout: Preparing : 1/1 2022-09-29T05:19:44.562 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : fuse-2.9.7-16.el8.x86_64 1/2 2022-09-29T05:19:44.730 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : ceph-fuse-2:16.2.10-932.g966bd07c.el8.x86_64 2/2 2022-09-29T05:19:45.085 INFO:teuthology.orchestra.run.smithi153.stdout: Running scriptlet: ceph-fuse-2:16.2.10-932.g966bd07c.el8.x86_64 2/2 2022-09-29T05:19:45.086 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : ceph-fuse-2:16.2.10-932.g966bd07c.el8.x86_64 1/2 2022-09-29T05:19:45.123 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-remoto-1.2.1-1.el8.noarch 3/6 2022-09-29T05:19:45.491 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : fuse-2.9.7-16.el8.x86_64 2/2 2022-09-29T05:19:45.491 INFO:teuthology.orchestra.run.smithi153.stdout: 2022-09-29T05:19:45.492 INFO:teuthology.orchestra.run.smithi153.stdout:Installed: 2022-09-29T05:19:45.492 INFO:teuthology.orchestra.run.smithi153.stdout: ceph-fuse-2:16.2.10-932.g966bd07c.el8.x86_64 fuse-2.9.7-16.el8.x86_64 2022-09-29T05:19:45.492 INFO:teuthology.orchestra.run.smithi153.stdout: 2022-09-29T05:19:45.492 INFO:teuthology.orchestra.run.smithi153.stdout:Complete! 2022-09-29T05:19:45.508 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-babel-2.5.1-7.el8.noarch 4/6 2022-09-29T05:19:45.673 DEBUG:teuthology.orchestra.run.smithi153:> sudo yum -y install librados-devel 2022-09-29T05:19:45.771 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-jinja2-2.10.1-3.el8.noarch 5/6 2022-09-29T05:19:45.997 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : ceph-mgr-cephadm-2:16.2.10-932.g966bd07c.el8.noarch 6/6 2022-09-29T05:19:46.242 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: ceph-mgr-cephadm-2:16.2.10-932.g966bd07c.el8.noarch 6/6 2022-09-29T05:19:46.243 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : ceph-mgr-cephadm-2:16.2.10-932.g966bd07c.el8.noarch 1/6 2022-09-29T05:19:46.243 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-babel-2.5.1-7.el8.noarch 2/6 2022-09-29T05:19:46.243 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-jinja2-2.10.1-3.el8.noarch 3/6 2022-09-29T05:19:46.243 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-apipkg-1.5-6.el8.noarch 4/6 2022-09-29T05:19:46.244 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-execnet-1.7.1-1.el8.noarch 5/6 2022-09-29T05:19:46.260 INFO:teuthology.orchestra.run.smithi153.stdout:Last metadata expiration check: 0:01:56 ago on Thu 29 Sep 2022 05:17:50 AM UTC. 2022-09-29T05:19:46.647 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-remoto-1.2.1-1.el8.noarch 6/6 2022-09-29T05:19:46.648 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:19:46.648 INFO:teuthology.orchestra.run.smithi083.stdout:Installed: 2022-09-29T05:19:46.648 INFO:teuthology.orchestra.run.smithi083.stdout: ceph-mgr-cephadm-2:16.2.10-932.g966bd07c.el8.noarch 2022-09-29T05:19:46.648 INFO:teuthology.orchestra.run.smithi083.stdout: python3-apipkg-1.5-6.el8.noarch 2022-09-29T05:19:46.649 INFO:teuthology.orchestra.run.smithi083.stdout: python3-babel-2.5.1-7.el8.noarch 2022-09-29T05:19:46.649 INFO:teuthology.orchestra.run.smithi083.stdout: python3-execnet-1.7.1-1.el8.noarch 2022-09-29T05:19:46.649 INFO:teuthology.orchestra.run.smithi083.stdout: python3-jinja2-2.10.1-3.el8.noarch 2022-09-29T05:19:46.649 INFO:teuthology.orchestra.run.smithi083.stdout: python3-remoto-1.2.1-1.el8.noarch 2022-09-29T05:19:46.650 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:19:46.650 INFO:teuthology.orchestra.run.smithi083.stdout:Complete! 2022-09-29T05:19:46.775 DEBUG:teuthology.orchestra.run.smithi083:> sudo yum -y install ceph-fuse 2022-09-29T05:19:47.340 INFO:teuthology.orchestra.run.smithi153.stdout:Dependencies resolved. 2022-09-29T05:19:47.341 INFO:teuthology.orchestra.run.smithi153.stdout:================================================================================ 2022-09-29T05:19:47.342 INFO:teuthology.orchestra.run.smithi153.stdout: Package Arch Version Repo Size 2022-09-29T05:19:47.342 INFO:teuthology.orchestra.run.smithi153.stdout:================================================================================ 2022-09-29T05:19:47.342 INFO:teuthology.orchestra.run.smithi153.stdout:Installing: 2022-09-29T05:19:47.342 INFO:teuthology.orchestra.run.smithi153.stdout: librados-devel x86_64 2:16.2.10-932.g966bd07c.el8 ceph 131 k 2022-09-29T05:19:47.343 INFO:teuthology.orchestra.run.smithi153.stdout: 2022-09-29T05:19:47.343 INFO:teuthology.orchestra.run.smithi153.stdout:Transaction Summary 2022-09-29T05:19:47.343 INFO:teuthology.orchestra.run.smithi153.stdout:================================================================================ 2022-09-29T05:19:47.343 INFO:teuthology.orchestra.run.smithi153.stdout:Install 1 Package 2022-09-29T05:19:47.344 INFO:teuthology.orchestra.run.smithi153.stdout: 2022-09-29T05:19:47.344 INFO:teuthology.orchestra.run.smithi153.stdout:Total download size: 131 k 2022-09-29T05:19:47.344 INFO:teuthology.orchestra.run.smithi153.stdout:Installed size: 432 k 2022-09-29T05:19:47.344 INFO:teuthology.orchestra.run.smithi153.stdout:Downloading Packages: 2022-09-29T05:19:47.347 INFO:teuthology.orchestra.run.smithi083.stdout:Last metadata expiration check: 0:01:57 ago on Thu 29 Sep 2022 05:17:50 AM UTC. 2022-09-29T05:19:47.565 INFO:teuthology.orchestra.run.smithi153.stdout:librados-devel-16.2.10-932.g966bd07c.el8.x86_64 589 kB/s | 131 kB 00:00 2022-09-29T05:19:47.566 INFO:teuthology.orchestra.run.smithi153.stdout:-------------------------------------------------------------------------------- 2022-09-29T05:19:47.566 INFO:teuthology.orchestra.run.smithi153.stdout:Total 586 kB/s | 131 kB 00:00 2022-09-29T05:19:47.566 INFO:teuthology.orchestra.run.smithi153.stdout:Running transaction check 2022-09-29T05:19:47.571 INFO:teuthology.orchestra.run.smithi153.stdout:Transaction check succeeded. 2022-09-29T05:19:47.571 INFO:teuthology.orchestra.run.smithi153.stdout:Running transaction test 2022-09-29T05:19:47.620 INFO:teuthology.orchestra.run.smithi153.stdout:Transaction test succeeded. 2022-09-29T05:19:47.623 INFO:teuthology.orchestra.run.smithi153.stdout:Running transaction 2022-09-29T05:19:47.738 INFO:teuthology.orchestra.run.smithi153.stdout: Preparing : 1/1 2022-09-29T05:19:48.120 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : librados-devel-2:16.2.10-932.g966bd07c.el8.x86_64 1/1 2022-09-29T05:19:48.404 INFO:teuthology.orchestra.run.smithi153.stdout: Running scriptlet: librados-devel-2:16.2.10-932.g966bd07c.el8.x86_64 1/1 2022-09-29T05:19:48.417 INFO:teuthology.orchestra.run.smithi083.stdout:Dependencies resolved. 2022-09-29T05:19:48.418 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2022-09-29T05:19:48.418 INFO:teuthology.orchestra.run.smithi083.stdout: Package Arch Version Repository Size 2022-09-29T05:19:48.419 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2022-09-29T05:19:48.419 INFO:teuthology.orchestra.run.smithi083.stdout:Installing: 2022-09-29T05:19:48.419 INFO:teuthology.orchestra.run.smithi083.stdout: ceph-fuse x86_64 2:16.2.10-932.g966bd07c.el8 ceph 906 k 2022-09-29T05:19:48.419 INFO:teuthology.orchestra.run.smithi083.stdout:Installing dependencies: 2022-09-29T05:19:48.420 INFO:teuthology.orchestra.run.smithi083.stdout: fuse x86_64 2.9.7-16.el8 CentOS-BaseOS 83 k 2022-09-29T05:19:48.420 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:19:48.420 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction Summary 2022-09-29T05:19:48.421 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2022-09-29T05:19:48.421 INFO:teuthology.orchestra.run.smithi083.stdout:Install 2 Packages 2022-09-29T05:19:48.421 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:19:48.422 INFO:teuthology.orchestra.run.smithi083.stdout:Total download size: 989 k 2022-09-29T05:19:48.422 INFO:teuthology.orchestra.run.smithi083.stdout:Installed size: 2.8 M 2022-09-29T05:19:48.422 INFO:teuthology.orchestra.run.smithi083.stdout:Downloading Packages: 2022-09-29T05:19:48.493 INFO:teuthology.orchestra.run.smithi083.stdout:(1/2): fuse-2.9.7-16.el8.x86_64.rpm 1.1 MB/s | 83 kB 00:00 2022-09-29T05:19:48.693 INFO:teuthology.orchestra.run.smithi083.stdout:(2/2): ceph-fuse-16.2.10-932.g966bd07c.el8.x86_ 3.2 MB/s | 906 kB 00:00 2022-09-29T05:19:48.694 INFO:teuthology.orchestra.run.smithi083.stdout:-------------------------------------------------------------------------------- 2022-09-29T05:19:48.695 INFO:teuthology.orchestra.run.smithi083.stdout:Total 3.5 MB/s | 989 kB 00:00 2022-09-29T05:19:48.704 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction check 2022-09-29T05:19:48.725 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction check succeeded. 2022-09-29T05:19:48.725 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction test 2022-09-29T05:19:48.799 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction test succeeded. 2022-09-29T05:19:48.802 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction 2022-09-29T05:19:48.812 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : librados-devel-2:16.2.10-932.g966bd07c.el8.x86_64 1/1 2022-09-29T05:19:48.812 INFO:teuthology.orchestra.run.smithi153.stdout: 2022-09-29T05:19:48.812 INFO:teuthology.orchestra.run.smithi153.stdout:Installed: 2022-09-29T05:19:48.813 INFO:teuthology.orchestra.run.smithi153.stdout: librados-devel-2:16.2.10-932.g966bd07c.el8.x86_64 2022-09-29T05:19:48.813 INFO:teuthology.orchestra.run.smithi153.stdout: 2022-09-29T05:19:48.813 INFO:teuthology.orchestra.run.smithi153.stdout:Complete! 2022-09-29T05:19:48.965 INFO:teuthology.orchestra.run.smithi083.stdout: Preparing : 1/1 2022-09-29T05:19:48.985 DEBUG:teuthology.orchestra.run.smithi153:> sudo yum -y install libcephfs2 2022-09-29T05:19:49.245 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : fuse-2.9.7-16.el8.x86_64 1/2 2022-09-29T05:19:49.429 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : ceph-fuse-2:16.2.10-932.g966bd07c.el8.x86_64 2/2 2022-09-29T05:19:49.568 INFO:teuthology.orchestra.run.smithi153.stdout:Last metadata expiration check: 0:01:59 ago on Thu 29 Sep 2022 05:17:50 AM UTC. 2022-09-29T05:19:49.768 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: ceph-fuse-2:16.2.10-932.g966bd07c.el8.x86_64 2/2 2022-09-29T05:19:49.768 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : ceph-fuse-2:16.2.10-932.g966bd07c.el8.x86_64 1/2 2022-09-29T05:19:50.197 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : fuse-2.9.7-16.el8.x86_64 2/2 2022-09-29T05:19:50.197 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:19:50.197 INFO:teuthology.orchestra.run.smithi083.stdout:Installed: 2022-09-29T05:19:50.198 INFO:teuthology.orchestra.run.smithi083.stdout: ceph-fuse-2:16.2.10-932.g966bd07c.el8.x86_64 fuse-2.9.7-16.el8.x86_64 2022-09-29T05:19:50.198 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:19:50.198 INFO:teuthology.orchestra.run.smithi083.stdout:Complete! 2022-09-29T05:19:50.418 DEBUG:teuthology.orchestra.run.smithi083:> sudo yum -y install librados-devel 2022-09-29T05:19:50.555 INFO:teuthology.orchestra.run.smithi153.stdout:Package libcephfs2-2:16.2.10-932.g966bd07c.el8.x86_64 is already installed. 2022-09-29T05:19:50.636 INFO:teuthology.orchestra.run.smithi153.stdout:Dependencies resolved. 2022-09-29T05:19:50.638 INFO:teuthology.orchestra.run.smithi153.stdout:Nothing to do. 2022-09-29T05:19:50.638 INFO:teuthology.orchestra.run.smithi153.stdout:Complete! 2022-09-29T05:19:50.709 DEBUG:teuthology.orchestra.run.smithi153:> sudo yum -y install libcephfs-devel 2022-09-29T05:19:50.997 INFO:teuthology.orchestra.run.smithi083.stdout:Last metadata expiration check: 0:02:00 ago on Thu 29 Sep 2022 05:17:50 AM UTC. 2022-09-29T05:19:51.285 INFO:teuthology.orchestra.run.smithi153.stdout:Last metadata expiration check: 0:02:01 ago on Thu 29 Sep 2022 05:17:50 AM UTC. 2022-09-29T05:19:52.058 INFO:teuthology.orchestra.run.smithi083.stdout:Dependencies resolved. 2022-09-29T05:19:52.060 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2022-09-29T05:19:52.060 INFO:teuthology.orchestra.run.smithi083.stdout: Package Arch Version Repo Size 2022-09-29T05:19:52.060 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2022-09-29T05:19:52.060 INFO:teuthology.orchestra.run.smithi083.stdout:Installing: 2022-09-29T05:19:52.061 INFO:teuthology.orchestra.run.smithi083.stdout: librados-devel x86_64 2:16.2.10-932.g966bd07c.el8 ceph 131 k 2022-09-29T05:19:52.061 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:19:52.061 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction Summary 2022-09-29T05:19:52.061 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2022-09-29T05:19:52.062 INFO:teuthology.orchestra.run.smithi083.stdout:Install 1 Package 2022-09-29T05:19:52.062 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:19:52.062 INFO:teuthology.orchestra.run.smithi083.stdout:Total download size: 131 k 2022-09-29T05:19:52.062 INFO:teuthology.orchestra.run.smithi083.stdout:Installed size: 432 k 2022-09-29T05:19:52.062 INFO:teuthology.orchestra.run.smithi083.stdout:Downloading Packages: 2022-09-29T05:19:52.242 INFO:teuthology.orchestra.run.smithi083.stdout:librados-devel-16.2.10-932.g966bd07c.el8.x86_64 722 kB/s | 131 kB 00:00 2022-09-29T05:19:52.243 INFO:teuthology.orchestra.run.smithi083.stdout:-------------------------------------------------------------------------------- 2022-09-29T05:19:52.243 INFO:teuthology.orchestra.run.smithi083.stdout:Total 718 kB/s | 131 kB 00:00 2022-09-29T05:19:52.243 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction check 2022-09-29T05:19:52.248 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction check succeeded. 2022-09-29T05:19:52.248 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction test 2022-09-29T05:19:52.296 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction test succeeded. 2022-09-29T05:19:52.299 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction 2022-09-29T05:19:52.355 INFO:teuthology.orchestra.run.smithi153.stdout:Dependencies resolved. 2022-09-29T05:19:52.356 INFO:teuthology.orchestra.run.smithi153.stdout:================================================================================ 2022-09-29T05:19:52.356 INFO:teuthology.orchestra.run.smithi153.stdout: Package Arch Version Repo Size 2022-09-29T05:19:52.356 INFO:teuthology.orchestra.run.smithi153.stdout:================================================================================ 2022-09-29T05:19:52.357 INFO:teuthology.orchestra.run.smithi153.stdout:Installing: 2022-09-29T05:19:52.357 INFO:teuthology.orchestra.run.smithi153.stdout: libcephfs-devel x86_64 2:16.2.10-932.g966bd07c.el8 ceph 25 k 2022-09-29T05:19:52.357 INFO:teuthology.orchestra.run.smithi153.stdout: 2022-09-29T05:19:52.357 INFO:teuthology.orchestra.run.smithi153.stdout:Transaction Summary 2022-09-29T05:19:52.357 INFO:teuthology.orchestra.run.smithi153.stdout:================================================================================ 2022-09-29T05:19:52.358 INFO:teuthology.orchestra.run.smithi153.stdout:Install 1 Package 2022-09-29T05:19:52.358 INFO:teuthology.orchestra.run.smithi153.stdout: 2022-09-29T05:19:52.359 INFO:teuthology.orchestra.run.smithi153.stdout:Total download size: 25 k 2022-09-29T05:19:52.359 INFO:teuthology.orchestra.run.smithi153.stdout:Installed size: 107 k 2022-09-29T05:19:52.359 INFO:teuthology.orchestra.run.smithi153.stdout:Downloading Packages: 2022-09-29T05:19:52.411 INFO:teuthology.orchestra.run.smithi083.stdout: Preparing : 1/1 2022-09-29T05:19:52.503 INFO:teuthology.orchestra.run.smithi153.stdout:libcephfs-devel-16.2.10-932.g966bd07c.el8.x86_6 175 kB/s | 25 kB 00:00 2022-09-29T05:19:52.504 INFO:teuthology.orchestra.run.smithi153.stdout:-------------------------------------------------------------------------------- 2022-09-29T05:19:52.504 INFO:teuthology.orchestra.run.smithi153.stdout:Total 174 kB/s | 25 kB 00:00 2022-09-29T05:19:52.504 INFO:teuthology.orchestra.run.smithi153.stdout:Running transaction check 2022-09-29T05:19:52.507 INFO:teuthology.orchestra.run.smithi153.stdout:Transaction check succeeded. 2022-09-29T05:19:52.507 INFO:teuthology.orchestra.run.smithi153.stdout:Running transaction test 2022-09-29T05:19:52.509 INFO:teuthology.orchestra.run.smithi153.stdout:Transaction test succeeded. 2022-09-29T05:19:52.512 INFO:teuthology.orchestra.run.smithi153.stdout:Running transaction 2022-09-29T05:19:52.564 INFO:teuthology.orchestra.run.smithi153.stdout: Preparing : 1/1 2022-09-29T05:19:52.607 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : librados-devel-2:16.2.10-932.g966bd07c.el8.x86_64 1/1 2022-09-29T05:19:52.745 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : libcephfs-devel-2:16.2.10-932.g966bd07c.el8.x86_64 1/1 2022-09-29T05:19:52.848 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: librados-devel-2:16.2.10-932.g966bd07c.el8.x86_64 1/1 2022-09-29T05:19:52.971 INFO:teuthology.orchestra.run.smithi153.stdout: Running scriptlet: libcephfs-devel-2:16.2.10-932.g966bd07c.el8.x86_64 1/1 2022-09-29T05:19:53.239 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : librados-devel-2:16.2.10-932.g966bd07c.el8.x86_64 1/1 2022-09-29T05:19:53.240 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:19:53.240 INFO:teuthology.orchestra.run.smithi083.stdout:Installed: 2022-09-29T05:19:53.240 INFO:teuthology.orchestra.run.smithi083.stdout: librados-devel-2:16.2.10-932.g966bd07c.el8.x86_64 2022-09-29T05:19:53.240 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:19:53.241 INFO:teuthology.orchestra.run.smithi083.stdout:Complete! 2022-09-29T05:19:53.332 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : libcephfs-devel-2:16.2.10-932.g966bd07c.el8.x86_64 1/1 2022-09-29T05:19:53.332 INFO:teuthology.orchestra.run.smithi153.stdout: 2022-09-29T05:19:53.332 INFO:teuthology.orchestra.run.smithi153.stdout:Installed: 2022-09-29T05:19:53.333 INFO:teuthology.orchestra.run.smithi153.stdout: libcephfs-devel-2:16.2.10-932.g966bd07c.el8.x86_64 2022-09-29T05:19:53.333 INFO:teuthology.orchestra.run.smithi153.stdout: 2022-09-29T05:19:53.333 INFO:teuthology.orchestra.run.smithi153.stdout:Complete! 2022-09-29T05:19:53.424 DEBUG:teuthology.orchestra.run.smithi153:> sudo yum -y install librados2 2022-09-29T05:19:53.431 DEBUG:teuthology.orchestra.run.smithi083:> sudo yum -y install libcephfs2 2022-09-29T05:19:54.005 INFO:teuthology.orchestra.run.smithi153.stdout:Last metadata expiration check: 0:02:04 ago on Thu 29 Sep 2022 05:17:50 AM UTC. 2022-09-29T05:19:54.011 INFO:teuthology.orchestra.run.smithi083.stdout:Last metadata expiration check: 0:02:04 ago on Thu 29 Sep 2022 05:17:50 AM UTC. 2022-09-29T05:19:54.982 INFO:teuthology.orchestra.run.smithi153.stdout:Package librados2-2:16.2.10-932.g966bd07c.el8.x86_64 is already installed. 2022-09-29T05:19:54.994 INFO:teuthology.orchestra.run.smithi083.stdout:Package libcephfs2-2:16.2.10-932.g966bd07c.el8.x86_64 is already installed. 2022-09-29T05:19:55.063 INFO:teuthology.orchestra.run.smithi153.stdout:Dependencies resolved. 2022-09-29T05:19:55.064 INFO:teuthology.orchestra.run.smithi153.stdout:Nothing to do. 2022-09-29T05:19:55.064 INFO:teuthology.orchestra.run.smithi153.stdout:Complete! 2022-09-29T05:19:55.077 INFO:teuthology.orchestra.run.smithi083.stdout:Dependencies resolved. 2022-09-29T05:19:55.078 INFO:teuthology.orchestra.run.smithi083.stdout:Nothing to do. 2022-09-29T05:19:55.079 INFO:teuthology.orchestra.run.smithi083.stdout:Complete! 2022-09-29T05:19:55.140 DEBUG:teuthology.orchestra.run.smithi153:> sudo yum -y install librbd1 2022-09-29T05:19:55.161 DEBUG:teuthology.orchestra.run.smithi083:> sudo yum -y install libcephfs-devel 2022-09-29T05:19:55.714 INFO:teuthology.orchestra.run.smithi153.stdout:Last metadata expiration check: 0:02:05 ago on Thu 29 Sep 2022 05:17:50 AM UTC. 2022-09-29T05:19:55.735 INFO:teuthology.orchestra.run.smithi083.stdout:Last metadata expiration check: 0:02:05 ago on Thu 29 Sep 2022 05:17:50 AM UTC. 2022-09-29T05:19:56.711 INFO:teuthology.orchestra.run.smithi153.stdout:Package librbd1-2:16.2.10-932.g966bd07c.el8.x86_64 is already installed. 2022-09-29T05:19:56.792 INFO:teuthology.orchestra.run.smithi153.stdout:Dependencies resolved. 2022-09-29T05:19:56.793 INFO:teuthology.orchestra.run.smithi153.stdout:Nothing to do. 2022-09-29T05:19:56.793 INFO:teuthology.orchestra.run.smithi153.stdout:Complete! 2022-09-29T05:19:56.811 INFO:teuthology.orchestra.run.smithi083.stdout:Dependencies resolved. 2022-09-29T05:19:56.811 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2022-09-29T05:19:56.811 INFO:teuthology.orchestra.run.smithi083.stdout: Package Arch Version Repo Size 2022-09-29T05:19:56.812 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2022-09-29T05:19:56.812 INFO:teuthology.orchestra.run.smithi083.stdout:Installing: 2022-09-29T05:19:56.812 INFO:teuthology.orchestra.run.smithi083.stdout: libcephfs-devel x86_64 2:16.2.10-932.g966bd07c.el8 ceph 25 k 2022-09-29T05:19:56.812 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:19:56.813 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction Summary 2022-09-29T05:19:56.813 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2022-09-29T05:19:56.813 INFO:teuthology.orchestra.run.smithi083.stdout:Install 1 Package 2022-09-29T05:19:56.813 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:19:56.814 INFO:teuthology.orchestra.run.smithi083.stdout:Total download size: 25 k 2022-09-29T05:19:56.814 INFO:teuthology.orchestra.run.smithi083.stdout:Installed size: 107 k 2022-09-29T05:19:56.815 INFO:teuthology.orchestra.run.smithi083.stdout:Downloading Packages: 2022-09-29T05:19:56.876 DEBUG:teuthology.orchestra.run.smithi153:> sudo yum -y install python3-rados 2022-09-29T05:19:56.954 INFO:teuthology.orchestra.run.smithi083.stdout:libcephfs-devel-16.2.10-932.g966bd07c.el8.x86_6 180 kB/s | 25 kB 00:00 2022-09-29T05:19:56.955 INFO:teuthology.orchestra.run.smithi083.stdout:-------------------------------------------------------------------------------- 2022-09-29T05:19:56.955 INFO:teuthology.orchestra.run.smithi083.stdout:Total 178 kB/s | 25 kB 00:00 2022-09-29T05:19:56.956 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction check 2022-09-29T05:19:56.959 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction check succeeded. 2022-09-29T05:19:56.959 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction test 2022-09-29T05:19:56.961 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction test succeeded. 2022-09-29T05:19:56.964 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction 2022-09-29T05:19:57.013 INFO:teuthology.orchestra.run.smithi083.stdout: Preparing : 1/1 2022-09-29T05:19:57.185 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : libcephfs-devel-2:16.2.10-932.g966bd07c.el8.x86_64 1/1 2022-09-29T05:19:57.413 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: libcephfs-devel-2:16.2.10-932.g966bd07c.el8.x86_64 1/1 2022-09-29T05:19:57.451 INFO:teuthology.orchestra.run.smithi153.stdout:Last metadata expiration check: 0:02:07 ago on Thu 29 Sep 2022 05:17:50 AM UTC. 2022-09-29T05:19:57.778 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : libcephfs-devel-2:16.2.10-932.g966bd07c.el8.x86_64 1/1 2022-09-29T05:19:57.778 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:19:57.778 INFO:teuthology.orchestra.run.smithi083.stdout:Installed: 2022-09-29T05:19:57.779 INFO:teuthology.orchestra.run.smithi083.stdout: libcephfs-devel-2:16.2.10-932.g966bd07c.el8.x86_64 2022-09-29T05:19:57.779 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:19:57.779 INFO:teuthology.orchestra.run.smithi083.stdout:Complete! 2022-09-29T05:19:57.880 DEBUG:teuthology.orchestra.run.smithi083:> sudo yum -y install librados2 2022-09-29T05:19:58.453 INFO:teuthology.orchestra.run.smithi153.stdout:Package python3-rados-2:16.2.10-932.g966bd07c.el8.x86_64 is already installed. 2022-09-29T05:19:58.454 INFO:teuthology.orchestra.run.smithi083.stdout:Last metadata expiration check: 0:02:08 ago on Thu 29 Sep 2022 05:17:50 AM UTC. 2022-09-29T05:19:58.536 INFO:teuthology.orchestra.run.smithi153.stdout:Dependencies resolved. 2022-09-29T05:19:58.537 INFO:teuthology.orchestra.run.smithi153.stdout:Nothing to do. 2022-09-29T05:19:58.538 INFO:teuthology.orchestra.run.smithi153.stdout:Complete! 2022-09-29T05:19:58.611 DEBUG:teuthology.orchestra.run.smithi153:> sudo yum -y install python3-rgw 2022-09-29T05:19:59.188 INFO:teuthology.orchestra.run.smithi153.stdout:Last metadata expiration check: 0:02:09 ago on Thu 29 Sep 2022 05:17:50 AM UTC. 2022-09-29T05:19:59.441 INFO:teuthology.orchestra.run.smithi083.stdout:Package librados2-2:16.2.10-932.g966bd07c.el8.x86_64 is already installed. 2022-09-29T05:19:59.521 INFO:teuthology.orchestra.run.smithi083.stdout:Dependencies resolved. 2022-09-29T05:19:59.522 INFO:teuthology.orchestra.run.smithi083.stdout:Nothing to do. 2022-09-29T05:19:59.522 INFO:teuthology.orchestra.run.smithi083.stdout:Complete! 2022-09-29T05:19:59.603 DEBUG:teuthology.orchestra.run.smithi083:> sudo yum -y install librbd1 2022-09-29T05:20:00.173 INFO:teuthology.orchestra.run.smithi153.stdout:Package python3-rgw-2:16.2.10-932.g966bd07c.el8.x86_64 is already installed. 2022-09-29T05:20:00.176 INFO:teuthology.orchestra.run.smithi083.stdout:Last metadata expiration check: 0:02:10 ago on Thu 29 Sep 2022 05:17:50 AM UTC. 2022-09-29T05:20:00.252 INFO:teuthology.orchestra.run.smithi153.stdout:Dependencies resolved. 2022-09-29T05:20:00.253 INFO:teuthology.orchestra.run.smithi153.stdout:Nothing to do. 2022-09-29T05:20:00.254 INFO:teuthology.orchestra.run.smithi153.stdout:Complete! 2022-09-29T05:20:00.452 DEBUG:teuthology.orchestra.run.smithi153:> sudo yum -y install python3-cephfs 2022-09-29T05:20:01.029 INFO:teuthology.orchestra.run.smithi153.stdout:Last metadata expiration check: 0:02:11 ago on Thu 29 Sep 2022 05:17:50 AM UTC. 2022-09-29T05:20:01.163 INFO:teuthology.orchestra.run.smithi083.stdout:Package librbd1-2:16.2.10-932.g966bd07c.el8.x86_64 is already installed. 2022-09-29T05:20:01.244 INFO:teuthology.orchestra.run.smithi083.stdout:Dependencies resolved. 2022-09-29T05:20:01.245 INFO:teuthology.orchestra.run.smithi083.stdout:Nothing to do. 2022-09-29T05:20:01.245 INFO:teuthology.orchestra.run.smithi083.stdout:Complete! 2022-09-29T05:20:01.336 DEBUG:teuthology.orchestra.run.smithi083:> sudo yum -y install python3-rados 2022-09-29T05:20:01.911 INFO:teuthology.orchestra.run.smithi083.stdout:Last metadata expiration check: 0:02:11 ago on Thu 29 Sep 2022 05:17:50 AM UTC. 2022-09-29T05:20:02.022 INFO:teuthology.orchestra.run.smithi153.stdout:Package python3-cephfs-2:16.2.10-932.g966bd07c.el8.x86_64 is already installed. 2022-09-29T05:20:02.102 INFO:teuthology.orchestra.run.smithi153.stdout:Dependencies resolved. 2022-09-29T05:20:02.103 INFO:teuthology.orchestra.run.smithi153.stdout:Nothing to do. 2022-09-29T05:20:02.104 INFO:teuthology.orchestra.run.smithi153.stdout:Complete! 2022-09-29T05:20:02.176 DEBUG:teuthology.orchestra.run.smithi153:> sudo yum -y install python3-rbd 2022-09-29T05:20:02.756 INFO:teuthology.orchestra.run.smithi153.stdout:Last metadata expiration check: 0:02:12 ago on Thu 29 Sep 2022 05:17:50 AM UTC. 2022-09-29T05:20:02.902 INFO:teuthology.orchestra.run.smithi083.stdout:Package python3-rados-2:16.2.10-932.g966bd07c.el8.x86_64 is already installed. 2022-09-29T05:20:02.983 INFO:teuthology.orchestra.run.smithi083.stdout:Dependencies resolved. 2022-09-29T05:20:02.983 INFO:teuthology.orchestra.run.smithi083.stdout:Nothing to do. 2022-09-29T05:20:02.984 INFO:teuthology.orchestra.run.smithi083.stdout:Complete! 2022-09-29T05:20:03.119 DEBUG:teuthology.orchestra.run.smithi083:> sudo yum -y install python3-rgw 2022-09-29T05:20:03.692 INFO:teuthology.orchestra.run.smithi083.stdout:Last metadata expiration check: 0:02:13 ago on Thu 29 Sep 2022 05:17:50 AM UTC. 2022-09-29T05:20:03.758 INFO:teuthology.orchestra.run.smithi153.stdout:Package python3-rbd-2:16.2.10-932.g966bd07c.el8.x86_64 is already installed. 2022-09-29T05:20:03.837 INFO:teuthology.orchestra.run.smithi153.stdout:Dependencies resolved. 2022-09-29T05:20:03.838 INFO:teuthology.orchestra.run.smithi153.stdout:Nothing to do. 2022-09-29T05:20:03.838 INFO:teuthology.orchestra.run.smithi153.stdout:Complete! 2022-09-29T05:20:03.915 DEBUG:teuthology.orchestra.run.smithi153:> sudo yum -y install rbd-fuse 2022-09-29T05:20:04.493 INFO:teuthology.orchestra.run.smithi153.stdout:Last metadata expiration check: 0:02:14 ago on Thu 29 Sep 2022 05:17:50 AM UTC. 2022-09-29T05:20:04.688 INFO:teuthology.orchestra.run.smithi083.stdout:Package python3-rgw-2:16.2.10-932.g966bd07c.el8.x86_64 is already installed. 2022-09-29T05:20:04.768 INFO:teuthology.orchestra.run.smithi083.stdout:Dependencies resolved. 2022-09-29T05:20:04.769 INFO:teuthology.orchestra.run.smithi083.stdout:Nothing to do. 2022-09-29T05:20:04.769 INFO:teuthology.orchestra.run.smithi083.stdout:Complete! 2022-09-29T05:20:04.860 DEBUG:teuthology.orchestra.run.smithi083:> sudo yum -y install python3-cephfs 2022-09-29T05:20:05.433 INFO:teuthology.orchestra.run.smithi083.stdout:Last metadata expiration check: 0:02:15 ago on Thu 29 Sep 2022 05:17:50 AM UTC. 2022-09-29T05:20:05.559 INFO:teuthology.orchestra.run.smithi153.stdout:Dependencies resolved. 2022-09-29T05:20:05.560 INFO:teuthology.orchestra.run.smithi153.stdout:================================================================================ 2022-09-29T05:20:05.560 INFO:teuthology.orchestra.run.smithi153.stdout: Package Architecture Version Repository Size 2022-09-29T05:20:05.560 INFO:teuthology.orchestra.run.smithi153.stdout:================================================================================ 2022-09-29T05:20:05.560 INFO:teuthology.orchestra.run.smithi153.stdout:Installing: 2022-09-29T05:20:05.561 INFO:teuthology.orchestra.run.smithi153.stdout: rbd-fuse x86_64 2:16.2.10-932.g966bd07c.el8 ceph 88 k 2022-09-29T05:20:05.561 INFO:teuthology.orchestra.run.smithi153.stdout: 2022-09-29T05:20:05.561 INFO:teuthology.orchestra.run.smithi153.stdout:Transaction Summary 2022-09-29T05:20:05.561 INFO:teuthology.orchestra.run.smithi153.stdout:================================================================================ 2022-09-29T05:20:05.562 INFO:teuthology.orchestra.run.smithi153.stdout:Install 1 Package 2022-09-29T05:20:05.562 INFO:teuthology.orchestra.run.smithi153.stdout: 2022-09-29T05:20:05.562 INFO:teuthology.orchestra.run.smithi153.stdout:Total download size: 88 k 2022-09-29T05:20:05.562 INFO:teuthology.orchestra.run.smithi153.stdout:Installed size: 240 k 2022-09-29T05:20:05.563 INFO:teuthology.orchestra.run.smithi153.stdout:Downloading Packages: 2022-09-29T05:20:05.735 INFO:teuthology.orchestra.run.smithi153.stdout:rbd-fuse-16.2.10-932.g966bd07c.el8.x86_64.rpm 507 kB/s | 88 kB 00:00 2022-09-29T05:20:05.736 INFO:teuthology.orchestra.run.smithi153.stdout:-------------------------------------------------------------------------------- 2022-09-29T05:20:05.736 INFO:teuthology.orchestra.run.smithi153.stdout:Total 504 kB/s | 88 kB 00:00 2022-09-29T05:20:05.737 INFO:teuthology.orchestra.run.smithi153.stdout:Running transaction check 2022-09-29T05:20:05.745 INFO:teuthology.orchestra.run.smithi153.stdout:Transaction check succeeded. 2022-09-29T05:20:05.746 INFO:teuthology.orchestra.run.smithi153.stdout:Running transaction test 2022-09-29T05:20:05.794 INFO:teuthology.orchestra.run.smithi153.stdout:Transaction test succeeded. 2022-09-29T05:20:05.797 INFO:teuthology.orchestra.run.smithi153.stdout:Running transaction 2022-09-29T05:20:05.905 INFO:teuthology.orchestra.run.smithi153.stdout: Preparing : 1/1 2022-09-29T05:20:06.084 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : rbd-fuse-2:16.2.10-932.g966bd07c.el8.x86_64 1/1 2022-09-29T05:20:06.345 INFO:teuthology.orchestra.run.smithi153.stdout: Running scriptlet: rbd-fuse-2:16.2.10-932.g966bd07c.el8.x86_64 1/1 2022-09-29T05:20:06.416 INFO:teuthology.orchestra.run.smithi083.stdout:Package python3-cephfs-2:16.2.10-932.g966bd07c.el8.x86_64 is already installed. 2022-09-29T05:20:06.495 INFO:teuthology.orchestra.run.smithi083.stdout:Dependencies resolved. 2022-09-29T05:20:06.496 INFO:teuthology.orchestra.run.smithi083.stdout:Nothing to do. 2022-09-29T05:20:06.496 INFO:teuthology.orchestra.run.smithi083.stdout:Complete! 2022-09-29T05:20:06.577 DEBUG:teuthology.orchestra.run.smithi083:> sudo yum -y install python3-rbd 2022-09-29T05:20:06.745 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : rbd-fuse-2:16.2.10-932.g966bd07c.el8.x86_64 1/1 2022-09-29T05:20:06.745 INFO:teuthology.orchestra.run.smithi153.stdout: 2022-09-29T05:20:06.745 INFO:teuthology.orchestra.run.smithi153.stdout:Installed: 2022-09-29T05:20:06.746 INFO:teuthology.orchestra.run.smithi153.stdout: rbd-fuse-2:16.2.10-932.g966bd07c.el8.x86_64 2022-09-29T05:20:06.746 INFO:teuthology.orchestra.run.smithi153.stdout: 2022-09-29T05:20:06.746 INFO:teuthology.orchestra.run.smithi153.stdout:Complete! 2022-09-29T05:20:06.907 DEBUG:teuthology.orchestra.run.smithi153:> sudo yum -y install rbd-mirror 2022-09-29T05:20:07.147 INFO:teuthology.orchestra.run.smithi083.stdout:Last metadata expiration check: 0:02:17 ago on Thu 29 Sep 2022 05:17:50 AM UTC. 2022-09-29T05:20:07.493 INFO:teuthology.orchestra.run.smithi153.stdout:Last metadata expiration check: 0:02:17 ago on Thu 29 Sep 2022 05:17:50 AM UTC. 2022-09-29T05:20:08.137 INFO:teuthology.orchestra.run.smithi083.stdout:Package python3-rbd-2:16.2.10-932.g966bd07c.el8.x86_64 is already installed. 2022-09-29T05:20:08.216 INFO:teuthology.orchestra.run.smithi083.stdout:Dependencies resolved. 2022-09-29T05:20:08.218 INFO:teuthology.orchestra.run.smithi083.stdout:Nothing to do. 2022-09-29T05:20:08.218 INFO:teuthology.orchestra.run.smithi083.stdout:Complete! 2022-09-29T05:20:08.297 DEBUG:teuthology.orchestra.run.smithi083:> sudo yum -y install rbd-fuse 2022-09-29T05:20:08.559 INFO:teuthology.orchestra.run.smithi153.stdout:Dependencies resolved. 2022-09-29T05:20:08.560 INFO:teuthology.orchestra.run.smithi153.stdout:================================================================================ 2022-09-29T05:20:08.561 INFO:teuthology.orchestra.run.smithi153.stdout: Package Arch Version Repo Size 2022-09-29T05:20:08.561 INFO:teuthology.orchestra.run.smithi153.stdout:================================================================================ 2022-09-29T05:20:08.561 INFO:teuthology.orchestra.run.smithi153.stdout:Installing: 2022-09-29T05:20:08.561 INFO:teuthology.orchestra.run.smithi153.stdout: rbd-mirror x86_64 2:16.2.10-932.g966bd07c.el8 ceph 3.7 M 2022-09-29T05:20:08.562 INFO:teuthology.orchestra.run.smithi153.stdout: 2022-09-29T05:20:08.562 INFO:teuthology.orchestra.run.smithi153.stdout:Transaction Summary 2022-09-29T05:20:08.562 INFO:teuthology.orchestra.run.smithi153.stdout:================================================================================ 2022-09-29T05:20:08.562 INFO:teuthology.orchestra.run.smithi153.stdout:Install 1 Package 2022-09-29T05:20:08.563 INFO:teuthology.orchestra.run.smithi153.stdout: 2022-09-29T05:20:08.563 INFO:teuthology.orchestra.run.smithi153.stdout:Total download size: 3.7 M 2022-09-29T05:20:08.563 INFO:teuthology.orchestra.run.smithi153.stdout:Installed size: 15 M 2022-09-29T05:20:08.563 INFO:teuthology.orchestra.run.smithi153.stdout:Downloading Packages: 2022-09-29T05:20:08.867 INFO:teuthology.orchestra.run.smithi083.stdout:Last metadata expiration check: 0:02:18 ago on Thu 29 Sep 2022 05:17:50 AM UTC. 2022-09-29T05:20:08.920 INFO:teuthology.orchestra.run.smithi153.stdout:rbd-mirror-16.2.10-932.g966bd07c.el8.x86_64.rpm 10 MB/s | 3.7 MB 00:00 2022-09-29T05:20:08.921 INFO:teuthology.orchestra.run.smithi153.stdout:-------------------------------------------------------------------------------- 2022-09-29T05:20:08.922 INFO:teuthology.orchestra.run.smithi153.stdout:Total 10 MB/s | 3.7 MB 00:00 2022-09-29T05:20:08.922 INFO:teuthology.orchestra.run.smithi153.stdout:Running transaction check 2022-09-29T05:20:08.933 INFO:teuthology.orchestra.run.smithi153.stdout:Transaction check succeeded. 2022-09-29T05:20:08.933 INFO:teuthology.orchestra.run.smithi153.stdout:Running transaction test 2022-09-29T05:20:08.995 INFO:teuthology.orchestra.run.smithi153.stdout:Transaction test succeeded. 2022-09-29T05:20:08.998 INFO:teuthology.orchestra.run.smithi153.stdout:Running transaction 2022-09-29T05:20:09.437 INFO:teuthology.orchestra.run.smithi153.stdout: Preparing : 1/1 2022-09-29T05:20:09.628 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : rbd-mirror-2:16.2.10-932.g966bd07c.el8.x86_64 1/1 2022-09-29T05:20:09.936 INFO:teuthology.orchestra.run.smithi083.stdout:Dependencies resolved. 2022-09-29T05:20:09.937 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2022-09-29T05:20:09.937 INFO:teuthology.orchestra.run.smithi083.stdout: Package Architecture Version Repository Size 2022-09-29T05:20:09.938 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2022-09-29T05:20:09.938 INFO:teuthology.orchestra.run.smithi083.stdout:Installing: 2022-09-29T05:20:09.938 INFO:teuthology.orchestra.run.smithi083.stdout: rbd-fuse x86_64 2:16.2.10-932.g966bd07c.el8 ceph 88 k 2022-09-29T05:20:09.938 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:20:09.939 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction Summary 2022-09-29T05:20:09.939 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2022-09-29T05:20:09.939 INFO:teuthology.orchestra.run.smithi083.stdout:Install 1 Package 2022-09-29T05:20:09.939 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:20:09.940 INFO:teuthology.orchestra.run.smithi083.stdout:Total download size: 88 k 2022-09-29T05:20:09.940 INFO:teuthology.orchestra.run.smithi083.stdout:Installed size: 240 k 2022-09-29T05:20:09.940 INFO:teuthology.orchestra.run.smithi083.stdout:Downloading Packages: 2022-09-29T05:20:10.007 INFO:teuthology.orchestra.run.smithi153.stdout: Running scriptlet: rbd-mirror-2:16.2.10-932.g966bd07c.el8.x86_64 1/1 2022-09-29T05:20:10.095 INFO:teuthology.orchestra.run.smithi083.stdout:rbd-fuse-16.2.10-932.g966bd07c.el8.x86_64.rpm 564 kB/s | 88 kB 00:00 2022-09-29T05:20:10.096 INFO:teuthology.orchestra.run.smithi083.stdout:-------------------------------------------------------------------------------- 2022-09-29T05:20:10.096 INFO:teuthology.orchestra.run.smithi083.stdout:Total 560 kB/s | 88 kB 00:00 2022-09-29T05:20:10.096 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction check 2022-09-29T05:20:10.105 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction check succeeded. 2022-09-29T05:20:10.105 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction test 2022-09-29T05:20:10.153 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction test succeeded. 2022-09-29T05:20:10.156 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction 2022-09-29T05:20:10.263 INFO:teuthology.orchestra.run.smithi083.stdout: Preparing : 1/1 2022-09-29T05:20:10.366 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : rbd-mirror-2:16.2.10-932.g966bd07c.el8.x86_64 1/1 2022-09-29T05:20:10.367 INFO:teuthology.orchestra.run.smithi153.stdout: 2022-09-29T05:20:10.367 INFO:teuthology.orchestra.run.smithi153.stdout:Installed: 2022-09-29T05:20:10.367 INFO:teuthology.orchestra.run.smithi153.stdout: rbd-mirror-2:16.2.10-932.g966bd07c.el8.x86_64 2022-09-29T05:20:10.368 INFO:teuthology.orchestra.run.smithi153.stdout: 2022-09-29T05:20:10.368 INFO:teuthology.orchestra.run.smithi153.stdout:Complete! 2022-09-29T05:20:10.475 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : rbd-fuse-2:16.2.10-932.g966bd07c.el8.x86_64 1/1 2022-09-29T05:20:10.530 DEBUG:teuthology.orchestra.run.smithi153:> sudo yum -y install rbd-nbd 2022-09-29T05:20:10.728 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: rbd-fuse-2:16.2.10-932.g966bd07c.el8.x86_64 1/1 2022-09-29T05:20:11.111 INFO:teuthology.orchestra.run.smithi153.stdout:Last metadata expiration check: 0:02:21 ago on Thu 29 Sep 2022 05:17:50 AM UTC. 2022-09-29T05:20:11.178 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : rbd-fuse-2:16.2.10-932.g966bd07c.el8.x86_64 1/1 2022-09-29T05:20:11.178 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:20:11.179 INFO:teuthology.orchestra.run.smithi083.stdout:Installed: 2022-09-29T05:20:11.179 INFO:teuthology.orchestra.run.smithi083.stdout: rbd-fuse-2:16.2.10-932.g966bd07c.el8.x86_64 2022-09-29T05:20:11.179 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:20:11.179 INFO:teuthology.orchestra.run.smithi083.stdout:Complete! 2022-09-29T05:20:11.397 DEBUG:teuthology.orchestra.run.smithi083:> sudo yum -y install rbd-mirror 2022-09-29T05:20:11.975 INFO:teuthology.orchestra.run.smithi083.stdout:Last metadata expiration check: 0:02:21 ago on Thu 29 Sep 2022 05:17:50 AM UTC. 2022-09-29T05:20:12.181 INFO:teuthology.orchestra.run.smithi153.stdout:Dependencies resolved. 2022-09-29T05:20:12.182 INFO:teuthology.orchestra.run.smithi153.stdout:================================================================================ 2022-09-29T05:20:12.183 INFO:teuthology.orchestra.run.smithi153.stdout: Package Architecture Version Repository Size 2022-09-29T05:20:12.183 INFO:teuthology.orchestra.run.smithi153.stdout:================================================================================ 2022-09-29T05:20:12.183 INFO:teuthology.orchestra.run.smithi153.stdout:Installing: 2022-09-29T05:20:12.183 INFO:teuthology.orchestra.run.smithi153.stdout: rbd-nbd x86_64 2:16.2.10-932.g966bd07c.el8 ceph 173 k 2022-09-29T05:20:12.184 INFO:teuthology.orchestra.run.smithi153.stdout: 2022-09-29T05:20:12.184 INFO:teuthology.orchestra.run.smithi153.stdout:Transaction Summary 2022-09-29T05:20:12.184 INFO:teuthology.orchestra.run.smithi153.stdout:================================================================================ 2022-09-29T05:20:12.184 INFO:teuthology.orchestra.run.smithi153.stdout:Install 1 Package 2022-09-29T05:20:12.184 INFO:teuthology.orchestra.run.smithi153.stdout: 2022-09-29T05:20:12.185 INFO:teuthology.orchestra.run.smithi153.stdout:Total download size: 173 k 2022-09-29T05:20:12.185 INFO:teuthology.orchestra.run.smithi153.stdout:Installed size: 515 k 2022-09-29T05:20:12.185 INFO:teuthology.orchestra.run.smithi153.stdout:Downloading Packages: 2022-09-29T05:20:12.375 INFO:teuthology.orchestra.run.smithi153.stdout:rbd-nbd-16.2.10-932.g966bd07c.el8.x86_64.rpm 905 kB/s | 173 kB 00:00 2022-09-29T05:20:12.375 INFO:teuthology.orchestra.run.smithi153.stdout:-------------------------------------------------------------------------------- 2022-09-29T05:20:12.376 INFO:teuthology.orchestra.run.smithi153.stdout:Total 900 kB/s | 173 kB 00:00 2022-09-29T05:20:12.376 INFO:teuthology.orchestra.run.smithi153.stdout:Running transaction check 2022-09-29T05:20:12.386 INFO:teuthology.orchestra.run.smithi153.stdout:Transaction check succeeded. 2022-09-29T05:20:12.386 INFO:teuthology.orchestra.run.smithi153.stdout:Running transaction test 2022-09-29T05:20:12.437 INFO:teuthology.orchestra.run.smithi153.stdout:Transaction test succeeded. 2022-09-29T05:20:12.440 INFO:teuthology.orchestra.run.smithi153.stdout:Running transaction 2022-09-29T05:20:12.557 INFO:teuthology.orchestra.run.smithi153.stdout: Preparing : 1/1 2022-09-29T05:20:12.708 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : rbd-nbd-2:16.2.10-932.g966bd07c.el8.x86_64 1/1 2022-09-29T05:20:12.943 INFO:teuthology.orchestra.run.smithi153.stdout: Running scriptlet: rbd-nbd-2:16.2.10-932.g966bd07c.el8.x86_64 1/1 2022-09-29T05:20:13.035 INFO:teuthology.orchestra.run.smithi083.stdout:Dependencies resolved. 2022-09-29T05:20:13.036 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2022-09-29T05:20:13.037 INFO:teuthology.orchestra.run.smithi083.stdout: Package Arch Version Repo Size 2022-09-29T05:20:13.037 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2022-09-29T05:20:13.037 INFO:teuthology.orchestra.run.smithi083.stdout:Installing: 2022-09-29T05:20:13.037 INFO:teuthology.orchestra.run.smithi083.stdout: rbd-mirror x86_64 2:16.2.10-932.g966bd07c.el8 ceph 3.7 M 2022-09-29T05:20:13.038 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:20:13.038 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction Summary 2022-09-29T05:20:13.038 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2022-09-29T05:20:13.038 INFO:teuthology.orchestra.run.smithi083.stdout:Install 1 Package 2022-09-29T05:20:13.039 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:20:13.039 INFO:teuthology.orchestra.run.smithi083.stdout:Total download size: 3.7 M 2022-09-29T05:20:13.039 INFO:teuthology.orchestra.run.smithi083.stdout:Installed size: 15 M 2022-09-29T05:20:13.039 INFO:teuthology.orchestra.run.smithi083.stdout:Downloading Packages: 2022-09-29T05:20:13.330 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : rbd-nbd-2:16.2.10-932.g966bd07c.el8.x86_64 1/1 2022-09-29T05:20:13.330 INFO:teuthology.orchestra.run.smithi153.stdout: 2022-09-29T05:20:13.331 INFO:teuthology.orchestra.run.smithi153.stdout:Installed: 2022-09-29T05:20:13.331 INFO:teuthology.orchestra.run.smithi153.stdout: rbd-nbd-2:16.2.10-932.g966bd07c.el8.x86_64 2022-09-29T05:20:13.331 INFO:teuthology.orchestra.run.smithi153.stdout: 2022-09-29T05:20:13.331 INFO:teuthology.orchestra.run.smithi153.stdout:Complete! 2022-09-29T05:20:13.417 INFO:teuthology.orchestra.run.smithi083.stdout:rbd-mirror-16.2.10-932.g966bd07c.el8.x86_64.rpm 9.7 MB/s | 3.7 MB 00:00 2022-09-29T05:20:13.417 INFO:teuthology.orchestra.run.smithi083.stdout:-------------------------------------------------------------------------------- 2022-09-29T05:20:13.418 INFO:teuthology.orchestra.run.smithi083.stdout:Total 9.6 MB/s | 3.7 MB 00:00 2022-09-29T05:20:13.418 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction check 2022-09-29T05:20:13.430 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction check succeeded. 2022-09-29T05:20:13.430 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction test 2022-09-29T05:20:13.493 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction test succeeded. 2022-09-29T05:20:13.496 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction 2022-09-29T05:20:13.546 DEBUG:teuthology.parallel:result is None 2022-09-29T05:20:13.926 INFO:teuthology.orchestra.run.smithi083.stdout: Preparing : 1/1 2022-09-29T05:20:14.093 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : rbd-mirror-2:16.2.10-932.g966bd07c.el8.x86_64 1/1 2022-09-29T05:20:14.473 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: rbd-mirror-2:16.2.10-932.g966bd07c.el8.x86_64 1/1 2022-09-29T05:20:14.913 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : rbd-mirror-2:16.2.10-932.g966bd07c.el8.x86_64 1/1 2022-09-29T05:20:14.913 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:20:14.914 INFO:teuthology.orchestra.run.smithi083.stdout:Installed: 2022-09-29T05:20:14.914 INFO:teuthology.orchestra.run.smithi083.stdout: rbd-mirror-2:16.2.10-932.g966bd07c.el8.x86_64 2022-09-29T05:20:14.914 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:20:14.915 INFO:teuthology.orchestra.run.smithi083.stdout:Complete! 2022-09-29T05:20:15.115 DEBUG:teuthology.orchestra.run.smithi083:> sudo yum -y install rbd-nbd 2022-09-29T05:20:15.695 INFO:teuthology.orchestra.run.smithi083.stdout:Last metadata expiration check: 0:02:25 ago on Thu 29 Sep 2022 05:17:50 AM UTC. 2022-09-29T05:20:16.772 INFO:teuthology.orchestra.run.smithi083.stdout:Dependencies resolved. 2022-09-29T05:20:16.773 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2022-09-29T05:20:16.773 INFO:teuthology.orchestra.run.smithi083.stdout: Package Architecture Version Repository Size 2022-09-29T05:20:16.773 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2022-09-29T05:20:16.774 INFO:teuthology.orchestra.run.smithi083.stdout:Installing: 2022-09-29T05:20:16.774 INFO:teuthology.orchestra.run.smithi083.stdout: rbd-nbd x86_64 2:16.2.10-932.g966bd07c.el8 ceph 173 k 2022-09-29T05:20:16.774 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:20:16.774 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction Summary 2022-09-29T05:20:16.775 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2022-09-29T05:20:16.775 INFO:teuthology.orchestra.run.smithi083.stdout:Install 1 Package 2022-09-29T05:20:16.775 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:20:16.775 INFO:teuthology.orchestra.run.smithi083.stdout:Total download size: 173 k 2022-09-29T05:20:16.776 INFO:teuthology.orchestra.run.smithi083.stdout:Installed size: 515 k 2022-09-29T05:20:16.776 INFO:teuthology.orchestra.run.smithi083.stdout:Downloading Packages: 2022-09-29T05:20:16.962 INFO:teuthology.orchestra.run.smithi083.stdout:rbd-nbd-16.2.10-932.g966bd07c.el8.x86_64.rpm 920 kB/s | 173 kB 00:00 2022-09-29T05:20:16.963 INFO:teuthology.orchestra.run.smithi083.stdout:-------------------------------------------------------------------------------- 2022-09-29T05:20:16.963 INFO:teuthology.orchestra.run.smithi083.stdout:Total 914 kB/s | 173 kB 00:00 2022-09-29T05:20:16.964 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction check 2022-09-29T05:20:16.973 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction check succeeded. 2022-09-29T05:20:16.974 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction test 2022-09-29T05:20:17.023 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction test succeeded. 2022-09-29T05:20:17.026 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction 2022-09-29T05:20:17.151 INFO:teuthology.orchestra.run.smithi083.stdout: Preparing : 1/1 2022-09-29T05:20:17.357 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : rbd-nbd-2:16.2.10-932.g966bd07c.el8.x86_64 1/1 2022-09-29T05:20:17.614 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: rbd-nbd-2:16.2.10-932.g966bd07c.el8.x86_64 1/1 2022-09-29T05:20:18.020 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : rbd-nbd-2:16.2.10-932.g966bd07c.el8.x86_64 1/1 2022-09-29T05:20:18.020 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:20:18.020 INFO:teuthology.orchestra.run.smithi083.stdout:Installed: 2022-09-29T05:20:18.020 INFO:teuthology.orchestra.run.smithi083.stdout: rbd-nbd-2:16.2.10-932.g966bd07c.el8.x86_64 2022-09-29T05:20:18.021 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:20:18.021 INFO:teuthology.orchestra.run.smithi083.stdout:Complete! 2022-09-29T05:20:18.632 DEBUG:teuthology.parallel:result is None 2022-09-29T05:20:18.633 DEBUG:teuthology.orchestra.run.smithi083:> sudo lsb_release -is 2022-09-29T05:20:18.670 INFO:teuthology.orchestra.run.smithi083.stdout:CentOSStream 2022-09-29T05:20:18.672 DEBUG:teuthology.misc:System to be installed: CentOSStream 2022-09-29T05:20:18.673 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2022-09-29T05:20:18.673 INFO:teuthology.packaging:ref: None 2022-09-29T05:20:18.673 INFO:teuthology.packaging:tag: None 2022-09-29T05:20:18.674 INFO:teuthology.packaging:branch: pacific 2022-09-29T05:20:18.674 INFO:teuthology.packaging:sha1: 652bf75409c45c34ae939b241b00adba20f12869 2022-09-29T05:20:18.675 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=pacific 2022-09-29T05:20:18.901 DEBUG:teuthology.orchestra.run.smithi083:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2022-09-29T05:20:18.923 INFO:teuthology.orchestra.run.smithi083.stdout:16.2.10-932.g966bd07c.el8 2022-09-29T05:20:18.924 INFO:teuthology.packaging:The installed version of ceph is 16.2.10-932.g966bd07c.el8 2022-09-29T05:20:18.925 INFO:teuthology.task.install:The correct ceph version 16.2.10-932.g966bd07c is installed. 2022-09-29T05:20:18.925 DEBUG:teuthology.orchestra.run.smithi153:> sudo lsb_release -is 2022-09-29T05:20:18.964 INFO:teuthology.orchestra.run.smithi153.stdout:CentOSStream 2022-09-29T05:20:18.965 DEBUG:teuthology.misc:System to be installed: CentOSStream 2022-09-29T05:20:18.966 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2022-09-29T05:20:18.966 INFO:teuthology.packaging:ref: None 2022-09-29T05:20:18.967 INFO:teuthology.packaging:tag: None 2022-09-29T05:20:18.967 INFO:teuthology.packaging:branch: pacific 2022-09-29T05:20:18.967 INFO:teuthology.packaging:sha1: 652bf75409c45c34ae939b241b00adba20f12869 2022-09-29T05:20:18.968 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=pacific 2022-09-29T05:20:19.189 DEBUG:teuthology.orchestra.run.smithi153:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2022-09-29T05:20:19.212 INFO:teuthology.orchestra.run.smithi153.stdout:16.2.10-932.g966bd07c.el8 2022-09-29T05:20:19.212 INFO:teuthology.packaging:The installed version of ceph is 16.2.10-932.g966bd07c.el8 2022-09-29T05:20:19.213 INFO:teuthology.task.install:The correct ceph version 16.2.10-932.g966bd07c is installed. 2022-09-29T05:20:19.213 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2022-09-29T05:20:19.214 DEBUG:teuthology.orchestra.run.smithi083:> set -ex 2022-09-29T05:20:19.214 DEBUG:teuthology.orchestra.run.smithi083:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2022-09-29T05:20:19.242 DEBUG:teuthology.orchestra.run.smithi153:> set -ex 2022-09-29T05:20:19.242 DEBUG:teuthology.orchestra.run.smithi153:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2022-09-29T05:20:19.279 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2022-09-29T05:20:19.280 DEBUG:teuthology.orchestra.run.smithi083:> set -ex 2022-09-29T05:20:19.280 DEBUG:teuthology.orchestra.run.smithi083:> sudo dd of=/usr/bin/daemon-helper 2022-09-29T05:20:19.309 DEBUG:teuthology.orchestra.run.smithi083:> sudo chmod a=rx -- /usr/bin/daemon-helper 2022-09-29T05:20:19.376 DEBUG:teuthology.orchestra.run.smithi153:> set -ex 2022-09-29T05:20:19.376 DEBUG:teuthology.orchestra.run.smithi153:> sudo dd of=/usr/bin/daemon-helper 2022-09-29T05:20:19.403 DEBUG:teuthology.orchestra.run.smithi153:> sudo chmod a=rx -- /usr/bin/daemon-helper 2022-09-29T05:20:19.469 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2022-09-29T05:20:19.469 DEBUG:teuthology.orchestra.run.smithi083:> set -ex 2022-09-29T05:20:19.470 DEBUG:teuthology.orchestra.run.smithi083:> sudo dd of=/usr/bin/adjust-ulimits 2022-09-29T05:20:19.496 DEBUG:teuthology.orchestra.run.smithi083:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2022-09-29T05:20:19.563 DEBUG:teuthology.orchestra.run.smithi153:> set -ex 2022-09-29T05:20:19.563 DEBUG:teuthology.orchestra.run.smithi153:> sudo dd of=/usr/bin/adjust-ulimits 2022-09-29T05:20:19.590 DEBUG:teuthology.orchestra.run.smithi153:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2022-09-29T05:20:19.656 INFO:teuthology.run_tasks:Running task print... 2022-09-29T05:20:19.667 INFO:teuthology.task.print:**** done install task... 2022-09-29T05:20:19.668 INFO:teuthology.run_tasks:Running task print... 2022-09-29T05:20:19.689 INFO:teuthology.task.print:**** done start installing pacific cephadm ... 2022-09-29T05:20:19.689 INFO:teuthology.run_tasks:Running task cephadm... 2022-09-29T05:20:19.766 INFO:tasks.cephadm:Config: {'cephadm_branch': 'pacific', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'conf': {'osd': {'osd_class_default_list': '*', 'osd_class_load_list': '*', 'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}}, 'image': 'quay.io/ceph/daemon-base:latest-pacific', 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': '652bf75409c45c34ae939b241b00adba20f12869'} 2022-09-29T05:20:19.766 INFO:tasks.cephadm:Cluster image is quay.io/ceph/daemon-base:latest-pacific 2022-09-29T05:20:19.767 INFO:tasks.cephadm:Cluster fsid is 69111522-3fb6-11ed-8431-001a4aab830c 2022-09-29T05:20:19.768 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2022-09-29T05:20:19.768 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.83', 'mon.c': '[v2:172.21.15.83:3301,v1:172.21.15.83:6790]', 'mon.b': '172.21.15.153'} 2022-09-29T05:20:19.769 INFO:tasks.cephadm:First mon is mon.a on smithi083 2022-09-29T05:20:19.769 INFO:tasks.cephadm:First mgr is y 2022-09-29T05:20:19.769 INFO:tasks.cephadm:Normalizing hostnames... 2022-09-29T05:20:19.770 DEBUG:teuthology.orchestra.run.smithi083:> sudo hostname $(hostname -s) 2022-09-29T05:20:19.797 DEBUG:teuthology.orchestra.run.smithi153:> sudo hostname $(hostname -s) 2022-09-29T05:20:19.825 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref pacific)... 2022-09-29T05:20:19.825 DEBUG:teuthology.orchestra.run.smithi083:> curl --silent https://raw.githubusercontent.com/ceph/ceph/pacific/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2022-09-29T05:20:20.083 INFO:teuthology.orchestra.run.smithi083.stdout:-rw-rw-r--. 1 ubuntu ubuntu 343445 Sep 29 05:20 /home/ubuntu/cephtest/cephadm 2022-09-29T05:20:20.083 DEBUG:teuthology.orchestra.run.smithi153:> curl --silent https://raw.githubusercontent.com/ceph/ceph/pacific/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2022-09-29T05:20:20.228 INFO:teuthology.orchestra.run.smithi153.stdout:-rw-rw-r--. 1 ubuntu ubuntu 343445 Sep 29 05:20 /home/ubuntu/cephtest/cephadm 2022-09-29T05:20:20.229 DEBUG:teuthology.orchestra.run.smithi083:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2022-09-29T05:20:20.248 DEBUG:teuthology.orchestra.run.smithi153:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2022-09-29T05:20:20.278 INFO:tasks.cephadm:Pulling image quay.io/ceph/daemon-base:latest-pacific on all hosts... 2022-09-29T05:20:20.278 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/daemon-base:latest-pacific pull 2022-09-29T05:20:20.291 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/daemon-base:latest-pacific pull 2022-09-29T05:20:21.666 INFO:teuthology.orchestra.run.smithi153.stderr:Pulling container image quay.io/ceph/daemon-base:latest-pacific... 2022-09-29T05:20:21.741 INFO:teuthology.orchestra.run.smithi083.stderr:Pulling container image quay.io/ceph/daemon-base:latest-pacific... 2022-09-29T05:20:36.014 INFO:teuthology.orchestra.run.smithi153.stdout:{ 2022-09-29T05:20:36.015 INFO:teuthology.orchestra.run.smithi153.stdout: "ceph_version": "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)", 2022-09-29T05:20:36.015 INFO:teuthology.orchestra.run.smithi153.stdout: "image_id": "c56e13ff90f29390179b843381d9c68e2c904b63a5e0e0391586709c63efc8f5", 2022-09-29T05:20:36.015 INFO:teuthology.orchestra.run.smithi153.stdout: "repo_digests": [ 2022-09-29T05:20:36.016 INFO:teuthology.orchestra.run.smithi153.stdout: "quay.io/ceph/daemon-base@sha256:fd28d170be2b3bb27a7afd5b7cee14cb38c8e29be31f32256f0687a2c108d800" 2022-09-29T05:20:36.016 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2022-09-29T05:20:36.016 INFO:teuthology.orchestra.run.smithi153.stdout:} 2022-09-29T05:20:36.206 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T05:20:36.206 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph_version": "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)", 2022-09-29T05:20:36.206 INFO:teuthology.orchestra.run.smithi083.stdout: "image_id": "c56e13ff90f29390179b843381d9c68e2c904b63a5e0e0391586709c63efc8f5", 2022-09-29T05:20:36.207 INFO:teuthology.orchestra.run.smithi083.stdout: "repo_digests": [ 2022-09-29T05:20:36.207 INFO:teuthology.orchestra.run.smithi083.stdout: "quay.io/ceph/daemon-base@sha256:fd28d170be2b3bb27a7afd5b7cee14cb38c8e29be31f32256f0687a2c108d800" 2022-09-29T05:20:36.207 INFO:teuthology.orchestra.run.smithi083.stdout: ] 2022-09-29T05:20:36.207 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T05:20:36.225 DEBUG:teuthology.orchestra.run.smithi083:> sudo mkdir -p /etc/ceph 2022-09-29T05:20:36.256 DEBUG:teuthology.orchestra.run.smithi153:> sudo mkdir -p /etc/ceph 2022-09-29T05:20:36.288 DEBUG:teuthology.orchestra.run.smithi083:> sudo chmod 777 /etc/ceph 2022-09-29T05:20:36.323 DEBUG:teuthology.orchestra.run.smithi153:> sudo chmod 777 /etc/ceph 2022-09-29T05:20:36.356 INFO:tasks.cephadm:Writing seed config... 2022-09-29T05:20:36.358 INFO:tasks.cephadm: override: [osd] osd_class_default_list = * 2022-09-29T05:20:36.358 INFO:tasks.cephadm: override: [osd] osd_class_load_list = * 2022-09-29T05:20:36.358 INFO:tasks.cephadm: override: [osd] debug ms = 1 2022-09-29T05:20:36.359 INFO:tasks.cephadm: override: [osd] debug osd = 20 2022-09-29T05:20:36.359 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2022-09-29T05:20:36.359 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2022-09-29T05:20:36.360 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2022-09-29T05:20:36.360 INFO:tasks.cephadm: override: [mon] debug mon = 20 2022-09-29T05:20:36.360 INFO:tasks.cephadm: override: [mon] debug ms = 1 2022-09-29T05:20:36.361 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2022-09-29T05:20:36.361 DEBUG:teuthology.orchestra.run.smithi083:> set -ex 2022-09-29T05:20:36.362 DEBUG:teuthology.orchestra.run.smithi083:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2022-09-29T05:20:36.381 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = 69111522-3fb6-11ed-8431-001a4aab830c [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true osd_class_default_list = * osd_class_load_list = * debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2022-09-29T05:20:36.382 DEBUG:teuthology.orchestra.run.smithi083:mon.a> sudo journalctl -f -n 0 -u ceph-69111522-3fb6-11ed-8431-001a4aab830c@mon.a.service 2022-09-29T05:20:36.424 DEBUG:teuthology.orchestra.run.smithi083:mgr.y> sudo journalctl -f -n 0 -u ceph-69111522-3fb6-11ed-8431-001a4aab830c@mgr.y.service 2022-09-29T05:20:36.450 INFO:journalctl@ceph.mon.a.smithi083.stdout:-- Logs begin at Thu 2022-09-29 05:12:45 UTC. -- 2022-09-29T05:20:36.451 INFO:tasks.cephadm:Bootstrapping... 2022-09-29T05:20:36.451 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/daemon-base:latest-pacific -v bootstrap --fsid 69111522-3fb6-11ed-8431-001a4aab830c --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.83 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2022-09-29T05:20:36.476 INFO:journalctl@ceph.mgr.y.smithi083.stdout:-- Logs begin at Thu 2022-09-29 05:12:45 UTC. -- 2022-09-29T05:20:36.649 INFO:teuthology.orchestra.run.smithi083.stderr:-------------------------------------------------------------------------------- 2022-09-29T05:20:36.650 INFO:teuthology.orchestra.run.smithi083.stderr:cephadm ['--image', 'quay.io/ceph/daemon-base:latest-pacific', '-v', 'bootstrap', '--fsid', '69111522-3fb6-11ed-8431-001a4aab830c', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.83', '--skip-admin-label'] 2022-09-29T05:20:36.680 INFO:teuthology.orchestra.run.smithi083.stderr:/bin/podman: 4.2.0 2022-09-29T05:20:36.693 INFO:teuthology.orchestra.run.smithi083.stderr:Verifying podman|docker is present... 2022-09-29T05:20:36.723 INFO:teuthology.orchestra.run.smithi083.stderr:/bin/podman: 4.2.0 2022-09-29T05:20:36.830 INFO:teuthology.orchestra.run.smithi083.stderr:Verifying lvm2 is present... 2022-09-29T05:20:36.830 INFO:teuthology.orchestra.run.smithi083.stderr:Verifying time synchronization is in place... 2022-09-29T05:20:36.836 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2022-09-29T05:20:36.842 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: inactive 2022-09-29T05:20:36.848 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: enabled 2022-09-29T05:20:36.854 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: active 2022-09-29T05:20:36.855 INFO:teuthology.orchestra.run.smithi083.stderr:Unit chronyd.service is enabled and running 2022-09-29T05:20:36.855 INFO:teuthology.orchestra.run.smithi083.stderr:Repeating the final host check... 2022-09-29T05:20:36.883 INFO:teuthology.orchestra.run.smithi083.stderr:/bin/podman: 4.2.0 2022-09-29T05:20:36.994 INFO:teuthology.orchestra.run.smithi083.stderr:podman (/bin/podman) version 4.2.0 is present 2022-09-29T05:20:36.994 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl is present 2022-09-29T05:20:36.995 INFO:teuthology.orchestra.run.smithi083.stderr:lvcreate is present 2022-09-29T05:20:37.000 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2022-09-29T05:20:37.005 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: inactive 2022-09-29T05:20:37.011 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: enabled 2022-09-29T05:20:37.016 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: active 2022-09-29T05:20:37.017 INFO:teuthology.orchestra.run.smithi083.stderr:Unit chronyd.service is enabled and running 2022-09-29T05:20:37.018 INFO:teuthology.orchestra.run.smithi083.stderr:Host looks OK 2022-09-29T05:20:37.018 INFO:teuthology.orchestra.run.smithi083.stderr:Cluster fsid: 69111522-3fb6-11ed-8431-001a4aab830c 2022-09-29T05:20:37.019 INFO:teuthology.orchestra.run.smithi083.stderr:Acquiring lock 139937352341992 on /run/cephadm/69111522-3fb6-11ed-8431-001a4aab830c.lock 2022-09-29T05:20:37.019 INFO:teuthology.orchestra.run.smithi083.stderr:Lock 139937352341992 acquired on /run/cephadm/69111522-3fb6-11ed-8431-001a4aab830c.lock 2022-09-29T05:20:37.019 INFO:teuthology.orchestra.run.smithi083.stderr:Verifying IP 172.21.15.83 port 3300 ... 2022-09-29T05:20:37.020 INFO:teuthology.orchestra.run.smithi083.stderr:Verifying IP 172.21.15.83 port 6789 ... 2022-09-29T05:20:37.020 INFO:teuthology.orchestra.run.smithi083.stderr:Base mon IP(s) is [172.21.15.83:3300, 172.21.15.83:6789], mon addrv is [v2:172.21.15.83:3300,v1:172.21.15.83:6789] 2022-09-29T05:20:37.021 INFO:teuthology.orchestra.run.smithi083.stderr:/sbin/ip: default via 172.21.15.254 dev enp3s0f1 proto dhcp src 172.21.15.83 metric 100 2022-09-29T05:20:37.022 INFO:teuthology.orchestra.run.smithi083.stderr:/sbin/ip: 172.21.0.0/20 dev enp3s0f1 proto kernel scope link src 172.21.15.83 metric 100 2022-09-29T05:20:37.024 INFO:teuthology.orchestra.run.smithi083.stderr:/sbin/ip: ::1 dev lo proto kernel metric 256 pref medium 2022-09-29T05:20:37.024 INFO:teuthology.orchestra.run.smithi083.stderr:/sbin/ip: fe80::/64 dev enp3s0f1 proto kernel metric 256 pref medium 2022-09-29T05:20:37.025 INFO:teuthology.orchestra.run.smithi083.stderr:/sbin/ip: default via fe80::327c:5e00:6487:71e0 dev enp3s0f1 proto ra metric 1024 expires 1767sec hoplimit 64 pref medium 2022-09-29T05:20:37.027 INFO:teuthology.orchestra.run.smithi083.stderr:/sbin/ip: 1: lo: mtu 65536 state UNKNOWN qlen 1000 2022-09-29T05:20:37.027 INFO:teuthology.orchestra.run.smithi083.stderr:/sbin/ip: inet6 ::1/128 scope host 2022-09-29T05:20:37.027 INFO:teuthology.orchestra.run.smithi083.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2022-09-29T05:20:37.027 INFO:teuthology.orchestra.run.smithi083.stderr:/sbin/ip: 5: enp3s0f1: mtu 1500 state UP qlen 1000 2022-09-29T05:20:37.028 INFO:teuthology.orchestra.run.smithi083.stderr:/sbin/ip: inet6 fe80::ec4:7aff:fe8f:ce91/64 scope link 2022-09-29T05:20:37.028 INFO:teuthology.orchestra.run.smithi083.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2022-09-29T05:20:37.029 INFO:teuthology.orchestra.run.smithi083.stderr:Mon IP `172.21.15.83` is in CIDR network `172.21.0.0/20` 2022-09-29T05:20:37.029 INFO:teuthology.orchestra.run.smithi083.stderr:Mon IP `172.21.15.83` is in CIDR network `172.21.0.0/20` 2022-09-29T05:20:37.029 INFO:teuthology.orchestra.run.smithi083.stderr:Inferred mon public CIDR from local network configuration ['172.21.0.0/20', '172.21.0.0/20'] 2022-09-29T05:20:37.030 INFO:teuthology.orchestra.run.smithi083.stderr:Internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2022-09-29T05:20:37.030 INFO:teuthology.orchestra.run.smithi083.stderr:Pulling container image quay.io/ceph/daemon-base:latest-pacific... 2022-09-29T05:20:37.116 INFO:teuthology.orchestra.run.smithi083.stderr:/bin/podman: Trying to pull quay.io/ceph/daemon-base:latest-pacific... 2022-09-29T05:20:37.479 INFO:teuthology.orchestra.run.smithi083.stderr:/bin/podman: Getting image source signatures 2022-09-29T05:20:37.479 INFO:teuthology.orchestra.run.smithi083.stderr:/bin/podman: Copying blob sha256:a093434cc8ca9e9fc32d0e8f38a37a620c22f22487f6e7cf63499e77957a861b 2022-09-29T05:20:37.479 INFO:teuthology.orchestra.run.smithi083.stderr:/bin/podman: Copying blob sha256:bc30081ecdfbe9983f51216360c0f2a39b6f82a41e0a82717157d0f2d116ea10 2022-09-29T05:20:37.480 INFO:teuthology.orchestra.run.smithi083.stderr:/bin/podman: Copying blob sha256:9bdecea3fbaa2ecc085daa5a2a5651dc4ff5c6cbbd30cf90d922b5e7e247ebbb 2022-09-29T05:20:37.480 INFO:teuthology.orchestra.run.smithi083.stderr:/bin/podman: Copying blob sha256:a05dea350572433af71ec016d378a13d5ab49308ddb6346daf8fcf95e20f3c45 2022-09-29T05:20:37.480 INFO:teuthology.orchestra.run.smithi083.stderr:/bin/podman: Copying blob sha256:632957bc3aa15aa8e77742ddc0525255559e18bf130948cb02a586049652b604 2022-09-29T05:20:37.480 INFO:teuthology.orchestra.run.smithi083.stderr:/bin/podman: Copying config sha256:c56e13ff90f29390179b843381d9c68e2c904b63a5e0e0391586709c63efc8f5 2022-09-29T05:20:37.530 INFO:teuthology.orchestra.run.smithi083.stderr:/bin/podman: Writing manifest to image destination 2022-09-29T05:20:37.530 INFO:teuthology.orchestra.run.smithi083.stderr:/bin/podman: Storing signatures 2022-09-29T05:20:37.693 INFO:teuthology.orchestra.run.smithi083.stderr:/bin/podman: c56e13ff90f29390179b843381d9c68e2c904b63a5e0e0391586709c63efc8f5 2022-09-29T05:20:38.273 INFO:teuthology.orchestra.run.smithi083.stderr:ceph: ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable) 2022-09-29T05:20:38.716 INFO:teuthology.orchestra.run.smithi083.stderr:Ceph version: ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable) 2022-09-29T05:20:38.717 INFO:teuthology.orchestra.run.smithi083.stderr:Extracting ceph user uid/gid from container image... 2022-09-29T05:20:39.127 INFO:teuthology.orchestra.run.smithi083.stderr:stat: 167 167 2022-09-29T05:20:39.562 INFO:teuthology.orchestra.run.smithi083.stderr:Creating initial keys... 2022-09-29T05:20:39.955 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-authtool: AQAnKzVj1ZHiNxAAw95N2tKkUuXcAjblY2i1sA== 2022-09-29T05:20:40.655 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-authtool: AQAoKzVjy5BSJhAAvZRVgkvIZDsaNHsvZNV2Jw== 2022-09-29T05:20:41.438 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-authtool: AQApKzVjTA2PGRAAb17PngYbaa30jSRVYGs9IQ== 2022-09-29T05:20:41.892 INFO:teuthology.orchestra.run.smithi083.stderr:Creating initial monmap... 2022-09-29T05:20:42.330 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: monmap file /tmp/monmap 2022-09-29T05:20:42.330 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: set fsid to 69111522-3fb6-11ed-8431-001a4aab830c 2022-09-29T05:20:42.330 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2022-09-29T05:20:42.833 INFO:teuthology.orchestra.run.smithi083.stderr:monmaptool for a [v2:172.21.15.83:3300,v1:172.21.15.83:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2022-09-29T05:20:42.834 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/monmaptool: set fsid to 69111522-3fb6-11ed-8431-001a4aab830c 2022-09-29T05:20:42.834 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2022-09-29T05:20:42.834 INFO:teuthology.orchestra.run.smithi083.stderr: 2022-09-29T05:20:42.835 INFO:teuthology.orchestra.run.smithi083.stderr:Creating mon... 2022-09-29T05:20:43.360 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.287+0000 7f5b5050a880 0 set uid:gid to 167:167 (ceph:ceph) 2022-09-29T05:20:43.360 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.287+0000 7f5b5050a880 1 imported monmap: 2022-09-29T05:20:43.361 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: epoch 0 2022-09-29T05:20:43.361 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: fsid 69111522-3fb6-11ed-8431-001a4aab830c 2022-09-29T05:20:43.361 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: last_changed 2022-09-29T05:20:42.317979+0000 2022-09-29T05:20:43.361 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: created 2022-09-29T05:20:42.317979+0000 2022-09-29T05:20:43.362 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: min_mon_release 0 (unknown) 2022-09-29T05:20:43.362 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: election_strategy: 1 2022-09-29T05:20:43.362 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: 0: [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] mon.a 2022-09-29T05:20:43.362 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: 2022-09-29T05:20:43.363 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.287+0000 7f5b5050a880 0 /usr/bin/ceph-mon: set fsid to 69111522-3fb6-11ed-8431-001a4aab830c 2022-09-29T05:20:43.363 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: RocksDB version: 6.8.1 2022-09-29T05:20:43.363 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: 2022-09-29T05:20:43.363 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2022-09-29T05:20:43.363 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: Compile date Jul 21 2022 2022-09-29T05:20:43.364 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: DB SUMMARY 2022-09-29T05:20:43.364 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: 2022-09-29T05:20:43.364 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 0, files: 2022-09-29T05:20:43.364 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: 2022-09-29T05:20:43.365 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 2022-09-29T05:20:43.365 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: 2022-09-29T05:20:43.365 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: Options.error_if_exists: 0 2022-09-29T05:20:43.365 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: Options.create_if_missing: 1 2022-09-29T05:20:43.365 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: Options.paranoid_checks: 1 2022-09-29T05:20:43.366 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: Options.env: 0x556389709000 2022-09-29T05:20:43.366 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: Options.fs: Posix File System 2022-09-29T05:20:43.366 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: Options.info_log: 0x55638b95b200 2022-09-29T05:20:43.366 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: Options.max_file_opening_threads: 16 2022-09-29T05:20:43.367 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: Options.statistics: (nil) 2022-09-29T05:20:43.367 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: Options.use_fsync: 0 2022-09-29T05:20:43.367 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: Options.max_log_file_size: 0 2022-09-29T05:20:43.367 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: Options.max_manifest_file_size: 1073741824 2022-09-29T05:20:43.367 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: Options.log_file_time_to_roll: 0 2022-09-29T05:20:43.368 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: Options.keep_log_file_num: 1000 2022-09-29T05:20:43.368 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: Options.recycle_log_file_num: 0 2022-09-29T05:20:43.368 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: Options.allow_fallocate: 1 2022-09-29T05:20:43.368 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: Options.allow_mmap_reads: 0 2022-09-29T05:20:43.369 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: Options.allow_mmap_writes: 0 2022-09-29T05:20:43.369 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: Options.use_direct_reads: 0 2022-09-29T05:20:43.369 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2022-09-29T05:20:43.369 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: Options.create_missing_column_families: 0 2022-09-29T05:20:43.370 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: Options.db_log_dir: 2022-09-29T05:20:43.370 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2022-09-29T05:20:43.370 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: Options.table_cache_numshardbits: 6 2022-09-29T05:20:43.370 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: Options.max_subcompactions: 1 2022-09-29T05:20:43.371 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: Options.max_background_flushes: -1 2022-09-29T05:20:43.371 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: Options.WAL_ttl_seconds: 0 2022-09-29T05:20:43.371 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: Options.WAL_size_limit_MB: 0 2022-09-29T05:20:43.371 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2022-09-29T05:20:43.371 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: Options.manifest_preallocation_size: 4194304 2022-09-29T05:20:43.372 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: Options.is_fd_close_on_exec: 1 2022-09-29T05:20:43.372 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: Options.advise_random_on_open: 1 2022-09-29T05:20:43.372 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: Options.db_write_buffer_size: 0 2022-09-29T05:20:43.372 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: Options.write_buffer_manager: 0x55638b963650 2022-09-29T05:20:43.373 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: Options.access_hint_on_compaction_start: 1 2022-09-29T05:20:43.373 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2022-09-29T05:20:43.373 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2022-09-29T05:20:43.373 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: Options.use_adaptive_mutex: 0 2022-09-29T05:20:43.373 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: Options.rate_limiter: (nil) 2022-09-29T05:20:43.374 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2022-09-29T05:20:43.374 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: Options.wal_recovery_mode: 2 2022-09-29T05:20:43.374 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: Options.enable_thread_tracking: 0 2022-09-29T05:20:43.374 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: Options.enable_pipelined_write: 0 2022-09-29T05:20:43.375 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: Options.unordered_write: 0 2022-09-29T05:20:43.375 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2022-09-29T05:20:43.375 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2022-09-29T05:20:43.375 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: Options.write_thread_max_yield_usec: 100 2022-09-29T05:20:43.375 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2022-09-29T05:20:43.376 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: Options.row_cache: None 2022-09-29T05:20:43.376 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: Options.wal_filter: None 2022-09-29T05:20:43.376 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: Options.avoid_flush_during_recovery: 0 2022-09-29T05:20:43.376 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: Options.allow_ingest_behind: 0 2022-09-29T05:20:43.376 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: Options.preserve_deletes: 0 2022-09-29T05:20:43.377 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: Options.two_write_queues: 0 2022-09-29T05:20:43.377 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: Options.manual_wal_flush: 0 2022-09-29T05:20:43.377 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: Options.atomic_flush: 0 2022-09-29T05:20:43.377 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2022-09-29T05:20:43.378 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: Options.persist_stats_to_disk: 0 2022-09-29T05:20:43.378 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: Options.write_dbid_to_manifest: 0 2022-09-29T05:20:43.378 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: Options.log_readahead_size: 0 2022-09-29T05:20:43.378 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: Options.sst_file_checksum_func: Unknown 2022-09-29T05:20:43.378 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: Options.max_background_jobs: 2 2022-09-29T05:20:43.379 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: Options.max_background_compactions: -1 2022-09-29T05:20:43.379 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2022-09-29T05:20:43.379 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2022-09-29T05:20:43.379 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: Options.delayed_write_rate : 16777216 2022-09-29T05:20:43.380 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: Options.max_total_wal_size: 0 2022-09-29T05:20:43.380 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2022-09-29T05:20:43.380 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: Options.stats_dump_period_sec: 600 2022-09-29T05:20:43.380 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: Options.stats_persist_period_sec: 600 2022-09-29T05:20:43.381 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: Options.stats_history_buffer_size: 1048576 2022-09-29T05:20:43.381 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: Options.max_open_files: -1 2022-09-29T05:20:43.381 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: Options.bytes_per_sync: 0 2022-09-29T05:20:43.381 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: Options.wal_bytes_per_sync: 0 2022-09-29T05:20:43.381 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: Options.strict_bytes_per_sync: 0 2022-09-29T05:20:43.382 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: Options.compaction_readahead_size: 0 2022-09-29T05:20:43.382 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: Compression algorithms supported: 2022-09-29T05:20:43.382 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: kZSTDNotFinalCompression supported: 0 2022-09-29T05:20:43.382 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: kZSTD supported: 0 2022-09-29T05:20:43.383 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: kXpressCompression supported: 0 2022-09-29T05:20:43.383 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: kLZ4HCCompression supported: 1 2022-09-29T05:20:43.383 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: kLZ4Compression supported: 1 2022-09-29T05:20:43.383 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: kBZip2Compression supported: 0 2022-09-29T05:20:43.383 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: kZlibCompression supported: 1 2022-09-29T05:20:43.384 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: kSnappyCompression supported: 1 2022-09-29T05:20:43.384 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: Fast CRC32 supported: Supported on x86 2022-09-29T05:20:43.384 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.288+0000 7f5b5050a880 4 rocksdb: [db_impl/db_impl_open.cc:273] Creating manifest 1 2022-09-29T05:20:43.384 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: 2022-09-29T05:20:43.384 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.289+0000 7f5b5050a880 4 rocksdb: [version_set.cc:4413] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000001 2022-09-29T05:20:43.385 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: 2022-09-29T05:20:43.385 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.289+0000 7f5b5050a880 4 rocksdb: [column_family.cc:552] --------------- Options for column family [default]: 2022-09-29T05:20:43.385 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: 2022-09-29T05:20:43.385 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.289+0000 7f5b5050a880 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2022-09-29T05:20:43.386 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.289+0000 7f5b5050a880 4 rocksdb: Options.merge_operator: 2022-09-29T05:20:43.386 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.289+0000 7f5b5050a880 4 rocksdb: Options.compaction_filter: None 2022-09-29T05:20:43.386 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.289+0000 7f5b5050a880 4 rocksdb: Options.compaction_filter_factory: None 2022-09-29T05:20:43.386 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.289+0000 7f5b5050a880 4 rocksdb: Options.memtable_factory: SkipListFactory 2022-09-29T05:20:43.386 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.289+0000 7f5b5050a880 4 rocksdb: Options.table_factory: BlockBasedTable 2022-09-29T05:20:43.387 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.289+0000 7f5b5050a880 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55638b86dd20) 2022-09-29T05:20:43.387 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: cache_index_and_filter_blocks: 1 2022-09-29T05:20:43.387 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: cache_index_and_filter_blocks_with_high_priority: 0 2022-09-29T05:20:43.387 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: pin_l0_filter_and_index_blocks_in_cache: 0 2022-09-29T05:20:43.388 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: pin_top_level_index_and_filter: 1 2022-09-29T05:20:43.388 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: index_type: 0 2022-09-29T05:20:43.388 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: data_block_index_type: 0 2022-09-29T05:20:43.388 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: index_shortening: 1 2022-09-29T05:20:43.388 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: data_block_hash_table_util_ratio: 0.750000 2022-09-29T05:20:43.389 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: hash_index_allow_collision: 1 2022-09-29T05:20:43.389 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: checksum: 1 2022-09-29T05:20:43.389 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: no_block_cache: 0 2022-09-29T05:20:43.389 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: block_cache: 0x55638b8a6d10 2022-09-29T05:20:43.390 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: block_cache_name: BinnedLRUCache 2022-09-29T05:20:43.390 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: block_cache_options: 2022-09-29T05:20:43.390 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: capacity : 536870912 2022-09-29T05:20:43.390 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: num_shard_bits : 4 2022-09-29T05:20:43.390 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: strict_capacity_limit : 0 2022-09-29T05:20:43.391 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: high_pri_pool_ratio: 0.000 2022-09-29T05:20:43.391 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: block_cache_compressed: (nil) 2022-09-29T05:20:43.391 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: persistent_cache: (nil) 2022-09-29T05:20:43.391 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: block_size: 4096 2022-09-29T05:20:43.391 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: block_size_deviation: 10 2022-09-29T05:20:43.392 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: block_restart_interval: 16 2022-09-29T05:20:43.392 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: index_block_restart_interval: 1 2022-09-29T05:20:43.392 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: metadata_block_size: 4096 2022-09-29T05:20:43.392 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: partition_filters: 0 2022-09-29T05:20:43.393 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: use_delta_encoding: 1 2022-09-29T05:20:43.393 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: filter_policy: rocksdb.BuiltinBloomFilter 2022-09-29T05:20:43.393 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: whole_key_filtering: 1 2022-09-29T05:20:43.393 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: verify_compression: 0 2022-09-29T05:20:43.393 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: read_amp_bytes_per_bit: 0 2022-09-29T05:20:43.394 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: format_version: 2 2022-09-29T05:20:43.394 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: enable_index_compression: 1 2022-09-29T05:20:43.394 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: block_align: 0 2022-09-29T05:20:43.394 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: 2022-09-29T05:20:43.395 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.289+0000 7f5b5050a880 4 rocksdb: Options.write_buffer_size: 33554432 2022-09-29T05:20:43.395 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.289+0000 7f5b5050a880 4 rocksdb: Options.max_write_buffer_number: 2 2022-09-29T05:20:43.395 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.289+0000 7f5b5050a880 4 rocksdb: Options.compression: NoCompression 2022-09-29T05:20:43.395 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.289+0000 7f5b5050a880 4 rocksdb: Options.bottommost_compression: Disabled 2022-09-29T05:20:43.395 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.289+0000 7f5b5050a880 4 rocksdb: Options.prefix_extractor: nullptr 2022-09-29T05:20:43.396 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.289+0000 7f5b5050a880 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2022-09-29T05:20:43.396 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.289+0000 7f5b5050a880 4 rocksdb: Options.num_levels: 7 2022-09-29T05:20:43.396 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.289+0000 7f5b5050a880 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2022-09-29T05:20:43.396 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.289+0000 7f5b5050a880 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2022-09-29T05:20:43.397 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.289+0000 7f5b5050a880 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2022-09-29T05:20:43.397 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.289+0000 7f5b5050a880 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2022-09-29T05:20:43.397 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.289+0000 7f5b5050a880 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2022-09-29T05:20:43.397 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.289+0000 7f5b5050a880 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2022-09-29T05:20:43.397 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.289+0000 7f5b5050a880 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2022-09-29T05:20:43.398 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.289+0000 7f5b5050a880 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2022-09-29T05:20:43.398 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.289+0000 7f5b5050a880 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2022-09-29T05:20:43.398 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.289+0000 7f5b5050a880 4 rocksdb: Options.compression_opts.window_bits: -14 2022-09-29T05:20:43.398 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.289+0000 7f5b5050a880 4 rocksdb: Options.compression_opts.level: 32767 2022-09-29T05:20:43.399 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.289+0000 7f5b5050a880 4 rocksdb: Options.compression_opts.strategy: 0 2022-09-29T05:20:43.399 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.289+0000 7f5b5050a880 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2022-09-29T05:20:43.399 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.289+0000 7f5b5050a880 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2022-09-29T05:20:43.399 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.289+0000 7f5b5050a880 4 rocksdb: Options.compression_opts.enabled: false 2022-09-29T05:20:43.399 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.289+0000 7f5b5050a880 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2022-09-29T05:20:43.400 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.289+0000 7f5b5050a880 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2022-09-29T05:20:43.400 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.289+0000 7f5b5050a880 4 rocksdb: Options.level0_stop_writes_trigger: 36 2022-09-29T05:20:43.400 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.289+0000 7f5b5050a880 4 rocksdb: Options.target_file_size_base: 67108864 2022-09-29T05:20:43.400 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.289+0000 7f5b5050a880 4 rocksdb: Options.target_file_size_multiplier: 1 2022-09-29T05:20:43.401 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.289+0000 7f5b5050a880 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2022-09-29T05:20:43.401 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.289+0000 7f5b5050a880 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2022-09-29T05:20:43.401 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.289+0000 7f5b5050a880 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2022-09-29T05:20:43.401 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.289+0000 7f5b5050a880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2022-09-29T05:20:43.401 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.289+0000 7f5b5050a880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2022-09-29T05:20:43.402 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.289+0000 7f5b5050a880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2022-09-29T05:20:43.402 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.289+0000 7f5b5050a880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2022-09-29T05:20:43.402 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.289+0000 7f5b5050a880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2022-09-29T05:20:43.402 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.289+0000 7f5b5050a880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2022-09-29T05:20:43.402 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.289+0000 7f5b5050a880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2022-09-29T05:20:43.403 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.289+0000 7f5b5050a880 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2022-09-29T05:20:43.403 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.289+0000 7f5b5050a880 4 rocksdb: Options.max_compaction_bytes: 1677721600 2022-09-29T05:20:43.403 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.289+0000 7f5b5050a880 4 rocksdb: Options.arena_block_size: 4194304 2022-09-29T05:20:43.403 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.289+0000 7f5b5050a880 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2022-09-29T05:20:43.404 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.289+0000 7f5b5050a880 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2022-09-29T05:20:43.404 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.289+0000 7f5b5050a880 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2022-09-29T05:20:43.404 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.289+0000 7f5b5050a880 4 rocksdb: Options.disable_auto_compactions: 0 2022-09-29T05:20:43.404 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.289+0000 7f5b5050a880 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2022-09-29T05:20:43.404 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.289+0000 7f5b5050a880 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2022-09-29T05:20:43.405 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.289+0000 7f5b5050a880 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2022-09-29T05:20:43.405 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.289+0000 7f5b5050a880 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2022-09-29T05:20:43.405 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.289+0000 7f5b5050a880 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2022-09-29T05:20:43.405 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.289+0000 7f5b5050a880 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2022-09-29T05:20:43.405 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.289+0000 7f5b5050a880 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2022-09-29T05:20:43.406 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.289+0000 7f5b5050a880 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2022-09-29T05:20:43.406 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.289+0000 7f5b5050a880 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2022-09-29T05:20:43.406 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.289+0000 7f5b5050a880 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2022-09-29T05:20:43.406 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.289+0000 7f5b5050a880 4 rocksdb: Options.table_properties_collectors: 2022-09-29T05:20:43.407 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.289+0000 7f5b5050a880 4 rocksdb: Options.inplace_update_support: 0 2022-09-29T05:20:43.407 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.289+0000 7f5b5050a880 4 rocksdb: Options.inplace_update_num_locks: 10000 2022-09-29T05:20:43.407 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.289+0000 7f5b5050a880 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2022-09-29T05:20:43.407 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.289+0000 7f5b5050a880 4 rocksdb: Options.memtable_whole_key_filtering: 0 2022-09-29T05:20:43.407 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.289+0000 7f5b5050a880 4 rocksdb: Options.memtable_huge_page_size: 0 2022-09-29T05:20:43.408 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.289+0000 7f5b5050a880 4 rocksdb: Options.bloom_locality: 0 2022-09-29T05:20:43.408 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.289+0000 7f5b5050a880 4 rocksdb: Options.max_successive_merges: 0 2022-09-29T05:20:43.408 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.289+0000 7f5b5050a880 4 rocksdb: Options.optimize_filters_for_hits: 0 2022-09-29T05:20:43.408 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.289+0000 7f5b5050a880 4 rocksdb: Options.paranoid_file_checks: 0 2022-09-29T05:20:43.408 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.289+0000 7f5b5050a880 4 rocksdb: Options.force_consistency_checks: 0 2022-09-29T05:20:43.409 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.289+0000 7f5b5050a880 4 rocksdb: Options.report_bg_io_stats: 0 2022-09-29T05:20:43.409 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.289+0000 7f5b5050a880 4 rocksdb: Options.ttl: 2592000 2022-09-29T05:20:43.409 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.289+0000 7f5b5050a880 4 rocksdb: Options.periodic_compaction_seconds: 0 2022-09-29T05:20:43.409 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.290+0000 7f5b5050a880 4 rocksdb: [version_set.cc:4568] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000001 succeeded,manifest_file_number is 1, next_file_number is 3, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2022-09-29T05:20:43.410 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: 2022-09-29T05:20:43.410 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.290+0000 7f5b5050a880 4 rocksdb: [version_set.cc:4577] Column family [default] (ID 0), log number is 0 2022-09-29T05:20:43.410 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: 2022-09-29T05:20:43.410 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.291+0000 7f5b5050a880 4 rocksdb: DB pointer 0x55638b971800 2022-09-29T05:20:43.410 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.291+0000 7f5b393a5700 4 rocksdb: [db_impl/db_impl.cc:850] ------- DUMPING STATS ------- 2022-09-29T05:20:43.411 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.291+0000 7f5b393a5700 4 rocksdb: [db_impl/db_impl.cc:851] 2022-09-29T05:20:43.411 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: ** DB Stats ** 2022-09-29T05:20:43.411 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: Uptime(secs): 0.0 total, 0.0 interval 2022-09-29T05:20:43.411 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2022-09-29T05:20:43.411 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2022-09-29T05:20:43.412 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2022-09-29T05:20:43.412 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2022-09-29T05:20:43.412 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2022-09-29T05:20:43.412 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2022-09-29T05:20:43.413 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: 2022-09-29T05:20:43.413 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: ** Compaction Stats [default] ** 2022-09-29T05:20:43.413 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2022-09-29T05:20:43.413 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2022-09-29T05:20:43.413 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: Sum 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2022-09-29T05:20:43.414 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2022-09-29T05:20:43.414 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: 2022-09-29T05:20:43.414 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: ** Compaction Stats [default] ** 2022-09-29T05:20:43.414 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2022-09-29T05:20:43.415 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2022-09-29T05:20:43.415 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: Uptime(secs): 0.0 total, 0.0 interval 2022-09-29T05:20:43.415 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: Flush(GB): cumulative 0.000, interval 0.000 2022-09-29T05:20:43.415 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: AddFile(GB): cumulative 0.000, interval 0.000 2022-09-29T05:20:43.415 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: AddFile(Total Files): cumulative 0, interval 0 2022-09-29T05:20:43.416 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: AddFile(L0 Files): cumulative 0, interval 0 2022-09-29T05:20:43.416 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: AddFile(Keys): cumulative 0, interval 0 2022-09-29T05:20:43.416 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2022-09-29T05:20:43.416 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2022-09-29T05:20:43.417 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2022-09-29T05:20:43.417 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: 2022-09-29T05:20:43.417 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: ** File Read Latency Histogram By Level [default] ** 2022-09-29T05:20:43.417 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: 2022-09-29T05:20:43.417 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: ** Compaction Stats [default] ** 2022-09-29T05:20:43.418 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2022-09-29T05:20:43.418 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2022-09-29T05:20:43.418 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: Sum 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2022-09-29T05:20:43.418 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2022-09-29T05:20:43.419 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: 2022-09-29T05:20:43.419 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: ** Compaction Stats [default] ** 2022-09-29T05:20:43.419 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2022-09-29T05:20:43.419 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2022-09-29T05:20:43.420 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: Uptime(secs): 0.0 total, 0.0 interval 2022-09-29T05:20:43.420 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: Flush(GB): cumulative 0.000, interval 0.000 2022-09-29T05:20:43.420 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: AddFile(GB): cumulative 0.000, interval 0.000 2022-09-29T05:20:43.420 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: AddFile(Total Files): cumulative 0, interval 0 2022-09-29T05:20:43.420 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: AddFile(L0 Files): cumulative 0, interval 0 2022-09-29T05:20:43.421 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: AddFile(Keys): cumulative 0, interval 0 2022-09-29T05:20:43.421 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2022-09-29T05:20:43.421 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2022-09-29T05:20:43.421 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2022-09-29T05:20:43.422 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: 2022-09-29T05:20:43.422 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: ** File Read Latency Histogram By Level [default] ** 2022-09-29T05:20:43.422 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: 2022-09-29T05:20:43.422 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.292+0000 7f5b5050a880 4 rocksdb: [db_impl/db_impl.cc:397] Shutdown: canceling all background work 2022-09-29T05:20:43.423 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.292+0000 7f5b5050a880 4 rocksdb: [db_impl/db_impl.cc:573] Shutdown complete 2022-09-29T05:20:43.423 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-mon: debug 2022-09-29T05:20:43.292+0000 7f5b5050a880 0 /usr/bin/ceph-mon: created monfs at /var/lib/ceph/mon/ceph-a for mon.a 2022-09-29T05:20:43.673 INFO:teuthology.orchestra.run.smithi083.stderr:create mon.a on 2022-09-29T05:20:43.685 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: Removed /etc/systemd/system/multi-user.target.wants/ceph.target. 2022-09-29T05:20:43.775 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2022-09-29T05:20:43.874 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-69111522-3fb6-11ed-8431-001a4aab830c.target → /etc/systemd/system/ceph-69111522-3fb6-11ed-8431-001a4aab830c.target. 2022-09-29T05:20:43.875 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: Created symlink /etc/systemd/system/ceph.target.wants/ceph-69111522-3fb6-11ed-8431-001a4aab830c.target → /etc/systemd/system/ceph-69111522-3fb6-11ed-8431-001a4aab830c.target. 2022-09-29T05:20:44.070 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: Failed to reset failed state of unit ceph-69111522-3fb6-11ed-8431-001a4aab830c@mon.a.service: Unit ceph-69111522-3fb6-11ed-8431-001a4aab830c@mon.a.service not loaded. 2022-09-29T05:20:44.077 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: Created symlink /etc/systemd/system/ceph-69111522-3fb6-11ed-8431-001a4aab830c.target.wants/ceph-69111522-3fb6-11ed-8431-001a4aab830c@mon.a.service → /etc/systemd/system/ceph-69111522-3fb6-11ed-8431-001a4aab830c@.service. 2022-09-29T05:20:44.209 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:44 smithi083 systemd[1]: Starting Ceph mon.a for 69111522-3fb6-11ed-8431-001a4aab830c... 2022-09-29T05:20:45.062 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:44 smithi083 podman[38299]: 2022-09-29T05:20:45.063 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:45 smithi083 bash[38299]: 75e1727ae3fc3519e283a6938aaed8c0922c1dec0ab177f9fa16a9eec885d801 2022-09-29T05:20:45.063 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38316]: cluster 2022-09-29T05:20:45.037844+0000 mon.a (mon.0) 0 : [INF] mkfs 69111522-3fb6-11ed-8431-001a4aab830c 2022-09-29T05:20:45.063 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38316]: cluster 2022-09-29T05:20:45.020883+0000 mon.a (mon.0) 1 : cluster [INF] mon.a is new leader, mons a in quorum (ranks 0) 2022-09-29T05:20:45.064 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:45 smithi083 systemd[1]: Started Ceph mon.a for 69111522-3fb6-11ed-8431-001a4aab830c. 2022-09-29T05:20:45.072 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: disabled 2022-09-29T05:20:45.079 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: inactive 2022-09-29T05:20:45.080 INFO:teuthology.orchestra.run.smithi083.stderr:firewalld.service is not enabled 2022-09-29T05:20:45.080 INFO:teuthology.orchestra.run.smithi083.stderr:Not possible to enable service . firewalld.service is not available 2022-09-29T05:20:45.081 INFO:teuthology.orchestra.run.smithi083.stderr:Waiting for mon to start... 2022-09-29T05:20:45.081 INFO:teuthology.orchestra.run.smithi083.stderr:Waiting for mon... 2022-09-29T05:20:45.643 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: cluster: 2022-09-29T05:20:45.643 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: id: 69111522-3fb6-11ed-8431-001a4aab830c 2022-09-29T05:20:45.643 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: health: HEALTH_OK 2022-09-29T05:20:45.644 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: 2022-09-29T05:20:45.644 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: services: 2022-09-29T05:20:45.644 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: mon: 1 daemons, quorum a (age 0.601636s) 2022-09-29T05:20:45.644 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: mgr: no daemons active 2022-09-29T05:20:45.645 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: osd: 0 osds: 0 up, 0 in 2022-09-29T05:20:45.645 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: 2022-09-29T05:20:45.645 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: data: 2022-09-29T05:20:45.645 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: pools: 0 pools, 0 pgs 2022-09-29T05:20:45.646 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: objects: 0 objects, 0 B 2022-09-29T05:20:45.646 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: usage: 0 B used, 0 B / 0 B avail 2022-09-29T05:20:45.646 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: pgs: 2022-09-29T05:20:45.646 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: 2022-09-29T05:20:46.020 INFO:teuthology.orchestra.run.smithi083.stderr:mon is available 2022-09-29T05:20:46.020 INFO:teuthology.orchestra.run.smithi083.stderr:Assimilating anything we can from ceph.conf... 2022-09-29T05:20:46.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38316]: cluster 2022-09-29T05:20:45.036610+0000 mon.a (mon.0) 2 : cluster [INF] mon.a is new leader, mons a in quorum (ranks 0) 2022-09-29T05:20:46.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38316]: cluster 2022-09-29T05:20:45.036791+0000 mon.a (mon.0) 3 : cluster [DBG] monmap e1: 1 mons at {a=[v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0]} 2022-09-29T05:20:46.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38316]: cluster 2022-09-29T05:20:45.039773+0000 mon.a (mon.0) 4 : cluster [DBG] fsmap 2022-09-29T05:20:46.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38316]: cluster 2022-09-29T05:20:45.042234+0000 mon.a (mon.0) 5 : cluster [DBG] osdmap e1: 0 total, 0 up, 0 in 2022-09-29T05:20:46.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38316]: cluster 2022-09-29T05:20:45.042615+0000 mon.a (mon.0) 6 : cluster [DBG] mgrmap e1: no daemons active 2022-09-29T05:20:46.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38316]: audit 2022-09-29T05:20:45.638205+0000 mon.a (mon.0) 7 : audit [DBG] from='client.? 172.21.15.83:0/3758375792' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2022-09-29T05:20:46.561 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: 2022-09-29T05:20:46.561 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: [global] 2022-09-29T05:20:46.561 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: fsid = 69111522-3fb6-11ed-8431-001a4aab830c 2022-09-29T05:20:46.561 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: log_to_journald = false 2022-09-29T05:20:46.562 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: mon_host = [v2:172.21.15.83:3300,v1:172.21.15.83:6789] 2022-09-29T05:20:46.562 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2022-09-29T05:20:46.562 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2022-09-29T05:20:46.562 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2022-09-29T05:20:46.563 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2022-09-29T05:20:46.563 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: 2022-09-29T05:20:46.563 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: [mgr] 2022-09-29T05:20:46.563 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2022-09-29T05:20:46.564 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: 2022-09-29T05:20:46.564 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: [osd] 2022-09-29T05:20:46.564 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2022-09-29T05:20:46.564 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2022-09-29T05:20:46.955 INFO:teuthology.orchestra.run.smithi083.stderr:Generating new minimal ceph.conf... 2022-09-29T05:20:47.262 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:47 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38316]: audit 2022-09-29T05:20:46.553596+0000 mon.a (mon.0) 8 : audit [INF] from='client.? 172.21.15.83:0/4177842500' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2022-09-29T05:20:47.263 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:47 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38316]: audit 2022-09-29T05:20:46.556191+0000 mon.a (mon.0) 9 : audit [INF] from='client.? 172.21.15.83:0/4177842500' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2022-09-29T05:20:47.991 INFO:teuthology.orchestra.run.smithi083.stderr:Restarting the monitor... 2022-09-29T05:20:48.397 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:47 smithi083 systemd[1]: Stopping Ceph mon.a for 69111522-3fb6-11ed-8431-001a4aab830c... 2022-09-29T05:20:48.397 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:48 smithi083 bash[38645]: Error: no container with name or ID "ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon.a" found: no such container 2022-09-29T05:20:48.398 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:48 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38316]: audit 2022-09-29T05:20:47.530860+0000 mon.a (mon.0) 10 : audit [DBG] from='client.? 172.21.15.83:0/2084424148' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:20:48.398 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:48 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38316]: debug 2022-09-29T05:20:48.105+0000 7f9122465700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2022-09-29T05:20:48.398 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:48 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38316]: debug 2022-09-29T05:20:48.105+0000 7f9122465700 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2022-09-29T05:20:48.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:48 smithi083 bash[38659]: ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a 2022-09-29T05:20:48.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:48 smithi083 bash[38698]: Error: no container with name or ID "ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon.a" found: no such container 2022-09-29T05:20:48.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:48 smithi083 systemd[1]: ceph-69111522-3fb6-11ed-8431-001a4aab830c@mon.a.service: Succeeded. 2022-09-29T05:20:48.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:48 smithi083 systemd[1]: Stopped Ceph mon.a for 69111522-3fb6-11ed-8431-001a4aab830c. 2022-09-29T05:20:48.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:48 smithi083 systemd[1]: Starting Ceph mon.a for 69111522-3fb6-11ed-8431-001a4aab830c... 2022-09-29T05:20:49.612 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 podman[38771]: 2022-09-29T05:20:49.612 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 bash[38771]: 0e7e560a682f94cdde2d8be5e333a352063048ee6ed55138bde80daa831a3183 2022-09-29T05:20:49.612 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.526+0000 7f4240125880 0 set uid:gid to 167:167 (ceph:ceph) 2022-09-29T05:20:49.613 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.526+0000 7f4240125880 0 ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable), process ceph-mon, pid 7 2022-09-29T05:20:49.613 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.526+0000 7f4240125880 0 pidfile_write: ignore empty --pid-file 2022-09-29T05:20:49.613 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 0 load: jerasure load: lrc load: isa 2022-09-29T05:20:49.613 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: RocksDB version: 6.8.1 2022-09-29T05:20:49.613 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: 2022-09-29T05:20:49.614 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2022-09-29T05:20:49.614 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: Compile date Jul 21 2022 2022-09-29T05:20:49.614 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: DB SUMMARY 2022-09-29T05:20:49.614 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: 2022-09-29T05:20:49.615 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: CURRENT file: CURRENT 2022-09-29T05:20:49.615 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: 2022-09-29T05:20:49.615 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: IDENTITY file: IDENTITY 2022-09-29T05:20:49.615 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: 2022-09-29T05:20:49.616 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: MANIFEST file: MANIFEST-000005 size: 131 Bytes 2022-09-29T05:20:49.616 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: 2022-09-29T05:20:49.616 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000004.sst 2022-09-29T05:20:49.616 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: 2022-09-29T05:20:49.616 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000006.log size: 85463 ; 2022-09-29T05:20:49.617 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: 2022-09-29T05:20:49.617 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: Options.error_if_exists: 0 2022-09-29T05:20:49.617 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: Options.create_if_missing: 0 2022-09-29T05:20:49.617 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: Options.paranoid_checks: 1 2022-09-29T05:20:49.618 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: Options.env: 0x563c13d82000 2022-09-29T05:20:49.618 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: Options.fs: Posix File System 2022-09-29T05:20:49.618 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: Options.info_log: 0x563c158cba20 2022-09-29T05:20:49.618 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: Options.max_file_opening_threads: 16 2022-09-29T05:20:49.619 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: Options.statistics: (nil) 2022-09-29T05:20:49.619 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: Options.use_fsync: 0 2022-09-29T05:20:49.619 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: Options.max_log_file_size: 0 2022-09-29T05:20:49.619 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: Options.max_manifest_file_size: 1073741824 2022-09-29T05:20:49.620 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: Options.log_file_time_to_roll: 0 2022-09-29T05:20:49.620 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: Options.keep_log_file_num: 1000 2022-09-29T05:20:49.620 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: Options.recycle_log_file_num: 0 2022-09-29T05:20:49.620 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: Options.allow_fallocate: 1 2022-09-29T05:20:49.620 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: Options.allow_mmap_reads: 0 2022-09-29T05:20:49.621 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: Options.allow_mmap_writes: 0 2022-09-29T05:20:49.621 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: Options.use_direct_reads: 0 2022-09-29T05:20:49.621 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2022-09-29T05:20:49.621 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: Options.create_missing_column_families: 0 2022-09-29T05:20:49.621 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: Options.db_log_dir: 2022-09-29T05:20:49.622 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2022-09-29T05:20:49.622 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: Options.table_cache_numshardbits: 6 2022-09-29T05:20:49.622 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: Options.max_subcompactions: 1 2022-09-29T05:20:49.622 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: Options.max_background_flushes: -1 2022-09-29T05:20:49.623 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: Options.WAL_ttl_seconds: 0 2022-09-29T05:20:49.623 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: Options.WAL_size_limit_MB: 0 2022-09-29T05:20:49.623 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2022-09-29T05:20:49.623 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: Options.manifest_preallocation_size: 4194304 2022-09-29T05:20:49.623 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: Options.is_fd_close_on_exec: 1 2022-09-29T05:20:49.624 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: Options.advise_random_on_open: 1 2022-09-29T05:20:49.624 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: Options.db_write_buffer_size: 0 2022-09-29T05:20:49.624 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: Options.write_buffer_manager: 0x563c165d40f0 2022-09-29T05:20:49.624 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: Options.access_hint_on_compaction_start: 1 2022-09-29T05:20:49.625 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2022-09-29T05:20:49.625 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2022-09-29T05:20:49.625 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: Options.use_adaptive_mutex: 0 2022-09-29T05:20:49.625 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: Options.rate_limiter: (nil) 2022-09-29T05:20:49.625 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2022-09-29T05:20:49.626 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: Options.wal_recovery_mode: 2 2022-09-29T05:20:49.626 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: Options.enable_thread_tracking: 0 2022-09-29T05:20:49.626 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: Options.enable_pipelined_write: 0 2022-09-29T05:20:49.626 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: Options.unordered_write: 0 2022-09-29T05:20:49.627 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2022-09-29T05:20:49.627 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2022-09-29T05:20:49.627 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: Options.write_thread_max_yield_usec: 100 2022-09-29T05:20:49.627 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2022-09-29T05:20:49.627 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: Options.row_cache: None 2022-09-29T05:20:49.628 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: Options.wal_filter: None 2022-09-29T05:20:49.628 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: Options.avoid_flush_during_recovery: 0 2022-09-29T05:20:49.628 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: Options.allow_ingest_behind: 0 2022-09-29T05:20:49.628 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: Options.preserve_deletes: 0 2022-09-29T05:20:49.628 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: Options.two_write_queues: 0 2022-09-29T05:20:49.629 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: Options.manual_wal_flush: 0 2022-09-29T05:20:49.629 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: Options.atomic_flush: 0 2022-09-29T05:20:49.629 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2022-09-29T05:20:49.629 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: Options.persist_stats_to_disk: 0 2022-09-29T05:20:49.630 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: Options.write_dbid_to_manifest: 0 2022-09-29T05:20:49.630 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: Options.log_readahead_size: 0 2022-09-29T05:20:49.630 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: Options.sst_file_checksum_func: Unknown 2022-09-29T05:20:49.630 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: Options.max_background_jobs: 2 2022-09-29T05:20:49.631 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: Options.max_background_compactions: -1 2022-09-29T05:20:49.631 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2022-09-29T05:20:49.631 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2022-09-29T05:20:49.631 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: Options.delayed_write_rate : 16777216 2022-09-29T05:20:49.631 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: Options.max_total_wal_size: 0 2022-09-29T05:20:49.632 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2022-09-29T05:20:49.632 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: Options.stats_dump_period_sec: 600 2022-09-29T05:20:49.632 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: Options.stats_persist_period_sec: 600 2022-09-29T05:20:49.632 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: Options.stats_history_buffer_size: 1048576 2022-09-29T05:20:49.633 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: Options.max_open_files: -1 2022-09-29T05:20:49.633 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: Options.bytes_per_sync: 0 2022-09-29T05:20:49.633 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: Options.wal_bytes_per_sync: 0 2022-09-29T05:20:49.633 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: Options.strict_bytes_per_sync: 0 2022-09-29T05:20:49.633 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: Options.compaction_readahead_size: 0 2022-09-29T05:20:49.634 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: Compression algorithms supported: 2022-09-29T05:20:49.634 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: kZSTDNotFinalCompression supported: 0 2022-09-29T05:20:49.634 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: kZSTD supported: 0 2022-09-29T05:20:49.634 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: kXpressCompression supported: 0 2022-09-29T05:20:49.635 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: kLZ4HCCompression supported: 1 2022-09-29T05:20:49.635 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: kLZ4Compression supported: 1 2022-09-29T05:20:49.635 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: kBZip2Compression supported: 0 2022-09-29T05:20:49.635 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: kZlibCompression supported: 1 2022-09-29T05:20:49.635 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: kSnappyCompression supported: 1 2022-09-29T05:20:49.636 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.531+0000 7f4240125880 4 rocksdb: Fast CRC32 supported: Supported on x86 2022-09-29T05:20:49.636 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.532+0000 7f4240125880 4 rocksdb: [version_set.cc:4413] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000005 2022-09-29T05:20:49.636 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: 2022-09-29T05:20:49.636 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.532+0000 7f4240125880 4 rocksdb: [column_family.cc:552] --------------- Options for column family [default]: 2022-09-29T05:20:49.637 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: 2022-09-29T05:20:49.637 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.532+0000 7f4240125880 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2022-09-29T05:20:49.637 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.532+0000 7f4240125880 4 rocksdb: Options.merge_operator: 2022-09-29T05:20:49.637 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.532+0000 7f4240125880 4 rocksdb: Options.compaction_filter: None 2022-09-29T05:20:49.637 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.532+0000 7f4240125880 4 rocksdb: Options.compaction_filter_factory: None 2022-09-29T05:20:49.638 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.532+0000 7f4240125880 4 rocksdb: Options.memtable_factory: SkipListFactory 2022-09-29T05:20:49.638 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.532+0000 7f4240125880 4 rocksdb: Options.table_factory: BlockBasedTable 2022-09-29T05:20:49.638 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.532+0000 7f4240125880 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x563c15875d10) 2022-09-29T05:20:49.638 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cache_index_and_filter_blocks: 1 2022-09-29T05:20:49.639 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cache_index_and_filter_blocks_with_high_priority: 0 2022-09-29T05:20:49.639 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: pin_l0_filter_and_index_blocks_in_cache: 0 2022-09-29T05:20:49.639 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: pin_top_level_index_and_filter: 1 2022-09-29T05:20:49.639 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: index_type: 0 2022-09-29T05:20:49.639 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: data_block_index_type: 0 2022-09-29T05:20:49.640 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: index_shortening: 1 2022-09-29T05:20:49.640 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: data_block_hash_table_util_ratio: 0.750000 2022-09-29T05:20:49.640 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: hash_index_allow_collision: 1 2022-09-29T05:20:49.640 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: checksum: 1 2022-09-29T05:20:49.640 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: no_block_cache: 0 2022-09-29T05:20:49.641 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: block_cache: 0x563c158aef10 2022-09-29T05:20:49.641 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: block_cache_name: BinnedLRUCache 2022-09-29T05:20:49.641 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: block_cache_options: 2022-09-29T05:20:49.641 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: capacity : 536870912 2022-09-29T05:20:49.642 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: num_shard_bits : 4 2022-09-29T05:20:49.642 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: strict_capacity_limit : 0 2022-09-29T05:20:49.642 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: high_pri_pool_ratio: 0.000 2022-09-29T05:20:49.642 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: block_cache_compressed: (nil) 2022-09-29T05:20:49.642 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: persistent_cache: (nil) 2022-09-29T05:20:49.643 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: block_size: 4096 2022-09-29T05:20:49.643 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: block_size_deviation: 10 2022-09-29T05:20:49.643 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: block_restart_interval: 16 2022-09-29T05:20:49.643 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: index_block_restart_interval: 1 2022-09-29T05:20:49.643 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: metadata_block_size: 4096 2022-09-29T05:20:49.644 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: partition_filters: 0 2022-09-29T05:20:49.644 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: use_delta_encoding: 1 2022-09-29T05:20:49.644 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: filter_policy: rocksdb.BuiltinBloomFilter 2022-09-29T05:20:49.644 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: whole_key_filtering: 1 2022-09-29T05:20:49.645 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: verify_compression: 0 2022-09-29T05:20:49.645 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: read_amp_bytes_per_bit: 0 2022-09-29T05:20:49.645 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: format_version: 2 2022-09-29T05:20:49.645 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: enable_index_compression: 1 2022-09-29T05:20:49.646 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: block_align: 0 2022-09-29T05:20:49.646 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: 2022-09-29T05:20:49.646 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.532+0000 7f4240125880 4 rocksdb: Options.write_buffer_size: 33554432 2022-09-29T05:20:49.646 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.532+0000 7f4240125880 4 rocksdb: Options.max_write_buffer_number: 2 2022-09-29T05:20:49.646 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.532+0000 7f4240125880 4 rocksdb: Options.compression: NoCompression 2022-09-29T05:20:49.647 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.532+0000 7f4240125880 4 rocksdb: Options.bottommost_compression: Disabled 2022-09-29T05:20:49.647 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.532+0000 7f4240125880 4 rocksdb: Options.prefix_extractor: nullptr 2022-09-29T05:20:49.647 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.532+0000 7f4240125880 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2022-09-29T05:20:49.647 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.532+0000 7f4240125880 4 rocksdb: Options.num_levels: 7 2022-09-29T05:20:49.647 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.532+0000 7f4240125880 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2022-09-29T05:20:49.648 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.532+0000 7f4240125880 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2022-09-29T05:20:49.648 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.532+0000 7f4240125880 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2022-09-29T05:20:49.648 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.532+0000 7f4240125880 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2022-09-29T05:20:49.648 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.532+0000 7f4240125880 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2022-09-29T05:20:49.649 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.532+0000 7f4240125880 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2022-09-29T05:20:49.649 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.532+0000 7f4240125880 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2022-09-29T05:20:49.649 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.532+0000 7f4240125880 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2022-09-29T05:20:49.649 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.532+0000 7f4240125880 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2022-09-29T05:20:49.650 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.532+0000 7f4240125880 4 rocksdb: Options.compression_opts.window_bits: -14 2022-09-29T05:20:49.650 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.532+0000 7f4240125880 4 rocksdb: Options.compression_opts.level: 32767 2022-09-29T05:20:49.650 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.532+0000 7f4240125880 4 rocksdb: Options.compression_opts.strategy: 0 2022-09-29T05:20:49.650 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.532+0000 7f4240125880 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2022-09-29T05:20:49.650 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.532+0000 7f4240125880 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2022-09-29T05:20:49.651 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.532+0000 7f4240125880 4 rocksdb: Options.compression_opts.enabled: false 2022-09-29T05:20:49.651 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.532+0000 7f4240125880 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2022-09-29T05:20:49.651 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.532+0000 7f4240125880 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2022-09-29T05:20:49.651 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.532+0000 7f4240125880 4 rocksdb: Options.level0_stop_writes_trigger: 36 2022-09-29T05:20:49.651 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.532+0000 7f4240125880 4 rocksdb: Options.target_file_size_base: 67108864 2022-09-29T05:20:49.652 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.532+0000 7f4240125880 4 rocksdb: Options.target_file_size_multiplier: 1 2022-09-29T05:20:49.652 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.532+0000 7f4240125880 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2022-09-29T05:20:49.652 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.532+0000 7f4240125880 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2022-09-29T05:20:49.652 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.532+0000 7f4240125880 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2022-09-29T05:20:49.653 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.532+0000 7f4240125880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2022-09-29T05:20:49.653 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.532+0000 7f4240125880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2022-09-29T05:20:49.653 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.532+0000 7f4240125880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2022-09-29T05:20:49.653 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.532+0000 7f4240125880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2022-09-29T05:20:49.653 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.532+0000 7f4240125880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2022-09-29T05:20:49.654 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.532+0000 7f4240125880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2022-09-29T05:20:49.654 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.532+0000 7f4240125880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2022-09-29T05:20:49.654 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.532+0000 7f4240125880 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2022-09-29T05:20:49.654 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.532+0000 7f4240125880 4 rocksdb: Options.max_compaction_bytes: 1677721600 2022-09-29T05:20:49.655 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.532+0000 7f4240125880 4 rocksdb: Options.arena_block_size: 4194304 2022-09-29T05:20:49.655 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.532+0000 7f4240125880 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2022-09-29T05:20:49.655 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.532+0000 7f4240125880 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2022-09-29T05:20:49.655 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.532+0000 7f4240125880 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2022-09-29T05:20:49.655 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.532+0000 7f4240125880 4 rocksdb: Options.disable_auto_compactions: 0 2022-09-29T05:20:49.656 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.532+0000 7f4240125880 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2022-09-29T05:20:49.656 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.532+0000 7f4240125880 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2022-09-29T05:20:49.656 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.532+0000 7f4240125880 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2022-09-29T05:20:49.656 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.532+0000 7f4240125880 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2022-09-29T05:20:49.657 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.532+0000 7f4240125880 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2022-09-29T05:20:49.657 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.532+0000 7f4240125880 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2022-09-29T05:20:49.657 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.532+0000 7f4240125880 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2022-09-29T05:20:49.657 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.532+0000 7f4240125880 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2022-09-29T05:20:49.657 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.532+0000 7f4240125880 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2022-09-29T05:20:49.658 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.532+0000 7f4240125880 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2022-09-29T05:20:49.658 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.532+0000 7f4240125880 4 rocksdb: Options.table_properties_collectors: 2022-09-29T05:20:49.658 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.532+0000 7f4240125880 4 rocksdb: Options.inplace_update_support: 0 2022-09-29T05:20:49.658 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.532+0000 7f4240125880 4 rocksdb: Options.inplace_update_num_locks: 10000 2022-09-29T05:20:49.659 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.532+0000 7f4240125880 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2022-09-29T05:20:49.659 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.532+0000 7f4240125880 4 rocksdb: Options.memtable_whole_key_filtering: 0 2022-09-29T05:20:49.659 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.532+0000 7f4240125880 4 rocksdb: Options.memtable_huge_page_size: 0 2022-09-29T05:20:49.659 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.532+0000 7f4240125880 4 rocksdb: Options.bloom_locality: 0 2022-09-29T05:20:49.660 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.532+0000 7f4240125880 4 rocksdb: Options.max_successive_merges: 0 2022-09-29T05:20:49.660 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.532+0000 7f4240125880 4 rocksdb: Options.optimize_filters_for_hits: 0 2022-09-29T05:20:49.660 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.532+0000 7f4240125880 4 rocksdb: Options.paranoid_file_checks: 0 2022-09-29T05:20:49.660 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.532+0000 7f4240125880 4 rocksdb: Options.force_consistency_checks: 0 2022-09-29T05:20:49.660 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.532+0000 7f4240125880 4 rocksdb: Options.report_bg_io_stats: 0 2022-09-29T05:20:49.661 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.532+0000 7f4240125880 4 rocksdb: Options.ttl: 2592000 2022-09-29T05:20:49.661 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.532+0000 7f4240125880 4 rocksdb: Options.periodic_compaction_seconds: 0 2022-09-29T05:20:49.661 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.532+0000 7f4240125880 4 rocksdb: [version_set.cc:4568] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 7, last_sequence is 5, log_number is 4,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2022-09-29T05:20:49.661 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: 2022-09-29T05:20:49.662 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.532+0000 7f4240125880 4 rocksdb: [version_set.cc:4577] Column family [default] (ID 0), log number is 4 2022-09-29T05:20:49.662 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: 2022-09-29T05:20:49.662 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.532+0000 7f4240125880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1664428849534010, "job": 1, "event": "recovery_started", "log_files": [6]} 2022-09-29T05:20:49.662 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.532+0000 7f4240125880 4 rocksdb: [db_impl/db_impl_open.cc:760] Recovering log #6 mode 2 2022-09-29T05:20:49.662 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.533+0000 7f4240125880 3 rocksdb: [le/block_based/filter_policy.cc:584] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2022-09-29T05:20:49.663 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.534+0000 7f4240125880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1664428849535157, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 7, "file_size": 82184, "table_properties": {"data_size": 80453, "index_size": 336, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 0, "index_value_is_delta_encoded": 0, "filter_size": 581, "raw_key_size": 9571, "raw_average_key_size": 47, "raw_value_size": 74895, "raw_average_value_size": 370, "num_data_blocks": 11, "num_entries": 202, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1664428849, "oldest_key_time": 3, "file_creation_time": 0}} 2022-09-29T05:20:49.663 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.534+0000 7f4240125880 4 rocksdb: [version_set.cc:3826] Creating manifest 8 2022-09-29T05:20:49.663 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: 2022-09-29T05:20:49.663 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.534+0000 7f4240125880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1664428849535653, "job": 1, "event": "recovery_finished"} 2022-09-29T05:20:49.664 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.535+0000 7f4240125880 4 rocksdb: DB pointer 0x563c15979800 2022-09-29T05:20:49.664 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.535+0000 7f4226f3f700 4 rocksdb: [db_impl/db_impl.cc:850] ------- DUMPING STATS ------- 2022-09-29T05:20:49.664 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.535+0000 7f4226f3f700 4 rocksdb: [db_impl/db_impl.cc:851] 2022-09-29T05:20:49.664 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: ** DB Stats ** 2022-09-29T05:20:49.665 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: Uptime(secs): 0.0 total, 0.0 interval 2022-09-29T05:20:49.665 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2022-09-29T05:20:49.665 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2022-09-29T05:20:49.665 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2022-09-29T05:20:49.665 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2022-09-29T05:20:49.666 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2022-09-29T05:20:49.666 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2022-09-29T05:20:49.666 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: 2022-09-29T05:20:49.666 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: ** Compaction Stats [default] ** 2022-09-29T05:20:49.667 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2022-09-29T05:20:49.667 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2022-09-29T05:20:49.667 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: L0 2/0 81.91 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 103.4 0.00 0.00 1 0.001 0 0 2022-09-29T05:20:49.667 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: Sum 2/0 81.91 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 103.4 0.00 0.00 1 0.001 0 0 2022-09-29T05:20:49.668 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 103.4 0.00 0.00 1 0.001 0 0 2022-09-29T05:20:49.669 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: 2022-09-29T05:20:49.669 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: ** Compaction Stats [default] ** 2022-09-29T05:20:49.669 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2022-09-29T05:20:49.669 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2022-09-29T05:20:49.669 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 103.4 0.00 0.00 1 0.001 0 0 2022-09-29T05:20:49.670 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: Uptime(secs): 0.0 total, 0.0 interval 2022-09-29T05:20:49.670 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: Flush(GB): cumulative 0.000, interval 0.000 2022-09-29T05:20:49.670 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: AddFile(GB): cumulative 0.000, interval 0.000 2022-09-29T05:20:49.670 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: AddFile(Total Files): cumulative 0, interval 0 2022-09-29T05:20:49.671 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: AddFile(L0 Files): cumulative 0, interval 0 2022-09-29T05:20:49.671 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: AddFile(Keys): cumulative 0, interval 0 2022-09-29T05:20:49.671 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: Cumulative compaction: 0.00 GB write, 22.05 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2022-09-29T05:20:49.671 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: Interval compaction: 0.00 GB write, 22.05 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2022-09-29T05:20:49.672 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2022-09-29T05:20:49.672 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: 2022-09-29T05:20:49.672 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: ** File Read Latency Histogram By Level [default] ** 2022-09-29T05:20:49.672 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: 2022-09-29T05:20:49.672 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: ** Compaction Stats [default] ** 2022-09-29T05:20:49.673 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2022-09-29T05:20:49.673 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2022-09-29T05:20:49.673 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: L0 2/0 81.91 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 103.4 0.00 0.00 1 0.001 0 0 2022-09-29T05:20:49.673 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: Sum 2/0 81.91 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 103.4 0.00 0.00 1 0.001 0 0 2022-09-29T05:20:49.674 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2022-09-29T05:20:49.674 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: 2022-09-29T05:20:49.674 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: ** Compaction Stats [default] ** 2022-09-29T05:20:49.674 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2022-09-29T05:20:49.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2022-09-29T05:20:49.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 103.4 0.00 0.00 1 0.001 0 0 2022-09-29T05:20:49.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: Uptime(secs): 0.0 total, 0.0 interval 2022-09-29T05:20:49.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: Flush(GB): cumulative 0.000, interval 0.000 2022-09-29T05:20:49.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: AddFile(GB): cumulative 0.000, interval 0.000 2022-09-29T05:20:49.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: AddFile(Total Files): cumulative 0, interval 0 2022-09-29T05:20:49.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: AddFile(L0 Files): cumulative 0, interval 0 2022-09-29T05:20:49.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: AddFile(Keys): cumulative 0, interval 0 2022-09-29T05:20:49.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: Cumulative compaction: 0.00 GB write, 21.75 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2022-09-29T05:20:49.677 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2022-09-29T05:20:49.677 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2022-09-29T05:20:49.677 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: 2022-09-29T05:20:49.677 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: ** File Read Latency Histogram By Level [default] ** 2022-09-29T05:20:49.677 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: 2022-09-29T05:20:49.678 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.536+0000 7f4240125880 0 starting mon.a rank 0 at public addrs [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] at bind addrs [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 69111522-3fb6-11ed-8431-001a4aab830c 2022-09-29T05:20:49.678 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.538+0000 7f4240125880 1 mon.a@-1(???) e1 preinit fsid 69111522-3fb6-11ed-8431-001a4aab830c 2022-09-29T05:20:49.678 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.538+0000 7f4240125880 0 mon.a@-1(???).mds e1 new map 2022-09-29T05:20:49.678 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.538+0000 7f4240125880 0 mon.a@-1(???).mds e1 print_map 2022-09-29T05:20:49.679 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: e1 2022-09-29T05:20:49.679 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: enable_multiple, ever_enabled_multiple: 1,1 2022-09-29T05:20:49.679 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2022-09-29T05:20:49.679 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: legacy client fscid: -1 2022-09-29T05:20:49.679 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: 2022-09-29T05:20:49.680 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: No filesystems configured 2022-09-29T05:20:49.680 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: 2022-09-29T05:20:49.680 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.538+0000 7f4240125880 0 mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2022-09-29T05:20:49.680 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.538+0000 7f4240125880 0 mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2022-09-29T05:20:49.681 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.538+0000 7f4240125880 0 mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2022-09-29T05:20:49.681 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.538+0000 7f4240125880 0 mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2022-09-29T05:20:49.681 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:20:47.530860+0000 mon.a (mon.0) 10 : audit [DBG] from='client.? 172.21.15.83:0/2084424148' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:20:49.681 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:20:49.538+0000 7f4240125880 1 mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2022-09-29T05:20:49.682 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:20:49.555912+0000 mon.a (mon.0) 1 : cluster [INF] mon.a is new leader, mons a in quorum (ranks 0) 2022-09-29T05:20:49.682 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:20:49.555968+0000 mon.a (mon.0) 2 : cluster [DBG] monmap e1: 1 mons at {a=[v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0]} 2022-09-29T05:20:49.682 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:20:49.556023+0000 mon.a (mon.0) 3 : cluster [DBG] fsmap 2022-09-29T05:20:49.682 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:20:49.556056+0000 mon.a (mon.0) 4 : cluster [DBG] osdmap e1: 0 total, 0 up, 0 in 2022-09-29T05:20:49.682 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:20:49.556427+0000 mon.a (mon.0) 5 : cluster [DBG] mgrmap e1: no daemons active 2022-09-29T05:20:49.683 INFO:teuthology.orchestra.run.smithi083.stderr:Setting mon public_network to 172.21.0.0/20 2022-09-29T05:20:49.863 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:49 smithi083 systemd[1]: Started Ceph mon.a for 69111522-3fb6-11ed-8431-001a4aab830c. 2022-09-29T05:20:50.669 INFO:teuthology.orchestra.run.smithi083.stderr:Wrote config to /etc/ceph/ceph.conf 2022-09-29T05:20:50.669 INFO:teuthology.orchestra.run.smithi083.stderr:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2022-09-29T05:20:50.669 INFO:teuthology.orchestra.run.smithi083.stderr:Creating mgr... 2022-09-29T05:20:50.670 INFO:teuthology.orchestra.run.smithi083.stderr:Verifying port 9283 ... 2022-09-29T05:20:50.799 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: Failed to reset failed state of unit ceph-69111522-3fb6-11ed-8431-001a4aab830c@mgr.y.service: Unit ceph-69111522-3fb6-11ed-8431-001a4aab830c@mgr.y.service not loaded. 2022-09-29T05:20:50.805 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: Created symlink /etc/systemd/system/ceph-69111522-3fb6-11ed-8431-001a4aab830c.target.wants/ceph-69111522-3fb6-11ed-8431-001a4aab830c@mgr.y.service → /etc/systemd/system/ceph-69111522-3fb6-11ed-8431-001a4aab830c@.service. 2022-09-29T05:20:50.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:20:50 smithi083 systemd[1]: Starting Ceph mgr.y for 69111522-3fb6-11ed-8431-001a4aab830c... 2022-09-29T05:20:51.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:20:50.203961+0000 mon.a (mon.0) 6 : audit [INF] from='client.? 172.21.15.83:0/2695033540' entity='client.admin' 2022-09-29T05:20:51.870 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:20:51 smithi083 podman[39049]: 2022-09-29T05:20:51.962 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: disabled 2022-09-29T05:20:51.970 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: inactive 2022-09-29T05:20:51.971 INFO:teuthology.orchestra.run.smithi083.stderr:firewalld.service is not enabled 2022-09-29T05:20:51.971 INFO:teuthology.orchestra.run.smithi083.stderr:Not possible to enable service . firewalld.service is not available 2022-09-29T05:20:51.978 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: disabled 2022-09-29T05:20:51.987 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: inactive 2022-09-29T05:20:51.988 INFO:teuthology.orchestra.run.smithi083.stderr:firewalld.service is not enabled 2022-09-29T05:20:51.988 INFO:teuthology.orchestra.run.smithi083.stderr:Not possible to open ports <[9283]>. firewalld.service is not available 2022-09-29T05:20:51.988 INFO:teuthology.orchestra.run.smithi083.stderr:Waiting for mgr to start... 2022-09-29T05:20:51.989 INFO:teuthology.orchestra.run.smithi083.stderr:Waiting for mgr... 2022-09-29T05:20:52.213 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:20:51 smithi083 bash[39049]: 41ec428786d7a380ead284a917bc512def09c1d91f9b019c88435a89cd90fddd 2022-09-29T05:20:52.213 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:20:51 smithi083 systemd[1]: Started Ceph mgr.y for 69111522-3fb6-11ed-8431-001a4aab830c. 2022-09-29T05:20:52.214 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:20:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:20:51.985+0000 7fae3928d680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2022-09-29T05:20:52.214 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:20:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:20:52.075+0000 7fae3928d680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2022-09-29T05:20:52.536 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: 2022-09-29T05:20:52.542 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: { 2022-09-29T05:20:52.543 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "fsid": "69111522-3fb6-11ed-8431-001a4aab830c", 2022-09-29T05:20:52.543 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "health": { 2022-09-29T05:20:52.543 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2022-09-29T05:20:52.543 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "checks": {}, 2022-09-29T05:20:52.544 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "mutes": [] 2022-09-29T05:20:52.544 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2022-09-29T05:20:52.544 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "election_epoch": 5, 2022-09-29T05:20:52.544 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "quorum": [ 2022-09-29T05:20:52.544 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: 0 2022-09-29T05:20:52.545 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: ], 2022-09-29T05:20:52.545 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "quorum_names": [ 2022-09-29T05:20:52.545 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "a" 2022-09-29T05:20:52.545 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: ], 2022-09-29T05:20:52.545 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "quorum_age": 2, 2022-09-29T05:20:52.546 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "monmap": { 2022-09-29T05:20:52.546 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "epoch": 1, 2022-09-29T05:20:52.546 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "min_mon_release_name": "pacific", 2022-09-29T05:20:52.546 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_mons": 1 2022-09-29T05:20:52.547 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2022-09-29T05:20:52.547 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "osdmap": { 2022-09-29T05:20:52.547 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "epoch": 1, 2022-09-29T05:20:52.548 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_osds": 0, 2022-09-29T05:20:52.548 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_up_osds": 0, 2022-09-29T05:20:52.548 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "osd_up_since": 0, 2022-09-29T05:20:52.549 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_in_osds": 0, 2022-09-29T05:20:52.549 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "osd_in_since": 0, 2022-09-29T05:20:52.549 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2022-09-29T05:20:52.549 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2022-09-29T05:20:52.550 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "pgmap": { 2022-09-29T05:20:52.550 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "pgs_by_state": [], 2022-09-29T05:20:52.550 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_pgs": 0, 2022-09-29T05:20:52.550 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_pools": 0, 2022-09-29T05:20:52.550 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_objects": 0, 2022-09-29T05:20:52.551 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "data_bytes": 0, 2022-09-29T05:20:52.551 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "bytes_used": 0, 2022-09-29T05:20:52.551 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "bytes_avail": 0, 2022-09-29T05:20:52.551 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "bytes_total": 0 2022-09-29T05:20:52.552 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2022-09-29T05:20:52.552 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "fsmap": { 2022-09-29T05:20:52.552 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "epoch": 1, 2022-09-29T05:20:52.552 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "by_rank": [], 2022-09-29T05:20:52.552 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "up:standby": 0 2022-09-29T05:20:52.553 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2022-09-29T05:20:52.553 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "mgrmap": { 2022-09-29T05:20:52.553 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "available": false, 2022-09-29T05:20:52.553 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_standbys": 0, 2022-09-29T05:20:52.553 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "modules": [ 2022-09-29T05:20:52.554 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "iostat", 2022-09-29T05:20:52.554 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "nfs", 2022-09-29T05:20:52.554 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "restful" 2022-09-29T05:20:52.554 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: ], 2022-09-29T05:20:52.555 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "services": {} 2022-09-29T05:20:52.555 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2022-09-29T05:20:52.555 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "servicemap": { 2022-09-29T05:20:52.555 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "epoch": 1, 2022-09-29T05:20:52.556 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "modified": "2022-09-29T05:20:45.038226+0000", 2022-09-29T05:20:52.556 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "services": {} 2022-09-29T05:20:52.556 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2022-09-29T05:20:52.556 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "progress_events": {} 2022-09-29T05:20:52.556 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: } 2022-09-29T05:20:52.560 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:20:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:20:52.449+0000 7fae3928d680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2022-09-29T05:20:52.931 INFO:teuthology.orchestra.run.smithi083.stderr:mgr not available, waiting (1/15)... 2022-09-29T05:20:52.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:20:52.533125+0000 mon.a (mon.0) 7 : audit [DBG] from='client.? 172.21.15.83:0/463430486' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2022-09-29T05:20:53.295 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:20:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:20:53.209+0000 7fae3928d680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2022-09-29T05:20:53.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:20:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:20:53.292+0000 7fae3928d680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2022-09-29T05:20:53.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:20:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:20:53.499+0000 7fae3928d680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2022-09-29T05:20:54.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:20:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:20:54.443+0000 7fae3928d680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2022-09-29T05:20:54.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:20:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:20:54.569+0000 7fae3928d680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2022-09-29T05:20:54.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:20:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:20:54.646+0000 7fae3928d680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2022-09-29T05:20:55.179 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:20:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:20:54.734+0000 7fae3928d680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2022-09-29T05:20:55.180 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:20:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:20:54.926+0000 7fae3928d680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2022-09-29T05:20:55.463 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:20:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:20:55.227+0000 7fae3928d680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2022-09-29T05:20:55.463 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:20:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:20:55.328+0000 7fae3928d680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2022-09-29T05:20:55.544 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: 2022-09-29T05:20:55.544 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: { 2022-09-29T05:20:55.544 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "fsid": "69111522-3fb6-11ed-8431-001a4aab830c", 2022-09-29T05:20:55.545 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "health": { 2022-09-29T05:20:55.545 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2022-09-29T05:20:55.545 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "checks": {}, 2022-09-29T05:20:55.545 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "mutes": [] 2022-09-29T05:20:55.546 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2022-09-29T05:20:55.546 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "election_epoch": 5, 2022-09-29T05:20:55.546 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "quorum": [ 2022-09-29T05:20:55.546 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: 0 2022-09-29T05:20:55.547 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: ], 2022-09-29T05:20:55.547 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "quorum_names": [ 2022-09-29T05:20:55.547 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "a" 2022-09-29T05:20:55.547 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: ], 2022-09-29T05:20:55.548 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "quorum_age": 5, 2022-09-29T05:20:55.548 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "monmap": { 2022-09-29T05:20:55.548 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "epoch": 1, 2022-09-29T05:20:55.548 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "min_mon_release_name": "pacific", 2022-09-29T05:20:55.549 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_mons": 1 2022-09-29T05:20:55.549 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2022-09-29T05:20:55.549 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "osdmap": { 2022-09-29T05:20:55.549 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "epoch": 1, 2022-09-29T05:20:55.549 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_osds": 0, 2022-09-29T05:20:55.550 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_up_osds": 0, 2022-09-29T05:20:55.550 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "osd_up_since": 0, 2022-09-29T05:20:55.550 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_in_osds": 0, 2022-09-29T05:20:55.550 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "osd_in_since": 0, 2022-09-29T05:20:55.550 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2022-09-29T05:20:55.551 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2022-09-29T05:20:55.551 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "pgmap": { 2022-09-29T05:20:55.551 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "pgs_by_state": [], 2022-09-29T05:20:55.551 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_pgs": 0, 2022-09-29T05:20:55.551 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_pools": 0, 2022-09-29T05:20:55.552 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_objects": 0, 2022-09-29T05:20:55.552 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "data_bytes": 0, 2022-09-29T05:20:55.552 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "bytes_used": 0, 2022-09-29T05:20:55.552 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "bytes_avail": 0, 2022-09-29T05:20:55.552 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "bytes_total": 0 2022-09-29T05:20:55.553 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2022-09-29T05:20:55.553 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "fsmap": { 2022-09-29T05:20:55.553 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "epoch": 1, 2022-09-29T05:20:55.553 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "by_rank": [], 2022-09-29T05:20:55.553 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "up:standby": 0 2022-09-29T05:20:55.554 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2022-09-29T05:20:55.554 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "mgrmap": { 2022-09-29T05:20:55.554 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "available": false, 2022-09-29T05:20:55.554 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_standbys": 0, 2022-09-29T05:20:55.554 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "modules": [ 2022-09-29T05:20:55.555 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "iostat", 2022-09-29T05:20:55.555 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "nfs", 2022-09-29T05:20:55.555 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "restful" 2022-09-29T05:20:55.555 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: ], 2022-09-29T05:20:55.555 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "services": {} 2022-09-29T05:20:55.556 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2022-09-29T05:20:55.556 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "servicemap": { 2022-09-29T05:20:55.556 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "epoch": 1, 2022-09-29T05:20:55.556 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "modified": "2022-09-29T05:20:45.038226+0000", 2022-09-29T05:20:55.556 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "services": {} 2022-09-29T05:20:55.557 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2022-09-29T05:20:55.557 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "progress_events": {} 2022-09-29T05:20:55.557 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: } 2022-09-29T05:20:55.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:20:55.535106+0000 mon.a (mon.0) 8 : audit [DBG] from='client.? 172.21.15.83:0/1903754858' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2022-09-29T05:20:55.981 INFO:teuthology.orchestra.run.smithi083.stderr:mgr not available, waiting (2/15)... 2022-09-29T05:20:56.421 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:20:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:20:56.072+0000 7fae3928d680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2022-09-29T05:20:56.421 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:20:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:20:56.154+0000 7fae3928d680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2022-09-29T05:20:56.422 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:20:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:20:56.245+0000 7fae3928d680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2022-09-29T05:20:56.936 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:20:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:20:56.417+0000 7fae3928d680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2022-09-29T05:20:56.936 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:20:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:20:56.673+0000 7fae3928d680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2022-09-29T05:20:56.937 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:20:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:20:56.802+0000 7fae3928d680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2022-09-29T05:20:57.213 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:20:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:20:56.932+0000 7fae3928d680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2022-09-29T05:20:57.213 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:20:57 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:20:57.100+0000 7fae3928d680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2022-09-29T05:20:57.213 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:20:57 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:20:57.183+0000 7fae3928d680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2022-09-29T05:20:57.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:57 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:20:57.195871+0000 mon.a (mon.0) 9 : cluster [INF] Activating manager daemon y 2022-09-29T05:20:57.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:57 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:20:57.198593+0000 mon.a (mon.0) 10 : cluster [DBG] mgrmap e2: y(active, starting, since 0.00281058s) 2022-09-29T05:20:57.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:57 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:20:57.199475+0000 mon.a (mon.0) 11 : audit [DBG] from='mgr.14100 172.21.15.83:0/1031978342' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2022-09-29T05:20:57.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:57 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:20:57.199658+0000 mon.a (mon.0) 12 : audit [DBG] from='mgr.14100 172.21.15.83:0/1031978342' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2022-09-29T05:20:57.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:57 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:20:57.199770+0000 mon.a (mon.0) 13 : audit [DBG] from='mgr.14100 172.21.15.83:0/1031978342' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2022-09-29T05:20:57.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:57 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:20:57.199913+0000 mon.a (mon.0) 14 : audit [DBG] from='mgr.14100 172.21.15.83:0/1031978342' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2022-09-29T05:20:57.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:57 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:20:57.200118+0000 mon.a (mon.0) 15 : audit [DBG] from='mgr.14100 172.21.15.83:0/1031978342' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2022-09-29T05:20:57.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:57 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:20:57.213579+0000 mon.a (mon.0) 16 : cluster [INF] Manager daemon y is now available 2022-09-29T05:20:57.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:57 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:20:57.223588+0000 mon.a (mon.0) 17 : audit [INF] from='mgr.14100 172.21.15.83:0/1031978342' entity='mgr.y' 2022-09-29T05:20:57.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:57 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:20:57.225117+0000 mon.a (mon.0) 18 : audit [INF] from='mgr.14100 172.21.15.83:0/1031978342' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:20:57.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:57 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:20:57.226264+0000 mon.a (mon.0) 19 : audit [INF] from='mgr.14100 172.21.15.83:0/1031978342' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:20:57.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:57 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:20:57.233984+0000 mon.a (mon.0) 20 : audit [INF] from='mgr.14100 172.21.15.83:0/1031978342' entity='mgr.y' 2022-09-29T05:20:57.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:57 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:20:57.238999+0000 mon.a (mon.0) 21 : audit [INF] from='mgr.14100 172.21.15.83:0/1031978342' entity='mgr.y' 2022-09-29T05:20:59.250 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:20:59 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:20:58.201100+0000 mon.a (mon.0) 22 : cluster [DBG] mgrmap e3: y(active, since 1.00531s) 2022-09-29T05:20:59.987 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: 2022-09-29T05:20:59.987 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: { 2022-09-29T05:20:59.987 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "fsid": "69111522-3fb6-11ed-8431-001a4aab830c", 2022-09-29T05:20:59.988 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "health": { 2022-09-29T05:20:59.988 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2022-09-29T05:20:59.988 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "checks": {}, 2022-09-29T05:20:59.988 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "mutes": [] 2022-09-29T05:20:59.989 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2022-09-29T05:20:59.989 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "election_epoch": 5, 2022-09-29T05:20:59.989 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "quorum": [ 2022-09-29T05:20:59.989 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: 0 2022-09-29T05:20:59.989 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: ], 2022-09-29T05:20:59.990 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "quorum_names": [ 2022-09-29T05:20:59.990 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "a" 2022-09-29T05:20:59.990 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: ], 2022-09-29T05:20:59.990 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "quorum_age": 10, 2022-09-29T05:20:59.991 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "monmap": { 2022-09-29T05:20:59.991 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "epoch": 1, 2022-09-29T05:20:59.991 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "min_mon_release_name": "pacific", 2022-09-29T05:20:59.991 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_mons": 1 2022-09-29T05:20:59.992 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2022-09-29T05:20:59.992 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "osdmap": { 2022-09-29T05:20:59.992 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "epoch": 1, 2022-09-29T05:20:59.992 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_osds": 0, 2022-09-29T05:20:59.992 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_up_osds": 0, 2022-09-29T05:20:59.993 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "osd_up_since": 0, 2022-09-29T05:20:59.996 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_in_osds": 0, 2022-09-29T05:20:59.997 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "osd_in_since": 0, 2022-09-29T05:20:59.997 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2022-09-29T05:20:59.997 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2022-09-29T05:20:59.997 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "pgmap": { 2022-09-29T05:20:59.998 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "pgs_by_state": [], 2022-09-29T05:20:59.998 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_pgs": 0, 2022-09-29T05:20:59.998 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_pools": 0, 2022-09-29T05:20:59.998 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_objects": 0, 2022-09-29T05:20:59.999 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "data_bytes": 0, 2022-09-29T05:20:59.999 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "bytes_used": 0, 2022-09-29T05:20:59.999 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "bytes_avail": 0, 2022-09-29T05:20:59.999 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "bytes_total": 0 2022-09-29T05:21:00.000 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2022-09-29T05:21:00.000 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "fsmap": { 2022-09-29T05:21:00.000 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "epoch": 1, 2022-09-29T05:21:00.000 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "by_rank": [], 2022-09-29T05:21:00.000 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "up:standby": 0 2022-09-29T05:21:00.001 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2022-09-29T05:21:00.001 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "mgrmap": { 2022-09-29T05:21:00.001 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "available": true, 2022-09-29T05:21:00.001 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_standbys": 0, 2022-09-29T05:21:00.001 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "modules": [ 2022-09-29T05:21:00.002 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "iostat", 2022-09-29T05:21:00.002 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "nfs", 2022-09-29T05:21:00.002 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "restful" 2022-09-29T05:21:00.002 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: ], 2022-09-29T05:21:00.003 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "services": {} 2022-09-29T05:21:00.003 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2022-09-29T05:21:00.003 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "servicemap": { 2022-09-29T05:21:00.003 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "epoch": 1, 2022-09-29T05:21:00.003 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "modified": "2022-09-29T05:20:45.038226+0000", 2022-09-29T05:21:00.004 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "services": {} 2022-09-29T05:21:00.004 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2022-09-29T05:21:00.004 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "progress_events": {} 2022-09-29T05:21:00.004 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: } 2022-09-29T05:21:00.893 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:20:59.981205+0000 mon.a (mon.0) 23 : audit [DBG] from='client.? 172.21.15.83:0/2244902821' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2022-09-29T05:21:00.893 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:21:00.201453+0000 mon.a (mon.0) 24 : cluster [DBG] mgrmap e4: y(active, since 3s) 2022-09-29T05:21:02.230 INFO:teuthology.orchestra.run.smithi083.stderr:mgr is available 2022-09-29T05:21:05.061 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: 2022-09-29T05:21:05.061 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: [global] 2022-09-29T05:21:05.062 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: fsid = 69111522-3fb6-11ed-8431-001a4aab830c 2022-09-29T05:21:05.062 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: log_to_journald = false 2022-09-29T05:21:05.062 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2022-09-29T05:21:05.062 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2022-09-29T05:21:05.063 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2022-09-29T05:21:05.063 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2022-09-29T05:21:05.063 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: 2022-09-29T05:21:05.064 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: [mgr] 2022-09-29T05:21:05.064 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2022-09-29T05:21:05.064 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: 2022-09-29T05:21:05.064 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: [osd] 2022-09-29T05:21:05.064 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2022-09-29T05:21:05.065 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2022-09-29T05:21:05.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:05.055643+0000 mon.a (mon.0) 25 : audit [INF] from='client.? 172.21.15.83:0/3725972393' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2022-09-29T05:21:07.982 INFO:teuthology.orchestra.run.smithi083.stderr:Enabling cephadm module... 2022-09-29T05:21:10.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:10.219841+0000 mon.a (mon.0) 26 : audit [INF] from='client.? 172.21.15.83:0/3448295615' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2022-09-29T05:21:10.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:21:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: ignoring --setuser ceph since I am not root 2022-09-29T05:21:10.964 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:21:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: ignoring --setgroup ceph since I am not root 2022-09-29T05:21:10.964 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:21:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:21:10.704+0000 7f4fb0ba3680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2022-09-29T05:21:10.964 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:21:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:21:10.783+0000 7f4fb0ba3680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2022-09-29T05:21:11.332 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:21:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:21:11.147+0000 7f4fb0ba3680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2022-09-29T05:21:11.592 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:10.551521+0000 mon.a (mon.0) 27 : audit [INF] from='client.? 172.21.15.83:0/3448295615' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2022-09-29T05:21:11.592 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:21:10.551616+0000 mon.a (mon.0) 28 : cluster [DBG] mgrmap e5: y(active, since 13s) 2022-09-29T05:21:11.908 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:21:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:21:11.904+0000 7f4fb0ba3680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2022-09-29T05:21:12.062 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: { 2022-09-29T05:21:12.062 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "epoch": 5, 2022-09-29T05:21:12.062 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "available": true, 2022-09-29T05:21:12.063 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "active_name": "y", 2022-09-29T05:21:12.063 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_standby": 0 2022-09-29T05:21:12.063 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: } 2022-09-29T05:21:12.212 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:21:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:21:11.989+0000 7f4fb0ba3680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2022-09-29T05:21:12.463 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:21:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:21:12.207+0000 7f4fb0ba3680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2022-09-29T05:21:12.563 INFO:teuthology.orchestra.run.smithi083.stderr:Waiting for the mgr to restart... 2022-09-29T05:21:12.564 INFO:teuthology.orchestra.run.smithi083.stderr:Waiting for mgr epoch 5... 2022-09-29T05:21:12.808 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:12.058247+0000 mon.a (mon.0) 29 : audit [DBG] from='client.? 172.21.15.83:0/2217834285' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2022-09-29T05:21:13.463 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:21:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:21:13.164+0000 7f4fb0ba3680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2022-09-29T05:21:13.463 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:21:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:21:13.291+0000 7f4fb0ba3680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2022-09-29T05:21:13.463 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:21:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:21:13.370+0000 7f4fb0ba3680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2022-09-29T05:21:13.934 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:21:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:21:13.459+0000 7f4fb0ba3680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2022-09-29T05:21:13.934 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:21:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:21:13.652+0000 7f4fb0ba3680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2022-09-29T05:21:14.213 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:21:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:21:13.930+0000 7f4fb0ba3680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2022-09-29T05:21:14.213 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:21:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:21:14.028+0000 7f4fb0ba3680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2022-09-29T05:21:15.091 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:21:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:21:14.740+0000 7f4fb0ba3680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2022-09-29T05:21:15.092 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:21:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:21:14.822+0000 7f4fb0ba3680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2022-09-29T05:21:15.092 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:21:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:21:14.914+0000 7f4fb0ba3680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2022-09-29T05:21:15.092 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:21:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:21:15.087+0000 7f4fb0ba3680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2022-09-29T05:21:15.606 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:21:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:21:15.342+0000 7f4fb0ba3680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2022-09-29T05:21:15.607 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:21:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:21:15.472+0000 7f4fb0ba3680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2022-09-29T05:21:15.924 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:21:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:21:15.603+0000 7f4fb0ba3680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2022-09-29T05:21:15.925 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:21:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:21:15.769+0000 7f4fb0ba3680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2022-09-29T05:21:15.925 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:21:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:21:15.852+0000 7f4fb0ba3680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2022-09-29T05:21:16.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:21:15.865991+0000 mon.a (mon.0) 30 : cluster [INF] Active manager daemon y restarted 2022-09-29T05:21:16.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:21:15.866413+0000 mon.a (mon.0) 31 : cluster [INF] Activating manager daemon y 2022-09-29T05:21:16.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:21:15.870219+0000 mon.a (mon.0) 32 : cluster [DBG] osdmap e2: 0 total, 0 up, 0 in 2022-09-29T05:21:16.935 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: { 2022-09-29T05:21:16.936 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "mgrmap_epoch": 7, 2022-09-29T05:21:16.936 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "initialized": true 2022-09-29T05:21:16.936 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: } 2022-09-29T05:21:17.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:21:15.919075+0000 mon.a (mon.0) 33 : cluster [DBG] mgrmap e6: y(active, starting, since 0.0527573s) 2022-09-29T05:21:17.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:15.919932+0000 mon.a (mon.0) 34 : audit [DBG] from='mgr.14116 172.21.15.83:0/2807403123' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2022-09-29T05:21:17.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:15.920435+0000 mon.a (mon.0) 35 : audit [DBG] from='mgr.14116 172.21.15.83:0/2807403123' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2022-09-29T05:21:17.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:15.921348+0000 mon.a (mon.0) 36 : audit [DBG] from='mgr.14116 172.21.15.83:0/2807403123' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2022-09-29T05:21:17.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:15.921472+0000 mon.a (mon.0) 37 : audit [DBG] from='mgr.14116 172.21.15.83:0/2807403123' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2022-09-29T05:21:17.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:15.921589+0000 mon.a (mon.0) 38 : audit [DBG] from='mgr.14116 172.21.15.83:0/2807403123' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2022-09-29T05:21:17.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:21:15.935751+0000 mon.a (mon.0) 39 : cluster [INF] Manager daemon y is now available 2022-09-29T05:21:17.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:15.949954+0000 mon.a (mon.0) 40 : audit [INF] from='mgr.14116 172.21.15.83:0/2807403123' entity='mgr.y' 2022-09-29T05:21:17.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:16.038469+0000 mon.a (mon.0) 41 : audit [INF] from='mgr.14116 172.21.15.83:0/2807403123' entity='mgr.y' 2022-09-29T05:21:17.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:16.039565+0000 mon.a (mon.0) 42 : audit [DBG] from='mgr.14116 172.21.15.83:0/2807403123' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:21:17.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:16.040721+0000 mon.a (mon.0) 43 : audit [DBG] from='mgr.14116 172.21.15.83:0/2807403123' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:21:17.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:16.042358+0000 mon.a (mon.0) 44 : audit [DBG] from='mgr.14116 172.21.15.83:0/2807403123' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:21:17.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:16.054322+0000 mon.a (mon.0) 45 : audit [INF] from='mgr.14116 172.21.15.83:0/2807403123' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:21:17.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:16.055531+0000 mon.a (mon.0) 46 : audit [INF] from='mgr.14116 172.21.15.83:0/2807403123' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:21:17.408 INFO:teuthology.orchestra.run.smithi083.stderr:mgr epoch 5 is available 2022-09-29T05:21:17.409 INFO:teuthology.orchestra.run.smithi083.stderr:Setting orchestrator backend to cephadm... 2022-09-29T05:21:18.184 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:17 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:16.925302+0000 mgr.y (mgr.14116) 1 : audit [DBG] from='client.14120 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2022-09-29T05:21:18.185 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:17 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:21:16.925892+0000 mon.a (mon.0) 47 : cluster [DBG] mgrmap e7: y(active, since 1.05957s) 2022-09-29T05:21:18.185 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:17 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:16.931854+0000 mgr.y (mgr.14116) 2 : audit [DBG] from='client.14120 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2022-09-29T05:21:19.103 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: value unchanged 2022-09-29T05:21:19.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:18 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:21:17.935751+0000 mon.a (mon.0) 48 : cluster [DBG] mgrmap e8: y(active, since 2s) 2022-09-29T05:21:19.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:18 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:18.030373+0000 mgr.y (mgr.14116) 3 : audit [DBG] from='client.14126 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:21:19.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:18 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:18.035004+0000 mon.a (mon.0) 49 : audit [INF] from='mgr.14116 172.21.15.83:0/2807403123' entity='mgr.y' 2022-09-29T05:21:19.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:18 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:18.063175+0000 mon.a (mon.0) 50 : audit [DBG] from='mgr.14116 172.21.15.83:0/2807403123' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:21:19.478 INFO:teuthology.orchestra.run.smithi083.stderr:Generating ssh key... 2022-09-29T05:21:20.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:19.099763+0000 mgr.y (mgr.14116) 4 : audit [DBG] from='client.14128 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:21:20.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:20.371833+0000 mon.a (mon.0) 51 : audit [INF] from='mgr.14116 172.21.15.83:0/2807403123' entity='mgr.y' 2022-09-29T05:21:20.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:20.373599+0000 mon.a (mon.0) 52 : audit [INF] from='mgr.14116 172.21.15.83:0/2807403123' entity='mgr.y' 2022-09-29T05:21:20.714 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:21:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: Generating public/private rsa key pair. 2022-09-29T05:21:20.714 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:21:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: Your identification has been saved in /tmp/tmpa3hel0m9/key. 2022-09-29T05:21:20.715 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:21:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: Your public key has been saved in /tmp/tmpa3hel0m9/key.pub. 2022-09-29T05:21:20.715 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:21:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: The key fingerprint is: 2022-09-29T05:21:20.715 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:21:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: SHA256:768acA7DXkls6xkgrmCHuEVJAbN5q0B7zwzVh8GDbIE ceph-69111522-3fb6-11ed-8431-001a4aab830c 2022-09-29T05:21:20.715 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:21:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: The key's randomart image is: 2022-09-29T05:21:20.716 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:21:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: +---[RSA 3072]----+ 2022-09-29T05:21:20.716 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:21:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: |o.o.o.+. | 2022-09-29T05:21:20.716 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:21:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: | = E +.+o | 2022-09-29T05:21:20.717 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:21:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: |o.+ o..o=. | 2022-09-29T05:21:20.717 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:21:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: |oooo.o +.o | 2022-09-29T05:21:20.717 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:21:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: |++o+. = S | 2022-09-29T05:21:20.717 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:21:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: |o=o.=. O + | 2022-09-29T05:21:20.718 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:21:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: |o . +. = . | 2022-09-29T05:21:20.718 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:21:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: | o | 2022-09-29T05:21:20.718 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:21:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: | ..oo. | 2022-09-29T05:21:20.718 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:21:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: +----[SHA256]-----+ 2022-09-29T05:21:21.487 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC3SV2SQYCDaE+LfnVnNmOO2b4VrqjhBpbXhE/Y7OXx57l1As3nSekNOSy/h75pLckO2HhLcLUVwWWLwy3jdsfKdoQ4p1JJQ3RHM75Js6iJT1dJRLijs1MMXikRM3QlXW+9VY6h5uORQ0b033M8EPduoe/7UbVZ33p3zm+5nU21Lji5WvtC/y400+EgYlq1Y/+7GNeqD0Xj9gSqKXqYqzxSXGQi4NL0yq5gH1TuNiSPgDXKI23TVMFlMH7m1DtW0CYawcWNPPzQtjJc1zbuAM/uBguFGn19mVVS3RwhnH4U6Jyc8qNRUqldQ95q9rqQstUJq9Vsjwht7UCiewSjDhzLNswqf4b7+nIyN9pKx0rprnZ1zqY9DONv69KwOQuuyFdCbd+Q806o5r9vVrR/0QX5XikXCNP9I++Bhu5VLLD6xgpK+xYpLsG/jd4bhj08j4LROCPbzpOIumvdKExFa8EPMHE1LS81kef3jy/47xZWKXF0j9Itf+Qn/LFdC5gLoOE= ceph-69111522-3fb6-11ed-8431-001a4aab830c 2022-09-29T05:21:21.925 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:21:20.083175+0000 mgr.y (mgr.14116) 6 : cephadm [INF] Generating ssh key... 2022-09-29T05:21:21.949 INFO:teuthology.orchestra.run.smithi083.stderr:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2022-09-29T05:21:21.950 INFO:teuthology.orchestra.run.smithi083.stderr:Adding key to root@localhost authorized_keys... 2022-09-29T05:21:21.950 INFO:teuthology.orchestra.run.smithi083.stderr:Adding host smithi083... 2022-09-29T05:21:22.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:21.483795+0000 mgr.y (mgr.14116) 7 : audit [DBG] from='client.14132 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:21:22.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:21:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: [79B blob data] 2022-09-29T05:21:23.314 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: Added host 'smithi083' with addr '172.21.15.83' 2022-09-29T05:21:23.788 INFO:teuthology.orchestra.run.smithi083.stderr:Deploying unmanaged mon service... 2022-09-29T05:21:24.406 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: Scheduled mon update... 2022-09-29T05:21:24.575 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:22.597729+0000 mgr.y (mgr.14116) 8 : audit [DBG] from='client.14134 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi083", "addr": "172.21.15.83", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:21:24.575 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:23.309510+0000 mon.a (mon.0) 53 : audit [INF] from='mgr.14116 172.21.15.83:0/2807403123' entity='mgr.y' 2022-09-29T05:21:24.576 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:21:23.309811+0000 mgr.y (mgr.14116) 9 : cephadm [INF] Added host smithi083 2022-09-29T05:21:24.576 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:23.339011+0000 mon.a (mon.0) 54 : audit [DBG] from='mgr.14116 172.21.15.83:0/2807403123' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:21:24.826 INFO:teuthology.orchestra.run.smithi083.stderr:Deploying unmanaged mgr service... 2022-09-29T05:21:25.581 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: Scheduled mgr update... 2022-09-29T05:21:25.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:24.398788+0000 mgr.y (mgr.14116) 10 : audit [DBG] from='client.14136 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:21:25.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:21:24.400008+0000 mgr.y (mgr.14116) 11 : cephadm [INF] Saving service mon spec with placement count:5 2022-09-29T05:21:25.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:24.402898+0000 mon.a (mon.0) 55 : audit [INF] from='mgr.14116 172.21.15.83:0/2807403123' entity='mgr.y' 2022-09-29T05:21:26.940 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:25.572597+0000 mgr.y (mgr.14116) 12 : audit [DBG] from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:21:26.940 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:21:25.573488+0000 mgr.y (mgr.14116) 13 : cephadm [INF] Saving service mgr spec with placement count:2 2022-09-29T05:21:26.941 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:25.578279+0000 mon.a (mon.0) 56 : audit [INF] from='mgr.14116 172.21.15.83:0/2807403123' entity='mgr.y' 2022-09-29T05:21:26.941 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:25.965307+0000 mon.a (mon.0) 57 : audit [INF] from='mgr.14116 172.21.15.83:0/2807403123' entity='mgr.y' 2022-09-29T05:21:26.941 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:26.416518+0000 mon.a (mon.0) 58 : audit [INF] from='mgr.14116 172.21.15.83:0/2807403123' entity='mgr.y' 2022-09-29T05:21:28.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:27 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:26.911832+0000 mon.a (mon.0) 59 : audit [INF] from='client.? 172.21.15.83:0/1622640961' entity='client.admin' 2022-09-29T05:21:28.333 INFO:teuthology.orchestra.run.smithi083.stderr:Enabling the dashboard module... 2022-09-29T05:21:29.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:28 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:27.946959+0000 mon.a (mon.0) 60 : audit [INF] from='client.? 172.21.15.83:0/3346236509' entity='client.admin' 2022-09-29T05:21:29.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:28 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:28.164116+0000 mon.a (mon.0) 61 : audit [INF] from='mgr.14116 172.21.15.83:0/2807403123' entity='mgr.y' 2022-09-29T05:21:30.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:29 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:29.028270+0000 mon.a (mon.0) 62 : audit [INF] from='client.? 172.21.15.83:0/271795143' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2022-09-29T05:21:30.212 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:21:29 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: ignoring --setuser ceph since I am not root 2022-09-29T05:21:30.212 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:21:29 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: ignoring --setgroup ceph since I am not root 2022-09-29T05:21:30.213 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:21:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:21:30.126+0000 7f3b357ee680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2022-09-29T05:21:30.463 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:21:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:21:30.207+0000 7f3b357ee680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2022-09-29T05:21:30.951 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:21:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:21:30.592+0000 7f3b357ee680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2022-09-29T05:21:31.104 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: { 2022-09-29T05:21:31.104 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "epoch": 9, 2022-09-29T05:21:31.104 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "available": true, 2022-09-29T05:21:31.105 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "active_name": "y", 2022-09-29T05:21:31.105 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_standby": 0 2022-09-29T05:21:31.105 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: } 2022-09-29T05:21:31.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:29.950975+0000 mon.a (mon.0) 63 : audit [INF] from='client.? 172.21.15.83:0/271795143' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2022-09-29T05:21:31.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:21:29.951040+0000 mon.a (mon.0) 64 : cluster [DBG] mgrmap e9: y(active, since 14s) 2022-09-29T05:21:31.649 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:21:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:21:31.353+0000 7f3b357ee680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2022-09-29T05:21:31.650 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:21:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:21:31.437+0000 7f3b357ee680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2022-09-29T05:21:31.925 INFO:teuthology.orchestra.run.smithi083.stderr:Waiting for the mgr to restart... 2022-09-29T05:21:31.926 INFO:teuthology.orchestra.run.smithi083.stderr:Waiting for mgr epoch 9... 2022-09-29T05:21:31.945 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:21:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:21:31.645+0000 7f3b357ee680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2022-09-29T05:21:32.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:31.100442+0000 mon.a (mon.0) 65 : audit [DBG] from='client.? 172.21.15.83:0/2897450455' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2022-09-29T05:21:32.904 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:21:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:21:32.603+0000 7f3b357ee680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2022-09-29T05:21:32.904 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:21:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:21:32.731+0000 7f3b357ee680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2022-09-29T05:21:32.905 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:21:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:21:32.809+0000 7f3b357ee680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2022-09-29T05:21:33.213 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:21:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:21:32.900+0000 7f3b357ee680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2022-09-29T05:21:33.213 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:21:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:21:33.094+0000 7f3b357ee680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2022-09-29T05:21:33.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:21:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:21:33.374+0000 7f3b357ee680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2022-09-29T05:21:33.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:21:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:21:33.474+0000 7f3b357ee680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2022-09-29T05:21:34.463 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:21:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:21:34.186+0000 7f3b357ee680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2022-09-29T05:21:34.463 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:21:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:21:34.268+0000 7f3b357ee680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2022-09-29T05:21:34.463 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:21:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:21:34.359+0000 7f3b357ee680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2022-09-29T05:21:34.793 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:21:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:21:34.534+0000 7f3b357ee680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2022-09-29T05:21:35.057 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:21:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:21:34.789+0000 7f3b357ee680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2022-09-29T05:21:35.058 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:21:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:21:34.920+0000 7f3b357ee680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2022-09-29T05:21:35.309 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:21:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:21:35.054+0000 7f3b357ee680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2022-09-29T05:21:35.310 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:21:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:21:35.221+0000 7f3b357ee680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2022-09-29T05:21:35.696 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:21:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:21:35.305+0000 7f3b357ee680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2022-09-29T05:21:35.697 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:21:35.319792+0000 mon.a (mon.0) 66 : cluster [INF] Active manager daemon y restarted 2022-09-29T05:21:35.697 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:21:35.320175+0000 mon.a (mon.0) 67 : cluster [INF] Activating manager daemon y 2022-09-29T05:21:35.697 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:21:35.321881+0000 mon.a (mon.0) 68 : cluster [DBG] osdmap e3: 0 total, 0 up, 0 in 2022-09-29T05:21:35.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:21:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: [79B blob data] 2022-09-29T05:21:36.567 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: { 2022-09-29T05:21:36.568 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "mgrmap_epoch": 11, 2022-09-29T05:21:36.568 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "initialized": true 2022-09-29T05:21:36.568 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: } 2022-09-29T05:21:36.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:21:35.553663+0000 mon.a (mon.0) 69 : cluster [DBG] mgrmap e10: y(active, starting, since 0.233571s) 2022-09-29T05:21:36.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:35.554668+0000 mon.a (mon.0) 70 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2022-09-29T05:21:36.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:35.555223+0000 mon.a (mon.0) 71 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2022-09-29T05:21:36.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:35.555969+0000 mon.a (mon.0) 72 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2022-09-29T05:21:36.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:35.556157+0000 mon.a (mon.0) 73 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2022-09-29T05:21:36.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:35.556284+0000 mon.a (mon.0) 74 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2022-09-29T05:21:36.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:21:35.569416+0000 mon.a (mon.0) 75 : cluster [INF] Manager daemon y is now available 2022-09-29T05:21:36.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:35.663413+0000 mon.a (mon.0) 76 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:21:36.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:35.664636+0000 mon.a (mon.0) 77 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:21:36.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:35.688337+0000 mon.a (mon.0) 78 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:21:36.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:35.689689+0000 mon.a (mon.0) 79 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:21:37.370 INFO:teuthology.orchestra.run.smithi083.stderr:mgr epoch 9 is available 2022-09-29T05:21:37.371 INFO:teuthology.orchestra.run.smithi083.stderr:Generating a dashboard self-signed certificate... 2022-09-29T05:21:37.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:37 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:36.557308+0000 mgr.y (mgr.14146) 1 : audit [DBG] from='client.14150 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2022-09-29T05:21:37.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:37 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:21:36.557983+0000 mon.a (mon.0) 80 : cluster [DBG] mgrmap e11: y(active, since 1.23789s) 2022-09-29T05:21:37.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:37 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:36.563577+0000 mgr.y (mgr.14146) 2 : audit [DBG] from='client.14150 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2022-09-29T05:21:37.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:37 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:37.533126+0000 mon.a (mon.0) 81 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:21:38.036 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: Self-signed certificate created 2022-09-29T05:21:38.509 INFO:teuthology.orchestra.run.smithi083.stderr:Creating initial admin user... 2022-09-29T05:21:38.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:38 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:21:37.566930+0000 mon.a (mon.0) 82 : cluster [DBG] mgrmap e12: y(active, since 2s) 2022-09-29T05:21:38.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:38 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:38.031114+0000 mon.a (mon.0) 83 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:21:38.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:38 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:38.033351+0000 mon.a (mon.0) 84 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:21:39.617 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: {"username": "admin", "password": "$2b$12$0mwqU3yYYvm0fZMpSCuIlOM7cauB0h73z3HDtpxhVLgmhYkz.IIP2", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1664428899, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2022-09-29T05:21:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:39 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:37.998571+0000 mgr.y (mgr.14146) 3 : audit [DBG] from='client.14156 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:21:40.047 INFO:teuthology.orchestra.run.smithi083.stderr:Fetching dashboard port number... 2022-09-29T05:21:40.633 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: 8443 2022-09-29T05:21:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:39.395900+0000 mgr.y (mgr.14146) 4 : audit [DBG] from='client.14158 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:21:40.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:39.613861+0000 mon.a (mon.0) 85 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:21:41.092 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: disabled 2022-09-29T05:21:41.100 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: inactive 2022-09-29T05:21:41.100 INFO:teuthology.orchestra.run.smithi083.stderr:firewalld.service is not enabled 2022-09-29T05:21:41.100 INFO:teuthology.orchestra.run.smithi083.stderr:Not possible to open ports <[8443]>. firewalld.service is not available 2022-09-29T05:21:41.102 INFO:teuthology.orchestra.run.smithi083.stderr:Ceph Dashboard is now available at: 2022-09-29T05:21:41.102 INFO:teuthology.orchestra.run.smithi083.stderr: 2022-09-29T05:21:41.102 INFO:teuthology.orchestra.run.smithi083.stderr: URL: https://smithi083:8443/ 2022-09-29T05:21:41.102 INFO:teuthology.orchestra.run.smithi083.stderr: User: admin 2022-09-29T05:21:41.103 INFO:teuthology.orchestra.run.smithi083.stderr: Password: 547jczimrd 2022-09-29T05:21:41.103 INFO:teuthology.orchestra.run.smithi083.stderr: 2022-09-29T05:21:41.103 INFO:teuthology.orchestra.run.smithi083.stderr:Enabling autotune for osd_memory_target 2022-09-29T05:21:41.872 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:40.630485+0000 mon.a (mon.0) 86 : audit [DBG] from='client.? 172.21.15.83:0/3342053454' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2022-09-29T05:21:42.858 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: set mgr/dashboard/cluster/status 2022-09-29T05:21:42.881 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:21:41.621191+0000 mon.a (mon.0) 87 : cluster [DBG] mgrmap e13: y(active, since 6s) 2022-09-29T05:21:43.373 INFO:teuthology.orchestra.run.smithi083.stderr:You can access the Ceph CLI as following in case of multi-cluster or non-default config: 2022-09-29T05:21:43.373 INFO:teuthology.orchestra.run.smithi083.stderr: 2022-09-29T05:21:43.373 INFO:teuthology.orchestra.run.smithi083.stderr: sudo /home/ubuntu/cephtest/cephadm shell --fsid 69111522-3fb6-11ed-8431-001a4aab830c -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2022-09-29T05:21:43.373 INFO:teuthology.orchestra.run.smithi083.stderr: 2022-09-29T05:21:43.374 INFO:teuthology.orchestra.run.smithi083.stderr:Or, if you are only running a single cluster on this host: 2022-09-29T05:21:43.374 INFO:teuthology.orchestra.run.smithi083.stderr: 2022-09-29T05:21:43.374 INFO:teuthology.orchestra.run.smithi083.stderr: sudo /home/ubuntu/cephtest/cephadm shell 2022-09-29T05:21:43.374 INFO:teuthology.orchestra.run.smithi083.stderr: 2022-09-29T05:21:43.375 INFO:teuthology.orchestra.run.smithi083.stderr:Please consider enabling telemetry to help improve Ceph: 2022-09-29T05:21:43.375 INFO:teuthology.orchestra.run.smithi083.stderr: 2022-09-29T05:21:43.375 INFO:teuthology.orchestra.run.smithi083.stderr: ceph telemetry on 2022-09-29T05:21:43.375 INFO:teuthology.orchestra.run.smithi083.stderr: 2022-09-29T05:21:43.376 INFO:teuthology.orchestra.run.smithi083.stderr:For more information see: 2022-09-29T05:21:43.376 INFO:teuthology.orchestra.run.smithi083.stderr: 2022-09-29T05:21:43.376 INFO:teuthology.orchestra.run.smithi083.stderr: https://docs.ceph.com/en/pacific/mgr/telemetry/ 2022-09-29T05:21:43.376 INFO:teuthology.orchestra.run.smithi083.stderr: 2022-09-29T05:21:43.376 INFO:teuthology.orchestra.run.smithi083.stderr:Bootstrap complete. 2022-09-29T05:21:43.409 INFO:tasks.cephadm:Fetching config... 2022-09-29T05:21:43.410 DEBUG:teuthology.orchestra.run.smithi083:> set -ex 2022-09-29T05:21:43.410 DEBUG:teuthology.orchestra.run.smithi083:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2022-09-29T05:21:43.430 INFO:tasks.cephadm:Fetching client.admin keyring... 2022-09-29T05:21:43.431 DEBUG:teuthology.orchestra.run.smithi083:> set -ex 2022-09-29T05:21:43.431 DEBUG:teuthology.orchestra.run.smithi083:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2022-09-29T05:21:43.488 INFO:tasks.cephadm:Fetching mon keyring... 2022-09-29T05:21:43.489 DEBUG:teuthology.orchestra.run.smithi083:> set -ex 2022-09-29T05:21:43.489 DEBUG:teuthology.orchestra.run.smithi083:> sudo dd if=/var/lib/ceph/69111522-3fb6-11ed-8431-001a4aab830c/mon.a/keyring of=/dev/stdout 2022-09-29T05:21:43.556 INFO:tasks.cephadm:Fetching pub ssh key... 2022-09-29T05:21:43.556 DEBUG:teuthology.orchestra.run.smithi083:> set -ex 2022-09-29T05:21:43.556 DEBUG:teuthology.orchestra.run.smithi083:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2022-09-29T05:21:43.615 INFO:tasks.cephadm:Installing pub ssh key for root users... 2022-09-29T05:21:43.616 DEBUG:teuthology.orchestra.run.smithi083:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC3SV2SQYCDaE+LfnVnNmOO2b4VrqjhBpbXhE/Y7OXx57l1As3nSekNOSy/h75pLckO2HhLcLUVwWWLwy3jdsfKdoQ4p1JJQ3RHM75Js6iJT1dJRLijs1MMXikRM3QlXW+9VY6h5uORQ0b033M8EPduoe/7UbVZ33p3zm+5nU21Lji5WvtC/y400+EgYlq1Y/+7GNeqD0Xj9gSqKXqYqzxSXGQi4NL0yq5gH1TuNiSPgDXKI23TVMFlMH7m1DtW0CYawcWNPPzQtjJc1zbuAM/uBguFGn19mVVS3RwhnH4U6Jyc8qNRUqldQ95q9rqQstUJq9Vsjwht7UCiewSjDhzLNswqf4b7+nIyN9pKx0rprnZ1zqY9DONv69KwOQuuyFdCbd+Q806o5r9vVrR/0QX5XikXCNP9I++Bhu5VLLD6xgpK+xYpLsG/jd4bhj08j4LROCPbzpOIumvdKExFa8EPMHE1LS81kef3jy/47xZWKXF0j9Itf+Qn/LFdC5gLoOE= ceph-69111522-3fb6-11ed-8431-001a4aab830c' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2022-09-29T05:21:43.695 INFO:teuthology.orchestra.run.smithi083.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC3SV2SQYCDaE+LfnVnNmOO2b4VrqjhBpbXhE/Y7OXx57l1As3nSekNOSy/h75pLckO2HhLcLUVwWWLwy3jdsfKdoQ4p1JJQ3RHM75Js6iJT1dJRLijs1MMXikRM3QlXW+9VY6h5uORQ0b033M8EPduoe/7UbVZ33p3zm+5nU21Lji5WvtC/y400+EgYlq1Y/+7GNeqD0Xj9gSqKXqYqzxSXGQi4NL0yq5gH1TuNiSPgDXKI23TVMFlMH7m1DtW0CYawcWNPPzQtjJc1zbuAM/uBguFGn19mVVS3RwhnH4U6Jyc8qNRUqldQ95q9rqQstUJq9Vsjwht7UCiewSjDhzLNswqf4b7+nIyN9pKx0rprnZ1zqY9DONv69KwOQuuyFdCbd+Q806o5r9vVrR/0QX5XikXCNP9I++Bhu5VLLD6xgpK+xYpLsG/jd4bhj08j4LROCPbzpOIumvdKExFa8EPMHE1LS81kef3jy/47xZWKXF0j9Itf+Qn/LFdC5gLoOE= ceph-69111522-3fb6-11ed-8431-001a4aab830c 2022-09-29T05:21:43.706 DEBUG:teuthology.orchestra.run.smithi153:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC3SV2SQYCDaE+LfnVnNmOO2b4VrqjhBpbXhE/Y7OXx57l1As3nSekNOSy/h75pLckO2HhLcLUVwWWLwy3jdsfKdoQ4p1JJQ3RHM75Js6iJT1dJRLijs1MMXikRM3QlXW+9VY6h5uORQ0b033M8EPduoe/7UbVZ33p3zm+5nU21Lji5WvtC/y400+EgYlq1Y/+7GNeqD0Xj9gSqKXqYqzxSXGQi4NL0yq5gH1TuNiSPgDXKI23TVMFlMH7m1DtW0CYawcWNPPzQtjJc1zbuAM/uBguFGn19mVVS3RwhnH4U6Jyc8qNRUqldQ95q9rqQstUJq9Vsjwht7UCiewSjDhzLNswqf4b7+nIyN9pKx0rprnZ1zqY9DONv69KwOQuuyFdCbd+Q806o5r9vVrR/0QX5XikXCNP9I++Bhu5VLLD6xgpK+xYpLsG/jd4bhj08j4LROCPbzpOIumvdKExFa8EPMHE1LS81kef3jy/47xZWKXF0j9Itf+Qn/LFdC5gLoOE= ceph-69111522-3fb6-11ed-8431-001a4aab830c' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2022-09-29T05:21:43.746 INFO:teuthology.orchestra.run.smithi153.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC3SV2SQYCDaE+LfnVnNmOO2b4VrqjhBpbXhE/Y7OXx57l1As3nSekNOSy/h75pLckO2HhLcLUVwWWLwy3jdsfKdoQ4p1JJQ3RHM75Js6iJT1dJRLijs1MMXikRM3QlXW+9VY6h5uORQ0b033M8EPduoe/7UbVZ33p3zm+5nU21Lji5WvtC/y400+EgYlq1Y/+7GNeqD0Xj9gSqKXqYqzxSXGQi4NL0yq5gH1TuNiSPgDXKI23TVMFlMH7m1DtW0CYawcWNPPzQtjJc1zbuAM/uBguFGn19mVVS3RwhnH4U6Jyc8qNRUqldQ95q9rqQstUJq9Vsjwht7UCiewSjDhzLNswqf4b7+nIyN9pKx0rprnZ1zqY9DONv69KwOQuuyFdCbd+Q806o5r9vVrR/0QX5XikXCNP9I++Bhu5VLLD6xgpK+xYpLsG/jd4bhj08j4LROCPbzpOIumvdKExFa8EPMHE1LS81kef3jy/47xZWKXF0j9Itf+Qn/LFdC5gLoOE= ceph-69111522-3fb6-11ed-8431-001a4aab830c 2022-09-29T05:21:43.757 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/daemon-base:latest-pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 69111522-3fb6-11ed-8431-001a4aab830c -- ceph config set mgr mgr/cephadm/allow_ptrace true 2022-09-29T05:21:44.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:42.852814+0000 mon.a (mon.0) 88 : audit [INF] from='client.? 172.21.15.83:0/2815260982' entity='client.admin' 2022-09-29T05:21:46.301 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2022-09-29T05:21:46.302 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/daemon-base:latest-pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 69111522-3fb6-11ed-8431-001a4aab830c -- ceph orch client-keyring set client.admin '*' --mode 0755 2022-09-29T05:21:47.047 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:45.775822+0000 mon.a (mon.0) 89 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:21:47.047 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:45.784207+0000 mon.a (mon.0) 90 : audit [INF] from='client.? 172.21.15.83:0/3056946270' entity='client.admin' 2022-09-29T05:21:47.047 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:46.428556+0000 mon.a (mon.0) 91 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi083", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:21:47.048 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:46.431459+0000 mon.a (mon.0) 92 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:21:47.048 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:46.432577+0000 mon.a (mon.0) 93 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:21:47.048 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:46.436389+0000 mon.a (mon.0) 94 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:21:48.778 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi153 2022-09-29T05:21:48.779 DEBUG:teuthology.orchestra.run.smithi153:> set -ex 2022-09-29T05:21:48.780 DEBUG:teuthology.orchestra.run.smithi153:> dd of=/etc/ceph/ceph.conf 2022-09-29T05:21:48.799 DEBUG:teuthology.orchestra.run.smithi153:> set -ex 2022-09-29T05:21:48.800 DEBUG:teuthology.orchestra.run.smithi153:> dd of=/etc/ceph/ceph.client.admin.keyring 2022-09-29T05:21:48.864 INFO:tasks.cephadm:Adding host smithi153 to orchestrator... 2022-09-29T05:21:48.864 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/daemon-base:latest-pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 69111522-3fb6-11ed-8431-001a4aab830c -- ceph orch host add smithi153 2022-09-29T05:21:49.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:48.305466+0000 mon.a (mon.0) 95 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:21:49.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:48.354411+0000 mon.a (mon.0) 96 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:21:49.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:48.355517+0000 mon.a (mon.0) 97 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:21:49.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:48.356136+0000 mon.a (mon.0) 98 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:21:49.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:21:48.357329+0000 mgr.y (mgr.14146) 6 : cephadm [INF] Updating smithi083:/etc/ceph/ceph.conf 2022-09-29T05:21:49.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:21:48.387087+0000 mgr.y (mgr.14146) 7 : cephadm [INF] Updating smithi083:/etc/ceph/ceph.client.admin.keyring 2022-09-29T05:21:49.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:48.431016+0000 mon.a (mon.0) 99 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:21:49.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:48.433746+0000 mon.a (mon.0) 100 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:21:50.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:21:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: [90B blob data] 2022-09-29T05:21:51.396 INFO:teuthology.orchestra.run.smithi083.stdout:Added host 'smithi153' with addr '172.21.15.153' 2022-09-29T05:21:51.437 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:50.634018+0000 mgr.y (mgr.14146) 8 : audit [DBG] from='client.14170 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi153", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:21:51.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:21:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: [80B blob data] 2022-09-29T05:21:51.899 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/daemon-base:latest-pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 69111522-3fb6-11ed-8431-001a4aab830c -- ceph orch host ls --format=json 2022-09-29T05:21:52.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:51.383117+0000 mon.a (mon.0) 101 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:21:52.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:21:51.393529+0000 mgr.y (mgr.14146) 9 : cephadm [INF] Added host smithi153 2022-09-29T05:21:52.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:51.404059+0000 mon.a (mon.0) 102 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:21:52.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:51.405323+0000 mon.a (mon.0) 103 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:21:52.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:51.406043+0000 mon.a (mon.0) 104 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:21:52.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:51.952826+0000 mon.a (mon.0) 105 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:21:53.657 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:21:53.658 INFO:teuthology.orchestra.run.smithi083.stdout:[{"addr": "172.21.15.83", "hostname": "smithi083", "labels": [], "status": ""}, {"addr": "172.21.15.153", "hostname": "smithi153", "labels": [], "status": ""}] 2022-09-29T05:21:53.679 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:52.406669+0000 mon.a (mon.0) 106 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:21:54.121 INFO:tasks.cephadm:Setting crush tunables to default 2022-09-29T05:21:54.122 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/daemon-base:latest-pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 69111522-3fb6-11ed-8431-001a4aab830c -- ceph osd crush tunables default 2022-09-29T05:21:54.707 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:53.653556+0000 mgr.y (mgr.14146) 10 : audit [DBG] from='client.14172 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2022-09-29T05:21:56.571 INFO:teuthology.orchestra.run.smithi083.stderr:adjusted tunables profile to default 2022-09-29T05:21:56.591 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:55.303800+0000 mon.a (mon.0) 107 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:21:56.591 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:21:55.557194+0000 mgr.y (mgr.14146) 11 : cluster [DBG] pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2022-09-29T05:21:56.591 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:55.779942+0000 mon.a (mon.0) 108 : audit [INF] from='client.? 172.21.15.83:0/1719885960' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2022-09-29T05:21:56.592 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:55.951401+0000 mon.a (mon.0) 109 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:21:56.592 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:55.951890+0000 mon.a (mon.0) 110 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi153", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:21:56.592 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:55.954984+0000 mon.a (mon.0) 111 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:21:57.055 INFO:tasks.cephadm:Adding mon.a on smithi083 2022-09-29T05:21:57.056 INFO:tasks.cephadm:Adding mon.c on smithi083 2022-09-29T05:21:57.056 INFO:tasks.cephadm:Adding mon.b on smithi153 2022-09-29T05:21:57.056 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/daemon-base:latest-pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 69111522-3fb6-11ed-8431-001a4aab830c -- ceph orch apply mon '3;smithi083:172.21.15.83=a;smithi083:[v2:172.21.15.83:3301,v1:172.21.15.83:6790]=c;smithi153:172.21.15.153=b' 2022-09-29T05:21:57.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:57 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:56.565693+0000 mon.a (mon.0) 112 : audit [INF] from='client.? 172.21.15.83:0/1719885960' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2022-09-29T05:21:57.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:57 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:21:56.565756+0000 mon.a (mon.0) 113 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2022-09-29T05:21:58.734 INFO:teuthology.orchestra.run.smithi153.stdout:Scheduled mon update... 2022-09-29T05:21:58.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:21:57.557540+0000 mgr.y (mgr.14146) 12 : cluster [DBG] pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2022-09-29T05:21:59.214 DEBUG:teuthology.orchestra.run.smithi083:mon.c> sudo journalctl -f -n 0 -u ceph-69111522-3fb6-11ed-8431-001a4aab830c@mon.c.service 2022-09-29T05:21:59.217 DEBUG:teuthology.orchestra.run.smithi153:mon.b> sudo journalctl -f -n 0 -u ceph-69111522-3fb6-11ed-8431-001a4aab830c@mon.b.service 2022-09-29T05:21:59.220 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2022-09-29T05:21:59.220 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/daemon-base:latest-pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 69111522-3fb6-11ed-8431-001a4aab830c -- ceph mon dump -f json 2022-09-29T05:21:59.248 INFO:journalctl@ceph.mon.c.smithi083.stdout:-- Logs begin at Thu 2022-09-29 05:12:45 UTC. -- 2022-09-29T05:21:59.249 INFO:journalctl@ceph.mon.b.smithi153.stdout:-- Logs begin at Thu 2022-09-29 05:12:53 UTC. -- 2022-09-29T05:21:59.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:59 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:58.726528+0000 mgr.y (mgr.14146) 13 : audit [DBG] from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi083:172.21.15.83=a;smithi083:[v2:172.21.15.83:3301,v1:172.21.15.83:6790]=c;smithi153:172.21.15.153=b", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:21:59.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:59 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:21:58.728688+0000 mgr.y (mgr.14146) 14 : cephadm [INF] Saving service mon spec with placement smithi083:172.21.15.83=a;smithi083:[v2:172.21.15.83:3301,v1:172.21.15.83:6790]=c;smithi153:172.21.15.153=b;count:3 2022-09-29T05:21:59.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:59 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:58.731105+0000 mon.a (mon.0) 114 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:21:59.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:59 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:58.775249+0000 mon.a (mon.0) 115 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:21:59.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:59 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:58.776417+0000 mon.a (mon.0) 116 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:21:59.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:59 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:58.777187+0000 mon.a (mon.0) 117 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:21:59.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:59 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:21:58.778369+0000 mgr.y (mgr.14146) 15 : cephadm [INF] Updating smithi153:/etc/ceph/ceph.conf 2022-09-29T05:21:59.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:59 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:21:58.800411+0000 mgr.y (mgr.14146) 16 : cephadm [INF] Updating smithi153:/etc/ceph/ceph.client.admin.keyring 2022-09-29T05:21:59.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:59 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:58.836634+0000 mon.a (mon.0) 118 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:21:59.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:59 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:58.839332+0000 mon.a (mon.0) 119 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:21:59.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:59 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:58.840871+0000 mon.a (mon.0) 120 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2022-09-29T05:21:59.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:59 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:21:58.841688+0000 mon.a (mon.0) 121 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:21:59.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:21:59 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:21:58.842407+0000 mgr.y (mgr.14146) 17 : cephadm [INF] Deploying daemon mon.c on smithi083 2022-09-29T05:22:00.799 INFO:teuthology.orchestra.run.smithi153.stdout: 2022-09-29T05:22:00.800 INFO:teuthology.orchestra.run.smithi153.stdout:{"epoch":1,"fsid":"69111522-3fb6-11ed-8431-001a4aab830c","modified":"2022-09-29T05:20:42.317979Z","created":"2022-09-29T05:20:42.317979Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:3300","nonce":0},{"type":"v1","addr":"172.21.15.83:6789","nonce":0}]},"addr":"172.21.15.83:6789/0","public_addr":"172.21.15.83:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2022-09-29T05:22:00.801 INFO:teuthology.orchestra.run.smithi153.stderr:dumped monmap epoch 1 2022-09-29T05:22:01.028 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:21:59.557816+0000 mgr.y (mgr.14146) 18 : cluster [DBG] pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2022-09-29T05:22:01.879 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:00.795722+0000 mon.a (mon.0) 122 : audit [DBG] from='client.? 172.21.15.153:0/2080035991' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2022-09-29T05:22:01.881 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 systemd[1]: Starting Ceph mon.c for 69111522-3fb6-11ed-8431-001a4aab830c... 2022-09-29T05:22:01.881 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 podman[44044]: 2022-09-29T05:22:01.882 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.850+0000 7f65ecc44880 0 set uid:gid to 167:167 (ceph:ceph) 2022-09-29T05:22:01.882 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.850+0000 7f65ecc44880 0 ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable), process ceph-mon, pid 7 2022-09-29T05:22:01.882 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.850+0000 7f65ecc44880 0 pidfile_write: ignore empty --pid-file 2022-09-29T05:22:01.883 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 0 load: jerasure load: lrc load: isa 2022-09-29T05:22:01.883 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: RocksDB version: 6.8.1 2022-09-29T05:22:01.883 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: 2022-09-29T05:22:01.883 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2022-09-29T05:22:01.883 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: Compile date Jul 21 2022 2022-09-29T05:22:01.884 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: DB SUMMARY 2022-09-29T05:22:01.884 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: 2022-09-29T05:22:01.884 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: CURRENT file: CURRENT 2022-09-29T05:22:01.884 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: 2022-09-29T05:22:01.885 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: IDENTITY file: IDENTITY 2022-09-29T05:22:01.885 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: 2022-09-29T05:22:01.885 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: MANIFEST file: MANIFEST-000001 size: 13 Bytes 2022-09-29T05:22:01.885 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: 2022-09-29T05:22:01.886 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 0, files: 2022-09-29T05:22:01.886 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: 2022-09-29T05:22:01.886 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000003.log size: 511 ; 2022-09-29T05:22:01.886 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: 2022-09-29T05:22:01.887 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: Options.error_if_exists: 0 2022-09-29T05:22:01.887 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: Options.create_if_missing: 0 2022-09-29T05:22:01.887 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: Options.paranoid_checks: 1 2022-09-29T05:22:01.887 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: Options.env: 0x55dfec269000 2022-09-29T05:22:01.888 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: Options.fs: Posix File System 2022-09-29T05:22:01.888 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: Options.info_log: 0x55dfed33ba20 2022-09-29T05:22:01.888 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: Options.max_file_opening_threads: 16 2022-09-29T05:22:01.888 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: Options.statistics: (nil) 2022-09-29T05:22:01.888 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: Options.use_fsync: 0 2022-09-29T05:22:01.889 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: Options.max_log_file_size: 0 2022-09-29T05:22:01.889 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: Options.max_manifest_file_size: 1073741824 2022-09-29T05:22:01.889 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: Options.log_file_time_to_roll: 0 2022-09-29T05:22:01.889 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: Options.keep_log_file_num: 1000 2022-09-29T05:22:01.890 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: Options.recycle_log_file_num: 0 2022-09-29T05:22:01.890 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: Options.allow_fallocate: 1 2022-09-29T05:22:01.890 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: Options.allow_mmap_reads: 0 2022-09-29T05:22:01.890 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: Options.allow_mmap_writes: 0 2022-09-29T05:22:01.891 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: Options.use_direct_reads: 0 2022-09-29T05:22:01.891 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2022-09-29T05:22:01.891 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: Options.create_missing_column_families: 0 2022-09-29T05:22:01.891 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: Options.db_log_dir: 2022-09-29T05:22:01.892 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-c/store.db 2022-09-29T05:22:01.892 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: Options.table_cache_numshardbits: 6 2022-09-29T05:22:01.892 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: Options.max_subcompactions: 1 2022-09-29T05:22:01.892 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: Options.max_background_flushes: -1 2022-09-29T05:22:01.892 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: Options.WAL_ttl_seconds: 0 2022-09-29T05:22:01.893 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: Options.WAL_size_limit_MB: 0 2022-09-29T05:22:01.893 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2022-09-29T05:22:01.893 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: Options.manifest_preallocation_size: 4194304 2022-09-29T05:22:01.893 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: Options.is_fd_close_on_exec: 1 2022-09-29T05:22:01.894 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: Options.advise_random_on_open: 1 2022-09-29T05:22:01.894 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: Options.db_write_buffer_size: 0 2022-09-29T05:22:01.894 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: Options.write_buffer_manager: 0x55dfee044120 2022-09-29T05:22:01.894 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: Options.access_hint_on_compaction_start: 1 2022-09-29T05:22:01.895 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2022-09-29T05:22:01.895 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2022-09-29T05:22:01.895 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: Options.use_adaptive_mutex: 0 2022-09-29T05:22:01.895 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: Options.rate_limiter: (nil) 2022-09-29T05:22:01.895 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2022-09-29T05:22:01.896 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: Options.wal_recovery_mode: 2 2022-09-29T05:22:01.896 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: Options.enable_thread_tracking: 0 2022-09-29T05:22:01.896 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: Options.enable_pipelined_write: 0 2022-09-29T05:22:01.896 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: Options.unordered_write: 0 2022-09-29T05:22:01.896 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2022-09-29T05:22:01.897 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2022-09-29T05:22:01.897 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: Options.write_thread_max_yield_usec: 100 2022-09-29T05:22:01.897 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2022-09-29T05:22:01.897 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: Options.row_cache: None 2022-09-29T05:22:01.897 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: Options.wal_filter: None 2022-09-29T05:22:01.898 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: Options.avoid_flush_during_recovery: 0 2022-09-29T05:22:01.898 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: Options.allow_ingest_behind: 0 2022-09-29T05:22:01.898 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: Options.preserve_deletes: 0 2022-09-29T05:22:01.898 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: Options.two_write_queues: 0 2022-09-29T05:22:01.899 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: Options.manual_wal_flush: 0 2022-09-29T05:22:01.899 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: Options.atomic_flush: 0 2022-09-29T05:22:01.899 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2022-09-29T05:22:01.899 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: Options.persist_stats_to_disk: 0 2022-09-29T05:22:01.899 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: Options.write_dbid_to_manifest: 0 2022-09-29T05:22:01.900 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: Options.log_readahead_size: 0 2022-09-29T05:22:01.900 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: Options.sst_file_checksum_func: Unknown 2022-09-29T05:22:01.900 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: Options.max_background_jobs: 2 2022-09-29T05:22:01.900 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: Options.max_background_compactions: -1 2022-09-29T05:22:01.900 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2022-09-29T05:22:01.901 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2022-09-29T05:22:01.901 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: Options.delayed_write_rate : 16777216 2022-09-29T05:22:01.901 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: Options.max_total_wal_size: 0 2022-09-29T05:22:01.901 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2022-09-29T05:22:01.901 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: Options.stats_dump_period_sec: 600 2022-09-29T05:22:01.902 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: Options.stats_persist_period_sec: 600 2022-09-29T05:22:01.902 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: Options.stats_history_buffer_size: 1048576 2022-09-29T05:22:01.902 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: Options.max_open_files: -1 2022-09-29T05:22:01.902 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: Options.bytes_per_sync: 0 2022-09-29T05:22:01.903 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: Options.wal_bytes_per_sync: 0 2022-09-29T05:22:01.903 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: Options.strict_bytes_per_sync: 0 2022-09-29T05:22:01.903 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: Options.compaction_readahead_size: 0 2022-09-29T05:22:01.903 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: Compression algorithms supported: 2022-09-29T05:22:01.904 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: kZSTDNotFinalCompression supported: 0 2022-09-29T05:22:01.904 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: kZSTD supported: 0 2022-09-29T05:22:01.904 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: kXpressCompression supported: 0 2022-09-29T05:22:01.904 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: kLZ4HCCompression supported: 1 2022-09-29T05:22:01.905 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: kLZ4Compression supported: 1 2022-09-29T05:22:01.905 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: kBZip2Compression supported: 0 2022-09-29T05:22:01.905 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: kZlibCompression supported: 1 2022-09-29T05:22:01.905 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: kSnappyCompression supported: 1 2022-09-29T05:22:01.905 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.855+0000 7f65ecc44880 4 rocksdb: Fast CRC32 supported: Supported on x86 2022-09-29T05:22:01.906 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.856+0000 7f65ecc44880 4 rocksdb: [version_set.cc:4413] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000001 2022-09-29T05:22:01.906 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: 2022-09-29T05:22:01.906 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.856+0000 7f65ecc44880 4 rocksdb: [column_family.cc:552] --------------- Options for column family [default]: 2022-09-29T05:22:01.906 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: 2022-09-29T05:22:01.907 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.856+0000 7f65ecc44880 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2022-09-29T05:22:01.907 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.856+0000 7f65ecc44880 4 rocksdb: Options.merge_operator: 2022-09-29T05:22:01.907 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.856+0000 7f65ecc44880 4 rocksdb: Options.compaction_filter: None 2022-09-29T05:22:01.907 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.856+0000 7f65ecc44880 4 rocksdb: Options.compaction_filter_factory: None 2022-09-29T05:22:01.908 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.856+0000 7f65ecc44880 4 rocksdb: Options.memtable_factory: SkipListFactory 2022-09-29T05:22:01.908 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.856+0000 7f65ecc44880 4 rocksdb: Options.table_factory: BlockBasedTable 2022-09-29T05:22:01.908 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.856+0000 7f65ecc44880 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55dfed2e5d10) 2022-09-29T05:22:01.908 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cache_index_and_filter_blocks: 1 2022-09-29T05:22:01.908 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cache_index_and_filter_blocks_with_high_priority: 0 2022-09-29T05:22:01.909 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: pin_l0_filter_and_index_blocks_in_cache: 0 2022-09-29T05:22:01.909 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: pin_top_level_index_and_filter: 1 2022-09-29T05:22:01.909 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: index_type: 0 2022-09-29T05:22:01.909 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: data_block_index_type: 0 2022-09-29T05:22:01.909 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: index_shortening: 1 2022-09-29T05:22:01.910 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: data_block_hash_table_util_ratio: 0.750000 2022-09-29T05:22:01.910 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: hash_index_allow_collision: 1 2022-09-29T05:22:01.910 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: checksum: 1 2022-09-29T05:22:01.910 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: no_block_cache: 0 2022-09-29T05:22:01.911 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: block_cache: 0x55dfed31ef10 2022-09-29T05:22:01.911 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: block_cache_name: BinnedLRUCache 2022-09-29T05:22:01.911 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: block_cache_options: 2022-09-29T05:22:01.911 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: capacity : 536870912 2022-09-29T05:22:01.911 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: num_shard_bits : 4 2022-09-29T05:22:01.912 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: strict_capacity_limit : 0 2022-09-29T05:22:01.912 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: high_pri_pool_ratio: 0.000 2022-09-29T05:22:01.912 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: block_cache_compressed: (nil) 2022-09-29T05:22:01.912 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: persistent_cache: (nil) 2022-09-29T05:22:01.912 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: block_size: 4096 2022-09-29T05:22:01.913 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: block_size_deviation: 10 2022-09-29T05:22:01.913 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: block_restart_interval: 16 2022-09-29T05:22:01.913 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: index_block_restart_interval: 1 2022-09-29T05:22:01.913 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: metadata_block_size: 4096 2022-09-29T05:22:01.914 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: partition_filters: 0 2022-09-29T05:22:01.914 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: use_delta_encoding: 1 2022-09-29T05:22:01.915 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: filter_policy: rocksdb.BuiltinBloomFilter 2022-09-29T05:22:01.916 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: whole_key_filtering: 1 2022-09-29T05:22:01.916 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: verify_compression: 0 2022-09-29T05:22:01.916 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: read_amp_bytes_per_bit: 0 2022-09-29T05:22:01.917 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: format_version: 2 2022-09-29T05:22:01.917 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: enable_index_compression: 1 2022-09-29T05:22:01.917 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: block_align: 0 2022-09-29T05:22:01.917 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: 2022-09-29T05:22:01.918 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.856+0000 7f65ecc44880 4 rocksdb: Options.write_buffer_size: 33554432 2022-09-29T05:22:01.918 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.856+0000 7f65ecc44880 4 rocksdb: Options.max_write_buffer_number: 2 2022-09-29T05:22:01.918 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.856+0000 7f65ecc44880 4 rocksdb: Options.compression: NoCompression 2022-09-29T05:22:01.918 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.856+0000 7f65ecc44880 4 rocksdb: Options.bottommost_compression: Disabled 2022-09-29T05:22:01.919 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.856+0000 7f65ecc44880 4 rocksdb: Options.prefix_extractor: nullptr 2022-09-29T05:22:01.919 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.856+0000 7f65ecc44880 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2022-09-29T05:22:01.919 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.856+0000 7f65ecc44880 4 rocksdb: Options.num_levels: 7 2022-09-29T05:22:01.919 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.856+0000 7f65ecc44880 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2022-09-29T05:22:01.919 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.856+0000 7f65ecc44880 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2022-09-29T05:22:01.920 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.856+0000 7f65ecc44880 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2022-09-29T05:22:01.920 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.856+0000 7f65ecc44880 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2022-09-29T05:22:01.920 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.856+0000 7f65ecc44880 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2022-09-29T05:22:01.920 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.856+0000 7f65ecc44880 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2022-09-29T05:22:01.921 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.856+0000 7f65ecc44880 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2022-09-29T05:22:01.921 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.856+0000 7f65ecc44880 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2022-09-29T05:22:01.921 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.856+0000 7f65ecc44880 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2022-09-29T05:22:01.921 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.856+0000 7f65ecc44880 4 rocksdb: Options.compression_opts.window_bits: -14 2022-09-29T05:22:01.922 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.856+0000 7f65ecc44880 4 rocksdb: Options.compression_opts.level: 32767 2022-09-29T05:22:01.922 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.856+0000 7f65ecc44880 4 rocksdb: Options.compression_opts.strategy: 0 2022-09-29T05:22:01.922 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.856+0000 7f65ecc44880 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2022-09-29T05:22:01.922 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.856+0000 7f65ecc44880 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2022-09-29T05:22:01.923 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.856+0000 7f65ecc44880 4 rocksdb: Options.compression_opts.enabled: false 2022-09-29T05:22:01.923 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.856+0000 7f65ecc44880 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2022-09-29T05:22:01.923 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.856+0000 7f65ecc44880 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2022-09-29T05:22:01.923 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.856+0000 7f65ecc44880 4 rocksdb: Options.level0_stop_writes_trigger: 36 2022-09-29T05:22:01.923 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.856+0000 7f65ecc44880 4 rocksdb: Options.target_file_size_base: 67108864 2022-09-29T05:22:01.924 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.856+0000 7f65ecc44880 4 rocksdb: Options.target_file_size_multiplier: 1 2022-09-29T05:22:01.924 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.856+0000 7f65ecc44880 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2022-09-29T05:22:01.924 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.856+0000 7f65ecc44880 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2022-09-29T05:22:01.924 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.856+0000 7f65ecc44880 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2022-09-29T05:22:01.925 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.856+0000 7f65ecc44880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2022-09-29T05:22:01.925 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.856+0000 7f65ecc44880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2022-09-29T05:22:01.925 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.856+0000 7f65ecc44880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2022-09-29T05:22:01.925 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.856+0000 7f65ecc44880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2022-09-29T05:22:01.925 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.856+0000 7f65ecc44880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2022-09-29T05:22:01.926 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.856+0000 7f65ecc44880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2022-09-29T05:22:01.926 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.856+0000 7f65ecc44880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2022-09-29T05:22:01.926 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.856+0000 7f65ecc44880 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2022-09-29T05:22:01.926 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.856+0000 7f65ecc44880 4 rocksdb: Options.max_compaction_bytes: 1677721600 2022-09-29T05:22:01.927 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.856+0000 7f65ecc44880 4 rocksdb: Options.arena_block_size: 4194304 2022-09-29T05:22:01.927 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.856+0000 7f65ecc44880 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2022-09-29T05:22:01.927 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.856+0000 7f65ecc44880 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2022-09-29T05:22:01.927 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.856+0000 7f65ecc44880 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2022-09-29T05:22:01.927 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.856+0000 7f65ecc44880 4 rocksdb: Options.disable_auto_compactions: 0 2022-09-29T05:22:01.928 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.856+0000 7f65ecc44880 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2022-09-29T05:22:01.928 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.856+0000 7f65ecc44880 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2022-09-29T05:22:01.928 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.856+0000 7f65ecc44880 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2022-09-29T05:22:01.928 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.856+0000 7f65ecc44880 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2022-09-29T05:22:01.929 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.856+0000 7f65ecc44880 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2022-09-29T05:22:01.929 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.856+0000 7f65ecc44880 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2022-09-29T05:22:01.929 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.856+0000 7f65ecc44880 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2022-09-29T05:22:01.929 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.856+0000 7f65ecc44880 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2022-09-29T05:22:01.930 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.856+0000 7f65ecc44880 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2022-09-29T05:22:01.930 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.856+0000 7f65ecc44880 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2022-09-29T05:22:01.930 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.856+0000 7f65ecc44880 4 rocksdb: Options.table_properties_collectors: 2022-09-29T05:22:01.930 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.856+0000 7f65ecc44880 4 rocksdb: Options.inplace_update_support: 0 2022-09-29T05:22:01.930 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.856+0000 7f65ecc44880 4 rocksdb: Options.inplace_update_num_locks: 10000 2022-09-29T05:22:01.931 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.856+0000 7f65ecc44880 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2022-09-29T05:22:01.931 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.856+0000 7f65ecc44880 4 rocksdb: Options.memtable_whole_key_filtering: 0 2022-09-29T05:22:01.931 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.856+0000 7f65ecc44880 4 rocksdb: Options.memtable_huge_page_size: 0 2022-09-29T05:22:01.931 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.856+0000 7f65ecc44880 4 rocksdb: Options.bloom_locality: 0 2022-09-29T05:22:01.931 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.856+0000 7f65ecc44880 4 rocksdb: Options.max_successive_merges: 0 2022-09-29T05:22:01.932 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.856+0000 7f65ecc44880 4 rocksdb: Options.optimize_filters_for_hits: 0 2022-09-29T05:22:01.932 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.856+0000 7f65ecc44880 4 rocksdb: Options.paranoid_file_checks: 0 2022-09-29T05:22:01.932 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.856+0000 7f65ecc44880 4 rocksdb: Options.force_consistency_checks: 0 2022-09-29T05:22:01.932 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.856+0000 7f65ecc44880 4 rocksdb: Options.report_bg_io_stats: 0 2022-09-29T05:22:01.933 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.856+0000 7f65ecc44880 4 rocksdb: Options.ttl: 2592000 2022-09-29T05:22:01.933 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.856+0000 7f65ecc44880 4 rocksdb: Options.periodic_compaction_seconds: 0 2022-09-29T05:22:01.933 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.856+0000 7f65ecc44880 4 rocksdb: [version_set.cc:4568] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000001 succeeded,manifest_file_number is 1, next_file_number is 3, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2022-09-29T05:22:01.933 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: 2022-09-29T05:22:01.933 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.856+0000 7f65ecc44880 4 rocksdb: [version_set.cc:4577] Column family [default] (ID 0), log number is 0 2022-09-29T05:22:01.934 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: 2022-09-29T05:22:01.934 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.856+0000 7f65ecc44880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1664428921857932, "job": 1, "event": "recovery_started", "log_files": [3]} 2022-09-29T05:22:01.934 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.856+0000 7f65ecc44880 4 rocksdb: [db_impl/db_impl_open.cc:760] Recovering log #3 mode 2 2022-09-29T05:22:01.934 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.857+0000 7f65ecc44880 3 rocksdb: [le/block_based/filter_policy.cc:584] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2022-09-29T05:22:01.935 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.857+0000 7f65ecc44880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1664428921858381, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 4, "file_size": 1437, "table_properties": {"data_size": 523, "index_size": 40, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 0, "index_value_is_delta_encoded": 0, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1664428921, "oldest_key_time": 3, "file_creation_time": 0}} 2022-09-29T05:22:01.935 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.857+0000 7f65ecc44880 4 rocksdb: [version_set.cc:3826] Creating manifest 5 2022-09-29T05:22:01.935 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: 2022-09-29T05:22:01.935 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.857+0000 7f65ecc44880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1664428921858759, "job": 1, "event": "recovery_finished"} 2022-09-29T05:22:01.936 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.858+0000 7f65ecc44880 4 rocksdb: DB pointer 0x55dfed3e9800 2022-09-29T05:22:01.936 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.858+0000 7f65d3a5e700 4 rocksdb: [db_impl/db_impl.cc:850] ------- DUMPING STATS ------- 2022-09-29T05:22:01.936 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.858+0000 7f65d3a5e700 4 rocksdb: [db_impl/db_impl.cc:851] 2022-09-29T05:22:01.936 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: ** DB Stats ** 2022-09-29T05:22:01.937 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: Uptime(secs): 0.0 total, 0.0 interval 2022-09-29T05:22:01.937 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2022-09-29T05:22:01.937 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2022-09-29T05:22:01.937 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2022-09-29T05:22:01.938 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2022-09-29T05:22:01.938 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2022-09-29T05:22:01.938 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2022-09-29T05:22:01.938 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: 2022-09-29T05:22:01.938 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: ** Compaction Stats [default] ** 2022-09-29T05:22:01.939 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2022-09-29T05:22:01.939 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2022-09-29T05:22:01.939 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: L0 1/0 1.40 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.4 0.00 0.00 1 0.000 0 0 2022-09-29T05:22:01.939 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: Sum 1/0 1.40 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.4 0.00 0.00 1 0.000 0 0 2022-09-29T05:22:01.939 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.4 0.00 0.00 1 0.000 0 0 2022-09-29T05:22:01.940 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: 2022-09-29T05:22:01.940 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: ** Compaction Stats [default] ** 2022-09-29T05:22:01.940 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2022-09-29T05:22:01.940 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2022-09-29T05:22:01.940 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 3.4 0.00 0.00 1 0.000 0 0 2022-09-29T05:22:01.941 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: Uptime(secs): 0.0 total, 0.0 interval 2022-09-29T05:22:01.941 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: Flush(GB): cumulative 0.000, interval 0.000 2022-09-29T05:22:01.941 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: AddFile(GB): cumulative 0.000, interval 0.000 2022-09-29T05:22:01.941 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: AddFile(Total Files): cumulative 0, interval 0 2022-09-29T05:22:01.942 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: AddFile(L0 Files): cumulative 0, interval 0 2022-09-29T05:22:01.942 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: AddFile(Keys): cumulative 0, interval 0 2022-09-29T05:22:01.942 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: Cumulative compaction: 0.00 GB write, 0.58 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2022-09-29T05:22:01.942 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: Interval compaction: 0.00 GB write, 0.58 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2022-09-29T05:22:01.943 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2022-09-29T05:22:01.943 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: 2022-09-29T05:22:01.943 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: ** File Read Latency Histogram By Level [default] ** 2022-09-29T05:22:01.943 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: 2022-09-29T05:22:01.943 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: ** Compaction Stats [default] ** 2022-09-29T05:22:01.944 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2022-09-29T05:22:01.944 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2022-09-29T05:22:01.944 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: L0 1/0 1.40 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.4 0.00 0.00 1 0.000 0 0 2022-09-29T05:22:01.944 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: Sum 1/0 1.40 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.4 0.00 0.00 1 0.000 0 0 2022-09-29T05:22:01.944 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2022-09-29T05:22:01.945 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: 2022-09-29T05:22:01.945 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: ** Compaction Stats [default] ** 2022-09-29T05:22:01.945 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2022-09-29T05:22:01.945 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2022-09-29T05:22:01.945 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 3.4 0.00 0.00 1 0.000 0 0 2022-09-29T05:22:01.946 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: Uptime(secs): 0.0 total, 0.0 interval 2022-09-29T05:22:01.946 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: Flush(GB): cumulative 0.000, interval 0.000 2022-09-29T05:22:01.946 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: AddFile(GB): cumulative 0.000, interval 0.000 2022-09-29T05:22:01.946 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: AddFile(Total Files): cumulative 0, interval 0 2022-09-29T05:22:01.947 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: AddFile(L0 Files): cumulative 0, interval 0 2022-09-29T05:22:01.947 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: AddFile(Keys): cumulative 0, interval 0 2022-09-29T05:22:01.947 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: Cumulative compaction: 0.00 GB write, 0.57 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2022-09-29T05:22:01.947 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2022-09-29T05:22:01.947 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2022-09-29T05:22:01.948 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: 2022-09-29T05:22:01.948 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: ** File Read Latency Histogram By Level [default] ** 2022-09-29T05:22:01.948 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: 2022-09-29T05:22:01.948 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.858+0000 7f65ecc44880 0 mon.c does not exist in monmap, will attempt to join an existing cluster 2022-09-29T05:22:01.949 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.858+0000 7f65ecc44880 0 using public_addrv [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] 2022-09-29T05:22:01.949 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.859+0000 7f65ecc44880 0 starting mon.c rank -1 at public addrs [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] at bind addrs [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 69111522-3fb6-11ed-8431-001a4aab830c 2022-09-29T05:22:01.949 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.860+0000 7f65ecc44880 1 mon.c@-1(???) e0 preinit fsid 69111522-3fb6-11ed-8431-001a4aab830c 2022-09-29T05:22:01.949 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.873+0000 7f65d6263700 0 mon.c@-1(synchronizing).mds e1 new map 2022-09-29T05:22:01.950 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.873+0000 7f65d6263700 0 mon.c@-1(synchronizing).mds e1 print_map 2022-09-29T05:22:01.950 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: e1 2022-09-29T05:22:01.950 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: enable_multiple, ever_enabled_multiple: 1,1 2022-09-29T05:22:01.950 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2022-09-29T05:22:01.951 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: legacy client fscid: -1 2022-09-29T05:22:01.951 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: 2022-09-29T05:22:01.951 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: No filesystems configured 2022-09-29T05:22:01.951 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: 2022-09-29T05:22:01.951 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.873+0000 7f65d6263700 1 mon.c@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2022-09-29T05:22:01.952 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.874+0000 7f65d6263700 1 mon.c@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2022-09-29T05:22:01.952 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.874+0000 7f65d6263700 1 mon.c@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2022-09-29T05:22:01.952 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.874+0000 7f65d6263700 1 mon.c@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2022-09-29T05:22:01.952 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.874+0000 7f65d6263700 1 mon.c@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2022-09-29T05:22:01.953 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.874+0000 7f65d6263700 1 mon.c@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2022-09-29T05:22:01.953 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.874+0000 7f65d6263700 0 mon.c@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2022-09-29T05:22:01.953 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.874+0000 7f65d6263700 0 mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2022-09-29T05:22:01.953 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.874+0000 7f65d6263700 0 mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2022-09-29T05:22:01.953 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.874+0000 7f65d6263700 0 mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2022-09-29T05:22:01.954 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:00.795722+0000 mon.a (mon.0) 122 : audit [DBG] from='client.? 172.21.15.153:0/2080035991' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2022-09-29T05:22:01.954 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.874+0000 7f65d6263700 1 mon.c@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2022-09-29T05:22:01.954 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:22:01.876+0000 7f65d6263700 10 mon.c@-1(synchronizing) e1 handle_conf_change mon_allow_pool_delete 2022-09-29T05:22:01.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 bash[44044]: a7179c86361b5ca1ef15cdd8880cc1643b059ed5bfb76f1e2732c7b93e5f1972 2022-09-29T05:22:02.200 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2022-09-29T05:22:02.201 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/daemon-base:latest-pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 69111522-3fb6-11ed-8431-001a4aab830c -- ceph mon dump -f json 2022-09-29T05:22:02.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:01 smithi083 systemd[1]: Started Ceph mon.c for 69111522-3fb6-11ed-8431-001a4aab830c. 2022-09-29T05:22:07.394 INFO:teuthology.orchestra.run.smithi153.stdout: 2022-09-29T05:22:07.395 INFO:teuthology.orchestra.run.smithi153.stdout:{"epoch":2,"fsid":"69111522-3fb6-11ed-8431-001a4aab830c","modified":"2022-09-29T05:22:01.880351Z","created":"2022-09-29T05:20:42.317979Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:3300","nonce":0},{"type":"v1","addr":"172.21.15.83:6789","nonce":0}]},"addr":"172.21.15.83:6789/0","public_addr":"172.21.15.83:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:3301","nonce":0},{"type":"v1","addr":"172.21.15.83:6790","nonce":0}]},"addr":"172.21.15.83:6790/0","public_addr":"172.21.15.83:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2022-09-29T05:22:07.396 INFO:teuthology.orchestra.run.smithi153.stderr:dumped monmap epoch 2 2022-09-29T05:22:07.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:22:01.558082+0000 mgr.y (mgr.14146) 19 : cluster [DBG] pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2022-09-29T05:22:07.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:01.882330+0000 mon.a (mon.0) 124 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2022-09-29T05:22:07.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:22:01.882509+0000 mon.a (mon.0) 125 : cluster [INF] mon.a calling monitor election 2022-09-29T05:22:07.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:01.894720+0000 mon.a (mon.0) 126 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2022-09-29T05:22:07.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:02.879106+0000 mon.a (mon.0) 127 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2022-09-29T05:22:07.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:22:03.558419+0000 mgr.y (mgr.14146) 20 : cluster [DBG] pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2022-09-29T05:22:07.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:03.879283+0000 mon.a (mon.0) 128 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2022-09-29T05:22:07.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:22:03.881275+0000 mon.c (mon.1) 1 : cluster [INF] mon.c calling monitor election 2022-09-29T05:22:07.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:04.879252+0000 mon.a (mon.0) 129 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2022-09-29T05:22:07.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:22:05.558720+0000 mgr.y (mgr.14146) 21 : cluster [DBG] pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2022-09-29T05:22:07.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:05.879578+0000 mon.a (mon.0) 130 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2022-09-29T05:22:07.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:06.879479+0000 mon.a (mon.0) 131 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2022-09-29T05:22:07.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:22:06.908008+0000 mon.a (mon.0) 132 : cluster [INF] mon.a is new leader, mons a,c in quorum (ranks 0,1) 2022-09-29T05:22:07.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:22:06.911620+0000 mon.a (mon.0) 133 : cluster [DBG] monmap e2: 2 mons at {a=[v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0],c=[v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0]} 2022-09-29T05:22:07.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:22:06.911699+0000 mon.a (mon.0) 134 : cluster [DBG] fsmap 2022-09-29T05:22:07.467 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:22:06.911746+0000 mon.a (mon.0) 135 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2022-09-29T05:22:07.467 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:22:06.911954+0000 mon.a (mon.0) 136 : cluster [DBG] mgrmap e13: y(active, since 31s) 2022-09-29T05:22:07.467 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:22:06.915158+0000 mon.a (mon.0) 137 : cluster [INF] overall HEALTH_OK 2022-09-29T05:22:07.468 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:22:01.558082+0000 mgr.y (mgr.14146) 19 : cluster [DBG] pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2022-09-29T05:22:07.468 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:01.882330+0000 mon.a (mon.0) 124 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2022-09-29T05:22:07.468 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:22:01.882509+0000 mon.a (mon.0) 125 : cluster [INF] mon.a calling monitor election 2022-09-29T05:22:07.468 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:01.894720+0000 mon.a (mon.0) 126 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2022-09-29T05:22:07.469 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:02.879106+0000 mon.a (mon.0) 127 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2022-09-29T05:22:07.469 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:22:03.558419+0000 mgr.y (mgr.14146) 20 : cluster [DBG] pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2022-09-29T05:22:07.469 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:03.879283+0000 mon.a (mon.0) 128 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2022-09-29T05:22:07.469 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:22:03.881275+0000 mon.c (mon.1) 1 : cluster [INF] mon.c calling monitor election 2022-09-29T05:22:07.469 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:04.879252+0000 mon.a (mon.0) 129 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2022-09-29T05:22:07.470 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:22:05.558720+0000 mgr.y (mgr.14146) 21 : cluster [DBG] pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2022-09-29T05:22:07.470 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:05.879578+0000 mon.a (mon.0) 130 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2022-09-29T05:22:07.470 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:06.879479+0000 mon.a (mon.0) 131 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2022-09-29T05:22:07.470 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:22:06.908008+0000 mon.a (mon.0) 132 : cluster [INF] mon.a is new leader, mons a,c in quorum (ranks 0,1) 2022-09-29T05:22:07.471 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:22:06.911620+0000 mon.a (mon.0) 133 : cluster [DBG] monmap e2: 2 mons at {a=[v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0],c=[v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0]} 2022-09-29T05:22:07.471 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:22:06.911699+0000 mon.a (mon.0) 134 : cluster [DBG] fsmap 2022-09-29T05:22:07.471 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:22:06.911746+0000 mon.a (mon.0) 135 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2022-09-29T05:22:07.471 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:22:06.911954+0000 mon.a (mon.0) 136 : cluster [DBG] mgrmap e13: y(active, since 31s) 2022-09-29T05:22:07.471 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:22:06.915158+0000 mon.a (mon.0) 137 : cluster [INF] overall HEALTH_OK 2022-09-29T05:22:07.472 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:06.917147+0000 mon.a (mon.0) 138 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:22:07.472 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:06.918159+0000 mon.a (mon.0) 139 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2022-09-29T05:22:07.472 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:06.918931+0000 mon.a (mon.0) 140 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:22:07.473 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:06.917147+0000 mon.a (mon.0) 138 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:22:07.473 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:06.918159+0000 mon.a (mon.0) 139 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2022-09-29T05:22:07.473 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:06.918931+0000 mon.a (mon.0) 140 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:22:08.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:07 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:22:06.919594+0000 mgr.y (mgr.14146) 22 : cephadm [INF] Deploying daemon mon.b on smithi153 2022-09-29T05:22:08.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:07 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:07.391283+0000 mon.a (mon.0) 141 : audit [DBG] from='client.? 172.21.15.153:0/267976349' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2022-09-29T05:22:08.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:07 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:07.879841+0000 mon.a (mon.0) 142 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2022-09-29T05:22:08.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:07 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:22:06.919594+0000 mgr.y (mgr.14146) 22 : cephadm [INF] Deploying daemon mon.b on smithi153 2022-09-29T05:22:08.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:07 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:07.391283+0000 mon.a (mon.0) 141 : audit [DBG] from='client.? 172.21.15.153:0/267976349' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2022-09-29T05:22:08.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:07 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:07.879841+0000 mon.a (mon.0) 142 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2022-09-29T05:22:08.960 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2022-09-29T05:22:08.961 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/daemon-base:latest-pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 69111522-3fb6-11ed-8431-001a4aab830c -- ceph mon dump -f json 2022-09-29T05:22:09.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:08 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:22:07.559019+0000 mgr.y (mgr.14146) 23 : cluster [DBG] pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2022-09-29T05:22:09.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:08 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:22:07.559019+0000 mgr.y (mgr.14146) 23 : cluster [DBG] pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2022-09-29T05:22:10.667 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:10 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: debug 2022-09-29T05:22:10.612+0000 7f7a99f71700 1 mon.b@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2022-09-29T05:22:10.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:22:09.559250+0000 mgr.y (mgr.14146) 24 : cluster [DBG] pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2022-09-29T05:22:10.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:22:09.559250+0000 mgr.y (mgr.14146) 24 : cluster [DBG] pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2022-09-29T05:22:11.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:10 smithi153 systemd[1]: Started Ceph mon.b for 69111522-3fb6-11ed-8431-001a4aab830c. 2022-09-29T05:22:15.657 INFO:teuthology.orchestra.run.smithi153.stdout: 2022-09-29T05:22:15.658 INFO:teuthology.orchestra.run.smithi153.stdout:{"epoch":3,"fsid":"69111522-3fb6-11ed-8431-001a4aab830c","modified":"2022-09-29T05:22:10.617397Z","created":"2022-09-29T05:20:42.317979Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:3300","nonce":0},{"type":"v1","addr":"172.21.15.83:6789","nonce":0}]},"addr":"172.21.15.83:6789/0","public_addr":"172.21.15.83:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:3301","nonce":0},{"type":"v1","addr":"172.21.15.83:6790","nonce":0}]},"addr":"172.21.15.83:6790/0","public_addr":"172.21.15.83:6790/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:3300","nonce":0},{"type":"v1","addr":"172.21.15.153:6789","nonce":0}]},"addr":"172.21.15.153:6789/0","public_addr":"172.21.15.153:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1,2]} 2022-09-29T05:22:15.660 INFO:teuthology.orchestra.run.smithi153.stderr:dumped monmap epoch 3 2022-09-29T05:22:15.945 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:15 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:10.620706+0000 mon.a (mon.0) 144 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2022-09-29T05:22:15.945 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:15 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:22:10.620934+0000 mon.a (mon.0) 145 : cluster [INF] mon.a calling monitor election 2022-09-29T05:22:15.945 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:15 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:22:10.622380+0000 mon.c (mon.1) 2 : cluster [INF] mon.c calling monitor election 2022-09-29T05:22:15.946 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:15 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:10.633746+0000 mon.a (mon.0) 146 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2022-09-29T05:22:15.946 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:15 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:10.633911+0000 mon.a (mon.0) 147 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2022-09-29T05:22:15.946 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:15 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:10.827022+0000 mon.a (mon.0) 148 : audit [DBG] from='client.? 172.21.15.153:0/222339591' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2022-09-29T05:22:15.946 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:15 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:22:11.559469+0000 mgr.y (mgr.14146) 25 : cluster [DBG] pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2022-09-29T05:22:15.947 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:15 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:11.617213+0000 mon.a (mon.0) 149 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2022-09-29T05:22:15.947 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:15 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:12.617267+0000 mon.a (mon.0) 150 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2022-09-29T05:22:15.947 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:15 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:22:12.618059+0000 mon.b (mon.2) 1 : cluster [INF] mon.b calling monitor election 2022-09-29T05:22:15.947 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:15 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:22:13.559802+0000 mgr.y (mgr.14146) 26 : cluster [DBG] pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2022-09-29T05:22:15.948 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:15 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:13.617017+0000 mon.a (mon.0) 151 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2022-09-29T05:22:15.948 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:15 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:14.617065+0000 mon.a (mon.0) 152 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2022-09-29T05:22:15.948 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:15 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:15.617451+0000 mon.a (mon.0) 153 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2022-09-29T05:22:15.948 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:15 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:22:15.623929+0000 mon.a (mon.0) 154 : cluster [INF] mon.a calling monitor election 2022-09-29T05:22:15.948 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:15 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:22:15.648651+0000 mon.a (mon.0) 155 : cluster [INF] mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2022-09-29T05:22:15.949 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:15 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:22:15.652148+0000 mon.a (mon.0) 156 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0],b=[v2:172.21.15.153:3300/0,v1:172.21.15.153:6789/0],c=[v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0]} 2022-09-29T05:22:15.949 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:15 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:22:15.652207+0000 mon.a (mon.0) 157 : cluster [DBG] fsmap 2022-09-29T05:22:15.949 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:15 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:22:15.652247+0000 mon.a (mon.0) 158 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2022-09-29T05:22:15.949 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:15 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:22:15.652460+0000 mon.a (mon.0) 159 : cluster [DBG] mgrmap e13: y(active, since 40s) 2022-09-29T05:22:15.950 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:15 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:22:15.661131+0000 mon.a (mon.0) 160 : cluster [INF] overall HEALTH_OK 2022-09-29T05:22:15.950 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:15 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:15.666055+0000 mon.a (mon.0) 161 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:22:15.950 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:15 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:15.667223+0000 mon.a (mon.0) 162 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:22:15.950 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:15 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:15.668522+0000 mon.a (mon.0) 163 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:22:15.951 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:15 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:15.669258+0000 mon.a (mon.0) 164 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:22:15.951 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:15 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:15.691242+0000 mon.a (mon.0) 165 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:22:15.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:10.620706+0000 mon.a (mon.0) 144 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2022-09-29T05:22:15.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:22:10.620934+0000 mon.a (mon.0) 145 : cluster [INF] mon.a calling monitor election 2022-09-29T05:22:15.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:22:10.622380+0000 mon.c (mon.1) 2 : cluster [INF] mon.c calling monitor election 2022-09-29T05:22:15.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:10.633746+0000 mon.a (mon.0) 146 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2022-09-29T05:22:15.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:10.633911+0000 mon.a (mon.0) 147 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2022-09-29T05:22:15.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:10.827022+0000 mon.a (mon.0) 148 : audit [DBG] from='client.? 172.21.15.153:0/222339591' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2022-09-29T05:22:15.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:22:11.559469+0000 mgr.y (mgr.14146) 25 : cluster [DBG] pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2022-09-29T05:22:15.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:11.617213+0000 mon.a (mon.0) 149 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2022-09-29T05:22:15.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:12.617267+0000 mon.a (mon.0) 150 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2022-09-29T05:22:15.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:22:12.618059+0000 mon.b (mon.2) 1 : cluster [INF] mon.b calling monitor election 2022-09-29T05:22:15.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:22:13.559802+0000 mgr.y (mgr.14146) 26 : cluster [DBG] pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2022-09-29T05:22:15.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:13.617017+0000 mon.a (mon.0) 151 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2022-09-29T05:22:15.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:14.617065+0000 mon.a (mon.0) 152 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2022-09-29T05:22:15.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:15.617451+0000 mon.a (mon.0) 153 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2022-09-29T05:22:15.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:22:15.623929+0000 mon.a (mon.0) 154 : cluster [INF] mon.a calling monitor election 2022-09-29T05:22:15.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:22:15.648651+0000 mon.a (mon.0) 155 : cluster [INF] mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2022-09-29T05:22:15.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:10.620706+0000 mon.a (mon.0) 144 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2022-09-29T05:22:15.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:22:10.620934+0000 mon.a (mon.0) 145 : cluster [INF] mon.a calling monitor election 2022-09-29T05:22:15.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:22:10.622380+0000 mon.c (mon.1) 2 : cluster [INF] mon.c calling monitor election 2022-09-29T05:22:15.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:10.633746+0000 mon.a (mon.0) 146 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2022-09-29T05:22:15.968 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:10.633911+0000 mon.a (mon.0) 147 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2022-09-29T05:22:15.968 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:10.827022+0000 mon.a (mon.0) 148 : audit [DBG] from='client.? 172.21.15.153:0/222339591' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2022-09-29T05:22:15.968 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:22:11.559469+0000 mgr.y (mgr.14146) 25 : cluster [DBG] pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2022-09-29T05:22:15.968 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:11.617213+0000 mon.a (mon.0) 149 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2022-09-29T05:22:15.969 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:12.617267+0000 mon.a (mon.0) 150 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2022-09-29T05:22:15.969 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:22:12.618059+0000 mon.b (mon.2) 1 : cluster [INF] mon.b calling monitor election 2022-09-29T05:22:15.969 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:22:13.559802+0000 mgr.y (mgr.14146) 26 : cluster [DBG] pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2022-09-29T05:22:15.969 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:13.617017+0000 mon.a (mon.0) 151 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2022-09-29T05:22:15.970 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:14.617065+0000 mon.a (mon.0) 152 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2022-09-29T05:22:15.970 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:15.617451+0000 mon.a (mon.0) 153 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2022-09-29T05:22:15.970 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:22:15.623929+0000 mon.a (mon.0) 154 : cluster [INF] mon.a calling monitor election 2022-09-29T05:22:15.970 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:22:15.648651+0000 mon.a (mon.0) 155 : cluster [INF] mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2022-09-29T05:22:15.971 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:22:15.652148+0000 mon.a (mon.0) 156 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0],b=[v2:172.21.15.153:3300/0,v1:172.21.15.153:6789/0],c=[v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0]} 2022-09-29T05:22:15.971 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:22:15.652207+0000 mon.a (mon.0) 157 : cluster [DBG] fsmap 2022-09-29T05:22:15.971 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:22:15.652247+0000 mon.a (mon.0) 158 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2022-09-29T05:22:15.972 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:22:15.652460+0000 mon.a (mon.0) 159 : cluster [DBG] mgrmap e13: y(active, since 40s) 2022-09-29T05:22:15.972 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:22:15.661131+0000 mon.a (mon.0) 160 : cluster [INF] overall HEALTH_OK 2022-09-29T05:22:15.972 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:15.666055+0000 mon.a (mon.0) 161 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:22:15.972 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:15.667223+0000 mon.a (mon.0) 162 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:22:15.972 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:15.668522+0000 mon.a (mon.0) 163 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:22:15.973 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:15.669258+0000 mon.a (mon.0) 164 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:22:15.973 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:15.691242+0000 mon.a (mon.0) 165 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:22:15.973 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:22:15.652148+0000 mon.a (mon.0) 156 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0],b=[v2:172.21.15.153:3300/0,v1:172.21.15.153:6789/0],c=[v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0]} 2022-09-29T05:22:15.974 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:22:15.652207+0000 mon.a (mon.0) 157 : cluster [DBG] fsmap 2022-09-29T05:22:15.974 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:22:15.652247+0000 mon.a (mon.0) 158 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2022-09-29T05:22:15.974 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:22:15.652460+0000 mon.a (mon.0) 159 : cluster [DBG] mgrmap e13: y(active, since 40s) 2022-09-29T05:22:15.974 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:22:15.661131+0000 mon.a (mon.0) 160 : cluster [INF] overall HEALTH_OK 2022-09-29T05:22:15.975 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:15.666055+0000 mon.a (mon.0) 161 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:22:15.975 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:15.667223+0000 mon.a (mon.0) 162 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:22:15.975 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:15.668522+0000 mon.a (mon.0) 163 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:22:15.975 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:15.669258+0000 mon.a (mon.0) 164 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:22:15.975 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:15.691242+0000 mon.a (mon.0) 165 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:22:16.072 INFO:tasks.cephadm:Generating final ceph.conf file... 2022-09-29T05:22:16.073 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/daemon-base:latest-pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 69111522-3fb6-11ed-8431-001a4aab830c -- ceph config generate-minimal-conf 2022-09-29T05:22:17.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:16 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:22:15.560168+0000 mgr.y (mgr.14146) 27 : cluster [DBG] pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2022-09-29T05:22:17.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:16 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:16.617425+0000 mon.a (mon.0) 166 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2022-09-29T05:22:17.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:22:15.560168+0000 mgr.y (mgr.14146) 27 : cluster [DBG] pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2022-09-29T05:22:17.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:16.617425+0000 mon.a (mon.0) 166 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2022-09-29T05:22:17.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:22:15.560168+0000 mgr.y (mgr.14146) 27 : cluster [DBG] pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2022-09-29T05:22:17.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:16.617425+0000 mon.a (mon.0) 166 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2022-09-29T05:22:17.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:17 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:16.770305+0000 mon.a (mon.0) 167 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:22:17.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:17 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:22:16.770686+0000 mgr.y (mgr.14146) 28 : cephadm [INF] Updating smithi153:/etc/ceph/ceph.conf 2022-09-29T05:22:17.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:17 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:16.792042+0000 mon.a (mon.0) 168 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:22:17.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:17 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:17.499134+0000 mon.a (mon.0) 169 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:22:17.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:17 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:17.572424+0000 mon.a (mon.0) 170 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:22:17.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:17 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:17.577839+0000 mon.a (mon.0) 171 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:22:17.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:17 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:17.578880+0000 mon.a (mon.0) 172 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2022-09-29T05:22:17.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:17 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:17.579377+0000 mon.a (mon.0) 173 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2022-09-29T05:22:17.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:17 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:17.579822+0000 mon.a (mon.0) 174 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:22:17.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:17 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:16.770305+0000 mon.a (mon.0) 167 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:22:17.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:17 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:22:16.770686+0000 mgr.y (mgr.14146) 28 : cephadm [INF] Updating smithi153:/etc/ceph/ceph.conf 2022-09-29T05:22:17.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:17 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:16.792042+0000 mon.a (mon.0) 168 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:22:17.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:17 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:17.499134+0000 mon.a (mon.0) 169 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:22:17.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:17 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:17.572424+0000 mon.a (mon.0) 170 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:22:17.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:17 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:17.577839+0000 mon.a (mon.0) 171 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:22:17.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:17 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:17.578880+0000 mon.a (mon.0) 172 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2022-09-29T05:22:17.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:17 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:17.579377+0000 mon.a (mon.0) 173 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2022-09-29T05:22:17.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:17 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:17.579822+0000 mon.a (mon.0) 174 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:22:17.976 INFO:teuthology.orchestra.run.smithi083.stdout:# minimal ceph.conf for 69111522-3fb6-11ed-8431-001a4aab830c 2022-09-29T05:22:17.976 INFO:teuthology.orchestra.run.smithi083.stdout:[global] 2022-09-29T05:22:17.976 INFO:teuthology.orchestra.run.smithi083.stdout: fsid = 69111522-3fb6-11ed-8431-001a4aab830c 2022-09-29T05:22:17.976 INFO:teuthology.orchestra.run.smithi083.stdout: mon_host = [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] [v2:172.21.15.153:3300/0,v1:172.21.15.153:6789/0] [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] 2022-09-29T05:22:18.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:17 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:16.770305+0000 mon.a (mon.0) 167 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:22:18.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:17 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:22:16.770686+0000 mgr.y (mgr.14146) 28 : cephadm [INF] Updating smithi153:/etc/ceph/ceph.conf 2022-09-29T05:22:18.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:17 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:16.792042+0000 mon.a (mon.0) 168 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:22:18.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:17 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:17.499134+0000 mon.a (mon.0) 169 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:22:18.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:17 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:17.572424+0000 mon.a (mon.0) 170 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:22:18.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:17 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:17.577839+0000 mon.a (mon.0) 171 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:22:18.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:17 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:17.578880+0000 mon.a (mon.0) 172 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2022-09-29T05:22:18.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:17 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:17.579377+0000 mon.a (mon.0) 173 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2022-09-29T05:22:18.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:17 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:17.579822+0000 mon.a (mon.0) 174 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:22:18.516 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2022-09-29T05:22:18.517 DEBUG:teuthology.orchestra.run.smithi083:> set -ex 2022-09-29T05:22:18.517 DEBUG:teuthology.orchestra.run.smithi083:> sudo dd of=/etc/ceph/ceph.conf 2022-09-29T05:22:18.549 DEBUG:teuthology.orchestra.run.smithi083:> set -ex 2022-09-29T05:22:18.549 DEBUG:teuthology.orchestra.run.smithi083:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2022-09-29T05:22:18.616 DEBUG:teuthology.orchestra.run.smithi153:> set -ex 2022-09-29T05:22:18.617 DEBUG:teuthology.orchestra.run.smithi153:> sudo dd of=/etc/ceph/ceph.conf 2022-09-29T05:22:18.654 DEBUG:teuthology.orchestra.run.smithi153:> set -ex 2022-09-29T05:22:18.654 DEBUG:teuthology.orchestra.run.smithi153:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2022-09-29T05:22:18.724 INFO:tasks.cephadm:Adding mgr.y on smithi083 2022-09-29T05:22:18.725 INFO:tasks.cephadm:Adding mgr.x on smithi153 2022-09-29T05:22:18.725 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/daemon-base:latest-pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 69111522-3fb6-11ed-8431-001a4aab830c -- ceph orch apply mgr '2;smithi083=y;smithi153=x' 2022-09-29T05:22:19.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:18 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:22:17.499511+0000 mgr.y (mgr.14146) 29 : cephadm [INF] Updating smithi083:/etc/ceph/ceph.conf 2022-09-29T05:22:19.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:18 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:22:17.560371+0000 mgr.y (mgr.14146) 30 : cluster [DBG] pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2022-09-29T05:22:19.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:18 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:22:17.578660+0000 mgr.y (mgr.14146) 31 : cephadm [INF] Reconfiguring mon.a (unknown last config time)... 2022-09-29T05:22:19.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:18 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:22:17.580239+0000 mgr.y (mgr.14146) 32 : cephadm [INF] Reconfiguring daemon mon.a on smithi083 2022-09-29T05:22:19.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:18 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:17.973339+0000 mon.c (mon.1) 3 : audit [DBG] from='client.? 172.21.15.83:0/247819308' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:22:19.133 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:18 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:22:17.499511+0000 mgr.y (mgr.14146) 29 : cephadm [INF] Updating smithi083:/etc/ceph/ceph.conf 2022-09-29T05:22:19.133 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:18 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:22:17.560371+0000 mgr.y (mgr.14146) 30 : cluster [DBG] pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2022-09-29T05:22:19.134 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:18 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:22:17.578660+0000 mgr.y (mgr.14146) 31 : cephadm [INF] Reconfiguring mon.a (unknown last config time)... 2022-09-29T05:22:19.134 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:18 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:22:17.580239+0000 mgr.y (mgr.14146) 32 : cephadm [INF] Reconfiguring daemon mon.a on smithi083 2022-09-29T05:22:19.134 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:18 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:17.973339+0000 mon.c (mon.1) 3 : audit [DBG] from='client.? 172.21.15.83:0/247819308' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:22:19.134 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:18 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:22:17.499511+0000 mgr.y (mgr.14146) 29 : cephadm [INF] Updating smithi083:/etc/ceph/ceph.conf 2022-09-29T05:22:19.135 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:18 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:22:17.560371+0000 mgr.y (mgr.14146) 30 : cluster [DBG] pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2022-09-29T05:22:19.135 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:18 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:22:17.578660+0000 mgr.y (mgr.14146) 31 : cephadm [INF] Reconfiguring mon.a (unknown last config time)... 2022-09-29T05:22:19.135 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:18 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:22:17.580239+0000 mgr.y (mgr.14146) 32 : cephadm [INF] Reconfiguring daemon mon.a on smithi083 2022-09-29T05:22:19.135 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:18 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:17.973339+0000 mon.c (mon.1) 3 : audit [DBG] from='client.? 172.21.15.83:0/247819308' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:22:20.422 INFO:teuthology.orchestra.run.smithi153.stdout:Scheduled mgr update... 2022-09-29T05:22:20.443 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:19.174229+0000 mon.a (mon.0) 175 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:22:20.443 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:22:19.174686+0000 mgr.y (mgr.14146) 33 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2022-09-29T05:22:20.444 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:19.174879+0000 mon.a (mon.0) 176 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2022-09-29T05:22:20.444 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:19.175432+0000 mon.a (mon.0) 177 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2022-09-29T05:22:20.444 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:19.175914+0000 mon.a (mon.0) 178 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:22:20.444 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:22:19.176381+0000 mgr.y (mgr.14146) 34 : cephadm [INF] Reconfiguring daemon mon.c on smithi083 2022-09-29T05:22:20.445 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:22:19.560582+0000 mgr.y (mgr.14146) 35 : cluster [DBG] pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2022-09-29T05:22:20.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:19.174229+0000 mon.a (mon.0) 175 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:22:20.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:22:19.174686+0000 mgr.y (mgr.14146) 33 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2022-09-29T05:22:20.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:19.174879+0000 mon.a (mon.0) 176 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2022-09-29T05:22:20.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:19.175432+0000 mon.a (mon.0) 177 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2022-09-29T05:22:20.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:19.175914+0000 mon.a (mon.0) 178 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:22:20.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:22:19.176381+0000 mgr.y (mgr.14146) 34 : cephadm [INF] Reconfiguring daemon mon.c on smithi083 2022-09-29T05:22:20.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:22:19.560582+0000 mgr.y (mgr.14146) 35 : cluster [DBG] pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2022-09-29T05:22:20.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:19.174229+0000 mon.a (mon.0) 175 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:22:20.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:22:19.174686+0000 mgr.y (mgr.14146) 33 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2022-09-29T05:22:20.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:19.174879+0000 mon.a (mon.0) 176 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2022-09-29T05:22:20.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:19.175432+0000 mon.a (mon.0) 177 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2022-09-29T05:22:20.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:19.175914+0000 mon.a (mon.0) 178 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:22:20.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:22:19.176381+0000 mgr.y (mgr.14146) 34 : cephadm [INF] Reconfiguring daemon mon.c on smithi083 2022-09-29T05:22:20.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:22:19.560582+0000 mgr.y (mgr.14146) 35 : cluster [DBG] pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2022-09-29T05:22:20.790 DEBUG:teuthology.orchestra.run.smithi153:mgr.x> sudo journalctl -f -n 0 -u ceph-69111522-3fb6-11ed-8431-001a4aab830c@mgr.x.service 2022-09-29T05:22:20.793 INFO:tasks.cephadm:Deploying OSDs... 2022-09-29T05:22:20.793 DEBUG:teuthology.orchestra.run.smithi083:> set -ex 2022-09-29T05:22:20.794 DEBUG:teuthology.orchestra.run.smithi083:> dd if=/scratch_devs of=/dev/stdout 2022-09-29T05:22:20.813 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2022-09-29T05:22:20.814 DEBUG:teuthology.orchestra.run.smithi083:> stat /dev/vg_nvme/lv_1 2022-09-29T05:22:20.822 INFO:journalctl@ceph.mgr.x.smithi153.stdout:-- Logs begin at Thu 2022-09-29 05:12:53 UTC. -- 2022-09-29T05:22:20.871 INFO:teuthology.orchestra.run.smithi083.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2022-09-29T05:22:20.872 INFO:teuthology.orchestra.run.smithi083.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2022-09-29T05:22:20.872 INFO:teuthology.orchestra.run.smithi083.stdout:Device: 6h/6d Inode: 125056 Links: 1 2022-09-29T05:22:20.872 INFO:teuthology.orchestra.run.smithi083.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2022-09-29T05:22:20.872 INFO:teuthology.orchestra.run.smithi083.stdout:Context: system_u:object_r:device_t:s0 2022-09-29T05:22:20.873 INFO:teuthology.orchestra.run.smithi083.stdout:Access: 2022-09-29 05:22:00.977474763 +0000 2022-09-29T05:22:20.873 INFO:teuthology.orchestra.run.smithi083.stdout:Modify: 2022-09-29 05:16:49.851940637 +0000 2022-09-29T05:22:20.873 INFO:teuthology.orchestra.run.smithi083.stdout:Change: 2022-09-29 05:16:49.851940637 +0000 2022-09-29T05:22:20.873 INFO:teuthology.orchestra.run.smithi083.stdout: Birth: - 2022-09-29T05:22:20.874 DEBUG:teuthology.orchestra.run.smithi083:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2022-09-29T05:22:20.939 INFO:teuthology.orchestra.run.smithi083.stderr:1+0 records in 2022-09-29T05:22:20.940 INFO:teuthology.orchestra.run.smithi083.stderr:1+0 records out 2022-09-29T05:22:20.940 INFO:teuthology.orchestra.run.smithi083.stderr:512 bytes copied, 0.00024032 s, 2.1 MB/s 2022-09-29T05:22:20.941 DEBUG:teuthology.orchestra.run.smithi083:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2022-09-29T05:22:21.001 DEBUG:teuthology.orchestra.run.smithi083:> stat /dev/vg_nvme/lv_2 2022-09-29T05:22:21.059 INFO:teuthology.orchestra.run.smithi083.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2022-09-29T05:22:21.060 INFO:teuthology.orchestra.run.smithi083.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2022-09-29T05:22:21.060 INFO:teuthology.orchestra.run.smithi083.stdout:Device: 6h/6d Inode: 125121 Links: 1 2022-09-29T05:22:21.060 INFO:teuthology.orchestra.run.smithi083.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2022-09-29T05:22:21.060 INFO:teuthology.orchestra.run.smithi083.stdout:Context: system_u:object_r:device_t:s0 2022-09-29T05:22:21.061 INFO:teuthology.orchestra.run.smithi083.stdout:Access: 2022-09-29 05:22:00.977474763 +0000 2022-09-29T05:22:21.061 INFO:teuthology.orchestra.run.smithi083.stdout:Modify: 2022-09-29 05:16:50.405926959 +0000 2022-09-29T05:22:21.061 INFO:teuthology.orchestra.run.smithi083.stdout:Change: 2022-09-29 05:16:50.405926959 +0000 2022-09-29T05:22:21.061 INFO:teuthology.orchestra.run.smithi083.stdout: Birth: - 2022-09-29T05:22:21.062 DEBUG:teuthology.orchestra.run.smithi083:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2022-09-29T05:22:21.126 INFO:teuthology.orchestra.run.smithi083.stderr:1+0 records in 2022-09-29T05:22:21.126 INFO:teuthology.orchestra.run.smithi083.stderr:1+0 records out 2022-09-29T05:22:21.127 INFO:teuthology.orchestra.run.smithi083.stderr:512 bytes copied, 0.000295603 s, 1.7 MB/s 2022-09-29T05:22:21.128 DEBUG:teuthology.orchestra.run.smithi083:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2022-09-29T05:22:21.188 DEBUG:teuthology.orchestra.run.smithi083:> stat /dev/vg_nvme/lv_3 2022-09-29T05:22:21.245 INFO:teuthology.orchestra.run.smithi083.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2022-09-29T05:22:21.245 INFO:teuthology.orchestra.run.smithi083.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2022-09-29T05:22:21.245 INFO:teuthology.orchestra.run.smithi083.stdout:Device: 6h/6d Inode: 111402 Links: 1 2022-09-29T05:22:21.246 INFO:teuthology.orchestra.run.smithi083.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2022-09-29T05:22:21.246 INFO:teuthology.orchestra.run.smithi083.stdout:Context: system_u:object_r:device_t:s0 2022-09-29T05:22:21.246 INFO:teuthology.orchestra.run.smithi083.stdout:Access: 2022-09-29 05:22:00.978474739 +0000 2022-09-29T05:22:21.246 INFO:teuthology.orchestra.run.smithi083.stdout:Modify: 2022-09-29 05:16:50.942913700 +0000 2022-09-29T05:22:21.247 INFO:teuthology.orchestra.run.smithi083.stdout:Change: 2022-09-29 05:16:50.942913700 +0000 2022-09-29T05:22:21.247 INFO:teuthology.orchestra.run.smithi083.stdout: Birth: - 2022-09-29T05:22:21.247 DEBUG:teuthology.orchestra.run.smithi083:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2022-09-29T05:22:21.311 INFO:teuthology.orchestra.run.smithi083.stderr:1+0 records in 2022-09-29T05:22:21.311 INFO:teuthology.orchestra.run.smithi083.stderr:1+0 records out 2022-09-29T05:22:21.312 INFO:teuthology.orchestra.run.smithi083.stderr:512 bytes copied, 0.000206199 s, 2.5 MB/s 2022-09-29T05:22:21.313 DEBUG:teuthology.orchestra.run.smithi083:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2022-09-29T05:22:21.372 DEBUG:teuthology.orchestra.run.smithi083:> stat /dev/vg_nvme/lv_4 2022-09-29T05:22:21.431 INFO:teuthology.orchestra.run.smithi083.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2022-09-29T05:22:21.431 INFO:teuthology.orchestra.run.smithi083.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2022-09-29T05:22:21.431 INFO:teuthology.orchestra.run.smithi083.stdout:Device: 6h/6d Inode: 124781 Links: 1 2022-09-29T05:22:21.432 INFO:teuthology.orchestra.run.smithi083.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2022-09-29T05:22:21.432 INFO:teuthology.orchestra.run.smithi083.stdout:Context: system_u:object_r:device_t:s0 2022-09-29T05:22:21.432 INFO:teuthology.orchestra.run.smithi083.stdout:Access: 2022-09-29 05:22:00.978474739 +0000 2022-09-29T05:22:21.432 INFO:teuthology.orchestra.run.smithi083.stdout:Modify: 2022-09-29 05:16:51.480900416 +0000 2022-09-29T05:22:21.433 INFO:teuthology.orchestra.run.smithi083.stdout:Change: 2022-09-29 05:16:51.480900416 +0000 2022-09-29T05:22:21.433 INFO:teuthology.orchestra.run.smithi083.stdout: Birth: - 2022-09-29T05:22:21.433 DEBUG:teuthology.orchestra.run.smithi083:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2022-09-29T05:22:21.497 INFO:teuthology.orchestra.run.smithi083.stderr:1+0 records in 2022-09-29T05:22:21.497 INFO:teuthology.orchestra.run.smithi083.stderr:1+0 records out 2022-09-29T05:22:21.498 INFO:teuthology.orchestra.run.smithi083.stderr:512 bytes copied, 0.000210772 s, 2.4 MB/s 2022-09-29T05:22:21.499 DEBUG:teuthology.orchestra.run.smithi083:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2022-09-29T05:22:21.558 DEBUG:teuthology.orchestra.run.smithi153:> set -ex 2022-09-29T05:22:21.559 DEBUG:teuthology.orchestra.run.smithi153:> dd if=/scratch_devs of=/dev/stdout 2022-09-29T05:22:21.578 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2022-09-29T05:22:21.578 DEBUG:teuthology.orchestra.run.smithi153:> stat /dev/vg_nvme/lv_1 2022-09-29T05:22:21.608 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:20.414699+0000 mgr.y (mgr.14146) 36 : audit [DBG] from='client.24110 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi083=y;smithi153=x", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:22:21.609 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:22:20.415879+0000 mgr.y (mgr.14146) 37 : cephadm [INF] Saving service mgr spec with placement smithi083=y;smithi153=x;count:2 2022-09-29T05:22:21.609 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:20.418995+0000 mon.a (mon.0) 179 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:22:21.609 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:20.452273+0000 mon.a (mon.0) 180 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:22:21.609 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:22:20.452728+0000 mgr.y (mgr.14146) 38 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2022-09-29T05:22:21.610 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:20.452928+0000 mon.a (mon.0) 181 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2022-09-29T05:22:21.610 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:20.453479+0000 mon.a (mon.0) 182 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2022-09-29T05:22:21.610 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:20.453970+0000 mon.a (mon.0) 183 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:22:21.611 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:22:20.454431+0000 mgr.y (mgr.14146) 39 : cephadm [INF] Reconfiguring daemon mon.b on smithi153 2022-09-29T05:22:21.628 INFO:teuthology.orchestra.run.smithi153.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2022-09-29T05:22:21.628 INFO:teuthology.orchestra.run.smithi153.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2022-09-29T05:22:21.628 INFO:teuthology.orchestra.run.smithi153.stdout:Device: 6h/6d Inode: 113327 Links: 1 2022-09-29T05:22:21.629 INFO:teuthology.orchestra.run.smithi153.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2022-09-29T05:22:21.629 INFO:teuthology.orchestra.run.smithi153.stdout:Context: system_u:object_r:device_t:s0 2022-09-29T05:22:21.629 INFO:teuthology.orchestra.run.smithi153.stdout:Access: 2022-09-29 05:22:09.372026033 +0000 2022-09-29T05:22:21.629 INFO:teuthology.orchestra.run.smithi153.stdout:Modify: 2022-09-29 05:16:49.870202448 +0000 2022-09-29T05:22:21.630 INFO:teuthology.orchestra.run.smithi153.stdout:Change: 2022-09-29 05:16:49.870202448 +0000 2022-09-29T05:22:21.630 INFO:teuthology.orchestra.run.smithi153.stdout: Birth: - 2022-09-29T05:22:21.630 DEBUG:teuthology.orchestra.run.smithi153:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2022-09-29T05:22:21.696 INFO:teuthology.orchestra.run.smithi153.stderr:1+0 records in 2022-09-29T05:22:21.696 INFO:teuthology.orchestra.run.smithi153.stderr:1+0 records out 2022-09-29T05:22:21.697 INFO:teuthology.orchestra.run.smithi153.stderr:512 bytes copied, 0.000220351 s, 2.3 MB/s 2022-09-29T05:22:21.698 DEBUG:teuthology.orchestra.run.smithi153:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2022-09-29T05:22:21.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:20.414699+0000 mgr.y (mgr.14146) 36 : audit [DBG] from='client.24110 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi083=y;smithi153=x", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:22:21.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:22:20.415879+0000 mgr.y (mgr.14146) 37 : cephadm [INF] Saving service mgr spec with placement smithi083=y;smithi153=x;count:2 2022-09-29T05:22:21.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:20.418995+0000 mon.a (mon.0) 179 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:22:21.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:20.452273+0000 mon.a (mon.0) 180 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:22:21.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:22:20.452728+0000 mgr.y (mgr.14146) 38 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2022-09-29T05:22:21.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:20.452928+0000 mon.a (mon.0) 181 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2022-09-29T05:22:21.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:20.453479+0000 mon.a (mon.0) 182 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2022-09-29T05:22:21.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:20.453970+0000 mon.a (mon.0) 183 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:22:21.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:22:20.454431+0000 mgr.y (mgr.14146) 39 : cephadm [INF] Reconfiguring daemon mon.b on smithi153 2022-09-29T05:22:21.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:20.414699+0000 mgr.y (mgr.14146) 36 : audit [DBG] from='client.24110 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi083=y;smithi153=x", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:22:21.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:22:20.415879+0000 mgr.y (mgr.14146) 37 : cephadm [INF] Saving service mgr spec with placement smithi083=y;smithi153=x;count:2 2022-09-29T05:22:21.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:20.418995+0000 mon.a (mon.0) 179 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:22:21.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:20.452273+0000 mon.a (mon.0) 180 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:22:21.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:22:20.452728+0000 mgr.y (mgr.14146) 38 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2022-09-29T05:22:21.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:20.452928+0000 mon.a (mon.0) 181 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2022-09-29T05:22:21.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:20.453479+0000 mon.a (mon.0) 182 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2022-09-29T05:22:21.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:20.453970+0000 mon.a (mon.0) 183 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:22:21.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:22:20.454431+0000 mgr.y (mgr.14146) 39 : cephadm [INF] Reconfiguring daemon mon.b on smithi153 2022-09-29T05:22:21.756 DEBUG:teuthology.orchestra.run.smithi153:> stat /dev/vg_nvme/lv_2 2022-09-29T05:22:21.813 INFO:teuthology.orchestra.run.smithi153.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2022-09-29T05:22:21.814 INFO:teuthology.orchestra.run.smithi153.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2022-09-29T05:22:21.814 INFO:teuthology.orchestra.run.smithi153.stdout:Device: 6h/6d Inode: 123142 Links: 1 2022-09-29T05:22:21.814 INFO:teuthology.orchestra.run.smithi153.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2022-09-29T05:22:21.815 INFO:teuthology.orchestra.run.smithi153.stdout:Context: system_u:object_r:device_t:s0 2022-09-29T05:22:21.815 INFO:teuthology.orchestra.run.smithi153.stdout:Access: 2022-09-29 05:22:09.372026033 +0000 2022-09-29T05:22:21.815 INFO:teuthology.orchestra.run.smithi153.stdout:Modify: 2022-09-29 05:16:50.415192141 +0000 2022-09-29T05:22:21.815 INFO:teuthology.orchestra.run.smithi153.stdout:Change: 2022-09-29 05:16:50.415192141 +0000 2022-09-29T05:22:21.816 INFO:teuthology.orchestra.run.smithi153.stdout: Birth: - 2022-09-29T05:22:21.816 DEBUG:teuthology.orchestra.run.smithi153:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2022-09-29T05:22:21.880 INFO:teuthology.orchestra.run.smithi153.stderr:1+0 records in 2022-09-29T05:22:21.880 INFO:teuthology.orchestra.run.smithi153.stderr:1+0 records out 2022-09-29T05:22:21.881 INFO:teuthology.orchestra.run.smithi153.stderr:512 bytes copied, 0.000205413 s, 2.5 MB/s 2022-09-29T05:22:21.882 DEBUG:teuthology.orchestra.run.smithi153:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2022-09-29T05:22:21.941 DEBUG:teuthology.orchestra.run.smithi153:> stat /dev/vg_nvme/lv_3 2022-09-29T05:22:21.999 INFO:teuthology.orchestra.run.smithi153.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2022-09-29T05:22:21.999 INFO:teuthology.orchestra.run.smithi153.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2022-09-29T05:22:22.000 INFO:teuthology.orchestra.run.smithi153.stdout:Device: 6h/6d Inode: 113434 Links: 1 2022-09-29T05:22:22.000 INFO:teuthology.orchestra.run.smithi153.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2022-09-29T05:22:22.000 INFO:teuthology.orchestra.run.smithi153.stdout:Context: system_u:object_r:device_t:s0 2022-09-29T05:22:22.000 INFO:teuthology.orchestra.run.smithi153.stdout:Access: 2022-09-29 05:22:09.372026033 +0000 2022-09-29T05:22:22.001 INFO:teuthology.orchestra.run.smithi153.stdout:Modify: 2022-09-29 05:16:50.942182175 +0000 2022-09-29T05:22:22.001 INFO:teuthology.orchestra.run.smithi153.stdout:Change: 2022-09-29 05:16:50.942182175 +0000 2022-09-29T05:22:22.001 INFO:teuthology.orchestra.run.smithi153.stdout: Birth: - 2022-09-29T05:22:22.001 DEBUG:teuthology.orchestra.run.smithi153:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2022-09-29T05:22:22.068 INFO:teuthology.orchestra.run.smithi153.stderr:1+0 records in 2022-09-29T05:22:22.068 INFO:teuthology.orchestra.run.smithi153.stderr:1+0 records out 2022-09-29T05:22:22.068 INFO:teuthology.orchestra.run.smithi153.stderr:512 bytes copied, 0.00020496 s, 2.5 MB/s 2022-09-29T05:22:22.069 DEBUG:teuthology.orchestra.run.smithi153:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2022-09-29T05:22:22.129 DEBUG:teuthology.orchestra.run.smithi153:> stat /dev/vg_nvme/lv_4 2022-09-29T05:22:22.187 INFO:teuthology.orchestra.run.smithi153.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2022-09-29T05:22:22.187 INFO:teuthology.orchestra.run.smithi153.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2022-09-29T05:22:22.187 INFO:teuthology.orchestra.run.smithi153.stdout:Device: 6h/6d Inode: 113506 Links: 1 2022-09-29T05:22:22.188 INFO:teuthology.orchestra.run.smithi153.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2022-09-29T05:22:22.188 INFO:teuthology.orchestra.run.smithi153.stdout:Context: system_u:object_r:device_t:s0 2022-09-29T05:22:22.188 INFO:teuthology.orchestra.run.smithi153.stdout:Access: 2022-09-29 05:22:09.372026033 +0000 2022-09-29T05:22:22.188 INFO:teuthology.orchestra.run.smithi153.stdout:Modify: 2022-09-29 05:16:51.474172114 +0000 2022-09-29T05:22:22.188 INFO:teuthology.orchestra.run.smithi153.stdout:Change: 2022-09-29 05:16:51.474172114 +0000 2022-09-29T05:22:22.189 INFO:teuthology.orchestra.run.smithi153.stdout: Birth: - 2022-09-29T05:22:22.189 DEBUG:teuthology.orchestra.run.smithi153:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2022-09-29T05:22:22.256 INFO:teuthology.orchestra.run.smithi153.stderr:1+0 records in 2022-09-29T05:22:22.257 INFO:teuthology.orchestra.run.smithi153.stderr:1+0 records out 2022-09-29T05:22:22.257 INFO:teuthology.orchestra.run.smithi153.stderr:512 bytes copied, 0.000227319 s, 2.3 MB/s 2022-09-29T05:22:22.258 DEBUG:teuthology.orchestra.run.smithi153:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2022-09-29T05:22:22.317 INFO:tasks.cephadm:Deploying osd.0 on smithi083 with /dev/vg_nvme/lv_4... 2022-09-29T05:22:22.317 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/daemon-base:latest-pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 69111522-3fb6-11ed-8431-001a4aab830c -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2022-09-29T05:22:22.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:22:21.560827+0000 mgr.y (mgr.14146) 40 : cluster [DBG] pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2022-09-29T05:22:22.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:21.635813+0000 mon.a (mon.0) 184 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:22:22.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:21.636397+0000 mon.a (mon.0) 185 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:22:22.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:21.637201+0000 mon.a (mon.0) 186 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:22:22.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:21.637725+0000 mon.a (mon.0) 187 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:22:22.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:21.641244+0000 mon.a (mon.0) 188 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:22:22.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:21.642209+0000 mon.a (mon.0) 189 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2022-09-29T05:22:22.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:21.644944+0000 mon.a (mon.0) 190 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2022-09-29T05:22:22.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:21.645532+0000 mon.a (mon.0) 191 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2022-09-29T05:22:22.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:21.646127+0000 mon.a (mon.0) 192 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:22:22.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:22:21.646558+0000 mgr.y (mgr.14146) 41 : cephadm [INF] Deploying daemon mgr.x on smithi153 2022-09-29T05:22:22.897 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:22:21.560827+0000 mgr.y (mgr.14146) 40 : cluster [DBG] pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2022-09-29T05:22:22.898 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:21.635813+0000 mon.a (mon.0) 184 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:22:22.898 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:21.636397+0000 mon.a (mon.0) 185 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:22:22.898 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:21.637201+0000 mon.a (mon.0) 186 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:22:22.898 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:21.637725+0000 mon.a (mon.0) 187 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:22:22.899 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:21.641244+0000 mon.a (mon.0) 188 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:22:22.899 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:21.642209+0000 mon.a (mon.0) 189 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2022-09-29T05:22:22.899 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:21.644944+0000 mon.a (mon.0) 190 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2022-09-29T05:22:22.899 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:21.645532+0000 mon.a (mon.0) 191 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2022-09-29T05:22:22.900 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:21.646127+0000 mon.a (mon.0) 192 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:22:22.900 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:22:21.646558+0000 mgr.y (mgr.14146) 41 : cephadm [INF] Deploying daemon mgr.x on smithi153 2022-09-29T05:22:22.900 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:22:21.560827+0000 mgr.y (mgr.14146) 40 : cluster [DBG] pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2022-09-29T05:22:22.901 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:21.635813+0000 mon.a (mon.0) 184 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:22:22.901 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:21.636397+0000 mon.a (mon.0) 185 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:22:22.901 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:21.637201+0000 mon.a (mon.0) 186 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:22:22.901 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:21.637725+0000 mon.a (mon.0) 187 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:22:22.901 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:21.641244+0000 mon.a (mon.0) 188 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:22:22.902 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:21.642209+0000 mon.a (mon.0) 189 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2022-09-29T05:22:22.902 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:21.644944+0000 mon.a (mon.0) 190 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2022-09-29T05:22:22.902 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:21.645532+0000 mon.a (mon.0) 191 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2022-09-29T05:22:22.902 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:21.646127+0000 mon.a (mon.0) 192 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:22:22.903 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:22:21.646558+0000 mgr.y (mgr.14146) 41 : cephadm [INF] Deploying daemon mgr.x on smithi153 2022-09-29T05:22:23.059 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:22:22 smithi153 systemd[1]: Starting Ceph mgr.x for 69111522-3fb6-11ed-8431-001a4aab830c... 2022-09-29T05:22:23.949 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:22:23 smithi153 podman[40216]: 2022-09-29T05:22:23.949 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:22:23 smithi153 bash[40216]: f73d53839dcbd1d88b20b4b759aff91a3fcf6a9930b6f87b17fcb81e06ec0053 2022-09-29T05:22:23.949 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:22:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[40232]: debug 2022-09-29T05:22:23.815+0000 7f999bac1700 1 -- 172.21.15.153:0/3390766055 <== mon.2 v2:172.21.15.153:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55ccbba584e0 con 0x55ccbba2ec00 2022-09-29T05:22:23.950 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:22:23 smithi153 systemd[1]: Started Ceph mgr.x for 69111522-3fb6-11ed-8431-001a4aab830c. 2022-09-29T05:22:23.950 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:22:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[40232]: debug 2022-09-29T05:22:23.944+0000 7f99ac095680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2022-09-29T05:22:24.296 INFO:teuthology.orchestra.run.smithi083.stderr:--> Zapping: /dev/vg_nvme/lv_4 2022-09-29T05:22:24.296 INFO:teuthology.orchestra.run.smithi083.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2022-09-29T05:22:24.296 INFO:teuthology.orchestra.run.smithi083.stderr: stderr: 10+0 records in 2022-09-29T05:22:24.297 INFO:teuthology.orchestra.run.smithi083.stderr:10+0 records out 2022-09-29T05:22:24.297 INFO:teuthology.orchestra.run.smithi083.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0108207 s, 969 MB/s 2022-09-29T05:22:24.297 INFO:teuthology.orchestra.run.smithi083.stderr:--> Zapping successful for: 2022-09-29T05:22:24.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:22:24 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[40232]: debug 2022-09-29T05:22:24.028+0000 7f99ac095680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2022-09-29T05:22:24.738 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/daemon-base:latest-pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 69111522-3fb6-11ed-8431-001a4aab830c -- ceph orch daemon add osd smithi083:vg_nvme/lv_4 2022-09-29T05:22:24.809 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:22:24 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[40232]: debug 2022-09-29T05:22:24.409+0000 7f99ac095680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2022-09-29T05:22:25.193 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:24 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:22:23.561184+0000 mgr.y (mgr.14146) 42 : cluster [DBG] pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2022-09-29T05:22:25.193 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:24 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:23.906175+0000 mon.a (mon.0) 193 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:22:25.193 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:24 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:23.907488+0000 mon.a (mon.0) 194 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:22:25.194 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:24 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:23.908301+0000 mon.a (mon.0) 195 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:22:25.194 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:24 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:23.908793+0000 mon.a (mon.0) 196 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:22:25.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:22:23.561184+0000 mgr.y (mgr.14146) 42 : cluster [DBG] pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2022-09-29T05:22:25.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:23.906175+0000 mon.a (mon.0) 193 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:22:25.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:23.907488+0000 mon.a (mon.0) 194 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:22:25.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:23.908301+0000 mon.a (mon.0) 195 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:22:25.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:23.908793+0000 mon.a (mon.0) 196 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:22:25.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:22:23.561184+0000 mgr.y (mgr.14146) 42 : cluster [DBG] pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2022-09-29T05:22:25.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:23.906175+0000 mon.a (mon.0) 193 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:22:25.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:23.907488+0000 mon.a (mon.0) 194 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:22:25.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:23.908301+0000 mon.a (mon.0) 195 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:22:25.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:23.908793+0000 mon.a (mon.0) 196 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:22:25.485 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:22:25 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[40232]: debug 2022-09-29T05:22:25.187+0000 7f99ac095680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2022-09-29T05:22:25.485 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:22:25 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[40232]: debug 2022-09-29T05:22:25.270+0000 7f99ac095680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2022-09-29T05:22:25.809 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:22:25 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[40232]: debug 2022-09-29T05:22:25.480+0000 7f99ac095680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2022-09-29T05:22:26.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:25.124007+0000 mon.a (mon.0) 197 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:22:26.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:25.129479+0000 mon.a (mon.0) 198 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:22:26.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:22:25.130570+0000 mgr.y (mgr.14146) 43 : cephadm [INF] Reconfiguring mgr.y (unknown last config time)... 2022-09-29T05:22:26.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:25.130822+0000 mon.a (mon.0) 199 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2022-09-29T05:22:26.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:25.131466+0000 mon.a (mon.0) 200 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2022-09-29T05:22:26.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:25.131980+0000 mon.a (mon.0) 201 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:22:26.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:22:25.132488+0000 mgr.y (mgr.14146) 44 : cephadm [INF] Reconfiguring daemon mgr.y on smithi083 2022-09-29T05:22:26.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:22:25.561450+0000 mgr.y (mgr.14146) 45 : cluster [DBG] pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2022-09-29T05:22:26.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:25.695057+0000 mon.a (mon.0) 202 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:22:26.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:25.124007+0000 mon.a (mon.0) 197 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:22:26.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:25.129479+0000 mon.a (mon.0) 198 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:22:26.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:22:25.130570+0000 mgr.y (mgr.14146) 43 : cephadm [INF] Reconfiguring mgr.y (unknown last config time)... 2022-09-29T05:22:26.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:25.130822+0000 mon.a (mon.0) 199 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2022-09-29T05:22:26.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:25.131466+0000 mon.a (mon.0) 200 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2022-09-29T05:22:26.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:25.131980+0000 mon.a (mon.0) 201 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:22:26.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:22:25.132488+0000 mgr.y (mgr.14146) 44 : cephadm [INF] Reconfiguring daemon mgr.y on smithi083 2022-09-29T05:22:26.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:22:25.561450+0000 mgr.y (mgr.14146) 45 : cluster [DBG] pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2022-09-29T05:22:26.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:25.695057+0000 mon.a (mon.0) 202 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:22:26.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:26 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:25.124007+0000 mon.a (mon.0) 197 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:22:26.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:26 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:25.129479+0000 mon.a (mon.0) 198 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:22:26.427 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:26 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:22:25.130570+0000 mgr.y (mgr.14146) 43 : cephadm [INF] Reconfiguring mgr.y (unknown last config time)... 2022-09-29T05:22:26.427 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:26 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:25.130822+0000 mon.a (mon.0) 199 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2022-09-29T05:22:26.427 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:26 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:25.131466+0000 mon.a (mon.0) 200 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2022-09-29T05:22:26.428 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:26 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:25.131980+0000 mon.a (mon.0) 201 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:22:26.428 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:26 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:22:25.132488+0000 mgr.y (mgr.14146) 44 : cephadm [INF] Reconfiguring daemon mgr.y on smithi083 2022-09-29T05:22:26.428 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:26 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:22:25.561450+0000 mgr.y (mgr.14146) 45 : cluster [DBG] pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2022-09-29T05:22:26.428 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:26 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:25.695057+0000 mon.a (mon.0) 202 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:22:26.720 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:22:26 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[40232]: debug 2022-09-29T05:22:26.421+0000 7f99ac095680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2022-09-29T05:22:26.720 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:22:26 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[40232]: debug 2022-09-29T05:22:26.547+0000 7f99ac095680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2022-09-29T05:22:26.721 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:22:26 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[40232]: debug 2022-09-29T05:22:26.626+0000 7f99ac095680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2022-09-29T05:22:27.059 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:22:26 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[40232]: debug 2022-09-29T05:22:26.716+0000 7f99ac095680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2022-09-29T05:22:27.059 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:22:26 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[40232]: debug 2022-09-29T05:22:26.912+0000 7f99ac095680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2022-09-29T05:22:27.399 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:27 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:26.763968+0000 mgr.y (mgr.14146) 46 : audit [DBG] from='client.14208 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi083:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:22:27.399 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:27 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:26.766313+0000 mon.a (mon.0) 203 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2022-09-29T05:22:27.400 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:27 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:26.768000+0000 mon.a (mon.0) 204 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2022-09-29T05:22:27.400 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:27 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:26.768511+0000 mon.a (mon.0) 205 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:22:27.400 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:27 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:26.877836+0000 mon.a (mon.0) 206 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:22:27.400 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:27 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:26.878612+0000 mon.a (mon.0) 207 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:22:27.401 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:27 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:26.879448+0000 mon.a (mon.0) 208 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:22:27.401 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:27 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:26.879967+0000 mon.a (mon.0) 209 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:22:27.401 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:27 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:26.885692+0000 mon.a (mon.0) 210 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:22:27.401 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:27 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:26.763968+0000 mgr.y (mgr.14146) 46 : audit [DBG] from='client.14208 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi083:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:22:27.402 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:27 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:26.766313+0000 mon.a (mon.0) 203 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2022-09-29T05:22:27.402 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:27 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:26.768000+0000 mon.a (mon.0) 204 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2022-09-29T05:22:27.402 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:27 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:26.768511+0000 mon.a (mon.0) 205 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:22:27.403 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:27 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:26.877836+0000 mon.a (mon.0) 206 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:22:27.403 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:27 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:26.878612+0000 mon.a (mon.0) 207 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:22:27.403 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:27 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:26.879448+0000 mon.a (mon.0) 208 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:22:27.403 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:27 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:26.879967+0000 mon.a (mon.0) 209 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:22:27.403 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:27 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:26.885692+0000 mon.a (mon.0) 210 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:22:27.559 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:22:27 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[40232]: debug 2022-09-29T05:22:27.193+0000 7f99ac095680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2022-09-29T05:22:27.559 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:22:27 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[40232]: debug 2022-09-29T05:22:27.290+0000 7f99ac095680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2022-09-29T05:22:27.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:27 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:26.763968+0000 mgr.y (mgr.14146) 46 : audit [DBG] from='client.14208 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi083:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:22:27.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:27 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:26.766313+0000 mon.a (mon.0) 203 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2022-09-29T05:22:27.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:27 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:26.768000+0000 mon.a (mon.0) 204 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2022-09-29T05:22:27.561 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:27 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:26.768511+0000 mon.a (mon.0) 205 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:22:27.561 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:27 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:26.877836+0000 mon.a (mon.0) 206 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:22:27.561 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:27 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:26.878612+0000 mon.a (mon.0) 207 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:22:27.561 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:27 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:26.879448+0000 mon.a (mon.0) 208 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:22:27.562 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:27 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:26.879967+0000 mon.a (mon.0) 209 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:22:27.562 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:27 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:26.885692+0000 mon.a (mon.0) 210 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:22:28.242 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:28 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:22:27.561719+0000 mgr.y (mgr.14146) 47 : cluster [DBG] pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2022-09-29T05:22:28.242 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:28 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:22:27.561719+0000 mgr.y (mgr.14146) 47 : cluster [DBG] pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2022-09-29T05:22:28.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:28 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:22:27.561719+0000 mgr.y (mgr.14146) 47 : cluster [DBG] pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2022-09-29T05:22:28.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:22:28 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[40232]: debug 2022-09-29T05:22:28.008+0000 7f99ac095680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2022-09-29T05:22:28.310 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:22:28 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[40232]: debug 2022-09-29T05:22:28.090+0000 7f99ac095680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2022-09-29T05:22:28.310 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:22:28 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[40232]: debug 2022-09-29T05:22:28.182+0000 7f99ac095680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2022-09-29T05:22:28.618 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:22:28 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[40232]: debug 2022-09-29T05:22:28.354+0000 7f99ac095680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2022-09-29T05:22:28.880 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:22:28 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[40232]: debug 2022-09-29T05:22:28.614+0000 7f99ac095680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2022-09-29T05:22:28.881 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:22:28 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[40232]: debug 2022-09-29T05:22:28.745+0000 7f99ac095680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2022-09-29T05:22:29.144 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:22:28 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[40232]: debug 2022-09-29T05:22:28.876+0000 7f99ac095680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2022-09-29T05:22:29.144 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:22:29 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[40232]: debug 2022-09-29T05:22:29.041+0000 7f99ac095680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2022-09-29T05:22:29.145 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:22:29 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[40232]: debug 2022-09-29T05:22:29.125+0000 7f99ac095680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2022-09-29T05:22:29.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:29 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:28.851236+0000 mon.c (mon.1) 4 : audit [INF] from='client.? 172.21.15.83:0/3142610309' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9b8b79ad-3d00-4dad-8da7-7cff0e65d302"}]: dispatch 2022-09-29T05:22:29.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:29 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:28.851578+0000 mon.a (mon.0) 211 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9b8b79ad-3d00-4dad-8da7-7cff0e65d302"}]: dispatch 2022-09-29T05:22:29.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:29 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:28.854605+0000 mon.a (mon.0) 212 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9b8b79ad-3d00-4dad-8da7-7cff0e65d302"}]': finished 2022-09-29T05:22:29.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:29 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:22:28.854650+0000 mon.a (mon.0) 213 : cluster [DBG] osdmap e5: 1 total, 0 up, 1 in 2022-09-29T05:22:29.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:29 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:28.854732+0000 mon.a (mon.0) 214 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2022-09-29T05:22:29.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:29 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:28.851236+0000 mon.c (mon.1) 4 : audit [INF] from='client.? 172.21.15.83:0/3142610309' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9b8b79ad-3d00-4dad-8da7-7cff0e65d302"}]: dispatch 2022-09-29T05:22:29.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:29 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:28.851578+0000 mon.a (mon.0) 211 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9b8b79ad-3d00-4dad-8da7-7cff0e65d302"}]: dispatch 2022-09-29T05:22:29.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:29 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:28.854605+0000 mon.a (mon.0) 212 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9b8b79ad-3d00-4dad-8da7-7cff0e65d302"}]': finished 2022-09-29T05:22:29.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:29 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:22:28.854650+0000 mon.a (mon.0) 213 : cluster [DBG] osdmap e5: 1 total, 0 up, 1 in 2022-09-29T05:22:29.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:29 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:28.854732+0000 mon.a (mon.0) 214 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2022-09-29T05:22:29.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:29 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:28.851236+0000 mon.c (mon.1) 4 : audit [INF] from='client.? 172.21.15.83:0/3142610309' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9b8b79ad-3d00-4dad-8da7-7cff0e65d302"}]: dispatch 2022-09-29T05:22:29.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:29 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:28.851578+0000 mon.a (mon.0) 211 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9b8b79ad-3d00-4dad-8da7-7cff0e65d302"}]: dispatch 2022-09-29T05:22:29.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:29 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:28.854605+0000 mon.a (mon.0) 212 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9b8b79ad-3d00-4dad-8da7-7cff0e65d302"}]': finished 2022-09-29T05:22:29.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:29 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:22:28.854650+0000 mon.a (mon.0) 213 : cluster [DBG] osdmap e5: 1 total, 0 up, 1 in 2022-09-29T05:22:29.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:29 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:28.854732+0000 mon.a (mon.0) 214 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2022-09-29T05:22:30.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:29.143426+0000 mon.b (mon.2) 2 : audit [DBG] from='mgr.? 172.21.15.153:0/2370690477' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2022-09-29T05:22:30.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:29.143739+0000 mon.b (mon.2) 3 : audit [DBG] from='mgr.? 172.21.15.153:0/2370690477' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2022-09-29T05:22:30.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:22:29.143930+0000 mon.a (mon.0) 215 : cluster [DBG] Standby manager daemon x started 2022-09-29T05:22:30.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:29.144565+0000 mon.b (mon.2) 4 : audit [DBG] from='mgr.? 172.21.15.153:0/2370690477' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2022-09-29T05:22:30.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:29.144845+0000 mon.b (mon.2) 5 : audit [DBG] from='mgr.? 172.21.15.153:0/2370690477' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2022-09-29T05:22:30.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:29.428853+0000 mon.c (mon.1) 5 : audit [DBG] from='client.? 172.21.15.83:0/1939851742' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2022-09-29T05:22:30.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:22:29.561941+0000 mgr.y (mgr.14146) 48 : cluster [DBG] pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2022-09-29T05:22:30.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:29.143426+0000 mon.b (mon.2) 2 : audit [DBG] from='mgr.? 172.21.15.153:0/2370690477' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2022-09-29T05:22:30.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:29.143739+0000 mon.b (mon.2) 3 : audit [DBG] from='mgr.? 172.21.15.153:0/2370690477' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2022-09-29T05:22:30.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:22:29.143930+0000 mon.a (mon.0) 215 : cluster [DBG] Standby manager daemon x started 2022-09-29T05:22:30.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:29.144565+0000 mon.b (mon.2) 4 : audit [DBG] from='mgr.? 172.21.15.153:0/2370690477' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2022-09-29T05:22:30.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:29.144845+0000 mon.b (mon.2) 5 : audit [DBG] from='mgr.? 172.21.15.153:0/2370690477' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2022-09-29T05:22:30.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:29.428853+0000 mon.c (mon.1) 5 : audit [DBG] from='client.? 172.21.15.83:0/1939851742' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2022-09-29T05:22:30.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:22:29.561941+0000 mgr.y (mgr.14146) 48 : cluster [DBG] pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2022-09-29T05:22:30.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:30 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:29.143426+0000 mon.b (mon.2) 2 : audit [DBG] from='mgr.? 172.21.15.153:0/2370690477' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2022-09-29T05:22:30.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:30 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:29.143739+0000 mon.b (mon.2) 3 : audit [DBG] from='mgr.? 172.21.15.153:0/2370690477' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2022-09-29T05:22:30.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:30 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:22:29.143930+0000 mon.a (mon.0) 215 : cluster [DBG] Standby manager daemon x started 2022-09-29T05:22:30.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:30 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:29.144565+0000 mon.b (mon.2) 4 : audit [DBG] from='mgr.? 172.21.15.153:0/2370690477' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2022-09-29T05:22:30.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:30 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:29.144845+0000 mon.b (mon.2) 5 : audit [DBG] from='mgr.? 172.21.15.153:0/2370690477' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2022-09-29T05:22:30.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:30 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:29.428853+0000 mon.c (mon.1) 5 : audit [DBG] from='client.? 172.21.15.83:0/1939851742' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2022-09-29T05:22:30.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:30 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:22:29.561941+0000 mgr.y (mgr.14146) 48 : cluster [DBG] pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2022-09-29T05:22:31.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:22:30.152671+0000 mon.a (mon.0) 216 : cluster [DBG] mgrmap e14: y(active, since 54s), standbys: x 2022-09-29T05:22:31.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:30.152786+0000 mon.a (mon.0) 217 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2022-09-29T05:22:31.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:22:30.152671+0000 mon.a (mon.0) 216 : cluster [DBG] mgrmap e14: y(active, since 54s), standbys: x 2022-09-29T05:22:31.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:30.152786+0000 mon.a (mon.0) 217 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2022-09-29T05:22:31.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:31 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:22:30.152671+0000 mon.a (mon.0) 216 : cluster [DBG] mgrmap e14: y(active, since 54s), standbys: x 2022-09-29T05:22:31.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:31 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:30.152786+0000 mon.a (mon.0) 217 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2022-09-29T05:22:32.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:22:31.562213+0000 mgr.y (mgr.14146) 49 : cluster [DBG] pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2022-09-29T05:22:32.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:22:31.562213+0000 mgr.y (mgr.14146) 49 : cluster [DBG] pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2022-09-29T05:22:32.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:22:31.562213+0000 mgr.y (mgr.14146) 49 : cluster [DBG] pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2022-09-29T05:22:34.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:22:33.562481+0000 mgr.y (mgr.14146) 50 : cluster [DBG] pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2022-09-29T05:22:34.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:34.444180+0000 mon.a (mon.0) 218 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2022-09-29T05:22:34.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:34.444740+0000 mon.a (mon.0) 219 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:22:34.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:22:33.562481+0000 mgr.y (mgr.14146) 50 : cluster [DBG] pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2022-09-29T05:22:34.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:34.444180+0000 mon.a (mon.0) 218 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2022-09-29T05:22:34.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:34.444740+0000 mon.a (mon.0) 219 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:22:35.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:34 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:22:33.562481+0000 mgr.y (mgr.14146) 50 : cluster [DBG] pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2022-09-29T05:22:35.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:34 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:34.444180+0000 mon.a (mon.0) 218 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2022-09-29T05:22:35.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:34 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:34.444740+0000 mon.a (mon.0) 219 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:22:35.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:22:34.445234+0000 mgr.y (mgr.14146) 51 : cephadm [INF] Deploying daemon osd.0 on smithi083 2022-09-29T05:22:35.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:22:34.445234+0000 mgr.y (mgr.14146) 51 : cephadm [INF] Deploying daemon osd.0 on smithi083 2022-09-29T05:22:36.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:35 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:22:34.445234+0000 mgr.y (mgr.14146) 51 : cephadm [INF] Deploying daemon osd.0 on smithi083 2022-09-29T05:22:36.881 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:22:35.562889+0000 mgr.y (mgr.14146) 52 : cluster [DBG] pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2022-09-29T05:22:36.882 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:35.689703+0000 mon.a (mon.0) 220 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:22:36.882 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:35.690567+0000 mon.a (mon.0) 221 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:22:36.882 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:22:35.562889+0000 mgr.y (mgr.14146) 52 : cluster [DBG] pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2022-09-29T05:22:36.883 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:35.689703+0000 mon.a (mon.0) 220 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:22:36.883 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:35.690567+0000 mon.a (mon.0) 221 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:22:37.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:36 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:22:35.562889+0000 mgr.y (mgr.14146) 52 : cluster [DBG] pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2022-09-29T05:22:37.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:36 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:35.689703+0000 mon.a (mon.0) 220 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:22:37.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:36 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:35.690567+0000 mon.a (mon.0) 221 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:22:38.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:38 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:22:37.563175+0000 mgr.y (mgr.14146) 53 : cluster [DBG] pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2022-09-29T05:22:38.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:38 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:22:37.563175+0000 mgr.y (mgr.14146) 53 : cluster [DBG] pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2022-09-29T05:22:39.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:38 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:22:37.563175+0000 mgr.y (mgr.14146) 53 : cluster [DBG] pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2022-09-29T05:22:39.444 INFO:teuthology.orchestra.run.smithi083.stdout:Created osd(s) 0 on host 'smithi083' 2022-09-29T05:22:39.900 DEBUG:teuthology.orchestra.run.smithi083:osd.0> sudo journalctl -f -n 0 -u ceph-69111522-3fb6-11ed-8431-001a4aab830c@osd.0.service 2022-09-29T05:22:39.903 INFO:tasks.cephadm:Deploying osd.1 on smithi083 with /dev/vg_nvme/lv_3... 2022-09-29T05:22:39.904 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/daemon-base:latest-pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 69111522-3fb6-11ed-8431-001a4aab830c -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2022-09-29T05:22:39.933 INFO:journalctl@ceph.osd.0.smithi083.stdout:-- Logs begin at Thu 2022-09-29 05:12:45 UTC. -- 2022-09-29T05:22:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:39.436065+0000 mon.a (mon.0) 222 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:22:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:39.439452+0000 mon.a (mon.0) 223 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:22:40.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:39.475426+0000 mon.a (mon.0) 224 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:22:40.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:39.476348+0000 mon.a (mon.0) 225 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:22:40.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:39.476888+0000 mon.a (mon.0) 226 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:22:40.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:22:39.563441+0000 mgr.y (mgr.14146) 54 : cluster [DBG] pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2022-09-29T05:22:40.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:39.436065+0000 mon.a (mon.0) 222 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:22:40.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:39.439452+0000 mon.a (mon.0) 223 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:22:40.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:39.475426+0000 mon.a (mon.0) 224 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:22:40.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:39.476348+0000 mon.a (mon.0) 225 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:22:40.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:39.476888+0000 mon.a (mon.0) 226 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:22:40.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:22:39.563441+0000 mgr.y (mgr.14146) 54 : cluster [DBG] pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2022-09-29T05:22:40.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:40 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:39.436065+0000 mon.a (mon.0) 222 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:22:40.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:40 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:39.439452+0000 mon.a (mon.0) 223 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:22:40.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:40 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:39.475426+0000 mon.a (mon.0) 224 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:22:40.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:40 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:39.476348+0000 mon.a (mon.0) 225 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:22:40.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:40 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:39.476888+0000 mon.a (mon.0) 226 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:22:40.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:40 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:22:39.563441+0000 mgr.y (mgr.14146) 54 : cluster [DBG] pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2022-09-29T05:22:41.913 INFO:teuthology.orchestra.run.smithi083.stderr:--> Zapping: /dev/vg_nvme/lv_3 2022-09-29T05:22:41.914 INFO:teuthology.orchestra.run.smithi083.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2022-09-29T05:22:41.914 INFO:teuthology.orchestra.run.smithi083.stderr: stderr: 10+0 records in 2022-09-29T05:22:41.914 INFO:teuthology.orchestra.run.smithi083.stderr:10+0 records out 2022-09-29T05:22:41.915 INFO:teuthology.orchestra.run.smithi083.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0193086 s, 543 MB/s 2022-09-29T05:22:41.915 INFO:teuthology.orchestra.run.smithi083.stderr:--> Zapping successful for: 2022-09-29T05:22:42.568 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/daemon-base:latest-pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 69111522-3fb6-11ed-8431-001a4aab830c -- ceph orch daemon add osd smithi083:vg_nvme/lv_3 2022-09-29T05:22:42.713 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 29 05:22:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-osd-0[47139]: debug 2022-09-29T05:22:42.564+0000 7fc3fd494200 -1 osd.0 0 log_to_monitors {default=true} 2022-09-29T05:22:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:22:41.563678+0000 mgr.y (mgr.14146) 55 : cluster [DBG] pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2022-09-29T05:22:42.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:41.571060+0000 mon.a (mon.0) 227 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:22:42.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:42.568645+0000 mon.c (mon.1) 6 : audit [INF] from='osd.0 [v2:172.21.15.83:6802/2080129456,v1:172.21.15.83:6803/2080129456]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2022-09-29T05:22:42.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:42.569008+0000 mon.a (mon.0) 228 : audit [INF] from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2022-09-29T05:22:42.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:22:41.563678+0000 mgr.y (mgr.14146) 55 : cluster [DBG] pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2022-09-29T05:22:42.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:41.571060+0000 mon.a (mon.0) 227 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:22:42.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:42.568645+0000 mon.c (mon.1) 6 : audit [INF] from='osd.0 [v2:172.21.15.83:6802/2080129456,v1:172.21.15.83:6803/2080129456]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2022-09-29T05:22:42.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:42.569008+0000 mon.a (mon.0) 228 : audit [INF] from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2022-09-29T05:22:43.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:22:41.563678+0000 mgr.y (mgr.14146) 55 : cluster [DBG] pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2022-09-29T05:22:43.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:41.571060+0000 mon.a (mon.0) 227 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:22:43.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:42.568645+0000 mon.c (mon.1) 6 : audit [INF] from='osd.0 [v2:172.21.15.83:6802/2080129456,v1:172.21.15.83:6803/2080129456]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2022-09-29T05:22:43.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:42.569008+0000 mon.a (mon.0) 228 : audit [INF] from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2022-09-29T05:22:43.682 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:42.620070+0000 mon.a (mon.0) 229 : audit [INF] from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2022-09-29T05:22:43.682 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:22:42.620128+0000 mon.a (mon.0) 230 : cluster [DBG] osdmap e6: 1 total, 0 up, 1 in 2022-09-29T05:22:43.683 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:42.620227+0000 mon.a (mon.0) 231 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2022-09-29T05:22:43.683 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:42.622286+0000 mon.c (mon.1) 7 : audit [INF] from='osd.0 [v2:172.21.15.83:6802/2080129456,v1:172.21.15.83:6803/2080129456]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]: dispatch 2022-09-29T05:22:43.683 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:42.623963+0000 mon.a (mon.0) 232 : audit [INF] from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]: dispatch 2022-09-29T05:22:43.684 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 29 05:22:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-osd-0[47139]: debug 2022-09-29T05:22:43.629+0000 7fc3f3a1a700 -1 osd.0 0 waiting for initial osdmap 2022-09-29T05:22:43.684 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 29 05:22:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-osd-0[47139]: debug 2022-09-29T05:22:43.632+0000 7fc3ec18e700 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2022-09-29T05:22:43.684 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:42.620070+0000 mon.a (mon.0) 229 : audit [INF] from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2022-09-29T05:22:43.685 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:22:42.620128+0000 mon.a (mon.0) 230 : cluster [DBG] osdmap e6: 1 total, 0 up, 1 in 2022-09-29T05:22:43.685 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:42.620227+0000 mon.a (mon.0) 231 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2022-09-29T05:22:43.685 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:42.622286+0000 mon.c (mon.1) 7 : audit [INF] from='osd.0 [v2:172.21.15.83:6802/2080129456,v1:172.21.15.83:6803/2080129456]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]: dispatch 2022-09-29T05:22:43.685 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:42.623963+0000 mon.a (mon.0) 232 : audit [INF] from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]: dispatch 2022-09-29T05:22:44.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:42.620070+0000 mon.a (mon.0) 229 : audit [INF] from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2022-09-29T05:22:44.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:22:42.620128+0000 mon.a (mon.0) 230 : cluster [DBG] osdmap e6: 1 total, 0 up, 1 in 2022-09-29T05:22:44.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:42.620227+0000 mon.a (mon.0) 231 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2022-09-29T05:22:44.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:42.622286+0000 mon.c (mon.1) 7 : audit [INF] from='osd.0 [v2:172.21.15.83:6802/2080129456,v1:172.21.15.83:6803/2080129456]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]: dispatch 2022-09-29T05:22:44.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:42.623963+0000 mon.a (mon.0) 232 : audit [INF] from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]: dispatch 2022-09-29T05:22:44.887 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:22:43.564026+0000 mgr.y (mgr.14146) 56 : cluster [DBG] pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2022-09-29T05:22:44.887 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:43.623675+0000 mon.a (mon.0) 233 : audit [INF] from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]': finished 2022-09-29T05:22:44.887 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:22:43.623757+0000 mon.a (mon.0) 234 : cluster [DBG] osdmap e7: 1 total, 0 up, 1 in 2022-09-29T05:22:44.887 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:43.624305+0000 mon.a (mon.0) 235 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2022-09-29T05:22:44.888 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:43.630820+0000 mon.a (mon.0) 236 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2022-09-29T05:22:44.888 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:22:43.564026+0000 mgr.y (mgr.14146) 56 : cluster [DBG] pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2022-09-29T05:22:44.888 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:43.623675+0000 mon.a (mon.0) 233 : audit [INF] from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]': finished 2022-09-29T05:22:44.889 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:22:43.623757+0000 mon.a (mon.0) 234 : cluster [DBG] osdmap e7: 1 total, 0 up, 1 in 2022-09-29T05:22:44.889 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:43.624305+0000 mon.a (mon.0) 235 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2022-09-29T05:22:44.889 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:43.630820+0000 mon.a (mon.0) 236 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2022-09-29T05:22:45.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:44 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:22:43.564026+0000 mgr.y (mgr.14146) 56 : cluster [DBG] pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2022-09-29T05:22:45.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:44 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:43.623675+0000 mon.a (mon.0) 233 : audit [INF] from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]': finished 2022-09-29T05:22:45.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:44 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:22:43.623757+0000 mon.a (mon.0) 234 : cluster [DBG] osdmap e7: 1 total, 0 up, 1 in 2022-09-29T05:22:45.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:44 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:43.624305+0000 mon.a (mon.0) 235 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2022-09-29T05:22:45.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:44 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:43.630820+0000 mon.a (mon.0) 236 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2022-09-29T05:22:45.951 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:22:43.518129+0000 osd.0 (osd.0) 1 : cluster [DBG] purged_snaps scrub starts 2022-09-29T05:22:45.951 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:22:43.518219+0000 osd.0 (osd.0) 2 : cluster [DBG] purged_snaps scrub ok 2022-09-29T05:22:45.952 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:44.627936+0000 mon.a (mon.0) 237 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2022-09-29T05:22:45.952 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:22:44.639672+0000 mon.a (mon.0) 238 : cluster [INF] osd.0 [v2:172.21.15.83:6802/2080129456,v1:172.21.15.83:6803/2080129456] boot 2022-09-29T05:22:45.952 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:22:44.639705+0000 mon.a (mon.0) 239 : cluster [DBG] osdmap e8: 1 total, 1 up, 1 in 2022-09-29T05:22:45.952 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:44.639795+0000 mon.a (mon.0) 240 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2022-09-29T05:22:45.953 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:44.694541+0000 mgr.y (mgr.14146) 57 : audit [DBG] from='client.24148 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi083:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:22:45.953 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:44.696455+0000 mon.a (mon.0) 241 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2022-09-29T05:22:45.953 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:44.698175+0000 mon.a (mon.0) 242 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2022-09-29T05:22:45.953 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:44.698712+0000 mon.a (mon.0) 243 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:22:45.953 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:45.184329+0000 mon.a (mon.0) 244 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:22:45.954 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:45.185066+0000 mon.a (mon.0) 245 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:22:45.954 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:45.190213+0000 mon.a (mon.0) 246 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:22:45.954 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:45.195006+0000 mon.a (mon.0) 247 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:22:45.955 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:22:43.518129+0000 osd.0 (osd.0) 1 : cluster [DBG] purged_snaps scrub starts 2022-09-29T05:22:45.955 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:22:43.518219+0000 osd.0 (osd.0) 2 : cluster [DBG] purged_snaps scrub ok 2022-09-29T05:22:45.955 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:44.627936+0000 mon.a (mon.0) 237 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2022-09-29T05:22:45.956 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:22:44.639672+0000 mon.a (mon.0) 238 : cluster [INF] osd.0 [v2:172.21.15.83:6802/2080129456,v1:172.21.15.83:6803/2080129456] boot 2022-09-29T05:22:45.956 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:22:44.639705+0000 mon.a (mon.0) 239 : cluster [DBG] osdmap e8: 1 total, 1 up, 1 in 2022-09-29T05:22:45.956 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:44.639795+0000 mon.a (mon.0) 240 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2022-09-29T05:22:45.956 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:44.694541+0000 mgr.y (mgr.14146) 57 : audit [DBG] from='client.24148 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi083:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:22:45.957 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:44.696455+0000 mon.a (mon.0) 241 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2022-09-29T05:22:45.957 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:44.698175+0000 mon.a (mon.0) 242 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2022-09-29T05:22:45.957 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:44.698712+0000 mon.a (mon.0) 243 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:22:45.957 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:45.184329+0000 mon.a (mon.0) 244 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:22:45.958 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:45.185066+0000 mon.a (mon.0) 245 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:22:45.958 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:45.190213+0000 mon.a (mon.0) 246 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:22:45.958 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:45.195006+0000 mon.a (mon.0) 247 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:22:46.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:45 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:22:43.518129+0000 osd.0 (osd.0) 1 : cluster [DBG] purged_snaps scrub starts 2022-09-29T05:22:46.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:45 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:22:43.518219+0000 osd.0 (osd.0) 2 : cluster [DBG] purged_snaps scrub ok 2022-09-29T05:22:46.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:45 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:44.627936+0000 mon.a (mon.0) 237 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2022-09-29T05:22:46.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:45 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:22:44.639672+0000 mon.a (mon.0) 238 : cluster [INF] osd.0 [v2:172.21.15.83:6802/2080129456,v1:172.21.15.83:6803/2080129456] boot 2022-09-29T05:22:46.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:45 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:22:44.639705+0000 mon.a (mon.0) 239 : cluster [DBG] osdmap e8: 1 total, 1 up, 1 in 2022-09-29T05:22:46.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:45 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:44.639795+0000 mon.a (mon.0) 240 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2022-09-29T05:22:46.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:45 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:44.694541+0000 mgr.y (mgr.14146) 57 : audit [DBG] from='client.24148 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi083:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:22:46.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:45 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:44.696455+0000 mon.a (mon.0) 241 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2022-09-29T05:22:46.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:45 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:44.698175+0000 mon.a (mon.0) 242 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2022-09-29T05:22:46.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:45 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:44.698712+0000 mon.a (mon.0) 243 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:22:46.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:45 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:45.184329+0000 mon.a (mon.0) 244 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:22:46.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:45 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:45.185066+0000 mon.a (mon.0) 245 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:22:46.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:45 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:45.190213+0000 mon.a (mon.0) 246 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:22:46.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:45 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:45.195006+0000 mon.a (mon.0) 247 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:22:46.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:22:45.179068+0000 mgr.y (mgr.14146) 58 : cephadm [INF] Detected new or changed devices on smithi083 2022-09-29T05:22:46.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:22:45.185569+0000 mgr.y (mgr.14146) 59 : cephadm [INF] Adjusting osd_memory_target on smithi083 to 16144M 2022-09-29T05:22:46.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:22:45.564475+0000 mgr.y (mgr.14146) 60 : cluster [DBG] pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2022-09-29T05:22:46.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:22:46.200758+0000 mon.a (mon.0) 248 : cluster [DBG] osdmap e9: 1 total, 1 up, 1 in 2022-09-29T05:22:46.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:22:45.179068+0000 mgr.y (mgr.14146) 58 : cephadm [INF] Detected new or changed devices on smithi083 2022-09-29T05:22:46.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:22:45.185569+0000 mgr.y (mgr.14146) 59 : cephadm [INF] Adjusting osd_memory_target on smithi083 to 16144M 2022-09-29T05:22:46.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:22:45.564475+0000 mgr.y (mgr.14146) 60 : cluster [DBG] pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2022-09-29T05:22:46.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:22:46.200758+0000 mon.a (mon.0) 248 : cluster [DBG] osdmap e9: 1 total, 1 up, 1 in 2022-09-29T05:22:47.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:46 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:22:45.179068+0000 mgr.y (mgr.14146) 58 : cephadm [INF] Detected new or changed devices on smithi083 2022-09-29T05:22:47.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:46 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:22:45.185569+0000 mgr.y (mgr.14146) 59 : cephadm [INF] Adjusting osd_memory_target on smithi083 to 16144M 2022-09-29T05:22:47.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:46 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:22:45.564475+0000 mgr.y (mgr.14146) 60 : cluster [DBG] pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2022-09-29T05:22:47.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:46 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:22:46.200758+0000 mon.a (mon.0) 248 : cluster [DBG] osdmap e9: 1 total, 1 up, 1 in 2022-09-29T05:22:47.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:47 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:46.845887+0000 mon.a (mon.0) 249 : audit [INF] from='client.? 172.21.15.83:0/437801724' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e098cff5-70cc-437e-8759-e6c7e3d452d2"}]: dispatch 2022-09-29T05:22:47.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:47 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:46.849424+0000 mon.a (mon.0) 250 : audit [INF] from='client.? 172.21.15.83:0/437801724' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "e098cff5-70cc-437e-8759-e6c7e3d452d2"}]': finished 2022-09-29T05:22:47.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:47 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:22:46.849472+0000 mon.a (mon.0) 251 : cluster [DBG] osdmap e10: 2 total, 1 up, 2 in 2022-09-29T05:22:47.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:47 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:46.849579+0000 mon.a (mon.0) 252 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2022-09-29T05:22:47.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:47 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:47.450839+0000 mon.c (mon.1) 8 : audit [DBG] from='client.? 172.21.15.83:0/1993641018' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2022-09-29T05:22:47.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:47 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:46.845887+0000 mon.a (mon.0) 249 : audit [INF] from='client.? 172.21.15.83:0/437801724' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e098cff5-70cc-437e-8759-e6c7e3d452d2"}]: dispatch 2022-09-29T05:22:47.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:47 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:46.849424+0000 mon.a (mon.0) 250 : audit [INF] from='client.? 172.21.15.83:0/437801724' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "e098cff5-70cc-437e-8759-e6c7e3d452d2"}]': finished 2022-09-29T05:22:47.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:47 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:22:46.849472+0000 mon.a (mon.0) 251 : cluster [DBG] osdmap e10: 2 total, 1 up, 2 in 2022-09-29T05:22:47.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:47 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:46.849579+0000 mon.a (mon.0) 252 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2022-09-29T05:22:47.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:47 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:47.450839+0000 mon.c (mon.1) 8 : audit [DBG] from='client.? 172.21.15.83:0/1993641018' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2022-09-29T05:22:48.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:47 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:46.845887+0000 mon.a (mon.0) 249 : audit [INF] from='client.? 172.21.15.83:0/437801724' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e098cff5-70cc-437e-8759-e6c7e3d452d2"}]: dispatch 2022-09-29T05:22:48.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:47 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:46.849424+0000 mon.a (mon.0) 250 : audit [INF] from='client.? 172.21.15.83:0/437801724' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "e098cff5-70cc-437e-8759-e6c7e3d452d2"}]': finished 2022-09-29T05:22:48.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:47 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:22:46.849472+0000 mon.a (mon.0) 251 : cluster [DBG] osdmap e10: 2 total, 1 up, 2 in 2022-09-29T05:22:48.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:47 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:46.849579+0000 mon.a (mon.0) 252 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2022-09-29T05:22:48.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:47 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:47.450839+0000 mon.c (mon.1) 8 : audit [DBG] from='client.? 172.21.15.83:0/1993641018' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2022-09-29T05:22:49.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:48 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:22:47.564800+0000 mgr.y (mgr.14146) 61 : cluster [DBG] pgmap v36: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2022-09-29T05:22:49.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:48 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:22:47.564800+0000 mgr.y (mgr.14146) 61 : cluster [DBG] pgmap v36: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2022-09-29T05:22:49.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:48 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:22:47.564800+0000 mgr.y (mgr.14146) 61 : cluster [DBG] pgmap v36: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2022-09-29T05:22:50.903 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:22:49.565080+0000 mgr.y (mgr.14146) 62 : cluster [DBG] pgmap v37: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2022-09-29T05:22:50.904 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:22:49.565080+0000 mgr.y (mgr.14146) 62 : cluster [DBG] pgmap v37: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2022-09-29T05:22:51.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:50 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:22:49.565080+0000 mgr.y (mgr.14146) 62 : cluster [DBG] pgmap v37: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2022-09-29T05:22:52.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:22:51.565369+0000 mgr.y (mgr.14146) 63 : cluster [DBG] pgmap v38: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2022-09-29T05:22:52.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:22:51.565369+0000 mgr.y (mgr.14146) 63 : cluster [DBG] pgmap v38: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2022-09-29T05:22:53.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:22:51.565369+0000 mgr.y (mgr.14146) 63 : cluster [DBG] pgmap v38: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2022-09-29T05:22:53.580 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:52.680628+0000 mon.a (mon.0) 253 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2022-09-29T05:22:53.581 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:52.681204+0000 mon.a (mon.0) 254 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:22:53.581 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:22:52.681644+0000 mgr.y (mgr.14146) 64 : cephadm [INF] Deploying daemon osd.1 on smithi083 2022-09-29T05:22:53.886 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:52.680628+0000 mon.a (mon.0) 253 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2022-09-29T05:22:53.887 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:52.681204+0000 mon.a (mon.0) 254 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:22:53.887 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:22:52.681644+0000 mgr.y (mgr.14146) 64 : cephadm [INF] Deploying daemon osd.1 on smithi083 2022-09-29T05:22:54.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:52.680628+0000 mon.a (mon.0) 253 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2022-09-29T05:22:54.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:52.681204+0000 mon.a (mon.0) 254 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:22:54.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:22:52.681644+0000 mgr.y (mgr.14146) 64 : cephadm [INF] Deploying daemon osd.1 on smithi083 2022-09-29T05:22:54.916 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:22:53.565702+0000 mgr.y (mgr.14146) 65 : cluster [DBG] pgmap v39: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2022-09-29T05:22:54.916 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:22:53.565702+0000 mgr.y (mgr.14146) 65 : cluster [DBG] pgmap v39: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2022-09-29T05:22:55.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:54 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:22:53.565702+0000 mgr.y (mgr.14146) 65 : cluster [DBG] pgmap v39: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2022-09-29T05:22:56.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:22:55.565971+0000 mgr.y (mgr.14146) 66 : cluster [DBG] pgmap v40: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2022-09-29T05:22:56.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:22:55.565971+0000 mgr.y (mgr.14146) 66 : cluster [DBG] pgmap v40: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2022-09-29T05:22:57.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:56 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:22:55.565971+0000 mgr.y (mgr.14146) 66 : cluster [DBG] pgmap v40: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2022-09-29T05:22:57.763 INFO:teuthology.orchestra.run.smithi083.stdout:Created osd(s) 1 on host 'smithi083' 2022-09-29T05:22:58.257 DEBUG:teuthology.orchestra.run.smithi083:osd.1> sudo journalctl -f -n 0 -u ceph-69111522-3fb6-11ed-8431-001a4aab830c@osd.1.service 2022-09-29T05:22:58.260 INFO:tasks.cephadm:Deploying osd.2 on smithi083 with /dev/vg_nvme/lv_2... 2022-09-29T05:22:58.260 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/daemon-base:latest-pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 69111522-3fb6-11ed-8431-001a4aab830c -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2022-09-29T05:22:58.290 INFO:journalctl@ceph.osd.1.smithi083.stdout:-- Logs begin at Thu 2022-09-29 05:12:45 UTC. -- 2022-09-29T05:22:59.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:58 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:22:57.566294+0000 mgr.y (mgr.14146) 67 : cluster [DBG] pgmap v41: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2022-09-29T05:22:59.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:58 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:57.754756+0000 mon.a (mon.0) 255 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:22:59.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:58 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:57.758937+0000 mon.a (mon.0) 256 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:22:59.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:58 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:57.784124+0000 mon.a (mon.0) 257 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:22:59.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:58 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:57.785134+0000 mon.a (mon.0) 258 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:22:59.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:22:58 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:22:57.785651+0000 mon.a (mon.0) 259 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:22:59.193 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:22:57.566294+0000 mgr.y (mgr.14146) 67 : cluster [DBG] pgmap v41: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2022-09-29T05:22:59.193 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:57.754756+0000 mon.a (mon.0) 255 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:22:59.193 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:57.758937+0000 mon.a (mon.0) 256 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:22:59.193 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:57.784124+0000 mon.a (mon.0) 257 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:22:59.194 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:57.785134+0000 mon.a (mon.0) 258 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:22:59.194 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:22:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:22:57.785651+0000 mon.a (mon.0) 259 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:22:59.194 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:22:57.566294+0000 mgr.y (mgr.14146) 67 : cluster [DBG] pgmap v41: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2022-09-29T05:22:59.195 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:57.754756+0000 mon.a (mon.0) 255 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:22:59.195 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:57.758937+0000 mon.a (mon.0) 256 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:22:59.195 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:57.784124+0000 mon.a (mon.0) 257 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:22:59.195 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:57.785134+0000 mon.a (mon.0) 258 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:22:59.196 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:22:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:22:57.785651+0000 mon.a (mon.0) 259 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:23:00.346 INFO:teuthology.orchestra.run.smithi083.stderr:--> Zapping: /dev/vg_nvme/lv_2 2022-09-29T05:23:00.347 INFO:teuthology.orchestra.run.smithi083.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2022-09-29T05:23:00.347 INFO:teuthology.orchestra.run.smithi083.stderr: stderr: 10+0 records in 2022-09-29T05:23:00.347 INFO:teuthology.orchestra.run.smithi083.stderr:10+0 records out 2022-09-29T05:23:00.348 INFO:teuthology.orchestra.run.smithi083.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0172043 s, 609 MB/s 2022-09-29T05:23:00.348 INFO:teuthology.orchestra.run.smithi083.stderr:--> Zapping successful for: 2022-09-29T05:23:00.915 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:22:59.566550+0000 mgr.y (mgr.14146) 68 : cluster [DBG] pgmap v42: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2022-09-29T05:23:00.916 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:00.011307+0000 mon.a (mon.0) 260 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:23:00.916 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 29 05:23:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-osd-1[50140]: debug 2022-09-29T05:23:00.888+0000 7f1b675c7200 -1 osd.1 0 log_to_monitors {default=true} 2022-09-29T05:23:00.917 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:22:59.566550+0000 mgr.y (mgr.14146) 68 : cluster [DBG] pgmap v42: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2022-09-29T05:23:00.917 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:00.011307+0000 mon.a (mon.0) 260 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:23:00.926 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/daemon-base:latest-pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 69111522-3fb6-11ed-8431-001a4aab830c -- ceph orch daemon add osd smithi083:vg_nvme/lv_2 2022-09-29T05:23:01.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:00 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:22:59.566550+0000 mgr.y (mgr.14146) 68 : cluster [DBG] pgmap v42: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2022-09-29T05:23:01.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:00 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:00.011307+0000 mon.a (mon.0) 260 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:23:01.816 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:00.891124+0000 mon.a (mon.0) 261 : audit [INF] from='osd.1 [v2:172.21.15.83:6810/605787053,v1:172.21.15.83:6811/605787053]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2022-09-29T05:23:01.817 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:00.891124+0000 mon.a (mon.0) 261 : audit [INF] from='osd.1 [v2:172.21.15.83:6810/605787053,v1:172.21.15.83:6811/605787053]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2022-09-29T05:23:02.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:01 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:00.891124+0000 mon.a (mon.0) 261 : audit [INF] from='osd.1 [v2:172.21.15.83:6810/605787053,v1:172.21.15.83:6811/605787053]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2022-09-29T05:23:02.963 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 29 05:23:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-osd-1[50140]: debug 2022-09-29T05:23:02.766+0000 7f1b5db4d700 -1 osd.1 0 waiting for initial osdmap 2022-09-29T05:23:02.963 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 29 05:23:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-osd-1[50140]: debug 2022-09-29T05:23:02.769+0000 7f1b57ac4700 -1 osd.1 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2022-09-29T05:23:02.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:23:01.566776+0000 mgr.y (mgr.14146) 69 : cluster [DBG] pgmap v43: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2022-09-29T05:23:02.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:01.762426+0000 mon.a (mon.0) 262 : audit [INF] from='osd.1 [v2:172.21.15.83:6810/605787053,v1:172.21.15.83:6811/605787053]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2022-09-29T05:23:02.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:23:01.762491+0000 mon.a (mon.0) 263 : cluster [DBG] osdmap e11: 2 total, 1 up, 2 in 2022-09-29T05:23:02.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:01.762614+0000 mon.a (mon.0) 264 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2022-09-29T05:23:02.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:01.763087+0000 mon.a (mon.0) 265 : audit [INF] from='osd.1 [v2:172.21.15.83:6810/605787053,v1:172.21.15.83:6811/605787053]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]: dispatch 2022-09-29T05:23:02.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:23:01.566776+0000 mgr.y (mgr.14146) 69 : cluster [DBG] pgmap v43: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2022-09-29T05:23:02.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:01.762426+0000 mon.a (mon.0) 262 : audit [INF] from='osd.1 [v2:172.21.15.83:6810/605787053,v1:172.21.15.83:6811/605787053]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2022-09-29T05:23:02.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:23:01.762491+0000 mon.a (mon.0) 263 : cluster [DBG] osdmap e11: 2 total, 1 up, 2 in 2022-09-29T05:23:02.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:01.762614+0000 mon.a (mon.0) 264 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2022-09-29T05:23:02.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:01.763087+0000 mon.a (mon.0) 265 : audit [INF] from='osd.1 [v2:172.21.15.83:6810/605787053,v1:172.21.15.83:6811/605787053]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]: dispatch 2022-09-29T05:23:03.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:23:01.566776+0000 mgr.y (mgr.14146) 69 : cluster [DBG] pgmap v43: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2022-09-29T05:23:03.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:01.762426+0000 mon.a (mon.0) 262 : audit [INF] from='osd.1 [v2:172.21.15.83:6810/605787053,v1:172.21.15.83:6811/605787053]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2022-09-29T05:23:03.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:23:01.762491+0000 mon.a (mon.0) 263 : cluster [DBG] osdmap e11: 2 total, 1 up, 2 in 2022-09-29T05:23:03.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:01.762614+0000 mon.a (mon.0) 264 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2022-09-29T05:23:03.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:01.763087+0000 mon.a (mon.0) 265 : audit [INF] from='osd.1 [v2:172.21.15.83:6810/605787053,v1:172.21.15.83:6811/605787053]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]: dispatch 2022-09-29T05:23:04.044 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:02.763508+0000 mon.a (mon.0) 266 : audit [INF] from='osd.1 [v2:172.21.15.83:6810/605787053,v1:172.21.15.83:6811/605787053]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]': finished 2022-09-29T05:23:04.045 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:23:02.763557+0000 mon.a (mon.0) 267 : cluster [DBG] osdmap e12: 2 total, 1 up, 2 in 2022-09-29T05:23:04.045 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:02.763885+0000 mon.a (mon.0) 268 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2022-09-29T05:23:04.045 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:02.767644+0000 mon.a (mon.0) 269 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2022-09-29T05:23:04.045 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:03.059217+0000 mon.a (mon.0) 270 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2022-09-29T05:23:04.046 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:03.061085+0000 mon.a (mon.0) 271 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2022-09-29T05:23:04.046 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:03.061628+0000 mon.a (mon.0) 272 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:23:04.046 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:03.763410+0000 mon.a (mon.0) 273 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:23:04.046 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:02.763508+0000 mon.a (mon.0) 266 : audit [INF] from='osd.1 [v2:172.21.15.83:6810/605787053,v1:172.21.15.83:6811/605787053]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]': finished 2022-09-29T05:23:04.047 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:23:02.763557+0000 mon.a (mon.0) 267 : cluster [DBG] osdmap e12: 2 total, 1 up, 2 in 2022-09-29T05:23:04.047 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:02.763885+0000 mon.a (mon.0) 268 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2022-09-29T05:23:04.047 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:02.767644+0000 mon.a (mon.0) 269 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2022-09-29T05:23:04.047 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:03.059217+0000 mon.a (mon.0) 270 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2022-09-29T05:23:04.048 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:03.061085+0000 mon.a (mon.0) 271 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2022-09-29T05:23:04.048 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:03.061628+0000 mon.a (mon.0) 272 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:23:04.048 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:03.763410+0000 mon.a (mon.0) 273 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:23:04.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:02.763508+0000 mon.a (mon.0) 266 : audit [INF] from='osd.1 [v2:172.21.15.83:6810/605787053,v1:172.21.15.83:6811/605787053]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]': finished 2022-09-29T05:23:04.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:23:02.763557+0000 mon.a (mon.0) 267 : cluster [DBG] osdmap e12: 2 total, 1 up, 2 in 2022-09-29T05:23:04.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:02.763885+0000 mon.a (mon.0) 268 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2022-09-29T05:23:04.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:02.767644+0000 mon.a (mon.0) 269 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2022-09-29T05:23:04.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:03.059217+0000 mon.a (mon.0) 270 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2022-09-29T05:23:04.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:03.061085+0000 mon.a (mon.0) 271 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2022-09-29T05:23:04.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:03.061628+0000 mon.a (mon.0) 272 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:23:04.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:03.763410+0000 mon.a (mon.0) 273 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:23:05.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:04 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:23:01.859124+0000 osd.1 (osd.1) 1 : cluster [DBG] purged_snaps scrub starts 2022-09-29T05:23:05.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:04 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:23:01.859211+0000 osd.1 (osd.1) 2 : cluster [DBG] purged_snaps scrub ok 2022-09-29T05:23:05.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:04 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:03.057189+0000 mgr.y (mgr.14146) 70 : audit [DBG] from='client.14262 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi083:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:23:05.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:04 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:23:03.567049+0000 mgr.y (mgr.14146) 71 : cluster [DBG] pgmap v46: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2022-09-29T05:23:05.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:04 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:23:03.758741+0000 mgr.y (mgr.14146) 72 : cephadm [INF] Detected new or changed devices on smithi083 2022-09-29T05:23:05.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:04 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:03.766874+0000 mon.a (mon.0) 274 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2022-09-29T05:23:05.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:04 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:23:03.773311+0000 mon.a (mon.0) 275 : cluster [INF] osd.1 [v2:172.21.15.83:6810/605787053,v1:172.21.15.83:6811/605787053] boot 2022-09-29T05:23:05.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:04 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:23:03.773355+0000 mon.a (mon.0) 276 : cluster [DBG] osdmap e13: 2 total, 2 up, 2 in 2022-09-29T05:23:05.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:04 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:03.774585+0000 mon.a (mon.0) 277 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2022-09-29T05:23:05.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:04 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:04.216183+0000 mon.a (mon.0) 278 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:23:05.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:04 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:04.216802+0000 mon.a (mon.0) 279 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:23:05.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:04 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:04.220747+0000 mon.a (mon.0) 280 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:23:05.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:04 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:04.226641+0000 mon.a (mon.0) 281 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:23:05.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:23:01.859124+0000 osd.1 (osd.1) 1 : cluster [DBG] purged_snaps scrub starts 2022-09-29T05:23:05.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:23:01.859211+0000 osd.1 (osd.1) 2 : cluster [DBG] purged_snaps scrub ok 2022-09-29T05:23:05.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:03.057189+0000 mgr.y (mgr.14146) 70 : audit [DBG] from='client.14262 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi083:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:23:05.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:23:03.567049+0000 mgr.y (mgr.14146) 71 : cluster [DBG] pgmap v46: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2022-09-29T05:23:05.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:23:03.758741+0000 mgr.y (mgr.14146) 72 : cephadm [INF] Detected new or changed devices on smithi083 2022-09-29T05:23:05.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:03.766874+0000 mon.a (mon.0) 274 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2022-09-29T05:23:05.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:23:03.773311+0000 mon.a (mon.0) 275 : cluster [INF] osd.1 [v2:172.21.15.83:6810/605787053,v1:172.21.15.83:6811/605787053] boot 2022-09-29T05:23:05.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:23:03.773355+0000 mon.a (mon.0) 276 : cluster [DBG] osdmap e13: 2 total, 2 up, 2 in 2022-09-29T05:23:05.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:03.774585+0000 mon.a (mon.0) 277 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2022-09-29T05:23:05.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:04.216183+0000 mon.a (mon.0) 278 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:23:05.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:04.216802+0000 mon.a (mon.0) 279 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:23:05.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:04.220747+0000 mon.a (mon.0) 280 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:23:05.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:04.226641+0000 mon.a (mon.0) 281 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:23:05.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:23:01.859124+0000 osd.1 (osd.1) 1 : cluster [DBG] purged_snaps scrub starts 2022-09-29T05:23:05.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:23:01.859211+0000 osd.1 (osd.1) 2 : cluster [DBG] purged_snaps scrub ok 2022-09-29T05:23:05.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:03.057189+0000 mgr.y (mgr.14146) 70 : audit [DBG] from='client.14262 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi083:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:23:05.218 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:23:03.567049+0000 mgr.y (mgr.14146) 71 : cluster [DBG] pgmap v46: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2022-09-29T05:23:05.218 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:23:03.758741+0000 mgr.y (mgr.14146) 72 : cephadm [INF] Detected new or changed devices on smithi083 2022-09-29T05:23:05.218 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:03.766874+0000 mon.a (mon.0) 274 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2022-09-29T05:23:05.218 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:23:03.773311+0000 mon.a (mon.0) 275 : cluster [INF] osd.1 [v2:172.21.15.83:6810/605787053,v1:172.21.15.83:6811/605787053] boot 2022-09-29T05:23:05.218 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:23:03.773355+0000 mon.a (mon.0) 276 : cluster [DBG] osdmap e13: 2 total, 2 up, 2 in 2022-09-29T05:23:05.219 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:03.774585+0000 mon.a (mon.0) 277 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2022-09-29T05:23:05.219 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:04.216183+0000 mon.a (mon.0) 278 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:23:05.219 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:04.216802+0000 mon.a (mon.0) 279 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:23:05.219 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:04.220747+0000 mon.a (mon.0) 280 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:23:05.220 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:04.226641+0000 mon.a (mon.0) 281 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:23:06.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:05 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:23:04.217206+0000 mgr.y (mgr.14146) 73 : cephadm [INF] Adjusting osd_memory_target on smithi083 to 8072M 2022-09-29T05:23:06.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:05 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:23:04.791109+0000 mon.a (mon.0) 282 : cluster [DBG] osdmap e14: 2 total, 2 up, 2 in 2022-09-29T05:23:06.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:05 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:05.378041+0000 mon.a (mon.0) 283 : audit [INF] from='client.? 172.21.15.83:0/1010174808' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2c14853b-5b64-496d-a011-5d8a9b17b731"}]: dispatch 2022-09-29T05:23:06.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:05 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:05.386308+0000 mon.a (mon.0) 284 : audit [INF] from='client.? 172.21.15.83:0/1010174808' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "2c14853b-5b64-496d-a011-5d8a9b17b731"}]': finished 2022-09-29T05:23:06.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:05 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:23:05.386353+0000 mon.a (mon.0) 285 : cluster [DBG] osdmap e15: 3 total, 2 up, 3 in 2022-09-29T05:23:06.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:05 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:05.386449+0000 mon.a (mon.0) 286 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2022-09-29T05:23:06.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:23:04.217206+0000 mgr.y (mgr.14146) 73 : cephadm [INF] Adjusting osd_memory_target on smithi083 to 8072M 2022-09-29T05:23:06.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:23:04.791109+0000 mon.a (mon.0) 282 : cluster [DBG] osdmap e14: 2 total, 2 up, 2 in 2022-09-29T05:23:06.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:05.378041+0000 mon.a (mon.0) 283 : audit [INF] from='client.? 172.21.15.83:0/1010174808' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2c14853b-5b64-496d-a011-5d8a9b17b731"}]: dispatch 2022-09-29T05:23:06.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:05.386308+0000 mon.a (mon.0) 284 : audit [INF] from='client.? 172.21.15.83:0/1010174808' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "2c14853b-5b64-496d-a011-5d8a9b17b731"}]': finished 2022-09-29T05:23:06.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:23:05.386353+0000 mon.a (mon.0) 285 : cluster [DBG] osdmap e15: 3 total, 2 up, 3 in 2022-09-29T05:23:06.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:05.386449+0000 mon.a (mon.0) 286 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2022-09-29T05:23:06.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:23:04.217206+0000 mgr.y (mgr.14146) 73 : cephadm [INF] Adjusting osd_memory_target on smithi083 to 8072M 2022-09-29T05:23:06.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:23:04.791109+0000 mon.a (mon.0) 282 : cluster [DBG] osdmap e14: 2 total, 2 up, 2 in 2022-09-29T05:23:06.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:05.378041+0000 mon.a (mon.0) 283 : audit [INF] from='client.? 172.21.15.83:0/1010174808' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2c14853b-5b64-496d-a011-5d8a9b17b731"}]: dispatch 2022-09-29T05:23:06.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:05.386308+0000 mon.a (mon.0) 284 : audit [INF] from='client.? 172.21.15.83:0/1010174808' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "2c14853b-5b64-496d-a011-5d8a9b17b731"}]': finished 2022-09-29T05:23:06.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:23:05.386353+0000 mon.a (mon.0) 285 : cluster [DBG] osdmap e15: 3 total, 2 up, 3 in 2022-09-29T05:23:06.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:05.386449+0000 mon.a (mon.0) 286 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2022-09-29T05:23:07.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:06 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:23:05.567305+0000 mgr.y (mgr.14146) 74 : cluster [DBG] pgmap v50: 0 pgs: ; 0 B data, 9.5 MiB used, 179 GiB / 179 GiB avail 2022-09-29T05:23:07.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:06 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:05.966462+0000 mon.c (mon.1) 9 : audit [DBG] from='client.? 172.21.15.83:0/1676537475' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2022-09-29T05:23:07.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:23:05.567305+0000 mgr.y (mgr.14146) 74 : cluster [DBG] pgmap v50: 0 pgs: ; 0 B data, 9.5 MiB used, 179 GiB / 179 GiB avail 2022-09-29T05:23:07.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:05.966462+0000 mon.c (mon.1) 9 : audit [DBG] from='client.? 172.21.15.83:0/1676537475' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2022-09-29T05:23:07.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:23:05.567305+0000 mgr.y (mgr.14146) 74 : cluster [DBG] pgmap v50: 0 pgs: ; 0 B data, 9.5 MiB used, 179 GiB / 179 GiB avail 2022-09-29T05:23:07.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:05.966462+0000 mon.c (mon.1) 9 : audit [DBG] from='client.? 172.21.15.83:0/1676537475' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2022-09-29T05:23:08.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:08 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:23:07.567580+0000 mgr.y (mgr.14146) 75 : cluster [DBG] pgmap v51: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2022-09-29T05:23:08.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:08 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:23:07.567580+0000 mgr.y (mgr.14146) 75 : cluster [DBG] pgmap v51: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2022-09-29T05:23:09.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:08 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:23:07.567580+0000 mgr.y (mgr.14146) 75 : cluster [DBG] pgmap v51: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2022-09-29T05:23:11.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:10 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:23:09.567928+0000 mgr.y (mgr.14146) 76 : cluster [DBG] pgmap v52: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2022-09-29T05:23:11.146 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:23:09.567928+0000 mgr.y (mgr.14146) 76 : cluster [DBG] pgmap v52: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2022-09-29T05:23:11.147 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:23:09.567928+0000 mgr.y (mgr.14146) 76 : cluster [DBG] pgmap v52: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2022-09-29T05:23:12.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:11 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:11.221133+0000 mon.a (mon.0) 287 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2022-09-29T05:23:12.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:11 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:11.221759+0000 mon.a (mon.0) 288 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:23:12.185 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:11.221133+0000 mon.a (mon.0) 287 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2022-09-29T05:23:12.185 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:11.221759+0000 mon.a (mon.0) 288 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:23:12.186 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:11.221133+0000 mon.a (mon.0) 287 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2022-09-29T05:23:12.186 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:11.221759+0000 mon.a (mon.0) 288 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:23:13.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:23:11.222238+0000 mgr.y (mgr.14146) 77 : cephadm [INF] Deploying daemon osd.2 on smithi083 2022-09-29T05:23:13.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:23:11.568453+0000 mgr.y (mgr.14146) 78 : cluster [DBG] pgmap v53: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2022-09-29T05:23:13.152 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:23:11.222238+0000 mgr.y (mgr.14146) 77 : cephadm [INF] Deploying daemon osd.2 on smithi083 2022-09-29T05:23:13.153 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:23:11.568453+0000 mgr.y (mgr.14146) 78 : cluster [DBG] pgmap v53: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2022-09-29T05:23:13.153 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:23:11.222238+0000 mgr.y (mgr.14146) 77 : cephadm [INF] Deploying daemon osd.2 on smithi083 2022-09-29T05:23:13.153 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:23:11.568453+0000 mgr.y (mgr.14146) 78 : cluster [DBG] pgmap v53: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2022-09-29T05:23:15.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:14 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:23:13.570703+0000 mgr.y (mgr.14146) 79 : cluster [DBG] pgmap v54: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2022-09-29T05:23:15.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:23:13.570703+0000 mgr.y (mgr.14146) 79 : cluster [DBG] pgmap v54: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2022-09-29T05:23:15.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:23:13.570703+0000 mgr.y (mgr.14146) 79 : cluster [DBG] pgmap v54: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2022-09-29T05:23:16.120 INFO:teuthology.orchestra.run.smithi083.stdout:Created osd(s) 2 on host 'smithi083' 2022-09-29T05:23:16.595 DEBUG:teuthology.orchestra.run.smithi083:osd.2> sudo journalctl -f -n 0 -u ceph-69111522-3fb6-11ed-8431-001a4aab830c@osd.2.service 2022-09-29T05:23:16.598 INFO:tasks.cephadm:Deploying osd.3 on smithi083 with /dev/vg_nvme/lv_1... 2022-09-29T05:23:16.599 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/daemon-base:latest-pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 69111522-3fb6-11ed-8431-001a4aab830c -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2022-09-29T05:23:16.630 INFO:journalctl@ceph.osd.2.smithi083.stdout:-- Logs begin at Thu 2022-09-29 05:12:45 UTC. -- 2022-09-29T05:23:16.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:23:15.571116+0000 mgr.y (mgr.14146) 80 : cluster [DBG] pgmap v55: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2022-09-29T05:23:16.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:16.111126+0000 mon.a (mon.0) 289 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:23:16.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:16.115843+0000 mon.a (mon.0) 290 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:23:16.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:16.116102+0000 mon.a (mon.0) 291 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:23:16.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:16.118286+0000 mon.a (mon.0) 292 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:23:16.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:16.118912+0000 mon.a (mon.0) 293 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:23:16.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:23:15.571116+0000 mgr.y (mgr.14146) 80 : cluster [DBG] pgmap v55: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2022-09-29T05:23:16.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:16.111126+0000 mon.a (mon.0) 289 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:23:16.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:16.115843+0000 mon.a (mon.0) 290 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:23:16.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:16.116102+0000 mon.a (mon.0) 291 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:23:16.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:16.118286+0000 mon.a (mon.0) 292 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:23:16.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:16.118912+0000 mon.a (mon.0) 293 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:23:17.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:16 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:23:15.571116+0000 mgr.y (mgr.14146) 80 : cluster [DBG] pgmap v55: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2022-09-29T05:23:17.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:16 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:16.111126+0000 mon.a (mon.0) 289 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:23:17.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:16 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:16.115843+0000 mon.a (mon.0) 290 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:23:17.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:16 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:16.116102+0000 mon.a (mon.0) 291 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:23:17.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:16 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:16.118286+0000 mon.a (mon.0) 292 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:23:17.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:16 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:16.118912+0000 mon.a (mon.0) 293 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:23:18.571 INFO:teuthology.orchestra.run.smithi083.stderr:--> Zapping: /dev/vg_nvme/lv_1 2022-09-29T05:23:18.571 INFO:teuthology.orchestra.run.smithi083.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2022-09-29T05:23:18.572 INFO:teuthology.orchestra.run.smithi083.stderr: stderr: 10+0 records in 2022-09-29T05:23:18.572 INFO:teuthology.orchestra.run.smithi083.stderr:10+0 records out 2022-09-29T05:23:18.572 INFO:teuthology.orchestra.run.smithi083.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0147572 s, 711 MB/s 2022-09-29T05:23:18.572 INFO:teuthology.orchestra.run.smithi083.stderr:--> Zapping successful for: 2022-09-29T05:23:18.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:18 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:23:17.571395+0000 mgr.y (mgr.14146) 81 : cluster [DBG] pgmap v56: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2022-09-29T05:23:18.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:18 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:18.508665+0000 mon.a (mon.0) 294 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:23:18.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:18 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:23:17.571395+0000 mgr.y (mgr.14146) 81 : cluster [DBG] pgmap v56: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2022-09-29T05:23:18.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:18 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:18.508665+0000 mon.a (mon.0) 294 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:23:19.052 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/daemon-base:latest-pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 69111522-3fb6-11ed-8431-001a4aab830c -- ceph orch daemon add osd smithi083:vg_nvme/lv_1 2022-09-29T05:23:19.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:18 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:23:17.571395+0000 mgr.y (mgr.14146) 81 : cluster [DBG] pgmap v56: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2022-09-29T05:23:19.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:18 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:18.508665+0000 mon.a (mon.0) 294 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:23:19.713 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 29 05:23:19 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-osd-2[53187]: debug 2022-09-29T05:23:19.297+0000 7fe8a9be6200 -1 osd.2 0 log_to_monitors {default=true} 2022-09-29T05:23:20.077 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:19 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:19.301098+0000 mon.c (mon.1) 10 : audit [INF] from='osd.2 [v2:172.21.15.83:6818/1928326991,v1:172.21.15.83:6819/1928326991]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2022-09-29T05:23:20.078 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:19 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:19.301457+0000 mon.a (mon.0) 295 : audit [INF] from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2022-09-29T05:23:20.078 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:19 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:19.301098+0000 mon.c (mon.1) 10 : audit [INF] from='osd.2 [v2:172.21.15.83:6818/1928326991,v1:172.21.15.83:6819/1928326991]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2022-09-29T05:23:20.078 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:19 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:19.301457+0000 mon.a (mon.0) 295 : audit [INF] from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2022-09-29T05:23:20.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:19 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:19.301098+0000 mon.c (mon.1) 10 : audit [INF] from='osd.2 [v2:172.21.15.83:6818/1928326991,v1:172.21.15.83:6819/1928326991]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2022-09-29T05:23:20.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:19 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:19.301457+0000 mon.a (mon.0) 295 : audit [INF] from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2022-09-29T05:23:21.213 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 29 05:23:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-osd-2[53187]: debug 2022-09-29T05:23:20.823+0000 7fe8a016c700 -1 osd.2 0 waiting for initial osdmap 2022-09-29T05:23:21.213 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 29 05:23:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-osd-2[53187]: debug 2022-09-29T05:23:20.827+0000 7fe89a8e4700 -1 osd.2 17 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2022-09-29T05:23:21.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:23:19.571645+0000 mgr.y (mgr.14146) 82 : cluster [DBG] pgmap v57: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2022-09-29T05:23:21.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:19.818369+0000 mon.a (mon.0) 296 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2022-09-29T05:23:21.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:23:19.818436+0000 mon.a (mon.0) 297 : cluster [DBG] osdmap e16: 3 total, 2 up, 3 in 2022-09-29T05:23:21.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:19.818555+0000 mon.a (mon.0) 298 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2022-09-29T05:23:21.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:19.818943+0000 mon.c (mon.1) 11 : audit [INF] from='osd.2 [v2:172.21.15.83:6818/1928326991,v1:172.21.15.83:6819/1928326991]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]: dispatch 2022-09-29T05:23:21.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:19.819229+0000 mon.a (mon.0) 299 : audit [INF] from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]: dispatch 2022-09-29T05:23:21.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:23:19.571645+0000 mgr.y (mgr.14146) 82 : cluster [DBG] pgmap v57: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2022-09-29T05:23:21.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:19.818369+0000 mon.a (mon.0) 296 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2022-09-29T05:23:21.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:23:19.818436+0000 mon.a (mon.0) 297 : cluster [DBG] osdmap e16: 3 total, 2 up, 3 in 2022-09-29T05:23:21.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:19.818555+0000 mon.a (mon.0) 298 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2022-09-29T05:23:21.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:19.818943+0000 mon.c (mon.1) 11 : audit [INF] from='osd.2 [v2:172.21.15.83:6818/1928326991,v1:172.21.15.83:6819/1928326991]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]: dispatch 2022-09-29T05:23:21.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:19.819229+0000 mon.a (mon.0) 299 : audit [INF] from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]: dispatch 2022-09-29T05:23:21.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:23:19.571645+0000 mgr.y (mgr.14146) 82 : cluster [DBG] pgmap v57: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2022-09-29T05:23:21.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:19.818369+0000 mon.a (mon.0) 296 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2022-09-29T05:23:21.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:23:19.818436+0000 mon.a (mon.0) 297 : cluster [DBG] osdmap e16: 3 total, 2 up, 3 in 2022-09-29T05:23:21.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:19.818555+0000 mon.a (mon.0) 298 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2022-09-29T05:23:21.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:19.818943+0000 mon.c (mon.1) 11 : audit [INF] from='osd.2 [v2:172.21.15.83:6818/1928326991,v1:172.21.15.83:6819/1928326991]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]: dispatch 2022-09-29T05:23:21.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:19.819229+0000 mon.a (mon.0) 299 : audit [INF] from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]: dispatch 2022-09-29T05:23:22.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:20.820073+0000 mon.a (mon.0) 300 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]': finished 2022-09-29T05:23:22.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:23:20.820151+0000 mon.a (mon.0) 301 : cluster [DBG] osdmap e17: 3 total, 2 up, 3 in 2022-09-29T05:23:22.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:20.820522+0000 mon.a (mon.0) 302 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2022-09-29T05:23:22.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:20.824439+0000 mon.a (mon.0) 303 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2022-09-29T05:23:22.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:21.611569+0000 mon.a (mon.0) 304 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2022-09-29T05:23:22.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:21.613784+0000 mon.a (mon.0) 305 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2022-09-29T05:23:22.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:21.614324+0000 mon.a (mon.0) 306 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:23:22.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:20.820073+0000 mon.a (mon.0) 300 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]': finished 2022-09-29T05:23:22.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:23:20.820151+0000 mon.a (mon.0) 301 : cluster [DBG] osdmap e17: 3 total, 2 up, 3 in 2022-09-29T05:23:22.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:20.820522+0000 mon.a (mon.0) 302 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2022-09-29T05:23:22.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:20.824439+0000 mon.a (mon.0) 303 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2022-09-29T05:23:22.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:21.611569+0000 mon.a (mon.0) 304 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2022-09-29T05:23:22.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:21.613784+0000 mon.a (mon.0) 305 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2022-09-29T05:23:22.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:21.614324+0000 mon.a (mon.0) 306 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:23:22.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:20.820073+0000 mon.a (mon.0) 300 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]': finished 2022-09-29T05:23:22.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:23:20.820151+0000 mon.a (mon.0) 301 : cluster [DBG] osdmap e17: 3 total, 2 up, 3 in 2022-09-29T05:23:22.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:20.820522+0000 mon.a (mon.0) 302 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2022-09-29T05:23:22.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:20.824439+0000 mon.a (mon.0) 303 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2022-09-29T05:23:22.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:21.611569+0000 mon.a (mon.0) 304 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2022-09-29T05:23:22.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:21.613784+0000 mon.a (mon.0) 305 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2022-09-29T05:23:22.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:21.614324+0000 mon.a (mon.0) 306 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:23:22.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:23:20.325648+0000 osd.2 (osd.2) 1 : cluster [DBG] purged_snaps scrub starts 2022-09-29T05:23:22.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:23:20.325745+0000 osd.2 (osd.2) 2 : cluster [DBG] purged_snaps scrub ok 2022-09-29T05:23:22.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:23:21.571905+0000 mgr.y (mgr.14146) 83 : cluster [DBG] pgmap v60: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2022-09-29T05:23:22.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:21.609483+0000 mgr.y (mgr.14146) 84 : audit [DBG] from='client.14289 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi083:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:23:22.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:23:21.827308+0000 mon.a (mon.0) 307 : cluster [INF] osd.2 [v2:172.21.15.83:6818/1928326991,v1:172.21.15.83:6819/1928326991] boot 2022-09-29T05:23:22.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:23:21.827365+0000 mon.a (mon.0) 308 : cluster [DBG] osdmap e18: 3 total, 3 up, 3 in 2022-09-29T05:23:22.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:21.827731+0000 mon.a (mon.0) 309 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2022-09-29T05:23:22.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:22.676731+0000 mon.a (mon.0) 310 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:23:22.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:22.677529+0000 mon.a (mon.0) 311 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:23:22.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:22.678055+0000 mon.a (mon.0) 312 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:23:22.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:22.678572+0000 mon.a (mon.0) 313 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:23:22.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:22.682287+0000 mon.a (mon.0) 314 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:23:22.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:22.685432+0000 mon.a (mon.0) 315 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:23:22.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:23:20.325648+0000 osd.2 (osd.2) 1 : cluster [DBG] purged_snaps scrub starts 2022-09-29T05:23:22.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:23:20.325745+0000 osd.2 (osd.2) 2 : cluster [DBG] purged_snaps scrub ok 2022-09-29T05:23:22.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:23:21.571905+0000 mgr.y (mgr.14146) 83 : cluster [DBG] pgmap v60: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2022-09-29T05:23:22.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:21.609483+0000 mgr.y (mgr.14146) 84 : audit [DBG] from='client.14289 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi083:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:23:22.968 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:23:21.827308+0000 mon.a (mon.0) 307 : cluster [INF] osd.2 [v2:172.21.15.83:6818/1928326991,v1:172.21.15.83:6819/1928326991] boot 2022-09-29T05:23:22.968 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:23:21.827365+0000 mon.a (mon.0) 308 : cluster [DBG] osdmap e18: 3 total, 3 up, 3 in 2022-09-29T05:23:22.968 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:21.827731+0000 mon.a (mon.0) 309 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2022-09-29T05:23:22.968 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:22.676731+0000 mon.a (mon.0) 310 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:23:22.968 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:22.677529+0000 mon.a (mon.0) 311 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:23:22.969 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:22.678055+0000 mon.a (mon.0) 312 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:23:22.969 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:22.678572+0000 mon.a (mon.0) 313 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:23:22.969 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:22.682287+0000 mon.a (mon.0) 314 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:23:22.969 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:22.685432+0000 mon.a (mon.0) 315 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:23:23.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:23:20.325648+0000 osd.2 (osd.2) 1 : cluster [DBG] purged_snaps scrub starts 2022-09-29T05:23:23.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:23:20.325745+0000 osd.2 (osd.2) 2 : cluster [DBG] purged_snaps scrub ok 2022-09-29T05:23:23.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:23:21.571905+0000 mgr.y (mgr.14146) 83 : cluster [DBG] pgmap v60: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2022-09-29T05:23:23.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:21.609483+0000 mgr.y (mgr.14146) 84 : audit [DBG] from='client.14289 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi083:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:23:23.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:23:21.827308+0000 mon.a (mon.0) 307 : cluster [INF] osd.2 [v2:172.21.15.83:6818/1928326991,v1:172.21.15.83:6819/1928326991] boot 2022-09-29T05:23:23.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:23:21.827365+0000 mon.a (mon.0) 308 : cluster [DBG] osdmap e18: 3 total, 3 up, 3 in 2022-09-29T05:23:23.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:21.827731+0000 mon.a (mon.0) 309 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2022-09-29T05:23:23.312 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:22.676731+0000 mon.a (mon.0) 310 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:23:23.312 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:22.677529+0000 mon.a (mon.0) 311 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:23:23.312 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:22.678055+0000 mon.a (mon.0) 312 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:23:23.313 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:22.678572+0000 mon.a (mon.0) 313 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:23:23.313 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:22.682287+0000 mon.a (mon.0) 314 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:23:23.313 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:22.685432+0000 mon.a (mon.0) 315 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:23:24.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:23:22.672547+0000 mgr.y (mgr.14146) 85 : cephadm [INF] Detected new or changed devices on smithi083 2022-09-29T05:23:24.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:23:22.678951+0000 mgr.y (mgr.14146) 86 : cephadm [INF] Adjusting osd_memory_target on smithi083 to 5381M 2022-09-29T05:23:24.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:23:22.832493+0000 mon.a (mon.0) 316 : cluster [DBG] osdmap e19: 3 total, 3 up, 3 in 2022-09-29T05:23:24.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:23:22.672547+0000 mgr.y (mgr.14146) 85 : cephadm [INF] Detected new or changed devices on smithi083 2022-09-29T05:23:24.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:23:22.678951+0000 mgr.y (mgr.14146) 86 : cephadm [INF] Adjusting osd_memory_target on smithi083 to 5381M 2022-09-29T05:23:24.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:23:22.832493+0000 mon.a (mon.0) 316 : cluster [DBG] osdmap e19: 3 total, 3 up, 3 in 2022-09-29T05:23:24.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:23:22.672547+0000 mgr.y (mgr.14146) 85 : cephadm [INF] Detected new or changed devices on smithi083 2022-09-29T05:23:24.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:23:22.678951+0000 mgr.y (mgr.14146) 86 : cephadm [INF] Adjusting osd_memory_target on smithi083 to 5381M 2022-09-29T05:23:24.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:23:22.832493+0000 mon.a (mon.0) 316 : cluster [DBG] osdmap e19: 3 total, 3 up, 3 in 2022-09-29T05:23:25.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:23:23.572219+0000 mgr.y (mgr.14146) 87 : cluster [DBG] pgmap v63: 0 pgs: ; 0 B data, 14 MiB used, 268 GiB / 268 GiB avail 2022-09-29T05:23:25.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:23.862937+0000 mon.a (mon.0) 317 : audit [INF] from='client.? 172.21.15.83:0/1108968896' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7336f148-261e-468e-8d3c-28f840150f36"}]: dispatch 2022-09-29T05:23:25.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:23.869080+0000 mon.a (mon.0) 318 : audit [INF] from='client.? 172.21.15.83:0/1108968896' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7336f148-261e-468e-8d3c-28f840150f36"}]': finished 2022-09-29T05:23:25.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:23:23.869120+0000 mon.a (mon.0) 319 : cluster [DBG] osdmap e20: 4 total, 3 up, 4 in 2022-09-29T05:23:25.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:23.869204+0000 mon.a (mon.0) 320 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2022-09-29T05:23:25.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:24.432069+0000 mon.a (mon.0) 321 : audit [DBG] from='client.? 172.21.15.83:0/3171352231' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2022-09-29T05:23:25.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:23:23.572219+0000 mgr.y (mgr.14146) 87 : cluster [DBG] pgmap v63: 0 pgs: ; 0 B data, 14 MiB used, 268 GiB / 268 GiB avail 2022-09-29T05:23:25.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:23.862937+0000 mon.a (mon.0) 317 : audit [INF] from='client.? 172.21.15.83:0/1108968896' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7336f148-261e-468e-8d3c-28f840150f36"}]: dispatch 2022-09-29T05:23:25.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:23.869080+0000 mon.a (mon.0) 318 : audit [INF] from='client.? 172.21.15.83:0/1108968896' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7336f148-261e-468e-8d3c-28f840150f36"}]': finished 2022-09-29T05:23:25.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:23:23.869120+0000 mon.a (mon.0) 319 : cluster [DBG] osdmap e20: 4 total, 3 up, 4 in 2022-09-29T05:23:25.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:23.869204+0000 mon.a (mon.0) 320 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2022-09-29T05:23:25.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:24.432069+0000 mon.a (mon.0) 321 : audit [DBG] from='client.? 172.21.15.83:0/3171352231' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2022-09-29T05:23:25.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:24 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:23:23.572219+0000 mgr.y (mgr.14146) 87 : cluster [DBG] pgmap v63: 0 pgs: ; 0 B data, 14 MiB used, 268 GiB / 268 GiB avail 2022-09-29T05:23:25.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:24 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:23.862937+0000 mon.a (mon.0) 317 : audit [INF] from='client.? 172.21.15.83:0/1108968896' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7336f148-261e-468e-8d3c-28f840150f36"}]: dispatch 2022-09-29T05:23:25.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:24 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:23.869080+0000 mon.a (mon.0) 318 : audit [INF] from='client.? 172.21.15.83:0/1108968896' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7336f148-261e-468e-8d3c-28f840150f36"}]': finished 2022-09-29T05:23:25.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:24 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:23:23.869120+0000 mon.a (mon.0) 319 : cluster [DBG] osdmap e20: 4 total, 3 up, 4 in 2022-09-29T05:23:25.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:24 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:23.869204+0000 mon.a (mon.0) 320 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2022-09-29T05:23:25.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:24 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:24.432069+0000 mon.a (mon.0) 321 : audit [DBG] from='client.? 172.21.15.83:0/3171352231' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2022-09-29T05:23:26.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:23:25.572598+0000 mgr.y (mgr.14146) 88 : cluster [DBG] pgmap v65: 0 pgs: ; 0 B data, 14 MiB used, 268 GiB / 268 GiB avail 2022-09-29T05:23:26.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:23:25.572598+0000 mgr.y (mgr.14146) 88 : cluster [DBG] pgmap v65: 0 pgs: ; 0 B data, 14 MiB used, 268 GiB / 268 GiB avail 2022-09-29T05:23:27.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:26 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:23:25.572598+0000 mgr.y (mgr.14146) 88 : cluster [DBG] pgmap v65: 0 pgs: ; 0 B data, 14 MiB used, 268 GiB / 268 GiB avail 2022-09-29T05:23:29.107 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:28 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:23:27.572933+0000 mgr.y (mgr.14146) 89 : cluster [DBG] pgmap v66: 0 pgs: ; 0 B data, 14 MiB used, 268 GiB / 268 GiB avail 2022-09-29T05:23:29.108 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:28 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:23:27.572933+0000 mgr.y (mgr.14146) 89 : cluster [DBG] pgmap v66: 0 pgs: ; 0 B data, 14 MiB used, 268 GiB / 268 GiB avail 2022-09-29T05:23:29.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:28 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:23:27.572933+0000 mgr.y (mgr.14146) 89 : cluster [DBG] pgmap v66: 0 pgs: ; 0 B data, 14 MiB used, 268 GiB / 268 GiB avail 2022-09-29T05:23:30.148 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:29 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:29.526205+0000 mon.a (mon.0) 322 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2022-09-29T05:23:30.149 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:29 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:29.526782+0000 mon.a (mon.0) 323 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:23:30.149 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:29 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:29.526205+0000 mon.a (mon.0) 322 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2022-09-29T05:23:30.149 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:29 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:29.526782+0000 mon.a (mon.0) 323 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:23:30.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:29 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:29.526205+0000 mon.a (mon.0) 322 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2022-09-29T05:23:30.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:29 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:29.526782+0000 mon.a (mon.0) 323 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:23:31.009 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:23:29.527237+0000 mgr.y (mgr.14146) 90 : cephadm [INF] Deploying daemon osd.3 on smithi083 2022-09-29T05:23:31.009 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:23:29.573177+0000 mgr.y (mgr.14146) 91 : cluster [DBG] pgmap v67: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2022-09-29T05:23:31.010 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:23:29.527237+0000 mgr.y (mgr.14146) 90 : cephadm [INF] Deploying daemon osd.3 on smithi083 2022-09-29T05:23:31.010 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:23:29.573177+0000 mgr.y (mgr.14146) 91 : cluster [DBG] pgmap v67: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2022-09-29T05:23:31.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:30 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:23:29.527237+0000 mgr.y (mgr.14146) 90 : cephadm [INF] Deploying daemon osd.3 on smithi083 2022-09-29T05:23:31.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:30 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:23:29.573177+0000 mgr.y (mgr.14146) 91 : cluster [DBG] pgmap v67: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2022-09-29T05:23:33.165 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:23:31.573591+0000 mgr.y (mgr.14146) 92 : cluster [DBG] pgmap v68: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2022-09-29T05:23:33.166 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:23:31.573591+0000 mgr.y (mgr.14146) 92 : cluster [DBG] pgmap v68: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2022-09-29T05:23:33.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:23:31.573591+0000 mgr.y (mgr.14146) 92 : cluster [DBG] pgmap v68: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2022-09-29T05:23:34.550 INFO:teuthology.orchestra.run.smithi083.stdout:Created osd(s) 3 on host 'smithi083' 2022-09-29T05:23:35.018 DEBUG:teuthology.orchestra.run.smithi083:osd.3> sudo journalctl -f -n 0 -u ceph-69111522-3fb6-11ed-8431-001a4aab830c@osd.3.service 2022-09-29T05:23:35.021 INFO:tasks.cephadm:Deploying osd.4 on smithi153 with /dev/vg_nvme/lv_4... 2022-09-29T05:23:35.021 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/daemon-base:latest-pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 69111522-3fb6-11ed-8431-001a4aab830c -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2022-09-29T05:23:35.051 INFO:journalctl@ceph.osd.3.smithi083.stdout:-- Logs begin at Thu 2022-09-29 05:12:45 UTC. -- 2022-09-29T05:23:35.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:23:33.573964+0000 mgr.y (mgr.14146) 93 : cluster [DBG] pgmap v69: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2022-09-29T05:23:35.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:34.541505+0000 mon.a (mon.0) 324 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:23:35.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:34.545757+0000 mon.a (mon.0) 325 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:23:35.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:34.546968+0000 mon.a (mon.0) 326 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2022-09-29T05:23:35.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:34.797444+0000 mon.a (mon.0) 327 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:23:35.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:34.800524+0000 mon.a (mon.0) 328 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:23:35.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:34.801094+0000 mon.a (mon.0) 329 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:23:35.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:23:33.573964+0000 mgr.y (mgr.14146) 93 : cluster [DBG] pgmap v69: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2022-09-29T05:23:35.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:34.541505+0000 mon.a (mon.0) 324 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:23:35.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:34.545757+0000 mon.a (mon.0) 325 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:23:35.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:34.546968+0000 mon.a (mon.0) 326 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2022-09-29T05:23:35.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:34.797444+0000 mon.a (mon.0) 327 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:23:35.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:34.800524+0000 mon.a (mon.0) 328 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:23:35.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:34.801094+0000 mon.a (mon.0) 329 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:23:35.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:34 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:23:33.573964+0000 mgr.y (mgr.14146) 93 : cluster [DBG] pgmap v69: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2022-09-29T05:23:35.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:34 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:34.541505+0000 mon.a (mon.0) 324 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:23:35.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:34 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:34.545757+0000 mon.a (mon.0) 325 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:23:35.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:34 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:34.546968+0000 mon.a (mon.0) 326 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2022-09-29T05:23:35.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:34 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:34.797444+0000 mon.a (mon.0) 327 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:23:35.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:34 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:34.800524+0000 mon.a (mon.0) 328 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:23:35.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:34 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:34.801094+0000 mon.a (mon.0) 329 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:23:36.733 INFO:teuthology.orchestra.run.smithi153.stderr:--> Zapping: /dev/vg_nvme/lv_4 2022-09-29T05:23:36.734 INFO:teuthology.orchestra.run.smithi153.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2022-09-29T05:23:36.734 INFO:teuthology.orchestra.run.smithi153.stderr: stderr: 10+0 records in 2022-09-29T05:23:36.734 INFO:teuthology.orchestra.run.smithi153.stderr:10+0 records out 2022-09-29T05:23:36.735 INFO:teuthology.orchestra.run.smithi153.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0154228 s, 680 MB/s 2022-09-29T05:23:36.735 INFO:teuthology.orchestra.run.smithi153.stderr:--> Zapping successful for: 2022-09-29T05:23:36.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:36 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:35.547597+0000 mon.a (mon.0) 330 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2022-09-29T05:23:36.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:36 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:23:35.547651+0000 mon.a (mon.0) 331 : cluster [DBG] osdmap e21: 4 total, 3 up, 4 in 2022-09-29T05:23:36.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:36 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:35.547828+0000 mon.a (mon.0) 332 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2022-09-29T05:23:36.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:36 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:35.549090+0000 mon.a (mon.0) 333 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2022-09-29T05:23:36.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:36 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:23:35.574182+0000 mgr.y (mgr.14146) 94 : cluster [DBG] pgmap v71: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2022-09-29T05:23:36.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:36 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:35.690601+0000 mon.a (mon.0) 334 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:23:36.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:36 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:35.691433+0000 mon.a (mon.0) 335 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:23:36.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:35.547597+0000 mon.a (mon.0) 330 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2022-09-29T05:23:36.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:23:35.547651+0000 mon.a (mon.0) 331 : cluster [DBG] osdmap e21: 4 total, 3 up, 4 in 2022-09-29T05:23:36.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:35.547828+0000 mon.a (mon.0) 332 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2022-09-29T05:23:36.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:35.549090+0000 mon.a (mon.0) 333 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2022-09-29T05:23:36.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:23:35.574182+0000 mgr.y (mgr.14146) 94 : cluster [DBG] pgmap v71: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2022-09-29T05:23:36.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:35.690601+0000 mon.a (mon.0) 334 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:23:36.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:35.691433+0000 mon.a (mon.0) 335 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:23:36.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:35.547597+0000 mon.a (mon.0) 330 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2022-09-29T05:23:36.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:23:35.547651+0000 mon.a (mon.0) 331 : cluster [DBG] osdmap e21: 4 total, 3 up, 4 in 2022-09-29T05:23:36.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:35.547828+0000 mon.a (mon.0) 332 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2022-09-29T05:23:36.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:35.549090+0000 mon.a (mon.0) 333 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2022-09-29T05:23:36.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:23:35.574182+0000 mgr.y (mgr.14146) 94 : cluster [DBG] pgmap v71: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2022-09-29T05:23:36.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:35.690601+0000 mon.a (mon.0) 334 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:23:36.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:35.691433+0000 mon.a (mon.0) 335 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:23:37.146 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/daemon-base:latest-pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 69111522-3fb6-11ed-8431-001a4aab830c -- ceph orch daemon add osd smithi153:vg_nvme/lv_4 2022-09-29T05:23:37.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:37 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:36.552215+0000 mon.a (mon.0) 336 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2022-09-29T05:23:37.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:37 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:23:36.552289+0000 mon.a (mon.0) 337 : cluster [DBG] osdmap e22: 4 total, 3 up, 4 in 2022-09-29T05:23:37.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:37 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:36.552537+0000 mon.a (mon.0) 338 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2022-09-29T05:23:37.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:37 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:36.731815+0000 mon.a (mon.0) 339 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:23:37.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:37 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:36.552215+0000 mon.a (mon.0) 336 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2022-09-29T05:23:37.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:37 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:23:36.552289+0000 mon.a (mon.0) 337 : cluster [DBG] osdmap e22: 4 total, 3 up, 4 in 2022-09-29T05:23:37.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:37 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:36.552537+0000 mon.a (mon.0) 338 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2022-09-29T05:23:37.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:37 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:36.731815+0000 mon.a (mon.0) 339 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:23:37.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:37 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:36.552215+0000 mon.a (mon.0) 336 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2022-09-29T05:23:37.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:37 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:23:36.552289+0000 mon.a (mon.0) 337 : cluster [DBG] osdmap e22: 4 total, 3 up, 4 in 2022-09-29T05:23:37.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:37 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:36.552537+0000 mon.a (mon.0) 338 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2022-09-29T05:23:37.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:37 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:36.731815+0000 mon.a (mon.0) 339 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:23:38.029 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 29 05:23:37 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-osd-3[56220]: debug 2022-09-29T05:23:37.746+0000 7f0415d12200 -1 osd.3 0 log_to_monitors {default=true} 2022-09-29T05:23:39.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:38 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:23:37.574569+0000 mgr.y (mgr.14146) 95 : cluster [DBG] pgmap v73: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2022-09-29T05:23:39.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:38 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:23:37.733935+0000 mon.a (mon.0) 340 : cluster [DBG] osdmap e23: 4 total, 3 up, 4 in 2022-09-29T05:23:39.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:38 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:37.734103+0000 mon.a (mon.0) 341 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2022-09-29T05:23:39.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:38 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:37.750498+0000 mon.a (mon.0) 342 : audit [INF] from='osd.3 [v2:172.21.15.83:6826/54976712,v1:172.21.15.83:6827/54976712]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2022-09-29T05:23:39.152 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:38 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:23:37.574569+0000 mgr.y (mgr.14146) 95 : cluster [DBG] pgmap v73: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2022-09-29T05:23:39.153 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:38 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:23:37.733935+0000 mon.a (mon.0) 340 : cluster [DBG] osdmap e23: 4 total, 3 up, 4 in 2022-09-29T05:23:39.153 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:38 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:37.734103+0000 mon.a (mon.0) 341 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2022-09-29T05:23:39.153 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:38 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:37.750498+0000 mon.a (mon.0) 342 : audit [INF] from='osd.3 [v2:172.21.15.83:6826/54976712,v1:172.21.15.83:6827/54976712]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2022-09-29T05:23:39.154 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:38 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:23:37.574569+0000 mgr.y (mgr.14146) 95 : cluster [DBG] pgmap v73: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2022-09-29T05:23:39.154 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:38 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:23:37.733935+0000 mon.a (mon.0) 340 : cluster [DBG] osdmap e23: 4 total, 3 up, 4 in 2022-09-29T05:23:39.154 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:38 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:37.734103+0000 mon.a (mon.0) 341 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2022-09-29T05:23:39.154 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:38 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:37.750498+0000 mon.a (mon.0) 342 : audit [INF] from='osd.3 [v2:172.21.15.83:6826/54976712,v1:172.21.15.83:6827/54976712]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2022-09-29T05:23:39.963 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 29 05:23:39 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-osd-3[56220]: debug 2022-09-29T05:23:39.739+0000 7f040c298700 -1 osd.3 0 waiting for initial osdmap 2022-09-29T05:23:39.963 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 29 05:23:39 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-osd-3[56220]: debug 2022-09-29T05:23:39.745+0000 7f0407a12700 -1 osd.3 25 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2022-09-29T05:23:39.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:39 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:38.735350+0000 mon.a (mon.0) 343 : audit [INF] from='osd.3 [v2:172.21.15.83:6826/54976712,v1:172.21.15.83:6827/54976712]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2022-09-29T05:23:39.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:39 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:23:38.735406+0000 mon.a (mon.0) 344 : cluster [DBG] osdmap e24: 4 total, 3 up, 4 in 2022-09-29T05:23:39.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:39 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:38.735752+0000 mon.a (mon.0) 345 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2022-09-29T05:23:39.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:39 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:38.736017+0000 mon.a (mon.0) 346 : audit [INF] from='osd.3 [v2:172.21.15.83:6826/54976712,v1:172.21.15.83:6827/54976712]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]: dispatch 2022-09-29T05:23:39.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:39 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:38.788280+0000 mgr.y (mgr.14146) 96 : audit [DBG] from='client.24212 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi153:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:23:39.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:39 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:38.790413+0000 mon.a (mon.0) 347 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2022-09-29T05:23:39.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:39 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:38.792237+0000 mon.a (mon.0) 348 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2022-09-29T05:23:39.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:39 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:38.792796+0000 mon.a (mon.0) 349 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:23:39.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:39 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:38.735350+0000 mon.a (mon.0) 343 : audit [INF] from='osd.3 [v2:172.21.15.83:6826/54976712,v1:172.21.15.83:6827/54976712]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2022-09-29T05:23:39.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:39 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:23:38.735406+0000 mon.a (mon.0) 344 : cluster [DBG] osdmap e24: 4 total, 3 up, 4 in 2022-09-29T05:23:39.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:39 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:38.735752+0000 mon.a (mon.0) 345 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2022-09-29T05:23:39.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:39 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:38.736017+0000 mon.a (mon.0) 346 : audit [INF] from='osd.3 [v2:172.21.15.83:6826/54976712,v1:172.21.15.83:6827/54976712]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]: dispatch 2022-09-29T05:23:39.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:39 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:38.788280+0000 mgr.y (mgr.14146) 96 : audit [DBG] from='client.24212 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi153:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:23:39.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:39 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:38.790413+0000 mon.a (mon.0) 347 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2022-09-29T05:23:39.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:39 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:38.792237+0000 mon.a (mon.0) 348 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2022-09-29T05:23:39.968 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:39 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:38.792796+0000 mon.a (mon.0) 349 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:23:40.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:39 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:38.735350+0000 mon.a (mon.0) 343 : audit [INF] from='osd.3 [v2:172.21.15.83:6826/54976712,v1:172.21.15.83:6827/54976712]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2022-09-29T05:23:40.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:39 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:23:38.735406+0000 mon.a (mon.0) 344 : cluster [DBG] osdmap e24: 4 total, 3 up, 4 in 2022-09-29T05:23:40.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:39 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:38.735752+0000 mon.a (mon.0) 345 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2022-09-29T05:23:40.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:39 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:38.736017+0000 mon.a (mon.0) 346 : audit [INF] from='osd.3 [v2:172.21.15.83:6826/54976712,v1:172.21.15.83:6827/54976712]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]: dispatch 2022-09-29T05:23:40.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:39 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:38.788280+0000 mgr.y (mgr.14146) 96 : audit [DBG] from='client.24212 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi153:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:23:40.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:39 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:38.790413+0000 mon.a (mon.0) 347 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2022-09-29T05:23:40.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:39 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:38.792237+0000 mon.a (mon.0) 348 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2022-09-29T05:23:40.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:39 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:38.792796+0000 mon.a (mon.0) 349 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:23:41.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:40 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:23:39.574868+0000 mgr.y (mgr.14146) 97 : cluster [DBG] pgmap v76: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2022-09-29T05:23:41.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:40 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:39.737013+0000 mon.a (mon.0) 350 : audit [INF] from='osd.3 [v2:172.21.15.83:6826/54976712,v1:172.21.15.83:6827/54976712]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]': finished 2022-09-29T05:23:41.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:40 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:23:39.737093+0000 mon.a (mon.0) 351 : cluster [DBG] osdmap e25: 4 total, 3 up, 4 in 2022-09-29T05:23:41.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:40 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:39.737217+0000 mon.a (mon.0) 352 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2022-09-29T05:23:41.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:40 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:39.741569+0000 mon.a (mon.0) 353 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2022-09-29T05:23:41.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:40 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:23:39.885837+0000 mgr.y (mgr.14146) 98 : cephadm [INF] Detected new or changed devices on smithi083 2022-09-29T05:23:41.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:40 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:39.890522+0000 mon.a (mon.0) 354 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:23:41.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:40 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:39.891324+0000 mon.a (mon.0) 355 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:23:41.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:40 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:39.891857+0000 mon.a (mon.0) 356 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:23:41.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:40 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:39.892358+0000 mon.a (mon.0) 357 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:23:41.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:40 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:39.892837+0000 mon.a (mon.0) 358 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:23:41.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:40 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:23:39.893223+0000 mgr.y (mgr.14146) 99 : cephadm [INF] Adjusting osd_memory_target on smithi083 to 4036M 2022-09-29T05:23:41.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:40 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:39.896331+0000 mon.a (mon.0) 359 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:23:41.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:40 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:39.899586+0000 mon.a (mon.0) 360 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:23:41.063 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:40 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:40.740463+0000 mon.a (mon.0) 361 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2022-09-29T05:23:41.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:23:39.574868+0000 mgr.y (mgr.14146) 97 : cluster [DBG] pgmap v76: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2022-09-29T05:23:41.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:39.737013+0000 mon.a (mon.0) 350 : audit [INF] from='osd.3 [v2:172.21.15.83:6826/54976712,v1:172.21.15.83:6827/54976712]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]': finished 2022-09-29T05:23:41.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:23:39.737093+0000 mon.a (mon.0) 351 : cluster [DBG] osdmap e25: 4 total, 3 up, 4 in 2022-09-29T05:23:41.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:39.737217+0000 mon.a (mon.0) 352 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2022-09-29T05:23:41.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:39.741569+0000 mon.a (mon.0) 353 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2022-09-29T05:23:41.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:23:39.885837+0000 mgr.y (mgr.14146) 98 : cephadm [INF] Detected new or changed devices on smithi083 2022-09-29T05:23:41.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:39.890522+0000 mon.a (mon.0) 354 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:23:41.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:39.891324+0000 mon.a (mon.0) 355 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:23:41.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:39.891857+0000 mon.a (mon.0) 356 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:23:41.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:39.892358+0000 mon.a (mon.0) 357 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:23:41.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:39.892837+0000 mon.a (mon.0) 358 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:23:41.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:23:39.893223+0000 mgr.y (mgr.14146) 99 : cephadm [INF] Adjusting osd_memory_target on smithi083 to 4036M 2022-09-29T05:23:41.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:39.896331+0000 mon.a (mon.0) 359 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:23:41.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:39.899586+0000 mon.a (mon.0) 360 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:23:41.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:40.740463+0000 mon.a (mon.0) 361 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2022-09-29T05:23:41.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:23:39.574868+0000 mgr.y (mgr.14146) 97 : cluster [DBG] pgmap v76: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2022-09-29T05:23:41.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:39.737013+0000 mon.a (mon.0) 350 : audit [INF] from='osd.3 [v2:172.21.15.83:6826/54976712,v1:172.21.15.83:6827/54976712]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]': finished 2022-09-29T05:23:41.218 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:23:39.737093+0000 mon.a (mon.0) 351 : cluster [DBG] osdmap e25: 4 total, 3 up, 4 in 2022-09-29T05:23:41.218 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:39.737217+0000 mon.a (mon.0) 352 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2022-09-29T05:23:41.218 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:39.741569+0000 mon.a (mon.0) 353 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2022-09-29T05:23:41.218 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:23:39.885837+0000 mgr.y (mgr.14146) 98 : cephadm [INF] Detected new or changed devices on smithi083 2022-09-29T05:23:41.219 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:39.890522+0000 mon.a (mon.0) 354 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:23:41.219 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:39.891324+0000 mon.a (mon.0) 355 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:23:41.219 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:39.891857+0000 mon.a (mon.0) 356 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:23:41.219 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:39.892358+0000 mon.a (mon.0) 357 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:23:41.220 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:39.892837+0000 mon.a (mon.0) 358 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:23:41.220 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:23:39.893223+0000 mgr.y (mgr.14146) 99 : cephadm [INF] Adjusting osd_memory_target on smithi083 to 4036M 2022-09-29T05:23:41.220 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:39.896331+0000 mon.a (mon.0) 359 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:23:41.220 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:39.899586+0000 mon.a (mon.0) 360 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:23:41.220 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:40.740463+0000 mon.a (mon.0) 361 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2022-09-29T05:23:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:41 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:23:38.716932+0000 osd.3 (osd.3) 1 : cluster [DBG] purged_snaps scrub starts 2022-09-29T05:23:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:41 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:23:38.717064+0000 osd.3 (osd.3) 2 : cluster [DBG] purged_snaps scrub ok 2022-09-29T05:23:42.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:41 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:23:40.752194+0000 mon.a (mon.0) 362 : cluster [INF] osd.3 [v2:172.21.15.83:6826/54976712,v1:172.21.15.83:6827/54976712] boot 2022-09-29T05:23:42.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:41 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:23:40.752234+0000 mon.a (mon.0) 363 : cluster [DBG] osdmap e26: 4 total, 4 up, 4 in 2022-09-29T05:23:42.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:41 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:40.752550+0000 mon.a (mon.0) 364 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2022-09-29T05:23:42.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:41 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:40.822046+0000 mon.b (mon.2) 6 : audit [INF] from='client.? 172.21.15.153:0/823517087' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "247d5c19-e8af-4620-a378-699dc2000db1"}]: dispatch 2022-09-29T05:23:42.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:41 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:40.822785+0000 mon.a (mon.0) 365 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "247d5c19-e8af-4620-a378-699dc2000db1"}]: dispatch 2022-09-29T05:23:42.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:41 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:40.827632+0000 mon.a (mon.0) 366 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "247d5c19-e8af-4620-a378-699dc2000db1"}]': finished 2022-09-29T05:23:42.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:41 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:23:40.827695+0000 mon.a (mon.0) 367 : cluster [DBG] osdmap e27: 5 total, 4 up, 5 in 2022-09-29T05:23:42.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:41 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:40.827843+0000 mon.a (mon.0) 368 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2022-09-29T05:23:42.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:41 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:41.378232+0000 mon.b (mon.2) 7 : audit [DBG] from='client.? 172.21.15.153:0/3524550779' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2022-09-29T05:23:42.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:23:38.716932+0000 osd.3 (osd.3) 1 : cluster [DBG] purged_snaps scrub starts 2022-09-29T05:23:42.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:23:38.717064+0000 osd.3 (osd.3) 2 : cluster [DBG] purged_snaps scrub ok 2022-09-29T05:23:42.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:23:40.752194+0000 mon.a (mon.0) 362 : cluster [INF] osd.3 [v2:172.21.15.83:6826/54976712,v1:172.21.15.83:6827/54976712] boot 2022-09-29T05:23:42.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:23:40.752234+0000 mon.a (mon.0) 363 : cluster [DBG] osdmap e26: 4 total, 4 up, 4 in 2022-09-29T05:23:42.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:40.752550+0000 mon.a (mon.0) 364 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2022-09-29T05:23:42.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:40.822046+0000 mon.b (mon.2) 6 : audit [INF] from='client.? 172.21.15.153:0/823517087' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "247d5c19-e8af-4620-a378-699dc2000db1"}]: dispatch 2022-09-29T05:23:42.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:40.822785+0000 mon.a (mon.0) 365 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "247d5c19-e8af-4620-a378-699dc2000db1"}]: dispatch 2022-09-29T05:23:42.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:40.827632+0000 mon.a (mon.0) 366 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "247d5c19-e8af-4620-a378-699dc2000db1"}]': finished 2022-09-29T05:23:42.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:23:40.827695+0000 mon.a (mon.0) 367 : cluster [DBG] osdmap e27: 5 total, 4 up, 5 in 2022-09-29T05:23:42.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:40.827843+0000 mon.a (mon.0) 368 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2022-09-29T05:23:42.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:41.378232+0000 mon.b (mon.2) 7 : audit [DBG] from='client.? 172.21.15.153:0/3524550779' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2022-09-29T05:23:42.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:23:38.716932+0000 osd.3 (osd.3) 1 : cluster [DBG] purged_snaps scrub starts 2022-09-29T05:23:42.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:23:38.717064+0000 osd.3 (osd.3) 2 : cluster [DBG] purged_snaps scrub ok 2022-09-29T05:23:42.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:23:40.752194+0000 mon.a (mon.0) 362 : cluster [INF] osd.3 [v2:172.21.15.83:6826/54976712,v1:172.21.15.83:6827/54976712] boot 2022-09-29T05:23:42.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:23:40.752234+0000 mon.a (mon.0) 363 : cluster [DBG] osdmap e26: 4 total, 4 up, 4 in 2022-09-29T05:23:42.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:40.752550+0000 mon.a (mon.0) 364 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2022-09-29T05:23:42.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:40.822046+0000 mon.b (mon.2) 6 : audit [INF] from='client.? 172.21.15.153:0/823517087' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "247d5c19-e8af-4620-a378-699dc2000db1"}]: dispatch 2022-09-29T05:23:42.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:40.822785+0000 mon.a (mon.0) 365 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "247d5c19-e8af-4620-a378-699dc2000db1"}]: dispatch 2022-09-29T05:23:42.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:40.827632+0000 mon.a (mon.0) 366 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "247d5c19-e8af-4620-a378-699dc2000db1"}]': finished 2022-09-29T05:23:42.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:23:40.827695+0000 mon.a (mon.0) 367 : cluster [DBG] osdmap e27: 5 total, 4 up, 5 in 2022-09-29T05:23:42.218 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:40.827843+0000 mon.a (mon.0) 368 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2022-09-29T05:23:42.218 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:41.378232+0000 mon.b (mon.2) 7 : audit [DBG] from='client.? 172.21.15.153:0/3524550779' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2022-09-29T05:23:43.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:23:41.575331+0000 mgr.y (mgr.14146) 100 : cluster [DBG] pgmap v80: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2022-09-29T05:23:43.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:23:41.575331+0000 mgr.y (mgr.14146) 100 : cluster [DBG] pgmap v80: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2022-09-29T05:23:43.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:23:41.575331+0000 mgr.y (mgr.14146) 100 : cluster [DBG] pgmap v80: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2022-09-29T05:23:44.914 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:44 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:23:43.575695+0000 mgr.y (mgr.14146) 101 : cluster [DBG] pgmap v81: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2022-09-29T05:23:45.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:23:43.575695+0000 mgr.y (mgr.14146) 101 : cluster [DBG] pgmap v81: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2022-09-29T05:23:45.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:23:43.575695+0000 mgr.y (mgr.14146) 101 : cluster [DBG] pgmap v81: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2022-09-29T05:23:46.955 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:46 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:23:45.576039+0000 mgr.y (mgr.14146) 102 : cluster [DBG] pgmap v82: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2022-09-29T05:23:46.956 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:46 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:46.065056+0000 mon.a (mon.0) 369 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2022-09-29T05:23:46.956 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:46 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:46.065883+0000 mon.a (mon.0) 370 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:23:47.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:23:45.576039+0000 mgr.y (mgr.14146) 102 : cluster [DBG] pgmap v82: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2022-09-29T05:23:47.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:46.065056+0000 mon.a (mon.0) 369 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2022-09-29T05:23:47.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:46.065883+0000 mon.a (mon.0) 370 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:23:47.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:23:45.576039+0000 mgr.y (mgr.14146) 102 : cluster [DBG] pgmap v82: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2022-09-29T05:23:47.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:46.065056+0000 mon.a (mon.0) 369 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2022-09-29T05:23:47.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:46.065883+0000 mon.a (mon.0) 370 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:23:48.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:47 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:23:46.066550+0000 mgr.y (mgr.14146) 103 : cephadm [INF] Deploying daemon osd.4 on smithi153 2022-09-29T05:23:48.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:47 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:23:46.066550+0000 mgr.y (mgr.14146) 103 : cephadm [INF] Deploying daemon osd.4 on smithi153 2022-09-29T05:23:48.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:47 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:23:46.066550+0000 mgr.y (mgr.14146) 103 : cephadm [INF] Deploying daemon osd.4 on smithi153 2022-09-29T05:23:49.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:48 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:23:47.576416+0000 mgr.y (mgr.14146) 104 : cluster [DBG] pgmap v83: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2022-09-29T05:23:49.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:48 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:23:47.576416+0000 mgr.y (mgr.14146) 104 : cluster [DBG] pgmap v83: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2022-09-29T05:23:49.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:48 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:23:47.576416+0000 mgr.y (mgr.14146) 104 : cluster [DBG] pgmap v83: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2022-09-29T05:23:50.653 INFO:teuthology.orchestra.run.smithi153.stdout:Created osd(s) 4 on host 'smithi153' 2022-09-29T05:23:51.054 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:50 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:23:49.576966+0000 mgr.y (mgr.14146) 105 : cluster [DBG] pgmap v84: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2022-09-29T05:23:51.055 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:50 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:50.641223+0000 mon.a (mon.0) 371 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:23:51.055 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:50 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:50.641454+0000 mon.a (mon.0) 372 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:23:51.055 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:50 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:50.642774+0000 mon.a (mon.0) 373 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:23:51.055 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:50 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:50.643632+0000 mon.a (mon.0) 374 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:23:51.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:50 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:50.647927+0000 mon.a (mon.0) 375 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:23:51.104 DEBUG:teuthology.orchestra.run.smithi153:osd.4> sudo journalctl -f -n 0 -u ceph-69111522-3fb6-11ed-8431-001a4aab830c@osd.4.service 2022-09-29T05:23:51.108 INFO:tasks.cephadm:Deploying osd.5 on smithi153 with /dev/vg_nvme/lv_3... 2022-09-29T05:23:51.108 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/daemon-base:latest-pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 69111522-3fb6-11ed-8431-001a4aab830c -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2022-09-29T05:23:51.137 INFO:journalctl@ceph.osd.4.smithi153.stdout:-- Logs begin at Thu 2022-09-29 05:12:53 UTC. -- 2022-09-29T05:23:51.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:23:49.576966+0000 mgr.y (mgr.14146) 105 : cluster [DBG] pgmap v84: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2022-09-29T05:23:51.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:50.641223+0000 mon.a (mon.0) 371 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:23:51.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:50.641454+0000 mon.a (mon.0) 372 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:23:51.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:50.642774+0000 mon.a (mon.0) 373 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:23:51.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:50.643632+0000 mon.a (mon.0) 374 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:23:51.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:50.647927+0000 mon.a (mon.0) 375 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:23:51.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:23:49.576966+0000 mgr.y (mgr.14146) 105 : cluster [DBG] pgmap v84: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2022-09-29T05:23:51.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:50.641223+0000 mon.a (mon.0) 371 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:23:51.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:50.641454+0000 mon.a (mon.0) 372 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:23:51.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:50.642774+0000 mon.a (mon.0) 373 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:23:51.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:50.643632+0000 mon.a (mon.0) 374 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:23:51.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:50.647927+0000 mon.a (mon.0) 375 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:23:53.004 INFO:teuthology.orchestra.run.smithi153.stderr:--> Zapping: /dev/vg_nvme/lv_3 2022-09-29T05:23:53.005 INFO:teuthology.orchestra.run.smithi153.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2022-09-29T05:23:53.005 INFO:teuthology.orchestra.run.smithi153.stderr: stderr: 10+0 records in 2022-09-29T05:23:53.005 INFO:teuthology.orchestra.run.smithi153.stderr:10+0 records out 2022-09-29T05:23:53.005 INFO:teuthology.orchestra.run.smithi153.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.012549 s, 836 MB/s 2022-09-29T05:23:53.006 INFO:teuthology.orchestra.run.smithi153.stderr:--> Zapping successful for: 2022-09-29T05:23:53.113 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:23:51.577446+0000 mgr.y (mgr.14146) 106 : cluster [DBG] pgmap v85: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2022-09-29T05:23:53.114 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:52.477539+0000 mon.a (mon.0) 376 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:23:53.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:23:51.577446+0000 mgr.y (mgr.14146) 106 : cluster [DBG] pgmap v85: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2022-09-29T05:23:53.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:52.477539+0000 mon.a (mon.0) 376 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:23:53.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:23:51.577446+0000 mgr.y (mgr.14146) 106 : cluster [DBG] pgmap v85: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2022-09-29T05:23:53.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:52.477539+0000 mon.a (mon.0) 376 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:23:53.535 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/daemon-base:latest-pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 69111522-3fb6-11ed-8431-001a4aab830c -- ceph orch daemon add osd smithi153:vg_nvme/lv_3 2022-09-29T05:23:53.983 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:53.804163+0000 mon.b (mon.2) 8 : audit [INF] from='osd.4 [v2:172.21.15.153:6800/4053209633,v1:172.21.15.153:6801/4053209633]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2022-09-29T05:23:53.983 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:53.804671+0000 mon.a (mon.0) 377 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2022-09-29T05:23:53.984 INFO:journalctl@ceph.osd.4.smithi153.stdout:Sep 29 05:23:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-osd-4[42555]: debug 2022-09-29T05:23:53.800+0000 7fa630af4200 -1 osd.4 0 log_to_monitors {default=true} 2022-09-29T05:23:54.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:53.804163+0000 mon.b (mon.2) 8 : audit [INF] from='osd.4 [v2:172.21.15.153:6800/4053209633,v1:172.21.15.153:6801/4053209633]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2022-09-29T05:23:54.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:53.804671+0000 mon.a (mon.0) 377 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2022-09-29T05:23:54.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:53.804163+0000 mon.b (mon.2) 8 : audit [INF] from='osd.4 [v2:172.21.15.153:6800/4053209633,v1:172.21.15.153:6801/4053209633]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2022-09-29T05:23:54.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:53.804671+0000 mon.a (mon.0) 377 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2022-09-29T05:23:55.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:23:53.577795+0000 mgr.y (mgr.14146) 107 : cluster [DBG] pgmap v86: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2022-09-29T05:23:55.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:53.861656+0000 mon.a (mon.0) 378 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2022-09-29T05:23:55.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:23:53.861729+0000 mon.a (mon.0) 379 : cluster [DBG] osdmap e28: 5 total, 4 up, 5 in 2022-09-29T05:23:55.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:53.861873+0000 mon.a (mon.0) 380 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2022-09-29T05:23:55.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:53.862131+0000 mon.b (mon.2) 9 : audit [INF] from='osd.4 [v2:172.21.15.153:6800/4053209633,v1:172.21.15.153:6801/4053209633]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2022-09-29T05:23:55.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:53.862621+0000 mon.a (mon.0) 381 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2022-09-29T05:23:55.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:23:53.577795+0000 mgr.y (mgr.14146) 107 : cluster [DBG] pgmap v86: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2022-09-29T05:23:55.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:53.861656+0000 mon.a (mon.0) 378 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2022-09-29T05:23:55.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:23:53.861729+0000 mon.a (mon.0) 379 : cluster [DBG] osdmap e28: 5 total, 4 up, 5 in 2022-09-29T05:23:55.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:53.861873+0000 mon.a (mon.0) 380 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2022-09-29T05:23:55.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:53.862131+0000 mon.b (mon.2) 9 : audit [INF] from='osd.4 [v2:172.21.15.153:6800/4053209633,v1:172.21.15.153:6801/4053209633]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2022-09-29T05:23:55.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:53.862621+0000 mon.a (mon.0) 381 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2022-09-29T05:23:55.303 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:54 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:23:53.577795+0000 mgr.y (mgr.14146) 107 : cluster [DBG] pgmap v86: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2022-09-29T05:23:55.304 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:54 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:53.861656+0000 mon.a (mon.0) 378 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2022-09-29T05:23:55.304 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:54 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:23:53.861729+0000 mon.a (mon.0) 379 : cluster [DBG] osdmap e28: 5 total, 4 up, 5 in 2022-09-29T05:23:55.304 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:54 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:53.861873+0000 mon.a (mon.0) 380 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2022-09-29T05:23:55.305 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:54 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:53.862131+0000 mon.b (mon.2) 9 : audit [INF] from='osd.4 [v2:172.21.15.153:6800/4053209633,v1:172.21.15.153:6801/4053209633]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2022-09-29T05:23:55.305 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:54 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:53.862621+0000 mon.a (mon.0) 381 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2022-09-29T05:23:55.305 INFO:journalctl@ceph.osd.4.smithi153.stdout:Sep 29 05:23:54 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-osd-4[42555]: debug 2022-09-29T05:23:54.876+0000 7fa62707a700 -1 osd.4 0 waiting for initial osdmap 2022-09-29T05:23:55.306 INFO:journalctl@ceph.osd.4.smithi153.stdout:Sep 29 05:23:54 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-osd-4[42555]: debug 2022-09-29T05:23:54.883+0000 7fa61f7ee700 -1 osd.4 29 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2022-09-29T05:23:56.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:54.868073+0000 mon.a (mon.0) 382 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]': finished 2022-09-29T05:23:56.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:23:54.868171+0000 mon.a (mon.0) 383 : cluster [DBG] osdmap e29: 5 total, 4 up, 5 in 2022-09-29T05:23:56.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:54.868839+0000 mon.a (mon.0) 384 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2022-09-29T05:23:56.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:54.871580+0000 mon.a (mon.0) 385 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2022-09-29T05:23:56.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:55.442948+0000 mon.a (mon.0) 386 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2022-09-29T05:23:56.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:55.445227+0000 mon.a (mon.0) 387 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2022-09-29T05:23:56.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:55.445924+0000 mon.a (mon.0) 388 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:23:56.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:55.664885+0000 mon.a (mon.0) 389 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:23:56.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:55.665863+0000 mon.a (mon.0) 390 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:23:56.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:55.670901+0000 mon.a (mon.0) 391 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:23:56.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:55.675380+0000 mon.a (mon.0) 392 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:23:56.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:55.677400+0000 mon.a (mon.0) 393 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:23:56.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:55.678482+0000 mon.a (mon.0) 394 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:23:56.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:55.679144+0000 mon.a (mon.0) 395 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:23:56.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:55.683954+0000 mon.a (mon.0) 396 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:23:56.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:23:55.868635+0000 mon.a (mon.0) 397 : cluster [INF] osd.4 [v2:172.21.15.153:6800/4053209633,v1:172.21.15.153:6801/4053209633] boot 2022-09-29T05:23:56.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:23:55.868689+0000 mon.a (mon.0) 398 : cluster [DBG] osdmap e30: 5 total, 5 up, 5 in 2022-09-29T05:23:56.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:55.868851+0000 mon.a (mon.0) 399 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2022-09-29T05:23:56.218 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:54.868073+0000 mon.a (mon.0) 382 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]': finished 2022-09-29T05:23:56.218 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:23:54.868171+0000 mon.a (mon.0) 383 : cluster [DBG] osdmap e29: 5 total, 4 up, 5 in 2022-09-29T05:23:56.218 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:54.868839+0000 mon.a (mon.0) 384 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2022-09-29T05:23:56.218 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:54.871580+0000 mon.a (mon.0) 385 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2022-09-29T05:23:56.219 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:55.442948+0000 mon.a (mon.0) 386 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2022-09-29T05:23:56.219 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:55.445227+0000 mon.a (mon.0) 387 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2022-09-29T05:23:56.219 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:55.445924+0000 mon.a (mon.0) 388 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:23:56.219 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:55.664885+0000 mon.a (mon.0) 389 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:23:56.219 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:55.665863+0000 mon.a (mon.0) 390 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:23:56.220 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:55.670901+0000 mon.a (mon.0) 391 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:23:56.220 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:55.675380+0000 mon.a (mon.0) 392 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:23:56.220 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:55.677400+0000 mon.a (mon.0) 393 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:23:56.220 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:55.678482+0000 mon.a (mon.0) 394 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:23:56.221 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:55.679144+0000 mon.a (mon.0) 395 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:23:56.221 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:55.683954+0000 mon.a (mon.0) 396 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:23:56.221 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:23:55.868635+0000 mon.a (mon.0) 397 : cluster [INF] osd.4 [v2:172.21.15.153:6800/4053209633,v1:172.21.15.153:6801/4053209633] boot 2022-09-29T05:23:56.221 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:23:55.868689+0000 mon.a (mon.0) 398 : cluster [DBG] osdmap e30: 5 total, 5 up, 5 in 2022-09-29T05:23:56.222 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:55.868851+0000 mon.a (mon.0) 399 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2022-09-29T05:23:56.236 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:55 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:54.868073+0000 mon.a (mon.0) 382 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]': finished 2022-09-29T05:23:56.236 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:55 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:23:54.868171+0000 mon.a (mon.0) 383 : cluster [DBG] osdmap e29: 5 total, 4 up, 5 in 2022-09-29T05:23:56.237 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:55 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:54.868839+0000 mon.a (mon.0) 384 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2022-09-29T05:23:56.237 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:55 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:54.871580+0000 mon.a (mon.0) 385 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2022-09-29T05:23:56.237 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:55 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:55.442948+0000 mon.a (mon.0) 386 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2022-09-29T05:23:56.237 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:55 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:55.445227+0000 mon.a (mon.0) 387 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2022-09-29T05:23:56.237 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:55 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:55.445924+0000 mon.a (mon.0) 388 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:23:56.238 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:55 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:55.664885+0000 mon.a (mon.0) 389 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:23:56.238 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:55 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:55.665863+0000 mon.a (mon.0) 390 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:23:56.238 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:55 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:55.670901+0000 mon.a (mon.0) 391 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:23:56.238 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:55 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:55.675380+0000 mon.a (mon.0) 392 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:23:56.239 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:55 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:55.677400+0000 mon.a (mon.0) 393 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:23:56.239 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:55 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:55.678482+0000 mon.a (mon.0) 394 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:23:56.239 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:55 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:55.679144+0000 mon.a (mon.0) 395 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:23:56.239 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:55 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:55.683954+0000 mon.a (mon.0) 396 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:23:56.240 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:55 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:23:55.868635+0000 mon.a (mon.0) 397 : cluster [INF] osd.4 [v2:172.21.15.153:6800/4053209633,v1:172.21.15.153:6801/4053209633] boot 2022-09-29T05:23:56.240 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:55 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:23:55.868689+0000 mon.a (mon.0) 398 : cluster [DBG] osdmap e30: 5 total, 5 up, 5 in 2022-09-29T05:23:56.240 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:55 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:55.868851+0000 mon.a (mon.0) 399 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2022-09-29T05:23:57.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:23:54.756054+0000 osd.4 (osd.4) 1 : cluster [DBG] purged_snaps scrub starts 2022-09-29T05:23:57.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:23:54.756144+0000 osd.4 (osd.4) 2 : cluster [DBG] purged_snaps scrub ok 2022-09-29T05:23:57.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:55.440753+0000 mgr.y (mgr.14146) 108 : audit [DBG] from='client.24239 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi153:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:23:57.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:23:55.578152+0000 mgr.y (mgr.14146) 109 : cluster [DBG] pgmap v89: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2022-09-29T05:23:57.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:23:55.660195+0000 mgr.y (mgr.14146) 110 : cephadm [INF] Detected new or changed devices on smithi153 2022-09-29T05:23:57.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:23:55.666498+0000 mgr.y (mgr.14146) 111 : cephadm [INF] Adjusting osd_memory_target on smithi153 to 17168M 2022-09-29T05:23:57.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:23:56.871006+0000 mon.a (mon.0) 400 : cluster [DBG] osdmap e31: 5 total, 5 up, 5 in 2022-09-29T05:23:57.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:23:54.756054+0000 osd.4 (osd.4) 1 : cluster [DBG] purged_snaps scrub starts 2022-09-29T05:23:57.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:23:54.756144+0000 osd.4 (osd.4) 2 : cluster [DBG] purged_snaps scrub ok 2022-09-29T05:23:57.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:55.440753+0000 mgr.y (mgr.14146) 108 : audit [DBG] from='client.24239 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi153:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:23:57.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:23:55.578152+0000 mgr.y (mgr.14146) 109 : cluster [DBG] pgmap v89: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2022-09-29T05:23:57.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:23:55.660195+0000 mgr.y (mgr.14146) 110 : cephadm [INF] Detected new or changed devices on smithi153 2022-09-29T05:23:57.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:23:55.666498+0000 mgr.y (mgr.14146) 111 : cephadm [INF] Adjusting osd_memory_target on smithi153 to 17168M 2022-09-29T05:23:57.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:23:56.871006+0000 mon.a (mon.0) 400 : cluster [DBG] osdmap e31: 5 total, 5 up, 5 in 2022-09-29T05:23:57.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:56 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:23:54.756054+0000 osd.4 (osd.4) 1 : cluster [DBG] purged_snaps scrub starts 2022-09-29T05:23:57.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:56 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:23:54.756144+0000 osd.4 (osd.4) 2 : cluster [DBG] purged_snaps scrub ok 2022-09-29T05:23:57.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:56 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:55.440753+0000 mgr.y (mgr.14146) 108 : audit [DBG] from='client.24239 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi153:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:23:57.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:56 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:23:55.578152+0000 mgr.y (mgr.14146) 109 : cluster [DBG] pgmap v89: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2022-09-29T05:23:57.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:56 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:23:55.660195+0000 mgr.y (mgr.14146) 110 : cephadm [INF] Detected new or changed devices on smithi153 2022-09-29T05:23:57.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:56 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:23:55.666498+0000 mgr.y (mgr.14146) 111 : cephadm [INF] Adjusting osd_memory_target on smithi153 to 17168M 2022-09-29T05:23:57.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:56 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:23:56.871006+0000 mon.a (mon.0) 400 : cluster [DBG] osdmap e31: 5 total, 5 up, 5 in 2022-09-29T05:23:58.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:57 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:57.496471+0000 mon.b (mon.2) 10 : audit [INF] from='client.? 172.21.15.153:0/3800060268' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c69e2263-5dd8-402d-b901-295c1f4840e2"}]: dispatch 2022-09-29T05:23:58.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:57 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:57.497222+0000 mon.a (mon.0) 401 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c69e2263-5dd8-402d-b901-295c1f4840e2"}]: dispatch 2022-09-29T05:23:58.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:57 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:57.502643+0000 mon.a (mon.0) 402 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c69e2263-5dd8-402d-b901-295c1f4840e2"}]': finished 2022-09-29T05:23:58.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:57 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:23:57.502737+0000 mon.a (mon.0) 403 : cluster [DBG] osdmap e32: 6 total, 5 up, 6 in 2022-09-29T05:23:58.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:57 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:57.502916+0000 mon.a (mon.0) 404 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2022-09-29T05:23:58.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:57 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:57.496471+0000 mon.b (mon.2) 10 : audit [INF] from='client.? 172.21.15.153:0/3800060268' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c69e2263-5dd8-402d-b901-295c1f4840e2"}]: dispatch 2022-09-29T05:23:58.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:57 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:57.497222+0000 mon.a (mon.0) 401 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c69e2263-5dd8-402d-b901-295c1f4840e2"}]: dispatch 2022-09-29T05:23:58.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:57 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:57.502643+0000 mon.a (mon.0) 402 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c69e2263-5dd8-402d-b901-295c1f4840e2"}]': finished 2022-09-29T05:23:58.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:57 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:23:57.502737+0000 mon.a (mon.0) 403 : cluster [DBG] osdmap e32: 6 total, 5 up, 6 in 2022-09-29T05:23:58.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:57 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:57.502916+0000 mon.a (mon.0) 404 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2022-09-29T05:23:58.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:57 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:57.496471+0000 mon.b (mon.2) 10 : audit [INF] from='client.? 172.21.15.153:0/3800060268' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c69e2263-5dd8-402d-b901-295c1f4840e2"}]: dispatch 2022-09-29T05:23:58.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:57 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:57.497222+0000 mon.a (mon.0) 401 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c69e2263-5dd8-402d-b901-295c1f4840e2"}]: dispatch 2022-09-29T05:23:58.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:57 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:57.502643+0000 mon.a (mon.0) 402 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c69e2263-5dd8-402d-b901-295c1f4840e2"}]': finished 2022-09-29T05:23:58.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:57 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:23:57.502737+0000 mon.a (mon.0) 403 : cluster [DBG] osdmap e32: 6 total, 5 up, 6 in 2022-09-29T05:23:58.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:57 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:57.502916+0000 mon.a (mon.0) 404 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2022-09-29T05:23:59.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:23:57.578642+0000 mgr.y (mgr.14146) 112 : cluster [DBG] pgmap v93: 1 pgs: 1 active+clean; 0 B data, 24 MiB used, 447 GiB / 447 GiB avail 2022-09-29T05:23:59.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:23:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:23:58.043080+0000 mon.b (mon.2) 11 : audit [DBG] from='client.? 172.21.15.153:0/2001485617' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2022-09-29T05:23:59.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:23:57.578642+0000 mgr.y (mgr.14146) 112 : cluster [DBG] pgmap v93: 1 pgs: 1 active+clean; 0 B data, 24 MiB used, 447 GiB / 447 GiB avail 2022-09-29T05:23:59.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:23:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:23:58.043080+0000 mon.b (mon.2) 11 : audit [DBG] from='client.? 172.21.15.153:0/2001485617' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2022-09-29T05:23:59.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:58 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:23:57.578642+0000 mgr.y (mgr.14146) 112 : cluster [DBG] pgmap v93: 1 pgs: 1 active+clean; 0 B data, 24 MiB used, 447 GiB / 447 GiB avail 2022-09-29T05:23:59.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:23:58 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:23:58.043080+0000 mon.b (mon.2) 11 : audit [DBG] from='client.? 172.21.15.153:0/2001485617' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2022-09-29T05:24:01.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:23:59.578969+0000 mgr.y (mgr.14146) 113 : cluster [DBG] pgmap v94: 1 pgs: 1 active+clean; 0 B data, 24 MiB used, 447 GiB / 447 GiB avail 2022-09-29T05:24:01.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:23:59.578969+0000 mgr.y (mgr.14146) 113 : cluster [DBG] pgmap v94: 1 pgs: 1 active+clean; 0 B data, 24 MiB used, 447 GiB / 447 GiB avail 2022-09-29T05:24:01.301 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:00 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:23:59.578969+0000 mgr.y (mgr.14146) 113 : cluster [DBG] pgmap v94: 1 pgs: 1 active+clean; 0 B data, 24 MiB used, 447 GiB / 447 GiB avail 2022-09-29T05:24:03.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:24:01.579393+0000 mgr.y (mgr.14146) 114 : cluster [DBG] pgmap v95: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2022-09-29T05:24:03.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:02.822616+0000 mon.a (mon.0) 405 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2022-09-29T05:24:03.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:02.823472+0000 mon.a (mon.0) 406 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:24:03.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:24:01.579393+0000 mgr.y (mgr.14146) 114 : cluster [DBG] pgmap v95: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2022-09-29T05:24:03.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:02.822616+0000 mon.a (mon.0) 405 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2022-09-29T05:24:03.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:02.823472+0000 mon.a (mon.0) 406 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:24:03.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:24:01.579393+0000 mgr.y (mgr.14146) 114 : cluster [DBG] pgmap v95: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2022-09-29T05:24:03.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:02.822616+0000 mon.a (mon.0) 405 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2022-09-29T05:24:03.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:02.823472+0000 mon.a (mon.0) 406 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:24:04.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:24:02.824172+0000 mgr.y (mgr.14146) 115 : cephadm [INF] Deploying daemon osd.5 on smithi153 2022-09-29T05:24:04.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:24:02.824172+0000 mgr.y (mgr.14146) 115 : cephadm [INF] Deploying daemon osd.5 on smithi153 2022-09-29T05:24:04.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:24:02.824172+0000 mgr.y (mgr.14146) 115 : cephadm [INF] Deploying daemon osd.5 on smithi153 2022-09-29T05:24:05.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:04 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:24:03.579885+0000 mgr.y (mgr.14146) 116 : cluster [DBG] pgmap v96: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2022-09-29T05:24:05.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:24:03.579885+0000 mgr.y (mgr.14146) 116 : cluster [DBG] pgmap v96: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2022-09-29T05:24:05.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:24:03.579885+0000 mgr.y (mgr.14146) 116 : cluster [DBG] pgmap v96: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2022-09-29T05:24:07.194 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:06 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:24:05.580403+0000 mgr.y (mgr.14146) 117 : cluster [DBG] pgmap v97: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2022-09-29T05:24:07.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:24:05.580403+0000 mgr.y (mgr.14146) 117 : cluster [DBG] pgmap v97: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2022-09-29T05:24:07.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:24:05.580403+0000 mgr.y (mgr.14146) 117 : cluster [DBG] pgmap v97: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2022-09-29T05:24:07.433 INFO:teuthology.orchestra.run.smithi153.stdout:Created osd(s) 5 on host 'smithi153' 2022-09-29T05:24:07.900 DEBUG:teuthology.orchestra.run.smithi153:osd.5> sudo journalctl -f -n 0 -u ceph-69111522-3fb6-11ed-8431-001a4aab830c@osd.5.service 2022-09-29T05:24:07.902 INFO:tasks.cephadm:Deploying osd.6 on smithi153 with /dev/vg_nvme/lv_2... 2022-09-29T05:24:07.903 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/daemon-base:latest-pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 69111522-3fb6-11ed-8431-001a4aab830c -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2022-09-29T05:24:07.935 INFO:journalctl@ceph.osd.5.smithi153.stdout:-- Logs begin at Thu 2022-09-29 05:12:53 UTC. -- 2022-09-29T05:24:08.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:08 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:07.421293+0000 mon.a (mon.0) 407 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:24:08.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:08 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:07.428321+0000 mon.a (mon.0) 408 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:24:08.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:08 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:07.428588+0000 mon.a (mon.0) 409 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:24:08.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:08 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:07.430516+0000 mon.a (mon.0) 410 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:24:08.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:08 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:07.431356+0000 mon.a (mon.0) 411 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:24:08.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:08 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:24:07.580716+0000 mgr.y (mgr.14146) 118 : cluster [DBG] pgmap v98: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2022-09-29T05:24:08.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:08 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:07.421293+0000 mon.a (mon.0) 407 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:24:08.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:08 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:07.428321+0000 mon.a (mon.0) 408 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:24:08.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:08 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:07.428588+0000 mon.a (mon.0) 409 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:24:08.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:08 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:07.430516+0000 mon.a (mon.0) 410 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:24:08.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:08 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:07.431356+0000 mon.a (mon.0) 411 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:24:08.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:08 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:24:07.580716+0000 mgr.y (mgr.14146) 118 : cluster [DBG] pgmap v98: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2022-09-29T05:24:08.737 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:08 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:07.421293+0000 mon.a (mon.0) 407 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:24:08.737 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:08 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:07.428321+0000 mon.a (mon.0) 408 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:24:08.738 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:08 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:07.428588+0000 mon.a (mon.0) 409 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:24:08.738 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:08 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:07.430516+0000 mon.a (mon.0) 410 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:24:08.738 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:08 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:07.431356+0000 mon.a (mon.0) 411 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:24:08.738 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:08 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:24:07.580716+0000 mgr.y (mgr.14146) 118 : cluster [DBG] pgmap v98: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2022-09-29T05:24:09.793 INFO:teuthology.orchestra.run.smithi153.stderr:--> Zapping: /dev/vg_nvme/lv_2 2022-09-29T05:24:09.794 INFO:teuthology.orchestra.run.smithi153.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2022-09-29T05:24:09.794 INFO:teuthology.orchestra.run.smithi153.stderr: stderr: 10+0 records in 2022-09-29T05:24:09.794 INFO:teuthology.orchestra.run.smithi153.stderr:10+0 records out 2022-09-29T05:24:09.794 INFO:teuthology.orchestra.run.smithi153.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.012638 s, 830 MB/s 2022-09-29T05:24:09.795 INFO:teuthology.orchestra.run.smithi153.stderr:--> Zapping successful for: 2022-09-29T05:24:10.477 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/daemon-base:latest-pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 69111522-3fb6-11ed-8431-001a4aab830c -- ceph orch daemon add osd smithi153:vg_nvme/lv_2 2022-09-29T05:24:10.494 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:10 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:09.293687+0000 mon.a (mon.0) 412 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:24:10.494 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:10 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:24:09.581011+0000 mgr.y (mgr.14146) 119 : cluster [DBG] pgmap v99: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2022-09-29T05:24:10.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:09.293687+0000 mon.a (mon.0) 412 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:24:10.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:24:09.581011+0000 mgr.y (mgr.14146) 119 : cluster [DBG] pgmap v99: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2022-09-29T05:24:10.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:09.293687+0000 mon.a (mon.0) 412 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:24:10.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:24:09.581011+0000 mgr.y (mgr.14146) 119 : cluster [DBG] pgmap v99: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2022-09-29T05:24:10.809 INFO:journalctl@ceph.osd.5.smithi153.stdout:Sep 29 05:24:10 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-osd-5[45562]: debug 2022-09-29T05:24:10.619+0000 7f79842fb200 -1 osd.5 0 log_to_monitors {default=true} 2022-09-29T05:24:11.494 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:11 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:10.623994+0000 mon.b (mon.2) 12 : audit [INF] from='osd.5 [v2:172.21.15.153:6808/2536485968,v1:172.21.15.153:6809/2536485968]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2022-09-29T05:24:11.495 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:11 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:10.624542+0000 mon.a (mon.0) 413 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2022-09-29T05:24:11.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:10.623994+0000 mon.b (mon.2) 12 : audit [INF] from='osd.5 [v2:172.21.15.153:6808/2536485968,v1:172.21.15.153:6809/2536485968]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2022-09-29T05:24:11.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:10.624542+0000 mon.a (mon.0) 413 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2022-09-29T05:24:11.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:10.623994+0000 mon.b (mon.2) 12 : audit [INF] from='osd.5 [v2:172.21.15.153:6808/2536485968,v1:172.21.15.153:6809/2536485968]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2022-09-29T05:24:11.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:10.624542+0000 mon.a (mon.0) 413 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2022-09-29T05:24:12.559 INFO:journalctl@ceph.osd.5.smithi153.stdout:Sep 29 05:24:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-osd-5[45562]: debug 2022-09-29T05:24:12.311+0000 7f797a881700 -1 osd.5 0 waiting for initial osdmap 2022-09-29T05:24:12.559 INFO:journalctl@ceph.osd.5.smithi153.stdout:Sep 29 05:24:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-osd-5[45562]: debug 2022-09-29T05:24:12.316+0000 7f7972ff5700 -1 osd.5 34 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2022-09-29T05:24:12.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:11.304534+0000 mon.a (mon.0) 414 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2022-09-29T05:24:12.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:24:11.304647+0000 mon.a (mon.0) 415 : cluster [DBG] osdmap e33: 6 total, 5 up, 6 in 2022-09-29T05:24:12.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:11.305238+0000 mon.a (mon.0) 416 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2022-09-29T05:24:12.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:11.306318+0000 mon.b (mon.2) 13 : audit [INF] from='osd.5 [v2:172.21.15.153:6808/2536485968,v1:172.21.15.153:6809/2536485968]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2022-09-29T05:24:12.561 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:11.306834+0000 mon.a (mon.0) 417 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2022-09-29T05:24:12.561 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:24:11.581371+0000 mgr.y (mgr.14146) 120 : cluster [DBG] pgmap v101: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2022-09-29T05:24:12.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:11.304534+0000 mon.a (mon.0) 414 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2022-09-29T05:24:12.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:24:11.304647+0000 mon.a (mon.0) 415 : cluster [DBG] osdmap e33: 6 total, 5 up, 6 in 2022-09-29T05:24:12.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:11.305238+0000 mon.a (mon.0) 416 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2022-09-29T05:24:12.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:11.306318+0000 mon.b (mon.2) 13 : audit [INF] from='osd.5 [v2:172.21.15.153:6808/2536485968,v1:172.21.15.153:6809/2536485968]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2022-09-29T05:24:12.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:11.306834+0000 mon.a (mon.0) 417 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2022-09-29T05:24:12.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:24:11.581371+0000 mgr.y (mgr.14146) 120 : cluster [DBG] pgmap v101: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2022-09-29T05:24:12.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:11.304534+0000 mon.a (mon.0) 414 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2022-09-29T05:24:12.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:24:11.304647+0000 mon.a (mon.0) 415 : cluster [DBG] osdmap e33: 6 total, 5 up, 6 in 2022-09-29T05:24:12.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:11.305238+0000 mon.a (mon.0) 416 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2022-09-29T05:24:12.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:11.306318+0000 mon.b (mon.2) 13 : audit [INF] from='osd.5 [v2:172.21.15.153:6808/2536485968,v1:172.21.15.153:6809/2536485968]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2022-09-29T05:24:12.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:11.306834+0000 mon.a (mon.0) 417 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2022-09-29T05:24:12.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:24:11.581371+0000 mgr.y (mgr.14146) 120 : cluster [DBG] pgmap v101: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2022-09-29T05:24:13.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:12.306007+0000 mon.a (mon.0) 418 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]': finished 2022-09-29T05:24:13.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:24:12.306117+0000 mon.a (mon.0) 419 : cluster [DBG] osdmap e34: 6 total, 5 up, 6 in 2022-09-29T05:24:13.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:12.306525+0000 mon.a (mon.0) 420 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2022-09-29T05:24:13.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:12.310256+0000 mon.a (mon.0) 421 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2022-09-29T05:24:13.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:12.380298+0000 mgr.y (mgr.14146) 121 : audit [DBG] from='client.24266 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi153:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:24:13.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:12.382519+0000 mon.a (mon.0) 422 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2022-09-29T05:24:13.561 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:12.384666+0000 mon.a (mon.0) 423 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2022-09-29T05:24:13.561 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:12.385344+0000 mon.a (mon.0) 424 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:24:13.561 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:24:12.585379+0000 mgr.y (mgr.14146) 122 : cephadm [INF] Detected new or changed devices on smithi153 2022-09-29T05:24:13.561 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:12.590571+0000 mon.a (mon.0) 425 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:24:13.561 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:12.988802+0000 mon.a (mon.0) 426 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:24:13.562 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:12.989754+0000 mon.a (mon.0) 427 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:24:13.562 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:24:12.990405+0000 mgr.y (mgr.14146) 123 : cephadm [INF] Adjusting osd_memory_target on smithi153 to 8584M 2022-09-29T05:24:13.562 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:12.994868+0000 mon.a (mon.0) 428 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:24:13.562 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:12.999621+0000 mon.a (mon.0) 429 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:24:13.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:12.306007+0000 mon.a (mon.0) 418 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]': finished 2022-09-29T05:24:13.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:24:12.306117+0000 mon.a (mon.0) 419 : cluster [DBG] osdmap e34: 6 total, 5 up, 6 in 2022-09-29T05:24:13.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:12.306525+0000 mon.a (mon.0) 420 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2022-09-29T05:24:13.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:12.310256+0000 mon.a (mon.0) 421 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2022-09-29T05:24:13.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:12.380298+0000 mgr.y (mgr.14146) 121 : audit [DBG] from='client.24266 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi153:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:24:13.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:12.382519+0000 mon.a (mon.0) 422 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2022-09-29T05:24:13.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:12.384666+0000 mon.a (mon.0) 423 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2022-09-29T05:24:13.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:12.385344+0000 mon.a (mon.0) 424 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:24:13.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:24:12.585379+0000 mgr.y (mgr.14146) 122 : cephadm [INF] Detected new or changed devices on smithi153 2022-09-29T05:24:13.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:12.590571+0000 mon.a (mon.0) 425 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:24:13.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:12.988802+0000 mon.a (mon.0) 426 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:24:13.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:12.989754+0000 mon.a (mon.0) 427 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:24:13.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:24:12.990405+0000 mgr.y (mgr.14146) 123 : cephadm [INF] Adjusting osd_memory_target on smithi153 to 8584M 2022-09-29T05:24:13.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:12.994868+0000 mon.a (mon.0) 428 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:24:13.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:12.999621+0000 mon.a (mon.0) 429 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:24:13.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:12.306007+0000 mon.a (mon.0) 418 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]': finished 2022-09-29T05:24:13.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:24:12.306117+0000 mon.a (mon.0) 419 : cluster [DBG] osdmap e34: 6 total, 5 up, 6 in 2022-09-29T05:24:13.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:12.306525+0000 mon.a (mon.0) 420 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2022-09-29T05:24:13.718 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:12.310256+0000 mon.a (mon.0) 421 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2022-09-29T05:24:13.718 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:12.380298+0000 mgr.y (mgr.14146) 121 : audit [DBG] from='client.24266 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi153:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:24:13.718 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:12.382519+0000 mon.a (mon.0) 422 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2022-09-29T05:24:13.718 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:12.384666+0000 mon.a (mon.0) 423 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2022-09-29T05:24:13.719 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:12.385344+0000 mon.a (mon.0) 424 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:24:13.719 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:24:12.585379+0000 mgr.y (mgr.14146) 122 : cephadm [INF] Detected new or changed devices on smithi153 2022-09-29T05:24:13.719 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:12.590571+0000 mon.a (mon.0) 425 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:24:13.719 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:12.988802+0000 mon.a (mon.0) 426 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:24:13.720 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:12.989754+0000 mon.a (mon.0) 427 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:24:13.720 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:24:12.990405+0000 mgr.y (mgr.14146) 123 : cephadm [INF] Adjusting osd_memory_target on smithi153 to 8584M 2022-09-29T05:24:13.720 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:12.994868+0000 mon.a (mon.0) 428 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:24:13.720 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:12.999621+0000 mon.a (mon.0) 429 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:24:14.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:24:11.670397+0000 osd.5 (osd.5) 1 : cluster [DBG] purged_snaps scrub starts 2022-09-29T05:24:14.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:24:11.670496+0000 osd.5 (osd.5) 2 : cluster [DBG] purged_snaps scrub ok 2022-09-29T05:24:14.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:13.312732+0000 mon.a (mon.0) 430 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2022-09-29T05:24:14.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:24:13.316329+0000 mon.a (mon.0) 431 : cluster [INF] osd.5 [v2:172.21.15.153:6808/2536485968,v1:172.21.15.153:6809/2536485968] boot 2022-09-29T05:24:14.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:24:13.316392+0000 mon.a (mon.0) 432 : cluster [DBG] osdmap e35: 6 total, 6 up, 6 in 2022-09-29T05:24:14.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:13.316540+0000 mon.a (mon.0) 433 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2022-09-29T05:24:14.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:24:13.581748+0000 mgr.y (mgr.14146) 124 : cluster [DBG] pgmap v104: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2022-09-29T05:24:14.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:14.302906+0000 mon.b (mon.2) 14 : audit [INF] from='client.? 172.21.15.153:0/2974615079' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "72aa13b6-e7b7-449b-9fa8-f284e8f0735a"}]: dispatch 2022-09-29T05:24:14.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:14.303685+0000 mon.a (mon.0) 434 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "72aa13b6-e7b7-449b-9fa8-f284e8f0735a"}]: dispatch 2022-09-29T05:24:14.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:24:11.670397+0000 osd.5 (osd.5) 1 : cluster [DBG] purged_snaps scrub starts 2022-09-29T05:24:14.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:24:11.670496+0000 osd.5 (osd.5) 2 : cluster [DBG] purged_snaps scrub ok 2022-09-29T05:24:14.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:13.312732+0000 mon.a (mon.0) 430 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2022-09-29T05:24:14.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:24:13.316329+0000 mon.a (mon.0) 431 : cluster [INF] osd.5 [v2:172.21.15.153:6808/2536485968,v1:172.21.15.153:6809/2536485968] boot 2022-09-29T05:24:14.718 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:24:13.316392+0000 mon.a (mon.0) 432 : cluster [DBG] osdmap e35: 6 total, 6 up, 6 in 2022-09-29T05:24:14.718 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:13.316540+0000 mon.a (mon.0) 433 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2022-09-29T05:24:14.719 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:24:13.581748+0000 mgr.y (mgr.14146) 124 : cluster [DBG] pgmap v104: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2022-09-29T05:24:14.719 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:14.302906+0000 mon.b (mon.2) 14 : audit [INF] from='client.? 172.21.15.153:0/2974615079' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "72aa13b6-e7b7-449b-9fa8-f284e8f0735a"}]: dispatch 2022-09-29T05:24:14.719 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:14.303685+0000 mon.a (mon.0) 434 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "72aa13b6-e7b7-449b-9fa8-f284e8f0735a"}]: dispatch 2022-09-29T05:24:14.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:14 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:24:11.670397+0000 osd.5 (osd.5) 1 : cluster [DBG] purged_snaps scrub starts 2022-09-29T05:24:14.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:14 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:24:11.670496+0000 osd.5 (osd.5) 2 : cluster [DBG] purged_snaps scrub ok 2022-09-29T05:24:14.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:14 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:13.312732+0000 mon.a (mon.0) 430 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2022-09-29T05:24:14.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:14 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:24:13.316329+0000 mon.a (mon.0) 431 : cluster [INF] osd.5 [v2:172.21.15.153:6808/2536485968,v1:172.21.15.153:6809/2536485968] boot 2022-09-29T05:24:14.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:14 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:24:13.316392+0000 mon.a (mon.0) 432 : cluster [DBG] osdmap e35: 6 total, 6 up, 6 in 2022-09-29T05:24:14.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:14 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:13.316540+0000 mon.a (mon.0) 433 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2022-09-29T05:24:14.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:14 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:24:13.581748+0000 mgr.y (mgr.14146) 124 : cluster [DBG] pgmap v104: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2022-09-29T05:24:14.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:14 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:14.302906+0000 mon.b (mon.2) 14 : audit [INF] from='client.? 172.21.15.153:0/2974615079' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "72aa13b6-e7b7-449b-9fa8-f284e8f0735a"}]: dispatch 2022-09-29T05:24:14.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:14 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:14.303685+0000 mon.a (mon.0) 434 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "72aa13b6-e7b7-449b-9fa8-f284e8f0735a"}]: dispatch 2022-09-29T05:24:15.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:14.309914+0000 mon.a (mon.0) 435 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "72aa13b6-e7b7-449b-9fa8-f284e8f0735a"}]': finished 2022-09-29T05:24:15.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:24:14.310004+0000 mon.a (mon.0) 436 : cluster [DBG] osdmap e36: 7 total, 6 up, 7 in 2022-09-29T05:24:15.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:14.310832+0000 mon.a (mon.0) 437 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2022-09-29T05:24:15.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:14.885161+0000 mon.b (mon.2) 15 : audit [DBG] from='client.? 172.21.15.153:0/1040351708' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2022-09-29T05:24:15.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:24:15.310587+0000 mon.a (mon.0) 438 : cluster [DBG] osdmap e37: 7 total, 6 up, 7 in 2022-09-29T05:24:15.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:15.310791+0000 mon.a (mon.0) 439 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2022-09-29T05:24:15.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:14.309914+0000 mon.a (mon.0) 435 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "72aa13b6-e7b7-449b-9fa8-f284e8f0735a"}]': finished 2022-09-29T05:24:15.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:24:14.310004+0000 mon.a (mon.0) 436 : cluster [DBG] osdmap e36: 7 total, 6 up, 7 in 2022-09-29T05:24:15.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:14.310832+0000 mon.a (mon.0) 437 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2022-09-29T05:24:15.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:14.885161+0000 mon.b (mon.2) 15 : audit [DBG] from='client.? 172.21.15.153:0/1040351708' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2022-09-29T05:24:15.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:24:15.310587+0000 mon.a (mon.0) 438 : cluster [DBG] osdmap e37: 7 total, 6 up, 7 in 2022-09-29T05:24:15.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:15.310791+0000 mon.a (mon.0) 439 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2022-09-29T05:24:15.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:15 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:14.309914+0000 mon.a (mon.0) 435 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "72aa13b6-e7b7-449b-9fa8-f284e8f0735a"}]': finished 2022-09-29T05:24:15.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:15 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:24:14.310004+0000 mon.a (mon.0) 436 : cluster [DBG] osdmap e36: 7 total, 6 up, 7 in 2022-09-29T05:24:15.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:15 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:14.310832+0000 mon.a (mon.0) 437 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2022-09-29T05:24:15.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:15 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:14.885161+0000 mon.b (mon.2) 15 : audit [DBG] from='client.? 172.21.15.153:0/1040351708' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2022-09-29T05:24:15.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:15 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:24:15.310587+0000 mon.a (mon.0) 438 : cluster [DBG] osdmap e37: 7 total, 6 up, 7 in 2022-09-29T05:24:15.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:15 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:15.310791+0000 mon.a (mon.0) 439 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2022-09-29T05:24:16.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:24:15.582264+0000 mgr.y (mgr.14146) 125 : cluster [DBG] pgmap v107: 1 pgs: 1 active+clean; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2022-09-29T05:24:16.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:24:15.582264+0000 mgr.y (mgr.14146) 125 : cluster [DBG] pgmap v107: 1 pgs: 1 active+clean; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2022-09-29T05:24:16.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:16 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:24:15.582264+0000 mgr.y (mgr.14146) 125 : cluster [DBG] pgmap v107: 1 pgs: 1 active+clean; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2022-09-29T05:24:18.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:18 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:24:17.582703+0000 mgr.y (mgr.14146) 126 : cluster [DBG] pgmap v108: 1 pgs: 1 active+clean; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2022-09-29T05:24:18.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:18 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:24:17.582703+0000 mgr.y (mgr.14146) 126 : cluster [DBG] pgmap v108: 1 pgs: 1 active+clean; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2022-09-29T05:24:18.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:18 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:24:17.582703+0000 mgr.y (mgr.14146) 126 : cluster [DBG] pgmap v108: 1 pgs: 1 active+clean; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2022-09-29T05:24:20.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:24:19.583084+0000 mgr.y (mgr.14146) 127 : cluster [DBG] pgmap v109: 1 pgs: 1 active+clean; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2022-09-29T05:24:20.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:19.745599+0000 mon.a (mon.0) 440 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2022-09-29T05:24:20.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:19.746483+0000 mon.a (mon.0) 441 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:24:20.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:24:19.747217+0000 mgr.y (mgr.14146) 128 : cephadm [INF] Deploying daemon osd.6 on smithi153 2022-09-29T05:24:20.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:24:19.583084+0000 mgr.y (mgr.14146) 127 : cluster [DBG] pgmap v109: 1 pgs: 1 active+clean; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2022-09-29T05:24:20.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:19.745599+0000 mon.a (mon.0) 440 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2022-09-29T05:24:20.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:19.746483+0000 mon.a (mon.0) 441 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:24:20.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:24:19.747217+0000 mgr.y (mgr.14146) 128 : cephadm [INF] Deploying daemon osd.6 on smithi153 2022-09-29T05:24:21.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:24:19.583084+0000 mgr.y (mgr.14146) 127 : cluster [DBG] pgmap v109: 1 pgs: 1 active+clean; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2022-09-29T05:24:21.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:19.745599+0000 mon.a (mon.0) 440 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2022-09-29T05:24:21.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:19.746483+0000 mon.a (mon.0) 441 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:24:21.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:24:19.747217+0000 mgr.y (mgr.14146) 128 : cephadm [INF] Deploying daemon osd.6 on smithi153 2022-09-29T05:24:22.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:24:21.583443+0000 mgr.y (mgr.14146) 129 : cluster [DBG] pgmap v110: 1 pgs: 1 active+clean; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2022-09-29T05:24:22.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:24:21.583443+0000 mgr.y (mgr.14146) 129 : cluster [DBG] pgmap v110: 1 pgs: 1 active+clean; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2022-09-29T05:24:23.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:24:21.583443+0000 mgr.y (mgr.14146) 129 : cluster [DBG] pgmap v110: 1 pgs: 1 active+clean; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2022-09-29T05:24:24.437 INFO:teuthology.orchestra.run.smithi153.stdout:Created osd(s) 6 on host 'smithi153' 2022-09-29T05:24:24.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:24 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:24:23.583915+0000 mgr.y (mgr.14146) 130 : cluster [DBG] pgmap v111: 1 pgs: 1 active+clean; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2022-09-29T05:24:24.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:24 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:24.427613+0000 mon.a (mon.0) 442 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:24:24.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:24 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:24.432372+0000 mon.a (mon.0) 443 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:24:24.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:24 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:24.442698+0000 mon.a (mon.0) 444 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:24:24.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:24 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:24.443868+0000 mon.a (mon.0) 445 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:24:24.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:24 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:24.444682+0000 mon.a (mon.0) 446 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:24:24.813 DEBUG:teuthology.orchestra.run.smithi153:osd.6> sudo journalctl -f -n 0 -u ceph-69111522-3fb6-11ed-8431-001a4aab830c@osd.6.service 2022-09-29T05:24:24.855 INFO:tasks.cephadm:Deploying osd.7 on smithi153 with /dev/vg_nvme/lv_1... 2022-09-29T05:24:24.856 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/daemon-base:latest-pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 69111522-3fb6-11ed-8431-001a4aab830c -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2022-09-29T05:24:24.886 INFO:journalctl@ceph.osd.6.smithi153.stdout:-- Logs begin at Thu 2022-09-29 05:12:53 UTC. -- 2022-09-29T05:24:24.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:24:23.583915+0000 mgr.y (mgr.14146) 130 : cluster [DBG] pgmap v111: 1 pgs: 1 active+clean; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2022-09-29T05:24:24.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:24.427613+0000 mon.a (mon.0) 442 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:24:24.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:24.432372+0000 mon.a (mon.0) 443 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:24:24.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:24.442698+0000 mon.a (mon.0) 444 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:24:24.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:24.443868+0000 mon.a (mon.0) 445 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:24:24.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:24.444682+0000 mon.a (mon.0) 446 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:24:24.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:24:23.583915+0000 mgr.y (mgr.14146) 130 : cluster [DBG] pgmap v111: 1 pgs: 1 active+clean; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2022-09-29T05:24:24.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:24.427613+0000 mon.a (mon.0) 442 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:24:24.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:24.432372+0000 mon.a (mon.0) 443 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:24:24.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:24.442698+0000 mon.a (mon.0) 444 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:24:24.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:24.443868+0000 mon.a (mon.0) 445 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:24:24.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:24.444682+0000 mon.a (mon.0) 446 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:24:26.713 INFO:teuthology.orchestra.run.smithi153.stderr:--> Zapping: /dev/vg_nvme/lv_1 2022-09-29T05:24:26.714 INFO:teuthology.orchestra.run.smithi153.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2022-09-29T05:24:26.714 INFO:teuthology.orchestra.run.smithi153.stderr: stderr: 10+0 records in 2022-09-29T05:24:26.714 INFO:teuthology.orchestra.run.smithi153.stderr:10+0 records out 2022-09-29T05:24:26.714 INFO:teuthology.orchestra.run.smithi153.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0125641 s, 835 MB/s 2022-09-29T05:24:26.715 INFO:teuthology.orchestra.run.smithi153.stderr:--> Zapping successful for: 2022-09-29T05:24:27.005 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:26 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:24:25.584333+0000 mgr.y (mgr.14146) 131 : cluster [DBG] pgmap v112: 1 pgs: 1 active+clean; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2022-09-29T05:24:27.005 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:26 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:26.462713+0000 mon.a (mon.0) 447 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:24:27.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:24:25.584333+0000 mgr.y (mgr.14146) 131 : cluster [DBG] pgmap v112: 1 pgs: 1 active+clean; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2022-09-29T05:24:27.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:26.462713+0000 mon.a (mon.0) 447 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:24:27.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:24:25.584333+0000 mgr.y (mgr.14146) 131 : cluster [DBG] pgmap v112: 1 pgs: 1 active+clean; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2022-09-29T05:24:27.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:26.462713+0000 mon.a (mon.0) 447 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:24:27.309 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/daemon-base:latest-pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 69111522-3fb6-11ed-8431-001a4aab830c -- ceph orch daemon add osd smithi153:vg_nvme/lv_1 2022-09-29T05:24:27.823 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:27 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:27.609900+0000 mon.b (mon.2) 16 : audit [INF] from='osd.6 [v2:172.21.15.153:6816/3662388207,v1:172.21.15.153:6817/3662388207]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2022-09-29T05:24:27.823 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:27 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:27.610462+0000 mon.a (mon.0) 448 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2022-09-29T05:24:27.824 INFO:journalctl@ceph.osd.6.smithi153.stdout:Sep 29 05:24:27 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-osd-6[48611]: debug 2022-09-29T05:24:27.606+0000 7f7b5f7a6200 -1 osd.6 0 log_to_monitors {default=true} 2022-09-29T05:24:28.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:27 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:27.609900+0000 mon.b (mon.2) 16 : audit [INF] from='osd.6 [v2:172.21.15.153:6816/3662388207,v1:172.21.15.153:6817/3662388207]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2022-09-29T05:24:28.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:27 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:27.610462+0000 mon.a (mon.0) 448 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2022-09-29T05:24:28.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:27 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:27.609900+0000 mon.b (mon.2) 16 : audit [INF] from='osd.6 [v2:172.21.15.153:6816/3662388207,v1:172.21.15.153:6817/3662388207]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2022-09-29T05:24:28.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:27 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:27.610462+0000 mon.a (mon.0) 448 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2022-09-29T05:24:29.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:28 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:24:27.584690+0000 mgr.y (mgr.14146) 132 : cluster [DBG] pgmap v113: 1 pgs: 1 active+clean; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2022-09-29T05:24:29.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:28 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:27.822330+0000 mon.a (mon.0) 449 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2022-09-29T05:24:29.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:28 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:24:27.822389+0000 mon.a (mon.0) 450 : cluster [DBG] osdmap e38: 7 total, 6 up, 7 in 2022-09-29T05:24:29.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:28 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:27.822559+0000 mon.a (mon.0) 451 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2022-09-29T05:24:29.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:28 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:27.822762+0000 mon.b (mon.2) 17 : audit [INF] from='osd.6 [v2:172.21.15.153:6816/3662388207,v1:172.21.15.153:6817/3662388207]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2022-09-29T05:24:29.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:28 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:27.823177+0000 mon.a (mon.0) 452 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2022-09-29T05:24:29.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:28 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:24:27.584690+0000 mgr.y (mgr.14146) 132 : cluster [DBG] pgmap v113: 1 pgs: 1 active+clean; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2022-09-29T05:24:29.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:28 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:27.822330+0000 mon.a (mon.0) 449 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2022-09-29T05:24:29.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:28 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:24:27.822389+0000 mon.a (mon.0) 450 : cluster [DBG] osdmap e38: 7 total, 6 up, 7 in 2022-09-29T05:24:29.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:28 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:27.822559+0000 mon.a (mon.0) 451 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2022-09-29T05:24:29.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:28 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:27.822762+0000 mon.b (mon.2) 17 : audit [INF] from='osd.6 [v2:172.21.15.153:6816/3662388207,v1:172.21.15.153:6817/3662388207]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2022-09-29T05:24:29.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:28 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:27.823177+0000 mon.a (mon.0) 452 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2022-09-29T05:24:29.309 INFO:journalctl@ceph.osd.6.smithi153.stdout:Sep 29 05:24:28 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-osd-6[48611]: debug 2022-09-29T05:24:28.828+0000 7f7b55d2c700 -1 osd.6 0 waiting for initial osdmap 2022-09-29T05:24:29.309 INFO:journalctl@ceph.osd.6.smithi153.stdout:Sep 29 05:24:28 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-osd-6[48611]: debug 2022-09-29T05:24:28.834+0000 7f7b4e4a0700 -1 osd.6 39 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2022-09-29T05:24:29.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:28 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:24:27.584690+0000 mgr.y (mgr.14146) 132 : cluster [DBG] pgmap v113: 1 pgs: 1 active+clean; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2022-09-29T05:24:29.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:28 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:27.822330+0000 mon.a (mon.0) 449 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2022-09-29T05:24:29.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:28 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:24:27.822389+0000 mon.a (mon.0) 450 : cluster [DBG] osdmap e38: 7 total, 6 up, 7 in 2022-09-29T05:24:29.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:28 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:27.822559+0000 mon.a (mon.0) 451 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2022-09-29T05:24:29.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:28 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:27.822762+0000 mon.b (mon.2) 17 : audit [INF] from='osd.6 [v2:172.21.15.153:6816/3662388207,v1:172.21.15.153:6817/3662388207]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2022-09-29T05:24:29.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:28 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:27.823177+0000 mon.a (mon.0) 452 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2022-09-29T05:24:29.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:29 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:28.823507+0000 mon.a (mon.0) 453 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]': finished 2022-09-29T05:24:29.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:29 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:24:28.823645+0000 mon.a (mon.0) 454 : cluster [DBG] osdmap e39: 7 total, 6 up, 7 in 2022-09-29T05:24:29.928 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:29 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:28.824143+0000 mon.a (mon.0) 455 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2022-09-29T05:24:29.928 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:29 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:28.829100+0000 mon.a (mon.0) 456 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2022-09-29T05:24:29.928 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:29 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:29.204670+0000 mon.a (mon.0) 457 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2022-09-29T05:24:29.928 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:29 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:29.207064+0000 mon.a (mon.0) 458 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2022-09-29T05:24:29.929 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:29 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:29.207795+0000 mon.a (mon.0) 459 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:24:29.929 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:29 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:24:29.811980+0000 mon.a (mon.0) 460 : cluster [INF] osd.6 [v2:172.21.15.153:6816/3662388207,v1:172.21.15.153:6817/3662388207] boot 2022-09-29T05:24:29.929 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:29 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:24:29.812054+0000 mon.a (mon.0) 461 : cluster [DBG] osdmap e40: 7 total, 7 up, 7 in 2022-09-29T05:24:29.929 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:29 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:29.812471+0000 mon.a (mon.0) 462 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2022-09-29T05:24:30.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:29 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:28.823507+0000 mon.a (mon.0) 453 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]': finished 2022-09-29T05:24:30.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:29 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:24:28.823645+0000 mon.a (mon.0) 454 : cluster [DBG] osdmap e39: 7 total, 6 up, 7 in 2022-09-29T05:24:30.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:29 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:28.824143+0000 mon.a (mon.0) 455 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2022-09-29T05:24:30.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:29 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:28.829100+0000 mon.a (mon.0) 456 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2022-09-29T05:24:30.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:29 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:29.204670+0000 mon.a (mon.0) 457 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2022-09-29T05:24:30.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:29 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:29.207064+0000 mon.a (mon.0) 458 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2022-09-29T05:24:30.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:29 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:29.207795+0000 mon.a (mon.0) 459 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:24:30.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:29 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:24:29.811980+0000 mon.a (mon.0) 460 : cluster [INF] osd.6 [v2:172.21.15.153:6816/3662388207,v1:172.21.15.153:6817/3662388207] boot 2022-09-29T05:24:30.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:29 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:24:29.812054+0000 mon.a (mon.0) 461 : cluster [DBG] osdmap e40: 7 total, 7 up, 7 in 2022-09-29T05:24:30.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:29 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:29.812471+0000 mon.a (mon.0) 462 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2022-09-29T05:24:30.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:29 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:28.823507+0000 mon.a (mon.0) 453 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]': finished 2022-09-29T05:24:30.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:29 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:24:28.823645+0000 mon.a (mon.0) 454 : cluster [DBG] osdmap e39: 7 total, 6 up, 7 in 2022-09-29T05:24:30.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:29 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:28.824143+0000 mon.a (mon.0) 455 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2022-09-29T05:24:30.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:29 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:28.829100+0000 mon.a (mon.0) 456 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2022-09-29T05:24:30.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:29 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:29.204670+0000 mon.a (mon.0) 457 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2022-09-29T05:24:30.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:29 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:29.207064+0000 mon.a (mon.0) 458 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2022-09-29T05:24:30.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:29 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:29.207795+0000 mon.a (mon.0) 459 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:24:30.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:29 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:24:29.811980+0000 mon.a (mon.0) 460 : cluster [INF] osd.6 [v2:172.21.15.153:6816/3662388207,v1:172.21.15.153:6817/3662388207] boot 2022-09-29T05:24:30.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:29 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:24:29.812054+0000 mon.a (mon.0) 461 : cluster [DBG] osdmap e40: 7 total, 7 up, 7 in 2022-09-29T05:24:30.218 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:29 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:29.812471+0000 mon.a (mon.0) 462 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2022-09-29T05:24:31.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:30 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:24:28.640996+0000 osd.6 (osd.6) 1 : cluster [DBG] purged_snaps scrub starts 2022-09-29T05:24:31.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:30 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:24:28.641101+0000 osd.6 (osd.6) 2 : cluster [DBG] purged_snaps scrub ok 2022-09-29T05:24:31.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:30 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:29.202330+0000 mgr.y (mgr.14146) 133 : audit [DBG] from='client.24293 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi153:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:24:31.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:30 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:24:29.585147+0000 mgr.y (mgr.14146) 134 : cluster [DBG] pgmap v116: 1 pgs: 1 active+clean; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2022-09-29T05:24:31.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:30 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:30.084327+0000 mon.a (mon.0) 463 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:24:31.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:30 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:30.085338+0000 mon.a (mon.0) 464 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:24:31.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:30 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:30.086157+0000 mon.a (mon.0) 465 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:24:31.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:30 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:30.086883+0000 mon.a (mon.0) 466 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:24:31.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:30 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:30.091793+0000 mon.a (mon.0) 467 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:24:31.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:30 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:30.096279+0000 mon.a (mon.0) 468 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:24:31.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:30 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:24:30.819958+0000 mon.a (mon.0) 469 : cluster [DBG] osdmap e41: 7 total, 7 up, 7 in 2022-09-29T05:24:31.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:24:28.640996+0000 osd.6 (osd.6) 1 : cluster [DBG] purged_snaps scrub starts 2022-09-29T05:24:31.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:24:28.641101+0000 osd.6 (osd.6) 2 : cluster [DBG] purged_snaps scrub ok 2022-09-29T05:24:31.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:29.202330+0000 mgr.y (mgr.14146) 133 : audit [DBG] from='client.24293 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi153:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:24:31.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:24:29.585147+0000 mgr.y (mgr.14146) 134 : cluster [DBG] pgmap v116: 1 pgs: 1 active+clean; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2022-09-29T05:24:31.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:30.084327+0000 mon.a (mon.0) 463 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:24:31.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:30.085338+0000 mon.a (mon.0) 464 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:24:31.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:30.086157+0000 mon.a (mon.0) 465 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:24:31.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:30.086883+0000 mon.a (mon.0) 466 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:24:31.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:30.091793+0000 mon.a (mon.0) 467 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:24:31.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:30.096279+0000 mon.a (mon.0) 468 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:24:31.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:24:30.819958+0000 mon.a (mon.0) 469 : cluster [DBG] osdmap e41: 7 total, 7 up, 7 in 2022-09-29T05:24:31.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:24:28.640996+0000 osd.6 (osd.6) 1 : cluster [DBG] purged_snaps scrub starts 2022-09-29T05:24:31.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:24:28.641101+0000 osd.6 (osd.6) 2 : cluster [DBG] purged_snaps scrub ok 2022-09-29T05:24:31.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:29.202330+0000 mgr.y (mgr.14146) 133 : audit [DBG] from='client.24293 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi153:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:24:31.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:24:29.585147+0000 mgr.y (mgr.14146) 134 : cluster [DBG] pgmap v116: 1 pgs: 1 active+clean; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2022-09-29T05:24:31.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:30.084327+0000 mon.a (mon.0) 463 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:24:31.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:30.085338+0000 mon.a (mon.0) 464 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:24:31.218 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:30.086157+0000 mon.a (mon.0) 465 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:24:31.218 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:30.086883+0000 mon.a (mon.0) 466 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:24:31.218 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:30.091793+0000 mon.a (mon.0) 467 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:24:31.218 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:30.096279+0000 mon.a (mon.0) 468 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:24:31.219 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:24:30.819958+0000 mon.a (mon.0) 469 : cluster [DBG] osdmap e41: 7 total, 7 up, 7 in 2022-09-29T05:24:32.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:24:30.079539+0000 mgr.y (mgr.14146) 135 : cephadm [INF] Detected new or changed devices on smithi153 2022-09-29T05:24:32.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:24:30.087450+0000 mgr.y (mgr.14146) 136 : cephadm [INF] Adjusting osd_memory_target on smithi153 to 5722M 2022-09-29T05:24:32.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:31.375210+0000 mon.b (mon.2) 18 : audit [INF] from='client.? 172.21.15.153:0/869208304' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b5fdd156-d2dc-47fb-b44b-803275092ac0"}]: dispatch 2022-09-29T05:24:32.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:31.375830+0000 mon.a (mon.0) 470 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b5fdd156-d2dc-47fb-b44b-803275092ac0"}]: dispatch 2022-09-29T05:24:32.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:31.383020+0000 mon.a (mon.0) 471 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b5fdd156-d2dc-47fb-b44b-803275092ac0"}]': finished 2022-09-29T05:24:32.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:24:31.383154+0000 mon.a (mon.0) 472 : cluster [DBG] osdmap e42: 8 total, 7 up, 8 in 2022-09-29T05:24:32.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:31.383303+0000 mon.a (mon.0) 473 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2022-09-29T05:24:32.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:24:30.079539+0000 mgr.y (mgr.14146) 135 : cephadm [INF] Detected new or changed devices on smithi153 2022-09-29T05:24:32.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:24:30.087450+0000 mgr.y (mgr.14146) 136 : cephadm [INF] Adjusting osd_memory_target on smithi153 to 5722M 2022-09-29T05:24:32.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:31.375210+0000 mon.b (mon.2) 18 : audit [INF] from='client.? 172.21.15.153:0/869208304' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b5fdd156-d2dc-47fb-b44b-803275092ac0"}]: dispatch 2022-09-29T05:24:32.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:31.375830+0000 mon.a (mon.0) 470 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b5fdd156-d2dc-47fb-b44b-803275092ac0"}]: dispatch 2022-09-29T05:24:32.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:31.383020+0000 mon.a (mon.0) 471 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b5fdd156-d2dc-47fb-b44b-803275092ac0"}]': finished 2022-09-29T05:24:32.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:24:31.383154+0000 mon.a (mon.0) 472 : cluster [DBG] osdmap e42: 8 total, 7 up, 8 in 2022-09-29T05:24:32.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:31.383303+0000 mon.a (mon.0) 473 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2022-09-29T05:24:32.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:31 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:24:30.079539+0000 mgr.y (mgr.14146) 135 : cephadm [INF] Detected new or changed devices on smithi153 2022-09-29T05:24:32.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:31 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:24:30.087450+0000 mgr.y (mgr.14146) 136 : cephadm [INF] Adjusting osd_memory_target on smithi153 to 5722M 2022-09-29T05:24:32.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:31 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:31.375210+0000 mon.b (mon.2) 18 : audit [INF] from='client.? 172.21.15.153:0/869208304' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b5fdd156-d2dc-47fb-b44b-803275092ac0"}]: dispatch 2022-09-29T05:24:32.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:31 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:31.375830+0000 mon.a (mon.0) 470 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b5fdd156-d2dc-47fb-b44b-803275092ac0"}]: dispatch 2022-09-29T05:24:32.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:31 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:31.383020+0000 mon.a (mon.0) 471 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b5fdd156-d2dc-47fb-b44b-803275092ac0"}]': finished 2022-09-29T05:24:32.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:31 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:24:31.383154+0000 mon.a (mon.0) 472 : cluster [DBG] osdmap e42: 8 total, 7 up, 8 in 2022-09-29T05:24:32.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:31 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:31.383303+0000 mon.a (mon.0) 473 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2022-09-29T05:24:33.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:24:31.585481+0000 mgr.y (mgr.14146) 137 : cluster [DBG] pgmap v120: 1 pgs: 1 remapped+peering; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2022-09-29T05:24:33.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:31.971418+0000 mon.b (mon.2) 19 : audit [DBG] from='client.? 172.21.15.153:0/1790299156' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2022-09-29T05:24:33.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:24:31.585481+0000 mgr.y (mgr.14146) 137 : cluster [DBG] pgmap v120: 1 pgs: 1 remapped+peering; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2022-09-29T05:24:33.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:31.971418+0000 mon.b (mon.2) 19 : audit [DBG] from='client.? 172.21.15.153:0/1790299156' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2022-09-29T05:24:33.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:24:31.585481+0000 mgr.y (mgr.14146) 137 : cluster [DBG] pgmap v120: 1 pgs: 1 remapped+peering; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2022-09-29T05:24:33.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:31.971418+0000 mon.b (mon.2) 19 : audit [DBG] from='client.? 172.21.15.153:0/1790299156' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2022-09-29T05:24:35.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:24:33.585810+0000 mgr.y (mgr.14146) 138 : cluster [DBG] pgmap v121: 1 pgs: 1 remapped+peering; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2022-09-29T05:24:35.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:24:33.585810+0000 mgr.y (mgr.14146) 138 : cluster [DBG] pgmap v121: 1 pgs: 1 remapped+peering; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2022-09-29T05:24:35.272 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:34 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:24:33.585810+0000 mgr.y (mgr.14146) 138 : cluster [DBG] pgmap v121: 1 pgs: 1 remapped+peering; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2022-09-29T05:24:36.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:35.691813+0000 mon.a (mon.0) 474 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:24:36.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:35.693002+0000 mon.a (mon.0) 475 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:24:36.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:35.691813+0000 mon.a (mon.0) 474 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:24:36.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:35.693002+0000 mon.a (mon.0) 475 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:24:36.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:35 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:35.691813+0000 mon.a (mon.0) 474 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:24:36.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:35 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:35.693002+0000 mon.a (mon.0) 475 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:24:37.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:24:35.586312+0000 mgr.y (mgr.14146) 139 : cluster [DBG] pgmap v122: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2022-09-29T05:24:37.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:24:35.586312+0000 mgr.y (mgr.14146) 139 : cluster [DBG] pgmap v122: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2022-09-29T05:24:37.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:36 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:24:35.586312+0000 mgr.y (mgr.14146) 139 : cluster [DBG] pgmap v122: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2022-09-29T05:24:38.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:37 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:36.886003+0000 mon.a (mon.0) 476 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2022-09-29T05:24:38.109 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:37 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:36.886842+0000 mon.a (mon.0) 477 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:24:38.110 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:37 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:24:36.887486+0000 mgr.y (mgr.14146) 140 : cephadm [INF] Deploying daemon osd.7 on smithi153 2022-09-29T05:24:38.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:37 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:36.886003+0000 mon.a (mon.0) 476 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2022-09-29T05:24:38.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:37 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:36.886842+0000 mon.a (mon.0) 477 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:24:38.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:37 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:24:36.887486+0000 mgr.y (mgr.14146) 140 : cephadm [INF] Deploying daemon osd.7 on smithi153 2022-09-29T05:24:38.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:37 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:36.886003+0000 mon.a (mon.0) 476 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2022-09-29T05:24:38.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:37 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:36.886842+0000 mon.a (mon.0) 477 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:24:38.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:37 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:24:36.887486+0000 mgr.y (mgr.14146) 140 : cephadm [INF] Deploying daemon osd.7 on smithi153 2022-09-29T05:24:39.151 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:38 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:24:37.586673+0000 mgr.y (mgr.14146) 141 : cluster [DBG] pgmap v123: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2022-09-29T05:24:39.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:38 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:24:37.586673+0000 mgr.y (mgr.14146) 141 : cluster [DBG] pgmap v123: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2022-09-29T05:24:39.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:38 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:24:37.586673+0000 mgr.y (mgr.14146) 141 : cluster [DBG] pgmap v123: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2022-09-29T05:24:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:24:39.587172+0000 mgr.y (mgr.14146) 142 : cluster [DBG] pgmap v124: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2022-09-29T05:24:41.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:24:39.587172+0000 mgr.y (mgr.14146) 142 : cluster [DBG] pgmap v124: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2022-09-29T05:24:41.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:40 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:24:39.587172+0000 mgr.y (mgr.14146) 142 : cluster [DBG] pgmap v124: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2022-09-29T05:24:41.618 INFO:teuthology.orchestra.run.smithi153.stdout:Created osd(s) 7 on host 'smithi153' 2022-09-29T05:24:42.003 DEBUG:teuthology.orchestra.run.smithi153:osd.7> sudo journalctl -f -n 0 -u ceph-69111522-3fb6-11ed-8431-001a4aab830c@osd.7.service 2022-09-29T05:24:42.006 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2022-09-29T05:24:42.007 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/daemon-base:latest-pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 69111522-3fb6-11ed-8431-001a4aab830c -- ceph osd stat -f json 2022-09-29T05:24:42.038 INFO:journalctl@ceph.osd.7.smithi153.stdout:-- Logs begin at Thu 2022-09-29 05:12:53 UTC. -- 2022-09-29T05:24:42.309 INFO:journalctl@ceph.osd.7.smithi153.stdout:Sep 29 05:24:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-osd-7[51632]: debug 2022-09-29T05:24:42.047+0000 7f3e694dc200 -1 Falling back to public interface 2022-09-29T05:24:42.865 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:24:41.587544+0000 mgr.y (mgr.14146) 143 : cluster [DBG] pgmap v125: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2022-09-29T05:24:42.865 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:41.608626+0000 mon.a (mon.0) 478 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:24:42.865 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:41.613772+0000 mon.a (mon.0) 479 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:24:42.866 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:41.639853+0000 mon.a (mon.0) 480 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:24:42.866 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:41.641122+0000 mon.a (mon.0) 481 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:24:42.866 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:41.641917+0000 mon.a (mon.0) 482 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:24:42.867 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:24:41.587544+0000 mgr.y (mgr.14146) 143 : cluster [DBG] pgmap v125: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2022-09-29T05:24:42.867 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:41.608626+0000 mon.a (mon.0) 478 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:24:42.867 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:41.613772+0000 mon.a (mon.0) 479 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:24:42.867 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:41.639853+0000 mon.a (mon.0) 480 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:24:42.868 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:41.641122+0000 mon.a (mon.0) 481 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:24:42.868 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:41.641917+0000 mon.a (mon.0) 482 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:24:43.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:24:41.587544+0000 mgr.y (mgr.14146) 143 : cluster [DBG] pgmap v125: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2022-09-29T05:24:43.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:41.608626+0000 mon.a (mon.0) 478 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:24:43.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:41.613772+0000 mon.a (mon.0) 479 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:24:43.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:41.639853+0000 mon.a (mon.0) 480 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:24:43.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:41.641122+0000 mon.a (mon.0) 481 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:24:43.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:41.641917+0000 mon.a (mon.0) 482 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:24:43.743 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:24:44.237 INFO:teuthology.orchestra.run.smithi083.stdout:{"epoch":42,"num_osds":8,"num_up_osds":7,"osd_up_since":1664429069,"num_in_osds":8,"osd_in_since":1664429071,"num_remapped_pgs":0} 2022-09-29T05:24:44.809 INFO:journalctl@ceph.osd.7.smithi153.stdout:Sep 29 05:24:44 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-osd-7[51632]: debug 2022-09-29T05:24:44.509+0000 7f3e694dc200 -1 osd.7 0 log_to_monitors {default=true} 2022-09-29T05:24:44.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:44 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:43.519379+0000 mon.a (mon.0) 483 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:24:44.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:44 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:24:43.587866+0000 mgr.y (mgr.14146) 144 : cluster [DBG] pgmap v126: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2022-09-29T05:24:44.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:44 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:43.740163+0000 mon.a (mon.0) 484 : audit [DBG] from='client.? 172.21.15.83:0/2252917532' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2022-09-29T05:24:44.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:44 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:44.512812+0000 mon.b (mon.2) 20 : audit [INF] from='osd.7 [v2:172.21.15.153:6824/1223012606,v1:172.21.15.153:6825/1223012606]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2022-09-29T05:24:44.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:44 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:44.513282+0000 mon.a (mon.0) 485 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2022-09-29T05:24:44.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:43.519379+0000 mon.a (mon.0) 483 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:24:44.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:24:43.587866+0000 mgr.y (mgr.14146) 144 : cluster [DBG] pgmap v126: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2022-09-29T05:24:44.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:43.740163+0000 mon.a (mon.0) 484 : audit [DBG] from='client.? 172.21.15.83:0/2252917532' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2022-09-29T05:24:44.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:44.512812+0000 mon.b (mon.2) 20 : audit [INF] from='osd.7 [v2:172.21.15.153:6824/1223012606,v1:172.21.15.153:6825/1223012606]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2022-09-29T05:24:44.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:44.513282+0000 mon.a (mon.0) 485 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2022-09-29T05:24:44.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:43.519379+0000 mon.a (mon.0) 483 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:24:44.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:24:43.587866+0000 mgr.y (mgr.14146) 144 : cluster [DBG] pgmap v126: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2022-09-29T05:24:44.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:43.740163+0000 mon.a (mon.0) 484 : audit [DBG] from='client.? 172.21.15.83:0/2252917532' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2022-09-29T05:24:44.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:44.512812+0000 mon.b (mon.2) 20 : audit [INF] from='osd.7 [v2:172.21.15.153:6824/1223012606,v1:172.21.15.153:6825/1223012606]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2022-09-29T05:24:44.968 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:44.513282+0000 mon.a (mon.0) 485 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2022-09-29T05:24:45.239 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/daemon-base:latest-pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 69111522-3fb6-11ed-8431-001a4aab830c -- ceph osd stat -f json 2022-09-29T05:24:45.531 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:44.524302+0000 mon.a (mon.0) 486 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2022-09-29T05:24:45.532 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:24:44.524376+0000 mon.a (mon.0) 487 : cluster [DBG] osdmap e43: 8 total, 7 up, 8 in 2022-09-29T05:24:45.532 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:44.524626+0000 mon.a (mon.0) 488 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2022-09-29T05:24:45.532 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:44.524726+0000 mon.b (mon.2) 21 : audit [INF] from='osd.7 [v2:172.21.15.153:6824/1223012606,v1:172.21.15.153:6825/1223012606]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2022-09-29T05:24:45.533 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:44.525108+0000 mon.a (mon.0) 489 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2022-09-29T05:24:45.796 INFO:journalctl@ceph.osd.7.smithi153.stdout:Sep 29 05:24:45 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-osd-7[51632]: debug 2022-09-29T05:24:45.532+0000 7f3e5fa62700 -1 osd.7 0 waiting for initial osdmap 2022-09-29T05:24:45.796 INFO:journalctl@ceph.osd.7.smithi153.stdout:Sep 29 05:24:45 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-osd-7[51632]: debug 2022-09-29T05:24:45.538+0000 7f3e5b1dc700 -1 osd.7 44 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2022-09-29T05:24:45.797 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:45 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:44.524302+0000 mon.a (mon.0) 486 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2022-09-29T05:24:45.797 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:45 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:24:44.524376+0000 mon.a (mon.0) 487 : cluster [DBG] osdmap e43: 8 total, 7 up, 8 in 2022-09-29T05:24:45.797 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:45 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:44.524626+0000 mon.a (mon.0) 488 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2022-09-29T05:24:45.797 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:45 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:44.524726+0000 mon.b (mon.2) 21 : audit [INF] from='osd.7 [v2:172.21.15.153:6824/1223012606,v1:172.21.15.153:6825/1223012606]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2022-09-29T05:24:45.798 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:45 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:44.525108+0000 mon.a (mon.0) 489 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2022-09-29T05:24:45.798 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:44.524302+0000 mon.a (mon.0) 486 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2022-09-29T05:24:45.798 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:24:44.524376+0000 mon.a (mon.0) 487 : cluster [DBG] osdmap e43: 8 total, 7 up, 8 in 2022-09-29T05:24:45.799 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:44.524626+0000 mon.a (mon.0) 488 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2022-09-29T05:24:45.799 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:44.524726+0000 mon.b (mon.2) 21 : audit [INF] from='osd.7 [v2:172.21.15.153:6824/1223012606,v1:172.21.15.153:6825/1223012606]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2022-09-29T05:24:45.799 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:44.525108+0000 mon.a (mon.0) 489 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2022-09-29T05:24:46.648 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:45.525768+0000 mon.a (mon.0) 490 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]': finished 2022-09-29T05:24:46.649 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:24:45.525844+0000 mon.a (mon.0) 491 : cluster [DBG] osdmap e44: 8 total, 7 up, 8 in 2022-09-29T05:24:46.649 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:45.526175+0000 mon.a (mon.0) 492 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2022-09-29T05:24:46.649 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:45.530428+0000 mon.a (mon.0) 493 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2022-09-29T05:24:46.649 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:24:45.588184+0000 mgr.y (mgr.14146) 145 : cluster [DBG] pgmap v129: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2022-09-29T05:24:46.650 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:46.378132+0000 mon.a (mon.0) 494 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:24:46.650 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:46.379043+0000 mon.a (mon.0) 495 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:24:46.650 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:46.379665+0000 mon.a (mon.0) 496 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:24:46.650 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:46.380209+0000 mon.a (mon.0) 497 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:24:46.651 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:46.380746+0000 mon.a (mon.0) 498 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:24:46.651 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:46.384958+0000 mon.a (mon.0) 499 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:24:46.651 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:46.388978+0000 mon.a (mon.0) 500 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:24:46.651 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:45.525768+0000 mon.a (mon.0) 490 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]': finished 2022-09-29T05:24:46.652 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:24:45.525844+0000 mon.a (mon.0) 491 : cluster [DBG] osdmap e44: 8 total, 7 up, 8 in 2022-09-29T05:24:46.652 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:45.526175+0000 mon.a (mon.0) 492 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2022-09-29T05:24:46.652 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:45.530428+0000 mon.a (mon.0) 493 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2022-09-29T05:24:46.652 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:24:45.588184+0000 mgr.y (mgr.14146) 145 : cluster [DBG] pgmap v129: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2022-09-29T05:24:46.653 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:46.378132+0000 mon.a (mon.0) 494 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:24:46.653 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:46.379043+0000 mon.a (mon.0) 495 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:24:46.653 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:46.379665+0000 mon.a (mon.0) 496 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:24:46.653 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:46.380209+0000 mon.a (mon.0) 497 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:24:46.654 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:46.380746+0000 mon.a (mon.0) 498 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:24:46.654 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:46.384958+0000 mon.a (mon.0) 499 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:24:46.654 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:46.388978+0000 mon.a (mon.0) 500 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:24:46.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:46 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:45.525768+0000 mon.a (mon.0) 490 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]': finished 2022-09-29T05:24:46.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:46 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:24:45.525844+0000 mon.a (mon.0) 491 : cluster [DBG] osdmap e44: 8 total, 7 up, 8 in 2022-09-29T05:24:46.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:46 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:45.526175+0000 mon.a (mon.0) 492 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2022-09-29T05:24:46.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:46 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:45.530428+0000 mon.a (mon.0) 493 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2022-09-29T05:24:46.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:46 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:24:45.588184+0000 mgr.y (mgr.14146) 145 : cluster [DBG] pgmap v129: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2022-09-29T05:24:46.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:46 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:46.378132+0000 mon.a (mon.0) 494 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:24:46.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:46 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:46.379043+0000 mon.a (mon.0) 495 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:24:46.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:46 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:46.379665+0000 mon.a (mon.0) 496 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:24:46.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:46 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:46.380209+0000 mon.a (mon.0) 497 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:24:46.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:46 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:46.380746+0000 mon.a (mon.0) 498 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:24:46.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:46 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:46.384958+0000 mon.a (mon.0) 499 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:24:46.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:46 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:46.388978+0000 mon.a (mon.0) 500 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:24:46.941 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:24:47.462 INFO:teuthology.orchestra.run.smithi083.stdout:{"epoch":45,"num_osds":8,"num_up_osds":8,"osd_up_since":1664429086,"num_in_osds":8,"osd_in_since":1664429071,"num_remapped_pgs":1} 2022-09-29T05:24:47.463 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/daemon-base:latest-pacific shell --fsid 69111522-3fb6-11ed-8431-001a4aab830c -- ceph osd dump --format=json 2022-09-29T05:24:47.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:47 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:24:45.518802+0000 osd.7 (osd.7) 1 : cluster [DBG] purged_snaps scrub starts 2022-09-29T05:24:47.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:47 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:24:45.518892+0000 osd.7 (osd.7) 2 : cluster [DBG] purged_snaps scrub ok 2022-09-29T05:24:47.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:47 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:24:46.373722+0000 mgr.y (mgr.14146) 146 : cephadm [INF] Detected new or changed devices on smithi153 2022-09-29T05:24:47.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:47 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:24:46.381182+0000 mgr.y (mgr.14146) 147 : cephadm [INF] Adjusting osd_memory_target on smithi153 to 4292M 2022-09-29T05:24:47.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:47 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:24:46.533584+0000 mon.a (mon.0) 501 : cluster [INF] osd.7 [v2:172.21.15.153:6824/1223012606,v1:172.21.15.153:6825/1223012606] boot 2022-09-29T05:24:47.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:47 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:24:46.533656+0000 mon.a (mon.0) 502 : cluster [DBG] osdmap e45: 8 total, 8 up, 8 in 2022-09-29T05:24:47.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:47 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:46.534050+0000 mon.a (mon.0) 503 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2022-09-29T05:24:47.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:47 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:46.938404+0000 mon.c (mon.1) 12 : audit [DBG] from='client.? 172.21.15.83:0/1355505508' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2022-09-29T05:24:47.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:47 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:24:45.518802+0000 osd.7 (osd.7) 1 : cluster [DBG] purged_snaps scrub starts 2022-09-29T05:24:47.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:47 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:24:45.518892+0000 osd.7 (osd.7) 2 : cluster [DBG] purged_snaps scrub ok 2022-09-29T05:24:47.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:47 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:24:46.373722+0000 mgr.y (mgr.14146) 146 : cephadm [INF] Detected new or changed devices on smithi153 2022-09-29T05:24:47.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:47 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:24:46.381182+0000 mgr.y (mgr.14146) 147 : cephadm [INF] Adjusting osd_memory_target on smithi153 to 4292M 2022-09-29T05:24:47.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:47 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:24:46.533584+0000 mon.a (mon.0) 501 : cluster [INF] osd.7 [v2:172.21.15.153:6824/1223012606,v1:172.21.15.153:6825/1223012606] boot 2022-09-29T05:24:47.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:47 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:24:46.533656+0000 mon.a (mon.0) 502 : cluster [DBG] osdmap e45: 8 total, 8 up, 8 in 2022-09-29T05:24:47.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:47 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:46.534050+0000 mon.a (mon.0) 503 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2022-09-29T05:24:47.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:47 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:46.938404+0000 mon.c (mon.1) 12 : audit [DBG] from='client.? 172.21.15.83:0/1355505508' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2022-09-29T05:24:47.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:47 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:24:45.518802+0000 osd.7 (osd.7) 1 : cluster [DBG] purged_snaps scrub starts 2022-09-29T05:24:47.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:47 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:24:45.518892+0000 osd.7 (osd.7) 2 : cluster [DBG] purged_snaps scrub ok 2022-09-29T05:24:47.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:47 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:24:46.373722+0000 mgr.y (mgr.14146) 146 : cephadm [INF] Detected new or changed devices on smithi153 2022-09-29T05:24:47.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:47 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:24:46.381182+0000 mgr.y (mgr.14146) 147 : cephadm [INF] Adjusting osd_memory_target on smithi153 to 4292M 2022-09-29T05:24:47.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:47 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:24:46.533584+0000 mon.a (mon.0) 501 : cluster [INF] osd.7 [v2:172.21.15.153:6824/1223012606,v1:172.21.15.153:6825/1223012606] boot 2022-09-29T05:24:47.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:47 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:24:46.533656+0000 mon.a (mon.0) 502 : cluster [DBG] osdmap e45: 8 total, 8 up, 8 in 2022-09-29T05:24:47.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:47 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:46.534050+0000 mon.a (mon.0) 503 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2022-09-29T05:24:47.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:47 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:46.938404+0000 mon.c (mon.1) 12 : audit [DBG] from='client.? 172.21.15.83:0/1355505508' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2022-09-29T05:24:48.046 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/69111522-3fb6-11ed-8431-001a4aab830c/mon.a/config 2022-09-29T05:24:48.549 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:48 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:24:47.543312+0000 mon.a (mon.0) 504 : cluster [DBG] osdmap e46: 8 total, 8 up, 8 in 2022-09-29T05:24:48.549 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:48 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:24:47.588481+0000 mgr.y (mgr.14146) 148 : cluster [DBG] pgmap v132: 1 pgs: 1 active+clean; 0 B data, 41 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:24:48.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:48 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:24:47.543312+0000 mon.a (mon.0) 504 : cluster [DBG] osdmap e46: 8 total, 8 up, 8 in 2022-09-29T05:24:48.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:48 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:24:47.588481+0000 mgr.y (mgr.14146) 148 : cluster [DBG] pgmap v132: 1 pgs: 1 active+clean; 0 B data, 41 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:24:48.917 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:48 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:24:47.543312+0000 mon.a (mon.0) 504 : cluster [DBG] osdmap e46: 8 total, 8 up, 8 in 2022-09-29T05:24:48.917 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:48 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:24:47.588481+0000 mgr.y (mgr.14146) 148 : cluster [DBG] pgmap v132: 1 pgs: 1 active+clean; 0 B data, 41 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:24:49.492 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:24:49.493 INFO:teuthology.orchestra.run.smithi083.stdout:{"epoch":47,"fsid":"69111522-3fb6-11ed-8431-001a4aab830c","created":"2022-09-29T05:20:45.037726+0000","modified":"2022-09-29T05:24:48.540721+0000","last_up_change":"2022-09-29T05:24:46.528124+0000","last_in_change":"2022-09-29T05:24:31.376431+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2022-09-29T05:23:34.795850+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"23","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}}],"osds":[{"osd":0,"uuid":"9b8b79ad-3d00-4dad-8da7-7cff0e65d302","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":45,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6802","nonce":2080129456},{"type":"v1","addr":"172.21.15.83:6803","nonce":2080129456}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6804","nonce":2080129456},{"type":"v1","addr":"172.21.15.83:6805","nonce":2080129456}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6808","nonce":2080129456},{"type":"v1","addr":"172.21.15.83:6809","nonce":2080129456}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6806","nonce":2080129456},{"type":"v1","addr":"172.21.15.83:6807","nonce":2080129456}]},"public_addr":"172.21.15.83:6803/2080129456","cluster_addr":"172.21.15.83:6805/2080129456","heartbeat_back_addr":"172.21.15.83:6809/2080129456","heartbeat_front_addr":"172.21.15.83:6807/2080129456","state":["exists","up"]},{"osd":1,"uuid":"e098cff5-70cc-437e-8759-e6c7e3d452d2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6810","nonce":605787053},{"type":"v1","addr":"172.21.15.83:6811","nonce":605787053}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6812","nonce":605787053},{"type":"v1","addr":"172.21.15.83:6813","nonce":605787053}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6816","nonce":605787053},{"type":"v1","addr":"172.21.15.83:6817","nonce":605787053}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6814","nonce":605787053},{"type":"v1","addr":"172.21.15.83:6815","nonce":605787053}]},"public_addr":"172.21.15.83:6811/605787053","cluster_addr":"172.21.15.83:6813/605787053","heartbeat_back_addr":"172.21.15.83:6817/605787053","heartbeat_front_addr":"172.21.15.83:6815/605787053","state":["exists","up"]},{"osd":2,"uuid":"2c14853b-5b64-496d-a011-5d8a9b17b731","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6818","nonce":1928326991},{"type":"v1","addr":"172.21.15.83:6819","nonce":1928326991}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6820","nonce":1928326991},{"type":"v1","addr":"172.21.15.83:6821","nonce":1928326991}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6824","nonce":1928326991},{"type":"v1","addr":"172.21.15.83:6825","nonce":1928326991}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6822","nonce":1928326991},{"type":"v1","addr":"172.21.15.83:6823","nonce":1928326991}]},"public_addr":"172.21.15.83:6819/1928326991","cluster_addr":"172.21.15.83:6821/1928326991","heartbeat_back_addr":"172.21.15.83:6825/1928326991","heartbeat_front_addr":"172.21.15.83:6823/1928326991","state":["exists","up"]},{"osd":3,"uuid":"7336f148-261e-468e-8d3c-28f840150f36","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6826","nonce":54976712},{"type":"v1","addr":"172.21.15.83:6827","nonce":54976712}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6828","nonce":54976712},{"type":"v1","addr":"172.21.15.83:6829","nonce":54976712}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6832","nonce":54976712},{"type":"v1","addr":"172.21.15.83:6833","nonce":54976712}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6830","nonce":54976712},{"type":"v1","addr":"172.21.15.83:6831","nonce":54976712}]},"public_addr":"172.21.15.83:6827/54976712","cluster_addr":"172.21.15.83:6829/54976712","heartbeat_back_addr":"172.21.15.83:6833/54976712","heartbeat_front_addr":"172.21.15.83:6831/54976712","state":["exists","up"]},{"osd":4,"uuid":"247d5c19-e8af-4620-a378-699dc2000db1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6800","nonce":4053209633},{"type":"v1","addr":"172.21.15.153:6801","nonce":4053209633}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6802","nonce":4053209633},{"type":"v1","addr":"172.21.15.153:6803","nonce":4053209633}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6806","nonce":4053209633},{"type":"v1","addr":"172.21.15.153:6807","nonce":4053209633}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6804","nonce":4053209633},{"type":"v1","addr":"172.21.15.153:6805","nonce":4053209633}]},"public_addr":"172.21.15.153:6801/4053209633","cluster_addr":"172.21.15.153:6803/4053209633","heartbeat_back_addr":"172.21.15.153:6807/4053209633","heartbeat_front_addr":"172.21.15.153:6805/4053209633","state":["exists","up"]},{"osd":5,"uuid":"c69e2263-5dd8-402d-b901-295c1f4840e2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":35,"up_thru":36,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6808","nonce":2536485968},{"type":"v1","addr":"172.21.15.153:6809","nonce":2536485968}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6810","nonce":2536485968},{"type":"v1","addr":"172.21.15.153:6811","nonce":2536485968}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6814","nonce":2536485968},{"type":"v1","addr":"172.21.15.153:6815","nonce":2536485968}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6812","nonce":2536485968},{"type":"v1","addr":"172.21.15.153:6813","nonce":2536485968}]},"public_addr":"172.21.15.153:6809/2536485968","cluster_addr":"172.21.15.153:6811/2536485968","heartbeat_back_addr":"172.21.15.153:6815/2536485968","heartbeat_front_addr":"172.21.15.153:6813/2536485968","state":["exists","up"]},{"osd":6,"uuid":"72aa13b6-e7b7-449b-9fa8-f284e8f0735a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":40,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6816","nonce":3662388207},{"type":"v1","addr":"172.21.15.153:6817","nonce":3662388207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6818","nonce":3662388207},{"type":"v1","addr":"172.21.15.153:6819","nonce":3662388207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6822","nonce":3662388207},{"type":"v1","addr":"172.21.15.153:6823","nonce":3662388207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6820","nonce":3662388207},{"type":"v1","addr":"172.21.15.153:6821","nonce":3662388207}]},"public_addr":"172.21.15.153:6817/3662388207","cluster_addr":"172.21.15.153:6819/3662388207","heartbeat_back_addr":"172.21.15.153:6823/3662388207","heartbeat_front_addr":"172.21.15.153:6821/3662388207","state":["exists","up"]},{"osd":7,"uuid":"b5fdd156-d2dc-47fb-b44b-803275092ac0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":45,"up_thru":46,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6824","nonce":1223012606},{"type":"v1","addr":"172.21.15.153:6825","nonce":1223012606}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6826","nonce":1223012606},{"type":"v1","addr":"172.21.15.153:6827","nonce":1223012606}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6830","nonce":1223012606},{"type":"v1","addr":"172.21.15.153:6831","nonce":1223012606}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6828","nonce":1223012606},{"type":"v1","addr":"172.21.15.153:6829","nonce":1223012606}]},"public_addr":"172.21.15.153:6825/1223012606","cluster_addr":"172.21.15.153:6827/1223012606","heartbeat_back_addr":"172.21.15.153:6831/1223012606","heartbeat_front_addr":"172.21.15.153:6829/1223012606","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2022-09-29T05:22:43.518222+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2022-09-29T05:23:01.859214+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2022-09-29T05:23:20.325748+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2022-09-29T05:23:38.717068+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2022-09-29T05:23:54.756146+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2022-09-29T05:24:11.670499+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2022-09-29T05:24:28.641104+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2022-09-29T05:24:45.518895+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.83:0/2269625481":"2022-09-30T05:21:35.319810+0000","172.21.15.83:0/2173437316":"2022-09-30T05:21:35.319810+0000","172.21.15.83:6800/2702658771":"2022-09-30T05:21:35.319810+0000","172.21.15.83:0/4005486955":"2022-09-30T05:21:15.866011+0000","172.21.15.83:6800/2006042278":"2022-09-30T05:21:15.866011+0000","172.21.15.83:6801/2006042278":"2022-09-30T05:21:15.866011+0000","172.21.15.83:6801/2702658771":"2022-09-30T05:21:35.319810+0000","172.21.15.83:0/4256024697":"2022-09-30T05:21:15.866011+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2022-09-29T05:24:49.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:49 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:24:48.547318+0000 mon.a (mon.0) 505 : cluster [DBG] osdmap e47: 8 total, 8 up, 8 in 2022-09-29T05:24:49.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:49 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:49.488655+0000 mon.c (mon.1) 13 : audit [DBG] from='client.? 172.21.15.83:0/868080085' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2022-09-29T05:24:49.918 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:24:48.547318+0000 mon.a (mon.0) 505 : cluster [DBG] osdmap e47: 8 total, 8 up, 8 in 2022-09-29T05:24:49.918 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:49.488655+0000 mon.c (mon.1) 13 : audit [DBG] from='client.? 172.21.15.83:0/868080085' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2022-09-29T05:24:49.919 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:24:48.547318+0000 mon.a (mon.0) 505 : cluster [DBG] osdmap e47: 8 total, 8 up, 8 in 2022-09-29T05:24:49.919 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:49.488655+0000 mon.c (mon.1) 13 : audit [DBG] from='client.? 172.21.15.83:0/868080085' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2022-09-29T05:24:49.964 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': 'device_health_metrics', 'create_time': '2022-09-29T05:23:34.795850+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '23', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr_devicehealth': {}}}] 2022-09-29T05:24:49.965 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/daemon-base:latest-pacific shell --fsid 69111522-3fb6-11ed-8431-001a4aab830c -- ceph osd pool get device_health_metrics pg_num 2022-09-29T05:24:50.540 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/69111522-3fb6-11ed-8431-001a4aab830c/mon.a/config 2022-09-29T05:24:51.080 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:24:49.588981+0000 mgr.y (mgr.14146) 149 : cluster [DBG] pgmap v134: 1 pgs: 1 active+clean; 0 B data, 41 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:24:51.081 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:24:49.588981+0000 mgr.y (mgr.14146) 149 : cluster [DBG] pgmap v134: 1 pgs: 1 active+clean; 0 B data, 41 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:24:51.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:50 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:24:49.588981+0000 mgr.y (mgr.14146) 149 : cluster [DBG] pgmap v134: 1 pgs: 1 active+clean; 0 B data, 41 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:24:51.936 INFO:teuthology.orchestra.run.smithi083.stdout:pg_num: 1 2022-09-29T05:24:52.455 INFO:tasks.cephadm:Adding mds.a on smithi083 2022-09-29T05:24:52.456 INFO:tasks.cephadm:Adding mds.b on smithi153 2022-09-29T05:24:52.456 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/daemon-base:latest-pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 69111522-3fb6-11ed-8431-001a4aab830c -- ceph orch apply mds all '2;smithi083=a;smithi153=b' 2022-09-29T05:24:53.113 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:24:51.589283+0000 mgr.y (mgr.14146) 150 : cluster [DBG] pgmap v135: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:24:53.113 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:51.933308+0000 mon.c (mon.1) 14 : audit [DBG] from='client.? 172.21.15.83:0/1002049539' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "device_health_metrics", "var": "pg_num"}]: dispatch 2022-09-29T05:24:53.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:24:51.589283+0000 mgr.y (mgr.14146) 150 : cluster [DBG] pgmap v135: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:24:53.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:51.933308+0000 mon.c (mon.1) 14 : audit [DBG] from='client.? 172.21.15.83:0/1002049539' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "device_health_metrics", "var": "pg_num"}]: dispatch 2022-09-29T05:24:53.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:24:51.589283+0000 mgr.y (mgr.14146) 150 : cluster [DBG] pgmap v135: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:24:53.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:51.933308+0000 mon.c (mon.1) 14 : audit [DBG] from='client.? 172.21.15.83:0/1002049539' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "device_health_metrics", "var": "pg_num"}]: dispatch 2022-09-29T05:24:54.083 INFO:teuthology.orchestra.run.smithi153.stdout:Scheduled mds.all update... 2022-09-29T05:24:54.500 DEBUG:teuthology.orchestra.run.smithi083:mds.a> sudo journalctl -f -n 0 -u ceph-69111522-3fb6-11ed-8431-001a4aab830c@mds.a.service 2022-09-29T05:24:54.503 DEBUG:teuthology.orchestra.run.smithi153:mds.b> sudo journalctl -f -n 0 -u ceph-69111522-3fb6-11ed-8431-001a4aab830c@mds.b.service 2022-09-29T05:24:54.506 INFO:tasks.cephadm:Setting up CephFS filesystem(s)... 2022-09-29T05:24:54.506 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2022-09-29T05:24:54.535 INFO:journalctl@ceph.mds.b.smithi153.stdout:-- Logs begin at Thu 2022-09-29 05:12:53 UTC. -- 2022-09-29T05:24:54.537 INFO:journalctl@ceph.mds.a.smithi083.stdout:-- Logs begin at Thu 2022-09-29 05:12:45 UTC. -- 2022-09-29T05:24:54.882 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:24:54.883 INFO:teuthology.orchestra.run.smithi083.stdout:{"epoch":47,"fsid":"69111522-3fb6-11ed-8431-001a4aab830c","created":"2022-09-29T05:20:45.037726+0000","modified":"2022-09-29T05:24:48.540721+0000","last_up_change":"2022-09-29T05:24:46.528124+0000","last_in_change":"2022-09-29T05:24:31.376431+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2022-09-29T05:23:34.795850+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"23","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}}],"osds":[{"osd":0,"uuid":"9b8b79ad-3d00-4dad-8da7-7cff0e65d302","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":45,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6802","nonce":2080129456},{"type":"v1","addr":"172.21.15.83:6803","nonce":2080129456}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6804","nonce":2080129456},{"type":"v1","addr":"172.21.15.83:6805","nonce":2080129456}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6808","nonce":2080129456},{"type":"v1","addr":"172.21.15.83:6809","nonce":2080129456}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6806","nonce":2080129456},{"type":"v1","addr":"172.21.15.83:6807","nonce":2080129456}]},"public_addr":"172.21.15.83:6803/2080129456","cluster_addr":"172.21.15.83:6805/2080129456","heartbeat_back_addr":"172.21.15.83:6809/2080129456","heartbeat_front_addr":"172.21.15.83:6807/2080129456","state":["exists","up"]},{"osd":1,"uuid":"e098cff5-70cc-437e-8759-e6c7e3d452d2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6810","nonce":605787053},{"type":"v1","addr":"172.21.15.83:6811","nonce":605787053}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6812","nonce":605787053},{"type":"v1","addr":"172.21.15.83:6813","nonce":605787053}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6816","nonce":605787053},{"type":"v1","addr":"172.21.15.83:6817","nonce":605787053}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6814","nonce":605787053},{"type":"v1","addr":"172.21.15.83:6815","nonce":605787053}]},"public_addr":"172.21.15.83:6811/605787053","cluster_addr":"172.21.15.83:6813/605787053","heartbeat_back_addr":"172.21.15.83:6817/605787053","heartbeat_front_addr":"172.21.15.83:6815/605787053","state":["exists","up"]},{"osd":2,"uuid":"2c14853b-5b64-496d-a011-5d8a9b17b731","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6818","nonce":1928326991},{"type":"v1","addr":"172.21.15.83:6819","nonce":1928326991}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6820","nonce":1928326991},{"type":"v1","addr":"172.21.15.83:6821","nonce":1928326991}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6824","nonce":1928326991},{"type":"v1","addr":"172.21.15.83:6825","nonce":1928326991}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6822","nonce":1928326991},{"type":"v1","addr":"172.21.15.83:6823","nonce":1928326991}]},"public_addr":"172.21.15.83:6819/1928326991","cluster_addr":"172.21.15.83:6821/1928326991","heartbeat_back_addr":"172.21.15.83:6825/1928326991","heartbeat_front_addr":"172.21.15.83:6823/1928326991","state":["exists","up"]},{"osd":3,"uuid":"7336f148-261e-468e-8d3c-28f840150f36","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6826","nonce":54976712},{"type":"v1","addr":"172.21.15.83:6827","nonce":54976712}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6828","nonce":54976712},{"type":"v1","addr":"172.21.15.83:6829","nonce":54976712}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6832","nonce":54976712},{"type":"v1","addr":"172.21.15.83:6833","nonce":54976712}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6830","nonce":54976712},{"type":"v1","addr":"172.21.15.83:6831","nonce":54976712}]},"public_addr":"172.21.15.83:6827/54976712","cluster_addr":"172.21.15.83:6829/54976712","heartbeat_back_addr":"172.21.15.83:6833/54976712","heartbeat_front_addr":"172.21.15.83:6831/54976712","state":["exists","up"]},{"osd":4,"uuid":"247d5c19-e8af-4620-a378-699dc2000db1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6800","nonce":4053209633},{"type":"v1","addr":"172.21.15.153:6801","nonce":4053209633}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6802","nonce":4053209633},{"type":"v1","addr":"172.21.15.153:6803","nonce":4053209633}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6806","nonce":4053209633},{"type":"v1","addr":"172.21.15.153:6807","nonce":4053209633}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6804","nonce":4053209633},{"type":"v1","addr":"172.21.15.153:6805","nonce":4053209633}]},"public_addr":"172.21.15.153:6801/4053209633","cluster_addr":"172.21.15.153:6803/4053209633","heartbeat_back_addr":"172.21.15.153:6807/4053209633","heartbeat_front_addr":"172.21.15.153:6805/4053209633","state":["exists","up"]},{"osd":5,"uuid":"c69e2263-5dd8-402d-b901-295c1f4840e2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":35,"up_thru":36,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6808","nonce":2536485968},{"type":"v1","addr":"172.21.15.153:6809","nonce":2536485968}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6810","nonce":2536485968},{"type":"v1","addr":"172.21.15.153:6811","nonce":2536485968}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6814","nonce":2536485968},{"type":"v1","addr":"172.21.15.153:6815","nonce":2536485968}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6812","nonce":2536485968},{"type":"v1","addr":"172.21.15.153:6813","nonce":2536485968}]},"public_addr":"172.21.15.153:6809/2536485968","cluster_addr":"172.21.15.153:6811/2536485968","heartbeat_back_addr":"172.21.15.153:6815/2536485968","heartbeat_front_addr":"172.21.15.153:6813/2536485968","state":["exists","up"]},{"osd":6,"uuid":"72aa13b6-e7b7-449b-9fa8-f284e8f0735a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":40,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6816","nonce":3662388207},{"type":"v1","addr":"172.21.15.153:6817","nonce":3662388207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6818","nonce":3662388207},{"type":"v1","addr":"172.21.15.153:6819","nonce":3662388207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6822","nonce":3662388207},{"type":"v1","addr":"172.21.15.153:6823","nonce":3662388207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6820","nonce":3662388207},{"type":"v1","addr":"172.21.15.153:6821","nonce":3662388207}]},"public_addr":"172.21.15.153:6817/3662388207","cluster_addr":"172.21.15.153:6819/3662388207","heartbeat_back_addr":"172.21.15.153:6823/3662388207","heartbeat_front_addr":"172.21.15.153:6821/3662388207","state":["exists","up"]},{"osd":7,"uuid":"b5fdd156-d2dc-47fb-b44b-803275092ac0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":45,"up_thru":46,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6824","nonce":1223012606},{"type":"v1","addr":"172.21.15.153:6825","nonce":1223012606}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6826","nonce":1223012606},{"type":"v1","addr":"172.21.15.153:6827","nonce":1223012606}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6830","nonce":1223012606},{"type":"v1","addr":"172.21.15.153:6831","nonce":1223012606}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6828","nonce":1223012606},{"type":"v1","addr":"172.21.15.153:6829","nonce":1223012606}]},"public_addr":"172.21.15.153:6825/1223012606","cluster_addr":"172.21.15.153:6827/1223012606","heartbeat_back_addr":"172.21.15.153:6831/1223012606","heartbeat_front_addr":"172.21.15.153:6829/1223012606","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2022-09-29T05:22:43.518222+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2022-09-29T05:23:01.859214+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2022-09-29T05:23:20.325748+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2022-09-29T05:23:38.717068+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2022-09-29T05:23:54.756146+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2022-09-29T05:24:11.670499+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2022-09-29T05:24:28.641104+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2022-09-29T05:24:45.518895+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.83:0/2269625481":"2022-09-30T05:21:35.319810+0000","172.21.15.83:0/2173437316":"2022-09-30T05:21:35.319810+0000","172.21.15.83:6800/2702658771":"2022-09-30T05:21:35.319810+0000","172.21.15.83:0/4005486955":"2022-09-30T05:21:15.866011+0000","172.21.15.83:6800/2006042278":"2022-09-30T05:21:15.866011+0000","172.21.15.83:6801/2006042278":"2022-09-30T05:21:15.866011+0000","172.21.15.83:6801/2702658771":"2022-09-30T05:21:35.319810+0000","172.21.15.83:0/4256024697":"2022-09-30T05:21:15.866011+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2022-09-29T05:24:54.896 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': 'device_health_metrics', 'create_time': '2022-09-29T05:23:34.795850+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '23', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr_devicehealth': {}}}] 2022-09-29T05:24:54.896 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get device_health_metrics pg_num 2022-09-29T05:24:55.172 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:24:53.589628+0000 mgr.y (mgr.14146) 151 : cluster [DBG] pgmap v136: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:24:55.172 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:54.080666+0000 mon.a (mon.0) 506 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:24:55.172 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:54.120713+0000 mon.a (mon.0) 507 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:24:55.173 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:54.121979+0000 mon.a (mon.0) 508 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:24:55.173 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:54.122810+0000 mon.a (mon.0) 509 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:24:55.173 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:54.130150+0000 mon.a (mon.0) 510 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:24:55.173 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:54.137882+0000 mon.a (mon.0) 511 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:24:55.174 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:54.138819+0000 mon.a (mon.0) 512 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2022-09-29T05:24:55.174 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:54.144127+0000 mon.a (mon.0) 513 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2022-09-29T05:24:55.174 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:54.144907+0000 mon.a (mon.0) 514 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:24:55.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:24:53.589628+0000 mgr.y (mgr.14146) 151 : cluster [DBG] pgmap v136: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:24:55.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:54.080666+0000 mon.a (mon.0) 506 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:24:55.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:54.120713+0000 mon.a (mon.0) 507 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:24:55.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:54.121979+0000 mon.a (mon.0) 508 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:24:55.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:54.122810+0000 mon.a (mon.0) 509 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:24:55.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:54.130150+0000 mon.a (mon.0) 510 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:24:55.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:54.137882+0000 mon.a (mon.0) 511 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:24:55.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:54.138819+0000 mon.a (mon.0) 512 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2022-09-29T05:24:55.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:54.144127+0000 mon.a (mon.0) 513 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2022-09-29T05:24:55.177 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:54.144907+0000 mon.a (mon.0) 514 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:24:55.195 INFO:teuthology.orchestra.run.smithi083.stdout:pg_num: 1 2022-09-29T05:24:55.209 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2022-09-29T05:24:55.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:54 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:24:53.589628+0000 mgr.y (mgr.14146) 151 : cluster [DBG] pgmap v136: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:24:55.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:54 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:54.080666+0000 mon.a (mon.0) 506 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:24:55.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:54 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:54.120713+0000 mon.a (mon.0) 507 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:24:55.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:54 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:54.121979+0000 mon.a (mon.0) 508 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:24:55.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:54 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:54.122810+0000 mon.a (mon.0) 509 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:24:55.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:54 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:54.130150+0000 mon.a (mon.0) 510 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:24:55.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:54 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:54.137882+0000 mon.a (mon.0) 511 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:24:55.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:54 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:54.138819+0000 mon.a (mon.0) 512 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2022-09-29T05:24:55.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:54 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:54.144127+0000 mon.a (mon.0) 513 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2022-09-29T05:24:55.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:54 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:54.144907+0000 mon.a (mon.0) 514 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:24:55.545 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:24:55.546 INFO:teuthology.orchestra.run.smithi083.stdout:{"epoch":1,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[]} 2022-09-29T05:24:55.548 INFO:teuthology.orchestra.run.smithi083.stderr:dumped fsmap epoch 1 2022-09-29T05:24:55.831 INFO:journalctl@ceph.mds.a.smithi083.stdout:Sep 29 05:24:55 smithi083 systemd[1]: Starting Ceph mds.a for 69111522-3fb6-11ed-8431-001a4aab830c... 2022-09-29T05:24:55.832 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:54.073117+0000 mgr.y (mgr.14146) 152 : audit [DBG] from='client.24329 -' entity='client.admin' cmd=[{"prefix": "orch apply mds", "fs_name": "all", "placement": "2;smithi083=a;smithi153=b", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:24:56.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:54.073117+0000 mgr.y (mgr.14146) 152 : audit [DBG] from='client.24329 -' entity='client.admin' cmd=[{"prefix": "orch apply mds", "fs_name": "all", "placement": "2;smithi083=a;smithi153=b", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:24:56.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:24:54.074663+0000 mgr.y (mgr.14146) 153 : cephadm [INF] Saving service mds.all spec with placement smithi083=a;smithi153=b;count:2 2022-09-29T05:24:56.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:24:54.145639+0000 mgr.y (mgr.14146) 154 : cephadm [INF] Deploying daemon mds.a on smithi083 2022-09-29T05:24:56.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:54.879568+0000 mon.a (mon.0) 515 : audit [DBG] from='client.? 172.21.15.83:0/3135786038' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2022-09-29T05:24:56.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:55.192767+0000 mon.a (mon.0) 516 : audit [DBG] from='client.? 172.21.15.83:0/1375200508' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "device_health_metrics", "var": "pg_num"}]: dispatch 2022-09-29T05:24:56.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:55.542889+0000 mon.a (mon.0) 517 : audit [DBG] from='client.? 172.21.15.83:0/1243629448' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2022-09-29T05:24:56.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:24:54.074663+0000 mgr.y (mgr.14146) 153 : cephadm [INF] Saving service mds.all spec with placement smithi083=a;smithi153=b;count:2 2022-09-29T05:24:56.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:24:54.145639+0000 mgr.y (mgr.14146) 154 : cephadm [INF] Deploying daemon mds.a on smithi083 2022-09-29T05:24:56.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:54.879568+0000 mon.a (mon.0) 515 : audit [DBG] from='client.? 172.21.15.83:0/3135786038' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2022-09-29T05:24:56.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:55.192767+0000 mon.a (mon.0) 516 : audit [DBG] from='client.? 172.21.15.83:0/1375200508' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "device_health_metrics", "var": "pg_num"}]: dispatch 2022-09-29T05:24:56.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:55.542889+0000 mon.a (mon.0) 517 : audit [DBG] from='client.? 172.21.15.83:0/1243629448' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2022-09-29T05:24:56.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:55 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:54.073117+0000 mgr.y (mgr.14146) 152 : audit [DBG] from='client.24329 -' entity='client.admin' cmd=[{"prefix": "orch apply mds", "fs_name": "all", "placement": "2;smithi083=a;smithi153=b", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:24:56.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:55 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:24:54.074663+0000 mgr.y (mgr.14146) 153 : cephadm [INF] Saving service mds.all spec with placement smithi083=a;smithi153=b;count:2 2022-09-29T05:24:56.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:55 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:24:54.145639+0000 mgr.y (mgr.14146) 154 : cephadm [INF] Deploying daemon mds.a on smithi083 2022-09-29T05:24:56.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:55 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:54.879568+0000 mon.a (mon.0) 515 : audit [DBG] from='client.? 172.21.15.83:0/3135786038' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2022-09-29T05:24:56.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:55 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:55.192767+0000 mon.a (mon.0) 516 : audit [DBG] from='client.? 172.21.15.83:0/1375200508' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "device_health_metrics", "var": "pg_num"}]: dispatch 2022-09-29T05:24:56.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:55 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:55.542889+0000 mon.a (mon.0) 517 : audit [DBG] from='client.? 172.21.15.83:0/1243629448' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2022-09-29T05:24:56.674 INFO:journalctl@ceph.mds.a.smithi083.stdout:Sep 29 05:24:56 smithi083 podman[58520]: 2022-09-29T05:24:56.963 INFO:journalctl@ceph.mds.a.smithi083.stdout:Sep 29 05:24:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mds-a[58536]: starting mds.a at 2022-09-29T05:24:56.963 INFO:journalctl@ceph.mds.a.smithi083.stdout:Sep 29 05:24:56 smithi083 bash[58520]: 7741d74b494df6251d848fd12a2d5b3fc60be8943b2fe749372f09dc630086d8 2022-09-29T05:24:56.964 INFO:journalctl@ceph.mds.a.smithi083.stdout:Sep 29 05:24:56 smithi083 systemd[1]: Started Ceph mds.a for 69111522-3fb6-11ed-8431-001a4aab830c. 2022-09-29T05:24:56.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:24:55.589973+0000 mgr.y (mgr.14146) 155 : cluster [DBG] pgmap v137: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:24:56.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:24:56.692356+0000 mon.a (mon.0) 518 : cluster [DBG] mds.? [v2:172.21.15.83:6834/4236763119,v1:172.21.15.83:6835/4236763119] up:boot 2022-09-29T05:24:56.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:24:56.692393+0000 mon.a (mon.0) 519 : cluster [DBG] fsmap 1 up:standby 2022-09-29T05:24:56.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:56.692504+0000 mon.a (mon.0) 520 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2022-09-29T05:24:56.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:56.823218+0000 mon.a (mon.0) 521 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:24:56.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:56.824179+0000 mon.a (mon.0) 522 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2022-09-29T05:24:56.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:56.826886+0000 mon.a (mon.0) 523 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2022-09-29T05:24:56.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:56.827478+0000 mon.a (mon.0) 524 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:24:56.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:24:55.589973+0000 mgr.y (mgr.14146) 155 : cluster [DBG] pgmap v137: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:24:56.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:24:56.692356+0000 mon.a (mon.0) 518 : cluster [DBG] mds.? [v2:172.21.15.83:6834/4236763119,v1:172.21.15.83:6835/4236763119] up:boot 2022-09-29T05:24:56.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:24:56.692393+0000 mon.a (mon.0) 519 : cluster [DBG] fsmap 1 up:standby 2022-09-29T05:24:56.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:56.692504+0000 mon.a (mon.0) 520 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2022-09-29T05:24:56.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:56.823218+0000 mon.a (mon.0) 521 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:24:56.968 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:56.824179+0000 mon.a (mon.0) 522 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2022-09-29T05:24:56.968 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:56.826886+0000 mon.a (mon.0) 523 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2022-09-29T05:24:56.968 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:56.827478+0000 mon.a (mon.0) 524 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:24:57.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:56 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:24:55.589973+0000 mgr.y (mgr.14146) 155 : cluster [DBG] pgmap v137: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:24:57.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:56 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:24:56.692356+0000 mon.a (mon.0) 518 : cluster [DBG] mds.? [v2:172.21.15.83:6834/4236763119,v1:172.21.15.83:6835/4236763119] up:boot 2022-09-29T05:24:57.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:56 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:24:56.692393+0000 mon.a (mon.0) 519 : cluster [DBG] fsmap 1 up:standby 2022-09-29T05:24:57.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:56 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:56.692504+0000 mon.a (mon.0) 520 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2022-09-29T05:24:57.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:56 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:56.823218+0000 mon.a (mon.0) 521 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:24:57.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:56 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:56.824179+0000 mon.a (mon.0) 522 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2022-09-29T05:24:57.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:56 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:56.826886+0000 mon.a (mon.0) 523 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2022-09-29T05:24:57.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:56 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:56.827478+0000 mon.a (mon.0) 524 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:24:57.561 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2022-09-29T05:24:57.897 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:24:57.898 INFO:teuthology.orchestra.run.smithi083.stdout:{"epoch":2,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":24343,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.83:6835/4236763119","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6834","nonce":4236763119},{"type":"v1","addr":"172.21.15.83:6835","nonce":4236763119}]},"join_fscid":-1,"export_targets":[],"features":4540138297136906239,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2}],"filesystems":[]} 2022-09-29T05:24:57.900 INFO:teuthology.orchestra.run.smithi083.stderr:dumped fsmap epoch 2 2022-09-29T05:24:57.910 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:57 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:24:56.827965+0000 mgr.y (mgr.14146) 156 : cephadm [INF] Deploying daemon mds.b on smithi153 2022-09-29T05:24:57.910 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:57 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:24:56.827965+0000 mgr.y (mgr.14146) 156 : cephadm [INF] Deploying daemon mds.b on smithi153 2022-09-29T05:24:57.996 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:57 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:24:56.827965+0000 mgr.y (mgr.14146) 156 : cephadm [INF] Deploying daemon mds.b on smithi153 2022-09-29T05:24:58.309 INFO:journalctl@ceph.mds.b.smithi153.stdout:Sep 29 05:24:58 smithi153 systemd[1]: Starting Ceph mds.b for 69111522-3fb6-11ed-8431-001a4aab830c... 2022-09-29T05:24:59.110 INFO:journalctl@ceph.mds.b.smithi153.stdout:Sep 29 05:24:58 smithi153 podman[53069]: 2022-09-29T05:24:59.111 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:58 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:24:57.590307+0000 mgr.y (mgr.14146) 157 : cluster [DBG] pgmap v138: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:24:59.111 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:24:58 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:57.894956+0000 mon.a (mon.0) 525 : audit [DBG] from='client.? 172.21.15.83:0/3369733736' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2022-09-29T05:24:59.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:24:57.590307+0000 mgr.y (mgr.14146) 157 : cluster [DBG] pgmap v138: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:24:59.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:24:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:57.894956+0000 mon.a (mon.0) 525 : audit [DBG] from='client.? 172.21.15.83:0/3369733736' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2022-09-29T05:24:59.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:24:57.590307+0000 mgr.y (mgr.14146) 157 : cluster [DBG] pgmap v138: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:24:59.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:24:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:57.894956+0000 mon.a (mon.0) 525 : audit [DBG] from='client.? 172.21.15.83:0/3369733736' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2022-09-29T05:24:59.559 INFO:journalctl@ceph.mds.b.smithi153.stdout:Sep 29 05:24:59 smithi153 bash[53069]: 365a49055b33f1e47c863bb976a459e4c944fbc3367142ff0b4a32b97cf1dad1 2022-09-29T05:24:59.559 INFO:journalctl@ceph.mds.b.smithi153.stdout:Sep 29 05:24:59 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mds-b[53085]: starting mds.b at 2022-09-29T05:24:59.559 INFO:journalctl@ceph.mds.b.smithi153.stdout:Sep 29 05:24:59 smithi153 systemd[1]: Started Ceph mds.b for 69111522-3fb6-11ed-8431-001a4aab830c. 2022-09-29T05:24:59.912 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2022-09-29T05:25:00.245 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:25:00.245 INFO:teuthology.orchestra.run.smithi083.stdout:{"epoch":3,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":24343,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.83:6835/4236763119","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6834","nonce":4236763119},{"type":"v1","addr":"172.21.15.83:6835","nonce":4236763119}]},"join_fscid":-1,"export_targets":[],"features":4540138297136906239,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2},{"gid":24356,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.153:6833/673217382","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6832","nonce":673217382},{"type":"v1","addr":"172.21.15.153:6833","nonce":673217382}]},"join_fscid":-1,"export_targets":[],"features":4540138297136906239,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3}],"filesystems":[]} 2022-09-29T05:25:00.247 INFO:teuthology.orchestra.run.smithi083.stderr:dumped fsmap epoch 3 2022-09-29T05:25:00.260 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2022-09-29T05:25:00.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:00 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:59.292099+0000 mon.a (mon.0) 526 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:25:00.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:00 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:59.293381+0000 mon.a (mon.0) 527 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:25:00.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:00 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:59.294550+0000 mon.a (mon.0) 528 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:25:00.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:00 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:59.295310+0000 mon.a (mon.0) 529 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:25:00.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:00 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:24:59.590830+0000 mgr.y (mgr.14146) 158 : cluster [DBG] pgmap v139: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:25:00.561 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:00 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:24:59.838459+0000 mon.a (mon.0) 530 : cluster [DBG] mds.? [v2:172.21.15.153:6832/673217382,v1:172.21.15.153:6833/673217382] up:boot 2022-09-29T05:25:00.561 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:00 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:24:59.838501+0000 mon.a (mon.0) 531 : cluster [DBG] fsmap 2 up:standby 2022-09-29T05:25:00.561 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:00 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:24:59.838597+0000 mon.a (mon.0) 532 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2022-09-29T05:25:00.561 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:00 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:00.242232+0000 mon.a (mon.0) 533 : audit [DBG] from='client.? 172.21.15.83:0/1512125027' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2022-09-29T05:25:00.574 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:25:00.574 INFO:teuthology.orchestra.run.smithi083.stdout:{"epoch":47,"fsid":"69111522-3fb6-11ed-8431-001a4aab830c","created":"2022-09-29T05:20:45.037726+0000","modified":"2022-09-29T05:24:48.540721+0000","last_up_change":"2022-09-29T05:24:46.528124+0000","last_in_change":"2022-09-29T05:24:31.376431+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2022-09-29T05:23:34.795850+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"23","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}}],"osds":[{"osd":0,"uuid":"9b8b79ad-3d00-4dad-8da7-7cff0e65d302","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":45,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6802","nonce":2080129456},{"type":"v1","addr":"172.21.15.83:6803","nonce":2080129456}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6804","nonce":2080129456},{"type":"v1","addr":"172.21.15.83:6805","nonce":2080129456}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6808","nonce":2080129456},{"type":"v1","addr":"172.21.15.83:6809","nonce":2080129456}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6806","nonce":2080129456},{"type":"v1","addr":"172.21.15.83:6807","nonce":2080129456}]},"public_addr":"172.21.15.83:6803/2080129456","cluster_addr":"172.21.15.83:6805/2080129456","heartbeat_back_addr":"172.21.15.83:6809/2080129456","heartbeat_front_addr":"172.21.15.83:6807/2080129456","state":["exists","up"]},{"osd":1,"uuid":"e098cff5-70cc-437e-8759-e6c7e3d452d2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6810","nonce":605787053},{"type":"v1","addr":"172.21.15.83:6811","nonce":605787053}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6812","nonce":605787053},{"type":"v1","addr":"172.21.15.83:6813","nonce":605787053}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6816","nonce":605787053},{"type":"v1","addr":"172.21.15.83:6817","nonce":605787053}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6814","nonce":605787053},{"type":"v1","addr":"172.21.15.83:6815","nonce":605787053}]},"public_addr":"172.21.15.83:6811/605787053","cluster_addr":"172.21.15.83:6813/605787053","heartbeat_back_addr":"172.21.15.83:6817/605787053","heartbeat_front_addr":"172.21.15.83:6815/605787053","state":["exists","up"]},{"osd":2,"uuid":"2c14853b-5b64-496d-a011-5d8a9b17b731","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6818","nonce":1928326991},{"type":"v1","addr":"172.21.15.83:6819","nonce":1928326991}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6820","nonce":1928326991},{"type":"v1","addr":"172.21.15.83:6821","nonce":1928326991}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6824","nonce":1928326991},{"type":"v1","addr":"172.21.15.83:6825","nonce":1928326991}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6822","nonce":1928326991},{"type":"v1","addr":"172.21.15.83:6823","nonce":1928326991}]},"public_addr":"172.21.15.83:6819/1928326991","cluster_addr":"172.21.15.83:6821/1928326991","heartbeat_back_addr":"172.21.15.83:6825/1928326991","heartbeat_front_addr":"172.21.15.83:6823/1928326991","state":["exists","up"]},{"osd":3,"uuid":"7336f148-261e-468e-8d3c-28f840150f36","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6826","nonce":54976712},{"type":"v1","addr":"172.21.15.83:6827","nonce":54976712}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6828","nonce":54976712},{"type":"v1","addr":"172.21.15.83:6829","nonce":54976712}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6832","nonce":54976712},{"type":"v1","addr":"172.21.15.83:6833","nonce":54976712}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6830","nonce":54976712},{"type":"v1","addr":"172.21.15.83:6831","nonce":54976712}]},"public_addr":"172.21.15.83:6827/54976712","cluster_addr":"172.21.15.83:6829/54976712","heartbeat_back_addr":"172.21.15.83:6833/54976712","heartbeat_front_addr":"172.21.15.83:6831/54976712","state":["exists","up"]},{"osd":4,"uuid":"247d5c19-e8af-4620-a378-699dc2000db1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6800","nonce":4053209633},{"type":"v1","addr":"172.21.15.153:6801","nonce":4053209633}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6802","nonce":4053209633},{"type":"v1","addr":"172.21.15.153:6803","nonce":4053209633}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6806","nonce":4053209633},{"type":"v1","addr":"172.21.15.153:6807","nonce":4053209633}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6804","nonce":4053209633},{"type":"v1","addr":"172.21.15.153:6805","nonce":4053209633}]},"public_addr":"172.21.15.153:6801/4053209633","cluster_addr":"172.21.15.153:6803/4053209633","heartbeat_back_addr":"172.21.15.153:6807/4053209633","heartbeat_front_addr":"172.21.15.153:6805/4053209633","state":["exists","up"]},{"osd":5,"uuid":"c69e2263-5dd8-402d-b901-295c1f4840e2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":35,"up_thru":36,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6808","nonce":2536485968},{"type":"v1","addr":"172.21.15.153:6809","nonce":2536485968}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6810","nonce":2536485968},{"type":"v1","addr":"172.21.15.153:6811","nonce":2536485968}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6814","nonce":2536485968},{"type":"v1","addr":"172.21.15.153:6815","nonce":2536485968}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6812","nonce":2536485968},{"type":"v1","addr":"172.21.15.153:6813","nonce":2536485968}]},"public_addr":"172.21.15.153:6809/2536485968","cluster_addr":"172.21.15.153:6811/2536485968","heartbeat_back_addr":"172.21.15.153:6815/2536485968","heartbeat_front_addr":"172.21.15.153:6813/2536485968","state":["exists","up"]},{"osd":6,"uuid":"72aa13b6-e7b7-449b-9fa8-f284e8f0735a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":40,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6816","nonce":3662388207},{"type":"v1","addr":"172.21.15.153:6817","nonce":3662388207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6818","nonce":3662388207},{"type":"v1","addr":"172.21.15.153:6819","nonce":3662388207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6822","nonce":3662388207},{"type":"v1","addr":"172.21.15.153:6823","nonce":3662388207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6820","nonce":3662388207},{"type":"v1","addr":"172.21.15.153:6821","nonce":3662388207}]},"public_addr":"172.21.15.153:6817/3662388207","cluster_addr":"172.21.15.153:6819/3662388207","heartbeat_back_addr":"172.21.15.153:6823/3662388207","heartbeat_front_addr":"172.21.15.153:6821/3662388207","state":["exists","up"]},{"osd":7,"uuid":"b5fdd156-d2dc-47fb-b44b-803275092ac0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":45,"up_thru":46,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6824","nonce":1223012606},{"type":"v1","addr":"172.21.15.153:6825","nonce":1223012606}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6826","nonce":1223012606},{"type":"v1","addr":"172.21.15.153:6827","nonce":1223012606}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6830","nonce":1223012606},{"type":"v1","addr":"172.21.15.153:6831","nonce":1223012606}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6828","nonce":1223012606},{"type":"v1","addr":"172.21.15.153:6829","nonce":1223012606}]},"public_addr":"172.21.15.153:6825/1223012606","cluster_addr":"172.21.15.153:6827/1223012606","heartbeat_back_addr":"172.21.15.153:6831/1223012606","heartbeat_front_addr":"172.21.15.153:6829/1223012606","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2022-09-29T05:22:43.518222+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2022-09-29T05:23:01.859214+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2022-09-29T05:23:20.325748+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2022-09-29T05:23:38.717068+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2022-09-29T05:23:54.756146+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2022-09-29T05:24:11.670499+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2022-09-29T05:24:28.641104+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2022-09-29T05:24:45.518895+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.83:0/2269625481":"2022-09-30T05:21:35.319810+0000","172.21.15.83:0/2173437316":"2022-09-30T05:21:35.319810+0000","172.21.15.83:6800/2702658771":"2022-09-30T05:21:35.319810+0000","172.21.15.83:0/4005486955":"2022-09-30T05:21:15.866011+0000","172.21.15.83:6800/2006042278":"2022-09-30T05:21:15.866011+0000","172.21.15.83:6801/2006042278":"2022-09-30T05:21:15.866011+0000","172.21.15.83:6801/2702658771":"2022-09-30T05:21:35.319810+0000","172.21.15.83:0/4256024697":"2022-09-30T05:21:15.866011+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2022-09-29T05:25:00.587 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:59.292099+0000 mon.a (mon.0) 526 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:25:00.587 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:59.293381+0000 mon.a (mon.0) 527 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:25:00.587 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:59.294550+0000 mon.a (mon.0) 528 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:25:00.588 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:59.295310+0000 mon.a (mon.0) 529 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:25:00.588 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:24:59.590830+0000 mgr.y (mgr.14146) 158 : cluster [DBG] pgmap v139: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:25:00.588 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:24:59.838459+0000 mon.a (mon.0) 530 : cluster [DBG] mds.? [v2:172.21.15.153:6832/673217382,v1:172.21.15.153:6833/673217382] up:boot 2022-09-29T05:25:00.588 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:24:59.838501+0000 mon.a (mon.0) 531 : cluster [DBG] fsmap 2 up:standby 2022-09-29T05:25:00.589 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:24:59.838597+0000 mon.a (mon.0) 532 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2022-09-29T05:25:00.589 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:00.242232+0000 mon.a (mon.0) 533 : audit [DBG] from='client.? 172.21.15.83:0/1512125027' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2022-09-29T05:25:00.589 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:59.292099+0000 mon.a (mon.0) 526 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:25:00.590 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:59.293381+0000 mon.a (mon.0) 527 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:25:00.590 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:59.294550+0000 mon.a (mon.0) 528 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:25:00.590 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:59.295310+0000 mon.a (mon.0) 529 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:25:00.590 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:24:59.590830+0000 mgr.y (mgr.14146) 158 : cluster [DBG] pgmap v139: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:25:00.591 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:24:59.838459+0000 mon.a (mon.0) 530 : cluster [DBG] mds.? [v2:172.21.15.153:6832/673217382,v1:172.21.15.153:6833/673217382] up:boot 2022-09-29T05:25:00.591 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:24:59.838501+0000 mon.a (mon.0) 531 : cluster [DBG] fsmap 2 up:standby 2022-09-29T05:25:00.591 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:24:59.838597+0000 mon.a (mon.0) 532 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2022-09-29T05:25:00.591 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:00.242232+0000 mon.a (mon.0) 533 : audit [DBG] from='client.? 172.21.15.83:0/1512125027' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2022-09-29T05:25:00.593 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': 'device_health_metrics', 'create_time': '2022-09-29T05:23:34.795850+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '23', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr_devicehealth': {}}}] 2022-09-29T05:25:00.593 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get device_health_metrics pg_num 2022-09-29T05:25:00.897 INFO:teuthology.orchestra.run.smithi083.stdout:pg_num: 1 2022-09-29T05:25:00.911 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --format=json-pretty osd lspools 2022-09-29T05:25:01.215 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:25:01.215 INFO:teuthology.orchestra.run.smithi083.stdout:[ 2022-09-29T05:25:01.216 INFO:teuthology.orchestra.run.smithi083.stdout: { 2022-09-29T05:25:01.216 INFO:teuthology.orchestra.run.smithi083.stdout: "poolnum": 1, 2022-09-29T05:25:01.216 INFO:teuthology.orchestra.run.smithi083.stdout: "poolname": "device_health_metrics" 2022-09-29T05:25:01.216 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T05:25:01.217 INFO:teuthology.orchestra.run.smithi083.stdout:] 2022-09-29T05:25:01.228 DEBUG:tasks.cephfs.filesystem:Creating filesystem 'cephfs' 2022-09-29T05:25:01.229 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool create cephfs_metadata --pg_num_min 64 2022-09-29T05:25:01.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:01 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:00.571203+0000 mon.c (mon.1) 15 : audit [DBG] from='client.? 172.21.15.83:0/967126417' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2022-09-29T05:25:01.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:01 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:00.727781+0000 mon.a (mon.0) 534 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:25:01.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:01 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:00.894950+0000 mon.c (mon.1) 16 : audit [DBG] from='client.? 172.21.15.83:0/3812569146' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "device_health_metrics", "var": "pg_num"}]: dispatch 2022-09-29T05:25:01.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:01 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:01.212667+0000 mon.c (mon.1) 17 : audit [DBG] from='client.? 172.21.15.83:0/950479021' entity='client.admin' cmd=[{"prefix": "osd lspools", "format": "json-pretty"}]: dispatch 2022-09-29T05:25:01.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:00.571203+0000 mon.c (mon.1) 15 : audit [DBG] from='client.? 172.21.15.83:0/967126417' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2022-09-29T05:25:01.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:00.727781+0000 mon.a (mon.0) 534 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:25:01.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:00.894950+0000 mon.c (mon.1) 16 : audit [DBG] from='client.? 172.21.15.83:0/3812569146' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "device_health_metrics", "var": "pg_num"}]: dispatch 2022-09-29T05:25:01.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:01.212667+0000 mon.c (mon.1) 17 : audit [DBG] from='client.? 172.21.15.83:0/950479021' entity='client.admin' cmd=[{"prefix": "osd lspools", "format": "json-pretty"}]: dispatch 2022-09-29T05:25:01.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:00.571203+0000 mon.c (mon.1) 15 : audit [DBG] from='client.? 172.21.15.83:0/967126417' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2022-09-29T05:25:01.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:00.727781+0000 mon.a (mon.0) 534 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:25:01.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:00.894950+0000 mon.c (mon.1) 16 : audit [DBG] from='client.? 172.21.15.83:0/3812569146' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "device_health_metrics", "var": "pg_num"}]: dispatch 2022-09-29T05:25:01.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:01.212667+0000 mon.c (mon.1) 17 : audit [DBG] from='client.? 172.21.15.83:0/950479021' entity='client.admin' cmd=[{"prefix": "osd lspools", "format": "json-pretty"}]: dispatch 2022-09-29T05:25:02.445 INFO:teuthology.orchestra.run.smithi083.stderr:pool 'cephfs_metadata' created 2022-09-29T05:25:02.456 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool create cephfs_data 64 --pg_num_min 64 --target_size_ratio 0.9 2022-09-29T05:25:02.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:01.316672+0000 mon.a (mon.0) 535 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:25:02.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:01.410789+0000 mon.a (mon.0) 536 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:25:02.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:01.414849+0000 mon.a (mon.0) 537 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:25:02.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:01.416893+0000 mon.a (mon.0) 538 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:25:02.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:01.417742+0000 mon.a (mon.0) 539 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:25:02.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:01.418284+0000 mon.a (mon.0) 540 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:25:02.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:01.424842+0000 mon.a (mon.0) 541 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:25:02.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:01.527795+0000 mon.c (mon.1) 18 : audit [INF] from='client.? 172.21.15.83:0/1048763046' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]: dispatch 2022-09-29T05:25:02.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:01.528109+0000 mon.a (mon.0) 542 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]: dispatch 2022-09-29T05:25:02.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:25:01.591137+0000 mgr.y (mgr.14146) 159 : cluster [DBG] pgmap v140: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:25:02.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:01.316672+0000 mon.a (mon.0) 535 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:25:02.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:01.410789+0000 mon.a (mon.0) 536 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:25:02.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:01.414849+0000 mon.a (mon.0) 537 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:25:02.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:01.416893+0000 mon.a (mon.0) 538 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:25:02.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:01.417742+0000 mon.a (mon.0) 539 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:25:02.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:01.418284+0000 mon.a (mon.0) 540 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:25:02.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:01.424842+0000 mon.a (mon.0) 541 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:25:02.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:01.527795+0000 mon.c (mon.1) 18 : audit [INF] from='client.? 172.21.15.83:0/1048763046' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]: dispatch 2022-09-29T05:25:02.718 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:01.528109+0000 mon.a (mon.0) 542 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]: dispatch 2022-09-29T05:25:02.718 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:25:01.591137+0000 mgr.y (mgr.14146) 159 : cluster [DBG] pgmap v140: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:25:02.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:01.316672+0000 mon.a (mon.0) 535 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:25:02.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:01.410789+0000 mon.a (mon.0) 536 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:25:02.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:01.414849+0000 mon.a (mon.0) 537 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:25:02.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:01.416893+0000 mon.a (mon.0) 538 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:25:02.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:01.417742+0000 mon.a (mon.0) 539 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:25:02.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:01.418284+0000 mon.a (mon.0) 540 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:25:02.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:01.424842+0000 mon.a (mon.0) 541 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:25:02.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:01.527795+0000 mon.c (mon.1) 18 : audit [INF] from='client.? 172.21.15.83:0/1048763046' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]: dispatch 2022-09-29T05:25:02.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:01.528109+0000 mon.a (mon.0) 542 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]: dispatch 2022-09-29T05:25:02.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:25:01.591137+0000 mgr.y (mgr.14146) 159 : cluster [DBG] pgmap v140: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:25:03.456 INFO:teuthology.orchestra.run.smithi083.stderr:pool 'cephfs_data' created 2022-09-29T05:25:03.470 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs new cephfs cephfs_metadata cephfs_data 2022-09-29T05:25:03.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:02.434288+0000 mon.a (mon.0) 543 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]': finished 2022-09-29T05:25:03.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:25:02.434344+0000 mon.a (mon.0) 544 : cluster [DBG] osdmap e48: 8 total, 8 up, 8 in 2022-09-29T05:25:03.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:02.740116+0000 mon.a (mon.0) 545 : audit [INF] from='client.? 172.21.15.83:0/250632544' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]: dispatch 2022-09-29T05:25:03.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:02.434288+0000 mon.a (mon.0) 543 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]': finished 2022-09-29T05:25:03.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:25:02.434344+0000 mon.a (mon.0) 544 : cluster [DBG] osdmap e48: 8 total, 8 up, 8 in 2022-09-29T05:25:03.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:02.740116+0000 mon.a (mon.0) 545 : audit [INF] from='client.? 172.21.15.83:0/250632544' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]: dispatch 2022-09-29T05:25:03.813 INFO:teuthology.orchestra.run.smithi083.stderr:new fs with metadata pool 2 and data pool 3 2022-09-29T05:25:03.814 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:02.434288+0000 mon.a (mon.0) 543 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]': finished 2022-09-29T05:25:03.814 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:25:02.434344+0000 mon.a (mon.0) 544 : cluster [DBG] osdmap e48: 8 total, 8 up, 8 in 2022-09-29T05:25:03.814 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:02.740116+0000 mon.a (mon.0) 545 : audit [INF] from='client.? 172.21.15.83:0/250632544' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]: dispatch 2022-09-29T05:25:03.830 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2022-09-29T05:25:04.109 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:25:04.110 INFO:teuthology.orchestra.run.smithi083.stdout:{"epoch":50,"fsid":"69111522-3fb6-11ed-8431-001a4aab830c","created":"2022-09-29T05:20:45.037726+0000","modified":"2022-09-29T05:25:03.789266+0000","last_up_change":"2022-09-29T05:24:46.528124+0000","last_in_change":"2022-09-29T05:24:31.376431+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2022-09-29T05:23:34.795850+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"23","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2022-09-29T05:25:01.755735+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"50","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}}},{"pool":3,"pool_name":"cephfs_data","create_time":"2022-09-29T05:25:02.966273+0000","flags":32769,"flags_names":"hashpspool,creating","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"50","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}}],"osds":[{"osd":0,"uuid":"9b8b79ad-3d00-4dad-8da7-7cff0e65d302","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6802","nonce":2080129456},{"type":"v1","addr":"172.21.15.83:6803","nonce":2080129456}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6804","nonce":2080129456},{"type":"v1","addr":"172.21.15.83:6805","nonce":2080129456}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6808","nonce":2080129456},{"type":"v1","addr":"172.21.15.83:6809","nonce":2080129456}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6806","nonce":2080129456},{"type":"v1","addr":"172.21.15.83:6807","nonce":2080129456}]},"public_addr":"172.21.15.83:6803/2080129456","cluster_addr":"172.21.15.83:6805/2080129456","heartbeat_back_addr":"172.21.15.83:6809/2080129456","heartbeat_front_addr":"172.21.15.83:6807/2080129456","state":["exists","up"]},{"osd":1,"uuid":"e098cff5-70cc-437e-8759-e6c7e3d452d2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6810","nonce":605787053},{"type":"v1","addr":"172.21.15.83:6811","nonce":605787053}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6812","nonce":605787053},{"type":"v1","addr":"172.21.15.83:6813","nonce":605787053}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6816","nonce":605787053},{"type":"v1","addr":"172.21.15.83:6817","nonce":605787053}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6814","nonce":605787053},{"type":"v1","addr":"172.21.15.83:6815","nonce":605787053}]},"public_addr":"172.21.15.83:6811/605787053","cluster_addr":"172.21.15.83:6813/605787053","heartbeat_back_addr":"172.21.15.83:6817/605787053","heartbeat_front_addr":"172.21.15.83:6815/605787053","state":["exists","up"]},{"osd":2,"uuid":"2c14853b-5b64-496d-a011-5d8a9b17b731","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6818","nonce":1928326991},{"type":"v1","addr":"172.21.15.83:6819","nonce":1928326991}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6820","nonce":1928326991},{"type":"v1","addr":"172.21.15.83:6821","nonce":1928326991}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6824","nonce":1928326991},{"type":"v1","addr":"172.21.15.83:6825","nonce":1928326991}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6822","nonce":1928326991},{"type":"v1","addr":"172.21.15.83:6823","nonce":1928326991}]},"public_addr":"172.21.15.83:6819/1928326991","cluster_addr":"172.21.15.83:6821/1928326991","heartbeat_back_addr":"172.21.15.83:6825/1928326991","heartbeat_front_addr":"172.21.15.83:6823/1928326991","state":["exists","up"]},{"osd":3,"uuid":"7336f148-261e-468e-8d3c-28f840150f36","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6826","nonce":54976712},{"type":"v1","addr":"172.21.15.83:6827","nonce":54976712}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6828","nonce":54976712},{"type":"v1","addr":"172.21.15.83:6829","nonce":54976712}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6832","nonce":54976712},{"type":"v1","addr":"172.21.15.83:6833","nonce":54976712}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6830","nonce":54976712},{"type":"v1","addr":"172.21.15.83:6831","nonce":54976712}]},"public_addr":"172.21.15.83:6827/54976712","cluster_addr":"172.21.15.83:6829/54976712","heartbeat_back_addr":"172.21.15.83:6833/54976712","heartbeat_front_addr":"172.21.15.83:6831/54976712","state":["exists","up"]},{"osd":4,"uuid":"247d5c19-e8af-4620-a378-699dc2000db1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6800","nonce":4053209633},{"type":"v1","addr":"172.21.15.153:6801","nonce":4053209633}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6802","nonce":4053209633},{"type":"v1","addr":"172.21.15.153:6803","nonce":4053209633}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6806","nonce":4053209633},{"type":"v1","addr":"172.21.15.153:6807","nonce":4053209633}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6804","nonce":4053209633},{"type":"v1","addr":"172.21.15.153:6805","nonce":4053209633}]},"public_addr":"172.21.15.153:6801/4053209633","cluster_addr":"172.21.15.153:6803/4053209633","heartbeat_back_addr":"172.21.15.153:6807/4053209633","heartbeat_front_addr":"172.21.15.153:6805/4053209633","state":["exists","up"]},{"osd":5,"uuid":"c69e2263-5dd8-402d-b901-295c1f4840e2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":35,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6808","nonce":2536485968},{"type":"v1","addr":"172.21.15.153:6809","nonce":2536485968}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6810","nonce":2536485968},{"type":"v1","addr":"172.21.15.153:6811","nonce":2536485968}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6814","nonce":2536485968},{"type":"v1","addr":"172.21.15.153:6815","nonce":2536485968}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6812","nonce":2536485968},{"type":"v1","addr":"172.21.15.153:6813","nonce":2536485968}]},"public_addr":"172.21.15.153:6809/2536485968","cluster_addr":"172.21.15.153:6811/2536485968","heartbeat_back_addr":"172.21.15.153:6815/2536485968","heartbeat_front_addr":"172.21.15.153:6813/2536485968","state":["exists","up"]},{"osd":6,"uuid":"72aa13b6-e7b7-449b-9fa8-f284e8f0735a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":40,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6816","nonce":3662388207},{"type":"v1","addr":"172.21.15.153:6817","nonce":3662388207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6818","nonce":3662388207},{"type":"v1","addr":"172.21.15.153:6819","nonce":3662388207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6822","nonce":3662388207},{"type":"v1","addr":"172.21.15.153:6823","nonce":3662388207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6820","nonce":3662388207},{"type":"v1","addr":"172.21.15.153:6821","nonce":3662388207}]},"public_addr":"172.21.15.153:6817/3662388207","cluster_addr":"172.21.15.153:6819/3662388207","heartbeat_back_addr":"172.21.15.153:6823/3662388207","heartbeat_front_addr":"172.21.15.153:6821/3662388207","state":["exists","up"]},{"osd":7,"uuid":"b5fdd156-d2dc-47fb-b44b-803275092ac0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":45,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6824","nonce":1223012606},{"type":"v1","addr":"172.21.15.153:6825","nonce":1223012606}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6826","nonce":1223012606},{"type":"v1","addr":"172.21.15.153:6827","nonce":1223012606}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6830","nonce":1223012606},{"type":"v1","addr":"172.21.15.153:6831","nonce":1223012606}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6828","nonce":1223012606},{"type":"v1","addr":"172.21.15.153:6829","nonce":1223012606}]},"public_addr":"172.21.15.153:6825/1223012606","cluster_addr":"172.21.15.153:6827/1223012606","heartbeat_back_addr":"172.21.15.153:6831/1223012606","heartbeat_front_addr":"172.21.15.153:6829/1223012606","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2022-09-29T05:22:43.518222+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2022-09-29T05:23:01.859214+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2022-09-29T05:23:20.325748+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2022-09-29T05:23:38.717068+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2022-09-29T05:23:54.756146+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2022-09-29T05:24:11.670499+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2022-09-29T05:24:28.641104+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2022-09-29T05:24:45.518895+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.83:0/2269625481":"2022-09-30T05:21:35.319810+0000","172.21.15.83:0/2173437316":"2022-09-30T05:21:35.319810+0000","172.21.15.83:6800/2702658771":"2022-09-30T05:21:35.319810+0000","172.21.15.83:0/4005486955":"2022-09-30T05:21:15.866011+0000","172.21.15.83:6800/2006042278":"2022-09-30T05:21:15.866011+0000","172.21.15.83:6801/2006042278":"2022-09-30T05:21:15.866011+0000","172.21.15.83:6801/2702658771":"2022-09-30T05:21:35.319810+0000","172.21.15.83:0/4256024697":"2022-09-30T05:21:15.866011+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2022-09-29T05:25:04.122 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2022-09-29T05:25:04.401 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:25:04.401 INFO:teuthology.orchestra.run.smithi083.stdout:{"epoch":50,"fsid":"69111522-3fb6-11ed-8431-001a4aab830c","created":"2022-09-29T05:20:45.037726+0000","modified":"2022-09-29T05:25:03.789266+0000","last_up_change":"2022-09-29T05:24:46.528124+0000","last_in_change":"2022-09-29T05:24:31.376431+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2022-09-29T05:23:34.795850+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"23","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2022-09-29T05:25:01.755735+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"50","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}}},{"pool":3,"pool_name":"cephfs_data","create_time":"2022-09-29T05:25:02.966273+0000","flags":32769,"flags_names":"hashpspool,creating","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"50","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}}],"osds":[{"osd":0,"uuid":"9b8b79ad-3d00-4dad-8da7-7cff0e65d302","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6802","nonce":2080129456},{"type":"v1","addr":"172.21.15.83:6803","nonce":2080129456}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6804","nonce":2080129456},{"type":"v1","addr":"172.21.15.83:6805","nonce":2080129456}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6808","nonce":2080129456},{"type":"v1","addr":"172.21.15.83:6809","nonce":2080129456}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6806","nonce":2080129456},{"type":"v1","addr":"172.21.15.83:6807","nonce":2080129456}]},"public_addr":"172.21.15.83:6803/2080129456","cluster_addr":"172.21.15.83:6805/2080129456","heartbeat_back_addr":"172.21.15.83:6809/2080129456","heartbeat_front_addr":"172.21.15.83:6807/2080129456","state":["exists","up"]},{"osd":1,"uuid":"e098cff5-70cc-437e-8759-e6c7e3d452d2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6810","nonce":605787053},{"type":"v1","addr":"172.21.15.83:6811","nonce":605787053}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6812","nonce":605787053},{"type":"v1","addr":"172.21.15.83:6813","nonce":605787053}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6816","nonce":605787053},{"type":"v1","addr":"172.21.15.83:6817","nonce":605787053}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6814","nonce":605787053},{"type":"v1","addr":"172.21.15.83:6815","nonce":605787053}]},"public_addr":"172.21.15.83:6811/605787053","cluster_addr":"172.21.15.83:6813/605787053","heartbeat_back_addr":"172.21.15.83:6817/605787053","heartbeat_front_addr":"172.21.15.83:6815/605787053","state":["exists","up"]},{"osd":2,"uuid":"2c14853b-5b64-496d-a011-5d8a9b17b731","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6818","nonce":1928326991},{"type":"v1","addr":"172.21.15.83:6819","nonce":1928326991}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6820","nonce":1928326991},{"type":"v1","addr":"172.21.15.83:6821","nonce":1928326991}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6824","nonce":1928326991},{"type":"v1","addr":"172.21.15.83:6825","nonce":1928326991}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6822","nonce":1928326991},{"type":"v1","addr":"172.21.15.83:6823","nonce":1928326991}]},"public_addr":"172.21.15.83:6819/1928326991","cluster_addr":"172.21.15.83:6821/1928326991","heartbeat_back_addr":"172.21.15.83:6825/1928326991","heartbeat_front_addr":"172.21.15.83:6823/1928326991","state":["exists","up"]},{"osd":3,"uuid":"7336f148-261e-468e-8d3c-28f840150f36","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6826","nonce":54976712},{"type":"v1","addr":"172.21.15.83:6827","nonce":54976712}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6828","nonce":54976712},{"type":"v1","addr":"172.21.15.83:6829","nonce":54976712}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6832","nonce":54976712},{"type":"v1","addr":"172.21.15.83:6833","nonce":54976712}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6830","nonce":54976712},{"type":"v1","addr":"172.21.15.83:6831","nonce":54976712}]},"public_addr":"172.21.15.83:6827/54976712","cluster_addr":"172.21.15.83:6829/54976712","heartbeat_back_addr":"172.21.15.83:6833/54976712","heartbeat_front_addr":"172.21.15.83:6831/54976712","state":["exists","up"]},{"osd":4,"uuid":"247d5c19-e8af-4620-a378-699dc2000db1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6800","nonce":4053209633},{"type":"v1","addr":"172.21.15.153:6801","nonce":4053209633}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6802","nonce":4053209633},{"type":"v1","addr":"172.21.15.153:6803","nonce":4053209633}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6806","nonce":4053209633},{"type":"v1","addr":"172.21.15.153:6807","nonce":4053209633}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6804","nonce":4053209633},{"type":"v1","addr":"172.21.15.153:6805","nonce":4053209633}]},"public_addr":"172.21.15.153:6801/4053209633","cluster_addr":"172.21.15.153:6803/4053209633","heartbeat_back_addr":"172.21.15.153:6807/4053209633","heartbeat_front_addr":"172.21.15.153:6805/4053209633","state":["exists","up"]},{"osd":5,"uuid":"c69e2263-5dd8-402d-b901-295c1f4840e2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":35,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6808","nonce":2536485968},{"type":"v1","addr":"172.21.15.153:6809","nonce":2536485968}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6810","nonce":2536485968},{"type":"v1","addr":"172.21.15.153:6811","nonce":2536485968}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6814","nonce":2536485968},{"type":"v1","addr":"172.21.15.153:6815","nonce":2536485968}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6812","nonce":2536485968},{"type":"v1","addr":"172.21.15.153:6813","nonce":2536485968}]},"public_addr":"172.21.15.153:6809/2536485968","cluster_addr":"172.21.15.153:6811/2536485968","heartbeat_back_addr":"172.21.15.153:6815/2536485968","heartbeat_front_addr":"172.21.15.153:6813/2536485968","state":["exists","up"]},{"osd":6,"uuid":"72aa13b6-e7b7-449b-9fa8-f284e8f0735a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":40,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6816","nonce":3662388207},{"type":"v1","addr":"172.21.15.153:6817","nonce":3662388207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6818","nonce":3662388207},{"type":"v1","addr":"172.21.15.153:6819","nonce":3662388207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6822","nonce":3662388207},{"type":"v1","addr":"172.21.15.153:6823","nonce":3662388207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6820","nonce":3662388207},{"type":"v1","addr":"172.21.15.153:6821","nonce":3662388207}]},"public_addr":"172.21.15.153:6817/3662388207","cluster_addr":"172.21.15.153:6819/3662388207","heartbeat_back_addr":"172.21.15.153:6823/3662388207","heartbeat_front_addr":"172.21.15.153:6821/3662388207","state":["exists","up"]},{"osd":7,"uuid":"b5fdd156-d2dc-47fb-b44b-803275092ac0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":45,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6824","nonce":1223012606},{"type":"v1","addr":"172.21.15.153:6825","nonce":1223012606}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6826","nonce":1223012606},{"type":"v1","addr":"172.21.15.153:6827","nonce":1223012606}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6830","nonce":1223012606},{"type":"v1","addr":"172.21.15.153:6831","nonce":1223012606}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6828","nonce":1223012606},{"type":"v1","addr":"172.21.15.153:6829","nonce":1223012606}]},"public_addr":"172.21.15.153:6825/1223012606","cluster_addr":"172.21.15.153:6827/1223012606","heartbeat_back_addr":"172.21.15.153:6831/1223012606","heartbeat_front_addr":"172.21.15.153:6829/1223012606","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2022-09-29T05:22:43.518222+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2022-09-29T05:23:01.859214+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2022-09-29T05:23:20.325748+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2022-09-29T05:23:38.717068+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2022-09-29T05:23:54.756146+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2022-09-29T05:24:11.670499+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2022-09-29T05:24:28.641104+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2022-09-29T05:24:45.518895+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.83:0/2269625481":"2022-09-30T05:21:35.319810+0000","172.21.15.83:0/2173437316":"2022-09-30T05:21:35.319810+0000","172.21.15.83:6800/2702658771":"2022-09-30T05:21:35.319810+0000","172.21.15.83:0/4005486955":"2022-09-30T05:21:15.866011+0000","172.21.15.83:6800/2006042278":"2022-09-30T05:21:15.866011+0000","172.21.15.83:6801/2006042278":"2022-09-30T05:21:15.866011+0000","172.21.15.83:6801/2702658771":"2022-09-30T05:21:35.319810+0000","172.21.15.83:0/4256024697":"2022-09-30T05:21:15.866011+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2022-09-29T05:25:04.416 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs set cephfs standby_count_wanted 0 2022-09-29T05:25:04.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:03.439622+0000 mon.a (mon.0) 546 : audit [INF] from='client.? 172.21.15.83:0/250632544' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]': finished 2022-09-29T05:25:04.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:25:03.439691+0000 mon.a (mon.0) 547 : cluster [DBG] osdmap e49: 8 total, 8 up, 8 in 2022-09-29T05:25:04.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:25:03.591409+0000 mgr.y (mgr.14146) 160 : cluster [DBG] pgmap v143: 97 pgs: 10 creating+peering, 86 unknown, 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:25:04.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:03.789155+0000 mon.a (mon.0) 548 : audit [INF] from='client.? 172.21.15.83:0/2964492618' entity='client.admin' cmd=[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]: dispatch 2022-09-29T05:25:04.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:25:03.789621+0000 mon.a (mon.0) 549 : cluster [INF] daemon mds.b assigned to filesystem cephfs as rank 0 2022-09-29T05:25:04.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:25:03.793806+0000 mon.a (mon.0) 550 : cluster [DBG] osdmap e50: 8 total, 8 up, 8 in 2022-09-29T05:25:04.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:03.794238+0000 mon.a (mon.0) 551 : audit [INF] from='client.? 172.21.15.83:0/2964492618' entity='client.admin' cmd='[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]': finished 2022-09-29T05:25:04.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:25:03.794310+0000 mon.a (mon.0) 552 : cluster [DBG] fsmap cephfs:1 {0=b=up:creating} 1 up:standby 2022-09-29T05:25:04.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:25:03.830557+0000 mon.a (mon.0) 553 : cluster [INF] daemon mds.b is now active in filesystem cephfs as rank 0 2022-09-29T05:25:04.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:04.105970+0000 mon.a (mon.0) 554 : audit [DBG] from='client.? 172.21.15.83:0/1238186036' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2022-09-29T05:25:04.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:04.397928+0000 mon.c (mon.1) 19 : audit [DBG] from='client.? 172.21.15.83:0/1845761390' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2022-09-29T05:25:04.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:03.439622+0000 mon.a (mon.0) 546 : audit [INF] from='client.? 172.21.15.83:0/250632544' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]': finished 2022-09-29T05:25:04.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:25:03.439691+0000 mon.a (mon.0) 547 : cluster [DBG] osdmap e49: 8 total, 8 up, 8 in 2022-09-29T05:25:04.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:25:03.591409+0000 mgr.y (mgr.14146) 160 : cluster [DBG] pgmap v143: 97 pgs: 10 creating+peering, 86 unknown, 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:25:04.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:03.789155+0000 mon.a (mon.0) 548 : audit [INF] from='client.? 172.21.15.83:0/2964492618' entity='client.admin' cmd=[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]: dispatch 2022-09-29T05:25:04.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:25:03.789621+0000 mon.a (mon.0) 549 : cluster [INF] daemon mds.b assigned to filesystem cephfs as rank 0 2022-09-29T05:25:04.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:25:03.793806+0000 mon.a (mon.0) 550 : cluster [DBG] osdmap e50: 8 total, 8 up, 8 in 2022-09-29T05:25:04.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:03.794238+0000 mon.a (mon.0) 551 : audit [INF] from='client.? 172.21.15.83:0/2964492618' entity='client.admin' cmd='[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]': finished 2022-09-29T05:25:04.718 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:25:03.794310+0000 mon.a (mon.0) 552 : cluster [DBG] fsmap cephfs:1 {0=b=up:creating} 1 up:standby 2022-09-29T05:25:04.718 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:25:03.830557+0000 mon.a (mon.0) 553 : cluster [INF] daemon mds.b is now active in filesystem cephfs as rank 0 2022-09-29T05:25:04.718 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:04.105970+0000 mon.a (mon.0) 554 : audit [DBG] from='client.? 172.21.15.83:0/1238186036' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2022-09-29T05:25:04.718 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:04.397928+0000 mon.c (mon.1) 19 : audit [DBG] from='client.? 172.21.15.83:0/1845761390' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2022-09-29T05:25:04.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:04 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:03.439622+0000 mon.a (mon.0) 546 : audit [INF] from='client.? 172.21.15.83:0/250632544' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]': finished 2022-09-29T05:25:04.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:04 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:25:03.439691+0000 mon.a (mon.0) 547 : cluster [DBG] osdmap e49: 8 total, 8 up, 8 in 2022-09-29T05:25:04.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:04 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:25:03.591409+0000 mgr.y (mgr.14146) 160 : cluster [DBG] pgmap v143: 97 pgs: 10 creating+peering, 86 unknown, 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:25:04.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:04 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:03.789155+0000 mon.a (mon.0) 548 : audit [INF] from='client.? 172.21.15.83:0/2964492618' entity='client.admin' cmd=[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]: dispatch 2022-09-29T05:25:04.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:04 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:25:03.789621+0000 mon.a (mon.0) 549 : cluster [INF] daemon mds.b assigned to filesystem cephfs as rank 0 2022-09-29T05:25:04.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:04 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:25:03.793806+0000 mon.a (mon.0) 550 : cluster [DBG] osdmap e50: 8 total, 8 up, 8 in 2022-09-29T05:25:04.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:04 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:03.794238+0000 mon.a (mon.0) 551 : audit [INF] from='client.? 172.21.15.83:0/2964492618' entity='client.admin' cmd='[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]': finished 2022-09-29T05:25:04.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:04 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:25:03.794310+0000 mon.a (mon.0) 552 : cluster [DBG] fsmap cephfs:1 {0=b=up:creating} 1 up:standby 2022-09-29T05:25:04.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:04 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:25:03.830557+0000 mon.a (mon.0) 553 : cluster [INF] daemon mds.b is now active in filesystem cephfs as rank 0 2022-09-29T05:25:04.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:04 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:04.105970+0000 mon.a (mon.0) 554 : audit [DBG] from='client.? 172.21.15.83:0/1238186036' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2022-09-29T05:25:04.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:04 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:04.397928+0000 mon.c (mon.1) 19 : audit [DBG] from='client.? 172.21.15.83:0/1845761390' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2022-09-29T05:25:04.813 DEBUG:tasks.cephfs.filesystem:fs_config: {} 2022-09-29T05:25:04.813 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs set cephfs allow_standby_replay false 2022-09-29T05:25:05.309 INFO:journalctl@ceph.mds.b.smithi153.stdout:Sep 29 05:25:04 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mds-b[53085]: debug 2022-09-29T05:25:04.839+0000 7f391d3c1700 -1 mds.pinger is_rank_lagging: rank=0 was never sent ping request. 2022-09-29T05:25:05.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:04.737902+0000 mon.a (mon.0) 555 : audit [INF] from='client.? 172.21.15.83:0/2517039209' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]: dispatch 2022-09-29T05:25:05.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:25:04.794340+0000 mon.a (mon.0) 556 : cluster [DBG] mds.? [v2:172.21.15.153:6832/673217382,v1:172.21.15.153:6833/673217382] up:active 2022-09-29T05:25:05.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:04.794402+0000 mon.a (mon.0) 557 : audit [INF] from='client.? 172.21.15.83:0/2517039209' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]': finished 2022-09-29T05:25:05.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:25:04.794459+0000 mon.a (mon.0) 558 : cluster [DBG] fsmap cephfs:1 {0=b=up:active} 1 up:standby 2022-09-29T05:25:05.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:25:04.799762+0000 mon.a (mon.0) 559 : cluster [DBG] osdmap e51: 8 total, 8 up, 8 in 2022-09-29T05:25:05.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:05.132670+0000 mon.c (mon.1) 20 : audit [INF] from='client.? 172.21.15.83:0/3401996256' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "false"}]: dispatch 2022-09-29T05:25:05.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:05.133014+0000 mon.a (mon.0) 560 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "false"}]: dispatch 2022-09-29T05:25:05.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:04.737902+0000 mon.a (mon.0) 555 : audit [INF] from='client.? 172.21.15.83:0/2517039209' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]: dispatch 2022-09-29T05:25:05.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:25:04.794340+0000 mon.a (mon.0) 556 : cluster [DBG] mds.? [v2:172.21.15.153:6832/673217382,v1:172.21.15.153:6833/673217382] up:active 2022-09-29T05:25:05.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:04.794402+0000 mon.a (mon.0) 557 : audit [INF] from='client.? 172.21.15.83:0/2517039209' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]': finished 2022-09-29T05:25:05.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:25:04.794459+0000 mon.a (mon.0) 558 : cluster [DBG] fsmap cephfs:1 {0=b=up:active} 1 up:standby 2022-09-29T05:25:05.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:25:04.799762+0000 mon.a (mon.0) 559 : cluster [DBG] osdmap e51: 8 total, 8 up, 8 in 2022-09-29T05:25:05.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:05.132670+0000 mon.c (mon.1) 20 : audit [INF] from='client.? 172.21.15.83:0/3401996256' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "false"}]: dispatch 2022-09-29T05:25:05.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:05.133014+0000 mon.a (mon.0) 560 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "false"}]: dispatch 2022-09-29T05:25:05.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:05 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:04.737902+0000 mon.a (mon.0) 555 : audit [INF] from='client.? 172.21.15.83:0/2517039209' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]: dispatch 2022-09-29T05:25:05.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:05 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:25:04.794340+0000 mon.a (mon.0) 556 : cluster [DBG] mds.? [v2:172.21.15.153:6832/673217382,v1:172.21.15.153:6833/673217382] up:active 2022-09-29T05:25:05.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:05 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:04.794402+0000 mon.a (mon.0) 557 : audit [INF] from='client.? 172.21.15.83:0/2517039209' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]': finished 2022-09-29T05:25:05.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:05 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:25:04.794459+0000 mon.a (mon.0) 558 : cluster [DBG] fsmap cephfs:1 {0=b=up:active} 1 up:standby 2022-09-29T05:25:05.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:05 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:25:04.799762+0000 mon.a (mon.0) 559 : cluster [DBG] osdmap e51: 8 total, 8 up, 8 in 2022-09-29T05:25:05.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:05 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:05.132670+0000 mon.c (mon.1) 20 : audit [INF] from='client.? 172.21.15.83:0/3401996256' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "false"}]: dispatch 2022-09-29T05:25:05.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:05 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:05.133014+0000 mon.a (mon.0) 560 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "false"}]: dispatch 2022-09-29T05:25:05.836 DEBUG:tasks.cephfs.filesystem:Not Creating any subvols for filesystem 'cephfs' 2022-09-29T05:25:05.837 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2022-09-29T05:25:06.152 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:25:06.152 INFO:teuthology.orchestra.run.smithi083.stdout:{"epoch":6,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":24343,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.83:6835/4236763119","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6834","nonce":4236763119},{"type":"v1","addr":"172.21.15.83:6835","nonce":4236763119}]},"join_fscid":-1,"export_targets":[],"features":4540138297136906239,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2}],"filesystems":[{"mdsmap":{"epoch":6,"flags":18,"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2022-09-29T05:25:03.789615+0000","modified":"2022-09-29T05:25:05.816761+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"max_mds":1,"in":[0],"up":{"mds_0":24356},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24356":{"gid":24356,"name":"b","rank":0,"incarnation":4,"state":"up:active","state_seq":3,"addr":"172.21.15.153:6833/673217382","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6832","nonce":673217382},{"type":"v1","addr":"172.21.15.153:6833","nonce":673217382}]},"join_fscid":-1,"export_targets":[],"features":4540138297136906239,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","standby_count_wanted":0},"id":1}]} 2022-09-29T05:25:06.154 INFO:teuthology.orchestra.run.smithi083.stderr:dumped fsmap epoch 6 2022-09-29T05:25:06.165 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2022-09-29T05:25:06.449 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:25:06.450 INFO:teuthology.orchestra.run.smithi083.stdout:{"epoch":51,"fsid":"69111522-3fb6-11ed-8431-001a4aab830c","created":"2022-09-29T05:20:45.037726+0000","modified":"2022-09-29T05:25:04.791432+0000","last_up_change":"2022-09-29T05:24:46.528124+0000","last_in_change":"2022-09-29T05:24:31.376431+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2022-09-29T05:23:34.795850+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"23","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2022-09-29T05:25:01.755735+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"50","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}}},{"pool":3,"pool_name":"cephfs_data","create_time":"2022-09-29T05:25:02.966273+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"51","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}}],"osds":[{"osd":0,"uuid":"9b8b79ad-3d00-4dad-8da7-7cff0e65d302","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6802","nonce":2080129456},{"type":"v1","addr":"172.21.15.83:6803","nonce":2080129456}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6804","nonce":2080129456},{"type":"v1","addr":"172.21.15.83:6805","nonce":2080129456}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6808","nonce":2080129456},{"type":"v1","addr":"172.21.15.83:6809","nonce":2080129456}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6806","nonce":2080129456},{"type":"v1","addr":"172.21.15.83:6807","nonce":2080129456}]},"public_addr":"172.21.15.83:6803/2080129456","cluster_addr":"172.21.15.83:6805/2080129456","heartbeat_back_addr":"172.21.15.83:6809/2080129456","heartbeat_front_addr":"172.21.15.83:6807/2080129456","state":["exists","up"]},{"osd":1,"uuid":"e098cff5-70cc-437e-8759-e6c7e3d452d2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6810","nonce":605787053},{"type":"v1","addr":"172.21.15.83:6811","nonce":605787053}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6812","nonce":605787053},{"type":"v1","addr":"172.21.15.83:6813","nonce":605787053}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6816","nonce":605787053},{"type":"v1","addr":"172.21.15.83:6817","nonce":605787053}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6814","nonce":605787053},{"type":"v1","addr":"172.21.15.83:6815","nonce":605787053}]},"public_addr":"172.21.15.83:6811/605787053","cluster_addr":"172.21.15.83:6813/605787053","heartbeat_back_addr":"172.21.15.83:6817/605787053","heartbeat_front_addr":"172.21.15.83:6815/605787053","state":["exists","up"]},{"osd":2,"uuid":"2c14853b-5b64-496d-a011-5d8a9b17b731","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6818","nonce":1928326991},{"type":"v1","addr":"172.21.15.83:6819","nonce":1928326991}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6820","nonce":1928326991},{"type":"v1","addr":"172.21.15.83:6821","nonce":1928326991}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6824","nonce":1928326991},{"type":"v1","addr":"172.21.15.83:6825","nonce":1928326991}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6822","nonce":1928326991},{"type":"v1","addr":"172.21.15.83:6823","nonce":1928326991}]},"public_addr":"172.21.15.83:6819/1928326991","cluster_addr":"172.21.15.83:6821/1928326991","heartbeat_back_addr":"172.21.15.83:6825/1928326991","heartbeat_front_addr":"172.21.15.83:6823/1928326991","state":["exists","up"]},{"osd":3,"uuid":"7336f148-261e-468e-8d3c-28f840150f36","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6826","nonce":54976712},{"type":"v1","addr":"172.21.15.83:6827","nonce":54976712}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6828","nonce":54976712},{"type":"v1","addr":"172.21.15.83:6829","nonce":54976712}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6832","nonce":54976712},{"type":"v1","addr":"172.21.15.83:6833","nonce":54976712}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6830","nonce":54976712},{"type":"v1","addr":"172.21.15.83:6831","nonce":54976712}]},"public_addr":"172.21.15.83:6827/54976712","cluster_addr":"172.21.15.83:6829/54976712","heartbeat_back_addr":"172.21.15.83:6833/54976712","heartbeat_front_addr":"172.21.15.83:6831/54976712","state":["exists","up"]},{"osd":4,"uuid":"247d5c19-e8af-4620-a378-699dc2000db1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6800","nonce":4053209633},{"type":"v1","addr":"172.21.15.153:6801","nonce":4053209633}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6802","nonce":4053209633},{"type":"v1","addr":"172.21.15.153:6803","nonce":4053209633}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6806","nonce":4053209633},{"type":"v1","addr":"172.21.15.153:6807","nonce":4053209633}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6804","nonce":4053209633},{"type":"v1","addr":"172.21.15.153:6805","nonce":4053209633}]},"public_addr":"172.21.15.153:6801/4053209633","cluster_addr":"172.21.15.153:6803/4053209633","heartbeat_back_addr":"172.21.15.153:6807/4053209633","heartbeat_front_addr":"172.21.15.153:6805/4053209633","state":["exists","up"]},{"osd":5,"uuid":"c69e2263-5dd8-402d-b901-295c1f4840e2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":35,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6808","nonce":2536485968},{"type":"v1","addr":"172.21.15.153:6809","nonce":2536485968}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6810","nonce":2536485968},{"type":"v1","addr":"172.21.15.153:6811","nonce":2536485968}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6814","nonce":2536485968},{"type":"v1","addr":"172.21.15.153:6815","nonce":2536485968}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6812","nonce":2536485968},{"type":"v1","addr":"172.21.15.153:6813","nonce":2536485968}]},"public_addr":"172.21.15.153:6809/2536485968","cluster_addr":"172.21.15.153:6811/2536485968","heartbeat_back_addr":"172.21.15.153:6815/2536485968","heartbeat_front_addr":"172.21.15.153:6813/2536485968","state":["exists","up"]},{"osd":6,"uuid":"72aa13b6-e7b7-449b-9fa8-f284e8f0735a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":40,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6816","nonce":3662388207},{"type":"v1","addr":"172.21.15.153:6817","nonce":3662388207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6818","nonce":3662388207},{"type":"v1","addr":"172.21.15.153:6819","nonce":3662388207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6822","nonce":3662388207},{"type":"v1","addr":"172.21.15.153:6823","nonce":3662388207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6820","nonce":3662388207},{"type":"v1","addr":"172.21.15.153:6821","nonce":3662388207}]},"public_addr":"172.21.15.153:6817/3662388207","cluster_addr":"172.21.15.153:6819/3662388207","heartbeat_back_addr":"172.21.15.153:6823/3662388207","heartbeat_front_addr":"172.21.15.153:6821/3662388207","state":["exists","up"]},{"osd":7,"uuid":"b5fdd156-d2dc-47fb-b44b-803275092ac0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":45,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6824","nonce":1223012606},{"type":"v1","addr":"172.21.15.153:6825","nonce":1223012606}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6826","nonce":1223012606},{"type":"v1","addr":"172.21.15.153:6827","nonce":1223012606}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6830","nonce":1223012606},{"type":"v1","addr":"172.21.15.153:6831","nonce":1223012606}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6828","nonce":1223012606},{"type":"v1","addr":"172.21.15.153:6829","nonce":1223012606}]},"public_addr":"172.21.15.153:6825/1223012606","cluster_addr":"172.21.15.153:6827/1223012606","heartbeat_back_addr":"172.21.15.153:6831/1223012606","heartbeat_front_addr":"172.21.15.153:6829/1223012606","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2022-09-29T05:22:43.518222+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2022-09-29T05:23:01.859214+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2022-09-29T05:23:20.325748+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2022-09-29T05:23:38.717068+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2022-09-29T05:23:54.756146+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2022-09-29T05:24:11.670499+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2022-09-29T05:24:28.641104+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2022-09-29T05:24:45.518895+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.83:0/2269625481":"2022-09-30T05:21:35.319810+0000","172.21.15.83:0/2173437316":"2022-09-30T05:21:35.319810+0000","172.21.15.83:6800/2702658771":"2022-09-30T05:21:35.319810+0000","172.21.15.83:0/4005486955":"2022-09-30T05:21:15.866011+0000","172.21.15.83:6800/2006042278":"2022-09-30T05:21:15.866011+0000","172.21.15.83:6801/2006042278":"2022-09-30T05:21:15.866011+0000","172.21.15.83:6801/2702658771":"2022-09-30T05:21:35.319810+0000","172.21.15.83:0/4256024697":"2022-09-30T05:21:15.866011+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2022-09-29T05:25:06.463 INFO:tasks.cephfs.filesystem.ceph_manager:waiting for clean 2022-09-29T05:25:06.463 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2022-09-29T05:25:06.753 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:25:06.757 INFO:teuthology.orchestra.run.smithi083.stderr:dumped all 2022-09-29T05:25:06.761 INFO:teuthology.orchestra.run.smithi083.stdout:{"pg_ready":true,"pg_map":{"version":146,"stamp":"2022-09-29T05:25:05.591588+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1696,"num_objects":8,"num_object_clones":0,"num_object_copies":24,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14,"num_write_kb":5,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":13,"ondisk_log_size":13,"up":156,"acting":156,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":749961216,"kb_used":43412,"kb_used_data":3604,"kb_used_omap":0,"kb_used_meta":39744,"kb_avail":749917804,"statfs":{"total":767960285184,"available":767915831296,"internally_reserved":0,"allocated":3690496,"data_stored":1460340,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":40697856},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1536,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"0.794136"},"pg_stats":[{"pgid":"3.3f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.3e","version":"0'0","reported_seq":9,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.816168+0000","last_change":"2022-09-29T05:25:03.816168+0000","last_active":"2022-09-29T05:25:03.816168+0000","last_peered":"2022-09-29T05:25:03.816168+0000","last_clean":"2022-09-29T05:25:03.816168+0000","last_became_active":"2022-09-29T05:25:03.815888+0000","last_became_peered":"2022-09-29T05:25:03.815888+0000","last_unstale":"2022-09-29T05:25:03.816168+0000","last_undegraded":"2022-09-29T05:25:03.816168+0000","last_fullsized":"2022-09-29T05:25:03.816168+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3d","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.3c","version":"0'0","reported_seq":9,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.819554+0000","last_change":"2022-09-29T05:25:03.819554+0000","last_active":"2022-09-29T05:25:03.819554+0000","last_peered":"2022-09-29T05:25:03.819554+0000","last_clean":"2022-09-29T05:25:03.819554+0000","last_became_active":"2022-09-29T05:25:03.819207+0000","last_became_peered":"2022-09-29T05:25:03.819207+0000","last_unstale":"2022-09-29T05:25:03.819554+0000","last_undegraded":"2022-09-29T05:25:03.819554+0000","last_fullsized":"2022-09-29T05:25:03.819554+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3b","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.3a","version":"0'0","reported_seq":9,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.817871+0000","last_change":"2022-09-29T05:25:03.817871+0000","last_active":"2022-09-29T05:25:03.817871+0000","last_peered":"2022-09-29T05:25:03.817871+0000","last_clean":"2022-09-29T05:25:03.817871+0000","last_became_active":"2022-09-29T05:25:03.816800+0000","last_became_peered":"2022-09-29T05:25:03.816800+0000","last_unstale":"2022-09-29T05:25:03.817871+0000","last_undegraded":"2022-09-29T05:25:03.817871+0000","last_fullsized":"2022-09-29T05:25:03.817871+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.39","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.38","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.37","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.36","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.825144+0000","last_change":"2022-09-29T05:25:03.823656+0000","last_active":"2022-09-29T05:25:03.825144+0000","last_peered":"2022-09-29T05:25:03.825144+0000","last_clean":"2022-09-29T05:25:03.825144+0000","last_became_active":"2022-09-29T05:25:03.823109+0000","last_became_peered":"2022-09-29T05:25:03.823109+0000","last_unstale":"2022-09-29T05:25:03.825144+0000","last_undegraded":"2022-09-29T05:25:03.825144+0000","last_fullsized":"2022-09-29T05:25:03.825144+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.35","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.825174+0000","last_change":"2022-09-29T05:25:03.824377+0000","last_active":"2022-09-29T05:25:03.825174+0000","last_peered":"2022-09-29T05:25:03.825174+0000","last_clean":"2022-09-29T05:25:03.825174+0000","last_became_active":"2022-09-29T05:25:03.823869+0000","last_became_peered":"2022-09-29T05:25:03.823869+0000","last_unstale":"2022-09-29T05:25:03.825174+0000","last_undegraded":"2022-09-29T05:25:03.825174+0000","last_fullsized":"2022-09-29T05:25:03.825174+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.34","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.33","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.32","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.31","version":"0'0","reported_seq":9,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.805590+0000","last_change":"2022-09-29T05:25:03.805590+0000","last_active":"2022-09-29T05:25:03.805590+0000","last_peered":"2022-09-29T05:25:03.805590+0000","last_clean":"2022-09-29T05:25:03.805590+0000","last_became_active":"2022-09-29T05:25:03.805310+0000","last_became_peered":"2022-09-29T05:25:03.805310+0000","last_unstale":"2022-09-29T05:25:03.805590+0000","last_undegraded":"2022-09-29T05:25:03.805590+0000","last_fullsized":"2022-09-29T05:25:03.805590+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.30","version":"0'0","reported_seq":9,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.815886+0000","last_change":"2022-09-29T05:25:03.815886+0000","last_active":"2022-09-29T05:25:03.815886+0000","last_peered":"2022-09-29T05:25:03.815886+0000","last_clean":"2022-09-29T05:25:03.815886+0000","last_became_active":"2022-09-29T05:25:03.815285+0000","last_became_peered":"2022-09-29T05:25:03.815285+0000","last_unstale":"2022-09-29T05:25:03.815886+0000","last_undegraded":"2022-09-29T05:25:03.815886+0000","last_fullsized":"2022-09-29T05:25:03.815886+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2f","version":"0'0","reported_seq":9,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.813784+0000","last_change":"2022-09-29T05:25:03.813784+0000","last_active":"2022-09-29T05:25:03.813784+0000","last_peered":"2022-09-29T05:25:03.813784+0000","last_clean":"2022-09-29T05:25:03.813784+0000","last_became_active":"2022-09-29T05:25:03.813420+0000","last_became_peered":"2022-09-29T05:25:03.813420+0000","last_unstale":"2022-09-29T05:25:03.813784+0000","last_undegraded":"2022-09-29T05:25:03.813784+0000","last_fullsized":"2022-09-29T05:25:03.813784+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2e","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.2d","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.1f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:02.423315+0000","last_change":"2022-09-29T05:25:02.423315+0000","last_active":"2022-09-29T05:25:02.423315+0000","last_peered":"2022-09-29T05:25:02.423315+0000","last_clean":"2022-09-29T05:25:02.423315+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:02.423315+0000","last_undegraded":"2022-09-29T05:25:02.423315+0000","last_fullsized":"2022-09-29T05:25:02.423315+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.1e","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.1e","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:02.423315+0000","last_change":"2022-09-29T05:25:02.423315+0000","last_active":"2022-09-29T05:25:02.423315+0000","last_peered":"2022-09-29T05:25:02.423315+0000","last_clean":"2022-09-29T05:25:02.423315+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:02.423315+0000","last_undegraded":"2022-09-29T05:25:02.423315+0000","last_fullsized":"2022-09-29T05:25:02.423315+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.1f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.1d","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2022-09-29T05:25:02.423315+0000","last_change":"2022-09-29T05:25:02.433891+0000","last_active":"2022-09-29T05:25:02.423315+0000","last_peered":"2022-09-29T05:25:02.423315+0000","last_clean":"2022-09-29T05:25:02.423315+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:02.423315+0000","last_undegraded":"2022-09-29T05:25:02.423315+0000","last_fullsized":"2022-09-29T05:25:02.423315+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,6],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"0'0","reported_seq":9,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.816581+0000","last_change":"2022-09-29T05:25:03.816581+0000","last_active":"2022-09-29T05:25:03.816581+0000","last_peered":"2022-09-29T05:25:03.816581+0000","last_clean":"2022-09-29T05:25:03.816581+0000","last_became_active":"2022-09-29T05:25:03.816206+0000","last_became_peered":"2022-09-29T05:25:03.816206+0000","last_unstale":"2022-09-29T05:25:03.816581+0000","last_undegraded":"2022-09-29T05:25:03.816581+0000","last_fullsized":"2022-09-29T05:25:03.816581+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1c","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2022-09-29T05:25:02.423315+0000","last_change":"2022-09-29T05:25:02.433333+0000","last_active":"2022-09-29T05:25:02.423315+0000","last_peered":"2022-09-29T05:25:02.423315+0000","last_clean":"2022-09-29T05:25:02.423315+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:02.423315+0000","last_undegraded":"2022-09-29T05:25:02.423315+0000","last_fullsized":"2022-09-29T05:25:02.423315+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[2,5],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1d","version":"0'0","reported_seq":9,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.820523+0000","last_change":"2022-09-29T05:25:03.820523+0000","last_active":"2022-09-29T05:25:03.820523+0000","last_peered":"2022-09-29T05:25:03.820523+0000","last_clean":"2022-09-29T05:25:03.820523+0000","last_became_active":"2022-09-29T05:25:03.819546+0000","last_became_peered":"2022-09-29T05:25:03.819546+0000","last_unstale":"2022-09-29T05:25:03.820523+0000","last_undegraded":"2022-09-29T05:25:03.820523+0000","last_fullsized":"2022-09-29T05:25:03.820523+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1b","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:02.423315+0000","last_change":"2022-09-29T05:25:02.423315+0000","last_active":"2022-09-29T05:25:02.423315+0000","last_peered":"2022-09-29T05:25:02.423315+0000","last_clean":"2022-09-29T05:25:02.423315+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:02.423315+0000","last_undegraded":"2022-09-29T05:25:02.423315+0000","last_fullsized":"2022-09-29T05:25:02.423315+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.1a","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.1a","version":"50'1","reported_seq":12,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.815499+0000","last_change":"2022-09-29T05:25:03.454747+0000","last_active":"2022-09-29T05:25:03.815499+0000","last_peered":"2022-09-29T05:25:03.815499+0000","last_clean":"2022-09-29T05:25:03.815499+0000","last_became_active":"2022-09-29T05:25:03.454395+0000","last_became_peered":"2022-09-29T05:25:03.454395+0000","last_unstale":"2022-09-29T05:25:03.815499+0000","last_undegraded":"2022-09-29T05:25:03.815499+0000","last_fullsized":"2022-09-29T05:25:03.815499+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1b","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.19","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:02.423315+0000","last_change":"2022-09-29T05:25:02.423315+0000","last_active":"2022-09-29T05:25:02.423315+0000","last_peered":"2022-09-29T05:25:02.423315+0000","last_clean":"2022-09-29T05:25:02.423315+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:02.423315+0000","last_undegraded":"2022-09-29T05:25:02.423315+0000","last_fullsized":"2022-09-29T05:25:02.423315+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.18","version":"0'0","reported_seq":10,"reported_epoch":49,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.464050+0000","last_change":"2022-09-29T05:25:03.460675+0000","last_active":"2022-09-29T05:25:03.464050+0000","last_peered":"2022-09-29T05:25:03.464050+0000","last_clean":"2022-09-29T05:25:03.464050+0000","last_became_active":"2022-09-29T05:25:03.460351+0000","last_became_peered":"2022-09-29T05:25:03.460351+0000","last_unstale":"2022-09-29T05:25:03.464050+0000","last_undegraded":"2022-09-29T05:25:03.464050+0000","last_fullsized":"2022-09-29T05:25:03.464050+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":9,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.816170+0000","last_change":"2022-09-29T05:25:03.816170+0000","last_active":"2022-09-29T05:25:03.816170+0000","last_peered":"2022-09-29T05:25:03.816170+0000","last_clean":"2022-09-29T05:25:03.816170+0000","last_became_active":"2022-09-29T05:25:03.815838+0000","last_became_peered":"2022-09-29T05:25:03.815838+0000","last_unstale":"2022-09-29T05:25:03.816170+0000","last_undegraded":"2022-09-29T05:25:03.816170+0000","last_fullsized":"2022-09-29T05:25:03.816170+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.17","version":"0'0","reported_seq":11,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.805981+0000","last_change":"2022-09-29T05:25:03.461045+0000","last_active":"2022-09-29T05:25:03.805981+0000","last_peered":"2022-09-29T05:25:03.805981+0000","last_clean":"2022-09-29T05:25:03.805981+0000","last_became_active":"2022-09-29T05:25:03.460729+0000","last_became_peered":"2022-09-29T05:25:03.460729+0000","last_unstale":"2022-09-29T05:25:03.805981+0000","last_undegraded":"2022-09-29T05:25:03.805981+0000","last_fullsized":"2022-09-29T05:25:03.805981+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":9,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.818733+0000","last_change":"2022-09-29T05:25:03.818733+0000","last_active":"2022-09-29T05:25:03.818733+0000","last_peered":"2022-09-29T05:25:03.818733+0000","last_clean":"2022-09-29T05:25:03.818733+0000","last_became_active":"2022-09-29T05:25:03.816983+0000","last_became_peered":"2022-09-29T05:25:03.816983+0000","last_unstale":"2022-09-29T05:25:03.818733+0000","last_undegraded":"2022-09-29T05:25:03.818733+0000","last_fullsized":"2022-09-29T05:25:03.818733+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.16","version":"0'0","reported_seq":10,"reported_epoch":49,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.464237+0000","last_change":"2022-09-29T05:25:03.463477+0000","last_active":"2022-09-29T05:25:03.464237+0000","last_peered":"2022-09-29T05:25:03.464237+0000","last_clean":"2022-09-29T05:25:03.464237+0000","last_became_active":"2022-09-29T05:25:03.463051+0000","last_became_peered":"2022-09-29T05:25:03.463051+0000","last_unstale":"2022-09-29T05:25:03.464237+0000","last_undegraded":"2022-09-29T05:25:03.464237+0000","last_fullsized":"2022-09-29T05:25:03.464237+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.15","version":"50'3","reported_seq":13,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.811766+0000","last_change":"2022-09-29T05:25:03.457077+0000","last_active":"2022-09-29T05:25:03.811766+0000","last_peered":"2022-09-29T05:25:03.811766+0000","last_clean":"2022-09-29T05:25:03.811766+0000","last_became_active":"2022-09-29T05:25:03.456823+0000","last_became_peered":"2022-09-29T05:25:03.456823+0000","last_unstale":"2022-09-29T05:25:03.811766+0000","last_undegraded":"2022-09-29T05:25:03.811766+0000","last_fullsized":"2022-09-29T05:25:03.811766+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":3,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.14","version":"50'6","reported_seq":17,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.827688+0000","last_change":"2022-09-29T05:25:03.461139+0000","last_active":"2022-09-29T05:25:03.827688+0000","last_peered":"2022-09-29T05:25:03.827688+0000","last_clean":"2022-09-29T05:25:03.827688+0000","last_became_active":"2022-09-29T05:25:03.460829+0000","last_became_peered":"2022-09-29T05:25:03.460829+0000","last_unstale":"2022-09-29T05:25:03.827688+0000","last_undegraded":"2022-09-29T05:25:03.827688+0000","last_fullsized":"2022-09-29T05:25:03.827688+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":6,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":1536,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.13","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:02.423315+0000","last_change":"2022-09-29T05:25:02.423315+0000","last_active":"2022-09-29T05:25:02.423315+0000","last_peered":"2022-09-29T05:25:02.423315+0000","last_clean":"2022-09-29T05:25:02.423315+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:02.423315+0000","last_undegraded":"2022-09-29T05:25:02.423315+0000","last_fullsized":"2022-09-29T05:25:02.423315+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.12","version":"0'0","reported_seq":10,"reported_epoch":49,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.464191+0000","last_change":"2022-09-29T05:25:03.462145+0000","last_active":"2022-09-29T05:25:03.464191+0000","last_peered":"2022-09-29T05:25:03.464191+0000","last_clean":"2022-09-29T05:25:03.464191+0000","last_became_active":"2022-09-29T05:25:03.461864+0000","last_became_peered":"2022-09-29T05:25:03.461864+0000","last_unstale":"2022-09-29T05:25:03.464191+0000","last_undegraded":"2022-09-29T05:25:03.464191+0000","last_fullsized":"2022-09-29T05:25:03.464191+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.11","version":"0'0","reported_seq":11,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.805505+0000","last_change":"2022-09-29T05:25:03.454654+0000","last_active":"2022-09-29T05:25:03.805505+0000","last_peered":"2022-09-29T05:25:03.805505+0000","last_clean":"2022-09-29T05:25:03.805505+0000","last_became_active":"2022-09-29T05:25:03.454319+0000","last_became_peered":"2022-09-29T05:25:03.454319+0000","last_unstale":"2022-09-29T05:25:03.805505+0000","last_undegraded":"2022-09-29T05:25:03.805505+0000","last_fullsized":"2022-09-29T05:25:03.805505+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.813586+0000","last_change":"2022-09-29T05:25:03.811425+0000","last_active":"2022-09-29T05:25:03.813586+0000","last_peered":"2022-09-29T05:25:03.813586+0000","last_clean":"2022-09-29T05:25:03.813586+0000","last_became_active":"2022-09-29T05:25:03.811120+0000","last_became_peered":"2022-09-29T05:25:03.811120+0000","last_unstale":"2022-09-29T05:25:03.813586+0000","last_undegraded":"2022-09-29T05:25:03.813586+0000","last_fullsized":"2022-09-29T05:25:03.813586+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.10","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2022-09-29T05:25:02.423315+0000","last_change":"2022-09-29T05:25:02.439448+0000","last_active":"2022-09-29T05:25:02.423315+0000","last_peered":"2022-09-29T05:25:02.423315+0000","last_clean":"2022-09-29T05:25:02.423315+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:02.423315+0000","last_undegraded":"2022-09-29T05:25:02.423315+0000","last_fullsized":"2022-09-29T05:25:02.423315+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,1],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:02.423315+0000","last_change":"2022-09-29T05:25:02.423315+0000","last_active":"2022-09-29T05:25:02.423315+0000","last_peered":"2022-09-29T05:25:02.423315+0000","last_clean":"2022-09-29T05:25:02.423315+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:02.423315+0000","last_undegraded":"2022-09-29T05:25:02.423315+0000","last_fullsized":"2022-09-29T05:25:02.423315+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.e","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":10,"reported_epoch":49,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.454814+0000","last_change":"2022-09-29T05:25:03.454507+0000","last_active":"2022-09-29T05:25:03.454814+0000","last_peered":"2022-09-29T05:25:03.454814+0000","last_clean":"2022-09-29T05:25:03.454814+0000","last_became_active":"2022-09-29T05:25:03.454195+0000","last_became_peered":"2022-09-29T05:25:03.454195+0000","last_unstale":"2022-09-29T05:25:03.454814+0000","last_undegraded":"2022-09-29T05:25:03.454814+0000","last_fullsized":"2022-09-29T05:25:03.454814+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":9,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.819862+0000","last_change":"2022-09-29T05:25:03.819862+0000","last_active":"2022-09-29T05:25:03.819862+0000","last_peered":"2022-09-29T05:25:03.819862+0000","last_clean":"2022-09-29T05:25:03.819862+0000","last_became_active":"2022-09-29T05:25:03.818117+0000","last_became_peered":"2022-09-29T05:25:03.818117+0000","last_unstale":"2022-09-29T05:25:03.819862+0000","last_undegraded":"2022-09-29T05:25:03.819862+0000","last_fullsized":"2022-09-29T05:25:03.819862+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":10,"reported_epoch":49,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.463942+0000","last_change":"2022-09-29T05:25:03.459646+0000","last_active":"2022-09-29T05:25:03.463942+0000","last_peered":"2022-09-29T05:25:03.463942+0000","last_clean":"2022-09-29T05:25:03.463942+0000","last_became_active":"2022-09-29T05:25:03.459320+0000","last_became_peered":"2022-09-29T05:25:03.459320+0000","last_unstale":"2022-09-29T05:25:03.463942+0000","last_undegraded":"2022-09-29T05:25:03.463942+0000","last_fullsized":"2022-09-29T05:25:03.463942+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":9,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.812390+0000","last_change":"2022-09-29T05:25:03.812390+0000","last_active":"2022-09-29T05:25:03.812390+0000","last_peered":"2022-09-29T05:25:03.812390+0000","last_clean":"2022-09-29T05:25:03.812390+0000","last_became_active":"2022-09-29T05:25:03.811925+0000","last_became_peered":"2022-09-29T05:25:03.811925+0000","last_unstale":"2022-09-29T05:25:03.812390+0000","last_undegraded":"2022-09-29T05:25:03.812390+0000","last_fullsized":"2022-09-29T05:25:03.812390+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2022-09-29T05:25:02.423315+0000","last_change":"2022-09-29T05:25:02.435837+0000","last_active":"2022-09-29T05:25:02.423315+0000","last_peered":"2022-09-29T05:25:02.423315+0000","last_clean":"2022-09-29T05:25:02.423315+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:02.423315+0000","last_undegraded":"2022-09-29T05:25:02.423315+0000","last_fullsized":"2022-09-29T05:25:02.423315+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,3],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":10,"reported_epoch":49,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.464203+0000","last_change":"2022-09-29T05:25:03.462271+0000","last_active":"2022-09-29T05:25:03.464203+0000","last_peered":"2022-09-29T05:25:03.464203+0000","last_clean":"2022-09-29T05:25:03.464203+0000","last_became_active":"2022-09-29T05:25:03.461959+0000","last_became_peered":"2022-09-29T05:25:03.461959+0000","last_unstale":"2022-09-29T05:25:03.464203+0000","last_undegraded":"2022-09-29T05:25:03.464203+0000","last_fullsized":"2022-09-29T05:25:03.464203+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":11,"reported_epoch":47,"state":"active+clean","last_fresh":"2022-09-29T05:24:48.628557+0000","last_change":"2022-09-29T05:24:48.628243+0000","last_active":"2022-09-29T05:24:48.628557+0000","last_peered":"2022-09-29T05:24:48.628557+0000","last_clean":"2022-09-29T05:24:48.628557+0000","last_became_active":"2022-09-29T05:24:48.627801+0000","last_became_peered":"2022-09-29T05:24:48.627801+0000","last_unstale":"2022-09-29T05:24:48.628557+0000","last_undegraded":"2022-09-29T05:24:48.628557+0000","last_fullsized":"2022-09-29T05:24:48.628557+0000","mapping_epoch":46,"log_start":"0'0","ondisk_log_start":"0'0","created":21,"last_epoch_clean":47,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:23:35.544281+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:23:35.544281+0000","last_clean_scrub_stamp":"2022-09-29T05:23:35.544281+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2022-09-29T05:25:02.423315+0000","last_change":"2022-09-29T05:25:02.433346+0000","last_active":"2022-09-29T05:25:02.423315+0000","last_peered":"2022-09-29T05:25:02.423315+0000","last_clean":"2022-09-29T05:25:02.423315+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:02.423315+0000","last_undegraded":"2022-09-29T05:25:02.423315+0000","last_fullsized":"2022-09-29T05:25:02.423315+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,1],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2022-09-29T05:25:02.423315+0000","last_change":"2022-09-29T05:25:02.433376+0000","last_active":"2022-09-29T05:25:02.423315+0000","last_peered":"2022-09-29T05:25:02.423315+0000","last_clean":"2022-09-29T05:25:02.423315+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:02.423315+0000","last_undegraded":"2022-09-29T05:25:02.423315+0000","last_fullsized":"2022-09-29T05:25:02.423315+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,4],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":9,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.803195+0000","last_change":"2022-09-29T05:25:03.803195+0000","last_active":"2022-09-29T05:25:03.803195+0000","last_peered":"2022-09-29T05:25:03.803195+0000","last_clean":"2022-09-29T05:25:03.803195+0000","last_became_active":"2022-09-29T05:25:03.802909+0000","last_became_peered":"2022-09-29T05:25:03.802909+0000","last_unstale":"2022-09-29T05:25:03.803195+0000","last_undegraded":"2022-09-29T05:25:03.803195+0000","last_fullsized":"2022-09-29T05:25:03.803195+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":10,"reported_epoch":49,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.457008+0000","last_change":"2022-09-29T05:25:03.456735+0000","last_active":"2022-09-29T05:25:03.457008+0000","last_peered":"2022-09-29T05:25:03.457008+0000","last_clean":"2022-09-29T05:25:03.457008+0000","last_became_active":"2022-09-29T05:25:03.456427+0000","last_became_peered":"2022-09-29T05:25:03.456427+0000","last_unstale":"2022-09-29T05:25:03.457008+0000","last_undegraded":"2022-09-29T05:25:03.457008+0000","last_fullsized":"2022-09-29T05:25:03.457008+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.7","version":"50'2","reported_seq":13,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.824910+0000","last_change":"2022-09-29T05:25:03.462138+0000","last_active":"2022-09-29T05:25:03.824910+0000","last_peered":"2022-09-29T05:25:03.824910+0000","last_clean":"2022-09-29T05:25:03.824910+0000","last_became_active":"2022-09-29T05:25:03.461809+0000","last_became_peered":"2022-09-29T05:25:03.461809+0000","last_unstale":"2022-09-29T05:25:03.824910+0000","last_undegraded":"2022-09-29T05:25:03.824910+0000","last_fullsized":"2022-09-29T05:25:03.824910+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.8","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2022-09-29T05:25:02.423315+0000","last_change":"2022-09-29T05:25:02.433909+0000","last_active":"2022-09-29T05:25:02.423315+0000","last_peered":"2022-09-29T05:25:02.423315+0000","last_clean":"2022-09-29T05:25:02.423315+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:02.423315+0000","last_undegraded":"2022-09-29T05:25:02.423315+0000","last_fullsized":"2022-09-29T05:25:02.423315+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,1],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.9","version":"0'0","reported_seq":10,"reported_epoch":49,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.452964+0000","last_change":"2022-09-29T05:25:03.452512+0000","last_active":"2022-09-29T05:25:03.452964+0000","last_peered":"2022-09-29T05:25:03.452964+0000","last_clean":"2022-09-29T05:25:03.452964+0000","last_became_active":"2022-09-29T05:25:03.452222+0000","last_became_peered":"2022-09-29T05:25:03.452222+0000","last_unstale":"2022-09-29T05:25:03.452964+0000","last_undegraded":"2022-09-29T05:25:03.452964+0000","last_fullsized":"2022-09-29T05:25:03.452964+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.a","version":"0'0","reported_seq":10,"reported_epoch":49,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.452852+0000","last_change":"2022-09-29T05:25:03.451868+0000","last_active":"2022-09-29T05:25:03.452852+0000","last_peered":"2022-09-29T05:25:03.452852+0000","last_clean":"2022-09-29T05:25:03.452852+0000","last_became_active":"2022-09-29T05:25:03.451576+0000","last_became_peered":"2022-09-29T05:25:03.451576+0000","last_unstale":"2022-09-29T05:25:03.452852+0000","last_undegraded":"2022-09-29T05:25:03.452852+0000","last_fullsized":"2022-09-29T05:25:03.452852+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.b","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2022-09-29T05:25:02.423315+0000","last_change":"2022-09-29T05:25:02.433388+0000","last_active":"2022-09-29T05:25:02.423315+0000","last_peered":"2022-09-29T05:25:02.423315+0000","last_clean":"2022-09-29T05:25:02.423315+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:02.423315+0000","last_undegraded":"2022-09-29T05:25:02.423315+0000","last_fullsized":"2022-09-29T05:25:02.423315+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[4,5],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.824054+0000","last_change":"2022-09-29T05:25:03.818102+0000","last_active":"2022-09-29T05:25:03.824054+0000","last_peered":"2022-09-29T05:25:03.824054+0000","last_clean":"2022-09-29T05:25:03.824054+0000","last_became_active":"2022-09-29T05:25:03.817772+0000","last_became_peered":"2022-09-29T05:25:03.817772+0000","last_unstale":"2022-09-29T05:25:03.824054+0000","last_undegraded":"2022-09-29T05:25:03.824054+0000","last_fullsized":"2022-09-29T05:25:03.824054+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.c","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2022-09-29T05:25:02.423315+0000","last_change":"2022-09-29T05:25:02.436663+0000","last_active":"2022-09-29T05:25:02.423315+0000","last_peered":"2022-09-29T05:25:02.423315+0000","last_clean":"2022-09-29T05:25:02.423315+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:02.423315+0000","last_undegraded":"2022-09-29T05:25:02.423315+0000","last_fullsized":"2022-09-29T05:25:02.423315+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,5],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.d","version":"50'1","reported_seq":11,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.798141+0000","last_change":"2022-09-29T05:25:03.452648+0000","last_active":"2022-09-29T05:25:03.798141+0000","last_peered":"2022-09-29T05:25:03.798141+0000","last_clean":"2022-09-29T05:25:03.798141+0000","last_became_active":"2022-09-29T05:25:03.452242+0000","last_became_peered":"2022-09-29T05:25:03.452242+0000","last_unstale":"2022-09-29T05:25:03.798141+0000","last_undegraded":"2022-09-29T05:25:03.798141+0000","last_fullsized":"2022-09-29T05:25:03.798141+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":9,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.809640+0000","last_change":"2022-09-29T05:25:03.809640+0000","last_active":"2022-09-29T05:25:03.809640+0000","last_peered":"2022-09-29T05:25:03.809640+0000","last_clean":"2022-09-29T05:25:03.809640+0000","last_became_active":"2022-09-29T05:25:03.809298+0000","last_became_peered":"2022-09-29T05:25:03.809298+0000","last_unstale":"2022-09-29T05:25:03.809640+0000","last_undegraded":"2022-09-29T05:25:03.809640+0000","last_fullsized":"2022-09-29T05:25:03.809640+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.e","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2022-09-29T05:25:02.423315+0000","last_change":"2022-09-29T05:25:02.439603+0000","last_active":"2022-09-29T05:25:02.423315+0000","last_peered":"2022-09-29T05:25:02.423315+0000","last_clean":"2022-09-29T05:25:02.423315+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:02.423315+0000","last_undegraded":"2022-09-29T05:25:02.423315+0000","last_fullsized":"2022-09-29T05:25:02.423315+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[3,7],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.20","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.813607+0000","last_change":"2022-09-29T05:25:03.811486+0000","last_active":"2022-09-29T05:25:03.813607+0000","last_peered":"2022-09-29T05:25:03.813607+0000","last_clean":"2022-09-29T05:25:03.813607+0000","last_became_active":"2022-09-29T05:25:03.811222+0000","last_became_peered":"2022-09-29T05:25:03.811222+0000","last_unstale":"2022-09-29T05:25:03.813607+0000","last_undegraded":"2022-09-29T05:25:03.813607+0000","last_fullsized":"2022-09-29T05:25:03.813607+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.21","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.22","version":"0'0","reported_seq":9,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.817645+0000","last_change":"2022-09-29T05:25:03.817645+0000","last_active":"2022-09-29T05:25:03.817645+0000","last_peered":"2022-09-29T05:25:03.817645+0000","last_clean":"2022-09-29T05:25:03.817645+0000","last_became_active":"2022-09-29T05:25:03.816740+0000","last_became_peered":"2022-09-29T05:25:03.816740+0000","last_unstale":"2022-09-29T05:25:03.817645+0000","last_undegraded":"2022-09-29T05:25:03.817645+0000","last_fullsized":"2022-09-29T05:25:03.817645+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.23","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.24","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.25","version":"0'0","reported_seq":9,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.817066+0000","last_change":"2022-09-29T05:25:03.817066+0000","last_active":"2022-09-29T05:25:03.817066+0000","last_peered":"2022-09-29T05:25:03.817066+0000","last_clean":"2022-09-29T05:25:03.817066+0000","last_became_active":"2022-09-29T05:25:03.816747+0000","last_became_peered":"2022-09-29T05:25:03.816747+0000","last_unstale":"2022-09-29T05:25:03.817066+0000","last_undegraded":"2022-09-29T05:25:03.817066+0000","last_fullsized":"2022-09-29T05:25:03.817066+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.26","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.824138+0000","last_change":"2022-09-29T05:25:03.823457+0000","last_active":"2022-09-29T05:25:03.824138+0000","last_peered":"2022-09-29T05:25:03.824138+0000","last_clean":"2022-09-29T05:25:03.824138+0000","last_became_active":"2022-09-29T05:25:03.823149+0000","last_became_peered":"2022-09-29T05:25:03.823149+0000","last_unstale":"2022-09-29T05:25:03.824138+0000","last_undegraded":"2022-09-29T05:25:03.824138+0000","last_fullsized":"2022-09-29T05:25:03.824138+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,4],"acting":[6,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.27","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.28","version":"0'0","reported_seq":9,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.815620+0000","last_change":"2022-09-29T05:25:03.815620+0000","last_active":"2022-09-29T05:25:03.815620+0000","last_peered":"2022-09-29T05:25:03.815620+0000","last_clean":"2022-09-29T05:25:03.815620+0000","last_became_active":"2022-09-29T05:25:03.814336+0000","last_became_peered":"2022-09-29T05:25:03.814336+0000","last_unstale":"2022-09-29T05:25:03.815620+0000","last_undegraded":"2022-09-29T05:25:03.815620+0000","last_fullsized":"2022-09-29T05:25:03.815620+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.29","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.2a","version":"0'0","reported_seq":9,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.819098+0000","last_change":"2022-09-29T05:25:03.819098+0000","last_active":"2022-09-29T05:25:03.819098+0000","last_peered":"2022-09-29T05:25:03.819098+0000","last_clean":"2022-09-29T05:25:03.819098+0000","last_became_active":"2022-09-29T05:25:03.818690+0000","last_became_peered":"2022-09-29T05:25:03.818690+0000","last_unstale":"2022-09-29T05:25:03.819098+0000","last_undegraded":"2022-09-29T05:25:03.819098+0000","last_fullsized":"2022-09-29T05:25:03.819098+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2b","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.2c","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.815650+0000","last_change":"2022-09-29T05:25:03.812644+0000","last_active":"2022-09-29T05:25:03.815650+0000","last_peered":"2022-09-29T05:25:03.815650+0000","last_clean":"2022-09-29T05:25:03.815650+0000","last_became_active":"2022-09-29T05:25:03.812339+0000","last_became_peered":"2022-09-29T05:25:03.812339+0000","last_unstale":"2022-09-29T05:25:03.815650+0000","last_undegraded":"2022-09-29T05:25:03.815650+0000","last_fullsized":"2022-09-29T05:25:03.815650+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":3,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":75,"acting":75,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":7},{"poolid":2,"num_pg":32,"stat_sum":{"num_bytes":1696,"num_objects":8,"num_object_clones":0,"num_object_copies":24,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14,"num_write_kb":5,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":36864,"data_stored":3322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":13,"ondisk_log_size":13,"up":78,"acting":78,"num_store_stats":5}],"osd_stats":[{"osd":7,"up_from":45,"seq":193273528326,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5176,"kb_used_data":432,"kb_used_omap":0,"kb_used_meta":4736,"kb_avail":93739976,"statfs":{"total":95995035648,"available":95989735424,"internally_reserved":0,"allocated":442368,"data_stored":174771,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":4849664},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.081}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.59399999999999997}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.028}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.69299999999999995}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.103}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.65400000000000003}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.63}]}]},{"osd":6,"up_from":40,"seq":171798691849,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5480,"kb_used_data":480,"kb_used_omap":0,"kb_used_meta":4992,"kb_avail":93739672,"statfs":{"total":95995035648,"available":95989424128,"internally_reserved":0,"allocated":491520,"data_stored":196260,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5111808},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.38200000000000001}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45100000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.439}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.36199999999999999}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.433}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.46200000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.42399999999999999}]}]},{"osd":1,"up_from":13,"seq":55834574874,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5736,"kb_used_data":480,"kb_used_omap":0,"kb_used_meta":5248,"kb_avail":93739416,"statfs":{"total":95995035648,"available":95989161984,"internally_reserved":0,"allocated":491520,"data_stored":195199,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5373952},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Sep 29 05:24:04 2022","interfaces":[{"interface":"back","average":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"last":0.35799999999999998},{"interface":"front","average":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"last":0.27800000000000002}]},{"osd":2,"last update":"Thu Sep 29 05:24:22 2022","interfaces":[{"interface":"back","average":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"last":0.32000000000000001},{"interface":"front","average":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.374}]},{"osd":3,"last update":"Thu Sep 29 05:24:43 2022","interfaces":[{"interface":"back","average":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"last":0.34699999999999998},{"interface":"front","average":{"1min":0.442,"5min":0.442,"15min":0.442},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"last":0.30199999999999999}]},{"osd":4,"last update":"Thu Sep 29 05:25:00 2022","interfaces":[{"interface":"back","average":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":1.1619999999999999,"5min":1.1619999999999999,"15min":1.1619999999999999},"last":0.67700000000000005},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":1.1799999999999999,"5min":1.1799999999999999,"15min":1.1799999999999999},"last":0.66800000000000004}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56200000000000006}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.69799999999999995}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.61599999999999999}]}]},{"osd":0,"up_from":8,"seq":34359738397,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5500,"kb_used_data":436,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739652,"statfs":{"total":95995035648,"available":95989403648,"internally_reserved":0,"allocated":446464,"data_stored":175086,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Sep 29 05:24:05 2022","interfaces":[{"interface":"back","average":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"min":{"1min":0.17699999999999999,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.28000000000000003},{"interface":"front","average":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.35399999999999998}]},{"osd":2,"last update":"Thu Sep 29 05:24:29 2022","interfaces":[{"interface":"back","average":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"last":0.29399999999999998},{"interface":"front","average":{"1min":0.377,"5min":0.377,"15min":0.377},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"last":0.36699999999999999}]},{"osd":3,"last update":"Thu Sep 29 05:24:41 2022","interfaces":[{"interface":"back","average":{"1min":0.437,"5min":0.437,"15min":0.437},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"last":0.32900000000000001},{"interface":"front","average":{"1min":0.434,"5min":0.434,"15min":0.434},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"last":0.314}]},{"osd":4,"last update":"Thu Sep 29 05:24:57 2022","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.50800000000000001}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.46000000000000002}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.47899999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56999999999999995}]}]},{"osd":2,"up_from":18,"seq":77309411350,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5448,"kb_used_data":448,"kb_used_omap":0,"kb_used_meta":4992,"kb_avail":93739704,"statfs":{"total":95995035648,"available":95989456896,"internally_reserved":0,"allocated":458752,"data_stored":181265,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5111808},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Sep 29 05:24:26 2022","interfaces":[{"interface":"back","average":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"last":0.318},{"interface":"front","average":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":0.72099999999999997,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.28000000000000003}]},{"osd":1,"last update":"Thu Sep 29 05:24:26 2022","interfaces":[{"interface":"back","average":{"1min":0.38,"5min":0.38,"15min":0.38},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"last":0.33300000000000002},{"interface":"front","average":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"last":0.35599999999999998}]},{"osd":3,"last update":"Thu Sep 29 05:24:41 2022","interfaces":[{"interface":"back","average":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"last":0.34899999999999998},{"interface":"front","average":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.71799999999999997,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.38600000000000001}]},{"osd":4,"last update":"Thu Sep 29 05:24:56 2022","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.55700000000000005}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.59499999999999997}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.64200000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.61899999999999999}]}]},{"osd":3,"up_from":26,"seq":111669149714,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5372,"kb_used_data":436,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739780,"statfs":{"total":95995035648,"available":95989534720,"internally_reserved":0,"allocated":446464,"data_stored":175086,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Sep 29 05:24:44 2022","interfaces":[{"interface":"back","average":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"last":0.29399999999999998},{"interface":"front","average":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"last":0.33900000000000002}]},{"osd":1,"last update":"Thu Sep 29 05:24:44 2022","interfaces":[{"interface":"back","average":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"last":0.30399999999999999}]},{"osd":2,"last update":"Thu Sep 29 05:24:44 2022","interfaces":[{"interface":"back","average":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"last":0.35299999999999998},{"interface":"front","average":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"last":0.32300000000000001}]},{"osd":4,"last update":"Thu Sep 29 05:25:00 2022","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.84099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.46999999999999997}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51900000000000002}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51200000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.50700000000000001}]}]},{"osd":4,"up_from":30,"seq":128849018894,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5308,"kb_used_data":436,"kb_used_omap":0,"kb_used_meta":4864,"kb_avail":93739844,"statfs":{"total":95995035648,"available":95989600256,"internally_reserved":0,"allocated":446464,"data_stored":175086,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":4980736},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Sep 29 05:24:59 2022","interfaces":[{"interface":"back","average":{"1min":0.47799999999999998,"5min":0.47799999999999998,"15min":0.47799999999999998},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.71999999999999997,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.67500000000000004,"5min":0.67500000000000004,"15min":0.67500000000000004},"last":0.434}]},{"osd":1,"last update":"Thu Sep 29 05:24:59 2022","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.45000000000000001},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.749,"5min":0.749,"15min":0.749},"last":0.56100000000000005}]},{"osd":2,"last update":"Thu Sep 29 05:24:59 2022","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.47199999999999998},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.51700000000000002,"15min":0.51700000000000002},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.50800000000000001}]},{"osd":3,"last update":"Thu Sep 29 05:24:59 2022","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.57499999999999996},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.54000000000000004}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.53000000000000003}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.52400000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.55500000000000005}]}]},{"osd":5,"up_from":35,"seq":150323855373,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5392,"kb_used_data":456,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739760,"statfs":{"total":95995035648,"available":95989514240,"internally_reserved":0,"allocated":466944,"data_stored":187587,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.55900000000000005}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.50700000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.49299999999999999}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45500000000000002}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.41199999999999998}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.46300000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.61199999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1626,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":1536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2022-09-29T05:25:06.775 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2022-09-29T05:25:07.084 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:25:07.086 INFO:teuthology.orchestra.run.smithi083.stderr:dumped all 2022-09-29T05:25:07.095 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:25:05.591809+0000 mgr.y (mgr.14146) 161 : cluster [DBG] pgmap v146: 97 pgs: 10 creating+peering, 45 unknown, 42 active+clean; 1.7 KiB data, 42 MiB used, 715 GiB / 715 GiB avail; 3.8 KiB/s wr, 11 op/s 2022-09-29T05:25:07.095 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:05.819902+0000 mon.a (mon.0) 561 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "false"}]': finished 2022-09-29T05:25:07.095 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:25:05.819996+0000 mon.a (mon.0) 562 : cluster [DBG] fsmap cephfs:1 {0=b=up:active} 1 up:standby 2022-09-29T05:25:07.095 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:06.149184+0000 mon.c (mon.1) 21 : audit [DBG] from='client.? 172.21.15.83:0/794198628' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2022-09-29T05:25:07.096 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:06.443576+0000 mon.c (mon.1) 22 : audit [DBG] from='client.? 172.21.15.83:0/601179928' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2022-09-29T05:25:07.096 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:25:05.591809+0000 mgr.y (mgr.14146) 161 : cluster [DBG] pgmap v146: 97 pgs: 10 creating+peering, 45 unknown, 42 active+clean; 1.7 KiB data, 42 MiB used, 715 GiB / 715 GiB avail; 3.8 KiB/s wr, 11 op/s 2022-09-29T05:25:07.096 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:05.819902+0000 mon.a (mon.0) 561 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "false"}]': finished 2022-09-29T05:25:07.097 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:25:05.819996+0000 mon.a (mon.0) 562 : cluster [DBG] fsmap cephfs:1 {0=b=up:active} 1 up:standby 2022-09-29T05:25:07.097 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:06.149184+0000 mon.c (mon.1) 21 : audit [DBG] from='client.? 172.21.15.83:0/794198628' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2022-09-29T05:25:07.097 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:06.443576+0000 mon.c (mon.1) 22 : audit [DBG] from='client.? 172.21.15.83:0/601179928' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2022-09-29T05:25:07.098 INFO:teuthology.orchestra.run.smithi083.stdout:{"pg_ready":true,"pg_map":{"version":146,"stamp":"2022-09-29T05:25:05.591588+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1696,"num_objects":8,"num_object_clones":0,"num_object_copies":24,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14,"num_write_kb":5,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":13,"ondisk_log_size":13,"up":156,"acting":156,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":749961216,"kb_used":43412,"kb_used_data":3604,"kb_used_omap":0,"kb_used_meta":39744,"kb_avail":749917804,"statfs":{"total":767960285184,"available":767915831296,"internally_reserved":0,"allocated":3690496,"data_stored":1460340,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":40697856},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1536,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"0.794136"},"pg_stats":[{"pgid":"3.3f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.3e","version":"0'0","reported_seq":9,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.816168+0000","last_change":"2022-09-29T05:25:03.816168+0000","last_active":"2022-09-29T05:25:03.816168+0000","last_peered":"2022-09-29T05:25:03.816168+0000","last_clean":"2022-09-29T05:25:03.816168+0000","last_became_active":"2022-09-29T05:25:03.815888+0000","last_became_peered":"2022-09-29T05:25:03.815888+0000","last_unstale":"2022-09-29T05:25:03.816168+0000","last_undegraded":"2022-09-29T05:25:03.816168+0000","last_fullsized":"2022-09-29T05:25:03.816168+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3d","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.3c","version":"0'0","reported_seq":9,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.819554+0000","last_change":"2022-09-29T05:25:03.819554+0000","last_active":"2022-09-29T05:25:03.819554+0000","last_peered":"2022-09-29T05:25:03.819554+0000","last_clean":"2022-09-29T05:25:03.819554+0000","last_became_active":"2022-09-29T05:25:03.819207+0000","last_became_peered":"2022-09-29T05:25:03.819207+0000","last_unstale":"2022-09-29T05:25:03.819554+0000","last_undegraded":"2022-09-29T05:25:03.819554+0000","last_fullsized":"2022-09-29T05:25:03.819554+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3b","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.3a","version":"0'0","reported_seq":9,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.817871+0000","last_change":"2022-09-29T05:25:03.817871+0000","last_active":"2022-09-29T05:25:03.817871+0000","last_peered":"2022-09-29T05:25:03.817871+0000","last_clean":"2022-09-29T05:25:03.817871+0000","last_became_active":"2022-09-29T05:25:03.816800+0000","last_became_peered":"2022-09-29T05:25:03.816800+0000","last_unstale":"2022-09-29T05:25:03.817871+0000","last_undegraded":"2022-09-29T05:25:03.817871+0000","last_fullsized":"2022-09-29T05:25:03.817871+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.39","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.38","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.37","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.36","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.825144+0000","last_change":"2022-09-29T05:25:03.823656+0000","last_active":"2022-09-29T05:25:03.825144+0000","last_peered":"2022-09-29T05:25:03.825144+0000","last_clean":"2022-09-29T05:25:03.825144+0000","last_became_active":"2022-09-29T05:25:03.823109+0000","last_became_peered":"2022-09-29T05:25:03.823109+0000","last_unstale":"2022-09-29T05:25:03.825144+0000","last_undegraded":"2022-09-29T05:25:03.825144+0000","last_fullsized":"2022-09-29T05:25:03.825144+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.35","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.825174+0000","last_change":"2022-09-29T05:25:03.824377+0000","last_active":"2022-09-29T05:25:03.825174+0000","last_peered":"2022-09-29T05:25:03.825174+0000","last_clean":"2022-09-29T05:25:03.825174+0000","last_became_active":"2022-09-29T05:25:03.823869+0000","last_became_peered":"2022-09-29T05:25:03.823869+0000","last_unstale":"2022-09-29T05:25:03.825174+0000","last_undegraded":"2022-09-29T05:25:03.825174+0000","last_fullsized":"2022-09-29T05:25:03.825174+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.34","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.33","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.32","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.31","version":"0'0","reported_seq":9,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.805590+0000","last_change":"2022-09-29T05:25:03.805590+0000","last_active":"2022-09-29T05:25:03.805590+0000","last_peered":"2022-09-29T05:25:03.805590+0000","last_clean":"2022-09-29T05:25:03.805590+0000","last_became_active":"2022-09-29T05:25:03.805310+0000","last_became_peered":"2022-09-29T05:25:03.805310+0000","last_unstale":"2022-09-29T05:25:03.805590+0000","last_undegraded":"2022-09-29T05:25:03.805590+0000","last_fullsized":"2022-09-29T05:25:03.805590+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.30","version":"0'0","reported_seq":9,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.815886+0000","last_change":"2022-09-29T05:25:03.815886+0000","last_active":"2022-09-29T05:25:03.815886+0000","last_peered":"2022-09-29T05:25:03.815886+0000","last_clean":"2022-09-29T05:25:03.815886+0000","last_became_active":"2022-09-29T05:25:03.815285+0000","last_became_peered":"2022-09-29T05:25:03.815285+0000","last_unstale":"2022-09-29T05:25:03.815886+0000","last_undegraded":"2022-09-29T05:25:03.815886+0000","last_fullsized":"2022-09-29T05:25:03.815886+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2f","version":"0'0","reported_seq":9,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.813784+0000","last_change":"2022-09-29T05:25:03.813784+0000","last_active":"2022-09-29T05:25:03.813784+0000","last_peered":"2022-09-29T05:25:03.813784+0000","last_clean":"2022-09-29T05:25:03.813784+0000","last_became_active":"2022-09-29T05:25:03.813420+0000","last_became_peered":"2022-09-29T05:25:03.813420+0000","last_unstale":"2022-09-29T05:25:03.813784+0000","last_undegraded":"2022-09-29T05:25:03.813784+0000","last_fullsized":"2022-09-29T05:25:03.813784+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2e","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.2d","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.1f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:02.423315+0000","last_change":"2022-09-29T05:25:02.423315+0000","last_active":"2022-09-29T05:25:02.423315+0000","last_peered":"2022-09-29T05:25:02.423315+0000","last_clean":"2022-09-29T05:25:02.423315+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:02.423315+0000","last_undegraded":"2022-09-29T05:25:02.423315+0000","last_fullsized":"2022-09-29T05:25:02.423315+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.1e","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.1e","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:02.423315+0000","last_change":"2022-09-29T05:25:02.423315+0000","last_active":"2022-09-29T05:25:02.423315+0000","last_peered":"2022-09-29T05:25:02.423315+0000","last_clean":"2022-09-29T05:25:02.423315+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:02.423315+0000","last_undegraded":"2022-09-29T05:25:02.423315+0000","last_fullsized":"2022-09-29T05:25:02.423315+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.1f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.1d","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2022-09-29T05:25:02.423315+0000","last_change":"2022-09-29T05:25:02.433891+0000","last_active":"2022-09-29T05:25:02.423315+0000","last_peered":"2022-09-29T05:25:02.423315+0000","last_clean":"2022-09-29T05:25:02.423315+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:02.423315+0000","last_undegraded":"2022-09-29T05:25:02.423315+0000","last_fullsized":"2022-09-29T05:25:02.423315+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,6],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"0'0","reported_seq":9,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.816581+0000","last_change":"2022-09-29T05:25:03.816581+0000","last_active":"2022-09-29T05:25:03.816581+0000","last_peered":"2022-09-29T05:25:03.816581+0000","last_clean":"2022-09-29T05:25:03.816581+0000","last_became_active":"2022-09-29T05:25:03.816206+0000","last_became_peered":"2022-09-29T05:25:03.816206+0000","last_unstale":"2022-09-29T05:25:03.816581+0000","last_undegraded":"2022-09-29T05:25:03.816581+0000","last_fullsized":"2022-09-29T05:25:03.816581+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1c","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2022-09-29T05:25:02.423315+0000","last_change":"2022-09-29T05:25:02.433333+0000","last_active":"2022-09-29T05:25:02.423315+0000","last_peered":"2022-09-29T05:25:02.423315+0000","last_clean":"2022-09-29T05:25:02.423315+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:02.423315+0000","last_undegraded":"2022-09-29T05:25:02.423315+0000","last_fullsized":"2022-09-29T05:25:02.423315+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[2,5],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1d","version":"0'0","reported_seq":9,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.820523+0000","last_change":"2022-09-29T05:25:03.820523+0000","last_active":"2022-09-29T05:25:03.820523+0000","last_peered":"2022-09-29T05:25:03.820523+0000","last_clean":"2022-09-29T05:25:03.820523+0000","last_became_active":"2022-09-29T05:25:03.819546+0000","last_became_peered":"2022-09-29T05:25:03.819546+0000","last_unstale":"2022-09-29T05:25:03.820523+0000","last_undegraded":"2022-09-29T05:25:03.820523+0000","last_fullsized":"2022-09-29T05:25:03.820523+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1b","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:02.423315+0000","last_change":"2022-09-29T05:25:02.423315+0000","last_active":"2022-09-29T05:25:02.423315+0000","last_peered":"2022-09-29T05:25:02.423315+0000","last_clean":"2022-09-29T05:25:02.423315+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:02.423315+0000","last_undegraded":"2022-09-29T05:25:02.423315+0000","last_fullsized":"2022-09-29T05:25:02.423315+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.1a","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.1a","version":"50'1","reported_seq":12,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.815499+0000","last_change":"2022-09-29T05:25:03.454747+0000","last_active":"2022-09-29T05:25:03.815499+0000","last_peered":"2022-09-29T05:25:03.815499+0000","last_clean":"2022-09-29T05:25:03.815499+0000","last_became_active":"2022-09-29T05:25:03.454395+0000","last_became_peered":"2022-09-29T05:25:03.454395+0000","last_unstale":"2022-09-29T05:25:03.815499+0000","last_undegraded":"2022-09-29T05:25:03.815499+0000","last_fullsized":"2022-09-29T05:25:03.815499+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1b","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.19","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:02.423315+0000","last_change":"2022-09-29T05:25:02.423315+0000","last_active":"2022-09-29T05:25:02.423315+0000","last_peered":"2022-09-29T05:25:02.423315+0000","last_clean":"2022-09-29T05:25:02.423315+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:02.423315+0000","last_undegraded":"2022-09-29T05:25:02.423315+0000","last_fullsized":"2022-09-29T05:25:02.423315+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.18","version":"0'0","reported_seq":10,"reported_epoch":49,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.464050+0000","last_change":"2022-09-29T05:25:03.460675+0000","last_active":"2022-09-29T05:25:03.464050+0000","last_peered":"2022-09-29T05:25:03.464050+0000","last_clean":"2022-09-29T05:25:03.464050+0000","last_became_active":"2022-09-29T05:25:03.460351+0000","last_became_peered":"2022-09-29T05:25:03.460351+0000","last_unstale":"2022-09-29T05:25:03.464050+0000","last_undegraded":"2022-09-29T05:25:03.464050+0000","last_fullsized":"2022-09-29T05:25:03.464050+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":9,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.816170+0000","last_change":"2022-09-29T05:25:03.816170+0000","last_active":"2022-09-29T05:25:03.816170+0000","last_peered":"2022-09-29T05:25:03.816170+0000","last_clean":"2022-09-29T05:25:03.816170+0000","last_became_active":"2022-09-29T05:25:03.815838+0000","last_became_peered":"2022-09-29T05:25:03.815838+0000","last_unstale":"2022-09-29T05:25:03.816170+0000","last_undegraded":"2022-09-29T05:25:03.816170+0000","last_fullsized":"2022-09-29T05:25:03.816170+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.17","version":"0'0","reported_seq":11,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.805981+0000","last_change":"2022-09-29T05:25:03.461045+0000","last_active":"2022-09-29T05:25:03.805981+0000","last_peered":"2022-09-29T05:25:03.805981+0000","last_clean":"2022-09-29T05:25:03.805981+0000","last_became_active":"2022-09-29T05:25:03.460729+0000","last_became_peered":"2022-09-29T05:25:03.460729+0000","last_unstale":"2022-09-29T05:25:03.805981+0000","last_undegraded":"2022-09-29T05:25:03.805981+0000","last_fullsized":"2022-09-29T05:25:03.805981+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":9,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.818733+0000","last_change":"2022-09-29T05:25:03.818733+0000","last_active":"2022-09-29T05:25:03.818733+0000","last_peered":"2022-09-29T05:25:03.818733+0000","last_clean":"2022-09-29T05:25:03.818733+0000","last_became_active":"2022-09-29T05:25:03.816983+0000","last_became_peered":"2022-09-29T05:25:03.816983+0000","last_unstale":"2022-09-29T05:25:03.818733+0000","last_undegraded":"2022-09-29T05:25:03.818733+0000","last_fullsized":"2022-09-29T05:25:03.818733+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.16","version":"0'0","reported_seq":10,"reported_epoch":49,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.464237+0000","last_change":"2022-09-29T05:25:03.463477+0000","last_active":"2022-09-29T05:25:03.464237+0000","last_peered":"2022-09-29T05:25:03.464237+0000","last_clean":"2022-09-29T05:25:03.464237+0000","last_became_active":"2022-09-29T05:25:03.463051+0000","last_became_peered":"2022-09-29T05:25:03.463051+0000","last_unstale":"2022-09-29T05:25:03.464237+0000","last_undegraded":"2022-09-29T05:25:03.464237+0000","last_fullsized":"2022-09-29T05:25:03.464237+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.15","version":"50'3","reported_seq":13,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.811766+0000","last_change":"2022-09-29T05:25:03.457077+0000","last_active":"2022-09-29T05:25:03.811766+0000","last_peered":"2022-09-29T05:25:03.811766+0000","last_clean":"2022-09-29T05:25:03.811766+0000","last_became_active":"2022-09-29T05:25:03.456823+0000","last_became_peered":"2022-09-29T05:25:03.456823+0000","last_unstale":"2022-09-29T05:25:03.811766+0000","last_undegraded":"2022-09-29T05:25:03.811766+0000","last_fullsized":"2022-09-29T05:25:03.811766+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":3,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.14","version":"50'6","reported_seq":17,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.827688+0000","last_change":"2022-09-29T05:25:03.461139+0000","last_active":"2022-09-29T05:25:03.827688+0000","last_peered":"2022-09-29T05:25:03.827688+0000","last_clean":"2022-09-29T05:25:03.827688+0000","last_became_active":"2022-09-29T05:25:03.460829+0000","last_became_peered":"2022-09-29T05:25:03.460829+0000","last_unstale":"2022-09-29T05:25:03.827688+0000","last_undegraded":"2022-09-29T05:25:03.827688+0000","last_fullsized":"2022-09-29T05:25:03.827688+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":6,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":1536,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.13","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:02.423315+0000","last_change":"2022-09-29T05:25:02.423315+0000","last_active":"2022-09-29T05:25:02.423315+0000","last_peered":"2022-09-29T05:25:02.423315+0000","last_clean":"2022-09-29T05:25:02.423315+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:02.423315+0000","last_undegraded":"2022-09-29T05:25:02.423315+0000","last_fullsized":"2022-09-29T05:25:02.423315+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.12","version":"0'0","reported_seq":10,"reported_epoch":49,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.464191+0000","last_change":"2022-09-29T05:25:03.462145+0000","last_active":"2022-09-29T05:25:03.464191+0000","last_peered":"2022-09-29T05:25:03.464191+0000","last_clean":"2022-09-29T05:25:03.464191+0000","last_became_active":"2022-09-29T05:25:03.461864+0000","last_became_peered":"2022-09-29T05:25:03.461864+0000","last_unstale":"2022-09-29T05:25:03.464191+0000","last_undegraded":"2022-09-29T05:25:03.464191+0000","last_fullsized":"2022-09-29T05:25:03.464191+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.11","version":"0'0","reported_seq":11,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.805505+0000","last_change":"2022-09-29T05:25:03.454654+0000","last_active":"2022-09-29T05:25:03.805505+0000","last_peered":"2022-09-29T05:25:03.805505+0000","last_clean":"2022-09-29T05:25:03.805505+0000","last_became_active":"2022-09-29T05:25:03.454319+0000","last_became_peered":"2022-09-29T05:25:03.454319+0000","last_unstale":"2022-09-29T05:25:03.805505+0000","last_undegraded":"2022-09-29T05:25:03.805505+0000","last_fullsized":"2022-09-29T05:25:03.805505+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.813586+0000","last_change":"2022-09-29T05:25:03.811425+0000","last_active":"2022-09-29T05:25:03.813586+0000","last_peered":"2022-09-29T05:25:03.813586+0000","last_clean":"2022-09-29T05:25:03.813586+0000","last_became_active":"2022-09-29T05:25:03.811120+0000","last_became_peered":"2022-09-29T05:25:03.811120+0000","last_unstale":"2022-09-29T05:25:03.813586+0000","last_undegraded":"2022-09-29T05:25:03.813586+0000","last_fullsized":"2022-09-29T05:25:03.813586+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.10","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2022-09-29T05:25:02.423315+0000","last_change":"2022-09-29T05:25:02.439448+0000","last_active":"2022-09-29T05:25:02.423315+0000","last_peered":"2022-09-29T05:25:02.423315+0000","last_clean":"2022-09-29T05:25:02.423315+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:02.423315+0000","last_undegraded":"2022-09-29T05:25:02.423315+0000","last_fullsized":"2022-09-29T05:25:02.423315+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,1],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:02.423315+0000","last_change":"2022-09-29T05:25:02.423315+0000","last_active":"2022-09-29T05:25:02.423315+0000","last_peered":"2022-09-29T05:25:02.423315+0000","last_clean":"2022-09-29T05:25:02.423315+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:02.423315+0000","last_undegraded":"2022-09-29T05:25:02.423315+0000","last_fullsized":"2022-09-29T05:25:02.423315+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.e","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":10,"reported_epoch":49,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.454814+0000","last_change":"2022-09-29T05:25:03.454507+0000","last_active":"2022-09-29T05:25:03.454814+0000","last_peered":"2022-09-29T05:25:03.454814+0000","last_clean":"2022-09-29T05:25:03.454814+0000","last_became_active":"2022-09-29T05:25:03.454195+0000","last_became_peered":"2022-09-29T05:25:03.454195+0000","last_unstale":"2022-09-29T05:25:03.454814+0000","last_undegraded":"2022-09-29T05:25:03.454814+0000","last_fullsized":"2022-09-29T05:25:03.454814+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":9,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.819862+0000","last_change":"2022-09-29T05:25:03.819862+0000","last_active":"2022-09-29T05:25:03.819862+0000","last_peered":"2022-09-29T05:25:03.819862+0000","last_clean":"2022-09-29T05:25:03.819862+0000","last_became_active":"2022-09-29T05:25:03.818117+0000","last_became_peered":"2022-09-29T05:25:03.818117+0000","last_unstale":"2022-09-29T05:25:03.819862+0000","last_undegraded":"2022-09-29T05:25:03.819862+0000","last_fullsized":"2022-09-29T05:25:03.819862+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":10,"reported_epoch":49,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.463942+0000","last_change":"2022-09-29T05:25:03.459646+0000","last_active":"2022-09-29T05:25:03.463942+0000","last_peered":"2022-09-29T05:25:03.463942+0000","last_clean":"2022-09-29T05:25:03.463942+0000","last_became_active":"2022-09-29T05:25:03.459320+0000","last_became_peered":"2022-09-29T05:25:03.459320+0000","last_unstale":"2022-09-29T05:25:03.463942+0000","last_undegraded":"2022-09-29T05:25:03.463942+0000","last_fullsized":"2022-09-29T05:25:03.463942+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":9,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.812390+0000","last_change":"2022-09-29T05:25:03.812390+0000","last_active":"2022-09-29T05:25:03.812390+0000","last_peered":"2022-09-29T05:25:03.812390+0000","last_clean":"2022-09-29T05:25:03.812390+0000","last_became_active":"2022-09-29T05:25:03.811925+0000","last_became_peered":"2022-09-29T05:25:03.811925+0000","last_unstale":"2022-09-29T05:25:03.812390+0000","last_undegraded":"2022-09-29T05:25:03.812390+0000","last_fullsized":"2022-09-29T05:25:03.812390+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2022-09-29T05:25:02.423315+0000","last_change":"2022-09-29T05:25:02.435837+0000","last_active":"2022-09-29T05:25:02.423315+0000","last_peered":"2022-09-29T05:25:02.423315+0000","last_clean":"2022-09-29T05:25:02.423315+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:02.423315+0000","last_undegraded":"2022-09-29T05:25:02.423315+0000","last_fullsized":"2022-09-29T05:25:02.423315+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,3],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":10,"reported_epoch":49,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.464203+0000","last_change":"2022-09-29T05:25:03.462271+0000","last_active":"2022-09-29T05:25:03.464203+0000","last_peered":"2022-09-29T05:25:03.464203+0000","last_clean":"2022-09-29T05:25:03.464203+0000","last_became_active":"2022-09-29T05:25:03.461959+0000","last_became_peered":"2022-09-29T05:25:03.461959+0000","last_unstale":"2022-09-29T05:25:03.464203+0000","last_undegraded":"2022-09-29T05:25:03.464203+0000","last_fullsized":"2022-09-29T05:25:03.464203+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":11,"reported_epoch":47,"state":"active+clean","last_fresh":"2022-09-29T05:24:48.628557+0000","last_change":"2022-09-29T05:24:48.628243+0000","last_active":"2022-09-29T05:24:48.628557+0000","last_peered":"2022-09-29T05:24:48.628557+0000","last_clean":"2022-09-29T05:24:48.628557+0000","last_became_active":"2022-09-29T05:24:48.627801+0000","last_became_peered":"2022-09-29T05:24:48.627801+0000","last_unstale":"2022-09-29T05:24:48.628557+0000","last_undegraded":"2022-09-29T05:24:48.628557+0000","last_fullsized":"2022-09-29T05:24:48.628557+0000","mapping_epoch":46,"log_start":"0'0","ondisk_log_start":"0'0","created":21,"last_epoch_clean":47,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:23:35.544281+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:23:35.544281+0000","last_clean_scrub_stamp":"2022-09-29T05:23:35.544281+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2022-09-29T05:25:02.423315+0000","last_change":"2022-09-29T05:25:02.433346+0000","last_active":"2022-09-29T05:25:02.423315+0000","last_peered":"2022-09-29T05:25:02.423315+0000","last_clean":"2022-09-29T05:25:02.423315+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:02.423315+0000","last_undegraded":"2022-09-29T05:25:02.423315+0000","last_fullsized":"2022-09-29T05:25:02.423315+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,1],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2022-09-29T05:25:02.423315+0000","last_change":"2022-09-29T05:25:02.433376+0000","last_active":"2022-09-29T05:25:02.423315+0000","last_peered":"2022-09-29T05:25:02.423315+0000","last_clean":"2022-09-29T05:25:02.423315+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:02.423315+0000","last_undegraded":"2022-09-29T05:25:02.423315+0000","last_fullsized":"2022-09-29T05:25:02.423315+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,4],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":9,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.803195+0000","last_change":"2022-09-29T05:25:03.803195+0000","last_active":"2022-09-29T05:25:03.803195+0000","last_peered":"2022-09-29T05:25:03.803195+0000","last_clean":"2022-09-29T05:25:03.803195+0000","last_became_active":"2022-09-29T05:25:03.802909+0000","last_became_peered":"2022-09-29T05:25:03.802909+0000","last_unstale":"2022-09-29T05:25:03.803195+0000","last_undegraded":"2022-09-29T05:25:03.803195+0000","last_fullsized":"2022-09-29T05:25:03.803195+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":10,"reported_epoch":49,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.457008+0000","last_change":"2022-09-29T05:25:03.456735+0000","last_active":"2022-09-29T05:25:03.457008+0000","last_peered":"2022-09-29T05:25:03.457008+0000","last_clean":"2022-09-29T05:25:03.457008+0000","last_became_active":"2022-09-29T05:25:03.456427+0000","last_became_peered":"2022-09-29T05:25:03.456427+0000","last_unstale":"2022-09-29T05:25:03.457008+0000","last_undegraded":"2022-09-29T05:25:03.457008+0000","last_fullsized":"2022-09-29T05:25:03.457008+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.7","version":"50'2","reported_seq":13,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.824910+0000","last_change":"2022-09-29T05:25:03.462138+0000","last_active":"2022-09-29T05:25:03.824910+0000","last_peered":"2022-09-29T05:25:03.824910+0000","last_clean":"2022-09-29T05:25:03.824910+0000","last_became_active":"2022-09-29T05:25:03.461809+0000","last_became_peered":"2022-09-29T05:25:03.461809+0000","last_unstale":"2022-09-29T05:25:03.824910+0000","last_undegraded":"2022-09-29T05:25:03.824910+0000","last_fullsized":"2022-09-29T05:25:03.824910+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.8","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2022-09-29T05:25:02.423315+0000","last_change":"2022-09-29T05:25:02.433909+0000","last_active":"2022-09-29T05:25:02.423315+0000","last_peered":"2022-09-29T05:25:02.423315+0000","last_clean":"2022-09-29T05:25:02.423315+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:02.423315+0000","last_undegraded":"2022-09-29T05:25:02.423315+0000","last_fullsized":"2022-09-29T05:25:02.423315+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,1],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.9","version":"0'0","reported_seq":10,"reported_epoch":49,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.452964+0000","last_change":"2022-09-29T05:25:03.452512+0000","last_active":"2022-09-29T05:25:03.452964+0000","last_peered":"2022-09-29T05:25:03.452964+0000","last_clean":"2022-09-29T05:25:03.452964+0000","last_became_active":"2022-09-29T05:25:03.452222+0000","last_became_peered":"2022-09-29T05:25:03.452222+0000","last_unstale":"2022-09-29T05:25:03.452964+0000","last_undegraded":"2022-09-29T05:25:03.452964+0000","last_fullsized":"2022-09-29T05:25:03.452964+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.a","version":"0'0","reported_seq":10,"reported_epoch":49,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.452852+0000","last_change":"2022-09-29T05:25:03.451868+0000","last_active":"2022-09-29T05:25:03.452852+0000","last_peered":"2022-09-29T05:25:03.452852+0000","last_clean":"2022-09-29T05:25:03.452852+0000","last_became_active":"2022-09-29T05:25:03.451576+0000","last_became_peered":"2022-09-29T05:25:03.451576+0000","last_unstale":"2022-09-29T05:25:03.452852+0000","last_undegraded":"2022-09-29T05:25:03.452852+0000","last_fullsized":"2022-09-29T05:25:03.452852+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.b","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2022-09-29T05:25:02.423315+0000","last_change":"2022-09-29T05:25:02.433388+0000","last_active":"2022-09-29T05:25:02.423315+0000","last_peered":"2022-09-29T05:25:02.423315+0000","last_clean":"2022-09-29T05:25:02.423315+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:02.423315+0000","last_undegraded":"2022-09-29T05:25:02.423315+0000","last_fullsized":"2022-09-29T05:25:02.423315+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[4,5],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.824054+0000","last_change":"2022-09-29T05:25:03.818102+0000","last_active":"2022-09-29T05:25:03.824054+0000","last_peered":"2022-09-29T05:25:03.824054+0000","last_clean":"2022-09-29T05:25:03.824054+0000","last_became_active":"2022-09-29T05:25:03.817772+0000","last_became_peered":"2022-09-29T05:25:03.817772+0000","last_unstale":"2022-09-29T05:25:03.824054+0000","last_undegraded":"2022-09-29T05:25:03.824054+0000","last_fullsized":"2022-09-29T05:25:03.824054+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.c","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2022-09-29T05:25:02.423315+0000","last_change":"2022-09-29T05:25:02.436663+0000","last_active":"2022-09-29T05:25:02.423315+0000","last_peered":"2022-09-29T05:25:02.423315+0000","last_clean":"2022-09-29T05:25:02.423315+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:02.423315+0000","last_undegraded":"2022-09-29T05:25:02.423315+0000","last_fullsized":"2022-09-29T05:25:02.423315+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,5],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.d","version":"50'1","reported_seq":11,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.798141+0000","last_change":"2022-09-29T05:25:03.452648+0000","last_active":"2022-09-29T05:25:03.798141+0000","last_peered":"2022-09-29T05:25:03.798141+0000","last_clean":"2022-09-29T05:25:03.798141+0000","last_became_active":"2022-09-29T05:25:03.452242+0000","last_became_peered":"2022-09-29T05:25:03.452242+0000","last_unstale":"2022-09-29T05:25:03.798141+0000","last_undegraded":"2022-09-29T05:25:03.798141+0000","last_fullsized":"2022-09-29T05:25:03.798141+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":9,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.809640+0000","last_change":"2022-09-29T05:25:03.809640+0000","last_active":"2022-09-29T05:25:03.809640+0000","last_peered":"2022-09-29T05:25:03.809640+0000","last_clean":"2022-09-29T05:25:03.809640+0000","last_became_active":"2022-09-29T05:25:03.809298+0000","last_became_peered":"2022-09-29T05:25:03.809298+0000","last_unstale":"2022-09-29T05:25:03.809640+0000","last_undegraded":"2022-09-29T05:25:03.809640+0000","last_fullsized":"2022-09-29T05:25:03.809640+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.e","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2022-09-29T05:25:02.423315+0000","last_change":"2022-09-29T05:25:02.439603+0000","last_active":"2022-09-29T05:25:02.423315+0000","last_peered":"2022-09-29T05:25:02.423315+0000","last_clean":"2022-09-29T05:25:02.423315+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:02.423315+0000","last_undegraded":"2022-09-29T05:25:02.423315+0000","last_fullsized":"2022-09-29T05:25:02.423315+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[3,7],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.20","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.813607+0000","last_change":"2022-09-29T05:25:03.811486+0000","last_active":"2022-09-29T05:25:03.813607+0000","last_peered":"2022-09-29T05:25:03.813607+0000","last_clean":"2022-09-29T05:25:03.813607+0000","last_became_active":"2022-09-29T05:25:03.811222+0000","last_became_peered":"2022-09-29T05:25:03.811222+0000","last_unstale":"2022-09-29T05:25:03.813607+0000","last_undegraded":"2022-09-29T05:25:03.813607+0000","last_fullsized":"2022-09-29T05:25:03.813607+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.21","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.22","version":"0'0","reported_seq":9,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.817645+0000","last_change":"2022-09-29T05:25:03.817645+0000","last_active":"2022-09-29T05:25:03.817645+0000","last_peered":"2022-09-29T05:25:03.817645+0000","last_clean":"2022-09-29T05:25:03.817645+0000","last_became_active":"2022-09-29T05:25:03.816740+0000","last_became_peered":"2022-09-29T05:25:03.816740+0000","last_unstale":"2022-09-29T05:25:03.817645+0000","last_undegraded":"2022-09-29T05:25:03.817645+0000","last_fullsized":"2022-09-29T05:25:03.817645+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.23","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.24","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.25","version":"0'0","reported_seq":9,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.817066+0000","last_change":"2022-09-29T05:25:03.817066+0000","last_active":"2022-09-29T05:25:03.817066+0000","last_peered":"2022-09-29T05:25:03.817066+0000","last_clean":"2022-09-29T05:25:03.817066+0000","last_became_active":"2022-09-29T05:25:03.816747+0000","last_became_peered":"2022-09-29T05:25:03.816747+0000","last_unstale":"2022-09-29T05:25:03.817066+0000","last_undegraded":"2022-09-29T05:25:03.817066+0000","last_fullsized":"2022-09-29T05:25:03.817066+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.26","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.824138+0000","last_change":"2022-09-29T05:25:03.823457+0000","last_active":"2022-09-29T05:25:03.824138+0000","last_peered":"2022-09-29T05:25:03.824138+0000","last_clean":"2022-09-29T05:25:03.824138+0000","last_became_active":"2022-09-29T05:25:03.823149+0000","last_became_peered":"2022-09-29T05:25:03.823149+0000","last_unstale":"2022-09-29T05:25:03.824138+0000","last_undegraded":"2022-09-29T05:25:03.824138+0000","last_fullsized":"2022-09-29T05:25:03.824138+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,4],"acting":[6,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.27","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.28","version":"0'0","reported_seq":9,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.815620+0000","last_change":"2022-09-29T05:25:03.815620+0000","last_active":"2022-09-29T05:25:03.815620+0000","last_peered":"2022-09-29T05:25:03.815620+0000","last_clean":"2022-09-29T05:25:03.815620+0000","last_became_active":"2022-09-29T05:25:03.814336+0000","last_became_peered":"2022-09-29T05:25:03.814336+0000","last_unstale":"2022-09-29T05:25:03.815620+0000","last_undegraded":"2022-09-29T05:25:03.815620+0000","last_fullsized":"2022-09-29T05:25:03.815620+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.29","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.2a","version":"0'0","reported_seq":9,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.819098+0000","last_change":"2022-09-29T05:25:03.819098+0000","last_active":"2022-09-29T05:25:03.819098+0000","last_peered":"2022-09-29T05:25:03.819098+0000","last_clean":"2022-09-29T05:25:03.819098+0000","last_became_active":"2022-09-29T05:25:03.818690+0000","last_became_peered":"2022-09-29T05:25:03.818690+0000","last_unstale":"2022-09-29T05:25:03.819098+0000","last_undegraded":"2022-09-29T05:25:03.819098+0000","last_fullsized":"2022-09-29T05:25:03.819098+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2b","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.2c","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.815650+0000","last_change":"2022-09-29T05:25:03.812644+0000","last_active":"2022-09-29T05:25:03.815650+0000","last_peered":"2022-09-29T05:25:03.815650+0000","last_clean":"2022-09-29T05:25:03.815650+0000","last_became_active":"2022-09-29T05:25:03.812339+0000","last_became_peered":"2022-09-29T05:25:03.812339+0000","last_unstale":"2022-09-29T05:25:03.815650+0000","last_undegraded":"2022-09-29T05:25:03.815650+0000","last_fullsized":"2022-09-29T05:25:03.815650+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":3,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":75,"acting":75,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":7},{"poolid":2,"num_pg":32,"stat_sum":{"num_bytes":1696,"num_objects":8,"num_object_clones":0,"num_object_copies":24,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14,"num_write_kb":5,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":36864,"data_stored":3322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":13,"ondisk_log_size":13,"up":78,"acting":78,"num_store_stats":5}],"osd_stats":[{"osd":7,"up_from":45,"seq":193273528326,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5176,"kb_used_data":432,"kb_used_omap":0,"kb_used_meta":4736,"kb_avail":93739976,"statfs":{"total":95995035648,"available":95989735424,"internally_reserved":0,"allocated":442368,"data_stored":174771,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":4849664},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.081}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.59399999999999997}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.028}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.69299999999999995}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.103}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.65400000000000003}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.63}]}]},{"osd":6,"up_from":40,"seq":171798691849,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5480,"kb_used_data":480,"kb_used_omap":0,"kb_used_meta":4992,"kb_avail":93739672,"statfs":{"total":95995035648,"available":95989424128,"internally_reserved":0,"allocated":491520,"data_stored":196260,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5111808},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.38200000000000001}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45100000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.439}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.36199999999999999}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.433}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.46200000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.42399999999999999}]}]},{"osd":1,"up_from":13,"seq":55834574874,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5736,"kb_used_data":480,"kb_used_omap":0,"kb_used_meta":5248,"kb_avail":93739416,"statfs":{"total":95995035648,"available":95989161984,"internally_reserved":0,"allocated":491520,"data_stored":195199,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5373952},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Sep 29 05:24:04 2022","interfaces":[{"interface":"back","average":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"last":0.35799999999999998},{"interface":"front","average":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"last":0.27800000000000002}]},{"osd":2,"last update":"Thu Sep 29 05:24:22 2022","interfaces":[{"interface":"back","average":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"last":0.32000000000000001},{"interface":"front","average":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.374}]},{"osd":3,"last update":"Thu Sep 29 05:24:43 2022","interfaces":[{"interface":"back","average":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"last":0.34699999999999998},{"interface":"front","average":{"1min":0.442,"5min":0.442,"15min":0.442},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"last":0.30199999999999999}]},{"osd":4,"last update":"Thu Sep 29 05:25:00 2022","interfaces":[{"interface":"back","average":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":1.1619999999999999,"5min":1.1619999999999999,"15min":1.1619999999999999},"last":0.67700000000000005},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":1.1799999999999999,"5min":1.1799999999999999,"15min":1.1799999999999999},"last":0.66800000000000004}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56200000000000006}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.69799999999999995}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.61599999999999999}]}]},{"osd":0,"up_from":8,"seq":34359738397,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5500,"kb_used_data":436,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739652,"statfs":{"total":95995035648,"available":95989403648,"internally_reserved":0,"allocated":446464,"data_stored":175086,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Sep 29 05:24:05 2022","interfaces":[{"interface":"back","average":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"min":{"1min":0.17699999999999999,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.28000000000000003},{"interface":"front","average":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.35399999999999998}]},{"osd":2,"last update":"Thu Sep 29 05:24:29 2022","interfaces":[{"interface":"back","average":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"last":0.29399999999999998},{"interface":"front","average":{"1min":0.377,"5min":0.377,"15min":0.377},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"last":0.36699999999999999}]},{"osd":3,"last update":"Thu Sep 29 05:24:41 2022","interfaces":[{"interface":"back","average":{"1min":0.437,"5min":0.437,"15min":0.437},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"last":0.32900000000000001},{"interface":"front","average":{"1min":0.434,"5min":0.434,"15min":0.434},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"last":0.314}]},{"osd":4,"last update":"Thu Sep 29 05:24:57 2022","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.50800000000000001}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.46000000000000002}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.47899999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56999999999999995}]}]},{"osd":2,"up_from":18,"seq":77309411350,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5448,"kb_used_data":448,"kb_used_omap":0,"kb_used_meta":4992,"kb_avail":93739704,"statfs":{"total":95995035648,"available":95989456896,"internally_reserved":0,"allocated":458752,"data_stored":181265,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5111808},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Sep 29 05:24:26 2022","interfaces":[{"interface":"back","average":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"last":0.318},{"interface":"front","average":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":0.72099999999999997,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.28000000000000003}]},{"osd":1,"last update":"Thu Sep 29 05:24:26 2022","interfaces":[{"interface":"back","average":{"1min":0.38,"5min":0.38,"15min":0.38},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"last":0.33300000000000002},{"interface":"front","average":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"last":0.35599999999999998}]},{"osd":3,"last update":"Thu Sep 29 05:24:41 2022","interfaces":[{"interface":"back","average":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"last":0.34899999999999998},{"interface":"front","average":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.71799999999999997,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.38600000000000001}]},{"osd":4,"last update":"Thu Sep 29 05:24:56 2022","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.55700000000000005}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.59499999999999997}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.64200000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.61899999999999999}]}]},{"osd":3,"up_from":26,"seq":111669149714,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5372,"kb_used_data":436,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739780,"statfs":{"total":95995035648,"available":95989534720,"internally_reserved":0,"allocated":446464,"data_stored":175086,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Sep 29 05:24:44 2022","interfaces":[{"interface":"back","average":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"last":0.29399999999999998},{"interface":"front","average":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"last":0.33900000000000002}]},{"osd":1,"last update":"Thu Sep 29 05:24:44 2022","interfaces":[{"interface":"back","average":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"last":0.30399999999999999}]},{"osd":2,"last update":"Thu Sep 29 05:24:44 2022","interfaces":[{"interface":"back","average":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"last":0.35299999999999998},{"interface":"front","average":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"last":0.32300000000000001}]},{"osd":4,"last update":"Thu Sep 29 05:25:00 2022","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.84099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.46999999999999997}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51900000000000002}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51200000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.50700000000000001}]}]},{"osd":4,"up_from":30,"seq":128849018894,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5308,"kb_used_data":436,"kb_used_omap":0,"kb_used_meta":4864,"kb_avail":93739844,"statfs":{"total":95995035648,"available":95989600256,"internally_reserved":0,"allocated":446464,"data_stored":175086,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":4980736},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Sep 29 05:24:59 2022","interfaces":[{"interface":"back","average":{"1min":0.47799999999999998,"5min":0.47799999999999998,"15min":0.47799999999999998},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.71999999999999997,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.67500000000000004,"5min":0.67500000000000004,"15min":0.67500000000000004},"last":0.434}]},{"osd":1,"last update":"Thu Sep 29 05:24:59 2022","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.45000000000000001},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.749,"5min":0.749,"15min":0.749},"last":0.56100000000000005}]},{"osd":2,"last update":"Thu Sep 29 05:24:59 2022","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.47199999999999998},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.51700000000000002,"15min":0.51700000000000002},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.50800000000000001}]},{"osd":3,"last update":"Thu Sep 29 05:24:59 2022","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.57499999999999996},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.54000000000000004}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.53000000000000003}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.52400000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.55500000000000005}]}]},{"osd":5,"up_from":35,"seq":150323855373,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5392,"kb_used_data":456,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739760,"statfs":{"total":95995035648,"available":95989514240,"internally_reserved":0,"allocated":466944,"data_stored":187587,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.55900000000000005}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.50700000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.49299999999999999}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45500000000000002}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.41199999999999998}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.46300000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.61199999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1626,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":1536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2022-09-29T05:25:07.104 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2022-09-29T05:25:07.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:06 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:25:05.591809+0000 mgr.y (mgr.14146) 161 : cluster [DBG] pgmap v146: 97 pgs: 10 creating+peering, 45 unknown, 42 active+clean; 1.7 KiB data, 42 MiB used, 715 GiB / 715 GiB avail; 3.8 KiB/s wr, 11 op/s 2022-09-29T05:25:07.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:06 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:05.819902+0000 mon.a (mon.0) 561 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "false"}]': finished 2022-09-29T05:25:07.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:06 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:25:05.819996+0000 mon.a (mon.0) 562 : cluster [DBG] fsmap cephfs:1 {0=b=up:active} 1 up:standby 2022-09-29T05:25:07.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:06 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:06.149184+0000 mon.c (mon.1) 21 : audit [DBG] from='client.? 172.21.15.83:0/794198628' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2022-09-29T05:25:07.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:06 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:06.443576+0000 mon.c (mon.1) 22 : audit [DBG] from='client.? 172.21.15.83:0/601179928' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2022-09-29T05:25:07.423 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:25:07.425 INFO:teuthology.orchestra.run.smithi083.stderr:dumped all 2022-09-29T05:25:07.434 INFO:teuthology.orchestra.run.smithi083.stdout:{"pg_ready":true,"pg_map":{"version":146,"stamp":"2022-09-29T05:25:05.591588+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1696,"num_objects":8,"num_object_clones":0,"num_object_copies":24,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14,"num_write_kb":5,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":13,"ondisk_log_size":13,"up":156,"acting":156,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":749961216,"kb_used":43412,"kb_used_data":3604,"kb_used_omap":0,"kb_used_meta":39744,"kb_avail":749917804,"statfs":{"total":767960285184,"available":767915831296,"internally_reserved":0,"allocated":3690496,"data_stored":1460340,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":40697856},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1536,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"0.794136"},"pg_stats":[{"pgid":"3.3f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.3e","version":"0'0","reported_seq":9,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.816168+0000","last_change":"2022-09-29T05:25:03.816168+0000","last_active":"2022-09-29T05:25:03.816168+0000","last_peered":"2022-09-29T05:25:03.816168+0000","last_clean":"2022-09-29T05:25:03.816168+0000","last_became_active":"2022-09-29T05:25:03.815888+0000","last_became_peered":"2022-09-29T05:25:03.815888+0000","last_unstale":"2022-09-29T05:25:03.816168+0000","last_undegraded":"2022-09-29T05:25:03.816168+0000","last_fullsized":"2022-09-29T05:25:03.816168+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3d","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.3c","version":"0'0","reported_seq":9,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.819554+0000","last_change":"2022-09-29T05:25:03.819554+0000","last_active":"2022-09-29T05:25:03.819554+0000","last_peered":"2022-09-29T05:25:03.819554+0000","last_clean":"2022-09-29T05:25:03.819554+0000","last_became_active":"2022-09-29T05:25:03.819207+0000","last_became_peered":"2022-09-29T05:25:03.819207+0000","last_unstale":"2022-09-29T05:25:03.819554+0000","last_undegraded":"2022-09-29T05:25:03.819554+0000","last_fullsized":"2022-09-29T05:25:03.819554+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3b","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.3a","version":"0'0","reported_seq":9,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.817871+0000","last_change":"2022-09-29T05:25:03.817871+0000","last_active":"2022-09-29T05:25:03.817871+0000","last_peered":"2022-09-29T05:25:03.817871+0000","last_clean":"2022-09-29T05:25:03.817871+0000","last_became_active":"2022-09-29T05:25:03.816800+0000","last_became_peered":"2022-09-29T05:25:03.816800+0000","last_unstale":"2022-09-29T05:25:03.817871+0000","last_undegraded":"2022-09-29T05:25:03.817871+0000","last_fullsized":"2022-09-29T05:25:03.817871+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.39","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.38","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.37","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.36","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.825144+0000","last_change":"2022-09-29T05:25:03.823656+0000","last_active":"2022-09-29T05:25:03.825144+0000","last_peered":"2022-09-29T05:25:03.825144+0000","last_clean":"2022-09-29T05:25:03.825144+0000","last_became_active":"2022-09-29T05:25:03.823109+0000","last_became_peered":"2022-09-29T05:25:03.823109+0000","last_unstale":"2022-09-29T05:25:03.825144+0000","last_undegraded":"2022-09-29T05:25:03.825144+0000","last_fullsized":"2022-09-29T05:25:03.825144+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.35","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.825174+0000","last_change":"2022-09-29T05:25:03.824377+0000","last_active":"2022-09-29T05:25:03.825174+0000","last_peered":"2022-09-29T05:25:03.825174+0000","last_clean":"2022-09-29T05:25:03.825174+0000","last_became_active":"2022-09-29T05:25:03.823869+0000","last_became_peered":"2022-09-29T05:25:03.823869+0000","last_unstale":"2022-09-29T05:25:03.825174+0000","last_undegraded":"2022-09-29T05:25:03.825174+0000","last_fullsized":"2022-09-29T05:25:03.825174+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.34","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.33","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.32","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.31","version":"0'0","reported_seq":9,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.805590+0000","last_change":"2022-09-29T05:25:03.805590+0000","last_active":"2022-09-29T05:25:03.805590+0000","last_peered":"2022-09-29T05:25:03.805590+0000","last_clean":"2022-09-29T05:25:03.805590+0000","last_became_active":"2022-09-29T05:25:03.805310+0000","last_became_peered":"2022-09-29T05:25:03.805310+0000","last_unstale":"2022-09-29T05:25:03.805590+0000","last_undegraded":"2022-09-29T05:25:03.805590+0000","last_fullsized":"2022-09-29T05:25:03.805590+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.30","version":"0'0","reported_seq":9,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.815886+0000","last_change":"2022-09-29T05:25:03.815886+0000","last_active":"2022-09-29T05:25:03.815886+0000","last_peered":"2022-09-29T05:25:03.815886+0000","last_clean":"2022-09-29T05:25:03.815886+0000","last_became_active":"2022-09-29T05:25:03.815285+0000","last_became_peered":"2022-09-29T05:25:03.815285+0000","last_unstale":"2022-09-29T05:25:03.815886+0000","last_undegraded":"2022-09-29T05:25:03.815886+0000","last_fullsized":"2022-09-29T05:25:03.815886+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2f","version":"0'0","reported_seq":9,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.813784+0000","last_change":"2022-09-29T05:25:03.813784+0000","last_active":"2022-09-29T05:25:03.813784+0000","last_peered":"2022-09-29T05:25:03.813784+0000","last_clean":"2022-09-29T05:25:03.813784+0000","last_became_active":"2022-09-29T05:25:03.813420+0000","last_became_peered":"2022-09-29T05:25:03.813420+0000","last_unstale":"2022-09-29T05:25:03.813784+0000","last_undegraded":"2022-09-29T05:25:03.813784+0000","last_fullsized":"2022-09-29T05:25:03.813784+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2e","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.2d","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.1f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:02.423315+0000","last_change":"2022-09-29T05:25:02.423315+0000","last_active":"2022-09-29T05:25:02.423315+0000","last_peered":"2022-09-29T05:25:02.423315+0000","last_clean":"2022-09-29T05:25:02.423315+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:02.423315+0000","last_undegraded":"2022-09-29T05:25:02.423315+0000","last_fullsized":"2022-09-29T05:25:02.423315+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.1e","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.1e","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:02.423315+0000","last_change":"2022-09-29T05:25:02.423315+0000","last_active":"2022-09-29T05:25:02.423315+0000","last_peered":"2022-09-29T05:25:02.423315+0000","last_clean":"2022-09-29T05:25:02.423315+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:02.423315+0000","last_undegraded":"2022-09-29T05:25:02.423315+0000","last_fullsized":"2022-09-29T05:25:02.423315+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.1f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.1d","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2022-09-29T05:25:02.423315+0000","last_change":"2022-09-29T05:25:02.433891+0000","last_active":"2022-09-29T05:25:02.423315+0000","last_peered":"2022-09-29T05:25:02.423315+0000","last_clean":"2022-09-29T05:25:02.423315+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:02.423315+0000","last_undegraded":"2022-09-29T05:25:02.423315+0000","last_fullsized":"2022-09-29T05:25:02.423315+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,6],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"0'0","reported_seq":9,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.816581+0000","last_change":"2022-09-29T05:25:03.816581+0000","last_active":"2022-09-29T05:25:03.816581+0000","last_peered":"2022-09-29T05:25:03.816581+0000","last_clean":"2022-09-29T05:25:03.816581+0000","last_became_active":"2022-09-29T05:25:03.816206+0000","last_became_peered":"2022-09-29T05:25:03.816206+0000","last_unstale":"2022-09-29T05:25:03.816581+0000","last_undegraded":"2022-09-29T05:25:03.816581+0000","last_fullsized":"2022-09-29T05:25:03.816581+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1c","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2022-09-29T05:25:02.423315+0000","last_change":"2022-09-29T05:25:02.433333+0000","last_active":"2022-09-29T05:25:02.423315+0000","last_peered":"2022-09-29T05:25:02.423315+0000","last_clean":"2022-09-29T05:25:02.423315+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:02.423315+0000","last_undegraded":"2022-09-29T05:25:02.423315+0000","last_fullsized":"2022-09-29T05:25:02.423315+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[2,5],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1d","version":"0'0","reported_seq":9,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.820523+0000","last_change":"2022-09-29T05:25:03.820523+0000","last_active":"2022-09-29T05:25:03.820523+0000","last_peered":"2022-09-29T05:25:03.820523+0000","last_clean":"2022-09-29T05:25:03.820523+0000","last_became_active":"2022-09-29T05:25:03.819546+0000","last_became_peered":"2022-09-29T05:25:03.819546+0000","last_unstale":"2022-09-29T05:25:03.820523+0000","last_undegraded":"2022-09-29T05:25:03.820523+0000","last_fullsized":"2022-09-29T05:25:03.820523+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1b","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:02.423315+0000","last_change":"2022-09-29T05:25:02.423315+0000","last_active":"2022-09-29T05:25:02.423315+0000","last_peered":"2022-09-29T05:25:02.423315+0000","last_clean":"2022-09-29T05:25:02.423315+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:02.423315+0000","last_undegraded":"2022-09-29T05:25:02.423315+0000","last_fullsized":"2022-09-29T05:25:02.423315+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.1a","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.1a","version":"50'1","reported_seq":12,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.815499+0000","last_change":"2022-09-29T05:25:03.454747+0000","last_active":"2022-09-29T05:25:03.815499+0000","last_peered":"2022-09-29T05:25:03.815499+0000","last_clean":"2022-09-29T05:25:03.815499+0000","last_became_active":"2022-09-29T05:25:03.454395+0000","last_became_peered":"2022-09-29T05:25:03.454395+0000","last_unstale":"2022-09-29T05:25:03.815499+0000","last_undegraded":"2022-09-29T05:25:03.815499+0000","last_fullsized":"2022-09-29T05:25:03.815499+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1b","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.19","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:02.423315+0000","last_change":"2022-09-29T05:25:02.423315+0000","last_active":"2022-09-29T05:25:02.423315+0000","last_peered":"2022-09-29T05:25:02.423315+0000","last_clean":"2022-09-29T05:25:02.423315+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:02.423315+0000","last_undegraded":"2022-09-29T05:25:02.423315+0000","last_fullsized":"2022-09-29T05:25:02.423315+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.18","version":"0'0","reported_seq":10,"reported_epoch":49,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.464050+0000","last_change":"2022-09-29T05:25:03.460675+0000","last_active":"2022-09-29T05:25:03.464050+0000","last_peered":"2022-09-29T05:25:03.464050+0000","last_clean":"2022-09-29T05:25:03.464050+0000","last_became_active":"2022-09-29T05:25:03.460351+0000","last_became_peered":"2022-09-29T05:25:03.460351+0000","last_unstale":"2022-09-29T05:25:03.464050+0000","last_undegraded":"2022-09-29T05:25:03.464050+0000","last_fullsized":"2022-09-29T05:25:03.464050+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":9,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.816170+0000","last_change":"2022-09-29T05:25:03.816170+0000","last_active":"2022-09-29T05:25:03.816170+0000","last_peered":"2022-09-29T05:25:03.816170+0000","last_clean":"2022-09-29T05:25:03.816170+0000","last_became_active":"2022-09-29T05:25:03.815838+0000","last_became_peered":"2022-09-29T05:25:03.815838+0000","last_unstale":"2022-09-29T05:25:03.816170+0000","last_undegraded":"2022-09-29T05:25:03.816170+0000","last_fullsized":"2022-09-29T05:25:03.816170+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.17","version":"0'0","reported_seq":11,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.805981+0000","last_change":"2022-09-29T05:25:03.461045+0000","last_active":"2022-09-29T05:25:03.805981+0000","last_peered":"2022-09-29T05:25:03.805981+0000","last_clean":"2022-09-29T05:25:03.805981+0000","last_became_active":"2022-09-29T05:25:03.460729+0000","last_became_peered":"2022-09-29T05:25:03.460729+0000","last_unstale":"2022-09-29T05:25:03.805981+0000","last_undegraded":"2022-09-29T05:25:03.805981+0000","last_fullsized":"2022-09-29T05:25:03.805981+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":9,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.818733+0000","last_change":"2022-09-29T05:25:03.818733+0000","last_active":"2022-09-29T05:25:03.818733+0000","last_peered":"2022-09-29T05:25:03.818733+0000","last_clean":"2022-09-29T05:25:03.818733+0000","last_became_active":"2022-09-29T05:25:03.816983+0000","last_became_peered":"2022-09-29T05:25:03.816983+0000","last_unstale":"2022-09-29T05:25:03.818733+0000","last_undegraded":"2022-09-29T05:25:03.818733+0000","last_fullsized":"2022-09-29T05:25:03.818733+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.16","version":"0'0","reported_seq":10,"reported_epoch":49,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.464237+0000","last_change":"2022-09-29T05:25:03.463477+0000","last_active":"2022-09-29T05:25:03.464237+0000","last_peered":"2022-09-29T05:25:03.464237+0000","last_clean":"2022-09-29T05:25:03.464237+0000","last_became_active":"2022-09-29T05:25:03.463051+0000","last_became_peered":"2022-09-29T05:25:03.463051+0000","last_unstale":"2022-09-29T05:25:03.464237+0000","last_undegraded":"2022-09-29T05:25:03.464237+0000","last_fullsized":"2022-09-29T05:25:03.464237+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.15","version":"50'3","reported_seq":13,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.811766+0000","last_change":"2022-09-29T05:25:03.457077+0000","last_active":"2022-09-29T05:25:03.811766+0000","last_peered":"2022-09-29T05:25:03.811766+0000","last_clean":"2022-09-29T05:25:03.811766+0000","last_became_active":"2022-09-29T05:25:03.456823+0000","last_became_peered":"2022-09-29T05:25:03.456823+0000","last_unstale":"2022-09-29T05:25:03.811766+0000","last_undegraded":"2022-09-29T05:25:03.811766+0000","last_fullsized":"2022-09-29T05:25:03.811766+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":3,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.14","version":"50'6","reported_seq":17,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.827688+0000","last_change":"2022-09-29T05:25:03.461139+0000","last_active":"2022-09-29T05:25:03.827688+0000","last_peered":"2022-09-29T05:25:03.827688+0000","last_clean":"2022-09-29T05:25:03.827688+0000","last_became_active":"2022-09-29T05:25:03.460829+0000","last_became_peered":"2022-09-29T05:25:03.460829+0000","last_unstale":"2022-09-29T05:25:03.827688+0000","last_undegraded":"2022-09-29T05:25:03.827688+0000","last_fullsized":"2022-09-29T05:25:03.827688+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":6,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":1536,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.13","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:02.423315+0000","last_change":"2022-09-29T05:25:02.423315+0000","last_active":"2022-09-29T05:25:02.423315+0000","last_peered":"2022-09-29T05:25:02.423315+0000","last_clean":"2022-09-29T05:25:02.423315+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:02.423315+0000","last_undegraded":"2022-09-29T05:25:02.423315+0000","last_fullsized":"2022-09-29T05:25:02.423315+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.12","version":"0'0","reported_seq":10,"reported_epoch":49,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.464191+0000","last_change":"2022-09-29T05:25:03.462145+0000","last_active":"2022-09-29T05:25:03.464191+0000","last_peered":"2022-09-29T05:25:03.464191+0000","last_clean":"2022-09-29T05:25:03.464191+0000","last_became_active":"2022-09-29T05:25:03.461864+0000","last_became_peered":"2022-09-29T05:25:03.461864+0000","last_unstale":"2022-09-29T05:25:03.464191+0000","last_undegraded":"2022-09-29T05:25:03.464191+0000","last_fullsized":"2022-09-29T05:25:03.464191+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.11","version":"0'0","reported_seq":11,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.805505+0000","last_change":"2022-09-29T05:25:03.454654+0000","last_active":"2022-09-29T05:25:03.805505+0000","last_peered":"2022-09-29T05:25:03.805505+0000","last_clean":"2022-09-29T05:25:03.805505+0000","last_became_active":"2022-09-29T05:25:03.454319+0000","last_became_peered":"2022-09-29T05:25:03.454319+0000","last_unstale":"2022-09-29T05:25:03.805505+0000","last_undegraded":"2022-09-29T05:25:03.805505+0000","last_fullsized":"2022-09-29T05:25:03.805505+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.813586+0000","last_change":"2022-09-29T05:25:03.811425+0000","last_active":"2022-09-29T05:25:03.813586+0000","last_peered":"2022-09-29T05:25:03.813586+0000","last_clean":"2022-09-29T05:25:03.813586+0000","last_became_active":"2022-09-29T05:25:03.811120+0000","last_became_peered":"2022-09-29T05:25:03.811120+0000","last_unstale":"2022-09-29T05:25:03.813586+0000","last_undegraded":"2022-09-29T05:25:03.813586+0000","last_fullsized":"2022-09-29T05:25:03.813586+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.10","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2022-09-29T05:25:02.423315+0000","last_change":"2022-09-29T05:25:02.439448+0000","last_active":"2022-09-29T05:25:02.423315+0000","last_peered":"2022-09-29T05:25:02.423315+0000","last_clean":"2022-09-29T05:25:02.423315+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:02.423315+0000","last_undegraded":"2022-09-29T05:25:02.423315+0000","last_fullsized":"2022-09-29T05:25:02.423315+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,1],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:02.423315+0000","last_change":"2022-09-29T05:25:02.423315+0000","last_active":"2022-09-29T05:25:02.423315+0000","last_peered":"2022-09-29T05:25:02.423315+0000","last_clean":"2022-09-29T05:25:02.423315+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:02.423315+0000","last_undegraded":"2022-09-29T05:25:02.423315+0000","last_fullsized":"2022-09-29T05:25:02.423315+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.e","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":10,"reported_epoch":49,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.454814+0000","last_change":"2022-09-29T05:25:03.454507+0000","last_active":"2022-09-29T05:25:03.454814+0000","last_peered":"2022-09-29T05:25:03.454814+0000","last_clean":"2022-09-29T05:25:03.454814+0000","last_became_active":"2022-09-29T05:25:03.454195+0000","last_became_peered":"2022-09-29T05:25:03.454195+0000","last_unstale":"2022-09-29T05:25:03.454814+0000","last_undegraded":"2022-09-29T05:25:03.454814+0000","last_fullsized":"2022-09-29T05:25:03.454814+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":9,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.819862+0000","last_change":"2022-09-29T05:25:03.819862+0000","last_active":"2022-09-29T05:25:03.819862+0000","last_peered":"2022-09-29T05:25:03.819862+0000","last_clean":"2022-09-29T05:25:03.819862+0000","last_became_active":"2022-09-29T05:25:03.818117+0000","last_became_peered":"2022-09-29T05:25:03.818117+0000","last_unstale":"2022-09-29T05:25:03.819862+0000","last_undegraded":"2022-09-29T05:25:03.819862+0000","last_fullsized":"2022-09-29T05:25:03.819862+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":10,"reported_epoch":49,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.463942+0000","last_change":"2022-09-29T05:25:03.459646+0000","last_active":"2022-09-29T05:25:03.463942+0000","last_peered":"2022-09-29T05:25:03.463942+0000","last_clean":"2022-09-29T05:25:03.463942+0000","last_became_active":"2022-09-29T05:25:03.459320+0000","last_became_peered":"2022-09-29T05:25:03.459320+0000","last_unstale":"2022-09-29T05:25:03.463942+0000","last_undegraded":"2022-09-29T05:25:03.463942+0000","last_fullsized":"2022-09-29T05:25:03.463942+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":9,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.812390+0000","last_change":"2022-09-29T05:25:03.812390+0000","last_active":"2022-09-29T05:25:03.812390+0000","last_peered":"2022-09-29T05:25:03.812390+0000","last_clean":"2022-09-29T05:25:03.812390+0000","last_became_active":"2022-09-29T05:25:03.811925+0000","last_became_peered":"2022-09-29T05:25:03.811925+0000","last_unstale":"2022-09-29T05:25:03.812390+0000","last_undegraded":"2022-09-29T05:25:03.812390+0000","last_fullsized":"2022-09-29T05:25:03.812390+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2022-09-29T05:25:02.423315+0000","last_change":"2022-09-29T05:25:02.435837+0000","last_active":"2022-09-29T05:25:02.423315+0000","last_peered":"2022-09-29T05:25:02.423315+0000","last_clean":"2022-09-29T05:25:02.423315+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:02.423315+0000","last_undegraded":"2022-09-29T05:25:02.423315+0000","last_fullsized":"2022-09-29T05:25:02.423315+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,3],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":10,"reported_epoch":49,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.464203+0000","last_change":"2022-09-29T05:25:03.462271+0000","last_active":"2022-09-29T05:25:03.464203+0000","last_peered":"2022-09-29T05:25:03.464203+0000","last_clean":"2022-09-29T05:25:03.464203+0000","last_became_active":"2022-09-29T05:25:03.461959+0000","last_became_peered":"2022-09-29T05:25:03.461959+0000","last_unstale":"2022-09-29T05:25:03.464203+0000","last_undegraded":"2022-09-29T05:25:03.464203+0000","last_fullsized":"2022-09-29T05:25:03.464203+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":11,"reported_epoch":47,"state":"active+clean","last_fresh":"2022-09-29T05:24:48.628557+0000","last_change":"2022-09-29T05:24:48.628243+0000","last_active":"2022-09-29T05:24:48.628557+0000","last_peered":"2022-09-29T05:24:48.628557+0000","last_clean":"2022-09-29T05:24:48.628557+0000","last_became_active":"2022-09-29T05:24:48.627801+0000","last_became_peered":"2022-09-29T05:24:48.627801+0000","last_unstale":"2022-09-29T05:24:48.628557+0000","last_undegraded":"2022-09-29T05:24:48.628557+0000","last_fullsized":"2022-09-29T05:24:48.628557+0000","mapping_epoch":46,"log_start":"0'0","ondisk_log_start":"0'0","created":21,"last_epoch_clean":47,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:23:35.544281+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:23:35.544281+0000","last_clean_scrub_stamp":"2022-09-29T05:23:35.544281+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2022-09-29T05:25:02.423315+0000","last_change":"2022-09-29T05:25:02.433346+0000","last_active":"2022-09-29T05:25:02.423315+0000","last_peered":"2022-09-29T05:25:02.423315+0000","last_clean":"2022-09-29T05:25:02.423315+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:02.423315+0000","last_undegraded":"2022-09-29T05:25:02.423315+0000","last_fullsized":"2022-09-29T05:25:02.423315+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,1],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2022-09-29T05:25:02.423315+0000","last_change":"2022-09-29T05:25:02.433376+0000","last_active":"2022-09-29T05:25:02.423315+0000","last_peered":"2022-09-29T05:25:02.423315+0000","last_clean":"2022-09-29T05:25:02.423315+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:02.423315+0000","last_undegraded":"2022-09-29T05:25:02.423315+0000","last_fullsized":"2022-09-29T05:25:02.423315+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,4],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":9,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.803195+0000","last_change":"2022-09-29T05:25:03.803195+0000","last_active":"2022-09-29T05:25:03.803195+0000","last_peered":"2022-09-29T05:25:03.803195+0000","last_clean":"2022-09-29T05:25:03.803195+0000","last_became_active":"2022-09-29T05:25:03.802909+0000","last_became_peered":"2022-09-29T05:25:03.802909+0000","last_unstale":"2022-09-29T05:25:03.803195+0000","last_undegraded":"2022-09-29T05:25:03.803195+0000","last_fullsized":"2022-09-29T05:25:03.803195+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":10,"reported_epoch":49,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.457008+0000","last_change":"2022-09-29T05:25:03.456735+0000","last_active":"2022-09-29T05:25:03.457008+0000","last_peered":"2022-09-29T05:25:03.457008+0000","last_clean":"2022-09-29T05:25:03.457008+0000","last_became_active":"2022-09-29T05:25:03.456427+0000","last_became_peered":"2022-09-29T05:25:03.456427+0000","last_unstale":"2022-09-29T05:25:03.457008+0000","last_undegraded":"2022-09-29T05:25:03.457008+0000","last_fullsized":"2022-09-29T05:25:03.457008+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.7","version":"50'2","reported_seq":13,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.824910+0000","last_change":"2022-09-29T05:25:03.462138+0000","last_active":"2022-09-29T05:25:03.824910+0000","last_peered":"2022-09-29T05:25:03.824910+0000","last_clean":"2022-09-29T05:25:03.824910+0000","last_became_active":"2022-09-29T05:25:03.461809+0000","last_became_peered":"2022-09-29T05:25:03.461809+0000","last_unstale":"2022-09-29T05:25:03.824910+0000","last_undegraded":"2022-09-29T05:25:03.824910+0000","last_fullsized":"2022-09-29T05:25:03.824910+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.8","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2022-09-29T05:25:02.423315+0000","last_change":"2022-09-29T05:25:02.433909+0000","last_active":"2022-09-29T05:25:02.423315+0000","last_peered":"2022-09-29T05:25:02.423315+0000","last_clean":"2022-09-29T05:25:02.423315+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:02.423315+0000","last_undegraded":"2022-09-29T05:25:02.423315+0000","last_fullsized":"2022-09-29T05:25:02.423315+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,1],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.9","version":"0'0","reported_seq":10,"reported_epoch":49,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.452964+0000","last_change":"2022-09-29T05:25:03.452512+0000","last_active":"2022-09-29T05:25:03.452964+0000","last_peered":"2022-09-29T05:25:03.452964+0000","last_clean":"2022-09-29T05:25:03.452964+0000","last_became_active":"2022-09-29T05:25:03.452222+0000","last_became_peered":"2022-09-29T05:25:03.452222+0000","last_unstale":"2022-09-29T05:25:03.452964+0000","last_undegraded":"2022-09-29T05:25:03.452964+0000","last_fullsized":"2022-09-29T05:25:03.452964+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.a","version":"0'0","reported_seq":10,"reported_epoch":49,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.452852+0000","last_change":"2022-09-29T05:25:03.451868+0000","last_active":"2022-09-29T05:25:03.452852+0000","last_peered":"2022-09-29T05:25:03.452852+0000","last_clean":"2022-09-29T05:25:03.452852+0000","last_became_active":"2022-09-29T05:25:03.451576+0000","last_became_peered":"2022-09-29T05:25:03.451576+0000","last_unstale":"2022-09-29T05:25:03.452852+0000","last_undegraded":"2022-09-29T05:25:03.452852+0000","last_fullsized":"2022-09-29T05:25:03.452852+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.b","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2022-09-29T05:25:02.423315+0000","last_change":"2022-09-29T05:25:02.433388+0000","last_active":"2022-09-29T05:25:02.423315+0000","last_peered":"2022-09-29T05:25:02.423315+0000","last_clean":"2022-09-29T05:25:02.423315+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:02.423315+0000","last_undegraded":"2022-09-29T05:25:02.423315+0000","last_fullsized":"2022-09-29T05:25:02.423315+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[4,5],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.824054+0000","last_change":"2022-09-29T05:25:03.818102+0000","last_active":"2022-09-29T05:25:03.824054+0000","last_peered":"2022-09-29T05:25:03.824054+0000","last_clean":"2022-09-29T05:25:03.824054+0000","last_became_active":"2022-09-29T05:25:03.817772+0000","last_became_peered":"2022-09-29T05:25:03.817772+0000","last_unstale":"2022-09-29T05:25:03.824054+0000","last_undegraded":"2022-09-29T05:25:03.824054+0000","last_fullsized":"2022-09-29T05:25:03.824054+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.c","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2022-09-29T05:25:02.423315+0000","last_change":"2022-09-29T05:25:02.436663+0000","last_active":"2022-09-29T05:25:02.423315+0000","last_peered":"2022-09-29T05:25:02.423315+0000","last_clean":"2022-09-29T05:25:02.423315+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:02.423315+0000","last_undegraded":"2022-09-29T05:25:02.423315+0000","last_fullsized":"2022-09-29T05:25:02.423315+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,5],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.d","version":"50'1","reported_seq":11,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.798141+0000","last_change":"2022-09-29T05:25:03.452648+0000","last_active":"2022-09-29T05:25:03.798141+0000","last_peered":"2022-09-29T05:25:03.798141+0000","last_clean":"2022-09-29T05:25:03.798141+0000","last_became_active":"2022-09-29T05:25:03.452242+0000","last_became_peered":"2022-09-29T05:25:03.452242+0000","last_unstale":"2022-09-29T05:25:03.798141+0000","last_undegraded":"2022-09-29T05:25:03.798141+0000","last_fullsized":"2022-09-29T05:25:03.798141+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":9,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.809640+0000","last_change":"2022-09-29T05:25:03.809640+0000","last_active":"2022-09-29T05:25:03.809640+0000","last_peered":"2022-09-29T05:25:03.809640+0000","last_clean":"2022-09-29T05:25:03.809640+0000","last_became_active":"2022-09-29T05:25:03.809298+0000","last_became_peered":"2022-09-29T05:25:03.809298+0000","last_unstale":"2022-09-29T05:25:03.809640+0000","last_undegraded":"2022-09-29T05:25:03.809640+0000","last_fullsized":"2022-09-29T05:25:03.809640+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.e","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2022-09-29T05:25:02.423315+0000","last_change":"2022-09-29T05:25:02.439603+0000","last_active":"2022-09-29T05:25:02.423315+0000","last_peered":"2022-09-29T05:25:02.423315+0000","last_clean":"2022-09-29T05:25:02.423315+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:02.423315+0000","last_undegraded":"2022-09-29T05:25:02.423315+0000","last_fullsized":"2022-09-29T05:25:02.423315+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[3,7],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.20","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.813607+0000","last_change":"2022-09-29T05:25:03.811486+0000","last_active":"2022-09-29T05:25:03.813607+0000","last_peered":"2022-09-29T05:25:03.813607+0000","last_clean":"2022-09-29T05:25:03.813607+0000","last_became_active":"2022-09-29T05:25:03.811222+0000","last_became_peered":"2022-09-29T05:25:03.811222+0000","last_unstale":"2022-09-29T05:25:03.813607+0000","last_undegraded":"2022-09-29T05:25:03.813607+0000","last_fullsized":"2022-09-29T05:25:03.813607+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.21","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.22","version":"0'0","reported_seq":9,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.817645+0000","last_change":"2022-09-29T05:25:03.817645+0000","last_active":"2022-09-29T05:25:03.817645+0000","last_peered":"2022-09-29T05:25:03.817645+0000","last_clean":"2022-09-29T05:25:03.817645+0000","last_became_active":"2022-09-29T05:25:03.816740+0000","last_became_peered":"2022-09-29T05:25:03.816740+0000","last_unstale":"2022-09-29T05:25:03.817645+0000","last_undegraded":"2022-09-29T05:25:03.817645+0000","last_fullsized":"2022-09-29T05:25:03.817645+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.23","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.24","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.25","version":"0'0","reported_seq":9,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.817066+0000","last_change":"2022-09-29T05:25:03.817066+0000","last_active":"2022-09-29T05:25:03.817066+0000","last_peered":"2022-09-29T05:25:03.817066+0000","last_clean":"2022-09-29T05:25:03.817066+0000","last_became_active":"2022-09-29T05:25:03.816747+0000","last_became_peered":"2022-09-29T05:25:03.816747+0000","last_unstale":"2022-09-29T05:25:03.817066+0000","last_undegraded":"2022-09-29T05:25:03.817066+0000","last_fullsized":"2022-09-29T05:25:03.817066+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.26","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.824138+0000","last_change":"2022-09-29T05:25:03.823457+0000","last_active":"2022-09-29T05:25:03.824138+0000","last_peered":"2022-09-29T05:25:03.824138+0000","last_clean":"2022-09-29T05:25:03.824138+0000","last_became_active":"2022-09-29T05:25:03.823149+0000","last_became_peered":"2022-09-29T05:25:03.823149+0000","last_unstale":"2022-09-29T05:25:03.824138+0000","last_undegraded":"2022-09-29T05:25:03.824138+0000","last_fullsized":"2022-09-29T05:25:03.824138+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,4],"acting":[6,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.27","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.28","version":"0'0","reported_seq":9,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.815620+0000","last_change":"2022-09-29T05:25:03.815620+0000","last_active":"2022-09-29T05:25:03.815620+0000","last_peered":"2022-09-29T05:25:03.815620+0000","last_clean":"2022-09-29T05:25:03.815620+0000","last_became_active":"2022-09-29T05:25:03.814336+0000","last_became_peered":"2022-09-29T05:25:03.814336+0000","last_unstale":"2022-09-29T05:25:03.815620+0000","last_undegraded":"2022-09-29T05:25:03.815620+0000","last_fullsized":"2022-09-29T05:25:03.815620+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.29","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.2a","version":"0'0","reported_seq":9,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.819098+0000","last_change":"2022-09-29T05:25:03.819098+0000","last_active":"2022-09-29T05:25:03.819098+0000","last_peered":"2022-09-29T05:25:03.819098+0000","last_clean":"2022-09-29T05:25:03.819098+0000","last_became_active":"2022-09-29T05:25:03.818690+0000","last_became_peered":"2022-09-29T05:25:03.818690+0000","last_unstale":"2022-09-29T05:25:03.819098+0000","last_undegraded":"2022-09-29T05:25:03.819098+0000","last_fullsized":"2022-09-29T05:25:03.819098+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2b","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.2c","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.815650+0000","last_change":"2022-09-29T05:25:03.812644+0000","last_active":"2022-09-29T05:25:03.815650+0000","last_peered":"2022-09-29T05:25:03.815650+0000","last_clean":"2022-09-29T05:25:03.815650+0000","last_became_active":"2022-09-29T05:25:03.812339+0000","last_became_peered":"2022-09-29T05:25:03.812339+0000","last_unstale":"2022-09-29T05:25:03.815650+0000","last_undegraded":"2022-09-29T05:25:03.815650+0000","last_fullsized":"2022-09-29T05:25:03.815650+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":3,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":75,"acting":75,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":7},{"poolid":2,"num_pg":32,"stat_sum":{"num_bytes":1696,"num_objects":8,"num_object_clones":0,"num_object_copies":24,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14,"num_write_kb":5,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":36864,"data_stored":3322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":13,"ondisk_log_size":13,"up":78,"acting":78,"num_store_stats":5}],"osd_stats":[{"osd":7,"up_from":45,"seq":193273528326,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5176,"kb_used_data":432,"kb_used_omap":0,"kb_used_meta":4736,"kb_avail":93739976,"statfs":{"total":95995035648,"available":95989735424,"internally_reserved":0,"allocated":442368,"data_stored":174771,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":4849664},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.081}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.59399999999999997}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.028}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.69299999999999995}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.103}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.65400000000000003}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.63}]}]},{"osd":6,"up_from":40,"seq":171798691849,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5480,"kb_used_data":480,"kb_used_omap":0,"kb_used_meta":4992,"kb_avail":93739672,"statfs":{"total":95995035648,"available":95989424128,"internally_reserved":0,"allocated":491520,"data_stored":196260,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5111808},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.38200000000000001}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45100000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.439}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.36199999999999999}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.433}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.46200000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.42399999999999999}]}]},{"osd":1,"up_from":13,"seq":55834574874,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5736,"kb_used_data":480,"kb_used_omap":0,"kb_used_meta":5248,"kb_avail":93739416,"statfs":{"total":95995035648,"available":95989161984,"internally_reserved":0,"allocated":491520,"data_stored":195199,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5373952},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Sep 29 05:24:04 2022","interfaces":[{"interface":"back","average":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"last":0.35799999999999998},{"interface":"front","average":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"last":0.27800000000000002}]},{"osd":2,"last update":"Thu Sep 29 05:24:22 2022","interfaces":[{"interface":"back","average":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"last":0.32000000000000001},{"interface":"front","average":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.374}]},{"osd":3,"last update":"Thu Sep 29 05:24:43 2022","interfaces":[{"interface":"back","average":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"last":0.34699999999999998},{"interface":"front","average":{"1min":0.442,"5min":0.442,"15min":0.442},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"last":0.30199999999999999}]},{"osd":4,"last update":"Thu Sep 29 05:25:00 2022","interfaces":[{"interface":"back","average":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":1.1619999999999999,"5min":1.1619999999999999,"15min":1.1619999999999999},"last":0.67700000000000005},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":1.1799999999999999,"5min":1.1799999999999999,"15min":1.1799999999999999},"last":0.66800000000000004}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56200000000000006}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.69799999999999995}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.61599999999999999}]}]},{"osd":0,"up_from":8,"seq":34359738397,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5500,"kb_used_data":436,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739652,"statfs":{"total":95995035648,"available":95989403648,"internally_reserved":0,"allocated":446464,"data_stored":175086,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Sep 29 05:24:05 2022","interfaces":[{"interface":"back","average":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"min":{"1min":0.17699999999999999,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.28000000000000003},{"interface":"front","average":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.35399999999999998}]},{"osd":2,"last update":"Thu Sep 29 05:24:29 2022","interfaces":[{"interface":"back","average":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"last":0.29399999999999998},{"interface":"front","average":{"1min":0.377,"5min":0.377,"15min":0.377},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"last":0.36699999999999999}]},{"osd":3,"last update":"Thu Sep 29 05:24:41 2022","interfaces":[{"interface":"back","average":{"1min":0.437,"5min":0.437,"15min":0.437},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"last":0.32900000000000001},{"interface":"front","average":{"1min":0.434,"5min":0.434,"15min":0.434},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"last":0.314}]},{"osd":4,"last update":"Thu Sep 29 05:24:57 2022","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.50800000000000001}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.46000000000000002}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.47899999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56999999999999995}]}]},{"osd":2,"up_from":18,"seq":77309411350,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5448,"kb_used_data":448,"kb_used_omap":0,"kb_used_meta":4992,"kb_avail":93739704,"statfs":{"total":95995035648,"available":95989456896,"internally_reserved":0,"allocated":458752,"data_stored":181265,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5111808},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Sep 29 05:24:26 2022","interfaces":[{"interface":"back","average":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"last":0.318},{"interface":"front","average":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":0.72099999999999997,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.28000000000000003}]},{"osd":1,"last update":"Thu Sep 29 05:24:26 2022","interfaces":[{"interface":"back","average":{"1min":0.38,"5min":0.38,"15min":0.38},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"last":0.33300000000000002},{"interface":"front","average":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"last":0.35599999999999998}]},{"osd":3,"last update":"Thu Sep 29 05:24:41 2022","interfaces":[{"interface":"back","average":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"last":0.34899999999999998},{"interface":"front","average":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.71799999999999997,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.38600000000000001}]},{"osd":4,"last update":"Thu Sep 29 05:24:56 2022","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.55700000000000005}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.59499999999999997}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.64200000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.61899999999999999}]}]},{"osd":3,"up_from":26,"seq":111669149714,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5372,"kb_used_data":436,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739780,"statfs":{"total":95995035648,"available":95989534720,"internally_reserved":0,"allocated":446464,"data_stored":175086,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Sep 29 05:24:44 2022","interfaces":[{"interface":"back","average":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"last":0.29399999999999998},{"interface":"front","average":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"last":0.33900000000000002}]},{"osd":1,"last update":"Thu Sep 29 05:24:44 2022","interfaces":[{"interface":"back","average":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"last":0.30399999999999999}]},{"osd":2,"last update":"Thu Sep 29 05:24:44 2022","interfaces":[{"interface":"back","average":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"last":0.35299999999999998},{"interface":"front","average":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"last":0.32300000000000001}]},{"osd":4,"last update":"Thu Sep 29 05:25:00 2022","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.84099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.46999999999999997}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51900000000000002}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51200000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.50700000000000001}]}]},{"osd":4,"up_from":30,"seq":128849018894,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5308,"kb_used_data":436,"kb_used_omap":0,"kb_used_meta":4864,"kb_avail":93739844,"statfs":{"total":95995035648,"available":95989600256,"internally_reserved":0,"allocated":446464,"data_stored":175086,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":4980736},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Sep 29 05:24:59 2022","interfaces":[{"interface":"back","average":{"1min":0.47799999999999998,"5min":0.47799999999999998,"15min":0.47799999999999998},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.71999999999999997,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.67500000000000004,"5min":0.67500000000000004,"15min":0.67500000000000004},"last":0.434}]},{"osd":1,"last update":"Thu Sep 29 05:24:59 2022","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.45000000000000001},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.749,"5min":0.749,"15min":0.749},"last":0.56100000000000005}]},{"osd":2,"last update":"Thu Sep 29 05:24:59 2022","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.47199999999999998},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.51700000000000002,"15min":0.51700000000000002},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.50800000000000001}]},{"osd":3,"last update":"Thu Sep 29 05:24:59 2022","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.57499999999999996},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.54000000000000004}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.53000000000000003}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.52400000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.55500000000000005}]}]},{"osd":5,"up_from":35,"seq":150323855373,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5392,"kb_used_data":456,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739760,"statfs":{"total":95995035648,"available":95989514240,"internally_reserved":0,"allocated":466944,"data_stored":187587,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.55900000000000005}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.50700000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.49299999999999999}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45500000000000002}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.41199999999999998}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.46300000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.61199999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1626,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":1536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2022-09-29T05:25:07.440 INFO:tasks.cephfs.filesystem.ceph_manager:PG 3.3f is not active+clean 2022-09-29T05:25:07.441 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '3.3f', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2022-09-29T05:25:03.430113+0000', 'last_change': '2022-09-29T05:25:03.430113+0000', 'last_active': '2022-09-29T05:25:03.430113+0000', 'last_peered': '2022-09-29T05:25:03.430113+0000', 'last_clean': '2022-09-29T05:25:03.430113+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2022-09-29T05:25:03.430113+0000', 'last_undegraded': '2022-09-29T05:25:03.430113+0000', 'last_fullsized': '2022-09-29T05:25:03.430113+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'last_clean_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'log_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2022-09-29T05:25:07.441 INFO:tasks.cephfs.filesystem.ceph_manager:PG 3.3d is not active+clean 2022-09-29T05:25:07.442 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '3.3d', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2022-09-29T05:25:03.430113+0000', 'last_change': '2022-09-29T05:25:03.430113+0000', 'last_active': '2022-09-29T05:25:03.430113+0000', 'last_peered': '2022-09-29T05:25:03.430113+0000', 'last_clean': '2022-09-29T05:25:03.430113+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2022-09-29T05:25:03.430113+0000', 'last_undegraded': '2022-09-29T05:25:03.430113+0000', 'last_fullsized': '2022-09-29T05:25:03.430113+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'last_clean_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'log_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2022-09-29T05:25:07.442 INFO:tasks.cephfs.filesystem.ceph_manager:PG 3.3b is not active+clean 2022-09-29T05:25:07.442 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '3.3b', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2022-09-29T05:25:03.430113+0000', 'last_change': '2022-09-29T05:25:03.430113+0000', 'last_active': '2022-09-29T05:25:03.430113+0000', 'last_peered': '2022-09-29T05:25:03.430113+0000', 'last_clean': '2022-09-29T05:25:03.430113+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2022-09-29T05:25:03.430113+0000', 'last_undegraded': '2022-09-29T05:25:03.430113+0000', 'last_fullsized': '2022-09-29T05:25:03.430113+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'last_clean_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'log_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2022-09-29T05:25:07.443 INFO:tasks.cephfs.filesystem.ceph_manager:PG 3.39 is not active+clean 2022-09-29T05:25:07.443 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '3.39', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2022-09-29T05:25:03.430113+0000', 'last_change': '2022-09-29T05:25:03.430113+0000', 'last_active': '2022-09-29T05:25:03.430113+0000', 'last_peered': '2022-09-29T05:25:03.430113+0000', 'last_clean': '2022-09-29T05:25:03.430113+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2022-09-29T05:25:03.430113+0000', 'last_undegraded': '2022-09-29T05:25:03.430113+0000', 'last_fullsized': '2022-09-29T05:25:03.430113+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'last_clean_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'log_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2022-09-29T05:25:07.443 INFO:tasks.cephfs.filesystem.ceph_manager:PG 3.38 is not active+clean 2022-09-29T05:25:07.444 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '3.38', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2022-09-29T05:25:03.430113+0000', 'last_change': '2022-09-29T05:25:03.430113+0000', 'last_active': '2022-09-29T05:25:03.430113+0000', 'last_peered': '2022-09-29T05:25:03.430113+0000', 'last_clean': '2022-09-29T05:25:03.430113+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2022-09-29T05:25:03.430113+0000', 'last_undegraded': '2022-09-29T05:25:03.430113+0000', 'last_fullsized': '2022-09-29T05:25:03.430113+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'last_clean_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'log_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2022-09-29T05:25:07.444 INFO:tasks.cephfs.filesystem.ceph_manager:PG 3.37 is not active+clean 2022-09-29T05:25:07.445 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '3.37', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2022-09-29T05:25:03.430113+0000', 'last_change': '2022-09-29T05:25:03.430113+0000', 'last_active': '2022-09-29T05:25:03.430113+0000', 'last_peered': '2022-09-29T05:25:03.430113+0000', 'last_clean': '2022-09-29T05:25:03.430113+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2022-09-29T05:25:03.430113+0000', 'last_undegraded': '2022-09-29T05:25:03.430113+0000', 'last_fullsized': '2022-09-29T05:25:03.430113+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'last_clean_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'log_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2022-09-29T05:25:07.445 INFO:tasks.cephfs.filesystem.ceph_manager:PG 3.34 is not active+clean 2022-09-29T05:25:07.445 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '3.34', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2022-09-29T05:25:03.430113+0000', 'last_change': '2022-09-29T05:25:03.430113+0000', 'last_active': '2022-09-29T05:25:03.430113+0000', 'last_peered': '2022-09-29T05:25:03.430113+0000', 'last_clean': '2022-09-29T05:25:03.430113+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2022-09-29T05:25:03.430113+0000', 'last_undegraded': '2022-09-29T05:25:03.430113+0000', 'last_fullsized': '2022-09-29T05:25:03.430113+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'last_clean_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'log_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2022-09-29T05:25:07.446 INFO:tasks.cephfs.filesystem.ceph_manager:PG 3.33 is not active+clean 2022-09-29T05:25:07.446 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '3.33', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2022-09-29T05:25:03.430113+0000', 'last_change': '2022-09-29T05:25:03.430113+0000', 'last_active': '2022-09-29T05:25:03.430113+0000', 'last_peered': '2022-09-29T05:25:03.430113+0000', 'last_clean': '2022-09-29T05:25:03.430113+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2022-09-29T05:25:03.430113+0000', 'last_undegraded': '2022-09-29T05:25:03.430113+0000', 'last_fullsized': '2022-09-29T05:25:03.430113+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'last_clean_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'log_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2022-09-29T05:25:07.447 INFO:tasks.cephfs.filesystem.ceph_manager:PG 3.32 is not active+clean 2022-09-29T05:25:07.447 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '3.32', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2022-09-29T05:25:03.430113+0000', 'last_change': '2022-09-29T05:25:03.430113+0000', 'last_active': '2022-09-29T05:25:03.430113+0000', 'last_peered': '2022-09-29T05:25:03.430113+0000', 'last_clean': '2022-09-29T05:25:03.430113+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2022-09-29T05:25:03.430113+0000', 'last_undegraded': '2022-09-29T05:25:03.430113+0000', 'last_fullsized': '2022-09-29T05:25:03.430113+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'last_clean_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'log_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2022-09-29T05:25:07.447 INFO:tasks.cephfs.filesystem.ceph_manager:PG 3.2e is not active+clean 2022-09-29T05:25:07.448 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '3.2e', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2022-09-29T05:25:03.430113+0000', 'last_change': '2022-09-29T05:25:03.430113+0000', 'last_active': '2022-09-29T05:25:03.430113+0000', 'last_peered': '2022-09-29T05:25:03.430113+0000', 'last_clean': '2022-09-29T05:25:03.430113+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2022-09-29T05:25:03.430113+0000', 'last_undegraded': '2022-09-29T05:25:03.430113+0000', 'last_fullsized': '2022-09-29T05:25:03.430113+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'last_clean_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'log_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2022-09-29T05:25:07.448 INFO:tasks.cephfs.filesystem.ceph_manager:PG 3.2d is not active+clean 2022-09-29T05:25:07.448 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '3.2d', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2022-09-29T05:25:03.430113+0000', 'last_change': '2022-09-29T05:25:03.430113+0000', 'last_active': '2022-09-29T05:25:03.430113+0000', 'last_peered': '2022-09-29T05:25:03.430113+0000', 'last_clean': '2022-09-29T05:25:03.430113+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2022-09-29T05:25:03.430113+0000', 'last_undegraded': '2022-09-29T05:25:03.430113+0000', 'last_fullsized': '2022-09-29T05:25:03.430113+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'last_clean_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'log_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2022-09-29T05:25:07.449 INFO:tasks.cephfs.filesystem.ceph_manager:PG 2.1f is not active+clean 2022-09-29T05:25:07.449 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '2.1f', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2022-09-29T05:25:02.423315+0000', 'last_change': '2022-09-29T05:25:02.423315+0000', 'last_active': '2022-09-29T05:25:02.423315+0000', 'last_peered': '2022-09-29T05:25:02.423315+0000', 'last_clean': '2022-09-29T05:25:02.423315+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2022-09-29T05:25:02.423315+0000', 'last_undegraded': '2022-09-29T05:25:02.423315+0000', 'last_fullsized': '2022-09-29T05:25:02.423315+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2022-09-29T05:25:02.423315+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2022-09-29T05:25:02.423315+0000', 'last_clean_scrub_stamp': '2022-09-29T05:25:02.423315+0000', 'log_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2022-09-29T05:25:07.450 INFO:tasks.cephfs.filesystem.ceph_manager:PG 3.1e is not active+clean 2022-09-29T05:25:07.450 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '3.1e', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2022-09-29T05:25:03.430113+0000', 'last_change': '2022-09-29T05:25:03.430113+0000', 'last_active': '2022-09-29T05:25:03.430113+0000', 'last_peered': '2022-09-29T05:25:03.430113+0000', 'last_clean': '2022-09-29T05:25:03.430113+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2022-09-29T05:25:03.430113+0000', 'last_undegraded': '2022-09-29T05:25:03.430113+0000', 'last_fullsized': '2022-09-29T05:25:03.430113+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'last_clean_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'log_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2022-09-29T05:25:07.450 INFO:tasks.cephfs.filesystem.ceph_manager:PG 2.1e is not active+clean 2022-09-29T05:25:07.451 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '2.1e', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2022-09-29T05:25:02.423315+0000', 'last_change': '2022-09-29T05:25:02.423315+0000', 'last_active': '2022-09-29T05:25:02.423315+0000', 'last_peered': '2022-09-29T05:25:02.423315+0000', 'last_clean': '2022-09-29T05:25:02.423315+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2022-09-29T05:25:02.423315+0000', 'last_undegraded': '2022-09-29T05:25:02.423315+0000', 'last_fullsized': '2022-09-29T05:25:02.423315+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2022-09-29T05:25:02.423315+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2022-09-29T05:25:02.423315+0000', 'last_clean_scrub_stamp': '2022-09-29T05:25:02.423315+0000', 'log_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2022-09-29T05:25:07.451 INFO:tasks.cephfs.filesystem.ceph_manager:PG 3.1f is not active+clean 2022-09-29T05:25:07.452 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '3.1f', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2022-09-29T05:25:03.430113+0000', 'last_change': '2022-09-29T05:25:03.430113+0000', 'last_active': '2022-09-29T05:25:03.430113+0000', 'last_peered': '2022-09-29T05:25:03.430113+0000', 'last_clean': '2022-09-29T05:25:03.430113+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2022-09-29T05:25:03.430113+0000', 'last_undegraded': '2022-09-29T05:25:03.430113+0000', 'last_fullsized': '2022-09-29T05:25:03.430113+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'last_clean_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'log_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2022-09-29T05:25:07.452 INFO:tasks.cephfs.filesystem.ceph_manager:PG 2.1d is not active+clean 2022-09-29T05:25:07.452 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '2.1d', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'creating+peering', 'last_fresh': '2022-09-29T05:25:02.423315+0000', 'last_change': '2022-09-29T05:25:02.433891+0000', 'last_active': '2022-09-29T05:25:02.423315+0000', 'last_peered': '2022-09-29T05:25:02.423315+0000', 'last_clean': '2022-09-29T05:25:02.423315+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2022-09-29T05:25:02.423315+0000', 'last_undegraded': '2022-09-29T05:25:02.423315+0000', 'last_fullsized': '2022-09-29T05:25:02.423315+0000', 'mapping_epoch': 48, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 48, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2022-09-29T05:25:02.423315+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2022-09-29T05:25:02.423315+0000', 'last_clean_scrub_stamp': '2022-09-29T05:25:02.423315+0000', 'log_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [7, 6, 0], 'acting': [7, 6, 0], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [0, 6], 'up_primary': 7, 'acting_primary': 7, 'purged_snaps': []} 2022-09-29T05:25:07.453 INFO:tasks.cephfs.filesystem.ceph_manager:PG 2.1c is not active+clean 2022-09-29T05:25:07.453 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '2.1c', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'creating+peering', 'last_fresh': '2022-09-29T05:25:02.423315+0000', 'last_change': '2022-09-29T05:25:02.433333+0000', 'last_active': '2022-09-29T05:25:02.423315+0000', 'last_peered': '2022-09-29T05:25:02.423315+0000', 'last_clean': '2022-09-29T05:25:02.423315+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2022-09-29T05:25:02.423315+0000', 'last_undegraded': '2022-09-29T05:25:02.423315+0000', 'last_fullsized': '2022-09-29T05:25:02.423315+0000', 'mapping_epoch': 48, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 48, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2022-09-29T05:25:02.423315+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2022-09-29T05:25:02.423315+0000', 'last_clean_scrub_stamp': '2022-09-29T05:25:02.423315+0000', 'log_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [7, 5, 2], 'acting': [7, 5, 2], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [2, 5], 'up_primary': 7, 'acting_primary': 7, 'purged_snaps': []} 2022-09-29T05:25:07.453 INFO:tasks.cephfs.filesystem.ceph_manager:PG 2.1b is not active+clean 2022-09-29T05:25:07.454 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '2.1b', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2022-09-29T05:25:02.423315+0000', 'last_change': '2022-09-29T05:25:02.423315+0000', 'last_active': '2022-09-29T05:25:02.423315+0000', 'last_peered': '2022-09-29T05:25:02.423315+0000', 'last_clean': '2022-09-29T05:25:02.423315+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2022-09-29T05:25:02.423315+0000', 'last_undegraded': '2022-09-29T05:25:02.423315+0000', 'last_fullsized': '2022-09-29T05:25:02.423315+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2022-09-29T05:25:02.423315+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2022-09-29T05:25:02.423315+0000', 'last_clean_scrub_stamp': '2022-09-29T05:25:02.423315+0000', 'log_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2022-09-29T05:25:07.454 INFO:tasks.cephfs.filesystem.ceph_manager:PG 3.1a is not active+clean 2022-09-29T05:25:07.455 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '3.1a', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2022-09-29T05:25:03.430113+0000', 'last_change': '2022-09-29T05:25:03.430113+0000', 'last_active': '2022-09-29T05:25:03.430113+0000', 'last_peered': '2022-09-29T05:25:03.430113+0000', 'last_clean': '2022-09-29T05:25:03.430113+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2022-09-29T05:25:03.430113+0000', 'last_undegraded': '2022-09-29T05:25:03.430113+0000', 'last_fullsized': '2022-09-29T05:25:03.430113+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'last_clean_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'log_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2022-09-29T05:25:07.455 INFO:tasks.cephfs.filesystem.ceph_manager:PG 3.1b is not active+clean 2022-09-29T05:25:07.455 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '3.1b', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2022-09-29T05:25:03.430113+0000', 'last_change': '2022-09-29T05:25:03.430113+0000', 'last_active': '2022-09-29T05:25:03.430113+0000', 'last_peered': '2022-09-29T05:25:03.430113+0000', 'last_clean': '2022-09-29T05:25:03.430113+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2022-09-29T05:25:03.430113+0000', 'last_undegraded': '2022-09-29T05:25:03.430113+0000', 'last_fullsized': '2022-09-29T05:25:03.430113+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'last_clean_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'log_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2022-09-29T05:25:07.456 INFO:tasks.cephfs.filesystem.ceph_manager:PG 2.19 is not active+clean 2022-09-29T05:25:07.456 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '2.19', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2022-09-29T05:25:02.423315+0000', 'last_change': '2022-09-29T05:25:02.423315+0000', 'last_active': '2022-09-29T05:25:02.423315+0000', 'last_peered': '2022-09-29T05:25:02.423315+0000', 'last_clean': '2022-09-29T05:25:02.423315+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2022-09-29T05:25:02.423315+0000', 'last_undegraded': '2022-09-29T05:25:02.423315+0000', 'last_fullsized': '2022-09-29T05:25:02.423315+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2022-09-29T05:25:02.423315+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2022-09-29T05:25:02.423315+0000', 'last_clean_scrub_stamp': '2022-09-29T05:25:02.423315+0000', 'log_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2022-09-29T05:25:07.457 INFO:tasks.cephfs.filesystem.ceph_manager:PG 3.18 is not active+clean 2022-09-29T05:25:07.457 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '3.18', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2022-09-29T05:25:03.430113+0000', 'last_change': '2022-09-29T05:25:03.430113+0000', 'last_active': '2022-09-29T05:25:03.430113+0000', 'last_peered': '2022-09-29T05:25:03.430113+0000', 'last_clean': '2022-09-29T05:25:03.430113+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2022-09-29T05:25:03.430113+0000', 'last_undegraded': '2022-09-29T05:25:03.430113+0000', 'last_fullsized': '2022-09-29T05:25:03.430113+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'last_clean_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'log_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2022-09-29T05:25:07.457 INFO:tasks.cephfs.filesystem.ceph_manager:PG 3.17 is not active+clean 2022-09-29T05:25:07.458 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '3.17', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2022-09-29T05:25:03.430113+0000', 'last_change': '2022-09-29T05:25:03.430113+0000', 'last_active': '2022-09-29T05:25:03.430113+0000', 'last_peered': '2022-09-29T05:25:03.430113+0000', 'last_clean': '2022-09-29T05:25:03.430113+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2022-09-29T05:25:03.430113+0000', 'last_undegraded': '2022-09-29T05:25:03.430113+0000', 'last_fullsized': '2022-09-29T05:25:03.430113+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'last_clean_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'log_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2022-09-29T05:25:07.458 INFO:tasks.cephfs.filesystem.ceph_manager:PG 3.14 is not active+clean 2022-09-29T05:25:07.458 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '3.14', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2022-09-29T05:25:03.430113+0000', 'last_change': '2022-09-29T05:25:03.430113+0000', 'last_active': '2022-09-29T05:25:03.430113+0000', 'last_peered': '2022-09-29T05:25:03.430113+0000', 'last_clean': '2022-09-29T05:25:03.430113+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2022-09-29T05:25:03.430113+0000', 'last_undegraded': '2022-09-29T05:25:03.430113+0000', 'last_fullsized': '2022-09-29T05:25:03.430113+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'last_clean_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'log_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2022-09-29T05:25:07.459 INFO:tasks.cephfs.filesystem.ceph_manager:PG 3.15 is not active+clean 2022-09-29T05:25:07.459 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '3.15', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2022-09-29T05:25:03.430113+0000', 'last_change': '2022-09-29T05:25:03.430113+0000', 'last_active': '2022-09-29T05:25:03.430113+0000', 'last_peered': '2022-09-29T05:25:03.430113+0000', 'last_clean': '2022-09-29T05:25:03.430113+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2022-09-29T05:25:03.430113+0000', 'last_undegraded': '2022-09-29T05:25:03.430113+0000', 'last_fullsized': '2022-09-29T05:25:03.430113+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'last_clean_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'log_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2022-09-29T05:25:07.460 INFO:tasks.cephfs.filesystem.ceph_manager:PG 2.13 is not active+clean 2022-09-29T05:25:07.460 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '2.13', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2022-09-29T05:25:02.423315+0000', 'last_change': '2022-09-29T05:25:02.423315+0000', 'last_active': '2022-09-29T05:25:02.423315+0000', 'last_peered': '2022-09-29T05:25:02.423315+0000', 'last_clean': '2022-09-29T05:25:02.423315+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2022-09-29T05:25:02.423315+0000', 'last_undegraded': '2022-09-29T05:25:02.423315+0000', 'last_fullsized': '2022-09-29T05:25:02.423315+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2022-09-29T05:25:02.423315+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2022-09-29T05:25:02.423315+0000', 'last_clean_scrub_stamp': '2022-09-29T05:25:02.423315+0000', 'log_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2022-09-29T05:25:07.460 INFO:tasks.cephfs.filesystem.ceph_manager:PG 3.12 is not active+clean 2022-09-29T05:25:07.461 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '3.12', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2022-09-29T05:25:03.430113+0000', 'last_change': '2022-09-29T05:25:03.430113+0000', 'last_active': '2022-09-29T05:25:03.430113+0000', 'last_peered': '2022-09-29T05:25:03.430113+0000', 'last_clean': '2022-09-29T05:25:03.430113+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2022-09-29T05:25:03.430113+0000', 'last_undegraded': '2022-09-29T05:25:03.430113+0000', 'last_fullsized': '2022-09-29T05:25:03.430113+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'last_clean_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'log_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2022-09-29T05:25:07.461 INFO:tasks.cephfs.filesystem.ceph_manager:PG 3.13 is not active+clean 2022-09-29T05:25:07.461 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '3.13', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2022-09-29T05:25:03.430113+0000', 'last_change': '2022-09-29T05:25:03.430113+0000', 'last_active': '2022-09-29T05:25:03.430113+0000', 'last_peered': '2022-09-29T05:25:03.430113+0000', 'last_clean': '2022-09-29T05:25:03.430113+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2022-09-29T05:25:03.430113+0000', 'last_undegraded': '2022-09-29T05:25:03.430113+0000', 'last_fullsized': '2022-09-29T05:25:03.430113+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'last_clean_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'log_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2022-09-29T05:25:07.462 INFO:tasks.cephfs.filesystem.ceph_manager:PG 2.10 is not active+clean 2022-09-29T05:25:07.462 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '2.10', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'creating+peering', 'last_fresh': '2022-09-29T05:25:02.423315+0000', 'last_change': '2022-09-29T05:25:02.439448+0000', 'last_active': '2022-09-29T05:25:02.423315+0000', 'last_peered': '2022-09-29T05:25:02.423315+0000', 'last_clean': '2022-09-29T05:25:02.423315+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2022-09-29T05:25:02.423315+0000', 'last_undegraded': '2022-09-29T05:25:02.423315+0000', 'last_fullsized': '2022-09-29T05:25:02.423315+0000', 'mapping_epoch': 48, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 48, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2022-09-29T05:25:02.423315+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2022-09-29T05:25:02.423315+0000', 'last_clean_scrub_stamp': '2022-09-29T05:25:02.423315+0000', 'log_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1, 0], 'acting': [2, 1, 0], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [0, 1], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} 2022-09-29T05:25:07.463 INFO:tasks.cephfs.filesystem.ceph_manager:PG 3.11 is not active+clean 2022-09-29T05:25:07.463 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '3.11', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2022-09-29T05:25:03.430113+0000', 'last_change': '2022-09-29T05:25:03.430113+0000', 'last_active': '2022-09-29T05:25:03.430113+0000', 'last_peered': '2022-09-29T05:25:03.430113+0000', 'last_clean': '2022-09-29T05:25:03.430113+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2022-09-29T05:25:03.430113+0000', 'last_undegraded': '2022-09-29T05:25:03.430113+0000', 'last_fullsized': '2022-09-29T05:25:03.430113+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'last_clean_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'log_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2022-09-29T05:25:07.463 INFO:tasks.cephfs.filesystem.ceph_manager:PG 2.f is not active+clean 2022-09-29T05:25:07.464 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '2.f', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2022-09-29T05:25:02.423315+0000', 'last_change': '2022-09-29T05:25:02.423315+0000', 'last_active': '2022-09-29T05:25:02.423315+0000', 'last_peered': '2022-09-29T05:25:02.423315+0000', 'last_clean': '2022-09-29T05:25:02.423315+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2022-09-29T05:25:02.423315+0000', 'last_undegraded': '2022-09-29T05:25:02.423315+0000', 'last_fullsized': '2022-09-29T05:25:02.423315+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2022-09-29T05:25:02.423315+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2022-09-29T05:25:02.423315+0000', 'last_clean_scrub_stamp': '2022-09-29T05:25:02.423315+0000', 'log_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2022-09-29T05:25:07.464 INFO:tasks.cephfs.filesystem.ceph_manager:PG 3.e is not active+clean 2022-09-29T05:25:07.464 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '3.e', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2022-09-29T05:25:03.430113+0000', 'last_change': '2022-09-29T05:25:03.430113+0000', 'last_active': '2022-09-29T05:25:03.430113+0000', 'last_peered': '2022-09-29T05:25:03.430113+0000', 'last_clean': '2022-09-29T05:25:03.430113+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2022-09-29T05:25:03.430113+0000', 'last_undegraded': '2022-09-29T05:25:03.430113+0000', 'last_fullsized': '2022-09-29T05:25:03.430113+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'last_clean_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'log_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2022-09-29T05:25:07.465 INFO:tasks.cephfs.filesystem.ceph_manager:PG 3.3 is not active+clean 2022-09-29T05:25:07.465 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '3.3', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2022-09-29T05:25:03.430113+0000', 'last_change': '2022-09-29T05:25:03.430113+0000', 'last_active': '2022-09-29T05:25:03.430113+0000', 'last_peered': '2022-09-29T05:25:03.430113+0000', 'last_clean': '2022-09-29T05:25:03.430113+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2022-09-29T05:25:03.430113+0000', 'last_undegraded': '2022-09-29T05:25:03.430113+0000', 'last_fullsized': '2022-09-29T05:25:03.430113+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'last_clean_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'log_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2022-09-29T05:25:07.466 INFO:tasks.cephfs.filesystem.ceph_manager:PG 2.1 is not active+clean 2022-09-29T05:25:07.466 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '2.1', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'creating+peering', 'last_fresh': '2022-09-29T05:25:02.423315+0000', 'last_change': '2022-09-29T05:25:02.435837+0000', 'last_active': '2022-09-29T05:25:02.423315+0000', 'last_peered': '2022-09-29T05:25:02.423315+0000', 'last_clean': '2022-09-29T05:25:02.423315+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2022-09-29T05:25:02.423315+0000', 'last_undegraded': '2022-09-29T05:25:02.423315+0000', 'last_fullsized': '2022-09-29T05:25:02.423315+0000', 'mapping_epoch': 48, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 48, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2022-09-29T05:25:02.423315+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2022-09-29T05:25:02.423315+0000', 'last_clean_scrub_stamp': '2022-09-29T05:25:02.423315+0000', 'log_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 3, 0], 'acting': [2, 3, 0], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [0, 3], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} 2022-09-29T05:25:07.466 INFO:tasks.cephfs.filesystem.ceph_manager:PG 3.2 is not active+clean 2022-09-29T05:25:07.467 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '3.2', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2022-09-29T05:25:03.430113+0000', 'last_change': '2022-09-29T05:25:03.430113+0000', 'last_active': '2022-09-29T05:25:03.430113+0000', 'last_peered': '2022-09-29T05:25:03.430113+0000', 'last_clean': '2022-09-29T05:25:03.430113+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2022-09-29T05:25:03.430113+0000', 'last_undegraded': '2022-09-29T05:25:03.430113+0000', 'last_fullsized': '2022-09-29T05:25:03.430113+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'last_clean_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'log_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2022-09-29T05:25:07.467 INFO:tasks.cephfs.filesystem.ceph_manager:PG 3.1 is not active+clean 2022-09-29T05:25:07.468 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '3.1', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2022-09-29T05:25:03.430113+0000', 'last_change': '2022-09-29T05:25:03.430113+0000', 'last_active': '2022-09-29T05:25:03.430113+0000', 'last_peered': '2022-09-29T05:25:03.430113+0000', 'last_clean': '2022-09-29T05:25:03.430113+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2022-09-29T05:25:03.430113+0000', 'last_undegraded': '2022-09-29T05:25:03.430113+0000', 'last_fullsized': '2022-09-29T05:25:03.430113+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'last_clean_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'log_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2022-09-29T05:25:07.468 INFO:tasks.cephfs.filesystem.ceph_manager:PG 2.0 is not active+clean 2022-09-29T05:25:07.468 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '2.0', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'creating+peering', 'last_fresh': '2022-09-29T05:25:02.423315+0000', 'last_change': '2022-09-29T05:25:02.433346+0000', 'last_active': '2022-09-29T05:25:02.423315+0000', 'last_peered': '2022-09-29T05:25:02.423315+0000', 'last_clean': '2022-09-29T05:25:02.423315+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2022-09-29T05:25:02.423315+0000', 'last_undegraded': '2022-09-29T05:25:02.423315+0000', 'last_fullsized': '2022-09-29T05:25:02.423315+0000', 'mapping_epoch': 48, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 48, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2022-09-29T05:25:02.423315+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2022-09-29T05:25:02.423315+0000', 'last_clean_scrub_stamp': '2022-09-29T05:25:02.423315+0000', 'log_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [7, 1, 0], 'acting': [7, 1, 0], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [0, 1], 'up_primary': 7, 'acting_primary': 7, 'purged_snaps': []} 2022-09-29T05:25:07.469 INFO:tasks.cephfs.filesystem.ceph_manager:PG 2.5 is not active+clean 2022-09-29T05:25:07.469 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '2.5', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'creating+peering', 'last_fresh': '2022-09-29T05:25:02.423315+0000', 'last_change': '2022-09-29T05:25:02.433376+0000', 'last_active': '2022-09-29T05:25:02.423315+0000', 'last_peered': '2022-09-29T05:25:02.423315+0000', 'last_clean': '2022-09-29T05:25:02.423315+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2022-09-29T05:25:02.423315+0000', 'last_undegraded': '2022-09-29T05:25:02.423315+0000', 'last_fullsized': '2022-09-29T05:25:02.423315+0000', 'mapping_epoch': 48, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 48, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2022-09-29T05:25:02.423315+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2022-09-29T05:25:02.423315+0000', 'last_clean_scrub_stamp': '2022-09-29T05:25:02.423315+0000', 'log_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [7, 0, 4], 'acting': [7, 0, 4], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [0, 4], 'up_primary': 7, 'acting_primary': 7, 'purged_snaps': []} 2022-09-29T05:25:07.469 INFO:tasks.cephfs.filesystem.ceph_manager:PG 3.7 is not active+clean 2022-09-29T05:25:07.470 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '3.7', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2022-09-29T05:25:03.430113+0000', 'last_change': '2022-09-29T05:25:03.430113+0000', 'last_active': '2022-09-29T05:25:03.430113+0000', 'last_peered': '2022-09-29T05:25:03.430113+0000', 'last_clean': '2022-09-29T05:25:03.430113+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2022-09-29T05:25:03.430113+0000', 'last_undegraded': '2022-09-29T05:25:03.430113+0000', 'last_fullsized': '2022-09-29T05:25:03.430113+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'last_clean_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'log_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2022-09-29T05:25:07.470 INFO:tasks.cephfs.filesystem.ceph_manager:PG 3.6 is not active+clean 2022-09-29T05:25:07.471 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '3.6', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2022-09-29T05:25:03.430113+0000', 'last_change': '2022-09-29T05:25:03.430113+0000', 'last_active': '2022-09-29T05:25:03.430113+0000', 'last_peered': '2022-09-29T05:25:03.430113+0000', 'last_clean': '2022-09-29T05:25:03.430113+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2022-09-29T05:25:03.430113+0000', 'last_undegraded': '2022-09-29T05:25:03.430113+0000', 'last_fullsized': '2022-09-29T05:25:03.430113+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'last_clean_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'log_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2022-09-29T05:25:07.471 INFO:tasks.cephfs.filesystem.ceph_manager:PG 2.8 is not active+clean 2022-09-29T05:25:07.471 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '2.8', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'creating+peering', 'last_fresh': '2022-09-29T05:25:02.423315+0000', 'last_change': '2022-09-29T05:25:02.433909+0000', 'last_active': '2022-09-29T05:25:02.423315+0000', 'last_peered': '2022-09-29T05:25:02.423315+0000', 'last_clean': '2022-09-29T05:25:02.423315+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2022-09-29T05:25:02.423315+0000', 'last_undegraded': '2022-09-29T05:25:02.423315+0000', 'last_fullsized': '2022-09-29T05:25:02.423315+0000', 'mapping_epoch': 48, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 48, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2022-09-29T05:25:02.423315+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2022-09-29T05:25:02.423315+0000', 'last_clean_scrub_stamp': '2022-09-29T05:25:02.423315+0000', 'log_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [7, 0, 1], 'acting': [7, 0, 1], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [0, 1], 'up_primary': 7, 'acting_primary': 7, 'purged_snaps': []} 2022-09-29T05:25:07.472 INFO:tasks.cephfs.filesystem.ceph_manager:PG 3.9 is not active+clean 2022-09-29T05:25:07.472 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '3.9', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2022-09-29T05:25:03.430113+0000', 'last_change': '2022-09-29T05:25:03.430113+0000', 'last_active': '2022-09-29T05:25:03.430113+0000', 'last_peered': '2022-09-29T05:25:03.430113+0000', 'last_clean': '2022-09-29T05:25:03.430113+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2022-09-29T05:25:03.430113+0000', 'last_undegraded': '2022-09-29T05:25:03.430113+0000', 'last_fullsized': '2022-09-29T05:25:03.430113+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'last_clean_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'log_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2022-09-29T05:25:07.473 INFO:tasks.cephfs.filesystem.ceph_manager:PG 3.8 is not active+clean 2022-09-29T05:25:07.473 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '3.8', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2022-09-29T05:25:03.430113+0000', 'last_change': '2022-09-29T05:25:03.430113+0000', 'last_active': '2022-09-29T05:25:03.430113+0000', 'last_peered': '2022-09-29T05:25:03.430113+0000', 'last_clean': '2022-09-29T05:25:03.430113+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2022-09-29T05:25:03.430113+0000', 'last_undegraded': '2022-09-29T05:25:03.430113+0000', 'last_fullsized': '2022-09-29T05:25:03.430113+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'last_clean_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'log_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2022-09-29T05:25:07.473 INFO:tasks.cephfs.filesystem.ceph_manager:PG 3.b is not active+clean 2022-09-29T05:25:07.474 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '3.b', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2022-09-29T05:25:03.430113+0000', 'last_change': '2022-09-29T05:25:03.430113+0000', 'last_active': '2022-09-29T05:25:03.430113+0000', 'last_peered': '2022-09-29T05:25:03.430113+0000', 'last_clean': '2022-09-29T05:25:03.430113+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2022-09-29T05:25:03.430113+0000', 'last_undegraded': '2022-09-29T05:25:03.430113+0000', 'last_fullsized': '2022-09-29T05:25:03.430113+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'last_clean_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'log_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2022-09-29T05:25:07.474 INFO:tasks.cephfs.filesystem.ceph_manager:PG 2.b is not active+clean 2022-09-29T05:25:07.474 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '2.b', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'creating+peering', 'last_fresh': '2022-09-29T05:25:02.423315+0000', 'last_change': '2022-09-29T05:25:02.433388+0000', 'last_active': '2022-09-29T05:25:02.423315+0000', 'last_peered': '2022-09-29T05:25:02.423315+0000', 'last_clean': '2022-09-29T05:25:02.423315+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2022-09-29T05:25:02.423315+0000', 'last_undegraded': '2022-09-29T05:25:02.423315+0000', 'last_fullsized': '2022-09-29T05:25:02.423315+0000', 'mapping_epoch': 48, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 48, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2022-09-29T05:25:02.423315+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2022-09-29T05:25:02.423315+0000', 'last_clean_scrub_stamp': '2022-09-29T05:25:02.423315+0000', 'log_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [7, 4, 5], 'acting': [7, 4, 5], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [4, 5], 'up_primary': 7, 'acting_primary': 7, 'purged_snaps': []} 2022-09-29T05:25:07.475 INFO:tasks.cephfs.filesystem.ceph_manager:PG 2.c is not active+clean 2022-09-29T05:25:07.475 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '2.c', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'creating+peering', 'last_fresh': '2022-09-29T05:25:02.423315+0000', 'last_change': '2022-09-29T05:25:02.436663+0000', 'last_active': '2022-09-29T05:25:02.423315+0000', 'last_peered': '2022-09-29T05:25:02.423315+0000', 'last_clean': '2022-09-29T05:25:02.423315+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2022-09-29T05:25:02.423315+0000', 'last_undegraded': '2022-09-29T05:25:02.423315+0000', 'last_fullsized': '2022-09-29T05:25:02.423315+0000', 'mapping_epoch': 48, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 48, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2022-09-29T05:25:02.423315+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2022-09-29T05:25:02.423315+0000', 'last_clean_scrub_stamp': '2022-09-29T05:25:02.423315+0000', 'log_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 5, 0], 'acting': [2, 5, 0], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [0, 5], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} 2022-09-29T05:25:07.476 INFO:tasks.cephfs.filesystem.ceph_manager:PG 3.d is not active+clean 2022-09-29T05:25:07.476 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '3.d', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2022-09-29T05:25:03.430113+0000', 'last_change': '2022-09-29T05:25:03.430113+0000', 'last_active': '2022-09-29T05:25:03.430113+0000', 'last_peered': '2022-09-29T05:25:03.430113+0000', 'last_clean': '2022-09-29T05:25:03.430113+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2022-09-29T05:25:03.430113+0000', 'last_undegraded': '2022-09-29T05:25:03.430113+0000', 'last_fullsized': '2022-09-29T05:25:03.430113+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'last_clean_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'log_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2022-09-29T05:25:07.476 INFO:tasks.cephfs.filesystem.ceph_manager:PG 2.e is not active+clean 2022-09-29T05:25:07.477 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '2.e', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'creating+peering', 'last_fresh': '2022-09-29T05:25:02.423315+0000', 'last_change': '2022-09-29T05:25:02.439603+0000', 'last_active': '2022-09-29T05:25:02.423315+0000', 'last_peered': '2022-09-29T05:25:02.423315+0000', 'last_clean': '2022-09-29T05:25:02.423315+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2022-09-29T05:25:02.423315+0000', 'last_undegraded': '2022-09-29T05:25:02.423315+0000', 'last_fullsized': '2022-09-29T05:25:02.423315+0000', 'mapping_epoch': 48, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 48, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2022-09-29T05:25:02.423315+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2022-09-29T05:25:02.423315+0000', 'last_clean_scrub_stamp': '2022-09-29T05:25:02.423315+0000', 'log_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 3, 7], 'acting': [2, 3, 7], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [3, 7], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} 2022-09-29T05:25:07.477 INFO:tasks.cephfs.filesystem.ceph_manager:PG 3.f is not active+clean 2022-09-29T05:25:07.478 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '3.f', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2022-09-29T05:25:03.430113+0000', 'last_change': '2022-09-29T05:25:03.430113+0000', 'last_active': '2022-09-29T05:25:03.430113+0000', 'last_peered': '2022-09-29T05:25:03.430113+0000', 'last_clean': '2022-09-29T05:25:03.430113+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2022-09-29T05:25:03.430113+0000', 'last_undegraded': '2022-09-29T05:25:03.430113+0000', 'last_fullsized': '2022-09-29T05:25:03.430113+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'last_clean_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'log_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2022-09-29T05:25:07.478 INFO:tasks.cephfs.filesystem.ceph_manager:PG 3.21 is not active+clean 2022-09-29T05:25:07.478 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '3.21', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2022-09-29T05:25:03.430113+0000', 'last_change': '2022-09-29T05:25:03.430113+0000', 'last_active': '2022-09-29T05:25:03.430113+0000', 'last_peered': '2022-09-29T05:25:03.430113+0000', 'last_clean': '2022-09-29T05:25:03.430113+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2022-09-29T05:25:03.430113+0000', 'last_undegraded': '2022-09-29T05:25:03.430113+0000', 'last_fullsized': '2022-09-29T05:25:03.430113+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'last_clean_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'log_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2022-09-29T05:25:07.479 INFO:tasks.cephfs.filesystem.ceph_manager:PG 3.23 is not active+clean 2022-09-29T05:25:07.479 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '3.23', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2022-09-29T05:25:03.430113+0000', 'last_change': '2022-09-29T05:25:03.430113+0000', 'last_active': '2022-09-29T05:25:03.430113+0000', 'last_peered': '2022-09-29T05:25:03.430113+0000', 'last_clean': '2022-09-29T05:25:03.430113+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2022-09-29T05:25:03.430113+0000', 'last_undegraded': '2022-09-29T05:25:03.430113+0000', 'last_fullsized': '2022-09-29T05:25:03.430113+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'last_clean_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'log_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2022-09-29T05:25:07.480 INFO:tasks.cephfs.filesystem.ceph_manager:PG 3.24 is not active+clean 2022-09-29T05:25:07.480 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '3.24', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2022-09-29T05:25:03.430113+0000', 'last_change': '2022-09-29T05:25:03.430113+0000', 'last_active': '2022-09-29T05:25:03.430113+0000', 'last_peered': '2022-09-29T05:25:03.430113+0000', 'last_clean': '2022-09-29T05:25:03.430113+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2022-09-29T05:25:03.430113+0000', 'last_undegraded': '2022-09-29T05:25:03.430113+0000', 'last_fullsized': '2022-09-29T05:25:03.430113+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'last_clean_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'log_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2022-09-29T05:25:07.480 INFO:tasks.cephfs.filesystem.ceph_manager:PG 3.27 is not active+clean 2022-09-29T05:25:07.481 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '3.27', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2022-09-29T05:25:03.430113+0000', 'last_change': '2022-09-29T05:25:03.430113+0000', 'last_active': '2022-09-29T05:25:03.430113+0000', 'last_peered': '2022-09-29T05:25:03.430113+0000', 'last_clean': '2022-09-29T05:25:03.430113+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2022-09-29T05:25:03.430113+0000', 'last_undegraded': '2022-09-29T05:25:03.430113+0000', 'last_fullsized': '2022-09-29T05:25:03.430113+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'last_clean_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'log_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2022-09-29T05:25:07.481 INFO:tasks.cephfs.filesystem.ceph_manager:PG 3.29 is not active+clean 2022-09-29T05:25:07.482 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '3.29', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2022-09-29T05:25:03.430113+0000', 'last_change': '2022-09-29T05:25:03.430113+0000', 'last_active': '2022-09-29T05:25:03.430113+0000', 'last_peered': '2022-09-29T05:25:03.430113+0000', 'last_clean': '2022-09-29T05:25:03.430113+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2022-09-29T05:25:03.430113+0000', 'last_undegraded': '2022-09-29T05:25:03.430113+0000', 'last_fullsized': '2022-09-29T05:25:03.430113+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'last_clean_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'log_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2022-09-29T05:25:07.482 INFO:tasks.cephfs.filesystem.ceph_manager:PG 3.2b is not active+clean 2022-09-29T05:25:07.482 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '3.2b', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2022-09-29T05:25:03.430113+0000', 'last_change': '2022-09-29T05:25:03.430113+0000', 'last_active': '2022-09-29T05:25:03.430113+0000', 'last_peered': '2022-09-29T05:25:03.430113+0000', 'last_clean': '2022-09-29T05:25:03.430113+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2022-09-29T05:25:03.430113+0000', 'last_undegraded': '2022-09-29T05:25:03.430113+0000', 'last_fullsized': '2022-09-29T05:25:03.430113+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'last_clean_scrub_stamp': '2022-09-29T05:25:03.430113+0000', 'log_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2022-09-29T05:25:07.483 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json 2022-09-29T05:25:07.819 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:25:07.820 INFO:teuthology.orchestra.run.smithi083.stdout:{"fsid":"69111522-3fb6-11ed-8431-001a4aab830c","health":{"status":"HEALTH_OK","checks":{},"mutes":[]},"election_epoch":16,"quorum":[0,1,2],"quorum_names":["a","c","b"],"quorum_age":172,"monmap":{"epoch":3,"min_mon_release_name":"pacific","num_mons":3},"osdmap":{"epoch":51,"num_osds":8,"num_up_osds":8,"osd_up_since":1664429086,"num_in_osds":8,"osd_in_since":1664429071,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"unknown","count":45},{"state_name":"active+clean","count":42},{"state_name":"creating+peering","count":10}],"num_pgs":97,"num_pools":3,"num_objects":8,"data_bytes":1696,"bytes_used":44453888,"bytes_avail":767915831296,"bytes_total":767960285184,"unknown_pgs_ratio":0.46391752362251282,"inactive_pgs_ratio":0.10309278219938278,"write_bytes_sec":3868,"read_op_per_sec":0,"write_op_per_sec":11},"fsmap":{"epoch":6,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"b","status":"up:active","gid":24356}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","restful"],"services":{"dashboard":"https://172.21.15.83:8443/"}},"servicemap":{"epoch":10,"modified":"2022-09-29T05:24:57.590189+0000","services":{"mds":{"daemons":{"summary":"","a":{"start_epoch":0,"start_stamp":"0.000000","gid":0,"addr":"(unrecognized address family 0)/0","metadata":{},"task_status":{}}}},"osd":{"daemons":{"summary":"","5":{"start_epoch":0,"start_stamp":"0.000000","gid":0,"addr":"(unrecognized address family 0)/0","metadata":{},"task_status":{}},"6":{"start_epoch":0,"start_stamp":"0.000000","gid":0,"addr":"(unrecognized address family 0)/0","metadata":{},"task_status":{}},"7":{"start_epoch":0,"start_stamp":"0.000000","gid":0,"addr":"(unrecognized address family 0)/0","metadata":{},"task_status":{}}}}}},"progress_events":{}} 2022-09-29T05:25:07.836 INFO:tasks.cephfs.filesystem.ceph_manager:no progress seen, keeping timeout for now 2022-09-29T05:25:07.837 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2022-09-29T05:25:08.122 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:25:08.124 INFO:teuthology.orchestra.run.smithi083.stderr:dumped all 2022-09-29T05:25:08.132 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:07 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:06.736581+0000 mgr.y (mgr.14146) 162 : audit [DBG] from='client.24439 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2022-09-29T05:25:08.132 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:07 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:07.816962+0000 mon.a (mon.0) 563 : audit [DBG] from='client.? 172.21.15.83:0/3501954724' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2022-09-29T05:25:08.133 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:07 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:06.736581+0000 mgr.y (mgr.14146) 162 : audit [DBG] from='client.24439 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2022-09-29T05:25:08.133 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:07 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:07.816962+0000 mon.a (mon.0) 563 : audit [DBG] from='client.? 172.21.15.83:0/3501954724' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2022-09-29T05:25:08.134 INFO:teuthology.orchestra.run.smithi083.stdout:{"pg_ready":true,"pg_map":{"version":147,"stamp":"2022-09-29T05:25:07.591998+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2306,"num_objects":16,"num_object_clones":0,"num_object_copies":48,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":16,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":29,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":8,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":28,"ondisk_log_size":28,"up":285,"acting":285,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":118,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":44624,"kb_used_data":3856,"kb_used_omap":0,"kb_used_meta":40704,"kb_avail":749916592,"statfs":{"total":767960285184,"available":767914590208,"internally_reserved":0,"allocated":3948544,"data_stored":1570396,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":41680896},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":13,"apply_latency_ms":13,"commit_latency_ns":13000000,"apply_latency_ns":13000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":2146,"num_objects":13,"num_object_clones":0,"num_object_copies":39,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":13,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":24,"num_write_kb":11,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":7,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"2.794545"},"pg_stats":[{"pgid":"3.3f","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.831454+0000","last_change":"2022-09-29T05:25:03.829987+0000","last_active":"2022-09-29T05:25:03.831454+0000","last_peered":"2022-09-29T05:25:03.831454+0000","last_clean":"2022-09-29T05:25:03.831454+0000","last_became_active":"2022-09-29T05:25:03.829703+0000","last_became_peered":"2022-09-29T05:25:03.829703+0000","last_unstale":"2022-09-29T05:25:03.831454+0000","last_undegraded":"2022-09-29T05:25:03.831454+0000","last_fullsized":"2022-09-29T05:25:03.831454+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3e","version":"0'0","reported_seq":9,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.816168+0000","last_change":"2022-09-29T05:25:03.816168+0000","last_active":"2022-09-29T05:25:03.816168+0000","last_peered":"2022-09-29T05:25:03.816168+0000","last_clean":"2022-09-29T05:25:03.816168+0000","last_became_active":"2022-09-29T05:25:03.815888+0000","last_became_peered":"2022-09-29T05:25:03.815888+0000","last_unstale":"2022-09-29T05:25:03.816168+0000","last_undegraded":"2022-09-29T05:25:03.816168+0000","last_fullsized":"2022-09-29T05:25:03.816168+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3d","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.816176+0000","last_change":"2022-09-29T05:25:03.814221+0000","last_active":"2022-09-29T05:25:03.816176+0000","last_peered":"2022-09-29T05:25:03.816176+0000","last_clean":"2022-09-29T05:25:03.816176+0000","last_became_active":"2022-09-29T05:25:03.813927+0000","last_became_peered":"2022-09-29T05:25:03.813927+0000","last_unstale":"2022-09-29T05:25:03.816176+0000","last_undegraded":"2022-09-29T05:25:03.816176+0000","last_fullsized":"2022-09-29T05:25:03.816176+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3c","version":"0'0","reported_seq":9,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.819554+0000","last_change":"2022-09-29T05:25:03.819554+0000","last_active":"2022-09-29T05:25:03.819554+0000","last_peered":"2022-09-29T05:25:03.819554+0000","last_clean":"2022-09-29T05:25:03.819554+0000","last_became_active":"2022-09-29T05:25:03.819207+0000","last_became_peered":"2022-09-29T05:25:03.819207+0000","last_unstale":"2022-09-29T05:25:03.819554+0000","last_undegraded":"2022-09-29T05:25:03.819554+0000","last_fullsized":"2022-09-29T05:25:03.819554+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3b","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.831380+0000","last_change":"2022-09-29T05:25:03.829034+0000","last_active":"2022-09-29T05:25:03.831380+0000","last_peered":"2022-09-29T05:25:03.831380+0000","last_clean":"2022-09-29T05:25:03.831380+0000","last_became_active":"2022-09-29T05:25:03.828384+0000","last_became_peered":"2022-09-29T05:25:03.828384+0000","last_unstale":"2022-09-29T05:25:03.831380+0000","last_undegraded":"2022-09-29T05:25:03.831380+0000","last_fullsized":"2022-09-29T05:25:03.831380+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3a","version":"0'0","reported_seq":9,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.817871+0000","last_change":"2022-09-29T05:25:03.817871+0000","last_active":"2022-09-29T05:25:03.817871+0000","last_peered":"2022-09-29T05:25:03.817871+0000","last_clean":"2022-09-29T05:25:03.817871+0000","last_became_active":"2022-09-29T05:25:03.816800+0000","last_became_peered":"2022-09-29T05:25:03.816800+0000","last_unstale":"2022-09-29T05:25:03.817871+0000","last_undegraded":"2022-09-29T05:25:03.817871+0000","last_fullsized":"2022-09-29T05:25:03.817871+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.39","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.831254+0000","last_change":"2022-09-29T05:25:03.825279+0000","last_active":"2022-09-29T05:25:03.831254+0000","last_peered":"2022-09-29T05:25:03.831254+0000","last_clean":"2022-09-29T05:25:03.831254+0000","last_became_active":"2022-09-29T05:25:03.825002+0000","last_became_peered":"2022-09-29T05:25:03.825002+0000","last_unstale":"2022-09-29T05:25:03.831254+0000","last_undegraded":"2022-09-29T05:25:03.831254+0000","last_fullsized":"2022-09-29T05:25:03.831254+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.38","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.816333+0000","last_change":"2022-09-29T05:25:03.813803+0000","last_active":"2022-09-29T05:25:03.816333+0000","last_peered":"2022-09-29T05:25:03.816333+0000","last_clean":"2022-09-29T05:25:03.816333+0000","last_became_active":"2022-09-29T05:25:03.813488+0000","last_became_peered":"2022-09-29T05:25:03.813488+0000","last_unstale":"2022-09-29T05:25:03.816333+0000","last_undegraded":"2022-09-29T05:25:03.816333+0000","last_fullsized":"2022-09-29T05:25:03.816333+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.37","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.831437+0000","last_change":"2022-09-29T05:25:03.829880+0000","last_active":"2022-09-29T05:25:03.831437+0000","last_peered":"2022-09-29T05:25:03.831437+0000","last_clean":"2022-09-29T05:25:03.831437+0000","last_became_active":"2022-09-29T05:25:03.829318+0000","last_became_peered":"2022-09-29T05:25:03.829318+0000","last_unstale":"2022-09-29T05:25:03.831437+0000","last_undegraded":"2022-09-29T05:25:03.831437+0000","last_fullsized":"2022-09-29T05:25:03.831437+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,0],"acting":[7,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.36","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.825144+0000","last_change":"2022-09-29T05:25:03.823656+0000","last_active":"2022-09-29T05:25:03.825144+0000","last_peered":"2022-09-29T05:25:03.825144+0000","last_clean":"2022-09-29T05:25:03.825144+0000","last_became_active":"2022-09-29T05:25:03.823109+0000","last_became_peered":"2022-09-29T05:25:03.823109+0000","last_unstale":"2022-09-29T05:25:03.825144+0000","last_undegraded":"2022-09-29T05:25:03.825144+0000","last_fullsized":"2022-09-29T05:25:03.825144+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.35","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.825174+0000","last_change":"2022-09-29T05:25:03.824377+0000","last_active":"2022-09-29T05:25:03.825174+0000","last_peered":"2022-09-29T05:25:03.825174+0000","last_clean":"2022-09-29T05:25:03.825174+0000","last_became_active":"2022-09-29T05:25:03.823869+0000","last_became_peered":"2022-09-29T05:25:03.823869+0000","last_unstale":"2022-09-29T05:25:03.825174+0000","last_undegraded":"2022-09-29T05:25:03.825174+0000","last_fullsized":"2022-09-29T05:25:03.825174+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.34","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.821389+0000","last_change":"2022-09-29T05:25:03.812233+0000","last_active":"2022-09-29T05:25:03.821389+0000","last_peered":"2022-09-29T05:25:03.821389+0000","last_clean":"2022-09-29T05:25:03.821389+0000","last_became_active":"2022-09-29T05:25:03.811765+0000","last_became_peered":"2022-09-29T05:25:03.811765+0000","last_unstale":"2022-09-29T05:25:03.821389+0000","last_undegraded":"2022-09-29T05:25:03.821389+0000","last_fullsized":"2022-09-29T05:25:03.821389+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.33","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.816299+0000","last_change":"2022-09-29T05:25:03.813308+0000","last_active":"2022-09-29T05:25:03.816299+0000","last_peered":"2022-09-29T05:25:03.816299+0000","last_clean":"2022-09-29T05:25:03.816299+0000","last_became_active":"2022-09-29T05:25:03.813019+0000","last_became_peered":"2022-09-29T05:25:03.813019+0000","last_unstale":"2022-09-29T05:25:03.816299+0000","last_undegraded":"2022-09-29T05:25:03.816299+0000","last_fullsized":"2022-09-29T05:25:03.816299+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.32","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.823741+0000","last_change":"2022-09-29T05:25:03.823321+0000","last_active":"2022-09-29T05:25:03.823741+0000","last_peered":"2022-09-29T05:25:03.823741+0000","last_clean":"2022-09-29T05:25:03.823741+0000","last_became_active":"2022-09-29T05:25:03.822987+0000","last_became_peered":"2022-09-29T05:25:03.822987+0000","last_unstale":"2022-09-29T05:25:03.823741+0000","last_undegraded":"2022-09-29T05:25:03.823741+0000","last_fullsized":"2022-09-29T05:25:03.823741+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.31","version":"0'0","reported_seq":9,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.805590+0000","last_change":"2022-09-29T05:25:03.805590+0000","last_active":"2022-09-29T05:25:03.805590+0000","last_peered":"2022-09-29T05:25:03.805590+0000","last_clean":"2022-09-29T05:25:03.805590+0000","last_became_active":"2022-09-29T05:25:03.805310+0000","last_became_peered":"2022-09-29T05:25:03.805310+0000","last_unstale":"2022-09-29T05:25:03.805590+0000","last_undegraded":"2022-09-29T05:25:03.805590+0000","last_fullsized":"2022-09-29T05:25:03.805590+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.30","version":"0'0","reported_seq":9,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.815886+0000","last_change":"2022-09-29T05:25:03.815886+0000","last_active":"2022-09-29T05:25:03.815886+0000","last_peered":"2022-09-29T05:25:03.815886+0000","last_clean":"2022-09-29T05:25:03.815886+0000","last_became_active":"2022-09-29T05:25:03.815285+0000","last_became_peered":"2022-09-29T05:25:03.815285+0000","last_unstale":"2022-09-29T05:25:03.815886+0000","last_undegraded":"2022-09-29T05:25:03.815886+0000","last_fullsized":"2022-09-29T05:25:03.815886+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2f","version":"0'0","reported_seq":9,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.813784+0000","last_change":"2022-09-29T05:25:03.813784+0000","last_active":"2022-09-29T05:25:03.813784+0000","last_peered":"2022-09-29T05:25:03.813784+0000","last_clean":"2022-09-29T05:25:03.813784+0000","last_became_active":"2022-09-29T05:25:03.813420+0000","last_became_peered":"2022-09-29T05:25:03.813420+0000","last_unstale":"2022-09-29T05:25:03.813784+0000","last_undegraded":"2022-09-29T05:25:03.813784+0000","last_fullsized":"2022-09-29T05:25:03.813784+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2e","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.2d","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.831407+0000","last_change":"2022-09-29T05:25:03.828324+0000","last_active":"2022-09-29T05:25:03.831407+0000","last_peered":"2022-09-29T05:25:03.831407+0000","last_clean":"2022-09-29T05:25:03.831407+0000","last_became_active":"2022-09-29T05:25:03.827685+0000","last_became_peered":"2022-09-29T05:25:03.827685+0000","last_unstale":"2022-09-29T05:25:03.831407+0000","last_undegraded":"2022-09-29T05:25:03.831407+0000","last_fullsized":"2022-09-29T05:25:03.831407+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1f","version":"50'4","reported_seq":15,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.824599+0000","last_change":"2022-09-29T05:25:03.449078+0000","last_active":"2022-09-29T05:25:03.824599+0000","last_peered":"2022-09-29T05:25:03.824599+0000","last_clean":"2022-09-29T05:25:03.824599+0000","last_became_active":"2022-09-29T05:25:03.448793+0000","last_became_peered":"2022-09-29T05:25:03.448793+0000","last_unstale":"2022-09-29T05:25:03.824599+0000","last_undegraded":"2022-09-29T05:25:03.824599+0000","last_fullsized":"2022-09-29T05:25:03.824599+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":4,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.816314+0000","last_change":"2022-09-29T05:25:03.814538+0000","last_active":"2022-09-29T05:25:03.816314+0000","last_peered":"2022-09-29T05:25:03.816314+0000","last_clean":"2022-09-29T05:25:03.816314+0000","last_became_active":"2022-09-29T05:25:03.814264+0000","last_became_peered":"2022-09-29T05:25:03.814264+0000","last_unstale":"2022-09-29T05:25:03.816314+0000","last_undegraded":"2022-09-29T05:25:03.816314+0000","last_fullsized":"2022-09-29T05:25:03.816314+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1e","version":"50'1","reported_seq":12,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.820630+0000","last_change":"2022-09-29T05:25:03.459868+0000","last_active":"2022-09-29T05:25:03.820630+0000","last_peered":"2022-09-29T05:25:03.820630+0000","last_clean":"2022-09-29T05:25:03.820630+0000","last_became_active":"2022-09-29T05:25:03.459458+0000","last_became_peered":"2022-09-29T05:25:03.459458+0000","last_unstale":"2022-09-29T05:25:03.820630+0000","last_undegraded":"2022-09-29T05:25:03.820630+0000","last_fullsized":"2022-09-29T05:25:03.820630+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.822676+0000","last_change":"2022-09-29T05:25:03.822430+0000","last_active":"2022-09-29T05:25:03.822676+0000","last_peered":"2022-09-29T05:25:03.822676+0000","last_clean":"2022-09-29T05:25:03.822676+0000","last_became_active":"2022-09-29T05:25:03.821898+0000","last_became_peered":"2022-09-29T05:25:03.821898+0000","last_unstale":"2022-09-29T05:25:03.822676+0000","last_undegraded":"2022-09-29T05:25:03.822676+0000","last_fullsized":"2022-09-29T05:25:03.822676+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1d","version":"50'2","reported_seq":13,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.828716+0000","last_change":"2022-09-29T05:25:03.455815+0000","last_active":"2022-09-29T05:25:03.828716+0000","last_peered":"2022-09-29T05:25:03.828716+0000","last_clean":"2022-09-29T05:25:03.828716+0000","last_became_active":"2022-09-29T05:25:03.455218+0000","last_became_peered":"2022-09-29T05:25:03.455218+0000","last_unstale":"2022-09-29T05:25:03.828716+0000","last_undegraded":"2022-09-29T05:25:03.828716+0000","last_fullsized":"2022-09-29T05:25:03.828716+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"0'0","reported_seq":9,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.816581+0000","last_change":"2022-09-29T05:25:03.816581+0000","last_active":"2022-09-29T05:25:03.816581+0000","last_peered":"2022-09-29T05:25:03.816581+0000","last_clean":"2022-09-29T05:25:03.816581+0000","last_became_active":"2022-09-29T05:25:03.816206+0000","last_became_peered":"2022-09-29T05:25:03.816206+0000","last_unstale":"2022-09-29T05:25:03.816581+0000","last_undegraded":"2022-09-29T05:25:03.816581+0000","last_fullsized":"2022-09-29T05:25:03.816581+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1c","version":"50'2","reported_seq":13,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.826639+0000","last_change":"2022-09-29T05:25:03.460578+0000","last_active":"2022-09-29T05:25:03.826639+0000","last_peered":"2022-09-29T05:25:03.826639+0000","last_clean":"2022-09-29T05:25:03.826639+0000","last_became_active":"2022-09-29T05:25:03.460273+0000","last_became_peered":"2022-09-29T05:25:03.460273+0000","last_unstale":"2022-09-29T05:25:03.826639+0000","last_undegraded":"2022-09-29T05:25:03.826639+0000","last_fullsized":"2022-09-29T05:25:03.826639+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1d","version":"0'0","reported_seq":9,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.820523+0000","last_change":"2022-09-29T05:25:03.820523+0000","last_active":"2022-09-29T05:25:03.820523+0000","last_peered":"2022-09-29T05:25:03.820523+0000","last_clean":"2022-09-29T05:25:03.820523+0000","last_became_active":"2022-09-29T05:25:03.819546+0000","last_became_peered":"2022-09-29T05:25:03.819546+0000","last_unstale":"2022-09-29T05:25:03.820523+0000","last_undegraded":"2022-09-29T05:25:03.820523+0000","last_fullsized":"2022-09-29T05:25:03.820523+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1b","version":"50'1","reported_seq":12,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.824161+0000","last_change":"2022-09-29T05:25:03.455758+0000","last_active":"2022-09-29T05:25:03.824161+0000","last_peered":"2022-09-29T05:25:03.824161+0000","last_clean":"2022-09-29T05:25:03.824161+0000","last_became_active":"2022-09-29T05:25:03.455438+0000","last_became_peered":"2022-09-29T05:25:03.455438+0000","last_unstale":"2022-09-29T05:25:03.824161+0000","last_undegraded":"2022-09-29T05:25:03.824161+0000","last_fullsized":"2022-09-29T05:25:03.824161+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.820593+0000","last_change":"2022-09-29T05:25:03.817393+0000","last_active":"2022-09-29T05:25:03.820593+0000","last_peered":"2022-09-29T05:25:03.820593+0000","last_clean":"2022-09-29T05:25:03.820593+0000","last_became_active":"2022-09-29T05:25:03.817103+0000","last_became_peered":"2022-09-29T05:25:03.817103+0000","last_unstale":"2022-09-29T05:25:03.820593+0000","last_undegraded":"2022-09-29T05:25:03.820593+0000","last_fullsized":"2022-09-29T05:25:03.820593+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1a","version":"50'1","reported_seq":12,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.815499+0000","last_change":"2022-09-29T05:25:03.454747+0000","last_active":"2022-09-29T05:25:03.815499+0000","last_peered":"2022-09-29T05:25:03.815499+0000","last_clean":"2022-09-29T05:25:03.815499+0000","last_became_active":"2022-09-29T05:25:03.454395+0000","last_became_peered":"2022-09-29T05:25:03.454395+0000","last_unstale":"2022-09-29T05:25:03.815499+0000","last_undegraded":"2022-09-29T05:25:03.815499+0000","last_fullsized":"2022-09-29T05:25:03.815499+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1b","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.820424+0000","last_change":"2022-09-29T05:25:03.819253+0000","last_active":"2022-09-29T05:25:03.820424+0000","last_peered":"2022-09-29T05:25:03.820424+0000","last_clean":"2022-09-29T05:25:03.820424+0000","last_became_active":"2022-09-29T05:25:03.818951+0000","last_became_peered":"2022-09-29T05:25:03.818951+0000","last_unstale":"2022-09-29T05:25:03.820424+0000","last_undegraded":"2022-09-29T05:25:03.820424+0000","last_fullsized":"2022-09-29T05:25:03.820424+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.19","version":"0'0","reported_seq":11,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.794770+0000","last_change":"2022-09-29T05:25:03.455193+0000","last_active":"2022-09-29T05:25:03.794770+0000","last_peered":"2022-09-29T05:25:03.794770+0000","last_clean":"2022-09-29T05:25:03.794770+0000","last_became_active":"2022-09-29T05:25:03.454916+0000","last_became_peered":"2022-09-29T05:25:03.454916+0000","last_unstale":"2022-09-29T05:25:03.794770+0000","last_undegraded":"2022-09-29T05:25:03.794770+0000","last_fullsized":"2022-09-29T05:25:03.794770+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.809444+0000","last_change":"2022-09-29T05:25:03.806930+0000","last_active":"2022-09-29T05:25:03.809444+0000","last_peered":"2022-09-29T05:25:03.809444+0000","last_clean":"2022-09-29T05:25:03.809444+0000","last_became_active":"2022-09-29T05:25:03.806565+0000","last_became_peered":"2022-09-29T05:25:03.806565+0000","last_unstale":"2022-09-29T05:25:03.809444+0000","last_undegraded":"2022-09-29T05:25:03.809444+0000","last_fullsized":"2022-09-29T05:25:03.809444+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.18","version":"0'0","reported_seq":10,"reported_epoch":49,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.464050+0000","last_change":"2022-09-29T05:25:03.460675+0000","last_active":"2022-09-29T05:25:03.464050+0000","last_peered":"2022-09-29T05:25:03.464050+0000","last_clean":"2022-09-29T05:25:03.464050+0000","last_became_active":"2022-09-29T05:25:03.460351+0000","last_became_peered":"2022-09-29T05:25:03.460351+0000","last_unstale":"2022-09-29T05:25:03.464050+0000","last_undegraded":"2022-09-29T05:25:03.464050+0000","last_fullsized":"2022-09-29T05:25:03.464050+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":9,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.816170+0000","last_change":"2022-09-29T05:25:03.816170+0000","last_active":"2022-09-29T05:25:03.816170+0000","last_peered":"2022-09-29T05:25:03.816170+0000","last_clean":"2022-09-29T05:25:03.816170+0000","last_became_active":"2022-09-29T05:25:03.815838+0000","last_became_peered":"2022-09-29T05:25:03.815838+0000","last_unstale":"2022-09-29T05:25:03.816170+0000","last_undegraded":"2022-09-29T05:25:03.816170+0000","last_fullsized":"2022-09-29T05:25:03.816170+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.17","version":"0'0","reported_seq":11,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.805981+0000","last_change":"2022-09-29T05:25:03.461045+0000","last_active":"2022-09-29T05:25:03.805981+0000","last_peered":"2022-09-29T05:25:03.805981+0000","last_clean":"2022-09-29T05:25:03.805981+0000","last_became_active":"2022-09-29T05:25:03.460729+0000","last_became_peered":"2022-09-29T05:25:03.460729+0000","last_unstale":"2022-09-29T05:25:03.805981+0000","last_undegraded":"2022-09-29T05:25:03.805981+0000","last_fullsized":"2022-09-29T05:25:03.805981+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":9,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.818733+0000","last_change":"2022-09-29T05:25:03.818733+0000","last_active":"2022-09-29T05:25:03.818733+0000","last_peered":"2022-09-29T05:25:03.818733+0000","last_clean":"2022-09-29T05:25:03.818733+0000","last_became_active":"2022-09-29T05:25:03.816983+0000","last_became_peered":"2022-09-29T05:25:03.816983+0000","last_unstale":"2022-09-29T05:25:03.818733+0000","last_undegraded":"2022-09-29T05:25:03.818733+0000","last_fullsized":"2022-09-29T05:25:03.818733+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.16","version":"0'0","reported_seq":10,"reported_epoch":49,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.464237+0000","last_change":"2022-09-29T05:25:03.463477+0000","last_active":"2022-09-29T05:25:03.464237+0000","last_peered":"2022-09-29T05:25:03.464237+0000","last_clean":"2022-09-29T05:25:03.464237+0000","last_became_active":"2022-09-29T05:25:03.463051+0000","last_became_peered":"2022-09-29T05:25:03.463051+0000","last_unstale":"2022-09-29T05:25:03.464237+0000","last_undegraded":"2022-09-29T05:25:03.464237+0000","last_fullsized":"2022-09-29T05:25:03.464237+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.822495+0000","last_change":"2022-09-29T05:25:03.822200+0000","last_active":"2022-09-29T05:25:03.822495+0000","last_peered":"2022-09-29T05:25:03.822495+0000","last_clean":"2022-09-29T05:25:03.822495+0000","last_became_active":"2022-09-29T05:25:03.821488+0000","last_became_peered":"2022-09-29T05:25:03.821488+0000","last_unstale":"2022-09-29T05:25:03.822495+0000","last_undegraded":"2022-09-29T05:25:03.822495+0000","last_fullsized":"2022-09-29T05:25:03.822495+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.15","version":"50'3","reported_seq":13,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.811766+0000","last_change":"2022-09-29T05:25:03.457077+0000","last_active":"2022-09-29T05:25:03.811766+0000","last_peered":"2022-09-29T05:25:03.811766+0000","last_clean":"2022-09-29T05:25:03.811766+0000","last_became_active":"2022-09-29T05:25:03.456823+0000","last_became_peered":"2022-09-29T05:25:03.456823+0000","last_unstale":"2022-09-29T05:25:03.811766+0000","last_undegraded":"2022-09-29T05:25:03.811766+0000","last_fullsized":"2022-09-29T05:25:03.811766+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":3,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.825581+0000","last_change":"2022-09-29T05:25:03.822638+0000","last_active":"2022-09-29T05:25:03.825581+0000","last_peered":"2022-09-29T05:25:03.825581+0000","last_clean":"2022-09-29T05:25:03.825581+0000","last_became_active":"2022-09-29T05:25:03.822297+0000","last_became_peered":"2022-09-29T05:25:03.822297+0000","last_unstale":"2022-09-29T05:25:03.825581+0000","last_undegraded":"2022-09-29T05:25:03.825581+0000","last_fullsized":"2022-09-29T05:25:03.825581+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.14","version":"50'6","reported_seq":17,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.827688+0000","last_change":"2022-09-29T05:25:03.461139+0000","last_active":"2022-09-29T05:25:03.827688+0000","last_peered":"2022-09-29T05:25:03.827688+0000","last_clean":"2022-09-29T05:25:03.827688+0000","last_became_active":"2022-09-29T05:25:03.460829+0000","last_became_peered":"2022-09-29T05:25:03.460829+0000","last_unstale":"2022-09-29T05:25:03.827688+0000","last_undegraded":"2022-09-29T05:25:03.827688+0000","last_fullsized":"2022-09-29T05:25:03.827688+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":6,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":1536,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.831388+0000","last_change":"2022-09-29T05:25:03.828186+0000","last_active":"2022-09-29T05:25:03.831388+0000","last_peered":"2022-09-29T05:25:03.831388+0000","last_clean":"2022-09-29T05:25:03.831388+0000","last_became_active":"2022-09-29T05:25:03.826167+0000","last_became_peered":"2022-09-29T05:25:03.826167+0000","last_unstale":"2022-09-29T05:25:03.831388+0000","last_undegraded":"2022-09-29T05:25:03.831388+0000","last_fullsized":"2022-09-29T05:25:03.831388+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.13","version":"50'2","reported_seq":13,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.817560+0000","last_change":"2022-09-29T05:25:03.451661+0000","last_active":"2022-09-29T05:25:03.817560+0000","last_peered":"2022-09-29T05:25:03.817560+0000","last_clean":"2022-09-29T05:25:03.817560+0000","last_became_active":"2022-09-29T05:25:03.451389+0000","last_became_peered":"2022-09-29T05:25:03.451389+0000","last_unstale":"2022-09-29T05:25:03.817560+0000","last_undegraded":"2022-09-29T05:25:03.817560+0000","last_fullsized":"2022-09-29T05:25:03.817560+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.825490+0000","last_change":"2022-09-29T05:25:03.824608+0000","last_active":"2022-09-29T05:25:03.825490+0000","last_peered":"2022-09-29T05:25:03.825490+0000","last_clean":"2022-09-29T05:25:03.825490+0000","last_became_active":"2022-09-29T05:25:03.824335+0000","last_became_peered":"2022-09-29T05:25:03.824335+0000","last_unstale":"2022-09-29T05:25:03.825490+0000","last_undegraded":"2022-09-29T05:25:03.825490+0000","last_fullsized":"2022-09-29T05:25:03.825490+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.12","version":"0'0","reported_seq":10,"reported_epoch":49,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.464191+0000","last_change":"2022-09-29T05:25:03.462145+0000","last_active":"2022-09-29T05:25:03.464191+0000","last_peered":"2022-09-29T05:25:03.464191+0000","last_clean":"2022-09-29T05:25:03.464191+0000","last_became_active":"2022-09-29T05:25:03.461864+0000","last_became_peered":"2022-09-29T05:25:03.461864+0000","last_unstale":"2022-09-29T05:25:03.464191+0000","last_undegraded":"2022-09-29T05:25:03.464191+0000","last_fullsized":"2022-09-29T05:25:03.464191+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.831361+0000","last_change":"2022-09-29T05:25:03.828734+0000","last_active":"2022-09-29T05:25:03.831361+0000","last_peered":"2022-09-29T05:25:03.831361+0000","last_clean":"2022-09-29T05:25:03.831361+0000","last_became_active":"2022-09-29T05:25:03.827929+0000","last_became_peered":"2022-09-29T05:25:03.827929+0000","last_unstale":"2022-09-29T05:25:03.831361+0000","last_undegraded":"2022-09-29T05:25:03.831361+0000","last_fullsized":"2022-09-29T05:25:03.831361+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.11","version":"0'0","reported_seq":11,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.805505+0000","last_change":"2022-09-29T05:25:03.454654+0000","last_active":"2022-09-29T05:25:03.805505+0000","last_peered":"2022-09-29T05:25:03.805505+0000","last_clean":"2022-09-29T05:25:03.805505+0000","last_became_active":"2022-09-29T05:25:03.454319+0000","last_became_peered":"2022-09-29T05:25:03.454319+0000","last_unstale":"2022-09-29T05:25:03.805505+0000","last_undegraded":"2022-09-29T05:25:03.805505+0000","last_fullsized":"2022-09-29T05:25:03.805505+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.813586+0000","last_change":"2022-09-29T05:25:03.811425+0000","last_active":"2022-09-29T05:25:03.813586+0000","last_peered":"2022-09-29T05:25:03.813586+0000","last_clean":"2022-09-29T05:25:03.813586+0000","last_became_active":"2022-09-29T05:25:03.811120+0000","last_became_peered":"2022-09-29T05:25:03.811120+0000","last_unstale":"2022-09-29T05:25:03.813586+0000","last_undegraded":"2022-09-29T05:25:03.813586+0000","last_fullsized":"2022-09-29T05:25:03.813586+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.10","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2022-09-29T05:25:02.423315+0000","last_change":"2022-09-29T05:25:02.439448+0000","last_active":"2022-09-29T05:25:02.423315+0000","last_peered":"2022-09-29T05:25:02.423315+0000","last_clean":"2022-09-29T05:25:02.423315+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:02.423315+0000","last_undegraded":"2022-09-29T05:25:02.423315+0000","last_fullsized":"2022-09-29T05:25:02.423315+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,1],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.831276+0000","last_change":"2022-09-29T05:25:03.826279+0000","last_active":"2022-09-29T05:25:03.831276+0000","last_peered":"2022-09-29T05:25:03.831276+0000","last_clean":"2022-09-29T05:25:03.831276+0000","last_became_active":"2022-09-29T05:25:03.825969+0000","last_became_peered":"2022-09-29T05:25:03.825969+0000","last_unstale":"2022-09-29T05:25:03.831276+0000","last_undegraded":"2022-09-29T05:25:03.831276+0000","last_fullsized":"2022-09-29T05:25:03.831276+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.f","version":"50'1","reported_seq":12,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.824523+0000","last_change":"2022-09-29T05:25:03.452230+0000","last_active":"2022-09-29T05:25:03.824523+0000","last_peered":"2022-09-29T05:25:03.824523+0000","last_clean":"2022-09-29T05:25:03.824523+0000","last_became_active":"2022-09-29T05:25:03.451747+0000","last_became_peered":"2022-09-29T05:25:03.451747+0000","last_unstale":"2022-09-29T05:25:03.824523+0000","last_undegraded":"2022-09-29T05:25:03.824523+0000","last_fullsized":"2022-09-29T05:25:03.824523+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.831346+0000","last_change":"2022-09-29T05:25:03.827453+0000","last_active":"2022-09-29T05:25:03.831346+0000","last_peered":"2022-09-29T05:25:03.831346+0000","last_clean":"2022-09-29T05:25:03.831346+0000","last_became_active":"2022-09-29T05:25:03.827151+0000","last_became_peered":"2022-09-29T05:25:03.827151+0000","last_unstale":"2022-09-29T05:25:03.831346+0000","last_undegraded":"2022-09-29T05:25:03.831346+0000","last_fullsized":"2022-09-29T05:25:03.831346+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":10,"reported_epoch":49,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.454814+0000","last_change":"2022-09-29T05:25:03.454507+0000","last_active":"2022-09-29T05:25:03.454814+0000","last_peered":"2022-09-29T05:25:03.454814+0000","last_clean":"2022-09-29T05:25:03.454814+0000","last_became_active":"2022-09-29T05:25:03.454195+0000","last_became_peered":"2022-09-29T05:25:03.454195+0000","last_unstale":"2022-09-29T05:25:03.454814+0000","last_undegraded":"2022-09-29T05:25:03.454814+0000","last_fullsized":"2022-09-29T05:25:03.454814+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":9,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.819862+0000","last_change":"2022-09-29T05:25:03.819862+0000","last_active":"2022-09-29T05:25:03.819862+0000","last_peered":"2022-09-29T05:25:03.819862+0000","last_clean":"2022-09-29T05:25:03.819862+0000","last_became_active":"2022-09-29T05:25:03.818117+0000","last_became_peered":"2022-09-29T05:25:03.818117+0000","last_unstale":"2022-09-29T05:25:03.819862+0000","last_undegraded":"2022-09-29T05:25:03.819862+0000","last_fullsized":"2022-09-29T05:25:03.819862+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":10,"reported_epoch":49,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.463942+0000","last_change":"2022-09-29T05:25:03.459646+0000","last_active":"2022-09-29T05:25:03.463942+0000","last_peered":"2022-09-29T05:25:03.463942+0000","last_clean":"2022-09-29T05:25:03.463942+0000","last_became_active":"2022-09-29T05:25:03.459320+0000","last_became_peered":"2022-09-29T05:25:03.459320+0000","last_unstale":"2022-09-29T05:25:03.463942+0000","last_undegraded":"2022-09-29T05:25:03.463942+0000","last_fullsized":"2022-09-29T05:25:03.463942+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.815521+0000","last_change":"2022-09-29T05:25:03.810358+0000","last_active":"2022-09-29T05:25:03.815521+0000","last_peered":"2022-09-29T05:25:03.815521+0000","last_clean":"2022-09-29T05:25:03.815521+0000","last_became_active":"2022-09-29T05:25:03.809959+0000","last_became_peered":"2022-09-29T05:25:03.809959+0000","last_unstale":"2022-09-29T05:25:03.815521+0000","last_undegraded":"2022-09-29T05:25:03.815521+0000","last_fullsized":"2022-09-29T05:25:03.815521+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":9,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.812390+0000","last_change":"2022-09-29T05:25:03.812390+0000","last_active":"2022-09-29T05:25:03.812390+0000","last_peered":"2022-09-29T05:25:03.812390+0000","last_clean":"2022-09-29T05:25:03.812390+0000","last_became_active":"2022-09-29T05:25:03.811925+0000","last_became_peered":"2022-09-29T05:25:03.811925+0000","last_unstale":"2022-09-29T05:25:03.812390+0000","last_undegraded":"2022-09-29T05:25:03.812390+0000","last_fullsized":"2022-09-29T05:25:03.812390+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2022-09-29T05:25:02.423315+0000","last_change":"2022-09-29T05:25:02.435837+0000","last_active":"2022-09-29T05:25:02.423315+0000","last_peered":"2022-09-29T05:25:02.423315+0000","last_clean":"2022-09-29T05:25:02.423315+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:02.423315+0000","last_undegraded":"2022-09-29T05:25:02.423315+0000","last_fullsized":"2022-09-29T05:25:02.423315+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,3],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.816328+0000","last_change":"2022-09-29T05:25:03.813170+0000","last_active":"2022-09-29T05:25:03.816328+0000","last_peered":"2022-09-29T05:25:03.816328+0000","last_clean":"2022-09-29T05:25:03.816328+0000","last_became_active":"2022-09-29T05:25:03.812861+0000","last_became_peered":"2022-09-29T05:25:03.812861+0000","last_unstale":"2022-09-29T05:25:03.816328+0000","last_undegraded":"2022-09-29T05:25:03.816328+0000","last_fullsized":"2022-09-29T05:25:03.816328+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":10,"reported_epoch":49,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.464203+0000","last_change":"2022-09-29T05:25:03.462271+0000","last_active":"2022-09-29T05:25:03.464203+0000","last_peered":"2022-09-29T05:25:03.464203+0000","last_clean":"2022-09-29T05:25:03.464203+0000","last_became_active":"2022-09-29T05:25:03.461959+0000","last_became_peered":"2022-09-29T05:25:03.461959+0000","last_unstale":"2022-09-29T05:25:03.464203+0000","last_undegraded":"2022-09-29T05:25:03.464203+0000","last_fullsized":"2022-09-29T05:25:03.464203+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":14,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.803350+0000","last_change":"2022-09-29T05:24:48.628243+0000","last_active":"2022-09-29T05:25:03.803350+0000","last_peered":"2022-09-29T05:25:03.803350+0000","last_clean":"2022-09-29T05:25:03.803350+0000","last_became_active":"2022-09-29T05:24:48.627801+0000","last_became_peered":"2022-09-29T05:24:48.627801+0000","last_unstale":"2022-09-29T05:25:03.803350+0000","last_undegraded":"2022-09-29T05:25:03.803350+0000","last_fullsized":"2022-09-29T05:25:03.803350+0000","mapping_epoch":46,"log_start":"0'0","ondisk_log_start":"0'0","created":21,"last_epoch_clean":47,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:23:35.544281+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:23:35.544281+0000","last_clean_scrub_stamp":"2022-09-29T05:23:35.544281+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.812499+0000","last_change":"2022-09-29T05:25:03.812201+0000","last_active":"2022-09-29T05:25:03.812499+0000","last_peered":"2022-09-29T05:25:03.812499+0000","last_clean":"2022-09-29T05:25:03.812499+0000","last_became_active":"2022-09-29T05:25:03.811903+0000","last_became_peered":"2022-09-29T05:25:03.811903+0000","last_unstale":"2022-09-29T05:25:03.812499+0000","last_undegraded":"2022-09-29T05:25:03.812499+0000","last_fullsized":"2022-09-29T05:25:03.812499+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.804063+0000","last_change":"2022-09-29T05:25:03.451876+0000","last_active":"2022-09-29T05:25:03.804063+0000","last_peered":"2022-09-29T05:25:03.804063+0000","last_clean":"2022-09-29T05:25:03.804063+0000","last_became_active":"2022-09-29T05:25:03.451371+0000","last_became_peered":"2022-09-29T05:25:03.451371+0000","last_unstale":"2022-09-29T05:25:03.804063+0000","last_undegraded":"2022-09-29T05:25:03.804063+0000","last_fullsized":"2022-09-29T05:25:03.804063+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.816054+0000","last_change":"2022-09-29T05:25:03.453060+0000","last_active":"2022-09-29T05:25:03.816054+0000","last_peered":"2022-09-29T05:25:03.816054+0000","last_clean":"2022-09-29T05:25:03.816054+0000","last_became_active":"2022-09-29T05:25:03.452758+0000","last_became_peered":"2022-09-29T05:25:03.452758+0000","last_unstale":"2022-09-29T05:25:03.816054+0000","last_undegraded":"2022-09-29T05:25:03.816054+0000","last_fullsized":"2022-09-29T05:25:03.816054+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":9,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.803195+0000","last_change":"2022-09-29T05:25:03.803195+0000","last_active":"2022-09-29T05:25:03.803195+0000","last_peered":"2022-09-29T05:25:03.803195+0000","last_clean":"2022-09-29T05:25:03.803195+0000","last_became_active":"2022-09-29T05:25:03.802909+0000","last_became_peered":"2022-09-29T05:25:03.802909+0000","last_unstale":"2022-09-29T05:25:03.803195+0000","last_undegraded":"2022-09-29T05:25:03.803195+0000","last_fullsized":"2022-09-29T05:25:03.803195+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":10,"reported_epoch":49,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.457008+0000","last_change":"2022-09-29T05:25:03.456735+0000","last_active":"2022-09-29T05:25:03.457008+0000","last_peered":"2022-09-29T05:25:03.457008+0000","last_clean":"2022-09-29T05:25:03.457008+0000","last_became_active":"2022-09-29T05:25:03.456427+0000","last_became_peered":"2022-09-29T05:25:03.456427+0000","last_unstale":"2022-09-29T05:25:03.457008+0000","last_undegraded":"2022-09-29T05:25:03.457008+0000","last_fullsized":"2022-09-29T05:25:03.457008+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.820572+0000","last_change":"2022-09-29T05:25:03.819725+0000","last_active":"2022-09-29T05:25:03.820572+0000","last_peered":"2022-09-29T05:25:03.820572+0000","last_clean":"2022-09-29T05:25:03.820572+0000","last_became_active":"2022-09-29T05:25:03.819455+0000","last_became_peered":"2022-09-29T05:25:03.819455+0000","last_unstale":"2022-09-29T05:25:03.820572+0000","last_undegraded":"2022-09-29T05:25:03.820572+0000","last_fullsized":"2022-09-29T05:25:03.820572+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"50'2","reported_seq":13,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.824910+0000","last_change":"2022-09-29T05:25:03.462138+0000","last_active":"2022-09-29T05:25:03.824910+0000","last_peered":"2022-09-29T05:25:03.824910+0000","last_clean":"2022-09-29T05:25:03.824910+0000","last_became_active":"2022-09-29T05:25:03.461809+0000","last_became_peered":"2022-09-29T05:25:03.461809+0000","last_unstale":"2022-09-29T05:25:03.824910+0000","last_undegraded":"2022-09-29T05:25:03.824910+0000","last_fullsized":"2022-09-29T05:25:03.824910+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.816152+0000","last_change":"2022-09-29T05:25:03.813902+0000","last_active":"2022-09-29T05:25:03.816152+0000","last_peered":"2022-09-29T05:25:03.816152+0000","last_clean":"2022-09-29T05:25:03.816152+0000","last_became_active":"2022-09-29T05:25:03.813617+0000","last_became_peered":"2022-09-29T05:25:03.813617+0000","last_unstale":"2022-09-29T05:25:03.816152+0000","last_undegraded":"2022-09-29T05:25:03.816152+0000","last_fullsized":"2022-09-29T05:25:03.816152+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.8","version":"0'0","reported_seq":11,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.805654+0000","last_change":"2022-09-29T05:25:03.451677+0000","last_active":"2022-09-29T05:25:03.805654+0000","last_peered":"2022-09-29T05:25:03.805654+0000","last_clean":"2022-09-29T05:25:03.805654+0000","last_became_active":"2022-09-29T05:25:03.450961+0000","last_became_peered":"2022-09-29T05:25:03.450961+0000","last_unstale":"2022-09-29T05:25:03.805654+0000","last_undegraded":"2022-09-29T05:25:03.805654+0000","last_fullsized":"2022-09-29T05:25:03.805654+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.825550+0000","last_change":"2022-09-29T05:25:03.822108+0000","last_active":"2022-09-29T05:25:03.825550+0000","last_peered":"2022-09-29T05:25:03.825550+0000","last_clean":"2022-09-29T05:25:03.825550+0000","last_became_active":"2022-09-29T05:25:03.821784+0000","last_became_peered":"2022-09-29T05:25:03.821784+0000","last_unstale":"2022-09-29T05:25:03.825550+0000","last_undegraded":"2022-09-29T05:25:03.825550+0000","last_fullsized":"2022-09-29T05:25:03.825550+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.9","version":"0'0","reported_seq":10,"reported_epoch":49,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.452964+0000","last_change":"2022-09-29T05:25:03.452512+0000","last_active":"2022-09-29T05:25:03.452964+0000","last_peered":"2022-09-29T05:25:03.452964+0000","last_clean":"2022-09-29T05:25:03.452964+0000","last_became_active":"2022-09-29T05:25:03.452222+0000","last_became_peered":"2022-09-29T05:25:03.452222+0000","last_unstale":"2022-09-29T05:25:03.452964+0000","last_undegraded":"2022-09-29T05:25:03.452964+0000","last_fullsized":"2022-09-29T05:25:03.452964+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.820040+0000","last_change":"2022-09-29T05:25:03.819591+0000","last_active":"2022-09-29T05:25:03.820040+0000","last_peered":"2022-09-29T05:25:03.820040+0000","last_clean":"2022-09-29T05:25:03.820040+0000","last_became_active":"2022-09-29T05:25:03.819231+0000","last_became_peered":"2022-09-29T05:25:03.819231+0000","last_unstale":"2022-09-29T05:25:03.820040+0000","last_undegraded":"2022-09-29T05:25:03.820040+0000","last_fullsized":"2022-09-29T05:25:03.820040+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.a","version":"0'0","reported_seq":10,"reported_epoch":49,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.452852+0000","last_change":"2022-09-29T05:25:03.451868+0000","last_active":"2022-09-29T05:25:03.452852+0000","last_peered":"2022-09-29T05:25:03.452852+0000","last_clean":"2022-09-29T05:25:03.452852+0000","last_became_active":"2022-09-29T05:25:03.451576+0000","last_became_peered":"2022-09-29T05:25:03.451576+0000","last_unstale":"2022-09-29T05:25:03.452852+0000","last_undegraded":"2022-09-29T05:25:03.452852+0000","last_fullsized":"2022-09-29T05:25:03.452852+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.810083+0000","last_change":"2022-09-29T05:25:03.808836+0000","last_active":"2022-09-29T05:25:03.810083+0000","last_peered":"2022-09-29T05:25:03.810083+0000","last_clean":"2022-09-29T05:25:03.810083+0000","last_became_active":"2022-09-29T05:25:03.808557+0000","last_became_peered":"2022-09-29T05:25:03.808557+0000","last_unstale":"2022-09-29T05:25:03.810083+0000","last_undegraded":"2022-09-29T05:25:03.810083+0000","last_fullsized":"2022-09-29T05:25:03.810083+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.b","version":"50'2","reported_seq":13,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.829344+0000","last_change":"2022-09-29T05:25:03.460369+0000","last_active":"2022-09-29T05:25:03.829344+0000","last_peered":"2022-09-29T05:25:03.829344+0000","last_clean":"2022-09-29T05:25:03.829344+0000","last_became_active":"2022-09-29T05:25:03.459796+0000","last_became_peered":"2022-09-29T05:25:03.459796+0000","last_unstale":"2022-09-29T05:25:03.829344+0000","last_undegraded":"2022-09-29T05:25:03.829344+0000","last_fullsized":"2022-09-29T05:25:03.829344+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.824054+0000","last_change":"2022-09-29T05:25:03.818102+0000","last_active":"2022-09-29T05:25:03.824054+0000","last_peered":"2022-09-29T05:25:03.824054+0000","last_clean":"2022-09-29T05:25:03.824054+0000","last_became_active":"2022-09-29T05:25:03.817772+0000","last_became_peered":"2022-09-29T05:25:03.817772+0000","last_unstale":"2022-09-29T05:25:03.824054+0000","last_undegraded":"2022-09-29T05:25:03.824054+0000","last_fullsized":"2022-09-29T05:25:03.824054+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.c","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2022-09-29T05:25:02.423315+0000","last_change":"2022-09-29T05:25:02.436663+0000","last_active":"2022-09-29T05:25:02.423315+0000","last_peered":"2022-09-29T05:25:02.423315+0000","last_clean":"2022-09-29T05:25:02.423315+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:02.423315+0000","last_undegraded":"2022-09-29T05:25:02.423315+0000","last_fullsized":"2022-09-29T05:25:02.423315+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,5],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.831425+0000","last_change":"2022-09-29T05:25:03.828523+0000","last_active":"2022-09-29T05:25:03.831425+0000","last_peered":"2022-09-29T05:25:03.831425+0000","last_clean":"2022-09-29T05:25:03.831425+0000","last_became_active":"2022-09-29T05:25:03.827990+0000","last_became_peered":"2022-09-29T05:25:03.827990+0000","last_unstale":"2022-09-29T05:25:03.831425+0000","last_undegraded":"2022-09-29T05:25:03.831425+0000","last_fullsized":"2022-09-29T05:25:03.831425+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.d","version":"50'1","reported_seq":11,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.798141+0000","last_change":"2022-09-29T05:25:03.452648+0000","last_active":"2022-09-29T05:25:03.798141+0000","last_peered":"2022-09-29T05:25:03.798141+0000","last_clean":"2022-09-29T05:25:03.798141+0000","last_became_active":"2022-09-29T05:25:03.452242+0000","last_became_peered":"2022-09-29T05:25:03.452242+0000","last_unstale":"2022-09-29T05:25:03.798141+0000","last_undegraded":"2022-09-29T05:25:03.798141+0000","last_fullsized":"2022-09-29T05:25:03.798141+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":9,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.809640+0000","last_change":"2022-09-29T05:25:03.809640+0000","last_active":"2022-09-29T05:25:03.809640+0000","last_peered":"2022-09-29T05:25:03.809640+0000","last_clean":"2022-09-29T05:25:03.809640+0000","last_became_active":"2022-09-29T05:25:03.809298+0000","last_became_peered":"2022-09-29T05:25:03.809298+0000","last_unstale":"2022-09-29T05:25:03.809640+0000","last_undegraded":"2022-09-29T05:25:03.809640+0000","last_fullsized":"2022-09-29T05:25:03.809640+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.e","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2022-09-29T05:25:02.423315+0000","last_change":"2022-09-29T05:25:02.439603+0000","last_active":"2022-09-29T05:25:02.423315+0000","last_peered":"2022-09-29T05:25:02.423315+0000","last_clean":"2022-09-29T05:25:02.423315+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:02.423315+0000","last_undegraded":"2022-09-29T05:25:02.423315+0000","last_fullsized":"2022-09-29T05:25:02.423315+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[3,7],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.831419+0000","last_change":"2022-09-29T05:25:03.829664+0000","last_active":"2022-09-29T05:25:03.831419+0000","last_peered":"2022-09-29T05:25:03.831419+0000","last_clean":"2022-09-29T05:25:03.831419+0000","last_became_active":"2022-09-29T05:25:03.828931+0000","last_became_peered":"2022-09-29T05:25:03.828931+0000","last_unstale":"2022-09-29T05:25:03.831419+0000","last_undegraded":"2022-09-29T05:25:03.831419+0000","last_fullsized":"2022-09-29T05:25:03.831419+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.20","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.813607+0000","last_change":"2022-09-29T05:25:03.811486+0000","last_active":"2022-09-29T05:25:03.813607+0000","last_peered":"2022-09-29T05:25:03.813607+0000","last_clean":"2022-09-29T05:25:03.813607+0000","last_became_active":"2022-09-29T05:25:03.811222+0000","last_became_peered":"2022-09-29T05:25:03.811222+0000","last_unstale":"2022-09-29T05:25:03.813607+0000","last_undegraded":"2022-09-29T05:25:03.813607+0000","last_fullsized":"2022-09-29T05:25:03.813607+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.21","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2022-09-29T05:25:03.430113+0000","last_change":"2022-09-29T05:25:03.430113+0000","last_active":"2022-09-29T05:25:03.430113+0000","last_peered":"2022-09-29T05:25:03.430113+0000","last_clean":"2022-09-29T05:25:03.430113+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2022-09-29T05:25:03.430113+0000","last_undegraded":"2022-09-29T05:25:03.430113+0000","last_fullsized":"2022-09-29T05:25:03.430113+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.22","version":"0'0","reported_seq":9,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.817645+0000","last_change":"2022-09-29T05:25:03.817645+0000","last_active":"2022-09-29T05:25:03.817645+0000","last_peered":"2022-09-29T05:25:03.817645+0000","last_clean":"2022-09-29T05:25:03.817645+0000","last_became_active":"2022-09-29T05:25:03.816740+0000","last_became_peered":"2022-09-29T05:25:03.816740+0000","last_unstale":"2022-09-29T05:25:03.817645+0000","last_undegraded":"2022-09-29T05:25:03.817645+0000","last_fullsized":"2022-09-29T05:25:03.817645+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.23","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.811445+0000","last_change":"2022-09-29T05:25:03.810436+0000","last_active":"2022-09-29T05:25:03.811445+0000","last_peered":"2022-09-29T05:25:03.811445+0000","last_clean":"2022-09-29T05:25:03.811445+0000","last_became_active":"2022-09-29T05:25:03.810178+0000","last_became_peered":"2022-09-29T05:25:03.810178+0000","last_unstale":"2022-09-29T05:25:03.811445+0000","last_undegraded":"2022-09-29T05:25:03.811445+0000","last_fullsized":"2022-09-29T05:25:03.811445+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.24","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.810303+0000","last_change":"2022-09-29T05:25:03.809114+0000","last_active":"2022-09-29T05:25:03.810303+0000","last_peered":"2022-09-29T05:25:03.810303+0000","last_clean":"2022-09-29T05:25:03.810303+0000","last_became_active":"2022-09-29T05:25:03.808686+0000","last_became_peered":"2022-09-29T05:25:03.808686+0000","last_unstale":"2022-09-29T05:25:03.810303+0000","last_undegraded":"2022-09-29T05:25:03.810303+0000","last_fullsized":"2022-09-29T05:25:03.810303+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.25","version":"0'0","reported_seq":9,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.817066+0000","last_change":"2022-09-29T05:25:03.817066+0000","last_active":"2022-09-29T05:25:03.817066+0000","last_peered":"2022-09-29T05:25:03.817066+0000","last_clean":"2022-09-29T05:25:03.817066+0000","last_became_active":"2022-09-29T05:25:03.816747+0000","last_became_peered":"2022-09-29T05:25:03.816747+0000","last_unstale":"2022-09-29T05:25:03.817066+0000","last_undegraded":"2022-09-29T05:25:03.817066+0000","last_fullsized":"2022-09-29T05:25:03.817066+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.26","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.824138+0000","last_change":"2022-09-29T05:25:03.823457+0000","last_active":"2022-09-29T05:25:03.824138+0000","last_peered":"2022-09-29T05:25:03.824138+0000","last_clean":"2022-09-29T05:25:03.824138+0000","last_became_active":"2022-09-29T05:25:03.823149+0000","last_became_peered":"2022-09-29T05:25:03.823149+0000","last_unstale":"2022-09-29T05:25:03.824138+0000","last_undegraded":"2022-09-29T05:25:03.824138+0000","last_fullsized":"2022-09-29T05:25:03.824138+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,4],"acting":[6,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.27","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.810175+0000","last_change":"2022-09-29T05:25:03.808766+0000","last_active":"2022-09-29T05:25:03.810175+0000","last_peered":"2022-09-29T05:25:03.810175+0000","last_clean":"2022-09-29T05:25:03.810175+0000","last_became_active":"2022-09-29T05:25:03.808474+0000","last_became_peered":"2022-09-29T05:25:03.808474+0000","last_unstale":"2022-09-29T05:25:03.810175+0000","last_undegraded":"2022-09-29T05:25:03.810175+0000","last_fullsized":"2022-09-29T05:25:03.810175+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.28","version":"0'0","reported_seq":9,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.815620+0000","last_change":"2022-09-29T05:25:03.815620+0000","last_active":"2022-09-29T05:25:03.815620+0000","last_peered":"2022-09-29T05:25:03.815620+0000","last_clean":"2022-09-29T05:25:03.815620+0000","last_became_active":"2022-09-29T05:25:03.814336+0000","last_became_peered":"2022-09-29T05:25:03.814336+0000","last_unstale":"2022-09-29T05:25:03.815620+0000","last_undegraded":"2022-09-29T05:25:03.815620+0000","last_fullsized":"2022-09-29T05:25:03.815620+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.29","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.810113+0000","last_change":"2022-09-29T05:25:03.808727+0000","last_active":"2022-09-29T05:25:03.810113+0000","last_peered":"2022-09-29T05:25:03.810113+0000","last_clean":"2022-09-29T05:25:03.810113+0000","last_became_active":"2022-09-29T05:25:03.808440+0000","last_became_peered":"2022-09-29T05:25:03.808440+0000","last_unstale":"2022-09-29T05:25:03.810113+0000","last_undegraded":"2022-09-29T05:25:03.810113+0000","last_fullsized":"2022-09-29T05:25:03.810113+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.2a","version":"0'0","reported_seq":9,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.819098+0000","last_change":"2022-09-29T05:25:03.819098+0000","last_active":"2022-09-29T05:25:03.819098+0000","last_peered":"2022-09-29T05:25:03.819098+0000","last_clean":"2022-09-29T05:25:03.819098+0000","last_became_active":"2022-09-29T05:25:03.818690+0000","last_became_peered":"2022-09-29T05:25:03.818690+0000","last_unstale":"2022-09-29T05:25:03.819098+0000","last_undegraded":"2022-09-29T05:25:03.819098+0000","last_fullsized":"2022-09-29T05:25:03.819098+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2b","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.825526+0000","last_change":"2022-09-29T05:25:03.824844+0000","last_active":"2022-09-29T05:25:03.825526+0000","last_peered":"2022-09-29T05:25:03.825526+0000","last_clean":"2022-09-29T05:25:03.825526+0000","last_became_active":"2022-09-29T05:25:03.824457+0000","last_became_peered":"2022-09-29T05:25:03.824457+0000","last_unstale":"2022-09-29T05:25:03.825526+0000","last_undegraded":"2022-09-29T05:25:03.825526+0000","last_fullsized":"2022-09-29T05:25:03.825526+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2c","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.815650+0000","last_change":"2022-09-29T05:25:03.812644+0000","last_active":"2022-09-29T05:25:03.815650+0000","last_peered":"2022-09-29T05:25:03.815650+0000","last_clean":"2022-09-29T05:25:03.815650+0000","last_became_active":"2022-09-29T05:25:03.812339+0000","last_became_peered":"2022-09-29T05:25:03.812339+0000","last_unstale":"2022-09-29T05:25:03.815650+0000","last_undegraded":"2022-09-29T05:25:03.815650+0000","last_fullsized":"2022-09-29T05:25:03.815650+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":3,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":186,"acting":186,"num_store_stats":7},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":7},{"poolid":2,"num_pg":32,"stat_sum":{"num_bytes":2306,"num_objects":16,"num_object_clones":0,"num_object_copies":48,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":16,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":29,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":8,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":6918,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":28,"ondisk_log_size":28,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":45,"seq":193273528327,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5488,"kb_used_data":488,"kb_used_omap":0,"kb_used_meta":4992,"kb_avail":93739664,"statfs":{"total":95995035648,"available":95989415936,"internally_reserved":0,"allocated":499712,"data_stored":201962,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5111808},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45100000000000001}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.42899999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58099999999999996}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.40999999999999998}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.53800000000000003}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.52500000000000002}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54900000000000004}]}]},{"osd":6,"up_from":40,"seq":171798691849,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5480,"kb_used_data":480,"kb_used_omap":0,"kb_used_meta":4992,"kb_avail":93739672,"statfs":{"total":95995035648,"available":95989424128,"internally_reserved":0,"allocated":491520,"data_stored":196260,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5111808},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.38200000000000001}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45100000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.439}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.36199999999999999}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.433}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.46200000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.42399999999999999}]}]},{"osd":1,"up_from":13,"seq":55834574874,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5736,"kb_used_data":480,"kb_used_omap":0,"kb_used_meta":5248,"kb_avail":93739416,"statfs":{"total":95995035648,"available":95989161984,"internally_reserved":0,"allocated":491520,"data_stored":195199,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5373952},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Sep 29 05:24:04 2022","interfaces":[{"interface":"back","average":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"last":0.35799999999999998},{"interface":"front","average":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"last":0.27800000000000002}]},{"osd":2,"last update":"Thu Sep 29 05:24:22 2022","interfaces":[{"interface":"back","average":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"last":0.32000000000000001},{"interface":"front","average":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.374}]},{"osd":3,"last update":"Thu Sep 29 05:24:43 2022","interfaces":[{"interface":"back","average":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"last":0.34699999999999998},{"interface":"front","average":{"1min":0.442,"5min":0.442,"15min":0.442},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"last":0.30199999999999999}]},{"osd":4,"last update":"Thu Sep 29 05:25:00 2022","interfaces":[{"interface":"back","average":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":1.1619999999999999,"5min":1.1619999999999999,"15min":1.1619999999999999},"last":0.67700000000000005},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":1.1799999999999999,"5min":1.1799999999999999,"15min":1.1799999999999999},"last":0.66800000000000004}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56200000000000006}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.69799999999999995}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.61599999999999999}]}]},{"osd":0,"up_from":8,"seq":34359738398,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5820,"kb_used_data":500,"kb_used_omap":0,"kb_used_meta":5312,"kb_avail":93739332,"statfs":{"total":95995035648,"available":95989075968,"internally_reserved":0,"allocated":512000,"data_stored":202401,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5439488},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Sep 29 05:24:05 2022","interfaces":[{"interface":"back","average":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"min":{"1min":0.17699999999999999,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.503},{"interface":"front","average":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.371}]},{"osd":2,"last update":"Thu Sep 29 05:24:29 2022","interfaces":[{"interface":"back","average":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"last":0.35899999999999999},{"interface":"front","average":{"1min":0.377,"5min":0.377,"15min":0.377},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"last":0.47499999999999998}]},{"osd":3,"last update":"Thu Sep 29 05:24:41 2022","interfaces":[{"interface":"back","average":{"1min":0.437,"5min":0.437,"15min":0.437},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.434,"5min":0.434,"15min":0.434},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"last":0.39700000000000002}]},{"osd":4,"last update":"Thu Sep 29 05:24:57 2022","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.72899999999999998},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.72599999999999998}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.68700000000000006}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.66500000000000004}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.65700000000000003}]}]},{"osd":2,"up_from":18,"seq":77309411350,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5448,"kb_used_data":448,"kb_used_omap":0,"kb_used_meta":4992,"kb_avail":93739704,"statfs":{"total":95995035648,"available":95989456896,"internally_reserved":0,"allocated":458752,"data_stored":181265,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5111808},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Sep 29 05:24:26 2022","interfaces":[{"interface":"back","average":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"last":0.318},{"interface":"front","average":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":0.72099999999999997,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.28000000000000003}]},{"osd":1,"last update":"Thu Sep 29 05:24:26 2022","interfaces":[{"interface":"back","average":{"1min":0.38,"5min":0.38,"15min":0.38},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"last":0.33300000000000002},{"interface":"front","average":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"last":0.35599999999999998}]},{"osd":3,"last update":"Thu Sep 29 05:24:41 2022","interfaces":[{"interface":"back","average":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"last":0.34899999999999998},{"interface":"front","average":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.71799999999999997,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.38600000000000001}]},{"osd":4,"last update":"Thu Sep 29 05:24:56 2022","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.55700000000000005}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.59499999999999997}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.64200000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.61899999999999999}]}]},{"osd":3,"up_from":26,"seq":111669149715,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5632,"kb_used_data":504,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739520,"statfs":{"total":95995035648,"available":95989268480,"internally_reserved":0,"allocated":516096,"data_stored":203341,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Sep 29 05:24:44 2022","interfaces":[{"interface":"back","average":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"last":0.28199999999999997},{"interface":"front","average":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"last":0.45600000000000002}]},{"osd":1,"last update":"Thu Sep 29 05:24:44 2022","interfaces":[{"interface":"back","average":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"last":0.253},{"interface":"front","average":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"last":0.29699999999999999}]},{"osd":2,"last update":"Thu Sep 29 05:24:44 2022","interfaces":[{"interface":"back","average":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"last":0.34399999999999997},{"interface":"front","average":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"last":0.318}]},{"osd":4,"last update":"Thu Sep 29 05:25:00 2022","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.84099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.41199999999999998},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.49299999999999999}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.42199999999999999}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54700000000000004}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.504}]}]},{"osd":4,"up_from":30,"seq":128849018895,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5628,"kb_used_data":500,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739524,"statfs":{"total":95995035648,"available":95989272576,"internally_reserved":0,"allocated":512000,"data_stored":202381,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Sep 29 05:24:59 2022","interfaces":[{"interface":"back","average":{"1min":0.47799999999999998,"5min":0.47799999999999998,"15min":0.47799999999999998},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.71999999999999997,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.65000000000000002},{"interface":"front","average":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.67500000000000004,"5min":0.67500000000000004,"15min":0.67500000000000004},"last":0.59999999999999998}]},{"osd":1,"last update":"Thu Sep 29 05:24:59 2022","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.55400000000000005},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.749,"5min":0.749,"15min":0.749},"last":0.498}]},{"osd":2,"last update":"Thu Sep 29 05:24:59 2022","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.71499999999999997},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.51700000000000002,"15min":0.51700000000000002},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.57799999999999996}]},{"osd":3,"last update":"Thu Sep 29 05:24:59 2022","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.63500000000000001},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.70399999999999996}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.76400000000000001}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.748}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.73499999999999999}]}]},{"osd":5,"up_from":35,"seq":150323855373,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5392,"kb_used_data":456,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739760,"statfs":{"total":95995035648,"available":95989514240,"internally_reserved":0,"allocated":466944,"data_stored":187587,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.55900000000000005}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.50700000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.49299999999999999}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45500000000000002}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.41199999999999998}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.46300000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.61199999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":700,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":1640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1626,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":1536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2022-09-29T05:25:08.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:07 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:06.736581+0000 mgr.y (mgr.14146) 162 : audit [DBG] from='client.24439 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2022-09-29T05:25:08.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:07 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:07.816962+0000 mon.a (mon.0) 563 : audit [DBG] from='client.? 172.21.15.83:0/3501954724' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2022-09-29T05:25:09.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:08 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:07.072904+0000 mgr.y (mgr.14146) 163 : audit [DBG] from='client.14535 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2022-09-29T05:25:09.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:08 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:07.411274+0000 mgr.y (mgr.14146) 164 : audit [DBG] from='client.24451 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2022-09-29T05:25:09.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:08 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:25:07.592360+0000 mgr.y (mgr.14146) 165 : cluster [DBG] pgmap v147: 97 pgs: 4 creating+peering, 2 unknown, 91 active+clean; 2.3 KiB data, 44 MiB used, 715 GiB / 715 GiB avail; 3.9 KiB/s wr, 8 op/s 2022-09-29T05:25:09.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:08 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:07.072904+0000 mgr.y (mgr.14146) 163 : audit [DBG] from='client.14535 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2022-09-29T05:25:09.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:08 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:07.411274+0000 mgr.y (mgr.14146) 164 : audit [DBG] from='client.24451 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2022-09-29T05:25:09.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:08 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:25:07.592360+0000 mgr.y (mgr.14146) 165 : cluster [DBG] pgmap v147: 97 pgs: 4 creating+peering, 2 unknown, 91 active+clean; 2.3 KiB data, 44 MiB used, 715 GiB / 715 GiB avail; 3.9 KiB/s wr, 8 op/s 2022-09-29T05:25:09.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:08 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:07.072904+0000 mgr.y (mgr.14146) 163 : audit [DBG] from='client.14535 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2022-09-29T05:25:09.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:08 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:07.411274+0000 mgr.y (mgr.14146) 164 : audit [DBG] from='client.24451 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2022-09-29T05:25:09.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:08 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:25:07.592360+0000 mgr.y (mgr.14146) 165 : cluster [DBG] pgmap v147: 97 pgs: 4 creating+peering, 2 unknown, 91 active+clean; 2.3 KiB data, 44 MiB used, 715 GiB / 715 GiB avail; 3.9 KiB/s wr, 8 op/s 2022-09-29T05:25:10.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:09 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:08.109880+0000 mgr.y (mgr.14146) 166 : audit [DBG] from='client.24463 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2022-09-29T05:25:10.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:09 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:08.109880+0000 mgr.y (mgr.14146) 166 : audit [DBG] from='client.24463 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2022-09-29T05:25:10.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:09 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:08.109880+0000 mgr.y (mgr.14146) 166 : audit [DBG] from='client.24463 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2022-09-29T05:25:11.141 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2022-09-29T05:25:11.171 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:25:09.592833+0000 mgr.y (mgr.14146) 167 : cluster [DBG] pgmap v148: 97 pgs: 97 active+clean; 2.3 KiB data, 44 MiB used, 715 GiB / 715 GiB avail; 2.3 KiB/s wr, 8 op/s 2022-09-29T05:25:11.172 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:25:09.592833+0000 mgr.y (mgr.14146) 167 : cluster [DBG] pgmap v148: 97 pgs: 97 active+clean; 2.3 KiB data, 44 MiB used, 715 GiB / 715 GiB avail; 2.3 KiB/s wr, 8 op/s 2022-09-29T05:25:11.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:10 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:25:09.592833+0000 mgr.y (mgr.14146) 167 : cluster [DBG] pgmap v148: 97 pgs: 97 active+clean; 2.3 KiB data, 44 MiB used, 715 GiB / 715 GiB avail; 2.3 KiB/s wr, 8 op/s 2022-09-29T05:25:11.439 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:25:11.441 INFO:teuthology.orchestra.run.smithi083.stderr:dumped all 2022-09-29T05:25:11.450 INFO:teuthology.orchestra.run.smithi083.stdout:{"pg_ready":true,"pg_map":{"version":148,"stamp":"2022-09-29T05:25:09.592580+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2306,"num_objects":21,"num_object_clones":0,"num_object_copies":63,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":21,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":44,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":13,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":38,"ondisk_log_size":38,"up":291,"acting":291,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":181,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":44956,"kb_used_data":3932,"kb_used_omap":0,"kb_used_meta":40960,"kb_avail":749916260,"statfs":{"total":767960285184,"available":767914250240,"internally_reserved":0,"allocated":4026368,"data_stored":1606257,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":41943040},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":2146,"num_objects":18,"num_object_clones":0,"num_object_copies":54,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":18,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":39,"num_write_kb":11,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":12,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.795128"},"pg_stats":[{"pgid":"3.3f","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.831454+0000","last_change":"2022-09-29T05:25:03.829987+0000","last_active":"2022-09-29T05:25:03.831454+0000","last_peered":"2022-09-29T05:25:03.831454+0000","last_clean":"2022-09-29T05:25:03.831454+0000","last_became_active":"2022-09-29T05:25:03.829703+0000","last_became_peered":"2022-09-29T05:25:03.829703+0000","last_unstale":"2022-09-29T05:25:03.831454+0000","last_undegraded":"2022-09-29T05:25:03.831454+0000","last_fullsized":"2022-09-29T05:25:03.831454+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3e","version":"0'0","reported_seq":9,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.816168+0000","last_change":"2022-09-29T05:25:03.816168+0000","last_active":"2022-09-29T05:25:03.816168+0000","last_peered":"2022-09-29T05:25:03.816168+0000","last_clean":"2022-09-29T05:25:03.816168+0000","last_became_active":"2022-09-29T05:25:03.815888+0000","last_became_peered":"2022-09-29T05:25:03.815888+0000","last_unstale":"2022-09-29T05:25:03.816168+0000","last_undegraded":"2022-09-29T05:25:03.816168+0000","last_fullsized":"2022-09-29T05:25:03.816168+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3d","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.816176+0000","last_change":"2022-09-29T05:25:03.814221+0000","last_active":"2022-09-29T05:25:03.816176+0000","last_peered":"2022-09-29T05:25:03.816176+0000","last_clean":"2022-09-29T05:25:03.816176+0000","last_became_active":"2022-09-29T05:25:03.813927+0000","last_became_peered":"2022-09-29T05:25:03.813927+0000","last_unstale":"2022-09-29T05:25:03.816176+0000","last_undegraded":"2022-09-29T05:25:03.816176+0000","last_fullsized":"2022-09-29T05:25:03.816176+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3c","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.820613+0000","last_change":"2022-09-29T05:25:03.819554+0000","last_active":"2022-09-29T05:25:03.820613+0000","last_peered":"2022-09-29T05:25:03.820613+0000","last_clean":"2022-09-29T05:25:03.820613+0000","last_became_active":"2022-09-29T05:25:03.819207+0000","last_became_peered":"2022-09-29T05:25:03.819207+0000","last_unstale":"2022-09-29T05:25:03.820613+0000","last_undegraded":"2022-09-29T05:25:03.820613+0000","last_fullsized":"2022-09-29T05:25:03.820613+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3b","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.831380+0000","last_change":"2022-09-29T05:25:03.829034+0000","last_active":"2022-09-29T05:25:03.831380+0000","last_peered":"2022-09-29T05:25:03.831380+0000","last_clean":"2022-09-29T05:25:03.831380+0000","last_became_active":"2022-09-29T05:25:03.828384+0000","last_became_peered":"2022-09-29T05:25:03.828384+0000","last_unstale":"2022-09-29T05:25:03.831380+0000","last_undegraded":"2022-09-29T05:25:03.831380+0000","last_fullsized":"2022-09-29T05:25:03.831380+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3a","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.818790+0000","last_change":"2022-09-29T05:25:03.817871+0000","last_active":"2022-09-29T05:25:03.818790+0000","last_peered":"2022-09-29T05:25:03.818790+0000","last_clean":"2022-09-29T05:25:03.818790+0000","last_became_active":"2022-09-29T05:25:03.816800+0000","last_became_peered":"2022-09-29T05:25:03.816800+0000","last_unstale":"2022-09-29T05:25:03.818790+0000","last_undegraded":"2022-09-29T05:25:03.818790+0000","last_fullsized":"2022-09-29T05:25:03.818790+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.39","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.831254+0000","last_change":"2022-09-29T05:25:03.825279+0000","last_active":"2022-09-29T05:25:03.831254+0000","last_peered":"2022-09-29T05:25:03.831254+0000","last_clean":"2022-09-29T05:25:03.831254+0000","last_became_active":"2022-09-29T05:25:03.825002+0000","last_became_peered":"2022-09-29T05:25:03.825002+0000","last_unstale":"2022-09-29T05:25:03.831254+0000","last_undegraded":"2022-09-29T05:25:03.831254+0000","last_fullsized":"2022-09-29T05:25:03.831254+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.38","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.816333+0000","last_change":"2022-09-29T05:25:03.813803+0000","last_active":"2022-09-29T05:25:03.816333+0000","last_peered":"2022-09-29T05:25:03.816333+0000","last_clean":"2022-09-29T05:25:03.816333+0000","last_became_active":"2022-09-29T05:25:03.813488+0000","last_became_peered":"2022-09-29T05:25:03.813488+0000","last_unstale":"2022-09-29T05:25:03.816333+0000","last_undegraded":"2022-09-29T05:25:03.816333+0000","last_fullsized":"2022-09-29T05:25:03.816333+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.37","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.831437+0000","last_change":"2022-09-29T05:25:03.829880+0000","last_active":"2022-09-29T05:25:03.831437+0000","last_peered":"2022-09-29T05:25:03.831437+0000","last_clean":"2022-09-29T05:25:03.831437+0000","last_became_active":"2022-09-29T05:25:03.829318+0000","last_became_peered":"2022-09-29T05:25:03.829318+0000","last_unstale":"2022-09-29T05:25:03.831437+0000","last_undegraded":"2022-09-29T05:25:03.831437+0000","last_fullsized":"2022-09-29T05:25:03.831437+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,0],"acting":[7,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.36","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.825144+0000","last_change":"2022-09-29T05:25:03.823656+0000","last_active":"2022-09-29T05:25:03.825144+0000","last_peered":"2022-09-29T05:25:03.825144+0000","last_clean":"2022-09-29T05:25:03.825144+0000","last_became_active":"2022-09-29T05:25:03.823109+0000","last_became_peered":"2022-09-29T05:25:03.823109+0000","last_unstale":"2022-09-29T05:25:03.825144+0000","last_undegraded":"2022-09-29T05:25:03.825144+0000","last_fullsized":"2022-09-29T05:25:03.825144+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.35","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.825174+0000","last_change":"2022-09-29T05:25:03.824377+0000","last_active":"2022-09-29T05:25:03.825174+0000","last_peered":"2022-09-29T05:25:03.825174+0000","last_clean":"2022-09-29T05:25:03.825174+0000","last_became_active":"2022-09-29T05:25:03.823869+0000","last_became_peered":"2022-09-29T05:25:03.823869+0000","last_unstale":"2022-09-29T05:25:03.825174+0000","last_undegraded":"2022-09-29T05:25:03.825174+0000","last_fullsized":"2022-09-29T05:25:03.825174+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.34","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.821389+0000","last_change":"2022-09-29T05:25:03.812233+0000","last_active":"2022-09-29T05:25:03.821389+0000","last_peered":"2022-09-29T05:25:03.821389+0000","last_clean":"2022-09-29T05:25:03.821389+0000","last_became_active":"2022-09-29T05:25:03.811765+0000","last_became_peered":"2022-09-29T05:25:03.811765+0000","last_unstale":"2022-09-29T05:25:03.821389+0000","last_undegraded":"2022-09-29T05:25:03.821389+0000","last_fullsized":"2022-09-29T05:25:03.821389+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.33","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.816299+0000","last_change":"2022-09-29T05:25:03.813308+0000","last_active":"2022-09-29T05:25:03.816299+0000","last_peered":"2022-09-29T05:25:03.816299+0000","last_clean":"2022-09-29T05:25:03.816299+0000","last_became_active":"2022-09-29T05:25:03.813019+0000","last_became_peered":"2022-09-29T05:25:03.813019+0000","last_unstale":"2022-09-29T05:25:03.816299+0000","last_undegraded":"2022-09-29T05:25:03.816299+0000","last_fullsized":"2022-09-29T05:25:03.816299+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.32","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.823741+0000","last_change":"2022-09-29T05:25:03.823321+0000","last_active":"2022-09-29T05:25:03.823741+0000","last_peered":"2022-09-29T05:25:03.823741+0000","last_clean":"2022-09-29T05:25:03.823741+0000","last_became_active":"2022-09-29T05:25:03.822987+0000","last_became_peered":"2022-09-29T05:25:03.822987+0000","last_unstale":"2022-09-29T05:25:03.823741+0000","last_undegraded":"2022-09-29T05:25:03.823741+0000","last_fullsized":"2022-09-29T05:25:03.823741+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.31","version":"0'0","reported_seq":9,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.805590+0000","last_change":"2022-09-29T05:25:03.805590+0000","last_active":"2022-09-29T05:25:03.805590+0000","last_peered":"2022-09-29T05:25:03.805590+0000","last_clean":"2022-09-29T05:25:03.805590+0000","last_became_active":"2022-09-29T05:25:03.805310+0000","last_became_peered":"2022-09-29T05:25:03.805310+0000","last_unstale":"2022-09-29T05:25:03.805590+0000","last_undegraded":"2022-09-29T05:25:03.805590+0000","last_fullsized":"2022-09-29T05:25:03.805590+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.30","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.816720+0000","last_change":"2022-09-29T05:25:03.815886+0000","last_active":"2022-09-29T05:25:03.816720+0000","last_peered":"2022-09-29T05:25:03.816720+0000","last_clean":"2022-09-29T05:25:03.816720+0000","last_became_active":"2022-09-29T05:25:03.815285+0000","last_became_peered":"2022-09-29T05:25:03.815285+0000","last_unstale":"2022-09-29T05:25:03.816720+0000","last_undegraded":"2022-09-29T05:25:03.816720+0000","last_fullsized":"2022-09-29T05:25:03.816720+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2f","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.816214+0000","last_change":"2022-09-29T05:25:03.813784+0000","last_active":"2022-09-29T05:25:03.816214+0000","last_peered":"2022-09-29T05:25:03.816214+0000","last_clean":"2022-09-29T05:25:03.816214+0000","last_became_active":"2022-09-29T05:25:03.813420+0000","last_became_peered":"2022-09-29T05:25:03.813420+0000","last_unstale":"2022-09-29T05:25:03.816214+0000","last_undegraded":"2022-09-29T05:25:03.816214+0000","last_fullsized":"2022-09-29T05:25:03.816214+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2e","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.816847+0000","last_change":"2022-09-29T05:25:03.813894+0000","last_active":"2022-09-29T05:25:03.816847+0000","last_peered":"2022-09-29T05:25:03.816847+0000","last_clean":"2022-09-29T05:25:03.816847+0000","last_became_active":"2022-09-29T05:25:03.813620+0000","last_became_peered":"2022-09-29T05:25:03.813620+0000","last_unstale":"2022-09-29T05:25:03.816847+0000","last_undegraded":"2022-09-29T05:25:03.816847+0000","last_fullsized":"2022-09-29T05:25:03.816847+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,7],"acting":[2,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2d","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.831407+0000","last_change":"2022-09-29T05:25:03.828324+0000","last_active":"2022-09-29T05:25:03.831407+0000","last_peered":"2022-09-29T05:25:03.831407+0000","last_clean":"2022-09-29T05:25:03.831407+0000","last_became_active":"2022-09-29T05:25:03.827685+0000","last_became_peered":"2022-09-29T05:25:03.827685+0000","last_unstale":"2022-09-29T05:25:03.831407+0000","last_undegraded":"2022-09-29T05:25:03.831407+0000","last_fullsized":"2022-09-29T05:25:03.831407+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1f","version":"50'4","reported_seq":15,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.824599+0000","last_change":"2022-09-29T05:25:03.449078+0000","last_active":"2022-09-29T05:25:03.824599+0000","last_peered":"2022-09-29T05:25:03.824599+0000","last_clean":"2022-09-29T05:25:03.824599+0000","last_became_active":"2022-09-29T05:25:03.448793+0000","last_became_peered":"2022-09-29T05:25:03.448793+0000","last_unstale":"2022-09-29T05:25:03.824599+0000","last_undegraded":"2022-09-29T05:25:03.824599+0000","last_fullsized":"2022-09-29T05:25:03.824599+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":4,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.816314+0000","last_change":"2022-09-29T05:25:03.814538+0000","last_active":"2022-09-29T05:25:03.816314+0000","last_peered":"2022-09-29T05:25:03.816314+0000","last_clean":"2022-09-29T05:25:03.816314+0000","last_became_active":"2022-09-29T05:25:03.814264+0000","last_became_peered":"2022-09-29T05:25:03.814264+0000","last_unstale":"2022-09-29T05:25:03.816314+0000","last_undegraded":"2022-09-29T05:25:03.816314+0000","last_fullsized":"2022-09-29T05:25:03.816314+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1e","version":"50'1","reported_seq":12,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.820630+0000","last_change":"2022-09-29T05:25:03.459868+0000","last_active":"2022-09-29T05:25:03.820630+0000","last_peered":"2022-09-29T05:25:03.820630+0000","last_clean":"2022-09-29T05:25:03.820630+0000","last_became_active":"2022-09-29T05:25:03.459458+0000","last_became_peered":"2022-09-29T05:25:03.459458+0000","last_unstale":"2022-09-29T05:25:03.820630+0000","last_undegraded":"2022-09-29T05:25:03.820630+0000","last_fullsized":"2022-09-29T05:25:03.820630+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.822676+0000","last_change":"2022-09-29T05:25:03.822430+0000","last_active":"2022-09-29T05:25:03.822676+0000","last_peered":"2022-09-29T05:25:03.822676+0000","last_clean":"2022-09-29T05:25:03.822676+0000","last_became_active":"2022-09-29T05:25:03.821898+0000","last_became_peered":"2022-09-29T05:25:03.821898+0000","last_unstale":"2022-09-29T05:25:03.822676+0000","last_undegraded":"2022-09-29T05:25:03.822676+0000","last_fullsized":"2022-09-29T05:25:03.822676+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1d","version":"50'2","reported_seq":13,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.828716+0000","last_change":"2022-09-29T05:25:03.455815+0000","last_active":"2022-09-29T05:25:03.828716+0000","last_peered":"2022-09-29T05:25:03.828716+0000","last_clean":"2022-09-29T05:25:03.828716+0000","last_became_active":"2022-09-29T05:25:03.455218+0000","last_became_peered":"2022-09-29T05:25:03.455218+0000","last_unstale":"2022-09-29T05:25:03.828716+0000","last_undegraded":"2022-09-29T05:25:03.828716+0000","last_fullsized":"2022-09-29T05:25:03.828716+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.820593+0000","last_change":"2022-09-29T05:25:03.816581+0000","last_active":"2022-09-29T05:25:03.820593+0000","last_peered":"2022-09-29T05:25:03.820593+0000","last_clean":"2022-09-29T05:25:03.820593+0000","last_became_active":"2022-09-29T05:25:03.816206+0000","last_became_peered":"2022-09-29T05:25:03.816206+0000","last_unstale":"2022-09-29T05:25:03.820593+0000","last_undegraded":"2022-09-29T05:25:03.820593+0000","last_fullsized":"2022-09-29T05:25:03.820593+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1c","version":"50'2","reported_seq":13,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.826639+0000","last_change":"2022-09-29T05:25:03.460578+0000","last_active":"2022-09-29T05:25:03.826639+0000","last_peered":"2022-09-29T05:25:03.826639+0000","last_clean":"2022-09-29T05:25:03.826639+0000","last_became_active":"2022-09-29T05:25:03.460273+0000","last_became_peered":"2022-09-29T05:25:03.460273+0000","last_unstale":"2022-09-29T05:25:03.826639+0000","last_undegraded":"2022-09-29T05:25:03.826639+0000","last_fullsized":"2022-09-29T05:25:03.826639+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1d","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.822057+0000","last_change":"2022-09-29T05:25:03.820523+0000","last_active":"2022-09-29T05:25:03.822057+0000","last_peered":"2022-09-29T05:25:03.822057+0000","last_clean":"2022-09-29T05:25:03.822057+0000","last_became_active":"2022-09-29T05:25:03.819546+0000","last_became_peered":"2022-09-29T05:25:03.819546+0000","last_unstale":"2022-09-29T05:25:03.822057+0000","last_undegraded":"2022-09-29T05:25:03.822057+0000","last_fullsized":"2022-09-29T05:25:03.822057+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1b","version":"50'1","reported_seq":12,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.824161+0000","last_change":"2022-09-29T05:25:03.455758+0000","last_active":"2022-09-29T05:25:03.824161+0000","last_peered":"2022-09-29T05:25:03.824161+0000","last_clean":"2022-09-29T05:25:03.824161+0000","last_became_active":"2022-09-29T05:25:03.455438+0000","last_became_peered":"2022-09-29T05:25:03.455438+0000","last_unstale":"2022-09-29T05:25:03.824161+0000","last_undegraded":"2022-09-29T05:25:03.824161+0000","last_fullsized":"2022-09-29T05:25:03.824161+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.820593+0000","last_change":"2022-09-29T05:25:03.817393+0000","last_active":"2022-09-29T05:25:03.820593+0000","last_peered":"2022-09-29T05:25:03.820593+0000","last_clean":"2022-09-29T05:25:03.820593+0000","last_became_active":"2022-09-29T05:25:03.817103+0000","last_became_peered":"2022-09-29T05:25:03.817103+0000","last_unstale":"2022-09-29T05:25:03.820593+0000","last_undegraded":"2022-09-29T05:25:03.820593+0000","last_fullsized":"2022-09-29T05:25:03.820593+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1a","version":"50'1","reported_seq":12,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.815499+0000","last_change":"2022-09-29T05:25:03.454747+0000","last_active":"2022-09-29T05:25:03.815499+0000","last_peered":"2022-09-29T05:25:03.815499+0000","last_clean":"2022-09-29T05:25:03.815499+0000","last_became_active":"2022-09-29T05:25:03.454395+0000","last_became_peered":"2022-09-29T05:25:03.454395+0000","last_unstale":"2022-09-29T05:25:03.815499+0000","last_undegraded":"2022-09-29T05:25:03.815499+0000","last_fullsized":"2022-09-29T05:25:03.815499+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1b","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.820424+0000","last_change":"2022-09-29T05:25:03.819253+0000","last_active":"2022-09-29T05:25:03.820424+0000","last_peered":"2022-09-29T05:25:03.820424+0000","last_clean":"2022-09-29T05:25:03.820424+0000","last_became_active":"2022-09-29T05:25:03.818951+0000","last_became_peered":"2022-09-29T05:25:03.818951+0000","last_unstale":"2022-09-29T05:25:03.820424+0000","last_undegraded":"2022-09-29T05:25:03.820424+0000","last_fullsized":"2022-09-29T05:25:03.820424+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.19","version":"0'0","reported_seq":11,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.794770+0000","last_change":"2022-09-29T05:25:03.455193+0000","last_active":"2022-09-29T05:25:03.794770+0000","last_peered":"2022-09-29T05:25:03.794770+0000","last_clean":"2022-09-29T05:25:03.794770+0000","last_became_active":"2022-09-29T05:25:03.454916+0000","last_became_peered":"2022-09-29T05:25:03.454916+0000","last_unstale":"2022-09-29T05:25:03.794770+0000","last_undegraded":"2022-09-29T05:25:03.794770+0000","last_fullsized":"2022-09-29T05:25:03.794770+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.809444+0000","last_change":"2022-09-29T05:25:03.806930+0000","last_active":"2022-09-29T05:25:03.809444+0000","last_peered":"2022-09-29T05:25:03.809444+0000","last_clean":"2022-09-29T05:25:03.809444+0000","last_became_active":"2022-09-29T05:25:03.806565+0000","last_became_peered":"2022-09-29T05:25:03.806565+0000","last_unstale":"2022-09-29T05:25:03.809444+0000","last_undegraded":"2022-09-29T05:25:03.809444+0000","last_fullsized":"2022-09-29T05:25:03.809444+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.18","version":"0'0","reported_seq":11,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.796508+0000","last_change":"2022-09-29T05:25:03.460675+0000","last_active":"2022-09-29T05:25:03.796508+0000","last_peered":"2022-09-29T05:25:03.796508+0000","last_clean":"2022-09-29T05:25:03.796508+0000","last_became_active":"2022-09-29T05:25:03.460351+0000","last_became_peered":"2022-09-29T05:25:03.460351+0000","last_unstale":"2022-09-29T05:25:03.796508+0000","last_undegraded":"2022-09-29T05:25:03.796508+0000","last_fullsized":"2022-09-29T05:25:03.796508+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":9,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.816170+0000","last_change":"2022-09-29T05:25:03.816170+0000","last_active":"2022-09-29T05:25:03.816170+0000","last_peered":"2022-09-29T05:25:03.816170+0000","last_clean":"2022-09-29T05:25:03.816170+0000","last_became_active":"2022-09-29T05:25:03.815838+0000","last_became_peered":"2022-09-29T05:25:03.815838+0000","last_unstale":"2022-09-29T05:25:03.816170+0000","last_undegraded":"2022-09-29T05:25:03.816170+0000","last_fullsized":"2022-09-29T05:25:03.816170+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.17","version":"0'0","reported_seq":11,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.805981+0000","last_change":"2022-09-29T05:25:03.461045+0000","last_active":"2022-09-29T05:25:03.805981+0000","last_peered":"2022-09-29T05:25:03.805981+0000","last_clean":"2022-09-29T05:25:03.805981+0000","last_became_active":"2022-09-29T05:25:03.460729+0000","last_became_peered":"2022-09-29T05:25:03.460729+0000","last_unstale":"2022-09-29T05:25:03.805981+0000","last_undegraded":"2022-09-29T05:25:03.805981+0000","last_fullsized":"2022-09-29T05:25:03.805981+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.819351+0000","last_change":"2022-09-29T05:25:03.818733+0000","last_active":"2022-09-29T05:25:03.819351+0000","last_peered":"2022-09-29T05:25:03.819351+0000","last_clean":"2022-09-29T05:25:03.819351+0000","last_became_active":"2022-09-29T05:25:03.816983+0000","last_became_peered":"2022-09-29T05:25:03.816983+0000","last_unstale":"2022-09-29T05:25:03.819351+0000","last_undegraded":"2022-09-29T05:25:03.819351+0000","last_fullsized":"2022-09-29T05:25:03.819351+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.16","version":"0'0","reported_seq":11,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.796278+0000","last_change":"2022-09-29T05:25:03.463477+0000","last_active":"2022-09-29T05:25:03.796278+0000","last_peered":"2022-09-29T05:25:03.796278+0000","last_clean":"2022-09-29T05:25:03.796278+0000","last_became_active":"2022-09-29T05:25:03.463051+0000","last_became_peered":"2022-09-29T05:25:03.463051+0000","last_unstale":"2022-09-29T05:25:03.796278+0000","last_undegraded":"2022-09-29T05:25:03.796278+0000","last_fullsized":"2022-09-29T05:25:03.796278+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.822495+0000","last_change":"2022-09-29T05:25:03.822200+0000","last_active":"2022-09-29T05:25:03.822495+0000","last_peered":"2022-09-29T05:25:03.822495+0000","last_clean":"2022-09-29T05:25:03.822495+0000","last_became_active":"2022-09-29T05:25:03.821488+0000","last_became_peered":"2022-09-29T05:25:03.821488+0000","last_unstale":"2022-09-29T05:25:03.822495+0000","last_undegraded":"2022-09-29T05:25:03.822495+0000","last_fullsized":"2022-09-29T05:25:03.822495+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.15","version":"50'3","reported_seq":13,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.811766+0000","last_change":"2022-09-29T05:25:03.457077+0000","last_active":"2022-09-29T05:25:03.811766+0000","last_peered":"2022-09-29T05:25:03.811766+0000","last_clean":"2022-09-29T05:25:03.811766+0000","last_became_active":"2022-09-29T05:25:03.456823+0000","last_became_peered":"2022-09-29T05:25:03.456823+0000","last_unstale":"2022-09-29T05:25:03.811766+0000","last_undegraded":"2022-09-29T05:25:03.811766+0000","last_fullsized":"2022-09-29T05:25:03.811766+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":3,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.825581+0000","last_change":"2022-09-29T05:25:03.822638+0000","last_active":"2022-09-29T05:25:03.825581+0000","last_peered":"2022-09-29T05:25:03.825581+0000","last_clean":"2022-09-29T05:25:03.825581+0000","last_became_active":"2022-09-29T05:25:03.822297+0000","last_became_peered":"2022-09-29T05:25:03.822297+0000","last_unstale":"2022-09-29T05:25:03.825581+0000","last_undegraded":"2022-09-29T05:25:03.825581+0000","last_fullsized":"2022-09-29T05:25:03.825581+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.14","version":"50'6","reported_seq":17,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.827688+0000","last_change":"2022-09-29T05:25:03.461139+0000","last_active":"2022-09-29T05:25:03.827688+0000","last_peered":"2022-09-29T05:25:03.827688+0000","last_clean":"2022-09-29T05:25:03.827688+0000","last_became_active":"2022-09-29T05:25:03.460829+0000","last_became_peered":"2022-09-29T05:25:03.460829+0000","last_unstale":"2022-09-29T05:25:03.827688+0000","last_undegraded":"2022-09-29T05:25:03.827688+0000","last_fullsized":"2022-09-29T05:25:03.827688+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":6,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":1536,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.831388+0000","last_change":"2022-09-29T05:25:03.828186+0000","last_active":"2022-09-29T05:25:03.831388+0000","last_peered":"2022-09-29T05:25:03.831388+0000","last_clean":"2022-09-29T05:25:03.831388+0000","last_became_active":"2022-09-29T05:25:03.826167+0000","last_became_peered":"2022-09-29T05:25:03.826167+0000","last_unstale":"2022-09-29T05:25:03.831388+0000","last_undegraded":"2022-09-29T05:25:03.831388+0000","last_fullsized":"2022-09-29T05:25:03.831388+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.13","version":"50'2","reported_seq":13,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.817560+0000","last_change":"2022-09-29T05:25:03.451661+0000","last_active":"2022-09-29T05:25:03.817560+0000","last_peered":"2022-09-29T05:25:03.817560+0000","last_clean":"2022-09-29T05:25:03.817560+0000","last_became_active":"2022-09-29T05:25:03.451389+0000","last_became_peered":"2022-09-29T05:25:03.451389+0000","last_unstale":"2022-09-29T05:25:03.817560+0000","last_undegraded":"2022-09-29T05:25:03.817560+0000","last_fullsized":"2022-09-29T05:25:03.817560+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.825490+0000","last_change":"2022-09-29T05:25:03.824608+0000","last_active":"2022-09-29T05:25:03.825490+0000","last_peered":"2022-09-29T05:25:03.825490+0000","last_clean":"2022-09-29T05:25:03.825490+0000","last_became_active":"2022-09-29T05:25:03.824335+0000","last_became_peered":"2022-09-29T05:25:03.824335+0000","last_unstale":"2022-09-29T05:25:03.825490+0000","last_undegraded":"2022-09-29T05:25:03.825490+0000","last_fullsized":"2022-09-29T05:25:03.825490+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.12","version":"0'0","reported_seq":11,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.796204+0000","last_change":"2022-09-29T05:25:03.462145+0000","last_active":"2022-09-29T05:25:03.796204+0000","last_peered":"2022-09-29T05:25:03.796204+0000","last_clean":"2022-09-29T05:25:03.796204+0000","last_became_active":"2022-09-29T05:25:03.461864+0000","last_became_peered":"2022-09-29T05:25:03.461864+0000","last_unstale":"2022-09-29T05:25:03.796204+0000","last_undegraded":"2022-09-29T05:25:03.796204+0000","last_fullsized":"2022-09-29T05:25:03.796204+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.831361+0000","last_change":"2022-09-29T05:25:03.828734+0000","last_active":"2022-09-29T05:25:03.831361+0000","last_peered":"2022-09-29T05:25:03.831361+0000","last_clean":"2022-09-29T05:25:03.831361+0000","last_became_active":"2022-09-29T05:25:03.827929+0000","last_became_peered":"2022-09-29T05:25:03.827929+0000","last_unstale":"2022-09-29T05:25:03.831361+0000","last_undegraded":"2022-09-29T05:25:03.831361+0000","last_fullsized":"2022-09-29T05:25:03.831361+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.11","version":"0'0","reported_seq":11,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.805505+0000","last_change":"2022-09-29T05:25:03.454654+0000","last_active":"2022-09-29T05:25:03.805505+0000","last_peered":"2022-09-29T05:25:03.805505+0000","last_clean":"2022-09-29T05:25:03.805505+0000","last_became_active":"2022-09-29T05:25:03.454319+0000","last_became_peered":"2022-09-29T05:25:03.454319+0000","last_unstale":"2022-09-29T05:25:03.805505+0000","last_undegraded":"2022-09-29T05:25:03.805505+0000","last_fullsized":"2022-09-29T05:25:03.805505+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.813586+0000","last_change":"2022-09-29T05:25:03.811425+0000","last_active":"2022-09-29T05:25:03.813586+0000","last_peered":"2022-09-29T05:25:03.813586+0000","last_clean":"2022-09-29T05:25:03.813586+0000","last_became_active":"2022-09-29T05:25:03.811120+0000","last_became_peered":"2022-09-29T05:25:03.811120+0000","last_unstale":"2022-09-29T05:25:03.813586+0000","last_undegraded":"2022-09-29T05:25:03.813586+0000","last_fullsized":"2022-09-29T05:25:03.813586+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.10","version":"50'8","reported_seq":19,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.813615+0000","last_change":"2022-09-29T05:25:03.449137+0000","last_active":"2022-09-29T05:25:03.813615+0000","last_peered":"2022-09-29T05:25:03.813615+0000","last_clean":"2022-09-29T05:25:03.813615+0000","last_became_active":"2022-09-29T05:25:03.448728+0000","last_became_peered":"2022-09-29T05:25:03.448728+0000","last_unstale":"2022-09-29T05:25:03.813615+0000","last_undegraded":"2022-09-29T05:25:03.813615+0000","last_fullsized":"2022-09-29T05:25:03.813615+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":8,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.831276+0000","last_change":"2022-09-29T05:25:03.826279+0000","last_active":"2022-09-29T05:25:03.831276+0000","last_peered":"2022-09-29T05:25:03.831276+0000","last_clean":"2022-09-29T05:25:03.831276+0000","last_became_active":"2022-09-29T05:25:03.825969+0000","last_became_peered":"2022-09-29T05:25:03.825969+0000","last_unstale":"2022-09-29T05:25:03.831276+0000","last_undegraded":"2022-09-29T05:25:03.831276+0000","last_fullsized":"2022-09-29T05:25:03.831276+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.f","version":"50'1","reported_seq":12,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.824523+0000","last_change":"2022-09-29T05:25:03.452230+0000","last_active":"2022-09-29T05:25:03.824523+0000","last_peered":"2022-09-29T05:25:03.824523+0000","last_clean":"2022-09-29T05:25:03.824523+0000","last_became_active":"2022-09-29T05:25:03.451747+0000","last_became_peered":"2022-09-29T05:25:03.451747+0000","last_unstale":"2022-09-29T05:25:03.824523+0000","last_undegraded":"2022-09-29T05:25:03.824523+0000","last_fullsized":"2022-09-29T05:25:03.824523+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.831346+0000","last_change":"2022-09-29T05:25:03.827453+0000","last_active":"2022-09-29T05:25:03.831346+0000","last_peered":"2022-09-29T05:25:03.831346+0000","last_clean":"2022-09-29T05:25:03.831346+0000","last_became_active":"2022-09-29T05:25:03.827151+0000","last_became_peered":"2022-09-29T05:25:03.827151+0000","last_unstale":"2022-09-29T05:25:03.831346+0000","last_undegraded":"2022-09-29T05:25:03.831346+0000","last_fullsized":"2022-09-29T05:25:03.831346+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":10,"reported_epoch":49,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.454814+0000","last_change":"2022-09-29T05:25:03.454507+0000","last_active":"2022-09-29T05:25:03.454814+0000","last_peered":"2022-09-29T05:25:03.454814+0000","last_clean":"2022-09-29T05:25:03.454814+0000","last_became_active":"2022-09-29T05:25:03.454195+0000","last_became_peered":"2022-09-29T05:25:03.454195+0000","last_unstale":"2022-09-29T05:25:03.454814+0000","last_undegraded":"2022-09-29T05:25:03.454814+0000","last_fullsized":"2022-09-29T05:25:03.454814+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.822030+0000","last_change":"2022-09-29T05:25:03.819862+0000","last_active":"2022-09-29T05:25:03.822030+0000","last_peered":"2022-09-29T05:25:03.822030+0000","last_clean":"2022-09-29T05:25:03.822030+0000","last_became_active":"2022-09-29T05:25:03.818117+0000","last_became_peered":"2022-09-29T05:25:03.818117+0000","last_unstale":"2022-09-29T05:25:03.822030+0000","last_undegraded":"2022-09-29T05:25:03.822030+0000","last_fullsized":"2022-09-29T05:25:03.822030+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":11,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.796134+0000","last_change":"2022-09-29T05:25:03.459646+0000","last_active":"2022-09-29T05:25:03.796134+0000","last_peered":"2022-09-29T05:25:03.796134+0000","last_clean":"2022-09-29T05:25:03.796134+0000","last_became_active":"2022-09-29T05:25:03.459320+0000","last_became_peered":"2022-09-29T05:25:03.459320+0000","last_unstale":"2022-09-29T05:25:03.796134+0000","last_undegraded":"2022-09-29T05:25:03.796134+0000","last_fullsized":"2022-09-29T05:25:03.796134+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.815521+0000","last_change":"2022-09-29T05:25:03.810358+0000","last_active":"2022-09-29T05:25:03.815521+0000","last_peered":"2022-09-29T05:25:03.815521+0000","last_clean":"2022-09-29T05:25:03.815521+0000","last_became_active":"2022-09-29T05:25:03.809959+0000","last_became_peered":"2022-09-29T05:25:03.809959+0000","last_unstale":"2022-09-29T05:25:03.815521+0000","last_undegraded":"2022-09-29T05:25:03.815521+0000","last_fullsized":"2022-09-29T05:25:03.815521+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":9,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.812390+0000","last_change":"2022-09-29T05:25:03.812390+0000","last_active":"2022-09-29T05:25:03.812390+0000","last_peered":"2022-09-29T05:25:03.812390+0000","last_clean":"2022-09-29T05:25:03.812390+0000","last_became_active":"2022-09-29T05:25:03.811925+0000","last_became_peered":"2022-09-29T05:25:03.811925+0000","last_unstale":"2022-09-29T05:25:03.812390+0000","last_undegraded":"2022-09-29T05:25:03.812390+0000","last_fullsized":"2022-09-29T05:25:03.812390+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":11,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.793975+0000","last_change":"2022-09-29T05:25:03.449156+0000","last_active":"2022-09-29T05:25:03.793975+0000","last_peered":"2022-09-29T05:25:03.793975+0000","last_clean":"2022-09-29T05:25:03.793975+0000","last_became_active":"2022-09-29T05:25:03.448885+0000","last_became_peered":"2022-09-29T05:25:03.448885+0000","last_unstale":"2022-09-29T05:25:03.793975+0000","last_undegraded":"2022-09-29T05:25:03.793975+0000","last_fullsized":"2022-09-29T05:25:03.793975+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.816328+0000","last_change":"2022-09-29T05:25:03.813170+0000","last_active":"2022-09-29T05:25:03.816328+0000","last_peered":"2022-09-29T05:25:03.816328+0000","last_clean":"2022-09-29T05:25:03.816328+0000","last_became_active":"2022-09-29T05:25:03.812861+0000","last_became_peered":"2022-09-29T05:25:03.812861+0000","last_unstale":"2022-09-29T05:25:03.816328+0000","last_undegraded":"2022-09-29T05:25:03.816328+0000","last_fullsized":"2022-09-29T05:25:03.816328+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":11,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.796275+0000","last_change":"2022-09-29T05:25:03.462271+0000","last_active":"2022-09-29T05:25:03.796275+0000","last_peered":"2022-09-29T05:25:03.796275+0000","last_clean":"2022-09-29T05:25:03.796275+0000","last_became_active":"2022-09-29T05:25:03.461959+0000","last_became_peered":"2022-09-29T05:25:03.461959+0000","last_unstale":"2022-09-29T05:25:03.796275+0000","last_undegraded":"2022-09-29T05:25:03.796275+0000","last_fullsized":"2022-09-29T05:25:03.796275+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":14,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.803350+0000","last_change":"2022-09-29T05:24:48.628243+0000","last_active":"2022-09-29T05:25:03.803350+0000","last_peered":"2022-09-29T05:25:03.803350+0000","last_clean":"2022-09-29T05:25:03.803350+0000","last_became_active":"2022-09-29T05:24:48.627801+0000","last_became_peered":"2022-09-29T05:24:48.627801+0000","last_unstale":"2022-09-29T05:25:03.803350+0000","last_undegraded":"2022-09-29T05:25:03.803350+0000","last_fullsized":"2022-09-29T05:25:03.803350+0000","mapping_epoch":46,"log_start":"0'0","ondisk_log_start":"0'0","created":21,"last_epoch_clean":47,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:23:35.544281+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:23:35.544281+0000","last_clean_scrub_stamp":"2022-09-29T05:23:35.544281+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.812499+0000","last_change":"2022-09-29T05:25:03.812201+0000","last_active":"2022-09-29T05:25:03.812499+0000","last_peered":"2022-09-29T05:25:03.812499+0000","last_clean":"2022-09-29T05:25:03.812499+0000","last_became_active":"2022-09-29T05:25:03.811903+0000","last_became_peered":"2022-09-29T05:25:03.811903+0000","last_unstale":"2022-09-29T05:25:03.812499+0000","last_undegraded":"2022-09-29T05:25:03.812499+0000","last_fullsized":"2022-09-29T05:25:03.812499+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.804063+0000","last_change":"2022-09-29T05:25:03.451876+0000","last_active":"2022-09-29T05:25:03.804063+0000","last_peered":"2022-09-29T05:25:03.804063+0000","last_clean":"2022-09-29T05:25:03.804063+0000","last_became_active":"2022-09-29T05:25:03.451371+0000","last_became_peered":"2022-09-29T05:25:03.451371+0000","last_unstale":"2022-09-29T05:25:03.804063+0000","last_undegraded":"2022-09-29T05:25:03.804063+0000","last_fullsized":"2022-09-29T05:25:03.804063+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.816054+0000","last_change":"2022-09-29T05:25:03.453060+0000","last_active":"2022-09-29T05:25:03.816054+0000","last_peered":"2022-09-29T05:25:03.816054+0000","last_clean":"2022-09-29T05:25:03.816054+0000","last_became_active":"2022-09-29T05:25:03.452758+0000","last_became_peered":"2022-09-29T05:25:03.452758+0000","last_unstale":"2022-09-29T05:25:03.816054+0000","last_undegraded":"2022-09-29T05:25:03.816054+0000","last_fullsized":"2022-09-29T05:25:03.816054+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":9,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.803195+0000","last_change":"2022-09-29T05:25:03.803195+0000","last_active":"2022-09-29T05:25:03.803195+0000","last_peered":"2022-09-29T05:25:03.803195+0000","last_clean":"2022-09-29T05:25:03.803195+0000","last_became_active":"2022-09-29T05:25:03.802909+0000","last_became_peered":"2022-09-29T05:25:03.802909+0000","last_unstale":"2022-09-29T05:25:03.803195+0000","last_undegraded":"2022-09-29T05:25:03.803195+0000","last_fullsized":"2022-09-29T05:25:03.803195+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":10,"reported_epoch":49,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.457008+0000","last_change":"2022-09-29T05:25:03.456735+0000","last_active":"2022-09-29T05:25:03.457008+0000","last_peered":"2022-09-29T05:25:03.457008+0000","last_clean":"2022-09-29T05:25:03.457008+0000","last_became_active":"2022-09-29T05:25:03.456427+0000","last_became_peered":"2022-09-29T05:25:03.456427+0000","last_unstale":"2022-09-29T05:25:03.457008+0000","last_undegraded":"2022-09-29T05:25:03.457008+0000","last_fullsized":"2022-09-29T05:25:03.457008+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.820572+0000","last_change":"2022-09-29T05:25:03.819725+0000","last_active":"2022-09-29T05:25:03.820572+0000","last_peered":"2022-09-29T05:25:03.820572+0000","last_clean":"2022-09-29T05:25:03.820572+0000","last_became_active":"2022-09-29T05:25:03.819455+0000","last_became_peered":"2022-09-29T05:25:03.819455+0000","last_unstale":"2022-09-29T05:25:03.820572+0000","last_undegraded":"2022-09-29T05:25:03.820572+0000","last_fullsized":"2022-09-29T05:25:03.820572+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"50'2","reported_seq":13,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.824910+0000","last_change":"2022-09-29T05:25:03.462138+0000","last_active":"2022-09-29T05:25:03.824910+0000","last_peered":"2022-09-29T05:25:03.824910+0000","last_clean":"2022-09-29T05:25:03.824910+0000","last_became_active":"2022-09-29T05:25:03.461809+0000","last_became_peered":"2022-09-29T05:25:03.461809+0000","last_unstale":"2022-09-29T05:25:03.824910+0000","last_undegraded":"2022-09-29T05:25:03.824910+0000","last_fullsized":"2022-09-29T05:25:03.824910+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.816152+0000","last_change":"2022-09-29T05:25:03.813902+0000","last_active":"2022-09-29T05:25:03.816152+0000","last_peered":"2022-09-29T05:25:03.816152+0000","last_clean":"2022-09-29T05:25:03.816152+0000","last_became_active":"2022-09-29T05:25:03.813617+0000","last_became_peered":"2022-09-29T05:25:03.813617+0000","last_unstale":"2022-09-29T05:25:03.816152+0000","last_undegraded":"2022-09-29T05:25:03.816152+0000","last_fullsized":"2022-09-29T05:25:03.816152+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.8","version":"0'0","reported_seq":11,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.805654+0000","last_change":"2022-09-29T05:25:03.451677+0000","last_active":"2022-09-29T05:25:03.805654+0000","last_peered":"2022-09-29T05:25:03.805654+0000","last_clean":"2022-09-29T05:25:03.805654+0000","last_became_active":"2022-09-29T05:25:03.450961+0000","last_became_peered":"2022-09-29T05:25:03.450961+0000","last_unstale":"2022-09-29T05:25:03.805654+0000","last_undegraded":"2022-09-29T05:25:03.805654+0000","last_fullsized":"2022-09-29T05:25:03.805654+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.825550+0000","last_change":"2022-09-29T05:25:03.822108+0000","last_active":"2022-09-29T05:25:03.825550+0000","last_peered":"2022-09-29T05:25:03.825550+0000","last_clean":"2022-09-29T05:25:03.825550+0000","last_became_active":"2022-09-29T05:25:03.821784+0000","last_became_peered":"2022-09-29T05:25:03.821784+0000","last_unstale":"2022-09-29T05:25:03.825550+0000","last_undegraded":"2022-09-29T05:25:03.825550+0000","last_fullsized":"2022-09-29T05:25:03.825550+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.9","version":"0'0","reported_seq":10,"reported_epoch":49,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.452964+0000","last_change":"2022-09-29T05:25:03.452512+0000","last_active":"2022-09-29T05:25:03.452964+0000","last_peered":"2022-09-29T05:25:03.452964+0000","last_clean":"2022-09-29T05:25:03.452964+0000","last_became_active":"2022-09-29T05:25:03.452222+0000","last_became_peered":"2022-09-29T05:25:03.452222+0000","last_unstale":"2022-09-29T05:25:03.452964+0000","last_undegraded":"2022-09-29T05:25:03.452964+0000","last_fullsized":"2022-09-29T05:25:03.452964+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.820040+0000","last_change":"2022-09-29T05:25:03.819591+0000","last_active":"2022-09-29T05:25:03.820040+0000","last_peered":"2022-09-29T05:25:03.820040+0000","last_clean":"2022-09-29T05:25:03.820040+0000","last_became_active":"2022-09-29T05:25:03.819231+0000","last_became_peered":"2022-09-29T05:25:03.819231+0000","last_unstale":"2022-09-29T05:25:03.820040+0000","last_undegraded":"2022-09-29T05:25:03.820040+0000","last_fullsized":"2022-09-29T05:25:03.820040+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.a","version":"0'0","reported_seq":10,"reported_epoch":49,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.452852+0000","last_change":"2022-09-29T05:25:03.451868+0000","last_active":"2022-09-29T05:25:03.452852+0000","last_peered":"2022-09-29T05:25:03.452852+0000","last_clean":"2022-09-29T05:25:03.452852+0000","last_became_active":"2022-09-29T05:25:03.451576+0000","last_became_peered":"2022-09-29T05:25:03.451576+0000","last_unstale":"2022-09-29T05:25:03.452852+0000","last_undegraded":"2022-09-29T05:25:03.452852+0000","last_fullsized":"2022-09-29T05:25:03.452852+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.810083+0000","last_change":"2022-09-29T05:25:03.808836+0000","last_active":"2022-09-29T05:25:03.810083+0000","last_peered":"2022-09-29T05:25:03.810083+0000","last_clean":"2022-09-29T05:25:03.810083+0000","last_became_active":"2022-09-29T05:25:03.808557+0000","last_became_peered":"2022-09-29T05:25:03.808557+0000","last_unstale":"2022-09-29T05:25:03.810083+0000","last_undegraded":"2022-09-29T05:25:03.810083+0000","last_fullsized":"2022-09-29T05:25:03.810083+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.b","version":"50'2","reported_seq":13,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.829344+0000","last_change":"2022-09-29T05:25:03.460369+0000","last_active":"2022-09-29T05:25:03.829344+0000","last_peered":"2022-09-29T05:25:03.829344+0000","last_clean":"2022-09-29T05:25:03.829344+0000","last_became_active":"2022-09-29T05:25:03.459796+0000","last_became_peered":"2022-09-29T05:25:03.459796+0000","last_unstale":"2022-09-29T05:25:03.829344+0000","last_undegraded":"2022-09-29T05:25:03.829344+0000","last_fullsized":"2022-09-29T05:25:03.829344+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.824054+0000","last_change":"2022-09-29T05:25:03.818102+0000","last_active":"2022-09-29T05:25:03.824054+0000","last_peered":"2022-09-29T05:25:03.824054+0000","last_clean":"2022-09-29T05:25:03.824054+0000","last_became_active":"2022-09-29T05:25:03.817772+0000","last_became_peered":"2022-09-29T05:25:03.817772+0000","last_unstale":"2022-09-29T05:25:03.824054+0000","last_undegraded":"2022-09-29T05:25:03.824054+0000","last_fullsized":"2022-09-29T05:25:03.824054+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.c","version":"0'0","reported_seq":11,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.793777+0000","last_change":"2022-09-29T05:25:03.460119+0000","last_active":"2022-09-29T05:25:03.793777+0000","last_peered":"2022-09-29T05:25:03.793777+0000","last_clean":"2022-09-29T05:25:03.793777+0000","last_became_active":"2022-09-29T05:25:03.459795+0000","last_became_peered":"2022-09-29T05:25:03.459795+0000","last_unstale":"2022-09-29T05:25:03.793777+0000","last_undegraded":"2022-09-29T05:25:03.793777+0000","last_fullsized":"2022-09-29T05:25:03.793777+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.831425+0000","last_change":"2022-09-29T05:25:03.828523+0000","last_active":"2022-09-29T05:25:03.831425+0000","last_peered":"2022-09-29T05:25:03.831425+0000","last_clean":"2022-09-29T05:25:03.831425+0000","last_became_active":"2022-09-29T05:25:03.827990+0000","last_became_peered":"2022-09-29T05:25:03.827990+0000","last_unstale":"2022-09-29T05:25:03.831425+0000","last_undegraded":"2022-09-29T05:25:03.831425+0000","last_fullsized":"2022-09-29T05:25:03.831425+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.d","version":"50'1","reported_seq":11,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.798141+0000","last_change":"2022-09-29T05:25:03.452648+0000","last_active":"2022-09-29T05:25:03.798141+0000","last_peered":"2022-09-29T05:25:03.798141+0000","last_clean":"2022-09-29T05:25:03.798141+0000","last_became_active":"2022-09-29T05:25:03.452242+0000","last_became_peered":"2022-09-29T05:25:03.452242+0000","last_unstale":"2022-09-29T05:25:03.798141+0000","last_undegraded":"2022-09-29T05:25:03.798141+0000","last_fullsized":"2022-09-29T05:25:03.798141+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.811131+0000","last_change":"2022-09-29T05:25:03.809640+0000","last_active":"2022-09-29T05:25:03.811131+0000","last_peered":"2022-09-29T05:25:03.811131+0000","last_clean":"2022-09-29T05:25:03.811131+0000","last_became_active":"2022-09-29T05:25:03.809298+0000","last_became_peered":"2022-09-29T05:25:03.809298+0000","last_unstale":"2022-09-29T05:25:03.811131+0000","last_undegraded":"2022-09-29T05:25:03.811131+0000","last_fullsized":"2022-09-29T05:25:03.811131+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.e","version":"50'2","reported_seq":13,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.818957+0000","last_change":"2022-09-29T05:25:03.450361+0000","last_active":"2022-09-29T05:25:03.818957+0000","last_peered":"2022-09-29T05:25:03.818957+0000","last_clean":"2022-09-29T05:25:03.818957+0000","last_became_active":"2022-09-29T05:25:03.450020+0000","last_became_peered":"2022-09-29T05:25:03.450020+0000","last_unstale":"2022-09-29T05:25:03.818957+0000","last_undegraded":"2022-09-29T05:25:03.818957+0000","last_fullsized":"2022-09-29T05:25:03.818957+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.831419+0000","last_change":"2022-09-29T05:25:03.829664+0000","last_active":"2022-09-29T05:25:03.831419+0000","last_peered":"2022-09-29T05:25:03.831419+0000","last_clean":"2022-09-29T05:25:03.831419+0000","last_became_active":"2022-09-29T05:25:03.828931+0000","last_became_peered":"2022-09-29T05:25:03.828931+0000","last_unstale":"2022-09-29T05:25:03.831419+0000","last_undegraded":"2022-09-29T05:25:03.831419+0000","last_fullsized":"2022-09-29T05:25:03.831419+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.20","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.813607+0000","last_change":"2022-09-29T05:25:03.811486+0000","last_active":"2022-09-29T05:25:03.813607+0000","last_peered":"2022-09-29T05:25:03.813607+0000","last_clean":"2022-09-29T05:25:03.813607+0000","last_became_active":"2022-09-29T05:25:03.811222+0000","last_became_peered":"2022-09-29T05:25:03.811222+0000","last_unstale":"2022-09-29T05:25:03.813607+0000","last_undegraded":"2022-09-29T05:25:03.813607+0000","last_fullsized":"2022-09-29T05:25:03.813607+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.21","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.811049+0000","last_change":"2022-09-29T05:25:03.809062+0000","last_active":"2022-09-29T05:25:03.811049+0000","last_peered":"2022-09-29T05:25:03.811049+0000","last_clean":"2022-09-29T05:25:03.811049+0000","last_became_active":"2022-09-29T05:25:03.808752+0000","last_became_peered":"2022-09-29T05:25:03.808752+0000","last_unstale":"2022-09-29T05:25:03.811049+0000","last_undegraded":"2022-09-29T05:25:03.811049+0000","last_fullsized":"2022-09-29T05:25:03.811049+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.22","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.818770+0000","last_change":"2022-09-29T05:25:03.817645+0000","last_active":"2022-09-29T05:25:03.818770+0000","last_peered":"2022-09-29T05:25:03.818770+0000","last_clean":"2022-09-29T05:25:03.818770+0000","last_became_active":"2022-09-29T05:25:03.816740+0000","last_became_peered":"2022-09-29T05:25:03.816740+0000","last_unstale":"2022-09-29T05:25:03.818770+0000","last_undegraded":"2022-09-29T05:25:03.818770+0000","last_fullsized":"2022-09-29T05:25:03.818770+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.23","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.811445+0000","last_change":"2022-09-29T05:25:03.810436+0000","last_active":"2022-09-29T05:25:03.811445+0000","last_peered":"2022-09-29T05:25:03.811445+0000","last_clean":"2022-09-29T05:25:03.811445+0000","last_became_active":"2022-09-29T05:25:03.810178+0000","last_became_peered":"2022-09-29T05:25:03.810178+0000","last_unstale":"2022-09-29T05:25:03.811445+0000","last_undegraded":"2022-09-29T05:25:03.811445+0000","last_fullsized":"2022-09-29T05:25:03.811445+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.24","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.810303+0000","last_change":"2022-09-29T05:25:03.809114+0000","last_active":"2022-09-29T05:25:03.810303+0000","last_peered":"2022-09-29T05:25:03.810303+0000","last_clean":"2022-09-29T05:25:03.810303+0000","last_became_active":"2022-09-29T05:25:03.808686+0000","last_became_peered":"2022-09-29T05:25:03.808686+0000","last_unstale":"2022-09-29T05:25:03.810303+0000","last_undegraded":"2022-09-29T05:25:03.810303+0000","last_fullsized":"2022-09-29T05:25:03.810303+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.25","version":"0'0","reported_seq":9,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.817066+0000","last_change":"2022-09-29T05:25:03.817066+0000","last_active":"2022-09-29T05:25:03.817066+0000","last_peered":"2022-09-29T05:25:03.817066+0000","last_clean":"2022-09-29T05:25:03.817066+0000","last_became_active":"2022-09-29T05:25:03.816747+0000","last_became_peered":"2022-09-29T05:25:03.816747+0000","last_unstale":"2022-09-29T05:25:03.817066+0000","last_undegraded":"2022-09-29T05:25:03.817066+0000","last_fullsized":"2022-09-29T05:25:03.817066+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.26","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.824138+0000","last_change":"2022-09-29T05:25:03.823457+0000","last_active":"2022-09-29T05:25:03.824138+0000","last_peered":"2022-09-29T05:25:03.824138+0000","last_clean":"2022-09-29T05:25:03.824138+0000","last_became_active":"2022-09-29T05:25:03.823149+0000","last_became_peered":"2022-09-29T05:25:03.823149+0000","last_unstale":"2022-09-29T05:25:03.824138+0000","last_undegraded":"2022-09-29T05:25:03.824138+0000","last_fullsized":"2022-09-29T05:25:03.824138+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,4],"acting":[6,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.27","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.810175+0000","last_change":"2022-09-29T05:25:03.808766+0000","last_active":"2022-09-29T05:25:03.810175+0000","last_peered":"2022-09-29T05:25:03.810175+0000","last_clean":"2022-09-29T05:25:03.810175+0000","last_became_active":"2022-09-29T05:25:03.808474+0000","last_became_peered":"2022-09-29T05:25:03.808474+0000","last_unstale":"2022-09-29T05:25:03.810175+0000","last_undegraded":"2022-09-29T05:25:03.810175+0000","last_fullsized":"2022-09-29T05:25:03.810175+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.28","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.816688+0000","last_change":"2022-09-29T05:25:03.815620+0000","last_active":"2022-09-29T05:25:03.816688+0000","last_peered":"2022-09-29T05:25:03.816688+0000","last_clean":"2022-09-29T05:25:03.816688+0000","last_became_active":"2022-09-29T05:25:03.814336+0000","last_became_peered":"2022-09-29T05:25:03.814336+0000","last_unstale":"2022-09-29T05:25:03.816688+0000","last_undegraded":"2022-09-29T05:25:03.816688+0000","last_fullsized":"2022-09-29T05:25:03.816688+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.29","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.810113+0000","last_change":"2022-09-29T05:25:03.808727+0000","last_active":"2022-09-29T05:25:03.810113+0000","last_peered":"2022-09-29T05:25:03.810113+0000","last_clean":"2022-09-29T05:25:03.810113+0000","last_became_active":"2022-09-29T05:25:03.808440+0000","last_became_peered":"2022-09-29T05:25:03.808440+0000","last_unstale":"2022-09-29T05:25:03.810113+0000","last_undegraded":"2022-09-29T05:25:03.810113+0000","last_fullsized":"2022-09-29T05:25:03.810113+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.2a","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.819620+0000","last_change":"2022-09-29T05:25:03.819098+0000","last_active":"2022-09-29T05:25:03.819620+0000","last_peered":"2022-09-29T05:25:03.819620+0000","last_clean":"2022-09-29T05:25:03.819620+0000","last_became_active":"2022-09-29T05:25:03.818690+0000","last_became_peered":"2022-09-29T05:25:03.818690+0000","last_unstale":"2022-09-29T05:25:03.819620+0000","last_undegraded":"2022-09-29T05:25:03.819620+0000","last_fullsized":"2022-09-29T05:25:03.819620+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2b","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.825526+0000","last_change":"2022-09-29T05:25:03.824844+0000","last_active":"2022-09-29T05:25:03.825526+0000","last_peered":"2022-09-29T05:25:03.825526+0000","last_clean":"2022-09-29T05:25:03.825526+0000","last_became_active":"2022-09-29T05:25:03.824457+0000","last_became_peered":"2022-09-29T05:25:03.824457+0000","last_unstale":"2022-09-29T05:25:03.825526+0000","last_undegraded":"2022-09-29T05:25:03.825526+0000","last_fullsized":"2022-09-29T05:25:03.825526+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2c","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-09-29T05:25:03.815650+0000","last_change":"2022-09-29T05:25:03.812644+0000","last_active":"2022-09-29T05:25:03.815650+0000","last_peered":"2022-09-29T05:25:03.815650+0000","last_clean":"2022-09-29T05:25:03.815650+0000","last_became_active":"2022-09-29T05:25:03.812339+0000","last_became_peered":"2022-09-29T05:25:03.812339+0000","last_unstale":"2022-09-29T05:25:03.815650+0000","last_undegraded":"2022-09-29T05:25:03.815650+0000","last_fullsized":"2022-09-29T05:25:03.815650+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":3,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":192,"acting":192,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":7},{"poolid":2,"num_pg":32,"stat_sum":{"num_bytes":2306,"num_objects":21,"num_object_clones":0,"num_object_copies":63,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":21,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":44,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":13,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":6918,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":38,"ondisk_log_size":38,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":45,"seq":193273528327,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5488,"kb_used_data":488,"kb_used_omap":0,"kb_used_meta":4992,"kb_avail":93739664,"statfs":{"total":95995035648,"available":95989415936,"internally_reserved":0,"allocated":499712,"data_stored":201962,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5111808},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45100000000000001}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.42899999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58099999999999996}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.40999999999999998}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.53800000000000003}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.52500000000000002}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54900000000000004}]}]},{"osd":6,"up_from":40,"seq":171798691849,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5480,"kb_used_data":480,"kb_used_omap":0,"kb_used_meta":4992,"kb_avail":93739672,"statfs":{"total":95995035648,"available":95989424128,"internally_reserved":0,"allocated":491520,"data_stored":196260,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5111808},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.38200000000000001}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45100000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.439}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.36199999999999999}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.433}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.46200000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.42399999999999999}]}]},{"osd":1,"up_from":13,"seq":55834574874,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5736,"kb_used_data":480,"kb_used_omap":0,"kb_used_meta":5248,"kb_avail":93739416,"statfs":{"total":95995035648,"available":95989161984,"internally_reserved":0,"allocated":491520,"data_stored":195199,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5373952},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Sep 29 05:24:04 2022","interfaces":[{"interface":"back","average":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"last":0.35799999999999998},{"interface":"front","average":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"last":0.27800000000000002}]},{"osd":2,"last update":"Thu Sep 29 05:24:22 2022","interfaces":[{"interface":"back","average":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"last":0.32000000000000001},{"interface":"front","average":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.374}]},{"osd":3,"last update":"Thu Sep 29 05:24:43 2022","interfaces":[{"interface":"back","average":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"last":0.34699999999999998},{"interface":"front","average":{"1min":0.442,"5min":0.442,"15min":0.442},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"last":0.30199999999999999}]},{"osd":4,"last update":"Thu Sep 29 05:25:00 2022","interfaces":[{"interface":"back","average":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":1.1619999999999999,"5min":1.1619999999999999,"15min":1.1619999999999999},"last":0.67700000000000005},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":1.1799999999999999,"5min":1.1799999999999999,"15min":1.1799999999999999},"last":0.66800000000000004}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56200000000000006}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.69799999999999995}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.61599999999999999}]}]},{"osd":0,"up_from":8,"seq":34359738398,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5820,"kb_used_data":500,"kb_used_omap":0,"kb_used_meta":5312,"kb_avail":93739332,"statfs":{"total":95995035648,"available":95989075968,"internally_reserved":0,"allocated":512000,"data_stored":202401,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5439488},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Sep 29 05:24:05 2022","interfaces":[{"interface":"back","average":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"min":{"1min":0.17699999999999999,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.503},{"interface":"front","average":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.371}]},{"osd":2,"last update":"Thu Sep 29 05:24:29 2022","interfaces":[{"interface":"back","average":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"last":0.35899999999999999},{"interface":"front","average":{"1min":0.377,"5min":0.377,"15min":0.377},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"last":0.47499999999999998}]},{"osd":3,"last update":"Thu Sep 29 05:24:41 2022","interfaces":[{"interface":"back","average":{"1min":0.437,"5min":0.437,"15min":0.437},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.434,"5min":0.434,"15min":0.434},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"last":0.39700000000000002}]},{"osd":4,"last update":"Thu Sep 29 05:24:57 2022","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.72899999999999998},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.72599999999999998}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.68700000000000006}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.66500000000000004}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.65700000000000003}]}]},{"osd":2,"up_from":18,"seq":77309411351,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5612,"kb_used_data":484,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739540,"statfs":{"total":95995035648,"available":95989288960,"internally_reserved":0,"allocated":495616,"data_stored":201701,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Sep 29 05:24:26 2022","interfaces":[{"interface":"back","average":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"last":0.23200000000000001},{"interface":"front","average":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":0.72099999999999997,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.307}]},{"osd":1,"last update":"Thu Sep 29 05:24:26 2022","interfaces":[{"interface":"back","average":{"1min":0.38,"5min":0.38,"15min":0.38},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"last":0.26800000000000002},{"interface":"front","average":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"last":0.249}]},{"osd":3,"last update":"Thu Sep 29 05:24:41 2022","interfaces":[{"interface":"back","average":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"last":0.33500000000000002},{"interface":"front","average":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.71799999999999997,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.34499999999999997}]},{"osd":4,"last update":"Thu Sep 29 05:24:56 2022","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.56399999999999995}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.59199999999999997}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54000000000000004}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51500000000000001}]}]},{"osd":3,"up_from":26,"seq":111669149715,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5632,"kb_used_data":504,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739520,"statfs":{"total":95995035648,"available":95989268480,"internally_reserved":0,"allocated":516096,"data_stored":203341,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Sep 29 05:24:44 2022","interfaces":[{"interface":"back","average":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"last":0.28199999999999997},{"interface":"front","average":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"last":0.45600000000000002}]},{"osd":1,"last update":"Thu Sep 29 05:24:44 2022","interfaces":[{"interface":"back","average":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"last":0.253},{"interface":"front","average":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"last":0.29699999999999999}]},{"osd":2,"last update":"Thu Sep 29 05:24:44 2022","interfaces":[{"interface":"back","average":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"last":0.34399999999999997},{"interface":"front","average":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"last":0.318}]},{"osd":4,"last update":"Thu Sep 29 05:25:00 2022","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.84099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.41199999999999998},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.49299999999999999}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.42199999999999999}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54700000000000004}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.504}]}]},{"osd":4,"up_from":30,"seq":128849018895,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5628,"kb_used_data":500,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739524,"statfs":{"total":95995035648,"available":95989272576,"internally_reserved":0,"allocated":512000,"data_stored":202381,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Sep 29 05:24:59 2022","interfaces":[{"interface":"back","average":{"1min":0.47799999999999998,"5min":0.47799999999999998,"15min":0.47799999999999998},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.71999999999999997,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.65000000000000002},{"interface":"front","average":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.67500000000000004,"5min":0.67500000000000004,"15min":0.67500000000000004},"last":0.59999999999999998}]},{"osd":1,"last update":"Thu Sep 29 05:24:59 2022","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.55400000000000005},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.749,"5min":0.749,"15min":0.749},"last":0.498}]},{"osd":2,"last update":"Thu Sep 29 05:24:59 2022","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.71499999999999997},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.51700000000000002,"15min":0.51700000000000002},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.57799999999999996}]},{"osd":3,"last update":"Thu Sep 29 05:24:59 2022","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.63500000000000001},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.70399999999999996}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.76400000000000001}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.748}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.73499999999999999}]}]},{"osd":5,"up_from":35,"seq":150323855374,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5560,"kb_used_data":496,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739592,"statfs":{"total":95995035648,"available":95989342208,"internally_reserved":0,"allocated":507904,"data_stored":203012,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58299999999999996}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.40400000000000003}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.501}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.433}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.47199999999999998}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.60799999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.53500000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":700,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":1640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1626,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":1536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2022-09-29T05:25:11.456 INFO:tasks.cephfs.filesystem.ceph_manager:clean! 2022-09-29T05:25:11.457 INFO:tasks.cephadm:Adding prometheus.a on smithi153 2022-09-29T05:25:11.457 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/daemon-base:latest-pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 69111522-3fb6-11ed-8431-001a4aab830c -- ceph orch apply prometheus '1;smithi153=a' 2022-09-29T05:25:13.058 INFO:teuthology.orchestra.run.smithi153.stdout:Scheduled prometheus update... 2022-09-29T05:25:13.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:11.427180+0000 mgr.y (mgr.14146) 168 : audit [DBG] from='client.14559 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2022-09-29T05:25:13.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:25:11.593349+0000 mgr.y (mgr.14146) 169 : cluster [DBG] pgmap v149: 97 pgs: 97 active+clean; 2.3 KiB data, 44 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s wr, 5 op/s 2022-09-29T05:25:13.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:11.427180+0000 mgr.y (mgr.14146) 168 : audit [DBG] from='client.14559 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2022-09-29T05:25:13.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:25:11.593349+0000 mgr.y (mgr.14146) 169 : cluster [DBG] pgmap v149: 97 pgs: 97 active+clean; 2.3 KiB data, 44 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s wr, 5 op/s 2022-09-29T05:25:13.241 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:11.427180+0000 mgr.y (mgr.14146) 168 : audit [DBG] from='client.14559 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2022-09-29T05:25:13.241 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:25:11.593349+0000 mgr.y (mgr.14146) 169 : cluster [DBG] pgmap v149: 97 pgs: 97 active+clean; 2.3 KiB data, 44 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s wr, 5 op/s 2022-09-29T05:25:13.488 DEBUG:teuthology.orchestra.run.smithi153:prometheus.a> sudo journalctl -f -n 0 -u ceph-69111522-3fb6-11ed-8431-001a4aab830c@prometheus.a.service 2022-09-29T05:25:13.491 INFO:tasks.cephadm:Adding node-exporter.a on smithi083 2022-09-29T05:25:13.491 INFO:tasks.cephadm:Adding node-exporter.b on smithi153 2022-09-29T05:25:13.492 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/daemon-base:latest-pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 69111522-3fb6-11ed-8431-001a4aab830c -- ceph orch apply node-exporter '2;smithi083=a;smithi153=b' 2022-09-29T05:25:13.522 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:-- Logs begin at Thu 2022-09-29 05:12:53 UTC. -- 2022-09-29T05:25:14.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:14 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:13.047195+0000 mgr.y (mgr.14146) 170 : audit [DBG] from='client.24446 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi153=a", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:25:14.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:14 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:25:13.048730+0000 mgr.y (mgr.14146) 171 : cephadm [INF] Saving service prometheus spec with placement smithi153=a;count:1 2022-09-29T05:25:14.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:14 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:13.054966+0000 mon.a (mon.0) 564 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:25:14.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:14 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:13.063981+0000 mon.a (mon.0) 565 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:25:14.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:14 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:13.065196+0000 mon.a (mon.0) 566 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:25:14.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:14 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:13.065939+0000 mon.a (mon.0) 567 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:25:14.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:14 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:13.500706+0000 mon.a (mon.0) 568 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:25:14.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:14 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:13.504367+0000 mon.a (mon.0) 569 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2022-09-29T05:25:14.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:14 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:25:13.593893+0000 mgr.y (mgr.14146) 172 : cluster [DBG] pgmap v150: 97 pgs: 97 active+clean; 2.3 KiB data, 44 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 4 op/s 2022-09-29T05:25:14.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:13.047195+0000 mgr.y (mgr.14146) 170 : audit [DBG] from='client.24446 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi153=a", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:25:14.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:25:13.048730+0000 mgr.y (mgr.14146) 171 : cephadm [INF] Saving service prometheus spec with placement smithi153=a;count:1 2022-09-29T05:25:14.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:13.054966+0000 mon.a (mon.0) 564 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:25:14.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:13.063981+0000 mon.a (mon.0) 565 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:25:14.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:13.065196+0000 mon.a (mon.0) 566 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:25:14.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:13.065939+0000 mon.a (mon.0) 567 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:25:14.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:13.500706+0000 mon.a (mon.0) 568 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:25:14.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:13.504367+0000 mon.a (mon.0) 569 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2022-09-29T05:25:14.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:25:13.593893+0000 mgr.y (mgr.14146) 172 : cluster [DBG] pgmap v150: 97 pgs: 97 active+clean; 2.3 KiB data, 44 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 4 op/s 2022-09-29T05:25:14.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:13.047195+0000 mgr.y (mgr.14146) 170 : audit [DBG] from='client.24446 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi153=a", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:25:14.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:25:13.048730+0000 mgr.y (mgr.14146) 171 : cephadm [INF] Saving service prometheus spec with placement smithi153=a;count:1 2022-09-29T05:25:14.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:13.054966+0000 mon.a (mon.0) 564 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:25:14.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:13.063981+0000 mon.a (mon.0) 565 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:25:14.467 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:13.065196+0000 mon.a (mon.0) 566 : audit [DBG] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:25:14.467 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:13.065939+0000 mon.a (mon.0) 567 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:25:14.467 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:13.500706+0000 mon.a (mon.0) 568 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' 2022-09-29T05:25:14.467 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:13.504367+0000 mon.a (mon.0) 569 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2022-09-29T05:25:14.468 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:25:13.593893+0000 mgr.y (mgr.14146) 172 : cluster [DBG] pgmap v150: 97 pgs: 97 active+clean; 2.3 KiB data, 44 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 4 op/s 2022-09-29T05:25:14.672 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:25:14 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[40232]: ignoring --setuser ceph since I am not root 2022-09-29T05:25:14.672 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:25:14 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[40232]: ignoring --setgroup ceph since I am not root 2022-09-29T05:25:14.673 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:25:14 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[40232]: debug 2022-09-29T05:25:14.544+0000 7fbc80b16700 1 -- 172.21.15.153:0/3424037036 <== mon.2 v2:172.21.15.153:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5590ffdea4e0 con 0x5590ffdc2c00 2022-09-29T05:25:14.673 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:25:14 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[40232]: debug 2022-09-29T05:25:14.667+0000 7fbc90ccc680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2022-09-29T05:25:14.950 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:25:14 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[40232]: debug 2022-09-29T05:25:14.755+0000 7fbc90ccc680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2022-09-29T05:25:14.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:25:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: ignoring --setuser ceph since I am not root 2022-09-29T05:25:14.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:25:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: ignoring --setgroup ceph since I am not root 2022-09-29T05:25:14.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:25:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:25:14.552+0000 7ffa72837700 1 -- 172.21.15.83:0/399582339 <== mon.0 v2:172.21.15.83:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5627b67044e0 con 0x5627b66dd400 2022-09-29T05:25:14.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:25:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:25:14.672+0000 7ffa829ed680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2022-09-29T05:25:14.964 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:25:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:25:14.752+0000 7ffa829ed680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2022-09-29T05:25:15.463 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:25:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:25:15.104+0000 7ffa829ed680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2022-09-29T05:25:15.512 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:25:15 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[40232]: debug 2022-09-29T05:25:15.141+0000 7fbc90ccc680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2022-09-29T05:25:15.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:15 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:14.513490+0000 mon.a (mon.0) 570 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2022-09-29T05:25:15.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:15 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:25:14.513589+0000 mon.a (mon.0) 571 : cluster [DBG] mgrmap e15: y(active, since 3m), standbys: x 2022-09-29T05:25:15.835 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:14.513490+0000 mon.a (mon.0) 570 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2022-09-29T05:25:15.835 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:25:14.513589+0000 mon.a (mon.0) 571 : cluster [DBG] mgrmap e15: y(active, since 3m), standbys: x 2022-09-29T05:25:15.836 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:14.513490+0000 mon.a (mon.0) 570 : audit [INF] from='mgr.14146 172.21.15.83:0/2484586199' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2022-09-29T05:25:15.836 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:25:14.513589+0000 mon.a (mon.0) 571 : cluster [DBG] mgrmap e15: y(active, since 3m), standbys: x 2022-09-29T05:25:16.131 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:25:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:25:15.831+0000 7ffa829ed680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2022-09-29T05:25:16.131 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:25:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:25:15.915+0000 7ffa829ed680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2022-09-29T05:25:16.174 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:25:15 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[40232]: debug 2022-09-29T05:25:15.875+0000 7fbc90ccc680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2022-09-29T05:25:16.175 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:25:15 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[40232]: debug 2022-09-29T05:25:15.960+0000 7fbc90ccc680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2022-09-29T05:25:16.462 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:25:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:25:16.127+0000 7ffa829ed680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2022-09-29T05:25:16.559 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:25:16 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[40232]: debug 2022-09-29T05:25:16.170+0000 7fbc90ccc680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2022-09-29T05:25:17.346 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:25:17 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:25:17.049+0000 7ffa829ed680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2022-09-29T05:25:17.347 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:25:17 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:25:17.174+0000 7ffa829ed680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2022-09-29T05:25:17.347 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:25:17 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:25:17.253+0000 7ffa829ed680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2022-09-29T05:25:17.405 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:25:17 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[40232]: debug 2022-09-29T05:25:17.105+0000 7fbc90ccc680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2022-09-29T05:25:17.405 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:25:17 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[40232]: debug 2022-09-29T05:25:17.231+0000 7fbc90ccc680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2022-09-29T05:25:17.405 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:25:17 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[40232]: debug 2022-09-29T05:25:17.310+0000 7fbc90ccc680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2022-09-29T05:25:17.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:25:17 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:25:17.342+0000 7ffa829ed680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2022-09-29T05:25:17.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:25:17 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:25:17.535+0000 7ffa829ed680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2022-09-29T05:25:17.809 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:25:17 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[40232]: debug 2022-09-29T05:25:17.400+0000 7fbc90ccc680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2022-09-29T05:25:17.809 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:25:17 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[40232]: debug 2022-09-29T05:25:17.597+0000 7fbc90ccc680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2022-09-29T05:25:18.213 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:25:17 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:25:17.815+0000 7ffa829ed680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2022-09-29T05:25:18.213 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:25:17 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:25:17.914+0000 7ffa829ed680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2022-09-29T05:25:18.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:25:17 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[40232]: debug 2022-09-29T05:25:17.878+0000 7fbc90ccc680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2022-09-29T05:25:18.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:25:17 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[40232]: debug 2022-09-29T05:25:17.976+0000 7fbc90ccc680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2022-09-29T05:25:18.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:25:18 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:25:18.620+0000 7ffa829ed680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2022-09-29T05:25:18.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:25:18 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:25:18.703+0000 7ffa829ed680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2022-09-29T05:25:18.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:25:18 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:25:18.794+0000 7ffa829ed680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2022-09-29T05:25:19.051 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:25:18 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[40232]: debug 2022-09-29T05:25:18.698+0000 7fbc90ccc680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2022-09-29T05:25:19.051 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:25:18 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[40232]: debug 2022-09-29T05:25:18.781+0000 7fbc90ccc680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2022-09-29T05:25:19.052 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:25:18 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[40232]: debug 2022-09-29T05:25:18.872+0000 7fbc90ccc680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2022-09-29T05:25:19.228 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:25:18 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:25:18.969+0000 7ffa829ed680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2022-09-29T05:25:19.489 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:25:19 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:25:19.225+0000 7ffa829ed680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2022-09-29T05:25:19.490 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:25:19 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:25:19.353+0000 7ffa829ed680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2022-09-29T05:25:19.577 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:25:19 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[40232]: debug 2022-09-29T05:25:19.047+0000 7fbc90ccc680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2022-09-29T05:25:19.577 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:25:19 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[40232]: debug 2022-09-29T05:25:19.308+0000 7fbc90ccc680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2022-09-29T05:25:19.577 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:25:19 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[40232]: debug 2022-09-29T05:25:19.439+0000 7fbc90ccc680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2022-09-29T05:25:19.807 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:25:19 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:25:19.486+0000 7ffa829ed680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2022-09-29T05:25:19.808 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:25:19 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:25:19.650+0000 7ffa829ed680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2022-09-29T05:25:19.808 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:25:19 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:25:19.733+0000 7ffa829ed680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2022-09-29T05:25:19.808 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:19 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:25:19.746923+0000 mon.a (mon.0) 572 : cluster [INF] Active manager daemon y restarted 2022-09-29T05:25:19.809 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:19 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:25:19.747439+0000 mon.a (mon.0) 573 : cluster [INF] Activating manager daemon y 2022-09-29T05:25:19.809 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:19 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:25:19.750587+0000 mon.a (mon.0) 574 : cluster [DBG] osdmap e52: 8 total, 8 up, 8 in 2022-09-29T05:25:19.832 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:25:19 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[40232]: debug 2022-09-29T05:25:19.572+0000 7fbc90ccc680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2022-09-29T05:25:19.832 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:25:19 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[40232]: debug 2022-09-29T05:25:19.739+0000 7fbc90ccc680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2022-09-29T05:25:19.833 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:19 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:25:19.746923+0000 mon.a (mon.0) 572 : cluster [INF] Active manager daemon y restarted 2022-09-29T05:25:19.833 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:19 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:25:19.747439+0000 mon.a (mon.0) 573 : cluster [INF] Activating manager daemon y 2022-09-29T05:25:19.833 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:19 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:25:19.750587+0000 mon.a (mon.0) 574 : cluster [DBG] osdmap e52: 8 total, 8 up, 8 in 2022-09-29T05:25:20.124 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:19 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:25:19.746923+0000 mon.a (mon.0) 572 : cluster [INF] Active manager daemon y restarted 2022-09-29T05:25:20.125 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:19 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:25:19.747439+0000 mon.a (mon.0) 573 : cluster [INF] Activating manager daemon y 2022-09-29T05:25:20.125 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:19 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:25:19.750587+0000 mon.a (mon.0) 574 : cluster [DBG] osdmap e52: 8 total, 8 up, 8 in 2022-09-29T05:25:20.125 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:25:19 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: [29/Sep/2022:05:25:19] ENGINE Bus STARTING 2022-09-29T05:25:20.125 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:25:19 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: [80B blob data] 2022-09-29T05:25:20.126 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:25:19 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: [79B blob data] 2022-09-29T05:25:20.126 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:25:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:25:20.000+0000 7ff9ec176700 -1 client.0 error registering admin socket command: (17) File exists 2022-09-29T05:25:20.126 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:25:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:25:20.000+0000 7ff9ec176700 -1 client.0 error registering admin socket command: (17) File exists 2022-09-29T05:25:20.126 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:25:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:25:20.000+0000 7ff9ec176700 -1 client.0 error registering admin socket command: (17) File exists 2022-09-29T05:25:20.127 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:25:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:25:20.000+0000 7ff9ec176700 -1 client.0 error registering admin socket command: (17) File exists 2022-09-29T05:25:20.127 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:25:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:25:20.000+0000 7ff9ec176700 -1 client.0 error registering admin socket command: (17) File exists 2022-09-29T05:25:20.127 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:25:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:25:20.001+0000 7ff9e9170700 -1 client.0 error registering admin socket command: (17) File exists 2022-09-29T05:25:20.127 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:25:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:25:20.001+0000 7ff9e9170700 -1 client.0 error registering admin socket command: (17) File exists 2022-09-29T05:25:20.128 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:25:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:25:20.001+0000 7ff9e9170700 -1 client.0 error registering admin socket command: (17) File exists 2022-09-29T05:25:20.128 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:25:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:25:20.001+0000 7ff9e9170700 -1 client.0 error registering admin socket command: (17) File exists 2022-09-29T05:25:20.128 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:25:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:25:20.001+0000 7ff9e9170700 -1 client.0 error registering admin socket command: (17) File exists 2022-09-29T05:25:20.128 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:25:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: CherryPy Checker: 2022-09-29T05:25:20.129 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:25:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: The Application mounted at '' has an empty config. 2022-09-29T05:25:20.129 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:25:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: 2022-09-29T05:25:20.129 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:25:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: [29/Sep/2022:05:25:20] ENGINE Serving on http://:::9283 2022-09-29T05:25:20.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:25:19 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[40232]: debug 2022-09-29T05:25:19.828+0000 7fbc90ccc680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2022-09-29T05:25:20.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:25:19 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[40232]: [29/Sep/2022:05:25:19] ENGINE Bus STARTING 2022-09-29T05:25:20.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:25:19 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[40232]: CherryPy Checker: 2022-09-29T05:25:20.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:25:19 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[40232]: The Application mounted at '' has an empty config. 2022-09-29T05:25:20.310 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:25:19 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[40232]: 2022-09-29T05:25:20.310 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:25:19 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[40232]: [29/Sep/2022:05:25:19] ENGINE Serving on http://:::9283 2022-09-29T05:25:20.310 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:25:19 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[40232]: [29/Sep/2022:05:25:19] ENGINE Bus STARTED 2022-09-29T05:25:20.463 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:25:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: [29/Sep/2022:05:25:20] ENGINE Bus STARTED 2022-09-29T05:25:20.845 INFO:teuthology.orchestra.run.smithi153.stdout:Scheduled node-exporter update... 2022-09-29T05:25:21.090 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:25:19.803466+0000 mon.a (mon.0) 575 : cluster [DBG] mgrmap e16: y(active, starting, since 0.0561336s), standbys: x 2022-09-29T05:25:21.091 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:19.804749+0000 mon.a (mon.0) 576 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2022-09-29T05:25:21.091 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:19.807924+0000 mon.a (mon.0) 577 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2022-09-29T05:25:21.091 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:19.808120+0000 mon.a (mon.0) 578 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2022-09-29T05:25:21.092 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:19.808281+0000 mon.a (mon.0) 579 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2022-09-29T05:25:21.092 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:19.808495+0000 mon.a (mon.0) 580 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2022-09-29T05:25:21.092 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:19.808733+0000 mon.a (mon.0) 581 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2022-09-29T05:25:21.092 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:19.808986+0000 mon.a (mon.0) 582 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2022-09-29T05:25:21.093 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:19.809264+0000 mon.a (mon.0) 583 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2022-09-29T05:25:21.093 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:19.809493+0000 mon.a (mon.0) 584 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2022-09-29T05:25:21.093 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:19.809643+0000 mon.a (mon.0) 585 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2022-09-29T05:25:21.093 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:19.809858+0000 mon.a (mon.0) 586 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2022-09-29T05:25:21.094 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:19.810039+0000 mon.a (mon.0) 587 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2022-09-29T05:25:21.094 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:19.810247+0000 mon.a (mon.0) 588 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2022-09-29T05:25:21.094 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:19.810510+0000 mon.a (mon.0) 589 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2022-09-29T05:25:21.094 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:19.810741+0000 mon.a (mon.0) 590 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2022-09-29T05:25:21.094 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:19.810981+0000 mon.a (mon.0) 591 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2022-09-29T05:25:21.095 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:19.811272+0000 mon.a (mon.0) 592 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2022-09-29T05:25:21.095 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:19.811725+0000 mon.a (mon.0) 593 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2022-09-29T05:25:21.095 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:25:19.825518+0000 mon.a (mon.0) 594 : cluster [INF] Manager daemon y is now available 2022-09-29T05:25:21.095 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:25:19.843170+0000 mon.a (mon.0) 595 : cluster [DBG] Standby manager daemon x restarted 2022-09-29T05:25:21.096 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:25:19.843259+0000 mon.a (mon.0) 596 : cluster [DBG] Standby manager daemon x started 2022-09-29T05:25:21.096 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:19.844782+0000 mon.b (mon.2) 22 : audit [DBG] from='mgr.? 172.21.15.153:0/1513592501' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2022-09-29T05:25:21.096 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:19.845127+0000 mon.b (mon.2) 23 : audit [DBG] from='mgr.? 172.21.15.153:0/1513592501' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2022-09-29T05:25:21.096 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:19.847234+0000 mon.b (mon.2) 24 : audit [DBG] from='mgr.? 172.21.15.153:0/1513592501' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2022-09-29T05:25:21.097 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:19.848247+0000 mon.b (mon.2) 25 : audit [DBG] from='mgr.? 172.21.15.153:0/1513592501' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2022-09-29T05:25:21.097 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:19.924049+0000 mon.a (mon.0) 597 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:25:21.097 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:19.925241+0000 mon.a (mon.0) 598 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:25:21.097 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:19.927601+0000 mon.a (mon.0) 599 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:25:21.098 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:19.929284+0000 mon.a (mon.0) 600 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:25:21.098 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:19.946473+0000 mon.a (mon.0) 601 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:25:21.098 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:19.965838+0000 mon.a (mon.0) 602 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:25:21.098 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:19.972634+0000 mon.a (mon.0) 603 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:25:21.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:25:19.803466+0000 mon.a (mon.0) 575 : cluster [DBG] mgrmap e16: y(active, starting, since 0.0561336s), standbys: x 2022-09-29T05:25:21.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:19.804749+0000 mon.a (mon.0) 576 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2022-09-29T05:25:21.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:19.807924+0000 mon.a (mon.0) 577 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2022-09-29T05:25:21.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:19.808120+0000 mon.a (mon.0) 578 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2022-09-29T05:25:21.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:19.808281+0000 mon.a (mon.0) 579 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2022-09-29T05:25:21.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:19.808495+0000 mon.a (mon.0) 580 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2022-09-29T05:25:21.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:19.808733+0000 mon.a (mon.0) 581 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2022-09-29T05:25:21.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:19.808986+0000 mon.a (mon.0) 582 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2022-09-29T05:25:21.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:19.809264+0000 mon.a (mon.0) 583 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2022-09-29T05:25:21.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:19.809493+0000 mon.a (mon.0) 584 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2022-09-29T05:25:21.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:19.809643+0000 mon.a (mon.0) 585 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2022-09-29T05:25:21.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:19.809858+0000 mon.a (mon.0) 586 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2022-09-29T05:25:21.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:19.810039+0000 mon.a (mon.0) 587 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2022-09-29T05:25:21.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:19.810247+0000 mon.a (mon.0) 588 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2022-09-29T05:25:21.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:19.810510+0000 mon.a (mon.0) 589 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2022-09-29T05:25:21.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:25:19.803466+0000 mon.a (mon.0) 575 : cluster [DBG] mgrmap e16: y(active, starting, since 0.0561336s), standbys: x 2022-09-29T05:25:21.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:19.804749+0000 mon.a (mon.0) 576 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2022-09-29T05:25:21.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:19.807924+0000 mon.a (mon.0) 577 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2022-09-29T05:25:21.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:19.808120+0000 mon.a (mon.0) 578 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2022-09-29T05:25:21.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:19.808281+0000 mon.a (mon.0) 579 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2022-09-29T05:25:21.218 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:19.808495+0000 mon.a (mon.0) 580 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2022-09-29T05:25:21.218 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:19.808733+0000 mon.a (mon.0) 581 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2022-09-29T05:25:21.218 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:19.808986+0000 mon.a (mon.0) 582 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2022-09-29T05:25:21.218 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:19.809264+0000 mon.a (mon.0) 583 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2022-09-29T05:25:21.219 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:19.809493+0000 mon.a (mon.0) 584 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2022-09-29T05:25:21.219 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:19.809643+0000 mon.a (mon.0) 585 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2022-09-29T05:25:21.219 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:19.809858+0000 mon.a (mon.0) 586 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2022-09-29T05:25:21.219 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:19.810039+0000 mon.a (mon.0) 587 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2022-09-29T05:25:21.219 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:19.810247+0000 mon.a (mon.0) 588 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2022-09-29T05:25:21.220 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:19.810510+0000 mon.a (mon.0) 589 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2022-09-29T05:25:21.220 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:19.810741+0000 mon.a (mon.0) 590 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2022-09-29T05:25:21.220 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:19.810981+0000 mon.a (mon.0) 591 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2022-09-29T05:25:21.221 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:19.811272+0000 mon.a (mon.0) 592 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2022-09-29T05:25:21.221 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:19.811725+0000 mon.a (mon.0) 593 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2022-09-29T05:25:21.221 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:25:19.825518+0000 mon.a (mon.0) 594 : cluster [INF] Manager daemon y is now available 2022-09-29T05:25:21.222 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:25:19.843170+0000 mon.a (mon.0) 595 : cluster [DBG] Standby manager daemon x restarted 2022-09-29T05:25:21.222 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:25:19.843259+0000 mon.a (mon.0) 596 : cluster [DBG] Standby manager daemon x started 2022-09-29T05:25:21.222 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:19.844782+0000 mon.b (mon.2) 22 : audit [DBG] from='mgr.? 172.21.15.153:0/1513592501' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2022-09-29T05:25:21.222 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:19.845127+0000 mon.b (mon.2) 23 : audit [DBG] from='mgr.? 172.21.15.153:0/1513592501' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2022-09-29T05:25:21.222 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:19.847234+0000 mon.b (mon.2) 24 : audit [DBG] from='mgr.? 172.21.15.153:0/1513592501' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2022-09-29T05:25:21.223 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:19.848247+0000 mon.b (mon.2) 25 : audit [DBG] from='mgr.? 172.21.15.153:0/1513592501' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2022-09-29T05:25:21.223 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:19.924049+0000 mon.a (mon.0) 597 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:25:21.223 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:19.925241+0000 mon.a (mon.0) 598 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:25:21.223 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:19.927601+0000 mon.a (mon.0) 599 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:25:21.223 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:19.929284+0000 mon.a (mon.0) 600 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:25:21.224 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:19.946473+0000 mon.a (mon.0) 601 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:25:21.224 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:19.965838+0000 mon.a (mon.0) 602 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:25:21.224 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:19.972634+0000 mon.a (mon.0) 603 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:25:21.225 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:19.810741+0000 mon.a (mon.0) 590 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2022-09-29T05:25:21.225 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:19.810981+0000 mon.a (mon.0) 591 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2022-09-29T05:25:21.225 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:19.811272+0000 mon.a (mon.0) 592 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2022-09-29T05:25:21.225 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:19.811725+0000 mon.a (mon.0) 593 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2022-09-29T05:25:21.226 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:25:19.825518+0000 mon.a (mon.0) 594 : cluster [INF] Manager daemon y is now available 2022-09-29T05:25:21.226 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:25:19.843170+0000 mon.a (mon.0) 595 : cluster [DBG] Standby manager daemon x restarted 2022-09-29T05:25:21.226 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:25:19.843259+0000 mon.a (mon.0) 596 : cluster [DBG] Standby manager daemon x started 2022-09-29T05:25:21.226 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:19.844782+0000 mon.b (mon.2) 22 : audit [DBG] from='mgr.? 172.21.15.153:0/1513592501' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2022-09-29T05:25:21.226 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:19.845127+0000 mon.b (mon.2) 23 : audit [DBG] from='mgr.? 172.21.15.153:0/1513592501' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2022-09-29T05:25:21.227 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:19.847234+0000 mon.b (mon.2) 24 : audit [DBG] from='mgr.? 172.21.15.153:0/1513592501' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2022-09-29T05:25:21.227 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:19.848247+0000 mon.b (mon.2) 25 : audit [DBG] from='mgr.? 172.21.15.153:0/1513592501' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2022-09-29T05:25:21.227 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:19.924049+0000 mon.a (mon.0) 597 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:25:21.227 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:19.925241+0000 mon.a (mon.0) 598 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:25:21.228 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:19.927601+0000 mon.a (mon.0) 599 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:25:21.228 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:19.929284+0000 mon.a (mon.0) 600 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:25:21.228 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:19.946473+0000 mon.a (mon.0) 601 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:25:21.228 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:19.965838+0000 mon.a (mon.0) 602 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:25:21.228 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:19.972634+0000 mon.a (mon.0) 603 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:25:21.363 DEBUG:teuthology.orchestra.run.smithi083:node-exporter.a> sudo journalctl -f -n 0 -u ceph-69111522-3fb6-11ed-8431-001a4aab830c@node-exporter.a.service 2022-09-29T05:25:21.366 DEBUG:teuthology.orchestra.run.smithi153:node-exporter.b> sudo journalctl -f -n 0 -u ceph-69111522-3fb6-11ed-8431-001a4aab830c@node-exporter.b.service 2022-09-29T05:25:21.369 INFO:tasks.cephadm:Adding alertmanager.a on smithi083 2022-09-29T05:25:21.369 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/daemon-base:latest-pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 69111522-3fb6-11ed-8431-001a4aab830c -- ceph orch apply alertmanager '1;smithi083=a' 2022-09-29T05:25:21.398 INFO:journalctl@ceph.node-exporter.a.smithi083.stdout:-- Logs begin at Thu 2022-09-29 05:12:45 UTC. -- 2022-09-29T05:25:21.399 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:-- Logs begin at Thu 2022-09-29 05:12:53 UTC. -- 2022-09-29T05:25:22.198 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:25:20.825603+0000 mon.a (mon.0) 604 : cluster [DBG] mgrmap e17: y(active, since 1.07827s), standbys: x 2022-09-29T05:25:22.198 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:25:20.838316+0000 mgr.y (mgr.14577) 3 : cephadm [INF] Saving service node-exporter spec with placement smithi083=a;smithi153=b;count:2 2022-09-29T05:25:22.198 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:20.841215+0000 mon.a (mon.0) 605 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:25:22.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:25:20.825603+0000 mon.a (mon.0) 604 : cluster [DBG] mgrmap e17: y(active, since 1.07827s), standbys: x 2022-09-29T05:25:22.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:25:20.838316+0000 mgr.y (mgr.14577) 3 : cephadm [INF] Saving service node-exporter spec with placement smithi083=a;smithi153=b;count:2 2022-09-29T05:25:22.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:20.841215+0000 mon.a (mon.0) 605 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:25:22.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:25:20.825603+0000 mon.a (mon.0) 604 : cluster [DBG] mgrmap e17: y(active, since 1.07827s), standbys: x 2022-09-29T05:25:22.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:25:20.838316+0000 mgr.y (mgr.14577) 3 : cephadm [INF] Saving service node-exporter spec with placement smithi083=a;smithi153=b;count:2 2022-09-29T05:25:22.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:20.841215+0000 mon.a (mon.0) 605 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:25:22.988 INFO:teuthology.orchestra.run.smithi153.stdout:Scheduled alertmanager update... 2022-09-29T05:25:23.012 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:25:21.807422+0000 mgr.y (mgr.14577) 4 : cluster [DBG] pgmap v4: 97 pgs: 97 active+clean; 3.0 KiB data, 44 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:25:23.012 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:25:21.845059+0000 mon.a (mon.0) 606 : cluster [DBG] mgrmap e18: y(active, since 2s), standbys: x 2022-09-29T05:25:23.012 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:22.227391+0000 mon.a (mon.0) 607 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:25:23.012 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:22.463058+0000 mon.a (mon.0) 608 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:25:23.013 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:22.597384+0000 mon.a (mon.0) 609 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:25:23.013 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:22.598039+0000 mon.a (mon.0) 610 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:25:23.013 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:22.598591+0000 mon.a (mon.0) 611 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:25:23.013 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:22.599121+0000 mon.a (mon.0) 612 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:25:23.014 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:22.603485+0000 mon.a (mon.0) 613 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:25:23.014 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:22.653323+0000 mon.a (mon.0) 614 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:25:23.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:25:21.807422+0000 mgr.y (mgr.14577) 4 : cluster [DBG] pgmap v4: 97 pgs: 97 active+clean; 3.0 KiB data, 44 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:25:23.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:25:21.845059+0000 mon.a (mon.0) 606 : cluster [DBG] mgrmap e18: y(active, since 2s), standbys: x 2022-09-29T05:25:23.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:22.227391+0000 mon.a (mon.0) 607 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:25:23.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:22.463058+0000 mon.a (mon.0) 608 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:25:23.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:22.597384+0000 mon.a (mon.0) 609 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:25:23.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:22.598039+0000 mon.a (mon.0) 610 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:25:23.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:22.598591+0000 mon.a (mon.0) 611 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:25:23.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:22.599121+0000 mon.a (mon.0) 612 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:25:23.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:22.603485+0000 mon.a (mon.0) 613 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:25:23.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:22.653323+0000 mon.a (mon.0) 614 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:25:23.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:25:21.807422+0000 mgr.y (mgr.14577) 4 : cluster [DBG] pgmap v4: 97 pgs: 97 active+clean; 3.0 KiB data, 44 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:25:23.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:25:21.845059+0000 mon.a (mon.0) 606 : cluster [DBG] mgrmap e18: y(active, since 2s), standbys: x 2022-09-29T05:25:23.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:22.227391+0000 mon.a (mon.0) 607 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:25:23.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:22.463058+0000 mon.a (mon.0) 608 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:25:23.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:22.597384+0000 mon.a (mon.0) 609 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:25:23.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:22.598039+0000 mon.a (mon.0) 610 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:25:23.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:22.598591+0000 mon.a (mon.0) 611 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:25:23.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:22.599121+0000 mon.a (mon.0) 612 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:25:23.218 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:22.603485+0000 mon.a (mon.0) 613 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:25:23.218 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:22.653323+0000 mon.a (mon.0) 614 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:25:23.426 DEBUG:teuthology.orchestra.run.smithi083:alertmanager.a> sudo journalctl -f -n 0 -u ceph-69111522-3fb6-11ed-8431-001a4aab830c@alertmanager.a.service 2022-09-29T05:25:23.429 INFO:tasks.cephadm:Adding grafana.a on smithi153 2022-09-29T05:25:23.429 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/daemon-base:latest-pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 69111522-3fb6-11ed-8431-001a4aab830c -- ceph orch apply grafana '1;smithi153=a' 2022-09-29T05:25:23.458 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:-- Logs begin at Thu 2022-09-29 05:12:45 UTC. -- 2022-09-29T05:25:24.153 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:25:22.599547+0000 mgr.y (mgr.14577) 5 : cephadm [INF] Adjusting osd_memory_target on smithi083 to 3012M 2022-09-29T05:25:24.154 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:25:22.604004+0000 mgr.y (mgr.14577) 6 : cephadm [INF] Updating smithi083:/etc/ceph/ceph.conf 2022-09-29T05:25:24.154 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:25:22.623814+0000 mgr.y (mgr.14577) 7 : cephadm [INF] Updating smithi083:/etc/ceph/ceph.client.admin.keyring 2022-09-29T05:25:24.154 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:22.861732+0000 mon.a (mon.0) 615 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:25:24.155 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:22.862443+0000 mon.a (mon.0) 616 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:25:24.155 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:22.863068+0000 mon.a (mon.0) 617 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:25:24.155 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:22.863667+0000 mon.a (mon.0) 618 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:25:24.155 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:25:22.864142+0000 mgr.y (mgr.14577) 8 : cephadm [INF] Adjusting osd_memory_target on smithi153 to 3268M 2022-09-29T05:25:24.156 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:22.867642+0000 mon.a (mon.0) 619 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:25:24.156 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:25:22.868176+0000 mgr.y (mgr.14577) 9 : cephadm [INF] Updating smithi153:/etc/ceph/ceph.conf 2022-09-29T05:25:24.156 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:25:22.884755+0000 mgr.y (mgr.14577) 10 : cephadm [INF] Updating smithi153:/etc/ceph/ceph.client.admin.keyring 2022-09-29T05:25:24.156 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:22.916240+0000 mon.a (mon.0) 620 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:25:24.157 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:22.919508+0000 mon.a (mon.0) 621 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:25:24.157 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:25:22.932904+0000 mgr.y (mgr.14577) 11 : cephadm [INF] Deploying daemon prometheus.a on smithi153 2022-09-29T05:25:24.157 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:22.979662+0000 mgr.y (mgr.14577) 12 : audit [DBG] from='client.24494 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi083=a", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:25:24.157 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:25:22.980691+0000 mgr.y (mgr.14577) 13 : cephadm [INF] Saving service alertmanager spec with placement smithi083=a;count:1 2022-09-29T05:25:24.158 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:22.984114+0000 mon.a (mon.0) 622 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:25:24.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:25:22.599547+0000 mgr.y (mgr.14577) 5 : cephadm [INF] Adjusting osd_memory_target on smithi083 to 3012M 2022-09-29T05:25:24.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:25:22.604004+0000 mgr.y (mgr.14577) 6 : cephadm [INF] Updating smithi083:/etc/ceph/ceph.conf 2022-09-29T05:25:24.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:25:22.623814+0000 mgr.y (mgr.14577) 7 : cephadm [INF] Updating smithi083:/etc/ceph/ceph.client.admin.keyring 2022-09-29T05:25:24.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:22.861732+0000 mon.a (mon.0) 615 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:25:24.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:22.862443+0000 mon.a (mon.0) 616 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:25:24.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:22.863068+0000 mon.a (mon.0) 617 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:25:24.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:22.863667+0000 mon.a (mon.0) 618 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:25:24.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:25:22.864142+0000 mgr.y (mgr.14577) 8 : cephadm [INF] Adjusting osd_memory_target on smithi153 to 3268M 2022-09-29T05:25:24.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:22.867642+0000 mon.a (mon.0) 619 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:25:24.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:25:22.868176+0000 mgr.y (mgr.14577) 9 : cephadm [INF] Updating smithi153:/etc/ceph/ceph.conf 2022-09-29T05:25:24.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:25:22.884755+0000 mgr.y (mgr.14577) 10 : cephadm [INF] Updating smithi153:/etc/ceph/ceph.client.admin.keyring 2022-09-29T05:25:24.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:22.916240+0000 mon.a (mon.0) 620 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:25:24.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:22.919508+0000 mon.a (mon.0) 621 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:25:24.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:25:22.932904+0000 mgr.y (mgr.14577) 11 : cephadm [INF] Deploying daemon prometheus.a on smithi153 2022-09-29T05:25:24.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:22.979662+0000 mgr.y (mgr.14577) 12 : audit [DBG] from='client.24494 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi083=a", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:25:24.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:25:22.980691+0000 mgr.y (mgr.14577) 13 : cephadm [INF] Saving service alertmanager spec with placement smithi083=a;count:1 2022-09-29T05:25:24.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:22.984114+0000 mon.a (mon.0) 622 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:25:24.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:25:22.599547+0000 mgr.y (mgr.14577) 5 : cephadm [INF] Adjusting osd_memory_target on smithi083 to 3012M 2022-09-29T05:25:24.218 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:25:22.604004+0000 mgr.y (mgr.14577) 6 : cephadm [INF] Updating smithi083:/etc/ceph/ceph.conf 2022-09-29T05:25:24.218 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:25:22.623814+0000 mgr.y (mgr.14577) 7 : cephadm [INF] Updating smithi083:/etc/ceph/ceph.client.admin.keyring 2022-09-29T05:25:24.218 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:22.861732+0000 mon.a (mon.0) 615 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:25:24.218 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:22.862443+0000 mon.a (mon.0) 616 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:25:24.218 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:22.863068+0000 mon.a (mon.0) 617 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:25:24.219 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:22.863667+0000 mon.a (mon.0) 618 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:25:24.219 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:25:22.864142+0000 mgr.y (mgr.14577) 8 : cephadm [INF] Adjusting osd_memory_target on smithi153 to 3268M 2022-09-29T05:25:24.219 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:22.867642+0000 mon.a (mon.0) 619 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:25:24.219 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:25:22.868176+0000 mgr.y (mgr.14577) 9 : cephadm [INF] Updating smithi153:/etc/ceph/ceph.conf 2022-09-29T05:25:24.220 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:25:22.884755+0000 mgr.y (mgr.14577) 10 : cephadm [INF] Updating smithi153:/etc/ceph/ceph.client.admin.keyring 2022-09-29T05:25:24.220 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:22.916240+0000 mon.a (mon.0) 620 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:25:24.220 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:22.919508+0000 mon.a (mon.0) 621 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:25:24.220 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:25:22.932904+0000 mgr.y (mgr.14577) 11 : cephadm [INF] Deploying daemon prometheus.a on smithi153 2022-09-29T05:25:24.220 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:22.979662+0000 mgr.y (mgr.14577) 12 : audit [DBG] from='client.24494 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi083=a", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:25:24.221 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:25:22.980691+0000 mgr.y (mgr.14577) 13 : cephadm [INF] Saving service alertmanager spec with placement smithi083=a;count:1 2022-09-29T05:25:24.221 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:22.984114+0000 mon.a (mon.0) 622 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:25:25.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:24 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:25:23.807805+0000 mgr.y (mgr.14577) 14 : cluster [DBG] pgmap v5: 97 pgs: 97 active+clean; 3.0 KiB data, 44 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:25:25.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:25:23.807805+0000 mgr.y (mgr.14577) 14 : cluster [DBG] pgmap v5: 97 pgs: 97 active+clean; 3.0 KiB data, 44 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:25:25.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:25:23.807805+0000 mgr.y (mgr.14577) 14 : cluster [DBG] pgmap v5: 97 pgs: 97 active+clean; 3.0 KiB data, 44 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:25:26.356 INFO:teuthology.orchestra.run.smithi153.stdout:Scheduled grafana update... 2022-09-29T05:25:27.306 DEBUG:teuthology.orchestra.run.smithi153:grafana.a> sudo journalctl -f -n 0 -u ceph-69111522-3fb6-11ed-8431-001a4aab830c@grafana.a.service 2022-09-29T05:25:27.309 INFO:tasks.cephadm:Setting up client nodes... 2022-09-29T05:25:27.310 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/daemon-base:latest-pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 69111522-3fb6-11ed-8431-001a4aab830c -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2022-09-29T05:25:27.342 INFO:journalctl@ceph.grafana.a.smithi153.stdout:-- Logs begin at Thu 2022-09-29 05:12:53 UTC. -- 2022-09-29T05:25:27.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:27 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:25:25.808344+0000 mgr.y (mgr.14577) 15 : cluster [DBG] pgmap v6: 97 pgs: 97 active+clean; 3.0 KiB data, 44 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:25:27.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:27 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:26.343367+0000 mgr.y (mgr.14577) 16 : audit [DBG] from='client.24500 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi153=a", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:25:27.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:27 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:25:26.344806+0000 mgr.y (mgr.14577) 17 : cephadm [INF] Saving service grafana spec with placement smithi153=a;count:1 2022-09-29T05:25:27.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:27 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:26.352073+0000 mon.a (mon.0) 623 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:25:27.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:27 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:25:25.808344+0000 mgr.y (mgr.14577) 15 : cluster [DBG] pgmap v6: 97 pgs: 97 active+clean; 3.0 KiB data, 44 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:25:27.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:27 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:26.343367+0000 mgr.y (mgr.14577) 16 : audit [DBG] from='client.24500 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi153=a", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:25:27.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:27 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:25:26.344806+0000 mgr.y (mgr.14577) 17 : cephadm [INF] Saving service grafana spec with placement smithi153=a;count:1 2022-09-29T05:25:27.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:27 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:26.352073+0000 mon.a (mon.0) 623 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:25:27.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:27 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:25:25.808344+0000 mgr.y (mgr.14577) 15 : cluster [DBG] pgmap v6: 97 pgs: 97 active+clean; 3.0 KiB data, 44 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:25:27.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:27 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:26.343367+0000 mgr.y (mgr.14577) 16 : audit [DBG] from='client.24500 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi153=a", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:25:27.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:27 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:25:26.344806+0000 mgr.y (mgr.14577) 17 : cephadm [INF] Saving service grafana spec with placement smithi153=a;count:1 2022-09-29T05:25:27.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:27 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:26.352073+0000 mon.a (mon.0) 623 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:25:29.138 INFO:teuthology.orchestra.run.smithi083.stdout:[client.0] 2022-09-29T05:25:29.138 INFO:teuthology.orchestra.run.smithi083.stdout: key = AQBJLDVjwR7eBxAA2PPtsXvRAuzPzmbpuAOLuQ== 2022-09-29T05:25:29.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:29 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:25:27.808837+0000 mgr.y (mgr.14577) 18 : cluster [DBG] pgmap v7: 97 pgs: 97 active+clean; 3.0 KiB data, 44 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:25:29.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:29 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:29.131844+0000 mon.a (mon.0) 624 : audit [INF] from='client.? 172.21.15.83:0/3156947810' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2022-09-29T05:25:29.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:29 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:29.135150+0000 mon.a (mon.0) 625 : audit [INF] from='client.? 172.21.15.83:0/3156947810' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2022-09-29T05:25:29.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:29 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:25:27.808837+0000 mgr.y (mgr.14577) 18 : cluster [DBG] pgmap v7: 97 pgs: 97 active+clean; 3.0 KiB data, 44 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:25:29.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:29 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:29.131844+0000 mon.a (mon.0) 624 : audit [INF] from='client.? 172.21.15.83:0/3156947810' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2022-09-29T05:25:29.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:29 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:29.135150+0000 mon.a (mon.0) 625 : audit [INF] from='client.? 172.21.15.83:0/3156947810' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2022-09-29T05:25:29.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:29 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:25:27.808837+0000 mgr.y (mgr.14577) 18 : cluster [DBG] pgmap v7: 97 pgs: 97 active+clean; 3.0 KiB data, 44 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:25:29.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:29 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:29.131844+0000 mon.a (mon.0) 624 : audit [INF] from='client.? 172.21.15.83:0/3156947810' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2022-09-29T05:25:29.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:29 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:29.135150+0000 mon.a (mon.0) 625 : audit [INF] from='client.? 172.21.15.83:0/3156947810' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2022-09-29T05:25:29.652 DEBUG:teuthology.orchestra.run.smithi083:> set -ex 2022-09-29T05:25:29.653 DEBUG:teuthology.orchestra.run.smithi083:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2022-09-29T05:25:29.653 DEBUG:teuthology.orchestra.run.smithi083:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2022-09-29T05:25:29.694 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/daemon-base:latest-pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 69111522-3fb6-11ed-8431-001a4aab830c -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2022-09-29T05:25:30.221 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:25:30 smithi153 systemd[1]: Starting Ceph prometheus.a for 69111522-3fb6-11ed-8431-001a4aab830c... 2022-09-29T05:25:31.132 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:25:30 smithi153 podman[55183]: 2022-09-29T05:25:31.549 INFO:teuthology.orchestra.run.smithi153.stdout:[client.1] 2022-09-29T05:25:31.550 INFO:teuthology.orchestra.run.smithi153.stdout: key = AQBLLDVjjd48IBAAPWgGFdzJqpoTYD+eyTJUHw== 2022-09-29T05:25:31.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:31 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:25:29.809180+0000 mgr.y (mgr.14577) 19 : cluster [DBG] pgmap v8: 97 pgs: 97 active+clean; 3.0 KiB data, 44 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:25:31.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:31 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:31.255707+0000 mon.a (mon.0) 626 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:25:31.559 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:25:31 smithi153 bash[55183]: 8505555ec40afd52da84b6479ac8ee116b68e48dacb11d59ec1acff6982b9461 2022-09-29T05:25:31.560 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:25:31 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[55220]: ts=2022-09-29T05:25:31.140Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2022-09-29T05:25:31.560 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:25:31 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[55220]: ts=2022-09-29T05:25:31.140Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2022-09-29T05:25:31.560 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:25:31 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[55220]: ts=2022-09-29T05:25:31.140Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2022-09-29T05:25:31.560 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:25:31 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[55220]: ts=2022-09-29T05:25:31.140Z caller=main.go:518 level=info host_details="(Linux 4.18.0-408.el8.x86_64 #1 SMP Mon Jul 18 17:42:52 UTC 2022 x86_64 smithi153 (none))" 2022-09-29T05:25:31.561 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:25:31 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[55220]: ts=2022-09-29T05:25:31.140Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2022-09-29T05:25:31.561 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:25:31 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[55220]: ts=2022-09-29T05:25:31.140Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2022-09-29T05:25:31.561 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:25:31 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[55220]: ts=2022-09-29T05:25:31.141Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2022-09-29T05:25:31.561 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:25:31 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[55220]: ts=2022-09-29T05:25:31.142Z caller=main.go:923 level=info msg="Starting TSDB ..." 2022-09-29T05:25:31.561 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:25:31 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[55220]: ts=2022-09-29T05:25:31.143Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2022-09-29T05:25:31.562 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:25:31 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[55220]: ts=2022-09-29T05:25:31.144Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2022-09-29T05:25:31.562 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:25:31 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[55220]: ts=2022-09-29T05:25:31.144Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.47µs 2022-09-29T05:25:31.562 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:25:31 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[55220]: ts=2022-09-29T05:25:31.144Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2022-09-29T05:25:31.562 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:25:31 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[55220]: ts=2022-09-29T05:25:31.144Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0 2022-09-29T05:25:31.563 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:25:31 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[55220]: ts=2022-09-29T05:25:31.144Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=21.044µs wal_replay_duration=286.609µs total_replay_duration=321.436µs 2022-09-29T05:25:31.563 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:25:31 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[55220]: ts=2022-09-29T05:25:31.145Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2022-09-29T05:25:31.563 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:25:31 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[55220]: ts=2022-09-29T05:25:31.145Z caller=main.go:947 level=info msg="TSDB started" 2022-09-29T05:25:31.563 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:25:31 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[55220]: ts=2022-09-29T05:25:31.145Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2022-09-29T05:25:31.564 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:25:31 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[55220]: ts=2022-09-29T05:25:31.161Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=15.745894ms db_storage=711ns remote_storage=1.219µs web_handler=176ns query_engine=658ns scrape=466.628µs scrape_sd=16.45µs notify=709ns notify_sd=1.405µs rules=15.109535ms 2022-09-29T05:25:31.564 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:25:31 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[55220]: ts=2022-09-29T05:25:31.161Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2022-09-29T05:25:31.564 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:25:31 smithi153 systemd[1]: Started Ceph prometheus.a for 69111522-3fb6-11ed-8431-001a4aab830c. 2022-09-29T05:25:31.650 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:25:29.809180+0000 mgr.y (mgr.14577) 19 : cluster [DBG] pgmap v8: 97 pgs: 97 active+clean; 3.0 KiB data, 44 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:25:31.650 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:31.255707+0000 mon.a (mon.0) 626 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:25:31.651 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:25:29.809180+0000 mgr.y (mgr.14577) 19 : cluster [DBG] pgmap v8: 97 pgs: 97 active+clean; 3.0 KiB data, 44 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:25:31.651 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:31.255707+0000 mon.a (mon.0) 626 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:25:31.963 INFO:journalctl@ceph.node-exporter.a.smithi083.stdout:Sep 29 05:25:31 smithi083 systemd[1]: Starting Ceph node-exporter.a for 69111522-3fb6-11ed-8431-001a4aab830c... 2022-09-29T05:25:31.981 DEBUG:teuthology.orchestra.run.smithi153:> set -ex 2022-09-29T05:25:31.981 DEBUG:teuthology.orchestra.run.smithi153:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2022-09-29T05:25:31.982 DEBUG:teuthology.orchestra.run.smithi153:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2022-09-29T05:25:32.024 INFO:tasks.cephadm:Waiting for OSDs to come up 2022-09-29T05:25:32.024 DEBUG:teuthology.orchestra.run.smithi083:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster ceph osd dump --format=json 2022-09-29T05:25:32.305 INFO:teuthology.misc.health.smithi083.stdout: 2022-09-29T05:25:32.306 INFO:teuthology.misc.health.smithi083.stdout:{"epoch":52,"fsid":"69111522-3fb6-11ed-8431-001a4aab830c","created":"2022-09-29T05:20:45.037726+0000","modified":"2022-09-29T05:25:19.746970+0000","last_up_change":"2022-09-29T05:24:46.528124+0000","last_in_change":"2022-09-29T05:24:31.376431+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2022-09-29T05:23:34.795850+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"23","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2022-09-29T05:25:01.755735+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"50","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}}},{"pool":3,"pool_name":"cephfs_data","create_time":"2022-09-29T05:25:02.966273+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"51","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}}],"osds":[{"osd":0,"uuid":"9b8b79ad-3d00-4dad-8da7-7cff0e65d302","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6802","nonce":2080129456},{"type":"v1","addr":"172.21.15.83:6803","nonce":2080129456}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6804","nonce":2080129456},{"type":"v1","addr":"172.21.15.83:6805","nonce":2080129456}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6808","nonce":2080129456},{"type":"v1","addr":"172.21.15.83:6809","nonce":2080129456}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6806","nonce":2080129456},{"type":"v1","addr":"172.21.15.83:6807","nonce":2080129456}]},"public_addr":"172.21.15.83:6803/2080129456","cluster_addr":"172.21.15.83:6805/2080129456","heartbeat_back_addr":"172.21.15.83:6809/2080129456","heartbeat_front_addr":"172.21.15.83:6807/2080129456","state":["exists","up"]},{"osd":1,"uuid":"e098cff5-70cc-437e-8759-e6c7e3d452d2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6810","nonce":605787053},{"type":"v1","addr":"172.21.15.83:6811","nonce":605787053}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6812","nonce":605787053},{"type":"v1","addr":"172.21.15.83:6813","nonce":605787053}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6816","nonce":605787053},{"type":"v1","addr":"172.21.15.83:6817","nonce":605787053}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6814","nonce":605787053},{"type":"v1","addr":"172.21.15.83:6815","nonce":605787053}]},"public_addr":"172.21.15.83:6811/605787053","cluster_addr":"172.21.15.83:6813/605787053","heartbeat_back_addr":"172.21.15.83:6817/605787053","heartbeat_front_addr":"172.21.15.83:6815/605787053","state":["exists","up"]},{"osd":2,"uuid":"2c14853b-5b64-496d-a011-5d8a9b17b731","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6818","nonce":1928326991},{"type":"v1","addr":"172.21.15.83:6819","nonce":1928326991}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6820","nonce":1928326991},{"type":"v1","addr":"172.21.15.83:6821","nonce":1928326991}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6824","nonce":1928326991},{"type":"v1","addr":"172.21.15.83:6825","nonce":1928326991}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6822","nonce":1928326991},{"type":"v1","addr":"172.21.15.83:6823","nonce":1928326991}]},"public_addr":"172.21.15.83:6819/1928326991","cluster_addr":"172.21.15.83:6821/1928326991","heartbeat_back_addr":"172.21.15.83:6825/1928326991","heartbeat_front_addr":"172.21.15.83:6823/1928326991","state":["exists","up"]},{"osd":3,"uuid":"7336f148-261e-468e-8d3c-28f840150f36","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6826","nonce":54976712},{"type":"v1","addr":"172.21.15.83:6827","nonce":54976712}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6828","nonce":54976712},{"type":"v1","addr":"172.21.15.83:6829","nonce":54976712}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6832","nonce":54976712},{"type":"v1","addr":"172.21.15.83:6833","nonce":54976712}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6830","nonce":54976712},{"type":"v1","addr":"172.21.15.83:6831","nonce":54976712}]},"public_addr":"172.21.15.83:6827/54976712","cluster_addr":"172.21.15.83:6829/54976712","heartbeat_back_addr":"172.21.15.83:6833/54976712","heartbeat_front_addr":"172.21.15.83:6831/54976712","state":["exists","up"]},{"osd":4,"uuid":"247d5c19-e8af-4620-a378-699dc2000db1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6800","nonce":4053209633},{"type":"v1","addr":"172.21.15.153:6801","nonce":4053209633}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6802","nonce":4053209633},{"type":"v1","addr":"172.21.15.153:6803","nonce":4053209633}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6806","nonce":4053209633},{"type":"v1","addr":"172.21.15.153:6807","nonce":4053209633}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6804","nonce":4053209633},{"type":"v1","addr":"172.21.15.153:6805","nonce":4053209633}]},"public_addr":"172.21.15.153:6801/4053209633","cluster_addr":"172.21.15.153:6803/4053209633","heartbeat_back_addr":"172.21.15.153:6807/4053209633","heartbeat_front_addr":"172.21.15.153:6805/4053209633","state":["exists","up"]},{"osd":5,"uuid":"c69e2263-5dd8-402d-b901-295c1f4840e2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":35,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6808","nonce":2536485968},{"type":"v1","addr":"172.21.15.153:6809","nonce":2536485968}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6810","nonce":2536485968},{"type":"v1","addr":"172.21.15.153:6811","nonce":2536485968}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6814","nonce":2536485968},{"type":"v1","addr":"172.21.15.153:6815","nonce":2536485968}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6812","nonce":2536485968},{"type":"v1","addr":"172.21.15.153:6813","nonce":2536485968}]},"public_addr":"172.21.15.153:6809/2536485968","cluster_addr":"172.21.15.153:6811/2536485968","heartbeat_back_addr":"172.21.15.153:6815/2536485968","heartbeat_front_addr":"172.21.15.153:6813/2536485968","state":["exists","up"]},{"osd":6,"uuid":"72aa13b6-e7b7-449b-9fa8-f284e8f0735a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":40,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6816","nonce":3662388207},{"type":"v1","addr":"172.21.15.153:6817","nonce":3662388207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6818","nonce":3662388207},{"type":"v1","addr":"172.21.15.153:6819","nonce":3662388207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6822","nonce":3662388207},{"type":"v1","addr":"172.21.15.153:6823","nonce":3662388207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6820","nonce":3662388207},{"type":"v1","addr":"172.21.15.153:6821","nonce":3662388207}]},"public_addr":"172.21.15.153:6817/3662388207","cluster_addr":"172.21.15.153:6819/3662388207","heartbeat_back_addr":"172.21.15.153:6823/3662388207","heartbeat_front_addr":"172.21.15.153:6821/3662388207","state":["exists","up"]},{"osd":7,"uuid":"b5fdd156-d2dc-47fb-b44b-803275092ac0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":45,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6824","nonce":1223012606},{"type":"v1","addr":"172.21.15.153:6825","nonce":1223012606}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6826","nonce":1223012606},{"type":"v1","addr":"172.21.15.153:6827","nonce":1223012606}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6830","nonce":1223012606},{"type":"v1","addr":"172.21.15.153:6831","nonce":1223012606}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6828","nonce":1223012606},{"type":"v1","addr":"172.21.15.153:6829","nonce":1223012606}]},"public_addr":"172.21.15.153:6825/1223012606","cluster_addr":"172.21.15.153:6827/1223012606","heartbeat_back_addr":"172.21.15.153:6831/1223012606","heartbeat_front_addr":"172.21.15.153:6829/1223012606","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2022-09-29T05:22:43.518222+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2022-09-29T05:23:01.859214+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2022-09-29T05:23:20.325748+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2022-09-29T05:23:38.717068+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2022-09-29T05:23:54.756146+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2022-09-29T05:24:11.670499+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2022-09-29T05:24:28.641104+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2022-09-29T05:24:45.518895+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.83:6801/2790412241":"2022-09-30T05:25:19.746941+0000","172.21.15.83:0/4071118407":"2022-09-30T05:25:19.746941+0000","172.21.15.83:0/2269625481":"2022-09-30T05:21:35.319810+0000","172.21.15.83:0/2173437316":"2022-09-30T05:21:35.319810+0000","172.21.15.83:6800/2702658771":"2022-09-30T05:21:35.319810+0000","172.21.15.83:0/4005486955":"2022-09-30T05:21:15.866011+0000","172.21.15.83:6800/2790412241":"2022-09-30T05:25:19.746941+0000","172.21.15.83:6800/2006042278":"2022-09-30T05:21:15.866011+0000","172.21.15.83:6801/2006042278":"2022-09-30T05:21:15.866011+0000","172.21.15.83:0/664253330":"2022-09-30T05:25:19.746941+0000","172.21.15.83:6801/2702658771":"2022-09-30T05:21:35.319810+0000","172.21.15.83:0/4256024697":"2022-09-30T05:21:15.866011+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2022-09-29T05:25:32.322 DEBUG:teuthology.misc:8 of 8 OSDs are up 2022-09-29T05:25:32.323 INFO:tasks.cephadm:Creating RBD pool 2022-09-29T05:25:32.323 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/daemon-base:latest-pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 69111522-3fb6-11ed-8431-001a4aab830c -- sudo ceph --cluster ceph osd pool create rbd 8 2022-09-29T05:25:32.713 INFO:journalctl@ceph.node-exporter.a.smithi083.stdout:Sep 29 05:25:32 smithi083 bash[60841]: Trying to pull quay.io/prometheus/node-exporter:v1.3.1... 2022-09-29T05:25:32.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:25:31.257978+0000 mgr.y (mgr.14577) 20 : cephadm [INF] Deploying daemon node-exporter.a on smithi083 2022-09-29T05:25:32.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:31.540057+0000 mon.b (mon.2) 26 : audit [INF] from='client.? 172.21.15.153:0/405302576' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2022-09-29T05:25:32.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:31.540713+0000 mon.a (mon.0) 627 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2022-09-29T05:25:32.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:31.546454+0000 mon.a (mon.0) 628 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2022-09-29T05:25:32.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:32.302121+0000 mon.a (mon.0) 629 : audit [DBG] from='client.? 172.21.15.83:0/2974808000' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2022-09-29T05:25:32.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:25:31.257978+0000 mgr.y (mgr.14577) 20 : cephadm [INF] Deploying daemon node-exporter.a on smithi083 2022-09-29T05:25:32.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:31.540057+0000 mon.b (mon.2) 26 : audit [INF] from='client.? 172.21.15.153:0/405302576' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2022-09-29T05:25:32.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:31.540713+0000 mon.a (mon.0) 627 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2022-09-29T05:25:32.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:31.546454+0000 mon.a (mon.0) 628 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2022-09-29T05:25:32.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:32.302121+0000 mon.a (mon.0) 629 : audit [DBG] from='client.? 172.21.15.83:0/2974808000' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2022-09-29T05:25:32.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:25:31.257978+0000 mgr.y (mgr.14577) 20 : cephadm [INF] Deploying daemon node-exporter.a on smithi083 2022-09-29T05:25:32.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:31.540057+0000 mon.b (mon.2) 26 : audit [INF] from='client.? 172.21.15.153:0/405302576' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2022-09-29T05:25:32.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:31.540713+0000 mon.a (mon.0) 627 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2022-09-29T05:25:32.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:31.546454+0000 mon.a (mon.0) 628 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2022-09-29T05:25:32.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:32.302121+0000 mon.a (mon.0) 629 : audit [DBG] from='client.? 172.21.15.83:0/2974808000' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2022-09-29T05:25:33.213 INFO:journalctl@ceph.node-exporter.a.smithi083.stdout:Sep 29 05:25:32 smithi083 bash[60841]: Getting image source signatures 2022-09-29T05:25:33.213 INFO:journalctl@ceph.node-exporter.a.smithi083.stdout:Sep 29 05:25:32 smithi083 bash[60841]: Copying blob sha256:b5db1e299295edf3005515ab7879c1df64a33c185d3a7a23aa4dcaa17d26f7b3 2022-09-29T05:25:33.213 INFO:journalctl@ceph.node-exporter.a.smithi083.stdout:Sep 29 05:25:32 smithi083 bash[60841]: Copying blob sha256:aa2a8d90b84cb2a9c422e7005cd166a008ccf22ef5d7d4f07128478585ce35ea 2022-09-29T05:25:33.214 INFO:journalctl@ceph.node-exporter.a.smithi083.stdout:Sep 29 05:25:32 smithi083 bash[60841]: Copying blob sha256:b45d31ee2d7f9f452678a85b0c837c29e12089f31ee8dbac6c8c24dfa4054a30 2022-09-29T05:25:33.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:25:31.810559+0000 mgr.y (mgr.14577) 21 : cluster [DBG] pgmap v9: 97 pgs: 97 active+clean; 3.0 KiB data, 44 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:25:33.938 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:25:31.810559+0000 mgr.y (mgr.14577) 21 : cluster [DBG] pgmap v9: 97 pgs: 97 active+clean; 3.0 KiB data, 44 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:25:33.939 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:25:31.810559+0000 mgr.y (mgr.14577) 21 : cluster [DBG] pgmap v9: 97 pgs: 97 active+clean; 3.0 KiB data, 44 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:25:34.212 INFO:journalctl@ceph.node-exporter.a.smithi083.stdout:Sep 29 05:25:34 smithi083 bash[60841]: Copying config sha256:1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 2022-09-29T05:25:34.213 INFO:journalctl@ceph.node-exporter.a.smithi083.stdout:Sep 29 05:25:34 smithi083 bash[60841]: Writing manifest to image destination 2022-09-29T05:25:34.213 INFO:journalctl@ceph.node-exporter.a.smithi083.stdout:Sep 29 05:25:34 smithi083 bash[60841]: Storing signatures 2022-09-29T05:25:34.874 INFO:journalctl@ceph.node-exporter.a.smithi083.stdout:Sep 29 05:25:34 smithi083 podman[60841]: 2022-09-29T05:25:35.463 INFO:journalctl@ceph.node-exporter.a.smithi083.stdout:Sep 29 05:25:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-a[60992]: ts=2022-09-29T05:25:35.034Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2022-09-29T05:25:35.463 INFO:journalctl@ceph.node-exporter.a.smithi083.stdout:Sep 29 05:25:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-a[60992]: ts=2022-09-29T05:25:35.034Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2022-09-29T05:25:35.464 INFO:journalctl@ceph.node-exporter.a.smithi083.stdout:Sep 29 05:25:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-a[60992]: ts=2022-09-29T05:25:35.034Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2022-09-29T05:25:35.464 INFO:journalctl@ceph.node-exporter.a.smithi083.stdout:Sep 29 05:25:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-a[60992]: ts=2022-09-29T05:25:35.035Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2022-09-29T05:25:35.464 INFO:journalctl@ceph.node-exporter.a.smithi083.stdout:Sep 29 05:25:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-a[60992]: ts=2022-09-29T05:25:35.035Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2022-09-29T05:25:35.465 INFO:journalctl@ceph.node-exporter.a.smithi083.stdout:Sep 29 05:25:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-a[60992]: ts=2022-09-29T05:25:35.035Z caller=node_exporter.go:115 level=info collector=arp 2022-09-29T05:25:35.465 INFO:journalctl@ceph.node-exporter.a.smithi083.stdout:Sep 29 05:25:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-a[60992]: ts=2022-09-29T05:25:35.035Z caller=node_exporter.go:115 level=info collector=bcache 2022-09-29T05:25:35.465 INFO:journalctl@ceph.node-exporter.a.smithi083.stdout:Sep 29 05:25:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-a[60992]: ts=2022-09-29T05:25:35.035Z caller=node_exporter.go:115 level=info collector=bonding 2022-09-29T05:25:35.465 INFO:journalctl@ceph.node-exporter.a.smithi083.stdout:Sep 29 05:25:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-a[60992]: ts=2022-09-29T05:25:35.035Z caller=node_exporter.go:115 level=info collector=btrfs 2022-09-29T05:25:35.466 INFO:journalctl@ceph.node-exporter.a.smithi083.stdout:Sep 29 05:25:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-a[60992]: ts=2022-09-29T05:25:35.035Z caller=node_exporter.go:115 level=info collector=conntrack 2022-09-29T05:25:35.466 INFO:journalctl@ceph.node-exporter.a.smithi083.stdout:Sep 29 05:25:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-a[60992]: ts=2022-09-29T05:25:35.035Z caller=node_exporter.go:115 level=info collector=cpu 2022-09-29T05:25:35.466 INFO:journalctl@ceph.node-exporter.a.smithi083.stdout:Sep 29 05:25:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-a[60992]: ts=2022-09-29T05:25:35.035Z caller=node_exporter.go:115 level=info collector=cpufreq 2022-09-29T05:25:35.466 INFO:journalctl@ceph.node-exporter.a.smithi083.stdout:Sep 29 05:25:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-a[60992]: ts=2022-09-29T05:25:35.035Z caller=node_exporter.go:115 level=info collector=diskstats 2022-09-29T05:25:35.467 INFO:journalctl@ceph.node-exporter.a.smithi083.stdout:Sep 29 05:25:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-a[60992]: ts=2022-09-29T05:25:35.035Z caller=node_exporter.go:115 level=info collector=dmi 2022-09-29T05:25:35.467 INFO:journalctl@ceph.node-exporter.a.smithi083.stdout:Sep 29 05:25:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-a[60992]: ts=2022-09-29T05:25:35.035Z caller=node_exporter.go:115 level=info collector=edac 2022-09-29T05:25:35.467 INFO:journalctl@ceph.node-exporter.a.smithi083.stdout:Sep 29 05:25:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-a[60992]: ts=2022-09-29T05:25:35.035Z caller=node_exporter.go:115 level=info collector=entropy 2022-09-29T05:25:35.467 INFO:journalctl@ceph.node-exporter.a.smithi083.stdout:Sep 29 05:25:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-a[60992]: ts=2022-09-29T05:25:35.035Z caller=node_exporter.go:115 level=info collector=fibrechannel 2022-09-29T05:25:35.467 INFO:journalctl@ceph.node-exporter.a.smithi083.stdout:Sep 29 05:25:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-a[60992]: ts=2022-09-29T05:25:35.035Z caller=node_exporter.go:115 level=info collector=filefd 2022-09-29T05:25:35.468 INFO:journalctl@ceph.node-exporter.a.smithi083.stdout:Sep 29 05:25:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-a[60992]: ts=2022-09-29T05:25:35.035Z caller=node_exporter.go:115 level=info collector=filesystem 2022-09-29T05:25:35.468 INFO:journalctl@ceph.node-exporter.a.smithi083.stdout:Sep 29 05:25:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-a[60992]: ts=2022-09-29T05:25:35.035Z caller=node_exporter.go:115 level=info collector=hwmon 2022-09-29T05:25:35.468 INFO:journalctl@ceph.node-exporter.a.smithi083.stdout:Sep 29 05:25:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-a[60992]: ts=2022-09-29T05:25:35.035Z caller=node_exporter.go:115 level=info collector=infiniband 2022-09-29T05:25:35.468 INFO:journalctl@ceph.node-exporter.a.smithi083.stdout:Sep 29 05:25:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-a[60992]: ts=2022-09-29T05:25:35.035Z caller=node_exporter.go:115 level=info collector=ipvs 2022-09-29T05:25:35.469 INFO:journalctl@ceph.node-exporter.a.smithi083.stdout:Sep 29 05:25:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-a[60992]: ts=2022-09-29T05:25:35.035Z caller=node_exporter.go:115 level=info collector=loadavg 2022-09-29T05:25:35.469 INFO:journalctl@ceph.node-exporter.a.smithi083.stdout:Sep 29 05:25:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-a[60992]: ts=2022-09-29T05:25:35.035Z caller=node_exporter.go:115 level=info collector=mdadm 2022-09-29T05:25:35.469 INFO:journalctl@ceph.node-exporter.a.smithi083.stdout:Sep 29 05:25:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-a[60992]: ts=2022-09-29T05:25:35.035Z caller=node_exporter.go:115 level=info collector=meminfo 2022-09-29T05:25:35.469 INFO:journalctl@ceph.node-exporter.a.smithi083.stdout:Sep 29 05:25:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-a[60992]: ts=2022-09-29T05:25:35.035Z caller=node_exporter.go:115 level=info collector=netclass 2022-09-29T05:25:35.469 INFO:journalctl@ceph.node-exporter.a.smithi083.stdout:Sep 29 05:25:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-a[60992]: ts=2022-09-29T05:25:35.035Z caller=node_exporter.go:115 level=info collector=netdev 2022-09-29T05:25:35.470 INFO:journalctl@ceph.node-exporter.a.smithi083.stdout:Sep 29 05:25:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-a[60992]: ts=2022-09-29T05:25:35.035Z caller=node_exporter.go:115 level=info collector=netstat 2022-09-29T05:25:35.470 INFO:journalctl@ceph.node-exporter.a.smithi083.stdout:Sep 29 05:25:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-a[60992]: ts=2022-09-29T05:25:35.035Z caller=node_exporter.go:115 level=info collector=nfs 2022-09-29T05:25:35.470 INFO:journalctl@ceph.node-exporter.a.smithi083.stdout:Sep 29 05:25:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-a[60992]: ts=2022-09-29T05:25:35.035Z caller=node_exporter.go:115 level=info collector=nfsd 2022-09-29T05:25:35.470 INFO:journalctl@ceph.node-exporter.a.smithi083.stdout:Sep 29 05:25:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-a[60992]: ts=2022-09-29T05:25:35.035Z caller=node_exporter.go:115 level=info collector=nvme 2022-09-29T05:25:35.471 INFO:journalctl@ceph.node-exporter.a.smithi083.stdout:Sep 29 05:25:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-a[60992]: ts=2022-09-29T05:25:35.035Z caller=node_exporter.go:115 level=info collector=os 2022-09-29T05:25:35.471 INFO:journalctl@ceph.node-exporter.a.smithi083.stdout:Sep 29 05:25:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-a[60992]: ts=2022-09-29T05:25:35.035Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2022-09-29T05:25:35.471 INFO:journalctl@ceph.node-exporter.a.smithi083.stdout:Sep 29 05:25:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-a[60992]: ts=2022-09-29T05:25:35.035Z caller=node_exporter.go:115 level=info collector=pressure 2022-09-29T05:25:35.472 INFO:journalctl@ceph.node-exporter.a.smithi083.stdout:Sep 29 05:25:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-a[60992]: ts=2022-09-29T05:25:35.035Z caller=node_exporter.go:115 level=info collector=rapl 2022-09-29T05:25:35.472 INFO:journalctl@ceph.node-exporter.a.smithi083.stdout:Sep 29 05:25:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-a[60992]: ts=2022-09-29T05:25:35.035Z caller=node_exporter.go:115 level=info collector=schedstat 2022-09-29T05:25:35.472 INFO:journalctl@ceph.node-exporter.a.smithi083.stdout:Sep 29 05:25:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-a[60992]: ts=2022-09-29T05:25:35.035Z caller=node_exporter.go:115 level=info collector=sockstat 2022-09-29T05:25:35.472 INFO:journalctl@ceph.node-exporter.a.smithi083.stdout:Sep 29 05:25:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-a[60992]: ts=2022-09-29T05:25:35.035Z caller=node_exporter.go:115 level=info collector=softnet 2022-09-29T05:25:35.473 INFO:journalctl@ceph.node-exporter.a.smithi083.stdout:Sep 29 05:25:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-a[60992]: ts=2022-09-29T05:25:35.035Z caller=node_exporter.go:115 level=info collector=stat 2022-09-29T05:25:35.473 INFO:journalctl@ceph.node-exporter.a.smithi083.stdout:Sep 29 05:25:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-a[60992]: ts=2022-09-29T05:25:35.035Z caller=node_exporter.go:115 level=info collector=tapestats 2022-09-29T05:25:35.473 INFO:journalctl@ceph.node-exporter.a.smithi083.stdout:Sep 29 05:25:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-a[60992]: ts=2022-09-29T05:25:35.036Z caller=node_exporter.go:115 level=info collector=textfile 2022-09-29T05:25:35.473 INFO:journalctl@ceph.node-exporter.a.smithi083.stdout:Sep 29 05:25:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-a[60992]: ts=2022-09-29T05:25:35.036Z caller=node_exporter.go:115 level=info collector=thermal_zone 2022-09-29T05:25:35.474 INFO:journalctl@ceph.node-exporter.a.smithi083.stdout:Sep 29 05:25:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-a[60992]: ts=2022-09-29T05:25:35.036Z caller=node_exporter.go:115 level=info collector=time 2022-09-29T05:25:35.474 INFO:journalctl@ceph.node-exporter.a.smithi083.stdout:Sep 29 05:25:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-a[60992]: ts=2022-09-29T05:25:35.036Z caller=node_exporter.go:115 level=info collector=udp_queues 2022-09-29T05:25:35.474 INFO:journalctl@ceph.node-exporter.a.smithi083.stdout:Sep 29 05:25:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-a[60992]: ts=2022-09-29T05:25:35.036Z caller=node_exporter.go:115 level=info collector=uname 2022-09-29T05:25:35.474 INFO:journalctl@ceph.node-exporter.a.smithi083.stdout:Sep 29 05:25:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-a[60992]: ts=2022-09-29T05:25:35.036Z caller=node_exporter.go:115 level=info collector=vmstat 2022-09-29T05:25:35.474 INFO:journalctl@ceph.node-exporter.a.smithi083.stdout:Sep 29 05:25:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-a[60992]: ts=2022-09-29T05:25:35.036Z caller=node_exporter.go:115 level=info collector=xfs 2022-09-29T05:25:35.475 INFO:journalctl@ceph.node-exporter.a.smithi083.stdout:Sep 29 05:25:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-a[60992]: ts=2022-09-29T05:25:35.036Z caller=node_exporter.go:115 level=info collector=zfs 2022-09-29T05:25:35.475 INFO:journalctl@ceph.node-exporter.a.smithi083.stdout:Sep 29 05:25:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-a[60992]: ts=2022-09-29T05:25:35.036Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2022-09-29T05:25:35.475 INFO:journalctl@ceph.node-exporter.a.smithi083.stdout:Sep 29 05:25:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-a[60992]: ts=2022-09-29T05:25:35.036Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2022-09-29T05:25:35.475 INFO:journalctl@ceph.node-exporter.a.smithi083.stdout:Sep 29 05:25:35 smithi083 bash[60841]: 98012608edcd5f49d95a6d0b313594cd6ad016491f0ca91be36db13966605c15 2022-09-29T05:25:35.476 INFO:journalctl@ceph.node-exporter.a.smithi083.stdout:Sep 29 05:25:35 smithi083 systemd[1]: Started Ceph node-exporter.a for 69111522-3fb6-11ed-8431-001a4aab830c. 2022-09-29T05:25:35.809 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Sep 29 05:25:35 smithi153 systemd[1]: Starting Ceph node-exporter.b for 69111522-3fb6-11ed-8431-001a4aab830c... 2022-09-29T05:25:35.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:35 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:25:33.810858+0000 mgr.y (mgr.14577) 22 : cluster [DBG] pgmap v10: 97 pgs: 97 active+clean; 3.0 KiB data, 44 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:25:35.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:35 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:34.952696+0000 mon.a (mon.0) 630 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:25:35.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:35 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:35.132211+0000 mon.a (mon.0) 631 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:25:35.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:35 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:35.197965+0000 mon.c (mon.1) 23 : audit [INF] from='client.? 172.21.15.83:0/1754740832' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2022-09-29T05:25:35.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:35 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:35.198298+0000 mon.a (mon.0) 632 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2022-09-29T05:25:35.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:25:33.810858+0000 mgr.y (mgr.14577) 22 : cluster [DBG] pgmap v10: 97 pgs: 97 active+clean; 3.0 KiB data, 44 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:25:35.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:34.952696+0000 mon.a (mon.0) 630 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:25:35.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:35.132211+0000 mon.a (mon.0) 631 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:25:35.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:35.197965+0000 mon.c (mon.1) 23 : audit [INF] from='client.? 172.21.15.83:0/1754740832' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2022-09-29T05:25:35.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:35.198298+0000 mon.a (mon.0) 632 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2022-09-29T05:25:35.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:25:33.810858+0000 mgr.y (mgr.14577) 22 : cluster [DBG] pgmap v10: 97 pgs: 97 active+clean; 3.0 KiB data, 44 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:25:35.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:34.952696+0000 mon.a (mon.0) 630 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:25:35.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:35.132211+0000 mon.a (mon.0) 631 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:25:35.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:35.197965+0000 mon.c (mon.1) 23 : audit [INF] from='client.? 172.21.15.83:0/1754740832' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2022-09-29T05:25:35.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:35.198298+0000 mon.a (mon.0) 632 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2022-09-29T05:25:36.143 INFO:teuthology.orchestra.run.smithi083.stderr:pool 'rbd' created 2022-09-29T05:25:36.627 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/daemon-base:latest-pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 69111522-3fb6-11ed-8431-001a4aab830c -- sudo ceph --cluster ceph osd pool application enable rbd rbd --yes-i-really-mean-it 2022-09-29T05:25:36.788 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Sep 29 05:25:36 smithi153 bash[55517]: Trying to pull quay.io/prometheus/node-exporter:v1.3.1... 2022-09-29T05:25:36.788 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:36 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:25:35.133107+0000 mgr.y (mgr.14577) 23 : cephadm [INF] Deploying daemon node-exporter.b on smithi153 2022-09-29T05:25:36.788 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:36 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:36.134473+0000 mon.a (mon.0) 633 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2022-09-29T05:25:36.789 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:36 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:25:36.134576+0000 mon.a (mon.0) 634 : cluster [DBG] osdmap e53: 8 total, 8 up, 8 in 2022-09-29T05:25:36.789 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:36 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:36.135216+0000 mon.a (mon.0) 635 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2022-09-29T05:25:36.789 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:36 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:36.136149+0000 mon.a (mon.0) 636 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2022-09-29T05:25:36.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:25:35.133107+0000 mgr.y (mgr.14577) 23 : cephadm [INF] Deploying daemon node-exporter.b on smithi153 2022-09-29T05:25:36.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:36.134473+0000 mon.a (mon.0) 633 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2022-09-29T05:25:36.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:25:36.134576+0000 mon.a (mon.0) 634 : cluster [DBG] osdmap e53: 8 total, 8 up, 8 in 2022-09-29T05:25:36.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:36.135216+0000 mon.a (mon.0) 635 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2022-09-29T05:25:36.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:36.136149+0000 mon.a (mon.0) 636 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2022-09-29T05:25:36.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:25:35.133107+0000 mgr.y (mgr.14577) 23 : cephadm [INF] Deploying daemon node-exporter.b on smithi153 2022-09-29T05:25:36.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:36.134473+0000 mon.a (mon.0) 633 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2022-09-29T05:25:36.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:25:36.134576+0000 mon.a (mon.0) 634 : cluster [DBG] osdmap e53: 8 total, 8 up, 8 in 2022-09-29T05:25:36.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:36.135216+0000 mon.a (mon.0) 635 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2022-09-29T05:25:36.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:36.136149+0000 mon.a (mon.0) 636 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2022-09-29T05:25:37.058 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Sep 29 05:25:36 smithi153 bash[55517]: Getting image source signatures 2022-09-29T05:25:37.059 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Sep 29 05:25:36 smithi153 bash[55517]: Copying blob sha256:b5db1e299295edf3005515ab7879c1df64a33c185d3a7a23aa4dcaa17d26f7b3 2022-09-29T05:25:37.059 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Sep 29 05:25:36 smithi153 bash[55517]: Copying blob sha256:aa2a8d90b84cb2a9c422e7005cd166a008ccf22ef5d7d4f07128478585ce35ea 2022-09-29T05:25:37.059 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Sep 29 05:25:36 smithi153 bash[55517]: Copying blob sha256:b45d31ee2d7f9f452678a85b0c837c29e12089f31ee8dbac6c8c24dfa4054a30 2022-09-29T05:25:37.776 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:37 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:25:35.811150+0000 mgr.y (mgr.14577) 24 : cluster [DBG] pgmap v11: 97 pgs: 97 active+clean; 3.0 KiB data, 44 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:25:37.777 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:37 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:37.136350+0000 mon.a (mon.0) 637 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2022-09-29T05:25:37.777 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:37 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:25:37.136479+0000 mon.a (mon.0) 638 : cluster [DBG] osdmap e54: 8 total, 8 up, 8 in 2022-09-29T05:25:37.778 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:37 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:25:35.811150+0000 mgr.y (mgr.14577) 24 : cluster [DBG] pgmap v11: 97 pgs: 97 active+clean; 3.0 KiB data, 44 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:25:37.778 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:37 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:37.136350+0000 mon.a (mon.0) 637 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2022-09-29T05:25:37.778 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:37 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:25:37.136479+0000 mon.a (mon.0) 638 : cluster [DBG] osdmap e54: 8 total, 8 up, 8 in 2022-09-29T05:25:37.809 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Sep 29 05:25:37 smithi153 bash[55517]: Copying config sha256:1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 2022-09-29T05:25:37.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:37 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:25:35.811150+0000 mgr.y (mgr.14577) 24 : cluster [DBG] pgmap v11: 97 pgs: 97 active+clean; 3.0 KiB data, 44 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:25:37.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:37 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:37.136350+0000 mon.a (mon.0) 637 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2022-09-29T05:25:37.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:37 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:25:37.136479+0000 mon.a (mon.0) 638 : cluster [DBG] osdmap e54: 8 total, 8 up, 8 in 2022-09-29T05:25:38.305 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Sep 29 05:25:37 smithi153 bash[55517]: Writing manifest to image destination 2022-09-29T05:25:38.306 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Sep 29 05:25:37 smithi153 bash[55517]: Storing signatures 2022-09-29T05:25:38.559 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Sep 29 05:25:38 smithi153 podman[55517]: 2022-09-29T05:25:38.560 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Sep 29 05:25:38 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-b[55569]: ts=2022-09-29T05:25:38.503Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2022-09-29T05:25:38.560 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Sep 29 05:25:38 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-b[55569]: ts=2022-09-29T05:25:38.503Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2022-09-29T05:25:38.560 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Sep 29 05:25:38 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-b[55569]: ts=2022-09-29T05:25:38.504Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2022-09-29T05:25:38.560 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Sep 29 05:25:38 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-b[55569]: ts=2022-09-29T05:25:38.504Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2022-09-29T05:25:38.561 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Sep 29 05:25:38 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-b[55569]: ts=2022-09-29T05:25:38.504Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2022-09-29T05:25:38.561 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Sep 29 05:25:38 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-b[55569]: ts=2022-09-29T05:25:38.504Z caller=node_exporter.go:115 level=info collector=arp 2022-09-29T05:25:38.561 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Sep 29 05:25:38 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-b[55569]: ts=2022-09-29T05:25:38.504Z caller=node_exporter.go:115 level=info collector=bcache 2022-09-29T05:25:38.561 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Sep 29 05:25:38 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-b[55569]: ts=2022-09-29T05:25:38.504Z caller=node_exporter.go:115 level=info collector=bonding 2022-09-29T05:25:38.562 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Sep 29 05:25:38 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-b[55569]: ts=2022-09-29T05:25:38.504Z caller=node_exporter.go:115 level=info collector=btrfs 2022-09-29T05:25:38.562 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Sep 29 05:25:38 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-b[55569]: ts=2022-09-29T05:25:38.504Z caller=node_exporter.go:115 level=info collector=conntrack 2022-09-29T05:25:38.562 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Sep 29 05:25:38 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-b[55569]: ts=2022-09-29T05:25:38.504Z caller=node_exporter.go:115 level=info collector=cpu 2022-09-29T05:25:38.562 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Sep 29 05:25:38 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-b[55569]: ts=2022-09-29T05:25:38.504Z caller=node_exporter.go:115 level=info collector=cpufreq 2022-09-29T05:25:38.563 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Sep 29 05:25:38 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-b[55569]: ts=2022-09-29T05:25:38.504Z caller=node_exporter.go:115 level=info collector=diskstats 2022-09-29T05:25:38.563 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Sep 29 05:25:38 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-b[55569]: ts=2022-09-29T05:25:38.504Z caller=node_exporter.go:115 level=info collector=dmi 2022-09-29T05:25:38.563 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Sep 29 05:25:38 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-b[55569]: ts=2022-09-29T05:25:38.504Z caller=node_exporter.go:115 level=info collector=edac 2022-09-29T05:25:38.563 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Sep 29 05:25:38 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-b[55569]: ts=2022-09-29T05:25:38.504Z caller=node_exporter.go:115 level=info collector=entropy 2022-09-29T05:25:38.564 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Sep 29 05:25:38 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-b[55569]: ts=2022-09-29T05:25:38.504Z caller=node_exporter.go:115 level=info collector=fibrechannel 2022-09-29T05:25:38.564 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Sep 29 05:25:38 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-b[55569]: ts=2022-09-29T05:25:38.504Z caller=node_exporter.go:115 level=info collector=filefd 2022-09-29T05:25:38.564 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Sep 29 05:25:38 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-b[55569]: ts=2022-09-29T05:25:38.504Z caller=node_exporter.go:115 level=info collector=filesystem 2022-09-29T05:25:38.564 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Sep 29 05:25:38 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-b[55569]: ts=2022-09-29T05:25:38.504Z caller=node_exporter.go:115 level=info collector=hwmon 2022-09-29T05:25:38.565 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Sep 29 05:25:38 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-b[55569]: ts=2022-09-29T05:25:38.504Z caller=node_exporter.go:115 level=info collector=infiniband 2022-09-29T05:25:38.565 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Sep 29 05:25:38 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-b[55569]: ts=2022-09-29T05:25:38.504Z caller=node_exporter.go:115 level=info collector=ipvs 2022-09-29T05:25:38.565 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Sep 29 05:25:38 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-b[55569]: ts=2022-09-29T05:25:38.504Z caller=node_exporter.go:115 level=info collector=loadavg 2022-09-29T05:25:38.565 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Sep 29 05:25:38 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-b[55569]: ts=2022-09-29T05:25:38.504Z caller=node_exporter.go:115 level=info collector=mdadm 2022-09-29T05:25:38.566 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Sep 29 05:25:38 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-b[55569]: ts=2022-09-29T05:25:38.504Z caller=node_exporter.go:115 level=info collector=meminfo 2022-09-29T05:25:38.566 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Sep 29 05:25:38 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-b[55569]: ts=2022-09-29T05:25:38.504Z caller=node_exporter.go:115 level=info collector=netclass 2022-09-29T05:25:38.566 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Sep 29 05:25:38 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-b[55569]: ts=2022-09-29T05:25:38.504Z caller=node_exporter.go:115 level=info collector=netdev 2022-09-29T05:25:38.566 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Sep 29 05:25:38 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-b[55569]: ts=2022-09-29T05:25:38.504Z caller=node_exporter.go:115 level=info collector=netstat 2022-09-29T05:25:38.566 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Sep 29 05:25:38 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-b[55569]: ts=2022-09-29T05:25:38.504Z caller=node_exporter.go:115 level=info collector=nfs 2022-09-29T05:25:38.567 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Sep 29 05:25:38 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-b[55569]: ts=2022-09-29T05:25:38.504Z caller=node_exporter.go:115 level=info collector=nfsd 2022-09-29T05:25:38.567 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Sep 29 05:25:38 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-b[55569]: ts=2022-09-29T05:25:38.504Z caller=node_exporter.go:115 level=info collector=nvme 2022-09-29T05:25:38.567 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Sep 29 05:25:38 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-b[55569]: ts=2022-09-29T05:25:38.504Z caller=node_exporter.go:115 level=info collector=os 2022-09-29T05:25:38.567 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Sep 29 05:25:38 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-b[55569]: ts=2022-09-29T05:25:38.504Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2022-09-29T05:25:38.568 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Sep 29 05:25:38 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-b[55569]: ts=2022-09-29T05:25:38.504Z caller=node_exporter.go:115 level=info collector=pressure 2022-09-29T05:25:38.568 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Sep 29 05:25:38 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-b[55569]: ts=2022-09-29T05:25:38.504Z caller=node_exporter.go:115 level=info collector=rapl 2022-09-29T05:25:38.568 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Sep 29 05:25:38 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-b[55569]: ts=2022-09-29T05:25:38.504Z caller=node_exporter.go:115 level=info collector=schedstat 2022-09-29T05:25:38.568 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Sep 29 05:25:38 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-b[55569]: ts=2022-09-29T05:25:38.504Z caller=node_exporter.go:115 level=info collector=sockstat 2022-09-29T05:25:38.569 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Sep 29 05:25:38 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-b[55569]: ts=2022-09-29T05:25:38.504Z caller=node_exporter.go:115 level=info collector=softnet 2022-09-29T05:25:38.569 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Sep 29 05:25:38 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-b[55569]: ts=2022-09-29T05:25:38.504Z caller=node_exporter.go:115 level=info collector=stat 2022-09-29T05:25:38.569 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Sep 29 05:25:38 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-b[55569]: ts=2022-09-29T05:25:38.504Z caller=node_exporter.go:115 level=info collector=tapestats 2022-09-29T05:25:38.569 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Sep 29 05:25:38 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-b[55569]: ts=2022-09-29T05:25:38.504Z caller=node_exporter.go:115 level=info collector=textfile 2022-09-29T05:25:38.570 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Sep 29 05:25:38 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-b[55569]: ts=2022-09-29T05:25:38.504Z caller=node_exporter.go:115 level=info collector=thermal_zone 2022-09-29T05:25:38.570 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Sep 29 05:25:38 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-b[55569]: ts=2022-09-29T05:25:38.504Z caller=node_exporter.go:115 level=info collector=time 2022-09-29T05:25:38.570 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Sep 29 05:25:38 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-b[55569]: ts=2022-09-29T05:25:38.504Z caller=node_exporter.go:115 level=info collector=udp_queues 2022-09-29T05:25:38.570 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Sep 29 05:25:38 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-b[55569]: ts=2022-09-29T05:25:38.504Z caller=node_exporter.go:115 level=info collector=uname 2022-09-29T05:25:38.571 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Sep 29 05:25:38 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-b[55569]: ts=2022-09-29T05:25:38.504Z caller=node_exporter.go:115 level=info collector=vmstat 2022-09-29T05:25:38.571 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Sep 29 05:25:38 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-b[55569]: ts=2022-09-29T05:25:38.504Z caller=node_exporter.go:115 level=info collector=xfs 2022-09-29T05:25:38.571 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Sep 29 05:25:38 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-b[55569]: ts=2022-09-29T05:25:38.504Z caller=node_exporter.go:115 level=info collector=zfs 2022-09-29T05:25:38.571 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Sep 29 05:25:38 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-b[55569]: ts=2022-09-29T05:25:38.504Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2022-09-29T05:25:38.572 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Sep 29 05:25:38 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-node-exporter-b[55569]: ts=2022-09-29T05:25:38.505Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2022-09-29T05:25:38.572 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Sep 29 05:25:38 smithi153 bash[55517]: a95f214c24de6c9378c44aee1b4316cbe342dcba80abcd0808428575f6b0505a 2022-09-29T05:25:38.572 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Sep 29 05:25:38 smithi153 systemd[1]: Started Ceph node-exporter.b for 69111522-3fb6-11ed-8431-001a4aab830c. 2022-09-29T05:25:39.149 INFO:teuthology.orchestra.run.smithi083.stderr:enabled application 'rbd' on pool 'rbd' 2022-09-29T05:25:39.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:39 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:25:37.811520+0000 mgr.y (mgr.14577) 25 : cluster [DBG] pgmap v14: 105 pgs: 8 unknown, 97 active+clean; 3.0 KiB data, 44 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:25:39.533 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:39 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:25:38.142080+0000 mon.a (mon.0) 639 : cluster [DBG] osdmap e55: 8 total, 8 up, 8 in 2022-09-29T05:25:39.533 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:39 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:38.381896+0000 mon.c (mon.1) 24 : audit [INF] from='client.? 172.21.15.83:0/2058012094' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2022-09-29T05:25:39.534 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:39 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:38.382210+0000 mon.a (mon.0) 640 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2022-09-29T05:25:39.534 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:39 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:38.589992+0000 mon.a (mon.0) 641 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:25:39.534 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:39 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:38.591106+0000 mon.a (mon.0) 642 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:25:39.535 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:39 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:38.592152+0000 mon.a (mon.0) 643 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:25:39.535 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:39 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:38.592788+0000 mon.a (mon.0) 644 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:25:39.535 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:39 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:25:37.811520+0000 mgr.y (mgr.14577) 25 : cluster [DBG] pgmap v14: 105 pgs: 8 unknown, 97 active+clean; 3.0 KiB data, 44 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:25:39.536 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:39 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:25:38.142080+0000 mon.a (mon.0) 639 : cluster [DBG] osdmap e55: 8 total, 8 up, 8 in 2022-09-29T05:25:39.536 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:39 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:38.381896+0000 mon.c (mon.1) 24 : audit [INF] from='client.? 172.21.15.83:0/2058012094' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2022-09-29T05:25:39.536 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:39 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:38.382210+0000 mon.a (mon.0) 640 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2022-09-29T05:25:39.536 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:39 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:38.589992+0000 mon.a (mon.0) 641 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:25:39.537 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:39 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:38.591106+0000 mon.a (mon.0) 642 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:25:39.537 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:39 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:38.592152+0000 mon.a (mon.0) 643 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:25:39.537 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:39 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:38.592788+0000 mon.a (mon.0) 644 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:25:39.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:39 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:25:37.811520+0000 mgr.y (mgr.14577) 25 : cluster [DBG] pgmap v14: 105 pgs: 8 unknown, 97 active+clean; 3.0 KiB data, 44 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:25:39.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:39 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:25:38.142080+0000 mon.a (mon.0) 639 : cluster [DBG] osdmap e55: 8 total, 8 up, 8 in 2022-09-29T05:25:39.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:39 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:38.381896+0000 mon.c (mon.1) 24 : audit [INF] from='client.? 172.21.15.83:0/2058012094' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2022-09-29T05:25:39.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:39 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:38.382210+0000 mon.a (mon.0) 640 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2022-09-29T05:25:39.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:39 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:38.589992+0000 mon.a (mon.0) 641 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:25:39.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:39 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:38.591106+0000 mon.a (mon.0) 642 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:25:39.561 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:39 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:38.592152+0000 mon.a (mon.0) 643 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:25:39.561 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:39 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:38.592788+0000 mon.a (mon.0) 644 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:25:39.652 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2022-09-29T05:25:39.653 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2022-09-29T05:25:39.654 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/daemon-base:latest-pacific shell --fsid 69111522-3fb6-11ed-8431-001a4aab830c -- ceph mgr dump --format=json 2022-09-29T05:25:40.253 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/69111522-3fb6-11ed-8431-001a4aab830c/mon.a/config 2022-09-29T05:25:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:39.140749+0000 mon.a (mon.0) 645 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2022-09-29T05:25:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:25:39.140808+0000 mon.a (mon.0) 646 : cluster [DBG] osdmap e56: 8 total, 8 up, 8 in 2022-09-29T05:25:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:39.959595+0000 mon.a (mon.0) 647 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:25:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:39.140749+0000 mon.a (mon.0) 645 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2022-09-29T05:25:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:25:39.140808+0000 mon.a (mon.0) 646 : cluster [DBG] osdmap e56: 8 total, 8 up, 8 in 2022-09-29T05:25:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:39.959595+0000 mon.a (mon.0) 647 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:25:40.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:40 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:39.140749+0000 mon.a (mon.0) 645 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2022-09-29T05:25:40.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:40 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:25:39.140808+0000 mon.a (mon.0) 646 : cluster [DBG] osdmap e56: 8 total, 8 up, 8 in 2022-09-29T05:25:40.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:40 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:39.959595+0000 mon.a (mon.0) 647 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:25:41.445 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:25:39.812088+0000 mgr.y (mgr.14577) 26 : cluster [DBG] pgmap v17: 105 pgs: 8 unknown, 97 active+clean; 3.0 KiB data, 44 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:25:41.445 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:25:39.812088+0000 mgr.y (mgr.14577) 26 : cluster [DBG] pgmap v17: 105 pgs: 8 unknown, 97 active+clean; 3.0 KiB data, 44 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:25:41.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:41 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:25:39.812088+0000 mgr.y (mgr.14577) 26 : cluster [DBG] pgmap v17: 105 pgs: 8 unknown, 97 active+clean; 3.0 KiB data, 44 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:25:41.860 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:25:42.557 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:25:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:25:42] "GET /metrics HTTP/1.1" 200 173489 "" "Prometheus/2.33.4" 2022-09-29T05:25:42.559 INFO:teuthology.orchestra.run.smithi083.stdout:{"epoch":18,"active_gid":14577,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6800","nonce":2966685565},{"type":"v1","addr":"172.21.15.83:6801","nonce":2966685565}]},"active_addr":"172.21.15.83:6801/2966685565","active_change":"2022-09-29T05:25:19.747318+0000","active_mgr_features":4540138297136906239,"available":true,"standbys":[{"gid":24452,"name":"x","mgr_features":4540138297136906239,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"str","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"str","level":"advanced","flags":0,"default_value":"30","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"str","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"str","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"str","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"str","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"str","level":"advanced","flags":0,"default_value":"30","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"str","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"str","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"str","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.83:8443/","prometheus":"http://172.21.15.83:9283/"},"always_on_modules":{"nautilus":["balancer","crash","devicehealth","orchestrator_cli","progress","rbd_support","status","volumes"],"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"last_failure_osd_epoch":52,"active_clients":[{"addrvec":[{"type":"v2","addr":"172.21.15.83:0","nonce":3550954306}]},{"addrvec":[{"type":"v2","addr":"172.21.15.83:0","nonce":1770843109}]},{"addrvec":[{"type":"v2","addr":"172.21.15.83:0","nonce":1598359380}]},{"addrvec":[{"type":"v2","addr":"172.21.15.83:0","nonce":3041220294}]}]}} 2022-09-29T05:25:42.564 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2022-09-29T05:25:42.564 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2022-09-29T05:25:42.565 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/daemon-base:latest-pacific shell --fsid 69111522-3fb6-11ed-8431-001a4aab830c -- ceph osd dump --format=json 2022-09-29T05:25:42.566 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:25:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[40232]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:25:42] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:25:42.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:41.731426+0000 mon.a (mon.0) 648 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:25:42.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:25:41.812613+0000 mgr.y (mgr.14577) 27 : cluster [DBG] pgmap v18: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:25:42.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:41.850580+0000 mon.a (mon.0) 649 : audit [DBG] from='client.? 172.21.15.83:0/382466294' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2022-09-29T05:25:42.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:42.258535+0000 mon.a (mon.0) 650 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:25:42.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:42.266591+0000 mon.a (mon.0) 651 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:25:42.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:25:42.274761+0000 mgr.y (mgr.14577) 28 : cephadm [INF] Deploying daemon alertmanager.a on smithi083 2022-09-29T05:25:42.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:41.731426+0000 mon.a (mon.0) 648 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:25:42.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:25:41.812613+0000 mgr.y (mgr.14577) 27 : cluster [DBG] pgmap v18: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:25:42.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:41.850580+0000 mon.a (mon.0) 649 : audit [DBG] from='client.? 172.21.15.83:0/382466294' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2022-09-29T05:25:42.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:42.258535+0000 mon.a (mon.0) 650 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:25:42.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:42.266591+0000 mon.a (mon.0) 651 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:25:42.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:25:42.274761+0000 mgr.y (mgr.14577) 28 : cephadm [INF] Deploying daemon alertmanager.a on smithi083 2022-09-29T05:25:43.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:41.731426+0000 mon.a (mon.0) 648 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:25:43.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:25:41.812613+0000 mgr.y (mgr.14577) 27 : cluster [DBG] pgmap v18: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:25:43.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:41.850580+0000 mon.a (mon.0) 649 : audit [DBG] from='client.? 172.21.15.83:0/382466294' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2022-09-29T05:25:43.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:42.258535+0000 mon.a (mon.0) 650 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:25:43.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:42.266591+0000 mon.a (mon.0) 651 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:25:43.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:25:42.274761+0000 mgr.y (mgr.14577) 28 : cephadm [INF] Deploying daemon alertmanager.a on smithi083 2022-09-29T05:25:43.183 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/69111522-3fb6-11ed-8431-001a4aab830c/mon.a/config 2022-09-29T05:25:45.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:25:43.812963+0000 mgr.y (mgr.14577) 29 : cluster [DBG] pgmap v19: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:25:45.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:25:43.812963+0000 mgr.y (mgr.14577) 29 : cluster [DBG] pgmap v19: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:25:45.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:44 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:25:43.812963+0000 mgr.y (mgr.14577) 29 : cluster [DBG] pgmap v19: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:25:46.097 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:25:46.098 INFO:teuthology.orchestra.run.smithi083.stdout:{"epoch":56,"fsid":"69111522-3fb6-11ed-8431-001a4aab830c","created":"2022-09-29T05:20:45.037726+0000","modified":"2022-09-29T05:25:39.134644+0000","last_up_change":"2022-09-29T05:24:46.528124+0000","last_in_change":"2022-09-29T05:24:31.376431+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2022-09-29T05:23:34.795850+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"54","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2022-09-29T05:25:01.755735+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"50","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}}},{"pool":3,"pool_name":"cephfs_data","create_time":"2022-09-29T05:25:02.966273+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"51","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}},{"pool":4,"pool_name":"rbd","create_time":"2022-09-29T05:25:35.425789+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"56","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"9b8b79ad-3d00-4dad-8da7-7cff0e65d302","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6802","nonce":2080129456},{"type":"v1","addr":"172.21.15.83:6803","nonce":2080129456}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6804","nonce":2080129456},{"type":"v1","addr":"172.21.15.83:6805","nonce":2080129456}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6808","nonce":2080129456},{"type":"v1","addr":"172.21.15.83:6809","nonce":2080129456}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6806","nonce":2080129456},{"type":"v1","addr":"172.21.15.83:6807","nonce":2080129456}]},"public_addr":"172.21.15.83:6803/2080129456","cluster_addr":"172.21.15.83:6805/2080129456","heartbeat_back_addr":"172.21.15.83:6809/2080129456","heartbeat_front_addr":"172.21.15.83:6807/2080129456","state":["exists","up"]},{"osd":1,"uuid":"e098cff5-70cc-437e-8759-e6c7e3d452d2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6810","nonce":605787053},{"type":"v1","addr":"172.21.15.83:6811","nonce":605787053}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6812","nonce":605787053},{"type":"v1","addr":"172.21.15.83:6813","nonce":605787053}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6816","nonce":605787053},{"type":"v1","addr":"172.21.15.83:6817","nonce":605787053}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6814","nonce":605787053},{"type":"v1","addr":"172.21.15.83:6815","nonce":605787053}]},"public_addr":"172.21.15.83:6811/605787053","cluster_addr":"172.21.15.83:6813/605787053","heartbeat_back_addr":"172.21.15.83:6817/605787053","heartbeat_front_addr":"172.21.15.83:6815/605787053","state":["exists","up"]},{"osd":2,"uuid":"2c14853b-5b64-496d-a011-5d8a9b17b731","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6818","nonce":1928326991},{"type":"v1","addr":"172.21.15.83:6819","nonce":1928326991}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6820","nonce":1928326991},{"type":"v1","addr":"172.21.15.83:6821","nonce":1928326991}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6824","nonce":1928326991},{"type":"v1","addr":"172.21.15.83:6825","nonce":1928326991}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6822","nonce":1928326991},{"type":"v1","addr":"172.21.15.83:6823","nonce":1928326991}]},"public_addr":"172.21.15.83:6819/1928326991","cluster_addr":"172.21.15.83:6821/1928326991","heartbeat_back_addr":"172.21.15.83:6825/1928326991","heartbeat_front_addr":"172.21.15.83:6823/1928326991","state":["exists","up"]},{"osd":3,"uuid":"7336f148-261e-468e-8d3c-28f840150f36","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6826","nonce":54976712},{"type":"v1","addr":"172.21.15.83:6827","nonce":54976712}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6828","nonce":54976712},{"type":"v1","addr":"172.21.15.83:6829","nonce":54976712}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6832","nonce":54976712},{"type":"v1","addr":"172.21.15.83:6833","nonce":54976712}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6830","nonce":54976712},{"type":"v1","addr":"172.21.15.83:6831","nonce":54976712}]},"public_addr":"172.21.15.83:6827/54976712","cluster_addr":"172.21.15.83:6829/54976712","heartbeat_back_addr":"172.21.15.83:6833/54976712","heartbeat_front_addr":"172.21.15.83:6831/54976712","state":["exists","up"]},{"osd":4,"uuid":"247d5c19-e8af-4620-a378-699dc2000db1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6800","nonce":4053209633},{"type":"v1","addr":"172.21.15.153:6801","nonce":4053209633}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6802","nonce":4053209633},{"type":"v1","addr":"172.21.15.153:6803","nonce":4053209633}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6806","nonce":4053209633},{"type":"v1","addr":"172.21.15.153:6807","nonce":4053209633}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6804","nonce":4053209633},{"type":"v1","addr":"172.21.15.153:6805","nonce":4053209633}]},"public_addr":"172.21.15.153:6801/4053209633","cluster_addr":"172.21.15.153:6803/4053209633","heartbeat_back_addr":"172.21.15.153:6807/4053209633","heartbeat_front_addr":"172.21.15.153:6805/4053209633","state":["exists","up"]},{"osd":5,"uuid":"c69e2263-5dd8-402d-b901-295c1f4840e2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":35,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6808","nonce":2536485968},{"type":"v1","addr":"172.21.15.153:6809","nonce":2536485968}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6810","nonce":2536485968},{"type":"v1","addr":"172.21.15.153:6811","nonce":2536485968}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6814","nonce":2536485968},{"type":"v1","addr":"172.21.15.153:6815","nonce":2536485968}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6812","nonce":2536485968},{"type":"v1","addr":"172.21.15.153:6813","nonce":2536485968}]},"public_addr":"172.21.15.153:6809/2536485968","cluster_addr":"172.21.15.153:6811/2536485968","heartbeat_back_addr":"172.21.15.153:6815/2536485968","heartbeat_front_addr":"172.21.15.153:6813/2536485968","state":["exists","up"]},{"osd":6,"uuid":"72aa13b6-e7b7-449b-9fa8-f284e8f0735a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":40,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6816","nonce":3662388207},{"type":"v1","addr":"172.21.15.153:6817","nonce":3662388207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6818","nonce":3662388207},{"type":"v1","addr":"172.21.15.153:6819","nonce":3662388207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6822","nonce":3662388207},{"type":"v1","addr":"172.21.15.153:6823","nonce":3662388207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6820","nonce":3662388207},{"type":"v1","addr":"172.21.15.153:6821","nonce":3662388207}]},"public_addr":"172.21.15.153:6817/3662388207","cluster_addr":"172.21.15.153:6819/3662388207","heartbeat_back_addr":"172.21.15.153:6823/3662388207","heartbeat_front_addr":"172.21.15.153:6821/3662388207","state":["exists","up"]},{"osd":7,"uuid":"b5fdd156-d2dc-47fb-b44b-803275092ac0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":45,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6824","nonce":1223012606},{"type":"v1","addr":"172.21.15.153:6825","nonce":1223012606}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6826","nonce":1223012606},{"type":"v1","addr":"172.21.15.153:6827","nonce":1223012606}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6830","nonce":1223012606},{"type":"v1","addr":"172.21.15.153:6831","nonce":1223012606}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6828","nonce":1223012606},{"type":"v1","addr":"172.21.15.153:6829","nonce":1223012606}]},"public_addr":"172.21.15.153:6825/1223012606","cluster_addr":"172.21.15.153:6827/1223012606","heartbeat_back_addr":"172.21.15.153:6831/1223012606","heartbeat_front_addr":"172.21.15.153:6829/1223012606","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2022-09-29T05:22:43.518222+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2022-09-29T05:23:01.859214+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2022-09-29T05:23:20.325748+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2022-09-29T05:23:38.717068+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2022-09-29T05:23:54.756146+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2022-09-29T05:24:11.670499+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2022-09-29T05:24:28.641104+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2022-09-29T05:24:45.518895+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.83:6801/2790412241":"2022-09-30T05:25:19.746941+0000","172.21.15.83:0/4071118407":"2022-09-30T05:25:19.746941+0000","172.21.15.83:0/2269625481":"2022-09-30T05:21:35.319810+0000","172.21.15.83:0/2173437316":"2022-09-30T05:21:35.319810+0000","172.21.15.83:6800/2702658771":"2022-09-30T05:21:35.319810+0000","172.21.15.83:0/4005486955":"2022-09-30T05:21:15.866011+0000","172.21.15.83:6800/2790412241":"2022-09-30T05:25:19.746941+0000","172.21.15.83:6800/2006042278":"2022-09-30T05:21:15.866011+0000","172.21.15.83:6801/2006042278":"2022-09-30T05:21:15.866011+0000","172.21.15.83:0/664253330":"2022-09-30T05:25:19.746941+0000","172.21.15.83:6801/2702658771":"2022-09-30T05:21:35.319810+0000","172.21.15.83:0/4256024697":"2022-09-30T05:21:15.866011+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2022-09-29T05:25:46.849 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2022-09-29T05:25:46.850 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/daemon-base:latest-pacific shell --fsid 69111522-3fb6-11ed-8431-001a4aab830c -- ceph osd dump --format=json 2022-09-29T05:25:47.103 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:25:45.813300+0000 mgr.y (mgr.14577) 30 : cluster [DBG] pgmap v20: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:25:47.103 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:46.093834+0000 mon.c (mon.1) 25 : audit [DBG] from='client.? 172.21.15.83:0/2746071253' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2022-09-29T05:25:47.103 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:25:45.813300+0000 mgr.y (mgr.14577) 30 : cluster [DBG] pgmap v20: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:25:47.104 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:46.093834+0000 mon.c (mon.1) 25 : audit [DBG] from='client.? 172.21.15.83:0/2746071253' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2022-09-29T05:25:47.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:46 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:25:45.813300+0000 mgr.y (mgr.14577) 30 : cluster [DBG] pgmap v20: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:25:47.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:46 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:46.093834+0000 mon.c (mon.1) 25 : audit [DBG] from='client.? 172.21.15.83:0/2746071253' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2022-09-29T05:25:47.748 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/69111522-3fb6-11ed-8431-001a4aab830c/mon.a/config 2022-09-29T05:25:49.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:48 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:25:47.813775+0000 mgr.y (mgr.14577) 31 : cluster [DBG] pgmap v21: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:25:49.214 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:25:48 smithi083 podman[62554]: 2022-09-29T05:25:49.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:48 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:25:47.813775+0000 mgr.y (mgr.14577) 31 : cluster [DBG] pgmap v21: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:25:49.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:48 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:25:47.813775+0000 mgr.y (mgr.14577) 31 : cluster [DBG] pgmap v21: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:25:49.387 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:25:49.388 INFO:teuthology.orchestra.run.smithi083.stdout:{"epoch":56,"fsid":"69111522-3fb6-11ed-8431-001a4aab830c","created":"2022-09-29T05:20:45.037726+0000","modified":"2022-09-29T05:25:39.134644+0000","last_up_change":"2022-09-29T05:24:46.528124+0000","last_in_change":"2022-09-29T05:24:31.376431+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2022-09-29T05:23:34.795850+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"54","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2022-09-29T05:25:01.755735+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"50","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}}},{"pool":3,"pool_name":"cephfs_data","create_time":"2022-09-29T05:25:02.966273+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"51","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}},{"pool":4,"pool_name":"rbd","create_time":"2022-09-29T05:25:35.425789+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"56","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"9b8b79ad-3d00-4dad-8da7-7cff0e65d302","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6802","nonce":2080129456},{"type":"v1","addr":"172.21.15.83:6803","nonce":2080129456}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6804","nonce":2080129456},{"type":"v1","addr":"172.21.15.83:6805","nonce":2080129456}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6808","nonce":2080129456},{"type":"v1","addr":"172.21.15.83:6809","nonce":2080129456}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6806","nonce":2080129456},{"type":"v1","addr":"172.21.15.83:6807","nonce":2080129456}]},"public_addr":"172.21.15.83:6803/2080129456","cluster_addr":"172.21.15.83:6805/2080129456","heartbeat_back_addr":"172.21.15.83:6809/2080129456","heartbeat_front_addr":"172.21.15.83:6807/2080129456","state":["exists","up"]},{"osd":1,"uuid":"e098cff5-70cc-437e-8759-e6c7e3d452d2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6810","nonce":605787053},{"type":"v1","addr":"172.21.15.83:6811","nonce":605787053}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6812","nonce":605787053},{"type":"v1","addr":"172.21.15.83:6813","nonce":605787053}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6816","nonce":605787053},{"type":"v1","addr":"172.21.15.83:6817","nonce":605787053}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6814","nonce":605787053},{"type":"v1","addr":"172.21.15.83:6815","nonce":605787053}]},"public_addr":"172.21.15.83:6811/605787053","cluster_addr":"172.21.15.83:6813/605787053","heartbeat_back_addr":"172.21.15.83:6817/605787053","heartbeat_front_addr":"172.21.15.83:6815/605787053","state":["exists","up"]},{"osd":2,"uuid":"2c14853b-5b64-496d-a011-5d8a9b17b731","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6818","nonce":1928326991},{"type":"v1","addr":"172.21.15.83:6819","nonce":1928326991}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6820","nonce":1928326991},{"type":"v1","addr":"172.21.15.83:6821","nonce":1928326991}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6824","nonce":1928326991},{"type":"v1","addr":"172.21.15.83:6825","nonce":1928326991}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6822","nonce":1928326991},{"type":"v1","addr":"172.21.15.83:6823","nonce":1928326991}]},"public_addr":"172.21.15.83:6819/1928326991","cluster_addr":"172.21.15.83:6821/1928326991","heartbeat_back_addr":"172.21.15.83:6825/1928326991","heartbeat_front_addr":"172.21.15.83:6823/1928326991","state":["exists","up"]},{"osd":3,"uuid":"7336f148-261e-468e-8d3c-28f840150f36","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6826","nonce":54976712},{"type":"v1","addr":"172.21.15.83:6827","nonce":54976712}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6828","nonce":54976712},{"type":"v1","addr":"172.21.15.83:6829","nonce":54976712}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6832","nonce":54976712},{"type":"v1","addr":"172.21.15.83:6833","nonce":54976712}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6830","nonce":54976712},{"type":"v1","addr":"172.21.15.83:6831","nonce":54976712}]},"public_addr":"172.21.15.83:6827/54976712","cluster_addr":"172.21.15.83:6829/54976712","heartbeat_back_addr":"172.21.15.83:6833/54976712","heartbeat_front_addr":"172.21.15.83:6831/54976712","state":["exists","up"]},{"osd":4,"uuid":"247d5c19-e8af-4620-a378-699dc2000db1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6800","nonce":4053209633},{"type":"v1","addr":"172.21.15.153:6801","nonce":4053209633}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6802","nonce":4053209633},{"type":"v1","addr":"172.21.15.153:6803","nonce":4053209633}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6806","nonce":4053209633},{"type":"v1","addr":"172.21.15.153:6807","nonce":4053209633}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6804","nonce":4053209633},{"type":"v1","addr":"172.21.15.153:6805","nonce":4053209633}]},"public_addr":"172.21.15.153:6801/4053209633","cluster_addr":"172.21.15.153:6803/4053209633","heartbeat_back_addr":"172.21.15.153:6807/4053209633","heartbeat_front_addr":"172.21.15.153:6805/4053209633","state":["exists","up"]},{"osd":5,"uuid":"c69e2263-5dd8-402d-b901-295c1f4840e2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":35,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6808","nonce":2536485968},{"type":"v1","addr":"172.21.15.153:6809","nonce":2536485968}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6810","nonce":2536485968},{"type":"v1","addr":"172.21.15.153:6811","nonce":2536485968}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6814","nonce":2536485968},{"type":"v1","addr":"172.21.15.153:6815","nonce":2536485968}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6812","nonce":2536485968},{"type":"v1","addr":"172.21.15.153:6813","nonce":2536485968}]},"public_addr":"172.21.15.153:6809/2536485968","cluster_addr":"172.21.15.153:6811/2536485968","heartbeat_back_addr":"172.21.15.153:6815/2536485968","heartbeat_front_addr":"172.21.15.153:6813/2536485968","state":["exists","up"]},{"osd":6,"uuid":"72aa13b6-e7b7-449b-9fa8-f284e8f0735a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":40,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6816","nonce":3662388207},{"type":"v1","addr":"172.21.15.153:6817","nonce":3662388207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6818","nonce":3662388207},{"type":"v1","addr":"172.21.15.153:6819","nonce":3662388207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6822","nonce":3662388207},{"type":"v1","addr":"172.21.15.153:6823","nonce":3662388207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6820","nonce":3662388207},{"type":"v1","addr":"172.21.15.153:6821","nonce":3662388207}]},"public_addr":"172.21.15.153:6817/3662388207","cluster_addr":"172.21.15.153:6819/3662388207","heartbeat_back_addr":"172.21.15.153:6823/3662388207","heartbeat_front_addr":"172.21.15.153:6821/3662388207","state":["exists","up"]},{"osd":7,"uuid":"b5fdd156-d2dc-47fb-b44b-803275092ac0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":45,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6824","nonce":1223012606},{"type":"v1","addr":"172.21.15.153:6825","nonce":1223012606}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6826","nonce":1223012606},{"type":"v1","addr":"172.21.15.153:6827","nonce":1223012606}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6830","nonce":1223012606},{"type":"v1","addr":"172.21.15.153:6831","nonce":1223012606}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6828","nonce":1223012606},{"type":"v1","addr":"172.21.15.153:6829","nonce":1223012606}]},"public_addr":"172.21.15.153:6825/1223012606","cluster_addr":"172.21.15.153:6827/1223012606","heartbeat_back_addr":"172.21.15.153:6831/1223012606","heartbeat_front_addr":"172.21.15.153:6829/1223012606","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2022-09-29T05:22:43.518222+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2022-09-29T05:23:01.859214+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2022-09-29T05:23:20.325748+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2022-09-29T05:23:38.717068+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2022-09-29T05:23:54.756146+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2022-09-29T05:24:11.670499+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2022-09-29T05:24:28.641104+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2022-09-29T05:24:45.518895+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.83:6801/2790412241":"2022-09-30T05:25:19.746941+0000","172.21.15.83:0/4071118407":"2022-09-30T05:25:19.746941+0000","172.21.15.83:0/2269625481":"2022-09-30T05:21:35.319810+0000","172.21.15.83:0/2173437316":"2022-09-30T05:21:35.319810+0000","172.21.15.83:6800/2702658771":"2022-09-30T05:21:35.319810+0000","172.21.15.83:0/4005486955":"2022-09-30T05:21:15.866011+0000","172.21.15.83:6800/2790412241":"2022-09-30T05:25:19.746941+0000","172.21.15.83:6800/2006042278":"2022-09-30T05:21:15.866011+0000","172.21.15.83:6801/2006042278":"2022-09-30T05:21:15.866011+0000","172.21.15.83:0/664253330":"2022-09-30T05:25:19.746941+0000","172.21.15.83:6801/2702658771":"2022-09-30T05:21:35.319810+0000","172.21.15.83:0/4256024697":"2022-09-30T05:21:15.866011+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2022-09-29T05:25:49.552 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:25:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[62589]: level=info ts=2022-09-29T05:25:49.229Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2022-09-29T05:25:49.552 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:25:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[62589]: level=info ts=2022-09-29T05:25:49.229Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2022-09-29T05:25:49.552 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:25:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[62589]: level=info ts=2022-09-29T05:25:49.230Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.83 port=9094 2022-09-29T05:25:49.553 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:25:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[62589]: level=info ts=2022-09-29T05:25:49.231Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2022-09-29T05:25:49.553 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:25:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[62589]: level=info ts=2022-09-29T05:25:49.262Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2022-09-29T05:25:49.553 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:25:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[62589]: level=info ts=2022-09-29T05:25:49.262Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2022-09-29T05:25:49.553 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:25:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[62589]: level=info ts=2022-09-29T05:25:49.264Z caller=main.go:518 msg=Listening address=:9093 2022-09-29T05:25:49.554 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:25:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[62589]: level=info ts=2022-09-29T05:25:49.264Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2022-09-29T05:25:49.554 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:25:49 smithi083 bash[62554]: f49ce68e8e50bc821467e8986edfe1078ceb6820f04294e93a30e3736d12c8ba 2022-09-29T05:25:49.554 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:25:49 smithi083 systemd[1]: Started Ceph alertmanager.a for 69111522-3fb6-11ed-8431-001a4aab830c. 2022-09-29T05:25:49.821 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:25:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: [29/Sep/2022:05:25:49] ENGINE Bus STOPPING 2022-09-29T05:25:49.822 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:25:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: [29/Sep/2022:05:25:49] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2022-09-29T05:25:49.822 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:25:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: [29/Sep/2022:05:25:49] ENGINE Bus STOPPED 2022-09-29T05:25:49.822 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:25:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: [29/Sep/2022:05:25:49] ENGINE Bus STARTING 2022-09-29T05:25:49.822 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:25:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: [29/Sep/2022:05:25:49] ENGINE Serving on http://:::9283 2022-09-29T05:25:49.823 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:25:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: [29/Sep/2022:05:25:49] ENGINE Bus STARTED 2022-09-29T05:25:49.859 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/daemon-base:latest-pacific shell --fsid 69111522-3fb6-11ed-8431-001a4aab830c -- ceph tell osd.0 flush_pg_stats 2022-09-29T05:25:49.860 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/daemon-base:latest-pacific shell --fsid 69111522-3fb6-11ed-8431-001a4aab830c -- ceph tell osd.1 flush_pg_stats 2022-09-29T05:25:49.860 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/daemon-base:latest-pacific shell --fsid 69111522-3fb6-11ed-8431-001a4aab830c -- ceph tell osd.2 flush_pg_stats 2022-09-29T05:25:49.861 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/daemon-base:latest-pacific shell --fsid 69111522-3fb6-11ed-8431-001a4aab830c -- ceph tell osd.3 flush_pg_stats 2022-09-29T05:25:49.862 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/daemon-base:latest-pacific shell --fsid 69111522-3fb6-11ed-8431-001a4aab830c -- ceph tell osd.4 flush_pg_stats 2022-09-29T05:25:49.862 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/daemon-base:latest-pacific shell --fsid 69111522-3fb6-11ed-8431-001a4aab830c -- ceph tell osd.5 flush_pg_stats 2022-09-29T05:25:49.863 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/daemon-base:latest-pacific shell --fsid 69111522-3fb6-11ed-8431-001a4aab830c -- ceph tell osd.6 flush_pg_stats 2022-09-29T05:25:49.864 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/daemon-base:latest-pacific shell --fsid 69111522-3fb6-11ed-8431-001a4aab830c -- ceph tell osd.7 flush_pg_stats 2022-09-29T05:25:50.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:49.345014+0000 mon.a (mon.0) 652 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:25:50.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:49.383245+0000 mon.c (mon.1) 26 : audit [DBG] from='client.? 172.21.15.83:0/1273429349' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2022-09-29T05:25:50.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:49.527535+0000 mon.a (mon.0) 653 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:25:50.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:49.532917+0000 mon.a (mon.0) 654 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:25:50.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:49.535564+0000 mon.a (mon.0) 655 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2022-09-29T05:25:50.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:49.535940+0000 mgr.y (mgr.14577) 32 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2022-09-29T05:25:50.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:49.540105+0000 mon.a (mon.0) 656 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:25:50.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:25:49.544108+0000 mgr.y (mgr.14577) 33 : cephadm [INF] Deploying daemon grafana.a on smithi153 2022-09-29T05:25:50.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:25:50.022122+0000 mon.a (mon.0) 657 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:25:50.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:49.345014+0000 mon.a (mon.0) 652 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:25:50.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:49.383245+0000 mon.c (mon.1) 26 : audit [DBG] from='client.? 172.21.15.83:0/1273429349' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2022-09-29T05:25:50.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:49.527535+0000 mon.a (mon.0) 653 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:25:50.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:49.532917+0000 mon.a (mon.0) 654 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:25:50.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:49.535564+0000 mon.a (mon.0) 655 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2022-09-29T05:25:50.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:49.535940+0000 mgr.y (mgr.14577) 32 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2022-09-29T05:25:50.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:49.540105+0000 mon.a (mon.0) 656 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:25:50.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:25:49.544108+0000 mgr.y (mgr.14577) 33 : cephadm [INF] Deploying daemon grafana.a on smithi153 2022-09-29T05:25:50.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:25:50.022122+0000 mon.a (mon.0) 657 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:25:50.733 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/69111522-3fb6-11ed-8431-001a4aab830c/mon.a/config 2022-09-29T05:25:50.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:50 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:49.345014+0000 mon.a (mon.0) 652 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:25:50.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:50 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:49.383245+0000 mon.c (mon.1) 26 : audit [DBG] from='client.? 172.21.15.83:0/1273429349' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2022-09-29T05:25:50.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:50 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:49.527535+0000 mon.a (mon.0) 653 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:25:50.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:50 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:49.532917+0000 mon.a (mon.0) 654 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:25:50.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:50 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:49.535564+0000 mon.a (mon.0) 655 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2022-09-29T05:25:50.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:50 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:49.535940+0000 mgr.y (mgr.14577) 32 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2022-09-29T05:25:50.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:50 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:49.540105+0000 mon.a (mon.0) 656 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:25:50.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:50 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:25:49.544108+0000 mgr.y (mgr.14577) 33 : cephadm [INF] Deploying daemon grafana.a on smithi153 2022-09-29T05:25:50.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:50 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:25:50.022122+0000 mon.a (mon.0) 657 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:25:50.849 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/69111522-3fb6-11ed-8431-001a4aab830c/mon.a/config 2022-09-29T05:25:51.065 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/69111522-3fb6-11ed-8431-001a4aab830c/mon.a/config 2022-09-29T05:25:51.123 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/69111522-3fb6-11ed-8431-001a4aab830c/mon.a/config 2022-09-29T05:25:51.137 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/69111522-3fb6-11ed-8431-001a4aab830c/mon.a/config 2022-09-29T05:25:51.350 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/69111522-3fb6-11ed-8431-001a4aab830c/mon.a/config 2022-09-29T05:25:51.542 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:25:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[62589]: level=info ts=2022-09-29T05:25:51.232Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000443988s 2022-09-29T05:25:51.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:51 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:25:49.814053+0000 mgr.y (mgr.14577) 34 : cluster [DBG] pgmap v22: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:25:51.829 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:25:49.814053+0000 mgr.y (mgr.14577) 34 : cluster [DBG] pgmap v22: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:25:51.829 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:25:49.814053+0000 mgr.y (mgr.14577) 34 : cluster [DBG] pgmap v22: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:25:51.992 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/69111522-3fb6-11ed-8431-001a4aab830c/mon.a/config 2022-09-29T05:25:52.253 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/69111522-3fb6-11ed-8431-001a4aab830c/mon.a/config 2022-09-29T05:25:52.559 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:25:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[40232]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:25:52] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:25:52.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:25:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:25:52] "GET /metrics HTTP/1.1" 200 174574 "" "Prometheus/2.33.4" 2022-09-29T05:25:53.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:25:51.814612+0000 mgr.y (mgr.14577) 35 : cluster [DBG] pgmap v23: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:25:53.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:25:51.814612+0000 mgr.y (mgr.14577) 35 : cluster [DBG] pgmap v23: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:25:53.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:25:51.814612+0000 mgr.y (mgr.14577) 35 : cluster [DBG] pgmap v23: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:25:55.702 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:25:53.814938+0000 mgr.y (mgr.14577) 36 : cluster [DBG] pgmap v24: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:25:55.703 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:25:53.814938+0000 mgr.y (mgr.14577) 36 : cluster [DBG] pgmap v24: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:25:55.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:55 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:25:53.814938+0000 mgr.y (mgr.14577) 36 : cluster [DBG] pgmap v24: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:25:57.657 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:57 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:25:55.815541+0000 mgr.y (mgr.14577) 37 : cluster [DBG] pgmap v25: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:25:57.658 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:57 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:25:55.815541+0000 mgr.y (mgr.14577) 37 : cluster [DBG] pgmap v25: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:25:57.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:57 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:25:55.815541+0000 mgr.y (mgr.14577) 37 : cluster [DBG] pgmap v25: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:25:58.884 INFO:teuthology.orchestra.run.smithi083.stdout:77309411361 2022-09-29T05:25:58.885 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/daemon-base:latest-pacific shell --fsid 69111522-3fb6-11ed-8431-001a4aab830c -- ceph osd last-stat-seq osd.2 2022-09-29T05:25:59.330 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:25:59 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[62589]: level=info ts=2022-09-29T05:25:59.234Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.002953611s 2022-09-29T05:25:59.456 INFO:teuthology.orchestra.run.smithi083.stdout:193273528337 2022-09-29T05:25:59.457 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/daemon-base:latest-pacific shell --fsid 69111522-3fb6-11ed-8431-001a4aab830c -- ceph osd last-stat-seq osd.7 2022-09-29T05:25:59.626 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:25:59 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:25:57.816080+0000 mgr.y (mgr.14577) 38 : cluster [DBG] pgmap v26: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:25:59.626 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:25:59 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:25:57.816080+0000 mgr.y (mgr.14577) 38 : cluster [DBG] pgmap v26: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:25:59.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:25:59 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:25:57.816080+0000 mgr.y (mgr.14577) 38 : cluster [DBG] pgmap v26: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:25:59.920 INFO:teuthology.orchestra.run.smithi083.stdout:150323855384 2022-09-29T05:25:59.921 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/daemon-base:latest-pacific shell --fsid 69111522-3fb6-11ed-8431-001a4aab830c -- ceph osd last-stat-seq osd.5 2022-09-29T05:25:59.990 INFO:teuthology.orchestra.run.smithi083.stdout:55834574884 2022-09-29T05:25:59.990 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/daemon-base:latest-pacific shell --fsid 69111522-3fb6-11ed-8431-001a4aab830c -- ceph osd last-stat-seq osd.1 2022-09-29T05:26:00.310 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:00 smithi153 systemd[1]: Starting Ceph grafana.a for 69111522-3fb6-11ed-8431-001a4aab830c... 2022-09-29T05:26:00.594 INFO:teuthology.orchestra.run.smithi083.stdout:128849018905 2022-09-29T05:26:00.595 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/daemon-base:latest-pacific shell --fsid 69111522-3fb6-11ed-8431-001a4aab830c -- ceph osd last-stat-seq osd.4 2022-09-29T05:26:01.077 INFO:teuthology.orchestra.run.smithi083.stdout:171798691859 2022-09-29T05:26:01.077 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/daemon-base:latest-pacific shell --fsid 69111522-3fb6-11ed-8431-001a4aab830c -- ceph osd last-stat-seq osd.6 2022-09-29T05:26:01.110 INFO:teuthology.orchestra.run.smithi083.stdout:111669149725 2022-09-29T05:26:01.111 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/daemon-base:latest-pacific shell --fsid 69111522-3fb6-11ed-8431-001a4aab830c -- ceph osd last-stat-seq osd.3 2022-09-29T05:26:01.130 INFO:teuthology.orchestra.run.smithi083.stdout:34359738408 2022-09-29T05:26:01.131 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/daemon-base:latest-pacific shell --fsid 69111522-3fb6-11ed-8431-001a4aab830c -- ceph osd last-stat-seq osd.0 2022-09-29T05:26:01.383 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/69111522-3fb6-11ed-8431-001a4aab830c/mon.a/config 2022-09-29T05:26:01.410 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/69111522-3fb6-11ed-8431-001a4aab830c/mon.a/config 2022-09-29T05:26:01.533 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/69111522-3fb6-11ed-8431-001a4aab830c/mon.a/config 2022-09-29T05:26:01.534 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/69111522-3fb6-11ed-8431-001a4aab830c/mon.a/config 2022-09-29T05:26:01.701 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/69111522-3fb6-11ed-8431-001a4aab830c/mon.a/config 2022-09-29T05:26:01.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:01 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:25:59.816462+0000 mgr.y (mgr.14577) 39 : cluster [DBG] pgmap v27: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:26:01.812 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:25:59.816462+0000 mgr.y (mgr.14577) 39 : cluster [DBG] pgmap v27: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:26:01.812 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:25:59.816462+0000 mgr.y (mgr.14577) 39 : cluster [DBG] pgmap v27: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:26:01.822 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/69111522-3fb6-11ed-8431-001a4aab830c/mon.a/config 2022-09-29T05:26:02.261 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[40232]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:26:02] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:26:02.262 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 podman[56299]: 2022-09-29T05:26:02.262 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 bash[56299]: faa72bb81473dd5c31264c35ee3bef828b1430104f7b0c6d4cdbb9c27521736c 2022-09-29T05:26:02.262 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 systemd[1]: Started Ceph grafana.a for 69111522-3fb6-11ed-8431-001a4aab830c. 2022-09-29T05:26:02.263 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="The state of unified alerting is still not defined. The decision will be made during as we run the database migrations" logger=settings 2022-09-29T05:26:02.263 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=warn msg="falling back to legacy setting of 'min_interval_seconds'; please use the configuration option in the `unified_alerting` section if Grafana 8 alerts are enabled." logger=settings 2022-09-29T05:26:02.263 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Config loaded from" logger=settings file=/usr/share/grafana/conf/defaults.ini 2022-09-29T05:26:02.263 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Config loaded from" logger=settings file=/etc/grafana/grafana.ini 2022-09-29T05:26:02.264 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_DATA=/var/lib/grafana" 2022-09-29T05:26:02.264 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_LOGS=/var/log/grafana" 2022-09-29T05:26:02.264 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2022-09-29T05:26:02.264 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2022-09-29T05:26:02.265 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Path Home" logger=settings path=/usr/share/grafana 2022-09-29T05:26:02.265 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Path Data" logger=settings path=/var/lib/grafana 2022-09-29T05:26:02.265 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Path Logs" logger=settings path=/var/log/grafana 2022-09-29T05:26:02.265 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Path Plugins" logger=settings path=/var/lib/grafana/plugins 2022-09-29T05:26:02.266 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Path Provisioning" logger=settings path=/etc/grafana/provisioning 2022-09-29T05:26:02.266 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="App mode production" logger=settings 2022-09-29T05:26:02.266 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Connecting to DB" logger=sqlstore dbtype=sqlite3 2022-09-29T05:26:02.266 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=warn msg="SQLite database file has broader permissions than it should" logger=sqlstore path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2022-09-29T05:26:02.267 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Starting DB migrations" logger=migrator 2022-09-29T05:26:02.267 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="create migration_log table" 2022-09-29T05:26:02.267 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="create user table" 2022-09-29T05:26:02.267 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user.login" 2022-09-29T05:26:02.267 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user.email" 2022-09-29T05:26:02.268 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_user_login - v1" 2022-09-29T05:26:02.268 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_user_email - v1" 2022-09-29T05:26:02.268 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table user to user_v1 - v1" 2022-09-29T05:26:02.268 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="create user table v2" 2022-09-29T05:26:02.269 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_user_login - v2" 2022-09-29T05:26:02.269 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_user_email - v2" 2022-09-29T05:26:02.269 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="copy data_source v1 to v2" 2022-09-29T05:26:02.269 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table user_v1" 2022-09-29T05:26:02.270 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Add column help_flags1 to user table" 2022-09-29T05:26:02.270 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Update user table charset" 2022-09-29T05:26:02.270 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Add last_seen_at column to user" 2022-09-29T05:26:02.270 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Add missing user data" 2022-09-29T05:26:02.271 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Add is_disabled column to user" 2022-09-29T05:26:02.271 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Add index user.login/user.email" 2022-09-29T05:26:02.271 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Add is_service_account column to user" 2022-09-29T05:26:02.271 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="create temp user table v1-7" 2022-09-29T05:26:02.271 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_email - v1-7" 2022-09-29T05:26:02.272 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_org_id - v1-7" 2022-09-29T05:26:02.272 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_code - v1-7" 2022-09-29T05:26:02.494 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:26:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:26:02] "GET /metrics HTTP/1.1" 200 174574 "" "Prometheus/2.33.4" 2022-09-29T05:26:02.512 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_status - v1-7" 2022-09-29T05:26:02.513 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Update temp_user table charset" 2022-09-29T05:26:02.513 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_email - v1" 2022-09-29T05:26:02.513 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_org_id - v1" 2022-09-29T05:26:02.513 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_code - v1" 2022-09-29T05:26:02.514 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_status - v1" 2022-09-29T05:26:02.514 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2022-09-29T05:26:02.514 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="create temp_user v2" 2022-09-29T05:26:02.514 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_email - v2" 2022-09-29T05:26:02.515 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_org_id - v2" 2022-09-29T05:26:02.515 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_code - v2" 2022-09-29T05:26:02.515 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_status - v2" 2022-09-29T05:26:02.515 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="copy temp_user v1 to v2" 2022-09-29T05:26:02.515 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="drop temp_user_tmp_qwerty" 2022-09-29T05:26:02.516 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Set created for temp users that will otherwise prematurely expire" 2022-09-29T05:26:02.516 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="create star table" 2022-09-29T05:26:02.516 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index star.user_id_dashboard_id" 2022-09-29T05:26:02.516 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="create org table v1" 2022-09-29T05:26:02.517 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_org_name - v1" 2022-09-29T05:26:02.517 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="create org_user table v1" 2022-09-29T05:26:02.517 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_org_user_org_id - v1" 2022-09-29T05:26:02.517 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_org_user_org_id_user_id - v1" 2022-09-29T05:26:02.517 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_org_user_user_id - v1" 2022-09-29T05:26:02.518 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Update org table charset" 2022-09-29T05:26:02.518 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Update org_user table charset" 2022-09-29T05:26:02.518 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Migrate all Read Only Viewers to Viewers" 2022-09-29T05:26:02.518 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard table" 2022-09-29T05:26:02.519 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard.account_id" 2022-09-29T05:26:02.519 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_account_id_slug" 2022-09-29T05:26:02.519 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_tag table" 2022-09-29T05:26:02.519 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_tag.dasboard_id_term" 2022-09-29T05:26:02.519 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2022-09-29T05:26:02.520 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table dashboard to dashboard_v1 - v1" 2022-09-29T05:26:02.520 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard v2" 2022-09-29T05:26:02.520 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_org_id - v2" 2022-09-29T05:26:02.520 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_org_id_slug - v2" 2022-09-29T05:26:02.520 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="copy dashboard v1 to v2" 2022-09-29T05:26:02.521 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="drop table dashboard_v1" 2022-09-29T05:26:02.521 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard.data to mediumtext v1" 2022-09-29T05:26:02.521 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Add column updated_by in dashboard - v2" 2022-09-29T05:26:02.521 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Add column created_by in dashboard - v2" 2022-09-29T05:26:02.522 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Add column gnetId in dashboard" 2022-09-29T05:26:02.522 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for gnetId in dashboard" 2022-09-29T05:26:02.522 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Add column plugin_id in dashboard" 2022-09-29T05:26:02.522 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for plugin_id in dashboard" 2022-09-29T05:26:02.522 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_id in dashboard_tag" 2022-09-29T05:26:02.523 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard table charset" 2022-09-29T05:26:02.523 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard_tag table charset" 2022-09-29T05:26:02.523 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Add column folder_id in dashboard" 2022-09-29T05:26:02.523 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Add column isFolder in dashboard" 2022-09-29T05:26:02.524 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Add column has_acl in dashboard" 2022-09-29T05:26:02.524 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Add column uid in dashboard" 2022-09-29T05:26:02.524 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid column values in dashboard" 2022-09-29T05:26:02.524 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index dashboard_org_id_uid" 2022-09-29T05:26:02.524 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index org_id_slug" 2022-09-29T05:26:02.525 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard title length" 2022-09-29T05:26:02.525 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index for dashboard_org_id_title_folder_id" 2022-09-29T05:26:02.525 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_provisioning" 2022-09-29T05:26:02.525 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2022-09-29T05:26:02.525 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_provisioning v2" 2022-09-29T05:26:02.526 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2022-09-29T05:26:02.526 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2022-09-29T05:26:02.526 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="copy dashboard_provisioning v1 to v2" 2022-09-29T05:26:02.526 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="drop dashboard_provisioning_tmp_qwerty" 2022-09-29T05:26:02.527 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Add check_sum column" 2022-09-29T05:26:02.527 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_title" 2022-09-29T05:26:02.527 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="delete tags for deleted dashboards" 2022-09-29T05:26:02.527 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="delete stars for deleted dashboards" 2022-09-29T05:26:02.527 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_is_folder" 2022-09-29T05:26:02.528 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="create data_source table" 2022-09-29T05:26:02.528 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index data_source.account_id" 2022-09-29T05:26:02.528 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index data_source.account_id_name" 2022-09-29T05:26:02.528 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_data_source_account_id - v1" 2022-09-29T05:26:02.529 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_data_source_account_id_name - v1" 2022-09-29T05:26:02.529 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table data_source to data_source_v1 - v1" 2022-09-29T05:26:02.529 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="create data_source table v2" 2022-09-29T05:26:02.529 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_data_source_org_id - v2" 2022-09-29T05:26:02.529 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_data_source_org_id_name - v2" 2022-09-29T05:26:02.530 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="copy data_source v1 to v2" 2022-09-29T05:26:02.531 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table data_source_v1 #2" 2022-09-29T05:26:02.531 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Add column with_credentials" 2022-09-29T05:26:02.532 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Add secure json data column" 2022-09-29T05:26:02.532 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Update data_source table charset" 2022-09-29T05:26:02.532 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Update initial version to 1" 2022-09-29T05:26:02.532 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Add read_only data column" 2022-09-29T05:26:02.532 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Migrate logging ds to loki ds" 2022-09-29T05:26:02.533 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Update json_data with nulls" 2022-09-29T05:26:02.533 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Add uid column" 2022-09-29T05:26:02.533 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid value" 2022-09-29T05:26:02.533 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index datasource_org_id_uid" 2022-09-29T05:26:02.534 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index datasource_org_id_is_default" 2022-09-29T05:26:02.534 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="create api_key table" 2022-09-29T05:26:02.534 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.account_id" 2022-09-29T05:26:02.534 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.key" 2022-09-29T05:26:02.535 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.account_id_name" 2022-09-29T05:26:02.535 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_api_key_account_id - v1" 2022-09-29T05:26:02.535 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_api_key_key - v1" 2022-09-29T05:26:02.535 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_api_key_account_id_name - v1" 2022-09-29T05:26:02.535 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table api_key to api_key_v1 - v1" 2022-09-29T05:26:02.536 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="create api_key table v2" 2022-09-29T05:26:02.536 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_api_key_org_id - v2" 2022-09-29T05:26:02.536 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_api_key_key - v2" 2022-09-29T05:26:02.536 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_api_key_org_id_name - v2" 2022-09-29T05:26:02.537 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="copy api_key v1 to v2" 2022-09-29T05:26:02.537 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table api_key_v1" 2022-09-29T05:26:02.537 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Update api_key table charset" 2022-09-29T05:26:02.537 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Add expires to api_key table" 2022-09-29T05:26:02.537 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Add service account foreign key" 2022-09-29T05:26:02.538 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_snapshot table v4" 2022-09-29T05:26:02.538 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="drop table dashboard_snapshot_v4 #1" 2022-09-29T05:26:02.538 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_snapshot table v5 #2" 2022-09-29T05:26:02.538 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_snapshot_key - v5" 2022-09-29T05:26:02.539 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_snapshot_delete_key - v5" 2022-09-29T05:26:02.539 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_snapshot_user_id - v5" 2022-09-29T05:26:02.539 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard_snapshot to mediumtext v2" 2022-09-29T05:26:02.539 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard_snapshot table charset" 2022-09-29T05:26:02.539 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Add column external_delete_url to dashboard_snapshots table" 2022-09-29T05:26:02.540 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Add encrypted dashboard json column" 2022-09-29T05:26:02.540 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Change dashboard_encrypted column to MEDIUMBLOB" 2022-09-29T05:26:02.540 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="create quota table v1" 2022-09-29T05:26:02.540 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_quota_org_id_user_id_target - v1" 2022-09-29T05:26:02.541 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Update quota table charset" 2022-09-29T05:26:02.541 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="create plugin_setting table" 2022-09-29T05:26:02.541 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2022-09-29T05:26:02.541 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Add column plugin_version to plugin_settings" 2022-09-29T05:26:02.541 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Update plugin_setting table charset" 2022-09-29T05:26:02.542 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="create session table" 2022-09-29T05:26:02.542 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table playlist table" 2022-09-29T05:26:02.542 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table playlist_item table" 2022-09-29T05:26:02.542 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="create playlist table v2" 2022-09-29T05:26:02.543 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="create playlist item table v2" 2022-09-29T05:26:02.543 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Update playlist table charset" 2022-09-29T05:26:02.543 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Update playlist_item table charset" 2022-09-29T05:26:02.543 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="drop preferences table v2" 2022-09-29T05:26:02.543 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="drop preferences table v3" 2022-09-29T05:26:02.544 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="create preferences table v3" 2022-09-29T05:26:02.544 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Update preferences table charset" 2022-09-29T05:26:02.544 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Add column team_id in preferences" 2022-09-29T05:26:02.544 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Update team_id column values in preferences" 2022-09-29T05:26:02.545 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Add column week_start in preferences" 2022-09-29T05:26:02.545 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="create alert table v1" 2022-09-29T05:26:02.545 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert org_id & id " 2022-09-29T05:26:02.545 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert state" 2022-09-29T05:26:02.545 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert dashboard_id" 2022-09-29T05:26:02.546 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Create alert_rule_tag table v1" 2022-09-29T05:26:02.546 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index alert_rule_tag.alert_id_tag_id" 2022-09-29T05:26:02.546 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2022-09-29T05:26:02.546 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2022-09-29T05:26:02.546 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Create alert_rule_tag table v2" 2022-09-29T05:26:02.547 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2022-09-29T05:26:02.547 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="copy alert_rule_tag v1 to v2" 2022-09-29T05:26:02.547 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="drop table alert_rule_tag_v1" 2022-09-29T05:26:02.547 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_notification table v1" 2022-09-29T05:26:02.548 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Add column is_default" 2022-09-29T05:26:02.548 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Add column frequency" 2022-09-29T05:26:02.548 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Add column send_reminder" 2022-09-29T05:26:02.548 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Add column disable_resolve_message" 2022-09-29T05:26:02.548 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert_notification org_id & name" 2022-09-29T05:26:02.549 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert table charset" 2022-09-29T05:26:02.549 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert_notification table charset" 2022-09-29T05:26:02.549 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="create notification_journal table v1" 2022-09-29T05:26:02.549 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index notification_journal org_id & alert_id & notifier_id" 2022-09-29T05:26:02.550 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_notification_journal" 2022-09-29T05:26:02.550 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_notification_state table v1" 2022-09-29T05:26:02.550 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert_notification_state org_id & alert_id & notifier_id" 2022-09-29T05:26:02.550 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Add for to alert table" 2022-09-29T05:26:02.550 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Add column uid in alert_notification" 2022-09-29T05:26:02.551 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid column values in alert_notification" 2022-09-29T05:26:02.551 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index alert_notification_org_id_uid" 2022-09-29T05:26:02.551 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index org_id_name" 2022-09-29T05:26:02.551 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Add column secure_settings in alert_notification" 2022-09-29T05:26:02.552 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert.settings to mediumtext" 2022-09-29T05:26:02.552 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Add non-unique index alert_notification_state_alert_id" 2022-09-29T05:26:02.552 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Add non-unique index alert_rule_tag_alert_id" 2022-09-29T05:26:02.552 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old annotation table v4" 2022-09-29T05:26:02.552 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="create annotation table v5" 2022-09-29T05:26:02.553 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 0 v3" 2022-09-29T05:26:02.553 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 1 v3" 2022-09-29T05:26:02.553 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 2 v3" 2022-09-29T05:26:02.553 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 3 v3" 2022-09-29T05:26:02.553 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 4 v3" 2022-09-29T05:26:02.554 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Update annotation table charset" 2022-09-29T05:26:02.554 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Add column region_id to annotation table" 2022-09-29T05:26:02.554 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Drop category_id index" 2022-09-29T05:26:02.554 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Add column tags to annotation table" 2022-09-29T05:26:02.555 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Create annotation_tag table v2" 2022-09-29T05:26:02.555 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index annotation_tag.annotation_id_tag_id" 2022-09-29T05:26:02.555 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2022-09-29T05:26:02.555 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table annotation_tag to annotation_tag_v2 - v2" 2022-09-29T05:26:02.555 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Create annotation_tag table v3" 2022-09-29T05:26:02.556 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2022-09-29T05:26:02.556 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="copy annotation_tag v2 to v3" 2022-09-29T05:26:02.556 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="drop table annotation_tag_v2" 2022-09-29T05:26:02.556 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert annotations and set TEXT to empty" 2022-09-29T05:26:02.557 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Add created time to annotation table" 2022-09-29T05:26:02.557 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Add updated time to annotation table" 2022-09-29T05:26:02.557 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for created in annotation table" 2022-09-29T05:26:02.557 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for updated in annotation table" 2022-09-29T05:26:02.557 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Convert existing annotations from seconds to milliseconds" 2022-09-29T05:26:02.558 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Add epoch_end column" 2022-09-29T05:26:02.558 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for epoch_end" 2022-09-29T05:26:02.558 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Make epoch_end the same as epoch" 2022-09-29T05:26:02.558 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Move region to single row" 2022-09-29T05:26:02.559 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_epoch from annotation table" 2022-09-29T05:26:02.559 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2022-09-29T05:26:02.632 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/69111522-3fb6-11ed-8431-001a4aab830c/mon.a/config 2022-09-29T05:26:02.701 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/69111522-3fb6-11ed-8431-001a4aab830c/mon.a/config 2022-09-29T05:26:02.792 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2022-09-29T05:26:02.792 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for org_id_epoch_end_epoch on annotation table" 2022-09-29T05:26:02.793 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_epoch_epoch_end from annotation table" 2022-09-29T05:26:02.793 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for alert_id on annotation table" 2022-09-29T05:26:02.793 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="create test_data table" 2022-09-29T05:26:02.793 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_version table v1" 2022-09-29T05:26:02.793 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_version.dashboard_id" 2022-09-29T05:26:02.794 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2022-09-29T05:26:02.794 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Set dashboard version to 1 where 0" 2022-09-29T05:26:02.794 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="save existing dashboard data in dashboard_version table v1" 2022-09-29T05:26:02.794 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard_version.data to mediumtext v1" 2022-09-29T05:26:02.795 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="create team table" 2022-09-29T05:26:02.795 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index team.org_id" 2022-09-29T05:26:02.795 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_org_id_name" 2022-09-29T05:26:02.795 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="create team member table" 2022-09-29T05:26:02.796 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_member.org_id" 2022-09-29T05:26:02.796 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_member_org_id_team_id_user_id" 2022-09-29T05:26:02.796 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_member.team_id" 2022-09-29T05:26:02.796 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Add column email to team table" 2022-09-29T05:26:02.797 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Add column external to team_member table" 2022-09-29T05:26:02.797 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Add column permission to team_member table" 2022-09-29T05:26:02.797 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard acl table" 2022-09-29T05:26:02.797 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_dashboard_id" 2022-09-29T05:26:02.798 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_acl_dashboard_id_user_id" 2022-09-29T05:26:02.798 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_acl_dashboard_id_team_id" 2022-09-29T05:26:02.798 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_user_id" 2022-09-29T05:26:02.798 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_team_id" 2022-09-29T05:26:02.798 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_org_id_role" 2022-09-29T05:26:02.799 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_permission" 2022-09-29T05:26:02.799 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="save default acl rules in dashboard_acl table" 2022-09-29T05:26:02.799 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="delete acl rules for deleted dashboards and folders" 2022-09-29T05:26:02.799 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="create tag table" 2022-09-29T05:26:02.800 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index tag.key_value" 2022-09-29T05:26:02.800 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="create login attempt table" 2022-09-29T05:26:02.800 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index login_attempt.username" 2022-09-29T05:26:02.800 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_login_attempt_username - v1" 2022-09-29T05:26:02.801 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2022-09-29T05:26:02.801 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="create login_attempt v2" 2022-09-29T05:26:02.801 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_login_attempt_username - v2" 2022-09-29T05:26:02.801 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="copy login_attempt v1 to v2" 2022-09-29T05:26:02.801 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="drop login_attempt_tmp_qwerty" 2022-09-29T05:26:02.802 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="create user auth table" 2022-09-29T05:26:02.802 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_user_auth_auth_module_auth_id - v1" 2022-09-29T05:26:02.802 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="alter user_auth.auth_id to length 190" 2022-09-29T05:26:02.802 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth access token to user_auth" 2022-09-29T05:26:02.802 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth refresh token to user_auth" 2022-09-29T05:26:02.803 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth token type to user_auth" 2022-09-29T05:26:02.803 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth expiry to user_auth" 2022-09-29T05:26:02.803 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Add index to user_id column in user_auth" 2022-09-29T05:26:02.803 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="create server_lock table" 2022-09-29T05:26:02.804 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index server_lock.operation_uid" 2022-09-29T05:26:02.804 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="create user auth token table" 2022-09-29T05:26:02.804 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_auth_token.auth_token" 2022-09-29T05:26:02.804 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_auth_token.prev_auth_token" 2022-09-29T05:26:02.804 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_auth_token.user_id" 2022-09-29T05:26:02.805 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Add revoked_at to the user auth token" 2022-09-29T05:26:02.805 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="create cache_data table" 2022-09-29T05:26:02.805 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index cache_data.cache_key" 2022-09-29T05:26:02.805 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="create short_url table v1" 2022-09-29T05:26:02.806 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index short_url.org_id-uid" 2022-09-29T05:26:02.806 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="delete alert_definition table" 2022-09-29T05:26:02.806 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="recreate alert_definition table" 2022-09-29T05:26:02.806 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition on org_id and title columns" 2022-09-29T05:26:02.806 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition on org_id and uid columns" 2022-09-29T05:26:02.807 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_definition table data column to mediumtext in mysql" 2022-09-29T05:26:02.807 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="drop index in alert_definition on org_id and title columns" 2022-09-29T05:26:02.807 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="drop index in alert_definition on org_id and uid columns" 2022-09-29T05:26:02.807 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index in alert_definition on org_id and title columns" 2022-09-29T05:26:02.808 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index in alert_definition on org_id and uid columns" 2022-09-29T05:26:02.808 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Add column paused in alert_definition" 2022-09-29T05:26:02.808 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_definition table" 2022-09-29T05:26:02.808 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="delete alert_definition_version table" 2022-09-29T05:26:02.809 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="recreate alert_definition_version table" 2022-09-29T05:26:02.809 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition_version table on alert_definition_id and version columns" 2022-09-29T05:26:02.809 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition_version table on alert_definition_uid and version columns" 2022-09-29T05:26:02.809 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_definition_version table data column to mediumtext in mysql" 2022-09-29T05:26:02.809 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_definition_version table" 2022-09-29T05:26:02.810 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_instance table" 2022-09-29T05:26:02.810 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2022-09-29T05:26:02.810 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_instance table on def_org_id, current_state columns" 2022-09-29T05:26:02.810 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add column current_state_end to alert_instance" 2022-09-29T05:26:02.811 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="remove index def_org_id, def_uid, current_state on alert_instance" 2022-09-29T05:26:02.811 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="remove index def_org_id, current_state on alert_instance" 2022-09-29T05:26:02.811 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="rename def_org_id to rule_org_id in alert_instance" 2022-09-29T05:26:02.811 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="rename def_uid to rule_uid in alert_instance" 2022-09-29T05:26:02.812 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index rule_org_id, rule_uid, current_state on alert_instance" 2022-09-29T05:26:02.812 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index rule_org_id, current_state on alert_instance" 2022-09-29T05:26:02.812 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_rule table" 2022-09-29T05:26:02.812 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id and title columns" 2022-09-29T05:26:02.812 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id and uid columns" 2022-09-29T05:26:02.813 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2022-09-29T05:26:02.813 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_rule table data column to mediumtext in mysql" 2022-09-29T05:26:02.813 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add column for to alert_rule" 2022-09-29T05:26:02.813 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add column annotations to alert_rule" 2022-09-29T05:26:02.814 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add column labels to alert_rule" 2022-09-29T05:26:02.814 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="remove unique index from alert_rule on org_id, title columns" 2022-09-29T05:26:02.814 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, namespase_uid and title columns" 2022-09-29T05:26:02.814 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add dashboard_uid column to alert_rule" 2022-09-29T05:26:02.815 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add panel_id column to alert_rule" 2022-09-29T05:26:02.815 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2022-09-29T05:26:02.815 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_rule_version table" 2022-09-29T05:26:02.815 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2022-09-29T05:26:02.815 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2022-09-29T05:26:02.816 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_rule_version table data column to mediumtext in mysql" 2022-09-29T05:26:02.816 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add column for to alert_rule_version" 2022-09-29T05:26:02.816 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add column annotations to alert_rule_version" 2022-09-29T05:26:02.816 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add column labels to alert_rule_version" 2022-09-29T05:26:02.817 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id=create_alert_configuration_table 2022-09-29T05:26:02.817 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Add column default in alert_configuration" 2022-09-29T05:26:02.817 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2022-09-29T05:26:02.817 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add column org_id in alert_configuration" 2022-09-29T05:26:02.817 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_configuration table on org_id column" 2022-09-29T05:26:02.818 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id=create_ngalert_configuration_table 2022-09-29T05:26:02.818 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index in ngalert_configuration on org_id column" 2022-09-29T05:26:02.818 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="clear migration entry \"remove unified alerting data\"" 2022-09-29T05:26:02.818 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="move dashboard alerts to unified alerting" 2022-09-29T05:26:02.819 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="create library_element table v1" 2022-09-29T05:26:02.819 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index library_element org_id-folder_id-name-kind" 2022-09-29T05:26:02.819 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="create library_element_connection table v1" 2022-09-29T05:26:02.819 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index library_element_connection element_id-kind-connection_id" 2022-09-29T05:26:02.819 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index library_element org_id_uid" 2022-09-29T05:26:02.820 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="clone move dashboard alerts to unified alerting" 2022-09-29T05:26:02.820 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="create data_keys table" 2022-09-29T05:26:02.820 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="create kv_store table v1" 2022-09-29T05:26:02.820 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index kv_store.org_id-namespace-key" 2022-09-29T05:26:02.821 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="update dashboard_uid and panel_id from existing annotations" 2022-09-29T05:26:02.821 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="create permission table" 2022-09-29T05:26:02.821 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index permission.role_id" 2022-09-29T05:26:02.821 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role_id_action_scope" 2022-09-29T05:26:02.821 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="create role table" 2022-09-29T05:26:02.822 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add column display_name" 2022-09-29T05:26:02.822 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add column group_name" 2022-09-29T05:26:02.822 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index role.org_id" 2022-09-29T05:26:02.822 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role_org_id_name" 2022-09-29T05:26:02.822 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index role_org_id_uid" 2022-09-29T05:26:02.823 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="create team role table" 2022-09-29T05:26:02.823 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_role.org_id" 2022-09-29T05:26:02.823 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_role_org_id_team_id_role_id" 2022-09-29T05:26:02.823 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_role.team_id" 2022-09-29T05:26:02.824 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="create user role table" 2022-09-29T05:26:02.824 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_role.org_id" 2022-09-29T05:26:02.824 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_role_org_id_user_id_role_id" 2022-09-29T05:26:02.824 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_role.user_id" 2022-09-29T05:26:02.825 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="create builtin role table" 2022-09-29T05:26:02.825 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.role_id" 2022-09-29T05:26:02.825 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.name" 2022-09-29T05:26:02.825 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Add column org_id to builtin_role table" 2022-09-29T05:26:02.825 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.org_id" 2022-09-29T05:26:02.826 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index builtin_role_org_id_role_id_role" 2022-09-29T05:26:02.826 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index role_org_id_uid" 2022-09-29T05:26:02.826 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role.uid" 2022-09-29T05:26:02.826 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="create seed assignment table" 2022-09-29T05:26:02.827 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index builtin_role_role_name" 2022-09-29T05:26:02.827 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="migrations completed" logger=migrator performed=381 skipped=0 duration=488.065281ms 2022-09-29T05:26:02.827 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Created default organization" logger=sqlstore 2022-09-29T05:26:02.827 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Initialising plugins" logger=plugin.manager 2022-09-29T05:26:02.828 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=input 2022-09-29T05:26:02.828 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=grafana-piechart-panel 2022-09-29T05:26:02.828 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=vonage-status-panel 2022-09-29T05:26:03.059 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="Live Push Gateway initialization" logger=live.push_http 2022-09-29T05:26:03.059 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=warn msg="[Deprecated] the datasource provisioning config is outdated. please upgrade" logger=provisioning.datasources filename=/etc/grafana/provisioning/datasources/ceph-dashboard.yml 2022-09-29T05:26:03.060 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Dashboard1 uid=P43CA22E17D0F9596 2022-09-29T05:26:03.060 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="warming cache for startup" logger=ngalert 2022-09-29T05:26:03.060 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="HTTP Server Listen" logger=http.server address=[::]:3000 protocol=https subUrl= socket= 2022-09-29T05:26:03.060 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:26:02+0000 lvl=info msg="starting MultiOrg Alertmanager" logger=ngalert.multiorg.alertmanager 2022-09-29T05:26:03.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:26:01.817093+0000 mgr.y (mgr.14577) 40 : cluster [DBG] pgmap v28: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:26:03.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:26:02.298110+0000 mon.a (mon.0) 658 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:26:03.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:26:02.299162+0000 mon.a (mon.0) 659 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:26:03.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:26:02.300204+0000 mon.a (mon.0) 660 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:26:03.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:26:02.300840+0000 mon.a (mon.0) 661 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:26:03.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:26:01.817093+0000 mgr.y (mgr.14577) 40 : cluster [DBG] pgmap v28: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:26:03.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:26:02.298110+0000 mon.a (mon.0) 658 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:26:03.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:26:02.299162+0000 mon.a (mon.0) 659 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:26:03.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:26:02.300204+0000 mon.a (mon.0) 660 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:26:03.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:26:02.300840+0000 mon.a (mon.0) 661 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:26:03.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:26:01.817093+0000 mgr.y (mgr.14577) 40 : cluster [DBG] pgmap v28: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:26:03.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:26:02.298110+0000 mon.a (mon.0) 658 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:26:03.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:26:02.299162+0000 mon.a (mon.0) 659 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:26:03.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:26:02.300204+0000 mon.a (mon.0) 660 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:26:03.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:26:02.300840+0000 mon.a (mon.0) 661 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:26:05.010 INFO:teuthology.orchestra.run.smithi083.stdout:77309411362 2022-09-29T05:26:05.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:05 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:26:03.817615+0000 mgr.y (mgr.14577) 41 : cluster [DBG] pgmap v29: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:26:05.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:05 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:26:05.007544+0000 mon.a (mon.0) 662 : audit [DBG] from='client.? 172.21.15.83:0/377607505' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2022-09-29T05:26:05.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:05 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:26:05.075743+0000 mon.a (mon.0) 663 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:26:05.562 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:26:03.817615+0000 mgr.y (mgr.14577) 41 : cluster [DBG] pgmap v29: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:26:05.562 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:26:05.007544+0000 mon.a (mon.0) 662 : audit [DBG] from='client.? 172.21.15.83:0/377607505' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2022-09-29T05:26:05.562 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:26:05.075743+0000 mon.a (mon.0) 663 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:26:05.563 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:26:03.817615+0000 mgr.y (mgr.14577) 41 : cluster [DBG] pgmap v29: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:26:05.563 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:26:05.007544+0000 mon.a (mon.0) 662 : audit [DBG] from='client.? 172.21.15.83:0/377607505' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2022-09-29T05:26:05.563 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:26:05.075743+0000 mon.a (mon.0) 663 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:26:07.098 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411361 got 77309411362 for osd.2 2022-09-29T05:26:07.099 DEBUG:teuthology.parallel:result is None 2022-09-29T05:26:07.379 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:07 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:26:05.818129+0000 mgr.y (mgr.14577) 42 : cluster [DBG] pgmap v30: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:26:07.380 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:07 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:26:06.174333+0000 mon.a (mon.0) 664 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:26:07.380 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:07 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:26:05.818129+0000 mgr.y (mgr.14577) 42 : cluster [DBG] pgmap v30: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:26:07.381 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:07 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:26:06.174333+0000 mon.a (mon.0) 664 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:26:07.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:07 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:26:05.818129+0000 mgr.y (mgr.14577) 42 : cluster [DBG] pgmap v30: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:26:07.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:07 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:26:06.174333+0000 mon.a (mon.0) 664 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:26:07.840 INFO:teuthology.orchestra.run.smithi083.stdout:150323855385 2022-09-29T05:26:08.430 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:08 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:26:07.837709+0000 mon.a (mon.0) 665 : audit [DBG] from='client.? 172.21.15.83:0/3053138967' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2022-09-29T05:26:08.431 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:08 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:26:07.837709+0000 mon.a (mon.0) 665 : audit [DBG] from='client.? 172.21.15.83:0/3053138967' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2022-09-29T05:26:08.468 INFO:teuthology.orchestra.run.smithi083.stdout:55834574886 2022-09-29T05:26:08.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:08 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:26:07.837709+0000 mon.a (mon.0) 665 : audit [DBG] from='client.? 172.21.15.83:0/3053138967' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2022-09-29T05:26:08.853 INFO:teuthology.orchestra.run.smithi083.stdout:128849018907 2022-09-29T05:26:08.993 INFO:teuthology.orchestra.run.smithi083.stdout:34359738410 2022-09-29T05:26:09.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:09 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:26:07.818640+0000 mgr.y (mgr.14577) 43 : cluster [DBG] pgmap v31: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:26:09.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:09 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:26:08.465233+0000 mon.a (mon.0) 666 : audit [DBG] from='client.? 172.21.15.83:0/2643453670' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2022-09-29T05:26:09.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:09 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:26:08.850968+0000 mon.c (mon.1) 27 : audit [DBG] from='client.? 172.21.15.83:0/3495704069' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2022-09-29T05:26:09.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:09 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:26:08.989880+0000 mon.c (mon.1) 28 : audit [DBG] from='client.? 172.21.15.83:0/38156721' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2022-09-29T05:26:09.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:09 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:26:07.818640+0000 mgr.y (mgr.14577) 43 : cluster [DBG] pgmap v31: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:26:09.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:09 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:26:08.465233+0000 mon.a (mon.0) 666 : audit [DBG] from='client.? 172.21.15.83:0/2643453670' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2022-09-29T05:26:09.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:09 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:26:08.850968+0000 mon.c (mon.1) 27 : audit [DBG] from='client.? 172.21.15.83:0/3495704069' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2022-09-29T05:26:09.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:09 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:26:08.989880+0000 mon.c (mon.1) 28 : audit [DBG] from='client.? 172.21.15.83:0/38156721' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2022-09-29T05:26:09.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:09 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:26:07.818640+0000 mgr.y (mgr.14577) 43 : cluster [DBG] pgmap v31: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:26:09.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:09 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:26:08.465233+0000 mon.a (mon.0) 666 : audit [DBG] from='client.? 172.21.15.83:0/2643453670' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2022-09-29T05:26:09.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:09 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:26:08.850968+0000 mon.c (mon.1) 27 : audit [DBG] from='client.? 172.21.15.83:0/3495704069' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2022-09-29T05:26:09.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:09 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:26:08.989880+0000 mon.c (mon.1) 28 : audit [DBG] from='client.? 172.21.15.83:0/38156721' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2022-09-29T05:26:09.957 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574884 got 55834574886 for osd.1 2022-09-29T05:26:09.958 DEBUG:teuthology.parallel:result is None 2022-09-29T05:26:09.990 INFO:tasks.cephadm.ceph_manager.ceph:need seq 150323855384 got 150323855385 for osd.5 2022-09-29T05:26:09.990 DEBUG:teuthology.parallel:result is None 2022-09-29T05:26:10.013 INFO:teuthology.orchestra.run.smithi083.stdout:193273528339 2022-09-29T05:26:10.343 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849018905 got 128849018907 for osd.4 2022-09-29T05:26:10.343 DEBUG:teuthology.parallel:result is None 2022-09-29T05:26:10.979 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:26:09.818958+0000 mgr.y (mgr.14577) 44 : cluster [DBG] pgmap v32: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:26:10.979 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:26:10.010333+0000 mon.a (mon.0) 667 : audit [DBG] from='client.? 172.21.15.83:0/538194182' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2022-09-29T05:26:10.979 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:26:09.818958+0000 mgr.y (mgr.14577) 44 : cluster [DBG] pgmap v32: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:26:10.980 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:26:10.010333+0000 mon.a (mon.0) 667 : audit [DBG] from='client.? 172.21.15.83:0/538194182' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2022-09-29T05:26:10.981 INFO:tasks.cephadm.ceph_manager.ceph:need seq 193273528337 got 193273528339 for osd.7 2022-09-29T05:26:10.981 DEBUG:teuthology.parallel:result is None 2022-09-29T05:26:11.104 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738408 got 34359738410 for osd.0 2022-09-29T05:26:11.104 DEBUG:teuthology.parallel:result is None 2022-09-29T05:26:11.147 INFO:teuthology.orchestra.run.smithi083.stdout:111669149727 2022-09-29T05:26:11.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:10 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:26:09.818958+0000 mgr.y (mgr.14577) 44 : cluster [DBG] pgmap v32: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:26:11.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:10 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:26:10.010333+0000 mon.a (mon.0) 667 : audit [DBG] from='client.? 172.21.15.83:0/538194182' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2022-09-29T05:26:11.457 INFO:teuthology.orchestra.run.smithi083.stdout:171798691861 2022-09-29T05:26:12.015 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149725 got 111669149727 for osd.3 2022-09-29T05:26:12.016 DEBUG:teuthology.parallel:result is None 2022-09-29T05:26:12.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:26:11.144846+0000 mon.a (mon.0) 668 : audit [DBG] from='client.? 172.21.15.83:0/499032919' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2022-09-29T05:26:12.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:26:11.454505+0000 mon.a (mon.0) 669 : audit [DBG] from='client.? 172.21.15.83:0/635115371' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2022-09-29T05:26:12.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:26:11.144846+0000 mon.a (mon.0) 668 : audit [DBG] from='client.? 172.21.15.83:0/499032919' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2022-09-29T05:26:12.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:26:11.454505+0000 mon.a (mon.0) 669 : audit [DBG] from='client.? 172.21.15.83:0/635115371' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2022-09-29T05:26:12.223 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:11 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:26:11.144846+0000 mon.a (mon.0) 668 : audit [DBG] from='client.? 172.21.15.83:0/499032919' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2022-09-29T05:26:12.223 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:11 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:26:11.454505+0000 mon.a (mon.0) 669 : audit [DBG] from='client.? 172.21.15.83:0/635115371' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2022-09-29T05:26:12.330 INFO:tasks.cephadm.ceph_manager.ceph:need seq 171798691859 got 171798691861 for osd.6 2022-09-29T05:26:12.330 DEBUG:teuthology.parallel:result is None 2022-09-29T05:26:12.331 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2022-09-29T05:26:12.331 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/daemon-base:latest-pacific shell --fsid 69111522-3fb6-11ed-8431-001a4aab830c -- ceph pg dump --format=json 2022-09-29T05:26:12.559 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:26:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[40232]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:26:12] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:26:12.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:26:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:26:12] "GET /metrics HTTP/1.1" 200 174567 "" "Prometheus/2.33.4" 2022-09-29T05:26:13.025 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/69111522-3fb6-11ed-8431-001a4aab830c/mon.a/config 2022-09-29T05:26:13.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:26:11.819682+0000 mgr.y (mgr.14577) 45 : cluster [DBG] pgmap v33: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:26:13.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:26:12.679668+0000 mon.a (mon.0) 670 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:26:13.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:26:12.683498+0000 mon.a (mon.0) 671 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:26:13.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:26:11.819682+0000 mgr.y (mgr.14577) 45 : cluster [DBG] pgmap v33: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:26:13.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:26:12.679668+0000 mon.a (mon.0) 670 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:26:13.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:26:12.683498+0000 mon.a (mon.0) 671 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:26:13.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:26:11.819682+0000 mgr.y (mgr.14577) 45 : cluster [DBG] pgmap v33: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:26:13.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:26:12.679668+0000 mon.a (mon.0) 670 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:26:13.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:26:12.683498+0000 mon.a (mon.0) 671 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:26:14.144 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:26:12.686477+0000 mgr.y (mgr.14577) 46 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2022-09-29T05:26:14.144 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:26:12.689172+0000 mgr.y (mgr.14577) 47 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi083 2022-09-29T05:26:14.144 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:26:12.686477+0000 mgr.y (mgr.14577) 46 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2022-09-29T05:26:14.145 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:26:12.689172+0000 mgr.y (mgr.14577) 47 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi083 2022-09-29T05:26:14.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:26:12.686477+0000 mgr.y (mgr.14577) 46 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2022-09-29T05:26:14.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:26:12.689172+0000 mgr.y (mgr.14577) 47 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi083 2022-09-29T05:26:14.847 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:26:13.820107+0000 mgr.y (mgr.14577) 48 : cluster [DBG] pgmap v34: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:26:15.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:26:13.820107+0000 mgr.y (mgr.14577) 48 : cluster [DBG] pgmap v34: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:26:15.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:14 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:26:13.820107+0000 mgr.y (mgr.14577) 48 : cluster [DBG] pgmap v34: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:26:15.439 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:26:15.441 INFO:teuthology.orchestra.run.smithi083.stderr:dumped all 2022-09-29T05:26:15.852 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:26:15 smithi083 systemd[1]: Stopping Ceph alertmanager.a for 69111522-3fb6-11ed-8431-001a4aab830c... 2022-09-29T05:26:15.852 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:26:15 smithi083 bash[66963]: Error: no container with name or ID "ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager.a" found: no such container 2022-09-29T05:26:16.009 INFO:teuthology.orchestra.run.smithi083.stdout:{"pg_ready":true,"pg_map":{"version":34,"stamp":"2022-09-29T05:26:13.819921+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3070,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":48,"num_write_kb":16,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":42,"ondisk_log_size":42,"up":315,"acting":315,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":315,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":45972,"kb_used_data":4436,"kb_used_omap":0,"kb_used_meta":41472,"kb_avail":749915244,"statfs":{"total":767960285184,"available":767913209856,"internally_reserved":0,"allocated":4542464,"data_stored":1903601,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":42467328},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003310"},"pg_stats":[{"pgid":"3.3f","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.518942+0000","last_change":"2022-09-29T05:25:03.829987+0000","last_active":"2022-09-29T05:25:38.518942+0000","last_peered":"2022-09-29T05:25:38.518942+0000","last_clean":"2022-09-29T05:25:38.518942+0000","last_became_active":"2022-09-29T05:25:03.829703+0000","last_became_peered":"2022-09-29T05:25:03.829703+0000","last_unstale":"2022-09-29T05:25:38.518942+0000","last_undegraded":"2022-09-29T05:25:38.518942+0000","last_fullsized":"2022-09-29T05:25:38.518942+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3e","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.141088+0000","last_change":"2022-09-29T05:25:03.816168+0000","last_active":"2022-09-29T05:25:38.141088+0000","last_peered":"2022-09-29T05:25:38.141088+0000","last_clean":"2022-09-29T05:25:38.141088+0000","last_became_active":"2022-09-29T05:25:03.815888+0000","last_became_peered":"2022-09-29T05:25:03.815888+0000","last_unstale":"2022-09-29T05:25:38.141088+0000","last_undegraded":"2022-09-29T05:25:38.141088+0000","last_fullsized":"2022-09-29T05:25:38.141088+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3d","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.518173+0000","last_change":"2022-09-29T05:25:03.814221+0000","last_active":"2022-09-29T05:25:38.518173+0000","last_peered":"2022-09-29T05:25:38.518173+0000","last_clean":"2022-09-29T05:25:38.518173+0000","last_became_active":"2022-09-29T05:25:03.813927+0000","last_became_peered":"2022-09-29T05:25:03.813927+0000","last_unstale":"2022-09-29T05:25:38.518173+0000","last_undegraded":"2022-09-29T05:25:38.518173+0000","last_fullsized":"2022-09-29T05:25:38.518173+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3c","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.518789+0000","last_change":"2022-09-29T05:25:03.819554+0000","last_active":"2022-09-29T05:25:38.518789+0000","last_peered":"2022-09-29T05:25:38.518789+0000","last_clean":"2022-09-29T05:25:38.518789+0000","last_became_active":"2022-09-29T05:25:03.819207+0000","last_became_peered":"2022-09-29T05:25:03.819207+0000","last_unstale":"2022-09-29T05:25:38.518789+0000","last_undegraded":"2022-09-29T05:25:38.518789+0000","last_fullsized":"2022-09-29T05:25:38.518789+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3b","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.519460+0000","last_change":"2022-09-29T05:25:03.829034+0000","last_active":"2022-09-29T05:25:38.519460+0000","last_peered":"2022-09-29T05:25:38.519460+0000","last_clean":"2022-09-29T05:25:38.519460+0000","last_became_active":"2022-09-29T05:25:03.828384+0000","last_became_peered":"2022-09-29T05:25:03.828384+0000","last_unstale":"2022-09-29T05:25:38.519460+0000","last_undegraded":"2022-09-29T05:25:38.519460+0000","last_fullsized":"2022-09-29T05:25:38.519460+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3a","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.517990+0000","last_change":"2022-09-29T05:25:03.817871+0000","last_active":"2022-09-29T05:25:38.517990+0000","last_peered":"2022-09-29T05:25:38.517990+0000","last_clean":"2022-09-29T05:25:38.517990+0000","last_became_active":"2022-09-29T05:25:03.816800+0000","last_became_peered":"2022-09-29T05:25:03.816800+0000","last_unstale":"2022-09-29T05:25:38.517990+0000","last_undegraded":"2022-09-29T05:25:38.517990+0000","last_fullsized":"2022-09-29T05:25:38.517990+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.39","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.517638+0000","last_change":"2022-09-29T05:25:03.825279+0000","last_active":"2022-09-29T05:25:38.517638+0000","last_peered":"2022-09-29T05:25:38.517638+0000","last_clean":"2022-09-29T05:25:38.517638+0000","last_became_active":"2022-09-29T05:25:03.825002+0000","last_became_peered":"2022-09-29T05:25:03.825002+0000","last_unstale":"2022-09-29T05:25:38.517638+0000","last_undegraded":"2022-09-29T05:25:38.517638+0000","last_fullsized":"2022-09-29T05:25:38.517638+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.38","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.517254+0000","last_change":"2022-09-29T05:25:03.813803+0000","last_active":"2022-09-29T05:25:38.517254+0000","last_peered":"2022-09-29T05:25:38.517254+0000","last_clean":"2022-09-29T05:25:38.517254+0000","last_became_active":"2022-09-29T05:25:03.813488+0000","last_became_peered":"2022-09-29T05:25:03.813488+0000","last_unstale":"2022-09-29T05:25:38.517254+0000","last_undegraded":"2022-09-29T05:25:38.517254+0000","last_fullsized":"2022-09-29T05:25:38.517254+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.37","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.518868+0000","last_change":"2022-09-29T05:25:03.829880+0000","last_active":"2022-09-29T05:25:38.518868+0000","last_peered":"2022-09-29T05:25:38.518868+0000","last_clean":"2022-09-29T05:25:38.518868+0000","last_became_active":"2022-09-29T05:25:03.829318+0000","last_became_peered":"2022-09-29T05:25:03.829318+0000","last_unstale":"2022-09-29T05:25:38.518868+0000","last_undegraded":"2022-09-29T05:25:38.518868+0000","last_fullsized":"2022-09-29T05:25:38.518868+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,0],"acting":[7,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.36","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.517536+0000","last_change":"2022-09-29T05:25:03.823656+0000","last_active":"2022-09-29T05:25:38.517536+0000","last_peered":"2022-09-29T05:25:38.517536+0000","last_clean":"2022-09-29T05:25:38.517536+0000","last_became_active":"2022-09-29T05:25:03.823109+0000","last_became_peered":"2022-09-29T05:25:03.823109+0000","last_unstale":"2022-09-29T05:25:38.517536+0000","last_undegraded":"2022-09-29T05:25:38.517536+0000","last_fullsized":"2022-09-29T05:25:38.517536+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.35","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.518474+0000","last_change":"2022-09-29T05:25:03.824377+0000","last_active":"2022-09-29T05:25:38.518474+0000","last_peered":"2022-09-29T05:25:38.518474+0000","last_clean":"2022-09-29T05:25:38.518474+0000","last_became_active":"2022-09-29T05:25:03.823869+0000","last_became_peered":"2022-09-29T05:25:03.823869+0000","last_unstale":"2022-09-29T05:25:38.518474+0000","last_undegraded":"2022-09-29T05:25:38.518474+0000","last_fullsized":"2022-09-29T05:25:38.518474+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.34","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.136241+0000","last_change":"2022-09-29T05:25:03.812233+0000","last_active":"2022-09-29T05:25:38.136241+0000","last_peered":"2022-09-29T05:25:38.136241+0000","last_clean":"2022-09-29T05:25:38.136241+0000","last_became_active":"2022-09-29T05:25:03.811765+0000","last_became_peered":"2022-09-29T05:25:03.811765+0000","last_unstale":"2022-09-29T05:25:38.136241+0000","last_undegraded":"2022-09-29T05:25:38.136241+0000","last_fullsized":"2022-09-29T05:25:38.136241+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.33","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.517404+0000","last_change":"2022-09-29T05:25:03.813308+0000","last_active":"2022-09-29T05:25:38.517404+0000","last_peered":"2022-09-29T05:25:38.517404+0000","last_clean":"2022-09-29T05:25:38.517404+0000","last_became_active":"2022-09-29T05:25:03.813019+0000","last_became_peered":"2022-09-29T05:25:03.813019+0000","last_unstale":"2022-09-29T05:25:38.517404+0000","last_undegraded":"2022-09-29T05:25:38.517404+0000","last_fullsized":"2022-09-29T05:25:38.517404+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.32","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.517710+0000","last_change":"2022-09-29T05:25:03.823321+0000","last_active":"2022-09-29T05:25:38.517710+0000","last_peered":"2022-09-29T05:25:38.517710+0000","last_clean":"2022-09-29T05:25:38.517710+0000","last_became_active":"2022-09-29T05:25:03.822987+0000","last_became_peered":"2022-09-29T05:25:03.822987+0000","last_unstale":"2022-09-29T05:25:38.517710+0000","last_undegraded":"2022-09-29T05:25:38.517710+0000","last_fullsized":"2022-09-29T05:25:38.517710+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.31","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.141395+0000","last_change":"2022-09-29T05:25:03.805590+0000","last_active":"2022-09-29T05:25:38.141395+0000","last_peered":"2022-09-29T05:25:38.141395+0000","last_clean":"2022-09-29T05:25:38.141395+0000","last_became_active":"2022-09-29T05:25:03.805310+0000","last_became_peered":"2022-09-29T05:25:03.805310+0000","last_unstale":"2022-09-29T05:25:38.141395+0000","last_undegraded":"2022-09-29T05:25:38.141395+0000","last_fullsized":"2022-09-29T05:25:38.141395+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.30","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.517306+0000","last_change":"2022-09-29T05:25:03.815886+0000","last_active":"2022-09-29T05:25:38.517306+0000","last_peered":"2022-09-29T05:25:38.517306+0000","last_clean":"2022-09-29T05:25:38.517306+0000","last_became_active":"2022-09-29T05:25:03.815285+0000","last_became_peered":"2022-09-29T05:25:03.815285+0000","last_unstale":"2022-09-29T05:25:38.517306+0000","last_undegraded":"2022-09-29T05:25:38.517306+0000","last_fullsized":"2022-09-29T05:25:38.517306+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2f","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.518592+0000","last_change":"2022-09-29T05:25:03.813784+0000","last_active":"2022-09-29T05:25:38.518592+0000","last_peered":"2022-09-29T05:25:38.518592+0000","last_clean":"2022-09-29T05:25:38.518592+0000","last_became_active":"2022-09-29T05:25:03.813420+0000","last_became_peered":"2022-09-29T05:25:03.813420+0000","last_unstale":"2022-09-29T05:25:38.518592+0000","last_undegraded":"2022-09-29T05:25:38.518592+0000","last_fullsized":"2022-09-29T05:25:38.518592+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2e","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.141808+0000","last_change":"2022-09-29T05:25:03.813894+0000","last_active":"2022-09-29T05:25:38.141808+0000","last_peered":"2022-09-29T05:25:38.141808+0000","last_clean":"2022-09-29T05:25:38.141808+0000","last_became_active":"2022-09-29T05:25:03.813620+0000","last_became_peered":"2022-09-29T05:25:03.813620+0000","last_unstale":"2022-09-29T05:25:38.141808+0000","last_undegraded":"2022-09-29T05:25:38.141808+0000","last_fullsized":"2022-09-29T05:25:38.141808+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,7],"acting":[2,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2d","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.519104+0000","last_change":"2022-09-29T05:25:03.828324+0000","last_active":"2022-09-29T05:25:38.519104+0000","last_peered":"2022-09-29T05:25:38.519104+0000","last_clean":"2022-09-29T05:25:38.519104+0000","last_became_active":"2022-09-29T05:25:03.827685+0000","last_became_peered":"2022-09-29T05:25:03.827685+0000","last_unstale":"2022-09-29T05:25:38.519104+0000","last_undegraded":"2022-09-29T05:25:38.519104+0000","last_fullsized":"2022-09-29T05:25:38.519104+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1f","version":"50'4","reported_seq":20,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.518334+0000","last_change":"2022-09-29T05:25:03.449078+0000","last_active":"2022-09-29T05:25:38.518334+0000","last_peered":"2022-09-29T05:25:38.518334+0000","last_clean":"2022-09-29T05:25:38.518334+0000","last_became_active":"2022-09-29T05:25:03.448793+0000","last_became_peered":"2022-09-29T05:25:03.448793+0000","last_unstale":"2022-09-29T05:25:38.518334+0000","last_undegraded":"2022-09-29T05:25:38.518334+0000","last_fullsized":"2022-09-29T05:25:38.518334+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":4,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.517840+0000","last_change":"2022-09-29T05:25:03.814538+0000","last_active":"2022-09-29T05:25:38.517840+0000","last_peered":"2022-09-29T05:25:38.517840+0000","last_clean":"2022-09-29T05:25:38.517840+0000","last_became_active":"2022-09-29T05:25:03.814264+0000","last_became_peered":"2022-09-29T05:25:03.814264+0000","last_unstale":"2022-09-29T05:25:38.517840+0000","last_undegraded":"2022-09-29T05:25:38.517840+0000","last_fullsized":"2022-09-29T05:25:38.517840+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1e","version":"50'1","reported_seq":17,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.517756+0000","last_change":"2022-09-29T05:25:03.459868+0000","last_active":"2022-09-29T05:25:38.517756+0000","last_peered":"2022-09-29T05:25:38.517756+0000","last_clean":"2022-09-29T05:25:38.517756+0000","last_became_active":"2022-09-29T05:25:03.459458+0000","last_became_peered":"2022-09-29T05:25:03.459458+0000","last_unstale":"2022-09-29T05:25:38.517756+0000","last_undegraded":"2022-09-29T05:25:38.517756+0000","last_fullsized":"2022-09-29T05:25:38.517756+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.518123+0000","last_change":"2022-09-29T05:25:03.822430+0000","last_active":"2022-09-29T05:25:38.518123+0000","last_peered":"2022-09-29T05:25:38.518123+0000","last_clean":"2022-09-29T05:25:38.518123+0000","last_became_active":"2022-09-29T05:25:03.821898+0000","last_became_peered":"2022-09-29T05:25:03.821898+0000","last_unstale":"2022-09-29T05:25:38.518123+0000","last_undegraded":"2022-09-29T05:25:38.518123+0000","last_fullsized":"2022-09-29T05:25:38.518123+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1d","version":"50'2","reported_seq":18,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.519405+0000","last_change":"2022-09-29T05:25:03.455815+0000","last_active":"2022-09-29T05:25:38.519405+0000","last_peered":"2022-09-29T05:25:38.519405+0000","last_clean":"2022-09-29T05:25:38.519405+0000","last_became_active":"2022-09-29T05:25:03.455218+0000","last_became_peered":"2022-09-29T05:25:03.455218+0000","last_unstale":"2022-09-29T05:25:38.519405+0000","last_undegraded":"2022-09-29T05:25:38.519405+0000","last_fullsized":"2022-09-29T05:25:38.519405+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.518874+0000","last_change":"2022-09-29T05:25:03.816581+0000","last_active":"2022-09-29T05:25:38.518874+0000","last_peered":"2022-09-29T05:25:38.518874+0000","last_clean":"2022-09-29T05:25:38.518874+0000","last_became_active":"2022-09-29T05:25:03.816206+0000","last_became_peered":"2022-09-29T05:25:03.816206+0000","last_unstale":"2022-09-29T05:25:38.518874+0000","last_undegraded":"2022-09-29T05:25:38.518874+0000","last_fullsized":"2022-09-29T05:25:38.518874+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1c","version":"50'2","reported_seq":18,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.518052+0000","last_change":"2022-09-29T05:25:03.460578+0000","last_active":"2022-09-29T05:25:38.518052+0000","last_peered":"2022-09-29T05:25:38.518052+0000","last_clean":"2022-09-29T05:25:38.518052+0000","last_became_active":"2022-09-29T05:25:03.460273+0000","last_became_peered":"2022-09-29T05:25:03.460273+0000","last_unstale":"2022-09-29T05:25:38.518052+0000","last_undegraded":"2022-09-29T05:25:38.518052+0000","last_fullsized":"2022-09-29T05:25:38.518052+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1d","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.518215+0000","last_change":"2022-09-29T05:25:03.820523+0000","last_active":"2022-09-29T05:25:38.518215+0000","last_peered":"2022-09-29T05:25:38.518215+0000","last_clean":"2022-09-29T05:25:38.518215+0000","last_became_active":"2022-09-29T05:25:03.819546+0000","last_became_peered":"2022-09-29T05:25:03.819546+0000","last_unstale":"2022-09-29T05:25:38.518215+0000","last_undegraded":"2022-09-29T05:25:38.518215+0000","last_fullsized":"2022-09-29T05:25:38.518215+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1b","version":"50'1","reported_seq":17,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.517635+0000","last_change":"2022-09-29T05:25:03.455758+0000","last_active":"2022-09-29T05:25:38.517635+0000","last_peered":"2022-09-29T05:25:38.517635+0000","last_clean":"2022-09-29T05:25:38.517635+0000","last_became_active":"2022-09-29T05:25:03.455438+0000","last_became_peered":"2022-09-29T05:25:03.455438+0000","last_unstale":"2022-09-29T05:25:38.517635+0000","last_undegraded":"2022-09-29T05:25:38.517635+0000","last_fullsized":"2022-09-29T05:25:38.517635+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.135947+0000","last_change":"2022-09-29T05:25:03.817393+0000","last_active":"2022-09-29T05:25:38.135947+0000","last_peered":"2022-09-29T05:25:38.135947+0000","last_clean":"2022-09-29T05:25:38.135947+0000","last_became_active":"2022-09-29T05:25:03.817103+0000","last_became_peered":"2022-09-29T05:25:03.817103+0000","last_unstale":"2022-09-29T05:25:38.135947+0000","last_undegraded":"2022-09-29T05:25:38.135947+0000","last_fullsized":"2022-09-29T05:25:38.135947+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1a","version":"50'1","reported_seq":17,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.517949+0000","last_change":"2022-09-29T05:25:03.454747+0000","last_active":"2022-09-29T05:25:38.517949+0000","last_peered":"2022-09-29T05:25:38.517949+0000","last_clean":"2022-09-29T05:25:38.517949+0000","last_became_active":"2022-09-29T05:25:03.454395+0000","last_became_peered":"2022-09-29T05:25:03.454395+0000","last_unstale":"2022-09-29T05:25:38.517949+0000","last_undegraded":"2022-09-29T05:25:38.517949+0000","last_fullsized":"2022-09-29T05:25:38.517949+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1b","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.518041+0000","last_change":"2022-09-29T05:25:03.819253+0000","last_active":"2022-09-29T05:25:38.518041+0000","last_peered":"2022-09-29T05:25:38.518041+0000","last_clean":"2022-09-29T05:25:38.518041+0000","last_became_active":"2022-09-29T05:25:03.818951+0000","last_became_peered":"2022-09-29T05:25:03.818951+0000","last_unstale":"2022-09-29T05:25:38.518041+0000","last_undegraded":"2022-09-29T05:25:38.518041+0000","last_fullsized":"2022-09-29T05:25:38.518041+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.19","version":"0'0","reported_seq":16,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.518293+0000","last_change":"2022-09-29T05:25:03.455193+0000","last_active":"2022-09-29T05:25:38.518293+0000","last_peered":"2022-09-29T05:25:38.518293+0000","last_clean":"2022-09-29T05:25:38.518293+0000","last_became_active":"2022-09-29T05:25:03.454916+0000","last_became_peered":"2022-09-29T05:25:03.454916+0000","last_unstale":"2022-09-29T05:25:38.518293+0000","last_undegraded":"2022-09-29T05:25:38.518293+0000","last_fullsized":"2022-09-29T05:25:38.518293+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.517315+0000","last_change":"2022-09-29T05:25:03.806930+0000","last_active":"2022-09-29T05:25:38.517315+0000","last_peered":"2022-09-29T05:25:38.517315+0000","last_clean":"2022-09-29T05:25:38.517315+0000","last_became_active":"2022-09-29T05:25:03.806565+0000","last_became_peered":"2022-09-29T05:25:03.806565+0000","last_unstale":"2022-09-29T05:25:38.517315+0000","last_undegraded":"2022-09-29T05:25:38.517315+0000","last_fullsized":"2022-09-29T05:25:38.517315+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.18","version":"0'0","reported_seq":16,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.517443+0000","last_change":"2022-09-29T05:25:03.460675+0000","last_active":"2022-09-29T05:25:38.517443+0000","last_peered":"2022-09-29T05:25:38.517443+0000","last_clean":"2022-09-29T05:25:38.517443+0000","last_became_active":"2022-09-29T05:25:03.460351+0000","last_became_peered":"2022-09-29T05:25:03.460351+0000","last_unstale":"2022-09-29T05:25:38.517443+0000","last_undegraded":"2022-09-29T05:25:38.517443+0000","last_fullsized":"2022-09-29T05:25:38.517443+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.141604+0000","last_change":"2022-09-29T05:25:03.816170+0000","last_active":"2022-09-29T05:25:38.141604+0000","last_peered":"2022-09-29T05:25:38.141604+0000","last_clean":"2022-09-29T05:25:38.141604+0000","last_became_active":"2022-09-29T05:25:03.815838+0000","last_became_peered":"2022-09-29T05:25:03.815838+0000","last_unstale":"2022-09-29T05:25:38.141604+0000","last_undegraded":"2022-09-29T05:25:38.141604+0000","last_fullsized":"2022-09-29T05:25:38.141604+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.17","version":"0'0","reported_seq":16,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.518267+0000","last_change":"2022-09-29T05:25:03.461045+0000","last_active":"2022-09-29T05:25:38.518267+0000","last_peered":"2022-09-29T05:25:38.518267+0000","last_clean":"2022-09-29T05:25:38.518267+0000","last_became_active":"2022-09-29T05:25:03.460729+0000","last_became_peered":"2022-09-29T05:25:03.460729+0000","last_unstale":"2022-09-29T05:25:38.518267+0000","last_undegraded":"2022-09-29T05:25:38.518267+0000","last_fullsized":"2022-09-29T05:25:38.518267+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.517866+0000","last_change":"2022-09-29T05:25:03.818733+0000","last_active":"2022-09-29T05:25:38.517866+0000","last_peered":"2022-09-29T05:25:38.517866+0000","last_clean":"2022-09-29T05:25:38.517866+0000","last_became_active":"2022-09-29T05:25:03.816983+0000","last_became_peered":"2022-09-29T05:25:03.816983+0000","last_unstale":"2022-09-29T05:25:38.517866+0000","last_undegraded":"2022-09-29T05:25:38.517866+0000","last_fullsized":"2022-09-29T05:25:38.517866+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.16","version":"0'0","reported_seq":16,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.517787+0000","last_change":"2022-09-29T05:25:03.463477+0000","last_active":"2022-09-29T05:25:38.517787+0000","last_peered":"2022-09-29T05:25:38.517787+0000","last_clean":"2022-09-29T05:25:38.517787+0000","last_became_active":"2022-09-29T05:25:03.463051+0000","last_became_peered":"2022-09-29T05:25:03.463051+0000","last_unstale":"2022-09-29T05:25:38.517787+0000","last_undegraded":"2022-09-29T05:25:38.517787+0000","last_fullsized":"2022-09-29T05:25:38.517787+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.517956+0000","last_change":"2022-09-29T05:25:03.822200+0000","last_active":"2022-09-29T05:25:38.517956+0000","last_peered":"2022-09-29T05:25:38.517956+0000","last_clean":"2022-09-29T05:25:38.517956+0000","last_became_active":"2022-09-29T05:25:03.821488+0000","last_became_peered":"2022-09-29T05:25:03.821488+0000","last_unstale":"2022-09-29T05:25:38.517956+0000","last_undegraded":"2022-09-29T05:25:38.517956+0000","last_fullsized":"2022-09-29T05:25:38.517956+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.15","version":"50'3","reported_seq":19,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.142344+0000","last_change":"2022-09-29T05:25:03.457077+0000","last_active":"2022-09-29T05:25:38.142344+0000","last_peered":"2022-09-29T05:25:38.142344+0000","last_clean":"2022-09-29T05:25:38.142344+0000","last_became_active":"2022-09-29T05:25:03.456823+0000","last_became_peered":"2022-09-29T05:25:03.456823+0000","last_unstale":"2022-09-29T05:25:38.142344+0000","last_undegraded":"2022-09-29T05:25:38.142344+0000","last_fullsized":"2022-09-29T05:25:38.142344+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":3,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.136217+0000","last_change":"2022-09-29T05:25:03.822638+0000","last_active":"2022-09-29T05:25:38.136217+0000","last_peered":"2022-09-29T05:25:38.136217+0000","last_clean":"2022-09-29T05:25:38.136217+0000","last_became_active":"2022-09-29T05:25:03.822297+0000","last_became_peered":"2022-09-29T05:25:03.822297+0000","last_unstale":"2022-09-29T05:25:38.136217+0000","last_undegraded":"2022-09-29T05:25:38.136217+0000","last_fullsized":"2022-09-29T05:25:38.136217+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.14","version":"52'9","reported_seq":25,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.517947+0000","last_change":"2022-09-29T05:25:03.461139+0000","last_active":"2022-09-29T05:25:38.517947+0000","last_peered":"2022-09-29T05:25:38.517947+0000","last_clean":"2022-09-29T05:25:38.517947+0000","last_became_active":"2022-09-29T05:25:03.460829+0000","last_became_peered":"2022-09-29T05:25:03.460829+0000","last_unstale":"2022-09-29T05:25:38.517947+0000","last_undegraded":"2022-09-29T05:25:38.517947+0000","last_fullsized":"2022-09-29T05:25:38.517947+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":9,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":2300,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.519257+0000","last_change":"2022-09-29T05:25:03.828186+0000","last_active":"2022-09-29T05:25:38.519257+0000","last_peered":"2022-09-29T05:25:38.519257+0000","last_clean":"2022-09-29T05:25:38.519257+0000","last_became_active":"2022-09-29T05:25:03.826167+0000","last_became_peered":"2022-09-29T05:25:03.826167+0000","last_unstale":"2022-09-29T05:25:38.519257+0000","last_undegraded":"2022-09-29T05:25:38.519257+0000","last_fullsized":"2022-09-29T05:25:38.519257+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.13","version":"50'2","reported_seq":18,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.518229+0000","last_change":"2022-09-29T05:25:03.451661+0000","last_active":"2022-09-29T05:25:38.518229+0000","last_peered":"2022-09-29T05:25:38.518229+0000","last_clean":"2022-09-29T05:25:38.518229+0000","last_became_active":"2022-09-29T05:25:03.451389+0000","last_became_peered":"2022-09-29T05:25:03.451389+0000","last_unstale":"2022-09-29T05:25:38.518229+0000","last_undegraded":"2022-09-29T05:25:38.518229+0000","last_fullsized":"2022-09-29T05:25:38.518229+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.517628+0000","last_change":"2022-09-29T05:25:03.824608+0000","last_active":"2022-09-29T05:25:38.517628+0000","last_peered":"2022-09-29T05:25:38.517628+0000","last_clean":"2022-09-29T05:25:38.517628+0000","last_became_active":"2022-09-29T05:25:03.824335+0000","last_became_peered":"2022-09-29T05:25:03.824335+0000","last_unstale":"2022-09-29T05:25:38.517628+0000","last_undegraded":"2022-09-29T05:25:38.517628+0000","last_fullsized":"2022-09-29T05:25:38.517628+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.12","version":"0'0","reported_seq":16,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.517688+0000","last_change":"2022-09-29T05:25:03.462145+0000","last_active":"2022-09-29T05:25:38.517688+0000","last_peered":"2022-09-29T05:25:38.517688+0000","last_clean":"2022-09-29T05:25:38.517688+0000","last_became_active":"2022-09-29T05:25:03.461864+0000","last_became_peered":"2022-09-29T05:25:03.461864+0000","last_unstale":"2022-09-29T05:25:38.517688+0000","last_undegraded":"2022-09-29T05:25:38.517688+0000","last_fullsized":"2022-09-29T05:25:38.517688+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.519372+0000","last_change":"2022-09-29T05:25:03.828734+0000","last_active":"2022-09-29T05:25:38.519372+0000","last_peered":"2022-09-29T05:25:38.519372+0000","last_clean":"2022-09-29T05:25:38.519372+0000","last_became_active":"2022-09-29T05:25:03.827929+0000","last_became_peered":"2022-09-29T05:25:03.827929+0000","last_unstale":"2022-09-29T05:25:38.519372+0000","last_undegraded":"2022-09-29T05:25:38.519372+0000","last_fullsized":"2022-09-29T05:25:38.519372+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.11","version":"0'0","reported_seq":16,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.518489+0000","last_change":"2022-09-29T05:25:03.454654+0000","last_active":"2022-09-29T05:25:38.518489+0000","last_peered":"2022-09-29T05:25:38.518489+0000","last_clean":"2022-09-29T05:25:38.518489+0000","last_became_active":"2022-09-29T05:25:03.454319+0000","last_became_peered":"2022-09-29T05:25:03.454319+0000","last_unstale":"2022-09-29T05:25:38.518489+0000","last_undegraded":"2022-09-29T05:25:38.518489+0000","last_fullsized":"2022-09-29T05:25:38.518489+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.518057+0000","last_change":"2022-09-29T05:25:03.811425+0000","last_active":"2022-09-29T05:25:38.518057+0000","last_peered":"2022-09-29T05:25:38.518057+0000","last_clean":"2022-09-29T05:25:38.518057+0000","last_became_active":"2022-09-29T05:25:03.811120+0000","last_became_peered":"2022-09-29T05:25:03.811120+0000","last_unstale":"2022-09-29T05:25:38.518057+0000","last_undegraded":"2022-09-29T05:25:38.518057+0000","last_fullsized":"2022-09-29T05:25:38.518057+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.10","version":"50'8","reported_seq":24,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.140816+0000","last_change":"2022-09-29T05:25:03.449137+0000","last_active":"2022-09-29T05:25:38.140816+0000","last_peered":"2022-09-29T05:25:38.140816+0000","last_clean":"2022-09-29T05:25:38.140816+0000","last_became_active":"2022-09-29T05:25:03.448728+0000","last_became_peered":"2022-09-29T05:25:03.448728+0000","last_unstale":"2022-09-29T05:25:38.140816+0000","last_undegraded":"2022-09-29T05:25:38.140816+0000","last_fullsized":"2022-09-29T05:25:38.140816+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":8,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.517782+0000","last_change":"2022-09-29T05:25:03.826279+0000","last_active":"2022-09-29T05:25:38.517782+0000","last_peered":"2022-09-29T05:25:38.517782+0000","last_clean":"2022-09-29T05:25:38.517782+0000","last_became_active":"2022-09-29T05:25:03.825969+0000","last_became_peered":"2022-09-29T05:25:03.825969+0000","last_unstale":"2022-09-29T05:25:38.517782+0000","last_undegraded":"2022-09-29T05:25:38.517782+0000","last_fullsized":"2022-09-29T05:25:38.517782+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.f","version":"50'1","reported_seq":17,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.136419+0000","last_change":"2022-09-29T05:25:03.452230+0000","last_active":"2022-09-29T05:25:38.136419+0000","last_peered":"2022-09-29T05:25:38.136419+0000","last_clean":"2022-09-29T05:25:38.136419+0000","last_became_active":"2022-09-29T05:25:03.451747+0000","last_became_peered":"2022-09-29T05:25:03.451747+0000","last_unstale":"2022-09-29T05:25:38.136419+0000","last_undegraded":"2022-09-29T05:25:38.136419+0000","last_fullsized":"2022-09-29T05:25:38.136419+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.518349+0000","last_change":"2022-09-29T05:25:03.827453+0000","last_active":"2022-09-29T05:25:38.518349+0000","last_peered":"2022-09-29T05:25:38.518349+0000","last_clean":"2022-09-29T05:25:38.518349+0000","last_became_active":"2022-09-29T05:25:03.827151+0000","last_became_peered":"2022-09-29T05:25:03.827151+0000","last_unstale":"2022-09-29T05:25:38.518349+0000","last_undegraded":"2022-09-29T05:25:38.518349+0000","last_fullsized":"2022-09-29T05:25:38.518349+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.141816+0000","last_change":"2022-09-29T05:25:37.268300+0000","last_active":"2022-09-29T05:25:38.141816+0000","last_peered":"2022-09-29T05:25:38.141816+0000","last_clean":"2022-09-29T05:25:38.141816+0000","last_became_active":"2022-09-29T05:25:37.267961+0000","last_became_peered":"2022-09-29T05:25:37.267961+0000","last_unstale":"2022-09-29T05:25:38.141816+0000","last_undegraded":"2022-09-29T05:25:38.141816+0000","last_fullsized":"2022-09-29T05:25:38.141816+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:36.130315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:36.130315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:36.130315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":16,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.141681+0000","last_change":"2022-09-29T05:25:03.454507+0000","last_active":"2022-09-29T05:25:38.141681+0000","last_peered":"2022-09-29T05:25:38.141681+0000","last_clean":"2022-09-29T05:25:38.141681+0000","last_became_active":"2022-09-29T05:25:03.454195+0000","last_became_peered":"2022-09-29T05:25:03.454195+0000","last_unstale":"2022-09-29T05:25:38.141681+0000","last_undegraded":"2022-09-29T05:25:38.141681+0000","last_fullsized":"2022-09-29T05:25:38.141681+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.518292+0000","last_change":"2022-09-29T05:25:03.819862+0000","last_active":"2022-09-29T05:25:38.518292+0000","last_peered":"2022-09-29T05:25:38.518292+0000","last_clean":"2022-09-29T05:25:38.518292+0000","last_became_active":"2022-09-29T05:25:03.818117+0000","last_became_peered":"2022-09-29T05:25:03.818117+0000","last_unstale":"2022-09-29T05:25:38.518292+0000","last_undegraded":"2022-09-29T05:25:38.518292+0000","last_fullsized":"2022-09-29T05:25:38.518292+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.141770+0000","last_change":"2022-09-29T05:25:37.140714+0000","last_active":"2022-09-29T05:25:38.141770+0000","last_peered":"2022-09-29T05:25:38.141770+0000","last_clean":"2022-09-29T05:25:38.141770+0000","last_became_active":"2022-09-29T05:25:37.140393+0000","last_became_peered":"2022-09-29T05:25:37.140393+0000","last_unstale":"2022-09-29T05:25:38.141770+0000","last_undegraded":"2022-09-29T05:25:38.141770+0000","last_fullsized":"2022-09-29T05:25:38.141770+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:36.130315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:36.130315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:36.130315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":16,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.517596+0000","last_change":"2022-09-29T05:25:03.459646+0000","last_active":"2022-09-29T05:25:38.517596+0000","last_peered":"2022-09-29T05:25:38.517596+0000","last_clean":"2022-09-29T05:25:38.517596+0000","last_became_active":"2022-09-29T05:25:03.459320+0000","last_became_peered":"2022-09-29T05:25:03.459320+0000","last_unstale":"2022-09-29T05:25:38.517596+0000","last_undegraded":"2022-09-29T05:25:38.517596+0000","last_fullsized":"2022-09-29T05:25:38.517596+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.136122+0000","last_change":"2022-09-29T05:25:03.810358+0000","last_active":"2022-09-29T05:25:38.136122+0000","last_peered":"2022-09-29T05:25:38.136122+0000","last_clean":"2022-09-29T05:25:38.136122+0000","last_became_active":"2022-09-29T05:25:03.809959+0000","last_became_peered":"2022-09-29T05:25:03.809959+0000","last_unstale":"2022-09-29T05:25:38.136122+0000","last_undegraded":"2022-09-29T05:25:38.136122+0000","last_fullsized":"2022-09-29T05:25:38.136122+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.141097+0000","last_change":"2022-09-29T05:25:37.268374+0000","last_active":"2022-09-29T05:25:38.141097+0000","last_peered":"2022-09-29T05:25:38.141097+0000","last_clean":"2022-09-29T05:25:38.141097+0000","last_became_active":"2022-09-29T05:25:37.267928+0000","last_became_peered":"2022-09-29T05:25:37.267928+0000","last_unstale":"2022-09-29T05:25:38.141097+0000","last_undegraded":"2022-09-29T05:25:38.141097+0000","last_fullsized":"2022-09-29T05:25:38.141097+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:36.130315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:36.130315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:36.130315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.141135+0000","last_change":"2022-09-29T05:25:03.812390+0000","last_active":"2022-09-29T05:25:38.141135+0000","last_peered":"2022-09-29T05:25:38.141135+0000","last_clean":"2022-09-29T05:25:38.141135+0000","last_became_active":"2022-09-29T05:25:03.811925+0000","last_became_peered":"2022-09-29T05:25:03.811925+0000","last_unstale":"2022-09-29T05:25:38.141135+0000","last_undegraded":"2022-09-29T05:25:38.141135+0000","last_fullsized":"2022-09-29T05:25:38.141135+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":16,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.140903+0000","last_change":"2022-09-29T05:25:03.449156+0000","last_active":"2022-09-29T05:25:38.140903+0000","last_peered":"2022-09-29T05:25:38.140903+0000","last_clean":"2022-09-29T05:25:38.140903+0000","last_became_active":"2022-09-29T05:25:03.448885+0000","last_became_peered":"2022-09-29T05:25:03.448885+0000","last_unstale":"2022-09-29T05:25:38.140903+0000","last_undegraded":"2022-09-29T05:25:38.140903+0000","last_fullsized":"2022-09-29T05:25:38.140903+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.518158+0000","last_change":"2022-09-29T05:25:37.269299+0000","last_active":"2022-09-29T05:25:38.518158+0000","last_peered":"2022-09-29T05:25:38.518158+0000","last_clean":"2022-09-29T05:25:38.518158+0000","last_became_active":"2022-09-29T05:25:37.268903+0000","last_became_peered":"2022-09-29T05:25:37.268903+0000","last_unstale":"2022-09-29T05:25:38.518158+0000","last_undegraded":"2022-09-29T05:25:38.518158+0000","last_fullsized":"2022-09-29T05:25:38.518158+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:36.130315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:36.130315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:36.130315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.518001+0000","last_change":"2022-09-29T05:25:03.813170+0000","last_active":"2022-09-29T05:25:38.518001+0000","last_peered":"2022-09-29T05:25:38.518001+0000","last_clean":"2022-09-29T05:25:38.518001+0000","last_became_active":"2022-09-29T05:25:03.812861+0000","last_became_peered":"2022-09-29T05:25:03.812861+0000","last_unstale":"2022-09-29T05:25:38.518001+0000","last_undegraded":"2022-09-29T05:25:38.518001+0000","last_fullsized":"2022-09-29T05:25:38.518001+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":16,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.517396+0000","last_change":"2022-09-29T05:25:03.462271+0000","last_active":"2022-09-29T05:25:38.517396+0000","last_peered":"2022-09-29T05:25:38.517396+0000","last_clean":"2022-09-29T05:25:38.517396+0000","last_became_active":"2022-09-29T05:25:03.461959+0000","last_became_peered":"2022-09-29T05:25:03.461959+0000","last_unstale":"2022-09-29T05:25:38.517396+0000","last_undegraded":"2022-09-29T05:25:38.517396+0000","last_fullsized":"2022-09-29T05:25:38.517396+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":19,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.519716+0000","last_change":"2022-09-29T05:24:48.628243+0000","last_active":"2022-09-29T05:25:38.519716+0000","last_peered":"2022-09-29T05:25:38.519716+0000","last_clean":"2022-09-29T05:25:38.519716+0000","last_became_active":"2022-09-29T05:24:48.627801+0000","last_became_peered":"2022-09-29T05:24:48.627801+0000","last_unstale":"2022-09-29T05:25:38.519716+0000","last_undegraded":"2022-09-29T05:25:38.519716+0000","last_fullsized":"2022-09-29T05:25:38.519716+0000","mapping_epoch":46,"log_start":"0'0","ondisk_log_start":"0'0","created":21,"last_epoch_clean":47,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:23:35.544281+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:23:35.544281+0000","last_clean_scrub_stamp":"2022-09-29T05:23:35.544281+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.518259+0000","last_change":"2022-09-29T05:25:37.141541+0000","last_active":"2022-09-29T05:25:38.518259+0000","last_peered":"2022-09-29T05:25:38.518259+0000","last_clean":"2022-09-29T05:25:38.518259+0000","last_became_active":"2022-09-29T05:25:37.141199+0000","last_became_peered":"2022-09-29T05:25:37.141199+0000","last_unstale":"2022-09-29T05:25:38.518259+0000","last_undegraded":"2022-09-29T05:25:38.518259+0000","last_fullsized":"2022-09-29T05:25:38.518259+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:36.130315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:36.130315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:36.130315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.517516+0000","last_change":"2022-09-29T05:25:03.812201+0000","last_active":"2022-09-29T05:25:38.517516+0000","last_peered":"2022-09-29T05:25:38.517516+0000","last_clean":"2022-09-29T05:25:38.517516+0000","last_became_active":"2022-09-29T05:25:03.811903+0000","last_became_peered":"2022-09-29T05:25:03.811903+0000","last_unstale":"2022-09-29T05:25:38.517516+0000","last_undegraded":"2022-09-29T05:25:38.517516+0000","last_fullsized":"2022-09-29T05:25:38.517516+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":16,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.519580+0000","last_change":"2022-09-29T05:25:03.451876+0000","last_active":"2022-09-29T05:25:38.519580+0000","last_peered":"2022-09-29T05:25:38.519580+0000","last_clean":"2022-09-29T05:25:38.519580+0000","last_became_active":"2022-09-29T05:25:03.451371+0000","last_became_peered":"2022-09-29T05:25:03.451371+0000","last_unstale":"2022-09-29T05:25:38.519580+0000","last_undegraded":"2022-09-29T05:25:38.519580+0000","last_fullsized":"2022-09-29T05:25:38.519580+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.517944+0000","last_change":"2022-09-29T05:25:37.269163+0000","last_active":"2022-09-29T05:25:38.517944+0000","last_peered":"2022-09-29T05:25:38.517944+0000","last_clean":"2022-09-29T05:25:38.517944+0000","last_became_active":"2022-09-29T05:25:37.268871+0000","last_became_peered":"2022-09-29T05:25:37.268871+0000","last_unstale":"2022-09-29T05:25:38.517944+0000","last_undegraded":"2022-09-29T05:25:38.517944+0000","last_fullsized":"2022-09-29T05:25:38.517944+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:36.130315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:36.130315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:36.130315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":16,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.519358+0000","last_change":"2022-09-29T05:25:03.453060+0000","last_active":"2022-09-29T05:25:38.519358+0000","last_peered":"2022-09-29T05:25:38.519358+0000","last_clean":"2022-09-29T05:25:38.519358+0000","last_became_active":"2022-09-29T05:25:03.452758+0000","last_became_peered":"2022-09-29T05:25:03.452758+0000","last_unstale":"2022-09-29T05:25:38.519358+0000","last_undegraded":"2022-09-29T05:25:38.519358+0000","last_fullsized":"2022-09-29T05:25:38.519358+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.141842+0000","last_change":"2022-09-29T05:25:03.803195+0000","last_active":"2022-09-29T05:25:38.141842+0000","last_peered":"2022-09-29T05:25:38.141842+0000","last_clean":"2022-09-29T05:25:38.141842+0000","last_became_active":"2022-09-29T05:25:03.802909+0000","last_became_peered":"2022-09-29T05:25:03.802909+0000","last_unstale":"2022-09-29T05:25:38.141842+0000","last_undegraded":"2022-09-29T05:25:38.141842+0000","last_fullsized":"2022-09-29T05:25:38.141842+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.517773+0000","last_change":"2022-09-29T05:25:37.268505+0000","last_active":"2022-09-29T05:25:38.517773+0000","last_peered":"2022-09-29T05:25:38.517773+0000","last_clean":"2022-09-29T05:25:38.517773+0000","last_became_active":"2022-09-29T05:25:37.268095+0000","last_became_peered":"2022-09-29T05:25:37.268095+0000","last_unstale":"2022-09-29T05:25:38.517773+0000","last_undegraded":"2022-09-29T05:25:38.517773+0000","last_fullsized":"2022-09-29T05:25:38.517773+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:36.130315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:36.130315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:36.130315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"51'1","reported_seq":17,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.141330+0000","last_change":"2022-09-29T05:25:03.456735+0000","last_active":"2022-09-29T05:25:38.141330+0000","last_peered":"2022-09-29T05:25:38.141330+0000","last_clean":"2022-09-29T05:25:38.141330+0000","last_became_active":"2022-09-29T05:25:03.456427+0000","last_became_peered":"2022-09-29T05:25:03.456427+0000","last_unstale":"2022-09-29T05:25:38.141330+0000","last_undegraded":"2022-09-29T05:25:38.141330+0000","last_fullsized":"2022-09-29T05:25:38.141330+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.517746+0000","last_change":"2022-09-29T05:25:03.819725+0000","last_active":"2022-09-29T05:25:38.517746+0000","last_peered":"2022-09-29T05:25:38.517746+0000","last_clean":"2022-09-29T05:25:38.517746+0000","last_became_active":"2022-09-29T05:25:03.819455+0000","last_became_peered":"2022-09-29T05:25:03.819455+0000","last_unstale":"2022-09-29T05:25:38.517746+0000","last_undegraded":"2022-09-29T05:25:38.517746+0000","last_fullsized":"2022-09-29T05:25:38.517746+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.135795+0000","last_change":"2022-09-29T05:25:37.269825+0000","last_active":"2022-09-29T05:25:38.135795+0000","last_peered":"2022-09-29T05:25:38.135795+0000","last_clean":"2022-09-29T05:25:38.135795+0000","last_became_active":"2022-09-29T05:25:37.269508+0000","last_became_peered":"2022-09-29T05:25:37.269508+0000","last_unstale":"2022-09-29T05:25:38.135795+0000","last_undegraded":"2022-09-29T05:25:38.135795+0000","last_fullsized":"2022-09-29T05:25:38.135795+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:36.130315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:36.130315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:36.130315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"50'2","reported_seq":18,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.518186+0000","last_change":"2022-09-29T05:25:03.462138+0000","last_active":"2022-09-29T05:25:38.518186+0000","last_peered":"2022-09-29T05:25:38.518186+0000","last_clean":"2022-09-29T05:25:38.518186+0000","last_became_active":"2022-09-29T05:25:03.461809+0000","last_became_peered":"2022-09-29T05:25:03.461809+0000","last_unstale":"2022-09-29T05:25:38.518186+0000","last_undegraded":"2022-09-29T05:25:38.518186+0000","last_fullsized":"2022-09-29T05:25:38.518186+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.518353+0000","last_change":"2022-09-29T05:25:03.813902+0000","last_active":"2022-09-29T05:25:38.518353+0000","last_peered":"2022-09-29T05:25:38.518353+0000","last_clean":"2022-09-29T05:25:38.518353+0000","last_became_active":"2022-09-29T05:25:03.813617+0000","last_became_peered":"2022-09-29T05:25:03.813617+0000","last_unstale":"2022-09-29T05:25:38.518353+0000","last_undegraded":"2022-09-29T05:25:38.518353+0000","last_fullsized":"2022-09-29T05:25:38.518353+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.8","version":"0'0","reported_seq":16,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.519792+0000","last_change":"2022-09-29T05:25:03.451677+0000","last_active":"2022-09-29T05:25:38.519792+0000","last_peered":"2022-09-29T05:25:38.519792+0000","last_clean":"2022-09-29T05:25:38.519792+0000","last_became_active":"2022-09-29T05:25:03.450961+0000","last_became_peered":"2022-09-29T05:25:03.450961+0000","last_unstale":"2022-09-29T05:25:38.519792+0000","last_undegraded":"2022-09-29T05:25:38.519792+0000","last_fullsized":"2022-09-29T05:25:38.519792+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.135920+0000","last_change":"2022-09-29T05:25:03.822108+0000","last_active":"2022-09-29T05:25:38.135920+0000","last_peered":"2022-09-29T05:25:38.135920+0000","last_clean":"2022-09-29T05:25:38.135920+0000","last_became_active":"2022-09-29T05:25:03.821784+0000","last_became_peered":"2022-09-29T05:25:03.821784+0000","last_unstale":"2022-09-29T05:25:38.135920+0000","last_undegraded":"2022-09-29T05:25:38.135920+0000","last_fullsized":"2022-09-29T05:25:38.135920+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.9","version":"0'0","reported_seq":16,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.141479+0000","last_change":"2022-09-29T05:25:03.452512+0000","last_active":"2022-09-29T05:25:38.141479+0000","last_peered":"2022-09-29T05:25:38.141479+0000","last_clean":"2022-09-29T05:25:38.141479+0000","last_became_active":"2022-09-29T05:25:03.452222+0000","last_became_peered":"2022-09-29T05:25:03.452222+0000","last_unstale":"2022-09-29T05:25:38.141479+0000","last_undegraded":"2022-09-29T05:25:38.141479+0000","last_fullsized":"2022-09-29T05:25:38.141479+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.517411+0000","last_change":"2022-09-29T05:25:03.819591+0000","last_active":"2022-09-29T05:25:38.517411+0000","last_peered":"2022-09-29T05:25:38.517411+0000","last_clean":"2022-09-29T05:25:38.517411+0000","last_became_active":"2022-09-29T05:25:03.819231+0000","last_became_peered":"2022-09-29T05:25:03.819231+0000","last_unstale":"2022-09-29T05:25:38.517411+0000","last_undegraded":"2022-09-29T05:25:38.517411+0000","last_fullsized":"2022-09-29T05:25:38.517411+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.a","version":"0'0","reported_seq":16,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.141930+0000","last_change":"2022-09-29T05:25:03.451868+0000","last_active":"2022-09-29T05:25:38.141930+0000","last_peered":"2022-09-29T05:25:38.141930+0000","last_clean":"2022-09-29T05:25:38.141930+0000","last_became_active":"2022-09-29T05:25:03.451576+0000","last_became_peered":"2022-09-29T05:25:03.451576+0000","last_unstale":"2022-09-29T05:25:38.141930+0000","last_undegraded":"2022-09-29T05:25:38.141930+0000","last_fullsized":"2022-09-29T05:25:38.141930+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.517482+0000","last_change":"2022-09-29T05:25:03.808836+0000","last_active":"2022-09-29T05:25:38.517482+0000","last_peered":"2022-09-29T05:25:38.517482+0000","last_clean":"2022-09-29T05:25:38.517482+0000","last_became_active":"2022-09-29T05:25:03.808557+0000","last_became_peered":"2022-09-29T05:25:03.808557+0000","last_unstale":"2022-09-29T05:25:38.517482+0000","last_undegraded":"2022-09-29T05:25:38.517482+0000","last_fullsized":"2022-09-29T05:25:38.517482+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.b","version":"50'2","reported_seq":18,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.519288+0000","last_change":"2022-09-29T05:25:03.460369+0000","last_active":"2022-09-29T05:25:38.519288+0000","last_peered":"2022-09-29T05:25:38.519288+0000","last_clean":"2022-09-29T05:25:38.519288+0000","last_became_active":"2022-09-29T05:25:03.459796+0000","last_became_peered":"2022-09-29T05:25:03.459796+0000","last_unstale":"2022-09-29T05:25:38.519288+0000","last_undegraded":"2022-09-29T05:25:38.519288+0000","last_fullsized":"2022-09-29T05:25:38.519288+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.517611+0000","last_change":"2022-09-29T05:25:03.818102+0000","last_active":"2022-09-29T05:25:38.517611+0000","last_peered":"2022-09-29T05:25:38.517611+0000","last_clean":"2022-09-29T05:25:38.517611+0000","last_became_active":"2022-09-29T05:25:03.817772+0000","last_became_peered":"2022-09-29T05:25:03.817772+0000","last_unstale":"2022-09-29T05:25:38.517611+0000","last_undegraded":"2022-09-29T05:25:38.517611+0000","last_fullsized":"2022-09-29T05:25:38.517611+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.c","version":"0'0","reported_seq":16,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.141464+0000","last_change":"2022-09-29T05:25:03.460119+0000","last_active":"2022-09-29T05:25:38.141464+0000","last_peered":"2022-09-29T05:25:38.141464+0000","last_clean":"2022-09-29T05:25:38.141464+0000","last_became_active":"2022-09-29T05:25:03.459795+0000","last_became_peered":"2022-09-29T05:25:03.459795+0000","last_unstale":"2022-09-29T05:25:38.141464+0000","last_undegraded":"2022-09-29T05:25:38.141464+0000","last_fullsized":"2022-09-29T05:25:38.141464+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.519315+0000","last_change":"2022-09-29T05:25:03.828523+0000","last_active":"2022-09-29T05:25:38.519315+0000","last_peered":"2022-09-29T05:25:38.519315+0000","last_clean":"2022-09-29T05:25:38.519315+0000","last_became_active":"2022-09-29T05:25:03.827990+0000","last_became_peered":"2022-09-29T05:25:03.827990+0000","last_unstale":"2022-09-29T05:25:38.519315+0000","last_undegraded":"2022-09-29T05:25:38.519315+0000","last_fullsized":"2022-09-29T05:25:38.519315+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.d","version":"50'1","reported_seq":17,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.142256+0000","last_change":"2022-09-29T05:25:03.452648+0000","last_active":"2022-09-29T05:25:38.142256+0000","last_peered":"2022-09-29T05:25:38.142256+0000","last_clean":"2022-09-29T05:25:38.142256+0000","last_became_active":"2022-09-29T05:25:03.452242+0000","last_became_peered":"2022-09-29T05:25:03.452242+0000","last_unstale":"2022-09-29T05:25:38.142256+0000","last_undegraded":"2022-09-29T05:25:38.142256+0000","last_fullsized":"2022-09-29T05:25:38.142256+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.518795+0000","last_change":"2022-09-29T05:25:03.809640+0000","last_active":"2022-09-29T05:25:38.518795+0000","last_peered":"2022-09-29T05:25:38.518795+0000","last_clean":"2022-09-29T05:25:38.518795+0000","last_became_active":"2022-09-29T05:25:03.809298+0000","last_became_peered":"2022-09-29T05:25:03.809298+0000","last_unstale":"2022-09-29T05:25:38.518795+0000","last_undegraded":"2022-09-29T05:25:38.518795+0000","last_fullsized":"2022-09-29T05:25:38.518795+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.e","version":"50'2","reported_seq":18,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.141902+0000","last_change":"2022-09-29T05:25:03.450361+0000","last_active":"2022-09-29T05:25:38.141902+0000","last_peered":"2022-09-29T05:25:38.141902+0000","last_clean":"2022-09-29T05:25:38.141902+0000","last_became_active":"2022-09-29T05:25:03.450020+0000","last_became_peered":"2022-09-29T05:25:03.450020+0000","last_unstale":"2022-09-29T05:25:38.141902+0000","last_undegraded":"2022-09-29T05:25:38.141902+0000","last_fullsized":"2022-09-29T05:25:38.141902+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.518792+0000","last_change":"2022-09-29T05:25:03.829664+0000","last_active":"2022-09-29T05:25:38.518792+0000","last_peered":"2022-09-29T05:25:38.518792+0000","last_clean":"2022-09-29T05:25:38.518792+0000","last_became_active":"2022-09-29T05:25:03.828931+0000","last_became_peered":"2022-09-29T05:25:03.828931+0000","last_unstale":"2022-09-29T05:25:38.518792+0000","last_undegraded":"2022-09-29T05:25:38.518792+0000","last_fullsized":"2022-09-29T05:25:38.518792+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.20","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.517921+0000","last_change":"2022-09-29T05:25:03.811486+0000","last_active":"2022-09-29T05:25:38.517921+0000","last_peered":"2022-09-29T05:25:38.517921+0000","last_clean":"2022-09-29T05:25:38.517921+0000","last_became_active":"2022-09-29T05:25:03.811222+0000","last_became_peered":"2022-09-29T05:25:03.811222+0000","last_unstale":"2022-09-29T05:25:38.517921+0000","last_undegraded":"2022-09-29T05:25:38.517921+0000","last_fullsized":"2022-09-29T05:25:38.517921+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.21","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.140986+0000","last_change":"2022-09-29T05:25:03.809062+0000","last_active":"2022-09-29T05:25:38.140986+0000","last_peered":"2022-09-29T05:25:38.140986+0000","last_clean":"2022-09-29T05:25:38.140986+0000","last_became_active":"2022-09-29T05:25:03.808752+0000","last_became_peered":"2022-09-29T05:25:03.808752+0000","last_unstale":"2022-09-29T05:25:38.140986+0000","last_undegraded":"2022-09-29T05:25:38.140986+0000","last_fullsized":"2022-09-29T05:25:38.140986+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.22","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.517769+0000","last_change":"2022-09-29T05:25:03.817645+0000","last_active":"2022-09-29T05:25:38.517769+0000","last_peered":"2022-09-29T05:25:38.517769+0000","last_clean":"2022-09-29T05:25:38.517769+0000","last_became_active":"2022-09-29T05:25:03.816740+0000","last_became_peered":"2022-09-29T05:25:03.816740+0000","last_unstale":"2022-09-29T05:25:38.517769+0000","last_undegraded":"2022-09-29T05:25:38.517769+0000","last_fullsized":"2022-09-29T05:25:38.517769+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.23","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.518117+0000","last_change":"2022-09-29T05:25:03.810436+0000","last_active":"2022-09-29T05:25:38.518117+0000","last_peered":"2022-09-29T05:25:38.518117+0000","last_clean":"2022-09-29T05:25:38.518117+0000","last_became_active":"2022-09-29T05:25:03.810178+0000","last_became_peered":"2022-09-29T05:25:03.810178+0000","last_unstale":"2022-09-29T05:25:38.518117+0000","last_undegraded":"2022-09-29T05:25:38.518117+0000","last_fullsized":"2022-09-29T05:25:38.518117+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.24","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.517982+0000","last_change":"2022-09-29T05:25:03.809114+0000","last_active":"2022-09-29T05:25:38.517982+0000","last_peered":"2022-09-29T05:25:38.517982+0000","last_clean":"2022-09-29T05:25:38.517982+0000","last_became_active":"2022-09-29T05:25:03.808686+0000","last_became_peered":"2022-09-29T05:25:03.808686+0000","last_unstale":"2022-09-29T05:25:38.517982+0000","last_undegraded":"2022-09-29T05:25:38.517982+0000","last_fullsized":"2022-09-29T05:25:38.517982+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.25","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.142167+0000","last_change":"2022-09-29T05:25:03.817066+0000","last_active":"2022-09-29T05:25:38.142167+0000","last_peered":"2022-09-29T05:25:38.142167+0000","last_clean":"2022-09-29T05:25:38.142167+0000","last_became_active":"2022-09-29T05:25:03.816747+0000","last_became_peered":"2022-09-29T05:25:03.816747+0000","last_unstale":"2022-09-29T05:25:38.142167+0000","last_undegraded":"2022-09-29T05:25:38.142167+0000","last_fullsized":"2022-09-29T05:25:38.142167+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.26","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.517712+0000","last_change":"2022-09-29T05:25:03.823457+0000","last_active":"2022-09-29T05:25:38.517712+0000","last_peered":"2022-09-29T05:25:38.517712+0000","last_clean":"2022-09-29T05:25:38.517712+0000","last_became_active":"2022-09-29T05:25:03.823149+0000","last_became_peered":"2022-09-29T05:25:03.823149+0000","last_unstale":"2022-09-29T05:25:38.517712+0000","last_undegraded":"2022-09-29T05:25:38.517712+0000","last_fullsized":"2022-09-29T05:25:38.517712+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,4],"acting":[6,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.27","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.517660+0000","last_change":"2022-09-29T05:25:03.808766+0000","last_active":"2022-09-29T05:25:38.517660+0000","last_peered":"2022-09-29T05:25:38.517660+0000","last_clean":"2022-09-29T05:25:38.517660+0000","last_became_active":"2022-09-29T05:25:03.808474+0000","last_became_peered":"2022-09-29T05:25:03.808474+0000","last_unstale":"2022-09-29T05:25:38.517660+0000","last_undegraded":"2022-09-29T05:25:38.517660+0000","last_fullsized":"2022-09-29T05:25:38.517660+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.28","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.517490+0000","last_change":"2022-09-29T05:25:03.815620+0000","last_active":"2022-09-29T05:25:38.517490+0000","last_peered":"2022-09-29T05:25:38.517490+0000","last_clean":"2022-09-29T05:25:38.517490+0000","last_became_active":"2022-09-29T05:25:03.814336+0000","last_became_peered":"2022-09-29T05:25:03.814336+0000","last_unstale":"2022-09-29T05:25:38.517490+0000","last_undegraded":"2022-09-29T05:25:38.517490+0000","last_fullsized":"2022-09-29T05:25:38.517490+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.29","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.518222+0000","last_change":"2022-09-29T05:25:03.808727+0000","last_active":"2022-09-29T05:25:38.518222+0000","last_peered":"2022-09-29T05:25:38.518222+0000","last_clean":"2022-09-29T05:25:38.518222+0000","last_became_active":"2022-09-29T05:25:03.808440+0000","last_became_peered":"2022-09-29T05:25:03.808440+0000","last_unstale":"2022-09-29T05:25:38.518222+0000","last_undegraded":"2022-09-29T05:25:38.518222+0000","last_fullsized":"2022-09-29T05:25:38.518222+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.2a","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.517848+0000","last_change":"2022-09-29T05:25:03.819098+0000","last_active":"2022-09-29T05:25:38.517848+0000","last_peered":"2022-09-29T05:25:38.517848+0000","last_clean":"2022-09-29T05:25:38.517848+0000","last_became_active":"2022-09-29T05:25:03.818690+0000","last_became_peered":"2022-09-29T05:25:03.818690+0000","last_unstale":"2022-09-29T05:25:38.517848+0000","last_undegraded":"2022-09-29T05:25:38.517848+0000","last_fullsized":"2022-09-29T05:25:38.517848+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2b","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.518130+0000","last_change":"2022-09-29T05:25:03.824844+0000","last_active":"2022-09-29T05:25:38.518130+0000","last_peered":"2022-09-29T05:25:38.518130+0000","last_clean":"2022-09-29T05:25:38.518130+0000","last_became_active":"2022-09-29T05:25:03.824457+0000","last_became_peered":"2022-09-29T05:25:03.824457+0000","last_unstale":"2022-09-29T05:25:38.518130+0000","last_undegraded":"2022-09-29T05:25:38.518130+0000","last_fullsized":"2022-09-29T05:25:38.518130+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2c","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.518029+0000","last_change":"2022-09-29T05:25:03.812644+0000","last_active":"2022-09-29T05:25:38.518029+0000","last_peered":"2022-09-29T05:25:38.518029+0000","last_clean":"2022-09-29T05:25:38.518029+0000","last_became_active":"2022-09-29T05:25:03.812339+0000","last_became_peered":"2022-09-29T05:25:03.812339+0000","last_unstale":"2022-09-29T05:25:38.518029+0000","last_undegraded":"2022-09-29T05:25:38.518029+0000","last_fullsized":"2022-09-29T05:25:38.518029+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":192,"acting":192,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":32,"stat_sum":{"num_bytes":3070,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":48,"num_write_kb":16,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":9210,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":42,"ondisk_log_size":42,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":45,"seq":193273528340,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5612,"kb_used_data":548,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739540,"statfs":{"total":95995035648,"available":95989288960,"internally_reserved":0,"allocated":561152,"data_stored":237178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Sep 29 05:25:46 2022","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":1.081,"5min":1.081,"15min":1.081},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.47899999999999998}]},{"osd":1,"last update":"Thu Sep 29 05:25:46 2022","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.38200000000000001},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":1.0569999999999999,"5min":1.0569999999999999,"15min":1.0569999999999999},"last":0.56599999999999995}]},{"osd":2,"last update":"Thu Sep 29 05:25:46 2022","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":1.028,"5min":1.028,"15min":1.028},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.95599999999999996,"5min":0.95599999999999996,"15min":0.95599999999999996},"last":0.51100000000000001}]},{"osd":3,"last update":"Thu Sep 29 05:25:46 2022","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.88700000000000001,"5min":0.88700000000000001,"15min":0.88700000000000001},"last":0.39900000000000002},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.84199999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.53900000000000003}]},{"osd":4,"last update":"Thu Sep 29 05:25:46 2022","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":1.103,"5min":1.103,"15min":1.103},"last":0.35599999999999998},{"interface":"front","average":{"1min":0.49399999999999999,"5min":0.49399999999999999,"15min":0.49399999999999999},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.96999999999999997,"5min":0.96999999999999997,"15min":0.96999999999999997},"last":0.436}]},{"osd":5,"last update":"Thu Sep 29 05:25:46 2022","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":1.1240000000000001,"5min":1.1240000000000001,"15min":1.1240000000000001},"last":0.52500000000000002}]},{"osd":6,"last update":"Thu Sep 29 05:25:46 2022","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.94199999999999995,"5min":0.94199999999999995,"15min":0.94199999999999995},"last":0.5},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":1.143,"5min":1.143,"15min":1.143},"last":0.58999999999999997}]}]},{"osd":6,"up_from":40,"seq":171798691862,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5616,"kb_used_data":552,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739536,"statfs":{"total":95995035648,"available":95989284864,"internally_reserved":0,"allocated":565248,"data_stored":238902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Sep 29 05:25:30 2022","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":2.0489999999999999,"5min":2.0489999999999999,"15min":2.0489999999999999},"last":0.82699999999999996},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":1.466,"5min":1.466,"15min":1.466},"last":0.624}]},{"osd":1,"last update":"Thu Sep 29 05:25:30 2022","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":2.0670000000000002,"5min":2.0670000000000002,"15min":2.0670000000000002},"last":0.63400000000000001},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":2.02,"5min":2.02,"15min":2.02},"last":0.76100000000000001}]},{"osd":2,"last update":"Thu Sep 29 05:25:30 2022","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":2.081,"5min":2.081,"15min":2.081},"last":0.85099999999999998},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":1.4910000000000001,"5min":1.4910000000000001,"15min":1.4910000000000001},"last":0.67100000000000004}]},{"osd":3,"last update":"Thu Sep 29 05:25:30 2022","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":2.1059999999999999,"5min":2.1059999999999999,"15min":2.1059999999999999},"last":0.66600000000000004},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":1.9890000000000001,"5min":1.9890000000000001,"15min":1.9890000000000001},"last":0.80200000000000005}]},{"osd":4,"last update":"Thu Sep 29 05:25:30 2022","interfaces":[{"interface":"back","average":{"1min":0.49399999999999999,"5min":0.49399999999999999,"15min":0.49399999999999999},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":1.3580000000000001,"5min":1.3580000000000001,"15min":1.3580000000000001},"last":0.45700000000000002},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":2.1360000000000001,"5min":2.1360000000000001,"15min":2.1360000000000001},"last":0.48899999999999999}]},{"osd":5,"last update":"Thu Sep 29 05:25:30 2022","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":1.405,"5min":1.405,"15min":1.405},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.47799999999999998,"5min":0.47799999999999998,"15min":0.47799999999999998},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.55800000000000005}]},{"osd":7,"last update":"Thu Sep 29 05:25:47 2022","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.53400000000000003},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":1.4379999999999999,"5min":1.4379999999999999,"15min":1.4379999999999999},"last":0.53900000000000003}]}]},{"osd":1,"up_from":13,"seq":55834574887,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5872,"kb_used_data":552,"kb_used_omap":0,"kb_used_meta":5312,"kb_avail":93739280,"statfs":{"total":95995035648,"available":95989022720,"internally_reserved":0,"allocated":565248,"data_stored":237077,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5439488},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Sep 29 05:26:07 2022","interfaces":[{"interface":"back","average":{"1min":0.318,"5min":0.33500000000000002,"15min":0.33700000000000002},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":0.48099999999999998,"5min":0.52100000000000002,"15min":0.52100000000000002},"last":0.28999999999999998},{"interface":"front","average":{"1min":0.33500000000000002,"5min":0.36199999999999999,"15min":0.36599999999999999},"min":{"1min":0.247,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.54800000000000004,"5min":0.55200000000000005,"15min":0.55200000000000005},"last":0.33000000000000002}]},{"osd":2,"last update":"Thu Sep 29 05:25:23 2022","interfaces":[{"interface":"back","average":{"1min":0.38,"5min":0.38800000000000001,"15min":0.38900000000000001},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.53400000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"last":0.34200000000000003},{"interface":"front","average":{"1min":0.41699999999999998,"5min":0.45000000000000001,"15min":0.45600000000000002},"min":{"1min":0.308,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.61399999999999999,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.29999999999999999}]},{"osd":3,"last update":"Thu Sep 29 05:25:45 2022","interfaces":[{"interface":"back","average":{"1min":0.34399999999999997,"5min":0.436,"15min":0.45200000000000001},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.49199999999999999,"5min":0.64400000000000002,"15min":0.64400000000000002},"last":0.46200000000000002},{"interface":"front","average":{"1min":0.34599999999999997,"5min":0.42199999999999999,"15min":0.435},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":0.46500000000000002,"5min":0.61699999999999999,"15min":0.61699999999999999},"last":0.40500000000000003}]},{"osd":4,"last update":"Thu Sep 29 05:26:02 2022","interfaces":[{"interface":"back","average":{"1min":0.59799999999999998,"5min":0.60999999999999999,"15min":0.61199999999999999},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.76700000000000002,"5min":1.1619999999999999,"15min":1.1619999999999999},"last":0.51400000000000001},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.59299999999999997,"15min":0.59499999999999997},"min":{"1min":0.432,"5min":0.38,"15min":0.38},"max":{"1min":0.75600000000000001,"5min":1.1799999999999999,"15min":1.1799999999999999},"last":0.45400000000000001}]},{"osd":5,"last update":"Thu Sep 29 05:25:17 2022","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":1.137,"5min":1.137,"15min":1.137},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.629,"5min":0.629,"15min":0.629},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":1.21,"5min":1.21,"15min":1.21},"last":0.53100000000000003}]},{"osd":6,"last update":"Thu Sep 29 05:25:34 2022","interfaces":[{"interface":"back","average":{"1min":0.749,"5min":0.749,"15min":0.749},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":4.173,"5min":4.173,"15min":4.173},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":1.196,"5min":1.196,"15min":1.196},"last":0.495}]},{"osd":7,"last update":"Thu Sep 29 05:25:46 2022","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":1.147,"5min":1.147,"15min":1.147},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":1.8049999999999999,"5min":1.8049999999999999,"15min":1.8049999999999999},"last":0.58799999999999997}]}]},{"osd":0,"up_from":8,"seq":34359738411,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5944,"kb_used_data":560,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":93739208,"statfs":{"total":95995035648,"available":95988948992,"internally_reserved":0,"allocated":573440,"data_stored":237617,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Sep 29 05:26:06 2022","interfaces":[{"interface":"back","average":{"1min":0.32900000000000001,"5min":0.33200000000000002,"15min":0.32400000000000001},"min":{"1min":0.22,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.46899999999999997,"5min":0.748,"15min":0.748},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.316,"5min":0.34499999999999997,"15min":0.34000000000000002},"min":{"1min":0.20699999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":0.498,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.35699999999999998}]},{"osd":2,"last update":"Thu Sep 29 05:25:31 2022","interfaces":[{"interface":"back","average":{"1min":0.36399999999999999,"5min":0.35799999999999998,"15min":0.35699999999999998},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"last":0.32300000000000001},{"interface":"front","average":{"1min":0.40100000000000002,"5min":0.38100000000000001,"15min":0.378},"min":{"1min":0.29799999999999999,"5min":0.22,"15min":0.22},"max":{"1min":0.65900000000000003,"5min":0.65900000000000003,"15min":0.65900000000000003},"last":0.33500000000000002}]},{"osd":3,"last update":"Thu Sep 29 05:25:42 2022","interfaces":[{"interface":"back","average":{"1min":0.39200000000000002,"5min":0.42799999999999999,"15min":0.434},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":0.54700000000000004,"5min":0.58999999999999997,"15min":0.58999999999999997},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.36799999999999999,"5min":0.41999999999999998,"15min":0.42899999999999999},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.57799999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"last":0.29399999999999998}]},{"osd":4,"last update":"Thu Sep 29 05:26:01 2022","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.53700000000000003,"15min":0.53200000000000003},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.503},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.56699999999999995,"15min":0.56399999999999995},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.77500000000000002,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.58999999999999997}]},{"osd":5,"last update":"Thu Sep 29 05:25:13 2022","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.81999999999999995,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.59799999999999998},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":0.85199999999999998,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.64700000000000002}]},{"osd":6,"last update":"Thu Sep 29 05:25:31 2022","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.47399999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"min":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.53300000000000003}]},{"osd":7,"last update":"Thu Sep 29 05:25:47 2022","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"min":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.58099999999999996},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.66300000000000003}]}]},{"osd":2,"up_from":18,"seq":77309411364,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5736,"kb_used_data":544,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739416,"statfs":{"total":95995035648,"available":95989161984,"internally_reserved":0,"allocated":557056,"data_stored":236917,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Sep 29 05:25:28 2022","interfaces":[{"interface":"back","average":{"1min":0.34699999999999998,"5min":0.35999999999999999,"15min":0.36199999999999999},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.84299999999999997,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.29699999999999999},{"interface":"front","average":{"1min":0.373,"5min":0.38900000000000001,"15min":0.39200000000000002},"min":{"1min":0.27400000000000002,"5min":0.219,"15min":0.219},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.40100000000000002}]},{"osd":1,"last update":"Thu Sep 29 05:25:28 2022","interfaces":[{"interface":"back","average":{"1min":0.35299999999999998,"5min":0.374,"15min":0.378},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.53200000000000003,"5min":0.59599999999999997,"15min":0.59599999999999997},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.40500000000000003,"5min":0.41199999999999998,"15min":0.41299999999999998},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.57899999999999996}]},{"osd":3,"last update":"Thu Sep 29 05:25:44 2022","interfaces":[{"interface":"back","average":{"1min":0.39100000000000001,"5min":0.44800000000000001,"15min":0.45800000000000002},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":1.0009999999999999,"5min":1.0009999999999999,"15min":1.0009999999999999},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.38900000000000001,"5min":0.44800000000000001,"15min":0.45800000000000002},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.61599999999999999,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.443}]},{"osd":4,"last update":"Thu Sep 29 05:25:59 2022","interfaces":[{"interface":"back","average":{"1min":0.49399999999999999,"5min":0.53500000000000003,"15min":0.54200000000000004},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.86499999999999999,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.505},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.57499999999999996,"15min":0.57799999999999996},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.91000000000000003,"5min":0.91000000000000003,"15min":0.91000000000000003},"last":0.59699999999999998}]},{"osd":5,"last update":"Thu Sep 29 05:25:14 2022","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.53900000000000003},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.73699999999999999,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.46400000000000002}]},{"osd":6,"last update":"Thu Sep 29 05:25:34 2022","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.97999999999999998,"5min":0.97999999999999998,"15min":0.97999999999999998},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"last":0.55400000000000005}]},{"osd":7,"last update":"Thu Sep 29 05:25:52 2022","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"min":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"max":{"1min":0.95799999999999996,"5min":0.95799999999999996,"15min":0.95799999999999996},"last":0.61699999999999999},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.58899999999999997}]}]},{"osd":3,"up_from":26,"seq":111669149728,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5756,"kb_used_data":564,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739396,"statfs":{"total":95995035648,"available":95989141504,"internally_reserved":0,"allocated":577536,"data_stored":239321,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Sep 29 05:25:48 2022","interfaces":[{"interface":"back","average":{"1min":0.317,"5min":0.35099999999999998,"15min":0.35699999999999998},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":0.495,"5min":0.495,"15min":0.495},"last":0.33800000000000002},{"interface":"front","average":{"1min":0.35299999999999998,"5min":0.39400000000000002,"15min":0.40100000000000002},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.46200000000000002,"5min":0.63500000000000001,"15min":0.63500000000000001},"last":0.32100000000000001}]},{"osd":1,"last update":"Thu Sep 29 05:25:48 2022","interfaces":[{"interface":"back","average":{"1min":0.34899999999999998,"5min":0.39900000000000002,"15min":0.40699999999999997},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":0.51900000000000002,"5min":0.57799999999999996,"15min":0.57799999999999996},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.36599999999999999,"5min":0.40100000000000002,"15min":0.40699999999999997},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.58099999999999996,"5min":0.60999999999999999,"15min":0.60999999999999999},"last":0.35499999999999998}]},{"osd":2,"last update":"Thu Sep 29 05:25:48 2022","interfaces":[{"interface":"back","average":{"1min":0.377,"5min":0.43099999999999999,"15min":0.44},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.58899999999999997,"5min":0.59999999999999998,"15min":0.59999999999999998},"last":0.307},{"interface":"front","average":{"1min":0.379,"5min":0.41899999999999998,"15min":0.42499999999999999},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.86099999999999999,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.377}]},{"osd":4,"last update":"Thu Sep 29 05:26:02 2022","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.57099999999999995,"15min":0.58099999999999996},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.65600000000000003,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.56799999999999995,"15min":0.57599999999999996},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.69399999999999995,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.46999999999999997}]},{"osd":5,"last update":"Thu Sep 29 05:25:15 2022","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.40200000000000002},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":0.876,"5min":0.876,"15min":0.876},"last":0.44400000000000001}]},{"osd":6,"last update":"Thu Sep 29 05:25:33 2022","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.80400000000000005,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.503},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.876,"5min":0.876,"15min":0.876},"last":0.45700000000000002}]},{"osd":7,"last update":"Thu Sep 29 05:25:48 2022","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":0.89000000000000001,"5min":0.89000000000000001,"15min":0.89000000000000001},"last":0.436},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.48699999999999999}]}]},{"osd":4,"up_from":30,"seq":128849018908,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5752,"kb_used_data":560,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739400,"statfs":{"total":95995035648,"available":95989145600,"internally_reserved":0,"allocated":573440,"data_stored":237597,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Sep 29 05:26:01 2022","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.48299999999999998,"15min":0.47899999999999998},"min":{"1min":0.39800000000000002,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.70899999999999996,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.58299999999999996},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.50900000000000001,"15min":0.50700000000000001},"min":{"1min":0.38800000000000001,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.436}]},{"osd":1,"last update":"Thu Sep 29 05:26:01 2022","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.51000000000000001,"15min":0.50800000000000001},"min":{"1min":0.36099999999999999,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.504,"5min":0.51000000000000001,"15min":0.51100000000000001},"min":{"1min":0.374,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.51000000000000001}]},{"osd":2,"last update":"Thu Sep 29 05:26:01 2022","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.52600000000000002,"15min":0.52800000000000002},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.72099999999999997,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.53500000000000003,"15min":0.52300000000000002},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":1.899,"5min":1.899,"15min":1.899},"last":0.44800000000000001}]},{"osd":3,"last update":"Thu Sep 29 05:26:01 2022","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.54900000000000004,"15min":0.54800000000000004},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.95199999999999996,"5min":0.95199999999999996,"15min":0.95199999999999996},"last":0.53800000000000003},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.54100000000000004,"15min":0.53600000000000003},"min":{"1min":0.45200000000000001,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.96099999999999997,"5min":0.96099999999999997,"15min":0.96099999999999997},"last":0.63500000000000001}]},{"osd":5,"last update":"Thu Sep 29 05:25:14 2022","interfaces":[{"interface":"back","average":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.56399999999999995}]},{"osd":6,"last update":"Thu Sep 29 05:25:30 2022","interfaces":[{"interface":"back","average":{"1min":0.432,"5min":0.432,"15min":0.432},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.60699999999999998},{"interface":"front","average":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.80400000000000005,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.57199999999999995}]},{"osd":7,"last update":"Thu Sep 29 05:25:47 2022","interfaces":[{"interface":"back","average":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.51400000000000001},{"interface":"front","average":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.65300000000000002}]}]},{"osd":5,"up_from":35,"seq":150323855387,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5684,"kb_used_data":556,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739468,"statfs":{"total":95995035648,"available":95989215232,"internally_reserved":0,"allocated":569344,"data_stored":238992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Sep 29 05:25:14 2022","interfaces":[{"interface":"back","average":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.627,"5min":0.627,"15min":0.627},"last":0.53000000000000003},{"interface":"front","average":{"1min":0.48999999999999999,"5min":0.48999999999999999,"15min":0.48999999999999999},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.60399999999999998}]},{"osd":1,"last update":"Thu Sep 29 05:25:14 2022","interfaces":[{"interface":"back","average":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.67600000000000005},{"interface":"front","average":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.63700000000000001}]},{"osd":2,"last update":"Thu Sep 29 05:25:14 2022","interfaces":[{"interface":"back","average":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"last":0.56699999999999995},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.72799999999999998}]},{"osd":3,"last update":"Thu Sep 29 05:25:14 2022","interfaces":[{"interface":"back","average":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.64200000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"last":0.624},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":0.68999999999999995,"5min":0.68999999999999995,"15min":0.68999999999999995},"last":0.75800000000000001}]},{"osd":4,"last update":"Thu Sep 29 05:25:14 2022","interfaces":[{"interface":"back","average":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"last":0.66600000000000004},{"interface":"front","average":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.73299999999999998}]},{"osd":6,"last update":"Thu Sep 29 05:25:31 2022","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.63,"5min":0.63,"15min":0.63},"last":0.59499999999999997},{"interface":"front","average":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"last":0.69399999999999995}]},{"osd":7,"last update":"Thu Sep 29 05:25:48 2022","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.65500000000000003},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.70899999999999996}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":700,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":2404,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":2390,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":2300,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2022-09-29T05:26:16.082 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/daemon-base:latest-pacific shell --fsid 69111522-3fb6-11ed-8431-001a4aab830c -- ceph pg dump --format=json 2022-09-29T05:26:16.111 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:26:15.424561+0000 mgr.y (mgr.14577) 49 : audit [DBG] from='client.24673 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2022-09-29T05:26:16.111 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:26:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[62589]: level=info ts=2022-09-29T05:26:15.944Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2022-09-29T05:26:16.112 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:26:15.424561+0000 mgr.y (mgr.14577) 49 : audit [DBG] from='client.24673 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2022-09-29T05:26:16.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:15 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:26:15.424561+0000 mgr.y (mgr.14577) 49 : audit [DBG] from='client.24673 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2022-09-29T05:26:16.385 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:26:16 smithi083 bash[66984]: ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a 2022-09-29T05:26:16.705 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/69111522-3fb6-11ed-8431-001a4aab830c/mon.a/config 2022-09-29T05:26:16.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:26:16 smithi083 bash[67053]: Error: no container with name or ID "ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager.a" found: no such container 2022-09-29T05:26:16.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:26:16 smithi083 systemd[1]: ceph-69111522-3fb6-11ed-8431-001a4aab830c@alertmanager.a.service: Succeeded. 2022-09-29T05:26:16.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:26:16 smithi083 systemd[1]: Stopped Ceph alertmanager.a for 69111522-3fb6-11ed-8431-001a4aab830c. 2022-09-29T05:26:16.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:26:16 smithi083 systemd[1]: Starting Ceph alertmanager.a for 69111522-3fb6-11ed-8431-001a4aab830c... 2022-09-29T05:26:17.104 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:26:15.820432+0000 mgr.y (mgr.14577) 50 : cluster [DBG] pgmap v35: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:26:17.104 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:26:15.820432+0000 mgr.y (mgr.14577) 50 : cluster [DBG] pgmap v35: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:26:17.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:16 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:26:15.820432+0000 mgr.y (mgr.14577) 50 : cluster [DBG] pgmap v35: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:26:17.902 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:26:17 smithi083 podman[67203]: 2022-09-29T05:26:18.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:26:17 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=info ts=2022-09-29T05:26:17.899Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2022-09-29T05:26:18.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:26:17 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=info ts=2022-09-29T05:26:17.899Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2022-09-29T05:26:18.214 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:26:17 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=info ts=2022-09-29T05:26:17.900Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.83 port=9094 2022-09-29T05:26:18.214 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:26:17 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=info ts=2022-09-29T05:26:17.901Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2022-09-29T05:26:18.214 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:26:17 smithi083 bash[67203]: 4af0d0fa9b288fca97bed7c961edf9a6cd1a053412b4763d7e4fb3d1bf47878a 2022-09-29T05:26:18.214 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:26:17 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=info ts=2022-09-29T05:26:17.927Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2022-09-29T05:26:18.214 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:26:17 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=info ts=2022-09-29T05:26:17.927Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2022-09-29T05:26:18.215 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:26:17 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=info ts=2022-09-29T05:26:17.929Z caller=main.go:518 msg=Listening address=:9093 2022-09-29T05:26:18.215 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:26:17 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=info ts=2022-09-29T05:26:17.929Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2022-09-29T05:26:18.215 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:26:17 smithi083 systemd[1]: Started Ceph alertmanager.a for 69111522-3fb6-11ed-8431-001a4aab830c. 2022-09-29T05:26:18.646 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:26:18.648 INFO:teuthology.orchestra.run.smithi083.stderr:dumped all 2022-09-29T05:26:19.117 INFO:teuthology.orchestra.run.smithi083.stdout:{"pg_ready":true,"pg_map":{"version":36,"stamp":"2022-09-29T05:26:17.820648+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3070,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":48,"num_write_kb":16,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":42,"ondisk_log_size":42,"up":315,"acting":315,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":315,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":45972,"kb_used_data":4436,"kb_used_omap":0,"kb_used_meta":41472,"kb_avail":749915244,"statfs":{"total":767960285184,"available":767913209856,"internally_reserved":0,"allocated":4542464,"data_stored":1903601,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":42467328},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002792"},"pg_stats":[{"pgid":"3.3f","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.518942+0000","last_change":"2022-09-29T05:25:03.829987+0000","last_active":"2022-09-29T05:25:38.518942+0000","last_peered":"2022-09-29T05:25:38.518942+0000","last_clean":"2022-09-29T05:25:38.518942+0000","last_became_active":"2022-09-29T05:25:03.829703+0000","last_became_peered":"2022-09-29T05:25:03.829703+0000","last_unstale":"2022-09-29T05:25:38.518942+0000","last_undegraded":"2022-09-29T05:25:38.518942+0000","last_fullsized":"2022-09-29T05:25:38.518942+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3e","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.141088+0000","last_change":"2022-09-29T05:25:03.816168+0000","last_active":"2022-09-29T05:25:38.141088+0000","last_peered":"2022-09-29T05:25:38.141088+0000","last_clean":"2022-09-29T05:25:38.141088+0000","last_became_active":"2022-09-29T05:25:03.815888+0000","last_became_peered":"2022-09-29T05:25:03.815888+0000","last_unstale":"2022-09-29T05:25:38.141088+0000","last_undegraded":"2022-09-29T05:25:38.141088+0000","last_fullsized":"2022-09-29T05:25:38.141088+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3d","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.518173+0000","last_change":"2022-09-29T05:25:03.814221+0000","last_active":"2022-09-29T05:25:38.518173+0000","last_peered":"2022-09-29T05:25:38.518173+0000","last_clean":"2022-09-29T05:25:38.518173+0000","last_became_active":"2022-09-29T05:25:03.813927+0000","last_became_peered":"2022-09-29T05:25:03.813927+0000","last_unstale":"2022-09-29T05:25:38.518173+0000","last_undegraded":"2022-09-29T05:25:38.518173+0000","last_fullsized":"2022-09-29T05:25:38.518173+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3c","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.518789+0000","last_change":"2022-09-29T05:25:03.819554+0000","last_active":"2022-09-29T05:25:38.518789+0000","last_peered":"2022-09-29T05:25:38.518789+0000","last_clean":"2022-09-29T05:25:38.518789+0000","last_became_active":"2022-09-29T05:25:03.819207+0000","last_became_peered":"2022-09-29T05:25:03.819207+0000","last_unstale":"2022-09-29T05:25:38.518789+0000","last_undegraded":"2022-09-29T05:25:38.518789+0000","last_fullsized":"2022-09-29T05:25:38.518789+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3b","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.519460+0000","last_change":"2022-09-29T05:25:03.829034+0000","last_active":"2022-09-29T05:25:38.519460+0000","last_peered":"2022-09-29T05:25:38.519460+0000","last_clean":"2022-09-29T05:25:38.519460+0000","last_became_active":"2022-09-29T05:25:03.828384+0000","last_became_peered":"2022-09-29T05:25:03.828384+0000","last_unstale":"2022-09-29T05:25:38.519460+0000","last_undegraded":"2022-09-29T05:25:38.519460+0000","last_fullsized":"2022-09-29T05:25:38.519460+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3a","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.517990+0000","last_change":"2022-09-29T05:25:03.817871+0000","last_active":"2022-09-29T05:25:38.517990+0000","last_peered":"2022-09-29T05:25:38.517990+0000","last_clean":"2022-09-29T05:25:38.517990+0000","last_became_active":"2022-09-29T05:25:03.816800+0000","last_became_peered":"2022-09-29T05:25:03.816800+0000","last_unstale":"2022-09-29T05:25:38.517990+0000","last_undegraded":"2022-09-29T05:25:38.517990+0000","last_fullsized":"2022-09-29T05:25:38.517990+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.39","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.517638+0000","last_change":"2022-09-29T05:25:03.825279+0000","last_active":"2022-09-29T05:25:38.517638+0000","last_peered":"2022-09-29T05:25:38.517638+0000","last_clean":"2022-09-29T05:25:38.517638+0000","last_became_active":"2022-09-29T05:25:03.825002+0000","last_became_peered":"2022-09-29T05:25:03.825002+0000","last_unstale":"2022-09-29T05:25:38.517638+0000","last_undegraded":"2022-09-29T05:25:38.517638+0000","last_fullsized":"2022-09-29T05:25:38.517638+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.38","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.517254+0000","last_change":"2022-09-29T05:25:03.813803+0000","last_active":"2022-09-29T05:25:38.517254+0000","last_peered":"2022-09-29T05:25:38.517254+0000","last_clean":"2022-09-29T05:25:38.517254+0000","last_became_active":"2022-09-29T05:25:03.813488+0000","last_became_peered":"2022-09-29T05:25:03.813488+0000","last_unstale":"2022-09-29T05:25:38.517254+0000","last_undegraded":"2022-09-29T05:25:38.517254+0000","last_fullsized":"2022-09-29T05:25:38.517254+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.37","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.518868+0000","last_change":"2022-09-29T05:25:03.829880+0000","last_active":"2022-09-29T05:25:38.518868+0000","last_peered":"2022-09-29T05:25:38.518868+0000","last_clean":"2022-09-29T05:25:38.518868+0000","last_became_active":"2022-09-29T05:25:03.829318+0000","last_became_peered":"2022-09-29T05:25:03.829318+0000","last_unstale":"2022-09-29T05:25:38.518868+0000","last_undegraded":"2022-09-29T05:25:38.518868+0000","last_fullsized":"2022-09-29T05:25:38.518868+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,0],"acting":[7,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.36","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.517536+0000","last_change":"2022-09-29T05:25:03.823656+0000","last_active":"2022-09-29T05:25:38.517536+0000","last_peered":"2022-09-29T05:25:38.517536+0000","last_clean":"2022-09-29T05:25:38.517536+0000","last_became_active":"2022-09-29T05:25:03.823109+0000","last_became_peered":"2022-09-29T05:25:03.823109+0000","last_unstale":"2022-09-29T05:25:38.517536+0000","last_undegraded":"2022-09-29T05:25:38.517536+0000","last_fullsized":"2022-09-29T05:25:38.517536+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.35","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.518474+0000","last_change":"2022-09-29T05:25:03.824377+0000","last_active":"2022-09-29T05:25:38.518474+0000","last_peered":"2022-09-29T05:25:38.518474+0000","last_clean":"2022-09-29T05:25:38.518474+0000","last_became_active":"2022-09-29T05:25:03.823869+0000","last_became_peered":"2022-09-29T05:25:03.823869+0000","last_unstale":"2022-09-29T05:25:38.518474+0000","last_undegraded":"2022-09-29T05:25:38.518474+0000","last_fullsized":"2022-09-29T05:25:38.518474+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.34","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.136241+0000","last_change":"2022-09-29T05:25:03.812233+0000","last_active":"2022-09-29T05:25:38.136241+0000","last_peered":"2022-09-29T05:25:38.136241+0000","last_clean":"2022-09-29T05:25:38.136241+0000","last_became_active":"2022-09-29T05:25:03.811765+0000","last_became_peered":"2022-09-29T05:25:03.811765+0000","last_unstale":"2022-09-29T05:25:38.136241+0000","last_undegraded":"2022-09-29T05:25:38.136241+0000","last_fullsized":"2022-09-29T05:25:38.136241+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.33","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.517404+0000","last_change":"2022-09-29T05:25:03.813308+0000","last_active":"2022-09-29T05:25:38.517404+0000","last_peered":"2022-09-29T05:25:38.517404+0000","last_clean":"2022-09-29T05:25:38.517404+0000","last_became_active":"2022-09-29T05:25:03.813019+0000","last_became_peered":"2022-09-29T05:25:03.813019+0000","last_unstale":"2022-09-29T05:25:38.517404+0000","last_undegraded":"2022-09-29T05:25:38.517404+0000","last_fullsized":"2022-09-29T05:25:38.517404+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.32","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.517710+0000","last_change":"2022-09-29T05:25:03.823321+0000","last_active":"2022-09-29T05:25:38.517710+0000","last_peered":"2022-09-29T05:25:38.517710+0000","last_clean":"2022-09-29T05:25:38.517710+0000","last_became_active":"2022-09-29T05:25:03.822987+0000","last_became_peered":"2022-09-29T05:25:03.822987+0000","last_unstale":"2022-09-29T05:25:38.517710+0000","last_undegraded":"2022-09-29T05:25:38.517710+0000","last_fullsized":"2022-09-29T05:25:38.517710+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.31","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.141395+0000","last_change":"2022-09-29T05:25:03.805590+0000","last_active":"2022-09-29T05:25:38.141395+0000","last_peered":"2022-09-29T05:25:38.141395+0000","last_clean":"2022-09-29T05:25:38.141395+0000","last_became_active":"2022-09-29T05:25:03.805310+0000","last_became_peered":"2022-09-29T05:25:03.805310+0000","last_unstale":"2022-09-29T05:25:38.141395+0000","last_undegraded":"2022-09-29T05:25:38.141395+0000","last_fullsized":"2022-09-29T05:25:38.141395+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.30","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.517306+0000","last_change":"2022-09-29T05:25:03.815886+0000","last_active":"2022-09-29T05:25:38.517306+0000","last_peered":"2022-09-29T05:25:38.517306+0000","last_clean":"2022-09-29T05:25:38.517306+0000","last_became_active":"2022-09-29T05:25:03.815285+0000","last_became_peered":"2022-09-29T05:25:03.815285+0000","last_unstale":"2022-09-29T05:25:38.517306+0000","last_undegraded":"2022-09-29T05:25:38.517306+0000","last_fullsized":"2022-09-29T05:25:38.517306+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2f","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.518592+0000","last_change":"2022-09-29T05:25:03.813784+0000","last_active":"2022-09-29T05:25:38.518592+0000","last_peered":"2022-09-29T05:25:38.518592+0000","last_clean":"2022-09-29T05:25:38.518592+0000","last_became_active":"2022-09-29T05:25:03.813420+0000","last_became_peered":"2022-09-29T05:25:03.813420+0000","last_unstale":"2022-09-29T05:25:38.518592+0000","last_undegraded":"2022-09-29T05:25:38.518592+0000","last_fullsized":"2022-09-29T05:25:38.518592+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2e","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.141808+0000","last_change":"2022-09-29T05:25:03.813894+0000","last_active":"2022-09-29T05:25:38.141808+0000","last_peered":"2022-09-29T05:25:38.141808+0000","last_clean":"2022-09-29T05:25:38.141808+0000","last_became_active":"2022-09-29T05:25:03.813620+0000","last_became_peered":"2022-09-29T05:25:03.813620+0000","last_unstale":"2022-09-29T05:25:38.141808+0000","last_undegraded":"2022-09-29T05:25:38.141808+0000","last_fullsized":"2022-09-29T05:25:38.141808+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,7],"acting":[2,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2d","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.519104+0000","last_change":"2022-09-29T05:25:03.828324+0000","last_active":"2022-09-29T05:25:38.519104+0000","last_peered":"2022-09-29T05:25:38.519104+0000","last_clean":"2022-09-29T05:25:38.519104+0000","last_became_active":"2022-09-29T05:25:03.827685+0000","last_became_peered":"2022-09-29T05:25:03.827685+0000","last_unstale":"2022-09-29T05:25:38.519104+0000","last_undegraded":"2022-09-29T05:25:38.519104+0000","last_fullsized":"2022-09-29T05:25:38.519104+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1f","version":"50'4","reported_seq":20,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.518334+0000","last_change":"2022-09-29T05:25:03.449078+0000","last_active":"2022-09-29T05:25:38.518334+0000","last_peered":"2022-09-29T05:25:38.518334+0000","last_clean":"2022-09-29T05:25:38.518334+0000","last_became_active":"2022-09-29T05:25:03.448793+0000","last_became_peered":"2022-09-29T05:25:03.448793+0000","last_unstale":"2022-09-29T05:25:38.518334+0000","last_undegraded":"2022-09-29T05:25:38.518334+0000","last_fullsized":"2022-09-29T05:25:38.518334+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":4,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.517840+0000","last_change":"2022-09-29T05:25:03.814538+0000","last_active":"2022-09-29T05:25:38.517840+0000","last_peered":"2022-09-29T05:25:38.517840+0000","last_clean":"2022-09-29T05:25:38.517840+0000","last_became_active":"2022-09-29T05:25:03.814264+0000","last_became_peered":"2022-09-29T05:25:03.814264+0000","last_unstale":"2022-09-29T05:25:38.517840+0000","last_undegraded":"2022-09-29T05:25:38.517840+0000","last_fullsized":"2022-09-29T05:25:38.517840+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1e","version":"50'1","reported_seq":17,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.517756+0000","last_change":"2022-09-29T05:25:03.459868+0000","last_active":"2022-09-29T05:25:38.517756+0000","last_peered":"2022-09-29T05:25:38.517756+0000","last_clean":"2022-09-29T05:25:38.517756+0000","last_became_active":"2022-09-29T05:25:03.459458+0000","last_became_peered":"2022-09-29T05:25:03.459458+0000","last_unstale":"2022-09-29T05:25:38.517756+0000","last_undegraded":"2022-09-29T05:25:38.517756+0000","last_fullsized":"2022-09-29T05:25:38.517756+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.518123+0000","last_change":"2022-09-29T05:25:03.822430+0000","last_active":"2022-09-29T05:25:38.518123+0000","last_peered":"2022-09-29T05:25:38.518123+0000","last_clean":"2022-09-29T05:25:38.518123+0000","last_became_active":"2022-09-29T05:25:03.821898+0000","last_became_peered":"2022-09-29T05:25:03.821898+0000","last_unstale":"2022-09-29T05:25:38.518123+0000","last_undegraded":"2022-09-29T05:25:38.518123+0000","last_fullsized":"2022-09-29T05:25:38.518123+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1d","version":"50'2","reported_seq":18,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.519405+0000","last_change":"2022-09-29T05:25:03.455815+0000","last_active":"2022-09-29T05:25:38.519405+0000","last_peered":"2022-09-29T05:25:38.519405+0000","last_clean":"2022-09-29T05:25:38.519405+0000","last_became_active":"2022-09-29T05:25:03.455218+0000","last_became_peered":"2022-09-29T05:25:03.455218+0000","last_unstale":"2022-09-29T05:25:38.519405+0000","last_undegraded":"2022-09-29T05:25:38.519405+0000","last_fullsized":"2022-09-29T05:25:38.519405+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.518874+0000","last_change":"2022-09-29T05:25:03.816581+0000","last_active":"2022-09-29T05:25:38.518874+0000","last_peered":"2022-09-29T05:25:38.518874+0000","last_clean":"2022-09-29T05:25:38.518874+0000","last_became_active":"2022-09-29T05:25:03.816206+0000","last_became_peered":"2022-09-29T05:25:03.816206+0000","last_unstale":"2022-09-29T05:25:38.518874+0000","last_undegraded":"2022-09-29T05:25:38.518874+0000","last_fullsized":"2022-09-29T05:25:38.518874+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1c","version":"50'2","reported_seq":18,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.518052+0000","last_change":"2022-09-29T05:25:03.460578+0000","last_active":"2022-09-29T05:25:38.518052+0000","last_peered":"2022-09-29T05:25:38.518052+0000","last_clean":"2022-09-29T05:25:38.518052+0000","last_became_active":"2022-09-29T05:25:03.460273+0000","last_became_peered":"2022-09-29T05:25:03.460273+0000","last_unstale":"2022-09-29T05:25:38.518052+0000","last_undegraded":"2022-09-29T05:25:38.518052+0000","last_fullsized":"2022-09-29T05:25:38.518052+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1d","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.518215+0000","last_change":"2022-09-29T05:25:03.820523+0000","last_active":"2022-09-29T05:25:38.518215+0000","last_peered":"2022-09-29T05:25:38.518215+0000","last_clean":"2022-09-29T05:25:38.518215+0000","last_became_active":"2022-09-29T05:25:03.819546+0000","last_became_peered":"2022-09-29T05:25:03.819546+0000","last_unstale":"2022-09-29T05:25:38.518215+0000","last_undegraded":"2022-09-29T05:25:38.518215+0000","last_fullsized":"2022-09-29T05:25:38.518215+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1b","version":"50'1","reported_seq":17,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.517635+0000","last_change":"2022-09-29T05:25:03.455758+0000","last_active":"2022-09-29T05:25:38.517635+0000","last_peered":"2022-09-29T05:25:38.517635+0000","last_clean":"2022-09-29T05:25:38.517635+0000","last_became_active":"2022-09-29T05:25:03.455438+0000","last_became_peered":"2022-09-29T05:25:03.455438+0000","last_unstale":"2022-09-29T05:25:38.517635+0000","last_undegraded":"2022-09-29T05:25:38.517635+0000","last_fullsized":"2022-09-29T05:25:38.517635+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.135947+0000","last_change":"2022-09-29T05:25:03.817393+0000","last_active":"2022-09-29T05:25:38.135947+0000","last_peered":"2022-09-29T05:25:38.135947+0000","last_clean":"2022-09-29T05:25:38.135947+0000","last_became_active":"2022-09-29T05:25:03.817103+0000","last_became_peered":"2022-09-29T05:25:03.817103+0000","last_unstale":"2022-09-29T05:25:38.135947+0000","last_undegraded":"2022-09-29T05:25:38.135947+0000","last_fullsized":"2022-09-29T05:25:38.135947+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1a","version":"50'1","reported_seq":17,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.517949+0000","last_change":"2022-09-29T05:25:03.454747+0000","last_active":"2022-09-29T05:25:38.517949+0000","last_peered":"2022-09-29T05:25:38.517949+0000","last_clean":"2022-09-29T05:25:38.517949+0000","last_became_active":"2022-09-29T05:25:03.454395+0000","last_became_peered":"2022-09-29T05:25:03.454395+0000","last_unstale":"2022-09-29T05:25:38.517949+0000","last_undegraded":"2022-09-29T05:25:38.517949+0000","last_fullsized":"2022-09-29T05:25:38.517949+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1b","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.518041+0000","last_change":"2022-09-29T05:25:03.819253+0000","last_active":"2022-09-29T05:25:38.518041+0000","last_peered":"2022-09-29T05:25:38.518041+0000","last_clean":"2022-09-29T05:25:38.518041+0000","last_became_active":"2022-09-29T05:25:03.818951+0000","last_became_peered":"2022-09-29T05:25:03.818951+0000","last_unstale":"2022-09-29T05:25:38.518041+0000","last_undegraded":"2022-09-29T05:25:38.518041+0000","last_fullsized":"2022-09-29T05:25:38.518041+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.19","version":"0'0","reported_seq":16,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.518293+0000","last_change":"2022-09-29T05:25:03.455193+0000","last_active":"2022-09-29T05:25:38.518293+0000","last_peered":"2022-09-29T05:25:38.518293+0000","last_clean":"2022-09-29T05:25:38.518293+0000","last_became_active":"2022-09-29T05:25:03.454916+0000","last_became_peered":"2022-09-29T05:25:03.454916+0000","last_unstale":"2022-09-29T05:25:38.518293+0000","last_undegraded":"2022-09-29T05:25:38.518293+0000","last_fullsized":"2022-09-29T05:25:38.518293+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.517315+0000","last_change":"2022-09-29T05:25:03.806930+0000","last_active":"2022-09-29T05:25:38.517315+0000","last_peered":"2022-09-29T05:25:38.517315+0000","last_clean":"2022-09-29T05:25:38.517315+0000","last_became_active":"2022-09-29T05:25:03.806565+0000","last_became_peered":"2022-09-29T05:25:03.806565+0000","last_unstale":"2022-09-29T05:25:38.517315+0000","last_undegraded":"2022-09-29T05:25:38.517315+0000","last_fullsized":"2022-09-29T05:25:38.517315+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.18","version":"0'0","reported_seq":16,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.517443+0000","last_change":"2022-09-29T05:25:03.460675+0000","last_active":"2022-09-29T05:25:38.517443+0000","last_peered":"2022-09-29T05:25:38.517443+0000","last_clean":"2022-09-29T05:25:38.517443+0000","last_became_active":"2022-09-29T05:25:03.460351+0000","last_became_peered":"2022-09-29T05:25:03.460351+0000","last_unstale":"2022-09-29T05:25:38.517443+0000","last_undegraded":"2022-09-29T05:25:38.517443+0000","last_fullsized":"2022-09-29T05:25:38.517443+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.141604+0000","last_change":"2022-09-29T05:25:03.816170+0000","last_active":"2022-09-29T05:25:38.141604+0000","last_peered":"2022-09-29T05:25:38.141604+0000","last_clean":"2022-09-29T05:25:38.141604+0000","last_became_active":"2022-09-29T05:25:03.815838+0000","last_became_peered":"2022-09-29T05:25:03.815838+0000","last_unstale":"2022-09-29T05:25:38.141604+0000","last_undegraded":"2022-09-29T05:25:38.141604+0000","last_fullsized":"2022-09-29T05:25:38.141604+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.17","version":"0'0","reported_seq":16,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.518267+0000","last_change":"2022-09-29T05:25:03.461045+0000","last_active":"2022-09-29T05:25:38.518267+0000","last_peered":"2022-09-29T05:25:38.518267+0000","last_clean":"2022-09-29T05:25:38.518267+0000","last_became_active":"2022-09-29T05:25:03.460729+0000","last_became_peered":"2022-09-29T05:25:03.460729+0000","last_unstale":"2022-09-29T05:25:38.518267+0000","last_undegraded":"2022-09-29T05:25:38.518267+0000","last_fullsized":"2022-09-29T05:25:38.518267+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.517866+0000","last_change":"2022-09-29T05:25:03.818733+0000","last_active":"2022-09-29T05:25:38.517866+0000","last_peered":"2022-09-29T05:25:38.517866+0000","last_clean":"2022-09-29T05:25:38.517866+0000","last_became_active":"2022-09-29T05:25:03.816983+0000","last_became_peered":"2022-09-29T05:25:03.816983+0000","last_unstale":"2022-09-29T05:25:38.517866+0000","last_undegraded":"2022-09-29T05:25:38.517866+0000","last_fullsized":"2022-09-29T05:25:38.517866+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.16","version":"0'0","reported_seq":16,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.517787+0000","last_change":"2022-09-29T05:25:03.463477+0000","last_active":"2022-09-29T05:25:38.517787+0000","last_peered":"2022-09-29T05:25:38.517787+0000","last_clean":"2022-09-29T05:25:38.517787+0000","last_became_active":"2022-09-29T05:25:03.463051+0000","last_became_peered":"2022-09-29T05:25:03.463051+0000","last_unstale":"2022-09-29T05:25:38.517787+0000","last_undegraded":"2022-09-29T05:25:38.517787+0000","last_fullsized":"2022-09-29T05:25:38.517787+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.517956+0000","last_change":"2022-09-29T05:25:03.822200+0000","last_active":"2022-09-29T05:25:38.517956+0000","last_peered":"2022-09-29T05:25:38.517956+0000","last_clean":"2022-09-29T05:25:38.517956+0000","last_became_active":"2022-09-29T05:25:03.821488+0000","last_became_peered":"2022-09-29T05:25:03.821488+0000","last_unstale":"2022-09-29T05:25:38.517956+0000","last_undegraded":"2022-09-29T05:25:38.517956+0000","last_fullsized":"2022-09-29T05:25:38.517956+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.15","version":"50'3","reported_seq":19,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.142344+0000","last_change":"2022-09-29T05:25:03.457077+0000","last_active":"2022-09-29T05:25:38.142344+0000","last_peered":"2022-09-29T05:25:38.142344+0000","last_clean":"2022-09-29T05:25:38.142344+0000","last_became_active":"2022-09-29T05:25:03.456823+0000","last_became_peered":"2022-09-29T05:25:03.456823+0000","last_unstale":"2022-09-29T05:25:38.142344+0000","last_undegraded":"2022-09-29T05:25:38.142344+0000","last_fullsized":"2022-09-29T05:25:38.142344+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":3,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.136217+0000","last_change":"2022-09-29T05:25:03.822638+0000","last_active":"2022-09-29T05:25:38.136217+0000","last_peered":"2022-09-29T05:25:38.136217+0000","last_clean":"2022-09-29T05:25:38.136217+0000","last_became_active":"2022-09-29T05:25:03.822297+0000","last_became_peered":"2022-09-29T05:25:03.822297+0000","last_unstale":"2022-09-29T05:25:38.136217+0000","last_undegraded":"2022-09-29T05:25:38.136217+0000","last_fullsized":"2022-09-29T05:25:38.136217+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.14","version":"52'9","reported_seq":25,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.517947+0000","last_change":"2022-09-29T05:25:03.461139+0000","last_active":"2022-09-29T05:25:38.517947+0000","last_peered":"2022-09-29T05:25:38.517947+0000","last_clean":"2022-09-29T05:25:38.517947+0000","last_became_active":"2022-09-29T05:25:03.460829+0000","last_became_peered":"2022-09-29T05:25:03.460829+0000","last_unstale":"2022-09-29T05:25:38.517947+0000","last_undegraded":"2022-09-29T05:25:38.517947+0000","last_fullsized":"2022-09-29T05:25:38.517947+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":9,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":2300,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.519257+0000","last_change":"2022-09-29T05:25:03.828186+0000","last_active":"2022-09-29T05:25:38.519257+0000","last_peered":"2022-09-29T05:25:38.519257+0000","last_clean":"2022-09-29T05:25:38.519257+0000","last_became_active":"2022-09-29T05:25:03.826167+0000","last_became_peered":"2022-09-29T05:25:03.826167+0000","last_unstale":"2022-09-29T05:25:38.519257+0000","last_undegraded":"2022-09-29T05:25:38.519257+0000","last_fullsized":"2022-09-29T05:25:38.519257+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.13","version":"50'2","reported_seq":18,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.518229+0000","last_change":"2022-09-29T05:25:03.451661+0000","last_active":"2022-09-29T05:25:38.518229+0000","last_peered":"2022-09-29T05:25:38.518229+0000","last_clean":"2022-09-29T05:25:38.518229+0000","last_became_active":"2022-09-29T05:25:03.451389+0000","last_became_peered":"2022-09-29T05:25:03.451389+0000","last_unstale":"2022-09-29T05:25:38.518229+0000","last_undegraded":"2022-09-29T05:25:38.518229+0000","last_fullsized":"2022-09-29T05:25:38.518229+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.517628+0000","last_change":"2022-09-29T05:25:03.824608+0000","last_active":"2022-09-29T05:25:38.517628+0000","last_peered":"2022-09-29T05:25:38.517628+0000","last_clean":"2022-09-29T05:25:38.517628+0000","last_became_active":"2022-09-29T05:25:03.824335+0000","last_became_peered":"2022-09-29T05:25:03.824335+0000","last_unstale":"2022-09-29T05:25:38.517628+0000","last_undegraded":"2022-09-29T05:25:38.517628+0000","last_fullsized":"2022-09-29T05:25:38.517628+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.12","version":"0'0","reported_seq":16,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.517688+0000","last_change":"2022-09-29T05:25:03.462145+0000","last_active":"2022-09-29T05:25:38.517688+0000","last_peered":"2022-09-29T05:25:38.517688+0000","last_clean":"2022-09-29T05:25:38.517688+0000","last_became_active":"2022-09-29T05:25:03.461864+0000","last_became_peered":"2022-09-29T05:25:03.461864+0000","last_unstale":"2022-09-29T05:25:38.517688+0000","last_undegraded":"2022-09-29T05:25:38.517688+0000","last_fullsized":"2022-09-29T05:25:38.517688+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.519372+0000","last_change":"2022-09-29T05:25:03.828734+0000","last_active":"2022-09-29T05:25:38.519372+0000","last_peered":"2022-09-29T05:25:38.519372+0000","last_clean":"2022-09-29T05:25:38.519372+0000","last_became_active":"2022-09-29T05:25:03.827929+0000","last_became_peered":"2022-09-29T05:25:03.827929+0000","last_unstale":"2022-09-29T05:25:38.519372+0000","last_undegraded":"2022-09-29T05:25:38.519372+0000","last_fullsized":"2022-09-29T05:25:38.519372+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.11","version":"0'0","reported_seq":16,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.518489+0000","last_change":"2022-09-29T05:25:03.454654+0000","last_active":"2022-09-29T05:25:38.518489+0000","last_peered":"2022-09-29T05:25:38.518489+0000","last_clean":"2022-09-29T05:25:38.518489+0000","last_became_active":"2022-09-29T05:25:03.454319+0000","last_became_peered":"2022-09-29T05:25:03.454319+0000","last_unstale":"2022-09-29T05:25:38.518489+0000","last_undegraded":"2022-09-29T05:25:38.518489+0000","last_fullsized":"2022-09-29T05:25:38.518489+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.518057+0000","last_change":"2022-09-29T05:25:03.811425+0000","last_active":"2022-09-29T05:25:38.518057+0000","last_peered":"2022-09-29T05:25:38.518057+0000","last_clean":"2022-09-29T05:25:38.518057+0000","last_became_active":"2022-09-29T05:25:03.811120+0000","last_became_peered":"2022-09-29T05:25:03.811120+0000","last_unstale":"2022-09-29T05:25:38.518057+0000","last_undegraded":"2022-09-29T05:25:38.518057+0000","last_fullsized":"2022-09-29T05:25:38.518057+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.10","version":"50'8","reported_seq":24,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.140816+0000","last_change":"2022-09-29T05:25:03.449137+0000","last_active":"2022-09-29T05:25:38.140816+0000","last_peered":"2022-09-29T05:25:38.140816+0000","last_clean":"2022-09-29T05:25:38.140816+0000","last_became_active":"2022-09-29T05:25:03.448728+0000","last_became_peered":"2022-09-29T05:25:03.448728+0000","last_unstale":"2022-09-29T05:25:38.140816+0000","last_undegraded":"2022-09-29T05:25:38.140816+0000","last_fullsized":"2022-09-29T05:25:38.140816+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":8,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.517782+0000","last_change":"2022-09-29T05:25:03.826279+0000","last_active":"2022-09-29T05:25:38.517782+0000","last_peered":"2022-09-29T05:25:38.517782+0000","last_clean":"2022-09-29T05:25:38.517782+0000","last_became_active":"2022-09-29T05:25:03.825969+0000","last_became_peered":"2022-09-29T05:25:03.825969+0000","last_unstale":"2022-09-29T05:25:38.517782+0000","last_undegraded":"2022-09-29T05:25:38.517782+0000","last_fullsized":"2022-09-29T05:25:38.517782+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.f","version":"50'1","reported_seq":17,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.136419+0000","last_change":"2022-09-29T05:25:03.452230+0000","last_active":"2022-09-29T05:25:38.136419+0000","last_peered":"2022-09-29T05:25:38.136419+0000","last_clean":"2022-09-29T05:25:38.136419+0000","last_became_active":"2022-09-29T05:25:03.451747+0000","last_became_peered":"2022-09-29T05:25:03.451747+0000","last_unstale":"2022-09-29T05:25:38.136419+0000","last_undegraded":"2022-09-29T05:25:38.136419+0000","last_fullsized":"2022-09-29T05:25:38.136419+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.518349+0000","last_change":"2022-09-29T05:25:03.827453+0000","last_active":"2022-09-29T05:25:38.518349+0000","last_peered":"2022-09-29T05:25:38.518349+0000","last_clean":"2022-09-29T05:25:38.518349+0000","last_became_active":"2022-09-29T05:25:03.827151+0000","last_became_peered":"2022-09-29T05:25:03.827151+0000","last_unstale":"2022-09-29T05:25:38.518349+0000","last_undegraded":"2022-09-29T05:25:38.518349+0000","last_fullsized":"2022-09-29T05:25:38.518349+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.141816+0000","last_change":"2022-09-29T05:25:37.268300+0000","last_active":"2022-09-29T05:25:38.141816+0000","last_peered":"2022-09-29T05:25:38.141816+0000","last_clean":"2022-09-29T05:25:38.141816+0000","last_became_active":"2022-09-29T05:25:37.267961+0000","last_became_peered":"2022-09-29T05:25:37.267961+0000","last_unstale":"2022-09-29T05:25:38.141816+0000","last_undegraded":"2022-09-29T05:25:38.141816+0000","last_fullsized":"2022-09-29T05:25:38.141816+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:36.130315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:36.130315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:36.130315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":16,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.141681+0000","last_change":"2022-09-29T05:25:03.454507+0000","last_active":"2022-09-29T05:25:38.141681+0000","last_peered":"2022-09-29T05:25:38.141681+0000","last_clean":"2022-09-29T05:25:38.141681+0000","last_became_active":"2022-09-29T05:25:03.454195+0000","last_became_peered":"2022-09-29T05:25:03.454195+0000","last_unstale":"2022-09-29T05:25:38.141681+0000","last_undegraded":"2022-09-29T05:25:38.141681+0000","last_fullsized":"2022-09-29T05:25:38.141681+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.518292+0000","last_change":"2022-09-29T05:25:03.819862+0000","last_active":"2022-09-29T05:25:38.518292+0000","last_peered":"2022-09-29T05:25:38.518292+0000","last_clean":"2022-09-29T05:25:38.518292+0000","last_became_active":"2022-09-29T05:25:03.818117+0000","last_became_peered":"2022-09-29T05:25:03.818117+0000","last_unstale":"2022-09-29T05:25:38.518292+0000","last_undegraded":"2022-09-29T05:25:38.518292+0000","last_fullsized":"2022-09-29T05:25:38.518292+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.141770+0000","last_change":"2022-09-29T05:25:37.140714+0000","last_active":"2022-09-29T05:25:38.141770+0000","last_peered":"2022-09-29T05:25:38.141770+0000","last_clean":"2022-09-29T05:25:38.141770+0000","last_became_active":"2022-09-29T05:25:37.140393+0000","last_became_peered":"2022-09-29T05:25:37.140393+0000","last_unstale":"2022-09-29T05:25:38.141770+0000","last_undegraded":"2022-09-29T05:25:38.141770+0000","last_fullsized":"2022-09-29T05:25:38.141770+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:36.130315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:36.130315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:36.130315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":16,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.517596+0000","last_change":"2022-09-29T05:25:03.459646+0000","last_active":"2022-09-29T05:25:38.517596+0000","last_peered":"2022-09-29T05:25:38.517596+0000","last_clean":"2022-09-29T05:25:38.517596+0000","last_became_active":"2022-09-29T05:25:03.459320+0000","last_became_peered":"2022-09-29T05:25:03.459320+0000","last_unstale":"2022-09-29T05:25:38.517596+0000","last_undegraded":"2022-09-29T05:25:38.517596+0000","last_fullsized":"2022-09-29T05:25:38.517596+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.136122+0000","last_change":"2022-09-29T05:25:03.810358+0000","last_active":"2022-09-29T05:25:38.136122+0000","last_peered":"2022-09-29T05:25:38.136122+0000","last_clean":"2022-09-29T05:25:38.136122+0000","last_became_active":"2022-09-29T05:25:03.809959+0000","last_became_peered":"2022-09-29T05:25:03.809959+0000","last_unstale":"2022-09-29T05:25:38.136122+0000","last_undegraded":"2022-09-29T05:25:38.136122+0000","last_fullsized":"2022-09-29T05:25:38.136122+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.141097+0000","last_change":"2022-09-29T05:25:37.268374+0000","last_active":"2022-09-29T05:25:38.141097+0000","last_peered":"2022-09-29T05:25:38.141097+0000","last_clean":"2022-09-29T05:25:38.141097+0000","last_became_active":"2022-09-29T05:25:37.267928+0000","last_became_peered":"2022-09-29T05:25:37.267928+0000","last_unstale":"2022-09-29T05:25:38.141097+0000","last_undegraded":"2022-09-29T05:25:38.141097+0000","last_fullsized":"2022-09-29T05:25:38.141097+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:36.130315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:36.130315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:36.130315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.141135+0000","last_change":"2022-09-29T05:25:03.812390+0000","last_active":"2022-09-29T05:25:38.141135+0000","last_peered":"2022-09-29T05:25:38.141135+0000","last_clean":"2022-09-29T05:25:38.141135+0000","last_became_active":"2022-09-29T05:25:03.811925+0000","last_became_peered":"2022-09-29T05:25:03.811925+0000","last_unstale":"2022-09-29T05:25:38.141135+0000","last_undegraded":"2022-09-29T05:25:38.141135+0000","last_fullsized":"2022-09-29T05:25:38.141135+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":16,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.140903+0000","last_change":"2022-09-29T05:25:03.449156+0000","last_active":"2022-09-29T05:25:38.140903+0000","last_peered":"2022-09-29T05:25:38.140903+0000","last_clean":"2022-09-29T05:25:38.140903+0000","last_became_active":"2022-09-29T05:25:03.448885+0000","last_became_peered":"2022-09-29T05:25:03.448885+0000","last_unstale":"2022-09-29T05:25:38.140903+0000","last_undegraded":"2022-09-29T05:25:38.140903+0000","last_fullsized":"2022-09-29T05:25:38.140903+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.518158+0000","last_change":"2022-09-29T05:25:37.269299+0000","last_active":"2022-09-29T05:25:38.518158+0000","last_peered":"2022-09-29T05:25:38.518158+0000","last_clean":"2022-09-29T05:25:38.518158+0000","last_became_active":"2022-09-29T05:25:37.268903+0000","last_became_peered":"2022-09-29T05:25:37.268903+0000","last_unstale":"2022-09-29T05:25:38.518158+0000","last_undegraded":"2022-09-29T05:25:38.518158+0000","last_fullsized":"2022-09-29T05:25:38.518158+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:36.130315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:36.130315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:36.130315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.518001+0000","last_change":"2022-09-29T05:25:03.813170+0000","last_active":"2022-09-29T05:25:38.518001+0000","last_peered":"2022-09-29T05:25:38.518001+0000","last_clean":"2022-09-29T05:25:38.518001+0000","last_became_active":"2022-09-29T05:25:03.812861+0000","last_became_peered":"2022-09-29T05:25:03.812861+0000","last_unstale":"2022-09-29T05:25:38.518001+0000","last_undegraded":"2022-09-29T05:25:38.518001+0000","last_fullsized":"2022-09-29T05:25:38.518001+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":16,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.517396+0000","last_change":"2022-09-29T05:25:03.462271+0000","last_active":"2022-09-29T05:25:38.517396+0000","last_peered":"2022-09-29T05:25:38.517396+0000","last_clean":"2022-09-29T05:25:38.517396+0000","last_became_active":"2022-09-29T05:25:03.461959+0000","last_became_peered":"2022-09-29T05:25:03.461959+0000","last_unstale":"2022-09-29T05:25:38.517396+0000","last_undegraded":"2022-09-29T05:25:38.517396+0000","last_fullsized":"2022-09-29T05:25:38.517396+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":19,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.519716+0000","last_change":"2022-09-29T05:24:48.628243+0000","last_active":"2022-09-29T05:25:38.519716+0000","last_peered":"2022-09-29T05:25:38.519716+0000","last_clean":"2022-09-29T05:25:38.519716+0000","last_became_active":"2022-09-29T05:24:48.627801+0000","last_became_peered":"2022-09-29T05:24:48.627801+0000","last_unstale":"2022-09-29T05:25:38.519716+0000","last_undegraded":"2022-09-29T05:25:38.519716+0000","last_fullsized":"2022-09-29T05:25:38.519716+0000","mapping_epoch":46,"log_start":"0'0","ondisk_log_start":"0'0","created":21,"last_epoch_clean":47,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:23:35.544281+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:23:35.544281+0000","last_clean_scrub_stamp":"2022-09-29T05:23:35.544281+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.518259+0000","last_change":"2022-09-29T05:25:37.141541+0000","last_active":"2022-09-29T05:25:38.518259+0000","last_peered":"2022-09-29T05:25:38.518259+0000","last_clean":"2022-09-29T05:25:38.518259+0000","last_became_active":"2022-09-29T05:25:37.141199+0000","last_became_peered":"2022-09-29T05:25:37.141199+0000","last_unstale":"2022-09-29T05:25:38.518259+0000","last_undegraded":"2022-09-29T05:25:38.518259+0000","last_fullsized":"2022-09-29T05:25:38.518259+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:36.130315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:36.130315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:36.130315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.517516+0000","last_change":"2022-09-29T05:25:03.812201+0000","last_active":"2022-09-29T05:25:38.517516+0000","last_peered":"2022-09-29T05:25:38.517516+0000","last_clean":"2022-09-29T05:25:38.517516+0000","last_became_active":"2022-09-29T05:25:03.811903+0000","last_became_peered":"2022-09-29T05:25:03.811903+0000","last_unstale":"2022-09-29T05:25:38.517516+0000","last_undegraded":"2022-09-29T05:25:38.517516+0000","last_fullsized":"2022-09-29T05:25:38.517516+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":16,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.519580+0000","last_change":"2022-09-29T05:25:03.451876+0000","last_active":"2022-09-29T05:25:38.519580+0000","last_peered":"2022-09-29T05:25:38.519580+0000","last_clean":"2022-09-29T05:25:38.519580+0000","last_became_active":"2022-09-29T05:25:03.451371+0000","last_became_peered":"2022-09-29T05:25:03.451371+0000","last_unstale":"2022-09-29T05:25:38.519580+0000","last_undegraded":"2022-09-29T05:25:38.519580+0000","last_fullsized":"2022-09-29T05:25:38.519580+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.517944+0000","last_change":"2022-09-29T05:25:37.269163+0000","last_active":"2022-09-29T05:25:38.517944+0000","last_peered":"2022-09-29T05:25:38.517944+0000","last_clean":"2022-09-29T05:25:38.517944+0000","last_became_active":"2022-09-29T05:25:37.268871+0000","last_became_peered":"2022-09-29T05:25:37.268871+0000","last_unstale":"2022-09-29T05:25:38.517944+0000","last_undegraded":"2022-09-29T05:25:38.517944+0000","last_fullsized":"2022-09-29T05:25:38.517944+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:36.130315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:36.130315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:36.130315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":16,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.519358+0000","last_change":"2022-09-29T05:25:03.453060+0000","last_active":"2022-09-29T05:25:38.519358+0000","last_peered":"2022-09-29T05:25:38.519358+0000","last_clean":"2022-09-29T05:25:38.519358+0000","last_became_active":"2022-09-29T05:25:03.452758+0000","last_became_peered":"2022-09-29T05:25:03.452758+0000","last_unstale":"2022-09-29T05:25:38.519358+0000","last_undegraded":"2022-09-29T05:25:38.519358+0000","last_fullsized":"2022-09-29T05:25:38.519358+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.141842+0000","last_change":"2022-09-29T05:25:03.803195+0000","last_active":"2022-09-29T05:25:38.141842+0000","last_peered":"2022-09-29T05:25:38.141842+0000","last_clean":"2022-09-29T05:25:38.141842+0000","last_became_active":"2022-09-29T05:25:03.802909+0000","last_became_peered":"2022-09-29T05:25:03.802909+0000","last_unstale":"2022-09-29T05:25:38.141842+0000","last_undegraded":"2022-09-29T05:25:38.141842+0000","last_fullsized":"2022-09-29T05:25:38.141842+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.517773+0000","last_change":"2022-09-29T05:25:37.268505+0000","last_active":"2022-09-29T05:25:38.517773+0000","last_peered":"2022-09-29T05:25:38.517773+0000","last_clean":"2022-09-29T05:25:38.517773+0000","last_became_active":"2022-09-29T05:25:37.268095+0000","last_became_peered":"2022-09-29T05:25:37.268095+0000","last_unstale":"2022-09-29T05:25:38.517773+0000","last_undegraded":"2022-09-29T05:25:38.517773+0000","last_fullsized":"2022-09-29T05:25:38.517773+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:36.130315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:36.130315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:36.130315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"51'1","reported_seq":17,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.141330+0000","last_change":"2022-09-29T05:25:03.456735+0000","last_active":"2022-09-29T05:25:38.141330+0000","last_peered":"2022-09-29T05:25:38.141330+0000","last_clean":"2022-09-29T05:25:38.141330+0000","last_became_active":"2022-09-29T05:25:03.456427+0000","last_became_peered":"2022-09-29T05:25:03.456427+0000","last_unstale":"2022-09-29T05:25:38.141330+0000","last_undegraded":"2022-09-29T05:25:38.141330+0000","last_fullsized":"2022-09-29T05:25:38.141330+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.517746+0000","last_change":"2022-09-29T05:25:03.819725+0000","last_active":"2022-09-29T05:25:38.517746+0000","last_peered":"2022-09-29T05:25:38.517746+0000","last_clean":"2022-09-29T05:25:38.517746+0000","last_became_active":"2022-09-29T05:25:03.819455+0000","last_became_peered":"2022-09-29T05:25:03.819455+0000","last_unstale":"2022-09-29T05:25:38.517746+0000","last_undegraded":"2022-09-29T05:25:38.517746+0000","last_fullsized":"2022-09-29T05:25:38.517746+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.135795+0000","last_change":"2022-09-29T05:25:37.269825+0000","last_active":"2022-09-29T05:25:38.135795+0000","last_peered":"2022-09-29T05:25:38.135795+0000","last_clean":"2022-09-29T05:25:38.135795+0000","last_became_active":"2022-09-29T05:25:37.269508+0000","last_became_peered":"2022-09-29T05:25:37.269508+0000","last_unstale":"2022-09-29T05:25:38.135795+0000","last_undegraded":"2022-09-29T05:25:38.135795+0000","last_fullsized":"2022-09-29T05:25:38.135795+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:36.130315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:36.130315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:36.130315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"50'2","reported_seq":18,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.518186+0000","last_change":"2022-09-29T05:25:03.462138+0000","last_active":"2022-09-29T05:25:38.518186+0000","last_peered":"2022-09-29T05:25:38.518186+0000","last_clean":"2022-09-29T05:25:38.518186+0000","last_became_active":"2022-09-29T05:25:03.461809+0000","last_became_peered":"2022-09-29T05:25:03.461809+0000","last_unstale":"2022-09-29T05:25:38.518186+0000","last_undegraded":"2022-09-29T05:25:38.518186+0000","last_fullsized":"2022-09-29T05:25:38.518186+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.518353+0000","last_change":"2022-09-29T05:25:03.813902+0000","last_active":"2022-09-29T05:25:38.518353+0000","last_peered":"2022-09-29T05:25:38.518353+0000","last_clean":"2022-09-29T05:25:38.518353+0000","last_became_active":"2022-09-29T05:25:03.813617+0000","last_became_peered":"2022-09-29T05:25:03.813617+0000","last_unstale":"2022-09-29T05:25:38.518353+0000","last_undegraded":"2022-09-29T05:25:38.518353+0000","last_fullsized":"2022-09-29T05:25:38.518353+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.8","version":"0'0","reported_seq":16,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.519792+0000","last_change":"2022-09-29T05:25:03.451677+0000","last_active":"2022-09-29T05:25:38.519792+0000","last_peered":"2022-09-29T05:25:38.519792+0000","last_clean":"2022-09-29T05:25:38.519792+0000","last_became_active":"2022-09-29T05:25:03.450961+0000","last_became_peered":"2022-09-29T05:25:03.450961+0000","last_unstale":"2022-09-29T05:25:38.519792+0000","last_undegraded":"2022-09-29T05:25:38.519792+0000","last_fullsized":"2022-09-29T05:25:38.519792+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.135920+0000","last_change":"2022-09-29T05:25:03.822108+0000","last_active":"2022-09-29T05:25:38.135920+0000","last_peered":"2022-09-29T05:25:38.135920+0000","last_clean":"2022-09-29T05:25:38.135920+0000","last_became_active":"2022-09-29T05:25:03.821784+0000","last_became_peered":"2022-09-29T05:25:03.821784+0000","last_unstale":"2022-09-29T05:25:38.135920+0000","last_undegraded":"2022-09-29T05:25:38.135920+0000","last_fullsized":"2022-09-29T05:25:38.135920+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.9","version":"0'0","reported_seq":16,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.141479+0000","last_change":"2022-09-29T05:25:03.452512+0000","last_active":"2022-09-29T05:25:38.141479+0000","last_peered":"2022-09-29T05:25:38.141479+0000","last_clean":"2022-09-29T05:25:38.141479+0000","last_became_active":"2022-09-29T05:25:03.452222+0000","last_became_peered":"2022-09-29T05:25:03.452222+0000","last_unstale":"2022-09-29T05:25:38.141479+0000","last_undegraded":"2022-09-29T05:25:38.141479+0000","last_fullsized":"2022-09-29T05:25:38.141479+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.517411+0000","last_change":"2022-09-29T05:25:03.819591+0000","last_active":"2022-09-29T05:25:38.517411+0000","last_peered":"2022-09-29T05:25:38.517411+0000","last_clean":"2022-09-29T05:25:38.517411+0000","last_became_active":"2022-09-29T05:25:03.819231+0000","last_became_peered":"2022-09-29T05:25:03.819231+0000","last_unstale":"2022-09-29T05:25:38.517411+0000","last_undegraded":"2022-09-29T05:25:38.517411+0000","last_fullsized":"2022-09-29T05:25:38.517411+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.a","version":"0'0","reported_seq":16,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.141930+0000","last_change":"2022-09-29T05:25:03.451868+0000","last_active":"2022-09-29T05:25:38.141930+0000","last_peered":"2022-09-29T05:25:38.141930+0000","last_clean":"2022-09-29T05:25:38.141930+0000","last_became_active":"2022-09-29T05:25:03.451576+0000","last_became_peered":"2022-09-29T05:25:03.451576+0000","last_unstale":"2022-09-29T05:25:38.141930+0000","last_undegraded":"2022-09-29T05:25:38.141930+0000","last_fullsized":"2022-09-29T05:25:38.141930+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.517482+0000","last_change":"2022-09-29T05:25:03.808836+0000","last_active":"2022-09-29T05:25:38.517482+0000","last_peered":"2022-09-29T05:25:38.517482+0000","last_clean":"2022-09-29T05:25:38.517482+0000","last_became_active":"2022-09-29T05:25:03.808557+0000","last_became_peered":"2022-09-29T05:25:03.808557+0000","last_unstale":"2022-09-29T05:25:38.517482+0000","last_undegraded":"2022-09-29T05:25:38.517482+0000","last_fullsized":"2022-09-29T05:25:38.517482+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.b","version":"50'2","reported_seq":18,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.519288+0000","last_change":"2022-09-29T05:25:03.460369+0000","last_active":"2022-09-29T05:25:38.519288+0000","last_peered":"2022-09-29T05:25:38.519288+0000","last_clean":"2022-09-29T05:25:38.519288+0000","last_became_active":"2022-09-29T05:25:03.459796+0000","last_became_peered":"2022-09-29T05:25:03.459796+0000","last_unstale":"2022-09-29T05:25:38.519288+0000","last_undegraded":"2022-09-29T05:25:38.519288+0000","last_fullsized":"2022-09-29T05:25:38.519288+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.517611+0000","last_change":"2022-09-29T05:25:03.818102+0000","last_active":"2022-09-29T05:25:38.517611+0000","last_peered":"2022-09-29T05:25:38.517611+0000","last_clean":"2022-09-29T05:25:38.517611+0000","last_became_active":"2022-09-29T05:25:03.817772+0000","last_became_peered":"2022-09-29T05:25:03.817772+0000","last_unstale":"2022-09-29T05:25:38.517611+0000","last_undegraded":"2022-09-29T05:25:38.517611+0000","last_fullsized":"2022-09-29T05:25:38.517611+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.c","version":"0'0","reported_seq":16,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.141464+0000","last_change":"2022-09-29T05:25:03.460119+0000","last_active":"2022-09-29T05:25:38.141464+0000","last_peered":"2022-09-29T05:25:38.141464+0000","last_clean":"2022-09-29T05:25:38.141464+0000","last_became_active":"2022-09-29T05:25:03.459795+0000","last_became_peered":"2022-09-29T05:25:03.459795+0000","last_unstale":"2022-09-29T05:25:38.141464+0000","last_undegraded":"2022-09-29T05:25:38.141464+0000","last_fullsized":"2022-09-29T05:25:38.141464+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.519315+0000","last_change":"2022-09-29T05:25:03.828523+0000","last_active":"2022-09-29T05:25:38.519315+0000","last_peered":"2022-09-29T05:25:38.519315+0000","last_clean":"2022-09-29T05:25:38.519315+0000","last_became_active":"2022-09-29T05:25:03.827990+0000","last_became_peered":"2022-09-29T05:25:03.827990+0000","last_unstale":"2022-09-29T05:25:38.519315+0000","last_undegraded":"2022-09-29T05:25:38.519315+0000","last_fullsized":"2022-09-29T05:25:38.519315+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.d","version":"50'1","reported_seq":17,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.142256+0000","last_change":"2022-09-29T05:25:03.452648+0000","last_active":"2022-09-29T05:25:38.142256+0000","last_peered":"2022-09-29T05:25:38.142256+0000","last_clean":"2022-09-29T05:25:38.142256+0000","last_became_active":"2022-09-29T05:25:03.452242+0000","last_became_peered":"2022-09-29T05:25:03.452242+0000","last_unstale":"2022-09-29T05:25:38.142256+0000","last_undegraded":"2022-09-29T05:25:38.142256+0000","last_fullsized":"2022-09-29T05:25:38.142256+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.518795+0000","last_change":"2022-09-29T05:25:03.809640+0000","last_active":"2022-09-29T05:25:38.518795+0000","last_peered":"2022-09-29T05:25:38.518795+0000","last_clean":"2022-09-29T05:25:38.518795+0000","last_became_active":"2022-09-29T05:25:03.809298+0000","last_became_peered":"2022-09-29T05:25:03.809298+0000","last_unstale":"2022-09-29T05:25:38.518795+0000","last_undegraded":"2022-09-29T05:25:38.518795+0000","last_fullsized":"2022-09-29T05:25:38.518795+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.e","version":"50'2","reported_seq":18,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.141902+0000","last_change":"2022-09-29T05:25:03.450361+0000","last_active":"2022-09-29T05:25:38.141902+0000","last_peered":"2022-09-29T05:25:38.141902+0000","last_clean":"2022-09-29T05:25:38.141902+0000","last_became_active":"2022-09-29T05:25:03.450020+0000","last_became_peered":"2022-09-29T05:25:03.450020+0000","last_unstale":"2022-09-29T05:25:38.141902+0000","last_undegraded":"2022-09-29T05:25:38.141902+0000","last_fullsized":"2022-09-29T05:25:38.141902+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:02.423315+0000","last_clean_scrub_stamp":"2022-09-29T05:25:02.423315+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.518792+0000","last_change":"2022-09-29T05:25:03.829664+0000","last_active":"2022-09-29T05:25:38.518792+0000","last_peered":"2022-09-29T05:25:38.518792+0000","last_clean":"2022-09-29T05:25:38.518792+0000","last_became_active":"2022-09-29T05:25:03.828931+0000","last_became_peered":"2022-09-29T05:25:03.828931+0000","last_unstale":"2022-09-29T05:25:38.518792+0000","last_undegraded":"2022-09-29T05:25:38.518792+0000","last_fullsized":"2022-09-29T05:25:38.518792+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.20","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.517921+0000","last_change":"2022-09-29T05:25:03.811486+0000","last_active":"2022-09-29T05:25:38.517921+0000","last_peered":"2022-09-29T05:25:38.517921+0000","last_clean":"2022-09-29T05:25:38.517921+0000","last_became_active":"2022-09-29T05:25:03.811222+0000","last_became_peered":"2022-09-29T05:25:03.811222+0000","last_unstale":"2022-09-29T05:25:38.517921+0000","last_undegraded":"2022-09-29T05:25:38.517921+0000","last_fullsized":"2022-09-29T05:25:38.517921+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.21","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.140986+0000","last_change":"2022-09-29T05:25:03.809062+0000","last_active":"2022-09-29T05:25:38.140986+0000","last_peered":"2022-09-29T05:25:38.140986+0000","last_clean":"2022-09-29T05:25:38.140986+0000","last_became_active":"2022-09-29T05:25:03.808752+0000","last_became_peered":"2022-09-29T05:25:03.808752+0000","last_unstale":"2022-09-29T05:25:38.140986+0000","last_undegraded":"2022-09-29T05:25:38.140986+0000","last_fullsized":"2022-09-29T05:25:38.140986+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.22","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.517769+0000","last_change":"2022-09-29T05:25:03.817645+0000","last_active":"2022-09-29T05:25:38.517769+0000","last_peered":"2022-09-29T05:25:38.517769+0000","last_clean":"2022-09-29T05:25:38.517769+0000","last_became_active":"2022-09-29T05:25:03.816740+0000","last_became_peered":"2022-09-29T05:25:03.816740+0000","last_unstale":"2022-09-29T05:25:38.517769+0000","last_undegraded":"2022-09-29T05:25:38.517769+0000","last_fullsized":"2022-09-29T05:25:38.517769+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.23","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.518117+0000","last_change":"2022-09-29T05:25:03.810436+0000","last_active":"2022-09-29T05:25:38.518117+0000","last_peered":"2022-09-29T05:25:38.518117+0000","last_clean":"2022-09-29T05:25:38.518117+0000","last_became_active":"2022-09-29T05:25:03.810178+0000","last_became_peered":"2022-09-29T05:25:03.810178+0000","last_unstale":"2022-09-29T05:25:38.518117+0000","last_undegraded":"2022-09-29T05:25:38.518117+0000","last_fullsized":"2022-09-29T05:25:38.518117+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.24","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.517982+0000","last_change":"2022-09-29T05:25:03.809114+0000","last_active":"2022-09-29T05:25:38.517982+0000","last_peered":"2022-09-29T05:25:38.517982+0000","last_clean":"2022-09-29T05:25:38.517982+0000","last_became_active":"2022-09-29T05:25:03.808686+0000","last_became_peered":"2022-09-29T05:25:03.808686+0000","last_unstale":"2022-09-29T05:25:38.517982+0000","last_undegraded":"2022-09-29T05:25:38.517982+0000","last_fullsized":"2022-09-29T05:25:38.517982+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.25","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.142167+0000","last_change":"2022-09-29T05:25:03.817066+0000","last_active":"2022-09-29T05:25:38.142167+0000","last_peered":"2022-09-29T05:25:38.142167+0000","last_clean":"2022-09-29T05:25:38.142167+0000","last_became_active":"2022-09-29T05:25:03.816747+0000","last_became_peered":"2022-09-29T05:25:03.816747+0000","last_unstale":"2022-09-29T05:25:38.142167+0000","last_undegraded":"2022-09-29T05:25:38.142167+0000","last_fullsized":"2022-09-29T05:25:38.142167+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.26","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.517712+0000","last_change":"2022-09-29T05:25:03.823457+0000","last_active":"2022-09-29T05:25:38.517712+0000","last_peered":"2022-09-29T05:25:38.517712+0000","last_clean":"2022-09-29T05:25:38.517712+0000","last_became_active":"2022-09-29T05:25:03.823149+0000","last_became_peered":"2022-09-29T05:25:03.823149+0000","last_unstale":"2022-09-29T05:25:38.517712+0000","last_undegraded":"2022-09-29T05:25:38.517712+0000","last_fullsized":"2022-09-29T05:25:38.517712+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,4],"acting":[6,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.27","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.517660+0000","last_change":"2022-09-29T05:25:03.808766+0000","last_active":"2022-09-29T05:25:38.517660+0000","last_peered":"2022-09-29T05:25:38.517660+0000","last_clean":"2022-09-29T05:25:38.517660+0000","last_became_active":"2022-09-29T05:25:03.808474+0000","last_became_peered":"2022-09-29T05:25:03.808474+0000","last_unstale":"2022-09-29T05:25:38.517660+0000","last_undegraded":"2022-09-29T05:25:38.517660+0000","last_fullsized":"2022-09-29T05:25:38.517660+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.28","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.517490+0000","last_change":"2022-09-29T05:25:03.815620+0000","last_active":"2022-09-29T05:25:38.517490+0000","last_peered":"2022-09-29T05:25:38.517490+0000","last_clean":"2022-09-29T05:25:38.517490+0000","last_became_active":"2022-09-29T05:25:03.814336+0000","last_became_peered":"2022-09-29T05:25:03.814336+0000","last_unstale":"2022-09-29T05:25:38.517490+0000","last_undegraded":"2022-09-29T05:25:38.517490+0000","last_fullsized":"2022-09-29T05:25:38.517490+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.29","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.518222+0000","last_change":"2022-09-29T05:25:03.808727+0000","last_active":"2022-09-29T05:25:38.518222+0000","last_peered":"2022-09-29T05:25:38.518222+0000","last_clean":"2022-09-29T05:25:38.518222+0000","last_became_active":"2022-09-29T05:25:03.808440+0000","last_became_peered":"2022-09-29T05:25:03.808440+0000","last_unstale":"2022-09-29T05:25:38.518222+0000","last_undegraded":"2022-09-29T05:25:38.518222+0000","last_fullsized":"2022-09-29T05:25:38.518222+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.2a","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.517848+0000","last_change":"2022-09-29T05:25:03.819098+0000","last_active":"2022-09-29T05:25:38.517848+0000","last_peered":"2022-09-29T05:25:38.517848+0000","last_clean":"2022-09-29T05:25:38.517848+0000","last_became_active":"2022-09-29T05:25:03.818690+0000","last_became_peered":"2022-09-29T05:25:03.818690+0000","last_unstale":"2022-09-29T05:25:38.517848+0000","last_undegraded":"2022-09-29T05:25:38.517848+0000","last_fullsized":"2022-09-29T05:25:38.517848+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2b","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.518130+0000","last_change":"2022-09-29T05:25:03.824844+0000","last_active":"2022-09-29T05:25:38.518130+0000","last_peered":"2022-09-29T05:25:38.518130+0000","last_clean":"2022-09-29T05:25:38.518130+0000","last_became_active":"2022-09-29T05:25:03.824457+0000","last_became_peered":"2022-09-29T05:25:03.824457+0000","last_unstale":"2022-09-29T05:25:38.518130+0000","last_undegraded":"2022-09-29T05:25:38.518130+0000","last_fullsized":"2022-09-29T05:25:38.518130+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2c","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-09-29T05:25:38.518029+0000","last_change":"2022-09-29T05:25:03.812644+0000","last_active":"2022-09-29T05:25:38.518029+0000","last_peered":"2022-09-29T05:25:38.518029+0000","last_clean":"2022-09-29T05:25:38.518029+0000","last_became_active":"2022-09-29T05:25:03.812339+0000","last_became_peered":"2022-09-29T05:25:03.812339+0000","last_unstale":"2022-09-29T05:25:38.518029+0000","last_undegraded":"2022-09-29T05:25:38.518029+0000","last_fullsized":"2022-09-29T05:25:38.518029+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":49,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-29T05:25:03.430113+0000","last_clean_scrub_stamp":"2022-09-29T05:25:03.430113+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":192,"acting":192,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":32,"stat_sum":{"num_bytes":3070,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":48,"num_write_kb":16,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":9210,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":42,"ondisk_log_size":42,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":45,"seq":193273528341,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5612,"kb_used_data":548,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739540,"statfs":{"total":95995035648,"available":95989288960,"internally_reserved":0,"allocated":561152,"data_stored":237178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Sep 29 05:25:46 2022","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":1.081,"5min":1.081,"15min":1.081},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.47899999999999998}]},{"osd":1,"last update":"Thu Sep 29 05:25:46 2022","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.38200000000000001},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":1.0569999999999999,"5min":1.0569999999999999,"15min":1.0569999999999999},"last":0.56599999999999995}]},{"osd":2,"last update":"Thu Sep 29 05:25:46 2022","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":1.028,"5min":1.028,"15min":1.028},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.95599999999999996,"5min":0.95599999999999996,"15min":0.95599999999999996},"last":0.51100000000000001}]},{"osd":3,"last update":"Thu Sep 29 05:25:46 2022","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.88700000000000001,"5min":0.88700000000000001,"15min":0.88700000000000001},"last":0.39900000000000002},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.84199999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.53900000000000003}]},{"osd":4,"last update":"Thu Sep 29 05:25:46 2022","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":1.103,"5min":1.103,"15min":1.103},"last":0.35599999999999998},{"interface":"front","average":{"1min":0.49399999999999999,"5min":0.49399999999999999,"15min":0.49399999999999999},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.96999999999999997,"5min":0.96999999999999997,"15min":0.96999999999999997},"last":0.436}]},{"osd":5,"last update":"Thu Sep 29 05:25:46 2022","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":1.1240000000000001,"5min":1.1240000000000001,"15min":1.1240000000000001},"last":0.52500000000000002}]},{"osd":6,"last update":"Thu Sep 29 05:25:46 2022","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.94199999999999995,"5min":0.94199999999999995,"15min":0.94199999999999995},"last":0.5},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":1.143,"5min":1.143,"15min":1.143},"last":0.58999999999999997}]}]},{"osd":6,"up_from":40,"seq":171798691863,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5616,"kb_used_data":552,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739536,"statfs":{"total":95995035648,"available":95989284864,"internally_reserved":0,"allocated":565248,"data_stored":238902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Sep 29 05:25:30 2022","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":2.0489999999999999,"5min":2.0489999999999999,"15min":2.0489999999999999},"last":0.64400000000000002},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":1.466,"5min":1.466,"15min":1.466},"last":0.60099999999999998}]},{"osd":1,"last update":"Thu Sep 29 05:25:30 2022","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":2.0670000000000002,"5min":2.0670000000000002,"15min":2.0670000000000002},"last":0.66200000000000003},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":2.02,"5min":2.02,"15min":2.02},"last":0.68700000000000006}]},{"osd":2,"last update":"Thu Sep 29 05:25:30 2022","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":2.081,"5min":2.081,"15min":2.081},"last":0.61399999999999999},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":1.4910000000000001,"5min":1.4910000000000001,"15min":1.4910000000000001},"last":0.67100000000000004}]},{"osd":3,"last update":"Thu Sep 29 05:25:30 2022","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":2.1059999999999999,"5min":2.1059999999999999,"15min":2.1059999999999999},"last":0.70299999999999996},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":1.9890000000000001,"5min":1.9890000000000001,"15min":1.9890000000000001},"last":0.56499999999999995}]},{"osd":4,"last update":"Thu Sep 29 05:25:30 2022","interfaces":[{"interface":"back","average":{"1min":0.49399999999999999,"5min":0.49399999999999999,"15min":0.49399999999999999},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":1.3580000000000001,"5min":1.3580000000000001,"15min":1.3580000000000001},"last":0.47799999999999998},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":2.1360000000000001,"5min":2.1360000000000001,"15min":2.1360000000000001},"last":0.496}]},{"osd":5,"last update":"Thu Sep 29 05:25:30 2022","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":1.405,"5min":1.405,"15min":1.405},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.47799999999999998,"5min":0.47799999999999998,"15min":0.47799999999999998},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.621}]},{"osd":7,"last update":"Thu Sep 29 05:25:47 2022","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.58499999999999996},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":1.4379999999999999,"5min":1.4379999999999999,"15min":1.4379999999999999},"last":0.63400000000000001}]}]},{"osd":1,"up_from":13,"seq":55834574888,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5872,"kb_used_data":552,"kb_used_omap":0,"kb_used_meta":5312,"kb_avail":93739280,"statfs":{"total":95995035648,"available":95989022720,"internally_reserved":0,"allocated":565248,"data_stored":237077,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5439488},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Sep 29 05:26:07 2022","interfaces":[{"interface":"back","average":{"1min":0.318,"5min":0.33500000000000002,"15min":0.33700000000000002},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":0.48099999999999998,"5min":0.52100000000000002,"15min":0.52100000000000002},"last":0.32500000000000001},{"interface":"front","average":{"1min":0.33500000000000002,"5min":0.36199999999999999,"15min":0.36599999999999999},"min":{"1min":0.247,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.54800000000000004,"5min":0.55200000000000005,"15min":0.55200000000000005},"last":0.26500000000000001}]},{"osd":2,"last update":"Thu Sep 29 05:25:23 2022","interfaces":[{"interface":"back","average":{"1min":0.38,"5min":0.38800000000000001,"15min":0.38900000000000001},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.53400000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.41699999999999998,"5min":0.45000000000000001,"15min":0.45600000000000002},"min":{"1min":0.308,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.61399999999999999,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.33900000000000002}]},{"osd":3,"last update":"Thu Sep 29 05:25:45 2022","interfaces":[{"interface":"back","average":{"1min":0.34399999999999997,"5min":0.436,"15min":0.45200000000000001},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.49199999999999999,"5min":0.64400000000000002,"15min":0.64400000000000002},"last":0.35799999999999998},{"interface":"front","average":{"1min":0.34599999999999997,"5min":0.42199999999999999,"15min":0.435},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":0.46500000000000002,"5min":0.61699999999999999,"15min":0.61699999999999999},"last":0.308}]},{"osd":4,"last update":"Thu Sep 29 05:26:02 2022","interfaces":[{"interface":"back","average":{"1min":0.59799999999999998,"5min":0.60999999999999999,"15min":0.61199999999999999},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.76700000000000002,"5min":1.1619999999999999,"15min":1.1619999999999999},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.59299999999999997,"15min":0.59499999999999997},"min":{"1min":0.432,"5min":0.38,"15min":0.38},"max":{"1min":0.75600000000000001,"5min":1.1799999999999999,"15min":1.1799999999999999},"last":0.52400000000000002}]},{"osd":5,"last update":"Thu Sep 29 05:25:17 2022","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":1.137,"5min":1.137,"15min":1.137},"last":0.58599999999999997},{"interface":"front","average":{"1min":0.629,"5min":0.629,"15min":0.629},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":1.21,"5min":1.21,"15min":1.21},"last":0.47299999999999998}]},{"osd":6,"last update":"Thu Sep 29 05:25:34 2022","interfaces":[{"interface":"back","average":{"1min":0.749,"5min":0.749,"15min":0.749},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":4.173,"5min":4.173,"15min":4.173},"last":0.505},{"interface":"front","average":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":1.196,"5min":1.196,"15min":1.196},"last":0.56200000000000006}]},{"osd":7,"last update":"Thu Sep 29 05:25:46 2022","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":1.147,"5min":1.147,"15min":1.147},"last":0.55500000000000005},{"interface":"front","average":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":1.8049999999999999,"5min":1.8049999999999999,"15min":1.8049999999999999},"last":0.498}]}]},{"osd":0,"up_from":8,"seq":34359738412,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5944,"kb_used_data":560,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":93739208,"statfs":{"total":95995035648,"available":95988948992,"internally_reserved":0,"allocated":573440,"data_stored":237617,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Sep 29 05:26:06 2022","interfaces":[{"interface":"back","average":{"1min":0.32900000000000001,"5min":0.33200000000000002,"15min":0.32400000000000001},"min":{"1min":0.22,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.46899999999999997,"5min":0.748,"15min":0.748},"last":0.30099999999999999},{"interface":"front","average":{"1min":0.316,"5min":0.34499999999999997,"15min":0.34000000000000002},"min":{"1min":0.20699999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":0.498,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.35499999999999998}]},{"osd":2,"last update":"Thu Sep 29 05:25:31 2022","interfaces":[{"interface":"back","average":{"1min":0.36399999999999999,"5min":0.35799999999999998,"15min":0.35699999999999998},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"last":0.313},{"interface":"front","average":{"1min":0.40100000000000002,"5min":0.38100000000000001,"15min":0.378},"min":{"1min":0.29799999999999999,"5min":0.22,"15min":0.22},"max":{"1min":0.65900000000000003,"5min":0.65900000000000003,"15min":0.65900000000000003},"last":0.38100000000000001}]},{"osd":3,"last update":"Thu Sep 29 05:25:42 2022","interfaces":[{"interface":"back","average":{"1min":0.39200000000000002,"5min":0.42799999999999999,"15min":0.434},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":0.54700000000000004,"5min":0.58999999999999997,"15min":0.58999999999999997},"last":0.41199999999999998},{"interface":"front","average":{"1min":0.36799999999999999,"5min":0.41999999999999998,"15min":0.42899999999999999},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.57799999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"last":0.32400000000000001}]},{"osd":4,"last update":"Thu Sep 29 05:26:01 2022","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.53700000000000003,"15min":0.53200000000000003},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.56699999999999995,"15min":0.56399999999999995},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.77500000000000002,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.60599999999999998}]},{"osd":5,"last update":"Thu Sep 29 05:26:13 2022","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.58099999999999996,"15min":0.58699999999999997},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.68400000000000005,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.49399999999999999},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.59699999999999998,"15min":0.60099999999999998},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.746,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.57999999999999996}]},{"osd":6,"last update":"Thu Sep 29 05:25:31 2022","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.47399999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"min":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.56100000000000005}]},{"osd":7,"last update":"Thu Sep 29 05:25:47 2022","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"min":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.59899999999999998},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.52200000000000002}]}]},{"osd":2,"up_from":18,"seq":77309411365,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5736,"kb_used_data":544,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739416,"statfs":{"total":95995035648,"available":95989161984,"internally_reserved":0,"allocated":557056,"data_stored":236917,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Sep 29 05:25:28 2022","interfaces":[{"interface":"back","average":{"1min":0.34699999999999998,"5min":0.35999999999999999,"15min":0.36199999999999999},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.84299999999999997,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.30299999999999999},{"interface":"front","average":{"1min":0.373,"5min":0.38900000000000001,"15min":0.39200000000000002},"min":{"1min":0.27400000000000002,"5min":0.219,"15min":0.219},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.31}]},{"osd":1,"last update":"Thu Sep 29 05:25:28 2022","interfaces":[{"interface":"back","average":{"1min":0.35299999999999998,"5min":0.374,"15min":0.378},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.53200000000000003,"5min":0.59599999999999997,"15min":0.59599999999999997},"last":0.29299999999999998},{"interface":"front","average":{"1min":0.40500000000000003,"5min":0.41199999999999998,"15min":0.41299999999999998},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.35099999999999998}]},{"osd":3,"last update":"Thu Sep 29 05:25:44 2022","interfaces":[{"interface":"back","average":{"1min":0.39100000000000001,"5min":0.44800000000000001,"15min":0.45800000000000002},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":1.0009999999999999,"5min":1.0009999999999999,"15min":1.0009999999999999},"last":0.33100000000000002},{"interface":"front","average":{"1min":0.38900000000000001,"5min":0.44800000000000001,"15min":0.45800000000000002},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.61599999999999999,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.33900000000000002}]},{"osd":4,"last update":"Thu Sep 29 05:25:59 2022","interfaces":[{"interface":"back","average":{"1min":0.49399999999999999,"5min":0.53500000000000003,"15min":0.54200000000000004},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.86499999999999999,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.63300000000000001},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.57499999999999996,"15min":0.57799999999999996},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.91000000000000003,"5min":0.91000000000000003,"15min":0.91000000000000003},"last":0.62}]},{"osd":5,"last update":"Thu Sep 29 05:25:14 2022","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.50900000000000001},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.73699999999999999,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.61199999999999999}]},{"osd":6,"last update":"Thu Sep 29 05:25:34 2022","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.97999999999999998,"5min":0.97999999999999998,"15min":0.97999999999999998},"last":0.58499999999999996},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"last":0.55100000000000005}]},{"osd":7,"last update":"Thu Sep 29 05:25:52 2022","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"min":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"max":{"1min":0.95799999999999996,"5min":0.95799999999999996,"15min":0.95799999999999996},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.51600000000000001}]}]},{"osd":3,"up_from":26,"seq":111669149729,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5756,"kb_used_data":564,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739396,"statfs":{"total":95995035648,"available":95989141504,"internally_reserved":0,"allocated":577536,"data_stored":239321,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Sep 29 05:25:48 2022","interfaces":[{"interface":"back","average":{"1min":0.317,"5min":0.35099999999999998,"15min":0.35699999999999998},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":0.495,"5min":0.495,"15min":0.495},"last":0.48299999999999998},{"interface":"front","average":{"1min":0.35299999999999998,"5min":0.39400000000000002,"15min":0.40100000000000002},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.46200000000000002,"5min":0.63500000000000001,"15min":0.63500000000000001},"last":0.53600000000000003}]},{"osd":1,"last update":"Thu Sep 29 05:25:48 2022","interfaces":[{"interface":"back","average":{"1min":0.34899999999999998,"5min":0.39900000000000002,"15min":0.40699999999999997},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":0.51900000000000002,"5min":0.57799999999999996,"15min":0.57799999999999996},"last":0.50800000000000001},{"interface":"front","average":{"1min":0.36599999999999999,"5min":0.40100000000000002,"15min":0.40699999999999997},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.58099999999999996,"5min":0.60999999999999999,"15min":0.60999999999999999},"last":0.44800000000000001}]},{"osd":2,"last update":"Thu Sep 29 05:25:48 2022","interfaces":[{"interface":"back","average":{"1min":0.377,"5min":0.43099999999999999,"15min":0.44},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.58899999999999997,"5min":0.59999999999999998,"15min":0.59999999999999998},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.379,"5min":0.41899999999999998,"15min":0.42499999999999999},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.86099999999999999,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.52100000000000002}]},{"osd":4,"last update":"Thu Sep 29 05:26:02 2022","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.57099999999999995,"15min":0.58099999999999996},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.65600000000000003,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.85099999999999998},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.56799999999999995,"15min":0.57599999999999996},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.69399999999999995,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.73999999999999999}]},{"osd":5,"last update":"Thu Sep 29 05:25:15 2022","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.85599999999999998},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":0.876,"5min":0.876,"15min":0.876},"last":0.89000000000000001}]},{"osd":6,"last update":"Thu Sep 29 05:25:33 2022","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.80400000000000005,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.75800000000000001},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.876,"5min":0.876,"15min":0.876},"last":0.82799999999999996}]},{"osd":7,"last update":"Thu Sep 29 05:25:48 2022","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":0.89000000000000001,"5min":0.89000000000000001,"15min":0.89000000000000001},"last":0.73399999999999999},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.83899999999999997}]}]},{"osd":4,"up_from":30,"seq":128849018909,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5752,"kb_used_data":560,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739400,"statfs":{"total":95995035648,"available":95989145600,"internally_reserved":0,"allocated":573440,"data_stored":237597,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Sep 29 05:26:01 2022","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.48299999999999998,"15min":0.47899999999999998},"min":{"1min":0.39800000000000002,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.70899999999999996,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.61899999999999999},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.50900000000000001,"15min":0.50700000000000001},"min":{"1min":0.38800000000000001,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.48399999999999999}]},{"osd":1,"last update":"Thu Sep 29 05:26:01 2022","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.51000000000000001,"15min":0.50800000000000001},"min":{"1min":0.36099999999999999,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.45700000000000002},{"interface":"front","average":{"1min":0.504,"5min":0.51000000000000001,"15min":0.51100000000000001},"min":{"1min":0.374,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.59399999999999997}]},{"osd":2,"last update":"Thu Sep 29 05:26:01 2022","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.52600000000000002,"15min":0.52800000000000002},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.72099999999999997,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.63800000000000001},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.53500000000000003,"15min":0.52300000000000002},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":1.899,"5min":1.899,"15min":1.899},"last":0.55100000000000005}]},{"osd":3,"last update":"Thu Sep 29 05:26:01 2022","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.54900000000000004,"15min":0.54800000000000004},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.95199999999999996,"5min":0.95199999999999996,"15min":0.95199999999999996},"last":0.68500000000000005},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.54100000000000004,"15min":0.53600000000000003},"min":{"1min":0.45200000000000001,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.96099999999999997,"5min":0.96099999999999997,"15min":0.96099999999999997},"last":0.67700000000000005}]},{"osd":5,"last update":"Thu Sep 29 05:25:14 2022","interfaces":[{"interface":"back","average":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.71099999999999997},{"interface":"front","average":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.78200000000000003}]},{"osd":6,"last update":"Thu Sep 29 05:25:30 2022","interfaces":[{"interface":"back","average":{"1min":0.432,"5min":0.432,"15min":0.432},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.73199999999999998},{"interface":"front","average":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.80400000000000005,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.75600000000000001}]},{"osd":7,"last update":"Thu Sep 29 05:25:47 2022","interfaces":[{"interface":"back","average":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.74399999999999999},{"interface":"front","average":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.76800000000000002}]}]},{"osd":5,"up_from":35,"seq":150323855388,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5684,"kb_used_data":556,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739468,"statfs":{"total":95995035648,"available":95989215232,"internally_reserved":0,"allocated":569344,"data_stored":238992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Sep 29 05:26:14 2022","interfaces":[{"interface":"back","average":{"1min":0.49399999999999999,"5min":0.48299999999999998,"15min":0.48099999999999998},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.621,"5min":0.627,"15min":0.627},"last":0.59299999999999997},{"interface":"front","average":{"1min":0.44900000000000001,"5min":0.48099999999999998,"15min":0.48699999999999999},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.64100000000000001,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.42499999999999999}]},{"osd":1,"last update":"Thu Sep 29 05:26:14 2022","interfaces":[{"interface":"back","average":{"1min":0.495,"5min":0.47899999999999998,"15min":0.47699999999999998},"min":{"1min":0.34799999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"last":0.40000000000000002},{"interface":"front","average":{"1min":0.47599999999999998,"5min":0.48799999999999999,"15min":0.48999999999999999},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.68300000000000005,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.68600000000000005}]},{"osd":2,"last update":"Thu Sep 29 05:26:14 2022","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":0.47999999999999998,"15min":0.47599999999999998},"min":{"1min":0.38400000000000001,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.53700000000000003},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.51500000000000001,"15min":0.51900000000000002},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.72799999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.67600000000000005}]},{"osd":3,"last update":"Thu Sep 29 05:26:14 2022","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.48499999999999999,"15min":0.48199999999999998},"min":{"1min":0.39600000000000002,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.69899999999999995},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.53000000000000003,"15min":0.52900000000000003},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.63200000000000001}]},{"osd":4,"last update":"Thu Sep 29 05:26:14 2022","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":0.40600000000000003,"15min":0.40200000000000002},"min":{"1min":0.31900000000000001,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.66600000000000004,"5min":0.66600000000000004,"15min":0.66600000000000004},"last":0.5},{"interface":"front","average":{"1min":0.45300000000000001,"5min":0.42899999999999999,"15min":0.42499999999999999},"min":{"1min":0.34200000000000003,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.73299999999999998,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.44600000000000001}]},{"osd":6,"last update":"Thu Sep 29 05:25:31 2022","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.63,"5min":0.63,"15min":0.63},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"last":0.57499999999999996}]},{"osd":7,"last update":"Thu Sep 29 05:25:48 2022","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.49299999999999999},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.65300000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":700,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":2404,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":2390,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":2300,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2022-09-29T05:26:19.124 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2022-09-29T05:26:19.124 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2022-09-29T05:26:19.125 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2022-09-29T05:26:19.125 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/daemon-base:latest-pacific shell --fsid 69111522-3fb6-11ed-8431-001a4aab830c -- ceph health --format=json 2022-09-29T05:26:19.133 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:18 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:26:17.821272+0000 mgr.y (mgr.14577) 51 : cluster [DBG] pgmap v36: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:26:19.134 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:18 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:26:17.991095+0000 mon.a (mon.0) 672 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:26:19.134 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:18 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:26:17.992207+0000 mgr.y (mgr.14577) 52 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2022-09-29T05:26:19.134 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:18 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:26:17.995411+0000 mgr.y (mgr.14577) 53 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi153 2022-09-29T05:26:19.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:18 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:26:17.821272+0000 mgr.y (mgr.14577) 51 : cluster [DBG] pgmap v36: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:26:19.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:18 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:26:17.991095+0000 mon.a (mon.0) 672 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:26:19.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:18 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:26:17.992207+0000 mgr.y (mgr.14577) 52 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2022-09-29T05:26:19.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:18 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:26:17.995411+0000 mgr.y (mgr.14577) 53 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi153 2022-09-29T05:26:19.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:18 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:26:17.821272+0000 mgr.y (mgr.14577) 51 : cluster [DBG] pgmap v36: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:26:19.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:18 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:26:17.991095+0000 mon.a (mon.0) 672 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:26:19.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:18 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:26:17.992207+0000 mgr.y (mgr.14577) 52 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2022-09-29T05:26:19.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:18 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:26:17.995411+0000 mgr.y (mgr.14577) 53 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi153 2022-09-29T05:26:19.704 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/69111522-3fb6-11ed-8431-001a4aab830c/mon.a/config 2022-09-29T05:26:20.164 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:19 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:26:18.631412+0000 mgr.y (mgr.14577) 54 : audit [DBG] from='client.14769 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2022-09-29T05:26:20.164 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:19 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:26:19.829757+0000 mon.a (mon.0) 673 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1a", "id": [7, 2]}]: dispatch 2022-09-29T05:26:20.165 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:19 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:26:19.829942+0000 mon.a (mon.0) 674 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.b", "id": [0, 2]}]: dispatch 2022-09-29T05:26:20.165 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:19 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:26:19.830124+0000 mon.a (mon.0) 675 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.17", "id": [0, 2]}]: dispatch 2022-09-29T05:26:20.165 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:19 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:26:19.830272+0000 mon.a (mon.0) 676 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.2c", "id": [4, 1]}]: dispatch 2022-09-29T05:26:20.165 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:19 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:26:19.830419+0000 mon.a (mon.0) 677 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.30", "id": [0, 1]}]: dispatch 2022-09-29T05:26:20.166 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:19 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:26:19.830563+0000 mon.a (mon.0) 678 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.32", "id": [0, 2]}]: dispatch 2022-09-29T05:26:20.166 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:19 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:26:19.830704+0000 mon.a (mon.0) 679 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.3b", "id": [4, 3]}]: dispatch 2022-09-29T05:26:20.166 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:19 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:26:19.969692+0000 mon.a (mon.0) 680 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:26:20.166 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:19 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:26:19.974343+0000 mon.a (mon.0) 681 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:26:20.167 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:26:19 smithi153 systemd[1]: Stopping Ceph prometheus.a for 69111522-3fb6-11ed-8431-001a4aab830c... 2022-09-29T05:26:20.167 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:26:19 smithi153 bash[57007]: Error: no container with name or ID "ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus.a" found: no such container 2022-09-29T05:26:20.167 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:26:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[55220]: ts=2022-09-29T05:26:20.043Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2022-09-29T05:26:20.168 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:26:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[55220]: ts=2022-09-29T05:26:20.043Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2022-09-29T05:26:20.168 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:26:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[55220]: ts=2022-09-29T05:26:20.043Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2022-09-29T05:26:20.168 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:26:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[55220]: ts=2022-09-29T05:26:20.043Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2022-09-29T05:26:20.168 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:26:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[55220]: ts=2022-09-29T05:26:20.043Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2022-09-29T05:26:20.168 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:26:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[55220]: ts=2022-09-29T05:26:20.043Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2022-09-29T05:26:20.169 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:26:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[55220]: ts=2022-09-29T05:26:20.043Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2022-09-29T05:26:20.169 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:26:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[55220]: ts=2022-09-29T05:26:20.043Z caller=main.go:828 level=info msg="Scrape manager stopped" 2022-09-29T05:26:20.169 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:26:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[55220]: ts=2022-09-29T05:26:20.043Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2022-09-29T05:26:20.169 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:26:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[55220]: ts=2022-09-29T05:26:20.043Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2022-09-29T05:26:20.170 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:26:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[55220]: ts=2022-09-29T05:26:20.043Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2022-09-29T05:26:20.170 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:26:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[55220]: ts=2022-09-29T05:26:20.043Z caller=main.go:1066 level=info msg="See you next time!" 2022-09-29T05:26:20.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:26:19 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=info ts=2022-09-29T05:26:19.901Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000250636s 2022-09-29T05:26:20.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:19 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:26:18.631412+0000 mgr.y (mgr.14577) 54 : audit [DBG] from='client.14769 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2022-09-29T05:26:20.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:19 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:26:19.829757+0000 mon.a (mon.0) 673 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1a", "id": [7, 2]}]: dispatch 2022-09-29T05:26:20.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:19 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:26:19.829942+0000 mon.a (mon.0) 674 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.b", "id": [0, 2]}]: dispatch 2022-09-29T05:26:20.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:19 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:26:19.830124+0000 mon.a (mon.0) 675 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.17", "id": [0, 2]}]: dispatch 2022-09-29T05:26:20.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:19 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:26:19.830272+0000 mon.a (mon.0) 676 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.2c", "id": [4, 1]}]: dispatch 2022-09-29T05:26:20.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:19 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:26:19.830419+0000 mon.a (mon.0) 677 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.30", "id": [0, 1]}]: dispatch 2022-09-29T05:26:20.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:19 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:26:19.830563+0000 mon.a (mon.0) 678 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.32", "id": [0, 2]}]: dispatch 2022-09-29T05:26:20.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:19 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:26:19.830704+0000 mon.a (mon.0) 679 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.3b", "id": [4, 3]}]: dispatch 2022-09-29T05:26:20.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:19 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:26:19.969692+0000 mon.a (mon.0) 680 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:26:20.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:19 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:26:19.974343+0000 mon.a (mon.0) 681 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:26:20.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:26:18.631412+0000 mgr.y (mgr.14577) 54 : audit [DBG] from='client.14769 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2022-09-29T05:26:20.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:26:19.829757+0000 mon.a (mon.0) 673 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1a", "id": [7, 2]}]: dispatch 2022-09-29T05:26:20.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:26:19.829942+0000 mon.a (mon.0) 674 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.b", "id": [0, 2]}]: dispatch 2022-09-29T05:26:20.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:26:19.830124+0000 mon.a (mon.0) 675 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.17", "id": [0, 2]}]: dispatch 2022-09-29T05:26:20.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:26:19.830272+0000 mon.a (mon.0) 676 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.2c", "id": [4, 1]}]: dispatch 2022-09-29T05:26:20.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:26:19.830419+0000 mon.a (mon.0) 677 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.30", "id": [0, 1]}]: dispatch 2022-09-29T05:26:20.218 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:26:19.830563+0000 mon.a (mon.0) 678 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.32", "id": [0, 2]}]: dispatch 2022-09-29T05:26:20.218 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:26:19.830704+0000 mon.a (mon.0) 679 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.3b", "id": [4, 3]}]: dispatch 2022-09-29T05:26:20.218 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:26:19.969692+0000 mon.a (mon.0) 680 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:26:20.218 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:26:19.974343+0000 mon.a (mon.0) 681 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:26:20.414 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:26:20 smithi153 bash[57022]: ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a 2022-09-29T05:26:20.414 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:26:20 smithi153 bash[57074]: Error: no container with name or ID "ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus.a" found: no such container 2022-09-29T05:26:20.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:26:20 smithi153 systemd[1]: ceph-69111522-3fb6-11ed-8431-001a4aab830c@prometheus.a.service: Succeeded. 2022-09-29T05:26:20.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:26:20 smithi153 systemd[1]: Stopped Ceph prometheus.a for 69111522-3fb6-11ed-8431-001a4aab830c. 2022-09-29T05:26:20.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:26:20 smithi153 systemd[1]: Starting Ceph prometheus.a for 69111522-3fb6-11ed-8431-001a4aab830c... 2022-09-29T05:26:21.189 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:26:21.277 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:26:21 smithi153 podman[57149]: 2022-09-29T05:26:21.277 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:26:19.821708+0000 mgr.y (mgr.14577) 55 : cluster [DBG] pgmap v37: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:26:21.277 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:26:20.000055+0000 mon.a (mon.0) 682 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1a", "id": [7, 2]}]': finished 2022-09-29T05:26:21.278 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:26:20.000126+0000 mon.a (mon.0) 683 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.b", "id": [0, 2]}]': finished 2022-09-29T05:26:21.278 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:26:20.000186+0000 mon.a (mon.0) 684 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.17", "id": [0, 2]}]': finished 2022-09-29T05:26:21.278 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:26:20.000265+0000 mon.a (mon.0) 685 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.2c", "id": [4, 1]}]': finished 2022-09-29T05:26:21.279 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:26:20.000333+0000 mon.a (mon.0) 686 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.30", "id": [0, 1]}]': finished 2022-09-29T05:26:21.279 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:26:20.000388+0000 mon.a (mon.0) 687 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.32", "id": [0, 2]}]': finished 2022-09-29T05:26:21.279 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:26:20.000447+0000 mon.a (mon.0) 688 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.3b", "id": [4, 3]}]': finished 2022-09-29T05:26:21.279 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:26:20.000505+0000 mon.a (mon.0) 689 : cluster [DBG] osdmap e57: 8 total, 8 up, 8 in 2022-09-29T05:26:21.376 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:26:19.821708+0000 mgr.y (mgr.14577) 55 : cluster [DBG] pgmap v37: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:26:21.377 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:26:20.000055+0000 mon.a (mon.0) 682 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1a", "id": [7, 2]}]': finished 2022-09-29T05:26:21.377 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:26:20.000126+0000 mon.a (mon.0) 683 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.b", "id": [0, 2]}]': finished 2022-09-29T05:26:21.377 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:26:20.000186+0000 mon.a (mon.0) 684 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.17", "id": [0, 2]}]': finished 2022-09-29T05:26:21.377 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:26:20.000265+0000 mon.a (mon.0) 685 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.2c", "id": [4, 1]}]': finished 2022-09-29T05:26:21.378 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:26:20.000333+0000 mon.a (mon.0) 686 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.30", "id": [0, 1]}]': finished 2022-09-29T05:26:21.378 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:26:20.000388+0000 mon.a (mon.0) 687 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.32", "id": [0, 2]}]': finished 2022-09-29T05:26:21.378 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:26:20.000447+0000 mon.a (mon.0) 688 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.3b", "id": [4, 3]}]': finished 2022-09-29T05:26:21.378 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:26:20.000505+0000 mon.a (mon.0) 689 : cluster [DBG] osdmap e57: 8 total, 8 up, 8 in 2022-09-29T05:26:21.379 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:26:19.821708+0000 mgr.y (mgr.14577) 55 : cluster [DBG] pgmap v37: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:26:21.379 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:26:20.000055+0000 mon.a (mon.0) 682 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1a", "id": [7, 2]}]': finished 2022-09-29T05:26:21.379 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:26:20.000126+0000 mon.a (mon.0) 683 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.b", "id": [0, 2]}]': finished 2022-09-29T05:26:21.380 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:26:20.000186+0000 mon.a (mon.0) 684 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.17", "id": [0, 2]}]': finished 2022-09-29T05:26:21.380 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:26:20.000265+0000 mon.a (mon.0) 685 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.2c", "id": [4, 1]}]': finished 2022-09-29T05:26:21.380 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:26:20.000333+0000 mon.a (mon.0) 686 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.30", "id": [0, 1]}]': finished 2022-09-29T05:26:21.380 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:26:20.000388+0000 mon.a (mon.0) 687 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.32", "id": [0, 2]}]': finished 2022-09-29T05:26:21.380 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:26:20.000447+0000 mon.a (mon.0) 688 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.3b", "id": [4, 3]}]': finished 2022-09-29T05:26:21.381 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:26:20.000505+0000 mon.a (mon.0) 689 : cluster [DBG] osdmap e57: 8 total, 8 up, 8 in 2022-09-29T05:26:21.559 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:26:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[57164]: ts=2022-09-29T05:26:21.273Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2022-09-29T05:26:21.559 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:26:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[57164]: ts=2022-09-29T05:26:21.274Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2022-09-29T05:26:21.560 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:26:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[57164]: ts=2022-09-29T05:26:21.274Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2022-09-29T05:26:21.560 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:26:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[57164]: ts=2022-09-29T05:26:21.274Z caller=main.go:518 level=info host_details="(Linux 4.18.0-408.el8.x86_64 #1 SMP Mon Jul 18 17:42:52 UTC 2022 x86_64 smithi153 (none))" 2022-09-29T05:26:21.560 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:26:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[57164]: ts=2022-09-29T05:26:21.274Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2022-09-29T05:26:21.560 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:26:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[57164]: ts=2022-09-29T05:26:21.274Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2022-09-29T05:26:21.561 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:26:21 smithi153 bash[57149]: fb1e43a0d8ba985dd60177317e489ea7f803a12c22ecd8195f328a07ac730b72 2022-09-29T05:26:21.561 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:26:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[57164]: ts=2022-09-29T05:26:21.275Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2022-09-29T05:26:21.561 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:26:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[57164]: ts=2022-09-29T05:26:21.276Z caller=main.go:923 level=info msg="Starting TSDB ..." 2022-09-29T05:26:21.562 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:26:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[57164]: ts=2022-09-29T05:26:21.276Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2022-09-29T05:26:21.562 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:26:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[57164]: ts=2022-09-29T05:26:21.279Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2022-09-29T05:26:21.562 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:26:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[57164]: ts=2022-09-29T05:26:21.279Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.899µs 2022-09-29T05:26:21.562 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:26:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[57164]: ts=2022-09-29T05:26:21.279Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2022-09-29T05:26:21.563 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:26:21 smithi153 systemd[1]: Started Ceph prometheus.a for 69111522-3fb6-11ed-8431-001a4aab830c. 2022-09-29T05:26:21.563 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:26:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[57164]: ts=2022-09-29T05:26:21.449Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2022-09-29T05:26:21.563 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:26:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[57164]: ts=2022-09-29T05:26:21.450Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2022-09-29T05:26:21.563 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:26:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[57164]: ts=2022-09-29T05:26:21.450Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=28.17µs wal_replay_duration=170.853529ms total_replay_duration=170.912391ms 2022-09-29T05:26:21.564 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:26:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[57164]: ts=2022-09-29T05:26:21.451Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2022-09-29T05:26:21.564 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:26:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[57164]: ts=2022-09-29T05:26:21.451Z caller=main.go:947 level=info msg="TSDB started" 2022-09-29T05:26:21.564 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:26:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[57164]: ts=2022-09-29T05:26:21.451Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2022-09-29T05:26:21.564 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:26:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[57164]: ts=2022-09-29T05:26:21.468Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=16.824433ms db_storage=693ns remote_storage=1.071µs web_handler=332ns query_engine=855ns scrape=400.759µs scrape_sd=32.937µs notify=33.783µs notify_sd=20.299µs rules=16.088737ms 2022-09-29T05:26:21.565 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:26:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[57164]: ts=2022-09-29T05:26:21.468Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2022-09-29T05:26:21.645 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:26:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: [29/Sep/2022:05:26:21] ENGINE Bus STOPPING 2022-09-29T05:26:21.646 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:26:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: [29/Sep/2022:05:26:21] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2022-09-29T05:26:21.646 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:26:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: [29/Sep/2022:05:26:21] ENGINE Bus STOPPED 2022-09-29T05:26:21.646 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:26:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: [29/Sep/2022:05:26:21] ENGINE Bus STARTING 2022-09-29T05:26:21.646 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:26:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: [29/Sep/2022:05:26:21] ENGINE Serving on http://:::9283 2022-09-29T05:26:21.647 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:26:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: [29/Sep/2022:05:26:21] ENGINE Bus STARTED 2022-09-29T05:26:21.647 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:26:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: [29/Sep/2022:05:26:21] ENGINE Bus STOPPING 2022-09-29T05:26:21.647 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:26:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: [29/Sep/2022:05:26:21] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2022-09-29T05:26:21.647 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:26:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: [29/Sep/2022:05:26:21] ENGINE Bus STOPPED 2022-09-29T05:26:21.648 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:26:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: [29/Sep/2022:05:26:21] ENGINE Bus STARTING 2022-09-29T05:26:21.712 INFO:teuthology.orchestra.run.smithi083.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2022-09-29T05:26:21.713 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2022-09-29T05:26:21.713 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2022-09-29T05:26:21.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:26:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: [29/Sep/2022:05:26:21] ENGINE Serving on http://:::9283 2022-09-29T05:26:21.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:26:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: [29/Sep/2022:05:26:21] ENGINE Bus STARTED 2022-09-29T05:26:21.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:26:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: [29/Sep/2022:05:26:21] ENGINE Bus STOPPING 2022-09-29T05:26:21.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:26:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: [29/Sep/2022:05:26:21] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2022-09-29T05:26:21.964 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:26:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: [29/Sep/2022:05:26:21] ENGINE Bus STOPPED 2022-09-29T05:26:21.964 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:26:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: [29/Sep/2022:05:26:21] ENGINE Bus STARTING 2022-09-29T05:26:21.964 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:26:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: [29/Sep/2022:05:26:21] ENGINE Serving on http://:::9283 2022-09-29T05:26:21.964 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:26:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: [29/Sep/2022:05:26:21] ENGINE Bus STARTED 2022-09-29T05:26:22.011 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:26:22.012 INFO:teuthology.orchestra.run.smithi083.stdout:{"epoch":58,"fsid":"69111522-3fb6-11ed-8431-001a4aab830c","created":"2022-09-29T05:20:45.037726+0000","modified":"2022-09-29T05:26:20.997298+0000","last_up_change":"2022-09-29T05:24:46.528124+0000","last_in_change":"2022-09-29T05:24:31.376431+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2022-09-29T05:23:34.795850+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"54","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2022-09-29T05:25:01.755735+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"50","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}}},{"pool":3,"pool_name":"cephfs_data","create_time":"2022-09-29T05:25:02.966273+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"51","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}},{"pool":4,"pool_name":"rbd","create_time":"2022-09-29T05:25:35.425789+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"56","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"9b8b79ad-3d00-4dad-8da7-7cff0e65d302","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6802","nonce":2080129456},{"type":"v1","addr":"172.21.15.83:6803","nonce":2080129456}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6804","nonce":2080129456},{"type":"v1","addr":"172.21.15.83:6805","nonce":2080129456}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6808","nonce":2080129456},{"type":"v1","addr":"172.21.15.83:6809","nonce":2080129456}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6806","nonce":2080129456},{"type":"v1","addr":"172.21.15.83:6807","nonce":2080129456}]},"public_addr":"172.21.15.83:6803/2080129456","cluster_addr":"172.21.15.83:6805/2080129456","heartbeat_back_addr":"172.21.15.83:6809/2080129456","heartbeat_front_addr":"172.21.15.83:6807/2080129456","state":["exists","up"]},{"osd":1,"uuid":"e098cff5-70cc-437e-8759-e6c7e3d452d2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6810","nonce":605787053},{"type":"v1","addr":"172.21.15.83:6811","nonce":605787053}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6812","nonce":605787053},{"type":"v1","addr":"172.21.15.83:6813","nonce":605787053}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6816","nonce":605787053},{"type":"v1","addr":"172.21.15.83:6817","nonce":605787053}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6814","nonce":605787053},{"type":"v1","addr":"172.21.15.83:6815","nonce":605787053}]},"public_addr":"172.21.15.83:6811/605787053","cluster_addr":"172.21.15.83:6813/605787053","heartbeat_back_addr":"172.21.15.83:6817/605787053","heartbeat_front_addr":"172.21.15.83:6815/605787053","state":["exists","up"]},{"osd":2,"uuid":"2c14853b-5b64-496d-a011-5d8a9b17b731","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6818","nonce":1928326991},{"type":"v1","addr":"172.21.15.83:6819","nonce":1928326991}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6820","nonce":1928326991},{"type":"v1","addr":"172.21.15.83:6821","nonce":1928326991}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6824","nonce":1928326991},{"type":"v1","addr":"172.21.15.83:6825","nonce":1928326991}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6822","nonce":1928326991},{"type":"v1","addr":"172.21.15.83:6823","nonce":1928326991}]},"public_addr":"172.21.15.83:6819/1928326991","cluster_addr":"172.21.15.83:6821/1928326991","heartbeat_back_addr":"172.21.15.83:6825/1928326991","heartbeat_front_addr":"172.21.15.83:6823/1928326991","state":["exists","up"]},{"osd":3,"uuid":"7336f148-261e-468e-8d3c-28f840150f36","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6826","nonce":54976712},{"type":"v1","addr":"172.21.15.83:6827","nonce":54976712}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6828","nonce":54976712},{"type":"v1","addr":"172.21.15.83:6829","nonce":54976712}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6832","nonce":54976712},{"type":"v1","addr":"172.21.15.83:6833","nonce":54976712}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6830","nonce":54976712},{"type":"v1","addr":"172.21.15.83:6831","nonce":54976712}]},"public_addr":"172.21.15.83:6827/54976712","cluster_addr":"172.21.15.83:6829/54976712","heartbeat_back_addr":"172.21.15.83:6833/54976712","heartbeat_front_addr":"172.21.15.83:6831/54976712","state":["exists","up"]},{"osd":4,"uuid":"247d5c19-e8af-4620-a378-699dc2000db1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6800","nonce":4053209633},{"type":"v1","addr":"172.21.15.153:6801","nonce":4053209633}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6802","nonce":4053209633},{"type":"v1","addr":"172.21.15.153:6803","nonce":4053209633}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6806","nonce":4053209633},{"type":"v1","addr":"172.21.15.153:6807","nonce":4053209633}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6804","nonce":4053209633},{"type":"v1","addr":"172.21.15.153:6805","nonce":4053209633}]},"public_addr":"172.21.15.153:6801/4053209633","cluster_addr":"172.21.15.153:6803/4053209633","heartbeat_back_addr":"172.21.15.153:6807/4053209633","heartbeat_front_addr":"172.21.15.153:6805/4053209633","state":["exists","up"]},{"osd":5,"uuid":"c69e2263-5dd8-402d-b901-295c1f4840e2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":35,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6808","nonce":2536485968},{"type":"v1","addr":"172.21.15.153:6809","nonce":2536485968}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6810","nonce":2536485968},{"type":"v1","addr":"172.21.15.153:6811","nonce":2536485968}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6814","nonce":2536485968},{"type":"v1","addr":"172.21.15.153:6815","nonce":2536485968}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6812","nonce":2536485968},{"type":"v1","addr":"172.21.15.153:6813","nonce":2536485968}]},"public_addr":"172.21.15.153:6809/2536485968","cluster_addr":"172.21.15.153:6811/2536485968","heartbeat_back_addr":"172.21.15.153:6815/2536485968","heartbeat_front_addr":"172.21.15.153:6813/2536485968","state":["exists","up"]},{"osd":6,"uuid":"72aa13b6-e7b7-449b-9fa8-f284e8f0735a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":40,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6816","nonce":3662388207},{"type":"v1","addr":"172.21.15.153:6817","nonce":3662388207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6818","nonce":3662388207},{"type":"v1","addr":"172.21.15.153:6819","nonce":3662388207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6822","nonce":3662388207},{"type":"v1","addr":"172.21.15.153:6823","nonce":3662388207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6820","nonce":3662388207},{"type":"v1","addr":"172.21.15.153:6821","nonce":3662388207}]},"public_addr":"172.21.15.153:6817/3662388207","cluster_addr":"172.21.15.153:6819/3662388207","heartbeat_back_addr":"172.21.15.153:6823/3662388207","heartbeat_front_addr":"172.21.15.153:6821/3662388207","state":["exists","up"]},{"osd":7,"uuid":"b5fdd156-d2dc-47fb-b44b-803275092ac0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":45,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6824","nonce":1223012606},{"type":"v1","addr":"172.21.15.153:6825","nonce":1223012606}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6826","nonce":1223012606},{"type":"v1","addr":"172.21.15.153:6827","nonce":1223012606}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6830","nonce":1223012606},{"type":"v1","addr":"172.21.15.153:6831","nonce":1223012606}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6828","nonce":1223012606},{"type":"v1","addr":"172.21.15.153:6829","nonce":1223012606}]},"public_addr":"172.21.15.153:6825/1223012606","cluster_addr":"172.21.15.153:6827/1223012606","heartbeat_back_addr":"172.21.15.153:6831/1223012606","heartbeat_front_addr":"172.21.15.153:6829/1223012606","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2022-09-29T05:22:43.518222+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2022-09-29T05:23:01.859214+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2022-09-29T05:23:20.325748+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2022-09-29T05:23:38.717068+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2022-09-29T05:23:54.756146+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2022-09-29T05:24:11.670499+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2022-09-29T05:24:28.641104+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2022-09-29T05:24:45.518895+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.1a","mappings":[{"from":7,"to":2}]},{"pgid":"3.b","mappings":[{"from":0,"to":2}]},{"pgid":"3.17","mappings":[{"from":0,"to":2}]},{"pgid":"3.2c","mappings":[{"from":4,"to":1}]},{"pgid":"3.30","mappings":[{"from":0,"to":1}]},{"pgid":"3.32","mappings":[{"from":0,"to":2}]},{"pgid":"3.3b","mappings":[{"from":4,"to":3}]}],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.83:6801/2790412241":"2022-09-30T05:25:19.746941+0000","172.21.15.83:0/4071118407":"2022-09-30T05:25:19.746941+0000","172.21.15.83:0/2269625481":"2022-09-30T05:21:35.319810+0000","172.21.15.83:0/2173437316":"2022-09-30T05:21:35.319810+0000","172.21.15.83:6800/2702658771":"2022-09-30T05:21:35.319810+0000","172.21.15.83:0/4005486955":"2022-09-30T05:21:15.866011+0000","172.21.15.83:6800/2790412241":"2022-09-30T05:25:19.746941+0000","172.21.15.83:6800/2006042278":"2022-09-30T05:21:15.866011+0000","172.21.15.83:6801/2006042278":"2022-09-30T05:21:15.866011+0000","172.21.15.83:0/664253330":"2022-09-30T05:25:19.746941+0000","172.21.15.83:6801/2702658771":"2022-09-30T05:21:35.319810+0000","172.21.15.83:0/4256024697":"2022-09-30T05:21:15.866011+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2022-09-29T05:26:22.027 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': 'device_health_metrics', 'create_time': '2022-09-29T05:23:34.795850+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '54', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr_devicehealth': {}}}, {'pool': 2, 'pool_name': 'cephfs_metadata', 'create_time': '2022-09-29T05:25:01.755735+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '50', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}}, {'pool': 3, 'pool_name': 'cephfs_data', 'create_time': '2022-09-29T05:25:02.966273+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '51', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}}, {'pool': 4, 'pool_name': 'rbd', 'create_time': '2022-09-29T05:25:35.425789+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 8, 'pg_placement_num': 8, 'pg_placement_num_target': 8, 'pg_num_target': 8, 'pg_num_pending': 8, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '56', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {}, 'application_metadata': {'rbd': {}}}] 2022-09-29T05:26:22.028 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get device_health_metrics pg_num 2022-09-29T05:26:22.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:26:21.005908+0000 mon.a (mon.0) 690 : cluster [DBG] osdmap e58: 8 total, 8 up, 8 in 2022-09-29T05:26:22.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:26:21.186163+0000 mon.a (mon.0) 691 : audit [DBG] from='client.? 172.21.15.83:0/1320790584' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2022-09-29T05:26:22.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:26:21.398987+0000 mon.a (mon.0) 692 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:26:22.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:26:21.402039+0000 mon.a (mon.0) 693 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2022-09-29T05:26:22.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:26:21.402437+0000 mgr.y (mgr.14577) 56 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2022-09-29T05:26:22.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:26:21.403191+0000 mon.a (mon.0) 694 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi083:9093"}]: dispatch 2022-09-29T05:26:22.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:26:21.403435+0000 mgr.y (mgr.14577) 57 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi083:9093"}]: dispatch 2022-09-29T05:26:22.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:26:21.407128+0000 mon.a (mon.0) 695 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:26:22.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:26:21.411398+0000 mon.a (mon.0) 696 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2022-09-29T05:26:22.312 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:26:21.411850+0000 mgr.y (mgr.14577) 58 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2022-09-29T05:26:22.312 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:26:21.529282+0000 mon.a (mon.0) 697 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi153.front.sepia.ceph.com:9095"}]: dispatch 2022-09-29T05:26:22.312 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:26:21.529636+0000 mgr.y (mgr.14577) 59 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi153.front.sepia.ceph.com:9095"}]: dispatch 2022-09-29T05:26:22.312 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:26:21.535486+0000 mon.a (mon.0) 698 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:26:22.313 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:26:21.539623+0000 mon.a (mon.0) 699 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2022-09-29T05:26:22.313 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:26:21.540053+0000 mgr.y (mgr.14577) 60 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2022-09-29T05:26:22.313 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:26:21.656728+0000 mon.a (mon.0) 700 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi153.front.sepia.ceph.com:3000"}]: dispatch 2022-09-29T05:26:22.313 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:26:21.660876+0000 mon.a (mon.0) 701 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:26:22.314 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:26:21.662696+0000 mon.a (mon.0) 702 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:26:22.314 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:26:21.663675+0000 mon.a (mon.0) 703 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:26:22.314 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:26:21.664338+0000 mon.a (mon.0) 704 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:26:22.314 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:26:22.008095+0000 mon.c (mon.1) 29 : audit [DBG] from='client.? 172.21.15.83:0/315449306' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2022-09-29T05:26:22.322 INFO:teuthology.orchestra.run.smithi083.stdout:pg_num: 1 2022-09-29T05:26:22.335 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:26:21.005908+0000 mon.a (mon.0) 690 : cluster [DBG] osdmap e58: 8 total, 8 up, 8 in 2022-09-29T05:26:22.335 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:26:21.186163+0000 mon.a (mon.0) 691 : audit [DBG] from='client.? 172.21.15.83:0/1320790584' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2022-09-29T05:26:22.336 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:26:21.398987+0000 mon.a (mon.0) 692 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:26:22.336 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:26:21.402039+0000 mon.a (mon.0) 693 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2022-09-29T05:26:22.336 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:26:21.402437+0000 mgr.y (mgr.14577) 56 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2022-09-29T05:26:22.336 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:26:21.403191+0000 mon.a (mon.0) 694 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi083:9093"}]: dispatch 2022-09-29T05:26:22.337 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:26:21.403435+0000 mgr.y (mgr.14577) 57 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi083:9093"}]: dispatch 2022-09-29T05:26:22.337 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:26:21.407128+0000 mon.a (mon.0) 695 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:26:22.337 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:26:21.411398+0000 mon.a (mon.0) 696 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2022-09-29T05:26:22.337 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:26:21.411850+0000 mgr.y (mgr.14577) 58 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2022-09-29T05:26:22.338 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:26:21.529282+0000 mon.a (mon.0) 697 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi153.front.sepia.ceph.com:9095"}]: dispatch 2022-09-29T05:26:22.338 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:26:21.529636+0000 mgr.y (mgr.14577) 59 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi153.front.sepia.ceph.com:9095"}]: dispatch 2022-09-29T05:26:22.338 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:26:21.535486+0000 mon.a (mon.0) 698 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:26:22.338 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:26:21.539623+0000 mon.a (mon.0) 699 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2022-09-29T05:26:22.339 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:26:21.540053+0000 mgr.y (mgr.14577) 60 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2022-09-29T05:26:22.339 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:26:21.656728+0000 mon.a (mon.0) 700 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi153.front.sepia.ceph.com:3000"}]: dispatch 2022-09-29T05:26:22.339 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:26:21.660876+0000 mon.a (mon.0) 701 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:26:22.339 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:26:21.662696+0000 mon.a (mon.0) 702 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:26:22.340 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:26:21.663675+0000 mon.a (mon.0) 703 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:26:22.340 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:26:21.664338+0000 mon.a (mon.0) 704 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:26:22.340 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:26:22.008095+0000 mon.c (mon.1) 29 : audit [DBG] from='client.? 172.21.15.83:0/315449306' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2022-09-29T05:26:22.341 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:26:21.005908+0000 mon.a (mon.0) 690 : cluster [DBG] osdmap e58: 8 total, 8 up, 8 in 2022-09-29T05:26:22.341 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:26:21.186163+0000 mon.a (mon.0) 691 : audit [DBG] from='client.? 172.21.15.83:0/1320790584' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2022-09-29T05:26:22.341 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:26:21.398987+0000 mon.a (mon.0) 692 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:26:22.342 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:26:21.402039+0000 mon.a (mon.0) 693 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2022-09-29T05:26:22.342 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:26:21.402437+0000 mgr.y (mgr.14577) 56 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2022-09-29T05:26:22.342 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:26:21.403191+0000 mon.a (mon.0) 694 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi083:9093"}]: dispatch 2022-09-29T05:26:22.342 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:26:21.403435+0000 mgr.y (mgr.14577) 57 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi083:9093"}]: dispatch 2022-09-29T05:26:22.343 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:26:21.407128+0000 mon.a (mon.0) 695 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:26:22.343 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:26:21.411398+0000 mon.a (mon.0) 696 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2022-09-29T05:26:22.343 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:26:21.411850+0000 mgr.y (mgr.14577) 58 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2022-09-29T05:26:22.344 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:26:21.529282+0000 mon.a (mon.0) 697 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi153.front.sepia.ceph.com:9095"}]: dispatch 2022-09-29T05:26:22.344 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:26:21.529636+0000 mgr.y (mgr.14577) 59 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi153.front.sepia.ceph.com:9095"}]: dispatch 2022-09-29T05:26:22.344 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:26:21.535486+0000 mon.a (mon.0) 698 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:26:22.344 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:26:21.539623+0000 mon.a (mon.0) 699 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2022-09-29T05:26:22.345 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:26:21.540053+0000 mgr.y (mgr.14577) 60 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2022-09-29T05:26:22.345 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:26:21.656728+0000 mon.a (mon.0) 700 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi153.front.sepia.ceph.com:3000"}]: dispatch 2022-09-29T05:26:22.345 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:26:21.660876+0000 mon.a (mon.0) 701 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:26:22.346 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:26:21.662696+0000 mon.a (mon.0) 702 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:26:22.346 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:26:21.663675+0000 mon.a (mon.0) 703 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:26:22.346 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:26:21.664338+0000 mon.a (mon.0) 704 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:26:22.347 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:26:22.008095+0000 mon.c (mon.1) 29 : audit [DBG] from='client.? 172.21.15.83:0/315449306' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2022-09-29T05:26:22.348 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2022-09-29T05:26:22.649 INFO:teuthology.orchestra.run.smithi083.stdout:pg_num: 32 2022-09-29T05:26:22.664 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2022-09-29T05:26:22.963 INFO:teuthology.orchestra.run.smithi083.stdout:pg_num: 64 2022-09-29T05:26:22.977 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get rbd pg_num 2022-09-29T05:26:23.275 INFO:teuthology.orchestra.run.smithi083.stdout:pg_num: 8 2022-09-29T05:26:23.288 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:26:21.657080+0000 mgr.y (mgr.14577) 61 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi153.front.sepia.ceph.com:3000"}]: dispatch 2022-09-29T05:26:23.288 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:26:21.822090+0000 mgr.y (mgr.14577) 62 : cluster [DBG] pgmap v40: 105 pgs: 7 peering, 98 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:26:23.288 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:26:22.319872+0000 mon.a (mon.0) 705 : audit [DBG] from='client.? 172.21.15.83:0/399229631' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "device_health_metrics", "var": "pg_num"}]: dispatch 2022-09-29T05:26:23.288 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:26:22.646994+0000 mon.c (mon.1) 30 : audit [DBG] from='client.? 172.21.15.83:0/4260892098' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2022-09-29T05:26:23.289 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:26:22.961026+0000 mon.a (mon.0) 706 : audit [DBG] from='client.? 172.21.15.83:0/996631072' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2022-09-29T05:26:23.290 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:26:21.657080+0000 mgr.y (mgr.14577) 61 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi153.front.sepia.ceph.com:3000"}]: dispatch 2022-09-29T05:26:23.290 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:26:21.822090+0000 mgr.y (mgr.14577) 62 : cluster [DBG] pgmap v40: 105 pgs: 7 peering, 98 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:26:23.291 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:26:22.319872+0000 mon.a (mon.0) 705 : audit [DBG] from='client.? 172.21.15.83:0/399229631' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "device_health_metrics", "var": "pg_num"}]: dispatch 2022-09-29T05:26:23.291 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:26:22.646994+0000 mon.c (mon.1) 30 : audit [DBG] from='client.? 172.21.15.83:0/4260892098' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2022-09-29T05:26:23.292 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:26:22.961026+0000 mon.a (mon.0) 706 : audit [DBG] from='client.? 172.21.15.83:0/996631072' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2022-09-29T05:26:23.293 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2022-09-29T05:26:23.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:26:21.657080+0000 mgr.y (mgr.14577) 61 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi153.front.sepia.ceph.com:3000"}]: dispatch 2022-09-29T05:26:23.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:26:21.822090+0000 mgr.y (mgr.14577) 62 : cluster [DBG] pgmap v40: 105 pgs: 7 peering, 98 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:26:23.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:26:22.319872+0000 mon.a (mon.0) 705 : audit [DBG] from='client.? 172.21.15.83:0/399229631' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "device_health_metrics", "var": "pg_num"}]: dispatch 2022-09-29T05:26:23.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:26:22.646994+0000 mon.c (mon.1) 30 : audit [DBG] from='client.? 172.21.15.83:0/4260892098' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2022-09-29T05:26:23.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:26:22.961026+0000 mon.a (mon.0) 706 : audit [DBG] from='client.? 172.21.15.83:0/996631072' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2022-09-29T05:26:23.627 INFO:teuthology.orchestra.run.smithi083.stdout: 2022-09-29T05:26:23.628 INFO:teuthology.orchestra.run.smithi083.stdout:{"epoch":6,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":24343,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.83:6835/4236763119","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6834","nonce":4236763119},{"type":"v1","addr":"172.21.15.83:6835","nonce":4236763119}]},"join_fscid":-1,"export_targets":[],"features":4540138297136906239,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2}],"filesystems":[{"mdsmap":{"epoch":6,"flags":18,"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2022-09-29T05:25:03.789615+0000","modified":"2022-09-29T05:25:05.816761+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"max_mds":1,"in":[0],"up":{"mds_0":24356},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24356":{"gid":24356,"name":"b","rank":0,"incarnation":4,"state":"up:active","state_seq":3,"addr":"172.21.15.153:6833/673217382","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6832","nonce":673217382},{"type":"v1","addr":"172.21.15.153:6833","nonce":673217382}]},"join_fscid":-1,"export_targets":[],"features":4540138297136906239,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","standby_count_wanted":0},"id":1}]} 2022-09-29T05:26:23.630 INFO:teuthology.orchestra.run.smithi083.stderr:dumped fsmap epoch 6 2022-09-29T05:26:23.643 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 6, 'flags': 18, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2022-09-29T05:25:03.789615+0000', 'modified': '2022-09-29T05:25:05.816761+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2'}}, 'max_mds': 1, 'in': [0], 'up': {'mds_0': 24356}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_24356': {'gid': 24356, 'name': 'b', 'rank': 0, 'incarnation': 4, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.153:6833/673217382', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.153:6832', 'nonce': 673217382}, {'type': 'v1', 'addr': '172.21.15.153:6833', 'nonce': 673217382}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540138297136906239, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2'}}}}, 'data_pools': [3], 'metadata_pool': 2, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'standby_count_wanted': 0} 2022-09-29T05:26:23.643 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: 1/1 2022-09-29T05:26:23.643 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.b status 2022-09-29T05:26:23.773 INFO:teuthology.orchestra.run.smithi083.stderr:2022-09-29T05:26:23.769+0000 7feed4ff9700 0 client.14823 ms_handle_reset on v2:172.21.15.153:6832/673217382 2022-09-29T05:26:23.795 INFO:teuthology.orchestra.run.smithi083.stderr:2022-09-29T05:26:23.791+0000 7feed4ff9700 0 client.24739 ms_handle_reset on v2:172.21.15.153:6832/673217382 2022-09-29T05:26:23.796 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T05:26:23.796 INFO:teuthology.orchestra.run.smithi083.stdout: "cluster_fsid": "69111522-3fb6-11ed-8431-001a4aab830c", 2022-09-29T05:26:23.797 INFO:teuthology.orchestra.run.smithi083.stdout: "whoami": 0, 2022-09-29T05:26:23.797 INFO:teuthology.orchestra.run.smithi083.stdout: "id": 24356, 2022-09-29T05:26:23.798 INFO:teuthology.orchestra.run.smithi083.stdout: "want_state": "up:active", 2022-09-29T05:26:23.798 INFO:teuthology.orchestra.run.smithi083.stdout: "state": "up:active", 2022-09-29T05:26:23.799 INFO:teuthology.orchestra.run.smithi083.stdout: "fs_name": "cephfs", 2022-09-29T05:26:23.799 INFO:teuthology.orchestra.run.smithi083.stdout: "rank_uptime": 79.999656931999994, 2022-09-29T05:26:23.800 INFO:teuthology.orchestra.run.smithi083.stdout: "mdsmap_epoch": 6, 2022-09-29T05:26:23.800 INFO:teuthology.orchestra.run.smithi083.stdout: "osdmap_epoch": 58, 2022-09-29T05:26:23.800 INFO:teuthology.orchestra.run.smithi083.stdout: "osdmap_epoch_barrier": 51, 2022-09-29T05:26:23.801 INFO:teuthology.orchestra.run.smithi083.stdout: "uptime": 84.564483721000002 2022-09-29T05:26:23.801 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T05:26:23.808 INFO:tasks.cephadm:Setup complete, yielding 2022-09-29T05:26:23.808 INFO:teuthology.run_tasks:Running task print... 2022-09-29T05:26:23.825 INFO:teuthology.task.print:**** done end installing pacific cephadm ... 2022-09-29T05:26:23.826 INFO:teuthology.run_tasks:Running task print... 2022-09-29T05:26:23.843 INFO:teuthology.task.print:**** done start cephadm.shell ceph config set mgr... 2022-09-29T05:26:23.843 INFO:teuthology.run_tasks:Running task cephadm.shell... 2022-09-29T05:26:23.861 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi083.front.sepia.ceph.com 2022-09-29T05:26:23.861 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/daemon-base:latest-pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 69111522-3fb6-11ed-8431-001a4aab830c -- bash -c 'ceph config set mgr mgr/cephadm/use_repo_digest true --force' 2022-09-29T05:26:24.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:24 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:26:23.272664+0000 mon.c (mon.1) 31 : audit [DBG] from='client.? 172.21.15.83:0/1221552052' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "rbd", "var": "pg_num"}]: dispatch 2022-09-29T05:26:24.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:24 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:26:23.624854+0000 mon.a (mon.0) 707 : audit [DBG] from='client.? 172.21.15.83:0/4053755474' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2022-09-29T05:26:24.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:26:23.272664+0000 mon.c (mon.1) 31 : audit [DBG] from='client.? 172.21.15.83:0/1221552052' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "rbd", "var": "pg_num"}]: dispatch 2022-09-29T05:26:24.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:26:23.624854+0000 mon.a (mon.0) 707 : audit [DBG] from='client.? 172.21.15.83:0/4053755474' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2022-09-29T05:26:24.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:26:23.272664+0000 mon.c (mon.1) 31 : audit [DBG] from='client.? 172.21.15.83:0/1221552052' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "rbd", "var": "pg_num"}]: dispatch 2022-09-29T05:26:24.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:26:23.624854+0000 mon.a (mon.0) 707 : audit [DBG] from='client.? 172.21.15.83:0/4053755474' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2022-09-29T05:26:25.292 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:26:23.822383+0000 mgr.y (mgr.14577) 63 : cluster [DBG] pgmap v41: 105 pgs: 7 peering, 98 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:26:25.293 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:26:23.822383+0000 mgr.y (mgr.14577) 63 : cluster [DBG] pgmap v41: 105 pgs: 7 peering, 98 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:26:25.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:25 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:26:23.822383+0000 mgr.y (mgr.14577) 63 : cluster [DBG] pgmap v41: 105 pgs: 7 peering, 98 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:26:26.415 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:26:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: [29/Sep/2022:05:26:26] ENGINE Bus STOPPING 2022-09-29T05:26:26.415 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:26:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: [29/Sep/2022:05:26:26] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2022-09-29T05:26:26.416 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:26:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: [29/Sep/2022:05:26:26] ENGINE Bus STOPPED 2022-09-29T05:26:26.416 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:26:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: [29/Sep/2022:05:26:26] ENGINE Bus STARTING 2022-09-29T05:26:26.416 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:26:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: [29/Sep/2022:05:26:26] ENGINE Serving on http://:::9283 2022-09-29T05:26:26.416 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:26:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: [29/Sep/2022:05:26:26] ENGINE Bus STARTED 2022-09-29T05:26:26.546 INFO:teuthology.run_tasks:Running task print... 2022-09-29T05:26:26.556 INFO:teuthology.task.print:**** done cephadm.shell ceph config set mgr... 2022-09-29T05:26:26.557 INFO:teuthology.run_tasks:Running task print... 2022-09-29T05:26:26.568 INFO:teuthology.task.print:**** done start telemetry pacific... 2022-09-29T05:26:26.568 INFO:teuthology.run_tasks:Running task workunit... 2022-09-29T05:26:26.582 INFO:tasks.workunit:Pulling workunits from ref dbaedf5bf154c5830301d16d6da9fb01a6c0c0c5 2022-09-29T05:26:26.583 INFO:tasks.workunit:Making a separate scratch dir for every client... 2022-09-29T05:26:26.583 DEBUG:teuthology.orchestra.run.smithi083:> stat -- /home/ubuntu/cephtest/mnt.0 2022-09-29T05:26:26.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2022-09-29T05:26:26.603 INFO:teuthology.orchestra.run.smithi083.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2022-09-29T05:26:26.603 DEBUG:teuthology.orchestra.run.smithi083:> mkdir -- /home/ubuntu/cephtest/mnt.0 2022-09-29T05:26:26.661 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2022-09-29T05:26:26.662 DEBUG:teuthology.orchestra.run.smithi083:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2022-09-29T05:26:26.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:26:25.410771+0000 mon.a (mon.0) 708 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:26:26.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:26:25.615719+0000 mon.a (mon.0) 709 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:26:26.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:26:26.007951+0000 mon.a (mon.0) 710 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:26:26.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:26:26.020124+0000 mon.a (mon.0) 711 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:26:26.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:26:26.020962+0000 mon.a (mon.0) 712 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:26:26.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:26:26.021477+0000 mon.a (mon.0) 713 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:26:26.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:26:26.037302+0000 mon.a (mon.0) 714 : audit [INF] from='client.? 172.21.15.83:0/2422289855' entity='client.admin' 2022-09-29T05:26:26.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:26:25.410771+0000 mon.a (mon.0) 708 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:26:26.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:26:25.615719+0000 mon.a (mon.0) 709 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:26:26.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:26:26.007951+0000 mon.a (mon.0) 710 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:26:26.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:26:26.020124+0000 mon.a (mon.0) 711 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:26:26.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:26:26.020962+0000 mon.a (mon.0) 712 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:26:26.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:26:26.021477+0000 mon.a (mon.0) 713 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:26:26.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:26:26.037302+0000 mon.a (mon.0) 714 : audit [INF] from='client.? 172.21.15.83:0/2422289855' entity='client.admin' 2022-09-29T05:26:26.719 INFO:tasks.workunit:timeout=3h 2022-09-29T05:26:26.719 INFO:tasks.workunit:cleanup=True 2022-09-29T05:26:26.720 DEBUG:teuthology.orchestra.run.smithi083:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone https://github.com/ljflores/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 && git checkout dbaedf5bf154c5830301d16d6da9fb01a6c0c0c5 2022-09-29T05:26:26.778 INFO:tasks.workunit.client.0.smithi083.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2022-09-29T05:26:26.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:26 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:26:25.410771+0000 mon.a (mon.0) 708 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:26:26.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:26 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:26:25.615719+0000 mon.a (mon.0) 709 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:26:26.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:26 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:26:26.007951+0000 mon.a (mon.0) 710 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:26:26.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:26 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:26:26.020124+0000 mon.a (mon.0) 711 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:26:26.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:26 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:26:26.020962+0000 mon.a (mon.0) 712 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:26:26.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:26 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:26:26.021477+0000 mon.a (mon.0) 713 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:26:26.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:26 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:26:26.037302+0000 mon.a (mon.0) 714 : audit [INF] from='client.? 172.21.15.83:0/2422289855' entity='client.admin' 2022-09-29T05:26:27.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:27 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:26:25.822699+0000 mgr.y (mgr.14577) 64 : cluster [DBG] pgmap v42: 105 pgs: 7 peering, 98 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:26:27.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:27 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:26:25.822699+0000 mgr.y (mgr.14577) 64 : cluster [DBG] pgmap v42: 105 pgs: 7 peering, 98 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:26:27.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:27 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:26:25.822699+0000 mgr.y (mgr.14577) 64 : cluster [DBG] pgmap v42: 105 pgs: 7 peering, 98 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:26:28.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:26:27 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=info ts=2022-09-29T05:26:27.904Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.003062695s 2022-09-29T05:26:29.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:29 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:26:27.823410+0000 mgr.y (mgr.14577) 65 : cluster [DBG] pgmap v43: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:26:29.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:29 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:26:27.823410+0000 mgr.y (mgr.14577) 65 : cluster [DBG] pgmap v43: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:26:29.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:29 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:26:27.823410+0000 mgr.y (mgr.14577) 65 : cluster [DBG] pgmap v43: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:26:31.427 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:31 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:26:29.823689+0000 mgr.y (mgr.14577) 66 : cluster [DBG] pgmap v44: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:26:31.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:26:29.823689+0000 mgr.y (mgr.14577) 66 : cluster [DBG] pgmap v44: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:26:31.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:26:29.823689+0000 mgr.y (mgr.14577) 66 : cluster [DBG] pgmap v44: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:26:32.559 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:26:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[40232]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:26:32] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:26:32.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:26:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:26:32] "GET /metrics HTTP/1.1" 200 174569 "" "Prometheus/2.33.4" 2022-09-29T05:26:33.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:26:31.824173+0000 mgr.y (mgr.14577) 67 : cluster [DBG] pgmap v45: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:26:33.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:26:31.824173+0000 mgr.y (mgr.14577) 67 : cluster [DBG] pgmap v45: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:26:33.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:26:31.824173+0000 mgr.y (mgr.14577) 67 : cluster [DBG] pgmap v45: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:26:34.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:26:33.593631+0000 mon.a (mon.0) 715 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:26:34.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:26:33.598544+0000 mon.a (mon.0) 716 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:26:34.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:26:33.602310+0000 mon.a (mon.0) 717 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:26:34.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:26:33.603200+0000 mon.a (mon.0) 718 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:26:34.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:26:33.603778+0000 mon.a (mon.0) 719 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:26:34.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:26:33.609977+0000 mon.a (mon.0) 720 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:26:34.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:26:33.824491+0000 mgr.y (mgr.14577) 68 : cluster [DBG] pgmap v46: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:26:34.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:26:33.593631+0000 mon.a (mon.0) 715 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:26:34.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:26:33.598544+0000 mon.a (mon.0) 716 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:26:34.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:26:33.602310+0000 mon.a (mon.0) 717 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:26:34.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:26:33.603200+0000 mon.a (mon.0) 718 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:26:34.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:26:33.603778+0000 mon.a (mon.0) 719 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:26:34.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:26:33.609977+0000 mon.a (mon.0) 720 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:26:34.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:26:33.824491+0000 mgr.y (mgr.14577) 68 : cluster [DBG] pgmap v46: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:26:35.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:34 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:26:33.593631+0000 mon.a (mon.0) 715 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:26:35.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:34 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:26:33.598544+0000 mon.a (mon.0) 716 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:26:35.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:34 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:26:33.602310+0000 mon.a (mon.0) 717 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:26:35.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:34 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:26:33.603200+0000 mon.a (mon.0) 718 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:26:35.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:34 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:26:33.603778+0000 mon.a (mon.0) 719 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:26:35.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:34 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:26:33.609977+0000 mon.a (mon.0) 720 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:26:35.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:34 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:26:33.824491+0000 mgr.y (mgr.14577) 68 : cluster [DBG] pgmap v46: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:26:37.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:26:35.824789+0000 mgr.y (mgr.14577) 69 : cluster [DBG] pgmap v47: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:26:37.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:26:35.824789+0000 mgr.y (mgr.14577) 69 : cluster [DBG] pgmap v47: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:26:37.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:36 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:26:35.824789+0000 mgr.y (mgr.14577) 69 : cluster [DBG] pgmap v47: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:26:39.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:38 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:26:37.825283+0000 mgr.y (mgr.14577) 70 : cluster [DBG] pgmap v48: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:26:39.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:38 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:26:37.825283+0000 mgr.y (mgr.14577) 70 : cluster [DBG] pgmap v48: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:26:39.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:38 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:26:37.825283+0000 mgr.y (mgr.14577) 70 : cluster [DBG] pgmap v48: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:26:41.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:26:39.825591+0000 mgr.y (mgr.14577) 71 : cluster [DBG] pgmap v49: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:26:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:26:39.825591+0000 mgr.y (mgr.14577) 71 : cluster [DBG] pgmap v49: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:26:41.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:40 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:26:39.825591+0000 mgr.y (mgr.14577) 71 : cluster [DBG] pgmap v49: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:26:42.559 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:26:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[40232]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:26:42] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:26:42.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:26:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:26:42] "GET /metrics HTTP/1.1" 200 174583 "" "Prometheus/2.33.4" 2022-09-29T05:26:43.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:26:41.826203+0000 mgr.y (mgr.14577) 72 : cluster [DBG] pgmap v50: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:26:43.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:26:41.826203+0000 mgr.y (mgr.14577) 72 : cluster [DBG] pgmap v50: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:26:43.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:26:41.826203+0000 mgr.y (mgr.14577) 72 : cluster [DBG] pgmap v50: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:26:45.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:26:43.826597+0000 mgr.y (mgr.14577) 73 : cluster [DBG] pgmap v51: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:26:45.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:26:43.826597+0000 mgr.y (mgr.14577) 73 : cluster [DBG] pgmap v51: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:26:45.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:44 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:26:43.826597+0000 mgr.y (mgr.14577) 73 : cluster [DBG] pgmap v51: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:26:47.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:26:45.826989+0000 mgr.y (mgr.14577) 74 : cluster [DBG] pgmap v52: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:26:47.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:26:45.826989+0000 mgr.y (mgr.14577) 74 : cluster [DBG] pgmap v52: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:26:47.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:46 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:26:45.826989+0000 mgr.y (mgr.14577) 74 : cluster [DBG] pgmap v52: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:26:49.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:48 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:26:47.827616+0000 mgr.y (mgr.14577) 75 : cluster [DBG] pgmap v53: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:26:49.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:48 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:26:47.827616+0000 mgr.y (mgr.14577) 75 : cluster [DBG] pgmap v53: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:26:49.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:48 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:26:47.827616+0000 mgr.y (mgr.14577) 75 : cluster [DBG] pgmap v53: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:26:51.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:26:49.827990+0000 mgr.y (mgr.14577) 76 : cluster [DBG] pgmap v54: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:26:51.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:26:49.827990+0000 mgr.y (mgr.14577) 76 : cluster [DBG] pgmap v54: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:26:51.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:50 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:26:49.827990+0000 mgr.y (mgr.14577) 76 : cluster [DBG] pgmap v54: 105 pgs: 105 active+clean; 3.0 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:26:52.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:26:50.866581+0000 mon.a (mon.0) 721 : cluster [DBG] mgrmap e19: y(active, since 91s), standbys: x 2022-09-29T05:26:52.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:26:50.866581+0000 mon.a (mon.0) 721 : cluster [DBG] mgrmap e19: y(active, since 91s), standbys: x 2022-09-29T05:26:52.223 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:51 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:26:50.866581+0000 mon.a (mon.0) 721 : cluster [DBG] mgrmap e19: y(active, since 91s), standbys: x 2022-09-29T05:26:52.559 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:26:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[40232]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:26:52] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:26:52.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:26:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:26:52] "GET /metrics HTTP/1.1" 200 174579 "" "Prometheus/2.33.4" 2022-09-29T05:26:53.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:26:51.828637+0000 mgr.y (mgr.14577) 77 : cluster [DBG] pgmap v55: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2022-09-29T05:26:53.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:26:51.828637+0000 mgr.y (mgr.14577) 77 : cluster [DBG] pgmap v55: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2022-09-29T05:26:53.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:26:51.828637+0000 mgr.y (mgr.14577) 77 : cluster [DBG] pgmap v55: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2022-09-29T05:26:55.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:26:53.829008+0000 mgr.y (mgr.14577) 78 : cluster [DBG] pgmap v56: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2022-09-29T05:26:55.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:26:53.829008+0000 mgr.y (mgr.14577) 78 : cluster [DBG] pgmap v56: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2022-09-29T05:26:55.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:54 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:26:53.829008+0000 mgr.y (mgr.14577) 78 : cluster [DBG] pgmap v56: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2022-09-29T05:26:57.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:26:55.829340+0000 mgr.y (mgr.14577) 79 : cluster [DBG] pgmap v57: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2022-09-29T05:26:57.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:26:55.829340+0000 mgr.y (mgr.14577) 79 : cluster [DBG] pgmap v57: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2022-09-29T05:26:57.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:56 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:26:55.829340+0000 mgr.y (mgr.14577) 79 : cluster [DBG] pgmap v57: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2022-09-29T05:26:59.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:26:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:26:57.829960+0000 mgr.y (mgr.14577) 80 : cluster [DBG] pgmap v58: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2022-09-29T05:26:59.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:26:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:26:57.829960+0000 mgr.y (mgr.14577) 80 : cluster [DBG] pgmap v58: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2022-09-29T05:26:59.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:26:58 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:26:57.829960+0000 mgr.y (mgr.14577) 80 : cluster [DBG] pgmap v58: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2022-09-29T05:27:01.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:26:59.830335+0000 mgr.y (mgr.14577) 81 : cluster [DBG] pgmap v59: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2022-09-29T05:27:01.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:26:59.830335+0000 mgr.y (mgr.14577) 81 : cluster [DBG] pgmap v59: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2022-09-29T05:27:01.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:00 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:26:59.830335+0000 mgr.y (mgr.14577) 81 : cluster [DBG] pgmap v59: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2022-09-29T05:27:02.559 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:27:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[40232]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:27:02] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:27:02.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:27:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:27:02] "GET /metrics HTTP/1.1" 200 174579 "" "Prometheus/2.33.4" 2022-09-29T05:27:03.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:27:01.831097+0000 mgr.y (mgr.14577) 82 : cluster [DBG] pgmap v60: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2022-09-29T05:27:03.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:27:01.831097+0000 mgr.y (mgr.14577) 82 : cluster [DBG] pgmap v60: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2022-09-29T05:27:03.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:27:01.831097+0000 mgr.y (mgr.14577) 82 : cluster [DBG] pgmap v60: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2022-09-29T05:27:04.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:27:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:27:03.826Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:27:05.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:27:03.831393+0000 mgr.y (mgr.14577) 83 : cluster [DBG] pgmap v61: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:27:05.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:27:03.831393+0000 mgr.y (mgr.14577) 83 : cluster [DBG] pgmap v61: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:27:05.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:04 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:27:03.831393+0000 mgr.y (mgr.14577) 83 : cluster [DBG] pgmap v61: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:27:05.542 INFO:tasks.workunit.client.0.smithi083.stderr:Note: switching to 'dbaedf5bf154c5830301d16d6da9fb01a6c0c0c5'. 2022-09-29T05:27:05.543 INFO:tasks.workunit.client.0.smithi083.stderr: 2022-09-29T05:27:05.543 INFO:tasks.workunit.client.0.smithi083.stderr:You are in 'detached HEAD' state. You can look around, make experimental 2022-09-29T05:27:05.543 INFO:tasks.workunit.client.0.smithi083.stderr:changes and commit them, and you can discard any commits you make in this 2022-09-29T05:27:05.544 INFO:tasks.workunit.client.0.smithi083.stderr:state without impacting any branches by switching back to a branch. 2022-09-29T05:27:05.544 INFO:tasks.workunit.client.0.smithi083.stderr: 2022-09-29T05:27:05.544 INFO:tasks.workunit.client.0.smithi083.stderr:If you want to create a new branch to retain commits you create, you may 2022-09-29T05:27:05.544 INFO:tasks.workunit.client.0.smithi083.stderr:do so (now or later) by using -c with the switch command. Example: 2022-09-29T05:27:05.545 INFO:tasks.workunit.client.0.smithi083.stderr: 2022-09-29T05:27:05.545 INFO:tasks.workunit.client.0.smithi083.stderr: git switch -c 2022-09-29T05:27:05.545 INFO:tasks.workunit.client.0.smithi083.stderr: 2022-09-29T05:27:05.546 INFO:tasks.workunit.client.0.smithi083.stderr:Or undo this operation with: 2022-09-29T05:27:05.546 INFO:tasks.workunit.client.0.smithi083.stderr: 2022-09-29T05:27:05.546 INFO:tasks.workunit.client.0.smithi083.stderr: git switch - 2022-09-29T05:27:05.546 INFO:tasks.workunit.client.0.smithi083.stderr: 2022-09-29T05:27:05.547 INFO:tasks.workunit.client.0.smithi083.stderr:Turn off this advice by setting config variable advice.detachedHead to false 2022-09-29T05:27:05.547 INFO:tasks.workunit.client.0.smithi083.stderr: 2022-09-29T05:27:05.547 INFO:tasks.workunit.client.0.smithi083.stderr:HEAD is now at dbaedf5bf15 qa/suites/upgrade/telemetry-upgrade: add new telemetry upgrade tests 2022-09-29T05:27:05.549 DEBUG:teuthology.orchestra.run.smithi083:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2022-09-29T05:27:05.615 INFO:tasks.workunit.client.0.smithi083.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2022-09-29T05:27:05.616 INFO:tasks.workunit.client.0.smithi083.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2022-09-29T05:27:05.617 INFO:tasks.workunit.client.0.smithi083.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2022-09-29T05:27:06.201 INFO:tasks.workunit.client.0.smithi083.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2022-09-29T05:27:06.260 INFO:tasks.workunit.client.0.smithi083.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2022-09-29T05:27:06.296 INFO:tasks.workunit.client.0.smithi083.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2022-09-29T05:27:06.298 INFO:tasks.workunit.client.0.smithi083.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2022-09-29T05:27:06.298 INFO:tasks.workunit.client.0.smithi083.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2022-09-29T05:27:06.334 INFO:tasks.workunit.client.0.smithi083.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2022-09-29T05:27:06.337 DEBUG:teuthology.orchestra.run.smithi083:> set -ex 2022-09-29T05:27:06.338 DEBUG:teuthology.orchestra.run.smithi083:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2022-09-29T05:27:06.396 INFO:tasks.workunit:Running workunits matching test_telemetry_pacific.sh on client.0... 2022-09-29T05:27:06.397 INFO:tasks.workunit:Running workunit test_telemetry_pacific.sh... 2022-09-29T05:27:06.398 DEBUG:teuthology.orchestra.run.smithi083:workunit test test_telemetry_pacific.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=dbaedf5bf154c5830301d16d6da9fb01a6c0c0c5 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/test_telemetry_pacific.sh 2022-09-29T05:27:06.457 INFO:tasks.workunit.client.0.smithi083.stderr:+ ceph config set mgr mgr/telemetry/contact ceph-org 2022-09-29T05:27:06.836 INFO:tasks.workunit.client.0.smithi083.stderr:+ ceph config set mgr mgr/telemetry/description 'upgrade test cluster' 2022-09-29T05:27:07.107 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:27:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: [29/Sep/2022:05:27:06] ENGINE Bus STOPPING 2022-09-29T05:27:07.107 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:27:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: [29/Sep/2022:05:27:06] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2022-09-29T05:27:07.107 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:27:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: [29/Sep/2022:05:27:06] ENGINE Bus STOPPED 2022-09-29T05:27:07.107 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:27:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: [29/Sep/2022:05:27:06] ENGINE Bus STARTING 2022-09-29T05:27:07.108 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:27:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: [29/Sep/2022:05:27:06] ENGINE Serving on http://:::9283 2022-09-29T05:27:07.108 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:27:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: [29/Sep/2022:05:27:06] ENGINE Bus STARTED 2022-09-29T05:27:07.108 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:27:05.831728+0000 mgr.y (mgr.14577) 84 : cluster [DBG] pgmap v62: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:27:07.109 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:27:06.714983+0000 mon.a (mon.0) 722 : audit [INF] from='client.? ' entity='client.admin' 2022-09-29T05:27:07.109 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:27:06.731437+0000 mon.a (mon.0) 723 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:27:07.109 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:27:06.732538+0000 mon.a (mon.0) 724 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:27:07.109 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:27:06.733092+0000 mon.a (mon.0) 725 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:27:07.109 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:27:06.737173+0000 mon.a (mon.0) 726 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:27:07.110 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:27:05.831728+0000 mgr.y (mgr.14577) 84 : cluster [DBG] pgmap v62: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:27:07.110 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:27:06.714983+0000 mon.a (mon.0) 722 : audit [INF] from='client.? ' entity='client.admin' 2022-09-29T05:27:07.110 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:27:06.731437+0000 mon.a (mon.0) 723 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:27:07.110 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:27:06.732538+0000 mon.a (mon.0) 724 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:27:07.111 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:27:06.733092+0000 mon.a (mon.0) 725 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:27:07.111 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:27:06.737173+0000 mon.a (mon.0) 726 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:27:07.217 INFO:tasks.workunit.client.0.smithi083.stderr:+ ceph config set mgr mgr/telemetry/channel_ident true 2022-09-29T05:27:07.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:06 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:27:05.831728+0000 mgr.y (mgr.14577) 84 : cluster [DBG] pgmap v62: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:27:07.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:06 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:27:06.714983+0000 mon.a (mon.0) 722 : audit [INF] from='client.? ' entity='client.admin' 2022-09-29T05:27:07.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:06 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:27:06.731437+0000 mon.a (mon.0) 723 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:27:07.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:06 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:27:06.732538+0000 mon.a (mon.0) 724 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:27:07.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:06 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:27:06.733092+0000 mon.a (mon.0) 725 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:27:07.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:06 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:27:06.737173+0000 mon.a (mon.0) 726 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:27:07.463 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:27:07 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: [29/Sep/2022:05:27:07] ENGINE Bus STOPPING 2022-09-29T05:27:07.463 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:27:07 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: [29/Sep/2022:05:27:07] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2022-09-29T05:27:07.463 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:27:07 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: [29/Sep/2022:05:27:07] ENGINE Bus STOPPED 2022-09-29T05:27:07.464 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:27:07 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: [29/Sep/2022:05:27:07] ENGINE Bus STARTING 2022-09-29T05:27:07.464 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:27:07 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: [29/Sep/2022:05:27:07] ENGINE Serving on http://:::9283 2022-09-29T05:27:07.464 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:27:07 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: [29/Sep/2022:05:27:07] ENGINE Bus STARTED 2022-09-29T05:27:07.593 INFO:tasks.workunit.client.0.smithi083.stderr:+ ceph telemetry on --license sharing-1-0 2022-09-29T05:27:07.909 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:27:07 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: [29/Sep/2022:05:27:07] ENGINE Bus STOPPING 2022-09-29T05:27:07.909 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:27:07 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: [29/Sep/2022:05:27:07] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2022-09-29T05:27:07.910 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:27:07 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: [29/Sep/2022:05:27:07] ENGINE Bus STOPPED 2022-09-29T05:27:07.910 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:27:07 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: [29/Sep/2022:05:27:07] ENGINE Bus STARTING 2022-09-29T05:27:07.910 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:27:07 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: [29/Sep/2022:05:27:07] ENGINE Serving on http://:::9283 2022-09-29T05:27:07.910 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:27:07 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: [29/Sep/2022:05:27:07] ENGINE Bus STARTED 2022-09-29T05:27:08.156 INFO:tasks.workunit.client.0.smithi083.stderr:++ ceph config get mgr mgr/telemetry/last_opt_revision 2022-09-29T05:27:08.213 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:27:07 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: [29/Sep/2022:05:27:07] ENGINE Bus STOPPING 2022-09-29T05:27:08.213 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:27:07 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: [29/Sep/2022:05:27:07] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2022-09-29T05:27:08.214 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:27:07 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: [29/Sep/2022:05:27:07] ENGINE Bus STOPPED 2022-09-29T05:27:08.214 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:27:07 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: [29/Sep/2022:05:27:07] ENGINE Bus STARTING 2022-09-29T05:27:08.214 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:27:08 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: [29/Sep/2022:05:27:08] ENGINE Serving on http://:::9283 2022-09-29T05:27:08.215 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:27:08 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: [29/Sep/2022:05:27:08] ENGINE Bus STARTED 2022-09-29T05:27:08.215 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:27:08 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: [29/Sep/2022:05:27:08] ENGINE Bus STOPPING 2022-09-29T05:27:08.215 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:27:08 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: [29/Sep/2022:05:27:08] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2022-09-29T05:27:08.215 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:27:08 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: [29/Sep/2022:05:27:08] ENGINE Bus STOPPED 2022-09-29T05:27:08.216 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:27:08 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: [29/Sep/2022:05:27:08] ENGINE Bus STARTING 2022-09-29T05:27:08.216 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:27:08 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: [29/Sep/2022:05:27:08] ENGINE Serving on http://:::9283 2022-09-29T05:27:08.216 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:27:08 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: [29/Sep/2022:05:27:08] ENGINE Bus STARTED 2022-09-29T05:27:08.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:08 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:27:07.098175+0000 mon.a (mon.0) 727 : audit [INF] from='client.? ' entity='client.admin' 2022-09-29T05:27:08.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:08 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:27:07.105097+0000 mon.a (mon.0) 728 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:27:08.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:08 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:27:07.105985+0000 mon.a (mon.0) 729 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:27:08.218 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:08 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:27:07.106536+0000 mon.a (mon.0) 730 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:27:08.218 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:08 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:27:07.110760+0000 mon.a (mon.0) 731 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:27:08.218 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:08 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:27:07.474494+0000 mon.a (mon.0) 732 : audit [INF] from='client.? ' entity='client.admin' 2022-09-29T05:27:08.218 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:08 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:27:07.478225+0000 mon.a (mon.0) 733 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:27:08.219 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:08 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:27:07.479197+0000 mon.a (mon.0) 734 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:27:08.219 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:08 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:27:07.479884+0000 mon.a (mon.0) 735 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:27:08.219 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:08 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:27:07.484004+0000 mon.a (mon.0) 736 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:27:08.219 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:08 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:27:07.897476+0000 mon.a (mon.0) 737 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:27:08.220 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:08 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:27:07.901977+0000 mon.a (mon.0) 738 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:27:08.220 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:08 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:27:07.951700+0000 mon.a (mon.0) 739 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:27:08.220 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:08 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:27:07.952715+0000 mon.a (mon.0) 740 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:27:08.220 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:08 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:27:07.953268+0000 mon.a (mon.0) 741 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:27:08.220 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:08 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:27:07.957397+0000 mon.a (mon.0) 742 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:27:08.221 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:08 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:27:08.025098+0000 mon.a (mon.0) 743 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:27:08.221 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:08 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:27:08.025457+0000 mon.a (mon.0) 744 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:27:08.221 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:08 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:27:08.026597+0000 mon.a (mon.0) 745 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:27:08.221 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:08 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:27:08.027281+0000 mon.a (mon.0) 746 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:27:08.222 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:08 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:27:08.033096+0000 mon.a (mon.0) 747 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:27:08.222 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:08 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:27:08.085989+0000 mon.a (mon.0) 748 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:27:08.222 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:08 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:27:07.098175+0000 mon.a (mon.0) 727 : audit [INF] from='client.? ' entity='client.admin' 2022-09-29T05:27:08.223 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:08 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:27:07.105097+0000 mon.a (mon.0) 728 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:27:08.223 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:08 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:27:07.105985+0000 mon.a (mon.0) 729 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:27:08.223 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:08 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:27:07.106536+0000 mon.a (mon.0) 730 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:27:08.223 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:08 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:27:07.110760+0000 mon.a (mon.0) 731 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:27:08.224 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:08 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:27:07.474494+0000 mon.a (mon.0) 732 : audit [INF] from='client.? ' entity='client.admin' 2022-09-29T05:27:08.224 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:08 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:27:07.478225+0000 mon.a (mon.0) 733 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:27:08.224 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:08 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:27:07.479197+0000 mon.a (mon.0) 734 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:27:08.224 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:08 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:27:07.479884+0000 mon.a (mon.0) 735 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:27:08.225 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:08 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:27:07.484004+0000 mon.a (mon.0) 736 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:27:08.225 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:08 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:27:07.897476+0000 mon.a (mon.0) 737 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:27:08.225 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:08 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:27:07.901977+0000 mon.a (mon.0) 738 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:27:08.225 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:08 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:27:07.951700+0000 mon.a (mon.0) 739 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:27:08.226 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:08 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:27:07.952715+0000 mon.a (mon.0) 740 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:27:08.226 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:08 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:27:07.953268+0000 mon.a (mon.0) 741 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:27:08.226 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:08 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:27:07.957397+0000 mon.a (mon.0) 742 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:27:08.226 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:08 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:27:08.025098+0000 mon.a (mon.0) 743 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:27:08.227 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:08 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:27:08.025457+0000 mon.a (mon.0) 744 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:27:08.227 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:08 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:27:08.026597+0000 mon.a (mon.0) 745 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:27:08.227 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:08 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:27:08.027281+0000 mon.a (mon.0) 746 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:27:08.227 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:08 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:27:08.033096+0000 mon.a (mon.0) 747 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:27:08.228 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:08 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:27:08.085989+0000 mon.a (mon.0) 748 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:27:08.526 INFO:tasks.workunit.client.0.smithi083.stderr:+ LAST_OPT_REVISION=3 2022-09-29T05:27:08.526 INFO:tasks.workunit.client.0.smithi083.stderr:+ '[' 3 -ne 3 ']' 2022-09-29T05:27:08.526 INFO:tasks.workunit.client.0.smithi083.stderr:+ ceph telemetry show 2022-09-29T05:27:08.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:08 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:27:07.098175+0000 mon.a (mon.0) 727 : audit [INF] from='client.? ' entity='client.admin' 2022-09-29T05:27:08.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:08 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:27:07.105097+0000 mon.a (mon.0) 728 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:27:08.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:08 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:27:07.105985+0000 mon.a (mon.0) 729 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:27:08.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:08 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:27:07.106536+0000 mon.a (mon.0) 730 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:27:08.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:08 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:27:07.110760+0000 mon.a (mon.0) 731 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:27:08.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:08 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:27:07.474494+0000 mon.a (mon.0) 732 : audit [INF] from='client.? ' entity='client.admin' 2022-09-29T05:27:08.561 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:08 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:27:07.478225+0000 mon.a (mon.0) 733 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:27:08.561 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:08 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:27:07.479197+0000 mon.a (mon.0) 734 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:27:08.561 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:08 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:27:07.479884+0000 mon.a (mon.0) 735 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:27:08.561 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:08 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:27:07.484004+0000 mon.a (mon.0) 736 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:27:08.561 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:08 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:27:07.897476+0000 mon.a (mon.0) 737 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:27:08.562 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:08 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:27:07.901977+0000 mon.a (mon.0) 738 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:27:08.562 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:08 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:27:07.951700+0000 mon.a (mon.0) 739 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:27:08.562 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:08 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:27:07.952715+0000 mon.a (mon.0) 740 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:27:08.562 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:08 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:27:07.953268+0000 mon.a (mon.0) 741 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:27:08.563 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:08 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:27:07.957397+0000 mon.a (mon.0) 742 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:27:08.563 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:08 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:27:08.025098+0000 mon.a (mon.0) 743 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:27:08.563 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:08 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:27:08.025457+0000 mon.a (mon.0) 744 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:27:08.563 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:08 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:27:08.026597+0000 mon.a (mon.0) 745 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:27:08.564 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:08 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:27:08.027281+0000 mon.a (mon.0) 746 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:27:08.564 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:08 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:27:08.033096+0000 mon.a (mon.0) 747 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:27:08.564 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:08 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:27:08.085989+0000 mon.a (mon.0) 748 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:27:08.988 INFO:tasks.workunit.client.0.smithi083.stdout:{ 2022-09-29T05:27:08.988 INFO:tasks.workunit.client.0.smithi083.stdout: "balancer": { 2022-09-29T05:27:08.989 INFO:tasks.workunit.client.0.smithi083.stdout: "active": true, 2022-09-29T05:27:08.989 INFO:tasks.workunit.client.0.smithi083.stdout: "mode": "" 2022-09-29T05:27:08.989 INFO:tasks.workunit.client.0.smithi083.stdout: }, 2022-09-29T05:27:08.989 INFO:tasks.workunit.client.0.smithi083.stdout: "channels": [ 2022-09-29T05:27:08.990 INFO:tasks.workunit.client.0.smithi083.stdout: "basic", 2022-09-29T05:27:08.990 INFO:tasks.workunit.client.0.smithi083.stdout: "crash", 2022-09-29T05:27:08.990 INFO:tasks.workunit.client.0.smithi083.stdout: "device", 2022-09-29T05:27:08.990 INFO:tasks.workunit.client.0.smithi083.stdout: "ident" 2022-09-29T05:27:08.991 INFO:tasks.workunit.client.0.smithi083.stdout: ], 2022-09-29T05:27:08.991 INFO:tasks.workunit.client.0.smithi083.stdout: "channels_available": [ 2022-09-29T05:27:08.991 INFO:tasks.workunit.client.0.smithi083.stdout: "basic", 2022-09-29T05:27:08.991 INFO:tasks.workunit.client.0.smithi083.stdout: "ident", 2022-09-29T05:27:08.992 INFO:tasks.workunit.client.0.smithi083.stdout: "crash", 2022-09-29T05:27:08.992 INFO:tasks.workunit.client.0.smithi083.stdout: "device" 2022-09-29T05:27:08.992 INFO:tasks.workunit.client.0.smithi083.stdout: ], 2022-09-29T05:27:08.992 INFO:tasks.workunit.client.0.smithi083.stdout: "config": { 2022-09-29T05:27:08.993 INFO:tasks.workunit.client.0.smithi083.stdout: "active_changed": [ 2022-09-29T05:27:08.993 INFO:tasks.workunit.client.0.smithi083.stdout: "auth_allow_insecure_global_id_reclaim", 2022-09-29T05:27:08.993 INFO:tasks.workunit.client.0.smithi083.stdout: "auth_debug", 2022-09-29T05:27:08.993 INFO:tasks.workunit.client.0.smithi083.stdout: "auth_mon_ticket_ttl", 2022-09-29T05:27:08.994 INFO:tasks.workunit.client.0.smithi083.stdout: "auth_service_ticket_ttl", 2022-09-29T05:27:08.994 INFO:tasks.workunit.client.0.smithi083.stdout: "bdev_debug_aio", 2022-09-29T05:27:08.994 INFO:tasks.workunit.client.0.smithi083.stdout: "client_check_pool_perm", 2022-09-29T05:27:08.994 INFO:tasks.workunit.client.0.smithi083.stdout: "client_mount_gid", 2022-09-29T05:27:08.995 INFO:tasks.workunit.client.0.smithi083.stdout: "client_mount_uid", 2022-09-29T05:27:08.995 INFO:tasks.workunit.client.0.smithi083.stdout: "container_image", 2022-09-29T05:27:08.995 INFO:tasks.workunit.client.0.smithi083.stdout: "daemonize", 2022-09-29T05:27:08.995 INFO:tasks.workunit.client.0.smithi083.stdout: "debug_asserts_on_shutdown", 2022-09-29T05:27:08.995 INFO:tasks.workunit.client.0.smithi083.stdout: "debug_mgr", 2022-09-29T05:27:08.996 INFO:tasks.workunit.client.0.smithi083.stdout: "debug_mon", 2022-09-29T05:27:08.996 INFO:tasks.workunit.client.0.smithi083.stdout: "debug_ms", 2022-09-29T05:27:08.996 INFO:tasks.workunit.client.0.smithi083.stdout: "debug_osd", 2022-09-29T05:27:08.997 INFO:tasks.workunit.client.0.smithi083.stdout: "debug_paxos", 2022-09-29T05:27:08.997 INFO:tasks.workunit.client.0.smithi083.stdout: "keyring", 2022-09-29T05:27:08.997 INFO:tasks.workunit.client.0.smithi083.stdout: "leveldb_block_size", 2022-09-29T05:27:08.997 INFO:tasks.workunit.client.0.smithi083.stdout: "leveldb_cache_size", 2022-09-29T05:27:08.998 INFO:tasks.workunit.client.0.smithi083.stdout: "leveldb_compression", 2022-09-29T05:27:08.998 INFO:tasks.workunit.client.0.smithi083.stdout: "leveldb_log", 2022-09-29T05:27:08.998 INFO:tasks.workunit.client.0.smithi083.stdout: "leveldb_write_buffer_size", 2022-09-29T05:27:08.998 INFO:tasks.workunit.client.0.smithi083.stdout: "log_stderr_prefix", 2022-09-29T05:27:08.999 INFO:tasks.workunit.client.0.smithi083.stdout: "log_to_file", 2022-09-29T05:27:08.999 INFO:tasks.workunit.client.0.smithi083.stdout: "log_to_stderr", 2022-09-29T05:27:08.999 INFO:tasks.workunit.client.0.smithi083.stdout: "mon_allow_pool_delete", 2022-09-29T05:27:08.999 INFO:tasks.workunit.client.0.smithi083.stdout: "mon_clock_drift_allowed", 2022-09-29T05:27:08.999 INFO:tasks.workunit.client.0.smithi083.stdout: "mon_cluster_log_file_level", 2022-09-29T05:27:09.000 INFO:tasks.workunit.client.0.smithi083.stdout: "mon_cluster_log_to_file", 2022-09-29T05:27:09.000 INFO:tasks.workunit.client.0.smithi083.stdout: "mon_cluster_log_to_stderr", 2022-09-29T05:27:09.000 INFO:tasks.workunit.client.0.smithi083.stdout: "mon_data_avail_warn", 2022-09-29T05:27:09.000 INFO:tasks.workunit.client.0.smithi083.stdout: "mon_host", 2022-09-29T05:27:09.001 INFO:tasks.workunit.client.0.smithi083.stdout: "mon_max_pg_per_osd", 2022-09-29T05:27:09.001 INFO:tasks.workunit.client.0.smithi083.stdout: "mon_mgr_mkfs_grace", 2022-09-29T05:27:09.001 INFO:tasks.workunit.client.0.smithi083.stdout: "mon_osd_prime_pg_temp", 2022-09-29T05:27:09.001 INFO:tasks.workunit.client.0.smithi083.stdout: "mon_osd_reporter_subtree_level", 2022-09-29T05:27:09.001 INFO:tasks.workunit.client.0.smithi083.stdout: "mon_pg_warn_max_object_skew", 2022-09-29T05:27:09.002 INFO:tasks.workunit.client.0.smithi083.stdout: "mon_reweight_min_bytes_per_osd", 2022-09-29T05:27:09.002 INFO:tasks.workunit.client.0.smithi083.stdout: "mon_reweight_min_pgs_per_osd", 2022-09-29T05:27:09.002 INFO:tasks.workunit.client.0.smithi083.stdout: "mon_warn_on_crush_straw_calc_version_zero", 2022-09-29T05:27:09.002 INFO:tasks.workunit.client.0.smithi083.stdout: "mon_warn_on_insecure_global_id_reclaim", 2022-09-29T05:27:09.003 INFO:tasks.workunit.client.0.smithi083.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed", 2022-09-29T05:27:09.003 INFO:tasks.workunit.client.0.smithi083.stdout: "mon_warn_on_legacy_crush_tunables", 2022-09-29T05:27:09.003 INFO:tasks.workunit.client.0.smithi083.stdout: "mon_warn_on_osd_down_out_interval_zero", 2022-09-29T05:27:09.003 INFO:tasks.workunit.client.0.smithi083.stdout: "mon_warn_on_pool_pg_num_not_power_of_two", 2022-09-29T05:27:09.003 INFO:tasks.workunit.client.0.smithi083.stdout: "mon_warn_on_too_few_osds", 2022-09-29T05:27:09.004 INFO:tasks.workunit.client.0.smithi083.stdout: "ms_die_on_bug", 2022-09-29T05:27:09.004 INFO:tasks.workunit.client.0.smithi083.stdout: "ms_die_on_old_message", 2022-09-29T05:27:09.004 INFO:tasks.workunit.client.0.smithi083.stdout: "no_config_file", 2022-09-29T05:27:09.004 INFO:tasks.workunit.client.0.smithi083.stdout: "osd_class_default_list", 2022-09-29T05:27:09.005 INFO:tasks.workunit.client.0.smithi083.stdout: "osd_class_load_list", 2022-09-29T05:27:09.005 INFO:tasks.workunit.client.0.smithi083.stdout: "osd_debug_misdirected_ops", 2022-09-29T05:27:09.005 INFO:tasks.workunit.client.0.smithi083.stdout: "osd_debug_op_order", 2022-09-29T05:27:09.005 INFO:tasks.workunit.client.0.smithi083.stdout: "osd_debug_pg_log_writeout", 2022-09-29T05:27:09.006 INFO:tasks.workunit.client.0.smithi083.stdout: "osd_debug_shutdown", 2022-09-29T05:27:09.006 INFO:tasks.workunit.client.0.smithi083.stdout: "osd_debug_verify_cached_snaps", 2022-09-29T05:27:09.006 INFO:tasks.workunit.client.0.smithi083.stdout: "osd_debug_verify_missing_on_start", 2022-09-29T05:27:09.006 INFO:tasks.workunit.client.0.smithi083.stdout: "osd_debug_verify_stray_on_activate", 2022-09-29T05:27:09.006 INFO:tasks.workunit.client.0.smithi083.stdout: "osd_deep_scrub_update_digest_min_age", 2022-09-29T05:27:09.007 INFO:tasks.workunit.client.0.smithi083.stdout: "osd_mclock_scheduler_background_best_effort_lim", 2022-09-29T05:27:09.007 INFO:tasks.workunit.client.0.smithi083.stdout: "osd_mclock_scheduler_background_best_effort_res", 2022-09-29T05:27:09.007 INFO:tasks.workunit.client.0.smithi083.stdout: "osd_mclock_scheduler_background_best_effort_wgt", 2022-09-29T05:27:09.007 INFO:tasks.workunit.client.0.smithi083.stdout: "osd_mclock_scheduler_background_recovery_lim", 2022-09-29T05:27:09.008 INFO:tasks.workunit.client.0.smithi083.stdout: "osd_mclock_scheduler_background_recovery_res", 2022-09-29T05:27:09.008 INFO:tasks.workunit.client.0.smithi083.stdout: "osd_mclock_scheduler_background_recovery_wgt", 2022-09-29T05:27:09.008 INFO:tasks.workunit.client.0.smithi083.stdout: "osd_mclock_scheduler_client_lim", 2022-09-29T05:27:09.009 INFO:tasks.workunit.client.0.smithi083.stdout: "osd_mclock_scheduler_client_res", 2022-09-29T05:27:09.009 INFO:tasks.workunit.client.0.smithi083.stdout: "osd_mclock_scheduler_client_wgt", 2022-09-29T05:27:09.009 INFO:tasks.workunit.client.0.smithi083.stdout: "osd_memory_target_autotune", 2022-09-29T05:27:09.009 INFO:tasks.workunit.client.0.smithi083.stdout: "osd_op_queue", 2022-09-29T05:27:09.009 INFO:tasks.workunit.client.0.smithi083.stdout: "osd_op_queue_cut_off", 2022-09-29T05:27:09.010 INFO:tasks.workunit.client.0.smithi083.stdout: "osd_pool_default_erasure_code_profile", 2022-09-29T05:27:09.010 INFO:tasks.workunit.client.0.smithi083.stdout: "osd_pool_default_pg_autoscale_mode", 2022-09-29T05:27:09.010 INFO:tasks.workunit.client.0.smithi083.stdout: "osd_recover_clone_overlap", 2022-09-29T05:27:09.010 INFO:tasks.workunit.client.0.smithi083.stdout: "osd_recovery_max_chunk", 2022-09-29T05:27:09.011 INFO:tasks.workunit.client.0.smithi083.stdout: "osd_scrub_load_threshold", 2022-09-29T05:27:09.011 INFO:tasks.workunit.client.0.smithi083.stdout: "osd_scrub_max_interval", 2022-09-29T05:27:09.011 INFO:tasks.workunit.client.0.smithi083.stdout: "osd_shutdown_pgref_assert", 2022-09-29T05:27:09.011 INFO:tasks.workunit.client.0.smithi083.stdout: "public_addr", 2022-09-29T05:27:09.012 INFO:tasks.workunit.client.0.smithi083.stdout: "public_addrv", 2022-09-29T05:27:09.012 INFO:tasks.workunit.client.0.smithi083.stdout: "public_network", 2022-09-29T05:27:09.012 INFO:tasks.workunit.client.0.smithi083.stdout: "rbd_default_features", 2022-09-29T05:27:09.012 INFO:tasks.workunit.client.0.smithi083.stdout: "setgroup", 2022-09-29T05:27:09.013 INFO:tasks.workunit.client.0.smithi083.stdout: "setuser" 2022-09-29T05:27:09.013 INFO:tasks.workunit.client.0.smithi083.stdout: ], 2022-09-29T05:27:09.013 INFO:tasks.workunit.client.0.smithi083.stdout: "cluster_changed": [ 2022-09-29T05:27:09.013 INFO:tasks.workunit.client.0.smithi083.stdout: "auth_allow_insecure_global_id_reclaim", 2022-09-29T05:27:09.013 INFO:tasks.workunit.client.0.smithi083.stdout: "auth_debug", 2022-09-29T05:27:09.014 INFO:tasks.workunit.client.0.smithi083.stdout: "auth_mon_ticket_ttl", 2022-09-29T05:27:09.014 INFO:tasks.workunit.client.0.smithi083.stdout: "auth_service_ticket_ttl", 2022-09-29T05:27:09.014 INFO:tasks.workunit.client.0.smithi083.stdout: "bdev_debug_aio", 2022-09-29T05:27:09.014 INFO:tasks.workunit.client.0.smithi083.stdout: "container_image", 2022-09-29T05:27:09.015 INFO:tasks.workunit.client.0.smithi083.stdout: "debug_asserts_on_shutdown", 2022-09-29T05:27:09.015 INFO:tasks.workunit.client.0.smithi083.stdout: "debug_mgr", 2022-09-29T05:27:09.015 INFO:tasks.workunit.client.0.smithi083.stdout: "debug_mon", 2022-09-29T05:27:09.015 INFO:tasks.workunit.client.0.smithi083.stdout: "debug_ms", 2022-09-29T05:27:09.016 INFO:tasks.workunit.client.0.smithi083.stdout: "debug_osd", 2022-09-29T05:27:09.016 INFO:tasks.workunit.client.0.smithi083.stdout: "debug_paxos", 2022-09-29T05:27:09.016 INFO:tasks.workunit.client.0.smithi083.stdout: "log_to_file", 2022-09-29T05:27:09.016 INFO:tasks.workunit.client.0.smithi083.stdout: "log_to_stderr", 2022-09-29T05:27:09.016 INFO:tasks.workunit.client.0.smithi083.stdout: "mds_join_fs", 2022-09-29T05:27:09.017 INFO:tasks.workunit.client.0.smithi083.stdout: "mgr/cephadm/allow_ptrace", 2022-09-29T05:27:09.017 INFO:tasks.workunit.client.0.smithi083.stdout: "mgr/cephadm/container_init", 2022-09-29T05:27:09.017 INFO:tasks.workunit.client.0.smithi083.stdout: "mgr/cephadm/migration_current", 2022-09-29T05:27:09.017 INFO:tasks.workunit.client.0.smithi083.stdout: "mgr/cephadm/use_repo_digest", 2022-09-29T05:27:09.018 INFO:tasks.workunit.client.0.smithi083.stdout: "mgr/dashboard/ALERTMANAGER_API_HOST", 2022-09-29T05:27:09.018 INFO:tasks.workunit.client.0.smithi083.stdout: "mgr/dashboard/GRAFANA_API_SSL_VERIFY", 2022-09-29T05:27:09.018 INFO:tasks.workunit.client.0.smithi083.stdout: "mgr/dashboard/GRAFANA_API_URL", 2022-09-29T05:27:09.018 INFO:tasks.workunit.client.0.smithi083.stdout: "mgr/dashboard/PROMETHEUS_API_HOST", 2022-09-29T05:27:09.019 INFO:tasks.workunit.client.0.smithi083.stdout: "mgr/dashboard/ssl_server_port", 2022-09-29T05:27:09.019 INFO:tasks.workunit.client.0.smithi083.stdout: "mgr/orchestrator/orchestrator", 2022-09-29T05:27:09.019 INFO:tasks.workunit.client.0.smithi083.stdout: "mgr/telemetry/channel_ident", 2022-09-29T05:27:09.019 INFO:tasks.workunit.client.0.smithi083.stdout: "mgr/telemetry/contact", 2022-09-29T05:27:09.019 INFO:tasks.workunit.client.0.smithi083.stdout: "mgr/telemetry/description", 2022-09-29T05:27:09.020 INFO:tasks.workunit.client.0.smithi083.stdout: "mgr/telemetry/enabled", 2022-09-29T05:27:09.020 INFO:tasks.workunit.client.0.smithi083.stdout: "mgr/telemetry/last_opt_revision", 2022-09-29T05:27:09.020 INFO:tasks.workunit.client.0.smithi083.stdout: "mon_allow_pool_delete", 2022-09-29T05:27:09.020 INFO:tasks.workunit.client.0.smithi083.stdout: "mon_clock_drift_allowed", 2022-09-29T05:27:09.021 INFO:tasks.workunit.client.0.smithi083.stdout: "mon_cluster_log_file_level", 2022-09-29T05:27:09.021 INFO:tasks.workunit.client.0.smithi083.stdout: "mon_data_avail_warn", 2022-09-29T05:27:09.021 INFO:tasks.workunit.client.0.smithi083.stdout: "mon_max_pg_per_osd", 2022-09-29T05:27:09.021 INFO:tasks.workunit.client.0.smithi083.stdout: "mon_mgr_mkfs_grace", 2022-09-29T05:27:09.022 INFO:tasks.workunit.client.0.smithi083.stdout: "mon_osd_prime_pg_temp", 2022-09-29T05:27:09.022 INFO:tasks.workunit.client.0.smithi083.stdout: "mon_osd_reporter_subtree_level", 2022-09-29T05:27:09.022 INFO:tasks.workunit.client.0.smithi083.stdout: "mon_pg_warn_max_object_skew", 2022-09-29T05:27:09.022 INFO:tasks.workunit.client.0.smithi083.stdout: "mon_reweight_min_bytes_per_osd", 2022-09-29T05:27:09.022 INFO:tasks.workunit.client.0.smithi083.stdout: "mon_reweight_min_pgs_per_osd", 2022-09-29T05:27:09.023 INFO:tasks.workunit.client.0.smithi083.stdout: "mon_warn_on_crush_straw_calc_version_zero", 2022-09-29T05:27:09.023 INFO:tasks.workunit.client.0.smithi083.stdout: "mon_warn_on_insecure_global_id_reclaim", 2022-09-29T05:27:09.023 INFO:tasks.workunit.client.0.smithi083.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed", 2022-09-29T05:27:09.024 INFO:tasks.workunit.client.0.smithi083.stdout: "mon_warn_on_legacy_crush_tunables", 2022-09-29T05:27:09.024 INFO:tasks.workunit.client.0.smithi083.stdout: "mon_warn_on_osd_down_out_interval_zero", 2022-09-29T05:27:09.024 INFO:tasks.workunit.client.0.smithi083.stdout: "mon_warn_on_pool_pg_num_not_power_of_two", 2022-09-29T05:27:09.024 INFO:tasks.workunit.client.0.smithi083.stdout: "mon_warn_on_too_few_osds", 2022-09-29T05:27:09.024 INFO:tasks.workunit.client.0.smithi083.stdout: "ms_die_on_bug", 2022-09-29T05:27:09.025 INFO:tasks.workunit.client.0.smithi083.stdout: "ms_die_on_old_message", 2022-09-29T05:27:09.025 INFO:tasks.workunit.client.0.smithi083.stdout: "osd_class_default_list", 2022-09-29T05:27:09.025 INFO:tasks.workunit.client.0.smithi083.stdout: "osd_class_load_list", 2022-09-29T05:27:09.025 INFO:tasks.workunit.client.0.smithi083.stdout: "osd_debug_misdirected_ops", 2022-09-29T05:27:09.026 INFO:tasks.workunit.client.0.smithi083.stdout: "osd_debug_op_order", 2022-09-29T05:27:09.026 INFO:tasks.workunit.client.0.smithi083.stdout: "osd_debug_pg_log_writeout", 2022-09-29T05:27:09.026 INFO:tasks.workunit.client.0.smithi083.stdout: "osd_debug_shutdown", 2022-09-29T05:27:09.026 INFO:tasks.workunit.client.0.smithi083.stdout: "osd_debug_verify_cached_snaps", 2022-09-29T05:27:09.027 INFO:tasks.workunit.client.0.smithi083.stdout: "osd_debug_verify_missing_on_start", 2022-09-29T05:27:09.027 INFO:tasks.workunit.client.0.smithi083.stdout: "osd_debug_verify_stray_on_activate", 2022-09-29T05:27:09.027 INFO:tasks.workunit.client.0.smithi083.stdout: "osd_deep_scrub_update_digest_min_age", 2022-09-29T05:27:09.027 INFO:tasks.workunit.client.0.smithi083.stdout: "osd_memory_target", 2022-09-29T05:27:09.028 INFO:tasks.workunit.client.0.smithi083.stdout: "osd_memory_target_autotune", 2022-09-29T05:27:09.028 INFO:tasks.workunit.client.0.smithi083.stdout: "osd_op_queue", 2022-09-29T05:27:09.028 INFO:tasks.workunit.client.0.smithi083.stdout: "osd_op_queue_cut_off", 2022-09-29T05:27:09.028 INFO:tasks.workunit.client.0.smithi083.stdout: "osd_pool_default_erasure_code_profile", 2022-09-29T05:27:09.028 INFO:tasks.workunit.client.0.smithi083.stdout: "osd_pool_default_pg_autoscale_mode", 2022-09-29T05:27:09.029 INFO:tasks.workunit.client.0.smithi083.stdout: "osd_recover_clone_overlap", 2022-09-29T05:27:09.029 INFO:tasks.workunit.client.0.smithi083.stdout: "osd_recovery_max_chunk", 2022-09-29T05:27:09.029 INFO:tasks.workunit.client.0.smithi083.stdout: "osd_scrub_load_threshold", 2022-09-29T05:27:09.029 INFO:tasks.workunit.client.0.smithi083.stdout: "osd_scrub_max_interval", 2022-09-29T05:27:09.030 INFO:tasks.workunit.client.0.smithi083.stdout: "osd_shutdown_pgref_assert", 2022-09-29T05:27:09.030 INFO:tasks.workunit.client.0.smithi083.stdout: "public_network", 2022-09-29T05:27:09.030 INFO:tasks.workunit.client.0.smithi083.stdout: "rgw_cache_enabled", 2022-09-29T05:27:09.030 INFO:tasks.workunit.client.0.smithi083.stdout: "rgw_enable_ops_log", 2022-09-29T05:27:09.031 INFO:tasks.workunit.client.0.smithi083.stdout: "rgw_enable_usage_log" 2022-09-29T05:27:09.031 INFO:tasks.workunit.client.0.smithi083.stdout: ] 2022-09-29T05:27:09.031 INFO:tasks.workunit.client.0.smithi083.stdout: }, 2022-09-29T05:27:09.031 INFO:tasks.workunit.client.0.smithi083.stdout: "contact": "ceph-org", 2022-09-29T05:27:09.031 INFO:tasks.workunit.client.0.smithi083.stdout: "crashes": [], 2022-09-29T05:27:09.032 INFO:tasks.workunit.client.0.smithi083.stdout: "created": "2022-09-29T05:20:42.317979Z", 2022-09-29T05:27:09.032 INFO:tasks.workunit.client.0.smithi083.stdout: "crush": { 2022-09-29T05:27:09.032 INFO:tasks.workunit.client.0.smithi083.stdout: "bucket_algs": { 2022-09-29T05:27:09.032 INFO:tasks.workunit.client.0.smithi083.stdout: "straw2": 3 2022-09-29T05:27:09.033 INFO:tasks.workunit.client.0.smithi083.stdout: }, 2022-09-29T05:27:09.033 INFO:tasks.workunit.client.0.smithi083.stdout: "bucket_sizes": { 2022-09-29T05:27:09.033 INFO:tasks.workunit.client.0.smithi083.stdout: "2": 1, 2022-09-29T05:27:09.033 INFO:tasks.workunit.client.0.smithi083.stdout: "4": 2 2022-09-29T05:27:09.034 INFO:tasks.workunit.client.0.smithi083.stdout: }, 2022-09-29T05:27:09.034 INFO:tasks.workunit.client.0.smithi083.stdout: "bucket_types": { 2022-09-29T05:27:09.034 INFO:tasks.workunit.client.0.smithi083.stdout: "1": 2, 2022-09-29T05:27:09.034 INFO:tasks.workunit.client.0.smithi083.stdout: "11": 1 2022-09-29T05:27:09.035 INFO:tasks.workunit.client.0.smithi083.stdout: }, 2022-09-29T05:27:09.035 INFO:tasks.workunit.client.0.smithi083.stdout: "compat_weight_set": false, 2022-09-29T05:27:09.035 INFO:tasks.workunit.client.0.smithi083.stdout: "device_classes": [ 2022-09-29T05:27:09.035 INFO:tasks.workunit.client.0.smithi083.stdout: 8 2022-09-29T05:27:09.035 INFO:tasks.workunit.client.0.smithi083.stdout: ], 2022-09-29T05:27:09.036 INFO:tasks.workunit.client.0.smithi083.stdout: "num_buckets": 6, 2022-09-29T05:27:09.036 INFO:tasks.workunit.client.0.smithi083.stdout: "num_devices": 8, 2022-09-29T05:27:09.036 INFO:tasks.workunit.client.0.smithi083.stdout: "num_rules": 1, 2022-09-29T05:27:09.036 INFO:tasks.workunit.client.0.smithi083.stdout: "num_types": 12, 2022-09-29T05:27:09.037 INFO:tasks.workunit.client.0.smithi083.stdout: "num_weight_sets": 0, 2022-09-29T05:27:09.037 INFO:tasks.workunit.client.0.smithi083.stdout: "tunables": { 2022-09-29T05:27:09.037 INFO:tasks.workunit.client.0.smithi083.stdout: "allowed_bucket_algs": 54, 2022-09-29T05:27:09.037 INFO:tasks.workunit.client.0.smithi083.stdout: "choose_local_fallback_tries": 0, 2022-09-29T05:27:09.037 INFO:tasks.workunit.client.0.smithi083.stdout: "choose_local_tries": 0, 2022-09-29T05:27:09.038 INFO:tasks.workunit.client.0.smithi083.stdout: "choose_total_tries": 50, 2022-09-29T05:27:09.038 INFO:tasks.workunit.client.0.smithi083.stdout: "chooseleaf_descend_once": 1, 2022-09-29T05:27:09.038 INFO:tasks.workunit.client.0.smithi083.stdout: "chooseleaf_stable": 1, 2022-09-29T05:27:09.039 INFO:tasks.workunit.client.0.smithi083.stdout: "chooseleaf_vary_r": 1, 2022-09-29T05:27:09.039 INFO:tasks.workunit.client.0.smithi083.stdout: "has_v2_rules": 0, 2022-09-29T05:27:09.039 INFO:tasks.workunit.client.0.smithi083.stdout: "has_v3_rules": 0, 2022-09-29T05:27:09.039 INFO:tasks.workunit.client.0.smithi083.stdout: "has_v4_buckets": 1, 2022-09-29T05:27:09.039 INFO:tasks.workunit.client.0.smithi083.stdout: "has_v5_rules": 0, 2022-09-29T05:27:09.040 INFO:tasks.workunit.client.0.smithi083.stdout: "legacy_tunables": 0, 2022-09-29T05:27:09.040 INFO:tasks.workunit.client.0.smithi083.stdout: "minimum_required_version": "jewel", 2022-09-29T05:27:09.040 INFO:tasks.workunit.client.0.smithi083.stdout: "optimal_tunables": 1, 2022-09-29T05:27:09.040 INFO:tasks.workunit.client.0.smithi083.stdout: "profile": "jewel", 2022-09-29T05:27:09.041 INFO:tasks.workunit.client.0.smithi083.stdout: "require_feature_tunables": 1, 2022-09-29T05:27:09.041 INFO:tasks.workunit.client.0.smithi083.stdout: "require_feature_tunables2": 1, 2022-09-29T05:27:09.041 INFO:tasks.workunit.client.0.smithi083.stdout: "require_feature_tunables3": 1, 2022-09-29T05:27:09.041 INFO:tasks.workunit.client.0.smithi083.stdout: "require_feature_tunables5": 1, 2022-09-29T05:27:09.041 INFO:tasks.workunit.client.0.smithi083.stdout: "straw_calc_version": 1 2022-09-29T05:27:09.042 INFO:tasks.workunit.client.0.smithi083.stdout: } 2022-09-29T05:27:09.042 INFO:tasks.workunit.client.0.smithi083.stdout: }, 2022-09-29T05:27:09.042 INFO:tasks.workunit.client.0.smithi083.stdout: "description": "upgrade test cluster", 2022-09-29T05:27:09.042 INFO:tasks.workunit.client.0.smithi083.stdout: "fs": { 2022-09-29T05:27:09.043 INFO:tasks.workunit.client.0.smithi083.stdout: "count": 1, 2022-09-29T05:27:09.043 INFO:tasks.workunit.client.0.smithi083.stdout: "feature_flags": { 2022-09-29T05:27:09.043 INFO:tasks.workunit.client.0.smithi083.stdout: "enable_multiple": true, 2022-09-29T05:27:09.043 INFO:tasks.workunit.client.0.smithi083.stdout: "ever_enabled_multiple": true 2022-09-29T05:27:09.044 INFO:tasks.workunit.client.0.smithi083.stdout: }, 2022-09-29T05:27:09.044 INFO:tasks.workunit.client.0.smithi083.stdout: "filesystems": [ 2022-09-29T05:27:09.044 INFO:tasks.workunit.client.0.smithi083.stdout: { 2022-09-29T05:27:09.044 INFO:tasks.workunit.client.0.smithi083.stdout: "approx_ctime": "2022-09", 2022-09-29T05:27:09.045 INFO:tasks.workunit.client.0.smithi083.stdout: "balancer_enabled": false, 2022-09-29T05:27:09.045 INFO:tasks.workunit.client.0.smithi083.stdout: "bytes": 0, 2022-09-29T05:27:09.045 INFO:tasks.workunit.client.0.smithi083.stdout: "cached_caps": 0, 2022-09-29T05:27:09.045 INFO:tasks.workunit.client.0.smithi083.stdout: "cached_dns": 10, 2022-09-29T05:27:09.046 INFO:tasks.workunit.client.0.smithi083.stdout: "cached_inos": 13, 2022-09-29T05:27:09.046 INFO:tasks.workunit.client.0.smithi083.stdout: "cached_subtrees": 2, 2022-09-29T05:27:09.046 INFO:tasks.workunit.client.0.smithi083.stdout: "ever_allowed_features": 0, 2022-09-29T05:27:09.046 INFO:tasks.workunit.client.0.smithi083.stdout: "explicitly_allowed_features": 0, 2022-09-29T05:27:09.047 INFO:tasks.workunit.client.0.smithi083.stdout: "files": 0, 2022-09-29T05:27:09.047 INFO:tasks.workunit.client.0.smithi083.stdout: "max_mds": 1, 2022-09-29T05:27:09.047 INFO:tasks.workunit.client.0.smithi083.stdout: "num_data_pools": 1, 2022-09-29T05:27:09.047 INFO:tasks.workunit.client.0.smithi083.stdout: "num_in": 1, 2022-09-29T05:27:09.048 INFO:tasks.workunit.client.0.smithi083.stdout: "num_mds": 1, 2022-09-29T05:27:09.048 INFO:tasks.workunit.client.0.smithi083.stdout: "num_sessions": 0, 2022-09-29T05:27:09.048 INFO:tasks.workunit.client.0.smithi083.stdout: "num_standby_replay": 0, 2022-09-29T05:27:09.049 INFO:tasks.workunit.client.0.smithi083.stdout: "num_up": 1, 2022-09-29T05:27:09.049 INFO:tasks.workunit.client.0.smithi083.stdout: "snaps": 0, 2022-09-29T05:27:09.049 INFO:tasks.workunit.client.0.smithi083.stdout: "standby_count_wanted": 0 2022-09-29T05:27:09.049 INFO:tasks.workunit.client.0.smithi083.stdout: } 2022-09-29T05:27:09.050 INFO:tasks.workunit.client.0.smithi083.stdout: ], 2022-09-29T05:27:09.050 INFO:tasks.workunit.client.0.smithi083.stdout: "num_standby_mds": 1, 2022-09-29T05:27:09.050 INFO:tasks.workunit.client.0.smithi083.stdout: "total_num_mds": 2 2022-09-29T05:27:09.050 INFO:tasks.workunit.client.0.smithi083.stdout: }, 2022-09-29T05:27:09.051 INFO:tasks.workunit.client.0.smithi083.stdout: "hosts": { 2022-09-29T05:27:09.051 INFO:tasks.workunit.client.0.smithi083.stdout: "num": 2, 2022-09-29T05:27:09.051 INFO:tasks.workunit.client.0.smithi083.stdout: "num_with_mds": 2, 2022-09-29T05:27:09.051 INFO:tasks.workunit.client.0.smithi083.stdout: "num_with_mgr": 2, 2022-09-29T05:27:09.052 INFO:tasks.workunit.client.0.smithi083.stdout: "num_with_mon": 2, 2022-09-29T05:27:09.052 INFO:tasks.workunit.client.0.smithi083.stdout: "num_with_osd": 2 2022-09-29T05:27:09.052 INFO:tasks.workunit.client.0.smithi083.stdout: }, 2022-09-29T05:27:09.052 INFO:tasks.workunit.client.0.smithi083.stdout: "leaderboard": false, 2022-09-29T05:27:09.053 INFO:tasks.workunit.client.0.smithi083.stdout: "license": "sharing-1-0", 2022-09-29T05:27:09.053 INFO:tasks.workunit.client.0.smithi083.stdout: "metadata": { 2022-09-29T05:27:09.053 INFO:tasks.workunit.client.0.smithi083.stdout: "mon": { 2022-09-29T05:27:09.053 INFO:tasks.workunit.client.0.smithi083.stdout: "arch": { 2022-09-29T05:27:09.053 INFO:tasks.workunit.client.0.smithi083.stdout: "x86_64": 3 2022-09-29T05:27:09.054 INFO:tasks.workunit.client.0.smithi083.stdout: }, 2022-09-29T05:27:09.054 INFO:tasks.workunit.client.0.smithi083.stdout: "ceph_version": { 2022-09-29T05:27:09.054 INFO:tasks.workunit.client.0.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 3 2022-09-29T05:27:09.054 INFO:tasks.workunit.client.0.smithi083.stdout: }, 2022-09-29T05:27:09.055 INFO:tasks.workunit.client.0.smithi083.stdout: "cpu": { 2022-09-29T05:27:09.055 INFO:tasks.workunit.client.0.smithi083.stdout: "Intel(R) Xeon(R) CPU E5-1620 v4 @ 3.50GHz": 3 2022-09-29T05:27:09.055 INFO:tasks.workunit.client.0.smithi083.stdout: }, 2022-09-29T05:27:09.055 INFO:tasks.workunit.client.0.smithi083.stdout: "distro": { 2022-09-29T05:27:09.056 INFO:tasks.workunit.client.0.smithi083.stdout: "centos": 3 2022-09-29T05:27:09.056 INFO:tasks.workunit.client.0.smithi083.stdout: }, 2022-09-29T05:27:09.056 INFO:tasks.workunit.client.0.smithi083.stdout: "distro_description": { 2022-09-29T05:27:09.056 INFO:tasks.workunit.client.0.smithi083.stdout: "CentOS Stream 8": 3 2022-09-29T05:27:09.057 INFO:tasks.workunit.client.0.smithi083.stdout: }, 2022-09-29T05:27:09.057 INFO:tasks.workunit.client.0.smithi083.stdout: "kernel_description": { 2022-09-29T05:27:09.057 INFO:tasks.workunit.client.0.smithi083.stdout: "#1 SMP Mon Jul 18 17:42:52 UTC 2022": 3 2022-09-29T05:27:09.057 INFO:tasks.workunit.client.0.smithi083.stdout: }, 2022-09-29T05:27:09.057 INFO:tasks.workunit.client.0.smithi083.stdout: "kernel_version": { 2022-09-29T05:27:09.058 INFO:tasks.workunit.client.0.smithi083.stdout: "4.18.0-408.el8.x86_64": 3 2022-09-29T05:27:09.058 INFO:tasks.workunit.client.0.smithi083.stdout: }, 2022-09-29T05:27:09.058 INFO:tasks.workunit.client.0.smithi083.stdout: "os": { 2022-09-29T05:27:09.058 INFO:tasks.workunit.client.0.smithi083.stdout: "Linux": 3 2022-09-29T05:27:09.059 INFO:tasks.workunit.client.0.smithi083.stdout: } 2022-09-29T05:27:09.059 INFO:tasks.workunit.client.0.smithi083.stdout: }, 2022-09-29T05:27:09.059 INFO:tasks.workunit.client.0.smithi083.stdout: "osd": { 2022-09-29T05:27:09.059 INFO:tasks.workunit.client.0.smithi083.stdout: "arch": { 2022-09-29T05:27:09.059 INFO:tasks.workunit.client.0.smithi083.stdout: "x86_64": 8 2022-09-29T05:27:09.060 INFO:tasks.workunit.client.0.smithi083.stdout: }, 2022-09-29T05:27:09.060 INFO:tasks.workunit.client.0.smithi083.stdout: "ceph_version": { 2022-09-29T05:27:09.060 INFO:tasks.workunit.client.0.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 8 2022-09-29T05:27:09.060 INFO:tasks.workunit.client.0.smithi083.stdout: }, 2022-09-29T05:27:09.061 INFO:tasks.workunit.client.0.smithi083.stdout: "cpu": { 2022-09-29T05:27:09.061 INFO:tasks.workunit.client.0.smithi083.stdout: "Intel(R) Xeon(R) CPU E5-1620 v4 @ 3.50GHz": 8 2022-09-29T05:27:09.061 INFO:tasks.workunit.client.0.smithi083.stdout: }, 2022-09-29T05:27:09.061 INFO:tasks.workunit.client.0.smithi083.stdout: "distro": { 2022-09-29T05:27:09.062 INFO:tasks.workunit.client.0.smithi083.stdout: "centos": 8 2022-09-29T05:27:09.062 INFO:tasks.workunit.client.0.smithi083.stdout: }, 2022-09-29T05:27:09.062 INFO:tasks.workunit.client.0.smithi083.stdout: "distro_description": { 2022-09-29T05:27:09.062 INFO:tasks.workunit.client.0.smithi083.stdout: "CentOS Stream 8": 8 2022-09-29T05:27:09.063 INFO:tasks.workunit.client.0.smithi083.stdout: }, 2022-09-29T05:27:09.063 INFO:tasks.workunit.client.0.smithi083.stdout: "kernel_description": { 2022-09-29T05:27:09.063 INFO:tasks.workunit.client.0.smithi083.stdout: "#1 SMP Mon Jul 18 17:42:52 UTC 2022": 8 2022-09-29T05:27:09.063 INFO:tasks.workunit.client.0.smithi083.stdout: }, 2022-09-29T05:27:09.063 INFO:tasks.workunit.client.0.smithi083.stdout: "kernel_version": { 2022-09-29T05:27:09.064 INFO:tasks.workunit.client.0.smithi083.stdout: "4.18.0-408.el8.x86_64": 8 2022-09-29T05:27:09.064 INFO:tasks.workunit.client.0.smithi083.stdout: }, 2022-09-29T05:27:09.064 INFO:tasks.workunit.client.0.smithi083.stdout: "os": { 2022-09-29T05:27:09.064 INFO:tasks.workunit.client.0.smithi083.stdout: "Linux": 8 2022-09-29T05:27:09.065 INFO:tasks.workunit.client.0.smithi083.stdout: }, 2022-09-29T05:27:09.065 INFO:tasks.workunit.client.0.smithi083.stdout: "osd_objectstore": { 2022-09-29T05:27:09.065 INFO:tasks.workunit.client.0.smithi083.stdout: "bluestore": 8 2022-09-29T05:27:09.065 INFO:tasks.workunit.client.0.smithi083.stdout: }, 2022-09-29T05:27:09.066 INFO:tasks.workunit.client.0.smithi083.stdout: "rotational": { 2022-09-29T05:27:09.066 INFO:tasks.workunit.client.0.smithi083.stdout: "0": 8 2022-09-29T05:27:09.066 INFO:tasks.workunit.client.0.smithi083.stdout: } 2022-09-29T05:27:09.066 INFO:tasks.workunit.client.0.smithi083.stdout: } 2022-09-29T05:27:09.067 INFO:tasks.workunit.client.0.smithi083.stdout: }, 2022-09-29T05:27:09.067 INFO:tasks.workunit.client.0.smithi083.stdout: "mon": { 2022-09-29T05:27:09.067 INFO:tasks.workunit.client.0.smithi083.stdout: "count": 3, 2022-09-29T05:27:09.067 INFO:tasks.workunit.client.0.smithi083.stdout: "features": { 2022-09-29T05:27:09.068 INFO:tasks.workunit.client.0.smithi083.stdout: "optional": [], 2022-09-29T05:27:09.068 INFO:tasks.workunit.client.0.smithi083.stdout: "persistent": [ 2022-09-29T05:27:09.068 INFO:tasks.workunit.client.0.smithi083.stdout: "kraken", 2022-09-29T05:27:09.068 INFO:tasks.workunit.client.0.smithi083.stdout: "luminous", 2022-09-29T05:27:09.068 INFO:tasks.workunit.client.0.smithi083.stdout: "mimic", 2022-09-29T05:27:09.069 INFO:tasks.workunit.client.0.smithi083.stdout: "osdmap-prune", 2022-09-29T05:27:09.069 INFO:tasks.workunit.client.0.smithi083.stdout: "nautilus", 2022-09-29T05:27:09.069 INFO:tasks.workunit.client.0.smithi083.stdout: "octopus", 2022-09-29T05:27:09.069 INFO:tasks.workunit.client.0.smithi083.stdout: "pacific", 2022-09-29T05:27:09.070 INFO:tasks.workunit.client.0.smithi083.stdout: "elector-pinging" 2022-09-29T05:27:09.070 INFO:tasks.workunit.client.0.smithi083.stdout: ] 2022-09-29T05:27:09.070 INFO:tasks.workunit.client.0.smithi083.stdout: }, 2022-09-29T05:27:09.070 INFO:tasks.workunit.client.0.smithi083.stdout: "ipv4_addr_mons": 6, 2022-09-29T05:27:09.071 INFO:tasks.workunit.client.0.smithi083.stdout: "ipv6_addr_mons": 0, 2022-09-29T05:27:09.071 INFO:tasks.workunit.client.0.smithi083.stdout: "min_mon_release": 16, 2022-09-29T05:27:09.071 INFO:tasks.workunit.client.0.smithi083.stdout: "v1_addr_mons": 3, 2022-09-29T05:27:09.071 INFO:tasks.workunit.client.0.smithi083.stdout: "v2_addr_mons": 3 2022-09-29T05:27:09.071 INFO:tasks.workunit.client.0.smithi083.stdout: }, 2022-09-29T05:27:09.072 INFO:tasks.workunit.client.0.smithi083.stdout: "organization": null, 2022-09-29T05:27:09.072 INFO:tasks.workunit.client.0.smithi083.stdout: "osd": { 2022-09-29T05:27:09.072 INFO:tasks.workunit.client.0.smithi083.stdout: "cluster_network": false, 2022-09-29T05:27:09.072 INFO:tasks.workunit.client.0.smithi083.stdout: "count": 8, 2022-09-29T05:27:09.073 INFO:tasks.workunit.client.0.smithi083.stdout: "require_min_compat_client": "luminous", 2022-09-29T05:27:09.073 INFO:tasks.workunit.client.0.smithi083.stdout: "require_osd_release": "pacific" 2022-09-29T05:27:09.073 INFO:tasks.workunit.client.0.smithi083.stdout: }, 2022-09-29T05:27:09.073 INFO:tasks.workunit.client.0.smithi083.stdout: "pools": [ 2022-09-29T05:27:09.074 INFO:tasks.workunit.client.0.smithi083.stdout: { 2022-09-29T05:27:09.074 INFO:tasks.workunit.client.0.smithi083.stdout: "cache_mode": "none", 2022-09-29T05:27:09.074 INFO:tasks.workunit.client.0.smithi083.stdout: "erasure_code_profile": {}, 2022-09-29T05:27:09.074 INFO:tasks.workunit.client.0.smithi083.stdout: "min_size": 2, 2022-09-29T05:27:09.074 INFO:tasks.workunit.client.0.smithi083.stdout: "pg_autoscale_mode": "off", 2022-09-29T05:27:09.075 INFO:tasks.workunit.client.0.smithi083.stdout: "pg_num": 1, 2022-09-29T05:27:09.075 INFO:tasks.workunit.client.0.smithi083.stdout: "pgp_num": 1, 2022-09-29T05:27:09.075 INFO:tasks.workunit.client.0.smithi083.stdout: "pool": 1, 2022-09-29T05:27:09.075 INFO:tasks.workunit.client.0.smithi083.stdout: "size": 3, 2022-09-29T05:27:09.076 INFO:tasks.workunit.client.0.smithi083.stdout: "target_max_bytes": 0, 2022-09-29T05:27:09.076 INFO:tasks.workunit.client.0.smithi083.stdout: "target_max_objects": 0, 2022-09-29T05:27:09.076 INFO:tasks.workunit.client.0.smithi083.stdout: "type": "replicated" 2022-09-29T05:27:09.076 INFO:tasks.workunit.client.0.smithi083.stdout: }, 2022-09-29T05:27:09.076 INFO:tasks.workunit.client.0.smithi083.stdout: { 2022-09-29T05:27:09.077 INFO:tasks.workunit.client.0.smithi083.stdout: "cache_mode": "none", 2022-09-29T05:27:09.077 INFO:tasks.workunit.client.0.smithi083.stdout: "erasure_code_profile": {}, 2022-09-29T05:27:09.077 INFO:tasks.workunit.client.0.smithi083.stdout: "min_size": 2, 2022-09-29T05:27:09.077 INFO:tasks.workunit.client.0.smithi083.stdout: "pg_autoscale_mode": "off", 2022-09-29T05:27:09.078 INFO:tasks.workunit.client.0.smithi083.stdout: "pg_num": 32, 2022-09-29T05:27:09.078 INFO:tasks.workunit.client.0.smithi083.stdout: "pgp_num": 32, 2022-09-29T05:27:09.078 INFO:tasks.workunit.client.0.smithi083.stdout: "pool": 2, 2022-09-29T05:27:09.078 INFO:tasks.workunit.client.0.smithi083.stdout: "size": 3, 2022-09-29T05:27:09.079 INFO:tasks.workunit.client.0.smithi083.stdout: "target_max_bytes": 0, 2022-09-29T05:27:09.079 INFO:tasks.workunit.client.0.smithi083.stdout: "target_max_objects": 0, 2022-09-29T05:27:09.079 INFO:tasks.workunit.client.0.smithi083.stdout: "type": "replicated" 2022-09-29T05:27:09.079 INFO:tasks.workunit.client.0.smithi083.stdout: }, 2022-09-29T05:27:09.080 INFO:tasks.workunit.client.0.smithi083.stdout: { 2022-09-29T05:27:09.080 INFO:tasks.workunit.client.0.smithi083.stdout: "cache_mode": "none", 2022-09-29T05:27:09.080 INFO:tasks.workunit.client.0.smithi083.stdout: "erasure_code_profile": {}, 2022-09-29T05:27:09.080 INFO:tasks.workunit.client.0.smithi083.stdout: "min_size": 2, 2022-09-29T05:27:09.080 INFO:tasks.workunit.client.0.smithi083.stdout: "pg_autoscale_mode": "off", 2022-09-29T05:27:09.081 INFO:tasks.workunit.client.0.smithi083.stdout: "pg_num": 64, 2022-09-29T05:27:09.081 INFO:tasks.workunit.client.0.smithi083.stdout: "pgp_num": 64, 2022-09-29T05:27:09.081 INFO:tasks.workunit.client.0.smithi083.stdout: "pool": 3, 2022-09-29T05:27:09.081 INFO:tasks.workunit.client.0.smithi083.stdout: "size": 3, 2022-09-29T05:27:09.081 INFO:tasks.workunit.client.0.smithi083.stdout: "target_max_bytes": 0, 2022-09-29T05:27:09.082 INFO:tasks.workunit.client.0.smithi083.stdout: "target_max_objects": 0, 2022-09-29T05:27:09.082 INFO:tasks.workunit.client.0.smithi083.stdout: "type": "replicated" 2022-09-29T05:27:09.082 INFO:tasks.workunit.client.0.smithi083.stdout: }, 2022-09-29T05:27:09.082 INFO:tasks.workunit.client.0.smithi083.stdout: { 2022-09-29T05:27:09.083 INFO:tasks.workunit.client.0.smithi083.stdout: "cache_mode": "none", 2022-09-29T05:27:09.083 INFO:tasks.workunit.client.0.smithi083.stdout: "erasure_code_profile": {}, 2022-09-29T05:27:09.083 INFO:tasks.workunit.client.0.smithi083.stdout: "min_size": 2, 2022-09-29T05:27:09.083 INFO:tasks.workunit.client.0.smithi083.stdout: "pg_autoscale_mode": "off", 2022-09-29T05:27:09.084 INFO:tasks.workunit.client.0.smithi083.stdout: "pg_num": 8, 2022-09-29T05:27:09.084 INFO:tasks.workunit.client.0.smithi083.stdout: "pgp_num": 8, 2022-09-29T05:27:09.084 INFO:tasks.workunit.client.0.smithi083.stdout: "pool": 4, 2022-09-29T05:27:09.084 INFO:tasks.workunit.client.0.smithi083.stdout: "size": 3, 2022-09-29T05:27:09.084 INFO:tasks.workunit.client.0.smithi083.stdout: "target_max_bytes": 0, 2022-09-29T05:27:09.085 INFO:tasks.workunit.client.0.smithi083.stdout: "target_max_objects": 0, 2022-09-29T05:27:09.085 INFO:tasks.workunit.client.0.smithi083.stdout: "type": "replicated" 2022-09-29T05:27:09.085 INFO:tasks.workunit.client.0.smithi083.stdout: } 2022-09-29T05:27:09.085 INFO:tasks.workunit.client.0.smithi083.stdout: ], 2022-09-29T05:27:09.086 INFO:tasks.workunit.client.0.smithi083.stdout: "rbd": { 2022-09-29T05:27:09.086 INFO:tasks.workunit.client.0.smithi083.stdout: "mirroring_by_pool": [ 2022-09-29T05:27:09.086 INFO:tasks.workunit.client.0.smithi083.stdout: false 2022-09-29T05:27:09.086 INFO:tasks.workunit.client.0.smithi083.stdout: ], 2022-09-29T05:27:09.087 INFO:tasks.workunit.client.0.smithi083.stdout: "num_images_by_pool": [ 2022-09-29T05:27:09.087 INFO:tasks.workunit.client.0.smithi083.stdout: 0 2022-09-29T05:27:09.087 INFO:tasks.workunit.client.0.smithi083.stdout: ], 2022-09-29T05:27:09.087 INFO:tasks.workunit.client.0.smithi083.stdout: "num_pools": 1 2022-09-29T05:27:09.087 INFO:tasks.workunit.client.0.smithi083.stdout: }, 2022-09-29T05:27:09.088 INFO:tasks.workunit.client.0.smithi083.stdout: "report_id": "be22a2fa-d304-4f1c-99b2-5d5b78cceb39", 2022-09-29T05:27:09.088 INFO:tasks.workunit.client.0.smithi083.stdout: "report_timestamp": "2022-09-29T05:27:08.981829", 2022-09-29T05:27:09.088 INFO:tasks.workunit.client.0.smithi083.stdout: "report_version": 1, 2022-09-29T05:27:09.088 INFO:tasks.workunit.client.0.smithi083.stdout: "services": {}, 2022-09-29T05:27:09.089 INFO:tasks.workunit.client.0.smithi083.stdout: "usage": { 2022-09-29T05:27:09.089 INFO:tasks.workunit.client.0.smithi083.stdout: "pg_num": 105, 2022-09-29T05:27:09.089 INFO:tasks.workunit.client.0.smithi083.stdout: "pools": 4, 2022-09-29T05:27:09.089 INFO:tasks.workunit.client.0.smithi083.stdout: "total_avail_bytes": 767912685568, 2022-09-29T05:27:09.090 INFO:tasks.workunit.client.0.smithi083.stdout: "total_bytes": 767960285184, 2022-09-29T05:27:09.090 INFO:tasks.workunit.client.0.smithi083.stdout: "total_used_bytes": 47599616 2022-09-29T05:27:09.090 INFO:tasks.workunit.client.0.smithi083.stdout: } 2022-09-29T05:27:09.090 INFO:tasks.workunit.client.0.smithi083.stdout:} 2022-09-29T05:27:09.091 INFO:tasks.workunit.client.0.smithi083.stdout: 2022-09-29T05:27:09.091 INFO:tasks.workunit.client.0.smithi083.stdout:Device report is generated separately. To see it run 'ceph telemetry show-device'. 2022-09-29T05:27:09.092 INFO:tasks.workunit.client.0.smithi083.stderr:+ ceph telemetry show-device 2022-09-29T05:27:09.462 INFO:tasks.workunit.client.0.smithi083.stdout:{ 2022-09-29T05:27:09.463 INFO:tasks.workunit.client.0.smithi083.stdout: "5c7273b4-3fb7-11ed-a65d-0cc47a8fce91": { 2022-09-29T05:27:09.463 INFO:tasks.workunit.client.0.smithi083.stdout: "INTEL_SSDPEDMD400G4_5c7323e0-3fb7-11ed-a65d-0cc47a8fce91": {} 2022-09-29T05:27:09.463 INFO:tasks.workunit.client.0.smithi083.stdout: }, 2022-09-29T05:27:09.463 INFO:tasks.workunit.client.0.smithi083.stdout: "5c7480d2-3fb7-11ed-a65d-0cc47a8fce91": { 2022-09-29T05:27:09.464 INFO:tasks.workunit.client.0.smithi083.stdout: "INTEL_SSDPEDMD400G4_5c75139e-3fb7-11ed-a65d-0cc47a8fce91": {} 2022-09-29T05:27:09.464 INFO:tasks.workunit.client.0.smithi083.stdout: } 2022-09-29T05:27:09.465 INFO:tasks.workunit.client.0.smithi083.stdout:} 2022-09-29T05:27:09.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:09 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:27:07.832264+0000 mgr.y (mgr.14577) 85 : cluster [DBG] pgmap v63: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:27:09.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:09 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:27:07.893727+0000 mgr.y (mgr.14577) 86 : audit [DBG] from='client.14859 -' entity='client.admin' cmd=[{"prefix": "telemetry on", "license": "sharing-1-0", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:27:09.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:09 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:27:08.069592+0000 mgr.y (mgr.14577) 87 : audit [DBG] from='client.14859 -' entity='client.admin' cmd=[{"prefix": "telemetry on", "license": "sharing-1-0", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:27:09.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:09 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:27:08.096595+0000 mon.a (mon.0) 749 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:27:09.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:09 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:27:08.104939+0000 mon.a (mon.0) 750 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:27:09.467 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:09 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:27:08.109315+0000 mon.a (mon.0) 751 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:27:09.467 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:09 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:27:08.112174+0000 mon.a (mon.0) 752 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:27:09.467 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:09 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:27:08.410199+0000 mon.a (mon.0) 753 : audit [DBG] from='client.? 172.21.15.83:0/1564786043' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/telemetry/last_opt_revision"}]: dispatch 2022-09-29T05:27:09.467 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:09 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:27:08.511106+0000 mon.a (mon.0) 754 : audit [DBG] from='client.? 172.21.15.83:0/1564786043' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/telemetry/last_opt_revision"}]: dispatch 2022-09-29T05:27:09.467 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:09 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:27:08.828992+0000 mon.a (mon.0) 755 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:27:09.468 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:09 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:27:08.982696+0000 mon.a (mon.0) 756 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:27:09.468 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:09 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:27:07.832264+0000 mgr.y (mgr.14577) 85 : cluster [DBG] pgmap v63: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:27:09.468 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:09 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:27:07.893727+0000 mgr.y (mgr.14577) 86 : audit [DBG] from='client.14859 -' entity='client.admin' cmd=[{"prefix": "telemetry on", "license": "sharing-1-0", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:27:09.469 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:09 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:27:08.069592+0000 mgr.y (mgr.14577) 87 : audit [DBG] from='client.14859 -' entity='client.admin' cmd=[{"prefix": "telemetry on", "license": "sharing-1-0", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:27:09.469 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:09 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:27:08.096595+0000 mon.a (mon.0) 749 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:27:09.469 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:09 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:27:08.104939+0000 mon.a (mon.0) 750 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:27:09.469 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:09 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:27:08.109315+0000 mon.a (mon.0) 751 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:27:09.470 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:09 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:27:08.112174+0000 mon.a (mon.0) 752 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:27:09.470 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:09 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:27:08.410199+0000 mon.a (mon.0) 753 : audit [DBG] from='client.? 172.21.15.83:0/1564786043' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/telemetry/last_opt_revision"}]: dispatch 2022-09-29T05:27:09.470 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:09 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:27:08.511106+0000 mon.a (mon.0) 754 : audit [DBG] from='client.? 172.21.15.83:0/1564786043' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/telemetry/last_opt_revision"}]: dispatch 2022-09-29T05:27:09.470 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:09 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:27:08.828992+0000 mon.a (mon.0) 755 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:27:09.471 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:09 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:27:08.982696+0000 mon.a (mon.0) 756 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:27:09.474 INFO:tasks.workunit.client.0.smithi083.stderr:+ echo OK 2022-09-29T05:27:09.475 INFO:tasks.workunit.client.0.smithi083.stdout:OK 2022-09-29T05:27:09.475 INFO:teuthology.orchestra.run:Running command with timeout 3600 2022-09-29T05:27:09.476 DEBUG:teuthology.orchestra.run.smithi083:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2022-09-29T05:27:09.546 INFO:tasks.workunit:Stopping ['test_telemetry_pacific.sh'] on client.0... 2022-09-29T05:27:09.547 DEBUG:teuthology.orchestra.run.smithi083:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2022-09-29T05:27:09.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:09 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:27:07.832264+0000 mgr.y (mgr.14577) 85 : cluster [DBG] pgmap v63: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:27:09.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:09 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:27:07.893727+0000 mgr.y (mgr.14577) 86 : audit [DBG] from='client.14859 -' entity='client.admin' cmd=[{"prefix": "telemetry on", "license": "sharing-1-0", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:27:09.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:09 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:27:08.069592+0000 mgr.y (mgr.14577) 87 : audit [DBG] from='client.14859 -' entity='client.admin' cmd=[{"prefix": "telemetry on", "license": "sharing-1-0", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:27:09.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:09 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:27:08.096595+0000 mon.a (mon.0) 749 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:27:09.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:09 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:27:08.104939+0000 mon.a (mon.0) 750 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:27:09.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:09 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:27:08.109315+0000 mon.a (mon.0) 751 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:27:09.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:09 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:27:08.112174+0000 mon.a (mon.0) 752 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:27:09.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:09 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:27:08.410199+0000 mon.a (mon.0) 753 : audit [DBG] from='client.? 172.21.15.83:0/1564786043' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/telemetry/last_opt_revision"}]: dispatch 2022-09-29T05:27:09.561 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:09 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:27:08.511106+0000 mon.a (mon.0) 754 : audit [DBG] from='client.? 172.21.15.83:0/1564786043' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/telemetry/last_opt_revision"}]: dispatch 2022-09-29T05:27:09.561 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:09 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:27:08.828992+0000 mon.a (mon.0) 755 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:27:09.561 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:09 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:27:08.982696+0000 mon.a (mon.0) 756 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:27:09.826 DEBUG:teuthology.parallel:result is None 2022-09-29T05:27:09.827 DEBUG:teuthology.orchestra.run.smithi083:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2022-09-29T05:27:09.855 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2022-09-29T05:27:09.856 DEBUG:teuthology.orchestra.run.smithi083:> rmdir -- /home/ubuntu/cephtest/mnt.0 2022-09-29T05:27:09.913 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2022-09-29T05:27:09.913 INFO:teuthology.run_tasks:Running task print... 2022-09-29T05:27:09.924 INFO:teuthology.task.print:**** done end telemetry pacific... 2022-09-29T05:27:09.925 INFO:teuthology.run_tasks:Running task print... 2022-09-29T05:27:09.935 INFO:teuthology.task.print:**** done start upgrade sequence... 2022-09-29T05:27:09.935 INFO:teuthology.run_tasks:Running task sequential... 2022-09-29T05:27:09.945 INFO:teuthology.task.sequential:In sequential, running task print... 2022-09-29T05:27:09.945 INFO:teuthology.task.print:**** done start upgrade, wait 2022-09-29T05:27:09.946 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2022-09-29T05:27:09.946 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi083.front.sepia.ceph.com 2022-09-29T05:27:09.947 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/daemon-base:latest-pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 69111522-3fb6-11ed-8431-001a4aab830c -e sha1=652bf75409c45c34ae939b241b00adba20f12869 -- bash -c 'ceph config set global log_to_journald false --force' 2022-09-29T05:27:10.385 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:27:08.827868+0000 mgr.y (mgr.14577) 88 : audit [DBG] from='client.14883 -' entity='client.admin' cmd=[{"prefix": "telemetry show", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:27:10.386 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:27:08.981772+0000 mgr.y (mgr.14577) 89 : audit [DBG] from='client.14883 -' entity='client.admin' cmd=[{"prefix": "telemetry show", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:27:10.386 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:27:09.119536+0000 mon.a (mon.0) 757 : cluster [DBG] mgrmap e20: y(active, since 109s), standbys: x 2022-09-29T05:27:10.387 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:27:09.305652+0000 mgr.y (mgr.14577) 90 : audit [DBG] from='client.14889 -' entity='client.admin' cmd=[{"prefix": "telemetry show-device", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:27:10.387 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:27:09.458680+0000 mgr.y (mgr.14577) 91 : audit [DBG] from='client.14889 -' entity='client.admin' cmd=[{"prefix": "telemetry show-device", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:27:10.387 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:27:08.827868+0000 mgr.y (mgr.14577) 88 : audit [DBG] from='client.14883 -' entity='client.admin' cmd=[{"prefix": "telemetry show", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:27:10.387 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:27:08.981772+0000 mgr.y (mgr.14577) 89 : audit [DBG] from='client.14883 -' entity='client.admin' cmd=[{"prefix": "telemetry show", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:27:10.388 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:27:09.119536+0000 mon.a (mon.0) 757 : cluster [DBG] mgrmap e20: y(active, since 109s), standbys: x 2022-09-29T05:27:10.388 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:27:09.305652+0000 mgr.y (mgr.14577) 90 : audit [DBG] from='client.14889 -' entity='client.admin' cmd=[{"prefix": "telemetry show-device", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:27:10.388 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:27:09.458680+0000 mgr.y (mgr.14577) 91 : audit [DBG] from='client.14889 -' entity='client.admin' cmd=[{"prefix": "telemetry show-device", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:27:10.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:10 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:27:08.827868+0000 mgr.y (mgr.14577) 88 : audit [DBG] from='client.14883 -' entity='client.admin' cmd=[{"prefix": "telemetry show", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:27:10.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:10 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:27:08.981772+0000 mgr.y (mgr.14577) 89 : audit [DBG] from='client.14883 -' entity='client.admin' cmd=[{"prefix": "telemetry show", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:27:10.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:10 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:27:09.119536+0000 mon.a (mon.0) 757 : cluster [DBG] mgrmap e20: y(active, since 109s), standbys: x 2022-09-29T05:27:10.561 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:10 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:27:09.305652+0000 mgr.y (mgr.14577) 90 : audit [DBG] from='client.14889 -' entity='client.admin' cmd=[{"prefix": "telemetry show-device", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:27:10.561 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:10 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:27:09.458680+0000 mgr.y (mgr.14577) 91 : audit [DBG] from='client.14889 -' entity='client.admin' cmd=[{"prefix": "telemetry show-device", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:27:11.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:27:09.832566+0000 mgr.y (mgr.14577) 92 : cluster [DBG] pgmap v64: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:27:11.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:27:09.832566+0000 mgr.y (mgr.14577) 92 : cluster [DBG] pgmap v64: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:27:11.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:11 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:27:09.832566+0000 mgr.y (mgr.14577) 92 : cluster [DBG] pgmap v64: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:27:11.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:27:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: [29/Sep/2022:05:27:11] ENGINE Bus STOPPING 2022-09-29T05:27:11.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:27:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: [29/Sep/2022:05:27:11] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2022-09-29T05:27:11.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:27:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: [29/Sep/2022:05:27:11] ENGINE Bus STOPPED 2022-09-29T05:27:11.964 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:27:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: [29/Sep/2022:05:27:11] ENGINE Bus STARTING 2022-09-29T05:27:11.964 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:27:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: [29/Sep/2022:05:27:11] ENGINE Serving on http://:::9283 2022-09-29T05:27:11.964 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:27:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: [29/Sep/2022:05:27:11] ENGINE Bus STARTED 2022-09-29T05:27:12.072 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/daemon-base:latest-pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 69111522-3fb6-11ed-8431-001a4aab830c -e sha1=652bf75409c45c34ae939b241b00adba20f12869 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2022-09-29T05:27:12.559 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:27:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[40232]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:27:12] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:27:12.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:27:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:27:12] "GET /metrics HTTP/1.1" 200 174575 "" "Prometheus/2.33.4" 2022-09-29T05:27:12.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:27:11.592105+0000 mon.a (mon.0) 758 : audit [INF] from='client.? ' entity='client.admin' 2022-09-29T05:27:12.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:27:11.611744+0000 mon.a (mon.0) 759 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:27:12.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:27:11.612761+0000 mon.a (mon.0) 760 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:27:12.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:27:11.613297+0000 mon.a (mon.0) 761 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:27:12.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:27:11.636930+0000 mon.a (mon.0) 762 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:27:12.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:27:11.833107+0000 mgr.y (mgr.14577) 93 : cluster [DBG] pgmap v65: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:27:12.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:27:11.592105+0000 mon.a (mon.0) 758 : audit [INF] from='client.? ' entity='client.admin' 2022-09-29T05:27:12.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:27:11.611744+0000 mon.a (mon.0) 759 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:27:12.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:27:11.612761+0000 mon.a (mon.0) 760 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:27:12.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:27:11.613297+0000 mon.a (mon.0) 761 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:27:12.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:27:11.636930+0000 mon.a (mon.0) 762 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:27:12.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:27:11.833107+0000 mgr.y (mgr.14577) 93 : cluster [DBG] pgmap v65: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:27:13.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:27:11.592105+0000 mon.a (mon.0) 758 : audit [INF] from='client.? ' entity='client.admin' 2022-09-29T05:27:13.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:27:11.611744+0000 mon.a (mon.0) 759 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:27:13.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:27:11.612761+0000 mon.a (mon.0) 760 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:27:13.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:27:11.613297+0000 mon.a (mon.0) 761 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:27:13.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:27:11.636930+0000 mon.a (mon.0) 762 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:27:13.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:27:11.833107+0000 mgr.y (mgr.14577) 93 : cluster [DBG] pgmap v65: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:27:13.867 INFO:teuthology.orchestra.run.smithi083.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:652bf75409c45c34ae939b241b00adba20f12869 2022-09-29T05:27:13.893 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:27:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:27:13.512Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:27:14.392 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/daemon-base:latest-pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 69111522-3fb6-11ed-8431-001a4aab830c -e sha1=652bf75409c45c34ae939b241b00adba20f12869 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done' 2022-09-29T05:27:15.131 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:27:13.833427+0000 mgr.y (mgr.14577) 94 : cluster [DBG] pgmap v66: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:27:15.131 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:27:13.860546+0000 mgr.y (mgr.14577) 95 : audit [DBG] from='client.24811 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:652bf75409c45c34ae939b241b00adba20f12869", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:27:15.131 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:27:13.861202+0000 mgr.y (mgr.14577) 96 : cephadm [INF] Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:652bf75409c45c34ae939b241b00adba20f12869 2022-09-29T05:27:15.132 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:27:13.864816+0000 mon.a (mon.0) 763 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:27:15.132 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:27:13.909256+0000 mon.a (mon.0) 764 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:27:15.132 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:27:13.910230+0000 mon.a (mon.0) 765 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:27:15.132 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:27:13.910725+0000 mon.a (mon.0) 766 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:27:15.133 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:27:13.915169+0000 mon.a (mon.0) 767 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:27:15.133 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:27:13.918484+0000 mgr.y (mgr.14577) 97 : cephadm [INF] Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:652bf75409c45c34ae939b241b00adba20f12869 2022-09-29T05:27:15.133 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:27:13.833427+0000 mgr.y (mgr.14577) 94 : cluster [DBG] pgmap v66: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:27:15.134 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:27:13.860546+0000 mgr.y (mgr.14577) 95 : audit [DBG] from='client.24811 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:652bf75409c45c34ae939b241b00adba20f12869", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:27:15.134 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:27:13.861202+0000 mgr.y (mgr.14577) 96 : cephadm [INF] Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:652bf75409c45c34ae939b241b00adba20f12869 2022-09-29T05:27:15.134 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:27:13.864816+0000 mon.a (mon.0) 763 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:27:15.134 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:27:13.909256+0000 mon.a (mon.0) 764 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:27:15.135 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:27:13.910230+0000 mon.a (mon.0) 765 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:27:15.135 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:27:13.910725+0000 mon.a (mon.0) 766 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:27:15.135 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:27:13.915169+0000 mon.a (mon.0) 767 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:27:15.135 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:27:13.918484+0000 mgr.y (mgr.14577) 97 : cephadm [INF] Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:652bf75409c45c34ae939b241b00adba20f12869 2022-09-29T05:27:15.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:14 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:27:13.833427+0000 mgr.y (mgr.14577) 94 : cluster [DBG] pgmap v66: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:27:15.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:14 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:27:13.860546+0000 mgr.y (mgr.14577) 95 : audit [DBG] from='client.24811 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:652bf75409c45c34ae939b241b00adba20f12869", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:27:15.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:14 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:27:13.861202+0000 mgr.y (mgr.14577) 96 : cephadm [INF] Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:652bf75409c45c34ae939b241b00adba20f12869 2022-09-29T05:27:15.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:14 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:27:13.864816+0000 mon.a (mon.0) 763 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:27:15.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:14 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:27:13.909256+0000 mon.a (mon.0) 764 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:27:15.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:14 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:27:13.910230+0000 mon.a (mon.0) 765 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:27:15.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:14 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:27:13.910725+0000 mon.a (mon.0) 766 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:27:15.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:14 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:27:13.915169+0000 mon.a (mon.0) 767 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:27:15.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:14 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:27:13.918484+0000 mgr.y (mgr.14577) 97 : cephadm [INF] Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:652bf75409c45c34ae939b241b00adba20f12869 2022-09-29T05:27:16.332 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T05:27:16.662 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T05:27:16.662 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (59s) 51s ago 87s 15.0M - ba2b418f427c 4af0d0fa9b28 2022-09-29T05:27:16.663 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (75s) 43s ago 74s 49.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T05:27:16.663 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2m) 51s ago 2m 13.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T05:27:16.663 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2m) 43s ago 2m 16.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T05:27:16.663 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443 running (4m) 43s ago 4m 393M - 16.2.10 c56e13ff90f2 f73d53839dcb 2022-09-29T05:27:16.664 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:9283 running (6m) 51s ago 6m 446M - 16.2.10 c56e13ff90f2 41ec428786d7 2022-09-29T05:27:16.664 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6m) 51s ago 6m 99.1M 2048M 16.2.10 c56e13ff90f2 0e7e560a682f 2022-09-29T05:27:16.664 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5m) 43s ago 5m 97.4M 2048M 16.2.10 c56e13ff90f2 b96ee39cc34f 2022-09-29T05:27:16.664 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5m) 51s ago 5m 93.5M 2048M 16.2.10 c56e13ff90f2 a7179c86361b 2022-09-29T05:27:16.664 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (102s) 51s ago 101s 13.6M - 1dbe0e931976 98012608edcd 2022-09-29T05:27:16.665 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (98s) 43s ago 98s 11.4M - 1dbe0e931976 a95f214c24de 2022-09-29T05:27:16.665 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4m) 51s ago 4m 52.6M 3012M 16.2.10 c56e13ff90f2 c12a21fd5897 2022-09-29T05:27:16.665 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4m) 51s ago 4m 50.6M 3012M 16.2.10 c56e13ff90f2 989e877de30a 2022-09-29T05:27:16.665 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4m) 51s ago 4m 46.6M 3012M 16.2.10 c56e13ff90f2 81b9b51f0eee 2022-09-29T05:27:16.665 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3m) 51s ago 3m 50.7M 3012M 16.2.10 c56e13ff90f2 4863a2083356 2022-09-29T05:27:16.666 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3m) 43s ago 3m 49.2M 3268M 16.2.10 c56e13ff90f2 ed61dc9353b0 2022-09-29T05:27:16.666 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3m) 43s ago 3m 54.4M 3268M 16.2.10 c56e13ff90f2 59f19fd15fed 2022-09-29T05:27:16.666 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2m) 43s ago 2m 53.8M 3268M 16.2.10 c56e13ff90f2 23decfd9c6b2 2022-09-29T05:27:16.666 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2m) 43s ago 2m 52.0M 3268M 16.2.10 c56e13ff90f2 23460a09c0a7 2022-09-29T05:27:16.667 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (55s) 43s ago 105s 34.0M - 514e6a882f6e fb1e43a0d8ba 2022-09-29T05:27:17.066 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T05:27:17.067 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T05:27:17.067 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 3 2022-09-29T05:27:17.067 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:27:17.068 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T05:27:17.068 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T05:27:17.068 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:27:17.068 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T05:27:17.069 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 8 2022-09-29T05:27:17.069 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:27:17.069 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T05:27:17.069 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T05:27:17.070 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:27:17.070 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T05:27:17.070 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 15 2022-09-29T05:27:17.070 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T05:27:17.071 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T05:27:17.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:27:15.833758+0000 mgr.y (mgr.14577) 98 : cluster [DBG] pgmap v67: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:27:17.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:27:16.315281+0000 mgr.y (mgr.14577) 99 : audit [DBG] from='client.14907 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:27:17.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:27:15.833758+0000 mgr.y (mgr.14577) 98 : cluster [DBG] pgmap v67: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:27:17.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:27:16.315281+0000 mgr.y (mgr.14577) 99 : audit [DBG] from='client.14907 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:27:17.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:16 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:27:15.833758+0000 mgr.y (mgr.14577) 98 : cluster [DBG] pgmap v67: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:27:17.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:16 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:27:16.315281+0000 mgr.y (mgr.14577) 99 : audit [DBG] from='client.14907 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:27:18.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:17 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:27:16.649886+0000 mgr.y (mgr.14577) 100 : audit [DBG] from='client.14913 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:27:18.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:17 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:27:17.063824+0000 mon.c (mon.1) 32 : audit [DBG] from='client.? 172.21.15.83:0/4087783281' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:27:18.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:17 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:27:16.649886+0000 mgr.y (mgr.14577) 100 : audit [DBG] from='client.14913 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:27:18.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:17 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:27:17.063824+0000 mon.c (mon.1) 32 : audit [DBG] from='client.? 172.21.15.83:0/4087783281' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:27:18.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:17 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:27:16.649886+0000 mgr.y (mgr.14577) 100 : audit [DBG] from='client.14913 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:27:18.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:17 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:27:17.063824+0000 mon.c (mon.1) 32 : audit [DBG] from='client.? 172.21.15.83:0/4087783281' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:27:19.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:18 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:27:17.834249+0000 mgr.y (mgr.14577) 101 : cluster [DBG] pgmap v68: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:27:19.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:18 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:27:17.834249+0000 mgr.y (mgr.14577) 101 : cluster [DBG] pgmap v68: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:27:19.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:18 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:27:17.834249+0000 mgr.y (mgr.14577) 101 : cluster [DBG] pgmap v68: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:27:21.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:27:19.834595+0000 mgr.y (mgr.14577) 102 : cluster [DBG] pgmap v69: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:27:21.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:27:19.973014+0000 mon.a (mon.0) 768 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:27:21.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:27:19.977204+0000 mon.a (mon.0) 769 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:27:21.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:27:19.834595+0000 mgr.y (mgr.14577) 102 : cluster [DBG] pgmap v69: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:27:21.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:27:19.973014+0000 mon.a (mon.0) 768 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:27:21.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:27:19.977204+0000 mon.a (mon.0) 769 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:27:21.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:27:19.834595+0000 mgr.y (mgr.14577) 102 : cluster [DBG] pgmap v69: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:27:21.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:27:19.973014+0000 mon.a (mon.0) 768 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:27:21.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:27:19.977204+0000 mon.a (mon.0) 769 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:27:22.559 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:27:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[40232]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:27:22] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:27:22.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:27:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:27:22] "GET /metrics HTTP/1.1" 200 174589 "" "Prometheus/2.33.4" 2022-09-29T05:27:23.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:27:21.835429+0000 mgr.y (mgr.14577) 103 : cluster [DBG] pgmap v70: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:27:23.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:27:21.835429+0000 mgr.y (mgr.14577) 103 : cluster [DBG] pgmap v70: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:27:23.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:27:21.835429+0000 mgr.y (mgr.14577) 103 : cluster [DBG] pgmap v70: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:27:23.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:27:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:27:23.521Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:27:25.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:27:23.835748+0000 mgr.y (mgr.14577) 104 : cluster [DBG] pgmap v71: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:27:25.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:27:23.835748+0000 mgr.y (mgr.14577) 104 : cluster [DBG] pgmap v71: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:27:25.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:24 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:27:23.835748+0000 mgr.y (mgr.14577) 104 : cluster [DBG] pgmap v71: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:27:27.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:27:25.836133+0000 mgr.y (mgr.14577) 105 : cluster [DBG] pgmap v72: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:27:27.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:27:25.836133+0000 mgr.y (mgr.14577) 105 : cluster [DBG] pgmap v72: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:27:27.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:26 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:27:25.836133+0000 mgr.y (mgr.14577) 105 : cluster [DBG] pgmap v72: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:27:29.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:28 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:27:27.836542+0000 mgr.y (mgr.14577) 106 : cluster [DBG] pgmap v73: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:27:29.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:28 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:27:27.836542+0000 mgr.y (mgr.14577) 106 : cluster [DBG] pgmap v73: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:27:29.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:28 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:27:27.836542+0000 mgr.y (mgr.14577) 106 : cluster [DBG] pgmap v73: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:27:31.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:27:29.836845+0000 mgr.y (mgr.14577) 107 : cluster [DBG] pgmap v74: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:27:31.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:27:29.836845+0000 mgr.y (mgr.14577) 107 : cluster [DBG] pgmap v74: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:27:31.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:30 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:27:29.836845+0000 mgr.y (mgr.14577) 107 : cluster [DBG] pgmap v74: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:27:32.559 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:27:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[40232]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:27:32] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:27:32.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:27:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:27:32] "GET /metrics HTTP/1.1" 200 174589 "" "Prometheus/2.33.4" 2022-09-29T05:27:33.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:27:31.837328+0000 mgr.y (mgr.14577) 108 : cluster [DBG] pgmap v75: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:27:33.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:27:31.837328+0000 mgr.y (mgr.14577) 108 : cluster [DBG] pgmap v75: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:27:33.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:27:31.837328+0000 mgr.y (mgr.14577) 108 : cluster [DBG] pgmap v75: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:27:33.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:27:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:27:33.514Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:27:35.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:27:33.837637+0000 mgr.y (mgr.14577) 109 : cluster [DBG] pgmap v76: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:27:35.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:27:33.837637+0000 mgr.y (mgr.14577) 109 : cluster [DBG] pgmap v76: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:27:35.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:34 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:27:33.837637+0000 mgr.y (mgr.14577) 109 : cluster [DBG] pgmap v76: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:27:37.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:27:35.838068+0000 mgr.y (mgr.14577) 110 : cluster [DBG] pgmap v77: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:27:37.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:27:35.838068+0000 mgr.y (mgr.14577) 110 : cluster [DBG] pgmap v77: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:27:37.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:36 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:27:35.838068+0000 mgr.y (mgr.14577) 110 : cluster [DBG] pgmap v77: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:27:39.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:38 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:27:37.838488+0000 mgr.y (mgr.14577) 111 : cluster [DBG] pgmap v78: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:27:39.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:38 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:27:37.838488+0000 mgr.y (mgr.14577) 111 : cluster [DBG] pgmap v78: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:27:39.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:38 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:27:37.838488+0000 mgr.y (mgr.14577) 111 : cluster [DBG] pgmap v78: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:27:41.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:27:39.838805+0000 mgr.y (mgr.14577) 112 : cluster [DBG] pgmap v79: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:27:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:27:39.838805+0000 mgr.y (mgr.14577) 112 : cluster [DBG] pgmap v79: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:27:41.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:40 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:27:39.838805+0000 mgr.y (mgr.14577) 112 : cluster [DBG] pgmap v79: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:27:42.559 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:27:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[40232]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:27:42] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:27:42.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:27:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:27:42] "GET /metrics HTTP/1.1" 200 174597 "" "Prometheus/2.33.4" 2022-09-29T05:27:43.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:27:41.839321+0000 mgr.y (mgr.14577) 113 : cluster [DBG] pgmap v80: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:27:43.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:27:42.740212+0000 mon.a (mon.0) 770 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:27:43.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:27:42.741308+0000 mon.a (mon.0) 771 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:27:43.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:27:42.745345+0000 mon.a (mon.0) 772 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:27:43.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:27:41.839321+0000 mgr.y (mgr.14577) 113 : cluster [DBG] pgmap v80: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:27:43.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:27:42.740212+0000 mon.a (mon.0) 770 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:27:43.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:27:42.741308+0000 mon.a (mon.0) 771 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:27:43.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:27:42.745345+0000 mon.a (mon.0) 772 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:27:43.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:27:41.839321+0000 mgr.y (mgr.14577) 113 : cluster [DBG] pgmap v80: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:27:43.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:27:42.740212+0000 mon.a (mon.0) 770 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:27:43.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:27:42.741308+0000 mon.a (mon.0) 771 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:27:43.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:27:42.745345+0000 mon.a (mon.0) 772 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:27:43.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:27:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:27:43.521Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:27:44.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:27:42.740474+0000 mgr.y (mgr.14577) 114 : cephadm [INF] Upgrade: Target is version 18.0.0-128-g652bf754 (unknown) 2022-09-29T05:27:44.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:27:42.740540+0000 mgr.y (mgr.14577) 115 : cephadm [INF] Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:ad4728ee5a4c5589854929c922a4943dafecc88384adc5458580332074f9dff1, digests ['quay.ceph.io/ceph-ci/ceph@sha256:ad4728ee5a4c5589854929c922a4943dafecc88384adc5458580332074f9dff1'] 2022-09-29T05:27:44.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:27:42.745904+0000 mgr.y (mgr.14577) 116 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.y) 2022-09-29T05:27:44.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:27:43.105727+0000 mgr.y (mgr.14577) 117 : cephadm [INF] Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:ad4728ee5a4c5589854929c922a4943dafecc88384adc5458580332074f9dff1 on smithi153 2022-09-29T05:27:44.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:27:42.740474+0000 mgr.y (mgr.14577) 114 : cephadm [INF] Upgrade: Target is version 18.0.0-128-g652bf754 (unknown) 2022-09-29T05:27:44.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:27:42.740540+0000 mgr.y (mgr.14577) 115 : cephadm [INF] Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:ad4728ee5a4c5589854929c922a4943dafecc88384adc5458580332074f9dff1, digests ['quay.ceph.io/ceph-ci/ceph@sha256:ad4728ee5a4c5589854929c922a4943dafecc88384adc5458580332074f9dff1'] 2022-09-29T05:27:44.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:27:42.745904+0000 mgr.y (mgr.14577) 116 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.y) 2022-09-29T05:27:44.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:27:43.105727+0000 mgr.y (mgr.14577) 117 : cephadm [INF] Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:ad4728ee5a4c5589854929c922a4943dafecc88384adc5458580332074f9dff1 on smithi153 2022-09-29T05:27:44.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:27:42.740474+0000 mgr.y (mgr.14577) 114 : cephadm [INF] Upgrade: Target is version 18.0.0-128-g652bf754 (unknown) 2022-09-29T05:27:44.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:27:42.740540+0000 mgr.y (mgr.14577) 115 : cephadm [INF] Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:ad4728ee5a4c5589854929c922a4943dafecc88384adc5458580332074f9dff1, digests ['quay.ceph.io/ceph-ci/ceph@sha256:ad4728ee5a4c5589854929c922a4943dafecc88384adc5458580332074f9dff1'] 2022-09-29T05:27:44.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:27:42.745904+0000 mgr.y (mgr.14577) 116 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.y) 2022-09-29T05:27:44.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:27:43.105727+0000 mgr.y (mgr.14577) 117 : cephadm [INF] Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:ad4728ee5a4c5589854929c922a4943dafecc88384adc5458580332074f9dff1 on smithi153 2022-09-29T05:27:45.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:27:43.839748+0000 mgr.y (mgr.14577) 118 : cluster [DBG] pgmap v81: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:27:45.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:27:43.839748+0000 mgr.y (mgr.14577) 118 : cluster [DBG] pgmap v81: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:27:45.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:44 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:27:43.839748+0000 mgr.y (mgr.14577) 118 : cluster [DBG] pgmap v81: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:27:47.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:27:45.840264+0000 mgr.y (mgr.14577) 119 : cluster [DBG] pgmap v82: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:27:47.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:27:45.840264+0000 mgr.y (mgr.14577) 119 : cluster [DBG] pgmap v82: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:27:47.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:46 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:27:45.840264+0000 mgr.y (mgr.14577) 119 : cluster [DBG] pgmap v82: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:27:47.372 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T05:27:47.641 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T05:27:47.641 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (90s) 82s ago 118s 15.0M - ba2b418f427c 4af0d0fa9b28 2022-09-29T05:27:47.642 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (106s) 74s ago 105s 49.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T05:27:47.642 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2m) 82s ago 2m 13.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T05:27:47.643 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2m) 74s ago 2m 16.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T05:27:47.643 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443 running (5m) 74s ago 5m 393M - 16.2.10 c56e13ff90f2 f73d53839dcb 2022-09-29T05:27:47.644 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:9283 running (6m) 82s ago 6m 446M - 16.2.10 c56e13ff90f2 41ec428786d7 2022-09-29T05:27:47.644 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6m) 82s ago 7m 99.1M 2048M 16.2.10 c56e13ff90f2 0e7e560a682f 2022-09-29T05:27:47.645 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5m) 74s ago 5m 97.4M 2048M 16.2.10 c56e13ff90f2 b96ee39cc34f 2022-09-29T05:27:47.645 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5m) 82s ago 5m 93.5M 2048M 16.2.10 c56e13ff90f2 a7179c86361b 2022-09-29T05:27:47.646 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2m) 82s ago 2m 13.6M - 1dbe0e931976 98012608edcd 2022-09-29T05:27:47.646 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2m) 74s ago 2m 11.4M - 1dbe0e931976 a95f214c24de 2022-09-29T05:27:47.647 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5m) 82s ago 5m 52.6M 3012M 16.2.10 c56e13ff90f2 c12a21fd5897 2022-09-29T05:27:47.647 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4m) 82s ago 4m 50.6M 3012M 16.2.10 c56e13ff90f2 989e877de30a 2022-09-29T05:27:47.647 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4m) 82s ago 4m 46.6M 3012M 16.2.10 c56e13ff90f2 81b9b51f0eee 2022-09-29T05:27:47.648 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4m) 82s ago 4m 50.7M 3012M 16.2.10 c56e13ff90f2 4863a2083356 2022-09-29T05:27:47.648 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3m) 74s ago 3m 49.2M 3268M 16.2.10 c56e13ff90f2 ed61dc9353b0 2022-09-29T05:27:47.649 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3m) 74s ago 3m 54.4M 3268M 16.2.10 c56e13ff90f2 59f19fd15fed 2022-09-29T05:27:47.649 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3m) 74s ago 3m 53.8M 3268M 16.2.10 c56e13ff90f2 23decfd9c6b2 2022-09-29T05:27:47.650 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3m) 74s ago 3m 52.0M 3268M 16.2.10 c56e13ff90f2 23460a09c0a7 2022-09-29T05:27:47.651 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (86s) 74s ago 2m 34.0M - 514e6a882f6e fb1e43a0d8ba 2022-09-29T05:27:47.967 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T05:27:47.967 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T05:27:47.968 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 3 2022-09-29T05:27:47.968 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:27:47.968 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T05:27:47.969 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T05:27:47.969 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:27:47.970 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T05:27:47.970 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 8 2022-09-29T05:27:47.971 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:27:47.971 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T05:27:47.971 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T05:27:47.971 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:27:47.972 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T05:27:47.972 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 15 2022-09-29T05:27:47.972 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T05:27:47.972 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T05:27:48.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:47 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:27:47.356608+0000 mgr.y (mgr.14577) 120 : audit [DBG] from='client.24835 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:27:48.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:47 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:27:47.356608+0000 mgr.y (mgr.14577) 120 : audit [DBG] from='client.24835 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:27:48.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:47 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:27:47.356608+0000 mgr.y (mgr.14577) 120 : audit [DBG] from='client.24835 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:27:49.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:48 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:27:47.630430+0000 mgr.y (mgr.14577) 121 : audit [DBG] from='client.24841 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:27:49.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:48 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:27:47.840714+0000 mgr.y (mgr.14577) 122 : cluster [DBG] pgmap v83: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:27:49.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:48 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:27:47.964022+0000 mon.c (mon.1) 33 : audit [DBG] from='client.? 172.21.15.83:0/2123160222' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:27:49.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:48 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:27:47.630430+0000 mgr.y (mgr.14577) 121 : audit [DBG] from='client.24841 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:27:49.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:48 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:27:47.840714+0000 mgr.y (mgr.14577) 122 : cluster [DBG] pgmap v83: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:27:49.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:48 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:27:47.964022+0000 mon.c (mon.1) 33 : audit [DBG] from='client.? 172.21.15.83:0/2123160222' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:27:49.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:48 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:27:47.630430+0000 mgr.y (mgr.14577) 121 : audit [DBG] from='client.24841 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:27:49.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:48 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:27:47.840714+0000 mgr.y (mgr.14577) 122 : cluster [DBG] pgmap v83: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:27:49.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:48 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:27:47.964022+0000 mon.c (mon.1) 33 : audit [DBG] from='client.? 172.21.15.83:0/2123160222' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:27:51.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:27:49.841156+0000 mgr.y (mgr.14577) 123 : cluster [DBG] pgmap v84: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:27:51.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:27:49.841156+0000 mgr.y (mgr.14577) 123 : cluster [DBG] pgmap v84: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:27:51.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:50 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:27:49.841156+0000 mgr.y (mgr.14577) 123 : cluster [DBG] pgmap v84: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:27:52.559 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:27:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[40232]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:27:52] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:27:52.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:27:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:27:52] "GET /metrics HTTP/1.1" 200 174595 "" "Prometheus/2.33.4" 2022-09-29T05:27:53.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:27:51.841733+0000 mgr.y (mgr.14577) 124 : cluster [DBG] pgmap v85: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:27:53.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:27:51.841733+0000 mgr.y (mgr.14577) 124 : cluster [DBG] pgmap v85: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:27:53.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:27:51.841733+0000 mgr.y (mgr.14577) 124 : cluster [DBG] pgmap v85: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:27:53.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:27:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:27:53.519Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:27:55.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:27:53.842174+0000 mgr.y (mgr.14577) 125 : cluster [DBG] pgmap v86: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:27:55.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:27:53.842174+0000 mgr.y (mgr.14577) 125 : cluster [DBG] pgmap v86: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:27:55.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:54 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:27:53.842174+0000 mgr.y (mgr.14577) 125 : cluster [DBG] pgmap v86: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:27:57.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:27:55.842881+0000 mgr.y (mgr.14577) 126 : cluster [DBG] pgmap v87: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:27:57.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:27:55.842881+0000 mgr.y (mgr.14577) 126 : cluster [DBG] pgmap v87: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:27:57.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:56 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:27:55.842881+0000 mgr.y (mgr.14577) 126 : cluster [DBG] pgmap v87: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:27:59.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:27:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:27:57.843463+0000 mgr.y (mgr.14577) 127 : cluster [DBG] pgmap v88: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:27:59.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:27:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:27:57.843463+0000 mgr.y (mgr.14577) 127 : cluster [DBG] pgmap v88: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:27:59.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:27:58 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:27:57.843463+0000 mgr.y (mgr.14577) 127 : cluster [DBG] pgmap v88: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:28:01.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:27:59.843875+0000 mgr.y (mgr.14577) 128 : cluster [DBG] pgmap v89: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:28:01.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:27:59.843875+0000 mgr.y (mgr.14577) 128 : cluster [DBG] pgmap v89: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:28:01.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:00 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:27:59.843875+0000 mgr.y (mgr.14577) 128 : cluster [DBG] pgmap v89: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:28:02.559 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:28:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[40232]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:28:02] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:28:02.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:28:02] "GET /metrics HTTP/1.1" 200 174595 "" "Prometheus/2.33.4" 2022-09-29T05:28:03.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:28:01.844717+0000 mgr.y (mgr.14577) 129 : cluster [DBG] pgmap v90: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:28:03.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:28:01.844717+0000 mgr.y (mgr.14577) 129 : cluster [DBG] pgmap v90: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:28:03.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:28:01.844717+0000 mgr.y (mgr.14577) 129 : cluster [DBG] pgmap v90: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:28:03.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:28:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:28:03.517Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:28:05.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:04 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:28:03.845269+0000 mgr.y (mgr.14577) 130 : cluster [DBG] pgmap v91: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:28:05.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:28:03.845269+0000 mgr.y (mgr.14577) 130 : cluster [DBG] pgmap v91: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:28:05.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:28:03.845269+0000 mgr.y (mgr.14577) 130 : cluster [DBG] pgmap v91: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:28:07.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:06 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:28:05.845740+0000 mgr.y (mgr.14577) 131 : cluster [DBG] pgmap v92: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:28:07.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:28:05.845740+0000 mgr.y (mgr.14577) 131 : cluster [DBG] pgmap v92: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:28:07.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:28:05.845740+0000 mgr.y (mgr.14577) 131 : cluster [DBG] pgmap v92: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:28:09.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:08 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:28:07.846259+0000 mgr.y (mgr.14577) 132 : cluster [DBG] pgmap v93: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:28:09.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:08 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:28:07.846259+0000 mgr.y (mgr.14577) 132 : cluster [DBG] pgmap v93: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:28:09.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:08 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:28:07.846259+0000 mgr.y (mgr.14577) 132 : cluster [DBG] pgmap v93: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:28:11.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:10 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:28:09.846627+0000 mgr.y (mgr.14577) 133 : cluster [DBG] pgmap v94: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:28:11.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:28:09.846627+0000 mgr.y (mgr.14577) 133 : cluster [DBG] pgmap v94: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:28:11.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:28:09.846627+0000 mgr.y (mgr.14577) 133 : cluster [DBG] pgmap v94: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:28:12.158 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:28:11.052863+0000 mgr.y (mgr.14577) 134 : cephadm [INF] Upgrade: Updating mgr.x 2022-09-29T05:28:12.158 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:11.057736+0000 mon.a (mon.0) 773 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:28:12.158 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:11.058763+0000 mon.a (mon.0) 774 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2022-09-29T05:28:12.159 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:11.059652+0000 mon.a (mon.0) 775 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2022-09-29T05:28:12.159 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:11.060547+0000 mon.a (mon.0) 776 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:28:12.159 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:28:11.061243+0000 mgr.y (mgr.14577) 135 : cephadm [INF] Deploying daemon mgr.x on smithi153 2022-09-29T05:28:12.416 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:28:11.052863+0000 mgr.y (mgr.14577) 134 : cephadm [INF] Upgrade: Updating mgr.x 2022-09-29T05:28:12.416 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:11.057736+0000 mon.a (mon.0) 773 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:28:12.416 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:11.058763+0000 mon.a (mon.0) 774 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2022-09-29T05:28:12.417 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:11.059652+0000 mon.a (mon.0) 775 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2022-09-29T05:28:12.417 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:11.060547+0000 mon.a (mon.0) 776 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:28:12.417 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:28:11.061243+0000 mgr.y (mgr.14577) 135 : cephadm [INF] Deploying daemon mgr.x on smithi153 2022-09-29T05:28:12.418 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:28:11.052863+0000 mgr.y (mgr.14577) 134 : cephadm [INF] Upgrade: Updating mgr.x 2022-09-29T05:28:12.418 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:11.057736+0000 mon.a (mon.0) 773 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:28:12.418 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:11.058763+0000 mon.a (mon.0) 774 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2022-09-29T05:28:12.418 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:11.059652+0000 mon.a (mon.0) 775 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2022-09-29T05:28:12.419 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:11.060547+0000 mon.a (mon.0) 776 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:28:12.419 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:28:11.061243+0000 mgr.y (mgr.14577) 135 : cephadm [INF] Deploying daemon mgr.x on smithi153 2022-09-29T05:28:12.537 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:28:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[40232]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:28:12] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:28:12.538 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:28:12 smithi153 systemd[1]: Stopping Ceph mgr.x for 69111522-3fb6-11ed-8431-001a4aab830c... 2022-09-29T05:28:12.538 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:28:12 smithi153 bash[58553]: Error: no container with name or ID "ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr.x" found: no such container 2022-09-29T05:28:12.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:28:12] "GET /metrics HTTP/1.1" 200 174600 "" "Prometheus/2.33.4" 2022-09-29T05:28:12.791 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:28:12 smithi153 bash[58568]: ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x 2022-09-29T05:28:12.791 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:28:12 smithi153 bash[58609]: Error: no container with name or ID "ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr.x" found: no such container 2022-09-29T05:28:13.059 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:28:12 smithi153 systemd[1]: ceph-69111522-3fb6-11ed-8431-001a4aab830c@mgr.x.service: Main process exited, code=exited, status=143/n/a 2022-09-29T05:28:13.059 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:28:12 smithi153 systemd[1]: ceph-69111522-3fb6-11ed-8431-001a4aab830c@mgr.x.service: Failed with result 'exit-code'. 2022-09-29T05:28:13.059 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:28:12 smithi153 systemd[1]: Stopped Ceph mgr.x for 69111522-3fb6-11ed-8431-001a4aab830c. 2022-09-29T05:28:13.060 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:28:12 smithi153 systemd[1]: Starting Ceph mgr.x for 69111522-3fb6-11ed-8431-001a4aab830c... 2022-09-29T05:28:13.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:28:11.847311+0000 mgr.y (mgr.14577) 136 : cluster [DBG] pgmap v95: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:28:13.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:28:11.847311+0000 mgr.y (mgr.14577) 136 : cluster [DBG] pgmap v95: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:28:13.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:28:11.847311+0000 mgr.y (mgr.14577) 136 : cluster [DBG] pgmap v95: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:28:13.877 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:28:13 smithi153 podman[58712]: 2022-09-29T05:28:13.877 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:28:13 smithi153 bash[58712]: 1b5d2346e3217b04e411b93e3e7ddb068bb95a3097d714f5478950e9b5f8cf4c 2022-09-29T05:28:13.878 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:28:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: debug 2022-09-29T05:28:13.806+0000 7f833fd39700 1 -- 172.21.15.153:0/1388721739 <== mon.2 v2:172.21.15.153:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55df80280820 con 0x55df80254800 2022-09-29T05:28:13.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:28:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:28:13.518Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:28:14.232 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:28:13 smithi153 systemd[1]: Started Ceph mgr.x for 69111522-3fb6-11ed-8431-001a4aab830c. 2022-09-29T05:28:14.232 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:28:14 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: debug 2022-09-29T05:28:14.024+0000 7f834b34f200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2022-09-29T05:28:14.559 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:28:14 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: debug 2022-09-29T05:28:14.228+0000 7f834b34f200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2022-09-29T05:28:15.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:28:13.847886+0000 mgr.y (mgr.14577) 137 : cluster [DBG] pgmap v96: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:28:15.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:13.929772+0000 mon.a (mon.0) 777 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:28:15.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:13.930624+0000 mon.a (mon.0) 778 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:28:15.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:13.931846+0000 mon.a (mon.0) 779 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:28:15.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:13.932718+0000 mon.a (mon.0) 780 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:28:15.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:28:13.847886+0000 mgr.y (mgr.14577) 137 : cluster [DBG] pgmap v96: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:28:15.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:13.929772+0000 mon.a (mon.0) 777 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:28:15.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:13.930624+0000 mon.a (mon.0) 778 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:28:15.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:13.931846+0000 mon.a (mon.0) 779 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:28:15.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:13.932718+0000 mon.a (mon.0) 780 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:28:15.218 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:28:15 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: debug 2022-09-29T05:28:15.034+0000 7f834b34f200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2022-09-29T05:28:15.219 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:14 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:28:13.847886+0000 mgr.y (mgr.14577) 137 : cluster [DBG] pgmap v96: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:28:15.219 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:14 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:13.929772+0000 mon.a (mon.0) 777 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:28:15.219 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:14 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:13.930624+0000 mon.a (mon.0) 778 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:28:15.219 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:14 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:13.931846+0000 mon.a (mon.0) 779 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:28:15.220 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:14 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:13.932718+0000 mon.a (mon.0) 780 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:28:15.559 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:28:15 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: debug 2022-09-29T05:28:15.214+0000 7f834b34f200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2022-09-29T05:28:15.559 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:28:15 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: debug 2022-09-29T05:28:15.445+0000 7f834b34f200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2022-09-29T05:28:15.971 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:28:15 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: debug 2022-09-29T05:28:15.618+0000 7f834b34f200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2022-09-29T05:28:15.971 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:28:15 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: debug 2022-09-29T05:28:15.967+0000 7f834b34f200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2022-09-29T05:28:16.559 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:28:16 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: debug 2022-09-29T05:28:16.290+0000 7f834b34f200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2022-09-29T05:28:16.833 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:28:16 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: debug 2022-09-29T05:28:16.556+0000 7f834b34f200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2022-09-29T05:28:16.834 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:28:16 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: debug 2022-09-29T05:28:16.649+0000 7f834b34f200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2022-09-29T05:28:17.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:28:15.848441+0000 mgr.y (mgr.14577) 138 : cluster [DBG] pgmap v97: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:28:17.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:28:15.848441+0000 mgr.y (mgr.14577) 138 : cluster [DBG] pgmap v97: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:28:17.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:16 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:28:15.848441+0000 mgr.y (mgr.14577) 138 : cluster [DBG] pgmap v97: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:28:17.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:28:16 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: debug 2022-09-29T05:28:16.830+0000 7f834b34f200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2022-09-29T05:28:17.310 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:28:16 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: debug 2022-09-29T05:28:16.918+0000 7f834b34f200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2022-09-29T05:28:17.751 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:28:17 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: debug 2022-09-29T05:28:17.345+0000 7f834b34f200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2022-09-29T05:28:17.752 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:28:17 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: debug 2022-09-29T05:28:17.436+0000 7f834b34f200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2022-09-29T05:28:17.752 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:28:17 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: debug 2022-09-29T05:28:17.529+0000 7f834b34f200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2022-09-29T05:28:18.059 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:28:17 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: debug 2022-09-29T05:28:17.748+0000 7f834b34f200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2022-09-29T05:28:18.270 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T05:28:18.539 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T05:28:18.539 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2m) 112s ago 2m 15.0M - ba2b418f427c 4af0d0fa9b28 2022-09-29T05:28:18.539 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2m) 0s ago 2m 60.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T05:28:18.540 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3m) 112s ago 3m 13.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T05:28:18.540 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3m) 0s ago 3m 16.8M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T05:28:18.540 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5s) 0s ago 5m 52.9M - 18.0.0-128-g652bf754 6d6412b0c71e 1b5d2346e321 2022-09-29T05:28:18.540 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:9283 running (7m) 112s ago 7m 446M - 16.2.10 c56e13ff90f2 41ec428786d7 2022-09-29T05:28:18.541 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7m) 112s ago 7m 99.1M 2048M 16.2.10 c56e13ff90f2 0e7e560a682f 2022-09-29T05:28:18.541 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6m) 0s ago 6m 72.8M 2048M 16.2.10 c56e13ff90f2 b96ee39cc34f 2022-09-29T05:28:18.541 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6m) 112s ago 6m 93.5M 2048M 16.2.10 c56e13ff90f2 a7179c86361b 2022-09-29T05:28:18.541 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2m) 112s ago 2m 13.6M - 1dbe0e931976 98012608edcd 2022-09-29T05:28:18.542 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2m) 0s ago 2m 24.3M - 1dbe0e931976 a95f214c24de 2022-09-29T05:28:18.542 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5m) 112s ago 5m 52.6M 3012M 16.2.10 c56e13ff90f2 c12a21fd5897 2022-09-29T05:28:18.542 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5m) 112s ago 5m 50.6M 3012M 16.2.10 c56e13ff90f2 989e877de30a 2022-09-29T05:28:18.543 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5m) 112s ago 5m 46.6M 3012M 16.2.10 c56e13ff90f2 81b9b51f0eee 2022-09-29T05:28:18.543 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4m) 112s ago 4m 50.7M 3012M 16.2.10 c56e13ff90f2 4863a2083356 2022-09-29T05:28:18.543 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4m) 0s ago 4m 52.9M 3268M 16.2.10 c56e13ff90f2 ed61dc9353b0 2022-09-29T05:28:18.543 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4m) 0s ago 4m 59.2M 3268M 16.2.10 c56e13ff90f2 59f19fd15fed 2022-09-29T05:28:18.543 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3m) 0s ago 3m 58.2M 3268M 16.2.10 c56e13ff90f2 23decfd9c6b2 2022-09-29T05:28:18.544 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3m) 0s ago 3m 57.1M 3268M 16.2.10 c56e13ff90f2 23460a09c0a7 2022-09-29T05:28:18.544 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (117s) 0s ago 2m 52.6M - 514e6a882f6e fb1e43a0d8ba 2022-09-29T05:28:18.803 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:28:18 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: debug 2022-09-29T05:28:18.511+0000 7f834b34f200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2022-09-29T05:28:18.803 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:28:18 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: debug 2022-09-29T05:28:18.609+0000 7f834b34f200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2022-09-29T05:28:18.803 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:28:18 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: debug 2022-09-29T05:28:18.705+0000 7f834b34f200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2022-09-29T05:28:18.866 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T05:28:18.866 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T05:28:18.866 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 3 2022-09-29T05:28:18.867 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:28:18.867 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T05:28:18.867 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T05:28:18.867 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:28:18.868 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T05:28:18.868 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 8 2022-09-29T05:28:18.868 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:28:18.868 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T05:28:18.868 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T05:28:18.869 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:28:18.869 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T05:28:18.869 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 15 2022-09-29T05:28:18.869 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T05:28:18.870 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T05:28:19.059 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:28:18 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: debug 2022-09-29T05:28:18.799+0000 7f834b34f200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2022-09-29T05:28:19.059 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:28:18 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: debug 2022-09-29T05:28:18.949+0000 7f834b34f200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2022-09-29T05:28:19.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:19 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:28:17.849040+0000 mgr.y (mgr.14577) 139 : cluster [DBG] pgmap v98: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:28:19.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:19 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:18.254917+0000 mgr.y (mgr.14577) 140 : audit [DBG] from='client.14949 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:28:19.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:19 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:18.414064+0000 mon.a (mon.0) 781 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:28:19.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:19 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:18.528654+0000 mgr.y (mgr.14577) 141 : audit [DBG] from='client.14955 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:28:19.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:19 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:18.816225+0000 mon.a (mon.0) 782 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:28:19.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:19 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:18.820316+0000 mon.a (mon.0) 783 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:28:19.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:19 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:18.823790+0000 mon.a (mon.0) 784 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2022-09-29T05:28:19.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:19 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:28:18.827581+0000 mon.a (mon.0) 785 : cluster [DBG] osdmap e59: 8 total, 8 up, 8 in 2022-09-29T05:28:19.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:19 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:18.863335+0000 mon.c (mon.1) 34 : audit [DBG] from='client.? 172.21.15.83:0/1663043668' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:28:19.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:19 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:28:17.849040+0000 mgr.y (mgr.14577) 139 : cluster [DBG] pgmap v98: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:28:19.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:19 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:18.254917+0000 mgr.y (mgr.14577) 140 : audit [DBG] from='client.14949 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:28:19.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:19 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:18.414064+0000 mon.a (mon.0) 781 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:28:19.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:19 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:18.528654+0000 mgr.y (mgr.14577) 141 : audit [DBG] from='client.14955 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:28:19.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:19 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:18.816225+0000 mon.a (mon.0) 782 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:28:19.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:19 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:18.820316+0000 mon.a (mon.0) 783 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:28:19.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:19 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:18.823790+0000 mon.a (mon.0) 784 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2022-09-29T05:28:19.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:19 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:28:18.827581+0000 mon.a (mon.0) 785 : cluster [DBG] osdmap e59: 8 total, 8 up, 8 in 2022-09-29T05:28:19.718 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:19 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:18.863335+0000 mon.c (mon.1) 34 : audit [DBG] from='client.? 172.21.15.83:0/1663043668' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:28:19.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:19 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:28:17.849040+0000 mgr.y (mgr.14577) 139 : cluster [DBG] pgmap v98: 105 pgs: 105 active+clean; 3.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:28:19.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:19 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:18.254917+0000 mgr.y (mgr.14577) 140 : audit [DBG] from='client.14949 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:28:19.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:19 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:18.414064+0000 mon.a (mon.0) 781 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:28:19.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:19 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:18.528654+0000 mgr.y (mgr.14577) 141 : audit [DBG] from='client.14955 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:28:19.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:19 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:18.816225+0000 mon.a (mon.0) 782 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' 2022-09-29T05:28:19.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:19 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:18.820316+0000 mon.a (mon.0) 783 : audit [DBG] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:28:19.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:19 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:18.823790+0000 mon.a (mon.0) 784 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2022-09-29T05:28:19.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:19 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:28:18.827581+0000 mon.a (mon.0) 785 : cluster [DBG] osdmap e59: 8 total, 8 up, 8 in 2022-09-29T05:28:19.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:19 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:18.863335+0000 mon.c (mon.1) 34 : audit [DBG] from='client.? 172.21.15.83:0/1663043668' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:28:20.164 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:28:19 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: debug 2022-09-29T05:28:19.950+0000 7f834b34f200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2022-09-29T05:28:20.165 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:28:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: debug 2022-09-29T05:28:20.067+0000 7f834b34f200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2022-09-29T05:28:20.165 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:28:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: debug 2022-09-29T05:28:20.161+0000 7f834b34f200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2022-09-29T05:28:20.213 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:19 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:28:19.816+0000 7ffa70032700 -1 mgr handle_mgr_map I was active but no longer am 2022-09-29T05:28:20.213 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:19 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: ignoring --setuser ceph since I am not root 2022-09-29T05:28:20.213 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:19 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: ignoring --setgroup ceph since I am not root 2022-09-29T05:28:20.214 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:19 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:28:19.862+0000 7f43fbbf8700 1 -- 172.21.15.83:0/249333473 <== mon.0 v2:172.21.15.83:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x559ea283c4e0 con 0x559ea2815400 2022-09-29T05:28:20.214 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:19 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:28:19.984+0000 7f440bdae680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2022-09-29T05:28:20.214 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:28:20.065+0000 7f440bdae680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2022-09-29T05:28:20.417 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:28:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: [29/Sep/2022:05:28:20] ENGINE Bus STARTING 2022-09-29T05:28:20.418 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:28:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: CherryPy Checker: 2022-09-29T05:28:20.418 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:28:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: The Application mounted at '' has an empty config. 2022-09-29T05:28:20.418 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:28:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: 2022-09-29T05:28:20.419 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:28:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: [29/Sep/2022:05:28:20] ENGINE Serving on http://:::9283 2022-09-29T05:28:20.419 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:28:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: [29/Sep/2022:05:28:20] ENGINE Bus STARTED 2022-09-29T05:28:20.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:28:20.423+0000 7f440bdae680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2022-09-29T05:28:20.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:28:18.821637+0000 mgr.y (mgr.14577) 142 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.y) 2022-09-29T05:28:20.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:28:18.823590+0000 mgr.y (mgr.14577) 143 : cephadm [INF] Failing over to other MGR 2022-09-29T05:28:20.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:19.819952+0000 mon.a (mon.0) 786 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2022-09-29T05:28:20.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:28:19.820018+0000 mon.a (mon.0) 787 : cluster [DBG] mgrmap e21: x(active, starting, since 0.995597s) 2022-09-29T05:28:20.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:28:20.163929+0000 mon.a (mon.0) 788 : cluster [INF] Active manager daemon x restarted 2022-09-29T05:28:20.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:28:20.164567+0000 mon.a (mon.0) 789 : cluster [INF] Activating manager daemon x 2022-09-29T05:28:20.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:20.165259+0000 mon.b (mon.2) 27 : audit [DBG] from='mgr.? 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2022-09-29T05:28:20.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:28:20.168298+0000 mon.a (mon.0) 790 : cluster [DBG] osdmap e60: 8 total, 8 up, 8 in 2022-09-29T05:28:20.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:20.169206+0000 mon.b (mon.2) 28 : audit [DBG] from='mgr.? 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2022-09-29T05:28:20.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:20.170699+0000 mon.b (mon.2) 29 : audit [DBG] from='mgr.? 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2022-09-29T05:28:20.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:20.170950+0000 mon.b (mon.2) 30 : audit [DBG] from='mgr.? 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2022-09-29T05:28:20.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:28:18.821637+0000 mgr.y (mgr.14577) 142 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.y) 2022-09-29T05:28:20.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:28:18.823590+0000 mgr.y (mgr.14577) 143 : cephadm [INF] Failing over to other MGR 2022-09-29T05:28:20.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:19.819952+0000 mon.a (mon.0) 786 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2022-09-29T05:28:20.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:28:19.820018+0000 mon.a (mon.0) 787 : cluster [DBG] mgrmap e21: x(active, starting, since 0.995597s) 2022-09-29T05:28:20.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:28:20.163929+0000 mon.a (mon.0) 788 : cluster [INF] Active manager daemon x restarted 2022-09-29T05:28:20.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:28:20.164567+0000 mon.a (mon.0) 789 : cluster [INF] Activating manager daemon x 2022-09-29T05:28:20.718 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:20.165259+0000 mon.b (mon.2) 27 : audit [DBG] from='mgr.? 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2022-09-29T05:28:20.718 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:28:20.168298+0000 mon.a (mon.0) 790 : cluster [DBG] osdmap e60: 8 total, 8 up, 8 in 2022-09-29T05:28:20.718 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:20.169206+0000 mon.b (mon.2) 28 : audit [DBG] from='mgr.? 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2022-09-29T05:28:20.719 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:20.170699+0000 mon.b (mon.2) 29 : audit [DBG] from='mgr.? 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2022-09-29T05:28:20.719 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:20.170950+0000 mon.b (mon.2) 30 : audit [DBG] from='mgr.? 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2022-09-29T05:28:20.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:28:18.821637+0000 mgr.y (mgr.14577) 142 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.y) 2022-09-29T05:28:20.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:28:18.823590+0000 mgr.y (mgr.14577) 143 : cephadm [INF] Failing over to other MGR 2022-09-29T05:28:20.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:19.819952+0000 mon.a (mon.0) 786 : audit [INF] from='mgr.14577 172.21.15.83:0/912821170' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2022-09-29T05:28:20.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:28:19.820018+0000 mon.a (mon.0) 787 : cluster [DBG] mgrmap e21: x(active, starting, since 0.995597s) 2022-09-29T05:28:20.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:28:20.163929+0000 mon.a (mon.0) 788 : cluster [INF] Active manager daemon x restarted 2022-09-29T05:28:20.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:28:20.164567+0000 mon.a (mon.0) 789 : cluster [INF] Activating manager daemon x 2022-09-29T05:28:20.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:20.165259+0000 mon.b (mon.2) 27 : audit [DBG] from='mgr.? 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2022-09-29T05:28:20.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:28:20.168298+0000 mon.a (mon.0) 790 : cluster [DBG] osdmap e60: 8 total, 8 up, 8 in 2022-09-29T05:28:20.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:20.169206+0000 mon.b (mon.2) 28 : audit [DBG] from='mgr.? 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2022-09-29T05:28:20.813 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:20.170699+0000 mon.b (mon.2) 29 : audit [DBG] from='mgr.? 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2022-09-29T05:28:20.813 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:20.170950+0000 mon.b (mon.2) 30 : audit [DBG] from='mgr.? 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2022-09-29T05:28:21.260 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:28:21.166+0000 7f440bdae680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2022-09-29T05:28:21.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:28:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: [29/Sep/2022:05:28:20] ENGINE Bus STOPPING 2022-09-29T05:28:21.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:28:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: [29/Sep/2022:05:28:20] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2022-09-29T05:28:21.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:28:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: [29/Sep/2022:05:28:20] ENGINE Bus STOPPED 2022-09-29T05:28:21.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:28:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: [29/Sep/2022:05:28:20] ENGINE Bus STARTING 2022-09-29T05:28:21.310 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:28:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: debug 2022-09-29T05:28:20.993+0000 7f82b30a5700 -1 client.0 error registering admin socket command: (17) File exists 2022-09-29T05:28:21.310 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:28:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: debug 2022-09-29T05:28:20.993+0000 7f82b30a5700 -1 client.0 error registering admin socket command: (17) File exists 2022-09-29T05:28:21.310 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:28:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: debug 2022-09-29T05:28:20.993+0000 7f82b30a5700 -1 client.0 error registering admin socket command: (17) File exists 2022-09-29T05:28:21.310 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:28:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: debug 2022-09-29T05:28:20.993+0000 7f82b30a5700 -1 client.0 error registering admin socket command: (17) File exists 2022-09-29T05:28:21.311 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:28:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: debug 2022-09-29T05:28:20.993+0000 7f82b30a5700 -1 client.0 error registering admin socket command: (17) File exists 2022-09-29T05:28:21.311 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:28:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: debug 2022-09-29T05:28:20.998+0000 7f82b009f700 -1 client.0 error registering admin socket command: (17) File exists 2022-09-29T05:28:21.311 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:28:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: debug 2022-09-29T05:28:20.998+0000 7f82b009f700 -1 client.0 error registering admin socket command: (17) File exists 2022-09-29T05:28:21.311 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:28:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: debug 2022-09-29T05:28:20.998+0000 7f82b009f700 -1 client.0 error registering admin socket command: (17) File exists 2022-09-29T05:28:21.312 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:28:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: debug 2022-09-29T05:28:20.998+0000 7f82b009f700 -1 client.0 error registering admin socket command: (17) File exists 2022-09-29T05:28:21.312 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:28:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: debug 2022-09-29T05:28:20.998+0000 7f82b009f700 -1 client.0 error registering admin socket command: (17) File exists 2022-09-29T05:28:21.312 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:28:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: [29/Sep/2022:05:28:21] ENGINE Serving on http://:::9283 2022-09-29T05:28:21.312 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:28:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: [29/Sep/2022:05:28:21] ENGINE Bus STARTED 2022-09-29T05:28:21.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:28:21.256+0000 7f440bdae680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2022-09-29T05:28:21.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:28:21.478+0000 7f440bdae680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2022-09-29T05:28:22.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:28:20.864618+0000 mon.a (mon.0) 791 : cluster [DBG] mgrmap e22: x(active, starting, since 0.700231s) 2022-09-29T05:28:22.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:20.866610+0000 mon.b (mon.2) 31 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2022-09-29T05:28:22.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:20.866841+0000 mon.b (mon.2) 32 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2022-09-29T05:28:22.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:20.867019+0000 mon.b (mon.2) 33 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2022-09-29T05:28:22.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:20.867207+0000 mon.b (mon.2) 34 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2022-09-29T05:28:22.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:20.867428+0000 mon.b (mon.2) 35 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2022-09-29T05:28:22.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:20.867655+0000 mon.b (mon.2) 36 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2022-09-29T05:28:22.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:20.867816+0000 mon.b (mon.2) 37 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2022-09-29T05:28:22.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:20.868013+0000 mon.b (mon.2) 38 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2022-09-29T05:28:22.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:20.868202+0000 mon.b (mon.2) 39 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2022-09-29T05:28:22.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:20.868407+0000 mon.b (mon.2) 40 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2022-09-29T05:28:22.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:20.868602+0000 mon.b (mon.2) 41 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2022-09-29T05:28:22.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:20.868820+0000 mon.b (mon.2) 42 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2022-09-29T05:28:22.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:20.869011+0000 mon.b (mon.2) 43 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2022-09-29T05:28:22.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:20.869207+0000 mon.b (mon.2) 44 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2022-09-29T05:28:22.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:28:20.864618+0000 mon.a (mon.0) 791 : cluster [DBG] mgrmap e22: x(active, starting, since 0.700231s) 2022-09-29T05:28:22.218 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:20.866610+0000 mon.b (mon.2) 31 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2022-09-29T05:28:22.218 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:20.866841+0000 mon.b (mon.2) 32 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2022-09-29T05:28:22.218 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:20.867019+0000 mon.b (mon.2) 33 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2022-09-29T05:28:22.218 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:20.867207+0000 mon.b (mon.2) 34 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2022-09-29T05:28:22.219 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:20.867428+0000 mon.b (mon.2) 35 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2022-09-29T05:28:22.219 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:20.867655+0000 mon.b (mon.2) 36 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2022-09-29T05:28:22.219 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:20.867816+0000 mon.b (mon.2) 37 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2022-09-29T05:28:22.219 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:20.868013+0000 mon.b (mon.2) 38 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2022-09-29T05:28:22.220 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:20.868202+0000 mon.b (mon.2) 39 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2022-09-29T05:28:22.220 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:20.868407+0000 mon.b (mon.2) 40 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2022-09-29T05:28:22.220 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:20.868602+0000 mon.b (mon.2) 41 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2022-09-29T05:28:22.220 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:20.868820+0000 mon.b (mon.2) 42 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2022-09-29T05:28:22.220 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:20.869011+0000 mon.b (mon.2) 43 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2022-09-29T05:28:22.221 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:20.869207+0000 mon.b (mon.2) 44 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2022-09-29T05:28:22.221 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:20.869973+0000 mon.b (mon.2) 45 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2022-09-29T05:28:22.222 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:20.870238+0000 mon.b (mon.2) 46 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2022-09-29T05:28:22.222 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:20.870714+0000 mon.b (mon.2) 47 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2022-09-29T05:28:22.222 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:28:20.902550+0000 mon.a (mon.0) 792 : cluster [INF] Manager daemon x is now available 2022-09-29T05:28:22.222 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:20.919988+0000 mon.b (mon.2) 48 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:28:22.223 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:20.922153+0000 mon.b (mon.2) 49 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:28:22.223 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:20.923834+0000 mon.b (mon.2) 50 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:28:22.223 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:20.925220+0000 mon.b (mon.2) 51 : audit [INF] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:28:22.223 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:20.930572+0000 mon.b (mon.2) 52 : audit [INF] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2022-09-29T05:28:22.224 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:20.931130+0000 mon.a (mon.0) 793 : audit [INF] from='mgr.24752 ' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2022-09-29T05:28:22.224 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:20.947262+0000 mon.b (mon.2) 53 : audit [INF] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:28:22.224 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:20.947713+0000 mon.a (mon.0) 794 : audit [INF] from='mgr.24752 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:28:22.224 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:20.966323+0000 mon.b (mon.2) 54 : audit [INF] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2022-09-29T05:28:22.225 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:20.966730+0000 mon.a (mon.0) 795 : audit [INF] from='mgr.24752 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2022-09-29T05:28:22.225 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:20.976611+0000 mon.a (mon.0) 796 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:22.225 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:20.869973+0000 mon.b (mon.2) 45 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2022-09-29T05:28:22.226 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:20.870238+0000 mon.b (mon.2) 46 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2022-09-29T05:28:22.226 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:20.870714+0000 mon.b (mon.2) 47 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2022-09-29T05:28:22.226 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:28:20.902550+0000 mon.a (mon.0) 792 : cluster [INF] Manager daemon x is now available 2022-09-29T05:28:22.226 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:20.919988+0000 mon.b (mon.2) 48 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:28:22.226 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:20.922153+0000 mon.b (mon.2) 49 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:28:22.227 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:20.923834+0000 mon.b (mon.2) 50 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:28:22.227 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:20.925220+0000 mon.b (mon.2) 51 : audit [INF] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:28:22.227 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:20.930572+0000 mon.b (mon.2) 52 : audit [INF] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2022-09-29T05:28:22.227 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:20.931130+0000 mon.a (mon.0) 793 : audit [INF] from='mgr.24752 ' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2022-09-29T05:28:22.228 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:20.947262+0000 mon.b (mon.2) 53 : audit [INF] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:28:22.228 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:20.947713+0000 mon.a (mon.0) 794 : audit [INF] from='mgr.24752 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:28:22.228 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:20.966323+0000 mon.b (mon.2) 54 : audit [INF] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2022-09-29T05:28:22.228 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:20.966730+0000 mon.a (mon.0) 795 : audit [INF] from='mgr.24752 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2022-09-29T05:28:22.228 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:20.976611+0000 mon.a (mon.0) 796 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:22.229 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:28:21.276099+0000 mgr.x (mgr.24752) 1 : cephadm [INF] Deploying cephadm binary to smithi153 2022-09-29T05:28:22.229 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:28:21.288179+0000 mgr.x (mgr.24752) 2 : cephadm [INF] Deploying cephadm binary to smithi083 2022-09-29T05:28:22.229 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:21.576281+0000 mon.a (mon.0) 797 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:22.229 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:21.587448+0000 mon.a (mon.0) 798 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:22.230 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:28:21.276099+0000 mgr.x (mgr.24752) 1 : cephadm [INF] Deploying cephadm binary to smithi153 2022-09-29T05:28:22.230 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:28:21.288179+0000 mgr.x (mgr.24752) 2 : cephadm [INF] Deploying cephadm binary to smithi083 2022-09-29T05:28:22.230 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:21.576281+0000 mon.a (mon.0) 797 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:22.230 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:21.587448+0000 mon.a (mon.0) 798 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:22.231 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:28:20.864618+0000 mon.a (mon.0) 791 : cluster [DBG] mgrmap e22: x(active, starting, since 0.700231s) 2022-09-29T05:28:22.232 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:20.866610+0000 mon.b (mon.2) 31 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2022-09-29T05:28:22.232 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:20.866841+0000 mon.b (mon.2) 32 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2022-09-29T05:28:22.232 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:20.867019+0000 mon.b (mon.2) 33 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2022-09-29T05:28:22.232 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:20.867207+0000 mon.b (mon.2) 34 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2022-09-29T05:28:22.233 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:20.867428+0000 mon.b (mon.2) 35 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2022-09-29T05:28:22.233 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:20.867655+0000 mon.b (mon.2) 36 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2022-09-29T05:28:22.233 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:20.867816+0000 mon.b (mon.2) 37 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2022-09-29T05:28:22.233 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:20.868013+0000 mon.b (mon.2) 38 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2022-09-29T05:28:22.234 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:20.868202+0000 mon.b (mon.2) 39 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2022-09-29T05:28:22.234 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:20.868407+0000 mon.b (mon.2) 40 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2022-09-29T05:28:22.234 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:20.868602+0000 mon.b (mon.2) 41 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2022-09-29T05:28:22.234 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:20.868820+0000 mon.b (mon.2) 42 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2022-09-29T05:28:22.235 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:20.869011+0000 mon.b (mon.2) 43 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2022-09-29T05:28:22.235 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:20.869207+0000 mon.b (mon.2) 44 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2022-09-29T05:28:22.235 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:20.869973+0000 mon.b (mon.2) 45 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2022-09-29T05:28:22.235 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:20.870238+0000 mon.b (mon.2) 46 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2022-09-29T05:28:22.236 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:20.870714+0000 mon.b (mon.2) 47 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2022-09-29T05:28:22.236 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:28:20.902550+0000 mon.a (mon.0) 792 : cluster [INF] Manager daemon x is now available 2022-09-29T05:28:22.236 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:20.919988+0000 mon.b (mon.2) 48 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:28:22.236 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:20.922153+0000 mon.b (mon.2) 49 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:28:22.237 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:20.923834+0000 mon.b (mon.2) 50 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:28:22.237 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:20.925220+0000 mon.b (mon.2) 51 : audit [INF] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:28:22.237 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:20.930572+0000 mon.b (mon.2) 52 : audit [INF] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2022-09-29T05:28:22.237 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:20.931130+0000 mon.a (mon.0) 793 : audit [INF] from='mgr.24752 ' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2022-09-29T05:28:22.237 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:20.947262+0000 mon.b (mon.2) 53 : audit [INF] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:28:22.238 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:20.947713+0000 mon.a (mon.0) 794 : audit [INF] from='mgr.24752 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:28:22.238 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:20.966323+0000 mon.b (mon.2) 54 : audit [INF] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2022-09-29T05:28:22.238 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:20.966730+0000 mon.a (mon.0) 795 : audit [INF] from='mgr.24752 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2022-09-29T05:28:22.238 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:20.976611+0000 mon.a (mon.0) 796 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:22.239 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:28:21.276099+0000 mgr.x (mgr.24752) 1 : cephadm [INF] Deploying cephadm binary to smithi153 2022-09-29T05:28:22.239 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:28:21.288179+0000 mgr.x (mgr.24752) 2 : cephadm [INF] Deploying cephadm binary to smithi083 2022-09-29T05:28:22.239 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:21.576281+0000 mon.a (mon.0) 797 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:22.239 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:21.587448+0000 mon.a (mon.0) 798 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:22.559 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:28:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:28:22] "GET /metrics HTTP/1.1" 200 25746 "" "Prometheus/2.33.4" 2022-09-29T05:28:22.782 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:28:22.462+0000 7f440bdae680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2022-09-29T05:28:22.783 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:28:22.601+0000 7f440bdae680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2022-09-29T05:28:22.783 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:28:22.687+0000 7f440bdae680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2022-09-29T05:28:23.213 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:28:22.779+0000 7f440bdae680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2022-09-29T05:28:23.213 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:28:22.990+0000 7f440bdae680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2022-09-29T05:28:23.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:28:21.873255+0000 mon.a (mon.0) 799 : cluster [DBG] mgrmap e23: x(active, since 1.70887s) 2022-09-29T05:28:23.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:21.873318+0000 mon.a (mon.0) 800 : audit [INF] from='mgr.24752 ' entity='mgr.x' cmd='[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]': finished 2022-09-29T05:28:23.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:28:21.873376+0000 mon.a (mon.0) 801 : cluster [DBG] osdmap e61: 8 total, 8 up, 8 in 2022-09-29T05:28:23.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:21.886887+0000 mon.b (mon.2) 55 : audit [INF] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2022-09-29T05:28:23.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:28:21.887025+0000 mgr.x (mgr.24752) 3 : cluster [DBG] pgmap v4: 105 pgs: 105 active+clean; 4.0 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:28:23.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:21.887390+0000 mon.a (mon.0) 802 : audit [INF] from='mgr.24752 ' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2022-09-29T05:28:23.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:22.608926+0000 mon.a (mon.0) 803 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:23.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:22.612729+0000 mon.a (mon.0) 804 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:23.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:28:21.873255+0000 mon.a (mon.0) 799 : cluster [DBG] mgrmap e23: x(active, since 1.70887s) 2022-09-29T05:28:23.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:21.873318+0000 mon.a (mon.0) 800 : audit [INF] from='mgr.24752 ' entity='mgr.x' cmd='[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]': finished 2022-09-29T05:28:23.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:28:21.873376+0000 mon.a (mon.0) 801 : cluster [DBG] osdmap e61: 8 total, 8 up, 8 in 2022-09-29T05:28:23.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:21.886887+0000 mon.b (mon.2) 55 : audit [INF] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2022-09-29T05:28:23.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:28:21.887025+0000 mgr.x (mgr.24752) 3 : cluster [DBG] pgmap v4: 105 pgs: 105 active+clean; 4.0 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:28:23.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:21.887390+0000 mon.a (mon.0) 802 : audit [INF] from='mgr.24752 ' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2022-09-29T05:28:23.218 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:22.608926+0000 mon.a (mon.0) 803 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:23.218 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:22.612729+0000 mon.a (mon.0) 804 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:23.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:28:21.873255+0000 mon.a (mon.0) 799 : cluster [DBG] mgrmap e23: x(active, since 1.70887s) 2022-09-29T05:28:23.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:21.873318+0000 mon.a (mon.0) 800 : audit [INF] from='mgr.24752 ' entity='mgr.x' cmd='[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]': finished 2022-09-29T05:28:23.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:28:21.873376+0000 mon.a (mon.0) 801 : cluster [DBG] osdmap e61: 8 total, 8 up, 8 in 2022-09-29T05:28:23.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:21.886887+0000 mon.b (mon.2) 55 : audit [INF] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2022-09-29T05:28:23.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:28:21.887025+0000 mgr.x (mgr.24752) 3 : cluster [DBG] pgmap v4: 105 pgs: 105 active+clean; 4.0 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:28:23.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:21.887390+0000 mon.a (mon.0) 802 : audit [INF] from='mgr.24752 ' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2022-09-29T05:28:23.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:22.608926+0000 mon.a (mon.0) 803 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:23.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:22.612729+0000 mon.a (mon.0) 804 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:23.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:28:23.304+0000 7f440bdae680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2022-09-29T05:28:23.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:28:23.406+0000 7f440bdae680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2022-09-29T05:28:23.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:28:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=warn ts=2022-09-29T05:28:23.514Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": dial tcp 172.21.15.83:8443: connect: connection refused" 2022-09-29T05:28:24.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:28:22.870157+0000 mgr.x (mgr.24752) 4 : cluster [DBG] pgmap v5: 105 pgs: 105 active+clean; 4.0 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:28:24.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:22.883659+0000 mon.a (mon.0) 805 : audit [INF] from='mgr.24752 ' entity='mgr.x' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2022-09-29T05:28:24.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:28:22.883729+0000 mon.a (mon.0) 806 : cluster [DBG] osdmap e62: 8 total, 8 up, 8 in 2022-09-29T05:28:24.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:28:22.883986+0000 mon.a (mon.0) 807 : cluster [DBG] mgrmap e24: x(active, since 2s) 2022-09-29T05:28:24.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:28:23.094287+0000 mgr.x (mgr.24752) 5 : cephadm [INF] [29/Sep/2022:05:28:23] ENGINE Bus STARTING 2022-09-29T05:28:24.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:28:23.207935+0000 mgr.x (mgr.24752) 6 : cephadm [INF] [29/Sep/2022:05:28:23] ENGINE Serving on https://:::8765 2022-09-29T05:28:24.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:28:23.319826+0000 mgr.x (mgr.24752) 7 : cephadm [INF] [29/Sep/2022:05:28:23] ENGINE Serving on https://172.21.15.153:7150 2022-09-29T05:28:24.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:28:23.319942+0000 mgr.x (mgr.24752) 8 : cephadm [INF] [29/Sep/2022:05:28:23] ENGINE Bus STARTED 2022-09-29T05:28:24.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:23.486240+0000 mon.a (mon.0) 808 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:24.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:28:22.870157+0000 mgr.x (mgr.24752) 4 : cluster [DBG] pgmap v5: 105 pgs: 105 active+clean; 4.0 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:28:24.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:22.883659+0000 mon.a (mon.0) 805 : audit [INF] from='mgr.24752 ' entity='mgr.x' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2022-09-29T05:28:24.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:28:22.883729+0000 mon.a (mon.0) 806 : cluster [DBG] osdmap e62: 8 total, 8 up, 8 in 2022-09-29T05:28:24.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:28:22.883986+0000 mon.a (mon.0) 807 : cluster [DBG] mgrmap e24: x(active, since 2s) 2022-09-29T05:28:24.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:28:23.094287+0000 mgr.x (mgr.24752) 5 : cephadm [INF] [29/Sep/2022:05:28:23] ENGINE Bus STARTING 2022-09-29T05:28:24.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:28:23.207935+0000 mgr.x (mgr.24752) 6 : cephadm [INF] [29/Sep/2022:05:28:23] ENGINE Serving on https://:::8765 2022-09-29T05:28:24.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:28:23.319826+0000 mgr.x (mgr.24752) 7 : cephadm [INF] [29/Sep/2022:05:28:23] ENGINE Serving on https://172.21.15.153:7150 2022-09-29T05:28:24.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:28:23.319942+0000 mgr.x (mgr.24752) 8 : cephadm [INF] [29/Sep/2022:05:28:23] ENGINE Bus STARTED 2022-09-29T05:28:24.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:23.486240+0000 mon.a (mon.0) 808 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:24.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:28:22.870157+0000 mgr.x (mgr.24752) 4 : cluster [DBG] pgmap v5: 105 pgs: 105 active+clean; 4.0 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:28:24.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:22.883659+0000 mon.a (mon.0) 805 : audit [INF] from='mgr.24752 ' entity='mgr.x' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2022-09-29T05:28:24.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:28:22.883729+0000 mon.a (mon.0) 806 : cluster [DBG] osdmap e62: 8 total, 8 up, 8 in 2022-09-29T05:28:24.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:28:22.883986+0000 mon.a (mon.0) 807 : cluster [DBG] mgrmap e24: x(active, since 2s) 2022-09-29T05:28:24.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:28:23.094287+0000 mgr.x (mgr.24752) 5 : cephadm [INF] [29/Sep/2022:05:28:23] ENGINE Bus STARTING 2022-09-29T05:28:24.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:28:23.207935+0000 mgr.x (mgr.24752) 6 : cephadm [INF] [29/Sep/2022:05:28:23] ENGINE Serving on https://:::8765 2022-09-29T05:28:24.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:28:23.319826+0000 mgr.x (mgr.24752) 7 : cephadm [INF] [29/Sep/2022:05:28:23] ENGINE Serving on https://172.21.15.153:7150 2022-09-29T05:28:24.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:28:23.319942+0000 mgr.x (mgr.24752) 8 : cephadm [INF] [29/Sep/2022:05:28:23] ENGINE Bus STARTED 2022-09-29T05:28:24.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:23.486240+0000 mon.a (mon.0) 808 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:24.635 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:28:24.227+0000 7f440bdae680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2022-09-29T05:28:24.635 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:28:24.316+0000 7f440bdae680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2022-09-29T05:28:24.636 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:28:24.428+0000 7f440bdae680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2022-09-29T05:28:24.906 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:28:24.631+0000 7f440bdae680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2022-09-29T05:28:25.190 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:28:24.902+0000 7f440bdae680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2022-09-29T05:28:25.190 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:28:25.047+0000 7f440bdae680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2022-09-29T05:28:25.463 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:28:25.186+0000 7f440bdae680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2022-09-29T05:28:25.463 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:28:25.368+0000 7f440bdae680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2022-09-29T05:28:25.885 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: debug 2022-09-29T05:28:25.462+0000 7f440bdae680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2022-09-29T05:28:25.886 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: [29/Sep/2022:05:28:25] ENGINE Bus STARTING 2022-09-29T05:28:25.886 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: CherryPy Checker: 2022-09-29T05:28:25.886 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: The Application mounted at '' has an empty config. 2022-09-29T05:28:25.886 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: 2022-09-29T05:28:25.887 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: [29/Sep/2022:05:28:25] ENGINE Serving on http://:::9283 2022-09-29T05:28:25.887 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: [29/Sep/2022:05:28:25] ENGINE Bus STARTED 2022-09-29T05:28:26.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:28:24.870610+0000 mgr.x (mgr.24752) 9 : cluster [DBG] pgmap v7: 105 pgs: 105 active+clean; 4.0 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:28:26.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:28:24.884218+0000 mon.a (mon.0) 809 : cluster [DBG] mgrmap e25: x(active, since 4s) 2022-09-29T05:28:26.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:28:25.465349+0000 mon.a (mon.0) 810 : cluster [DBG] Standby manager daemon y started 2022-09-29T05:28:26.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:25.465684+0000 mon.c (mon.1) 35 : audit [DBG] from='mgr.? 172.21.15.83:0/1431383079' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2022-09-29T05:28:26.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:25.468099+0000 mon.c (mon.1) 36 : audit [DBG] from='mgr.? 172.21.15.83:0/1431383079' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2022-09-29T05:28:26.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:25.470295+0000 mon.c (mon.1) 37 : audit [DBG] from='mgr.? 172.21.15.83:0/1431383079' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2022-09-29T05:28:26.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:25.470639+0000 mon.c (mon.1) 38 : audit [DBG] from='mgr.? 172.21.15.83:0/1431383079' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2022-09-29T05:28:26.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:25.523536+0000 mon.a (mon.0) 811 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:26.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:25.652014+0000 mon.a (mon.0) 812 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:26.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:25.659508+0000 mon.a (mon.0) 813 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:26.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:28:24.870610+0000 mgr.x (mgr.24752) 9 : cluster [DBG] pgmap v7: 105 pgs: 105 active+clean; 4.0 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:28:26.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:28:24.884218+0000 mon.a (mon.0) 809 : cluster [DBG] mgrmap e25: x(active, since 4s) 2022-09-29T05:28:26.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:28:25.465349+0000 mon.a (mon.0) 810 : cluster [DBG] Standby manager daemon y started 2022-09-29T05:28:26.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:25.465684+0000 mon.c (mon.1) 35 : audit [DBG] from='mgr.? 172.21.15.83:0/1431383079' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2022-09-29T05:28:26.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:25.468099+0000 mon.c (mon.1) 36 : audit [DBG] from='mgr.? 172.21.15.83:0/1431383079' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2022-09-29T05:28:26.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:25.470295+0000 mon.c (mon.1) 37 : audit [DBG] from='mgr.? 172.21.15.83:0/1431383079' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2022-09-29T05:28:26.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:25.470639+0000 mon.c (mon.1) 38 : audit [DBG] from='mgr.? 172.21.15.83:0/1431383079' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2022-09-29T05:28:26.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:25.523536+0000 mon.a (mon.0) 811 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:26.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:25.652014+0000 mon.a (mon.0) 812 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:26.218 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:25.659508+0000 mon.a (mon.0) 813 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:26.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:25 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:28:24.870610+0000 mgr.x (mgr.24752) 9 : cluster [DBG] pgmap v7: 105 pgs: 105 active+clean; 4.0 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:28:26.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:25 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:28:24.884218+0000 mon.a (mon.0) 809 : cluster [DBG] mgrmap e25: x(active, since 4s) 2022-09-29T05:28:26.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:25 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:28:25.465349+0000 mon.a (mon.0) 810 : cluster [DBG] Standby manager daemon y started 2022-09-29T05:28:26.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:25 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:25.465684+0000 mon.c (mon.1) 35 : audit [DBG] from='mgr.? 172.21.15.83:0/1431383079' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2022-09-29T05:28:26.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:25 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:25.468099+0000 mon.c (mon.1) 36 : audit [DBG] from='mgr.? 172.21.15.83:0/1431383079' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2022-09-29T05:28:26.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:25 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:25.470295+0000 mon.c (mon.1) 37 : audit [DBG] from='mgr.? 172.21.15.83:0/1431383079' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2022-09-29T05:28:26.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:25 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:25.470639+0000 mon.c (mon.1) 38 : audit [DBG] from='mgr.? 172.21.15.83:0/1431383079' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2022-09-29T05:28:26.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:25 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:25.523536+0000 mon.a (mon.0) 811 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:26.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:25 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:25.652014+0000 mon.a (mon.0) 812 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:26.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:25 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:25.659508+0000 mon.a (mon.0) 813 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:26.895 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:28:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=info ts=2022-09-29T05:28:26.488Z caller=notify.go:735 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify success" attempts=4 2022-09-29T05:28:26.898 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:28:25.893629+0000 mon.a (mon.0) 814 : cluster [DBG] mgrmap e26: x(active, since 5s), standbys: y 2022-09-29T05:28:26.898 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:25.896822+0000 mon.b (mon.2) 56 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2022-09-29T05:28:26.898 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:26.621365+0000 mon.a (mon.0) 815 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:26.899 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:26.626151+0000 mon.a (mon.0) 816 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:27.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:28:25.893629+0000 mon.a (mon.0) 814 : cluster [DBG] mgrmap e26: x(active, since 5s), standbys: y 2022-09-29T05:28:27.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:25.896822+0000 mon.b (mon.2) 56 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2022-09-29T05:28:27.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:26.621365+0000 mon.a (mon.0) 815 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:27.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:26.626151+0000 mon.a (mon.0) 816 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:27.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:26 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:28:25.893629+0000 mon.a (mon.0) 814 : cluster [DBG] mgrmap e26: x(active, since 5s), standbys: y 2022-09-29T05:28:27.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:26 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:25.896822+0000 mon.b (mon.2) 56 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2022-09-29T05:28:27.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:26 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:26.621365+0000 mon.a (mon.0) 815 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:27.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:26 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:26.626151+0000 mon.a (mon.0) 816 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:28.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:28 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:27.039149+0000 mon.a (mon.0) 817 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2022-09-29T05:28:28.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:28 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:27.039337+0000 mon.b (mon.2) 57 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2022-09-29T05:28:28.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:28 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:27.039558+0000 mon.b (mon.2) 58 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2022-09-29T05:28:28.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:28 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:27.039719+0000 mon.b (mon.2) 59 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2022-09-29T05:28:28.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:28 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:27.529973+0000 mon.a (mon.0) 818 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2022-09-29T05:28:28.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:28 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:27.532140+0000 mon.b (mon.2) 60 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2022-09-29T05:28:28.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:28 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:27.532319+0000 mon.b (mon.2) 61 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2022-09-29T05:28:28.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:28 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:27.532471+0000 mon.b (mon.2) 62 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2022-09-29T05:28:28.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:28 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:27.039149+0000 mon.a (mon.0) 817 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2022-09-29T05:28:28.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:28 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:27.039337+0000 mon.b (mon.2) 57 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2022-09-29T05:28:28.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:28 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:27.039558+0000 mon.b (mon.2) 58 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2022-09-29T05:28:28.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:28 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:27.039719+0000 mon.b (mon.2) 59 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2022-09-29T05:28:28.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:28 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:27.529973+0000 mon.a (mon.0) 818 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2022-09-29T05:28:28.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:28 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:27.532140+0000 mon.b (mon.2) 60 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2022-09-29T05:28:28.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:28 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:27.532319+0000 mon.b (mon.2) 61 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2022-09-29T05:28:28.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:28 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:27.532471+0000 mon.b (mon.2) 62 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2022-09-29T05:28:28.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:28 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:27.039149+0000 mon.a (mon.0) 817 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2022-09-29T05:28:28.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:28 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:27.039337+0000 mon.b (mon.2) 57 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2022-09-29T05:28:28.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:28 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:27.039558+0000 mon.b (mon.2) 58 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2022-09-29T05:28:28.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:28 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:27.039719+0000 mon.b (mon.2) 59 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2022-09-29T05:28:28.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:28 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:27.529973+0000 mon.a (mon.0) 818 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2022-09-29T05:28:28.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:28 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:27.532140+0000 mon.b (mon.2) 60 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2022-09-29T05:28:28.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:28 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:27.532319+0000 mon.b (mon.2) 61 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2022-09-29T05:28:28.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:28 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:27.532471+0000 mon.b (mon.2) 62 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2022-09-29T05:28:29.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:29 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:28:26.871009+0000 mgr.x (mgr.24752) 10 : cluster [DBG] pgmap v8: 105 pgs: 105 active+clean; 4.0 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:28:29.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:29 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:27.531882+0000 mon.c (mon.1) 39 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2022-09-29T05:28:29.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:29 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:28.024596+0000 mon.c (mon.1) 40 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2022-09-29T05:28:29.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:29 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:28.031224+0000 mon.b (mon.2) 63 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2022-09-29T05:28:29.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:29 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:28.516093+0000 mon.b (mon.2) 64 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2022-09-29T05:28:29.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:29 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:28.516379+0000 mon.b (mon.2) 65 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2022-09-29T05:28:29.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:29 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:28.516585+0000 mon.b (mon.2) 66 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2022-09-29T05:28:29.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:29 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:28.516764+0000 mon.b (mon.2) 67 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2022-09-29T05:28:29.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:29 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:28.520860+0000 mon.a (mon.0) 819 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:29.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:29 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:28.528215+0000 mon.a (mon.0) 820 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:29.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:29 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:28:26.871009+0000 mgr.x (mgr.24752) 10 : cluster [DBG] pgmap v8: 105 pgs: 105 active+clean; 4.0 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:28:29.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:29 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:27.531882+0000 mon.c (mon.1) 39 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2022-09-29T05:28:29.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:29 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:28.024596+0000 mon.c (mon.1) 40 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2022-09-29T05:28:29.427 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:29 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:28.031224+0000 mon.b (mon.2) 63 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2022-09-29T05:28:29.427 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:29 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:28.516093+0000 mon.b (mon.2) 64 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2022-09-29T05:28:29.427 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:29 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:28.516379+0000 mon.b (mon.2) 65 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2022-09-29T05:28:29.428 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:29 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:28.516585+0000 mon.b (mon.2) 66 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2022-09-29T05:28:29.428 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:29 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:28.516764+0000 mon.b (mon.2) 67 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2022-09-29T05:28:29.428 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:29 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:28.520860+0000 mon.a (mon.0) 819 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:29.428 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:29 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:28.528215+0000 mon.a (mon.0) 820 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:29.429 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:29 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:28:26.871009+0000 mgr.x (mgr.24752) 10 : cluster [DBG] pgmap v8: 105 pgs: 105 active+clean; 4.0 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:28:29.429 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:29 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:27.531882+0000 mon.c (mon.1) 39 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2022-09-29T05:28:29.429 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:29 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:28.024596+0000 mon.c (mon.1) 40 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2022-09-29T05:28:29.430 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:29 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:28.031224+0000 mon.b (mon.2) 63 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2022-09-29T05:28:29.430 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:29 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:28.516093+0000 mon.b (mon.2) 64 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2022-09-29T05:28:29.430 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:29 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:28.516379+0000 mon.b (mon.2) 65 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2022-09-29T05:28:29.430 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:29 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:28.516585+0000 mon.b (mon.2) 66 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2022-09-29T05:28:29.431 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:29 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:28.516764+0000 mon.b (mon.2) 67 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2022-09-29T05:28:29.431 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:29 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:28.520860+0000 mon.a (mon.0) 819 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:29.431 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:29 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:28.528215+0000 mon.a (mon.0) 820 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:30.651 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:30 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:28:28.871821+0000 mgr.x (mgr.24752) 11 : cluster [DBG] pgmap v9: 105 pgs: 105 active+clean; 581 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 308 KiB/s wr, 35 op/s 2022-09-29T05:28:30.651 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:30 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:29.473657+0000 mon.a (mon.0) 821 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:30.651 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:30 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:29.478490+0000 mon.a (mon.0) 822 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:30.652 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:30 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:28:29.715782+0000 mgr.x (mgr.24752) 12 : cephadm [INF] Detected new or changed devices on smithi083 2022-09-29T05:28:30.652 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:30 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:29.720539+0000 mon.a (mon.0) 823 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:30.652 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:30 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:29.725787+0000 mon.a (mon.0) 824 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:30.652 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:30 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:29.727110+0000 mon.b (mon.2) 68 : audit [INF] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:28:30.653 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:30 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:29.727435+0000 mon.a (mon.0) 825 : audit [INF] from='mgr.24752 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:28:30.653 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:30 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:29.728128+0000 mon.b (mon.2) 69 : audit [INF] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:28:30.653 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:30 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:29.728488+0000 mon.a (mon.0) 826 : audit [INF] from='mgr.24752 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:28:30.653 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:30 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:29.729156+0000 mon.b (mon.2) 70 : audit [INF] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:28:30.654 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:30 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:29.729511+0000 mon.a (mon.0) 827 : audit [INF] from='mgr.24752 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:28:30.654 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:30 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:29.730158+0000 mon.b (mon.2) 71 : audit [INF] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:28:30.654 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:30 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:29.730512+0000 mon.a (mon.0) 828 : audit [INF] from='mgr.24752 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:28:30.654 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:30 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:28:29.731051+0000 mgr.x (mgr.24752) 13 : cephadm [INF] Adjusting osd_memory_target on smithi083 to 2500M 2022-09-29T05:28:30.655 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:30 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:29.734480+0000 mon.a (mon.0) 829 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:30.655 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:30 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:28:29.735391+0000 mgr.x (mgr.24752) 14 : cephadm [INF] Updating smithi083:/etc/ceph/ceph.conf 2022-09-29T05:28:30.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:28:28.871821+0000 mgr.x (mgr.24752) 11 : cluster [DBG] pgmap v9: 105 pgs: 105 active+clean; 581 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 308 KiB/s wr, 35 op/s 2022-09-29T05:28:30.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:29.473657+0000 mon.a (mon.0) 821 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:30.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:29.478490+0000 mon.a (mon.0) 822 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:30.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:28:29.715782+0000 mgr.x (mgr.24752) 12 : cephadm [INF] Detected new or changed devices on smithi083 2022-09-29T05:28:30.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:29.720539+0000 mon.a (mon.0) 823 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:30.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:29.725787+0000 mon.a (mon.0) 824 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:30.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:29.727110+0000 mon.b (mon.2) 68 : audit [INF] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:28:30.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:29.727435+0000 mon.a (mon.0) 825 : audit [INF] from='mgr.24752 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:28:30.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:29.728128+0000 mon.b (mon.2) 69 : audit [INF] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:28:30.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:29.728488+0000 mon.a (mon.0) 826 : audit [INF] from='mgr.24752 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:28:30.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:29.729156+0000 mon.b (mon.2) 70 : audit [INF] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:28:30.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:29.729511+0000 mon.a (mon.0) 827 : audit [INF] from='mgr.24752 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:28:30.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:29.730158+0000 mon.b (mon.2) 71 : audit [INF] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:28:30.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:29.730512+0000 mon.a (mon.0) 828 : audit [INF] from='mgr.24752 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:28:30.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:28:29.731051+0000 mgr.x (mgr.24752) 13 : cephadm [INF] Adjusting osd_memory_target on smithi083 to 2500M 2022-09-29T05:28:30.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:29.734480+0000 mon.a (mon.0) 829 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:30.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:28:29.735391+0000 mgr.x (mgr.24752) 14 : cephadm [INF] Updating smithi083:/etc/ceph/ceph.conf 2022-09-29T05:28:30.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:28:28.871821+0000 mgr.x (mgr.24752) 11 : cluster [DBG] pgmap v9: 105 pgs: 105 active+clean; 581 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 308 KiB/s wr, 35 op/s 2022-09-29T05:28:30.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:29.473657+0000 mon.a (mon.0) 821 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:30.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:29.478490+0000 mon.a (mon.0) 822 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:30.968 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:28:29.715782+0000 mgr.x (mgr.24752) 12 : cephadm [INF] Detected new or changed devices on smithi083 2022-09-29T05:28:30.968 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:29.720539+0000 mon.a (mon.0) 823 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:30.968 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:29.725787+0000 mon.a (mon.0) 824 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:30.968 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:29.727110+0000 mon.b (mon.2) 68 : audit [INF] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:28:30.969 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:29.727435+0000 mon.a (mon.0) 825 : audit [INF] from='mgr.24752 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:28:30.969 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:29.728128+0000 mon.b (mon.2) 69 : audit [INF] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:28:30.969 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:29.728488+0000 mon.a (mon.0) 826 : audit [INF] from='mgr.24752 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:28:30.969 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:29.729156+0000 mon.b (mon.2) 70 : audit [INF] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:28:30.969 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:29.729511+0000 mon.a (mon.0) 827 : audit [INF] from='mgr.24752 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:28:30.970 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:29.730158+0000 mon.b (mon.2) 71 : audit [INF] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:28:30.970 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:29.730512+0000 mon.a (mon.0) 828 : audit [INF] from='mgr.24752 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:28:30.970 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:28:29.731051+0000 mgr.x (mgr.24752) 13 : cephadm [INF] Adjusting osd_memory_target on smithi083 to 2500M 2022-09-29T05:28:30.970 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:29.734480+0000 mon.a (mon.0) 829 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:30.971 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:28:29.735391+0000 mgr.x (mgr.24752) 14 : cephadm [INF] Updating smithi083:/etc/ceph/ceph.conf 2022-09-29T05:28:31.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:31 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:28:29.948725+0000 mgr.x (mgr.24752) 15 : cephadm [INF] Updating smithi083:/var/lib/ceph/69111522-3fb6-11ed-8431-001a4aab830c/config/ceph.conf 2022-09-29T05:28:31.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:31 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:28:30.159984+0000 mgr.x (mgr.24752) 16 : cephadm [INF] Updating smithi083:/etc/ceph/ceph.client.admin.keyring 2022-09-29T05:28:31.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:31 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:28:30.367415+0000 mgr.x (mgr.24752) 17 : cephadm [INF] Updating smithi083:/var/lib/ceph/69111522-3fb6-11ed-8431-001a4aab830c/config/ceph.client.admin.keyring 2022-09-29T05:28:31.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:31 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:30.581076+0000 mon.a (mon.0) 830 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:31.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:31 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:30.588312+0000 mon.a (mon.0) 831 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:31.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:28:29.948725+0000 mgr.x (mgr.24752) 15 : cephadm [INF] Updating smithi083:/var/lib/ceph/69111522-3fb6-11ed-8431-001a4aab830c/config/ceph.conf 2022-09-29T05:28:31.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:28:30.159984+0000 mgr.x (mgr.24752) 16 : cephadm [INF] Updating smithi083:/etc/ceph/ceph.client.admin.keyring 2022-09-29T05:28:31.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:28:30.367415+0000 mgr.x (mgr.24752) 17 : cephadm [INF] Updating smithi083:/var/lib/ceph/69111522-3fb6-11ed-8431-001a4aab830c/config/ceph.client.admin.keyring 2022-09-29T05:28:31.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:30.581076+0000 mon.a (mon.0) 830 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:31.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:30.588312+0000 mon.a (mon.0) 831 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:31.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:28:29.948725+0000 mgr.x (mgr.24752) 15 : cephadm [INF] Updating smithi083:/var/lib/ceph/69111522-3fb6-11ed-8431-001a4aab830c/config/ceph.conf 2022-09-29T05:28:31.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:28:30.159984+0000 mgr.x (mgr.24752) 16 : cephadm [INF] Updating smithi083:/etc/ceph/ceph.client.admin.keyring 2022-09-29T05:28:31.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:28:30.367415+0000 mgr.x (mgr.24752) 17 : cephadm [INF] Updating smithi083:/var/lib/ceph/69111522-3fb6-11ed-8431-001a4aab830c/config/ceph.client.admin.keyring 2022-09-29T05:28:31.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:30.581076+0000 mon.a (mon.0) 830 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:31.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:30.588312+0000 mon.a (mon.0) 831 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:32.559 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:28:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:28:32] "GET /metrics HTTP/1.1" 200 25746 "" "Prometheus/2.33.4" 2022-09-29T05:28:32.559 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:28:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[57164]: ts=2022-09-29T05:28:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: |\n Pool '{{ $labels.name }}' will be full in less than 5 days assuming the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed it's capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi153.front.sepia.ceph.com:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:28:32.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[39065]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:28:32] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:28:32.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:28:30.872219+0000 mgr.x (mgr.24752) 18 : cluster [DBG] pgmap v10: 105 pgs: 105 active+clean; 581 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 10 KiB/s rd, 240 KiB/s wr, 27 op/s 2022-09-29T05:28:32.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:28:30.988863+0000 mgr.x (mgr.24752) 19 : cluster [DBG] pgmap v11: 105 pgs: 105 active+clean; 581 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 265 KiB/s wr, 30 op/s 2022-09-29T05:28:32.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:28:31.586989+0000 mon.a (mon.0) 832 : cluster [WRN] Health check failed: Telemetry requires re-opt-in (TELEMETRY_CHANGED) 2022-09-29T05:28:32.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:32.277886+0000 mon.a (mon.0) 833 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:32.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:32.286056+0000 mon.a (mon.0) 834 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:32.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:32.287300+0000 mon.b (mon.2) 72 : audit [INF] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:28:32.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:32.287666+0000 mon.a (mon.0) 835 : audit [INF] from='mgr.24752 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:28:32.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:32.288305+0000 mon.b (mon.2) 73 : audit [INF] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:28:32.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:32.288665+0000 mon.a (mon.0) 836 : audit [INF] from='mgr.24752 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:28:32.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:32.289230+0000 mon.b (mon.2) 74 : audit [INF] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:28:32.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:32.289630+0000 mon.a (mon.0) 837 : audit [INF] from='mgr.24752 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:28:32.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:32.290176+0000 mon.b (mon.2) 75 : audit [INF] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:28:32.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:32.290524+0000 mon.a (mon.0) 838 : audit [INF] from='mgr.24752 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:28:32.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:32.295306+0000 mon.a (mon.0) 839 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:32.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:28:30.872219+0000 mgr.x (mgr.24752) 18 : cluster [DBG] pgmap v10: 105 pgs: 105 active+clean; 581 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 10 KiB/s rd, 240 KiB/s wr, 27 op/s 2022-09-29T05:28:32.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:28:30.988863+0000 mgr.x (mgr.24752) 19 : cluster [DBG] pgmap v11: 105 pgs: 105 active+clean; 581 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 265 KiB/s wr, 30 op/s 2022-09-29T05:28:32.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:28:31.586989+0000 mon.a (mon.0) 832 : cluster [WRN] Health check failed: Telemetry requires re-opt-in (TELEMETRY_CHANGED) 2022-09-29T05:28:32.718 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:32.277886+0000 mon.a (mon.0) 833 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:32.718 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:32.286056+0000 mon.a (mon.0) 834 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:32.718 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:32.287300+0000 mon.b (mon.2) 72 : audit [INF] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:28:32.718 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:32.287666+0000 mon.a (mon.0) 835 : audit [INF] from='mgr.24752 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:28:32.719 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:32.288305+0000 mon.b (mon.2) 73 : audit [INF] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:28:32.719 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:32.288665+0000 mon.a (mon.0) 836 : audit [INF] from='mgr.24752 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:28:32.719 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:32.289230+0000 mon.b (mon.2) 74 : audit [INF] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:28:32.719 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:32.289630+0000 mon.a (mon.0) 837 : audit [INF] from='mgr.24752 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:28:32.720 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:32.290176+0000 mon.b (mon.2) 75 : audit [INF] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:28:32.720 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:32.290524+0000 mon.a (mon.0) 838 : audit [INF] from='mgr.24752 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:28:32.720 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:32.295306+0000 mon.a (mon.0) 839 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:33.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:28:30.872219+0000 mgr.x (mgr.24752) 18 : cluster [DBG] pgmap v10: 105 pgs: 105 active+clean; 581 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 10 KiB/s rd, 240 KiB/s wr, 27 op/s 2022-09-29T05:28:33.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:28:30.988863+0000 mgr.x (mgr.24752) 19 : cluster [DBG] pgmap v11: 105 pgs: 105 active+clean; 581 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 265 KiB/s wr, 30 op/s 2022-09-29T05:28:33.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:28:31.586989+0000 mon.a (mon.0) 832 : cluster [WRN] Health check failed: Telemetry requires re-opt-in (TELEMETRY_CHANGED) 2022-09-29T05:28:33.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:32.277886+0000 mon.a (mon.0) 833 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:33.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:32.286056+0000 mon.a (mon.0) 834 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:33.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:32.287300+0000 mon.b (mon.2) 72 : audit [INF] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:28:33.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:32.287666+0000 mon.a (mon.0) 835 : audit [INF] from='mgr.24752 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:28:33.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:32.288305+0000 mon.b (mon.2) 73 : audit [INF] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:28:33.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:32.288665+0000 mon.a (mon.0) 836 : audit [INF] from='mgr.24752 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:28:33.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:32.289230+0000 mon.b (mon.2) 74 : audit [INF] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:28:33.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:32.289630+0000 mon.a (mon.0) 837 : audit [INF] from='mgr.24752 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:28:33.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:32.290176+0000 mon.b (mon.2) 75 : audit [INF] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:28:33.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:32.290524+0000 mon.a (mon.0) 838 : audit [INF] from='mgr.24752 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2022-09-29T05:28:33.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:32.295306+0000 mon.a (mon.0) 839 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:33.895 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:28:32.272427+0000 mgr.x (mgr.24752) 20 : cephadm [INF] Detected new or changed devices on smithi153 2022-09-29T05:28:33.895 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:28:32.290977+0000 mgr.x (mgr.24752) 21 : cephadm [INF] Adjusting osd_memory_target on smithi153 to 2500M 2022-09-29T05:28:33.896 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:28:32.295822+0000 mgr.x (mgr.24752) 22 : cephadm [INF] Updating smithi153:/etc/ceph/ceph.conf 2022-09-29T05:28:33.896 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:28:32.508367+0000 mgr.x (mgr.24752) 23 : cephadm [INF] Updating smithi153:/var/lib/ceph/69111522-3fb6-11ed-8431-001a4aab830c/config/ceph.conf 2022-09-29T05:28:33.896 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:28:32.716215+0000 mgr.x (mgr.24752) 24 : cephadm [INF] Updating smithi153:/etc/ceph/ceph.client.admin.keyring 2022-09-29T05:28:33.896 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:33.139261+0000 mon.a (mon.0) 840 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:33.896 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:33.147730+0000 mon.a (mon.0) 841 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:33.897 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:33.152883+0000 mon.a (mon.0) 842 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:33.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:28:32.272427+0000 mgr.x (mgr.24752) 20 : cephadm [INF] Detected new or changed devices on smithi153 2022-09-29T05:28:33.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:28:32.290977+0000 mgr.x (mgr.24752) 21 : cephadm [INF] Adjusting osd_memory_target on smithi153 to 2500M 2022-09-29T05:28:33.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:28:32.295822+0000 mgr.x (mgr.24752) 22 : cephadm [INF] Updating smithi153:/etc/ceph/ceph.conf 2022-09-29T05:28:33.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:28:32.508367+0000 mgr.x (mgr.24752) 23 : cephadm [INF] Updating smithi153:/var/lib/ceph/69111522-3fb6-11ed-8431-001a4aab830c/config/ceph.conf 2022-09-29T05:28:33.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:28:32.716215+0000 mgr.x (mgr.24752) 24 : cephadm [INF] Updating smithi153:/etc/ceph/ceph.client.admin.keyring 2022-09-29T05:28:33.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:33.139261+0000 mon.a (mon.0) 840 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:33.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:33.147730+0000 mon.a (mon.0) 841 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:33.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:33.152883+0000 mon.a (mon.0) 842 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:33.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:28:32.272427+0000 mgr.x (mgr.24752) 20 : cephadm [INF] Detected new or changed devices on smithi153 2022-09-29T05:28:33.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:28:32.290977+0000 mgr.x (mgr.24752) 21 : cephadm [INF] Adjusting osd_memory_target on smithi153 to 2500M 2022-09-29T05:28:33.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:28:32.295822+0000 mgr.x (mgr.24752) 22 : cephadm [INF] Updating smithi153:/etc/ceph/ceph.conf 2022-09-29T05:28:33.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:28:32.508367+0000 mgr.x (mgr.24752) 23 : cephadm [INF] Updating smithi153:/var/lib/ceph/69111522-3fb6-11ed-8431-001a4aab830c/config/ceph.conf 2022-09-29T05:28:33.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:28:32.716215+0000 mgr.x (mgr.24752) 24 : cephadm [INF] Updating smithi153:/etc/ceph/ceph.client.admin.keyring 2022-09-29T05:28:33.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:33.139261+0000 mon.a (mon.0) 840 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:33.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:33.147730+0000 mon.a (mon.0) 841 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:33.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:33.152883+0000 mon.a (mon.0) 842 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:34.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:28:32.923839+0000 mgr.x (mgr.24752) 25 : cephadm [INF] Updating smithi153:/var/lib/ceph/69111522-3fb6-11ed-8431-001a4aab830c/config/ceph.client.admin.keyring 2022-09-29T05:28:34.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:28:32.989614+0000 mgr.x (mgr.24752) 26 : cluster [DBG] pgmap v12: 105 pgs: 105 active+clean; 581 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 9.9 KiB/s rd, 239 KiB/s wr, 27 op/s 2022-09-29T05:28:34.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:28:33.226047+0000 mgr.x (mgr.24752) 27 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2022-09-29T05:28:34.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:28:33.232958+0000 mgr.x (mgr.24752) 28 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi153 2022-09-29T05:28:34.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:28:32.923839+0000 mgr.x (mgr.24752) 25 : cephadm [INF] Updating smithi153:/var/lib/ceph/69111522-3fb6-11ed-8431-001a4aab830c/config/ceph.client.admin.keyring 2022-09-29T05:28:34.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:28:32.989614+0000 mgr.x (mgr.24752) 26 : cluster [DBG] pgmap v12: 105 pgs: 105 active+clean; 581 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 9.9 KiB/s rd, 239 KiB/s wr, 27 op/s 2022-09-29T05:28:34.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:28:33.226047+0000 mgr.x (mgr.24752) 27 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2022-09-29T05:28:34.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:28:33.232958+0000 mgr.x (mgr.24752) 28 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi153 2022-09-29T05:28:35.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:34 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:28:32.923839+0000 mgr.x (mgr.24752) 25 : cephadm [INF] Updating smithi153:/var/lib/ceph/69111522-3fb6-11ed-8431-001a4aab830c/config/ceph.client.admin.keyring 2022-09-29T05:28:35.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:34 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:28:32.989614+0000 mgr.x (mgr.24752) 26 : cluster [DBG] pgmap v12: 105 pgs: 105 active+clean; 581 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 9.9 KiB/s rd, 239 KiB/s wr, 27 op/s 2022-09-29T05:28:35.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:34 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:28:33.226047+0000 mgr.x (mgr.24752) 27 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2022-09-29T05:28:35.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:34 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:28:33.232958+0000 mgr.x (mgr.24752) 28 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi153 2022-09-29T05:28:36.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:28:34.990193+0000 mgr.x (mgr.24752) 29 : cluster [DBG] pgmap v13: 105 pgs: 105 active+clean; 581 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 9.9 KiB/s rd, 239 KiB/s wr, 27 op/s 2022-09-29T05:28:36.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:28:34.990193+0000 mgr.x (mgr.24752) 29 : cluster [DBG] pgmap v13: 105 pgs: 105 active+clean; 581 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 9.9 KiB/s rd, 239 KiB/s wr, 27 op/s 2022-09-29T05:28:36.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:35 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:28:34.990193+0000 mgr.x (mgr.24752) 29 : cluster [DBG] pgmap v13: 105 pgs: 105 active+clean; 581 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 9.9 KiB/s rd, 239 KiB/s wr, 27 op/s 2022-09-29T05:28:37.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:36 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:35.947499+0000 mon.a (mon.0) 843 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:37.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:35.947499+0000 mon.a (mon.0) 843 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:37.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:35.947499+0000 mon.a (mon.0) 843 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:38.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:37 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:28:36.990959+0000 mgr.x (mgr.24752) 30 : cluster [DBG] pgmap v14: 105 pgs: 105 active+clean; 581 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 9.9 KiB/s rd, 239 KiB/s wr, 27 op/s 2022-09-29T05:28:38.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:37 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:28:36.990959+0000 mgr.x (mgr.24752) 30 : cluster [DBG] pgmap v14: 105 pgs: 105 active+clean; 581 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 9.9 KiB/s rd, 239 KiB/s wr, 27 op/s 2022-09-29T05:28:38.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:37 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:28:36.990959+0000 mgr.x (mgr.24752) 30 : cluster [DBG] pgmap v14: 105 pgs: 105 active+clean; 581 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 9.9 KiB/s rd, 239 KiB/s wr, 27 op/s 2022-09-29T05:28:39.559 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:28:39 smithi153 systemd[1]: Stopping Ceph prometheus.a for 69111522-3fb6-11ed-8431-001a4aab830c... 2022-09-29T05:28:39.560 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:28:39 smithi153 bash[62229]: Error: no container with name or ID "ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus.a" found: no such container 2022-09-29T05:28:40.051 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:28:39 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[57164]: ts=2022-09-29T05:28:39.597Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2022-09-29T05:28:40.051 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:28:39 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[57164]: ts=2022-09-29T05:28:39.597Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2022-09-29T05:28:40.051 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:28:39 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[57164]: ts=2022-09-29T05:28:39.597Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2022-09-29T05:28:40.052 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:28:39 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[57164]: ts=2022-09-29T05:28:39.597Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2022-09-29T05:28:40.052 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:28:39 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[57164]: ts=2022-09-29T05:28:39.597Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2022-09-29T05:28:40.052 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:28:39 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[57164]: ts=2022-09-29T05:28:39.597Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2022-09-29T05:28:40.052 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:28:39 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[57164]: ts=2022-09-29T05:28:39.598Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2022-09-29T05:28:40.053 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:28:39 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[57164]: ts=2022-09-29T05:28:39.598Z caller=main.go:828 level=info msg="Scrape manager stopped" 2022-09-29T05:28:40.053 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:28:39 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[57164]: ts=2022-09-29T05:28:39.598Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2022-09-29T05:28:40.053 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:28:39 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[57164]: ts=2022-09-29T05:28:39.598Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2022-09-29T05:28:40.053 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:28:39 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[57164]: ts=2022-09-29T05:28:39.598Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2022-09-29T05:28:40.053 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:28:39 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[57164]: ts=2022-09-29T05:28:39.598Z caller=main.go:1066 level=info msg="See you next time!" 2022-09-29T05:28:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:40 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:28:38.991555+0000 mgr.x (mgr.24752) 31 : cluster [DBG] pgmap v15: 105 pgs: 105 active+clean; 581 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 910 B/s rd, 26 KiB/s wr, 2 op/s 2022-09-29T05:28:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:28:38.991555+0000 mgr.x (mgr.24752) 31 : cluster [DBG] pgmap v15: 105 pgs: 105 active+clean; 581 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 910 B/s rd, 26 KiB/s wr, 2 op/s 2022-09-29T05:28:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:28:38.991555+0000 mgr.x (mgr.24752) 31 : cluster [DBG] pgmap v15: 105 pgs: 105 active+clean; 581 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 910 B/s rd, 26 KiB/s wr, 2 op/s 2022-09-29T05:28:41.059 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:28:40 smithi153 bash[62243]: ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a 2022-09-29T05:28:42.056 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:28:41 smithi153 bash[62285]: Error: no container with name or ID "ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus.a" found: no such container 2022-09-29T05:28:42.056 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:28:41 smithi153 systemd[1]: ceph-69111522-3fb6-11ed-8431-001a4aab830c@prometheus.a.service: Succeeded. 2022-09-29T05:28:42.057 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:28:41 smithi153 systemd[1]: Stopped Ceph prometheus.a for 69111522-3fb6-11ed-8431-001a4aab830c. 2022-09-29T05:28:42.057 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:28:41 smithi153 systemd[1]: Starting Ceph prometheus.a for 69111522-3fb6-11ed-8431-001a4aab830c... 2022-09-29T05:28:42.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:28:40.992194+0000 mgr.x (mgr.24752) 32 : cluster [DBG] pgmap v16: 105 pgs: 105 active+clean; 581 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 910 B/s rd, 26 KiB/s wr, 2 op/s 2022-09-29T05:28:42.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:28:40.992194+0000 mgr.x (mgr.24752) 32 : cluster [DBG] pgmap v16: 105 pgs: 105 active+clean; 581 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 910 B/s rd, 26 KiB/s wr, 2 op/s 2022-09-29T05:28:42.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:28:40.992194+0000 mgr.x (mgr.24752) 32 : cluster [DBG] pgmap v16: 105 pgs: 105 active+clean; 581 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 910 B/s rd, 26 KiB/s wr, 2 op/s 2022-09-29T05:28:43.139 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:28:42 smithi153 podman[62358]: 2022-09-29T05:28:43.139 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:28:43 smithi153 bash[62358]: 7d12806b2b388e94b8ca7dbf7cd0ed3146f2d7cb690528ebb4fa1ce6293879dd 2022-09-29T05:28:43.139 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:28:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[62373]: ts=2022-09-29T05:28:43.092Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2022-09-29T05:28:43.140 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:28:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[62373]: ts=2022-09-29T05:28:43.092Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2022-09-29T05:28:43.140 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:28:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[62373]: ts=2022-09-29T05:28:43.092Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2022-09-29T05:28:43.140 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:28:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[62373]: ts=2022-09-29T05:28:43.092Z caller=main.go:518 level=info host_details="(Linux 4.18.0-408.el8.x86_64 #1 SMP Mon Jul 18 17:42:52 UTC 2022 x86_64 smithi153 (none))" 2022-09-29T05:28:43.140 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:28:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[62373]: ts=2022-09-29T05:28:43.092Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2022-09-29T05:28:43.141 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:28:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[62373]: ts=2022-09-29T05:28:43.092Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2022-09-29T05:28:43.141 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:28:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[62373]: ts=2022-09-29T05:28:43.094Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2022-09-29T05:28:43.141 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:28:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[62373]: ts=2022-09-29T05:28:43.094Z caller=main.go:923 level=info msg="Starting TSDB ..." 2022-09-29T05:28:43.141 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:28:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[62373]: ts=2022-09-29T05:28:43.095Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2022-09-29T05:28:43.142 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:28:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[62373]: ts=2022-09-29T05:28:43.096Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2022-09-29T05:28:43.142 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:28:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[62373]: ts=2022-09-29T05:28:43.096Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.499µs 2022-09-29T05:28:43.142 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:28:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[62373]: ts=2022-09-29T05:28:43.096Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2022-09-29T05:28:43.439 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:28:43 smithi153 systemd[1]: Started Ceph prometheus.a for 69111522-3fb6-11ed-8431-001a4aab830c. 2022-09-29T05:28:43.439 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:28:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[62373]: ts=2022-09-29T05:28:43.262Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2022-09-29T05:28:43.440 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:28:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: [29/Sep/2022:05:28:43] ENGINE Bus STOPPING 2022-09-29T05:28:43.440 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:28:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: [29/Sep/2022:05:28:43] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2022-09-29T05:28:43.441 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:28:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: [29/Sep/2022:05:28:43] ENGINE Bus STOPPED 2022-09-29T05:28:43.441 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:28:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: [29/Sep/2022:05:28:43] ENGINE Bus STARTING 2022-09-29T05:28:43.441 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:28:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: [29/Sep/2022:05:28:43] ENGINE Serving on http://:::9283 2022-09-29T05:28:43.441 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:28:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: [29/Sep/2022:05:28:43] ENGINE Bus STARTED 2022-09-29T05:28:43.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:28:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[62373]: ts=2022-09-29T05:28:43.436Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2022-09-29T05:28:43.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:28:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[62373]: ts=2022-09-29T05:28:43.436Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2022-09-29T05:28:43.810 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:28:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[62373]: ts=2022-09-29T05:28:43.436Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=31.061µs wal_replay_duration=339.799361ms total_replay_duration=339.846914ms 2022-09-29T05:28:43.810 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:28:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[62373]: ts=2022-09-29T05:28:43.439Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2022-09-29T05:28:43.810 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:28:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[62373]: ts=2022-09-29T05:28:43.439Z caller=main.go:947 level=info msg="TSDB started" 2022-09-29T05:28:43.811 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:28:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[62373]: ts=2022-09-29T05:28:43.439Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2022-09-29T05:28:43.811 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:28:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[62373]: ts=2022-09-29T05:28:43.456Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=17.048377ms db_storage=688ns remote_storage=1.188µs web_handler=720ns query_engine=665ns scrape=414.408µs scrape_sd=179.596µs notify=11.795µs notify_sd=57.44µs rules=15.976961ms 2022-09-29T05:28:43.811 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:28:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[62373]: ts=2022-09-29T05:28:43.456Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2022-09-29T05:28:44.273 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:28:42.992855+0000 mgr.x (mgr.24752) 33 : cluster [DBG] pgmap v17: 105 pgs: 105 active+clean; 581 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 1 op/s 2022-09-29T05:28:44.273 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:43.164154+0000 mon.a (mon.0) 844 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:44.273 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:43.169874+0000 mon.a (mon.0) 845 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:44.274 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:43.173660+0000 mon.b (mon.2) 76 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2022-09-29T05:28:44.274 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:43.174126+0000 mgr.x (mgr.24752) 34 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2022-09-29T05:28:44.274 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:43.174930+0000 mon.b (mon.2) 77 : audit [INF] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi153:9095"}]: dispatch 2022-09-29T05:28:44.274 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:43.175243+0000 mgr.x (mgr.24752) 35 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi153:9095"}]: dispatch 2022-09-29T05:28:44.275 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:43.179622+0000 mon.a (mon.0) 846 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:44.275 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:43.182859+0000 mon.b (mon.2) 78 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:28:44.275 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:28:43.185338+0000 mgr.x (mgr.24752) 36 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2022-09-29T05:28:44.275 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:28:43.185875+0000 mgr.x (mgr.24752) 37 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2022-09-29T05:28:44.276 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:28:42.992855+0000 mgr.x (mgr.24752) 33 : cluster [DBG] pgmap v17: 105 pgs: 105 active+clean; 581 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 1 op/s 2022-09-29T05:28:44.276 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:43.164154+0000 mon.a (mon.0) 844 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:44.276 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:43.169874+0000 mon.a (mon.0) 845 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:44.277 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:43.173660+0000 mon.b (mon.2) 76 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2022-09-29T05:28:44.277 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:43.174126+0000 mgr.x (mgr.24752) 34 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2022-09-29T05:28:44.277 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:43.174930+0000 mon.b (mon.2) 77 : audit [INF] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi153:9095"}]: dispatch 2022-09-29T05:28:44.277 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:43.175243+0000 mgr.x (mgr.24752) 35 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi153:9095"}]: dispatch 2022-09-29T05:28:44.277 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:43.179622+0000 mon.a (mon.0) 846 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:44.278 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:43.182859+0000 mon.b (mon.2) 78 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:28:44.278 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:28:43.185338+0000 mgr.x (mgr.24752) 36 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2022-09-29T05:28:44.278 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:28:43.185875+0000 mgr.x (mgr.24752) 37 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2022-09-29T05:28:44.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:44 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:28:42.992855+0000 mgr.x (mgr.24752) 33 : cluster [DBG] pgmap v17: 105 pgs: 105 active+clean; 581 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 1 op/s 2022-09-29T05:28:44.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:44 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:43.164154+0000 mon.a (mon.0) 844 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:44.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:44 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:43.169874+0000 mon.a (mon.0) 845 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:44.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:44 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:43.173660+0000 mon.b (mon.2) 76 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2022-09-29T05:28:44.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:44 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:43.174126+0000 mgr.x (mgr.24752) 34 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2022-09-29T05:28:44.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:44 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:43.174930+0000 mon.b (mon.2) 77 : audit [INF] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi153:9095"}]: dispatch 2022-09-29T05:28:44.561 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:44 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:43.175243+0000 mgr.x (mgr.24752) 35 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi153:9095"}]: dispatch 2022-09-29T05:28:44.561 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:44 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:43.179622+0000 mon.a (mon.0) 846 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:44.561 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:44 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:43.182859+0000 mon.b (mon.2) 78 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:28:44.562 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:44 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:28:43.185338+0000 mgr.x (mgr.24752) 36 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2022-09-29T05:28:44.562 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:44 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:28:43.185875+0000 mgr.x (mgr.24752) 37 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2022-09-29T05:28:46.041 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:28:44.776101+0000 mgr.x (mgr.24752) 38 : cephadm [INF] Upgrade: Updating mgr.y 2022-09-29T05:28:46.042 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:44.780756+0000 mon.a (mon.0) 847 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:46.042 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:44.782741+0000 mon.b (mon.2) 79 : audit [INF] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2022-09-29T05:28:46.043 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:44.783257+0000 mon.a (mon.0) 848 : audit [INF] from='mgr.24752 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2022-09-29T05:28:46.043 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:44.784308+0000 mon.b (mon.2) 80 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2022-09-29T05:28:46.043 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:44.785223+0000 mon.b (mon.2) 81 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:28:46.043 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:28:44.786188+0000 mgr.x (mgr.24752) 39 : cephadm [INF] Deploying daemon mgr.y on smithi083 2022-09-29T05:28:46.044 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:28:44.776101+0000 mgr.x (mgr.24752) 38 : cephadm [INF] Upgrade: Updating mgr.y 2022-09-29T05:28:46.044 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:44.780756+0000 mon.a (mon.0) 847 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:46.044 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:44.782741+0000 mon.b (mon.2) 79 : audit [INF] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2022-09-29T05:28:46.045 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:44.783257+0000 mon.a (mon.0) 848 : audit [INF] from='mgr.24752 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2022-09-29T05:28:46.045 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:44.784308+0000 mon.b (mon.2) 80 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2022-09-29T05:28:46.045 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:44.785223+0000 mon.b (mon.2) 81 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:28:46.045 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:28:44.786188+0000 mgr.x (mgr.24752) 39 : cephadm [INF] Deploying daemon mgr.y on smithi083 2022-09-29T05:28:46.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:45 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:28:44.776101+0000 mgr.x (mgr.24752) 38 : cephadm [INF] Upgrade: Updating mgr.y 2022-09-29T05:28:46.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:45 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:44.780756+0000 mon.a (mon.0) 847 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:46.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:45 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:44.782741+0000 mon.b (mon.2) 79 : audit [INF] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2022-09-29T05:28:46.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:45 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:44.783257+0000 mon.a (mon.0) 848 : audit [INF] from='mgr.24752 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2022-09-29T05:28:46.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:45 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:44.784308+0000 mon.b (mon.2) 80 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2022-09-29T05:28:46.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:45 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:44.785223+0000 mon.b (mon.2) 81 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:28:46.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:45 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:28:44.786188+0000 mgr.x (mgr.24752) 39 : cephadm [INF] Deploying daemon mgr.y on smithi083 2022-09-29T05:28:46.463 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:46 smithi083 systemd[1]: Stopping Ceph mgr.y for 69111522-3fb6-11ed-8431-001a4aab830c... 2022-09-29T05:28:47.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:46 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:28:44.993272+0000 mgr.x (mgr.24752) 40 : cluster [DBG] pgmap v18: 105 pgs: 105 active+clean; 581 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:28:47.120 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:28:44.993272+0000 mgr.x (mgr.24752) 40 : cluster [DBG] pgmap v18: 105 pgs: 105 active+clean; 581 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:28:47.121 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:46 smithi083 bash[72924]: ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y 2022-09-29T05:28:47.121 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:47 smithi083 systemd[1]: ceph-69111522-3fb6-11ed-8431-001a4aab830c@mgr.y.service: Main process exited, code=exited, status=143/n/a 2022-09-29T05:28:47.122 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:28:44.993272+0000 mgr.x (mgr.24752) 40 : cluster [DBG] pgmap v18: 105 pgs: 105 active+clean; 581 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:28:47.463 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:47 smithi083 systemd[1]: ceph-69111522-3fb6-11ed-8431-001a4aab830c@mgr.y.service: Failed with result 'exit-code'. 2022-09-29T05:28:47.463 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:47 smithi083 systemd[1]: Stopped Ceph mgr.y for 69111522-3fb6-11ed-8431-001a4aab830c. 2022-09-29T05:28:47.463 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:47 smithi083 systemd[1]: Starting Ceph mgr.y for 69111522-3fb6-11ed-8431-001a4aab830c... 2022-09-29T05:28:48.196 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:47 smithi083 podman[73051]: 2022-09-29T05:28:48.463 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:48 smithi083 bash[73051]: 4ff0215b55e85519e5f2170dff313318073b642d8cceda5cba44e4682ae70f2f 2022-09-29T05:28:48.463 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:48 smithi083 ceph-mgr[73090]: -- 172.21.15.83:0/1506078336 <== mon.1 v2:172.21.15.83:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55770d544820 con 0x55770d519000 2022-09-29T05:28:48.463 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:48 smithi083 systemd[1]: Started Ceph mgr.y for 69111522-3fb6-11ed-8431-001a4aab830c. 2022-09-29T05:28:48.464 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:48 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: 2022-09-29T05:28:48.410+0000 7f4cf2fe8200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2022-09-29T05:28:48.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:48 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: 2022-09-29T05:28:48.623+0000 7f4cf2fe8200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2022-09-29T05:28:48.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:48 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:28:46.994136+0000 mgr.x (mgr.24752) 41 : cluster [DBG] pgmap v19: 105 pgs: 105 active+clean; 581 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:28:48.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:48 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:48.253113+0000 mon.a (mon.0) 849 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:48.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:48 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:48.259966+0000 mon.a (mon.0) 850 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:48.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:48 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:48.261362+0000 mon.b (mon.2) 82 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:28:48.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:48 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:48.262489+0000 mon.b (mon.2) 83 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:28:48.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:48 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:48.263216+0000 mon.b (mon.2) 84 : audit [INF] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:28:48.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:48 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:28:46.994136+0000 mgr.x (mgr.24752) 41 : cluster [DBG] pgmap v19: 105 pgs: 105 active+clean; 581 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:28:48.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:48 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:48.253113+0000 mon.a (mon.0) 849 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:48.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:48 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:48.259966+0000 mon.a (mon.0) 850 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:48.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:48 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:48.261362+0000 mon.b (mon.2) 82 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:28:48.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:48 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:48.262489+0000 mon.b (mon.2) 83 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:28:48.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:48 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:48.263216+0000 mon.b (mon.2) 84 : audit [INF] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:28:49.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:48 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:28:46.994136+0000 mgr.x (mgr.24752) 41 : cluster [DBG] pgmap v19: 105 pgs: 105 active+clean; 581 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:28:49.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:48 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:48.253113+0000 mon.a (mon.0) 849 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:49.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:48 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:48.259966+0000 mon.a (mon.0) 850 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:49.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:48 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:48.261362+0000 mon.b (mon.2) 82 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:28:49.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:48 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:48.262489+0000 mon.b (mon.2) 83 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:28:49.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:48 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:48.263216+0000 mon.b (mon.2) 84 : audit [INF] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:28:49.206 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T05:28:49.513 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T05:28:49.513 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2m) 23s ago 3m 26.3M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T05:28:49.514 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2m) 21s ago 2m 60.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T05:28:49.514 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3m) 23s ago 3m 14.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T05:28:49.514 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3m) 21s ago 3m 17.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T05:28:49.514 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (36s) 21s ago 6m 458M - 18.0.0-128-g652bf754 6d6412b0c71e 1b5d2346e321 2022-09-29T05:28:49.515 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 starting - - - - 2022-09-29T05:28:49.515 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8m) 23s ago 8m 99.3M 2048M 16.2.10 c56e13ff90f2 0e7e560a682f 2022-09-29T05:28:49.515 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6m) 21s ago 6m 78.0M 2048M 16.2.10 c56e13ff90f2 b96ee39cc34f 2022-09-29T05:28:49.515 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6m) 23s ago 6m 74.9M 2048M 16.2.10 c56e13ff90f2 a7179c86361b 2022-09-29T05:28:49.516 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3m) 23s ago 3m 25.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T05:28:49.516 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3m) 21s ago 3m 24.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T05:28:49.516 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6m) 23s ago 6m 58.4M 2500M 16.2.10 c56e13ff90f2 c12a21fd5897 2022-09-29T05:28:49.516 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5m) 23s ago 5m 57.4M 2500M 16.2.10 c56e13ff90f2 989e877de30a 2022-09-29T05:28:49.516 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5m) 23s ago 5m 51.1M 2500M 16.2.10 c56e13ff90f2 81b9b51f0eee 2022-09-29T05:28:49.517 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5m) 23s ago 5m 56.7M 2500M 16.2.10 c56e13ff90f2 4863a2083356 2022-09-29T05:28:49.517 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4m) 21s ago 4m 54.1M 2500M 16.2.10 c56e13ff90f2 ed61dc9353b0 2022-09-29T05:28:49.517 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4m) 21s ago 4m 60.7M 2500M 16.2.10 c56e13ff90f2 59f19fd15fed 2022-09-29T05:28:49.517 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4m) 21s ago 4m 59.3M 2500M 16.2.10 c56e13ff90f2 23decfd9c6b2 2022-09-29T05:28:49.518 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4m) 21s ago 4m 58.2M 2500M 16.2.10 c56e13ff90f2 23460a09c0a7 2022-09-29T05:28:49.518 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 starting - - - - 2022-09-29T05:28:49.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: 2022-09-29T05:28:49.450+0000 7f4cf2fe8200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2022-09-29T05:28:49.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: 2022-09-29T05:28:49.637+0000 7f4cf2fe8200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2022-09-29T05:28:49.892 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T05:28:49.892 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T05:28:49.893 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 3 2022-09-29T05:28:49.893 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:28:49.893 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T05:28:49.894 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 1, 2022-09-29T05:28:49.894 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 1 2022-09-29T05:28:49.895 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:28:49.895 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T05:28:49.896 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 8 2022-09-29T05:28:49.896 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:28:49.896 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T05:28:49.897 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T05:28:49.897 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:28:49.898 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T05:28:49.898 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 14, 2022-09-29T05:28:49.898 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 1 2022-09-29T05:28:49.899 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T05:28:49.899 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T05:28:50.213 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: 2022-09-29T05:28:49.886+0000 7f4cf2fe8200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2022-09-29T05:28:50.213 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: 2022-09-29T05:28:50.062+0000 7f4cf2fe8200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2022-09-29T05:28:50.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: 2022-09-29T05:28:50.406+0000 7f4cf2fe8200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2022-09-29T05:28:50.988 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: 2022-09-29T05:28:50.715+0000 7f4cf2fe8200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2022-09-29T05:28:50.988 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:28:48.994719+0000 mgr.x (mgr.24752) 42 : cluster [DBG] pgmap v20: 105 pgs: 105 active+clean; 581 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:28:50.989 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:49.189006+0000 mgr.x (mgr.24752) 43 : audit [DBG] from='client.24925 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:28:50.989 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:49.502607+0000 mgr.x (mgr.24752) 44 : audit [DBG] from='client.24931 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:28:50.989 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:49.888975+0000 mon.c (mon.1) 41 : audit [DBG] from='client.? 172.21.15.83:0/2707318507' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:28:50.989 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:28:48.994719+0000 mgr.x (mgr.24752) 42 : cluster [DBG] pgmap v20: 105 pgs: 105 active+clean; 581 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:28:50.990 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:49.189006+0000 mgr.x (mgr.24752) 43 : audit [DBG] from='client.24925 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:28:50.990 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:49.502607+0000 mgr.x (mgr.24752) 44 : audit [DBG] from='client.24931 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:28:50.990 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:49.888975+0000 mon.c (mon.1) 41 : audit [DBG] from='client.? 172.21.15.83:0/2707318507' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:28:51.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:50 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:28:48.994719+0000 mgr.x (mgr.24752) 42 : cluster [DBG] pgmap v20: 105 pgs: 105 active+clean; 581 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:28:51.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:50 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:49.189006+0000 mgr.x (mgr.24752) 43 : audit [DBG] from='client.24925 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:28:51.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:50 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:49.502607+0000 mgr.x (mgr.24752) 44 : audit [DBG] from='client.24931 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:28:51.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:50 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:49.888975+0000 mon.c (mon.1) 41 : audit [DBG] from='client.? 172.21.15.83:0/2707318507' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:28:51.264 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: 2022-09-29T05:28:50.984+0000 7f4cf2fe8200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2022-09-29T05:28:51.264 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: 2022-09-29T05:28:51.079+0000 7f4cf2fe8200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2022-09-29T05:28:51.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: 2022-09-29T05:28:51.260+0000 7f4cf2fe8200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2022-09-29T05:28:51.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: 2022-09-29T05:28:51.348+0000 7f4cf2fe8200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2022-09-29T05:28:52.182 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: 2022-09-29T05:28:51.762+0000 7f4cf2fe8200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2022-09-29T05:28:52.183 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: 2022-09-29T05:28:51.861+0000 7f4cf2fe8200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2022-09-29T05:28:52.183 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: 2022-09-29T05:28:51.951+0000 7f4cf2fe8200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2022-09-29T05:28:52.794 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:28:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[62373]: ts=2022-09-29T05:28:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi153.front.sepia.ceph.com:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:28:53.059 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:28:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:28:53] "GET /metrics HTTP/1.1" 200 176112 "" "Prometheus/2.33.4" 2022-09-29T05:28:53.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:28:50.995162+0000 mgr.x (mgr.24752) 45 : cluster [DBG] pgmap v21: 105 pgs: 105 active+clean; 581 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:28:53.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:52.718765+0000 mon.a (mon.0) 851 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:53.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:52.724912+0000 mon.a (mon.0) 852 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:53.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:52.754218+0000 mon.a (mon.0) 853 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:53.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:52.762876+0000 mon.a (mon.0) 854 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:53.127 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: 2022-09-29T05:28:52.178+0000 7f4cf2fe8200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2022-09-29T05:28:53.127 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: 2022-09-29T05:28:52.927+0000 7f4cf2fe8200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2022-09-29T05:28:53.128 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: 2022-09-29T05:28:53.024+0000 7f4cf2fe8200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2022-09-29T05:28:53.128 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:28:50.995162+0000 mgr.x (mgr.24752) 45 : cluster [DBG] pgmap v21: 105 pgs: 105 active+clean; 581 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:28:53.128 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:52.718765+0000 mon.a (mon.0) 851 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:53.129 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:52.724912+0000 mon.a (mon.0) 852 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:53.129 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:52.754218+0000 mon.a (mon.0) 853 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:53.129 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:52.762876+0000 mon.a (mon.0) 854 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:53.129 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:28:50.995162+0000 mgr.x (mgr.24752) 45 : cluster [DBG] pgmap v21: 105 pgs: 105 active+clean; 581 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:28:53.130 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:52.718765+0000 mon.a (mon.0) 851 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:53.130 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:52.724912+0000 mon.a (mon.0) 852 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:53.130 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:52.754218+0000 mon.a (mon.0) 853 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:53.130 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:52.762876+0000 mon.a (mon.0) 854 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:53.380 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: 2022-09-29T05:28:53.123+0000 7f4cf2fe8200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2022-09-29T05:28:53.381 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: 2022-09-29T05:28:53.214+0000 7f4cf2fe8200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2022-09-29T05:28:53.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: 2022-09-29T05:28:53.376+0000 7f4cf2fe8200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2022-09-29T05:28:54.702 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: 2022-09-29T05:28:54.435+0000 7f4cf2fe8200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2022-09-29T05:28:54.702 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: 2022-09-29T05:28:54.564+0000 7f4cf2fe8200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2022-09-29T05:28:54.703 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: 2022-09-29T05:28:54.660+0000 7f4cf2fe8200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2022-09-29T05:28:54.703 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: [29/Sep/2022:05:28:54] ENGINE Bus STARTING 2022-09-29T05:28:54.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: CherryPy Checker: 2022-09-29T05:28:54.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: The Application mounted at '' has an empty config. 2022-09-29T05:28:54.964 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: 2022-09-29T05:28:54.964 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: [29/Sep/2022:05:28:54] ENGINE Serving on http://:::9283 2022-09-29T05:28:54.965 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: [29/Sep/2022:05:28:54] ENGINE Bus STARTED 2022-09-29T05:28:54.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:28:52.995947+0000 mgr.x (mgr.24752) 46 : cluster [DBG] pgmap v22: 105 pgs: 105 active+clean; 581 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:28:54.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:53.745158+0000 mon.a (mon.0) 855 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:54.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:53.750347+0000 mon.a (mon.0) 856 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:54.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:28:54.675143+0000 mon.a (mon.0) 857 : cluster [DBG] Standby manager daemon y restarted 2022-09-29T05:28:54.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:28:54.675286+0000 mon.a (mon.0) 858 : cluster [DBG] Standby manager daemon y started 2022-09-29T05:28:54.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:54.677820+0000 mon.a (mon.0) 859 : audit [DBG] from='mgr.? 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2022-09-29T05:28:54.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:54.678179+0000 mon.a (mon.0) 860 : audit [DBG] from='mgr.? 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2022-09-29T05:28:54.968 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:54.680963+0000 mon.a (mon.0) 861 : audit [DBG] from='mgr.? 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2022-09-29T05:28:54.968 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:54.681292+0000 mon.a (mon.0) 862 : audit [DBG] from='mgr.? 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2022-09-29T05:28:54.969 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:28:52.995947+0000 mgr.x (mgr.24752) 46 : cluster [DBG] pgmap v22: 105 pgs: 105 active+clean; 581 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:28:54.970 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:53.745158+0000 mon.a (mon.0) 855 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:54.970 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:53.750347+0000 mon.a (mon.0) 856 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:54.971 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:28:54.675143+0000 mon.a (mon.0) 857 : cluster [DBG] Standby manager daemon y restarted 2022-09-29T05:28:54.971 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:28:54.675286+0000 mon.a (mon.0) 858 : cluster [DBG] Standby manager daemon y started 2022-09-29T05:28:54.971 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:54.677820+0000 mon.a (mon.0) 859 : audit [DBG] from='mgr.? 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2022-09-29T05:28:54.972 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:54.678179+0000 mon.a (mon.0) 860 : audit [DBG] from='mgr.? 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2022-09-29T05:28:54.972 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:54.680963+0000 mon.a (mon.0) 861 : audit [DBG] from='mgr.? 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2022-09-29T05:28:54.972 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:54.681292+0000 mon.a (mon.0) 862 : audit [DBG] from='mgr.? 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2022-09-29T05:28:55.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:54 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:28:52.995947+0000 mgr.x (mgr.24752) 46 : cluster [DBG] pgmap v22: 105 pgs: 105 active+clean; 581 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:28:55.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:54 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:53.745158+0000 mon.a (mon.0) 855 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:55.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:54 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:53.750347+0000 mon.a (mon.0) 856 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:55.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:54 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:28:54.675143+0000 mon.a (mon.0) 857 : cluster [DBG] Standby manager daemon y restarted 2022-09-29T05:28:55.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:54 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:28:54.675286+0000 mon.a (mon.0) 858 : cluster [DBG] Standby manager daemon y started 2022-09-29T05:28:55.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:54 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:54.677820+0000 mon.a (mon.0) 859 : audit [DBG] from='mgr.? 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2022-09-29T05:28:55.063 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:54 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:54.678179+0000 mon.a (mon.0) 860 : audit [DBG] from='mgr.? 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2022-09-29T05:28:55.063 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:54 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:54.680963+0000 mon.a (mon.0) 861 : audit [DBG] from='mgr.? 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2022-09-29T05:28:55.064 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:54 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:54.681292+0000 mon.a (mon.0) 862 : audit [DBG] from='mgr.? 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2022-09-29T05:28:55.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:28:54.800156+0000 mon.a (mon.0) 863 : cluster [DBG] mgrmap e27: x(active, since 34s), standbys: y 2022-09-29T05:28:55.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:28:54.800156+0000 mon.a (mon.0) 863 : cluster [DBG] mgrmap e27: x(active, since 34s), standbys: y 2022-09-29T05:28:56.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:55 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:28:54.800156+0000 mon.a (mon.0) 863 : cluster [DBG] mgrmap e27: x(active, since 34s), standbys: y 2022-09-29T05:28:57.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:56 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:28:54.996535+0000 mgr.x (mgr.24752) 47 : cluster [DBG] pgmap v23: 105 pgs: 105 active+clean; 581 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:28:57.059 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:28:56 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: debug 2022-09-29T05:28:56.996+0000 7f82aa894700 -1 client.24794 I was blocklisted at osd epoch 63 2022-09-29T05:28:57.060 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:28:56 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: debug 2022-09-29T05:28:56.996+0000 7f82a608b700 -1 client.24904 I was blocklisted at osd epoch 63 2022-09-29T05:28:57.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:28:54.996535+0000 mgr.x (mgr.24752) 47 : cluster [DBG] pgmap v23: 105 pgs: 105 active+clean; 581 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:28:57.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:28:54.996535+0000 mgr.x (mgr.24752) 47 : cluster [DBG] pgmap v23: 105 pgs: 105 active+clean; 581 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:28:58.167 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:28:57 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: debug 2022-09-29T05:28:57.901+0000 7f833d534700 -1 mgr handle_mgr_map I was active but no longer am 2022-09-29T05:28:58.168 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:28:57 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: ignoring --setuser ceph since I am not root 2022-09-29T05:28:58.168 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:28:57 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: ignoring --setgroup ceph since I am not root 2022-09-29T05:28:58.168 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:28:57 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: debug 2022-09-29T05:28:57.952+0000 7fa665c73700 1 -- 172.21.15.153:0/2815132503 <== mon.2 v2:172.21.15.153:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55d1bb26c820 con 0x55d1bb241000 2022-09-29T05:28:58.169 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:57 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:28:56.876543+0000 mgr.x (mgr.24752) 48 : cephadm [INF] Detected new or changed devices on smithi083 2022-09-29T05:28:58.169 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:57 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:56.883700+0000 mon.a (mon.0) 864 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:58.169 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:57 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:56.892164+0000 mon.a (mon.0) 865 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:58.169 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:57 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:56.897575+0000 mon.a (mon.0) 866 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:58.170 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:57 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:56.979670+0000 mon.b (mon.2) 85 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:28:58.170 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:57 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:28:56.982017+0000 mgr.x (mgr.24752) 49 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2022-09-29T05:28:58.170 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:57 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:28:56.982446+0000 mgr.x (mgr.24752) 50 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2022-09-29T05:28:58.171 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:57 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:28:56.984615+0000 mgr.x (mgr.24752) 51 : cephadm [INF] Failing over to other MGR 2022-09-29T05:28:58.171 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:57 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:56.984833+0000 mon.b (mon.2) 86 : audit [INF] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2022-09-29T05:28:58.172 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:57 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:56.985451+0000 mon.a (mon.0) 867 : audit [INF] from='mgr.24752 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2022-09-29T05:28:58.172 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:57 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:28:56.990588+0000 mon.a (mon.0) 868 : cluster [DBG] osdmap e63: 8 total, 8 up, 8 in 2022-09-29T05:28:58.173 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:57 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:28:56.997069+0000 mgr.x (mgr.24752) 52 : cluster [DBG] pgmap v25: 105 pgs: 105 active+clean; 581 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:28:58.213 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:57 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: [29/Sep/2022:05:28:57] ENGINE Bus STOPPING 2022-09-29T05:28:58.213 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:57 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: [29/Sep/2022:05:28:57] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2022-09-29T05:28:58.213 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:57 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: [29/Sep/2022:05:28:57] ENGINE Bus STOPPED 2022-09-29T05:28:58.214 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:57 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: [29/Sep/2022:05:28:57] ENGINE Bus STARTING 2022-09-29T05:28:58.214 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: 2022-09-29T05:28:58.039+0000 7f4c54d32700 -1 client.0 error registering admin socket command: (17) File exists 2022-09-29T05:28:58.214 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: 2022-09-29T05:28:58.039+0000 7f4c54d32700 -1 client.0 error registering admin socket command: (17) File exists 2022-09-29T05:28:58.215 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: 2022-09-29T05:28:58.039+0000 7f4c54d32700 -1 client.0 error registering admin socket command: (17) File exists 2022-09-29T05:28:58.215 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: 2022-09-29T05:28:58.039+0000 7f4c54d32700 -1 client.0 error registering admin socket command: (17) File exists 2022-09-29T05:28:58.216 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: 2022-09-29T05:28:58.039+0000 7f4c54d32700 -1 client.0 error registering admin socket command: (17) File exists 2022-09-29T05:28:58.216 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: 2022-09-29T05:28:58.039+0000 7f4c57d38700 -1 client.0 error registering admin socket command: (17) File exists 2022-09-29T05:28:58.217 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: 2022-09-29T05:28:58.039+0000 7f4c57d38700 -1 client.0 error registering admin socket command: (17) File exists 2022-09-29T05:28:58.217 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: 2022-09-29T05:28:58.039+0000 7f4c57d38700 -1 client.0 error registering admin socket command: (17) File exists 2022-09-29T05:28:58.218 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: 2022-09-29T05:28:58.039+0000 7f4c57d38700 -1 client.0 error registering admin socket command: (17) File exists 2022-09-29T05:28:58.218 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: 2022-09-29T05:28:58.039+0000 7f4c57d38700 -1 client.0 error registering admin socket command: (17) File exists 2022-09-29T05:28:58.219 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: [29/Sep/2022:05:28:58] ENGINE Serving on http://:::9283 2022-09-29T05:28:58.219 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:28:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: [29/Sep/2022:05:28:58] ENGINE Bus STARTED 2022-09-29T05:28:58.220 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:57 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:28:56.876543+0000 mgr.x (mgr.24752) 48 : cephadm [INF] Detected new or changed devices on smithi083 2022-09-29T05:28:58.220 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:57 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:56.883700+0000 mon.a (mon.0) 864 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:58.221 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:57 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:56.892164+0000 mon.a (mon.0) 865 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:58.221 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:57 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:56.897575+0000 mon.a (mon.0) 866 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:58.222 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:57 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:56.979670+0000 mon.b (mon.2) 85 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:28:58.222 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:57 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:28:56.982017+0000 mgr.x (mgr.24752) 49 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2022-09-29T05:28:58.223 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:57 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:28:56.982446+0000 mgr.x (mgr.24752) 50 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2022-09-29T05:28:58.224 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:57 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:28:56.984615+0000 mgr.x (mgr.24752) 51 : cephadm [INF] Failing over to other MGR 2022-09-29T05:28:58.224 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:57 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:56.984833+0000 mon.b (mon.2) 86 : audit [INF] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2022-09-29T05:28:58.225 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:57 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:56.985451+0000 mon.a (mon.0) 867 : audit [INF] from='mgr.24752 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2022-09-29T05:28:58.225 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:57 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:28:56.990588+0000 mon.a (mon.0) 868 : cluster [DBG] osdmap e63: 8 total, 8 up, 8 in 2022-09-29T05:28:58.226 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:57 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:28:56.997069+0000 mgr.x (mgr.24752) 52 : cluster [DBG] pgmap v25: 105 pgs: 105 active+clean; 581 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:28:58.226 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:57 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:28:56.876543+0000 mgr.x (mgr.24752) 48 : cephadm [INF] Detected new or changed devices on smithi083 2022-09-29T05:28:58.227 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:57 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:56.883700+0000 mon.a (mon.0) 864 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:58.227 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:57 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:56.892164+0000 mon.a (mon.0) 865 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:58.227 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:57 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:56.897575+0000 mon.a (mon.0) 866 : audit [INF] from='mgr.24752 ' entity='mgr.x' 2022-09-29T05:28:58.228 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:57 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:56.979670+0000 mon.b (mon.2) 85 : audit [DBG] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:28:58.228 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:57 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:28:56.982017+0000 mgr.x (mgr.24752) 49 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2022-09-29T05:28:58.229 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:57 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:28:56.982446+0000 mgr.x (mgr.24752) 50 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2022-09-29T05:28:58.229 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:57 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:28:56.984615+0000 mgr.x (mgr.24752) 51 : cephadm [INF] Failing over to other MGR 2022-09-29T05:28:58.230 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:57 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:56.984833+0000 mon.b (mon.2) 86 : audit [INF] from='mgr.24752 172.21.15.153:0/1556245530' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2022-09-29T05:28:58.230 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:57 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:56.985451+0000 mon.a (mon.0) 867 : audit [INF] from='mgr.24752 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2022-09-29T05:28:58.231 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:57 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:28:56.990588+0000 mon.a (mon.0) 868 : cluster [DBG] osdmap e63: 8 total, 8 up, 8 in 2022-09-29T05:28:58.231 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:57 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:28:56.997069+0000 mgr.x (mgr.24752) 52 : cluster [DBG] pgmap v25: 105 pgs: 105 active+clean; 581 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:28:58.560 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:28:58 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: debug 2022-09-29T05:28:58.162+0000 7fa670e6b200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2022-09-29T05:28:58.561 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:28:58 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: debug 2022-09-29T05:28:58.370+0000 7fa670e6b200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2022-09-29T05:28:59.059 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:28:58 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[62373]: ts=2022-09-29T05:28:58.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi153:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi153.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:28:59.060 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:28:58 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[62373]: ts=2022-09-29T05:28:58.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephPGImbalance\nexpr: abs(((ceph_osd_numpg > 0) - on(job) group_left() avg by(job) (ceph_osd_numpg\n > 0)) / on(job) group_left() avg by(job) (ceph_osd_numpg > 0)) * on(ceph_daemon)\n group_left(hostname) ceph_osd_metadata > 0.3\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.5\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} deviates by\n more than 30% from average PG count.\n summary: PGs are not balanced across OSDs\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi153:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi153.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:28:59.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:58 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:57.901746+0000 mon.a (mon.0) 869 : audit [INF] from='mgr.24752 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2022-09-29T05:28:59.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:58 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:28:57.901851+0000 mon.a (mon.0) 870 : cluster [DBG] mgrmap e28: y(active, starting, since 0.915697s) 2022-09-29T05:28:59.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:58 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:57.903370+0000 mon.c (mon.1) 42 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2022-09-29T05:28:59.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:58 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:57.903645+0000 mon.c (mon.1) 43 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2022-09-29T05:28:59.063 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:58 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:57.903901+0000 mon.c (mon.1) 44 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2022-09-29T05:28:59.063 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:58 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:57.904179+0000 mon.c (mon.1) 45 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2022-09-29T05:28:59.064 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:58 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:57.904465+0000 mon.c (mon.1) 46 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2022-09-29T05:28:59.064 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:58 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:57.904760+0000 mon.c (mon.1) 47 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2022-09-29T05:28:59.065 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:58 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:57.905004+0000 mon.c (mon.1) 48 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2022-09-29T05:28:59.065 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:58 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:57.905338+0000 mon.c (mon.1) 49 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2022-09-29T05:28:59.066 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:58 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:57.905660+0000 mon.c (mon.1) 50 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2022-09-29T05:28:59.066 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:58 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:57.905948+0000 mon.c (mon.1) 51 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2022-09-29T05:28:59.067 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:58 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:57.906247+0000 mon.c (mon.1) 52 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2022-09-29T05:28:59.067 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:58 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:57.906532+0000 mon.c (mon.1) 53 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2022-09-29T05:28:59.068 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:58 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:57.906826+0000 mon.c (mon.1) 54 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2022-09-29T05:28:59.068 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:58 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:57.907131+0000 mon.c (mon.1) 55 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2022-09-29T05:28:59.069 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:58 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:57.907445+0000 mon.c (mon.1) 56 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2022-09-29T05:28:59.069 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:58 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:57.907762+0000 mon.c (mon.1) 57 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2022-09-29T05:28:59.069 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:58 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:57.908323+0000 mon.c (mon.1) 58 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2022-09-29T05:28:59.070 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:58 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:28:57.941525+0000 mon.a (mon.0) 871 : cluster [INF] Manager daemon y is now available 2022-09-29T05:28:59.070 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:58 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:57.960155+0000 mon.c (mon.1) 59 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:28:59.070 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:58 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:57.962448+0000 mon.c (mon.1) 60 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:28:59.070 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:58 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:57.964290+0000 mon.c (mon.1) 61 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:28:59.071 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:58 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:57.966267+0000 mon.c (mon.1) 62 : audit [INF] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:28:59.071 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:58 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:57.987260+0000 mon.c (mon.1) 63 : audit [INF] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:28:59.072 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:58 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:57.987730+0000 mon.a (mon.0) 872 : audit [INF] from='mgr.15009 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:28:59.072 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:58 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:58.004394+0000 mon.c (mon.1) 64 : audit [INF] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:28:59.072 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:58 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:28:58.004828+0000 mon.a (mon.0) 873 : audit [INF] from='mgr.15009 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:28:59.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:57.901746+0000 mon.a (mon.0) 869 : audit [INF] from='mgr.24752 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2022-09-29T05:28:59.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:28:57.901851+0000 mon.a (mon.0) 870 : cluster [DBG] mgrmap e28: y(active, starting, since 0.915697s) 2022-09-29T05:28:59.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:57.903370+0000 mon.c (mon.1) 42 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2022-09-29T05:28:59.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:57.903645+0000 mon.c (mon.1) 43 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2022-09-29T05:28:59.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:57.903901+0000 mon.c (mon.1) 44 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2022-09-29T05:28:59.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:57.904179+0000 mon.c (mon.1) 45 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2022-09-29T05:28:59.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:57.904465+0000 mon.c (mon.1) 46 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2022-09-29T05:28:59.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:57.904760+0000 mon.c (mon.1) 47 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2022-09-29T05:28:59.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:57.905004+0000 mon.c (mon.1) 48 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2022-09-29T05:28:59.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:57.905338+0000 mon.c (mon.1) 49 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2022-09-29T05:28:59.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:57.905660+0000 mon.c (mon.1) 50 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2022-09-29T05:28:59.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:57.905948+0000 mon.c (mon.1) 51 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2022-09-29T05:28:59.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:57.906247+0000 mon.c (mon.1) 52 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2022-09-29T05:28:59.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:57.906532+0000 mon.c (mon.1) 53 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2022-09-29T05:28:59.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:57.906826+0000 mon.c (mon.1) 54 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2022-09-29T05:28:59.218 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:57.901746+0000 mon.a (mon.0) 869 : audit [INF] from='mgr.24752 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2022-09-29T05:28:59.218 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:28:57.901851+0000 mon.a (mon.0) 870 : cluster [DBG] mgrmap e28: y(active, starting, since 0.915697s) 2022-09-29T05:28:59.218 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:57.903370+0000 mon.c (mon.1) 42 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2022-09-29T05:28:59.219 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:57.903645+0000 mon.c (mon.1) 43 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2022-09-29T05:28:59.219 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:57.903901+0000 mon.c (mon.1) 44 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2022-09-29T05:28:59.219 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:57.904179+0000 mon.c (mon.1) 45 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2022-09-29T05:28:59.219 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:57.904465+0000 mon.c (mon.1) 46 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2022-09-29T05:28:59.220 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:57.904760+0000 mon.c (mon.1) 47 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2022-09-29T05:28:59.220 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:57.905004+0000 mon.c (mon.1) 48 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2022-09-29T05:28:59.220 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:57.905338+0000 mon.c (mon.1) 49 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2022-09-29T05:28:59.221 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:57.905660+0000 mon.c (mon.1) 50 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2022-09-29T05:28:59.221 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:57.905948+0000 mon.c (mon.1) 51 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2022-09-29T05:28:59.221 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:57.906247+0000 mon.c (mon.1) 52 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2022-09-29T05:28:59.222 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:57.906532+0000 mon.c (mon.1) 53 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2022-09-29T05:28:59.222 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:57.906826+0000 mon.c (mon.1) 54 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2022-09-29T05:28:59.223 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:57.907131+0000 mon.c (mon.1) 55 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2022-09-29T05:28:59.223 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:57.907445+0000 mon.c (mon.1) 56 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2022-09-29T05:28:59.223 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:57.907762+0000 mon.c (mon.1) 57 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2022-09-29T05:28:59.224 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:57.908323+0000 mon.c (mon.1) 58 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2022-09-29T05:28:59.224 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:28:57.941525+0000 mon.a (mon.0) 871 : cluster [INF] Manager daemon y is now available 2022-09-29T05:28:59.224 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:57.960155+0000 mon.c (mon.1) 59 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:28:59.225 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:57.962448+0000 mon.c (mon.1) 60 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:28:59.225 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:57.964290+0000 mon.c (mon.1) 61 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:28:59.225 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:57.966267+0000 mon.c (mon.1) 62 : audit [INF] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:28:59.226 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:57.987260+0000 mon.c (mon.1) 63 : audit [INF] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:28:59.226 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:57.987730+0000 mon.a (mon.0) 872 : audit [INF] from='mgr.15009 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:28:59.226 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:58.004394+0000 mon.c (mon.1) 64 : audit [INF] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:28:59.226 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:28:58.004828+0000 mon.a (mon.0) 873 : audit [INF] from='mgr.15009 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:28:59.227 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:57.907131+0000 mon.c (mon.1) 55 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2022-09-29T05:28:59.227 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:57.907445+0000 mon.c (mon.1) 56 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2022-09-29T05:28:59.227 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:57.907762+0000 mon.c (mon.1) 57 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2022-09-29T05:28:59.228 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:57.908323+0000 mon.c (mon.1) 58 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2022-09-29T05:28:59.228 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:28:57.941525+0000 mon.a (mon.0) 871 : cluster [INF] Manager daemon y is now available 2022-09-29T05:28:59.228 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:57.960155+0000 mon.c (mon.1) 59 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:28:59.228 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:57.962448+0000 mon.c (mon.1) 60 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:28:59.229 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:57.964290+0000 mon.c (mon.1) 61 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:28:59.229 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:57.966267+0000 mon.c (mon.1) 62 : audit [INF] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:28:59.229 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:57.987260+0000 mon.c (mon.1) 63 : audit [INF] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:28:59.229 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:57.987730+0000 mon.a (mon.0) 872 : audit [INF] from='mgr.15009 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:28:59.230 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:58.004394+0000 mon.c (mon.1) 64 : audit [INF] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:28:59.230 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:28:58.004828+0000 mon.a (mon.0) 873 : audit [INF] from='mgr.15009 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:28:59.559 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:28:59 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: debug 2022-09-29T05:28:59.152+0000 7fa670e6b200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2022-09-29T05:28:59.559 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:28:59 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: debug 2022-09-29T05:28:59.343+0000 7fa670e6b200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2022-09-29T05:28:59.914 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:28:59 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: debug 2022-09-29T05:28:59.582+0000 7fa670e6b200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2022-09-29T05:28:59.915 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:28:59 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: debug 2022-09-29T05:28:59.756+0000 7fa670e6b200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2022-09-29T05:29:00.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:59 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:28:58.829331+0000 mgr.y (mgr.15009) 1 : cephadm [INF] [29/Sep/2022:05:28:58] ENGINE Bus STARTING 2022-09-29T05:29:00.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:59 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:28:58.904254+0000 mon.a (mon.0) 874 : cluster [DBG] mgrmap e29: y(active, since 1.9181s) 2022-09-29T05:29:00.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:59 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:28:58.918451+0000 mgr.y (mgr.15009) 2 : cluster [DBG] pgmap v3: 105 pgs: 105 active+clean; 582 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:29:00.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:59 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:28:58.942869+0000 mgr.y (mgr.15009) 3 : cephadm [INF] [29/Sep/2022:05:28:58] ENGINE Serving on https://172.21.15.83:7150 2022-09-29T05:29:00.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:59 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:28:59.055189+0000 mgr.y (mgr.15009) 4 : cephadm [INF] [29/Sep/2022:05:28:59] ENGINE Serving on https://:::8765 2022-09-29T05:29:00.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:28:59 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:28:59.055312+0000 mgr.y (mgr.15009) 5 : cephadm [INF] [29/Sep/2022:05:28:59] ENGINE Bus STARTED 2022-09-29T05:29:00.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:59 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:28:58.829331+0000 mgr.y (mgr.15009) 1 : cephadm [INF] [29/Sep/2022:05:28:58] ENGINE Bus STARTING 2022-09-29T05:29:00.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:59 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:28:58.904254+0000 mon.a (mon.0) 874 : cluster [DBG] mgrmap e29: y(active, since 1.9181s) 2022-09-29T05:29:00.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:59 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:28:58.918451+0000 mgr.y (mgr.15009) 2 : cluster [DBG] pgmap v3: 105 pgs: 105 active+clean; 582 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:29:00.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:59 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:28:58.942869+0000 mgr.y (mgr.15009) 3 : cephadm [INF] [29/Sep/2022:05:28:58] ENGINE Serving on https://172.21.15.83:7150 2022-09-29T05:29:00.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:59 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:28:59.055189+0000 mgr.y (mgr.15009) 4 : cephadm [INF] [29/Sep/2022:05:28:59] ENGINE Serving on https://:::8765 2022-09-29T05:29:00.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:28:59 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:28:59.055312+0000 mgr.y (mgr.15009) 5 : cephadm [INF] [29/Sep/2022:05:28:59] ENGINE Bus STARTED 2022-09-29T05:29:00.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:00 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: debug 2022-09-29T05:29:00.101+0000 7fa670e6b200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2022-09-29T05:29:00.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:59 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:28:58.829331+0000 mgr.y (mgr.15009) 1 : cephadm [INF] [29/Sep/2022:05:28:58] ENGINE Bus STARTING 2022-09-29T05:29:00.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:59 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:28:58.904254+0000 mon.a (mon.0) 874 : cluster [DBG] mgrmap e29: y(active, since 1.9181s) 2022-09-29T05:29:00.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:59 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:28:58.918451+0000 mgr.y (mgr.15009) 2 : cluster [DBG] pgmap v3: 105 pgs: 105 active+clean; 582 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:29:00.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:59 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:28:58.942869+0000 mgr.y (mgr.15009) 3 : cephadm [INF] [29/Sep/2022:05:28:58] ENGINE Serving on https://172.21.15.83:7150 2022-09-29T05:29:00.312 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:59 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:28:59.055189+0000 mgr.y (mgr.15009) 4 : cephadm [INF] [29/Sep/2022:05:28:59] ENGINE Serving on https://:::8765 2022-09-29T05:29:00.312 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:28:59 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:28:59.055312+0000 mgr.y (mgr.15009) 5 : cephadm [INF] [29/Sep/2022:05:28:59] ENGINE Bus STARTED 2022-09-29T05:29:00.692 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:00 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: debug 2022-09-29T05:29:00.407+0000 7fa670e6b200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2022-09-29T05:29:00.692 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:00 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: debug 2022-09-29T05:29:00.677+0000 7fa670e6b200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2022-09-29T05:29:00.973 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:00 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:28:59.906244+0000 mon.a (mon.0) 875 : cluster [INF] Health check cleared: TELEMETRY_CHANGED (was: Telemetry requires re-opt-in) 2022-09-29T05:29:00.973 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:00 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:28:59.906302+0000 mon.a (mon.0) 876 : cluster [INF] Cluster is now healthy 2022-09-29T05:29:00.974 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:00 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:28:59.906454+0000 mgr.y (mgr.15009) 6 : cluster [DBG] pgmap v4: 105 pgs: 105 active+clean; 582 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:29:00.974 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:00 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: debug 2022-09-29T05:29:00.774+0000 7fa670e6b200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2022-09-29T05:29:01.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:28:59.906244+0000 mon.a (mon.0) 875 : cluster [INF] Health check cleared: TELEMETRY_CHANGED (was: Telemetry requires re-opt-in) 2022-09-29T05:29:01.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:28:59.906302+0000 mon.a (mon.0) 876 : cluster [INF] Cluster is now healthy 2022-09-29T05:29:01.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:28:59.906454+0000 mgr.y (mgr.15009) 6 : cluster [DBG] pgmap v4: 105 pgs: 105 active+clean; 582 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:29:01.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:28:59.906244+0000 mon.a (mon.0) 875 : cluster [INF] Health check cleared: TELEMETRY_CHANGED (was: Telemetry requires re-opt-in) 2022-09-29T05:29:01.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:28:59.906302+0000 mon.a (mon.0) 876 : cluster [INF] Cluster is now healthy 2022-09-29T05:29:01.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:28:59.906454+0000 mgr.y (mgr.15009) 6 : cluster [DBG] pgmap v4: 105 pgs: 105 active+clean; 582 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:29:01.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:00 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: debug 2022-09-29T05:29:00.968+0000 7fa670e6b200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2022-09-29T05:29:01.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:01 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: debug 2022-09-29T05:29:01.062+0000 7fa670e6b200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2022-09-29T05:29:01.809 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:01 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: debug 2022-09-29T05:29:01.506+0000 7fa670e6b200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2022-09-29T05:29:01.809 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:01 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: debug 2022-09-29T05:29:01.600+0000 7fa670e6b200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2022-09-29T05:29:01.809 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:01 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: debug 2022-09-29T05:29:01.693+0000 7fa670e6b200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2022-09-29T05:29:01.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:29:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:29:01] "GET /metrics HTTP/1.1" 200 25804 "" "Prometheus/2.33.4" 2022-09-29T05:29:02.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:01 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: debug 2022-09-29T05:29:01.923+0000 7fa670e6b200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2022-09-29T05:29:02.683 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:29:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[62373]: ts=2022-09-29T05:29:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi153.front.sepia.ceph.com:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:29:02.970 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:29:01.906990+0000 mgr.y (mgr.15009) 7 : cluster [DBG] pgmap v5: 105 pgs: 105 active+clean; 582 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:29:02.970 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:02.689780+0000 mon.a (mon.0) 877 : audit [INF] from='mgr.15009 ' entity='mgr.y' 2022-09-29T05:29:02.970 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:02.695237+0000 mon.a (mon.0) 878 : audit [INF] from='mgr.15009 ' entity='mgr.y' 2022-09-29T05:29:02.971 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: debug 2022-09-29T05:29:02.678+0000 7fa670e6b200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2022-09-29T05:29:02.971 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: debug 2022-09-29T05:29:02.776+0000 7fa670e6b200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2022-09-29T05:29:02.971 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: debug 2022-09-29T05:29:02.872+0000 7fa670e6b200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2022-09-29T05:29:03.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:29:01.906990+0000 mgr.y (mgr.15009) 7 : cluster [DBG] pgmap v5: 105 pgs: 105 active+clean; 582 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:29:03.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:02.689780+0000 mon.a (mon.0) 877 : audit [INF] from='mgr.15009 ' entity='mgr.y' 2022-09-29T05:29:03.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:02.695237+0000 mon.a (mon.0) 878 : audit [INF] from='mgr.15009 ' entity='mgr.y' 2022-09-29T05:29:03.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:29:01.906990+0000 mgr.y (mgr.15009) 7 : cluster [DBG] pgmap v5: 105 pgs: 105 active+clean; 582 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:29:03.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:29:02.689780+0000 mon.a (mon.0) 877 : audit [INF] from='mgr.15009 ' entity='mgr.y' 2022-09-29T05:29:03.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:29:02.695237+0000 mon.a (mon.0) 878 : audit [INF] from='mgr.15009 ' entity='mgr.y' 2022-09-29T05:29:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: debug 2022-09-29T05:29:02.965+0000 7fa670e6b200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2022-09-29T05:29:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: debug 2022-09-29T05:29:03.129+0000 7fa670e6b200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2022-09-29T05:29:04.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:02.957395+0000 mon.a (mon.0) 879 : audit [INF] from='mgr.15009 ' entity='mgr.y' 2022-09-29T05:29:04.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:02.963241+0000 mon.a (mon.0) 880 : audit [INF] from='mgr.15009 ' entity='mgr.y' 2022-09-29T05:29:04.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:03.557585+0000 mon.a (mon.0) 881 : audit [INF] from='mgr.15009 ' entity='mgr.y' 2022-09-29T05:29:04.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:03.562567+0000 mon.a (mon.0) 882 : audit [INF] from='mgr.15009 ' entity='mgr.y' 2022-09-29T05:29:04.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:29:02.957395+0000 mon.a (mon.0) 879 : audit [INF] from='mgr.15009 ' entity='mgr.y' 2022-09-29T05:29:04.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:29:02.963241+0000 mon.a (mon.0) 880 : audit [INF] from='mgr.15009 ' entity='mgr.y' 2022-09-29T05:29:04.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:29:03.557585+0000 mon.a (mon.0) 881 : audit [INF] from='mgr.15009 ' entity='mgr.y' 2022-09-29T05:29:04.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:29:03.562567+0000 mon.a (mon.0) 882 : audit [INF] from='mgr.15009 ' entity='mgr.y' 2022-09-29T05:29:04.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:04 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: debug 2022-09-29T05:29:04.205+0000 7fa670e6b200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2022-09-29T05:29:04.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:02.957395+0000 mon.a (mon.0) 879 : audit [INF] from='mgr.15009 ' entity='mgr.y' 2022-09-29T05:29:04.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:02.963241+0000 mon.a (mon.0) 880 : audit [INF] from='mgr.15009 ' entity='mgr.y' 2022-09-29T05:29:04.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:03.557585+0000 mon.a (mon.0) 881 : audit [INF] from='mgr.15009 ' entity='mgr.y' 2022-09-29T05:29:04.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:03.562567+0000 mon.a (mon.0) 882 : audit [INF] from='mgr.15009 ' entity='mgr.y' 2022-09-29T05:29:04.714 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:04 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: debug 2022-09-29T05:29:04.328+0000 7fa670e6b200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2022-09-29T05:29:04.715 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:04 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: debug 2022-09-29T05:29:04.425+0000 7fa670e6b200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2022-09-29T05:29:04.715 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:04 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: [29/Sep/2022:05:29:04] ENGINE Bus STARTING 2022-09-29T05:29:04.715 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:04 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: CherryPy Checker: 2022-09-29T05:29:04.982 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:04 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: The Application mounted at '' has an empty config. 2022-09-29T05:29:04.982 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:04 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: 2022-09-29T05:29:04.982 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:04 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: [29/Sep/2022:05:29:04] ENGINE Serving on http://:::9283 2022-09-29T05:29:04.982 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:04 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[58728]: [29/Sep/2022:05:29:04] ENGINE Bus STARTED 2022-09-29T05:29:05.044 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:29:03.907410+0000 mgr.y (mgr.15009) 8 : cluster [DBG] pgmap v6: 105 pgs: 105 active+clean; 582 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:29:05.045 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:03.979698+0000 mon.a (mon.0) 883 : audit [INF] from='mgr.15009 ' entity='mgr.y' 2022-09-29T05:29:05.045 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:03.985466+0000 mon.a (mon.0) 884 : audit [INF] from='mgr.15009 ' entity='mgr.y' 2022-09-29T05:29:05.045 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:29:03.987395+0000 mgr.y (mgr.15009) 9 : cephadm [INF] Updating smithi153:/etc/ceph/ceph.conf 2022-09-29T05:29:05.045 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:29:04.197022+0000 mgr.y (mgr.15009) 10 : cephadm [INF] Updating smithi153:/var/lib/ceph/69111522-3fb6-11ed-8431-001a4aab830c/config/ceph.conf 2022-09-29T05:29:05.046 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:29:04.441519+0000 mon.a (mon.0) 885 : cluster [DBG] Standby manager daemon x started 2022-09-29T05:29:05.046 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:04.443238+0000 mon.c (mon.1) 65 : audit [DBG] from='mgr.? 172.21.15.153:0/2947530482' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2022-09-29T05:29:05.046 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:04.444091+0000 mon.c (mon.1) 66 : audit [DBG] from='mgr.? 172.21.15.153:0/2947530482' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2022-09-29T05:29:05.046 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:04.445888+0000 mon.c (mon.1) 67 : audit [DBG] from='mgr.? 172.21.15.153:0/2947530482' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2022-09-29T05:29:05.047 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:04.446996+0000 mon.c (mon.1) 68 : audit [DBG] from='mgr.? 172.21.15.153:0/2947530482' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2022-09-29T05:29:05.047 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:04.836786+0000 mon.a (mon.0) 886 : audit [INF] from='mgr.15009 ' entity='mgr.y' 2022-09-29T05:29:05.047 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:04.841324+0000 mon.a (mon.0) 887 : audit [INF] from='mgr.15009 ' entity='mgr.y' 2022-09-29T05:29:05.047 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:29:03.907410+0000 mgr.y (mgr.15009) 8 : cluster [DBG] pgmap v6: 105 pgs: 105 active+clean; 582 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:29:05.048 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:29:03.979698+0000 mon.a (mon.0) 883 : audit [INF] from='mgr.15009 ' entity='mgr.y' 2022-09-29T05:29:05.048 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:29:03.985466+0000 mon.a (mon.0) 884 : audit [INF] from='mgr.15009 ' entity='mgr.y' 2022-09-29T05:29:05.048 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:29:03.987395+0000 mgr.y (mgr.15009) 9 : cephadm [INF] Updating smithi153:/etc/ceph/ceph.conf 2022-09-29T05:29:05.048 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:29:04.197022+0000 mgr.y (mgr.15009) 10 : cephadm [INF] Updating smithi153:/var/lib/ceph/69111522-3fb6-11ed-8431-001a4aab830c/config/ceph.conf 2022-09-29T05:29:05.049 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:29:04.441519+0000 mon.a (mon.0) 885 : cluster [DBG] Standby manager daemon x started 2022-09-29T05:29:05.049 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:29:04.443238+0000 mon.c (mon.1) 65 : audit [DBG] from='mgr.? 172.21.15.153:0/2947530482' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2022-09-29T05:29:05.049 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:29:04.444091+0000 mon.c (mon.1) 66 : audit [DBG] from='mgr.? 172.21.15.153:0/2947530482' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2022-09-29T05:29:05.049 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:29:04.445888+0000 mon.c (mon.1) 67 : audit [DBG] from='mgr.? 172.21.15.153:0/2947530482' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2022-09-29T05:29:05.050 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:29:04.446996+0000 mon.c (mon.1) 68 : audit [DBG] from='mgr.? 172.21.15.153:0/2947530482' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2022-09-29T05:29:05.050 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:29:04.836786+0000 mon.a (mon.0) 886 : audit [INF] from='mgr.15009 ' entity='mgr.y' 2022-09-29T05:29:05.050 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:29:04.841324+0000 mon.a (mon.0) 887 : audit [INF] from='mgr.15009 ' entity='mgr.y' 2022-09-29T05:29:05.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:04 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:29:03.907410+0000 mgr.y (mgr.15009) 8 : cluster [DBG] pgmap v6: 105 pgs: 105 active+clean; 582 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:29:05.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:04 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:03.979698+0000 mon.a (mon.0) 883 : audit [INF] from='mgr.15009 ' entity='mgr.y' 2022-09-29T05:29:05.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:04 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:03.985466+0000 mon.a (mon.0) 884 : audit [INF] from='mgr.15009 ' entity='mgr.y' 2022-09-29T05:29:05.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:04 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:29:03.987395+0000 mgr.y (mgr.15009) 9 : cephadm [INF] Updating smithi153:/etc/ceph/ceph.conf 2022-09-29T05:29:05.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:04 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:29:04.197022+0000 mgr.y (mgr.15009) 10 : cephadm [INF] Updating smithi153:/var/lib/ceph/69111522-3fb6-11ed-8431-001a4aab830c/config/ceph.conf 2022-09-29T05:29:05.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:04 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:29:04.441519+0000 mon.a (mon.0) 885 : cluster [DBG] Standby manager daemon x started 2022-09-29T05:29:05.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:04 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:04.443238+0000 mon.c (mon.1) 65 : audit [DBG] from='mgr.? 172.21.15.153:0/2947530482' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2022-09-29T05:29:05.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:04 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:04.444091+0000 mon.c (mon.1) 66 : audit [DBG] from='mgr.? 172.21.15.153:0/2947530482' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2022-09-29T05:29:05.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:04 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:04.445888+0000 mon.c (mon.1) 67 : audit [DBG] from='mgr.? 172.21.15.153:0/2947530482' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2022-09-29T05:29:05.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:04 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:04.446996+0000 mon.c (mon.1) 68 : audit [DBG] from='mgr.? 172.21.15.153:0/2947530482' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2022-09-29T05:29:05.312 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:04 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:04.836786+0000 mon.a (mon.0) 886 : audit [INF] from='mgr.15009 ' entity='mgr.y' 2022-09-29T05:29:05.312 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:04 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:04.841324+0000 mon.a (mon.0) 887 : audit [INF] from='mgr.15009 ' entity='mgr.y' 2022-09-29T05:29:06.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:05 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:29:04.402959+0000 mgr.y (mgr.15009) 11 : cephadm [INF] Updating smithi153:/etc/ceph/ceph.client.admin.keyring 2022-09-29T05:29:06.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:05 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:29:04.621337+0000 mgr.y (mgr.15009) 12 : cephadm [INF] Updating smithi153:/var/lib/ceph/69111522-3fb6-11ed-8431-001a4aab830c/config/ceph.client.admin.keyring 2022-09-29T05:29:06.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:05 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:29:04.992890+0000 mon.a (mon.0) 888 : cluster [DBG] mgrmap e30: y(active, since 8s), standbys: x 2022-09-29T05:29:06.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:05 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:04.993582+0000 mon.c (mon.1) 69 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2022-09-29T05:29:06.321 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:29:04.402959+0000 mgr.y (mgr.15009) 11 : cephadm [INF] Updating smithi153:/etc/ceph/ceph.client.admin.keyring 2022-09-29T05:29:06.322 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:29:04.621337+0000 mgr.y (mgr.15009) 12 : cephadm [INF] Updating smithi153:/var/lib/ceph/69111522-3fb6-11ed-8431-001a4aab830c/config/ceph.client.admin.keyring 2022-09-29T05:29:06.322 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:29:04.992890+0000 mon.a (mon.0) 888 : cluster [DBG] mgrmap e30: y(active, since 8s), standbys: x 2022-09-29T05:29:06.322 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:29:04.993582+0000 mon.c (mon.1) 69 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2022-09-29T05:29:06.323 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:29:04.402959+0000 mgr.y (mgr.15009) 11 : cephadm [INF] Updating smithi153:/etc/ceph/ceph.client.admin.keyring 2022-09-29T05:29:06.323 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:29:04.621337+0000 mgr.y (mgr.15009) 12 : cephadm [INF] Updating smithi153:/var/lib/ceph/69111522-3fb6-11ed-8431-001a4aab830c/config/ceph.client.admin.keyring 2022-09-29T05:29:06.323 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:29:04.992890+0000 mon.a (mon.0) 888 : cluster [DBG] mgrmap e30: y(active, since 8s), standbys: x 2022-09-29T05:29:06.323 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:04.993582+0000 mon.c (mon.1) 69 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2022-09-29T05:29:07.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:06 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:29:05.908098+0000 mgr.y (mgr.15009) 13 : cluster [DBG] pgmap v7: 105 pgs: 105 active+clean; 582 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:29:07.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:06 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:06.569542+0000 mon.a (mon.0) 889 : audit [INF] from='mgr.15009 ' entity='mgr.y' 2022-09-29T05:29:07.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:06 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:06.577297+0000 mon.a (mon.0) 890 : audit [INF] from='mgr.15009 ' entity='mgr.y' 2022-09-29T05:29:07.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:29:05.908098+0000 mgr.y (mgr.15009) 13 : cluster [DBG] pgmap v7: 105 pgs: 105 active+clean; 582 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:29:07.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:06.569542+0000 mon.a (mon.0) 889 : audit [INF] from='mgr.15009 ' entity='mgr.y' 2022-09-29T05:29:07.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:06.577297+0000 mon.a (mon.0) 890 : audit [INF] from='mgr.15009 ' entity='mgr.y' 2022-09-29T05:29:07.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:29:05.908098+0000 mgr.y (mgr.15009) 13 : cluster [DBG] pgmap v7: 105 pgs: 105 active+clean; 582 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:29:07.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:29:06.569542+0000 mon.a (mon.0) 889 : audit [INF] from='mgr.15009 ' entity='mgr.y' 2022-09-29T05:29:07.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:29:06.577297+0000 mon.a (mon.0) 890 : audit [INF] from='mgr.15009 ' entity='mgr.y' 2022-09-29T05:29:08.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:07 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:29:06.564430+0000 mgr.y (mgr.15009) 14 : cephadm [INF] Detected new or changed devices on smithi083 2022-09-29T05:29:08.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:07 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:29:06.578567+0000 mgr.y (mgr.15009) 15 : cephadm [INF] Updating smithi083:/etc/ceph/ceph.conf 2022-09-29T05:29:08.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:08 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:29:06.790255+0000 mgr.y (mgr.15009) 16 : cephadm [INF] Updating smithi083:/var/lib/ceph/69111522-3fb6-11ed-8431-001a4aab830c/config/ceph.conf 2022-09-29T05:29:08.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:08 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:29:06.999297+0000 mgr.y (mgr.15009) 17 : cephadm [INF] Updating smithi083:/etc/ceph/ceph.client.admin.keyring 2022-09-29T05:29:08.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:08 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:07.419799+0000 mon.a (mon.0) 891 : audit [INF] from='mgr.15009 ' entity='mgr.y' 2022-09-29T05:29:08.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:08 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:07.427041+0000 mon.a (mon.0) 892 : audit [INF] from='mgr.15009 ' entity='mgr.y' 2022-09-29T05:29:08.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:08 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:07.431414+0000 mon.a (mon.0) 893 : audit [INF] from='mgr.15009 ' entity='mgr.y' 2022-09-29T05:29:08.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:07 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:29:06.564430+0000 mgr.y (mgr.15009) 14 : cephadm [INF] Detected new or changed devices on smithi083 2022-09-29T05:29:08.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:07 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:29:06.578567+0000 mgr.y (mgr.15009) 15 : cephadm [INF] Updating smithi083:/etc/ceph/ceph.conf 2022-09-29T05:29:08.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:07 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:29:06.790255+0000 mgr.y (mgr.15009) 16 : cephadm [INF] Updating smithi083:/var/lib/ceph/69111522-3fb6-11ed-8431-001a4aab830c/config/ceph.conf 2022-09-29T05:29:08.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:07 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:29:06.999297+0000 mgr.y (mgr.15009) 17 : cephadm [INF] Updating smithi083:/etc/ceph/ceph.client.admin.keyring 2022-09-29T05:29:08.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:07 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:29:07.419799+0000 mon.a (mon.0) 891 : audit [INF] from='mgr.15009 ' entity='mgr.y' 2022-09-29T05:29:08.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:07 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:29:07.427041+0000 mon.a (mon.0) 892 : audit [INF] from='mgr.15009 ' entity='mgr.y' 2022-09-29T05:29:08.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:07 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:29:07.431414+0000 mon.a (mon.0) 893 : audit [INF] from='mgr.15009 ' entity='mgr.y' 2022-09-29T05:29:08.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:07 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:29:06.564430+0000 mgr.y (mgr.15009) 14 : cephadm [INF] Detected new or changed devices on smithi083 2022-09-29T05:29:08.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:07 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:29:06.578567+0000 mgr.y (mgr.15009) 15 : cephadm [INF] Updating smithi083:/etc/ceph/ceph.conf 2022-09-29T05:29:08.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:07 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:29:06.790255+0000 mgr.y (mgr.15009) 16 : cephadm [INF] Updating smithi083:/var/lib/ceph/69111522-3fb6-11ed-8431-001a4aab830c/config/ceph.conf 2022-09-29T05:29:08.467 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:07 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:29:06.999297+0000 mgr.y (mgr.15009) 17 : cephadm [INF] Updating smithi083:/etc/ceph/ceph.client.admin.keyring 2022-09-29T05:29:08.467 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:07 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:07.419799+0000 mon.a (mon.0) 891 : audit [INF] from='mgr.15009 ' entity='mgr.y' 2022-09-29T05:29:08.467 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:07 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:07.427041+0000 mon.a (mon.0) 892 : audit [INF] from='mgr.15009 ' entity='mgr.y' 2022-09-29T05:29:08.468 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:07 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:07.431414+0000 mon.a (mon.0) 893 : audit [INF] from='mgr.15009 ' entity='mgr.y' 2022-09-29T05:29:08.972 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:29:08 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[62373]: ts=2022-09-29T05:29:08.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi153.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi083:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:29:09.252 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:09 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:29:07.206539+0000 mgr.y (mgr.15009) 18 : cephadm [INF] Updating smithi083:/var/lib/ceph/69111522-3fb6-11ed-8431-001a4aab830c/config/ceph.client.admin.keyring 2022-09-29T05:29:09.252 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:09 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:29:07.504680+0000 mgr.y (mgr.15009) 19 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2022-09-29T05:29:09.253 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:09 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:29:07.511745+0000 mgr.y (mgr.15009) 20 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi153 2022-09-29T05:29:09.253 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:09 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:29:07.908424+0000 mgr.y (mgr.15009) 21 : cluster [DBG] pgmap v8: 105 pgs: 105 active+clean; 582 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:29:09.254 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:09 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:29:08.025906+0000 mgr.y (mgr.15009) 22 : cluster [DBG] pgmap v9: 105 pgs: 105 active+clean; 582 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:29:09.255 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:09 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:29:08.429815+0000 mon.a (mon.0) 894 : cluster [WRN] Health check failed: Telemetry requires re-opt-in (TELEMETRY_CHANGED) 2022-09-29T05:29:09.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:09 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:29:07.206539+0000 mgr.y (mgr.15009) 18 : cephadm [INF] Updating smithi083:/var/lib/ceph/69111522-3fb6-11ed-8431-001a4aab830c/config/ceph.client.admin.keyring 2022-09-29T05:29:09.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:09 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:29:07.504680+0000 mgr.y (mgr.15009) 19 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2022-09-29T05:29:09.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:09 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:29:07.511745+0000 mgr.y (mgr.15009) 20 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi153 2022-09-29T05:29:09.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:09 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:29:07.908424+0000 mgr.y (mgr.15009) 21 : cluster [DBG] pgmap v8: 105 pgs: 105 active+clean; 582 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:29:09.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:09 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:29:08.025906+0000 mgr.y (mgr.15009) 22 : cluster [DBG] pgmap v9: 105 pgs: 105 active+clean; 582 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:29:09.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:09 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:29:08.429815+0000 mon.a (mon.0) 894 : cluster [WRN] Health check failed: Telemetry requires re-opt-in (TELEMETRY_CHANGED) 2022-09-29T05:29:09.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:09 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:29:07.206539+0000 mgr.y (mgr.15009) 18 : cephadm [INF] Updating smithi083:/var/lib/ceph/69111522-3fb6-11ed-8431-001a4aab830c/config/ceph.client.admin.keyring 2022-09-29T05:29:09.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:09 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:29:07.504680+0000 mgr.y (mgr.15009) 19 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2022-09-29T05:29:09.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:09 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:29:07.511745+0000 mgr.y (mgr.15009) 20 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi153 2022-09-29T05:29:09.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:09 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:29:07.908424+0000 mgr.y (mgr.15009) 21 : cluster [DBG] pgmap v8: 105 pgs: 105 active+clean; 582 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:29:09.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:09 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:29:08.025906+0000 mgr.y (mgr.15009) 22 : cluster [DBG] pgmap v9: 105 pgs: 105 active+clean; 582 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:29:09.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:09 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:29:08.429815+0000 mon.a (mon.0) 894 : cluster [WRN] Health check failed: Telemetry requires re-opt-in (TELEMETRY_CHANGED) 2022-09-29T05:29:09.936 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:29:09 smithi153 systemd[1]: Stopping Ceph prometheus.a for 69111522-3fb6-11ed-8431-001a4aab830c... 2022-09-29T05:29:09.937 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:29:09 smithi153 bash[65226]: Error: no container with name or ID "ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus.a" found: no such container 2022-09-29T05:29:09.937 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:29:09 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[62373]: ts=2022-09-29T05:29:09.827Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2022-09-29T05:29:09.937 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:29:09 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[62373]: ts=2022-09-29T05:29:09.827Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2022-09-29T05:29:09.937 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:29:09 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[62373]: ts=2022-09-29T05:29:09.827Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2022-09-29T05:29:09.938 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:29:09 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[62373]: ts=2022-09-29T05:29:09.827Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2022-09-29T05:29:09.938 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:29:09 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[62373]: ts=2022-09-29T05:29:09.827Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2022-09-29T05:29:09.938 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:29:09 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[62373]: ts=2022-09-29T05:29:09.827Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2022-09-29T05:29:09.938 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:29:09 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[62373]: ts=2022-09-29T05:29:09.828Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2022-09-29T05:29:09.939 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:29:09 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[62373]: ts=2022-09-29T05:29:09.828Z caller=main.go:828 level=info msg="Scrape manager stopped" 2022-09-29T05:29:09.939 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:29:09 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[62373]: ts=2022-09-29T05:29:09.828Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2022-09-29T05:29:09.939 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:29:09 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[62373]: ts=2022-09-29T05:29:09.828Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2022-09-29T05:29:09.939 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:29:09 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[62373]: ts=2022-09-29T05:29:09.828Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2022-09-29T05:29:09.939 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:29:09 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[62373]: ts=2022-09-29T05:29:09.828Z caller=main.go:1066 level=info msg="See you next time!" 2022-09-29T05:29:10.225 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:29:10 smithi153 bash[65239]: ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a 2022-09-29T05:29:10.559 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:29:10 smithi153 bash[65286]: Error: no container with name or ID "ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus.a" found: no such container 2022-09-29T05:29:10.559 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:29:10 smithi153 systemd[1]: ceph-69111522-3fb6-11ed-8431-001a4aab830c@prometheus.a.service: Succeeded. 2022-09-29T05:29:10.560 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:29:10 smithi153 systemd[1]: Stopped Ceph prometheus.a for 69111522-3fb6-11ed-8431-001a4aab830c. 2022-09-29T05:29:10.560 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:29:10 smithi153 systemd[1]: Starting Ceph prometheus.a for 69111522-3fb6-11ed-8431-001a4aab830c... 2022-09-29T05:29:11.152 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:29:10.026572+0000 mgr.y (mgr.15009) 23 : cluster [DBG] pgmap v10: 105 pgs: 105 active+clean; 582 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:29:11.152 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:29:10.026572+0000 mgr.y (mgr.15009) 23 : cluster [DBG] pgmap v10: 105 pgs: 105 active+clean; 582 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:29:11.259 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:10 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:29:10.026572+0000 mgr.y (mgr.15009) 23 : cluster [DBG] pgmap v10: 105 pgs: 105 active+clean; 582 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:29:11.260 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:29:10 smithi153 podman[65358]: 2022-09-29T05:29:11.260 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:29:11 smithi153 bash[65358]: 40a54c5a196a59bc1b0df022eb49720b583534b6e6b4b4b7ab870a609a938e99 2022-09-29T05:29:11.260 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:29:11 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:29:11.077Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2022-09-29T05:29:11.261 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:29:11 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:29:11.077Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2022-09-29T05:29:11.261 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:29:11 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:29:11.077Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2022-09-29T05:29:11.261 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:29:11 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:29:11.077Z caller=main.go:518 level=info host_details="(Linux 4.18.0-408.el8.x86_64 #1 SMP Mon Jul 18 17:42:52 UTC 2022 x86_64 smithi153 (none))" 2022-09-29T05:29:11.261 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:29:11 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:29:11.077Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2022-09-29T05:29:11.262 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:29:11 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:29:11.077Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2022-09-29T05:29:11.262 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:29:11 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:29:11.079Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2022-09-29T05:29:11.262 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:29:11 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:29:11.079Z caller=main.go:923 level=info msg="Starting TSDB ..." 2022-09-29T05:29:11.262 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:29:11 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:29:11.080Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2022-09-29T05:29:11.263 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:29:11 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:29:11.082Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2022-09-29T05:29:11.263 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:29:11 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:29:11.082Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.557µs 2022-09-29T05:29:11.263 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:29:11 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:29:11.082Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2022-09-29T05:29:11.263 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:29:11 smithi153 systemd[1]: Started Ceph prometheus.a for 69111522-3fb6-11ed-8431-001a4aab830c. 2022-09-29T05:29:11.264 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:29:11 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:29:11.256Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3 2022-09-29T05:29:11.463 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:29:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: [29/Sep/2022:05:29:11] ENGINE Bus STOPPING 2022-09-29T05:29:11.463 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:29:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: [29/Sep/2022:05:29:11] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2022-09-29T05:29:11.463 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:29:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: [29/Sep/2022:05:29:11] ENGINE Bus STOPPED 2022-09-29T05:29:11.464 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:29:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: [29/Sep/2022:05:29:11] ENGINE Bus STARTING 2022-09-29T05:29:11.464 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:29:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: [29/Sep/2022:05:29:11] ENGINE Serving on http://:::9283 2022-09-29T05:29:11.464 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:29:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: [29/Sep/2022:05:29:11] ENGINE Bus STARTED 2022-09-29T05:29:11.559 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:29:11 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:29:11.430Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3 2022-09-29T05:29:11.892 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:29:11 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:29:11.629Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=3 2022-09-29T05:29:11.893 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:29:11 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:29:11.629Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=3 2022-09-29T05:29:11.893 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:29:11 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:29:11.629Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=34.736µs wal_replay_duration=546.611063ms total_replay_duration=546.65995ms 2022-09-29T05:29:11.893 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:29:11 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:29:11.632Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2022-09-29T05:29:11.893 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:29:11 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:29:11.632Z caller=main.go:947 level=info msg="TSDB started" 2022-09-29T05:29:11.894 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:29:11 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:29:11.632Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2022-09-29T05:29:11.894 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:29:11 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:29:11.652Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=19.217272ms db_storage=765ns remote_storage=1.126µs web_handler=330ns query_engine=633ns scrape=470.107µs scrape_sd=230.765µs notify=14.768µs notify_sd=50.025µs rules=18.222461ms 2022-09-29T05:29:11.894 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:29:11 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:29:11.652Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2022-09-29T05:29:12.160 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:11.114594+0000 mon.a (mon.0) 895 : audit [INF] from='mgr.15009 ' entity='mgr.y' 2022-09-29T05:29:12.161 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:11.121208+0000 mon.a (mon.0) 896 : audit [INF] from='mgr.15009 ' entity='mgr.y' 2022-09-29T05:29:12.161 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:11.126671+0000 mon.c (mon.1) 70 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2022-09-29T05:29:12.161 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:11.127157+0000 mgr.y (mgr.15009) 24 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2022-09-29T05:29:12.161 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:11.128206+0000 mon.c (mon.1) 71 : audit [INF] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi153.front.sepia.ceph.com:9095"}]: dispatch 2022-09-29T05:29:12.162 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:11.128562+0000 mgr.y (mgr.15009) 25 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi153.front.sepia.ceph.com:9095"}]: dispatch 2022-09-29T05:29:12.162 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:11.134683+0000 mon.a (mon.0) 897 : audit [INF] from='mgr.15009 ' entity='mgr.y' 2022-09-29T05:29:12.162 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:11.139652+0000 mon.c (mon.1) 72 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:29:12.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:29:11.114594+0000 mon.a (mon.0) 895 : audit [INF] from='mgr.15009 ' entity='mgr.y' 2022-09-29T05:29:12.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:29:11.121208+0000 mon.a (mon.0) 896 : audit [INF] from='mgr.15009 ' entity='mgr.y' 2022-09-29T05:29:12.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:29:11.126671+0000 mon.c (mon.1) 70 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2022-09-29T05:29:12.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:29:11.127157+0000 mgr.y (mgr.15009) 24 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2022-09-29T05:29:12.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:29:11.128206+0000 mon.c (mon.1) 71 : audit [INF] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi153.front.sepia.ceph.com:9095"}]: dispatch 2022-09-29T05:29:12.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:29:11.128562+0000 mgr.y (mgr.15009) 25 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi153.front.sepia.ceph.com:9095"}]: dispatch 2022-09-29T05:29:12.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:29:11.134683+0000 mon.a (mon.0) 897 : audit [INF] from='mgr.15009 ' entity='mgr.y' 2022-09-29T05:29:12.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:29:11.139652+0000 mon.c (mon.1) 72 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:29:12.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:11.114594+0000 mon.a (mon.0) 895 : audit [INF] from='mgr.15009 ' entity='mgr.y' 2022-09-29T05:29:12.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:11.121208+0000 mon.a (mon.0) 896 : audit [INF] from='mgr.15009 ' entity='mgr.y' 2022-09-29T05:29:12.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:11.126671+0000 mon.c (mon.1) 70 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2022-09-29T05:29:12.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:11.127157+0000 mgr.y (mgr.15009) 24 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2022-09-29T05:29:12.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:11.128206+0000 mon.c (mon.1) 71 : audit [INF] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi153.front.sepia.ceph.com:9095"}]: dispatch 2022-09-29T05:29:12.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:11.128562+0000 mgr.y (mgr.15009) 25 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi153.front.sepia.ceph.com:9095"}]: dispatch 2022-09-29T05:29:12.467 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:11.134683+0000 mon.a (mon.0) 897 : audit [INF] from='mgr.15009 ' entity='mgr.y' 2022-09-29T05:29:12.467 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:11.139652+0000 mon.c (mon.1) 72 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:29:12.559 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:29:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:29:12.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi153.front.sepia.ceph.com:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:29:13.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:29:12.027185+0000 mgr.y (mgr.15009) 26 : cluster [DBG] pgmap v11: 105 pgs: 105 active+clean; 582 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:29:13.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:29:12.544057+0000 mon.c (mon.1) 73 : audit [INF] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2022-09-29T05:29:13.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:29:12.544435+0000 mon.a (mon.0) 898 : audit [INF] from='mgr.15009 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2022-09-29T05:29:13.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:29:12.545578+0000 mon.c (mon.1) 74 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2022-09-29T05:29:13.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:29:12.546610+0000 mon.c (mon.1) 75 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:29:13.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:29:12.027185+0000 mgr.y (mgr.15009) 26 : cluster [DBG] pgmap v11: 105 pgs: 105 active+clean; 582 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:29:13.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:12.544057+0000 mon.c (mon.1) 73 : audit [INF] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2022-09-29T05:29:13.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:12.544435+0000 mon.a (mon.0) 898 : audit [INF] from='mgr.15009 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2022-09-29T05:29:13.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:12.545578+0000 mon.c (mon.1) 74 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2022-09-29T05:29:13.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:12.546610+0000 mon.c (mon.1) 75 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:29:13.485 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:29:12.027185+0000 mgr.y (mgr.15009) 26 : cluster [DBG] pgmap v11: 105 pgs: 105 active+clean; 582 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:29:13.485 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:12.544057+0000 mon.c (mon.1) 73 : audit [INF] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2022-09-29T05:29:13.485 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:12.544435+0000 mon.a (mon.0) 898 : audit [INF] from='mgr.15009 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2022-09-29T05:29:13.486 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:12.545578+0000 mon.c (mon.1) 74 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2022-09-29T05:29:13.486 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:12.546610+0000 mon.c (mon.1) 75 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:29:14.059 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:13 smithi153 systemd[1]: Stopping Ceph mgr.x for 69111522-3fb6-11ed-8431-001a4aab830c... 2022-09-29T05:29:14.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:29:12.543580+0000 mgr.y (mgr.15009) 27 : cephadm [INF] Upgrade: Updating mgr.x 2022-09-29T05:29:14.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:29:12.547584+0000 mgr.y (mgr.15009) 28 : cephadm [INF] Deploying daemon mgr.x on smithi153 2022-09-29T05:29:14.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:29:12.543580+0000 mgr.y (mgr.15009) 27 : cephadm [INF] Upgrade: Updating mgr.x 2022-09-29T05:29:14.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:29:12.547584+0000 mgr.y (mgr.15009) 28 : cephadm [INF] Deploying daemon mgr.x on smithi153 2022-09-29T05:29:14.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:14 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:29:12.543580+0000 mgr.y (mgr.15009) 27 : cephadm [INF] Upgrade: Updating mgr.x 2022-09-29T05:29:14.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:14 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:29:12.547584+0000 mgr.y (mgr.15009) 28 : cephadm [INF] Deploying daemon mgr.x on smithi153 2022-09-29T05:29:14.560 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:14 smithi153 bash[65788]: ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x 2022-09-29T05:29:15.059 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:14 smithi153 systemd[1]: ceph-69111522-3fb6-11ed-8431-001a4aab830c@mgr.x.service: Main process exited, code=exited, status=143/n/a 2022-09-29T05:29:15.059 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:14 smithi153 systemd[1]: ceph-69111522-3fb6-11ed-8431-001a4aab830c@mgr.x.service: Failed with result 'exit-code'. 2022-09-29T05:29:15.060 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:14 smithi153 systemd[1]: Stopped Ceph mgr.x for 69111522-3fb6-11ed-8431-001a4aab830c. 2022-09-29T05:29:15.060 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:14 smithi153 systemd[1]: Starting Ceph mgr.x for 69111522-3fb6-11ed-8431-001a4aab830c... 2022-09-29T05:29:15.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:29:14.027745+0000 mgr.y (mgr.15009) 29 : cluster [DBG] pgmap v12: 105 pgs: 105 active+clean; 582 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:29:15.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:29:14.027745+0000 mgr.y (mgr.15009) 29 : cluster [DBG] pgmap v12: 105 pgs: 105 active+clean; 582 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:29:15.472 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:15 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:29:14.027745+0000 mgr.y (mgr.15009) 29 : cluster [DBG] pgmap v12: 105 pgs: 105 active+clean; 582 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:29:15.472 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:15 smithi153 podman[65916]: 2022-09-29T05:29:16.046 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:15 smithi153 bash[65916]: d539b7e85b69fbf795432771d29bb8182a4736d07c5fb9db807033ffddf7b934 2022-09-29T05:29:16.047 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:15 smithi153 ceph-mgr[65956]: -- 172.21.15.153:0/593524574 <== mon.2 v2:172.21.15.153:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x563088430820 con 0x563088405000 2022-09-29T05:29:16.047 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:15 smithi153 systemd[1]: Started Ceph mgr.x for 69111522-3fb6-11ed-8431-001a4aab830c. 2022-09-29T05:29:16.047 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:15 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: 2022-09-29T05:29:15.834+0000 7f3396690200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2022-09-29T05:29:16.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:29:15.700312+0000 mon.a (mon.0) 899 : audit [INF] from='mgr.15009 ' entity='mgr.y' 2022-09-29T05:29:16.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:29:15.711085+0000 mon.a (mon.0) 900 : audit [INF] from='mgr.15009 ' entity='mgr.y' 2022-09-29T05:29:16.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:29:15.712611+0000 mon.c (mon.1) 76 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:29:16.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:29:15.713911+0000 mon.c (mon.1) 77 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:29:16.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:29:15.714729+0000 mon.c (mon.1) 78 : audit [INF] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:29:16.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:29:16.028416+0000 mgr.y (mgr.15009) 30 : cluster [DBG] pgmap v13: 105 pgs: 105 active+clean; 582 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:29:16.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:15.700312+0000 mon.a (mon.0) 899 : audit [INF] from='mgr.15009 ' entity='mgr.y' 2022-09-29T05:29:16.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:15.711085+0000 mon.a (mon.0) 900 : audit [INF] from='mgr.15009 ' entity='mgr.y' 2022-09-29T05:29:16.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:15.712611+0000 mon.c (mon.1) 76 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:29:16.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:15.713911+0000 mon.c (mon.1) 77 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:29:16.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:15.714729+0000 mon.c (mon.1) 78 : audit [INF] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:29:16.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:29:16.028416+0000 mgr.y (mgr.15009) 30 : cluster [DBG] pgmap v13: 105 pgs: 105 active+clean; 582 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:29:17.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:16 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:15.700312+0000 mon.a (mon.0) 899 : audit [INF] from='mgr.15009 ' entity='mgr.y' 2022-09-29T05:29:17.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:16 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:15.711085+0000 mon.a (mon.0) 900 : audit [INF] from='mgr.15009 ' entity='mgr.y' 2022-09-29T05:29:17.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:16 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:15.712611+0000 mon.c (mon.1) 76 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:29:17.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:16 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:15.713911+0000 mon.c (mon.1) 77 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:29:17.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:16 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:15.714729+0000 mon.c (mon.1) 78 : audit [INF] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:29:17.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:16 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:29:16.028416+0000 mgr.y (mgr.15009) 30 : cluster [DBG] pgmap v13: 105 pgs: 105 active+clean; 582 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:29:17.025 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:16 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: 2022-09-29T05:29:16.042+0000 7f3396690200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2022-09-29T05:29:17.025 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:16 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: 2022-09-29T05:29:16.831+0000 7f3396690200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2022-09-29T05:29:17.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:17 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: 2022-09-29T05:29:17.019+0000 7f3396690200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2022-09-29T05:29:17.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:17 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: 2022-09-29T05:29:17.250+0000 7f3396690200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2022-09-29T05:29:17.765 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:17 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: 2022-09-29T05:29:17.424+0000 7f3396690200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2022-09-29T05:29:17.765 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:17 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: 2022-09-29T05:29:17.761+0000 7f3396690200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2022-09-29T05:29:18.349 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:18 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: 2022-09-29T05:29:18.072+0000 7f3396690200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2022-09-29T05:29:18.629 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:18 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: 2022-09-29T05:29:18.346+0000 7f3396690200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2022-09-29T05:29:18.629 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:18 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: 2022-09-29T05:29:18.434+0000 7f3396690200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2022-09-29T05:29:19.059 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:29:18 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:29:18.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi153.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi083:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:29:19.059 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:18 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: 2022-09-29T05:29:18.625+0000 7f3396690200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2022-09-29T05:29:19.060 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:18 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: 2022-09-29T05:29:18.708+0000 7f3396690200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2022-09-29T05:29:19.338 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:19 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:29:18.028991+0000 mgr.y (mgr.15009) 31 : cluster [DBG] pgmap v14: 105 pgs: 105 active+clean; 582 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:29:19.339 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:19 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: 2022-09-29T05:29:19.148+0000 7f3396690200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2022-09-29T05:29:19.339 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:19 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: 2022-09-29T05:29:19.248+0000 7f3396690200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2022-09-29T05:29:19.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:19 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:29:18.028991+0000 mgr.y (mgr.15009) 31 : cluster [DBG] pgmap v14: 105 pgs: 105 active+clean; 582 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:29:19.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:19 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:29:18.028991+0000 mgr.y (mgr.15009) 31 : cluster [DBG] pgmap v14: 105 pgs: 105 active+clean; 582 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:29:19.809 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:19 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: 2022-09-29T05:29:19.335+0000 7f3396690200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2022-09-29T05:29:19.809 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:19 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: 2022-09-29T05:29:19.564+0000 7f3396690200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2022-09-29T05:29:20.223 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T05:29:20.506 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T05:29:20.506 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3m) 17s ago 3m 26.3M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T05:29:20.507 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3m) 17s ago 3m 60.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T05:29:20.507 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4m) 17s ago 4m 14.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T05:29:20.507 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4m) 17s ago 4m 17.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T05:29:20.508 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 starting - - - - 2022-09-29T05:29:20.508 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (32s) 17s ago 8m 454M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T05:29:20.508 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8m) 17s ago 8m 108M 2048M 16.2.10 c56e13ff90f2 0e7e560a682f 2022-09-29T05:29:20.508 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7m) 17s ago 7m 90.9M 2048M 16.2.10 c56e13ff90f2 b96ee39cc34f 2022-09-29T05:29:20.508 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7m) 17s ago 7m 86.5M 2048M 16.2.10 c56e13ff90f2 a7179c86361b 2022-09-29T05:29:20.509 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3m) 17s ago 3m 25.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T05:29:20.509 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3m) 17s ago 3m 24.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T05:29:20.509 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6m) 17s ago 6m 64.0M 2500M 16.2.10 c56e13ff90f2 c12a21fd5897 2022-09-29T05:29:20.509 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6m) 17s ago 6m 59.1M 2500M 16.2.10 c56e13ff90f2 989e877de30a 2022-09-29T05:29:20.510 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6m) 17s ago 6m 53.0M 2500M 16.2.10 c56e13ff90f2 81b9b51f0eee 2022-09-29T05:29:20.510 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5m) 17s ago 5m 59.2M 2500M 16.2.10 c56e13ff90f2 4863a2083356 2022-09-29T05:29:20.510 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5m) 17s ago 5m 55.0M 2500M 16.2.10 c56e13ff90f2 ed61dc9353b0 2022-09-29T05:29:20.510 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5m) 17s ago 5m 62.4M 2500M 16.2.10 c56e13ff90f2 59f19fd15fed 2022-09-29T05:29:20.511 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4m) 17s ago 4m 64.0M 2500M 16.2.10 c56e13ff90f2 23decfd9c6b2 2022-09-29T05:29:20.511 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4m) 17s ago 4m 67.0M 2500M 16.2.10 c56e13ff90f2 23460a09c0a7 2022-09-29T05:29:20.511 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 starting - - - - 2022-09-29T05:29:20.636 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: 2022-09-29T05:29:20.329+0000 7f3396690200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2022-09-29T05:29:20.636 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: 2022-09-29T05:29:20.430+0000 7f3396690200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2022-09-29T05:29:20.636 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: 2022-09-29T05:29:20.530+0000 7f3396690200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2022-09-29T05:29:20.867 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T05:29:20.867 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T05:29:20.868 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 3 2022-09-29T05:29:20.868 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:29:20.868 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T05:29:20.868 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T05:29:20.869 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:29:20.870 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T05:29:20.870 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 8 2022-09-29T05:29:20.870 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:29:20.871 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T05:29:20.871 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T05:29:20.871 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:29:20.871 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T05:29:20.872 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 13, 2022-09-29T05:29:20.872 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T05:29:20.872 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T05:29:20.872 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T05:29:21.059 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: 2022-09-29T05:29:20.632+0000 7f3396690200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2022-09-29T05:29:21.059 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: 2022-09-29T05:29:20.797+0000 7f3396690200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2022-09-29T05:29:21.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:29:20.029593+0000 mgr.y (mgr.15009) 32 : cluster [DBG] pgmap v15: 105 pgs: 105 active+clean; 582 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:29:21.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:20.768302+0000 mon.a (mon.0) 901 : audit [INF] from='mgr.15009 ' entity='mgr.y' 2022-09-29T05:29:21.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:20.773685+0000 mon.a (mon.0) 902 : audit [INF] from='mgr.15009 ' entity='mgr.y' 2022-09-29T05:29:21.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:20.864407+0000 mon.c (mon.1) 79 : audit [DBG] from='client.? 172.21.15.83:0/4045087813' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:29:21.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:29:20.029593+0000 mgr.y (mgr.15009) 32 : cluster [DBG] pgmap v15: 105 pgs: 105 active+clean; 582 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:29:21.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:29:20.768302+0000 mon.a (mon.0) 901 : audit [INF] from='mgr.15009 ' entity='mgr.y' 2022-09-29T05:29:21.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:29:20.773685+0000 mon.a (mon.0) 902 : audit [INF] from='mgr.15009 ' entity='mgr.y' 2022-09-29T05:29:21.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:29:20.864407+0000 mon.c (mon.1) 79 : audit [DBG] from='client.? 172.21.15.83:0/4045087813' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:29:21.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:29:20.029593+0000 mgr.y (mgr.15009) 32 : cluster [DBG] pgmap v15: 105 pgs: 105 active+clean; 582 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:29:21.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:20.768302+0000 mon.a (mon.0) 901 : audit [INF] from='mgr.15009 ' entity='mgr.y' 2022-09-29T05:29:21.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:20.773685+0000 mon.a (mon.0) 902 : audit [INF] from='mgr.15009 ' entity='mgr.y' 2022-09-29T05:29:21.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:20.864407+0000 mon.c (mon.1) 79 : audit [DBG] from='client.? 172.21.15.83:0/4045087813' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:29:21.880 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:29:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:29:21] "GET /metrics HTTP/1.1" 200 176136 "" "Prometheus/2.33.4" 2022-09-29T05:29:22.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:29:20.202737+0000 mgr.y (mgr.15009) 33 : audit [DBG] from='client.15069 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:29:22.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:29:20.495413+0000 mgr.y (mgr.15009) 34 : audit [DBG] from='client.24994 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:29:22.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:29:21.758071+0000 mon.a (mon.0) 903 : audit [INF] from='mgr.15009 ' entity='mgr.y' 2022-09-29T05:29:22.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:29:21.763305+0000 mon.a (mon.0) 904 : audit [INF] from='mgr.15009 ' entity='mgr.y' 2022-09-29T05:29:22.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:20.202737+0000 mgr.y (mgr.15009) 33 : audit [DBG] from='client.15069 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:29:22.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:20.495413+0000 mgr.y (mgr.15009) 34 : audit [DBG] from='client.24994 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:29:22.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:21.758071+0000 mon.a (mon.0) 903 : audit [INF] from='mgr.15009 ' entity='mgr.y' 2022-09-29T05:29:22.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:21.763305+0000 mon.a (mon.0) 904 : audit [INF] from='mgr.15009 ' entity='mgr.y' 2022-09-29T05:29:22.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: 2022-09-29T05:29:21.857+0000 7f3396690200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2022-09-29T05:29:22.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: 2022-09-29T05:29:21.983+0000 7f3396690200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2022-09-29T05:29:22.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: 2022-09-29T05:29:22.081+0000 7f3396690200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2022-09-29T05:29:22.310 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: [29/Sep/2022:05:29:22] ENGINE Bus STARTING 2022-09-29T05:29:22.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:20.202737+0000 mgr.y (mgr.15009) 33 : audit [DBG] from='client.15069 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:29:22.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:20.495413+0000 mgr.y (mgr.15009) 34 : audit [DBG] from='client.24994 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:29:22.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:21.758071+0000 mon.a (mon.0) 903 : audit [INF] from='mgr.15009 ' entity='mgr.y' 2022-09-29T05:29:22.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:21 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:21.763305+0000 mon.a (mon.0) 904 : audit [INF] from='mgr.15009 ' entity='mgr.y' 2022-09-29T05:29:22.632 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:29:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:29:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:29:22.633 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: CherryPy Checker: 2022-09-29T05:29:22.633 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: The Application mounted at '' has an empty config. 2022-09-29T05:29:22.633 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: 2022-09-29T05:29:22.634 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: [29/Sep/2022:05:29:22] ENGINE Serving on http://:::9283 2022-09-29T05:29:22.634 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: [29/Sep/2022:05:29:22] ENGINE Bus STARTED 2022-09-29T05:29:23.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:29:22.030063+0000 mgr.y (mgr.15009) 35 : cluster [DBG] pgmap v16: 105 pgs: 105 active+clean; 582 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:29:23.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:29:22.095665+0000 mon.a (mon.0) 905 : cluster [DBG] Standby manager daemon x restarted 2022-09-29T05:29:23.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:29:22.095861+0000 mon.a (mon.0) 906 : cluster [DBG] Standby manager daemon x started 2022-09-29T05:29:23.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:22.097193+0000 mon.b (mon.2) 87 : audit [DBG] from='mgr.? 172.21.15.153:0/311536859' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2022-09-29T05:29:23.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:22.097612+0000 mon.b (mon.2) 88 : audit [DBG] from='mgr.? 172.21.15.153:0/311536859' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2022-09-29T05:29:23.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:22.099449+0000 mon.b (mon.2) 89 : audit [DBG] from='mgr.? 172.21.15.153:0/311536859' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2022-09-29T05:29:23.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:22.100418+0000 mon.b (mon.2) 90 : audit [DBG] from='mgr.? 172.21.15.153:0/311536859' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2022-09-29T05:29:23.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:29:22.030063+0000 mgr.y (mgr.15009) 35 : cluster [DBG] pgmap v16: 105 pgs: 105 active+clean; 582 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:29:23.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:29:22.095665+0000 mon.a (mon.0) 905 : cluster [DBG] Standby manager daemon x restarted 2022-09-29T05:29:23.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:29:22.095861+0000 mon.a (mon.0) 906 : cluster [DBG] Standby manager daemon x started 2022-09-29T05:29:23.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:29:22.097193+0000 mon.b (mon.2) 87 : audit [DBG] from='mgr.? 172.21.15.153:0/311536859' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2022-09-29T05:29:23.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:29:22.097612+0000 mon.b (mon.2) 88 : audit [DBG] from='mgr.? 172.21.15.153:0/311536859' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2022-09-29T05:29:23.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:29:22.099449+0000 mon.b (mon.2) 89 : audit [DBG] from='mgr.? 172.21.15.153:0/311536859' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2022-09-29T05:29:23.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:29:22.100418+0000 mon.b (mon.2) 90 : audit [DBG] from='mgr.? 172.21.15.153:0/311536859' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2022-09-29T05:29:23.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:29:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:29:23.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:29:22.030063+0000 mgr.y (mgr.15009) 35 : cluster [DBG] pgmap v16: 105 pgs: 105 active+clean; 582 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:29:23.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:29:22.095665+0000 mon.a (mon.0) 905 : cluster [DBG] Standby manager daemon x restarted 2022-09-29T05:29:23.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:29:22.095861+0000 mon.a (mon.0) 906 : cluster [DBG] Standby manager daemon x started 2022-09-29T05:29:23.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:22.097193+0000 mon.b (mon.2) 87 : audit [DBG] from='mgr.? 172.21.15.153:0/311536859' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2022-09-29T05:29:23.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:22.097612+0000 mon.b (mon.2) 88 : audit [DBG] from='mgr.? 172.21.15.153:0/311536859' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2022-09-29T05:29:23.312 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:22.099449+0000 mon.b (mon.2) 89 : audit [DBG] from='mgr.? 172.21.15.153:0/311536859' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2022-09-29T05:29:23.312 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:22.100418+0000 mon.b (mon.2) 90 : audit [DBG] from='mgr.? 172.21.15.153:0/311536859' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2022-09-29T05:29:24.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:29:22.891675+0000 mon.a (mon.0) 907 : cluster [DBG] mgrmap e31: y(active, since 25s), standbys: x 2022-09-29T05:29:24.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:29:22.891675+0000 mon.a (mon.0) 907 : cluster [DBG] mgrmap e31: y(active, since 25s), standbys: x 2022-09-29T05:29:24.257 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:29:22.891675+0000 mon.a (mon.0) 907 : cluster [DBG] mgrmap e31: y(active, since 25s), standbys: x 2022-09-29T05:29:25.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:24 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:29:24.030621+0000 mgr.y (mgr.15009) 36 : cluster [DBG] pgmap v17: 105 pgs: 105 active+clean; 582 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:29:25.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:24 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:24.690199+0000 mon.a (mon.0) 908 : audit [INF] from='mgr.15009 ' entity='mgr.y' 2022-09-29T05:29:25.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:24 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:24.700906+0000 mon.a (mon.0) 909 : audit [INF] from='mgr.15009 ' entity='mgr.y' 2022-09-29T05:29:25.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:24 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:24.711188+0000 mon.a (mon.0) 910 : audit [INF] from='mgr.15009 ' entity='mgr.y' 2022-09-29T05:29:25.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:24 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:24.792098+0000 mon.c (mon.1) 80 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:29:25.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:24 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:24.794855+0000 mon.c (mon.1) 81 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:29:25.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:24 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:24.802208+0000 mon.a (mon.0) 911 : audit [INF] from='mgr.15009 ' entity='mgr.y' 2022-09-29T05:29:25.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:24 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:24.805651+0000 mon.c (mon.1) 82 : audit [INF] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2022-09-29T05:29:25.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:24 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:24.805934+0000 mon.a (mon.0) 912 : audit [INF] from='mgr.15009 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2022-09-29T05:29:25.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:24 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:24.811706+0000 mon.a (mon.0) 913 : audit [INF] from='mgr.15009 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2022-09-29T05:29:25.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:24 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:24.814971+0000 mon.c (mon.1) 83 : audit [INF] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2022-09-29T05:29:25.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:24 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:24.815250+0000 mon.a (mon.0) 914 : audit [INF] from='mgr.15009 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2022-09-29T05:29:25.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:24 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:24.821072+0000 mon.a (mon.0) 915 : audit [INF] from='mgr.15009 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2022-09-29T05:29:25.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:24 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:24.824541+0000 mon.c (mon.1) 84 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2022-09-29T05:29:25.063 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:24 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:24.825284+0000 mon.c (mon.1) 85 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2022-09-29T05:29:25.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:29:24.030621+0000 mgr.y (mgr.15009) 36 : cluster [DBG] pgmap v17: 105 pgs: 105 active+clean; 582 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:29:25.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:24.690199+0000 mon.a (mon.0) 908 : audit [INF] from='mgr.15009 ' entity='mgr.y' 2022-09-29T05:29:25.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:24.700906+0000 mon.a (mon.0) 909 : audit [INF] from='mgr.15009 ' entity='mgr.y' 2022-09-29T05:29:25.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:24.711188+0000 mon.a (mon.0) 910 : audit [INF] from='mgr.15009 ' entity='mgr.y' 2022-09-29T05:29:25.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:24.792098+0000 mon.c (mon.1) 80 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:29:25.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:24.794855+0000 mon.c (mon.1) 81 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:29:25.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:24.802208+0000 mon.a (mon.0) 911 : audit [INF] from='mgr.15009 ' entity='mgr.y' 2022-09-29T05:29:25.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:24.805651+0000 mon.c (mon.1) 82 : audit [INF] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2022-09-29T05:29:25.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:24.805934+0000 mon.a (mon.0) 912 : audit [INF] from='mgr.15009 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2022-09-29T05:29:25.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:24.811706+0000 mon.a (mon.0) 913 : audit [INF] from='mgr.15009 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2022-09-29T05:29:25.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:24.814971+0000 mon.c (mon.1) 83 : audit [INF] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2022-09-29T05:29:25.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:24.815250+0000 mon.a (mon.0) 914 : audit [INF] from='mgr.15009 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2022-09-29T05:29:25.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:24.821072+0000 mon.a (mon.0) 915 : audit [INF] from='mgr.15009 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2022-09-29T05:29:25.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:24.824541+0000 mon.c (mon.1) 84 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2022-09-29T05:29:25.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:24.825284+0000 mon.c (mon.1) 85 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2022-09-29T05:29:25.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:29:24.030621+0000 mgr.y (mgr.15009) 36 : cluster [DBG] pgmap v17: 105 pgs: 105 active+clean; 582 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:29:25.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:29:24.690199+0000 mon.a (mon.0) 908 : audit [INF] from='mgr.15009 ' entity='mgr.y' 2022-09-29T05:29:25.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:29:24.700906+0000 mon.a (mon.0) 909 : audit [INF] from='mgr.15009 ' entity='mgr.y' 2022-09-29T05:29:25.218 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:29:24.711188+0000 mon.a (mon.0) 910 : audit [INF] from='mgr.15009 ' entity='mgr.y' 2022-09-29T05:29:25.218 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:29:24.792098+0000 mon.c (mon.1) 80 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:29:25.218 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:29:24.794855+0000 mon.c (mon.1) 81 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:29:25.218 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:29:24.802208+0000 mon.a (mon.0) 911 : audit [INF] from='mgr.15009 ' entity='mgr.y' 2022-09-29T05:29:25.219 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:29:24.805651+0000 mon.c (mon.1) 82 : audit [INF] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2022-09-29T05:29:25.219 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:29:24.805934+0000 mon.a (mon.0) 912 : audit [INF] from='mgr.15009 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2022-09-29T05:29:25.219 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:29:24.811706+0000 mon.a (mon.0) 913 : audit [INF] from='mgr.15009 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2022-09-29T05:29:25.219 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:29:24.814971+0000 mon.c (mon.1) 83 : audit [INF] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2022-09-29T05:29:25.220 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:29:24.815250+0000 mon.a (mon.0) 914 : audit [INF] from='mgr.15009 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2022-09-29T05:29:25.220 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:29:24.821072+0000 mon.a (mon.0) 915 : audit [INF] from='mgr.15009 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2022-09-29T05:29:25.220 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:29:24.824541+0000 mon.c (mon.1) 84 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2022-09-29T05:29:25.220 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:29:24.825284+0000 mon.c (mon.1) 85 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2022-09-29T05:29:26.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:29:24.684686+0000 mgr.y (mgr.15009) 37 : cephadm [INF] Detected new or changed devices on smithi153 2022-09-29T05:29:26.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:29:24.795518+0000 mgr.y (mgr.15009) 38 : cephadm [INF] Upgrade: Setting container_image for all mgr 2022-09-29T05:29:26.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:29:24.825779+0000 mgr.y (mgr.15009) 39 : cephadm [INF] Upgrade: It appears safe to stop mon.a 2022-09-29T05:29:26.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:29:24.684686+0000 mgr.y (mgr.15009) 37 : cephadm [INF] Detected new or changed devices on smithi153 2022-09-29T05:29:26.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:29:24.795518+0000 mgr.y (mgr.15009) 38 : cephadm [INF] Upgrade: Setting container_image for all mgr 2022-09-29T05:29:26.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:29:24.825779+0000 mgr.y (mgr.15009) 39 : cephadm [INF] Upgrade: It appears safe to stop mon.a 2022-09-29T05:29:26.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:25 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:29:24.684686+0000 mgr.y (mgr.15009) 37 : cephadm [INF] Detected new or changed devices on smithi153 2022-09-29T05:29:26.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:25 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:29:24.795518+0000 mgr.y (mgr.15009) 38 : cephadm [INF] Upgrade: Setting container_image for all mgr 2022-09-29T05:29:26.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:25 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:29:24.825779+0000 mgr.y (mgr.15009) 39 : cephadm [INF] Upgrade: It appears safe to stop mon.a 2022-09-29T05:29:27.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:29:26.031002+0000 mgr.y (mgr.15009) 40 : cluster [DBG] pgmap v18: 105 pgs: 105 active+clean; 582 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:29:27.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:26.318081+0000 mon.a (mon.0) 916 : audit [INF] from='mgr.15009 ' entity='mgr.y' 2022-09-29T05:29:27.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:26.318932+0000 mon.c (mon.1) 86 : audit [INF] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2022-09-29T05:29:27.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:26.319496+0000 mon.c (mon.1) 87 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2022-09-29T05:29:27.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:26.320040+0000 mon.c (mon.1) 88 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:29:27.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cluster 2022-09-29T05:29:26.031002+0000 mgr.y (mgr.15009) 40 : cluster [DBG] pgmap v18: 105 pgs: 105 active+clean; 582 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:29:27.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:29:26.318081+0000 mon.a (mon.0) 916 : audit [INF] from='mgr.15009 ' entity='mgr.y' 2022-09-29T05:29:27.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:29:26.318932+0000 mon.c (mon.1) 86 : audit [INF] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2022-09-29T05:29:27.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:29:26.319496+0000 mon.c (mon.1) 87 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2022-09-29T05:29:27.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: audit 2022-09-29T05:29:26.320040+0000 mon.c (mon.1) 88 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:29:27.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:26 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:29:26.031002+0000 mgr.y (mgr.15009) 40 : cluster [DBG] pgmap v18: 105 pgs: 105 active+clean; 582 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:29:27.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:26 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:26.318081+0000 mon.a (mon.0) 916 : audit [INF] from='mgr.15009 ' entity='mgr.y' 2022-09-29T05:29:27.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:26 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:26.318932+0000 mon.c (mon.1) 86 : audit [INF] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2022-09-29T05:29:27.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:26 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:26.319496+0000 mon.c (mon.1) 87 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2022-09-29T05:29:27.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:26 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:26.320040+0000 mon.c (mon.1) 88 : audit [DBG] from='mgr.15009 172.21.15.83:0/3722019916' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:29:27.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:27 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:29:26.311333+0000 mgr.y (mgr.15009) 41 : cephadm [INF] Upgrade: Updating mon.a 2022-09-29T05:29:27.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:27 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:29:26.320873+0000 mgr.y (mgr.15009) 42 : cephadm [INF] Deploying daemon mon.a on smithi083 2022-09-29T05:29:27.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:27 smithi083 systemd[1]: Stopping Ceph mon.a for 69111522-3fb6-11ed-8431-001a4aab830c... 2022-09-29T05:29:27.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:27 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:29:26.311333+0000 mgr.y (mgr.15009) 41 : cephadm [INF] Upgrade: Updating mon.a 2022-09-29T05:29:27.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:27 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: cephadm 2022-09-29T05:29:26.320873+0000 mgr.y (mgr.15009) 42 : cephadm [INF] Deploying daemon mon.a on smithi083 2022-09-29T05:29:28.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:27 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:29:26.311333+0000 mgr.y (mgr.15009) 41 : cephadm [INF] Upgrade: Updating mon.a 2022-09-29T05:29:28.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:27 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:29:26.320873+0000 mgr.y (mgr.15009) 42 : cephadm [INF] Deploying daemon mon.a on smithi083 2022-09-29T05:29:28.390 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:28 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:29:28.100+0000 7f422ff51700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2022-09-29T05:29:28.391 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:28 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[38787]: debug 2022-09-29T05:29:28.100+0000 7f422ff51700 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2022-09-29T05:29:28.652 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:28 smithi083 bash[77076]: ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a 2022-09-29T05:29:28.652 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:28 smithi083 systemd[1]: ceph-69111522-3fb6-11ed-8431-001a4aab830c@mon.a.service: Succeeded. 2022-09-29T05:29:28.652 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:28 smithi083 systemd[1]: Stopped Ceph mon.a for 69111522-3fb6-11ed-8431-001a4aab830c. 2022-09-29T05:29:28.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:28 smithi083 systemd[1]: Starting Ceph mon.a for 69111522-3fb6-11ed-8431-001a4aab830c... 2022-09-29T05:29:29.059 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:29:28 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:29:28.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi153.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi083:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:29:29.059 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:29:28 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:29:28.662Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephPGImbalance\nexpr: abs(((ceph_osd_numpg > 0) - on(job) group_left() avg by(job) (ceph_osd_numpg\n > 0)) / on(job) group_left() avg by(job) (ceph_osd_numpg > 0)) * on(ceph_daemon)\n group_left(hostname) ceph_osd_metadata > 0.3\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.5\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} deviates by\n more than 30% from average PG count.\n summary: PGs are not balanced across OSDs\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi153.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi083:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:29:29.679 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 podman[77203]: 2022-09-29T05:29:29.680 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: set uid:gid to 167:167 (ceph:ceph) 2022-09-29T05:29:29.680 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev), process ceph-mon, pid 7 2022-09-29T05:29:29.680 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: pidfile_write: ignore empty --pid-file 2022-09-29T05:29:29.680 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 bash[77203]: 5252f1d434075b96b57eefd1b193522fa878b4004c56a19c652affa8e667a34c 2022-09-29T05:29:29.681 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: load: jerasure load: lrc 2022-09-29T05:29:29.681 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: RocksDB version: 6.15.5 2022-09-29T05:29:29.681 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2022-09-29T05:29:29.681 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Compile date Sep 28 2022 2022-09-29T05:29:29.682 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: DB SUMMARY 2022-09-29T05:29:29.682 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: DB Session ID: AE3SROYLISI6MGTVDIZB 2022-09-29T05:29:29.682 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: CURRENT file: CURRENT 2022-09-29T05:29:29.682 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: IDENTITY file: IDENTITY 2022-09-29T05:29:29.683 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: MANIFEST file: MANIFEST-000008 size: 579 Bytes 2022-09-29T05:29:29.683 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000017.sst 2022-09-29T05:29:29.683 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000015.log size: 1046220 ; 2022-09-29T05:29:29.683 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.error_if_exists: 0 2022-09-29T05:29:29.683 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.create_if_missing: 0 2022-09-29T05:29:29.684 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.paranoid_checks: 1 2022-09-29T05:29:29.684 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2022-09-29T05:29:29.684 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.env: 0x55d1bf386280 2022-09-29T05:29:29.684 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.fs: Posix File System 2022-09-29T05:29:29.685 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.info_log: 0x55d1bfbf5b00 2022-09-29T05:29:29.685 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.max_file_opening_threads: 16 2022-09-29T05:29:29.685 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.statistics: (nil) 2022-09-29T05:29:29.685 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.use_fsync: 0 2022-09-29T05:29:29.686 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.max_log_file_size: 0 2022-09-29T05:29:29.686 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.max_manifest_file_size: 1073741824 2022-09-29T05:29:29.686 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.log_file_time_to_roll: 0 2022-09-29T05:29:29.686 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.keep_log_file_num: 1000 2022-09-29T05:29:29.686 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.recycle_log_file_num: 0 2022-09-29T05:29:29.687 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.allow_fallocate: 1 2022-09-29T05:29:29.687 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.allow_mmap_reads: 0 2022-09-29T05:29:29.687 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.allow_mmap_writes: 0 2022-09-29T05:29:29.687 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.use_direct_reads: 0 2022-09-29T05:29:29.688 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2022-09-29T05:29:29.688 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.create_missing_column_families: 0 2022-09-29T05:29:29.688 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.db_log_dir: 2022-09-29T05:29:29.688 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2022-09-29T05:29:29.689 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.table_cache_numshardbits: 6 2022-09-29T05:29:29.689 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.WAL_ttl_seconds: 0 2022-09-29T05:29:29.689 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.WAL_size_limit_MB: 0 2022-09-29T05:29:29.689 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2022-09-29T05:29:29.690 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.manifest_preallocation_size: 4194304 2022-09-29T05:29:29.690 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.is_fd_close_on_exec: 1 2022-09-29T05:29:29.690 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.advise_random_on_open: 1 2022-09-29T05:29:29.690 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.db_write_buffer_size: 0 2022-09-29T05:29:29.691 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.write_buffer_manager: 0x55d1bfce0ba0 2022-09-29T05:29:29.691 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.access_hint_on_compaction_start: 1 2022-09-29T05:29:29.691 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2022-09-29T05:29:29.691 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.random_access_max_buffer_size: 1048576 2022-09-29T05:29:29.692 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.use_adaptive_mutex: 0 2022-09-29T05:29:29.692 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.rate_limiter: (nil) 2022-09-29T05:29:29.692 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2022-09-29T05:29:29.692 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.wal_recovery_mode: 2 2022-09-29T05:29:29.693 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.enable_thread_tracking: 0 2022-09-29T05:29:29.693 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.enable_pipelined_write: 0 2022-09-29T05:29:29.693 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.unordered_write: 0 2022-09-29T05:29:29.693 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.allow_concurrent_memtable_write: 1 2022-09-29T05:29:29.694 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2022-09-29T05:29:29.694 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.write_thread_max_yield_usec: 100 2022-09-29T05:29:29.695 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.write_thread_slow_yield_usec: 3 2022-09-29T05:29:29.695 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.row_cache: None 2022-09-29T05:29:29.696 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.wal_filter: None 2022-09-29T05:29:29.696 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.avoid_flush_during_recovery: 0 2022-09-29T05:29:29.696 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.allow_ingest_behind: 0 2022-09-29T05:29:29.697 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.preserve_deletes: 0 2022-09-29T05:29:29.697 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.two_write_queues: 0 2022-09-29T05:29:29.698 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.manual_wal_flush: 0 2022-09-29T05:29:29.698 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.atomic_flush: 0 2022-09-29T05:29:29.698 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2022-09-29T05:29:29.699 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.persist_stats_to_disk: 0 2022-09-29T05:29:29.699 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.write_dbid_to_manifest: 0 2022-09-29T05:29:29.699 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.log_readahead_size: 0 2022-09-29T05:29:29.700 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.file_checksum_gen_factory: Unknown 2022-09-29T05:29:29.700 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.best_efforts_recovery: 0 2022-09-29T05:29:29.700 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2022-09-29T05:29:29.701 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2022-09-29T05:29:29.701 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.allow_data_in_errors: 0 2022-09-29T05:29:29.702 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.db_host_id: __hostname__ 2022-09-29T05:29:29.703 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.max_background_jobs: 2 2022-09-29T05:29:29.704 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.max_background_compactions: -1 2022-09-29T05:29:29.704 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.max_subcompactions: 1 2022-09-29T05:29:29.704 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.avoid_flush_during_shutdown: 0 2022-09-29T05:29:29.704 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2022-09-29T05:29:29.704 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.delayed_write_rate : 16777216 2022-09-29T05:29:29.705 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.max_total_wal_size: 0 2022-09-29T05:29:29.705 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2022-09-29T05:29:29.705 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.stats_dump_period_sec: 600 2022-09-29T05:29:29.705 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.stats_persist_period_sec: 600 2022-09-29T05:29:29.706 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.stats_history_buffer_size: 1048576 2022-09-29T05:29:29.706 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.max_open_files: -1 2022-09-29T05:29:29.706 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.bytes_per_sync: 0 2022-09-29T05:29:29.706 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.wal_bytes_per_sync: 0 2022-09-29T05:29:29.707 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.strict_bytes_per_sync: 0 2022-09-29T05:29:29.707 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.compaction_readahead_size: 0 2022-09-29T05:29:29.707 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.max_background_flushes: -1 2022-09-29T05:29:29.707 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Compression algorithms supported: 2022-09-29T05:29:29.708 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: kZSTDNotFinalCompression supported: 0 2022-09-29T05:29:29.708 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: kZSTD supported: 0 2022-09-29T05:29:29.708 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: kXpressCompression supported: 0 2022-09-29T05:29:29.708 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: kLZ4HCCompression supported: 1 2022-09-29T05:29:29.709 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: kZlibCompression supported: 1 2022-09-29T05:29:29.709 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: kSnappyCompression supported: 1 2022-09-29T05:29:29.709 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: kLZ4Compression supported: 1 2022-09-29T05:29:29.709 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: kBZip2Compression supported: 0 2022-09-29T05:29:29.710 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Fast CRC32 supported: Supported on x86 2022-09-29T05:29:29.710 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000008 2022-09-29T05:29:29.710 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2022-09-29T05:29:29.710 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2022-09-29T05:29:29.710 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.merge_operator: 2022-09-29T05:29:29.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.compaction_filter: None 2022-09-29T05:29:29.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.compaction_filter_factory: None 2022-09-29T05:29:29.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.sst_partitioner_factory: None 2022-09-29T05:29:29.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.memtable_factory: SkipListFactory 2022-09-29T05:29:29.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.table_factory: BlockBasedTable 2022-09-29T05:29:29.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55d1bfbf2e40) 2022-09-29T05:29:29.712 INFO:journalctl@ceph.mon.a.smithi083.stdout: cache_index_and_filter_blocks: 1 2022-09-29T05:29:29.712 INFO:journalctl@ceph.mon.a.smithi083.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2022-09-29T05:29:29.713 INFO:journalctl@ceph.mon.a.smithi083.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2022-09-29T05:29:29.713 INFO:journalctl@ceph.mon.a.smithi083.stdout: pin_top_level_index_and_filter: 1 2022-09-29T05:29:29.713 INFO:journalctl@ceph.mon.a.smithi083.stdout: index_type: 0 2022-09-29T05:29:29.713 INFO:journalctl@ceph.mon.a.smithi083.stdout: data_block_index_type: 0 2022-09-29T05:29:29.713 INFO:journalctl@ceph.mon.a.smithi083.stdout: index_shortening: 1 2022-09-29T05:29:29.714 INFO:journalctl@ceph.mon.a.smithi083.stdout: data_block_hash_table_util_ratio: 0.750000 2022-09-29T05:29:29.714 INFO:journalctl@ceph.mon.a.smithi083.stdout: hash_index_allow_collision: 1 2022-09-29T05:29:29.714 INFO:journalctl@ceph.mon.a.smithi083.stdout: checksum: 1 2022-09-29T05:29:29.714 INFO:journalctl@ceph.mon.a.smithi083.stdout: no_block_cache: 0 2022-09-29T05:29:29.715 INFO:journalctl@ceph.mon.a.smithi083.stdout: block_cache: 0x55d1bfc5e9b0 2022-09-29T05:29:29.715 INFO:journalctl@ceph.mon.a.smithi083.stdout: block_cache_name: BinnedLRUCache 2022-09-29T05:29:29.715 INFO:journalctl@ceph.mon.a.smithi083.stdout: block_cache_options: 2022-09-29T05:29:29.715 INFO:journalctl@ceph.mon.a.smithi083.stdout: capacity : 536870912 2022-09-29T05:29:29.716 INFO:journalctl@ceph.mon.a.smithi083.stdout: num_shard_bits : 4 2022-09-29T05:29:29.716 INFO:journalctl@ceph.mon.a.smithi083.stdout: strict_capacity_limit : 0 2022-09-29T05:29:29.716 INFO:journalctl@ceph.mon.a.smithi083.stdout: high_pri_pool_ratio: 0.000 2022-09-29T05:29:29.716 INFO:journalctl@ceph.mon.a.smithi083.stdout: block_cache_compressed: (nil) 2022-09-29T05:29:29.716 INFO:journalctl@ceph.mon.a.smithi083.stdout: persistent_cache: (nil) 2022-09-29T05:29:29.717 INFO:journalctl@ceph.mon.a.smithi083.stdout: block_size: 4096 2022-09-29T05:29:29.717 INFO:journalctl@ceph.mon.a.smithi083.stdout: block_size_deviation: 10 2022-09-29T05:29:29.717 INFO:journalctl@ceph.mon.a.smithi083.stdout: block_restart_interval: 16 2022-09-29T05:29:29.717 INFO:journalctl@ceph.mon.a.smithi083.stdout: index_block_restart_interval: 1 2022-09-29T05:29:29.718 INFO:journalctl@ceph.mon.a.smithi083.stdout: metadata_block_size: 4096 2022-09-29T05:29:29.718 INFO:journalctl@ceph.mon.a.smithi083.stdout: partition_filters: 0 2022-09-29T05:29:29.718 INFO:journalctl@ceph.mon.a.smithi083.stdout: use_delta_encoding: 1 2022-09-29T05:29:29.718 INFO:journalctl@ceph.mon.a.smithi083.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2022-09-29T05:29:29.719 INFO:journalctl@ceph.mon.a.smithi083.stdout: whole_key_filtering: 1 2022-09-29T05:29:29.719 INFO:journalctl@ceph.mon.a.smithi083.stdout: verify_compression: 0 2022-09-29T05:29:29.719 INFO:journalctl@ceph.mon.a.smithi083.stdout: read_amp_bytes_per_bit: 0 2022-09-29T05:29:29.719 INFO:journalctl@ceph.mon.a.smithi083.stdout: format_version: 4 2022-09-29T05:29:29.719 INFO:journalctl@ceph.mon.a.smithi083.stdout: enable_index_compression: 1 2022-09-29T05:29:29.720 INFO:journalctl@ceph.mon.a.smithi083.stdout: block_align: 0 2022-09-29T05:29:29.720 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.write_buffer_size: 33554432 2022-09-29T05:29:29.720 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.max_write_buffer_number: 2 2022-09-29T05:29:29.720 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.compression: NoCompression 2022-09-29T05:29:29.720 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.bottommost_compression: Disabled 2022-09-29T05:29:29.721 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.prefix_extractor: nullptr 2022-09-29T05:29:29.721 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2022-09-29T05:29:29.721 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.num_levels: 7 2022-09-29T05:29:29.721 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2022-09-29T05:29:29.722 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2022-09-29T05:29:29.722 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2022-09-29T05:29:29.722 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2022-09-29T05:29:29.722 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.bottommost_compression_opts.level: 32767 2022-09-29T05:29:29.723 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2022-09-29T05:29:29.723 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2022-09-29T05:29:29.723 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2022-09-29T05:29:29.723 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2022-09-29T05:29:29.723 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.bottommost_compression_opts.enabled: false 2022-09-29T05:29:29.724 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.compression_opts.window_bits: -14 2022-09-29T05:29:29.724 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.compression_opts.level: 32767 2022-09-29T05:29:29.724 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.compression_opts.strategy: 0 2022-09-29T05:29:29.724 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2022-09-29T05:29:29.725 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2022-09-29T05:29:29.725 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.compression_opts.parallel_threads: 1 2022-09-29T05:29:29.725 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.compression_opts.enabled: false 2022-09-29T05:29:29.725 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2022-09-29T05:29:29.725 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2022-09-29T05:29:29.726 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.level0_stop_writes_trigger: 36 2022-09-29T05:29:29.726 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.target_file_size_base: 67108864 2022-09-29T05:29:29.726 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.target_file_size_multiplier: 1 2022-09-29T05:29:29.726 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.max_bytes_for_level_base: 268435456 2022-09-29T05:29:29.727 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2022-09-29T05:29:29.727 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2022-09-29T05:29:29.727 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2022-09-29T05:29:29.727 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2022-09-29T05:29:29.727 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2022-09-29T05:29:29.728 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2022-09-29T05:29:29.728 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2022-09-29T05:29:29.728 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2022-09-29T05:29:29.728 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2022-09-29T05:29:29.729 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2022-09-29T05:29:29.729 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.max_compaction_bytes: 1677721600 2022-09-29T05:29:29.729 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.arena_block_size: 4194304 2022-09-29T05:29:29.729 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2022-09-29T05:29:29.729 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2022-09-29T05:29:29.730 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2022-09-29T05:29:29.730 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.disable_auto_compactions: 0 2022-09-29T05:29:29.730 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2022-09-29T05:29:29.730 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2022-09-29T05:29:29.731 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2022-09-29T05:29:29.731 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2022-09-29T05:29:29.731 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2022-09-29T05:29:29.731 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2022-09-29T05:29:29.732 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2022-09-29T05:29:29.732 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2022-09-29T05:29:29.732 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2022-09-29T05:29:29.732 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2022-09-29T05:29:29.733 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.table_properties_collectors: 2022-09-29T05:29:29.733 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.inplace_update_support: 0 2022-09-29T05:29:29.733 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.inplace_update_num_locks: 10000 2022-09-29T05:29:29.733 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2022-09-29T05:29:29.733 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.memtable_whole_key_filtering: 0 2022-09-29T05:29:29.734 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.memtable_huge_page_size: 0 2022-09-29T05:29:29.734 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.bloom_locality: 0 2022-09-29T05:29:29.734 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.max_successive_merges: 0 2022-09-29T05:29:29.734 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.optimize_filters_for_hits: 0 2022-09-29T05:29:29.735 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.paranoid_file_checks: 0 2022-09-29T05:29:29.735 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.force_consistency_checks: 1 2022-09-29T05:29:29.735 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.report_bg_io_stats: 0 2022-09-29T05:29:29.735 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.ttl: 2592000 2022-09-29T05:29:29.735 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.periodic_compaction_seconds: 0 2022-09-29T05:29:29.736 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.enable_blob_files: false 2022-09-29T05:29:29.736 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.min_blob_size: 0 2022-09-29T05:29:29.736 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.blob_file_size: 268435456 2022-09-29T05:29:29.736 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.blob_compression_type: NoCompression 2022-09-29T05:29:29.737 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.enable_blob_garbage_collection: false 2022-09-29T05:29:29.737 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2022-09-29T05:29:29.737 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: [db/version_set.cc:3457] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2022-09-29T05:29:29.737 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000008 succeeded,manifest_file_number is 8, next_file_number is 19, last_sequence is 8822, log_number is 15,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2022-09-29T05:29:29.737 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 15 2022-09-29T05:29:29.738 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: [db/version_set.cc:4082] Creating manifest 19 2022-09-29T05:29:29.738 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: [db/version_set.cc:3457] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2022-09-29T05:29:29.738 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: EVENT_LOG_v1 {"time_micros": 1664429369601350, "job": 1, "event": "recovery_started", "wal_files": [15]} 2022-09-29T05:29:29.738 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #15 mode 2 2022-09-29T05:29:29.739 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2022-09-29T05:29:29.739 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: EVENT_LOG_v1 {"time_micros": 1664429369604193, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 20, "file_size": 1008526, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 1006177, "index_size": 1094, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 325, "raw_key_size": 2489, "raw_average_key_size": 23, "raw_value_size": 1003241, "raw_average_value_size": 9554, "num_data_blocks": 49, "num_entries": 105, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1664429369, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "1ebb2a7f-7401-485f-883f-9deb2133f9ca", "db_session_id": "AE3SROYLISI6MGTVDIZB"}} 2022-09-29T05:29:29.739 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: [db/version_set.cc:4082] Creating manifest 21 2022-09-29T05:29:29.739 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: [db/version_set.cc:3457] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2022-09-29T05:29:29.739 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: EVENT_LOG_v1 {"time_micros": 1664429369604555, "job": 1, "event": "recovery_finished"} 2022-09-29T05:29:29.740 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000015.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2022-09-29T05:29:29.740 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x55d1bfc45880 2022-09-29T05:29:29.740 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: DB pointer 0x55d1bfc54000 2022-09-29T05:29:29.740 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2022-09-29T05:29:29.741 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: rocksdb: [db/db_impl/db_impl.cc:903] 2022-09-29T05:29:29.741 INFO:journalctl@ceph.mon.a.smithi083.stdout: ** DB Stats ** 2022-09-29T05:29:29.741 INFO:journalctl@ceph.mon.a.smithi083.stdout: Uptime(secs): 0.0 total, 0.0 interval 2022-09-29T05:29:29.741 INFO:journalctl@ceph.mon.a.smithi083.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2022-09-29T05:29:29.741 INFO:journalctl@ceph.mon.a.smithi083.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2022-09-29T05:29:29.742 INFO:journalctl@ceph.mon.a.smithi083.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2022-09-29T05:29:29.742 INFO:journalctl@ceph.mon.a.smithi083.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2022-09-29T05:29:29.742 INFO:journalctl@ceph.mon.a.smithi083.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2022-09-29T05:29:29.742 INFO:journalctl@ceph.mon.a.smithi083.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2022-09-29T05:29:29.743 INFO:journalctl@ceph.mon.a.smithi083.stdout: 2022-09-29T05:29:29.743 INFO:journalctl@ceph.mon.a.smithi083.stdout: ** Compaction Stats [default] ** 2022-09-29T05:29:29.743 INFO:journalctl@ceph.mon.a.smithi083.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2022-09-29T05:29:29.743 INFO:journalctl@ceph.mon.a.smithi083.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2022-09-29T05:29:29.743 INFO:journalctl@ceph.mon.a.smithi083.stdout: L0 1/0 984.89 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 527.9 0.00 0.00 1 0.002 0 0 2022-09-29T05:29:29.744 INFO:journalctl@ceph.mon.a.smithi083.stdout: L6 1/0 20.41 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2022-09-29T05:29:29.744 INFO:journalctl@ceph.mon.a.smithi083.stdout: Sum 2/0 21.37 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 527.9 0.00 0.00 1 0.002 0 0 2022-09-29T05:29:29.744 INFO:journalctl@ceph.mon.a.smithi083.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 527.9 0.00 0.00 1 0.002 0 0 2022-09-29T05:29:29.744 INFO:journalctl@ceph.mon.a.smithi083.stdout: 2022-09-29T05:29:29.745 INFO:journalctl@ceph.mon.a.smithi083.stdout: ** Compaction Stats [default] ** 2022-09-29T05:29:29.745 INFO:journalctl@ceph.mon.a.smithi083.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2022-09-29T05:29:29.745 INFO:journalctl@ceph.mon.a.smithi083.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2022-09-29T05:29:29.745 INFO:journalctl@ceph.mon.a.smithi083.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 527.9 0.00 0.00 1 0.002 0 0 2022-09-29T05:29:29.745 INFO:journalctl@ceph.mon.a.smithi083.stdout: Uptime(secs): 0.0 total, 0.0 interval 2022-09-29T05:29:29.746 INFO:journalctl@ceph.mon.a.smithi083.stdout: Flush(GB): cumulative 0.001, interval 0.001 2022-09-29T05:29:29.746 INFO:journalctl@ceph.mon.a.smithi083.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2022-09-29T05:29:29.746 INFO:journalctl@ceph.mon.a.smithi083.stdout: AddFile(Total Files): cumulative 0, interval 0 2022-09-29T05:29:29.746 INFO:journalctl@ceph.mon.a.smithi083.stdout: AddFile(L0 Files): cumulative 0, interval 0 2022-09-29T05:29:29.747 INFO:journalctl@ceph.mon.a.smithi083.stdout: AddFile(Keys): cumulative 0, interval 0 2022-09-29T05:29:29.747 INFO:journalctl@ceph.mon.a.smithi083.stdout: Cumulative compaction: 0.00 GB write, 136.58 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2022-09-29T05:29:29.747 INFO:journalctl@ceph.mon.a.smithi083.stdout: Interval compaction: 0.00 GB write, 136.58 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2022-09-29T05:29:29.747 INFO:journalctl@ceph.mon.a.smithi083.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2022-09-29T05:29:29.747 INFO:journalctl@ceph.mon.a.smithi083.stdout: 2022-09-29T05:29:29.748 INFO:journalctl@ceph.mon.a.smithi083.stdout: ** File Read Latency Histogram By Level [default] ** 2022-09-29T05:29:29.748 INFO:journalctl@ceph.mon.a.smithi083.stdout: 2022-09-29T05:29:29.748 INFO:journalctl@ceph.mon.a.smithi083.stdout: ** Compaction Stats [default] ** 2022-09-29T05:29:29.748 INFO:journalctl@ceph.mon.a.smithi083.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2022-09-29T05:29:29.749 INFO:journalctl@ceph.mon.a.smithi083.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2022-09-29T05:29:29.749 INFO:journalctl@ceph.mon.a.smithi083.stdout: L0 1/0 984.89 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 527.9 0.00 0.00 1 0.002 0 0 2022-09-29T05:29:29.749 INFO:journalctl@ceph.mon.a.smithi083.stdout: L6 1/0 20.41 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2022-09-29T05:29:29.749 INFO:journalctl@ceph.mon.a.smithi083.stdout: Sum 2/0 21.37 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 527.9 0.00 0.00 1 0.002 0 0 2022-09-29T05:29:29.749 INFO:journalctl@ceph.mon.a.smithi083.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2022-09-29T05:29:29.750 INFO:journalctl@ceph.mon.a.smithi083.stdout: 2022-09-29T05:29:29.750 INFO:journalctl@ceph.mon.a.smithi083.stdout: ** Compaction Stats [default] ** 2022-09-29T05:29:29.750 INFO:journalctl@ceph.mon.a.smithi083.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2022-09-29T05:29:29.750 INFO:journalctl@ceph.mon.a.smithi083.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2022-09-29T05:29:29.751 INFO:journalctl@ceph.mon.a.smithi083.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 527.9 0.00 0.00 1 0.002 0 0 2022-09-29T05:29:29.751 INFO:journalctl@ceph.mon.a.smithi083.stdout: Uptime(secs): 0.0 total, 0.0 interval 2022-09-29T05:29:29.751 INFO:journalctl@ceph.mon.a.smithi083.stdout: Flush(GB): cumulative 0.001, interval 0.000 2022-09-29T05:29:29.751 INFO:journalctl@ceph.mon.a.smithi083.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2022-09-29T05:29:29.752 INFO:journalctl@ceph.mon.a.smithi083.stdout: AddFile(Total Files): cumulative 0, interval 0 2022-09-29T05:29:29.752 INFO:journalctl@ceph.mon.a.smithi083.stdout: AddFile(L0 Files): cumulative 0, interval 0 2022-09-29T05:29:29.752 INFO:journalctl@ceph.mon.a.smithi083.stdout: AddFile(Keys): cumulative 0, interval 0 2022-09-29T05:29:29.752 INFO:journalctl@ceph.mon.a.smithi083.stdout: Cumulative compaction: 0.00 GB write, 135.93 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2022-09-29T05:29:29.753 INFO:journalctl@ceph.mon.a.smithi083.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2022-09-29T05:29:29.753 INFO:journalctl@ceph.mon.a.smithi083.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2022-09-29T05:29:29.753 INFO:journalctl@ceph.mon.a.smithi083.stdout: 2022-09-29T05:29:29.753 INFO:journalctl@ceph.mon.a.smithi083.stdout: ** File Read Latency Histogram By Level [default] ** 2022-09-29T05:29:29.753 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: starting mon.a rank 0 at public addrs [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] at bind addrs [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 69111522-3fb6-11ed-8431-001a4aab830c 2022-09-29T05:29:29.754 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: mon.a@-1(???) e3 preinit fsid 69111522-3fb6-11ed-8431-001a4aab830c 2022-09-29T05:29:29.754 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: mon.a@-1(???).mds e6 new map 2022-09-29T05:29:29.754 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: mon.a@-1(???).mds e6 print_map 2022-09-29T05:29:29.754 INFO:journalctl@ceph.mon.a.smithi083.stdout: e6 2022-09-29T05:29:29.755 INFO:journalctl@ceph.mon.a.smithi083.stdout: enable_multiple, ever_enabled_multiple: 1,1 2022-09-29T05:29:29.755 INFO:journalctl@ceph.mon.a.smithi083.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2022-09-29T05:29:29.755 INFO:journalctl@ceph.mon.a.smithi083.stdout: legacy client fscid: 1 2022-09-29T05:29:29.755 INFO:journalctl@ceph.mon.a.smithi083.stdout: 2022-09-29T05:29:29.755 INFO:journalctl@ceph.mon.a.smithi083.stdout: Filesystem 'cephfs' (1) 2022-09-29T05:29:29.756 INFO:journalctl@ceph.mon.a.smithi083.stdout: fs_name cephfs 2022-09-29T05:29:29.756 INFO:journalctl@ceph.mon.a.smithi083.stdout: epoch 6 2022-09-29T05:29:29.756 INFO:journalctl@ceph.mon.a.smithi083.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2022-09-29T05:29:29.757 INFO:journalctl@ceph.mon.a.smithi083.stdout: created 2022-09-29T05:25:03.789615+0000 2022-09-29T05:29:29.757 INFO:journalctl@ceph.mon.a.smithi083.stdout: modified 2022-09-29T05:25:05.816761+0000 2022-09-29T05:29:29.757 INFO:journalctl@ceph.mon.a.smithi083.stdout: tableserver 0 2022-09-29T05:29:29.758 INFO:journalctl@ceph.mon.a.smithi083.stdout: root 0 2022-09-29T05:29:29.758 INFO:journalctl@ceph.mon.a.smithi083.stdout: session_timeout 60 2022-09-29T05:29:29.759 INFO:journalctl@ceph.mon.a.smithi083.stdout: session_autoclose 300 2022-09-29T05:29:29.759 INFO:journalctl@ceph.mon.a.smithi083.stdout: max_file_size 1099511627776 2022-09-29T05:29:29.760 INFO:journalctl@ceph.mon.a.smithi083.stdout: required_client_features {} 2022-09-29T05:29:29.760 INFO:journalctl@ceph.mon.a.smithi083.stdout: last_failure 0 2022-09-29T05:29:29.760 INFO:journalctl@ceph.mon.a.smithi083.stdout: last_failure_osd_epoch 0 2022-09-29T05:29:29.761 INFO:journalctl@ceph.mon.a.smithi083.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2022-09-29T05:29:29.761 INFO:journalctl@ceph.mon.a.smithi083.stdout: max_mds 1 2022-09-29T05:29:29.762 INFO:journalctl@ceph.mon.a.smithi083.stdout: in 0 2022-09-29T05:29:29.762 INFO:journalctl@ceph.mon.a.smithi083.stdout: up {0=24356} 2022-09-29T05:29:29.763 INFO:journalctl@ceph.mon.a.smithi083.stdout: failed 2022-09-29T05:29:29.763 INFO:journalctl@ceph.mon.a.smithi083.stdout: damaged 2022-09-29T05:29:29.763 INFO:journalctl@ceph.mon.a.smithi083.stdout: stopped 2022-09-29T05:29:29.764 INFO:journalctl@ceph.mon.a.smithi083.stdout: data_pools [3] 2022-09-29T05:29:29.764 INFO:journalctl@ceph.mon.a.smithi083.stdout: metadata_pool 2 2022-09-29T05:29:29.765 INFO:journalctl@ceph.mon.a.smithi083.stdout: inline_data disabled 2022-09-29T05:29:29.765 INFO:journalctl@ceph.mon.a.smithi083.stdout: balancer 2022-09-29T05:29:29.765 INFO:journalctl@ceph.mon.a.smithi083.stdout: standby_count_wanted 0 2022-09-29T05:29:29.766 INFO:journalctl@ceph.mon.a.smithi083.stdout: [mds.b{0:24356} state up:active seq 3 addr [v2:172.21.15.153:6832/673217382,v1:172.21.15.153:6833/673217382] compat {c=[1],r=[1],i=[7ff]}] 2022-09-29T05:29:29.766 INFO:journalctl@ceph.mon.a.smithi083.stdout: 2022-09-29T05:29:29.767 INFO:journalctl@ceph.mon.a.smithi083.stdout: 2022-09-29T05:29:29.767 INFO:journalctl@ceph.mon.a.smithi083.stdout: Standby daemons: 2022-09-29T05:29:29.768 INFO:journalctl@ceph.mon.a.smithi083.stdout: 2022-09-29T05:29:29.768 INFO:journalctl@ceph.mon.a.smithi083.stdout: [mds.a{-1:24343} state up:standby seq 1 addr [v2:172.21.15.83:6834/4236763119,v1:172.21.15.83:6835/4236763119] compat {c=[1],r=[1],i=[7ff]}] 2022-09-29T05:29:29.768 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: mon.a@-1(???).osd e63 crush map has features 3314933000854323200, adjusting msgr requires 2022-09-29T05:29:29.769 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: mon.a@-1(???).osd e63 crush map has features 432629239337189376, adjusting msgr requires 2022-09-29T05:29:29.769 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: mon.a@-1(???).osd e63 crush map has features 432629239337189376, adjusting msgr requires 2022-09-29T05:29:29.770 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: mon.a@-1(???).osd e63 crush map has features 432629239337189376, adjusting msgr requires 2022-09-29T05:29:29.770 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 ceph-mon[77242]: mon.a@-1(???).paxosservice(auth 1..18) refresh upgraded, format 0 -> 3 2022-09-29T05:29:29.771 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:29:29 smithi083 systemd[1]: Started Ceph mon.a for 69111522-3fb6-11ed-8431-001a4aab830c. 2022-09-29T05:29:31.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:29:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:29:31] "GET /metrics HTTP/1.1" 200 176143 "" "Prometheus/2.33.4" 2022-09-29T05:29:32.691 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:29:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:29:32.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:29:32.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:29:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ignoring --setuser ceph since I am not root 2022-09-29T05:29:32.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:29:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ignoring --setgroup ceph since I am not root 2022-09-29T05:29:32.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:29:32 smithi083 ceph-mgr[73090]: -- 172.21.15.83:0/1906365463 <== mon.1 v2:172.21.15.83:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55e43866e820 con 0x55e438642800 2022-09-29T05:29:32.964 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:29:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: 2022-09-29T05:29:32.910+0000 7f9b6df98200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2022-09-29T05:29:32.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:29:28.031345+0000 mgr.y (mgr.15009) 43 : cluster [DBG] pgmap v19: 105 pgs: 105 active+clean; 582 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:29:32.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:29:30.031953+0000 mgr.y (mgr.15009) 44 : cluster [DBG] pgmap v20: 105 pgs: 105 active+clean; 582 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:29:32.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:29:32.032386+0000 mgr.y (mgr.15009) 45 : cluster [DBG] pgmap v21: 105 pgs: 105 active+clean; 582 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:29:32.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:29:32.621691+0000 mon.a (mon.0) 1 : cluster [INF] mon.a calling monitor election 2022-09-29T05:29:32.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:29:32.635189+0000 mon.a (mon.0) 2 : cluster [INF] mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2022-09-29T05:29:32.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:29:32.641477+0000 mon.a (mon.0) 3 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0],b=[v2:172.21.15.153:3300/0,v1:172.21.15.153:6789/0],c=[v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0]} 2022-09-29T05:29:32.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:29:32.642132+0000 mon.a (mon.0) 4 : cluster [DBG] fsmap cephfs:1 {0=b=up:active} 1 up:standby 2022-09-29T05:29:32.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:29:32.642166+0000 mon.a (mon.0) 5 : cluster [DBG] osdmap e63: 8 total, 8 up, 8 in 2022-09-29T05:29:32.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:29:32.643297+0000 mon.a (mon.0) 6 : cluster [DBG] mgrmap e31: y(active, since 35s), standbys: x 2022-09-29T05:29:32.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:29:32.654930+0000 mon.a (mon.0) 7 : cluster [DBG] mgrmap e32: y(active, since 35s), standbys: x 2022-09-29T05:29:32.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:29:32.654972+0000 mon.a (mon.0) 8 : cluster [WRN] Health detail: HEALTH_WARN Telemetry requires re-opt-in 2022-09-29T05:29:32.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:29:32.654986+0000 mon.a (mon.0) 9 : cluster [WRN] [WRN] TELEMETRY_CHANGED: Telemetry requires re-opt-in 2022-09-29T05:29:32.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:29:32.654999+0000 mon.a (mon.0) 10 : cluster [WRN] telemetry module includes new collections; please re-opt-in to new collections with `ceph telemetry on` 2022-09-29T05:29:32.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:32.655093+0000 mon.a (mon.0) 11 : audit [INF] from='mgr.15009 ' entity='' 2022-09-29T05:29:33.059 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ignoring --setuser ceph since I am not root 2022-09-29T05:29:33.059 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ignoring --setgroup ceph since I am not root 2022-09-29T05:29:33.059 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:32 smithi153 ceph-mgr[65956]: -- 172.21.15.153:0/2941219813 <== mon.2 v2:172.21.15.153:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x561e66e0e820 con 0x561e66de2c00 2022-09-29T05:29:33.060 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: 2022-09-29T05:29:32.905+0000 7f7b3b8f7200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2022-09-29T05:29:33.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:29:28.031345+0000 mgr.y (mgr.15009) 43 : cluster [DBG] pgmap v19: 105 pgs: 105 active+clean; 582 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:29:33.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:29:30.031953+0000 mgr.y (mgr.15009) 44 : cluster [DBG] pgmap v20: 105 pgs: 105 active+clean; 582 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:29:33.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:29:32.032386+0000 mgr.y (mgr.15009) 45 : cluster [DBG] pgmap v21: 105 pgs: 105 active+clean; 582 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:29:33.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:29:32.621691+0000 mon.a (mon.0) 1 : cluster [INF] mon.a calling monitor election 2022-09-29T05:29:33.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:29:32.635189+0000 mon.a (mon.0) 2 : cluster [INF] mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2022-09-29T05:29:33.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:29:32.641477+0000 mon.a (mon.0) 3 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0],b=[v2:172.21.15.153:3300/0,v1:172.21.15.153:6789/0],c=[v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0]} 2022-09-29T05:29:33.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:29:32.642132+0000 mon.a (mon.0) 4 : cluster [DBG] fsmap cephfs:1 {0=b=up:active} 1 up:standby 2022-09-29T05:29:33.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:29:32.642166+0000 mon.a (mon.0) 5 : cluster [DBG] osdmap e63: 8 total, 8 up, 8 in 2022-09-29T05:29:33.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:29:32.643297+0000 mon.a (mon.0) 6 : cluster [DBG] mgrmap e31: y(active, since 35s), standbys: x 2022-09-29T05:29:33.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:29:32.654930+0000 mon.a (mon.0) 7 : cluster [DBG] mgrmap e32: y(active, since 35s), standbys: x 2022-09-29T05:29:33.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:29:32.654972+0000 mon.a (mon.0) 8 : cluster [WRN] Health detail: HEALTH_WARN Telemetry requires re-opt-in 2022-09-29T05:29:33.063 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:29:32.654986+0000 mon.a (mon.0) 9 : cluster [WRN] [WRN] TELEMETRY_CHANGED: Telemetry requires re-opt-in 2022-09-29T05:29:33.063 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:29:32.654999+0000 mon.a (mon.0) 10 : cluster [WRN] telemetry module includes new collections; please re-opt-in to new collections with `ceph telemetry on` 2022-09-29T05:29:33.063 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:32.655093+0000 mon.a (mon.0) 11 : audit [INF] from='mgr.15009 ' entity='' 2022-09-29T05:29:33.463 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:29:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: 2022-09-29T05:29:33.103+0000 7f9b6df98200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2022-09-29T05:29:33.559 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: 2022-09-29T05:29:33.100+0000 7f7b3b8f7200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2022-09-29T05:29:34.213 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:29:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: 2022-09-29T05:29:33.858+0000 7f9b6df98200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2022-09-29T05:29:34.213 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:29:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: 2022-09-29T05:29:34.031+0000 7f9b6df98200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2022-09-29T05:29:34.243 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: 2022-09-29T05:29:33.841+0000 7f7b3b8f7200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2022-09-29T05:29:34.243 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:34 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: 2022-09-29T05:29:34.019+0000 7f7b3b8f7200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2022-09-29T05:29:34.560 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:34 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: 2022-09-29T05:29:34.239+0000 7f7b3b8f7200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2022-09-29T05:29:34.560 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:34 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: 2022-09-29T05:29:34.400+0000 7f7b3b8f7200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2022-09-29T05:29:34.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:29:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: 2022-09-29T05:29:34.250+0000 7f9b6df98200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2022-09-29T05:29:34.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:29:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: 2022-09-29T05:29:34.409+0000 7f9b6df98200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2022-09-29T05:29:35.015 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:34 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: 2022-09-29T05:29:34.722+0000 7f7b3b8f7200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2022-09-29T05:29:35.092 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:29:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: 2022-09-29T05:29:34.728+0000 7f9b6df98200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2022-09-29T05:29:35.271 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:35 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: 2022-09-29T05:29:35.010+0000 7f7b3b8f7200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2022-09-29T05:29:35.273 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:29:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: 2022-09-29T05:29:35.015+0000 7f9b6df98200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2022-09-29T05:29:35.536 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:35 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: 2022-09-29T05:29:35.267+0000 7f7b3b8f7200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2022-09-29T05:29:35.536 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:35 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: 2022-09-29T05:29:35.355+0000 7f7b3b8f7200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2022-09-29T05:29:35.536 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:35 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: 2022-09-29T05:29:35.532+0000 7f7b3b8f7200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2022-09-29T05:29:35.538 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:29:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: 2022-09-29T05:29:35.270+0000 7f9b6df98200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2022-09-29T05:29:35.538 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:29:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: 2022-09-29T05:29:35.357+0000 7f9b6df98200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2022-09-29T05:29:35.809 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:35 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: 2022-09-29T05:29:35.614+0000 7f7b3b8f7200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2022-09-29T05:29:35.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:29:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: 2022-09-29T05:29:35.534+0000 7f9b6df98200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2022-09-29T05:29:35.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:29:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: 2022-09-29T05:29:35.617+0000 7f9b6df98200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2022-09-29T05:29:36.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:36 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: 2022-09-29T05:29:36.019+0000 7f7b3b8f7200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2022-09-29T05:29:36.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:36 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: 2022-09-29T05:29:36.110+0000 7f7b3b8f7200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2022-09-29T05:29:36.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:36 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: 2022-09-29T05:29:36.196+0000 7f7b3b8f7200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2022-09-29T05:29:36.407 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:29:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: 2022-09-29T05:29:36.020+0000 7f9b6df98200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2022-09-29T05:29:36.408 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:29:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: 2022-09-29T05:29:36.112+0000 7f9b6df98200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2022-09-29T05:29:36.408 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:29:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: 2022-09-29T05:29:36.198+0000 7f9b6df98200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2022-09-29T05:29:36.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:29:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: 2022-09-29T05:29:36.403+0000 7f9b6df98200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2022-09-29T05:29:36.809 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:36 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: 2022-09-29T05:29:36.405+0000 7f7b3b8f7200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2022-09-29T05:29:37.380 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:29:37 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: 2022-09-29T05:29:37.103+0000 7f9b6df98200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2022-09-29T05:29:37.380 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:29:37 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: 2022-09-29T05:29:37.195+0000 7f9b6df98200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2022-09-29T05:29:37.381 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:29:37 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: 2022-09-29T05:29:37.289+0000 7f9b6df98200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2022-09-29T05:29:37.386 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:37 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: 2022-09-29T05:29:37.108+0000 7f7b3b8f7200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2022-09-29T05:29:37.387 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:37 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: 2022-09-29T05:29:37.200+0000 7f7b3b8f7200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2022-09-29T05:29:37.387 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:37 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: 2022-09-29T05:29:37.295+0000 7f7b3b8f7200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2022-09-29T05:29:37.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:29:37 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: 2022-09-29T05:29:37.376+0000 7f9b6df98200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2022-09-29T05:29:37.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:29:37 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: 2022-09-29T05:29:37.524+0000 7f9b6df98200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2022-09-29T05:29:37.809 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:37 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: 2022-09-29T05:29:37.383+0000 7f7b3b8f7200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2022-09-29T05:29:37.809 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:37 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: 2022-09-29T05:29:37.531+0000 7f7b3b8f7200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2022-09-29T05:29:38.792 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:29:38 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: 2022-09-29T05:29:38.504+0000 7f9b6df98200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2022-09-29T05:29:38.793 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:29:38 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: 2022-09-29T05:29:38.619+0000 7f9b6df98200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2022-09-29T05:29:38.793 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:29:38 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: 2022-09-29T05:29:38.712+0000 7f9b6df98200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2022-09-29T05:29:38.794 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:38 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: 2022-09-29T05:29:38.514+0000 7f7b3b8f7200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2022-09-29T05:29:38.794 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:38 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: 2022-09-29T05:29:38.629+0000 7f7b3b8f7200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2022-09-29T05:29:38.795 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:38 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: 2022-09-29T05:29:38.722+0000 7f7b3b8f7200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2022-09-29T05:29:38.795 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:38 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: [29/Sep/2022:05:29:38] ENGINE Bus STARTING 2022-09-29T05:29:38.796 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:29:38 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:29:38.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi153.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi083:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:29:38.796 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:29:38 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:29:38.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephPGImbalance\nexpr: abs(((ceph_osd_numpg > 0) - on(job) group_left() avg by(job) (ceph_osd_numpg\n > 0)) / on(job) group_left() avg by(job) (ceph_osd_numpg > 0)) * on(ceph_daemon)\n group_left(hostname) ceph_osd_metadata > 0.3\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.5\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} deviates by\n more than 30% from average PG count.\n summary: PGs are not balanced across OSDs\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi153.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi083:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:29:39.059 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:38 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: CherryPy Checker: 2022-09-29T05:29:39.059 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:38 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: The Application mounted at '' has an empty config. 2022-09-29T05:29:39.060 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:38 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: 2022-09-29T05:29:39.060 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:38 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: [29/Sep/2022:05:29:38] ENGINE Serving on http://:::9283 2022-09-29T05:29:39.060 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:38 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: [29/Sep/2022:05:29:38] ENGINE Bus STARTED 2022-09-29T05:29:39.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:38 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:29:38.726859+0000 mon.a (mon.0) 12 : cluster [INF] Active manager daemon y restarted 2022-09-29T05:29:39.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:38 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:29:38.727430+0000 mon.a (mon.0) 13 : cluster [INF] Activating manager daemon y 2022-09-29T05:29:39.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:38 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:29:38.732678+0000 mon.a (mon.0) 14 : cluster [DBG] osdmap e64: 8 total, 8 up, 8 in 2022-09-29T05:29:39.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:38 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:29:38.736034+0000 mon.a (mon.0) 15 : cluster [DBG] Standby manager daemon x restarted 2022-09-29T05:29:39.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:38 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:29:38.736118+0000 mon.a (mon.0) 16 : cluster [DBG] Standby manager daemon x started 2022-09-29T05:29:39.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:38 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:38.737874+0000 mon.b (mon.2) 91 : audit [DBG] from='mgr.? 172.21.15.153:0/2095332738' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2022-09-29T05:29:39.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:38 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:38.738408+0000 mon.b (mon.2) 92 : audit [DBG] from='mgr.? 172.21.15.153:0/2095332738' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2022-09-29T05:29:39.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:38 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:38.740291+0000 mon.b (mon.2) 93 : audit [DBG] from='mgr.? 172.21.15.153:0/2095332738' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2022-09-29T05:29:39.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:38 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:38.741408+0000 mon.b (mon.2) 94 : audit [DBG] from='mgr.? 172.21.15.153:0/2095332738' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2022-09-29T05:29:39.131 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:29:38 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: [29/Sep/2022:05:29:38] ENGINE Bus STARTING 2022-09-29T05:29:39.132 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:38 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:29:38.726859+0000 mon.a (mon.0) 12 : cluster [INF] Active manager daemon y restarted 2022-09-29T05:29:39.132 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:38 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:29:38.727430+0000 mon.a (mon.0) 13 : cluster [INF] Activating manager daemon y 2022-09-29T05:29:39.133 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:38 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:29:38.732678+0000 mon.a (mon.0) 14 : cluster [DBG] osdmap e64: 8 total, 8 up, 8 in 2022-09-29T05:29:39.133 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:38 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:29:38.736034+0000 mon.a (mon.0) 15 : cluster [DBG] Standby manager daemon x restarted 2022-09-29T05:29:39.133 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:38 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:29:38.736118+0000 mon.a (mon.0) 16 : cluster [DBG] Standby manager daemon x started 2022-09-29T05:29:39.133 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:38 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:38.737874+0000 mon.b (mon.2) 91 : audit [DBG] from='mgr.? 172.21.15.153:0/2095332738' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2022-09-29T05:29:39.133 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:38 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:38.738408+0000 mon.b (mon.2) 92 : audit [DBG] from='mgr.? 172.21.15.153:0/2095332738' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2022-09-29T05:29:39.134 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:38 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:38.740291+0000 mon.b (mon.2) 93 : audit [DBG] from='mgr.? 172.21.15.153:0/2095332738' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2022-09-29T05:29:39.134 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:38 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:38.741408+0000 mon.b (mon.2) 94 : audit [DBG] from='mgr.? 172.21.15.153:0/2095332738' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2022-09-29T05:29:39.463 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:29:39 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: 2022-09-29T05:29:39.138+0000 7f9acec20700 -1 client.0 error registering admin socket command: (17) File exists 2022-09-29T05:29:39.463 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:29:39 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: 2022-09-29T05:29:39.138+0000 7f9acec20700 -1 client.0 error registering admin socket command: (17) File exists 2022-09-29T05:29:39.463 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:29:39 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: 2022-09-29T05:29:39.138+0000 7f9acec20700 -1 client.0 error registering admin socket command: (17) File exists 2022-09-29T05:29:39.464 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:29:39 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: 2022-09-29T05:29:39.138+0000 7f9acec20700 -1 client.0 error registering admin socket command: (17) File exists 2022-09-29T05:29:39.464 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:29:39 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: 2022-09-29T05:29:39.138+0000 7f9acec20700 -1 client.0 error registering admin socket command: (17) File exists 2022-09-29T05:29:39.464 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:29:39 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: 2022-09-29T05:29:39.138+0000 7f9ad1c26700 -1 client.0 error registering admin socket command: (17) File exists 2022-09-29T05:29:39.464 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:29:39 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: 2022-09-29T05:29:39.138+0000 7f9ad1c26700 -1 client.0 error registering admin socket command: (17) File exists 2022-09-29T05:29:39.465 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:29:39 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: 2022-09-29T05:29:39.138+0000 7f9ad1c26700 -1 client.0 error registering admin socket command: (17) File exists 2022-09-29T05:29:39.465 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:29:39 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: 2022-09-29T05:29:39.138+0000 7f9ad1c26700 -1 client.0 error registering admin socket command: (17) File exists 2022-09-29T05:29:39.465 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:29:39 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: 2022-09-29T05:29:39.138+0000 7f9ad1c26700 -1 client.0 error registering admin socket command: (17) File exists 2022-09-29T05:29:39.465 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:29:39 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: CherryPy Checker: 2022-09-29T05:29:39.466 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:29:39 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: The Application mounted at '' has an empty config. 2022-09-29T05:29:39.466 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:29:39 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: 2022-09-29T05:29:39.466 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:29:39 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: [29/Sep/2022:05:29:39] ENGINE Serving on http://:::9283 2022-09-29T05:29:39.466 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:29:39 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: [29/Sep/2022:05:29:39] ENGINE Bus STARTED 2022-09-29T05:29:40.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:39 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:29:38.784429+0000 mon.a (mon.0) 17 : cluster [DBG] mgrmap e33: y(active, starting, since 0.0570875s), standbys: x 2022-09-29T05:29:40.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:39 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:38.786827+0000 mon.c (mon.1) 89 : audit [DBG] from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2022-09-29T05:29:40.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:39 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:38.786950+0000 mon.c (mon.1) 90 : audit [DBG] from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2022-09-29T05:29:40.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:39 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:38.787070+0000 mon.c (mon.1) 91 : audit [DBG] from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2022-09-29T05:29:40.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:39 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:38.787747+0000 mon.c (mon.1) 92 : audit [DBG] from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2022-09-29T05:29:40.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:39 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:38.787897+0000 mon.c (mon.1) 93 : audit [DBG] from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2022-09-29T05:29:40.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:39 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:38.788188+0000 mon.c (mon.1) 94 : audit [DBG] from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2022-09-29T05:29:40.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:39 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:38.788344+0000 mon.c (mon.1) 95 : audit [DBG] from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2022-09-29T05:29:40.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:39 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:38.788618+0000 mon.c (mon.1) 96 : audit [DBG] from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2022-09-29T05:29:40.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:39 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:38.788771+0000 mon.c (mon.1) 97 : audit [DBG] from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2022-09-29T05:29:40.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:39 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:38.793009+0000 mon.c (mon.1) 98 : audit [DBG] from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2022-09-29T05:29:40.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:39 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:38.793239+0000 mon.c (mon.1) 99 : audit [DBG] from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2022-09-29T05:29:40.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:39 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:38.793469+0000 mon.c (mon.1) 100 : audit [DBG] from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2022-09-29T05:29:40.063 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:39 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:38.793690+0000 mon.c (mon.1) 101 : audit [DBG] from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2022-09-29T05:29:40.063 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:39 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:38.793915+0000 mon.c (mon.1) 102 : audit [DBG] from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2022-09-29T05:29:40.063 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:39 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:38.794094+0000 mon.c (mon.1) 103 : audit [DBG] from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2022-09-29T05:29:40.063 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:39 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:38.794381+0000 mon.c (mon.1) 104 : audit [DBG] from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2022-09-29T05:29:40.063 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:39 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:38.794620+0000 mon.c (mon.1) 105 : audit [DBG] from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2022-09-29T05:29:40.064 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:39 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:38.795122+0000 mon.c (mon.1) 106 : audit [DBG] from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2022-09-29T05:29:40.064 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:39 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:29:38.813489+0000 mon.a (mon.0) 18 : cluster [INF] Manager daemon y is now available 2022-09-29T05:29:40.064 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:39 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:38.833920+0000 mon.c (mon.1) 107 : audit [DBG] from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:29:40.064 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:39 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:38.836911+0000 mon.c (mon.1) 108 : audit [DBG] from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:29:40.065 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:39 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:38.838498+0000 mon.c (mon.1) 109 : audit [DBG] from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:29:40.065 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:39 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:38.839983+0000 mon.c (mon.1) 110 : audit [INF] from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:29:40.065 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:39 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:39.089417+0000 mon.c (mon.1) 111 : audit [INF] from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:29:40.065 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:39 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:39.089857+0000 mon.a (mon.0) 19 : audit [INF] from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:29:40.065 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:39 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:39.103532+0000 mon.c (mon.1) 112 : audit [INF] from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:29:40.066 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:39 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:39.103944+0000 mon.a (mon.0) 20 : audit [INF] from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:29:40.213 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:29:39 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: 2022-09-29T05:29:39.804+0000 7f9b26e95700 -1 mgr.server handle_report got status from non-daemon mon.a 2022-09-29T05:29:40.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:39 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:29:38.784429+0000 mon.a (mon.0) 17 : cluster [DBG] mgrmap e33: y(active, starting, since 0.0570875s), standbys: x 2022-09-29T05:29:40.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:39 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:38.786827+0000 mon.c (mon.1) 89 : audit [DBG] from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2022-09-29T05:29:40.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:39 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:38.786950+0000 mon.c (mon.1) 90 : audit [DBG] from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2022-09-29T05:29:40.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:39 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:38.787070+0000 mon.c (mon.1) 91 : audit [DBG] from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2022-09-29T05:29:40.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:39 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:38.787747+0000 mon.c (mon.1) 92 : audit [DBG] from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2022-09-29T05:29:40.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:39 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:38.787897+0000 mon.c (mon.1) 93 : audit [DBG] from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2022-09-29T05:29:40.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:39 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:38.788188+0000 mon.c (mon.1) 94 : audit [DBG] from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2022-09-29T05:29:40.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:39 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:38.788344+0000 mon.c (mon.1) 95 : audit [DBG] from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2022-09-29T05:29:40.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:39 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:38.788618+0000 mon.c (mon.1) 96 : audit [DBG] from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2022-09-29T05:29:40.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:39 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:38.788771+0000 mon.c (mon.1) 97 : audit [DBG] from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2022-09-29T05:29:40.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:39 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:38.793009+0000 mon.c (mon.1) 98 : audit [DBG] from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2022-09-29T05:29:40.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:39 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:38.793239+0000 mon.c (mon.1) 99 : audit [DBG] from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2022-09-29T05:29:40.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:39 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:38.793469+0000 mon.c (mon.1) 100 : audit [DBG] from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2022-09-29T05:29:40.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:39 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:38.793690+0000 mon.c (mon.1) 101 : audit [DBG] from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2022-09-29T05:29:40.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:39 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:38.793915+0000 mon.c (mon.1) 102 : audit [DBG] from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2022-09-29T05:29:40.218 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:39 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:38.794094+0000 mon.c (mon.1) 103 : audit [DBG] from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2022-09-29T05:29:40.218 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:39 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:38.794381+0000 mon.c (mon.1) 104 : audit [DBG] from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2022-09-29T05:29:40.218 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:39 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:38.794620+0000 mon.c (mon.1) 105 : audit [DBG] from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2022-09-29T05:29:40.218 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:39 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:38.795122+0000 mon.c (mon.1) 106 : audit [DBG] from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2022-09-29T05:29:40.219 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:39 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:29:38.813489+0000 mon.a (mon.0) 18 : cluster [INF] Manager daemon y is now available 2022-09-29T05:29:40.219 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:39 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:38.833920+0000 mon.c (mon.1) 107 : audit [DBG] from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:29:40.219 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:39 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:38.836911+0000 mon.c (mon.1) 108 : audit [DBG] from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:29:40.219 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:39 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:38.838498+0000 mon.c (mon.1) 109 : audit [DBG] from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:29:40.220 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:39 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:38.839983+0000 mon.c (mon.1) 110 : audit [INF] from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:29:40.220 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:39 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:39.089417+0000 mon.c (mon.1) 111 : audit [INF] from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:29:40.220 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:39 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:39.089857+0000 mon.a (mon.0) 19 : audit [INF] from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:29:40.220 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:39 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:39.103532+0000 mon.c (mon.1) 112 : audit [INF] from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:29:40.221 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:39 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:39.103944+0000 mon.a (mon.0) 20 : audit [INF] from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:29:41.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:40 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:29:39.779652+0000 mgr.y (mgr.34107) 1 : cephadm [INF] [29/Sep/2022:05:29:39] ENGINE Bus STARTING 2022-09-29T05:29:41.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:40 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:29:39.800717+0000 mon.a (mon.0) 21 : cluster [DBG] mgrmap e34: y(active, since 1.07338s), standbys: x 2022-09-29T05:29:41.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:40 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:29:39.806206+0000 mgr.y (mgr.34107) 2 : cluster [DBG] pgmap v3: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:29:41.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:40 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:29:39.895095+0000 mgr.y (mgr.34107) 3 : cephadm [INF] [29/Sep/2022:05:29:39] ENGINE Serving on https://:::8765 2022-09-29T05:29:41.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:40 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:29:40.007614+0000 mgr.y (mgr.34107) 4 : cephadm [INF] [29/Sep/2022:05:29:40] ENGINE Serving on https://172.21.15.83:7150 2022-09-29T05:29:41.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:40 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:29:40.007714+0000 mgr.y (mgr.34107) 5 : cephadm [INF] [29/Sep/2022:05:29:40] ENGINE Bus STARTED 2022-09-29T05:29:41.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:29:39.779652+0000 mgr.y (mgr.34107) 1 : cephadm [INF] [29/Sep/2022:05:29:39] ENGINE Bus STARTING 2022-09-29T05:29:41.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:29:39.800717+0000 mon.a (mon.0) 21 : cluster [DBG] mgrmap e34: y(active, since 1.07338s), standbys: x 2022-09-29T05:29:41.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:29:39.806206+0000 mgr.y (mgr.34107) 2 : cluster [DBG] pgmap v3: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:29:41.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:29:39.895095+0000 mgr.y (mgr.34107) 3 : cephadm [INF] [29/Sep/2022:05:29:39] ENGINE Serving on https://:::8765 2022-09-29T05:29:41.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:29:40.007614+0000 mgr.y (mgr.34107) 4 : cephadm [INF] [29/Sep/2022:05:29:40] ENGINE Serving on https://172.21.15.83:7150 2022-09-29T05:29:41.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:29:40.007714+0000 mgr.y (mgr.34107) 5 : cephadm [INF] [29/Sep/2022:05:29:40] ENGINE Bus STARTED 2022-09-29T05:29:41.805 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:29:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:29:41] "GET /metrics HTTP/1.1" 200 26022 "" "Prometheus/2.33.4" 2022-09-29T05:29:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:41 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:29:40.789654+0000 mgr.y (mgr.34107) 6 : cluster [DBG] pgmap v4: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:29:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:41 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:29:40.796892+0000 mon.a (mon.0) 22 : cluster [INF] Health check cleared: TELEMETRY_CHANGED (was: Telemetry requires re-opt-in) 2022-09-29T05:29:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:41 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:29:40.796920+0000 mon.a (mon.0) 23 : cluster [INF] Cluster is now healthy 2022-09-29T05:29:42.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:41 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:29:40.805124+0000 mon.a (mon.0) 24 : cluster [DBG] mgrmap e35: y(active, since 2s), standbys: x 2022-09-29T05:29:42.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:29:40.789654+0000 mgr.y (mgr.34107) 6 : cluster [DBG] pgmap v4: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:29:42.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:29:40.796892+0000 mon.a (mon.0) 22 : cluster [INF] Health check cleared: TELEMETRY_CHANGED (was: Telemetry requires re-opt-in) 2022-09-29T05:29:42.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:29:40.796920+0000 mon.a (mon.0) 23 : cluster [INF] Cluster is now healthy 2022-09-29T05:29:42.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:29:40.805124+0000 mon.a (mon.0) 24 : cluster [DBG] mgrmap e35: y(active, since 2s), standbys: x 2022-09-29T05:29:42.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:29:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:29:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:29:43.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:29:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:29:44.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:29:42.790195+0000 mgr.y (mgr.34107) 7 : cluster [DBG] pgmap v5: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:29:44.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:43.606897+0000 mon.a (mon.0) 25 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:29:44.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:43.614717+0000 mon.a (mon.0) 26 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:29:44.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:44.271387+0000 mon.a (mon.0) 27 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:29:44.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:44.278507+0000 mon.a (mon.0) 28 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:29:44.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:44.579506+0000 mon.a (mon.0) 29 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:29:44.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:44.585442+0000 mon.a (mon.0) 30 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:29:45.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:44 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:29:42.790195+0000 mgr.y (mgr.34107) 7 : cluster [DBG] pgmap v5: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:29:45.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:44 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:43.606897+0000 mon.a (mon.0) 25 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:29:45.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:44 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:43.614717+0000 mon.a (mon.0) 26 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:29:45.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:44 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:44.271387+0000 mon.a (mon.0) 27 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:29:45.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:44 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:44.278507+0000 mon.a (mon.0) 28 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:29:45.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:44 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:44.579506+0000 mon.a (mon.0) 29 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:29:45.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:44 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:44.585442+0000 mon.a (mon.0) 30 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:29:45.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:29:44.586866+0000 mgr.y (mgr.34107) 8 : cephadm [INF] Updating smithi083:/etc/ceph/ceph.conf 2022-09-29T05:29:45.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:45.245180+0000 mon.a (mon.0) 31 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:29:45.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:45.252019+0000 mon.a (mon.0) 32 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:29:45.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:45.471175+0000 mon.a (mon.0) 33 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:29:45.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:45.477381+0000 mon.a (mon.0) 34 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:29:46.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:45 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:29:44.586866+0000 mgr.y (mgr.34107) 8 : cephadm [INF] Updating smithi083:/etc/ceph/ceph.conf 2022-09-29T05:29:46.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:45 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:45.245180+0000 mon.a (mon.0) 31 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:29:46.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:45 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:45.252019+0000 mon.a (mon.0) 32 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:29:46.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:45 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:45.471175+0000 mon.a (mon.0) 33 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:29:46.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:45 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:45.477381+0000 mon.a (mon.0) 34 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:29:46.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:46 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:29:44.790538+0000 mgr.y (mgr.34107) 9 : cluster [DBG] pgmap v6: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:29:46.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:46 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:29:44.836904+0000 mgr.y (mgr.34107) 10 : cephadm [INF] Updating smithi083:/var/lib/ceph/69111522-3fb6-11ed-8431-001a4aab830c/config/ceph.conf 2022-09-29T05:29:46.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:46 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:29:45.048817+0000 mgr.y (mgr.34107) 11 : cephadm [INF] Updating smithi083:/etc/ceph/ceph.client.admin.keyring 2022-09-29T05:29:46.678 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:46 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:29:45.258871+0000 mgr.y (mgr.34107) 12 : cephadm [INF] Updating smithi083:/var/lib/ceph/69111522-3fb6-11ed-8431-001a4aab830c/config/ceph.client.admin.keyring 2022-09-29T05:29:46.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:29:44.790538+0000 mgr.y (mgr.34107) 9 : cluster [DBG] pgmap v6: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:29:46.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:29:44.836904+0000 mgr.y (mgr.34107) 10 : cephadm [INF] Updating smithi083:/var/lib/ceph/69111522-3fb6-11ed-8431-001a4aab830c/config/ceph.conf 2022-09-29T05:29:46.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:29:45.048817+0000 mgr.y (mgr.34107) 11 : cephadm [INF] Updating smithi083:/etc/ceph/ceph.client.admin.keyring 2022-09-29T05:29:46.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:29:45.258871+0000 mgr.y (mgr.34107) 12 : cephadm [INF] Updating smithi083:/var/lib/ceph/69111522-3fb6-11ed-8431-001a4aab830c/config/ceph.client.admin.keyring 2022-09-29T05:29:48.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:48 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:29:46.791388+0000 mgr.y (mgr.34107) 13 : cluster [DBG] pgmap v7: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:29:48.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:48 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:48.161865+0000 mon.a (mon.0) 35 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:29:48.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:48 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:48.169549+0000 mon.a (mon.0) 36 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:29:49.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:48 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:29:46.791388+0000 mgr.y (mgr.34107) 13 : cluster [DBG] pgmap v7: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:29:49.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:48 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:48.161865+0000 mon.a (mon.0) 35 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:29:49.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:48 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:48.169549+0000 mon.a (mon.0) 36 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:29:49.060 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:29:48 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:29:48.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi153.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi083:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:29:49.899 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:29:48.155590+0000 mgr.y (mgr.34107) 14 : cephadm [INF] Detected new or changed devices on smithi153 2022-09-29T05:29:49.899 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:29:48.171163+0000 mgr.y (mgr.34107) 15 : cephadm [INF] Updating smithi153:/etc/ceph/ceph.conf 2022-09-29T05:29:49.899 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:29:48.388766+0000 mgr.y (mgr.34107) 16 : cephadm [INF] Updating smithi153:/var/lib/ceph/69111522-3fb6-11ed-8431-001a4aab830c/config/ceph.conf 2022-09-29T05:29:49.900 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:29:48.602767+0000 mgr.y (mgr.34107) 17 : cephadm [INF] Updating smithi153:/etc/ceph/ceph.client.admin.keyring 2022-09-29T05:29:49.900 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:49.032110+0000 mon.a (mon.0) 37 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:29:49.900 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:49.040882+0000 mon.a (mon.0) 38 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:29:49.900 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:49.045636+0000 mon.a (mon.0) 39 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:29:49.901 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:49.119737+0000 mon.c (mon.1) 113 : audit [DBG] from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:29:49.901 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:49.122563+0000 mon.c (mon.1) 114 : audit [DBG] from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:29:49.901 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:49.123792+0000 mon.c (mon.1) 115 : audit [DBG] from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2022-09-29T05:29:49.901 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:49.124464+0000 mon.c (mon.1) 116 : audit [DBG] from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2022-09-29T05:29:50.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:49 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:29:48.155590+0000 mgr.y (mgr.34107) 14 : cephadm [INF] Detected new or changed devices on smithi153 2022-09-29T05:29:50.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:49 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:29:48.171163+0000 mgr.y (mgr.34107) 15 : cephadm [INF] Updating smithi153:/etc/ceph/ceph.conf 2022-09-29T05:29:50.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:49 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:29:48.388766+0000 mgr.y (mgr.34107) 16 : cephadm [INF] Updating smithi153:/var/lib/ceph/69111522-3fb6-11ed-8431-001a4aab830c/config/ceph.conf 2022-09-29T05:29:50.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:49 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:29:48.602767+0000 mgr.y (mgr.34107) 17 : cephadm [INF] Updating smithi153:/etc/ceph/ceph.client.admin.keyring 2022-09-29T05:29:50.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:49 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:49.032110+0000 mon.a (mon.0) 37 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:29:50.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:49 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:49.040882+0000 mon.a (mon.0) 38 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:29:50.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:49 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:49.045636+0000 mon.a (mon.0) 39 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:29:50.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:49 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:49.119737+0000 mon.c (mon.1) 113 : audit [DBG] from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:29:50.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:49 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:49.122563+0000 mon.c (mon.1) 114 : audit [DBG] from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:29:50.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:49 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:49.123792+0000 mon.c (mon.1) 115 : audit [DBG] from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2022-09-29T05:29:50.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:49 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:49.124464+0000 mon.c (mon.1) 116 : audit [DBG] from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2022-09-29T05:29:50.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:29:48.791774+0000 mgr.y (mgr.34107) 18 : cluster [DBG] pgmap v8: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:29:50.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:29:48.815271+0000 mgr.y (mgr.34107) 19 : cephadm [INF] Updating smithi153:/var/lib/ceph/69111522-3fb6-11ed-8431-001a4aab830c/config/ceph.client.admin.keyring 2022-09-29T05:29:50.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:29:49.124958+0000 mgr.y (mgr.34107) 20 : cephadm [INF] Upgrade: It appears safe to stop mon.c 2022-09-29T05:29:50.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:29:49.127397+0000 mgr.y (mgr.34107) 21 : cluster [DBG] pgmap v9: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:29:50.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cluster 2022-09-29T05:29:49.626684+0000 mon.a (mon.0) 40 : cluster [WRN] Health check failed: Telemetry requires re-opt-in (TELEMETRY_CHANGED) 2022-09-29T05:29:50.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:50.606446+0000 mon.a (mon.0) 41 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:29:50.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:50.607780+0000 mon.c (mon.1) 117 : audit [INF] from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2022-09-29T05:29:50.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:50.608583+0000 mon.a (mon.0) 42 : audit [DBG] from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2022-09-29T05:29:50.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: audit 2022-09-29T05:29:50.609273+0000 mon.c (mon.1) 118 : audit [DBG] from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:29:51.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:50 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:29:48.791774+0000 mgr.y (mgr.34107) 18 : cluster [DBG] pgmap v8: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:29:51.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:50 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:29:48.815271+0000 mgr.y (mgr.34107) 19 : cephadm [INF] Updating smithi153:/var/lib/ceph/69111522-3fb6-11ed-8431-001a4aab830c/config/ceph.client.admin.keyring 2022-09-29T05:29:51.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:50 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:29:49.124958+0000 mgr.y (mgr.34107) 20 : cephadm [INF] Upgrade: It appears safe to stop mon.c 2022-09-29T05:29:51.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:50 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:29:49.127397+0000 mgr.y (mgr.34107) 21 : cluster [DBG] pgmap v9: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:29:51.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:50 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:29:49.626684+0000 mon.a (mon.0) 40 : cluster [WRN] Health check failed: Telemetry requires re-opt-in (TELEMETRY_CHANGED) 2022-09-29T05:29:51.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:50 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:50.606446+0000 mon.a (mon.0) 41 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:29:51.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:50 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:50.607780+0000 mon.c (mon.1) 117 : audit [INF] from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2022-09-29T05:29:51.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:50 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:50.608583+0000 mon.a (mon.0) 42 : audit [DBG] from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2022-09-29T05:29:51.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:50 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:50.609273+0000 mon.c (mon.1) 118 : audit [DBG] from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:29:51.201 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T05:29:51.506 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T05:29:51.507 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3m) 7s ago 4m 26.3M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T05:29:51.507 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3m) 7s ago 3m 60.1M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T05:29:51.508 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4m) 7s ago 4m 15.1M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T05:29:51.508 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4m) 7s ago 4m 17.8M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T05:29:51.509 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (36s) 7s ago 7m 426M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T05:29:51.509 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (63s) 7s ago 8m 457M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T05:29:51.510 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (22s) 7s ago 9m 33.7M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T05:29:51.510 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7m) 7s ago 7m 85.0M 2048M 16.2.10 c56e13ff90f2 b96ee39cc34f 2022-09-29T05:29:51.511 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7m) 7s ago 7m 83.2M 2048M 16.2.10 c56e13ff90f2 a7179c86361b 2022-09-29T05:29:51.511 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4m) 7s ago 4m 24.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T05:29:51.512 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4m) 7s ago 4m 29.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T05:29:51.512 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7m) 7s ago 7m 66.4M 2500M 16.2.10 c56e13ff90f2 c12a21fd5897 2022-09-29T05:29:51.513 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6m) 7s ago 6m 61.1M 2500M 16.2.10 c56e13ff90f2 989e877de30a 2022-09-29T05:29:51.513 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6m) 7s ago 6m 54.5M 2500M 16.2.10 c56e13ff90f2 81b9b51f0eee 2022-09-29T05:29:51.514 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6m) 7s ago 6m 61.7M 2500M 16.2.10 c56e13ff90f2 4863a2083356 2022-09-29T05:29:51.514 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6m) 7s ago 6m 57.1M 2500M 16.2.10 c56e13ff90f2 ed61dc9353b0 2022-09-29T05:29:51.515 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5m) 7s ago 5m 64.7M 2500M 16.2.10 c56e13ff90f2 59f19fd15fed 2022-09-29T05:29:51.515 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5m) 7s ago 5m 66.3M 2500M 16.2.10 c56e13ff90f2 23decfd9c6b2 2022-09-29T05:29:51.516 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5m) 7s ago 5m 70.2M 2500M 16.2.10 c56e13ff90f2 23460a09c0a7 2022-09-29T05:29:51.516 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (40s) 7s ago 4m 62.1M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T05:29:51.869 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:29:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:29:51] "GET /metrics HTTP/1.1" 200 26022 "" "Prometheus/2.33.4" 2022-09-29T05:29:51.869 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:29:50.600710+0000 mgr.y (mgr.34107) 22 : cephadm [INF] Upgrade: Updating mon.c 2022-09-29T05:29:51.869 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: cephadm 2022-09-29T05:29:50.609993+0000 mgr.y (mgr.34107) 23 : cephadm [INF] Deploying daemon mon.c on smithi083 2022-09-29T05:29:51.878 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T05:29:51.879 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T05:29:51.879 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T05:29:51.879 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 1 2022-09-29T05:29:51.879 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:29:51.879 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T05:29:51.880 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T05:29:51.880 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:29:51.880 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T05:29:51.881 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 8 2022-09-29T05:29:51.881 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:29:51.881 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T05:29:51.882 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T05:29:51.882 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:29:51.883 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T05:29:51.883 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 12, 2022-09-29T05:29:51.884 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T05:29:51.884 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T05:29:51.884 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T05:29:52.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:51 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:29:50.600710+0000 mgr.y (mgr.34107) 22 : cephadm [INF] Upgrade: Updating mon.c 2022-09-29T05:29:52.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:51 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:29:50.609993+0000 mgr.y (mgr.34107) 23 : cephadm [INF] Deploying daemon mon.c on smithi083 2022-09-29T05:29:52.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:51 smithi083 systemd[1]: Stopping Ceph mon.c for 69111522-3fb6-11ed-8431-001a4aab830c... 2022-09-29T05:29:52.606 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:29:52.276+0000 7f65dca70700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2022-09-29T05:29:52.606 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c[44060]: debug 2022-09-29T05:29:52.276+0000 7f65dca70700 -1 mon.c@1(peon) e3 *** Got Signal Terminated *** 2022-09-29T05:29:52.607 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:29:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: [29/Sep/2022:05:29:52] ENGINE Bus STOPPING 2022-09-29T05:29:52.607 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:29:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: [29/Sep/2022:05:29:52] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2022-09-29T05:29:52.607 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:29:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: [29/Sep/2022:05:29:52] ENGINE Bus STOPPED 2022-09-29T05:29:52.607 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:29:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: [29/Sep/2022:05:29:52] ENGINE Bus STARTING 2022-09-29T05:29:52.608 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:29:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: [29/Sep/2022:05:29:52] ENGINE Serving on http://:::9283 2022-09-29T05:29:52.608 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:29:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: [29/Sep/2022:05:29:52] ENGINE Bus STARTED 2022-09-29T05:29:52.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:29:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:29:52.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:29:52.866 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:52 smithi083 bash[79751]: ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-c 2022-09-29T05:29:53.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:52 smithi083 systemd[1]: ceph-69111522-3fb6-11ed-8431-001a4aab830c@mon.c.service: Succeeded. 2022-09-29T05:29:53.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:52 smithi083 systemd[1]: Stopped Ceph mon.c for 69111522-3fb6-11ed-8431-001a4aab830c. 2022-09-29T05:29:53.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:52 smithi083 systemd[1]: Starting Ceph mon.c for 69111522-3fb6-11ed-8431-001a4aab830c... 2022-09-29T05:29:53.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:29:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:29:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:29:53.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 podman[79891]: 2022-09-29T05:29:53.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: set uid:gid to 167:167 (ceph:ceph) 2022-09-29T05:29:53.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev), process ceph-mon, pid 7 2022-09-29T05:29:53.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: pidfile_write: ignore empty --pid-file 2022-09-29T05:29:53.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: load: jerasure load: lrc 2022-09-29T05:29:53.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 bash[79891]: 54044e5b4f35da1e44f4aa92973dfcae02f4bc732177f6a1b369971bae6ec057 2022-09-29T05:29:53.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: RocksDB version: 6.15.5 2022-09-29T05:29:53.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2022-09-29T05:29:53.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Compile date Sep 28 2022 2022-09-29T05:29:53.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: DB SUMMARY 2022-09-29T05:29:53.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: DB Session ID: 6P05JZY4QRE27DHNKRWA 2022-09-29T05:29:53.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: CURRENT file: CURRENT 2022-09-29T05:29:53.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: IDENTITY file: IDENTITY 2022-09-29T05:29:53.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: MANIFEST file: MANIFEST-000005 size: 503 Bytes 2022-09-29T05:29:53.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 1, files: 000014.sst 2022-09-29T05:29:53.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000012.log size: 4680988 ; 2022-09-29T05:29:53.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.error_if_exists: 0 2022-09-29T05:29:53.968 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.create_if_missing: 0 2022-09-29T05:29:53.968 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.paranoid_checks: 1 2022-09-29T05:29:53.968 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2022-09-29T05:29:53.968 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.env: 0x556aa9b18280 2022-09-29T05:29:53.968 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.fs: Posix File System 2022-09-29T05:29:53.969 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.info_log: 0x556aabb33b00 2022-09-29T05:29:53.969 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.max_file_opening_threads: 16 2022-09-29T05:29:53.969 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.statistics: (nil) 2022-09-29T05:29:53.969 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.use_fsync: 0 2022-09-29T05:29:53.969 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.max_log_file_size: 0 2022-09-29T05:29:53.970 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.max_manifest_file_size: 1073741824 2022-09-29T05:29:53.970 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.log_file_time_to_roll: 0 2022-09-29T05:29:53.970 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.keep_log_file_num: 1000 2022-09-29T05:29:53.970 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.recycle_log_file_num: 0 2022-09-29T05:29:53.971 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.allow_fallocate: 1 2022-09-29T05:29:53.971 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.allow_mmap_reads: 0 2022-09-29T05:29:53.971 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.allow_mmap_writes: 0 2022-09-29T05:29:53.971 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.use_direct_reads: 0 2022-09-29T05:29:53.971 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2022-09-29T05:29:53.972 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.create_missing_column_families: 0 2022-09-29T05:29:53.972 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.db_log_dir: 2022-09-29T05:29:53.972 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-c/store.db 2022-09-29T05:29:53.972 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.table_cache_numshardbits: 6 2022-09-29T05:29:53.974 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.WAL_ttl_seconds: 0 2022-09-29T05:29:53.974 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.WAL_size_limit_MB: 0 2022-09-29T05:29:53.974 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2022-09-29T05:29:53.975 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.manifest_preallocation_size: 4194304 2022-09-29T05:29:53.975 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.is_fd_close_on_exec: 1 2022-09-29T05:29:53.975 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.advise_random_on_open: 1 2022-09-29T05:29:53.975 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.db_write_buffer_size: 0 2022-09-29T05:29:53.976 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.write_buffer_manager: 0x556aabc1eba0 2022-09-29T05:29:53.976 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.access_hint_on_compaction_start: 1 2022-09-29T05:29:53.976 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2022-09-29T05:29:53.976 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.random_access_max_buffer_size: 1048576 2022-09-29T05:29:53.976 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.use_adaptive_mutex: 0 2022-09-29T05:29:53.977 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.rate_limiter: (nil) 2022-09-29T05:29:53.977 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2022-09-29T05:29:53.977 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.wal_recovery_mode: 2 2022-09-29T05:29:53.977 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.enable_thread_tracking: 0 2022-09-29T05:29:53.978 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.enable_pipelined_write: 0 2022-09-29T05:29:53.978 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.unordered_write: 0 2022-09-29T05:29:53.978 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.allow_concurrent_memtable_write: 1 2022-09-29T05:29:53.978 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2022-09-29T05:29:53.978 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.write_thread_max_yield_usec: 100 2022-09-29T05:29:53.979 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.write_thread_slow_yield_usec: 3 2022-09-29T05:29:53.979 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.row_cache: None 2022-09-29T05:29:53.979 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.wal_filter: None 2022-09-29T05:29:53.979 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.avoid_flush_during_recovery: 0 2022-09-29T05:29:53.979 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.allow_ingest_behind: 0 2022-09-29T05:29:53.980 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.preserve_deletes: 0 2022-09-29T05:29:53.980 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.two_write_queues: 0 2022-09-29T05:29:53.980 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.manual_wal_flush: 0 2022-09-29T05:29:53.980 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.atomic_flush: 0 2022-09-29T05:29:53.980 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2022-09-29T05:29:53.981 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.persist_stats_to_disk: 0 2022-09-29T05:29:53.981 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.write_dbid_to_manifest: 0 2022-09-29T05:29:53.981 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.log_readahead_size: 0 2022-09-29T05:29:53.981 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.file_checksum_gen_factory: Unknown 2022-09-29T05:29:53.981 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.best_efforts_recovery: 0 2022-09-29T05:29:53.982 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2022-09-29T05:29:53.982 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2022-09-29T05:29:53.982 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.allow_data_in_errors: 0 2022-09-29T05:29:53.982 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.db_host_id: __hostname__ 2022-09-29T05:29:53.983 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.max_background_jobs: 2 2022-09-29T05:29:53.983 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.max_background_compactions: -1 2022-09-29T05:29:53.983 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.max_subcompactions: 1 2022-09-29T05:29:53.983 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.avoid_flush_during_shutdown: 0 2022-09-29T05:29:53.983 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2022-09-29T05:29:53.984 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.delayed_write_rate : 16777216 2022-09-29T05:29:53.984 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.max_total_wal_size: 0 2022-09-29T05:29:53.984 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2022-09-29T05:29:53.984 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.stats_dump_period_sec: 600 2022-09-29T05:29:53.985 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.stats_persist_period_sec: 600 2022-09-29T05:29:53.985 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.stats_history_buffer_size: 1048576 2022-09-29T05:29:53.985 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.max_open_files: -1 2022-09-29T05:29:53.985 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.bytes_per_sync: 0 2022-09-29T05:29:53.985 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.wal_bytes_per_sync: 0 2022-09-29T05:29:53.986 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.strict_bytes_per_sync: 0 2022-09-29T05:29:53.986 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.compaction_readahead_size: 0 2022-09-29T05:29:53.986 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.max_background_flushes: -1 2022-09-29T05:29:53.986 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Compression algorithms supported: 2022-09-29T05:29:53.987 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: kZSTDNotFinalCompression supported: 0 2022-09-29T05:29:53.987 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: kZSTD supported: 0 2022-09-29T05:29:53.987 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: kXpressCompression supported: 0 2022-09-29T05:29:53.987 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: kLZ4HCCompression supported: 1 2022-09-29T05:29:53.987 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: kZlibCompression supported: 1 2022-09-29T05:29:53.988 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: kSnappyCompression supported: 1 2022-09-29T05:29:53.988 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: kLZ4Compression supported: 1 2022-09-29T05:29:53.988 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: kBZip2Compression supported: 0 2022-09-29T05:29:53.988 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Fast CRC32 supported: Supported on x86 2022-09-29T05:29:53.989 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000005 2022-09-29T05:29:53.989 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2022-09-29T05:29:53.989 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2022-09-29T05:29:53.989 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.merge_operator: 2022-09-29T05:29:53.989 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.compaction_filter: None 2022-09-29T05:29:53.990 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.compaction_filter_factory: None 2022-09-29T05:29:53.990 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.sst_partitioner_factory: None 2022-09-29T05:29:53.990 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.memtable_factory: SkipListFactory 2022-09-29T05:29:53.990 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.table_factory: BlockBasedTable 2022-09-29T05:29:53.991 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x556aabb30e40) 2022-09-29T05:29:53.991 INFO:journalctl@ceph.mon.c.smithi083.stdout: cache_index_and_filter_blocks: 1 2022-09-29T05:29:53.991 INFO:journalctl@ceph.mon.c.smithi083.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2022-09-29T05:29:53.991 INFO:journalctl@ceph.mon.c.smithi083.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2022-09-29T05:29:53.992 INFO:journalctl@ceph.mon.c.smithi083.stdout: pin_top_level_index_and_filter: 1 2022-09-29T05:29:53.992 INFO:journalctl@ceph.mon.c.smithi083.stdout: index_type: 0 2022-09-29T05:29:53.992 INFO:journalctl@ceph.mon.c.smithi083.stdout: data_block_index_type: 0 2022-09-29T05:29:53.992 INFO:journalctl@ceph.mon.c.smithi083.stdout: index_shortening: 1 2022-09-29T05:29:53.993 INFO:journalctl@ceph.mon.c.smithi083.stdout: data_block_hash_table_util_ratio: 0.750000 2022-09-29T05:29:53.993 INFO:journalctl@ceph.mon.c.smithi083.stdout: hash_index_allow_collision: 1 2022-09-29T05:29:53.993 INFO:journalctl@ceph.mon.c.smithi083.stdout: checksum: 1 2022-09-29T05:29:53.993 INFO:journalctl@ceph.mon.c.smithi083.stdout: no_block_cache: 0 2022-09-29T05:29:53.993 INFO:journalctl@ceph.mon.c.smithi083.stdout: block_cache: 0x556aabb9c9b0 2022-09-29T05:29:53.994 INFO:journalctl@ceph.mon.c.smithi083.stdout: block_cache_name: BinnedLRUCache 2022-09-29T05:29:53.994 INFO:journalctl@ceph.mon.c.smithi083.stdout: block_cache_options: 2022-09-29T05:29:53.994 INFO:journalctl@ceph.mon.c.smithi083.stdout: capacity : 536870912 2022-09-29T05:29:53.994 INFO:journalctl@ceph.mon.c.smithi083.stdout: num_shard_bits : 4 2022-09-29T05:29:53.995 INFO:journalctl@ceph.mon.c.smithi083.stdout: strict_capacity_limit : 0 2022-09-29T05:29:53.995 INFO:journalctl@ceph.mon.c.smithi083.stdout: high_pri_pool_ratio: 0.000 2022-09-29T05:29:53.995 INFO:journalctl@ceph.mon.c.smithi083.stdout: block_cache_compressed: (nil) 2022-09-29T05:29:53.995 INFO:journalctl@ceph.mon.c.smithi083.stdout: persistent_cache: (nil) 2022-09-29T05:29:53.995 INFO:journalctl@ceph.mon.c.smithi083.stdout: block_size: 4096 2022-09-29T05:29:53.996 INFO:journalctl@ceph.mon.c.smithi083.stdout: block_size_deviation: 10 2022-09-29T05:29:53.996 INFO:journalctl@ceph.mon.c.smithi083.stdout: block_restart_interval: 16 2022-09-29T05:29:53.996 INFO:journalctl@ceph.mon.c.smithi083.stdout: index_block_restart_interval: 1 2022-09-29T05:29:53.996 INFO:journalctl@ceph.mon.c.smithi083.stdout: metadata_block_size: 4096 2022-09-29T05:29:53.997 INFO:journalctl@ceph.mon.c.smithi083.stdout: partition_filters: 0 2022-09-29T05:29:53.997 INFO:journalctl@ceph.mon.c.smithi083.stdout: use_delta_encoding: 1 2022-09-29T05:29:53.997 INFO:journalctl@ceph.mon.c.smithi083.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2022-09-29T05:29:53.997 INFO:journalctl@ceph.mon.c.smithi083.stdout: whole_key_filtering: 1 2022-09-29T05:29:53.997 INFO:journalctl@ceph.mon.c.smithi083.stdout: verify_compression: 0 2022-09-29T05:29:53.998 INFO:journalctl@ceph.mon.c.smithi083.stdout: read_amp_bytes_per_bit: 0 2022-09-29T05:29:53.998 INFO:journalctl@ceph.mon.c.smithi083.stdout: format_version: 4 2022-09-29T05:29:53.998 INFO:journalctl@ceph.mon.c.smithi083.stdout: enable_index_compression: 1 2022-09-29T05:29:53.998 INFO:journalctl@ceph.mon.c.smithi083.stdout: block_align: 0 2022-09-29T05:29:53.999 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.write_buffer_size: 33554432 2022-09-29T05:29:53.999 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.max_write_buffer_number: 2 2022-09-29T05:29:53.999 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.compression: NoCompression 2022-09-29T05:29:53.999 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.bottommost_compression: Disabled 2022-09-29T05:29:54.000 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.prefix_extractor: nullptr 2022-09-29T05:29:54.000 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2022-09-29T05:29:54.000 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.num_levels: 7 2022-09-29T05:29:54.000 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2022-09-29T05:29:54.001 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2022-09-29T05:29:54.001 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2022-09-29T05:29:54.001 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2022-09-29T05:29:54.001 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.bottommost_compression_opts.level: 32767 2022-09-29T05:29:54.002 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2022-09-29T05:29:54.002 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2022-09-29T05:29:54.002 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2022-09-29T05:29:54.002 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2022-09-29T05:29:54.002 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.bottommost_compression_opts.enabled: false 2022-09-29T05:29:54.003 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.compression_opts.window_bits: -14 2022-09-29T05:29:54.003 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.compression_opts.level: 32767 2022-09-29T05:29:54.003 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.compression_opts.strategy: 0 2022-09-29T05:29:54.003 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2022-09-29T05:29:54.003 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2022-09-29T05:29:54.004 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.compression_opts.parallel_threads: 1 2022-09-29T05:29:54.004 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.compression_opts.enabled: false 2022-09-29T05:29:54.004 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2022-09-29T05:29:54.004 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2022-09-29T05:29:54.005 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.level0_stop_writes_trigger: 36 2022-09-29T05:29:54.005 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.target_file_size_base: 67108864 2022-09-29T05:29:54.005 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.target_file_size_multiplier: 1 2022-09-29T05:29:54.005 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.max_bytes_for_level_base: 268435456 2022-09-29T05:29:54.005 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2022-09-29T05:29:54.006 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2022-09-29T05:29:54.006 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2022-09-29T05:29:54.006 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2022-09-29T05:29:54.006 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2022-09-29T05:29:54.007 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2022-09-29T05:29:54.007 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2022-09-29T05:29:54.007 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2022-09-29T05:29:54.007 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2022-09-29T05:29:54.007 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2022-09-29T05:29:54.008 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.max_compaction_bytes: 1677721600 2022-09-29T05:29:54.008 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.arena_block_size: 4194304 2022-09-29T05:29:54.008 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2022-09-29T05:29:54.008 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2022-09-29T05:29:54.008 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2022-09-29T05:29:54.009 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.disable_auto_compactions: 0 2022-09-29T05:29:54.009 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2022-09-29T05:29:54.009 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2022-09-29T05:29:54.009 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2022-09-29T05:29:54.010 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2022-09-29T05:29:54.010 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2022-09-29T05:29:54.010 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2022-09-29T05:29:54.010 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2022-09-29T05:29:54.010 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2022-09-29T05:29:54.011 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2022-09-29T05:29:54.011 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2022-09-29T05:29:54.011 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.table_properties_collectors: 2022-09-29T05:29:54.011 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.inplace_update_support: 0 2022-09-29T05:29:54.012 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.inplace_update_num_locks: 10000 2022-09-29T05:29:54.012 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2022-09-29T05:29:54.012 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.memtable_whole_key_filtering: 0 2022-09-29T05:29:54.012 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.memtable_huge_page_size: 0 2022-09-29T05:29:54.013 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.bloom_locality: 0 2022-09-29T05:29:54.013 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.max_successive_merges: 0 2022-09-29T05:29:54.013 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.optimize_filters_for_hits: 0 2022-09-29T05:29:54.013 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.paranoid_file_checks: 0 2022-09-29T05:29:54.013 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.force_consistency_checks: 1 2022-09-29T05:29:54.014 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.report_bg_io_stats: 0 2022-09-29T05:29:54.014 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.ttl: 2592000 2022-09-29T05:29:54.014 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.periodic_compaction_seconds: 0 2022-09-29T05:29:54.014 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.enable_blob_files: false 2022-09-29T05:29:54.014 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.min_blob_size: 0 2022-09-29T05:29:54.015 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.blob_file_size: 268435456 2022-09-29T05:29:54.015 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.blob_compression_type: NoCompression 2022-09-29T05:29:54.015 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.enable_blob_garbage_collection: false 2022-09-29T05:29:54.015 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2022-09-29T05:29:54.016 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: [db/version_set.cc:3457] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2022-09-29T05:29:54.016 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 16, last_sequence is 9297, log_number is 12,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2022-09-29T05:29:54.016 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 12 2022-09-29T05:29:54.016 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: [db/version_set.cc:4082] Creating manifest 16 2022-09-29T05:29:54.017 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: [db/version_set.cc:3457] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2022-09-29T05:29:54.017 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: EVENT_LOG_v1 {"time_micros": 1664429393844563, "job": 1, "event": "recovery_started", "wal_files": [12]} 2022-09-29T05:29:54.017 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #12 mode 2 2022-09-29T05:29:54.017 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2022-09-29T05:29:54.017 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 systemd[1]: Started Ceph mon.c for 69111522-3fb6-11ed-8431-001a4aab830c. 2022-09-29T05:29:54.018 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: EVENT_LOG_v1 {"time_micros": 1664429393854357, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 17, "file_size": 3002457, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 2998176, "index_size": 2765, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 4788, "raw_average_key_size": 21, "raw_value_size": 2991809, "raw_average_value_size": 13537, "num_data_blocks": 125, "num_entries": 221, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1664429393, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "55b0e340-601c-4fbb-b83d-72354d36bd4c", "db_session_id": "6P05JZY4QRE27DHNKRWA"}} 2022-09-29T05:29:54.018 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: [db/version_set.cc:4082] Creating manifest 18 2022-09-29T05:29:54.018 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: [db/version_set.cc:3457] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2022-09-29T05:29:54.018 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: EVENT_LOG_v1 {"time_micros": 1664429393854951, "job": 1, "event": "recovery_finished"} 2022-09-29T05:29:54.019 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000012.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2022-09-29T05:29:54.019 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x556aabb83880 2022-09-29T05:29:54.019 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: DB pointer 0x556aabbf6000 2022-09-29T05:29:54.019 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2022-09-29T05:29:54.020 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: rocksdb: [db/db_impl/db_impl.cc:903] 2022-09-29T05:29:54.020 INFO:journalctl@ceph.mon.c.smithi083.stdout: ** DB Stats ** 2022-09-29T05:29:54.020 INFO:journalctl@ceph.mon.c.smithi083.stdout: Uptime(secs): 0.0 total, 0.0 interval 2022-09-29T05:29:54.020 INFO:journalctl@ceph.mon.c.smithi083.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2022-09-29T05:29:54.020 INFO:journalctl@ceph.mon.c.smithi083.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2022-09-29T05:29:54.021 INFO:journalctl@ceph.mon.c.smithi083.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2022-09-29T05:29:54.021 INFO:journalctl@ceph.mon.c.smithi083.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2022-09-29T05:29:54.021 INFO:journalctl@ceph.mon.c.smithi083.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2022-09-29T05:29:54.021 INFO:journalctl@ceph.mon.c.smithi083.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2022-09-29T05:29:54.022 INFO:journalctl@ceph.mon.c.smithi083.stdout: 2022-09-29T05:29:54.022 INFO:journalctl@ceph.mon.c.smithi083.stdout: ** Compaction Stats [default] ** 2022-09-29T05:29:54.022 INFO:journalctl@ceph.mon.c.smithi083.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2022-09-29T05:29:54.022 INFO:journalctl@ceph.mon.c.smithi083.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2022-09-29T05:29:54.023 INFO:journalctl@ceph.mon.c.smithi083.stdout: L0 1/0 2.86 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 491.5 0.01 0.00 1 0.006 0 0 2022-09-29T05:29:54.023 INFO:journalctl@ceph.mon.c.smithi083.stdout: L6 1/0 20.41 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2022-09-29T05:29:54.023 INFO:journalctl@ceph.mon.c.smithi083.stdout: Sum 2/0 23.27 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 491.5 0.01 0.00 1 0.006 0 0 2022-09-29T05:29:54.023 INFO:journalctl@ceph.mon.c.smithi083.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 491.5 0.01 0.00 1 0.006 0 0 2022-09-29T05:29:54.023 INFO:journalctl@ceph.mon.c.smithi083.stdout: 2022-09-29T05:29:54.024 INFO:journalctl@ceph.mon.c.smithi083.stdout: ** Compaction Stats [default] ** 2022-09-29T05:29:54.024 INFO:journalctl@ceph.mon.c.smithi083.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2022-09-29T05:29:54.024 INFO:journalctl@ceph.mon.c.smithi083.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2022-09-29T05:29:54.024 INFO:journalctl@ceph.mon.c.smithi083.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 491.5 0.01 0.00 1 0.006 0 0 2022-09-29T05:29:54.025 INFO:journalctl@ceph.mon.c.smithi083.stdout: Uptime(secs): 0.0 total, 0.0 interval 2022-09-29T05:29:54.025 INFO:journalctl@ceph.mon.c.smithi083.stdout: Flush(GB): cumulative 0.003, interval 0.003 2022-09-29T05:29:54.025 INFO:journalctl@ceph.mon.c.smithi083.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2022-09-29T05:29:54.025 INFO:journalctl@ceph.mon.c.smithi083.stdout: AddFile(Total Files): cumulative 0, interval 0 2022-09-29T05:29:54.026 INFO:journalctl@ceph.mon.c.smithi083.stdout: AddFile(L0 Files): cumulative 0, interval 0 2022-09-29T05:29:54.026 INFO:journalctl@ceph.mon.c.smithi083.stdout: AddFile(Keys): cumulative 0, interval 0 2022-09-29T05:29:54.026 INFO:journalctl@ceph.mon.c.smithi083.stdout: Cumulative compaction: 0.00 GB write, 224.67 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2022-09-29T05:29:54.026 INFO:journalctl@ceph.mon.c.smithi083.stdout: Interval compaction: 0.00 GB write, 224.67 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2022-09-29T05:29:54.026 INFO:journalctl@ceph.mon.c.smithi083.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2022-09-29T05:29:54.027 INFO:journalctl@ceph.mon.c.smithi083.stdout: 2022-09-29T05:29:54.027 INFO:journalctl@ceph.mon.c.smithi083.stdout: ** File Read Latency Histogram By Level [default] ** 2022-09-29T05:29:54.027 INFO:journalctl@ceph.mon.c.smithi083.stdout: 2022-09-29T05:29:54.027 INFO:journalctl@ceph.mon.c.smithi083.stdout: ** Compaction Stats [default] ** 2022-09-29T05:29:54.028 INFO:journalctl@ceph.mon.c.smithi083.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2022-09-29T05:29:54.028 INFO:journalctl@ceph.mon.c.smithi083.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2022-09-29T05:29:54.028 INFO:journalctl@ceph.mon.c.smithi083.stdout: L0 1/0 2.86 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 491.5 0.01 0.00 1 0.006 0 0 2022-09-29T05:29:54.028 INFO:journalctl@ceph.mon.c.smithi083.stdout: L6 1/0 20.41 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2022-09-29T05:29:54.028 INFO:journalctl@ceph.mon.c.smithi083.stdout: Sum 2/0 23.27 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 491.5 0.01 0.00 1 0.006 0 0 2022-09-29T05:29:54.029 INFO:journalctl@ceph.mon.c.smithi083.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2022-09-29T05:29:54.029 INFO:journalctl@ceph.mon.c.smithi083.stdout: 2022-09-29T05:29:54.029 INFO:journalctl@ceph.mon.c.smithi083.stdout: ** Compaction Stats [default] ** 2022-09-29T05:29:54.029 INFO:journalctl@ceph.mon.c.smithi083.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2022-09-29T05:29:54.030 INFO:journalctl@ceph.mon.c.smithi083.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2022-09-29T05:29:54.030 INFO:journalctl@ceph.mon.c.smithi083.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 491.5 0.01 0.00 1 0.006 0 0 2022-09-29T05:29:54.030 INFO:journalctl@ceph.mon.c.smithi083.stdout: Uptime(secs): 0.0 total, 0.0 interval 2022-09-29T05:29:54.030 INFO:journalctl@ceph.mon.c.smithi083.stdout: Flush(GB): cumulative 0.003, interval 0.000 2022-09-29T05:29:54.031 INFO:journalctl@ceph.mon.c.smithi083.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2022-09-29T05:29:54.031 INFO:journalctl@ceph.mon.c.smithi083.stdout: AddFile(Total Files): cumulative 0, interval 0 2022-09-29T05:29:54.031 INFO:journalctl@ceph.mon.c.smithi083.stdout: AddFile(L0 Files): cumulative 0, interval 0 2022-09-29T05:29:54.031 INFO:journalctl@ceph.mon.c.smithi083.stdout: AddFile(Keys): cumulative 0, interval 0 2022-09-29T05:29:54.031 INFO:journalctl@ceph.mon.c.smithi083.stdout: Cumulative compaction: 0.00 GB write, 223.91 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2022-09-29T05:29:54.032 INFO:journalctl@ceph.mon.c.smithi083.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2022-09-29T05:29:54.032 INFO:journalctl@ceph.mon.c.smithi083.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2022-09-29T05:29:54.032 INFO:journalctl@ceph.mon.c.smithi083.stdout: 2022-09-29T05:29:54.032 INFO:journalctl@ceph.mon.c.smithi083.stdout: ** File Read Latency Histogram By Level [default] ** 2022-09-29T05:29:54.033 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: starting mon.c rank 1 at public addrs [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] at bind addrs [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 69111522-3fb6-11ed-8431-001a4aab830c 2022-09-29T05:29:54.033 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: mon.c@-1(???) e3 preinit fsid 69111522-3fb6-11ed-8431-001a4aab830c 2022-09-29T05:29:54.033 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: mon.c@-1(???).mds e6 new map 2022-09-29T05:29:54.033 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: mon.c@-1(???).mds e6 print_map 2022-09-29T05:29:54.033 INFO:journalctl@ceph.mon.c.smithi083.stdout: e6 2022-09-29T05:29:54.034 INFO:journalctl@ceph.mon.c.smithi083.stdout: enable_multiple, ever_enabled_multiple: 1,1 2022-09-29T05:29:54.034 INFO:journalctl@ceph.mon.c.smithi083.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2022-09-29T05:29:54.034 INFO:journalctl@ceph.mon.c.smithi083.stdout: legacy client fscid: 1 2022-09-29T05:29:54.034 INFO:journalctl@ceph.mon.c.smithi083.stdout: 2022-09-29T05:29:54.035 INFO:journalctl@ceph.mon.c.smithi083.stdout: Filesystem 'cephfs' (1) 2022-09-29T05:29:54.035 INFO:journalctl@ceph.mon.c.smithi083.stdout: fs_name cephfs 2022-09-29T05:29:54.035 INFO:journalctl@ceph.mon.c.smithi083.stdout: epoch 6 2022-09-29T05:29:54.035 INFO:journalctl@ceph.mon.c.smithi083.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2022-09-29T05:29:54.035 INFO:journalctl@ceph.mon.c.smithi083.stdout: created 2022-09-29T05:25:03.789615+0000 2022-09-29T05:29:54.036 INFO:journalctl@ceph.mon.c.smithi083.stdout: modified 2022-09-29T05:25:05.816761+0000 2022-09-29T05:29:54.036 INFO:journalctl@ceph.mon.c.smithi083.stdout: tableserver 0 2022-09-29T05:29:54.036 INFO:journalctl@ceph.mon.c.smithi083.stdout: root 0 2022-09-29T05:29:54.036 INFO:journalctl@ceph.mon.c.smithi083.stdout: session_timeout 60 2022-09-29T05:29:54.037 INFO:journalctl@ceph.mon.c.smithi083.stdout: session_autoclose 300 2022-09-29T05:29:54.037 INFO:journalctl@ceph.mon.c.smithi083.stdout: max_file_size 1099511627776 2022-09-29T05:29:54.037 INFO:journalctl@ceph.mon.c.smithi083.stdout: required_client_features {} 2022-09-29T05:29:54.037 INFO:journalctl@ceph.mon.c.smithi083.stdout: last_failure 0 2022-09-29T05:29:54.037 INFO:journalctl@ceph.mon.c.smithi083.stdout: last_failure_osd_epoch 0 2022-09-29T05:29:54.038 INFO:journalctl@ceph.mon.c.smithi083.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2022-09-29T05:29:54.038 INFO:journalctl@ceph.mon.c.smithi083.stdout: max_mds 1 2022-09-29T05:29:54.038 INFO:journalctl@ceph.mon.c.smithi083.stdout: in 0 2022-09-29T05:29:54.038 INFO:journalctl@ceph.mon.c.smithi083.stdout: up {0=24356} 2022-09-29T05:29:54.039 INFO:journalctl@ceph.mon.c.smithi083.stdout: failed 2022-09-29T05:29:54.039 INFO:journalctl@ceph.mon.c.smithi083.stdout: damaged 2022-09-29T05:29:54.039 INFO:journalctl@ceph.mon.c.smithi083.stdout: stopped 2022-09-29T05:29:54.039 INFO:journalctl@ceph.mon.c.smithi083.stdout: data_pools [3] 2022-09-29T05:29:54.039 INFO:journalctl@ceph.mon.c.smithi083.stdout: metadata_pool 2 2022-09-29T05:29:54.040 INFO:journalctl@ceph.mon.c.smithi083.stdout: inline_data disabled 2022-09-29T05:29:54.040 INFO:journalctl@ceph.mon.c.smithi083.stdout: balancer 2022-09-29T05:29:54.040 INFO:journalctl@ceph.mon.c.smithi083.stdout: standby_count_wanted 0 2022-09-29T05:29:54.040 INFO:journalctl@ceph.mon.c.smithi083.stdout: [mds.b{0:24356} state up:active seq 3 addr [v2:172.21.15.153:6832/673217382,v1:172.21.15.153:6833/673217382] compat {c=[1],r=[1],i=[7ff]}] 2022-09-29T05:29:54.041 INFO:journalctl@ceph.mon.c.smithi083.stdout: 2022-09-29T05:29:54.041 INFO:journalctl@ceph.mon.c.smithi083.stdout: 2022-09-29T05:29:54.041 INFO:journalctl@ceph.mon.c.smithi083.stdout: Standby daemons: 2022-09-29T05:29:54.041 INFO:journalctl@ceph.mon.c.smithi083.stdout: 2022-09-29T05:29:54.041 INFO:journalctl@ceph.mon.c.smithi083.stdout: [mds.a{-1:24343} state up:standby seq 1 addr [v2:172.21.15.83:6834/4236763119,v1:172.21.15.83:6835/4236763119] compat {c=[1],r=[1],i=[7ff]}] 2022-09-29T05:29:54.042 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: mon.c@-1(???).osd e64 crush map has features 3314933000854323200, adjusting msgr requires 2022-09-29T05:29:54.042 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: mon.c@-1(???).osd e64 crush map has features 432629239337189376, adjusting msgr requires 2022-09-29T05:29:54.042 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: mon.c@-1(???).osd e64 crush map has features 432629239337189376, adjusting msgr requires 2022-09-29T05:29:54.042 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: mon.c@-1(???).osd e64 crush map has features 432629239337189376, adjusting msgr requires 2022-09-29T05:29:54.043 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:29:53 smithi083 ceph-mon[79930]: mon.c@-1(???).paxosservice(auth 1..19) refresh upgraded, format 0 -> 3 2022-09-29T05:29:54.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:54 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:29:51.127931+0000 mgr.y (mgr.34107) 24 : cluster [DBG] pgmap v10: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:29:54.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:54 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:51.185176+0000 mgr.y (mgr.34107) 25 : audit [DBG] from='client.25048 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:29:54.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:54 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:51.494811+0000 mgr.y (mgr.34107) 26 : audit [DBG] from='client.25054 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:29:54.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:54 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:51.875271+0000 mon.a (mon.0) 43 : audit [DBG] from='client.? 172.21.15.83:0/4148291325' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:29:54.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:54 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:52.281647+0000 mon.b (mon.2) 95 : audit [DBG] from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2022-09-29T05:29:54.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:54 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:52.281936+0000 mon.b (mon.2) 96 : audit [DBG] from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2022-09-29T05:29:54.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:54 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:52.282108+0000 mon.b (mon.2) 97 : audit [DBG] from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2022-09-29T05:29:55.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:55 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:29:53.128277+0000 mgr.y (mgr.34107) 27 : cluster [DBG] pgmap v11: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:29:55.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:55 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:29:54.033854+0000 mon.c (mon.1) 1 : cluster [INF] mon.c calling monitor election 2022-09-29T05:29:55.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:55 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:29:54.034420+0000 mon.a (mon.0) 44 : cluster [INF] mon.a calling monitor election 2022-09-29T05:29:55.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:55 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:29:54.051065+0000 mon.a (mon.0) 45 : cluster [INF] mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2022-09-29T05:29:55.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:55 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:29:54.062979+0000 mon.a (mon.0) 46 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0],b=[v2:172.21.15.153:3300/0,v1:172.21.15.153:6789/0],c=[v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0]} 2022-09-29T05:29:55.561 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:55 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:29:54.063486+0000 mon.a (mon.0) 47 : cluster [DBG] fsmap cephfs:1 {0=b=up:active} 1 up:standby 2022-09-29T05:29:55.561 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:55 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:29:54.063513+0000 mon.a (mon.0) 48 : cluster [DBG] osdmap e64: 8 total, 8 up, 8 in 2022-09-29T05:29:55.561 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:55 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:29:54.063815+0000 mon.a (mon.0) 49 : cluster [DBG] mgrmap e35: y(active, since 15s), standbys: x 2022-09-29T05:29:55.561 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:55 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:29:54.066836+0000 mon.a (mon.0) 50 : cluster [WRN] Health detail: HEALTH_WARN Telemetry requires re-opt-in 2022-09-29T05:29:55.561 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:55 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:29:54.066852+0000 mon.a (mon.0) 51 : cluster [WRN] [WRN] TELEMETRY_CHANGED: Telemetry requires re-opt-in 2022-09-29T05:29:55.562 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:55 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:29:54.066859+0000 mon.a (mon.0) 52 : cluster [WRN] telemetry module includes new collections; please re-opt-in to new collections with `ceph telemetry on` 2022-09-29T05:29:55.562 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:55 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:54.074335+0000 mon.a (mon.0) 53 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:29:55.562 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:55 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:54.081884+0000 mon.a (mon.0) 54 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:29:55.562 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:55 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:54.083533+0000 mon.b (mon.2) 98 : audit [DBG] from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:29:55.563 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:55 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:54.084803+0000 mon.b (mon.2) 99 : audit [DBG] from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:29:55.563 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:55 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:54.085540+0000 mon.b (mon.2) 100 : audit [INF] from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:29:56.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:56 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:29:55.128818+0000 mgr.y (mgr.34107) 28 : cluster [DBG] pgmap v12: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:29:58.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:58 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:29:57.129261+0000 mgr.y (mgr.34107) 29 : cluster [DBG] pgmap v13: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:29:59.059 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:29:58 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:29:58.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi153.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi083:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:29:59.213 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:29:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: 2022-09-29T05:29:58.863+0000 7f9b26e95700 -1 mgr.server handle_report got status from non-daemon mon.c 2022-09-29T05:29:59.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:59 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:58.264696+0000 mon.a (mon.0) 55 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:29:59.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:59 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:58.270897+0000 mon.a (mon.0) 56 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:29:59.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:59 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:59.228162+0000 mon.a (mon.0) 57 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:29:59.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:29:59 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:29:59.236659+0000 mon.a (mon.0) 58 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:01.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:00 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:29:59.129706+0000 mgr.y (mgr.34107) 30 : cluster [DBG] pgmap v14: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:01.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:00 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:30:00.000119+0000 mon.a (mon.0) 59 : cluster [WRN] overall HEALTH_WARN Telemetry requires re-opt-in 2022-09-29T05:30:01.826 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:30:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:30:01] "GET /metrics HTTP/1.1" 200 173027 "" "Prometheus/2.33.4" 2022-09-29T05:30:02.629 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:30:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:30:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:30:03.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:30:01.130275+0000 mgr.y (mgr.34107) 31 : cluster [DBG] pgmap v15: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:03.026 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:30:02.344132+0000 mon.a (mon.0) 60 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:03.026 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:30:02.350822+0000 mon.a (mon.0) 61 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:03.026 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:30:02.356051+0000 mon.a (mon.0) 62 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:03.026 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:30:02.435082+0000 mon.b (mon.2) 101 : audit [DBG] from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:30:03.027 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:30:02.438124+0000 mon.b (mon.2) 102 : audit [DBG] from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:30:03.027 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:30:02.439372+0000 mon.b (mon.2) 103 : audit [DBG] from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2022-09-29T05:30:03.027 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:30:02.440114+0000 mon.b (mon.2) 104 : audit [DBG] from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2022-09-29T05:30:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:30:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:30:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:30:03.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:30:02.339072+0000 mgr.y (mgr.34107) 32 : cephadm [INF] Detected new or changed devices on smithi083 2022-09-29T05:30:03.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cephadm 2022-09-29T05:30:02.440681+0000 mgr.y (mgr.34107) 33 : cephadm [INF] Upgrade: It appears safe to stop mon.b 2022-09-29T05:30:04.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:04 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: cluster 2022-09-29T05:30:03.130798+0000 mgr.y (mgr.34107) 34 : cluster [DBG] pgmap v16: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:04.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:04 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:30:03.931648+0000 mon.a (mon.0) 63 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:04.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:04 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:30:03.932559+0000 mon.b (mon.2) 105 : audit [INF] from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2022-09-29T05:30:04.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:04 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:30:03.933617+0000 mon.a (mon.0) 64 : audit [DBG] from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2022-09-29T05:30:04.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:04 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: audit 2022-09-29T05:30:03.934381+0000 mon.b (mon.2) 106 : audit [DBG] from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:30:05.490 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:05 smithi153 systemd[1]: Stopping Ceph mon.b for 69111522-3fb6-11ed-8431-001a4aab830c... 2022-09-29T05:30:05.490 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:05 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: debug 2022-09-29T05:30:05.484+0000 7f7aa077e700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2022-09-29T05:30:05.491 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:05 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b[39083]: debug 2022-09-29T05:30:05.484+0000 7f7aa077e700 -1 mon.b@2(peon) e3 *** Got Signal Terminated *** 2022-09-29T05:30:05.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:30:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: [29/Sep/2022:05:30:05] ENGINE Bus STOPPING 2022-09-29T05:30:05.964 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:30:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: [29/Sep/2022:05:30:05] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2022-09-29T05:30:05.964 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:30:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: [29/Sep/2022:05:30:05] ENGINE Bus STOPPED 2022-09-29T05:30:05.964 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:30:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: [29/Sep/2022:05:30:05] ENGINE Bus STARTING 2022-09-29T05:30:05.964 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:30:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: [29/Sep/2022:05:30:05] ENGINE Serving on http://:::9283 2022-09-29T05:30:05.965 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:30:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: [29/Sep/2022:05:30:05] ENGINE Bus STARTED 2022-09-29T05:30:06.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:05 smithi153 bash[69871]: ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-b 2022-09-29T05:30:06.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:05 smithi153 systemd[1]: ceph-69111522-3fb6-11ed-8431-001a4aab830c@mon.b.service: Succeeded. 2022-09-29T05:30:06.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:05 smithi153 systemd[1]: Stopped Ceph mon.b for 69111522-3fb6-11ed-8431-001a4aab830c. 2022-09-29T05:30:06.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 systemd[1]: Starting Ceph mon.b for 69111522-3fb6-11ed-8431-001a4aab830c... 2022-09-29T05:30:07.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 podman[70005]: 2022-09-29T05:30:07.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 bash[70005]: f8e8b54cae07726f70e5cf27bd26ae900cb7d56bb9c78a045d60ecec386474dd 2022-09-29T05:30:07.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: set uid:gid to 167:167 (ceph:ceph) 2022-09-29T05:30:07.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev), process ceph-mon, pid 7 2022-09-29T05:30:07.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: pidfile_write: ignore empty --pid-file 2022-09-29T05:30:07.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 systemd[1]: Started Ceph mon.b for 69111522-3fb6-11ed-8431-001a4aab830c. 2022-09-29T05:30:07.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: load: jerasure load: lrc 2022-09-29T05:30:07.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: RocksDB version: 6.15.5 2022-09-29T05:30:07.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2022-09-29T05:30:07.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Compile date Sep 28 2022 2022-09-29T05:30:07.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: DB SUMMARY 2022-09-29T05:30:07.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: DB Session ID: XFDOS3D2W6R8HB9NX9PW 2022-09-29T05:30:07.063 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: CURRENT file: CURRENT 2022-09-29T05:30:07.063 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: IDENTITY file: IDENTITY 2022-09-29T05:30:07.063 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: MANIFEST file: MANIFEST-000005 size: 503 Bytes 2022-09-29T05:30:07.063 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000014.sst 2022-09-29T05:30:07.064 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000012.log size: 5931870 ; 2022-09-29T05:30:07.064 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.error_if_exists: 0 2022-09-29T05:30:07.064 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.create_if_missing: 0 2022-09-29T05:30:07.064 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.paranoid_checks: 1 2022-09-29T05:30:07.064 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2022-09-29T05:30:07.065 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.env: 0x5617b2eb4280 2022-09-29T05:30:07.065 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.fs: Posix File System 2022-09-29T05:30:07.065 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.info_log: 0x5617b3e77b00 2022-09-29T05:30:07.065 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.max_file_opening_threads: 16 2022-09-29T05:30:07.066 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.statistics: (nil) 2022-09-29T05:30:07.066 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.use_fsync: 0 2022-09-29T05:30:07.066 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.max_log_file_size: 0 2022-09-29T05:30:07.067 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.max_manifest_file_size: 1073741824 2022-09-29T05:30:07.067 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.log_file_time_to_roll: 0 2022-09-29T05:30:07.067 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.keep_log_file_num: 1000 2022-09-29T05:30:07.067 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.recycle_log_file_num: 0 2022-09-29T05:30:07.068 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.allow_fallocate: 1 2022-09-29T05:30:07.068 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.allow_mmap_reads: 0 2022-09-29T05:30:07.068 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.allow_mmap_writes: 0 2022-09-29T05:30:07.068 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.use_direct_reads: 0 2022-09-29T05:30:07.068 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2022-09-29T05:30:07.069 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.create_missing_column_families: 0 2022-09-29T05:30:07.069 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.db_log_dir: 2022-09-29T05:30:07.069 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-b/store.db 2022-09-29T05:30:07.069 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.table_cache_numshardbits: 6 2022-09-29T05:30:07.070 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.WAL_ttl_seconds: 0 2022-09-29T05:30:07.070 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.WAL_size_limit_MB: 0 2022-09-29T05:30:07.070 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2022-09-29T05:30:07.070 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.manifest_preallocation_size: 4194304 2022-09-29T05:30:07.071 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.is_fd_close_on_exec: 1 2022-09-29T05:30:07.071 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.advise_random_on_open: 1 2022-09-29T05:30:07.071 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.db_write_buffer_size: 0 2022-09-29T05:30:07.071 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.write_buffer_manager: 0x5617b3f62ba0 2022-09-29T05:30:07.071 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.access_hint_on_compaction_start: 1 2022-09-29T05:30:07.072 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2022-09-29T05:30:07.072 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.random_access_max_buffer_size: 1048576 2022-09-29T05:30:07.072 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.use_adaptive_mutex: 0 2022-09-29T05:30:07.072 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.rate_limiter: (nil) 2022-09-29T05:30:07.073 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2022-09-29T05:30:07.073 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.wal_recovery_mode: 2 2022-09-29T05:30:07.073 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.enable_thread_tracking: 0 2022-09-29T05:30:07.073 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.enable_pipelined_write: 0 2022-09-29T05:30:07.074 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.unordered_write: 0 2022-09-29T05:30:07.074 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.allow_concurrent_memtable_write: 1 2022-09-29T05:30:07.074 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2022-09-29T05:30:07.074 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.write_thread_max_yield_usec: 100 2022-09-29T05:30:07.075 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.write_thread_slow_yield_usec: 3 2022-09-29T05:30:07.075 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.row_cache: None 2022-09-29T05:30:07.075 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.wal_filter: None 2022-09-29T05:30:07.075 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.avoid_flush_during_recovery: 0 2022-09-29T05:30:07.075 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.allow_ingest_behind: 0 2022-09-29T05:30:07.076 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.preserve_deletes: 0 2022-09-29T05:30:07.076 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.two_write_queues: 0 2022-09-29T05:30:07.076 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.manual_wal_flush: 0 2022-09-29T05:30:07.076 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.atomic_flush: 0 2022-09-29T05:30:07.077 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2022-09-29T05:30:07.077 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.persist_stats_to_disk: 0 2022-09-29T05:30:07.077 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.write_dbid_to_manifest: 0 2022-09-29T05:30:07.077 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.log_readahead_size: 0 2022-09-29T05:30:07.078 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.file_checksum_gen_factory: Unknown 2022-09-29T05:30:07.078 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.best_efforts_recovery: 0 2022-09-29T05:30:07.078 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2022-09-29T05:30:07.078 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2022-09-29T05:30:07.079 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.allow_data_in_errors: 0 2022-09-29T05:30:07.080 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.db_host_id: __hostname__ 2022-09-29T05:30:07.081 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.max_background_jobs: 2 2022-09-29T05:30:07.081 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.max_background_compactions: -1 2022-09-29T05:30:07.081 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.max_subcompactions: 1 2022-09-29T05:30:07.081 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.avoid_flush_during_shutdown: 0 2022-09-29T05:30:07.082 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2022-09-29T05:30:07.082 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.delayed_write_rate : 16777216 2022-09-29T05:30:07.082 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.max_total_wal_size: 0 2022-09-29T05:30:07.082 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2022-09-29T05:30:07.083 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.stats_dump_period_sec: 600 2022-09-29T05:30:07.083 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.stats_persist_period_sec: 600 2022-09-29T05:30:07.083 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.stats_history_buffer_size: 1048576 2022-09-29T05:30:07.083 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.max_open_files: -1 2022-09-29T05:30:07.083 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.bytes_per_sync: 0 2022-09-29T05:30:07.084 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.wal_bytes_per_sync: 0 2022-09-29T05:30:07.084 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.strict_bytes_per_sync: 0 2022-09-29T05:30:07.084 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.compaction_readahead_size: 0 2022-09-29T05:30:07.084 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.max_background_flushes: -1 2022-09-29T05:30:07.085 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Compression algorithms supported: 2022-09-29T05:30:07.085 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: kZSTDNotFinalCompression supported: 0 2022-09-29T05:30:07.085 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: kZSTD supported: 0 2022-09-29T05:30:07.085 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: kXpressCompression supported: 0 2022-09-29T05:30:07.085 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: kLZ4HCCompression supported: 1 2022-09-29T05:30:07.086 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: kZlibCompression supported: 1 2022-09-29T05:30:07.086 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: kSnappyCompression supported: 1 2022-09-29T05:30:07.086 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: kLZ4Compression supported: 1 2022-09-29T05:30:07.086 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: kBZip2Compression supported: 0 2022-09-29T05:30:07.087 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Fast CRC32 supported: Supported on x86 2022-09-29T05:30:07.087 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000005 2022-09-29T05:30:07.087 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2022-09-29T05:30:07.087 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2022-09-29T05:30:07.087 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.merge_operator: 2022-09-29T05:30:07.088 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.compaction_filter: None 2022-09-29T05:30:07.088 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.compaction_filter_factory: None 2022-09-29T05:30:07.088 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.sst_partitioner_factory: None 2022-09-29T05:30:07.088 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.memtable_factory: SkipListFactory 2022-09-29T05:30:07.089 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.table_factory: BlockBasedTable 2022-09-29T05:30:07.089 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5617b3e74e40) 2022-09-29T05:30:07.089 INFO:journalctl@ceph.mon.b.smithi153.stdout: cache_index_and_filter_blocks: 1 2022-09-29T05:30:07.089 INFO:journalctl@ceph.mon.b.smithi153.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2022-09-29T05:30:07.089 INFO:journalctl@ceph.mon.b.smithi153.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2022-09-29T05:30:07.090 INFO:journalctl@ceph.mon.b.smithi153.stdout: pin_top_level_index_and_filter: 1 2022-09-29T05:30:07.090 INFO:journalctl@ceph.mon.b.smithi153.stdout: index_type: 0 2022-09-29T05:30:07.090 INFO:journalctl@ceph.mon.b.smithi153.stdout: data_block_index_type: 0 2022-09-29T05:30:07.090 INFO:journalctl@ceph.mon.b.smithi153.stdout: index_shortening: 1 2022-09-29T05:30:07.091 INFO:journalctl@ceph.mon.b.smithi153.stdout: data_block_hash_table_util_ratio: 0.750000 2022-09-29T05:30:07.091 INFO:journalctl@ceph.mon.b.smithi153.stdout: hash_index_allow_collision: 1 2022-09-29T05:30:07.091 INFO:journalctl@ceph.mon.b.smithi153.stdout: checksum: 1 2022-09-29T05:30:07.092 INFO:journalctl@ceph.mon.b.smithi153.stdout: no_block_cache: 0 2022-09-29T05:30:07.092 INFO:journalctl@ceph.mon.b.smithi153.stdout: block_cache: 0x5617b3ee09b0 2022-09-29T05:30:07.092 INFO:journalctl@ceph.mon.b.smithi153.stdout: block_cache_name: BinnedLRUCache 2022-09-29T05:30:07.093 INFO:journalctl@ceph.mon.b.smithi153.stdout: block_cache_options: 2022-09-29T05:30:07.093 INFO:journalctl@ceph.mon.b.smithi153.stdout: capacity : 536870912 2022-09-29T05:30:07.094 INFO:journalctl@ceph.mon.b.smithi153.stdout: num_shard_bits : 4 2022-09-29T05:30:07.094 INFO:journalctl@ceph.mon.b.smithi153.stdout: strict_capacity_limit : 0 2022-09-29T05:30:07.095 INFO:journalctl@ceph.mon.b.smithi153.stdout: high_pri_pool_ratio: 0.000 2022-09-29T05:30:07.095 INFO:journalctl@ceph.mon.b.smithi153.stdout: block_cache_compressed: (nil) 2022-09-29T05:30:07.096 INFO:journalctl@ceph.mon.b.smithi153.stdout: persistent_cache: (nil) 2022-09-29T05:30:07.096 INFO:journalctl@ceph.mon.b.smithi153.stdout: block_size: 4096 2022-09-29T05:30:07.096 INFO:journalctl@ceph.mon.b.smithi153.stdout: block_size_deviation: 10 2022-09-29T05:30:07.097 INFO:journalctl@ceph.mon.b.smithi153.stdout: block_restart_interval: 16 2022-09-29T05:30:07.097 INFO:journalctl@ceph.mon.b.smithi153.stdout: index_block_restart_interval: 1 2022-09-29T05:30:07.097 INFO:journalctl@ceph.mon.b.smithi153.stdout: metadata_block_size: 4096 2022-09-29T05:30:07.098 INFO:journalctl@ceph.mon.b.smithi153.stdout: partition_filters: 0 2022-09-29T05:30:07.098 INFO:journalctl@ceph.mon.b.smithi153.stdout: use_delta_encoding: 1 2022-09-29T05:30:07.099 INFO:journalctl@ceph.mon.b.smithi153.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2022-09-29T05:30:07.099 INFO:journalctl@ceph.mon.b.smithi153.stdout: whole_key_filtering: 1 2022-09-29T05:30:07.099 INFO:journalctl@ceph.mon.b.smithi153.stdout: verify_compression: 0 2022-09-29T05:30:07.099 INFO:journalctl@ceph.mon.b.smithi153.stdout: read_amp_bytes_per_bit: 0 2022-09-29T05:30:07.100 INFO:journalctl@ceph.mon.b.smithi153.stdout: format_version: 4 2022-09-29T05:30:07.100 INFO:journalctl@ceph.mon.b.smithi153.stdout: enable_index_compression: 1 2022-09-29T05:30:07.100 INFO:journalctl@ceph.mon.b.smithi153.stdout: block_align: 0 2022-09-29T05:30:07.100 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.write_buffer_size: 33554432 2022-09-29T05:30:07.101 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.max_write_buffer_number: 2 2022-09-29T05:30:07.101 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.compression: NoCompression 2022-09-29T05:30:07.101 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.bottommost_compression: Disabled 2022-09-29T05:30:07.101 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.prefix_extractor: nullptr 2022-09-29T05:30:07.102 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2022-09-29T05:30:07.102 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.num_levels: 7 2022-09-29T05:30:07.102 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2022-09-29T05:30:07.102 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2022-09-29T05:30:07.102 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2022-09-29T05:30:07.103 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2022-09-29T05:30:07.103 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.bottommost_compression_opts.level: 32767 2022-09-29T05:30:07.103 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2022-09-29T05:30:07.103 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2022-09-29T05:30:07.104 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2022-09-29T05:30:07.104 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2022-09-29T05:30:07.104 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.bottommost_compression_opts.enabled: false 2022-09-29T05:30:07.104 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.compression_opts.window_bits: -14 2022-09-29T05:30:07.104 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.compression_opts.level: 32767 2022-09-29T05:30:07.105 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.compression_opts.strategy: 0 2022-09-29T05:30:07.105 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2022-09-29T05:30:07.105 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2022-09-29T05:30:07.105 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.compression_opts.parallel_threads: 1 2022-09-29T05:30:07.106 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.compression_opts.enabled: false 2022-09-29T05:30:07.106 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2022-09-29T05:30:07.106 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2022-09-29T05:30:07.106 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.level0_stop_writes_trigger: 36 2022-09-29T05:30:07.107 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.target_file_size_base: 67108864 2022-09-29T05:30:07.107 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.target_file_size_multiplier: 1 2022-09-29T05:30:07.107 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.max_bytes_for_level_base: 268435456 2022-09-29T05:30:07.107 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2022-09-29T05:30:07.107 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2022-09-29T05:30:07.108 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2022-09-29T05:30:07.108 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2022-09-29T05:30:07.108 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2022-09-29T05:30:07.108 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2022-09-29T05:30:07.109 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2022-09-29T05:30:07.109 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2022-09-29T05:30:07.109 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2022-09-29T05:30:07.109 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2022-09-29T05:30:07.109 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.max_compaction_bytes: 1677721600 2022-09-29T05:30:07.110 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.arena_block_size: 4194304 2022-09-29T05:30:07.110 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2022-09-29T05:30:07.110 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2022-09-29T05:30:07.110 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2022-09-29T05:30:07.111 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.disable_auto_compactions: 0 2022-09-29T05:30:07.111 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2022-09-29T05:30:07.111 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2022-09-29T05:30:07.111 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2022-09-29T05:30:07.111 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2022-09-29T05:30:07.112 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2022-09-29T05:30:07.112 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2022-09-29T05:30:07.112 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2022-09-29T05:30:07.112 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2022-09-29T05:30:07.113 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2022-09-29T05:30:07.113 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2022-09-29T05:30:07.113 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.table_properties_collectors: 2022-09-29T05:30:07.113 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.inplace_update_support: 0 2022-09-29T05:30:07.113 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.inplace_update_num_locks: 10000 2022-09-29T05:30:07.114 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2022-09-29T05:30:07.114 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.memtable_whole_key_filtering: 0 2022-09-29T05:30:07.114 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.memtable_huge_page_size: 0 2022-09-29T05:30:07.114 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.bloom_locality: 0 2022-09-29T05:30:07.115 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.max_successive_merges: 0 2022-09-29T05:30:07.115 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.optimize_filters_for_hits: 0 2022-09-29T05:30:07.115 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.paranoid_file_checks: 0 2022-09-29T05:30:07.115 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.force_consistency_checks: 1 2022-09-29T05:30:07.116 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.report_bg_io_stats: 0 2022-09-29T05:30:07.116 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.ttl: 2592000 2022-09-29T05:30:07.116 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.periodic_compaction_seconds: 0 2022-09-29T05:30:07.116 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.enable_blob_files: false 2022-09-29T05:30:07.117 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.min_blob_size: 0 2022-09-29T05:30:07.117 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.blob_file_size: 268435456 2022-09-29T05:30:07.117 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.blob_compression_type: NoCompression 2022-09-29T05:30:07.117 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.enable_blob_garbage_collection: false 2022-09-29T05:30:07.118 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2022-09-29T05:30:07.118 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: [db/version_set.cc:3457] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2022-09-29T05:30:07.118 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 16, last_sequence is 9244, log_number is 12,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2022-09-29T05:30:07.119 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 12 2022-09-29T05:30:07.119 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: [db/version_set.cc:4082] Creating manifest 16 2022-09-29T05:30:07.119 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: [db/version_set.cc:3457] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2022-09-29T05:30:07.119 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: EVENT_LOG_v1 {"time_micros": 1664429406919607, "job": 1, "event": "recovery_started", "wal_files": [12]} 2022-09-29T05:30:07.120 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #12 mode 2 2022-09-29T05:30:07.120 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2022-09-29T05:30:07.120 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: EVENT_LOG_v1 {"time_micros": 1664429406930287, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 17, "file_size": 3760629, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 3755104, "index_size": 3752, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 837, "raw_key_size": 6085, "raw_average_key_size": 21, "raw_value_size": 3746808, "raw_average_value_size": 12964, "num_data_blocks": 169, "num_entries": 289, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1664429406, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "f1fee407-bb0c-4f67-b072-d75ed423cb91", "db_session_id": "XFDOS3D2W6R8HB9NX9PW"}} 2022-09-29T05:30:07.120 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: [db/version_set.cc:4082] Creating manifest 18 2022-09-29T05:30:07.121 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: [db/version_set.cc:3457] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2022-09-29T05:30:07.121 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: EVENT_LOG_v1 {"time_micros": 1664429406931113, "job": 1, "event": "recovery_finished"} 2022-09-29T05:30:07.121 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000012.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2022-09-29T05:30:07.121 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x5617b3ec7880 2022-09-29T05:30:07.122 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: DB pointer 0x5617b3f3a000 2022-09-29T05:30:07.122 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2022-09-29T05:30:07.122 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: rocksdb: [db/db_impl/db_impl.cc:903] 2022-09-29T05:30:07.122 INFO:journalctl@ceph.mon.b.smithi153.stdout: ** DB Stats ** 2022-09-29T05:30:07.122 INFO:journalctl@ceph.mon.b.smithi153.stdout: Uptime(secs): 0.0 total, 0.0 interval 2022-09-29T05:30:07.123 INFO:journalctl@ceph.mon.b.smithi153.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2022-09-29T05:30:07.123 INFO:journalctl@ceph.mon.b.smithi153.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2022-09-29T05:30:07.123 INFO:journalctl@ceph.mon.b.smithi153.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2022-09-29T05:30:07.123 INFO:journalctl@ceph.mon.b.smithi153.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2022-09-29T05:30:07.124 INFO:journalctl@ceph.mon.b.smithi153.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2022-09-29T05:30:07.124 INFO:journalctl@ceph.mon.b.smithi153.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2022-09-29T05:30:07.124 INFO:journalctl@ceph.mon.b.smithi153.stdout: 2022-09-29T05:30:07.124 INFO:journalctl@ceph.mon.b.smithi153.stdout: ** Compaction Stats [default] ** 2022-09-29T05:30:07.124 INFO:journalctl@ceph.mon.b.smithi153.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2022-09-29T05:30:07.125 INFO:journalctl@ceph.mon.b.smithi153.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2022-09-29T05:30:07.125 INFO:journalctl@ceph.mon.b.smithi153.stdout: L0 1/0 3.59 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 624.3 0.01 0.00 1 0.006 0 0 2022-09-29T05:30:07.125 INFO:journalctl@ceph.mon.b.smithi153.stdout: L6 1/0 20.41 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2022-09-29T05:30:07.125 INFO:journalctl@ceph.mon.b.smithi153.stdout: Sum 2/0 23.99 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 624.3 0.01 0.00 1 0.006 0 0 2022-09-29T05:30:07.126 INFO:journalctl@ceph.mon.b.smithi153.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 624.3 0.01 0.00 1 0.006 0 0 2022-09-29T05:30:07.126 INFO:journalctl@ceph.mon.b.smithi153.stdout: 2022-09-29T05:30:07.126 INFO:journalctl@ceph.mon.b.smithi153.stdout: ** Compaction Stats [default] ** 2022-09-29T05:30:07.127 INFO:journalctl@ceph.mon.b.smithi153.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2022-09-29T05:30:07.127 INFO:journalctl@ceph.mon.b.smithi153.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2022-09-29T05:30:07.127 INFO:journalctl@ceph.mon.b.smithi153.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 624.3 0.01 0.00 1 0.006 0 0 2022-09-29T05:30:07.128 INFO:journalctl@ceph.mon.b.smithi153.stdout: Uptime(secs): 0.0 total, 0.0 interval 2022-09-29T05:30:07.128 INFO:journalctl@ceph.mon.b.smithi153.stdout: Flush(GB): cumulative 0.004, interval 0.004 2022-09-29T05:30:07.128 INFO:journalctl@ceph.mon.b.smithi153.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2022-09-29T05:30:07.128 INFO:journalctl@ceph.mon.b.smithi153.stdout: AddFile(Total Files): cumulative 0, interval 0 2022-09-29T05:30:07.129 INFO:journalctl@ceph.mon.b.smithi153.stdout: AddFile(L0 Files): cumulative 0, interval 0 2022-09-29T05:30:07.129 INFO:journalctl@ceph.mon.b.smithi153.stdout: AddFile(Keys): cumulative 0, interval 0 2022-09-29T05:30:07.129 INFO:journalctl@ceph.mon.b.smithi153.stdout: Cumulative compaction: 0.00 GB write, 257.18 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2022-09-29T05:30:07.129 INFO:journalctl@ceph.mon.b.smithi153.stdout: Interval compaction: 0.00 GB write, 257.18 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2022-09-29T05:30:07.130 INFO:journalctl@ceph.mon.b.smithi153.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2022-09-29T05:30:07.130 INFO:journalctl@ceph.mon.b.smithi153.stdout: 2022-09-29T05:30:07.130 INFO:journalctl@ceph.mon.b.smithi153.stdout: ** File Read Latency Histogram By Level [default] ** 2022-09-29T05:30:07.130 INFO:journalctl@ceph.mon.b.smithi153.stdout: 2022-09-29T05:30:07.131 INFO:journalctl@ceph.mon.b.smithi153.stdout: ** Compaction Stats [default] ** 2022-09-29T05:30:07.131 INFO:journalctl@ceph.mon.b.smithi153.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2022-09-29T05:30:07.131 INFO:journalctl@ceph.mon.b.smithi153.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2022-09-29T05:30:07.131 INFO:journalctl@ceph.mon.b.smithi153.stdout: L0 1/0 3.59 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 624.3 0.01 0.00 1 0.006 0 0 2022-09-29T05:30:07.131 INFO:journalctl@ceph.mon.b.smithi153.stdout: L6 1/0 20.41 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2022-09-29T05:30:07.132 INFO:journalctl@ceph.mon.b.smithi153.stdout: Sum 2/0 23.99 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 624.3 0.01 0.00 1 0.006 0 0 2022-09-29T05:30:07.132 INFO:journalctl@ceph.mon.b.smithi153.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2022-09-29T05:30:07.132 INFO:journalctl@ceph.mon.b.smithi153.stdout: 2022-09-29T05:30:07.132 INFO:journalctl@ceph.mon.b.smithi153.stdout: ** Compaction Stats [default] ** 2022-09-29T05:30:07.132 INFO:journalctl@ceph.mon.b.smithi153.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2022-09-29T05:30:07.133 INFO:journalctl@ceph.mon.b.smithi153.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2022-09-29T05:30:07.133 INFO:journalctl@ceph.mon.b.smithi153.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 624.3 0.01 0.00 1 0.006 0 0 2022-09-29T05:30:07.133 INFO:journalctl@ceph.mon.b.smithi153.stdout: Uptime(secs): 0.0 total, 0.0 interval 2022-09-29T05:30:07.133 INFO:journalctl@ceph.mon.b.smithi153.stdout: Flush(GB): cumulative 0.004, interval 0.000 2022-09-29T05:30:07.134 INFO:journalctl@ceph.mon.b.smithi153.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2022-09-29T05:30:07.134 INFO:journalctl@ceph.mon.b.smithi153.stdout: AddFile(Total Files): cumulative 0, interval 0 2022-09-29T05:30:07.134 INFO:journalctl@ceph.mon.b.smithi153.stdout: AddFile(L0 Files): cumulative 0, interval 0 2022-09-29T05:30:07.134 INFO:journalctl@ceph.mon.b.smithi153.stdout: AddFile(Keys): cumulative 0, interval 0 2022-09-29T05:30:07.135 INFO:journalctl@ceph.mon.b.smithi153.stdout: Cumulative compaction: 0.00 GB write, 256.52 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2022-09-29T05:30:07.135 INFO:journalctl@ceph.mon.b.smithi153.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2022-09-29T05:30:07.135 INFO:journalctl@ceph.mon.b.smithi153.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2022-09-29T05:30:07.135 INFO:journalctl@ceph.mon.b.smithi153.stdout: 2022-09-29T05:30:07.135 INFO:journalctl@ceph.mon.b.smithi153.stdout: ** File Read Latency Histogram By Level [default] ** 2022-09-29T05:30:07.136 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: starting mon.b rank 2 at public addrs [v2:172.21.15.153:3300/0,v1:172.21.15.153:6789/0] at bind addrs [v2:172.21.15.153:3300/0,v1:172.21.15.153:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 69111522-3fb6-11ed-8431-001a4aab830c 2022-09-29T05:30:07.136 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: mon.b@-1(???) e3 preinit fsid 69111522-3fb6-11ed-8431-001a4aab830c 2022-09-29T05:30:07.136 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: mon.b@-1(???).mds e6 new map 2022-09-29T05:30:07.136 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: mon.b@-1(???).mds e6 print_map 2022-09-29T05:30:07.137 INFO:journalctl@ceph.mon.b.smithi153.stdout: e6 2022-09-29T05:30:07.137 INFO:journalctl@ceph.mon.b.smithi153.stdout: enable_multiple, ever_enabled_multiple: 1,1 2022-09-29T05:30:07.137 INFO:journalctl@ceph.mon.b.smithi153.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2022-09-29T05:30:07.137 INFO:journalctl@ceph.mon.b.smithi153.stdout: legacy client fscid: 1 2022-09-29T05:30:07.137 INFO:journalctl@ceph.mon.b.smithi153.stdout: 2022-09-29T05:30:07.138 INFO:journalctl@ceph.mon.b.smithi153.stdout: Filesystem 'cephfs' (1) 2022-09-29T05:30:07.138 INFO:journalctl@ceph.mon.b.smithi153.stdout: fs_name cephfs 2022-09-29T05:30:07.138 INFO:journalctl@ceph.mon.b.smithi153.stdout: epoch 6 2022-09-29T05:30:07.138 INFO:journalctl@ceph.mon.b.smithi153.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2022-09-29T05:30:07.139 INFO:journalctl@ceph.mon.b.smithi153.stdout: created 2022-09-29T05:25:03.789615+0000 2022-09-29T05:30:07.139 INFO:journalctl@ceph.mon.b.smithi153.stdout: modified 2022-09-29T05:25:05.816761+0000 2022-09-29T05:30:07.139 INFO:journalctl@ceph.mon.b.smithi153.stdout: tableserver 0 2022-09-29T05:30:07.139 INFO:journalctl@ceph.mon.b.smithi153.stdout: root 0 2022-09-29T05:30:07.139 INFO:journalctl@ceph.mon.b.smithi153.stdout: session_timeout 60 2022-09-29T05:30:07.140 INFO:journalctl@ceph.mon.b.smithi153.stdout: session_autoclose 300 2022-09-29T05:30:07.140 INFO:journalctl@ceph.mon.b.smithi153.stdout: max_file_size 1099511627776 2022-09-29T05:30:07.140 INFO:journalctl@ceph.mon.b.smithi153.stdout: required_client_features {} 2022-09-29T05:30:07.140 INFO:journalctl@ceph.mon.b.smithi153.stdout: last_failure 0 2022-09-29T05:30:07.140 INFO:journalctl@ceph.mon.b.smithi153.stdout: last_failure_osd_epoch 0 2022-09-29T05:30:07.141 INFO:journalctl@ceph.mon.b.smithi153.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2022-09-29T05:30:07.141 INFO:journalctl@ceph.mon.b.smithi153.stdout: max_mds 1 2022-09-29T05:30:07.141 INFO:journalctl@ceph.mon.b.smithi153.stdout: in 0 2022-09-29T05:30:07.141 INFO:journalctl@ceph.mon.b.smithi153.stdout: up {0=24356} 2022-09-29T05:30:07.142 INFO:journalctl@ceph.mon.b.smithi153.stdout: failed 2022-09-29T05:30:07.142 INFO:journalctl@ceph.mon.b.smithi153.stdout: damaged 2022-09-29T05:30:07.142 INFO:journalctl@ceph.mon.b.smithi153.stdout: stopped 2022-09-29T05:30:07.142 INFO:journalctl@ceph.mon.b.smithi153.stdout: data_pools [3] 2022-09-29T05:30:07.143 INFO:journalctl@ceph.mon.b.smithi153.stdout: metadata_pool 2 2022-09-29T05:30:07.143 INFO:journalctl@ceph.mon.b.smithi153.stdout: inline_data disabled 2022-09-29T05:30:07.143 INFO:journalctl@ceph.mon.b.smithi153.stdout: balancer 2022-09-29T05:30:07.143 INFO:journalctl@ceph.mon.b.smithi153.stdout: standby_count_wanted 0 2022-09-29T05:30:07.143 INFO:journalctl@ceph.mon.b.smithi153.stdout: [mds.b{0:24356} state up:active seq 3 addr [v2:172.21.15.153:6832/673217382,v1:172.21.15.153:6833/673217382] compat {c=[1],r=[1],i=[7ff]}] 2022-09-29T05:30:07.144 INFO:journalctl@ceph.mon.b.smithi153.stdout: 2022-09-29T05:30:07.144 INFO:journalctl@ceph.mon.b.smithi153.stdout: 2022-09-29T05:30:07.144 INFO:journalctl@ceph.mon.b.smithi153.stdout: Standby daemons: 2022-09-29T05:30:07.144 INFO:journalctl@ceph.mon.b.smithi153.stdout: 2022-09-29T05:30:07.145 INFO:journalctl@ceph.mon.b.smithi153.stdout: [mds.a{-1:24343} state up:standby seq 1 addr [v2:172.21.15.83:6834/4236763119,v1:172.21.15.83:6835/4236763119] compat {c=[1],r=[1],i=[7ff]}] 2022-09-29T05:30:07.145 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: mon.b@-1(???).osd e64 crush map has features 3314933000854323200, adjusting msgr requires 2022-09-29T05:30:07.145 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: mon.b@-1(???).osd e64 crush map has features 432629239337189376, adjusting msgr requires 2022-09-29T05:30:07.145 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: mon.b@-1(???).osd e64 crush map has features 432629239337189376, adjusting msgr requires 2022-09-29T05:30:07.146 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: mon.b@-1(???).osd e64 crush map has features 432629239337189376, adjusting msgr requires 2022-09-29T05:30:07.146 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:06 smithi153 ceph-mon[70044]: mon.b@-1(???).paxosservice(auth 1..20) refresh upgraded, format 0 -> 3 2022-09-29T05:30:09.059 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:30:08 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:30:08.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi153.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi083:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:30:09.059 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:30:08 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:30:08.662Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephPGImbalance\nexpr: abs(((ceph_osd_numpg > 0) - on(job) group_left() avg by(job) (ceph_osd_numpg\n > 0)) / on(job) group_left() avg by(job) (ceph_osd_numpg > 0)) * on(ceph_daemon)\n group_left(hostname) ceph_osd_metadata > 0.3\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.5\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} deviates by\n more than 30% from average PG count.\n summary: PGs are not balanced across OSDs\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi153.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi083:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:30:11.943 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:30:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:30:11] "GET /metrics HTTP/1.1" 200 170063 "" "Prometheus/2.33.4" 2022-09-29T05:30:12.213 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:30:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: 2022-09-29T05:30:11.940+0000 7f9b26e95700 -1 mgr.server handle_report got status from non-daemon mon.b 2022-09-29T05:30:12.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: mkfs 69111522-3fb6-11ed-8431-001a4aab830c 2022-09-29T05:30:12.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: Upgrade: Need to upgrade myself (mgr.x) 2022-09-29T05:30:12.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: Upgrade: Need to upgrade myself (mgr.x) 2022-09-29T05:30:12.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: Upgrade: Updating mgr.y 2022-09-29T05:30:12.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: Deploying daemon mgr.y on smithi083 2022-09-29T05:30:12.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: Detected new or changed devices on smithi083 2022-09-29T05:30:12.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: Upgrade: Need to upgrade myself (mgr.x) 2022-09-29T05:30:12.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: Upgrade: Need to upgrade myself (mgr.x) 2022-09-29T05:30:12.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: Failing over to other MGR 2022-09-29T05:30:12.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: [29/Sep/2022:05:28:58] ENGINE Bus STARTING 2022-09-29T05:30:12.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: [29/Sep/2022:05:28:58] ENGINE Serving on https://172.21.15.83:7150 2022-09-29T05:30:12.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: [29/Sep/2022:05:28:59] ENGINE Serving on https://:::8765 2022-09-29T05:30:12.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: [29/Sep/2022:05:28:59] ENGINE Bus STARTED 2022-09-29T05:30:12.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: Updating smithi153:/etc/ceph/ceph.conf 2022-09-29T05:30:12.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: Updating smithi153:/var/lib/ceph/69111522-3fb6-11ed-8431-001a4aab830c/config/ceph.conf 2022-09-29T05:30:12.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: Updating smithi153:/etc/ceph/ceph.client.admin.keyring 2022-09-29T05:30:12.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: Updating smithi153:/var/lib/ceph/69111522-3fb6-11ed-8431-001a4aab830c/config/ceph.client.admin.keyring 2022-09-29T05:30:12.218 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: Detected new or changed devices on smithi083 2022-09-29T05:30:12.218 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: Updating smithi083:/etc/ceph/ceph.conf 2022-09-29T05:30:12.218 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: Updating smithi083:/var/lib/ceph/69111522-3fb6-11ed-8431-001a4aab830c/config/ceph.conf 2022-09-29T05:30:12.218 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: Updating smithi083:/etc/ceph/ceph.client.admin.keyring 2022-09-29T05:30:12.219 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: Updating smithi083:/var/lib/ceph/69111522-3fb6-11ed-8431-001a4aab830c/config/ceph.client.admin.keyring 2022-09-29T05:30:12.219 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: Reconfiguring prometheus.a (dependencies changed)... 2022-09-29T05:30:12.219 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: Reconfiguring daemon prometheus.a on smithi153 2022-09-29T05:30:12.219 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: Upgrade: Updating mgr.x 2022-09-29T05:30:12.220 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: Deploying daemon mgr.x on smithi153 2022-09-29T05:30:12.220 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: Detected new or changed devices on smithi153 2022-09-29T05:30:12.220 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: Upgrade: Setting container_image for all mgr 2022-09-29T05:30:12.220 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: Upgrade: It appears safe to stop mon.a 2022-09-29T05:30:12.220 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: Upgrade: Updating mon.a 2022-09-29T05:30:12.221 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: Deploying daemon mon.a on smithi083 2022-09-29T05:30:12.221 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: pgmap v20: 105 pgs: 105 active+clean; 582 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:12.221 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: pgmap v21: 105 pgs: 105 active+clean; 582 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:12.221 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: mon.a calling monitor election 2022-09-29T05:30:12.222 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2022-09-29T05:30:12.222 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: monmap e3: 3 mons at {a=[v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0],b=[v2:172.21.15.153:3300/0,v1:172.21.15.153:6789/0],c=[v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0]} 2022-09-29T05:30:12.222 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: fsmap cephfs:1 {0=b=up:active} 1 up:standby 2022-09-29T05:30:12.222 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: osdmap e63: 8 total, 8 up, 8 in 2022-09-29T05:30:12.223 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: mgrmap e31: y(active, since 35s), standbys: x 2022-09-29T05:30:12.223 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: mgrmap e32: y(active, since 35s), standbys: x 2022-09-29T05:30:12.223 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: Health detail: HEALTH_WARN Telemetry requires re-opt-in 2022-09-29T05:30:12.223 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: [WRN] TELEMETRY_CHANGED: Telemetry requires re-opt-in 2022-09-29T05:30:12.224 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: mkfs 69111522-3fb6-11ed-8431-001a4aab830c 2022-09-29T05:30:12.224 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: Upgrade: Need to upgrade myself (mgr.x) 2022-09-29T05:30:12.224 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: Upgrade: Need to upgrade myself (mgr.x) 2022-09-29T05:30:12.225 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: Upgrade: Updating mgr.y 2022-09-29T05:30:12.225 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: Deploying daemon mgr.y on smithi083 2022-09-29T05:30:12.225 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: Detected new or changed devices on smithi083 2022-09-29T05:30:12.225 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: Upgrade: Need to upgrade myself (mgr.x) 2022-09-29T05:30:12.226 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: Upgrade: Need to upgrade myself (mgr.x) 2022-09-29T05:30:12.226 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: Failing over to other MGR 2022-09-29T05:30:12.226 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: [29/Sep/2022:05:28:58] ENGINE Bus STARTING 2022-09-29T05:30:12.226 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: [29/Sep/2022:05:28:58] ENGINE Serving on https://172.21.15.83:7150 2022-09-29T05:30:12.227 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: [29/Sep/2022:05:28:59] ENGINE Serving on https://:::8765 2022-09-29T05:30:12.227 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: [29/Sep/2022:05:28:59] ENGINE Bus STARTED 2022-09-29T05:30:12.227 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: Updating smithi153:/etc/ceph/ceph.conf 2022-09-29T05:30:12.227 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: Updating smithi153:/var/lib/ceph/69111522-3fb6-11ed-8431-001a4aab830c/config/ceph.conf 2022-09-29T05:30:12.227 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: Updating smithi153:/etc/ceph/ceph.client.admin.keyring 2022-09-29T05:30:12.228 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: Updating smithi153:/var/lib/ceph/69111522-3fb6-11ed-8431-001a4aab830c/config/ceph.client.admin.keyring 2022-09-29T05:30:12.228 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: Detected new or changed devices on smithi083 2022-09-29T05:30:12.228 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: Updating smithi083:/etc/ceph/ceph.conf 2022-09-29T05:30:12.228 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: Updating smithi083:/var/lib/ceph/69111522-3fb6-11ed-8431-001a4aab830c/config/ceph.conf 2022-09-29T05:30:12.229 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: Updating smithi083:/etc/ceph/ceph.client.admin.keyring 2022-09-29T05:30:12.229 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: Updating smithi083:/var/lib/ceph/69111522-3fb6-11ed-8431-001a4aab830c/config/ceph.client.admin.keyring 2022-09-29T05:30:12.229 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: Reconfiguring prometheus.a (dependencies changed)... 2022-09-29T05:30:12.229 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: Reconfiguring daemon prometheus.a on smithi153 2022-09-29T05:30:12.230 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: Upgrade: Updating mgr.x 2022-09-29T05:30:12.230 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: Deploying daemon mgr.x on smithi153 2022-09-29T05:30:12.230 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: Detected new or changed devices on smithi153 2022-09-29T05:30:12.230 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: Upgrade: Setting container_image for all mgr 2022-09-29T05:30:12.231 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: Upgrade: It appears safe to stop mon.a 2022-09-29T05:30:12.231 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: Upgrade: Updating mon.a 2022-09-29T05:30:12.231 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: Deploying daemon mon.a on smithi083 2022-09-29T05:30:12.232 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: pgmap v20: 105 pgs: 105 active+clean; 582 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:12.232 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: pgmap v21: 105 pgs: 105 active+clean; 582 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:12.232 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: mon.a calling monitor election 2022-09-29T05:30:12.232 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2022-09-29T05:30:12.232 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: monmap e3: 3 mons at {a=[v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0],b=[v2:172.21.15.153:3300/0,v1:172.21.15.153:6789/0],c=[v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0]} 2022-09-29T05:30:12.233 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: fsmap cephfs:1 {0=b=up:active} 1 up:standby 2022-09-29T05:30:12.233 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: osdmap e63: 8 total, 8 up, 8 in 2022-09-29T05:30:12.233 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: mgrmap e31: y(active, since 35s), standbys: x 2022-09-29T05:30:12.234 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: mgrmap e32: y(active, since 35s), standbys: x 2022-09-29T05:30:12.234 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: Health detail: HEALTH_WARN Telemetry requires re-opt-in 2022-09-29T05:30:12.234 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: [WRN] TELEMETRY_CHANGED: Telemetry requires re-opt-in 2022-09-29T05:30:12.235 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: telemetry module includes new collections; please re-opt-in to new collections with `ceph telemetry on` 2022-09-29T05:30:12.235 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: Active manager daemon y restarted 2022-09-29T05:30:12.235 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: Activating manager daemon y 2022-09-29T05:30:12.236 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: osdmap e64: 8 total, 8 up, 8 in 2022-09-29T05:30:12.236 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: Standby manager daemon x restarted 2022-09-29T05:30:12.236 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: Standby manager daemon x started 2022-09-29T05:30:12.236 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: mgrmap e33: y(active, starting, since 0.0570875s), standbys: x 2022-09-29T05:30:12.237 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: Manager daemon y is now available 2022-09-29T05:30:12.237 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: [29/Sep/2022:05:29:39] ENGINE Bus STARTING 2022-09-29T05:30:12.237 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: mgrmap e34: y(active, since 1.07338s), standbys: x 2022-09-29T05:30:12.237 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: pgmap v3: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:12.238 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: [29/Sep/2022:05:29:39] ENGINE Serving on https://:::8765 2022-09-29T05:30:12.238 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: [29/Sep/2022:05:29:40] ENGINE Serving on https://172.21.15.83:7150 2022-09-29T05:30:12.238 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: [29/Sep/2022:05:29:40] ENGINE Bus STARTED 2022-09-29T05:30:12.238 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: pgmap v4: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:12.239 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: Health check cleared: TELEMETRY_CHANGED (was: Telemetry requires re-opt-in) 2022-09-29T05:30:12.239 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: Cluster is now healthy 2022-09-29T05:30:12.239 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: mgrmap e35: y(active, since 2s), standbys: x 2022-09-29T05:30:12.239 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: pgmap v5: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:12.239 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:12.240 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:12.240 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:12.240 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:12.240 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: Updating smithi083:/etc/ceph/ceph.conf 2022-09-29T05:30:12.241 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: pgmap v6: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:12.241 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: Updating smithi083:/var/lib/ceph/69111522-3fb6-11ed-8431-001a4aab830c/config/ceph.conf 2022-09-29T05:30:12.241 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: Updating smithi083:/etc/ceph/ceph.client.admin.keyring 2022-09-29T05:30:12.241 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:12.241 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:12.242 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: Updating smithi083:/var/lib/ceph/69111522-3fb6-11ed-8431-001a4aab830c/config/ceph.client.admin.keyring 2022-09-29T05:30:12.242 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:12.242 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:12.242 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: pgmap v7: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:12.243 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: Detected new or changed devices on smithi153 2022-09-29T05:30:12.243 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:12.243 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:12.243 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: Updating smithi153:/etc/ceph/ceph.conf 2022-09-29T05:30:12.244 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: Updating smithi153:/var/lib/ceph/69111522-3fb6-11ed-8431-001a4aab830c/config/ceph.conf 2022-09-29T05:30:12.244 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: Updating smithi153:/etc/ceph/ceph.client.admin.keyring 2022-09-29T05:30:12.244 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: pgmap v8: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:12.244 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: Updating smithi153:/var/lib/ceph/69111522-3fb6-11ed-8431-001a4aab830c/config/ceph.client.admin.keyring 2022-09-29T05:30:12.245 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:12.245 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: telemetry module includes new collections; please re-opt-in to new collections with `ceph telemetry on` 2022-09-29T05:30:12.245 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: Active manager daemon y restarted 2022-09-29T05:30:12.246 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: Activating manager daemon y 2022-09-29T05:30:12.246 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: osdmap e64: 8 total, 8 up, 8 in 2022-09-29T05:30:12.246 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: Standby manager daemon x restarted 2022-09-29T05:30:12.246 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: Standby manager daemon x started 2022-09-29T05:30:12.247 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: mgrmap e33: y(active, starting, since 0.0570875s), standbys: x 2022-09-29T05:30:12.247 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: Manager daemon y is now available 2022-09-29T05:30:12.247 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: [29/Sep/2022:05:29:39] ENGINE Bus STARTING 2022-09-29T05:30:12.247 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: mgrmap e34: y(active, since 1.07338s), standbys: x 2022-09-29T05:30:12.248 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: pgmap v3: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:12.248 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: [29/Sep/2022:05:29:39] ENGINE Serving on https://:::8765 2022-09-29T05:30:12.248 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: [29/Sep/2022:05:29:40] ENGINE Serving on https://172.21.15.83:7150 2022-09-29T05:30:12.248 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: [29/Sep/2022:05:29:40] ENGINE Bus STARTED 2022-09-29T05:30:12.249 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: pgmap v4: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:12.249 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: Health check cleared: TELEMETRY_CHANGED (was: Telemetry requires re-opt-in) 2022-09-29T05:30:12.249 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: Cluster is now healthy 2022-09-29T05:30:12.249 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: mgrmap e35: y(active, since 2s), standbys: x 2022-09-29T05:30:12.250 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: pgmap v5: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:12.250 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:12.250 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:12.250 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:12.251 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:12.251 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: Updating smithi083:/etc/ceph/ceph.conf 2022-09-29T05:30:12.251 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: pgmap v6: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:12.251 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: Updating smithi083:/var/lib/ceph/69111522-3fb6-11ed-8431-001a4aab830c/config/ceph.conf 2022-09-29T05:30:12.251 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: Updating smithi083:/etc/ceph/ceph.client.admin.keyring 2022-09-29T05:30:12.252 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:12.252 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:12.252 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: Updating smithi083:/var/lib/ceph/69111522-3fb6-11ed-8431-001a4aab830c/config/ceph.client.admin.keyring 2022-09-29T05:30:12.253 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:12.253 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:12.253 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: pgmap v7: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:12.254 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: Detected new or changed devices on smithi153 2022-09-29T05:30:12.254 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:12.255 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:12.255 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: Updating smithi153:/etc/ceph/ceph.conf 2022-09-29T05:30:12.255 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: Updating smithi153:/var/lib/ceph/69111522-3fb6-11ed-8431-001a4aab830c/config/ceph.conf 2022-09-29T05:30:12.256 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: Updating smithi153:/etc/ceph/ceph.client.admin.keyring 2022-09-29T05:30:12.256 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: pgmap v8: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:12.256 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: Updating smithi153:/var/lib/ceph/69111522-3fb6-11ed-8431-001a4aab830c/config/ceph.client.admin.keyring 2022-09-29T05:30:12.257 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:12.258 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:12.258 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:12.258 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:30:12.259 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:30:12.259 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2022-09-29T05:30:12.260 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2022-09-29T05:30:12.260 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: Upgrade: It appears safe to stop mon.c 2022-09-29T05:30:12.260 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: pgmap v9: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:12.260 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: Health check failed: Telemetry requires re-opt-in (TELEMETRY_CHANGED) 2022-09-29T05:30:12.261 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: Upgrade: Updating mon.c 2022-09-29T05:30:12.261 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:12.261 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2022-09-29T05:30:12.261 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2022-09-29T05:30:12.261 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:30:12.262 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: Deploying daemon mon.c on smithi083 2022-09-29T05:30:12.262 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: pgmap v10: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:12.262 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: from='client.25048 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:30:12.262 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: from='client.25054 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:30:12.263 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4148291325' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:30:12.263 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2022-09-29T05:30:12.263 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2022-09-29T05:30:12.263 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2022-09-29T05:30:12.264 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: pgmap v11: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:12.264 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: mon.c calling monitor election 2022-09-29T05:30:12.264 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: mon.a calling monitor election 2022-09-29T05:30:12.264 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2022-09-29T05:30:12.264 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: monmap e3: 3 mons at {a=[v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0],b=[v2:172.21.15.153:3300/0,v1:172.21.15.153:6789/0],c=[v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0]} 2022-09-29T05:30:12.265 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: fsmap cephfs:1 {0=b=up:active} 1 up:standby 2022-09-29T05:30:12.265 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: osdmap e64: 8 total, 8 up, 8 in 2022-09-29T05:30:12.265 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: mgrmap e35: y(active, since 15s), standbys: x 2022-09-29T05:30:12.265 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: Health detail: HEALTH_WARN Telemetry requires re-opt-in 2022-09-29T05:30:12.266 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: [WRN] TELEMETRY_CHANGED: Telemetry requires re-opt-in 2022-09-29T05:30:12.266 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: telemetry module includes new collections; please re-opt-in to new collections with `ceph telemetry on` 2022-09-29T05:30:12.266 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:12.267 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:12.267 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:12.267 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:30:12.267 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:30:12.268 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2022-09-29T05:30:12.268 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2022-09-29T05:30:12.268 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: Upgrade: It appears safe to stop mon.c 2022-09-29T05:30:12.268 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: pgmap v9: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:12.269 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: Health check failed: Telemetry requires re-opt-in (TELEMETRY_CHANGED) 2022-09-29T05:30:12.269 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: Upgrade: Updating mon.c 2022-09-29T05:30:12.269 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:12.269 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2022-09-29T05:30:12.270 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2022-09-29T05:30:12.270 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:30:12.270 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: Deploying daemon mon.c on smithi083 2022-09-29T05:30:12.270 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: pgmap v10: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:12.271 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: from='client.25048 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:30:12.271 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: from='client.25054 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:30:12.271 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4148291325' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:30:12.271 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2022-09-29T05:30:12.272 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2022-09-29T05:30:12.272 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2022-09-29T05:30:12.272 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: pgmap v11: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:12.272 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: mon.c calling monitor election 2022-09-29T05:30:12.272 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: mon.a calling monitor election 2022-09-29T05:30:12.273 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2022-09-29T05:30:12.273 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: monmap e3: 3 mons at {a=[v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0],b=[v2:172.21.15.153:3300/0,v1:172.21.15.153:6789/0],c=[v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0]} 2022-09-29T05:30:12.273 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: fsmap cephfs:1 {0=b=up:active} 1 up:standby 2022-09-29T05:30:12.273 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: osdmap e64: 8 total, 8 up, 8 in 2022-09-29T05:30:12.274 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: mgrmap e35: y(active, since 15s), standbys: x 2022-09-29T05:30:12.274 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: Health detail: HEALTH_WARN Telemetry requires re-opt-in 2022-09-29T05:30:12.274 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: [WRN] TELEMETRY_CHANGED: Telemetry requires re-opt-in 2022-09-29T05:30:12.274 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: telemetry module includes new collections; please re-opt-in to new collections with `ceph telemetry on` 2022-09-29T05:30:12.275 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:12.275 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:12.276 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:30:12.276 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:30:12.276 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:30:12.276 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: pgmap v12: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:12.276 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: pgmap v13: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:12.277 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:12.277 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:12.277 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: pgmap v14: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:12.277 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:12.278 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:12.278 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: overall HEALTH_WARN Telemetry requires re-opt-in 2022-09-29T05:30:12.278 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: pgmap v15: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:12.278 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: Detected new or changed devices on smithi083 2022-09-29T05:30:12.279 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:12.279 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:12.279 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:12.279 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:30:12.280 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:30:12.280 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2022-09-29T05:30:12.280 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2022-09-29T05:30:12.280 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: Upgrade: It appears safe to stop mon.b 2022-09-29T05:30:12.280 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: pgmap v16: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:12.281 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: Upgrade: Updating mon.b 2022-09-29T05:30:12.281 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:12.281 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2022-09-29T05:30:12.281 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2022-09-29T05:30:12.282 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:30:12.282 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: Deploying daemon mon.b on smithi153 2022-09-29T05:30:12.282 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: pgmap v17: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:12.283 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2022-09-29T05:30:12.283 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2022-09-29T05:30:12.283 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2022-09-29T05:30:12.283 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: Health detail: HEALTH_WARN Telemetry requires re-opt-in 2022-09-29T05:30:12.284 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: [WRN] TELEMETRY_CHANGED: Telemetry requires re-opt-in 2022-09-29T05:30:12.284 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: telemetry module includes new collections; please re-opt-in to new collections with `ceph telemetry on` 2022-09-29T05:30:12.284 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2022-09-29T05:30:12.284 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: mon.a calling monitor election 2022-09-29T05:30:12.284 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: mon.c calling monitor election 2022-09-29T05:30:12.285 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: mon.b calling monitor election 2022-09-29T05:30:12.285 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2022-09-29T05:30:12.285 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2022-09-29T05:30:12.285 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: pgmap v18: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:12.286 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: pgmap v19: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:12.286 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: pgmap v20: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:12.286 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2022-09-29T05:30:12.286 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: Health detail: HEALTH_WARN Telemetry requires re-opt-in 2022-09-29T05:30:12.287 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: [WRN] TELEMETRY_CHANGED: Telemetry requires re-opt-in 2022-09-29T05:30:12.287 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: telemetry module includes new collections; please re-opt-in to new collections with `ceph telemetry on` 2022-09-29T05:30:12.287 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: mon.c calling monitor election 2022-09-29T05:30:12.287 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: mon.a calling monitor election 2022-09-29T05:30:12.288 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2022-09-29T05:30:12.288 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: monmap e4: 3 mons at {a=[v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0],b=[v2:172.21.15.153:3300/0,v1:172.21.15.153:6789/0],c=[v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0]} 2022-09-29T05:30:12.288 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: fsmap cephfs:1 {0=b=up:active} 1 up:standby 2022-09-29T05:30:12.288 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: osdmap e64: 8 total, 8 up, 8 in 2022-09-29T05:30:12.289 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: mgrmap e35: y(active, since 33s), standbys: x 2022-09-29T05:30:12.289 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: Health detail: HEALTH_WARN Telemetry requires re-opt-in 2022-09-29T05:30:12.289 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: [WRN] TELEMETRY_CHANGED: Telemetry requires re-opt-in 2022-09-29T05:30:12.289 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: telemetry module includes new collections; please re-opt-in to new collections with `ceph telemetry on` 2022-09-29T05:30:12.290 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='' 2022-09-29T05:30:12.290 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:12.290 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:30:12.290 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:30:12.290 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:30:12.291 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:12.291 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:30:12.292 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:30:12.292 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:30:12.292 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: pgmap v12: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:12.293 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: pgmap v13: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:12.293 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:12.293 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:12.293 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: pgmap v14: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:12.294 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:12.294 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:12.294 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: overall HEALTH_WARN Telemetry requires re-opt-in 2022-09-29T05:30:12.294 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: pgmap v15: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:12.294 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: Detected new or changed devices on smithi083 2022-09-29T05:30:12.295 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:12.295 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:12.295 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:12.295 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:30:12.296 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:30:12.296 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2022-09-29T05:30:12.296 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2022-09-29T05:30:12.296 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: Upgrade: It appears safe to stop mon.b 2022-09-29T05:30:12.297 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: pgmap v16: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:12.297 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: Upgrade: Updating mon.b 2022-09-29T05:30:12.297 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:12.297 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2022-09-29T05:30:12.297 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2022-09-29T05:30:12.298 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:30:12.298 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: Deploying daemon mon.b on smithi153 2022-09-29T05:30:12.298 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: pgmap v17: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:12.298 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2022-09-29T05:30:12.299 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2022-09-29T05:30:12.299 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2022-09-29T05:30:12.299 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: Health detail: HEALTH_WARN Telemetry requires re-opt-in 2022-09-29T05:30:12.299 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: [WRN] TELEMETRY_CHANGED: Telemetry requires re-opt-in 2022-09-29T05:30:12.300 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: telemetry module includes new collections; please re-opt-in to new collections with `ceph telemetry on` 2022-09-29T05:30:12.300 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2022-09-29T05:30:12.300 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: mon.a calling monitor election 2022-09-29T05:30:12.300 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: mon.c calling monitor election 2022-09-29T05:30:12.300 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: mon.b calling monitor election 2022-09-29T05:30:12.301 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2022-09-29T05:30:12.301 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2022-09-29T05:30:12.301 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: pgmap v18: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:12.301 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: pgmap v19: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:12.302 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: pgmap v20: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:12.302 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2022-09-29T05:30:12.302 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: Health detail: HEALTH_WARN Telemetry requires re-opt-in 2022-09-29T05:30:12.302 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: [WRN] TELEMETRY_CHANGED: Telemetry requires re-opt-in 2022-09-29T05:30:12.303 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: telemetry module includes new collections; please re-opt-in to new collections with `ceph telemetry on` 2022-09-29T05:30:12.303 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: mon.c calling monitor election 2022-09-29T05:30:12.303 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: mon.a calling monitor election 2022-09-29T05:30:12.303 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2022-09-29T05:30:12.303 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: monmap e4: 3 mons at {a=[v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0],b=[v2:172.21.15.153:3300/0,v1:172.21.15.153:6789/0],c=[v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0]} 2022-09-29T05:30:12.304 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: fsmap cephfs:1 {0=b=up:active} 1 up:standby 2022-09-29T05:30:12.304 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: osdmap e64: 8 total, 8 up, 8 in 2022-09-29T05:30:12.304 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: mgrmap e35: y(active, since 33s), standbys: x 2022-09-29T05:30:12.304 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: Health detail: HEALTH_WARN Telemetry requires re-opt-in 2022-09-29T05:30:12.305 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: [WRN] TELEMETRY_CHANGED: Telemetry requires re-opt-in 2022-09-29T05:30:12.305 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: telemetry module includes new collections; please re-opt-in to new collections with `ceph telemetry on` 2022-09-29T05:30:12.305 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='' 2022-09-29T05:30:12.305 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:12.305 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:30:12.306 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:30:12.306 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:12 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:30:12.559 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:30:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:30:12.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: mkfs 69111522-3fb6-11ed-8431-001a4aab830c 2022-09-29T05:30:12.561 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: Upgrade: Need to upgrade myself (mgr.x) 2022-09-29T05:30:12.561 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: Upgrade: Need to upgrade myself (mgr.x) 2022-09-29T05:30:12.562 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: Upgrade: Updating mgr.y 2022-09-29T05:30:12.562 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: Deploying daemon mgr.y on smithi083 2022-09-29T05:30:12.562 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: Detected new or changed devices on smithi083 2022-09-29T05:30:12.563 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: Upgrade: Need to upgrade myself (mgr.x) 2022-09-29T05:30:12.563 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: Upgrade: Need to upgrade myself (mgr.x) 2022-09-29T05:30:12.563 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: Failing over to other MGR 2022-09-29T05:30:12.564 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: [29/Sep/2022:05:28:58] ENGINE Bus STARTING 2022-09-29T05:30:12.564 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: [29/Sep/2022:05:28:58] ENGINE Serving on https://172.21.15.83:7150 2022-09-29T05:30:12.564 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: [29/Sep/2022:05:28:59] ENGINE Serving on https://:::8765 2022-09-29T05:30:12.564 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: [29/Sep/2022:05:28:59] ENGINE Bus STARTED 2022-09-29T05:30:12.565 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: Updating smithi153:/etc/ceph/ceph.conf 2022-09-29T05:30:12.565 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: Updating smithi153:/var/lib/ceph/69111522-3fb6-11ed-8431-001a4aab830c/config/ceph.conf 2022-09-29T05:30:12.565 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: Updating smithi153:/etc/ceph/ceph.client.admin.keyring 2022-09-29T05:30:12.565 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: Updating smithi153:/var/lib/ceph/69111522-3fb6-11ed-8431-001a4aab830c/config/ceph.client.admin.keyring 2022-09-29T05:30:12.566 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: Detected new or changed devices on smithi083 2022-09-29T05:30:12.566 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: Updating smithi083:/etc/ceph/ceph.conf 2022-09-29T05:30:12.566 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: Updating smithi083:/var/lib/ceph/69111522-3fb6-11ed-8431-001a4aab830c/config/ceph.conf 2022-09-29T05:30:12.567 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: Updating smithi083:/etc/ceph/ceph.client.admin.keyring 2022-09-29T05:30:12.567 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: Updating smithi083:/var/lib/ceph/69111522-3fb6-11ed-8431-001a4aab830c/config/ceph.client.admin.keyring 2022-09-29T05:30:12.567 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: Reconfiguring prometheus.a (dependencies changed)... 2022-09-29T05:30:12.568 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: Reconfiguring daemon prometheus.a on smithi153 2022-09-29T05:30:12.568 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: Upgrade: Updating mgr.x 2022-09-29T05:30:12.568 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: Deploying daemon mgr.x on smithi153 2022-09-29T05:30:12.569 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: Detected new or changed devices on smithi153 2022-09-29T05:30:12.569 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: Upgrade: Setting container_image for all mgr 2022-09-29T05:30:12.569 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: Upgrade: It appears safe to stop mon.a 2022-09-29T05:30:12.569 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: Upgrade: Updating mon.a 2022-09-29T05:30:12.570 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: Deploying daemon mon.a on smithi083 2022-09-29T05:30:12.570 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: pgmap v20: 105 pgs: 105 active+clean; 582 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:12.570 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: pgmap v21: 105 pgs: 105 active+clean; 582 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:12.570 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: mon.a calling monitor election 2022-09-29T05:30:12.571 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2022-09-29T05:30:12.571 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: monmap e3: 3 mons at {a=[v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0],b=[v2:172.21.15.153:3300/0,v1:172.21.15.153:6789/0],c=[v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0]} 2022-09-29T05:30:12.571 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: fsmap cephfs:1 {0=b=up:active} 1 up:standby 2022-09-29T05:30:12.571 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: osdmap e63: 8 total, 8 up, 8 in 2022-09-29T05:30:12.572 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: mgrmap e31: y(active, since 35s), standbys: x 2022-09-29T05:30:12.572 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: mgrmap e32: y(active, since 35s), standbys: x 2022-09-29T05:30:12.572 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: Health detail: HEALTH_WARN Telemetry requires re-opt-in 2022-09-29T05:30:12.572 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: [WRN] TELEMETRY_CHANGED: Telemetry requires re-opt-in 2022-09-29T05:30:12.572 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: telemetry module includes new collections; please re-opt-in to new collections with `ceph telemetry on` 2022-09-29T05:30:12.573 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: Active manager daemon y restarted 2022-09-29T05:30:12.573 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: Activating manager daemon y 2022-09-29T05:30:12.573 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: osdmap e64: 8 total, 8 up, 8 in 2022-09-29T05:30:12.573 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: Standby manager daemon x restarted 2022-09-29T05:30:12.574 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: Standby manager daemon x started 2022-09-29T05:30:12.574 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: mgrmap e33: y(active, starting, since 0.0570875s), standbys: x 2022-09-29T05:30:12.574 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: Manager daemon y is now available 2022-09-29T05:30:12.574 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: [29/Sep/2022:05:29:39] ENGINE Bus STARTING 2022-09-29T05:30:12.574 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: mgrmap e34: y(active, since 1.07338s), standbys: x 2022-09-29T05:30:12.575 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: pgmap v3: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:12.575 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: [29/Sep/2022:05:29:39] ENGINE Serving on https://:::8765 2022-09-29T05:30:12.575 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: [29/Sep/2022:05:29:40] ENGINE Serving on https://172.21.15.83:7150 2022-09-29T05:30:12.575 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: [29/Sep/2022:05:29:40] ENGINE Bus STARTED 2022-09-29T05:30:12.576 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: pgmap v4: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:12.576 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: Health check cleared: TELEMETRY_CHANGED (was: Telemetry requires re-opt-in) 2022-09-29T05:30:12.576 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: Cluster is now healthy 2022-09-29T05:30:12.576 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: mgrmap e35: y(active, since 2s), standbys: x 2022-09-29T05:30:12.576 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: pgmap v5: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:12.577 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:12.577 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:12.577 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:12.577 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:12.578 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: Updating smithi083:/etc/ceph/ceph.conf 2022-09-29T05:30:12.578 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: pgmap v6: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:12.578 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: Updating smithi083:/var/lib/ceph/69111522-3fb6-11ed-8431-001a4aab830c/config/ceph.conf 2022-09-29T05:30:12.578 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: Updating smithi083:/etc/ceph/ceph.client.admin.keyring 2022-09-29T05:30:12.578 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:12.579 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:12.579 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: Updating smithi083:/var/lib/ceph/69111522-3fb6-11ed-8431-001a4aab830c/config/ceph.client.admin.keyring 2022-09-29T05:30:12.579 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:12.579 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:12.580 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: pgmap v7: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:12.580 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: Detected new or changed devices on smithi153 2022-09-29T05:30:12.580 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:12.580 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:12.581 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: Updating smithi153:/etc/ceph/ceph.conf 2022-09-29T05:30:12.581 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: Updating smithi153:/var/lib/ceph/69111522-3fb6-11ed-8431-001a4aab830c/config/ceph.conf 2022-09-29T05:30:12.581 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: Updating smithi153:/etc/ceph/ceph.client.admin.keyring 2022-09-29T05:30:12.581 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: pgmap v8: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:12.581 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: Updating smithi153:/var/lib/ceph/69111522-3fb6-11ed-8431-001a4aab830c/config/ceph.client.admin.keyring 2022-09-29T05:30:12.582 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:12.582 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:12.582 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:12.583 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:30:12.583 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:30:12.583 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2022-09-29T05:30:12.583 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2022-09-29T05:30:12.584 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: Upgrade: It appears safe to stop mon.c 2022-09-29T05:30:12.584 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: pgmap v9: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:12.584 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: Health check failed: Telemetry requires re-opt-in (TELEMETRY_CHANGED) 2022-09-29T05:30:12.585 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: Upgrade: Updating mon.c 2022-09-29T05:30:12.585 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:12.585 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2022-09-29T05:30:12.585 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2022-09-29T05:30:12.586 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:30:12.586 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: Deploying daemon mon.c on smithi083 2022-09-29T05:30:12.586 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: pgmap v10: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:12.586 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: from='client.25048 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:30:12.586 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: from='client.25054 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:30:12.587 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4148291325' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:30:12.587 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2022-09-29T05:30:12.587 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2022-09-29T05:30:12.587 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2022-09-29T05:30:12.588 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: pgmap v11: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:12.588 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: mon.c calling monitor election 2022-09-29T05:30:12.588 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: mon.a calling monitor election 2022-09-29T05:30:12.588 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2022-09-29T05:30:12.589 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: monmap e3: 3 mons at {a=[v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0],b=[v2:172.21.15.153:3300/0,v1:172.21.15.153:6789/0],c=[v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0]} 2022-09-29T05:30:12.589 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: fsmap cephfs:1 {0=b=up:active} 1 up:standby 2022-09-29T05:30:12.589 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: osdmap e64: 8 total, 8 up, 8 in 2022-09-29T05:30:12.589 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: mgrmap e35: y(active, since 15s), standbys: x 2022-09-29T05:30:12.589 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: Health detail: HEALTH_WARN Telemetry requires re-opt-in 2022-09-29T05:30:12.590 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: [WRN] TELEMETRY_CHANGED: Telemetry requires re-opt-in 2022-09-29T05:30:12.590 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: telemetry module includes new collections; please re-opt-in to new collections with `ceph telemetry on` 2022-09-29T05:30:12.590 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:12.590 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:12.591 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:30:12.591 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:30:12.591 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:30:12.591 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: pgmap v12: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:12.592 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: pgmap v13: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:12.592 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:12.592 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:12.592 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: pgmap v14: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:12.593 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:12.593 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:12.593 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: overall HEALTH_WARN Telemetry requires re-opt-in 2022-09-29T05:30:12.593 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: pgmap v15: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:12.593 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: Detected new or changed devices on smithi083 2022-09-29T05:30:12.594 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:12.594 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:12.594 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:12.595 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:30:12.595 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:30:12.595 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2022-09-29T05:30:12.595 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2022-09-29T05:30:12.596 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: Upgrade: It appears safe to stop mon.b 2022-09-29T05:30:12.596 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: pgmap v16: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:12.596 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: Upgrade: Updating mon.b 2022-09-29T05:30:12.596 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:12.596 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2022-09-29T05:30:12.597 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2022-09-29T05:30:12.597 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:30:12.597 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: Deploying daemon mon.b on smithi153 2022-09-29T05:30:12.597 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: pgmap v17: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:12.598 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2022-09-29T05:30:12.598 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2022-09-29T05:30:12.598 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2022-09-29T05:30:12.598 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: Health detail: HEALTH_WARN Telemetry requires re-opt-in 2022-09-29T05:30:12.598 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: [WRN] TELEMETRY_CHANGED: Telemetry requires re-opt-in 2022-09-29T05:30:12.599 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: telemetry module includes new collections; please re-opt-in to new collections with `ceph telemetry on` 2022-09-29T05:30:12.599 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2022-09-29T05:30:12.599 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: mon.a calling monitor election 2022-09-29T05:30:12.599 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: mon.c calling monitor election 2022-09-29T05:30:12.600 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: mon.b calling monitor election 2022-09-29T05:30:12.600 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2022-09-29T05:30:12.600 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2022-09-29T05:30:12.600 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: pgmap v18: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:12.601 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: pgmap v19: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:12.601 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: pgmap v20: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:12.601 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2022-09-29T05:30:12.602 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: Health detail: HEALTH_WARN Telemetry requires re-opt-in 2022-09-29T05:30:12.602 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: [WRN] TELEMETRY_CHANGED: Telemetry requires re-opt-in 2022-09-29T05:30:12.602 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: telemetry module includes new collections; please re-opt-in to new collections with `ceph telemetry on` 2022-09-29T05:30:12.602 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: mon.c calling monitor election 2022-09-29T05:30:12.602 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: mon.a calling monitor election 2022-09-29T05:30:12.603 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2022-09-29T05:30:12.603 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: monmap e4: 3 mons at {a=[v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0],b=[v2:172.21.15.153:3300/0,v1:172.21.15.153:6789/0],c=[v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0]} 2022-09-29T05:30:12.603 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: fsmap cephfs:1 {0=b=up:active} 1 up:standby 2022-09-29T05:30:12.603 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: osdmap e64: 8 total, 8 up, 8 in 2022-09-29T05:30:12.604 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: mgrmap e35: y(active, since 33s), standbys: x 2022-09-29T05:30:12.604 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: Health detail: HEALTH_WARN Telemetry requires re-opt-in 2022-09-29T05:30:12.604 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: [WRN] TELEMETRY_CHANGED: Telemetry requires re-opt-in 2022-09-29T05:30:12.604 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: telemetry module includes new collections; please re-opt-in to new collections with `ceph telemetry on` 2022-09-29T05:30:12.604 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='' 2022-09-29T05:30:12.605 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:12.605 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:30:12.605 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:30:12.605 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:12 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:30:13.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:30:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:30:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:30:14.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:14 smithi083 ceph-mon[79930]: pgmap v21: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:14.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:14 smithi083 ceph-mon[77242]: pgmap v21: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:14.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:14 smithi153 ceph-mon[70044]: pgmap v21: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:16.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:16 smithi083 ceph-mon[77242]: pgmap v22: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:16.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:16 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:16.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:16 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:16.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:16 smithi083 ceph-mon[79930]: pgmap v22: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:16.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:16 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:16.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:16 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:17.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:16 smithi153 ceph-mon[70044]: pgmap v22: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:17.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:16 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:17.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:16 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:18.664 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:18 smithi153 ceph-mon[70044]: pgmap v23: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:18.664 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:18 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:18.664 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:18 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:18.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:18 smithi083 ceph-mon[79930]: pgmap v23: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:18.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:18 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:18.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:18 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:18.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:18 smithi083 ceph-mon[77242]: pgmap v23: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:18.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:18 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:18.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:18 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:18.997 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:30:18 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:30:18.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi153.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi083:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:30:18.998 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:30:18 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:30:18.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephPGImbalance\nexpr: abs(((ceph_osd_numpg > 0) - on(job) group_left() avg by(job) (ceph_osd_numpg\n > 0)) / on(job) group_left() avg by(job) (ceph_osd_numpg > 0)) * on(ceph_daemon)\n group_left(hostname) ceph_osd_metadata > 0.3\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.5\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} deviates by\n more than 30% from average PG count.\n summary: PGs are not balanced across OSDs\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi153.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi083:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:30:20.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:20 smithi153 ceph-mon[70044]: pgmap v24: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:20.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:20 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:20.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:20 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:20.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:20 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:20.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:20 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2022-09-29T05:30:20.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:20 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2022-09-29T05:30:20.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:20 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:30:20.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:20 smithi083 ceph-mon[77242]: pgmap v24: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:20.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:20 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:20.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:20 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:20.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:20 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:20.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:20 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2022-09-29T05:30:20.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:20 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2022-09-29T05:30:20.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:20 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:30:20.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:20 smithi083 ceph-mon[79930]: pgmap v24: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:20.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:20 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:20.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:20 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:20.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:20 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:20.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:20 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2022-09-29T05:30:20.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:20 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2022-09-29T05:30:20.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:20 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:30:21.761 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:21 smithi083 ceph-mon[79930]: Detected new or changed devices on smithi153 2022-09-29T05:30:21.761 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:21 smithi083 ceph-mon[79930]: Reconfiguring mon.a (monmap changed)... 2022-09-29T05:30:21.761 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:21 smithi083 ceph-mon[79930]: Reconfiguring daemon mon.a on smithi083 2022-09-29T05:30:21.762 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:30:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:30:21] "GET /metrics HTTP/1.1" 200 170063 "" "Prometheus/2.33.4" 2022-09-29T05:30:21.762 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:21 smithi083 ceph-mon[77242]: Detected new or changed devices on smithi153 2022-09-29T05:30:21.763 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:21 smithi083 ceph-mon[77242]: Reconfiguring mon.a (monmap changed)... 2022-09-29T05:30:21.763 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:21 smithi083 ceph-mon[77242]: Reconfiguring daemon mon.a on smithi083 2022-09-29T05:30:22.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:21 smithi153 ceph-mon[70044]: Detected new or changed devices on smithi153 2022-09-29T05:30:22.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:21 smithi153 ceph-mon[70044]: Reconfiguring mon.a (monmap changed)... 2022-09-29T05:30:22.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:21 smithi153 ceph-mon[70044]: Reconfiguring daemon mon.a on smithi083 2022-09-29T05:30:22.217 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T05:30:22.519 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T05:30:22.520 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4m) 24s ago 4m 26.4M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T05:30:22.520 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4m) 5s ago 4m 60.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T05:30:22.520 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5m) 24s ago 5m 15.2M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T05:30:22.520 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5m) 5s ago 5m 18.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T05:30:22.521 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (67s) 5s ago 7m 427M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T05:30:22.521 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (94s) 24s ago 9m 475M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T05:30:22.521 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (53s) 24s ago 9m 46.3M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T05:30:22.521 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (15s) 5s ago 8m 30.0M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T05:30:22.522 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (28s) 24s ago 8m 25.8M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T05:30:22.522 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4m) 24s ago 4m 24.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T05:30:22.522 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4m) 5s ago 4m 24.7M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T05:30:22.522 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7m) 24s ago 7m 67.1M 2500M 16.2.10 c56e13ff90f2 c12a21fd5897 2022-09-29T05:30:22.523 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7m) 24s ago 7m 61.7M 2500M 16.2.10 c56e13ff90f2 989e877de30a 2022-09-29T05:30:22.523 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7m) 24s ago 7m 55.4M 2500M 16.2.10 c56e13ff90f2 81b9b51f0eee 2022-09-29T05:30:22.523 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6m) 24s ago 6m 62.2M 2500M 16.2.10 c56e13ff90f2 4863a2083356 2022-09-29T05:30:22.523 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6m) 5s ago 6m 58.3M 2500M 16.2.10 c56e13ff90f2 ed61dc9353b0 2022-09-29T05:30:22.524 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6m) 5s ago 6m 66.5M 2500M 16.2.10 c56e13ff90f2 59f19fd15fed 2022-09-29T05:30:22.524 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5m) 5s ago 5m 68.0M 2500M 16.2.10 c56e13ff90f2 23decfd9c6b2 2022-09-29T05:30:22.524 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5m) 5s ago 5m 72.0M 2500M 16.2.10 c56e13ff90f2 23460a09c0a7 2022-09-29T05:30:22.524 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (71s) 5s ago 4m 66.9M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T05:30:22.634 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:30:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:30:22.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:30:22.893 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:22 smithi083 ceph-mon[79930]: pgmap v25: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:22.894 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:22 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:22.894 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:22 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:22.894 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:22 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2022-09-29T05:30:22.894 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:22 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2022-09-29T05:30:22.895 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:22 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2022-09-29T05:30:22.895 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:22 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:30:22.895 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:22 smithi083 ceph-mon[77242]: pgmap v25: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:22.896 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:22 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:22.896 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:22 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:22.896 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:22 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2022-09-29T05:30:22.896 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:22 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2022-09-29T05:30:22.897 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:22 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2022-09-29T05:30:22.897 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:22 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:30:22.900 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T05:30:22.901 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T05:30:22.901 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T05:30:22.901 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:30:22.901 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T05:30:22.902 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T05:30:22.902 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:30:22.902 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T05:30:22.902 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 8 2022-09-29T05:30:22.902 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:30:22.903 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T05:30:22.903 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T05:30:22.903 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:30:22.903 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T05:30:22.904 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 10, 2022-09-29T05:30:22.904 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 5 2022-09-29T05:30:22.904 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T05:30:22.904 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T05:30:23.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:22 smithi153 ceph-mon[70044]: pgmap v25: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:23.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:22 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:23.026 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:22 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:23.026 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:22 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2022-09-29T05:30:23.026 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:22 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2022-09-29T05:30:23.026 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:22 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2022-09-29T05:30:23.027 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:22 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:30:23.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:30:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:30:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:30:23.919 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:23 smithi083 ceph-mon[79930]: Reconfiguring mgr.y (monmap changed)... 2022-09-29T05:30:23.920 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:23 smithi083 ceph-mon[79930]: Reconfiguring daemon mgr.y on smithi083 2022-09-29T05:30:23.920 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:23 smithi083 ceph-mon[79930]: from='client.44101 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:30:23.920 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:23 smithi083 ceph-mon[79930]: from='client.34173 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:30:23.920 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:23 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2183909592' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:30:23.921 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:23 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:23.921 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:23 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:23.921 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:23 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2022-09-29T05:30:23.921 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:23 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2022-09-29T05:30:23.921 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:23 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:30:23.922 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:23 smithi083 ceph-mon[77242]: Reconfiguring mgr.y (monmap changed)... 2022-09-29T05:30:23.922 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:23 smithi083 ceph-mon[77242]: Reconfiguring daemon mgr.y on smithi083 2022-09-29T05:30:23.923 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:23 smithi083 ceph-mon[77242]: from='client.44101 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:30:23.923 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:23 smithi083 ceph-mon[77242]: from='client.34173 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:30:23.923 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:23 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2183909592' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:30:23.923 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:23 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:23.924 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:23 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:23.924 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:23 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2022-09-29T05:30:23.924 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:23 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2022-09-29T05:30:23.924 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:23 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:30:24.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:23 smithi153 ceph-mon[70044]: Reconfiguring mgr.y (monmap changed)... 2022-09-29T05:30:24.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:23 smithi153 ceph-mon[70044]: Reconfiguring daemon mgr.y on smithi083 2022-09-29T05:30:24.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:23 smithi153 ceph-mon[70044]: from='client.44101 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:30:24.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:23 smithi153 ceph-mon[70044]: from='client.34173 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:30:24.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:23 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2183909592' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:30:24.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:23 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:24.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:23 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:24.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:23 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2022-09-29T05:30:24.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:23 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2022-09-29T05:30:24.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:23 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:30:24.647 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:24 smithi083 ceph-mon[77242]: pgmap v26: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:24.648 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:24 smithi083 ceph-mon[77242]: Reconfiguring mon.c (monmap changed)... 2022-09-29T05:30:24.648 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:24 smithi083 ceph-mon[77242]: Reconfiguring daemon mon.c on smithi083 2022-09-29T05:30:24.648 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:24 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:24.648 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:24 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:24.649 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:24 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2022-09-29T05:30:24.649 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:24 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:30:24.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:24 smithi083 ceph-mon[79930]: pgmap v26: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:24.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:24 smithi083 ceph-mon[79930]: Reconfiguring mon.c (monmap changed)... 2022-09-29T05:30:24.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:24 smithi083 ceph-mon[79930]: Reconfiguring daemon mon.c on smithi083 2022-09-29T05:30:24.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:24 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:24.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:24 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:24.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:24 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2022-09-29T05:30:24.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:24 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:30:25.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:24 smithi153 ceph-mon[70044]: pgmap v26: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:25.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:24 smithi153 ceph-mon[70044]: Reconfiguring mon.c (monmap changed)... 2022-09-29T05:30:25.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:24 smithi153 ceph-mon[70044]: Reconfiguring daemon mon.c on smithi083 2022-09-29T05:30:25.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:24 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:25.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:24 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:25.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:24 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2022-09-29T05:30:25.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:24 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:30:25.878 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:25 smithi083 ceph-mon[79930]: Reconfiguring osd.0 (monmap changed)... 2022-09-29T05:30:25.878 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:25 smithi083 ceph-mon[79930]: Reconfiguring daemon osd.0 on smithi083 2022-09-29T05:30:25.879 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:25 smithi083 ceph-mon[77242]: Reconfiguring osd.0 (monmap changed)... 2022-09-29T05:30:25.879 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:25 smithi083 ceph-mon[77242]: Reconfiguring daemon osd.0 on smithi083 2022-09-29T05:30:26.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:25 smithi153 ceph-mon[70044]: Reconfiguring osd.0 (monmap changed)... 2022-09-29T05:30:26.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:25 smithi153 ceph-mon[70044]: Reconfiguring daemon osd.0 on smithi083 2022-09-29T05:30:26.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:26 smithi083 ceph-mon[77242]: pgmap v27: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:26.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:26 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:26.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:26 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:26.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:26 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2022-09-29T05:30:26.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:26 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:30:26.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:26 smithi083 ceph-mon[79930]: pgmap v27: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:26.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:26 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:26.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:26 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:26.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:26 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2022-09-29T05:30:26.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:26 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:30:27.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:26 smithi153 ceph-mon[70044]: pgmap v27: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:27.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:26 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:27.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:26 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:27.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:26 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2022-09-29T05:30:27.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:26 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:30:27.931 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:27 smithi083 ceph-mon[79930]: Reconfiguring osd.1 (monmap changed)... 2022-09-29T05:30:27.932 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:27 smithi083 ceph-mon[79930]: Reconfiguring daemon osd.1 on smithi083 2022-09-29T05:30:27.932 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:27 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:27.932 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:27 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:27.933 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:27 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2022-09-29T05:30:27.933 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:27 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:30:27.933 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:27 smithi083 ceph-mon[77242]: Reconfiguring osd.1 (monmap changed)... 2022-09-29T05:30:27.933 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:27 smithi083 ceph-mon[77242]: Reconfiguring daemon osd.1 on smithi083 2022-09-29T05:30:27.934 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:27 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:27.934 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:27 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:27.935 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:27 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2022-09-29T05:30:27.935 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:27 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:30:28.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:27 smithi153 ceph-mon[70044]: Reconfiguring osd.1 (monmap changed)... 2022-09-29T05:30:28.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:27 smithi153 ceph-mon[70044]: Reconfiguring daemon osd.1 on smithi083 2022-09-29T05:30:28.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:27 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:28.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:27 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:28.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:27 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2022-09-29T05:30:28.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:27 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:30:28.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:28 smithi083 ceph-mon[77242]: pgmap v28: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:28.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:28 smithi083 ceph-mon[77242]: Reconfiguring osd.2 (monmap changed)... 2022-09-29T05:30:28.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:28 smithi083 ceph-mon[77242]: Reconfiguring daemon osd.2 on smithi083 2022-09-29T05:30:28.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:28 smithi083 ceph-mon[79930]: pgmap v28: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:28.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:28 smithi083 ceph-mon[79930]: Reconfiguring osd.2 (monmap changed)... 2022-09-29T05:30:28.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:28 smithi083 ceph-mon[79930]: Reconfiguring daemon osd.2 on smithi083 2022-09-29T05:30:29.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:28 smithi153 ceph-mon[70044]: pgmap v28: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:29.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:28 smithi153 ceph-mon[70044]: Reconfiguring osd.2 (monmap changed)... 2022-09-29T05:30:29.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:28 smithi153 ceph-mon[70044]: Reconfiguring daemon osd.2 on smithi083 2022-09-29T05:30:29.060 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:30:28 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:30:28.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi153.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi083:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:30:29.060 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:30:28 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:30:28.662Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephPGImbalance\nexpr: abs(((ceph_osd_numpg > 0) - on(job) group_left() avg by(job) (ceph_osd_numpg\n > 0)) / on(job) group_left() avg by(job) (ceph_osd_numpg > 0)) * on(ceph_daemon)\n group_left(hostname) ceph_osd_metadata > 0.3\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.5\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} deviates by\n more than 30% from average PG count.\n summary: PGs are not balanced across OSDs\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi153.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi083:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:30:29.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:29 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:29.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:29 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:29.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:29 smithi083 ceph-mon[77242]: Reconfiguring osd.3 (monmap changed)... 2022-09-29T05:30:29.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:29 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2022-09-29T05:30:29.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:29 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:30:29.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:29 smithi083 ceph-mon[77242]: Reconfiguring daemon osd.3 on smithi083 2022-09-29T05:30:29.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:29 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:29.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:29 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:29.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:29 smithi083 ceph-mon[79930]: Reconfiguring osd.3 (monmap changed)... 2022-09-29T05:30:29.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:29 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2022-09-29T05:30:29.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:29 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:30:29.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:29 smithi083 ceph-mon[79930]: Reconfiguring daemon osd.3 on smithi083 2022-09-29T05:30:30.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:29 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:30.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:29 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:30.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:29 smithi153 ceph-mon[70044]: Reconfiguring osd.3 (monmap changed)... 2022-09-29T05:30:30.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:29 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2022-09-29T05:30:30.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:29 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:30:30.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:29 smithi153 ceph-mon[70044]: Reconfiguring daemon osd.3 on smithi083 2022-09-29T05:30:30.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:30 smithi083 ceph-mon[79930]: pgmap v29: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:30.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:30 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:30.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:30 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:30.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:30 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2022-09-29T05:30:30.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:30 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2022-09-29T05:30:30.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:30 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:30:30.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:30 smithi083 ceph-mon[77242]: pgmap v29: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:30.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:30 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:30.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:30 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:30.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:30 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2022-09-29T05:30:30.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:30 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2022-09-29T05:30:30.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:30 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:30:31.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:30 smithi153 ceph-mon[70044]: pgmap v29: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:31.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:30 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:31.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:30 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:31.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:30 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2022-09-29T05:30:31.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:30 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2022-09-29T05:30:31.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:30 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:30:31.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:30:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:30:31] "GET /metrics HTTP/1.1" 200 170148 "" "Prometheus/2.33.4" 2022-09-29T05:30:32.031 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:31 smithi153 ceph-mon[70044]: Reconfiguring mds.a (monmap changed)... 2022-09-29T05:30:32.031 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:31 smithi153 ceph-mon[70044]: Reconfiguring daemon mds.a on smithi083 2022-09-29T05:30:32.031 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:31 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:32.032 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:31 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:32.032 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:31 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2022-09-29T05:30:32.032 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:31 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2022-09-29T05:30:32.032 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:31 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:30:32.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:31 smithi083 ceph-mon[77242]: Reconfiguring mds.a (monmap changed)... 2022-09-29T05:30:32.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:31 smithi083 ceph-mon[77242]: Reconfiguring daemon mds.a on smithi083 2022-09-29T05:30:32.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:31 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:32.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:31 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:32.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:31 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2022-09-29T05:30:32.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:31 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2022-09-29T05:30:32.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:31 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:30:32.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:31 smithi083 ceph-mon[79930]: Reconfiguring mds.a (monmap changed)... 2022-09-29T05:30:32.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:31 smithi083 ceph-mon[79930]: Reconfiguring daemon mds.a on smithi083 2022-09-29T05:30:32.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:31 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:32.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:31 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:32.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:31 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2022-09-29T05:30:32.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:31 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2022-09-29T05:30:32.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:31 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:30:32.715 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:30:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:30:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:30:33.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:32 smithi153 ceph-mon[70044]: pgmap v30: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:33.026 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:32 smithi153 ceph-mon[70044]: Reconfiguring mon.b (monmap changed)... 2022-09-29T05:30:33.026 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:32 smithi153 ceph-mon[70044]: Reconfiguring daemon mon.b on smithi153 2022-09-29T05:30:33.026 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:32 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:33.026 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:32 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:33.027 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:32 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2022-09-29T05:30:33.027 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:32 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2022-09-29T05:30:33.027 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:32 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2022-09-29T05:30:33.027 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:32 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:30:33.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:32 smithi083 ceph-mon[79930]: pgmap v30: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:33.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:32 smithi083 ceph-mon[79930]: Reconfiguring mon.b (monmap changed)... 2022-09-29T05:30:33.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:32 smithi083 ceph-mon[79930]: Reconfiguring daemon mon.b on smithi153 2022-09-29T05:30:33.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:32 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:33.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:32 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:33.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:32 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2022-09-29T05:30:33.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:32 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2022-09-29T05:30:33.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:32 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2022-09-29T05:30:33.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:32 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:30:33.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:32 smithi083 ceph-mon[77242]: pgmap v30: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:33.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:32 smithi083 ceph-mon[77242]: Reconfiguring mon.b (monmap changed)... 2022-09-29T05:30:33.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:32 smithi083 ceph-mon[77242]: Reconfiguring daemon mon.b on smithi153 2022-09-29T05:30:33.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:32 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:33.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:32 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:33.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:32 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2022-09-29T05:30:33.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:32 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2022-09-29T05:30:33.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:32 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2022-09-29T05:30:33.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:32 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:30:33.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:30:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:30:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:30:34.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:33 smithi153 ceph-mon[70044]: Reconfiguring mgr.x (monmap changed)... 2022-09-29T05:30:34.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:33 smithi153 ceph-mon[70044]: Reconfiguring daemon mgr.x on smithi153 2022-09-29T05:30:34.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:33 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:34.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:33 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:34.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:33 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2022-09-29T05:30:34.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:33 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:30:34.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:33 smithi083 ceph-mon[79930]: Reconfiguring mgr.x (monmap changed)... 2022-09-29T05:30:34.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:33 smithi083 ceph-mon[79930]: Reconfiguring daemon mgr.x on smithi153 2022-09-29T05:30:34.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:33 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:34.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:33 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:34.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:33 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2022-09-29T05:30:34.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:33 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:30:34.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:33 smithi083 ceph-mon[77242]: Reconfiguring mgr.x (monmap changed)... 2022-09-29T05:30:34.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:33 smithi083 ceph-mon[77242]: Reconfiguring daemon mgr.x on smithi153 2022-09-29T05:30:34.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:33 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:34.218 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:33 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:34.218 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:33 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2022-09-29T05:30:34.219 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:33 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:30:35.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:34 smithi153 ceph-mon[70044]: pgmap v31: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:35.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:34 smithi153 ceph-mon[70044]: Reconfiguring osd.4 (monmap changed)... 2022-09-29T05:30:35.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:34 smithi153 ceph-mon[70044]: Reconfiguring daemon osd.4 on smithi153 2022-09-29T05:30:35.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:34 smithi083 ceph-mon[79930]: pgmap v31: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:35.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:34 smithi083 ceph-mon[79930]: Reconfiguring osd.4 (monmap changed)... 2022-09-29T05:30:35.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:34 smithi083 ceph-mon[79930]: Reconfiguring daemon osd.4 on smithi153 2022-09-29T05:30:35.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:34 smithi083 ceph-mon[77242]: pgmap v31: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:35.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:34 smithi083 ceph-mon[77242]: Reconfiguring osd.4 (monmap changed)... 2022-09-29T05:30:35.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:34 smithi083 ceph-mon[77242]: Reconfiguring daemon osd.4 on smithi153 2022-09-29T05:30:36.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:35 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:36.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:35 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:36.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:35 smithi153 ceph-mon[70044]: Reconfiguring osd.5 (monmap changed)... 2022-09-29T05:30:36.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:35 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2022-09-29T05:30:36.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:35 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:30:36.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:35 smithi153 ceph-mon[70044]: Reconfiguring daemon osd.5 on smithi153 2022-09-29T05:30:36.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:35 smithi153 ceph-mon[70044]: pgmap v32: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:36.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:35 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:36.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:35 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:36.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:35 smithi083 ceph-mon[79930]: Reconfiguring osd.5 (monmap changed)... 2022-09-29T05:30:36.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:35 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2022-09-29T05:30:36.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:35 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:30:36.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:35 smithi083 ceph-mon[79930]: Reconfiguring daemon osd.5 on smithi153 2022-09-29T05:30:36.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:35 smithi083 ceph-mon[79930]: pgmap v32: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:36.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:35 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:36.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:35 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:36.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:35 smithi083 ceph-mon[77242]: Reconfiguring osd.5 (monmap changed)... 2022-09-29T05:30:36.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:35 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2022-09-29T05:30:36.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:35 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:30:36.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:35 smithi083 ceph-mon[77242]: Reconfiguring daemon osd.5 on smithi153 2022-09-29T05:30:36.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:35 smithi083 ceph-mon[77242]: pgmap v32: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:37.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:37 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:37.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:37 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:37.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:37 smithi153 ceph-mon[70044]: Reconfiguring osd.6 (monmap changed)... 2022-09-29T05:30:37.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:37 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2022-09-29T05:30:37.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:37 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:30:37.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:37 smithi153 ceph-mon[70044]: Reconfiguring daemon osd.6 on smithi153 2022-09-29T05:30:37.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:37 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:37.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:37 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:37.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:37 smithi083 ceph-mon[79930]: Reconfiguring osd.6 (monmap changed)... 2022-09-29T05:30:37.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:37 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2022-09-29T05:30:37.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:37 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:30:37.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:37 smithi083 ceph-mon[79930]: Reconfiguring daemon osd.6 on smithi153 2022-09-29T05:30:37.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:37 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:37.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:37 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:37.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:37 smithi083 ceph-mon[77242]: Reconfiguring osd.6 (monmap changed)... 2022-09-29T05:30:37.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:37 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2022-09-29T05:30:37.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:37 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:30:37.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:37 smithi083 ceph-mon[77242]: Reconfiguring daemon osd.6 on smithi153 2022-09-29T05:30:38.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:38 smithi153 ceph-mon[70044]: pgmap v33: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:38.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:38 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:38.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:38 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:38.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:38 smithi153 ceph-mon[70044]: Reconfiguring osd.7 (monmap changed)... 2022-09-29T05:30:38.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:38 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2022-09-29T05:30:38.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:38 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:30:38.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:38 smithi153 ceph-mon[70044]: Reconfiguring daemon osd.7 on smithi153 2022-09-29T05:30:38.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:38 smithi083 ceph-mon[79930]: pgmap v33: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:38.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:38 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:38.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:38 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:38.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:38 smithi083 ceph-mon[79930]: Reconfiguring osd.7 (monmap changed)... 2022-09-29T05:30:38.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:38 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2022-09-29T05:30:38.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:38 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:30:38.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:38 smithi083 ceph-mon[79930]: Reconfiguring daemon osd.7 on smithi153 2022-09-29T05:30:38.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:38 smithi083 ceph-mon[77242]: pgmap v33: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:38.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:38 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:38.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:38 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:38.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:38 smithi083 ceph-mon[77242]: Reconfiguring osd.7 (monmap changed)... 2022-09-29T05:30:38.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:38 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2022-09-29T05:30:38.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:38 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:30:38.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:38 smithi083 ceph-mon[77242]: Reconfiguring daemon osd.7 on smithi153 2022-09-29T05:30:39.059 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:30:38 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:30:38.662Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi153.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi083:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:30:39.059 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:30:38 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:30:38.664Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephPGImbalance\nexpr: abs(((ceph_osd_numpg > 0) - on(job) group_left() avg by(job) (ceph_osd_numpg\n > 0)) / on(job) group_left() avg by(job) (ceph_osd_numpg > 0)) * on(ceph_daemon)\n group_left(hostname) ceph_osd_metadata > 0.3\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.5\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} deviates by\n more than 30% from average PG count.\n summary: PGs are not balanced across OSDs\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi153.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi083:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:30:39.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:39.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:39.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:39 smithi153 ceph-mon[70044]: Reconfiguring mds.b (monmap changed)... 2022-09-29T05:30:39.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2022-09-29T05:30:39.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2022-09-29T05:30:39.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:30:39.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:39 smithi153 ceph-mon[70044]: Reconfiguring daemon mds.b on smithi153 2022-09-29T05:30:39.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:30:39.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:30:39.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:30:39.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:30:39.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:39.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:39.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:39 smithi083 ceph-mon[79930]: Reconfiguring mds.b (monmap changed)... 2022-09-29T05:30:39.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2022-09-29T05:30:39.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2022-09-29T05:30:39.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:30:39.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:39 smithi083 ceph-mon[79930]: Reconfiguring daemon mds.b on smithi153 2022-09-29T05:30:39.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:30:39.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:30:39.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:30:39.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:30:39.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:39.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:39.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:39 smithi083 ceph-mon[77242]: Reconfiguring mds.b (monmap changed)... 2022-09-29T05:30:39.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2022-09-29T05:30:39.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2022-09-29T05:30:39.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:30:39.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:39 smithi083 ceph-mon[77242]: Reconfiguring daemon mds.b on smithi153 2022-09-29T05:30:39.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:30:39.968 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:30:39.968 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:30:39.968 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:30:40.792 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:40 smithi083 ceph-mon[79930]: pgmap v34: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:40.793 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:40.793 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:40.793 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:30:40.793 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:30:40.794 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:30:40.794 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:40.794 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2022-09-29T05:30:40.794 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2022-09-29T05:30:40.795 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2022-09-29T05:30:40.795 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2022-09-29T05:30:40.795 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2022-09-29T05:30:40.796 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2022-09-29T05:30:40.796 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2022-09-29T05:30:40.796 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2022-09-29T05:30:40.796 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2022-09-29T05:30:40.796 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:30:40.797 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:40.797 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2022-09-29T05:30:40.797 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:40 smithi083 ceph-mon[77242]: pgmap v34: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:40.798 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:40.798 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:40.798 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:30:40.798 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:30:40.799 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:30:40.799 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:40.799 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2022-09-29T05:30:40.799 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2022-09-29T05:30:40.800 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2022-09-29T05:30:40.800 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2022-09-29T05:30:40.800 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2022-09-29T05:30:40.800 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2022-09-29T05:30:40.801 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2022-09-29T05:30:40.801 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2022-09-29T05:30:40.801 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2022-09-29T05:30:40.801 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:30:40.802 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:40.802 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2022-09-29T05:30:41.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:40 smithi153 ceph-mon[70044]: pgmap v34: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:41.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:41.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:41.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:30:41.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:30:41.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:30:41.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:41.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2022-09-29T05:30:41.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2022-09-29T05:30:41.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2022-09-29T05:30:41.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2022-09-29T05:30:41.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2022-09-29T05:30:41.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2022-09-29T05:30:41.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2022-09-29T05:30:41.063 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2022-09-29T05:30:41.063 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2022-09-29T05:30:41.063 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:30:41.063 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:41.063 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2022-09-29T05:30:41.897 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:41 smithi083 ceph-mon[77242]: Upgrade: Setting container_image for all mon 2022-09-29T05:30:41.897 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:41 smithi083 ceph-mon[77242]: Upgrade: Setting container_image for all crash 2022-09-29T05:30:41.897 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:41 smithi083 ceph-mon[77242]: from='mon.1 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2022-09-29T05:30:41.898 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:41 smithi083 ceph-mon[77242]: Upgrade: osd.0 is safe to restart 2022-09-29T05:30:41.898 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:41.898 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2022-09-29T05:30:41.898 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:30:41.899 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:30:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:30:41] "GET /metrics HTTP/1.1" 200 170169 "" "Prometheus/2.33.4" 2022-09-29T05:30:41.899 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:41 smithi083 ceph-mon[79930]: Upgrade: Setting container_image for all mon 2022-09-29T05:30:41.899 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:41 smithi083 ceph-mon[79930]: Upgrade: Setting container_image for all crash 2022-09-29T05:30:41.900 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:41 smithi083 ceph-mon[79930]: from='mon.1 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2022-09-29T05:30:41.900 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:41 smithi083 ceph-mon[79930]: Upgrade: osd.0 is safe to restart 2022-09-29T05:30:41.900 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:41.900 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2022-09-29T05:30:41.900 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:30:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:41 smithi153 ceph-mon[70044]: Upgrade: Setting container_image for all mon 2022-09-29T05:30:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:41 smithi153 ceph-mon[70044]: Upgrade: Setting container_image for all crash 2022-09-29T05:30:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:41 smithi153 ceph-mon[70044]: from='mon.1 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2022-09-29T05:30:42.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:41 smithi153 ceph-mon[70044]: Upgrade: osd.0 is safe to restart 2022-09-29T05:30:42.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:42.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2022-09-29T05:30:42.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:30:42.639 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:30:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:30:42.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:30:42.816 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:42 smithi083 ceph-mon[79930]: pgmap v35: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:42.816 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:42 smithi083 ceph-mon[79930]: Upgrade: Updating osd.0 2022-09-29T05:30:42.816 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:42 smithi083 ceph-mon[79930]: Deploying daemon osd.0 on smithi083 2022-09-29T05:30:42.817 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:42 smithi083 ceph-mon[77242]: pgmap v35: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:42.817 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:42 smithi083 ceph-mon[77242]: Upgrade: Updating osd.0 2022-09-29T05:30:42.817 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:42 smithi083 ceph-mon[77242]: Deploying daemon osd.0 on smithi083 2022-09-29T05:30:43.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:42 smithi153 ceph-mon[70044]: pgmap v35: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:43.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:42 smithi153 ceph-mon[70044]: Upgrade: Updating osd.0 2022-09-29T05:30:43.026 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:42 smithi153 ceph-mon[70044]: Deploying daemon osd.0 on smithi083 2022-09-29T05:30:43.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:30:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:30:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:30:44.136 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 29 05:30:43 smithi083 systemd[1]: Stopping Ceph osd.0 for 69111522-3fb6-11ed-8431-001a4aab830c... 2022-09-29T05:30:44.463 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 29 05:30:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-osd-0[47139]: debug 2022-09-29T05:30:44.132+0000 7fc3f4a1c700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2022-09-29T05:30:44.463 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 29 05:30:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-osd-0[47139]: debug 2022-09-29T05:30:44.132+0000 7fc3f4a1c700 -1 osd.0 64 *** Got signal Terminated *** 2022-09-29T05:30:44.463 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 29 05:30:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-osd-0[47139]: debug 2022-09-29T05:30:44.132+0000 7fc3f4a1c700 -1 osd.0 64 *** Immediate shutdown (osd_fast_shutdown=true) *** 2022-09-29T05:30:44.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:44 smithi083 ceph-mon[79930]: pgmap v36: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:44.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:44 smithi083 ceph-mon[79930]: osd.0 marked itself down and dead 2022-09-29T05:30:44.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:44 smithi083 ceph-mon[77242]: pgmap v36: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:44.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:44 smithi083 ceph-mon[77242]: osd.0 marked itself down and dead 2022-09-29T05:30:45.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:44 smithi153 ceph-mon[70044]: pgmap v36: 105 pgs: 105 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:45.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:44 smithi153 ceph-mon[70044]: osd.0 marked itself down and dead 2022-09-29T05:30:45.463 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 29 05:30:44 smithi083 bash[83000]: ceph-69111522-3fb6-11ed-8431-001a4aab830c-osd-0 2022-09-29T05:30:45.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:45 smithi083 ceph-mon[77242]: Health check failed: 1 osds down (OSD_DOWN) 2022-09-29T05:30:45.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:45 smithi083 ceph-mon[77242]: osdmap e65: 8 total, 7 up, 8 in 2022-09-29T05:30:45.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:45 smithi083 ceph-mon[77242]: osdmap e66: 8 total, 7 up, 8 in 2022-09-29T05:30:45.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:45 smithi083 ceph-mon[79930]: Health check failed: 1 osds down (OSD_DOWN) 2022-09-29T05:30:45.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:45 smithi083 ceph-mon[79930]: osdmap e65: 8 total, 7 up, 8 in 2022-09-29T05:30:45.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:45 smithi083 ceph-mon[79930]: osdmap e66: 8 total, 7 up, 8 in 2022-09-29T05:30:46.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:45 smithi153 ceph-mon[70044]: Health check failed: 1 osds down (OSD_DOWN) 2022-09-29T05:30:46.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:45 smithi153 ceph-mon[70044]: osdmap e65: 8 total, 7 up, 8 in 2022-09-29T05:30:46.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:45 smithi153 ceph-mon[70044]: osdmap e66: 8 total, 7 up, 8 in 2022-09-29T05:30:46.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:46 smithi083 ceph-mon[79930]: pgmap v38: 105 pgs: 43 peering, 62 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:46.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:46 smithi083 ceph-mon[79930]: Health check failed: Reduced data availability: 10 pgs inactive, 43 pgs peering (PG_AVAILABILITY) 2022-09-29T05:30:46.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:46 smithi083 ceph-mon[77242]: pgmap v38: 105 pgs: 43 peering, 62 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:46.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:46 smithi083 ceph-mon[77242]: Health check failed: Reduced data availability: 10 pgs inactive, 43 pgs peering (PG_AVAILABILITY) 2022-09-29T05:30:46.965 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 29 05:30:46 smithi083 systemd[1]: ceph-69111522-3fb6-11ed-8431-001a4aab830c@osd.0.service: Succeeded. 2022-09-29T05:30:46.965 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 29 05:30:46 smithi083 systemd[1]: Stopped Ceph osd.0 for 69111522-3fb6-11ed-8431-001a4aab830c. 2022-09-29T05:30:46.965 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 29 05:30:46 smithi083 systemd[1]: Starting Ceph osd.0 for 69111522-3fb6-11ed-8431-001a4aab830c... 2022-09-29T05:30:47.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:46 smithi153 ceph-mon[70044]: pgmap v38: 105 pgs: 43 peering, 62 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:47.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:46 smithi153 ceph-mon[70044]: Health check failed: Reduced data availability: 10 pgs inactive, 43 pgs peering (PG_AVAILABILITY) 2022-09-29T05:30:48.463 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 29 05:30:48 smithi083 bash[83250]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2022-09-29T05:30:48.464 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 29 05:30:48 smithi083 bash[83250]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2022-09-29T05:30:48.464 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 29 05:30:48 smithi083 bash[83250]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2022-09-29T05:30:48.464 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 29 05:30:48 smithi083 bash[83250]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2022-09-29T05:30:48.465 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 29 05:30:48 smithi083 bash[83250]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2022-09-29T05:30:48.465 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 29 05:30:48 smithi083 bash[83250]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2022-09-29T05:30:48.465 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 29 05:30:48 smithi083 bash[83250]: --> ceph-volume raw activate successful for osd ID: 0 2022-09-29T05:30:48.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:48 smithi083 ceph-mon[79930]: pgmap v40: 105 pgs: 43 peering, 62 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:48.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:48 smithi083 ceph-mon[77242]: pgmap v40: 105 pgs: 43 peering, 62 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:49.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:48 smithi153 ceph-mon[70044]: pgmap v40: 105 pgs: 43 peering, 62 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:49.059 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:30:48 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:30:48.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi153.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi083:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:30:49.060 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:30:48 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:30:48.662Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephPGImbalance\nexpr: abs(((ceph_osd_numpg > 0) - on(job) group_left() avg by(job) (ceph_osd_numpg\n > 0)) / on(job) group_left() avg by(job) (ceph_osd_numpg > 0)) * on(ceph_daemon)\n group_left(hostname) ceph_osd_metadata > 0.3\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.5\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} deviates by\n more than 30% from average PG count.\n summary: PGs are not balanced across OSDs\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi153.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi083:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:30:49.463 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 29 05:30:49 smithi083 podman[83509]: 2022-09-29T05:30:49.463 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 29 05:30:49 smithi083 ceph-osd[83548]: -- 172.21.15.83:0/2928676683 <== mon.0 v2:172.21.15.83:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x565357790680 con 0x565357755000 2022-09-29T05:30:49.463 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 29 05:30:49 smithi083 bash[83509]: bf55222fed6d1ee40a366aac47fc7bf2a8a003e57eb5679cd2cf9cc3eab54101 2022-09-29T05:30:49.897 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 29 05:30:49 smithi083 systemd[1]: Started Ceph osd.0 for 69111522-3fb6-11ed-8431-001a4aab830c. 2022-09-29T05:30:50.213 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 29 05:30:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-osd-0[83525]: 2022-09-29T05:30:49.893+0000 7efe8abf55c0 -1 Falling back to public interface 2022-09-29T05:30:50.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:50 smithi153 ceph-mon[70044]: pgmap v41: 105 pgs: 43 peering, 62 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:50.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:50 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:50.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:50 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:50.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:50 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:30:50.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:50 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:30:50.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:50 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:30:50.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:50 smithi083 ceph-mon[79930]: pgmap v41: 105 pgs: 43 peering, 62 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:50.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:50 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:50.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:50 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:50.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:50 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:30:50.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:50 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:30:50.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:50 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:30:50.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:50 smithi083 ceph-mon[77242]: pgmap v41: 105 pgs: 43 peering, 62 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:30:50.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:50 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:50.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:50 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:50.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:50 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:30:50.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:50 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:30:50.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:50 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:30:51.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:30:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:30:51] "GET /metrics HTTP/1.1" 200 170169 "" "Prometheus/2.33.4" 2022-09-29T05:30:51.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:51 smithi083 ceph-mon[77242]: Health check failed: Degraded data redundancy: 14/72 objects degraded (19.444%), 7 pgs degraded (PG_DEGRADED) 2022-09-29T05:30:51.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:51 smithi083 ceph-mon[77242]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 10 pgs inactive, 43 pgs peering) 2022-09-29T05:30:51.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:51 smithi083 ceph-mon[79930]: Health check failed: Degraded data redundancy: 14/72 objects degraded (19.444%), 7 pgs degraded (PG_DEGRADED) 2022-09-29T05:30:51.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:51 smithi083 ceph-mon[79930]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 10 pgs inactive, 43 pgs peering) 2022-09-29T05:30:52.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:51 smithi153 ceph-mon[70044]: Health check failed: Degraded data redundancy: 14/72 objects degraded (19.444%), 7 pgs degraded (PG_DEGRADED) 2022-09-29T05:30:52.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:51 smithi153 ceph-mon[70044]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 10 pgs inactive, 43 pgs peering) 2022-09-29T05:30:52.575 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:30:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:30:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:30:52.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:52 smithi083 ceph-mon[79930]: pgmap v42: 105 pgs: 7 active+undersized+degraded, 36 active+undersized, 62 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 14/72 objects degraded (19.444%) 2022-09-29T05:30:52.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:52 smithi083 ceph-mon[77242]: pgmap v42: 105 pgs: 7 active+undersized+degraded, 36 active+undersized, 62 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 14/72 objects degraded (19.444%) 2022-09-29T05:30:52.964 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 29 05:30:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-osd-0[83525]: 2022-09-29T05:30:52.620+0000 7efe8abf55c0 -1 osd.0 64 log_to_monitors true 2022-09-29T05:30:53.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:52 smithi153 ceph-mon[70044]: pgmap v42: 105 pgs: 7 active+undersized+degraded, 36 active+undersized, 62 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 14/72 objects degraded (19.444%) 2022-09-29T05:30:53.231 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T05:30:53.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:30:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:30:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:30:53.531 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T05:30:53.531 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4m) 55s ago 5m 26.4M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T05:30:53.531 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4m) 36s ago 4m 60.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T05:30:53.531 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5m) 55s ago 5m 15.2M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T05:30:53.532 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5m) 36s ago 5m 18.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T05:30:53.532 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (98s) 36s ago 8m 427M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T05:30:53.532 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2m) 55s ago 10m 475M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T05:30:53.532 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (84s) 55s ago 10m 46.3M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T05:30:53.533 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (46s) 36s ago 8m 30.0M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T05:30:53.533 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (59s) 55s ago 8m 25.8M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T05:30:53.533 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5m) 55s ago 5m 24.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T05:30:53.533 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5m) 36s ago 5m 24.7M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T05:30:53.533 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 starting - - - 2500M 2022-09-29T05:30:53.534 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7m) 55s ago 7m 61.7M 2500M 16.2.10 c56e13ff90f2 989e877de30a 2022-09-29T05:30:53.534 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7m) 55s ago 7m 55.4M 2500M 16.2.10 c56e13ff90f2 81b9b51f0eee 2022-09-29T05:30:53.534 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7m) 55s ago 7m 62.2M 2500M 16.2.10 c56e13ff90f2 4863a2083356 2022-09-29T05:30:53.534 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7m) 36s ago 7m 58.3M 2500M 16.2.10 c56e13ff90f2 ed61dc9353b0 2022-09-29T05:30:53.535 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6m) 36s ago 6m 66.5M 2500M 16.2.10 c56e13ff90f2 59f19fd15fed 2022-09-29T05:30:53.535 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6m) 36s ago 6m 68.0M 2500M 16.2.10 c56e13ff90f2 23decfd9c6b2 2022-09-29T05:30:53.535 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6m) 36s ago 6m 72.0M 2500M 16.2.10 c56e13ff90f2 23460a09c0a7 2022-09-29T05:30:53.535 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (102s) 36s ago 5m 66.9M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T05:30:53.922 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T05:30:53.922 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T05:30:53.922 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T05:30:53.923 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:30:53.923 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T05:30:53.923 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T05:30:53.923 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:30:53.923 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T05:30:53.924 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 7 2022-09-29T05:30:53.924 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:30:53.924 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T05:30:53.924 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T05:30:53.924 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:30:53.925 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T05:30:53.925 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 9, 2022-09-29T05:30:53.925 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 5 2022-09-29T05:30:53.926 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T05:30:53.926 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T05:30:53.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:53 smithi083 ceph-mon[79930]: from='osd.0 [v2:172.21.15.83:6802/3034839700,v1:172.21.15.83:6803/3034839700]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2022-09-29T05:30:53.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:53 smithi083 ceph-mon[79930]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2022-09-29T05:30:53.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:53 smithi083 ceph-mon[77242]: from='osd.0 [v2:172.21.15.83:6802/3034839700,v1:172.21.15.83:6803/3034839700]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2022-09-29T05:30:53.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:53 smithi083 ceph-mon[77242]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2022-09-29T05:30:53.964 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 29 05:30:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-osd-0[83525]: 2022-09-29T05:30:53.600+0000 7efe79374700 -1 osd.0 64 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2022-09-29T05:30:54.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:53 smithi153 ceph-mon[70044]: from='osd.0 [v2:172.21.15.83:6802/3034839700,v1:172.21.15.83:6803/3034839700]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2022-09-29T05:30:54.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:53 smithi153 ceph-mon[70044]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2022-09-29T05:30:54.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:54 smithi083 ceph-mon[77242]: pgmap v43: 105 pgs: 7 active+undersized+degraded, 36 active+undersized, 62 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 14/72 objects degraded (19.444%) 2022-09-29T05:30:54.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:54 smithi083 ceph-mon[77242]: from='client.34191 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:30:54.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:54 smithi083 ceph-mon[77242]: from='client.44131 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:30:54.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:54 smithi083 ceph-mon[77242]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2022-09-29T05:30:54.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:54 smithi083 ceph-mon[77242]: osdmap e67: 8 total, 7 up, 8 in 2022-09-29T05:30:54.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:54 smithi083 ceph-mon[77242]: from='osd.0 [v2:172.21.15.83:6802/3034839700,v1:172.21.15.83:6803/3034839700]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]: dispatch 2022-09-29T05:30:54.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:54 smithi083 ceph-mon[77242]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]: dispatch 2022-09-29T05:30:54.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:54 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:54.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:54 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3959745238' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:30:54.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:54 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:54.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:54 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:54.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:54 smithi083 ceph-mon[79930]: pgmap v43: 105 pgs: 7 active+undersized+degraded, 36 active+undersized, 62 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 14/72 objects degraded (19.444%) 2022-09-29T05:30:54.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:54 smithi083 ceph-mon[79930]: from='client.34191 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:30:54.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:54 smithi083 ceph-mon[79930]: from='client.44131 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:30:54.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:54 smithi083 ceph-mon[79930]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2022-09-29T05:30:54.968 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:54 smithi083 ceph-mon[79930]: osdmap e67: 8 total, 7 up, 8 in 2022-09-29T05:30:54.968 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:54 smithi083 ceph-mon[79930]: from='osd.0 [v2:172.21.15.83:6802/3034839700,v1:172.21.15.83:6803/3034839700]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]: dispatch 2022-09-29T05:30:54.968 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:54 smithi083 ceph-mon[79930]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]: dispatch 2022-09-29T05:30:54.969 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:54 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:54.969 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:54 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3959745238' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:30:54.969 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:54 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:54.970 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:54 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:55.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:54 smithi153 ceph-mon[70044]: pgmap v43: 105 pgs: 7 active+undersized+degraded, 36 active+undersized, 62 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 14/72 objects degraded (19.444%) 2022-09-29T05:30:55.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:54 smithi153 ceph-mon[70044]: from='client.34191 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:30:55.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:54 smithi153 ceph-mon[70044]: from='client.44131 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:30:55.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:54 smithi153 ceph-mon[70044]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2022-09-29T05:30:55.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:54 smithi153 ceph-mon[70044]: osdmap e67: 8 total, 7 up, 8 in 2022-09-29T05:30:55.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:54 smithi153 ceph-mon[70044]: from='osd.0 [v2:172.21.15.83:6802/3034839700,v1:172.21.15.83:6803/3034839700]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]: dispatch 2022-09-29T05:30:55.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:54 smithi153 ceph-mon[70044]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]: dispatch 2022-09-29T05:30:55.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:54 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:55.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:54 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3959745238' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:30:55.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:54 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:55.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:54 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:55.871 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:55 smithi083 ceph-mon[79930]: Health check cleared: OSD_DOWN (was: 1 osds down) 2022-09-29T05:30:55.871 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:55 smithi083 ceph-mon[79930]: osd.0 [v2:172.21.15.83:6802/3034839700,v1:172.21.15.83:6803/3034839700] boot 2022-09-29T05:30:55.872 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:55 smithi083 ceph-mon[79930]: osdmap e68: 8 total, 8 up, 8 in 2022-09-29T05:30:55.872 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:55 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2022-09-29T05:30:55.873 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:55 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:55.873 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:55 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:55.874 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:55 smithi083 ceph-mon[79930]: osdmap e69: 8 total, 8 up, 8 in 2022-09-29T05:30:55.874 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:55 smithi083 ceph-mon[77242]: Health check cleared: OSD_DOWN (was: 1 osds down) 2022-09-29T05:30:55.875 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:55 smithi083 ceph-mon[77242]: osd.0 [v2:172.21.15.83:6802/3034839700,v1:172.21.15.83:6803/3034839700] boot 2022-09-29T05:30:55.875 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:55 smithi083 ceph-mon[77242]: osdmap e68: 8 total, 8 up, 8 in 2022-09-29T05:30:55.876 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:55 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2022-09-29T05:30:55.876 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:55 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:55.876 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:55 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:55.877 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:55 smithi083 ceph-mon[77242]: osdmap e69: 8 total, 8 up, 8 in 2022-09-29T05:30:56.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:55 smithi153 ceph-mon[70044]: Health check cleared: OSD_DOWN (was: 1 osds down) 2022-09-29T05:30:56.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:55 smithi153 ceph-mon[70044]: osd.0 [v2:172.21.15.83:6802/3034839700,v1:172.21.15.83:6803/3034839700] boot 2022-09-29T05:30:56.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:55 smithi153 ceph-mon[70044]: osdmap e68: 8 total, 8 up, 8 in 2022-09-29T05:30:56.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:55 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2022-09-29T05:30:56.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:55 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:56.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:55 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:56.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:55 smithi153 ceph-mon[70044]: osdmap e69: 8 total, 8 up, 8 in 2022-09-29T05:30:56.882 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:56 smithi083 ceph-mon[79930]: pgmap v46: 105 pgs: 31 peering, 2 active+undersized+degraded, 10 active+undersized, 62 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2022-09-29T05:30:56.883 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:56 smithi083 ceph-mon[77242]: pgmap v46: 105 pgs: 31 peering, 2 active+undersized+degraded, 10 active+undersized, 62 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2022-09-29T05:30:57.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:56 smithi153 ceph-mon[70044]: pgmap v46: 105 pgs: 31 peering, 2 active+undersized+degraded, 10 active+undersized, 62 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2022-09-29T05:30:58.463 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:30:58 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: 2022-09-29T05:30:58.232+0000 7f9b26e95700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (14 PGs are or would become offline) 2022-09-29T05:30:58.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:58 smithi083 ceph-mon[77242]: pgmap v48: 105 pgs: 31 peering, 2 active+undersized+degraded, 10 active+undersized, 62 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2022-09-29T05:30:58.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:58 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:58.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:58 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:58.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:58 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:58.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:58 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:30:58.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:58 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:30:58.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:58 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:30:58.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:58 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:30:58.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:58 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2022-09-29T05:30:58.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:58 smithi083 ceph-mon[79930]: pgmap v48: 105 pgs: 31 peering, 2 active+undersized+degraded, 10 active+undersized, 62 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2022-09-29T05:30:58.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:58 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:58.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:58 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:58.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:58 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:58.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:58 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:30:58.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:58 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:30:58.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:58 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:30:58.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:58 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:30:58.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:58 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2022-09-29T05:30:59.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:58 smithi153 ceph-mon[70044]: pgmap v48: 105 pgs: 31 peering, 2 active+undersized+degraded, 10 active+undersized, 62 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2022-09-29T05:30:59.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:58 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:59.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:58 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:59.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:58 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:30:59.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:58 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:30:59.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:58 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:30:59.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:58 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:30:59.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:58 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:30:59.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:58 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2022-09-29T05:30:59.062 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:30:58 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:30:58.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi153.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi083:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:30:59.062 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:30:58 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:30:58.662Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephPGImbalance\nexpr: abs(((ceph_osd_numpg > 0) - on(job) group_left() avg by(job) (ceph_osd_numpg\n > 0)) / on(job) group_left() avg by(job) (ceph_osd_numpg > 0)) * on(ceph_daemon)\n group_left(hostname) ceph_osd_metadata > 0.3\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.5\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} deviates by\n more than 30% from average PG count.\n summary: PGs are not balanced across OSDs\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi153.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi083:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:30:59.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:59 smithi083 ceph-mon[79930]: Detected new or changed devices on smithi083 2022-09-29T05:30:59.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:59 smithi083 ceph-mon[79930]: from='mon.1 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2022-09-29T05:30:59.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:30:59 smithi083 ceph-mon[79930]: Upgrade: unsafe to stop osd(s) at this time (14 PGs are or would become offline) 2022-09-29T05:30:59.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:59 smithi083 ceph-mon[77242]: Detected new or changed devices on smithi083 2022-09-29T05:30:59.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:59 smithi083 ceph-mon[77242]: from='mon.1 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2022-09-29T05:30:59.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:30:59 smithi083 ceph-mon[77242]: Upgrade: unsafe to stop osd(s) at this time (14 PGs are or would become offline) 2022-09-29T05:31:00.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:59 smithi153 ceph-mon[70044]: Detected new or changed devices on smithi083 2022-09-29T05:31:00.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:59 smithi153 ceph-mon[70044]: from='mon.1 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2022-09-29T05:31:00.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:30:59 smithi153 ceph-mon[70044]: Upgrade: unsafe to stop osd(s) at this time (14 PGs are or would become offline) 2022-09-29T05:31:00.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:00 smithi083 ceph-mon[79930]: pgmap v49: 105 pgs: 31 peering, 74 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:31:00.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:00 smithi083 ceph-mon[79930]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/72 objects degraded (5.556%), 2 pgs degraded) 2022-09-29T05:31:00.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:00 smithi083 ceph-mon[77242]: pgmap v49: 105 pgs: 31 peering, 74 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:31:00.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:00 smithi083 ceph-mon[77242]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/72 objects degraded (5.556%), 2 pgs degraded) 2022-09-29T05:31:01.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:00 smithi153 ceph-mon[70044]: pgmap v49: 105 pgs: 31 peering, 74 active+clean; 583 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:31:01.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:00 smithi153 ceph-mon[70044]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/72 objects degraded (5.556%), 2 pgs degraded) 2022-09-29T05:31:01.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:31:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:31:01] "GET /metrics HTTP/1.1" 200 192027 "" "Prometheus/2.33.4" 2022-09-29T05:31:02.610 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:31:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:31:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:31:02.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:02 smithi083 ceph-mon[77242]: pgmap v50: 105 pgs: 105 active+clean; 583 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:31:02.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:02 smithi083 ceph-mon[79930]: pgmap v50: 105 pgs: 105 active+clean; 583 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:31:03.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:02 smithi153 ceph-mon[70044]: pgmap v50: 105 pgs: 105 active+clean; 583 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:31:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:31:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:31:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:31:04.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:04 smithi083 ceph-mon[79930]: pgmap v51: 105 pgs: 105 active+clean; 583 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:31:04.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:04 smithi083 ceph-mon[77242]: pgmap v51: 105 pgs: 105 active+clean; 583 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:31:05.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:04 smithi153 ceph-mon[70044]: pgmap v51: 105 pgs: 105 active+clean; 583 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:31:06.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:06 smithi083 ceph-mon[79930]: pgmap v52: 105 pgs: 105 active+clean; 583 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:31:06.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:06 smithi083 ceph-mon[77242]: pgmap v52: 105 pgs: 105 active+clean; 583 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:31:07.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:06 smithi153 ceph-mon[70044]: pgmap v52: 105 pgs: 105 active+clean; 583 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:31:08.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:08 smithi083 ceph-mon[79930]: pgmap v53: 105 pgs: 105 active+clean; 583 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:31:08.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:08 smithi083 ceph-mon[77242]: pgmap v53: 105 pgs: 105 active+clean; 583 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:31:09.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:08 smithi153 ceph-mon[70044]: pgmap v53: 105 pgs: 105 active+clean; 583 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:31:09.060 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:31:08 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:31:08.661Z caller=alerting.go:353 level=warn component="rule manager" alert="unsupported value type" msg="Expanding alert template failed" err="error executing template __alert_CephOSDDown: template: __alert_CephOSDDown:1:358: executing \"__alert_CephOSDDown\" at : error calling query: found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi153.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi083:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}];many-to-many matching not allowed: matching labels must be unique on one side" data="unsupported value type" 2022-09-29T05:31:09.060 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:31:08 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:31:08.662Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi153.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi083:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:31:09.060 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:31:08 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:31:08.663Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephPGImbalance\nexpr: abs(((ceph_osd_numpg > 0) - on(job) group_left() avg by(job) (ceph_osd_numpg\n > 0)) / on(job) group_left() avg by(job) (ceph_osd_numpg > 0)) * on(ceph_daemon)\n group_left(hostname) ceph_osd_metadata > 0.3\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.5\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} deviates by\n more than 30% from average PG count.\n summary: PGs are not balanced across OSDs\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi153.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi083:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:31:10.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:09 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:10.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:09 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:10.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:09 smithi083 ceph-mon[79930]: pgmap v54: 105 pgs: 105 active+clean; 583 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:31:10.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:09 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:10.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:09 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:10.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:09 smithi083 ceph-mon[77242]: pgmap v54: 105 pgs: 105 active+clean; 583 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:31:10.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:09 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:10.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:09 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:10.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:09 smithi153 ceph-mon[70044]: pgmap v54: 105 pgs: 105 active+clean; 583 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:31:11.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:10 smithi083 ceph-mon[79930]: mgrmap e36: y(active, since 91s), standbys: x 2022-09-29T05:31:11.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:10 smithi083 ceph-mon[77242]: mgrmap e36: y(active, since 91s), standbys: x 2022-09-29T05:31:11.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:10 smithi153 ceph-mon[70044]: mgrmap e36: y(active, since 91s), standbys: x 2022-09-29T05:31:11.889 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:31:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:31:11] "GET /metrics HTTP/1.1" 200 191898 "" "Prometheus/2.33.4" 2022-09-29T05:31:12.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:11 smithi083 ceph-mon[77242]: pgmap v55: 105 pgs: 105 active+clean; 583 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2022-09-29T05:31:12.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:11 smithi083 ceph-mon[79930]: pgmap v55: 105 pgs: 105 active+clean; 583 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2022-09-29T05:31:12.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:11 smithi153 ceph-mon[70044]: pgmap v55: 105 pgs: 105 active+clean; 583 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2022-09-29T05:31:12.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:31:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:31:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:31:13.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:31:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:31:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:31:14.411 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:14 smithi083 ceph-mon[79930]: pgmap v56: 105 pgs: 105 active+clean; 583 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2022-09-29T05:31:14.412 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:14 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2022-09-29T05:31:14.412 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:14 smithi083 ceph-mon[79930]: from='mon.1 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2022-09-29T05:31:14.412 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:14 smithi083 ceph-mon[79930]: Upgrade: osd.1 is safe to restart 2022-09-29T05:31:14.413 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:14 smithi083 ceph-mon[77242]: pgmap v56: 105 pgs: 105 active+clean; 583 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2022-09-29T05:31:14.413 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:14 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2022-09-29T05:31:14.413 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:14 smithi083 ceph-mon[77242]: from='mon.1 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2022-09-29T05:31:14.414 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:14 smithi083 ceph-mon[77242]: Upgrade: osd.1 is safe to restart 2022-09-29T05:31:14.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:14 smithi153 ceph-mon[70044]: pgmap v56: 105 pgs: 105 active+clean; 583 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2022-09-29T05:31:14.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:14 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2022-09-29T05:31:14.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:14 smithi153 ceph-mon[70044]: from='mon.1 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2022-09-29T05:31:14.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:14 smithi153 ceph-mon[70044]: Upgrade: osd.1 is safe to restart 2022-09-29T05:31:15.923 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:15 smithi083 ceph-mon[79930]: Upgrade: Updating osd.1 2022-09-29T05:31:15.924 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:15 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:15.924 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:15 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2022-09-29T05:31:15.924 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:15 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:31:15.925 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:15 smithi083 ceph-mon[79930]: Deploying daemon osd.1 on smithi083 2022-09-29T05:31:15.925 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:15 smithi083 ceph-mon[77242]: Upgrade: Updating osd.1 2022-09-29T05:31:15.926 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:15 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:15.926 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:15 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2022-09-29T05:31:15.927 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:15 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:31:15.927 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:15 smithi083 ceph-mon[77242]: Deploying daemon osd.1 on smithi083 2022-09-29T05:31:16.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:15 smithi153 ceph-mon[70044]: Upgrade: Updating osd.1 2022-09-29T05:31:16.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:15 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:16.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:15 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2022-09-29T05:31:16.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:15 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:31:16.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:15 smithi153 ceph-mon[70044]: Deploying daemon osd.1 on smithi083 2022-09-29T05:31:16.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:16 smithi083 ceph-mon[79930]: pgmap v57: 105 pgs: 105 active+clean; 583 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2022-09-29T05:31:16.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:16 smithi083 ceph-mon[77242]: pgmap v57: 105 pgs: 105 active+clean; 583 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2022-09-29T05:31:17.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:16 smithi153 ceph-mon[70044]: pgmap v57: 105 pgs: 105 active+clean; 583 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2022-09-29T05:31:17.461 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 29 05:31:17 smithi083 systemd[1]: Stopping Ceph osd.1 for 69111522-3fb6-11ed-8431-001a4aab830c... 2022-09-29T05:31:17.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:17 smithi083 ceph-mon[77242]: osd.1 marked itself down and dead 2022-09-29T05:31:17.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:17 smithi083 ceph-mon[79930]: osd.1 marked itself down and dead 2022-09-29T05:31:17.714 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 29 05:31:17 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-osd-1[50140]: debug 2022-09-29T05:31:17.455+0000 7f1b5eb4f700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2022-09-29T05:31:17.714 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 29 05:31:17 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-osd-1[50140]: debug 2022-09-29T05:31:17.455+0000 7f1b5eb4f700 -1 osd.1 69 *** Got signal Terminated *** 2022-09-29T05:31:17.715 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 29 05:31:17 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-osd-1[50140]: debug 2022-09-29T05:31:17.455+0000 7f1b5eb4f700 -1 osd.1 69 *** Immediate shutdown (osd_fast_shutdown=true) *** 2022-09-29T05:31:18.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:17 smithi153 ceph-mon[70044]: osd.1 marked itself down and dead 2022-09-29T05:31:18.213 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 29 05:31:17 smithi083 bash[85688]: ceph-69111522-3fb6-11ed-8431-001a4aab830c-osd-1 2022-09-29T05:31:18.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:18 smithi083 ceph-mon[77242]: pgmap v58: 105 pgs: 105 active+clean; 583 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2022-09-29T05:31:18.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:18 smithi083 ceph-mon[77242]: Health check failed: 1 osds down (OSD_DOWN) 2022-09-29T05:31:18.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:18 smithi083 ceph-mon[77242]: osdmap e70: 8 total, 7 up, 8 in 2022-09-29T05:31:18.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:18 smithi083 ceph-mon[79930]: pgmap v58: 105 pgs: 105 active+clean; 583 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2022-09-29T05:31:18.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:18 smithi083 ceph-mon[79930]: Health check failed: 1 osds down (OSD_DOWN) 2022-09-29T05:31:18.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:18 smithi083 ceph-mon[79930]: osdmap e70: 8 total, 7 up, 8 in 2022-09-29T05:31:19.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:18 smithi153 ceph-mon[70044]: pgmap v58: 105 pgs: 105 active+clean; 583 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2022-09-29T05:31:19.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:18 smithi153 ceph-mon[70044]: Health check failed: 1 osds down (OSD_DOWN) 2022-09-29T05:31:19.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:18 smithi153 ceph-mon[70044]: osdmap e70: 8 total, 7 up, 8 in 2022-09-29T05:31:19.060 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:31:18 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:31:18.662Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi083:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi153.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:31:19.060 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:31:18 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:31:18.663Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephPGImbalance\nexpr: abs(((ceph_osd_numpg > 0) - on(job) group_left() avg by(job) (ceph_osd_numpg\n > 0)) / on(job) group_left() avg by(job) (ceph_osd_numpg > 0)) * on(ceph_daemon)\n group_left(hostname) ceph_osd_metadata > 0.3\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.5\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} deviates by\n more than 30% from average PG count.\n summary: PGs are not balanced across OSDs\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi083:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi153.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:31:19.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:19 smithi083 ceph-mon[77242]: osdmap e71: 8 total, 7 up, 8 in 2022-09-29T05:31:19.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:19 smithi083 ceph-mon[79930]: osdmap e71: 8 total, 7 up, 8 in 2022-09-29T05:31:19.964 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 29 05:31:19 smithi083 systemd[1]: ceph-69111522-3fb6-11ed-8431-001a4aab830c@osd.1.service: Succeeded. 2022-09-29T05:31:19.965 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 29 05:31:19 smithi083 systemd[1]: Stopped Ceph osd.1 for 69111522-3fb6-11ed-8431-001a4aab830c. 2022-09-29T05:31:19.965 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 29 05:31:19 smithi083 systemd[1]: Starting Ceph osd.1 for 69111522-3fb6-11ed-8431-001a4aab830c... 2022-09-29T05:31:20.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:19 smithi153 ceph-mon[70044]: osdmap e71: 8 total, 7 up, 8 in 2022-09-29T05:31:20.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:20 smithi083 ceph-mon[79930]: pgmap v61: 105 pgs: 6 peering, 13 stale+active+clean, 86 active+clean; 583 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:31:20.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:20 smithi083 ceph-mon[77242]: pgmap v61: 105 pgs: 6 peering, 13 stale+active+clean, 86 active+clean; 583 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:31:21.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:20 smithi153 ceph-mon[70044]: pgmap v61: 105 pgs: 6 peering, 13 stale+active+clean, 86 active+clean; 583 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:31:21.463 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 29 05:31:21 smithi083 bash[85940]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2022-09-29T05:31:21.464 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 29 05:31:21 smithi083 bash[85940]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2022-09-29T05:31:21.464 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 29 05:31:21 smithi083 bash[85940]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2022-09-29T05:31:21.464 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 29 05:31:21 smithi083 bash[85940]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2022-09-29T05:31:21.464 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 29 05:31:21 smithi083 bash[85940]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2022-09-29T05:31:21.465 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 29 05:31:21 smithi083 bash[85940]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2022-09-29T05:31:21.465 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 29 05:31:21 smithi083 bash[85940]: --> ceph-volume raw activate successful for osd ID: 1 2022-09-29T05:31:21.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:31:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:31:21] "GET /metrics HTTP/1.1" 200 191898 "" "Prometheus/2.33.4" 2022-09-29T05:31:21.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:21 smithi083 ceph-mon[79930]: Health check failed: Degraded data redundancy: 8/72 objects degraded (11.111%), 4 pgs degraded (PG_DEGRADED) 2022-09-29T05:31:21.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:21 smithi083 ceph-mon[77242]: Health check failed: Degraded data redundancy: 8/72 objects degraded (11.111%), 4 pgs degraded (PG_DEGRADED) 2022-09-29T05:31:22.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:21 smithi153 ceph-mon[70044]: Health check failed: Degraded data redundancy: 8/72 objects degraded (11.111%), 4 pgs degraded (PG_DEGRADED) 2022-09-29T05:31:22.469 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 29 05:31:22 smithi083 podman[86218]: 2022-09-29T05:31:22.470 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 29 05:31:22 smithi083 ceph-osd[86257]: -- 172.21.15.83:0/3896422331 <== mon.1 v2:172.21.15.83:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5597b694e680 con 0x5597b6912c00 2022-09-29T05:31:22.470 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 29 05:31:22 smithi083 bash[86218]: 6e1a78274241beb2055018d72f2494494173976dbe98af88db885b0367baff46 2022-09-29T05:31:22.738 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 29 05:31:22 smithi083 systemd[1]: Started Ceph osd.1 for 69111522-3fb6-11ed-8431-001a4aab830c. 2022-09-29T05:31:22.739 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:31:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:31:22.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:31:23.026 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:22 smithi153 ceph-mon[70044]: pgmap v62: 105 pgs: 4 active+undersized+degraded, 6 peering, 28 active+undersized, 67 active+clean; 583 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 8/72 objects degraded (11.111%) 2022-09-29T05:31:23.026 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:22 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:23.026 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:22 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:23.027 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:22 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:31:23.027 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:22 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:31:23.027 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:22 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:31:23.213 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 29 05:31:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-osd-1[86234]: 2022-09-29T05:31:22.919+0000 7fed358315c0 -1 Falling back to public interface 2022-09-29T05:31:23.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:22 smithi083 ceph-mon[77242]: pgmap v62: 105 pgs: 4 active+undersized+degraded, 6 peering, 28 active+undersized, 67 active+clean; 583 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 8/72 objects degraded (11.111%) 2022-09-29T05:31:23.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:22 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:23.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:22 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:23.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:22 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:31:23.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:22 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:31:23.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:22 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:31:23.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:22 smithi083 ceph-mon[79930]: pgmap v62: 105 pgs: 4 active+undersized+degraded, 6 peering, 28 active+undersized, 67 active+clean; 583 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 8/72 objects degraded (11.111%) 2022-09-29T05:31:23.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:22 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:23.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:22 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:23.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:22 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:31:23.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:22 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:31:23.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:22 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:31:23.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:31:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:31:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:31:24.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:23 smithi153 ceph-mon[70044]: pgmap v63: 105 pgs: 4 active+undersized+degraded, 6 peering, 28 active+undersized, 67 active+clean; 583 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 8/72 objects degraded (11.111%) 2022-09-29T05:31:24.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:23 smithi083 ceph-mon[77242]: pgmap v63: 105 pgs: 4 active+undersized+degraded, 6 peering, 28 active+undersized, 67 active+clean; 583 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 8/72 objects degraded (11.111%) 2022-09-29T05:31:24.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:23 smithi083 ceph-mon[79930]: pgmap v63: 105 pgs: 4 active+undersized+degraded, 6 peering, 28 active+undersized, 67 active+clean; 583 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 8/72 objects degraded (11.111%) 2022-09-29T05:31:24.278 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T05:31:24.591 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T05:31:24.591 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5m) 30s ago 5m 26.4M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T05:31:24.592 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5m) 68s ago 5m 60.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T05:31:24.592 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6m) 30s ago 6m 15.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T05:31:24.592 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6m) 68s ago 6m 18.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T05:31:24.592 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2m) 68s ago 9m 427M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T05:31:24.593 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2m) 30s ago 10m 483M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T05:31:24.593 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (115s) 30s ago 10m 79.8M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T05:31:24.593 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (77s) 68s ago 9m 30.0M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T05:31:24.593 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (91s) 30s ago 9m 67.0M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T05:31:24.594 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5m) 30s ago 5m 25.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T05:31:24.594 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5m) 68s ago 5m 24.7M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T05:31:24.594 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (35s) 30s ago 8m 13.7M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T05:31:24.594 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 starting - - - 2500M 2022-09-29T05:31:24.595 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8m) 30s ago 8m 57.9M 2500M 16.2.10 c56e13ff90f2 81b9b51f0eee 2022-09-29T05:31:24.595 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7m) 30s ago 7m 66.0M 2500M 16.2.10 c56e13ff90f2 4863a2083356 2022-09-29T05:31:24.595 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7m) 68s ago 7m 58.3M 2500M 16.2.10 c56e13ff90f2 ed61dc9353b0 2022-09-29T05:31:24.595 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7m) 68s ago 7m 66.5M 2500M 16.2.10 c56e13ff90f2 59f19fd15fed 2022-09-29T05:31:24.596 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7m) 68s ago 7m 68.0M 2500M 16.2.10 c56e13ff90f2 23decfd9c6b2 2022-09-29T05:31:24.596 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6m) 68s ago 6m 72.0M 2500M 16.2.10 c56e13ff90f2 23460a09c0a7 2022-09-29T05:31:24.596 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2m) 68s ago 5m 66.9M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T05:31:24.983 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T05:31:24.983 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T05:31:24.984 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T05:31:24.984 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:31:24.984 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T05:31:24.984 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T05:31:24.985 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:31:24.985 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T05:31:24.985 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 6, 2022-09-29T05:31:24.985 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 1 2022-09-29T05:31:24.986 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:31:24.986 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T05:31:24.986 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T05:31:24.986 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:31:24.987 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T05:31:24.987 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 8, 2022-09-29T05:31:24.987 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 6 2022-09-29T05:31:24.987 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T05:31:24.987 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T05:31:25.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:24 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:25.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:24 smithi083 ceph-mon[79930]: from='client.34212 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:31:25.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:24 smithi083 ceph-mon[79930]: from='client.44152 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:31:25.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:24 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:25.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:24 smithi083 ceph-mon[77242]: from='client.34212 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:31:25.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:24 smithi083 ceph-mon[77242]: from='client.44152 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:31:25.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:24 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:25.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:24 smithi153 ceph-mon[70044]: from='client.34212 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:31:25.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:24 smithi153 ceph-mon[70044]: from='client.44152 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:31:25.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:25 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2041439416' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:31:25.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:25 smithi083 ceph-mon[79930]: pgmap v64: 105 pgs: 4 active+undersized+degraded, 34 active+undersized, 67 active+clean; 583 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 8/72 objects degraded (11.111%) 2022-09-29T05:31:25.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:25 smithi083 ceph-mon[79930]: from='osd.1 [v2:172.21.15.83:6810/1189016617,v1:172.21.15.83:6811/1189016617]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2022-09-29T05:31:25.964 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 29 05:31:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-osd-1[86234]: 2022-09-29T05:31:25.660+0000 7fed358315c0 -1 osd.1 69 log_to_monitors true 2022-09-29T05:31:25.964 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 29 05:31:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-osd-1[86234]: 2022-09-29T05:31:25.884+0000 7fed227ad700 -1 osd.1 69 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2022-09-29T05:31:25.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:25 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2041439416' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:31:25.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:25 smithi083 ceph-mon[77242]: pgmap v64: 105 pgs: 4 active+undersized+degraded, 34 active+undersized, 67 active+clean; 583 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 8/72 objects degraded (11.111%) 2022-09-29T05:31:25.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:25 smithi083 ceph-mon[77242]: from='osd.1 [v2:172.21.15.83:6810/1189016617,v1:172.21.15.83:6811/1189016617]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2022-09-29T05:31:26.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:25 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2041439416' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:31:26.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:25 smithi153 ceph-mon[70044]: pgmap v64: 105 pgs: 4 active+undersized+degraded, 34 active+undersized, 67 active+clean; 583 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 8/72 objects degraded (11.111%) 2022-09-29T05:31:26.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:25 smithi153 ceph-mon[70044]: from='osd.1 [v2:172.21.15.83:6810/1189016617,v1:172.21.15.83:6811/1189016617]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2022-09-29T05:31:27.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:26 smithi083 ceph-mon[79930]: from='osd.1 [v2:172.21.15.83:6810/1189016617,v1:172.21.15.83:6811/1189016617]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2022-09-29T05:31:27.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:26 smithi083 ceph-mon[79930]: osdmap e72: 8 total, 7 up, 8 in 2022-09-29T05:31:27.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:26 smithi083 ceph-mon[79930]: from='osd.1 [v2:172.21.15.83:6810/1189016617,v1:172.21.15.83:6811/1189016617]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]: dispatch 2022-09-29T05:31:27.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:26 smithi083 ceph-mon[77242]: from='osd.1 [v2:172.21.15.83:6810/1189016617,v1:172.21.15.83:6811/1189016617]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2022-09-29T05:31:27.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:26 smithi083 ceph-mon[77242]: osdmap e72: 8 total, 7 up, 8 in 2022-09-29T05:31:27.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:26 smithi083 ceph-mon[77242]: from='osd.1 [v2:172.21.15.83:6810/1189016617,v1:172.21.15.83:6811/1189016617]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]: dispatch 2022-09-29T05:31:27.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:26 smithi153 ceph-mon[70044]: from='osd.1 [v2:172.21.15.83:6810/1189016617,v1:172.21.15.83:6811/1189016617]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2022-09-29T05:31:27.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:26 smithi153 ceph-mon[70044]: osdmap e72: 8 total, 7 up, 8 in 2022-09-29T05:31:27.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:26 smithi153 ceph-mon[70044]: from='osd.1 [v2:172.21.15.83:6810/1189016617,v1:172.21.15.83:6811/1189016617]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]: dispatch 2022-09-29T05:31:28.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:27 smithi083 ceph-mon[79930]: Health check cleared: OSD_DOWN (was: 1 osds down) 2022-09-29T05:31:28.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:27 smithi083 ceph-mon[79930]: osd.1 [v2:172.21.15.83:6810/1189016617,v1:172.21.15.83:6811/1189016617] boot 2022-09-29T05:31:28.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:27 smithi083 ceph-mon[79930]: osdmap e73: 8 total, 8 up, 8 in 2022-09-29T05:31:28.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:27 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2022-09-29T05:31:28.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:27 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:28.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:27 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:28.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:27 smithi083 ceph-mon[79930]: pgmap v67: 105 pgs: 4 active+undersized+degraded, 34 active+undersized, 67 active+clean; 583 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 8/72 objects degraded (11.111%) 2022-09-29T05:31:28.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:27 smithi083 ceph-mon[79930]: osdmap e74: 8 total, 8 up, 8 in 2022-09-29T05:31:28.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:27 smithi083 ceph-mon[77242]: Health check cleared: OSD_DOWN (was: 1 osds down) 2022-09-29T05:31:28.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:27 smithi083 ceph-mon[77242]: osd.1 [v2:172.21.15.83:6810/1189016617,v1:172.21.15.83:6811/1189016617] boot 2022-09-29T05:31:28.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:27 smithi083 ceph-mon[77242]: osdmap e73: 8 total, 8 up, 8 in 2022-09-29T05:31:28.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:27 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2022-09-29T05:31:28.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:27 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:28.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:27 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:28.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:27 smithi083 ceph-mon[77242]: pgmap v67: 105 pgs: 4 active+undersized+degraded, 34 active+undersized, 67 active+clean; 583 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 8/72 objects degraded (11.111%) 2022-09-29T05:31:28.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:27 smithi083 ceph-mon[77242]: osdmap e74: 8 total, 8 up, 8 in 2022-09-29T05:31:28.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:27 smithi153 ceph-mon[70044]: Health check cleared: OSD_DOWN (was: 1 osds down) 2022-09-29T05:31:28.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:27 smithi153 ceph-mon[70044]: osd.1 [v2:172.21.15.83:6810/1189016617,v1:172.21.15.83:6811/1189016617] boot 2022-09-29T05:31:28.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:27 smithi153 ceph-mon[70044]: osdmap e73: 8 total, 8 up, 8 in 2022-09-29T05:31:28.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:27 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2022-09-29T05:31:28.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:27 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:28.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:27 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:28.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:27 smithi153 ceph-mon[70044]: pgmap v67: 105 pgs: 4 active+undersized+degraded, 34 active+undersized, 67 active+clean; 583 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 8/72 objects degraded (11.111%) 2022-09-29T05:31:28.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:27 smithi153 ceph-mon[70044]: osdmap e74: 8 total, 8 up, 8 in 2022-09-29T05:31:29.053 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:31:28 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:31:28.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi083:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi153.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:31:29.054 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:31:28 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:31:28.662Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephPGImbalance\nexpr: abs(((ceph_osd_numpg > 0) - on(job) group_left() avg by(job) (ceph_osd_numpg\n > 0)) / on(job) group_left() avg by(job) (ceph_osd_numpg > 0)) * on(ceph_daemon)\n group_left(hostname) ceph_osd_metadata > 0.3\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.5\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} deviates by\n more than 30% from average PG count.\n summary: PGs are not balanced across OSDs\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi083:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi153.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:31:29.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:29 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:29.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:29 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:29.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:29 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:29.376 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:29 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:29.376 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:29 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:29.377 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:29 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:29.377 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:29 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:29.377 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:29 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:29.378 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:29 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:30.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:30 smithi083 ceph-mon[79930]: pgmap v69: 105 pgs: 4 active+undersized+degraded, 30 active+undersized, 71 active+clean; 583 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 8/72 objects degraded (11.111%) 2022-09-29T05:31:30.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:30 smithi083 ceph-mon[77242]: pgmap v69: 105 pgs: 4 active+undersized+degraded, 30 active+undersized, 71 active+clean; 583 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 8/72 objects degraded (11.111%) 2022-09-29T05:31:31.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:30 smithi153 ceph-mon[70044]: pgmap v69: 105 pgs: 4 active+undersized+degraded, 30 active+undersized, 71 active+clean; 583 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 8/72 objects degraded (11.111%) 2022-09-29T05:31:31.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:31:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:31:31] "GET /metrics HTTP/1.1" 200 192148 "" "Prometheus/2.33.4" 2022-09-29T05:31:32.247 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:32 smithi083 ceph-mon[79930]: Detected new or changed devices on smithi083 2022-09-29T05:31:32.247 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:32 smithi083 ceph-mon[79930]: pgmap v70: 105 pgs: 105 active+clean; 583 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:31:32.248 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:32 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:32.248 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:32 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:32.248 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:32 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:32.248 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:32 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:31:32.249 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:32 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:31:32.249 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:32 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:31:32.249 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:32 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:31:32.249 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:32 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2022-09-29T05:31:32.249 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:32 smithi083 ceph-mon[79930]: from='mon.1 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2022-09-29T05:31:32.250 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:32 smithi083 ceph-mon[79930]: Upgrade: osd.2 is safe to restart 2022-09-29T05:31:32.250 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:32 smithi083 ceph-mon[79930]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 8/72 objects degraded (11.111%), 4 pgs degraded) 2022-09-29T05:31:32.250 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:32 smithi083 ceph-mon[77242]: Detected new or changed devices on smithi083 2022-09-29T05:31:32.251 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:32 smithi083 ceph-mon[77242]: pgmap v70: 105 pgs: 105 active+clean; 583 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:31:32.251 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:32 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:32.251 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:32 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:32.251 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:32 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:32.252 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:32 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:31:32.252 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:32 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:31:32.252 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:32 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:31:32.252 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:32 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:31:32.252 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:32 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2022-09-29T05:31:32.253 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:32 smithi083 ceph-mon[77242]: from='mon.1 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2022-09-29T05:31:32.253 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:32 smithi083 ceph-mon[77242]: Upgrade: osd.2 is safe to restart 2022-09-29T05:31:32.253 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:32 smithi083 ceph-mon[77242]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 8/72 objects degraded (11.111%), 4 pgs degraded) 2022-09-29T05:31:32.559 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:31:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:31:32.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:31:32.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:32 smithi153 ceph-mon[70044]: Detected new or changed devices on smithi083 2022-09-29T05:31:32.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:32 smithi153 ceph-mon[70044]: pgmap v70: 105 pgs: 105 active+clean; 583 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:31:32.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:32 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:32.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:32 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:32.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:32 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:32.561 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:32 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:31:32.561 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:32 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:31:32.561 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:32 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:31:32.561 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:32 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:31:32.562 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:32 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2022-09-29T05:31:32.562 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:32 smithi153 ceph-mon[70044]: from='mon.1 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2022-09-29T05:31:32.562 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:32 smithi153 ceph-mon[70044]: Upgrade: osd.2 is safe to restart 2022-09-29T05:31:32.562 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:32 smithi153 ceph-mon[70044]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 8/72 objects degraded (11.111%), 4 pgs degraded) 2022-09-29T05:31:33.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:31:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:31:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:31:33.939 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:33 smithi083 ceph-mon[79930]: Upgrade: Updating osd.2 2022-09-29T05:31:33.939 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:33 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:33.939 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:33 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2022-09-29T05:31:33.940 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:33 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:31:33.940 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:33 smithi083 ceph-mon[79930]: Deploying daemon osd.2 on smithi083 2022-09-29T05:31:33.940 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:33 smithi083 ceph-mon[77242]: Upgrade: Updating osd.2 2022-09-29T05:31:33.941 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:33 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:33.941 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:33 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2022-09-29T05:31:33.941 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:33 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:31:33.941 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:33 smithi083 ceph-mon[77242]: Deploying daemon osd.2 on smithi083 2022-09-29T05:31:34.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:33 smithi153 ceph-mon[70044]: Upgrade: Updating osd.2 2022-09-29T05:31:34.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:33 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:34.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:33 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2022-09-29T05:31:34.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:33 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:31:34.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:33 smithi153 ceph-mon[70044]: Deploying daemon osd.2 on smithi083 2022-09-29T05:31:34.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:34 smithi083 ceph-mon[79930]: pgmap v71: 105 pgs: 105 active+clean; 583 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:31:34.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:34 smithi083 ceph-mon[77242]: pgmap v71: 105 pgs: 105 active+clean; 583 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:31:35.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:34 smithi153 ceph-mon[70044]: pgmap v71: 105 pgs: 105 active+clean; 583 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:31:35.463 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 29 05:31:35 smithi083 systemd[1]: Stopping Ceph osd.2 for 69111522-3fb6-11ed-8431-001a4aab830c... 2022-09-29T05:31:35.898 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:35 smithi083 ceph-mon[77242]: osd.2 marked itself down and dead 2022-09-29T05:31:35.899 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:35 smithi083 ceph-mon[79930]: osd.2 marked itself down and dead 2022-09-29T05:31:35.899 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 29 05:31:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-osd-2[53187]: debug 2022-09-29T05:31:35.640+0000 7fe8a116e700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2022-09-29T05:31:35.899 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 29 05:31:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-osd-2[53187]: debug 2022-09-29T05:31:35.640+0000 7fe8a116e700 -1 osd.2 74 *** Got signal Terminated *** 2022-09-29T05:31:35.899 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 29 05:31:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-osd-2[53187]: debug 2022-09-29T05:31:35.640+0000 7fe8a116e700 -1 osd.2 74 *** Immediate shutdown (osd_fast_shutdown=true) *** 2022-09-29T05:31:36.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:35 smithi153 ceph-mon[70044]: osd.2 marked itself down and dead 2022-09-29T05:31:36.213 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 29 05:31:35 smithi083 bash[88376]: ceph-69111522-3fb6-11ed-8431-001a4aab830c-osd-2 2022-09-29T05:31:36.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:36 smithi083 ceph-mon[79930]: pgmap v72: 105 pgs: 105 active+clean; 583 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:31:36.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:36 smithi083 ceph-mon[79930]: Health check failed: 1 osds down (OSD_DOWN) 2022-09-29T05:31:36.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:36 smithi083 ceph-mon[79930]: osdmap e75: 8 total, 7 up, 8 in 2022-09-29T05:31:36.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:36 smithi083 ceph-mon[77242]: pgmap v72: 105 pgs: 105 active+clean; 583 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:31:36.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:36 smithi083 ceph-mon[77242]: Health check failed: 1 osds down (OSD_DOWN) 2022-09-29T05:31:36.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:36 smithi083 ceph-mon[77242]: osdmap e75: 8 total, 7 up, 8 in 2022-09-29T05:31:37.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:36 smithi153 ceph-mon[70044]: pgmap v72: 105 pgs: 105 active+clean; 583 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:31:37.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:36 smithi153 ceph-mon[70044]: Health check failed: 1 osds down (OSD_DOWN) 2022-09-29T05:31:37.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:36 smithi153 ceph-mon[70044]: osdmap e75: 8 total, 7 up, 8 in 2022-09-29T05:31:37.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:37 smithi083 ceph-mon[77242]: osdmap e76: 8 total, 7 up, 8 in 2022-09-29T05:31:37.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:37 smithi083 ceph-mon[79930]: osdmap e76: 8 total, 7 up, 8 in 2022-09-29T05:31:37.964 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 29 05:31:37 smithi083 systemd[1]: ceph-69111522-3fb6-11ed-8431-001a4aab830c@osd.2.service: Succeeded. 2022-09-29T05:31:37.964 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 29 05:31:37 smithi083 systemd[1]: Stopped Ceph osd.2 for 69111522-3fb6-11ed-8431-001a4aab830c. 2022-09-29T05:31:37.965 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 29 05:31:37 smithi083 systemd[1]: Starting Ceph osd.2 for 69111522-3fb6-11ed-8431-001a4aab830c... 2022-09-29T05:31:38.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:37 smithi153 ceph-mon[70044]: osdmap e76: 8 total, 7 up, 8 in 2022-09-29T05:31:38.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:38 smithi083 ceph-mon[79930]: pgmap v75: 105 pgs: 6 peering, 6 stale+active+clean, 93 active+clean; 583 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:31:38.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:38 smithi083 ceph-mon[79930]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2022-09-29T05:31:38.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:38 smithi083 ceph-mon[77242]: pgmap v75: 105 pgs: 6 peering, 6 stale+active+clean, 93 active+clean; 583 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:31:38.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:38 smithi083 ceph-mon[77242]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2022-09-29T05:31:39.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:38 smithi153 ceph-mon[70044]: pgmap v75: 105 pgs: 6 peering, 6 stale+active+clean, 93 active+clean; 583 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:31:39.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:38 smithi153 ceph-mon[70044]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2022-09-29T05:31:39.060 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:31:38 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:31:38.661Z caller=alerting.go:353 level=warn component="rule manager" alert="unsupported value type" msg="Expanding alert template failed" err="error executing template __alert_CephOSDDown: template: __alert_CephOSDDown:1:358: executing \"__alert_CephOSDDown\" at : error calling query: found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi083:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi153.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}];many-to-many matching not allowed: matching labels must be unique on one side" data="unsupported value type" 2022-09-29T05:31:39.060 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:31:38 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:31:38.662Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi083:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi153.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:31:39.060 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:31:38 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:31:38.663Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephPGImbalance\nexpr: abs(((ceph_osd_numpg > 0) - on(job) group_left() avg by(job) (ceph_osd_numpg\n > 0)) / on(job) group_left() avg by(job) (ceph_osd_numpg > 0)) * on(ceph_daemon)\n group_left(hostname) ceph_osd_metadata > 0.3\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.5\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} deviates by\n more than 30% from average PG count.\n summary: PGs are not balanced across OSDs\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi083:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi153.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:31:39.213 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 29 05:31:38 smithi083 bash[88633]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2022-09-29T05:31:39.213 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 29 05:31:38 smithi083 bash[88633]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2022-09-29T05:31:39.214 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 29 05:31:38 smithi083 bash[88633]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2022-09-29T05:31:39.214 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 29 05:31:38 smithi083 bash[88633]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2022-09-29T05:31:39.214 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 29 05:31:38 smithi083 bash[88633]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2022-09-29T05:31:39.214 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 29 05:31:38 smithi083 bash[88633]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2022-09-29T05:31:39.215 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 29 05:31:38 smithi083 bash[88633]: --> ceph-volume raw activate successful for osd ID: 2 2022-09-29T05:31:40.181 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 29 05:31:40 smithi083 podman[88895]: 2022-09-29T05:31:40.181 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:40.182 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:40.182 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:31:40.182 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:31:40.182 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:31:40.183 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:31:40.183 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:39 smithi083 ceph-mon[77242]: pgmap v76: 105 pgs: 6 peering, 6 stale+active+clean, 3 active+undersized, 90 active+clean; 583 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:31:40.183 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:40.183 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:40.184 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:31:40.184 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:31:40.184 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:31:40.184 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:31:40.184 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:39 smithi083 ceph-mon[79930]: pgmap v76: 105 pgs: 6 peering, 6 stale+active+clean, 3 active+undersized, 90 active+clean; 583 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:31:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:31:40.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:31:40.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:31:40.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:31:40.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:39 smithi153 ceph-mon[70044]: pgmap v76: 105 pgs: 6 peering, 6 stale+active+clean, 3 active+undersized, 90 active+clean; 583 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:31:40.713 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 29 05:31:40 smithi083 ceph-osd[88934]: -- 172.21.15.83:0/3096965169 <== mon.1 v2:172.21.15.83:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5597312ec680 con 0x5597312b0c00 2022-09-29T05:31:40.713 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 29 05:31:40 smithi083 bash[88895]: 4eebe9e46fcb70d113ea96a669de5e32dbfc1b8811bfad6103094ea696527b7e 2022-09-29T05:31:40.713 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 29 05:31:40 smithi083 systemd[1]: Started Ceph osd.2 for 69111522-3fb6-11ed-8431-001a4aab830c. 2022-09-29T05:31:41.213 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 29 05:31:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-osd-2[88911]: 2022-09-29T05:31:40.757+0000 7ff3aee7d5c0 -1 Falling back to public interface 2022-09-29T05:31:41.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:31:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:31:41] "GET /metrics HTTP/1.1" 200 196871 "" "Prometheus/2.33.4" 2022-09-29T05:31:41.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:41.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:41.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:31:41.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:31:41.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:31:41.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:41.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:41.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:41.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:41.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:31:41.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:31:41.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:31:41.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:41.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:41.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:41.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:41.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:31:41.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:31:41.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:31:41.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:41.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:42.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:42 smithi083 ceph-mon[79930]: pgmap v77: 105 pgs: 4 active+undersized+degraded, 27 active+undersized, 74 active+clean; 583 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 7/72 objects degraded (9.722%) 2022-09-29T05:31:42.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:42 smithi083 ceph-mon[79930]: Health check failed: Degraded data redundancy: 7/72 objects degraded (9.722%), 4 pgs degraded (PG_DEGRADED) 2022-09-29T05:31:42.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:42 smithi083 ceph-mon[79930]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2022-09-29T05:31:42.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:42 smithi083 ceph-mon[77242]: pgmap v77: 105 pgs: 4 active+undersized+degraded, 27 active+undersized, 74 active+clean; 583 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 7/72 objects degraded (9.722%) 2022-09-29T05:31:42.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:42 smithi083 ceph-mon[77242]: Health check failed: Degraded data redundancy: 7/72 objects degraded (9.722%), 4 pgs degraded (PG_DEGRADED) 2022-09-29T05:31:42.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:42 smithi083 ceph-mon[77242]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2022-09-29T05:31:42.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:31:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:31:42.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:31:42.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:42 smithi153 ceph-mon[70044]: pgmap v77: 105 pgs: 4 active+undersized+degraded, 27 active+undersized, 74 active+clean; 583 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 7/72 objects degraded (9.722%) 2022-09-29T05:31:42.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:42 smithi153 ceph-mon[70044]: Health check failed: Degraded data redundancy: 7/72 objects degraded (9.722%), 4 pgs degraded (PG_DEGRADED) 2022-09-29T05:31:42.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:42 smithi153 ceph-mon[70044]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2022-09-29T05:31:43.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:31:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:31:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:31:43.963 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 29 05:31:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-osd-2[88911]: 2022-09-29T05:31:43.484+0000 7ff3aee7d5c0 -1 osd.2 74 log_to_monitors true 2022-09-29T05:31:44.713 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 29 05:31:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-osd-2[88911]: 2022-09-29T05:31:44.394+0000 7ff39b5f8700 -1 osd.2 74 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2022-09-29T05:31:44.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:44 smithi083 ceph-mon[77242]: pgmap v78: 105 pgs: 4 active+undersized+degraded, 27 active+undersized, 74 active+clean; 583 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 7/72 objects degraded (9.722%) 2022-09-29T05:31:44.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:44 smithi083 ceph-mon[77242]: from='osd.2 [v2:172.21.15.83:6818/128426567,v1:172.21.15.83:6819/128426567]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2022-09-29T05:31:44.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:44 smithi083 ceph-mon[77242]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2022-09-29T05:31:44.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:44 smithi083 ceph-mon[79930]: pgmap v78: 105 pgs: 4 active+undersized+degraded, 27 active+undersized, 74 active+clean; 583 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 7/72 objects degraded (9.722%) 2022-09-29T05:31:44.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:44 smithi083 ceph-mon[79930]: from='osd.2 [v2:172.21.15.83:6818/128426567,v1:172.21.15.83:6819/128426567]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2022-09-29T05:31:44.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:44 smithi083 ceph-mon[79930]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2022-09-29T05:31:44.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:44 smithi153 ceph-mon[70044]: pgmap v78: 105 pgs: 4 active+undersized+degraded, 27 active+undersized, 74 active+clean; 583 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 7/72 objects degraded (9.722%) 2022-09-29T05:31:44.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:44 smithi153 ceph-mon[70044]: from='osd.2 [v2:172.21.15.83:6818/128426567,v1:172.21.15.83:6819/128426567]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2022-09-29T05:31:44.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:44 smithi153 ceph-mon[70044]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2022-09-29T05:31:45.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:45 smithi083 ceph-mon[79930]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2022-09-29T05:31:45.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:45 smithi083 ceph-mon[79930]: osdmap e77: 8 total, 7 up, 8 in 2022-09-29T05:31:45.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:45 smithi083 ceph-mon[79930]: from='osd.2 [v2:172.21.15.83:6818/128426567,v1:172.21.15.83:6819/128426567]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]: dispatch 2022-09-29T05:31:45.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:45 smithi083 ceph-mon[79930]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]: dispatch 2022-09-29T05:31:45.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:45 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:45.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:45 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:45.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:45 smithi083 ceph-mon[77242]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2022-09-29T05:31:45.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:45 smithi083 ceph-mon[77242]: osdmap e77: 8 total, 7 up, 8 in 2022-09-29T05:31:45.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:45 smithi083 ceph-mon[77242]: from='osd.2 [v2:172.21.15.83:6818/128426567,v1:172.21.15.83:6819/128426567]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]: dispatch 2022-09-29T05:31:45.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:45 smithi083 ceph-mon[77242]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]: dispatch 2022-09-29T05:31:45.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:45 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:45.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:45 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:45.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:45 smithi153 ceph-mon[70044]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2022-09-29T05:31:45.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:45 smithi153 ceph-mon[70044]: osdmap e77: 8 total, 7 up, 8 in 2022-09-29T05:31:45.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:45 smithi153 ceph-mon[70044]: from='osd.2 [v2:172.21.15.83:6818/128426567,v1:172.21.15.83:6819/128426567]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]: dispatch 2022-09-29T05:31:45.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:45 smithi153 ceph-mon[70044]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]: dispatch 2022-09-29T05:31:45.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:45 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:45.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:45 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:46.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:46 smithi083 ceph-mon[79930]: pgmap v80: 105 pgs: 4 active+undersized+degraded, 27 active+undersized, 74 active+clean; 583 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 7/72 objects degraded (9.722%) 2022-09-29T05:31:46.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:46 smithi083 ceph-mon[79930]: Health check cleared: OSD_DOWN (was: 1 osds down) 2022-09-29T05:31:46.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:46 smithi083 ceph-mon[79930]: osd.2 [v2:172.21.15.83:6818/128426567,v1:172.21.15.83:6819/128426567] boot 2022-09-29T05:31:46.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:46 smithi083 ceph-mon[79930]: osdmap e78: 8 total, 8 up, 8 in 2022-09-29T05:31:46.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:46 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2022-09-29T05:31:46.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:46 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:46.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:46 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:46.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:46 smithi083 ceph-mon[79930]: osdmap e79: 8 total, 8 up, 8 in 2022-09-29T05:31:46.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:46 smithi083 ceph-mon[77242]: pgmap v80: 105 pgs: 4 active+undersized+degraded, 27 active+undersized, 74 active+clean; 583 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 7/72 objects degraded (9.722%) 2022-09-29T05:31:46.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:46 smithi083 ceph-mon[77242]: Health check cleared: OSD_DOWN (was: 1 osds down) 2022-09-29T05:31:46.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:46 smithi083 ceph-mon[77242]: osd.2 [v2:172.21.15.83:6818/128426567,v1:172.21.15.83:6819/128426567] boot 2022-09-29T05:31:46.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:46 smithi083 ceph-mon[77242]: osdmap e78: 8 total, 8 up, 8 in 2022-09-29T05:31:46.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:46 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2022-09-29T05:31:46.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:46 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:46.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:46 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:46.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:46 smithi083 ceph-mon[77242]: osdmap e79: 8 total, 8 up, 8 in 2022-09-29T05:31:46.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:46 smithi153 ceph-mon[70044]: pgmap v80: 105 pgs: 4 active+undersized+degraded, 27 active+undersized, 74 active+clean; 583 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 7/72 objects degraded (9.722%) 2022-09-29T05:31:46.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:46 smithi153 ceph-mon[70044]: Health check cleared: OSD_DOWN (was: 1 osds down) 2022-09-29T05:31:46.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:46 smithi153 ceph-mon[70044]: osd.2 [v2:172.21.15.83:6818/128426567,v1:172.21.15.83:6819/128426567] boot 2022-09-29T05:31:46.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:46 smithi153 ceph-mon[70044]: osdmap e78: 8 total, 8 up, 8 in 2022-09-29T05:31:46.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:46 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2022-09-29T05:31:46.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:46 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:46.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:46 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:46.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:46 smithi153 ceph-mon[70044]: osdmap e79: 8 total, 8 up, 8 in 2022-09-29T05:31:48.664 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:48 smithi153 ceph-mon[70044]: pgmap v83: 105 pgs: 4 peering, 4 active+undersized+degraded, 23 active+undersized, 74 active+clean; 583 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 7/72 objects degraded (9.722%) 2022-09-29T05:31:48.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:48 smithi083 ceph-mon[77242]: pgmap v83: 105 pgs: 4 peering, 4 active+undersized+degraded, 23 active+undersized, 74 active+clean; 583 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 7/72 objects degraded (9.722%) 2022-09-29T05:31:48.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:48 smithi083 ceph-mon[79930]: pgmap v83: 105 pgs: 4 peering, 4 active+undersized+degraded, 23 active+undersized, 74 active+clean; 583 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 7/72 objects degraded (9.722%) 2022-09-29T05:31:49.059 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:31:48 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:31:48.661Z caller=alerting.go:353 level=warn component="rule manager" alert="unsupported value type" msg="Expanding alert template failed" err="error executing template __alert_CephOSDDown: template: __alert_CephOSDDown:1:358: executing \"__alert_CephOSDDown\" at : error calling query: found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi083:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi153.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}];many-to-many matching not allowed: matching labels must be unique on one side" data="unsupported value type" 2022-09-29T05:31:49.059 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:31:48 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:31:48.663Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi083:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi153.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:31:49.060 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:31:48 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:31:48.663Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephPGImbalance\nexpr: abs(((ceph_osd_numpg > 0) - on(job) group_left() avg by(job) (ceph_osd_numpg\n > 0)) / on(job) group_left() avg by(job) (ceph_osd_numpg > 0)) * on(ceph_daemon)\n group_left(hostname) ceph_osd_metadata > 0.3\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.5\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} deviates by\n more than 30% from average PG count.\n summary: PGs are not balanced across OSDs\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi083:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi153.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:31:49.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:31:49 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: 2022-09-29T05:31:49.490+0000 7f9b26e95700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (11 PGs are or would become offline) 2022-09-29T05:31:50.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:49 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:50.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:49 smithi083 ceph-mon[77242]: pgmap v84: 105 pgs: 4 peering, 4 active+undersized+degraded, 20 active+undersized, 77 active+clean; 583 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 7/72 objects degraded (9.722%) 2022-09-29T05:31:50.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:49 smithi083 ceph-mon[77242]: Detected new or changed devices on smithi083 2022-09-29T05:31:50.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:49 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:50.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:49 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:50.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:49 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:50.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:49 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:31:50.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:49 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:31:50.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:49 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:31:50.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:49 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:31:50.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:49 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2022-09-29T05:31:50.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:49 smithi083 ceph-mon[77242]: from='mon.1 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2022-09-29T05:31:50.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:49 smithi083 ceph-mon[77242]: Upgrade: unsafe to stop osd(s) at this time (11 PGs are or would become offline) 2022-09-29T05:31:50.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:49 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:50.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:49 smithi083 ceph-mon[79930]: pgmap v84: 105 pgs: 4 peering, 4 active+undersized+degraded, 20 active+undersized, 77 active+clean; 583 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 7/72 objects degraded (9.722%) 2022-09-29T05:31:50.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:49 smithi083 ceph-mon[79930]: Detected new or changed devices on smithi083 2022-09-29T05:31:50.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:49 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:50.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:49 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:50.218 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:49 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:50.218 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:49 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:31:50.218 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:49 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:31:50.218 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:49 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:31:50.218 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:49 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:31:50.219 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:49 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2022-09-29T05:31:50.219 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:49 smithi083 ceph-mon[79930]: from='mon.1 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2022-09-29T05:31:50.219 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:49 smithi083 ceph-mon[79930]: Upgrade: unsafe to stop osd(s) at this time (11 PGs are or would become offline) 2022-09-29T05:31:50.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:49 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:50.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:49 smithi153 ceph-mon[70044]: pgmap v84: 105 pgs: 4 peering, 4 active+undersized+degraded, 20 active+undersized, 77 active+clean; 583 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 7/72 objects degraded (9.722%) 2022-09-29T05:31:50.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:49 smithi153 ceph-mon[70044]: Detected new or changed devices on smithi083 2022-09-29T05:31:50.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:49 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:50.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:49 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:50.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:49 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:50.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:49 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:31:50.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:49 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:31:50.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:49 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:31:50.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:49 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:31:50.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:49 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2022-09-29T05:31:50.312 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:49 smithi153 ceph-mon[70044]: from='mon.1 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2022-09-29T05:31:50.312 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:49 smithi153 ceph-mon[70044]: Upgrade: unsafe to stop osd(s) at this time (11 PGs are or would become offline) 2022-09-29T05:31:51.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:51 smithi153 ceph-mon[70044]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 7/72 objects degraded (9.722%), 4 pgs degraded) 2022-09-29T05:31:51.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:51 smithi083 ceph-mon[79930]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 7/72 objects degraded (9.722%), 4 pgs degraded) 2022-09-29T05:31:51.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:51 smithi083 ceph-mon[77242]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 7/72 objects degraded (9.722%), 4 pgs degraded) 2022-09-29T05:31:51.714 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:31:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:31:51] "GET /metrics HTTP/1.1" 200 196871 "" "Prometheus/2.33.4" 2022-09-29T05:31:52.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:52 smithi153 ceph-mon[70044]: pgmap v85: 105 pgs: 105 active+clean; 583 KiB data, 54 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:31:52.559 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:31:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:31:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:31:52.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:52 smithi083 ceph-mon[79930]: pgmap v85: 105 pgs: 105 active+clean; 583 KiB data, 54 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:31:52.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:52 smithi083 ceph-mon[77242]: pgmap v85: 105 pgs: 105 active+clean; 583 KiB data, 54 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:31:53.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:31:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:31:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:31:54.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:54 smithi153 ceph-mon[70044]: pgmap v86: 105 pgs: 105 active+clean; 583 KiB data, 54 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:31:54.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:54 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:54.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:54 smithi083 ceph-mon[79930]: pgmap v86: 105 pgs: 105 active+clean; 583 KiB data, 54 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:31:54.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:54 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:54.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:54 smithi083 ceph-mon[77242]: pgmap v86: 105 pgs: 105 active+clean; 583 KiB data, 54 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:31:54.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:54 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:31:55.312 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T05:31:55.601 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T05:31:55.602 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5m) 10s ago 6m 26.4M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T05:31:55.602 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5m) 99s ago 5m 60.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T05:31:55.602 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6m) 10s ago 6m 15.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T05:31:55.603 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6m) 99s ago 6m 18.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T05:31:55.603 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2m) 99s ago 9m 427M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T05:31:55.603 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3m) 10s ago 11m 488M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T05:31:55.603 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2m) 10s ago 11m 82.8M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T05:31:55.603 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (108s) 99s ago 9m 30.0M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T05:31:55.604 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2m) 10s ago 9m 67.9M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T05:31:55.604 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6m) 10s ago 6m 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T05:31:55.604 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6m) 99s ago 6m 24.7M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T05:31:55.604 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (66s) 10s ago 9m 58.7M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T05:31:55.605 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (33s) 10s ago 8m 51.4M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T05:31:55.605 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (15s) 10s ago 8m 13.1M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T05:31:55.605 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8m) 10s ago 8m 73.8M 2500M 16.2.10 c56e13ff90f2 4863a2083356 2022-09-29T05:31:55.605 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8m) 99s ago 8m 58.3M 2500M 16.2.10 c56e13ff90f2 ed61dc9353b0 2022-09-29T05:31:55.606 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7m) 99s ago 7m 66.5M 2500M 16.2.10 c56e13ff90f2 59f19fd15fed 2022-09-29T05:31:55.606 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7m) 99s ago 7m 68.0M 2500M 16.2.10 c56e13ff90f2 23decfd9c6b2 2022-09-29T05:31:55.606 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7m) 99s ago 7m 72.0M 2500M 16.2.10 c56e13ff90f2 23460a09c0a7 2022-09-29T05:31:55.606 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2m) 99s ago 6m 66.9M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T05:31:55.959 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T05:31:55.960 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T05:31:55.960 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T05:31:55.960 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:31:55.960 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T05:31:55.960 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T05:31:55.961 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:31:55.961 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T05:31:55.961 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 5, 2022-09-29T05:31:55.961 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T05:31:55.962 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:31:55.962 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T05:31:55.962 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T05:31:55.962 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:31:55.962 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T05:31:55.963 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 7, 2022-09-29T05:31:55.963 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T05:31:55.963 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T05:31:55.963 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T05:31:56.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:56 smithi083 ceph-mon[77242]: pgmap v87: 105 pgs: 105 active+clean; 583 KiB data, 54 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:31:56.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:56 smithi083 ceph-mon[77242]: from='client.44173 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:31:56.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:56 smithi083 ceph-mon[77242]: from='client.44179 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:31:56.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:56 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2495760106' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:31:56.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:56 smithi083 ceph-mon[79930]: pgmap v87: 105 pgs: 105 active+clean; 583 KiB data, 54 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:31:56.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:56 smithi083 ceph-mon[79930]: from='client.44173 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:31:56.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:56 smithi083 ceph-mon[79930]: from='client.44179 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:31:56.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:56 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2495760106' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:31:57.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:56 smithi153 ceph-mon[70044]: pgmap v87: 105 pgs: 105 active+clean; 583 KiB data, 54 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:31:57.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:56 smithi153 ceph-mon[70044]: from='client.44173 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:31:57.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:56 smithi153 ceph-mon[70044]: from='client.44179 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:31:57.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:56 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2495760106' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:31:58.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:58 smithi083 ceph-mon[79930]: pgmap v88: 105 pgs: 105 active+clean; 583 KiB data, 54 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:31:58.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:58 smithi083 ceph-mon[77242]: pgmap v88: 105 pgs: 105 active+clean; 583 KiB data, 54 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:31:59.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:58 smithi153 ceph-mon[70044]: pgmap v88: 105 pgs: 105 active+clean; 583 KiB data, 54 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:31:59.060 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:31:58 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:31:58.660Z caller=alerting.go:353 level=warn component="rule manager" alert="unsupported value type" msg="Expanding alert template failed" err="error executing template __alert_CephOSDDown: template: __alert_CephOSDDown:1:358: executing \"__alert_CephOSDDown\" at : error calling query: found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi083:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi153.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}];many-to-many matching not allowed: matching labels must be unique on one side" data="unsupported value type" 2022-09-29T05:31:59.060 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:31:58 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:31:58.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi083:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi153.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:31:59.060 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:31:58 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:31:58.662Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephPGImbalance\nexpr: abs(((ceph_osd_numpg > 0) - on(job) group_left() avg by(job) (ceph_osd_numpg\n > 0)) / on(job) group_left() avg by(job) (ceph_osd_numpg > 0)) * on(ceph_daemon)\n group_left(hostname) ceph_osd_metadata > 0.3\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.5\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} deviates by\n more than 30% from average PG count.\n summary: PGs are not balanced across OSDs\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi083:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi153.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:32:00.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:59 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:32:00.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:31:59 smithi083 ceph-mon[79930]: pgmap v89: 105 pgs: 105 active+clean; 583 KiB data, 54 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:32:00.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:59 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:32:00.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:31:59 smithi083 ceph-mon[77242]: pgmap v89: 105 pgs: 105 active+clean; 583 KiB data, 54 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:32:00.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:59 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:32:00.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:31:59 smithi153 ceph-mon[70044]: pgmap v89: 105 pgs: 105 active+clean; 583 KiB data, 54 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:32:01.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:32:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:32:01] "GET /metrics HTTP/1.1" 200 201445 "" "Prometheus/2.33.4" 2022-09-29T05:32:02.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:02 smithi153 ceph-mon[70044]: pgmap v90: 105 pgs: 105 active+clean; 583 KiB data, 54 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:32:02.560 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:32:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:32:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:32:02.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:02 smithi083 ceph-mon[77242]: pgmap v90: 105 pgs: 105 active+clean; 583 KiB data, 54 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:32:02.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:02 smithi083 ceph-mon[79930]: pgmap v90: 105 pgs: 105 active+clean; 583 KiB data, 54 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:32:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:32:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:32:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:32:04.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:04 smithi153 ceph-mon[70044]: pgmap v91: 105 pgs: 105 active+clean; 583 KiB data, 54 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:32:04.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:04 smithi083 ceph-mon[77242]: pgmap v91: 105 pgs: 105 active+clean; 583 KiB data, 54 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:32:04.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:04 smithi083 ceph-mon[79930]: pgmap v91: 105 pgs: 105 active+clean; 583 KiB data, 54 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:32:05.555 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:05 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2022-09-29T05:32:05.556 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:05 smithi083 ceph-mon[77242]: from='mon.1 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2022-09-29T05:32:05.556 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:05 smithi083 ceph-mon[77242]: Upgrade: osd.3 is safe to restart 2022-09-29T05:32:05.556 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:05 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2022-09-29T05:32:05.557 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:05 smithi083 ceph-mon[79930]: from='mon.1 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2022-09-29T05:32:05.557 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:05 smithi083 ceph-mon[79930]: Upgrade: osd.3 is safe to restart 2022-09-29T05:32:05.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:05 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2022-09-29T05:32:05.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:05 smithi153 ceph-mon[70044]: from='mon.1 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2022-09-29T05:32:05.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:05 smithi153 ceph-mon[70044]: Upgrade: osd.3 is safe to restart 2022-09-29T05:32:06.942 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:06 smithi083 ceph-mon[79930]: pgmap v92: 105 pgs: 105 active+clean; 583 KiB data, 54 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:32:06.942 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:06 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:32:06.942 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:06 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2022-09-29T05:32:06.943 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:06 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:32:06.943 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:06 smithi083 ceph-mon[77242]: pgmap v92: 105 pgs: 105 active+clean; 583 KiB data, 54 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:32:06.943 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:06 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:32:06.944 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:06 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2022-09-29T05:32:06.944 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:06 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:32:07.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:06 smithi153 ceph-mon[70044]: pgmap v92: 105 pgs: 105 active+clean; 583 KiB data, 54 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:32:07.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:06 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:32:07.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:06 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2022-09-29T05:32:07.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:06 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:32:07.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:07 smithi083 ceph-mon[79930]: Upgrade: Updating osd.3 2022-09-29T05:32:07.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:07 smithi083 ceph-mon[79930]: Deploying daemon osd.3 on smithi083 2022-09-29T05:32:07.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:07 smithi083 ceph-mon[77242]: Upgrade: Updating osd.3 2022-09-29T05:32:07.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:07 smithi083 ceph-mon[77242]: Deploying daemon osd.3 on smithi083 2022-09-29T05:32:08.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:07 smithi153 ceph-mon[70044]: Upgrade: Updating osd.3 2022-09-29T05:32:08.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:07 smithi153 ceph-mon[70044]: Deploying daemon osd.3 on smithi083 2022-09-29T05:32:08.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:08 smithi083 ceph-mon[79930]: pgmap v93: 105 pgs: 105 active+clean; 583 KiB data, 54 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:32:08.964 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 29 05:32:08 smithi083 systemd[1]: Stopping Ceph osd.3 for 69111522-3fb6-11ed-8431-001a4aab830c... 2022-09-29T05:32:08.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:08 smithi083 ceph-mon[77242]: pgmap v93: 105 pgs: 105 active+clean; 583 KiB data, 54 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:32:09.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:08 smithi153 ceph-mon[70044]: pgmap v93: 105 pgs: 105 active+clean; 583 KiB data, 54 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:32:09.059 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:32:08 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:32:08.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi083:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi153.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:32:09.060 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:32:08 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:32:08.662Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephPGImbalance\nexpr: abs(((ceph_osd_numpg > 0) - on(job) group_left() avg by(job) (ceph_osd_numpg\n > 0)) / on(job) group_left() avg by(job) (ceph_osd_numpg > 0)) * on(ceph_daemon)\n group_left(hostname) ceph_osd_metadata > 0.3\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.5\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} deviates by\n more than 30% from average PG count.\n summary: PGs are not balanced across OSDs\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi083:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi153.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:32:09.463 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 29 05:32:09 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-osd-3[56220]: debug 2022-09-29T05:32:09.007+0000 7f040d29a700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2022-09-29T05:32:09.463 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 29 05:32:09 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-osd-3[56220]: debug 2022-09-29T05:32:09.007+0000 7f040d29a700 -1 osd.3 79 *** Got signal Terminated *** 2022-09-29T05:32:09.463 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 29 05:32:09 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-osd-3[56220]: debug 2022-09-29T05:32:09.007+0000 7f040d29a700 -1 osd.3 79 *** Immediate shutdown (osd_fast_shutdown=true) *** 2022-09-29T05:32:09.929 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:09 smithi083 ceph-mon[79930]: osd.3 marked itself down and dead 2022-09-29T05:32:09.929 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:09 smithi083 ceph-mon[77242]: osd.3 marked itself down and dead 2022-09-29T05:32:10.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:09 smithi153 ceph-mon[70044]: osd.3 marked itself down and dead 2022-09-29T05:32:10.213 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 29 05:32:10 smithi083 bash[91176]: ceph-69111522-3fb6-11ed-8431-001a4aab830c-osd-3 2022-09-29T05:32:10.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:10 smithi083 ceph-mon[77242]: pgmap v94: 105 pgs: 105 active+clean; 583 KiB data, 54 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:32:10.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:10 smithi083 ceph-mon[77242]: Health check failed: 1 osds down (OSD_DOWN) 2022-09-29T05:32:10.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:10 smithi083 ceph-mon[77242]: osdmap e80: 8 total, 7 up, 8 in 2022-09-29T05:32:10.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:10 smithi083 ceph-mon[79930]: pgmap v94: 105 pgs: 105 active+clean; 583 KiB data, 54 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:32:10.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:10 smithi083 ceph-mon[79930]: Health check failed: 1 osds down (OSD_DOWN) 2022-09-29T05:32:10.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:10 smithi083 ceph-mon[79930]: osdmap e80: 8 total, 7 up, 8 in 2022-09-29T05:32:11.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:10 smithi153 ceph-mon[70044]: pgmap v94: 105 pgs: 105 active+clean; 583 KiB data, 54 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:32:11.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:10 smithi153 ceph-mon[70044]: Health check failed: 1 osds down (OSD_DOWN) 2022-09-29T05:32:11.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:10 smithi153 ceph-mon[70044]: osdmap e80: 8 total, 7 up, 8 in 2022-09-29T05:32:11.789 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:32:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:32:11] "GET /metrics HTTP/1.1" 200 201436 "" "Prometheus/2.33.4" 2022-09-29T05:32:11.789 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:11 smithi083 ceph-mon[79930]: osdmap e81: 8 total, 7 up, 8 in 2022-09-29T05:32:11.790 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:11 smithi083 ceph-mon[77242]: osdmap e81: 8 total, 7 up, 8 in 2022-09-29T05:32:11.790 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 29 05:32:11 smithi083 systemd[1]: ceph-69111522-3fb6-11ed-8431-001a4aab830c@osd.3.service: Succeeded. 2022-09-29T05:32:11.791 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 29 05:32:11 smithi083 systemd[1]: Stopped Ceph osd.3 for 69111522-3fb6-11ed-8431-001a4aab830c. 2022-09-29T05:32:12.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:11 smithi153 ceph-mon[70044]: osdmap e81: 8 total, 7 up, 8 in 2022-09-29T05:32:12.682 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:32:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:32:12.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:32:12.963 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 29 05:32:11 smithi083 systemd[1]: Starting Ceph osd.3 for 69111522-3fb6-11ed-8431-001a4aab830c... 2022-09-29T05:32:12.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:12 smithi083 ceph-mon[79930]: pgmap v97: 105 pgs: 1 active+undersized+degraded, 20 peering, 3 stale+active+clean, 6 active+undersized, 75 active+clean; 583 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 1/72 objects degraded (1.389%) 2022-09-29T05:32:12.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:12 smithi083 ceph-mon[79930]: Health check failed: Reduced data availability: 7 pgs inactive, 7 pgs peering (PG_AVAILABILITY) 2022-09-29T05:32:12.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:12 smithi083 ceph-mon[79930]: Health check failed: Degraded data redundancy: 1/72 objects degraded (1.389%), 1 pg degraded (PG_DEGRADED) 2022-09-29T05:32:12.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:12 smithi083 ceph-mon[77242]: pgmap v97: 105 pgs: 1 active+undersized+degraded, 20 peering, 3 stale+active+clean, 6 active+undersized, 75 active+clean; 583 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 1/72 objects degraded (1.389%) 2022-09-29T05:32:12.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:12 smithi083 ceph-mon[77242]: Health check failed: Reduced data availability: 7 pgs inactive, 7 pgs peering (PG_AVAILABILITY) 2022-09-29T05:32:12.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:12 smithi083 ceph-mon[77242]: Health check failed: Degraded data redundancy: 1/72 objects degraded (1.389%), 1 pg degraded (PG_DEGRADED) 2022-09-29T05:32:13.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:12 smithi153 ceph-mon[70044]: pgmap v97: 105 pgs: 1 active+undersized+degraded, 20 peering, 3 stale+active+clean, 6 active+undersized, 75 active+clean; 583 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 1/72 objects degraded (1.389%) 2022-09-29T05:32:13.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:12 smithi153 ceph-mon[70044]: Health check failed: Reduced data availability: 7 pgs inactive, 7 pgs peering (PG_AVAILABILITY) 2022-09-29T05:32:13.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:12 smithi153 ceph-mon[70044]: Health check failed: Degraded data redundancy: 1/72 objects degraded (1.389%), 1 pg degraded (PG_DEGRADED) 2022-09-29T05:32:13.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:32:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:32:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:32:13.713 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 29 05:32:13 smithi083 bash[91427]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2022-09-29T05:32:13.713 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 29 05:32:13 smithi083 bash[91427]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2022-09-29T05:32:13.713 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 29 05:32:13 smithi083 bash[91427]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2022-09-29T05:32:13.714 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 29 05:32:13 smithi083 bash[91427]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2022-09-29T05:32:13.714 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 29 05:32:13 smithi083 bash[91427]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2022-09-29T05:32:13.714 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 29 05:32:13 smithi083 bash[91427]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2022-09-29T05:32:13.714 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 29 05:32:13 smithi083 bash[91427]: --> ceph-volume raw activate successful for osd ID: 3 2022-09-29T05:32:14.689 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:14 smithi083 ceph-mon[77242]: pgmap v98: 105 pgs: 1 active+undersized+degraded, 20 peering, 3 stale+active+clean, 6 active+undersized, 75 active+clean; 583 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 1/72 objects degraded (1.389%) 2022-09-29T05:32:14.689 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 29 05:32:14 smithi083 podman[91711]: 2022-09-29T05:32:14.689 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 29 05:32:14 smithi083 ceph-osd[91750]: -- 172.21.15.83:0/1498925307 <== mon.1 v2:172.21.15.83:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55e0b031a680 con 0x55e0b02df000 2022-09-29T05:32:14.690 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 29 05:32:14 smithi083 bash[91711]: f045a5c0be19a1e44c6659f4991784bbbe9a18dd7ed78fe5366f89335c6919e6 2022-09-29T05:32:14.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:14 smithi083 ceph-mon[79930]: pgmap v98: 105 pgs: 1 active+undersized+degraded, 20 peering, 3 stale+active+clean, 6 active+undersized, 75 active+clean; 583 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 1/72 objects degraded (1.389%) 2022-09-29T05:32:14.964 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 29 05:32:14 smithi083 systemd[1]: Started Ceph osd.3 for 69111522-3fb6-11ed-8431-001a4aab830c. 2022-09-29T05:32:15.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:14 smithi153 ceph-mon[70044]: pgmap v98: 105 pgs: 1 active+undersized+degraded, 20 peering, 3 stale+active+clean, 6 active+undersized, 75 active+clean; 583 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 1/72 objects degraded (1.389%) 2022-09-29T05:32:15.463 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 29 05:32:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-osd-3[91727]: 2022-09-29T05:32:15.133+0000 7fb021f115c0 -1 Falling back to public interface 2022-09-29T05:32:16.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:15 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:32:16.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:15 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:32:16.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:15 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:32:16.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:15 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:32:16.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:15 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:32:16.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:15 smithi153 ceph-mon[70044]: pgmap v99: 105 pgs: 4 active+undersized+degraded, 33 active+undersized, 68 active+clean; 583 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 8/72 objects degraded (11.111%) 2022-09-29T05:32:16.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:15 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:32:16.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:15 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:32:16.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:15 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:32:16.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:15 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:32:16.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:15 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:32:16.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:15 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:32:16.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:15 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:32:16.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:15 smithi083 ceph-mon[79930]: pgmap v99: 105 pgs: 4 active+undersized+degraded, 33 active+undersized, 68 active+clean; 583 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 8/72 objects degraded (11.111%) 2022-09-29T05:32:16.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:15 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:32:16.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:15 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:32:16.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:15 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:32:16.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:15 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:32:16.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:15 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:32:16.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:15 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:32:16.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:15 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:32:16.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:15 smithi083 ceph-mon[77242]: pgmap v99: 105 pgs: 4 active+undersized+degraded, 33 active+undersized, 68 active+clean; 583 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 8/72 objects degraded (11.111%) 2022-09-29T05:32:16.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:15 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:32:16.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:15 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:32:17.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:16 smithi153 ceph-mon[70044]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 7 pgs inactive, 7 pgs peering) 2022-09-29T05:32:17.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:16 smithi083 ceph-mon[79930]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 7 pgs inactive, 7 pgs peering) 2022-09-29T05:32:17.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:16 smithi083 ceph-mon[77242]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 7 pgs inactive, 7 pgs peering) 2022-09-29T05:32:18.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:17 smithi153 ceph-mon[70044]: pgmap v100: 105 pgs: 4 active+undersized+degraded, 33 active+undersized, 68 active+clean; 583 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 8/72 objects degraded (11.111%) 2022-09-29T05:32:18.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:17 smithi083 ceph-mon[79930]: pgmap v100: 105 pgs: 4 active+undersized+degraded, 33 active+undersized, 68 active+clean; 583 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 8/72 objects degraded (11.111%) 2022-09-29T05:32:18.214 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 29 05:32:17 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-osd-3[91727]: 2022-09-29T05:32:17.876+0000 7fb021f115c0 -1 osd.3 79 log_to_monitors true 2022-09-29T05:32:18.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:17 smithi083 ceph-mon[77242]: pgmap v100: 105 pgs: 4 active+undersized+degraded, 33 active+undersized, 68 active+clean; 583 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 8/72 objects degraded (11.111%) 2022-09-29T05:32:19.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:18 smithi153 ceph-mon[70044]: from='osd.3 [v2:172.21.15.83:6826/494706257,v1:172.21.15.83:6827/494706257]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2022-09-29T05:32:19.059 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:32:18 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:32:18.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi083:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi153.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:32:19.060 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:32:18 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:32:18.662Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephPGImbalance\nexpr: abs(((ceph_osd_numpg > 0) - on(job) group_left() avg by(job) (ceph_osd_numpg\n > 0)) / on(job) group_left() avg by(job) (ceph_osd_numpg > 0)) * on(ceph_daemon)\n group_left(hostname) ceph_osd_metadata > 0.3\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.5\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} deviates by\n more than 30% from average PG count.\n summary: PGs are not balanced across OSDs\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi083:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi153.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:32:19.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:18 smithi083 ceph-mon[79930]: from='osd.3 [v2:172.21.15.83:6826/494706257,v1:172.21.15.83:6827/494706257]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2022-09-29T05:32:19.213 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 29 05:32:18 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-osd-3[91727]: 2022-09-29T05:32:18.781+0000 7fb011e93700 -1 osd.3 79 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2022-09-29T05:32:19.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:18 smithi083 ceph-mon[77242]: from='osd.3 [v2:172.21.15.83:6826/494706257,v1:172.21.15.83:6827/494706257]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2022-09-29T05:32:20.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:19 smithi153 ceph-mon[70044]: from='osd.3 [v2:172.21.15.83:6826/494706257,v1:172.21.15.83:6827/494706257]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2022-09-29T05:32:20.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:19 smithi153 ceph-mon[70044]: osdmap e82: 8 total, 7 up, 8 in 2022-09-29T05:32:20.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:19 smithi153 ceph-mon[70044]: from='osd.3 [v2:172.21.15.83:6826/494706257,v1:172.21.15.83:6827/494706257]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]: dispatch 2022-09-29T05:32:20.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:19 smithi153 ceph-mon[70044]: pgmap v102: 105 pgs: 4 active+undersized+degraded, 33 active+undersized, 68 active+clean; 583 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 8/72 objects degraded (11.111%) 2022-09-29T05:32:20.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:19 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:32:20.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:19 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:32:20.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:19 smithi153 ceph-mon[70044]: Health check update: Degraded data redundancy: 8/72 objects degraded (11.111%), 4 pgs degraded (PG_DEGRADED) 2022-09-29T05:32:20.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:19 smithi083 ceph-mon[77242]: from='osd.3 [v2:172.21.15.83:6826/494706257,v1:172.21.15.83:6827/494706257]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2022-09-29T05:32:20.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:19 smithi083 ceph-mon[77242]: osdmap e82: 8 total, 7 up, 8 in 2022-09-29T05:32:20.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:19 smithi083 ceph-mon[77242]: from='osd.3 [v2:172.21.15.83:6826/494706257,v1:172.21.15.83:6827/494706257]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]: dispatch 2022-09-29T05:32:20.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:19 smithi083 ceph-mon[77242]: pgmap v102: 105 pgs: 4 active+undersized+degraded, 33 active+undersized, 68 active+clean; 583 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 8/72 objects degraded (11.111%) 2022-09-29T05:32:20.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:19 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:32:20.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:19 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:32:20.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:19 smithi083 ceph-mon[77242]: Health check update: Degraded data redundancy: 8/72 objects degraded (11.111%), 4 pgs degraded (PG_DEGRADED) 2022-09-29T05:32:20.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:19 smithi083 ceph-mon[79930]: from='osd.3 [v2:172.21.15.83:6826/494706257,v1:172.21.15.83:6827/494706257]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2022-09-29T05:32:20.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:19 smithi083 ceph-mon[79930]: osdmap e82: 8 total, 7 up, 8 in 2022-09-29T05:32:20.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:19 smithi083 ceph-mon[79930]: from='osd.3 [v2:172.21.15.83:6826/494706257,v1:172.21.15.83:6827/494706257]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]: dispatch 2022-09-29T05:32:20.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:19 smithi083 ceph-mon[79930]: pgmap v102: 105 pgs: 4 active+undersized+degraded, 33 active+undersized, 68 active+clean; 583 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 8/72 objects degraded (11.111%) 2022-09-29T05:32:20.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:19 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:32:20.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:19 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:32:20.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:19 smithi083 ceph-mon[79930]: Health check update: Degraded data redundancy: 8/72 objects degraded (11.111%), 4 pgs degraded (PG_DEGRADED) 2022-09-29T05:32:21.036 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:20 smithi083 ceph-mon[77242]: Health check cleared: OSD_DOWN (was: 1 osds down) 2022-09-29T05:32:21.036 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:20 smithi083 ceph-mon[77242]: osd.3 [v2:172.21.15.83:6826/494706257,v1:172.21.15.83:6827/494706257] boot 2022-09-29T05:32:21.036 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:20 smithi083 ceph-mon[77242]: osdmap e83: 8 total, 8 up, 8 in 2022-09-29T05:32:21.037 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:20 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2022-09-29T05:32:21.037 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:20 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:32:21.037 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:20 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:32:21.038 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:20 smithi083 ceph-mon[79930]: Health check cleared: OSD_DOWN (was: 1 osds down) 2022-09-29T05:32:21.038 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:20 smithi083 ceph-mon[79930]: osd.3 [v2:172.21.15.83:6826/494706257,v1:172.21.15.83:6827/494706257] boot 2022-09-29T05:32:21.038 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:20 smithi083 ceph-mon[79930]: osdmap e83: 8 total, 8 up, 8 in 2022-09-29T05:32:21.038 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:20 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2022-09-29T05:32:21.039 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:20 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:32:21.039 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:20 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:32:21.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:20 smithi153 ceph-mon[70044]: Health check cleared: OSD_DOWN (was: 1 osds down) 2022-09-29T05:32:21.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:20 smithi153 ceph-mon[70044]: osd.3 [v2:172.21.15.83:6826/494706257,v1:172.21.15.83:6827/494706257] boot 2022-09-29T05:32:21.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:20 smithi153 ceph-mon[70044]: osdmap e83: 8 total, 8 up, 8 in 2022-09-29T05:32:21.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:20 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2022-09-29T05:32:21.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:20 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:32:21.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:20 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:32:21.650 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:32:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:32:21] "GET /metrics HTTP/1.1" 200 201436 "" "Prometheus/2.33.4" 2022-09-29T05:32:21.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:21 smithi083 ceph-mon[79930]: osdmap e84: 8 total, 8 up, 8 in 2022-09-29T05:32:21.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:21 smithi083 ceph-mon[79930]: pgmap v105: 105 pgs: 9 peering, 2 active+undersized+degraded, 20 active+undersized, 74 active+clean; 583 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2022-09-29T05:32:21.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:21 smithi083 ceph-mon[77242]: osdmap e84: 8 total, 8 up, 8 in 2022-09-29T05:32:21.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:21 smithi083 ceph-mon[77242]: pgmap v105: 105 pgs: 9 peering, 2 active+undersized+degraded, 20 active+undersized, 74 active+clean; 583 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2022-09-29T05:32:22.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:21 smithi153 ceph-mon[70044]: osdmap e84: 8 total, 8 up, 8 in 2022-09-29T05:32:22.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:21 smithi153 ceph-mon[70044]: pgmap v105: 105 pgs: 9 peering, 2 active+undersized+degraded, 20 active+undersized, 74 active+clean; 583 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2022-09-29T05:32:22.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:32:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:32:22.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:32:23.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:32:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:32:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:32:23.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:32:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: 2022-09-29T05:32:23.422+0000 7f9b26e95700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2022-09-29T05:32:24.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:24 smithi083 ceph-mon[79930]: pgmap v106: 105 pgs: 9 peering, 2 active+undersized+degraded, 20 active+undersized, 74 active+clean; 583 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2022-09-29T05:32:24.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:24 smithi083 ceph-mon[79930]: Detected new or changed devices on smithi083 2022-09-29T05:32:24.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:24 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:32:24.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:24 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:32:24.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:24 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:32:24.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:24 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:32:24.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:24 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:32:24.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:24 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:32:24.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:24 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:32:24.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:24 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2022-09-29T05:32:24.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:24 smithi083 ceph-mon[79930]: from='mon.1 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2022-09-29T05:32:24.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:24 smithi083 ceph-mon[79930]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2022-09-29T05:32:24.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:24 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:32:24.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:24 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:32:24.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:24 smithi083 ceph-mon[77242]: pgmap v106: 105 pgs: 9 peering, 2 active+undersized+degraded, 20 active+undersized, 74 active+clean; 583 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2022-09-29T05:32:24.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:24 smithi083 ceph-mon[77242]: Detected new or changed devices on smithi083 2022-09-29T05:32:24.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:24 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:32:24.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:24 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:32:24.718 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:24 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:32:24.718 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:24 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:32:24.718 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:24 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:32:24.718 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:24 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:32:24.719 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:24 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:32:24.719 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:24 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2022-09-29T05:32:24.719 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:24 smithi083 ceph-mon[77242]: from='mon.1 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2022-09-29T05:32:24.719 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:24 smithi083 ceph-mon[77242]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2022-09-29T05:32:24.720 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:24 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:32:24.720 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:24 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:32:24.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:24 smithi153 ceph-mon[70044]: pgmap v106: 105 pgs: 9 peering, 2 active+undersized+degraded, 20 active+undersized, 74 active+clean; 583 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2022-09-29T05:32:24.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:24 smithi153 ceph-mon[70044]: Detected new or changed devices on smithi083 2022-09-29T05:32:24.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:24 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:32:24.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:24 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:32:24.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:24 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:32:24.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:24 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:32:24.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:24 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:32:24.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:24 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:32:24.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:24 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:32:24.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:24 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2022-09-29T05:32:24.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:24 smithi153 ceph-mon[70044]: from='mon.1 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2022-09-29T05:32:24.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:24 smithi153 ceph-mon[70044]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2022-09-29T05:32:24.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:24 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:32:24.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:24 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:32:25.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:25 smithi083 ceph-mon[79930]: Health check update: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded (PG_DEGRADED) 2022-09-29T05:32:25.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:25 smithi083 ceph-mon[77242]: Health check update: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded (PG_DEGRADED) 2022-09-29T05:32:25.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:25 smithi153 ceph-mon[70044]: Health check update: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded (PG_DEGRADED) 2022-09-29T05:32:26.284 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T05:32:26.576 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T05:32:26.576 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6m) 7s ago 6m 26.4M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T05:32:26.577 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6m) 2m ago 6m 60.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T05:32:26.577 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7m) 7s ago 7m 15.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T05:32:26.577 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7m) 2m ago 7m 18.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T05:32:26.577 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3m) 2m ago 10m 427M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T05:32:26.577 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3m) 7s ago 11m 490M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T05:32:26.578 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2m) 7s ago 11m 86.0M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T05:32:26.578 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2m) 2m ago 10m 30.0M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T05:32:26.578 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2m) 7s ago 10m 70.7M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T05:32:26.578 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6m) 7s ago 6m 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T05:32:26.579 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6m) 2m ago 6m 24.7M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T05:32:26.579 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (97s) 7s ago 9m 63.4M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T05:32:26.579 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (64s) 7s ago 9m 56.5M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T05:32:26.579 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (46s) 7s ago 9m 51.9M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T05:32:26.579 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (12s) 7s ago 8m 12.9M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T05:32:26.580 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8m) 2m ago 8m 58.3M 2500M 16.2.10 c56e13ff90f2 ed61dc9353b0 2022-09-29T05:32:26.580 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8m) 2m ago 8m 66.5M 2500M 16.2.10 c56e13ff90f2 59f19fd15fed 2022-09-29T05:32:26.580 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8m) 2m ago 8m 68.0M 2500M 16.2.10 c56e13ff90f2 23decfd9c6b2 2022-09-29T05:32:26.580 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7m) 2m ago 7m 72.0M 2500M 16.2.10 c56e13ff90f2 23460a09c0a7 2022-09-29T05:32:26.580 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3m) 2m ago 6m 66.9M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T05:32:26.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:26 smithi083 ceph-mon[77242]: pgmap v107: 105 pgs: 105 active+clean; 583 KiB data, 55 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:32:26.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:26 smithi083 ceph-mon[77242]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded) 2022-09-29T05:32:26.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:26 smithi083 ceph-mon[79930]: pgmap v107: 105 pgs: 105 active+clean; 583 KiB data, 55 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:32:26.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:26 smithi083 ceph-mon[79930]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded) 2022-09-29T05:32:26.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:26 smithi153 ceph-mon[70044]: pgmap v107: 105 pgs: 105 active+clean; 583 KiB data, 55 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:32:26.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:26 smithi153 ceph-mon[70044]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded) 2022-09-29T05:32:26.933 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T05:32:26.933 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T05:32:26.933 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T05:32:26.933 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:32:26.934 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T05:32:26.934 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T05:32:26.934 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:32:26.934 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T05:32:26.935 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 4, 2022-09-29T05:32:26.935 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 4 2022-09-29T05:32:26.935 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:32:26.935 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T05:32:26.935 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T05:32:26.936 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:32:26.936 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T05:32:26.936 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 6, 2022-09-29T05:32:26.936 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 9 2022-09-29T05:32:26.937 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T05:32:26.937 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T05:32:27.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:27 smithi083 ceph-mon[79930]: from='client.34263 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:32:27.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:27 smithi083 ceph-mon[79930]: from='client.44203 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:32:27.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:27 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2536569828' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:32:27.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:27 smithi083 ceph-mon[77242]: from='client.34263 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:32:27.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:27 smithi083 ceph-mon[77242]: from='client.44203 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:32:27.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:27 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2536569828' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:32:27.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:27 smithi153 ceph-mon[70044]: from='client.34263 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:32:27.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:27 smithi153 ceph-mon[70044]: from='client.44203 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:32:27.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:27 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2536569828' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:32:28.664 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:28 smithi153 ceph-mon[70044]: pgmap v108: 105 pgs: 105 active+clean; 583 KiB data, 55 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:32:28.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:28 smithi083 ceph-mon[79930]: pgmap v108: 105 pgs: 105 active+clean; 583 KiB data, 55 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:32:28.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:28 smithi083 ceph-mon[77242]: pgmap v108: 105 pgs: 105 active+clean; 583 KiB data, 55 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:32:29.059 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:32:28 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:32:28.662Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi083:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi153.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:32:29.201 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:32:28 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:32:28.662Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephPGImbalance\nexpr: abs(((ceph_osd_numpg > 0) - on(job) group_left() avg by(job) (ceph_osd_numpg\n > 0)) / on(job) group_left() avg by(job) (ceph_osd_numpg > 0)) * on(ceph_daemon)\n group_left(hostname) ceph_osd_metadata > 0.3\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.5\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} deviates by\n more than 30% from average PG count.\n summary: PGs are not balanced across OSDs\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi083:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi153.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:32:30.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:30 smithi083 ceph-mon[77242]: pgmap v109: 105 pgs: 105 active+clean; 583 KiB data, 55 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:32:30.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:30 smithi083 ceph-mon[79930]: pgmap v109: 105 pgs: 105 active+clean; 583 KiB data, 55 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:32:31.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:30 smithi153 ceph-mon[70044]: pgmap v109: 105 pgs: 105 active+clean; 583 KiB data, 55 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:32:31.972 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:32:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:32:31] "GET /metrics HTTP/1.1" 200 206425 "" "Prometheus/2.33.4" 2022-09-29T05:32:32.658 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:32:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:32:32.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:32:32.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:32 smithi083 ceph-mon[77242]: pgmap v110: 105 pgs: 105 active+clean; 583 KiB data, 55 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:32:32.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:32 smithi083 ceph-mon[79930]: pgmap v110: 105 pgs: 105 active+clean; 583 KiB data, 55 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:32:33.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:32 smithi153 ceph-mon[70044]: pgmap v110: 105 pgs: 105 active+clean; 583 KiB data, 55 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:32:33.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:32:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:32:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:32:34.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:34 smithi083 ceph-mon[79930]: pgmap v111: 105 pgs: 105 active+clean; 583 KiB data, 55 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:32:34.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:34 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:32:34.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:34 smithi083 ceph-mon[77242]: pgmap v111: 105 pgs: 105 active+clean; 583 KiB data, 55 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:32:34.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:34 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:32:35.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:34 smithi153 ceph-mon[70044]: pgmap v111: 105 pgs: 105 active+clean; 583 KiB data, 55 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:32:35.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:34 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:32:36.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:36 smithi083 ceph-mon[79930]: pgmap v112: 105 pgs: 105 active+clean; 583 KiB data, 55 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:32:36.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:36 smithi083 ceph-mon[77242]: pgmap v112: 105 pgs: 105 active+clean; 583 KiB data, 55 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:32:37.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:36 smithi153 ceph-mon[70044]: pgmap v112: 105 pgs: 105 active+clean; 583 KiB data, 55 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:32:38.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:38 smithi083 ceph-mon[79930]: pgmap v113: 105 pgs: 105 active+clean; 583 KiB data, 55 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:32:38.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:38 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2022-09-29T05:32:38.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:38 smithi083 ceph-mon[77242]: pgmap v113: 105 pgs: 105 active+clean; 583 KiB data, 55 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:32:38.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:38 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2022-09-29T05:32:39.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:38 smithi153 ceph-mon[70044]: pgmap v113: 105 pgs: 105 active+clean; 583 KiB data, 55 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:32:39.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:38 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2022-09-29T05:32:39.059 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:32:38 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:32:38.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi083:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi153.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:32:39.060 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:32:38 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:32:38.662Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephPGImbalance\nexpr: abs(((ceph_osd_numpg > 0) - on(job) group_left() avg by(job) (ceph_osd_numpg\n > 0)) / on(job) group_left() avg by(job) (ceph_osd_numpg > 0)) * on(ceph_daemon)\n group_left(hostname) ceph_osd_metadata > 0.3\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.5\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} deviates by\n more than 30% from average PG count.\n summary: PGs are not balanced across OSDs\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi083:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi153.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:32:39.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:39 smithi153 ceph-mon[70044]: from='mon.1 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2022-09-29T05:32:39.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:39 smithi153 ceph-mon[70044]: Upgrade: osd.4 is safe to restart 2022-09-29T05:32:39.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:32:39.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:32:39.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:32:39.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:32:39.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:32:39.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:39 smithi083 ceph-mon[79930]: from='mon.1 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2022-09-29T05:32:39.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:39 smithi083 ceph-mon[79930]: Upgrade: osd.4 is safe to restart 2022-09-29T05:32:39.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:32:39.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:32:39.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:32:39.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:32:39.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:32:39.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:39 smithi083 ceph-mon[77242]: from='mon.1 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2022-09-29T05:32:39.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:39 smithi083 ceph-mon[77242]: Upgrade: osd.4 is safe to restart 2022-09-29T05:32:39.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:32:39.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:32:39.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:32:39.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:32:39.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:32:40.776 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:40 smithi153 ceph-mon[70044]: pgmap v114: 105 pgs: 105 active+clean; 583 KiB data, 55 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:32:40.776 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:32:40.777 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2022-09-29T05:32:40.777 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:32:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:40 smithi083 ceph-mon[79930]: pgmap v114: 105 pgs: 105 active+clean; 583 KiB data, 55 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:32:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:32:40.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2022-09-29T05:32:40.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:32:40.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:40 smithi083 ceph-mon[77242]: pgmap v114: 105 pgs: 105 active+clean; 583 KiB data, 55 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:32:40.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:32:40.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2022-09-29T05:32:40.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:32:41.897 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:41 smithi153 ceph-mon[70044]: Upgrade: Updating osd.4 2022-09-29T05:32:41.897 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:41 smithi153 ceph-mon[70044]: Deploying daemon osd.4 on smithi153 2022-09-29T05:32:41.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:32:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:32:41] "GET /metrics HTTP/1.1" 200 206218 "" "Prometheus/2.33.4" 2022-09-29T05:32:41.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:41 smithi083 ceph-mon[79930]: Upgrade: Updating osd.4 2022-09-29T05:32:41.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:41 smithi083 ceph-mon[79930]: Deploying daemon osd.4 on smithi153 2022-09-29T05:32:41.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:41 smithi083 ceph-mon[77242]: Upgrade: Updating osd.4 2022-09-29T05:32:41.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:41 smithi083 ceph-mon[77242]: Deploying daemon osd.4 on smithi153 2022-09-29T05:32:42.309 INFO:journalctl@ceph.osd.4.smithi153.stdout:Sep 29 05:32:42 smithi153 systemd[1]: Stopping Ceph osd.4 for 69111522-3fb6-11ed-8431-001a4aab830c... 2022-09-29T05:32:42.692 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:32:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:32:42.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:32:42.692 INFO:journalctl@ceph.osd.4.smithi153.stdout:Sep 29 05:32:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-osd-4[42555]: debug 2022-09-29T05:32:42.399+0000 7fa62807c700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2022-09-29T05:32:42.693 INFO:journalctl@ceph.osd.4.smithi153.stdout:Sep 29 05:32:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-osd-4[42555]: debug 2022-09-29T05:32:42.400+0000 7fa62807c700 -1 osd.4 84 *** Got signal Terminated *** 2022-09-29T05:32:42.693 INFO:journalctl@ceph.osd.4.smithi153.stdout:Sep 29 05:32:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-osd-4[42555]: debug 2022-09-29T05:32:42.400+0000 7fa62807c700 -1 osd.4 84 *** Immediate shutdown (osd_fast_shutdown=true) *** 2022-09-29T05:32:42.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:42 smithi083 ceph-mon[79930]: pgmap v115: 105 pgs: 105 active+clean; 583 KiB data, 55 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:32:42.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:42 smithi083 ceph-mon[79930]: osd.4 marked itself down and dead 2022-09-29T05:32:42.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:42 smithi083 ceph-mon[77242]: pgmap v115: 105 pgs: 105 active+clean; 583 KiB data, 55 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:32:42.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:42 smithi083 ceph-mon[77242]: osd.4 marked itself down and dead 2022-09-29T05:32:43.006 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:42 smithi153 ceph-mon[70044]: pgmap v115: 105 pgs: 105 active+clean; 583 KiB data, 55 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:32:43.006 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:42 smithi153 ceph-mon[70044]: osd.4 marked itself down and dead 2022-09-29T05:32:43.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:32:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:32:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:32:43.309 INFO:journalctl@ceph.osd.4.smithi153.stdout:Sep 29 05:32:43 smithi153 bash[73274]: ceph-69111522-3fb6-11ed-8431-001a4aab830c-osd-4 2022-09-29T05:32:43.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:43 smithi083 ceph-mon[77242]: Health check failed: 1 osds down (OSD_DOWN) 2022-09-29T05:32:43.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:43 smithi083 ceph-mon[77242]: osdmap e85: 8 total, 7 up, 8 in 2022-09-29T05:32:43.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:43 smithi083 ceph-mon[77242]: osdmap e86: 8 total, 7 up, 8 in 2022-09-29T05:32:43.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:43 smithi083 ceph-mon[79930]: Health check failed: 1 osds down (OSD_DOWN) 2022-09-29T05:32:43.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:43 smithi083 ceph-mon[79930]: osdmap e85: 8 total, 7 up, 8 in 2022-09-29T05:32:43.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:43 smithi083 ceph-mon[79930]: osdmap e86: 8 total, 7 up, 8 in 2022-09-29T05:32:44.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:43 smithi153 ceph-mon[70044]: Health check failed: 1 osds down (OSD_DOWN) 2022-09-29T05:32:44.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:43 smithi153 ceph-mon[70044]: osdmap e85: 8 total, 7 up, 8 in 2022-09-29T05:32:44.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:43 smithi153 ceph-mon[70044]: osdmap e86: 8 total, 7 up, 8 in 2022-09-29T05:32:44.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:44 smithi153 ceph-mon[70044]: pgmap v117: 105 pgs: 7 stale+active+clean, 98 active+clean; 583 KiB data, 55 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:32:44.810 INFO:journalctl@ceph.osd.4.smithi153.stdout:Sep 29 05:32:44 smithi153 systemd[1]: ceph-69111522-3fb6-11ed-8431-001a4aab830c@osd.4.service: Succeeded. 2022-09-29T05:32:44.810 INFO:journalctl@ceph.osd.4.smithi153.stdout:Sep 29 05:32:44 smithi153 systemd[1]: Stopped Ceph osd.4 for 69111522-3fb6-11ed-8431-001a4aab830c. 2022-09-29T05:32:44.811 INFO:journalctl@ceph.osd.4.smithi153.stdout:Sep 29 05:32:44 smithi153 systemd[1]: Starting Ceph osd.4 for 69111522-3fb6-11ed-8431-001a4aab830c... 2022-09-29T05:32:44.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:44 smithi083 ceph-mon[79930]: pgmap v117: 105 pgs: 7 stale+active+clean, 98 active+clean; 583 KiB data, 55 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:32:44.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:44 smithi083 ceph-mon[77242]: pgmap v117: 105 pgs: 7 stale+active+clean, 98 active+clean; 583 KiB data, 55 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:32:46.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:45 smithi153 ceph-mon[70044]: pgmap v119: 105 pgs: 1 active+undersized+degraded, 8 peering, 1 stale+active+clean, 24 active+undersized, 71 active+clean; 583 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 1/72 objects degraded (1.389%) 2022-09-29T05:32:46.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:45 smithi153 ceph-mon[70044]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2022-09-29T05:32:46.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:45 smithi153 ceph-mon[70044]: Health check failed: Degraded data redundancy: 1/72 objects degraded (1.389%), 1 pg degraded (PG_DEGRADED) 2022-09-29T05:32:46.060 INFO:journalctl@ceph.osd.4.smithi153.stdout:Sep 29 05:32:45 smithi153 bash[73531]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2022-09-29T05:32:46.060 INFO:journalctl@ceph.osd.4.smithi153.stdout:Sep 29 05:32:45 smithi153 bash[73531]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2022-09-29T05:32:46.060 INFO:journalctl@ceph.osd.4.smithi153.stdout:Sep 29 05:32:45 smithi153 bash[73531]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2022-09-29T05:32:46.061 INFO:journalctl@ceph.osd.4.smithi153.stdout:Sep 29 05:32:45 smithi153 bash[73531]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2022-09-29T05:32:46.061 INFO:journalctl@ceph.osd.4.smithi153.stdout:Sep 29 05:32:45 smithi153 bash[73531]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2022-09-29T05:32:46.061 INFO:journalctl@ceph.osd.4.smithi153.stdout:Sep 29 05:32:45 smithi153 bash[73531]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2022-09-29T05:32:46.061 INFO:journalctl@ceph.osd.4.smithi153.stdout:Sep 29 05:32:45 smithi153 bash[73531]: --> ceph-volume raw activate successful for osd ID: 4 2022-09-29T05:32:46.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:45 smithi083 ceph-mon[79930]: pgmap v119: 105 pgs: 1 active+undersized+degraded, 8 peering, 1 stale+active+clean, 24 active+undersized, 71 active+clean; 583 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 1/72 objects degraded (1.389%) 2022-09-29T05:32:46.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:45 smithi083 ceph-mon[79930]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2022-09-29T05:32:46.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:45 smithi083 ceph-mon[79930]: Health check failed: Degraded data redundancy: 1/72 objects degraded (1.389%), 1 pg degraded (PG_DEGRADED) 2022-09-29T05:32:46.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:45 smithi083 ceph-mon[77242]: pgmap v119: 105 pgs: 1 active+undersized+degraded, 8 peering, 1 stale+active+clean, 24 active+undersized, 71 active+clean; 583 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 1/72 objects degraded (1.389%) 2022-09-29T05:32:46.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:45 smithi083 ceph-mon[77242]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2022-09-29T05:32:46.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:45 smithi083 ceph-mon[77242]: Health check failed: Degraded data redundancy: 1/72 objects degraded (1.389%), 1 pg degraded (PG_DEGRADED) 2022-09-29T05:32:47.309 INFO:journalctl@ceph.osd.4.smithi153.stdout:Sep 29 05:32:46 smithi153 podman[73818]: 2022-09-29T05:32:47.309 INFO:journalctl@ceph.osd.4.smithi153.stdout:Sep 29 05:32:47 smithi153 bash[73818]: 4642d6bd58aee68444fcd90758868831582d5214162895c9df1e7665401fba7a 2022-09-29T05:32:47.309 INFO:journalctl@ceph.osd.4.smithi153.stdout:Sep 29 05:32:47 smithi153 ceph-osd[73857]: -- 172.21.15.153:0/161742500 <== mon.2 v2:172.21.15.153:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x56163bb9c680 con 0x56163bb60c00 2022-09-29T05:32:47.310 INFO:journalctl@ceph.osd.4.smithi153.stdout:Sep 29 05:32:47 smithi153 systemd[1]: Started Ceph osd.4 for 69111522-3fb6-11ed-8431-001a4aab830c. 2022-09-29T05:32:48.059 INFO:journalctl@ceph.osd.4.smithi153.stdout:Sep 29 05:32:47 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-osd-4[73834]: 2022-09-29T05:32:47.676+0000 7fd4279ac5c0 -1 Falling back to public interface 2022-09-29T05:32:48.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:48 smithi153 ceph-mon[70044]: pgmap v120: 105 pgs: 3 active+undersized+degraded, 8 peering, 28 active+undersized, 66 active+clean; 583 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2022-09-29T05:32:48.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:48 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:32:48.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:48 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:32:48.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:48 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:32:48.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:48 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:32:48.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:48 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:32:48.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:48 smithi083 ceph-mon[79930]: pgmap v120: 105 pgs: 3 active+undersized+degraded, 8 peering, 28 active+undersized, 66 active+clean; 583 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2022-09-29T05:32:48.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:48 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:32:48.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:48 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:32:48.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:48 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:32:48.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:48 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:32:48.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:48 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:32:48.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:48 smithi083 ceph-mon[77242]: pgmap v120: 105 pgs: 3 active+undersized+degraded, 8 peering, 28 active+undersized, 66 active+clean; 583 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2022-09-29T05:32:48.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:48 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:32:48.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:48 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:32:48.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:48 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:32:48.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:48 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:32:48.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:48 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:32:49.059 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:32:48 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:32:48.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi083:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi153.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:32:49.059 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:32:48 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:32:48.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephPGImbalance\nexpr: abs(((ceph_osd_numpg > 0) - on(job) group_left() avg by(job) (ceph_osd_numpg\n > 0)) / on(job) group_left() avg by(job) (ceph_osd_numpg > 0)) * on(ceph_daemon)\n group_left(hostname) ceph_osd_metadata > 0.3\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.5\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} deviates by\n more than 30% from average PG count.\n summary: PGs are not balanced across OSDs\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi083:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi153.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:32:49.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:49 smithi153 ceph-mon[70044]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2022-09-29T05:32:49.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:49 smithi083 ceph-mon[79930]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2022-09-29T05:32:49.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:49 smithi083 ceph-mon[77242]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2022-09-29T05:32:50.673 INFO:journalctl@ceph.osd.4.smithi153.stdout:Sep 29 05:32:50 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-osd-4[73834]: 2022-09-29T05:32:50.402+0000 7fd4279ac5c0 -1 osd.4 84 log_to_monitors true 2022-09-29T05:32:50.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:50 smithi083 ceph-mon[79930]: pgmap v121: 105 pgs: 6 active+undersized+degraded, 33 active+undersized, 66 active+clean; 583 KiB data, 56 MiB used, 715 GiB / 715 GiB avail; 8/72 objects degraded (11.111%) 2022-09-29T05:32:50.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:50 smithi083 ceph-mon[79930]: from='osd.4 [v2:172.21.15.153:6800/2296787106,v1:172.21.15.153:6801/2296787106]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2022-09-29T05:32:50.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:50 smithi083 ceph-mon[79930]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2022-09-29T05:32:50.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:50 smithi083 ceph-mon[77242]: pgmap v121: 105 pgs: 6 active+undersized+degraded, 33 active+undersized, 66 active+clean; 583 KiB data, 56 MiB used, 715 GiB / 715 GiB avail; 8/72 objects degraded (11.111%) 2022-09-29T05:32:50.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:50 smithi083 ceph-mon[77242]: from='osd.4 [v2:172.21.15.153:6800/2296787106,v1:172.21.15.153:6801/2296787106]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2022-09-29T05:32:50.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:50 smithi083 ceph-mon[77242]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2022-09-29T05:32:51.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:50 smithi153 ceph-mon[70044]: pgmap v121: 105 pgs: 6 active+undersized+degraded, 33 active+undersized, 66 active+clean; 583 KiB data, 56 MiB used, 715 GiB / 715 GiB avail; 8/72 objects degraded (11.111%) 2022-09-29T05:32:51.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:50 smithi153 ceph-mon[70044]: from='osd.4 [v2:172.21.15.153:6800/2296787106,v1:172.21.15.153:6801/2296787106]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2022-09-29T05:32:51.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:50 smithi153 ceph-mon[70044]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2022-09-29T05:32:51.060 INFO:journalctl@ceph.osd.4.smithi153.stdout:Sep 29 05:32:50 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-osd-4[73834]: 2022-09-29T05:32:50.689+0000 7fd41792e700 -1 osd.4 84 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2022-09-29T05:32:51.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:32:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:32:51] "GET /metrics HTTP/1.1" 200 206218 "" "Prometheus/2.33.4" 2022-09-29T05:32:51.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:51 smithi083 ceph-mon[79930]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2022-09-29T05:32:51.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:51 smithi083 ceph-mon[79930]: osdmap e87: 8 total, 7 up, 8 in 2022-09-29T05:32:51.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:51 smithi083 ceph-mon[79930]: from='osd.4 [v2:172.21.15.153:6800/2296787106,v1:172.21.15.153:6801/2296787106]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2022-09-29T05:32:51.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:51 smithi083 ceph-mon[79930]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2022-09-29T05:32:51.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:51 smithi083 ceph-mon[77242]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2022-09-29T05:32:51.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:51 smithi083 ceph-mon[77242]: osdmap e87: 8 total, 7 up, 8 in 2022-09-29T05:32:51.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:51 smithi083 ceph-mon[77242]: from='osd.4 [v2:172.21.15.153:6800/2296787106,v1:172.21.15.153:6801/2296787106]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2022-09-29T05:32:51.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:51 smithi083 ceph-mon[77242]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2022-09-29T05:32:52.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:51 smithi153 ceph-mon[70044]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2022-09-29T05:32:52.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:51 smithi153 ceph-mon[70044]: osdmap e87: 8 total, 7 up, 8 in 2022-09-29T05:32:52.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:51 smithi153 ceph-mon[70044]: from='osd.4 [v2:172.21.15.153:6800/2296787106,v1:172.21.15.153:6801/2296787106]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2022-09-29T05:32:52.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:51 smithi153 ceph-mon[70044]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2022-09-29T05:32:52.694 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:52 smithi153 ceph-mon[70044]: pgmap v123: 105 pgs: 6 active+undersized+degraded, 33 active+undersized, 66 active+clean; 583 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 8/72 objects degraded (11.111%) 2022-09-29T05:32:52.694 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:52 smithi153 ceph-mon[70044]: Health check cleared: OSD_DOWN (was: 1 osds down) 2022-09-29T05:32:52.695 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:52 smithi153 ceph-mon[70044]: osd.4 [v2:172.21.15.153:6800/2296787106,v1:172.21.15.153:6801/2296787106] boot 2022-09-29T05:32:52.695 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:52 smithi153 ceph-mon[70044]: osdmap e88: 8 total, 8 up, 8 in 2022-09-29T05:32:52.695 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:52 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2022-09-29T05:32:52.695 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:52 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:32:52.696 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:52 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:32:52.696 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:32:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:32:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:32:52.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:52 smithi083 ceph-mon[79930]: pgmap v123: 105 pgs: 6 active+undersized+degraded, 33 active+undersized, 66 active+clean; 583 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 8/72 objects degraded (11.111%) 2022-09-29T05:32:52.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:52 smithi083 ceph-mon[79930]: Health check cleared: OSD_DOWN (was: 1 osds down) 2022-09-29T05:32:52.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:52 smithi083 ceph-mon[79930]: osd.4 [v2:172.21.15.153:6800/2296787106,v1:172.21.15.153:6801/2296787106] boot 2022-09-29T05:32:52.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:52 smithi083 ceph-mon[79930]: osdmap e88: 8 total, 8 up, 8 in 2022-09-29T05:32:52.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:52 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2022-09-29T05:32:52.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:52 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:32:52.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:52 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:32:52.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:52 smithi083 ceph-mon[77242]: pgmap v123: 105 pgs: 6 active+undersized+degraded, 33 active+undersized, 66 active+clean; 583 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 8/72 objects degraded (11.111%) 2022-09-29T05:32:52.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:52 smithi083 ceph-mon[77242]: Health check cleared: OSD_DOWN (was: 1 osds down) 2022-09-29T05:32:52.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:52 smithi083 ceph-mon[77242]: osd.4 [v2:172.21.15.153:6800/2296787106,v1:172.21.15.153:6801/2296787106] boot 2022-09-29T05:32:52.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:52 smithi083 ceph-mon[77242]: osdmap e88: 8 total, 8 up, 8 in 2022-09-29T05:32:52.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:52 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2022-09-29T05:32:52.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:52 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:32:52.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:52 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:32:53.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:32:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:32:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:32:53.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:53 smithi083 ceph-mon[79930]: osdmap e89: 8 total, 8 up, 8 in 2022-09-29T05:32:53.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:53 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:32:53.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:53 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:32:53.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:53 smithi083 ceph-mon[77242]: osdmap e89: 8 total, 8 up, 8 in 2022-09-29T05:32:53.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:53 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:32:53.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:53 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:32:54.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:53 smithi153 ceph-mon[70044]: osdmap e89: 8 total, 8 up, 8 in 2022-09-29T05:32:54.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:53 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:32:54.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:53 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:32:54.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:54 smithi083 ceph-mon[79930]: pgmap v126: 105 pgs: 6 active+undersized+degraded, 33 active+undersized, 66 active+clean; 583 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 8/72 objects degraded (11.111%) 2022-09-29T05:32:54.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:54 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:32:54.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:54 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:32:54.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:54 smithi083 ceph-mon[79930]: Health check update: Degraded data redundancy: 8/72 objects degraded (11.111%), 6 pgs degraded (PG_DEGRADED) 2022-09-29T05:32:54.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:54 smithi083 ceph-mon[77242]: pgmap v126: 105 pgs: 6 active+undersized+degraded, 33 active+undersized, 66 active+clean; 583 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 8/72 objects degraded (11.111%) 2022-09-29T05:32:54.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:54 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:32:54.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:54 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:32:54.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:54 smithi083 ceph-mon[77242]: Health check update: Degraded data redundancy: 8/72 objects degraded (11.111%), 6 pgs degraded (PG_DEGRADED) 2022-09-29T05:32:55.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:54 smithi153 ceph-mon[70044]: pgmap v126: 105 pgs: 6 active+undersized+degraded, 33 active+undersized, 66 active+clean; 583 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 8/72 objects degraded (11.111%) 2022-09-29T05:32:55.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:54 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:32:55.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:54 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:32:55.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:54 smithi153 ceph-mon[70044]: Health check update: Degraded data redundancy: 8/72 objects degraded (11.111%), 6 pgs degraded (PG_DEGRADED) 2022-09-29T05:32:56.213 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:32:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: 2022-09-29T05:32:55.783+0000 7f9b26e95700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2022-09-29T05:32:56.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:56 smithi083 ceph-mon[79930]: pgmap v127: 105 pgs: 3 active+undersized+degraded, 9 active+undersized, 93 active+clean; 583 KiB data, 56 MiB used, 715 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2022-09-29T05:32:56.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:56 smithi083 ceph-mon[79930]: Detected new or changed devices on smithi153 2022-09-29T05:32:56.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:56 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:32:56.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:56 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:32:56.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:56 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:32:56.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:56 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:32:56.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:56 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:32:56.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:56 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:32:56.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:56 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:32:56.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:56 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2022-09-29T05:32:56.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:56 smithi083 ceph-mon[77242]: pgmap v127: 105 pgs: 3 active+undersized+degraded, 9 active+undersized, 93 active+clean; 583 KiB data, 56 MiB used, 715 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2022-09-29T05:32:56.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:56 smithi083 ceph-mon[77242]: Detected new or changed devices on smithi153 2022-09-29T05:32:56.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:56 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:32:56.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:56 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:32:56.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:56 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:32:56.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:56 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:32:56.968 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:56 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:32:56.968 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:56 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:32:56.968 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:56 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:32:56.968 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:56 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2022-09-29T05:32:57.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:56 smithi153 ceph-mon[70044]: pgmap v127: 105 pgs: 3 active+undersized+degraded, 9 active+undersized, 93 active+clean; 583 KiB data, 56 MiB used, 715 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2022-09-29T05:32:57.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:56 smithi153 ceph-mon[70044]: Detected new or changed devices on smithi153 2022-09-29T05:32:57.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:56 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:32:57.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:56 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:32:57.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:56 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:32:57.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:56 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:32:57.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:56 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:32:57.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:56 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:32:57.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:56 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:32:57.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:56 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2022-09-29T05:32:57.260 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T05:32:57.546 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T05:32:57.546 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6m) 38s ago 7m 26.4M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T05:32:57.546 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6m) 5s ago 6m 60.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T05:32:57.547 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8m) 38s ago 8m 15.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T05:32:57.547 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7m) 5s ago 7m 18.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T05:32:57.547 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3m) 5s ago 10m 429M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T05:32:57.547 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4m) 38s ago 12m 490M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T05:32:57.548 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3m) 38s ago 12m 86.0M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T05:32:57.548 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2m) 5s ago 10m 66.9M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T05:32:57.548 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3m) 38s ago 10m 70.7M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T05:32:57.548 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7m) 38s ago 7m 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T05:32:57.549 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7m) 5s ago 7m 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T05:32:57.549 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2m) 38s ago 10m 63.4M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T05:32:57.549 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (95s) 38s ago 9m 56.5M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T05:32:57.549 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (77s) 38s ago 9m 51.9M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T05:32:57.550 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (43s) 38s ago 9m 12.9M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T05:32:57.550 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10s) 5s ago 9m 12.6M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T05:32:57.551 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8m) 5s ago 8m 87.9M 2500M 16.2.10 c56e13ff90f2 59f19fd15fed 2022-09-29T05:32:57.551 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8m) 5s ago 8m 87.6M 2500M 16.2.10 c56e13ff90f2 23decfd9c6b2 2022-09-29T05:32:57.551 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8m) 5s ago 8m 94.3M 2500M 16.2.10 c56e13ff90f2 23460a09c0a7 2022-09-29T05:32:57.551 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3m) 5s ago 7m 78.8M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T05:32:57.906 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T05:32:57.907 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T05:32:57.907 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T05:32:57.907 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:32:57.907 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T05:32:57.908 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T05:32:57.908 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:32:57.908 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T05:32:57.908 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 3, 2022-09-29T05:32:57.909 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 5 2022-09-29T05:32:57.909 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:32:57.909 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T05:32:57.909 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T05:32:57.910 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:32:57.910 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T05:32:57.910 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 5, 2022-09-29T05:32:57.910 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 10 2022-09-29T05:32:57.911 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T05:32:57.911 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T05:32:57.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:57 smithi083 ceph-mon[79930]: from='mon.1 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2022-09-29T05:32:57.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:57 smithi083 ceph-mon[79930]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2022-09-29T05:32:57.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:57 smithi083 ceph-mon[77242]: from='mon.1 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2022-09-29T05:32:57.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:57 smithi083 ceph-mon[77242]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2022-09-29T05:32:58.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:57 smithi153 ceph-mon[70044]: from='mon.1 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2022-09-29T05:32:58.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:57 smithi153 ceph-mon[70044]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2022-09-29T05:32:58.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:58 smithi083 ceph-mon[79930]: pgmap v128: 105 pgs: 105 active+clean; 583 KiB data, 56 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:32:58.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:58 smithi083 ceph-mon[79930]: from='client.34287 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:32:58.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:58 smithi083 ceph-mon[79930]: from='client.44227 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:32:58.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:58 smithi083 ceph-mon[79930]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/72 objects degraded (4.167%), 3 pgs degraded) 2022-09-29T05:32:58.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:32:58 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1224779536' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:32:58.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:58 smithi083 ceph-mon[77242]: pgmap v128: 105 pgs: 105 active+clean; 583 KiB data, 56 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:32:58.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:58 smithi083 ceph-mon[77242]: from='client.34287 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:32:58.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:58 smithi083 ceph-mon[77242]: from='client.44227 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:32:58.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:58 smithi083 ceph-mon[77242]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/72 objects degraded (4.167%), 3 pgs degraded) 2022-09-29T05:32:58.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:32:58 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1224779536' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:32:59.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:58 smithi153 ceph-mon[70044]: pgmap v128: 105 pgs: 105 active+clean; 583 KiB data, 56 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:32:59.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:58 smithi153 ceph-mon[70044]: from='client.34287 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:32:59.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:58 smithi153 ceph-mon[70044]: from='client.44227 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:32:59.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:58 smithi153 ceph-mon[70044]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/72 objects degraded (4.167%), 3 pgs degraded) 2022-09-29T05:32:59.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:32:58 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1224779536' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:32:59.060 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:32:58 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:32:58.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi083:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi153.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:32:59.061 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:32:58 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:32:58.662Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephPGImbalance\nexpr: abs(((ceph_osd_numpg > 0) - on(job) group_left() avg by(job) (ceph_osd_numpg\n > 0)) / on(job) group_left() avg by(job) (ceph_osd_numpg > 0)) * on(ceph_daemon)\n group_left(hostname) ceph_osd_metadata > 0.3\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.5\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} deviates by\n more than 30% from average PG count.\n summary: PGs are not balanced across OSDs\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi083:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi153.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:33:00.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:00 smithi083 ceph-mon[79930]: pgmap v129: 105 pgs: 105 active+clean; 583 KiB data, 56 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:33:00.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:00 smithi083 ceph-mon[77242]: pgmap v129: 105 pgs: 105 active+clean; 583 KiB data, 56 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:33:01.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:00 smithi153 ceph-mon[70044]: pgmap v129: 105 pgs: 105 active+clean; 583 KiB data, 56 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:33:01.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:33:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:33:01] "GET /metrics HTTP/1.1" 200 211090 "" "Prometheus/2.33.4" 2022-09-29T05:33:02.696 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:33:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:33:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:33:02.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:02 smithi083 ceph-mon[79930]: pgmap v130: 105 pgs: 105 active+clean; 583 KiB data, 56 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:33:02.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:02 smithi083 ceph-mon[77242]: pgmap v130: 105 pgs: 105 active+clean; 583 KiB data, 56 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:33:03.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:02 smithi153 ceph-mon[70044]: pgmap v130: 105 pgs: 105 active+clean; 583 KiB data, 56 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:33:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:33:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:33:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:33:04.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:04 smithi083 ceph-mon[77242]: pgmap v131: 105 pgs: 105 active+clean; 583 KiB data, 56 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:33:04.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:04 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:33:04.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:04 smithi083 ceph-mon[79930]: pgmap v131: 105 pgs: 105 active+clean; 583 KiB data, 56 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:33:04.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:04 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:33:05.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:04 smithi153 ceph-mon[70044]: pgmap v131: 105 pgs: 105 active+clean; 583 KiB data, 56 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:33:05.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:04 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:33:06.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:06 smithi083 ceph-mon[79930]: pgmap v132: 105 pgs: 105 active+clean; 583 KiB data, 56 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:33:06.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:06 smithi083 ceph-mon[77242]: pgmap v132: 105 pgs: 105 active+clean; 583 KiB data, 56 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:33:07.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:06 smithi153 ceph-mon[70044]: pgmap v132: 105 pgs: 105 active+clean; 583 KiB data, 56 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:33:08.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:08 smithi083 ceph-mon[79930]: pgmap v133: 105 pgs: 105 active+clean; 583 KiB data, 56 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:33:08.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:08 smithi083 ceph-mon[77242]: pgmap v133: 105 pgs: 105 active+clean; 583 KiB data, 56 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:33:09.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:08 smithi153 ceph-mon[70044]: pgmap v133: 105 pgs: 105 active+clean; 583 KiB data, 56 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:33:09.059 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:33:08 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:33:08.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi083:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi153.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:33:09.060 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:33:08 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:33:08.663Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephPGImbalance\nexpr: abs(((ceph_osd_numpg > 0) - on(job) group_left() avg by(job) (ceph_osd_numpg\n > 0)) / on(job) group_left() avg by(job) (ceph_osd_numpg > 0)) * on(ceph_daemon)\n group_left(hostname) ceph_osd_metadata > 0.3\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.5\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} deviates by\n more than 30% from average PG count.\n summary: PGs are not balanced across OSDs\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi083:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi153.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:33:10.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:09 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:33:10.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:09 smithi083 ceph-mon[79930]: pgmap v134: 105 pgs: 105 active+clean; 583 KiB data, 56 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:33:10.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:09 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:33:10.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:09 smithi083 ceph-mon[77242]: pgmap v134: 105 pgs: 105 active+clean; 583 KiB data, 56 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:33:10.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:09 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:33:10.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:09 smithi153 ceph-mon[70044]: pgmap v134: 105 pgs: 105 active+clean; 583 KiB data, 56 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:33:11.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:10 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2022-09-29T05:33:11.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:10 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2022-09-29T05:33:11.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:10 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2022-09-29T05:33:11.875 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:33:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:33:11] "GET /metrics HTTP/1.1" 200 210962 "" "Prometheus/2.33.4" 2022-09-29T05:33:12.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:11 smithi083 ceph-mon[79930]: from='mon.1 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2022-09-29T05:33:12.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:11 smithi083 ceph-mon[79930]: Upgrade: osd.5 is safe to restart 2022-09-29T05:33:12.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:11 smithi083 ceph-mon[79930]: pgmap v135: 105 pgs: 105 active+clean; 583 KiB data, 56 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:33:12.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:11 smithi083 ceph-mon[77242]: from='mon.1 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2022-09-29T05:33:12.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:11 smithi083 ceph-mon[77242]: Upgrade: osd.5 is safe to restart 2022-09-29T05:33:12.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:11 smithi083 ceph-mon[77242]: pgmap v135: 105 pgs: 105 active+clean; 583 KiB data, 56 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:33:12.287 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:11 smithi153 ceph-mon[70044]: from='mon.1 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2022-09-29T05:33:12.288 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:11 smithi153 ceph-mon[70044]: Upgrade: osd.5 is safe to restart 2022-09-29T05:33:12.288 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:11 smithi153 ceph-mon[70044]: pgmap v135: 105 pgs: 105 active+clean; 583 KiB data, 56 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:33:12.559 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:33:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:33:12.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:33:13.237 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:33:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:33:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:33:13.498 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:13 smithi153 ceph-mon[70044]: Upgrade: Updating osd.5 2022-09-29T05:33:13.498 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:13 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:33:13.499 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:13 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2022-09-29T05:33:13.499 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:13 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:33:13.499 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:13 smithi153 ceph-mon[70044]: Deploying daemon osd.5 on smithi153 2022-09-29T05:33:13.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:13 smithi083 ceph-mon[79930]: Upgrade: Updating osd.5 2022-09-29T05:33:13.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:13 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:33:13.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:13 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2022-09-29T05:33:13.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:13 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:33:13.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:13 smithi083 ceph-mon[79930]: Deploying daemon osd.5 on smithi153 2022-09-29T05:33:13.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:13 smithi083 ceph-mon[77242]: Upgrade: Updating osd.5 2022-09-29T05:33:13.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:13 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:33:13.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:13 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2022-09-29T05:33:13.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:13 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:33:13.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:13 smithi083 ceph-mon[77242]: Deploying daemon osd.5 on smithi153 2022-09-29T05:33:14.384 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:14 smithi153 ceph-mon[70044]: pgmap v136: 105 pgs: 105 active+clean; 583 KiB data, 56 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:33:14.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:14 smithi083 ceph-mon[79930]: pgmap v136: 105 pgs: 105 active+clean; 583 KiB data, 56 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:33:14.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:14 smithi083 ceph-mon[77242]: pgmap v136: 105 pgs: 105 active+clean; 583 KiB data, 56 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:33:15.309 INFO:journalctl@ceph.osd.5.smithi153.stdout:Sep 29 05:33:14 smithi153 systemd[1]: Stopping Ceph osd.5 for 69111522-3fb6-11ed-8431-001a4aab830c... 2022-09-29T05:33:15.309 INFO:journalctl@ceph.osd.5.smithi153.stdout:Sep 29 05:33:14 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-osd-5[45562]: debug 2022-09-29T05:33:14.982+0000 7f797b883700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2022-09-29T05:33:15.309 INFO:journalctl@ceph.osd.5.smithi153.stdout:Sep 29 05:33:14 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-osd-5[45562]: debug 2022-09-29T05:33:14.982+0000 7f797b883700 -1 osd.5 89 *** Got signal Terminated *** 2022-09-29T05:33:15.310 INFO:journalctl@ceph.osd.5.smithi153.stdout:Sep 29 05:33:14 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-osd-5[45562]: debug 2022-09-29T05:33:14.982+0000 7f797b883700 -1 osd.5 89 *** Immediate shutdown (osd_fast_shutdown=true) *** 2022-09-29T05:33:15.935 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:15 smithi153 ceph-mon[70044]: osd.5 marked itself down and dead 2022-09-29T05:33:15.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:15 smithi083 ceph-mon[79930]: osd.5 marked itself down and dead 2022-09-29T05:33:15.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:15 smithi083 ceph-mon[77242]: osd.5 marked itself down and dead 2022-09-29T05:33:16.309 INFO:journalctl@ceph.osd.5.smithi153.stdout:Sep 29 05:33:15 smithi153 bash[76002]: ceph-69111522-3fb6-11ed-8431-001a4aab830c-osd-5 2022-09-29T05:33:16.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:16 smithi083 ceph-mon[79930]: pgmap v137: 105 pgs: 105 active+clean; 583 KiB data, 56 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:33:16.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:16 smithi083 ceph-mon[79930]: Health check failed: 1 osds down (OSD_DOWN) 2022-09-29T05:33:16.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:16 smithi083 ceph-mon[79930]: osdmap e90: 8 total, 7 up, 8 in 2022-09-29T05:33:16.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:16 smithi083 ceph-mon[77242]: pgmap v137: 105 pgs: 105 active+clean; 583 KiB data, 56 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:33:16.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:16 smithi083 ceph-mon[77242]: Health check failed: 1 osds down (OSD_DOWN) 2022-09-29T05:33:16.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:16 smithi083 ceph-mon[77242]: osdmap e90: 8 total, 7 up, 8 in 2022-09-29T05:33:17.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:16 smithi153 ceph-mon[70044]: pgmap v137: 105 pgs: 105 active+clean; 583 KiB data, 56 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:33:17.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:16 smithi153 ceph-mon[70044]: Health check failed: 1 osds down (OSD_DOWN) 2022-09-29T05:33:17.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:16 smithi153 ceph-mon[70044]: osdmap e90: 8 total, 7 up, 8 in 2022-09-29T05:33:17.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:17 smithi083 ceph-mon[79930]: osdmap e91: 8 total, 7 up, 8 in 2022-09-29T05:33:17.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:17 smithi083 ceph-mon[77242]: osdmap e91: 8 total, 7 up, 8 in 2022-09-29T05:33:18.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:17 smithi153 ceph-mon[70044]: osdmap e91: 8 total, 7 up, 8 in 2022-09-29T05:33:18.060 INFO:journalctl@ceph.osd.5.smithi153.stdout:Sep 29 05:33:17 smithi153 systemd[1]: ceph-69111522-3fb6-11ed-8431-001a4aab830c@osd.5.service: Succeeded. 2022-09-29T05:33:18.060 INFO:journalctl@ceph.osd.5.smithi153.stdout:Sep 29 05:33:17 smithi153 systemd[1]: Stopped Ceph osd.5 for 69111522-3fb6-11ed-8431-001a4aab830c. 2022-09-29T05:33:18.061 INFO:journalctl@ceph.osd.5.smithi153.stdout:Sep 29 05:33:17 smithi153 systemd[1]: Starting Ceph osd.5 for 69111522-3fb6-11ed-8431-001a4aab830c... 2022-09-29T05:33:18.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:18 smithi153 ceph-mon[70044]: pgmap v140: 105 pgs: 11 peering, 12 stale+active+clean, 82 active+clean; 583 KiB data, 56 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:33:18.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:18 smithi153 ceph-mon[70044]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2022-09-29T05:33:18.810 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:33:18 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:33:18.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi083:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi153.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:33:18.810 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:33:18 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:33:18.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephPGImbalance\nexpr: abs(((ceph_osd_numpg > 0) - on(job) group_left() avg by(job) (ceph_osd_numpg\n > 0)) / on(job) group_left() avg by(job) (ceph_osd_numpg > 0)) * on(ceph_daemon)\n group_left(hostname) ceph_osd_metadata > 0.3\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.5\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} deviates by\n more than 30% from average PG count.\n summary: PGs are not balanced across OSDs\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi083:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi153.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:33:18.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:18 smithi083 ceph-mon[79930]: pgmap v140: 105 pgs: 11 peering, 12 stale+active+clean, 82 active+clean; 583 KiB data, 56 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:33:18.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:18 smithi083 ceph-mon[79930]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2022-09-29T05:33:18.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:18 smithi083 ceph-mon[77242]: pgmap v140: 105 pgs: 11 peering, 12 stale+active+clean, 82 active+clean; 583 KiB data, 56 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:33:18.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:18 smithi083 ceph-mon[77242]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2022-09-29T05:33:19.309 INFO:journalctl@ceph.osd.5.smithi153.stdout:Sep 29 05:33:19 smithi153 bash[76253]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2022-09-29T05:33:19.309 INFO:journalctl@ceph.osd.5.smithi153.stdout:Sep 29 05:33:19 smithi153 bash[76253]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2022-09-29T05:33:19.309 INFO:journalctl@ceph.osd.5.smithi153.stdout:Sep 29 05:33:19 smithi153 bash[76253]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2022-09-29T05:33:19.310 INFO:journalctl@ceph.osd.5.smithi153.stdout:Sep 29 05:33:19 smithi153 bash[76253]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2022-09-29T05:33:19.310 INFO:journalctl@ceph.osd.5.smithi153.stdout:Sep 29 05:33:19 smithi153 bash[76253]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2022-09-29T05:33:19.310 INFO:journalctl@ceph.osd.5.smithi153.stdout:Sep 29 05:33:19 smithi153 bash[76253]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2022-09-29T05:33:19.310 INFO:journalctl@ceph.osd.5.smithi153.stdout:Sep 29 05:33:19 smithi153 bash[76253]: --> ceph-volume raw activate successful for osd ID: 5 2022-09-29T05:33:20.559 INFO:journalctl@ceph.osd.5.smithi153.stdout:Sep 29 05:33:20 smithi153 podman[76534]: 2022-09-29T05:33:20.559 INFO:journalctl@ceph.osd.5.smithi153.stdout:Sep 29 05:33:20 smithi153 bash[76534]: a041d8e19416821f13575b3fa37d9d704c69b190c0afabd3a411d6b244aae806 2022-09-29T05:33:20.560 INFO:journalctl@ceph.osd.5.smithi153.stdout:Sep 29 05:33:20 smithi153 ceph-osd[76572]: -- 172.21.15.153:0/2248491587 <== mon.2 v2:172.21.15.153:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x559886c3a680 con 0x559886bff400 2022-09-29T05:33:20.560 INFO:journalctl@ceph.osd.5.smithi153.stdout:Sep 29 05:33:20 smithi153 systemd[1]: Started Ceph osd.5 for 69111522-3fb6-11ed-8431-001a4aab830c. 2022-09-29T05:33:20.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:20 smithi083 ceph-mon[79930]: pgmap v141: 105 pgs: 11 peering, 5 stale+active+clean, 12 active+undersized, 77 active+clean; 583 KiB data, 60 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:33:20.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:20 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:33:20.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:20 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:33:20.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:20 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:33:20.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:20 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:33:20.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:20 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:33:20.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:20 smithi083 ceph-mon[77242]: pgmap v141: 105 pgs: 11 peering, 5 stale+active+clean, 12 active+undersized, 77 active+clean; 583 KiB data, 60 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:33:20.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:20 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:33:20.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:20 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:33:20.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:20 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:33:20.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:20 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:33:20.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:20 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:33:21.059 INFO:journalctl@ceph.osd.5.smithi153.stdout:Sep 29 05:33:20 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-osd-5[76550]: 2022-09-29T05:33:20.793+0000 7ff7b376c5c0 -1 Falling back to public interface 2022-09-29T05:33:21.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:20 smithi153 ceph-mon[70044]: pgmap v141: 105 pgs: 11 peering, 5 stale+active+clean, 12 active+undersized, 77 active+clean; 583 KiB data, 60 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:33:21.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:20 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:33:21.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:20 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:33:21.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:20 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:33:21.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:20 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:33:21.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:20 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:33:21.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:33:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:33:21] "GET /metrics HTTP/1.1" 200 210962 "" "Prometheus/2.33.4" 2022-09-29T05:33:21.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:21 smithi083 ceph-mon[79930]: Health check failed: Degraded data redundancy: 8/72 objects degraded (11.111%), 4 pgs degraded (PG_DEGRADED) 2022-09-29T05:33:21.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:21 smithi083 ceph-mon[79930]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2022-09-29T05:33:21.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:21 smithi083 ceph-mon[77242]: Health check failed: Degraded data redundancy: 8/72 objects degraded (11.111%), 4 pgs degraded (PG_DEGRADED) 2022-09-29T05:33:21.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:21 smithi083 ceph-mon[77242]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2022-09-29T05:33:22.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:21 smithi153 ceph-mon[70044]: Health check failed: Degraded data redundancy: 8/72 objects degraded (11.111%), 4 pgs degraded (PG_DEGRADED) 2022-09-29T05:33:22.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:21 smithi153 ceph-mon[70044]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2022-09-29T05:33:22.736 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:33:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:33:22.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:33:23.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:22 smithi153 ceph-mon[70044]: pgmap v142: 105 pgs: 4 active+undersized+degraded, 38 active+undersized, 63 active+clean; 583 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 8/72 objects degraded (11.111%) 2022-09-29T05:33:23.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:22 smithi083 ceph-mon[79930]: pgmap v142: 105 pgs: 4 active+undersized+degraded, 38 active+undersized, 63 active+clean; 583 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 8/72 objects degraded (11.111%) 2022-09-29T05:33:23.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:22 smithi083 ceph-mon[77242]: pgmap v142: 105 pgs: 4 active+undersized+degraded, 38 active+undersized, 63 active+clean; 583 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 8/72 objects degraded (11.111%) 2022-09-29T05:33:23.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:33:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:33:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:33:24.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:23 smithi153 ceph-mon[70044]: from='osd.5 [v2:172.21.15.153:6808/1581188016,v1:172.21.15.153:6809/1581188016]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2022-09-29T05:33:24.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:23 smithi153 ceph-mon[70044]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2022-09-29T05:33:24.060 INFO:journalctl@ceph.osd.5.smithi153.stdout:Sep 29 05:33:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-osd-5[76550]: 2022-09-29T05:33:23.557+0000 7ff7b376c5c0 -1 osd.5 89 log_to_monitors true 2022-09-29T05:33:24.060 INFO:journalctl@ceph.osd.5.smithi153.stdout:Sep 29 05:33:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-osd-5[76550]: 2022-09-29T05:33:23.762+0000 7ff79fee7700 -1 osd.5 89 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2022-09-29T05:33:24.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:23 smithi083 ceph-mon[79930]: from='osd.5 [v2:172.21.15.153:6808/1581188016,v1:172.21.15.153:6809/1581188016]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2022-09-29T05:33:24.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:23 smithi083 ceph-mon[79930]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2022-09-29T05:33:24.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:23 smithi083 ceph-mon[77242]: from='osd.5 [v2:172.21.15.153:6808/1581188016,v1:172.21.15.153:6809/1581188016]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2022-09-29T05:33:24.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:23 smithi083 ceph-mon[77242]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2022-09-29T05:33:25.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:24 smithi153 ceph-mon[70044]: pgmap v143: 105 pgs: 4 active+undersized+degraded, 38 active+undersized, 63 active+clean; 583 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 8/72 objects degraded (11.111%) 2022-09-29T05:33:25.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:24 smithi153 ceph-mon[70044]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2022-09-29T05:33:25.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:24 smithi153 ceph-mon[70044]: osdmap e92: 8 total, 7 up, 8 in 2022-09-29T05:33:25.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:24 smithi153 ceph-mon[70044]: from='osd.5 [v2:172.21.15.153:6808/1581188016,v1:172.21.15.153:6809/1581188016]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2022-09-29T05:33:25.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:24 smithi153 ceph-mon[70044]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2022-09-29T05:33:25.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:24 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:33:25.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:24 smithi083 ceph-mon[79930]: pgmap v143: 105 pgs: 4 active+undersized+degraded, 38 active+undersized, 63 active+clean; 583 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 8/72 objects degraded (11.111%) 2022-09-29T05:33:25.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:24 smithi083 ceph-mon[79930]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2022-09-29T05:33:25.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:24 smithi083 ceph-mon[79930]: osdmap e92: 8 total, 7 up, 8 in 2022-09-29T05:33:25.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:24 smithi083 ceph-mon[79930]: from='osd.5 [v2:172.21.15.153:6808/1581188016,v1:172.21.15.153:6809/1581188016]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2022-09-29T05:33:25.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:24 smithi083 ceph-mon[79930]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2022-09-29T05:33:25.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:24 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:33:25.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:24 smithi083 ceph-mon[77242]: pgmap v143: 105 pgs: 4 active+undersized+degraded, 38 active+undersized, 63 active+clean; 583 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 8/72 objects degraded (11.111%) 2022-09-29T05:33:25.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:24 smithi083 ceph-mon[77242]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2022-09-29T05:33:25.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:24 smithi083 ceph-mon[77242]: osdmap e92: 8 total, 7 up, 8 in 2022-09-29T05:33:25.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:24 smithi083 ceph-mon[77242]: from='osd.5 [v2:172.21.15.153:6808/1581188016,v1:172.21.15.153:6809/1581188016]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2022-09-29T05:33:25.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:24 smithi083 ceph-mon[77242]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2022-09-29T05:33:25.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:24 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:33:26.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:25 smithi153 ceph-mon[70044]: Health check cleared: OSD_DOWN (was: 1 osds down) 2022-09-29T05:33:26.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:25 smithi153 ceph-mon[70044]: osd.5 [v2:172.21.15.153:6808/1581188016,v1:172.21.15.153:6809/1581188016] boot 2022-09-29T05:33:26.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:25 smithi153 ceph-mon[70044]: osdmap e93: 8 total, 8 up, 8 in 2022-09-29T05:33:26.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:25 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2022-09-29T05:33:26.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:25 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:33:26.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:25 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:33:26.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:25 smithi153 ceph-mon[70044]: osdmap e94: 8 total, 8 up, 8 in 2022-09-29T05:33:26.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:25 smithi083 ceph-mon[79930]: Health check cleared: OSD_DOWN (was: 1 osds down) 2022-09-29T05:33:26.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:25 smithi083 ceph-mon[79930]: osd.5 [v2:172.21.15.153:6808/1581188016,v1:172.21.15.153:6809/1581188016] boot 2022-09-29T05:33:26.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:25 smithi083 ceph-mon[79930]: osdmap e93: 8 total, 8 up, 8 in 2022-09-29T05:33:26.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:25 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2022-09-29T05:33:26.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:25 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:33:26.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:25 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:33:26.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:25 smithi083 ceph-mon[79930]: osdmap e94: 8 total, 8 up, 8 in 2022-09-29T05:33:26.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:25 smithi083 ceph-mon[77242]: Health check cleared: OSD_DOWN (was: 1 osds down) 2022-09-29T05:33:26.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:25 smithi083 ceph-mon[77242]: osd.5 [v2:172.21.15.153:6808/1581188016,v1:172.21.15.153:6809/1581188016] boot 2022-09-29T05:33:26.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:25 smithi083 ceph-mon[77242]: osdmap e93: 8 total, 8 up, 8 in 2022-09-29T05:33:26.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:25 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2022-09-29T05:33:26.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:25 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:33:26.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:25 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:33:26.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:25 smithi083 ceph-mon[77242]: osdmap e94: 8 total, 8 up, 8 in 2022-09-29T05:33:27.020 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:26 smithi153 ceph-mon[70044]: pgmap v146: 105 pgs: 10 peering, 1 active+undersized+degraded, 31 active+undersized, 63 active+clean; 583 KiB data, 60 MiB used, 715 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2022-09-29T05:33:27.020 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:26 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:33:27.020 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:26 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:33:27.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:26 smithi083 ceph-mon[79930]: pgmap v146: 105 pgs: 10 peering, 1 active+undersized+degraded, 31 active+undersized, 63 active+clean; 583 KiB data, 60 MiB used, 715 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2022-09-29T05:33:27.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:26 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:33:27.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:26 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:33:27.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:26 smithi083 ceph-mon[77242]: pgmap v146: 105 pgs: 10 peering, 1 active+undersized+degraded, 31 active+undersized, 63 active+clean; 583 KiB data, 60 MiB used, 715 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2022-09-29T05:33:27.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:26 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:33:27.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:26 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:33:28.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:27 smithi153 ceph-mon[70044]: pgmap v148: 105 pgs: 11 peering, 26 active+undersized, 68 active+clean; 583 KiB data, 60 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:33:28.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:27 smithi083 ceph-mon[79930]: pgmap v148: 105 pgs: 11 peering, 26 active+undersized, 68 active+clean; 583 KiB data, 60 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:33:28.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:27 smithi083 ceph-mon[77242]: pgmap v148: 105 pgs: 11 peering, 26 active+undersized, 68 active+clean; 583 KiB data, 60 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:33:28.227 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T05:33:28.514 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T05:33:28.514 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7m) 69s ago 7m 26.4M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T05:33:28.515 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7m) 3s ago 7m 60.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T05:33:28.515 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8m) 69s ago 8m 15.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T05:33:28.515 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8m) 3s ago 8m 19.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T05:33:28.515 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4m) 3s ago 11m 429M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T05:33:28.516 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4m) 69s ago 12m 490M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T05:33:28.516 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3m) 69s ago 12m 86.0M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T05:33:28.516 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3m) 3s ago 11m 69.7M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T05:33:28.516 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3m) 69s ago 11m 70.7M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T05:33:28.517 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7m) 69s ago 7m 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T05:33:28.517 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7m) 3s ago 7m 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T05:33:28.517 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2m) 69s ago 10m 63.4M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T05:33:28.517 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2m) 69s ago 10m 56.5M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T05:33:28.518 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (108s) 69s ago 10m 51.9M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T05:33:28.518 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (74s) 69s ago 9m 12.9M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T05:33:28.518 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (41s) 3s ago 9m 52.9M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T05:33:28.519 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8s) 3s ago 9m 16.3M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T05:33:28.519 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9m) 3s ago 9m 92.7M 2500M 16.2.10 c56e13ff90f2 23decfd9c6b2 2022-09-29T05:33:28.519 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8m) 3s ago 8m 100M 2500M 16.2.10 c56e13ff90f2 23460a09c0a7 2022-09-29T05:33:28.519 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4m) 3s ago 7m 79.1M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T05:33:28.896 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T05:33:28.896 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T05:33:28.896 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T05:33:28.897 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:33:28.897 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T05:33:28.897 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T05:33:28.898 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:33:28.898 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T05:33:28.898 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T05:33:28.898 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 6 2022-09-29T05:33:28.899 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:33:28.899 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T05:33:28.899 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T05:33:28.899 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:33:28.900 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T05:33:28.900 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 4, 2022-09-29T05:33:28.900 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 11 2022-09-29T05:33:28.900 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T05:33:28.901 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T05:33:29.059 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:33:28 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:33:28.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi083:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi153.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:33:29.059 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:33:28 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:33:28.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephPGImbalance\nexpr: abs(((ceph_osd_numpg > 0) - on(job) group_left() avg by(job) (ceph_osd_numpg\n > 0)) / on(job) group_left() avg by(job) (ceph_osd_numpg > 0)) * on(ceph_daemon)\n group_left(hostname) ceph_osd_metadata > 0.3\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.5\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} deviates by\n more than 30% from average PG count.\n summary: PGs are not balanced across OSDs\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi083:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)\", cluster_addr=\"172.21.15.83\", device_class=\"ssd\", hostname=\"smithi083\", instance=\"smithi153.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.83\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:33:29.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:28 smithi153 ceph-mon[70044]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded) 2022-09-29T05:33:29.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:28 smithi153 ceph-mon[70044]: from='client.44245 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:33:29.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:28 smithi153 ceph-mon[70044]: from='client.44251 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:33:29.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:28 smithi153 ceph-mon[70044]: Detected new or changed devices on smithi153 2022-09-29T05:33:29.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:28 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:33:29.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:28 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:33:29.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:28 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:33:29.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:28 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:33:29.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:28 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:33:29.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:28 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:33:29.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:28 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:33:29.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:28 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2022-09-29T05:33:29.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:28 smithi153 ceph-mon[70044]: from='mon.1 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2022-09-29T05:33:29.063 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:28 smithi153 ceph-mon[70044]: Upgrade: unsafe to stop osd(s) at this time (16 PGs are or would become offline) 2022-09-29T05:33:29.213 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:33:28 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: 2022-09-29T05:33:28.743+0000 7f9b26e95700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (16 PGs are or would become offline) 2022-09-29T05:33:29.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:28 smithi083 ceph-mon[79930]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded) 2022-09-29T05:33:29.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:28 smithi083 ceph-mon[79930]: from='client.44245 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:33:29.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:28 smithi083 ceph-mon[79930]: from='client.44251 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:33:29.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:28 smithi083 ceph-mon[79930]: Detected new or changed devices on smithi153 2022-09-29T05:33:29.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:28 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:33:29.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:28 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:33:29.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:28 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:33:29.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:28 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:33:29.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:28 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:33:29.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:28 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:33:29.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:28 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:33:29.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:28 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2022-09-29T05:33:29.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:28 smithi083 ceph-mon[79930]: from='mon.1 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2022-09-29T05:33:29.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:28 smithi083 ceph-mon[79930]: Upgrade: unsafe to stop osd(s) at this time (16 PGs are or would become offline) 2022-09-29T05:33:29.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:28 smithi083 ceph-mon[77242]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded) 2022-09-29T05:33:29.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:28 smithi083 ceph-mon[77242]: from='client.44245 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:33:29.218 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:28 smithi083 ceph-mon[77242]: from='client.44251 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:33:29.218 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:28 smithi083 ceph-mon[77242]: Detected new or changed devices on smithi153 2022-09-29T05:33:29.218 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:28 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:33:29.218 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:28 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:33:29.219 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:28 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:33:29.219 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:28 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:33:29.219 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:28 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:33:29.219 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:28 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:33:29.220 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:28 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:33:29.220 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:28 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2022-09-29T05:33:29.220 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:28 smithi083 ceph-mon[77242]: from='mon.1 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2022-09-29T05:33:29.220 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:28 smithi083 ceph-mon[77242]: Upgrade: unsafe to stop osd(s) at this time (16 PGs are or would become offline) 2022-09-29T05:33:30.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:29 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/466236654' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:33:30.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:29 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:33:30.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:29 smithi153 ceph-mon[70044]: pgmap v149: 105 pgs: 11 peering, 4 active+undersized, 90 active+clean; 583 KiB data, 61 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:33:30.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:29 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/466236654' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:33:30.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:29 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:33:30.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:29 smithi083 ceph-mon[79930]: pgmap v149: 105 pgs: 11 peering, 4 active+undersized, 90 active+clean; 583 KiB data, 61 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:33:30.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:29 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/466236654' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:33:30.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:29 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:33:30.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:29 smithi083 ceph-mon[77242]: pgmap v149: 105 pgs: 11 peering, 4 active+undersized, 90 active+clean; 583 KiB data, 61 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:33:31.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:33:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:33:31] "GET /metrics HTTP/1.1" 200 215842 "" "Prometheus/2.33.4" 2022-09-29T05:33:32.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:32 smithi153 ceph-mon[70044]: pgmap v150: 105 pgs: 105 active+clean; 583 KiB data, 61 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:33:32.559 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:33:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:33:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:33:32.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:32 smithi083 ceph-mon[79930]: pgmap v150: 105 pgs: 105 active+clean; 583 KiB data, 61 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:33:32.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:32 smithi083 ceph-mon[77242]: pgmap v150: 105 pgs: 105 active+clean; 583 KiB data, 61 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:33:33.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:33:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:33:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:33:33.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:33:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:33:33.615Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:33:34.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:34 smithi153 ceph-mon[70044]: pgmap v151: 105 pgs: 105 active+clean; 583 KiB data, 61 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:33:34.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:34 smithi083 ceph-mon[79930]: pgmap v151: 105 pgs: 105 active+clean; 583 KiB data, 61 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:33:34.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:34 smithi083 ceph-mon[77242]: pgmap v151: 105 pgs: 105 active+clean; 583 KiB data, 61 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:33:36.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:36 smithi083 ceph-mon[79930]: pgmap v152: 105 pgs: 105 active+clean; 583 KiB data, 61 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:33:36.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:36 smithi083 ceph-mon[77242]: pgmap v152: 105 pgs: 105 active+clean; 583 KiB data, 61 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:33:37.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:36 smithi153 ceph-mon[70044]: pgmap v152: 105 pgs: 105 active+clean; 583 KiB data, 61 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:33:38.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:38 smithi083 ceph-mon[79930]: pgmap v153: 105 pgs: 105 active+clean; 583 KiB data, 61 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:33:38.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:38 smithi083 ceph-mon[77242]: pgmap v153: 105 pgs: 105 active+clean; 583 KiB data, 61 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:33:39.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:38 smithi153 ceph-mon[70044]: pgmap v153: 105 pgs: 105 active+clean; 583 KiB data, 61 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:33:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:33:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:33:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:33:40.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:33:40.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:33:40.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:33:40.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:39 smithi083 ceph-mon[77242]: pgmap v154: 105 pgs: 105 active+clean; 583 KiB data, 61 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:33:40.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:33:40.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:33:40.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:33:40.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:33:40.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:33:40.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:33:40.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:39 smithi083 ceph-mon[79930]: pgmap v154: 105 pgs: 105 active+clean; 583 KiB data, 61 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:33:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:33:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:33:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:33:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:33:40.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:33:40.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:33:40.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:39 smithi153 ceph-mon[70044]: pgmap v154: 105 pgs: 105 active+clean; 583 KiB data, 61 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:33:41.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:33:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:33:41] "GET /metrics HTTP/1.1" 200 215770 "" "Prometheus/2.33.4" 2022-09-29T05:33:42.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:42 smithi153 ceph-mon[70044]: pgmap v155: 105 pgs: 105 active+clean; 583 KiB data, 61 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:33:42.560 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:33:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:33:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:33:42.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:42 smithi083 ceph-mon[79930]: pgmap v155: 105 pgs: 105 active+clean; 583 KiB data, 61 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:33:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:42 smithi083 ceph-mon[77242]: pgmap v155: 105 pgs: 105 active+clean; 583 KiB data, 61 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:33:43.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:33:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:33:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:33:43.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:33:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:33:43.537Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:33:44.521 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:44 smithi153 ceph-mon[70044]: pgmap v156: 105 pgs: 105 active+clean; 583 KiB data, 61 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:33:44.522 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:44 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2022-09-29T05:33:44.522 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:44 smithi153 ceph-mon[70044]: from='mon.1 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2022-09-29T05:33:44.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:44 smithi083 ceph-mon[79930]: pgmap v156: 105 pgs: 105 active+clean; 583 KiB data, 61 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:33:44.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:44 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2022-09-29T05:33:44.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:44 smithi083 ceph-mon[79930]: from='mon.1 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2022-09-29T05:33:44.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:44 smithi083 ceph-mon[77242]: pgmap v156: 105 pgs: 105 active+clean; 583 KiB data, 61 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:33:44.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:44 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2022-09-29T05:33:44.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:44 smithi083 ceph-mon[77242]: from='mon.1 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2022-09-29T05:33:45.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:45 smithi153 ceph-mon[70044]: Upgrade: osd.6 is safe to restart 2022-09-29T05:33:45.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:45 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:33:45.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:45 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2022-09-29T05:33:45.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:45 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:33:45.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:45 smithi083 ceph-mon[77242]: Upgrade: osd.6 is safe to restart 2022-09-29T05:33:45.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:45 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:33:45.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:45 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2022-09-29T05:33:45.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:45 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:33:45.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:45 smithi083 ceph-mon[79930]: Upgrade: osd.6 is safe to restart 2022-09-29T05:33:45.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:45 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:33:45.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:45 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2022-09-29T05:33:45.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:45 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:33:46.940 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:46 smithi153 ceph-mon[70044]: Upgrade: Updating osd.6 2022-09-29T05:33:46.941 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:46 smithi153 ceph-mon[70044]: Deploying daemon osd.6 on smithi153 2022-09-29T05:33:46.941 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:46 smithi153 ceph-mon[70044]: pgmap v157: 105 pgs: 105 active+clean; 583 KiB data, 61 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:33:46.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:46 smithi083 ceph-mon[79930]: Upgrade: Updating osd.6 2022-09-29T05:33:46.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:46 smithi083 ceph-mon[79930]: Deploying daemon osd.6 on smithi153 2022-09-29T05:33:46.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:46 smithi083 ceph-mon[79930]: pgmap v157: 105 pgs: 105 active+clean; 583 KiB data, 61 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:33:46.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:46 smithi083 ceph-mon[77242]: Upgrade: Updating osd.6 2022-09-29T05:33:46.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:46 smithi083 ceph-mon[77242]: Deploying daemon osd.6 on smithi153 2022-09-29T05:33:46.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:46 smithi083 ceph-mon[77242]: pgmap v157: 105 pgs: 105 active+clean; 583 KiB data, 61 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:33:47.559 INFO:journalctl@ceph.osd.6.smithi153.stdout:Sep 29 05:33:47 smithi153 systemd[1]: Stopping Ceph osd.6 for 69111522-3fb6-11ed-8431-001a4aab830c... 2022-09-29T05:33:47.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:47 smithi083 ceph-mon[79930]: osd.6 marked itself down and dead 2022-09-29T05:33:47.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:47 smithi083 ceph-mon[77242]: osd.6 marked itself down and dead 2022-09-29T05:33:48.012 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:47 smithi153 ceph-mon[70044]: osd.6 marked itself down and dead 2022-09-29T05:33:48.013 INFO:journalctl@ceph.osd.6.smithi153.stdout:Sep 29 05:33:47 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-osd-6[48611]: debug 2022-09-29T05:33:47.617+0000 7f7b56d2e700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2022-09-29T05:33:48.013 INFO:journalctl@ceph.osd.6.smithi153.stdout:Sep 29 05:33:47 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-osd-6[48611]: debug 2022-09-29T05:33:47.617+0000 7f7b56d2e700 -1 osd.6 94 *** Got signal Terminated *** 2022-09-29T05:33:48.013 INFO:journalctl@ceph.osd.6.smithi153.stdout:Sep 29 05:33:47 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-osd-6[48611]: debug 2022-09-29T05:33:47.617+0000 7f7b56d2e700 -1 osd.6 94 *** Immediate shutdown (osd_fast_shutdown=true) *** 2022-09-29T05:33:48.309 INFO:journalctl@ceph.osd.6.smithi153.stdout:Sep 29 05:33:48 smithi153 bash[78716]: ceph-69111522-3fb6-11ed-8431-001a4aab830c-osd-6 2022-09-29T05:33:48.933 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:48 smithi153 ceph-mon[70044]: pgmap v158: 105 pgs: 105 active+clean; 583 KiB data, 61 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:33:48.934 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:48 smithi153 ceph-mon[70044]: Health check failed: 1 osds down (OSD_DOWN) 2022-09-29T05:33:48.934 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:48 smithi153 ceph-mon[70044]: osdmap e95: 8 total, 7 up, 8 in 2022-09-29T05:33:48.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:48 smithi083 ceph-mon[79930]: pgmap v158: 105 pgs: 105 active+clean; 583 KiB data, 61 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:33:48.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:48 smithi083 ceph-mon[79930]: Health check failed: 1 osds down (OSD_DOWN) 2022-09-29T05:33:48.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:48 smithi083 ceph-mon[79930]: osdmap e95: 8 total, 7 up, 8 in 2022-09-29T05:33:48.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:48 smithi083 ceph-mon[77242]: pgmap v158: 105 pgs: 105 active+clean; 583 KiB data, 61 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:33:48.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:48 smithi083 ceph-mon[77242]: Health check failed: 1 osds down (OSD_DOWN) 2022-09-29T05:33:48.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:48 smithi083 ceph-mon[77242]: osdmap e95: 8 total, 7 up, 8 in 2022-09-29T05:33:49.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:49 smithi083 ceph-mon[79930]: osdmap e96: 8 total, 7 up, 8 in 2022-09-29T05:33:49.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:49 smithi083 ceph-mon[77242]: osdmap e96: 8 total, 7 up, 8 in 2022-09-29T05:33:50.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:49 smithi153 ceph-mon[70044]: osdmap e96: 8 total, 7 up, 8 in 2022-09-29T05:33:50.059 INFO:journalctl@ceph.osd.6.smithi153.stdout:Sep 29 05:33:49 smithi153 systemd[1]: ceph-69111522-3fb6-11ed-8431-001a4aab830c@osd.6.service: Succeeded. 2022-09-29T05:33:50.060 INFO:journalctl@ceph.osd.6.smithi153.stdout:Sep 29 05:33:49 smithi153 systemd[1]: Stopped Ceph osd.6 for 69111522-3fb6-11ed-8431-001a4aab830c. 2022-09-29T05:33:50.060 INFO:journalctl@ceph.osd.6.smithi153.stdout:Sep 29 05:33:49 smithi153 systemd[1]: Starting Ceph osd.6 for 69111522-3fb6-11ed-8431-001a4aab830c... 2022-09-29T05:33:50.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:50 smithi153 ceph-mon[70044]: pgmap v161: 105 pgs: 1 active+undersized+degraded, 2 peering, 7 stale+active+clean, 14 active+undersized, 81 active+clean; 583 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2022-09-29T05:33:50.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:50 smithi153 ceph-mon[70044]: Health check failed: Degraded data redundancy: 4/72 objects degraded (5.556%), 1 pg degraded (PG_DEGRADED) 2022-09-29T05:33:50.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:50 smithi083 ceph-mon[79930]: pgmap v161: 105 pgs: 1 active+undersized+degraded, 2 peering, 7 stale+active+clean, 14 active+undersized, 81 active+clean; 583 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2022-09-29T05:33:50.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:50 smithi083 ceph-mon[79930]: Health check failed: Degraded data redundancy: 4/72 objects degraded (5.556%), 1 pg degraded (PG_DEGRADED) 2022-09-29T05:33:50.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:50 smithi083 ceph-mon[77242]: pgmap v161: 105 pgs: 1 active+undersized+degraded, 2 peering, 7 stale+active+clean, 14 active+undersized, 81 active+clean; 583 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2022-09-29T05:33:50.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:50 smithi083 ceph-mon[77242]: Health check failed: Degraded data redundancy: 4/72 objects degraded (5.556%), 1 pg degraded (PG_DEGRADED) 2022-09-29T05:33:51.309 INFO:journalctl@ceph.osd.6.smithi153.stdout:Sep 29 05:33:50 smithi153 bash[78970]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2022-09-29T05:33:51.309 INFO:journalctl@ceph.osd.6.smithi153.stdout:Sep 29 05:33:50 smithi153 bash[78970]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2022-09-29T05:33:51.309 INFO:journalctl@ceph.osd.6.smithi153.stdout:Sep 29 05:33:50 smithi153 bash[78970]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2022-09-29T05:33:51.310 INFO:journalctl@ceph.osd.6.smithi153.stdout:Sep 29 05:33:50 smithi153 bash[78970]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2022-09-29T05:33:51.310 INFO:journalctl@ceph.osd.6.smithi153.stdout:Sep 29 05:33:50 smithi153 bash[78970]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2022-09-29T05:33:51.311 INFO:journalctl@ceph.osd.6.smithi153.stdout:Sep 29 05:33:50 smithi153 bash[78970]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2022-09-29T05:33:51.311 INFO:journalctl@ceph.osd.6.smithi153.stdout:Sep 29 05:33:50 smithi153 bash[78970]: --> ceph-volume raw activate successful for osd ID: 6 2022-09-29T05:33:51.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:33:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:33:51] "GET /metrics HTTP/1.1" 200 215770 "" "Prometheus/2.33.4" 2022-09-29T05:33:52.559 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:33:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:33:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:33:52.559 INFO:journalctl@ceph.osd.6.smithi153.stdout:Sep 29 05:33:52 smithi153 podman[79257]: 2022-09-29T05:33:52.560 INFO:journalctl@ceph.osd.6.smithi153.stdout:Sep 29 05:33:52 smithi153 bash[79257]: 760baeb754de5890bd8f6855078813cfcb0c70343718b4716adfcbcdfd64ba26 2022-09-29T05:33:52.560 INFO:journalctl@ceph.osd.6.smithi153.stdout:Sep 29 05:33:52 smithi153 ceph-osd[79296]: -- 172.21.15.153:0/349288191 <== mon.2 v2:172.21.15.153:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55a782480680 con 0x55a782444c00 2022-09-29T05:33:52.560 INFO:journalctl@ceph.osd.6.smithi153.stdout:Sep 29 05:33:52 smithi153 systemd[1]: Started Ceph osd.6 for 69111522-3fb6-11ed-8431-001a4aab830c. 2022-09-29T05:33:52.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:52 smithi083 ceph-mon[79930]: pgmap v162: 105 pgs: 5 active+undersized+degraded, 2 peering, 32 active+undersized, 66 active+clean; 583 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 9/72 objects degraded (12.500%) 2022-09-29T05:33:52.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:52 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:33:52.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:52 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:33:52.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:52 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:33:52.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:52 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:33:52.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:52 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:33:52.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:52 smithi083 ceph-mon[77242]: pgmap v162: 105 pgs: 5 active+undersized+degraded, 2 peering, 32 active+undersized, 66 active+clean; 583 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 9/72 objects degraded (12.500%) 2022-09-29T05:33:52.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:52 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:33:52.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:52 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:33:52.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:52 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:33:52.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:52 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:33:52.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:52 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:33:53.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:52 smithi153 ceph-mon[70044]: pgmap v162: 105 pgs: 5 active+undersized+degraded, 2 peering, 32 active+undersized, 66 active+clean; 583 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 9/72 objects degraded (12.500%) 2022-09-29T05:33:53.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:52 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:33:53.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:52 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:33:53.026 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:52 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:33:53.026 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:52 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:33:53.026 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:52 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:33:53.026 INFO:journalctl@ceph.osd.6.smithi153.stdout:Sep 29 05:33:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-osd-6[79273]: 2022-09-29T05:33:52.786+0000 7fb6d0f5e5c0 -1 Falling back to public interface 2022-09-29T05:33:53.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:33:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:33:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:33:53.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:33:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:33:53.538Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:33:54.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:54 smithi083 ceph-mon[77242]: pgmap v163: 105 pgs: 5 active+undersized+degraded, 2 peering, 32 active+undersized, 66 active+clean; 583 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 9/72 objects degraded (12.500%) 2022-09-29T05:33:54.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:54 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:33:54.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:54 smithi083 ceph-mon[79930]: pgmap v163: 105 pgs: 5 active+undersized+degraded, 2 peering, 32 active+undersized, 66 active+clean; 583 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 9/72 objects degraded (12.500%) 2022-09-29T05:33:54.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:54 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:33:55.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:54 smithi153 ceph-mon[70044]: pgmap v163: 105 pgs: 5 active+undersized+degraded, 2 peering, 32 active+undersized, 66 active+clean; 583 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 9/72 objects degraded (12.500%) 2022-09-29T05:33:55.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:54 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:33:55.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:55 smithi153 ceph-mon[70044]: from='osd.6 [v2:172.21.15.153:6816/1643674275,v1:172.21.15.153:6817/1643674275]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2022-09-29T05:33:55.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:55 smithi153 ceph-mon[70044]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2022-09-29T05:33:55.809 INFO:journalctl@ceph.osd.6.smithi153.stdout:Sep 29 05:33:55 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-osd-6[79273]: 2022-09-29T05:33:55.507+0000 7fb6d0f5e5c0 -1 osd.6 94 log_to_monitors true 2022-09-29T05:33:55.810 INFO:journalctl@ceph.osd.6.smithi153.stdout:Sep 29 05:33:55 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-osd-6[79273]: 2022-09-29T05:33:55.731+0000 7fb6beedc700 -1 osd.6 94 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2022-09-29T05:33:55.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:55 smithi083 ceph-mon[79930]: from='osd.6 [v2:172.21.15.153:6816/1643674275,v1:172.21.15.153:6817/1643674275]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2022-09-29T05:33:55.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:55 smithi083 ceph-mon[79930]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2022-09-29T05:33:55.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:55 smithi083 ceph-mon[77242]: from='osd.6 [v2:172.21.15.153:6816/1643674275,v1:172.21.15.153:6817/1643674275]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2022-09-29T05:33:55.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:55 smithi083 ceph-mon[77242]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2022-09-29T05:33:57.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:56 smithi153 ceph-mon[70044]: pgmap v164: 105 pgs: 5 active+undersized+degraded, 34 active+undersized, 66 active+clean; 583 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 9/72 objects degraded (12.500%) 2022-09-29T05:33:57.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:56 smithi153 ceph-mon[70044]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2022-09-29T05:33:57.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:56 smithi153 ceph-mon[70044]: osdmap e97: 8 total, 7 up, 8 in 2022-09-29T05:33:57.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:56 smithi153 ceph-mon[70044]: from='osd.6 [v2:172.21.15.153:6816/1643674275,v1:172.21.15.153:6817/1643674275]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2022-09-29T05:33:57.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:56 smithi153 ceph-mon[70044]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2022-09-29T05:33:57.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:56 smithi083 ceph-mon[79930]: pgmap v164: 105 pgs: 5 active+undersized+degraded, 34 active+undersized, 66 active+clean; 583 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 9/72 objects degraded (12.500%) 2022-09-29T05:33:57.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:56 smithi083 ceph-mon[79930]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2022-09-29T05:33:57.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:56 smithi083 ceph-mon[79930]: osdmap e97: 8 total, 7 up, 8 in 2022-09-29T05:33:57.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:56 smithi083 ceph-mon[79930]: from='osd.6 [v2:172.21.15.153:6816/1643674275,v1:172.21.15.153:6817/1643674275]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2022-09-29T05:33:57.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:56 smithi083 ceph-mon[79930]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2022-09-29T05:33:57.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:56 smithi083 ceph-mon[77242]: pgmap v164: 105 pgs: 5 active+undersized+degraded, 34 active+undersized, 66 active+clean; 583 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 9/72 objects degraded (12.500%) 2022-09-29T05:33:57.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:56 smithi083 ceph-mon[77242]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2022-09-29T05:33:57.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:56 smithi083 ceph-mon[77242]: osdmap e97: 8 total, 7 up, 8 in 2022-09-29T05:33:57.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:56 smithi083 ceph-mon[77242]: from='osd.6 [v2:172.21.15.153:6816/1643674275,v1:172.21.15.153:6817/1643674275]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2022-09-29T05:33:57.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:56 smithi083 ceph-mon[77242]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2022-09-29T05:33:58.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:57 smithi153 ceph-mon[70044]: Health check cleared: OSD_DOWN (was: 1 osds down) 2022-09-29T05:33:58.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:57 smithi153 ceph-mon[70044]: osd.6 [v2:172.21.15.153:6816/1643674275,v1:172.21.15.153:6817/1643674275] boot 2022-09-29T05:33:58.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:57 smithi153 ceph-mon[70044]: osdmap e98: 8 total, 8 up, 8 in 2022-09-29T05:33:58.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:57 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2022-09-29T05:33:58.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:57 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:33:58.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:57 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:33:58.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:57 smithi153 ceph-mon[70044]: osdmap e99: 8 total, 8 up, 8 in 2022-09-29T05:33:58.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:57 smithi083 ceph-mon[79930]: Health check cleared: OSD_DOWN (was: 1 osds down) 2022-09-29T05:33:58.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:57 smithi083 ceph-mon[79930]: osd.6 [v2:172.21.15.153:6816/1643674275,v1:172.21.15.153:6817/1643674275] boot 2022-09-29T05:33:58.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:57 smithi083 ceph-mon[79930]: osdmap e98: 8 total, 8 up, 8 in 2022-09-29T05:33:58.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:57 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2022-09-29T05:33:58.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:57 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:33:58.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:57 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:33:58.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:57 smithi083 ceph-mon[79930]: osdmap e99: 8 total, 8 up, 8 in 2022-09-29T05:33:58.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:57 smithi083 ceph-mon[77242]: Health check cleared: OSD_DOWN (was: 1 osds down) 2022-09-29T05:33:58.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:57 smithi083 ceph-mon[77242]: osd.6 [v2:172.21.15.153:6816/1643674275,v1:172.21.15.153:6817/1643674275] boot 2022-09-29T05:33:58.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:57 smithi083 ceph-mon[77242]: osdmap e98: 8 total, 8 up, 8 in 2022-09-29T05:33:58.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:57 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2022-09-29T05:33:58.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:57 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:33:58.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:57 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:33:58.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:57 smithi083 ceph-mon[77242]: osdmap e99: 8 total, 8 up, 8 in 2022-09-29T05:33:58.938 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:58 smithi153 ceph-mon[70044]: pgmap v167: 105 pgs: 5 active+undersized+degraded, 34 active+undersized, 66 active+clean; 583 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 9/72 objects degraded (12.500%) 2022-09-29T05:33:58.938 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:58 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:33:58.938 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:33:58 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:33:59.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:58 smithi083 ceph-mon[79930]: pgmap v167: 105 pgs: 5 active+undersized+degraded, 34 active+undersized, 66 active+clean; 583 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 9/72 objects degraded (12.500%) 2022-09-29T05:33:59.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:58 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:33:59.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:33:58 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:33:59.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:58 smithi083 ceph-mon[77242]: pgmap v167: 105 pgs: 5 active+undersized+degraded, 34 active+undersized, 66 active+clean; 583 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 9/72 objects degraded (12.500%) 2022-09-29T05:33:59.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:58 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:33:59.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:33:58 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:33:59.224 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T05:33:59.512 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T05:33:59.512 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7m) 100s ago 8m 26.4M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T05:33:59.512 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7m) 2s ago 7m 60.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T05:33:59.512 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9m) 100s ago 9m 15.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T05:33:59.513 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9m) 2s ago 9m 19.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T05:33:59.513 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4m) 2s ago 11m 430M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T05:33:59.513 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5m) 100s ago 13m 490M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T05:33:59.513 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4m) 100s ago 13m 86.0M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T05:33:59.513 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3m) 2s ago 11m 72.4M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T05:33:59.514 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4m) 100s ago 11m 70.7M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T05:33:59.514 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8m) 100s ago 8m 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T05:33:59.514 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8m) 2s ago 8m 28.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T05:33:59.514 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3m) 100s ago 11m 63.4M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T05:33:59.515 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2m) 100s ago 11m 56.5M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T05:33:59.515 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2m) 100s ago 10m 51.9M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T05:33:59.515 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (105s) 100s ago 10m 12.9M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T05:33:59.515 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (72s) 2s ago 10m 56.7M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T05:33:59.515 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (39s) 2s ago 9m 57.3M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T05:33:59.515 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7s) 2s ago 9m 12.7M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T05:33:59.516 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9m) 2s ago 9m 104M 2500M 16.2.10 c56e13ff90f2 23460a09c0a7 2022-09-29T05:33:59.516 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4m) 2s ago 8m 79.2M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T05:33:59.868 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T05:33:59.868 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T05:33:59.868 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T05:33:59.868 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:33:59.869 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T05:33:59.869 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T05:33:59.869 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:33:59.869 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T05:33:59.869 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 1, 2022-09-29T05:33:59.870 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 7 2022-09-29T05:33:59.870 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:33:59.870 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T05:33:59.870 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T05:33:59.870 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:33:59.871 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T05:33:59.871 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 3, 2022-09-29T05:33:59.871 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 12 2022-09-29T05:33:59.871 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T05:33:59.871 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T05:34:00.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:00 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:34:00.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:00 smithi153 ceph-mon[70044]: pgmap v169: 105 pgs: 5 active+undersized+degraded, 23 active+undersized, 77 active+clean; 583 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 9/72 objects degraded (12.500%) 2022-09-29T05:34:00.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:00 smithi153 ceph-mon[70044]: from='client.34335 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:34:00.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:00 smithi153 ceph-mon[70044]: from='client.44275 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:34:00.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:00 smithi153 ceph-mon[70044]: Health check update: Degraded data redundancy: 9/72 objects degraded (12.500%), 5 pgs degraded (PG_DEGRADED) 2022-09-29T05:34:00.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:00 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4176962352' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:34:00.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:00 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:34:00.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:00 smithi083 ceph-mon[79930]: pgmap v169: 105 pgs: 5 active+undersized+degraded, 23 active+undersized, 77 active+clean; 583 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 9/72 objects degraded (12.500%) 2022-09-29T05:34:00.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:00 smithi083 ceph-mon[79930]: from='client.34335 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:34:00.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:00 smithi083 ceph-mon[79930]: from='client.44275 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:34:00.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:00 smithi083 ceph-mon[79930]: Health check update: Degraded data redundancy: 9/72 objects degraded (12.500%), 5 pgs degraded (PG_DEGRADED) 2022-09-29T05:34:00.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:00 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4176962352' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:34:00.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:00 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:34:00.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:00 smithi083 ceph-mon[77242]: pgmap v169: 105 pgs: 5 active+undersized+degraded, 23 active+undersized, 77 active+clean; 583 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 9/72 objects degraded (12.500%) 2022-09-29T05:34:00.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:00 smithi083 ceph-mon[77242]: from='client.34335 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:34:00.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:00 smithi083 ceph-mon[77242]: from='client.44275 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:34:00.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:00 smithi083 ceph-mon[77242]: Health check update: Degraded data redundancy: 9/72 objects degraded (12.500%), 5 pgs degraded (PG_DEGRADED) 2022-09-29T05:34:00.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:00 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4176962352' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:34:01.213 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:34:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: 2022-09-29T05:34:00.762+0000 7f9b26e95700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (11 PGs are or would become offline) 2022-09-29T05:34:01.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:34:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:34:01] "GET /metrics HTTP/1.1" 200 216011 "" "Prometheus/2.33.4" 2022-09-29T05:34:01.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:01 smithi083 ceph-mon[79930]: Detected new or changed devices on smithi153 2022-09-29T05:34:01.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:01 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:34:01.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:01 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:34:01.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:01 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:34:01.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:01 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:34:01.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:01 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:34:01.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:01 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:34:01.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:01 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:34:01.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:01 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2022-09-29T05:34:01.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:01 smithi083 ceph-mon[77242]: Detected new or changed devices on smithi153 2022-09-29T05:34:01.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:01 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:34:01.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:01 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:34:01.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:01 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:34:01.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:01 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:34:01.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:01 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:34:01.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:01 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:34:01.968 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:01 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:34:01.968 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:01 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2022-09-29T05:34:02.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:01 smithi153 ceph-mon[70044]: Detected new or changed devices on smithi153 2022-09-29T05:34:02.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:01 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:34:02.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:01 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:34:02.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:01 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:34:02.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:01 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:34:02.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:01 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:34:02.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:01 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:34:02.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:01 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:34:02.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:01 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2022-09-29T05:34:02.670 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:34:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:34:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:34:02.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:02 smithi083 ceph-mon[79930]: from='mon.1 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2022-09-29T05:34:02.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:02 smithi083 ceph-mon[79930]: Upgrade: unsafe to stop osd(s) at this time (11 PGs are or would become offline) 2022-09-29T05:34:02.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:02 smithi083 ceph-mon[79930]: pgmap v170: 105 pgs: 105 active+clean; 583 KiB data, 66 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:34:02.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:02 smithi083 ceph-mon[79930]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 9/72 objects degraded (12.500%), 5 pgs degraded) 2022-09-29T05:34:02.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:02 smithi083 ceph-mon[77242]: from='mon.1 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2022-09-29T05:34:02.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:02 smithi083 ceph-mon[77242]: Upgrade: unsafe to stop osd(s) at this time (11 PGs are or would become offline) 2022-09-29T05:34:02.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:02 smithi083 ceph-mon[77242]: pgmap v170: 105 pgs: 105 active+clean; 583 KiB data, 66 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:34:02.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:02 smithi083 ceph-mon[77242]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 9/72 objects degraded (12.500%), 5 pgs degraded) 2022-09-29T05:34:03.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:02 smithi153 ceph-mon[70044]: from='mon.1 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2022-09-29T05:34:03.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:02 smithi153 ceph-mon[70044]: Upgrade: unsafe to stop osd(s) at this time (11 PGs are or would become offline) 2022-09-29T05:34:03.026 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:02 smithi153 ceph-mon[70044]: pgmap v170: 105 pgs: 105 active+clean; 583 KiB data, 66 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:34:03.026 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:02 smithi153 ceph-mon[70044]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 9/72 objects degraded (12.500%), 5 pgs degraded) 2022-09-29T05:34:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:34:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:34:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:34:03.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:34:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:34:03.540Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:34:04.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:04 smithi083 ceph-mon[79930]: pgmap v171: 105 pgs: 105 active+clean; 583 KiB data, 66 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:34:04.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:04 smithi083 ceph-mon[77242]: pgmap v171: 105 pgs: 105 active+clean; 583 KiB data, 66 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:34:05.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:04 smithi153 ceph-mon[70044]: pgmap v171: 105 pgs: 105 active+clean; 583 KiB data, 66 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:34:06.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:06 smithi083 ceph-mon[79930]: pgmap v172: 105 pgs: 105 active+clean; 583 KiB data, 66 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:34:06.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:06 smithi083 ceph-mon[77242]: pgmap v172: 105 pgs: 105 active+clean; 583 KiB data, 66 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:34:07.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:06 smithi153 ceph-mon[70044]: pgmap v172: 105 pgs: 105 active+clean; 583 KiB data, 66 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:34:08.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:08 smithi083 ceph-mon[79930]: pgmap v173: 105 pgs: 105 active+clean; 583 KiB data, 66 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:34:08.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:08 smithi083 ceph-mon[77242]: pgmap v173: 105 pgs: 105 active+clean; 583 KiB data, 66 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:34:09.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:08 smithi153 ceph-mon[70044]: pgmap v173: 105 pgs: 105 active+clean; 583 KiB data, 66 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:34:10.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:09 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:34:10.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:09 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:34:10.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:09 smithi083 ceph-mon[79930]: pgmap v174: 105 pgs: 105 active+clean; 583 KiB data, 66 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:34:10.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:09 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:34:10.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:09 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:34:10.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:09 smithi083 ceph-mon[77242]: pgmap v174: 105 pgs: 105 active+clean; 583 KiB data, 66 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:34:10.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:09 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:34:10.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:09 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:34:10.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:09 smithi153 ceph-mon[70044]: pgmap v174: 105 pgs: 105 active+clean; 583 KiB data, 66 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:34:11.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:34:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:34:11] "GET /metrics HTTP/1.1" 200 220504 "" "Prometheus/2.33.4" 2022-09-29T05:34:12.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:12 smithi153 ceph-mon[70044]: pgmap v175: 105 pgs: 105 active+clean; 583 KiB data, 66 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:34:12.559 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:34:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:34:12.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:34:12.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:12 smithi083 ceph-mon[79930]: pgmap v175: 105 pgs: 105 active+clean; 583 KiB data, 66 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:34:12.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:12 smithi083 ceph-mon[77242]: pgmap v175: 105 pgs: 105 active+clean; 583 KiB data, 66 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:34:13.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:34:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:34:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:34:13.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:34:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:34:13.540Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:34:14.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:14 smithi153 ceph-mon[70044]: pgmap v176: 105 pgs: 105 active+clean; 583 KiB data, 66 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:34:14.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:14 smithi083 ceph-mon[77242]: pgmap v176: 105 pgs: 105 active+clean; 583 KiB data, 66 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:34:14.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:14 smithi083 ceph-mon[79930]: pgmap v176: 105 pgs: 105 active+clean; 583 KiB data, 66 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:34:16.776 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:16 smithi153 ceph-mon[70044]: pgmap v177: 105 pgs: 105 active+clean; 583 KiB data, 66 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:34:16.777 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:16 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2022-09-29T05:34:16.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:16 smithi083 ceph-mon[77242]: pgmap v177: 105 pgs: 105 active+clean; 583 KiB data, 66 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:34:16.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:16 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2022-09-29T05:34:16.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:16 smithi083 ceph-mon[79930]: pgmap v177: 105 pgs: 105 active+clean; 583 KiB data, 66 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:34:16.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:16 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2022-09-29T05:34:17.940 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:17 smithi153 ceph-mon[70044]: from='mon.1 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2022-09-29T05:34:17.941 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:17 smithi153 ceph-mon[70044]: Upgrade: osd.7 is safe to restart 2022-09-29T05:34:17.941 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:17 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:34:17.941 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:17 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2022-09-29T05:34:17.942 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:17 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:34:17.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:17 smithi083 ceph-mon[79930]: from='mon.1 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2022-09-29T05:34:17.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:17 smithi083 ceph-mon[79930]: Upgrade: osd.7 is safe to restart 2022-09-29T05:34:17.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:17 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:34:17.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:17 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2022-09-29T05:34:17.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:17 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:34:17.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:17 smithi083 ceph-mon[77242]: from='mon.1 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2022-09-29T05:34:17.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:17 smithi083 ceph-mon[77242]: Upgrade: osd.7 is safe to restart 2022-09-29T05:34:17.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:17 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:34:17.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:17 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2022-09-29T05:34:17.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:17 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:34:18.956 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:18 smithi153 ceph-mon[70044]: Upgrade: Updating osd.7 2022-09-29T05:34:18.957 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:18 smithi153 ceph-mon[70044]: Deploying daemon osd.7 on smithi153 2022-09-29T05:34:18.957 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:18 smithi153 ceph-mon[70044]: pgmap v178: 105 pgs: 105 active+clean; 583 KiB data, 66 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:34:18.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:18 smithi083 ceph-mon[79930]: Upgrade: Updating osd.7 2022-09-29T05:34:18.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:18 smithi083 ceph-mon[79930]: Deploying daemon osd.7 on smithi153 2022-09-29T05:34:18.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:18 smithi083 ceph-mon[79930]: pgmap v178: 105 pgs: 105 active+clean; 583 KiB data, 66 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:34:18.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:18 smithi083 ceph-mon[77242]: Upgrade: Updating osd.7 2022-09-29T05:34:18.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:18 smithi083 ceph-mon[77242]: Deploying daemon osd.7 on smithi153 2022-09-29T05:34:18.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:18 smithi083 ceph-mon[77242]: pgmap v178: 105 pgs: 105 active+clean; 583 KiB data, 66 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:34:19.809 INFO:journalctl@ceph.osd.7.smithi153.stdout:Sep 29 05:34:19 smithi153 systemd[1]: Stopping Ceph osd.7 for 69111522-3fb6-11ed-8431-001a4aab830c... 2022-09-29T05:34:20.309 INFO:journalctl@ceph.osd.7.smithi153.stdout:Sep 29 05:34:19 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-osd-7[51632]: debug 2022-09-29T05:34:19.843+0000 7f3e60a64700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2022-09-29T05:34:20.309 INFO:journalctl@ceph.osd.7.smithi153.stdout:Sep 29 05:34:19 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-osd-7[51632]: debug 2022-09-29T05:34:19.843+0000 7f3e60a64700 -1 osd.7 99 *** Got signal Terminated *** 2022-09-29T05:34:20.309 INFO:journalctl@ceph.osd.7.smithi153.stdout:Sep 29 05:34:19 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-osd-7[51632]: debug 2022-09-29T05:34:19.843+0000 7f3e60a64700 -1 osd.7 99 *** Immediate shutdown (osd_fast_shutdown=true) *** 2022-09-29T05:34:20.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:20 smithi083 ceph-mon[79930]: pgmap v179: 105 pgs: 105 active+clean; 583 KiB data, 66 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:34:20.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:20 smithi083 ceph-mon[79930]: osd.7 marked itself down and dead 2022-09-29T05:34:20.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:20 smithi083 ceph-mon[77242]: pgmap v179: 105 pgs: 105 active+clean; 583 KiB data, 66 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:34:20.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:20 smithi083 ceph-mon[77242]: osd.7 marked itself down and dead 2022-09-29T05:34:21.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:20 smithi153 ceph-mon[70044]: pgmap v179: 105 pgs: 105 active+clean; 583 KiB data, 66 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:34:21.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:20 smithi153 ceph-mon[70044]: osd.7 marked itself down and dead 2022-09-29T05:34:21.059 INFO:journalctl@ceph.osd.7.smithi153.stdout:Sep 29 05:34:20 smithi153 bash[81439]: ceph-69111522-3fb6-11ed-8431-001a4aab830c-osd-7 2022-09-29T05:34:21.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:34:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:34:21] "GET /metrics HTTP/1.1" 200 220504 "" "Prometheus/2.33.4" 2022-09-29T05:34:21.988 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:21 smithi083 ceph-mon[77242]: Health check failed: 1 osds down (OSD_DOWN) 2022-09-29T05:34:21.988 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:21 smithi083 ceph-mon[77242]: Health check failed: all OSDs are running quincy or later but require_osd_release < quincy (OSD_UPGRADE_FINISHED) 2022-09-29T05:34:21.988 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:21 smithi083 ceph-mon[77242]: osdmap e100: 8 total, 7 up, 8 in 2022-09-29T05:34:21.989 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:21 smithi083 ceph-mon[79930]: Health check failed: 1 osds down (OSD_DOWN) 2022-09-29T05:34:21.989 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:21 smithi083 ceph-mon[79930]: Health check failed: all OSDs are running quincy or later but require_osd_release < quincy (OSD_UPGRADE_FINISHED) 2022-09-29T05:34:21.989 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:21 smithi083 ceph-mon[79930]: osdmap e100: 8 total, 7 up, 8 in 2022-09-29T05:34:22.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:21 smithi153 ceph-mon[70044]: Health check failed: 1 osds down (OSD_DOWN) 2022-09-29T05:34:22.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:21 smithi153 ceph-mon[70044]: Health check failed: all OSDs are running quincy or later but require_osd_release < quincy (OSD_UPGRADE_FINISHED) 2022-09-29T05:34:22.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:21 smithi153 ceph-mon[70044]: osdmap e100: 8 total, 7 up, 8 in 2022-09-29T05:34:22.467 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:34:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:34:22.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:34:22.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:22 smithi153 ceph-mon[70044]: pgmap v181: 105 pgs: 17 peering, 10 stale+active+clean, 78 active+clean; 583 KiB data, 66 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:34:22.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:22 smithi153 ceph-mon[70044]: Health check failed: Reduced data availability: 4 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2022-09-29T05:34:22.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:22 smithi153 ceph-mon[70044]: osdmap e101: 8 total, 7 up, 8 in 2022-09-29T05:34:22.810 INFO:journalctl@ceph.osd.7.smithi153.stdout:Sep 29 05:34:22 smithi153 systemd[1]: ceph-69111522-3fb6-11ed-8431-001a4aab830c@osd.7.service: Succeeded. 2022-09-29T05:34:22.810 INFO:journalctl@ceph.osd.7.smithi153.stdout:Sep 29 05:34:22 smithi153 systemd[1]: Stopped Ceph osd.7 for 69111522-3fb6-11ed-8431-001a4aab830c. 2022-09-29T05:34:22.810 INFO:journalctl@ceph.osd.7.smithi153.stdout:Sep 29 05:34:22 smithi153 systemd[1]: Starting Ceph osd.7 for 69111522-3fb6-11ed-8431-001a4aab830c... 2022-09-29T05:34:22.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:22 smithi083 ceph-mon[79930]: pgmap v181: 105 pgs: 17 peering, 10 stale+active+clean, 78 active+clean; 583 KiB data, 66 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:34:22.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:22 smithi083 ceph-mon[79930]: Health check failed: Reduced data availability: 4 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2022-09-29T05:34:22.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:22 smithi083 ceph-mon[79930]: osdmap e101: 8 total, 7 up, 8 in 2022-09-29T05:34:22.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:22 smithi083 ceph-mon[77242]: pgmap v181: 105 pgs: 17 peering, 10 stale+active+clean, 78 active+clean; 583 KiB data, 66 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:34:22.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:22 smithi083 ceph-mon[77242]: Health check failed: Reduced data availability: 4 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2022-09-29T05:34:22.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:22 smithi083 ceph-mon[77242]: osdmap e101: 8 total, 7 up, 8 in 2022-09-29T05:34:23.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:34:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:34:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:34:23.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:34:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:34:23.541Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:34:24.309 INFO:journalctl@ceph.osd.7.smithi153.stdout:Sep 29 05:34:23 smithi153 bash[81689]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2022-09-29T05:34:24.310 INFO:journalctl@ceph.osd.7.smithi153.stdout:Sep 29 05:34:23 smithi153 bash[81689]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2022-09-29T05:34:24.310 INFO:journalctl@ceph.osd.7.smithi153.stdout:Sep 29 05:34:23 smithi153 bash[81689]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2022-09-29T05:34:24.310 INFO:journalctl@ceph.osd.7.smithi153.stdout:Sep 29 05:34:23 smithi153 bash[81689]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2022-09-29T05:34:24.310 INFO:journalctl@ceph.osd.7.smithi153.stdout:Sep 29 05:34:23 smithi153 bash[81689]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2022-09-29T05:34:24.311 INFO:journalctl@ceph.osd.7.smithi153.stdout:Sep 29 05:34:23 smithi153 bash[81689]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2022-09-29T05:34:24.311 INFO:journalctl@ceph.osd.7.smithi153.stdout:Sep 29 05:34:23 smithi153 bash[81689]: --> ceph-volume raw activate successful for osd ID: 7 2022-09-29T05:34:24.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:24 smithi083 ceph-mon[79930]: pgmap v183: 105 pgs: 17 peering, 10 stale+active+clean, 78 active+clean; 583 KiB data, 66 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:34:24.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:24 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:34:24.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:24 smithi083 ceph-mon[77242]: pgmap v183: 105 pgs: 17 peering, 10 stale+active+clean, 78 active+clean; 583 KiB data, 66 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:34:24.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:24 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:34:25.024 INFO:journalctl@ceph.osd.7.smithi153.stdout:Sep 29 05:34:24 smithi153 podman[81957]: 2022-09-29T05:34:25.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:24 smithi153 ceph-mon[70044]: pgmap v183: 105 pgs: 17 peering, 10 stale+active+clean, 78 active+clean; 583 KiB data, 66 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:34:25.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:24 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:34:25.559 INFO:journalctl@ceph.osd.7.smithi153.stdout:Sep 29 05:34:25 smithi153 bash[81957]: 6b12b90816000f80b25bf8396c0ff70b0deff3f615168ddc9f28a08b7ec3c9c2 2022-09-29T05:34:25.559 INFO:journalctl@ceph.osd.7.smithi153.stdout:Sep 29 05:34:25 smithi153 ceph-osd[81997]: -- 172.21.15.153:0/2786447703 <== mon.2 v2:172.21.15.153:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5636349e2680 con 0x5636349a6c00 2022-09-29T05:34:25.559 INFO:journalctl@ceph.osd.7.smithi153.stdout:Sep 29 05:34:25 smithi153 systemd[1]: Started Ceph osd.7 for 69111522-3fb6-11ed-8431-001a4aab830c. 2022-09-29T05:34:26.059 INFO:journalctl@ceph.osd.7.smithi153.stdout:Sep 29 05:34:25 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-osd-7[81974]: 2022-09-29T05:34:25.614+0000 7fe7e07435c0 -1 Falling back to public interface 2022-09-29T05:34:26.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:26 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:34:26.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:26 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:34:26.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:26 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:34:26.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:26 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:34:26.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:26 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:34:26.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:26 smithi083 ceph-mon[79930]: pgmap v184: 105 pgs: 4 active+undersized+degraded, 17 peering, 25 active+undersized, 59 active+clean; 583 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 6/72 objects degraded (8.333%) 2022-09-29T05:34:26.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:26 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:34:26.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:26 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:34:26.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:26 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:34:26.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:26 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:34:26.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:26 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:34:26.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:26 smithi083 ceph-mon[77242]: pgmap v184: 105 pgs: 4 active+undersized+degraded, 17 peering, 25 active+undersized, 59 active+clean; 583 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 6/72 objects degraded (8.333%) 2022-09-29T05:34:26.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:26 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:34:26.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:26 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:34:26.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:26 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:34:26.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:26 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:34:26.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:26 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:34:26.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:26 smithi153 ceph-mon[70044]: pgmap v184: 105 pgs: 4 active+undersized+degraded, 17 peering, 25 active+undersized, 59 active+clean; 583 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 6/72 objects degraded (8.333%) 2022-09-29T05:34:27.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:27 smithi083 ceph-mon[79930]: Health check failed: Degraded data redundancy: 6/72 objects degraded (8.333%), 4 pgs degraded (PG_DEGRADED) 2022-09-29T05:34:27.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:27 smithi083 ceph-mon[77242]: Health check failed: Degraded data redundancy: 6/72 objects degraded (8.333%), 4 pgs degraded (PG_DEGRADED) 2022-09-29T05:34:27.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:27 smithi153 ceph-mon[70044]: Health check failed: Degraded data redundancy: 6/72 objects degraded (8.333%), 4 pgs degraded (PG_DEGRADED) 2022-09-29T05:34:28.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:28 smithi083 ceph-mon[79930]: pgmap v185: 105 pgs: 8 active+undersized+degraded, 38 active+undersized, 59 active+clean; 583 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 10/72 objects degraded (13.889%) 2022-09-29T05:34:28.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:28 smithi083 ceph-mon[77242]: pgmap v185: 105 pgs: 8 active+undersized+degraded, 38 active+undersized, 59 active+clean; 583 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 10/72 objects degraded (13.889%) 2022-09-29T05:34:28.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:28 smithi153 ceph-mon[70044]: pgmap v185: 105 pgs: 8 active+undersized+degraded, 38 active+undersized, 59 active+clean; 583 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 10/72 objects degraded (13.889%) 2022-09-29T05:34:28.559 INFO:journalctl@ceph.osd.7.smithi153.stdout:Sep 29 05:34:28 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-osd-7[81974]: 2022-09-29T05:34:28.353+0000 7fe7e07435c0 -1 osd.7 99 log_to_monitors true 2022-09-29T05:34:29.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:29 smithi083 ceph-mon[77242]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 6 pgs peering) 2022-09-29T05:34:29.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:29 smithi083 ceph-mon[77242]: from='osd.7 [v2:172.21.15.153:6824/3970422470,v1:172.21.15.153:6825/3970422470]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2022-09-29T05:34:29.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:29 smithi083 ceph-mon[77242]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2022-09-29T05:34:29.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:29 smithi083 ceph-mon[79930]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 6 pgs peering) 2022-09-29T05:34:29.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:29 smithi083 ceph-mon[79930]: from='osd.7 [v2:172.21.15.153:6824/3970422470,v1:172.21.15.153:6825/3970422470]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2022-09-29T05:34:29.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:29 smithi083 ceph-mon[79930]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2022-09-29T05:34:29.559 INFO:journalctl@ceph.osd.7.smithi153.stdout:Sep 29 05:34:29 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-osd-7[81974]: 2022-09-29T05:34:29.196+0000 7fe7ce6c1700 -1 osd.7 99 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2022-09-29T05:34:29.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:29 smithi153 ceph-mon[70044]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 6 pgs peering) 2022-09-29T05:34:29.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:29 smithi153 ceph-mon[70044]: from='osd.7 [v2:172.21.15.153:6824/3970422470,v1:172.21.15.153:6825/3970422470]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2022-09-29T05:34:29.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:29 smithi153 ceph-mon[70044]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2022-09-29T05:34:30.206 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T05:34:30.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:30 smithi083 ceph-mon[79930]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2022-09-29T05:34:30.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:30 smithi083 ceph-mon[79930]: osdmap e102: 8 total, 7 up, 8 in 2022-09-29T05:34:30.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:30 smithi083 ceph-mon[79930]: from='osd.7 [v2:172.21.15.153:6824/3970422470,v1:172.21.15.153:6825/3970422470]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2022-09-29T05:34:30.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:30 smithi083 ceph-mon[79930]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2022-09-29T05:34:30.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:30 smithi083 ceph-mon[79930]: pgmap v187: 105 pgs: 8 active+undersized+degraded, 38 active+undersized, 59 active+clean; 583 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 10/72 objects degraded (13.889%) 2022-09-29T05:34:30.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:30 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:34:30.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:30 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:34:30.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:30 smithi083 ceph-mon[77242]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2022-09-29T05:34:30.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:30 smithi083 ceph-mon[77242]: osdmap e102: 8 total, 7 up, 8 in 2022-09-29T05:34:30.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:30 smithi083 ceph-mon[77242]: from='osd.7 [v2:172.21.15.153:6824/3970422470,v1:172.21.15.153:6825/3970422470]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2022-09-29T05:34:30.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:30 smithi083 ceph-mon[77242]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2022-09-29T05:34:30.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:30 smithi083 ceph-mon[77242]: pgmap v187: 105 pgs: 8 active+undersized+degraded, 38 active+undersized, 59 active+clean; 583 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 10/72 objects degraded (13.889%) 2022-09-29T05:34:30.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:30 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:34:30.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:30 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:34:30.499 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T05:34:30.499 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8m) 2m ago 8m 26.4M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T05:34:30.499 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8m) 0s ago 8m 59.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T05:34:30.500 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9m) 2m ago 9m 15.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T05:34:30.500 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9m) 0s ago 9m 19.4M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T05:34:30.500 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5m) 0s ago 12m 430M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T05:34:30.500 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5m) 2m ago 13m 490M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T05:34:30.501 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5m) 2m ago 13m 86.0M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T05:34:30.501 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4m) 0s ago 12m 75.3M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T05:34:30.501 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4m) 2m ago 12m 70.7M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T05:34:30.501 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8m) 2m ago 8m 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T05:34:30.501 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8m) 0s ago 8m 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T05:34:30.502 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3m) 2m ago 11m 63.4M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T05:34:30.502 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3m) 2m ago 11m 56.5M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T05:34:30.502 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2m) 2m ago 11m 51.9M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T05:34:30.502 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2m) 2m ago 10m 12.9M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T05:34:30.503 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (103s) 0s ago 10m 61.8M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T05:34:30.503 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (70s) 0s ago 10m 63.4M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T05:34:30.503 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (38s) 0s ago 10m 56.2M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T05:34:30.503 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5s) 0s ago 9m 15.3M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T05:34:30.504 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5m) 0s ago 8m 80.9M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T05:34:30.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:30 smithi153 ceph-mon[70044]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2022-09-29T05:34:30.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:30 smithi153 ceph-mon[70044]: osdmap e102: 8 total, 7 up, 8 in 2022-09-29T05:34:30.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:30 smithi153 ceph-mon[70044]: from='osd.7 [v2:172.21.15.153:6824/3970422470,v1:172.21.15.153:6825/3970422470]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2022-09-29T05:34:30.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:30 smithi153 ceph-mon[70044]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2022-09-29T05:34:30.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:30 smithi153 ceph-mon[70044]: pgmap v187: 105 pgs: 8 active+undersized+degraded, 38 active+undersized, 59 active+clean; 583 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 10/72 objects degraded (13.889%) 2022-09-29T05:34:30.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:30 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:34:30.561 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:30 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:34:30.861 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T05:34:30.861 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T05:34:30.861 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T05:34:30.861 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:34:30.862 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T05:34:30.862 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T05:34:30.862 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:34:30.862 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T05:34:30.863 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T05:34:30.863 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:34:30.863 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T05:34:30.863 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T05:34:30.864 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:34:30.864 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T05:34:30.864 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T05:34:30.864 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T05:34:30.865 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T05:34:30.865 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T05:34:31.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:31 smithi083 ceph-mon[79930]: Health check cleared: OSD_DOWN (was: 1 osds down) 2022-09-29T05:34:31.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:31 smithi083 ceph-mon[79930]: from='client.34359 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:34:31.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:31 smithi083 ceph-mon[79930]: osd.7 [v2:172.21.15.153:6824/3970422470,v1:172.21.15.153:6825/3970422470] boot 2022-09-29T05:34:31.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:31 smithi083 ceph-mon[79930]: osdmap e103: 8 total, 8 up, 8 in 2022-09-29T05:34:31.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:31 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2022-09-29T05:34:31.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:31 smithi083 ceph-mon[79930]: from='client.44299 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:34:31.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:31 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:34:31.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:31 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:34:31.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:31 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/418085742' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:34:31.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:31 smithi083 ceph-mon[79930]: osdmap e104: 8 total, 8 up, 8 in 2022-09-29T05:34:31.467 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:31 smithi083 ceph-mon[77242]: Health check cleared: OSD_DOWN (was: 1 osds down) 2022-09-29T05:34:31.467 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:31 smithi083 ceph-mon[77242]: from='client.34359 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:34:31.468 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:31 smithi083 ceph-mon[77242]: osd.7 [v2:172.21.15.153:6824/3970422470,v1:172.21.15.153:6825/3970422470] boot 2022-09-29T05:34:31.468 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:31 smithi083 ceph-mon[77242]: osdmap e103: 8 total, 8 up, 8 in 2022-09-29T05:34:31.469 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:31 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2022-09-29T05:34:31.469 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:31 smithi083 ceph-mon[77242]: from='client.44299 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:34:31.469 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:31 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:34:31.470 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:31 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:34:31.470 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:31 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/418085742' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:34:31.470 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:31 smithi083 ceph-mon[77242]: osdmap e104: 8 total, 8 up, 8 in 2022-09-29T05:34:31.507 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:31 smithi153 ceph-mon[70044]: Health check cleared: OSD_DOWN (was: 1 osds down) 2022-09-29T05:34:31.507 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:31 smithi153 ceph-mon[70044]: from='client.34359 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:34:31.508 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:31 smithi153 ceph-mon[70044]: osd.7 [v2:172.21.15.153:6824/3970422470,v1:172.21.15.153:6825/3970422470] boot 2022-09-29T05:34:31.508 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:31 smithi153 ceph-mon[70044]: osdmap e103: 8 total, 8 up, 8 in 2022-09-29T05:34:31.508 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:31 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2022-09-29T05:34:31.508 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:31 smithi153 ceph-mon[70044]: from='client.44299 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:34:31.509 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:31 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:34:31.509 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:31 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:34:31.509 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:31 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/418085742' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:34:31.509 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:31 smithi153 ceph-mon[70044]: osdmap e104: 8 total, 8 up, 8 in 2022-09-29T05:34:31.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:34:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:34:31] "GET /metrics HTTP/1.1" 200 220774 "" "Prometheus/2.33.4" 2022-09-29T05:34:32.363 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:32 smithi153 ceph-mon[70044]: pgmap v190: 105 pgs: 9 peering, 6 active+undersized+degraded, 31 active+undersized, 59 active+clean; 583 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 8/72 objects degraded (11.111%) 2022-09-29T05:34:32.364 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:34:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:34:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:34:32.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:32 smithi083 ceph-mon[79930]: pgmap v190: 105 pgs: 9 peering, 6 active+undersized+degraded, 31 active+undersized, 59 active+clean; 583 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 8/72 objects degraded (11.111%) 2022-09-29T05:34:32.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:32 smithi083 ceph-mon[77242]: pgmap v190: 105 pgs: 9 peering, 6 active+undersized+degraded, 31 active+undersized, 59 active+clean; 583 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 8/72 objects degraded (11.111%) 2022-09-29T05:34:33.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:34:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:34:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:34:33.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:33 smithi153 ceph-mon[70044]: Health check update: Degraded data redundancy: 8/72 objects degraded (11.111%), 6 pgs degraded (PG_DEGRADED) 2022-09-29T05:34:33.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:33 smithi083 ceph-mon[79930]: Health check update: Degraded data redundancy: 8/72 objects degraded (11.111%), 6 pgs degraded (PG_DEGRADED) 2022-09-29T05:34:33.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:33 smithi083 ceph-mon[77242]: Health check update: Degraded data redundancy: 8/72 objects degraded (11.111%), 6 pgs degraded (PG_DEGRADED) 2022-09-29T05:34:33.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:34:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: 2022-09-29T05:34:33.663+0000 7f9af51ca700 -1 log_channel(cephadm) log [ERR] : Upgrade: Paused due to UPGRADE_EXCEPTION: Upgrade: failed due to an unexpected exception 2022-09-29T05:34:33.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:34:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:34:33.542Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:34:34.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[77219]: 2022-09-29T05:34:34.199+0000 7f160f665700 -1 log_channel(cluster) log [ERR] : Health check failed: Upgrade: failed due to an unexpected exception (UPGRADE_EXCEPTION) 2022-09-29T05:34:34.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:34 smithi153 ceph-mon[70044]: pgmap v191: 105 pgs: 9 peering, 6 active+undersized+degraded, 31 active+undersized, 59 active+clean; 583 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 8/72 objects degraded (11.111%) 2022-09-29T05:34:34.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:34 smithi153 ceph-mon[70044]: Detected new or changed devices on smithi153 2022-09-29T05:34:34.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:34 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:34:34.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:34 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:34:34.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:34 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:34:34.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:34 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:34:34.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:34 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:34:34.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:34 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:34:34.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:34 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:34:34.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:34 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:34:34.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:34 smithi153 ceph-mon[70044]: Upgrade: Setting container_image for all osd 2022-09-29T05:34:34.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:34 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:34:34.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:34 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2022-09-29T05:34:34.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:34 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2022-09-29T05:34:34.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:34 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2022-09-29T05:34:34.813 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:34 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2022-09-29T05:34:34.813 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:34 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2022-09-29T05:34:34.813 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:34 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2022-09-29T05:34:34.813 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:34 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2022-09-29T05:34:34.813 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:34 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2022-09-29T05:34:34.814 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:34 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2022-09-29T05:34:34.814 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:34 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2022-09-29T05:34:34.814 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:34 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2022-09-29T05:34:34.814 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:34 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2022-09-29T05:34:34.814 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:34 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2022-09-29T05:34:34.815 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:34 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2022-09-29T05:34:34.815 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:34 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2022-09-29T05:34:34.815 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:34 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2022-09-29T05:34:34.815 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:34 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2022-09-29T05:34:34.816 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:34 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2022-09-29T05:34:34.816 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:34 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2022-09-29T05:34:34.816 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:34 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2022-09-29T05:34:34.816 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:34 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2022-09-29T05:34:34.816 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:34 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2022-09-29T05:34:34.817 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:34 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2022-09-29T05:34:34.817 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:34 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2022-09-29T05:34:34.817 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:34 smithi153 ceph-mon[70044]: Upgrade: Setting require_osd_release to 18 unknown 2022-09-29T05:34:34.817 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:34 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "osd require-osd-release", "release": "unknown"}]: dispatch 2022-09-29T05:34:34.817 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:34 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd require-osd-release", "release": "unknown"}]: dispatch 2022-09-29T05:34:34.818 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:34 smithi153 ceph-mon[70044]: Upgrade: Paused due to UPGRADE_EXCEPTION: Upgrade: failed due to an unexpected exception 2022-09-29T05:34:34.818 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:34 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:34:34.818 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:34 smithi153 ceph-mon[70044]: pgmap v192: 105 pgs: 9 peering, 5 active+undersized+degraded, 27 active+undersized, 64 active+clean; 583 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 6/72 objects degraded (8.333%) 2022-09-29T05:34:34.818 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:34 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:34:34.819 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:34 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:34:34.819 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:34 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:34:34.819 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:34 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:34:34.819 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:34 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:34:34.819 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:34 smithi153 ceph-mon[70044]: Health check failed: Upgrade: failed due to an unexpected exception (UPGRADE_EXCEPTION) 2022-09-29T05:34:34.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[79930]: pgmap v191: 105 pgs: 9 peering, 6 active+undersized+degraded, 31 active+undersized, 59 active+clean; 583 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 8/72 objects degraded (11.111%) 2022-09-29T05:34:34.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[79930]: Detected new or changed devices on smithi153 2022-09-29T05:34:34.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:34:34.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:34:34.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:34:34.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:34:34.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:34:34.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:34:34.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:34:34.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:34:34.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[79930]: Upgrade: Setting container_image for all osd 2022-09-29T05:34:34.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:34:34.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2022-09-29T05:34:34.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2022-09-29T05:34:34.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2022-09-29T05:34:34.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2022-09-29T05:34:34.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2022-09-29T05:34:34.968 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2022-09-29T05:34:34.968 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2022-09-29T05:34:34.968 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2022-09-29T05:34:34.968 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2022-09-29T05:34:34.968 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2022-09-29T05:34:34.969 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2022-09-29T05:34:34.969 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2022-09-29T05:34:34.969 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2022-09-29T05:34:34.969 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2022-09-29T05:34:34.970 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2022-09-29T05:34:34.970 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[77242]: pgmap v191: 105 pgs: 9 peering, 6 active+undersized+degraded, 31 active+undersized, 59 active+clean; 583 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 8/72 objects degraded (11.111%) 2022-09-29T05:34:34.970 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[77242]: Detected new or changed devices on smithi153 2022-09-29T05:34:34.971 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:34:34.971 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:34:34.971 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:34:34.971 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:34:34.972 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:34:34.972 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:34:34.972 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:34:34.972 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:34:34.972 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[77242]: Upgrade: Setting container_image for all osd 2022-09-29T05:34:34.973 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:34:34.973 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2022-09-29T05:34:34.973 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2022-09-29T05:34:34.973 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2022-09-29T05:34:34.973 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2022-09-29T05:34:34.974 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2022-09-29T05:34:34.974 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2022-09-29T05:34:34.974 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2022-09-29T05:34:34.974 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2022-09-29T05:34:34.975 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2022-09-29T05:34:34.975 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2022-09-29T05:34:34.975 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2022-09-29T05:34:34.975 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2022-09-29T05:34:34.975 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2022-09-29T05:34:34.976 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2022-09-29T05:34:34.976 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2022-09-29T05:34:34.976 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2022-09-29T05:34:34.976 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2022-09-29T05:34:34.977 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2022-09-29T05:34:34.977 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2022-09-29T05:34:34.977 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2022-09-29T05:34:34.977 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2022-09-29T05:34:34.978 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2022-09-29T05:34:34.978 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2022-09-29T05:34:34.978 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2022-09-29T05:34:34.978 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[79930]: Upgrade: Setting require_osd_release to 18 unknown 2022-09-29T05:34:34.978 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "osd require-osd-release", "release": "unknown"}]: dispatch 2022-09-29T05:34:34.979 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd require-osd-release", "release": "unknown"}]: dispatch 2022-09-29T05:34:34.979 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[79930]: Upgrade: Paused due to UPGRADE_EXCEPTION: Upgrade: failed due to an unexpected exception 2022-09-29T05:34:34.979 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:34:34.979 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[79930]: pgmap v192: 105 pgs: 9 peering, 5 active+undersized+degraded, 27 active+undersized, 64 active+clean; 583 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 6/72 objects degraded (8.333%) 2022-09-29T05:34:34.980 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:34:34.980 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:34:34.980 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:34:34.980 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:34:34.981 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:34:34.981 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[79930]: Health check failed: Upgrade: failed due to an unexpected exception (UPGRADE_EXCEPTION) 2022-09-29T05:34:34.981 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2022-09-29T05:34:34.982 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2022-09-29T05:34:34.982 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2022-09-29T05:34:34.982 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2022-09-29T05:34:34.982 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2022-09-29T05:34:34.983 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2022-09-29T05:34:34.983 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2022-09-29T05:34:34.983 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2022-09-29T05:34:34.983 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2022-09-29T05:34:34.984 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[77242]: Upgrade: Setting require_osd_release to 18 unknown 2022-09-29T05:34:34.984 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "osd require-osd-release", "release": "unknown"}]: dispatch 2022-09-29T05:34:34.984 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd require-osd-release", "release": "unknown"}]: dispatch 2022-09-29T05:34:34.984 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[77242]: Upgrade: Paused due to UPGRADE_EXCEPTION: Upgrade: failed due to an unexpected exception 2022-09-29T05:34:34.985 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:34:34.985 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[77242]: pgmap v192: 105 pgs: 9 peering, 5 active+undersized+degraded, 27 active+undersized, 64 active+clean; 583 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 6/72 objects degraded (8.333%) 2022-09-29T05:34:34.985 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:34:34.985 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:34:34.985 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:34:34.986 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:34:34.986 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:34:34.986 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:34 smithi083 ceph-mon[77242]: Health check failed: Upgrade: failed due to an unexpected exception (UPGRADE_EXCEPTION) 2022-09-29T05:34:36.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:35 smithi153 ceph-mon[70044]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 6/72 objects degraded (8.333%), 5 pgs degraded) 2022-09-29T05:34:36.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:35 smithi083 ceph-mon[79930]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 6/72 objects degraded (8.333%), 5 pgs degraded) 2022-09-29T05:34:36.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:35 smithi083 ceph-mon[77242]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 6/72 objects degraded (8.333%), 5 pgs degraded) 2022-09-29T05:34:37.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:36 smithi153 ceph-mon[70044]: pgmap v193: 105 pgs: 9 peering, 96 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:34:37.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:36 smithi083 ceph-mon[79930]: pgmap v193: 105 pgs: 9 peering, 96 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:34:37.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:36 smithi083 ceph-mon[77242]: pgmap v193: 105 pgs: 9 peering, 96 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:34:38.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:38 smithi083 ceph-mon[79930]: pgmap v194: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:34:38.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:34:38 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:34:38.672Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:34:38.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:38 smithi083 ceph-mon[77242]: pgmap v194: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:34:39.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:38 smithi153 ceph-mon[70044]: pgmap v194: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:34:40.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:34:40.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:34:40.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:34:40.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:34:40.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:34:40.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:39 smithi083 ceph-mon[79930]: pgmap v195: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:34:40.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:34:40.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:34:40.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:34:40.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:34:40.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:34:40.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:39 smithi083 ceph-mon[77242]: pgmap v195: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:34:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:34:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:34:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:34:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:34:40.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:34:40.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:39 smithi153 ceph-mon[70044]: pgmap v195: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:34:41.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:34:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:34:41] "GET /metrics HTTP/1.1" 200 225327 "" "Prometheus/2.33.4" 2022-09-29T05:34:42.728 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:34:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:34:42.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:34:43.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:42 smithi153 ceph-mon[70044]: pgmap v196: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:34:43.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:42 smithi083 ceph-mon[79930]: pgmap v196: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:34:43.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:42 smithi083 ceph-mon[77242]: pgmap v196: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:34:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:34:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:34:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:34:43.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:34:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:34:43.542Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:34:45.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:44 smithi153 ceph-mon[70044]: pgmap v197: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:34:45.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:44 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:34:45.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:44 smithi083 ceph-mon[79930]: pgmap v197: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:34:45.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:44 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:34:45.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:44 smithi083 ceph-mon[77242]: pgmap v197: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:34:45.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:44 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:34:47.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:46 smithi153 ceph-mon[70044]: pgmap v198: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:34:47.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:46 smithi083 ceph-mon[79930]: pgmap v198: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:34:47.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:46 smithi083 ceph-mon[77242]: pgmap v198: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:34:49.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:48 smithi153 ceph-mon[70044]: pgmap v199: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:34:49.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:48 smithi083 ceph-mon[79930]: pgmap v199: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:34:49.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:48 smithi083 ceph-mon[77242]: pgmap v199: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:34:51.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:50 smithi153 ceph-mon[70044]: pgmap v200: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:34:51.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:50 smithi083 ceph-mon[79930]: pgmap v200: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:34:51.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:50 smithi083 ceph-mon[77242]: pgmap v200: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:34:51.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:34:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:34:51] "GET /metrics HTTP/1.1" 200 225327 "" "Prometheus/2.33.4" 2022-09-29T05:34:52.742 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:34:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:34:52.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:34:53.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:52 smithi153 ceph-mon[70044]: pgmap v201: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:34:53.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:52 smithi083 ceph-mon[79930]: pgmap v201: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:34:53.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:52 smithi083 ceph-mon[77242]: pgmap v201: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:34:53.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:34:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:34:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:34:53.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:34:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:34:53.542Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:34:55.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:54 smithi153 ceph-mon[70044]: pgmap v202: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:34:55.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:54 smithi083 ceph-mon[79930]: pgmap v202: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:34:55.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:54 smithi083 ceph-mon[77242]: pgmap v202: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:34:57.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:56 smithi153 ceph-mon[70044]: pgmap v203: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:34:57.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:56 smithi083 ceph-mon[79930]: pgmap v203: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:34:57.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:56 smithi083 ceph-mon[77242]: pgmap v203: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:34:59.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:34:58 smithi153 ceph-mon[70044]: pgmap v204: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:34:59.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:34:58 smithi083 ceph-mon[79930]: pgmap v204: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:34:59.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:34:58 smithi083 ceph-mon[77242]: pgmap v204: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:01.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:35:00 smithi153 ceph-mon[70044]: pgmap v205: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:01.189 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T05:35:01.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:35:00 smithi083 ceph-mon[79930]: pgmap v205: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:01.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:35:00 smithi083 ceph-mon[77242]: pgmap v205: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:01.476 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T05:35:01.477 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8m) 2m ago 9m 26.4M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T05:35:01.477 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9m) 31s ago 8m 59.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T05:35:01.477 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10m) 2m ago 10m 15.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T05:35:01.477 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10m) 31s ago 10m 19.4M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T05:35:01.478 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5m) 31s ago 12m 430M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T05:35:01.478 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6m) 2m ago 14m 490M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T05:35:01.478 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5m) 2m ago 14m 86.0M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T05:35:01.478 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4m) 31s ago 12m 75.3M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T05:35:01.479 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5m) 2m ago 12m 70.7M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T05:35:01.479 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9m) 2m ago 9m 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T05:35:01.479 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9m) 31s ago 9m 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T05:35:01.480 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4m) 2m ago 12m 63.4M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T05:35:01.480 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3m) 2m ago 12m 56.5M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T05:35:01.480 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3m) 2m ago 11m 51.9M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T05:35:01.480 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2m) 2m ago 11m 12.9M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T05:35:01.480 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2m) 31s ago 11m 61.8M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T05:35:01.481 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (101s) 31s ago 10m 63.4M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T05:35:01.481 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (69s) 31s ago 10m 56.2M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T05:35:01.481 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (36s) 31s ago 10m 15.3M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T05:35:01.481 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5m) 31s ago 9m 80.9M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T05:35:01.837 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T05:35:01.837 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T05:35:01.838 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T05:35:01.838 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:35:01.838 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T05:35:01.838 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T05:35:01.839 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:35:01.839 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T05:35:01.839 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T05:35:01.840 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:35:01.840 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T05:35:01.840 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T05:35:01.840 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:35:01.841 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T05:35:01.841 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T05:35:01.842 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T05:35:01.842 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T05:35:01.842 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T05:35:01.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:35:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:35:01] "GET /metrics HTTP/1.1" 200 225311 "" "Prometheus/2.33.4" 2022-09-29T05:35:02.757 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:35:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:35:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:35:03.026 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:35:02 smithi153 ceph-mon[70044]: from='client.44311 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:35:03.026 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:35:02 smithi153 ceph-mon[70044]: from='client.44317 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:35:03.026 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:35:02 smithi153 ceph-mon[70044]: pgmap v206: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:03.027 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:35:02 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/986298973' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:35:03.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:35:02 smithi083 ceph-mon[79930]: from='client.44311 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:35:03.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:35:02 smithi083 ceph-mon[79930]: from='client.44317 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:35:03.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:35:02 smithi083 ceph-mon[79930]: pgmap v206: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:03.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:35:02 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/986298973' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:35:03.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:35:02 smithi083 ceph-mon[77242]: from='client.44311 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:35:03.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:35:02 smithi083 ceph-mon[77242]: from='client.44317 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:35:03.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:35:02 smithi083 ceph-mon[77242]: pgmap v206: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:03.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:35:02 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/986298973' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:35:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:35:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:35:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:35:03.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:35:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:35:03.543Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:35:05.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:35:04 smithi153 ceph-mon[70044]: pgmap v207: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:05.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:35:04 smithi083 ceph-mon[79930]: pgmap v207: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:05.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:35:04 smithi083 ceph-mon[77242]: pgmap v207: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:07.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:35:06 smithi153 ceph-mon[70044]: pgmap v208: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:07.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:35:06 smithi083 ceph-mon[79930]: pgmap v208: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:07.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:35:06 smithi083 ceph-mon[77242]: pgmap v208: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:09.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:35:08 smithi153 ceph-mon[70044]: pgmap v209: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:09.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:35:08 smithi083 ceph-mon[79930]: pgmap v209: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:09.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:35:08 smithi083 ceph-mon[77242]: pgmap v209: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:11.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:35:10 smithi153 ceph-mon[70044]: pgmap v210: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:11.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:35:10 smithi083 ceph-mon[79930]: pgmap v210: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:11.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:35:10 smithi083 ceph-mon[77242]: pgmap v210: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:11.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:35:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:35:11] "GET /metrics HTTP/1.1" 200 225286 "" "Prometheus/2.33.4" 2022-09-29T05:35:12.773 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:35:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:35:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:35:13.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:35:12 smithi153 ceph-mon[70044]: pgmap v211: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:13.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:35:12 smithi083 ceph-mon[79930]: pgmap v211: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:13.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:35:12 smithi083 ceph-mon[77242]: pgmap v211: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:13.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:35:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:35:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:35:13.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:35:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:35:13.544Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:35:15.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:35:14 smithi153 ceph-mon[70044]: pgmap v212: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:15.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:35:14 smithi083 ceph-mon[79930]: pgmap v212: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:15.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:35:14 smithi083 ceph-mon[77242]: pgmap v212: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:17.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:35:16 smithi153 ceph-mon[70044]: pgmap v213: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:17.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:35:16 smithi083 ceph-mon[79930]: pgmap v213: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:17.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:35:16 smithi083 ceph-mon[77242]: pgmap v213: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:18.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:35:17 smithi153 ceph-mon[70044]: pgmap v214: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:18.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:35:17 smithi083 ceph-mon[79930]: pgmap v214: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:18.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:35:17 smithi083 ceph-mon[77242]: pgmap v214: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:21.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:35:20 smithi153 ceph-mon[70044]: pgmap v215: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:21.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:35:20 smithi083 ceph-mon[79930]: pgmap v215: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:21.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:35:20 smithi083 ceph-mon[77242]: pgmap v215: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:21.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:35:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:35:21] "GET /metrics HTTP/1.1" 200 225286 "" "Prometheus/2.33.4" 2022-09-29T05:35:22.743 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:35:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:35:22.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:35:23.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:35:22 smithi153 ceph-mon[70044]: pgmap v216: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:23.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:35:22 smithi083 ceph-mon[79930]: pgmap v216: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:23.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:35:22 smithi083 ceph-mon[77242]: pgmap v216: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:23.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:35:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:35:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:35:23.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:35:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:35:23.545Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:35:24.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:35:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:35:24.239Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:35:25.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:35:24 smithi153 ceph-mon[70044]: pgmap v217: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:25.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:35:24 smithi083 ceph-mon[79930]: pgmap v217: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:25.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:35:24 smithi083 ceph-mon[77242]: pgmap v217: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:27.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:35:26 smithi153 ceph-mon[70044]: pgmap v218: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:27.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:35:26 smithi083 ceph-mon[77242]: pgmap v218: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:27.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:35:26 smithi083 ceph-mon[79930]: pgmap v218: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:29.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:35:28 smithi153 ceph-mon[70044]: pgmap v219: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:29.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:35:28 smithi083 ceph-mon[79930]: pgmap v219: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:29.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:35:28 smithi083 ceph-mon[77242]: pgmap v219: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:31.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:35:30 smithi153 ceph-mon[70044]: pgmap v220: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:31.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:35:30 smithi083 ceph-mon[79930]: pgmap v220: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:31.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:35:30 smithi083 ceph-mon[77242]: pgmap v220: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:31.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:35:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:35:31] "GET /metrics HTTP/1.1" 200 225288 "" "Prometheus/2.33.4" 2022-09-29T05:35:32.163 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T05:35:32.450 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T05:35:32.451 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9m) 3m ago 9m 26.4M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T05:35:32.451 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9m) 62s ago 9m 59.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T05:35:32.451 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10m) 3m ago 10m 15.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T05:35:32.451 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10m) 62s ago 10m 19.4M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T05:35:32.452 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6m) 62s ago 13m 430M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T05:35:32.452 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6m) 3m ago 14m 490M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T05:35:32.452 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6m) 3m ago 14m 86.0M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T05:35:32.452 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5m) 62s ago 13m 75.3M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T05:35:32.453 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5m) 3m ago 13m 70.7M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T05:35:32.453 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9m) 3m ago 9m 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T05:35:32.453 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9m) 62s ago 9m 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T05:35:32.453 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4m) 3m ago 12m 63.4M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T05:35:32.454 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4m) 3m ago 12m 56.5M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T05:35:32.454 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3m) 3m ago 12m 51.9M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T05:35:32.454 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3m) 3m ago 11m 12.9M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T05:35:32.454 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2m) 62s ago 11m 61.8M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T05:35:32.455 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2m) 62s ago 11m 63.4M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T05:35:32.455 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (100s) 62s ago 11m 56.2M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T05:35:32.455 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (67s) 62s ago 10m 15.3M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T05:35:32.455 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6m) 62s ago 10m 80.9M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T05:35:32.763 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:35:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:35:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:35:32.808 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T05:35:32.808 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T05:35:32.809 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T05:35:32.809 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:35:32.809 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T05:35:32.809 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T05:35:32.810 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:35:32.810 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T05:35:32.810 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T05:35:32.810 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:35:32.811 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T05:35:32.811 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T05:35:32.811 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:35:32.811 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T05:35:32.812 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T05:35:32.812 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T05:35:32.812 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T05:35:32.812 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T05:35:33.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:35:32 smithi153 ceph-mon[70044]: pgmap v221: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:33.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:35:32 smithi083 ceph-mon[77242]: pgmap v221: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:33.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:35:32 smithi083 ceph-mon[79930]: pgmap v221: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:33.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:35:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:35:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:35:33.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:35:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:35:33.543Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:35:33.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:35:33 smithi083 ceph-mon[79930]: from='client.44329 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:35:33.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:35:33 smithi083 ceph-mon[79930]: from='client.44335 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:35:33.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:35:33 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4110563114' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:35:33.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:35:33 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:35:33.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:35:33 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:35:33.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:35:33 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:35:33.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:35:33 smithi083 ceph-mon[77242]: from='client.44329 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:35:33.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:35:33 smithi083 ceph-mon[77242]: from='client.44335 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:35:33.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:35:33 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4110563114' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:35:33.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:35:33 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:35:33.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:35:33 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:35:33.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:35:33 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:35:34.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:35:33 smithi153 ceph-mon[70044]: from='client.44329 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:35:34.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:35:33 smithi153 ceph-mon[70044]: from='client.44335 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:35:34.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:35:33 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4110563114' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:35:34.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:35:33 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:35:34.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:35:33 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:35:34.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:35:33 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:35:34.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:35:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:35:34.240Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:35:35.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:35:34 smithi153 ceph-mon[70044]: pgmap v222: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:35.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:35:34 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:35:35.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:35:34 smithi083 ceph-mon[79930]: pgmap v222: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:35.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:35:34 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:35:35.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:35:34 smithi083 ceph-mon[77242]: pgmap v222: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:35.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:35:34 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:35:37.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:35:36 smithi153 ceph-mon[70044]: pgmap v223: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:37.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:35:36 smithi083 ceph-mon[77242]: pgmap v223: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:37.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:35:36 smithi083 ceph-mon[79930]: pgmap v223: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:39.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:35:38 smithi153 ceph-mon[70044]: pgmap v224: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:39.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:35:38 smithi083 ceph-mon[79930]: pgmap v224: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:39.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:35:38 smithi083 ceph-mon[77242]: pgmap v224: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:40.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:35:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:35:40.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:35:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:35:40.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:35:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:35:40.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:35:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:35:40.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:35:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:35:40.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:35:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:35:40.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:35:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:35:40.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:35:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:35:40.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:35:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:35:40.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:35:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:35:40.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:35:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:35:40.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:35:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:35:41.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:35:40 smithi153 ceph-mon[70044]: pgmap v225: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:41.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:35:40 smithi083 ceph-mon[79930]: pgmap v225: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:35:40 smithi083 ceph-mon[77242]: pgmap v225: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:41.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:35:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:35:41] "GET /metrics HTTP/1.1" 200 225290 "" "Prometheus/2.33.4" 2022-09-29T05:35:42.780 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:35:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:35:42.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:35:43.059 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:35:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:35:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:35:43.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:35:42 smithi153 ceph-mon[70044]: pgmap v226: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:43.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:35:42 smithi083 ceph-mon[79930]: pgmap v226: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:43.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:35:42 smithi083 ceph-mon[77242]: pgmap v226: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:43.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:35:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:35:43.545Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:35:44.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:35:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:35:44.241Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:35:45.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:35:44 smithi153 ceph-mon[70044]: pgmap v227: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:45.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:35:44 smithi083 ceph-mon[79930]: pgmap v227: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:45.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:35:44 smithi083 ceph-mon[77242]: pgmap v227: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:46.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:35:45 smithi153 ceph-mon[70044]: pgmap v228: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:46.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:35:45 smithi083 ceph-mon[79930]: pgmap v228: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:46.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:35:45 smithi083 ceph-mon[77242]: pgmap v228: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:49.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:35:48 smithi153 ceph-mon[70044]: pgmap v229: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:49.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:35:48 smithi083 ceph-mon[79930]: pgmap v229: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:49.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:35:48 smithi083 ceph-mon[77242]: pgmap v229: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:50.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:35:50 smithi083 ceph-mon[77242]: pgmap v230: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:50.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:35:50 smithi083 ceph-mon[79930]: pgmap v230: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:51.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:35:50 smithi153 ceph-mon[70044]: pgmap v230: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:51.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:35:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:35:51] "GET /metrics HTTP/1.1" 200 225290 "" "Prometheus/2.33.4" 2022-09-29T05:35:52.704 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:35:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:35:52.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:35:52.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:35:52 smithi083 ceph-mon[79930]: pgmap v231: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:52.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:35:52 smithi083 ceph-mon[77242]: pgmap v231: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:53.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:35:52 smithi153 ceph-mon[70044]: pgmap v231: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:53.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:35:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:35:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:35:53.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:35:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:35:53.545Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:35:54.706 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:35:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:35:54.241Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:35:54.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:35:54 smithi083 ceph-mon[79930]: pgmap v232: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:54.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:35:54 smithi083 ceph-mon[77242]: pgmap v232: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:55.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:35:54 smithi153 ceph-mon[70044]: pgmap v232: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:56.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:35:56 smithi083 ceph-mon[79930]: pgmap v233: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:56.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:35:56 smithi083 ceph-mon[77242]: pgmap v233: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:57.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:35:56 smithi153 ceph-mon[70044]: pgmap v233: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:58.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:35:58 smithi083 ceph-mon[79930]: pgmap v234: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:58.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:35:58 smithi083 ceph-mon[77242]: pgmap v234: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:35:59.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:35:58 smithi153 ceph-mon[70044]: pgmap v234: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:00.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:36:00 smithi083 ceph-mon[79930]: pgmap v235: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:00.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:36:00 smithi083 ceph-mon[77242]: pgmap v235: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:01.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:36:00 smithi153 ceph-mon[70044]: pgmap v235: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:01.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:36:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:36:01] "GET /metrics HTTP/1.1" 200 225293 "" "Prometheus/2.33.4" 2022-09-29T05:36:02.709 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:36:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:36:02.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:36:02.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:36:02 smithi083 ceph-mon[79930]: pgmap v236: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:02.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:36:02 smithi083 ceph-mon[77242]: pgmap v236: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:03.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:36:02 smithi153 ceph-mon[70044]: pgmap v236: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:03.136 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T05:36:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:36:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:36:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:36:03.431 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T05:36:03.431 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9m) 3m ago 10m 26.4M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T05:36:03.431 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10m) 93s ago 10m 59.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T05:36:03.431 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11m) 3m ago 11m 15.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T05:36:03.432 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11m) 93s ago 11m 19.4M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T05:36:03.432 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6m) 93s ago 13m 430M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T05:36:03.432 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7m) 3m ago 15m 490M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T05:36:03.432 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6m) 3m ago 15m 86.0M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T05:36:03.433 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5m) 93s ago 13m 75.3M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T05:36:03.433 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6m) 3m ago 14m 70.7M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T05:36:03.433 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10m) 3m ago 10m 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T05:36:03.433 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10m) 93s ago 10m 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T05:36:03.434 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5m) 3m ago 13m 63.4M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T05:36:03.434 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4m) 3m ago 13m 56.5M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T05:36:03.434 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4m) 3m ago 12m 51.9M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T05:36:03.434 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3m) 3m ago 12m 12.9M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T05:36:03.435 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3m) 93s ago 12m 61.8M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T05:36:03.435 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2m) 93s ago 11m 63.4M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T05:36:03.435 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2m) 93s ago 11m 56.2M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T05:36:03.435 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (98s) 93s ago 11m 15.3M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T05:36:03.435 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6m) 93s ago 10m 80.9M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T05:36:03.798 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T05:36:03.799 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T05:36:03.799 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T05:36:03.799 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:36:03.799 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T05:36:03.800 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T05:36:03.800 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:36:03.800 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T05:36:03.800 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T05:36:03.800 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:36:03.801 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T05:36:03.801 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T05:36:03.801 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:36:03.801 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T05:36:03.802 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T05:36:03.802 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T05:36:03.802 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T05:36:03.802 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T05:36:03.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:36:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:36:03.545Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:36:04.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:36:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:36:04.241Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:36:04.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:36:04 smithi083 ceph-mon[77242]: from='client.44347 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:36:04.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:36:04 smithi083 ceph-mon[77242]: from='client.34419 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:36:04.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:36:04 smithi083 ceph-mon[77242]: pgmap v237: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:04.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:36:04 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3926668812' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:36:04.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:36:04 smithi083 ceph-mon[79930]: from='client.44347 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:36:04.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:36:04 smithi083 ceph-mon[79930]: from='client.34419 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:36:04.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:36:04 smithi083 ceph-mon[79930]: pgmap v237: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:04.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:36:04 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3926668812' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:36:05.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:36:04 smithi153 ceph-mon[70044]: from='client.44347 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:36:05.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:36:04 smithi153 ceph-mon[70044]: from='client.34419 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:36:05.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:36:04 smithi153 ceph-mon[70044]: pgmap v237: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:05.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:36:04 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3926668812' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:36:07.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:36:06 smithi153 ceph-mon[70044]: pgmap v238: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:07.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:36:06 smithi083 ceph-mon[79930]: pgmap v238: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:07.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:36:06 smithi083 ceph-mon[77242]: pgmap v238: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:09.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:36:08 smithi153 ceph-mon[70044]: pgmap v239: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:09.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:36:08 smithi083 ceph-mon[79930]: pgmap v239: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:09.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:36:08 smithi083 ceph-mon[77242]: pgmap v239: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:10.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:36:10 smithi083 ceph-mon[77242]: pgmap v240: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:10.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:36:10 smithi083 ceph-mon[79930]: pgmap v240: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:11.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:36:10 smithi153 ceph-mon[70044]: pgmap v240: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:11.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:36:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:36:11] "GET /metrics HTTP/1.1" 200 225286 "" "Prometheus/2.33.4" 2022-09-29T05:36:12.709 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:36:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:36:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:36:12.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:36:12 smithi083 ceph-mon[79930]: pgmap v241: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:12.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:36:12 smithi083 ceph-mon[77242]: pgmap v241: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:13.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:36:12 smithi153 ceph-mon[70044]: pgmap v241: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:13.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:36:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:36:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:36:13.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:36:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:36:13.546Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:36:14.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:36:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:36:14.242Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:36:14.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:36:14 smithi083 ceph-mon[77242]: pgmap v242: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:14.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:36:14 smithi083 ceph-mon[79930]: pgmap v242: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:15.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:36:14 smithi153 ceph-mon[70044]: pgmap v242: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:17.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:36:16 smithi153 ceph-mon[70044]: pgmap v243: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:17.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:36:16 smithi083 ceph-mon[79930]: pgmap v243: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:17.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:36:16 smithi083 ceph-mon[77242]: pgmap v243: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:19.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:36:18 smithi153 ceph-mon[70044]: pgmap v244: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:19.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:36:18 smithi083 ceph-mon[77242]: pgmap v244: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:19.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:36:18 smithi083 ceph-mon[79930]: pgmap v244: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:20.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:36:20 smithi083 ceph-mon[77242]: pgmap v245: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:20.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:36:20 smithi083 ceph-mon[79930]: pgmap v245: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:21.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:36:20 smithi153 ceph-mon[70044]: pgmap v245: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:21.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:36:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:36:21] "GET /metrics HTTP/1.1" 200 225286 "" "Prometheus/2.33.4" 2022-09-29T05:36:22.710 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:36:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:36:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:36:22.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:36:22 smithi083 ceph-mon[79930]: pgmap v246: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:22.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:36:22 smithi083 ceph-mon[77242]: pgmap v246: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:23.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:36:22 smithi153 ceph-mon[70044]: pgmap v246: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:23.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:36:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:36:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:36:23.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:36:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:36:23.548Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:36:24.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:36:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:36:24.243Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:36:25.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:36:24 smithi153 ceph-mon[70044]: pgmap v247: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:25.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:36:24 smithi083 ceph-mon[77242]: pgmap v247: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:25.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:36:24 smithi083 ceph-mon[79930]: pgmap v247: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:27.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:36:26 smithi153 ceph-mon[70044]: pgmap v248: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:27.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:36:26 smithi083 ceph-mon[77242]: pgmap v248: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:27.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:36:26 smithi083 ceph-mon[79930]: pgmap v248: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:29.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:36:28 smithi153 ceph-mon[70044]: pgmap v249: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:29.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:36:28 smithi083 ceph-mon[77242]: pgmap v249: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:29.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:36:28 smithi083 ceph-mon[79930]: pgmap v249: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:30.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:36:30 smithi083 ceph-mon[79930]: pgmap v250: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:30.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:36:30 smithi083 ceph-mon[77242]: pgmap v250: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:31.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:36:30 smithi153 ceph-mon[70044]: pgmap v250: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:31.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:36:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:36:31] "GET /metrics HTTP/1.1" 200 225295 "" "Prometheus/2.33.4" 2022-09-29T05:36:32.713 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:36:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:36:32.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:36:33.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:36:32 smithi153 ceph-mon[70044]: pgmap v251: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:33.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:36:32 smithi083 ceph-mon[77242]: pgmap v251: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:33.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:36:32 smithi083 ceph-mon[79930]: pgmap v251: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:33.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:36:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:36:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:36:33.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:36:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:36:33.548Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:36:34.126 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T05:36:34.422 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T05:36:34.422 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10m) 4m ago 10m 26.4M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T05:36:34.423 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10m) 2m ago 10m 59.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T05:36:34.423 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11m) 4m ago 11m 15.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T05:36:34.423 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11m) 2m ago 11m 19.4M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T05:36:34.423 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7m) 2m ago 14m 430M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T05:36:34.424 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7m) 4m ago 15m 490M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T05:36:34.424 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7m) 4m ago 15m 86.0M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T05:36:34.424 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6m) 2m ago 14m 75.3M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T05:36:34.424 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6m) 4m ago 14m 70.7M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T05:36:34.425 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10m) 4m ago 10m 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T05:36:34.425 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10m) 2m ago 10m 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T05:36:34.425 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5m) 4m ago 13m 63.4M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T05:36:34.426 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5m) 4m ago 13m 56.5M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T05:36:34.427 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4m) 4m ago 13m 51.9M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T05:36:34.427 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4m) 4m ago 12m 12.9M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T05:36:34.427 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3m) 2m ago 12m 61.8M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T05:36:34.428 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3m) 2m ago 12m 63.4M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T05:36:34.428 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2m) 2m ago 12m 56.2M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T05:36:34.429 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2m) 2m ago 11m 15.3M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T05:36:34.429 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7m) 2m ago 11m 80.9M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T05:36:34.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:36:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:36:34.244Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:36:34.801 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T05:36:34.802 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T05:36:34.802 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T05:36:34.802 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:36:34.803 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T05:36:34.803 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T05:36:34.803 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:36:34.803 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T05:36:34.804 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T05:36:34.804 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:36:34.804 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T05:36:34.804 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T05:36:34.804 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:36:34.805 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T05:36:34.805 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T05:36:34.805 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T05:36:34.805 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T05:36:34.806 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T05:36:35.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:36:34 smithi153 ceph-mon[70044]: pgmap v252: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:35.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:36:34 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:36:35.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:36:34 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:36:35.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:36:34 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:36:35.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:36:34 smithi083 ceph-mon[79930]: pgmap v252: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:35.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:36:34 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:36:35.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:36:34 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:36:35.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:36:34 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:36:35.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:36:34 smithi083 ceph-mon[77242]: pgmap v252: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:35.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:36:34 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:36:35.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:36:34 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:36:35.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:36:34 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:36:36.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:36:35 smithi153 ceph-mon[70044]: from='client.44365 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:36:36.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:36:35 smithi153 ceph-mon[70044]: from='client.44371 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:36:36.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:36:35 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2401813190' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:36:36.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:36:35 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:36:36.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:36:35 smithi083 ceph-mon[79930]: from='client.44365 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:36:36.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:36:35 smithi083 ceph-mon[79930]: from='client.44371 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:36:36.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:36:35 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2401813190' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:36:36.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:36:35 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:36:36.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:36:35 smithi083 ceph-mon[77242]: from='client.44365 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:36:36.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:36:35 smithi083 ceph-mon[77242]: from='client.44371 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:36:36.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:36:35 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2401813190' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:36:36.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:36:35 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:36:37.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:36:36 smithi153 ceph-mon[70044]: pgmap v253: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:37.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:36:36 smithi083 ceph-mon[79930]: pgmap v253: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:37.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:36:36 smithi083 ceph-mon[77242]: pgmap v253: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:39.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:36:38 smithi153 ceph-mon[70044]: pgmap v254: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:39.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:36:38 smithi083 ceph-mon[79930]: pgmap v254: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:39.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:36:38 smithi083 ceph-mon[77242]: pgmap v254: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:40.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:36:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:36:40.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:36:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:36:40.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:36:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:36:40.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:36:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:36:40.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:36:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:36:40.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:36:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:36:40.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:36:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:36:40.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:36:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:36:40.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:36:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:36:40.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:36:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:36:40.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:36:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:36:40.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:36:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:36:41.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:36:40 smithi153 ceph-mon[70044]: pgmap v255: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:41.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:36:40 smithi083 ceph-mon[79930]: pgmap v255: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:36:40 smithi083 ceph-mon[77242]: pgmap v255: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:41.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:36:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:36:41] "GET /metrics HTTP/1.1" 200 225259 "" "Prometheus/2.33.4" 2022-09-29T05:36:42.728 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:36:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:36:42.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:36:43.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:36:42 smithi153 ceph-mon[70044]: pgmap v256: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:43.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:36:42 smithi083 ceph-mon[77242]: pgmap v256: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:43.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:36:42 smithi083 ceph-mon[79930]: pgmap v256: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:43.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:36:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:36:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:36:43.967 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:36:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:36:43.548Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:36:44.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:36:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:36:44.244Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:36:45.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:36:44 smithi153 ceph-mon[70044]: pgmap v257: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:45.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:36:44 smithi083 ceph-mon[77242]: pgmap v257: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:45.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:36:44 smithi083 ceph-mon[79930]: pgmap v257: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:47.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:36:46 smithi153 ceph-mon[70044]: pgmap v258: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:47.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:36:46 smithi083 ceph-mon[79930]: pgmap v258: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:47.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:36:46 smithi083 ceph-mon[77242]: pgmap v258: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:49.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:36:48 smithi153 ceph-mon[70044]: pgmap v259: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:49.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:36:48 smithi083 ceph-mon[79930]: pgmap v259: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:49.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:36:48 smithi083 ceph-mon[77242]: pgmap v259: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:50.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:36:50 smithi083 ceph-mon[77242]: pgmap v260: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:50.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:36:50 smithi083 ceph-mon[79930]: pgmap v260: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:51.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:36:50 smithi153 ceph-mon[70044]: pgmap v260: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:51.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:36:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:36:51] "GET /metrics HTTP/1.1" 200 225259 "" "Prometheus/2.33.4" 2022-09-29T05:36:52.768 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:36:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:36:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:36:53.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:36:52 smithi153 ceph-mon[70044]: pgmap v261: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:53.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:36:52 smithi083 ceph-mon[79930]: pgmap v261: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:53.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:36:52 smithi083 ceph-mon[77242]: pgmap v261: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:53.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:36:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:36:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:36:53.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:36:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:36:53.548Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:36:54.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:36:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:36:54.245Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:36:55.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:36:54 smithi153 ceph-mon[70044]: pgmap v262: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:55.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:36:54 smithi083 ceph-mon[79930]: pgmap v262: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:55.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:36:54 smithi083 ceph-mon[77242]: pgmap v262: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:57.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:36:56 smithi153 ceph-mon[70044]: pgmap v263: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:57.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:36:56 smithi083 ceph-mon[79930]: pgmap v263: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:57.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:36:56 smithi083 ceph-mon[77242]: pgmap v263: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:59.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:36:58 smithi153 ceph-mon[70044]: pgmap v264: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:59.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:36:58 smithi083 ceph-mon[79930]: pgmap v264: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:36:59.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:36:58 smithi083 ceph-mon[77242]: pgmap v264: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:01.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:37:00 smithi153 ceph-mon[70044]: pgmap v265: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:01.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:37:00 smithi083 ceph-mon[79930]: pgmap v265: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:01.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:37:00 smithi083 ceph-mon[77242]: pgmap v265: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:01.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:37:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:37:01] "GET /metrics HTTP/1.1" 200 225263 "" "Prometheus/2.33.4" 2022-09-29T05:37:02.771 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:37:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:37:02.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:37:03.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:37:02 smithi153 ceph-mon[70044]: pgmap v266: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:03.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:37:02 smithi083 ceph-mon[77242]: pgmap v266: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:03.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:37:02 smithi083 ceph-mon[79930]: pgmap v266: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:37:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:37:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:37:03.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:37:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:37:03.549Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:37:04.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:37:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:37:04.246Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:37:05.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:37:04 smithi153 ceph-mon[70044]: pgmap v267: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:05.129 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T05:37:05.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:37:04 smithi083 ceph-mon[79930]: pgmap v267: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:05.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:37:04 smithi083 ceph-mon[77242]: pgmap v267: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:05.416 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T05:37:05.416 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10m) 4m ago 11m 26.4M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T05:37:05.416 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11m) 2m ago 11m 59.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T05:37:05.417 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (12m) 4m ago 12m 15.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T05:37:05.417 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (12m) 2m ago 12m 19.4M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T05:37:05.417 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7m) 2m ago 14m 430M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T05:37:05.417 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8m) 4m ago 16m 490M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T05:37:05.417 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7m) 4m ago 16m 86.0M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T05:37:05.418 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6m) 2m ago 14m 75.3M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T05:37:05.418 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7m) 4m ago 15m 70.7M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T05:37:05.418 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11m) 4m ago 11m 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T05:37:05.418 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11m) 2m ago 11m 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T05:37:05.419 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6m) 4m ago 14m 63.4M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T05:37:05.419 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5m) 4m ago 14m 56.5M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T05:37:05.419 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5m) 4m ago 13m 51.9M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T05:37:05.419 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4m) 4m ago 13m 12.9M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T05:37:05.420 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4m) 2m ago 13m 61.8M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T05:37:05.420 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3m) 2m ago 12m 63.4M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T05:37:05.420 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3m) 2m ago 12m 56.2M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T05:37:05.420 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2m) 2m ago 12m 15.3M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T05:37:05.420 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7m) 2m ago 11m 80.9M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T05:37:05.773 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T05:37:05.774 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T05:37:05.774 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T05:37:05.774 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:37:05.775 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T05:37:05.775 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T05:37:05.775 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:37:05.775 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T05:37:05.775 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T05:37:05.776 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:37:05.776 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T05:37:05.776 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T05:37:05.776 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:37:05.777 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T05:37:05.777 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T05:37:05.777 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T05:37:05.777 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T05:37:05.777 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T05:37:06.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:37:05 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/56846715' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:37:06.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:37:05 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/56846715' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:37:06.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:37:05 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/56846715' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:37:07.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:37:06 smithi153 ceph-mon[70044]: from='client.34449 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:37:07.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:37:06 smithi153 ceph-mon[70044]: from='client.44389 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:37:07.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:37:06 smithi153 ceph-mon[70044]: pgmap v268: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:07.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:37:06 smithi083 ceph-mon[79930]: from='client.34449 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:37:07.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:37:06 smithi083 ceph-mon[79930]: from='client.44389 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:37:07.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:37:06 smithi083 ceph-mon[79930]: pgmap v268: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:07.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:37:06 smithi083 ceph-mon[77242]: from='client.34449 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:37:07.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:37:06 smithi083 ceph-mon[77242]: from='client.44389 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:37:07.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:37:06 smithi083 ceph-mon[77242]: pgmap v268: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:09.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:37:08 smithi153 ceph-mon[70044]: pgmap v269: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:09.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:37:08 smithi083 ceph-mon[79930]: pgmap v269: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:09.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:37:08 smithi083 ceph-mon[77242]: pgmap v269: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:11.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:37:10 smithi153 ceph-mon[70044]: pgmap v270: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:11.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:37:10 smithi083 ceph-mon[79930]: pgmap v270: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:11.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:37:10 smithi083 ceph-mon[77242]: pgmap v270: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:11.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:37:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:37:11] "GET /metrics HTTP/1.1" 200 225249 "" "Prometheus/2.33.4" 2022-09-29T05:37:12.774 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:37:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:37:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:37:13.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:37:12 smithi153 ceph-mon[70044]: pgmap v271: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:13.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:37:12 smithi083 ceph-mon[79930]: pgmap v271: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:13.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:37:12 smithi083 ceph-mon[77242]: pgmap v271: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:13.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:37:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:37:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:37:13.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:37:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:37:13.550Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:37:14.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:37:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:37:14.247Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:37:15.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:37:14 smithi153 ceph-mon[70044]: pgmap v272: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:15.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:37:14 smithi083 ceph-mon[79930]: pgmap v272: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:15.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:37:14 smithi083 ceph-mon[77242]: pgmap v272: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:17.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:37:16 smithi153 ceph-mon[70044]: pgmap v273: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:17.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:37:16 smithi083 ceph-mon[77242]: pgmap v273: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:17.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:37:16 smithi083 ceph-mon[79930]: pgmap v273: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:19.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:37:18 smithi153 ceph-mon[70044]: pgmap v274: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:19.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:37:18 smithi083 ceph-mon[79930]: pgmap v274: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:19.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:37:18 smithi083 ceph-mon[77242]: pgmap v274: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:21.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:37:20 smithi153 ceph-mon[70044]: pgmap v275: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:21.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:37:20 smithi083 ceph-mon[77242]: pgmap v275: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:21.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:37:20 smithi083 ceph-mon[79930]: pgmap v275: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:21.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:37:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:37:21] "GET /metrics HTTP/1.1" 200 225249 "" "Prometheus/2.33.4" 2022-09-29T05:37:22.776 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:37:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:37:22.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:37:23.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:37:22 smithi153 ceph-mon[70044]: pgmap v276: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:23.059 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:37:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:37:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:37:23.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:37:22 smithi083 ceph-mon[77242]: pgmap v276: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:23.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:37:22 smithi083 ceph-mon[79930]: pgmap v276: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:23.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:37:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:37:23.550Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:37:24.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:37:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:37:24.248Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:37:25.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:37:24 smithi153 ceph-mon[70044]: pgmap v277: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:25.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:37:24 smithi083 ceph-mon[77242]: pgmap v277: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:25.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:37:24 smithi083 ceph-mon[79930]: pgmap v277: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:27.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:37:26 smithi153 ceph-mon[70044]: pgmap v278: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:27.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:37:26 smithi083 ceph-mon[77242]: pgmap v278: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:27.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:37:26 smithi083 ceph-mon[79930]: pgmap v278: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:29.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:37:28 smithi153 ceph-mon[70044]: pgmap v279: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:29.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:37:28 smithi083 ceph-mon[77242]: pgmap v279: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:29.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:37:28 smithi083 ceph-mon[79930]: pgmap v279: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:31.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:37:30 smithi153 ceph-mon[70044]: pgmap v280: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:31.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:37:30 smithi083 ceph-mon[79930]: pgmap v280: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:31.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:37:30 smithi083 ceph-mon[77242]: pgmap v280: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:31.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:37:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:37:31] "GET /metrics HTTP/1.1" 200 225264 "" "Prometheus/2.33.4" 2022-09-29T05:37:32.780 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:37:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:37:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:37:33.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:37:32 smithi153 ceph-mon[70044]: pgmap v281: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:33.059 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:37:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:37:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:37:33.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:37:32 smithi083 ceph-mon[77242]: pgmap v281: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:33.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:37:32 smithi083 ceph-mon[79930]: pgmap v281: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:33.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:37:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:37:33.550Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:37:34.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:37:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:37:34.248Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:37:35.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:37:34 smithi153 ceph-mon[70044]: pgmap v282: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:35.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:37:34 smithi083 ceph-mon[79930]: pgmap v282: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:35.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:37:34 smithi083 ceph-mon[77242]: pgmap v282: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:36.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:37:35 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:37:36.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:37:35 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:37:36.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:37:35 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:37:36.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:37:35 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:37:36.090 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T05:37:36.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:37:35 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:37:36.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:37:35 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:37:36.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:37:35 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:37:36.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:37:35 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:37:36.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:37:35 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:37:36.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:37:35 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:37:36.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:37:35 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:37:36.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:37:35 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:37:36.384 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T05:37:36.385 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11m) 5m ago 11m 26.4M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T05:37:36.385 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11m) 3m ago 11m 59.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T05:37:36.385 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (12m) 5m ago 12m 15.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T05:37:36.385 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (12m) 3m ago 12m 19.4M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T05:37:36.386 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8m) 3m ago 15m 430M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T05:37:36.386 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8m) 5m ago 16m 490M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T05:37:36.387 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8m) 5m ago 16m 86.0M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T05:37:36.387 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7m) 3m ago 15m 75.3M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T05:37:36.387 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7m) 5m ago 15m 70.7M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T05:37:36.387 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (12m) 5m ago 12m 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T05:37:36.388 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11m) 3m ago 11m 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T05:37:36.388 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6m) 5m ago 14m 63.4M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T05:37:36.388 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6m) 5m ago 14m 56.5M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T05:37:36.388 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5m) 5m ago 14m 51.9M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T05:37:36.389 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5m) 5m ago 14m 12.9M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T05:37:36.389 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4m) 3m ago 13m 61.8M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T05:37:36.389 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4m) 3m ago 13m 63.4M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T05:37:36.389 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3m) 3m ago 13m 56.2M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T05:37:36.390 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3m) 3m ago 12m 15.3M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T05:37:36.390 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8m) 3m ago 12m 80.9M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T05:37:36.739 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T05:37:36.739 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T05:37:36.739 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T05:37:36.739 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:37:36.740 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T05:37:36.740 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T05:37:36.740 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:37:36.740 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T05:37:36.741 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T05:37:36.741 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:37:36.741 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T05:37:36.741 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T05:37:36.741 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:37:36.742 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T05:37:36.742 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T05:37:36.742 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T05:37:36.742 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T05:37:36.743 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T05:37:37.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:37:36 smithi153 ceph-mon[70044]: pgmap v283: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:37.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:37:36 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1244566145' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:37:37.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:37:36 smithi083 ceph-mon[79930]: pgmap v283: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:37.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:37:36 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1244566145' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:37:37.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:37:36 smithi083 ceph-mon[77242]: pgmap v283: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:37.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:37:36 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1244566145' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:37:38.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:37:37 smithi153 ceph-mon[70044]: from='client.34467 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:37:38.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:37:37 smithi153 ceph-mon[70044]: from='client.34473 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:37:38.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:37:37 smithi083 ceph-mon[79930]: from='client.34467 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:37:38.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:37:37 smithi083 ceph-mon[79930]: from='client.34473 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:37:38.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:37:37 smithi083 ceph-mon[77242]: from='client.34467 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:37:38.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:37:37 smithi083 ceph-mon[77242]: from='client.34473 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:37:39.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:37:38 smithi153 ceph-mon[70044]: pgmap v284: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:39.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:37:38 smithi083 ceph-mon[79930]: pgmap v284: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:39.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:37:38 smithi083 ceph-mon[77242]: pgmap v284: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:40.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:37:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:37:40.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:37:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:37:40.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:37:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:37:40.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:37:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:37:40.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:37:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:37:40.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:37:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:37:40.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:37:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:37:40.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:37:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:37:40.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:37:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:37:40.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:37:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:37:40.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:37:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:37:40.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:37:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:37:41.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:37:40 smithi153 ceph-mon[70044]: pgmap v285: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:41.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:37:40 smithi083 ceph-mon[79930]: pgmap v285: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:37:40 smithi083 ceph-mon[77242]: pgmap v285: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:41.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:37:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:37:41] "GET /metrics HTTP/1.1" 200 225261 "" "Prometheus/2.33.4" 2022-09-29T05:37:42.799 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:37:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:37:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:37:43.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:37:42 smithi153 ceph-mon[70044]: pgmap v286: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:43.059 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:37:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:37:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:37:43.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:37:42 smithi083 ceph-mon[79930]: pgmap v286: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:43.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:37:42 smithi083 ceph-mon[77242]: pgmap v286: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:43.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:37:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:37:43.552Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:37:44.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:37:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:37:44.249Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:37:45.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:37:44 smithi153 ceph-mon[70044]: pgmap v287: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:45.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:37:44 smithi083 ceph-mon[79930]: pgmap v287: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:45.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:37:44 smithi083 ceph-mon[77242]: pgmap v287: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:46.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:37:45 smithi083 ceph-mon[79930]: pgmap v288: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:46.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:37:45 smithi083 ceph-mon[77242]: pgmap v288: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:46.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:37:45 smithi153 ceph-mon[70044]: pgmap v288: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:49.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:37:48 smithi153 ceph-mon[70044]: pgmap v289: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:49.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:37:48 smithi083 ceph-mon[79930]: pgmap v289: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:49.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:37:48 smithi083 ceph-mon[77242]: pgmap v289: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:51.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:37:50 smithi153 ceph-mon[70044]: pgmap v290: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:51.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:37:50 smithi083 ceph-mon[79930]: pgmap v290: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:51.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:37:50 smithi083 ceph-mon[77242]: pgmap v290: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:51.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:37:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:37:51] "GET /metrics HTTP/1.1" 200 225261 "" "Prometheus/2.33.4" 2022-09-29T05:37:52.786 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:37:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:37:52.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:37:53.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:37:52 smithi153 ceph-mon[70044]: pgmap v291: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:53.059 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:37:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:37:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:37:53.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:37:52 smithi083 ceph-mon[77242]: pgmap v291: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:53.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:37:52 smithi083 ceph-mon[79930]: pgmap v291: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:53.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:37:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:37:53.552Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:37:54.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:37:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:37:54.249Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:37:55.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:37:54 smithi153 ceph-mon[70044]: pgmap v292: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:55.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:37:54 smithi083 ceph-mon[79930]: pgmap v292: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:55.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:37:54 smithi083 ceph-mon[77242]: pgmap v292: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:57.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:37:56 smithi153 ceph-mon[70044]: pgmap v293: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:57.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:37:56 smithi083 ceph-mon[77242]: pgmap v293: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:57.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:37:56 smithi083 ceph-mon[79930]: pgmap v293: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:59.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:37:58 smithi153 ceph-mon[70044]: pgmap v294: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:59.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:37:58 smithi083 ceph-mon[79930]: pgmap v294: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:37:59.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:37:58 smithi083 ceph-mon[77242]: pgmap v294: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:01.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:38:00 smithi153 ceph-mon[70044]: pgmap v295: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:01.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:38:00 smithi083 ceph-mon[79930]: pgmap v295: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:01.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:38:00 smithi083 ceph-mon[77242]: pgmap v295: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:01.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:38:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:38:01] "GET /metrics HTTP/1.1" 200 225263 "" "Prometheus/2.33.4" 2022-09-29T05:38:02.789 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:38:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:38:02.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:38:03.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:38:02 smithi153 ceph-mon[70044]: pgmap v296: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:03.059 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:38:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:38:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:38:03.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:38:02 smithi083 ceph-mon[77242]: pgmap v296: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:03.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:38:02 smithi083 ceph-mon[79930]: pgmap v296: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:03.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:38:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:38:03.552Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:38:04.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:38:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:38:04.250Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:38:05.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:38:04 smithi153 ceph-mon[70044]: pgmap v297: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:05.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:38:04 smithi083 ceph-mon[77242]: pgmap v297: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:05.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:38:04 smithi083 ceph-mon[79930]: pgmap v297: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:07.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:38:06 smithi153 ceph-mon[70044]: pgmap v298: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:07.068 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T05:38:07.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:38:06 smithi083 ceph-mon[79930]: pgmap v298: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:07.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:38:06 smithi083 ceph-mon[77242]: pgmap v298: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:07.361 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T05:38:07.362 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11m) 5m ago 12m 26.4M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T05:38:07.362 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (12m) 3m ago 12m 59.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T05:38:07.362 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (13m) 5m ago 13m 15.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T05:38:07.362 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (13m) 3m ago 13m 19.4M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T05:38:07.363 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8m) 3m ago 15m 430M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T05:38:07.363 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9m) 5m ago 17m 490M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T05:38:07.363 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8m) 5m ago 17m 86.0M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T05:38:07.363 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8m) 3m ago 15m 75.3M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T05:38:07.363 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8m) 5m ago 16m 70.7M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T05:38:07.364 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (12m) 5m ago 12m 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T05:38:07.364 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (12m) 3m ago 12m 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T05:38:07.364 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7m) 5m ago 15m 63.4M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T05:38:07.364 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6m) 5m ago 15m 56.5M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T05:38:07.364 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6m) 5m ago 14m 51.9M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T05:38:07.365 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5m) 5m ago 14m 12.9M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T05:38:07.365 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5m) 3m ago 14m 61.8M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T05:38:07.365 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4m) 3m ago 13m 63.4M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T05:38:07.365 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4m) 3m ago 13m 56.2M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T05:38:07.365 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3m) 3m ago 13m 15.3M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T05:38:07.366 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8m) 3m ago 12m 80.9M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T05:38:07.715 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T05:38:07.715 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T05:38:07.716 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T05:38:07.716 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:38:07.716 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T05:38:07.716 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T05:38:07.717 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:38:07.717 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T05:38:07.717 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T05:38:07.717 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:38:07.718 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T05:38:07.718 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T05:38:07.718 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:38:07.718 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T05:38:07.718 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T05:38:07.719 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T05:38:07.719 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T05:38:07.719 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T05:38:08.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:38:07 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4235755022' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:38:08.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:38:07 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4235755022' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:38:08.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:38:07 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4235755022' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:38:09.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:38:08 smithi153 ceph-mon[70044]: from='client.44419 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:38:09.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:38:08 smithi153 ceph-mon[70044]: from='client.34491 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:38:09.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:38:08 smithi153 ceph-mon[70044]: pgmap v299: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:09.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:38:08 smithi083 ceph-mon[79930]: from='client.44419 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:38:09.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:38:08 smithi083 ceph-mon[79930]: from='client.34491 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:38:09.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:38:08 smithi083 ceph-mon[79930]: pgmap v299: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:09.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:38:08 smithi083 ceph-mon[77242]: from='client.44419 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:38:09.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:38:08 smithi083 ceph-mon[77242]: from='client.34491 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:38:09.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:38:08 smithi083 ceph-mon[77242]: pgmap v299: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:11.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:38:10 smithi153 ceph-mon[70044]: pgmap v300: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:11.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:38:10 smithi083 ceph-mon[79930]: pgmap v300: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:11.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:38:10 smithi083 ceph-mon[77242]: pgmap v300: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:11.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:38:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:38:11] "GET /metrics HTTP/1.1" 200 225267 "" "Prometheus/2.33.4" 2022-09-29T05:38:12.792 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:38:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:38:12.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:38:13.059 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:38:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:38:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:38:13.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:38:12 smithi153 ceph-mon[70044]: pgmap v301: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:13.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:38:12 smithi083 ceph-mon[79930]: pgmap v301: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:13.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:38:12 smithi083 ceph-mon[77242]: pgmap v301: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:13.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:38:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:38:13.553Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:38:14.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:38:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:38:14.250Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:38:15.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:38:14 smithi153 ceph-mon[70044]: pgmap v302: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:15.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:38:14 smithi083 ceph-mon[79930]: pgmap v302: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:15.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:38:14 smithi083 ceph-mon[77242]: pgmap v302: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:17.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:38:16 smithi153 ceph-mon[70044]: pgmap v303: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:17.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:38:16 smithi083 ceph-mon[79930]: pgmap v303: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:17.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:38:16 smithi083 ceph-mon[77242]: pgmap v303: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:19.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:38:18 smithi153 ceph-mon[70044]: pgmap v304: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:19.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:38:18 smithi083 ceph-mon[79930]: pgmap v304: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:19.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:38:18 smithi083 ceph-mon[77242]: pgmap v304: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:21.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:38:20 smithi153 ceph-mon[70044]: pgmap v305: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:21.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:38:20 smithi083 ceph-mon[79930]: pgmap v305: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:21.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:38:20 smithi083 ceph-mon[77242]: pgmap v305: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:21.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:38:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:38:21] "GET /metrics HTTP/1.1" 200 225267 "" "Prometheus/2.33.4" 2022-09-29T05:38:22.795 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:38:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:38:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:38:23.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:38:22 smithi153 ceph-mon[70044]: pgmap v306: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:23.059 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:38:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:38:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:38:23.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:38:22 smithi083 ceph-mon[79930]: pgmap v306: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:23.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:38:22 smithi083 ceph-mon[77242]: pgmap v306: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:23.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:38:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:38:23.553Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:38:24.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:38:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:38:24.250Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:38:25.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:38:24 smithi153 ceph-mon[70044]: pgmap v307: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:25.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:38:24 smithi083 ceph-mon[79930]: pgmap v307: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:25.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:38:24 smithi083 ceph-mon[77242]: pgmap v307: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:27.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:38:26 smithi153 ceph-mon[70044]: pgmap v308: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:27.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:38:26 smithi083 ceph-mon[79930]: pgmap v308: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:27.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:38:26 smithi083 ceph-mon[77242]: pgmap v308: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:29.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:38:28 smithi153 ceph-mon[70044]: pgmap v309: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:29.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:38:28 smithi083 ceph-mon[77242]: pgmap v309: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:29.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:38:28 smithi083 ceph-mon[79930]: pgmap v309: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:31.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:38:30 smithi153 ceph-mon[70044]: pgmap v310: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:31.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:38:30 smithi083 ceph-mon[77242]: pgmap v310: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:31.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:38:30 smithi083 ceph-mon[79930]: pgmap v310: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:31.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:38:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:38:31] "GET /metrics HTTP/1.1" 200 225258 "" "Prometheus/2.33.4" 2022-09-29T05:38:32.799 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:38:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:38:32.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:38:33.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:38:32 smithi153 ceph-mon[70044]: pgmap v311: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:33.059 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:38:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:38:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:38:33.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:38:32 smithi083 ceph-mon[77242]: pgmap v311: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:33.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:38:32 smithi083 ceph-mon[79930]: pgmap v311: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:33.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:38:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:38:33.554Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:38:34.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:38:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:38:34.251Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:38:35.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:38:34 smithi153 ceph-mon[70044]: pgmap v312: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:35.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:38:34 smithi083 ceph-mon[77242]: pgmap v312: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:35.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:38:34 smithi083 ceph-mon[79930]: pgmap v312: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:36.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:38:35 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:38:36.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:38:35 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:38:36.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:38:35 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:38:36.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:38:35 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:38:36.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:38:35 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:38:36.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:38:35 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:38:36.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:38:35 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:38:36.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:38:35 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:38:36.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:38:35 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:38:37.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:38:36 smithi083 ceph-mon[79930]: pgmap v313: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:37.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:38:36 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:38:37.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:38:36 smithi083 ceph-mon[77242]: pgmap v313: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:37.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:38:36 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:38:37.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:38:36 smithi153 ceph-mon[70044]: pgmap v313: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:37.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:38:36 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:38:38.050 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T05:38:38.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:38:37 smithi083 ceph-mon[79930]: pgmap v314: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:38.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:38:37 smithi083 ceph-mon[77242]: pgmap v314: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:38.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:38:37 smithi153 ceph-mon[70044]: pgmap v314: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:38.339 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T05:38:38.340 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (12m) 6m ago 12m 26.4M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T05:38:38.340 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (12m) 4m ago 12m 59.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T05:38:38.340 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (13m) 6m ago 13m 15.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T05:38:38.340 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (13m) 4m ago 13m 19.4M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T05:38:38.341 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9m) 4m ago 16m 430M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T05:38:38.341 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9m) 6m ago 17m 490M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T05:38:38.341 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9m) 6m ago 17m 86.0M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T05:38:38.341 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8m) 4m ago 16m 75.3M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T05:38:38.341 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8m) 6m ago 16m 70.7M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T05:38:38.342 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (13m) 6m ago 13m 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T05:38:38.342 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (13m) 4m ago 12m 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T05:38:38.342 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7m) 6m ago 15m 63.4M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T05:38:38.342 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7m) 6m ago 15m 56.5M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T05:38:38.343 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6m) 6m ago 15m 51.9M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T05:38:38.343 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6m) 6m ago 15m 12.9M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T05:38:38.343 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5m) 4m ago 14m 61.8M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T05:38:38.343 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5m) 4m ago 14m 63.4M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T05:38:38.343 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4m) 4m ago 14m 56.2M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T05:38:38.344 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4m) 4m ago 13m 15.3M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T05:38:38.344 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9m) 4m ago 13m 80.9M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T05:38:38.698 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T05:38:38.699 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T05:38:38.699 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T05:38:38.699 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:38:38.699 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T05:38:38.700 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T05:38:38.700 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:38:38.700 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T05:38:38.700 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T05:38:38.700 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:38:38.701 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T05:38:38.701 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T05:38:38.701 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:38:38.701 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T05:38:38.702 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T05:38:38.702 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T05:38:38.702 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T05:38:38.702 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T05:38:39.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:38:38 smithi083 ceph-mon[79930]: from='client.34503 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:38:39.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:38:38 smithi083 ceph-mon[79930]: from='client.44443 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:38:39.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:38:38 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4144842483' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:38:39.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:38:38 smithi083 ceph-mon[77242]: from='client.34503 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:38:39.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:38:38 smithi083 ceph-mon[77242]: from='client.44443 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:38:39.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:38:38 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4144842483' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:38:39.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:38:38 smithi153 ceph-mon[70044]: from='client.34503 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:38:39.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:38:38 smithi153 ceph-mon[70044]: from='client.44443 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:38:39.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:38:38 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4144842483' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:38:40.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:38:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:38:40.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:38:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:38:40.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:38:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:38:40.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:38:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:38:40.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:38:39 smithi083 ceph-mon[79930]: pgmap v315: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:40.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:38:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:38:40.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:38:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:38:40.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:38:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:38:40.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:38:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:38:40.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:38:39 smithi083 ceph-mon[77242]: pgmap v315: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:38:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:38:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:38:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:38:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:38:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:38:40.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:38:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:38:40.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:38:39 smithi153 ceph-mon[70044]: pgmap v315: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:41.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:38:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:38:41] "GET /metrics HTTP/1.1" 200 225260 "" "Prometheus/2.33.4" 2022-09-29T05:38:42.801 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:38:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:38:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:38:43.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:38:42 smithi153 ceph-mon[70044]: pgmap v316: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:43.059 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:38:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:38:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:38:43.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:38:42 smithi083 ceph-mon[79930]: pgmap v316: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:43.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:38:42 smithi083 ceph-mon[77242]: pgmap v316: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:43.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:38:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:38:43.554Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:38:44.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:38:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:38:44.251Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:38:45.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:38:44 smithi153 ceph-mon[70044]: pgmap v317: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:45.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:38:44 smithi083 ceph-mon[77242]: pgmap v317: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:45.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:38:44 smithi083 ceph-mon[79930]: pgmap v317: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:47.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:38:46 smithi153 ceph-mon[70044]: pgmap v318: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:47.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:38:46 smithi083 ceph-mon[79930]: pgmap v318: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:47.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:38:46 smithi083 ceph-mon[77242]: pgmap v318: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:49.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:38:48 smithi083 ceph-mon[77242]: pgmap v319: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:49.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:38:48 smithi083 ceph-mon[79930]: pgmap v319: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:49.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:38:48 smithi153 ceph-mon[70044]: pgmap v319: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:51.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:38:50 smithi153 ceph-mon[70044]: pgmap v320: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:51.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:38:50 smithi083 ceph-mon[79930]: pgmap v320: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:51.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:38:50 smithi083 ceph-mon[77242]: pgmap v320: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:51.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:38:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:38:51] "GET /metrics HTTP/1.1" 200 225260 "" "Prometheus/2.33.4" 2022-09-29T05:38:52.805 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:38:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:38:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:38:53.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:38:52 smithi153 ceph-mon[70044]: pgmap v321: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:53.059 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:38:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:38:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:38:53.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:38:52 smithi083 ceph-mon[79930]: pgmap v321: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:53.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:38:52 smithi083 ceph-mon[77242]: pgmap v321: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:53.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:38:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:38:53.555Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:38:54.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:38:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:38:54.252Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:38:55.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:38:54 smithi083 ceph-mon[79930]: pgmap v322: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:55.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:38:54 smithi083 ceph-mon[77242]: pgmap v322: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:55.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:38:54 smithi153 ceph-mon[70044]: pgmap v322: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:57.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:38:56 smithi083 ceph-mon[79930]: pgmap v323: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:57.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:38:56 smithi083 ceph-mon[77242]: pgmap v323: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:57.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:38:56 smithi153 ceph-mon[70044]: pgmap v323: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:59.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:38:58 smithi083 ceph-mon[79930]: pgmap v324: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:59.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:38:58 smithi083 ceph-mon[77242]: pgmap v324: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:38:59.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:38:58 smithi153 ceph-mon[70044]: pgmap v324: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:01.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:39:00 smithi153 ceph-mon[70044]: pgmap v325: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:01.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:39:00 smithi083 ceph-mon[77242]: pgmap v325: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:01.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:39:00 smithi083 ceph-mon[79930]: pgmap v325: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:01.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:39:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:39:01] "GET /metrics HTTP/1.1" 200 225263 "" "Prometheus/2.33.4" 2022-09-29T05:39:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:39:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:39:02.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:39:03.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:39:02 smithi153 ceph-mon[70044]: pgmap v326: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:03.059 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:39:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:39:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:39:03.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:39:02 smithi083 ceph-mon[79930]: pgmap v326: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:03.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:39:02 smithi083 ceph-mon[77242]: pgmap v326: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:03.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:39:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:39:03.556Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:39:04.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:39:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:39:04.253Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:39:05.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:39:04 smithi083 ceph-mon[79930]: pgmap v327: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:05.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:39:04 smithi083 ceph-mon[77242]: pgmap v327: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:05.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:39:04 smithi153 ceph-mon[70044]: pgmap v327: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:07.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:39:06 smithi083 ceph-mon[79930]: pgmap v328: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:07.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:39:06 smithi083 ceph-mon[77242]: pgmap v328: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:07.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:39:06 smithi153 ceph-mon[70044]: pgmap v328: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:09.030 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T05:39:09.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:39:08 smithi083 ceph-mon[79930]: pgmap v329: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:09.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:39:08 smithi083 ceph-mon[77242]: pgmap v329: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:09.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:39:08 smithi153 ceph-mon[70044]: pgmap v329: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:09.324 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T05:39:09.324 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (12m) 6m ago 13m 26.4M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T05:39:09.324 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (13m) 4m ago 13m 59.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T05:39:09.325 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (14m) 6m ago 14m 15.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T05:39:09.325 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (14m) 4m ago 14m 19.4M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T05:39:09.325 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9m) 4m ago 16m 430M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T05:39:09.325 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10m) 6m ago 18m 490M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T05:39:09.326 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9m) 6m ago 18m 86.0M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T05:39:09.326 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9m) 4m ago 16m 75.3M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T05:39:09.326 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9m) 6m ago 17m 70.7M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T05:39:09.326 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (13m) 6m ago 13m 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T05:39:09.327 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (13m) 4m ago 13m 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T05:39:09.327 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8m) 6m ago 16m 63.4M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T05:39:09.327 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7m) 6m ago 16m 56.5M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T05:39:09.327 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7m) 6m ago 15m 51.9M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T05:39:09.328 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6m) 6m ago 15m 12.9M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T05:39:09.328 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6m) 4m ago 15m 61.8M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T05:39:09.328 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5m) 4m ago 15m 63.4M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T05:39:09.328 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5m) 4m ago 14m 56.2M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T05:39:09.328 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4m) 4m ago 14m 15.3M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T05:39:09.329 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9m) 4m ago 13m 80.9M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T05:39:09.685 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T05:39:09.685 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T05:39:09.686 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T05:39:09.686 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:39:09.686 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T05:39:09.686 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T05:39:09.687 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:39:09.687 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T05:39:09.687 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T05:39:09.687 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:39:09.688 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T05:39:09.688 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T05:39:09.688 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:39:09.688 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T05:39:09.689 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T05:39:09.689 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T05:39:09.689 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T05:39:09.689 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T05:39:10.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:39:09 smithi083 ceph-mon[77242]: from='client.34521 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:39:10.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:39:09 smithi083 ceph-mon[77242]: from='client.44461 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:39:10.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:39:09 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3722813429' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:39:10.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:39:09 smithi083 ceph-mon[77242]: pgmap v330: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:10.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:39:09 smithi083 ceph-mon[79930]: from='client.34521 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:39:10.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:39:09 smithi083 ceph-mon[79930]: from='client.44461 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:39:10.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:39:09 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3722813429' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:39:10.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:39:09 smithi083 ceph-mon[79930]: pgmap v330: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:10.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:39:09 smithi153 ceph-mon[70044]: from='client.34521 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:39:10.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:39:09 smithi153 ceph-mon[70044]: from='client.44461 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:39:10.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:39:09 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3722813429' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:39:10.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:39:09 smithi153 ceph-mon[70044]: pgmap v330: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:11.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:39:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:39:11] "GET /metrics HTTP/1.1" 200 225275 "" "Prometheus/2.33.4" 2022-09-29T05:39:12.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:39:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:39:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:39:13.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:39:12 smithi083 ceph-mon[79930]: pgmap v331: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:13.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:39:12 smithi083 ceph-mon[77242]: pgmap v331: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:13.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:39:12 smithi153 ceph-mon[70044]: pgmap v331: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:13.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:39:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:39:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:39:13.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:39:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:39:13.557Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:39:14.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:39:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:39:14.254Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:39:15.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:39:14 smithi083 ceph-mon[79930]: pgmap v332: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:15.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:39:14 smithi083 ceph-mon[77242]: pgmap v332: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:15.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:39:14 smithi153 ceph-mon[70044]: pgmap v332: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:17.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:39:16 smithi083 ceph-mon[77242]: pgmap v333: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:17.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:39:16 smithi083 ceph-mon[79930]: pgmap v333: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:17.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:39:16 smithi153 ceph-mon[70044]: pgmap v333: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:18.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:39:17 smithi083 ceph-mon[79930]: pgmap v334: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:18.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:39:17 smithi083 ceph-mon[77242]: pgmap v334: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:18.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:39:17 smithi153 ceph-mon[70044]: pgmap v334: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:21.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:39:20 smithi153 ceph-mon[70044]: pgmap v335: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:21.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:39:20 smithi083 ceph-mon[79930]: pgmap v335: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:21.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:39:20 smithi083 ceph-mon[77242]: pgmap v335: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:21.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:39:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:39:21] "GET /metrics HTTP/1.1" 200 225275 "" "Prometheus/2.33.4" 2022-09-29T05:39:22.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:39:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:39:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:39:23.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:39:22 smithi083 ceph-mon[79930]: pgmap v336: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:23.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:39:22 smithi083 ceph-mon[77242]: pgmap v336: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:23.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:39:22 smithi153 ceph-mon[70044]: pgmap v336: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:23.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:39:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:39:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:39:23.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:39:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:39:23.558Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:39:24.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:39:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:39:24.254Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:39:25.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:39:24 smithi083 ceph-mon[77242]: pgmap v337: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:25.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:39:24 smithi083 ceph-mon[79930]: pgmap v337: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:25.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:39:24 smithi153 ceph-mon[70044]: pgmap v337: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:26.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:39:25 smithi083 ceph-mon[79930]: pgmap v338: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:26.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:39:25 smithi083 ceph-mon[77242]: pgmap v338: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:26.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:39:25 smithi153 ceph-mon[70044]: pgmap v338: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:29.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:39:28 smithi083 ceph-mon[79930]: pgmap v339: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:29.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:39:28 smithi083 ceph-mon[77242]: pgmap v339: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:29.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:39:28 smithi153 ceph-mon[70044]: pgmap v339: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:31.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:39:30 smithi153 ceph-mon[70044]: pgmap v340: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:31.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:39:30 smithi083 ceph-mon[79930]: pgmap v340: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:31.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:39:30 smithi083 ceph-mon[77242]: pgmap v340: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:31.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:39:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:39:31] "GET /metrics HTTP/1.1" 200 225269 "" "Prometheus/2.33.4" 2022-09-29T05:39:32.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:39:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:39:32.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:39:33.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:39:32 smithi083 ceph-mon[79930]: pgmap v341: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:33.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:39:32 smithi083 ceph-mon[77242]: pgmap v341: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:33.312 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:39:32 smithi153 ceph-mon[70044]: pgmap v341: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:33.313 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:39:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:39:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:39:33.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:39:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:39:33.558Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:39:34.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:39:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:39:34.255Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:39:35.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:39:34 smithi083 ceph-mon[79930]: pgmap v342: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:35.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:39:34 smithi083 ceph-mon[77242]: pgmap v342: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:35.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:39:34 smithi153 ceph-mon[70044]: pgmap v342: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:36.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:39:35 smithi083 ceph-mon[79930]: pgmap v343: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:36.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:39:35 smithi083 ceph-mon[77242]: pgmap v343: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:36.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:39:35 smithi153 ceph-mon[70044]: pgmap v343: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:37.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:39:36 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:39:37.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:39:36 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:39:37.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:39:36 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:39:37.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:39:36 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:39:37.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:39:36 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:39:37.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:39:36 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:39:37.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:39:36 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:39:37.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:39:36 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:39:37.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:39:36 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:39:38.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:39:37 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:39:38.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:39:37 smithi083 ceph-mon[77242]: pgmap v344: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:38.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:39:37 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:39:38.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:39:37 smithi083 ceph-mon[79930]: pgmap v344: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:38.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:39:37 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:39:38.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:39:37 smithi153 ceph-mon[70044]: pgmap v344: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:39.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:39:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:39:39.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:39:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:39:39.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:39:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:39:39.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:39:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:39:39.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:39:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:39:39.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:39:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:39:39.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:39:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:39:39.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:39:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:39:39.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:39:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:39:39.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:39:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:39:39.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:39:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:39:39.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:39:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:39:40.012 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T05:39:40.299 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T05:39:40.299 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (13m) 7m ago 13m 26.4M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T05:39:40.300 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (13m) 5m ago 13m 59.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T05:39:40.300 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (14m) 7m ago 14m 15.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T05:39:40.300 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (14m) 5m ago 14m 19.4M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T05:39:40.300 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10m) 5m ago 17m 430M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T05:39:40.301 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10m) 7m ago 18m 490M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T05:39:40.301 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10m) 7m ago 18m 86.0M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T05:39:40.301 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9m) 5m ago 17m 75.3M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T05:39:40.301 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9m) 7m ago 17m 70.7M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T05:39:40.302 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (14m) 7m ago 14m 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T05:39:40.302 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (14m) 5m ago 14m 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T05:39:40.303 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8m) 7m ago 17m 63.4M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T05:39:40.303 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8m) 7m ago 16m 56.5M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T05:39:40.303 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8m) 7m ago 16m 51.9M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T05:39:40.303 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7m) 7m ago 16m 12.9M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T05:39:40.304 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6m) 5m ago 15m 61.8M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T05:39:40.304 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6m) 5m ago 15m 63.4M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T05:39:40.305 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5m) 5m ago 15m 56.2M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T05:39:40.305 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5m) 5m ago 14m 15.3M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T05:39:40.305 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10m) 5m ago 14m 80.9M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T05:39:40.654 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T05:39:40.655 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T05:39:40.655 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T05:39:40.655 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:39:40.656 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T05:39:40.656 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T05:39:40.656 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:39:40.656 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T05:39:40.657 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T05:39:40.657 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:39:40.657 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T05:39:40.657 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T05:39:40.658 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:39:40.658 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T05:39:40.658 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T05:39:40.658 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T05:39:40.659 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T05:39:40.659 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T05:39:41.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:39:40 smithi153 ceph-mon[70044]: pgmap v345: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:41.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:39:40 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2609922328' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:39:41.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:39:40 smithi083 ceph-mon[79930]: pgmap v345: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:41.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:39:40 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2609922328' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:39:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:39:40 smithi083 ceph-mon[77242]: pgmap v345: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:41.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:39:40 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2609922328' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:39:41.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:39:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:39:41] "GET /metrics HTTP/1.1" 200 225276 "" "Prometheus/2.33.4" 2022-09-29T05:39:41.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:39:41 smithi083 ceph-mon[79930]: from='client.44473 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:39:41.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:39:41 smithi083 ceph-mon[79930]: from='client.44479 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:39:41.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:39:41 smithi083 ceph-mon[77242]: from='client.44473 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:39:41.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:39:41 smithi083 ceph-mon[77242]: from='client.44479 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:39:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:39:41 smithi153 ceph-mon[70044]: from='client.44473 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:39:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:39:41 smithi153 ceph-mon[70044]: from='client.44479 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:39:42.760 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:39:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:39:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:39:43.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:39:42 smithi153 ceph-mon[70044]: pgmap v346: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:43.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:39:42 smithi083 ceph-mon[79930]: pgmap v346: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:43.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:39:42 smithi083 ceph-mon[77242]: pgmap v346: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:39:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:39:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:39:43.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:39:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:39:43.559Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:39:44.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:39:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:39:44.256Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:39:45.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:39:44 smithi083 ceph-mon[79930]: pgmap v347: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:45.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:39:44 smithi083 ceph-mon[77242]: pgmap v347: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:45.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:39:44 smithi153 ceph-mon[70044]: pgmap v347: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:46.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:39:45 smithi083 ceph-mon[77242]: pgmap v348: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:46.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:39:45 smithi083 ceph-mon[79930]: pgmap v348: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:46.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:39:45 smithi153 ceph-mon[70044]: pgmap v348: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:49.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:39:48 smithi083 ceph-mon[79930]: pgmap v349: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:49.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:39:48 smithi083 ceph-mon[77242]: pgmap v349: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:49.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:39:48 smithi153 ceph-mon[70044]: pgmap v349: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:51.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:39:50 smithi153 ceph-mon[70044]: pgmap v350: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:51.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:39:50 smithi083 ceph-mon[79930]: pgmap v350: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:51.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:39:50 smithi083 ceph-mon[77242]: pgmap v350: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:51.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:39:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:39:51] "GET /metrics HTTP/1.1" 200 225276 "" "Prometheus/2.33.4" 2022-09-29T05:39:52.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:39:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:39:52.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:39:53.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:39:52 smithi083 ceph-mon[79930]: pgmap v351: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:53.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:39:52 smithi083 ceph-mon[77242]: pgmap v351: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:53.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:39:52 smithi153 ceph-mon[70044]: pgmap v351: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:53.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:39:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:39:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:39:53.830 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:39:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:39:53.559Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:39:54.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:39:53 smithi083 ceph-mon[79930]: pgmap v352: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:54.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:39:53 smithi083 ceph-mon[77242]: pgmap v352: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:54.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:39:53 smithi153 ceph-mon[70044]: pgmap v352: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:54.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:39:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:39:54.256Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:39:56.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:39:55 smithi083 ceph-mon[79930]: pgmap v353: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:56.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:39:55 smithi083 ceph-mon[77242]: pgmap v353: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:56.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:39:55 smithi153 ceph-mon[70044]: pgmap v353: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:58.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:39:57 smithi083 ceph-mon[79930]: pgmap v354: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:58.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:39:57 smithi083 ceph-mon[77242]: pgmap v354: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:39:58.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:39:57 smithi153 ceph-mon[70044]: pgmap v354: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:00.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:40:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:40:00.148Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:40:00.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:40:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[77219]: 2022-09-29T05:39:59.999+0000 7f160f665700 -1 log_channel(cluster) log [ERR] : Health detail: HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T05:40:00.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:40:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[77219]: 2022-09-29T05:39:59.999+0000 7f160f665700 -1 log_channel(cluster) log [ERR] : [WRN] OSD_UPGRADE_FINISHED: all OSDs are running quincy or later but require_osd_release < quincy 2022-09-29T05:40:00.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:40:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[77219]: 2022-09-29T05:39:59.999+0000 7f160f665700 -1 log_channel(cluster) log [ERR] : all OSDs are running quincy or later but require_osd_release < quincy 2022-09-29T05:40:00.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:40:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[77219]: 2022-09-29T05:39:59.999+0000 7f160f665700 -1 log_channel(cluster) log [ERR] : [WRN] TELEMETRY_CHANGED: Telemetry requires re-opt-in 2022-09-29T05:40:00.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:40:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[77219]: 2022-09-29T05:39:59.999+0000 7f160f665700 -1 log_channel(cluster) log [ERR] : telemetry module includes new collections; please re-opt-in to new collections with `ceph telemetry on` 2022-09-29T05:40:00.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:40:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[77219]: 2022-09-29T05:39:59.999+0000 7f160f665700 -1 log_channel(cluster) log [ERR] : [ERR] UPGRADE_EXCEPTION: Upgrade: failed due to an unexpected exception 2022-09-29T05:40:00.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:40:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[77219]: 2022-09-29T05:39:59.999+0000 7f160f665700 -1 log_channel(cluster) log [ERR] : Unexpected exception occurred during upgrade process: osd require-osd-release failed: unrecognized release unknown retval: -22 2022-09-29T05:40:01.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:40:00 smithi153 ceph-mon[70044]: pgmap v355: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:01.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:40:00 smithi153 ceph-mon[70044]: Health detail: HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T05:40:01.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:40:00 smithi153 ceph-mon[70044]: [WRN] OSD_UPGRADE_FINISHED: all OSDs are running quincy or later but require_osd_release < quincy 2022-09-29T05:40:01.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:40:00 smithi153 ceph-mon[70044]: all OSDs are running quincy or later but require_osd_release < quincy 2022-09-29T05:40:01.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:40:00 smithi153 ceph-mon[70044]: [WRN] TELEMETRY_CHANGED: Telemetry requires re-opt-in 2022-09-29T05:40:01.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:40:00 smithi153 ceph-mon[70044]: telemetry module includes new collections; please re-opt-in to new collections with `ceph telemetry on` 2022-09-29T05:40:01.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:40:00 smithi153 ceph-mon[70044]: [ERR] UPGRADE_EXCEPTION: Upgrade: failed due to an unexpected exception 2022-09-29T05:40:01.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:40:00 smithi153 ceph-mon[70044]: Unexpected exception occurred during upgrade process: osd require-osd-release failed: unrecognized release unknown retval: -22 2022-09-29T05:40:01.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:40:00 smithi083 ceph-mon[79930]: pgmap v355: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:01.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:40:00 smithi083 ceph-mon[79930]: Health detail: HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T05:40:01.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:40:00 smithi083 ceph-mon[79930]: [WRN] OSD_UPGRADE_FINISHED: all OSDs are running quincy or later but require_osd_release < quincy 2022-09-29T05:40:01.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:40:00 smithi083 ceph-mon[79930]: all OSDs are running quincy or later but require_osd_release < quincy 2022-09-29T05:40:01.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:40:00 smithi083 ceph-mon[79930]: [WRN] TELEMETRY_CHANGED: Telemetry requires re-opt-in 2022-09-29T05:40:01.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:40:00 smithi083 ceph-mon[79930]: telemetry module includes new collections; please re-opt-in to new collections with `ceph telemetry on` 2022-09-29T05:40:01.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:40:00 smithi083 ceph-mon[79930]: [ERR] UPGRADE_EXCEPTION: Upgrade: failed due to an unexpected exception 2022-09-29T05:40:01.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:40:00 smithi083 ceph-mon[79930]: Unexpected exception occurred during upgrade process: osd require-osd-release failed: unrecognized release unknown retval: -22 2022-09-29T05:40:01.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:40:00 smithi083 ceph-mon[77242]: pgmap v355: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:01.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:40:00 smithi083 ceph-mon[77242]: Health detail: HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T05:40:01.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:40:00 smithi083 ceph-mon[77242]: [WRN] OSD_UPGRADE_FINISHED: all OSDs are running quincy or later but require_osd_release < quincy 2022-09-29T05:40:01.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:40:00 smithi083 ceph-mon[77242]: all OSDs are running quincy or later but require_osd_release < quincy 2022-09-29T05:40:01.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:40:00 smithi083 ceph-mon[77242]: [WRN] TELEMETRY_CHANGED: Telemetry requires re-opt-in 2022-09-29T05:40:01.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:40:00 smithi083 ceph-mon[77242]: telemetry module includes new collections; please re-opt-in to new collections with `ceph telemetry on` 2022-09-29T05:40:01.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:40:00 smithi083 ceph-mon[77242]: [ERR] UPGRADE_EXCEPTION: Upgrade: failed due to an unexpected exception 2022-09-29T05:40:01.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:40:00 smithi083 ceph-mon[77242]: Unexpected exception occurred during upgrade process: osd require-osd-release failed: unrecognized release unknown retval: -22 2022-09-29T05:40:01.875 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:40:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:40:01] "GET /metrics HTTP/1.1" 200 225286 "" "Prometheus/2.33.4" 2022-09-29T05:40:02.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:40:01 smithi083 ceph-mon[79930]: pgmap v356: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:02.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:40:01 smithi083 ceph-mon[77242]: pgmap v356: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:02.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:40:01 smithi153 ceph-mon[70044]: pgmap v356: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:02.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:40:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:40:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:40:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:40:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:40:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:40:03.877 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:40:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:40:03.560Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:40:04.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:40:03 smithi083 ceph-mon[77242]: pgmap v357: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:04.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:40:03 smithi083 ceph-mon[79930]: pgmap v357: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:04.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:40:03 smithi153 ceph-mon[70044]: pgmap v357: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:04.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:40:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:40:04.258Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:40:06.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:40:05 smithi083 ceph-mon[79930]: pgmap v358: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:06.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:40:05 smithi083 ceph-mon[77242]: pgmap v358: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:06.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:40:05 smithi153 ceph-mon[70044]: pgmap v358: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:08.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:40:07 smithi083 ceph-mon[79930]: pgmap v359: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:08.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:40:07 smithi083 ceph-mon[77242]: pgmap v359: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:08.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:40:07 smithi153 ceph-mon[70044]: pgmap v359: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:10.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:40:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:40:10.147Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:40:10.983 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T05:40:11.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:40:10 smithi153 ceph-mon[70044]: pgmap v360: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:11.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:40:10 smithi083 ceph-mon[79930]: pgmap v360: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:11.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:40:10 smithi083 ceph-mon[77242]: pgmap v360: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:11.274 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T05:40:11.274 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (13m) 7m ago 14m 26.4M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T05:40:11.275 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (14m) 5m ago 14m 59.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T05:40:11.275 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (15m) 7m ago 15m 15.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T05:40:11.275 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (15m) 5m ago 15m 19.4M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T05:40:11.275 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10m) 5m ago 17m 430M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T05:40:11.276 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11m) 7m ago 19m 490M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T05:40:11.276 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10m) 7m ago 19m 86.0M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T05:40:11.276 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10m) 5m ago 18m 75.3M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T05:40:11.276 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10m) 7m ago 18m 70.7M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T05:40:11.276 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (14m) 7m ago 14m 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T05:40:11.277 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (14m) 5m ago 14m 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T05:40:11.277 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9m) 7m ago 17m 63.4M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T05:40:11.277 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8m) 7m ago 17m 56.5M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T05:40:11.277 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8m) 7m ago 16m 51.9M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T05:40:11.278 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7m) 7m ago 16m 12.9M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T05:40:11.278 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7m) 5m ago 16m 61.8M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T05:40:11.278 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6m) 5m ago 16m 63.4M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T05:40:11.278 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6m) 5m ago 15m 56.2M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T05:40:11.279 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5m) 5m ago 15m 15.3M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T05:40:11.279 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (11m) 5m ago 14m 80.9M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T05:40:11.632 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T05:40:11.633 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T05:40:11.633 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T05:40:11.633 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:40:11.634 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T05:40:11.634 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T05:40:11.634 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:40:11.634 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T05:40:11.635 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T05:40:11.635 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:40:11.635 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T05:40:11.635 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T05:40:11.635 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:40:11.636 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T05:40:11.636 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T05:40:11.636 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T05:40:11.636 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T05:40:11.637 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T05:40:11.768 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:40:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:40:11] "GET /metrics HTTP/1.1" 200 225292 "" "Prometheus/2.33.4" 2022-09-29T05:40:12.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:40:11 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4041124461' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:40:12.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:40:11 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4041124461' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:40:12.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:40:11 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4041124461' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:40:12.775 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:40:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:40:12.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:40:13.059 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:40:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:40:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:40:13.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:40:12 smithi153 ceph-mon[70044]: from='client.44491 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:40:13.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:40:12 smithi153 ceph-mon[70044]: from='client.34563 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:40:13.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:40:12 smithi153 ceph-mon[70044]: pgmap v361: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:13.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:40:12 smithi083 ceph-mon[79930]: from='client.44491 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:40:13.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:40:12 smithi083 ceph-mon[79930]: from='client.34563 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:40:13.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:40:12 smithi083 ceph-mon[79930]: pgmap v361: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:13.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:40:12 smithi083 ceph-mon[77242]: from='client.44491 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:40:13.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:40:12 smithi083 ceph-mon[77242]: from='client.34563 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:40:13.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:40:12 smithi083 ceph-mon[77242]: pgmap v361: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:13.877 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:40:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:40:13.560Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:40:14.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:40:13 smithi083 ceph-mon[79930]: pgmap v362: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:14.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:40:13 smithi083 ceph-mon[77242]: pgmap v362: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:14.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:40:13 smithi153 ceph-mon[70044]: pgmap v362: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:14.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:40:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:40:14.259Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:40:16.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:40:15 smithi083 ceph-mon[77242]: pgmap v363: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:16.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:40:15 smithi083 ceph-mon[79930]: pgmap v363: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:16.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:40:15 smithi153 ceph-mon[70044]: pgmap v363: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:18.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:40:17 smithi083 ceph-mon[79930]: pgmap v364: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:18.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:40:17 smithi083 ceph-mon[77242]: pgmap v364: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:18.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:40:17 smithi153 ceph-mon[70044]: pgmap v364: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:20.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:40:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:40:20.147Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:40:21.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:40:20 smithi153 ceph-mon[70044]: pgmap v365: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:21.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:40:20 smithi083 ceph-mon[79930]: pgmap v365: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:21.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:40:20 smithi083 ceph-mon[77242]: pgmap v365: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:21.878 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:40:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:40:21] "GET /metrics HTTP/1.1" 200 225292 "" "Prometheus/2.33.4" 2022-09-29T05:40:22.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:40:21 smithi083 ceph-mon[77242]: pgmap v366: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:22.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:40:21 smithi083 ceph-mon[79930]: pgmap v366: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:22.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:40:21 smithi153 ceph-mon[70044]: pgmap v366: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:22.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:40:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:40:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:40:23.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:40:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:40:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:40:23.881 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:40:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:40:23.561Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:40:24.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:40:23 smithi083 ceph-mon[77242]: pgmap v367: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:24.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:40:23 smithi083 ceph-mon[79930]: pgmap v367: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:24.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:40:23 smithi153 ceph-mon[70044]: pgmap v367: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:40:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:40:24.259Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:40:26.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:40:25 smithi083 ceph-mon[77242]: pgmap v368: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:26.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:40:25 smithi083 ceph-mon[79930]: pgmap v368: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:26.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:40:25 smithi153 ceph-mon[70044]: pgmap v368: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:28.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:40:27 smithi083 ceph-mon[77242]: pgmap v369: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:28.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:40:27 smithi083 ceph-mon[79930]: pgmap v369: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:28.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:40:27 smithi153 ceph-mon[70044]: pgmap v369: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:30.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:40:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:40:30.148Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:40:31.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:40:30 smithi153 ceph-mon[70044]: pgmap v370: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:31.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:40:30 smithi083 ceph-mon[77242]: pgmap v370: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:31.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:40:30 smithi083 ceph-mon[79930]: pgmap v370: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:31.882 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:40:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:40:31] "GET /metrics HTTP/1.1" 200 225294 "" "Prometheus/2.33.4" 2022-09-29T05:40:32.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:40:31 smithi083 ceph-mon[77242]: pgmap v371: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:32.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:40:31 smithi083 ceph-mon[79930]: pgmap v371: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:32.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:40:31 smithi153 ceph-mon[70044]: pgmap v371: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:32.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:40:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:40:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:40:33.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:40:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:40:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:40:33.882 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:40:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:40:33.561Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:40:34.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:40:33 smithi083 ceph-mon[79930]: pgmap v372: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:34.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:40:33 smithi083 ceph-mon[77242]: pgmap v372: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:34.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:40:33 smithi153 ceph-mon[70044]: pgmap v372: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:34.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:40:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:40:34.259Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:40:36.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:40:35 smithi083 ceph-mon[79930]: pgmap v373: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:36.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:40:35 smithi083 ceph-mon[77242]: pgmap v373: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:36.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:40:35 smithi153 ceph-mon[70044]: pgmap v373: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:37.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:40:37 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:40:37.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:40:37 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:40:37.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:40:37 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:40:37.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:40:37 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:40:37.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:40:37 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:40:37.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:40:37 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:40:37.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:40:37 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:40:37.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:40:37 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:40:37.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:40:37 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:40:38.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:40:38 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:40:38.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:40:38 smithi083 ceph-mon[79930]: pgmap v374: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:38.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:40:38 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:40:38.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:40:38 smithi083 ceph-mon[77242]: pgmap v374: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:39.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:40:38 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:40:39.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:40:38 smithi153 ceph-mon[70044]: pgmap v374: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:39.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:40:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:40:39.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:40:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:40:39.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:40:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:40:39.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:40:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:40:39.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:40:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:40:39.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:40:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:40:39.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:40:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:40:39.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:40:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:40:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:40:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:40:40.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:40:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:40:40.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:40:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:40:40.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:40:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:40:40.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:40:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:40:40.149Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:40:41.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:40:40 smithi153 ceph-mon[70044]: pgmap v375: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:41.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:40:40 smithi083 ceph-mon[79930]: pgmap v375: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:40:40 smithi083 ceph-mon[77242]: pgmap v375: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:41.881 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:40:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:40:41] "GET /metrics HTTP/1.1" 200 225293 "" "Prometheus/2.33.4" 2022-09-29T05:40:41.965 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T05:40:42.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:40:41 smithi083 ceph-mon[79930]: pgmap v376: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:42.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:40:41 smithi083 ceph-mon[77242]: pgmap v376: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:42.253 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T05:40:42.253 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (14m) 8m ago 14m 26.4M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T05:40:42.253 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (14m) 6m ago 14m 59.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T05:40:42.254 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (15m) 8m ago 15m 15.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T05:40:42.254 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (15m) 6m ago 15m 19.4M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T05:40:42.254 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (11m) 6m ago 18m 430M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T05:40:42.254 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11m) 8m ago 19m 490M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T05:40:42.255 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11m) 8m ago 19m 86.0M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T05:40:42.255 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10m) 6m ago 18m 75.3M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T05:40:42.255 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10m) 8m ago 18m 70.7M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T05:40:42.255 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (15m) 8m ago 15m 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T05:40:42.256 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (15m) 6m ago 15m 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T05:40:42.256 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9m) 8m ago 18m 63.4M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T05:40:42.256 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9m) 8m ago 17m 56.5M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T05:40:42.256 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9m) 8m ago 17m 51.9M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T05:40:42.257 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8m) 8m ago 17m 12.9M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T05:40:42.257 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7m) 6m ago 16m 61.8M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T05:40:42.257 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7m) 6m ago 16m 63.4M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T05:40:42.257 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6m) 6m ago 16m 56.2M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T05:40:42.257 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6m) 6m ago 16m 15.3M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T05:40:42.258 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (11m) 6m ago 15m 80.9M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T05:40:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:40:41 smithi153 ceph-mon[70044]: pgmap v376: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:42.608 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T05:40:42.609 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T05:40:42.609 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T05:40:42.609 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:40:42.609 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T05:40:42.610 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T05:40:42.610 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:40:42.610 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T05:40:42.610 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T05:40:42.611 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:40:42.611 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T05:40:42.611 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T05:40:42.611 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:40:42.612 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T05:40:42.612 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T05:40:42.612 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T05:40:42.612 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T05:40:42.613 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T05:40:42.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:40:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:40:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:40:43.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:40:42 smithi083 ceph-mon[77242]: from='client.34575 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:40:43.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:40:42 smithi083 ceph-mon[77242]: from='client.44515 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:40:43.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:40:42 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2912680898' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:40:43.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:40:42 smithi083 ceph-mon[79930]: from='client.34575 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:40:43.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:40:42 smithi083 ceph-mon[79930]: from='client.44515 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:40:43.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:40:42 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2912680898' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:40:43.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:40:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:40:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:40:43.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:40:42 smithi153 ceph-mon[70044]: from='client.34575 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:40:43.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:40:42 smithi153 ceph-mon[70044]: from='client.44515 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:40:43.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:40:42 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2912680898' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:40:43.887 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:40:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:40:43.562Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:40:44.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:40:43 smithi083 ceph-mon[79930]: pgmap v377: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:44.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:40:43 smithi083 ceph-mon[77242]: pgmap v377: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:44.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:40:43 smithi153 ceph-mon[70044]: pgmap v377: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:44.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:40:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:40:44.260Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:40:46.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:40:45 smithi083 ceph-mon[79930]: pgmap v378: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:46.244 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:40:45 smithi083 ceph-mon[77242]: pgmap v378: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:46.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:40:45 smithi153 ceph-mon[70044]: pgmap v378: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:48.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:40:47 smithi083 ceph-mon[79930]: pgmap v379: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:48.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:40:47 smithi083 ceph-mon[77242]: pgmap v379: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:48.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:40:47 smithi153 ceph-mon[70044]: pgmap v379: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:50.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:40:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:40:50.150Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:40:51.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:40:50 smithi153 ceph-mon[70044]: pgmap v380: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:51.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:40:50 smithi083 ceph-mon[79930]: pgmap v380: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:51.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:40:50 smithi083 ceph-mon[77242]: pgmap v380: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:51.883 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:40:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:40:51] "GET /metrics HTTP/1.1" 200 225293 "" "Prometheus/2.33.4" 2022-09-29T05:40:52.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:40:51 smithi083 ceph-mon[79930]: pgmap v381: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:52.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:40:51 smithi083 ceph-mon[77242]: pgmap v381: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:52.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:40:51 smithi153 ceph-mon[70044]: pgmap v381: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:40:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:40:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:40:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:40:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:40:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:40:53.883 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:40:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:40:53.562Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:40:54.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:40:53 smithi083 ceph-mon[77242]: pgmap v382: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:54.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:40:53 smithi083 ceph-mon[79930]: pgmap v382: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:54.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:40:53 smithi153 ceph-mon[70044]: pgmap v382: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:54.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:40:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:40:54.260Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:40:56.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:40:55 smithi083 ceph-mon[77242]: pgmap v383: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:56.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:40:55 smithi083 ceph-mon[79930]: pgmap v383: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:56.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:40:55 smithi153 ceph-mon[70044]: pgmap v383: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:58.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:40:57 smithi083 ceph-mon[79930]: pgmap v384: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:58.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:40:57 smithi083 ceph-mon[77242]: pgmap v384: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:40:58.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:40:57 smithi153 ceph-mon[70044]: pgmap v384: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:00.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:41:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:41:00.150Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:41:01.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:41:00 smithi153 ceph-mon[70044]: pgmap v385: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:01.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:41:00 smithi083 ceph-mon[79930]: pgmap v385: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:01.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:41:00 smithi083 ceph-mon[77242]: pgmap v385: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:01.885 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:41:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:41:01] "GET /metrics HTTP/1.1" 200 225296 "" "Prometheus/2.33.4" 2022-09-29T05:41:02.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:41:01 smithi083 ceph-mon[79930]: pgmap v386: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:02.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:41:01 smithi083 ceph-mon[77242]: pgmap v386: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:02.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:41:01 smithi153 ceph-mon[70044]: pgmap v386: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:02.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:41:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:41:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:41:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:41:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:41:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:41:03.310 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Sep 29 05:41:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-grafana-a[56315]: t=2022-09-29T05:41:02+0000 lvl=info msg="Database locked, sleeping then retrying" logger=sqlstore error="database is locked" retry=0 2022-09-29T05:41:03.885 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:41:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:41:03.563Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:41:04.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:41:03 smithi083 ceph-mon[79930]: pgmap v387: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:04.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:41:03 smithi083 ceph-mon[77242]: pgmap v387: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:04.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:41:03 smithi153 ceph-mon[70044]: pgmap v387: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:04.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:41:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:41:04.261Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:41:06.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:41:05 smithi083 ceph-mon[79930]: pgmap v388: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:06.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:41:05 smithi083 ceph-mon[77242]: pgmap v388: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:06.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:41:05 smithi153 ceph-mon[70044]: pgmap v388: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:08.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:41:07 smithi083 ceph-mon[79930]: pgmap v389: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:08.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:41:07 smithi083 ceph-mon[77242]: pgmap v389: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:08.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:41:07 smithi153 ceph-mon[70044]: pgmap v389: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:10.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:41:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:41:10.151Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:41:11.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:41:10 smithi153 ceph-mon[70044]: pgmap v390: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:11.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:41:10 smithi083 ceph-mon[79930]: pgmap v390: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:11.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:41:10 smithi083 ceph-mon[77242]: pgmap v390: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:11.886 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:41:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:41:11] "GET /metrics HTTP/1.1" 200 225296 "" "Prometheus/2.33.4" 2022-09-29T05:41:12.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:41:11 smithi083 ceph-mon[77242]: pgmap v391: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:12.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:41:11 smithi083 ceph-mon[79930]: pgmap v391: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:12.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:41:11 smithi153 ceph-mon[70044]: pgmap v391: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:12.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:41:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:41:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:41:12.934 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T05:41:13.222 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T05:41:13.222 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (14m) 8m ago 15m 26.4M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T05:41:13.222 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (15m) 6m ago 15m 59.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T05:41:13.222 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (16m) 8m ago 16m 15.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T05:41:13.223 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (16m) 6m ago 16m 19.4M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T05:41:13.223 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (11m) 6m ago 18m 430M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T05:41:13.223 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (12m) 8m ago 20m 490M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T05:41:13.223 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11m) 8m ago 20m 86.0M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T05:41:13.223 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (11m) 6m ago 19m 75.3M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T05:41:13.224 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (11m) 8m ago 19m 70.7M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T05:41:13.224 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (15m) 8m ago 15m 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T05:41:13.224 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (15m) 6m ago 15m 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T05:41:13.224 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10m) 8m ago 18m 63.4M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T05:41:13.225 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9m) 8m ago 18m 56.5M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T05:41:13.225 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9m) 8m ago 17m 51.9M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T05:41:13.225 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8m) 8m ago 17m 12.9M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T05:41:13.225 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8m) 6m ago 17m 61.8M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T05:41:13.225 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7m) 6m ago 17m 63.4M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T05:41:13.226 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7m) 6m ago 16m 56.2M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T05:41:13.226 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6m) 6m ago 16m 15.3M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T05:41:13.226 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (12m) 6m ago 15m 80.9M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T05:41:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:41:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:41:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:41:13.581 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T05:41:13.581 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T05:41:13.582 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T05:41:13.582 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:41:13.582 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T05:41:13.582 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T05:41:13.583 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:41:13.583 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T05:41:13.583 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T05:41:13.583 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:41:13.583 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T05:41:13.584 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T05:41:13.584 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:41:13.584 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T05:41:13.584 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T05:41:13.584 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T05:41:13.585 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T05:41:13.585 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T05:41:13.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:41:13 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/635966088' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:41:13.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:41:13 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/635966088' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:41:13.964 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:41:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:41:13.563Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:41:14.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:41:13 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/635966088' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:41:14.637 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:41:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:41:14.261Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:41:14.638 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:41:14 smithi083 ceph-mon[77242]: from='client.34593 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:41:14.639 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:41:14 smithi083 ceph-mon[77242]: from='client.44533 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:41:14.639 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:41:14 smithi083 ceph-mon[77242]: pgmap v392: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:14.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:41:14 smithi083 ceph-mon[79930]: from='client.34593 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:41:14.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:41:14 smithi083 ceph-mon[79930]: from='client.44533 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:41:14.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:41:14 smithi083 ceph-mon[79930]: pgmap v392: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:15.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:41:14 smithi153 ceph-mon[70044]: from='client.34593 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:41:15.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:41:14 smithi153 ceph-mon[70044]: from='client.44533 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:41:15.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:41:14 smithi153 ceph-mon[70044]: pgmap v392: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:16.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:41:15 smithi083 ceph-mon[77242]: pgmap v393: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:16.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:41:15 smithi083 ceph-mon[79930]: pgmap v393: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:16.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:41:15 smithi153 ceph-mon[70044]: pgmap v393: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:18.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:41:17 smithi083 ceph-mon[79930]: pgmap v394: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:18.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:41:17 smithi083 ceph-mon[77242]: pgmap v394: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:18.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:41:17 smithi153 ceph-mon[70044]: pgmap v394: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:20.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:41:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:41:20.151Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:41:21.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:41:20 smithi153 ceph-mon[70044]: pgmap v395: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:21.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:41:20 smithi083 ceph-mon[79930]: pgmap v395: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:21.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:41:20 smithi083 ceph-mon[77242]: pgmap v395: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:21.888 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:41:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:41:21] "GET /metrics HTTP/1.1" 200 225296 "" "Prometheus/2.33.4" 2022-09-29T05:41:22.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:41:21 smithi083 ceph-mon[77242]: pgmap v396: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:22.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:41:21 smithi083 ceph-mon[79930]: pgmap v396: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:22.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:41:21 smithi153 ceph-mon[70044]: pgmap v396: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:22.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:41:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:41:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:41:23.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:41:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:41:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:41:23.889 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:41:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:41:23.565Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:41:24.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:41:23 smithi083 ceph-mon[79930]: pgmap v397: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:24.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:41:23 smithi083 ceph-mon[77242]: pgmap v397: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:24.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:41:23 smithi153 ceph-mon[70044]: pgmap v397: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:24.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:41:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:41:24.263Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:41:26.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:41:25 smithi083 ceph-mon[77242]: pgmap v398: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:26.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:41:25 smithi083 ceph-mon[79930]: pgmap v398: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:26.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:41:25 smithi153 ceph-mon[70044]: pgmap v398: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:28.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:41:27 smithi083 ceph-mon[77242]: pgmap v399: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:28.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:41:27 smithi083 ceph-mon[79930]: pgmap v399: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:28.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:41:27 smithi153 ceph-mon[70044]: pgmap v399: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:30.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:41:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:41:30.151Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:41:31.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:41:30 smithi153 ceph-mon[70044]: pgmap v400: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:31.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:41:30 smithi083 ceph-mon[77242]: pgmap v400: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:31.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:41:30 smithi083 ceph-mon[79930]: pgmap v400: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:31.892 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:41:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:41:31] "GET /metrics HTTP/1.1" 200 225292 "" "Prometheus/2.33.4" 2022-09-29T05:41:32.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:41:31 smithi083 ceph-mon[79930]: pgmap v401: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:32.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:41:31 smithi083 ceph-mon[77242]: pgmap v401: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:32.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:41:31 smithi153 ceph-mon[70044]: pgmap v401: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:32.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:41:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:41:32.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:41:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:41:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:41:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:41:33.893 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:41:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:41:33.565Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:41:34.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:41:33 smithi083 ceph-mon[77242]: pgmap v402: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:34.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:41:33 smithi083 ceph-mon[79930]: pgmap v402: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:34.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:41:33 smithi153 ceph-mon[70044]: pgmap v402: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:34.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:41:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:41:34.263Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:41:36.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:41:35 smithi083 ceph-mon[79930]: pgmap v403: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:36.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:41:35 smithi083 ceph-mon[77242]: pgmap v403: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:36.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:41:35 smithi153 ceph-mon[70044]: pgmap v403: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:38.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:41:37 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:41:38.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:41:37 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:41:38.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:41:37 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:41:38.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:41:37 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:41:38.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:41:37 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:41:38.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:41:37 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:41:38.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:41:37 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:41:38.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:41:37 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:41:38.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:41:37 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:41:39.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:41:38 smithi153 ceph-mon[70044]: pgmap v404: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:39.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:41:38 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:41:39.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:41:38 smithi083 ceph-mon[77242]: pgmap v404: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:39.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:41:38 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:41:39.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:41:38 smithi083 ceph-mon[79930]: pgmap v404: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:39.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:41:38 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:41:40.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:41:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:41:40.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:41:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:41:40.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:41:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:41:40.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:41:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:41:40.155 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:41:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:41:40.155 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:41:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:41:40.155 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:41:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:41:40.156 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:41:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:41:40.156 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:41:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:41:40.156 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:41:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:41:40.156 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:41:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:41:40.157 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:41:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:41:40.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:41:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:41:40.152Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:41:41.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:41:40 smithi153 ceph-mon[70044]: pgmap v405: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:41:40 smithi083 ceph-mon[77242]: pgmap v405: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:41.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:41:40 smithi083 ceph-mon[79930]: pgmap v405: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:41.894 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:41:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:41:41] "GET /metrics HTTP/1.1" 200 225296 "" "Prometheus/2.33.4" 2022-09-29T05:41:42.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:41:41 smithi083 ceph-mon[79930]: pgmap v406: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:42.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:41:41 smithi083 ceph-mon[77242]: pgmap v406: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:41:41 smithi153 ceph-mon[70044]: pgmap v406: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:42.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:41:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:41:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:41:43.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:41:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:41:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:41:43.894 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:41:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:41:43.566Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:41:43.907 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T05:41:44.196 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T05:41:44.196 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (15m) 9m ago 15m 26.4M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T05:41:44.197 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (15m) 7m ago 15m 59.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T05:41:44.197 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (16m) 9m ago 16m 15.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T05:41:44.197 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (16m) 7m ago 16m 19.4M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T05:41:44.197 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (12m) 7m ago 19m 430M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T05:41:44.198 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (12m) 9m ago 20m 490M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T05:41:44.198 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (12m) 9m ago 20m 86.0M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T05:41:44.198 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (11m) 7m ago 19m 75.3M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T05:41:44.198 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (11m) 9m ago 19m 70.7M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T05:41:44.199 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (16m) 9m ago 16m 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T05:41:44.199 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (16m) 7m ago 16m 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T05:41:44.199 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10m) 9m ago 19m 63.4M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T05:41:44.199 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10m) 9m ago 18m 56.5M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T05:41:44.199 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10m) 9m ago 18m 51.9M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T05:41:44.200 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9m) 9m ago 18m 12.9M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T05:41:44.200 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8m) 7m ago 17m 61.8M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T05:41:44.200 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8m) 7m ago 17m 63.4M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T05:41:44.200 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7m) 7m ago 17m 56.2M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T05:41:44.201 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7m) 7m ago 17m 15.3M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T05:41:44.201 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (12m) 7m ago 16m 80.9M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T05:41:44.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:41:43 smithi083 ceph-mon[79930]: pgmap v407: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:44.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:41:43 smithi083 ceph-mon[77242]: pgmap v407: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:44.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:41:43 smithi153 ceph-mon[70044]: pgmap v407: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:44.554 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T05:41:44.554 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T05:41:44.555 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T05:41:44.555 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:41:44.555 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T05:41:44.555 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T05:41:44.556 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:41:44.556 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T05:41:44.556 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T05:41:44.556 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:41:44.557 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T05:41:44.557 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T05:41:44.557 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:41:44.557 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T05:41:44.557 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T05:41:44.558 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T05:41:44.558 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T05:41:44.558 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T05:41:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:41:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:41:44.263Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:41:45.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:41:44 smithi083 ceph-mon[79930]: from='client.34611 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:41:45.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:41:44 smithi083 ceph-mon[79930]: from='client.44551 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:41:45.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:41:44 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3882889023' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:41:45.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:41:44 smithi083 ceph-mon[77242]: from='client.34611 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:41:45.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:41:44 smithi083 ceph-mon[77242]: from='client.44551 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:41:45.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:41:44 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3882889023' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:41:45.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:41:44 smithi153 ceph-mon[70044]: from='client.34611 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:41:45.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:41:44 smithi153 ceph-mon[70044]: from='client.44551 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:41:45.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:41:44 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3882889023' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:41:46.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:41:45 smithi083 ceph-mon[79930]: pgmap v408: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:46.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:41:45 smithi083 ceph-mon[77242]: pgmap v408: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:46.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:41:45 smithi153 ceph-mon[70044]: pgmap v408: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:48.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:41:47 smithi083 ceph-mon[77242]: pgmap v409: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:48.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:41:47 smithi083 ceph-mon[79930]: pgmap v409: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:48.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:41:47 smithi153 ceph-mon[70044]: pgmap v409: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:50.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:41:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:41:50.153Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:41:51.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:41:50 smithi153 ceph-mon[70044]: pgmap v410: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:51.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:41:50 smithi083 ceph-mon[79930]: pgmap v410: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:51.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:41:50 smithi083 ceph-mon[77242]: pgmap v410: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:51.899 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:41:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:41:51] "GET /metrics HTTP/1.1" 200 225296 "" "Prometheus/2.33.4" 2022-09-29T05:41:51.900 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:41:51 smithi083 ceph-mon[77242]: pgmap v411: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:52.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:41:51 smithi083 ceph-mon[79930]: pgmap v411: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:52.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:41:51 smithi153 ceph-mon[70044]: pgmap v411: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:52.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:41:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:41:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:41:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:41:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:41:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:41:53.896 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:41:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:41:53.565Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:41:54.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:41:53 smithi083 ceph-mon[77242]: pgmap v412: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:54.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:41:53 smithi083 ceph-mon[79930]: pgmap v412: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:54.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:41:53 smithi153 ceph-mon[70044]: pgmap v412: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:54.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:41:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:41:54.264Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:41:56.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:41:55 smithi083 ceph-mon[79930]: pgmap v413: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:56.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:41:55 smithi083 ceph-mon[77242]: pgmap v413: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:56.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:41:55 smithi153 ceph-mon[70044]: pgmap v413: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:58.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:41:57 smithi083 ceph-mon[79930]: pgmap v414: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:58.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:41:57 smithi083 ceph-mon[77242]: pgmap v414: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:41:58.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:41:57 smithi153 ceph-mon[70044]: pgmap v414: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:00.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:42:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:42:00.154Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:42:01.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:42:00 smithi153 ceph-mon[70044]: pgmap v415: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:01.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:42:00 smithi083 ceph-mon[79930]: pgmap v415: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:01.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:42:00 smithi083 ceph-mon[77242]: pgmap v415: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:01.897 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:42:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:42:01] "GET /metrics HTTP/1.1" 200 225296 "" "Prometheus/2.33.4" 2022-09-29T05:42:02.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:42:01 smithi083 ceph-mon[79930]: pgmap v416: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:02.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:42:01 smithi083 ceph-mon[77242]: pgmap v416: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:02.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:42:01 smithi153 ceph-mon[70044]: pgmap v416: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:02.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:42:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:42:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:42:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:42:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:42:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:42:03.895 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:42:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:42:03.566Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:42:04.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:42:03 smithi083 ceph-mon[79930]: pgmap v417: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:04.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:42:03 smithi083 ceph-mon[77242]: pgmap v417: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:04.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:42:03 smithi153 ceph-mon[70044]: pgmap v417: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:04.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:42:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:42:04.265Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:42:06.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:42:05 smithi083 ceph-mon[77242]: pgmap v418: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:06.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:42:05 smithi083 ceph-mon[79930]: pgmap v418: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:06.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:42:05 smithi153 ceph-mon[70044]: pgmap v418: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:08.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:42:07 smithi083 ceph-mon[79930]: pgmap v419: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:08.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:42:07 smithi083 ceph-mon[77242]: pgmap v419: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:08.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:42:07 smithi153 ceph-mon[70044]: pgmap v419: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:10.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:42:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:42:10.154Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:42:11.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:42:10 smithi153 ceph-mon[70044]: pgmap v420: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:11.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:42:10 smithi083 ceph-mon[79930]: pgmap v420: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:11.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:42:10 smithi083 ceph-mon[77242]: pgmap v420: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:11.897 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:42:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:42:11] "GET /metrics HTTP/1.1" 200 225294 "" "Prometheus/2.33.4" 2022-09-29T05:42:12.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:42:11 smithi083 ceph-mon[79930]: pgmap v421: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:12.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:42:11 smithi083 ceph-mon[77242]: pgmap v421: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:12.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:42:11 smithi153 ceph-mon[70044]: pgmap v421: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:42:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:42:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:42:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:42:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:42:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:42:13.897 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:42:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:42:13.566Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:42:14.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:42:13 smithi083 ceph-mon[77242]: pgmap v422: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:14.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:42:13 smithi083 ceph-mon[79930]: pgmap v422: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:14.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:42:13 smithi153 ceph-mon[70044]: pgmap v422: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:14.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:42:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:42:14.266Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:42:14.882 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T05:42:15.169 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T05:42:15.169 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (15m) 9m ago 16m 26.4M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T05:42:15.169 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (16m) 7m ago 16m 59.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T05:42:15.170 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (17m) 9m ago 17m 15.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T05:42:15.170 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (17m) 7m ago 17m 19.4M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T05:42:15.170 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (12m) 7m ago 19m 430M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T05:42:15.171 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (13m) 9m ago 21m 490M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T05:42:15.171 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (12m) 9m ago 21m 86.0M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T05:42:15.171 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (12m) 7m ago 20m 75.3M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T05:42:15.171 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (12m) 9m ago 20m 70.7M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T05:42:15.172 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (16m) 9m ago 16m 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T05:42:15.172 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (16m) 7m ago 16m 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T05:42:15.172 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11m) 9m ago 19m 63.4M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T05:42:15.172 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10m) 9m ago 19m 56.5M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T05:42:15.173 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10m) 9m ago 18m 51.9M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T05:42:15.173 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10m) 9m ago 18m 12.9M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T05:42:15.173 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9m) 7m ago 18m 61.8M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T05:42:15.173 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8m) 7m ago 18m 63.4M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T05:42:15.174 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8m) 7m ago 17m 56.2M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T05:42:15.174 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7m) 7m ago 17m 15.3M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T05:42:15.174 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (13m) 7m ago 16m 80.9M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T05:42:15.523 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T05:42:15.524 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T05:42:15.524 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T05:42:15.524 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:42:15.525 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T05:42:15.525 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T05:42:15.525 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:42:15.525 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T05:42:15.526 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T05:42:15.526 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:42:15.526 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T05:42:15.526 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T05:42:15.527 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:42:15.527 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T05:42:15.527 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T05:42:15.527 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T05:42:15.527 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T05:42:15.528 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T05:42:16.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:42:15 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/446368831' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:42:16.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:42:15 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/446368831' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:42:16.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:42:15 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/446368831' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:42:17.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:42:16 smithi153 ceph-mon[70044]: from='client.44563 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:42:17.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:42:16 smithi153 ceph-mon[70044]: from='client.34635 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:42:17.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:42:16 smithi153 ceph-mon[70044]: pgmap v423: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:17.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:42:16 smithi083 ceph-mon[79930]: from='client.44563 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:42:17.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:42:16 smithi083 ceph-mon[79930]: from='client.34635 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:42:17.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:42:16 smithi083 ceph-mon[79930]: pgmap v423: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:17.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:42:16 smithi083 ceph-mon[77242]: from='client.44563 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:42:17.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:42:16 smithi083 ceph-mon[77242]: from='client.34635 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:42:17.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:42:16 smithi083 ceph-mon[77242]: pgmap v423: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:18.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:42:17 smithi083 ceph-mon[79930]: pgmap v424: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:18.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:42:17 smithi083 ceph-mon[77242]: pgmap v424: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:18.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:42:17 smithi153 ceph-mon[70044]: pgmap v424: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:20.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:42:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:42:20.155Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:42:21.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:42:20 smithi153 ceph-mon[70044]: pgmap v425: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:21.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:42:20 smithi083 ceph-mon[79930]: pgmap v425: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:21.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:42:20 smithi083 ceph-mon[77242]: pgmap v425: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:21.900 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:42:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:42:21] "GET /metrics HTTP/1.1" 200 225294 "" "Prometheus/2.33.4" 2022-09-29T05:42:22.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:42:21 smithi083 ceph-mon[77242]: pgmap v426: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:22.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:42:21 smithi083 ceph-mon[79930]: pgmap v426: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:22.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:42:21 smithi153 ceph-mon[70044]: pgmap v426: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:42:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:42:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:42:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:42:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:42:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:42:23.901 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:42:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:42:23.567Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:42:24.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:42:23 smithi083 ceph-mon[79930]: pgmap v427: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:24.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:42:23 smithi083 ceph-mon[77242]: pgmap v427: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:24.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:42:23 smithi153 ceph-mon[70044]: pgmap v427: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:24.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:42:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:42:24.266Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:42:26.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:42:25 smithi083 ceph-mon[77242]: pgmap v428: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:26.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:42:25 smithi083 ceph-mon[79930]: pgmap v428: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:26.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:42:25 smithi153 ceph-mon[70044]: pgmap v428: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:28.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:42:27 smithi083 ceph-mon[79930]: pgmap v429: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:28.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:42:27 smithi083 ceph-mon[77242]: pgmap v429: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:28.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:42:27 smithi153 ceph-mon[70044]: pgmap v429: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:30.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:42:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:42:30.156Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:42:31.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:42:30 smithi153 ceph-mon[70044]: pgmap v430: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:31.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:42:30 smithi083 ceph-mon[77242]: pgmap v430: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:31.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:42:30 smithi083 ceph-mon[79930]: pgmap v430: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:31.900 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:42:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:42:31] "GET /metrics HTTP/1.1" 200 225297 "" "Prometheus/2.33.4" 2022-09-29T05:42:32.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:42:31 smithi083 ceph-mon[79930]: pgmap v431: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:32.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:42:31 smithi083 ceph-mon[77242]: pgmap v431: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:32.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:42:31 smithi153 ceph-mon[70044]: pgmap v431: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:32.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:42:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:42:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:42:33.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:42:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:42:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:42:33.900 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:42:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:42:33.568Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:42:34.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:42:33 smithi083 ceph-mon[79930]: pgmap v432: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:34.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:42:33 smithi083 ceph-mon[77242]: pgmap v432: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:34.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:42:33 smithi153 ceph-mon[70044]: pgmap v432: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:34.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:42:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:42:34.267Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:42:36.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:42:35 smithi083 ceph-mon[79930]: pgmap v433: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:36.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:42:35 smithi083 ceph-mon[77242]: pgmap v433: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:36.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:42:35 smithi153 ceph-mon[70044]: pgmap v433: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:38.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:42:37 smithi083 ceph-mon[79930]: pgmap v434: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:38.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:42:37 smithi083 ceph-mon[77242]: pgmap v434: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:38.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:42:37 smithi153 ceph-mon[70044]: pgmap v434: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:39.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:42:38 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:42:39.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:42:38 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:42:39.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:42:38 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:42:39.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:42:38 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:42:39.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:42:38 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:42:39.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:42:38 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:42:39.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:42:38 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:42:39.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:42:38 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:42:39.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:42:38 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:42:39.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:42:38 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:42:39.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:42:38 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:42:39.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:42:38 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:42:39.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:42:38 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:42:39.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:42:38 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:42:39.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:42:38 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:42:39.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:42:38 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:42:39.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:42:38 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:42:39.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:42:38 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:42:39.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:42:38 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:42:39.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:42:38 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:42:39.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:42:38 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:42:40.160 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:42:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:42:40.160 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:42:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:42:40.160 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:42:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:42:40.161 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:42:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:42:40.161 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:42:39 smithi083 ceph-mon[79930]: pgmap v435: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:40.161 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:42:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:42:40.161 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:42:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:42:40.162 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:42:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:42:40.162 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:42:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:42:40.162 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:42:39 smithi083 ceph-mon[77242]: pgmap v435: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:42:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:42:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:42:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:42:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:42:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:42:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:42:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:42:40.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:42:39 smithi153 ceph-mon[70044]: pgmap v435: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:40.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:42:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:42:40.156Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:42:41.901 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:42:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:42:41] "GET /metrics HTTP/1.1" 200 225294 "" "Prometheus/2.33.4" 2022-09-29T05:42:42.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:42:41 smithi083 ceph-mon[77242]: pgmap v436: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:42.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:42:41 smithi083 ceph-mon[79930]: pgmap v436: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:42:41 smithi153 ceph-mon[70044]: pgmap v436: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:42:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:42:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:42:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:42:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:42:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:42:43.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:42:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:42:43.568Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:42:44.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:42:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:42:44.267Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:42:44.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:42:44 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:42:44.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:42:44 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:42:44.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:42:44 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:42:44.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:42:44 smithi083 ceph-mon[77242]: pgmap v437: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:44.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:42:44 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:42:44.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:42:44 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:42:44.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:42:44 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:42:44.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:42:44 smithi083 ceph-mon[79930]: pgmap v437: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:44.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:42:44 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:42:44.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:42:44 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:42:44.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:42:44 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:42:44.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:42:44 smithi153 ceph-mon[70044]: pgmap v437: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:45.853 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T05:42:46.144 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T05:42:46.144 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (16m) 2s ago 16m 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T05:42:46.145 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (16m) 8m ago 16m 59.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T05:42:46.145 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (17m) 2s ago 17m 17.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T05:42:46.145 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (17m) 8m ago 17m 19.4M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T05:42:46.145 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (13m) 8m ago 20m 430M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T05:42:46.145 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (13m) 2s ago 21m 516M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T05:42:46.146 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (13m) 2s ago 22m 128M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T05:42:46.146 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (12m) 8m ago 20m 75.3M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T05:42:46.146 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (12m) 2s ago 20m 88.2M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T05:42:46.146 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (17m) 2s ago 17m 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T05:42:46.147 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (17m) 8m ago 17m 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T05:42:46.147 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11m) 2s ago 20m 107M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T05:42:46.147 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11m) 2s ago 19m 99.8M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T05:42:46.147 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11m) 2s ago 19m 84.6M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T05:42:46.147 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10m) 2s ago 19m 90.7M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T05:42:46.148 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9m) 8m ago 18m 61.8M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T05:42:46.148 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9m) 8m ago 18m 63.4M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T05:42:46.148 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8m) 8m ago 18m 56.2M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T05:42:46.148 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8m) 8m ago 18m 15.3M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T05:42:46.148 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (13m) 8m ago 17m 80.9M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T05:42:46.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:42:45 smithi083 ceph-mon[79930]: pgmap v438: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:46.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:42:45 smithi083 ceph-mon[79930]: from='client.34647 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:42:46.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:42:45 smithi083 ceph-mon[77242]: pgmap v438: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:46.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:42:45 smithi083 ceph-mon[77242]: from='client.34647 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:42:46.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:42:45 smithi153 ceph-mon[70044]: pgmap v438: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:46.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:42:45 smithi153 ceph-mon[70044]: from='client.34647 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:42:46.503 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T05:42:46.503 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T05:42:46.503 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T05:42:46.504 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:42:46.504 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T05:42:46.504 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T05:42:46.504 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:42:46.504 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T05:42:46.505 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T05:42:46.505 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:42:46.505 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T05:42:46.505 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T05:42:46.505 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:42:46.506 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T05:42:46.506 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T05:42:46.506 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T05:42:46.506 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T05:42:46.506 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T05:42:47.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:42:46 smithi083 ceph-mon[79930]: from='client.44587 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:42:47.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:42:46 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3428711898' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:42:47.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:42:46 smithi083 ceph-mon[77242]: from='client.44587 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:42:47.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:42:46 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3428711898' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:42:47.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:42:46 smithi153 ceph-mon[70044]: from='client.44587 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:42:47.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:42:46 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3428711898' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:42:48.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:42:47 smithi083 ceph-mon[79930]: pgmap v439: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:48.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:42:47 smithi083 ceph-mon[77242]: pgmap v439: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:48.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:42:47 smithi153 ceph-mon[70044]: pgmap v439: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:50.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:42:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:42:50.158Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:42:51.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:42:50 smithi153 ceph-mon[70044]: pgmap v440: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:51.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:42:50 smithi083 ceph-mon[77242]: pgmap v440: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:51.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:42:50 smithi083 ceph-mon[79930]: pgmap v440: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:51.902 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:42:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:42:51] "GET /metrics HTTP/1.1" 200 225294 "" "Prometheus/2.33.4" 2022-09-29T05:42:52.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:42:51 smithi083 ceph-mon[79930]: pgmap v441: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:52.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:42:51 smithi083 ceph-mon[77242]: pgmap v441: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:52.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:42:51 smithi153 ceph-mon[70044]: pgmap v441: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:52.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:42:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:42:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:42:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:42:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:42:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:42:53.905 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:42:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:42:53.570Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:42:54.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:42:53 smithi083 ceph-mon[79930]: pgmap v442: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:54.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:42:53 smithi083 ceph-mon[77242]: pgmap v442: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:54.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:42:53 smithi153 ceph-mon[70044]: pgmap v442: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:54.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:42:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:42:54.268Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:42:56.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:42:55 smithi083 ceph-mon[79930]: pgmap v443: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:56.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:42:55 smithi083 ceph-mon[77242]: pgmap v443: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:56.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:42:55 smithi153 ceph-mon[70044]: pgmap v443: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:58.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:42:57 smithi083 ceph-mon[79930]: pgmap v444: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:58.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:42:57 smithi083 ceph-mon[77242]: pgmap v444: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:42:58.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:42:57 smithi153 ceph-mon[70044]: pgmap v444: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:00.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:43:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:43:00.159Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:43:01.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:43:00 smithi153 ceph-mon[70044]: pgmap v445: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:01.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:43:00 smithi083 ceph-mon[79930]: pgmap v445: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:01.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:43:00 smithi083 ceph-mon[77242]: pgmap v445: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:01.904 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:43:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:43:01] "GET /metrics HTTP/1.1" 200 225299 "" "Prometheus/2.33.4" 2022-09-29T05:43:02.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:43:01 smithi083 ceph-mon[77242]: pgmap v446: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:02.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:43:01 smithi083 ceph-mon[79930]: pgmap v446: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:02.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:43:01 smithi153 ceph-mon[70044]: pgmap v446: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:43:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:43:02.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:43:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:43:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:43:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:43:03.905 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:43:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:43:03.571Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:43:04.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:43:03 smithi083 ceph-mon[79930]: pgmap v447: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:04.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:43:03 smithi083 ceph-mon[77242]: pgmap v447: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:04.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:43:03 smithi153 ceph-mon[70044]: pgmap v447: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:04.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:43:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:43:04.269Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:43:06.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:43:05 smithi083 ceph-mon[79930]: pgmap v448: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:06.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:43:05 smithi083 ceph-mon[77242]: pgmap v448: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:06.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:43:05 smithi153 ceph-mon[70044]: pgmap v448: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:08.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:43:07 smithi083 ceph-mon[79930]: pgmap v449: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:08.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:43:07 smithi083 ceph-mon[77242]: pgmap v449: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:08.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:43:07 smithi153 ceph-mon[70044]: pgmap v449: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:10.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:43:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:43:10.160Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:43:11.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:43:10 smithi153 ceph-mon[70044]: pgmap v450: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:11.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:43:10 smithi083 ceph-mon[79930]: pgmap v450: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:11.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:43:10 smithi083 ceph-mon[77242]: pgmap v450: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:11.907 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:43:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:43:11] "GET /metrics HTTP/1.1" 200 225285 "" "Prometheus/2.33.4" 2022-09-29T05:43:12.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:43:11 smithi083 ceph-mon[77242]: pgmap v451: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:12.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:43:11 smithi083 ceph-mon[79930]: pgmap v451: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:12.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:43:11 smithi153 ceph-mon[70044]: pgmap v451: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:12.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:43:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:43:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:43:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:43:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:43:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:43:13.908 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:43:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:43:13.572Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:43:14.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:43:13 smithi083 ceph-mon[79930]: pgmap v452: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:14.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:43:13 smithi083 ceph-mon[77242]: pgmap v452: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:14.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:43:13 smithi153 ceph-mon[70044]: pgmap v452: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:14.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:43:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:43:14.269Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:43:16.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:43:15 smithi083 ceph-mon[77242]: pgmap v453: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:16.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:43:15 smithi083 ceph-mon[79930]: pgmap v453: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:16.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:43:15 smithi153 ceph-mon[70044]: pgmap v453: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:16.830 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T05:43:17.122 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T05:43:17.122 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (16m) 33s ago 17m 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T05:43:17.123 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (17m) 8m ago 17m 59.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T05:43:17.123 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (18m) 33s ago 18m 17.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T05:43:17.123 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (18m) 8m ago 18m 19.4M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T05:43:17.123 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (14m) 8m ago 20m 430M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T05:43:17.124 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (14m) 33s ago 22m 516M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T05:43:17.124 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (13m) 33s ago 22m 128M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T05:43:17.124 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (13m) 8m ago 21m 75.3M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T05:43:17.124 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (13m) 33s ago 21m 88.2M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T05:43:17.124 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (17m) 33s ago 17m 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T05:43:17.125 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (17m) 8m ago 17m 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T05:43:17.125 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (12m) 33s ago 20m 107M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T05:43:17.125 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11m) 33s ago 20m 99.8M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T05:43:17.125 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11m) 33s ago 20m 84.6M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T05:43:17.125 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (11m) 33s ago 19m 90.7M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T05:43:17.126 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10m) 8m ago 19m 61.8M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T05:43:17.126 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9m) 8m ago 19m 63.4M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T05:43:17.126 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9m) 8m ago 18m 56.2M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T05:43:17.126 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8m) 8m ago 18m 15.3M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T05:43:17.127 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (14m) 8m ago 17m 80.9M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T05:43:17.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:43:16 smithi083 ceph-mon[79930]: from='client.34665 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:43:17.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:43:16 smithi083 ceph-mon[77242]: from='client.34665 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:43:17.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:43:16 smithi153 ceph-mon[70044]: from='client.34665 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:43:17.473 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T05:43:17.473 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T05:43:17.474 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T05:43:17.474 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:43:17.474 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T05:43:17.474 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T05:43:17.475 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:43:17.475 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T05:43:17.475 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T05:43:17.475 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:43:17.475 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T05:43:17.476 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T05:43:17.476 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:43:17.476 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T05:43:17.476 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T05:43:17.477 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T05:43:17.477 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T05:43:17.477 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T05:43:18.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:43:17 smithi083 ceph-mon[77242]: from='client.44605 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:43:18.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:43:17 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3222577127' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:43:18.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:43:17 smithi083 ceph-mon[77242]: pgmap v454: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:18.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:43:17 smithi083 ceph-mon[79930]: from='client.44605 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:43:18.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:43:17 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3222577127' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:43:18.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:43:17 smithi083 ceph-mon[79930]: pgmap v454: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:18.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:43:17 smithi153 ceph-mon[70044]: from='client.44605 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:43:18.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:43:17 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3222577127' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:43:18.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:43:17 smithi153 ceph-mon[70044]: pgmap v454: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:20.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:43:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:43:20.160Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:43:21.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:43:20 smithi083 ceph-mon[79930]: pgmap v455: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:21.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:43:20 smithi083 ceph-mon[77242]: pgmap v455: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:21.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:43:20 smithi153 ceph-mon[70044]: pgmap v455: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:21.910 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:43:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:43:21] "GET /metrics HTTP/1.1" 200 225285 "" "Prometheus/2.33.4" 2022-09-29T05:43:22.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:43:21 smithi083 ceph-mon[79930]: pgmap v456: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:22.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:43:21 smithi083 ceph-mon[77242]: pgmap v456: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:22.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:43:21 smithi153 ceph-mon[70044]: pgmap v456: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:43:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:43:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:43:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:43:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:43:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:43:23.910 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:43:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:43:23.572Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:43:23.911 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:43:23 smithi083 ceph-mon[77242]: pgmap v457: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:24.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:43:23 smithi083 ceph-mon[79930]: pgmap v457: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:24.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:43:23 smithi153 ceph-mon[70044]: pgmap v457: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:24.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:43:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:43:24.270Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:43:26.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:43:25 smithi083 ceph-mon[79930]: pgmap v458: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:26.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:43:25 smithi083 ceph-mon[77242]: pgmap v458: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:26.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:43:25 smithi153 ceph-mon[70044]: pgmap v458: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:28.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:43:27 smithi083 ceph-mon[79930]: pgmap v459: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:28.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:43:27 smithi083 ceph-mon[77242]: pgmap v459: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:28.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:43:27 smithi153 ceph-mon[70044]: pgmap v459: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:30.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:43:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:43:30.161Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:43:31.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:43:30 smithi083 ceph-mon[79930]: pgmap v460: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:31.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:43:30 smithi083 ceph-mon[77242]: pgmap v460: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:31.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:43:30 smithi153 ceph-mon[70044]: pgmap v460: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:31.910 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:43:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:43:31] "GET /metrics HTTP/1.1" 200 225285 "" "Prometheus/2.33.4" 2022-09-29T05:43:32.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:43:31 smithi083 ceph-mon[77242]: pgmap v461: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:32.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:43:31 smithi083 ceph-mon[79930]: pgmap v461: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:32.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:43:31 smithi153 ceph-mon[70044]: pgmap v461: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:32.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:43:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:43:32.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:43:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:43:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:43:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:43:33.910 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:43:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:43:33.573Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:43:34.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:43:33 smithi083 ceph-mon[79930]: pgmap v462: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:34.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:43:33 smithi083 ceph-mon[77242]: pgmap v462: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:34.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:43:33 smithi153 ceph-mon[70044]: pgmap v462: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:34.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:43:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:43:34.270Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:43:36.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:43:35 smithi083 ceph-mon[77242]: pgmap v463: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:36.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:43:35 smithi083 ceph-mon[79930]: pgmap v463: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:36.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:43:35 smithi153 ceph-mon[70044]: pgmap v463: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:38.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:43:37 smithi083 ceph-mon[79930]: pgmap v464: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:38.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:43:37 smithi083 ceph-mon[77242]: pgmap v464: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:38.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:43:37 smithi153 ceph-mon[70044]: pgmap v464: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:39.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:43:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:43:39.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:43:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:43:39.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:43:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:43:39.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:43:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:43:39.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:43:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:43:39.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:43:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:43:39.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:43:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:43:39.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:43:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:43:39.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:43:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:43:39.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:43:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:43:39.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:43:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:43:39.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:43:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:43:40.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:43:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:43:40.161Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:43:41.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:43:40 smithi083 ceph-mon[79930]: pgmap v465: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:43:40 smithi083 ceph-mon[77242]: pgmap v465: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:41.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:43:40 smithi153 ceph-mon[70044]: pgmap v465: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:41.913 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:43:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:43:41] "GET /metrics HTTP/1.1" 200 225274 "" "Prometheus/2.33.4" 2022-09-29T05:43:42.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:43:41 smithi083 ceph-mon[77242]: pgmap v466: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:42.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:43:41 smithi083 ceph-mon[79930]: pgmap v466: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:43:41 smithi153 ceph-mon[70044]: pgmap v466: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:43:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:43:42.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:43:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:43:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:43:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:43:43.831 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:43:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:43:43.573Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:43:44.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:43:43 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:43:44.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:43:43 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:43:44.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:43:43 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:43:44.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:43:43 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:43:44.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:43:43 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:43:44.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:43:43 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:43:44.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:43:43 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:43:44.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:43:43 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:43:44.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:43:43 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:43:44.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:43:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:43:44.270Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:43:45.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:43:44 smithi083 ceph-mon[79930]: pgmap v467: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:45.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:43:44 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:43:45.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:43:44 smithi083 ceph-mon[77242]: pgmap v467: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:45.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:43:44 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:43:45.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:43:44 smithi153 ceph-mon[70044]: pgmap v467: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:45.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:43:44 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:43:46.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:43:45 smithi083 ceph-mon[79930]: pgmap v468: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:46.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:43:45 smithi083 ceph-mon[77242]: pgmap v468: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:46.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:43:45 smithi153 ceph-mon[70044]: pgmap v468: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:47.798 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T05:43:48.091 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T05:43:48.092 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (17m) 64s ago 17m 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T05:43:48.092 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (17m) 9m ago 17m 59.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T05:43:48.092 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (18m) 64s ago 18m 17.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T05:43:48.092 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (18m) 9m ago 18m 19.4M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T05:43:48.093 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (14m) 9m ago 21m 430M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T05:43:48.093 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (15m) 64s ago 22m 516M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T05:43:48.093 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (14m) 64s ago 23m 128M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T05:43:48.093 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (13m) 9m ago 21m 75.3M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T05:43:48.094 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (13m) 64s ago 21m 88.2M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T05:43:48.094 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (18m) 64s ago 18m 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T05:43:48.094 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (18m) 9m ago 18m 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T05:43:48.094 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (12m) 64s ago 21m 107M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T05:43:48.095 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (12m) 64s ago 20m 99.8M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T05:43:48.095 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (12m) 64s ago 20m 84.6M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T05:43:48.095 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (11m) 64s ago 20m 90.7M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T05:43:48.095 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (11m) 9m ago 19m 61.8M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T05:43:48.095 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10m) 9m ago 19m 63.4M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T05:43:48.096 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9m) 9m ago 19m 56.2M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T05:43:48.096 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9m) 9m ago 19m 15.3M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T05:43:48.096 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (14m) 9m ago 18m 80.9M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T05:43:48.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:43:47 smithi083 ceph-mon[77242]: from='client.44617 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:43:48.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:43:47 smithi083 ceph-mon[77242]: pgmap v469: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:48.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:43:47 smithi083 ceph-mon[79930]: from='client.44617 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:43:48.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:43:47 smithi083 ceph-mon[79930]: pgmap v469: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:48.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:43:47 smithi153 ceph-mon[70044]: from='client.44617 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:43:48.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:43:47 smithi153 ceph-mon[70044]: pgmap v469: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:48.445 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T05:43:48.446 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T05:43:48.446 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T05:43:48.446 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:43:48.447 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T05:43:48.447 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T05:43:48.447 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:43:48.447 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T05:43:48.448 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T05:43:48.448 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:43:48.448 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T05:43:48.448 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T05:43:48.448 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:43:48.449 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T05:43:48.449 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T05:43:48.449 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T05:43:48.449 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T05:43:48.450 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T05:43:49.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:43:48 smithi083 ceph-mon[79930]: from='client.34689 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:43:49.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:43:48 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1010346402' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:43:49.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:43:48 smithi083 ceph-mon[77242]: from='client.34689 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:43:49.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:43:48 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1010346402' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:43:49.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:43:48 smithi153 ceph-mon[70044]: from='client.34689 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:43:49.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:43:48 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1010346402' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:43:50.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:43:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:43:50.162Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:43:51.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:43:50 smithi083 ceph-mon[79930]: pgmap v470: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:51.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:43:50 smithi083 ceph-mon[77242]: pgmap v470: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:51.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:43:50 smithi153 ceph-mon[70044]: pgmap v470: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:51.915 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:43:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:43:51] "GET /metrics HTTP/1.1" 200 225274 "" "Prometheus/2.33.4" 2022-09-29T05:43:52.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:43:51 smithi083 ceph-mon[79930]: pgmap v471: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:52.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:43:51 smithi083 ceph-mon[77242]: pgmap v471: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:52.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:43:51 smithi153 ceph-mon[70044]: pgmap v471: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:52.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:43:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:43:52.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:43:53.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:43:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:43:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:43:53.913 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:43:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:43:53.574Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:43:54.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:43:53 smithi083 ceph-mon[79930]: pgmap v472: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:54.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:43:53 smithi083 ceph-mon[77242]: pgmap v472: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:54.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:43:53 smithi153 ceph-mon[70044]: pgmap v472: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:43:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:43:54.271Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:43:56.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:43:55 smithi083 ceph-mon[79930]: pgmap v473: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:56.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:43:55 smithi083 ceph-mon[77242]: pgmap v473: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:56.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:43:55 smithi153 ceph-mon[70044]: pgmap v473: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:58.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:43:57 smithi083 ceph-mon[79930]: pgmap v474: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:58.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:43:57 smithi083 ceph-mon[77242]: pgmap v474: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:43:58.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:43:57 smithi153 ceph-mon[70044]: pgmap v474: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:00.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:44:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:44:00.163Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:44:01.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:44:00 smithi083 ceph-mon[79930]: pgmap v475: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:01.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:44:00 smithi083 ceph-mon[77242]: pgmap v475: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:01.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:44:00 smithi153 ceph-mon[70044]: pgmap v475: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:01.914 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:44:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:44:01] "GET /metrics HTTP/1.1" 200 225269 "" "Prometheus/2.33.4" 2022-09-29T05:44:02.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:44:01 smithi083 ceph-mon[79930]: pgmap v476: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:02.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:44:01 smithi083 ceph-mon[77242]: pgmap v476: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:02.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:44:01 smithi153 ceph-mon[70044]: pgmap v476: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:44:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:44:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:44:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:44:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:44:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:44:03.915 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:44:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:44:03.575Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:44:04.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:44:03 smithi083 ceph-mon[79930]: pgmap v477: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:04.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:44:03 smithi083 ceph-mon[77242]: pgmap v477: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:04.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:44:03 smithi153 ceph-mon[70044]: pgmap v477: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:04.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:44:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:44:04.272Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:44:06.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:44:05 smithi083 ceph-mon[77242]: pgmap v478: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:06.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:44:05 smithi083 ceph-mon[79930]: pgmap v478: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:06.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:44:05 smithi153 ceph-mon[70044]: pgmap v478: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:08.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:44:07 smithi083 ceph-mon[77242]: pgmap v479: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:08.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:44:07 smithi083 ceph-mon[79930]: pgmap v479: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:08.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:44:07 smithi153 ceph-mon[70044]: pgmap v479: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:10.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:44:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:44:10.163Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:44:11.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:44:10 smithi083 ceph-mon[79930]: pgmap v480: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:11.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:44:10 smithi083 ceph-mon[77242]: pgmap v480: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:11.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:44:10 smithi153 ceph-mon[70044]: pgmap v480: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:11.916 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:44:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:44:11] "GET /metrics HTTP/1.1" 200 225266 "" "Prometheus/2.33.4" 2022-09-29T05:44:12.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:44:11 smithi083 ceph-mon[77242]: pgmap v481: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:12.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:44:11 smithi083 ceph-mon[79930]: pgmap v481: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:12.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:44:11 smithi153 ceph-mon[70044]: pgmap v481: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:12.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:44:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:44:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:44:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:44:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:44:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:44:13.916 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:44:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:44:13.575Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:44:14.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:44:13 smithi083 ceph-mon[77242]: pgmap v482: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:14.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:44:13 smithi083 ceph-mon[79930]: pgmap v482: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:14.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:44:13 smithi153 ceph-mon[70044]: pgmap v482: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:14.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:44:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:44:14.272Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:44:16.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:44:15 smithi083 ceph-mon[77242]: pgmap v483: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:16.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:44:15 smithi083 ceph-mon[79930]: pgmap v483: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:16.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:44:15 smithi153 ceph-mon[70044]: pgmap v483: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:18.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:44:17 smithi083 ceph-mon[79930]: pgmap v484: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:18.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:44:17 smithi083 ceph-mon[77242]: pgmap v484: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:18.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:44:17 smithi153 ceph-mon[70044]: pgmap v484: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:18.769 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T05:44:19.059 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T05:44:19.059 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (18m) 95s ago 18m 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T05:44:19.059 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (18m) 9m ago 18m 59.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T05:44:19.060 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (19m) 95s ago 19m 17.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T05:44:19.060 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (19m) 9m ago 19m 19.4M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T05:44:19.060 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (15m) 9m ago 21m 430M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T05:44:19.060 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (15m) 95s ago 23m 516M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T05:44:19.061 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (14m) 95s ago 23m 128M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T05:44:19.061 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (14m) 9m ago 22m 75.3M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T05:44:19.061 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (14m) 95s ago 22m 88.2M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T05:44:19.061 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (18m) 95s ago 18m 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T05:44:19.062 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (18m) 9m ago 18m 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T05:44:19.062 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (13m) 95s ago 21m 107M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T05:44:19.062 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (12m) 95s ago 21m 99.8M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T05:44:19.062 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (12m) 95s ago 21m 84.6M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T05:44:19.063 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (12m) 95s ago 20m 90.7M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T05:44:19.063 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (11m) 9m ago 20m 61.8M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T05:44:19.063 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10m) 9m ago 20m 63.4M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T05:44:19.063 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10m) 9m ago 19m 56.2M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T05:44:19.064 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9m) 9m ago 19m 15.3M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T05:44:19.064 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (15m) 9m ago 18m 80.9M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T05:44:19.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:44:18 smithi083 ceph-mon[79930]: from='client.34701 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:44:19.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:44:18 smithi083 ceph-mon[77242]: from='client.34701 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:44:19.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:44:18 smithi153 ceph-mon[70044]: from='client.34701 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:44:19.414 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T05:44:19.415 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T05:44:19.415 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T05:44:19.415 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:44:19.416 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T05:44:19.416 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T05:44:19.416 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:44:19.416 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T05:44:19.416 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T05:44:19.417 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:44:19.417 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T05:44:19.417 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T05:44:19.417 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:44:19.418 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T05:44:19.418 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T05:44:19.418 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T05:44:19.418 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T05:44:19.419 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T05:44:20.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:44:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:44:20.163Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:44:20.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:44:19 smithi083 ceph-mon[77242]: from='client.44641 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:44:20.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:44:19 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1506822070' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:44:20.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:44:19 smithi083 ceph-mon[77242]: pgmap v485: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:20.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:44:19 smithi083 ceph-mon[79930]: from='client.44641 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:44:20.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:44:19 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1506822070' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:44:20.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:44:19 smithi083 ceph-mon[79930]: pgmap v485: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:20.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:44:19 smithi153 ceph-mon[70044]: from='client.44641 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:44:20.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:44:19 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1506822070' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:44:20.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:44:19 smithi153 ceph-mon[70044]: pgmap v485: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:21.920 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:44:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:44:21] "GET /metrics HTTP/1.1" 200 225266 "" "Prometheus/2.33.4" 2022-09-29T05:44:22.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:44:21 smithi083 ceph-mon[77242]: pgmap v486: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:22.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:44:21 smithi083 ceph-mon[79930]: pgmap v486: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:22.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:44:21 smithi153 ceph-mon[70044]: pgmap v486: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:44:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:44:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:44:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:44:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:44:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:44:23.918 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:44:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:44:23.576Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:44:24.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:44:23 smithi083 ceph-mon[79930]: pgmap v487: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:24.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:44:23 smithi083 ceph-mon[77242]: pgmap v487: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:24.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:44:23 smithi153 ceph-mon[70044]: pgmap v487: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:44:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:44:24.272Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:44:26.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:44:25 smithi083 ceph-mon[79930]: pgmap v488: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:26.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:44:25 smithi083 ceph-mon[77242]: pgmap v488: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:26.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:44:25 smithi153 ceph-mon[70044]: pgmap v488: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:28.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:44:27 smithi083 ceph-mon[79930]: pgmap v489: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:28.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:44:27 smithi083 ceph-mon[77242]: pgmap v489: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:28.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:44:27 smithi153 ceph-mon[70044]: pgmap v489: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:30.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:44:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:44:30.164Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:44:31.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:44:30 smithi083 ceph-mon[79930]: pgmap v490: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:31.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:44:30 smithi083 ceph-mon[77242]: pgmap v490: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:31.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:44:30 smithi153 ceph-mon[70044]: pgmap v490: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:31.919 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:44:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:44:31] "GET /metrics HTTP/1.1" 200 225263 "" "Prometheus/2.33.4" 2022-09-29T05:44:32.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:44:31 smithi083 ceph-mon[79930]: pgmap v491: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:32.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:44:31 smithi083 ceph-mon[77242]: pgmap v491: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:32.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:44:31 smithi153 ceph-mon[70044]: pgmap v491: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:44:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:44:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:44:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:44:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:44:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:44:33.920 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:44:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:44:33.577Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:44:34.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:44:33 smithi083 ceph-mon[79930]: pgmap v492: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:34.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:44:33 smithi083 ceph-mon[77242]: pgmap v492: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:34.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:44:33 smithi153 ceph-mon[70044]: pgmap v492: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:34.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:44:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:44:34.272Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:44:36.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:44:35 smithi083 ceph-mon[79930]: pgmap v493: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:36.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:44:35 smithi083 ceph-mon[77242]: pgmap v493: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:36.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:44:35 smithi153 ceph-mon[70044]: pgmap v493: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:38.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:44:37 smithi083 ceph-mon[79930]: pgmap v494: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:38.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:44:37 smithi083 ceph-mon[77242]: pgmap v494: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:38.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:44:37 smithi153 ceph-mon[70044]: pgmap v494: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:39.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:44:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:44:39.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:44:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:44:39.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:44:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:44:39.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:44:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:44:39.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:44:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:44:39.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:44:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:44:39.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:44:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:44:39.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:44:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:44:39.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:44:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:44:39.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:44:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:44:39.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:44:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:44:39.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:44:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:44:40.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:44:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:44:40.165Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:44:41.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:44:40 smithi083 ceph-mon[79930]: pgmap v495: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:44:40 smithi083 ceph-mon[77242]: pgmap v495: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:44:40 smithi153 ceph-mon[70044]: pgmap v495: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:41.923 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:44:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:44:41] "GET /metrics HTTP/1.1" 200 225252 "" "Prometheus/2.33.4" 2022-09-29T05:44:42.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:44:41 smithi083 ceph-mon[77242]: pgmap v496: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:42.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:44:41 smithi083 ceph-mon[79930]: pgmap v496: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:44:41 smithi153 ceph-mon[70044]: pgmap v496: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:44:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:44:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:44:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:44:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:44:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:44:43.923 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:44:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:44:43.578Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:44:44.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:44:43 smithi083 ceph-mon[79930]: pgmap v497: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:44.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:44:43 smithi083 ceph-mon[77242]: pgmap v497: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:44.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:44:43 smithi153 ceph-mon[70044]: pgmap v497: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:44:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:44:44.273Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:44:45.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:44:44 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:44:45.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:44:44 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:44:45.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:44:44 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:44:45.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:44:44 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:44:45.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:44:44 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:44:45.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:44:44 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:44:45.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:44:44 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:44:45.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:44:44 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:44:45.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:44:44 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:44:46.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:44:45 smithi083 ceph-mon[79930]: pgmap v498: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:46.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:44:45 smithi083 ceph-mon[77242]: pgmap v498: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:46.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:44:45 smithi153 ceph-mon[70044]: pgmap v498: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:48.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:44:47 smithi083 ceph-mon[79930]: pgmap v499: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:48.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:44:47 smithi083 ceph-mon[77242]: pgmap v499: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:48.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:44:47 smithi153 ceph-mon[70044]: pgmap v499: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:49.732 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T05:44:50.023 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T05:44:50.024 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (18m) 2m ago 19m 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T05:44:50.024 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (18m) 1s ago 18m 61.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T05:44:50.024 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (19m) 2m ago 19m 17.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T05:44:50.024 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (19m) 1s ago 19m 22.5M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T05:44:50.025 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (15m) 1s ago 22m 435M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T05:44:50.025 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (16m) 2m ago 23m 516M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T05:44:50.025 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (15m) 2m ago 24m 128M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T05:44:50.025 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (14m) 1s ago 22m 77.5M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T05:44:50.026 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (14m) 2m ago 22m 88.2M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T05:44:50.026 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (19m) 2m ago 19m 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T05:44:50.026 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (19m) 1s ago 19m 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T05:44:50.026 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (14m) 2m ago 22m 107M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T05:44:50.027 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (13m) 2m ago 21m 99.8M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T05:44:50.027 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (13m) 2m ago 21m 84.6M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T05:44:50.027 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (12m) 2m ago 21m 90.7M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T05:44:50.027 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (12m) 1s ago 20m 89.2M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T05:44:50.028 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (11m) 1s ago 20m 97.0M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T05:44:50.028 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10m) 1s ago 20m 85.6M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T05:44:50.028 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10m) 1s ago 20m 91.5M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T05:44:50.028 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (15m) 1s ago 19m 88.2M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T05:44:50.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:44:49 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:44:50.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:44:49 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:44:50.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:44:49 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:44:50.167 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:44:49 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:44:50.167 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:44:49 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:44:50.167 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:44:49 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:44:50.168 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:44:49 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:44:50.168 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:44:49 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:44:50.168 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:44:49 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:44:50.382 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T05:44:50.382 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T05:44:50.383 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T05:44:50.383 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:44:50.383 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T05:44:50.383 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T05:44:50.383 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:44:50.384 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T05:44:50.384 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T05:44:50.384 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:44:50.384 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T05:44:50.385 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T05:44:50.385 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:44:50.385 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T05:44:50.385 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T05:44:50.386 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T05:44:50.386 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T05:44:50.386 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T05:44:50.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:44:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:44:50.164Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:44:51.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:44:50 smithi083 ceph-mon[79930]: from='client.44653 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:44:51.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:44:50 smithi083 ceph-mon[79930]: pgmap v500: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:51.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:44:50 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/518506586' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:44:51.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:44:50 smithi083 ceph-mon[77242]: from='client.44653 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:44:51.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:44:50 smithi083 ceph-mon[77242]: pgmap v500: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:51.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:44:50 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/518506586' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:44:51.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:44:50 smithi153 ceph-mon[70044]: from='client.44653 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:44:51.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:44:50 smithi153 ceph-mon[70044]: pgmap v500: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:51.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:44:50 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/518506586' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:44:51.830 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:44:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:44:51] "GET /metrics HTTP/1.1" 200 225252 "" "Prometheus/2.33.4" 2022-09-29T05:44:52.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:44:51 smithi083 ceph-mon[79930]: from='client.44659 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:44:52.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:44:51 smithi083 ceph-mon[77242]: from='client.44659 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:44:52.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:44:51 smithi153 ceph-mon[70044]: from='client.44659 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:44:52.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:44:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:44:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:44:53.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:44:52 smithi083 ceph-mon[79930]: pgmap v501: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:53.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:44:52 smithi083 ceph-mon[77242]: pgmap v501: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:53.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:44:52 smithi153 ceph-mon[70044]: pgmap v501: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:53.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:44:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:44:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:44:53.923 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:44:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:44:53.579Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:44:54.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:44:53 smithi083 ceph-mon[79930]: pgmap v502: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:54.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:44:53 smithi083 ceph-mon[77242]: pgmap v502: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:54.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:44:53 smithi153 ceph-mon[70044]: pgmap v502: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:54.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:44:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:44:54.273Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:44:56.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:44:55 smithi083 ceph-mon[77242]: pgmap v503: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:56.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:44:55 smithi083 ceph-mon[79930]: pgmap v503: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:56.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:44:55 smithi153 ceph-mon[70044]: pgmap v503: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:58.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:44:57 smithi083 ceph-mon[79930]: pgmap v504: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:58.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:44:57 smithi083 ceph-mon[77242]: pgmap v504: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:44:58.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:44:57 smithi153 ceph-mon[70044]: pgmap v504: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:45:00.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:45:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:45:00.165Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:45:01.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:45:00 smithi083 ceph-mon[77242]: pgmap v505: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:45:01.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:45:00 smithi083 ceph-mon[79930]: pgmap v505: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:45:01.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:45:00 smithi153 ceph-mon[70044]: pgmap v505: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:45:01.927 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:45:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:45:01] "GET /metrics HTTP/1.1" 200 225253 "" "Prometheus/2.33.4" 2022-09-29T05:45:02.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:45:01 smithi083 ceph-mon[79930]: pgmap v506: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:45:02.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:45:01 smithi083 ceph-mon[77242]: pgmap v506: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:45:02.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:45:01 smithi153 ceph-mon[70044]: pgmap v506: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:45:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:45:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:45:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:45:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:45:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:45:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:45:03.927 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:45:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:45:03.580Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:45:04.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:45:03 smithi083 ceph-mon[77242]: pgmap v507: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:45:04.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:45:03 smithi083 ceph-mon[79930]: pgmap v507: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:45:04.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:45:03 smithi153 ceph-mon[70044]: pgmap v507: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:45:04.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:45:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:45:04.274Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:45:06.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:45:05 smithi083 ceph-mon[79930]: pgmap v508: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:45:06.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:45:05 smithi083 ceph-mon[77242]: pgmap v508: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:45:06.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:45:05 smithi153 ceph-mon[70044]: pgmap v508: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:45:08.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:45:07 smithi083 ceph-mon[79930]: pgmap v509: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:45:08.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:45:07 smithi083 ceph-mon[77242]: pgmap v509: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:45:08.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:45:07 smithi153 ceph-mon[70044]: pgmap v509: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:45:10.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:45:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:45:10.167Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:45:11.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:45:10 smithi083 ceph-mon[79930]: pgmap v510: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:45:11.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:45:10 smithi083 ceph-mon[77242]: pgmap v510: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:45:11.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:45:10 smithi153 ceph-mon[70044]: pgmap v510: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:45:11.926 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:45:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:45:11] "GET /metrics HTTP/1.1" 200 225246 "" "Prometheus/2.33.4" 2022-09-29T05:45:11.927 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:45:11 smithi083 ceph-mon[77242]: pgmap v511: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:45:12.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:45:11 smithi083 ceph-mon[79930]: pgmap v511: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:45:12.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:45:11 smithi153 ceph-mon[70044]: pgmap v511: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:45:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:45:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:45:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:45:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:45:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:45:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:45:13.927 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:45:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:45:13.581Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:45:14.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:45:13 smithi083 ceph-mon[79930]: pgmap v512: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:45:14.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:45:13 smithi083 ceph-mon[77242]: pgmap v512: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:45:14.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:45:13 smithi153 ceph-mon[70044]: pgmap v512: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:45:14.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:45:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:45:14.275Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:45:16.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:45:15 smithi083 ceph-mon[79930]: pgmap v513: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:45:16.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:45:15 smithi083 ceph-mon[77242]: pgmap v513: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:45:16.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:45:15 smithi153 ceph-mon[70044]: pgmap v513: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:45:18.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:45:17 smithi083 ceph-mon[79930]: pgmap v514: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:45:18.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:45:17 smithi083 ceph-mon[77242]: pgmap v514: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:45:18.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:45:17 smithi153 ceph-mon[70044]: pgmap v514: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:45:20.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:45:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:45:20.167Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:45:20.709 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T05:45:21.001 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T05:45:21.001 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (19m) 2m ago 19m 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T05:45:21.002 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (19m) 32s ago 19m 61.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T05:45:21.002 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (20m) 2m ago 20m 17.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T05:45:21.002 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (20m) 32s ago 20m 22.5M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T05:45:21.002 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (16m) 32s ago 22m 435M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T05:45:21.003 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (16m) 2m ago 24m 516M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T05:45:21.003 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (15m) 2m ago 24m 128M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T05:45:21.003 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (15m) 32s ago 23m 77.5M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T05:45:21.003 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (15m) 2m ago 23m 88.2M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T05:45:21.003 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (19m) 2m ago 19m 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T05:45:21.004 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (19m) 32s ago 19m 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T05:45:21.004 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (14m) 2m ago 22m 107M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T05:45:21.004 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (13m) 2m ago 22m 99.8M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T05:45:21.004 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (13m) 2m ago 22m 84.6M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T05:45:21.004 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (13m) 2m ago 21m 90.7M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T05:45:21.005 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (12m) 32s ago 21m 89.2M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T05:45:21.005 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (12m) 32s ago 21m 97.0M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T05:45:21.005 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (11m) 32s ago 20m 85.6M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T05:45:21.005 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10m) 32s ago 20m 91.5M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T05:45:21.005 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (16m) 32s ago 19m 88.2M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T05:45:21.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:45:20 smithi083 ceph-mon[77242]: pgmap v515: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:45:21.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:45:20 smithi083 ceph-mon[79930]: pgmap v515: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:45:21.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:45:20 smithi153 ceph-mon[70044]: pgmap v515: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:45:21.355 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T05:45:21.355 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T05:45:21.356 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T05:45:21.356 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:45:21.356 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T05:45:21.356 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T05:45:21.357 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:45:21.357 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T05:45:21.357 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T05:45:21.357 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:45:21.357 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T05:45:21.358 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T05:45:21.358 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:45:21.358 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T05:45:21.358 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T05:45:21.359 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T05:45:21.359 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T05:45:21.359 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T05:45:21.837 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:45:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:45:21] "GET /metrics HTTP/1.1" 200 225246 "" "Prometheus/2.33.4" 2022-09-29T05:45:22.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:45:21 smithi083 ceph-mon[79930]: from='client.34737 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:45:22.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:45:21 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1674360650' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:45:22.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:45:21 smithi083 ceph-mon[77242]: from='client.34737 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:45:22.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:45:21 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1674360650' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:45:22.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:45:21 smithi153 ceph-mon[70044]: from='client.34737 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:45:22.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:45:21 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1674360650' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:45:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:45:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:45:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:45:23.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:45:22 smithi083 ceph-mon[79930]: from='client.34743 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:45:23.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:45:22 smithi083 ceph-mon[79930]: pgmap v516: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:45:23.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:45:22 smithi083 ceph-mon[77242]: from='client.34743 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:45:23.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:45:22 smithi083 ceph-mon[77242]: pgmap v516: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:45:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:45:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:45:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:45:23.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:45:22 smithi153 ceph-mon[70044]: from='client.34743 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:45:23.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:45:22 smithi153 ceph-mon[70044]: pgmap v516: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:45:23.929 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:45:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:45:23.581Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:45:24.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:45:23 smithi083 ceph-mon[77242]: pgmap v517: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:45:24.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:45:23 smithi083 ceph-mon[79930]: pgmap v517: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:45:24.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:45:23 smithi153 ceph-mon[70044]: pgmap v517: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:45:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:45:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:45:24.276Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:45:26.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:45:25 smithi083 ceph-mon[77242]: pgmap v518: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:45:26.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:45:25 smithi083 ceph-mon[79930]: pgmap v518: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:45:26.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:45:25 smithi153 ceph-mon[70044]: pgmap v518: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:45:28.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:45:27 smithi083 ceph-mon[77242]: pgmap v519: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:45:28.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:45:27 smithi083 ceph-mon[79930]: pgmap v519: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:45:28.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:45:27 smithi153 ceph-mon[70044]: pgmap v519: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:45:30.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:45:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:45:30.168Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:45:31.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:45:30 smithi083 ceph-mon[79930]: pgmap v520: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:45:31.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:45:30 smithi083 ceph-mon[77242]: pgmap v520: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:45:31.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:45:30 smithi153 ceph-mon[70044]: pgmap v520: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:45:31.932 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:45:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:45:31] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T05:45:32.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:45:31 smithi083 ceph-mon[77242]: pgmap v521: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:45:32.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:45:31 smithi083 ceph-mon[79930]: pgmap v521: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:45:32.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:45:31 smithi153 ceph-mon[70044]: pgmap v521: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:45:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:45:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:45:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:45:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:45:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:45:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:45:33.934 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:45:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:45:33.581Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:45:34.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:45:33 smithi083 ceph-mon[77242]: pgmap v522: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:45:34.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:45:33 smithi083 ceph-mon[79930]: pgmap v522: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:45:34.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:45:33 smithi153 ceph-mon[70044]: pgmap v522: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:45:34.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:45:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:45:34.276Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:45:36.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:45:35 smithi083 ceph-mon[77242]: pgmap v523: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:45:36.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:45:35 smithi083 ceph-mon[79930]: pgmap v523: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:45:36.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:45:35 smithi153 ceph-mon[70044]: pgmap v523: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:45:38.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:45:37 smithi083 ceph-mon[79930]: pgmap v524: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:45:38.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:45:37 smithi083 ceph-mon[77242]: pgmap v524: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:45:38.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:45:37 smithi153 ceph-mon[70044]: pgmap v524: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:45:39.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:45:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:45:39.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:45:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:45:39.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:45:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:45:39.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:45:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:45:39.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:45:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:45:39.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:45:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:45:39.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:45:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:45:39.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:45:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:45:39.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:45:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:45:39.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:45:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:45:39.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:45:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:45:39.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:45:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:45:40.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:45:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:45:40.168Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:45:41.835 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:45:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:45:41] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T05:45:42.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:45:41 smithi083 ceph-mon[79930]: pgmap v525: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:45:42.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:45:41 smithi083 ceph-mon[77242]: pgmap v525: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:45:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:45:41 smithi153 ceph-mon[70044]: pgmap v525: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:45:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:45:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:45:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:45:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:45:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:45:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:45:43.838 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:45:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:45:43.582Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:45:44.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:45:43 smithi083 ceph-mon[79930]: pgmap v526: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:45:44.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:45:43 smithi083 ceph-mon[77242]: pgmap v526: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:45:44.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:45:43 smithi153 ceph-mon[70044]: pgmap v526: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:45:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:45:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:45:44.277Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:45:46.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:45:45 smithi083 ceph-mon[79930]: pgmap v527: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:45:46.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:45:45 smithi083 ceph-mon[77242]: pgmap v527: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:45:46.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:45:45 smithi153 ceph-mon[70044]: pgmap v527: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:45:48.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:45:47 smithi083 ceph-mon[79930]: pgmap v528: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:45:48.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:45:47 smithi083 ceph-mon[77242]: pgmap v528: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:45:48.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:45:47 smithi153 ceph-mon[70044]: pgmap v528: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:45:50.170 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:45:49 smithi083 ceph-mon[79930]: pgmap v529: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:45:50.170 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:45:49 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:45:50.171 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:45:49 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:45:50.171 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:45:49 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:45:50.171 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:45:49 smithi083 ceph-mon[77242]: pgmap v529: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:45:50.172 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:45:49 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:45:50.172 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:45:49 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:45:50.172 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:45:49 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:45:50.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:45:49 smithi153 ceph-mon[70044]: pgmap v529: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:45:50.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:45:49 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:45:50.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:45:49 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:45:50.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:45:49 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:45:50.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:45:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:45:50.167Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:45:51.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:45:51 smithi153 ceph-mon[70044]: pgmap v530: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:45:51.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:45:51 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:45:51.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:45:51 smithi083 ceph-mon[79930]: pgmap v530: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:45:51.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:45:51 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:45:51.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:45:51 smithi083 ceph-mon[77242]: pgmap v530: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:45:51.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:45:51 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:45:51.682 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T05:45:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:45:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:45:51] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T05:45:51.970 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T05:45:51.970 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (19m) 3m ago 20m 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T05:45:51.971 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (19m) 63s ago 19m 61.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T05:45:51.971 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (20m) 3m ago 20m 17.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T05:45:51.971 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (20m) 63s ago 20m 22.5M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T05:45:51.971 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (16m) 63s ago 23m 435M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T05:45:51.972 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (17m) 3m ago 25m 516M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T05:45:51.972 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (16m) 3m ago 25m 128M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T05:45:51.972 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (15m) 63s ago 23m 77.5M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T05:45:51.972 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (15m) 3m ago 23m 88.2M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T05:45:51.972 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (20m) 3m ago 20m 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T05:45:51.973 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (20m) 63s ago 20m 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T05:45:51.973 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (15m) 3m ago 23m 107M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T05:45:51.973 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (14m) 3m ago 22m 99.8M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T05:45:51.973 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (14m) 3m ago 22m 84.6M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T05:45:51.973 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (13m) 3m ago 22m 90.7M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T05:45:51.974 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (13m) 63s ago 22m 89.2M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T05:45:51.974 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (12m) 63s ago 21m 97.0M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T05:45:51.974 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (11m) 63s ago 21m 85.6M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T05:45:51.974 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (11m) 63s ago 21m 91.5M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T05:45:51.974 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (16m) 63s ago 20m 88.2M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T05:45:52.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:45:52 smithi153 ceph-mon[70044]: from='client.44689 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:45:52.326 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T05:45:52.326 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T05:45:52.326 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T05:45:52.327 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:45:52.327 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T05:45:52.327 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T05:45:52.327 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:45:52.328 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T05:45:52.328 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T05:45:52.328 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:45:52.328 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T05:45:52.328 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T05:45:52.329 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:45:52.329 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T05:45:52.329 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T05:45:52.329 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T05:45:52.330 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T05:45:52.330 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T05:45:52.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:45:52 smithi083 ceph-mon[77242]: from='client.44689 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:45:52.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:45:52 smithi083 ceph-mon[79930]: from='client.44689 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:45:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:45:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:45:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:45:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:45:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:45:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:45:53.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:45:53 smithi153 ceph-mon[70044]: pgmap v531: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:45:53.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:45:53 smithi153 ceph-mon[70044]: from='client.44695 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:45:53.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:45:53 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2225103334' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:45:53.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:45:53 smithi083 ceph-mon[77242]: pgmap v531: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:45:53.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:45:53 smithi083 ceph-mon[77242]: from='client.44695 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:45:53.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:45:53 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2225103334' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:45:53.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:45:53 smithi083 ceph-mon[79930]: pgmap v531: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:45:53.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:45:53 smithi083 ceph-mon[79930]: from='client.44695 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:45:53.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:45:53 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2225103334' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:45:53.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:45:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:45:53.582Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:45:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:45:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:45:54.277Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:45:56.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:45:55 smithi083 ceph-mon[79930]: pgmap v532: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:45:56.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:45:55 smithi083 ceph-mon[77242]: pgmap v532: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:45:56.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:45:55 smithi153 ceph-mon[70044]: pgmap v532: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:45:58.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:45:57 smithi083 ceph-mon[79930]: pgmap v533: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:45:58.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:45:57 smithi083 ceph-mon[77242]: pgmap v533: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:45:58.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:45:57 smithi153 ceph-mon[70044]: pgmap v533: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:00.171 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:45:59 smithi083 ceph-mon[79930]: pgmap v534: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:00.172 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:45:59 smithi083 ceph-mon[77242]: pgmap v534: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:00.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:45:59 smithi153 ceph-mon[70044]: pgmap v534: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:00.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:46:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:46:00.169Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:46:01.845 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:46:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:46:01] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T05:46:02.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:46:01 smithi083 ceph-mon[77242]: pgmap v535: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:02.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:46:01 smithi083 ceph-mon[79930]: pgmap v535: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:02.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:46:01 smithi153 ceph-mon[70044]: pgmap v535: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:46:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:46:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:46:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:46:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:46:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:46:03.848 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:46:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:46:03.583Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:46:04.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:46:03 smithi083 ceph-mon[79930]: pgmap v536: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:04.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:46:03 smithi083 ceph-mon[77242]: pgmap v536: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:04.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:46:03 smithi153 ceph-mon[70044]: pgmap v536: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:04.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:46:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:46:04.278Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:46:06.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:46:05 smithi083 ceph-mon[77242]: pgmap v537: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:06.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:46:05 smithi083 ceph-mon[79930]: pgmap v537: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:06.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:46:05 smithi153 ceph-mon[70044]: pgmap v537: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:08.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:46:07 smithi083 ceph-mon[77242]: pgmap v538: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:08.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:46:07 smithi083 ceph-mon[79930]: pgmap v538: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:08.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:46:07 smithi153 ceph-mon[70044]: pgmap v538: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:10.172 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:46:09 smithi083 ceph-mon[79930]: pgmap v539: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:10.173 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:46:09 smithi083 ceph-mon[77242]: pgmap v539: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:10.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:46:09 smithi153 ceph-mon[70044]: pgmap v539: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:10.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:46:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:46:10.170Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:46:11.861 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:46:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:46:11] "GET /metrics HTTP/1.1" 200 225217 "" "Prometheus/2.33.4" 2022-09-29T05:46:12.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:46:11 smithi083 ceph-mon[77242]: pgmap v540: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:12.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:46:11 smithi083 ceph-mon[79930]: pgmap v540: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:12.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:46:11 smithi153 ceph-mon[70044]: pgmap v540: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:46:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:46:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:46:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:46:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:46:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:46:13.863 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:46:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:46:13.584Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:46:14.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:46:13 smithi083 ceph-mon[79930]: pgmap v541: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:14.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:46:13 smithi083 ceph-mon[77242]: pgmap v541: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:14.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:46:13 smithi153 ceph-mon[70044]: pgmap v541: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:14.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:46:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:46:14.279Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:46:16.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:46:15 smithi083 ceph-mon[79930]: pgmap v542: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:16.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:46:15 smithi083 ceph-mon[77242]: pgmap v542: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:16.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:46:15 smithi153 ceph-mon[70044]: pgmap v542: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:18.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:46:17 smithi083 ceph-mon[79930]: pgmap v543: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:18.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:46:17 smithi083 ceph-mon[77242]: pgmap v543: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:18.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:46:17 smithi153 ceph-mon[70044]: pgmap v543: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:20.173 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:46:19 smithi083 ceph-mon[79930]: pgmap v544: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:20.173 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:46:19 smithi083 ceph-mon[77242]: pgmap v544: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:20.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:46:19 smithi153 ceph-mon[70044]: pgmap v544: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:20.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:46:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:46:20.170Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:46:21.874 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:46:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:46:21] "GET /metrics HTTP/1.1" 200 225217 "" "Prometheus/2.33.4" 2022-09-29T05:46:22.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:46:21 smithi083 ceph-mon[79930]: pgmap v545: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:22.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:46:21 smithi083 ceph-mon[77242]: pgmap v545: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:22.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:46:21 smithi153 ceph-mon[70044]: pgmap v545: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:22.656 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T05:46:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:46:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:46:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:46:22.943 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T05:46:22.943 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (20m) 3m ago 20m 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T05:46:22.943 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (20m) 94s ago 20m 61.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T05:46:22.944 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (21m) 3m ago 21m 17.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T05:46:22.944 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (21m) 94s ago 21m 22.5M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T05:46:22.944 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (17m) 94s ago 23m 435M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T05:46:22.944 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (17m) 3m ago 25m 516M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T05:46:22.944 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (16m) 3m ago 25m 128M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T05:46:22.945 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (16m) 94s ago 24m 77.5M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T05:46:22.945 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (16m) 3m ago 24m 88.2M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T05:46:22.945 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (20m) 3m ago 20m 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T05:46:22.945 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (20m) 94s ago 20m 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T05:46:22.945 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (15m) 3m ago 23m 107M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T05:46:22.946 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (15m) 3m ago 23m 99.8M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T05:46:22.946 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (14m) 3m ago 23m 84.6M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T05:46:22.947 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (14m) 3m ago 22m 90.7M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T05:46:22.947 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (13m) 94s ago 22m 89.2M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T05:46:22.947 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (13m) 94s ago 22m 97.0M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T05:46:22.947 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (12m) 94s ago 21m 85.6M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T05:46:22.948 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (11m) 94s ago 21m 91.5M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T05:46:22.948 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (17m) 94s ago 20m 88.2M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T05:46:23.297 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T05:46:23.297 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T05:46:23.297 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T05:46:23.298 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:46:23.298 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T05:46:23.298 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T05:46:23.298 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:46:23.299 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T05:46:23.299 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T05:46:23.299 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:46:23.299 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T05:46:23.300 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T05:46:23.300 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:46:23.300 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T05:46:23.300 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T05:46:23.300 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T05:46:23.301 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T05:46:23.301 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T05:46:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:46:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:46:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:46:23.876 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:46:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:46:23.584Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:46:24.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:46:23 smithi083 ceph-mon[79930]: pgmap v546: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:24.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:46:23 smithi083 ceph-mon[79930]: from='client.34773 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:46:24.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:46:23 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3997131510' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:46:24.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:46:23 smithi083 ceph-mon[77242]: pgmap v546: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:24.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:46:23 smithi083 ceph-mon[77242]: from='client.34773 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:46:24.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:46:23 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3997131510' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:46:24.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:46:23 smithi153 ceph-mon[70044]: pgmap v546: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:24.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:46:23 smithi153 ceph-mon[70044]: from='client.34773 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:46:24.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:46:23 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3997131510' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:46:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:46:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:46:24.279Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:46:25.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:46:24 smithi083 ceph-mon[79930]: from='client.44713 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:46:25.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:46:24 smithi083 ceph-mon[77242]: from='client.44713 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:46:25.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:46:24 smithi153 ceph-mon[70044]: from='client.44713 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:46:26.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:46:25 smithi083 ceph-mon[79930]: pgmap v547: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:26.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:46:25 smithi083 ceph-mon[77242]: pgmap v547: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:26.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:46:25 smithi153 ceph-mon[70044]: pgmap v547: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:28.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:46:27 smithi083 ceph-mon[79930]: pgmap v548: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:28.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:46:27 smithi083 ceph-mon[77242]: pgmap v548: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:28.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:46:27 smithi153 ceph-mon[70044]: pgmap v548: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:30.174 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:46:29 smithi083 ceph-mon[79930]: pgmap v549: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:30.174 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:46:29 smithi083 ceph-mon[77242]: pgmap v549: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:30.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:46:29 smithi153 ceph-mon[70044]: pgmap v549: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:30.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:46:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:46:30.171Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:46:31.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:46:30 smithi083 ceph-mon[79930]: pgmap v550: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:31.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:46:30 smithi083 ceph-mon[77242]: pgmap v550: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:31.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:46:30 smithi153 ceph-mon[70044]: pgmap v550: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:46:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:46:31] "GET /metrics HTTP/1.1" 200 225218 "" "Prometheus/2.33.4" 2022-09-29T05:46:32.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:46:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:46:32.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:46:33.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:46:32 smithi083 ceph-mon[77242]: pgmap v551: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:33.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:46:32 smithi083 ceph-mon[79930]: pgmap v551: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:33.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:46:32 smithi153 ceph-mon[70044]: pgmap v551: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:33.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:46:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:46:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:46:33.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:46:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:46:33.585Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:46:34.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:46:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:46:34.281Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:46:36.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:46:35 smithi083 ceph-mon[79930]: pgmap v552: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:36.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:46:35 smithi083 ceph-mon[77242]: pgmap v552: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:36.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:46:35 smithi153 ceph-mon[70044]: pgmap v552: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:38.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:46:37 smithi083 ceph-mon[79930]: pgmap v553: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:38.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:46:37 smithi083 ceph-mon[77242]: pgmap v553: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:38.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:46:37 smithi153 ceph-mon[70044]: pgmap v553: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:40.175 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:46:39 smithi083 ceph-mon[79930]: pgmap v554: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:40.175 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:46:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:46:40.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:46:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:46:40.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:46:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:46:40.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:46:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:46:40.177 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:46:39 smithi083 ceph-mon[77242]: pgmap v554: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:40.177 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:46:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:46:40.177 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:46:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:46:40.177 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:46:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:46:40.177 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:46:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:46:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:46:39 smithi153 ceph-mon[70044]: pgmap v554: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:46:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:46:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:46:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:46:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:46:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:46:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:46:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:46:40.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:46:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:46:40.172Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:46:41.855 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:46:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:46:41] "GET /metrics HTTP/1.1" 200 225170 "" "Prometheus/2.33.4" 2022-09-29T05:46:42.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:46:41 smithi083 ceph-mon[79930]: pgmap v555: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:42.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:46:41 smithi083 ceph-mon[77242]: pgmap v555: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:46:41 smithi153 ceph-mon[70044]: pgmap v555: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:46:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:46:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:46:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:46:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:46:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:46:43.858 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:46:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:46:43.587Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:46:44.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:46:43 smithi083 ceph-mon[77242]: pgmap v556: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:44.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:46:43 smithi083 ceph-mon[79930]: pgmap v556: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:44.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:46:43 smithi153 ceph-mon[70044]: pgmap v556: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:44.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:46:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:46:44.281Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:46:46.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:46:45 smithi083 ceph-mon[77242]: pgmap v557: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:46.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:46:45 smithi083 ceph-mon[79930]: pgmap v557: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:46.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:46:45 smithi153 ceph-mon[70044]: pgmap v557: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:48.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:46:47 smithi083 ceph-mon[77242]: pgmap v558: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:48.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:46:47 smithi083 ceph-mon[79930]: pgmap v558: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:48.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:46:47 smithi153 ceph-mon[70044]: pgmap v558: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:50.175 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:46:49 smithi083 ceph-mon[79930]: pgmap v559: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:50.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:46:49 smithi083 ceph-mon[77242]: pgmap v559: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:50.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:46:49 smithi153 ceph-mon[70044]: pgmap v559: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:50.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:46:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:46:50.172Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:46:51.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:46:50 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:46:51.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:46:50 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:46:51.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:46:50 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:46:51.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:46:50 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:46:51.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:46:50 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:46:51.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:46:50 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:46:51.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:46:50 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:46:51.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:46:50 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:46:51.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:46:50 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:46:51.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:46:50 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:46:51.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:46:50 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:46:51.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:46:50 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:46:51.880 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:46:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:46:51] "GET /metrics HTTP/1.1" 200 225170 "" "Prometheus/2.33.4" 2022-09-29T05:46:52.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:46:51 smithi083 ceph-mon[79930]: pgmap v560: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:52.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:46:51 smithi083 ceph-mon[77242]: pgmap v560: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:52.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:46:51 smithi153 ceph-mon[70044]: pgmap v560: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:46:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:46:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:46:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:46:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:46:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:46:53.627 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T05:46:53.883 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:46:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:46:53.587Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:46:53.883 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:46:53 smithi083 ceph-mon[77242]: pgmap v561: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:53.920 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T05:46:53.920 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (20m) 4m ago 21m 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T05:46:53.921 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (20m) 2m ago 20m 61.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T05:46:53.921 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (21m) 4m ago 21m 17.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T05:46:53.921 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (21m) 2m ago 21m 22.5M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T05:46:53.921 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (17m) 2m ago 24m 435M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T05:46:53.921 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (18m) 4m ago 26m 516M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T05:46:53.922 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (17m) 4m ago 26m 128M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T05:46:53.922 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (16m) 2m ago 24m 77.5M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T05:46:53.922 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (17m) 4m ago 24m 88.2M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T05:46:53.922 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (21m) 4m ago 21m 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T05:46:53.922 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (21m) 2m ago 21m 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T05:46:53.923 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (16m) 4m ago 24m 107M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T05:46:53.923 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (15m) 4m ago 23m 99.8M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T05:46:53.923 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (15m) 4m ago 23m 84.6M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T05:46:53.923 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (14m) 4m ago 23m 90.7M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T05:46:53.924 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (14m) 2m ago 23m 89.2M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T05:46:53.924 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (13m) 2m ago 22m 97.0M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T05:46:53.924 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (13m) 2m ago 22m 85.6M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T05:46:53.924 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (12m) 2m ago 22m 91.5M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T05:46:53.924 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (17m) 2m ago 21m 88.2M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T05:46:54.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:46:53 smithi083 ceph-mon[79930]: pgmap v561: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:54.277 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T05:46:54.277 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T05:46:54.277 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T05:46:54.277 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:46:54.278 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T05:46:54.278 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T05:46:54.278 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:46:54.278 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T05:46:54.279 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T05:46:54.279 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:46:54.279 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T05:46:54.279 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T05:46:54.280 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:46:54.280 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T05:46:54.280 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T05:46:54.280 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T05:46:54.281 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T05:46:54.281 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T05:46:54.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:46:53 smithi153 ceph-mon[70044]: pgmap v561: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:46:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:46:54.281Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:46:55.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:46:54 smithi083 ceph-mon[79930]: from='client.44725 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:46:55.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:46:54 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4092917775' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:46:55.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:46:54 smithi083 ceph-mon[77242]: from='client.44725 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:46:55.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:46:54 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4092917775' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:46:55.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:46:54 smithi153 ceph-mon[70044]: from='client.44725 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:46:55.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:46:54 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4092917775' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:46:56.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:46:55 smithi083 ceph-mon[79930]: pgmap v562: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:56.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:46:55 smithi083 ceph-mon[79930]: from='client.44731 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:46:56.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:46:55 smithi083 ceph-mon[77242]: pgmap v562: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:56.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:46:55 smithi083 ceph-mon[77242]: from='client.44731 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:46:56.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:46:55 smithi153 ceph-mon[70044]: pgmap v562: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:56.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:46:55 smithi153 ceph-mon[70044]: from='client.44731 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:46:57.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:46:56 smithi083 ceph-mon[79930]: pgmap v563: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:57.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:46:56 smithi083 ceph-mon[77242]: pgmap v563: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:57.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:46:56 smithi153 ceph-mon[70044]: pgmap v563: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:59.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:46:58 smithi083 ceph-mon[79930]: pgmap v564: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:59.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:46:58 smithi083 ceph-mon[77242]: pgmap v564: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:46:59.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:46:58 smithi153 ceph-mon[70044]: pgmap v564: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:00.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:47:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:47:00.174Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:47:01.852 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:47:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:47:01] "GET /metrics HTTP/1.1" 200 225157 "" "Prometheus/2.33.4" 2022-09-29T05:47:02.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:47:01 smithi083 ceph-mon[77242]: pgmap v565: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:02.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:47:01 smithi083 ceph-mon[79930]: pgmap v565: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:02.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:47:01 smithi153 ceph-mon[70044]: pgmap v565: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:47:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:47:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:47:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:47:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:47:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:47:03.855 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:47:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:47:03.588Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:47:04.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:47:03 smithi083 ceph-mon[79930]: pgmap v566: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:04.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:47:03 smithi083 ceph-mon[77242]: pgmap v566: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:04.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:47:03 smithi153 ceph-mon[70044]: pgmap v566: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:04.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:47:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:47:04.283Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:47:06.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:47:05 smithi083 ceph-mon[79930]: pgmap v567: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:06.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:47:05 smithi083 ceph-mon[77242]: pgmap v567: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:06.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:47:05 smithi153 ceph-mon[70044]: pgmap v567: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:08.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:47:07 smithi083 ceph-mon[79930]: pgmap v568: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:08.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:47:07 smithi083 ceph-mon[77242]: pgmap v568: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:08.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:47:07 smithi153 ceph-mon[70044]: pgmap v568: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:10.178 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:47:09 smithi083 ceph-mon[79930]: pgmap v569: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:10.178 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:47:09 smithi083 ceph-mon[77242]: pgmap v569: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:10.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:47:09 smithi153 ceph-mon[70044]: pgmap v569: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:10.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:47:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:47:10.175Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:47:11.870 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:47:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:47:11] "GET /metrics HTTP/1.1" 200 225156 "" "Prometheus/2.33.4" 2022-09-29T05:47:12.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:47:11 smithi083 ceph-mon[79930]: pgmap v570: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:12.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:47:11 smithi083 ceph-mon[77242]: pgmap v570: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:12.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:47:11 smithi153 ceph-mon[70044]: pgmap v570: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:47:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:47:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:47:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:47:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:47:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:47:13.873 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:47:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:47:13.588Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:47:14.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:47:13 smithi083 ceph-mon[79930]: pgmap v571: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:14.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:47:13 smithi083 ceph-mon[77242]: pgmap v571: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:14.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:47:13 smithi153 ceph-mon[70044]: pgmap v571: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:14.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:47:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:47:14.283Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:47:16.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:47:15 smithi083 ceph-mon[79930]: pgmap v572: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:16.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:47:15 smithi083 ceph-mon[77242]: pgmap v572: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:16.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:47:15 smithi153 ceph-mon[70044]: pgmap v572: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:18.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:47:17 smithi083 ceph-mon[79930]: pgmap v573: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:18.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:47:17 smithi083 ceph-mon[77242]: pgmap v573: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:18.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:47:17 smithi153 ceph-mon[70044]: pgmap v573: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:20.179 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:47:19 smithi083 ceph-mon[79930]: pgmap v574: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:20.179 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:47:19 smithi083 ceph-mon[77242]: pgmap v574: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:20.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:47:19 smithi153 ceph-mon[70044]: pgmap v574: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:20.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:47:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:47:20.176Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:47:21.886 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:47:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:47:21] "GET /metrics HTTP/1.1" 200 225156 "" "Prometheus/2.33.4" 2022-09-29T05:47:22.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:47:21 smithi083 ceph-mon[79930]: pgmap v575: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:22.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:47:21 smithi083 ceph-mon[77242]: pgmap v575: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:22.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:47:21 smithi153 ceph-mon[70044]: pgmap v575: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:47:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:47:22.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:47:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:47:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:47:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:47:23.889 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:47:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:47:23.588Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:47:24.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:47:23 smithi083 ceph-mon[79930]: pgmap v576: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:24.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:47:23 smithi083 ceph-mon[77242]: pgmap v576: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:24.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:47:23 smithi153 ceph-mon[70044]: pgmap v576: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:24.602 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T05:47:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:47:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:47:24.284Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:47:24.896 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T05:47:24.896 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (21m) 4m ago 21m 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T05:47:24.897 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (21m) 2m ago 21m 61.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T05:47:24.897 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (22m) 4m ago 22m 17.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T05:47:24.897 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (22m) 2m ago 22m 22.5M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T05:47:24.897 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (18m) 2m ago 25m 435M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T05:47:24.898 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (18m) 4m ago 26m 516M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T05:47:24.898 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (17m) 4m ago 26m 128M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T05:47:24.898 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (17m) 2m ago 25m 77.5M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T05:47:24.898 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (17m) 4m ago 25m 88.2M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T05:47:24.899 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (21m) 4m ago 21m 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T05:47:24.899 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (21m) 2m ago 21m 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T05:47:24.899 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (16m) 4m ago 24m 107M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T05:47:24.899 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (16m) 4m ago 24m 99.8M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T05:47:24.900 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (15m) 4m ago 24m 84.6M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T05:47:24.900 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (15m) 4m ago 23m 90.7M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T05:47:24.900 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (14m) 2m ago 23m 89.2M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T05:47:24.900 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (14m) 2m ago 23m 97.0M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T05:47:24.901 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (13m) 2m ago 23m 85.6M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T05:47:24.901 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (13m) 2m ago 22m 91.5M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T05:47:24.901 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (18m) 2m ago 21m 88.2M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T05:47:25.254 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T05:47:25.255 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T05:47:25.255 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T05:47:25.255 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:47:25.255 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T05:47:25.256 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T05:47:25.256 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:47:25.256 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T05:47:25.256 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T05:47:25.257 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:47:25.257 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T05:47:25.257 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T05:47:25.257 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:47:25.257 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T05:47:25.258 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T05:47:25.258 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T05:47:25.258 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T05:47:25.259 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T05:47:26.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:47:25 smithi083 ceph-mon[79930]: pgmap v577: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:26.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:47:25 smithi083 ceph-mon[79930]: from='client.34809 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:47:26.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:47:25 smithi083 ceph-mon[79930]: from='client.34815 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:47:26.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:47:25 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/988940301' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:47:26.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:47:25 smithi083 ceph-mon[77242]: pgmap v577: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:26.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:47:25 smithi083 ceph-mon[77242]: from='client.34809 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:47:26.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:47:25 smithi083 ceph-mon[77242]: from='client.34815 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:47:26.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:47:25 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/988940301' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:47:26.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:47:25 smithi153 ceph-mon[70044]: pgmap v577: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:26.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:47:25 smithi153 ceph-mon[70044]: from='client.34809 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:47:26.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:47:25 smithi153 ceph-mon[70044]: from='client.34815 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:47:26.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:47:25 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/988940301' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:47:28.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:47:27 smithi083 ceph-mon[79930]: pgmap v578: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:28.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:47:27 smithi083 ceph-mon[77242]: pgmap v578: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:28.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:47:27 smithi153 ceph-mon[70044]: pgmap v578: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:30.179 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:47:29 smithi083 ceph-mon[79930]: pgmap v579: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:30.180 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:47:29 smithi083 ceph-mon[77242]: pgmap v579: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:30.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:47:29 smithi153 ceph-mon[70044]: pgmap v579: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:30.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:47:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:47:30.177Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:47:31.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:47:30 smithi083 ceph-mon[79930]: pgmap v580: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:31.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:47:30 smithi083 ceph-mon[77242]: pgmap v580: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:31.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:47:30 smithi153 ceph-mon[70044]: pgmap v580: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:47:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:47:31] "GET /metrics HTTP/1.1" 200 225150 "" "Prometheus/2.33.4" 2022-09-29T05:47:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:47:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:47:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:47:33.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:47:32 smithi083 ceph-mon[79930]: pgmap v581: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:33.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:47:32 smithi083 ceph-mon[77242]: pgmap v581: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:33.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:47:32 smithi153 ceph-mon[70044]: pgmap v581: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:33.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:47:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:47:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:47:33.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:47:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:47:33.589Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:47:34.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:47:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:47:34.284Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:47:36.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:47:35 smithi083 ceph-mon[79930]: pgmap v582: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:36.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:47:35 smithi083 ceph-mon[77242]: pgmap v582: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:36.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:47:35 smithi153 ceph-mon[70044]: pgmap v582: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:38.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:47:37 smithi083 ceph-mon[79930]: pgmap v583: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:38.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:47:37 smithi083 ceph-mon[77242]: pgmap v583: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:38.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:47:37 smithi153 ceph-mon[70044]: pgmap v583: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:40.179 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:47:39 smithi083 ceph-mon[79930]: pgmap v584: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:40.180 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:47:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:47:40.180 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:47:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:47:40.180 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:47:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:47:40.181 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:47:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:47:40.181 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:47:39 smithi083 ceph-mon[77242]: pgmap v584: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:40.181 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:47:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:47:40.182 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:47:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:47:40.182 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:47:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:47:40.183 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:47:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:47:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:47:39 smithi153 ceph-mon[70044]: pgmap v584: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:47:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:47:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:47:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:47:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:47:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:47:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:47:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:47:40.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:47:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:47:40.177Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:47:41.877 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:47:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:47:41] "GET /metrics HTTP/1.1" 200 225152 "" "Prometheus/2.33.4" 2022-09-29T05:47:42.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:47:41 smithi083 ceph-mon[79930]: pgmap v585: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:42.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:47:41 smithi083 ceph-mon[77242]: pgmap v585: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:47:41 smithi153 ceph-mon[70044]: pgmap v585: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:47:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:47:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:47:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:47:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:47:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:47:43.880 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:47:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:47:43.590Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:47:44.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:47:43 smithi083 ceph-mon[77242]: pgmap v586: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:44.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:47:43 smithi083 ceph-mon[79930]: pgmap v586: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:44.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:47:43 smithi153 ceph-mon[70044]: pgmap v586: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:47:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:47:44.285Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:47:46.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:47:45 smithi083 ceph-mon[79930]: pgmap v587: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:46.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:47:45 smithi083 ceph-mon[77242]: pgmap v587: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:46.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:47:45 smithi153 ceph-mon[70044]: pgmap v587: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:48.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:47:47 smithi083 ceph-mon[79930]: pgmap v588: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:48.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:47:47 smithi083 ceph-mon[77242]: pgmap v588: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:48.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:47:47 smithi153 ceph-mon[70044]: pgmap v588: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:50.181 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:47:49 smithi083 ceph-mon[79930]: pgmap v589: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:50.181 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:47:49 smithi083 ceph-mon[77242]: pgmap v589: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:50.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:47:49 smithi153 ceph-mon[70044]: pgmap v589: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:50.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:47:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:47:50.178Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:47:51.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:47:50 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:47:51.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:47:50 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:47:51.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:47:50 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:47:51.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:47:50 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:47:51.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:47:50 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:47:51.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:47:50 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:47:51.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:47:50 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:47:51.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:47:50 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:47:51.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:47:50 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:47:51.895 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:47:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:47:51] "GET /metrics HTTP/1.1" 200 225152 "" "Prometheus/2.33.4" 2022-09-29T05:47:52.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:47:51 smithi083 ceph-mon[77242]: pgmap v590: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:52.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:47:51 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:47:52.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:47:51 smithi083 ceph-mon[79930]: pgmap v590: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:52.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:47:51 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:47:52.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:47:51 smithi153 ceph-mon[70044]: pgmap v590: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:52.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:47:51 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:47:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:47:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:47:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:47:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:47:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:47:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:47:53.898 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:47:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:47:53.590Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:47:54.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:47:53 smithi083 ceph-mon[79930]: pgmap v591: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:54.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:47:53 smithi083 ceph-mon[77242]: pgmap v591: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:54.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:47:53 smithi153 ceph-mon[70044]: pgmap v591: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:47:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:47:54.285Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:47:55.580 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T05:47:55.869 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T05:47:55.870 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (21m) 5m ago 22m 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T05:47:55.870 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (21m) 3m ago 21m 61.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T05:47:55.870 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (22m) 5m ago 22m 17.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T05:47:55.870 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (22m) 3m ago 22m 22.5M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T05:47:55.871 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (18m) 3m ago 25m 435M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T05:47:55.871 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (19m) 5m ago 27m 516M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T05:47:55.871 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (18m) 5m ago 27m 128M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T05:47:55.871 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (17m) 3m ago 25m 77.5M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T05:47:55.872 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (18m) 5m ago 25m 88.2M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T05:47:55.872 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (22m) 5m ago 22m 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T05:47:55.872 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (22m) 3m ago 22m 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T05:47:55.872 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (17m) 5m ago 25m 107M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T05:47:55.873 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (16m) 5m ago 24m 99.8M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T05:47:55.873 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (16m) 5m ago 24m 84.6M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T05:47:55.873 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (15m) 5m ago 24m 90.7M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T05:47:55.874 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (15m) 3m ago 24m 89.2M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T05:47:55.874 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (14m) 3m ago 23m 97.0M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T05:47:55.874 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (14m) 3m ago 23m 85.6M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T05:47:55.874 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (13m) 3m ago 23m 91.5M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T05:47:55.875 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (18m) 3m ago 22m 88.2M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T05:47:56.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:47:55 smithi083 ceph-mon[79930]: pgmap v592: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:56.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:47:55 smithi083 ceph-mon[77242]: pgmap v592: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:56.228 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T05:47:56.228 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T05:47:56.228 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T05:47:56.229 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:47:56.229 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T05:47:56.229 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T05:47:56.229 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:47:56.229 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T05:47:56.230 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T05:47:56.230 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:47:56.230 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T05:47:56.230 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T05:47:56.231 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:47:56.231 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T05:47:56.231 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T05:47:56.231 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T05:47:56.232 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T05:47:56.232 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T05:47:56.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:47:55 smithi153 ceph-mon[70044]: pgmap v592: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:57.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:47:56 smithi083 ceph-mon[79930]: from='client.44761 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:47:57.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:47:56 smithi083 ceph-mon[79930]: from='client.44767 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:47:57.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:47:56 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/83407663' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:47:57.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:47:56 smithi083 ceph-mon[77242]: from='client.44761 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:47:57.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:47:56 smithi083 ceph-mon[77242]: from='client.44767 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:47:57.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:47:56 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/83407663' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:47:57.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:47:56 smithi153 ceph-mon[70044]: from='client.44761 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:47:57.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:47:56 smithi153 ceph-mon[70044]: from='client.44767 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:47:57.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:47:56 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/83407663' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:47:58.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:47:57 smithi083 ceph-mon[79930]: pgmap v593: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:58.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:47:57 smithi083 ceph-mon[77242]: pgmap v593: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:58.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:47:57 smithi153 ceph-mon[70044]: pgmap v593: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:59.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:47:58 smithi083 ceph-mon[79930]: pgmap v594: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:59.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:47:58 smithi083 ceph-mon[77242]: pgmap v594: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:47:59.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:47:58 smithi153 ceph-mon[70044]: pgmap v594: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:48:00.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:48:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:48:00.178Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:48:01.871 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:48:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:48:01] "GET /metrics HTTP/1.1" 200 225154 "" "Prometheus/2.33.4" 2022-09-29T05:48:02.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:48:01 smithi083 ceph-mon[77242]: pgmap v595: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:48:02.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:48:01 smithi083 ceph-mon[79930]: pgmap v595: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:48:02.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:48:01 smithi153 ceph-mon[70044]: pgmap v595: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:48:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:48:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:48:02.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:48:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:48:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:48:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:48:03.873 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:48:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:48:03.591Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:48:04.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:48:03 smithi083 ceph-mon[79930]: pgmap v596: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:48:04.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:48:03 smithi083 ceph-mon[77242]: pgmap v596: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:48:04.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:48:03 smithi153 ceph-mon[70044]: pgmap v596: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:48:04.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:48:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:48:04.286Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:48:06.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:48:05 smithi083 ceph-mon[77242]: pgmap v597: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:48:06.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:48:05 smithi083 ceph-mon[79930]: pgmap v597: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:48:06.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:48:05 smithi153 ceph-mon[70044]: pgmap v597: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:48:08.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:48:07 smithi083 ceph-mon[79930]: pgmap v598: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:48:08.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:48:07 smithi083 ceph-mon[77242]: pgmap v598: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:48:08.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:48:07 smithi153 ceph-mon[70044]: pgmap v598: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:48:10.181 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:48:09 smithi083 ceph-mon[79930]: pgmap v599: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:48:10.181 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:48:09 smithi083 ceph-mon[77242]: pgmap v599: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:48:10.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:48:09 smithi153 ceph-mon[70044]: pgmap v599: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:48:10.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:48:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:48:10.178Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:48:11.888 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:48:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:48:11] "GET /metrics HTTP/1.1" 200 225151 "" "Prometheus/2.33.4" 2022-09-29T05:48:12.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:48:11 smithi083 ceph-mon[77242]: pgmap v600: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:48:12.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:48:11 smithi083 ceph-mon[79930]: pgmap v600: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:48:12.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:48:11 smithi153 ceph-mon[70044]: pgmap v600: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:48:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:48:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:48:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:48:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:48:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:48:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:48:13.890 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:48:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:48:13.592Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:48:13.891 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:48:13 smithi083 ceph-mon[77242]: pgmap v601: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:48:14.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:48:13 smithi083 ceph-mon[79930]: pgmap v601: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:48:14.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:48:13 smithi153 ceph-mon[70044]: pgmap v601: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:48:14.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:48:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:48:14.286Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:48:16.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:48:15 smithi083 ceph-mon[79930]: pgmap v602: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:48:16.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:48:15 smithi083 ceph-mon[77242]: pgmap v602: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:48:16.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:48:15 smithi153 ceph-mon[70044]: pgmap v602: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:48:18.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:48:17 smithi083 ceph-mon[77242]: pgmap v603: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:48:18.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:48:17 smithi083 ceph-mon[79930]: pgmap v603: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:48:18.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:48:17 smithi153 ceph-mon[70044]: pgmap v603: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:48:20.183 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:48:19 smithi083 ceph-mon[79930]: pgmap v604: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:48:20.183 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:48:19 smithi083 ceph-mon[77242]: pgmap v604: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:48:20.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:48:19 smithi153 ceph-mon[70044]: pgmap v604: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:48:20.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:48:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:48:20.180Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:48:21.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:48:20 smithi083 ceph-mon[79930]: pgmap v605: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:48:21.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:48:20 smithi083 ceph-mon[77242]: pgmap v605: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:48:21.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:48:20 smithi153 ceph-mon[70044]: pgmap v605: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:48:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:48:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:48:21] "GET /metrics HTTP/1.1" 200 225151 "" "Prometheus/2.33.4" 2022-09-29T05:48:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:48:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:48:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:48:23.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:48:22 smithi153 ceph-mon[70044]: pgmap v606: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:48:23.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:48:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:48:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:48:23.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:48:22 smithi083 ceph-mon[79930]: pgmap v606: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:48:23.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:48:22 smithi083 ceph-mon[77242]: pgmap v606: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:48:23.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:48:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:48:23.592Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:48:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:48:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:48:24.286Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:48:26.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:48:25 smithi083 ceph-mon[79930]: pgmap v607: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:48:26.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:48:25 smithi083 ceph-mon[77242]: pgmap v607: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:48:26.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:48:25 smithi153 ceph-mon[70044]: pgmap v607: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:48:26.556 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T05:48:26.848 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T05:48:26.848 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (22m) 5m ago 22m 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T05:48:26.849 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (22m) 3m ago 22m 61.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T05:48:26.849 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (23m) 5m ago 23m 17.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T05:48:26.849 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (23m) 3m ago 23m 22.5M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T05:48:26.849 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (19m) 3m ago 26m 435M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T05:48:26.850 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (19m) 5m ago 27m 516M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T05:48:26.850 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (18m) 5m ago 27m 128M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T05:48:26.850 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (18m) 3m ago 26m 77.5M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T05:48:26.850 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (18m) 5m ago 26m 88.2M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T05:48:26.851 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (22m) 5m ago 22m 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T05:48:26.851 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (22m) 3m ago 22m 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T05:48:26.851 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (17m) 5m ago 25m 107M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T05:48:26.851 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (17m) 5m ago 25m 99.8M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T05:48:26.851 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (16m) 5m ago 25m 84.6M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T05:48:26.852 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (16m) 5m ago 24m 90.7M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T05:48:26.852 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (15m) 3m ago 24m 89.2M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T05:48:26.852 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (15m) 3m ago 24m 97.0M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T05:48:26.852 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (14m) 3m ago 24m 85.6M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T05:48:26.853 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (14m) 3m ago 23m 91.5M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T05:48:26.853 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (19m) 3m ago 22m 88.2M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T05:48:27.204 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T05:48:27.204 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T05:48:27.204 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T05:48:27.205 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:48:27.205 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T05:48:27.205 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T05:48:27.205 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:48:27.206 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T05:48:27.206 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T05:48:27.206 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:48:27.206 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T05:48:27.206 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T05:48:27.207 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:48:27.207 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T05:48:27.207 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T05:48:27.207 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T05:48:27.208 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T05:48:27.208 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T05:48:28.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:48:27 smithi083 ceph-mon[79930]: pgmap v608: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:48:28.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:48:27 smithi083 ceph-mon[79930]: from='client.44779 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:48:28.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:48:27 smithi083 ceph-mon[79930]: from='client.44785 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:48:28.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:48:27 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2452445641' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:48:28.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:48:27 smithi083 ceph-mon[77242]: pgmap v608: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:48:28.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:48:27 smithi083 ceph-mon[77242]: from='client.44779 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:48:28.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:48:27 smithi083 ceph-mon[77242]: from='client.44785 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:48:28.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:48:27 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2452445641' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:48:28.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:48:27 smithi153 ceph-mon[70044]: pgmap v608: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:48:28.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:48:27 smithi153 ceph-mon[70044]: from='client.44779 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:48:28.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:48:27 smithi153 ceph-mon[70044]: from='client.44785 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:48:28.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:48:27 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2452445641' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:48:30.183 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:48:29 smithi083 ceph-mon[79930]: pgmap v609: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:48:30.183 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:48:29 smithi083 ceph-mon[77242]: pgmap v609: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:48:30.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:48:29 smithi153 ceph-mon[70044]: pgmap v609: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:48:30.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:48:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:48:30.180Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:48:31.882 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:48:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:48:31] "GET /metrics HTTP/1.1" 200 225157 "" "Prometheus/2.33.4" 2022-09-29T05:48:32.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:48:31 smithi083 ceph-mon[77242]: pgmap v610: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:48:32.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:48:31 smithi083 ceph-mon[79930]: pgmap v610: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:48:32.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:48:31 smithi153 ceph-mon[70044]: pgmap v610: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:48:32.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:48:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:48:32.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:48:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:48:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:48:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:48:33.885 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:48:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:48:33.592Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:48:34.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:48:33 smithi083 ceph-mon[77242]: pgmap v611: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:48:34.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:48:33 smithi083 ceph-mon[79930]: pgmap v611: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:48:34.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:48:33 smithi153 ceph-mon[70044]: pgmap v611: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:48:34.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:48:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:48:34.287Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:48:36.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:48:35 smithi083 ceph-mon[77242]: pgmap v612: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:48:36.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:48:35 smithi083 ceph-mon[79930]: pgmap v612: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:48:36.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:48:35 smithi153 ceph-mon[70044]: pgmap v612: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:48:38.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:48:37 smithi083 ceph-mon[79930]: pgmap v613: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:48:38.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:48:37 smithi083 ceph-mon[77242]: pgmap v613: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:48:38.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:48:37 smithi153 ceph-mon[70044]: pgmap v613: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:48:40.184 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:48:39 smithi083 ceph-mon[79930]: pgmap v614: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:48:40.185 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:48:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:48:40.185 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:48:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:48:40.185 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:48:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:48:40.185 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:48:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:48:40.186 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:48:39 smithi083 ceph-mon[77242]: pgmap v614: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:48:40.186 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:48:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:48:40.186 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:48:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:48:40.187 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:48:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:48:40.187 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:48:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:48:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:48:39 smithi153 ceph-mon[70044]: pgmap v614: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:48:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:48:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:48:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:48:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:48:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:48:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:48:40.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:48:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:48:40.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:48:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:48:40.182Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:48:41.899 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:48:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:48:41] "GET /metrics HTTP/1.1" 200 225157 "" "Prometheus/2.33.4" 2022-09-29T05:48:42.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:48:41 smithi083 ceph-mon[79930]: pgmap v615: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:48:42.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:48:41 smithi083 ceph-mon[77242]: pgmap v615: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:48:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:48:41 smithi153 ceph-mon[70044]: pgmap v615: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:48:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:48:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:48:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:48:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:48:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:48:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:48:43.901 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:48:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:48:43.593Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:48:44.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:48:43 smithi083 ceph-mon[79930]: pgmap v616: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:48:44.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:48:43 smithi083 ceph-mon[77242]: pgmap v616: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:48:44.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:48:43 smithi153 ceph-mon[70044]: pgmap v616: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:48:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:48:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:48:44.288Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:48:46.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:48:45 smithi083 ceph-mon[79930]: pgmap v617: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:48:46.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:48:45 smithi083 ceph-mon[77242]: pgmap v617: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:48:46.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:48:45 smithi153 ceph-mon[70044]: pgmap v617: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:48:48.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:48:47 smithi083 ceph-mon[79930]: pgmap v618: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:48:48.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:48:47 smithi083 ceph-mon[77242]: pgmap v618: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:48:48.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:48:47 smithi153 ceph-mon[70044]: pgmap v618: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:48:50.185 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:48:49 smithi083 ceph-mon[79930]: pgmap v619: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:48:50.185 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:48:49 smithi083 ceph-mon[77242]: pgmap v619: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:48:50.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:48:49 smithi153 ceph-mon[70044]: pgmap v619: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:48:50.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:48:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:48:50.182Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:48:51.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:48:50 smithi083 ceph-mon[79930]: pgmap v620: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:48:51.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:48:50 smithi083 ceph-mon[77242]: pgmap v620: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:48:51.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:48:50 smithi153 ceph-mon[70044]: pgmap v620: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:48:51.917 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:48:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:48:51] "GET /metrics HTTP/1.1" 200 225157 "" "Prometheus/2.33.4" 2022-09-29T05:48:51.918 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:48:51 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:48:51.918 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:48:51 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:48:51.918 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:48:51 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:48:51.918 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:48:51 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:48:52.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:48:51 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:48:52.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:48:51 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:48:52.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:48:51 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:48:52.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:48:51 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:48:52.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:48:51 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:48:52.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:48:51 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:48:52.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:48:51 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:48:52.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:48:51 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:48:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:48:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:48:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:48:53.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:48:52 smithi083 ceph-mon[79930]: pgmap v621: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:48:53.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:48:52 smithi083 ceph-mon[77242]: pgmap v621: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:48:53.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:48:52 smithi153 ceph-mon[70044]: pgmap v621: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:48:53.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:48:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:48:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:48:53.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:48:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:48:53.593Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:48:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:48:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:48:54.289Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:48:56.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:48:55 smithi083 ceph-mon[77242]: pgmap v622: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:48:56.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:48:55 smithi083 ceph-mon[79930]: pgmap v622: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:48:56.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:48:55 smithi153 ceph-mon[70044]: pgmap v622: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:48:57.531 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T05:48:57.817 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T05:48:57.817 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (22m) 6m ago 23m 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T05:48:57.817 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (22m) 4m ago 22m 61.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T05:48:57.818 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (24m) 6m ago 24m 17.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T05:48:57.818 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (23m) 4m ago 23m 22.5M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T05:48:57.818 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (19m) 4m ago 26m 435M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T05:48:57.818 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (20m) 6m ago 28m 516M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T05:48:57.819 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (19m) 6m ago 28m 128M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T05:48:57.819 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (18m) 4m ago 26m 77.5M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T05:48:57.819 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (19m) 6m ago 26m 88.2M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T05:48:57.819 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (23m) 6m ago 23m 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T05:48:57.819 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (23m) 4m ago 23m 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T05:48:57.820 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (18m) 6m ago 26m 107M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T05:48:57.820 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (17m) 6m ago 26m 99.8M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T05:48:57.820 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (17m) 6m ago 25m 84.6M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T05:48:57.820 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (16m) 6m ago 25m 90.7M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T05:48:57.820 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (16m) 4m ago 25m 89.2M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T05:48:57.821 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (15m) 4m ago 24m 97.0M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T05:48:57.821 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (15m) 4m ago 24m 85.6M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T05:48:57.821 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (14m) 4m ago 24m 91.5M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T05:48:57.821 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (19m) 4m ago 23m 88.2M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T05:48:58.172 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T05:48:58.172 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T05:48:58.172 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T05:48:58.172 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:48:58.173 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T05:48:58.173 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T05:48:58.173 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:48:58.173 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T05:48:58.174 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T05:48:58.174 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:48:58.174 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T05:48:58.174 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T05:48:58.175 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:48:58.175 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T05:48:58.175 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T05:48:58.175 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T05:48:58.176 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T05:48:58.176 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T05:48:58.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:48:57 smithi083 ceph-mon[79930]: pgmap v623: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:48:58.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:48:57 smithi083 ceph-mon[77242]: pgmap v623: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:48:58.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:48:57 smithi153 ceph-mon[70044]: pgmap v623: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:48:59.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:48:58 smithi083 ceph-mon[79930]: from='client.34863 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:48:59.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:48:58 smithi083 ceph-mon[79930]: from='client.34869 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:48:59.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:48:58 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2238398257' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:48:59.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:48:58 smithi083 ceph-mon[77242]: from='client.34863 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:48:59.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:48:58 smithi083 ceph-mon[77242]: from='client.34869 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:48:59.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:48:58 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2238398257' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:48:59.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:48:58 smithi153 ceph-mon[70044]: from='client.34863 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:48:59.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:48:58 smithi153 ceph-mon[70044]: from='client.34869 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:48:59.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:48:58 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2238398257' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:49:00.185 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:48:59 smithi083 ceph-mon[79930]: pgmap v624: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:00.186 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:48:59 smithi083 ceph-mon[77242]: pgmap v624: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:00.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:48:59 smithi153 ceph-mon[70044]: pgmap v624: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:00.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:49:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:49:00.183Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:49:01.896 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:49:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:49:01] "GET /metrics HTTP/1.1" 200 225151 "" "Prometheus/2.33.4" 2022-09-29T05:49:02.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:49:01 smithi083 ceph-mon[79930]: pgmap v625: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:02.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:49:01 smithi083 ceph-mon[77242]: pgmap v625: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:02.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:49:01 smithi153 ceph-mon[70044]: pgmap v625: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:49:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:49:02.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:49:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:49:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:49:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:49:03.899 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:49:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:49:03.594Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:49:04.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:49:03 smithi083 ceph-mon[79930]: pgmap v626: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:04.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:49:03 smithi083 ceph-mon[77242]: pgmap v626: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:04.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:49:03 smithi153 ceph-mon[70044]: pgmap v626: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:04.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:49:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:49:04.289Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:49:06.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:49:05 smithi083 ceph-mon[77242]: pgmap v627: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:06.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:49:05 smithi083 ceph-mon[79930]: pgmap v627: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:06.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:49:05 smithi153 ceph-mon[70044]: pgmap v627: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:08.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:49:07 smithi083 ceph-mon[77242]: pgmap v628: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:08.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:49:07 smithi083 ceph-mon[79930]: pgmap v628: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:08.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:49:07 smithi153 ceph-mon[70044]: pgmap v628: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:10.187 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:49:09 smithi083 ceph-mon[79930]: pgmap v629: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:10.187 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:49:09 smithi083 ceph-mon[77242]: pgmap v629: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:10.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:49:09 smithi153 ceph-mon[70044]: pgmap v629: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:10.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:49:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:49:10.184Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:49:11.911 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:49:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:49:11] "GET /metrics HTTP/1.1" 200 225155 "" "Prometheus/2.33.4" 2022-09-29T05:49:12.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:49:11 smithi083 ceph-mon[79930]: pgmap v630: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:12.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:49:11 smithi083 ceph-mon[77242]: pgmap v630: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:12.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:49:11 smithi153 ceph-mon[70044]: pgmap v630: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:49:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:49:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:49:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:49:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:49:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:49:13.914 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:49:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:49:13.595Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:49:14.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:49:13 smithi083 ceph-mon[79930]: pgmap v631: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:14.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:49:13 smithi083 ceph-mon[77242]: pgmap v631: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:14.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:49:13 smithi153 ceph-mon[70044]: pgmap v631: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:14.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:49:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:49:14.290Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:49:16.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:49:15 smithi083 ceph-mon[77242]: pgmap v632: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:16.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:49:15 smithi083 ceph-mon[79930]: pgmap v632: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:16.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:49:15 smithi153 ceph-mon[70044]: pgmap v632: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:18.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:49:17 smithi083 ceph-mon[79930]: pgmap v633: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:18.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:49:17 smithi083 ceph-mon[77242]: pgmap v633: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:18.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:49:17 smithi153 ceph-mon[70044]: pgmap v633: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:20.187 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:49:19 smithi083 ceph-mon[79930]: pgmap v634: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:20.188 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:49:19 smithi083 ceph-mon[77242]: pgmap v634: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:20.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:49:19 smithi153 ceph-mon[70044]: pgmap v634: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:20.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:49:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:49:20.185Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:49:21.896 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:49:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:49:21] "GET /metrics HTTP/1.1" 200 225155 "" "Prometheus/2.33.4" 2022-09-29T05:49:22.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:49:21 smithi083 ceph-mon[79930]: pgmap v635: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:22.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:49:21 smithi083 ceph-mon[77242]: pgmap v635: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:22.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:49:21 smithi153 ceph-mon[70044]: pgmap v635: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:49:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:49:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:49:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:49:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:49:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:49:23.899 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:49:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:49:23.595Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:49:24.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:49:23 smithi083 ceph-mon[79930]: pgmap v636: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:24.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:49:23 smithi083 ceph-mon[77242]: pgmap v636: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:24.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:49:23 smithi153 ceph-mon[70044]: pgmap v636: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:49:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:49:24.290Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:49:26.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:49:25 smithi083 ceph-mon[79930]: pgmap v637: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:26.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:49:25 smithi083 ceph-mon[77242]: pgmap v637: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:26.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:49:25 smithi153 ceph-mon[70044]: pgmap v637: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:28.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:49:27 smithi083 ceph-mon[79930]: pgmap v638: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:28.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:49:27 smithi083 ceph-mon[77242]: pgmap v638: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:28.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:49:27 smithi153 ceph-mon[70044]: pgmap v638: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:28.500 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T05:49:28.789 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T05:49:28.789 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (23m) 6m ago 23m 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T05:49:28.790 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (23m) 4m ago 23m 61.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T05:49:28.790 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (24m) 6m ago 24m 17.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T05:49:28.790 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (24m) 4m ago 24m 22.5M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T05:49:28.790 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (20m) 4m ago 27m 435M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T05:49:28.790 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (20m) 6m ago 28m 516M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T05:49:28.791 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (19m) 6m ago 28m 128M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T05:49:28.791 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (19m) 4m ago 27m 77.5M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T05:49:28.791 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (19m) 6m ago 27m 88.2M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T05:49:28.791 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (23m) 6m ago 23m 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T05:49:28.792 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (23m) 4m ago 23m 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T05:49:28.792 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (18m) 6m ago 26m 107M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T05:49:28.792 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (18m) 6m ago 26m 99.8M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T05:49:28.792 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (17m) 6m ago 26m 84.6M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T05:49:28.792 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (17m) 6m ago 25m 90.7M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T05:49:28.793 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (16m) 4m ago 25m 89.2M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T05:49:28.793 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (16m) 4m ago 25m 97.0M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T05:49:28.793 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (15m) 4m ago 25m 85.6M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T05:49:28.793 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (15m) 4m ago 24m 91.5M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T05:49:28.794 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (20m) 4m ago 23m 88.2M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T05:49:29.147 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T05:49:29.148 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T05:49:29.148 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T05:49:29.148 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:49:29.149 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T05:49:29.149 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T05:49:29.149 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:49:29.149 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T05:49:29.150 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T05:49:29.150 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:49:29.150 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T05:49:29.150 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T05:49:29.151 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:49:29.151 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T05:49:29.151 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T05:49:29.151 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T05:49:29.151 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T05:49:29.152 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T05:49:30.188 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:49:29 smithi083 ceph-mon[79930]: pgmap v639: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:30.189 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:49:29 smithi083 ceph-mon[79930]: from='client.44815 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:49:30.189 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:49:29 smithi083 ceph-mon[79930]: from='client.34887 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:49:30.189 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:49:29 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3865964354' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:49:30.189 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:49:29 smithi083 ceph-mon[77242]: pgmap v639: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:30.190 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:49:29 smithi083 ceph-mon[77242]: from='client.44815 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:49:30.190 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:49:29 smithi083 ceph-mon[77242]: from='client.34887 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:49:30.190 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:49:29 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3865964354' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:49:30.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:49:29 smithi153 ceph-mon[70044]: pgmap v639: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:30.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:49:29 smithi153 ceph-mon[70044]: from='client.44815 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:49:30.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:49:29 smithi153 ceph-mon[70044]: from='client.34887 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:49:30.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:49:29 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3865964354' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:49:30.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:49:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:49:30.186Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:49:31.910 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:49:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:49:31] "GET /metrics HTTP/1.1" 200 225156 "" "Prometheus/2.33.4" 2022-09-29T05:49:32.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:49:31 smithi083 ceph-mon[77242]: pgmap v640: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:32.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:49:31 smithi083 ceph-mon[79930]: pgmap v640: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:32.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:49:31 smithi153 ceph-mon[70044]: pgmap v640: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:49:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:49:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:49:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:49:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:49:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:49:33.913 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:49:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:49:33.596Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:49:34.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:49:33 smithi083 ceph-mon[77242]: pgmap v641: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:34.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:49:33 smithi083 ceph-mon[79930]: pgmap v641: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:34.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:49:33 smithi153 ceph-mon[70044]: pgmap v641: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:34.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:49:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:49:34.291Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:49:36.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:49:35 smithi083 ceph-mon[79930]: pgmap v642: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:36.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:49:35 smithi083 ceph-mon[77242]: pgmap v642: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:36.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:49:35 smithi153 ceph-mon[70044]: pgmap v642: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:37.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:49:36 smithi083 ceph-mon[79930]: pgmap v643: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:37.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:49:36 smithi083 ceph-mon[77242]: pgmap v643: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:37.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:49:36 smithi153 ceph-mon[70044]: pgmap v643: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:39.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:49:39 smithi153 ceph-mon[70044]: pgmap v644: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:39.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:49:39 smithi083 ceph-mon[77242]: pgmap v644: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:39.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:49:39 smithi083 ceph-mon[79930]: pgmap v644: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:49:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:49:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:49:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:49:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:49:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:49:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:49:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:49:40.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:49:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:49:40.187Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:49:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:49:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:49:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:49:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:49:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:49:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:49:40.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:49:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:49:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:49:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:49:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:49:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:49:40.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:49:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:49:40.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:49:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:49:41.892 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:49:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:49:41] "GET /metrics HTTP/1.1" 200 225146 "" "Prometheus/2.33.4" 2022-09-29T05:49:42.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:49:41 smithi083 ceph-mon[79930]: pgmap v645: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:42.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:49:41 smithi083 ceph-mon[77242]: pgmap v645: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:49:41 smithi153 ceph-mon[70044]: pgmap v645: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:49:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:49:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:49:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:49:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:49:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:49:43.894 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:49:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:49:43.597Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:49:44.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:49:43 smithi083 ceph-mon[79930]: pgmap v646: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:44.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:49:43 smithi083 ceph-mon[77242]: pgmap v646: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:44.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:49:43 smithi153 ceph-mon[70044]: pgmap v646: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:44.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:49:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:49:44.292Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:49:46.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:49:45 smithi083 ceph-mon[77242]: pgmap v647: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:46.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:49:45 smithi083 ceph-mon[79930]: pgmap v647: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:46.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:49:45 smithi153 ceph-mon[70044]: pgmap v647: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:48.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:49:47 smithi083 ceph-mon[79930]: pgmap v648: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:48.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:49:47 smithi083 ceph-mon[77242]: pgmap v648: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:48.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:49:47 smithi153 ceph-mon[70044]: pgmap v648: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:50.190 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:49:49 smithi083 ceph-mon[79930]: pgmap v649: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:50.191 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:49:49 smithi083 ceph-mon[77242]: pgmap v649: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:50.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:49:49 smithi153 ceph-mon[70044]: pgmap v649: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:50.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:49:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:49:50.188Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:49:51.907 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:49:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:49:51] "GET /metrics HTTP/1.1" 200 225146 "" "Prometheus/2.33.4" 2022-09-29T05:49:52.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:49:51 smithi083 ceph-mon[79930]: pgmap v650: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:52.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:49:51 smithi083 ceph-mon[77242]: pgmap v650: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:52.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:49:51 smithi153 ceph-mon[70044]: pgmap v650: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:49:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:49:52.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:49:53.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:49:52 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:49:53.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:49:52 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:49:53.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:49:52 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:49:53.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:49:52 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:49:53.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:49:52 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:49:53.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:49:52 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:49:53.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:49:52 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:49:53.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:49:52 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:49:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:49:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:49:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:49:53.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:49:52 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:49:53.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:49:52 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:49:53.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:49:52 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:49:53.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:49:52 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:49:53.917 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:49:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:49:53.597Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:49:54.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:49:53 smithi083 ceph-mon[79930]: pgmap v651: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:54.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:49:53 smithi083 ceph-mon[77242]: pgmap v651: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:54.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:49:53 smithi153 ceph-mon[70044]: pgmap v651: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:49:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:49:54.293Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:49:56.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:49:55 smithi083 ceph-mon[77242]: pgmap v652: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:56.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:49:55 smithi083 ceph-mon[79930]: pgmap v652: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:56.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:49:55 smithi153 ceph-mon[70044]: pgmap v652: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:57.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:49:56 smithi083 ceph-mon[77242]: pgmap v653: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:57.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:49:56 smithi083 ceph-mon[79930]: pgmap v653: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:57.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:49:56 smithi153 ceph-mon[70044]: pgmap v653: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:59.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:49:59 smithi153 ceph-mon[70044]: pgmap v654: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:59.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:49:59 smithi083 ceph-mon[79930]: pgmap v654: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:59.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:49:59 smithi083 ceph-mon[77242]: pgmap v654: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:49:59.476 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T05:49:59.764 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T05:49:59.785 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (23m) 7m ago 24m 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T05:49:59.785 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (23m) 5m ago 23m 61.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T05:49:59.785 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (25m) 7m ago 25m 17.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T05:49:59.786 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (25m) 5m ago 25m 22.5M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T05:49:59.786 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (20m) 5m ago 27m 435M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T05:49:59.786 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (21m) 7m ago 29m 516M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T05:49:59.786 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (20m) 7m ago 29m 128M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T05:49:59.787 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (19m) 5m ago 27m 77.5M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T05:49:59.787 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (20m) 7m ago 27m 88.2M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T05:49:59.787 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (24m) 7m ago 24m 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T05:49:59.787 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (24m) 5m ago 24m 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T05:49:59.788 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (19m) 7m ago 27m 107M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T05:49:59.788 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (18m) 7m ago 27m 99.8M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T05:49:59.788 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (18m) 7m ago 26m 84.6M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T05:49:59.788 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (17m) 7m ago 26m 90.7M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T05:49:59.789 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (17m) 5m ago 26m 89.2M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T05:49:59.789 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (16m) 5m ago 25m 97.0M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T05:49:59.789 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (16m) 5m ago 25m 85.6M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T05:49:59.789 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (15m) 5m ago 25m 91.5M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T05:49:59.790 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (20m) 5m ago 24m 88.2M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T05:50:00.128 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T05:50:00.129 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T05:50:00.129 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T05:50:00.129 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:50:00.129 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T05:50:00.130 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T05:50:00.130 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:50:00.130 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T05:50:00.130 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T05:50:00.131 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:50:00.131 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T05:50:00.131 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T05:50:00.131 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:50:00.132 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T05:50:00.132 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T05:50:00.132 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T05:50:00.132 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T05:50:00.133 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T05:50:00.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:50:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:50:00.187Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:50:00.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:50:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[77219]: 2022-09-29T05:49:59.999+0000 7f160f665700 -1 log_channel(cluster) log [ERR] : overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T05:50:01.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:50:00 smithi083 ceph-mon[79930]: from='client.44833 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:50:01.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:50:00 smithi083 ceph-mon[79930]: from='client.34905 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:50:01.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:50:00 smithi083 ceph-mon[79930]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T05:50:01.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:50:00 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/857025394' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:50:01.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:50:00 smithi083 ceph-mon[77242]: from='client.44833 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:50:01.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:50:00 smithi083 ceph-mon[77242]: from='client.34905 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:50:01.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:50:00 smithi083 ceph-mon[77242]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T05:50:01.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:50:00 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/857025394' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:50:01.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:50:00 smithi153 ceph-mon[70044]: from='client.44833 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:50:01.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:50:00 smithi153 ceph-mon[70044]: from='client.34905 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:50:01.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:50:00 smithi153 ceph-mon[70044]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T05:50:01.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:50:00 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/857025394' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:50:01.900 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:50:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:50:01] "GET /metrics HTTP/1.1" 200 225158 "" "Prometheus/2.33.4" 2022-09-29T05:50:02.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:50:01 smithi083 ceph-mon[79930]: pgmap v655: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:50:02.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:50:01 smithi083 ceph-mon[77242]: pgmap v655: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:50:02.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:50:01 smithi153 ceph-mon[70044]: pgmap v655: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:50:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:50:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:50:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:50:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:50:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:50:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:50:03.902 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:50:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:50:03.598Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:50:04.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:50:03 smithi083 ceph-mon[79930]: pgmap v656: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:50:04.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:50:03 smithi083 ceph-mon[77242]: pgmap v656: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:50:04.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:50:03 smithi153 ceph-mon[70044]: pgmap v656: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:50:04.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:50:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:50:04.293Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:50:06.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:50:05 smithi083 ceph-mon[77242]: pgmap v657: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:50:06.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:50:05 smithi083 ceph-mon[79930]: pgmap v657: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:50:06.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:50:05 smithi153 ceph-mon[70044]: pgmap v657: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:50:08.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:50:07 smithi083 ceph-mon[77242]: pgmap v658: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:50:08.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:50:07 smithi083 ceph-mon[79930]: pgmap v658: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:50:08.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:50:07 smithi153 ceph-mon[70044]: pgmap v658: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:50:10.191 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:50:09 smithi083 ceph-mon[79930]: pgmap v659: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:50:10.192 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:50:09 smithi083 ceph-mon[77242]: pgmap v659: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:50:10.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:50:09 smithi153 ceph-mon[70044]: pgmap v659: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:50:10.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:50:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:50:10.189Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:50:11.915 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:50:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:50:11] "GET /metrics HTTP/1.1" 200 225155 "" "Prometheus/2.33.4" 2022-09-29T05:50:12.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:50:11 smithi083 ceph-mon[79930]: pgmap v660: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:50:12.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:50:11 smithi083 ceph-mon[77242]: pgmap v660: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:50:12.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:50:11 smithi153 ceph-mon[70044]: pgmap v660: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:50:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:50:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:50:12.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:50:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:50:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:50:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:50:13.918 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:50:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:50:13.598Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:50:14.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:50:13 smithi083 ceph-mon[79930]: pgmap v661: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:50:14.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:50:13 smithi083 ceph-mon[77242]: pgmap v661: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:50:14.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:50:13 smithi153 ceph-mon[70044]: pgmap v661: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:50:14.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:50:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:50:14.294Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:50:16.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:50:15 smithi083 ceph-mon[79930]: pgmap v662: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:50:16.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:50:15 smithi083 ceph-mon[77242]: pgmap v662: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:50:16.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:50:15 smithi153 ceph-mon[70044]: pgmap v662: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:50:17.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:50:16 smithi153 ceph-mon[70044]: pgmap v663: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:50:17.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:50:16 smithi083 ceph-mon[77242]: pgmap v663: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:50:17.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:50:16 smithi083 ceph-mon[79930]: pgmap v663: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:50:19.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:50:18 smithi153 ceph-mon[70044]: pgmap v664: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:50:19.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:50:18 smithi083 ceph-mon[79930]: pgmap v664: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:50:19.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:50:18 smithi083 ceph-mon[77242]: pgmap v664: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:50:20.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:50:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:50:20.189Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:50:21.900 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:50:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:50:21] "GET /metrics HTTP/1.1" 200 225155 "" "Prometheus/2.33.4" 2022-09-29T05:50:22.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:50:21 smithi083 ceph-mon[79930]: pgmap v665: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:50:22.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:50:21 smithi083 ceph-mon[77242]: pgmap v665: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:50:22.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:50:21 smithi153 ceph-mon[70044]: pgmap v665: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:50:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:50:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:50:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:50:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:50:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:50:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:50:23.903 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:50:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:50:23.599Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:50:24.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:50:23 smithi083 ceph-mon[79930]: pgmap v666: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:50:24.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:50:23 smithi083 ceph-mon[77242]: pgmap v666: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:50:24.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:50:23 smithi153 ceph-mon[70044]: pgmap v666: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:50:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:50:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:50:24.294Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:50:26.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:50:25 smithi083 ceph-mon[79930]: pgmap v667: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:50:26.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:50:25 smithi083 ceph-mon[77242]: pgmap v667: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:50:26.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:50:25 smithi153 ceph-mon[70044]: pgmap v667: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:50:28.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:50:27 smithi083 ceph-mon[79930]: pgmap v668: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:50:28.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:50:27 smithi083 ceph-mon[77242]: pgmap v668: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:50:28.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:50:27 smithi153 ceph-mon[70044]: pgmap v668: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:50:30.192 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:50:29 smithi083 ceph-mon[79930]: pgmap v669: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:50:30.193 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:50:29 smithi083 ceph-mon[77242]: pgmap v669: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:50:30.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:50:29 smithi153 ceph-mon[70044]: pgmap v669: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:50:30.452 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T05:50:30.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:50:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:50:30.190Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:50:30.741 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T05:50:30.741 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (24m) 7m ago 24m 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T05:50:30.742 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (24m) 5m ago 24m 61.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T05:50:30.742 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (25m) 7m ago 25m 17.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T05:50:30.742 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (25m) 5m ago 25m 22.5M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T05:50:30.743 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (21m) 5m ago 28m 435M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T05:50:30.743 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (21m) 7m ago 29m 516M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T05:50:30.743 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (21m) 7m ago 29m 128M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T05:50:30.743 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (20m) 5m ago 28m 77.5M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T05:50:30.744 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (20m) 7m ago 28m 88.2M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T05:50:30.744 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (24m) 7m ago 24m 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T05:50:30.744 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (24m) 5m ago 24m 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T05:50:30.744 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (19m) 7m ago 27m 107M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T05:50:30.745 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (19m) 7m ago 27m 99.8M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T05:50:30.745 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (18m) 7m ago 27m 84.6M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T05:50:30.745 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (18m) 7m ago 26m 90.7M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T05:50:30.745 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (17m) 5m ago 26m 89.2M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T05:50:30.746 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (17m) 5m ago 26m 97.0M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T05:50:30.746 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (16m) 5m ago 26m 85.6M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T05:50:30.746 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (16m) 5m ago 25m 91.5M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T05:50:30.746 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (21m) 5m ago 24m 88.2M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T05:50:31.099 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T05:50:31.099 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T05:50:31.099 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T05:50:31.100 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:50:31.100 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T05:50:31.100 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T05:50:31.100 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:50:31.101 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T05:50:31.101 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T05:50:31.101 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:50:31.101 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T05:50:31.101 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T05:50:31.102 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:50:31.102 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T05:50:31.102 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T05:50:31.102 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T05:50:31.103 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T05:50:31.103 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T05:50:31.913 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:50:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:50:31] "GET /metrics HTTP/1.1" 200 225159 "" "Prometheus/2.33.4" 2022-09-29T05:50:32.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:50:31 smithi083 ceph-mon[77242]: pgmap v670: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:50:32.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:50:31 smithi083 ceph-mon[77242]: from='client.44851 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:50:32.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:50:31 smithi083 ceph-mon[77242]: from='client.34923 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:50:32.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:50:31 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2233913161' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:50:32.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:50:31 smithi083 ceph-mon[79930]: pgmap v670: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:50:32.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:50:31 smithi083 ceph-mon[79930]: from='client.44851 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:50:32.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:50:31 smithi083 ceph-mon[79930]: from='client.34923 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:50:32.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:50:31 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2233913161' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:50:32.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:50:31 smithi153 ceph-mon[70044]: pgmap v670: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:50:32.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:50:31 smithi153 ceph-mon[70044]: from='client.44851 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:50:32.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:50:31 smithi153 ceph-mon[70044]: from='client.34923 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:50:32.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:50:31 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2233913161' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:50:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:50:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:50:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:50:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:50:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:50:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:50:33.916 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:50:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:50:33.600Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:50:34.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:50:33 smithi083 ceph-mon[79930]: pgmap v671: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:50:34.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:50:33 smithi083 ceph-mon[77242]: pgmap v671: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:50:34.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:50:33 smithi153 ceph-mon[70044]: pgmap v671: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:50:34.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:50:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:50:34.294Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:50:36.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:50:35 smithi083 ceph-mon[77242]: pgmap v672: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:50:36.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:50:35 smithi083 ceph-mon[79930]: pgmap v672: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:50:36.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:50:35 smithi153 ceph-mon[70044]: pgmap v672: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:50:38.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:50:37 smithi083 ceph-mon[79930]: pgmap v673: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:50:38.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:50:37 smithi083 ceph-mon[77242]: pgmap v673: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:50:38.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:50:37 smithi153 ceph-mon[70044]: pgmap v673: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:50:40.193 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:50:39 smithi083 ceph-mon[79930]: pgmap v674: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:50:40.194 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:50:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:50:40.194 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:50:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:50:40.194 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:50:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:50:40.195 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:50:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:50:40.195 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:50:39 smithi083 ceph-mon[77242]: pgmap v674: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:50:40.195 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:50:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:50:40.196 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:50:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:50:40.196 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:50:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:50:40.196 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:50:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:50:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:50:39 smithi153 ceph-mon[70044]: pgmap v674: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:50:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:50:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:50:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:50:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:50:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:50:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:50:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:50:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:50:40.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:50:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:50:40.191Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:50:41.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:50:40 smithi083 ceph-mon[77242]: pgmap v675: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:50:41.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:50:40 smithi083 ceph-mon[79930]: pgmap v675: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:50:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:50:40 smithi153 ceph-mon[70044]: pgmap v675: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:50:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:50:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:50:41] "GET /metrics HTTP/1.1" 200 225153 "" "Prometheus/2.33.4" 2022-09-29T05:50:42.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:50:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:50:42.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:50:43.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:50:43 smithi153 ceph-mon[70044]: pgmap v676: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:50:43.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:50:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:50:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:50:43.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:50:43 smithi083 ceph-mon[79930]: pgmap v676: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:50:43.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:50:43 smithi083 ceph-mon[77242]: pgmap v676: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:50:43.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:50:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:50:43.601Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:50:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:50:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:50:44.295Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:50:46.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:50:45 smithi083 ceph-mon[79930]: pgmap v677: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:50:46.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:50:45 smithi083 ceph-mon[77242]: pgmap v677: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:50:46.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:50:45 smithi153 ceph-mon[70044]: pgmap v677: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:50:48.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:50:47 smithi083 ceph-mon[79930]: pgmap v678: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:50:48.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:50:47 smithi083 ceph-mon[77242]: pgmap v678: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:50:48.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:50:47 smithi153 ceph-mon[70044]: pgmap v678: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:50:50.194 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:50:49 smithi083 ceph-mon[79930]: pgmap v679: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:50:50.195 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:50:49 smithi083 ceph-mon[77242]: pgmap v679: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:50:50.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:50:49 smithi153 ceph-mon[70044]: pgmap v679: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:50:50.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:50:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:50:50.192Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:50:51.915 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:50:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:50:51] "GET /metrics HTTP/1.1" 200 225153 "" "Prometheus/2.33.4" 2022-09-29T05:50:52.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:50:51 smithi083 ceph-mon[79930]: pgmap v680: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:50:52.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:50:51 smithi083 ceph-mon[77242]: pgmap v680: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:50:52.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:50:51 smithi153 ceph-mon[70044]: pgmap v680: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:50:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:50:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:50:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:50:53.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:50:52 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:50:53.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:50:52 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:50:53.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:50:52 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:50:53.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:50:52 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:50:53.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:50:52 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:50:53.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:50:52 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:50:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:50:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:50:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:50:53.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:50:52 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:50:53.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:50:52 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:50:53.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:50:52 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:50:53.922 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:50:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:50:53.601Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:50:54.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:50:53 smithi083 ceph-mon[79930]: pgmap v681: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:50:54.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:50:53 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:50:54.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:50:53 smithi083 ceph-mon[77242]: pgmap v681: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:50:54.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:50:53 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:50:54.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:50:53 smithi153 ceph-mon[70044]: pgmap v681: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:50:54.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:50:53 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:50:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:50:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:50:54.295Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:50:56.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:50:55 smithi083 ceph-mon[77242]: pgmap v682: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:50:56.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:50:55 smithi083 ceph-mon[79930]: pgmap v682: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:50:56.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:50:55 smithi153 ceph-mon[70044]: pgmap v682: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:50:58.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:50:57 smithi083 ceph-mon[79930]: pgmap v683: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:50:58.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:50:57 smithi083 ceph-mon[77242]: pgmap v683: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:50:58.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:50:57 smithi153 ceph-mon[70044]: pgmap v683: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:00.196 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:50:59 smithi083 ceph-mon[79930]: pgmap v684: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:00.197 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:50:59 smithi083 ceph-mon[77242]: pgmap v684: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:00.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:50:59 smithi153 ceph-mon[70044]: pgmap v684: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:00.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:51:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:51:00.193Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:51:01.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:51:00 smithi083 ceph-mon[79930]: pgmap v685: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:01.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:51:00 smithi083 ceph-mon[77242]: pgmap v685: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:01.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:51:00 smithi153 ceph-mon[70044]: pgmap v685: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:01.425 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T05:51:01.712 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T05:51:01.713 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (24m) 8m ago 25m 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T05:51:01.713 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (25m) 6m ago 24m 61.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T05:51:01.713 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (26m) 8m ago 26m 17.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T05:51:01.713 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (26m) 6m ago 26m 22.5M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T05:51:01.713 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (21m) 6m ago 28m 435M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T05:51:01.714 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (22m) 8m ago 30m 516M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T05:51:01.714 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (21m) 8m ago 30m 128M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T05:51:01.714 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (20m) 6m ago 28m 77.5M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T05:51:01.714 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (21m) 8m ago 28m 88.2M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T05:51:01.715 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (25m) 8m ago 25m 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T05:51:01.715 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (25m) 6m ago 25m 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T05:51:01.715 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (20m) 8m ago 28m 107M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T05:51:01.715 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (19m) 8m ago 28m 99.8M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T05:51:01.715 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (19m) 8m ago 27m 84.6M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T05:51:01.716 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (18m) 8m ago 27m 90.7M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T05:51:01.716 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (18m) 6m ago 27m 89.2M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T05:51:01.716 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (17m) 6m ago 26m 97.0M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T05:51:01.716 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (17m) 6m ago 26m 85.6M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T05:51:01.716 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (16m) 6m ago 26m 91.5M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T05:51:01.717 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (21m) 6m ago 25m 88.2M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T05:51:01.940 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:51:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:51:01] "GET /metrics HTTP/1.1" 200 225155 "" "Prometheus/2.33.4" 2022-09-29T05:51:02.072 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T05:51:02.072 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T05:51:02.072 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T05:51:02.073 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:51:02.073 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T05:51:02.073 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T05:51:02.073 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:51:02.074 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T05:51:02.074 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T05:51:02.074 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:51:02.074 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T05:51:02.075 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T05:51:02.075 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:51:02.075 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T05:51:02.075 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T05:51:02.076 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T05:51:02.076 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T05:51:02.076 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T05:51:02.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:51:01 smithi083 ceph-mon[79930]: from='client.34935 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:51:02.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:51:01 smithi083 ceph-mon[79930]: from='client.34941 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:51:02.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:51:01 smithi083 ceph-mon[77242]: from='client.34935 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:51:02.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:51:01 smithi083 ceph-mon[77242]: from='client.34941 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:51:02.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:51:01 smithi153 ceph-mon[70044]: from='client.34935 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:51:02.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:51:01 smithi153 ceph-mon[70044]: from='client.34941 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:51:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:51:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:51:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:51:03.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:51:02 smithi083 ceph-mon[79930]: pgmap v686: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:03.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:51:02 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2491965429' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:51:03.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:51:02 smithi083 ceph-mon[77242]: pgmap v686: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:03.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:51:02 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2491965429' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:51:03.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:51:02 smithi153 ceph-mon[70044]: pgmap v686: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:03.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:51:02 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2491965429' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:51:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:51:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:51:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:51:03.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:51:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:51:03.603Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:51:04.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:51:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:51:04.296Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:51:06.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:51:05 smithi083 ceph-mon[77242]: pgmap v687: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:06.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:51:05 smithi083 ceph-mon[79930]: pgmap v687: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:06.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:51:05 smithi153 ceph-mon[70044]: pgmap v687: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:08.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:51:07 smithi083 ceph-mon[79930]: pgmap v688: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:08.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:51:07 smithi083 ceph-mon[77242]: pgmap v688: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:08.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:51:07 smithi153 ceph-mon[70044]: pgmap v688: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:10.195 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:51:09 smithi083 ceph-mon[79930]: pgmap v689: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:10.196 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:51:09 smithi083 ceph-mon[77242]: pgmap v689: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:10.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:51:09 smithi153 ceph-mon[70044]: pgmap v689: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:10.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:51:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:51:10.193Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:51:11.916 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:51:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:51:11] "GET /metrics HTTP/1.1" 200 225155 "" "Prometheus/2.33.4" 2022-09-29T05:51:12.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:51:11 smithi083 ceph-mon[79930]: pgmap v690: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:12.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:51:11 smithi083 ceph-mon[77242]: pgmap v690: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:12.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:51:11 smithi153 ceph-mon[70044]: pgmap v690: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:12.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:51:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:51:12.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:51:13.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:51:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:51:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:51:13.919 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:51:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:51:13.604Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:51:14.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:51:13 smithi083 ceph-mon[79930]: pgmap v691: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:14.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:51:13 smithi083 ceph-mon[77242]: pgmap v691: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:14.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:51:13 smithi153 ceph-mon[70044]: pgmap v691: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:14.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:51:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:51:14.297Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:51:16.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:51:15 smithi083 ceph-mon[79930]: pgmap v692: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:16.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:51:15 smithi083 ceph-mon[77242]: pgmap v692: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:16.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:51:15 smithi153 ceph-mon[70044]: pgmap v692: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:18.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:51:17 smithi083 ceph-mon[79930]: pgmap v693: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:18.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:51:17 smithi083 ceph-mon[77242]: pgmap v693: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:18.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:51:17 smithi153 ceph-mon[70044]: pgmap v693: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:20.197 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:51:19 smithi083 ceph-mon[79930]: pgmap v694: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:20.197 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:51:19 smithi083 ceph-mon[77242]: pgmap v694: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:20.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:51:19 smithi153 ceph-mon[70044]: pgmap v694: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:20.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:51:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:51:20.194Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:51:21.929 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:51:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:51:21] "GET /metrics HTTP/1.1" 200 225155 "" "Prometheus/2.33.4" 2022-09-29T05:51:22.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:51:21 smithi083 ceph-mon[79930]: pgmap v695: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:22.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:51:21 smithi083 ceph-mon[77242]: pgmap v695: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:22.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:51:21 smithi153 ceph-mon[70044]: pgmap v695: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:51:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:51:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:51:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:51:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:51:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:51:23.932 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:51:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:51:23.604Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:51:24.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:51:23 smithi083 ceph-mon[79930]: pgmap v696: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:24.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:51:23 smithi083 ceph-mon[77242]: pgmap v696: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:24.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:51:23 smithi153 ceph-mon[70044]: pgmap v696: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:51:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:51:24.298Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:51:26.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:51:25 smithi083 ceph-mon[79930]: pgmap v697: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:26.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:51:25 smithi083 ceph-mon[77242]: pgmap v697: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:26.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:51:25 smithi153 ceph-mon[70044]: pgmap v697: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:28.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:51:27 smithi083 ceph-mon[79930]: pgmap v698: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:28.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:51:27 smithi083 ceph-mon[77242]: pgmap v698: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:28.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:51:27 smithi153 ceph-mon[70044]: pgmap v698: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:30.198 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:51:29 smithi083 ceph-mon[79930]: pgmap v699: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:30.198 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:51:29 smithi083 ceph-mon[77242]: pgmap v699: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:30.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:51:29 smithi153 ceph-mon[70044]: pgmap v699: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:30.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:51:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:51:30.195Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:51:31.921 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:51:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:51:31] "GET /metrics HTTP/1.1" 200 225155 "" "Prometheus/2.33.4" 2022-09-29T05:51:32.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:51:31 smithi083 ceph-mon[79930]: pgmap v700: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:32.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:51:31 smithi083 ceph-mon[77242]: pgmap v700: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:32.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:51:31 smithi153 ceph-mon[70044]: pgmap v700: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:32.397 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T05:51:32.683 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T05:51:32.684 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (25m) 8m ago 25m 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T05:51:32.684 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (25m) 6m ago 25m 61.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T05:51:32.684 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (26m) 8m ago 26m 17.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T05:51:32.684 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (26m) 6m ago 26m 22.5M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T05:51:32.685 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (22m) 6m ago 29m 435M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T05:51:32.685 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (22m) 8m ago 30m 516M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T05:51:32.685 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (22m) 8m ago 30m 128M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T05:51:32.685 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (21m) 6m ago 29m 77.5M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T05:51:32.686 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (21m) 8m ago 29m 88.2M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T05:51:32.686 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (25m) 8m ago 25m 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T05:51:32.686 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (25m) 6m ago 25m 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T05:51:32.687 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (20m) 8m ago 28m 107M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T05:51:32.687 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (20m) 8m ago 28m 99.8M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T05:51:32.687 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (19m) 8m ago 28m 84.6M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T05:51:32.687 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (19m) 8m ago 27m 90.7M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T05:51:32.687 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (18m) 6m ago 27m 89.2M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T05:51:32.688 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (18m) 6m ago 27m 97.0M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T05:51:32.688 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (17m) 6m ago 27m 85.6M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T05:51:32.688 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (17m) 6m ago 26m 91.5M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T05:51:32.688 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (22m) 6m ago 26m 88.2M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T05:51:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:51:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:51:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:51:33.041 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T05:51:33.042 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T05:51:33.042 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T05:51:33.042 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:51:33.043 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T05:51:33.043 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T05:51:33.043 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:51:33.043 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T05:51:33.044 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T05:51:33.044 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:51:33.044 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T05:51:33.044 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T05:51:33.045 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:51:33.045 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T05:51:33.045 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T05:51:33.045 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T05:51:33.045 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T05:51:33.046 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T05:51:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:51:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:51:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:51:33.923 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:51:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:51:33.605Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:51:34.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:51:33 smithi083 ceph-mon[79930]: pgmap v701: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:34.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:51:33 smithi083 ceph-mon[79930]: from='client.34953 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:51:34.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:51:33 smithi083 ceph-mon[79930]: from='client.44893 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:51:34.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:51:33 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2173042235' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:51:34.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:51:33 smithi083 ceph-mon[77242]: pgmap v701: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:34.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:51:33 smithi083 ceph-mon[77242]: from='client.34953 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:51:34.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:51:33 smithi083 ceph-mon[77242]: from='client.44893 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:51:34.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:51:33 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2173042235' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:51:34.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:51:33 smithi153 ceph-mon[70044]: pgmap v701: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:34.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:51:33 smithi153 ceph-mon[70044]: from='client.34953 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:51:34.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:51:33 smithi153 ceph-mon[70044]: from='client.44893 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:51:34.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:51:33 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2173042235' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:51:34.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:51:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:51:34.299Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:51:36.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:51:35 smithi083 ceph-mon[79930]: pgmap v702: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:36.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:51:35 smithi083 ceph-mon[77242]: pgmap v702: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:36.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:51:35 smithi153 ceph-mon[70044]: pgmap v702: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:38.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:51:37 smithi083 ceph-mon[79930]: pgmap v703: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:38.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:51:37 smithi083 ceph-mon[77242]: pgmap v703: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:38.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:51:37 smithi153 ceph-mon[70044]: pgmap v703: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:39.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:51:38 smithi083 ceph-mon[77242]: pgmap v704: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:39.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:51:38 smithi083 ceph-mon[79930]: pgmap v704: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:39.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:51:38 smithi153 ceph-mon[70044]: pgmap v704: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:40.198 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:51:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:51:40.199 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:51:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:51:40.199 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:51:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:51:40.199 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:51:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:51:40.200 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:51:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:51:40.200 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:51:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:51:40.200 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:51:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:51:40.200 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:51:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:51:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:51:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:51:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:51:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:51:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:51:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:51:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:51:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:51:40.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:51:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:51:40.196Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:51:41.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:51:40 smithi083 ceph-mon[79930]: pgmap v705: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:51:40 smithi083 ceph-mon[77242]: pgmap v705: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:51:40 smithi153 ceph-mon[70044]: pgmap v705: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:51:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:51:41] "GET /metrics HTTP/1.1" 200 225152 "" "Prometheus/2.33.4" 2022-09-29T05:51:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:51:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:51:42.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:51:43.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:51:43 smithi153 ceph-mon[70044]: pgmap v706: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:43.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:51:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:51:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:51:43.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:51:43 smithi083 ceph-mon[79930]: pgmap v706: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:43.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:51:43 smithi083 ceph-mon[77242]: pgmap v706: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:43.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:51:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:51:43.606Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:51:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:51:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:51:44.299Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:51:46.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:51:45 smithi083 ceph-mon[77242]: pgmap v707: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:46.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:51:45 smithi083 ceph-mon[79930]: pgmap v707: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:46.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:51:45 smithi153 ceph-mon[70044]: pgmap v707: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:48.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:51:47 smithi083 ceph-mon[77242]: pgmap v708: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:48.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:51:47 smithi083 ceph-mon[79930]: pgmap v708: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:48.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:51:47 smithi153 ceph-mon[70044]: pgmap v708: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:50.199 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:51:49 smithi083 ceph-mon[79930]: pgmap v709: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:50.199 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:51:49 smithi083 ceph-mon[77242]: pgmap v709: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:50.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:51:49 smithi153 ceph-mon[70044]: pgmap v709: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:50.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:51:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:51:50.196Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:51:51.927 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:51:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:51:51] "GET /metrics HTTP/1.1" 200 225152 "" "Prometheus/2.33.4" 2022-09-29T05:51:52.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:51:51 smithi083 ceph-mon[77242]: pgmap v710: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:52.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:51:51 smithi083 ceph-mon[79930]: pgmap v710: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:52.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:51:51 smithi153 ceph-mon[70044]: pgmap v710: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:51:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:51:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:51:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:51:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:51:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:51:53.930 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:51:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:51:53.606Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:51:54.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:51:53 smithi083 ceph-mon[79930]: pgmap v711: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:54.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:51:53 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:51:54.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:51:53 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:51:54.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:51:53 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:51:54.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:51:53 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:51:54.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:51:53 smithi083 ceph-mon[77242]: pgmap v711: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:54.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:51:53 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:51:54.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:51:53 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:51:54.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:51:53 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:51:54.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:51:53 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:51:54.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:51:53 smithi153 ceph-mon[70044]: pgmap v711: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:54.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:51:53 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:51:54.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:51:53 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:51:54.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:51:53 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:51:54.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:51:53 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:51:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:51:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:51:54.300Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:51:56.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:51:55 smithi083 ceph-mon[79930]: pgmap v712: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:56.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:51:55 smithi083 ceph-mon[77242]: pgmap v712: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:56.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:51:55 smithi153 ceph-mon[70044]: pgmap v712: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:58.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:51:57 smithi083 ceph-mon[79930]: pgmap v713: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:58.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:51:57 smithi083 ceph-mon[77242]: pgmap v713: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:51:58.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:51:57 smithi153 ceph-mon[70044]: pgmap v713: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:00.200 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:51:59 smithi083 ceph-mon[79930]: pgmap v714: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:00.200 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:51:59 smithi083 ceph-mon[77242]: pgmap v714: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:00.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:51:59 smithi153 ceph-mon[70044]: pgmap v714: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:00.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:52:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:52:00.197Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:52:01.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:52:00 smithi083 ceph-mon[79930]: pgmap v715: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:01.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:52:00 smithi083 ceph-mon[77242]: pgmap v715: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:01.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:52:00 smithi153 ceph-mon[70044]: pgmap v715: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:52:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:52:01] "GET /metrics HTTP/1.1" 200 225155 "" "Prometheus/2.33.4" 2022-09-29T05:52:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:52:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:52:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:52:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:52:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:52:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:52:03.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:52:03 smithi153 ceph-mon[70044]: pgmap v716: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:03.372 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T05:52:03.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:52:03 smithi083 ceph-mon[79930]: pgmap v716: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:03.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:52:03 smithi083 ceph-mon[77242]: pgmap v716: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:03.667 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T05:52:03.667 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (25m) 9m ago 26m 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T05:52:03.667 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (26m) 7m ago 26m 61.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T05:52:03.668 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (27m) 9m ago 27m 17.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T05:52:03.668 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (27m) 7m ago 27m 22.5M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T05:52:03.668 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (22m) 7m ago 29m 435M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T05:52:03.668 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (23m) 9m ago 31m 516M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T05:52:03.669 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (22m) 9m ago 31m 128M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T05:52:03.669 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (21m) 7m ago 29m 77.5M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T05:52:03.669 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (22m) 9m ago 30m 88.2M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T05:52:03.669 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (26m) 9m ago 26m 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T05:52:03.670 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (26m) 7m ago 26m 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T05:52:03.670 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (21m) 9m ago 29m 107M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T05:52:03.670 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (20m) 9m ago 29m 99.8M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T05:52:03.670 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (20m) 9m ago 28m 84.6M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T05:52:03.670 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (19m) 9m ago 28m 90.7M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T05:52:03.671 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (19m) 7m ago 28m 89.2M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T05:52:03.671 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (18m) 7m ago 27m 97.0M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T05:52:03.671 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (18m) 7m ago 27m 85.6M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T05:52:03.671 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (17m) 7m ago 27m 91.5M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T05:52:03.672 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (22m) 7m ago 26m 88.2M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T05:52:03.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:52:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:52:03.607Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:52:04.032 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T05:52:04.032 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T05:52:04.032 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T05:52:04.033 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:52:04.033 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T05:52:04.033 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T05:52:04.033 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:52:04.034 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T05:52:04.034 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T05:52:04.034 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:52:04.034 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T05:52:04.035 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T05:52:04.035 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:52:04.035 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T05:52:04.035 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T05:52:04.035 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T05:52:04.036 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T05:52:04.036 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T05:52:04.302 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:52:04 smithi083 ceph-mon[79930]: from='client.44905 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:52:04.303 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:52:04 smithi083 ceph-mon[79930]: from='client.34977 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:52:04.303 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:52:04 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3525048848' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:52:04.303 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:52:04 smithi083 ceph-mon[77242]: from='client.44905 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:52:04.304 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:52:04 smithi083 ceph-mon[77242]: from='client.34977 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:52:04.304 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:52:04 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3525048848' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:52:04.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:52:04 smithi153 ceph-mon[70044]: from='client.44905 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:52:04.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:52:04 smithi153 ceph-mon[70044]: from='client.34977 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:52:04.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:52:04 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3525048848' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:52:04.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:52:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:52:04.300Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:52:06.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:52:05 smithi083 ceph-mon[79930]: pgmap v717: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:06.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:52:05 smithi083 ceph-mon[77242]: pgmap v717: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:06.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:52:05 smithi153 ceph-mon[70044]: pgmap v717: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:08.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:52:07 smithi083 ceph-mon[77242]: pgmap v718: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:08.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:52:07 smithi083 ceph-mon[79930]: pgmap v718: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:08.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:52:07 smithi153 ceph-mon[70044]: pgmap v718: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:10.200 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:52:09 smithi083 ceph-mon[79930]: pgmap v719: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:10.201 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:52:09 smithi083 ceph-mon[77242]: pgmap v719: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:10.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:52:09 smithi153 ceph-mon[70044]: pgmap v719: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:10.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:52:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:52:10.198Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:52:11.931 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:52:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:52:11] "GET /metrics HTTP/1.1" 200 225157 "" "Prometheus/2.33.4" 2022-09-29T05:52:12.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:52:11 smithi083 ceph-mon[79930]: pgmap v720: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:12.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:52:11 smithi083 ceph-mon[77242]: pgmap v720: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:12.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:52:11 smithi153 ceph-mon[70044]: pgmap v720: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:52:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:52:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:52:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:52:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:52:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:52:13.934 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:52:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:52:13.608Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:52:14.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:52:13 smithi083 ceph-mon[79930]: pgmap v721: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:14.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:52:13 smithi083 ceph-mon[77242]: pgmap v721: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:14.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:52:13 smithi153 ceph-mon[70044]: pgmap v721: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:14.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:52:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:52:14.302Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:52:16.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:52:15 smithi083 ceph-mon[79930]: pgmap v722: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:16.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:52:15 smithi083 ceph-mon[77242]: pgmap v722: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:16.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:52:15 smithi153 ceph-mon[70044]: pgmap v722: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:18.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:52:17 smithi083 ceph-mon[79930]: pgmap v723: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:18.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:52:17 smithi083 ceph-mon[77242]: pgmap v723: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:18.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:52:17 smithi153 ceph-mon[70044]: pgmap v723: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:20.201 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:52:19 smithi083 ceph-mon[79930]: pgmap v724: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:20.202 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:52:19 smithi083 ceph-mon[77242]: pgmap v724: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:20.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:52:19 smithi153 ceph-mon[70044]: pgmap v724: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:20.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:52:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:52:20.199Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:52:21.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:52:20 smithi083 ceph-mon[79930]: pgmap v725: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:21.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:52:20 smithi083 ceph-mon[77242]: pgmap v725: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:21.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:52:20 smithi153 ceph-mon[70044]: pgmap v725: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:52:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:52:21] "GET /metrics HTTP/1.1" 200 225157 "" "Prometheus/2.33.4" 2022-09-29T05:52:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:52:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:52:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:52:23.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:52:23 smithi153 ceph-mon[70044]: pgmap v726: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:23.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:52:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:52:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:52:23.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:52:23 smithi083 ceph-mon[79930]: pgmap v726: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:23.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:52:23 smithi083 ceph-mon[77242]: pgmap v726: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:23.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:52:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:52:23.608Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:52:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:52:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:52:24.302Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:52:26.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:52:25 smithi083 ceph-mon[77242]: pgmap v727: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:26.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:52:25 smithi083 ceph-mon[79930]: pgmap v727: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:26.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:52:25 smithi153 ceph-mon[70044]: pgmap v727: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:28.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:52:27 smithi083 ceph-mon[79930]: pgmap v728: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:28.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:52:27 smithi083 ceph-mon[77242]: pgmap v728: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:28.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:52:27 smithi153 ceph-mon[70044]: pgmap v728: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:30.202 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:52:29 smithi083 ceph-mon[79930]: pgmap v729: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:30.203 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:52:29 smithi083 ceph-mon[77242]: pgmap v729: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:30.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:52:29 smithi153 ceph-mon[70044]: pgmap v729: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:30.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:52:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:52:30.200Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:52:31.934 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:52:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:52:31] "GET /metrics HTTP/1.1" 200 225152 "" "Prometheus/2.33.4" 2022-09-29T05:52:32.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:52:31 smithi083 ceph-mon[79930]: pgmap v730: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:32.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:52:31 smithi083 ceph-mon[77242]: pgmap v730: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:32.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:52:31 smithi153 ceph-mon[70044]: pgmap v730: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:52:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:52:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:52:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:52:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:52:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:52:33.936 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:52:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:52:33.609Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:52:34.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:52:33 smithi083 ceph-mon[79930]: pgmap v731: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:34.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:52:33 smithi083 ceph-mon[77242]: pgmap v731: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:34.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:52:33 smithi153 ceph-mon[70044]: pgmap v731: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:34.360 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T05:52:34.648 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T05:52:34.648 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (26m) 9m ago 26m 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T05:52:34.649 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (26m) 7m ago 26m 61.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T05:52:34.649 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (27m) 9m ago 27m 17.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T05:52:34.649 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (27m) 7m ago 27m 22.5M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T05:52:34.649 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (23m) 7m ago 30m 435M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T05:52:34.649 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (23m) 9m ago 31m 516M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T05:52:34.650 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (23m) 9m ago 31m 128M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T05:52:34.650 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (22m) 7m ago 30m 77.5M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T05:52:34.650 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (22m) 9m ago 30m 88.2M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T05:52:34.650 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (27m) 9m ago 26m 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T05:52:34.651 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (26m) 7m ago 26m 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T05:52:34.651 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (21m) 9m ago 29m 107M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T05:52:34.651 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (21m) 9m ago 29m 99.8M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T05:52:34.651 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (20m) 9m ago 29m 84.6M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T05:52:34.651 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (20m) 9m ago 29m 90.7M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T05:52:34.652 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (19m) 7m ago 28m 89.2M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T05:52:34.652 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (19m) 7m ago 28m 97.0M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T05:52:34.652 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (18m) 7m ago 28m 85.6M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T05:52:34.652 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (18m) 7m ago 27m 91.5M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T05:52:34.652 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (23m) 7m ago 27m 88.2M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T05:52:34.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:52:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:52:34.302Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:52:35.009 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T05:52:35.009 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T05:52:35.010 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T05:52:35.010 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:52:35.010 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T05:52:35.010 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T05:52:35.011 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:52:35.011 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T05:52:35.011 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T05:52:35.011 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:52:35.012 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T05:52:35.012 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T05:52:35.012 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:52:35.012 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T05:52:35.012 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T05:52:35.013 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T05:52:35.013 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T05:52:35.013 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T05:52:36.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:52:35 smithi083 ceph-mon[79930]: pgmap v732: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:36.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:52:35 smithi083 ceph-mon[79930]: from='client.44923 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:52:36.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:52:35 smithi083 ceph-mon[79930]: from='client.44929 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:52:36.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:52:35 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2032493832' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:52:36.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:52:35 smithi083 ceph-mon[77242]: pgmap v732: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:36.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:52:35 smithi083 ceph-mon[77242]: from='client.44923 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:52:36.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:52:35 smithi083 ceph-mon[77242]: from='client.44929 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:52:36.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:52:35 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2032493832' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:52:36.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:52:35 smithi153 ceph-mon[70044]: pgmap v732: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:36.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:52:35 smithi153 ceph-mon[70044]: from='client.44923 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:52:36.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:52:35 smithi153 ceph-mon[70044]: from='client.44929 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:52:36.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:52:35 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2032493832' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:52:38.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:52:37 smithi083 ceph-mon[79930]: pgmap v733: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:38.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:52:37 smithi083 ceph-mon[77242]: pgmap v733: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:38.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:52:37 smithi153 ceph-mon[70044]: pgmap v733: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:40.203 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:52:39 smithi083 ceph-mon[77242]: pgmap v734: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:40.203 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:52:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:52:40.203 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:52:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:52:40.204 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:52:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:52:40.204 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:52:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:52:40.204 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:52:39 smithi083 ceph-mon[79930]: pgmap v734: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:40.204 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:52:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:52:40.205 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:52:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:52:40.205 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:52:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:52:40.205 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:52:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:52:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:52:39 smithi153 ceph-mon[70044]: pgmap v734: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:52:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:52:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:52:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:52:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:52:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:52:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:52:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:52:40.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:52:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:52:40.200Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:52:41.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:52:40 smithi083 ceph-mon[79930]: pgmap v735: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:52:40 smithi083 ceph-mon[77242]: pgmap v735: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:52:40 smithi153 ceph-mon[70044]: pgmap v735: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:52:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:52:41] "GET /metrics HTTP/1.1" 200 225154 "" "Prometheus/2.33.4" 2022-09-29T05:52:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:52:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:52:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:52:43.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:52:43 smithi153 ceph-mon[70044]: pgmap v736: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:43.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:52:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:52:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:52:43.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:52:43 smithi083 ceph-mon[79930]: pgmap v736: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:43.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:52:43 smithi083 ceph-mon[77242]: pgmap v736: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:43.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:52:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:52:43.610Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:52:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:52:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:52:44.304Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:52:46.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:52:45 smithi083 ceph-mon[79930]: pgmap v737: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:46.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:52:45 smithi083 ceph-mon[77242]: pgmap v737: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:46.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:52:45 smithi153 ceph-mon[70044]: pgmap v737: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:48.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:52:47 smithi083 ceph-mon[77242]: pgmap v738: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:48.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:52:47 smithi083 ceph-mon[79930]: pgmap v738: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:48.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:52:47 smithi153 ceph-mon[70044]: pgmap v738: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:50.204 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:52:49 smithi083 ceph-mon[79930]: pgmap v739: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:50.204 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:52:49 smithi083 ceph-mon[77242]: pgmap v739: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:50.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:52:49 smithi153 ceph-mon[70044]: pgmap v739: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:50.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:52:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:52:50.201Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:52:51.937 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:52:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:52:51] "GET /metrics HTTP/1.1" 200 225154 "" "Prometheus/2.33.4" 2022-09-29T05:52:52.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:52:51 smithi083 ceph-mon[79930]: pgmap v740: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:52.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:52:51 smithi083 ceph-mon[77242]: pgmap v740: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:52.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:52:51 smithi153 ceph-mon[70044]: pgmap v740: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:52:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:52:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:52:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:52:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:52:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:52:53.940 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:52:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:52:53.611Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:52:54.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:52:53 smithi083 ceph-mon[79930]: pgmap v741: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:54.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:52:53 smithi083 ceph-mon[77242]: pgmap v741: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:54.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:52:53 smithi153 ceph-mon[70044]: pgmap v741: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:52:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:52:54.304Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:52:55.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:52:54 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:52:55.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:52:54 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:52:55.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:52:54 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:52:55.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:52:54 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:52:55.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:52:54 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:52:55.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:52:54 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:52:55.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:52:54 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:52:55.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:52:54 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:52:55.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:52:54 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:52:55.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:52:54 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:52:55.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:52:54 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:52:55.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:52:54 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:52:55.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:52:54 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:52:55.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:52:54 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:52:55.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:52:54 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:52:55.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:52:54 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:52:55.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:52:54 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:52:55.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:52:54 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:52:55.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:52:54 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:52:55.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:52:54 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:52:55.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:52:54 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:52:56.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:52:55 smithi083 ceph-mon[79930]: pgmap v742: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:56.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:52:55 smithi083 ceph-mon[77242]: pgmap v742: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:56.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:52:55 smithi153 ceph-mon[70044]: pgmap v742: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:57.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:52:56 smithi083 ceph-mon[79930]: pgmap v743: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:57.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:52:56 smithi083 ceph-mon[77242]: pgmap v743: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:52:57.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:52:56 smithi153 ceph-mon[70044]: pgmap v743: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:00.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:52:59 smithi153 ceph-mon[70044]: pgmap v744: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:00.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:52:59 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:53:00.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:52:59 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:53:00.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:52:59 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:53:00.205 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:52:59 smithi083 ceph-mon[79930]: pgmap v744: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:00.205 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:52:59 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:53:00.205 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:52:59 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:53:00.206 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:52:59 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:53:00.206 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:52:59 smithi083 ceph-mon[77242]: pgmap v744: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:00.206 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:52:59 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:53:00.206 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:52:59 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:53:00.207 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:52:59 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:53:00.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:53:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:53:00.202Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:53:01.935 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:53:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:53:01] "GET /metrics HTTP/1.1" 200 225155 "" "Prometheus/2.33.4" 2022-09-29T05:53:02.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:53:01 smithi083 ceph-mon[79930]: pgmap v745: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:02.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:53:01 smithi083 ceph-mon[77242]: pgmap v745: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:02.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:53:01 smithi153 ceph-mon[70044]: pgmap v745: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:53:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:53:02.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:53:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:53:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:53:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:53:03.938 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:53:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:53:03.611Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:53:04.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:53:03 smithi083 ceph-mon[77242]: pgmap v746: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:04.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:53:03 smithi083 ceph-mon[79930]: pgmap v746: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:04.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:53:03 smithi153 ceph-mon[70044]: pgmap v746: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:04.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:53:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:53:04.305Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:53:05.333 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T05:53:05.621 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T05:53:05.622 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (26m) 6s ago 27m 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T05:53:05.622 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (27m) 8m ago 27m 61.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T05:53:05.622 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (28m) 6s ago 28m 19.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T05:53:05.622 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (28m) 8m ago 28m 22.5M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T05:53:05.623 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (23m) 8m ago 30m 435M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T05:53:05.623 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (24m) 6s ago 32m 527M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T05:53:05.623 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (23m) 6s ago 32m 166M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T05:53:05.623 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (22m) 8m ago 30m 77.5M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T05:53:05.624 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (23m) 6s ago 31m 103M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T05:53:05.624 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (27m) 6s ago 27m 26.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T05:53:05.624 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (27m) 8m ago 27m 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T05:53:05.624 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (22m) 6s ago 30m 133M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T05:53:05.624 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (21m) 6s ago 30m 125M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T05:53:05.625 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (21m) 6s ago 29m 105M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T05:53:05.625 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (20m) 6s ago 29m 116M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T05:53:05.625 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (20m) 8m ago 29m 89.2M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T05:53:05.625 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (19m) 8m ago 28m 97.0M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T05:53:05.626 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (19m) 8m ago 28m 85.6M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T05:53:05.626 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (18m) 8m ago 28m 91.5M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T05:53:05.626 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (23m) 8m ago 27m 88.2M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T05:53:05.981 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T05:53:05.982 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T05:53:05.982 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T05:53:05.982 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:53:05.983 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T05:53:05.983 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T05:53:05.983 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:53:05.983 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T05:53:05.984 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T05:53:05.984 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:53:05.984 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T05:53:05.984 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T05:53:05.984 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:53:05.985 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T05:53:05.985 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T05:53:05.985 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T05:53:05.985 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T05:53:05.986 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T05:53:06.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:53:05 smithi083 ceph-mon[79930]: pgmap v747: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:06.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:53:05 smithi083 ceph-mon[77242]: pgmap v747: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:06.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:53:05 smithi153 ceph-mon[70044]: pgmap v747: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:07.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:53:06 smithi083 ceph-mon[79930]: from='client.44941 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:53:07.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:53:06 smithi083 ceph-mon[79930]: from='client.44947 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:53:07.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:53:06 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3069131404' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:53:07.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:53:06 smithi083 ceph-mon[77242]: from='client.44941 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:53:07.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:53:06 smithi083 ceph-mon[77242]: from='client.44947 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:53:07.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:53:06 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3069131404' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:53:07.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:53:06 smithi153 ceph-mon[70044]: from='client.44941 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:53:07.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:53:06 smithi153 ceph-mon[70044]: from='client.44947 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:53:07.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:53:06 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3069131404' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:53:08.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:53:07 smithi083 ceph-mon[79930]: pgmap v748: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:08.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:53:07 smithi083 ceph-mon[77242]: pgmap v748: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:08.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:53:07 smithi153 ceph-mon[70044]: pgmap v748: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:10.204 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:53:09 smithi083 ceph-mon[79930]: pgmap v749: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:10.205 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:53:09 smithi083 ceph-mon[77242]: pgmap v749: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:10.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:53:09 smithi153 ceph-mon[70044]: pgmap v749: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:10.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:53:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:53:10.202Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:53:11.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:53:10 smithi083 ceph-mon[79930]: pgmap v750: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:11.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:53:10 smithi083 ceph-mon[77242]: pgmap v750: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:11.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:53:10 smithi153 ceph-mon[70044]: pgmap v750: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:53:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:53:11] "GET /metrics HTTP/1.1" 200 225149 "" "Prometheus/2.33.4" 2022-09-29T05:53:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:53:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:53:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:53:13.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:53:13 smithi153 ceph-mon[70044]: pgmap v751: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:13.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:53:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:53:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:53:13.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:53:13 smithi083 ceph-mon[77242]: pgmap v751: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:13.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:53:13 smithi083 ceph-mon[79930]: pgmap v751: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:13.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:53:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:53:13.611Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:53:14.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:53:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:53:14.306Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:53:16.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:53:15 smithi083 ceph-mon[79930]: pgmap v752: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:16.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:53:15 smithi083 ceph-mon[77242]: pgmap v752: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:16.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:53:15 smithi153 ceph-mon[70044]: pgmap v752: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:18.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:53:17 smithi083 ceph-mon[79930]: pgmap v753: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:18.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:53:17 smithi083 ceph-mon[77242]: pgmap v753: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:18.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:53:17 smithi153 ceph-mon[70044]: pgmap v753: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:20.205 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:53:19 smithi083 ceph-mon[79930]: pgmap v754: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:20.205 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:53:19 smithi083 ceph-mon[77242]: pgmap v754: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:20.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:53:19 smithi153 ceph-mon[70044]: pgmap v754: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:20.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:53:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:53:20.202Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:53:21.945 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:53:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:53:21] "GET /metrics HTTP/1.1" 200 225149 "" "Prometheus/2.33.4" 2022-09-29T05:53:22.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:53:21 smithi083 ceph-mon[79930]: pgmap v755: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:22.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:53:21 smithi083 ceph-mon[77242]: pgmap v755: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:22.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:53:21 smithi153 ceph-mon[70044]: pgmap v755: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:53:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:53:22.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:53:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:53:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:53:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:53:23.947 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:53:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:53:23.611Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:53:24.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:53:23 smithi083 ceph-mon[79930]: pgmap v756: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:24.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:53:23 smithi083 ceph-mon[77242]: pgmap v756: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:24.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:53:23 smithi153 ceph-mon[70044]: pgmap v756: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:53:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:53:24.307Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:53:26.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:53:25 smithi083 ceph-mon[77242]: pgmap v757: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:26.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:53:25 smithi083 ceph-mon[79930]: pgmap v757: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:26.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:53:25 smithi153 ceph-mon[70044]: pgmap v757: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:27.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:53:27 smithi153 ceph-mon[70044]: pgmap v758: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:27.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:53:27 smithi083 ceph-mon[77242]: pgmap v758: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:27.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:53:27 smithi083 ceph-mon[79930]: pgmap v758: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:29.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:53:29 smithi083 ceph-mon[77242]: pgmap v759: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:29.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:53:29 smithi083 ceph-mon[79930]: pgmap v759: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:29.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:53:29 smithi153 ceph-mon[70044]: pgmap v759: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:30.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:53:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:53:30.203Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:53:31.942 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:53:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:53:31] "GET /metrics HTTP/1.1" 200 225156 "" "Prometheus/2.33.4" 2022-09-29T05:53:32.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:53:31 smithi083 ceph-mon[77242]: pgmap v760: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:32.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:53:31 smithi083 ceph-mon[79930]: pgmap v760: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:32.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:53:31 smithi153 ceph-mon[70044]: pgmap v760: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:53:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:53:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:53:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:53:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:53:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:53:33.944 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:53:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:53:33.612Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:53:34.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:53:33 smithi083 ceph-mon[77242]: pgmap v761: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:34.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:53:33 smithi083 ceph-mon[79930]: pgmap v761: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:34.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:53:33 smithi153 ceph-mon[70044]: pgmap v761: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:34.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:53:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:53:34.307Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:53:36.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:53:35 smithi083 ceph-mon[79930]: pgmap v762: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:36.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:53:35 smithi083 ceph-mon[77242]: pgmap v762: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:36.308 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T05:53:36.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:53:35 smithi153 ceph-mon[70044]: pgmap v762: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:36.595 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T05:53:36.595 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (27m) 37s ago 27m 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T05:53:36.596 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (27m) 8m ago 27m 61.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T05:53:36.596 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (28m) 37s ago 28m 19.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T05:53:36.596 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (28m) 8m ago 28m 22.5M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T05:53:36.596 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (24m) 8m ago 31m 435M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T05:53:36.596 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (24m) 37s ago 32m 527M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T05:53:36.597 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (24m) 37s ago 32m 166M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T05:53:36.597 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (23m) 8m ago 31m 77.5M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T05:53:36.597 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (23m) 37s ago 31m 103M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T05:53:36.597 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (28m) 37s ago 28m 26.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T05:53:36.598 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (27m) 8m ago 27m 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T05:53:36.598 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (22m) 37s ago 30m 133M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T05:53:36.598 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (22m) 37s ago 30m 125M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T05:53:36.598 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (21m) 37s ago 30m 105M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T05:53:36.599 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (21m) 37s ago 30m 116M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T05:53:36.599 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (20m) 8m ago 29m 89.2M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T05:53:36.599 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (20m) 8m ago 29m 97.0M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T05:53:36.599 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (19m) 8m ago 29m 85.6M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T05:53:36.599 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (19m) 8m ago 28m 91.5M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T05:53:36.600 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (24m) 8m ago 28m 88.2M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T05:53:36.952 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T05:53:36.952 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T05:53:36.953 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T05:53:36.953 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:53:36.953 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T05:53:36.953 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T05:53:36.954 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:53:36.954 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T05:53:36.954 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T05:53:36.954 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:53:36.955 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T05:53:36.955 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T05:53:36.955 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:53:36.955 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T05:53:36.955 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T05:53:36.956 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T05:53:36.956 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T05:53:36.956 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T05:53:38.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:53:37 smithi083 ceph-mon[79930]: pgmap v763: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:38.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:53:37 smithi083 ceph-mon[79930]: from='client.35025 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:53:38.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:53:37 smithi083 ceph-mon[79930]: from='client.44965 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:53:38.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:53:37 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/541139493' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:53:38.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:53:37 smithi083 ceph-mon[77242]: pgmap v763: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:38.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:53:37 smithi083 ceph-mon[77242]: from='client.35025 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:53:38.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:53:37 smithi083 ceph-mon[77242]: from='client.44965 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:53:38.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:53:37 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/541139493' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:53:38.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:53:37 smithi153 ceph-mon[70044]: pgmap v763: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:38.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:53:37 smithi153 ceph-mon[70044]: from='client.35025 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:53:38.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:53:37 smithi153 ceph-mon[70044]: from='client.44965 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:53:38.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:53:37 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/541139493' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:53:40.206 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:53:39 smithi083 ceph-mon[79930]: pgmap v764: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:40.206 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:53:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:53:40.207 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:53:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:53:40.207 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:53:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:53:40.207 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:53:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:53:40.207 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:53:39 smithi083 ceph-mon[77242]: pgmap v764: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:40.208 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:53:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:53:40.208 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:53:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:53:40.208 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:53:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:53:40.208 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:53:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:53:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:53:39 smithi153 ceph-mon[70044]: pgmap v764: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:53:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:53:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:53:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:53:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:53:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:53:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:53:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:53:40.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:53:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:53:40.203Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:53:41.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:53:40 smithi083 ceph-mon[77242]: pgmap v765: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:41.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:53:40 smithi083 ceph-mon[79930]: pgmap v765: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:53:40 smithi153 ceph-mon[70044]: pgmap v765: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:53:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:53:41] "GET /metrics HTTP/1.1" 200 225152 "" "Prometheus/2.33.4" 2022-09-29T05:53:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:53:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:53:42.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:53:43.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:53:43 smithi153 ceph-mon[70044]: pgmap v766: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:43.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:53:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:53:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:53:43.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:53:43 smithi083 ceph-mon[79930]: pgmap v766: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:43.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:53:43 smithi083 ceph-mon[77242]: pgmap v766: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:43.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:53:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:53:43.613Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:53:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:53:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:53:44.308Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:53:46.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:53:45 smithi083 ceph-mon[79930]: pgmap v767: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:46.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:53:45 smithi083 ceph-mon[77242]: pgmap v767: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:46.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:53:45 smithi153 ceph-mon[70044]: pgmap v767: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:48.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:53:47 smithi083 ceph-mon[79930]: pgmap v768: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:48.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:53:47 smithi083 ceph-mon[77242]: pgmap v768: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:48.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:53:47 smithi153 ceph-mon[70044]: pgmap v768: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:50.206 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:53:49 smithi083 ceph-mon[79930]: pgmap v769: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:50.207 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:53:49 smithi083 ceph-mon[77242]: pgmap v769: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:50.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:53:49 smithi153 ceph-mon[70044]: pgmap v769: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:50.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:53:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:53:50.204Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:53:51.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:53:51 smithi153 ceph-mon[70044]: pgmap v770: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:51.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:53:51 smithi083 ceph-mon[77242]: pgmap v770: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:51.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:53:51 smithi083 ceph-mon[79930]: pgmap v770: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:53:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:53:51] "GET /metrics HTTP/1.1" 200 225152 "" "Prometheus/2.33.4" 2022-09-29T05:53:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:53:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:53:52.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:53:53.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:53:53 smithi153 ceph-mon[70044]: pgmap v771: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:53.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:53:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:53:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:53:53.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:53:53 smithi083 ceph-mon[79930]: pgmap v771: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:53.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:53:53 smithi083 ceph-mon[77242]: pgmap v771: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:53.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:53:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:53:53.613Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:53:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:53:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:53:54.309Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:53:56.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:53:55 smithi083 ceph-mon[79930]: pgmap v772: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:56.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:53:55 smithi083 ceph-mon[77242]: pgmap v772: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:56.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:53:55 smithi153 ceph-mon[70044]: pgmap v772: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:58.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:53:57 smithi083 ceph-mon[77242]: pgmap v773: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:58.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:53:57 smithi083 ceph-mon[79930]: pgmap v773: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:53:58.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:53:57 smithi153 ceph-mon[70044]: pgmap v773: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:00.207 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:53:59 smithi083 ceph-mon[79930]: pgmap v774: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:00.208 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:53:59 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:54:00.208 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:53:59 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:54:00.208 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:53:59 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:54:00.209 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:53:59 smithi083 ceph-mon[77242]: pgmap v774: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:00.209 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:53:59 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:54:00.209 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:53:59 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:54:00.209 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:53:59 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:54:00.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:53:59 smithi153 ceph-mon[70044]: pgmap v774: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:00.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:53:59 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:54:00.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:53:59 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:54:00.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:53:59 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:54:00.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:54:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:54:00.205Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:54:01.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:54:01 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:54:01.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:54:01 smithi153 ceph-mon[70044]: pgmap v775: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:01.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:54:00 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:54:01.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:54:00 smithi083 ceph-mon[79930]: pgmap v775: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:01.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:54:00 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:54:01.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:54:00 smithi083 ceph-mon[77242]: pgmap v775: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:54:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:54:01] "GET /metrics HTTP/1.1" 200 225156 "" "Prometheus/2.33.4" 2022-09-29T05:54:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:54:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:54:02.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:54:03.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:54:03 smithi153 ceph-mon[70044]: pgmap v776: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:54:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:54:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:54:03.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:54:03 smithi083 ceph-mon[77242]: pgmap v776: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:03.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:54:03 smithi083 ceph-mon[79930]: pgmap v776: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:03.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:54:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:54:03.614Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:54:04.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:54:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:54:04.310Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:54:06.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:54:05 smithi083 ceph-mon[79930]: pgmap v777: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:06.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:54:05 smithi083 ceph-mon[77242]: pgmap v777: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:06.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:54:05 smithi153 ceph-mon[70044]: pgmap v777: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:07.286 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T05:54:07.574 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T05:54:07.575 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (27m) 68s ago 28m 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T05:54:07.575 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (28m) 9m ago 28m 61.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T05:54:07.575 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (29m) 68s ago 29m 19.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T05:54:07.575 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (29m) 9m ago 29m 22.5M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T05:54:07.576 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (24m) 9m ago 31m 435M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T05:54:07.576 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (25m) 68s ago 33m 527M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T05:54:07.576 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (24m) 68s ago 33m 166M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T05:54:07.576 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (24m) 9m ago 31m 77.5M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T05:54:07.576 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (24m) 68s ago 32m 103M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T05:54:07.577 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (28m) 68s ago 28m 26.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T05:54:07.577 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (28m) 9m ago 28m 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T05:54:07.577 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (23m) 68s ago 31m 133M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T05:54:07.577 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (22m) 68s ago 31m 125M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T05:54:07.578 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (22m) 68s ago 30m 105M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T05:54:07.578 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (21m) 68s ago 30m 116M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T05:54:07.578 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (21m) 9m ago 30m 89.2M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T05:54:07.578 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (20m) 9m ago 30m 97.0M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T05:54:07.579 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (20m) 9m ago 29m 85.6M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T05:54:07.579 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (19m) 9m ago 29m 91.5M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T05:54:07.579 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (24m) 9m ago 28m 88.2M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T05:54:07.931 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T05:54:07.932 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T05:54:07.932 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T05:54:07.932 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:54:07.932 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T05:54:07.933 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T05:54:07.933 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:54:07.933 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T05:54:07.933 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T05:54:07.933 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:54:07.934 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T05:54:07.934 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T05:54:07.934 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:54:07.934 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T05:54:07.935 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T05:54:07.935 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T05:54:07.935 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T05:54:07.935 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T05:54:08.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:54:07 smithi083 ceph-mon[77242]: pgmap v778: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:08.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:54:07 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3122983916' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:54:08.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:54:07 smithi083 ceph-mon[79930]: pgmap v778: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:08.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:54:07 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3122983916' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:54:08.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:54:07 smithi153 ceph-mon[70044]: pgmap v778: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:08.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:54:07 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3122983916' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:54:09.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:54:08 smithi083 ceph-mon[79930]: from='client.35043 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:54:09.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:54:08 smithi083 ceph-mon[79930]: from='client.44983 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:54:09.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:54:08 smithi083 ceph-mon[77242]: from='client.35043 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:54:09.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:54:08 smithi083 ceph-mon[77242]: from='client.44983 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:54:09.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:54:08 smithi153 ceph-mon[70044]: from='client.35043 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:54:09.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:54:08 smithi153 ceph-mon[70044]: from='client.44983 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:54:10.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:54:09 smithi083 ceph-mon[79930]: pgmap v779: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:10.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:54:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:54:10.206Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:54:10.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:54:09 smithi083 ceph-mon[77242]: pgmap v779: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:10.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:54:09 smithi153 ceph-mon[70044]: pgmap v779: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:11.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:54:10 smithi153 ceph-mon[70044]: pgmap v780: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:11.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:54:10 smithi083 ceph-mon[79930]: pgmap v780: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:11.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:54:10 smithi083 ceph-mon[77242]: pgmap v780: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:54:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:54:11] "GET /metrics HTTP/1.1" 200 225156 "" "Prometheus/2.33.4" 2022-09-29T05:54:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:54:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:54:12.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:54:13.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:54:13 smithi153 ceph-mon[70044]: pgmap v781: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:13.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:54:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:54:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:54:13.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:54:13 smithi083 ceph-mon[79930]: pgmap v781: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:13.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:54:13 smithi083 ceph-mon[77242]: pgmap v781: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:13.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:54:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:54:13.614Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:54:14.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:54:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:54:14.311Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:54:16.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:54:15 smithi083 ceph-mon[79930]: pgmap v782: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:16.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:54:15 smithi083 ceph-mon[77242]: pgmap v782: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:16.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:54:15 smithi153 ceph-mon[70044]: pgmap v782: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:18.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:54:17 smithi083 ceph-mon[79930]: pgmap v783: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:18.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:54:17 smithi083 ceph-mon[77242]: pgmap v783: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:18.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:54:17 smithi153 ceph-mon[70044]: pgmap v783: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:20.209 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:54:19 smithi083 ceph-mon[79930]: pgmap v784: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:20.210 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:54:19 smithi083 ceph-mon[77242]: pgmap v784: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:20.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:54:19 smithi153 ceph-mon[70044]: pgmap v784: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:20.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:54:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:54:20.207Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:54:21.959 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:54:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:54:21] "GET /metrics HTTP/1.1" 200 225156 "" "Prometheus/2.33.4" 2022-09-29T05:54:22.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:54:21 smithi083 ceph-mon[79930]: pgmap v785: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:22.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:54:21 smithi083 ceph-mon[77242]: pgmap v785: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:22.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:54:21 smithi153 ceph-mon[70044]: pgmap v785: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:54:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:54:22.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:54:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:54:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:54:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:54:23.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:54:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:54:23.614Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:54:24.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:54:23 smithi083 ceph-mon[79930]: pgmap v786: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:24.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:54:23 smithi083 ceph-mon[77242]: pgmap v786: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:24.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:54:23 smithi153 ceph-mon[70044]: pgmap v786: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:54:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:54:24.311Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:54:26.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:54:25 smithi083 ceph-mon[79930]: pgmap v787: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:26.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:54:25 smithi083 ceph-mon[77242]: pgmap v787: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:26.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:54:25 smithi153 ceph-mon[70044]: pgmap v787: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:28.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:54:27 smithi083 ceph-mon[79930]: pgmap v788: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:28.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:54:27 smithi083 ceph-mon[77242]: pgmap v788: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:28.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:54:27 smithi153 ceph-mon[70044]: pgmap v788: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:30.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:54:29 smithi083 ceph-mon[79930]: pgmap v789: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:30.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:54:29 smithi083 ceph-mon[77242]: pgmap v789: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:30.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:54:29 smithi153 ceph-mon[70044]: pgmap v789: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:30.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:54:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:54:30.208Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:54:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:54:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:54:31] "GET /metrics HTTP/1.1" 200 225160 "" "Prometheus/2.33.4" 2022-09-29T05:54:32.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:54:31 smithi083 ceph-mon[77242]: pgmap v790: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:32.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:54:31 smithi083 ceph-mon[79930]: pgmap v790: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:32.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:54:31 smithi153 ceph-mon[70044]: pgmap v790: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:54:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:54:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:54:33.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:54:33 smithi153 ceph-mon[70044]: pgmap v791: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:33.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:54:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:54:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:54:33.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:54:33 smithi083 ceph-mon[79930]: pgmap v791: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:33.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:54:33 smithi083 ceph-mon[77242]: pgmap v791: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:33.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:54:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:54:33.615Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:54:34.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:54:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:54:34.312Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:54:36.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:54:35 smithi083 ceph-mon[79930]: pgmap v792: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:36.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:54:35 smithi083 ceph-mon[77242]: pgmap v792: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:36.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:54:35 smithi153 ceph-mon[70044]: pgmap v792: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:38.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:54:37 smithi083 ceph-mon[79930]: pgmap v793: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:38.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:54:37 smithi083 ceph-mon[77242]: pgmap v793: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:38.259 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T05:54:38.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:54:37 smithi153 ceph-mon[70044]: pgmap v793: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:38.547 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T05:54:38.548 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (28m) 99s ago 28m 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T05:54:38.548 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (28m) 9m ago 28m 61.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T05:54:38.548 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (29m) 99s ago 29m 19.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T05:54:38.548 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (29m) 9m ago 29m 22.5M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T05:54:38.549 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (25m) 9m ago 32m 435M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T05:54:38.549 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (25m) 99s ago 33m 527M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T05:54:38.549 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (25m) 99s ago 33m 166M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T05:54:38.549 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (24m) 9m ago 32m 77.5M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T05:54:38.549 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (24m) 99s ago 32m 103M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T05:54:38.550 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (29m) 99s ago 29m 26.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T05:54:38.550 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (29m) 9m ago 29m 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T05:54:38.550 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (23m) 99s ago 31m 133M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T05:54:38.550 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (23m) 99s ago 31m 125M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T05:54:38.551 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (22m) 99s ago 31m 105M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T05:54:38.551 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (22m) 99s ago 31m 116M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T05:54:38.551 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (21m) 9m ago 30m 89.2M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T05:54:38.551 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (21m) 9m ago 30m 97.0M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T05:54:38.551 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (20m) 9m ago 30m 85.6M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T05:54:38.552 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (20m) 9m ago 29m 91.5M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T05:54:38.552 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (25m) 9m ago 29m 88.2M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T05:54:38.906 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T05:54:38.907 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T05:54:38.907 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T05:54:38.907 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:54:38.907 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T05:54:38.908 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T05:54:38.908 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:54:38.908 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T05:54:38.908 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T05:54:38.909 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:54:38.909 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T05:54:38.909 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T05:54:38.909 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:54:38.910 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T05:54:38.910 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T05:54:38.910 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T05:54:38.910 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T05:54:38.911 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T05:54:39.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:54:38 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2767030143' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:54:39.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:54:38 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2767030143' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:54:39.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:54:38 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2767030143' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:54:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:54:39 smithi153 ceph-mon[70044]: pgmap v794: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:54:39 smithi153 ceph-mon[70044]: from='client.35061 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:54:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:54:39 smithi153 ceph-mon[70044]: from='client.45001 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:54:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:54:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:54:40.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:54:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:54:40.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:54:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:54:40.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:54:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:54:40.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:54:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:54:40.208Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:54:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:54:39 smithi083 ceph-mon[77242]: pgmap v794: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:54:39 smithi083 ceph-mon[77242]: from='client.35061 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:54:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:54:39 smithi083 ceph-mon[77242]: from='client.45001 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:54:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:54:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:54:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:54:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:54:40.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:54:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:54:40.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:54:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:54:40.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:54:39 smithi083 ceph-mon[79930]: pgmap v794: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:40.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:54:39 smithi083 ceph-mon[79930]: from='client.35061 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:54:40.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:54:39 smithi083 ceph-mon[79930]: from='client.45001 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:54:40.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:54:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:54:40.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:54:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:54:40.467 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:54:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:54:40.467 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:54:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:54:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:54:40 smithi153 ceph-mon[70044]: pgmap v795: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:41.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:54:40 smithi083 ceph-mon[79930]: pgmap v795: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:41.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:54:40 smithi083 ceph-mon[77242]: pgmap v795: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:54:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:54:41] "GET /metrics HTTP/1.1" 200 225167 "" "Prometheus/2.33.4" 2022-09-29T05:54:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:54:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:54:42.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:54:43.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:54:43 smithi153 ceph-mon[70044]: pgmap v796: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:43.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:54:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:54:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:54:43.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:54:43 smithi083 ceph-mon[79930]: pgmap v796: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:43.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:54:43 smithi083 ceph-mon[77242]: pgmap v796: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:43.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:54:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:54:43.615Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:54:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:54:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:54:44.313Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:54:46.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:54:45 smithi083 ceph-mon[79930]: pgmap v797: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:46.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:54:45 smithi083 ceph-mon[77242]: pgmap v797: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:46.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:54:45 smithi153 ceph-mon[70044]: pgmap v797: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:48.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:54:47 smithi083 ceph-mon[79930]: pgmap v798: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:48.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:54:47 smithi083 ceph-mon[77242]: pgmap v798: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:48.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:54:47 smithi153 ceph-mon[70044]: pgmap v798: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:50.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:54:49 smithi083 ceph-mon[79930]: pgmap v799: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:50.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:54:49 smithi083 ceph-mon[77242]: pgmap v799: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:50.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:54:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:54:50.208Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:54:50.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:54:49 smithi153 ceph-mon[70044]: pgmap v799: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:54:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:54:51] "GET /metrics HTTP/1.1" 200 225167 "" "Prometheus/2.33.4" 2022-09-29T05:54:52.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:54:51 smithi153 ceph-mon[70044]: pgmap v800: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:52.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:54:51 smithi083 ceph-mon[79930]: pgmap v800: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:52.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:54:51 smithi083 ceph-mon[77242]: pgmap v800: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:52.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:54:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:54:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:54:53.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:54:52 smithi153 ceph-mon[70044]: pgmap v801: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:53.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:54:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:54:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:54:53.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:54:52 smithi083 ceph-mon[79930]: pgmap v801: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:53.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:54:52 smithi083 ceph-mon[77242]: pgmap v801: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:53.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:54:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:54:53.616Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:54:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:54:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:54:54.313Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:54:56.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:54:55 smithi083 ceph-mon[79930]: pgmap v802: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:56.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:54:55 smithi083 ceph-mon[77242]: pgmap v802: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:56.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:54:55 smithi153 ceph-mon[70044]: pgmap v802: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:58.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:54:57 smithi083 ceph-mon[79930]: pgmap v803: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:58.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:54:57 smithi083 ceph-mon[77242]: pgmap v803: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:54:58.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:54:57 smithi153 ceph-mon[70044]: pgmap v803: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:00.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:54:59 smithi153 ceph-mon[70044]: pgmap v804: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:00.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:54:59 smithi083 ceph-mon[79930]: pgmap v804: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:00.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:55:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:55:00.208Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:55:00.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:54:59 smithi083 ceph-mon[77242]: pgmap v804: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:01.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:55:00 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:55:01.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:55:00 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:55:01.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:55:00 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:55:01.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:55:00 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:55:01.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:55:00 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:55:01.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:55:00 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:55:01.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:55:00 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:55:01.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:55:00 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:55:01.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:55:00 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:55:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:55:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:55:01] "GET /metrics HTTP/1.1" 200 225166 "" "Prometheus/2.33.4" 2022-09-29T05:55:02.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:55:01 smithi153 ceph-mon[70044]: pgmap v805: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:02.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:55:01 smithi083 ceph-mon[79930]: pgmap v805: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:02.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:55:01 smithi083 ceph-mon[77242]: pgmap v805: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:55:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:55:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:55:03.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:55:02 smithi153 ceph-mon[70044]: pgmap v806: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:55:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:55:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:55:03.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:55:02 smithi083 ceph-mon[79930]: pgmap v806: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:03.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:55:02 smithi083 ceph-mon[77242]: pgmap v806: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:03.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:55:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:55:03.617Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:55:04.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:55:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:55:04.314Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:55:05.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:55:05 smithi083 ceph-mon[77242]: pgmap v807: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:05.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:55:05 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:55:05.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:55:05 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:55:05.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:55:05 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:55:05.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:55:05 smithi083 ceph-mon[79930]: pgmap v807: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:05.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:55:05 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:55:05.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:55:05 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:55:05.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:55:05 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:55:06.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:55:05 smithi153 ceph-mon[70044]: pgmap v807: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:06.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:55:05 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:55:06.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:55:05 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:55:06.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:55:05 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:55:07.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:55:07 smithi083 ceph-mon[77242]: pgmap v808: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:07.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:55:07 smithi083 ceph-mon[79930]: pgmap v808: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:08.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:55:07 smithi153 ceph-mon[70044]: pgmap v808: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:09.231 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T05:55:09.520 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T05:55:09.520 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (28m) 2m ago 29m 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T05:55:09.521 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (29m) 4s ago 29m 60.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T05:55:09.521 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (30m) 2m ago 30m 19.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T05:55:09.521 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (30m) 4s ago 30m 25.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T05:55:09.521 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (25m) 4s ago 32m 438M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T05:55:09.521 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (26m) 2m ago 34m 527M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T05:55:09.522 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (25m) 2m ago 34m 166M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T05:55:09.522 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (25m) 4s ago 32m 87.1M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T05:55:09.522 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (25m) 2m ago 33m 103M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T05:55:09.522 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (29m) 2m ago 29m 26.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T05:55:09.523 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (29m) 4s ago 29m 25.6M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T05:55:09.523 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (24m) 2m ago 32m 133M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T05:55:09.523 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (23m) 2m ago 32m 125M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T05:55:09.523 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (23m) 2m ago 31m 105M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T05:55:09.523 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (22m) 2m ago 31m 116M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T05:55:09.524 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (22m) 4s ago 31m 111M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T05:55:09.524 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (21m) 4s ago 31m 125M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T05:55:09.524 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (21m) 4s ago 30m 110M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T05:55:09.524 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (20m) 4s ago 30m 121M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T05:55:09.525 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (25m) 4s ago 29m 89.8M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T05:55:09.882 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T05:55:09.882 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T05:55:09.883 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T05:55:09.883 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:55:09.883 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T05:55:09.883 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T05:55:09.884 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:55:09.884 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T05:55:09.884 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T05:55:09.884 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:55:09.884 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T05:55:09.885 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T05:55:09.885 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:55:09.885 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T05:55:09.885 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T05:55:09.886 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T05:55:09.886 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T05:55:09.886 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T05:55:09.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:55:09 smithi083 ceph-mon[79930]: pgmap v809: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:09.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:55:09 smithi083 ceph-mon[77242]: pgmap v809: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:10.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:55:09 smithi153 ceph-mon[70044]: pgmap v809: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:10.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:55:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:55:10.209Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:55:10.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:55:10 smithi083 ceph-mon[79930]: from='client.45013 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:55:10.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:55:10 smithi083 ceph-mon[79930]: from='client.45019 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:55:10.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:55:10 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2568489206' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:55:10.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:55:10 smithi083 ceph-mon[77242]: from='client.45013 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:55:10.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:55:10 smithi083 ceph-mon[77242]: from='client.45019 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:55:10.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:55:10 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2568489206' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:55:11.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:55:10 smithi153 ceph-mon[70044]: from='client.45013 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:55:11.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:55:10 smithi153 ceph-mon[70044]: from='client.45019 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:55:11.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:55:10 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2568489206' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:55:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:55:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:55:11] "GET /metrics HTTP/1.1" 200 225164 "" "Prometheus/2.33.4" 2022-09-29T05:55:12.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:55:11 smithi153 ceph-mon[70044]: pgmap v810: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:12.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:55:11 smithi083 ceph-mon[79930]: pgmap v810: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:12.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:55:11 smithi083 ceph-mon[77242]: pgmap v810: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:55:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:55:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:55:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:55:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:55:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:55:13.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:55:12 smithi153 ceph-mon[70044]: pgmap v811: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:13.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:55:12 smithi083 ceph-mon[77242]: pgmap v811: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:13.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:55:12 smithi083 ceph-mon[79930]: pgmap v811: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:13.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:55:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:55:13.617Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:55:14.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:55:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:55:14.315Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:55:16.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:55:15 smithi153 ceph-mon[70044]: pgmap v812: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:16.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:55:15 smithi083 ceph-mon[79930]: pgmap v812: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:16.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:55:15 smithi083 ceph-mon[77242]: pgmap v812: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:17.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:55:17 smithi153 ceph-mon[70044]: pgmap v813: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:17.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:55:17 smithi083 ceph-mon[79930]: pgmap v813: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:17.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:55:17 smithi083 ceph-mon[77242]: pgmap v813: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:19.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:55:19 smithi083 ceph-mon[79930]: pgmap v814: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:19.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:55:19 smithi083 ceph-mon[77242]: pgmap v814: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:19.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:55:19 smithi153 ceph-mon[70044]: pgmap v814: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:20.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:55:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:55:20.210Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:55:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:55:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:55:21] "GET /metrics HTTP/1.1" 200 225164 "" "Prometheus/2.33.4" 2022-09-29T05:55:22.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:55:21 smithi153 ceph-mon[70044]: pgmap v815: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:22.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:55:21 smithi083 ceph-mon[77242]: pgmap v815: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:22.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:55:21 smithi083 ceph-mon[79930]: pgmap v815: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:55:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:55:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:55:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:55:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:55:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:55:23.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:55:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:55:23.617Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:55:24.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:55:23 smithi153 ceph-mon[70044]: pgmap v816: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:24.318 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:55:23 smithi083 ceph-mon[79930]: pgmap v816: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:24.318 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:55:23 smithi083 ceph-mon[77242]: pgmap v816: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:55:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:55:24.316Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:55:26.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:55:25 smithi153 ceph-mon[70044]: pgmap v817: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:26.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:55:25 smithi083 ceph-mon[77242]: pgmap v817: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:26.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:55:25 smithi083 ceph-mon[79930]: pgmap v817: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:27.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:55:26 smithi153 ceph-mon[70044]: pgmap v818: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:27.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:55:26 smithi083 ceph-mon[79930]: pgmap v818: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:27.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:55:26 smithi083 ceph-mon[77242]: pgmap v818: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:29.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:55:29 smithi083 ceph-mon[79930]: pgmap v819: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:29.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:55:29 smithi083 ceph-mon[77242]: pgmap v819: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:29.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:55:29 smithi153 ceph-mon[70044]: pgmap v819: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:30.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:55:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:55:30.211Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:55:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:55:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:55:31] "GET /metrics HTTP/1.1" 200 225169 "" "Prometheus/2.33.4" 2022-09-29T05:55:32.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:55:31 smithi153 ceph-mon[70044]: pgmap v820: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:32.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:55:31 smithi083 ceph-mon[77242]: pgmap v820: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:32.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:55:31 smithi083 ceph-mon[79930]: pgmap v820: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:55:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:55:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:55:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:55:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:55:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:55:33.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:55:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:55:33.618Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:55:34.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:55:33 smithi153 ceph-mon[70044]: pgmap v821: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:34.318 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:55:33 smithi083 ceph-mon[79930]: pgmap v821: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:34.319 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:55:33 smithi083 ceph-mon[77242]: pgmap v821: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:34.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:55:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:55:34.316Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:55:36.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:55:35 smithi153 ceph-mon[70044]: pgmap v822: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:36.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:55:35 smithi083 ceph-mon[79930]: pgmap v822: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:36.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:55:35 smithi083 ceph-mon[77242]: pgmap v822: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:38.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:55:37 smithi153 ceph-mon[70044]: pgmap v823: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:38.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:55:37 smithi083 ceph-mon[79930]: pgmap v823: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:38.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:55:37 smithi083 ceph-mon[77242]: pgmap v823: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:39.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:55:38 smithi153 ceph-mon[70044]: pgmap v824: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:39.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:55:38 smithi083 ceph-mon[79930]: pgmap v824: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:39.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:55:38 smithi083 ceph-mon[77242]: pgmap v824: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:40.210 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T05:55:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:55:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:55:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:55:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:55:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:55:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:55:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:55:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:55:40.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:55:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:55:40.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:55:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:55:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:55:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:55:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:55:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:55:40.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:55:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:55:40.211Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:55:40.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:55:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:55:40.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:55:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:55:40.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:55:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:55:40.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:55:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:55:40.500 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T05:55:40.500 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (29m) 2m ago 29m 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T05:55:40.500 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (29m) 35s ago 29m 60.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T05:55:40.501 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (30m) 2m ago 30m 19.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T05:55:40.501 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (30m) 35s ago 30m 25.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T05:55:40.501 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (26m) 35s ago 33m 438M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T05:55:40.501 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (26m) 2m ago 34m 527M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T05:55:40.502 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (26m) 2m ago 34m 166M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T05:55:40.502 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (25m) 35s ago 33m 87.1M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T05:55:40.502 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (25m) 2m ago 33m 103M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T05:55:40.502 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (30m) 2m ago 30m 26.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T05:55:40.503 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (30m) 35s ago 30m 25.6M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T05:55:40.503 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (24m) 2m ago 33m 133M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T05:55:40.503 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (24m) 2m ago 32m 125M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T05:55:40.503 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (24m) 2m ago 32m 105M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T05:55:40.503 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (23m) 2m ago 32m 116M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T05:55:40.504 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (22m) 35s ago 31m 111M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T05:55:40.504 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (22m) 35s ago 31m 125M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T05:55:40.504 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (21m) 35s ago 31m 110M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T05:55:40.504 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (21m) 35s ago 30m 121M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T05:55:40.504 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (26m) 35s ago 30m 89.8M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T05:55:40.857 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T05:55:40.858 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T05:55:40.858 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T05:55:40.858 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:55:40.858 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T05:55:40.859 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T05:55:40.859 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:55:40.859 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T05:55:40.859 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T05:55:40.860 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:55:40.860 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T05:55:40.860 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T05:55:40.860 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:55:40.860 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T05:55:40.861 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T05:55:40.861 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T05:55:40.861 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T05:55:40.861 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T05:55:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:55:40 smithi153 ceph-mon[70044]: pgmap v825: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:55:40 smithi153 ceph-mon[70044]: from='client.35097 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:55:41.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:55:40 smithi153 ceph-mon[70044]: from='client.45037 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:55:41.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:55:40 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4163978124' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:55:41.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:55:40 smithi083 ceph-mon[77242]: pgmap v825: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:41.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:55:40 smithi083 ceph-mon[77242]: from='client.35097 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:55:41.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:55:40 smithi083 ceph-mon[77242]: from='client.45037 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:55:41.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:55:40 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4163978124' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:55:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:55:40 smithi083 ceph-mon[79930]: pgmap v825: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:41.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:55:40 smithi083 ceph-mon[79930]: from='client.35097 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:55:41.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:55:40 smithi083 ceph-mon[79930]: from='client.45037 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:55:41.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:55:40 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4163978124' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:55:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:55:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:55:41] "GET /metrics HTTP/1.1" 200 225169 "" "Prometheus/2.33.4" 2022-09-29T05:55:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:55:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:55:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:55:43.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:55:43 smithi153 ceph-mon[70044]: pgmap v826: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:43.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:55:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:55:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:55:43.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:55:43 smithi083 ceph-mon[79930]: pgmap v826: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:43.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:55:43 smithi083 ceph-mon[77242]: pgmap v826: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:43.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:55:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:55:43.618Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:55:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:55:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:55:44.316Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:55:46.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:55:45 smithi153 ceph-mon[70044]: pgmap v827: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:46.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:55:45 smithi083 ceph-mon[79930]: pgmap v827: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:46.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:55:45 smithi083 ceph-mon[77242]: pgmap v827: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:48.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:55:47 smithi153 ceph-mon[70044]: pgmap v828: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:48.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:55:47 smithi083 ceph-mon[79930]: pgmap v828: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:48.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:55:47 smithi083 ceph-mon[77242]: pgmap v828: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:49.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:55:48 smithi153 ceph-mon[70044]: pgmap v829: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:49.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:55:48 smithi083 ceph-mon[79930]: pgmap v829: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:49.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:55:48 smithi083 ceph-mon[77242]: pgmap v829: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:50.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:55:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:55:50.212Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:55:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:55:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:55:51] "GET /metrics HTTP/1.1" 200 225169 "" "Prometheus/2.33.4" 2022-09-29T05:55:52.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:55:51 smithi153 ceph-mon[70044]: pgmap v830: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:52.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:55:51 smithi083 ceph-mon[79930]: pgmap v830: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:52.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:55:51 smithi083 ceph-mon[77242]: pgmap v830: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:55:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:55:52.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:55:53.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:55:53 smithi153 ceph-mon[70044]: pgmap v831: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:53.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:55:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:55:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:55:53.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:55:53 smithi083 ceph-mon[77242]: pgmap v831: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:53.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:55:53 smithi083 ceph-mon[79930]: pgmap v831: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:53.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:55:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:55:53.619Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:55:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:55:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:55:54.317Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:55:56.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:55:55 smithi153 ceph-mon[70044]: pgmap v832: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:56.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:55:55 smithi083 ceph-mon[79930]: pgmap v832: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:56.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:55:55 smithi083 ceph-mon[77242]: pgmap v832: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:57.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:55:56 smithi153 ceph-mon[70044]: pgmap v833: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:57.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:55:56 smithi083 ceph-mon[77242]: pgmap v833: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:57.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:55:56 smithi083 ceph-mon[79930]: pgmap v833: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:59.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:55:59 smithi083 ceph-mon[79930]: pgmap v834: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:59.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:55:59 smithi083 ceph-mon[77242]: pgmap v834: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:55:59.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:55:59 smithi153 ceph-mon[70044]: pgmap v834: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:00.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:56:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:56:00.212Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:56:01.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:56:01 smithi153 ceph-mon[70044]: pgmap v835: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:01.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:56:01 smithi083 ceph-mon[79930]: pgmap v835: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:01.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:56:01 smithi083 ceph-mon[77242]: pgmap v835: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:56:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:56:01] "GET /metrics HTTP/1.1" 200 225167 "" "Prometheus/2.33.4" 2022-09-29T05:56:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:56:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:56:02.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:56:03.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:56:03 smithi153 ceph-mon[70044]: pgmap v836: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:56:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:56:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:56:03.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:56:03 smithi083 ceph-mon[77242]: pgmap v836: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:03.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:56:03 smithi083 ceph-mon[79930]: pgmap v836: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:03.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:56:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:56:03.620Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:56:04.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:56:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:56:04.317Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:56:06.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:56:05 smithi153 ceph-mon[70044]: pgmap v837: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:06.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:56:05 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:56:06.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:56:05 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:56:06.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:56:05 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:56:06.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:56:05 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:56:06.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:56:05 smithi083 ceph-mon[79930]: pgmap v837: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:06.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:56:05 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:56:06.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:56:05 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:56:06.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:56:05 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:56:06.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:56:05 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:56:06.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:56:05 smithi083 ceph-mon[77242]: pgmap v837: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:06.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:56:05 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:56:06.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:56:05 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:56:06.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:56:05 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:56:06.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:56:05 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:56:07.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:56:06 smithi153 ceph-mon[70044]: pgmap v838: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:07.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:56:06 smithi083 ceph-mon[77242]: pgmap v838: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:07.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:56:06 smithi083 ceph-mon[79930]: pgmap v838: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:09.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:56:09 smithi083 ceph-mon[79930]: pgmap v839: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:09.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:56:09 smithi083 ceph-mon[77242]: pgmap v839: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:09.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:56:09 smithi153 ceph-mon[70044]: pgmap v839: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:10.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:56:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:56:10.214Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:56:11.184 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T05:56:11.473 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T05:56:11.473 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (29m) 3m ago 30m 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T05:56:11.474 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (30m) 66s ago 30m 60.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T05:56:11.474 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (31m) 3m ago 31m 19.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T05:56:11.474 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (31m) 66s ago 31m 25.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T05:56:11.474 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (26m) 66s ago 33m 438M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T05:56:11.475 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (27m) 3m ago 35m 527M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T05:56:11.475 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (26m) 3m ago 35m 166M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T05:56:11.475 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (26m) 66s ago 34m 87.1M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T05:56:11.475 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (26m) 3m ago 34m 103M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T05:56:11.476 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (30m) 3m ago 30m 26.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T05:56:11.476 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (30m) 66s ago 30m 25.6M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T05:56:11.476 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (25m) 3m ago 33m 133M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T05:56:11.476 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (24m) 3m ago 33m 125M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T05:56:11.477 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (24m) 3m ago 32m 105M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T05:56:11.477 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (23m) 3m ago 32m 116M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T05:56:11.477 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (23m) 66s ago 32m 111M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T05:56:11.477 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (22m) 66s ago 32m 125M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T05:56:11.477 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (22m) 66s ago 31m 110M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T05:56:11.478 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (21m) 66s ago 31m 121M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T05:56:11.478 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (27m) 66s ago 30m 89.8M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T05:56:11.827 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T05:56:11.827 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T05:56:11.827 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T05:56:11.828 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:56:11.828 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T05:56:11.828 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T05:56:11.828 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:56:11.828 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T05:56:11.829 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T05:56:11.829 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:56:11.829 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T05:56:11.829 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T05:56:11.829 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:56:11.830 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T05:56:11.830 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T05:56:11.830 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T05:56:11.830 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T05:56:11.831 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T05:56:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:56:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:56:11] "GET /metrics HTTP/1.1" 200 225158 "" "Prometheus/2.33.4" 2022-09-29T05:56:12.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:56:11 smithi153 ceph-mon[70044]: pgmap v840: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:12.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:56:11 smithi153 ceph-mon[70044]: from='client.45049 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:56:12.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:56:11 smithi153 ceph-mon[70044]: from='client.45055 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:56:12.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:56:11 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1448195221' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:56:12.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:56:11 smithi083 ceph-mon[79930]: pgmap v840: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:12.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:56:11 smithi083 ceph-mon[79930]: from='client.45049 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:56:12.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:56:11 smithi083 ceph-mon[79930]: from='client.45055 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:56:12.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:56:11 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1448195221' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:56:12.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:56:11 smithi083 ceph-mon[77242]: pgmap v840: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:12.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:56:11 smithi083 ceph-mon[77242]: from='client.45049 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:56:12.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:56:11 smithi083 ceph-mon[77242]: from='client.45055 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:56:12.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:56:11 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1448195221' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:56:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:56:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:56:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:56:13.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:56:12 smithi153 ceph-mon[70044]: pgmap v841: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:13.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:56:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:56:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:56:13.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:56:12 smithi083 ceph-mon[79930]: pgmap v841: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:13.491 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:56:12 smithi083 ceph-mon[77242]: pgmap v841: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:13.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:56:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:56:13.621Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:56:14.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:56:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:56:14.318Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:56:15.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:56:15 smithi083 ceph-mon[79930]: pgmap v842: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:15.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:56:15 smithi083 ceph-mon[77242]: pgmap v842: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:15.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:56:15 smithi153 ceph-mon[70044]: pgmap v842: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:17.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:56:17 smithi083 ceph-mon[79930]: pgmap v843: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:17.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:56:17 smithi083 ceph-mon[77242]: pgmap v843: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:17.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:56:17 smithi153 ceph-mon[70044]: pgmap v843: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:19.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:56:19 smithi083 ceph-mon[79930]: pgmap v844: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:19.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:56:19 smithi083 ceph-mon[77242]: pgmap v844: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:19.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:56:19 smithi153 ceph-mon[70044]: pgmap v844: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:20.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:56:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:56:20.214Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:56:21.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:56:20 smithi153 ceph-mon[70044]: pgmap v845: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:21.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:56:20 smithi083 ceph-mon[77242]: pgmap v845: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:21.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:56:20 smithi083 ceph-mon[79930]: pgmap v845: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:56:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:56:21] "GET /metrics HTTP/1.1" 200 225158 "" "Prometheus/2.33.4" 2022-09-29T05:56:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:56:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:56:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:56:23.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:56:23 smithi153 ceph-mon[70044]: pgmap v846: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:23.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:56:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:56:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:56:23.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:56:23 smithi083 ceph-mon[79930]: pgmap v846: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:23.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:56:23 smithi083 ceph-mon[77242]: pgmap v846: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:23.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:56:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:56:23.622Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:56:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:56:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:56:24.319Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:56:25.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:56:25 smithi083 ceph-mon[79930]: pgmap v847: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:25.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:56:25 smithi083 ceph-mon[77242]: pgmap v847: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:25.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:56:25 smithi153 ceph-mon[70044]: pgmap v847: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:27.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:56:27 smithi083 ceph-mon[79930]: pgmap v848: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:27.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:56:27 smithi083 ceph-mon[77242]: pgmap v848: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:27.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:56:27 smithi153 ceph-mon[70044]: pgmap v848: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:29.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:56:29 smithi083 ceph-mon[79930]: pgmap v849: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:29.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:56:29 smithi083 ceph-mon[77242]: pgmap v849: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:29.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:56:29 smithi153 ceph-mon[70044]: pgmap v849: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:30.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:56:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:56:30.215Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:56:31.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:56:30 smithi153 ceph-mon[70044]: pgmap v850: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:31.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:56:30 smithi083 ceph-mon[77242]: pgmap v850: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:31.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:56:30 smithi083 ceph-mon[79930]: pgmap v850: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:56:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:56:31] "GET /metrics HTTP/1.1" 200 225166 "" "Prometheus/2.33.4" 2022-09-29T05:56:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:56:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:56:32.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:56:33.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:56:33 smithi153 ceph-mon[70044]: pgmap v851: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:33.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:56:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:56:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:56:33.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:56:33 smithi083 ceph-mon[79930]: pgmap v851: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:33.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:56:33 smithi083 ceph-mon[77242]: pgmap v851: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:33.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:56:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:56:33.622Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:56:34.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:56:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:56:34.319Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:56:35.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:56:35 smithi083 ceph-mon[77242]: pgmap v852: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:35.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:56:35 smithi083 ceph-mon[79930]: pgmap v852: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:35.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:56:35 smithi153 ceph-mon[70044]: pgmap v852: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:37.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:56:37 smithi083 ceph-mon[79930]: pgmap v853: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:37.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:56:37 smithi083 ceph-mon[77242]: pgmap v853: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:37.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:56:37 smithi153 ceph-mon[70044]: pgmap v853: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:39.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:56:39 smithi083 ceph-mon[79930]: pgmap v854: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:39.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:56:39 smithi083 ceph-mon[77242]: pgmap v854: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:39.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:56:39 smithi153 ceph-mon[70044]: pgmap v854: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:56:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:56:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:56:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:56:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:56:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:56:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:56:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:56:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:56:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:56:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:56:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:56:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:56:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:56:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:56:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:56:40.465 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:56:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:56:40.215Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:56:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:56:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:56:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:56:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:56:40.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:56:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:56:40.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:56:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:56:41.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:56:41 smithi083 ceph-mon[79930]: pgmap v855: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:41.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:56:41 smithi083 ceph-mon[77242]: pgmap v855: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:56:41 smithi153 ceph-mon[70044]: pgmap v855: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:56:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:56:41] "GET /metrics HTTP/1.1" 200 225169 "" "Prometheus/2.33.4" 2022-09-29T05:56:42.151 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T05:56:42.438 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T05:56:42.438 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (30m) 3m ago 30m 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T05:56:42.438 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (30m) 97s ago 30m 60.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T05:56:42.439 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (31m) 3m ago 31m 19.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T05:56:42.439 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (31m) 97s ago 31m 25.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T05:56:42.439 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (27m) 97s ago 34m 438M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T05:56:42.439 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (27m) 3m ago 35m 527M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T05:56:42.439 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (27m) 3m ago 35m 166M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T05:56:42.440 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (26m) 97s ago 34m 87.1M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T05:56:42.440 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (26m) 3m ago 34m 103M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T05:56:42.440 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (31m) 3m ago 31m 26.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T05:56:42.440 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (31m) 97s ago 31m 25.6M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T05:56:42.441 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (25m) 3m ago 34m 133M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T05:56:42.441 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (25m) 3m ago 33m 125M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T05:56:42.441 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (25m) 3m ago 33m 105M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T05:56:42.441 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (24m) 3m ago 33m 116M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T05:56:42.442 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (23m) 97s ago 32m 111M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T05:56:42.442 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (23m) 97s ago 32m 125M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T05:56:42.442 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (22m) 97s ago 32m 110M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T05:56:42.442 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (22m) 97s ago 32m 121M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T05:56:42.442 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (27m) 97s ago 31m 89.8M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T05:56:42.793 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T05:56:42.794 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T05:56:42.794 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T05:56:42.794 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:56:42.795 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T05:56:42.795 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T05:56:42.795 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:56:42.795 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T05:56:42.796 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T05:56:42.796 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:56:42.796 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T05:56:42.796 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T05:56:42.797 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:56:42.797 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T05:56:42.797 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T05:56:42.797 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T05:56:42.798 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T05:56:42.798 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T05:56:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:56:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:56:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:56:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:56:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:56:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:56:43.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:56:43 smithi153 ceph-mon[70044]: pgmap v856: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:43.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:56:43 smithi153 ceph-mon[70044]: from='client.45067 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:56:43.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:56:43 smithi153 ceph-mon[70044]: from='client.45073 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:56:43.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:56:43 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4023295449' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:56:43.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:56:43 smithi083 ceph-mon[79930]: pgmap v856: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:43.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:56:43 smithi083 ceph-mon[79930]: from='client.45067 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:56:43.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:56:43 smithi083 ceph-mon[79930]: from='client.45073 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:56:43.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:56:43 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4023295449' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:56:43.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:56:43 smithi083 ceph-mon[77242]: pgmap v856: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:43.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:56:43 smithi083 ceph-mon[77242]: from='client.45067 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:56:43.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:56:43 smithi083 ceph-mon[77242]: from='client.45073 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:56:43.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:56:43 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4023295449' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:56:43.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:56:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:56:43.622Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:56:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:56:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:56:44.319Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:56:45.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:56:45 smithi083 ceph-mon[79930]: pgmap v857: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:45.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:56:45 smithi083 ceph-mon[77242]: pgmap v857: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:45.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:56:45 smithi153 ceph-mon[70044]: pgmap v857: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:47.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:56:47 smithi083 ceph-mon[79930]: pgmap v858: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:47.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:56:47 smithi083 ceph-mon[77242]: pgmap v858: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:47.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:56:47 smithi153 ceph-mon[70044]: pgmap v858: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:49.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:56:49 smithi083 ceph-mon[77242]: pgmap v859: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:49.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:56:49 smithi083 ceph-mon[79930]: pgmap v859: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:49.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:56:49 smithi153 ceph-mon[70044]: pgmap v859: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:50.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:56:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:56:50.215Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:56:51.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:56:50 smithi153 ceph-mon[70044]: pgmap v860: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:51.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:56:50 smithi083 ceph-mon[79930]: pgmap v860: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:51.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:56:50 smithi083 ceph-mon[77242]: pgmap v860: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:56:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:56:51] "GET /metrics HTTP/1.1" 200 225169 "" "Prometheus/2.33.4" 2022-09-29T05:56:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:56:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:56:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:56:53.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:56:53 smithi153 ceph-mon[70044]: pgmap v861: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:53.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:56:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:56:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:56:53.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:56:53 smithi083 ceph-mon[79930]: pgmap v861: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:53.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:56:53 smithi083 ceph-mon[77242]: pgmap v861: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:53.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:56:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:56:53.622Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:56:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:56:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:56:54.320Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:56:55.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:56:55 smithi083 ceph-mon[79930]: pgmap v862: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:55.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:56:55 smithi083 ceph-mon[77242]: pgmap v862: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:55.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:56:55 smithi153 ceph-mon[70044]: pgmap v862: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:57.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:56:57 smithi083 ceph-mon[79930]: pgmap v863: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:57.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:56:57 smithi083 ceph-mon[77242]: pgmap v863: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:57.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:56:57 smithi153 ceph-mon[70044]: pgmap v863: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:59.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:56:59 smithi083 ceph-mon[79930]: pgmap v864: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:59.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:56:59 smithi083 ceph-mon[77242]: pgmap v864: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:56:59.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:56:59 smithi153 ceph-mon[70044]: pgmap v864: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:00.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:57:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:57:00.216Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:57:01.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:57:00 smithi153 ceph-mon[70044]: pgmap v865: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:01.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:57:00 smithi083 ceph-mon[77242]: pgmap v865: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:01.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:57:00 smithi083 ceph-mon[79930]: pgmap v865: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:57:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:57:01] "GET /metrics HTTP/1.1" 200 225167 "" "Prometheus/2.33.4" 2022-09-29T05:57:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:57:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:57:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:57:03.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:57:03 smithi153 ceph-mon[70044]: pgmap v866: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:57:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:57:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:57:03.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:57:03 smithi083 ceph-mon[79930]: pgmap v866: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:03.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:57:03 smithi083 ceph-mon[77242]: pgmap v866: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:03.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:57:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:57:03.623Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:57:04.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:57:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:57:04.321Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:57:05.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:57:05 smithi083 ceph-mon[77242]: pgmap v867: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:05.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:57:05 smithi083 ceph-mon[79930]: pgmap v867: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:05.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:57:05 smithi153 ceph-mon[70044]: pgmap v867: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:06.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:57:06 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:57:06.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:57:06 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:57:06.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:57:06 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:57:06.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:57:06 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:57:06.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:57:06 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:57:06.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:57:06 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:57:06.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:57:06 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:57:06.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:57:06 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:57:06.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:57:06 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:57:07.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:57:07 smithi153 ceph-mon[70044]: pgmap v868: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:07.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:57:07 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:57:07.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:57:07 smithi083 ceph-mon[77242]: pgmap v868: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:07.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:57:07 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:57:07.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:57:07 smithi083 ceph-mon[79930]: pgmap v868: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:07.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:57:07 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:57:09.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:57:09 smithi153 ceph-mon[70044]: pgmap v869: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:09.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:57:09 smithi083 ceph-mon[79930]: pgmap v869: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:09.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:57:09 smithi083 ceph-mon[77242]: pgmap v869: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:10.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:57:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:57:10.217Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:57:11.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:57:10 smithi153 ceph-mon[70044]: pgmap v870: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:11.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:57:10 smithi083 ceph-mon[79930]: pgmap v870: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:11.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:57:10 smithi083 ceph-mon[77242]: pgmap v870: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:57:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:57:11] "GET /metrics HTTP/1.1" 200 225168 "" "Prometheus/2.33.4" 2022-09-29T05:57:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:57:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:57:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:57:13.118 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T05:57:13.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:57:13 smithi153 ceph-mon[70044]: pgmap v871: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:13.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:57:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:57:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:57:13.414 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T05:57:13.415 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (30m) 4m ago 31m 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T05:57:13.415 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (31m) 2m ago 31m 60.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T05:57:13.415 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (32m) 4m ago 32m 19.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T05:57:13.415 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (32m) 2m ago 32m 25.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T05:57:13.415 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (27m) 2m ago 34m 438M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T05:57:13.416 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (28m) 4m ago 36m 527M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T05:57:13.416 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (27m) 4m ago 36m 166M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T05:57:13.416 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (27m) 2m ago 35m 87.1M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T05:57:13.416 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (27m) 4m ago 35m 103M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T05:57:13.417 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (31m) 4m ago 31m 26.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T05:57:13.417 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (31m) 2m ago 31m 25.6M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T05:57:13.417 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (26m) 4m ago 34m 133M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T05:57:13.417 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (25m) 4m ago 34m 125M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T05:57:13.417 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (25m) 4m ago 33m 105M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T05:57:13.418 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (24m) 4m ago 33m 116M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T05:57:13.418 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (24m) 2m ago 33m 111M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T05:57:13.418 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (23m) 2m ago 33m 125M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T05:57:13.418 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (23m) 2m ago 32m 110M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T05:57:13.418 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (22m) 2m ago 32m 121M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T05:57:13.419 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (28m) 2m ago 31m 89.8M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T05:57:13.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:57:13 smithi083 ceph-mon[77242]: pgmap v871: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:13.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:57:13 smithi083 ceph-mon[79930]: pgmap v871: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:13.784 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T05:57:13.785 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T05:57:13.785 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T05:57:13.785 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:57:13.785 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T05:57:13.786 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T05:57:13.786 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:57:13.786 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T05:57:13.786 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T05:57:13.786 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:57:13.787 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T05:57:13.787 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T05:57:13.787 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:57:13.787 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T05:57:13.788 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T05:57:13.788 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T05:57:13.788 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T05:57:13.788 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T05:57:13.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:57:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:57:13.623Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:57:14.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:57:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:57:14.321Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:57:14.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:57:14 smithi083 ceph-mon[77242]: from='client.45085 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:57:14.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:57:14 smithi083 ceph-mon[77242]: from='client.45091 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:57:14.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:57:14 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/764586103' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:57:14.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:57:14 smithi083 ceph-mon[79930]: from='client.45085 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:57:14.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:57:14 smithi083 ceph-mon[79930]: from='client.45091 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:57:14.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:57:14 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/764586103' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:57:14.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:57:14 smithi153 ceph-mon[70044]: from='client.45085 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:57:14.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:57:14 smithi153 ceph-mon[70044]: from='client.45091 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:57:14.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:57:14 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/764586103' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:57:15.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:57:15 smithi083 ceph-mon[79930]: pgmap v872: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:15.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:57:15 smithi083 ceph-mon[77242]: pgmap v872: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:15.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:57:15 smithi153 ceph-mon[70044]: pgmap v872: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:17.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:57:17 smithi083 ceph-mon[79930]: pgmap v873: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:17.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:57:17 smithi083 ceph-mon[77242]: pgmap v873: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:17.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:57:17 smithi153 ceph-mon[70044]: pgmap v873: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:19.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:57:19 smithi083 ceph-mon[77242]: pgmap v874: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:19.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:57:19 smithi083 ceph-mon[79930]: pgmap v874: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:19.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:57:19 smithi153 ceph-mon[70044]: pgmap v874: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:20.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:57:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:57:20.217Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:57:21.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:57:20 smithi153 ceph-mon[70044]: pgmap v875: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:21.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:57:20 smithi083 ceph-mon[79930]: pgmap v875: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:21.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:57:20 smithi083 ceph-mon[77242]: pgmap v875: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:57:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:57:21] "GET /metrics HTTP/1.1" 200 225168 "" "Prometheus/2.33.4" 2022-09-29T05:57:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:57:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:57:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:57:23.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:57:23 smithi153 ceph-mon[70044]: pgmap v876: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:23.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:57:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:57:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:57:23.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:57:23 smithi083 ceph-mon[79930]: pgmap v876: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:23.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:57:23 smithi083 ceph-mon[77242]: pgmap v876: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:23.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:57:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:57:23.624Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:57:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:57:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:57:24.323Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:57:25.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:57:25 smithi083 ceph-mon[79930]: pgmap v877: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:25.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:57:25 smithi083 ceph-mon[77242]: pgmap v877: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:25.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:57:25 smithi153 ceph-mon[70044]: pgmap v877: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:27.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:57:27 smithi083 ceph-mon[79930]: pgmap v878: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:27.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:57:27 smithi083 ceph-mon[77242]: pgmap v878: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:27.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:57:27 smithi153 ceph-mon[70044]: pgmap v878: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:29.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:57:29 smithi083 ceph-mon[79930]: pgmap v879: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:29.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:57:29 smithi083 ceph-mon[77242]: pgmap v879: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:29.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:57:29 smithi153 ceph-mon[70044]: pgmap v879: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:30.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:57:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:57:30.218Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:57:31.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:57:30 smithi153 ceph-mon[70044]: pgmap v880: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:31.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:57:30 smithi083 ceph-mon[79930]: pgmap v880: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:31.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:57:30 smithi083 ceph-mon[77242]: pgmap v880: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:57:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:57:31] "GET /metrics HTTP/1.1" 200 225170 "" "Prometheus/2.33.4" 2022-09-29T05:57:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:57:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:57:32.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:57:33.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:57:33 smithi153 ceph-mon[70044]: pgmap v881: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:33.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:57:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:57:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:57:33.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:57:33 smithi083 ceph-mon[79930]: pgmap v881: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:33.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:57:33 smithi083 ceph-mon[77242]: pgmap v881: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:33.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:57:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:57:33.625Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:57:34.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:57:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:57:34.324Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:57:35.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:57:35 smithi083 ceph-mon[79930]: pgmap v882: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:35.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:57:35 smithi083 ceph-mon[77242]: pgmap v882: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:35.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:57:35 smithi153 ceph-mon[70044]: pgmap v882: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:37.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:57:37 smithi083 ceph-mon[79930]: pgmap v883: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:37.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:57:37 smithi083 ceph-mon[77242]: pgmap v883: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:37.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:57:37 smithi153 ceph-mon[70044]: pgmap v883: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:39.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:57:39 smithi083 ceph-mon[77242]: pgmap v884: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:39.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:57:39 smithi083 ceph-mon[79930]: pgmap v884: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:39.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:57:39 smithi153 ceph-mon[70044]: pgmap v884: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:57:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:57:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:57:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:57:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:57:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:57:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:57:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:57:40.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:57:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:57:40.217Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:57:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:57:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:57:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:57:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:57:40.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:57:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:57:40.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:57:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:57:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:57:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:57:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:57:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:57:40.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:57:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:57:40.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:57:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:57:41.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:57:41 smithi083 ceph-mon[77242]: pgmap v885: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:57:41 smithi083 ceph-mon[79930]: pgmap v885: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:57:41 smithi153 ceph-mon[70044]: pgmap v885: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:57:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:57:41] "GET /metrics HTTP/1.1" 200 225170 "" "Prometheus/2.33.4" 2022-09-29T05:57:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:57:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:57:42.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:57:43.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:57:43 smithi153 ceph-mon[70044]: pgmap v886: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:43.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:57:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:57:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:57:43.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:57:43 smithi083 ceph-mon[77242]: pgmap v886: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:43.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:57:43 smithi083 ceph-mon[79930]: pgmap v886: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:43.968 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:57:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:57:43.626Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:57:44.108 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T05:57:44.399 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T05:57:44.399 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (31m) 4m ago 31m 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T05:57:44.400 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (31m) 2m ago 31m 60.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T05:57:44.400 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (32m) 4m ago 32m 19.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T05:57:44.400 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (32m) 2m ago 32m 25.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T05:57:44.400 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (28m) 2m ago 35m 438M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T05:57:44.401 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (28m) 4m ago 36m 527M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T05:57:44.401 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (28m) 4m ago 36m 166M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T05:57:44.401 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (27m) 2m ago 35m 87.1M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T05:57:44.401 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (27m) 4m ago 35m 103M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T05:57:44.402 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (32m) 4m ago 32m 26.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T05:57:44.402 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (32m) 2m ago 32m 25.6M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T05:57:44.402 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (26m) 4m ago 35m 133M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T05:57:44.402 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (26m) 4m ago 34m 125M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T05:57:44.403 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (26m) 4m ago 34m 105M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T05:57:44.403 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (25m) 4m ago 34m 116M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T05:57:44.403 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (24m) 2m ago 33m 111M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T05:57:44.403 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (24m) 2m ago 33m 125M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T05:57:44.403 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (23m) 2m ago 33m 110M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T05:57:44.404 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (23m) 2m ago 33m 121M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T05:57:44.404 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (28m) 2m ago 32m 89.8M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T05:57:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:57:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:57:44.324Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:57:44.754 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T05:57:44.755 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T05:57:44.755 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T05:57:44.755 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:57:44.755 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T05:57:44.756 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T05:57:44.756 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:57:44.756 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T05:57:44.756 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T05:57:44.757 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:57:44.757 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T05:57:44.757 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T05:57:44.757 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:57:44.758 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T05:57:44.758 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T05:57:44.758 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T05:57:44.758 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T05:57:44.758 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T05:57:45.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:57:45 smithi083 ceph-mon[77242]: from='client.35169 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:57:45.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:57:45 smithi083 ceph-mon[77242]: pgmap v887: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:45.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:57:45 smithi083 ceph-mon[77242]: from='client.45109 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:57:45.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:57:45 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/638198025' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:57:45.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:57:45 smithi083 ceph-mon[79930]: from='client.35169 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:57:45.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:57:45 smithi083 ceph-mon[79930]: pgmap v887: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:45.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:57:45 smithi083 ceph-mon[79930]: from='client.45109 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:57:45.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:57:45 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/638198025' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:57:45.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:57:45 smithi153 ceph-mon[70044]: from='client.35169 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:57:45.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:57:45 smithi153 ceph-mon[70044]: pgmap v887: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:45.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:57:45 smithi153 ceph-mon[70044]: from='client.45109 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:57:45.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:57:45 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/638198025' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:57:47.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:57:47 smithi083 ceph-mon[79930]: pgmap v888: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:47.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:57:47 smithi083 ceph-mon[77242]: pgmap v888: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:47.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:57:47 smithi153 ceph-mon[70044]: pgmap v888: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:49.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:57:49 smithi083 ceph-mon[79930]: pgmap v889: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:49.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:57:49 smithi083 ceph-mon[77242]: pgmap v889: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:49.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:57:49 smithi153 ceph-mon[70044]: pgmap v889: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:50.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:57:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:57:50.218Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:57:51.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:57:51 smithi153 ceph-mon[70044]: pgmap v890: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:51.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:57:51 smithi083 ceph-mon[79930]: pgmap v890: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:51.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:57:51 smithi083 ceph-mon[77242]: pgmap v890: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:57:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:57:51] "GET /metrics HTTP/1.1" 200 225170 "" "Prometheus/2.33.4" 2022-09-29T05:57:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:57:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:57:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:57:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:57:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:57:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:57:53.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:57:53 smithi153 ceph-mon[70044]: pgmap v891: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:53.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:57:53 smithi083 ceph-mon[79930]: pgmap v891: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:53.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:57:53 smithi083 ceph-mon[77242]: pgmap v891: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:53.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:57:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:57:53.626Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:57:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:57:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:57:54.326Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:57:55.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:57:55 smithi083 ceph-mon[79930]: pgmap v892: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:55.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:57:55 smithi083 ceph-mon[77242]: pgmap v892: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:55.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:57:55 smithi153 ceph-mon[70044]: pgmap v892: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:57.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:57:57 smithi083 ceph-mon[79930]: pgmap v893: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:57.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:57:57 smithi083 ceph-mon[77242]: pgmap v893: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:57.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:57:57 smithi153 ceph-mon[70044]: pgmap v893: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:59.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:57:59 smithi083 ceph-mon[79930]: pgmap v894: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:59.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:57:59 smithi083 ceph-mon[77242]: pgmap v894: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:57:59.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:57:59 smithi153 ceph-mon[70044]: pgmap v894: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:00.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:58:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:58:00.219Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:58:01.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:58:01 smithi153 ceph-mon[70044]: pgmap v895: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:01.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:58:01 smithi083 ceph-mon[79930]: pgmap v895: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:01.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:58:01 smithi083 ceph-mon[77242]: pgmap v895: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:58:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:58:01] "GET /metrics HTTP/1.1" 200 225168 "" "Prometheus/2.33.4" 2022-09-29T05:58:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:58:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:58:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:58:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:58:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:58:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:58:03.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:58:03 smithi153 ceph-mon[70044]: pgmap v896: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:03.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:58:03 smithi083 ceph-mon[79930]: pgmap v896: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:03.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:58:03 smithi083 ceph-mon[77242]: pgmap v896: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:03.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:58:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:58:03.626Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:58:04.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:58:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:58:04.326Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:58:05.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:58:05 smithi083 ceph-mon[79930]: pgmap v897: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:05.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:58:05 smithi083 ceph-mon[77242]: pgmap v897: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:05.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:58:05 smithi153 ceph-mon[70044]: pgmap v897: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:07.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:58:07 smithi083 ceph-mon[79930]: pgmap v898: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:07.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:58:07 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:58:07.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:58:07 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:58:07.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:58:07 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:58:07.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:58:07 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:58:07.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:58:07 smithi083 ceph-mon[77242]: pgmap v898: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:07.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:58:07 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:58:07.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:58:07 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:58:07.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:58:07 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:58:07.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:58:07 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:58:07.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:58:07 smithi153 ceph-mon[70044]: pgmap v898: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:07.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:58:07 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:58:07.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:58:07 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:58:07.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:58:07 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:58:07.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:58:07 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:58:09.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:58:09 smithi083 ceph-mon[79930]: pgmap v899: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:09.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:58:09 smithi083 ceph-mon[77242]: pgmap v899: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:09.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:58:09 smithi153 ceph-mon[70044]: pgmap v899: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:10.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:58:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:58:10.220Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:58:11.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:58:11 smithi153 ceph-mon[70044]: pgmap v900: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:11.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:58:11 smithi083 ceph-mon[79930]: pgmap v900: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:11.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:58:11 smithi083 ceph-mon[77242]: pgmap v900: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:58:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:58:11] "GET /metrics HTTP/1.1" 200 225171 "" "Prometheus/2.33.4" 2022-09-29T05:58:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:58:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:58:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:58:13.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:58:13 smithi153 ceph-mon[70044]: pgmap v901: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:13.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:58:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:58:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:58:13.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:58:13 smithi083 ceph-mon[79930]: pgmap v901: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:13.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:58:13 smithi083 ceph-mon[77242]: pgmap v901: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:13.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:58:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:58:13.627Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:58:14.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:58:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:58:14.327Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:58:15.081 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T05:58:15.374 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T05:58:15.374 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (31m) 5m ago 32m 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T05:58:15.374 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (32m) 3m ago 32m 60.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T05:58:15.375 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (33m) 5m ago 33m 19.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T05:58:15.375 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (33m) 3m ago 33m 25.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T05:58:15.375 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (29m) 3m ago 35m 438M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T05:58:15.375 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (29m) 5m ago 37m 527M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T05:58:15.375 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (28m) 5m ago 37m 166M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T05:58:15.376 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (28m) 3m ago 36m 87.1M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T05:58:15.376 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (28m) 5m ago 36m 103M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T05:58:15.376 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (32m) 5m ago 32m 26.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T05:58:15.376 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (32m) 3m ago 32m 25.6M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T05:58:15.377 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (27m) 5m ago 35m 133M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T05:58:15.377 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (26m) 5m ago 35m 125M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T05:58:15.377 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (26m) 5m ago 34m 105M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T05:58:15.377 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (26m) 5m ago 34m 116M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T05:58:15.378 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (25m) 3m ago 34m 111M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T05:58:15.378 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (24m) 3m ago 34m 125M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T05:58:15.378 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (24m) 3m ago 33m 110M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T05:58:15.378 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (23m) 3m ago 33m 121M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T05:58:15.378 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (29m) 3m ago 32m 89.8M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T05:58:15.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:58:15 smithi083 ceph-mon[79930]: pgmap v902: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:15.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:58:15 smithi083 ceph-mon[77242]: pgmap v902: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:15.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:58:15 smithi153 ceph-mon[70044]: pgmap v902: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:15.727 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T05:58:15.728 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T05:58:15.728 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T05:58:15.728 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:58:15.729 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T05:58:15.729 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T05:58:15.729 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:58:15.729 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T05:58:15.729 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T05:58:15.730 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:58:15.730 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T05:58:15.730 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T05:58:15.730 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:58:15.731 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T05:58:15.731 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T05:58:15.731 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T05:58:15.731 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T05:58:15.732 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T05:58:16.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:58:16 smithi083 ceph-mon[79930]: from='client.35187 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:58:16.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:58:16 smithi083 ceph-mon[79930]: from='client.45127 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:58:16.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:58:16 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3693176318' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:58:16.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:58:16 smithi083 ceph-mon[77242]: from='client.35187 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:58:16.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:58:16 smithi083 ceph-mon[77242]: from='client.45127 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:58:16.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:58:16 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3693176318' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:58:16.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:58:16 smithi153 ceph-mon[70044]: from='client.35187 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:58:16.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:58:16 smithi153 ceph-mon[70044]: from='client.45127 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:58:16.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:58:16 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3693176318' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:58:17.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:58:17 smithi083 ceph-mon[79930]: pgmap v903: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:17.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:58:17 smithi083 ceph-mon[77242]: pgmap v903: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:17.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:58:17 smithi153 ceph-mon[70044]: pgmap v903: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:19.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:58:19 smithi083 ceph-mon[79930]: pgmap v904: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:19.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:58:19 smithi083 ceph-mon[77242]: pgmap v904: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:19.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:58:19 smithi153 ceph-mon[70044]: pgmap v904: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:20.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:58:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:58:20.220Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:58:21.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:58:21 smithi153 ceph-mon[70044]: pgmap v905: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:21.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:58:21 smithi083 ceph-mon[79930]: pgmap v905: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:21.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:58:21 smithi083 ceph-mon[77242]: pgmap v905: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:58:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:58:21] "GET /metrics HTTP/1.1" 200 225171 "" "Prometheus/2.33.4" 2022-09-29T05:58:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:58:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:58:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:58:23.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:58:23 smithi153 ceph-mon[70044]: pgmap v906: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:23.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:58:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:58:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:58:23.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:58:23 smithi083 ceph-mon[77242]: pgmap v906: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:23.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:58:23 smithi083 ceph-mon[79930]: pgmap v906: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:23.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:58:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:58:23.627Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:58:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:58:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:58:24.327Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:58:25.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:58:25 smithi083 ceph-mon[77242]: pgmap v907: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:25.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:58:25 smithi083 ceph-mon[79930]: pgmap v907: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:25.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:58:25 smithi153 ceph-mon[70044]: pgmap v907: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:27.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:58:27 smithi083 ceph-mon[79930]: pgmap v908: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:27.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:58:27 smithi083 ceph-mon[77242]: pgmap v908: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:27.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:58:27 smithi153 ceph-mon[70044]: pgmap v908: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:29.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:58:29 smithi083 ceph-mon[79930]: pgmap v909: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:29.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:58:29 smithi083 ceph-mon[77242]: pgmap v909: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:29.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:58:29 smithi153 ceph-mon[70044]: pgmap v909: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:30.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:58:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:58:30.221Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:58:31.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:58:31 smithi153 ceph-mon[70044]: pgmap v910: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:31.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:58:31 smithi083 ceph-mon[79930]: pgmap v910: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:31.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:58:31 smithi083 ceph-mon[77242]: pgmap v910: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:58:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:58:31] "GET /metrics HTTP/1.1" 200 225168 "" "Prometheus/2.33.4" 2022-09-29T05:58:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:58:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:58:32.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:58:33.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:58:33 smithi153 ceph-mon[70044]: pgmap v911: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:33.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:58:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:58:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:58:33.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:58:33 smithi083 ceph-mon[79930]: pgmap v911: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:33.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:58:33 smithi083 ceph-mon[77242]: pgmap v911: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:33.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:58:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:58:33.628Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:58:34.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:58:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:58:34.328Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:58:35.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:58:35 smithi083 ceph-mon[79930]: pgmap v912: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:35.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:58:35 smithi083 ceph-mon[77242]: pgmap v912: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:35.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:58:35 smithi153 ceph-mon[70044]: pgmap v912: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:37.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:58:37 smithi083 ceph-mon[79930]: pgmap v913: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:37.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:58:37 smithi083 ceph-mon[77242]: pgmap v913: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:37.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:58:37 smithi153 ceph-mon[70044]: pgmap v913: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:39.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:58:39 smithi083 ceph-mon[79930]: pgmap v914: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:39.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:58:39 smithi083 ceph-mon[77242]: pgmap v914: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:39.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:58:39 smithi153 ceph-mon[70044]: pgmap v914: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:40.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:58:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:58:40.222Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:58:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:58:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:58:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:58:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:58:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:58:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:58:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:58:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:58:40.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:58:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:58:40.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:58:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:58:40.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:58:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:58:40.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:58:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:58:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:58:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:58:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:58:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:58:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:58:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:58:40.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:58:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:58:41.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:58:41 smithi083 ceph-mon[77242]: pgmap v915: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:58:41 smithi083 ceph-mon[79930]: pgmap v915: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:58:41 smithi153 ceph-mon[70044]: pgmap v915: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:58:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:58:41] "GET /metrics HTTP/1.1" 200 225170 "" "Prometheus/2.33.4" 2022-09-29T05:58:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:58:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:58:42.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:58:43.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:58:43 smithi153 ceph-mon[70044]: pgmap v916: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:43.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:58:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:58:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:58:43.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:58:43 smithi083 ceph-mon[79930]: pgmap v916: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:43.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:58:43 smithi083 ceph-mon[77242]: pgmap v916: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:43.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:58:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:58:43.629Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:58:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:58:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:58:44.329Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:58:45.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:58:45 smithi083 ceph-mon[79930]: pgmap v917: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:45.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:58:45 smithi083 ceph-mon[77242]: pgmap v917: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:45.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:58:45 smithi153 ceph-mon[70044]: pgmap v917: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:46.055 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T05:58:46.347 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T05:58:46.348 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (32m) 5m ago 32m 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T05:58:46.348 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (32m) 3m ago 32m 60.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T05:58:46.348 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (33m) 5m ago 33m 19.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T05:58:46.348 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (33m) 3m ago 33m 25.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T05:58:46.349 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (29m) 3m ago 36m 438M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T05:58:46.349 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (29m) 5m ago 37m 527M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T05:58:46.349 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (29m) 5m ago 38m 166M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T05:58:46.349 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (28m) 3m ago 36m 87.1M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T05:58:46.350 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (28m) 5m ago 36m 103M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T05:58:46.350 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (33m) 5m ago 33m 26.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T05:58:46.350 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (33m) 3m ago 33m 25.6M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T05:58:46.350 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (27m) 5m ago 36m 133M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T05:58:46.351 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (27m) 5m ago 35m 125M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T05:58:46.351 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (27m) 5m ago 35m 105M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T05:58:46.351 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (26m) 5m ago 35m 116M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T05:58:46.351 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (25m) 3m ago 34m 111M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T05:58:46.352 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (25m) 3m ago 34m 125M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T05:58:46.352 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (24m) 3m ago 34m 110M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T05:58:46.352 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (24m) 3m ago 34m 121M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T05:58:46.352 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (29m) 3m ago 33m 89.8M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T05:58:46.704 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T05:58:46.704 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T05:58:46.705 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T05:58:46.705 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:58:46.705 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T05:58:46.705 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T05:58:46.706 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:58:46.706 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T05:58:46.706 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T05:58:46.706 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:58:46.706 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T05:58:46.707 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T05:58:46.707 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:58:46.707 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T05:58:46.707 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T05:58:46.708 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T05:58:46.708 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T05:58:46.708 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T05:58:47.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:58:47 smithi083 ceph-mon[79930]: from='client.45139 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:58:47.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:58:47 smithi083 ceph-mon[79930]: pgmap v918: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:47.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:58:47 smithi083 ceph-mon[79930]: from='client.45145 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:58:47.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:58:47 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3831442517' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:58:47.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:58:47 smithi083 ceph-mon[77242]: from='client.45139 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:58:47.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:58:47 smithi083 ceph-mon[77242]: pgmap v918: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:47.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:58:47 smithi083 ceph-mon[77242]: from='client.45145 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:58:47.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:58:47 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3831442517' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:58:47.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:58:47 smithi153 ceph-mon[70044]: from='client.45139 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:58:47.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:58:47 smithi153 ceph-mon[70044]: pgmap v918: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:47.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:58:47 smithi153 ceph-mon[70044]: from='client.45145 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:58:47.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:58:47 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3831442517' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:58:49.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:58:49 smithi083 ceph-mon[79930]: pgmap v919: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:49.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:58:49 smithi083 ceph-mon[77242]: pgmap v919: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:49.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:58:49 smithi153 ceph-mon[70044]: pgmap v919: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:50.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:58:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:58:50.223Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:58:51.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:58:51 smithi153 ceph-mon[70044]: pgmap v920: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:51.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:58:51 smithi083 ceph-mon[79930]: pgmap v920: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:51.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:58:51 smithi083 ceph-mon[77242]: pgmap v920: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:58:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:58:51] "GET /metrics HTTP/1.1" 200 225170 "" "Prometheus/2.33.4" 2022-09-29T05:58:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:58:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:58:52.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:58:53.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:58:53 smithi153 ceph-mon[70044]: pgmap v921: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:53.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:58:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:58:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:58:53.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:58:53 smithi083 ceph-mon[79930]: pgmap v921: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:53.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:58:53 smithi083 ceph-mon[77242]: pgmap v921: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:53.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:58:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:58:53.630Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:58:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:58:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:58:54.329Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:58:55.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:58:55 smithi083 ceph-mon[79930]: pgmap v922: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:55.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:58:55 smithi083 ceph-mon[77242]: pgmap v922: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:55.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:58:55 smithi153 ceph-mon[70044]: pgmap v922: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:57.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:58:57 smithi083 ceph-mon[79930]: pgmap v923: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:57.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:58:57 smithi083 ceph-mon[77242]: pgmap v923: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:57.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:58:57 smithi153 ceph-mon[70044]: pgmap v923: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:59.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:58:59 smithi083 ceph-mon[79930]: pgmap v924: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:59.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:58:59 smithi083 ceph-mon[77242]: pgmap v924: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:58:59.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:58:59 smithi153 ceph-mon[70044]: pgmap v924: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:00.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:59:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:59:00.223Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:59:01.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:59:01 smithi153 ceph-mon[70044]: pgmap v925: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:01.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:59:01 smithi083 ceph-mon[79930]: pgmap v925: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:01.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:59:01 smithi083 ceph-mon[77242]: pgmap v925: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:59:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:59:01] "GET /metrics HTTP/1.1" 200 225166 "" "Prometheus/2.33.4" 2022-09-29T05:59:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:59:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:59:02.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:59:03.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:59:03 smithi153 ceph-mon[70044]: pgmap v926: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:59:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:59:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:59:03.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:59:03 smithi083 ceph-mon[77242]: pgmap v926: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:03.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:59:03 smithi083 ceph-mon[79930]: pgmap v926: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:03.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:59:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:59:03.631Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:59:04.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:59:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:59:04.330Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:59:05.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:59:05 smithi083 ceph-mon[79930]: pgmap v927: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:05.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:59:05 smithi083 ceph-mon[77242]: pgmap v927: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:05.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:59:05 smithi153 ceph-mon[70044]: pgmap v927: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:07.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:59:07 smithi083 ceph-mon[79930]: pgmap v928: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:07.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:59:07 smithi083 ceph-mon[77242]: pgmap v928: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:07.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:59:07 smithi153 ceph-mon[70044]: pgmap v928: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:08.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:59:08 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:59:08.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:59:08 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:59:08.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:59:08 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:59:08.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:59:08 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:59:08.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:59:08 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:59:08.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:59:08 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:59:08.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:59:08 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:59:08.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:59:08 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:59:08.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:59:08 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T05:59:08.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:59:08 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T05:59:08.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:59:08 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T05:59:08.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:59:08 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T05:59:10.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:59:09 smithi083 ceph-mon[77242]: pgmap v929: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:10.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:59:09 smithi083 ceph-mon[79930]: pgmap v929: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:10.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:59:09 smithi153 ceph-mon[70044]: pgmap v929: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:10.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:59:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:59:10.224Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:59:11.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:59:11 smithi153 ceph-mon[70044]: pgmap v930: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:11.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:59:11 smithi083 ceph-mon[79930]: pgmap v930: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:11.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:59:11 smithi083 ceph-mon[77242]: pgmap v930: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:59:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:59:11] "GET /metrics HTTP/1.1" 200 225169 "" "Prometheus/2.33.4" 2022-09-29T05:59:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:59:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:59:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:59:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:59:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:59:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:59:13.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:59:13 smithi153 ceph-mon[70044]: pgmap v931: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:13.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:59:13 smithi083 ceph-mon[79930]: pgmap v931: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:13.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:59:13 smithi083 ceph-mon[77242]: pgmap v931: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:13.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:59:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:59:13.631Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:59:14.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:59:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:59:14.330Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:59:15.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:59:15 smithi083 ceph-mon[77242]: pgmap v932: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:15.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:59:15 smithi083 ceph-mon[79930]: pgmap v932: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:15.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:59:15 smithi153 ceph-mon[70044]: pgmap v932: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:17.033 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T05:59:17.322 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T05:59:17.322 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (32m) 6m ago 33m 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T05:59:17.323 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (33m) 4m ago 33m 60.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T05:59:17.323 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (34m) 6m ago 34m 19.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T05:59:17.323 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (34m) 4m ago 34m 25.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T05:59:17.323 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (30m) 4m ago 36m 438M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T05:59:17.323 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (30m) 6m ago 38m 527M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T05:59:17.324 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (29m) 6m ago 38m 166M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T05:59:17.324 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (29m) 4m ago 37m 87.1M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T05:59:17.324 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (29m) 6m ago 37m 103M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T05:59:17.324 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (33m) 6m ago 33m 26.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T05:59:17.324 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (33m) 4m ago 33m 25.6M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T05:59:17.325 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (28m) 6m ago 36m 133M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T05:59:17.325 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (27m) 6m ago 36m 125M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T05:59:17.325 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (27m) 6m ago 36m 105M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T05:59:17.325 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (27m) 6m ago 35m 116M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T05:59:17.325 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (26m) 4m ago 35m 111M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T05:59:17.326 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (25m) 4m ago 35m 125M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T05:59:17.326 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (25m) 4m ago 34m 110M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T05:59:17.326 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (24m) 4m ago 34m 121M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T05:59:17.326 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (30m) 4m ago 33m 89.8M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T05:59:17.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:59:17 smithi083 ceph-mon[79930]: pgmap v933: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:17.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:59:17 smithi083 ceph-mon[77242]: pgmap v933: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:17.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:59:17 smithi153 ceph-mon[70044]: pgmap v933: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:17.678 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T05:59:17.679 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T05:59:17.679 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T05:59:17.679 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:59:17.679 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T05:59:17.679 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T05:59:17.680 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:59:17.680 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T05:59:17.680 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T05:59:17.680 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:59:17.680 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T05:59:17.681 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T05:59:17.681 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:59:17.681 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T05:59:17.681 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T05:59:17.682 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T05:59:17.682 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T05:59:17.682 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T05:59:18.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:59:18 smithi083 ceph-mon[79930]: from='client.45157 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:59:18.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:59:18 smithi083 ceph-mon[79930]: from='client.45163 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:59:18.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:59:18 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3138407454' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:59:18.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:59:18 smithi083 ceph-mon[77242]: from='client.45157 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:59:18.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:59:18 smithi083 ceph-mon[77242]: from='client.45163 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:59:18.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:59:18 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3138407454' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:59:18.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:59:18 smithi153 ceph-mon[70044]: from='client.45157 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:59:18.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:59:18 smithi153 ceph-mon[70044]: from='client.45163 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:59:18.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:59:18 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3138407454' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:59:19.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:59:19 smithi083 ceph-mon[79930]: pgmap v934: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:19.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:59:19 smithi083 ceph-mon[77242]: pgmap v934: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:19.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:59:19 smithi153 ceph-mon[70044]: pgmap v934: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:20.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:59:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:59:20.225Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:59:21.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:59:21 smithi153 ceph-mon[70044]: pgmap v935: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:21.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:59:21 smithi083 ceph-mon[79930]: pgmap v935: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:21.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:59:21 smithi083 ceph-mon[77242]: pgmap v935: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:59:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:59:21] "GET /metrics HTTP/1.1" 200 225169 "" "Prometheus/2.33.4" 2022-09-29T05:59:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:59:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:59:22.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:59:23.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:59:23 smithi153 ceph-mon[70044]: pgmap v936: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:23.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:59:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:59:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:59:23.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:59:23 smithi083 ceph-mon[77242]: pgmap v936: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:23.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:59:23 smithi083 ceph-mon[79930]: pgmap v936: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:23.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:59:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:59:23.632Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:59:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:59:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:59:24.330Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:59:25.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:59:25 smithi083 ceph-mon[79930]: pgmap v937: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:25.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:59:25 smithi083 ceph-mon[77242]: pgmap v937: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:25.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:59:25 smithi153 ceph-mon[70044]: pgmap v937: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:27.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:59:27 smithi083 ceph-mon[79930]: pgmap v938: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:27.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:59:27 smithi083 ceph-mon[77242]: pgmap v938: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:27.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:59:27 smithi153 ceph-mon[70044]: pgmap v938: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:29.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:59:29 smithi083 ceph-mon[79930]: pgmap v939: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:29.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:59:29 smithi083 ceph-mon[77242]: pgmap v939: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:29.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:59:29 smithi153 ceph-mon[70044]: pgmap v939: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:30.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:59:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:59:30.226Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:59:31.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:59:31 smithi153 ceph-mon[70044]: pgmap v940: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:31.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:59:31 smithi083 ceph-mon[77242]: pgmap v940: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:31.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:59:31 smithi083 ceph-mon[79930]: pgmap v940: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:59:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:59:31] "GET /metrics HTTP/1.1" 200 225172 "" "Prometheus/2.33.4" 2022-09-29T05:59:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:59:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:59:32.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:59:33.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:59:33 smithi153 ceph-mon[70044]: pgmap v941: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:33.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:59:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:59:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:59:33.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:59:33 smithi083 ceph-mon[79930]: pgmap v941: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:33.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:59:33 smithi083 ceph-mon[77242]: pgmap v941: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:33.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:59:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:59:33.633Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:59:34.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:59:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:59:34.331Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:59:35.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:59:35 smithi083 ceph-mon[79930]: pgmap v942: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:35.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:59:35 smithi083 ceph-mon[77242]: pgmap v942: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:35.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:59:35 smithi153 ceph-mon[70044]: pgmap v942: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:37.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:59:37 smithi083 ceph-mon[79930]: pgmap v943: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:37.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:59:37 smithi083 ceph-mon[77242]: pgmap v943: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:37.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:59:37 smithi153 ceph-mon[70044]: pgmap v943: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:39.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:59:39 smithi083 ceph-mon[79930]: pgmap v944: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:39.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:59:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:59:39.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:59:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:59:39.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:59:39 smithi083 ceph-mon[77242]: pgmap v944: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:39.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:59:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:59:39.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:59:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:59:39.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:59:39 smithi153 ceph-mon[70044]: pgmap v944: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:39.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:59:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:59:39.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:59:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T05:59:40.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:59:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:59:40.226Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:59:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:59:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:59:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:59:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:59:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:59:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:59:40.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:59:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:59:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:59:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:59:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:59:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T05:59:41.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:59:41 smithi083 ceph-mon[79930]: pgmap v945: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:41.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:59:41 smithi083 ceph-mon[77242]: pgmap v945: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:59:41 smithi153 ceph-mon[70044]: pgmap v945: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:59:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:59:41] "GET /metrics HTTP/1.1" 200 225169 "" "Prometheus/2.33.4" 2022-09-29T05:59:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:59:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:59:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:59:43.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:59:43 smithi153 ceph-mon[70044]: pgmap v946: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:43.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:59:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:59:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:59:43.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:59:43 smithi083 ceph-mon[77242]: pgmap v946: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:43.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:59:43 smithi083 ceph-mon[79930]: pgmap v946: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:43.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:59:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:59:43.634Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:59:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:59:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:59:44.332Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:59:45.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:59:45 smithi083 ceph-mon[77242]: pgmap v947: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:45.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:59:45 smithi083 ceph-mon[79930]: pgmap v947: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:45.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:59:45 smithi153 ceph-mon[70044]: pgmap v947: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:47.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:59:47 smithi083 ceph-mon[79930]: pgmap v948: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:47.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:59:47 smithi083 ceph-mon[77242]: pgmap v948: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:47.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:59:47 smithi153 ceph-mon[70044]: pgmap v948: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:48.004 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T05:59:48.293 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T05:59:48.293 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (33m) 6m ago 33m 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T05:59:48.293 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (33m) 4m ago 33m 60.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T05:59:48.294 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (34m) 6m ago 34m 19.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T05:59:48.294 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (34m) 4m ago 34m 25.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T05:59:48.294 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (30m) 4m ago 37m 438M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T05:59:48.294 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (31m) 6m ago 38m 527M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T05:59:48.295 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (30m) 6m ago 39m 166M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T05:59:48.295 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (29m) 4m ago 37m 87.1M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T05:59:48.295 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (29m) 6m ago 37m 103M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T05:59:48.295 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (34m) 6m ago 34m 26.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T05:59:48.295 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (34m) 4m ago 34m 25.6M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T05:59:48.296 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (28m) 6m ago 37m 133M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T05:59:48.296 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (28m) 6m ago 36m 125M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T05:59:48.296 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (28m) 6m ago 36m 105M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T05:59:48.296 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (27m) 6m ago 36m 116M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T05:59:48.297 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (27m) 4m ago 35m 111M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T05:59:48.297 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (26m) 4m ago 35m 125M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T05:59:48.297 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (25m) 4m ago 35m 110M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T05:59:48.297 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (25m) 4m ago 35m 121M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T05:59:48.297 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (30m) 4m ago 34m 89.8M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T05:59:48.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:59:48 smithi083 ceph-mon[79930]: from='client.35241 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:59:48.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:59:48 smithi083 ceph-mon[77242]: from='client.35241 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:59:48.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:59:48 smithi153 ceph-mon[70044]: from='client.35241 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:59:48.647 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T05:59:48.648 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T05:59:48.648 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T05:59:48.648 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:59:48.648 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T05:59:48.649 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T05:59:48.649 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:59:48.649 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T05:59:48.649 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T05:59:48.650 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:59:48.650 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T05:59:48.650 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T05:59:48.650 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T05:59:48.651 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T05:59:48.651 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T05:59:48.651 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T05:59:48.651 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T05:59:48.651 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T05:59:49.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:59:49 smithi083 ceph-mon[77242]: pgmap v949: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:49.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:59:49 smithi083 ceph-mon[77242]: from='client.35247 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:59:49.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:59:49 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3022365580' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:59:49.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:59:49 smithi083 ceph-mon[79930]: pgmap v949: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:49.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:59:49 smithi083 ceph-mon[79930]: from='client.35247 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:59:49.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:59:49 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3022365580' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:59:49.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:59:49 smithi153 ceph-mon[70044]: pgmap v949: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:49.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:59:49 smithi153 ceph-mon[70044]: from='client.35247 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T05:59:49.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:59:49 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3022365580' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T05:59:50.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:59:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:59:50.227Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:59:51.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:59:51 smithi153 ceph-mon[70044]: pgmap v950: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:51.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:59:51 smithi083 ceph-mon[77242]: pgmap v950: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:51.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:59:51 smithi083 ceph-mon[79930]: pgmap v950: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 05:59:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:59:51] "GET /metrics HTTP/1.1" 200 225169 "" "Prometheus/2.33.4" 2022-09-29T05:59:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 05:59:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T05:59:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T05:59:53.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:59:53 smithi153 ceph-mon[70044]: pgmap v951: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:53.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 05:59:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:05:59:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T05:59:53.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:59:53 smithi083 ceph-mon[79930]: pgmap v951: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:53.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:59:53 smithi083 ceph-mon[77242]: pgmap v951: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:53.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:59:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:59:53.634Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:59:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 05:59:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T05:59:54.332Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T05:59:55.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:59:55 smithi083 ceph-mon[79930]: pgmap v952: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:55.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:59:55 smithi083 ceph-mon[77242]: pgmap v952: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:55.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:59:55 smithi153 ceph-mon[70044]: pgmap v952: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:57.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:59:57 smithi083 ceph-mon[79930]: pgmap v953: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:57.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:59:57 smithi083 ceph-mon[77242]: pgmap v953: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:57.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:59:57 smithi153 ceph-mon[70044]: pgmap v953: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:59.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 05:59:59 smithi083 ceph-mon[77242]: pgmap v954: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:59.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 05:59:59 smithi083 ceph-mon[79930]: pgmap v954: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T05:59:59.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 05:59:59 smithi153 ceph-mon[70044]: pgmap v954: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:00.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:00:00 smithi083 ceph-mon[79930]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T06:00:00.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:00:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[77219]: 2022-09-29T05:59:59.999+0000 7f160f665700 -1 log_channel(cluster) log [ERR] : overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T06:00:00.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:00:00 smithi083 ceph-mon[77242]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T06:00:00.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:00:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:00:00.227Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:00:00.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:00:00 smithi153 ceph-mon[70044]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T06:00:01.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:00:01 smithi083 ceph-mon[79930]: pgmap v955: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:01.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:00:01 smithi083 ceph-mon[77242]: pgmap v955: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:01.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:00:01 smithi153 ceph-mon[70044]: pgmap v955: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:00:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:00:01] "GET /metrics HTTP/1.1" 200 225172 "" "Prometheus/2.33.4" 2022-09-29T06:00:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:00:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:00:02.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:00:03.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:00:03 smithi153 ceph-mon[70044]: pgmap v956: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:00:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:00:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:00:03.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:00:03 smithi083 ceph-mon[79930]: pgmap v956: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:03.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:00:03 smithi083 ceph-mon[77242]: pgmap v956: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:03.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:00:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:00:03.635Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:00:04.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:00:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:00:04.333Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:00:05.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:00:05 smithi083 ceph-mon[79930]: pgmap v957: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:05.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:00:05 smithi083 ceph-mon[77242]: pgmap v957: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:05.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:00:05 smithi153 ceph-mon[70044]: pgmap v957: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:07.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:00:07 smithi083 ceph-mon[79930]: pgmap v958: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:07.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:00:07 smithi083 ceph-mon[77242]: pgmap v958: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:07.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:00:07 smithi153 ceph-mon[70044]: pgmap v958: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:08.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:00:08 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:00:08.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:00:08 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:00:08.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:00:08 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:00:08.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:00:08 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:00:08.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:00:08 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:00:08.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:00:08 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:00:08.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:00:08 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:00:08.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:00:08 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:00:08.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:00:08 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:00:09.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:00:09 smithi153 ceph-mon[70044]: pgmap v959: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:09.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:00:09 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:00:09.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:00:09 smithi083 ceph-mon[79930]: pgmap v959: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:09.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:00:09 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:00:09.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:00:09 smithi083 ceph-mon[77242]: pgmap v959: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:09.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:00:09 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:00:10.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:00:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:00:10.227Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:00:11.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:00:11 smithi153 ceph-mon[70044]: pgmap v960: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:11.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:00:11 smithi083 ceph-mon[79930]: pgmap v960: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:11.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:00:11 smithi083 ceph-mon[77242]: pgmap v960: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:00:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:00:11] "GET /metrics HTTP/1.1" 200 225169 "" "Prometheus/2.33.4" 2022-09-29T06:00:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:00:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:00:12.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:00:13.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:00:13 smithi153 ceph-mon[70044]: pgmap v961: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:13.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:00:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:00:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:00:13.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:00:13 smithi083 ceph-mon[79930]: pgmap v961: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:13.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:00:13 smithi083 ceph-mon[77242]: pgmap v961: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:13.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:00:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:00:13.636Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:00:14.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:00:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:00:14.334Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:00:15.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:00:15 smithi083 ceph-mon[77242]: pgmap v962: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:15.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:00:15 smithi083 ceph-mon[79930]: pgmap v962: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:15.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:00:15 smithi153 ceph-mon[70044]: pgmap v962: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:17.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:00:17 smithi083 ceph-mon[77242]: pgmap v963: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:17.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:00:17 smithi083 ceph-mon[79930]: pgmap v963: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:17.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:00:17 smithi153 ceph-mon[70044]: pgmap v963: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:18.973 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:00:19.260 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:00:19.261 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (34m) 7m ago 34m 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:00:19.261 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (34m) 5m ago 34m 60.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:00:19.261 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (35m) 7m ago 35m 19.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:00:19.261 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (35m) 5m ago 35m 25.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:00:19.261 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (31m) 5m ago 37m 438M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:00:19.262 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (31m) 7m ago 39m 527M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:00:19.262 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (30m) 7m ago 39m 166M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:00:19.262 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (30m) 5m ago 38m 87.1M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:00:19.262 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (30m) 7m ago 38m 103M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:00:19.263 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (34m) 7m ago 34m 26.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:00:19.263 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (34m) 5m ago 34m 25.6M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:00:19.263 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (29m) 7m ago 37m 133M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:00:19.263 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (28m) 7m ago 37m 125M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:00:19.263 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (28m) 7m ago 37m 105M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:00:19.264 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (28m) 7m ago 36m 116M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:00:19.264 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (27m) 5m ago 36m 111M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:00:19.264 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (26m) 5m ago 36m 125M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:00:19.264 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (26m) 5m ago 35m 110M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:00:19.265 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (25m) 5m ago 35m 121M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:00:19.265 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (31m) 5m ago 34m 89.8M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:00:19.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:00:19 smithi083 ceph-mon[77242]: pgmap v964: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:19.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:00:19 smithi083 ceph-mon[77242]: from='client.45193 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:00:19.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:00:19 smithi083 ceph-mon[79930]: pgmap v964: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:19.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:00:19 smithi083 ceph-mon[79930]: from='client.45193 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:00:19.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:00:19 smithi153 ceph-mon[70044]: pgmap v964: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:19.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:00:19 smithi153 ceph-mon[70044]: from='client.45193 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:00:19.616 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:00:19.616 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:00:19.616 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:00:19.617 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:00:19.617 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:00:19.617 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:00:19.617 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:00:19.618 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:00:19.618 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:00:19.618 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:00:19.618 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:00:19.619 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:00:19.619 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:00:19.619 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:00:19.619 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:00:19.619 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:00:19.620 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:00:19.620 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:00:20.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:00:20 smithi083 ceph-mon[79930]: from='client.35265 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:00:20.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:00:20 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/304981308' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:00:20.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:00:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:00:20.227Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:00:20.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:00:20 smithi083 ceph-mon[77242]: from='client.35265 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:00:20.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:00:20 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/304981308' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:00:20.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:00:20 smithi153 ceph-mon[70044]: from='client.35265 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:00:20.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:00:20 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/304981308' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:00:21.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:00:21 smithi083 ceph-mon[79930]: pgmap v965: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:21.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:00:21 smithi083 ceph-mon[77242]: pgmap v965: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:21.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:00:21 smithi153 ceph-mon[70044]: pgmap v965: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:00:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:00:21] "GET /metrics HTTP/1.1" 200 225169 "" "Prometheus/2.33.4" 2022-09-29T06:00:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:00:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:00:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:00:23.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:00:23 smithi153 ceph-mon[70044]: pgmap v966: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:23.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:00:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:00:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:00:23.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:00:23 smithi083 ceph-mon[79930]: pgmap v966: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:23.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:00:23 smithi083 ceph-mon[77242]: pgmap v966: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:23.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:00:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:00:23.637Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:00:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:00:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:00:24.334Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:00:25.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:00:25 smithi153 ceph-mon[70044]: pgmap v967: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:25.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:00:25 smithi083 ceph-mon[79930]: pgmap v967: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:25.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:00:25 smithi083 ceph-mon[77242]: pgmap v967: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:27.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:00:27 smithi153 ceph-mon[70044]: pgmap v968: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:27.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:00:27 smithi083 ceph-mon[79930]: pgmap v968: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:27.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:00:27 smithi083 ceph-mon[77242]: pgmap v968: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:29.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:00:29 smithi153 ceph-mon[70044]: pgmap v969: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:29.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:00:29 smithi083 ceph-mon[79930]: pgmap v969: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:29.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:00:29 smithi083 ceph-mon[77242]: pgmap v969: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:30.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:00:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:00:30.229Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:00:31.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:00:31 smithi153 ceph-mon[70044]: pgmap v970: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:31.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:00:31 smithi083 ceph-mon[79930]: pgmap v970: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:31.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:00:31 smithi083 ceph-mon[77242]: pgmap v970: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:00:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:00:31] "GET /metrics HTTP/1.1" 200 225170 "" "Prometheus/2.33.4" 2022-09-29T06:00:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:00:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:00:32.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:00:33.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:00:33 smithi153 ceph-mon[70044]: pgmap v971: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:33.312 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:00:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:00:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:00:33.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:00:33 smithi083 ceph-mon[79930]: pgmap v971: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:33.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:00:33 smithi083 ceph-mon[77242]: pgmap v971: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:33.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:00:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:00:33.637Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:00:34.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:00:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:00:34.335Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:00:35.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:00:35 smithi083 ceph-mon[79930]: pgmap v972: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:35.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:00:35 smithi083 ceph-mon[77242]: pgmap v972: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:35.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:00:35 smithi153 ceph-mon[70044]: pgmap v972: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:37.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:00:37 smithi083 ceph-mon[79930]: pgmap v973: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:37.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:00:37 smithi083 ceph-mon[77242]: pgmap v973: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:37.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:00:37 smithi153 ceph-mon[70044]: pgmap v973: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:39.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:00:39 smithi083 ceph-mon[77242]: pgmap v974: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:39.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:00:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:00:39.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:00:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:00:39.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:00:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:00:39.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:00:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:00:39.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:00:39 smithi083 ceph-mon[79930]: pgmap v974: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:39.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:00:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:00:39.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:00:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:00:39.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:00:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:00:39.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:00:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:00:39.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:00:39 smithi153 ceph-mon[70044]: pgmap v974: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:39.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:00:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:00:39.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:00:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:00:39.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:00:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:00:39.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:00:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:00:40.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:00:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:00:40.229Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:00:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:00:41 smithi153 ceph-mon[70044]: pgmap v975: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:41.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:00:41 smithi083 ceph-mon[77242]: pgmap v975: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:00:41 smithi083 ceph-mon[79930]: pgmap v975: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:00:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:00:41] "GET /metrics HTTP/1.1" 200 225175 "" "Prometheus/2.33.4" 2022-09-29T06:00:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:00:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:00:42.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:00:43.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:00:43 smithi153 ceph-mon[70044]: pgmap v976: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:43.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:00:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:00:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:00:43.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:00:43 smithi083 ceph-mon[79930]: pgmap v976: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:43.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:00:43 smithi083 ceph-mon[77242]: pgmap v976: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:43.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:00:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:00:43.638Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:00:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:00:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:00:44.335Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:00:45.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:00:45 smithi083 ceph-mon[77242]: pgmap v977: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:45.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:00:45 smithi083 ceph-mon[79930]: pgmap v977: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:45.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:00:45 smithi153 ceph-mon[70044]: pgmap v977: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:47.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:00:47 smithi083 ceph-mon[77242]: pgmap v978: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:47.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:00:47 smithi083 ceph-mon[79930]: pgmap v978: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:47.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:00:47 smithi153 ceph-mon[70044]: pgmap v978: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:49.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:00:49 smithi153 ceph-mon[70044]: pgmap v979: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:49.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:00:49 smithi083 ceph-mon[79930]: pgmap v979: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:49.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:00:49 smithi083 ceph-mon[77242]: pgmap v979: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:49.941 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:00:50.230 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:00:50.231 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (34m) 7m ago 35m 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:00:50.231 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (34m) 5m ago 34m 60.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:00:50.231 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (35m) 7m ago 35m 19.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:00:50.231 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (35m) 5m ago 35m 25.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:00:50.232 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (31m) 5m ago 38m 438M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:00:50.232 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (32m) 7m ago 39m 527M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:00:50.232 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (31m) 7m ago 40m 166M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:00:50.232 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (30m) 5m ago 38m 87.1M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:00:50.233 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (30m) 7m ago 38m 103M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:00:50.233 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (35m) 7m ago 35m 26.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:00:50.233 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (35m) 5m ago 35m 25.6M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:00:50.233 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (30m) 7m ago 38m 133M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:00:50.233 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (29m) 7m ago 37m 125M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:00:50.234 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (29m) 7m ago 37m 105M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:00:50.234 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (28m) 7m ago 37m 116M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:00:50.234 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (28m) 5m ago 36m 111M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:00:50.234 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (27m) 5m ago 36m 125M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:00:50.234 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (26m) 5m ago 36m 110M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:00:50.235 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (26m) 5m ago 36m 121M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:00:50.235 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (31m) 5m ago 35m 89.8M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:00:50.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:00:50 smithi153 ceph-mon[70044]: from='client.35277 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:00:50.588 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:00:50.588 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:00:50.588 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:00:50.588 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:00:50.589 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:00:50.589 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:00:50.589 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:00:50.589 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:00:50.589 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:00:50.590 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:00:50.590 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:00:50.590 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:00:50.590 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:00:50.591 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:00:50.591 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:00:50.591 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:00:50.591 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:00:50.591 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:00:50.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:00:50 smithi083 ceph-mon[79930]: from='client.35277 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:00:50.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:00:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:00:50.231Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:00:50.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:00:50 smithi083 ceph-mon[77242]: from='client.35277 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:00:51.507 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:00:51 smithi083 ceph-mon[77242]: pgmap v980: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:51.508 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:00:51 smithi083 ceph-mon[77242]: from='client.45217 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:00:51.508 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:00:51 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/752445082' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:00:51.508 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:00:51 smithi083 ceph-mon[79930]: pgmap v980: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:51.509 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:00:51 smithi083 ceph-mon[79930]: from='client.45217 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:00:51.509 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:00:51 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/752445082' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:00:51.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:00:51 smithi153 ceph-mon[70044]: pgmap v980: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:51.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:00:51 smithi153 ceph-mon[70044]: from='client.45217 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:00:51.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:00:51 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/752445082' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:00:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:00:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:00:51] "GET /metrics HTTP/1.1" 200 225175 "" "Prometheus/2.33.4" 2022-09-29T06:00:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:00:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:00:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:00:53.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:00:53 smithi153 ceph-mon[70044]: pgmap v981: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:53.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:00:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:00:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:00:53.641 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:00:53 smithi083 ceph-mon[79930]: pgmap v981: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:53.642 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:00:53 smithi083 ceph-mon[77242]: pgmap v981: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:53.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:00:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:00:53.639Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:00:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:00:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:00:54.335Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:00:55.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:00:55 smithi153 ceph-mon[70044]: pgmap v982: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:55.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:00:55 smithi083 ceph-mon[79930]: pgmap v982: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:55.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:00:55 smithi083 ceph-mon[77242]: pgmap v982: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:57.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:00:57 smithi153 ceph-mon[70044]: pgmap v983: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:57.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:00:57 smithi083 ceph-mon[77242]: pgmap v983: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:57.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:00:57 smithi083 ceph-mon[79930]: pgmap v983: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:59.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:00:59 smithi153 ceph-mon[70044]: pgmap v984: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:59.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:00:59 smithi083 ceph-mon[79930]: pgmap v984: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:00:59.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:00:59 smithi083 ceph-mon[77242]: pgmap v984: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:00.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:01:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:01:00.230Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:01:01.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:01:01 smithi153 ceph-mon[70044]: pgmap v985: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:01.396 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:01:01 smithi083 ceph-mon[79930]: pgmap v985: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:01.397 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:01:01 smithi083 ceph-mon[77242]: pgmap v985: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:01.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:01:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:01:01] "GET /metrics HTTP/1.1" 200 225172 "" "Prometheus/2.33.4" 2022-09-29T06:01:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:01:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:01:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:01:03.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:01:03 smithi153 ceph-mon[70044]: pgmap v986: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:01:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:01:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:01:03.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:01:03 smithi083 ceph-mon[79930]: pgmap v986: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:03.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:01:03 smithi083 ceph-mon[77242]: pgmap v986: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:03.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:01:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:01:03.639Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:01:04.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:01:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:01:04.336Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:01:05.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:01:05 smithi153 ceph-mon[70044]: pgmap v987: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:05.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:01:05 smithi083 ceph-mon[77242]: pgmap v987: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:05.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:01:05 smithi083 ceph-mon[79930]: pgmap v987: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:07.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:01:07 smithi153 ceph-mon[70044]: pgmap v988: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:07.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:01:07 smithi083 ceph-mon[79930]: pgmap v988: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:07.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:01:07 smithi083 ceph-mon[77242]: pgmap v988: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:09.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:01:09 smithi153 ceph-mon[70044]: pgmap v989: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:09.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:01:09 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:01:09.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:01:09 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:01:09.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:01:09 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:01:09.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:01:09 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:01:09.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:01:09 smithi083 ceph-mon[77242]: pgmap v989: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:09.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:01:09 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:01:09.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:01:09 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:01:09.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:01:09 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:01:09.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:01:09 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:01:09.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:01:09 smithi083 ceph-mon[79930]: pgmap v989: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:09.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:01:09 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:01:09.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:01:09 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:01:09.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:01:09 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:01:09.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:01:09 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:01:10.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:01:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:01:10.231Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:01:11.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:01:11 smithi153 ceph-mon[70044]: pgmap v990: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:11.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:01:11 smithi083 ceph-mon[79930]: pgmap v990: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:11.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:01:11 smithi083 ceph-mon[77242]: pgmap v990: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:01:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:01:11] "GET /metrics HTTP/1.1" 200 225170 "" "Prometheus/2.33.4" 2022-09-29T06:01:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:01:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:01:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:01:13.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:01:13 smithi153 ceph-mon[70044]: pgmap v991: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:13.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:01:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:01:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:01:13.643 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:01:13 smithi083 ceph-mon[79930]: pgmap v991: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:13.643 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:01:13 smithi083 ceph-mon[77242]: pgmap v991: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:13.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:01:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:01:13.640Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:01:14.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:01:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:01:14.337Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:01:15.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:01:15 smithi153 ceph-mon[70044]: pgmap v992: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:15.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:01:15 smithi083 ceph-mon[79930]: pgmap v992: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:15.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:01:15 smithi083 ceph-mon[77242]: pgmap v992: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:17.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:01:17 smithi153 ceph-mon[70044]: pgmap v993: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:17.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:01:17 smithi083 ceph-mon[77242]: pgmap v993: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:17.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:01:17 smithi083 ceph-mon[79930]: pgmap v993: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:19.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:01:19 smithi153 ceph-mon[70044]: pgmap v994: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:19.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:01:19 smithi083 ceph-mon[79930]: pgmap v994: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:19.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:01:19 smithi083 ceph-mon[77242]: pgmap v994: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:20.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:01:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:01:20.232Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:01:20.913 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:01:21.204 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:01:21.204 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (35m) 8m ago 35m 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:01:21.204 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (35m) 6m ago 35m 60.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:01:21.205 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (36m) 8m ago 36m 19.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:01:21.205 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (36m) 6m ago 36m 25.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:01:21.205 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (32m) 6m ago 38m 438M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:01:21.206 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (32m) 8m ago 40m 527M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:01:21.206 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (31m) 8m ago 40m 166M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:01:21.206 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (31m) 6m ago 39m 87.1M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:01:21.206 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (31m) 8m ago 39m 103M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:01:21.206 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (35m) 8m ago 35m 26.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:01:21.207 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (35m) 6m ago 35m 25.6M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:01:21.207 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (30m) 8m ago 38m 133M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:01:21.207 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (29m) 8m ago 38m 125M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:01:21.207 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (29m) 8m ago 38m 105M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:01:21.208 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (29m) 8m ago 37m 116M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:01:21.208 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (28m) 6m ago 37m 111M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:01:21.208 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (28m) 6m ago 37m 125M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:01:21.208 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (27m) 6m ago 36m 110M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:01:21.209 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (26m) 6m ago 36m 121M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:01:21.209 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (32m) 6m ago 35m 89.8M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:01:21.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:01:21 smithi153 ceph-mon[70044]: pgmap v995: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:21.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:01:21 smithi153 ceph-mon[70044]: from='client.45229 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:01:21.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:01:21 smithi083 ceph-mon[79930]: pgmap v995: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:21.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:01:21 smithi083 ceph-mon[79930]: from='client.45229 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:01:21.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:01:21 smithi083 ceph-mon[77242]: pgmap v995: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:21.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:01:21 smithi083 ceph-mon[77242]: from='client.45229 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:01:21.560 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:01:21.560 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:01:21.560 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:01:21.561 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:01:21.561 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:01:21.561 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:01:21.561 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:01:21.561 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:01:21.562 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:01:21.562 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:01:21.562 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:01:21.562 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:01:21.563 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:01:21.563 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:01:21.563 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:01:21.563 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:01:21.564 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:01:21.564 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:01:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:01:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:01:21] "GET /metrics HTTP/1.1" 200 225170 "" "Prometheus/2.33.4" 2022-09-29T06:01:22.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:01:22 smithi153 ceph-mon[70044]: from='client.35301 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:01:22.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:01:22 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/909133522' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:01:22.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:01:22 smithi083 ceph-mon[79930]: from='client.35301 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:01:22.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:01:22 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/909133522' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:01:22.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:01:22 smithi083 ceph-mon[77242]: from='client.35301 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:01:22.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:01:22 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/909133522' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:01:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:01:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:01:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:01:23.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:01:23 smithi153 ceph-mon[70044]: pgmap v996: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:23.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:01:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:01:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:01:23.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:01:23 smithi083 ceph-mon[79930]: pgmap v996: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:23.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:01:23 smithi083 ceph-mon[77242]: pgmap v996: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:23.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:01:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:01:23.641Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:01:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:01:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:01:24.338Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:01:25.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:01:25 smithi153 ceph-mon[70044]: pgmap v997: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:25.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:01:25 smithi083 ceph-mon[79930]: pgmap v997: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:25.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:01:25 smithi083 ceph-mon[77242]: pgmap v997: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:27.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:01:27 smithi153 ceph-mon[70044]: pgmap v998: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:27.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:01:27 smithi083 ceph-mon[79930]: pgmap v998: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:27.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:01:27 smithi083 ceph-mon[77242]: pgmap v998: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:29.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:01:29 smithi153 ceph-mon[70044]: pgmap v999: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:29.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:01:29 smithi083 ceph-mon[79930]: pgmap v999: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:29.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:01:29 smithi083 ceph-mon[77242]: pgmap v999: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:30.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:01:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:01:30.232Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:01:31.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:01:31 smithi153 ceph-mon[70044]: pgmap v1000: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:31.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:01:31 smithi083 ceph-mon[79930]: pgmap v1000: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:31.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:01:31 smithi083 ceph-mon[77242]: pgmap v1000: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:01:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:01:31] "GET /metrics HTTP/1.1" 200 225169 "" "Prometheus/2.33.4" 2022-09-29T06:01:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:01:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:01:32.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:01:33.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:01:33 smithi153 ceph-mon[70044]: pgmap v1001: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:33.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:01:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:01:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:01:33.644 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:01:33 smithi083 ceph-mon[79930]: pgmap v1001: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:33.644 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:01:33 smithi083 ceph-mon[77242]: pgmap v1001: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:33.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:01:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:01:33.642Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:01:34.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:01:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:01:34.338Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:01:35.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:01:35 smithi153 ceph-mon[70044]: pgmap v1002: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:35.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:01:35 smithi083 ceph-mon[79930]: pgmap v1002: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:35.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:01:35 smithi083 ceph-mon[77242]: pgmap v1002: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:37.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:01:37 smithi153 ceph-mon[70044]: pgmap v1003: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:37.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:01:37 smithi083 ceph-mon[79930]: pgmap v1003: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:37.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:01:37 smithi083 ceph-mon[77242]: pgmap v1003: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:39.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:01:39 smithi153 ceph-mon[70044]: pgmap v1004: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:39.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:01:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:01:39.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:01:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:01:39.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:01:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:01:39.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:01:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:01:39.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:01:39 smithi083 ceph-mon[79930]: pgmap v1004: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:39.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:01:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:01:39.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:01:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:01:39.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:01:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:01:39.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:01:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:01:39.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:01:39 smithi083 ceph-mon[77242]: pgmap v1004: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:39.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:01:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:01:39.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:01:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:01:39.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:01:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:01:39.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:01:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:01:40.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:01:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:01:40.232Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:01:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:01:41 smithi153 ceph-mon[70044]: pgmap v1005: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:41.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:01:41 smithi083 ceph-mon[79930]: pgmap v1005: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:41.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:01:41 smithi083 ceph-mon[77242]: pgmap v1005: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:01:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:01:41] "GET /metrics HTTP/1.1" 200 225172 "" "Prometheus/2.33.4" 2022-09-29T06:01:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:01:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:01:42.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:01:43.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:01:43 smithi153 ceph-mon[70044]: pgmap v1006: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:43.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:01:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:01:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:01:43.645 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:01:43 smithi083 ceph-mon[79930]: pgmap v1006: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:43.645 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:01:43 smithi083 ceph-mon[77242]: pgmap v1006: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:43.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:01:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:01:43.643Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:01:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:01:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:01:44.339Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:01:45.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:01:45 smithi153 ceph-mon[70044]: pgmap v1007: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:45.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:01:45 smithi083 ceph-mon[79930]: pgmap v1007: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:45.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:01:45 smithi083 ceph-mon[77242]: pgmap v1007: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:47.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:01:47 smithi153 ceph-mon[70044]: pgmap v1008: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:47.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:01:47 smithi083 ceph-mon[79930]: pgmap v1008: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:47.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:01:47 smithi083 ceph-mon[77242]: pgmap v1008: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:49.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:01:49 smithi153 ceph-mon[70044]: pgmap v1009: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:49.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:01:49 smithi083 ceph-mon[79930]: pgmap v1009: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:49.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:01:49 smithi083 ceph-mon[77242]: pgmap v1009: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:50.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:01:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:01:50.232Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:01:51.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:01:51 smithi153 ceph-mon[70044]: pgmap v1010: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:51.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:01:51 smithi083 ceph-mon[77242]: pgmap v1010: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:51.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:01:51 smithi083 ceph-mon[79930]: pgmap v1010: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:51.884 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:01:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:01:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:01:51] "GET /metrics HTTP/1.1" 200 225172 "" "Prometheus/2.33.4" 2022-09-29T06:01:52.176 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:01:52.176 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (35m) 8m ago 36m 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:01:52.176 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (35m) 6m ago 35m 60.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:01:52.177 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (36m) 8m ago 36m 19.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:01:52.177 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (36m) 6m ago 36m 25.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:01:52.177 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (32m) 6m ago 39m 438M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:01:52.177 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (33m) 8m ago 41m 527M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:01:52.177 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (32m) 8m ago 41m 166M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:01:52.178 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (31m) 6m ago 39m 87.1M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:01:52.178 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (31m) 8m ago 39m 103M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:01:52.178 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (36m) 8m ago 36m 26.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:01:52.178 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (36m) 6m ago 36m 25.6M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:01:52.179 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (31m) 8m ago 39m 133M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:01:52.179 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (30m) 8m ago 38m 125M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:01:52.179 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (30m) 8m ago 38m 105M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:01:52.179 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (29m) 8m ago 38m 116M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:01:52.179 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (29m) 6m ago 38m 111M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:01:52.180 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (28m) 6m ago 37m 125M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:01:52.180 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (28m) 6m ago 37m 110M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:01:52.180 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (27m) 6m ago 37m 121M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:01:52.180 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (32m) 6m ago 36m 89.8M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:01:52.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:01:52 smithi153 ceph-mon[70044]: from='client.35313 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:01:52.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:01:52 smithi083 ceph-mon[79930]: from='client.35313 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:01:52.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:01:52 smithi083 ceph-mon[77242]: from='client.35313 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:01:52.534 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:01:52.534 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:01:52.534 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:01:52.534 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:01:52.535 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:01:52.535 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:01:52.535 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:01:52.535 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:01:52.536 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:01:52.536 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:01:52.536 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:01:52.536 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:01:52.536 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:01:52.537 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:01:52.537 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:01:52.537 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:01:52.537 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:01:52.538 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:01:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:01:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:01:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:01:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:01:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:01:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:01:53.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:01:53 smithi153 ceph-mon[70044]: from='client.35319 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:01:53.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:01:53 smithi153 ceph-mon[70044]: pgmap v1011: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:53.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:01:53 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2326339338' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:01:53.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:01:53 smithi083 ceph-mon[79930]: from='client.35319 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:01:53.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:01:53 smithi083 ceph-mon[79930]: pgmap v1011: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:53.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:01:53 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2326339338' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:01:53.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:01:53 smithi083 ceph-mon[77242]: from='client.35319 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:01:53.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:01:53 smithi083 ceph-mon[77242]: pgmap v1011: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:53.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:01:53 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2326339338' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:01:53.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:01:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:01:53.643Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:01:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:01:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:01:54.339Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:01:55.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:01:55 smithi153 ceph-mon[70044]: pgmap v1012: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:55.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:01:55 smithi083 ceph-mon[79930]: pgmap v1012: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:55.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:01:55 smithi083 ceph-mon[77242]: pgmap v1012: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:57.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:01:57 smithi153 ceph-mon[70044]: pgmap v1013: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:57.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:01:57 smithi083 ceph-mon[79930]: pgmap v1013: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:57.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:01:57 smithi083 ceph-mon[77242]: pgmap v1013: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:59.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:01:59 smithi153 ceph-mon[70044]: pgmap v1014: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:59.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:01:59 smithi083 ceph-mon[79930]: pgmap v1014: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:01:59.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:01:59 smithi083 ceph-mon[77242]: pgmap v1014: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:00.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:02:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:02:00.232Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:02:01.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:02:01 smithi153 ceph-mon[70044]: pgmap v1015: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:01.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:02:01 smithi083 ceph-mon[79930]: pgmap v1015: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:01.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:02:01 smithi083 ceph-mon[77242]: pgmap v1015: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:02:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:02:01] "GET /metrics HTTP/1.1" 200 225165 "" "Prometheus/2.33.4" 2022-09-29T06:02:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:02:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:02:02.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:02:03.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:02:03 smithi153 ceph-mon[70044]: pgmap v1016: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:02:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:02:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:02:03.646 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:02:03 smithi083 ceph-mon[79930]: pgmap v1016: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:03.646 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:02:03 smithi083 ceph-mon[77242]: pgmap v1016: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:03.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:02:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:02:03.644Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:02:04.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:02:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:02:04.340Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:02:05.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:02:05 smithi153 ceph-mon[70044]: pgmap v1017: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:05.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:02:05 smithi083 ceph-mon[79930]: pgmap v1017: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:05.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:02:05 smithi083 ceph-mon[77242]: pgmap v1017: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:07.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:02:07 smithi153 ceph-mon[70044]: pgmap v1018: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:07.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:02:07 smithi083 ceph-mon[79930]: pgmap v1018: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:07.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:02:07 smithi083 ceph-mon[77242]: pgmap v1018: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:09.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:02:09 smithi153 ceph-mon[70044]: pgmap v1019: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:09.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:02:09 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:02:09.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:02:09 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:02:09.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:02:09 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:02:09.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:02:09 smithi083 ceph-mon[79930]: pgmap v1019: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:09.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:02:09 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:02:09.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:02:09 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:02:09.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:02:09 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:02:09.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:02:09 smithi083 ceph-mon[77242]: pgmap v1019: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:09.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:02:09 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:02:09.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:02:09 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:02:09.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:02:09 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:02:10.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:02:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:02:10.232Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:02:11.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:02:10 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:02:11.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:02:10 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:02:11.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:02:10 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:02:11.812 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:02:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:02:11] "GET /metrics HTTP/1.1" 200 225169 "" "Prometheus/2.33.4" 2022-09-29T06:02:12.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:02:11 smithi083 ceph-mon[79930]: pgmap v1020: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:12.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:02:11 smithi083 ceph-mon[77242]: pgmap v1020: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:12.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:02:11 smithi153 ceph-mon[70044]: pgmap v1020: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:02:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:02:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:02:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:02:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:02:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:02:13.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:02:13 smithi083 ceph-mon[79930]: pgmap v1021: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:13.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:02:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:02:13.645Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:02:13.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:02:13 smithi083 ceph-mon[77242]: pgmap v1021: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:14.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:02:13 smithi153 ceph-mon[70044]: pgmap v1021: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:14.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:02:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:02:14.341Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:02:16.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:02:15 smithi083 ceph-mon[77242]: pgmap v1022: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:16.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:02:15 smithi083 ceph-mon[79930]: pgmap v1022: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:16.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:02:15 smithi153 ceph-mon[70044]: pgmap v1022: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:18.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:02:17 smithi083 ceph-mon[79930]: pgmap v1023: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:18.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:02:17 smithi083 ceph-mon[77242]: pgmap v1023: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:18.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:02:17 smithi153 ceph-mon[70044]: pgmap v1023: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:20.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:02:19 smithi083 ceph-mon[79930]: pgmap v1024: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:20.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:02:19 smithi083 ceph-mon[77242]: pgmap v1024: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:20.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:02:19 smithi153 ceph-mon[70044]: pgmap v1024: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:20.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:02:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:02:20.234Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:02:21.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:02:21 smithi083 ceph-mon[79930]: pgmap v1025: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:21.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:02:21 smithi083 ceph-mon[77242]: pgmap v1025: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:21.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:02:21 smithi153 ceph-mon[70044]: pgmap v1025: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:02:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:02:21] "GET /metrics HTTP/1.1" 200 225169 "" "Prometheus/2.33.4" 2022-09-29T06:02:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:02:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:02:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:02:22.860 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:02:23.151 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:02:23.152 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (36m) 9m ago 36m 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:02:23.152 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (36m) 7m ago 36m 60.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:02:23.152 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (37m) 9m ago 37m 19.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:02:23.152 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (37m) 7m ago 37m 25.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:02:23.153 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (33m) 7m ago 39m 438M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:02:23.153 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (33m) 9m ago 41m 527M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:02:23.153 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (32m) 9m ago 41m 166M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:02:23.153 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (32m) 7m ago 40m 87.1M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:02:23.153 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (32m) 9m ago 40m 103M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:02:23.154 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (36m) 9m ago 36m 26.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:02:23.154 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (36m) 7m ago 36m 25.6M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:02:23.154 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (31m) 9m ago 39m 133M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:02:23.155 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (31m) 9m ago 39m 125M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:02:23.155 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (30m) 9m ago 39m 105M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:02:23.155 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (30m) 9m ago 38m 116M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:02:23.155 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (29m) 7m ago 38m 111M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:02:23.155 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (29m) 7m ago 38m 125M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:02:23.156 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (28m) 7m ago 37m 110M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:02:23.156 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (27m) 7m ago 37m 121M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:02:23.156 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (33m) 7m ago 36m 89.8M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:02:23.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:02:23 smithi153 ceph-mon[70044]: pgmap v1026: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:23.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:02:23 smithi153 ceph-mon[70044]: from='client.45265 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:02:23.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:02:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:02:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:02:23.516 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:02:23.517 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:02:23.517 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:02:23.517 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:02:23.517 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:02:23.517 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:02:23.518 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:02:23.518 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:02:23.518 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:02:23.518 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:02:23.519 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:02:23.519 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:02:23.519 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:02:23.519 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:02:23.520 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:02:23.520 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:02:23.520 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:02:23.520 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:02:23.647 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:02:23 smithi083 ceph-mon[77242]: pgmap v1026: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:23.647 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:02:23 smithi083 ceph-mon[77242]: from='client.45265 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:02:23.647 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:02:23 smithi083 ceph-mon[79930]: pgmap v1026: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:23.648 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:02:23 smithi083 ceph-mon[79930]: from='client.45265 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:02:23.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:02:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:02:23.645Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:02:24.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:02:24 smithi153 ceph-mon[70044]: from='client.35337 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:02:24.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:02:24 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2426129485' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:02:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:02:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:02:24.341Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:02:24.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:02:24 smithi083 ceph-mon[77242]: from='client.35337 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:02:24.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:02:24 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2426129485' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:02:24.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:02:24 smithi083 ceph-mon[79930]: from='client.35337 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:02:24.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:02:24 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2426129485' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:02:25.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:02:25 smithi153 ceph-mon[70044]: pgmap v1027: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:25.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:02:25 smithi083 ceph-mon[79930]: pgmap v1027: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:25.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:02:25 smithi083 ceph-mon[77242]: pgmap v1027: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:27.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:02:27 smithi153 ceph-mon[70044]: pgmap v1028: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:27.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:02:27 smithi083 ceph-mon[79930]: pgmap v1028: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:27.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:02:27 smithi083 ceph-mon[77242]: pgmap v1028: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:29.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:02:29 smithi153 ceph-mon[70044]: pgmap v1029: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:29.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:02:29 smithi083 ceph-mon[79930]: pgmap v1029: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:29.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:02:29 smithi083 ceph-mon[77242]: pgmap v1029: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:30.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:02:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:02:30.235Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:02:31.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:02:31 smithi083 ceph-mon[77242]: pgmap v1030: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:31.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:02:31 smithi083 ceph-mon[79930]: pgmap v1030: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:31.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:02:31 smithi153 ceph-mon[70044]: pgmap v1030: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:02:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:02:31] "GET /metrics HTTP/1.1" 200 225171 "" "Prometheus/2.33.4" 2022-09-29T06:02:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:02:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:02:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:02:33.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:02:33 smithi153 ceph-mon[70044]: pgmap v1031: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:33.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:02:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:02:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:02:33.649 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:02:33 smithi083 ceph-mon[79930]: pgmap v1031: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:33.649 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:02:33 smithi083 ceph-mon[77242]: pgmap v1031: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:33.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:02:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:02:33.647Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:02:34.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:02:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:02:34.342Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:02:35.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:02:35 smithi153 ceph-mon[70044]: pgmap v1032: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:35.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:02:35 smithi083 ceph-mon[79930]: pgmap v1032: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:35.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:02:35 smithi083 ceph-mon[77242]: pgmap v1032: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:37.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:02:37 smithi153 ceph-mon[70044]: pgmap v1033: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:37.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:02:37 smithi083 ceph-mon[79930]: pgmap v1033: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:37.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:02:37 smithi083 ceph-mon[77242]: pgmap v1033: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:39.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:02:39 smithi153 ceph-mon[70044]: pgmap v1034: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:39.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:02:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:02:39.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:02:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:02:39.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:02:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:02:39.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:02:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:02:39.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:02:39 smithi083 ceph-mon[79930]: pgmap v1034: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:39.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:02:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:02:39.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:02:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:02:39.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:02:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:02:39.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:02:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:02:39.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:02:39 smithi083 ceph-mon[77242]: pgmap v1034: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:39.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:02:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:02:39.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:02:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:02:39.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:02:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:02:39.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:02:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:02:40.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:02:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:02:40.236Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:02:41.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:02:41 smithi083 ceph-mon[79930]: pgmap v1035: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:41.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:02:41 smithi083 ceph-mon[77242]: pgmap v1035: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:02:41 smithi153 ceph-mon[70044]: pgmap v1035: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:02:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:02:41] "GET /metrics HTTP/1.1" 200 225175 "" "Prometheus/2.33.4" 2022-09-29T06:02:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:02:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:02:42.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:02:43.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:02:43 smithi153 ceph-mon[70044]: pgmap v1036: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:02:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:02:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:02:43.649 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:02:43 smithi083 ceph-mon[79930]: pgmap v1036: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:43.650 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:02:43 smithi083 ceph-mon[77242]: pgmap v1036: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:43.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:02:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:02:43.647Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:02:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:02:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:02:44.343Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:02:45.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:02:45 smithi153 ceph-mon[70044]: pgmap v1037: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:45.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:02:45 smithi083 ceph-mon[79930]: pgmap v1037: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:45.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:02:45 smithi083 ceph-mon[77242]: pgmap v1037: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:47.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:02:47 smithi153 ceph-mon[70044]: pgmap v1038: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:47.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:02:47 smithi083 ceph-mon[79930]: pgmap v1038: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:47.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:02:47 smithi083 ceph-mon[77242]: pgmap v1038: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:49.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:02:49 smithi153 ceph-mon[70044]: pgmap v1039: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:49.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:02:49 smithi083 ceph-mon[79930]: pgmap v1039: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:49.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:02:49 smithi083 ceph-mon[77242]: pgmap v1039: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:50.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:02:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:02:50.237Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:02:51.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:02:51 smithi083 ceph-mon[79930]: pgmap v1040: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:51.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:02:51 smithi083 ceph-mon[77242]: pgmap v1040: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:51.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:02:51 smithi153 ceph-mon[70044]: pgmap v1040: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:02:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:02:51] "GET /metrics HTTP/1.1" 200 225175 "" "Prometheus/2.33.4" 2022-09-29T06:02:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:02:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:02:52.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:02:53.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:02:53 smithi153 ceph-mon[70044]: pgmap v1041: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:53.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:02:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:02:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:02:53.649 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:02:53 smithi083 ceph-mon[79930]: pgmap v1041: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:53.650 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:02:53 smithi083 ceph-mon[77242]: pgmap v1041: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:53.854 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:02:53.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:02:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:02:53.647Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:02:54.145 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:02:54.145 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (36m) 9m ago 37m 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:02:54.146 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (36m) 7m ago 36m 60.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:02:54.146 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (37m) 9m ago 37m 19.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:02:54.146 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (37m) 7m ago 37m 25.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:02:54.146 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (33m) 7m ago 40m 438M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:02:54.147 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (34m) 9m ago 42m 527M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:02:54.147 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (33m) 9m ago 42m 166M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:02:54.147 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (32m) 7m ago 40m 87.1M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:02:54.147 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (33m) 9m ago 40m 103M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:02:54.148 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (37m) 9m ago 37m 26.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:02:54.148 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (37m) 7m ago 37m 25.6M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:02:54.148 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (32m) 9m ago 40m 133M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:02:54.148 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (31m) 9m ago 39m 125M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:02:54.148 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (31m) 9m ago 39m 105M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:02:54.149 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (30m) 9m ago 39m 116M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:02:54.149 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (30m) 7m ago 39m 111M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:02:54.149 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (29m) 7m ago 38m 125M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:02:54.149 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (29m) 7m ago 38m 110M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:02:54.150 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (28m) 7m ago 38m 121M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:02:54.150 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (33m) 7m ago 37m 89.8M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:02:54.506 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:02:54.506 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:02:54.507 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:02:54.507 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:02:54.507 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:02:54.507 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:02:54.508 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:02:54.508 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:02:54.508 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:02:54.508 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:02:54.508 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:02:54.509 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:02:54.509 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:02:54.509 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:02:54.509 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:02:54.510 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:02:54.510 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:02:54.510 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:02:54.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:02:54 smithi153 ceph-mon[70044]: from='client.35349 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:02:54.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:02:54 smithi083 ceph-mon[79930]: from='client.35349 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:02:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:02:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:02:54.343Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:02:54.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:02:54 smithi083 ceph-mon[77242]: from='client.35349 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:02:55.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:02:55 smithi153 ceph-mon[70044]: from='client.45289 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:02:55.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:02:55 smithi153 ceph-mon[70044]: pgmap v1042: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:55.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:02:55 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/342349617' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:02:55.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:02:55 smithi083 ceph-mon[79930]: from='client.45289 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:02:55.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:02:55 smithi083 ceph-mon[79930]: pgmap v1042: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:55.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:02:55 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/342349617' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:02:55.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:02:55 smithi083 ceph-mon[77242]: from='client.45289 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:02:55.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:02:55 smithi083 ceph-mon[77242]: pgmap v1042: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:55.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:02:55 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/342349617' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:02:57.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:02:57 smithi153 ceph-mon[70044]: pgmap v1043: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:57.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:02:57 smithi083 ceph-mon[79930]: pgmap v1043: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:57.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:02:57 smithi083 ceph-mon[77242]: pgmap v1043: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:59.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:02:59 smithi153 ceph-mon[70044]: pgmap v1044: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:59.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:02:59 smithi083 ceph-mon[77242]: pgmap v1044: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:02:59.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:02:59 smithi083 ceph-mon[79930]: pgmap v1044: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:00.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:03:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:03:00.238Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:03:01.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:03:01 smithi083 ceph-mon[77242]: pgmap v1045: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:01.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:03:01 smithi083 ceph-mon[79930]: pgmap v1045: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:01.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:03:01 smithi153 ceph-mon[70044]: pgmap v1045: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:03:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:03:01] "GET /metrics HTTP/1.1" 200 225174 "" "Prometheus/2.33.4" 2022-09-29T06:03:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:03:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:03:02.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:03:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:03:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:03:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:03:03.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:03:03 smithi153 ceph-mon[70044]: pgmap v1046: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:03.651 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:03:03 smithi083 ceph-mon[79930]: pgmap v1046: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:03.651 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:03:03 smithi083 ceph-mon[77242]: pgmap v1046: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:03.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:03:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:03:03.649Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:03:04.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:03:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:03:04.344Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:03:05.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:03:05 smithi153 ceph-mon[70044]: pgmap v1047: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:05.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:03:05 smithi083 ceph-mon[79930]: pgmap v1047: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:05.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:03:05 smithi083 ceph-mon[77242]: pgmap v1047: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:07.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:03:07 smithi153 ceph-mon[70044]: pgmap v1048: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:07.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:03:07 smithi083 ceph-mon[79930]: pgmap v1048: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:07.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:03:07 smithi083 ceph-mon[77242]: pgmap v1048: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:09.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:03:09 smithi153 ceph-mon[70044]: pgmap v1049: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:09.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:03:09 smithi083 ceph-mon[79930]: pgmap v1049: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:09.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:03:09 smithi083 ceph-mon[77242]: pgmap v1049: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:10.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:03:10 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:03:10.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:03:10 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:03:10.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:03:10 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:03:10.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:03:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:03:10.237Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:03:10.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:03:10 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:03:10.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:03:10 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:03:10.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:03:10 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:03:10.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:03:10 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:03:10.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:03:10 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:03:10.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:03:10 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:03:11.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:03:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:03:11] "GET /metrics HTTP/1.1" 200 225174 "" "Prometheus/2.33.4" 2022-09-29T06:03:11.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:03:11 smithi083 ceph-mon[79930]: pgmap v1050: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:11.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:03:11 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:03:11.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:03:11 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:03:11.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:03:11 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:03:11.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:03:11 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:03:11.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:03:11 smithi083 ceph-mon[77242]: pgmap v1050: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:11.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:03:11 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:03:11.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:03:11 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:03:11.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:03:11 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:03:11.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:03:11 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:03:11.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:03:11 smithi153 ceph-mon[70044]: pgmap v1050: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:11.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:03:11 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:03:11.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:03:11 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:03:11.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:03:11 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:03:11.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:03:11 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:03:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:03:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:03:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:03:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:03:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:03:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:03:13.350 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:03:13 smithi083 ceph-mon[77242]: pgmap v1051: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:13.652 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:03:13 smithi083 ceph-mon[79930]: pgmap v1051: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:13.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:03:13 smithi153 ceph-mon[70044]: pgmap v1051: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:13.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:03:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:03:13.649Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:03:14.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:03:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:03:14.344Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:03:15.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:03:15 smithi083 ceph-mon[79930]: pgmap v1052: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:15.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:03:15 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:03:15.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:03:15 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:03:15.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:03:15 smithi083 ceph-mon[77242]: pgmap v1052: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:15.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:03:15 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:03:15.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:03:15 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:03:16.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:03:15 smithi153 ceph-mon[70044]: pgmap v1052: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:16.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:03:15 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:03:16.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:03:15 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:03:16.904 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:03:16 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:03:16.905 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:03:16 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:03:16.905 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:03:16 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:03:16.905 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:03:16 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:03:17.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:03:16 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:03:17.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:03:16 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:03:18.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:03:17 smithi153 ceph-mon[70044]: pgmap v1053: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:18.071 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:03:17 smithi083 ceph-mon[79930]: pgmap v1053: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:18.071 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:03:17 smithi083 ceph-mon[77242]: pgmap v1053: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:19.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:03:19 smithi083 ceph-mon[77242]: pgmap v1054: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:19.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:03:19 smithi083 ceph-mon[77242]: Detected new or changed devices on smithi083 2022-09-29T06:03:19.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:03:19 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:03:19.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:03:19 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:03:19.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:03:19 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:03:19.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:03:19 smithi083 ceph-mon[79930]: pgmap v1054: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:19.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:03:19 smithi083 ceph-mon[79930]: Detected new or changed devices on smithi083 2022-09-29T06:03:19.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:03:19 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:03:19.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:03:19 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:03:19.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:03:19 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:03:19.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:03:19 smithi153 ceph-mon[70044]: pgmap v1054: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:19.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:03:19 smithi153 ceph-mon[70044]: Detected new or changed devices on smithi083 2022-09-29T06:03:19.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:03:19 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:03:19.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:03:19 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:03:19.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:03:19 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:03:20.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:03:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:03:20.238Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:03:21.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:03:21 smithi083 ceph-mon[79930]: pgmap v1055: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:21.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:03:21 smithi083 ceph-mon[77242]: pgmap v1055: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:21.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:03:21 smithi153 ceph-mon[70044]: pgmap v1055: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:03:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:03:21] "GET /metrics HTTP/1.1" 200 225174 "" "Prometheus/2.33.4" 2022-09-29T06:03:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:03:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:03:22.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:03:23.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:03:23 smithi153 ceph-mon[70044]: pgmap v1056: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:23.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:03:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:03:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:03:23.653 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:03:23 smithi083 ceph-mon[79930]: pgmap v1056: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:23.653 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:03:23 smithi083 ceph-mon[77242]: pgmap v1056: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:23.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:03:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:03:23.650Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:03:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:03:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:03:24.345Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:03:24.831 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:03:25.120 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:03:25.120 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (37m) 10s ago 37m 26.7M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:03:25.121 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (37m) 8m ago 37m 60.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:03:25.121 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (38m) 10s ago 38m 21.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:03:25.121 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (38m) 8m ago 38m 25.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:03:25.121 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (34m) 8m ago 41m 438M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:03:25.121 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (34m) 10s ago 42m 538M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:03:25.122 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (33m) 10s ago 42m 204M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:03:25.122 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (33m) 8m ago 41m 87.1M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:03:25.122 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (33m) 10s ago 41m 124M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:03:25.122 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (37m) 10s ago 37m 25.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:03:25.123 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (37m) 8m ago 37m 25.6M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:03:25.123 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (32m) 10s ago 40m 158M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:03:25.123 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (32m) 10s ago 40m 150M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:03:25.123 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (31m) 10s ago 40m 124M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:03:25.123 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (31m) 10s ago 39m 141M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:03:25.124 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (30m) 8m ago 39m 111M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:03:25.124 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (30m) 8m ago 39m 125M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:03:25.124 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (29m) 8m ago 39m 110M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:03:25.124 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (29m) 8m ago 38m 121M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:03:25.124 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (34m) 8m ago 37m 89.8M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:03:25.480 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:03:25.480 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:03:25.480 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:03:25.480 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:03:25.481 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:03:25.481 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:03:25.481 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:03:25.481 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:03:25.481 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:03:25.482 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:03:25.482 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:03:25.482 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:03:25.482 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:03:25.483 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:03:25.483 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:03:25.483 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:03:25.483 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:03:25.483 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:03:25.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:03:25 smithi153 ceph-mon[70044]: pgmap v1057: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:25.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:03:25 smithi153 ceph-mon[70044]: from='client.45301 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:03:25.609 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:03:25 smithi083 ceph-mon[79930]: pgmap v1057: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:25.609 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:03:25 smithi083 ceph-mon[79930]: from='client.45301 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:03:25.609 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:03:25 smithi083 ceph-mon[77242]: pgmap v1057: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:25.609 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:03:25 smithi083 ceph-mon[77242]: from='client.45301 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:03:26.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:03:26 smithi153 ceph-mon[70044]: from='client.35373 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:03:26.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:03:26 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4155419564' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:03:26.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:03:26 smithi083 ceph-mon[79930]: from='client.35373 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:03:26.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:03:26 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4155419564' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:03:26.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:03:26 smithi083 ceph-mon[77242]: from='client.35373 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:03:26.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:03:26 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4155419564' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:03:27.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:03:27 smithi153 ceph-mon[70044]: pgmap v1058: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:27.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:03:27 smithi083 ceph-mon[79930]: pgmap v1058: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:27.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:03:27 smithi083 ceph-mon[77242]: pgmap v1058: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:29.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:03:29 smithi153 ceph-mon[70044]: pgmap v1059: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:29.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:03:29 smithi083 ceph-mon[79930]: pgmap v1059: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:29.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:03:29 smithi083 ceph-mon[77242]: pgmap v1059: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:30.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:03:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:03:30.239Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:03:31.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:03:31 smithi083 ceph-mon[79930]: pgmap v1060: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:31.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:03:31 smithi083 ceph-mon[77242]: pgmap v1060: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:31.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:03:31 smithi153 ceph-mon[70044]: pgmap v1060: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:03:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:03:31] "GET /metrics HTTP/1.1" 200 225173 "" "Prometheus/2.33.4" 2022-09-29T06:03:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:03:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:03:32.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:03:33.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:03:33 smithi153 ceph-mon[70044]: pgmap v1061: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:03:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:03:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:03:33.653 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:03:33 smithi083 ceph-mon[79930]: pgmap v1061: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:33.654 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:03:33 smithi083 ceph-mon[77242]: pgmap v1061: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:33.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:03:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:03:33.651Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:03:34.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:03:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:03:34.346Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:03:35.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:03:35 smithi153 ceph-mon[70044]: pgmap v1062: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:35.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:03:35 smithi083 ceph-mon[77242]: pgmap v1062: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:35.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:03:35 smithi083 ceph-mon[79930]: pgmap v1062: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:37.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:03:37 smithi153 ceph-mon[70044]: pgmap v1063: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:37.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:03:37 smithi083 ceph-mon[79930]: pgmap v1063: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:37.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:03:37 smithi083 ceph-mon[77242]: pgmap v1063: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:39.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:03:39 smithi153 ceph-mon[70044]: pgmap v1064: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:39.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:03:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:03:39.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:03:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:03:39.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:03:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:03:39.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:03:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:03:39.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:03:39 smithi083 ceph-mon[77242]: pgmap v1064: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:39.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:03:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:03:39.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:03:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:03:39.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:03:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:03:39.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:03:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:03:39.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:03:39 smithi083 ceph-mon[79930]: pgmap v1064: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:39.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:03:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:03:39.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:03:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:03:39.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:03:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:03:39.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:03:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:03:40.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:03:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:03:40.239Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:03:41.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:03:41 smithi083 ceph-mon[77242]: pgmap v1065: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:03:41 smithi083 ceph-mon[79930]: pgmap v1065: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:03:41 smithi153 ceph-mon[70044]: pgmap v1065: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:03:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:03:41] "GET /metrics HTTP/1.1" 200 225176 "" "Prometheus/2.33.4" 2022-09-29T06:03:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:03:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:03:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:03:43.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:03:43 smithi153 ceph-mon[70044]: pgmap v1066: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:43.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:03:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:03:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:03:43.654 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:03:43 smithi083 ceph-mon[79930]: pgmap v1066: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:43.654 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:03:43 smithi083 ceph-mon[77242]: pgmap v1066: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:43.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:03:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:03:43.652Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:03:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:03:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:03:44.347Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:03:45.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:03:45 smithi153 ceph-mon[70044]: pgmap v1067: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:45.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:03:45 smithi083 ceph-mon[79930]: pgmap v1067: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:45.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:03:45 smithi083 ceph-mon[77242]: pgmap v1067: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:47.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:03:47 smithi153 ceph-mon[70044]: pgmap v1068: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:47.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:03:47 smithi083 ceph-mon[79930]: pgmap v1068: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:47.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:03:47 smithi083 ceph-mon[77242]: pgmap v1068: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:49.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:03:49 smithi153 ceph-mon[70044]: pgmap v1069: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:49.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:03:49 smithi083 ceph-mon[79930]: pgmap v1069: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:49.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:03:49 smithi083 ceph-mon[77242]: pgmap v1069: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:50.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:03:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:03:50.240Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:03:51.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:03:51 smithi083 ceph-mon[79930]: pgmap v1070: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:51.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:03:51 smithi083 ceph-mon[77242]: pgmap v1070: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:51.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:03:51 smithi153 ceph-mon[70044]: pgmap v1070: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:03:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:03:51] "GET /metrics HTTP/1.1" 200 225176 "" "Prometheus/2.33.4" 2022-09-29T06:03:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:03:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:03:52.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:03:53.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:03:53 smithi153 ceph-mon[70044]: pgmap v1071: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:53.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:03:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:03:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:03:53.654 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:03:53 smithi083 ceph-mon[79930]: pgmap v1071: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:53.655 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:03:53 smithi083 ceph-mon[77242]: pgmap v1071: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:53.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:03:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:03:53.652Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:03:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:03:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:03:54.348Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:03:55.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:03:55 smithi153 ceph-mon[70044]: pgmap v1072: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:55.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:03:55 smithi083 ceph-mon[79930]: pgmap v1072: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:55.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:03:55 smithi083 ceph-mon[77242]: pgmap v1072: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:55.809 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:03:56.096 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:03:56.096 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (37m) 41s ago 38m 26.7M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:03:56.096 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (37m) 8m ago 37m 60.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:03:56.096 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (38m) 41s ago 38m 21.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:03:56.097 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (38m) 8m ago 38m 25.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:03:56.097 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (34m) 8m ago 41m 438M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:03:56.097 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (35m) 41s ago 43m 538M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:03:56.097 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (34m) 41s ago 43m 204M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:03:56.098 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (33m) 8m ago 41m 87.1M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:03:56.098 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (34m) 41s ago 41m 124M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:03:56.098 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (38m) 41s ago 38m 25.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:03:56.098 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (38m) 8m ago 38m 25.6M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:03:56.098 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (33m) 41s ago 41m 158M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:03:56.099 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (32m) 41s ago 40m 150M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:03:56.099 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (32m) 41s ago 40m 124M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:03:56.099 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (31m) 41s ago 40m 141M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:03:56.099 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (31m) 8m ago 40m 111M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:03:56.100 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (30m) 8m ago 39m 125M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:03:56.100 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (30m) 8m ago 39m 110M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:03:56.100 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (29m) 8m ago 39m 121M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:03:56.100 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (34m) 8m ago 38m 89.8M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:03:56.451 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:03:56.451 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:03:56.452 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:03:56.452 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:03:56.452 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:03:56.452 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:03:56.452 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:03:56.453 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:03:56.453 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:03:56.453 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:03:56.453 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:03:56.453 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:03:56.454 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:03:56.454 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:03:56.454 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:03:56.454 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:03:56.455 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:03:56.455 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:03:56.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:03:56 smithi153 ceph-mon[70044]: from='client.35385 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:03:56.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:03:56 smithi083 ceph-mon[79930]: from='client.35385 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:03:56.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:03:56 smithi083 ceph-mon[77242]: from='client.35385 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:03:57.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:03:57 smithi153 ceph-mon[70044]: from='client.35391 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:03:57.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:03:57 smithi153 ceph-mon[70044]: pgmap v1073: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:57.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:03:57 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3064262098' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:03:57.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:03:57 smithi083 ceph-mon[79930]: from='client.35391 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:03:57.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:03:57 smithi083 ceph-mon[79930]: pgmap v1073: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:57.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:03:57 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3064262098' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:03:57.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:03:57 smithi083 ceph-mon[77242]: from='client.35391 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:03:57.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:03:57 smithi083 ceph-mon[77242]: pgmap v1073: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:57.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:03:57 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3064262098' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:03:59.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:03:59 smithi153 ceph-mon[70044]: pgmap v1074: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:59.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:03:59 smithi083 ceph-mon[79930]: pgmap v1074: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:03:59.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:03:59 smithi083 ceph-mon[77242]: pgmap v1074: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:00.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:04:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:04:00.240Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:04:01.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:04:01 smithi083 ceph-mon[79930]: pgmap v1075: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:01.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:04:01 smithi083 ceph-mon[77242]: pgmap v1075: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:01.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:04:01 smithi153 ceph-mon[70044]: pgmap v1075: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:04:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:04:01] "GET /metrics HTTP/1.1" 200 225172 "" "Prometheus/2.33.4" 2022-09-29T06:04:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:04:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:04:02.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:04:03.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:04:03 smithi153 ceph-mon[70044]: pgmap v1076: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:04:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:04:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:04:03.655 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:04:03 smithi083 ceph-mon[79930]: pgmap v1076: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:03.656 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:04:03 smithi083 ceph-mon[77242]: pgmap v1076: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:03.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:04:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:04:03.653Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:04:04.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:04:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:04:04.348Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:04:05.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:04:05 smithi153 ceph-mon[70044]: pgmap v1077: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:05.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:04:05 smithi083 ceph-mon[79930]: pgmap v1077: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:05.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:04:05 smithi083 ceph-mon[77242]: pgmap v1077: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:07.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:04:07 smithi153 ceph-mon[70044]: pgmap v1078: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:07.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:04:07 smithi083 ceph-mon[79930]: pgmap v1078: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:07.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:04:07 smithi083 ceph-mon[77242]: pgmap v1078: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:09.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:04:09 smithi153 ceph-mon[70044]: pgmap v1079: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:09.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:04:09 smithi083 ceph-mon[79930]: pgmap v1079: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:09.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:04:09 smithi083 ceph-mon[77242]: pgmap v1079: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:10.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:04:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:04:10.241Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:04:11.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:04:11 smithi083 ceph-mon[79930]: pgmap v1080: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:11.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:04:11 smithi083 ceph-mon[77242]: pgmap v1080: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:11.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:04:11 smithi153 ceph-mon[70044]: pgmap v1080: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:04:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:04:11] "GET /metrics HTTP/1.1" 200 225175 "" "Prometheus/2.33.4" 2022-09-29T06:04:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:04:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:04:12.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:04:13.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:04:13 smithi153 ceph-mon[70044]: pgmap v1081: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:13.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:04:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:04:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:04:13.655 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:04:13 smithi083 ceph-mon[79930]: pgmap v1081: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:13.655 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:04:13 smithi083 ceph-mon[77242]: pgmap v1081: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:13.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:04:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:04:13.653Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:04:14.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:04:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:04:14.348Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:04:15.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:04:15 smithi153 ceph-mon[70044]: pgmap v1082: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:15.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:04:15 smithi083 ceph-mon[77242]: pgmap v1082: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:15.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:04:15 smithi083 ceph-mon[79930]: pgmap v1082: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:17.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:04:17 smithi153 ceph-mon[70044]: pgmap v1083: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:17.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:04:17 smithi083 ceph-mon[79930]: pgmap v1083: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:17.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:04:17 smithi083 ceph-mon[77242]: pgmap v1083: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:19.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:04:19 smithi153 ceph-mon[70044]: pgmap v1084: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:19.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:04:19 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:04:19.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:04:19 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:04:19.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:04:19 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:04:19.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:04:19 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:04:19.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:04:19 smithi083 ceph-mon[79930]: pgmap v1084: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:19.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:04:19 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:04:19.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:04:19 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:04:19.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:04:19 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:04:19.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:04:19 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:04:19.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:04:19 smithi083 ceph-mon[77242]: pgmap v1084: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:19.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:04:19 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:04:19.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:04:19 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:04:19.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:04:19 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:04:19.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:04:19 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:04:20.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:04:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:04:20.242Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:04:21.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:04:21 smithi083 ceph-mon[77242]: pgmap v1085: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:21.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:04:21 smithi083 ceph-mon[79930]: pgmap v1085: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:21.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:04:21 smithi153 ceph-mon[70044]: pgmap v1085: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:04:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:04:21] "GET /metrics HTTP/1.1" 200 225175 "" "Prometheus/2.33.4" 2022-09-29T06:04:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:04:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:04:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:04:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:04:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:04:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:04:23.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:04:23 smithi153 ceph-mon[70044]: pgmap v1086: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:23.657 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:04:23 smithi083 ceph-mon[79930]: pgmap v1086: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:23.657 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:04:23 smithi083 ceph-mon[77242]: pgmap v1086: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:23.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:04:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:04:23.654Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:04:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:04:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:04:24.349Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:04:25.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:04:25 smithi153 ceph-mon[70044]: pgmap v1087: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:25.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:04:25 smithi083 ceph-mon[79930]: pgmap v1087: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:25.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:04:25 smithi083 ceph-mon[77242]: pgmap v1087: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:26.778 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:04:27.068 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:04:27.068 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (38m) 72s ago 38m 26.7M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:04:27.069 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (38m) 9m ago 38m 60.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:04:27.069 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (39m) 72s ago 39m 21.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:04:27.069 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (39m) 9m ago 39m 25.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:04:27.069 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (35m) 9m ago 42m 438M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:04:27.070 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (35m) 72s ago 43m 538M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:04:27.070 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (34m) 72s ago 43m 204M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:04:27.070 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (34m) 9m ago 42m 87.1M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:04:27.070 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (34m) 72s ago 42m 124M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:04:27.071 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (38m) 72s ago 38m 25.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:04:27.071 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (38m) 9m ago 38m 25.6M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:04:27.071 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (33m) 72s ago 41m 158M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:04:27.072 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (33m) 72s ago 41m 150M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:04:27.072 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (32m) 72s ago 41m 124M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:04:27.072 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (32m) 72s ago 40m 141M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:04:27.072 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (31m) 9m ago 40m 111M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:04:27.073 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (31m) 9m ago 40m 125M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:04:27.073 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (30m) 9m ago 40m 110M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:04:27.073 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (30m) 9m ago 39m 121M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:04:27.073 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (35m) 9m ago 38m 89.8M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:04:27.424 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:04:27.424 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:04:27.424 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:04:27.425 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:04:27.425 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:04:27.425 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:04:27.425 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:04:27.426 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:04:27.426 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:04:27.426 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:04:27.426 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:04:27.426 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:04:27.427 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:04:27.427 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:04:27.427 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:04:27.427 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:04:27.428 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:04:27.428 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:04:27.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:04:27 smithi153 ceph-mon[70044]: pgmap v1088: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:27.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:04:27 smithi153 ceph-mon[70044]: from='client.35403 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:04:27.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:04:27 smithi083 ceph-mon[77242]: pgmap v1088: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:27.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:04:27 smithi083 ceph-mon[77242]: from='client.35403 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:04:27.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:04:27 smithi083 ceph-mon[79930]: pgmap v1088: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:27.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:04:27 smithi083 ceph-mon[79930]: from='client.35403 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:04:28.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:04:28 smithi153 ceph-mon[70044]: from='client.45343 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:04:28.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:04:28 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1864357346' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:04:28.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:04:28 smithi083 ceph-mon[77242]: from='client.45343 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:04:28.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:04:28 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1864357346' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:04:28.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:04:28 smithi083 ceph-mon[79930]: from='client.45343 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:04:28.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:04:28 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1864357346' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:04:29.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:04:29 smithi153 ceph-mon[70044]: pgmap v1089: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:29.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:04:29 smithi083 ceph-mon[77242]: pgmap v1089: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:29.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:04:29 smithi083 ceph-mon[79930]: pgmap v1089: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:30.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:04:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:04:30.241Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:04:31.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:04:31 smithi083 ceph-mon[79930]: pgmap v1090: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:31.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:04:31 smithi083 ceph-mon[77242]: pgmap v1090: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:31.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:04:31 smithi153 ceph-mon[70044]: pgmap v1090: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:04:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:04:31] "GET /metrics HTTP/1.1" 200 225175 "" "Prometheus/2.33.4" 2022-09-29T06:04:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:04:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:04:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:04:33.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:04:33 smithi153 ceph-mon[70044]: pgmap v1091: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:04:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:04:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:04:33.657 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:04:33 smithi083 ceph-mon[79930]: pgmap v1091: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:33.658 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:04:33 smithi083 ceph-mon[77242]: pgmap v1091: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:33.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:04:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:04:33.655Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:04:34.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:04:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:04:34.350Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:04:35.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:04:35 smithi153 ceph-mon[70044]: pgmap v1092: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:35.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:04:35 smithi083 ceph-mon[79930]: pgmap v1092: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:35.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:04:35 smithi083 ceph-mon[77242]: pgmap v1092: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:37.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:04:37 smithi153 ceph-mon[70044]: pgmap v1093: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:37.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:04:37 smithi083 ceph-mon[79930]: pgmap v1093: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:37.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:04:37 smithi083 ceph-mon[77242]: pgmap v1093: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:39.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:04:39 smithi153 ceph-mon[70044]: pgmap v1094: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:39.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:04:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:04:39.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:04:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:04:39.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:04:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:04:39.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:04:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:04:39.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:04:39 smithi083 ceph-mon[79930]: pgmap v1094: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:39.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:04:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:04:39.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:04:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:04:39.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:04:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:04:39.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:04:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:04:39.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:04:39 smithi083 ceph-mon[77242]: pgmap v1094: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:39.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:04:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:04:39.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:04:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:04:39.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:04:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:04:39.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:04:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:04:40.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:04:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:04:40.242Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:04:41.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:04:41 smithi083 ceph-mon[79930]: pgmap v1095: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:41.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:04:41 smithi083 ceph-mon[77242]: pgmap v1095: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:04:41 smithi153 ceph-mon[70044]: pgmap v1095: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:04:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:04:41] "GET /metrics HTTP/1.1" 200 225170 "" "Prometheus/2.33.4" 2022-09-29T06:04:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:04:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:04:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:04:43.277 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:04:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:04:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:04:43.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:04:43 smithi153 ceph-mon[70044]: pgmap v1096: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:43.658 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:04:43 smithi083 ceph-mon[79930]: pgmap v1096: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:43.659 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:04:43 smithi083 ceph-mon[77242]: pgmap v1096: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:43.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:04:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:04:43.656Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:04:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:04:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:04:44.350Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:04:45.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:04:45 smithi153 ceph-mon[70044]: pgmap v1097: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:45.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:04:45 smithi083 ceph-mon[79930]: pgmap v1097: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:45.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:04:45 smithi083 ceph-mon[77242]: pgmap v1097: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:47.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:04:47 smithi153 ceph-mon[70044]: pgmap v1098: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:47.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:04:47 smithi083 ceph-mon[77242]: pgmap v1098: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:47.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:04:47 smithi083 ceph-mon[79930]: pgmap v1098: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:49.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:04:49 smithi153 ceph-mon[70044]: pgmap v1099: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:49.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:04:49 smithi083 ceph-mon[79930]: pgmap v1099: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:49.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:04:49 smithi083 ceph-mon[77242]: pgmap v1099: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:50.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:04:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:04:50.243Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:04:51.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:04:51 smithi083 ceph-mon[79930]: pgmap v1100: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:51.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:04:51 smithi083 ceph-mon[77242]: pgmap v1100: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:51.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:04:51 smithi153 ceph-mon[70044]: pgmap v1100: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:04:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:04:51] "GET /metrics HTTP/1.1" 200 225170 "" "Prometheus/2.33.4" 2022-09-29T06:04:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:04:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:04:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:04:53.280 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:04:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:04:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:04:53.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:04:53 smithi153 ceph-mon[70044]: pgmap v1101: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:53.658 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:04:53 smithi083 ceph-mon[79930]: pgmap v1101: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:53.659 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:04:53 smithi083 ceph-mon[77242]: pgmap v1101: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:53.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:04:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:04:53.656Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:04:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:04:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:04:54.350Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:04:55.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:04:55 smithi153 ceph-mon[70044]: pgmap v1102: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:55.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:04:55 smithi083 ceph-mon[77242]: pgmap v1102: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:55.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:04:55 smithi083 ceph-mon[79930]: pgmap v1102: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:57.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:04:57 smithi153 ceph-mon[70044]: pgmap v1103: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:57.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:04:57 smithi083 ceph-mon[79930]: pgmap v1103: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:57.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:04:57 smithi083 ceph-mon[77242]: pgmap v1103: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:57.751 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:04:58.038 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:04:58.038 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (38m) 103s ago 39m 26.7M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:04:58.038 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (38m) 9m ago 38m 60.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:04:58.039 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (40m) 103s ago 40m 21.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:04:58.039 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (39m) 9m ago 39m 25.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:04:58.039 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (35m) 9m ago 42m 438M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:04:58.039 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (36m) 103s ago 44m 538M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:04:58.040 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (35m) 103s ago 44m 204M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:04:58.040 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (34m) 9m ago 42m 87.1M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:04:58.040 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (35m) 103s ago 42m 124M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:04:58.041 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (39m) 103s ago 39m 25.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:04:58.041 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (39m) 9m ago 39m 25.6M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:04:58.041 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (34m) 103s ago 42m 158M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:04:58.042 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (33m) 103s ago 42m 150M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:04:58.042 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (33m) 103s ago 41m 124M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:04:58.042 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (32m) 103s ago 41m 141M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:04:58.043 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (32m) 9m ago 41m 111M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:04:58.043 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (31m) 9m ago 40m 125M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:04:58.043 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (31m) 9m ago 40m 110M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:04:58.043 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (30m) 9m ago 40m 121M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:04:58.043 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (35m) 9m ago 39m 89.8M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:04:58.397 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:04:58.397 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:04:58.398 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:04:58.398 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:04:58.398 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:04:58.399 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:04:58.399 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:04:58.399 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:04:58.399 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:04:58.399 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:04:58.400 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:04:58.400 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:04:58.400 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:04:58.400 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:04:58.401 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:04:58.401 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:04:58.401 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:04:58.401 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:04:58.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:04:58 smithi153 ceph-mon[70044]: from='client.45355 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:04:58.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:04:58 smithi153 ceph-mon[70044]: from='client.45361 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:04:58.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:04:58 smithi083 ceph-mon[79930]: from='client.45355 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:04:58.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:04:58 smithi083 ceph-mon[79930]: from='client.45361 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:04:58.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:04:58 smithi083 ceph-mon[77242]: from='client.45355 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:04:58.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:04:58 smithi083 ceph-mon[77242]: from='client.45361 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:04:59.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:04:59 smithi153 ceph-mon[70044]: pgmap v1104: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:59.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:04:59 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3080487018' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:04:59.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:04:59 smithi083 ceph-mon[79930]: pgmap v1104: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:59.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:04:59 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3080487018' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:04:59.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:04:59 smithi083 ceph-mon[77242]: pgmap v1104: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:04:59.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:04:59 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3080487018' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:05:00.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:05:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:05:00.244Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:05:01.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:05:01 smithi083 ceph-mon[77242]: pgmap v1105: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:01.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:05:01 smithi083 ceph-mon[79930]: pgmap v1105: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:01.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:05:01 smithi153 ceph-mon[70044]: pgmap v1105: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:05:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:05:01] "GET /metrics HTTP/1.1" 200 225176 "" "Prometheus/2.33.4" 2022-09-29T06:05:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:05:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:05:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:05:03.284 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:05:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:05:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:05:03.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:05:03 smithi153 ceph-mon[70044]: pgmap v1106: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:03.659 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:05:03 smithi083 ceph-mon[79930]: pgmap v1106: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:03.659 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:05:03 smithi083 ceph-mon[77242]: pgmap v1106: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:03.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:05:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:05:03.657Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:05:04.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:05:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:05:04.351Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:05:05.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:05:05 smithi153 ceph-mon[70044]: pgmap v1107: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:05.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:05:05 smithi083 ceph-mon[79930]: pgmap v1107: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:05.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:05:05 smithi083 ceph-mon[77242]: pgmap v1107: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:07.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:05:07 smithi153 ceph-mon[70044]: pgmap v1108: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:07.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:05:07 smithi083 ceph-mon[79930]: pgmap v1108: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:07.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:05:07 smithi083 ceph-mon[77242]: pgmap v1108: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:09.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:05:09 smithi153 ceph-mon[70044]: pgmap v1109: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:09.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:05:09 smithi083 ceph-mon[79930]: pgmap v1109: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:09.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:05:09 smithi083 ceph-mon[77242]: pgmap v1109: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:10.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:05:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:05:10.244Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:05:11.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:05:11 smithi083 ceph-mon[79930]: pgmap v1110: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:11.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:05:11 smithi083 ceph-mon[77242]: pgmap v1110: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:11.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:05:11 smithi153 ceph-mon[70044]: pgmap v1110: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:05:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:05:11] "GET /metrics HTTP/1.1" 200 225169 "" "Prometheus/2.33.4" 2022-09-29T06:05:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:05:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:05:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:05:13.285 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:05:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:05:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:05:13.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:05:13 smithi153 ceph-mon[70044]: pgmap v1111: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:13.659 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:05:13 smithi083 ceph-mon[79930]: pgmap v1111: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:13.660 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:05:13 smithi083 ceph-mon[77242]: pgmap v1111: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:13.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:05:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:05:13.658Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:05:14.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:05:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:05:14.352Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:05:15.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:05:15 smithi153 ceph-mon[70044]: pgmap v1112: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:15.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:05:15 smithi083 ceph-mon[77242]: pgmap v1112: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:15.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:05:15 smithi083 ceph-mon[79930]: pgmap v1112: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:17.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:05:17 smithi153 ceph-mon[70044]: pgmap v1113: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:17.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:05:17 smithi083 ceph-mon[77242]: pgmap v1113: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:17.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:05:17 smithi083 ceph-mon[79930]: pgmap v1113: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:19.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:05:19 smithi153 ceph-mon[70044]: pgmap v1114: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:19.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:05:19 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:05:19.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:05:19 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:05:19.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:05:19 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:05:19.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:05:19 smithi083 ceph-mon[79930]: pgmap v1114: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:19.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:05:19 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:05:19.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:05:19 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:05:19.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:05:19 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:05:19.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:05:19 smithi083 ceph-mon[77242]: pgmap v1114: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:19.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:05:19 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:05:19.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:05:19 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:05:19.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:05:19 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:05:20.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:05:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:05:20.245Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:05:21.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:05:21 smithi083 ceph-mon[79930]: pgmap v1115: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:21.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:05:21 smithi083 ceph-mon[77242]: pgmap v1115: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:21.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:05:21 smithi153 ceph-mon[70044]: pgmap v1115: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:05:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:05:21] "GET /metrics HTTP/1.1" 200 225169 "" "Prometheus/2.33.4" 2022-09-29T06:05:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:05:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:05:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:05:23.288 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:05:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:05:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:05:23.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:05:23 smithi153 ceph-mon[70044]: pgmap v1116: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:23.660 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:05:23 smithi083 ceph-mon[79930]: pgmap v1116: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:23.660 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:05:23 smithi083 ceph-mon[77242]: pgmap v1116: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:23.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:05:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:05:23.658Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:05:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:05:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:05:24.352Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:05:24.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:05:24 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:05:24.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:05:24 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:05:24.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:05:24 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:05:24.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:05:24 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:05:24.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:05:24 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:05:24.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:05:24 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:05:24.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:05:24 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:05:24.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:05:24 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:05:24.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:05:24 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:05:24.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:05:24 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:05:24.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:05:24 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:05:24.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:05:24 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:05:25.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:05:25 smithi083 ceph-mon[77242]: pgmap v1117: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:25.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:05:25 smithi083 ceph-mon[79930]: pgmap v1117: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:26.052 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:05:25 smithi153 ceph-mon[70044]: pgmap v1117: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:28.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:05:28 smithi153 ceph-mon[70044]: pgmap v1118: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:28.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:05:28 smithi153 ceph-mon[70044]: Detected new or changed devices on smithi153 2022-09-29T06:05:28.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:05:28 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:05:28.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:05:28 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:05:28.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:05:28 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:05:28.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:05:28 smithi083 ceph-mon[79930]: pgmap v1118: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:28.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:05:28 smithi083 ceph-mon[79930]: Detected new or changed devices on smithi153 2022-09-29T06:05:28.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:05:28 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:05:28.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:05:28 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:05:28.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:05:28 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:05:28.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:05:28 smithi083 ceph-mon[77242]: pgmap v1118: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:28.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:05:28 smithi083 ceph-mon[77242]: Detected new or changed devices on smithi153 2022-09-29T06:05:28.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:05:28 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:05:28.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:05:28 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:05:28.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:05:28 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:05:28.724 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:05:29.015 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:05:29.016 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (39m) 2m ago 39m 26.7M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:05:29.016 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (39m) 5s ago 39m 61.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:05:29.016 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (40m) 2m ago 40m 21.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:05:29.016 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (40m) 5s ago 40m 27.6M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:05:29.017 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (36m) 5s ago 43m 439M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:05:29.017 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (36m) 2m ago 44m 538M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:05:29.017 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (35m) 2m ago 44m 204M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:05:29.017 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (35m) 5s ago 43m 101M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:05:29.018 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (35m) 2m ago 43m 124M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:05:29.018 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (39m) 2m ago 39m 25.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:05:29.018 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (39m) 5s ago 39m 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:05:29.018 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (34m) 2m ago 42m 158M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:05:29.019 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (34m) 2m ago 42m 150M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:05:29.019 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (33m) 2m ago 42m 124M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:05:29.019 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (33m) 2m ago 41m 141M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:05:29.019 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (32m) 5s ago 41m 133M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:05:29.020 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (32m) 5s ago 41m 152M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:05:29.020 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (31m) 5s ago 41m 135M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:05:29.020 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (31m) 5s ago 40m 150M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:05:29.020 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (36m) 5s ago 39m 100M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:05:29.371 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:05:29.371 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:05:29.371 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:05:29.372 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:05:29.372 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:05:29.372 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:05:29.372 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:05:29.373 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:05:29.373 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:05:29.373 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:05:29.373 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:05:29.374 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:05:29.374 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:05:29.374 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:05:29.374 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:05:29.375 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:05:29.375 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:05:29.375 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:05:30.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:05:30 smithi153 ceph-mon[70044]: pgmap v1119: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:30.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:05:30 smithi153 ceph-mon[70044]: from='client.45373 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:05:30.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:05:30 smithi153 ceph-mon[70044]: from='client.35445 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:05:30.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:05:30 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1304912538' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:05:30.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:05:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:05:30.245Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:05:30.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:05:30 smithi083 ceph-mon[77242]: pgmap v1119: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:30.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:05:30 smithi083 ceph-mon[77242]: from='client.45373 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:05:30.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:05:30 smithi083 ceph-mon[77242]: from='client.35445 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:05:30.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:05:30 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1304912538' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:05:30.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:05:30 smithi083 ceph-mon[79930]: pgmap v1119: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:30.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:05:30 smithi083 ceph-mon[79930]: from='client.45373 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:05:30.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:05:30 smithi083 ceph-mon[79930]: from='client.35445 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:05:30.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:05:30 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1304912538' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:05:31.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:05:31 smithi083 ceph-mon[79930]: pgmap v1120: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:31.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:05:31 smithi083 ceph-mon[77242]: pgmap v1120: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:31.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:05:31 smithi153 ceph-mon[70044]: pgmap v1120: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:05:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:05:31] "GET /metrics HTTP/1.1" 200 225173 "" "Prometheus/2.33.4" 2022-09-29T06:05:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:05:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:05:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:05:33.291 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:05:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:05:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:05:33.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:05:33 smithi153 ceph-mon[70044]: pgmap v1121: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:33.661 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:05:33 smithi083 ceph-mon[79930]: pgmap v1121: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:33.661 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:05:33 smithi083 ceph-mon[77242]: pgmap v1121: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:33.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:05:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:05:33.659Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:05:34.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:05:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:05:34.352Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:05:35.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:05:35 smithi153 ceph-mon[70044]: pgmap v1122: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:35.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:05:35 smithi083 ceph-mon[77242]: pgmap v1122: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:35.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:05:35 smithi083 ceph-mon[79930]: pgmap v1122: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:37.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:05:37 smithi153 ceph-mon[70044]: pgmap v1123: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:37.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:05:37 smithi083 ceph-mon[79930]: pgmap v1123: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:37.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:05:37 smithi083 ceph-mon[77242]: pgmap v1123: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:39.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:05:39 smithi153 ceph-mon[70044]: pgmap v1124: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:39.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:05:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:05:39.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:05:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:05:39.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:05:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:05:39.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:05:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:05:39.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:05:39 smithi083 ceph-mon[77242]: pgmap v1124: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:39.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:05:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:05:39.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:05:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:05:39.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:05:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:05:39.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:05:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:05:39.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:05:39 smithi083 ceph-mon[79930]: pgmap v1124: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:39.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:05:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:05:39.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:05:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:05:39.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:05:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:05:39.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:05:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:05:40.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:05:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:05:40.246Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:05:41.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:05:41 smithi083 ceph-mon[77242]: pgmap v1125: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:41.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:05:41 smithi083 ceph-mon[79930]: pgmap v1125: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:05:41 smithi153 ceph-mon[70044]: pgmap v1125: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:05:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:05:41] "GET /metrics HTTP/1.1" 200 225168 "" "Prometheus/2.33.4" 2022-09-29T06:05:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:05:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:05:42.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:05:43.295 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:05:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:05:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:05:43.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:05:43 smithi153 ceph-mon[70044]: pgmap v1126: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:43.661 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:05:43 smithi083 ceph-mon[79930]: pgmap v1126: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:43.662 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:05:43 smithi083 ceph-mon[77242]: pgmap v1126: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:43.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:05:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:05:43.659Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:05:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:05:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:05:44.353Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:05:45.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:05:45 smithi153 ceph-mon[70044]: pgmap v1127: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:45.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:05:45 smithi083 ceph-mon[79930]: pgmap v1127: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:45.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:05:45 smithi083 ceph-mon[77242]: pgmap v1127: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:47.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:05:47 smithi153 ceph-mon[70044]: pgmap v1128: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:47.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:05:47 smithi083 ceph-mon[79930]: pgmap v1128: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:47.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:05:47 smithi083 ceph-mon[77242]: pgmap v1128: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:49.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:05:49 smithi153 ceph-mon[70044]: pgmap v1129: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:49.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:05:49 smithi083 ceph-mon[79930]: pgmap v1129: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:49.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:05:49 smithi083 ceph-mon[77242]: pgmap v1129: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:50.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:05:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:05:50.246Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:05:51.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:05:51 smithi083 ceph-mon[77242]: pgmap v1130: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:51.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:05:51 smithi083 ceph-mon[79930]: pgmap v1130: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:51.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:05:51 smithi153 ceph-mon[70044]: pgmap v1130: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:05:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:05:51] "GET /metrics HTTP/1.1" 200 225168 "" "Prometheus/2.33.4" 2022-09-29T06:05:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:05:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:05:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:05:53.298 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:05:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:05:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:05:53.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:05:53 smithi153 ceph-mon[70044]: pgmap v1131: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:53.662 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:05:53 smithi083 ceph-mon[79930]: pgmap v1131: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:53.662 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:05:53 smithi083 ceph-mon[77242]: pgmap v1131: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:53.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:05:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:05:53.660Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:05:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:05:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:05:54.354Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:05:55.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:05:55 smithi153 ceph-mon[70044]: pgmap v1132: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:55.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:05:55 smithi083 ceph-mon[77242]: pgmap v1132: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:55.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:05:55 smithi083 ceph-mon[79930]: pgmap v1132: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:57.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:05:57 smithi153 ceph-mon[70044]: pgmap v1133: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:57.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:05:57 smithi083 ceph-mon[77242]: pgmap v1133: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:57.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:05:57 smithi083 ceph-mon[79930]: pgmap v1133: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:59.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:05:59 smithi153 ceph-mon[70044]: pgmap v1134: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:59.695 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:05:59.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:05:59 smithi083 ceph-mon[79930]: pgmap v1134: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:59.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:05:59 smithi083 ceph-mon[77242]: pgmap v1134: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:05:59.983 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:05:59.984 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (39m) 2m ago 40m 26.7M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:05:59.984 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (39m) 36s ago 39m 61.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:05:59.984 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (41m) 2m ago 41m 21.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:05:59.984 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (41m) 36s ago 41m 27.6M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:05:59.984 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (36m) 36s ago 43m 439M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:05:59.985 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (37m) 2m ago 45m 538M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:05:59.985 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (36m) 2m ago 45m 204M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:05:59.985 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (35m) 36s ago 43m 101M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:05:59.985 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (36m) 2m ago 43m 124M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:05:59.986 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (40m) 2m ago 40m 25.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:05:59.986 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (40m) 36s ago 40m 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:05:59.986 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (35m) 2m ago 43m 158M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:05:59.986 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (34m) 2m ago 43m 150M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:05:59.986 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (34m) 2m ago 42m 124M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:05:59.987 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (33m) 2m ago 42m 141M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:05:59.987 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (33m) 36s ago 42m 133M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:05:59.987 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (32m) 36s ago 41m 152M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:05:59.987 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (32m) 36s ago 41m 135M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:05:59.988 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (31m) 36s ago 41m 150M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:05:59.988 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (36m) 36s ago 40m 100M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:06:00.340 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:06:00.341 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:06:00.341 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:06:00.341 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:06:00.342 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:06:00.342 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:06:00.342 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:06:00.342 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:06:00.343 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:06:00.343 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:06:00.343 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:06:00.343 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:06:00.344 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:06:00.344 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:06:00.344 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:06:00.344 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:06:00.344 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:06:00.345 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:06:00.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:06:00 smithi083 ceph-mon[77242]: from='client.35457 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:06:00.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:06:00 smithi083 ceph-mon[77242]: from='client.45397 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:06:00.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:06:00 smithi083 ceph-mon[79930]: from='client.35457 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:06:00.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:06:00 smithi083 ceph-mon[79930]: from='client.45397 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:06:00.714 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:06:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:06:00.246Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:06:00.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:06:00 smithi153 ceph-mon[70044]: from='client.35457 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:06:00.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:06:00 smithi153 ceph-mon[70044]: from='client.45397 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:06:01.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:06:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:06:01] "GET /metrics HTTP/1.1" 200 225169 "" "Prometheus/2.33.4" 2022-09-29T06:06:01.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:06:01 smithi083 ceph-mon[79930]: pgmap v1135: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:06:01.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:06:01 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3943551377' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:06:01.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:06:01 smithi083 ceph-mon[77242]: pgmap v1135: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:06:01.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:06:01 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3943551377' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:06:01.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:06:01 smithi153 ceph-mon[70044]: pgmap v1135: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:06:01.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:06:01 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3943551377' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:06:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:06:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:06:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:06:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:06:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:06:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:06:03.663 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:06:03 smithi083 ceph-mon[79930]: pgmap v1136: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:06:03.663 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:06:03 smithi083 ceph-mon[77242]: pgmap v1136: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:06:03.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:06:03 smithi153 ceph-mon[70044]: pgmap v1136: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:06:03.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:06:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:06:03.661Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:06:04.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:06:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:06:04.355Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:06:05.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:06:05 smithi083 ceph-mon[79930]: pgmap v1137: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:06:05.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:06:05 smithi083 ceph-mon[77242]: pgmap v1137: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:06:05.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:06:05 smithi153 ceph-mon[70044]: pgmap v1137: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:06:07.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:06:07 smithi083 ceph-mon[79930]: pgmap v1138: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:06:07.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:06:07 smithi083 ceph-mon[77242]: pgmap v1138: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:06:07.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:06:07 smithi153 ceph-mon[70044]: pgmap v1138: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:06:09.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:06:09 smithi083 ceph-mon[79930]: pgmap v1139: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:06:09.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:06:09 smithi083 ceph-mon[77242]: pgmap v1139: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:06:09.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:06:09 smithi153 ceph-mon[70044]: pgmap v1139: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:06:10.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:06:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:06:10.247Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:06:11.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:06:11 smithi083 ceph-mon[79930]: pgmap v1140: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:06:11.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:06:11 smithi083 ceph-mon[77242]: pgmap v1140: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:06:11.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:06:11 smithi153 ceph-mon[70044]: pgmap v1140: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:06:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:06:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:06:11] "GET /metrics HTTP/1.1" 200 225170 "" "Prometheus/2.33.4" 2022-09-29T06:06:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:06:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:06:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:06:13.304 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:06:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:06:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:06:13.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:06:13 smithi153 ceph-mon[70044]: pgmap v1141: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:06:13.663 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:06:13 smithi083 ceph-mon[79930]: pgmap v1141: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:06:13.664 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:06:13 smithi083 ceph-mon[77242]: pgmap v1141: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:06:13.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:06:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:06:13.661Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:06:14.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:06:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:06:14.356Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:06:15.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:06:15 smithi153 ceph-mon[70044]: pgmap v1142: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:06:15.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:06:15 smithi083 ceph-mon[79930]: pgmap v1142: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:06:15.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:06:15 smithi083 ceph-mon[77242]: pgmap v1142: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:06:17.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:06:17 smithi083 ceph-mon[79930]: pgmap v1143: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:06:17.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:06:17 smithi083 ceph-mon[77242]: pgmap v1143: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:06:17.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:06:17 smithi153 ceph-mon[70044]: pgmap v1143: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:06:19.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:06:19 smithi083 ceph-mon[77242]: pgmap v1144: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:06:19.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:06:19 smithi083 ceph-mon[79930]: pgmap v1144: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:06:19.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:06:19 smithi153 ceph-mon[70044]: pgmap v1144: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:06:20.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:06:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:06:20.247Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:06:21.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:06:21 smithi083 ceph-mon[79930]: pgmap v1145: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:06:21.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:06:21 smithi083 ceph-mon[77242]: pgmap v1145: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:06:21.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:06:21 smithi153 ceph-mon[70044]: pgmap v1145: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:06:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:06:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:06:21] "GET /metrics HTTP/1.1" 200 225170 "" "Prometheus/2.33.4" 2022-09-29T06:06:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:06:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:06:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:06:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:06:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:06:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:06:23.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:06:23 smithi153 ceph-mon[70044]: pgmap v1146: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:06:23.665 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:06:23 smithi083 ceph-mon[79930]: pgmap v1146: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:06:23.665 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:06:23 smithi083 ceph-mon[77242]: pgmap v1146: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:06:23.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:06:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:06:23.663Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:06:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:06:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:06:24.356Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:06:25.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:06:25 smithi083 ceph-mon[79930]: pgmap v1147: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:06:25.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:06:25 smithi083 ceph-mon[77242]: pgmap v1147: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:06:25.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:06:25 smithi153 ceph-mon[70044]: pgmap v1147: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:06:27.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:06:27 smithi083 ceph-mon[79930]: pgmap v1148: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:06:27.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:06:27 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:06:27.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:06:27 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:06:27.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:06:27 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:06:27.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:06:27 smithi083 ceph-mon[77242]: pgmap v1148: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:06:27.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:06:27 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:06:27.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:06:27 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:06:27.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:06:27 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:06:27.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:06:27 smithi153 ceph-mon[70044]: pgmap v1148: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:06:27.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:06:27 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:06:27.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:06:27 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:06:27.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:06:27 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:06:28.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:06:28 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:06:28.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:06:28 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:06:29.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:06:28 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:06:29.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:06:29 smithi083 ceph-mon[79930]: pgmap v1149: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:06:29.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:06:29 smithi083 ceph-mon[77242]: pgmap v1149: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:06:30.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:06:29 smithi153 ceph-mon[70044]: pgmap v1149: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:06:30.669 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:06:30.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:06:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:06:30.248Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:06:30.957 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:06:30.958 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (40m) 3m ago 40m 26.7M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:06:30.958 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (40m) 67s ago 40m 61.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:06:30.958 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (41m) 3m ago 41m 21.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:06:30.958 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (41m) 67s ago 41m 27.6M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:06:30.959 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (37m) 67s ago 44m 439M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:06:30.959 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (37m) 3m ago 45m 538M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:06:30.959 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (37m) 3m ago 45m 204M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:06:30.959 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (36m) 67s ago 44m 101M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:06:30.960 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (36m) 3m ago 44m 124M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:06:30.960 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (40m) 3m ago 40m 25.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:06:30.960 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (40m) 67s ago 40m 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:06:30.960 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (35m) 3m ago 43m 158M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:06:30.961 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (35m) 3m ago 43m 150M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:06:30.961 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (34m) 3m ago 43m 124M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:06:30.961 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (34m) 3m ago 42m 141M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:06:30.961 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (33m) 67s ago 42m 133M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:06:30.961 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (33m) 67s ago 42m 152M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:06:30.962 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (32m) 67s ago 42m 135M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:06:30.962 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (32m) 67s ago 41m 150M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:06:30.962 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (37m) 67s ago 40m 100M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:06:31.314 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:06:31.314 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:06:31.315 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:06:31.315 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:06:31.315 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:06:31.315 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:06:31.316 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:06:31.316 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:06:31.316 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:06:31.316 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:06:31.316 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:06:31.317 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:06:31.317 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:06:31.317 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:06:31.317 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:06:31.318 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:06:31.318 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:06:31.318 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:06:31.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:06:31 smithi083 ceph-mon[77242]: pgmap v1150: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:06:31.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:06:31 smithi083 ceph-mon[77242]: from='client.45409 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:06:31.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:06:31 smithi083 ceph-mon[77242]: from='client.45415 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:06:31.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:06:31 smithi083 ceph-mon[79930]: pgmap v1150: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:06:31.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:06:31 smithi083 ceph-mon[79930]: from='client.45409 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:06:31.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:06:31 smithi083 ceph-mon[79930]: from='client.45415 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:06:31.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:06:31 smithi153 ceph-mon[70044]: pgmap v1150: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:06:31.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:06:31 smithi153 ceph-mon[70044]: from='client.45409 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:06:31.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:06:31 smithi153 ceph-mon[70044]: from='client.45415 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:06:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:06:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:06:31] "GET /metrics HTTP/1.1" 200 225170 "" "Prometheus/2.33.4" 2022-09-29T06:06:32.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:06:32 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3541251552' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:06:32.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:06:32 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3541251552' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:06:32.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:06:32 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3541251552' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:06:32.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:06:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:06:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:06:33.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:06:33 smithi153 ceph-mon[70044]: pgmap v1151: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:06:33.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:06:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:06:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:06:33.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:06:33 smithi083 ceph-mon[79930]: pgmap v1151: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:06:33.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:06:33 smithi083 ceph-mon[77242]: pgmap v1151: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:06:33.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:06:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:06:33.663Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:06:34.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:06:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:06:34.356Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:06:35.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:06:35 smithi083 ceph-mon[77242]: pgmap v1152: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:06:35.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:06:35 smithi083 ceph-mon[79930]: pgmap v1152: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:06:35.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:06:35 smithi153 ceph-mon[70044]: pgmap v1152: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:06:37.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:06:37 smithi083 ceph-mon[79930]: pgmap v1153: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:06:37.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:06:37 smithi083 ceph-mon[77242]: pgmap v1153: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:06:37.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:06:37 smithi153 ceph-mon[70044]: pgmap v1153: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:06:39.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:06:39 smithi083 ceph-mon[79930]: pgmap v1154: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:06:39.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:06:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:06:39.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:06:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:06:39.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:06:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:06:39.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:06:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:06:39.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:06:39 smithi083 ceph-mon[77242]: pgmap v1154: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:06:39.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:06:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:06:39.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:06:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:06:39.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:06:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:06:39.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:06:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:06:39.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:06:39 smithi153 ceph-mon[70044]: pgmap v1154: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:06:39.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:06:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:06:39.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:06:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:06:39.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:06:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:06:39.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:06:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:06:40.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:06:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:06:40.248Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:06:41.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:06:41 smithi083 ceph-mon[79930]: pgmap v1155: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:06:41.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:06:41 smithi083 ceph-mon[77242]: pgmap v1155: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:06:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:06:41 smithi153 ceph-mon[70044]: pgmap v1155: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:06:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:06:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:06:41] "GET /metrics HTTP/1.1" 200 225167 "" "Prometheus/2.33.4" 2022-09-29T06:06:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:06:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:06:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:06:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:06:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:06:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:06:43.666 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:06:43 smithi083 ceph-mon[79930]: pgmap v1156: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:06:43.667 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:06:43 smithi083 ceph-mon[77242]: pgmap v1156: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:06:43.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:06:43 smithi153 ceph-mon[70044]: pgmap v1156: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:06:43.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:06:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:06:43.664Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:06:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:06:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:06:44.357Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:06:45.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:06:45 smithi083 ceph-mon[79930]: pgmap v1157: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:06:45.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:06:45 smithi083 ceph-mon[77242]: pgmap v1157: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:06:45.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:06:45 smithi153 ceph-mon[70044]: pgmap v1157: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:06:47.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:06:47 smithi083 ceph-mon[79930]: pgmap v1158: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:06:47.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:06:47 smithi083 ceph-mon[77242]: pgmap v1158: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:06:47.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:06:47 smithi153 ceph-mon[70044]: pgmap v1158: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:06:49.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:06:49 smithi083 ceph-mon[77242]: pgmap v1159: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:06:49.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:06:49 smithi083 ceph-mon[79930]: pgmap v1159: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:06:49.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:06:49 smithi153 ceph-mon[70044]: pgmap v1159: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:06:50.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:06:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:06:50.249Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:06:51.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:06:51 smithi083 ceph-mon[79930]: pgmap v1160: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:06:51.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:06:51 smithi083 ceph-mon[77242]: pgmap v1160: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:06:51.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:06:51 smithi153 ceph-mon[70044]: pgmap v1160: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:06:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:06:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:06:51] "GET /metrics HTTP/1.1" 200 225167 "" "Prometheus/2.33.4" 2022-09-29T06:06:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:06:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:06:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:06:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:06:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:06:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:06:53.667 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:06:53 smithi083 ceph-mon[79930]: pgmap v1161: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:06:53.667 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:06:53 smithi083 ceph-mon[77242]: pgmap v1161: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:06:53.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:06:53 smithi153 ceph-mon[70044]: pgmap v1161: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:06:53.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:06:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:06:53.665Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:06:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:06:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:06:54.358Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:06:55.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:06:55 smithi083 ceph-mon[79930]: pgmap v1162: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:06:55.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:06:55 smithi083 ceph-mon[77242]: pgmap v1162: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:06:55.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:06:55 smithi153 ceph-mon[70044]: pgmap v1162: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:06:57.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:06:57 smithi083 ceph-mon[77242]: pgmap v1163: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:06:57.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:06:57 smithi083 ceph-mon[79930]: pgmap v1163: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:06:57.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:06:57 smithi153 ceph-mon[70044]: pgmap v1163: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:06:59.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:06:59 smithi083 ceph-mon[79930]: pgmap v1164: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:06:59.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:06:59 smithi083 ceph-mon[77242]: pgmap v1164: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:06:59.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:06:59 smithi153 ceph-mon[70044]: pgmap v1164: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:00.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:07:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:07:00.250Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:07:01.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:07:01 smithi083 ceph-mon[79930]: pgmap v1165: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:01.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:07:01 smithi083 ceph-mon[77242]: pgmap v1165: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:01.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:07:01 smithi153 ceph-mon[70044]: pgmap v1165: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:01.638 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:07:01.929 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:07:01.929 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (40m) 3m ago 41m 26.7M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:07:01.930 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (41m) 98s ago 40m 61.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:07:01.930 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (42m) 3m ago 42m 21.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:07:01.930 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (42m) 98s ago 42m 27.6M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:07:01.930 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (37m) 98s ago 44m 439M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:07:01.931 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (38m) 3m ago 46m 538M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:07:01.931 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (37m) 3m ago 46m 204M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:07:01.931 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (36m) 98s ago 44m 101M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:07:01.931 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (37m) 3m ago 44m 124M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:07:01.932 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (41m) 3m ago 41m 25.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:07:01.932 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (41m) 98s ago 41m 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:07:01.932 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (36m) 3m ago 44m 158M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:07:01.932 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (35m) 3m ago 44m 150M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:07:01.932 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (35m) 3m ago 43m 124M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:07:01.933 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (34m) 3m ago 43m 141M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:07:01.933 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (34m) 98s ago 43m 133M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:07:01.933 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (33m) 98s ago 42m 152M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:07:01.933 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (33m) 98s ago 42m 135M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:07:01.934 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (32m) 98s ago 42m 150M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:07:01.934 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (37m) 98s ago 41m 100M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:07:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:07:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:07:01] "GET /metrics HTTP/1.1" 200 225166 "" "Prometheus/2.33.4" 2022-09-29T06:07:02.285 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:07:02.285 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:07:02.285 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:07:02.286 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:07:02.286 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:07:02.286 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:07:02.286 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:07:02.287 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:07:02.287 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:07:02.287 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:07:02.287 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:07:02.288 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:07:02.288 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:07:02.288 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:07:02.288 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:07:02.288 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:07:02.289 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:07:02.289 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:07:02.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:07:02 smithi083 ceph-mon[79930]: from='client.45427 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:07:02.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:07:02 smithi083 ceph-mon[79930]: from='client.45433 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:07:02.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:07:02 smithi083 ceph-mon[77242]: from='client.45427 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:07:02.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:07:02 smithi083 ceph-mon[77242]: from='client.45433 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:07:02.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:07:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:07:02.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:07:02.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:07:02 smithi153 ceph-mon[70044]: from='client.45427 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:07:02.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:07:02 smithi153 ceph-mon[70044]: from='client.45433 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:07:03.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:07:03 smithi153 ceph-mon[70044]: pgmap v1166: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:03.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:07:03 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1468337918' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:07:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:07:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:07:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:07:03.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:07:03 smithi083 ceph-mon[79930]: pgmap v1166: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:03.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:07:03 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1468337918' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:07:03.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:07:03 smithi083 ceph-mon[77242]: pgmap v1166: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:03.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:07:03 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1468337918' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:07:03.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:07:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:07:03.665Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:07:04.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:07:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:07:04.359Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:07:05.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:07:05 smithi083 ceph-mon[79930]: pgmap v1167: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:05.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:07:05 smithi083 ceph-mon[77242]: pgmap v1167: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:05.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:07:05 smithi153 ceph-mon[70044]: pgmap v1167: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:07.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:07:07 smithi083 ceph-mon[79930]: pgmap v1168: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:07.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:07:07 smithi083 ceph-mon[77242]: pgmap v1168: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:07.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:07:07 smithi153 ceph-mon[70044]: pgmap v1168: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:09.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:07:09 smithi083 ceph-mon[77242]: pgmap v1169: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:09.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:07:09 smithi083 ceph-mon[79930]: pgmap v1169: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:09.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:07:09 smithi153 ceph-mon[70044]: pgmap v1169: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:10.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:07:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:07:10.251Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:07:11.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:07:11 smithi083 ceph-mon[79930]: pgmap v1170: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:11.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:07:11 smithi083 ceph-mon[77242]: pgmap v1170: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:11.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:07:11 smithi153 ceph-mon[70044]: pgmap v1170: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:07:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:07:11] "GET /metrics HTTP/1.1" 200 225171 "" "Prometheus/2.33.4" 2022-09-29T06:07:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:07:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:07:12.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:07:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:07:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:07:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:07:13.668 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:07:13 smithi083 ceph-mon[79930]: pgmap v1171: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:13.668 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:07:13 smithi083 ceph-mon[77242]: pgmap v1171: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:13.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:07:13 smithi153 ceph-mon[70044]: pgmap v1171: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:13.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:07:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:07:13.666Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:07:14.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:07:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:07:14.359Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:07:15.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:07:15 smithi083 ceph-mon[77242]: pgmap v1172: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:15.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:07:15 smithi083 ceph-mon[79930]: pgmap v1172: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:15.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:07:15 smithi153 ceph-mon[70044]: pgmap v1172: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:17.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:07:17 smithi083 ceph-mon[77242]: pgmap v1173: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:17.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:07:17 smithi083 ceph-mon[79930]: pgmap v1173: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:17.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:07:17 smithi153 ceph-mon[70044]: pgmap v1173: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:19.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:07:19 smithi083 ceph-mon[77242]: pgmap v1174: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:19.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:07:19 smithi083 ceph-mon[79930]: pgmap v1174: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:19.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:07:19 smithi153 ceph-mon[70044]: pgmap v1174: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:20.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:07:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:07:20.251Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:07:21.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:07:21 smithi083 ceph-mon[79930]: pgmap v1175: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:21.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:07:21 smithi083 ceph-mon[77242]: pgmap v1175: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:21.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:07:21 smithi153 ceph-mon[70044]: pgmap v1175: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:07:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:07:21] "GET /metrics HTTP/1.1" 200 225171 "" "Prometheus/2.33.4" 2022-09-29T06:07:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:07:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:07:22.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:07:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:07:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:07:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:07:23.668 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:07:23 smithi083 ceph-mon[79930]: pgmap v1176: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:23.669 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:07:23 smithi083 ceph-mon[77242]: pgmap v1176: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:23.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:07:23 smithi153 ceph-mon[70044]: pgmap v1176: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:23.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:07:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:07:23.666Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:07:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:07:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:07:24.359Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:07:25.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:07:25 smithi083 ceph-mon[77242]: pgmap v1177: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:25.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:07:25 smithi083 ceph-mon[79930]: pgmap v1177: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:25.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:07:25 smithi153 ceph-mon[70044]: pgmap v1177: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:27.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:07:27 smithi083 ceph-mon[77242]: pgmap v1178: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:27.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:07:27 smithi083 ceph-mon[79930]: pgmap v1178: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:27.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:07:27 smithi153 ceph-mon[70044]: pgmap v1178: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:28.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:07:28 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:07:28.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:07:28 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:07:28.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:07:28 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:07:28.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:07:28 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:07:28.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:07:28 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:07:28.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:07:28 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:07:28.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:07:28 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:07:28.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:07:28 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:07:28.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:07:28 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:07:29.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:07:29 smithi083 ceph-mon[79930]: pgmap v1179: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:29.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:07:29 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:07:29.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:07:29 smithi083 ceph-mon[77242]: pgmap v1179: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:29.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:07:29 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:07:29.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:07:29 smithi153 ceph-mon[70044]: pgmap v1179: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:29.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:07:29 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:07:30.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:07:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:07:30.252Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:07:31.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:07:31 smithi083 ceph-mon[79930]: pgmap v1180: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:31.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:07:31 smithi083 ceph-mon[77242]: pgmap v1180: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:31.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:07:31 smithi153 ceph-mon[70044]: pgmap v1180: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:07:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:07:31] "GET /metrics HTTP/1.1" 200 225171 "" "Prometheus/2.33.4" 2022-09-29T06:07:32.610 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:07:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:07:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:07:32.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:07:32.900 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:07:32.900 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (41m) 4m ago 41m 26.7M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:07:32.901 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (41m) 2m ago 41m 61.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:07:32.901 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (42m) 4m ago 42m 21.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:07:32.901 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (42m) 2m ago 42m 27.6M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:07:32.901 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (38m) 2m ago 45m 439M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:07:32.902 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (38m) 4m ago 46m 538M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:07:32.902 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (38m) 4m ago 46m 204M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:07:32.902 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (37m) 2m ago 45m 101M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:07:32.903 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (37m) 4m ago 45m 124M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:07:32.903 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (41m) 4m ago 41m 25.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:07:32.903 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (41m) 2m ago 41m 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:07:32.903 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (36m) 4m ago 44m 158M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:07:32.904 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (36m) 4m ago 44m 150M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:07:32.904 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (35m) 4m ago 44m 124M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:07:32.904 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (35m) 4m ago 43m 141M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:07:32.904 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (34m) 2m ago 43m 133M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:07:32.904 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (34m) 2m ago 43m 152M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:07:32.905 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (33m) 2m ago 43m 135M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:07:32.905 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (33m) 2m ago 42m 150M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:07:32.905 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (38m) 2m ago 42m 100M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:07:33.257 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:07:33.258 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:07:33.258 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:07:33.258 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:07:33.259 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:07:33.259 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:07:33.259 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:07:33.259 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:07:33.259 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:07:33.260 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:07:33.260 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:07:33.260 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:07:33.260 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:07:33.261 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:07:33.261 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:07:33.261 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:07:33.261 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:07:33.261 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:07:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:07:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:07:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:07:33.668 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:07:33 smithi083 ceph-mon[79930]: pgmap v1181: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:33.668 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:07:33 smithi083 ceph-mon[79930]: from='client.35511 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:07:33.669 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:07:33 smithi083 ceph-mon[79930]: from='client.45451 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:07:33.669 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:07:33 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3637714826' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:07:33.669 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:07:33 smithi083 ceph-mon[77242]: pgmap v1181: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:33.669 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:07:33 smithi083 ceph-mon[77242]: from='client.35511 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:07:33.670 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:07:33 smithi083 ceph-mon[77242]: from='client.45451 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:07:33.670 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:07:33 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3637714826' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:07:33.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:07:33 smithi153 ceph-mon[70044]: pgmap v1181: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:33.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:07:33 smithi153 ceph-mon[70044]: from='client.35511 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:07:33.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:07:33 smithi153 ceph-mon[70044]: from='client.45451 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:07:33.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:07:33 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3637714826' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:07:33.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:07:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:07:33.666Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:07:34.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:07:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:07:34.361Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:07:35.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:07:35 smithi083 ceph-mon[77242]: pgmap v1182: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:35.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:07:35 smithi083 ceph-mon[79930]: pgmap v1182: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:35.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:07:35 smithi153 ceph-mon[70044]: pgmap v1182: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:37.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:07:37 smithi083 ceph-mon[79930]: pgmap v1183: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:37.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:07:37 smithi083 ceph-mon[77242]: pgmap v1183: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:37.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:07:37 smithi153 ceph-mon[70044]: pgmap v1183: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:39.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:07:39 smithi083 ceph-mon[79930]: pgmap v1184: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:39.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:07:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:07:39.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:07:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:07:39.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:07:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:07:39.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:07:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:07:39.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:07:39 smithi083 ceph-mon[77242]: pgmap v1184: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:39.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:07:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:07:39.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:07:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:07:39.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:07:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:07:39.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:07:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:07:39.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:07:39 smithi153 ceph-mon[70044]: pgmap v1184: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:39.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:07:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:07:39.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:07:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:07:39.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:07:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:07:39.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:07:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:07:40.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:07:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:07:40.252Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:07:41.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:07:41 smithi083 ceph-mon[77242]: pgmap v1185: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:41.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:07:41 smithi083 ceph-mon[79930]: pgmap v1185: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:07:41 smithi153 ceph-mon[70044]: pgmap v1185: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:07:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:07:41] "GET /metrics HTTP/1.1" 200 225172 "" "Prometheus/2.33.4" 2022-09-29T06:07:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:07:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:07:42.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:07:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:07:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:07:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:07:43.669 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:07:43 smithi083 ceph-mon[79930]: pgmap v1186: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:43.670 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:07:43 smithi083 ceph-mon[77242]: pgmap v1186: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:43.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:07:43 smithi153 ceph-mon[70044]: pgmap v1186: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:43.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:07:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:07:43.667Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:07:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:07:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:07:44.361Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:07:45.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:07:45 smithi083 ceph-mon[77242]: pgmap v1187: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:45.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:07:45 smithi083 ceph-mon[79930]: pgmap v1187: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:45.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:07:45 smithi153 ceph-mon[70044]: pgmap v1187: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:47.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:07:47 smithi083 ceph-mon[79930]: pgmap v1188: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:47.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:07:47 smithi083 ceph-mon[77242]: pgmap v1188: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:47.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:07:47 smithi153 ceph-mon[70044]: pgmap v1188: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:49.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:07:49 smithi083 ceph-mon[79930]: pgmap v1189: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:49.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:07:49 smithi083 ceph-mon[77242]: pgmap v1189: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:49.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:07:49 smithi153 ceph-mon[70044]: pgmap v1189: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:50.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:07:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:07:50.253Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:07:51.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:07:51 smithi083 ceph-mon[79930]: pgmap v1190: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:51.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:07:51 smithi083 ceph-mon[77242]: pgmap v1190: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:51.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:07:51 smithi153 ceph-mon[70044]: pgmap v1190: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:07:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:07:51] "GET /metrics HTTP/1.1" 200 225172 "" "Prometheus/2.33.4" 2022-09-29T06:07:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:07:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:07:52.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:07:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:07:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:07:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:07:53.669 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:07:53 smithi083 ceph-mon[79930]: pgmap v1191: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:53.669 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:07:53 smithi083 ceph-mon[77242]: pgmap v1191: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:53.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:07:53 smithi153 ceph-mon[70044]: pgmap v1191: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:53.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:07:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:07:53.667Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:07:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:07:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:07:54.361Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:07:55.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:07:55 smithi083 ceph-mon[79930]: pgmap v1192: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:55.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:07:55 smithi083 ceph-mon[77242]: pgmap v1192: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:55.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:07:55 smithi153 ceph-mon[70044]: pgmap v1192: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:57.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:07:57 smithi083 ceph-mon[77242]: pgmap v1193: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:57.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:07:57 smithi083 ceph-mon[79930]: pgmap v1193: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:57.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:07:57 smithi153 ceph-mon[70044]: pgmap v1193: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:59.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:07:59 smithi083 ceph-mon[77242]: pgmap v1194: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:59.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:07:59 smithi083 ceph-mon[79930]: pgmap v1194: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:07:59.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:07:59 smithi153 ceph-mon[70044]: pgmap v1194: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:00.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:08:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:08:00.253Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:08:01.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:08:01 smithi083 ceph-mon[77242]: pgmap v1195: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:01.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:08:01 smithi083 ceph-mon[79930]: pgmap v1195: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:01.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:08:01 smithi153 ceph-mon[70044]: pgmap v1195: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:08:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:08:01] "GET /metrics HTTP/1.1" 200 225168 "" "Prometheus/2.33.4" 2022-09-29T06:08:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:08:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:08:02.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:08:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:08:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:08:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:08:03.595 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:08:03.670 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:08:03 smithi083 ceph-mon[79930]: pgmap v1196: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:03.671 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:08:03 smithi083 ceph-mon[77242]: pgmap v1196: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:03.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:08:03 smithi153 ceph-mon[70044]: pgmap v1196: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:03.896 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:08:03.896 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (41m) 4m ago 42m 26.7M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:08:03.896 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (42m) 2m ago 42m 61.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:08:03.897 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (43m) 4m ago 43m 21.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:08:03.897 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (43m) 2m ago 43m 27.6M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:08:03.897 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (38m) 2m ago 45m 439M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:08:03.897 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (39m) 4m ago 47m 538M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:08:03.898 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (38m) 4m ago 47m 204M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:08:03.898 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (37m) 2m ago 45m 101M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:08:03.898 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (38m) 4m ago 46m 124M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:08:03.898 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (42m) 4m ago 42m 25.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:08:03.899 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (42m) 2m ago 42m 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:08:03.899 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (37m) 4m ago 45m 158M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:08:03.899 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (36m) 4m ago 45m 150M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:08:03.899 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (36m) 4m ago 44m 124M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:08:03.900 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (35m) 4m ago 44m 141M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:08:03.900 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (35m) 2m ago 44m 133M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:08:03.900 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (34m) 2m ago 43m 152M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:08:03.900 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (34m) 2m ago 43m 135M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:08:03.900 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (33m) 2m ago 43m 150M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:08:03.901 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (38m) 2m ago 42m 100M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:08:03.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:08:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:08:03.668Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:08:04.255 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:08:04.256 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:08:04.256 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:08:04.256 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:08:04.256 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:08:04.257 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:08:04.257 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:08:04.257 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:08:04.257 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:08:04.258 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:08:04.258 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:08:04.258 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:08:04.258 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:08:04.258 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:08:04.259 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:08:04.259 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:08:04.259 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:08:04.259 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:08:04.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:08:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:08:04.361Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:08:04.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:08:04 smithi083 ceph-mon[77242]: from='client.45463 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:08:04.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:08:04 smithi083 ceph-mon[77242]: from='client.35535 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:08:04.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:08:04 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2041045622' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:08:04.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:08:04 smithi083 ceph-mon[79930]: from='client.45463 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:08:04.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:08:04 smithi083 ceph-mon[79930]: from='client.35535 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:08:04.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:08:04 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2041045622' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:08:04.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:08:04 smithi153 ceph-mon[70044]: from='client.45463 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:08:04.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:08:04 smithi153 ceph-mon[70044]: from='client.35535 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:08:04.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:08:04 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2041045622' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:08:05.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:08:05 smithi083 ceph-mon[79930]: pgmap v1197: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:05.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:08:05 smithi083 ceph-mon[77242]: pgmap v1197: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:05.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:08:05 smithi153 ceph-mon[70044]: pgmap v1197: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:07.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:08:07 smithi083 ceph-mon[79930]: pgmap v1198: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:07.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:08:07 smithi083 ceph-mon[77242]: pgmap v1198: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:07.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:08:07 smithi153 ceph-mon[70044]: pgmap v1198: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:09.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:08:09 smithi083 ceph-mon[79930]: pgmap v1199: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:09.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:08:09 smithi083 ceph-mon[77242]: pgmap v1199: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:09.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:08:09 smithi153 ceph-mon[70044]: pgmap v1199: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:10.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:08:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:08:10.254Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:08:11.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:08:11 smithi083 ceph-mon[79930]: pgmap v1200: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:11.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:08:11 smithi083 ceph-mon[77242]: pgmap v1200: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:11.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:08:11 smithi153 ceph-mon[70044]: pgmap v1200: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:08:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:08:11] "GET /metrics HTTP/1.1" 200 225170 "" "Prometheus/2.33.4" 2022-09-29T06:08:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:08:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:08:12.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:08:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:08:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:08:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:08:13.671 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:08:13 smithi083 ceph-mon[77242]: pgmap v1201: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:13.671 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:08:13 smithi083 ceph-mon[79930]: pgmap v1201: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:13.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:08:13 smithi153 ceph-mon[70044]: pgmap v1201: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:13.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:08:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:08:13.669Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:08:14.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:08:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:08:14.362Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:08:15.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:08:15 smithi083 ceph-mon[79930]: pgmap v1202: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:15.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:08:15 smithi083 ceph-mon[77242]: pgmap v1202: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:15.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:08:15 smithi153 ceph-mon[70044]: pgmap v1202: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:17.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:08:17 smithi083 ceph-mon[79930]: pgmap v1203: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:17.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:08:17 smithi083 ceph-mon[77242]: pgmap v1203: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:17.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:08:17 smithi153 ceph-mon[70044]: pgmap v1203: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:19.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:08:19 smithi083 ceph-mon[79930]: pgmap v1204: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:19.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:08:19 smithi083 ceph-mon[77242]: pgmap v1204: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:19.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:08:19 smithi153 ceph-mon[70044]: pgmap v1204: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:20.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:08:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:08:20.254Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:08:21.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:08:21 smithi083 ceph-mon[77242]: pgmap v1205: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:21.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:08:21 smithi083 ceph-mon[79930]: pgmap v1205: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:21.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:08:21 smithi153 ceph-mon[70044]: pgmap v1205: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:08:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:08:21] "GET /metrics HTTP/1.1" 200 225170 "" "Prometheus/2.33.4" 2022-09-29T06:08:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:08:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:08:22.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:08:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:08:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:08:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:08:23.672 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:08:23 smithi083 ceph-mon[79930]: pgmap v1206: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:23.672 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:08:23 smithi083 ceph-mon[77242]: pgmap v1206: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:23.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:08:23 smithi153 ceph-mon[70044]: pgmap v1206: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:23.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:08:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:08:23.670Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:08:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:08:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:08:24.363Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:08:25.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:08:25 smithi083 ceph-mon[77242]: pgmap v1207: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:25.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:08:25 smithi083 ceph-mon[79930]: pgmap v1207: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:25.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:08:25 smithi153 ceph-mon[70044]: pgmap v1207: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:27.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:08:27 smithi083 ceph-mon[77242]: pgmap v1208: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:27.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:08:27 smithi083 ceph-mon[79930]: pgmap v1208: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:27.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:08:27 smithi153 ceph-mon[70044]: pgmap v1208: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:29.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:08:29 smithi083 ceph-mon[77242]: pgmap v1209: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:29.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:08:29 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:08:29.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:08:29 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:08:29.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:08:29 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:08:29.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:08:29 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:08:29.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:08:29 smithi083 ceph-mon[79930]: pgmap v1209: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:29.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:08:29 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:08:29.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:08:29 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:08:29.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:08:29 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:08:29.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:08:29 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:08:29.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:08:29 smithi153 ceph-mon[70044]: pgmap v1209: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:29.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:08:29 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:08:29.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:08:29 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:08:29.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:08:29 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:08:29.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:08:29 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:08:30.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:08:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:08:30.255Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:08:31.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:08:31 smithi083 ceph-mon[79930]: pgmap v1210: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:31.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:08:31 smithi083 ceph-mon[77242]: pgmap v1210: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:31.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:08:31 smithi153 ceph-mon[70044]: pgmap v1210: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:08:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:08:31] "GET /metrics HTTP/1.1" 200 225171 "" "Prometheus/2.33.4" 2022-09-29T06:08:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:08:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:08:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:08:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:08:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:08:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:08:33.673 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:08:33 smithi083 ceph-mon[79930]: pgmap v1211: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:33.674 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:08:33 smithi083 ceph-mon[77242]: pgmap v1211: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:33.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:08:33 smithi153 ceph-mon[70044]: pgmap v1211: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:33.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:08:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:08:33.671Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:08:34.585 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:08:34.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:08:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:08:34.363Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:08:34.872 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:08:34.872 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (42m) 5m ago 42m 26.7M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:08:34.872 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (42m) 3m ago 42m 61.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:08:34.873 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (43m) 5m ago 43m 21.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:08:34.873 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (43m) 3m ago 43m 27.6M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:08:34.873 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (39m) 3m ago 46m 439M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:08:34.873 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (39m) 5m ago 47m 538M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:08:34.873 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (39m) 5m ago 47m 204M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:08:34.874 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (38m) 3m ago 46m 101M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:08:34.874 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (38m) 5m ago 46m 124M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:08:34.874 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (43m) 5m ago 42m 25.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:08:34.874 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (42m) 3m ago 42m 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:08:34.875 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (37m) 5m ago 45m 158M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:08:34.875 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (37m) 5m ago 45m 150M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:08:34.875 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (36m) 5m ago 45m 124M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:08:34.875 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (36m) 5m ago 45m 141M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:08:34.875 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (35m) 3m ago 44m 133M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:08:34.876 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (35m) 3m ago 44m 152M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:08:34.876 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (34m) 3m ago 44m 135M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:08:34.876 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (34m) 3m ago 43m 150M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:08:34.876 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (39m) 3m ago 43m 100M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:08:35.230 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:08:35.230 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:08:35.231 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:08:35.231 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:08:35.231 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:08:35.231 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:08:35.231 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:08:35.232 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:08:35.232 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:08:35.232 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:08:35.232 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:08:35.233 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:08:35.233 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:08:35.233 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:08:35.233 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:08:35.233 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:08:35.234 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:08:35.234 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:08:35.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:08:35 smithi083 ceph-mon[79930]: pgmap v1212: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:35.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:08:35 smithi083 ceph-mon[79930]: from='client.45481 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:08:35.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:08:35 smithi083 ceph-mon[79930]: from='client.35553 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:08:35.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:08:35 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1686289861' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:08:35.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:08:35 smithi083 ceph-mon[77242]: pgmap v1212: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:35.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:08:35 smithi083 ceph-mon[77242]: from='client.45481 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:08:35.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:08:35 smithi083 ceph-mon[77242]: from='client.35553 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:08:35.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:08:35 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1686289861' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:08:35.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:08:35 smithi153 ceph-mon[70044]: pgmap v1212: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:35.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:08:35 smithi153 ceph-mon[70044]: from='client.45481 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:08:35.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:08:35 smithi153 ceph-mon[70044]: from='client.35553 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:08:35.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:08:35 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1686289861' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:08:37.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:08:37 smithi083 ceph-mon[79930]: pgmap v1213: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:37.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:08:37 smithi083 ceph-mon[77242]: pgmap v1213: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:37.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:08:37 smithi153 ceph-mon[70044]: pgmap v1213: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:39.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:08:39 smithi083 ceph-mon[79930]: pgmap v1214: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:39.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:08:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:08:39.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:08:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:08:39.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:08:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:08:39.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:08:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:08:39.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:08:39 smithi083 ceph-mon[77242]: pgmap v1214: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:39.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:08:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:08:39.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:08:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:08:39.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:08:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:08:39.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:08:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:08:39.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:08:39 smithi153 ceph-mon[70044]: pgmap v1214: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:39.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:08:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:08:39.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:08:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:08:39.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:08:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:08:39.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:08:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:08:40.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:08:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:08:40.255Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:08:41.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:08:41 smithi083 ceph-mon[79930]: pgmap v1215: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:41.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:08:41 smithi083 ceph-mon[77242]: pgmap v1215: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:08:41 smithi153 ceph-mon[70044]: pgmap v1215: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:08:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:08:41] "GET /metrics HTTP/1.1" 200 225167 "" "Prometheus/2.33.4" 2022-09-29T06:08:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:08:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:08:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:08:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:08:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:08:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:08:43.673 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:08:43 smithi083 ceph-mon[79930]: pgmap v1216: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:43.674 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:08:43 smithi083 ceph-mon[77242]: pgmap v1216: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:43.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:08:43 smithi153 ceph-mon[70044]: pgmap v1216: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:43.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:08:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:08:43.671Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:08:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:08:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:08:44.364Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:08:45.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:08:45 smithi083 ceph-mon[79930]: pgmap v1217: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:45.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:08:45 smithi083 ceph-mon[77242]: pgmap v1217: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:45.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:08:45 smithi153 ceph-mon[70044]: pgmap v1217: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:47.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:08:47 smithi083 ceph-mon[79930]: pgmap v1218: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:47.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:08:47 smithi083 ceph-mon[77242]: pgmap v1218: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:47.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:08:47 smithi153 ceph-mon[70044]: pgmap v1218: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:49.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:08:49 smithi083 ceph-mon[79930]: pgmap v1219: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:49.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:08:49 smithi083 ceph-mon[77242]: pgmap v1219: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:49.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:08:49 smithi153 ceph-mon[70044]: pgmap v1219: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:50.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:08:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:08:50.255Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:08:51.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:08:51 smithi083 ceph-mon[77242]: pgmap v1220: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:51.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:08:51 smithi083 ceph-mon[79930]: pgmap v1220: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:51.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:08:51 smithi153 ceph-mon[70044]: pgmap v1220: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:08:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:08:51] "GET /metrics HTTP/1.1" 200 225167 "" "Prometheus/2.33.4" 2022-09-29T06:08:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:08:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:08:52.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:08:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:08:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:08:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:08:53.673 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:08:53 smithi083 ceph-mon[77242]: pgmap v1221: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:53.674 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:08:53 smithi083 ceph-mon[79930]: pgmap v1221: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:53.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:08:53 smithi153 ceph-mon[70044]: pgmap v1221: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:53.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:08:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:08:53.672Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:08:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:08:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:08:54.365Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:08:55.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:08:55 smithi083 ceph-mon[77242]: pgmap v1222: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:55.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:08:55 smithi083 ceph-mon[79930]: pgmap v1222: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:55.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:08:55 smithi153 ceph-mon[70044]: pgmap v1222: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:57.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:08:57 smithi083 ceph-mon[79930]: pgmap v1223: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:57.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:08:57 smithi083 ceph-mon[77242]: pgmap v1223: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:57.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:08:57 smithi153 ceph-mon[70044]: pgmap v1223: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:59.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:08:59 smithi083 ceph-mon[79930]: pgmap v1224: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:59.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:08:59 smithi083 ceph-mon[77242]: pgmap v1224: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:08:59.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:08:59 smithi153 ceph-mon[70044]: pgmap v1224: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:00.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:09:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:09:00.256Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:09:01.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:09:01 smithi083 ceph-mon[79930]: pgmap v1225: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:01.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:09:01 smithi083 ceph-mon[77242]: pgmap v1225: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:01.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:09:01 smithi153 ceph-mon[70044]: pgmap v1225: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:09:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:09:01] "GET /metrics HTTP/1.1" 200 225173 "" "Prometheus/2.33.4" 2022-09-29T06:09:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:09:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:09:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:09:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:09:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:09:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:09:03.674 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:09:03 smithi083 ceph-mon[79930]: pgmap v1226: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:03.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:09:03 smithi083 ceph-mon[77242]: pgmap v1226: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:03.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:09:03 smithi153 ceph-mon[70044]: pgmap v1226: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:03.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:09:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:09:03.672Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:09:04.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:09:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:09:04.366Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:09:05.558 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:09:05.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:09:05 smithi083 ceph-mon[79930]: pgmap v1227: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:05.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:09:05 smithi083 ceph-mon[77242]: pgmap v1227: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:05.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:09:05 smithi153 ceph-mon[70044]: pgmap v1227: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:05.847 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:09:05.847 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (42m) 5m ago 43m 26.7M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:09:05.847 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (43m) 3m ago 43m 61.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:09:05.847 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (44m) 5m ago 44m 21.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:09:05.848 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (44m) 3m ago 44m 27.6M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:09:05.848 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (39m) 3m ago 46m 439M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:09:05.848 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (40m) 5m ago 48m 538M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:09:05.848 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (39m) 5m ago 48m 204M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:09:05.849 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (38m) 3m ago 46m 101M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:09:05.849 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (39m) 5m ago 47m 124M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:09:05.849 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (43m) 5m ago 43m 25.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:09:05.849 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (43m) 3m ago 43m 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:09:05.850 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (38m) 5m ago 46m 158M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:09:05.850 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (37m) 5m ago 46m 150M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:09:05.850 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (37m) 5m ago 45m 124M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:09:05.850 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (36m) 5m ago 45m 141M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:09:05.850 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (36m) 3m ago 45m 133M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:09:05.851 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (35m) 3m ago 44m 152M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:09:05.851 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (35m) 3m ago 44m 135M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:09:05.851 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (34m) 3m ago 44m 150M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:09:05.851 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (39m) 3m ago 43m 100M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:09:06.205 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:09:06.206 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:09:06.206 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:09:06.206 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:09:06.206 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:09:06.207 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:09:06.207 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:09:06.207 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:09:06.207 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:09:06.207 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:09:06.208 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:09:06.208 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:09:06.208 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:09:06.208 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:09:06.209 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:09:06.209 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:09:06.209 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:09:06.209 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:09:06.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:09:06 smithi083 ceph-mon[79930]: from='client.35565 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:09:06.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:09:06 smithi083 ceph-mon[79930]: from='client.35571 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:09:06.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:09:06 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2124335722' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:09:06.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:09:06 smithi083 ceph-mon[77242]: from='client.35565 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:09:06.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:09:06 smithi083 ceph-mon[77242]: from='client.35571 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:09:06.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:09:06 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2124335722' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:09:06.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:09:06 smithi153 ceph-mon[70044]: from='client.35565 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:09:06.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:09:06 smithi153 ceph-mon[70044]: from='client.35571 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:09:06.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:09:06 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2124335722' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:09:07.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:09:07 smithi083 ceph-mon[79930]: pgmap v1228: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:07.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:09:07 smithi083 ceph-mon[77242]: pgmap v1228: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:07.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:09:07 smithi153 ceph-mon[70044]: pgmap v1228: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:09.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:09:09 smithi083 ceph-mon[79930]: pgmap v1229: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:09.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:09:09 smithi083 ceph-mon[77242]: pgmap v1229: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:09.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:09:09 smithi153 ceph-mon[70044]: pgmap v1229: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:10.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:09:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:09:10.257Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:09:11.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:09:11 smithi083 ceph-mon[79930]: pgmap v1230: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:11.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:09:11 smithi083 ceph-mon[77242]: pgmap v1230: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:11.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:09:11 smithi153 ceph-mon[70044]: pgmap v1230: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:09:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:09:11] "GET /metrics HTTP/1.1" 200 225174 "" "Prometheus/2.33.4" 2022-09-29T06:09:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:09:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:09:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:09:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:09:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:09:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:09:13.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:09:13 smithi083 ceph-mon[77242]: pgmap v1231: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:13.675 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:09:13 smithi083 ceph-mon[79930]: pgmap v1231: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:13.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:09:13 smithi153 ceph-mon[70044]: pgmap v1231: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:13.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:09:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:09:13.673Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:09:14.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:09:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:09:14.367Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:09:15.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:09:15 smithi083 ceph-mon[79930]: pgmap v1232: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:15.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:09:15 smithi083 ceph-mon[77242]: pgmap v1232: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:15.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:09:15 smithi153 ceph-mon[70044]: pgmap v1232: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:17.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:09:17 smithi083 ceph-mon[79930]: pgmap v1233: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:17.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:09:17 smithi083 ceph-mon[77242]: pgmap v1233: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:17.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:09:17 smithi153 ceph-mon[70044]: pgmap v1233: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:19.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:09:19 smithi083 ceph-mon[79930]: pgmap v1234: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:19.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:09:19 smithi083 ceph-mon[77242]: pgmap v1234: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:19.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:09:19 smithi153 ceph-mon[70044]: pgmap v1234: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:20.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:09:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:09:20.257Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:09:21.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:09:21 smithi083 ceph-mon[79930]: pgmap v1235: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:21.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:09:21 smithi083 ceph-mon[77242]: pgmap v1235: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:21.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:09:21 smithi153 ceph-mon[70044]: pgmap v1235: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:09:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:09:21] "GET /metrics HTTP/1.1" 200 225174 "" "Prometheus/2.33.4" 2022-09-29T06:09:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:09:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:09:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:09:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:09:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:09:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:09:23.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:09:23 smithi083 ceph-mon[77242]: pgmap v1236: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:23.675 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:09:23 smithi083 ceph-mon[79930]: pgmap v1236: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:23.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:09:23 smithi153 ceph-mon[70044]: pgmap v1236: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:23.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:09:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:09:23.673Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:09:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:09:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:09:24.368Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:09:25.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:09:25 smithi083 ceph-mon[79930]: pgmap v1237: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:25.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:09:25 smithi083 ceph-mon[77242]: pgmap v1237: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:25.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:09:25 smithi153 ceph-mon[70044]: pgmap v1237: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:27.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:09:27 smithi083 ceph-mon[79930]: pgmap v1238: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:27.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:09:27 smithi083 ceph-mon[77242]: pgmap v1238: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:27.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:09:27 smithi153 ceph-mon[70044]: pgmap v1238: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:29.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:09:29 smithi083 ceph-mon[79930]: pgmap v1239: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:29.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:09:29 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:09:29.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:09:29 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:09:29.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:09:29 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:09:29.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:09:29 smithi083 ceph-mon[77242]: pgmap v1239: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:29.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:09:29 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:09:29.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:09:29 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:09:29.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:09:29 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:09:29.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:09:29 smithi153 ceph-mon[70044]: pgmap v1239: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:29.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:09:29 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:09:29.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:09:29 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:09:29.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:09:29 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:09:30.654 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:09:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:09:30.258Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:09:30.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:09:30 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:09:30.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:09:30 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:09:31.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:09:30 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:09:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:09:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:09:31] "GET /metrics HTTP/1.1" 200 225172 "" "Prometheus/2.33.4" 2022-09-29T06:09:31.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:09:31 smithi083 ceph-mon[79930]: pgmap v1240: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:31.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:09:31 smithi083 ceph-mon[77242]: pgmap v1240: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:32.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:09:31 smithi153 ceph-mon[70044]: pgmap v1240: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:09:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:09:32.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:09:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:09:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:09:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:09:33.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:09:33 smithi083 ceph-mon[79930]: pgmap v1241: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:33.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:09:33 smithi083 ceph-mon[77242]: pgmap v1241: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:33.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:09:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:09:33.673Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:09:34.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:09:33 smithi153 ceph-mon[70044]: pgmap v1241: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:34.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:09:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:09:34.368Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:09:35.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:09:35 smithi083 ceph-mon[79930]: pgmap v1242: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:35.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:09:35 smithi083 ceph-mon[77242]: pgmap v1242: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:36.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:09:35 smithi153 ceph-mon[70044]: pgmap v1242: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:36.534 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:09:36.825 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:09:36.825 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (43m) 6m ago 43m 26.7M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:09:36.826 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (43m) 4m ago 43m 61.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:09:36.826 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (44m) 6m ago 44m 21.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:09:36.826 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (44m) 4m ago 44m 27.6M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:09:36.826 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (40m) 4m ago 47m 439M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:09:36.826 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (40m) 6m ago 48m 538M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:09:36.827 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (40m) 6m ago 48m 204M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:09:36.827 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (39m) 4m ago 47m 101M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:09:36.827 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (39m) 6m ago 47m 124M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:09:36.827 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (44m) 6m ago 44m 25.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:09:36.827 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (43m) 4m ago 43m 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:09:36.828 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (38m) 6m ago 46m 158M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:09:36.828 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (38m) 6m ago 46m 150M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:09:36.828 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (37m) 6m ago 46m 124M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:09:36.828 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (37m) 6m ago 46m 141M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:09:36.828 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (36m) 4m ago 45m 133M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:09:36.829 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (36m) 4m ago 45m 152M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:09:36.829 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (35m) 4m ago 45m 135M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:09:36.829 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (35m) 4m ago 44m 150M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:09:36.829 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (40m) 4m ago 44m 100M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:09:37.181 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:09:37.181 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:09:37.181 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:09:37.182 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:09:37.182 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:09:37.182 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:09:37.182 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:09:37.183 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:09:37.183 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:09:37.183 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:09:37.183 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:09:37.183 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:09:37.184 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:09:37.184 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:09:37.184 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:09:37.184 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:09:37.184 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:09:37.185 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:09:37.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:09:37 smithi083 ceph-mon[79930]: pgmap v1243: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:37.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:09:37 smithi083 ceph-mon[79930]: from='client.45517 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:09:37.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:09:37 smithi083 ceph-mon[79930]: from='client.45523 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:09:37.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:09:37 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/782983363' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:09:37.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:09:37 smithi083 ceph-mon[77242]: pgmap v1243: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:37.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:09:37 smithi083 ceph-mon[77242]: from='client.45517 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:09:37.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:09:37 smithi083 ceph-mon[77242]: from='client.45523 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:09:37.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:09:37 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/782983363' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:09:38.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:09:37 smithi153 ceph-mon[70044]: pgmap v1243: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:38.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:09:37 smithi153 ceph-mon[70044]: from='client.45517 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:09:38.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:09:37 smithi153 ceph-mon[70044]: from='client.45523 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:09:38.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:09:37 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/782983363' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:09:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:09:39 smithi083 ceph-mon[79930]: pgmap v1244: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:09:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:09:39.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:09:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:09:39.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:09:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:09:39.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:09:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:09:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:09:39 smithi083 ceph-mon[77242]: pgmap v1244: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:39.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:09:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:09:39.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:09:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:09:39.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:09:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:09:39.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:09:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:09:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:09:39 smithi153 ceph-mon[70044]: pgmap v1244: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:09:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:09:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:09:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:09:40.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:09:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:09:40.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:09:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:09:40.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:09:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:09:40.259Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:09:41.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:09:41 smithi083 ceph-mon[79930]: pgmap v1245: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:41.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:09:41 smithi083 ceph-mon[77242]: pgmap v1245: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:09:41 smithi153 ceph-mon[70044]: pgmap v1245: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:09:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:09:41] "GET /metrics HTTP/1.1" 200 225172 "" "Prometheus/2.33.4" 2022-09-29T06:09:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:09:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:09:42.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:09:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:09:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:09:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:09:43.679 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:09:43 smithi083 ceph-mon[79930]: pgmap v1246: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:43.679 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:09:43 smithi083 ceph-mon[77242]: pgmap v1246: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:43.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:09:43 smithi153 ceph-mon[70044]: pgmap v1246: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:43.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:09:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:09:43.674Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:09:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:09:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:09:44.370Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:09:45.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:09:45 smithi083 ceph-mon[79930]: pgmap v1247: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:45.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:09:45 smithi083 ceph-mon[77242]: pgmap v1247: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:45.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:09:45 smithi153 ceph-mon[70044]: pgmap v1247: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:47.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:09:47 smithi083 ceph-mon[79930]: pgmap v1248: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:47.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:09:47 smithi083 ceph-mon[77242]: pgmap v1248: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:47.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:09:47 smithi153 ceph-mon[70044]: pgmap v1248: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:49.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:09:49 smithi083 ceph-mon[79930]: pgmap v1249: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:49.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:09:49 smithi083 ceph-mon[77242]: pgmap v1249: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:49.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:09:49 smithi153 ceph-mon[70044]: pgmap v1249: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:50.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:09:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:09:50.259Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:09:51.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:09:51 smithi083 ceph-mon[79930]: pgmap v1250: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:51.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:09:51 smithi083 ceph-mon[77242]: pgmap v1250: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:51.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:09:51 smithi153 ceph-mon[70044]: pgmap v1250: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:09:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:09:51] "GET /metrics HTTP/1.1" 200 225172 "" "Prometheus/2.33.4" 2022-09-29T06:09:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:09:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:09:52.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:09:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:09:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:09:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:09:53.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:09:53 smithi083 ceph-mon[79930]: pgmap v1251: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:53.677 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:09:53 smithi083 ceph-mon[77242]: pgmap v1251: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:53.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:09:53 smithi153 ceph-mon[70044]: pgmap v1251: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:53.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:09:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:09:53.675Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:09:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:09:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:09:54.370Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:09:55.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:09:55 smithi083 ceph-mon[77242]: pgmap v1252: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:55.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:09:55 smithi083 ceph-mon[79930]: pgmap v1252: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:55.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:09:55 smithi153 ceph-mon[70044]: pgmap v1252: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:57.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:09:57 smithi083 ceph-mon[77242]: pgmap v1253: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:57.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:09:57 smithi083 ceph-mon[79930]: pgmap v1253: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:57.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:09:57 smithi153 ceph-mon[70044]: pgmap v1253: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:59.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:09:59 smithi083 ceph-mon[77242]: pgmap v1254: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:59.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:09:59 smithi083 ceph-mon[79930]: pgmap v1254: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:09:59.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:09:59 smithi153 ceph-mon[70044]: pgmap v1254: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:00.261 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:10:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[77219]: 2022-09-29T06:09:59.999+0000 7f160f665700 -1 log_channel(cluster) log [ERR] : overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T06:10:00.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:10:00 smithi083 ceph-mon[79930]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T06:10:00.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:10:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:10:00.260Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:10:00.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:10:00 smithi083 ceph-mon[77242]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T06:10:00.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:10:00 smithi153 ceph-mon[70044]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T06:10:01.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:10:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:10:01] "GET /metrics HTTP/1.1" 200 225168 "" "Prometheus/2.33.4" 2022-09-29T06:10:01.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:10:01 smithi083 ceph-mon[79930]: pgmap v1255: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:01.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:10:01 smithi083 ceph-mon[77242]: pgmap v1255: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:01.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:10:01 smithi153 ceph-mon[70044]: pgmap v1255: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:10:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:10:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:10:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:10:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:10:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:10:03.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:10:03 smithi083 ceph-mon[79930]: pgmap v1256: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:03.678 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:10:03 smithi083 ceph-mon[77242]: pgmap v1256: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:03.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:10:03 smithi153 ceph-mon[70044]: pgmap v1256: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:03.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:10:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:10:03.675Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:10:04.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:10:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:10:04.371Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:10:05.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:10:05 smithi083 ceph-mon[77242]: pgmap v1257: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:05.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:10:05 smithi083 ceph-mon[79930]: pgmap v1257: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:05.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:10:05 smithi153 ceph-mon[70044]: pgmap v1257: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:07.511 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:10:07.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:10:07 smithi083 ceph-mon[79930]: pgmap v1258: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:07.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:10:07 smithi083 ceph-mon[77242]: pgmap v1258: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:07.801 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:10:07.802 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (43m) 6m ago 44m 26.7M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:10:07.802 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (44m) 4m ago 44m 61.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:10:07.802 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (45m) 6m ago 45m 21.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:10:07.802 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (45m) 4m ago 45m 27.6M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:10:07.803 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (40m) 4m ago 47m 439M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:10:07.803 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (41m) 6m ago 49m 538M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:10:07.803 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (40m) 6m ago 49m 204M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:10:07.803 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (40m) 4m ago 47m 101M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:10:07.804 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (40m) 6m ago 48m 124M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:10:07.804 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (44m) 6m ago 44m 25.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:10:07.804 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (44m) 4m ago 44m 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:10:07.804 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (39m) 6m ago 47m 158M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:10:07.804 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (38m) 6m ago 47m 150M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:10:07.805 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (38m) 6m ago 46m 124M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:10:07.805 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (37m) 6m ago 46m 141M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:10:07.805 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (37m) 4m ago 46m 133M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:10:07.805 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (36m) 4m ago 46m 152M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:10:07.806 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (36m) 4m ago 45m 135M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:10:07.806 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (35m) 4m ago 45m 150M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:10:07.806 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (40m) 4m ago 44m 100M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:10:07.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:10:07 smithi153 ceph-mon[70044]: pgmap v1258: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:08.153 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:10:08.153 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:10:08.153 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:10:08.154 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:10:08.154 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:10:08.154 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:10:08.154 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:10:08.155 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:10:08.155 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:10:08.155 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:10:08.155 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:10:08.156 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:10:08.156 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:10:08.156 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:10:08.156 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:10:08.156 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:10:08.157 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:10:08.157 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:10:08.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:10:08 smithi083 ceph-mon[79930]: from='client.35601 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:10:08.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:10:08 smithi083 ceph-mon[79930]: from='client.45541 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:10:08.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:10:08 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3478463505' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:10:08.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:10:08 smithi083 ceph-mon[77242]: from='client.35601 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:10:08.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:10:08 smithi083 ceph-mon[77242]: from='client.45541 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:10:08.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:10:08 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3478463505' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:10:08.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:10:08 smithi153 ceph-mon[70044]: from='client.35601 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:10:08.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:10:08 smithi153 ceph-mon[70044]: from='client.45541 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:10:08.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:10:08 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3478463505' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:10:09.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:10:09 smithi083 ceph-mon[79930]: pgmap v1259: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:09.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:10:09 smithi083 ceph-mon[77242]: pgmap v1259: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:09.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:10:09 smithi153 ceph-mon[70044]: pgmap v1259: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:10.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:10:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:10:10.260Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:10:11.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:10:11 smithi083 ceph-mon[77242]: pgmap v1260: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:11.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:10:11 smithi083 ceph-mon[79930]: pgmap v1260: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:11.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:10:11 smithi153 ceph-mon[70044]: pgmap v1260: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:10:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:10:11] "GET /metrics HTTP/1.1" 200 225167 "" "Prometheus/2.33.4" 2022-09-29T06:10:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:10:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:10:12.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:10:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:10:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:10:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:10:13.678 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:10:13 smithi083 ceph-mon[79930]: pgmap v1261: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:13.678 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:10:13 smithi083 ceph-mon[77242]: pgmap v1261: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:13.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:10:13 smithi153 ceph-mon[70044]: pgmap v1261: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:13.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:10:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:10:13.676Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:10:14.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:10:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:10:14.371Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:10:15.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:10:15 smithi083 ceph-mon[77242]: pgmap v1262: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:15.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:10:15 smithi083 ceph-mon[79930]: pgmap v1262: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:15.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:10:15 smithi153 ceph-mon[70044]: pgmap v1262: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:17.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:10:17 smithi083 ceph-mon[79930]: pgmap v1263: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:17.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:10:17 smithi083 ceph-mon[77242]: pgmap v1263: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:17.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:10:17 smithi153 ceph-mon[70044]: pgmap v1263: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:19.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:10:19 smithi083 ceph-mon[79930]: pgmap v1264: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:19.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:10:19 smithi083 ceph-mon[77242]: pgmap v1264: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:19.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:10:19 smithi153 ceph-mon[70044]: pgmap v1264: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:20.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:10:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:10:20.261Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:10:21.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:10:21 smithi083 ceph-mon[79930]: pgmap v1265: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:21.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:10:21 smithi083 ceph-mon[77242]: pgmap v1265: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:21.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:10:21 smithi153 ceph-mon[70044]: pgmap v1265: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:10:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:10:21] "GET /metrics HTTP/1.1" 200 225167 "" "Prometheus/2.33.4" 2022-09-29T06:10:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:10:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:10:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:10:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:10:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:10:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:10:23.679 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:10:23 smithi083 ceph-mon[79930]: pgmap v1266: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:23.679 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:10:23 smithi083 ceph-mon[77242]: pgmap v1266: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:23.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:10:23 smithi153 ceph-mon[70044]: pgmap v1266: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:23.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:10:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:10:23.677Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:10:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:10:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:10:24.372Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:10:25.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:10:25 smithi083 ceph-mon[79930]: pgmap v1267: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:25.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:10:25 smithi083 ceph-mon[77242]: pgmap v1267: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:25.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:10:25 smithi153 ceph-mon[70044]: pgmap v1267: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:27.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:10:27 smithi083 ceph-mon[77242]: pgmap v1268: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:27.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:10:27 smithi083 ceph-mon[79930]: pgmap v1268: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:27.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:10:27 smithi153 ceph-mon[70044]: pgmap v1268: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:29.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:10:29 smithi083 ceph-mon[77242]: pgmap v1269: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:29.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:10:29 smithi083 ceph-mon[79930]: pgmap v1269: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:29.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:10:29 smithi153 ceph-mon[70044]: pgmap v1269: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:30.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:10:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:10:30.267Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:10:30.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:10:30 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:10:30.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:10:30 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:10:30.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:10:30 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:10:30.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:10:30 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:10:30.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:10:30 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:10:30.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:10:30 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:10:30.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:10:30 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:10:30.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:10:30 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:10:30.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:10:30 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:10:30.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:10:30 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:10:30.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:10:30 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:10:30.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:10:30 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:10:31.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:10:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:10:31] "GET /metrics HTTP/1.1" 200 225174 "" "Prometheus/2.33.4" 2022-09-29T06:10:31.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:10:31 smithi083 ceph-mon[77242]: pgmap v1270: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:31.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:10:31 smithi083 ceph-mon[79930]: pgmap v1270: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:31.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:10:31 smithi153 ceph-mon[70044]: pgmap v1270: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:10:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:10:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:10:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:10:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:10:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:10:33.679 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:10:33 smithi083 ceph-mon[79930]: pgmap v1271: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:33.680 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:10:33 smithi083 ceph-mon[77242]: pgmap v1271: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:33.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:10:33 smithi153 ceph-mon[70044]: pgmap v1271: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:33.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:10:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:10:33.677Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:10:34.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:10:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:10:34.372Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:10:35.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:10:35 smithi083 ceph-mon[77242]: pgmap v1272: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:35.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:10:35 smithi083 ceph-mon[79930]: pgmap v1272: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:35.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:10:35 smithi153 ceph-mon[70044]: pgmap v1272: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:37.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:10:37 smithi083 ceph-mon[79930]: pgmap v1273: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:37.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:10:37 smithi083 ceph-mon[77242]: pgmap v1273: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:37.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:10:37 smithi153 ceph-mon[70044]: pgmap v1273: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:38.481 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:10:38.771 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:10:38.771 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (44m) 7m ago 44m 26.7M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:10:38.771 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (44m) 5m ago 44m 61.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:10:38.772 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (45m) 7m ago 45m 21.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:10:38.772 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (45m) 5m ago 45m 27.6M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:10:38.772 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (41m) 5m ago 48m 439M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:10:38.772 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (41m) 7m ago 49m 538M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:10:38.772 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (41m) 7m ago 49m 204M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:10:38.773 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (40m) 5m ago 48m 101M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:10:38.773 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (40m) 7m ago 48m 124M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:10:38.773 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (45m) 7m ago 45m 25.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:10:38.773 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (45m) 5m ago 45m 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:10:38.774 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (39m) 7m ago 47m 158M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:10:38.774 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (39m) 7m ago 47m 150M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:10:38.774 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (38m) 7m ago 47m 124M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:10:38.774 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (38m) 7m ago 47m 141M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:10:38.774 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (37m) 5m ago 46m 133M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:10:38.775 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (37m) 5m ago 46m 152M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:10:38.775 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (36m) 5m ago 46m 135M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:10:38.775 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (36m) 5m ago 45m 150M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:10:38.775 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (41m) 5m ago 45m 100M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:10:39.129 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:10:39.130 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:10:39.130 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:10:39.130 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:10:39.131 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:10:39.131 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:10:39.131 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:10:39.132 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:10:39.132 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:10:39.132 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:10:39.133 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:10:39.133 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:10:39.133 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:10:39.133 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:10:39.134 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:10:39.134 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:10:39.134 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:10:39.135 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:10:39.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:10:39 smithi083 ceph-mon[79930]: pgmap v1274: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:39.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:10:39 smithi083 ceph-mon[79930]: from='client.35619 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:10:39.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:10:39 smithi083 ceph-mon[79930]: from='client.35625 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:10:39.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:10:39 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2245471553' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:10:39.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:10:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:10:39.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:10:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:10:39.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:10:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:10:39.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:10:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:10:39.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:10:39 smithi083 ceph-mon[77242]: pgmap v1274: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:39.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:10:39 smithi083 ceph-mon[77242]: from='client.35619 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:10:39.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:10:39 smithi083 ceph-mon[77242]: from='client.35625 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:10:39.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:10:39 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2245471553' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:10:39.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:10:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:10:39.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:10:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:10:39.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:10:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:10:39.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:10:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:10:39.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:10:39 smithi153 ceph-mon[70044]: pgmap v1274: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:39.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:10:39 smithi153 ceph-mon[70044]: from='client.35619 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:10:39.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:10:39 smithi153 ceph-mon[70044]: from='client.35625 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:10:39.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:10:39 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2245471553' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:10:39.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:10:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:10:39.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:10:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:10:39.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:10:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:10:39.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:10:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:10:40.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:10:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:10:40.263Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:10:41.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:10:41 smithi083 ceph-mon[79930]: pgmap v1275: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:41.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:10:41 smithi083 ceph-mon[77242]: pgmap v1275: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:10:41 smithi153 ceph-mon[70044]: pgmap v1275: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:10:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:10:41] "GET /metrics HTTP/1.1" 200 225173 "" "Prometheus/2.33.4" 2022-09-29T06:10:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:10:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:10:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:10:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:10:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:10:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:10:43.679 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:10:43 smithi083 ceph-mon[79930]: pgmap v1276: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:43.680 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:10:43 smithi083 ceph-mon[77242]: pgmap v1276: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:43.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:10:43 smithi153 ceph-mon[70044]: pgmap v1276: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:43.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:10:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:10:43.677Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:10:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:10:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:10:44.373Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:10:45.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:10:45 smithi083 ceph-mon[77242]: pgmap v1277: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:45.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:10:45 smithi083 ceph-mon[79930]: pgmap v1277: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:45.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:10:45 smithi153 ceph-mon[70044]: pgmap v1277: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:47.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:10:47 smithi083 ceph-mon[79930]: pgmap v1278: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:47.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:10:47 smithi083 ceph-mon[77242]: pgmap v1278: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:47.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:10:47 smithi153 ceph-mon[70044]: pgmap v1278: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:49.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:10:49 smithi083 ceph-mon[79930]: pgmap v1279: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:49.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:10:49 smithi083 ceph-mon[77242]: pgmap v1279: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:49.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:10:49 smithi153 ceph-mon[70044]: pgmap v1279: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:50.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:10:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:10:50.264Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:10:51.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:10:51 smithi083 ceph-mon[79930]: pgmap v1280: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:51.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:10:51 smithi083 ceph-mon[77242]: pgmap v1280: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:51.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:10:51 smithi153 ceph-mon[70044]: pgmap v1280: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:10:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:10:51] "GET /metrics HTTP/1.1" 200 225173 "" "Prometheus/2.33.4" 2022-09-29T06:10:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:10:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:10:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:10:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:10:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:10:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:10:53.679 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:10:53 smithi083 ceph-mon[79930]: pgmap v1281: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:53.680 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:10:53 smithi083 ceph-mon[77242]: pgmap v1281: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:53.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:10:53 smithi153 ceph-mon[70044]: pgmap v1281: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:53.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:10:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:10:53.678Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:10:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:10:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:10:54.374Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:10:55.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:10:55 smithi083 ceph-mon[79930]: pgmap v1282: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:55.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:10:55 smithi083 ceph-mon[77242]: pgmap v1282: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:55.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:10:55 smithi153 ceph-mon[70044]: pgmap v1282: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:57.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:10:57 smithi083 ceph-mon[79930]: pgmap v1283: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:57.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:10:57 smithi083 ceph-mon[77242]: pgmap v1283: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:57.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:10:57 smithi153 ceph-mon[70044]: pgmap v1283: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:59.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:10:59 smithi083 ceph-mon[79930]: pgmap v1284: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:59.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:10:59 smithi083 ceph-mon[77242]: pgmap v1284: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:10:59.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:10:59 smithi153 ceph-mon[70044]: pgmap v1284: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:00.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:11:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:11:00.265Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:11:01.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:11:01 smithi083 ceph-mon[77242]: pgmap v1285: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:01.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:11:01 smithi083 ceph-mon[79930]: pgmap v1285: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:01.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:11:01 smithi153 ceph-mon[70044]: pgmap v1285: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:11:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:11:01] "GET /metrics HTTP/1.1" 200 225171 "" "Prometheus/2.33.4" 2022-09-29T06:11:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:11:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:11:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:11:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:11:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:11:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:11:03.680 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:11:03 smithi083 ceph-mon[79930]: pgmap v1286: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:03.681 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:11:03 smithi083 ceph-mon[77242]: pgmap v1286: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:03.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:11:03 smithi153 ceph-mon[70044]: pgmap v1286: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:03.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:11:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:11:03.678Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:11:04.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:11:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:11:04.375Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:11:05.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:11:05 smithi083 ceph-mon[79930]: pgmap v1287: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:05.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:11:05 smithi083 ceph-mon[77242]: pgmap v1287: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:05.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:11:05 smithi153 ceph-mon[70044]: pgmap v1287: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:07.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:11:07 smithi083 ceph-mon[77242]: pgmap v1288: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:07.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:11:07 smithi083 ceph-mon[79930]: pgmap v1288: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:07.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:11:07 smithi153 ceph-mon[70044]: pgmap v1288: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:09.459 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:11:09.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:11:09 smithi083 ceph-mon[79930]: pgmap v1289: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:09.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:11:09 smithi083 ceph-mon[77242]: pgmap v1289: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:09.746 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:11:09.746 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (44m) 7m ago 45m 26.7M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:11:09.747 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (45m) 5m ago 45m 61.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:11:09.747 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (46m) 7m ago 46m 21.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:11:09.747 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (46m) 5m ago 46m 27.6M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:11:09.747 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (41m) 5m ago 48m 439M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:11:09.747 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (42m) 7m ago 50m 538M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:11:09.748 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (41m) 7m ago 50m 204M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:11:09.748 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (41m) 5m ago 48m 101M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:11:09.748 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (41m) 7m ago 49m 124M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:11:09.748 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (45m) 7m ago 45m 25.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:11:09.749 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (45m) 5m ago 45m 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:11:09.749 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (40m) 7m ago 48m 158M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:11:09.749 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (39m) 7m ago 48m 150M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:11:09.749 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (39m) 7m ago 47m 124M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:11:09.749 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (38m) 7m ago 47m 141M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:11:09.750 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (38m) 5m ago 47m 133M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:11:09.750 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (37m) 5m ago 47m 152M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:11:09.750 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (37m) 5m ago 46m 135M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:11:09.750 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (36m) 5m ago 46m 150M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:11:09.750 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (41m) 5m ago 45m 100M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:11:09.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:11:09 smithi153 ceph-mon[70044]: pgmap v1289: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:10.102 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:11:10.103 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:11:10.103 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:11:10.103 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:11:10.103 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:11:10.104 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:11:10.104 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:11:10.104 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:11:10.104 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:11:10.104 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:11:10.105 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:11:10.105 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:11:10.105 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:11:10.105 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:11:10.106 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:11:10.106 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:11:10.106 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:11:10.106 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:11:10.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:11:10 smithi083 ceph-mon[79930]: from='client.35637 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:11:10.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:11:10 smithi083 ceph-mon[79930]: from='client.45577 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:11:10.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:11:10 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/837234662' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:11:10.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:11:10 smithi083 ceph-mon[77242]: from='client.35637 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:11:10.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:11:10 smithi083 ceph-mon[77242]: from='client.45577 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:11:10.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:11:10 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/837234662' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:11:10.714 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:11:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:11:10.265Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:11:10.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:11:10 smithi153 ceph-mon[70044]: from='client.35637 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:11:10.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:11:10 smithi153 ceph-mon[70044]: from='client.45577 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:11:10.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:11:10 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/837234662' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:11:11.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:11:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:11:11] "GET /metrics HTTP/1.1" 200 225173 "" "Prometheus/2.33.4" 2022-09-29T06:11:11.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:11:11 smithi083 ceph-mon[79930]: pgmap v1290: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:11.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:11:11 smithi083 ceph-mon[77242]: pgmap v1290: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:11.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:11:11 smithi153 ceph-mon[70044]: pgmap v1290: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:11:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:11:12.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:11:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:11:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:11:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:11:13.681 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:11:13 smithi083 ceph-mon[79930]: pgmap v1291: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:13.681 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:11:13 smithi083 ceph-mon[77242]: pgmap v1291: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:13.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:11:13 smithi153 ceph-mon[70044]: pgmap v1291: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:13.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:11:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:11:13.679Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:11:14.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:11:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:11:14.375Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:11:15.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:11:15 smithi083 ceph-mon[79930]: pgmap v1292: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:15.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:11:15 smithi083 ceph-mon[77242]: pgmap v1292: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:15.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:11:15 smithi153 ceph-mon[70044]: pgmap v1292: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:17.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:11:17 smithi083 ceph-mon[79930]: pgmap v1293: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:17.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:11:17 smithi083 ceph-mon[77242]: pgmap v1293: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:17.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:11:17 smithi153 ceph-mon[70044]: pgmap v1293: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:19.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:11:19 smithi083 ceph-mon[77242]: pgmap v1294: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:19.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:11:19 smithi083 ceph-mon[79930]: pgmap v1294: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:19.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:11:19 smithi153 ceph-mon[70044]: pgmap v1294: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:20.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:11:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:11:20.267Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:11:21.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:11:21 smithi083 ceph-mon[79930]: pgmap v1295: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:21.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:11:21 smithi083 ceph-mon[77242]: pgmap v1295: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:21.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:11:21 smithi153 ceph-mon[70044]: pgmap v1295: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:11:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:11:21] "GET /metrics HTTP/1.1" 200 225173 "" "Prometheus/2.33.4" 2022-09-29T06:11:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:11:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:11:22.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:11:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:11:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:11:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:11:23.682 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:11:23 smithi083 ceph-mon[79930]: pgmap v1296: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:23.682 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:11:23 smithi083 ceph-mon[77242]: pgmap v1296: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:23.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:11:23 smithi153 ceph-mon[70044]: pgmap v1296: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:23.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:11:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:11:23.680Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:11:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:11:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:11:24.375Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:11:25.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:11:25 smithi083 ceph-mon[79930]: pgmap v1297: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:25.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:11:25 smithi083 ceph-mon[77242]: pgmap v1297: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:25.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:11:25 smithi153 ceph-mon[70044]: pgmap v1297: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:27.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:11:27 smithi083 ceph-mon[77242]: pgmap v1298: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:27.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:11:27 smithi083 ceph-mon[79930]: pgmap v1298: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:27.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:11:27 smithi153 ceph-mon[70044]: pgmap v1298: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:29.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:11:29 smithi083 ceph-mon[79930]: pgmap v1299: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:29.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:11:29 smithi083 ceph-mon[77242]: pgmap v1299: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:29.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:11:29 smithi153 ceph-mon[70044]: pgmap v1299: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:30.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:11:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:11:30.267Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:11:31.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:11:31 smithi083 ceph-mon[79930]: pgmap v1300: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:31.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:11:31 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:11:31.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:11:31 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:11:31.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:11:31 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:11:31.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:11:31 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:11:31.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:11:31 smithi083 ceph-mon[77242]: pgmap v1300: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:31.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:11:31 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:11:31.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:11:31 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:11:31.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:11:31 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:11:31.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:11:31 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:11:31.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:11:31 smithi153 ceph-mon[70044]: pgmap v1300: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:31.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:11:31 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:11:31.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:11:31 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:11:31.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:11:31 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:11:31.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:11:31 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:11:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:11:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:11:31] "GET /metrics HTTP/1.1" 200 225169 "" "Prometheus/2.33.4" 2022-09-29T06:11:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:11:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:11:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:11:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:11:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:11:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:11:33.683 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:11:33 smithi083 ceph-mon[79930]: pgmap v1301: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:33.683 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:11:33 smithi083 ceph-mon[77242]: pgmap v1301: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:33.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:11:33 smithi153 ceph-mon[70044]: pgmap v1301: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:33.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:11:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:11:33.680Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:11:34.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:11:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:11:34.376Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:11:35.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:11:35 smithi083 ceph-mon[79930]: pgmap v1302: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:35.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:11:35 smithi083 ceph-mon[77242]: pgmap v1302: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:35.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:11:35 smithi153 ceph-mon[70044]: pgmap v1302: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:37.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:11:37 smithi083 ceph-mon[79930]: pgmap v1303: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:37.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:11:37 smithi083 ceph-mon[77242]: pgmap v1303: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:37.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:11:37 smithi153 ceph-mon[70044]: pgmap v1303: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:39.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:11:39 smithi083 ceph-mon[79930]: pgmap v1304: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:39.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:11:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:11:39.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:11:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:11:39.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:11:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:11:39.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:11:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:11:39.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:11:39 smithi083 ceph-mon[77242]: pgmap v1304: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:39.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:11:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:11:39.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:11:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:11:39.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:11:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:11:39.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:11:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:11:39.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:11:39 smithi153 ceph-mon[70044]: pgmap v1304: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:39.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:11:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:11:39.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:11:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:11:39.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:11:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:11:39.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:11:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:11:40.433 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:11:40.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:11:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:11:40.268Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:11:40.721 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:11:40.721 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (45m) 8m ago 45m 26.7M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:11:40.722 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (45m) 6m ago 45m 61.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:11:40.722 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (46m) 8m ago 46m 21.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:11:40.722 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (46m) 6m ago 46m 27.6M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:11:40.722 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (42m) 6m ago 49m 439M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:11:40.723 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (42m) 8m ago 50m 538M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:11:40.723 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (42m) 8m ago 50m 204M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:11:40.723 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (41m) 6m ago 49m 101M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:11:40.723 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (41m) 8m ago 49m 124M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:11:40.724 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (46m) 8m ago 46m 25.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:11:40.724 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (46m) 6m ago 46m 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:11:40.724 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (40m) 8m ago 49m 158M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:11:40.724 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (40m) 8m ago 48m 150M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:11:40.724 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (40m) 8m ago 48m 124M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:11:40.725 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (39m) 8m ago 48m 141M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:11:40.725 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (38m) 6m ago 47m 133M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:11:40.725 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (38m) 6m ago 47m 152M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:11:40.725 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (37m) 6m ago 47m 135M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:11:40.726 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (37m) 6m ago 46m 150M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:11:40.726 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (42m) 6m ago 46m 100M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:11:41.079 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:11:41.079 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:11:41.079 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:11:41.080 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:11:41.080 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:11:41.080 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:11:41.081 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:11:41.081 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:11:41.081 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:11:41.081 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:11:41.082 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:11:41.082 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:11:41.082 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:11:41.082 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:11:41.083 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:11:41.083 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:11:41.083 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:11:41.083 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:11:41.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:11:41 smithi083 ceph-mon[79930]: pgmap v1305: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:41.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:11:41 smithi083 ceph-mon[79930]: from='client.45589 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:11:41.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:11:41 smithi083 ceph-mon[79930]: from='client.35661 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:11:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:11:41 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1963126515' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:11:41.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:11:41 smithi083 ceph-mon[77242]: pgmap v1305: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:41.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:11:41 smithi083 ceph-mon[77242]: from='client.45589 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:11:41.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:11:41 smithi083 ceph-mon[77242]: from='client.35661 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:11:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:11:41 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1963126515' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:11:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:11:41 smithi153 ceph-mon[70044]: pgmap v1305: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:11:41 smithi153 ceph-mon[70044]: from='client.45589 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:11:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:11:41 smithi153 ceph-mon[70044]: from='client.35661 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:11:41.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:11:41 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1963126515' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:11:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:11:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:11:41] "GET /metrics HTTP/1.1" 200 225164 "" "Prometheus/2.33.4" 2022-09-29T06:11:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:11:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:11:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:11:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:11:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:11:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:11:43.683 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:11:43 smithi083 ceph-mon[79930]: pgmap v1306: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:43.683 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:11:43 smithi083 ceph-mon[77242]: pgmap v1306: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:43.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:11:43 smithi153 ceph-mon[70044]: pgmap v1306: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:43.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:11:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:11:43.681Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:11:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:11:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:11:44.377Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:11:45.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:11:45 smithi083 ceph-mon[79930]: pgmap v1307: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:45.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:11:45 smithi083 ceph-mon[77242]: pgmap v1307: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:45.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:11:45 smithi153 ceph-mon[70044]: pgmap v1307: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:47.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:11:47 smithi083 ceph-mon[79930]: pgmap v1308: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:47.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:11:47 smithi083 ceph-mon[77242]: pgmap v1308: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:47.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:11:47 smithi153 ceph-mon[70044]: pgmap v1308: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:49.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:11:49 smithi083 ceph-mon[79930]: pgmap v1309: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:49.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:11:49 smithi083 ceph-mon[77242]: pgmap v1309: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:49.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:11:49 smithi153 ceph-mon[70044]: pgmap v1309: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:50.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:11:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:11:50.268Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:11:51.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:11:51 smithi083 ceph-mon[79930]: pgmap v1310: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:51.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:11:51 smithi083 ceph-mon[77242]: pgmap v1310: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:51.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:11:51 smithi153 ceph-mon[70044]: pgmap v1310: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:11:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:11:51] "GET /metrics HTTP/1.1" 200 225164 "" "Prometheus/2.33.4" 2022-09-29T06:11:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:11:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:11:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:11:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:11:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:11:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:11:53.683 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:11:53 smithi083 ceph-mon[79930]: pgmap v1311: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:53.684 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:11:53 smithi083 ceph-mon[77242]: pgmap v1311: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:53.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:11:53 smithi153 ceph-mon[70044]: pgmap v1311: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:53.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:11:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:11:53.682Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:11:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:11:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:11:54.377Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:11:55.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:11:55 smithi083 ceph-mon[79930]: pgmap v1312: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:55.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:11:55 smithi083 ceph-mon[77242]: pgmap v1312: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:55.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:11:55 smithi153 ceph-mon[70044]: pgmap v1312: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:57.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:11:57 smithi083 ceph-mon[77242]: pgmap v1313: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:57.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:11:57 smithi083 ceph-mon[79930]: pgmap v1313: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:57.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:11:57 smithi153 ceph-mon[70044]: pgmap v1313: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:59.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:11:59 smithi083 ceph-mon[77242]: pgmap v1314: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:59.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:11:59 smithi083 ceph-mon[79930]: pgmap v1314: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:11:59.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:11:59 smithi153 ceph-mon[70044]: pgmap v1314: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:00.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:12:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:12:00.269Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:12:01.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:12:01 smithi083 ceph-mon[79930]: pgmap v1315: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:01.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:12:01 smithi083 ceph-mon[77242]: pgmap v1315: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:01.485 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:12:01 smithi153 ceph-mon[70044]: pgmap v1315: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:12:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:12:01] "GET /metrics HTTP/1.1" 200 225170 "" "Prometheus/2.33.4" 2022-09-29T06:12:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:12:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:12:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:12:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:12:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:12:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:12:03.684 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:12:03 smithi083 ceph-mon[79930]: pgmap v1316: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:03.685 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:12:03 smithi083 ceph-mon[77242]: pgmap v1316: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:03.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:12:03 smithi153 ceph-mon[70044]: pgmap v1316: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:03.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:12:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:12:03.683Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:12:04.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:12:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:12:04.378Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:12:05.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:12:05 smithi083 ceph-mon[79930]: pgmap v1317: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:05.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:12:05 smithi083 ceph-mon[77242]: pgmap v1317: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:05.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:12:05 smithi153 ceph-mon[70044]: pgmap v1317: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:07.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:12:07 smithi083 ceph-mon[77242]: pgmap v1318: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:07.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:12:07 smithi083 ceph-mon[79930]: pgmap v1318: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:07.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:12:07 smithi153 ceph-mon[70044]: pgmap v1318: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:09.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:12:09 smithi083 ceph-mon[77242]: pgmap v1319: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:09.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:12:09 smithi083 ceph-mon[79930]: pgmap v1319: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:09.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:12:09 smithi153 ceph-mon[70044]: pgmap v1319: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:10.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:12:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:12:10.269Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:12:11.409 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:12:11.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:12:11 smithi083 ceph-mon[77242]: pgmap v1320: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:11.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:12:11 smithi083 ceph-mon[79930]: pgmap v1320: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:11.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:12:11 smithi153 ceph-mon[70044]: pgmap v1320: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:11.701 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:12:11.701 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (45m) 8m ago 46m 26.7M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:12:11.701 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (46m) 6m ago 46m 61.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:12:11.701 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (47m) 8m ago 47m 21.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:12:11.702 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (47m) 6m ago 47m 27.6M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:12:11.702 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (42m) 6m ago 49m 439M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:12:11.702 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (43m) 8m ago 51m 538M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:12:11.703 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (42m) 8m ago 51m 204M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:12:11.703 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (42m) 6m ago 50m 101M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:12:11.703 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (42m) 8m ago 50m 124M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:12:11.703 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (46m) 8m ago 46m 25.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:12:11.703 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (46m) 6m ago 46m 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:12:11.704 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (41m) 8m ago 49m 158M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:12:11.704 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (40m) 8m ago 49m 150M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:12:11.704 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (40m) 8m ago 48m 124M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:12:11.704 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (39m) 8m ago 48m 141M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:12:11.705 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (39m) 6m ago 48m 133M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:12:11.705 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (38m) 6m ago 48m 152M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:12:11.705 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (38m) 6m ago 47m 135M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:12:11.705 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (37m) 6m ago 47m 150M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:12:11.706 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (43m) 6m ago 46m 100M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:12:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:12:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:12:11] "GET /metrics HTTP/1.1" 200 225170 "" "Prometheus/2.33.4" 2022-09-29T06:12:12.059 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:12:12.059 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:12:12.060 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:12:12.060 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:12:12.060 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:12:12.060 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:12:12.060 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:12:12.061 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:12:12.061 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:12:12.061 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:12:12.061 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:12:12.062 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:12:12.062 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:12:12.062 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:12:12.062 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:12:12.063 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:12:12.063 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:12:12.063 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:12:12.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:12:12 smithi083 ceph-mon[79930]: from='client.35673 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:12:12.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:12:12 smithi083 ceph-mon[79930]: from='client.35679 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:12:12.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:12:12 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/750706475' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:12:12.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:12:12 smithi083 ceph-mon[77242]: from='client.35673 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:12:12.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:12:12 smithi083 ceph-mon[77242]: from='client.35679 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:12:12.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:12:12 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/750706475' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:12:12.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:12:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:12:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:12:12.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:12:12 smithi153 ceph-mon[70044]: from='client.35673 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:12:12.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:12:12 smithi153 ceph-mon[70044]: from='client.35679 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:12:12.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:12:12 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/750706475' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:12:13.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:12:13 smithi153 ceph-mon[70044]: pgmap v1321: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:12:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:12:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:12:13.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:12:13 smithi083 ceph-mon[79930]: pgmap v1321: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:13.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:12:13 smithi083 ceph-mon[77242]: pgmap v1321: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:13.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:12:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:12:13.683Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:12:14.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:12:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:12:14.379Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:12:15.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:12:15 smithi083 ceph-mon[79930]: pgmap v1322: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:15.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:12:15 smithi083 ceph-mon[77242]: pgmap v1322: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:15.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:12:15 smithi153 ceph-mon[70044]: pgmap v1322: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:17.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:12:17 smithi083 ceph-mon[79930]: pgmap v1323: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:17.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:12:17 smithi083 ceph-mon[77242]: pgmap v1323: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:17.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:12:17 smithi153 ceph-mon[70044]: pgmap v1323: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:19.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:12:19 smithi083 ceph-mon[79930]: pgmap v1324: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:19.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:12:19 smithi083 ceph-mon[77242]: pgmap v1324: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:19.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:12:19 smithi153 ceph-mon[70044]: pgmap v1324: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:20.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:12:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:12:20.270Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:12:21.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:12:21 smithi083 ceph-mon[79930]: pgmap v1325: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:21.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:12:21 smithi083 ceph-mon[77242]: pgmap v1325: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:21.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:12:21 smithi153 ceph-mon[70044]: pgmap v1325: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:12:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:12:21] "GET /metrics HTTP/1.1" 200 225170 "" "Prometheus/2.33.4" 2022-09-29T06:12:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:12:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:12:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:12:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:12:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:12:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:12:23.686 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:12:23 smithi083 ceph-mon[79930]: pgmap v1326: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:23.687 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:12:23 smithi083 ceph-mon[77242]: pgmap v1326: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:23.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:12:23 smithi153 ceph-mon[70044]: pgmap v1326: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:23.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:12:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:12:23.684Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:12:24.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:12:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:12:24.380Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:12:25.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:12:25 smithi083 ceph-mon[79930]: pgmap v1327: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:25.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:12:25 smithi083 ceph-mon[77242]: pgmap v1327: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:25.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:12:25 smithi153 ceph-mon[70044]: pgmap v1327: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:27.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:12:27 smithi083 ceph-mon[79930]: pgmap v1328: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:27.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:12:27 smithi083 ceph-mon[77242]: pgmap v1328: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:27.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:12:27 smithi153 ceph-mon[70044]: pgmap v1328: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:29.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:12:29 smithi083 ceph-mon[79930]: pgmap v1329: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:29.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:12:29 smithi083 ceph-mon[77242]: pgmap v1329: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:29.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:12:29 smithi153 ceph-mon[70044]: pgmap v1329: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:30.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:12:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:12:30.271Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:12:31.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:12:31 smithi083 ceph-mon[77242]: pgmap v1330: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:31.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:12:31 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:12:31.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:12:31 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:12:31.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:12:31 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:12:31.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:12:31 smithi083 ceph-mon[79930]: pgmap v1330: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:31.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:12:31 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:12:31.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:12:31 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:12:31.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:12:31 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:12:31.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:12:31 smithi153 ceph-mon[70044]: pgmap v1330: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:31.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:12:31 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:12:31.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:12:31 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:12:31.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:12:31 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:12:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:12:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:12:31] "GET /metrics HTTP/1.1" 200 225170 "" "Prometheus/2.33.4" 2022-09-29T06:12:32.630 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:12:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:12:32.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:12:32.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:12:32 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:12:32.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:12:32 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:12:33.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:12:32 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:12:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:12:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:12:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:12:33.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:12:33 smithi083 ceph-mon[79930]: pgmap v1331: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:33.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:12:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:12:33.685Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:12:33.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:12:33 smithi083 ceph-mon[77242]: pgmap v1331: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:34.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:12:33 smithi153 ceph-mon[70044]: pgmap v1331: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:34.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:12:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:12:34.380Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:12:35.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:12:35 smithi083 ceph-mon[77242]: pgmap v1332: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:35.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:12:35 smithi083 ceph-mon[79930]: pgmap v1332: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:36.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:12:35 smithi153 ceph-mon[70044]: pgmap v1332: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:37.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:12:37 smithi083 ceph-mon[79930]: pgmap v1333: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:37.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:12:37 smithi083 ceph-mon[77242]: pgmap v1333: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:38.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:12:37 smithi153 ceph-mon[70044]: pgmap v1333: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:12:39 smithi083 ceph-mon[79930]: pgmap v1334: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:39.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:12:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:12:39.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:12:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:12:39.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:12:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:12:39.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:12:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:12:39.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:12:39 smithi083 ceph-mon[77242]: pgmap v1334: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:39.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:12:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:12:39.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:12:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:12:39.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:12:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:12:39.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:12:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:12:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:12:39 smithi153 ceph-mon[70044]: pgmap v1334: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:40.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:12:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:12:40.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:12:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:12:40.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:12:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:12:40.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:12:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:12:40.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:12:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:12:40.272Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:12:41.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:12:41 smithi083 ceph-mon[79930]: pgmap v1335: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:41.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:12:41 smithi083 ceph-mon[77242]: pgmap v1335: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:12:41 smithi153 ceph-mon[70044]: pgmap v1335: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:12:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:12:41] "GET /metrics HTTP/1.1" 200 225168 "" "Prometheus/2.33.4" 2022-09-29T06:12:42.384 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:12:42.676 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:12:42.676 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (46m) 9m ago 46m 26.7M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:12:42.676 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (46m) 7m ago 46m 61.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:12:42.677 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (47m) 9m ago 47m 21.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:12:42.677 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (47m) 7m ago 47m 27.6M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:12:42.677 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (43m) 7m ago 50m 439M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:12:42.677 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (43m) 9m ago 51m 538M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:12:42.678 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (43m) 9m ago 51m 204M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:12:42.678 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (42m) 7m ago 50m 101M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:12:42.678 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (42m) 9m ago 50m 124M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:12:42.678 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (47m) 9m ago 47m 25.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:12:42.679 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (47m) 7m ago 47m 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:12:42.679 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (41m) 9m ago 50m 158M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:12:42.679 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (41m) 9m ago 49m 150M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:12:42.679 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (41m) 9m ago 49m 124M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:12:42.679 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (40m) 9m ago 49m 141M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:12:42.680 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (39m) 7m ago 48m 133M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:12:42.680 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (39m) 7m ago 48m 152M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:12:42.680 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (38m) 7m ago 48m 135M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:12:42.680 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (38m) 7m ago 48m 150M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:12:42.681 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (43m) 7m ago 47m 100M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:12:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:12:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:12:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:12:43.034 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:12:43.035 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:12:43.035 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:12:43.035 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:12:43.035 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:12:43.036 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:12:43.036 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:12:43.036 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:12:43.036 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:12:43.037 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:12:43.037 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:12:43.037 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:12:43.037 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:12:43.037 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:12:43.038 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:12:43.038 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:12:43.038 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:12:43.038 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:12:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:12:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:12:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:12:43.686 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:12:43 smithi083 ceph-mon[77242]: pgmap v1336: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:43.686 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:12:43 smithi083 ceph-mon[77242]: from='client.35691 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:12:43.687 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:12:43 smithi083 ceph-mon[77242]: from='client.45631 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:12:43.687 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:12:43 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2045204769' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:12:43.687 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:12:43 smithi083 ceph-mon[79930]: pgmap v1336: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:43.687 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:12:43 smithi083 ceph-mon[79930]: from='client.35691 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:12:43.688 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:12:43 smithi083 ceph-mon[79930]: from='client.45631 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:12:43.688 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:12:43 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2045204769' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:12:43.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:12:43 smithi153 ceph-mon[70044]: pgmap v1336: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:43.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:12:43 smithi153 ceph-mon[70044]: from='client.35691 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:12:43.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:12:43 smithi153 ceph-mon[70044]: from='client.45631 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:12:43.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:12:43 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2045204769' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:12:43.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:12:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:12:43.684Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:12:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:12:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:12:44.381Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:12:45.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:12:45 smithi083 ceph-mon[79930]: pgmap v1337: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:45.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:12:45 smithi083 ceph-mon[77242]: pgmap v1337: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:45.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:12:45 smithi153 ceph-mon[70044]: pgmap v1337: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:47.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:12:47 smithi083 ceph-mon[79930]: pgmap v1338: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:47.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:12:47 smithi083 ceph-mon[77242]: pgmap v1338: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:47.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:12:47 smithi153 ceph-mon[70044]: pgmap v1338: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:49.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:12:49 smithi083 ceph-mon[79930]: pgmap v1339: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:49.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:12:49 smithi083 ceph-mon[77242]: pgmap v1339: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:49.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:12:49 smithi153 ceph-mon[70044]: pgmap v1339: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:50.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:12:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:12:50.272Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:12:51.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:12:51 smithi083 ceph-mon[79930]: pgmap v1340: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:51.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:12:51 smithi083 ceph-mon[77242]: pgmap v1340: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:51.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:12:51 smithi153 ceph-mon[70044]: pgmap v1340: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:12:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:12:51] "GET /metrics HTTP/1.1" 200 225168 "" "Prometheus/2.33.4" 2022-09-29T06:12:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:12:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:12:52.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:12:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:12:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:12:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:12:53.687 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:12:53 smithi083 ceph-mon[79930]: pgmap v1341: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:53.687 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:12:53 smithi083 ceph-mon[77242]: pgmap v1341: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:53.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:12:53 smithi153 ceph-mon[70044]: pgmap v1341: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:53.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:12:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:12:53.685Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:12:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:12:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:12:54.380Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:12:55.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:12:55 smithi083 ceph-mon[79930]: pgmap v1342: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:55.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:12:55 smithi083 ceph-mon[77242]: pgmap v1342: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:55.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:12:55 smithi153 ceph-mon[70044]: pgmap v1342: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:57.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:12:57 smithi083 ceph-mon[79930]: pgmap v1343: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:57.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:12:57 smithi083 ceph-mon[77242]: pgmap v1343: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:57.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:12:57 smithi153 ceph-mon[70044]: pgmap v1343: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:59.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:12:59 smithi083 ceph-mon[79930]: pgmap v1344: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:59.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:12:59 smithi083 ceph-mon[77242]: pgmap v1344: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:12:59.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:12:59 smithi153 ceph-mon[70044]: pgmap v1344: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:00.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:13:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:13:00.273Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:13:01.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:13:01 smithi083 ceph-mon[79930]: pgmap v1345: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:01.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:13:01 smithi083 ceph-mon[77242]: pgmap v1345: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:01.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:13:01 smithi153 ceph-mon[70044]: pgmap v1345: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:13:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:13:01] "GET /metrics HTTP/1.1" 200 225173 "" "Prometheus/2.33.4" 2022-09-29T06:13:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:13:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:13:02.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:13:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:13:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:13:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:13:03.687 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:13:03 smithi083 ceph-mon[79930]: pgmap v1346: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:03.688 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:13:03 smithi083 ceph-mon[77242]: pgmap v1346: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:03.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:13:03 smithi153 ceph-mon[70044]: pgmap v1346: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:03.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:13:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:13:03.686Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:13:04.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:13:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:13:04.381Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:13:05.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:13:05 smithi083 ceph-mon[77242]: pgmap v1347: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:05.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:13:05 smithi083 ceph-mon[79930]: pgmap v1347: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:05.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:13:05 smithi153 ceph-mon[70044]: pgmap v1347: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:07.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:13:07 smithi083 ceph-mon[79930]: pgmap v1348: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:07.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:13:07 smithi083 ceph-mon[77242]: pgmap v1348: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:07.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:13:07 smithi153 ceph-mon[70044]: pgmap v1348: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:09.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:13:09 smithi083 ceph-mon[79930]: pgmap v1349: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:09.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:13:09 smithi083 ceph-mon[77242]: pgmap v1349: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:09.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:13:09 smithi153 ceph-mon[70044]: pgmap v1349: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:10.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:13:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:13:10.273Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:13:11.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:13:11 smithi083 ceph-mon[77242]: pgmap v1350: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:11.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:13:11 smithi083 ceph-mon[79930]: pgmap v1350: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:11.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:13:11 smithi153 ceph-mon[70044]: pgmap v1350: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:13:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:13:11] "GET /metrics HTTP/1.1" 200 225174 "" "Prometheus/2.33.4" 2022-09-29T06:13:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:13:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:13:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:13:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:13:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:13:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:13:13.363 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:13:13.659 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:13:13.660 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (46m) 9m ago 47m 26.7M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:13:13.660 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (47m) 7m ago 47m 61.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:13:13.660 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (48m) 9m ago 48m 21.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:13:13.661 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (48m) 7m ago 48m 27.6M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:13:13.661 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (43m) 7m ago 50m 439M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:13:13.661 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (44m) 9m ago 52m 538M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:13:13.661 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (43m) 9m ago 52m 204M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:13:13.662 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (43m) 7m ago 51m 101M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:13:13.662 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (43m) 9m ago 51m 124M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:13:13.662 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (47m) 9m ago 47m 25.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:13:13.662 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (47m) 7m ago 47m 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:13:13.663 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (42m) 9m ago 50m 158M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:13:13.663 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (41m) 9m ago 50m 150M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:13:13.663 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (41m) 9m ago 49m 124M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:13:13.663 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (40m) 9m ago 49m 141M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:13:13.663 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (40m) 7m ago 49m 133M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:13:13.664 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (39m) 7m ago 49m 152M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:13:13.664 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (39m) 7m ago 48m 135M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:13:13.664 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (38m) 7m ago 48m 150M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:13:13.664 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (44m) 7m ago 47m 100M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:13:13.687 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:13:13 smithi083 ceph-mon[79930]: pgmap v1351: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:13.687 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:13:13 smithi083 ceph-mon[77242]: pgmap v1351: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:13.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:13:13 smithi153 ceph-mon[70044]: pgmap v1351: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:13.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:13:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:13:13.685Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:13:14.018 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:13:14.018 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:13:14.018 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:13:14.019 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:13:14.019 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:13:14.019 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:13:14.019 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:13:14.019 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:13:14.020 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:13:14.020 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:13:14.020 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:13:14.020 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:13:14.021 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:13:14.021 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:13:14.021 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:13:14.021 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:13:14.021 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:13:14.022 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:13:14.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:13:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:13:14.381Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:13:14.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:13:14 smithi083 ceph-mon[77242]: from='client.45643 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:13:14.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:13:14 smithi083 ceph-mon[77242]: from='client.45649 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:13:14.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:13:14 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2845317612' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:13:14.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:13:14 smithi083 ceph-mon[79930]: from='client.45643 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:13:14.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:13:14 smithi083 ceph-mon[79930]: from='client.45649 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:13:14.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:13:14 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2845317612' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:13:14.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:13:14 smithi153 ceph-mon[70044]: from='client.45643 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:13:14.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:13:14 smithi153 ceph-mon[70044]: from='client.45649 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:13:14.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:13:14 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2845317612' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:13:15.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:13:15 smithi083 ceph-mon[79930]: pgmap v1352: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:15.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:13:15 smithi083 ceph-mon[77242]: pgmap v1352: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:15.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:13:15 smithi153 ceph-mon[70044]: pgmap v1352: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:17.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:13:17 smithi083 ceph-mon[77242]: pgmap v1353: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:17.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:13:17 smithi083 ceph-mon[79930]: pgmap v1353: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:17.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:13:17 smithi153 ceph-mon[70044]: pgmap v1353: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:19.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:13:19 smithi083 ceph-mon[79930]: pgmap v1354: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:19.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:13:19 smithi083 ceph-mon[77242]: pgmap v1354: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:19.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:13:19 smithi153 ceph-mon[70044]: pgmap v1354: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:20.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:13:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:13:20.273Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:13:21.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:13:21 smithi083 ceph-mon[79930]: pgmap v1355: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:21.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:13:21 smithi083 ceph-mon[77242]: pgmap v1355: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:21.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:13:21 smithi153 ceph-mon[70044]: pgmap v1355: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:13:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:13:21] "GET /metrics HTTP/1.1" 200 225174 "" "Prometheus/2.33.4" 2022-09-29T06:13:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:13:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:13:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:13:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:13:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:13:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:13:23.689 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:13:23 smithi083 ceph-mon[79930]: pgmap v1356: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:23.689 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:13:23 smithi083 ceph-mon[77242]: pgmap v1356: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:23.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:13:23 smithi153 ceph-mon[70044]: pgmap v1356: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:23.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:13:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:13:23.687Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:13:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:13:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:13:24.382Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:13:25.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:13:25 smithi083 ceph-mon[79930]: pgmap v1357: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:25.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:13:25 smithi083 ceph-mon[77242]: pgmap v1357: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:25.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:13:25 smithi153 ceph-mon[70044]: pgmap v1357: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:27.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:13:27 smithi083 ceph-mon[79930]: pgmap v1358: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:27.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:13:27 smithi083 ceph-mon[77242]: pgmap v1358: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:27.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:13:27 smithi153 ceph-mon[70044]: pgmap v1358: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:29.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:13:29 smithi083 ceph-mon[79930]: pgmap v1359: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:29.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:13:29 smithi083 ceph-mon[77242]: pgmap v1359: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:29.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:13:29 smithi153 ceph-mon[70044]: pgmap v1359: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:30.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:13:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:13:30.274Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:13:31.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:13:31 smithi083 ceph-mon[79930]: pgmap v1360: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:31.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:13:31 smithi083 ceph-mon[77242]: pgmap v1360: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:31.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:13:31 smithi153 ceph-mon[70044]: pgmap v1360: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:13:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:13:31] "GET /metrics HTTP/1.1" 200 225170 "" "Prometheus/2.33.4" 2022-09-29T06:13:32.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:13:32 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:13:32.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:13:32 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:13:32.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:13:32 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:13:32.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:13:32 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:13:32.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:13:32 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:13:32.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:13:32 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:13:32.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:13:32 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:13:32.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:13:32 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:13:32.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:13:32 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:13:32.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:13:32 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:13:32.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:13:32 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:13:32.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:13:32 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:13:32.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:13:32 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:13:32.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:13:32 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:13:32.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:13:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:13:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:13:32.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:13:32 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:13:32.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:13:32 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:13:32.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:13:32 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:13:32.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:13:32 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:13:32.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:13:32 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:13:32.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:13:32 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:13:32.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:13:32 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:13:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:13:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:13:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:13:33.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:13:33 smithi153 ceph-mon[70044]: pgmap v1361: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:33.689 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:13:33 smithi083 ceph-mon[79930]: pgmap v1361: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:33.690 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:13:33 smithi083 ceph-mon[77242]: pgmap v1361: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:33.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:13:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:13:33.687Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:13:34.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:13:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:13:34.382Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:13:35.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:13:35 smithi083 ceph-mon[79930]: pgmap v1362: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:35.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:13:35 smithi083 ceph-mon[77242]: pgmap v1362: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:35.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:13:35 smithi153 ceph-mon[70044]: pgmap v1362: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:37.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:13:37 smithi083 ceph-mon[79930]: pgmap v1363: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:37.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:13:37 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:13:37.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:13:37 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:13:37.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:13:37 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:13:37.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:13:37 smithi083 ceph-mon[77242]: pgmap v1363: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:37.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:13:37 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:13:37.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:13:37 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:13:37.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:13:37 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:13:37.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:13:37 smithi153 ceph-mon[70044]: pgmap v1363: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:37.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:13:37 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:13:37.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:13:37 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:13:37.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:13:37 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:13:39.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:13:39 smithi083 ceph-mon[79930]: pgmap v1364: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:39.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:13:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:13:39.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:13:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:13:39.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:13:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:13:39.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:13:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:13:39.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:13:39 smithi083 ceph-mon[77242]: pgmap v1364: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:39.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:13:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:13:39.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:13:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:13:39.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:13:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:13:39.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:13:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:13:39.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:13:39 smithi153 ceph-mon[70044]: pgmap v1364: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:39.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:13:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:13:39.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:13:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:13:39.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:13:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:13:39.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:13:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:13:40.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:13:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:13:40.275Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:13:41.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:13:41 smithi083 ceph-mon[79930]: pgmap v1365: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:41.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:13:41 smithi083 ceph-mon[77242]: pgmap v1365: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:13:41 smithi153 ceph-mon[70044]: pgmap v1365: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:13:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:13:41] "GET /metrics HTTP/1.1" 200 225171 "" "Prometheus/2.33.4" 2022-09-29T06:13:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:13:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:13:42.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:13:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:13:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:13:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:13:43.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:13:43 smithi083 ceph-mon[79930]: pgmap v1366: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:43.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:13:43 smithi083 ceph-mon[77242]: pgmap v1366: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:43.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:13:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:13:43.688Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:13:43.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:13:43 smithi153 ceph-mon[70044]: pgmap v1366: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:44.345 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:13:44.635 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:13:44.636 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (47m) 8s ago 47m 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:13:44.636 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (47m) 8m ago 47m 61.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:13:44.636 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (48m) 8s ago 48m 23.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:13:44.636 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (48m) 8m ago 48m 27.6M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:13:44.637 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (44m) 8m ago 51m 439M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:13:44.637 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (44m) 8s ago 52m 550M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:13:44.637 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (44m) 8s ago 52m 241M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:13:44.637 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (43m) 8m ago 51m 101M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:13:44.638 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (43m) 8s ago 51m 144M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:13:44.638 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (48m) 8s ago 48m 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:13:44.638 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (48m) 8m ago 48m 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:13:44.638 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (42m) 8s ago 51m 183M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:13:44.639 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (42m) 8s ago 50m 176M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:13:44.639 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (42m) 8s ago 50m 144M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:13:44.639 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (41m) 8s ago 50m 166M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:13:44.639 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (40m) 8m ago 49m 133M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:13:44.640 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (40m) 8m ago 49m 152M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:13:44.640 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (39m) 8m ago 49m 135M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:13:44.640 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (39m) 8m ago 49m 150M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:13:44.640 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (44m) 8m ago 48m 100M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:13:44.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:13:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:13:44.382Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:13:44.994 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:13:44.994 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:13:44.995 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:13:44.995 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:13:44.995 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:13:44.995 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:13:44.996 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:13:44.996 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:13:44.996 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:13:44.996 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:13:44.997 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:13:44.997 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:13:44.997 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:13:44.997 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:13:44.998 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:13:44.998 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:13:44.998 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:13:44.998 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:13:45.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:13:45 smithi083 ceph-mon[77242]: from='client.35727 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:13:45.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:13:45 smithi083 ceph-mon[77242]: pgmap v1367: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:45.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:13:45 smithi083 ceph-mon[77242]: from='client.45667 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:13:45.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:13:45 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/556784593' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:13:45.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:13:45 smithi083 ceph-mon[79930]: from='client.35727 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:13:45.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:13:45 smithi083 ceph-mon[79930]: pgmap v1367: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:45.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:13:45 smithi083 ceph-mon[79930]: from='client.45667 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:13:45.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:13:45 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/556784593' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:13:45.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:13:45 smithi153 ceph-mon[70044]: from='client.35727 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:13:45.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:13:45 smithi153 ceph-mon[70044]: pgmap v1367: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:45.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:13:45 smithi153 ceph-mon[70044]: from='client.45667 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:13:45.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:13:45 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/556784593' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:13:47.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:13:47 smithi083 ceph-mon[79930]: pgmap v1368: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:47.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:13:47 smithi083 ceph-mon[77242]: pgmap v1368: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:47.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:13:47 smithi153 ceph-mon[70044]: pgmap v1368: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:49.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:13:49 smithi083 ceph-mon[79930]: pgmap v1369: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:49.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:13:49 smithi083 ceph-mon[77242]: pgmap v1369: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:49.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:13:49 smithi153 ceph-mon[70044]: pgmap v1369: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:50.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:13:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:13:50.276Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:13:51.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:13:51 smithi083 ceph-mon[79930]: pgmap v1370: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:51.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:13:51 smithi083 ceph-mon[77242]: pgmap v1370: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:51.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:13:51 smithi153 ceph-mon[70044]: pgmap v1370: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:13:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:13:51] "GET /metrics HTTP/1.1" 200 225171 "" "Prometheus/2.33.4" 2022-09-29T06:13:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:13:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:13:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:13:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:13:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:13:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:13:53.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:13:53 smithi083 ceph-mon[79930]: pgmap v1371: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:53.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:13:53 smithi083 ceph-mon[77242]: pgmap v1371: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:53.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:13:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:13:53.689Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:13:53.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:13:53 smithi153 ceph-mon[70044]: pgmap v1371: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:13:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:13:54.384Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:13:55.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:13:55 smithi083 ceph-mon[79930]: pgmap v1372: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:55.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:13:55 smithi083 ceph-mon[77242]: pgmap v1372: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:55.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:13:55 smithi153 ceph-mon[70044]: pgmap v1372: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:57.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:13:57 smithi083 ceph-mon[79930]: pgmap v1373: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:57.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:13:57 smithi083 ceph-mon[77242]: pgmap v1373: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:57.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:13:57 smithi153 ceph-mon[70044]: pgmap v1373: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:59.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:13:59 smithi083 ceph-mon[79930]: pgmap v1374: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:59.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:13:59 smithi083 ceph-mon[77242]: pgmap v1374: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:13:59.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:13:59 smithi153 ceph-mon[70044]: pgmap v1374: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:00.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:14:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:14:00.276Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:14:01.507 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:14:01 smithi083 ceph-mon[79930]: pgmap v1375: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:01.508 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:14:01 smithi083 ceph-mon[77242]: pgmap v1375: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:01.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:14:01 smithi153 ceph-mon[70044]: pgmap v1375: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:14:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:14:01] "GET /metrics HTTP/1.1" 200 225173 "" "Prometheus/2.33.4" 2022-09-29T06:14:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:14:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:14:02.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:14:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:14:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:14:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:14:03.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:14:03 smithi083 ceph-mon[79930]: pgmap v1376: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:03.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:14:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:14:03.690Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:14:03.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:14:03 smithi083 ceph-mon[77242]: pgmap v1376: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:03.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:14:03 smithi153 ceph-mon[70044]: pgmap v1376: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:04.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:14:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:14:04.385Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:14:05.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:14:05 smithi083 ceph-mon[77242]: pgmap v1377: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:05.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:14:05 smithi083 ceph-mon[79930]: pgmap v1377: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:05.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:14:05 smithi153 ceph-mon[70044]: pgmap v1377: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:07.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:14:07 smithi083 ceph-mon[79930]: pgmap v1378: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:07.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:14:07 smithi083 ceph-mon[77242]: pgmap v1378: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:07.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:14:07 smithi153 ceph-mon[70044]: pgmap v1378: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:09.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:14:09 smithi083 ceph-mon[79930]: pgmap v1379: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:09.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:14:09 smithi083 ceph-mon[77242]: pgmap v1379: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:09.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:14:09 smithi153 ceph-mon[70044]: pgmap v1379: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:10.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:14:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:14:10.277Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:14:11.508 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:14:11 smithi083 ceph-mon[79930]: pgmap v1380: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:11.508 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:14:11 smithi083 ceph-mon[77242]: pgmap v1380: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:11.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:14:11 smithi153 ceph-mon[70044]: pgmap v1380: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:14:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:14:11] "GET /metrics HTTP/1.1" 200 225169 "" "Prometheus/2.33.4" 2022-09-29T06:14:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:14:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:14:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:14:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:14:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:14:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:14:13.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:14:13 smithi083 ceph-mon[79930]: pgmap v1381: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:13.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:14:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:14:13.690Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:14:13.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:14:13 smithi083 ceph-mon[77242]: pgmap v1381: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:13.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:14:13 smithi153 ceph-mon[70044]: pgmap v1381: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:14.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:14:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:14:14.385Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:14:15.328 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:14:15.617 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:14:15.618 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (47m) 39s ago 48m 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:14:15.618 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (48m) 8m ago 48m 61.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:14:15.619 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (49m) 39s ago 49m 23.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:14:15.619 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (49m) 8m ago 49m 27.6M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:14:15.619 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (45m) 8m ago 51m 439M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:14:15.619 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (45m) 39s ago 53m 550M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:14:15.619 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (44m) 39s ago 53m 241M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:14:15.620 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (44m) 8m ago 52m 101M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:14:15.620 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (44m) 39s ago 52m 144M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:14:15.620 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (48m) 39s ago 48m 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:14:15.620 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (48m) 8m ago 48m 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:14:15.621 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (43m) 39s ago 51m 183M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:14:15.621 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (42m) 39s ago 51m 176M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:14:15.621 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (42m) 39s ago 50m 144M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:14:15.621 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (42m) 39s ago 50m 166M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:14:15.622 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (41m) 8m ago 50m 133M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:14:15.622 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (40m) 8m ago 50m 152M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:14:15.622 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (40m) 8m ago 49m 135M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:14:15.622 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (39m) 8m ago 49m 150M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:14:15.623 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (45m) 8m ago 48m 100M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:14:15.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:14:15 smithi083 ceph-mon[79930]: pgmap v1382: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:15.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:14:15 smithi083 ceph-mon[77242]: pgmap v1382: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:15.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:14:15 smithi153 ceph-mon[70044]: pgmap v1382: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:15.976 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:14:15.977 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:14:15.977 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:14:15.978 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:14:15.978 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:14:15.978 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:14:15.978 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:14:15.979 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:14:15.979 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:14:15.979 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:14:15.979 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:14:15.980 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:14:15.980 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:14:15.980 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:14:15.980 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:14:15.981 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:14:15.981 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:14:15.981 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:14:16.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:14:16 smithi153 ceph-mon[70044]: from='client.35745 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:14:16.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:14:16 smithi153 ceph-mon[70044]: from='client.45685 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:14:16.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:14:16 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1380941372' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:14:16.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:14:16 smithi083 ceph-mon[79930]: from='client.35745 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:14:16.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:14:16 smithi083 ceph-mon[79930]: from='client.45685 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:14:16.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:14:16 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1380941372' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:14:16.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:14:16 smithi083 ceph-mon[77242]: from='client.35745 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:14:16.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:14:16 smithi083 ceph-mon[77242]: from='client.45685 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:14:16.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:14:16 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1380941372' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:14:17.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:14:17 smithi153 ceph-mon[70044]: pgmap v1383: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:17.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:14:17 smithi083 ceph-mon[79930]: pgmap v1383: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:17.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:14:17 smithi083 ceph-mon[77242]: pgmap v1383: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:19.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:14:19 smithi153 ceph-mon[70044]: pgmap v1384: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:19.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:14:19 smithi083 ceph-mon[79930]: pgmap v1384: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:19.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:14:19 smithi083 ceph-mon[77242]: pgmap v1384: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:20.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:14:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:14:20.278Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:14:21.508 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:14:21 smithi083 ceph-mon[79930]: pgmap v1385: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:21.508 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:14:21 smithi083 ceph-mon[77242]: pgmap v1385: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:21.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:14:21 smithi153 ceph-mon[70044]: pgmap v1385: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:14:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:14:21] "GET /metrics HTTP/1.1" 200 225169 "" "Prometheus/2.33.4" 2022-09-29T06:14:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:14:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:14:22.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:14:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:14:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:14:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:14:23.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:14:23 smithi083 ceph-mon[77242]: pgmap v1386: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:23.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:14:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:14:23.691Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:14:23.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:14:23 smithi083 ceph-mon[79930]: pgmap v1386: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:23.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:14:23 smithi153 ceph-mon[70044]: pgmap v1386: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:14:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:14:24.385Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:14:25.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:14:25 smithi083 ceph-mon[77242]: pgmap v1387: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:25.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:14:25 smithi083 ceph-mon[79930]: pgmap v1387: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:25.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:14:25 smithi153 ceph-mon[70044]: pgmap v1387: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:27.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:14:27 smithi083 ceph-mon[77242]: pgmap v1388: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:27.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:14:27 smithi083 ceph-mon[79930]: pgmap v1388: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:27.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:14:27 smithi153 ceph-mon[70044]: pgmap v1388: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:29.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:14:29 smithi083 ceph-mon[77242]: pgmap v1389: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:29.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:14:29 smithi083 ceph-mon[79930]: pgmap v1389: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:29.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:14:29 smithi153 ceph-mon[70044]: pgmap v1389: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:30.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:14:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:14:30.279Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:14:31.508 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:14:31 smithi083 ceph-mon[79930]: pgmap v1390: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:31.508 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:14:31 smithi083 ceph-mon[77242]: pgmap v1390: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:31.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:14:31 smithi153 ceph-mon[70044]: pgmap v1390: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:14:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:14:31] "GET /metrics HTTP/1.1" 200 225170 "" "Prometheus/2.33.4" 2022-09-29T06:14:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:14:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:14:32.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:14:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:14:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:14:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:14:33.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:14:33 smithi083 ceph-mon[79930]: pgmap v1391: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:33.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:14:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:14:33.692Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:14:33.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:14:33 smithi083 ceph-mon[77242]: pgmap v1391: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:33.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:14:33 smithi153 ceph-mon[70044]: pgmap v1391: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:34.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:14:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:14:34.386Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:14:35.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:14:35 smithi083 ceph-mon[77242]: pgmap v1392: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:35.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:14:35 smithi083 ceph-mon[79930]: pgmap v1392: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:35.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:14:35 smithi153 ceph-mon[70044]: pgmap v1392: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:37.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:14:37 smithi153 ceph-mon[70044]: pgmap v1393: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:37.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:14:37 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:14:37.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:14:37 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:14:37.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:14:37 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:14:37.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:14:37 smithi083 ceph-mon[79930]: pgmap v1393: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:37.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:14:37 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:14:37.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:14:37 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:14:37.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:14:37 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:14:37.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:14:37 smithi083 ceph-mon[77242]: pgmap v1393: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:37.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:14:37 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:14:37.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:14:37 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:14:37.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:14:37 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:14:38.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:14:38 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:14:38.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:14:38 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:14:39.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:14:38 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:14:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:14:39 smithi083 ceph-mon[79930]: pgmap v1394: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:14:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:14:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:14:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:14:39.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:14:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:14:39.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:14:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:14:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:14:39 smithi083 ceph-mon[77242]: pgmap v1394: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:39.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:14:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:14:39.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:14:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:14:39.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:14:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:14:39.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:14:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:14:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:14:39 smithi153 ceph-mon[70044]: pgmap v1394: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:14:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:14:40.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:14:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:14:40.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:14:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:14:40.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:14:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:14:40.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:14:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:14:40.280Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:14:41.507 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:14:41 smithi083 ceph-mon[77242]: pgmap v1395: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:41.508 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:14:41 smithi083 ceph-mon[79930]: pgmap v1395: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:14:41 smithi153 ceph-mon[70044]: pgmap v1395: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:14:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:14:41] "GET /metrics HTTP/1.1" 200 225176 "" "Prometheus/2.33.4" 2022-09-29T06:14:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:14:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:14:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:14:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:14:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:14:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:14:43.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:14:43 smithi083 ceph-mon[77242]: pgmap v1396: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:43.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:14:43 smithi083 ceph-mon[79930]: pgmap v1396: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:43.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:14:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:14:43.691Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:14:43.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:14:43 smithi153 ceph-mon[70044]: pgmap v1396: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:14:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:14:44.386Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:14:45.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:14:45 smithi083 ceph-mon[79930]: pgmap v1397: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:45.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:14:45 smithi083 ceph-mon[77242]: pgmap v1397: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:45.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:14:45 smithi153 ceph-mon[70044]: pgmap v1397: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:46.304 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:14:46.592 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:14:46.593 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (48m) 70s ago 48m 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:14:46.593 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (48m) 9m ago 48m 61.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:14:46.593 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (49m) 70s ago 49m 23.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:14:46.593 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (49m) 9m ago 49m 27.6M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:14:46.594 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (45m) 9m ago 52m 439M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:14:46.594 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (45m) 70s ago 53m 550M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:14:46.594 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (45m) 70s ago 54m 241M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:14:46.594 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (44m) 9m ago 52m 101M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:14:46.595 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (44m) 70s ago 52m 144M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:14:46.595 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (49m) 70s ago 49m 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:14:46.595 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (49m) 9m ago 49m 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:14:46.595 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (43m) 70s ago 52m 183M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:14:46.596 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (43m) 70s ago 51m 176M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:14:46.596 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (43m) 70s ago 51m 144M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:14:46.596 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (42m) 70s ago 51m 166M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:14:46.596 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (41m) 9m ago 50m 133M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:14:46.596 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (41m) 9m ago 50m 152M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:14:46.597 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (40m) 9m ago 50m 135M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:14:46.597 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (40m) 9m ago 50m 150M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:14:46.597 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (45m) 9m ago 49m 100M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:14:46.946 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:14:46.946 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:14:46.946 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:14:46.947 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:14:46.947 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:14:46.947 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:14:46.948 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:14:46.948 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:14:46.948 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:14:46.948 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:14:46.949 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:14:46.949 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:14:46.949 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:14:46.949 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:14:46.949 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:14:46.950 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:14:46.950 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:14:46.950 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:14:47.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:14:47 smithi153 ceph-mon[70044]: from='client.35763 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:14:47.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:14:47 smithi153 ceph-mon[70044]: pgmap v1398: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:47.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:14:47 smithi153 ceph-mon[70044]: from='client.45703 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:14:47.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:14:47 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4226677573' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:14:47.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:14:47 smithi083 ceph-mon[79930]: from='client.35763 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:14:47.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:14:47 smithi083 ceph-mon[79930]: pgmap v1398: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:47.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:14:47 smithi083 ceph-mon[79930]: from='client.45703 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:14:47.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:14:47 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4226677573' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:14:47.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:14:47 smithi083 ceph-mon[77242]: from='client.35763 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:14:47.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:14:47 smithi083 ceph-mon[77242]: pgmap v1398: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:47.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:14:47 smithi083 ceph-mon[77242]: from='client.45703 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:14:47.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:14:47 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4226677573' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:14:49.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:14:49 smithi153 ceph-mon[70044]: pgmap v1399: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:49.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:14:49 smithi083 ceph-mon[79930]: pgmap v1399: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:49.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:14:49 smithi083 ceph-mon[77242]: pgmap v1399: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:50.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:14:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:14:50.280Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:14:51.508 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:14:51 smithi083 ceph-mon[79930]: pgmap v1400: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:51.508 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:14:51 smithi083 ceph-mon[77242]: pgmap v1400: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:51.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:14:51 smithi153 ceph-mon[70044]: pgmap v1400: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:14:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:14:51] "GET /metrics HTTP/1.1" 200 225176 "" "Prometheus/2.33.4" 2022-09-29T06:14:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:14:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:14:52.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:14:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:14:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:14:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:14:53.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:14:53 smithi083 ceph-mon[79930]: pgmap v1401: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:53.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:14:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:14:53.692Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:14:53.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:14:53 smithi083 ceph-mon[77242]: pgmap v1401: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:53.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:14:53 smithi153 ceph-mon[70044]: pgmap v1401: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:14:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:14:54.387Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:14:55.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:14:55 smithi153 ceph-mon[70044]: pgmap v1402: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:55.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:14:55 smithi083 ceph-mon[77242]: pgmap v1402: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:55.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:14:55 smithi083 ceph-mon[79930]: pgmap v1402: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:57.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:14:57 smithi153 ceph-mon[70044]: pgmap v1403: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:57.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:14:57 smithi083 ceph-mon[77242]: pgmap v1403: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:57.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:14:57 smithi083 ceph-mon[79930]: pgmap v1403: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:59.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:14:59 smithi153 ceph-mon[70044]: pgmap v1404: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:59.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:14:59 smithi083 ceph-mon[77242]: pgmap v1404: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:14:59.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:14:59 smithi083 ceph-mon[79930]: pgmap v1404: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:00.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:15:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:15:00.281Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:15:01.508 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:15:01 smithi083 ceph-mon[77242]: pgmap v1405: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:01.508 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:15:01 smithi083 ceph-mon[79930]: pgmap v1405: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:01.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:15:01 smithi153 ceph-mon[70044]: pgmap v1405: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:15:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:15:01] "GET /metrics HTTP/1.1" 200 225170 "" "Prometheus/2.33.4" 2022-09-29T06:15:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:15:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:15:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:15:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:15:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:15:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:15:03.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:15:03 smithi153 ceph-mon[70044]: pgmap v1406: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:03.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:15:03 smithi083 ceph-mon[79930]: pgmap v1406: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:03.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:15:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:15:03.692Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:15:03.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:15:03 smithi083 ceph-mon[77242]: pgmap v1406: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:04.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:15:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:15:04.388Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:15:05.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:15:05 smithi153 ceph-mon[70044]: pgmap v1407: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:05.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:15:05 smithi083 ceph-mon[77242]: pgmap v1407: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:05.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:15:05 smithi083 ceph-mon[79930]: pgmap v1407: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:07.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:15:07 smithi153 ceph-mon[70044]: pgmap v1408: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:07.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:15:07 smithi083 ceph-mon[79930]: pgmap v1408: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:07.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:15:07 smithi083 ceph-mon[77242]: pgmap v1408: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:09.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:15:09 smithi153 ceph-mon[70044]: pgmap v1409: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:09.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:15:09 smithi083 ceph-mon[79930]: pgmap v1409: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:09.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:15:09 smithi083 ceph-mon[77242]: pgmap v1409: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:10.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:15:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:15:10.282Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:15:11.508 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:15:11 smithi083 ceph-mon[79930]: pgmap v1410: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:11.508 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:15:11 smithi083 ceph-mon[77242]: pgmap v1410: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:11.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:15:11 smithi153 ceph-mon[70044]: pgmap v1410: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:15:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:15:11] "GET /metrics HTTP/1.1" 200 225173 "" "Prometheus/2.33.4" 2022-09-29T06:15:12.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:15:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:15:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:15:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:15:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:15:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:15:13.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:15:13 smithi153 ceph-mon[70044]: pgmap v1411: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:13.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:15:13 smithi083 ceph-mon[79930]: pgmap v1411: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:13.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:15:13 smithi083 ceph-mon[77242]: pgmap v1411: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:13.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:15:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:15:13.693Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:15:14.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:15:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:15:14.389Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:15:15.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:15:15 smithi153 ceph-mon[70044]: pgmap v1412: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:15.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:15:15 smithi083 ceph-mon[79930]: pgmap v1412: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:15.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:15:15 smithi083 ceph-mon[77242]: pgmap v1412: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:17.272 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:15:17.562 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:15:17.563 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (49m) 101s ago 49m 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:15:17.563 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (49m) 9m ago 49m 61.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:15:17.563 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (50m) 101s ago 50m 23.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:15:17.563 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (50m) 9m ago 50m 27.6M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:15:17.564 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (46m) 9m ago 52m 439M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:15:17.564 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (46m) 101s ago 54m 550M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:15:17.564 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (45m) 101s ago 54m 241M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:15:17.564 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (45m) 9m ago 53m 101M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:15:17.564 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (45m) 101s ago 53m 144M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:15:17.565 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (49m) 101s ago 49m 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:15:17.565 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (49m) 9m ago 49m 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:15:17.565 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (44m) 101s ago 52m 183M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:15:17.565 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (43m) 101s ago 52m 176M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:15:17.566 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (43m) 101s ago 52m 144M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:15:17.566 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (43m) 101s ago 51m 166M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:15:17.566 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (42m) 9m ago 51m 133M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:15:17.566 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (41m) 9m ago 51m 152M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:15:17.567 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (41m) 9m ago 50m 135M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:15:17.567 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (40m) 9m ago 50m 150M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:15:17.567 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (46m) 9m ago 49m 100M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:15:17.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:15:17 smithi153 ceph-mon[70044]: pgmap v1413: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:17.919 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:15:17.920 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:15:17.920 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:15:17.920 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:15:17.920 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:15:17.921 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:15:17.921 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:15:17.921 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:15:17.921 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:15:17.922 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:15:17.922 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:15:17.922 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:15:17.922 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:15:17.922 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:15:17.923 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:15:17.923 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:15:17.923 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:15:17.923 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:15:17.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:15:17 smithi083 ceph-mon[79930]: pgmap v1413: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:17.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:15:17 smithi083 ceph-mon[77242]: pgmap v1413: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:18.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:15:18 smithi153 ceph-mon[70044]: from='client.35781 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:15:18.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:15:18 smithi153 ceph-mon[70044]: from='client.35787 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:15:18.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:15:18 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1801221622' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:15:18.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:15:18 smithi083 ceph-mon[77242]: from='client.35781 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:15:18.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:15:18 smithi083 ceph-mon[77242]: from='client.35787 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:15:18.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:15:18 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1801221622' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:15:18.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:15:18 smithi083 ceph-mon[79930]: from='client.35781 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:15:18.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:15:18 smithi083 ceph-mon[79930]: from='client.35787 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:15:18.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:15:18 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1801221622' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:15:19.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:15:19 smithi153 ceph-mon[70044]: pgmap v1414: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:19.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:15:19 smithi083 ceph-mon[79930]: pgmap v1414: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:19.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:15:19 smithi083 ceph-mon[77242]: pgmap v1414: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:20.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:15:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:15:20.283Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:15:21.507 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:15:21 smithi083 ceph-mon[79930]: pgmap v1415: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:21.508 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:15:21 smithi083 ceph-mon[77242]: pgmap v1415: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:21.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:15:21 smithi153 ceph-mon[70044]: pgmap v1415: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:15:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:15:21] "GET /metrics HTTP/1.1" 200 225173 "" "Prometheus/2.33.4" 2022-09-29T06:15:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:15:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:15:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:15:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:15:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:15:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:15:23.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:15:23 smithi153 ceph-mon[70044]: pgmap v1416: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:23.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:15:23 smithi083 ceph-mon[77242]: pgmap v1416: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:23.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:15:23 smithi083 ceph-mon[79930]: pgmap v1416: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:23.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:15:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:15:23.693Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:15:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:15:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:15:24.389Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:15:25.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:15:25 smithi153 ceph-mon[70044]: pgmap v1417: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:25.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:15:25 smithi083 ceph-mon[77242]: pgmap v1417: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:25.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:15:25 smithi083 ceph-mon[79930]: pgmap v1417: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:27.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:15:27 smithi153 ceph-mon[70044]: pgmap v1418: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:27.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:15:27 smithi083 ceph-mon[79930]: pgmap v1418: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:27.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:15:27 smithi083 ceph-mon[77242]: pgmap v1418: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:29.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:15:29 smithi153 ceph-mon[70044]: pgmap v1419: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:29.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:15:29 smithi083 ceph-mon[79930]: pgmap v1419: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:29.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:15:29 smithi083 ceph-mon[77242]: pgmap v1419: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:30.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:15:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:15:30.284Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:15:31.508 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:15:31 smithi083 ceph-mon[79930]: pgmap v1420: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:31.509 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:15:31 smithi083 ceph-mon[77242]: pgmap v1420: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:31.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:15:31 smithi153 ceph-mon[70044]: pgmap v1420: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:15:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:15:31] "GET /metrics HTTP/1.1" 200 225170 "" "Prometheus/2.33.4" 2022-09-29T06:15:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:15:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:15:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:15:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:15:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:15:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:15:33.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:15:33 smithi153 ceph-mon[70044]: pgmap v1421: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:33.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:15:33 smithi083 ceph-mon[79930]: pgmap v1421: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:33.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:15:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:15:33.693Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:15:33.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:15:33 smithi083 ceph-mon[77242]: pgmap v1421: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:34.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:15:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:15:34.389Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:15:35.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:15:35 smithi153 ceph-mon[70044]: pgmap v1422: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:35.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:15:35 smithi083 ceph-mon[79930]: pgmap v1422: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:35.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:15:35 smithi083 ceph-mon[77242]: pgmap v1422: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:37.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:15:37 smithi153 ceph-mon[70044]: pgmap v1423: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:37.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:15:37 smithi083 ceph-mon[77242]: pgmap v1423: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:37.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:15:37 smithi083 ceph-mon[79930]: pgmap v1423: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:38.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:15:38 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:15:38.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:15:38 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:15:38.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:15:38 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:15:38.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:15:38 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:15:38.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:15:38 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:15:38.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:15:38 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:15:38.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:15:38 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:15:38.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:15:38 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:15:38.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:15:38 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:15:39.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:15:39 smithi153 ceph-mon[70044]: pgmap v1424: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:39.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:15:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:15:39.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:15:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:15:39.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:15:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:15:39.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:15:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:15:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:15:39 smithi083 ceph-mon[79930]: pgmap v1424: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:15:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:15:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:15:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:15:39.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:15:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:15:39.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:15:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:15:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:15:39 smithi083 ceph-mon[77242]: pgmap v1424: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:39.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:15:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:15:39.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:15:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:15:39.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:15:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:15:39.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:15:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:15:40.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:15:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:15:40.285Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:15:41.508 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:15:41 smithi083 ceph-mon[79930]: pgmap v1425: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:41.509 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:15:41 smithi083 ceph-mon[77242]: pgmap v1425: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:15:41 smithi153 ceph-mon[70044]: pgmap v1425: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:15:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:15:41] "GET /metrics HTTP/1.1" 200 225171 "" "Prometheus/2.33.4" 2022-09-29T06:15:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:15:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:15:42.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:15:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:15:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:15:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:15:43.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:15:43 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:15:43.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:15:43 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:15:43.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:15:43 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:15:43.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:15:43 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:15:43.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:15:43 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:15:43.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:15:43 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:15:43.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:15:43 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:15:43.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:15:43 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:15:43.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:15:43 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:15:43.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:15:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:15:43.694Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:15:44.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:15:44 smithi083 ceph-mon[79930]: pgmap v1426: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:44.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:15:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:15:44.390Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:15:44.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:15:44 smithi083 ceph-mon[77242]: pgmap v1426: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:44.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:15:44 smithi153 ceph-mon[70044]: pgmap v1426: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:46.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:15:46 smithi083 ceph-mon[79930]: pgmap v1427: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:46.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:15:46 smithi083 ceph-mon[77242]: pgmap v1427: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:46.561 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:15:46 smithi153 ceph-mon[70044]: pgmap v1427: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:48.247 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:15:48.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:15:48 smithi083 ceph-mon[77242]: pgmap v1428: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:48.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:15:48 smithi083 ceph-mon[79930]: pgmap v1428: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:48.536 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:15:48.536 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (49m) 2m ago 49m 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:15:48.536 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (49m) 6s ago 49m 61.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:15:48.536 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (50m) 2m ago 50m 23.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:15:48.537 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (50m) 6s ago 50m 27.6M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:15:48.537 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (46m) 6s ago 53m 439M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:15:48.537 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (47m) 2m ago 54m 550M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:15:48.537 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (46m) 2m ago 55m 241M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:15:48.538 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (45m) 6s ago 53m 115M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:15:48.538 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (45m) 2m ago 53m 144M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:15:48.538 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (50m) 2m ago 50m 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:15:48.538 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (50m) 6s ago 50m 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:15:48.539 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (44m) 2m ago 53m 183M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:15:48.539 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (44m) 2m ago 52m 176M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:15:48.539 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (44m) 2m ago 52m 144M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:15:48.539 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (43m) 2m ago 52m 166M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:15:48.539 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (43m) 6s ago 51m 154M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:15:48.540 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (42m) 6s ago 51m 180M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:15:48.540 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (41m) 6s ago 51m 159M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:15:48.540 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (41m) 6s ago 51m 180M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:15:48.540 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (46m) 6s ago 50m 102M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:15:48.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:15:48 smithi153 ceph-mon[70044]: pgmap v1428: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:48.891 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:15:48.892 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:15:48.892 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:15:48.892 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:15:48.892 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:15:48.893 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:15:48.893 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:15:48.893 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:15:48.893 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:15:48.893 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:15:48.894 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:15:48.894 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:15:48.894 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:15:48.894 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:15:48.895 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:15:48.895 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:15:48.895 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:15:48.895 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:15:49.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:15:49 smithi083 ceph-mon[77242]: from='client.35799 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:15:49.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:15:49 smithi083 ceph-mon[77242]: pgmap v1429: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:49.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:15:49 smithi083 ceph-mon[77242]: from='client.45739 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:15:49.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:15:49 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4261328692' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:15:49.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:15:49 smithi083 ceph-mon[79930]: from='client.35799 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:15:49.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:15:49 smithi083 ceph-mon[79930]: pgmap v1429: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:49.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:15:49 smithi083 ceph-mon[79930]: from='client.45739 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:15:49.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:15:49 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4261328692' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:15:49.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:15:49 smithi153 ceph-mon[70044]: from='client.35799 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:15:49.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:15:49 smithi153 ceph-mon[70044]: pgmap v1429: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:49.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:15:49 smithi153 ceph-mon[70044]: from='client.45739 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:15:49.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:15:49 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4261328692' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:15:50.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:15:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:15:50.285Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:15:51.508 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:15:51 smithi083 ceph-mon[79930]: pgmap v1430: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:51.508 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:15:51 smithi083 ceph-mon[77242]: pgmap v1430: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:51.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:15:51 smithi153 ceph-mon[70044]: pgmap v1430: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:15:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:15:51] "GET /metrics HTTP/1.1" 200 225171 "" "Prometheus/2.33.4" 2022-09-29T06:15:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:15:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:15:52.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:15:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:15:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:15:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:15:53.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:15:53 smithi153 ceph-mon[70044]: pgmap v1431: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:53.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:15:53 smithi083 ceph-mon[79930]: pgmap v1431: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:53.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:15:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:15:53.695Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:15:53.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:15:53 smithi083 ceph-mon[77242]: pgmap v1431: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:54.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:15:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:15:54.390Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:15:55.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:15:55 smithi153 ceph-mon[70044]: pgmap v1432: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:55.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:15:55 smithi083 ceph-mon[79930]: pgmap v1432: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:55.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:15:55 smithi083 ceph-mon[77242]: pgmap v1432: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:57.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:15:57 smithi153 ceph-mon[70044]: pgmap v1433: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:57.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:15:57 smithi083 ceph-mon[79930]: pgmap v1433: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:57.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:15:57 smithi083 ceph-mon[77242]: pgmap v1433: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:59.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:15:59 smithi153 ceph-mon[70044]: pgmap v1434: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:59.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:15:59 smithi083 ceph-mon[77242]: pgmap v1434: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:15:59.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:15:59 smithi083 ceph-mon[79930]: pgmap v1434: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:00.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:16:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:16:00.285Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:16:01.508 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:16:01 smithi083 ceph-mon[79930]: pgmap v1435: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:01.508 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:16:01 smithi083 ceph-mon[77242]: pgmap v1435: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:01.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:16:01 smithi153 ceph-mon[70044]: pgmap v1435: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:16:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:16:01] "GET /metrics HTTP/1.1" 200 225174 "" "Prometheus/2.33.4" 2022-09-29T06:16:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:16:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:16:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:16:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:16:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:16:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:16:03.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:16:03 smithi153 ceph-mon[70044]: pgmap v1436: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:03.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:16:03 smithi083 ceph-mon[79930]: pgmap v1436: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:03.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:16:03 smithi083 ceph-mon[77242]: pgmap v1436: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:03.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:16:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:16:03.696Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:16:04.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:16:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:16:04.391Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:16:05.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:16:05 smithi153 ceph-mon[70044]: pgmap v1437: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:05.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:16:05 smithi083 ceph-mon[79930]: pgmap v1437: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:05.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:16:05 smithi083 ceph-mon[77242]: pgmap v1437: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:07.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:16:07 smithi153 ceph-mon[70044]: pgmap v1438: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:07.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:16:07 smithi083 ceph-mon[77242]: pgmap v1438: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:07.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:16:07 smithi083 ceph-mon[79930]: pgmap v1438: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:09.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:16:09 smithi153 ceph-mon[70044]: pgmap v1439: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:09.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:16:09 smithi083 ceph-mon[79930]: pgmap v1439: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:09.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:16:09 smithi083 ceph-mon[77242]: pgmap v1439: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:10.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:16:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:16:10.286Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:16:11.508 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:16:11 smithi083 ceph-mon[79930]: pgmap v1440: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:11.508 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:16:11 smithi083 ceph-mon[77242]: pgmap v1440: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:11.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:16:11 smithi153 ceph-mon[70044]: pgmap v1440: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:16:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:16:11] "GET /metrics HTTP/1.1" 200 225174 "" "Prometheus/2.33.4" 2022-09-29T06:16:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:16:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:16:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:16:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:16:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:16:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:16:13.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:16:13 smithi153 ceph-mon[70044]: pgmap v1441: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:13.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:16:13 smithi083 ceph-mon[79930]: pgmap v1441: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:13.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:16:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:16:13.696Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:16:13.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:16:13 smithi083 ceph-mon[77242]: pgmap v1441: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:14.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:16:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:16:14.391Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:16:15.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:16:15 smithi153 ceph-mon[70044]: pgmap v1442: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:15.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:16:15 smithi083 ceph-mon[79930]: pgmap v1442: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:15.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:16:15 smithi083 ceph-mon[77242]: pgmap v1442: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:17.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:16:17 smithi153 ceph-mon[70044]: pgmap v1443: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:17.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:16:17 smithi083 ceph-mon[79930]: pgmap v1443: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:17.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:16:17 smithi083 ceph-mon[77242]: pgmap v1443: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:19.217 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:16:19.509 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:16:19.509 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (50m) 2m ago 50m 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:16:19.510 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (50m) 37s ago 50m 61.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:16:19.510 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (51m) 2m ago 51m 23.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:16:19.511 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (51m) 37s ago 51m 27.6M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:16:19.511 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (47m) 37s ago 53m 439M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:16:19.512 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (47m) 2m ago 55m 550M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:16:19.512 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (46m) 2m ago 55m 241M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:16:19.513 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (46m) 37s ago 54m 115M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:16:19.513 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (46m) 2m ago 54m 144M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:16:19.513 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (50m) 2m ago 50m 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:16:19.514 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (50m) 37s ago 50m 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:16:19.514 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (45m) 2m ago 53m 183M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:16:19.515 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (44m) 2m ago 53m 176M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:16:19.515 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (44m) 2m ago 53m 144M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:16:19.515 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (44m) 2m ago 52m 166M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:16:19.516 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (43m) 37s ago 52m 154M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:16:19.516 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (42m) 37s ago 52m 180M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:16:19.517 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (42m) 37s ago 51m 159M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:16:19.517 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (41m) 37s ago 51m 180M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:16:19.518 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (47m) 37s ago 50m 102M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:16:19.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:16:19 smithi153 ceph-mon[70044]: pgmap v1444: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:19.865 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:16:19.865 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:16:19.866 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:16:19.866 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:16:19.867 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:16:19.867 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:16:19.867 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:16:19.868 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:16:19.868 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:16:19.869 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:16:19.869 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:16:19.870 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:16:19.870 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:16:19.870 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:16:19.871 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:16:19.871 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:16:19.872 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:16:19.872 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:16:19.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:16:19 smithi083 ceph-mon[79930]: pgmap v1444: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:19.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:16:19 smithi083 ceph-mon[77242]: pgmap v1444: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:20.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:16:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:16:20.287Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:16:20.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:16:20 smithi083 ceph-mon[79930]: from='client.35817 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:16:20.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:16:20 smithi083 ceph-mon[79930]: from='client.35823 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:16:20.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:16:20 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4279501066' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:16:20.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:16:20 smithi083 ceph-mon[77242]: from='client.35817 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:16:20.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:16:20 smithi083 ceph-mon[77242]: from='client.35823 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:16:20.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:16:20 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4279501066' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:16:20.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:16:20 smithi153 ceph-mon[70044]: from='client.35817 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:16:20.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:16:20 smithi153 ceph-mon[70044]: from='client.35823 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:16:20.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:16:20 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4279501066' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:16:21.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:16:21 smithi153 ceph-mon[70044]: pgmap v1445: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:16:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:16:21] "GET /metrics HTTP/1.1" 200 225174 "" "Prometheus/2.33.4" 2022-09-29T06:16:21.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:16:21 smithi083 ceph-mon[79930]: pgmap v1445: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:21.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:16:21 smithi083 ceph-mon[77242]: pgmap v1445: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:16:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:16:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:16:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:16:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:16:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:16:23.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:16:23 smithi153 ceph-mon[70044]: pgmap v1446: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:23.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:16:23 smithi083 ceph-mon[79930]: pgmap v1446: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:23.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:16:23 smithi083 ceph-mon[77242]: pgmap v1446: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:23.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:16:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:16:23.697Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:16:24.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:16:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:16:24.392Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:16:25.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:16:25 smithi153 ceph-mon[70044]: pgmap v1447: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:25.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:16:25 smithi083 ceph-mon[79930]: pgmap v1447: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:25.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:16:25 smithi083 ceph-mon[77242]: pgmap v1447: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:27.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:16:27 smithi153 ceph-mon[70044]: pgmap v1448: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:27.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:16:27 smithi083 ceph-mon[79930]: pgmap v1448: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:27.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:16:27 smithi083 ceph-mon[77242]: pgmap v1448: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:29.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:16:29 smithi153 ceph-mon[70044]: pgmap v1449: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:29.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:16:29 smithi083 ceph-mon[79930]: pgmap v1449: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:29.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:16:29 smithi083 ceph-mon[77242]: pgmap v1449: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:30.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:16:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:16:30.287Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:16:31.508 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:16:31 smithi083 ceph-mon[79930]: pgmap v1450: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:31.508 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:16:31 smithi083 ceph-mon[77242]: pgmap v1450: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:31.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:16:31 smithi153 ceph-mon[70044]: pgmap v1450: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:16:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:16:31] "GET /metrics HTTP/1.1" 200 225171 "" "Prometheus/2.33.4" 2022-09-29T06:16:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:16:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:16:32.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:16:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:16:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:16:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:16:33.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:16:33 smithi153 ceph-mon[70044]: pgmap v1451: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:33.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:16:33 smithi083 ceph-mon[79930]: pgmap v1451: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:33.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:16:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:16:33.697Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:16:33.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:16:33 smithi083 ceph-mon[77242]: pgmap v1451: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:34.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:16:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:16:34.393Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:16:35.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:16:35 smithi153 ceph-mon[70044]: pgmap v1452: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:35.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:16:35 smithi083 ceph-mon[77242]: pgmap v1452: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:35.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:16:35 smithi083 ceph-mon[79930]: pgmap v1452: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:37.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:16:37 smithi153 ceph-mon[70044]: pgmap v1453: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:37.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:16:37 smithi083 ceph-mon[79930]: pgmap v1453: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:37.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:16:37 smithi083 ceph-mon[77242]: pgmap v1453: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:39.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:16:39 smithi153 ceph-mon[70044]: pgmap v1454: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:39.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:16:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:16:39.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:16:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:16:39.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:16:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:16:39.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:16:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:16:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:16:39 smithi083 ceph-mon[79930]: pgmap v1454: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:16:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:16:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:16:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:16:39.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:16:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:16:39.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:16:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:16:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:16:39 smithi083 ceph-mon[77242]: pgmap v1454: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:39.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:16:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:16:39.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:16:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:16:39.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:16:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:16:39.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:16:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:16:40.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:16:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:16:40.288Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:16:41.508 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:16:41 smithi083 ceph-mon[79930]: pgmap v1455: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:41.509 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:16:41 smithi083 ceph-mon[77242]: pgmap v1455: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:16:41 smithi153 ceph-mon[70044]: pgmap v1455: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:16:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:16:41] "GET /metrics HTTP/1.1" 200 225173 "" "Prometheus/2.33.4" 2022-09-29T06:16:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:16:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:16:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:16:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:16:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:16:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:16:43.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:16:43 smithi153 ceph-mon[70044]: pgmap v1456: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:43.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:16:43 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:16:43.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:16:43 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:16:43.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:16:43 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:16:43.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:16:43 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:16:43.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:16:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:16:43.697Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:16:43.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:16:43 smithi083 ceph-mon[79930]: pgmap v1456: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:43.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:16:43 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:16:43.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:16:43 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:16:43.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:16:43 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:16:43.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:16:43 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:16:43.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:16:43 smithi083 ceph-mon[77242]: pgmap v1456: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:43.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:16:43 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:16:43.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:16:43 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:16:43.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:16:43 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:16:43.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:16:43 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:16:44.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:16:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:16:44.394Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:16:45.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:16:45 smithi153 ceph-mon[70044]: pgmap v1457: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:45.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:16:45 smithi083 ceph-mon[79930]: pgmap v1457: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:45.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:16:45 smithi083 ceph-mon[77242]: pgmap v1457: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:47.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:16:47 smithi153 ceph-mon[70044]: pgmap v1458: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:47.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:16:47 smithi083 ceph-mon[79930]: pgmap v1458: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:47.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:16:47 smithi083 ceph-mon[77242]: pgmap v1458: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:49.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:16:49 smithi153 ceph-mon[70044]: pgmap v1459: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:49.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:16:49 smithi083 ceph-mon[79930]: pgmap v1459: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:49.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:16:49 smithi083 ceph-mon[77242]: pgmap v1459: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:50.190 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:16:50.482 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:16:50.482 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (50m) 3m ago 51m 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:16:50.482 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (50m) 68s ago 50m 61.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:16:50.483 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (51m) 3m ago 51m 23.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:16:50.483 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (51m) 68s ago 51m 27.6M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:16:50.483 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (47m) 68s ago 54m 439M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:16:50.483 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (48m) 3m ago 55m 550M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:16:50.484 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (47m) 3m ago 56m 241M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:16:50.484 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (46m) 68s ago 54m 115M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:16:50.484 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (46m) 3m ago 54m 144M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:16:50.485 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (51m) 3m ago 51m 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:16:50.485 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (51m) 68s ago 51m 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:16:50.485 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (46m) 3m ago 54m 183M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:16:50.486 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (45m) 3m ago 53m 176M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:16:50.486 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (45m) 3m ago 53m 144M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:16:50.486 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (44m) 3m ago 53m 166M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:16:50.486 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (44m) 68s ago 52m 154M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:16:50.487 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (43m) 68s ago 52m 180M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:16:50.487 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (42m) 68s ago 52m 159M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:16:50.487 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (42m) 68s ago 52m 180M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:16:50.487 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (47m) 68s ago 51m 102M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:16:50.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:16:50 smithi083 ceph-mon[77242]: from='client.35835 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:16:50.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:16:50 smithi083 ceph-mon[79930]: from='client.35835 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:16:50.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:16:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:16:50.288Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:16:50.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:16:50 smithi153 ceph-mon[70044]: from='client.35835 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:16:50.845 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:16:50.845 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:16:50.846 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:16:50.846 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:16:50.846 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:16:50.846 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:16:50.847 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:16:50.849 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:16:50.850 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:16:50.850 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:16:50.850 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:16:50.851 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:16:50.851 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:16:50.852 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:16:50.852 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:16:50.853 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:16:50.853 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:16:50.854 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:16:51.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:16:51 smithi153 ceph-mon[70044]: pgmap v1460: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:51.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:16:51 smithi153 ceph-mon[70044]: from='client.35841 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:16:51.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:16:51 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1571486680' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:16:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:16:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:16:51] "GET /metrics HTTP/1.1" 200 225173 "" "Prometheus/2.33.4" 2022-09-29T06:16:51.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:16:51 smithi083 ceph-mon[77242]: pgmap v1460: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:51.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:16:51 smithi083 ceph-mon[77242]: from='client.35841 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:16:51.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:16:51 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1571486680' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:16:51.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:16:51 smithi083 ceph-mon[79930]: pgmap v1460: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:51.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:16:51 smithi083 ceph-mon[79930]: from='client.35841 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:16:51.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:16:51 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1571486680' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:16:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:16:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:16:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:16:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:16:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:16:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:16:53.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:16:53 smithi153 ceph-mon[70044]: pgmap v1461: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:53.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:16:53 smithi083 ceph-mon[79930]: pgmap v1461: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:53.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:16:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:16:53.699Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:16:53.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:16:53 smithi083 ceph-mon[77242]: pgmap v1461: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:54.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:16:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:16:54.394Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:16:55.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:16:55 smithi153 ceph-mon[70044]: pgmap v1462: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:55.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:16:55 smithi083 ceph-mon[79930]: pgmap v1462: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:55.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:16:55 smithi083 ceph-mon[77242]: pgmap v1462: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:57.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:16:57 smithi153 ceph-mon[70044]: pgmap v1463: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:57.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:16:57 smithi083 ceph-mon[77242]: pgmap v1463: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:57.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:16:57 smithi083 ceph-mon[79930]: pgmap v1463: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:59.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:16:59 smithi153 ceph-mon[70044]: pgmap v1464: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:59.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:16:59 smithi083 ceph-mon[79930]: pgmap v1464: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:16:59.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:16:59 smithi083 ceph-mon[77242]: pgmap v1464: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:00.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:17:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:17:00.288Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:17:01.508 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:17:01 smithi083 ceph-mon[79930]: pgmap v1465: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:01.508 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:17:01 smithi083 ceph-mon[77242]: pgmap v1465: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:01.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:17:01 smithi153 ceph-mon[70044]: pgmap v1465: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:17:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:17:01] "GET /metrics HTTP/1.1" 200 225172 "" "Prometheus/2.33.4" 2022-09-29T06:17:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:17:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:17:02.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:17:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:17:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:17:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:17:03.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:17:03 smithi153 ceph-mon[70044]: pgmap v1466: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:03.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:17:03 smithi083 ceph-mon[77242]: pgmap v1466: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:03.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:17:03 smithi083 ceph-mon[79930]: pgmap v1466: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:03.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:17:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:17:03.700Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:17:04.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:17:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:17:04.395Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:17:05.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:17:05 smithi153 ceph-mon[70044]: pgmap v1467: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:05.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:17:05 smithi083 ceph-mon[79930]: pgmap v1467: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:05.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:17:05 smithi083 ceph-mon[77242]: pgmap v1467: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:07.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:17:07 smithi153 ceph-mon[70044]: pgmap v1468: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:07.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:17:07 smithi083 ceph-mon[79930]: pgmap v1468: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:07.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:17:07 smithi083 ceph-mon[77242]: pgmap v1468: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:09.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:17:09 smithi153 ceph-mon[70044]: pgmap v1469: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:09.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:17:09 smithi083 ceph-mon[79930]: pgmap v1469: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:09.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:17:09 smithi083 ceph-mon[77242]: pgmap v1469: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:10.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:17:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:17:10.290Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:17:11.508 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:17:11 smithi083 ceph-mon[79930]: pgmap v1470: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:11.509 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:17:11 smithi083 ceph-mon[77242]: pgmap v1470: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:11.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:17:11 smithi153 ceph-mon[70044]: pgmap v1470: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:17:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:17:11] "GET /metrics HTTP/1.1" 200 225170 "" "Prometheus/2.33.4" 2022-09-29T06:17:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:17:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:17:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:17:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:17:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:17:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:17:13.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:17:13 smithi153 ceph-mon[70044]: pgmap v1471: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:13.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:17:13 smithi083 ceph-mon[79930]: pgmap v1471: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:13.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:17:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:17:13.700Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:17:13.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:17:13 smithi083 ceph-mon[77242]: pgmap v1471: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:14.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:17:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:17:14.396Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:17:15.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:17:15 smithi153 ceph-mon[70044]: pgmap v1472: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:15.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:17:15 smithi083 ceph-mon[79930]: pgmap v1472: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:15.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:17:15 smithi083 ceph-mon[77242]: pgmap v1472: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:17.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:17:17 smithi153 ceph-mon[70044]: pgmap v1473: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:17.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:17:17 smithi083 ceph-mon[77242]: pgmap v1473: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:17.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:17:17 smithi083 ceph-mon[79930]: pgmap v1473: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:19.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:17:19 smithi153 ceph-mon[70044]: pgmap v1474: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:19.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:17:19 smithi083 ceph-mon[77242]: pgmap v1474: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:19.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:17:19 smithi083 ceph-mon[79930]: pgmap v1474: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:20.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:17:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:17:20.290Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:17:21.164 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:17:21.457 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:17:21.457 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (51m) 3m ago 51m 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:17:21.458 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (51m) 99s ago 51m 61.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:17:21.458 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (52m) 3m ago 52m 23.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:17:21.458 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (52m) 99s ago 52m 27.6M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:17:21.458 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (48m) 99s ago 54m 439M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:17:21.459 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (48m) 3m ago 56m 550M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:17:21.459 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (47m) 3m ago 56m 241M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:17:21.459 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (47m) 99s ago 55m 115M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:17:21.459 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (47m) 3m ago 55m 144M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:17:21.460 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (51m) 3m ago 51m 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:17:21.460 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (51m) 99s ago 51m 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:17:21.460 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (46m) 3m ago 54m 183M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:17:21.460 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (45m) 3m ago 54m 176M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:17:21.461 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (45m) 3m ago 54m 144M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:17:21.461 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (45m) 3m ago 53m 166M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:17:21.461 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (44m) 99s ago 53m 154M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:17:21.461 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (44m) 99s ago 53m 180M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:17:21.462 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (43m) 99s ago 52m 159M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:17:21.462 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (42m) 99s ago 52m 180M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:17:21.462 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (48m) 99s ago 51m 102M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:17:21.507 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:17:21 smithi083 ceph-mon[79930]: pgmap v1475: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:21.507 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:17:21 smithi083 ceph-mon[79930]: from='client.35853 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:17:21.508 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:17:21 smithi083 ceph-mon[77242]: pgmap v1475: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:21.508 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:17:21 smithi083 ceph-mon[77242]: from='client.35853 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:17:21.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:17:21 smithi153 ceph-mon[70044]: pgmap v1475: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:21.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:17:21 smithi153 ceph-mon[70044]: from='client.35853 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:17:21.812 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:17:21.812 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:17:21.812 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:17:21.813 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:17:21.813 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:17:21.813 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:17:21.813 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:17:21.814 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:17:21.814 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:17:21.814 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:17:21.814 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:17:21.815 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:17:21.815 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:17:21.815 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:17:21.815 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:17:21.816 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:17:21.816 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:17:21.816 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:17:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:17:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:17:21] "GET /metrics HTTP/1.1" 200 225170 "" "Prometheus/2.33.4" 2022-09-29T06:17:22.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:17:22 smithi153 ceph-mon[70044]: from='client.35859 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:17:22.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:17:22 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2756519325' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:17:22.559 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:17:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:17:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:17:22.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:17:22 smithi083 ceph-mon[77242]: from='client.35859 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:17:22.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:17:22 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2756519325' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:17:22.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:17:22 smithi083 ceph-mon[79930]: from='client.35859 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:17:22.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:17:22 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2756519325' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:17:23.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:17:23 smithi153 ceph-mon[70044]: pgmap v1476: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:17:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:17:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:17:23.703 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:17:23 smithi083 ceph-mon[79930]: pgmap v1476: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:23.703 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:17:23 smithi083 ceph-mon[77242]: pgmap v1476: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:23.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:17:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:17:23.701Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:17:24.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:17:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:17:24.396Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:17:25.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:17:25 smithi153 ceph-mon[70044]: pgmap v1477: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:25.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:17:25 smithi083 ceph-mon[79930]: pgmap v1477: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:25.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:17:25 smithi083 ceph-mon[77242]: pgmap v1477: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:27.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:17:27 smithi153 ceph-mon[70044]: pgmap v1478: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:27.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:17:27 smithi083 ceph-mon[79930]: pgmap v1478: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:27.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:17:27 smithi083 ceph-mon[77242]: pgmap v1478: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:29.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:17:29 smithi153 ceph-mon[70044]: pgmap v1479: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:29.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:17:29 smithi083 ceph-mon[79930]: pgmap v1479: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:29.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:17:29 smithi083 ceph-mon[77242]: pgmap v1479: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:30.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:17:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:17:30.291Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:17:31.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:17:31 smithi153 ceph-mon[70044]: pgmap v1480: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:31.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:17:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:17:31] "GET /metrics HTTP/1.1" 200 225175 "" "Prometheus/2.33.4" 2022-09-29T06:17:31.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:17:31 smithi083 ceph-mon[79930]: pgmap v1480: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:31.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:17:31 smithi083 ceph-mon[77242]: pgmap v1480: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:17:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:17:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:17:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:17:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:17:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:17:33.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:17:33 smithi153 ceph-mon[70044]: pgmap v1481: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:33.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:17:33 smithi083 ceph-mon[79930]: pgmap v1481: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:33.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:17:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:17:33.701Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:17:33.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:17:33 smithi083 ceph-mon[77242]: pgmap v1481: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:34.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:17:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:17:34.397Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:17:35.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:17:35 smithi153 ceph-mon[70044]: pgmap v1482: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:35.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:17:35 smithi083 ceph-mon[77242]: pgmap v1482: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:35.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:17:35 smithi083 ceph-mon[79930]: pgmap v1482: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:37.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:17:37 smithi153 ceph-mon[70044]: pgmap v1483: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:37.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:17:37 smithi083 ceph-mon[79930]: pgmap v1483: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:37.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:17:37 smithi083 ceph-mon[77242]: pgmap v1483: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:39.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:17:39 smithi153 ceph-mon[70044]: pgmap v1484: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:39.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:17:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:17:39.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:17:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:17:39.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:17:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:17:39.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:17:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:17:39.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:17:39 smithi083 ceph-mon[77242]: pgmap v1484: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:39.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:17:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:17:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:17:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:17:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:17:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:17:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:17:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:17:39.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:17:39 smithi083 ceph-mon[79930]: pgmap v1484: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:39.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:17:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:17:39.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:17:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:17:39.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:17:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:17:39.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:17:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:17:40.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:17:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:17:40.291Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:17:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:17:41 smithi153 ceph-mon[70044]: pgmap v1485: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:41.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:17:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:17:41] "GET /metrics HTTP/1.1" 200 225170 "" "Prometheus/2.33.4" 2022-09-29T06:17:41.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:17:41 smithi083 ceph-mon[79930]: pgmap v1485: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:41.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:17:41 smithi083 ceph-mon[77242]: pgmap v1485: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:17:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:17:42.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:17:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:17:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:17:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:17:43.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:17:43 smithi153 ceph-mon[70044]: pgmap v1486: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:43.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:17:43 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:17:43.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:17:43 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:17:43.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:17:43 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:17:43.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:17:43 smithi083 ceph-mon[79930]: pgmap v1486: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:43.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:17:43 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:17:43.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:17:43 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:17:43.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:17:43 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:17:43.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:17:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:17:43.702Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:17:43.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:17:43 smithi083 ceph-mon[77242]: pgmap v1486: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:43.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:17:43 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:17:43.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:17:43 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:17:43.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:17:43 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:17:44.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:17:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:17:44.397Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:17:45.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:17:45 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:17:45.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:17:45 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:17:45.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:17:45 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:17:46.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:17:46 smithi153 ceph-mon[70044]: pgmap v1487: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:46.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:17:46 smithi083 ceph-mon[79930]: pgmap v1487: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:46.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:17:46 smithi083 ceph-mon[77242]: pgmap v1487: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:48.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:17:48 smithi153 ceph-mon[70044]: pgmap v1488: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:48.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:17:48 smithi083 ceph-mon[79930]: pgmap v1488: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:48.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:17:48 smithi083 ceph-mon[77242]: pgmap v1488: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:50.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:17:50 smithi153 ceph-mon[70044]: pgmap v1489: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:50.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:17:50 smithi083 ceph-mon[79930]: pgmap v1489: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:50.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:17:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:17:50.292Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:17:50.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:17:50 smithi083 ceph-mon[77242]: pgmap v1489: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:51.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:17:51 smithi153 ceph-mon[70044]: pgmap v1490: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:51.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:17:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:17:51] "GET /metrics HTTP/1.1" 200 225170 "" "Prometheus/2.33.4" 2022-09-29T06:17:51.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:17:51 smithi083 ceph-mon[79930]: pgmap v1490: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:51.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:17:51 smithi083 ceph-mon[77242]: pgmap v1490: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:52.138 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:17:52.430 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:17:52.430 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (51m) 4m ago 52m 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:17:52.431 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (51m) 2m ago 51m 61.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:17:52.431 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (52m) 4m ago 52m 23.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:17:52.431 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (52m) 2m ago 52m 27.6M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:17:52.431 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (48m) 2m ago 55m 439M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:17:52.432 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (49m) 4m ago 57m 550M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:17:52.432 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (48m) 4m ago 57m 241M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:17:52.432 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (47m) 2m ago 55m 115M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:17:52.432 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (47m) 4m ago 55m 144M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:17:52.433 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (52m) 4m ago 52m 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:17:52.433 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (52m) 2m ago 52m 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:17:52.433 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (47m) 4m ago 55m 183M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:17:52.434 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (46m) 4m ago 54m 176M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:17:52.434 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (46m) 4m ago 54m 144M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:17:52.434 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (45m) 4m ago 54m 166M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:17:52.434 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (45m) 2m ago 54m 154M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:17:52.435 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (44m) 2m ago 53m 180M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:17:52.435 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (44m) 2m ago 53m 159M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:17:52.435 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (43m) 2m ago 53m 180M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:17:52.435 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (48m) 2m ago 52m 102M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:17:52.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:17:52 smithi153 ceph-mon[70044]: from='client.45805 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:17:52.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:17:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:17:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:17:52.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:17:52 smithi083 ceph-mon[79930]: from='client.45805 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:17:52.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:17:52 smithi083 ceph-mon[77242]: from='client.45805 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:17:52.789 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:17:52.790 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:17:52.790 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:17:52.790 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:17:52.791 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:17:52.791 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:17:52.791 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:17:52.791 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:17:52.791 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:17:52.792 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:17:52.792 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:17:52.792 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:17:52.793 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:17:52.793 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:17:52.793 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:17:52.793 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:17:52.794 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:17:52.794 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:17:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:17:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:17:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:17:53.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:17:53 smithi153 ceph-mon[70044]: from='client.45811 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:17:53.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:17:53 smithi153 ceph-mon[70044]: pgmap v1491: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:53.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:17:53 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/604428870' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:17:53.704 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:17:53 smithi083 ceph-mon[79930]: from='client.45811 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:17:53.704 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:17:53 smithi083 ceph-mon[79930]: pgmap v1491: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:53.704 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:17:53 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/604428870' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:17:53.705 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:17:53 smithi083 ceph-mon[77242]: from='client.45811 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:17:53.705 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:17:53 smithi083 ceph-mon[77242]: pgmap v1491: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:53.705 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:17:53 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/604428870' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:17:53.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:17:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:17:53.702Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:17:54.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:17:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:17:54.398Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:17:55.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:17:55 smithi153 ceph-mon[70044]: pgmap v1492: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:55.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:17:55 smithi083 ceph-mon[77242]: pgmap v1492: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:55.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:17:55 smithi083 ceph-mon[79930]: pgmap v1492: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:57.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:17:57 smithi153 ceph-mon[70044]: pgmap v1493: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:57.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:17:57 smithi083 ceph-mon[79930]: pgmap v1493: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:57.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:17:57 smithi083 ceph-mon[77242]: pgmap v1493: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:59.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:17:59 smithi153 ceph-mon[70044]: pgmap v1494: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:59.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:17:59 smithi083 ceph-mon[79930]: pgmap v1494: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:17:59.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:17:59 smithi083 ceph-mon[77242]: pgmap v1494: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:00.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:18:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:18:00.293Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:18:01.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:18:01 smithi153 ceph-mon[70044]: pgmap v1495: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:01.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:18:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:18:01] "GET /metrics HTTP/1.1" 200 225173 "" "Prometheus/2.33.4" 2022-09-29T06:18:01.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:18:01 smithi083 ceph-mon[79930]: pgmap v1495: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:01.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:18:01 smithi083 ceph-mon[77242]: pgmap v1495: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:18:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:18:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:18:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:18:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:18:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:18:03.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:18:03 smithi153 ceph-mon[70044]: pgmap v1496: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:03.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:18:03 smithi083 ceph-mon[79930]: pgmap v1496: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:03.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:18:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:18:03.704Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:18:03.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:18:03 smithi083 ceph-mon[77242]: pgmap v1496: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:04.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:18:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:18:04.399Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:18:05.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:18:05 smithi153 ceph-mon[70044]: pgmap v1497: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:05.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:18:05 smithi083 ceph-mon[79930]: pgmap v1497: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:05.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:18:05 smithi083 ceph-mon[77242]: pgmap v1497: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:07.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:18:07 smithi153 ceph-mon[70044]: pgmap v1498: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:07.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:18:07 smithi083 ceph-mon[77242]: pgmap v1498: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:07.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:18:07 smithi083 ceph-mon[79930]: pgmap v1498: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:09.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:18:09 smithi153 ceph-mon[70044]: pgmap v1499: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:09.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:18:09 smithi083 ceph-mon[77242]: pgmap v1499: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:09.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:18:09 smithi083 ceph-mon[79930]: pgmap v1499: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:10.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:18:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:18:10.293Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:18:11.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:18:11 smithi153 ceph-mon[70044]: pgmap v1500: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:11.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:18:11 smithi083 ceph-mon[77242]: pgmap v1500: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:11.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:18:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:18:11] "GET /metrics HTTP/1.1" 200 225172 "" "Prometheus/2.33.4" 2022-09-29T06:18:11.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:18:11 smithi083 ceph-mon[79930]: pgmap v1500: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:18:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:18:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:18:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:18:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:18:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:18:13.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:18:13 smithi153 ceph-mon[70044]: pgmap v1501: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:13.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:18:13 smithi083 ceph-mon[79930]: pgmap v1501: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:13.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:18:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:18:13.705Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:18:13.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:18:13 smithi083 ceph-mon[77242]: pgmap v1501: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:14.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:18:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:18:14.400Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:18:15.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:18:15 smithi153 ceph-mon[70044]: pgmap v1502: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:15.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:18:15 smithi083 ceph-mon[79930]: pgmap v1502: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:15.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:18:15 smithi083 ceph-mon[77242]: pgmap v1502: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:17.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:18:17 smithi153 ceph-mon[70044]: pgmap v1503: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:17.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:18:17 smithi083 ceph-mon[79930]: pgmap v1503: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:17.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:18:17 smithi083 ceph-mon[77242]: pgmap v1503: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:19.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:18:19 smithi153 ceph-mon[70044]: pgmap v1504: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:19.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:18:19 smithi083 ceph-mon[79930]: pgmap v1504: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:19.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:18:19 smithi083 ceph-mon[77242]: pgmap v1504: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:20.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:18:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:18:20.293Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:18:21.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:18:21 smithi153 ceph-mon[70044]: pgmap v1505: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:21.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:18:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:18:21] "GET /metrics HTTP/1.1" 200 225172 "" "Prometheus/2.33.4" 2022-09-29T06:18:21.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:18:21 smithi083 ceph-mon[79930]: pgmap v1505: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:21.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:18:21 smithi083 ceph-mon[77242]: pgmap v1505: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:18:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:18:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:18:23.115 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:18:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:18:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:18:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:18:23.413 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:18:23.413 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (52m) 4m ago 52m 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:18:23.414 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (52m) 2m ago 52m 61.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:18:23.414 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (53m) 4m ago 53m 23.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:18:23.414 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (53m) 2m ago 53m 27.6M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:18:23.414 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (49m) 2m ago 55m 439M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:18:23.415 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (49m) 4m ago 57m 550M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:18:23.415 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (48m) 4m ago 57m 241M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:18:23.415 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (48m) 2m ago 56m 115M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:18:23.415 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (48m) 4m ago 56m 144M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:18:23.416 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (52m) 4m ago 52m 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:18:23.416 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (52m) 2m ago 52m 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:18:23.416 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (47m) 4m ago 55m 183M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:18:23.416 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (47m) 4m ago 55m 176M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:18:23.416 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (46m) 4m ago 55m 144M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:18:23.417 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (46m) 4m ago 54m 166M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:18:23.417 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (45m) 2m ago 54m 154M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:18:23.417 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (45m) 2m ago 54m 180M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:18:23.417 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (44m) 2m ago 53m 159M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:18:23.418 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (43m) 2m ago 53m 180M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:18:23.418 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (49m) 2m ago 52m 102M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:18:23.782 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:18:23.783 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:18:23.783 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:18:23.783 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:18:23.784 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:18:23.784 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:18:23.784 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:18:23.784 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:18:23.785 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:18:23.785 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:18:23.785 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:18:23.785 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:18:23.785 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:18:23.786 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:18:23.786 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:18:23.786 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:18:23.786 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:18:23.787 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:18:23.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:18:23 smithi153 ceph-mon[70044]: pgmap v1506: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:23.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:18:23 smithi153 ceph-mon[70044]: from='client.45823 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:18:23.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:18:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:18:23.704Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:18:23.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:18:23 smithi083 ceph-mon[77242]: pgmap v1506: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:23.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:18:23 smithi083 ceph-mon[77242]: from='client.45823 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:18:23.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:18:23 smithi083 ceph-mon[79930]: pgmap v1506: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:23.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:18:23 smithi083 ceph-mon[79930]: from='client.45823 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:18:24.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:18:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:18:24.400Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:18:24.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:18:24 smithi083 ceph-mon[77242]: from='client.45829 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:18:24.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:18:24 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3064186318' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:18:24.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:18:24 smithi083 ceph-mon[79930]: from='client.45829 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:18:24.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:18:24 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3064186318' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:18:24.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:18:24 smithi153 ceph-mon[70044]: from='client.45829 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:18:24.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:18:24 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3064186318' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:18:25.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:18:25 smithi153 ceph-mon[70044]: pgmap v1507: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:25.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:18:25 smithi083 ceph-mon[79930]: pgmap v1507: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:25.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:18:25 smithi083 ceph-mon[77242]: pgmap v1507: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:27.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:18:27 smithi153 ceph-mon[70044]: pgmap v1508: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:27.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:18:27 smithi083 ceph-mon[79930]: pgmap v1508: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:27.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:18:27 smithi083 ceph-mon[77242]: pgmap v1508: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:29.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:18:29 smithi153 ceph-mon[70044]: pgmap v1509: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:29.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:18:29 smithi083 ceph-mon[79930]: pgmap v1509: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:29.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:18:29 smithi083 ceph-mon[77242]: pgmap v1509: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:30.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:18:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:18:30.294Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:18:31.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:18:31 smithi153 ceph-mon[70044]: pgmap v1510: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:31.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:18:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:18:31] "GET /metrics HTTP/1.1" 200 225174 "" "Prometheus/2.33.4" 2022-09-29T06:18:31.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:18:31 smithi083 ceph-mon[79930]: pgmap v1510: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:31.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:18:31 smithi083 ceph-mon[77242]: pgmap v1510: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:18:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:18:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:18:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:18:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:18:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:18:33.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:18:33 smithi153 ceph-mon[70044]: pgmap v1511: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:33.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:18:33 smithi083 ceph-mon[79930]: pgmap v1511: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:33.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:18:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:18:33.706Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:18:33.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:18:33 smithi083 ceph-mon[77242]: pgmap v1511: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:34.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:18:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:18:34.402Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:18:35.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:18:35 smithi153 ceph-mon[70044]: pgmap v1512: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:35.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:18:35 smithi083 ceph-mon[77242]: pgmap v1512: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:35.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:18:35 smithi083 ceph-mon[79930]: pgmap v1512: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:37.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:18:37 smithi153 ceph-mon[70044]: pgmap v1513: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:37.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:18:37 smithi083 ceph-mon[79930]: pgmap v1513: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:37.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:18:37 smithi083 ceph-mon[77242]: pgmap v1513: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:39.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:18:39 smithi153 ceph-mon[70044]: pgmap v1514: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:39.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:18:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:18:39.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:18:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:18:39.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:18:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:18:39.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:18:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:18:39.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:18:39 smithi083 ceph-mon[79930]: pgmap v1514: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:18:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:18:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:18:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:18:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:18:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:18:39.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:18:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:18:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:18:39 smithi083 ceph-mon[77242]: pgmap v1514: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:18:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:18:39.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:18:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:18:39.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:18:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:18:39.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:18:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:18:40.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:18:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:18:40.294Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:18:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:18:41 smithi153 ceph-mon[70044]: pgmap v1515: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:41.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:18:41 smithi083 ceph-mon[77242]: pgmap v1515: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:41.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:18:41 smithi083 ceph-mon[79930]: pgmap v1515: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:41.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:18:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:18:41] "GET /metrics HTTP/1.1" 200 225176 "" "Prometheus/2.33.4" 2022-09-29T06:18:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:18:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:18:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:18:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:18:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:18:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:18:43.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:18:43 smithi153 ceph-mon[70044]: pgmap v1516: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:43.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:18:43 smithi083 ceph-mon[79930]: pgmap v1516: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:43.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:18:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:18:43.706Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:18:43.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:18:43 smithi083 ceph-mon[77242]: pgmap v1516: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:44.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:18:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:18:44.402Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:18:44.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:18:44 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:18:44.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:18:44 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:18:44.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:18:44 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:18:44.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:18:44 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:18:44.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:18:44 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:18:44.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:18:44 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:18:44.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:18:44 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:18:44.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:18:44 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:18:44.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:18:44 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:18:45.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:18:45 smithi083 ceph-mon[77242]: pgmap v1517: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:45.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:18:45 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:18:45.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:18:45 smithi083 ceph-mon[79930]: pgmap v1517: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:45.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:18:45 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:18:46.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:18:45 smithi153 ceph-mon[70044]: pgmap v1517: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:46.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:18:45 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:18:47.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:18:47 smithi083 ceph-mon[79930]: pgmap v1518: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:47.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:18:47 smithi083 ceph-mon[77242]: pgmap v1518: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:48.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:18:47 smithi153 ceph-mon[70044]: pgmap v1518: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:49.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:18:49 smithi083 ceph-mon[79930]: pgmap v1519: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:49.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:18:49 smithi083 ceph-mon[77242]: pgmap v1519: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:50.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:18:49 smithi153 ceph-mon[70044]: pgmap v1519: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:50.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:18:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:18:50.295Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:18:51.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:18:51 smithi153 ceph-mon[70044]: pgmap v1520: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:51.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:18:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:18:51] "GET /metrics HTTP/1.1" 200 225176 "" "Prometheus/2.33.4" 2022-09-29T06:18:51.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:18:51 smithi083 ceph-mon[79930]: pgmap v1520: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:51.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:18:51 smithi083 ceph-mon[77242]: pgmap v1520: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:18:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:18:52.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:18:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:18:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:18:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:18:53.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:18:53 smithi153 ceph-mon[70044]: pgmap v1521: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:53.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:18:53 smithi083 ceph-mon[79930]: pgmap v1521: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:53.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:18:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:18:53.707Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:18:53.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:18:53 smithi083 ceph-mon[77242]: pgmap v1521: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:54.124 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:18:54.413 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:18:54.413 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (52m) 5m ago 53m 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:18:54.413 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (52m) 3m ago 52m 61.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:18:54.414 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (53m) 5m ago 53m 23.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:18:54.414 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (53m) 3m ago 53m 27.6M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:18:54.414 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (49m) 3m ago 56m 439M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:18:54.414 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (50m) 5m ago 58m 550M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:18:54.415 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (49m) 5m ago 58m 241M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:18:54.415 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (48m) 3m ago 56m 115M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:18:54.415 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (49m) 5m ago 56m 144M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:18:54.415 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (53m) 5m ago 53m 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:18:54.416 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (53m) 3m ago 53m 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:18:54.416 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (48m) 5m ago 56m 183M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:18:54.416 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (47m) 5m ago 55m 176M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:18:54.416 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (47m) 5m ago 55m 144M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:18:54.417 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (46m) 5m ago 55m 166M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:18:54.417 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (46m) 3m ago 55m 154M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:18:54.417 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (45m) 3m ago 54m 180M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:18:54.417 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (45m) 3m ago 54m 159M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:18:54.418 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (44m) 3m ago 54m 180M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:18:54.418 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (49m) 3m ago 53m 102M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:18:54.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:18:54 smithi083 ceph-mon[79930]: from='client.45841 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:18:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:18:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:18:54.403Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:18:54.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:18:54 smithi083 ceph-mon[77242]: from='client.45841 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:18:54.768 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:18:54.768 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:18:54.768 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:18:54.768 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:18:54.769 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:18:54.769 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:18:54.769 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:18:54.770 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:18:54.770 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:18:54.770 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:18:54.770 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:18:54.771 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:18:54.771 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:18:54.771 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:18:54.771 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:18:54.771 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:18:54.772 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:18:54.772 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:18:54.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:18:54 smithi153 ceph-mon[70044]: from='client.45841 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:18:55.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:18:55 smithi153 ceph-mon[70044]: from='client.35913 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:18:55.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:18:55 smithi153 ceph-mon[70044]: pgmap v1522: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:55.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:18:55 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2651542062' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:18:55.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:18:55 smithi083 ceph-mon[79930]: from='client.35913 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:18:55.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:18:55 smithi083 ceph-mon[79930]: pgmap v1522: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:55.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:18:55 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2651542062' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:18:55.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:18:55 smithi083 ceph-mon[77242]: from='client.35913 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:18:55.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:18:55 smithi083 ceph-mon[77242]: pgmap v1522: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:55.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:18:55 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2651542062' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:18:57.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:18:57 smithi153 ceph-mon[70044]: pgmap v1523: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:57.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:18:57 smithi083 ceph-mon[79930]: pgmap v1523: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:57.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:18:57 smithi083 ceph-mon[77242]: pgmap v1523: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:59.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:18:59 smithi153 ceph-mon[70044]: pgmap v1524: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:59.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:18:59 smithi083 ceph-mon[79930]: pgmap v1524: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:18:59.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:18:59 smithi083 ceph-mon[77242]: pgmap v1524: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:19:00.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:19:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:19:00.296Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:19:01.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:19:01 smithi153 ceph-mon[70044]: pgmap v1525: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:19:01.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:19:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:19:01] "GET /metrics HTTP/1.1" 200 225174 "" "Prometheus/2.33.4" 2022-09-29T06:19:01.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:19:01 smithi083 ceph-mon[79930]: pgmap v1525: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:19:01.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:19:01 smithi083 ceph-mon[77242]: pgmap v1525: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:19:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:19:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:19:02.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:19:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:19:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:19:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:19:03.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:19:03 smithi153 ceph-mon[70044]: pgmap v1526: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:19:03.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:19:03 smithi083 ceph-mon[79930]: pgmap v1526: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:19:03.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:19:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:19:03.707Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:19:03.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:19:03 smithi083 ceph-mon[77242]: pgmap v1526: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:19:04.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:19:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:19:04.403Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:19:05.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:19:05 smithi153 ceph-mon[70044]: pgmap v1527: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:19:05.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:19:05 smithi083 ceph-mon[77242]: pgmap v1527: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:19:05.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:19:05 smithi083 ceph-mon[79930]: pgmap v1527: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:19:07.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:19:07 smithi153 ceph-mon[70044]: pgmap v1528: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:19:07.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:19:07 smithi083 ceph-mon[79930]: pgmap v1528: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:19:07.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:19:07 smithi083 ceph-mon[77242]: pgmap v1528: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:19:09.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:19:09 smithi153 ceph-mon[70044]: pgmap v1529: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:19:09.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:19:09 smithi083 ceph-mon[77242]: pgmap v1529: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:19:09.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:19:09 smithi083 ceph-mon[79930]: pgmap v1529: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:19:10.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:19:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:19:10.296Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:19:11.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:19:11 smithi153 ceph-mon[70044]: pgmap v1530: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:19:11.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:19:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:19:11] "GET /metrics HTTP/1.1" 200 225175 "" "Prometheus/2.33.4" 2022-09-29T06:19:11.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:19:11 smithi083 ceph-mon[79930]: pgmap v1530: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:19:11.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:19:11 smithi083 ceph-mon[77242]: pgmap v1530: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:19:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:19:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:19:12.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:19:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:19:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:19:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:19:13.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:19:13 smithi153 ceph-mon[70044]: pgmap v1531: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:19:13.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:19:13 smithi083 ceph-mon[79930]: pgmap v1531: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:19:13.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:19:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:19:13.708Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:19:13.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:19:13 smithi083 ceph-mon[77242]: pgmap v1531: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:19:14.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:19:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:19:14.404Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:19:15.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:19:15 smithi153 ceph-mon[70044]: pgmap v1532: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:19:15.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:19:15 smithi083 ceph-mon[79930]: pgmap v1532: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:19:15.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:19:15 smithi083 ceph-mon[77242]: pgmap v1532: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:19:17.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:19:17 smithi153 ceph-mon[70044]: pgmap v1533: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:19:17.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:19:17 smithi083 ceph-mon[79930]: pgmap v1533: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:19:17.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:19:17 smithi083 ceph-mon[77242]: pgmap v1533: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:19:19.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:19:19 smithi153 ceph-mon[70044]: pgmap v1534: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:19:19.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:19:19 smithi083 ceph-mon[79930]: pgmap v1534: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:19:19.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:19:19 smithi083 ceph-mon[77242]: pgmap v1534: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:19:20.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:19:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:19:20.297Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:19:21.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:19:21 smithi153 ceph-mon[70044]: pgmap v1535: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:19:21.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:19:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:19:21] "GET /metrics HTTP/1.1" 200 225175 "" "Prometheus/2.33.4" 2022-09-29T06:19:21.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:19:21 smithi083 ceph-mon[79930]: pgmap v1535: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:19:21.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:19:21 smithi083 ceph-mon[77242]: pgmap v1535: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:19:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:19:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:19:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:19:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:19:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:19:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:19:23.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:19:23 smithi153 ceph-mon[70044]: pgmap v1536: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:19:23.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:19:23 smithi083 ceph-mon[79930]: pgmap v1536: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:19:23.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:19:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:19:23.708Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:19:23.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:19:23 smithi083 ceph-mon[77242]: pgmap v1536: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:19:24.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:19:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:19:24.404Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:19:25.094 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:19:25.383 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:19:25.383 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (53m) 5m ago 53m 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:19:25.383 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (53m) 3m ago 53m 61.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:19:25.384 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (54m) 5m ago 54m 23.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:19:25.384 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (54m) 3m ago 54m 27.6M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:19:25.384 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (50m) 3m ago 57m 439M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:19:25.384 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (50m) 5m ago 58m 550M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:19:25.384 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (49m) 5m ago 58m 241M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:19:25.385 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (49m) 3m ago 57m 115M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:19:25.385 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (49m) 5m ago 57m 144M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:19:25.385 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (53m) 5m ago 53m 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:19:25.385 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (53m) 3m ago 53m 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:19:25.386 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (48m) 5m ago 56m 183M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:19:25.386 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (48m) 5m ago 56m 176M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:19:25.386 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (47m) 5m ago 56m 144M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:19:25.386 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (47m) 5m ago 55m 166M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:19:25.387 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (46m) 3m ago 55m 154M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:19:25.387 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (46m) 3m ago 55m 180M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:19:25.387 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (45m) 3m ago 55m 159M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:19:25.387 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (45m) 3m ago 54m 180M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:19:25.387 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (50m) 3m ago 53m 102M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:19:25.743 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:19:25.743 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:19:25.743 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:19:25.744 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:19:25.744 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:19:25.744 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:19:25.744 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:19:25.745 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:19:25.745 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:19:25.745 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:19:25.745 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:19:25.745 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:19:25.746 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:19:25.746 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:19:25.746 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:19:25.746 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:19:25.747 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:19:25.747 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:19:25.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:19:25 smithi153 ceph-mon[70044]: pgmap v1537: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:19:25.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:19:25 smithi153 ceph-mon[70044]: from='client.35925 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:19:25.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:19:25 smithi083 ceph-mon[77242]: pgmap v1537: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:19:25.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:19:25 smithi083 ceph-mon[77242]: from='client.35925 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:19:25.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:19:25 smithi083 ceph-mon[79930]: pgmap v1537: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:19:25.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:19:25 smithi083 ceph-mon[79930]: from='client.35925 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:19:26.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:19:26 smithi153 ceph-mon[70044]: from='client.35931 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:19:26.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:19:26 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3112257095' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:19:26.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:19:26 smithi083 ceph-mon[79930]: from='client.35931 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:19:26.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:19:26 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3112257095' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:19:26.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:19:26 smithi083 ceph-mon[77242]: from='client.35931 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:19:26.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:19:26 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3112257095' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:19:27.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:19:27 smithi153 ceph-mon[70044]: pgmap v1538: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:19:27.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:19:27 smithi083 ceph-mon[77242]: pgmap v1538: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:19:27.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:19:27 smithi083 ceph-mon[79930]: pgmap v1538: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:19:29.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:19:29 smithi083 ceph-mon[79930]: pgmap v1539: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:19:29.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:19:29 smithi083 ceph-mon[77242]: pgmap v1539: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:19:30.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:19:29 smithi153 ceph-mon[70044]: pgmap v1539: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:19:30.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:19:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:19:30.297Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:19:31.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:19:31 smithi153 ceph-mon[70044]: pgmap v1540: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:19:31.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:19:31 smithi083 ceph-mon[77242]: pgmap v1540: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:19:31.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:19:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:19:31] "GET /metrics HTTP/1.1" 200 225174 "" "Prometheus/2.33.4" 2022-09-29T06:19:31.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:19:31 smithi083 ceph-mon[79930]: pgmap v1540: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:19:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:19:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:19:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:19:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:19:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:19:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:19:33.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:19:33 smithi153 ceph-mon[70044]: pgmap v1541: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:19:33.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:19:33 smithi083 ceph-mon[79930]: pgmap v1541: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:19:33.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:19:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:19:33.708Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:19:33.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:19:33 smithi083 ceph-mon[77242]: pgmap v1541: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:19:34.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:19:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:19:34.405Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:19:35.802 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:19:35 smithi153 ceph-mon[70044]: pgmap v1542: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:19:35.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:19:35 smithi083 ceph-mon[79930]: pgmap v1542: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:19:35.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:19:35 smithi083 ceph-mon[77242]: pgmap v1542: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:19:37.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:19:37 smithi153 ceph-mon[70044]: pgmap v1543: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:19:37.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:19:37 smithi083 ceph-mon[79930]: pgmap v1543: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:19:37.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:19:37 smithi083 ceph-mon[77242]: pgmap v1543: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:19:39.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:19:39 smithi153 ceph-mon[70044]: pgmap v1544: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:19:39.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:19:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:19:39.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:19:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:19:39.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:19:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:19:39.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:19:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:19:39.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:19:39 smithi083 ceph-mon[79930]: pgmap v1544: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:19:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:19:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:19:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:19:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:19:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:19:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:19:39.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:19:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:19:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:19:39 smithi083 ceph-mon[77242]: pgmap v1544: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:19:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:19:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:19:39.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:19:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:19:39.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:19:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:19:39.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:19:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:19:40.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:19:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:19:40.298Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:19:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:19:41 smithi153 ceph-mon[70044]: pgmap v1545: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:19:41.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:19:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:19:41] "GET /metrics HTTP/1.1" 200 225174 "" "Prometheus/2.33.4" 2022-09-29T06:19:41.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:19:41 smithi083 ceph-mon[77242]: pgmap v1545: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:19:41.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:19:41 smithi083 ceph-mon[79930]: pgmap v1545: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:19:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:19:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:19:42.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:19:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:19:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:19:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:19:43.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:19:43 smithi153 ceph-mon[70044]: pgmap v1546: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:19:43.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:19:43 smithi083 ceph-mon[79930]: pgmap v1546: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:19:43.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:19:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:19:43.709Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:19:43.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:19:43 smithi083 ceph-mon[77242]: pgmap v1546: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:19:44.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:19:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:19:44.405Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:19:45.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:19:45 smithi153 ceph-mon[70044]: pgmap v1547: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:19:45.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:19:45 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:19:45.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:19:45 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:19:45.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:19:45 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:19:45.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:19:45 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:19:45.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:19:45 smithi083 ceph-mon[79930]: pgmap v1547: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:19:45.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:19:45 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:19:45.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:19:45 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:19:45.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:19:45 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:19:45.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:19:45 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:19:45.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:19:45 smithi083 ceph-mon[77242]: pgmap v1547: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:19:45.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:19:45 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:19:45.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:19:45 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:19:45.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:19:45 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:19:45.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:19:45 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:19:47.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:19:47 smithi153 ceph-mon[70044]: pgmap v1548: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:19:47.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:19:47 smithi083 ceph-mon[79930]: pgmap v1548: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:19:47.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:19:47 smithi083 ceph-mon[77242]: pgmap v1548: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:19:49.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:19:49 smithi083 ceph-mon[79930]: pgmap v1549: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:19:49.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:19:49 smithi083 ceph-mon[77242]: pgmap v1549: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:19:50.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:19:49 smithi153 ceph-mon[70044]: pgmap v1549: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:19:50.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:19:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:19:50.298Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:19:51.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:19:51 smithi153 ceph-mon[70044]: pgmap v1550: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:19:51.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:19:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:19:51] "GET /metrics HTTP/1.1" 200 225174 "" "Prometheus/2.33.4" 2022-09-29T06:19:51.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:19:51 smithi083 ceph-mon[79930]: pgmap v1550: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:19:51.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:19:51 smithi083 ceph-mon[77242]: pgmap v1550: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:19:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:19:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:19:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:19:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:19:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:19:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:19:53.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:19:53 smithi153 ceph-mon[70044]: pgmap v1551: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:19:53.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:19:53 smithi083 ceph-mon[79930]: pgmap v1551: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:19:53.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:19:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:19:53.710Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:19:53.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:19:53 smithi083 ceph-mon[77242]: pgmap v1551: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:19:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:19:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:19:54.406Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:19:55.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:19:55 smithi153 ceph-mon[70044]: pgmap v1552: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:19:55.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:19:55 smithi083 ceph-mon[79930]: pgmap v1552: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:19:55.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:19:55 smithi083 ceph-mon[77242]: pgmap v1552: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:19:56.070 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:19:56.358 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:19:56.359 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (53m) 6m ago 54m 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:19:56.359 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (53m) 4m ago 53m 61.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:19:56.359 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (55m) 6m ago 54m 23.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:19:56.359 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (54m) 4m ago 54m 27.6M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:19:56.360 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (50m) 4m ago 57m 439M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:19:56.360 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (51m) 6m ago 59m 550M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:19:56.360 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (50m) 6m ago 59m 241M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:19:56.360 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (49m) 4m ago 57m 115M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:19:56.360 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (50m) 6m ago 57m 144M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:19:56.361 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (54m) 6m ago 54m 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:19:56.361 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (54m) 4m ago 54m 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:19:56.361 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (49m) 6m ago 57m 183M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:19:56.361 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (48m) 6m ago 56m 176M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:19:56.362 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (48m) 6m ago 56m 144M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:19:56.362 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (47m) 6m ago 56m 166M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:19:56.362 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (47m) 4m ago 56m 154M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:19:56.362 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (46m) 4m ago 55m 180M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:19:56.362 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (46m) 4m ago 55m 159M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:19:56.363 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (45m) 4m ago 55m 180M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:19:56.363 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (50m) 4m ago 54m 102M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:19:56.714 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:19:56.715 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:19:56.715 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:19:56.715 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:19:56.715 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:19:56.716 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:19:56.716 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:19:56.716 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:19:56.716 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:19:56.717 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:19:56.717 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:19:56.717 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:19:56.717 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:19:56.717 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:19:56.718 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:19:56.718 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:19:56.718 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:19:56.718 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:19:56.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:19:56 smithi083 ceph-mon[79930]: from='client.35943 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:19:56.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:19:56 smithi083 ceph-mon[77242]: from='client.35943 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:19:57.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:19:56 smithi153 ceph-mon[70044]: from='client.35943 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:19:57.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:19:57 smithi083 ceph-mon[79930]: from='client.35949 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:19:57.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:19:57 smithi083 ceph-mon[79930]: pgmap v1553: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:19:57.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:19:57 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/543285560' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:19:57.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:19:57 smithi083 ceph-mon[77242]: from='client.35949 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:19:57.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:19:57 smithi083 ceph-mon[77242]: pgmap v1553: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:19:57.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:19:57 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/543285560' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:19:58.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:19:57 smithi153 ceph-mon[70044]: from='client.35949 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:19:58.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:19:57 smithi153 ceph-mon[70044]: pgmap v1553: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:19:58.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:19:57 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/543285560' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:19:59.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:19:59 smithi083 ceph-mon[79930]: pgmap v1554: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:19:59.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:19:59 smithi083 ceph-mon[77242]: pgmap v1554: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:00.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:19:59 smithi153 ceph-mon[70044]: pgmap v1554: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:00.301 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:20:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[77219]: 2022-09-29T06:19:59.999+0000 7f160f665700 -1 log_channel(cluster) log [ERR] : overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T06:20:00.567 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:20:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:20:00.299Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:20:00.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:20:00 smithi083 ceph-mon[79930]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T06:20:00.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:20:00 smithi083 ceph-mon[77242]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T06:20:01.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:20:00 smithi153 ceph-mon[70044]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T06:20:01.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:20:01 smithi083 ceph-mon[77242]: pgmap v1555: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:20:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:20:01] "GET /metrics HTTP/1.1" 200 225173 "" "Prometheus/2.33.4" 2022-09-29T06:20:01.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:20:01 smithi083 ceph-mon[79930]: pgmap v1555: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:02.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:20:01 smithi153 ceph-mon[70044]: pgmap v1555: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:20:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:20:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:20:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:20:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:20:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:20:03.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:20:03 smithi083 ceph-mon[79930]: pgmap v1556: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:03.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:20:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:20:03.711Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:20:03.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:20:03 smithi083 ceph-mon[77242]: pgmap v1556: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:04.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:20:03 smithi153 ceph-mon[70044]: pgmap v1556: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:04.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:20:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:20:04.407Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:20:05.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:20:05 smithi083 ceph-mon[77242]: pgmap v1557: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:05.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:20:05 smithi083 ceph-mon[79930]: pgmap v1557: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:06.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:20:05 smithi153 ceph-mon[70044]: pgmap v1557: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:07.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:20:07 smithi083 ceph-mon[79930]: pgmap v1558: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:08.157 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:20:07 smithi083 ceph-mon[77242]: pgmap v1558: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:08.158 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:20:07 smithi153 ceph-mon[70044]: pgmap v1558: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:09.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:20:09 smithi083 ceph-mon[79930]: pgmap v1559: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:09.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:20:09 smithi083 ceph-mon[77242]: pgmap v1559: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:10.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:20:09 smithi153 ceph-mon[70044]: pgmap v1559: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:10.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:20:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:20:10.300Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:20:11.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:20:11 smithi153 ceph-mon[70044]: pgmap v1560: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:11.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:20:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:20:11] "GET /metrics HTTP/1.1" 200 225172 "" "Prometheus/2.33.4" 2022-09-29T06:20:11.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:20:11 smithi083 ceph-mon[79930]: pgmap v1560: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:11.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:20:11 smithi083 ceph-mon[77242]: pgmap v1560: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:20:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:20:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:20:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:20:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:20:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:20:13.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:20:13 smithi083 ceph-mon[77242]: pgmap v1561: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:13.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:20:13 smithi083 ceph-mon[79930]: pgmap v1561: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:13.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:20:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:20:13.711Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:20:14.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:20:13 smithi153 ceph-mon[70044]: pgmap v1561: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:14.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:20:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:20:14.407Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:20:15.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:20:15 smithi083 ceph-mon[79930]: pgmap v1562: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:15.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:20:15 smithi083 ceph-mon[77242]: pgmap v1562: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:16.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:20:15 smithi153 ceph-mon[70044]: pgmap v1562: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:17.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:20:17 smithi083 ceph-mon[79930]: pgmap v1563: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:17.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:20:17 smithi083 ceph-mon[77242]: pgmap v1563: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:18.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:20:17 smithi153 ceph-mon[70044]: pgmap v1563: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:19.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:20:19 smithi083 ceph-mon[77242]: pgmap v1564: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:19.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:20:19 smithi083 ceph-mon[79930]: pgmap v1564: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:20.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:20:19 smithi153 ceph-mon[70044]: pgmap v1564: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:20.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:20:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:20:20.300Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:20:21.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:20:21 smithi153 ceph-mon[70044]: pgmap v1565: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:21.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:20:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:20:21] "GET /metrics HTTP/1.1" 200 225172 "" "Prometheus/2.33.4" 2022-09-29T06:20:21.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:20:21 smithi083 ceph-mon[79930]: pgmap v1565: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:21.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:20:21 smithi083 ceph-mon[77242]: pgmap v1565: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:20:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:20:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:20:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:20:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:20:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:20:23.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:20:23 smithi083 ceph-mon[77242]: pgmap v1566: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:23.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:20:23 smithi083 ceph-mon[79930]: pgmap v1566: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:23.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:20:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:20:23.711Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:20:24.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:20:23 smithi153 ceph-mon[70044]: pgmap v1566: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:24.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:20:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:20:24.408Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:20:25.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:20:25 smithi083 ceph-mon[77242]: pgmap v1567: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:25.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:20:25 smithi083 ceph-mon[79930]: pgmap v1567: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:26.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:20:25 smithi153 ceph-mon[70044]: pgmap v1567: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:27.042 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:20:27.330 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:20:27.331 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (54m) 6m ago 54m 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:20:27.331 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (54m) 4m ago 54m 61.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:20:27.331 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (55m) 6m ago 55m 23.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:20:27.331 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (55m) 4m ago 55m 27.6M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:20:27.332 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (51m) 4m ago 58m 439M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:20:27.332 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (51m) 6m ago 59m 550M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:20:27.332 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (50m) 6m ago 59m 241M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:20:27.332 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (50m) 4m ago 58m 115M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:20:27.333 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (50m) 6m ago 58m 144M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:20:27.333 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (54m) 6m ago 54m 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:20:27.333 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (54m) 4m ago 54m 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:20:27.333 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (49m) 6m ago 57m 183M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:20:27.333 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (49m) 6m ago 57m 176M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:20:27.334 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (48m) 6m ago 57m 144M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:20:27.334 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (48m) 6m ago 56m 166M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:20:27.334 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (47m) 4m ago 56m 154M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:20:27.334 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (47m) 4m ago 56m 180M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:20:27.335 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (46m) 4m ago 56m 159M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:20:27.335 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (46m) 4m ago 55m 180M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:20:27.335 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (51m) 4m ago 54m 102M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:20:27.686 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:20:27.686 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:20:27.687 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:20:27.687 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:20:27.687 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:20:27.687 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:20:27.688 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:20:27.688 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:20:27.688 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:20:27.688 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:20:27.688 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:20:27.689 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:20:27.689 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:20:27.689 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:20:27.689 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:20:27.690 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:20:27.690 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:20:27.690 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:20:27.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:20:27 smithi083 ceph-mon[77242]: pgmap v1568: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:27.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:20:27 smithi083 ceph-mon[77242]: from='client.45895 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:20:27.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:20:27 smithi083 ceph-mon[79930]: pgmap v1568: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:27.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:20:27 smithi083 ceph-mon[79930]: from='client.45895 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:20:28.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:20:27 smithi153 ceph-mon[70044]: pgmap v1568: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:28.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:20:27 smithi153 ceph-mon[70044]: from='client.45895 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:20:28.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:20:28 smithi083 ceph-mon[77242]: from='client.35967 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:20:28.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:20:28 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/312169056' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:20:28.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:20:28 smithi083 ceph-mon[79930]: from='client.35967 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:20:28.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:20:28 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/312169056' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:20:29.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:20:28 smithi153 ceph-mon[70044]: from='client.35967 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:20:29.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:20:28 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/312169056' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:20:29.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:20:29 smithi083 ceph-mon[77242]: pgmap v1569: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:29.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:20:29 smithi083 ceph-mon[79930]: pgmap v1569: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:30.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:20:29 smithi153 ceph-mon[70044]: pgmap v1569: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:30.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:20:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:20:30.301Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:20:31.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:20:31 smithi153 ceph-mon[70044]: pgmap v1570: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:31.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:20:31 smithi083 ceph-mon[77242]: pgmap v1570: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:31.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:20:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:20:31] "GET /metrics HTTP/1.1" 200 225173 "" "Prometheus/2.33.4" 2022-09-29T06:20:31.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:20:31 smithi083 ceph-mon[79930]: pgmap v1570: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:20:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:20:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:20:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:20:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:20:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:20:33.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:20:33 smithi083 ceph-mon[77242]: pgmap v1571: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:33.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:20:33 smithi083 ceph-mon[79930]: pgmap v1571: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:33.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:20:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:20:33.712Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:20:34.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:20:33 smithi153 ceph-mon[70044]: pgmap v1571: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:34.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:20:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:20:34.409Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:20:35.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:20:35 smithi083 ceph-mon[79930]: pgmap v1572: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:35.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:20:35 smithi083 ceph-mon[77242]: pgmap v1572: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:36.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:20:35 smithi153 ceph-mon[70044]: pgmap v1572: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:37.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:20:37 smithi083 ceph-mon[79930]: pgmap v1573: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:37.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:20:37 smithi083 ceph-mon[77242]: pgmap v1573: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:38.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:20:37 smithi153 ceph-mon[70044]: pgmap v1573: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:20:39 smithi083 ceph-mon[79930]: pgmap v1574: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:20:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:20:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:20:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:20:39.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:20:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:20:39.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:20:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:20:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:20:39 smithi083 ceph-mon[77242]: pgmap v1574: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:39.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:20:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:20:39.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:20:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:20:39.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:20:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:20:39.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:20:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:20:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:20:39 smithi153 ceph-mon[70044]: pgmap v1574: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:20:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:20:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:20:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:20:40.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:20:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:20:40.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:20:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:20:40.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:20:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:20:40.303Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:20:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:20:41 smithi153 ceph-mon[70044]: pgmap v1575: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:41.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:20:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:20:41] "GET /metrics HTTP/1.1" 200 225174 "" "Prometheus/2.33.4" 2022-09-29T06:20:41.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:20:41 smithi083 ceph-mon[77242]: pgmap v1575: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:41.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:20:41 smithi083 ceph-mon[79930]: pgmap v1575: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:20:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:20:42.319Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:20:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:20:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:20:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:20:43.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:20:43 smithi083 ceph-mon[79930]: pgmap v1576: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:43.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:20:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:20:43.713Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:20:43.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:20:43 smithi083 ceph-mon[77242]: pgmap v1576: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:44.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:20:43 smithi153 ceph-mon[70044]: pgmap v1576: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:44.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:20:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:20:44.409Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:20:45.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:20:45 smithi083 ceph-mon[77242]: pgmap v1577: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:45.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:20:45 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:20:45.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:20:45 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:20:45.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:20:45 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:20:45.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:20:45 smithi083 ceph-mon[79930]: pgmap v1577: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:45.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:20:45 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:20:45.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:20:45 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:20:45.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:20:45 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:20:46.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:20:45 smithi153 ceph-mon[70044]: pgmap v1577: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:46.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:20:45 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:20:46.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:20:45 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:20:46.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:20:45 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:20:47.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:20:47 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:20:47.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:20:47 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:20:47.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:20:47 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:20:48.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:20:48 smithi153 ceph-mon[70044]: pgmap v1578: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:48.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:20:48 smithi083 ceph-mon[79930]: pgmap v1578: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:48.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:20:48 smithi083 ceph-mon[77242]: pgmap v1578: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:50.305 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:20:50 smithi083 ceph-mon[79930]: pgmap v1579: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:50.305 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:20:50 smithi083 ceph-mon[77242]: pgmap v1579: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:50.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:20:50 smithi153 ceph-mon[70044]: pgmap v1579: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:50.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:20:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:20:50.303Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:20:51.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:20:51 smithi153 ceph-mon[70044]: pgmap v1580: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:51.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:20:51 smithi083 ceph-mon[79930]: pgmap v1580: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:51.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:20:51 smithi083 ceph-mon[77242]: pgmap v1580: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:51.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:20:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:20:51] "GET /metrics HTTP/1.1" 200 225174 "" "Prometheus/2.33.4" 2022-09-29T06:20:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:20:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:20:52.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:20:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:20:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:20:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:20:53.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:20:53 smithi083 ceph-mon[79930]: pgmap v1581: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:53.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:20:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:20:53.714Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:20:53.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:20:53 smithi083 ceph-mon[77242]: pgmap v1581: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:54.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:20:53 smithi153 ceph-mon[70044]: pgmap v1581: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:54.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:20:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:20:54.410Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:20:55.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:20:55 smithi083 ceph-mon[79930]: pgmap v1582: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:55.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:20:55 smithi083 ceph-mon[77242]: pgmap v1582: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:56.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:20:55 smithi153 ceph-mon[70044]: pgmap v1582: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:57.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:20:57 smithi083 ceph-mon[79930]: pgmap v1583: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:57.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:20:57 smithi083 ceph-mon[77242]: pgmap v1583: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:58.013 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:20:58.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:20:57 smithi153 ceph-mon[70044]: pgmap v1583: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:58.302 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:20:58.302 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (54m) 7m ago 55m 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:20:58.302 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (54m) 5m ago 54m 61.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:20:58.303 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (56m) 7m ago 56m 23.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:20:58.303 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (55m) 5m ago 55m 27.6M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:20:58.303 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (51m) 5m ago 58m 439M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:20:58.303 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (52m) 7m ago 60m 550M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:20:58.303 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (51m) 7m ago 60m 241M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:20:58.304 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (50m) 5m ago 58m 115M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:20:58.304 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (51m) 7m ago 58m 144M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:20:58.304 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (55m) 7m ago 55m 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:20:58.304 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (55m) 5m ago 55m 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:20:58.305 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (50m) 7m ago 58m 183M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:20:58.305 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (49m) 7m ago 58m 176M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:20:58.305 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (49m) 7m ago 57m 144M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:20:58.305 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (48m) 7m ago 57m 166M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:20:58.306 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (48m) 5m ago 57m 154M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:20:58.306 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (47m) 5m ago 56m 180M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:20:58.306 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (47m) 5m ago 56m 159M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:20:58.306 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (46m) 5m ago 56m 180M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:20:58.307 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (51m) 5m ago 55m 102M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:20:58.659 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:20:58.660 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:20:58.660 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:20:58.660 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:20:58.661 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:20:58.661 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:20:58.661 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:20:58.661 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:20:58.662 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:20:58.662 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:20:58.662 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:20:58.662 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:20:58.663 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:20:58.663 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:20:58.663 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:20:58.663 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:20:58.663 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:20:58.664 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:20:58.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:20:58 smithi083 ceph-mon[79930]: from='client.35979 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:20:58.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:20:58 smithi083 ceph-mon[77242]: from='client.35979 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:20:59.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:20:58 smithi153 ceph-mon[70044]: from='client.35979 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:20:59.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:20:59 smithi083 ceph-mon[79930]: from='client.35985 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:20:59.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:20:59 smithi083 ceph-mon[79930]: pgmap v1584: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:59.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:20:59 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3429135438' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:20:59.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:20:59 smithi083 ceph-mon[77242]: from='client.35985 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:20:59.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:20:59 smithi083 ceph-mon[77242]: pgmap v1584: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:20:59.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:20:59 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3429135438' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:21:00.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:20:59 smithi153 ceph-mon[70044]: from='client.35985 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:21:00.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:20:59 smithi153 ceph-mon[70044]: pgmap v1584: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:21:00.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:20:59 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3429135438' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:21:00.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:21:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:21:00.306Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:21:01.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:21:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:21:01] "GET /metrics HTTP/1.1" 200 225172 "" "Prometheus/2.33.4" 2022-09-29T06:21:01.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:21:01 smithi083 ceph-mon[79930]: pgmap v1585: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:21:01.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:21:01 smithi083 ceph-mon[77242]: pgmap v1585: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:21:01.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:21:01 smithi153 ceph-mon[70044]: pgmap v1585: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:21:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:21:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:21:02.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:21:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:21:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:21:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:21:03.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:21:03 smithi083 ceph-mon[79930]: pgmap v1586: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:21:03.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:21:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:21:03.715Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:21:03.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:21:03 smithi083 ceph-mon[77242]: pgmap v1586: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:21:04.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:21:03 smithi153 ceph-mon[70044]: pgmap v1586: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:21:04.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:21:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:21:04.410Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:21:05.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:21:05 smithi083 ceph-mon[77242]: pgmap v1587: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:21:05.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:21:05 smithi083 ceph-mon[79930]: pgmap v1587: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:21:06.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:21:05 smithi153 ceph-mon[70044]: pgmap v1587: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:21:07.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:21:07 smithi083 ceph-mon[79930]: pgmap v1588: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:21:07.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:21:07 smithi083 ceph-mon[77242]: pgmap v1588: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:21:08.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:21:07 smithi153 ceph-mon[70044]: pgmap v1588: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:21:09.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:21:09 smithi083 ceph-mon[79930]: pgmap v1589: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:21:09.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:21:09 smithi083 ceph-mon[77242]: pgmap v1589: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:21:10.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:21:09 smithi153 ceph-mon[70044]: pgmap v1589: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:21:10.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:21:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:21:10.306Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:21:11.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:21:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:21:11] "GET /metrics HTTP/1.1" 200 225171 "" "Prometheus/2.33.4" 2022-09-29T06:21:11.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:21:11 smithi083 ceph-mon[77242]: pgmap v1590: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:21:11.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:21:11 smithi083 ceph-mon[79930]: pgmap v1590: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:21:11.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:21:11 smithi153 ceph-mon[70044]: pgmap v1590: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:21:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:21:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:21:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:21:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:21:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:21:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:21:13.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:21:13 smithi083 ceph-mon[77242]: pgmap v1591: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:21:13.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:21:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:21:13.715Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:21:13.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:21:13 smithi083 ceph-mon[79930]: pgmap v1591: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:21:14.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:21:13 smithi153 ceph-mon[70044]: pgmap v1591: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:21:14.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:21:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:21:14.411Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:21:15.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:21:15 smithi083 ceph-mon[77242]: pgmap v1592: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:21:15.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:21:15 smithi083 ceph-mon[79930]: pgmap v1592: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:21:16.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:21:15 smithi153 ceph-mon[70044]: pgmap v1592: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:21:17.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:21:17 smithi083 ceph-mon[77242]: pgmap v1593: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:21:17.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:21:17 smithi083 ceph-mon[79930]: pgmap v1593: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:21:18.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:21:17 smithi153 ceph-mon[70044]: pgmap v1593: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:21:19.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:21:19 smithi083 ceph-mon[77242]: pgmap v1594: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:21:19.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:21:19 smithi083 ceph-mon[79930]: pgmap v1594: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:21:20.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:21:19 smithi153 ceph-mon[70044]: pgmap v1594: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:21:20.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:21:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:21:20.306Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:21:21.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:21:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:21:21] "GET /metrics HTTP/1.1" 200 225171 "" "Prometheus/2.33.4" 2022-09-29T06:21:21.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:21:21 smithi083 ceph-mon[79930]: pgmap v1595: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:21:21.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:21:21 smithi083 ceph-mon[77242]: pgmap v1595: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:21:21.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:21:21 smithi153 ceph-mon[70044]: pgmap v1595: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:21:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:21:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:21:22.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:21:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:21:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:21:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:21:23.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:21:23 smithi083 ceph-mon[79930]: pgmap v1596: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:21:23.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:21:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:21:23.715Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:21:23.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:21:23 smithi083 ceph-mon[77242]: pgmap v1596: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:21:24.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:21:23 smithi153 ceph-mon[70044]: pgmap v1596: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:21:24.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:21:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:21:24.411Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:21:25.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:21:25 smithi083 ceph-mon[79930]: pgmap v1597: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:21:25.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:21:25 smithi083 ceph-mon[77242]: pgmap v1597: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:21:26.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:21:25 smithi153 ceph-mon[70044]: pgmap v1597: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:21:27.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:21:27 smithi083 ceph-mon[79930]: pgmap v1598: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:21:27.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:21:27 smithi083 ceph-mon[77242]: pgmap v1598: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:21:28.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:21:27 smithi153 ceph-mon[70044]: pgmap v1598: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:21:28.985 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:21:29.272 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:21:29.273 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (55m) 7m ago 55m 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:21:29.273 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (55m) 5m ago 55m 61.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:21:29.273 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (56m) 7m ago 56m 23.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:21:29.274 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (56m) 5m ago 56m 27.6M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:21:29.274 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (52m) 5m ago 59m 439M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:21:29.274 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (52m) 7m ago 60m 550M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:21:29.275 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (51m) 7m ago 60m 241M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:21:29.275 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (51m) 5m ago 59m 115M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:21:29.275 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (51m) 7m ago 59m 144M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:21:29.275 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (55m) 7m ago 55m 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:21:29.276 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (55m) 5m ago 55m 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:21:29.276 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (50m) 7m ago 58m 183M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:21:29.276 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (50m) 7m ago 58m 176M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:21:29.276 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (49m) 7m ago 58m 144M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:21:29.277 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (49m) 7m ago 57m 166M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:21:29.277 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (48m) 5m ago 57m 154M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:21:29.277 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (48m) 5m ago 57m 180M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:21:29.278 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (47m) 5m ago 57m 159M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:21:29.278 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (47m) 5m ago 56m 180M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:21:29.278 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (52m) 5m ago 55m 102M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:21:29.632 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:21:29.632 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:21:29.632 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:21:29.633 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:21:29.633 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:21:29.633 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:21:29.633 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:21:29.633 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:21:29.634 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:21:29.634 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:21:29.634 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:21:29.634 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:21:29.635 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:21:29.635 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:21:29.635 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:21:29.635 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:21:29.635 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:21:29.636 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:21:29.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:21:29 smithi083 ceph-mon[79930]: pgmap v1599: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:21:29.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:21:29 smithi083 ceph-mon[79930]: from='client.35997 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:21:29.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:21:29 smithi083 ceph-mon[77242]: pgmap v1599: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:21:29.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:21:29 smithi083 ceph-mon[77242]: from='client.35997 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:21:30.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:21:29 smithi153 ceph-mon[70044]: pgmap v1599: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:21:30.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:21:29 smithi153 ceph-mon[70044]: from='client.35997 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:21:30.595 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:21:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:21:30.306Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:21:30.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:21:30 smithi083 ceph-mon[79930]: from='client.36003 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:21:30.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:21:30 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/791373652' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:21:30.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:21:30 smithi083 ceph-mon[77242]: from='client.36003 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:21:30.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:21:30 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/791373652' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:21:31.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:21:30 smithi153 ceph-mon[70044]: from='client.36003 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:21:31.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:21:30 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/791373652' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:21:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:21:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:21:31] "GET /metrics HTTP/1.1" 200 225175 "" "Prometheus/2.33.4" 2022-09-29T06:21:31.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:21:31 smithi083 ceph-mon[79930]: pgmap v1600: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:21:31.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:21:31 smithi083 ceph-mon[77242]: pgmap v1600: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:21:32.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:21:31 smithi153 ceph-mon[70044]: pgmap v1600: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:21:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:21:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:21:32.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:21:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:21:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:21:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:21:33.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:21:33 smithi083 ceph-mon[77242]: pgmap v1601: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:21:33.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:21:33 smithi083 ceph-mon[79930]: pgmap v1601: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:21:33.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:21:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:21:33.716Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:21:34.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:21:33 smithi153 ceph-mon[70044]: pgmap v1601: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:21:34.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:21:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:21:34.412Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:21:35.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:21:35 smithi083 ceph-mon[79930]: pgmap v1602: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:21:35.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:21:35 smithi083 ceph-mon[77242]: pgmap v1602: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:21:36.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:21:35 smithi153 ceph-mon[70044]: pgmap v1602: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:21:37.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:21:37 smithi083 ceph-mon[79930]: pgmap v1603: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:21:37.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:21:37 smithi083 ceph-mon[77242]: pgmap v1603: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:21:38.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:21:37 smithi153 ceph-mon[70044]: pgmap v1603: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:21:39.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:21:39 smithi083 ceph-mon[79930]: pgmap v1604: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:21:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:21:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:21:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:21:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:21:39.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:21:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:21:39.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:21:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:21:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:21:39 smithi083 ceph-mon[77242]: pgmap v1604: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:21:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:21:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:21:39.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:21:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:21:39.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:21:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:21:39.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:21:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:21:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:21:39 smithi153 ceph-mon[70044]: pgmap v1604: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:21:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:21:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:21:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:21:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:21:40.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:21:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:21:40.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:21:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:21:40.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:21:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:21:40.308Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:21:41.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:21:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:21:41] "GET /metrics HTTP/1.1" 200 225173 "" "Prometheus/2.33.4" 2022-09-29T06:21:41.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:21:41 smithi083 ceph-mon[79930]: pgmap v1605: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:21:41.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:21:41 smithi083 ceph-mon[77242]: pgmap v1605: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:21:41.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:21:41 smithi153 ceph-mon[70044]: pgmap v1605: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:21:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:21:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:21:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:21:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:21:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:21:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:21:43.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:21:43 smithi083 ceph-mon[79930]: pgmap v1606: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:21:43.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:21:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:21:43.717Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:21:43.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:21:43 smithi083 ceph-mon[77242]: pgmap v1606: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:21:44.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:21:43 smithi153 ceph-mon[70044]: pgmap v1606: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:21:44.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:21:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:21:44.413Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:21:45.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:21:45 smithi083 ceph-mon[79930]: pgmap v1607: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:21:45.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:21:45 smithi083 ceph-mon[77242]: pgmap v1607: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:21:46.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:21:45 smithi153 ceph-mon[70044]: pgmap v1607: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:21:46.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:21:46 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:21:46.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:21:46 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:21:46.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:21:46 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:21:46.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:21:46 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:21:46.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:21:46 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:21:46.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:21:46 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:21:47.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:21:46 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:21:47.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:21:46 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:21:47.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:21:46 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:21:47.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:21:47 smithi083 ceph-mon[79930]: pgmap v1608: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:21:47.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:21:47 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:21:47.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:21:47 smithi083 ceph-mon[77242]: pgmap v1608: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:21:47.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:21:47 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:21:48.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:21:47 smithi153 ceph-mon[70044]: pgmap v1608: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:21:48.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:21:47 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:21:49.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:21:49 smithi083 ceph-mon[79930]: pgmap v1609: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:21:49.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:21:49 smithi083 ceph-mon[77242]: pgmap v1609: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:21:50.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:21:49 smithi153 ceph-mon[70044]: pgmap v1609: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:21:50.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:21:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:21:50.308Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:21:51.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:21:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:21:51] "GET /metrics HTTP/1.1" 200 225173 "" "Prometheus/2.33.4" 2022-09-29T06:21:51.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:21:51 smithi083 ceph-mon[79930]: pgmap v1610: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:21:51.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:21:51 smithi083 ceph-mon[77242]: pgmap v1610: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:21:51.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:21:51 smithi153 ceph-mon[70044]: pgmap v1610: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:21:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:21:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:21:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:21:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:21:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:21:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:21:53.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:21:53 smithi083 ceph-mon[79930]: pgmap v1611: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:21:53.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:21:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:21:53.718Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:21:53.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:21:53 smithi083 ceph-mon[77242]: pgmap v1611: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:21:54.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:21:53 smithi153 ceph-mon[70044]: pgmap v1611: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:21:54.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:21:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:21:54.414Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:21:55.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:21:55 smithi083 ceph-mon[79930]: pgmap v1612: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:21:55.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:21:55 smithi083 ceph-mon[77242]: pgmap v1612: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:21:56.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:21:55 smithi153 ceph-mon[70044]: pgmap v1612: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:21:57.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:21:57 smithi083 ceph-mon[79930]: pgmap v1613: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:21:57.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:21:57 smithi083 ceph-mon[77242]: pgmap v1613: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:21:58.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:21:57 smithi153 ceph-mon[70044]: pgmap v1613: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:21:59.959 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:21:59.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:21:59 smithi083 ceph-mon[79930]: pgmap v1614: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:21:59.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:21:59 smithi083 ceph-mon[77242]: pgmap v1614: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:00.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:21:59 smithi153 ceph-mon[70044]: pgmap v1614: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:00.251 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:22:00.251 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (55m) 8m ago 56m 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:22:00.251 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (55m) 6m ago 55m 61.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:22:00.252 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (57m) 8m ago 57m 23.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:22:00.252 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (57m) 6m ago 57m 27.6M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:22:00.252 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (52m) 6m ago 59m 439M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:22:00.252 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (53m) 8m ago 61m 550M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:22:00.253 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (52m) 8m ago 61m 241M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:22:00.253 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (51m) 6m ago 59m 115M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:22:00.253 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (52m) 8m ago 59m 144M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:22:00.253 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (56m) 8m ago 56m 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:22:00.253 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (56m) 6m ago 56m 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:22:00.254 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (51m) 8m ago 59m 183M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:22:00.254 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (50m) 8m ago 59m 176M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:22:00.254 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (50m) 8m ago 58m 144M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:22:00.254 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (49m) 8m ago 58m 166M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:22:00.255 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (49m) 6m ago 58m 154M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:22:00.255 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (48m) 6m ago 57m 180M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:22:00.255 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (48m) 6m ago 57m 159M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:22:00.255 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (47m) 6m ago 57m 180M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:22:00.256 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (52m) 6m ago 56m 102M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:22:00.602 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:22:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:22:00.307Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:22:00.610 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:22:00.610 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:22:00.610 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:22:00.610 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:22:00.611 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:22:00.611 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:22:00.611 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:22:00.611 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:22:00.612 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:22:00.612 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:22:00.612 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:22:00.612 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:22:00.613 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:22:00.613 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:22:00.613 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:22:00.613 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:22:00.613 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:22:00.614 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:22:00.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:22:00 smithi083 ceph-mon[77242]: from='client.36015 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:22:00.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:22:00 smithi083 ceph-mon[79930]: from='client.36015 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:22:01.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:22:00 smithi153 ceph-mon[70044]: from='client.36015 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:22:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:22:01] "GET /metrics HTTP/1.1" 200 225177 "" "Prometheus/2.33.4" 2022-09-29T06:22:01.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:22:01 smithi083 ceph-mon[79930]: from='client.36021 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:22:01.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:22:01 smithi083 ceph-mon[79930]: pgmap v1615: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:01.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:22:01 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4123527308' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:22:01.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:22:01 smithi083 ceph-mon[77242]: from='client.36021 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:22:01.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:22:01 smithi083 ceph-mon[77242]: pgmap v1615: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:01.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:22:01 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4123527308' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:22:02.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:22:01 smithi153 ceph-mon[70044]: from='client.36021 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:22:02.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:22:01 smithi153 ceph-mon[70044]: pgmap v1615: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:02.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:22:01 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4123527308' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:22:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:22:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:22:02.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:22:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:22:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:22:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:22:03.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:22:03 smithi083 ceph-mon[79930]: pgmap v1616: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:03.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:22:03 smithi083 ceph-mon[77242]: pgmap v1616: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:03.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:22:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:22:03.718Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:22:04.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:22:03 smithi153 ceph-mon[70044]: pgmap v1616: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:04.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:22:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:22:04.415Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:22:05.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:22:05 smithi083 ceph-mon[79930]: pgmap v1617: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:05.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:22:05 smithi083 ceph-mon[77242]: pgmap v1617: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:06.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:22:05 smithi153 ceph-mon[70044]: pgmap v1617: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:07.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:22:07 smithi083 ceph-mon[79930]: pgmap v1618: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:07.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:22:07 smithi083 ceph-mon[77242]: pgmap v1618: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:08.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:22:07 smithi153 ceph-mon[70044]: pgmap v1618: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:09.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:22:09 smithi083 ceph-mon[79930]: pgmap v1619: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:09.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:22:09 smithi083 ceph-mon[77242]: pgmap v1619: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:10.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:22:09 smithi153 ceph-mon[70044]: pgmap v1619: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:10.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:22:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:22:10.310Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:22:11.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:22:11 smithi083 ceph-mon[77242]: pgmap v1620: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:11.715 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:22:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:22:11] "GET /metrics HTTP/1.1" 200 225176 "" "Prometheus/2.33.4" 2022-09-29T06:22:11.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:22:11 smithi083 ceph-mon[79930]: pgmap v1620: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:11.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:22:11 smithi153 ceph-mon[70044]: pgmap v1620: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:22:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:22:12.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:22:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:22:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:22:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:22:13.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:22:13 smithi083 ceph-mon[77242]: pgmap v1621: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:13.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:22:13 smithi083 ceph-mon[79930]: pgmap v1621: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:13.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:22:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:22:13.720Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:22:14.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:22:13 smithi153 ceph-mon[70044]: pgmap v1621: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:14.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:22:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:22:14.416Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:22:15.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:22:15 smithi083 ceph-mon[77242]: pgmap v1622: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:15.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:22:15 smithi083 ceph-mon[79930]: pgmap v1622: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:16.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:22:15 smithi153 ceph-mon[70044]: pgmap v1622: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:17.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:22:17 smithi083 ceph-mon[77242]: pgmap v1623: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:17.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:22:17 smithi083 ceph-mon[79930]: pgmap v1623: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:18.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:22:17 smithi153 ceph-mon[70044]: pgmap v1623: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:19.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:22:19 smithi083 ceph-mon[79930]: pgmap v1624: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:19.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:22:19 smithi083 ceph-mon[77242]: pgmap v1624: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:20.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:22:19 smithi153 ceph-mon[70044]: pgmap v1624: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:20.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:22:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:22:20.310Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:22:21.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:22:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:22:21] "GET /metrics HTTP/1.1" 200 225176 "" "Prometheus/2.33.4" 2022-09-29T06:22:21.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:22:21 smithi083 ceph-mon[79930]: pgmap v1625: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:21.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:22:21 smithi083 ceph-mon[77242]: pgmap v1625: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:21.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:22:21 smithi153 ceph-mon[70044]: pgmap v1625: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:22:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:22:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:22:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:22:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:22:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:22:23.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:22:23 smithi083 ceph-mon[79930]: pgmap v1626: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:23.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:22:23 smithi083 ceph-mon[77242]: pgmap v1626: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:23.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:22:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:22:23.721Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:22:24.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:22:23 smithi153 ceph-mon[70044]: pgmap v1626: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:24.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:22:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:22:24.416Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:22:25.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:22:25 smithi083 ceph-mon[79930]: pgmap v1627: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:25.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:22:25 smithi083 ceph-mon[77242]: pgmap v1627: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:26.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:22:25 smithi153 ceph-mon[70044]: pgmap v1627: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:27.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:22:27 smithi083 ceph-mon[77242]: pgmap v1628: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:27.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:22:27 smithi083 ceph-mon[79930]: pgmap v1628: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:28.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:22:27 smithi153 ceph-mon[70044]: pgmap v1628: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:29.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:22:29 smithi083 ceph-mon[79930]: pgmap v1629: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:29.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:22:29 smithi083 ceph-mon[77242]: pgmap v1629: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:30.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:22:29 smithi153 ceph-mon[70044]: pgmap v1629: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:30.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:22:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:22:30.310Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:22:30.934 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:22:31.222 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:22:31.222 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (56m) 8m ago 56m 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:22:31.222 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (56m) 6m ago 56m 61.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:22:31.223 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (57m) 8m ago 57m 23.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:22:31.223 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (57m) 6m ago 57m 27.6M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:22:31.223 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (53m) 6m ago 60m 439M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:22:31.223 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (53m) 8m ago 61m 550M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:22:31.224 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (53m) 8m ago 61m 241M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:22:31.224 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (52m) 6m ago 60m 115M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:22:31.224 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (52m) 8m ago 60m 144M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:22:31.224 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (56m) 8m ago 56m 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:22:31.225 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (56m) 6m ago 56m 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:22:31.225 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (51m) 8m ago 59m 183M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:22:31.225 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (51m) 8m ago 59m 176M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:22:31.225 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (50m) 8m ago 59m 144M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:22:31.225 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (50m) 8m ago 58m 166M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:22:31.226 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (49m) 6m ago 58m 154M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:22:31.226 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (49m) 6m ago 58m 180M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:22:31.226 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (48m) 6m ago 58m 159M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:22:31.226 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (48m) 6m ago 57m 180M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:22:31.227 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (53m) 6m ago 57m 102M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:22:31.584 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:22:31.584 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:22:31.585 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:22:31.585 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:22:31.585 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:22:31.585 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:22:31.586 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:22:31.586 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:22:31.586 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:22:31.586 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:22:31.587 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:22:31.587 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:22:31.587 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:22:31.587 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:22:31.587 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:22:31.588 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:22:31.588 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:22:31.588 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:22:31.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:22:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:22:31] "GET /metrics HTTP/1.1" 200 225176 "" "Prometheus/2.33.4" 2022-09-29T06:22:31.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:22:31 smithi083 ceph-mon[79930]: pgmap v1630: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:31.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:22:31 smithi083 ceph-mon[79930]: from='client.45967 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:22:31.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:22:31 smithi083 ceph-mon[79930]: from='client.36039 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:22:31.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:22:31 smithi083 ceph-mon[77242]: pgmap v1630: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:31.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:22:31 smithi083 ceph-mon[77242]: from='client.45967 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:22:31.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:22:31 smithi083 ceph-mon[77242]: from='client.36039 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:22:31.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:22:31 smithi153 ceph-mon[70044]: pgmap v1630: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:31.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:22:31 smithi153 ceph-mon[70044]: from='client.45967 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:22:31.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:22:31 smithi153 ceph-mon[70044]: from='client.36039 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:22:32.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:22:32 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1470743926' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:22:32.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:22:32 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1470743926' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:22:32.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:22:32 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1470743926' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:22:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:22:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:22:32.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:22:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:22:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:22:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:22:33.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:22:33 smithi083 ceph-mon[77242]: pgmap v1631: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:33.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:22:33 smithi083 ceph-mon[79930]: pgmap v1631: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:33.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:22:33 smithi153 ceph-mon[70044]: pgmap v1631: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:34.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:22:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:22:33.722Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:22:34.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:22:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:22:34.416Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:22:35.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:22:35 smithi083 ceph-mon[77242]: pgmap v1632: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:35.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:22:35 smithi083 ceph-mon[79930]: pgmap v1632: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:36.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:22:35 smithi153 ceph-mon[70044]: pgmap v1632: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:37.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:22:37 smithi083 ceph-mon[77242]: pgmap v1633: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:37.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:22:37 smithi083 ceph-mon[79930]: pgmap v1633: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:38.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:22:37 smithi153 ceph-mon[70044]: pgmap v1633: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:39.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:22:39 smithi083 ceph-mon[79930]: pgmap v1634: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:22:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:22:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:22:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:22:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:22:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:22:39.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:22:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:22:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:22:39 smithi083 ceph-mon[77242]: pgmap v1634: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:22:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:22:39.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:22:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:22:39.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:22:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:22:39.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:22:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:22:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:22:39 smithi153 ceph-mon[70044]: pgmap v1634: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:22:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:22:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:22:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:22:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:22:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:22:40.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:22:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:22:40.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:22:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:22:40.311Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:22:41.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:22:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:22:41] "GET /metrics HTTP/1.1" 200 225173 "" "Prometheus/2.33.4" 2022-09-29T06:22:41.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:22:41 smithi083 ceph-mon[79930]: pgmap v1635: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:41.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:22:41 smithi083 ceph-mon[77242]: pgmap v1635: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:41.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:22:41 smithi153 ceph-mon[70044]: pgmap v1635: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:22:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:22:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:22:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:22:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:22:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:22:43.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:22:43 smithi083 ceph-mon[79930]: pgmap v1636: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:43.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:22:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:22:43.722Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:22:43.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:22:43 smithi083 ceph-mon[77242]: pgmap v1636: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:44.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:22:43 smithi153 ceph-mon[70044]: pgmap v1636: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:44.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:22:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:22:44.418Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:22:45.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:22:45 smithi083 ceph-mon[77242]: pgmap v1637: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:45.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:22:45 smithi083 ceph-mon[79930]: pgmap v1637: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:46.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:22:45 smithi153 ceph-mon[70044]: pgmap v1637: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:47.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:22:47 smithi083 ceph-mon[79930]: pgmap v1638: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:47.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:22:47 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:22:47.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:22:47 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:22:47.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:22:47 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:22:47.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:22:47 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:22:47.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:22:47 smithi083 ceph-mon[77242]: pgmap v1638: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:47.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:22:47 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:22:47.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:22:47 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:22:47.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:22:47 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:22:47.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:22:47 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:22:48.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:22:47 smithi153 ceph-mon[70044]: pgmap v1638: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:48.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:22:47 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:22:48.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:22:47 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:22:48.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:22:47 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:22:48.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:22:47 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:22:49.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:22:49 smithi083 ceph-mon[79930]: pgmap v1639: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:49.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:22:49 smithi083 ceph-mon[77242]: pgmap v1639: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:50.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:22:49 smithi153 ceph-mon[70044]: pgmap v1639: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:50.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:22:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:22:50.311Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:22:51.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:22:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:22:51] "GET /metrics HTTP/1.1" 200 225173 "" "Prometheus/2.33.4" 2022-09-29T06:22:51.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:22:51 smithi083 ceph-mon[77242]: pgmap v1640: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:51.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:22:51 smithi083 ceph-mon[79930]: pgmap v1640: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:51.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:22:51 smithi153 ceph-mon[70044]: pgmap v1640: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:22:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:22:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:22:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:22:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:22:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:22:53.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:22:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:22:53.722Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:22:53.977 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:22:53 smithi083 ceph-mon[77242]: pgmap v1641: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:53.977 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:22:53 smithi083 ceph-mon[79930]: pgmap v1641: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:54.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:22:53 smithi153 ceph-mon[70044]: pgmap v1641: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:54.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:22:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:22:54.417Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:22:55.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:22:55 smithi083 ceph-mon[79930]: pgmap v1642: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:55.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:22:55 smithi083 ceph-mon[77242]: pgmap v1642: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:56.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:22:55 smithi153 ceph-mon[70044]: pgmap v1642: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:57.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:22:57 smithi083 ceph-mon[77242]: pgmap v1643: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:57.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:22:57 smithi083 ceph-mon[79930]: pgmap v1643: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:58.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:22:57 smithi153 ceph-mon[70044]: pgmap v1643: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:59.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:22:59 smithi083 ceph-mon[77242]: pgmap v1644: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:22:59.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:22:59 smithi083 ceph-mon[79930]: pgmap v1644: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:00.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:22:59 smithi153 ceph-mon[70044]: pgmap v1644: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:00.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:23:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:23:00.312Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:23:01.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:23:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:23:01] "GET /metrics HTTP/1.1" 200 225174 "" "Prometheus/2.33.4" 2022-09-29T06:23:01.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:23:01 smithi083 ceph-mon[79930]: pgmap v1645: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:01.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:23:01 smithi083 ceph-mon[77242]: pgmap v1645: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:01.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:23:01 smithi153 ceph-mon[70044]: pgmap v1645: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:01.913 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:23:02.203 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:23:02.203 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (56m) 9m ago 57m 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:23:02.204 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (57m) 7m ago 56m 61.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:23:02.204 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (58m) 9m ago 58m 23.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:23:02.204 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (58m) 7m ago 58m 27.6M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:23:02.204 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (53m) 7m ago 60m 439M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:23:02.205 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (54m) 9m ago 62m 550M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:23:02.205 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (53m) 9m ago 62m 241M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:23:02.205 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (52m) 7m ago 60m 115M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:23:02.205 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (53m) 9m ago 61m 144M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:23:02.206 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (57m) 9m ago 57m 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:23:02.206 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (57m) 7m ago 57m 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:23:02.206 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (52m) 9m ago 60m 183M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:23:02.206 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (51m) 9m ago 60m 176M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:23:02.207 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (51m) 9m ago 59m 144M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:23:02.207 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (50m) 9m ago 59m 166M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:23:02.207 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (50m) 7m ago 59m 154M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:23:02.207 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (49m) 7m ago 58m 180M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:23:02.208 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (49m) 7m ago 58m 159M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:23:02.208 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (48m) 7m ago 58m 180M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:23:02.208 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (53m) 7m ago 57m 102M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:23:02.560 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:23:02.560 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:23:02.560 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:23:02.560 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:23:02.561 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:23:02.561 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:23:02.561 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:23:02.561 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:23:02.562 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:23:02.562 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:23:02.562 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:23:02.562 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:23:02.563 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:23:02.563 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:23:02.563 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:23:02.563 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:23:02.563 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:23:02.564 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:23:02.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:23:02 smithi083 ceph-mon[79930]: from='client.36051 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:23:02.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:23:02 smithi083 ceph-mon[79930]: from='client.36057 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:23:02.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:23:02 smithi083 ceph-mon[77242]: from='client.36051 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:23:02.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:23:02 smithi083 ceph-mon[77242]: from='client.36057 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:23:02.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:23:02 smithi153 ceph-mon[70044]: from='client.36051 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:23:02.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:23:02 smithi153 ceph-mon[70044]: from='client.36057 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:23:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:23:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:23:02.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:23:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:23:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:23:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:23:03.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:23:03 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3213644300' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:23:03.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:23:03 smithi083 ceph-mon[79930]: pgmap v1646: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:03.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:23:03 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3213644300' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:23:03.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:23:03 smithi083 ceph-mon[77242]: pgmap v1646: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:03.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:23:03 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3213644300' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:23:03.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:23:03 smithi153 ceph-mon[70044]: pgmap v1646: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:04.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:23:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:23:03.723Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:23:04.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:23:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:23:04.418Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:23:05.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:23:05 smithi083 ceph-mon[77242]: pgmap v1647: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:05.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:23:05 smithi083 ceph-mon[79930]: pgmap v1647: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:06.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:23:05 smithi153 ceph-mon[70044]: pgmap v1647: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:07.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:23:07 smithi083 ceph-mon[79930]: pgmap v1648: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:07.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:23:07 smithi083 ceph-mon[77242]: pgmap v1648: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:08.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:23:07 smithi153 ceph-mon[70044]: pgmap v1648: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:09.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:23:09 smithi083 ceph-mon[79930]: pgmap v1649: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:09.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:23:09 smithi083 ceph-mon[77242]: pgmap v1649: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:10.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:23:09 smithi153 ceph-mon[70044]: pgmap v1649: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:10.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:23:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:23:10.313Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:23:11.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:23:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:23:11] "GET /metrics HTTP/1.1" 200 225166 "" "Prometheus/2.33.4" 2022-09-29T06:23:11.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:23:11 smithi083 ceph-mon[79930]: pgmap v1650: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:11.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:23:11 smithi083 ceph-mon[77242]: pgmap v1650: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:11.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:23:11 smithi153 ceph-mon[70044]: pgmap v1650: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:23:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:23:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:23:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:23:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:23:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:23:13.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:23:13 smithi083 ceph-mon[79930]: pgmap v1651: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:13.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:23:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:23:13.724Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:23:13.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:23:13 smithi083 ceph-mon[77242]: pgmap v1651: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:14.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:23:13 smithi153 ceph-mon[70044]: pgmap v1651: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:14.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:23:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:23:14.419Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:23:15.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:23:15 smithi083 ceph-mon[79930]: pgmap v1652: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:15.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:23:15 smithi083 ceph-mon[77242]: pgmap v1652: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:16.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:23:15 smithi153 ceph-mon[70044]: pgmap v1652: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:17.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:23:17 smithi083 ceph-mon[79930]: pgmap v1653: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:17.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:23:17 smithi083 ceph-mon[77242]: pgmap v1653: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:18.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:23:17 smithi153 ceph-mon[70044]: pgmap v1653: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:19.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:23:19 smithi083 ceph-mon[79930]: pgmap v1654: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:19.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:23:19 smithi083 ceph-mon[77242]: pgmap v1654: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:20.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:23:19 smithi153 ceph-mon[70044]: pgmap v1654: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:20.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:23:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:23:20.313Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:23:21.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:23:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:23:21] "GET /metrics HTTP/1.1" 200 225166 "" "Prometheus/2.33.4" 2022-09-29T06:23:21.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:23:21 smithi083 ceph-mon[79930]: pgmap v1655: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:21.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:23:21 smithi083 ceph-mon[77242]: pgmap v1655: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:21.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:23:21 smithi153 ceph-mon[70044]: pgmap v1655: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:23:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:23:22.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:23:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:23:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:23:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:23:23.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:23:23 smithi083 ceph-mon[79930]: pgmap v1656: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:23.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:23:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:23:23.724Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:23:23.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:23:23 smithi083 ceph-mon[77242]: pgmap v1656: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:24.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:23:23 smithi153 ceph-mon[70044]: pgmap v1656: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:24.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:23:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:23:24.420Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:23:25.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:23:25 smithi083 ceph-mon[79930]: pgmap v1657: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:25.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:23:25 smithi083 ceph-mon[77242]: pgmap v1657: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:26.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:23:25 smithi153 ceph-mon[70044]: pgmap v1657: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:27.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:23:27 smithi083 ceph-mon[79930]: pgmap v1658: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:27.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:23:27 smithi083 ceph-mon[77242]: pgmap v1658: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:28.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:23:27 smithi153 ceph-mon[70044]: pgmap v1658: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:29.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:23:29 smithi083 ceph-mon[79930]: pgmap v1659: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:29.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:23:29 smithi083 ceph-mon[77242]: pgmap v1659: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:30.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:23:29 smithi153 ceph-mon[70044]: pgmap v1659: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:30.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:23:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:23:30.314Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:23:31.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:23:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:23:31] "GET /metrics HTTP/1.1" 200 225176 "" "Prometheus/2.33.4" 2022-09-29T06:23:31.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:23:31 smithi083 ceph-mon[79930]: pgmap v1660: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:31.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:23:31 smithi083 ceph-mon[77242]: pgmap v1660: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:31.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:23:31 smithi153 ceph-mon[70044]: pgmap v1660: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:23:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:23:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:23:32.889 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:23:33.176 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:23:33.176 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (57m) 9m ago 57m 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:23:33.177 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (57m) 7m ago 57m 61.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:23:33.177 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (58m) 9m ago 58m 23.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:23:33.177 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (58m) 7m ago 58m 27.6M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:23:33.177 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (54m) 7m ago 61m 439M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:23:33.178 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (54m) 9m ago 62m 550M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:23:33.178 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (54m) 9m ago 62m 241M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:23:33.179 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (53m) 7m ago 61m 115M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:23:33.179 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (53m) 9m ago 61m 144M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:23:33.179 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (57m) 9m ago 57m 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:23:33.179 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (57m) 7m ago 57m 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:23:33.180 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (52m) 9m ago 60m 183M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:23:33.180 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (52m) 9m ago 60m 176M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:23:33.180 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (51m) 9m ago 60m 144M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:23:33.180 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (51m) 9m ago 59m 166M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:23:33.180 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (50m) 7m ago 59m 154M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:23:33.181 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (50m) 7m ago 59m 180M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:23:33.181 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (49m) 7m ago 59m 159M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:23:33.181 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (49m) 7m ago 58m 180M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:23:33.181 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (54m) 7m ago 58m 102M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:23:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:23:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:23:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:23:33.546 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:23:33.546 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:23:33.546 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:23:33.547 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:23:33.547 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:23:33.547 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:23:33.547 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:23:33.547 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:23:33.548 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:23:33.548 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:23:33.548 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:23:33.548 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:23:33.549 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:23:33.549 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:23:33.549 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:23:33.549 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:23:33.550 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:23:33.550 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:23:33.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:23:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:23:33.724Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:23:33.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:23:33 smithi083 ceph-mon[77242]: pgmap v1661: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:33.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:23:33 smithi083 ceph-mon[77242]: from='client.36069 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:23:33.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:23:33 smithi083 ceph-mon[77242]: from='client.36075 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:23:33.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:23:33 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2883305405' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:23:33.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:23:33 smithi083 ceph-mon[79930]: pgmap v1661: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:33.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:23:33 smithi083 ceph-mon[79930]: from='client.36069 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:23:33.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:23:33 smithi083 ceph-mon[79930]: from='client.36075 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:23:33.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:23:33 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2883305405' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:23:34.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:23:33 smithi153 ceph-mon[70044]: pgmap v1661: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:34.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:23:33 smithi153 ceph-mon[70044]: from='client.36069 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:23:34.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:23:33 smithi153 ceph-mon[70044]: from='client.36075 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:23:34.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:23:33 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2883305405' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:23:34.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:23:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:23:34.420Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:23:35.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:23:35 smithi083 ceph-mon[79930]: pgmap v1662: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:35.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:23:35 smithi083 ceph-mon[77242]: pgmap v1662: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:36.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:23:35 smithi153 ceph-mon[70044]: pgmap v1662: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:37.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:23:37 smithi083 ceph-mon[77242]: pgmap v1663: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:37.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:23:37 smithi083 ceph-mon[79930]: pgmap v1663: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:38.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:23:37 smithi153 ceph-mon[70044]: pgmap v1663: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:39.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:23:39 smithi083 ceph-mon[77242]: pgmap v1664: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:39.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:23:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:23:39.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:23:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:23:39.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:23:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:23:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:23:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:23:39.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:23:39 smithi083 ceph-mon[79930]: pgmap v1664: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:39.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:23:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:23:39.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:23:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:23:39.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:23:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:23:39.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:23:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:23:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:23:39 smithi153 ceph-mon[70044]: pgmap v1664: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:23:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:23:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:23:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:23:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:23:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:23:40.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:23:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:23:40.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:23:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:23:40.314Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:23:41.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:23:41 smithi083 ceph-mon[77242]: pgmap v1665: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:41.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:23:41 smithi083 ceph-mon[79930]: pgmap v1665: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:41.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:23:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:23:41] "GET /metrics HTTP/1.1" 200 225177 "" "Prometheus/2.33.4" 2022-09-29T06:23:41.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:23:41 smithi153 ceph-mon[70044]: pgmap v1665: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:23:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:23:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:23:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:23:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:23:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:23:43.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:23:43 smithi083 ceph-mon[79930]: pgmap v1666: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:43.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:23:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:23:43.726Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:23:43.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:23:43 smithi083 ceph-mon[77242]: pgmap v1666: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:44.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:23:43 smithi153 ceph-mon[70044]: pgmap v1666: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:44.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:23:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:23:44.422Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:23:45.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:23:45 smithi083 ceph-mon[79930]: pgmap v1667: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:45.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:23:45 smithi083 ceph-mon[77242]: pgmap v1667: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:46.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:23:45 smithi153 ceph-mon[70044]: pgmap v1667: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:47.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:23:47 smithi083 ceph-mon[79930]: pgmap v1668: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:47.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:23:47 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:23:47.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:23:47 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:23:47.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:23:47 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:23:47.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:23:47 smithi083 ceph-mon[77242]: pgmap v1668: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:47.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:23:47 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:23:47.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:23:47 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:23:47.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:23:47 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:23:48.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:23:47 smithi153 ceph-mon[70044]: pgmap v1668: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:48.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:23:47 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:23:48.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:23:47 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:23:48.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:23:47 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:23:49.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:23:48 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:23:49.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:23:48 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:23:49.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:23:48 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:23:49.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:23:48 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:23:49.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:23:48 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:23:49.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:23:48 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:23:49.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:23:48 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:23:49.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:23:48 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:23:49.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:23:48 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:23:49.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:23:48 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:23:49.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:23:48 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:23:49.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:23:48 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:23:50.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:23:49 smithi083 ceph-mon[79930]: pgmap v1669: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:50.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:23:49 smithi083 ceph-mon[77242]: pgmap v1669: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:50.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:23:49 smithi153 ceph-mon[70044]: pgmap v1669: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:50.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:23:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:23:50.315Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:23:51.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:23:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:23:51] "GET /metrics HTTP/1.1" 200 225177 "" "Prometheus/2.33.4" 2022-09-29T06:23:51.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:23:51 smithi083 ceph-mon[79930]: pgmap v1670: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:51.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:23:51 smithi083 ceph-mon[77242]: pgmap v1670: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:51.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:23:51 smithi153 ceph-mon[70044]: pgmap v1670: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:23:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:23:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:23:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:23:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:23:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:23:53.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:23:53 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:23:53.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:23:53 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:23:53.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:23:53 smithi083 ceph-mon[77242]: pgmap v1671: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:53.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:23:53 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:23:53.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:23:53 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:23:53.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:23:53 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:23:53.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:23:53 smithi083 ceph-mon[79930]: pgmap v1671: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:53.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:23:53 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:23:53.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:23:53 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:23:53.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:23:53 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:23:53.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:23:53 smithi153 ceph-mon[70044]: pgmap v1671: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:53.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:23:53 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:23:54.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:23:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:23:53.726Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:23:54.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:23:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:23:54.423Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:23:55.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:23:55 smithi083 ceph-mon[79930]: pgmap v1672: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:55.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:23:55 smithi083 ceph-mon[77242]: pgmap v1672: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:56.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:23:55 smithi153 ceph-mon[70044]: pgmap v1672: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:57.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:23:57 smithi083 ceph-mon[77242]: pgmap v1673: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:57.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:23:57 smithi083 ceph-mon[79930]: pgmap v1673: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:58.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:23:57 smithi153 ceph-mon[70044]: pgmap v1673: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:59.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:23:59 smithi083 ceph-mon[77242]: pgmap v1674: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:23:59.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:23:59 smithi083 ceph-mon[79930]: pgmap v1674: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:00.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:23:59 smithi153 ceph-mon[70044]: pgmap v1674: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:00.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:24:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:24:00.316Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:24:01.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:24:01 smithi083 ceph-mon[77242]: pgmap v1675: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:01.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:24:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:24:01] "GET /metrics HTTP/1.1" 200 225177 "" "Prometheus/2.33.4" 2022-09-29T06:24:01.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:24:01 smithi083 ceph-mon[79930]: pgmap v1675: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:01.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:24:01 smithi153 ceph-mon[70044]: pgmap v1675: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:24:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:24:02.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:24:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:24:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:24:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:24:03.887 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:24:03.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:24:03 smithi083 ceph-mon[77242]: pgmap v1676: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:03.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:24:03 smithi083 ceph-mon[79930]: pgmap v1676: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:03.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:24:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:24:03.726Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:24:04.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:24:03 smithi153 ceph-mon[70044]: pgmap v1676: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:04.176 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:24:04.176 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (57m) 11s ago 58m 26.5M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:24:04.177 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (58m) 8m ago 58m 61.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:24:04.177 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (59m) 11s ago 59m 25.3M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:24:04.177 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (59m) 8m ago 59m 27.6M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:24:04.177 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (54m) 8m ago 61m 439M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:24:04.178 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (55m) 11s ago 63m 560M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:24:04.178 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (54m) 11s ago 63m 275M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:24:04.178 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (53m) 8m ago 61m 115M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:24:04.178 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (54m) 11s ago 62m 166M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:24:04.179 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (58m) 11s ago 58m 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:24:04.179 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (58m) 8m ago 58m 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:24:04.179 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (53m) 11s ago 61m 209M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:24:04.179 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (52m) 11s ago 61m 201M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:24:04.179 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (52m) 11s ago 60m 163M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:24:04.180 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (51m) 11s ago 60m 191M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:24:04.180 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (51m) 8m ago 60m 154M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:24:04.180 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (50m) 8m ago 59m 180M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:24:04.181 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (50m) 8m ago 59m 159M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:24:04.181 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (49m) 8m ago 59m 180M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:24:04.181 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (54m) 8m ago 58m 102M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:24:04.535 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:24:04.535 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:24:04.535 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:24:04.536 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:24:04.536 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:24:04.536 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:24:04.536 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:24:04.537 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:24:04.537 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:24:04.537 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:24:04.537 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:24:04.537 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:24:04.538 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:24:04.538 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:24:04.538 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:24:04.538 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:24:04.539 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:24:04.539 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:24:04.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:24:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:24:04.423Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:24:04.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:24:04 smithi083 ceph-mon[77242]: from='client.46021 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:24:04.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:24:04 smithi083 ceph-mon[77242]: from='client.36093 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:24:04.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:24:04 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1173213673' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:24:04.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:24:04 smithi083 ceph-mon[79930]: from='client.46021 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:24:04.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:24:04 smithi083 ceph-mon[79930]: from='client.36093 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:24:04.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:24:04 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1173213673' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:24:05.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:24:04 smithi153 ceph-mon[70044]: from='client.46021 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:24:05.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:24:04 smithi153 ceph-mon[70044]: from='client.36093 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:24:05.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:24:04 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1173213673' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:24:05.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:24:05 smithi083 ceph-mon[77242]: pgmap v1677: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:05.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:24:05 smithi083 ceph-mon[79930]: pgmap v1677: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:06.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:24:05 smithi153 ceph-mon[70044]: pgmap v1677: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:07.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:24:07 smithi083 ceph-mon[77242]: pgmap v1678: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:07.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:24:07 smithi083 ceph-mon[79930]: pgmap v1678: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:08.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:24:07 smithi153 ceph-mon[70044]: pgmap v1678: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:09.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:24:09 smithi083 ceph-mon[79930]: pgmap v1679: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:09.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:24:09 smithi083 ceph-mon[77242]: pgmap v1679: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:10.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:24:09 smithi153 ceph-mon[70044]: pgmap v1679: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:10.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:24:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:24:10.317Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:24:11.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:24:11 smithi083 ceph-mon[77242]: pgmap v1680: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:11.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:24:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:24:11] "GET /metrics HTTP/1.1" 200 225166 "" "Prometheus/2.33.4" 2022-09-29T06:24:11.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:24:11 smithi083 ceph-mon[79930]: pgmap v1680: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:11.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:24:11 smithi153 ceph-mon[70044]: pgmap v1680: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:24:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:24:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:24:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:24:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:24:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:24:13.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:24:13 smithi083 ceph-mon[79930]: pgmap v1681: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:13.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:24:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:24:13.728Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:24:13.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:24:13 smithi083 ceph-mon[77242]: pgmap v1681: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:14.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:24:13 smithi153 ceph-mon[70044]: pgmap v1681: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:14.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:24:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:24:14.424Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:24:15.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:24:15 smithi083 ceph-mon[79930]: pgmap v1682: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:15.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:24:15 smithi083 ceph-mon[77242]: pgmap v1682: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:16.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:24:15 smithi153 ceph-mon[70044]: pgmap v1682: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:17.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:24:17 smithi083 ceph-mon[79930]: pgmap v1683: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:17.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:24:17 smithi083 ceph-mon[77242]: pgmap v1683: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:18.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:24:17 smithi153 ceph-mon[70044]: pgmap v1683: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:19.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:24:19 smithi083 ceph-mon[79930]: pgmap v1684: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:19.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:24:19 smithi083 ceph-mon[77242]: pgmap v1684: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:20.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:24:19 smithi153 ceph-mon[70044]: pgmap v1684: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:20.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:24:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:24:20.318Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:24:21.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:24:21 smithi083 ceph-mon[79930]: pgmap v1685: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:21.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:24:21 smithi083 ceph-mon[77242]: pgmap v1685: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:21.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:24:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:24:21] "GET /metrics HTTP/1.1" 200 225166 "" "Prometheus/2.33.4" 2022-09-29T06:24:21.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:24:21 smithi153 ceph-mon[70044]: pgmap v1685: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:24:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:24:22.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:24:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:24:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:24:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:24:23.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:24:23 smithi083 ceph-mon[79930]: pgmap v1686: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:23.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:24:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:24:23.729Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:24:23.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:24:23 smithi083 ceph-mon[77242]: pgmap v1686: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:24.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:24:23 smithi153 ceph-mon[70044]: pgmap v1686: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:24.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:24:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:24:24.425Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:24:25.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:24:25 smithi083 ceph-mon[77242]: pgmap v1687: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:25.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:24:25 smithi083 ceph-mon[79930]: pgmap v1687: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:26.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:24:25 smithi153 ceph-mon[70044]: pgmap v1687: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:27.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:24:27 smithi083 ceph-mon[79930]: pgmap v1688: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:27.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:24:27 smithi083 ceph-mon[77242]: pgmap v1688: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:28.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:24:27 smithi153 ceph-mon[70044]: pgmap v1688: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:29.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:24:29 smithi083 ceph-mon[79930]: pgmap v1689: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:29.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:24:29 smithi083 ceph-mon[77242]: pgmap v1689: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:30.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:24:29 smithi153 ceph-mon[70044]: pgmap v1689: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:30.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:24:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:24:30.318Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:24:31.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:24:31 smithi083 ceph-mon[77242]: pgmap v1690: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:31.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:24:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:24:31] "GET /metrics HTTP/1.1" 200 225178 "" "Prometheus/2.33.4" 2022-09-29T06:24:31.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:24:31 smithi083 ceph-mon[79930]: pgmap v1690: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:31.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:24:31 smithi153 ceph-mon[70044]: pgmap v1690: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:24:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:24:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:24:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:24:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:24:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:24:33.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:24:33 smithi083 ceph-mon[79930]: pgmap v1691: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:33.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:24:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:24:33.729Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:24:33.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:24:33 smithi083 ceph-mon[77242]: pgmap v1691: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:34.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:24:33 smithi153 ceph-mon[70044]: pgmap v1691: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:34.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:24:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:24:34.425Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:24:34.864 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:24:35.151 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:24:35.152 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (58m) 42s ago 58m 26.5M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:24:35.152 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (58m) 8m ago 58m 61.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:24:35.152 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (59m) 42s ago 59m 25.3M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:24:35.152 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (59m) 8m ago 59m 27.6M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:24:35.153 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (55m) 8m ago 62m 439M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:24:35.153 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (55m) 42s ago 63m 560M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:24:35.153 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (55m) 42s ago 63m 275M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:24:35.153 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (54m) 8m ago 62m 115M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:24:35.154 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (54m) 42s ago 62m 166M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:24:35.154 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (59m) 42s ago 59m 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:24:35.154 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (58m) 8m ago 58m 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:24:35.154 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (53m) 42s ago 61m 209M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:24:35.155 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (53m) 42s ago 61m 201M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:24:35.155 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (52m) 42s ago 61m 163M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:24:35.155 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (52m) 42s ago 61m 191M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:24:35.155 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (51m) 8m ago 60m 154M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:24:35.155 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (51m) 8m ago 60m 180M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:24:35.156 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (50m) 8m ago 60m 159M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:24:35.156 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (50m) 8m ago 59m 180M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:24:35.156 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (55m) 8m ago 59m 102M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:24:35.508 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:24:35.508 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:24:35.508 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:24:35.509 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:24:35.509 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:24:35.509 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:24:35.509 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:24:35.510 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:24:35.510 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:24:35.510 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:24:35.510 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:24:35.511 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:24:35.511 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:24:35.511 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:24:35.511 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:24:35.512 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:24:35.512 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:24:35.512 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:24:35.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:24:35 smithi083 ceph-mon[79930]: pgmap v1692: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:35.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:24:35 smithi083 ceph-mon[79930]: from='client.46039 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:24:35.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:24:35 smithi083 ceph-mon[79930]: from='client.36111 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:24:35.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:24:35 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2699879744' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:24:35.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:24:35 smithi083 ceph-mon[77242]: pgmap v1692: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:35.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:24:35 smithi083 ceph-mon[77242]: from='client.46039 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:24:35.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:24:35 smithi083 ceph-mon[77242]: from='client.36111 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:24:35.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:24:35 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2699879744' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:24:36.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:24:35 smithi153 ceph-mon[70044]: pgmap v1692: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:36.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:24:35 smithi153 ceph-mon[70044]: from='client.46039 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:24:36.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:24:35 smithi153 ceph-mon[70044]: from='client.36111 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:24:36.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:24:35 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2699879744' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:24:37.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:24:37 smithi083 ceph-mon[79930]: pgmap v1693: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:37.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:24:37 smithi083 ceph-mon[77242]: pgmap v1693: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:38.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:24:37 smithi153 ceph-mon[70044]: pgmap v1693: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:24:39 smithi083 ceph-mon[79930]: pgmap v1694: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:24:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:24:39.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:24:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:24:39.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:24:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:24:39.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:24:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:24:39.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:24:39 smithi083 ceph-mon[77242]: pgmap v1694: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:39.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:24:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:24:39.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:24:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:24:39.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:24:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:24:39.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:24:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:24:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:24:39 smithi153 ceph-mon[70044]: pgmap v1694: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:24:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:24:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:24:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:24:40.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:24:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:24:40.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:24:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:24:40.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:24:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:24:40.319Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:24:41.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:24:41 smithi083 ceph-mon[79930]: pgmap v1695: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:41.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:24:41 smithi083 ceph-mon[77242]: pgmap v1695: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:41.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:24:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:24:41] "GET /metrics HTTP/1.1" 200 225175 "" "Prometheus/2.33.4" 2022-09-29T06:24:41.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:24:41 smithi153 ceph-mon[70044]: pgmap v1695: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:24:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:24:42.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:24:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:24:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:24:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:24:43.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:24:43 smithi083 ceph-mon[77242]: pgmap v1696: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:43.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:24:43 smithi083 ceph-mon[79930]: pgmap v1696: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:43.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:24:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:24:43.730Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:24:44.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:24:43 smithi153 ceph-mon[70044]: pgmap v1696: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:44.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:24:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:24:44.426Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:24:45.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:24:45 smithi083 ceph-mon[77242]: pgmap v1697: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:45.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:24:45 smithi083 ceph-mon[79930]: pgmap v1697: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:46.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:24:45 smithi153 ceph-mon[70044]: pgmap v1697: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:47.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:24:47 smithi083 ceph-mon[77242]: pgmap v1698: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:47.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:24:47 smithi083 ceph-mon[79930]: pgmap v1698: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:48.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:24:47 smithi153 ceph-mon[70044]: pgmap v1698: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:49.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:24:49 smithi083 ceph-mon[77242]: pgmap v1699: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:49.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:24:49 smithi083 ceph-mon[79930]: pgmap v1699: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:50.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:24:49 smithi153 ceph-mon[70044]: pgmap v1699: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:50.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:24:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:24:50.319Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:24:51.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:24:51 smithi083 ceph-mon[79930]: pgmap v1700: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:51.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:24:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:24:51] "GET /metrics HTTP/1.1" 200 225175 "" "Prometheus/2.33.4" 2022-09-29T06:24:51.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:24:51 smithi083 ceph-mon[77242]: pgmap v1700: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:51.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:24:51 smithi153 ceph-mon[70044]: pgmap v1700: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:24:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:24:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:24:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:24:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:24:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:24:53.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:24:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:24:53.730Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:24:53.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:24:53 smithi083 ceph-mon[79930]: pgmap v1701: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:53.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:24:53 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:24:53.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:24:53 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:24:53.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:24:53 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:24:53.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:24:53 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:24:53.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:24:53 smithi083 ceph-mon[77242]: pgmap v1701: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:53.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:24:53 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:24:53.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:24:53 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:24:53.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:24:53 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:24:53.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:24:53 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:24:54.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:24:53 smithi153 ceph-mon[70044]: pgmap v1701: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:54.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:24:53 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:24:54.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:24:53 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:24:54.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:24:53 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:24:54.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:24:53 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:24:54.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:24:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:24:54.426Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:24:55.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:24:55 smithi083 ceph-mon[79930]: pgmap v1702: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:55.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:24:55 smithi083 ceph-mon[77242]: pgmap v1702: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:56.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:24:55 smithi153 ceph-mon[70044]: pgmap v1702: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:57.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:24:57 smithi083 ceph-mon[79930]: pgmap v1703: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:57.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:24:57 smithi083 ceph-mon[77242]: pgmap v1703: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:58.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:24:57 smithi153 ceph-mon[70044]: pgmap v1703: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:59.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:24:59 smithi083 ceph-mon[79930]: pgmap v1704: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:24:59.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:24:59 smithi083 ceph-mon[77242]: pgmap v1704: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:25:00.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:24:59 smithi153 ceph-mon[70044]: pgmap v1704: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:25:00.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:25:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:25:00.320Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:25:01.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:25:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:25:01] "GET /metrics HTTP/1.1" 200 225173 "" "Prometheus/2.33.4" 2022-09-29T06:25:01.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:25:01 smithi083 ceph-mon[79930]: pgmap v1705: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:25:01.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:25:01 smithi083 ceph-mon[77242]: pgmap v1705: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:25:01.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:25:01 smithi153 ceph-mon[70044]: pgmap v1705: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:25:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:25:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:25:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:25:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:25:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:25:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:25:03.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:25:03 smithi083 ceph-mon[77242]: pgmap v1706: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:25:03.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:25:03 smithi083 ceph-mon[79930]: pgmap v1706: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:25:03.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:25:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:25:03.731Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:25:04.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:25:03 smithi153 ceph-mon[70044]: pgmap v1706: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:25:04.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:25:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:25:04.427Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:25:05.835 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:25:05.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:25:05 smithi083 ceph-mon[79930]: pgmap v1707: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:25:05.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:25:05 smithi083 ceph-mon[77242]: pgmap v1707: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:25:06.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:25:05 smithi153 ceph-mon[70044]: pgmap v1707: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:25:06.126 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:25:06.126 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (58m) 73s ago 59m 26.5M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:25:06.127 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (59m) 9m ago 59m 61.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:25:06.127 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (60m) 73s ago 60m 25.3M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:25:06.127 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (60m) 9m ago 60m 27.6M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:25:06.127 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (55m) 9m ago 62m 439M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:25:06.127 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (56m) 73s ago 64m 560M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:25:06.128 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (55m) 73s ago 64m 275M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:25:06.128 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (54m) 9m ago 62m 115M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:25:06.128 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (55m) 73s ago 63m 166M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:25:06.128 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (59m) 73s ago 59m 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:25:06.129 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (59m) 9m ago 59m 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:25:06.129 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (54m) 73s ago 62m 209M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:25:06.129 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (53m) 73s ago 62m 201M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:25:06.129 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (53m) 73s ago 61m 163M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:25:06.130 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (52m) 73s ago 61m 191M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:25:06.130 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (52m) 9m ago 61m 154M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:25:06.130 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (51m) 9m ago 60m 180M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:25:06.130 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (51m) 9m ago 60m 159M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:25:06.130 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (50m) 9m ago 60m 180M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:25:06.131 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (55m) 9m ago 59m 102M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:25:06.483 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:25:06.483 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:25:06.484 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:25:06.484 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:25:06.484 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:25:06.484 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:25:06.485 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:25:06.485 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:25:06.485 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:25:06.485 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:25:06.486 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:25:06.486 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:25:06.486 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:25:06.486 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:25:06.487 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:25:06.487 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:25:06.487 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:25:06.487 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:25:06.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:25:06 smithi083 ceph-mon[77242]: from='client.36123 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:25:06.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:25:06 smithi083 ceph-mon[77242]: from='client.36129 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:25:06.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:25:06 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/303188438' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:25:06.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:25:06 smithi083 ceph-mon[79930]: from='client.36123 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:25:06.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:25:06 smithi083 ceph-mon[79930]: from='client.36129 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:25:06.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:25:06 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/303188438' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:25:07.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:25:06 smithi153 ceph-mon[70044]: from='client.36123 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:25:07.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:25:06 smithi153 ceph-mon[70044]: from='client.36129 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:25:07.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:25:06 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/303188438' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:25:07.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:25:07 smithi083 ceph-mon[79930]: pgmap v1708: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:25:07.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:25:07 smithi083 ceph-mon[77242]: pgmap v1708: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:25:08.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:25:07 smithi153 ceph-mon[70044]: pgmap v1708: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:25:09.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:25:09 smithi083 ceph-mon[79930]: pgmap v1709: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:25:09.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:25:09 smithi083 ceph-mon[77242]: pgmap v1709: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:25:10.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:25:09 smithi153 ceph-mon[70044]: pgmap v1709: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:25:10.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:25:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:25:10.321Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:25:11.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:25:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:25:11] "GET /metrics HTTP/1.1" 200 225176 "" "Prometheus/2.33.4" 2022-09-29T06:25:11.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:25:11 smithi083 ceph-mon[79930]: pgmap v1710: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:25:11.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:25:11 smithi083 ceph-mon[77242]: pgmap v1710: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:25:11.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:25:11 smithi153 ceph-mon[70044]: pgmap v1710: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:25:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:25:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:25:12.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:25:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:25:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:25:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:25:13.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:25:13 smithi083 ceph-mon[79930]: pgmap v1711: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:25:13.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:25:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:25:13.732Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:25:13.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:25:13 smithi083 ceph-mon[77242]: pgmap v1711: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:25:14.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:25:13 smithi153 ceph-mon[70044]: pgmap v1711: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:25:14.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:25:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:25:14.428Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:25:15.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:25:15 smithi083 ceph-mon[79930]: pgmap v1712: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:25:15.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:25:15 smithi083 ceph-mon[77242]: pgmap v1712: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:25:16.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:25:15 smithi153 ceph-mon[70044]: pgmap v1712: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:25:17.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:25:17 smithi083 ceph-mon[79930]: pgmap v1713: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:25:17.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:25:17 smithi083 ceph-mon[77242]: pgmap v1713: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:25:18.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:25:17 smithi153 ceph-mon[70044]: pgmap v1713: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:25:19.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:25:19 smithi083 ceph-mon[77242]: pgmap v1714: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:25:19.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:25:19 smithi083 ceph-mon[79930]: pgmap v1714: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:25:20.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:25:19 smithi153 ceph-mon[70044]: pgmap v1714: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:25:20.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:25:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:25:20.322Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:25:21.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:25:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:25:21] "GET /metrics HTTP/1.1" 200 225176 "" "Prometheus/2.33.4" 2022-09-29T06:25:21.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:25:21 smithi083 ceph-mon[79930]: pgmap v1715: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:25:21.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:25:21 smithi083 ceph-mon[77242]: pgmap v1715: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:25:21.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:25:21 smithi153 ceph-mon[70044]: pgmap v1715: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:25:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:25:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:25:22.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:25:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:25:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:25:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:25:23.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:25:23 smithi083 ceph-mon[79930]: pgmap v1716: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:25:23.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:25:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:25:23.732Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:25:23.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:25:23 smithi083 ceph-mon[77242]: pgmap v1716: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:25:24.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:25:23 smithi153 ceph-mon[70044]: pgmap v1716: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:25:24.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:25:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:25:24.428Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:25:25.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:25:25 smithi083 ceph-mon[79930]: pgmap v1717: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:25:25.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:25:25 smithi083 ceph-mon[77242]: pgmap v1717: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:25:26.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:25:25 smithi153 ceph-mon[70044]: pgmap v1717: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:25:27.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:25:27 smithi083 ceph-mon[79930]: pgmap v1718: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:25:27.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:25:27 smithi083 ceph-mon[77242]: pgmap v1718: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:25:28.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:25:27 smithi153 ceph-mon[70044]: pgmap v1718: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:25:29.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:25:29 smithi083 ceph-mon[77242]: pgmap v1719: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:25:29.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:25:29 smithi083 ceph-mon[79930]: pgmap v1719: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:25:30.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:25:29 smithi153 ceph-mon[70044]: pgmap v1719: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:25:30.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:25:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:25:30.322Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:25:31.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:25:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:25:31] "GET /metrics HTTP/1.1" 200 225174 "" "Prometheus/2.33.4" 2022-09-29T06:25:31.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:25:31 smithi083 ceph-mon[79930]: pgmap v1720: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:25:31.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:25:31 smithi083 ceph-mon[77242]: pgmap v1720: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:25:31.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:25:31 smithi153 ceph-mon[70044]: pgmap v1720: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:25:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:25:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:25:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:25:33.314 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:25:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:25:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:25:33.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:25:33 smithi083 ceph-mon[77242]: pgmap v1721: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:25:33.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:25:33 smithi083 ceph-mon[79930]: pgmap v1721: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:25:33.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:25:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:25:33.733Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:25:34.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:25:33 smithi153 ceph-mon[70044]: pgmap v1721: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:25:34.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:25:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:25:34.429Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:25:35.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:25:35 smithi083 ceph-mon[77242]: pgmap v1722: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:25:35.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:25:35 smithi083 ceph-mon[79930]: pgmap v1722: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:25:36.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:25:35 smithi153 ceph-mon[70044]: pgmap v1722: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:25:36.810 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:25:37.097 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:25:37.097 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (59m) 104s ago 59m 26.5M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:25:37.098 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (59m) 9m ago 59m 61.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:25:37.098 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (60m) 104s ago 60m 25.3M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:25:37.098 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (60m) 9m ago 60m 27.6M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:25:37.098 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (56m) 9m ago 63m 439M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:25:37.099 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (56m) 104s ago 64m 560M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:25:37.099 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (56m) 104s ago 64m 275M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:25:37.099 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (55m) 9m ago 63m 115M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:25:37.099 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (55m) 104s ago 63m 166M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:25:37.100 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (60m) 104s ago 60m 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:25:37.100 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (59m) 9m ago 59m 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:25:37.100 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (54m) 104s ago 62m 209M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:25:37.100 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (54m) 104s ago 62m 201M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:25:37.101 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (53m) 104s ago 62m 163M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:25:37.101 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (53m) 104s ago 62m 191M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:25:37.101 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (52m) 9m ago 61m 154M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:25:37.101 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (52m) 9m ago 61m 180M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:25:37.101 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (51m) 9m ago 61m 159M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:25:37.102 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (51m) 9m ago 60m 180M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:25:37.102 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (56m) 9m ago 60m 102M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:25:37.451 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:25:37.451 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:25:37.452 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:25:37.452 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:25:37.452 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:25:37.452 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:25:37.453 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:25:37.453 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:25:37.453 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:25:37.453 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:25:37.454 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:25:37.454 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:25:37.454 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:25:37.454 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:25:37.455 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:25:37.455 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:25:37.455 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:25:37.456 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:25:37.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:25:37 smithi083 ceph-mon[79930]: pgmap v1723: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:25:37.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:25:37 smithi083 ceph-mon[79930]: from='client.46075 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:25:37.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:25:37 smithi083 ceph-mon[79930]: from='client.46081 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:25:37.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:25:37 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1495598023' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:25:37.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:25:37 smithi083 ceph-mon[77242]: pgmap v1723: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:25:37.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:25:37 smithi083 ceph-mon[77242]: from='client.46075 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:25:37.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:25:37 smithi083 ceph-mon[77242]: from='client.46081 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:25:37.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:25:37 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1495598023' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:25:38.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:25:37 smithi153 ceph-mon[70044]: pgmap v1723: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:25:38.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:25:37 smithi153 ceph-mon[70044]: from='client.46075 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:25:38.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:25:37 smithi153 ceph-mon[70044]: from='client.46081 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:25:38.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:25:37 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1495598023' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:25:39.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:25:39 smithi083 ceph-mon[77242]: pgmap v1724: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:25:39.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:25:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:25:39.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:25:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:25:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:25:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:25:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:25:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:25:39.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:25:39 smithi083 ceph-mon[79930]: pgmap v1724: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:25:39.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:25:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:25:39.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:25:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:25:39.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:25:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:25:39.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:25:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:25:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:25:39 smithi153 ceph-mon[70044]: pgmap v1724: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:25:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:25:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:25:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:25:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:25:40.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:25:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:25:40.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:25:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:25:40.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:25:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:25:40.323Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:25:41.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:25:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:25:41] "GET /metrics HTTP/1.1" 200 225177 "" "Prometheus/2.33.4" 2022-09-29T06:25:41.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:25:41 smithi083 ceph-mon[79930]: pgmap v1725: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:25:41.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:25:41 smithi083 ceph-mon[77242]: pgmap v1725: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:25:41.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:25:41 smithi153 ceph-mon[70044]: pgmap v1725: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:25:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:25:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:25:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:25:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:25:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:25:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:25:43.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:25:43 smithi083 ceph-mon[79930]: pgmap v1726: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:25:43.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:25:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:25:43.734Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:25:43.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:25:43 smithi083 ceph-mon[77242]: pgmap v1726: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:25:44.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:25:43 smithi153 ceph-mon[70044]: pgmap v1726: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:25:44.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:25:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:25:44.429Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:25:45.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:25:45 smithi083 ceph-mon[79930]: pgmap v1727: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:25:45.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:25:45 smithi083 ceph-mon[77242]: pgmap v1727: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:25:46.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:25:45 smithi153 ceph-mon[70044]: pgmap v1727: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:25:47.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:25:47 smithi083 ceph-mon[79930]: pgmap v1728: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:25:47.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:25:47 smithi083 ceph-mon[77242]: pgmap v1728: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:25:48.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:25:47 smithi153 ceph-mon[70044]: pgmap v1728: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:25:49.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:25:49 smithi083 ceph-mon[79930]: pgmap v1729: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:25:49.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:25:49 smithi083 ceph-mon[77242]: pgmap v1729: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:25:50.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:25:49 smithi153 ceph-mon[70044]: pgmap v1729: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:25:50.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:25:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:25:50.324Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:25:51.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:25:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:25:51] "GET /metrics HTTP/1.1" 200 225177 "" "Prometheus/2.33.4" 2022-09-29T06:25:51.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:25:51 smithi083 ceph-mon[79930]: pgmap v1730: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:25:51.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:25:51 smithi083 ceph-mon[77242]: pgmap v1730: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:25:51.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:25:51 smithi153 ceph-mon[70044]: pgmap v1730: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:25:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:25:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:25:52.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:25:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:25:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:25:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:25:53.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:25:53 smithi083 ceph-mon[79930]: pgmap v1731: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:25:53.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:25:53 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:25:53.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:25:53 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:25:53.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:25:53 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:25:53.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:25:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:25:53.734Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:25:53.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:25:53 smithi083 ceph-mon[77242]: pgmap v1731: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:25:53.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:25:53 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:25:53.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:25:53 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:25:53.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:25:53 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:25:54.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:25:53 smithi153 ceph-mon[70044]: pgmap v1731: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:25:54.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:25:53 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:25:54.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:25:53 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:25:54.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:25:53 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:25:54.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:25:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:25:54.429Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:25:55.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:25:55 smithi083 ceph-mon[77242]: pgmap v1732: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:25:55.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:25:55 smithi083 ceph-mon[79930]: pgmap v1732: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:25:56.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:25:55 smithi153 ceph-mon[70044]: pgmap v1732: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:25:57.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:25:57 smithi083 ceph-mon[77242]: pgmap v1733: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:25:57.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:25:57 smithi083 ceph-mon[79930]: pgmap v1733: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:25:58.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:25:57 smithi153 ceph-mon[70044]: pgmap v1733: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:25:59.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:25:59 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:25:59.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:25:59 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:25:59.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:25:59 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:25:59.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:25:59 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:25:59.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:25:59 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:25:59.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:25:59 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:25:59.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:25:59 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:25:59.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:25:59 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:25:59.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:25:59 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:26:00.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:26:00 smithi083 ceph-mon[77242]: pgmap v1734: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:00.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:26:00 smithi083 ceph-mon[79930]: pgmap v1734: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:00.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:26:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:26:00.325Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:26:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:26:00 smithi153 ceph-mon[70044]: pgmap v1734: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:01.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:26:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:26:01] "GET /metrics HTTP/1.1" 200 225172 "" "Prometheus/2.33.4" 2022-09-29T06:26:01.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:26:01 smithi083 ceph-mon[79930]: pgmap v1735: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:01.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:26:01 smithi083 ceph-mon[77242]: pgmap v1735: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:01.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:26:01 smithi153 ceph-mon[70044]: pgmap v1735: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:26:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:26:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:26:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:26:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:26:03.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:26:03 smithi083 ceph-mon[77242]: pgmap v1736: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:03.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:26:03 smithi083 ceph-mon[79930]: pgmap v1736: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:03.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:26:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:26:03.735Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:26:04.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:26:03 smithi153 ceph-mon[70044]: pgmap v1736: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:04.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:26:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:26:04.431Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:26:05.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:26:05 smithi083 ceph-mon[79930]: pgmap v1737: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:05.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:26:05 smithi083 ceph-mon[77242]: pgmap v1737: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:06.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:26:05 smithi153 ceph-mon[70044]: pgmap v1737: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:07.778 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:26:07.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:26:07 smithi083 ceph-mon[77242]: pgmap v1738: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:07.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:26:07 smithi083 ceph-mon[79930]: pgmap v1738: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:08.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:26:07 smithi153 ceph-mon[70044]: pgmap v1738: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:08.065 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:26:08.065 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (59m) 2m ago 60m 26.5M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:26:08.066 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (60m) 9s ago 60m 61.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:26:08.066 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (61m) 2m ago 61m 25.3M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:26:08.066 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (61m) 9s ago 61m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:26:08.067 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (56m) 9s ago 63m 439M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:26:08.067 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (57m) 2m ago 65m 560M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:26:08.067 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (56m) 2m ago 65m 275M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:26:08.067 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (56m) 9s ago 63m 132M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:26:08.068 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (56m) 2m ago 64m 166M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:26:08.068 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (60m) 2m ago 60m 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:26:08.068 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (60m) 9s ago 60m 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:26:08.068 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (55m) 2m ago 63m 209M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:26:08.069 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (54m) 2m ago 63m 201M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:26:08.069 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (54m) 2m ago 62m 163M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:26:08.069 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (53m) 2m ago 62m 191M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:26:08.069 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (53m) 9s ago 62m 176M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:26:08.070 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (52m) 9s ago 62m 207M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:26:08.070 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (52m) 9s ago 61m 184M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:26:08.070 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (51m) 9s ago 61m 209M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:26:08.070 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (56m) 9s ago 60m 103M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:26:08.421 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:26:08.422 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:26:08.422 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:26:08.422 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:26:08.422 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:26:08.423 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:26:08.423 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:26:08.423 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:26:08.423 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:26:08.424 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:26:08.424 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:26:08.424 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:26:08.424 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:26:08.424 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:26:08.425 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:26:08.425 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:26:08.425 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:26:08.425 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:26:08.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:26:08 smithi083 ceph-mon[79930]: from='client.46093 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:26:08.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:26:08 smithi083 ceph-mon[79930]: from='client.36165 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:26:08.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:26:08 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/369837994' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:26:08.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:26:08 smithi083 ceph-mon[77242]: from='client.46093 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:26:08.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:26:08 smithi083 ceph-mon[77242]: from='client.36165 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:26:08.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:26:08 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/369837994' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:26:09.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:26:08 smithi153 ceph-mon[70044]: from='client.46093 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:26:09.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:26:08 smithi153 ceph-mon[70044]: from='client.36165 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:26:09.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:26:08 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/369837994' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:26:09.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:26:09 smithi083 ceph-mon[77242]: pgmap v1739: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:09.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:26:09 smithi083 ceph-mon[79930]: pgmap v1739: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:10.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:26:09 smithi153 ceph-mon[70044]: pgmap v1739: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:10.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:26:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:26:10.325Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:26:11.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:26:11 smithi083 ceph-mon[77242]: pgmap v1740: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:11.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:26:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:26:11] "GET /metrics HTTP/1.1" 200 225179 "" "Prometheus/2.33.4" 2022-09-29T06:26:11.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:26:11 smithi083 ceph-mon[79930]: pgmap v1740: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:11.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:26:11 smithi153 ceph-mon[70044]: pgmap v1740: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:26:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:26:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:26:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:26:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:26:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:26:13.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:26:13 smithi083 ceph-mon[79930]: pgmap v1741: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:13.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:26:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:26:13.735Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:26:13.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:26:13 smithi083 ceph-mon[77242]: pgmap v1741: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:14.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:26:13 smithi153 ceph-mon[70044]: pgmap v1741: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:14.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:26:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:26:14.432Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:26:15.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:26:15 smithi083 ceph-mon[79930]: pgmap v1742: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:15.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:26:15 smithi083 ceph-mon[77242]: pgmap v1742: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:16.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:26:15 smithi153 ceph-mon[70044]: pgmap v1742: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:17.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:26:17 smithi083 ceph-mon[79930]: pgmap v1743: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:17.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:26:17 smithi083 ceph-mon[77242]: pgmap v1743: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:18.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:26:17 smithi153 ceph-mon[70044]: pgmap v1743: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:19.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:26:19 smithi083 ceph-mon[79930]: pgmap v1744: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:19.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:26:19 smithi083 ceph-mon[77242]: pgmap v1744: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:20.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:26:19 smithi153 ceph-mon[70044]: pgmap v1744: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:20.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:26:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:26:20.325Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:26:21.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:26:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:26:21] "GET /metrics HTTP/1.1" 200 225179 "" "Prometheus/2.33.4" 2022-09-29T06:26:21.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:26:21 smithi083 ceph-mon[79930]: pgmap v1745: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:21.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:26:21 smithi083 ceph-mon[77242]: pgmap v1745: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:21.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:26:21 smithi153 ceph-mon[70044]: pgmap v1745: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:26:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:26:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:26:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:26:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:26:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:26:23.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:26:23 smithi083 ceph-mon[77242]: pgmap v1746: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:23.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:26:23 smithi083 ceph-mon[79930]: pgmap v1746: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:23.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:26:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:26:23.736Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:26:24.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:26:23 smithi153 ceph-mon[70044]: pgmap v1746: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:24.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:26:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:26:24.433Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:26:25.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:26:25 smithi083 ceph-mon[79930]: pgmap v1747: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:25.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:26:25 smithi083 ceph-mon[77242]: pgmap v1747: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:26.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:26:25 smithi153 ceph-mon[70044]: pgmap v1747: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:27.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:26:27 smithi083 ceph-mon[79930]: pgmap v1748: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:27.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:26:27 smithi083 ceph-mon[77242]: pgmap v1748: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:28.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:26:27 smithi153 ceph-mon[70044]: pgmap v1748: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:29.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:26:29 smithi083 ceph-mon[77242]: pgmap v1749: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:29.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:26:29 smithi083 ceph-mon[79930]: pgmap v1749: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:30.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:26:29 smithi153 ceph-mon[70044]: pgmap v1749: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:30.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:26:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:26:30.326Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:26:31.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:26:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:26:31] "GET /metrics HTTP/1.1" 200 225179 "" "Prometheus/2.33.4" 2022-09-29T06:26:31.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:26:31 smithi083 ceph-mon[79930]: pgmap v1750: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:31.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:26:31 smithi083 ceph-mon[77242]: pgmap v1750: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:31.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:26:31 smithi153 ceph-mon[70044]: pgmap v1750: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:26:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:26:32.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:26:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:26:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:26:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:26:33.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:26:33 smithi083 ceph-mon[79930]: pgmap v1751: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:33.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:26:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:26:33.736Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:26:33.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:26:33 smithi083 ceph-mon[77242]: pgmap v1751: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:34.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:26:33 smithi153 ceph-mon[70044]: pgmap v1751: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:34.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:26:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:26:34.433Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:26:35.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:26:35 smithi083 ceph-mon[77242]: pgmap v1752: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:35.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:26:35 smithi083 ceph-mon[79930]: pgmap v1752: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:36.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:26:35 smithi153 ceph-mon[70044]: pgmap v1752: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:37.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:26:37 smithi083 ceph-mon[77242]: pgmap v1753: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:37.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:26:37 smithi083 ceph-mon[79930]: pgmap v1753: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:38.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:26:37 smithi153 ceph-mon[70044]: pgmap v1753: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:38.749 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:26:39.042 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:26:39.042 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (60m) 2m ago 60m 26.5M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:26:39.042 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (60m) 40s ago 60m 61.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:26:39.043 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (61m) 2m ago 61m 25.3M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:26:39.043 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (61m) 40s ago 61m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:26:39.043 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (57m) 40s ago 64m 439M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:26:39.043 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (57m) 2m ago 65m 560M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:26:39.044 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (57m) 2m ago 65m 275M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:26:39.044 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (56m) 40s ago 64m 132M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:26:39.044 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (56m) 2m ago 64m 166M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:26:39.044 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (61m) 2m ago 61m 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:26:39.045 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (61m) 40s ago 61m 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:26:39.045 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (55m) 2m ago 63m 209M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:26:39.045 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (55m) 2m ago 63m 201M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:26:39.045 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (54m) 2m ago 63m 163M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:26:39.046 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (54m) 2m ago 63m 191M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:26:39.046 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (53m) 40s ago 62m 176M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:26:39.046 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (53m) 40s ago 62m 207M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:26:39.046 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (52m) 40s ago 62m 184M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:26:39.047 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (52m) 40s ago 61m 209M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:26:39.047 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (57m) 40s ago 61m 103M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:26:39.401 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:26:39.402 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:26:39.402 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:26:39.402 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:26:39.402 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:26:39.403 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:26:39.403 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:26:39.403 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:26:39.403 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:26:39.404 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:26:39.404 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:26:39.404 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:26:39.404 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:26:39.404 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:26:39.405 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:26:39.405 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:26:39.405 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:26:39.405 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:26:39.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:26:39 smithi083 ceph-mon[79930]: pgmap v1754: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:26:39 smithi083 ceph-mon[79930]: from='client.36177 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:26:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:26:39 smithi083 ceph-mon[79930]: from='client.46117 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:26:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:26:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:26:39.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:26:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:26:39.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:26:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:26:39.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:26:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:26:39.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:26:39 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2743765417' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:26:39.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:26:39 smithi083 ceph-mon[77242]: pgmap v1754: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:39.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:26:39 smithi083 ceph-mon[77242]: from='client.36177 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:26:39.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:26:39 smithi083 ceph-mon[77242]: from='client.46117 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:26:39.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:26:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:26:39.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:26:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:26:39.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:26:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:26:39.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:26:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:26:39.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:26:39 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2743765417' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:26:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:26:39 smithi153 ceph-mon[70044]: pgmap v1754: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:26:39 smithi153 ceph-mon[70044]: from='client.36177 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:26:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:26:39 smithi153 ceph-mon[70044]: from='client.46117 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:26:40.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:26:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:26:40.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:26:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:26:40.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:26:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:26:40.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:26:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:26:40.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:26:39 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2743765417' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:26:40.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:26:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:26:40.326Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:26:41.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:26:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:26:41] "GET /metrics HTTP/1.1" 200 225178 "" "Prometheus/2.33.4" 2022-09-29T06:26:41.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:26:41 smithi083 ceph-mon[79930]: pgmap v1755: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:41.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:26:41 smithi083 ceph-mon[77242]: pgmap v1755: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:41.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:26:41 smithi153 ceph-mon[70044]: pgmap v1755: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:26:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:26:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:26:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:26:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:26:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:26:43.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:26:43 smithi083 ceph-mon[79930]: pgmap v1756: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:43.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:26:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:26:43.737Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:26:43.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:26:43 smithi083 ceph-mon[77242]: pgmap v1756: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:44.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:26:43 smithi153 ceph-mon[70044]: pgmap v1756: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:44.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:26:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:26:44.434Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:26:45.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:26:45 smithi083 ceph-mon[79930]: pgmap v1757: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:45.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:26:45 smithi083 ceph-mon[77242]: pgmap v1757: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:46.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:26:45 smithi153 ceph-mon[70044]: pgmap v1757: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:47.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:26:47 smithi083 ceph-mon[77242]: pgmap v1758: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:47.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:26:47 smithi083 ceph-mon[79930]: pgmap v1758: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:48.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:26:47 smithi153 ceph-mon[70044]: pgmap v1758: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:49.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:26:49 smithi083 ceph-mon[79930]: pgmap v1759: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:49.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:26:49 smithi083 ceph-mon[77242]: pgmap v1759: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:50.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:26:49 smithi153 ceph-mon[70044]: pgmap v1759: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:50.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:26:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:26:50.327Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:26:51.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:26:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:26:51] "GET /metrics HTTP/1.1" 200 225178 "" "Prometheus/2.33.4" 2022-09-29T06:26:51.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:26:51 smithi083 ceph-mon[79930]: pgmap v1760: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:51.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:26:51 smithi083 ceph-mon[77242]: pgmap v1760: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:51.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:26:51 smithi153 ceph-mon[70044]: pgmap v1760: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:26:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:26:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:26:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:26:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:26:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:26:53.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:26:53 smithi083 ceph-mon[77242]: pgmap v1761: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:53.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:26:53 smithi083 ceph-mon[79930]: pgmap v1761: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:53.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:26:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:26:53.738Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:26:54.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:26:53 smithi153 ceph-mon[70044]: pgmap v1761: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:54.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:26:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:26:54.434Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:26:55.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:26:55 smithi083 ceph-mon[79930]: pgmap v1762: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:55.969 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:26:55 smithi083 ceph-mon[77242]: pgmap v1762: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:56.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:26:55 smithi153 ceph-mon[70044]: pgmap v1762: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:57.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:26:57 smithi083 ceph-mon[79930]: pgmap v1763: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:57.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:26:57 smithi083 ceph-mon[77242]: pgmap v1763: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:58.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:26:57 smithi153 ceph-mon[70044]: pgmap v1763: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:59.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:26:59 smithi083 ceph-mon[79930]: pgmap v1764: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:59.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:26:59 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:26:59.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:26:59 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:26:59.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:26:59 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:26:59.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:26:59 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:26:59.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:26:59 smithi083 ceph-mon[77242]: pgmap v1764: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:26:59.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:26:59 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:26:59.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:26:59 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:26:59.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:26:59 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:26:59.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:26:59 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:27:00.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:26:59 smithi153 ceph-mon[70044]: pgmap v1764: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:27:00.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:26:59 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:27:00.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:26:59 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:27:00.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:26:59 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:27:00.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:26:59 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:27:00.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:27:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:27:00.328Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:27:01.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:27:01 smithi083 ceph-mon[77242]: pgmap v1765: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:27:01.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:27:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:27:01] "GET /metrics HTTP/1.1" 200 225180 "" "Prometheus/2.33.4" 2022-09-29T06:27:01.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:27:01 smithi083 ceph-mon[79930]: pgmap v1765: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:27:01.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:27:01 smithi153 ceph-mon[70044]: pgmap v1765: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:27:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:27:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:27:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:27:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:27:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:27:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:27:03.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:27:03 smithi083 ceph-mon[79930]: pgmap v1766: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:27:03.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:27:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:27:03.738Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:27:03.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:27:03 smithi083 ceph-mon[77242]: pgmap v1766: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:27:04.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:27:03 smithi153 ceph-mon[70044]: pgmap v1766: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:27:04.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:27:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:27:04.434Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:27:05.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:27:05 smithi083 ceph-mon[79930]: pgmap v1767: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:27:05.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:27:05 smithi083 ceph-mon[77242]: pgmap v1767: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:27:06.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:27:05 smithi153 ceph-mon[70044]: pgmap v1767: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:27:07.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:27:07 smithi083 ceph-mon[79930]: pgmap v1768: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:27:07.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:27:07 smithi083 ceph-mon[77242]: pgmap v1768: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:27:08.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:27:07 smithi153 ceph-mon[70044]: pgmap v1768: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:27:09.729 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:27:09.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:27:09 smithi083 ceph-mon[79930]: pgmap v1769: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:27:09.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:27:09 smithi083 ceph-mon[77242]: pgmap v1769: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:27:10.019 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:27:10.020 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (60m) 3m ago 61m 26.5M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:27:10.020 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (61m) 71s ago 61m 61.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:27:10.020 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (62m) 3m ago 62m 25.3M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:27:10.020 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (62m) 71s ago 62m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:27:10.021 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (57m) 71s ago 64m 439M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:27:10.021 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (58m) 3m ago 66m 560M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:27:10.021 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (57m) 3m ago 66m 275M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:27:10.021 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (57m) 71s ago 64m 132M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:27:10.022 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (57m) 3m ago 65m 166M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:27:10.022 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (61m) 3m ago 61m 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:27:10.022 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (61m) 71s ago 61m 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:27:10.022 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (56m) 3m ago 64m 209M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:27:10.023 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (55m) 3m ago 64m 201M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:27:10.023 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (55m) 3m ago 63m 163M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:27:10.023 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (54m) 3m ago 63m 191M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:27:10.023 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (54m) 71s ago 63m 176M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:27:10.023 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (53m) 71s ago 63m 207M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:27:10.024 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (53m) 71s ago 62m 184M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:27:10.024 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (52m) 71s ago 62m 209M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:27:10.024 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (57m) 71s ago 61m 103M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:27:10.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:27:09 smithi153 ceph-mon[70044]: pgmap v1769: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:27:10.375 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:27:10.376 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:27:10.376 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:27:10.376 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:27:10.376 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:27:10.377 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:27:10.377 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:27:10.377 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:27:10.377 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:27:10.378 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:27:10.378 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:27:10.378 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:27:10.378 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:27:10.378 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:27:10.379 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:27:10.379 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:27:10.379 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:27:10.379 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:27:10.700 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:27:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:27:10.328Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:27:10.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:27:10 smithi083 ceph-mon[79930]: from='client.36195 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:27:10.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:27:10 smithi083 ceph-mon[79930]: from='client.36201 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:27:10.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:27:10 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1908262230' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:27:10.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:27:10 smithi083 ceph-mon[77242]: from='client.36195 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:27:10.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:27:10 smithi083 ceph-mon[77242]: from='client.36201 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:27:10.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:27:10 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1908262230' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:27:11.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:27:10 smithi153 ceph-mon[70044]: from='client.36195 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:27:11.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:27:10 smithi153 ceph-mon[70044]: from='client.36201 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:27:11.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:27:10 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1908262230' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:27:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:27:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:27:11] "GET /metrics HTTP/1.1" 200 225178 "" "Prometheus/2.33.4" 2022-09-29T06:27:11.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:27:11 smithi083 ceph-mon[79930]: pgmap v1770: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:27:11.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:27:11 smithi083 ceph-mon[77242]: pgmap v1770: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:27:12.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:27:11 smithi153 ceph-mon[70044]: pgmap v1770: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:27:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:27:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:27:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:27:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:27:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:27:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:27:13.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:27:13 smithi083 ceph-mon[79930]: pgmap v1771: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:27:13.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:27:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:27:13.739Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:27:13.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:27:13 smithi083 ceph-mon[77242]: pgmap v1771: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:27:14.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:27:13 smithi153 ceph-mon[70044]: pgmap v1771: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:27:14.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:27:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:27:14.435Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:27:15.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:27:15 smithi083 ceph-mon[79930]: pgmap v1772: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:27:15.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:27:15 smithi083 ceph-mon[77242]: pgmap v1772: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:27:16.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:27:15 smithi153 ceph-mon[70044]: pgmap v1772: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:27:18.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:27:17 smithi153 ceph-mon[70044]: pgmap v1773: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:27:18.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:27:17 smithi083 ceph-mon[79930]: pgmap v1773: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:27:18.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:27:17 smithi083 ceph-mon[77242]: pgmap v1773: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:27:20.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:27:19 smithi153 ceph-mon[70044]: pgmap v1774: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:27:20.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:27:19 smithi083 ceph-mon[79930]: pgmap v1774: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:27:20.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:27:19 smithi083 ceph-mon[77242]: pgmap v1774: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:27:20.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:27:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:27:20.329Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:27:21.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:27:21 smithi083 ceph-mon[79930]: pgmap v1775: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:27:21.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:27:21 smithi083 ceph-mon[77242]: pgmap v1775: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:27:21.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:27:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:27:21] "GET /metrics HTTP/1.1" 200 225178 "" "Prometheus/2.33.4" 2022-09-29T06:27:21.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:27:21 smithi153 ceph-mon[70044]: pgmap v1775: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:27:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:27:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:27:22.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:27:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:27:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:27:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:27:23.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:27:23 smithi083 ceph-mon[79930]: pgmap v1776: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:27:23.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:27:23 smithi083 ceph-mon[77242]: pgmap v1776: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:27:23.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:27:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:27:23.740Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:27:24.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:27:23 smithi153 ceph-mon[70044]: pgmap v1776: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:27:24.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:27:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:27:24.435Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:27:25.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:27:25 smithi083 ceph-mon[77242]: pgmap v1777: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:27:25.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:27:25 smithi083 ceph-mon[79930]: pgmap v1777: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:27:26.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:27:25 smithi153 ceph-mon[70044]: pgmap v1777: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:27:27.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:27:27 smithi083 ceph-mon[79930]: pgmap v1778: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:27:27.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:27:27 smithi083 ceph-mon[77242]: pgmap v1778: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:27:28.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:27:27 smithi153 ceph-mon[70044]: pgmap v1778: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:27:29.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:27:29 smithi083 ceph-mon[79930]: pgmap v1779: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:27:29.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:27:29 smithi083 ceph-mon[77242]: pgmap v1779: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:27:30.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:27:29 smithi153 ceph-mon[70044]: pgmap v1779: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:27:30.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:27:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:27:30.330Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:27:31.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:27:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:27:31] "GET /metrics HTTP/1.1" 200 225178 "" "Prometheus/2.33.4" 2022-09-29T06:27:31.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:27:31 smithi083 ceph-mon[79930]: pgmap v1780: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:27:31.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:27:31 smithi083 ceph-mon[77242]: pgmap v1780: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:27:31.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:27:31 smithi153 ceph-mon[70044]: pgmap v1780: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:27:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:27:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:27:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:27:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:27:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:27:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:27:33.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:27:33 smithi083 ceph-mon[79930]: pgmap v1781: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:27:33.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:27:33 smithi083 ceph-mon[77242]: pgmap v1781: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:27:33.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:27:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:27:33.740Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:27:34.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:27:33 smithi153 ceph-mon[70044]: pgmap v1781: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:27:34.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:27:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:27:34.437Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:27:35.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:27:35 smithi083 ceph-mon[79930]: pgmap v1782: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:27:35.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:27:35 smithi083 ceph-mon[77242]: pgmap v1782: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:27:36.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:27:35 smithi153 ceph-mon[70044]: pgmap v1782: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:27:37.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:27:37 smithi083 ceph-mon[79930]: pgmap v1783: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:27:37.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:27:37 smithi083 ceph-mon[77242]: pgmap v1783: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:27:38.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:27:37 smithi153 ceph-mon[70044]: pgmap v1783: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:27:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:27:39 smithi083 ceph-mon[79930]: pgmap v1784: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:27:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:27:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:27:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:27:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:27:39.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:27:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:27:39.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:27:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:27:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:27:39 smithi083 ceph-mon[77242]: pgmap v1784: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:27:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:27:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:27:39.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:27:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:27:39.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:27:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:27:39.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:27:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:27:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:27:39 smithi153 ceph-mon[70044]: pgmap v1784: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:27:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:27:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:27:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:27:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:27:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:27:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:27:40.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:27:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:27:40.702 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:27:40.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:27:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:27:40.331Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:27:40.994 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:27:40.994 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (61m) 3m ago 61m 26.5M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:27:40.995 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (61m) 102s ago 61m 61.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:27:40.995 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (62m) 3m ago 62m 25.3M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:27:40.995 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (62m) 102s ago 62m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:27:40.995 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (58m) 102s ago 65m 439M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:27:40.995 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (58m) 3m ago 66m 560M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:27:40.996 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (58m) 3m ago 66m 275M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:27:40.996 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (57m) 102s ago 65m 132M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:27:40.996 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (57m) 3m ago 65m 166M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:27:40.996 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (62m) 3m ago 62m 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:27:40.997 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (62m) 102s ago 62m 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:27:40.997 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (56m) 3m ago 65m 209M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:27:40.997 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (56m) 3m ago 64m 201M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:27:40.997 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (56m) 3m ago 64m 163M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:27:40.997 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (55m) 3m ago 64m 191M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:27:40.998 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (54m) 102s ago 63m 176M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:27:40.998 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (54m) 102s ago 63m 207M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:27:40.998 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (53m) 102s ago 63m 184M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:27:40.998 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (53m) 102s ago 62m 209M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:27:40.999 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (58m) 102s ago 62m 103M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:27:41.349 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:27:41.350 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:27:41.350 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:27:41.350 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:27:41.351 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:27:41.351 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:27:41.351 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:27:41.351 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:27:41.351 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:27:41.352 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:27:41.352 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:27:41.352 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:27:41.352 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:27:41.353 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:27:41.353 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:27:41.353 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:27:41.353 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:27:41.353 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:27:41.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:27:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:27:41] "GET /metrics HTTP/1.1" 200 225178 "" "Prometheus/2.33.4" 2022-09-29T06:27:41.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:27:41 smithi083 ceph-mon[79930]: pgmap v1785: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:27:41.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:27:41 smithi083 ceph-mon[79930]: from='client.46147 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:27:41.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:27:41 smithi083 ceph-mon[79930]: from='client.36219 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:27:41.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:27:41 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4047043817' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:27:41.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:27:41 smithi083 ceph-mon[77242]: pgmap v1785: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:27:41.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:27:41 smithi083 ceph-mon[77242]: from='client.46147 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:27:41.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:27:41 smithi083 ceph-mon[77242]: from='client.36219 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:27:41.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:27:41 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4047043817' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:27:41.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:27:41 smithi153 ceph-mon[70044]: pgmap v1785: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:27:41.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:27:41 smithi153 ceph-mon[70044]: from='client.46147 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:27:41.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:27:41 smithi153 ceph-mon[70044]: from='client.36219 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:27:41.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:27:41 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4047043817' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:27:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:27:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:27:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:27:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:27:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:27:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:27:43.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:27:43 smithi083 ceph-mon[77242]: pgmap v1786: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:27:43.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:27:43 smithi083 ceph-mon[79930]: pgmap v1786: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:27:43.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:27:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:27:43.741Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:27:44.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:27:43 smithi153 ceph-mon[70044]: pgmap v1786: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:27:44.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:27:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:27:44.437Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:27:45.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:27:45 smithi083 ceph-mon[79930]: pgmap v1787: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:27:45.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:27:45 smithi083 ceph-mon[77242]: pgmap v1787: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:27:46.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:27:45 smithi153 ceph-mon[70044]: pgmap v1787: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:27:47.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:27:47 smithi083 ceph-mon[79930]: pgmap v1788: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:27:47.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:27:47 smithi083 ceph-mon[77242]: pgmap v1788: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:27:48.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:27:47 smithi153 ceph-mon[70044]: pgmap v1788: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:27:49.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:27:49 smithi083 ceph-mon[79930]: pgmap v1789: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:27:49.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:27:49 smithi083 ceph-mon[77242]: pgmap v1789: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:27:50.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:27:49 smithi153 ceph-mon[70044]: pgmap v1789: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:27:50.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:27:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:27:50.332Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:27:51.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:27:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:27:51] "GET /metrics HTTP/1.1" 200 225178 "" "Prometheus/2.33.4" 2022-09-29T06:27:51.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:27:51 smithi083 ceph-mon[79930]: pgmap v1790: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:27:51.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:27:51 smithi083 ceph-mon[77242]: pgmap v1790: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:27:51.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:27:51 smithi153 ceph-mon[70044]: pgmap v1790: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:27:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:27:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:27:52.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:27:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:27:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:27:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:27:53.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:27:53 smithi083 ceph-mon[79930]: pgmap v1791: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:27:53.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:27:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:27:53.742Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:27:53.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:27:53 smithi083 ceph-mon[77242]: pgmap v1791: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:27:54.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:27:53 smithi153 ceph-mon[70044]: pgmap v1791: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:27:54.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:27:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:27:54.438Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:27:55.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:27:55 smithi083 ceph-mon[79930]: pgmap v1792: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:27:55.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:27:55 smithi083 ceph-mon[77242]: pgmap v1792: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:27:56.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:27:55 smithi153 ceph-mon[70044]: pgmap v1792: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:27:58.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:27:57 smithi153 ceph-mon[70044]: pgmap v1793: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:27:58.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:27:57 smithi083 ceph-mon[79930]: pgmap v1793: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:27:58.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:27:57 smithi083 ceph-mon[77242]: pgmap v1793: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:00.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:27:59 smithi153 ceph-mon[70044]: pgmap v1794: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:00.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:27:59 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:28:00.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:27:59 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:28:00.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:27:59 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:28:00.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:27:59 smithi083 ceph-mon[79930]: pgmap v1794: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:00.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:27:59 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:28:00.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:27:59 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:28:00.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:27:59 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:28:00.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:27:59 smithi083 ceph-mon[77242]: pgmap v1794: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:00.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:27:59 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:28:00.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:27:59 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:28:00.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:27:59 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:28:00.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:28:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:28:00.332Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:28:01.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:28:01 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:28:01.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:28:01 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:28:01.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:28:01 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:28:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:28:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:28:01] "GET /metrics HTTP/1.1" 200 225180 "" "Prometheus/2.33.4" 2022-09-29T06:28:02.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:28:02 smithi153 ceph-mon[70044]: pgmap v1795: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:02.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:28:02 smithi083 ceph-mon[79930]: pgmap v1795: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:02.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:28:02 smithi083 ceph-mon[77242]: pgmap v1795: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:28:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:28:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:28:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:28:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:28:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:28:04.037 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:28:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:28:03.743Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:28:04.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:28:04 smithi153 ceph-mon[70044]: pgmap v1796: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:04.440 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:28:04 smithi083 ceph-mon[79930]: pgmap v1796: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:04.440 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:28:04 smithi083 ceph-mon[77242]: pgmap v1796: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:04.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:28:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:28:04.438Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:28:06.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:28:06 smithi153 ceph-mon[70044]: pgmap v1797: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:06.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:28:06 smithi083 ceph-mon[79930]: pgmap v1797: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:06.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:28:06 smithi083 ceph-mon[77242]: pgmap v1797: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:08.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:28:08 smithi153 ceph-mon[70044]: pgmap v1798: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:08.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:28:08 smithi083 ceph-mon[77242]: pgmap v1798: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:08.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:28:08 smithi083 ceph-mon[79930]: pgmap v1798: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:10.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:28:10 smithi153 ceph-mon[70044]: pgmap v1799: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:10.334 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:28:10 smithi083 ceph-mon[79930]: pgmap v1799: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:10.335 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:28:10 smithi083 ceph-mon[77242]: pgmap v1799: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:10.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:28:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:28:10.333Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:28:11.678 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:28:11.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:28:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:28:11] "GET /metrics HTTP/1.1" 200 225174 "" "Prometheus/2.33.4" 2022-09-29T06:28:11.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:28:11 smithi083 ceph-mon[79930]: pgmap v1800: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:11.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:28:11 smithi083 ceph-mon[77242]: pgmap v1800: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:11.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:28:11 smithi153 ceph-mon[70044]: pgmap v1800: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:11.967 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:28:11.967 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (61m) 4m ago 62m 26.5M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:28:11.968 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (62m) 2m ago 62m 61.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:28:11.968 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (63m) 4m ago 63m 25.3M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:28:11.968 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (63m) 2m ago 63m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:28:11.968 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (58m) 2m ago 65m 439M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:28:11.968 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (59m) 4m ago 67m 560M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:28:11.969 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (58m) 4m ago 67m 275M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:28:11.969 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (58m) 2m ago 66m 132M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:28:11.969 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (58m) 4m ago 66m 166M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:28:11.970 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (62m) 4m ago 62m 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:28:11.970 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (62m) 2m ago 62m 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:28:11.970 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (57m) 4m ago 65m 209M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:28:11.970 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (56m) 4m ago 65m 201M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:28:11.971 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (56m) 4m ago 64m 163M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:28:11.971 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (55m) 4m ago 64m 191M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:28:11.971 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (55m) 2m ago 64m 176M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:28:11.971 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (54m) 2m ago 64m 207M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:28:11.971 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (54m) 2m ago 63m 184M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:28:11.972 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (53m) 2m ago 63m 209M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:28:11.972 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (59m) 2m ago 62m 103M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:28:12.320 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:28:12.321 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:28:12.321 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:28:12.321 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:28:12.321 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:28:12.322 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:28:12.322 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:28:12.322 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:28:12.322 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:28:12.323 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:28:12.323 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:28:12.323 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:28:12.323 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:28:12.324 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:28:12.324 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:28:12.324 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:28:12.325 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:28:12.325 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:28:12.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:28:12 smithi083 ceph-mon[79930]: from='client.36231 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:28:12.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:28:12 smithi083 ceph-mon[79930]: from='client.36237 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:28:12.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:28:12 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4174476123' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:28:12.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:28:12 smithi083 ceph-mon[77242]: from='client.36231 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:28:12.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:28:12 smithi083 ceph-mon[77242]: from='client.36237 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:28:12.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:28:12 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4174476123' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:28:12.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:28:12 smithi153 ceph-mon[70044]: from='client.36231 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:28:12.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:28:12 smithi153 ceph-mon[70044]: from='client.36237 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:28:12.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:28:12 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4174476123' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:28:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:28:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:28:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:28:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:28:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:28:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:28:13.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:28:13 smithi083 ceph-mon[79930]: pgmap v1801: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:13.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:28:13 smithi083 ceph-mon[77242]: pgmap v1801: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:13.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:28:13 smithi153 ceph-mon[70044]: pgmap v1801: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:14.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:28:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:28:13.743Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:28:14.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:28:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:28:14.439Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:28:16.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:28:15 smithi153 ceph-mon[70044]: pgmap v1802: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:16.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:28:15 smithi083 ceph-mon[79930]: pgmap v1802: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:16.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:28:15 smithi083 ceph-mon[77242]: pgmap v1802: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:18.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:28:17 smithi153 ceph-mon[70044]: pgmap v1803: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:18.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:28:17 smithi083 ceph-mon[79930]: pgmap v1803: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:18.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:28:17 smithi083 ceph-mon[77242]: pgmap v1803: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:20.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:28:19 smithi153 ceph-mon[70044]: pgmap v1804: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:20.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:28:19 smithi083 ceph-mon[79930]: pgmap v1804: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:20.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:28:19 smithi083 ceph-mon[77242]: pgmap v1804: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:20.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:28:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:28:20.334Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:28:21.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:28:21 smithi083 ceph-mon[79930]: pgmap v1805: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:21.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:28:21 smithi083 ceph-mon[77242]: pgmap v1805: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:21.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:28:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:28:21] "GET /metrics HTTP/1.1" 200 225174 "" "Prometheus/2.33.4" 2022-09-29T06:28:21.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:28:21 smithi153 ceph-mon[70044]: pgmap v1805: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:28:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:28:22.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:28:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:28:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:28:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:28:24.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:28:23 smithi153 ceph-mon[70044]: pgmap v1806: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:24.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:28:23 smithi083 ceph-mon[79930]: pgmap v1806: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:24.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:28:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:28:23.743Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:28:24.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:28:23 smithi083 ceph-mon[77242]: pgmap v1806: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:24.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:28:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:28:24.440Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:28:26.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:28:25 smithi153 ceph-mon[70044]: pgmap v1807: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:26.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:28:25 smithi083 ceph-mon[79930]: pgmap v1807: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:26.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:28:25 smithi083 ceph-mon[77242]: pgmap v1807: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:28.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:28:27 smithi153 ceph-mon[70044]: pgmap v1808: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:28.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:28:27 smithi083 ceph-mon[79930]: pgmap v1808: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:28.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:28:27 smithi083 ceph-mon[77242]: pgmap v1808: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:30.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:28:29 smithi153 ceph-mon[70044]: pgmap v1809: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:30.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:28:29 smithi083 ceph-mon[79930]: pgmap v1809: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:30.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:28:29 smithi083 ceph-mon[77242]: pgmap v1809: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:30.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:28:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:28:30.335Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:28:31.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:28:31 smithi083 ceph-mon[79930]: pgmap v1810: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:31.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:28:31 smithi083 ceph-mon[77242]: pgmap v1810: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:31.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:28:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:28:31] "GET /metrics HTTP/1.1" 200 225173 "" "Prometheus/2.33.4" 2022-09-29T06:28:31.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:28:31 smithi153 ceph-mon[70044]: pgmap v1810: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:28:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:28:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:28:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:28:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:28:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:28:34.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:28:33 smithi153 ceph-mon[70044]: pgmap v1811: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:34.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:28:33 smithi083 ceph-mon[79930]: pgmap v1811: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:34.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:28:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:28:33.744Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:28:34.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:28:33 smithi083 ceph-mon[77242]: pgmap v1811: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:34.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:28:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:28:34.440Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:28:36.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:28:35 smithi153 ceph-mon[70044]: pgmap v1812: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:36.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:28:35 smithi083 ceph-mon[79930]: pgmap v1812: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:36.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:28:35 smithi083 ceph-mon[77242]: pgmap v1812: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:38.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:28:37 smithi153 ceph-mon[70044]: pgmap v1813: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:38.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:28:37 smithi083 ceph-mon[77242]: pgmap v1813: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:38.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:28:37 smithi083 ceph-mon[79930]: pgmap v1813: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:28:39 smithi153 ceph-mon[70044]: pgmap v1814: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:28:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:28:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:28:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:28:40.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:28:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:28:40.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:28:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:28:40.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:28:39 smithi083 ceph-mon[79930]: pgmap v1814: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:28:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:28:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:28:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:28:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:28:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:28:40.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:28:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:28:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:28:39 smithi083 ceph-mon[77242]: pgmap v1814: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:28:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:28:40.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:28:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:28:40.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:28:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:28:40.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:28:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:28:40.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:28:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:28:40.336Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:28:41.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:28:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:28:41] "GET /metrics HTTP/1.1" 200 225177 "" "Prometheus/2.33.4" 2022-09-29T06:28:41.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:28:41 smithi083 ceph-mon[79930]: pgmap v1815: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:41.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:28:41 smithi083 ceph-mon[77242]: pgmap v1815: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:41.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:28:41 smithi153 ceph-mon[70044]: pgmap v1815: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:42.647 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:28:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:28:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:28:42.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:28:42.934 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:28:42.935 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (62m) 4m ago 62m 26.5M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:28:42.935 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (62m) 2m ago 62m 61.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:28:42.935 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (63m) 4m ago 63m 25.3M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:28:42.936 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (63m) 2m ago 63m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:28:42.936 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (59m) 2m ago 66m 439M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:28:42.936 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (59m) 4m ago 67m 560M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:28:42.936 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (59m) 4m ago 67m 275M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:28:42.937 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (58m) 2m ago 66m 132M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:28:42.937 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (58m) 4m ago 66m 166M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:28:42.937 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (63m) 4m ago 63m 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:28:42.937 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (63m) 2m ago 63m 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:28:42.937 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (57m) 4m ago 66m 209M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:28:42.938 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (57m) 4m ago 65m 201M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:28:42.938 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (57m) 4m ago 65m 163M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:28:42.938 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (56m) 4m ago 65m 191M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:28:42.938 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (55m) 2m ago 64m 176M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:28:42.939 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (55m) 2m ago 64m 207M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:28:42.939 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (54m) 2m ago 64m 184M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:28:42.939 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (54m) 2m ago 64m 209M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:28:42.939 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (59m) 2m ago 63m 103M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:28:43.290 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:28:43.290 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:28:43.291 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:28:43.291 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:28:43.291 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:28:43.291 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:28:43.292 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:28:43.292 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:28:43.292 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:28:43.292 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:28:43.293 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:28:43.293 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:28:43.293 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:28:43.293 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:28:43.293 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:28:43.294 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:28:43.294 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:28:43.294 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:28:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:28:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:28:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:28:44.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:28:43 smithi153 ceph-mon[70044]: from='client.36249 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:28:44.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:28:43 smithi153 ceph-mon[70044]: pgmap v1816: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:44.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:28:43 smithi153 ceph-mon[70044]: from='client.36255 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:28:44.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:28:43 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3348817810' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:28:44.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:28:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:28:43.744Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:28:44.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:28:43 smithi083 ceph-mon[79930]: from='client.36249 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:28:44.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:28:43 smithi083 ceph-mon[79930]: pgmap v1816: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:44.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:28:43 smithi083 ceph-mon[79930]: from='client.36255 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:28:44.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:28:43 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3348817810' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:28:44.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:28:43 smithi083 ceph-mon[77242]: from='client.36249 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:28:44.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:28:43 smithi083 ceph-mon[77242]: pgmap v1816: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:44.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:28:43 smithi083 ceph-mon[77242]: from='client.36255 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:28:44.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:28:43 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3348817810' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:28:44.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:28:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:28:44.440Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:28:46.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:28:45 smithi153 ceph-mon[70044]: pgmap v1817: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:46.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:28:45 smithi083 ceph-mon[79930]: pgmap v1817: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:46.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:28:45 smithi083 ceph-mon[77242]: pgmap v1817: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:48.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:28:47 smithi153 ceph-mon[70044]: pgmap v1818: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:48.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:28:47 smithi083 ceph-mon[79930]: pgmap v1818: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:48.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:28:47 smithi083 ceph-mon[77242]: pgmap v1818: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:50.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:28:49 smithi153 ceph-mon[70044]: pgmap v1819: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:50.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:28:49 smithi083 ceph-mon[77242]: pgmap v1819: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:50.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:28:49 smithi083 ceph-mon[79930]: pgmap v1819: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:50.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:28:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:28:50.337Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:28:51.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:28:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:28:51] "GET /metrics HTTP/1.1" 200 225177 "" "Prometheus/2.33.4" 2022-09-29T06:28:51.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:28:51 smithi083 ceph-mon[79930]: pgmap v1820: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:51.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:28:51 smithi083 ceph-mon[77242]: pgmap v1820: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:51.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:28:51 smithi153 ceph-mon[70044]: pgmap v1820: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:28:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:28:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:28:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:28:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:28:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:28:54.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:28:53 smithi153 ceph-mon[70044]: pgmap v1821: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:54.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:28:53 smithi083 ceph-mon[79930]: pgmap v1821: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:54.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:28:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:28:53.745Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:28:54.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:28:53 smithi083 ceph-mon[77242]: pgmap v1821: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:54.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:28:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:28:54.440Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:28:56.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:28:55 smithi153 ceph-mon[70044]: pgmap v1822: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:56.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:28:55 smithi083 ceph-mon[79930]: pgmap v1822: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:56.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:28:55 smithi083 ceph-mon[77242]: pgmap v1822: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:58.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:28:57 smithi153 ceph-mon[70044]: pgmap v1823: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:58.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:28:57 smithi083 ceph-mon[79930]: pgmap v1823: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:28:58.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:28:57 smithi083 ceph-mon[77242]: pgmap v1823: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:00.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:28:59 smithi153 ceph-mon[70044]: pgmap v1824: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:00.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:28:59 smithi083 ceph-mon[79930]: pgmap v1824: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:00.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:28:59 smithi083 ceph-mon[77242]: pgmap v1824: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:00.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:29:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:29:00.337Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:29:01.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:29:00 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:29:01.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:29:00 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:29:01.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:29:00 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:29:01.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:29:00 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:29:01.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:29:00 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:29:01.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:29:00 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:29:01.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:29:00 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:29:01.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:29:00 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:29:01.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:29:00 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:29:01.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:29:00 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:29:01.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:29:00 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:29:01.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:29:00 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:29:01.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:29:01 smithi083 ceph-mon[77242]: pgmap v1825: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:01.965 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:29:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:29:01] "GET /metrics HTTP/1.1" 200 225178 "" "Prometheus/2.33.4" 2022-09-29T06:29:01.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:29:01 smithi083 ceph-mon[79930]: pgmap v1825: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:02.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:29:01 smithi153 ceph-mon[70044]: pgmap v1825: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:29:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:29:02.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:29:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:29:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:29:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:29:04.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:29:03 smithi153 ceph-mon[70044]: pgmap v1826: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:04.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:29:03 smithi083 ceph-mon[79930]: pgmap v1826: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:04.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:29:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:29:03.746Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:29:04.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:29:03 smithi083 ceph-mon[77242]: pgmap v1826: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:04.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:29:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:29:04.441Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:29:06.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:29:05 smithi153 ceph-mon[70044]: pgmap v1827: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:06.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:29:05 smithi083 ceph-mon[77242]: pgmap v1827: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:06.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:29:05 smithi083 ceph-mon[79930]: pgmap v1827: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:08.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:29:07 smithi153 ceph-mon[70044]: pgmap v1828: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:08.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:29:07 smithi083 ceph-mon[79930]: pgmap v1828: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:08.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:29:07 smithi083 ceph-mon[77242]: pgmap v1828: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:10.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:29:09 smithi153 ceph-mon[70044]: pgmap v1829: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:10.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:29:09 smithi083 ceph-mon[79930]: pgmap v1829: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:10.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:29:09 smithi083 ceph-mon[77242]: pgmap v1829: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:10.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:29:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:29:10.338Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:29:11.716 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:29:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:29:11] "GET /metrics HTTP/1.1" 200 225173 "" "Prometheus/2.33.4" 2022-09-29T06:29:11.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:29:11 smithi083 ceph-mon[79930]: pgmap v1830: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:11.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:29:11 smithi083 ceph-mon[77242]: pgmap v1830: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:11.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:29:11 smithi153 ceph-mon[70044]: pgmap v1830: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:29:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:29:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:29:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:29:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:29:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:29:13.626 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:29:13.924 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:29:13.924 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (62m) 5m ago 63m 26.5M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:29:13.924 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (63m) 3m ago 63m 61.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:29:13.924 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (64m) 5m ago 64m 25.3M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:29:13.925 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (64m) 3m ago 64m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:29:13.925 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (59m) 3m ago 66m 439M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:29:13.925 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (60m) 5m ago 68m 560M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:29:13.926 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (59m) 5m ago 68m 275M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:29:13.926 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (59m) 3m ago 67m 132M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:29:13.926 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (59m) 5m ago 67m 166M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:29:13.926 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (63m) 5m ago 63m 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:29:13.926 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (63m) 3m ago 63m 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:29:13.927 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (58m) 5m ago 66m 209M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:29:13.927 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (57m) 5m ago 66m 201M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:29:13.927 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (57m) 5m ago 65m 163M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:29:13.927 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (56m) 5m ago 65m 191M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:29:13.928 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (56m) 3m ago 65m 176M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:29:13.928 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (55m) 3m ago 65m 207M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:29:13.928 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (55m) 3m ago 64m 184M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:29:13.928 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (54m) 3m ago 64m 209M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:29:13.929 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (60m) 3m ago 63m 103M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:29:14.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:29:13 smithi153 ceph-mon[70044]: pgmap v1831: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:14.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:29:13 smithi083 ceph-mon[79930]: pgmap v1831: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:14.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:29:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:29:13.745Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:29:14.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:29:13 smithi083 ceph-mon[77242]: pgmap v1831: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:14.284 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:29:14.285 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:29:14.285 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:29:14.285 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:29:14.285 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:29:14.286 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:29:14.286 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:29:14.286 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:29:14.286 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:29:14.287 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:29:14.287 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:29:14.287 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:29:14.287 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:29:14.288 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:29:14.288 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:29:14.288 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:29:14.288 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:29:14.289 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:29:14.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:29:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:29:14.441Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:29:15.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:29:14 smithi153 ceph-mon[70044]: from='client.46201 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:29:15.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:29:14 smithi153 ceph-mon[70044]: from='client.46207 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:29:15.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:29:14 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2889216443' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:29:15.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:29:14 smithi083 ceph-mon[77242]: from='client.46201 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:29:15.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:29:14 smithi083 ceph-mon[77242]: from='client.46207 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:29:15.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:29:14 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2889216443' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:29:15.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:29:14 smithi083 ceph-mon[79930]: from='client.46201 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:29:15.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:29:14 smithi083 ceph-mon[79930]: from='client.46207 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:29:15.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:29:14 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2889216443' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:29:16.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:29:15 smithi153 ceph-mon[70044]: pgmap v1832: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:16.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:29:15 smithi083 ceph-mon[79930]: pgmap v1832: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:16.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:29:15 smithi083 ceph-mon[77242]: pgmap v1832: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:18.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:29:17 smithi153 ceph-mon[70044]: pgmap v1833: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:18.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:29:17 smithi083 ceph-mon[79930]: pgmap v1833: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:18.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:29:17 smithi083 ceph-mon[77242]: pgmap v1833: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:20.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:29:19 smithi153 ceph-mon[70044]: pgmap v1834: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:20.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:29:19 smithi083 ceph-mon[77242]: pgmap v1834: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:20.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:29:19 smithi083 ceph-mon[79930]: pgmap v1834: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:20.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:29:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:29:20.338Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:29:21.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:29:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:29:21] "GET /metrics HTTP/1.1" 200 225173 "" "Prometheus/2.33.4" 2022-09-29T06:29:21.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:29:21 smithi083 ceph-mon[79930]: pgmap v1835: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:21.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:29:21 smithi083 ceph-mon[77242]: pgmap v1835: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:21.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:29:21 smithi153 ceph-mon[70044]: pgmap v1835: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:29:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:29:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:29:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:29:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:29:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:29:24.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:29:23 smithi153 ceph-mon[70044]: pgmap v1836: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:24.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:29:23 smithi083 ceph-mon[79930]: pgmap v1836: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:24.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:29:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:29:23.747Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:29:24.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:29:23 smithi083 ceph-mon[77242]: pgmap v1836: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:24.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:29:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:29:24.442Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:29:26.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:29:25 smithi153 ceph-mon[70044]: pgmap v1837: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:26.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:29:25 smithi083 ceph-mon[79930]: pgmap v1837: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:26.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:29:25 smithi083 ceph-mon[77242]: pgmap v1837: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:28.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:29:27 smithi153 ceph-mon[70044]: pgmap v1838: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:28.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:29:27 smithi083 ceph-mon[79930]: pgmap v1838: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:28.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:29:27 smithi083 ceph-mon[77242]: pgmap v1838: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:30.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:29:29 smithi153 ceph-mon[70044]: pgmap v1839: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:30.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:29:29 smithi083 ceph-mon[79930]: pgmap v1839: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:30.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:29:29 smithi083 ceph-mon[77242]: pgmap v1839: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:30.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:29:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:29:30.339Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:29:31.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:29:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:29:31] "GET /metrics HTTP/1.1" 200 225176 "" "Prometheus/2.33.4" 2022-09-29T06:29:31.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:29:31 smithi083 ceph-mon[79930]: pgmap v1840: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:31.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:29:31 smithi083 ceph-mon[77242]: pgmap v1840: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:31.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:29:31 smithi153 ceph-mon[70044]: pgmap v1840: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:29:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:29:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:29:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:29:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:29:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:29:34.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:29:33 smithi153 ceph-mon[70044]: pgmap v1841: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:34.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:29:33 smithi083 ceph-mon[79930]: pgmap v1841: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:34.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:29:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:29:33.747Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:29:34.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:29:33 smithi083 ceph-mon[77242]: pgmap v1841: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:34.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:29:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:29:34.444Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:29:36.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:29:35 smithi153 ceph-mon[70044]: pgmap v1842: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:36.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:29:35 smithi083 ceph-mon[77242]: pgmap v1842: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:36.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:29:35 smithi083 ceph-mon[79930]: pgmap v1842: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:38.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:29:37 smithi153 ceph-mon[70044]: pgmap v1843: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:38.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:29:37 smithi083 ceph-mon[79930]: pgmap v1843: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:38.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:29:37 smithi083 ceph-mon[77242]: pgmap v1843: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:29:39 smithi153 ceph-mon[70044]: pgmap v1844: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:29:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:29:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:29:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:29:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:29:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:29:40.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:29:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:29:40.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:29:39 smithi083 ceph-mon[79930]: pgmap v1844: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:29:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:29:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:29:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:29:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:29:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:29:40.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:29:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:29:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:29:39 smithi083 ceph-mon[77242]: pgmap v1844: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:29:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:29:40.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:29:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:29:40.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:29:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:29:40.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:29:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:29:40.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:29:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:29:40.340Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:29:41.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:29:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:29:41] "GET /metrics HTTP/1.1" 200 225181 "" "Prometheus/2.33.4" 2022-09-29T06:29:41.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:29:41 smithi083 ceph-mon[79930]: pgmap v1845: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:41.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:29:41 smithi083 ceph-mon[77242]: pgmap v1845: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:41.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:29:41 smithi153 ceph-mon[70044]: pgmap v1845: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:29:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:29:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:29:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:29:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:29:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:29:44.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:29:43 smithi153 ceph-mon[70044]: pgmap v1846: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:44.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:29:43 smithi083 ceph-mon[79930]: pgmap v1846: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:44.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:29:43 smithi083 ceph-mon[77242]: pgmap v1846: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:44.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:29:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:29:43.748Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:29:44.610 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:29:44.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:29:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:29:44.443Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:29:44.901 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:29:44.902 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (63m) 5m ago 63m 26.5M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:29:44.902 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (63m) 3m ago 63m 61.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:29:44.902 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (64m) 5m ago 64m 25.3M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:29:44.902 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (64m) 3m ago 64m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:29:44.903 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (60m) 3m ago 67m 439M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:29:44.903 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (60m) 5m ago 68m 560M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:29:44.903 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (60m) 5m ago 68m 275M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:29:44.903 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (59m) 3m ago 67m 132M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:29:44.904 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (59m) 5m ago 67m 166M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:29:44.904 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (64m) 5m ago 64m 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:29:44.904 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (64m) 3m ago 64m 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:29:44.904 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (58m) 5m ago 67m 209M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:29:44.904 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (58m) 5m ago 66m 201M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:29:44.905 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (58m) 5m ago 66m 163M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:29:44.905 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (57m) 5m ago 66m 191M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:29:44.905 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (56m) 3m ago 65m 176M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:29:44.905 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (56m) 3m ago 65m 207M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:29:44.906 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (55m) 3m ago 65m 184M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:29:44.906 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (55m) 3m ago 65m 209M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:29:44.906 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (60m) 3m ago 64m 103M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:29:45.253 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:29:45.254 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:29:45.254 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:29:45.254 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:29:45.255 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:29:45.255 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:29:45.255 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:29:45.255 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:29:45.255 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:29:45.256 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:29:45.256 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:29:45.256 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:29:45.256 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:29:45.257 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:29:45.257 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:29:45.257 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:29:45.257 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:29:45.258 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:29:46.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:29:45 smithi153 ceph-mon[70044]: from='client.36285 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:29:46.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:29:45 smithi153 ceph-mon[70044]: pgmap v1847: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:46.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:29:45 smithi153 ceph-mon[70044]: from='client.36291 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:29:46.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:29:45 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2349507677' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:29:46.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:29:45 smithi083 ceph-mon[79930]: from='client.36285 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:29:46.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:29:45 smithi083 ceph-mon[79930]: pgmap v1847: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:46.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:29:45 smithi083 ceph-mon[79930]: from='client.36291 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:29:46.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:29:45 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2349507677' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:29:46.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:29:45 smithi083 ceph-mon[77242]: from='client.36285 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:29:46.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:29:45 smithi083 ceph-mon[77242]: pgmap v1847: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:46.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:29:45 smithi083 ceph-mon[77242]: from='client.36291 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:29:46.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:29:45 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2349507677' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:29:48.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:29:47 smithi153 ceph-mon[70044]: pgmap v1848: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:48.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:29:47 smithi083 ceph-mon[79930]: pgmap v1848: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:48.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:29:47 smithi083 ceph-mon[77242]: pgmap v1848: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:50.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:29:49 smithi153 ceph-mon[70044]: pgmap v1849: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:50.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:29:49 smithi083 ceph-mon[77242]: pgmap v1849: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:50.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:29:49 smithi083 ceph-mon[79930]: pgmap v1849: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:50.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:29:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:29:50.340Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:29:51.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:29:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:29:51] "GET /metrics HTTP/1.1" 200 225181 "" "Prometheus/2.33.4" 2022-09-29T06:29:51.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:29:51 smithi083 ceph-mon[79930]: pgmap v1850: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:51.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:29:51 smithi083 ceph-mon[77242]: pgmap v1850: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:51.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:29:51 smithi153 ceph-mon[70044]: pgmap v1850: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:29:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:29:52.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:29:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:29:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:29:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:29:54.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:29:53 smithi153 ceph-mon[70044]: pgmap v1851: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:54.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:29:53 smithi083 ceph-mon[79930]: pgmap v1851: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:54.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:29:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:29:53.748Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:29:54.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:29:53 smithi083 ceph-mon[77242]: pgmap v1851: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:54.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:29:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:29:54.445Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:29:56.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:29:55 smithi153 ceph-mon[70044]: pgmap v1852: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:56.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:29:55 smithi083 ceph-mon[79930]: pgmap v1852: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:56.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:29:55 smithi083 ceph-mon[77242]: pgmap v1852: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:58.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:29:57 smithi153 ceph-mon[70044]: pgmap v1853: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:58.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:29:57 smithi083 ceph-mon[79930]: pgmap v1853: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:29:58.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:29:57 smithi083 ceph-mon[77242]: pgmap v1853: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:00.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:29:59 smithi153 ceph-mon[70044]: pgmap v1854: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:00.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:29:59 smithi083 ceph-mon[79930]: pgmap v1854: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:00.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:29:59 smithi083 ceph-mon[77242]: pgmap v1854: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:00.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:30:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[77219]: 2022-09-29T06:29:59.999+0000 7f160f665700 -1 log_channel(cluster) log [ERR] : overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T06:30:00.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:30:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:30:00.340Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:30:01.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:30:00 smithi153 ceph-mon[70044]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T06:30:01.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:30:00 smithi083 ceph-mon[79930]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T06:30:01.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:30:00 smithi083 ceph-mon[77242]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T06:30:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:30:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:30:01] "GET /metrics HTTP/1.1" 200 225175 "" "Prometheus/2.33.4" 2022-09-29T06:30:01.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:30:01 smithi083 ceph-mon[77242]: pgmap v1855: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:01.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:30:01 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:30:01.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:30:01 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:30:01.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:30:01 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:30:01.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:30:01 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:30:01.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:30:01 smithi083 ceph-mon[79930]: pgmap v1855: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:01.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:30:01 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:30:01.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:30:01 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:30:01.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:30:01 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:30:01.968 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:30:01 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:30:02.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:30:01 smithi153 ceph-mon[70044]: pgmap v1855: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:02.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:30:01 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:30:02.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:30:01 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:30:02.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:30:01 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:30:02.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:30:01 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:30:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:30:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:30:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:30:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:30:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:30:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:30:04.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:30:03 smithi153 ceph-mon[70044]: pgmap v1856: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:04.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:30:03 smithi083 ceph-mon[79930]: pgmap v1856: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:04.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:30:03 smithi083 ceph-mon[77242]: pgmap v1856: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:04.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:30:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:30:03.749Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:30:04.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:30:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:30:04.445Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:30:06.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:30:05 smithi153 ceph-mon[70044]: pgmap v1857: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:06.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:30:05 smithi083 ceph-mon[79930]: pgmap v1857: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:06.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:30:05 smithi083 ceph-mon[77242]: pgmap v1857: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:08.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:30:07 smithi153 ceph-mon[70044]: pgmap v1858: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:08.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:30:07 smithi083 ceph-mon[79930]: pgmap v1858: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:08.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:30:07 smithi083 ceph-mon[77242]: pgmap v1858: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:10.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:30:09 smithi153 ceph-mon[70044]: pgmap v1859: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:10.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:30:09 smithi083 ceph-mon[79930]: pgmap v1859: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:10.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:30:09 smithi083 ceph-mon[77242]: pgmap v1859: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:10.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:30:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:30:10.341Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:30:11.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:30:11 smithi083 ceph-mon[79930]: pgmap v1860: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:11.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:30:11 smithi083 ceph-mon[77242]: pgmap v1860: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:11.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:30:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:30:11] "GET /metrics HTTP/1.1" 200 225180 "" "Prometheus/2.33.4" 2022-09-29T06:30:11.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:30:11 smithi153 ceph-mon[70044]: pgmap v1860: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:30:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:30:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:30:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:30:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:30:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:30:14.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:30:13 smithi153 ceph-mon[70044]: pgmap v1861: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:14.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:30:13 smithi083 ceph-mon[79930]: pgmap v1861: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:14.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:30:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:30:13.750Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:30:14.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:30:13 smithi083 ceph-mon[77242]: pgmap v1861: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:14.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:30:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:30:14.446Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:30:15.583 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:30:15.874 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:30:15.875 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (63m) 6m ago 64m 26.5M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:30:15.875 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (64m) 4m ago 64m 61.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:30:15.875 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (65m) 6m ago 65m 25.3M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:30:15.876 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (65m) 4m ago 65m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:30:15.876 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (61m) 4m ago 67m 439M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:30:15.876 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (61m) 6m ago 69m 560M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:30:15.876 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (60m) 6m ago 69m 275M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:30:15.877 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (60m) 4m ago 68m 132M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:30:15.877 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (60m) 6m ago 68m 166M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:30:15.877 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (64m) 6m ago 64m 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:30:15.877 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (64m) 4m ago 64m 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:30:15.878 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (59m) 6m ago 67m 209M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:30:15.878 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (58m) 6m ago 67m 201M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:30:15.878 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (58m) 6m ago 66m 163M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:30:15.879 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (58m) 6m ago 66m 191M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:30:15.879 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (57m) 4m ago 66m 176M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:30:15.879 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (56m) 4m ago 66m 207M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:30:15.879 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (56m) 4m ago 65m 184M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:30:15.880 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (55m) 4m ago 65m 209M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:30:15.880 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (61m) 4m ago 64m 103M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:30:16.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:30:15 smithi153 ceph-mon[70044]: pgmap v1862: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:16.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:30:15 smithi083 ceph-mon[79930]: pgmap v1862: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:16.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:30:15 smithi083 ceph-mon[77242]: pgmap v1862: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:16.231 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:30:16.231 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:30:16.231 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:30:16.232 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:30:16.232 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:30:16.232 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:30:16.232 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:30:16.233 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:30:16.233 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:30:16.233 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:30:16.233 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:30:16.233 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:30:16.234 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:30:16.234 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:30:16.234 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:30:16.234 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:30:16.235 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:30:16.235 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:30:17.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:30:16 smithi153 ceph-mon[70044]: from='client.36303 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:30:17.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:30:16 smithi153 ceph-mon[70044]: from='client.36309 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:30:17.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:30:16 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2511329366' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:30:17.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:30:16 smithi083 ceph-mon[77242]: from='client.36303 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:30:17.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:30:16 smithi083 ceph-mon[77242]: from='client.36309 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:30:17.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:30:16 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2511329366' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:30:17.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:30:16 smithi083 ceph-mon[79930]: from='client.36303 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:30:17.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:30:16 smithi083 ceph-mon[79930]: from='client.36309 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:30:17.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:30:16 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2511329366' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:30:18.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:30:17 smithi153 ceph-mon[70044]: pgmap v1863: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:18.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:30:17 smithi083 ceph-mon[79930]: pgmap v1863: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:18.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:30:17 smithi083 ceph-mon[77242]: pgmap v1863: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:20.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:30:19 smithi153 ceph-mon[70044]: pgmap v1864: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:20.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:30:19 smithi083 ceph-mon[79930]: pgmap v1864: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:20.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:30:19 smithi083 ceph-mon[77242]: pgmap v1864: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:20.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:30:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:30:20.341Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:30:21.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:30:21 smithi083 ceph-mon[79930]: pgmap v1865: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:21.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:30:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:30:21] "GET /metrics HTTP/1.1" 200 225180 "" "Prometheus/2.33.4" 2022-09-29T06:30:21.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:30:21 smithi083 ceph-mon[77242]: pgmap v1865: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:21.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:30:21 smithi153 ceph-mon[70044]: pgmap v1865: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:30:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:30:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:30:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:30:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:30:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:30:24.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:30:23 smithi153 ceph-mon[70044]: pgmap v1866: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:24.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:30:23 smithi083 ceph-mon[79930]: pgmap v1866: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:24.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:30:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:30:23.750Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:30:24.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:30:23 smithi083 ceph-mon[77242]: pgmap v1866: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:24.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:30:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:30:24.446Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:30:26.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:30:25 smithi153 ceph-mon[70044]: pgmap v1867: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:26.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:30:25 smithi083 ceph-mon[79930]: pgmap v1867: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:26.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:30:25 smithi083 ceph-mon[77242]: pgmap v1867: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:28.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:30:27 smithi153 ceph-mon[70044]: pgmap v1868: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:28.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:30:27 smithi083 ceph-mon[79930]: pgmap v1868: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:28.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:30:27 smithi083 ceph-mon[77242]: pgmap v1868: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:30.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:30:29 smithi153 ceph-mon[70044]: pgmap v1869: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:30.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:30:29 smithi083 ceph-mon[79930]: pgmap v1869: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:30.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:30:29 smithi083 ceph-mon[77242]: pgmap v1869: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:30.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:30:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:30:30.342Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:30:31.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:30:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:30:31] "GET /metrics HTTP/1.1" 200 225176 "" "Prometheus/2.33.4" 2022-09-29T06:30:31.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:30:31 smithi083 ceph-mon[79930]: pgmap v1870: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:31.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:30:31 smithi083 ceph-mon[77242]: pgmap v1870: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:31.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:30:31 smithi153 ceph-mon[70044]: pgmap v1870: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:30:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:30:32.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:30:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:30:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:30:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:30:34.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:30:33 smithi153 ceph-mon[70044]: pgmap v1871: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:34.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:30:33 smithi083 ceph-mon[79930]: pgmap v1871: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:34.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:30:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:30:33.750Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:30:34.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:30:33 smithi083 ceph-mon[77242]: pgmap v1871: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:34.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:30:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:30:34.446Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:30:36.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:30:35 smithi153 ceph-mon[70044]: pgmap v1872: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:36.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:30:35 smithi083 ceph-mon[79930]: pgmap v1872: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:36.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:30:35 smithi083 ceph-mon[77242]: pgmap v1872: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:38.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:30:37 smithi153 ceph-mon[70044]: pgmap v1873: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:38.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:30:37 smithi083 ceph-mon[79930]: pgmap v1873: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:38.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:30:37 smithi083 ceph-mon[77242]: pgmap v1873: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:30:39 smithi153 ceph-mon[70044]: pgmap v1874: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:30:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:30:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:30:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:30:40.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:30:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:30:40.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:30:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:30:40.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:30:39 smithi083 ceph-mon[79930]: pgmap v1874: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:30:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:30:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:30:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:30:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:30:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:30:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:30:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:30:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:30:39 smithi083 ceph-mon[77242]: pgmap v1874: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:30:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:30:40.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:30:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:30:40.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:30:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:30:40.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:30:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:30:40.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:30:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:30:40.342Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:30:41.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:30:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:30:41] "GET /metrics HTTP/1.1" 200 225180 "" "Prometheus/2.33.4" 2022-09-29T06:30:41.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:30:41 smithi083 ceph-mon[79930]: pgmap v1875: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:41.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:30:41 smithi083 ceph-mon[77242]: pgmap v1875: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:41.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:30:41 smithi153 ceph-mon[70044]: pgmap v1875: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:30:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:30:42.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:30:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:30:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:30:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:30:44.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:30:43 smithi153 ceph-mon[70044]: pgmap v1876: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:44.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:30:43 smithi083 ceph-mon[79930]: pgmap v1876: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:44.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:30:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:30:43.751Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:30:44.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:30:43 smithi083 ceph-mon[77242]: pgmap v1876: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:44.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:30:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:30:44.447Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:30:46.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:30:45 smithi153 ceph-mon[70044]: pgmap v1877: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:46.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:30:45 smithi083 ceph-mon[79930]: pgmap v1877: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:46.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:30:45 smithi083 ceph-mon[77242]: pgmap v1877: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:46.559 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:30:46.847 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:30:46.848 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (64m) 6m ago 64m 26.5M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:30:46.848 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (64m) 4m ago 64m 61.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:30:46.848 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (65m) 6m ago 65m 25.3M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:30:46.848 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (65m) 4m ago 65m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:30:46.849 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (61m) 4m ago 68m 439M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:30:46.849 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (61m) 6m ago 69m 560M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:30:46.849 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (61m) 6m ago 70m 275M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:30:46.849 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (60m) 4m ago 68m 132M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:30:46.850 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (60m) 6m ago 68m 166M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:30:46.850 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (65m) 6m ago 65m 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:30:46.850 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (65m) 4m ago 65m 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:30:46.850 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (59m) 6m ago 68m 209M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:30:46.851 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (59m) 6m ago 67m 201M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:30:46.851 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (59m) 6m ago 67m 163M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:30:46.851 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (58m) 6m ago 67m 191M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:30:46.851 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (57m) 4m ago 66m 176M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:30:46.852 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (57m) 4m ago 66m 207M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:30:46.852 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (56m) 4m ago 66m 184M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:30:46.852 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (56m) 4m ago 66m 209M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:30:46.852 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (61m) 4m ago 65m 103M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:30:47.201 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:30:47.202 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:30:47.202 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:30:47.203 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:30:47.203 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:30:47.203 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:30:47.203 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:30:47.204 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:30:47.204 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:30:47.204 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:30:47.204 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:30:47.205 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:30:47.205 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:30:47.205 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:30:47.205 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:30:47.206 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:30:47.206 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:30:47.206 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:30:48.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:30:47 smithi153 ceph-mon[70044]: from='client.46255 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:30:48.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:30:47 smithi153 ceph-mon[70044]: pgmap v1878: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:48.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:30:47 smithi153 ceph-mon[70044]: from='client.46261 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:30:48.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:30:47 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4130789317' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:30:48.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:30:47 smithi083 ceph-mon[79930]: from='client.46255 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:30:48.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:30:47 smithi083 ceph-mon[79930]: pgmap v1878: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:48.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:30:47 smithi083 ceph-mon[79930]: from='client.46261 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:30:48.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:30:47 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4130789317' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:30:48.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:30:47 smithi083 ceph-mon[77242]: from='client.46255 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:30:48.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:30:47 smithi083 ceph-mon[77242]: pgmap v1878: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:48.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:30:47 smithi083 ceph-mon[77242]: from='client.46261 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:30:48.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:30:47 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4130789317' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:30:50.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:30:49 smithi153 ceph-mon[70044]: pgmap v1879: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:50.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:30:49 smithi083 ceph-mon[79930]: pgmap v1879: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:50.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:30:49 smithi083 ceph-mon[77242]: pgmap v1879: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:50.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:30:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:30:50.343Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:30:51.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:30:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:30:51] "GET /metrics HTTP/1.1" 200 225180 "" "Prometheus/2.33.4" 2022-09-29T06:30:51.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:30:51 smithi083 ceph-mon[79930]: pgmap v1880: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:51.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:30:51 smithi083 ceph-mon[77242]: pgmap v1880: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:51.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:30:51 smithi153 ceph-mon[70044]: pgmap v1880: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:30:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:30:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:30:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:30:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:30:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:30:54.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:30:53 smithi153 ceph-mon[70044]: pgmap v1881: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:54.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:30:53 smithi083 ceph-mon[77242]: pgmap v1881: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:54.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:30:53 smithi083 ceph-mon[79930]: pgmap v1881: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:54.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:30:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:30:53.751Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:30:54.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:30:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:30:54.448Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:30:56.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:30:55 smithi153 ceph-mon[70044]: pgmap v1882: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:56.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:30:55 smithi083 ceph-mon[77242]: pgmap v1882: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:56.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:30:55 smithi083 ceph-mon[79930]: pgmap v1882: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:58.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:30:57 smithi153 ceph-mon[70044]: pgmap v1883: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:58.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:30:57 smithi083 ceph-mon[79930]: pgmap v1883: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:30:58.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:30:57 smithi083 ceph-mon[77242]: pgmap v1883: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:00.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:30:59 smithi153 ceph-mon[70044]: pgmap v1884: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:00.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:30:59 smithi083 ceph-mon[77242]: pgmap v1884: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:00.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:30:59 smithi083 ceph-mon[79930]: pgmap v1884: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:00.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:31:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:31:00.344Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:31:01.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:31:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:31:01] "GET /metrics HTTP/1.1" 200 225181 "" "Prometheus/2.33.4" 2022-09-29T06:31:01.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:31:01 smithi083 ceph-mon[77242]: pgmap v1885: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:01.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:31:01 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:31:01.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:31:01 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:31:01.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:31:01 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:31:01.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:31:01 smithi083 ceph-mon[79930]: pgmap v1885: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:01.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:31:01 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:31:01.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:31:01 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:31:01.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:31:01 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:31:01.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:31:01 smithi153 ceph-mon[70044]: pgmap v1885: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:01.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:31:01 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:31:01.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:31:01 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:31:01.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:31:01 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:31:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:31:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:31:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:31:03.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:31:03 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:31:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:31:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:31:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:31:03.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:31:03 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:31:03.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:31:03 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:31:04.020 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:31:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:31:03.752Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:31:04.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:31:04 smithi153 ceph-mon[70044]: pgmap v1886: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:04.450 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:31:04 smithi083 ceph-mon[79930]: pgmap v1886: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:04.450 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:31:04 smithi083 ceph-mon[77242]: pgmap v1886: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:04.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:31:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:31:04.448Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:31:06.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:31:06 smithi153 ceph-mon[70044]: pgmap v1887: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:06.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:31:06 smithi083 ceph-mon[79930]: pgmap v1887: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:06.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:31:06 smithi083 ceph-mon[77242]: pgmap v1887: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:08.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:31:08 smithi153 ceph-mon[70044]: pgmap v1888: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:08.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:31:08 smithi083 ceph-mon[79930]: pgmap v1888: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:08.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:31:08 smithi083 ceph-mon[77242]: pgmap v1888: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:10.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:31:10 smithi153 ceph-mon[70044]: pgmap v1889: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:10.346 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:31:10 smithi083 ceph-mon[79930]: pgmap v1889: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:10.346 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:31:10 smithi083 ceph-mon[77242]: pgmap v1889: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:10.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:31:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:31:10.344Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:31:11.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:31:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:31:11] "GET /metrics HTTP/1.1" 200 225178 "" "Prometheus/2.33.4" 2022-09-29T06:31:11.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:31:11 smithi083 ceph-mon[79930]: pgmap v1890: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:11.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:31:11 smithi083 ceph-mon[77242]: pgmap v1890: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:11.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:31:11 smithi153 ceph-mon[70044]: pgmap v1890: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:31:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:31:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:31:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:31:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:31:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:31:14.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:31:13 smithi153 ceph-mon[70044]: pgmap v1891: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:14.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:31:13 smithi083 ceph-mon[79930]: pgmap v1891: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:14.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:31:13 smithi083 ceph-mon[77242]: pgmap v1891: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:14.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:31:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:31:13.752Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:31:14.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:31:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:31:14.448Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:31:16.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:31:15 smithi153 ceph-mon[70044]: pgmap v1892: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:16.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:31:15 smithi083 ceph-mon[79930]: pgmap v1892: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:16.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:31:15 smithi083 ceph-mon[77242]: pgmap v1892: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:17.528 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:31:17.814 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:31:17.814 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (65m) 7m ago 65m 26.5M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:31:17.814 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (65m) 5m ago 65m 61.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:31:17.815 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (66m) 7m ago 66m 25.3M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:31:17.815 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (66m) 5m ago 66m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:31:17.815 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (62m) 5m ago 68m 439M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:31:17.815 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (62m) 7m ago 70m 560M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:31:17.815 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (61m) 7m ago 70m 275M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:31:17.816 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (61m) 5m ago 69m 132M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:31:17.816 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (61m) 7m ago 69m 166M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:31:17.816 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (65m) 7m ago 65m 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:31:17.816 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (65m) 5m ago 65m 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:31:17.817 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (60m) 7m ago 68m 209M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:31:17.817 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (59m) 7m ago 68m 201M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:31:17.817 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (59m) 7m ago 68m 163M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:31:17.817 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (59m) 7m ago 67m 191M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:31:17.817 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (58m) 5m ago 67m 176M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:31:17.818 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (57m) 5m ago 67m 207M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:31:17.818 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (57m) 5m ago 66m 184M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:31:17.818 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (56m) 5m ago 66m 209M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:31:17.818 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (62m) 5m ago 65m 103M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:31:18.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:31:17 smithi153 ceph-mon[70044]: pgmap v1893: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:18.168 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:31:18.169 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:31:18.169 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:31:18.169 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:31:18.169 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:31:18.170 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:31:18.170 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:31:18.170 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:31:18.170 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:31:18.171 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:31:18.171 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:31:18.171 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:31:18.171 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:31:18.171 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:31:18.172 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:31:18.172 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:31:18.172 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:31:18.172 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:31:18.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:31:17 smithi083 ceph-mon[79930]: pgmap v1893: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:18.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:31:17 smithi083 ceph-mon[77242]: pgmap v1893: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:19.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:31:18 smithi153 ceph-mon[70044]: from='client.36339 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:31:19.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:31:18 smithi153 ceph-mon[70044]: from='client.46279 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:31:19.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:31:18 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1706222567' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:31:19.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:31:18 smithi083 ceph-mon[79930]: from='client.36339 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:31:19.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:31:18 smithi083 ceph-mon[79930]: from='client.46279 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:31:19.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:31:18 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1706222567' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:31:19.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:31:18 smithi083 ceph-mon[77242]: from='client.36339 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:31:19.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:31:18 smithi083 ceph-mon[77242]: from='client.46279 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:31:19.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:31:18 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1706222567' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:31:20.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:31:19 smithi153 ceph-mon[70044]: pgmap v1894: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:20.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:31:19 smithi083 ceph-mon[79930]: pgmap v1894: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:20.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:31:19 smithi083 ceph-mon[77242]: pgmap v1894: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:20.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:31:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:31:20.346Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:31:21.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:31:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:31:21] "GET /metrics HTTP/1.1" 200 225178 "" "Prometheus/2.33.4" 2022-09-29T06:31:21.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:31:21 smithi083 ceph-mon[79930]: pgmap v1895: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:21.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:31:21 smithi083 ceph-mon[77242]: pgmap v1895: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:21.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:31:21 smithi153 ceph-mon[70044]: pgmap v1895: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:31:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:31:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:31:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:31:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:31:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:31:24.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:31:23 smithi153 ceph-mon[70044]: pgmap v1896: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:24.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:31:23 smithi083 ceph-mon[77242]: pgmap v1896: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:24.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:31:23 smithi083 ceph-mon[79930]: pgmap v1896: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:24.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:31:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:31:23.753Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:31:24.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:31:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:31:24.449Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:31:26.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:31:25 smithi153 ceph-mon[70044]: pgmap v1897: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:26.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:31:25 smithi083 ceph-mon[79930]: pgmap v1897: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:26.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:31:25 smithi083 ceph-mon[77242]: pgmap v1897: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:28.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:31:27 smithi153 ceph-mon[70044]: pgmap v1898: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:28.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:31:27 smithi083 ceph-mon[79930]: pgmap v1898: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:28.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:31:27 smithi083 ceph-mon[77242]: pgmap v1898: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:30.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:31:29 smithi153 ceph-mon[70044]: pgmap v1899: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:30.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:31:29 smithi083 ceph-mon[79930]: pgmap v1899: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:30.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:31:29 smithi083 ceph-mon[77242]: pgmap v1899: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:30.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:31:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:31:30.346Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:31:31.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:31:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:31:31] "GET /metrics HTTP/1.1" 200 225176 "" "Prometheus/2.33.4" 2022-09-29T06:31:31.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:31:31 smithi083 ceph-mon[79930]: pgmap v1900: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:31.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:31:31 smithi083 ceph-mon[77242]: pgmap v1900: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:31.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:31:31 smithi153 ceph-mon[70044]: pgmap v1900: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:31:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:31:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:31:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:31:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:31:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:31:34.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:31:33 smithi153 ceph-mon[70044]: pgmap v1901: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:34.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:31:33 smithi083 ceph-mon[77242]: pgmap v1901: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:34.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:31:33 smithi083 ceph-mon[79930]: pgmap v1901: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:34.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:31:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:31:33.754Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:31:34.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:31:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:31:34.450Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:31:36.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:31:35 smithi153 ceph-mon[70044]: pgmap v1902: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:36.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:31:35 smithi083 ceph-mon[77242]: pgmap v1902: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:36.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:31:35 smithi083 ceph-mon[79930]: pgmap v1902: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:38.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:31:37 smithi153 ceph-mon[70044]: pgmap v1903: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:38.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:31:37 smithi083 ceph-mon[79930]: pgmap v1903: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:38.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:31:37 smithi083 ceph-mon[77242]: pgmap v1903: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:31:39 smithi153 ceph-mon[70044]: pgmap v1904: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:31:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:31:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:31:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:31:40.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:31:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:31:40.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:31:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:31:40.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:31:39 smithi083 ceph-mon[79930]: pgmap v1904: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:31:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:31:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:31:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:31:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:31:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:31:40.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:31:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:31:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:31:39 smithi083 ceph-mon[77242]: pgmap v1904: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:31:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:31:40.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:31:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:31:40.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:31:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:31:40.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:31:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:31:40.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:31:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:31:40.347Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:31:41.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:31:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:31:41] "GET /metrics HTTP/1.1" 200 225177 "" "Prometheus/2.33.4" 2022-09-29T06:31:41.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:31:41 smithi083 ceph-mon[79930]: pgmap v1905: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:41.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:31:41 smithi083 ceph-mon[77242]: pgmap v1905: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:41.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:31:41 smithi153 ceph-mon[70044]: pgmap v1905: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:31:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:31:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:31:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:31:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:31:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:31:44.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:31:43 smithi153 ceph-mon[70044]: pgmap v1906: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:44.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:31:43 smithi083 ceph-mon[79930]: pgmap v1906: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:44.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:31:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:31:43.755Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:31:44.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:31:43 smithi083 ceph-mon[77242]: pgmap v1906: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:44.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:31:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:31:44.451Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:31:46.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:31:45 smithi153 ceph-mon[70044]: pgmap v1907: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:46.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:31:45 smithi083 ceph-mon[77242]: pgmap v1907: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:46.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:31:45 smithi083 ceph-mon[79930]: pgmap v1907: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:48.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:31:47 smithi153 ceph-mon[70044]: pgmap v1908: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:48.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:31:47 smithi083 ceph-mon[79930]: pgmap v1908: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:48.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:31:47 smithi083 ceph-mon[77242]: pgmap v1908: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:48.496 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:31:48.788 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:31:48.789 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (65m) 7m ago 65m 26.5M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:31:48.789 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (65m) 5m ago 65m 61.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:31:48.789 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (66m) 7m ago 66m 25.3M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:31:48.790 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (66m) 5m ago 66m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:31:48.790 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (62m) 5m ago 69m 439M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:31:48.790 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (63m) 7m ago 70m 560M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:31:48.790 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (62m) 7m ago 71m 275M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:31:48.791 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (61m) 5m ago 69m 132M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:31:48.791 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (61m) 7m ago 69m 166M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:31:48.791 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (66m) 7m ago 66m 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:31:48.791 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (66m) 5m ago 66m 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:31:48.792 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (60m) 7m ago 69m 209M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:31:48.792 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (60m) 7m ago 68m 201M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:31:48.792 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (60m) 7m ago 68m 163M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:31:48.792 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (59m) 7m ago 68m 191M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:31:48.792 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (59m) 5m ago 67m 176M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:31:48.793 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (58m) 5m ago 67m 207M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:31:48.793 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (57m) 5m ago 67m 184M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:31:48.793 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (57m) 5m ago 67m 209M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:31:48.793 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (62m) 5m ago 66m 103M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:31:49.145 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:31:49.146 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:31:49.146 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:31:49.146 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:31:49.146 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:31:49.147 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:31:49.147 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:31:49.147 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:31:49.147 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:31:49.148 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:31:49.148 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:31:49.148 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:31:49.148 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:31:49.148 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:31:49.149 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:31:49.149 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:31:49.149 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:31:49.149 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:31:50.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:31:49 smithi153 ceph-mon[70044]: from='client.46291 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:31:50.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:31:49 smithi153 ceph-mon[70044]: pgmap v1909: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:50.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:31:49 smithi153 ceph-mon[70044]: from='client.36363 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:31:50.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:31:49 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/5426253' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:31:50.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:31:49 smithi083 ceph-mon[79930]: from='client.46291 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:31:50.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:31:49 smithi083 ceph-mon[79930]: pgmap v1909: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:50.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:31:49 smithi083 ceph-mon[79930]: from='client.36363 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:31:50.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:31:49 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/5426253' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:31:50.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:31:49 smithi083 ceph-mon[77242]: from='client.46291 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:31:50.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:31:49 smithi083 ceph-mon[77242]: pgmap v1909: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:50.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:31:49 smithi083 ceph-mon[77242]: from='client.36363 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:31:50.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:31:49 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/5426253' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:31:50.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:31:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:31:50.347Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:31:51.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:31:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:31:51] "GET /metrics HTTP/1.1" 200 225177 "" "Prometheus/2.33.4" 2022-09-29T06:31:51.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:31:51 smithi083 ceph-mon[79930]: pgmap v1910: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:51.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:31:51 smithi083 ceph-mon[77242]: pgmap v1910: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:51.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:31:51 smithi153 ceph-mon[70044]: pgmap v1910: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:31:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:31:52.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:31:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:31:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:31:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:31:54.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:31:53 smithi153 ceph-mon[70044]: pgmap v1911: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:54.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:31:53 smithi083 ceph-mon[79930]: pgmap v1911: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:54.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:31:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:31:53.755Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:31:54.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:31:53 smithi083 ceph-mon[77242]: pgmap v1911: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:54.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:31:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:31:54.452Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:31:56.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:31:55 smithi153 ceph-mon[70044]: pgmap v1912: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:56.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:31:55 smithi083 ceph-mon[79930]: pgmap v1912: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:56.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:31:55 smithi083 ceph-mon[77242]: pgmap v1912: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:58.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:31:57 smithi153 ceph-mon[70044]: pgmap v1913: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:58.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:31:57 smithi083 ceph-mon[79930]: pgmap v1913: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:31:58.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:31:57 smithi083 ceph-mon[77242]: pgmap v1913: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:00.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:31:59 smithi153 ceph-mon[70044]: pgmap v1914: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:00.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:31:59 smithi083 ceph-mon[79930]: pgmap v1914: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:00.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:31:59 smithi083 ceph-mon[77242]: pgmap v1914: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:00.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:32:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:32:00.348Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:32:01.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:32:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:32:01] "GET /metrics HTTP/1.1" 200 225175 "" "Prometheus/2.33.4" 2022-09-29T06:32:01.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:32:01 smithi083 ceph-mon[79930]: pgmap v1915: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:01.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:32:01 smithi083 ceph-mon[77242]: pgmap v1915: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:01.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:32:01 smithi153 ceph-mon[70044]: pgmap v1915: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:02.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:32:02 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:32:02.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:32:02 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:32:02.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:32:02 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:32:02.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:32:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:32:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:32:02.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:32:02 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:32:02.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:32:02 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:32:02.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:32:02 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:32:02.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:32:02 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:32:02.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:32:02 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:32:02.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:32:02 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:32:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:32:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:32:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:32:03.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:32:03 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:32:03.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:32:03 smithi083 ceph-mon[79930]: pgmap v1916: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:03.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:32:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:32:03.756Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:32:03.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:32:03 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:32:03.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:32:03 smithi083 ceph-mon[77242]: pgmap v1916: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:04.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:32:03 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:32:04.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:32:03 smithi153 ceph-mon[70044]: pgmap v1916: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:04.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:32:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:32:04.453Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:32:06.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:32:05 smithi153 ceph-mon[70044]: pgmap v1917: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:06.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:32:05 smithi083 ceph-mon[79930]: pgmap v1917: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:06.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:32:05 smithi083 ceph-mon[77242]: pgmap v1917: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:08.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:32:07 smithi153 ceph-mon[70044]: pgmap v1918: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:08.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:32:07 smithi083 ceph-mon[79930]: pgmap v1918: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:08.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:32:07 smithi083 ceph-mon[77242]: pgmap v1918: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:10.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:32:09 smithi153 ceph-mon[70044]: pgmap v1919: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:10.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:32:09 smithi083 ceph-mon[79930]: pgmap v1919: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:10.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:32:09 smithi083 ceph-mon[77242]: pgmap v1919: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:10.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:32:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:32:10.348Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:32:11.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:32:11 smithi153 ceph-mon[70044]: pgmap v1920: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:32:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:32:11] "GET /metrics HTTP/1.1" 200 225178 "" "Prometheus/2.33.4" 2022-09-29T06:32:11.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:32:11 smithi083 ceph-mon[79930]: pgmap v1920: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:11.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:32:11 smithi083 ceph-mon[77242]: pgmap v1920: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:32:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:32:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:32:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:32:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:32:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:32:14.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:32:13 smithi153 ceph-mon[70044]: pgmap v1921: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:14.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:32:13 smithi083 ceph-mon[77242]: pgmap v1921: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:14.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:32:13 smithi083 ceph-mon[79930]: pgmap v1921: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:14.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:32:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:32:13.757Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:32:14.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:32:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:32:14.454Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:32:16.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:32:15 smithi153 ceph-mon[70044]: pgmap v1922: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:16.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:32:15 smithi083 ceph-mon[77242]: pgmap v1922: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:16.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:32:15 smithi083 ceph-mon[79930]: pgmap v1922: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:18.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:32:17 smithi153 ceph-mon[70044]: pgmap v1923: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:18.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:32:17 smithi083 ceph-mon[79930]: pgmap v1923: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:18.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:32:17 smithi083 ceph-mon[77242]: pgmap v1923: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:19.473 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:32:19.763 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:32:19.763 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (66m) 8m ago 66m 26.5M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:32:19.763 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (66m) 6m ago 66m 61.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:32:19.763 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (67m) 8m ago 67m 25.3M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:32:19.764 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (67m) 6m ago 67m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:32:19.764 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (63m) 6m ago 69m 439M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:32:19.764 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (63m) 8m ago 71m 560M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:32:19.764 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (62m) 8m ago 71m 275M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:32:19.765 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (62m) 6m ago 70m 132M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:32:19.765 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (62m) 8m ago 70m 166M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:32:19.765 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (66m) 8m ago 66m 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:32:19.765 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (66m) 6m ago 66m 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:32:19.766 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (61m) 8m ago 69m 209M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:32:19.766 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (60m) 8m ago 69m 201M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:32:19.766 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (60m) 8m ago 69m 163M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:32:19.766 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (60m) 8m ago 68m 191M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:32:19.766 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (59m) 6m ago 68m 176M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:32:19.767 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (58m) 6m ago 68m 207M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:32:19.767 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (58m) 6m ago 67m 184M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:32:19.767 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (57m) 6m ago 67m 209M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:32:19.767 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (63m) 6m ago 66m 103M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:32:20.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:32:19 smithi153 ceph-mon[70044]: pgmap v1924: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:20.122 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:32:20.123 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:32:20.123 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:32:20.123 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:32:20.124 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:32:20.124 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:32:20.124 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:32:20.124 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:32:20.125 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:32:20.125 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:32:20.125 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:32:20.125 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:32:20.126 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:32:20.126 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:32:20.126 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:32:20.127 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:32:20.127 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:32:20.127 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:32:20.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:32:19 smithi083 ceph-mon[79930]: pgmap v1924: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:20.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:32:19 smithi083 ceph-mon[77242]: pgmap v1924: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:20.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:32:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:32:20.349Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:32:21.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:32:20 smithi153 ceph-mon[70044]: from='client.36375 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:32:21.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:32:20 smithi153 ceph-mon[70044]: from='client.46315 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:32:21.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:32:20 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1277396448' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:32:21.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:32:20 smithi083 ceph-mon[79930]: from='client.36375 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:32:21.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:32:20 smithi083 ceph-mon[79930]: from='client.46315 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:32:21.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:32:20 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1277396448' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:32:21.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:32:20 smithi083 ceph-mon[77242]: from='client.36375 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:32:21.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:32:20 smithi083 ceph-mon[77242]: from='client.46315 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:32:21.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:32:20 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1277396448' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:32:21.800 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:32:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:32:21] "GET /metrics HTTP/1.1" 200 225178 "" "Prometheus/2.33.4" 2022-09-29T06:32:22.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:32:21 smithi153 ceph-mon[70044]: pgmap v1925: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:22.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:32:21 smithi083 ceph-mon[79930]: pgmap v1925: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:22.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:32:21 smithi083 ceph-mon[77242]: pgmap v1925: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:32:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:32:22.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:32:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:32:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:32:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:32:24.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:32:23 smithi153 ceph-mon[70044]: pgmap v1926: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:24.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:32:23 smithi083 ceph-mon[79930]: pgmap v1926: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:24.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:32:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:32:23.757Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:32:24.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:32:23 smithi083 ceph-mon[77242]: pgmap v1926: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:24.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:32:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:32:24.454Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:32:26.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:32:25 smithi083 ceph-mon[79930]: pgmap v1927: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:26.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:32:25 smithi083 ceph-mon[77242]: pgmap v1927: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:26.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:32:25 smithi153 ceph-mon[70044]: pgmap v1927: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:28.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:32:27 smithi083 ceph-mon[77242]: pgmap v1928: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:28.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:32:27 smithi083 ceph-mon[79930]: pgmap v1928: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:28.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:32:27 smithi153 ceph-mon[70044]: pgmap v1928: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:30.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:32:29 smithi083 ceph-mon[77242]: pgmap v1929: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:30.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:32:29 smithi083 ceph-mon[79930]: pgmap v1929: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:30.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:32:29 smithi153 ceph-mon[70044]: pgmap v1929: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:30.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:32:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:32:30.350Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:32:31.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:32:31 smithi153 ceph-mon[70044]: pgmap v1930: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:32:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:32:31] "GET /metrics HTTP/1.1" 200 225178 "" "Prometheus/2.33.4" 2022-09-29T06:32:31.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:32:31 smithi083 ceph-mon[79930]: pgmap v1930: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:31.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:32:31 smithi083 ceph-mon[77242]: pgmap v1930: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:32:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:32:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:32:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:32:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:32:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:32:34.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:32:33 smithi153 ceph-mon[70044]: pgmap v1931: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:34.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:32:33 smithi083 ceph-mon[79930]: pgmap v1931: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:34.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:32:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:32:33.757Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:32:34.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:32:33 smithi083 ceph-mon[77242]: pgmap v1931: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:34.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:32:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:32:34.455Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:32:36.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:32:35 smithi153 ceph-mon[70044]: pgmap v1932: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:36.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:32:35 smithi083 ceph-mon[79930]: pgmap v1932: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:36.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:32:35 smithi083 ceph-mon[77242]: pgmap v1932: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:38.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:32:37 smithi153 ceph-mon[70044]: pgmap v1933: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:38.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:32:37 smithi083 ceph-mon[79930]: pgmap v1933: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:38.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:32:37 smithi083 ceph-mon[77242]: pgmap v1933: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:32:39 smithi153 ceph-mon[70044]: pgmap v1934: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:32:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:32:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:32:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:32:40.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:32:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:32:40.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:32:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:32:40.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:32:39 smithi083 ceph-mon[79930]: pgmap v1934: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:32:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:32:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:32:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:32:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:32:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:32:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:32:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:32:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:32:39 smithi083 ceph-mon[77242]: pgmap v1934: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:32:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:32:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:32:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:32:40.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:32:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:32:40.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:32:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:32:40.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:32:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:32:40.350Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:32:41.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:32:41 smithi153 ceph-mon[70044]: pgmap v1935: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:41.967 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:32:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:32:41] "GET /metrics HTTP/1.1" 200 225179 "" "Prometheus/2.33.4" 2022-09-29T06:32:41.968 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:32:41 smithi083 ceph-mon[79930]: pgmap v1935: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:41.969 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:32:41 smithi083 ceph-mon[77242]: pgmap v1935: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:32:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:32:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:32:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:32:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:32:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:32:44.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:32:43 smithi153 ceph-mon[70044]: pgmap v1936: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:44.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:32:43 smithi083 ceph-mon[79930]: pgmap v1936: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:44.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:32:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:32:43.758Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:32:44.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:32:43 smithi083 ceph-mon[77242]: pgmap v1936: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:44.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:32:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:32:44.455Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:32:46.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:32:45 smithi153 ceph-mon[70044]: pgmap v1937: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:46.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:32:45 smithi083 ceph-mon[79930]: pgmap v1937: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:46.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:32:45 smithi083 ceph-mon[77242]: pgmap v1937: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:48.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:32:47 smithi083 ceph-mon[79930]: pgmap v1938: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:48.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:32:47 smithi083 ceph-mon[77242]: pgmap v1938: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:48.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:32:47 smithi153 ceph-mon[70044]: pgmap v1938: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:50.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:32:49 smithi083 ceph-mon[79930]: pgmap v1939: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:50.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:32:49 smithi083 ceph-mon[77242]: pgmap v1939: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:50.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:32:49 smithi153 ceph-mon[70044]: pgmap v1939: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:50.450 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:32:50.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:32:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:32:50.350Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:32:50.739 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:32:50.740 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (66m) 8m ago 67m 26.5M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:32:50.740 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (66m) 6m ago 66m 61.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:32:50.740 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (67m) 8m ago 67m 25.3M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:32:50.741 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (67m) 6m ago 67m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:32:50.741 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (63m) 6m ago 70m 439M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:32:50.741 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (64m) 8m ago 71m 560M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:32:50.741 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (63m) 8m ago 72m 275M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:32:50.742 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (62m) 6m ago 70m 132M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:32:50.742 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (62m) 8m ago 70m 166M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:32:50.742 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (67m) 8m ago 67m 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:32:50.742 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (67m) 6m ago 67m 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:32:50.743 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (62m) 8m ago 70m 209M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:32:50.743 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (61m) 8m ago 69m 201M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:32:50.743 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (61m) 8m ago 69m 163M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:32:50.743 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (60m) 8m ago 69m 191M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:32:50.744 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (60m) 6m ago 69m 176M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:32:50.744 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (59m) 6m ago 68m 207M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:32:50.744 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (58m) 6m ago 68m 184M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:32:50.744 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (58m) 6m ago 68m 209M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:32:50.745 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (63m) 6m ago 67m 103M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:32:51.094 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:32:51.094 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:32:51.095 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:32:51.095 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:32:51.095 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:32:51.096 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:32:51.096 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:32:51.096 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:32:51.096 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:32:51.096 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:32:51.097 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:32:51.097 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:32:51.097 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:32:51.097 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:32:51.098 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:32:51.098 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:32:51.098 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:32:51.098 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:32:51.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:32:51 smithi153 ceph-mon[70044]: from='client.46327 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:32:51.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:32:51 smithi153 ceph-mon[70044]: from='client.36399 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:32:51.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:32:51 smithi153 ceph-mon[70044]: pgmap v1940: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:51.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:32:51 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2471121249' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:32:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:32:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:32:51] "GET /metrics HTTP/1.1" 200 225179 "" "Prometheus/2.33.4" 2022-09-29T06:32:51.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:32:51 smithi083 ceph-mon[77242]: from='client.46327 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:32:51.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:32:51 smithi083 ceph-mon[77242]: from='client.36399 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:32:51.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:32:51 smithi083 ceph-mon[77242]: pgmap v1940: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:51.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:32:51 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2471121249' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:32:51.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:32:51 smithi083 ceph-mon[79930]: from='client.46327 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:32:51.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:32:51 smithi083 ceph-mon[79930]: from='client.36399 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:32:51.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:32:51 smithi083 ceph-mon[79930]: pgmap v1940: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:51.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:32:51 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2471121249' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:32:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:32:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:32:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:32:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:32:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:32:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:32:54.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:32:53 smithi153 ceph-mon[70044]: pgmap v1941: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:54.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:32:53 smithi083 ceph-mon[79930]: pgmap v1941: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:54.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:32:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:32:53.758Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:32:54.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:32:53 smithi083 ceph-mon[77242]: pgmap v1941: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:54.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:32:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:32:54.456Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:32:56.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:32:55 smithi153 ceph-mon[70044]: pgmap v1942: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:56.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:32:55 smithi083 ceph-mon[77242]: pgmap v1942: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:56.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:32:55 smithi083 ceph-mon[79930]: pgmap v1942: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:58.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:32:57 smithi153 ceph-mon[70044]: pgmap v1943: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:58.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:32:57 smithi083 ceph-mon[79930]: pgmap v1943: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:32:58.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:32:57 smithi083 ceph-mon[77242]: pgmap v1943: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:00.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:32:59 smithi083 ceph-mon[79930]: pgmap v1944: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:00.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:32:59 smithi083 ceph-mon[77242]: pgmap v1944: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:32:59 smithi153 ceph-mon[70044]: pgmap v1944: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:00.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:33:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:33:00.351Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:33:01.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:33:01 smithi153 ceph-mon[70044]: pgmap v1945: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:01.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:33:01 smithi083 ceph-mon[77242]: pgmap v1945: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:33:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:33:01] "GET /metrics HTTP/1.1" 200 225175 "" "Prometheus/2.33.4" 2022-09-29T06:33:01.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:33:01 smithi083 ceph-mon[79930]: pgmap v1945: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:33:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:33:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:33:03.057 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:33:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:33:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:33:03.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:33:02 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:33:03.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:33:02 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:33:03.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:33:02 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:33:03.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:33:02 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:33:03.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:33:02 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:33:03.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:33:02 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:33:03.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:33:02 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:33:03.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:33:02 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:33:03.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:33:02 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:33:04.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:33:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:33:03.758Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:33:04.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:33:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:33:04.456Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:33:04.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:33:04 smithi083 ceph-mon[77242]: pgmap v1946: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:04.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:33:04 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:33:04.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:33:04 smithi083 ceph-mon[79930]: pgmap v1946: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:04.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:33:04 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:33:04.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:33:04 smithi153 ceph-mon[70044]: pgmap v1946: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:04.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:33:04 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:33:05.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:33:05 smithi083 ceph-mon[79930]: pgmap v1947: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:05.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:33:05 smithi083 ceph-mon[77242]: pgmap v1947: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:05.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:33:05 smithi153 ceph-mon[70044]: pgmap v1947: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:08.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:33:07 smithi153 ceph-mon[70044]: pgmap v1948: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:08.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:33:07 smithi083 ceph-mon[77242]: pgmap v1948: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:08.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:33:07 smithi083 ceph-mon[79930]: pgmap v1948: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:10.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:33:09 smithi083 ceph-mon[79930]: pgmap v1949: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:10.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:33:09 smithi083 ceph-mon[77242]: pgmap v1949: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:10.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:33:09 smithi153 ceph-mon[70044]: pgmap v1949: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:10.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:33:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:33:10.353Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:33:11.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:33:11 smithi153 ceph-mon[70044]: pgmap v1950: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:33:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:33:11] "GET /metrics HTTP/1.1" 200 225177 "" "Prometheus/2.33.4" 2022-09-29T06:33:11.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:33:11 smithi083 ceph-mon[79930]: pgmap v1950: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:11.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:33:11 smithi083 ceph-mon[77242]: pgmap v1950: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:33:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:33:12.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:33:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:33:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:33:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:33:14.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:33:13 smithi153 ceph-mon[70044]: pgmap v1951: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:14.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:33:13 smithi083 ceph-mon[79930]: pgmap v1951: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:14.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:33:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:33:13.759Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:33:14.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:33:13 smithi083 ceph-mon[77242]: pgmap v1951: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:14.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:33:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:33:14.457Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:33:16.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:33:15 smithi083 ceph-mon[79930]: pgmap v1952: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:16.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:33:15 smithi083 ceph-mon[77242]: pgmap v1952: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:16.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:33:15 smithi153 ceph-mon[70044]: pgmap v1952: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:18.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:33:17 smithi083 ceph-mon[79930]: pgmap v1953: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:18.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:33:17 smithi083 ceph-mon[77242]: pgmap v1953: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:18.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:33:17 smithi153 ceph-mon[70044]: pgmap v1953: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:20.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:33:19 smithi083 ceph-mon[79930]: pgmap v1954: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:20.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:33:19 smithi083 ceph-mon[77242]: pgmap v1954: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:33:19 smithi153 ceph-mon[70044]: pgmap v1954: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:20.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:33:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:33:20.353Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:33:21.419 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:33:21.709 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:33:21.709 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (67m) 9m ago 67m 26.5M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:33:21.709 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (67m) 7m ago 67m 61.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:33:21.709 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (68m) 9m ago 68m 25.3M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:33:21.710 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (68m) 7m ago 68m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:33:21.710 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (64m) 7m ago 70m 439M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:33:21.710 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (64m) 9m ago 72m 560M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:33:21.711 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (63m) 9m ago 72m 275M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:33:21.711 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (63m) 7m ago 71m 132M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:33:21.711 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (63m) 9m ago 71m 166M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:33:21.711 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (67m) 9m ago 67m 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:33:21.712 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (67m) 7m ago 67m 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:33:21.712 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (62m) 9m ago 70m 209M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:33:21.712 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (61m) 9m ago 70m 201M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:33:21.712 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (61m) 9m ago 70m 163M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:33:21.712 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (61m) 9m ago 69m 191M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:33:21.713 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (60m) 7m ago 69m 176M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:33:21.713 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (60m) 7m ago 69m 207M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:33:21.713 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (59m) 7m ago 68m 184M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:33:21.713 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (58m) 7m ago 68m 209M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:33:21.714 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (64m) 7m ago 67m 103M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:33:21.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:33:21 smithi153 ceph-mon[70044]: pgmap v1955: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:33:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:33:21] "GET /metrics HTTP/1.1" 200 225177 "" "Prometheus/2.33.4" 2022-09-29T06:33:21.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:33:21 smithi083 ceph-mon[79930]: pgmap v1955: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:21.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:33:21 smithi083 ceph-mon[77242]: pgmap v1955: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:22.069 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:33:22.069 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:33:22.070 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:33:22.070 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:33:22.070 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:33:22.071 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:33:22.071 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:33:22.071 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:33:22.071 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:33:22.071 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:33:22.072 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:33:22.072 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:33:22.072 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:33:22.072 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:33:22.073 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:33:22.073 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:33:22.073 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:33:22.073 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:33:22.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:33:22 smithi153 ceph-mon[70044]: from='client.46345 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:33:22.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:33:22 smithi153 ceph-mon[70044]: from='client.36417 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:33:22.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:33:22 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/247595141' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:33:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:33:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:33:22.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:33:22.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:33:22 smithi083 ceph-mon[79930]: from='client.46345 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:33:22.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:33:22 smithi083 ceph-mon[79930]: from='client.36417 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:33:22.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:33:22 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/247595141' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:33:22.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:33:22 smithi083 ceph-mon[77242]: from='client.46345 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:33:22.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:33:22 smithi083 ceph-mon[77242]: from='client.36417 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:33:22.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:33:22 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/247595141' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:33:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:33:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:33:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:33:23.761 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:33:23 smithi083 ceph-mon[79930]: pgmap v1956: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:23.762 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:33:23 smithi083 ceph-mon[77242]: pgmap v1956: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:23.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:33:23 smithi153 ceph-mon[70044]: pgmap v1956: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:24.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:33:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:33:23.760Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:33:24.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:33:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:33:24.458Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:33:26.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:33:25 smithi083 ceph-mon[79930]: pgmap v1957: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:26.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:33:25 smithi083 ceph-mon[77242]: pgmap v1957: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:26.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:33:25 smithi153 ceph-mon[70044]: pgmap v1957: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:28.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:33:27 smithi083 ceph-mon[79930]: pgmap v1958: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:28.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:33:27 smithi083 ceph-mon[77242]: pgmap v1958: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:28.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:33:27 smithi153 ceph-mon[70044]: pgmap v1958: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:30.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:33:29 smithi083 ceph-mon[79930]: pgmap v1959: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:30.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:33:29 smithi083 ceph-mon[77242]: pgmap v1959: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:30.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:33:29 smithi153 ceph-mon[70044]: pgmap v1959: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:30.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:33:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:33:30.354Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:33:31.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:33:31 smithi153 ceph-mon[70044]: pgmap v1960: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:31.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:33:31 smithi083 ceph-mon[79930]: pgmap v1960: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:31.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:33:31 smithi083 ceph-mon[77242]: pgmap v1960: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:33:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:33:31] "GET /metrics HTTP/1.1" 200 225181 "" "Prometheus/2.33.4" 2022-09-29T06:33:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:33:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:33:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:33:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:33:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:33:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:33:34.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:33:33 smithi083 ceph-mon[79930]: pgmap v1961: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:34.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:33:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:33:33.760Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:33:34.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:33:33 smithi083 ceph-mon[77242]: pgmap v1961: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:34.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:33:33 smithi153 ceph-mon[70044]: pgmap v1961: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:34.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:33:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:33:34.458Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:33:36.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:33:35 smithi083 ceph-mon[79930]: pgmap v1962: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:36.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:33:35 smithi083 ceph-mon[77242]: pgmap v1962: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:36.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:33:35 smithi153 ceph-mon[70044]: pgmap v1962: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:38.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:33:37 smithi083 ceph-mon[77242]: pgmap v1963: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:38.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:33:37 smithi083 ceph-mon[79930]: pgmap v1963: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:38.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:33:37 smithi153 ceph-mon[70044]: pgmap v1963: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:40.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:33:39 smithi083 ceph-mon[79930]: pgmap v1964: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:33:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:33:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:33:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:33:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:33:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:33:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:33:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:33:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:33:39 smithi083 ceph-mon[77242]: pgmap v1964: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:33:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:33:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:33:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:33:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:33:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:33:40.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:33:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:33:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:33:39 smithi153 ceph-mon[70044]: pgmap v1964: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:33:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:33:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:33:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:33:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:33:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:33:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:33:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:33:40.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:33:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:33:40.355Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:33:41.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:33:41 smithi153 ceph-mon[70044]: pgmap v1965: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:41.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:33:41 smithi083 ceph-mon[77242]: pgmap v1965: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:41.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:33:41 smithi083 ceph-mon[79930]: pgmap v1965: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:41.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:33:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:33:41] "GET /metrics HTTP/1.1" 200 225178 "" "Prometheus/2.33.4" 2022-09-29T06:33:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:33:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:33:42.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:33:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:33:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:33:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:33:44.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:33:43 smithi083 ceph-mon[79930]: pgmap v1966: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:44.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:33:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:33:43.761Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:33:44.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:33:43 smithi083 ceph-mon[77242]: pgmap v1966: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:44.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:33:43 smithi153 ceph-mon[70044]: pgmap v1966: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:44.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:33:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:33:44.459Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:33:46.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:33:45 smithi083 ceph-mon[79930]: pgmap v1967: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:46.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:33:45 smithi083 ceph-mon[77242]: pgmap v1967: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:46.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:33:45 smithi153 ceph-mon[70044]: pgmap v1967: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:48.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:33:47 smithi083 ceph-mon[79930]: pgmap v1968: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:48.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:33:47 smithi083 ceph-mon[77242]: pgmap v1968: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:48.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:33:47 smithi153 ceph-mon[70044]: pgmap v1968: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:50.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:33:49 smithi083 ceph-mon[79930]: pgmap v1969: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:50.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:33:49 smithi083 ceph-mon[77242]: pgmap v1969: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:50.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:33:49 smithi153 ceph-mon[70044]: pgmap v1969: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:50.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:33:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:33:50.355Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:33:51.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:33:51 smithi153 ceph-mon[70044]: pgmap v1970: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:33:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:33:51] "GET /metrics HTTP/1.1" 200 225178 "" "Prometheus/2.33.4" 2022-09-29T06:33:51.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:33:51 smithi083 ceph-mon[79930]: pgmap v1970: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:51.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:33:51 smithi083 ceph-mon[77242]: pgmap v1970: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:52.395 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:33:52.683 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:33:52.684 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (67m) 10m ago 68m 26.5M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:33:52.684 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (67m) 7m ago 67m 61.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:33:52.684 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (68m) 10m ago 68m 25.3M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:33:52.684 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (68m) 7m ago 68m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:33:52.685 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (64m) 7m ago 71m 439M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:33:52.685 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (65m) 10m ago 73m 560M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:33:52.685 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (64m) 10m ago 73m 275M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:33:52.686 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (63m) 7m ago 71m 132M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:33:52.686 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (63m) 10m ago 71m 166M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:33:52.686 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (68m) 10m ago 68m 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:33:52.686 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (68m) 7m ago 68m 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:33:52.687 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (63m) 10m ago 71m 209M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:33:52.687 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (62m) 10m ago 70m 201M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:33:52.688 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (62m) 10m ago 70m 163M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:33:52.688 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (61m) 10m ago 70m 191M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:33:52.688 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (61m) 7m ago 70m 176M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:33:52.689 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (60m) 7m ago 69m 207M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:33:52.689 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (60m) 7m ago 69m 184M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:33:52.689 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (59m) 7m ago 69m 209M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:33:52.689 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (64m) 7m ago 68m 103M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:33:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:33:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:33:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:33:53.039 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:33:53.040 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:33:53.040 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:33:53.040 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:33:53.041 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:33:53.041 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:33:53.041 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:33:53.041 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:33:53.042 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:33:53.042 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:33:53.042 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:33:53.043 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:33:53.043 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:33:53.043 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:33:53.043 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:33:53.044 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:33:53.044 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:33:53.044 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:33:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:33:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:33:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:33:54.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:33:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:33:53.761Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:33:54.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:33:53 smithi083 ceph-mon[77242]: from='client.36429 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:33:54.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:33:53 smithi083 ceph-mon[77242]: from='client.36435 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:33:54.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:33:53 smithi083 ceph-mon[77242]: pgmap v1971: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:54.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:33:53 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4246731776' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:33:54.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:33:53 smithi083 ceph-mon[79930]: from='client.36429 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:33:54.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:33:53 smithi083 ceph-mon[79930]: from='client.36435 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:33:54.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:33:53 smithi083 ceph-mon[79930]: pgmap v1971: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:54.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:33:53 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4246731776' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:33:54.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:33:53 smithi153 ceph-mon[70044]: from='client.36429 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:33:54.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:33:53 smithi153 ceph-mon[70044]: from='client.36435 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:33:54.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:33:53 smithi153 ceph-mon[70044]: pgmap v1971: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:54.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:33:53 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4246731776' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:33:54.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:33:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:33:54.459Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:33:56.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:33:55 smithi083 ceph-mon[79930]: pgmap v1972: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:56.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:33:55 smithi083 ceph-mon[77242]: pgmap v1972: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:56.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:33:55 smithi153 ceph-mon[70044]: pgmap v1972: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:58.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:33:57 smithi083 ceph-mon[79930]: pgmap v1973: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:58.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:33:57 smithi083 ceph-mon[77242]: pgmap v1973: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:33:58.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:33:57 smithi153 ceph-mon[70044]: pgmap v1973: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:00.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:33:59 smithi083 ceph-mon[77242]: pgmap v1974: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:00.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:33:59 smithi083 ceph-mon[79930]: pgmap v1974: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:33:59 smithi153 ceph-mon[70044]: pgmap v1974: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:00.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:34:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:34:00.356Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:34:01.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:34:01 smithi153 ceph-mon[70044]: pgmap v1975: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:34:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:34:01] "GET /metrics HTTP/1.1" 200 225169 "" "Prometheus/2.33.4" 2022-09-29T06:34:01.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:34:01 smithi083 ceph-mon[79930]: pgmap v1975: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:01.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:34:01 smithi083 ceph-mon[77242]: pgmap v1975: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:34:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:34:02.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:34:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:34:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:34:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:34:04.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:34:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:34:03.761Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:34:04.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:34:03 smithi083 ceph-mon[77242]: pgmap v1976: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:04.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:34:03 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:34:04.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:34:03 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:34:04.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:34:03 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:34:04.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:34:03 smithi083 ceph-mon[79930]: pgmap v1976: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:04.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:34:03 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:34:04.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:34:03 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:34:04.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:34:03 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:34:04.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:34:03 smithi153 ceph-mon[70044]: pgmap v1976: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:04.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:34:03 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:34:04.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:34:03 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:34:04.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:34:03 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:34:04.872 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:34:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:34:04.460Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:34:05.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:34:04 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:34:05.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:34:04 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:34:05.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:34:04 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:34:05.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:34:04 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:34:05.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:34:04 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:34:05.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:34:04 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:34:05.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:34:04 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:34:05.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:34:04 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:34:05.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:34:04 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:34:05.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:34:04 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:34:05.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:34:04 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:34:05.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:34:04 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:34:06.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:34:05 smithi083 ceph-mon[77242]: pgmap v1977: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:06.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:34:05 smithi083 ceph-mon[79930]: pgmap v1977: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:06.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:34:05 smithi153 ceph-mon[70044]: pgmap v1977: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:08.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:34:07 smithi083 ceph-mon[77242]: pgmap v1978: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:08.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:34:07 smithi083 ceph-mon[79930]: pgmap v1978: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:08.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:34:07 smithi153 ceph-mon[70044]: pgmap v1978: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:09.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:34:09 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:34:09.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:34:09 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:34:09.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:34:09 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:34:09.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:34:09 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:34:09.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:34:09 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:34:09.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:34:09 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:34:09.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:34:09 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:34:09.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:34:09 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:34:09.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:34:09 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:34:09.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:34:09 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:34:09.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:34:09 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:34:09.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:34:09 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:34:10.358 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:34:10 smithi083 ceph-mon[79930]: pgmap v1979: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:10.359 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:34:10 smithi083 ceph-mon[77242]: pgmap v1979: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:34:10 smithi153 ceph-mon[70044]: pgmap v1979: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:10.648 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:34:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:34:10.357Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:34:11.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:34:11 smithi083 ceph-mon[79930]: pgmap v1980: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:11.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:34:11 smithi083 ceph-mon[77242]: pgmap v1980: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:11.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:34:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:34:11] "GET /metrics HTTP/1.1" 200 225179 "" "Prometheus/2.33.4" 2022-09-29T06:34:11.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:34:11 smithi153 ceph-mon[70044]: pgmap v1980: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:12.803 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:34:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:34:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:34:13.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:34:12 smithi153 ceph-mon[70044]: Detected new or changed devices on smithi083 2022-09-29T06:34:13.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:34:12 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:34:13.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:34:12 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:34:13.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:34:12 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:34:13.059 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:34:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:34:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:34:13.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:34:12 smithi083 ceph-mon[79930]: Detected new or changed devices on smithi083 2022-09-29T06:34:13.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:34:12 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:34:13.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:34:12 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:34:13.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:34:12 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:34:13.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:34:12 smithi083 ceph-mon[77242]: Detected new or changed devices on smithi083 2022-09-29T06:34:13.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:34:12 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:34:13.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:34:12 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:34:13.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:34:12 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:34:14.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:34:13 smithi083 ceph-mon[77242]: pgmap v1981: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:14.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:34:13 smithi083 ceph-mon[79930]: pgmap v1981: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:14.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:34:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:34:13.762Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:34:14.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:34:13 smithi153 ceph-mon[70044]: pgmap v1981: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:14.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:34:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:34:14.461Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:34:16.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:34:15 smithi083 ceph-mon[77242]: pgmap v1982: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:16.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:34:15 smithi083 ceph-mon[79930]: pgmap v1982: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:16.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:34:15 smithi153 ceph-mon[70044]: pgmap v1982: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:18.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:34:17 smithi083 ceph-mon[79930]: pgmap v1983: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:18.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:34:17 smithi083 ceph-mon[77242]: pgmap v1983: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:18.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:34:17 smithi153 ceph-mon[70044]: pgmap v1983: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:20.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:34:19 smithi083 ceph-mon[79930]: pgmap v1984: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:20.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:34:19 smithi083 ceph-mon[77242]: pgmap v1984: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:34:19 smithi153 ceph-mon[70044]: pgmap v1984: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:20.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:34:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:34:20.358Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:34:21.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:34:21 smithi153 ceph-mon[70044]: pgmap v1985: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:34:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:34:21] "GET /metrics HTTP/1.1" 200 225179 "" "Prometheus/2.33.4" 2022-09-29T06:34:21.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:34:21 smithi083 ceph-mon[79930]: pgmap v1985: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:21.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:34:21 smithi083 ceph-mon[77242]: pgmap v1985: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:34:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:34:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:34:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:34:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:34:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:34:23.371 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:34:23.670 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:34:23.670 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (68m) 15s ago 68m 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:34:23.670 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (68m) 8m ago 68m 61.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:34:23.671 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (69m) 15s ago 69m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:34:23.671 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (69m) 8m ago 69m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:34:23.671 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (65m) 8m ago 71m 439M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:34:23.671 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (65m) 15s ago 73m 570M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:34:23.672 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (64m) 15s ago 73m 310M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:34:23.672 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (64m) 8m ago 72m 132M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:34:23.672 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (64m) 15s ago 72m 186M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:34:23.672 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (68m) 15s ago 68m 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:34:23.673 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (68m) 8m ago 68m 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:34:23.673 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (63m) 15s ago 71m 234M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:34:23.673 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (63m) 15s ago 71m 226M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:34:23.673 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (62m) 15s ago 71m 183M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:34:23.674 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (62m) 15s ago 70m 216M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:34:23.674 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (61m) 8m ago 70m 176M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:34:23.674 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (61m) 8m ago 70m 207M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:34:23.674 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (60m) 8m ago 69m 184M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:34:23.675 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (59m) 8m ago 69m 209M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:34:23.675 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (65m) 8m ago 68m 103M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:34:24.033 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:34:24.034 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:34:24.034 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:34:24.034 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:34:24.034 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:34:24.034 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:34:24.035 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:34:24.035 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:34:24.035 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:34:24.035 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:34:24.036 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:34:24.036 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:34:24.036 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:34:24.036 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:34:24.037 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:34:24.037 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:34:24.037 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:34:24.037 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:34:24.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:34:23 smithi083 ceph-mon[77242]: pgmap v1986: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:24.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:34:23 smithi083 ceph-mon[79930]: pgmap v1986: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:24.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:34:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:34:23.762Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:34:24.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:34:23 smithi153 ceph-mon[70044]: pgmap v1986: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:24.833 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:34:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:34:24.460Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:34:25.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:34:24 smithi083 ceph-mon[79930]: from='client.46381 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:34:25.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:34:24 smithi083 ceph-mon[79930]: from='client.36453 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:34:25.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:34:24 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/739386194' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:34:25.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:34:24 smithi083 ceph-mon[77242]: from='client.46381 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:34:25.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:34:24 smithi083 ceph-mon[77242]: from='client.36453 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:34:25.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:34:24 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/739386194' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:34:25.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:34:24 smithi153 ceph-mon[70044]: from='client.46381 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:34:25.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:34:24 smithi153 ceph-mon[70044]: from='client.36453 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:34:25.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:34:24 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/739386194' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:34:26.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:34:25 smithi083 ceph-mon[79930]: pgmap v1987: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:26.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:34:25 smithi083 ceph-mon[77242]: pgmap v1987: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:26.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:34:25 smithi153 ceph-mon[70044]: pgmap v1987: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:28.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:34:27 smithi083 ceph-mon[79930]: pgmap v1988: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:28.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:34:27 smithi083 ceph-mon[77242]: pgmap v1988: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:28.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:34:27 smithi153 ceph-mon[70044]: pgmap v1988: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:30.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:34:29 smithi083 ceph-mon[79930]: pgmap v1989: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:30.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:34:29 smithi083 ceph-mon[77242]: pgmap v1989: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:30.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:34:29 smithi153 ceph-mon[70044]: pgmap v1989: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:30.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:34:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:34:30.359Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:34:31.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:34:31 smithi153 ceph-mon[70044]: pgmap v1990: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:31.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:34:31 smithi083 ceph-mon[79930]: pgmap v1990: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:31.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:34:31 smithi083 ceph-mon[77242]: pgmap v1990: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:31.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:34:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:34:31] "GET /metrics HTTP/1.1" 200 225176 "" "Prometheus/2.33.4" 2022-09-29T06:34:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:34:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:34:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:34:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:34:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:34:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:34:34.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:34:33 smithi083 ceph-mon[77242]: pgmap v1991: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:34.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:34:33 smithi083 ceph-mon[79930]: pgmap v1991: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:34.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:34:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:34:33.764Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:34:34.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:34:33 smithi153 ceph-mon[70044]: pgmap v1991: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:34.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:34:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:34:34.462Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:34:36.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:34:35 smithi083 ceph-mon[79930]: pgmap v1992: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:36.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:34:35 smithi083 ceph-mon[77242]: pgmap v1992: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:36.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:34:35 smithi153 ceph-mon[70044]: pgmap v1992: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:38.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:34:37 smithi083 ceph-mon[79930]: pgmap v1993: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:38.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:34:37 smithi083 ceph-mon[77242]: pgmap v1993: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:38.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:34:37 smithi153 ceph-mon[70044]: pgmap v1993: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:40.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:34:39 smithi083 ceph-mon[79930]: pgmap v1994: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:34:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:34:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:34:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:34:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:34:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:34:40.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:34:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:34:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:34:39 smithi083 ceph-mon[77242]: pgmap v1994: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:34:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:34:40.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:34:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:34:40.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:34:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:34:40.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:34:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:34:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:34:39 smithi153 ceph-mon[70044]: pgmap v1994: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:34:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:34:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:34:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:34:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:34:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:34:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:34:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:34:40.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:34:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:34:40.360Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:34:41.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:34:41 smithi153 ceph-mon[70044]: pgmap v1995: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:41.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:34:41 smithi083 ceph-mon[79930]: pgmap v1995: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:41.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:34:41 smithi083 ceph-mon[77242]: pgmap v1995: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:41.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:34:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:34:41] "GET /metrics HTTP/1.1" 200 225180 "" "Prometheus/2.33.4" 2022-09-29T06:34:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:34:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:34:42.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:34:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:34:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:34:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:34:44.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:34:43 smithi083 ceph-mon[77242]: pgmap v1996: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:44.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:34:43 smithi083 ceph-mon[79930]: pgmap v1996: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:44.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:34:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:34:43.764Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:34:44.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:34:43 smithi153 ceph-mon[70044]: pgmap v1996: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:44.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:34:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:34:44.462Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:34:46.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:34:45 smithi083 ceph-mon[79930]: pgmap v1997: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:46.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:34:45 smithi083 ceph-mon[77242]: pgmap v1997: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:46.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:34:45 smithi153 ceph-mon[70044]: pgmap v1997: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:48.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:34:47 smithi083 ceph-mon[79930]: pgmap v1998: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:48.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:34:47 smithi083 ceph-mon[77242]: pgmap v1998: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:48.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:34:47 smithi153 ceph-mon[70044]: pgmap v1998: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:50.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:34:49 smithi083 ceph-mon[79930]: pgmap v1999: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:50.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:34:49 smithi083 ceph-mon[77242]: pgmap v1999: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:50.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:34:49 smithi153 ceph-mon[70044]: pgmap v1999: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:50.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:34:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:34:50.361Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:34:51.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:34:51 smithi153 ceph-mon[70044]: pgmap v2000: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:51.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:34:51 smithi083 ceph-mon[79930]: pgmap v2000: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:51.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:34:51 smithi083 ceph-mon[77242]: pgmap v2000: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:34:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:34:51] "GET /metrics HTTP/1.1" 200 225180 "" "Prometheus/2.33.4" 2022-09-29T06:34:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:34:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:34:52.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:34:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:34:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:34:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:34:54.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:34:53 smithi083 ceph-mon[79930]: pgmap v2001: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:54.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:34:53 smithi083 ceph-mon[77242]: pgmap v2001: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:54.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:34:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:34:53.764Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:34:54.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:34:53 smithi153 ceph-mon[70044]: pgmap v2001: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:54.355 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:34:54.642 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:34:54.643 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (68m) 46s ago 69m 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:34:54.643 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (68m) 8m ago 68m 61.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:34:54.643 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (69m) 46s ago 69m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:34:54.643 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (69m) 8m ago 69m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:34:54.644 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (65m) 8m ago 72m 439M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:34:54.644 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (66m) 46s ago 74m 570M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:34:54.644 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (65m) 46s ago 74m 310M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:34:54.644 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (64m) 8m ago 72m 132M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:34:54.645 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (65m) 46s ago 72m 186M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:34:54.645 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (69m) 46s ago 69m 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:34:54.645 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (69m) 8m ago 69m 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:34:54.645 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (64m) 46s ago 72m 234M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:34:54.646 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (63m) 46s ago 71m 226M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:34:54.646 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (63m) 46s ago 71m 183M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:34:54.646 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (62m) 46s ago 71m 216M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:34:54.646 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (62m) 8m ago 71m 176M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:34:54.647 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (61m) 8m ago 70m 207M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:34:54.647 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (61m) 8m ago 70m 184M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:34:54.647 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (60m) 8m ago 70m 209M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:34:54.647 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (65m) 8m ago 69m 103M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:34:54.841 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:34:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:34:54.463Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:34:54.842 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:34:54 smithi083 ceph-mon[77242]: from='client.46399 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:34:55.001 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:34:55.001 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:34:55.001 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:34:55.002 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:34:55.002 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:34:55.002 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:34:55.002 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:34:55.003 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:34:55.003 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:34:55.003 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:34:55.003 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:34:55.004 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:34:55.004 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:34:55.004 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:34:55.005 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:34:55.005 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:34:55.005 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:34:55.005 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:34:55.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:34:54 smithi083 ceph-mon[79930]: from='client.46399 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:34:55.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:34:54 smithi153 ceph-mon[70044]: from='client.46399 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:34:56.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:34:55 smithi083 ceph-mon[79930]: from='client.36471 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:34:56.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:34:55 smithi083 ceph-mon[79930]: pgmap v2002: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:56.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:34:55 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2218903539' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:34:56.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:34:55 smithi083 ceph-mon[77242]: from='client.36471 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:34:56.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:34:55 smithi083 ceph-mon[77242]: pgmap v2002: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:56.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:34:55 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2218903539' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:34:56.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:34:55 smithi153 ceph-mon[70044]: from='client.36471 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:34:56.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:34:55 smithi153 ceph-mon[70044]: pgmap v2002: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:56.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:34:55 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2218903539' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:34:58.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:34:57 smithi083 ceph-mon[79930]: pgmap v2003: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:58.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:34:57 smithi083 ceph-mon[77242]: pgmap v2003: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:34:58.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:34:57 smithi153 ceph-mon[70044]: pgmap v2003: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:00.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:34:59 smithi083 ceph-mon[79930]: pgmap v2004: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:00.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:34:59 smithi083 ceph-mon[77242]: pgmap v2004: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:34:59 smithi153 ceph-mon[70044]: pgmap v2004: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:00.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:35:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:35:00.361Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:35:01.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:35:01 smithi153 ceph-mon[70044]: pgmap v2005: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:35:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:35:01] "GET /metrics HTTP/1.1" 200 225177 "" "Prometheus/2.33.4" 2022-09-29T06:35:01.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:35:01 smithi083 ceph-mon[79930]: pgmap v2005: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:01.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:35:01 smithi083 ceph-mon[77242]: pgmap v2005: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:35:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:35:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:35:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:35:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:35:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:35:04.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:35:03 smithi083 ceph-mon[79930]: pgmap v2006: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:04.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:35:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:35:03.765Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:35:04.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:35:03 smithi083 ceph-mon[77242]: pgmap v2006: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:04.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:35:03 smithi153 ceph-mon[70044]: pgmap v2006: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:04.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:35:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:35:04.464Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:35:06.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:35:05 smithi083 ceph-mon[79930]: pgmap v2007: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:06.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:35:05 smithi083 ceph-mon[77242]: pgmap v2007: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:06.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:35:05 smithi153 ceph-mon[70044]: pgmap v2007: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:08.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:35:07 smithi083 ceph-mon[79930]: pgmap v2008: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:08.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:35:07 smithi083 ceph-mon[77242]: pgmap v2008: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:08.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:35:07 smithi153 ceph-mon[70044]: pgmap v2008: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:10.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:35:09 smithi083 ceph-mon[79930]: pgmap v2009: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:10.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:35:09 smithi083 ceph-mon[77242]: pgmap v2009: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:10.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:35:09 smithi153 ceph-mon[70044]: pgmap v2009: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:10.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:35:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:35:10.362Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:35:11.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:35:11 smithi153 ceph-mon[70044]: pgmap v2010: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:35:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:35:11] "GET /metrics HTTP/1.1" 200 225177 "" "Prometheus/2.33.4" 2022-09-29T06:35:11.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:35:11 smithi083 ceph-mon[79930]: pgmap v2010: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:11.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:35:11 smithi083 ceph-mon[77242]: pgmap v2010: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:12.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:35:12 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:35:12.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:35:12 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:35:12.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:35:12 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:35:12.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:35:12 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:35:12.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:35:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:35:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:35:12.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:35:12 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:35:12.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:35:12 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:35:12.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:35:12 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:35:12.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:35:12 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:35:12.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:35:12 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:35:12.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:35:12 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:35:12.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:35:12 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:35:12.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:35:12 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:35:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:35:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:35:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:35:13.767 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:35:13 smithi083 ceph-mon[79930]: pgmap v2011: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:13.768 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:35:13 smithi083 ceph-mon[77242]: pgmap v2011: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:13.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:35:13 smithi153 ceph-mon[70044]: pgmap v2011: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:14.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:35:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:35:13.765Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:35:14.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:35:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:35:14.464Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:35:16.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:35:15 smithi083 ceph-mon[77242]: pgmap v2012: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:16.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:35:15 smithi083 ceph-mon[79930]: pgmap v2012: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:16.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:35:15 smithi153 ceph-mon[70044]: pgmap v2012: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:18.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:35:17 smithi083 ceph-mon[79930]: pgmap v2013: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:18.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:35:17 smithi083 ceph-mon[77242]: pgmap v2013: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:18.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:35:17 smithi153 ceph-mon[70044]: pgmap v2013: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:20.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:35:19 smithi083 ceph-mon[79930]: pgmap v2014: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:20.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:35:19 smithi083 ceph-mon[77242]: pgmap v2014: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:35:19 smithi153 ceph-mon[70044]: pgmap v2014: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:20.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:35:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:35:20.363Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:35:21.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:35:21 smithi153 ceph-mon[70044]: pgmap v2015: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:35:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:35:21] "GET /metrics HTTP/1.1" 200 225177 "" "Prometheus/2.33.4" 2022-09-29T06:35:21.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:35:21 smithi083 ceph-mon[79930]: pgmap v2015: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:21.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:35:21 smithi083 ceph-mon[77242]: pgmap v2015: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:35:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:35:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:35:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:35:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:35:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:35:24.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:35:23 smithi083 ceph-mon[79930]: pgmap v2016: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:24.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:35:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:35:23.766Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:35:24.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:35:23 smithi083 ceph-mon[77242]: pgmap v2016: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:24.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:35:23 smithi153 ceph-mon[70044]: pgmap v2016: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:24.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:35:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:35:24.467Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:35:25.325 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:35:25.617 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:35:25.617 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (69m) 77s ago 69m 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:35:25.618 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (69m) 9m ago 69m 61.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:35:25.618 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (70m) 77s ago 70m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:35:25.618 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (70m) 9m ago 70m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:35:25.619 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (66m) 9m ago 73m 439M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:35:25.619 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (66m) 77s ago 74m 570M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:35:25.619 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (65m) 77s ago 74m 310M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:35:25.619 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (65m) 9m ago 73m 132M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:35:25.620 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (65m) 77s ago 73m 186M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:35:25.620 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (69m) 77s ago 69m 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:35:25.620 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (69m) 9m ago 69m 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:35:25.621 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (64m) 77s ago 72m 234M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:35:25.621 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (64m) 77s ago 72m 226M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:35:25.621 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (63m) 77s ago 72m 183M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:35:25.621 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (63m) 77s ago 71m 216M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:35:25.622 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (62m) 9m ago 71m 176M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:35:25.622 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (62m) 9m ago 71m 207M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:35:25.622 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (61m) 9m ago 71m 184M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:35:25.623 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (61m) 9m ago 70m 209M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:35:25.623 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (66m) 9m ago 69m 103M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:35:25.977 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:35:25.977 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:35:25.978 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:35:25.978 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:35:25.978 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:35:25.978 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:35:25.979 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:35:25.979 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:35:25.979 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:35:25.979 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:35:25.979 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:35:25.980 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:35:25.980 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:35:25.980 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:35:25.980 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:35:25.981 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:35:25.981 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:35:25.981 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:35:26.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:35:25 smithi083 ceph-mon[79930]: pgmap v2017: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:26.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:35:25 smithi083 ceph-mon[79930]: from='client.36483 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:35:26.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:35:25 smithi083 ceph-mon[77242]: pgmap v2017: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:26.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:35:25 smithi083 ceph-mon[77242]: from='client.36483 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:35:26.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:35:25 smithi153 ceph-mon[70044]: pgmap v2017: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:26.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:35:25 smithi153 ceph-mon[70044]: from='client.36483 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:35:27.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:35:26 smithi083 ceph-mon[79930]: from='client.46423 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:35:27.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:35:26 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2863389158' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:35:27.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:35:26 smithi083 ceph-mon[77242]: from='client.46423 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:35:27.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:35:26 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2863389158' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:35:27.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:35:26 smithi153 ceph-mon[70044]: from='client.46423 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:35:27.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:35:26 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2863389158' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:35:28.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:35:27 smithi083 ceph-mon[79930]: pgmap v2018: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:28.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:35:27 smithi083 ceph-mon[77242]: pgmap v2018: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:28.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:35:27 smithi153 ceph-mon[70044]: pgmap v2018: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:30.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:35:29 smithi083 ceph-mon[79930]: pgmap v2019: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:30.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:35:29 smithi083 ceph-mon[77242]: pgmap v2019: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:30.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:35:29 smithi153 ceph-mon[70044]: pgmap v2019: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:30.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:35:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:35:30.363Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:35:31.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:35:31 smithi153 ceph-mon[70044]: pgmap v2020: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:35:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:35:31] "GET /metrics HTTP/1.1" 200 225179 "" "Prometheus/2.33.4" 2022-09-29T06:35:31.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:35:31 smithi083 ceph-mon[79930]: pgmap v2020: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:31.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:35:31 smithi083 ceph-mon[77242]: pgmap v2020: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:35:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:35:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:35:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:35:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:35:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:35:34.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:35:33 smithi083 ceph-mon[79930]: pgmap v2021: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:34.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:35:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:35:33.767Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:35:34.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:35:33 smithi083 ceph-mon[77242]: pgmap v2021: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:34.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:35:33 smithi153 ceph-mon[70044]: pgmap v2021: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:34.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:35:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:35:34.465Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:35:36.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:35:35 smithi083 ceph-mon[79930]: pgmap v2022: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:36.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:35:35 smithi083 ceph-mon[77242]: pgmap v2022: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:36.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:35:35 smithi153 ceph-mon[70044]: pgmap v2022: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:38.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:35:37 smithi083 ceph-mon[79930]: pgmap v2023: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:38.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:35:37 smithi083 ceph-mon[77242]: pgmap v2023: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:38.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:35:37 smithi153 ceph-mon[70044]: pgmap v2023: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:40.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:35:39 smithi083 ceph-mon[79930]: pgmap v2024: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:35:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:35:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:35:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:35:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:35:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:35:40.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:35:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:35:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:35:39 smithi083 ceph-mon[77242]: pgmap v2024: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:35:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:35:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:35:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:35:40.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:35:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:35:40.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:35:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:35:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:35:39 smithi153 ceph-mon[70044]: pgmap v2024: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:35:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:35:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:35:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:35:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:35:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:35:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:35:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:35:40.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:35:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:35:40.363Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:35:41.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:35:41 smithi153 ceph-mon[70044]: pgmap v2025: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:41.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:35:41 smithi083 ceph-mon[77242]: pgmap v2025: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:35:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:35:41] "GET /metrics HTTP/1.1" 200 225181 "" "Prometheus/2.33.4" 2022-09-29T06:35:41.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:35:41 smithi083 ceph-mon[79930]: pgmap v2025: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:35:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:35:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:35:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:35:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:35:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:35:44.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:35:43 smithi083 ceph-mon[77242]: pgmap v2026: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:44.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:35:43 smithi083 ceph-mon[79930]: pgmap v2026: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:44.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:35:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:35:43.768Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:35:44.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:35:43 smithi153 ceph-mon[70044]: pgmap v2026: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:44.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:35:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:35:44.466Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:35:46.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:35:45 smithi083 ceph-mon[77242]: pgmap v2027: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:46.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:35:45 smithi083 ceph-mon[79930]: pgmap v2027: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:46.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:35:45 smithi153 ceph-mon[70044]: pgmap v2027: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:48.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:35:47 smithi083 ceph-mon[77242]: pgmap v2028: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:48.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:35:47 smithi083 ceph-mon[79930]: pgmap v2028: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:48.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:35:47 smithi153 ceph-mon[70044]: pgmap v2028: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:50.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:35:49 smithi083 ceph-mon[79930]: pgmap v2029: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:50.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:35:49 smithi083 ceph-mon[77242]: pgmap v2029: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:50.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:35:49 smithi153 ceph-mon[70044]: pgmap v2029: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:50.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:35:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:35:50.364Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:35:51.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:35:51 smithi153 ceph-mon[70044]: pgmap v2030: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:51.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:35:51 smithi083 ceph-mon[77242]: pgmap v2030: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:51.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:35:51 smithi083 ceph-mon[79930]: pgmap v2030: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:51.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:35:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:35:51] "GET /metrics HTTP/1.1" 200 225181 "" "Prometheus/2.33.4" 2022-09-29T06:35:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:35:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:35:52.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:35:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:35:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:35:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:35:54.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:35:53 smithi083 ceph-mon[77242]: pgmap v2031: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:54.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:35:53 smithi083 ceph-mon[79930]: pgmap v2031: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:54.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:35:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:35:53.768Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:35:54.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:35:53 smithi153 ceph-mon[70044]: pgmap v2031: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:54.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:35:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:35:54.466Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:35:56.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:35:55 smithi083 ceph-mon[79930]: pgmap v2032: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:56.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:35:55 smithi083 ceph-mon[77242]: pgmap v2032: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:56.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:35:55 smithi153 ceph-mon[70044]: pgmap v2032: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:56.308 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:35:56.597 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:35:56.597 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (69m) 108s ago 70m 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:35:56.597 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (69m) 9m ago 69m 61.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:35:56.597 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (71m) 108s ago 70m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:35:56.598 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (70m) 9m ago 70m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:35:56.598 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (66m) 9m ago 73m 439M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:35:56.598 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (67m) 108s ago 75m 570M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:35:56.598 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (66m) 108s ago 75m 310M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:35:56.599 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (65m) 9m ago 73m 132M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:35:56.599 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (66m) 108s ago 73m 186M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:35:56.599 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (70m) 108s ago 70m 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:35:56.599 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (70m) 9m ago 70m 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:35:56.600 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (65m) 108s ago 73m 234M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:35:56.600 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (64m) 108s ago 72m 226M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:35:56.600 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (64m) 108s ago 72m 183M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:35:56.600 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (63m) 108s ago 72m 216M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:35:56.600 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (63m) 9m ago 72m 176M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:35:56.601 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (62m) 9m ago 71m 207M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:35:56.601 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (62m) 9m ago 71m 184M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:35:56.601 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (61m) 9m ago 71m 209M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:35:56.601 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (66m) 9m ago 70m 103M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:35:56.954 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:35:56.954 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:35:56.955 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:35:56.955 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:35:56.955 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:35:56.955 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:35:56.956 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:35:56.956 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:35:56.956 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:35:56.956 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:35:56.956 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:35:56.957 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:35:56.957 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:35:56.957 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:35:56.957 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:35:56.958 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:35:56.958 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:35:56.958 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:35:57.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:35:56 smithi083 ceph-mon[79930]: from='client.46435 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:35:57.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:35:56 smithi083 ceph-mon[77242]: from='client.46435 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:35:57.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:35:56 smithi153 ceph-mon[70044]: from='client.46435 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:35:58.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:35:57 smithi083 ceph-mon[79930]: from='client.36507 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:35:58.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:35:57 smithi083 ceph-mon[79930]: pgmap v2033: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:58.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:35:57 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/711181143' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:35:58.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:35:57 smithi083 ceph-mon[77242]: from='client.36507 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:35:58.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:35:57 smithi083 ceph-mon[77242]: pgmap v2033: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:58.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:35:57 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/711181143' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:35:58.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:35:57 smithi153 ceph-mon[70044]: from='client.36507 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:35:58.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:35:57 smithi153 ceph-mon[70044]: pgmap v2033: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:35:58.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:35:57 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/711181143' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:36:00.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:35:59 smithi083 ceph-mon[79930]: pgmap v2034: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:00.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:35:59 smithi083 ceph-mon[77242]: pgmap v2034: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:35:59 smithi153 ceph-mon[70044]: pgmap v2034: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:00.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:36:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:36:00.365Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:36:01.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:36:01 smithi153 ceph-mon[70044]: pgmap v2035: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:01.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:36:01 smithi083 ceph-mon[79930]: pgmap v2035: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:36:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:36:01] "GET /metrics HTTP/1.1" 200 225178 "" "Prometheus/2.33.4" 2022-09-29T06:36:01.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:36:01 smithi083 ceph-mon[77242]: pgmap v2035: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:36:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:36:02.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:36:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:36:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:36:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:36:04.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:36:03 smithi083 ceph-mon[79930]: pgmap v2036: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:04.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:36:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:36:03.768Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:36:04.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:36:03 smithi083 ceph-mon[77242]: pgmap v2036: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:04.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:36:03 smithi153 ceph-mon[70044]: pgmap v2036: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:04.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:36:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:36:04.467Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:36:06.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:36:05 smithi083 ceph-mon[79930]: pgmap v2037: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:06.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:36:05 smithi083 ceph-mon[77242]: pgmap v2037: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:06.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:36:05 smithi153 ceph-mon[70044]: pgmap v2037: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:08.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:36:07 smithi083 ceph-mon[79930]: pgmap v2038: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:08.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:36:07 smithi083 ceph-mon[77242]: pgmap v2038: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:08.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:36:07 smithi153 ceph-mon[70044]: pgmap v2038: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:10.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:36:09 smithi083 ceph-mon[79930]: pgmap v2039: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:10.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:36:09 smithi083 ceph-mon[77242]: pgmap v2039: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:10.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:36:09 smithi153 ceph-mon[70044]: pgmap v2039: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:10.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:36:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:36:10.365Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:36:11.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:36:11 smithi153 ceph-mon[70044]: pgmap v2040: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:11.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:36:11 smithi083 ceph-mon[77242]: pgmap v2040: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:36:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:36:11] "GET /metrics HTTP/1.1" 200 225179 "" "Prometheus/2.33.4" 2022-09-29T06:36:11.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:36:11 smithi083 ceph-mon[79930]: pgmap v2040: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:12.605 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:36:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:36:12.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:36:12.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:36:12 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:36:12.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:36:12 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:36:12.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:36:12 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:36:12.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:36:12 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:36:12.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:36:12 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:36:12.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:36:12 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:36:13.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:36:12 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:36:13.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:36:12 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:36:13.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:36:12 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:36:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:36:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:36:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:36:13.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:36:13 smithi083 ceph-mon[79930]: pgmap v2041: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:13.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:36:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:36:13.768Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:36:13.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:36:13 smithi083 ceph-mon[77242]: pgmap v2041: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:14.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:36:13 smithi153 ceph-mon[70044]: pgmap v2041: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:14.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:36:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:36:14.468Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:36:16.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:36:15 smithi083 ceph-mon[79930]: pgmap v2042: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:16.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:36:15 smithi083 ceph-mon[77242]: pgmap v2042: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:16.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:36:15 smithi153 ceph-mon[70044]: pgmap v2042: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:18.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:36:18 smithi153 ceph-mon[70044]: pgmap v2043: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:18.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:36:18 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:36:18.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:36:18 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:36:18.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:36:18 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:36:18.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:36:18 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:36:18.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:36:18 smithi083 ceph-mon[77242]: pgmap v2043: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:18.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:36:18 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:36:18.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:36:18 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:36:18.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:36:18 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:36:18.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:36:18 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:36:18.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:36:18 smithi083 ceph-mon[79930]: pgmap v2043: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:18.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:36:18 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:36:18.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:36:18 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:36:18.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:36:18 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:36:18.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:36:18 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:36:20.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:36:20 smithi153 ceph-mon[70044]: pgmap v2044: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:20.367 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:36:20 smithi083 ceph-mon[79930]: pgmap v2044: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:20.368 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:36:20 smithi083 ceph-mon[77242]: pgmap v2044: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:20.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:36:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:36:20.366Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:36:21.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:36:21 smithi153 ceph-mon[70044]: Detected new or changed devices on smithi153 2022-09-29T06:36:21.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:36:21 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:36:21.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:36:21 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:36:21.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:36:21 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:36:21.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:36:21 smithi153 ceph-mon[70044]: pgmap v2045: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:36:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:36:21] "GET /metrics HTTP/1.1" 200 225179 "" "Prometheus/2.33.4" 2022-09-29T06:36:21.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:36:21 smithi083 ceph-mon[79930]: Detected new or changed devices on smithi153 2022-09-29T06:36:21.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:36:21 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:36:21.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:36:21 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:36:21.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:36:21 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:36:21.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:36:21 smithi083 ceph-mon[79930]: pgmap v2045: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:21.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:36:21 smithi083 ceph-mon[77242]: Detected new or changed devices on smithi153 2022-09-29T06:36:21.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:36:21 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:36:21.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:36:21 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:36:21.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:36:21 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:36:21.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:36:21 smithi083 ceph-mon[77242]: pgmap v2045: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:36:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:36:22.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:36:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:36:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:36:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:36:24.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:36:23 smithi083 ceph-mon[79930]: pgmap v2046: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:24.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:36:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:36:23.769Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:36:24.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:36:23 smithi083 ceph-mon[77242]: pgmap v2046: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:24.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:36:23 smithi153 ceph-mon[70044]: pgmap v2046: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:24.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:36:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:36:24.468Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:36:26.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:36:25 smithi083 ceph-mon[79930]: pgmap v2047: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:26.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:36:25 smithi083 ceph-mon[77242]: pgmap v2047: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:26.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:36:25 smithi153 ceph-mon[70044]: pgmap v2047: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:27.279 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:36:27.566 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:36:27.566 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (70m) 2m ago 70m 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:36:27.567 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (70m) 10s ago 70m 61.1M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:36:27.567 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (71m) 2m ago 71m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:36:27.568 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (71m) 10s ago 71m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:36:27.568 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (67m) 10s ago 74m 439M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:36:27.569 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (67m) 2m ago 75m 570M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:36:27.569 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (66m) 2m ago 75m 310M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:36:27.570 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (66m) 10s ago 74m 147M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:36:27.570 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (66m) 2m ago 74m 186M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:36:27.571 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (70m) 2m ago 70m 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:36:27.571 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (70m) 10s ago 70m 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:36:27.572 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (65m) 2m ago 73m 234M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:36:27.572 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (65m) 2m ago 73m 226M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:36:27.573 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (64m) 2m ago 73m 183M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:36:27.573 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (64m) 2m ago 72m 216M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:36:27.574 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (63m) 10s ago 72m 198M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:36:27.574 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (63m) 10s ago 72m 235M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:36:27.575 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (62m) 10s ago 72m 208M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:36:27.575 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (62m) 10s ago 71m 238M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:36:27.575 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (67m) 10s ago 70m 105M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:36:27.922 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:36:27.922 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:36:27.922 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:36:27.923 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:36:27.923 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:36:27.923 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:36:27.923 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:36:27.924 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:36:27.924 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:36:27.924 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:36:27.924 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:36:27.925 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:36:27.925 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:36:27.925 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:36:27.925 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:36:27.926 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:36:27.926 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:36:27.926 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:36:28.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:36:27 smithi083 ceph-mon[79930]: pgmap v2048: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:28.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:36:27 smithi083 ceph-mon[79930]: from='client.46453 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:36:28.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:36:27 smithi083 ceph-mon[77242]: pgmap v2048: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:28.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:36:27 smithi083 ceph-mon[77242]: from='client.46453 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:36:28.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:36:27 smithi153 ceph-mon[70044]: pgmap v2048: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:28.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:36:27 smithi153 ceph-mon[70044]: from='client.46453 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:36:29.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:36:28 smithi083 ceph-mon[79930]: from='client.46459 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:36:29.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:36:28 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1618426024' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:36:29.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:36:28 smithi083 ceph-mon[77242]: from='client.46459 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:36:29.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:36:28 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1618426024' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:36:29.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:36:28 smithi153 ceph-mon[70044]: from='client.46459 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:36:29.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:36:28 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1618426024' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:36:30.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:36:29 smithi083 ceph-mon[79930]: pgmap v2049: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:30.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:36:29 smithi083 ceph-mon[77242]: pgmap v2049: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:30.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:36:29 smithi153 ceph-mon[70044]: pgmap v2049: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:30.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:36:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:36:30.366Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:36:31.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:36:31 smithi153 ceph-mon[70044]: pgmap v2050: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:31.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:36:31 smithi083 ceph-mon[79930]: pgmap v2050: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:36:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:36:31] "GET /metrics HTTP/1.1" 200 225178 "" "Prometheus/2.33.4" 2022-09-29T06:36:31.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:36:31 smithi083 ceph-mon[77242]: pgmap v2050: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:36:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:36:32.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:36:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:36:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:36:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:36:34.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:36:33 smithi083 ceph-mon[79930]: pgmap v2051: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:34.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:36:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:36:33.769Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:36:34.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:36:33 smithi083 ceph-mon[77242]: pgmap v2051: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:34.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:36:33 smithi153 ceph-mon[70044]: pgmap v2051: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:34.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:36:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:36:34.469Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:36:36.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:36:35 smithi083 ceph-mon[77242]: pgmap v2052: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:36.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:36:35 smithi083 ceph-mon[79930]: pgmap v2052: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:36.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:36:35 smithi153 ceph-mon[70044]: pgmap v2052: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:38.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:36:37 smithi083 ceph-mon[79930]: pgmap v2053: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:38.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:36:37 smithi083 ceph-mon[77242]: pgmap v2053: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:38.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:36:37 smithi153 ceph-mon[70044]: pgmap v2053: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:40.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:36:39 smithi083 ceph-mon[79930]: pgmap v2054: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:36:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:36:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:36:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:36:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:36:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:36:40.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:36:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:36:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:36:39 smithi083 ceph-mon[77242]: pgmap v2054: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:36:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:36:40.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:36:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:36:40.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:36:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:36:40.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:36:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:36:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:36:39 smithi153 ceph-mon[70044]: pgmap v2054: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:36:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:36:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:36:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:36:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:36:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:36:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:36:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:36:40.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:36:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:36:40.367Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:36:41.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:36:41 smithi153 ceph-mon[70044]: pgmap v2055: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:41.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:36:41 smithi083 ceph-mon[79930]: pgmap v2055: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:36:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:36:41] "GET /metrics HTTP/1.1" 200 225176 "" "Prometheus/2.33.4" 2022-09-29T06:36:41.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:36:41 smithi083 ceph-mon[77242]: pgmap v2055: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:36:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:36:42.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:36:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:36:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:36:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:36:44.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:36:43 smithi083 ceph-mon[79930]: pgmap v2056: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:44.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:36:43 smithi083 ceph-mon[77242]: pgmap v2056: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:44.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:36:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:36:43.770Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:36:44.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:36:43 smithi153 ceph-mon[70044]: pgmap v2056: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:44.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:36:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:36:44.470Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:36:46.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:36:45 smithi083 ceph-mon[77242]: pgmap v2057: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:46.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:36:45 smithi083 ceph-mon[79930]: pgmap v2057: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:46.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:36:45 smithi153 ceph-mon[70044]: pgmap v2057: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:48.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:36:47 smithi083 ceph-mon[79930]: pgmap v2058: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:48.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:36:47 smithi083 ceph-mon[77242]: pgmap v2058: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:48.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:36:47 smithi153 ceph-mon[70044]: pgmap v2058: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:50.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:36:49 smithi083 ceph-mon[79930]: pgmap v2059: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:50.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:36:49 smithi083 ceph-mon[77242]: pgmap v2059: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:50.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:36:49 smithi153 ceph-mon[70044]: pgmap v2059: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:50.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:36:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:36:50.368Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:36:51.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:36:51 smithi153 ceph-mon[70044]: pgmap v2060: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:36:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:36:51] "GET /metrics HTTP/1.1" 200 225176 "" "Prometheus/2.33.4" 2022-09-29T06:36:51.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:36:51 smithi083 ceph-mon[79930]: pgmap v2060: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:51.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:36:51 smithi083 ceph-mon[77242]: pgmap v2060: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:36:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:36:52.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:36:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:36:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:36:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:36:54.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:36:53 smithi083 ceph-mon[79930]: pgmap v2061: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:54.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:36:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:36:53.771Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:36:54.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:36:53 smithi083 ceph-mon[77242]: pgmap v2061: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:54.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:36:53 smithi153 ceph-mon[70044]: pgmap v2061: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:54.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:36:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:36:54.470Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:36:56.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:36:55 smithi083 ceph-mon[79930]: pgmap v2062: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:56.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:36:55 smithi083 ceph-mon[77242]: pgmap v2062: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:56.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:36:55 smithi153 ceph-mon[70044]: pgmap v2062: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:58.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:36:57 smithi083 ceph-mon[77242]: pgmap v2063: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:58.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:36:57 smithi083 ceph-mon[79930]: pgmap v2063: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:58.245 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:36:58.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:36:57 smithi153 ceph-mon[70044]: pgmap v2063: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:36:58.532 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:36:58.532 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (70m) 2m ago 71m 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:36:58.533 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (70m) 41s ago 70m 61.1M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:36:58.533 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (72m) 2m ago 72m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:36:58.533 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (71m) 41s ago 71m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:36:58.533 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (67m) 41s ago 74m 439M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:36:58.534 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (68m) 2m ago 76m 570M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:36:58.534 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (67m) 2m ago 76m 310M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:36:58.534 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (66m) 41s ago 74m 147M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:36:58.535 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (67m) 2m ago 74m 186M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:36:58.535 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (71m) 2m ago 71m 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:36:58.535 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (71m) 41s ago 71m 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:36:58.535 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (66m) 2m ago 74m 234M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:36:58.535 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (65m) 2m ago 74m 226M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:36:58.536 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (65m) 2m ago 73m 183M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:36:58.536 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (64m) 2m ago 73m 216M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:36:58.536 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (64m) 41s ago 73m 198M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:36:58.536 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (63m) 41s ago 72m 235M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:36:58.537 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (63m) 41s ago 72m 208M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:36:58.537 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (62m) 41s ago 72m 238M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:36:58.537 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (67m) 41s ago 71m 105M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:36:58.887 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:36:58.888 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:36:58.888 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:36:58.888 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:36:58.888 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:36:58.888 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:36:58.889 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:36:58.889 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:36:58.889 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:36:58.889 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:36:58.890 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:36:58.890 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:36:58.890 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:36:58.890 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:36:58.891 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:36:58.891 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:36:58.891 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:36:58.891 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:36:59.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:36:58 smithi083 ceph-mon[77242]: from='client.46471 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:36:59.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:36:58 smithi083 ceph-mon[79930]: from='client.46471 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:36:59.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:36:58 smithi153 ceph-mon[70044]: from='client.46471 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:37:00.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:36:59 smithi083 ceph-mon[77242]: from='client.46477 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:37:00.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:36:59 smithi083 ceph-mon[77242]: pgmap v2064: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:37:00.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:36:59 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1611367264' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:37:00.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:36:59 smithi083 ceph-mon[79930]: from='client.46477 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:37:00.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:36:59 smithi083 ceph-mon[79930]: pgmap v2064: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:37:00.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:36:59 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1611367264' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:37:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:36:59 smithi153 ceph-mon[70044]: from='client.46477 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:37:00.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:36:59 smithi153 ceph-mon[70044]: pgmap v2064: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:37:00.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:36:59 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1611367264' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:37:00.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:37:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:37:00.368Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:37:01.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:37:01 smithi153 ceph-mon[70044]: pgmap v2065: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:37:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:37:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:37:01] "GET /metrics HTTP/1.1" 200 225182 "" "Prometheus/2.33.4" 2022-09-29T06:37:01.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:37:01 smithi083 ceph-mon[79930]: pgmap v2065: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:37:01.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:37:01 smithi083 ceph-mon[77242]: pgmap v2065: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:37:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:37:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:37:02.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:37:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:37:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:37:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:37:04.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:37:03 smithi083 ceph-mon[77242]: pgmap v2066: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:37:04.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:37:03 smithi083 ceph-mon[79930]: pgmap v2066: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:37:04.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:37:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:37:03.772Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:37:04.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:37:03 smithi153 ceph-mon[70044]: pgmap v2066: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:37:04.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:37:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:37:04.470Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:37:06.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:37:05 smithi083 ceph-mon[79930]: pgmap v2067: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:37:06.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:37:05 smithi083 ceph-mon[77242]: pgmap v2067: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:37:06.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:37:05 smithi153 ceph-mon[70044]: pgmap v2067: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:37:08.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:37:07 smithi083 ceph-mon[79930]: pgmap v2068: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:37:08.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:37:07 smithi083 ceph-mon[77242]: pgmap v2068: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:37:08.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:37:07 smithi153 ceph-mon[70044]: pgmap v2068: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:37:10.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:37:09 smithi083 ceph-mon[79930]: pgmap v2069: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:37:10.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:37:09 smithi083 ceph-mon[77242]: pgmap v2069: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:37:10.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:37:09 smithi153 ceph-mon[70044]: pgmap v2069: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:37:10.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:37:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:37:10.368Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:37:11.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:37:11 smithi153 ceph-mon[70044]: pgmap v2070: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:37:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:37:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:37:11] "GET /metrics HTTP/1.1" 200 225177 "" "Prometheus/2.33.4" 2022-09-29T06:37:11.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:37:11 smithi083 ceph-mon[79930]: pgmap v2070: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:37:11.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:37:11 smithi083 ceph-mon[77242]: pgmap v2070: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:37:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:37:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:37:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:37:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:37:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:37:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:37:14.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:37:13 smithi083 ceph-mon[79930]: pgmap v2071: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:37:14.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:37:13 smithi083 ceph-mon[77242]: pgmap v2071: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:37:14.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:37:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:37:13.772Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:37:14.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:37:13 smithi153 ceph-mon[70044]: pgmap v2071: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:37:14.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:37:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:37:14.471Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:37:16.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:37:15 smithi083 ceph-mon[79930]: pgmap v2072: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:37:16.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:37:15 smithi083 ceph-mon[77242]: pgmap v2072: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:37:16.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:37:15 smithi153 ceph-mon[70044]: pgmap v2072: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:37:18.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:37:17 smithi083 ceph-mon[79930]: pgmap v2073: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:37:18.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:37:17 smithi083 ceph-mon[77242]: pgmap v2073: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:37:18.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:37:17 smithi153 ceph-mon[70044]: pgmap v2073: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:37:20.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:37:19 smithi083 ceph-mon[79930]: pgmap v2074: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:37:20.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:37:19 smithi083 ceph-mon[77242]: pgmap v2074: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:37:20.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:37:19 smithi153 ceph-mon[70044]: pgmap v2074: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:37:20.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:37:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:37:20.370Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:37:21.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:37:21 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:37:21.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:37:21 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:37:21.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:37:21 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:37:21.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:37:21 smithi153 ceph-mon[70044]: pgmap v2075: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:37:21.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:37:21 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:37:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:37:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:37:21] "GET /metrics HTTP/1.1" 200 225177 "" "Prometheus/2.33.4" 2022-09-29T06:37:21.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:37:21 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:37:21.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:37:21 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:37:21.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:37:21 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:37:21.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:37:21 smithi083 ceph-mon[79930]: pgmap v2075: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:37:21.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:37:21 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:37:21.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:37:21 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:37:21.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:37:21 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:37:21.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:37:21 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:37:21.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:37:21 smithi083 ceph-mon[77242]: pgmap v2075: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:37:21.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:37:21 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:37:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:37:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:37:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:37:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:37:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:37:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:37:24.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:37:23 smithi083 ceph-mon[77242]: pgmap v2076: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:37:24.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:37:23 smithi083 ceph-mon[79930]: pgmap v2076: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:37:24.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:37:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:37:23.774Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:37:24.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:37:23 smithi153 ceph-mon[70044]: pgmap v2076: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:37:24.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:37:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:37:24.472Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:37:26.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:37:25 smithi083 ceph-mon[79930]: pgmap v2077: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:37:26.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:37:25 smithi083 ceph-mon[77242]: pgmap v2077: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:37:26.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:37:25 smithi153 ceph-mon[70044]: pgmap v2077: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:37:28.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:37:27 smithi083 ceph-mon[79930]: pgmap v2078: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:37:28.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:37:27 smithi083 ceph-mon[77242]: pgmap v2078: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:37:28.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:37:27 smithi153 ceph-mon[70044]: pgmap v2078: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:37:29.211 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:37:29.500 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:37:29.500 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (71m) 3m ago 71m 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:37:29.500 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (71m) 72s ago 71m 61.1M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:37:29.501 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (72m) 3m ago 72m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:37:29.501 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (72m) 72s ago 72m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:37:29.501 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (68m) 72s ago 75m 439M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:37:29.501 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (68m) 3m ago 76m 570M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:37:29.502 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (68m) 3m ago 76m 310M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:37:29.502 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (67m) 72s ago 75m 147M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:37:29.502 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (67m) 3m ago 75m 186M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:37:29.502 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (71m) 3m ago 71m 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:37:29.503 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (71m) 72s ago 71m 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:37:29.503 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (66m) 3m ago 74m 234M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:37:29.503 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (66m) 3m ago 74m 226M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:37:29.503 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (65m) 3m ago 74m 183M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:37:29.504 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (65m) 3m ago 73m 216M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:37:29.504 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (64m) 72s ago 73m 198M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:37:29.504 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (64m) 72s ago 73m 235M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:37:29.504 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (63m) 72s ago 73m 208M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:37:29.505 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (63m) 72s ago 72m 238M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:37:29.505 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (68m) 72s ago 71m 105M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:37:29.858 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:37:29.858 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:37:29.859 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:37:29.859 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:37:29.859 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:37:29.859 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:37:29.860 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:37:29.860 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:37:29.860 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:37:29.860 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:37:29.861 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:37:29.861 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:37:29.861 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:37:29.861 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:37:29.862 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:37:29.862 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:37:29.862 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:37:29.862 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:37:30.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:37:29 smithi083 ceph-mon[79930]: pgmap v2079: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:37:30.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:37:29 smithi083 ceph-mon[79930]: from='client.46489 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:37:30.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:37:29 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2325333102' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:37:30.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:37:29 smithi083 ceph-mon[77242]: pgmap v2079: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:37:30.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:37:29 smithi083 ceph-mon[77242]: from='client.46489 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:37:30.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:37:29 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2325333102' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:37:30.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:37:29 smithi153 ceph-mon[70044]: pgmap v2079: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:37:30.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:37:29 smithi153 ceph-mon[70044]: from='client.46489 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:37:30.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:37:29 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2325333102' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:37:30.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:37:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:37:30.369Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:37:31.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:37:30 smithi083 ceph-mon[79930]: from='client.46495 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:37:31.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:37:30 smithi083 ceph-mon[77242]: from='client.46495 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:37:31.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:37:30 smithi153 ceph-mon[70044]: from='client.46495 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:37:31.895 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:37:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:37:31] "GET /metrics HTTP/1.1" 200 225176 "" "Prometheus/2.33.4" 2022-09-29T06:37:32.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:37:31 smithi083 ceph-mon[79930]: pgmap v2080: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:37:32.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:37:31 smithi083 ceph-mon[77242]: pgmap v2080: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:37:32.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:37:31 smithi153 ceph-mon[70044]: pgmap v2080: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:37:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:37:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:37:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:37:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:37:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:37:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:37:34.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:37:33 smithi083 ceph-mon[79930]: pgmap v2081: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:37:34.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:37:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:37:33.774Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:37:34.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:37:33 smithi083 ceph-mon[77242]: pgmap v2081: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:37:34.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:37:33 smithi153 ceph-mon[70044]: pgmap v2081: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:37:34.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:37:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:37:34.473Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:37:36.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:37:35 smithi083 ceph-mon[79930]: pgmap v2082: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:37:36.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:37:35 smithi083 ceph-mon[77242]: pgmap v2082: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:37:36.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:37:35 smithi153 ceph-mon[70044]: pgmap v2082: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:37:38.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:37:37 smithi083 ceph-mon[79930]: pgmap v2083: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:37:38.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:37:37 smithi083 ceph-mon[77242]: pgmap v2083: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:37:38.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:37:37 smithi153 ceph-mon[70044]: pgmap v2083: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:37:40.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:37:39 smithi083 ceph-mon[77242]: pgmap v2084: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:37:40.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:37:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:37:40.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:37:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:37:40.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:37:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:37:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:37:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:37:40.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:37:39 smithi083 ceph-mon[79930]: pgmap v2084: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:37:40.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:37:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:37:40.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:37:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:37:40.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:37:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:37:40.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:37:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:37:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:37:39 smithi153 ceph-mon[70044]: pgmap v2084: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:37:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:37:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:37:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:37:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:37:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:37:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:37:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:37:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:37:40.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:37:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:37:40.370Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:37:41.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:37:41 smithi153 ceph-mon[70044]: pgmap v2085: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:37:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:37:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:37:41] "GET /metrics HTTP/1.1" 200 225174 "" "Prometheus/2.33.4" 2022-09-29T06:37:41.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:37:41 smithi083 ceph-mon[79930]: pgmap v2085: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:37:41.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:37:41 smithi083 ceph-mon[77242]: pgmap v2085: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:37:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:37:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:37:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:37:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:37:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:37:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:37:44.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:37:43 smithi083 ceph-mon[77242]: pgmap v2086: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:37:44.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:37:43 smithi083 ceph-mon[79930]: pgmap v2086: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:37:44.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:37:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:37:43.774Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:37:44.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:37:43 smithi153 ceph-mon[70044]: pgmap v2086: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:37:44.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:37:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:37:44.473Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:37:46.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:37:45 smithi083 ceph-mon[77242]: pgmap v2087: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:37:46.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:37:45 smithi083 ceph-mon[79930]: pgmap v2087: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:37:46.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:37:45 smithi153 ceph-mon[70044]: pgmap v2087: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:37:48.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:37:47 smithi083 ceph-mon[79930]: pgmap v2088: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:37:48.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:37:47 smithi083 ceph-mon[77242]: pgmap v2088: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:37:48.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:37:47 smithi153 ceph-mon[70044]: pgmap v2088: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:37:50.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:37:49 smithi083 ceph-mon[79930]: pgmap v2089: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:37:50.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:37:49 smithi083 ceph-mon[77242]: pgmap v2089: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:37:50.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:37:49 smithi153 ceph-mon[70044]: pgmap v2089: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:37:50.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:37:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:37:50.370Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:37:51.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:37:51 smithi153 ceph-mon[70044]: pgmap v2090: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:37:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:37:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:37:51] "GET /metrics HTTP/1.1" 200 225174 "" "Prometheus/2.33.4" 2022-09-29T06:37:51.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:37:51 smithi083 ceph-mon[79930]: pgmap v2090: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:37:51.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:37:51 smithi083 ceph-mon[77242]: pgmap v2090: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:37:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:37:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:37:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:37:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:37:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:37:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:37:54.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:37:53 smithi083 ceph-mon[79930]: pgmap v2091: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:37:54.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:37:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:37:53.774Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:37:54.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:37:53 smithi083 ceph-mon[77242]: pgmap v2091: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:37:54.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:37:53 smithi153 ceph-mon[70044]: pgmap v2091: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:37:54.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:37:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:37:54.475Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:37:56.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:37:55 smithi083 ceph-mon[79930]: pgmap v2092: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:37:56.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:37:55 smithi083 ceph-mon[77242]: pgmap v2092: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:37:56.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:37:55 smithi153 ceph-mon[70044]: pgmap v2092: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:37:58.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:37:57 smithi083 ceph-mon[79930]: pgmap v2093: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:37:58.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:37:57 smithi083 ceph-mon[77242]: pgmap v2093: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:37:58.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:37:57 smithi153 ceph-mon[70044]: pgmap v2093: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:00.186 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:38:00.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:37:59 smithi083 ceph-mon[79930]: pgmap v2094: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:00.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:37:59 smithi083 ceph-mon[77242]: pgmap v2094: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:37:59 smithi153 ceph-mon[70044]: pgmap v2094: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:00.474 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:38:00.474 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (71m) 3m ago 72m 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:38:00.475 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (71m) 103s ago 71m 61.1M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:38:00.475 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (73m) 3m ago 73m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:38:00.475 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (73m) 103s ago 73m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:38:00.475 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (68m) 103s ago 75m 439M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:38:00.476 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (69m) 3m ago 77m 570M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:38:00.476 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (68m) 3m ago 77m 310M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:38:00.476 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (67m) 103s ago 75m 147M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:38:00.476 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (68m) 3m ago 75m 186M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:38:00.476 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (72m) 3m ago 72m 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:38:00.477 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (72m) 103s ago 72m 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:38:00.477 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (67m) 3m ago 75m 234M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:38:00.477 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (66m) 3m ago 75m 226M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:38:00.478 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (66m) 3m ago 74m 183M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:38:00.478 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (65m) 3m ago 74m 216M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:38:00.478 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (65m) 103s ago 74m 198M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:38:00.478 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (64m) 103s ago 73m 235M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:38:00.479 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (64m) 103s ago 73m 208M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:38:00.479 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (63m) 103s ago 73m 238M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:38:00.479 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (68m) 103s ago 72m 105M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:38:00.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:38:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:38:00.371Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:38:00.832 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:38:00.832 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:38:00.832 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:38:00.833 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:38:00.833 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:38:00.833 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:38:00.833 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:38:00.834 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:38:00.834 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:38:00.834 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:38:00.834 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:38:00.835 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:38:00.835 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:38:00.835 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:38:00.835 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:38:00.836 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:38:00.836 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:38:00.836 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:38:01.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:38:00 smithi083 ceph-mon[79930]: from='client.46507 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:38:01.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:38:00 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3168075405' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:38:01.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:38:00 smithi083 ceph-mon[77242]: from='client.46507 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:38:01.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:38:00 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3168075405' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:38:01.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:38:00 smithi153 ceph-mon[70044]: from='client.46507 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:38:01.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:38:00 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3168075405' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:38:01.828 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:38:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:38:01] "GET /metrics HTTP/1.1" 200 225180 "" "Prometheus/2.33.4" 2022-09-29T06:38:02.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:38:01 smithi083 ceph-mon[79930]: from='client.36579 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:38:02.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:38:01 smithi083 ceph-mon[79930]: pgmap v2095: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:02.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:38:01 smithi083 ceph-mon[77242]: from='client.36579 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:38:02.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:38:01 smithi083 ceph-mon[77242]: pgmap v2095: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:02.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:38:01 smithi153 ceph-mon[70044]: from='client.36579 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:38:02.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:38:01 smithi153 ceph-mon[70044]: pgmap v2095: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:38:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:38:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:38:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:38:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:38:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:38:04.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:38:03 smithi083 ceph-mon[79930]: pgmap v2096: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:04.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:38:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:38:03.775Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:38:04.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:38:03 smithi083 ceph-mon[77242]: pgmap v2096: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:04.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:38:03 smithi153 ceph-mon[70044]: pgmap v2096: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:04.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:38:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:38:04.475Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:38:06.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:38:05 smithi083 ceph-mon[77242]: pgmap v2097: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:06.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:38:05 smithi083 ceph-mon[79930]: pgmap v2097: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:06.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:38:05 smithi153 ceph-mon[70044]: pgmap v2097: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:08.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:38:07 smithi083 ceph-mon[79930]: pgmap v2098: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:08.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:38:07 smithi083 ceph-mon[77242]: pgmap v2098: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:08.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:38:07 smithi153 ceph-mon[70044]: pgmap v2098: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:10.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:38:09 smithi083 ceph-mon[79930]: pgmap v2099: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:10.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:38:09 smithi083 ceph-mon[77242]: pgmap v2099: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:10.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:38:09 smithi153 ceph-mon[70044]: pgmap v2099: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:10.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:38:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:38:10.372Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:38:11.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:38:11 smithi153 ceph-mon[70044]: pgmap v2100: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:38:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:38:11] "GET /metrics HTTP/1.1" 200 225177 "" "Prometheus/2.33.4" 2022-09-29T06:38:11.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:38:11 smithi083 ceph-mon[79930]: pgmap v2100: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:11.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:38:11 smithi083 ceph-mon[77242]: pgmap v2100: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:38:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:38:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:38:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:38:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:38:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:38:14.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:38:13 smithi083 ceph-mon[79930]: pgmap v2101: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:14.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:38:13 smithi083 ceph-mon[77242]: pgmap v2101: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:14.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:38:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:38:13.775Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:38:14.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:38:13 smithi153 ceph-mon[70044]: pgmap v2101: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:14.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:38:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:38:14.475Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:38:16.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:38:15 smithi083 ceph-mon[79930]: pgmap v2102: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:16.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:38:15 smithi083 ceph-mon[77242]: pgmap v2102: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:16.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:38:15 smithi153 ceph-mon[70044]: pgmap v2102: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:18.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:38:17 smithi083 ceph-mon[79930]: pgmap v2103: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:18.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:38:17 smithi083 ceph-mon[77242]: pgmap v2103: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:18.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:38:17 smithi153 ceph-mon[70044]: pgmap v2103: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:20.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:38:19 smithi083 ceph-mon[79930]: pgmap v2104: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:20.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:38:19 smithi083 ceph-mon[77242]: pgmap v2104: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:38:19 smithi153 ceph-mon[70044]: pgmap v2104: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:20.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:38:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:38:20.373Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:38:21.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:38:21 smithi153 ceph-mon[70044]: pgmap v2105: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:21.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:38:21 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:38:21.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:38:21 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:38:21.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:38:21 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:38:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:38:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:38:21] "GET /metrics HTTP/1.1" 200 225177 "" "Prometheus/2.33.4" 2022-09-29T06:38:21.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:38:21 smithi083 ceph-mon[77242]: pgmap v2105: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:21.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:38:21 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:38:21.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:38:21 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:38:21.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:38:21 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:38:21.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:38:21 smithi083 ceph-mon[79930]: pgmap v2105: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:21.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:38:21 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:38:21.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:38:21 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:38:21.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:38:21 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:38:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:38:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:38:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:38:23.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:38:22 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:38:23.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:38:22 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:38:23.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:38:22 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:38:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:38:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:38:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:38:24.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:38:23 smithi083 ceph-mon[79930]: pgmap v2106: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:24.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:38:23 smithi083 ceph-mon[77242]: pgmap v2106: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:24.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:38:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:38:23.777Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:38:24.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:38:23 smithi153 ceph-mon[70044]: pgmap v2106: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:24.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:38:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:38:24.476Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:38:26.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:38:25 smithi083 ceph-mon[79930]: pgmap v2107: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:26.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:38:25 smithi083 ceph-mon[77242]: pgmap v2107: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:26.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:38:25 smithi153 ceph-mon[70044]: pgmap v2107: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:28.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:38:27 smithi083 ceph-mon[79930]: pgmap v2108: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:28.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:38:27 smithi083 ceph-mon[77242]: pgmap v2108: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:28.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:38:27 smithi153 ceph-mon[70044]: pgmap v2108: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:30.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:38:29 smithi083 ceph-mon[79930]: pgmap v2109: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:30.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:38:29 smithi083 ceph-mon[77242]: pgmap v2109: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:30.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:38:29 smithi153 ceph-mon[70044]: pgmap v2109: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:30.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:38:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:38:30.373Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:38:31.161 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:38:31.450 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:38:31.451 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (72m) 4m ago 72m 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:38:31.451 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (72m) 2m ago 72m 61.1M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:38:31.451 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (73m) 4m ago 73m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:38:31.452 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (73m) 2m ago 73m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:38:31.452 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (69m) 2m ago 76m 439M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:38:31.452 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (69m) 4m ago 77m 570M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:38:31.452 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (69m) 4m ago 77m 310M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:38:31.453 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (68m) 2m ago 76m 147M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:38:31.453 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (68m) 4m ago 76m 186M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:38:31.453 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (72m) 4m ago 72m 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:38:31.453 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (72m) 2m ago 72m 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:38:31.453 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (67m) 4m ago 75m 234M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:38:31.454 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (67m) 4m ago 75m 226M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:38:31.454 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (66m) 4m ago 75m 183M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:38:31.454 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (66m) 4m ago 74m 216M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:38:31.455 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (65m) 2m ago 74m 198M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:38:31.455 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (65m) 2m ago 74m 235M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:38:31.455 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (64m) 2m ago 74m 208M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:38:31.455 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (64m) 2m ago 73m 238M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:38:31.455 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (69m) 2m ago 73m 105M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:38:31.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:38:31 smithi153 ceph-mon[70044]: pgmap v2110: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:31.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:38:31 smithi153 ceph-mon[70044]: from='client.46525 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:38:31.810 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:38:31.810 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:38:31.810 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:38:31.810 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:38:31.811 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:38:31.811 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:38:31.811 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:38:31.811 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:38:31.812 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:38:31.812 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:38:31.812 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:38:31.812 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:38:31.813 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:38:31.813 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:38:31.813 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:38:31.814 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:38:31.814 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:38:31.814 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:38:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:38:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:38:31] "GET /metrics HTTP/1.1" 200 225177 "" "Prometheus/2.33.4" 2022-09-29T06:38:31.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:38:31 smithi083 ceph-mon[79930]: pgmap v2110: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:31.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:38:31 smithi083 ceph-mon[79930]: from='client.46525 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:38:31.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:38:31 smithi083 ceph-mon[77242]: pgmap v2110: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:31.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:38:31 smithi083 ceph-mon[77242]: from='client.46525 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:38:32.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:38:32 smithi153 ceph-mon[70044]: from='client.36597 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:38:32.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:38:32 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/70864448' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:38:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:38:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:38:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:38:32.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:38:32 smithi083 ceph-mon[79930]: from='client.36597 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:38:32.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:38:32 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/70864448' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:38:32.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:38:32 smithi083 ceph-mon[77242]: from='client.36597 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:38:32.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:38:32 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/70864448' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:38:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:38:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:38:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:38:33.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:38:33 smithi153 ceph-mon[70044]: pgmap v2111: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:33.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:38:33 smithi083 ceph-mon[79930]: pgmap v2111: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:33.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:38:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:38:33.777Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:38:33.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:38:33 smithi083 ceph-mon[77242]: pgmap v2111: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:34.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:38:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:38:34.477Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:38:36.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:38:35 smithi083 ceph-mon[79930]: pgmap v2112: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:36.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:38:35 smithi083 ceph-mon[77242]: pgmap v2112: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:36.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:38:35 smithi153 ceph-mon[70044]: pgmap v2112: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:38.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:38:37 smithi083 ceph-mon[77242]: pgmap v2113: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:38.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:38:37 smithi083 ceph-mon[79930]: pgmap v2113: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:38.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:38:37 smithi153 ceph-mon[70044]: pgmap v2113: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:40.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:38:39 smithi083 ceph-mon[77242]: pgmap v2114: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:40.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:38:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:38:40.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:38:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:38:40.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:38:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:38:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:38:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:38:40.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:38:39 smithi083 ceph-mon[79930]: pgmap v2114: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:40.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:38:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:38:40.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:38:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:38:40.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:38:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:38:40.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:38:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:38:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:38:39 smithi153 ceph-mon[70044]: pgmap v2114: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:38:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:38:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:38:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:38:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:38:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:38:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:38:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:38:40.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:38:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:38:40.374Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:38:41.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:38:41 smithi153 ceph-mon[70044]: pgmap v2115: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:38:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:38:41] "GET /metrics HTTP/1.1" 200 225178 "" "Prometheus/2.33.4" 2022-09-29T06:38:41.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:38:41 smithi083 ceph-mon[79930]: pgmap v2115: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:41.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:38:41 smithi083 ceph-mon[77242]: pgmap v2115: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:38:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:38:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:38:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:38:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:38:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:38:44.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:38:43 smithi083 ceph-mon[79930]: pgmap v2116: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:44.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:38:43 smithi083 ceph-mon[77242]: pgmap v2116: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:44.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:38:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:38:43.778Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:38:44.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:38:43 smithi153 ceph-mon[70044]: pgmap v2116: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:44.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:38:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:38:44.478Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:38:46.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:38:45 smithi083 ceph-mon[77242]: pgmap v2117: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:46.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:38:45 smithi083 ceph-mon[79930]: pgmap v2117: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:46.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:38:45 smithi153 ceph-mon[70044]: pgmap v2117: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:48.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:38:47 smithi083 ceph-mon[79930]: pgmap v2118: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:48.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:38:47 smithi083 ceph-mon[77242]: pgmap v2118: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:48.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:38:47 smithi153 ceph-mon[70044]: pgmap v2118: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:50.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:38:49 smithi083 ceph-mon[79930]: pgmap v2119: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:50.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:38:49 smithi083 ceph-mon[77242]: pgmap v2119: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:50.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:38:49 smithi153 ceph-mon[70044]: pgmap v2119: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:50.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:38:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:38:50.375Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:38:51.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:38:51 smithi153 ceph-mon[70044]: pgmap v2120: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:38:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:38:51] "GET /metrics HTTP/1.1" 200 225178 "" "Prometheus/2.33.4" 2022-09-29T06:38:51.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:38:51 smithi083 ceph-mon[79930]: pgmap v2120: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:51.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:38:51 smithi083 ceph-mon[77242]: pgmap v2120: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:38:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:38:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:38:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:38:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:38:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:38:54.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:38:53 smithi083 ceph-mon[79930]: pgmap v2121: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:54.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:38:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:38:53.779Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:38:54.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:38:53 smithi083 ceph-mon[77242]: pgmap v2121: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:54.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:38:53 smithi153 ceph-mon[70044]: pgmap v2121: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:54.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:38:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:38:54.478Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:38:56.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:38:55 smithi083 ceph-mon[77242]: pgmap v2122: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:56.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:38:55 smithi083 ceph-mon[79930]: pgmap v2122: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:56.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:38:55 smithi153 ceph-mon[70044]: pgmap v2122: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:58.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:38:57 smithi083 ceph-mon[77242]: pgmap v2123: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:58.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:38:57 smithi083 ceph-mon[79930]: pgmap v2123: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:38:58.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:38:57 smithi153 ceph-mon[70044]: pgmap v2123: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:00.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:38:59 smithi083 ceph-mon[79930]: pgmap v2124: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:00.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:38:59 smithi083 ceph-mon[77242]: pgmap v2124: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:38:59 smithi153 ceph-mon[70044]: pgmap v2124: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:00.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:39:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:39:00.376Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:39:01.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:39:01 smithi153 ceph-mon[70044]: pgmap v2125: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:39:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:39:01] "GET /metrics HTTP/1.1" 200 225176 "" "Prometheus/2.33.4" 2022-09-29T06:39:01.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:39:01 smithi083 ceph-mon[79930]: pgmap v2125: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:01.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:39:01 smithi083 ceph-mon[77242]: pgmap v2125: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:02.139 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:39:02.429 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:39:02.430 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (72m) 4m ago 73m 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:39:02.430 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (73m) 2m ago 73m 61.1M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:39:02.430 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (74m) 4m ago 74m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:39:02.430 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (74m) 2m ago 74m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:39:02.431 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (69m) 2m ago 76m 439M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:39:02.431 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (70m) 4m ago 78m 570M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:39:02.431 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (69m) 4m ago 78m 310M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:39:02.431 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (68m) 2m ago 76m 147M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:39:02.432 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (69m) 4m ago 77m 186M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:39:02.432 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (73m) 4m ago 73m 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:39:02.432 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (73m) 2m ago 73m 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:39:02.432 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (68m) 4m ago 76m 234M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:39:02.433 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (67m) 4m ago 76m 226M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:39:02.433 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (67m) 4m ago 75m 183M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:39:02.433 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (66m) 4m ago 75m 216M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:39:02.434 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (66m) 2m ago 75m 198M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:39:02.434 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (65m) 2m ago 74m 235M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:39:02.434 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (65m) 2m ago 74m 208M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:39:02.434 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (64m) 2m ago 74m 238M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:39:02.435 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (69m) 2m ago 73m 105M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:39:02.786 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:39:02.786 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:39:02.787 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:39:02.787 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:39:02.787 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:39:02.788 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:39:02.788 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:39:02.788 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:39:02.788 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:39:02.789 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:39:02.789 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:39:02.789 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:39:02.789 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:39:02.790 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:39:02.790 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:39:02.790 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:39:02.790 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:39:02.791 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:39:02.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:39:02 smithi153 ceph-mon[70044]: from='client.46543 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:39:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:39:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:39:02.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:39:02.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:39:02 smithi083 ceph-mon[77242]: from='client.46543 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:39:02.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:39:02 smithi083 ceph-mon[79930]: from='client.46543 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:39:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:39:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:39:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:39:03.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:39:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:39:03.779Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:39:03.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:39:03 smithi083 ceph-mon[79930]: from='client.46549 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:39:03.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:39:03 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3561718551' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:39:03.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:39:03 smithi083 ceph-mon[79930]: pgmap v2126: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:03.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:39:03 smithi083 ceph-mon[77242]: from='client.46549 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:39:03.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:39:03 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3561718551' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:39:03.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:39:03 smithi083 ceph-mon[77242]: pgmap v2126: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:04.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:39:03 smithi153 ceph-mon[70044]: from='client.46549 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:39:04.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:39:03 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3561718551' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:39:04.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:39:03 smithi153 ceph-mon[70044]: pgmap v2126: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:04.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:39:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:39:04.479Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:39:06.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:39:05 smithi083 ceph-mon[79930]: pgmap v2127: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:06.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:39:05 smithi083 ceph-mon[77242]: pgmap v2127: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:06.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:39:05 smithi153 ceph-mon[70044]: pgmap v2127: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:08.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:39:07 smithi083 ceph-mon[77242]: pgmap v2128: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:08.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:39:07 smithi083 ceph-mon[79930]: pgmap v2128: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:08.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:39:07 smithi153 ceph-mon[70044]: pgmap v2128: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:10.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:39:09 smithi083 ceph-mon[79930]: pgmap v2129: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:10.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:39:09 smithi083 ceph-mon[77242]: pgmap v2129: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:10.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:39:09 smithi153 ceph-mon[70044]: pgmap v2129: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:10.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:39:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:39:10.376Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:39:11.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:39:11 smithi153 ceph-mon[70044]: pgmap v2130: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:39:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:39:11] "GET /metrics HTTP/1.1" 200 225175 "" "Prometheus/2.33.4" 2022-09-29T06:39:11.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:39:11 smithi083 ceph-mon[79930]: pgmap v2130: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:11.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:39:11 smithi083 ceph-mon[77242]: pgmap v2130: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:12.810 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:39:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:39:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:39:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:39:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:39:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:39:14.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:39:13 smithi083 ceph-mon[79930]: pgmap v2131: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:14.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:39:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:39:13.780Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:39:14.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:39:13 smithi083 ceph-mon[77242]: pgmap v2131: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:14.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:39:13 smithi153 ceph-mon[70044]: pgmap v2131: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:14.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:39:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:39:14.479Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:39:16.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:39:15 smithi083 ceph-mon[79930]: pgmap v2132: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:16.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:39:15 smithi083 ceph-mon[77242]: pgmap v2132: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:16.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:39:15 smithi153 ceph-mon[70044]: pgmap v2132: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:18.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:39:17 smithi083 ceph-mon[79930]: pgmap v2133: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:18.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:39:17 smithi083 ceph-mon[77242]: pgmap v2133: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:18.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:39:17 smithi153 ceph-mon[70044]: pgmap v2133: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:20.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:39:19 smithi083 ceph-mon[79930]: pgmap v2134: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:20.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:39:19 smithi083 ceph-mon[77242]: pgmap v2134: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:39:19 smithi153 ceph-mon[70044]: pgmap v2134: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:20.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:39:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:39:20.376Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:39:21.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:39:21 smithi153 ceph-mon[70044]: pgmap v2135: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:39:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:39:21] "GET /metrics HTTP/1.1" 200 225175 "" "Prometheus/2.33.4" 2022-09-29T06:39:21.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:39:21 smithi083 ceph-mon[79930]: pgmap v2135: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:21.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:39:21 smithi083 ceph-mon[77242]: pgmap v2135: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:39:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:39:22.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:39:22.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:39:22 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:39:22.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:39:22 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:39:22.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:39:22 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:39:22.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:39:22 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:39:22.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:39:22 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:39:22.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:39:22 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:39:22.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:39:22 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:39:22.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:39:22 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:39:22.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:39:22 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:39:22.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:39:22 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:39:22.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:39:22 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:39:22.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:39:22 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:39:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:39:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:39:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:39:23.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:39:23 smithi083 ceph-mon[79930]: pgmap v2136: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:23.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:39:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:39:23.781Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:39:23.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:39:23 smithi083 ceph-mon[77242]: pgmap v2136: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:24.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:39:23 smithi153 ceph-mon[70044]: pgmap v2136: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:24.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:39:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:39:24.480Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:39:26.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:39:25 smithi083 ceph-mon[79930]: pgmap v2137: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:26.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:39:25 smithi083 ceph-mon[77242]: pgmap v2137: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:26.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:39:25 smithi153 ceph-mon[70044]: pgmap v2137: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:28.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:39:27 smithi083 ceph-mon[79930]: pgmap v2138: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:28.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:39:27 smithi083 ceph-mon[77242]: pgmap v2138: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:28.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:39:27 smithi153 ceph-mon[70044]: pgmap v2138: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:30.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:39:29 smithi083 ceph-mon[79930]: pgmap v2139: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:30.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:39:29 smithi083 ceph-mon[77242]: pgmap v2139: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:30.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:39:29 smithi153 ceph-mon[70044]: pgmap v2139: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:30.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:39:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:39:30.377Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:39:31.560 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:39:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:39:31] "GET /metrics HTTP/1.1" 200 225175 "" "Prometheus/2.33.4" 2022-09-29T06:39:31.560 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:39:31 smithi083 ceph-mon[77242]: pgmap v2140: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:31.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:39:31 smithi083 ceph-mon[79930]: pgmap v2140: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:32.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:39:31 smithi153 ceph-mon[70044]: pgmap v2140: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:39:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:39:32.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:39:33.115 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:39:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:39:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:39:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:39:33.410 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:39:33.411 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (73m) 5m ago 73m 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:39:33.411 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (73m) 3m ago 73m 61.1M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:39:33.411 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (74m) 5m ago 74m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:39:33.412 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (74m) 3m ago 74m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:39:33.412 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (70m) 3m ago 77m 439M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:39:33.412 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (70m) 5m ago 78m 570M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:39:33.412 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (70m) 5m ago 78m 310M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:39:33.413 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (69m) 3m ago 77m 147M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:39:33.413 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (69m) 5m ago 77m 186M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:39:33.413 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (73m) 5m ago 73m 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:39:33.413 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (73m) 3m ago 73m 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:39:33.414 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (68m) 5m ago 76m 234M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:39:33.414 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (68m) 5m ago 76m 226M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:39:33.414 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (67m) 5m ago 76m 183M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:39:33.414 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (67m) 5m ago 75m 216M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:39:33.415 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (66m) 3m ago 75m 198M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:39:33.415 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (66m) 3m ago 75m 235M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:39:33.415 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (65m) 3m ago 75m 208M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:39:33.415 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (65m) 3m ago 74m 238M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:39:33.416 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (70m) 3m ago 74m 105M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:39:33.776 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:39:33.776 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:39:33.776 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:39:33.777 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:39:33.777 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:39:33.777 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:39:33.777 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:39:33.778 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:39:33.778 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:39:33.778 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:39:33.778 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:39:33.779 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:39:33.779 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:39:33.779 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:39:33.779 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:39:33.780 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:39:33.780 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:39:33.780 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:39:34.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:39:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:39:33.781Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:39:34.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:39:33 smithi083 ceph-mon[77242]: pgmap v2141: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:34.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:39:33 smithi083 ceph-mon[77242]: from='client.36627 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:39:34.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:39:33 smithi083 ceph-mon[77242]: from='client.46567 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:39:34.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:39:33 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3680272569' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:39:34.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:39:33 smithi083 ceph-mon[79930]: pgmap v2141: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:34.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:39:33 smithi083 ceph-mon[79930]: from='client.36627 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:39:34.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:39:33 smithi083 ceph-mon[79930]: from='client.46567 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:39:34.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:39:33 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3680272569' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:39:34.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:39:33 smithi153 ceph-mon[70044]: pgmap v2141: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:34.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:39:33 smithi153 ceph-mon[70044]: from='client.36627 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:39:34.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:39:33 smithi153 ceph-mon[70044]: from='client.46567 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:39:34.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:39:33 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3680272569' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:39:34.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:39:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:39:34.480Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:39:36.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:39:35 smithi083 ceph-mon[79930]: pgmap v2142: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:36.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:39:35 smithi083 ceph-mon[77242]: pgmap v2142: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:36.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:39:35 smithi153 ceph-mon[70044]: pgmap v2142: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:38.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:39:37 smithi083 ceph-mon[79930]: pgmap v2143: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:38.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:39:37 smithi083 ceph-mon[77242]: pgmap v2143: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:38.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:39:37 smithi153 ceph-mon[70044]: pgmap v2143: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:40.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:39:39 smithi083 ceph-mon[79930]: pgmap v2144: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:39:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:39:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:39:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:39:40.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:39:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:39:40.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:39:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:39:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:39:39 smithi083 ceph-mon[77242]: pgmap v2144: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:39:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:39:40.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:39:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:39:40.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:39:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:39:40.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:39:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:39:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:39:39 smithi153 ceph-mon[70044]: pgmap v2144: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:39:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:39:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:39:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:39:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:39:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:39:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:39:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:39:40.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:39:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:39:40.377Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:39:41.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:39:41 smithi083 ceph-mon[79930]: pgmap v2145: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:41.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:39:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:39:41] "GET /metrics HTTP/1.1" 200 225176 "" "Prometheus/2.33.4" 2022-09-29T06:39:41.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:39:41 smithi083 ceph-mon[77242]: pgmap v2145: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:42.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:39:41 smithi153 ceph-mon[70044]: pgmap v2145: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:39:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:39:42.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:39:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:39:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:39:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:39:44.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:39:43 smithi083 ceph-mon[79930]: pgmap v2146: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:44.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:39:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:39:43.782Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:39:44.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:39:43 smithi083 ceph-mon[77242]: pgmap v2146: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:44.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:39:43 smithi153 ceph-mon[70044]: pgmap v2146: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:44.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:39:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:39:44.482Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:39:46.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:39:45 smithi083 ceph-mon[79930]: pgmap v2147: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:46.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:39:45 smithi083 ceph-mon[77242]: pgmap v2147: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:46.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:39:45 smithi153 ceph-mon[70044]: pgmap v2147: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:48.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:39:47 smithi083 ceph-mon[79930]: pgmap v2148: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:48.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:39:47 smithi083 ceph-mon[77242]: pgmap v2148: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:48.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:39:47 smithi153 ceph-mon[70044]: pgmap v2148: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:50.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:39:49 smithi083 ceph-mon[77242]: pgmap v2149: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:50.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:39:49 smithi083 ceph-mon[79930]: pgmap v2149: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:50.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:39:49 smithi153 ceph-mon[70044]: pgmap v2149: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:50.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:39:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:39:50.378Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:39:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:39:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:39:51] "GET /metrics HTTP/1.1" 200 225176 "" "Prometheus/2.33.4" 2022-09-29T06:39:51.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:39:51 smithi083 ceph-mon[79930]: pgmap v2150: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:51.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:39:51 smithi083 ceph-mon[77242]: pgmap v2150: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:52.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:39:51 smithi153 ceph-mon[70044]: pgmap v2150: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:39:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:39:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:39:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:39:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:39:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:39:54.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:39:53 smithi083 ceph-mon[77242]: pgmap v2151: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:54.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:39:53 smithi083 ceph-mon[79930]: pgmap v2151: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:54.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:39:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:39:53.783Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:39:54.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:39:53 smithi153 ceph-mon[70044]: pgmap v2151: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:54.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:39:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:39:54.482Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:39:56.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:39:55 smithi083 ceph-mon[79930]: pgmap v2152: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:56.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:39:55 smithi083 ceph-mon[77242]: pgmap v2152: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:56.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:39:55 smithi153 ceph-mon[70044]: pgmap v2152: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:58.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:39:57 smithi083 ceph-mon[79930]: pgmap v2153: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:58.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:39:57 smithi083 ceph-mon[77242]: pgmap v2153: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:39:58.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:39:57 smithi153 ceph-mon[70044]: pgmap v2153: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:00.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:39:59 smithi083 ceph-mon[79930]: pgmap v2154: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:00.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:39:59 smithi083 ceph-mon[77242]: pgmap v2154: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:00.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:40:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[77219]: 2022-09-29T06:39:59.999+0000 7f160f665700 -1 log_channel(cluster) log [ERR] : overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T06:40:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:39:59 smithi153 ceph-mon[70044]: pgmap v2154: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:00.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:40:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:40:00.381Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:40:01.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:40:00 smithi083 ceph-mon[79930]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T06:40:01.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:40:00 smithi083 ceph-mon[77242]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T06:40:01.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:40:00 smithi153 ceph-mon[70044]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T06:40:01.947 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:40:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:40:01] "GET /metrics HTTP/1.1" 200 225179 "" "Prometheus/2.33.4" 2022-09-29T06:40:02.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:40:01 smithi083 ceph-mon[79930]: pgmap v2155: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:02.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:40:01 smithi083 ceph-mon[77242]: pgmap v2155: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:02.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:40:01 smithi153 ceph-mon[70044]: pgmap v2155: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:40:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:40:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:40:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:40:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:40:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:40:04.107 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:40:04.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:40:03 smithi083 ceph-mon[79930]: pgmap v2156: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:04.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:40:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:40:03.783Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:40:04.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:40:03 smithi083 ceph-mon[77242]: pgmap v2156: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:04.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:40:03 smithi153 ceph-mon[70044]: pgmap v2156: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:04.396 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:40:04.396 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (73m) 5m ago 74m 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:40:04.396 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (74m) 3m ago 74m 61.1M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:40:04.397 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (75m) 5m ago 75m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:40:04.397 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (75m) 3m ago 75m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:40:04.397 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (70m) 3m ago 77m 439M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:40:04.397 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (71m) 5m ago 79m 570M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:40:04.398 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (70m) 5m ago 79m 310M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:40:04.398 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (69m) 3m ago 77m 147M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:40:04.398 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (70m) 5m ago 78m 186M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:40:04.398 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (74m) 5m ago 74m 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:40:04.399 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (74m) 3m ago 74m 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:40:04.399 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (69m) 5m ago 77m 234M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:40:04.399 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (68m) 5m ago 77m 226M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:40:04.399 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (68m) 5m ago 76m 183M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:40:04.399 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (67m) 5m ago 76m 216M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:40:04.400 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (67m) 3m ago 76m 198M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:40:04.400 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (66m) 3m ago 75m 235M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:40:04.400 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (66m) 3m ago 75m 208M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:40:04.400 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (65m) 3m ago 75m 238M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:40:04.401 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (70m) 3m ago 74m 105M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:40:04.750 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:40:04.751 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:40:04.751 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:40:04.751 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:40:04.751 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:40:04.752 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:40:04.752 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:40:04.752 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:40:04.752 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:40:04.753 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:40:04.753 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:40:04.753 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:40:04.753 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:40:04.754 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:40:04.754 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:40:04.754 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:40:04.754 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:40:04.755 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:40:04.957 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:40:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:40:04.482Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:40:05.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:40:04 smithi083 ceph-mon[79930]: from='client.36645 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:40:05.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:40:04 smithi083 ceph-mon[79930]: from='client.36651 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:40:05.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:40:04 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2087077553' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:40:05.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:40:04 smithi083 ceph-mon[77242]: from='client.36645 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:40:05.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:40:04 smithi083 ceph-mon[77242]: from='client.36651 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:40:05.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:40:04 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2087077553' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:40:05.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:40:04 smithi153 ceph-mon[70044]: from='client.36645 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:40:05.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:40:04 smithi153 ceph-mon[70044]: from='client.36651 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:40:05.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:40:04 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2087077553' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:40:06.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:40:05 smithi083 ceph-mon[79930]: pgmap v2157: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:06.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:40:05 smithi083 ceph-mon[77242]: pgmap v2157: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:06.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:40:05 smithi153 ceph-mon[70044]: pgmap v2157: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:08.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:40:07 smithi083 ceph-mon[77242]: pgmap v2158: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:08.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:40:07 smithi083 ceph-mon[79930]: pgmap v2158: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:08.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:40:07 smithi153 ceph-mon[70044]: pgmap v2158: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:10.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:40:09 smithi153 ceph-mon[70044]: pgmap v2159: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:10.381 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:40:09 smithi083 ceph-mon[79930]: pgmap v2159: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:10.382 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:40:09 smithi083 ceph-mon[77242]: pgmap v2159: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:10.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:40:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:40:10.380Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:40:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:40:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:40:11] "GET /metrics HTTP/1.1" 200 225178 "" "Prometheus/2.33.4" 2022-09-29T06:40:11.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:40:11 smithi083 ceph-mon[79930]: pgmap v2160: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:11.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:40:11 smithi083 ceph-mon[77242]: pgmap v2160: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:12.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:40:11 smithi153 ceph-mon[70044]: pgmap v2160: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:40:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:40:12.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:40:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:40:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:40:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:40:14.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:40:13 smithi083 ceph-mon[77242]: pgmap v2161: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:14.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:40:13 smithi083 ceph-mon[79930]: pgmap v2161: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:14.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:40:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:40:13.783Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:40:14.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:40:13 smithi153 ceph-mon[70044]: pgmap v2161: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:14.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:40:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:40:14.483Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:40:16.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:40:15 smithi083 ceph-mon[77242]: pgmap v2162: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:16.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:40:15 smithi083 ceph-mon[79930]: pgmap v2162: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:16.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:40:15 smithi153 ceph-mon[70044]: pgmap v2162: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:18.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:40:17 smithi083 ceph-mon[77242]: pgmap v2163: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:18.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:40:17 smithi083 ceph-mon[79930]: pgmap v2163: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:18.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:40:17 smithi153 ceph-mon[70044]: pgmap v2163: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:20.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:40:19 smithi083 ceph-mon[79930]: pgmap v2164: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:20.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:40:19 smithi083 ceph-mon[77242]: pgmap v2164: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:40:19 smithi153 ceph-mon[70044]: pgmap v2164: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:20.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:40:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:40:20.380Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:40:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:40:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:40:21] "GET /metrics HTTP/1.1" 200 225178 "" "Prometheus/2.33.4" 2022-09-29T06:40:21.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:40:21 smithi083 ceph-mon[79930]: pgmap v2165: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:21.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:40:21 smithi083 ceph-mon[77242]: pgmap v2165: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:22.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:40:21 smithi153 ceph-mon[70044]: pgmap v2165: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:22.571 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:40:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:40:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:40:22.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:40:22 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:40:22.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:40:22 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:40:22.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:40:22 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:40:22.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:40:22 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:40:22.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:40:22 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:40:22.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:40:22 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:40:23.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:40:22 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:40:23.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:40:22 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:40:23.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:40:22 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:40:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:40:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:40:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:40:24.112 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:40:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:40:23.783Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:40:24.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:40:24 smithi083 ceph-mon[77242]: pgmap v2166: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:24.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:40:24 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:40:24.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:40:24 smithi083 ceph-mon[79930]: pgmap v2166: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:24.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:40:24 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:40:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:40:24 smithi153 ceph-mon[70044]: pgmap v2166: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:24.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:40:24 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:40:24.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:40:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:40:24.483Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:40:26.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:40:26 smithi083 ceph-mon[77242]: pgmap v2167: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:26.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:40:26 smithi083 ceph-mon[79930]: pgmap v2167: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:40:26 smithi153 ceph-mon[70044]: pgmap v2167: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:28.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:40:28 smithi083 ceph-mon[77242]: pgmap v2168: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:28.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:40:28 smithi083 ceph-mon[79930]: pgmap v2168: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:40:28 smithi153 ceph-mon[70044]: pgmap v2168: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:30.382 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:40:30 smithi083 ceph-mon[79930]: pgmap v2169: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:30.383 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:40:30 smithi083 ceph-mon[77242]: pgmap v2169: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:30.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:40:30 smithi153 ceph-mon[70044]: pgmap v2169: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:30.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:40:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:40:30.381Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:40:31.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:40:31 smithi083 ceph-mon[77242]: pgmap v2170: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:40:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:40:31] "GET /metrics HTTP/1.1" 200 225176 "" "Prometheus/2.33.4" 2022-09-29T06:40:31.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:40:31 smithi083 ceph-mon[79930]: pgmap v2170: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:32.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:40:31 smithi153 ceph-mon[70044]: pgmap v2170: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:40:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:40:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:40:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:40:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:40:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:40:34.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:40:33 smithi083 ceph-mon[77242]: pgmap v2171: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:34.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:40:33 smithi083 ceph-mon[79930]: pgmap v2171: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:34.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:40:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:40:33.783Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:40:34.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:40:33 smithi153 ceph-mon[70044]: pgmap v2171: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:34.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:40:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:40:34.484Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:40:35.082 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:40:35.369 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:40:35.370 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (74m) 6m ago 74m 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:40:35.370 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (74m) 4m ago 74m 61.1M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:40:35.370 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (75m) 6m ago 75m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:40:35.370 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (75m) 4m ago 75m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:40:35.371 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (71m) 4m ago 78m 439M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:40:35.371 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (71m) 6m ago 79m 570M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:40:35.371 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (71m) 6m ago 79m 310M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:40:35.371 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (70m) 4m ago 78m 147M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:40:35.372 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (70m) 6m ago 78m 186M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:40:35.372 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (75m) 6m ago 75m 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:40:35.372 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (74m) 4m ago 74m 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:40:35.372 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (69m) 6m ago 77m 234M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:40:35.372 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (69m) 6m ago 77m 226M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:40:35.373 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (68m) 6m ago 77m 183M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:40:35.373 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (68m) 6m ago 77m 216M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:40:35.373 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (67m) 4m ago 76m 198M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:40:35.373 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (67m) 4m ago 76m 235M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:40:35.374 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (66m) 4m ago 76m 208M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:40:35.374 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (66m) 4m ago 75m 238M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:40:35.374 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (71m) 4m ago 75m 105M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:40:35.731 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:40:35.731 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:40:35.731 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:40:35.732 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:40:35.732 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:40:35.732 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:40:35.732 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:40:35.733 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:40:35.733 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:40:35.733 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:40:35.733 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:40:35.734 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:40:35.734 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:40:35.734 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:40:35.734 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:40:35.735 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:40:35.735 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:40:35.735 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:40:36.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:40:35 smithi083 ceph-mon[77242]: pgmap v2172: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:36.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:40:35 smithi083 ceph-mon[77242]: from='client.36663 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:40:36.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:40:35 smithi083 ceph-mon[77242]: from='client.36669 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:40:36.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:40:35 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3927607301' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:40:36.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:40:35 smithi083 ceph-mon[79930]: pgmap v2172: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:36.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:40:35 smithi083 ceph-mon[79930]: from='client.36663 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:40:36.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:40:35 smithi083 ceph-mon[79930]: from='client.36669 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:40:36.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:40:35 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3927607301' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:40:36.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:40:35 smithi153 ceph-mon[70044]: pgmap v2172: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:36.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:40:35 smithi153 ceph-mon[70044]: from='client.36663 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:40:36.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:40:35 smithi153 ceph-mon[70044]: from='client.36669 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:40:36.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:40:35 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3927607301' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:40:38.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:40:37 smithi083 ceph-mon[79930]: pgmap v2173: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:38.237 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:40:37 smithi083 ceph-mon[77242]: pgmap v2173: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:38.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:40:37 smithi153 ceph-mon[70044]: pgmap v2173: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:40:39 smithi083 ceph-mon[79930]: pgmap v2174: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:40.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:40:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:40:40.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:40:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:40:40.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:40:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:40:40.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:40:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:40:40.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:40:39 smithi083 ceph-mon[77242]: pgmap v2174: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:40.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:40:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:40:40.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:40:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:40:40.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:40:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:40:40.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:40:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:40:40.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:40:39 smithi153 ceph-mon[70044]: pgmap v2174: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:40.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:40:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:40:40.312 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:40:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:40:40.312 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:40:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:40:40.313 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:40:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:40:40.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:40:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:40:40.382Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:40:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:40:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:40:41] "GET /metrics HTTP/1.1" 200 225178 "" "Prometheus/2.33.4" 2022-09-29T06:40:41.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:40:41 smithi083 ceph-mon[79930]: pgmap v2175: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:41.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:40:41 smithi083 ceph-mon[77242]: pgmap v2175: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:42.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:40:41 smithi153 ceph-mon[70044]: pgmap v2175: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:40:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:40:42.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:40:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:40:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:40:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:40:44.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:40:43 smithi083 ceph-mon[79930]: pgmap v2176: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:44.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:40:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:40:43.784Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:40:44.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:40:43 smithi083 ceph-mon[77242]: pgmap v2176: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:44.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:40:43 smithi153 ceph-mon[70044]: pgmap v2176: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:44.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:40:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:40:44.485Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:40:46.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:40:45 smithi083 ceph-mon[79930]: pgmap v2177: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:46.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:40:45 smithi083 ceph-mon[77242]: pgmap v2177: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:46.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:40:45 smithi153 ceph-mon[70044]: pgmap v2177: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:48.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:40:47 smithi083 ceph-mon[79930]: pgmap v2178: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:48.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:40:47 smithi083 ceph-mon[77242]: pgmap v2178: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:48.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:40:47 smithi153 ceph-mon[70044]: pgmap v2178: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:50.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:40:49 smithi083 ceph-mon[77242]: pgmap v2179: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:50.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:40:49 smithi083 ceph-mon[79930]: pgmap v2179: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:50.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:40:49 smithi153 ceph-mon[70044]: pgmap v2179: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:50.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:40:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:40:50.383Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:40:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:40:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:40:51] "GET /metrics HTTP/1.1" 200 225178 "" "Prometheus/2.33.4" 2022-09-29T06:40:51.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:40:51 smithi083 ceph-mon[79930]: pgmap v2180: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:51.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:40:51 smithi083 ceph-mon[77242]: pgmap v2180: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:52.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:40:51 smithi153 ceph-mon[70044]: pgmap v2180: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:40:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:40:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:40:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:40:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:40:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:40:54.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:40:53 smithi083 ceph-mon[79930]: pgmap v2181: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:54.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:40:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:40:53.784Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:40:54.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:40:53 smithi083 ceph-mon[77242]: pgmap v2181: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:54.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:40:53 smithi153 ceph-mon[70044]: pgmap v2181: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:54.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:40:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:40:54.486Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:40:56.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:40:55 smithi083 ceph-mon[79930]: pgmap v2182: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:56.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:40:55 smithi083 ceph-mon[77242]: pgmap v2182: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:56.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:40:55 smithi153 ceph-mon[70044]: pgmap v2182: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:58.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:40:57 smithi083 ceph-mon[77242]: pgmap v2183: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:58.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:40:57 smithi083 ceph-mon[79930]: pgmap v2183: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:40:58.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:40:57 smithi153 ceph-mon[70044]: pgmap v2183: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:00.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:40:59 smithi083 ceph-mon[79930]: pgmap v2184: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:00.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:40:59 smithi083 ceph-mon[77242]: pgmap v2184: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:40:59 smithi153 ceph-mon[70044]: pgmap v2184: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:00.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:41:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:41:00.384Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:41:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:41:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:41:01] "GET /metrics HTTP/1.1" 200 225179 "" "Prometheus/2.33.4" 2022-09-29T06:41:01.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:41:01 smithi083 ceph-mon[79930]: pgmap v2185: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:01.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:41:01 smithi083 ceph-mon[77242]: pgmap v2185: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:02.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:41:01 smithi153 ceph-mon[70044]: pgmap v2185: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:41:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:41:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:41:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:41:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:41:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:41:04.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:41:03 smithi083 ceph-mon[79930]: pgmap v2186: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:04.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:41:03 smithi083 ceph-mon[77242]: pgmap v2186: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:04.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:41:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:41:03.785Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:41:04.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:41:03 smithi153 ceph-mon[70044]: pgmap v2186: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:04.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:41:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:41:04.486Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:41:06.058 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:41:06.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:41:05 smithi083 ceph-mon[77242]: pgmap v2187: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:06.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:41:05 smithi083 ceph-mon[79930]: pgmap v2187: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:06.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:41:05 smithi153 ceph-mon[70044]: pgmap v2187: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:06.346 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:41:06.347 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (74m) 6m ago 75m 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:41:06.347 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (75m) 4m ago 75m 61.1M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:41:06.348 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (76m) 6m ago 76m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:41:06.348 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (76m) 4m ago 76m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:41:06.348 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (71m) 4m ago 78m 439M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:41:06.349 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (72m) 6m ago 80m 570M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:41:06.349 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (71m) 6m ago 80m 310M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:41:06.350 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (70m) 4m ago 78m 147M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:41:06.350 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (71m) 6m ago 79m 186M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:41:06.351 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (75m) 6m ago 75m 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:41:06.351 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (75m) 4m ago 75m 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:41:06.351 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (70m) 6m ago 78m 234M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:41:06.352 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (69m) 6m ago 78m 226M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:41:06.352 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (69m) 6m ago 77m 183M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:41:06.352 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (68m) 6m ago 77m 216M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:41:06.353 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (68m) 4m ago 77m 198M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:41:06.353 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (67m) 4m ago 76m 235M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:41:06.354 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (67m) 4m ago 76m 208M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:41:06.354 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (66m) 4m ago 76m 238M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:41:06.354 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (71m) 4m ago 75m 105M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:41:06.702 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:41:06.703 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:41:06.703 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:41:06.703 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:41:06.703 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:41:06.704 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:41:06.704 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:41:06.704 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:41:06.704 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:41:06.704 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:41:06.705 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:41:06.705 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:41:06.705 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:41:06.705 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:41:06.706 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:41:06.706 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:41:06.706 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:41:06.706 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:41:07.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:41:06 smithi153 ceph-mon[70044]: from='client.36681 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:41:07.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:41:06 smithi153 ceph-mon[70044]: from='client.46621 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:41:07.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:41:06 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2854936080' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:41:07.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:41:06 smithi083 ceph-mon[79930]: from='client.36681 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:41:07.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:41:06 smithi083 ceph-mon[79930]: from='client.46621 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:41:07.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:41:06 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2854936080' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:41:07.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:41:06 smithi083 ceph-mon[77242]: from='client.36681 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:41:07.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:41:06 smithi083 ceph-mon[77242]: from='client.46621 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:41:07.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:41:06 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2854936080' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:41:08.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:41:07 smithi153 ceph-mon[70044]: pgmap v2188: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:08.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:41:07 smithi083 ceph-mon[79930]: pgmap v2188: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:08.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:41:07 smithi083 ceph-mon[77242]: pgmap v2188: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:10.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:41:09 smithi153 ceph-mon[70044]: pgmap v2189: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:10.385 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:41:09 smithi083 ceph-mon[79930]: pgmap v2189: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:10.386 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:41:09 smithi083 ceph-mon[77242]: pgmap v2189: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:10.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:41:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:41:10.384Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:41:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:41:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:41:11] "GET /metrics HTTP/1.1" 200 225181 "" "Prometheus/2.33.4" 2022-09-29T06:41:11.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:41:11 smithi083 ceph-mon[79930]: pgmap v2190: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:11.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:41:11 smithi083 ceph-mon[77242]: pgmap v2190: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:12.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:41:11 smithi153 ceph-mon[70044]: pgmap v2190: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:41:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:41:12.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:41:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:41:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:41:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:41:14.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:41:13 smithi083 ceph-mon[79930]: pgmap v2191: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:14.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:41:13 smithi083 ceph-mon[77242]: pgmap v2191: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:14.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:41:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:41:13.785Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:41:14.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:41:13 smithi153 ceph-mon[70044]: pgmap v2191: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:14.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:41:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:41:14.487Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:41:16.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:41:15 smithi153 ceph-mon[70044]: pgmap v2192: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:16.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:41:15 smithi083 ceph-mon[79930]: pgmap v2192: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:16.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:41:15 smithi083 ceph-mon[77242]: pgmap v2192: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:18.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:41:17 smithi153 ceph-mon[70044]: pgmap v2193: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:18.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:41:17 smithi083 ceph-mon[77242]: pgmap v2193: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:18.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:41:17 smithi083 ceph-mon[79930]: pgmap v2193: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:41:19 smithi153 ceph-mon[70044]: pgmap v2194: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:20.386 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:41:19 smithi083 ceph-mon[79930]: pgmap v2194: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:20.386 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:41:19 smithi083 ceph-mon[77242]: pgmap v2194: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:20.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:41:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:41:20.384Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:41:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:41:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:41:21] "GET /metrics HTTP/1.1" 200 225181 "" "Prometheus/2.33.4" 2022-09-29T06:41:21.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:41:21 smithi083 ceph-mon[79930]: pgmap v2195: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:21.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:41:21 smithi083 ceph-mon[77242]: pgmap v2195: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:22.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:41:21 smithi153 ceph-mon[70044]: pgmap v2195: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:41:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:41:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:41:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:41:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:41:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:41:24.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:41:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:41:23.784Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:41:24.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:41:23 smithi083 ceph-mon[77242]: pgmap v2196: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:24.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:41:23 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:41:24.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:41:23 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:41:24.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:41:23 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:41:24.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:41:23 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:41:24.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:41:23 smithi083 ceph-mon[79930]: pgmap v2196: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:24.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:41:23 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:41:24.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:41:23 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:41:24.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:41:23 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:41:24.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:41:23 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:41:24.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:41:23 smithi153 ceph-mon[70044]: pgmap v2196: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:24.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:41:23 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:41:24.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:41:23 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:41:24.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:41:23 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:41:24.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:41:23 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:41:24.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:41:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:41:24.487Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:41:26.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:41:25 smithi153 ceph-mon[70044]: pgmap v2197: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:26.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:41:25 smithi083 ceph-mon[79930]: pgmap v2197: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:26.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:41:25 smithi083 ceph-mon[77242]: pgmap v2197: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:28.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:41:27 smithi153 ceph-mon[70044]: pgmap v2198: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:28.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:41:27 smithi083 ceph-mon[79930]: pgmap v2198: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:28.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:41:27 smithi083 ceph-mon[77242]: pgmap v2198: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:30.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:41:29 smithi153 ceph-mon[70044]: pgmap v2199: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:30.386 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:41:29 smithi083 ceph-mon[79930]: pgmap v2199: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:30.386 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:41:29 smithi083 ceph-mon[77242]: pgmap v2199: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:30.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:41:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:41:30.384Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:41:31.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:41:31 smithi083 ceph-mon[77242]: pgmap v2200: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:41:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:41:31] "GET /metrics HTTP/1.1" 200 225180 "" "Prometheus/2.33.4" 2022-09-29T06:41:31.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:41:31 smithi083 ceph-mon[79930]: pgmap v2200: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:32.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:41:31 smithi153 ceph-mon[70044]: pgmap v2200: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:41:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:41:32.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:41:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:41:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:41:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:41:34.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:41:33 smithi083 ceph-mon[79930]: pgmap v2201: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:34.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:41:33 smithi083 ceph-mon[77242]: pgmap v2201: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:34.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:41:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:41:33.785Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:41:34.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:41:33 smithi153 ceph-mon[70044]: pgmap v2201: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:34.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:41:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:41:34.489Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:41:36.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:41:35 smithi153 ceph-mon[70044]: pgmap v2202: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:36.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:41:35 smithi083 ceph-mon[77242]: pgmap v2202: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:36.579 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:41:35 smithi083 ceph-mon[79930]: pgmap v2202: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:37.031 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:41:37.318 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:41:37.319 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (75m) 7m ago 75m 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:41:37.319 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (75m) 5m ago 75m 61.1M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:41:37.319 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (76m) 7m ago 76m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:41:37.320 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (76m) 5m ago 76m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:41:37.320 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (72m) 5m ago 79m 439M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:41:37.320 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (72m) 7m ago 80m 570M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:41:37.321 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (72m) 7m ago 80m 310M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:41:37.321 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (71m) 5m ago 79m 147M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:41:37.321 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (71m) 7m ago 79m 186M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:41:37.321 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (76m) 7m ago 76m 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:41:37.322 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (75m) 5m ago 75m 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:41:37.322 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (70m) 7m ago 78m 234M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:41:37.322 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (70m) 7m ago 78m 226M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:41:37.322 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (69m) 7m ago 78m 183M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:41:37.323 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (69m) 7m ago 78m 216M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:41:37.323 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (68m) 5m ago 77m 198M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:41:37.323 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (68m) 5m ago 77m 235M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:41:37.324 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (67m) 5m ago 77m 208M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:41:37.324 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (67m) 5m ago 76m 238M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:41:37.324 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (72m) 5m ago 76m 105M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:41:37.671 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:41:37.672 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:41:37.672 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:41:37.673 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:41:37.673 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:41:37.674 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:41:37.674 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:41:37.675 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:41:37.675 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:41:37.676 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:41:37.676 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:41:37.676 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:41:37.677 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:41:37.677 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:41:37.678 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:41:37.678 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:41:37.679 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:41:37.679 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:41:38.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:41:37 smithi153 ceph-mon[70044]: pgmap v2203: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:38.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:41:37 smithi153 ceph-mon[70044]: from='client.46633 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:41:38.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:41:37 smithi153 ceph-mon[70044]: from='client.46639 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:41:38.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:41:37 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4166134640' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:41:38.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:41:37 smithi083 ceph-mon[79930]: pgmap v2203: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:38.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:41:37 smithi083 ceph-mon[79930]: from='client.46633 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:41:38.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:41:37 smithi083 ceph-mon[79930]: from='client.46639 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:41:38.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:41:37 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4166134640' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:41:38.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:41:37 smithi083 ceph-mon[77242]: pgmap v2203: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:38.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:41:37 smithi083 ceph-mon[77242]: from='client.46633 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:41:38.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:41:37 smithi083 ceph-mon[77242]: from='client.46639 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:41:38.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:41:37 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4166134640' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:41:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:41:39 smithi153 ceph-mon[70044]: pgmap v2204: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:41:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:41:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:41:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:41:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:41:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:41:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:41:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:41:40.387 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:41:39 smithi083 ceph-mon[79930]: pgmap v2204: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:40.387 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:41:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:41:40.387 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:41:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:41:40.388 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:41:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:41:40.388 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:41:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:41:40.388 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:41:39 smithi083 ceph-mon[77242]: pgmap v2204: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:40.388 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:41:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:41:40.389 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:41:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:41:40.389 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:41:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:41:40.389 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:41:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:41:40.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:41:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:41:40.385Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:41:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:41:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:41:41] "GET /metrics HTTP/1.1" 200 225179 "" "Prometheus/2.33.4" 2022-09-29T06:41:41.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:41:41 smithi083 ceph-mon[79930]: pgmap v2205: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:41.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:41:41 smithi083 ceph-mon[77242]: pgmap v2205: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:42.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:41:41 smithi153 ceph-mon[70044]: pgmap v2205: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:41:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:41:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:41:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:41:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:41:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:41:44.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:41:43 smithi083 ceph-mon[79930]: pgmap v2206: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:44.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:41:43 smithi083 ceph-mon[77242]: pgmap v2206: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:44.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:41:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:41:43.786Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:41:44.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:41:43 smithi153 ceph-mon[70044]: pgmap v2206: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:44.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:41:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:41:44.489Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:41:46.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:41:45 smithi153 ceph-mon[70044]: pgmap v2207: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:46.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:41:45 smithi083 ceph-mon[79930]: pgmap v2207: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:46.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:41:45 smithi083 ceph-mon[77242]: pgmap v2207: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:48.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:41:47 smithi153 ceph-mon[70044]: pgmap v2208: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:48.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:41:47 smithi083 ceph-mon[79930]: pgmap v2208: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:48.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:41:47 smithi083 ceph-mon[77242]: pgmap v2208: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:50.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:41:49 smithi153 ceph-mon[70044]: pgmap v2209: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:50.388 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:41:49 smithi083 ceph-mon[79930]: pgmap v2209: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:50.388 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:41:49 smithi083 ceph-mon[77242]: pgmap v2209: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:50.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:41:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:41:50.386Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:41:51.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:41:51 smithi083 ceph-mon[79930]: pgmap v2210: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:41:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:41:51] "GET /metrics HTTP/1.1" 200 225179 "" "Prometheus/2.33.4" 2022-09-29T06:41:51.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:41:51 smithi083 ceph-mon[77242]: pgmap v2210: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:52.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:41:51 smithi153 ceph-mon[70044]: pgmap v2210: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:41:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:41:52.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:41:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:41:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:41:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:41:54.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:41:53 smithi083 ceph-mon[79930]: pgmap v2211: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:54.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:41:53 smithi083 ceph-mon[77242]: pgmap v2211: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:54.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:41:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:41:53.787Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:41:54.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:41:53 smithi153 ceph-mon[70044]: pgmap v2211: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:54.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:41:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:41:54.490Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:41:56.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:41:55 smithi153 ceph-mon[70044]: pgmap v2212: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:56.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:41:55 smithi083 ceph-mon[77242]: pgmap v2212: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:56.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:41:55 smithi083 ceph-mon[79930]: pgmap v2212: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:58.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:41:57 smithi153 ceph-mon[70044]: pgmap v2213: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:58.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:41:57 smithi083 ceph-mon[79930]: pgmap v2213: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:41:58.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:41:57 smithi083 ceph-mon[77242]: pgmap v2213: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:41:59 smithi153 ceph-mon[70044]: pgmap v2214: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:00.388 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:41:59 smithi083 ceph-mon[79930]: pgmap v2214: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:00.388 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:41:59 smithi083 ceph-mon[77242]: pgmap v2214: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:00.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:42:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:42:00.387Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:42:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:42:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:42:01] "GET /metrics HTTP/1.1" 200 225177 "" "Prometheus/2.33.4" 2022-09-29T06:42:01.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:42:01 smithi083 ceph-mon[79930]: pgmap v2215: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:01.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:42:01 smithi083 ceph-mon[77242]: pgmap v2215: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:02.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:42:01 smithi153 ceph-mon[70044]: pgmap v2215: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:42:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:42:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:42:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:42:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:42:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:42:04.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:42:03 smithi083 ceph-mon[79930]: pgmap v2216: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:04.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:42:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:42:03.788Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:42:04.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:42:03 smithi083 ceph-mon[77242]: pgmap v2216: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:04.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:42:03 smithi153 ceph-mon[70044]: pgmap v2216: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:04.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:42:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:42:04.491Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:42:06.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:42:05 smithi153 ceph-mon[70044]: pgmap v2217: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:06.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:42:05 smithi083 ceph-mon[79930]: pgmap v2217: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:06.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:42:05 smithi083 ceph-mon[77242]: pgmap v2217: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:08.000 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:42:08.288 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:42:08.288 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (75m) 8m ago 76m 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:42:08.289 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (76m) 5m ago 76m 61.1M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:42:08.289 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (77m) 8m ago 77m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:42:08.289 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (77m) 5m ago 77m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:42:08.289 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (72m) 5m ago 79m 439M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:42:08.290 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (73m) 8m ago 81m 570M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:42:08.290 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (72m) 8m ago 81m 310M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:42:08.290 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (72m) 5m ago 79m 147M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:42:08.290 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (72m) 8m ago 80m 186M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:42:08.291 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (76m) 8m ago 76m 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:42:08.291 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (76m) 5m ago 76m 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:42:08.291 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (71m) 8m ago 79m 234M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:42:08.291 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (70m) 8m ago 79m 226M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:42:08.291 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (70m) 8m ago 78m 183M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:42:08.292 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (69m) 8m ago 78m 216M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:42:08.292 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (69m) 5m ago 78m 198M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:42:08.292 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (68m) 5m ago 78m 235M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:42:08.292 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (68m) 5m ago 77m 208M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:42:08.293 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (67m) 5m ago 77m 238M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:42:08.293 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (72m) 5m ago 76m 105M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:42:08.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:42:07 smithi153 ceph-mon[70044]: pgmap v2218: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:08.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:42:07 smithi083 ceph-mon[79930]: pgmap v2218: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:08.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:42:07 smithi083 ceph-mon[77242]: pgmap v2218: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:08.645 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:42:08.645 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:42:08.645 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:42:08.646 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:42:08.646 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:42:08.646 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:42:08.646 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:42:08.647 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:42:08.647 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:42:08.647 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:42:08.647 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:42:08.647 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:42:08.648 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:42:08.648 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:42:08.648 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:42:08.648 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:42:08.649 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:42:08.649 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:42:09.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:42:08 smithi153 ceph-mon[70044]: from='client.36717 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:42:09.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:42:08 smithi153 ceph-mon[70044]: from='client.36723 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:42:09.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:42:08 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2372451592' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:42:09.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:42:08 smithi083 ceph-mon[79930]: from='client.36717 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:42:09.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:42:08 smithi083 ceph-mon[79930]: from='client.36723 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:42:09.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:42:08 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2372451592' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:42:09.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:42:08 smithi083 ceph-mon[77242]: from='client.36717 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:42:09.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:42:08 smithi083 ceph-mon[77242]: from='client.36723 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:42:09.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:42:08 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2372451592' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:42:10.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:42:09 smithi153 ceph-mon[70044]: pgmap v2219: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:10.388 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:42:09 smithi083 ceph-mon[79930]: pgmap v2219: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:10.389 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:42:09 smithi083 ceph-mon[77242]: pgmap v2219: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:10.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:42:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:42:10.387Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:42:11.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:42:11 smithi083 ceph-mon[77242]: pgmap v2220: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:42:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:42:11] "GET /metrics HTTP/1.1" 200 225175 "" "Prometheus/2.33.4" 2022-09-29T06:42:11.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:42:11 smithi083 ceph-mon[79930]: pgmap v2220: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:12.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:42:11 smithi153 ceph-mon[70044]: pgmap v2220: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:42:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:42:12.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:42:13.314 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:42:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:42:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:42:14.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:42:13 smithi083 ceph-mon[79930]: pgmap v2221: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:14.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:42:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:42:13.789Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:42:14.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:42:13 smithi083 ceph-mon[77242]: pgmap v2221: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:14.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:42:13 smithi153 ceph-mon[70044]: pgmap v2221: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:14.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:42:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:42:14.492Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:42:16.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:42:15 smithi153 ceph-mon[70044]: pgmap v2222: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:16.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:42:15 smithi083 ceph-mon[79930]: pgmap v2222: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:16.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:42:15 smithi083 ceph-mon[77242]: pgmap v2222: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:18.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:42:17 smithi153 ceph-mon[70044]: pgmap v2223: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:18.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:42:17 smithi083 ceph-mon[79930]: pgmap v2223: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:18.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:42:17 smithi083 ceph-mon[77242]: pgmap v2223: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:42:19 smithi153 ceph-mon[70044]: pgmap v2224: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:20.389 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:42:19 smithi083 ceph-mon[79930]: pgmap v2224: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:20.390 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:42:19 smithi083 ceph-mon[77242]: pgmap v2224: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:20.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:42:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:42:20.388Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:42:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:42:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:42:21] "GET /metrics HTTP/1.1" 200 225175 "" "Prometheus/2.33.4" 2022-09-29T06:42:21.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:42:21 smithi083 ceph-mon[77242]: pgmap v2225: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:21.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:42:21 smithi083 ceph-mon[79930]: pgmap v2225: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:22.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:42:21 smithi153 ceph-mon[70044]: pgmap v2225: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:42:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:42:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:42:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:42:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:42:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:42:24.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:42:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:42:23.790Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:42:24.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:42:23 smithi083 ceph-mon[79930]: pgmap v2226: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:24.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:42:23 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:42:24.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:42:23 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:42:24.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:42:23 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:42:24.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:42:23 smithi083 ceph-mon[77242]: pgmap v2226: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:24.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:42:23 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:42:24.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:42:23 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:42:24.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:42:23 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:42:24.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:42:23 smithi153 ceph-mon[70044]: pgmap v2226: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:24.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:42:23 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:42:24.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:42:23 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:42:24.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:42:23 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:42:24.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:42:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:42:24.492Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:42:25.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:42:25 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:42:25.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:42:25 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:42:25.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:42:25 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:42:26.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:42:26 smithi083 ceph-mon[79930]: pgmap v2227: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:26.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:42:26 smithi083 ceph-mon[77242]: pgmap v2227: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:26.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:42:26 smithi153 ceph-mon[70044]: pgmap v2227: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:28.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:42:28 smithi083 ceph-mon[79930]: pgmap v2228: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:28.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:42:28 smithi083 ceph-mon[77242]: pgmap v2228: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:28.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:42:28 smithi153 ceph-mon[70044]: pgmap v2228: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:29.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:42:29 smithi083 ceph-mon[79930]: pgmap v2229: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:29.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:42:29 smithi083 ceph-mon[77242]: pgmap v2229: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:29.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:42:29 smithi153 ceph-mon[70044]: pgmap v2229: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:30.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:42:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:42:30.388Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:42:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:42:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:42:31] "GET /metrics HTTP/1.1" 200 225176 "" "Prometheus/2.33.4" 2022-09-29T06:42:31.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:42:31 smithi083 ceph-mon[79930]: pgmap v2230: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:31.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:42:31 smithi083 ceph-mon[77242]: pgmap v2230: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:32.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:42:31 smithi153 ceph-mon[70044]: pgmap v2230: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:42:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:42:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:42:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:42:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:42:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:42:34.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:42:33 smithi083 ceph-mon[79930]: pgmap v2231: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:34.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:42:33 smithi083 ceph-mon[77242]: pgmap v2231: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:34.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:42:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:42:33.790Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:42:34.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:42:33 smithi153 ceph-mon[70044]: pgmap v2231: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:34.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:42:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:42:34.494Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:42:36.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:42:35 smithi153 ceph-mon[70044]: pgmap v2232: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:36.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:42:35 smithi083 ceph-mon[77242]: pgmap v2232: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:36.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:42:35 smithi083 ceph-mon[79930]: pgmap v2232: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:38.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:42:37 smithi153 ceph-mon[70044]: pgmap v2233: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:38.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:42:37 smithi083 ceph-mon[79930]: pgmap v2233: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:38.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:42:37 smithi083 ceph-mon[77242]: pgmap v2233: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:38.974 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:42:39.267 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:42:39.267 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (76m) 8m ago 76m 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:42:39.268 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (76m) 6m ago 76m 61.1M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:42:39.268 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (77m) 8m ago 77m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:42:39.268 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (77m) 6m ago 77m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:42:39.268 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (73m) 6m ago 80m 439M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:42:39.268 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (73m) 8m ago 81m 570M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:42:39.269 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (73m) 8m ago 81m 310M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:42:39.269 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (72m) 6m ago 80m 147M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:42:39.269 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (72m) 8m ago 80m 186M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:42:39.269 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (77m) 8m ago 77m 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:42:39.270 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (77m) 6m ago 77m 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:42:39.270 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (71m) 8m ago 79m 234M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:42:39.270 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (71m) 8m ago 79m 226M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:42:39.270 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (70m) 8m ago 79m 183M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:42:39.271 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (70m) 8m ago 79m 216M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:42:39.271 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (69m) 6m ago 78m 198M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:42:39.271 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (69m) 6m ago 78m 235M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:42:39.271 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (68m) 6m ago 78m 208M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:42:39.272 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (68m) 6m ago 77m 238M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:42:39.272 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (73m) 6m ago 77m 105M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:42:39.625 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:42:39.626 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:42:39.626 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:42:39.627 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:42:39.627 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:42:39.628 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:42:39.628 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:42:39.629 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:42:39.629 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:42:39.630 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:42:39.630 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:42:39.631 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:42:39.631 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:42:39.631 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:42:39.632 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:42:39.632 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:42:39.633 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:42:39.633 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:42:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:42:39 smithi153 ceph-mon[70044]: pgmap v2234: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:42:39 smithi153 ceph-mon[70044]: from='client.36735 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:42:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:42:39 smithi153 ceph-mon[70044]: from='client.36741 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:42:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:42:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:42:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:42:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:42:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:42:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:42:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:42:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:42:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:42:39 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4019852276' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:42:40.391 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:42:39 smithi083 ceph-mon[79930]: pgmap v2234: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:40.391 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:42:39 smithi083 ceph-mon[79930]: from='client.36735 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:42:40.392 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:42:39 smithi083 ceph-mon[79930]: from='client.36741 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:42:40.392 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:42:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:42:40.392 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:42:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:42:40.392 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:42:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:42:40.392 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:42:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:42:40.393 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:42:39 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4019852276' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:42:40.393 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:42:39 smithi083 ceph-mon[77242]: pgmap v2234: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:40.393 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:42:39 smithi083 ceph-mon[77242]: from='client.36735 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:42:40.394 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:42:39 smithi083 ceph-mon[77242]: from='client.36741 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:42:40.394 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:42:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:42:40.394 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:42:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:42:40.395 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:42:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:42:40.395 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:42:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:42:40.395 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:42:39 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4019852276' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:42:40.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:42:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:42:40.389Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:42:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:42:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:42:41] "GET /metrics HTTP/1.1" 200 225176 "" "Prometheus/2.33.4" 2022-09-29T06:42:41.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:42:41 smithi083 ceph-mon[79930]: pgmap v2235: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:41.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:42:41 smithi083 ceph-mon[77242]: pgmap v2235: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:42.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:42:41 smithi153 ceph-mon[70044]: pgmap v2235: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:42:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:42:42.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:42:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:42:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:42:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:42:44.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:42:43 smithi083 ceph-mon[79930]: pgmap v2236: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:44.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:42:43 smithi083 ceph-mon[77242]: pgmap v2236: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:44.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:42:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:42:43.790Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:42:44.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:42:43 smithi153 ceph-mon[70044]: pgmap v2236: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:44.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:42:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:42:44.494Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:42:46.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:42:45 smithi153 ceph-mon[70044]: pgmap v2237: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:46.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:42:45 smithi083 ceph-mon[79930]: pgmap v2237: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:46.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:42:45 smithi083 ceph-mon[77242]: pgmap v2237: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:48.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:42:47 smithi153 ceph-mon[70044]: pgmap v2238: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:48.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:42:47 smithi083 ceph-mon[79930]: pgmap v2238: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:48.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:42:47 smithi083 ceph-mon[77242]: pgmap v2238: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:50.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:42:49 smithi153 ceph-mon[70044]: pgmap v2239: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:50.392 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:42:49 smithi083 ceph-mon[79930]: pgmap v2239: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:50.392 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:42:49 smithi083 ceph-mon[77242]: pgmap v2239: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:50.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:42:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:42:50.390Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:42:51.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:42:51 smithi083 ceph-mon[79930]: pgmap v2240: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:42:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:42:51] "GET /metrics HTTP/1.1" 200 225176 "" "Prometheus/2.33.4" 2022-09-29T06:42:51.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:42:51 smithi083 ceph-mon[77242]: pgmap v2240: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:52.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:42:51 smithi153 ceph-mon[70044]: pgmap v2240: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:42:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:42:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:42:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:42:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:42:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:42:54.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:42:53 smithi083 ceph-mon[77242]: pgmap v2241: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:54.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:42:53 smithi083 ceph-mon[79930]: pgmap v2241: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:54.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:42:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:42:53.791Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:42:54.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:42:53 smithi153 ceph-mon[70044]: pgmap v2241: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:54.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:42:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:42:54.494Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:42:56.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:42:55 smithi153 ceph-mon[70044]: pgmap v2242: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:56.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:42:55 smithi083 ceph-mon[79930]: pgmap v2242: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:56.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:42:55 smithi083 ceph-mon[77242]: pgmap v2242: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:58.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:42:57 smithi153 ceph-mon[70044]: pgmap v2243: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:58.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:42:57 smithi083 ceph-mon[77242]: pgmap v2243: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:42:58.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:42:57 smithi083 ceph-mon[79930]: pgmap v2243: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:42:59 smithi153 ceph-mon[70044]: pgmap v2244: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:00.392 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:42:59 smithi083 ceph-mon[79930]: pgmap v2244: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:00.393 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:42:59 smithi083 ceph-mon[77242]: pgmap v2244: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:00.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:43:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:43:00.391Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:43:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:43:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:43:01] "GET /metrics HTTP/1.1" 200 225177 "" "Prometheus/2.33.4" 2022-09-29T06:43:01.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:43:01 smithi083 ceph-mon[79930]: pgmap v2245: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:01.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:43:01 smithi083 ceph-mon[77242]: pgmap v2245: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:02.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:43:01 smithi153 ceph-mon[70044]: pgmap v2245: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:43:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:43:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:43:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:43:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:43:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:43:04.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:43:03 smithi083 ceph-mon[77242]: pgmap v2246: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:04.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:43:03 smithi083 ceph-mon[79930]: pgmap v2246: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:04.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:43:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:43:03.792Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:43:04.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:43:03 smithi153 ceph-mon[70044]: pgmap v2246: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:04.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:43:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:43:04.495Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:43:06.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:43:05 smithi153 ceph-mon[70044]: pgmap v2247: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:06.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:43:05 smithi083 ceph-mon[79930]: pgmap v2247: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:06.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:43:05 smithi083 ceph-mon[77242]: pgmap v2247: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:08.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:43:07 smithi153 ceph-mon[70044]: pgmap v2248: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:08.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:43:07 smithi083 ceph-mon[79930]: pgmap v2248: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:08.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:43:07 smithi083 ceph-mon[77242]: pgmap v2248: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:09.954 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:43:10.243 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:43:10.244 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (76m) 9m ago 77m 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:43:10.244 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (77m) 6m ago 77m 61.1M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:43:10.245 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (78m) 9m ago 78m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:43:10.245 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (78m) 6m ago 78m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:43:10.245 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (73m) 6m ago 80m 439M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:43:10.245 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (74m) 9m ago 82m 570M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:43:10.245 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (73m) 9m ago 82m 310M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:43:10.246 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (73m) 6m ago 80m 147M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:43:10.246 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (73m) 9m ago 81m 186M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:43:10.246 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (77m) 9m ago 77m 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:43:10.247 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (77m) 6m ago 77m 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:43:10.247 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (72m) 9m ago 80m 234M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:43:10.247 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (71m) 9m ago 80m 226M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:43:10.247 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (71m) 9m ago 79m 183M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:43:10.248 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (70m) 9m ago 79m 216M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:43:10.248 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (70m) 6m ago 79m 198M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:43:10.248 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (69m) 6m ago 79m 235M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:43:10.249 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (69m) 6m ago 78m 208M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:43:10.249 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (68m) 6m ago 78m 238M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:43:10.249 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (73m) 6m ago 77m 105M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:43:10.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:43:09 smithi153 ceph-mon[70044]: pgmap v2249: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:10.392 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:43:09 smithi083 ceph-mon[79930]: pgmap v2249: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:10.392 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:43:09 smithi083 ceph-mon[77242]: pgmap v2249: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:10.606 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:43:10.607 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:43:10.607 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:43:10.607 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:43:10.608 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:43:10.608 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:43:10.608 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:43:10.608 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:43:10.609 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:43:10.609 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:43:10.609 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:43:10.609 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:43:10.610 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:43:10.610 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:43:10.610 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:43:10.610 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:43:10.611 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:43:10.611 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:43:10.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:43:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:43:10.390Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:43:11.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:43:11 smithi153 ceph-mon[70044]: from='client.46687 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:43:11.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:43:11 smithi153 ceph-mon[70044]: from='client.36759 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:43:11.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:43:11 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3983734253' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:43:11.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:43:11 smithi083 ceph-mon[79930]: from='client.46687 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:43:11.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:43:11 smithi083 ceph-mon[79930]: from='client.36759 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:43:11.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:43:11 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3983734253' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:43:11.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:43:10 smithi083 ceph-mon[77242]: from='client.46687 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:43:11.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:43:10 smithi083 ceph-mon[77242]: from='client.36759 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:43:11.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:43:10 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3983734253' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:43:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:43:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:43:11] "GET /metrics HTTP/1.1" 200 225178 "" "Prometheus/2.33.4" 2022-09-29T06:43:12.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:43:12 smithi153 ceph-mon[70044]: pgmap v2250: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:12.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:43:12 smithi083 ceph-mon[79930]: pgmap v2250: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:12.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:43:12 smithi083 ceph-mon[77242]: pgmap v2250: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:43:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:43:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:43:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:43:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:43:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:43:14.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:43:14 smithi083 ceph-mon[79930]: pgmap v2251: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:14.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:43:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:43:13.792Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:43:14.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:43:14 smithi083 ceph-mon[77242]: pgmap v2251: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:14.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:43:14 smithi153 ceph-mon[70044]: pgmap v2251: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:14.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:43:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:43:14.496Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:43:16.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:43:16 smithi153 ceph-mon[70044]: pgmap v2252: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:16.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:43:16 smithi083 ceph-mon[79930]: pgmap v2252: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:16.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:43:16 smithi083 ceph-mon[77242]: pgmap v2252: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:18.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:43:18 smithi153 ceph-mon[70044]: pgmap v2253: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:18.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:43:18 smithi083 ceph-mon[79930]: pgmap v2253: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:18.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:43:18 smithi083 ceph-mon[77242]: pgmap v2253: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:43:20 smithi153 ceph-mon[70044]: pgmap v2254: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:20.393 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:43:20 smithi083 ceph-mon[79930]: pgmap v2254: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:20.394 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:43:20 smithi083 ceph-mon[77242]: pgmap v2254: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:20.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:43:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:43:20.392Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:43:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:43:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:43:21] "GET /metrics HTTP/1.1" 200 225178 "" "Prometheus/2.33.4" 2022-09-29T06:43:21.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:43:21 smithi083 ceph-mon[79930]: pgmap v2255: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:21.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:43:21 smithi083 ceph-mon[77242]: pgmap v2255: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:22.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:43:21 smithi153 ceph-mon[70044]: pgmap v2255: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:43:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:43:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:43:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:43:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:43:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:43:24.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:43:23 smithi083 ceph-mon[79930]: pgmap v2256: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:24.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:43:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:43:23.793Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:43:24.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:43:23 smithi083 ceph-mon[77242]: pgmap v2256: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:24.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:43:23 smithi153 ceph-mon[70044]: pgmap v2256: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:24.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:43:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:43:24.496Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:43:25.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:43:25 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:43:25.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:43:25 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:43:25.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:43:25 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:43:25.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:43:25 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:43:25.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:43:25 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:43:25.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:43:25 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:43:25.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:43:25 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:43:25.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:43:25 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:43:25.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:43:25 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:43:26.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:43:26 smithi083 ceph-mon[79930]: pgmap v2257: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:26.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:43:26 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:43:26.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:43:26 smithi083 ceph-mon[77242]: pgmap v2257: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:26.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:43:26 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:43:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:43:26 smithi153 ceph-mon[70044]: pgmap v2257: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:26.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:43:26 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:43:28.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:43:28 smithi083 ceph-mon[79930]: pgmap v2258: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:28.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:43:28 smithi083 ceph-mon[77242]: pgmap v2258: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:43:28 smithi153 ceph-mon[70044]: pgmap v2258: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:30.394 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:43:30 smithi083 ceph-mon[79930]: pgmap v2259: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:30.395 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:43:30 smithi083 ceph-mon[77242]: pgmap v2259: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:43:30 smithi153 ceph-mon[70044]: pgmap v2259: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:30.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:43:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:43:30.392Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:43:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:43:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:43:31] "GET /metrics HTTP/1.1" 200 225176 "" "Prometheus/2.33.4" 2022-09-29T06:43:31.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:43:31 smithi083 ceph-mon[79930]: pgmap v2260: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:31.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:43:31 smithi083 ceph-mon[77242]: pgmap v2260: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:32.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:43:31 smithi153 ceph-mon[70044]: pgmap v2260: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:43:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:43:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:43:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:43:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:43:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:43:34.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:43:33 smithi083 ceph-mon[79930]: pgmap v2261: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:34.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:43:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:43:33.794Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:43:34.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:43:33 smithi083 ceph-mon[77242]: pgmap v2261: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:34.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:43:33 smithi153 ceph-mon[70044]: pgmap v2261: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:34.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:43:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:43:34.496Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:43:36.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:43:36 smithi153 ceph-mon[70044]: pgmap v2262: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:36.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:43:36 smithi083 ceph-mon[79930]: pgmap v2262: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:36.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:43:36 smithi083 ceph-mon[77242]: pgmap v2262: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:38.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:43:38 smithi153 ceph-mon[70044]: pgmap v2263: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:38.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:43:38 smithi083 ceph-mon[79930]: pgmap v2263: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:38.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:43:38 smithi083 ceph-mon[77242]: pgmap v2263: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:43:40 smithi153 ceph-mon[70044]: pgmap v2264: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:43:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:43:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:43:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:43:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:43:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:43:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:43:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:43:40.395 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:43:40 smithi083 ceph-mon[79930]: pgmap v2264: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:40.395 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:43:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:43:40.395 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:43:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:43:40.395 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:43:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:43:40.396 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:43:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:43:40.396 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:43:40 smithi083 ceph-mon[77242]: pgmap v2264: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:40.396 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:43:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:43:40.397 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:43:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:43:40.397 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:43:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:43:40.397 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:43:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:43:40.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:43:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:43:40.393Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:43:40.934 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:43:41.225 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:43:41.226 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (77m) 9m ago 77m 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:43:41.226 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (77m) 7m ago 77m 61.1M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:43:41.226 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (78m) 9m ago 78m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:43:41.226 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (78m) 7m ago 78m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:43:41.227 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (74m) 7m ago 81m 439M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:43:41.227 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (74m) 9m ago 82m 570M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:43:41.227 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (74m) 9m ago 82m 310M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:43:41.228 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (73m) 7m ago 81m 147M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:43:41.228 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (73m) 9m ago 81m 186M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:43:41.228 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (78m) 9m ago 78m 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:43:41.228 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (78m) 7m ago 78m 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:43:41.229 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (72m) 9m ago 81m 234M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:43:41.229 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (72m) 9m ago 80m 226M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:43:41.229 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (72m) 9m ago 80m 183M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:43:41.229 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (71m) 9m ago 80m 216M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:43:41.230 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (70m) 7m ago 79m 198M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:43:41.230 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (70m) 7m ago 79m 235M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:43:41.230 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (69m) 7m ago 79m 208M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:43:41.230 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (69m) 7m ago 78m 238M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:43:41.230 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (74m) 7m ago 78m 105M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:43:41.578 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:43:41.579 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:43:41.579 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:43:41.579 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:43:41.580 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:43:41.580 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:43:41.580 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:43:41.580 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:43:41.580 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:43:41.581 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:43:41.581 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:43:41.581 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:43:41.581 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:43:41.582 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:43:41.582 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:43:41.582 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:43:41.582 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:43:41.583 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:43:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:43:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:43:41] "GET /metrics HTTP/1.1" 200 225181 "" "Prometheus/2.33.4" 2022-09-29T06:43:41.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:43:41 smithi083 ceph-mon[79930]: from='client.36771 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:43:41.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:43:41 smithi083 ceph-mon[79930]: pgmap v2265: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:41.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:43:41 smithi083 ceph-mon[79930]: from='client.36777 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:43:41.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:43:41 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1326082668' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:43:41.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:43:41 smithi083 ceph-mon[77242]: from='client.36771 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:43:41.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:43:41 smithi083 ceph-mon[77242]: pgmap v2265: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:41.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:43:41 smithi083 ceph-mon[77242]: from='client.36777 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:43:41.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:43:41 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1326082668' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:43:42.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:43:41 smithi153 ceph-mon[70044]: from='client.36771 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:43:42.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:43:41 smithi153 ceph-mon[70044]: pgmap v2265: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:42.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:43:41 smithi153 ceph-mon[70044]: from='client.36777 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:43:42.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:43:41 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1326082668' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:43:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:43:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:43:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:43:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:43:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:43:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:43:44.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:43:44 smithi083 ceph-mon[79930]: pgmap v2266: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:44.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:43:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:43:43.795Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:43:44.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:43:44 smithi083 ceph-mon[77242]: pgmap v2266: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:44.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:43:44 smithi153 ceph-mon[70044]: pgmap v2266: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:44.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:43:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:43:44.498Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:43:46.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:43:46 smithi153 ceph-mon[70044]: pgmap v2267: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:46.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:43:46 smithi083 ceph-mon[79930]: pgmap v2267: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:46.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:43:46 smithi083 ceph-mon[77242]: pgmap v2267: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:48.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:43:48 smithi153 ceph-mon[70044]: pgmap v2268: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:48.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:43:48 smithi083 ceph-mon[79930]: pgmap v2268: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:48.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:43:48 smithi083 ceph-mon[77242]: pgmap v2268: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:50.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:43:50 smithi153 ceph-mon[70044]: pgmap v2269: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:50.395 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:43:50 smithi083 ceph-mon[79930]: pgmap v2269: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:50.396 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:43:50 smithi083 ceph-mon[77242]: pgmap v2269: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:50.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:43:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:43:50.394Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:43:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:43:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:43:51] "GET /metrics HTTP/1.1" 200 225181 "" "Prometheus/2.33.4" 2022-09-29T06:43:51.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:43:51 smithi083 ceph-mon[79930]: pgmap v2270: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:51.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:43:51 smithi083 ceph-mon[77242]: pgmap v2270: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:52.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:43:51 smithi153 ceph-mon[70044]: pgmap v2270: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:43:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:43:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:43:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:43:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:43:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:43:54.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:43:54 smithi083 ceph-mon[79930]: pgmap v2271: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:54.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:43:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:43:53.795Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:43:54.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:43:54 smithi083 ceph-mon[77242]: pgmap v2271: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:54.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:43:54 smithi153 ceph-mon[70044]: pgmap v2271: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:54.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:43:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:43:54.498Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:43:56.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:43:56 smithi153 ceph-mon[70044]: pgmap v2272: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:56.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:43:56 smithi083 ceph-mon[79930]: pgmap v2272: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:56.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:43:56 smithi083 ceph-mon[77242]: pgmap v2272: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:58.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:43:58 smithi153 ceph-mon[70044]: pgmap v2273: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:58.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:43:58 smithi083 ceph-mon[79930]: pgmap v2273: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:43:58.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:43:58 smithi083 ceph-mon[77242]: pgmap v2273: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:00.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:44:00 smithi153 ceph-mon[70044]: pgmap v2274: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:00.396 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:44:00 smithi083 ceph-mon[79930]: pgmap v2274: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:00.416 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:44:00 smithi083 ceph-mon[77242]: pgmap v2274: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:00.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:44:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:44:00.395Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:44:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:44:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:44:01] "GET /metrics HTTP/1.1" 200 225169 "" "Prometheus/2.33.4" 2022-09-29T06:44:01.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:44:01 smithi083 ceph-mon[79930]: pgmap v2275: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:01.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:44:01 smithi083 ceph-mon[77242]: pgmap v2275: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:02.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:44:01 smithi153 ceph-mon[70044]: pgmap v2275: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:44:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:44:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:44:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:44:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:44:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:44:04.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:44:04 smithi083 ceph-mon[79930]: pgmap v2276: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:04.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:44:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:44:03.795Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:44:04.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:44:04 smithi083 ceph-mon[77242]: pgmap v2276: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:04.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:44:04 smithi153 ceph-mon[70044]: pgmap v2276: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:04.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:44:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:44:04.498Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:44:06.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:44:06 smithi153 ceph-mon[70044]: pgmap v2277: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:06.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:44:06 smithi083 ceph-mon[79930]: pgmap v2277: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:06.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:44:06 smithi083 ceph-mon[77242]: pgmap v2277: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:08.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:44:08 smithi153 ceph-mon[70044]: pgmap v2278: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:08.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:44:08 smithi083 ceph-mon[79930]: pgmap v2278: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:08.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:44:08 smithi083 ceph-mon[77242]: pgmap v2278: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:10.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:44:10 smithi153 ceph-mon[70044]: pgmap v2279: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:10.397 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:44:10 smithi083 ceph-mon[79930]: pgmap v2279: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:10.397 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:44:10 smithi083 ceph-mon[77242]: pgmap v2279: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:10.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:44:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:44:10.395Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:44:11.907 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:44:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:44:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:44:11] "GET /metrics HTTP/1.1" 200 225175 "" "Prometheus/2.33.4" 2022-09-29T06:44:11.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:44:11 smithi083 ceph-mon[79930]: pgmap v2280: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:11.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:44:11 smithi083 ceph-mon[77242]: pgmap v2280: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:12.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:44:11 smithi153 ceph-mon[70044]: pgmap v2280: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:12.194 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:44:12.195 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (77m) 10m ago 78m 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:44:12.195 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (78m) 7m ago 78m 61.1M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:44:12.195 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (79m) 10m ago 79m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:44:12.195 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (79m) 7m ago 79m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:44:12.196 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (74m) 7m ago 81m 439M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:44:12.196 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (75m) 10m ago 83m 570M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:44:12.196 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (74m) 10m ago 83m 310M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:44:12.196 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (74m) 7m ago 82m 147M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:44:12.196 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (74m) 10m ago 82m 186M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:44:12.197 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (78m) 10m ago 78m 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:44:12.197 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (78m) 7m ago 78m 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:44:12.197 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (73m) 10m ago 81m 234M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:44:12.197 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (72m) 10m ago 81m 226M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:44:12.198 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (72m) 10m ago 80m 183M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:44:12.198 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (71m) 10m ago 80m 216M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:44:12.198 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (71m) 7m ago 80m 198M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:44:12.198 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (70m) 7m ago 80m 235M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:44:12.199 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (70m) 7m ago 79m 208M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:44:12.199 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (69m) 7m ago 79m 238M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:44:12.199 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (75m) 7m ago 78m 105M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:44:12.546 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:44:12.547 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:44:12.547 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:44:12.547 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:44:12.548 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:44:12.548 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:44:12.548 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:44:12.548 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:44:12.548 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:44:12.549 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:44:12.549 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:44:12.549 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:44:12.550 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:44:12.550 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:44:12.551 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:44:12.551 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:44:12.552 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:44:12.552 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:44:12.620 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:44:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:44:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:44:12.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:44:12 smithi083 ceph-mon[77242]: from='client.36789 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:44:12.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:44:12 smithi083 ceph-mon[77242]: from='client.36795 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:44:12.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:44:12 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/479176370' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:44:12.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:44:12 smithi083 ceph-mon[79930]: from='client.36789 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:44:12.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:44:12 smithi083 ceph-mon[79930]: from='client.36795 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:44:12.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:44:12 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/479176370' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:44:13.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:44:12 smithi153 ceph-mon[70044]: from='client.36789 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:44:13.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:44:12 smithi153 ceph-mon[70044]: from='client.36795 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:44:13.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:44:12 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/479176370' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:44:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:44:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:44:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:44:13.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:44:13 smithi083 ceph-mon[77242]: pgmap v2281: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:13.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:44:13 smithi083 ceph-mon[79930]: pgmap v2281: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:13.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:44:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:44:13.796Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:44:14.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:44:13 smithi153 ceph-mon[70044]: pgmap v2281: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:14.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:44:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:44:14.499Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:44:16.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:44:16 smithi153 ceph-mon[70044]: pgmap v2282: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:16.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:44:16 smithi083 ceph-mon[79930]: pgmap v2282: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:16.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:44:16 smithi083 ceph-mon[77242]: pgmap v2282: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:18.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:44:18 smithi153 ceph-mon[70044]: pgmap v2283: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:18.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:44:18 smithi083 ceph-mon[79930]: pgmap v2283: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:18.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:44:18 smithi083 ceph-mon[77242]: pgmap v2283: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:44:20 smithi153 ceph-mon[70044]: pgmap v2284: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:20.397 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:44:20 smithi083 ceph-mon[79930]: pgmap v2284: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:20.397 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:44:20 smithi083 ceph-mon[77242]: pgmap v2284: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:20.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:44:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:44:20.395Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:44:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:44:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:44:21] "GET /metrics HTTP/1.1" 200 225175 "" "Prometheus/2.33.4" 2022-09-29T06:44:21.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:44:21 smithi083 ceph-mon[79930]: pgmap v2285: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:21.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:44:21 smithi083 ceph-mon[77242]: pgmap v2285: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:22.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:44:21 smithi153 ceph-mon[70044]: pgmap v2285: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:44:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:44:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:44:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:44:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:44:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:44:24.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:44:24 smithi083 ceph-mon[79930]: pgmap v2286: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:24.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:44:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:44:23.797Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:44:24.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:44:24 smithi083 ceph-mon[77242]: pgmap v2286: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:24.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:44:24 smithi153 ceph-mon[70044]: pgmap v2286: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:24.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:44:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:44:24.500Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:44:26.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:44:26 smithi153 ceph-mon[70044]: pgmap v2287: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:26.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:44:26 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:44:26.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:44:26 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:44:26.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:44:26 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:44:26.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:44:26 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:44:26.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:44:26 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:44:26.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:44:26 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:44:26.338 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:44:26 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:44:26.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:44:26 smithi083 ceph-mon[79930]: pgmap v2287: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:26.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:44:26 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:44:26.469 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:44:26 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:44:26.469 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:44:26 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:44:26.470 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:44:26 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:44:26.470 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:44:26 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:44:26.470 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:44:26 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:44:26.471 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:44:26 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:44:26.471 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:44:26 smithi083 ceph-mon[77242]: pgmap v2287: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:26.471 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:44:26 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:44:26.472 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:44:26 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:44:26.472 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:44:26 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:44:26.472 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:44:26 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:44:26.472 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:44:26 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:44:26.473 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:44:26 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:44:26.473 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:44:26 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:44:28.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:44:28 smithi153 ceph-mon[70044]: pgmap v2288: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:28.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:44:28 smithi083 ceph-mon[79930]: pgmap v2288: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:28.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:44:28 smithi083 ceph-mon[77242]: pgmap v2288: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:30.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:44:30 smithi153 ceph-mon[70044]: pgmap v2289: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:30.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:44:30 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:44:30.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:44:30 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:44:30.397 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:44:30 smithi083 ceph-mon[79930]: pgmap v2289: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:30.397 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:44:30 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:44:30.398 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:44:30 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:44:30.398 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:44:30 smithi083 ceph-mon[77242]: pgmap v2289: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:30.399 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:44:30 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:44:30.399 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:44:30 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:44:30.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:44:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:44:30.395Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:44:31.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:44:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:44:31] "GET /metrics HTTP/1.1" 200 225179 "" "Prometheus/2.33.4" 2022-09-29T06:44:31.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:44:31 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:44:31.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:44:31 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:44:31.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:44:31 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:44:32.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:44:32 smithi083 ceph-mon[79930]: pgmap v2290: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:32.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:44:32 smithi083 ceph-mon[77242]: pgmap v2290: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:32.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:44:32 smithi153 ceph-mon[70044]: pgmap v2290: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:44:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:44:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:44:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:44:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:44:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:44:34.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:44:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:44:33.797Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:44:34.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:44:34 smithi083 ceph-mon[77242]: pgmap v2291: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:34.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:44:34 smithi083 ceph-mon[79930]: pgmap v2291: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:34.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:44:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:44:34.501Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:44:34.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:44:34 smithi153 ceph-mon[70044]: pgmap v2291: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:36.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:44:36 smithi083 ceph-mon[79930]: pgmap v2292: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:36.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:44:36 smithi083 ceph-mon[77242]: pgmap v2292: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:36.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:44:36 smithi153 ceph-mon[70044]: pgmap v2292: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:38.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:44:38 smithi083 ceph-mon[77242]: pgmap v2293: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:38.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:44:38 smithi083 ceph-mon[79930]: pgmap v2293: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:38.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:44:38 smithi153 ceph-mon[70044]: pgmap v2293: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:39.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:44:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:44:39.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:44:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:44:39.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:44:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:44:39.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:44:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:44:39.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:44:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:44:39.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:44:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:44:39.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:44:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:44:39.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:44:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:44:39.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:44:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:44:39.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:44:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:44:39.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:44:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:44:39.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:44:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:44:40.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:44:40 smithi083 ceph-mon[77242]: pgmap v2294: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:44:40 smithi083 ceph-mon[79930]: pgmap v2294: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:40.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:44:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:44:40.397Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:44:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:44:40 smithi153 ceph-mon[70044]: pgmap v2294: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:44:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:44:41] "GET /metrics HTTP/1.1" 200 225176 "" "Prometheus/2.33.4" 2022-09-29T06:44:41.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:44:41 smithi083 ceph-mon[79930]: pgmap v2295: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:41.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:44:41 smithi083 ceph-mon[77242]: pgmap v2295: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:42.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:44:41 smithi153 ceph-mon[70044]: pgmap v2295: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:44:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:44:42.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:44:42.875 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:44:43.166 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:44:43.166 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (78m) 13s ago 78m 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:44:43.166 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (78m) 8m ago 78m 61.1M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:44:43.167 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (79m) 13s ago 79m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:44:43.167 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (79m) 8m ago 79m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:44:43.167 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (75m) 8m ago 82m 439M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:44:43.168 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (75m) 13s ago 83m 580M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:44:43.168 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (75m) 13s ago 83m 348M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:44:43.168 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (74m) 8m ago 82m 147M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:44:43.168 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (74m) 13s ago 82m 207M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:44:43.169 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (79m) 13s ago 79m 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:44:43.169 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (79m) 8m ago 79m 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:44:43.169 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (73m) 13s ago 82m 259M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:44:43.169 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (73m) 13s ago 81m 251M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:44:43.170 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (73m) 13s ago 81m 203M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:44:43.170 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (72m) 13s ago 81m 241M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:44:43.170 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (71m) 8m ago 80m 198M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:44:43.170 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (71m) 8m ago 80m 235M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:44:43.171 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (70m) 8m ago 80m 208M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:44:43.171 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (70m) 8m ago 80m 238M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:44:43.171 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (75m) 8m ago 79m 105M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:44:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:44:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:44:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:44:43.531 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:44:43.531 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:44:43.531 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:44:43.532 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:44:43.532 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:44:43.532 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:44:43.533 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:44:43.533 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:44:43.533 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:44:43.533 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:44:43.534 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:44:43.534 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:44:43.534 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:44:43.535 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:44:43.535 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:44:43.535 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:44:43.535 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:44:43.536 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:44:44.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:44:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:44:43.797Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:44:44.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:44:44 smithi083 ceph-mon[79930]: from='client.46741 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:44:44.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:44:44 smithi083 ceph-mon[79930]: pgmap v2296: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:44.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:44:44 smithi083 ceph-mon[79930]: from='client.46747 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:44:44.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:44:44 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1322565554' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:44:44.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:44:44 smithi083 ceph-mon[77242]: from='client.46741 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:44:44.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:44:44 smithi083 ceph-mon[77242]: pgmap v2296: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:44.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:44:44 smithi083 ceph-mon[77242]: from='client.46747 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:44:44.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:44:44 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1322565554' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:44:44.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:44:44 smithi153 ceph-mon[70044]: from='client.46741 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:44:44.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:44:44 smithi153 ceph-mon[70044]: pgmap v2296: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:44.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:44:44 smithi153 ceph-mon[70044]: from='client.46747 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:44:44.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:44:44 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1322565554' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:44:44.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:44:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:44:44.501Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:44:46.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:44:46 smithi153 ceph-mon[70044]: pgmap v2297: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:46.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:44:46 smithi083 ceph-mon[79930]: pgmap v2297: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:46.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:44:46 smithi083 ceph-mon[77242]: pgmap v2297: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:48.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:44:48 smithi153 ceph-mon[70044]: pgmap v2298: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:48.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:44:48 smithi083 ceph-mon[79930]: pgmap v2298: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:48.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:44:48 smithi083 ceph-mon[77242]: pgmap v2298: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:50.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:44:50 smithi153 ceph-mon[70044]: pgmap v2299: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:50.398 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:44:50 smithi083 ceph-mon[79930]: pgmap v2299: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:50.399 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:44:50 smithi083 ceph-mon[77242]: pgmap v2299: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:50.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:44:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:44:50.397Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:44:51.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:44:51 smithi083 ceph-mon[77242]: pgmap v2300: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:44:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:44:51] "GET /metrics HTTP/1.1" 200 225176 "" "Prometheus/2.33.4" 2022-09-29T06:44:51.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:44:51 smithi083 ceph-mon[79930]: pgmap v2300: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:52.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:44:51 smithi153 ceph-mon[70044]: pgmap v2300: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:44:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:44:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:44:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:44:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:44:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:44:54.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:44:54 smithi083 ceph-mon[79930]: pgmap v2301: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:54.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:44:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:44:53.799Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:44:54.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:44:54 smithi083 ceph-mon[77242]: pgmap v2301: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:54.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:44:54 smithi153 ceph-mon[70044]: pgmap v2301: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:54.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:44:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:44:54.502Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:44:56.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:44:56 smithi153 ceph-mon[70044]: pgmap v2302: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:56.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:44:56 smithi083 ceph-mon[79930]: pgmap v2302: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:56.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:44:56 smithi083 ceph-mon[77242]: pgmap v2302: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:58.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:44:58 smithi153 ceph-mon[70044]: pgmap v2303: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:58.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:44:58 smithi083 ceph-mon[77242]: pgmap v2303: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:44:58.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:44:58 smithi083 ceph-mon[79930]: pgmap v2303: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:45:00 smithi153 ceph-mon[70044]: pgmap v2304: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:00.399 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:45:00 smithi083 ceph-mon[79930]: pgmap v2304: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:00.400 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:45:00 smithi083 ceph-mon[77242]: pgmap v2304: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:00.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:45:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:45:00.398Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:45:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:45:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:45:01] "GET /metrics HTTP/1.1" 200 225174 "" "Prometheus/2.33.4" 2022-09-29T06:45:01.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:45:01 smithi083 ceph-mon[79930]: pgmap v2305: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:01.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:45:01 smithi083 ceph-mon[77242]: pgmap v2305: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:02.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:45:01 smithi153 ceph-mon[70044]: pgmap v2305: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:45:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:45:02.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:45:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:45:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:45:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:45:04.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:45:04 smithi083 ceph-mon[79930]: pgmap v2306: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:04.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:45:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:45:03.799Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:45:04.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:45:04 smithi083 ceph-mon[77242]: pgmap v2306: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:04.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:45:04 smithi153 ceph-mon[70044]: pgmap v2306: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:04.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:45:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:45:04.502Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:45:06.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:45:06 smithi153 ceph-mon[70044]: pgmap v2307: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:06.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:45:06 smithi083 ceph-mon[79930]: pgmap v2307: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:06.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:45:06 smithi083 ceph-mon[77242]: pgmap v2307: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:08.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:45:08 smithi153 ceph-mon[70044]: pgmap v2308: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:08.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:45:08 smithi083 ceph-mon[79930]: pgmap v2308: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:08.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:45:08 smithi083 ceph-mon[77242]: pgmap v2308: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:10.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:45:10 smithi153 ceph-mon[70044]: pgmap v2309: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:10.400 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:45:10 smithi083 ceph-mon[79930]: pgmap v2309: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:10.401 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:45:10 smithi083 ceph-mon[77242]: pgmap v2309: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:10.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:45:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:45:10.399Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:45:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:45:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:45:11] "GET /metrics HTTP/1.1" 200 225177 "" "Prometheus/2.33.4" 2022-09-29T06:45:11.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:45:11 smithi083 ceph-mon[79930]: pgmap v2310: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:11.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:45:11 smithi083 ceph-mon[77242]: pgmap v2310: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:12.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:45:11 smithi153 ceph-mon[70044]: pgmap v2310: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:45:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:45:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:45:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:45:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:45:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:45:13.859 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:45:14.149 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:45:14.149 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (78m) 44s ago 79m 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:45:14.149 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (79m) 8m ago 79m 61.1M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:45:14.150 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (80m) 44s ago 80m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:45:14.150 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (80m) 8m ago 80m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:45:14.150 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (75m) 8m ago 82m 439M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:45:14.150 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (76m) 44s ago 84m 580M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:45:14.151 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (75m) 44s ago 84m 348M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:45:14.151 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (75m) 8m ago 83m 147M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:45:14.151 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (75m) 44s ago 83m 207M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:45:14.151 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (79m) 44s ago 79m 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:45:14.152 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (79m) 8m ago 79m 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:45:14.152 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (74m) 44s ago 82m 259M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:45:14.152 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (73m) 44s ago 82m 251M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:45:14.152 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (73m) 44s ago 81m 203M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:45:14.153 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (72m) 44s ago 81m 241M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:45:14.153 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (72m) 8m ago 81m 198M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:45:14.153 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (71m) 8m ago 81m 235M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:45:14.153 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (71m) 8m ago 80m 208M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:45:14.153 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (70m) 8m ago 80m 238M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:45:14.154 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (76m) 8m ago 79m 105M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:45:14.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:45:14 smithi083 ceph-mon[79930]: pgmap v2311: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:14.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:45:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:45:13.799Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:45:14.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:45:14 smithi083 ceph-mon[77242]: pgmap v2311: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:14.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:45:14 smithi153 ceph-mon[70044]: pgmap v2311: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:14.504 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:45:14.505 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:45:14.505 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:45:14.505 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:45:14.505 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:45:14.506 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:45:14.506 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:45:14.506 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:45:14.506 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:45:14.507 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:45:14.507 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:45:14.507 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:45:14.507 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:45:14.508 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:45:14.508 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:45:14.508 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:45:14.508 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:45:14.509 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:45:14.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:45:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:45:14.502Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:45:15.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:45:15 smithi153 ceph-mon[70044]: from='client.36825 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:45:15.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:45:15 smithi153 ceph-mon[70044]: from='client.36831 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:45:15.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:45:15 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3517198329' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:45:15.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:45:15 smithi083 ceph-mon[79930]: from='client.36825 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:45:15.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:45:15 smithi083 ceph-mon[79930]: from='client.36831 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:45:15.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:45:15 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3517198329' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:45:15.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:45:15 smithi083 ceph-mon[77242]: from='client.36825 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:45:15.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:45:15 smithi083 ceph-mon[77242]: from='client.36831 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:45:15.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:45:15 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3517198329' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:45:16.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:45:16 smithi153 ceph-mon[70044]: pgmap v2312: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:16.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:45:16 smithi083 ceph-mon[79930]: pgmap v2312: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:16.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:45:16 smithi083 ceph-mon[77242]: pgmap v2312: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:18.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:45:18 smithi153 ceph-mon[70044]: pgmap v2313: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:18.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:45:18 smithi083 ceph-mon[79930]: pgmap v2313: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:18.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:45:18 smithi083 ceph-mon[77242]: pgmap v2313: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:45:20 smithi153 ceph-mon[70044]: pgmap v2314: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:20.400 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:45:20 smithi083 ceph-mon[79930]: pgmap v2314: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:20.401 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:45:20 smithi083 ceph-mon[77242]: pgmap v2314: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:20.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:45:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:45:20.399Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:45:21.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:45:21 smithi083 ceph-mon[77242]: pgmap v2315: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:45:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:45:21] "GET /metrics HTTP/1.1" 200 225177 "" "Prometheus/2.33.4" 2022-09-29T06:45:21.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:45:21 smithi083 ceph-mon[79930]: pgmap v2315: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:22.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:45:21 smithi153 ceph-mon[70044]: pgmap v2315: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:45:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:45:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:45:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:45:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:45:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:45:24.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:45:24 smithi083 ceph-mon[77242]: pgmap v2316: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:24.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:45:24 smithi083 ceph-mon[79930]: pgmap v2316: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:24.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:45:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:45:23.801Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:45:24.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:45:24 smithi153 ceph-mon[70044]: pgmap v2316: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:24.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:45:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:45:24.503Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:45:26.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:45:26 smithi153 ceph-mon[70044]: pgmap v2317: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:26.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:45:26 smithi083 ceph-mon[77242]: pgmap v2317: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:26.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:45:26 smithi083 ceph-mon[79930]: pgmap v2317: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:28.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:45:28 smithi153 ceph-mon[70044]: pgmap v2318: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:28.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:45:28 smithi083 ceph-mon[79930]: pgmap v2318: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:28.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:45:28 smithi083 ceph-mon[77242]: pgmap v2318: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:30.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:45:30 smithi153 ceph-mon[70044]: pgmap v2319: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:30.402 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:45:30 smithi083 ceph-mon[79930]: pgmap v2319: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:30.402 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:45:30 smithi083 ceph-mon[77242]: pgmap v2319: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:30.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:45:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:45:30.400Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:45:31.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:45:31 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:45:31.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:45:31 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:45:31.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:45:31 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:45:31.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:45:31 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:45:31.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:45:31 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:45:31.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:45:31 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:45:31.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:45:31 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:45:31.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:45:31 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:45:31.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:45:31 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:45:31.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:45:31 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:45:31.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:45:31 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:45:31.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:45:31 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:45:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:45:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:45:31] "GET /metrics HTTP/1.1" 200 225179 "" "Prometheus/2.33.4" 2022-09-29T06:45:32.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:45:32 smithi153 ceph-mon[70044]: pgmap v2320: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:32.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:45:32 smithi083 ceph-mon[79930]: pgmap v2320: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:32.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:45:32 smithi083 ceph-mon[77242]: pgmap v2320: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:45:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:45:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:45:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:45:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:45:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:45:34.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:45:34 smithi083 ceph-mon[79930]: pgmap v2321: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:34.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:45:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:45:33.802Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:45:34.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:45:34 smithi083 ceph-mon[77242]: pgmap v2321: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:34.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:45:34 smithi153 ceph-mon[70044]: pgmap v2321: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:34.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:45:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:45:34.503Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:45:36.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:45:36 smithi153 ceph-mon[70044]: pgmap v2322: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:36.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:45:36 smithi083 ceph-mon[79930]: pgmap v2322: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:36.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:45:36 smithi083 ceph-mon[77242]: pgmap v2322: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:38.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:45:38 smithi083 ceph-mon[79930]: pgmap v2323: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:38.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:45:38 smithi083 ceph-mon[77242]: pgmap v2323: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:45:38 smithi153 ceph-mon[70044]: pgmap v2323: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:40.402 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:45:40 smithi083 ceph-mon[79930]: pgmap v2324: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:40.402 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:45:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:45:40.403 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:45:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:45:40.403 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:45:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:45:40.403 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:45:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:45:40.404 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:45:40 smithi083 ceph-mon[77242]: pgmap v2324: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:40.404 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:45:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:45:40.404 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:45:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:45:40.404 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:45:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:45:40.405 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:45:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:45:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:45:40 smithi153 ceph-mon[70044]: pgmap v2324: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:45:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:45:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:45:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:45:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:45:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:45:40.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:45:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:45:40.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:45:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:45:40.400Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:45:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:45:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:45:41] "GET /metrics HTTP/1.1" 200 225179 "" "Prometheus/2.33.4" 2022-09-29T06:45:41.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:45:41 smithi083 ceph-mon[79930]: pgmap v2325: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:41.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:45:41 smithi083 ceph-mon[77242]: pgmap v2325: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:42.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:45:41 smithi153 ceph-mon[70044]: pgmap v2325: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:45:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:45:42.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:45:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:45:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:45:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:45:44.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:45:44 smithi083 ceph-mon[79930]: pgmap v2326: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:44.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:45:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:45:43.803Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:45:44.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:45:44 smithi083 ceph-mon[77242]: pgmap v2326: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:44.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:45:44 smithi153 ceph-mon[70044]: pgmap v2326: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:44.831 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:45:44.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:45:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:45:44.504Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:45:45.124 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:45:45.124 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (79m) 75s ago 79m 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:45:45.125 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (79m) 9m ago 79m 61.1M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:45:45.125 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (80m) 75s ago 80m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:45:45.125 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (80m) 9m ago 80m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:45:45.125 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (76m) 9m ago 83m 439M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:45:45.126 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (76m) 75s ago 84m 580M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:45:45.126 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (76m) 75s ago 85m 348M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:45:45.126 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (75m) 9m ago 83m 147M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:45:45.127 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (75m) 75s ago 83m 207M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:45:45.127 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (80m) 75s ago 80m 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:45:45.127 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (80m) 9m ago 80m 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:45:45.127 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (74m) 75s ago 83m 259M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:45:45.128 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (74m) 75s ago 82m 251M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:45:45.128 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (74m) 75s ago 82m 203M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:45:45.128 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (73m) 75s ago 82m 241M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:45:45.128 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (72m) 9m ago 81m 198M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:45:45.129 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (72m) 9m ago 81m 235M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:45:45.129 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (71m) 9m ago 81m 208M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:45:45.129 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (71m) 9m ago 81m 238M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:45:45.129 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (76m) 9m ago 80m 105M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:45:45.483 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:45:45.483 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:45:45.484 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:45:45.484 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:45:45.484 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:45:45.485 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:45:45.485 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:45:45.485 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:45:45.485 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:45:45.486 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:45:45.486 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:45:45.486 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:45:45.486 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:45:45.487 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:45:45.487 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:45:45.487 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:45:45.487 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:45:45.488 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:45:46.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:45:46 smithi153 ceph-mon[70044]: from='client.46777 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:45:46.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:45:46 smithi153 ceph-mon[70044]: pgmap v2327: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:46.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:45:46 smithi153 ceph-mon[70044]: from='client.36849 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:45:46.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:45:46 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1613451341' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:45:46.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:45:46 smithi083 ceph-mon[79930]: from='client.46777 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:45:46.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:45:46 smithi083 ceph-mon[79930]: pgmap v2327: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:46.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:45:46 smithi083 ceph-mon[79930]: from='client.36849 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:45:46.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:45:46 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1613451341' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:45:46.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:45:46 smithi083 ceph-mon[77242]: from='client.46777 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:45:46.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:45:46 smithi083 ceph-mon[77242]: pgmap v2327: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:46.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:45:46 smithi083 ceph-mon[77242]: from='client.36849 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:45:46.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:45:46 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1613451341' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:45:48.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:45:48 smithi153 ceph-mon[70044]: pgmap v2328: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:48.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:45:48 smithi083 ceph-mon[79930]: pgmap v2328: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:48.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:45:48 smithi083 ceph-mon[77242]: pgmap v2328: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:50.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:45:50 smithi153 ceph-mon[70044]: pgmap v2329: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:50.403 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:45:50 smithi083 ceph-mon[79930]: pgmap v2329: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:50.404 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:45:50 smithi083 ceph-mon[77242]: pgmap v2329: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:50.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:45:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:45:50.402Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:45:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:45:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:45:51] "GET /metrics HTTP/1.1" 200 225179 "" "Prometheus/2.33.4" 2022-09-29T06:45:51.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:45:51 smithi083 ceph-mon[79930]: pgmap v2330: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:51.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:45:51 smithi083 ceph-mon[77242]: pgmap v2330: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:52.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:45:51 smithi153 ceph-mon[70044]: pgmap v2330: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:45:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:45:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:45:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:45:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:45:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:45:54.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:45:54 smithi083 ceph-mon[79930]: pgmap v2331: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:54.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:45:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:45:53.804Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:45:54.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:45:54 smithi083 ceph-mon[77242]: pgmap v2331: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:54.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:45:54 smithi153 ceph-mon[70044]: pgmap v2331: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:54.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:45:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:45:54.505Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:45:56.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:45:56 smithi153 ceph-mon[70044]: pgmap v2332: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:56.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:45:56 smithi083 ceph-mon[79930]: pgmap v2332: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:56.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:45:56 smithi083 ceph-mon[77242]: pgmap v2332: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:58.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:45:58 smithi153 ceph-mon[70044]: pgmap v2333: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:58.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:45:58 smithi083 ceph-mon[79930]: pgmap v2333: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:45:58.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:45:58 smithi083 ceph-mon[77242]: pgmap v2333: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:46:00 smithi153 ceph-mon[70044]: pgmap v2334: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:00.403 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:46:00 smithi083 ceph-mon[79930]: pgmap v2334: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:00.404 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:46:00 smithi083 ceph-mon[77242]: pgmap v2334: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:00.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:46:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:46:00.402Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:46:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:46:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:46:01] "GET /metrics HTTP/1.1" 200 225176 "" "Prometheus/2.33.4" 2022-09-29T06:46:01.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:46:01 smithi083 ceph-mon[79930]: pgmap v2335: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:01.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:46:01 smithi083 ceph-mon[77242]: pgmap v2335: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:02.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:46:01 smithi153 ceph-mon[70044]: pgmap v2335: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:46:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:46:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:46:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:46:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:46:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:46:04.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:46:04 smithi083 ceph-mon[79930]: pgmap v2336: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:04.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:46:04 smithi083 ceph-mon[77242]: pgmap v2336: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:04.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:46:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:46:03.804Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:46:04.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:46:04 smithi153 ceph-mon[70044]: pgmap v2336: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:04.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:46:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:46:04.506Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:46:06.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:46:06 smithi153 ceph-mon[70044]: pgmap v2337: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:06.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:46:06 smithi083 ceph-mon[79930]: pgmap v2337: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:06.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:46:06 smithi083 ceph-mon[77242]: pgmap v2337: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:08.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:46:08 smithi153 ceph-mon[70044]: pgmap v2338: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:08.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:46:08 smithi083 ceph-mon[79930]: pgmap v2338: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:08.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:46:08 smithi083 ceph-mon[77242]: pgmap v2338: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:10.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:46:10 smithi153 ceph-mon[70044]: pgmap v2339: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:10.404 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:46:10 smithi083 ceph-mon[79930]: pgmap v2339: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:10.405 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:46:10 smithi083 ceph-mon[77242]: pgmap v2339: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:10.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:46:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:46:10.403Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:46:11.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:46:11 smithi083 ceph-mon[77242]: pgmap v2340: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:46:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:46:11] "GET /metrics HTTP/1.1" 200 225176 "" "Prometheus/2.33.4" 2022-09-29T06:46:11.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:46:11 smithi083 ceph-mon[79930]: pgmap v2340: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:12.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:46:11 smithi153 ceph-mon[70044]: pgmap v2340: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:46:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:46:12.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:46:13.313 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:46:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:46:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:46:14.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:46:14 smithi083 ceph-mon[77242]: pgmap v2341: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:14.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:46:14 smithi083 ceph-mon[79930]: pgmap v2341: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:14.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:46:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:46:13.805Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:46:14.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:46:14 smithi153 ceph-mon[70044]: pgmap v2341: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:14.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:46:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:46:14.507Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:46:15.806 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:46:16.093 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:46:16.093 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (79m) 106s ago 80m 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:46:16.093 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (80m) 9m ago 80m 61.1M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:46:16.094 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (81m) 106s ago 81m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:46:16.094 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (81m) 9m ago 81m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:46:16.094 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (77m) 9m ago 83m 439M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:46:16.094 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (77m) 106s ago 85m 580M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:46:16.095 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (76m) 106s ago 85m 348M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:46:16.095 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (76m) 9m ago 84m 147M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:46:16.095 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (76m) 106s ago 84m 207M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:46:16.095 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (80m) 106s ago 80m 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:46:16.096 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (80m) 9m ago 80m 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:46:16.096 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (75m) 106s ago 83m 259M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:46:16.096 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (74m) 106s ago 83m 251M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:46:16.096 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (74m) 106s ago 83m 203M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:46:16.097 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (74m) 106s ago 82m 241M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:46:16.097 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (73m) 9m ago 82m 198M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:46:16.097 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (72m) 9m ago 82m 235M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:46:16.097 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (72m) 9m ago 81m 208M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:46:16.098 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (71m) 9m ago 81m 238M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:46:16.098 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (77m) 9m ago 80m 105M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:46:16.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:46:16 smithi153 ceph-mon[70044]: pgmap v2342: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:16.446 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:46:16.446 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:46:16.447 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:46:16.447 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:46:16.447 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:46:16.447 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:46:16.448 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:46:16.448 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:46:16.448 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:46:16.448 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:46:16.449 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:46:16.449 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:46:16.449 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:46:16.449 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:46:16.450 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:46:16.450 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:46:16.450 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:46:16.450 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:46:16.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:46:16 smithi083 ceph-mon[79930]: pgmap v2342: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:16.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:46:16 smithi083 ceph-mon[77242]: pgmap v2342: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:17.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:46:17 smithi153 ceph-mon[70044]: from='client.36861 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:46:17.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:46:17 smithi153 ceph-mon[70044]: from='client.36867 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:46:17.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:46:17 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4079805784' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:46:17.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:46:17 smithi083 ceph-mon[79930]: from='client.36861 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:46:17.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:46:17 smithi083 ceph-mon[79930]: from='client.36867 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:46:17.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:46:17 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4079805784' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:46:17.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:46:17 smithi083 ceph-mon[77242]: from='client.36861 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:46:17.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:46:17 smithi083 ceph-mon[77242]: from='client.36867 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:46:17.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:46:17 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4079805784' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:46:18.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:46:18 smithi083 ceph-mon[79930]: pgmap v2343: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:18.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:46:18 smithi083 ceph-mon[77242]: pgmap v2343: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:46:18 smithi153 ceph-mon[70044]: pgmap v2343: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:20.405 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:46:20 smithi083 ceph-mon[79930]: pgmap v2344: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:20.405 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:46:20 smithi083 ceph-mon[77242]: pgmap v2344: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:46:20 smithi153 ceph-mon[70044]: pgmap v2344: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:20.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:46:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:46:20.403Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:46:21.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:46:21 smithi083 ceph-mon[77242]: pgmap v2345: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:46:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:46:21] "GET /metrics HTTP/1.1" 200 225176 "" "Prometheus/2.33.4" 2022-09-29T06:46:21.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:46:21 smithi083 ceph-mon[79930]: pgmap v2345: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:22.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:46:21 smithi153 ceph-mon[70044]: pgmap v2345: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:46:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:46:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:46:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:46:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:46:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:46:24.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:46:24 smithi083 ceph-mon[79930]: pgmap v2346: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:24.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:46:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:46:23.807Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:46:24.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:46:24 smithi083 ceph-mon[77242]: pgmap v2346: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:24.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:46:24 smithi153 ceph-mon[70044]: pgmap v2346: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:24.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:46:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:46:24.507Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:46:26.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:46:26 smithi153 ceph-mon[70044]: pgmap v2347: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:26.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:46:26 smithi083 ceph-mon[77242]: pgmap v2347: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:26.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:46:26 smithi083 ceph-mon[79930]: pgmap v2347: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:28.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:46:28 smithi083 ceph-mon[77242]: pgmap v2348: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:28.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:46:28 smithi083 ceph-mon[79930]: pgmap v2348: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:46:28 smithi153 ceph-mon[70044]: pgmap v2348: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:30.406 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:46:30 smithi083 ceph-mon[79930]: pgmap v2349: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:30.406 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:46:30 smithi083 ceph-mon[77242]: pgmap v2349: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:46:30 smithi153 ceph-mon[70044]: pgmap v2349: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:30.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:46:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:46:30.404Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:46:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:46:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:46:31] "GET /metrics HTTP/1.1" 200 225176 "" "Prometheus/2.33.4" 2022-09-29T06:46:31.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:46:31 smithi083 ceph-mon[79930]: pgmap v2350: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:31.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:46:31 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:46:31.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:46:31 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:46:31.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:46:31 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:46:31.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:46:31 smithi083 ceph-mon[77242]: pgmap v2350: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:31.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:46:31 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:46:31.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:46:31 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:46:31.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:46:31 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:46:32.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:46:31 smithi153 ceph-mon[70044]: pgmap v2350: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:32.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:46:31 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:46:32.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:46:31 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:46:32.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:46:31 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:46:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:46:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:46:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:46:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:46:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:46:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:46:34.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:46:34 smithi083 ceph-mon[79930]: pgmap v2351: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:34.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:46:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:46:33.808Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:46:34.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:46:34 smithi083 ceph-mon[77242]: pgmap v2351: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:34.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:46:34 smithi153 ceph-mon[70044]: pgmap v2351: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:34.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:46:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:46:34.508Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:46:36.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:46:36 smithi083 ceph-mon[77242]: pgmap v2352: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:36.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:46:36 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:46:36.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:46:36 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:46:36.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:46:36 smithi083 ceph-mon[79930]: pgmap v2352: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:36.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:46:36 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:46:36.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:46:36 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:46:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:46:36 smithi153 ceph-mon[70044]: pgmap v2352: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:36.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:46:36 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:46:36.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:46:36 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:46:37.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:46:37 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:46:37.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:46:37 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:46:37.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:46:37 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:46:38.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:46:38 smithi083 ceph-mon[77242]: pgmap v2353: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:38.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:46:38 smithi083 ceph-mon[79930]: pgmap v2353: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:46:38 smithi153 ceph-mon[70044]: pgmap v2353: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:40.406 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:46:40 smithi083 ceph-mon[79930]: pgmap v2354: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:40.407 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:46:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:46:40.407 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:46:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:46:40.407 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:46:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:46:40.407 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:46:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:46:40.408 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:46:40 smithi083 ceph-mon[77242]: pgmap v2354: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:40.408 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:46:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:46:40.408 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:46:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:46:40.409 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:46:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:46:40.409 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:46:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:46:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:46:40 smithi153 ceph-mon[70044]: pgmap v2354: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:46:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:46:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:46:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:46:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:46:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:46:40.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:46:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:46:40.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:46:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:46:40.405Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:46:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:46:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:46:41] "GET /metrics HTTP/1.1" 200 225175 "" "Prometheus/2.33.4" 2022-09-29T06:46:41.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:46:41 smithi083 ceph-mon[79930]: pgmap v2355: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:41.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:46:41 smithi083 ceph-mon[77242]: pgmap v2355: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:42.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:46:41 smithi153 ceph-mon[70044]: pgmap v2355: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:46:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:46:42.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:46:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:46:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:46:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:46:44.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:46:44 smithi083 ceph-mon[79930]: pgmap v2356: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:44.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:46:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:46:43.808Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:46:44.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:46:44 smithi083 ceph-mon[77242]: pgmap v2356: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:44.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:46:44 smithi153 ceph-mon[70044]: pgmap v2356: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:44.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:46:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:46:44.509Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:46:46.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:46:46 smithi083 ceph-mon[79930]: pgmap v2357: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:46.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:46:46 smithi083 ceph-mon[77242]: pgmap v2357: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:46.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:46:46 smithi153 ceph-mon[70044]: pgmap v2357: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:46.768 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:46:47.058 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:46:47.059 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (80m) 2m ago 80m 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:46:47.059 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (80m) 11s ago 80m 61.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:46:47.059 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (81m) 2m ago 81m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:46:47.059 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (81m) 11s ago 81m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:46:47.060 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (77m) 11s ago 84m 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:46:47.060 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (77m) 2m ago 85m 580M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:46:47.060 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (77m) 2m ago 86m 348M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:46:47.060 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (76m) 11s ago 84m 161M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:46:47.061 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (76m) 2m ago 84m 207M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:46:47.061 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (81m) 2m ago 81m 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:46:47.061 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (81m) 11s ago 81m 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:46:47.061 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (75m) 2m ago 84m 259M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:46:47.062 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (75m) 2m ago 83m 251M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:46:47.062 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (75m) 2m ago 83m 203M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:46:47.062 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (74m) 2m ago 83m 241M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:46:47.062 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (74m) 11s ago 82m 219M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:46:47.063 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (73m) 11s ago 82m 262M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:46:47.063 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (72m) 11s ago 82m 233M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:46:47.063 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (72m) 11s ago 82m 268M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:46:47.064 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (77m) 11s ago 81m 109M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:46:47.417 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:46:47.417 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:46:47.418 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:46:47.418 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:46:47.418 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:46:47.418 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:46:47.419 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:46:47.419 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:46:47.419 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:46:47.419 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:46:47.420 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:46:47.420 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:46:47.420 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:46:47.420 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:46:47.421 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:46:47.421 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:46:47.421 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:46:47.421 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:46:48.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:46:48 smithi083 ceph-mon[79930]: from='client.36879 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:46:48.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:46:48 smithi083 ceph-mon[79930]: pgmap v2358: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:48.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:46:48 smithi083 ceph-mon[79930]: from='client.36885 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:46:48.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:46:48 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3786690430' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:46:48.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:46:48 smithi083 ceph-mon[77242]: from='client.36879 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:46:48.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:46:48 smithi083 ceph-mon[77242]: pgmap v2358: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:48.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:46:48 smithi083 ceph-mon[77242]: from='client.36885 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:46:48.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:46:48 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3786690430' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:46:48.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:46:48 smithi153 ceph-mon[70044]: from='client.36879 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:46:48.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:46:48 smithi153 ceph-mon[70044]: pgmap v2358: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:48.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:46:48 smithi153 ceph-mon[70044]: from='client.36885 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:46:48.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:46:48 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3786690430' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:46:50.406 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:46:50 smithi083 ceph-mon[79930]: pgmap v2359: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:50.407 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:46:50 smithi083 ceph-mon[77242]: pgmap v2359: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:46:50 smithi153 ceph-mon[70044]: pgmap v2359: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:50.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:46:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:46:50.405Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:46:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:46:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:46:51] "GET /metrics HTTP/1.1" 200 225175 "" "Prometheus/2.33.4" 2022-09-29T06:46:51.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:46:51 smithi083 ceph-mon[79930]: pgmap v2360: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:51.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:46:51 smithi083 ceph-mon[77242]: pgmap v2360: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:52.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:46:51 smithi153 ceph-mon[70044]: pgmap v2360: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:46:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:46:52.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:46:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:46:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:46:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:46:54.061 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:46:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:46:53.808Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:46:54.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:46:54 smithi083 ceph-mon[77242]: pgmap v2361: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:54.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:46:54 smithi083 ceph-mon[79930]: pgmap v2361: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:54.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:46:54 smithi153 ceph-mon[70044]: pgmap v2361: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:54.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:46:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:46:54.509Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:46:56.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:46:56 smithi083 ceph-mon[77242]: pgmap v2362: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:56.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:46:56 smithi083 ceph-mon[79930]: pgmap v2362: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:56.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:46:56 smithi153 ceph-mon[70044]: pgmap v2362: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:58.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:46:58 smithi083 ceph-mon[79930]: pgmap v2363: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:58.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:46:58 smithi083 ceph-mon[77242]: pgmap v2363: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:46:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:46:58 smithi153 ceph-mon[70044]: pgmap v2363: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:00.407 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:47:00 smithi083 ceph-mon[79930]: pgmap v2364: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:00.407 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:47:00 smithi083 ceph-mon[77242]: pgmap v2364: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:47:00 smithi153 ceph-mon[70044]: pgmap v2364: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:00.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:47:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:47:00.405Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:47:01.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:47:01 smithi083 ceph-mon[77242]: pgmap v2365: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:47:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:47:01] "GET /metrics HTTP/1.1" 200 225178 "" "Prometheus/2.33.4" 2022-09-29T06:47:01.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:47:01 smithi083 ceph-mon[79930]: pgmap v2365: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:02.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:47:01 smithi153 ceph-mon[70044]: pgmap v2365: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:47:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:47:02.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:47:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:47:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:47:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:47:04.064 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:47:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:47:03.810Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:47:04.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:47:04 smithi083 ceph-mon[79930]: pgmap v2366: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:04.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:47:04 smithi083 ceph-mon[77242]: pgmap v2366: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:47:04 smithi153 ceph-mon[70044]: pgmap v2366: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:04.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:47:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:47:04.510Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:47:06.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:47:06 smithi083 ceph-mon[77242]: pgmap v2367: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:06.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:47:06 smithi083 ceph-mon[79930]: pgmap v2367: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:06.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:47:06 smithi153 ceph-mon[70044]: pgmap v2367: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:08.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:47:08 smithi083 ceph-mon[77242]: pgmap v2368: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:08.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:47:08 smithi083 ceph-mon[79930]: pgmap v2368: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:47:08 smithi153 ceph-mon[70044]: pgmap v2368: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:10.408 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:47:10 smithi083 ceph-mon[79930]: pgmap v2369: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:10.408 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:47:10 smithi083 ceph-mon[77242]: pgmap v2369: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:47:10 smithi153 ceph-mon[70044]: pgmap v2369: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:10.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:47:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:47:10.406Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:47:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:47:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:47:11] "GET /metrics HTTP/1.1" 200 225173 "" "Prometheus/2.33.4" 2022-09-29T06:47:11.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:47:11 smithi083 ceph-mon[79930]: pgmap v2370: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:11.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:47:11 smithi083 ceph-mon[77242]: pgmap v2370: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:12.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:47:11 smithi153 ceph-mon[70044]: pgmap v2370: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:47:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:47:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:47:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:47:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:47:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:47:14.068 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:47:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:47:13.810Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:47:14.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:47:14 smithi083 ceph-mon[77242]: pgmap v2371: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:14.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:47:14 smithi083 ceph-mon[79930]: pgmap v2371: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:14.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:47:14 smithi153 ceph-mon[70044]: pgmap v2371: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:14.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:47:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:47:14.511Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:47:16.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:47:16 smithi083 ceph-mon[79930]: pgmap v2372: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:16.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:47:16 smithi083 ceph-mon[77242]: pgmap v2372: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:47:16 smithi153 ceph-mon[70044]: pgmap v2372: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:17.743 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:47:18.032 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:47:18.032 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (81m) 2m ago 81m 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:47:18.032 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (81m) 42s ago 81m 61.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:47:18.033 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (82m) 2m ago 82m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:47:18.033 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (82m) 42s ago 82m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:47:18.033 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (78m) 42s ago 84m 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:47:18.034 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (78m) 2m ago 86m 580M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:47:18.034 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (77m) 2m ago 86m 348M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:47:18.034 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (77m) 42s ago 85m 161M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:47:18.035 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (77m) 2m ago 85m 207M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:47:18.035 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (81m) 2m ago 81m 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:47:18.035 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (81m) 42s ago 81m 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:47:18.035 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (76m) 2m ago 84m 259M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:47:18.036 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (75m) 2m ago 84m 251M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:47:18.036 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (75m) 2m ago 84m 203M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:47:18.036 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (75m) 2m ago 83m 241M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:47:18.036 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (74m) 42s ago 83m 219M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:47:18.037 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (73m) 42s ago 83m 262M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:47:18.037 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (73m) 42s ago 82m 233M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:47:18.037 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (72m) 42s ago 82m 268M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:47:18.038 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (78m) 42s ago 81m 109M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:47:18.391 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:47:18.392 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:47:18.392 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:47:18.392 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:47:18.392 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:47:18.393 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:47:18.393 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:47:18.393 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:47:18.393 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:47:18.394 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:47:18.394 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:47:18.394 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:47:18.394 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:47:18.395 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:47:18.395 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:47:18.395 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:47:18.395 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:47:18.395 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:47:18.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:47:18 smithi083 ceph-mon[79930]: pgmap v2373: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:18.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:47:18 smithi083 ceph-mon[77242]: pgmap v2373: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:18.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:47:18 smithi153 ceph-mon[70044]: pgmap v2373: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:19.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:47:19 smithi083 ceph-mon[79930]: from='client.36897 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:47:19.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:47:19 smithi083 ceph-mon[79930]: from='client.46837 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:47:19.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:47:19 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2294711383' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:47:19.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:47:19 smithi083 ceph-mon[77242]: from='client.36897 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:47:19.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:47:19 smithi083 ceph-mon[77242]: from='client.46837 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:47:19.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:47:19 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2294711383' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:47:19.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:47:19 smithi153 ceph-mon[70044]: from='client.36897 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:47:19.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:47:19 smithi153 ceph-mon[70044]: from='client.46837 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:47:19.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:47:19 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2294711383' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:47:20.408 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:47:20 smithi083 ceph-mon[79930]: pgmap v2374: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:20.409 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:47:20 smithi083 ceph-mon[77242]: pgmap v2374: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:47:20 smithi153 ceph-mon[70044]: pgmap v2374: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:20.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:47:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:47:20.407Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:47:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:47:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:47:21] "GET /metrics HTTP/1.1" 200 225173 "" "Prometheus/2.33.4" 2022-09-29T06:47:21.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:47:21 smithi083 ceph-mon[79930]: pgmap v2375: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:21.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:47:21 smithi083 ceph-mon[77242]: pgmap v2375: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:22.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:47:21 smithi153 ceph-mon[70044]: pgmap v2375: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:47:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:47:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:47:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:47:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:47:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:47:24.071 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:47:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:47:23.811Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:47:24.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:47:24 smithi083 ceph-mon[79930]: pgmap v2376: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:24.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:47:24 smithi083 ceph-mon[77242]: pgmap v2376: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:47:24 smithi153 ceph-mon[70044]: pgmap v2376: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:24.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:47:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:47:24.511Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:47:26.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:47:26 smithi083 ceph-mon[79930]: pgmap v2377: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:26.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:47:26 smithi083 ceph-mon[77242]: pgmap v2377: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:47:26 smithi153 ceph-mon[70044]: pgmap v2377: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:28.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:47:28 smithi083 ceph-mon[79930]: pgmap v2378: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:28.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:47:28 smithi083 ceph-mon[77242]: pgmap v2378: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:47:28 smithi153 ceph-mon[70044]: pgmap v2378: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:30.408 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:47:30 smithi083 ceph-mon[79930]: pgmap v2379: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:30.409 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:47:30 smithi083 ceph-mon[77242]: pgmap v2379: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:47:30 smithi153 ceph-mon[70044]: pgmap v2379: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:30.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:47:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:47:30.407Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:47:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:47:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:47:31] "GET /metrics HTTP/1.1" 200 225174 "" "Prometheus/2.33.4" 2022-09-29T06:47:31.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:47:31 smithi083 ceph-mon[79930]: pgmap v2380: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:31.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:47:31 smithi083 ceph-mon[77242]: pgmap v2380: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:32.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:47:31 smithi153 ceph-mon[70044]: pgmap v2380: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:47:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:47:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:47:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:47:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:47:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:47:34.074 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:47:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:47:33.811Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:47:34.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:47:34 smithi083 ceph-mon[79930]: pgmap v2381: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:34.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:47:34 smithi083 ceph-mon[77242]: pgmap v2381: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:47:34 smithi153 ceph-mon[70044]: pgmap v2381: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:34.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:47:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:47:34.512Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:47:36.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:47:36 smithi083 ceph-mon[79930]: pgmap v2382: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:36.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:47:36 smithi083 ceph-mon[77242]: pgmap v2382: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:47:36 smithi153 ceph-mon[70044]: pgmap v2382: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:37.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:47:37 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:47:37.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:47:37 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:47:37.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:47:37 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:47:37.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:47:37 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:47:37.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:47:37 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:47:37.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:47:37 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:47:37.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:47:37 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:47:37.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:47:37 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:47:37.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:47:37 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:47:37.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:47:37 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:47:37.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:47:37 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:47:37.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:47:37 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:47:38.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:47:38 smithi083 ceph-mon[79930]: pgmap v2383: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:38.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:47:38 smithi083 ceph-mon[77242]: pgmap v2383: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:47:38 smithi153 ceph-mon[70044]: pgmap v2383: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:40.409 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:47:40 smithi083 ceph-mon[79930]: pgmap v2384: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:40.410 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:47:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:47:40.410 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:47:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:47:40.410 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:47:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:47:40.411 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:47:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:47:40.411 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:47:40 smithi083 ceph-mon[77242]: pgmap v2384: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:40.411 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:47:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:47:40.412 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:47:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:47:40.412 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:47:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:47:40.412 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:47:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:47:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:47:40 smithi153 ceph-mon[70044]: pgmap v2384: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:47:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:47:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:47:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:47:40.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:47:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:47:40.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:47:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:47:40.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:47:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:47:40.408Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:47:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:47:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:47:41] "GET /metrics HTTP/1.1" 200 225176 "" "Prometheus/2.33.4" 2022-09-29T06:47:41.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:47:41 smithi083 ceph-mon[79930]: pgmap v2385: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:41.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:47:41 smithi083 ceph-mon[77242]: pgmap v2385: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:42.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:47:41 smithi153 ceph-mon[70044]: pgmap v2385: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:47:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:47:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:47:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:47:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:47:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:47:44.077 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:47:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:47:43.812Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:47:44.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:47:44 smithi083 ceph-mon[79930]: pgmap v2386: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:44.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:47:44 smithi083 ceph-mon[77242]: pgmap v2386: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:44.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:47:44 smithi153 ceph-mon[70044]: pgmap v2386: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:44.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:47:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:47:44.512Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:47:46.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:47:46 smithi083 ceph-mon[79930]: pgmap v2387: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:46.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:47:46 smithi083 ceph-mon[77242]: pgmap v2387: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:46.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:47:46 smithi153 ceph-mon[70044]: pgmap v2387: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:48.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:47:48 smithi083 ceph-mon[79930]: pgmap v2388: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:48.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:47:48 smithi083 ceph-mon[77242]: pgmap v2388: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:48.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:47:48 smithi153 ceph-mon[70044]: pgmap v2388: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:48.718 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:47:49.005 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:47:49.006 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (81m) 3m ago 81m 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:47:49.006 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (81m) 73s ago 81m 61.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:47:49.006 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (82m) 3m ago 82m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:47:49.007 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (82m) 73s ago 82m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:47:49.007 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (78m) 73s ago 85m 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:47:49.007 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (79m) 3m ago 86m 580M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:47:49.007 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (78m) 3m ago 87m 348M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:47:49.007 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (77m) 73s ago 85m 161M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:47:49.008 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (77m) 3m ago 85m 207M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:47:49.008 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (82m) 3m ago 82m 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:47:49.008 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (82m) 73s ago 82m 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:47:49.008 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (76m) 3m ago 85m 259M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:47:49.009 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (76m) 3m ago 84m 251M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:47:49.009 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (76m) 3m ago 84m 203M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:47:49.009 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (75m) 3m ago 84m 241M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:47:49.009 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (75m) 73s ago 83m 219M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:47:49.009 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (74m) 73s ago 83m 262M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:47:49.010 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (73m) 73s ago 83m 233M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:47:49.010 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (73m) 73s ago 83m 268M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:47:49.010 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (78m) 73s ago 82m 109M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:47:49.362 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:47:49.362 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:47:49.362 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:47:49.363 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:47:49.363 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:47:49.363 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:47:49.363 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:47:49.363 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:47:49.364 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:47:49.364 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:47:49.364 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:47:49.364 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:47:49.365 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:47:49.365 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:47:49.365 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:47:49.365 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:47:49.365 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:47:49.366 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:47:50.410 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:47:50 smithi083 ceph-mon[79930]: from='client.36915 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:47:50.410 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:47:50 smithi083 ceph-mon[79930]: from='client.36921 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:47:50.411 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:47:50 smithi083 ceph-mon[79930]: pgmap v2389: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:50.411 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:47:50 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1240723992' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:47:50.411 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:47:50 smithi083 ceph-mon[77242]: from='client.36915 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:47:50.412 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:47:50 smithi083 ceph-mon[77242]: from='client.36921 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:47:50.412 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:47:50 smithi083 ceph-mon[77242]: pgmap v2389: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:50.412 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:47:50 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1240723992' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:47:50.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:47:50 smithi153 ceph-mon[70044]: from='client.36915 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:47:50.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:47:50 smithi153 ceph-mon[70044]: from='client.36921 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:47:50.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:47:50 smithi153 ceph-mon[70044]: pgmap v2389: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:50.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:47:50 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1240723992' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:47:50.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:47:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:47:50.408Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:47:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:47:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:47:51] "GET /metrics HTTP/1.1" 200 225176 "" "Prometheus/2.33.4" 2022-09-29T06:47:51.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:47:51 smithi083 ceph-mon[79930]: pgmap v2390: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:51.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:47:51 smithi083 ceph-mon[77242]: pgmap v2390: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:52.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:47:51 smithi153 ceph-mon[70044]: pgmap v2390: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:47:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:47:52.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:47:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:47:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:47:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:47:54.080 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:47:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:47:53.812Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:47:54.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:47:54 smithi083 ceph-mon[77242]: pgmap v2391: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:54.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:47:54 smithi083 ceph-mon[79930]: pgmap v2391: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:54.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:47:54 smithi153 ceph-mon[70044]: pgmap v2391: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:54.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:47:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:47:54.512Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:47:56.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:47:56 smithi083 ceph-mon[79930]: pgmap v2392: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:56.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:47:56 smithi083 ceph-mon[77242]: pgmap v2392: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:56.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:47:56 smithi153 ceph-mon[70044]: pgmap v2392: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:58.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:47:58 smithi083 ceph-mon[79930]: pgmap v2393: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:58.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:47:58 smithi083 ceph-mon[77242]: pgmap v2393: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:47:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:47:58 smithi153 ceph-mon[70044]: pgmap v2393: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:00.411 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:48:00 smithi083 ceph-mon[79930]: pgmap v2394: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:00.412 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:48:00 smithi083 ceph-mon[77242]: pgmap v2394: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:48:00 smithi153 ceph-mon[70044]: pgmap v2394: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:00.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:48:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:48:00.410Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:48:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:48:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:48:01] "GET /metrics HTTP/1.1" 200 225174 "" "Prometheus/2.33.4" 2022-09-29T06:48:01.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:48:01 smithi083 ceph-mon[79930]: pgmap v2395: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:01.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:48:01 smithi083 ceph-mon[77242]: pgmap v2395: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:02.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:48:01 smithi153 ceph-mon[70044]: pgmap v2395: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:48:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:48:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:48:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:48:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:48:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:48:04.083 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:48:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:48:03.813Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:48:04.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:48:04 smithi083 ceph-mon[79930]: pgmap v2396: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:04.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:48:04 smithi083 ceph-mon[77242]: pgmap v2396: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:48:04 smithi153 ceph-mon[70044]: pgmap v2396: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:04.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:48:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:48:04.513Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:48:06.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:48:06 smithi083 ceph-mon[79930]: pgmap v2397: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:06.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:48:06 smithi083 ceph-mon[77242]: pgmap v2397: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:06.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:48:06 smithi153 ceph-mon[70044]: pgmap v2397: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:08.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:48:08 smithi083 ceph-mon[79930]: pgmap v2398: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:08.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:48:08 smithi083 ceph-mon[77242]: pgmap v2398: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:08.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:48:08 smithi153 ceph-mon[70044]: pgmap v2398: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:10.412 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:48:10 smithi083 ceph-mon[79930]: pgmap v2399: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:10.412 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:48:10 smithi083 ceph-mon[77242]: pgmap v2399: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:48:10 smithi153 ceph-mon[70044]: pgmap v2399: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:10.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:48:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:48:10.410Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:48:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:48:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:48:11] "GET /metrics HTTP/1.1" 200 225176 "" "Prometheus/2.33.4" 2022-09-29T06:48:11.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:48:11 smithi083 ceph-mon[79930]: pgmap v2400: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:11.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:48:11 smithi083 ceph-mon[77242]: pgmap v2400: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:12.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:48:11 smithi153 ceph-mon[70044]: pgmap v2400: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:48:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:48:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:48:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:48:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:48:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:48:14.086 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:48:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:48:13.813Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:48:14.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:48:14 smithi083 ceph-mon[79930]: pgmap v2401: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:14.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:48:14 smithi083 ceph-mon[77242]: pgmap v2401: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:14.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:48:14 smithi153 ceph-mon[70044]: pgmap v2401: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:14.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:48:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:48:14.514Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:48:16.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:48:16 smithi083 ceph-mon[79930]: pgmap v2402: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:16.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:48:16 smithi083 ceph-mon[77242]: pgmap v2402: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:48:16 smithi153 ceph-mon[70044]: pgmap v2402: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:18.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:48:18 smithi083 ceph-mon[79930]: pgmap v2403: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:18.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:48:18 smithi083 ceph-mon[77242]: pgmap v2403: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:48:18 smithi153 ceph-mon[70044]: pgmap v2403: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:19.688 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:48:19.976 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:48:19.976 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (82m) 3m ago 82m 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:48:19.977 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (82m) 104s ago 82m 61.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:48:19.977 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (83m) 3m ago 83m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:48:19.977 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (83m) 104s ago 83m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:48:19.977 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (79m) 104s ago 85m 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:48:19.978 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (79m) 3m ago 87m 580M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:48:19.978 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (78m) 3m ago 87m 348M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:48:19.978 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (78m) 104s ago 86m 161M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:48:19.978 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (78m) 3m ago 86m 207M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:48:19.979 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (82m) 3m ago 82m 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:48:19.979 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (82m) 104s ago 82m 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:48:19.979 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (77m) 3m ago 85m 259M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:48:19.979 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (76m) 3m ago 85m 251M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:48:19.980 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (76m) 3m ago 85m 203M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:48:19.980 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (76m) 3m ago 84m 241M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:48:19.980 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (75m) 104s ago 84m 219M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:48:19.981 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (74m) 104s ago 84m 262M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:48:19.981 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (74m) 104s ago 83m 233M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:48:19.981 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (73m) 104s ago 83m 268M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:48:19.981 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (79m) 104s ago 82m 109M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:48:20.334 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:48:20.335 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:48:20.335 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:48:20.335 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:48:20.335 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:48:20.336 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:48:20.336 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:48:20.336 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:48:20.336 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:48:20.337 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:48:20.337 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:48:20.337 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:48:20.337 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:48:20.338 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:48:20.338 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:48:20.338 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:48:20.338 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:48:20.339 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:48:20.412 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:48:20 smithi083 ceph-mon[79930]: pgmap v2404: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:20.413 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:48:20 smithi083 ceph-mon[77242]: pgmap v2404: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:48:20 smithi153 ceph-mon[70044]: pgmap v2404: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:20.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:48:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:48:20.411Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:48:21.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:48:21 smithi083 ceph-mon[77242]: from='client.36933 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:48:21.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:48:21 smithi083 ceph-mon[77242]: from='client.36939 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:48:21.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:48:21 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1546376744' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:48:21.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:48:21 smithi083 ceph-mon[79930]: from='client.36933 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:48:21.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:48:21 smithi083 ceph-mon[79930]: from='client.36939 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:48:21.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:48:21 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1546376744' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:48:21.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:48:21 smithi153 ceph-mon[70044]: from='client.36933 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:48:21.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:48:21 smithi153 ceph-mon[70044]: from='client.36939 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:48:21.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:48:21 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1546376744' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:48:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:48:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:48:21] "GET /metrics HTTP/1.1" 200 225176 "" "Prometheus/2.33.4" 2022-09-29T06:48:22.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:48:22 smithi083 ceph-mon[77242]: pgmap v2405: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:22.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:48:22 smithi083 ceph-mon[79930]: pgmap v2405: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:48:22 smithi153 ceph-mon[70044]: pgmap v2405: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:22.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:48:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:48:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:48:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:48:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:48:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:48:24.100 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:48:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:48:23.813Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:48:24.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:48:24 smithi083 ceph-mon[79930]: pgmap v2406: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:24.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:48:24 smithi083 ceph-mon[77242]: pgmap v2406: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:48:24 smithi153 ceph-mon[70044]: pgmap v2406: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:24.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:48:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:48:24.514Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:48:26.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:48:26 smithi083 ceph-mon[79930]: pgmap v2407: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:26.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:48:26 smithi083 ceph-mon[77242]: pgmap v2407: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:48:26 smithi153 ceph-mon[70044]: pgmap v2407: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:28.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:48:28 smithi083 ceph-mon[79930]: pgmap v2408: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:28.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:48:28 smithi083 ceph-mon[77242]: pgmap v2408: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:48:28 smithi153 ceph-mon[70044]: pgmap v2408: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:30.413 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:48:30 smithi083 ceph-mon[79930]: pgmap v2409: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:30.413 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:48:30 smithi083 ceph-mon[77242]: pgmap v2409: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:48:30 smithi153 ceph-mon[70044]: pgmap v2409: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:30.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:48:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:48:30.411Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:48:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:48:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:48:31] "GET /metrics HTTP/1.1" 200 225176 "" "Prometheus/2.33.4" 2022-09-29T06:48:31.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:48:31 smithi083 ceph-mon[79930]: pgmap v2410: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:31.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:48:31 smithi083 ceph-mon[77242]: pgmap v2410: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:32.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:48:31 smithi153 ceph-mon[70044]: pgmap v2410: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:48:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:48:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:48:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:48:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:48:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:48:34.093 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:48:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:48:33.815Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:48:34.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:48:34 smithi083 ceph-mon[77242]: pgmap v2411: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:34.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:48:34 smithi083 ceph-mon[79930]: pgmap v2411: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:48:34 smithi153 ceph-mon[70044]: pgmap v2411: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:34.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:48:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:48:34.516Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:48:36.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:48:36 smithi083 ceph-mon[79930]: pgmap v2412: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:36.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:48:36 smithi083 ceph-mon[77242]: pgmap v2412: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:48:36 smithi153 ceph-mon[70044]: pgmap v2412: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:37.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:48:37 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:48:37.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:48:37 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:48:37.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:48:37 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:48:37.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:48:37 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:48:37.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:48:37 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:48:37.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:48:37 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:48:37.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:48:37 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:48:37.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:48:37 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:48:37.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:48:37 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:48:38.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:48:38 smithi083 ceph-mon[79930]: pgmap v2413: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:38.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:48:38 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:48:38.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:48:38 smithi083 ceph-mon[77242]: pgmap v2413: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:38.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:48:38 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:48:38.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:48:38 smithi153 ceph-mon[70044]: pgmap v2413: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:38.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:48:38 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:48:39.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:48:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:48:39.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:48:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:48:39.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:48:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:48:39.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:48:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:48:39.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:48:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:48:39.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:48:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:48:39.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:48:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:48:39.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:48:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:48:39.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:48:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:48:39.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:48:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:48:39.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:48:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:48:39.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:48:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:48:40.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:48:40 smithi083 ceph-mon[79930]: pgmap v2414: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:40.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:48:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:48:40.412Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:48:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:48:40 smithi083 ceph-mon[77242]: pgmap v2414: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:48:40 smithi153 ceph-mon[70044]: pgmap v2414: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:41.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:48:41 smithi083 ceph-mon[79930]: pgmap v2415: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:41.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:48:41 smithi083 ceph-mon[77242]: pgmap v2415: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:48:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:48:41] "GET /metrics HTTP/1.1" 200 225178 "" "Prometheus/2.33.4" 2022-09-29T06:48:42.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:48:41 smithi153 ceph-mon[70044]: pgmap v2415: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:48:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:48:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:48:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:48:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:48:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:48:44.096 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:48:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:48:43.815Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:48:44.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:48:44 smithi083 ceph-mon[79930]: pgmap v2416: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:44.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:48:44 smithi083 ceph-mon[77242]: pgmap v2416: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:44.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:48:44 smithi153 ceph-mon[70044]: pgmap v2416: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:44.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:48:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:48:44.516Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:48:46.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:48:46 smithi083 ceph-mon[79930]: pgmap v2417: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:46.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:48:46 smithi083 ceph-mon[77242]: pgmap v2417: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:46.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:48:46 smithi153 ceph-mon[70044]: pgmap v2417: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:48.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:48:48 smithi083 ceph-mon[79930]: pgmap v2418: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:48.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:48:48 smithi083 ceph-mon[77242]: pgmap v2418: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:48.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:48:48 smithi153 ceph-mon[70044]: pgmap v2418: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:50.413 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:48:50 smithi083 ceph-mon[79930]: pgmap v2419: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:50.414 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:48:50 smithi083 ceph-mon[77242]: pgmap v2419: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:50.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:48:50 smithi153 ceph-mon[70044]: pgmap v2419: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:50.656 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:48:50.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:48:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:48:50.412Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:48:50.948 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:48:50.949 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (82m) 4m ago 83m 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:48:50.949 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (82m) 2m ago 82m 61.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:48:50.949 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (83m) 4m ago 83m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:48:50.949 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (83m) 2m ago 83m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:48:50.950 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (79m) 2m ago 86m 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:48:50.950 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (80m) 4m ago 87m 580M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:48:50.950 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (79m) 4m ago 88m 348M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:48:50.950 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (78m) 2m ago 86m 161M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:48:50.951 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (78m) 4m ago 86m 207M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:48:50.951 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (83m) 4m ago 83m 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:48:50.951 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (83m) 2m ago 83m 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:48:50.951 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (78m) 4m ago 86m 259M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:48:50.952 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (77m) 4m ago 85m 251M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:48:50.952 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (77m) 4m ago 85m 203M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:48:50.952 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (76m) 4m ago 85m 241M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:48:50.952 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (76m) 2m ago 85m 219M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:48:50.953 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (75m) 2m ago 84m 262M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:48:50.953 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (74m) 2m ago 84m 233M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:48:50.953 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (74m) 2m ago 84m 268M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:48:50.953 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (79m) 2m ago 83m 109M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:48:51.303 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:48:51.304 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:48:51.304 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:48:51.304 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:48:51.305 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:48:51.305 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:48:51.305 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:48:51.306 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:48:51.306 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:48:51.306 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:48:51.306 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:48:51.307 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:48:51.307 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:48:51.307 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:48:51.308 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:48:51.308 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:48:51.308 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:48:51.308 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:48:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:48:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:48:51] "GET /metrics HTTP/1.1" 200 225178 "" "Prometheus/2.33.4" 2022-09-29T06:48:51.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:48:51 smithi083 ceph-mon[77242]: from='client.46885 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:48:51.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:48:51 smithi083 ceph-mon[77242]: from='client.36957 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:48:51.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:48:51 smithi083 ceph-mon[77242]: pgmap v2420: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:51.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:48:51 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2140462583' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:48:51.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:48:51 smithi083 ceph-mon[79930]: from='client.46885 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:48:51.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:48:51 smithi083 ceph-mon[79930]: from='client.36957 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:48:51.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:48:51 smithi083 ceph-mon[79930]: pgmap v2420: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:51.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:48:51 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2140462583' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:48:52.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:48:51 smithi153 ceph-mon[70044]: from='client.46885 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:48:52.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:48:51 smithi153 ceph-mon[70044]: from='client.36957 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:48:52.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:48:51 smithi153 ceph-mon[70044]: pgmap v2420: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:52.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:48:51 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2140462583' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:48:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:48:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:48:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:48:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:48:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:48:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:48:54.098 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:48:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:48:53.816Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:48:54.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:48:54 smithi083 ceph-mon[77242]: pgmap v2421: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:54.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:48:54 smithi083 ceph-mon[79930]: pgmap v2421: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:54.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:48:54 smithi153 ceph-mon[70044]: pgmap v2421: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:54.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:48:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:48:54.517Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:48:56.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:48:56 smithi083 ceph-mon[77242]: pgmap v2422: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:56.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:48:56 smithi083 ceph-mon[79930]: pgmap v2422: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:56.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:48:56 smithi153 ceph-mon[70044]: pgmap v2422: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:58.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:48:58 smithi083 ceph-mon[79930]: pgmap v2423: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:58.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:48:58 smithi083 ceph-mon[77242]: pgmap v2423: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:48:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:48:58 smithi153 ceph-mon[70044]: pgmap v2423: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:00.415 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:49:00 smithi083 ceph-mon[79930]: pgmap v2424: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:00.416 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:49:00 smithi083 ceph-mon[77242]: pgmap v2424: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:49:00 smithi153 ceph-mon[70044]: pgmap v2424: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:00.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:49:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:49:00.414Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:49:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:49:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:49:01] "GET /metrics HTTP/1.1" 200 225178 "" "Prometheus/2.33.4" 2022-09-29T06:49:01.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:49:01 smithi083 ceph-mon[79930]: pgmap v2425: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:01.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:49:01 smithi083 ceph-mon[77242]: pgmap v2425: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:02.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:49:01 smithi153 ceph-mon[70044]: pgmap v2425: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:49:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:49:02.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:49:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:49:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:49:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:49:04.101 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:49:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:49:03.817Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:49:04.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:49:04 smithi083 ceph-mon[79930]: pgmap v2426: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:04.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:49:04 smithi083 ceph-mon[77242]: pgmap v2426: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:49:04 smithi153 ceph-mon[70044]: pgmap v2426: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:04.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:49:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:49:04.518Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:49:06.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:49:06 smithi083 ceph-mon[79930]: pgmap v2427: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:06.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:49:06 smithi083 ceph-mon[77242]: pgmap v2427: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:06.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:49:06 smithi153 ceph-mon[70044]: pgmap v2427: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:08.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:49:08 smithi083 ceph-mon[79930]: pgmap v2428: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:08.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:49:08 smithi083 ceph-mon[77242]: pgmap v2428: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:49:08 smithi153 ceph-mon[70044]: pgmap v2428: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:10.415 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:49:10 smithi083 ceph-mon[79930]: pgmap v2429: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:10.416 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:49:10 smithi083 ceph-mon[77242]: pgmap v2429: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:49:10 smithi153 ceph-mon[70044]: pgmap v2429: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:10.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:49:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:49:10.414Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:49:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:49:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:49:11] "GET /metrics HTTP/1.1" 200 225178 "" "Prometheus/2.33.4" 2022-09-29T06:49:11.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:49:11 smithi083 ceph-mon[79930]: pgmap v2430: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:11.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:49:11 smithi083 ceph-mon[77242]: pgmap v2430: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:12.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:49:11 smithi153 ceph-mon[70044]: pgmap v2430: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:49:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:49:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:49:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:49:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:49:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:49:14.105 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:49:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:49:13.818Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:49:14.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:49:14 smithi083 ceph-mon[79930]: pgmap v2431: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:14.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:49:14 smithi083 ceph-mon[77242]: pgmap v2431: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:14.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:49:14 smithi153 ceph-mon[70044]: pgmap v2431: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:14.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:49:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:49:14.519Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:49:16.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:49:16 smithi083 ceph-mon[79930]: pgmap v2432: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:16.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:49:16 smithi083 ceph-mon[77242]: pgmap v2432: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:49:16 smithi153 ceph-mon[70044]: pgmap v2432: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:18.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:49:18 smithi083 ceph-mon[79930]: pgmap v2433: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:18.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:49:18 smithi083 ceph-mon[77242]: pgmap v2433: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:49:18 smithi153 ceph-mon[70044]: pgmap v2433: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:20.417 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:49:20 smithi083 ceph-mon[79930]: pgmap v2434: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:20.417 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:49:20 smithi083 ceph-mon[77242]: pgmap v2434: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:49:20 smithi153 ceph-mon[70044]: pgmap v2434: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:20.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:49:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:49:20.415Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:49:21.630 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:49:21.921 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:49:21.922 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (83m) 4m ago 83m 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:49:21.922 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (83m) 2m ago 83m 61.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:49:21.922 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (84m) 4m ago 84m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:49:21.923 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (84m) 2m ago 84m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:49:21.923 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (80m) 2m ago 86m 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:49:21.923 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (80m) 4m ago 88m 580M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:49:21.923 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (79m) 4m ago 88m 348M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:49:21.924 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (79m) 2m ago 87m 161M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:49:21.924 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (79m) 4m ago 87m 207M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:49:21.924 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (83m) 4m ago 83m 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:49:21.924 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (83m) 2m ago 83m 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:49:21.924 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (78m) 4m ago 86m 259M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:49:21.925 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (77m) 4m ago 86m 251M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:49:21.925 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (77m) 4m ago 86m 203M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:49:21.925 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (77m) 4m ago 85m 241M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:49:21.925 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (76m) 2m ago 85m 219M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:49:21.926 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (76m) 2m ago 85m 262M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:49:21.926 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (75m) 2m ago 84m 233M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:49:21.926 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (74m) 2m ago 84m 268M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:49:21.926 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (80m) 2m ago 83m 109M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:49:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:49:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:49:21] "GET /metrics HTTP/1.1" 200 225178 "" "Prometheus/2.33.4" 2022-09-29T06:49:21.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:49:21 smithi083 ceph-mon[79930]: pgmap v2435: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:21.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:49:21 smithi083 ceph-mon[77242]: pgmap v2435: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:22.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:49:21 smithi153 ceph-mon[70044]: pgmap v2435: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:22.278 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:49:22.279 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:49:22.279 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:49:22.280 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:49:22.280 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:49:22.280 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:49:22.280 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:49:22.281 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:49:22.281 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:49:22.281 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:49:22.281 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:49:22.282 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:49:22.282 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:49:22.282 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:49:22.282 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:49:22.283 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:49:22.283 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:49:22.283 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:49:22.691 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:49:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:49:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:49:22.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:49:22 smithi083 ceph-mon[79930]: from='client.36969 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:49:22.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:49:22 smithi083 ceph-mon[79930]: from='client.46909 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:49:22.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:49:22 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3079174256' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:49:22.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:49:22 smithi083 ceph-mon[77242]: from='client.36969 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:49:22.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:49:22 smithi083 ceph-mon[77242]: from='client.46909 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:49:22.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:49:22 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3079174256' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:49:23.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:49:22 smithi153 ceph-mon[70044]: from='client.36969 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:49:23.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:49:22 smithi153 ceph-mon[70044]: from='client.46909 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:49:23.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:49:22 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3079174256' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:49:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:49:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:49:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:49:23.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:49:23 smithi083 ceph-mon[79930]: pgmap v2436: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:23.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:49:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:49:23.819Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:49:23.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:49:23 smithi083 ceph-mon[77242]: pgmap v2436: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:24.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:49:23 smithi153 ceph-mon[70044]: pgmap v2436: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:24.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:49:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:49:24.520Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:49:26.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:49:26 smithi083 ceph-mon[79930]: pgmap v2437: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:26.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:49:26 smithi083 ceph-mon[77242]: pgmap v2437: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:49:26 smithi153 ceph-mon[70044]: pgmap v2437: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:28.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:49:28 smithi083 ceph-mon[79930]: pgmap v2438: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:28.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:49:28 smithi083 ceph-mon[77242]: pgmap v2438: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:49:28 smithi153 ceph-mon[70044]: pgmap v2438: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:30.417 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:49:30 smithi083 ceph-mon[79930]: pgmap v2439: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:30.417 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:49:30 smithi083 ceph-mon[77242]: pgmap v2439: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:49:30 smithi153 ceph-mon[70044]: pgmap v2439: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:30.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:49:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:49:30.415Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:49:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:49:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:49:31] "GET /metrics HTTP/1.1" 200 225170 "" "Prometheus/2.33.4" 2022-09-29T06:49:31.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:49:31 smithi083 ceph-mon[79930]: pgmap v2440: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:31.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:49:31 smithi083 ceph-mon[77242]: pgmap v2440: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:32.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:49:31 smithi153 ceph-mon[70044]: pgmap v2440: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:49:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:49:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:49:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:49:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:49:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:49:34.111 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:49:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:49:33.820Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:49:34.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:49:34 smithi083 ceph-mon[79930]: pgmap v2441: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:34.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:49:34 smithi083 ceph-mon[77242]: pgmap v2441: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:49:34 smithi153 ceph-mon[70044]: pgmap v2441: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:34.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:49:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:49:34.521Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:49:36.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:49:36 smithi083 ceph-mon[79930]: pgmap v2442: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:36.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:49:36 smithi083 ceph-mon[77242]: pgmap v2442: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:49:36 smithi153 ceph-mon[70044]: pgmap v2442: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:38.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:49:38 smithi083 ceph-mon[79930]: pgmap v2443: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:38.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:49:38 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:49:38.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:49:38 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:49:38.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:49:38 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:49:38.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:49:38 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:49:38.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:49:38 smithi083 ceph-mon[77242]: pgmap v2443: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:38.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:49:38 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:49:38.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:49:38 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:49:38.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:49:38 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:49:38.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:49:38 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:49:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:49:38 smithi153 ceph-mon[70044]: pgmap v2443: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:38.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:49:38 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:49:38.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:49:38 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:49:38.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:49:38 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:49:38.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:49:38 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:49:40.418 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:49:40 smithi083 ceph-mon[79930]: pgmap v2444: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:40.419 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:49:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:49:40.419 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:49:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:49:40.419 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:49:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:49:40.419 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:49:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:49:40.420 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:49:40 smithi083 ceph-mon[77242]: pgmap v2444: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:40.420 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:49:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:49:40.420 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:49:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:49:40.421 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:49:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:49:40.421 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:49:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:49:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:49:40 smithi153 ceph-mon[70044]: pgmap v2444: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:49:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:49:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:49:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:49:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:49:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:49:40.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:49:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:49:40.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:49:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:49:40.417Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:49:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:49:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:49:41] "GET /metrics HTTP/1.1" 200 225172 "" "Prometheus/2.33.4" 2022-09-29T06:49:41.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:49:41 smithi083 ceph-mon[79930]: pgmap v2445: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:41.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:49:41 smithi083 ceph-mon[77242]: pgmap v2445: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:42.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:49:41 smithi153 ceph-mon[70044]: pgmap v2445: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:49:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:49:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:49:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:49:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:49:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:49:44.114 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:49:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:49:43.820Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:49:44.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:49:44 smithi083 ceph-mon[79930]: pgmap v2446: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:44.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:49:44 smithi083 ceph-mon[77242]: pgmap v2446: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:44.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:49:44 smithi153 ceph-mon[70044]: pgmap v2446: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:44.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:49:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:49:44.522Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:49:46.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:49:46 smithi083 ceph-mon[77242]: pgmap v2447: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:46.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:49:46 smithi083 ceph-mon[79930]: pgmap v2447: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:46.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:49:46 smithi153 ceph-mon[70044]: pgmap v2447: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:48.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:49:48 smithi083 ceph-mon[79930]: pgmap v2448: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:48.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:49:48 smithi083 ceph-mon[77242]: pgmap v2448: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:48.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:49:48 smithi153 ceph-mon[70044]: pgmap v2448: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:50.418 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:49:50 smithi083 ceph-mon[79930]: pgmap v2449: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:50.419 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:49:50 smithi083 ceph-mon[77242]: pgmap v2449: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:49:50 smithi153 ceph-mon[70044]: pgmap v2449: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:50.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:49:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:49:50.417Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:49:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:49:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:49:51] "GET /metrics HTTP/1.1" 200 225172 "" "Prometheus/2.33.4" 2022-09-29T06:49:51.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:49:51 smithi083 ceph-mon[79930]: pgmap v2450: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:51.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:49:51 smithi083 ceph-mon[77242]: pgmap v2450: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:52.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:49:51 smithi153 ceph-mon[70044]: pgmap v2450: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:52.602 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:49:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:49:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:49:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:49:52.891 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:49:52.892 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (83m) 5m ago 84m 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:49:52.892 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (83m) 3m ago 83m 61.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:49:52.892 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (84m) 5m ago 84m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:49:52.893 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (84m) 3m ago 84m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:49:52.893 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (80m) 3m ago 87m 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:49:52.893 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (81m) 5m ago 89m 580M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:49:52.893 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (80m) 5m ago 89m 348M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:49:52.894 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (79m) 3m ago 87m 161M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:49:52.894 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (79m) 5m ago 87m 207M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:49:52.894 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (84m) 5m ago 84m 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:49:52.895 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (84m) 3m ago 84m 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:49:52.895 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (79m) 5m ago 87m 259M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:49:52.895 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (78m) 5m ago 86m 251M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:49:52.895 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (78m) 5m ago 86m 203M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:49:52.896 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (77m) 5m ago 86m 241M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:49:52.896 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (77m) 3m ago 86m 219M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:49:52.896 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (76m) 3m ago 85m 262M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:49:52.896 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (76m) 3m ago 85m 233M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:49:52.897 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (75m) 3m ago 85m 268M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:49:52.897 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (80m) 3m ago 84m 109M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:49:53.247 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:49:53.247 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:49:53.247 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:49:53.248 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:49:53.248 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:49:53.248 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:49:53.248 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:49:53.248 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:49:53.249 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:49:53.249 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:49:53.249 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:49:53.249 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:49:53.250 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:49:53.250 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:49:53.250 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:49:53.250 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:49:53.251 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:49:53.251 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:49:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:49:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:49:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:49:54.117 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:49:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:49:53.820Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:49:54.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:49:54 smithi083 ceph-mon[77242]: from='client.36987 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:49:54.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:49:54 smithi083 ceph-mon[77242]: from='client.36993 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:49:54.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:49:54 smithi083 ceph-mon[77242]: pgmap v2451: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:54.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:49:54 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/301373757' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:49:54.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:49:54 smithi083 ceph-mon[79930]: from='client.36987 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:49:54.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:49:54 smithi083 ceph-mon[79930]: from='client.36993 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:49:54.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:49:54 smithi083 ceph-mon[79930]: pgmap v2451: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:54.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:49:54 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/301373757' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:49:54.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:49:54 smithi153 ceph-mon[70044]: from='client.36987 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:49:54.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:49:54 smithi153 ceph-mon[70044]: from='client.36993 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:49:54.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:49:54 smithi153 ceph-mon[70044]: pgmap v2451: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:54.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:49:54 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/301373757' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:49:54.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:49:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:49:54.521Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:49:56.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:49:56 smithi083 ceph-mon[79930]: pgmap v2452: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:56.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:49:56 smithi083 ceph-mon[77242]: pgmap v2452: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:56.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:49:56 smithi153 ceph-mon[70044]: pgmap v2452: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:58.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:49:58 smithi083 ceph-mon[79930]: pgmap v2453: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:58.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:49:58 smithi083 ceph-mon[77242]: pgmap v2453: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:49:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:49:58 smithi153 ceph-mon[70044]: pgmap v2453: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:00.419 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:50:00 smithi083 ceph-mon[79930]: pgmap v2454: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:00.420 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:50:00 smithi083 ceph-mon[79930]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T06:50:00.420 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:50:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[77219]: 2022-09-29T06:49:59.999+0000 7f160f665700 -1 log_channel(cluster) log [ERR] : overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T06:50:00.420 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:50:00 smithi083 ceph-mon[77242]: pgmap v2454: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:00.421 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:50:00 smithi083 ceph-mon[77242]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T06:50:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:50:00 smithi153 ceph-mon[70044]: pgmap v2454: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:00.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:50:00 smithi153 ceph-mon[70044]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T06:50:00.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:50:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:50:00.418Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:50:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:50:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:50:01] "GET /metrics HTTP/1.1" 200 225177 "" "Prometheus/2.33.4" 2022-09-29T06:50:01.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:50:01 smithi083 ceph-mon[79930]: pgmap v2455: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:01.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:50:01 smithi083 ceph-mon[77242]: pgmap v2455: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:02.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:50:01 smithi153 ceph-mon[70044]: pgmap v2455: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:50:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:50:02.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:50:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:50:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:50:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:50:04.121 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:50:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:50:03.822Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:50:04.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:50:04 smithi083 ceph-mon[79930]: pgmap v2456: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:04.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:50:04 smithi083 ceph-mon[77242]: pgmap v2456: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:50:04 smithi153 ceph-mon[70044]: pgmap v2456: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:04.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:50:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:50:04.522Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:50:06.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:50:06 smithi083 ceph-mon[79930]: pgmap v2457: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:06.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:50:06 smithi083 ceph-mon[77242]: pgmap v2457: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:06.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:50:06 smithi153 ceph-mon[70044]: pgmap v2457: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:08.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:50:08 smithi083 ceph-mon[79930]: pgmap v2458: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:08.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:50:08 smithi083 ceph-mon[77242]: pgmap v2458: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:50:08 smithi153 ceph-mon[70044]: pgmap v2458: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:10.420 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:50:10 smithi083 ceph-mon[79930]: pgmap v2459: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:10.421 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:50:10 smithi083 ceph-mon[77242]: pgmap v2459: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:50:10 smithi153 ceph-mon[70044]: pgmap v2459: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:10.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:50:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:50:10.419Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:50:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:50:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:50:11] "GET /metrics HTTP/1.1" 200 225179 "" "Prometheus/2.33.4" 2022-09-29T06:50:11.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:50:11 smithi083 ceph-mon[79930]: pgmap v2460: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:11.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:50:11 smithi083 ceph-mon[77242]: pgmap v2460: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:12.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:50:11 smithi153 ceph-mon[70044]: pgmap v2460: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:50:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:50:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:50:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:50:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:50:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:50:14.124 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:50:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:50:13.822Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:50:14.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:50:14 smithi083 ceph-mon[79930]: pgmap v2461: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:14.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:50:14 smithi083 ceph-mon[77242]: pgmap v2461: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:14.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:50:14 smithi153 ceph-mon[70044]: pgmap v2461: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:14.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:50:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:50:14.524Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:50:16.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:50:16 smithi083 ceph-mon[79930]: pgmap v2462: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:16.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:50:16 smithi083 ceph-mon[77242]: pgmap v2462: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:16.561 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:50:16 smithi153 ceph-mon[70044]: pgmap v2462: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:18.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:50:18 smithi083 ceph-mon[79930]: pgmap v2463: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:18.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:50:18 smithi083 ceph-mon[77242]: pgmap v2463: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:50:18 smithi153 ceph-mon[70044]: pgmap v2463: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:20.421 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:50:20 smithi083 ceph-mon[79930]: pgmap v2464: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:20.422 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:50:20 smithi083 ceph-mon[77242]: pgmap v2464: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:20.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:50:20 smithi153 ceph-mon[70044]: pgmap v2464: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:20.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:50:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:50:20.419Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:50:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:50:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:50:21] "GET /metrics HTTP/1.1" 200 225179 "" "Prometheus/2.33.4" 2022-09-29T06:50:21.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:50:21 smithi083 ceph-mon[79930]: pgmap v2465: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:21.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:50:21 smithi083 ceph-mon[77242]: pgmap v2465: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:22.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:50:21 smithi153 ceph-mon[70044]: pgmap v2465: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:50:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:50:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:50:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:50:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:50:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:50:23.581 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:50:23.867 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:50:23.868 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (84m) 5m ago 84m 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:50:23.868 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (84m) 3m ago 84m 61.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:50:23.868 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (85m) 5m ago 85m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:50:23.869 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (85m) 3m ago 85m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:50:23.869 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (81m) 3m ago 88m 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:50:23.869 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (81m) 5m ago 89m 580M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:50:23.869 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (80m) 5m ago 89m 348M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:50:23.870 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (80m) 3m ago 88m 161M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:50:23.870 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (80m) 5m ago 88m 207M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:50:23.870 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (84m) 5m ago 84m 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:50:23.870 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (84m) 3m ago 84m 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:50:23.870 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (79m) 5m ago 87m 259M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:50:23.871 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (79m) 5m ago 87m 251M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:50:23.871 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (78m) 5m ago 87m 203M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:50:23.871 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (78m) 5m ago 86m 241M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:50:23.871 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (77m) 3m ago 86m 219M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:50:23.872 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (77m) 3m ago 86m 262M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:50:23.872 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (76m) 3m ago 85m 233M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:50:23.872 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (75m) 3m ago 85m 268M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:50:23.872 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (81m) 3m ago 84m 109M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:50:24.127 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:50:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:50:23.821Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:50:24.230 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:50:24.230 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:50:24.231 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:50:24.231 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:50:24.231 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:50:24.231 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:50:24.232 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:50:24.232 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:50:24.232 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:50:24.232 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:50:24.233 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:50:24.233 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:50:24.233 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:50:24.233 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:50:24.234 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:50:24.234 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:50:24.234 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:50:24.235 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:50:24.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:50:24 smithi083 ceph-mon[79930]: pgmap v2466: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:24.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:50:24 smithi083 ceph-mon[77242]: pgmap v2466: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:50:24 smithi153 ceph-mon[70044]: pgmap v2466: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:24.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:50:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:50:24.523Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:50:25.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:50:25 smithi083 ceph-mon[77242]: from='client.46939 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:50:25.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:50:25 smithi083 ceph-mon[77242]: from='client.46945 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:50:25.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:50:25 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1519457441' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:50:25.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:50:25 smithi083 ceph-mon[79930]: from='client.46939 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:50:25.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:50:25 smithi083 ceph-mon[79930]: from='client.46945 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:50:25.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:50:25 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1519457441' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:50:25.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:50:25 smithi153 ceph-mon[70044]: from='client.46939 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:50:25.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:50:25 smithi153 ceph-mon[70044]: from='client.46945 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:50:25.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:50:25 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1519457441' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:50:26.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:50:26 smithi083 ceph-mon[79930]: pgmap v2467: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:26.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:50:26 smithi083 ceph-mon[77242]: pgmap v2467: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:50:26 smithi153 ceph-mon[70044]: pgmap v2467: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:28.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:50:28 smithi083 ceph-mon[79930]: pgmap v2468: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:28.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:50:28 smithi083 ceph-mon[77242]: pgmap v2468: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:50:28 smithi153 ceph-mon[70044]: pgmap v2468: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:30.421 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:50:30 smithi083 ceph-mon[79930]: pgmap v2469: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:30.422 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:50:30 smithi083 ceph-mon[77242]: pgmap v2469: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:50:30 smithi153 ceph-mon[70044]: pgmap v2469: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:30.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:50:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:50:30.420Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:50:31.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:50:31 smithi083 ceph-mon[77242]: pgmap v2470: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:50:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:50:31] "GET /metrics HTTP/1.1" 200 225177 "" "Prometheus/2.33.4" 2022-09-29T06:50:31.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:50:31 smithi083 ceph-mon[79930]: pgmap v2470: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:32.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:50:31 smithi153 ceph-mon[70044]: pgmap v2470: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:50:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:50:32.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:50:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:50:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:50:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:50:34.129 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:50:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:50:33.823Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:50:34.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:50:34 smithi083 ceph-mon[79930]: pgmap v2471: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:34.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:50:34 smithi083 ceph-mon[77242]: pgmap v2471: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:50:34 smithi153 ceph-mon[70044]: pgmap v2471: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:34.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:50:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:50:34.524Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:50:36.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:50:36 smithi083 ceph-mon[79930]: pgmap v2472: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:36.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:50:36 smithi083 ceph-mon[77242]: pgmap v2472: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:50:36 smithi153 ceph-mon[70044]: pgmap v2472: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:38.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:50:38 smithi083 ceph-mon[79930]: pgmap v2473: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:38.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:50:38 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:50:38.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:50:38 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:50:38.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:50:38 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:50:38.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:50:38 smithi083 ceph-mon[77242]: pgmap v2473: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:38.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:50:38 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:50:38.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:50:38 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:50:38.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:50:38 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:50:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:50:38 smithi153 ceph-mon[70044]: pgmap v2473: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:38.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:50:38 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:50:38.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:50:38 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:50:38.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:50:38 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:50:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:50:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:50:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:50:39 smithi153 ceph-mon[70044]: pgmap v2474: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:50:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:50:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:50:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:50:40.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:50:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:50:40.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:50:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:50:40.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:50:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:50:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:50:39 smithi083 ceph-mon[79930]: pgmap v2474: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:50:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:50:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:50:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:50:40.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:50:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:50:40.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:50:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:50:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:50:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:50:40.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:50:39 smithi083 ceph-mon[77242]: pgmap v2474: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:40.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:50:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:50:40.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:50:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:50:40.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:50:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:50:40.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:50:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:50:40.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:50:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:50:40.420Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:50:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:50:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:50:41] "GET /metrics HTTP/1.1" 200 225176 "" "Prometheus/2.33.4" 2022-09-29T06:50:41.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:50:41 smithi083 ceph-mon[79930]: pgmap v2475: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:41.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:50:41 smithi083 ceph-mon[77242]: pgmap v2475: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:42.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:50:41 smithi153 ceph-mon[70044]: pgmap v2475: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:50:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:50:42.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:50:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:50:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:50:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:50:44.132 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:50:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:50:43.823Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:50:44.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:50:44 smithi083 ceph-mon[79930]: pgmap v2476: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:44.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:50:44 smithi083 ceph-mon[77242]: pgmap v2476: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:44.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:50:44 smithi153 ceph-mon[70044]: pgmap v2476: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:44.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:50:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:50:44.524Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:50:46.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:50:46 smithi083 ceph-mon[79930]: pgmap v2477: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:46.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:50:46 smithi083 ceph-mon[77242]: pgmap v2477: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:46.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:50:46 smithi153 ceph-mon[70044]: pgmap v2477: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:48.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:50:48 smithi083 ceph-mon[77242]: pgmap v2478: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:48.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:50:48 smithi083 ceph-mon[79930]: pgmap v2478: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:48.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:50:48 smithi153 ceph-mon[70044]: pgmap v2478: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:50.422 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:50:50 smithi083 ceph-mon[79930]: pgmap v2479: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:50.423 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:50:50 smithi083 ceph-mon[77242]: pgmap v2479: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:50:50 smithi153 ceph-mon[70044]: pgmap v2479: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:50.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:50:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:50:50.421Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:50:51.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:50:51 smithi083 ceph-mon[79930]: pgmap v2480: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:51.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:50:51 smithi083 ceph-mon[77242]: pgmap v2480: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:50:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:50:51] "GET /metrics HTTP/1.1" 200 225176 "" "Prometheus/2.33.4" 2022-09-29T06:50:52.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:50:51 smithi153 ceph-mon[70044]: pgmap v2480: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:50:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:50:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:50:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:50:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:50:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:50:54.136 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:50:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:50:53.824Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:50:54.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:50:54 smithi083 ceph-mon[79930]: pgmap v2481: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:54.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:50:54 smithi083 ceph-mon[77242]: pgmap v2481: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:54.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:50:54 smithi153 ceph-mon[70044]: pgmap v2481: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:54.558 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:50:54.846 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:50:54.847 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (84m) 6m ago 85m 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:50:54.847 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (84m) 4m ago 84m 61.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:50:54.847 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (85m) 6m ago 85m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:50:54.847 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (85m) 4m ago 85m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:50:54.848 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (81m) 4m ago 88m 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:50:54.848 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (82m) 6m ago 90m 580M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:50:54.848 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (81m) 6m ago 90m 348M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:50:54.849 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (80m) 4m ago 88m 161M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:50:54.849 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (81m) 6m ago 88m 207M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:50:54.849 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (85m) 6m ago 85m 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:50:54.849 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (85m) 4m ago 85m 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:50:54.850 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (80m) 6m ago 88m 259M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:50:54.850 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (79m) 6m ago 87m 251M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:50:54.850 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (79m) 6m ago 87m 203M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:50:54.850 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (78m) 6m ago 87m 241M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:50:54.851 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (78m) 4m ago 87m 219M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:50:54.851 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (77m) 4m ago 86m 262M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:50:54.851 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (77m) 4m ago 86m 233M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:50:54.851 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (76m) 4m ago 86m 268M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:50:54.852 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (81m) 4m ago 85m 109M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:50:54.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:50:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:50:54.526Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:50:55.205 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:50:55.206 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:50:55.206 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:50:55.206 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:50:55.207 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:50:55.207 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:50:55.207 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:50:55.208 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:50:55.208 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:50:55.208 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:50:55.208 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:50:55.208 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:50:55.209 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:50:55.209 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:50:55.209 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:50:55.209 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:50:55.210 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:50:55.210 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:50:56.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:50:56 smithi083 ceph-mon[79930]: from='client.46957 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:50:56.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:50:56 smithi083 ceph-mon[79930]: from='client.37029 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:50:56.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:50:56 smithi083 ceph-mon[79930]: pgmap v2482: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:56.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:50:56 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1871067270' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:50:56.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:50:56 smithi083 ceph-mon[77242]: from='client.46957 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:50:56.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:50:56 smithi083 ceph-mon[77242]: from='client.37029 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:50:56.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:50:56 smithi083 ceph-mon[77242]: pgmap v2482: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:56.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:50:56 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1871067270' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:50:56.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:50:56 smithi153 ceph-mon[70044]: from='client.46957 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:50:56.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:50:56 smithi153 ceph-mon[70044]: from='client.37029 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:50:56.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:50:56 smithi153 ceph-mon[70044]: pgmap v2482: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:56.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:50:56 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1871067270' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:50:58.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:50:58 smithi083 ceph-mon[79930]: pgmap v2483: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:58.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:50:58 smithi083 ceph-mon[77242]: pgmap v2483: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:50:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:50:58 smithi153 ceph-mon[70044]: pgmap v2483: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:00.424 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:51:00 smithi083 ceph-mon[79930]: pgmap v2484: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:00.424 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:51:00 smithi083 ceph-mon[77242]: pgmap v2484: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:51:00 smithi153 ceph-mon[70044]: pgmap v2484: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:00.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:51:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:51:00.422Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:51:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:51:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:51:01] "GET /metrics HTTP/1.1" 200 225176 "" "Prometheus/2.33.4" 2022-09-29T06:51:01.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:51:01 smithi083 ceph-mon[79930]: pgmap v2485: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:01.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:51:01 smithi083 ceph-mon[77242]: pgmap v2485: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:02.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:51:01 smithi153 ceph-mon[70044]: pgmap v2485: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:51:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:51:02.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:51:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:51:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:51:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:51:04.139 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:51:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:51:03.824Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:51:04.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:51:04 smithi083 ceph-mon[77242]: pgmap v2486: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:04.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:51:04 smithi083 ceph-mon[79930]: pgmap v2486: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:51:04 smithi153 ceph-mon[70044]: pgmap v2486: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:04.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:51:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:51:04.525Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:51:06.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:51:06 smithi083 ceph-mon[79930]: pgmap v2487: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:06.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:51:06 smithi083 ceph-mon[77242]: pgmap v2487: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:06.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:51:06 smithi153 ceph-mon[70044]: pgmap v2487: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:08.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:51:08 smithi083 ceph-mon[79930]: pgmap v2488: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:08.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:51:08 smithi083 ceph-mon[77242]: pgmap v2488: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:51:08 smithi153 ceph-mon[70044]: pgmap v2488: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:10.424 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:51:10 smithi083 ceph-mon[79930]: pgmap v2489: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:10.424 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:51:10 smithi083 ceph-mon[77242]: pgmap v2489: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:51:10 smithi153 ceph-mon[70044]: pgmap v2489: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:10.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:51:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:51:10.422Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:51:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:51:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:51:11] "GET /metrics HTTP/1.1" 200 225177 "" "Prometheus/2.33.4" 2022-09-29T06:51:11.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:51:11 smithi083 ceph-mon[79930]: pgmap v2490: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:11.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:51:11 smithi083 ceph-mon[77242]: pgmap v2490: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:12.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:51:11 smithi153 ceph-mon[70044]: pgmap v2490: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:51:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:51:12.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:51:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:51:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:51:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:51:14.142 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:51:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:51:13.824Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:51:14.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:51:14 smithi083 ceph-mon[79930]: pgmap v2491: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:14.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:51:14 smithi083 ceph-mon[77242]: pgmap v2491: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:14.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:51:14 smithi153 ceph-mon[70044]: pgmap v2491: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:14.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:51:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:51:14.527Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:51:16.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:51:16 smithi083 ceph-mon[79930]: pgmap v2492: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:16.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:51:16 smithi083 ceph-mon[77242]: pgmap v2492: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:51:16 smithi153 ceph-mon[70044]: pgmap v2492: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:18.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:51:18 smithi083 ceph-mon[79930]: pgmap v2493: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:18.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:51:18 smithi083 ceph-mon[77242]: pgmap v2493: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:18.562 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:51:18 smithi153 ceph-mon[70044]: pgmap v2493: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:20.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:51:20 smithi083 ceph-mon[79930]: pgmap v2494: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:20.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:51:20 smithi083 ceph-mon[77242]: pgmap v2494: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:51:20 smithi153 ceph-mon[70044]: pgmap v2494: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:20.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:51:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:51:20.423Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:51:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:51:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:51:21] "GET /metrics HTTP/1.1" 200 225177 "" "Prometheus/2.33.4" 2022-09-29T06:51:21.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:51:21 smithi083 ceph-mon[79930]: pgmap v2495: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:21.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:51:21 smithi083 ceph-mon[77242]: pgmap v2495: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:22.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:51:21 smithi153 ceph-mon[70044]: pgmap v2495: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:51:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:51:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:51:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:51:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:51:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:51:24.145 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:51:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:51:23.825Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:51:24.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:51:24 smithi083 ceph-mon[79930]: pgmap v2496: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:24.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:51:24 smithi083 ceph-mon[77242]: pgmap v2496: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:51:24 smithi153 ceph-mon[70044]: pgmap v2496: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:24.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:51:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:51:24.527Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:51:25.530 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:51:25.820 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:51:25.821 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (85m) 6m ago 85m 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:51:25.821 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (85m) 4m ago 85m 61.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:51:25.822 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (86m) 6m ago 86m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:51:25.822 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (86m) 4m ago 86m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:51:25.822 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (82m) 4m ago 89m 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:51:25.822 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (82m) 6m ago 90m 580M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:51:25.823 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (81m) 6m ago 90m 348M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:51:25.823 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (81m) 4m ago 89m 161M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:51:25.823 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (81m) 6m ago 89m 207M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:51:25.823 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (85m) 6m ago 85m 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:51:25.823 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (85m) 4m ago 85m 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:51:25.824 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (80m) 6m ago 88m 259M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:51:25.824 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (80m) 6m ago 88m 251M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:51:25.824 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (79m) 6m ago 88m 203M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:51:25.824 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (79m) 6m ago 87m 241M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:51:25.825 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (78m) 4m ago 87m 219M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:51:25.825 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (78m) 4m ago 87m 262M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:51:25.825 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (77m) 4m ago 87m 233M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:51:25.825 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (77m) 4m ago 86m 268M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:51:25.826 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (82m) 4m ago 85m 109M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:51:26.176 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:51:26.176 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:51:26.176 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:51:26.177 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:51:26.177 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:51:26.177 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:51:26.177 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:51:26.178 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:51:26.178 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:51:26.178 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:51:26.179 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:51:26.179 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:51:26.179 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:51:26.179 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:51:26.180 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:51:26.180 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:51:26.180 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:51:26.180 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:51:26.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:51:26 smithi083 ceph-mon[79930]: pgmap v2497: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:26.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:51:26 smithi083 ceph-mon[79930]: from='client.46975 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:51:26.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:51:26 smithi083 ceph-mon[77242]: pgmap v2497: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:26.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:51:26 smithi083 ceph-mon[77242]: from='client.46975 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:51:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:51:26 smithi153 ceph-mon[70044]: pgmap v2497: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:26.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:51:26 smithi153 ceph-mon[70044]: from='client.46975 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:51:27.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:51:27 smithi083 ceph-mon[79930]: from='client.46981 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:51:27.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:51:27 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3069192671' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:51:27.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:51:27 smithi083 ceph-mon[77242]: from='client.46981 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:51:27.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:51:27 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3069192671' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:51:27.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:51:27 smithi153 ceph-mon[70044]: from='client.46981 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:51:27.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:51:27 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3069192671' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:51:28.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:51:28 smithi083 ceph-mon[79930]: pgmap v2498: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:28.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:51:28 smithi083 ceph-mon[77242]: pgmap v2498: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:51:28 smithi153 ceph-mon[70044]: pgmap v2498: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:30.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:51:30 smithi083 ceph-mon[79930]: pgmap v2499: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:30.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:51:30 smithi083 ceph-mon[77242]: pgmap v2499: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:51:30 smithi153 ceph-mon[70044]: pgmap v2499: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:30.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:51:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:51:30.424Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:51:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:51:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:51:31] "GET /metrics HTTP/1.1" 200 225177 "" "Prometheus/2.33.4" 2022-09-29T06:51:31.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:51:31 smithi083 ceph-mon[79930]: pgmap v2500: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:31.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:51:31 smithi083 ceph-mon[77242]: pgmap v2500: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:32.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:51:31 smithi153 ceph-mon[70044]: pgmap v2500: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:51:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:51:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:51:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:51:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:51:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:51:34.148 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:51:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:51:33.826Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:51:34.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:51:34 smithi083 ceph-mon[79930]: pgmap v2501: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:34.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:51:34 smithi083 ceph-mon[77242]: pgmap v2501: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:51:34 smithi153 ceph-mon[70044]: pgmap v2501: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:34.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:51:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:51:34.528Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:51:36.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:51:36 smithi083 ceph-mon[79930]: pgmap v2502: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:36.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:51:36 smithi083 ceph-mon[77242]: pgmap v2502: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:51:36 smithi153 ceph-mon[70044]: pgmap v2502: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:38.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:51:38 smithi083 ceph-mon[79930]: pgmap v2503: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:38.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:51:38 smithi083 ceph-mon[77242]: pgmap v2503: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:51:38 smithi153 ceph-mon[70044]: pgmap v2503: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:39.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:51:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:51:39.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:51:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:51:39.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:51:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:51:39.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:51:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:51:39.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:51:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:51:39.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:51:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:51:39.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:51:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:51:39.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:51:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:51:39.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:51:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:51:40.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:51:40 smithi083 ceph-mon[77242]: pgmap v2504: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:40.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:51:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:51:40.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:51:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:51:40.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:51:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:51:40.427 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:51:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:51:40.427 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:51:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:51:40.427 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:51:40 smithi083 ceph-mon[79930]: pgmap v2504: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:40.428 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:51:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:51:40.428 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:51:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:51:40.428 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:51:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:51:40.428 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:51:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:51:40.429 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:51:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:51:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:51:40 smithi153 ceph-mon[70044]: pgmap v2504: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:51:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:51:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:51:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:51:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:51:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:51:40.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:51:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:51:40.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:51:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:51:40.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:51:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:51:40.424Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:51:41.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:51:41 smithi083 ceph-mon[77242]: pgmap v2505: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:51:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:51:41] "GET /metrics HTTP/1.1" 200 225176 "" "Prometheus/2.33.4" 2022-09-29T06:51:41.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:51:41 smithi083 ceph-mon[79930]: pgmap v2505: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:42.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:51:41 smithi153 ceph-mon[70044]: pgmap v2505: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:51:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:51:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:51:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:51:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:51:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:51:44.150 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:51:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:51:43.826Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:51:44.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:51:44 smithi083 ceph-mon[79930]: pgmap v2506: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:44.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:51:44 smithi083 ceph-mon[77242]: pgmap v2506: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:44.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:51:44 smithi153 ceph-mon[70044]: pgmap v2506: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:44.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:51:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:51:44.529Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:51:46.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:51:46 smithi083 ceph-mon[79930]: pgmap v2507: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:46.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:51:46 smithi083 ceph-mon[77242]: pgmap v2507: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:46.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:51:46 smithi153 ceph-mon[70044]: pgmap v2507: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:48.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:51:48 smithi083 ceph-mon[79930]: pgmap v2508: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:48.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:51:48 smithi083 ceph-mon[77242]: pgmap v2508: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:48.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:51:48 smithi153 ceph-mon[70044]: pgmap v2508: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:50.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:51:50 smithi083 ceph-mon[79930]: pgmap v2509: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:50.427 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:51:50 smithi083 ceph-mon[77242]: pgmap v2509: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:51:50 smithi153 ceph-mon[70044]: pgmap v2509: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:50.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:51:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:51:50.425Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:51:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:51:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:51:51] "GET /metrics HTTP/1.1" 200 225176 "" "Prometheus/2.33.4" 2022-09-29T06:51:51.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:51:51 smithi083 ceph-mon[79930]: pgmap v2510: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:51.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:51:51 smithi083 ceph-mon[77242]: pgmap v2510: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:52.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:51:51 smithi153 ceph-mon[70044]: pgmap v2510: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:51:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:51:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:51:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:51:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:51:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:51:54.153 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:51:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:51:53.826Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:51:54.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:51:54 smithi083 ceph-mon[79930]: pgmap v2511: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:54.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:51:54 smithi083 ceph-mon[77242]: pgmap v2511: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:54.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:51:54 smithi153 ceph-mon[70044]: pgmap v2511: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:54.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:51:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:51:54.529Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:51:56.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:51:56 smithi083 ceph-mon[79930]: pgmap v2512: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:56.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:51:56 smithi083 ceph-mon[77242]: pgmap v2512: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:56.505 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:51:56.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:51:56 smithi153 ceph-mon[70044]: pgmap v2512: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:56.793 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:51:56.794 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (85m) 7m ago 86m 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:51:56.794 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (85m) 5m ago 85m 61.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:51:56.794 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (87m) 7m ago 87m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:51:56.795 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (86m) 5m ago 86m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:51:56.795 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (82m) 5m ago 89m 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:51:56.795 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (83m) 7m ago 91m 580M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:51:56.795 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (82m) 7m ago 91m 348M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:51:56.796 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (81m) 5m ago 89m 161M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:51:56.796 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (82m) 7m ago 89m 207M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:51:56.796 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (86m) 7m ago 86m 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:51:56.796 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (86m) 5m ago 86m 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:51:56.796 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (81m) 7m ago 89m 259M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:51:56.797 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (80m) 7m ago 88m 251M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:51:56.797 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (80m) 7m ago 88m 203M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:51:56.797 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (79m) 7m ago 88m 241M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:51:56.797 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (79m) 5m ago 88m 219M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:51:56.798 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (78m) 5m ago 87m 262M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:51:56.798 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (78m) 5m ago 87m 233M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:51:56.798 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (77m) 5m ago 87m 268M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:51:56.798 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (82m) 5m ago 86m 109M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:51:57.150 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:51:57.151 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:51:57.151 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:51:57.151 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:51:57.151 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:51:57.152 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:51:57.152 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:51:57.152 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:51:57.152 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:51:57.153 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:51:57.153 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:51:57.153 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:51:57.153 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:51:57.154 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:51:57.154 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:51:57.154 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:51:57.154 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:51:57.155 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:51:57.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:51:57 smithi083 ceph-mon[79930]: from='client.37059 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:51:57.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:51:57 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4057858409' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:51:57.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:51:57 smithi083 ceph-mon[77242]: from='client.37059 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:51:57.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:51:57 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4057858409' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:51:57.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:51:57 smithi153 ceph-mon[70044]: from='client.37059 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:51:57.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:51:57 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4057858409' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:51:58.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:51:58 smithi083 ceph-mon[77242]: from='client.46999 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:51:58.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:51:58 smithi083 ceph-mon[77242]: pgmap v2513: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:58.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:51:58 smithi083 ceph-mon[79930]: from='client.46999 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:51:58.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:51:58 smithi083 ceph-mon[79930]: pgmap v2513: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:51:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:51:58 smithi153 ceph-mon[70044]: from='client.46999 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:51:58.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:51:58 smithi153 ceph-mon[70044]: pgmap v2513: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:00.427 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:52:00 smithi083 ceph-mon[79930]: pgmap v2514: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:00.427 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:52:00 smithi083 ceph-mon[77242]: pgmap v2514: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:52:00 smithi153 ceph-mon[70044]: pgmap v2514: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:00.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:52:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:52:00.425Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:52:01.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:52:01 smithi153 ceph-mon[70044]: pgmap v2515: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:52:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:52:01] "GET /metrics HTTP/1.1" 200 225174 "" "Prometheus/2.33.4" 2022-09-29T06:52:01.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:52:01 smithi083 ceph-mon[79930]: pgmap v2515: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:01.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:52:01 smithi083 ceph-mon[77242]: pgmap v2515: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:52:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:52:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:52:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:52:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:52:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:52:04.157 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:52:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:52:03.827Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:52:04.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:52:04 smithi083 ceph-mon[79930]: pgmap v2516: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:04.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:52:04 smithi083 ceph-mon[77242]: pgmap v2516: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:52:04 smithi153 ceph-mon[70044]: pgmap v2516: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:04.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:52:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:52:04.530Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:52:06.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:52:06 smithi083 ceph-mon[79930]: pgmap v2517: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:06.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:52:06 smithi083 ceph-mon[77242]: pgmap v2517: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:06.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:52:06 smithi153 ceph-mon[70044]: pgmap v2517: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:08.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:52:08 smithi083 ceph-mon[79930]: pgmap v2518: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:08.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:52:08 smithi083 ceph-mon[77242]: pgmap v2518: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:52:08 smithi153 ceph-mon[70044]: pgmap v2518: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:10.427 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:52:10 smithi083 ceph-mon[79930]: pgmap v2519: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:10.428 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:52:10 smithi083 ceph-mon[77242]: pgmap v2519: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:52:10 smithi153 ceph-mon[70044]: pgmap v2519: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:10.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:52:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:52:10.426Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:52:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:52:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:52:11] "GET /metrics HTTP/1.1" 200 225177 "" "Prometheus/2.33.4" 2022-09-29T06:52:11.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:52:11 smithi083 ceph-mon[79930]: pgmap v2520: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:11.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:52:11 smithi083 ceph-mon[77242]: pgmap v2520: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:12.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:52:11 smithi153 ceph-mon[70044]: pgmap v2520: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:52:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:52:12.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:52:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:52:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:52:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:52:14.159 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:52:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:52:13.827Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:52:14.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:52:14 smithi083 ceph-mon[79930]: pgmap v2521: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:14.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:52:14 smithi083 ceph-mon[77242]: pgmap v2521: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:14.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:52:14 smithi153 ceph-mon[70044]: pgmap v2521: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:14.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:52:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:52:14.530Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:52:16.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:52:16 smithi083 ceph-mon[79930]: pgmap v2522: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:16.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:52:16 smithi083 ceph-mon[77242]: pgmap v2522: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:52:16 smithi153 ceph-mon[70044]: pgmap v2522: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:18.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:52:18 smithi083 ceph-mon[77242]: pgmap v2523: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:18.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:52:18 smithi083 ceph-mon[79930]: pgmap v2523: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:52:18 smithi153 ceph-mon[70044]: pgmap v2523: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:20.428 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:52:20 smithi083 ceph-mon[79930]: pgmap v2524: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:20.428 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:52:20 smithi083 ceph-mon[77242]: pgmap v2524: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:52:20 smithi153 ceph-mon[70044]: pgmap v2524: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:20.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:52:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:52:20.427Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:52:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:52:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:52:21] "GET /metrics HTTP/1.1" 200 225177 "" "Prometheus/2.33.4" 2022-09-29T06:52:21.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:52:21 smithi083 ceph-mon[79930]: pgmap v2525: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:21.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:52:21 smithi083 ceph-mon[77242]: pgmap v2525: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:22.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:52:21 smithi153 ceph-mon[70044]: pgmap v2525: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:52:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:52:22.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:52:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:52:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:52:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:52:24.163 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:52:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:52:23.828Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:52:24.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:52:24 smithi083 ceph-mon[79930]: pgmap v2526: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:24.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:52:24 smithi083 ceph-mon[77242]: pgmap v2526: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:52:24 smithi153 ceph-mon[70044]: pgmap v2526: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:24.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:52:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:52:24.531Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:52:26.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:52:26 smithi083 ceph-mon[79930]: pgmap v2527: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:26.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:52:26 smithi083 ceph-mon[77242]: pgmap v2527: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:52:26 smithi153 ceph-mon[70044]: pgmap v2527: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:27.477 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:52:27.766 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:52:27.766 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (86m) 7m ago 86m 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:52:27.767 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (86m) 5m ago 86m 61.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:52:27.767 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (87m) 7m ago 87m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:52:27.767 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (87m) 5m ago 87m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:52:27.767 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (83m) 5m ago 90m 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:52:27.768 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (83m) 7m ago 91m 580M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:52:27.768 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (82m) 7m ago 91m 348M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:52:27.768 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (82m) 5m ago 90m 161M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:52:27.768 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (82m) 7m ago 90m 207M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:52:27.768 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (86m) 7m ago 86m 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:52:27.769 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (86m) 5m ago 86m 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:52:27.769 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (81m) 7m ago 89m 259M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:52:27.769 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (81m) 7m ago 89m 251M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:52:27.769 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (80m) 7m ago 89m 203M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:52:27.770 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (80m) 7m ago 88m 241M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:52:27.770 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (79m) 5m ago 88m 219M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:52:27.770 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (79m) 5m ago 88m 262M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:52:27.771 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (78m) 5m ago 88m 233M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:52:27.771 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (78m) 5m ago 87m 268M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:52:27.771 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (83m) 5m ago 86m 109M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:52:28.121 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:52:28.121 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:52:28.122 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:52:28.122 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:52:28.122 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:52:28.123 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:52:28.123 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:52:28.123 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:52:28.123 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:52:28.124 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:52:28.124 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:52:28.124 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:52:28.124 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:52:28.124 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:52:28.125 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:52:28.125 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:52:28.125 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:52:28.125 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:52:28.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:52:28 smithi083 ceph-mon[79930]: pgmap v2528: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:28.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:52:28 smithi083 ceph-mon[79930]: from='client.37077 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:52:28.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:52:28 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3896391166' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:52:28.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:52:28 smithi083 ceph-mon[77242]: pgmap v2528: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:28.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:52:28 smithi083 ceph-mon[77242]: from='client.37077 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:52:28.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:52:28 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3896391166' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:52:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:52:28 smithi153 ceph-mon[70044]: pgmap v2528: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:28.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:52:28 smithi153 ceph-mon[70044]: from='client.37077 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:52:28.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:52:28 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3896391166' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:52:29.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:52:29 smithi083 ceph-mon[79930]: from='client.37083 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:52:29.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:52:29 smithi083 ceph-mon[77242]: from='client.37083 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:52:29.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:52:29 smithi153 ceph-mon[70044]: from='client.37083 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:52:30.429 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:52:30 smithi083 ceph-mon[79930]: pgmap v2529: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:30.429 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:52:30 smithi083 ceph-mon[77242]: pgmap v2529: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:52:30 smithi153 ceph-mon[70044]: pgmap v2529: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:30.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:52:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:52:30.427Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:52:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:52:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:52:31] "GET /metrics HTTP/1.1" 200 225178 "" "Prometheus/2.33.4" 2022-09-29T06:52:31.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:52:31 smithi083 ceph-mon[79930]: pgmap v2530: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:31.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:52:31 smithi083 ceph-mon[77242]: pgmap v2530: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:32.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:52:31 smithi153 ceph-mon[70044]: pgmap v2530: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:52:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:52:32.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:52:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:52:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:52:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:52:34.165 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:52:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:52:33.828Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:52:34.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:52:34 smithi083 ceph-mon[79930]: pgmap v2531: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:34.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:52:34 smithi083 ceph-mon[77242]: pgmap v2531: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:52:34 smithi153 ceph-mon[70044]: pgmap v2531: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:34.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:52:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:52:34.531Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:52:36.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:52:36 smithi083 ceph-mon[79930]: pgmap v2532: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:36.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:52:36 smithi083 ceph-mon[77242]: pgmap v2532: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:52:36 smithi153 ceph-mon[70044]: pgmap v2532: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:38.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:52:38 smithi083 ceph-mon[79930]: pgmap v2533: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:38.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:52:38 smithi083 ceph-mon[77242]: pgmap v2533: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:52:38 smithi153 ceph-mon[70044]: pgmap v2533: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:40.428 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:52:40 smithi083 ceph-mon[79930]: pgmap v2534: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:40.429 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:52:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:52:40.429 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:52:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:52:40.430 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:52:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:52:40.430 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:52:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:52:40.430 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:52:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:52:40.430 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:52:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:52:40.431 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:52:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:52:40.431 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:52:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:52:40.431 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:52:40 smithi083 ceph-mon[77242]: pgmap v2534: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:40.432 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:52:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:52:40.432 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:52:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:52:40.432 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:52:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:52:40.432 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:52:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:52:40.433 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:52:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:52:40.433 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:52:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:52:40.433 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:52:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:52:40.433 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:52:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:52:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:52:40 smithi153 ceph-mon[70044]: pgmap v2534: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:52:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:52:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:52:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:52:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:52:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:52:40.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:52:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:52:40.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:52:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:52:40.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:52:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:52:40.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:52:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:52:40.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:52:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:52:40.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:52:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:52:40.427Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:52:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:52:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:52:41] "GET /metrics HTTP/1.1" 200 225175 "" "Prometheus/2.33.4" 2022-09-29T06:52:41.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:52:41 smithi083 ceph-mon[79930]: pgmap v2535: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:41.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:52:41 smithi083 ceph-mon[77242]: pgmap v2535: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:42.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:52:41 smithi153 ceph-mon[70044]: pgmap v2535: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:52:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:52:42.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:52:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:52:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:52:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:52:44.169 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:52:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:52:43.829Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:52:44.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:52:44 smithi083 ceph-mon[77242]: pgmap v2536: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:44.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:52:44 smithi083 ceph-mon[79930]: pgmap v2536: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:44.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:52:44 smithi153 ceph-mon[70044]: pgmap v2536: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:44.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:52:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:52:44.532Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:52:46.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:52:46 smithi083 ceph-mon[77242]: pgmap v2537: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:46.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:52:46 smithi083 ceph-mon[79930]: pgmap v2537: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:46.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:52:46 smithi153 ceph-mon[70044]: pgmap v2537: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:48.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:52:48 smithi083 ceph-mon[77242]: pgmap v2538: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:48.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:52:48 smithi083 ceph-mon[79930]: pgmap v2538: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:48.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:52:48 smithi153 ceph-mon[70044]: pgmap v2538: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:50.429 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:52:50 smithi083 ceph-mon[79930]: pgmap v2539: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:50.430 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:52:50 smithi083 ceph-mon[77242]: pgmap v2539: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:52:50 smithi153 ceph-mon[70044]: pgmap v2539: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:50.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:52:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:52:50.428Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:52:51.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:52:51 smithi083 ceph-mon[77242]: pgmap v2540: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:52:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:52:51] "GET /metrics HTTP/1.1" 200 225175 "" "Prometheus/2.33.4" 2022-09-29T06:52:51.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:52:51 smithi083 ceph-mon[79930]: pgmap v2540: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:52.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:52:51 smithi153 ceph-mon[70044]: pgmap v2540: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:52:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:52:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:52:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:52:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:52:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:52:54.173 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:52:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:52:53.830Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:52:54.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:52:54 smithi083 ceph-mon[79930]: pgmap v2541: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:54.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:52:54 smithi083 ceph-mon[77242]: pgmap v2541: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:54.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:52:54 smithi153 ceph-mon[70044]: pgmap v2541: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:54.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:52:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:52:54.533Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:52:56.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:52:56 smithi083 ceph-mon[79930]: pgmap v2542: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:56.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:52:56 smithi083 ceph-mon[77242]: pgmap v2542: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:56.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:52:56 smithi153 ceph-mon[70044]: pgmap v2542: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:58.448 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:52:58.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:52:58 smithi083 ceph-mon[79930]: pgmap v2543: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:58.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:52:58 smithi083 ceph-mon[77242]: pgmap v2543: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:52:58 smithi153 ceph-mon[70044]: pgmap v2543: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:52:58.735 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:52:58.735 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (86m) 8m ago 87m 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:52:58.736 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (86m) 6m ago 86m 61.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:52:58.736 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (88m) 8m ago 88m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:52:58.736 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (87m) 6m ago 87m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:52:58.736 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (83m) 6m ago 90m 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:52:58.737 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (84m) 8m ago 92m 580M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:52:58.737 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (83m) 8m ago 92m 348M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:52:58.737 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (82m) 6m ago 90m 161M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:52:58.737 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (83m) 8m ago 90m 207M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:52:58.738 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (87m) 8m ago 87m 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:52:58.738 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (87m) 6m ago 87m 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:52:58.738 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (82m) 8m ago 90m 259M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:52:58.738 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (81m) 8m ago 90m 251M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:52:58.739 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (81m) 8m ago 89m 203M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:52:58.739 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (80m) 8m ago 89m 241M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:52:58.739 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (80m) 6m ago 89m 219M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:52:58.739 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (79m) 6m ago 88m 262M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:52:58.739 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (79m) 6m ago 88m 233M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:52:58.740 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (78m) 6m ago 88m 268M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:52:58.740 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (83m) 6m ago 87m 109M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:52:59.088 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:52:59.088 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:52:59.089 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:52:59.089 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:52:59.089 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:52:59.089 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:52:59.090 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:52:59.090 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:52:59.090 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:52:59.091 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:52:59.091 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:52:59.091 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:52:59.091 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:52:59.092 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:52:59.092 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:52:59.092 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:52:59.092 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:52:59.093 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:52:59.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:52:59 smithi083 ceph-mon[79930]: from='client.37095 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:52:59.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:52:59 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2255817847' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:52:59.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:52:59 smithi083 ceph-mon[77242]: from='client.37095 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:52:59.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:52:59 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2255817847' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:52:59.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:52:59 smithi153 ceph-mon[70044]: from='client.37095 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:52:59.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:52:59 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2255817847' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:53:00.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:53:00 smithi083 ceph-mon[79930]: from='client.47035 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:53:00.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:53:00 smithi083 ceph-mon[79930]: pgmap v2544: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:53:00.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:53:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:53:00.428Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:53:00.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:53:00 smithi083 ceph-mon[77242]: from='client.47035 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:53:00.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:53:00 smithi083 ceph-mon[77242]: pgmap v2544: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:53:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:53:00 smithi153 ceph-mon[70044]: from='client.47035 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:53:00.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:53:00 smithi153 ceph-mon[70044]: pgmap v2544: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:53:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:53:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:53:01] "GET /metrics HTTP/1.1" 200 225178 "" "Prometheus/2.33.4" 2022-09-29T06:53:01.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:53:01 smithi083 ceph-mon[79930]: pgmap v2545: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:53:01.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:53:01 smithi083 ceph-mon[77242]: pgmap v2545: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:53:02.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:53:01 smithi153 ceph-mon[70044]: pgmap v2545: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:53:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:53:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:53:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:53:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:53:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:53:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:53:04.175 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:53:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:53:03.831Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:53:04.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:53:04 smithi083 ceph-mon[79930]: pgmap v2546: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:53:04.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:53:04 smithi083 ceph-mon[77242]: pgmap v2546: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:53:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:53:04 smithi153 ceph-mon[70044]: pgmap v2546: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:53:04.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:53:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:53:04.533Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:53:06.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:53:06 smithi083 ceph-mon[79930]: pgmap v2547: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:53:06.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:53:06 smithi083 ceph-mon[77242]: pgmap v2547: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:53:06.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:53:06 smithi153 ceph-mon[70044]: pgmap v2547: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:53:08.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:53:08 smithi083 ceph-mon[79930]: pgmap v2548: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:53:08.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:53:08 smithi083 ceph-mon[77242]: pgmap v2548: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:53:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:53:08 smithi153 ceph-mon[70044]: pgmap v2548: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:53:10.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:53:10 smithi083 ceph-mon[79930]: pgmap v2549: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:53:10.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:53:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:53:10.429Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:53:10.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:53:10 smithi083 ceph-mon[77242]: pgmap v2549: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:53:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:53:10 smithi153 ceph-mon[70044]: pgmap v2549: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:53:11.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:53:11 smithi083 ceph-mon[79930]: pgmap v2550: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:53:11.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:53:11 smithi083 ceph-mon[77242]: pgmap v2550: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:53:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:53:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:53:11] "GET /metrics HTTP/1.1" 200 225175 "" "Prometheus/2.33.4" 2022-09-29T06:53:12.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:53:11 smithi153 ceph-mon[70044]: pgmap v2550: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:53:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:53:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:53:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:53:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:53:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:53:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:53:14.178 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:53:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:53:13.831Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:53:14.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:53:14 smithi083 ceph-mon[79930]: pgmap v2551: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:53:14.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:53:14 smithi083 ceph-mon[77242]: pgmap v2551: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:53:14.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:53:14 smithi153 ceph-mon[70044]: pgmap v2551: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:53:14.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:53:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:53:14.535Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:53:16.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:53:16 smithi083 ceph-mon[79930]: pgmap v2552: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:53:16.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:53:16 smithi083 ceph-mon[77242]: pgmap v2552: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:53:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:53:16 smithi153 ceph-mon[70044]: pgmap v2552: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:53:18.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:53:18 smithi083 ceph-mon[79930]: pgmap v2553: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:53:18.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:53:18 smithi083 ceph-mon[77242]: pgmap v2553: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:53:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:53:18 smithi153 ceph-mon[70044]: pgmap v2553: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:53:20.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:53:20 smithi083 ceph-mon[79930]: pgmap v2554: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:53:20.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:53:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:53:20.429Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:53:20.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:53:20 smithi083 ceph-mon[77242]: pgmap v2554: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:53:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:53:20 smithi153 ceph-mon[70044]: pgmap v2554: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:53:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:53:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:53:21] "GET /metrics HTTP/1.1" 200 225175 "" "Prometheus/2.33.4" 2022-09-29T06:53:21.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:53:21 smithi083 ceph-mon[79930]: pgmap v2555: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:53:21.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:53:21 smithi083 ceph-mon[77242]: pgmap v2555: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:53:22.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:53:21 smithi153 ceph-mon[70044]: pgmap v2555: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:53:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:53:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:53:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:53:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:53:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:53:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:53:24.181 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:53:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:53:23.831Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:53:24.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:53:24 smithi083 ceph-mon[79930]: pgmap v2556: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:53:24.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:53:24 smithi083 ceph-mon[77242]: pgmap v2556: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:53:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:53:24 smithi153 ceph-mon[70044]: pgmap v2556: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:53:24.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:53:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:53:24.534Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:53:26.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:53:26 smithi083 ceph-mon[77242]: pgmap v2557: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:53:26.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:53:26 smithi083 ceph-mon[79930]: pgmap v2557: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:53:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:53:26 smithi153 ceph-mon[70044]: pgmap v2557: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:53:28.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:53:28 smithi083 ceph-mon[79930]: pgmap v2558: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:53:28.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:53:28 smithi083 ceph-mon[77242]: pgmap v2558: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:53:28.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:53:28 smithi153 ceph-mon[70044]: pgmap v2558: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:53:29.415 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:53:29.707 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:53:29.707 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (87m) 8m ago 87m 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:53:29.708 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (87m) 6m ago 87m 61.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:53:29.708 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (88m) 8m ago 88m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:53:29.708 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (88m) 6m ago 88m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:53:29.709 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (84m) 6m ago 91m 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:53:29.709 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (84m) 8m ago 92m 580M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:53:29.709 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (84m) 8m ago 92m 348M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:53:29.709 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (83m) 6m ago 91m 161M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:53:29.709 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (83m) 8m ago 91m 207M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:53:29.710 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (87m) 8m ago 87m 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:53:29.710 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (87m) 6m ago 87m 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:53:29.710 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (82m) 8m ago 90m 259M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:53:29.710 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (82m) 8m ago 90m 251M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:53:29.711 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (81m) 8m ago 90m 203M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:53:29.711 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (81m) 8m ago 89m 241M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:53:29.711 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (80m) 6m ago 89m 219M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:53:29.711 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (80m) 6m ago 89m 262M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:53:29.712 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (79m) 6m ago 89m 233M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:53:29.712 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (79m) 6m ago 88m 268M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:53:29.712 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (84m) 6m ago 87m 109M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:53:30.061 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:53:30.062 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:53:30.062 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:53:30.063 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:53:30.063 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:53:30.063 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:53:30.063 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:53:30.064 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:53:30.064 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:53:30.064 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:53:30.064 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:53:30.064 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:53:30.065 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:53:30.065 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:53:30.065 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:53:30.065 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:53:30.065 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:53:30.066 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:53:30.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:53:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:53:30.429Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:53:30.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:53:30 smithi083 ceph-mon[77242]: pgmap v2559: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:53:30.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:53:30 smithi083 ceph-mon[77242]: from='client.47047 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:53:30.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:53:30 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2484358099' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:53:30.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:53:30 smithi083 ceph-mon[79930]: pgmap v2559: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:53:30.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:53:30 smithi083 ceph-mon[79930]: from='client.47047 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:53:30.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:53:30 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2484358099' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:53:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:53:30 smithi153 ceph-mon[70044]: pgmap v2559: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:53:30.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:53:30 smithi153 ceph-mon[70044]: from='client.47047 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:53:30.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:53:30 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2484358099' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:53:31.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:53:31 smithi083 ceph-mon[79930]: from='client.47053 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:53:31.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:53:31 smithi083 ceph-mon[77242]: from='client.47053 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:53:31.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:53:31 smithi153 ceph-mon[70044]: from='client.47053 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:53:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:53:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:53:31] "GET /metrics HTTP/1.1" 200 225175 "" "Prometheus/2.33.4" 2022-09-29T06:53:32.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:53:32 smithi083 ceph-mon[77242]: pgmap v2560: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:53:32.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:53:32 smithi083 ceph-mon[79930]: pgmap v2560: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:53:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:53:32 smithi153 ceph-mon[70044]: pgmap v2560: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:53:32.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:53:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:53:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:53:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:53:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:53:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:53:34.202 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:53:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:53:33.832Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:53:34.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:53:34 smithi083 ceph-mon[79930]: pgmap v2561: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:53:34.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:53:34 smithi083 ceph-mon[77242]: pgmap v2561: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:53:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:53:34 smithi153 ceph-mon[70044]: pgmap v2561: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:53:34.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:53:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:53:34.535Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:53:36.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:53:36 smithi083 ceph-mon[79930]: pgmap v2562: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:53:36.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:53:36 smithi083 ceph-mon[77242]: pgmap v2562: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:53:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:53:36 smithi153 ceph-mon[70044]: pgmap v2562: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:53:38.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:53:38 smithi083 ceph-mon[79930]: pgmap v2563: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:53:38.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:53:38 smithi083 ceph-mon[77242]: pgmap v2563: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:53:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:53:38 smithi153 ceph-mon[70044]: pgmap v2563: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:53:40.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:53:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:53:40.429Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:53:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:53:40 smithi083 ceph-mon[79930]: pgmap v2564: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:53:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:53:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:53:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:53:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:53:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:53:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:53:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:53:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:53:40.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:53:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:53:40.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:53:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:53:40.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:53:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:53:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:53:40 smithi083 ceph-mon[77242]: pgmap v2564: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:53:40.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:53:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:53:40.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:53:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:53:40.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:53:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:53:40.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:53:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:53:40.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:53:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:53:40.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:53:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:53:40.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:53:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:53:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:53:40 smithi153 ceph-mon[70044]: pgmap v2564: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:53:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:53:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:53:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:53:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:53:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:53:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:53:40.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:53:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:53:40.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:53:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:53:40.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:53:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:53:40.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:53:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:53:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:53:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:53:41] "GET /metrics HTTP/1.1" 200 225176 "" "Prometheus/2.33.4" 2022-09-29T06:53:41.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:53:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:53:41.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:53:41 smithi083 ceph-mon[77242]: pgmap v2565: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:53:41.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:53:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:53:41.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:53:41 smithi083 ceph-mon[79930]: pgmap v2565: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:53:42.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:53:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:53:42.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:53:41 smithi153 ceph-mon[70044]: pgmap v2565: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:53:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:53:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:53:42.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:53:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:53:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:53:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:53:44.188 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:53:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:53:43.833Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:53:44.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:53:44 smithi083 ceph-mon[79930]: pgmap v2566: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:53:44.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:53:44 smithi083 ceph-mon[77242]: pgmap v2566: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:53:44.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:53:44 smithi153 ceph-mon[70044]: pgmap v2566: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:53:44.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:53:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:53:44.536Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:53:46.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:53:46 smithi083 ceph-mon[79930]: pgmap v2567: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:53:46.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:53:46 smithi083 ceph-mon[77242]: pgmap v2567: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:53:46.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:53:46 smithi153 ceph-mon[70044]: pgmap v2567: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:53:48.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:53:48 smithi083 ceph-mon[79930]: pgmap v2568: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:53:48.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:53:48 smithi083 ceph-mon[77242]: pgmap v2568: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:53:48.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:53:48 smithi153 ceph-mon[70044]: pgmap v2568: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:53:50.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:53:50 smithi083 ceph-mon[77242]: pgmap v2569: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:53:50.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:53:50 smithi083 ceph-mon[79930]: pgmap v2569: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:53:50.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:53:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:53:50.430Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:53:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:53:50 smithi153 ceph-mon[70044]: pgmap v2569: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:53:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:53:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:53:51] "GET /metrics HTTP/1.1" 200 225176 "" "Prometheus/2.33.4" 2022-09-29T06:53:51.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:53:51 smithi083 ceph-mon[79930]: pgmap v2570: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:53:51.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:53:51 smithi083 ceph-mon[77242]: pgmap v2570: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:53:52.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:53:51 smithi153 ceph-mon[70044]: pgmap v2570: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:53:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:53:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:53:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:53:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:53:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:53:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:53:54.191 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:53:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:53:53.834Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:53:54.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:53:54 smithi083 ceph-mon[79930]: pgmap v2571: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:53:54.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:53:54 smithi083 ceph-mon[77242]: pgmap v2571: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:53:54.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:53:54 smithi153 ceph-mon[70044]: pgmap v2571: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:53:54.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:53:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:53:54.536Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:53:56.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:53:56 smithi083 ceph-mon[77242]: pgmap v2572: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:53:56.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:53:56 smithi083 ceph-mon[79930]: pgmap v2572: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:53:56.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:53:56 smithi153 ceph-mon[70044]: pgmap v2572: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:53:58.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:53:58 smithi083 ceph-mon[79930]: pgmap v2573: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:53:58.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:53:58 smithi083 ceph-mon[77242]: pgmap v2573: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:53:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:53:58 smithi153 ceph-mon[70044]: pgmap v2573: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:00.387 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:54:00.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:54:00 smithi083 ceph-mon[79930]: pgmap v2574: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:00.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:54:00 smithi083 ceph-mon[77242]: pgmap v2574: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:00.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:54:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:54:00.429Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:54:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:54:00 smithi153 ceph-mon[70044]: pgmap v2574: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:00.674 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:54:00.675 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (87m) 9m ago 88m 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:54:00.675 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (87m) 7m ago 87m 61.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:54:00.675 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (89m) 9m ago 89m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:54:00.675 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (89m) 7m ago 89m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:54:00.675 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (84m) 7m ago 91m 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:54:00.676 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (85m) 9m ago 93m 580M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:54:00.676 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (84m) 9m ago 93m 348M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:54:00.676 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (83m) 7m ago 91m 161M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:54:00.676 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (84m) 9m ago 91m 207M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:54:00.677 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (88m) 9m ago 88m 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:54:00.677 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (88m) 7m ago 88m 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:54:00.677 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (83m) 9m ago 91m 259M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:54:00.678 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (82m) 9m ago 91m 251M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:54:00.678 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (82m) 9m ago 90m 203M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:54:00.678 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (81m) 9m ago 90m 241M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:54:00.678 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (81m) 7m ago 90m 219M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:54:00.679 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (80m) 7m ago 89m 262M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:54:00.679 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (80m) 7m ago 89m 233M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:54:00.679 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (79m) 7m ago 89m 268M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:54:00.680 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (84m) 7m ago 88m 109M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:54:01.031 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:54:01.032 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:54:01.032 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:54:01.032 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:54:01.033 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:54:01.033 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:54:01.033 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:54:01.033 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:54:01.034 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:54:01.034 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:54:01.034 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:54:01.034 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:54:01.035 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:54:01.035 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:54:01.035 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:54:01.035 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:54:01.035 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:54:01.036 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:54:01.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:54:01 smithi083 ceph-mon[79930]: from='client.47065 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:54:01.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:54:01 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1825195809' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:54:01.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:54:01 smithi083 ceph-mon[77242]: from='client.47065 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:54:01.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:54:01 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1825195809' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:54:01.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:54:01 smithi153 ceph-mon[70044]: from='client.47065 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:54:01.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:54:01 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1825195809' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:54:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:54:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:54:01] "GET /metrics HTTP/1.1" 200 225180 "" "Prometheus/2.33.4" 2022-09-29T06:54:02.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:54:02 smithi083 ceph-mon[77242]: from='client.47071 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:54:02.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:54:02 smithi083 ceph-mon[77242]: pgmap v2575: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:02.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:54:02 smithi083 ceph-mon[79930]: from='client.47071 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:54:02.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:54:02 smithi083 ceph-mon[79930]: pgmap v2575: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:54:02 smithi153 ceph-mon[70044]: from='client.47071 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:54:02.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:54:02 smithi153 ceph-mon[70044]: pgmap v2575: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:02.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:54:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:54:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:54:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:54:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:54:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:54:04.208 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:54:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:54:03.835Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:54:04.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:54:04 smithi083 ceph-mon[79930]: pgmap v2576: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:04.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:54:04 smithi083 ceph-mon[77242]: pgmap v2576: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:54:04 smithi153 ceph-mon[70044]: pgmap v2576: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:04.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:54:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:54:04.537Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:54:06.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:54:06 smithi083 ceph-mon[79930]: pgmap v2577: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:06.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:54:06 smithi083 ceph-mon[77242]: pgmap v2577: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:06.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:54:06 smithi153 ceph-mon[70044]: pgmap v2577: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:54:08 smithi153 ceph-mon[70044]: pgmap v2578: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:08.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:54:08 smithi083 ceph-mon[79930]: pgmap v2578: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:08.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:54:08 smithi083 ceph-mon[77242]: pgmap v2578: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:54:10 smithi153 ceph-mon[70044]: pgmap v2579: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:10.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:54:10 smithi083 ceph-mon[79930]: pgmap v2579: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:10.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:54:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:54:10.431Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:54:10.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:54:10 smithi083 ceph-mon[77242]: pgmap v2579: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:11.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:54:11 smithi083 ceph-mon[77242]: pgmap v2580: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:54:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:54:11] "GET /metrics HTTP/1.1" 200 225179 "" "Prometheus/2.33.4" 2022-09-29T06:54:11.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:54:11 smithi083 ceph-mon[79930]: pgmap v2580: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:12.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:54:11 smithi153 ceph-mon[70044]: pgmap v2580: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:54:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:54:12.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:54:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:54:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:54:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:54:14.198 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:54:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:54:13.836Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:54:14.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:54:14 smithi083 ceph-mon[79930]: pgmap v2581: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:14.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:54:14 smithi083 ceph-mon[77242]: pgmap v2581: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:14.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:54:14 smithi153 ceph-mon[70044]: pgmap v2581: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:14.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:54:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:54:14.538Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:54:16.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:54:16 smithi083 ceph-mon[79930]: pgmap v2582: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:16.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:54:16 smithi083 ceph-mon[77242]: pgmap v2582: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:54:16 smithi153 ceph-mon[70044]: pgmap v2582: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:18.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:54:18 smithi083 ceph-mon[79930]: pgmap v2583: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:18.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:54:18 smithi083 ceph-mon[77242]: pgmap v2583: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:54:18 smithi153 ceph-mon[70044]: pgmap v2583: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:20.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:54:20 smithi083 ceph-mon[77242]: pgmap v2584: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:20.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:54:20 smithi083 ceph-mon[79930]: pgmap v2584: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:20.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:54:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:54:20.432Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:54:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:54:20 smithi153 ceph-mon[70044]: pgmap v2584: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:21.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:54:21 smithi083 ceph-mon[77242]: pgmap v2585: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:54:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:54:21] "GET /metrics HTTP/1.1" 200 225179 "" "Prometheus/2.33.4" 2022-09-29T06:54:21.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:54:21 smithi083 ceph-mon[79930]: pgmap v2585: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:22.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:54:21 smithi153 ceph-mon[70044]: pgmap v2585: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:54:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:54:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:54:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:54:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:54:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:54:24.201 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:54:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:54:23.836Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:54:24.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:54:24 smithi083 ceph-mon[77242]: pgmap v2586: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:24.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:54:24 smithi083 ceph-mon[79930]: pgmap v2586: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:54:24 smithi153 ceph-mon[70044]: pgmap v2586: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:24.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:54:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:54:24.539Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:54:26.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:54:26 smithi083 ceph-mon[77242]: pgmap v2587: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:26.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:54:26 smithi083 ceph-mon[79930]: pgmap v2587: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:54:26 smithi153 ceph-mon[70044]: pgmap v2587: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:28.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:54:28 smithi083 ceph-mon[79930]: pgmap v2588: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:28.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:54:28 smithi083 ceph-mon[77242]: pgmap v2588: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:54:28 smithi153 ceph-mon[70044]: pgmap v2588: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:30.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:54:30 smithi083 ceph-mon[79930]: pgmap v2589: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:30.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:54:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:54:30.433Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:54:30.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:54:30 smithi083 ceph-mon[77242]: pgmap v2589: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:54:30 smithi153 ceph-mon[70044]: pgmap v2589: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:31.358 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:54:31.650 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:54:31.651 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (88m) 10m ago 88m 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:54:31.651 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (88m) 7m ago 88m 61.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:54:31.651 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (89m) 10m ago 89m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:54:31.652 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (89m) 7m ago 89m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:54:31.652 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (85m) 7m ago 92m 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:54:31.652 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (85m) 10m ago 93m 580M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:54:31.652 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (85m) 10m ago 93m 348M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:54:31.653 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (84m) 7m ago 92m 161M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:54:31.653 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (84m) 10m ago 92m 207M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:54:31.653 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (88m) 10m ago 88m 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:54:31.654 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (88m) 7m ago 88m 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:54:31.654 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (83m) 10m ago 91m 259M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:54:31.654 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (83m) 10m ago 91m 251M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:54:31.654 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (82m) 10m ago 91m 203M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:54:31.655 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (82m) 10m ago 90m 241M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:54:31.655 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (81m) 7m ago 90m 219M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:54:31.655 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (81m) 7m ago 90m 262M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:54:31.655 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (80m) 7m ago 90m 233M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:54:31.656 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (80m) 7m ago 89m 268M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:54:31.656 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (85m) 7m ago 89m 109M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:54:31.762 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:54:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:54:31] "GET /metrics HTTP/1.1" 200 225178 "" "Prometheus/2.33.4" 2022-09-29T06:54:32.014 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:54:32.015 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:54:32.015 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:54:32.015 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:54:32.016 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:54:32.016 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:54:32.016 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:54:32.016 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:54:32.017 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:54:32.017 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:54:32.017 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:54:32.018 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:54:32.018 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:54:32.018 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:54:32.018 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:54:32.019 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:54:32.019 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:54:32.019 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:54:32.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:54:31 smithi153 ceph-mon[70044]: pgmap v2590: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:32.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:54:31 smithi153 ceph-mon[70044]: from='client.47083 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:54:32.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:54:31 smithi083 ceph-mon[79930]: pgmap v2590: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:32.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:54:31 smithi083 ceph-mon[79930]: from='client.47083 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:54:32.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:54:31 smithi083 ceph-mon[77242]: pgmap v2590: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:32.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:54:31 smithi083 ceph-mon[77242]: from='client.47083 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:54:32.764 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:54:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:54:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:54:33.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:54:32 smithi153 ceph-mon[70044]: from='client.47089 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:54:33.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:54:32 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2514039550' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:54:33.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:54:32 smithi083 ceph-mon[79930]: from='client.47089 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:54:33.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:54:32 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2514039550' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:54:33.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:54:32 smithi083 ceph-mon[77242]: from='client.47089 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:54:33.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:54:32 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2514039550' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:54:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:54:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:54:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:54:34.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:54:33 smithi153 ceph-mon[70044]: pgmap v2591: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:34.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:54:33 smithi083 ceph-mon[79930]: pgmap v2591: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:34.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:54:33 smithi083 ceph-mon[77242]: pgmap v2591: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:34.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:54:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:54:33.837Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:54:34.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:54:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:54:34.539Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:54:36.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:54:36 smithi083 ceph-mon[79930]: pgmap v2592: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:36.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:54:36 smithi083 ceph-mon[77242]: pgmap v2592: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:54:36 smithi153 ceph-mon[70044]: pgmap v2592: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:38.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:54:38 smithi083 ceph-mon[79930]: pgmap v2593: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:38.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:54:38 smithi083 ceph-mon[77242]: pgmap v2593: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:54:38 smithi153 ceph-mon[70044]: pgmap v2593: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:40.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:54:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:54:40.434Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:54:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:54:40 smithi083 ceph-mon[79930]: pgmap v2594: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:54:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:54:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:54:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:54:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:54:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:54:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:54:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:54:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:54:40 smithi083 ceph-mon[77242]: pgmap v2594: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:54:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:54:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:54:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:54:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:54:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:54:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:54:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:54:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:54:40 smithi153 ceph-mon[70044]: pgmap v2594: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:54:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:54:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:54:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:54:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:54:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:54:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:54:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:54:41.761 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:54:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:54:41] "GET /metrics HTTP/1.1" 200 225176 "" "Prometheus/2.33.4" 2022-09-29T06:54:42.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:54:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:54:42.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:54:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:54:42.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:54:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:54:42.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:54:41 smithi153 ceph-mon[70044]: pgmap v2595: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:54:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:54:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:54:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:54:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:54:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:54:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:54:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:54:42.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:54:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:54:42.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:54:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:54:42.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:54:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:54:42.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:54:41 smithi083 ceph-mon[79930]: pgmap v2595: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:42.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:54:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:54:42.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:54:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:54:42.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:54:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:54:42.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:54:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:54:42.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:54:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:54:42.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:54:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:54:42.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:54:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:54:42.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:54:41 smithi083 ceph-mon[77242]: pgmap v2595: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:42.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:54:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:54:42.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:54:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:54:42.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:54:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:54:42.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:54:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:54:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:54:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:54:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:54:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:54:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:54:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:54:44.206 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:54:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:54:43.837Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:54:44.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:54:44 smithi083 ceph-mon[77242]: pgmap v2596: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:44.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:54:44 smithi083 ceph-mon[79930]: pgmap v2596: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:44.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:54:44 smithi153 ceph-mon[70044]: pgmap v2596: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:44.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:54:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:54:44.539Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:54:46.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:54:46 smithi153 ceph-mon[70044]: pgmap v2597: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:46.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:54:46 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:54:46.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:54:46 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:54:46.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:54:46 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:54:46.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:54:46 smithi083 ceph-mon[79930]: pgmap v2597: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:46.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:54:46 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:54:46.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:54:46 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:54:46.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:54:46 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:54:46.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:54:46 smithi083 ceph-mon[77242]: pgmap v2597: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:46.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:54:46 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:54:46.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:54:46 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:54:46.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:54:46 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:54:48.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:54:48 smithi153 ceph-mon[70044]: pgmap v2598: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:48.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:54:48 smithi083 ceph-mon[79930]: pgmap v2598: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:48.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:54:48 smithi083 ceph-mon[77242]: pgmap v2598: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:54:50 smithi153 ceph-mon[70044]: pgmap v2599: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:50.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:54:50 smithi083 ceph-mon[79930]: pgmap v2599: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:50.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:54:50 smithi083 ceph-mon[77242]: pgmap v2599: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:50.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:54:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:54:50.434Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:54:51.762 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:54:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:54:51] "GET /metrics HTTP/1.1" 200 225176 "" "Prometheus/2.33.4" 2022-09-29T06:54:52.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:54:51 smithi153 ceph-mon[70044]: pgmap v2600: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:52.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:54:51 smithi083 ceph-mon[77242]: pgmap v2600: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:52.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:54:51 smithi083 ceph-mon[79930]: pgmap v2600: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:54:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:54:52.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:54:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:54:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:54:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:54:54.209 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:54:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:54:53.839Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:54:54.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:54:54 smithi083 ceph-mon[79930]: pgmap v2601: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:54.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:54:54 smithi083 ceph-mon[77242]: pgmap v2601: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:54.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:54:54 smithi153 ceph-mon[70044]: pgmap v2601: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:54.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:54:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:54:54.540Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:54:56.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:54:56 smithi153 ceph-mon[70044]: pgmap v2602: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:56.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:54:56 smithi083 ceph-mon[79930]: pgmap v2602: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:56.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:54:56 smithi083 ceph-mon[77242]: pgmap v2602: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:54:58 smithi153 ceph-mon[70044]: pgmap v2603: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:58.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:54:58 smithi083 ceph-mon[79930]: pgmap v2603: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:54:58.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:54:58 smithi083 ceph-mon[77242]: pgmap v2603: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:55:00 smithi153 ceph-mon[70044]: pgmap v2604: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:00.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:55:00 smithi083 ceph-mon[79930]: pgmap v2604: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:00.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:55:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:55:00.435Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:55:00.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:55:00 smithi083 ceph-mon[77242]: pgmap v2604: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:01.763 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:55:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:55:01] "GET /metrics HTTP/1.1" 200 225179 "" "Prometheus/2.33.4" 2022-09-29T06:55:02.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:55:01 smithi153 ceph-mon[70044]: pgmap v2605: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:02.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:55:01 smithi083 ceph-mon[79930]: pgmap v2605: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:02.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:55:01 smithi083 ceph-mon[77242]: pgmap v2605: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:02.343 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:55:02.629 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:55:02.630 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (88m) 17s ago 89m 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:55:02.630 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (89m) 8m ago 89m 61.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:55:02.630 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (90m) 17s ago 90m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:55:02.631 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (90m) 8m ago 90m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:55:02.631 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (85m) 8m ago 92m 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:55:02.631 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (86m) 17s ago 94m 589M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:55:02.631 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (85m) 17s ago 94m 385M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:55:02.632 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (84m) 8m ago 92m 161M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:55:02.632 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (85m) 17s ago 93m 227M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:55:02.632 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (89m) 17s ago 89m 26.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:55:02.632 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (89m) 8m ago 89m 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:55:02.632 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (84m) 17s ago 92m 285M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:55:02.633 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (83m) 17s ago 92m 277M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:55:02.633 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (83m) 17s ago 91m 222M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:55:02.633 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (82m) 17s ago 91m 266M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:55:02.633 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (82m) 8m ago 91m 219M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:55:02.634 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (81m) 8m ago 90m 262M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:55:02.634 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (81m) 8m ago 90m 233M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:55:02.634 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (80m) 8m ago 90m 268M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:55:02.634 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (85m) 8m ago 89m 109M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:55:02.765 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:55:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:55:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:55:02.986 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:55:02.986 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:55:02.987 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:55:02.987 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:55:02.987 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:55:02.988 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:55:02.988 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:55:02.988 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:55:02.988 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:55:02.989 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:55:02.989 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:55:02.989 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:55:02.990 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:55:02.990 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:55:02.990 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:55:02.991 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:55:02.991 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:55:02.991 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:55:03.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:55:02 smithi153 ceph-mon[70044]: from='client.47101 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:55:03.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:55:02 smithi083 ceph-mon[79930]: from='client.47101 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:55:03.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:55:02 smithi083 ceph-mon[77242]: from='client.47101 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:55:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:55:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:55:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:55:04.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:55:03 smithi153 ceph-mon[70044]: from='client.47107 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:55:04.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:55:03 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/325693839' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:55:04.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:55:03 smithi153 ceph-mon[70044]: pgmap v2606: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:04.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:55:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:55:03.839Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:55:04.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:55:03 smithi083 ceph-mon[77242]: from='client.47107 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:55:04.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:55:03 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/325693839' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:55:04.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:55:03 smithi083 ceph-mon[77242]: pgmap v2606: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:04.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:55:03 smithi083 ceph-mon[79930]: from='client.47107 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:55:04.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:55:03 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/325693839' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:55:04.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:55:03 smithi083 ceph-mon[79930]: pgmap v2606: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:04.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:55:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:55:04.541Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:55:06.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:55:06 smithi153 ceph-mon[70044]: pgmap v2607: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:06.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:55:06 smithi083 ceph-mon[79930]: pgmap v2607: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:06.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:55:06 smithi083 ceph-mon[77242]: pgmap v2607: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:55:08 smithi153 ceph-mon[70044]: pgmap v2608: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:08.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:55:08 smithi083 ceph-mon[79930]: pgmap v2608: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:08.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:55:08 smithi083 ceph-mon[77242]: pgmap v2608: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:55:10 smithi153 ceph-mon[70044]: pgmap v2609: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:10.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:55:10 smithi083 ceph-mon[79930]: pgmap v2609: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:10.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:55:10 smithi083 ceph-mon[77242]: pgmap v2609: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:10.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:55:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:55:10.436Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:55:11.768 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:55:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:55:11] "GET /metrics HTTP/1.1" 200 225175 "" "Prometheus/2.33.4" 2022-09-29T06:55:12.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:55:11 smithi153 ceph-mon[70044]: pgmap v2610: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:12.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:55:11 smithi083 ceph-mon[77242]: pgmap v2610: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:12.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:55:11 smithi083 ceph-mon[79930]: pgmap v2610: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:55:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:55:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:55:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:55:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:55:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:55:14.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:55:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:55:13.840Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:55:14.543 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:55:14 smithi083 ceph-mon[79930]: pgmap v2611: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:14.544 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:55:14 smithi083 ceph-mon[77242]: pgmap v2611: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:14.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:55:14 smithi153 ceph-mon[70044]: pgmap v2611: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:14.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:55:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:55:14.542Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:55:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:55:16 smithi153 ceph-mon[70044]: pgmap v2612: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:16.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:55:16 smithi083 ceph-mon[79930]: pgmap v2612: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:16.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:55:16 smithi083 ceph-mon[77242]: pgmap v2612: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:18.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:55:18 smithi153 ceph-mon[70044]: pgmap v2613: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:18.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:55:18 smithi083 ceph-mon[79930]: pgmap v2613: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:18.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:55:18 smithi083 ceph-mon[77242]: pgmap v2613: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:55:20 smithi153 ceph-mon[70044]: pgmap v2614: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:20.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:55:20 smithi083 ceph-mon[79930]: pgmap v2614: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:20.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:55:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:55:20.436Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:55:20.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:55:20 smithi083 ceph-mon[77242]: pgmap v2614: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:21.768 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:55:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:55:21] "GET /metrics HTTP/1.1" 200 225175 "" "Prometheus/2.33.4" 2022-09-29T06:55:22.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:55:21 smithi153 ceph-mon[70044]: pgmap v2615: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:22.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:55:21 smithi083 ceph-mon[79930]: pgmap v2615: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:22.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:55:21 smithi083 ceph-mon[77242]: pgmap v2615: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:55:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:55:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:55:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:55:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:55:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:55:24.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:55:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:55:23.840Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:55:24.544 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:55:24 smithi083 ceph-mon[79930]: pgmap v2616: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:24.545 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:55:24 smithi083 ceph-mon[77242]: pgmap v2616: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:55:24 smithi153 ceph-mon[70044]: pgmap v2616: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:24.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:55:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:55:24.543Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:55:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:55:26 smithi153 ceph-mon[70044]: pgmap v2617: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:26.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:55:26 smithi083 ceph-mon[79930]: pgmap v2617: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:26.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:55:26 smithi083 ceph-mon[77242]: pgmap v2617: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:55:28 smithi153 ceph-mon[70044]: pgmap v2618: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:28.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:55:28 smithi083 ceph-mon[79930]: pgmap v2618: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:28.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:55:28 smithi083 ceph-mon[77242]: pgmap v2618: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:55:30 smithi153 ceph-mon[70044]: pgmap v2619: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:30.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:55:30 smithi083 ceph-mon[79930]: pgmap v2619: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:30.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:55:30 smithi083 ceph-mon[77242]: pgmap v2619: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:30.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:55:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:55:30.437Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:55:31.770 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:55:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:55:31] "GET /metrics HTTP/1.1" 200 225178 "" "Prometheus/2.33.4" 2022-09-29T06:55:31.771 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:55:31 smithi083 ceph-mon[77242]: pgmap v2620: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:32.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:55:31 smithi153 ceph-mon[70044]: pgmap v2620: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:32.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:55:31 smithi083 ceph-mon[79930]: pgmap v2620: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:55:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:55:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:55:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:55:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:55:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:55:33.314 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:55:33.612 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:55:33.613 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (89m) 48s ago 89m 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:55:33.613 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (89m) 8m ago 89m 61.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:55:33.613 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (90m) 48s ago 90m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:55:33.613 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (90m) 8m ago 90m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:55:33.614 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (86m) 8m ago 93m 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:55:33.614 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (86m) 48s ago 94m 589M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:55:33.614 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (86m) 48s ago 94m 385M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:55:33.614 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (85m) 8m ago 93m 161M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:55:33.615 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (85m) 48s ago 93m 227M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:55:33.615 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (89m) 48s ago 89m 26.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:55:33.615 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (89m) 8m ago 89m 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:55:33.615 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (84m) 48s ago 92m 285M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:55:33.615 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (84m) 48s ago 92m 277M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:55:33.616 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (83m) 48s ago 92m 222M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:55:33.616 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (83m) 48s ago 91m 266M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:55:33.616 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (82m) 8m ago 91m 219M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:55:33.616 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (82m) 8m ago 91m 262M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:55:33.617 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (81m) 8m ago 91m 233M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:55:33.617 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (81m) 8m ago 90m 268M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:55:33.617 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (86m) 8m ago 90m 109M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:55:33.981 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:55:33.981 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:55:33.982 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:55:33.982 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:55:33.982 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:55:33.982 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:55:33.983 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:55:33.983 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:55:33.983 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:55:33.983 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:55:33.983 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:55:33.984 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:55:33.984 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:55:33.984 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:55:33.984 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:55:33.984 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:55:33.985 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:55:33.985 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:55:34.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:55:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:55:33.841Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:55:34.544 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:55:34 smithi083 ceph-mon[79930]: pgmap v2621: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:34.545 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:55:34 smithi083 ceph-mon[79930]: from='client.37185 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:55:34.545 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:55:34 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2234953121' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:55:34.545 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:55:34 smithi083 ceph-mon[77242]: pgmap v2621: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:34.546 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:55:34 smithi083 ceph-mon[77242]: from='client.37185 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:55:34.546 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:55:34 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2234953121' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:55:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:55:34 smithi153 ceph-mon[70044]: pgmap v2621: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:55:34 smithi153 ceph-mon[70044]: from='client.37185 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:55:34.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:55:34 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2234953121' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:55:34.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:55:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:55:34.543Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:55:35.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:55:35 smithi153 ceph-mon[70044]: from='client.47125 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:55:35.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:55:35 smithi083 ceph-mon[79930]: from='client.47125 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:55:35.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:55:35 smithi083 ceph-mon[77242]: from='client.47125 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:55:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:55:36 smithi153 ceph-mon[70044]: pgmap v2622: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:36.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:55:36 smithi083 ceph-mon[77242]: pgmap v2622: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:36.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:55:36 smithi083 ceph-mon[79930]: pgmap v2622: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:55:38 smithi153 ceph-mon[70044]: pgmap v2623: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:38.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:55:38 smithi083 ceph-mon[79930]: pgmap v2623: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:38.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:55:38 smithi083 ceph-mon[77242]: pgmap v2623: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:55:40 smithi153 ceph-mon[70044]: pgmap v2624: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:55:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:55:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:55:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:55:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:55:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:55:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:55:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:55:40.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:55:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:55:40.438Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:55:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:55:40 smithi083 ceph-mon[77242]: pgmap v2624: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:55:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:55:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:55:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:55:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:55:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:55:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:55:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:55:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:55:40 smithi083 ceph-mon[79930]: pgmap v2624: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:55:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:55:40.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:55:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:55:40.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:55:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:55:40.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:55:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:55:41.772 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:55:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:55:41] "GET /metrics HTTP/1.1" 200 225172 "" "Prometheus/2.33.4" 2022-09-29T06:55:41.772 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:55:41 smithi083 ceph-mon[77242]: pgmap v2625: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:42.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:55:41 smithi153 ceph-mon[70044]: pgmap v2625: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:42.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:55:41 smithi083 ceph-mon[79930]: pgmap v2625: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:55:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:55:42.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:55:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:55:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:55:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:55:44.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:55:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:55:43.842Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:55:44.546 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:55:44 smithi083 ceph-mon[79930]: pgmap v2626: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:44.546 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:55:44 smithi083 ceph-mon[77242]: pgmap v2626: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:44.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:55:44 smithi153 ceph-mon[70044]: pgmap v2626: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:44.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:55:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:55:44.544Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:55:46.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:55:46 smithi153 ceph-mon[70044]: pgmap v2627: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:46.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:55:46 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:55:46.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:55:46 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:55:46.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:55:46 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:55:46.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:55:46 smithi083 ceph-mon[79930]: pgmap v2627: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:46.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:55:46 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:55:46.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:55:46 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:55:46.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:55:46 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:55:46.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:55:46 smithi083 ceph-mon[77242]: pgmap v2627: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:46.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:55:46 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:55:46.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:55:46 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:55:46.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:55:46 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:55:47.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:55:47 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:55:47.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:55:47 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:55:47.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:55:47 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:55:48.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:55:48 smithi153 ceph-mon[70044]: pgmap v2628: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:48.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:55:48 smithi083 ceph-mon[79930]: pgmap v2628: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:48.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:55:48 smithi083 ceph-mon[77242]: pgmap v2628: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:50.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:55:50 smithi083 ceph-mon[79930]: pgmap v2629: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:50.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:55:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:55:50.439Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:55:50.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:55:50 smithi083 ceph-mon[77242]: pgmap v2629: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:50.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:55:50 smithi153 ceph-mon[70044]: pgmap v2629: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:51.775 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:55:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:55:51] "GET /metrics HTTP/1.1" 200 225172 "" "Prometheus/2.33.4" 2022-09-29T06:55:52.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:55:51 smithi153 ceph-mon[70044]: pgmap v2630: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:52.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:55:51 smithi083 ceph-mon[79930]: pgmap v2630: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:52.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:55:51 smithi083 ceph-mon[77242]: pgmap v2630: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:55:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:55:52.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:55:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:55:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:55:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:55:54.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:55:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:55:53.843Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:55:54.546 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:55:54 smithi083 ceph-mon[79930]: pgmap v2631: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:54.546 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:55:54 smithi083 ceph-mon[77242]: pgmap v2631: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:54.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:55:54 smithi153 ceph-mon[70044]: pgmap v2631: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:54.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:55:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:55:54.544Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:55:56.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:55:56 smithi153 ceph-mon[70044]: pgmap v2632: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:56.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:55:56 smithi083 ceph-mon[79930]: pgmap v2632: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:56.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:55:56 smithi083 ceph-mon[77242]: pgmap v2632: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:55:58 smithi153 ceph-mon[70044]: pgmap v2633: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:58.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:55:58 smithi083 ceph-mon[79930]: pgmap v2633: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:55:58.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:55:58 smithi083 ceph-mon[77242]: pgmap v2633: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:56:00 smithi153 ceph-mon[70044]: pgmap v2634: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:00.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:56:00 smithi083 ceph-mon[79930]: pgmap v2634: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:00.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:56:00 smithi083 ceph-mon[77242]: pgmap v2634: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:00.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:56:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:56:00.439Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:56:01.778 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:56:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:56:01] "GET /metrics HTTP/1.1" 200 225176 "" "Prometheus/2.33.4" 2022-09-29T06:56:02.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:56:01 smithi153 ceph-mon[70044]: pgmap v2635: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:02.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:56:01 smithi083 ceph-mon[77242]: pgmap v2635: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:02.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:56:01 smithi083 ceph-mon[79930]: pgmap v2635: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:56:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:56:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:56:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:56:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:56:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:56:04.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:56:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:56:03.843Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:56:04.308 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:56:04.546 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:56:04 smithi083 ceph-mon[79930]: pgmap v2636: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:04.546 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:56:04 smithi083 ceph-mon[77242]: pgmap v2636: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:56:04 smithi153 ceph-mon[70044]: pgmap v2636: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:04.602 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:56:04.602 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (89m) 79s ago 90m 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:56:04.603 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (90m) 9m ago 90m 61.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:56:04.603 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (91m) 79s ago 91m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:56:04.603 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (91m) 9m ago 91m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:56:04.603 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (86m) 9m ago 93m 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:56:04.604 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (87m) 79s ago 95m 589M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:56:04.604 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (86m) 79s ago 95m 385M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:56:04.604 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (85m) 9m ago 93m 161M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:56:04.604 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (86m) 79s ago 94m 227M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:56:04.605 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (90m) 79s ago 90m 26.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:56:04.605 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (90m) 9m ago 90m 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:56:04.605 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (85m) 79s ago 93m 285M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:56:04.605 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (84m) 79s ago 93m 277M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:56:04.605 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (84m) 79s ago 92m 222M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:56:04.606 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (83m) 79s ago 92m 266M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:56:04.606 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (83m) 9m ago 92m 219M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:56:04.606 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (82m) 9m ago 91m 262M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:56:04.606 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (82m) 9m ago 91m 233M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:56:04.607 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (81m) 9m ago 91m 268M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:56:04.607 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (86m) 9m ago 90m 109M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:56:04.960 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:56:04.960 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:56:04.961 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:56:04.961 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:56:04.961 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:56:04.961 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:56:04.961 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:56:04.962 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:56:04.962 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:56:04.962 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:56:04.962 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:56:04.963 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:56:04.963 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:56:04.963 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:56:04.963 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:56:04.964 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:56:04.964 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:56:04.964 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:56:04.965 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:56:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:56:04.545Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:56:05.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:56:05 smithi153 ceph-mon[70044]: from='client.47137 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:56:05.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:56:05 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/382565942' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:56:05.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:56:05 smithi083 ceph-mon[79930]: from='client.47137 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:56:05.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:56:05 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/382565942' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:56:05.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:56:05 smithi083 ceph-mon[77242]: from='client.47137 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:56:05.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:56:05 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/382565942' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:56:06.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:56:06 smithi153 ceph-mon[70044]: from='client.47143 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:56:06.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:56:06 smithi153 ceph-mon[70044]: pgmap v2637: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:06.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:56:06 smithi083 ceph-mon[79930]: from='client.47143 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:56:06.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:56:06 smithi083 ceph-mon[79930]: pgmap v2637: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:06.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:56:06 smithi083 ceph-mon[77242]: from='client.47143 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:56:06.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:56:06 smithi083 ceph-mon[77242]: pgmap v2637: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:56:08 smithi153 ceph-mon[70044]: pgmap v2638: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:08.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:56:08 smithi083 ceph-mon[79930]: pgmap v2638: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:08.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:56:08 smithi083 ceph-mon[77242]: pgmap v2638: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:56:10 smithi153 ceph-mon[70044]: pgmap v2639: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:10.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:56:10 smithi083 ceph-mon[79930]: pgmap v2639: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:10.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:56:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:56:10.440Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:56:10.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:56:10 smithi083 ceph-mon[77242]: pgmap v2639: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:11.781 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:56:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:56:11] "GET /metrics HTTP/1.1" 200 225178 "" "Prometheus/2.33.4" 2022-09-29T06:56:12.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:56:11 smithi153 ceph-mon[70044]: pgmap v2640: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:12.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:56:11 smithi083 ceph-mon[77242]: pgmap v2640: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:12.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:56:11 smithi083 ceph-mon[79930]: pgmap v2640: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:56:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:56:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:56:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:56:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:56:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:56:14.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:56:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:56:13.844Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:56:14.548 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:56:14 smithi083 ceph-mon[79930]: pgmap v2641: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:14.549 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:56:14 smithi083 ceph-mon[77242]: pgmap v2641: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:14.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:56:14 smithi153 ceph-mon[70044]: pgmap v2641: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:14.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:56:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:56:14.547Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:56:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:56:16 smithi153 ceph-mon[70044]: pgmap v2642: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:16.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:56:16 smithi083 ceph-mon[79930]: pgmap v2642: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:16.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:56:16 smithi083 ceph-mon[77242]: pgmap v2642: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:56:18 smithi153 ceph-mon[70044]: pgmap v2643: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:18.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:56:18 smithi083 ceph-mon[79930]: pgmap v2643: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:18.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:56:18 smithi083 ceph-mon[77242]: pgmap v2643: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:56:20 smithi153 ceph-mon[70044]: pgmap v2644: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:20.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:56:20 smithi083 ceph-mon[79930]: pgmap v2644: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:20.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:56:20 smithi083 ceph-mon[77242]: pgmap v2644: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:20.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:56:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:56:20.442Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:56:21.780 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:56:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:56:21] "GET /metrics HTTP/1.1" 200 225178 "" "Prometheus/2.33.4" 2022-09-29T06:56:22.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:56:21 smithi153 ceph-mon[70044]: pgmap v2645: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:22.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:56:21 smithi083 ceph-mon[79930]: pgmap v2645: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:22.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:56:21 smithi083 ceph-mon[77242]: pgmap v2645: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:56:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:56:22.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:56:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:56:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:56:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:56:24.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:56:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:56:23.845Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:56:24.549 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:56:24 smithi083 ceph-mon[79930]: pgmap v2646: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:24.549 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:56:24 smithi083 ceph-mon[77242]: pgmap v2646: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:56:24 smithi153 ceph-mon[70044]: pgmap v2646: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:24.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:56:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:56:24.547Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:56:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:56:26 smithi153 ceph-mon[70044]: pgmap v2647: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:26.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:56:26 smithi083 ceph-mon[77242]: pgmap v2647: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:26.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:56:26 smithi083 ceph-mon[79930]: pgmap v2647: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:56:28 smithi153 ceph-mon[70044]: pgmap v2648: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:28.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:56:28 smithi083 ceph-mon[79930]: pgmap v2648: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:28.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:56:28 smithi083 ceph-mon[77242]: pgmap v2648: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:56:30 smithi153 ceph-mon[70044]: pgmap v2649: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:30.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:56:30 smithi083 ceph-mon[79930]: pgmap v2649: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:30.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:56:30 smithi083 ceph-mon[77242]: pgmap v2649: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:30.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:56:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:56:30.442Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:56:31.783 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:56:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:56:31] "GET /metrics HTTP/1.1" 200 225174 "" "Prometheus/2.33.4" 2022-09-29T06:56:31.784 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:56:31 smithi083 ceph-mon[77242]: pgmap v2650: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:32.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:56:31 smithi153 ceph-mon[70044]: pgmap v2650: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:32.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:56:31 smithi083 ceph-mon[79930]: pgmap v2650: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:56:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:56:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:56:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:56:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:56:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:56:34.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:56:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:56:33.845Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:56:34.549 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:56:34 smithi083 ceph-mon[79930]: pgmap v2651: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:34.550 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:56:34 smithi083 ceph-mon[77242]: pgmap v2651: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:56:34 smithi153 ceph-mon[70044]: pgmap v2651: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:34.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:56:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:56:34.548Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:56:35.287 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:56:35.576 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:56:35.576 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (90m) 110s ago 90m 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:56:35.576 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (90m) 10m ago 90m 61.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:56:35.577 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (91m) 110s ago 91m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:56:35.577 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (91m) 10m ago 91m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:56:35.577 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (87m) 10m ago 94m 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:56:35.577 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (87m) 110s ago 95m 589M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:56:35.578 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (87m) 110s ago 95m 385M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:56:35.578 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (86m) 10m ago 94m 161M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:56:35.578 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (86m) 110s ago 94m 227M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:56:35.578 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (91m) 110s ago 91m 26.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:56:35.578 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (90m) 10m ago 90m 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:56:35.579 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (85m) 110s ago 93m 285M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:56:35.579 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (85m) 110s ago 93m 277M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:56:35.579 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (84m) 110s ago 93m 222M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:56:35.579 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (84m) 110s ago 93m 266M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:56:35.580 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (83m) 10m ago 92m 219M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:56:35.580 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (83m) 10m ago 92m 262M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:56:35.580 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (82m) 10m ago 92m 233M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:56:35.580 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (82m) 10m ago 91m 268M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:56:35.580 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (87m) 10m ago 91m 109M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:56:35.932 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:56:35.933 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:56:35.933 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:56:35.933 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:56:35.933 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:56:35.934 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:56:35.934 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:56:35.934 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:56:35.934 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:56:35.935 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:56:35.935 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:56:35.935 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:56:35.935 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:56:35.936 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:56:35.936 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:56:35.936 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:56:35.936 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:56:35.937 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:56:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:56:36 smithi153 ceph-mon[70044]: pgmap v2652: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:36.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:56:36 smithi153 ceph-mon[70044]: from='client.47155 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:56:36.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:56:36 smithi153 ceph-mon[70044]: from='client.37227 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:56:36.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:56:36 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2429071329' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:56:36.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:56:36 smithi083 ceph-mon[79930]: pgmap v2652: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:36.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:56:36 smithi083 ceph-mon[79930]: from='client.47155 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:56:36.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:56:36 smithi083 ceph-mon[79930]: from='client.37227 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:56:36.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:56:36 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2429071329' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:56:36.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:56:36 smithi083 ceph-mon[77242]: pgmap v2652: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:36.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:56:36 smithi083 ceph-mon[77242]: from='client.47155 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:56:36.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:56:36 smithi083 ceph-mon[77242]: from='client.37227 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:56:36.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:56:36 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2429071329' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:56:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:56:38 smithi153 ceph-mon[70044]: pgmap v2653: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:38.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:56:38 smithi083 ceph-mon[79930]: pgmap v2653: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:38.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:56:38 smithi083 ceph-mon[77242]: pgmap v2653: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:56:40 smithi153 ceph-mon[70044]: pgmap v2654: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:56:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:56:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:56:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:56:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:56:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:56:40.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:56:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:56:40.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:56:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:56:40.442Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:56:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:56:40 smithi083 ceph-mon[79930]: pgmap v2654: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:56:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:56:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:56:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:56:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:56:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:56:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:56:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:56:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:56:40 smithi083 ceph-mon[77242]: pgmap v2654: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:56:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:56:40.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:56:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:56:40.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:56:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:56:40.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:56:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:56:41.785 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:56:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:56:41] "GET /metrics HTTP/1.1" 200 225177 "" "Prometheus/2.33.4" 2022-09-29T06:56:42.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:56:41 smithi153 ceph-mon[70044]: pgmap v2655: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:42.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:56:41 smithi083 ceph-mon[79930]: pgmap v2655: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:42.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:56:41 smithi083 ceph-mon[77242]: pgmap v2655: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:56:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:56:42.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:56:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:56:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:56:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:56:44.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:56:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:56:43.846Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:56:44.550 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:56:44 smithi083 ceph-mon[79930]: pgmap v2656: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:44.550 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:56:44 smithi083 ceph-mon[77242]: pgmap v2656: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:44.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:56:44 smithi153 ceph-mon[70044]: pgmap v2656: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:44.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:56:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:56:44.548Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:56:46.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:56:46 smithi153 ceph-mon[70044]: pgmap v2657: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:46.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:56:46 smithi083 ceph-mon[79930]: pgmap v2657: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:46.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:56:46 smithi083 ceph-mon[77242]: pgmap v2657: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:47.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:56:47 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:56:47.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:56:47 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:56:47.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:56:47 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:56:47.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:56:47 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:56:47.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:56:47 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:56:47.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:56:47 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:56:47.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:56:47 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:56:47.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:56:47 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:56:47.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:56:47 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:56:48.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:56:48 smithi153 ceph-mon[70044]: pgmap v2658: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:48.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:56:48 smithi083 ceph-mon[79930]: pgmap v2658: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:48.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:56:48 smithi083 ceph-mon[77242]: pgmap v2658: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:56:50 smithi153 ceph-mon[70044]: pgmap v2659: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:50.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:56:50 smithi083 ceph-mon[77242]: pgmap v2659: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:50.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:56:50 smithi083 ceph-mon[79930]: pgmap v2659: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:50.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:56:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:56:50.444Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:56:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:56:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:56:51] "GET /metrics HTTP/1.1" 200 225177 "" "Prometheus/2.33.4" 2022-09-29T06:56:52.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:56:52 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:56:52.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:56:52 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:56:52.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:56:52 smithi153 ceph-mon[70044]: pgmap v2660: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:52.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:56:52 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:56:52.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:56:52 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:56:52.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:56:52 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:56:52.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:56:52 smithi083 ceph-mon[79930]: pgmap v2660: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:52.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:56:52 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:56:52.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:56:52 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:56:52.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:56:52 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:56:52.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:56:52 smithi083 ceph-mon[77242]: pgmap v2660: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:52.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:56:52 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:56:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:56:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:56:52.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:56:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:56:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:56:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:56:54.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:56:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:56:53.847Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:56:54.550 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:56:54 smithi083 ceph-mon[79930]: pgmap v2661: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:54.551 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:56:54 smithi083 ceph-mon[77242]: pgmap v2661: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:54.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:56:54 smithi153 ceph-mon[70044]: pgmap v2661: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:54.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:56:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:56:54.549Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:56:56.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:56:56 smithi153 ceph-mon[70044]: pgmap v2662: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:56.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:56:56 smithi083 ceph-mon[79930]: pgmap v2662: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:56.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:56:56 smithi083 ceph-mon[77242]: pgmap v2662: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:56:58 smithi153 ceph-mon[70044]: pgmap v2663: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:58.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:56:58 smithi083 ceph-mon[79930]: pgmap v2663: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:56:58.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:56:58 smithi083 ceph-mon[77242]: pgmap v2663: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:57:00 smithi153 ceph-mon[70044]: pgmap v2664: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:00.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:57:00 smithi083 ceph-mon[79930]: pgmap v2664: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:00.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:57:00 smithi083 ceph-mon[77242]: pgmap v2664: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:00.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:57:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:57:00.445Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:57:01.791 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:57:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:57:01] "GET /metrics HTTP/1.1" 200 225178 "" "Prometheus/2.33.4" 2022-09-29T06:57:02.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:57:01 smithi153 ceph-mon[70044]: pgmap v2665: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:02.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:57:01 smithi083 ceph-mon[79930]: pgmap v2665: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:02.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:57:01 smithi083 ceph-mon[77242]: pgmap v2665: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:57:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:57:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:57:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:57:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:57:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:57:04.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:57:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:57:03.848Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:57:04.551 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:57:04 smithi083 ceph-mon[79930]: pgmap v2666: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:04.552 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:57:04 smithi083 ceph-mon[77242]: pgmap v2666: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:57:04 smithi153 ceph-mon[70044]: pgmap v2666: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:04.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:57:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:57:04.550Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:57:06.259 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:57:06.545 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:57:06.546 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (90m) 2m ago 91m 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:57:06.546 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (91m) 15s ago 91m 62.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:57:06.546 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (92m) 2m ago 92m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:57:06.546 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (92m) 15s ago 92m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:57:06.547 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (87m) 15s ago 94m 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:57:06.547 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (88m) 2m ago 96m 589M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:57:06.547 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (87m) 2m ago 96m 385M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:57:06.547 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (86m) 15s ago 94m 174M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:57:06.548 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (87m) 2m ago 95m 227M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:57:06.548 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (91m) 2m ago 91m 26.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:57:06.548 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (91m) 15s ago 91m 29.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:57:06.548 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (86m) 2m ago 94m 285M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:57:06.548 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (85m) 2m ago 94m 277M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:57:06.549 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (85m) 2m ago 93m 222M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:57:06.549 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (84m) 2m ago 93m 266M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:57:06.549 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (84m) 15s ago 93m 241M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:57:06.549 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (83m) 15s ago 92m 290M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:57:06.549 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (83m) 15s ago 92m 258M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:57:06.550 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (82m) 15s ago 92m 297M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:57:06.550 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (87m) 15s ago 91m 110M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:57:06.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:57:06 smithi153 ceph-mon[70044]: pgmap v2667: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:06.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:57:06 smithi083 ceph-mon[79930]: pgmap v2667: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:06.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:57:06 smithi083 ceph-mon[77242]: pgmap v2667: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:06.901 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:57:06.901 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:57:06.901 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:57:06.901 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:57:06.902 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:57:06.902 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:57:06.902 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:57:06.902 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:57:06.903 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:57:06.903 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:57:06.903 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:57:06.903 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:57:06.903 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:57:06.904 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:57:06.904 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:57:06.904 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:57:06.904 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:57:06.904 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:57:07.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:57:07 smithi153 ceph-mon[70044]: from='client.37239 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:57:07.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:57:07 smithi153 ceph-mon[70044]: from='client.37245 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:57:07.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:57:07 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/141179733' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:57:07.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:57:07 smithi083 ceph-mon[79930]: from='client.37239 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:57:07.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:57:07 smithi083 ceph-mon[79930]: from='client.37245 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:57:07.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:57:07 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/141179733' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:57:07.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:57:07 smithi083 ceph-mon[77242]: from='client.37239 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:57:07.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:57:07 smithi083 ceph-mon[77242]: from='client.37245 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:57:07.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:57:07 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/141179733' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:57:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:57:08 smithi153 ceph-mon[70044]: pgmap v2668: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:08.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:57:08 smithi083 ceph-mon[77242]: pgmap v2668: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:08.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:57:08 smithi083 ceph-mon[79930]: pgmap v2668: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:57:10 smithi153 ceph-mon[70044]: pgmap v2669: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:10.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:57:10 smithi083 ceph-mon[77242]: pgmap v2669: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:10.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:57:10 smithi083 ceph-mon[79930]: pgmap v2669: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:10.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:57:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:57:10.446Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:57:11.793 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:57:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:57:11] "GET /metrics HTTP/1.1" 200 225176 "" "Prometheus/2.33.4" 2022-09-29T06:57:11.794 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:57:11 smithi083 ceph-mon[77242]: pgmap v2670: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:12.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:57:11 smithi153 ceph-mon[70044]: pgmap v2670: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:12.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:57:11 smithi083 ceph-mon[79930]: pgmap v2670: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:57:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:57:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:57:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:57:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:57:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:57:14.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:57:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:57:13.849Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:57:14.552 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:57:14 smithi083 ceph-mon[79930]: pgmap v2671: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:14.553 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:57:14 smithi083 ceph-mon[77242]: pgmap v2671: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:14.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:57:14 smithi153 ceph-mon[70044]: pgmap v2671: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:14.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:57:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:57:14.551Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:57:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:57:16 smithi153 ceph-mon[70044]: pgmap v2672: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:16.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:57:16 smithi083 ceph-mon[79930]: pgmap v2672: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:16.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:57:16 smithi083 ceph-mon[77242]: pgmap v2672: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:57:18 smithi153 ceph-mon[70044]: pgmap v2673: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:18.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:57:18 smithi083 ceph-mon[77242]: pgmap v2673: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:18.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:57:18 smithi083 ceph-mon[79930]: pgmap v2673: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:57:20 smithi153 ceph-mon[70044]: pgmap v2674: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:20.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:57:20 smithi083 ceph-mon[77242]: pgmap v2674: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:20.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:57:20 smithi083 ceph-mon[79930]: pgmap v2674: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:20.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:57:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:57:20.446Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:57:21.795 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:57:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:57:21] "GET /metrics HTTP/1.1" 200 225176 "" "Prometheus/2.33.4" 2022-09-29T06:57:22.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:57:21 smithi153 ceph-mon[70044]: pgmap v2675: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:22.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:57:21 smithi083 ceph-mon[79930]: pgmap v2675: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:22.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:57:21 smithi083 ceph-mon[77242]: pgmap v2675: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:57:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:57:22.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:57:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:57:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:57:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:57:24.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:57:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:57:23.849Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:57:24.552 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:57:24 smithi083 ceph-mon[79930]: pgmap v2676: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:24.553 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:57:24 smithi083 ceph-mon[77242]: pgmap v2676: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:57:24 smithi153 ceph-mon[70044]: pgmap v2676: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:24.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:57:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:57:24.551Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:57:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:57:26 smithi153 ceph-mon[70044]: pgmap v2677: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:26.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:57:26 smithi083 ceph-mon[79930]: pgmap v2677: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:26.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:57:26 smithi083 ceph-mon[77242]: pgmap v2677: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:57:28 smithi153 ceph-mon[70044]: pgmap v2678: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:28.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:57:28 smithi083 ceph-mon[79930]: pgmap v2678: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:28.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:57:28 smithi083 ceph-mon[77242]: pgmap v2678: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:57:30 smithi153 ceph-mon[70044]: pgmap v2679: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:30.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:57:30 smithi083 ceph-mon[79930]: pgmap v2679: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:30.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:57:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:57:30.447Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:57:30.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:57:30 smithi083 ceph-mon[77242]: pgmap v2679: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:31.802 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:57:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:57:31] "GET /metrics HTTP/1.1" 200 225177 "" "Prometheus/2.33.4" 2022-09-29T06:57:32.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:57:31 smithi153 ceph-mon[70044]: pgmap v2680: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:32.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:57:31 smithi083 ceph-mon[79930]: pgmap v2680: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:32.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:57:31 smithi083 ceph-mon[77242]: pgmap v2680: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:57:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:57:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:57:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:57:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:57:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:57:34.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:57:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:57:33.850Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:57:34.554 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:57:34 smithi083 ceph-mon[79930]: pgmap v2681: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:34.554 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:57:34 smithi083 ceph-mon[77242]: pgmap v2681: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:57:34 smithi153 ceph-mon[70044]: pgmap v2681: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:34.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:57:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:57:34.552Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:57:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:57:36 smithi153 ceph-mon[70044]: pgmap v2682: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:36.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:57:36 smithi083 ceph-mon[77242]: pgmap v2682: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:36.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:57:36 smithi083 ceph-mon[79930]: pgmap v2682: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:37.227 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:57:37.519 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:57:37.520 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (91m) 2m ago 91m 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:57:37.520 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (91m) 46s ago 91m 62.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:57:37.520 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (92m) 2m ago 92m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:57:37.520 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (92m) 46s ago 92m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:57:37.521 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (88m) 46s ago 95m 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:57:37.521 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (88m) 2m ago 96m 589M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:57:37.521 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (88m) 2m ago 96m 385M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:57:37.521 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (87m) 46s ago 95m 174M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:57:37.521 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (87m) 2m ago 95m 227M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:57:37.522 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (92m) 2m ago 92m 26.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:57:37.522 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (91m) 46s ago 91m 29.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:57:37.522 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (86m) 2m ago 94m 285M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:57:37.522 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (86m) 2m ago 94m 277M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:57:37.523 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (85m) 2m ago 94m 222M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:57:37.523 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (85m) 2m ago 94m 266M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:57:37.523 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (84m) 46s ago 93m 241M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:57:37.523 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (84m) 46s ago 93m 290M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:57:37.523 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (83m) 46s ago 93m 258M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:57:37.524 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (83m) 46s ago 92m 297M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:57:37.524 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (88m) 46s ago 92m 110M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:57:37.877 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:57:37.878 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:57:37.878 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:57:37.878 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:57:37.878 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:57:37.879 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:57:37.879 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:57:37.879 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:57:37.879 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:57:37.880 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:57:37.880 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:57:37.880 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:57:37.880 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:57:37.881 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:57:37.881 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:57:37.881 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:57:37.881 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:57:37.882 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:57:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:57:38 smithi153 ceph-mon[70044]: pgmap v2683: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:38.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:57:38 smithi153 ceph-mon[70044]: from='client.47191 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:57:38.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:57:38 smithi153 ceph-mon[70044]: from='client.37263 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:57:38.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:57:38 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3469284866' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:57:38.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:57:38 smithi083 ceph-mon[77242]: pgmap v2683: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:38.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:57:38 smithi083 ceph-mon[77242]: from='client.47191 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:57:38.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:57:38 smithi083 ceph-mon[77242]: from='client.37263 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:57:38.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:57:38 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3469284866' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:57:38.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:57:38 smithi083 ceph-mon[79930]: pgmap v2683: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:38.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:57:38 smithi083 ceph-mon[79930]: from='client.47191 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:57:38.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:57:38 smithi083 ceph-mon[79930]: from='client.37263 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:57:38.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:57:38 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3469284866' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:57:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:57:40 smithi153 ceph-mon[70044]: pgmap v2684: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:57:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:57:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:57:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:57:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:57:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:57:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:57:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:57:40.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:57:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:57:40.448Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:57:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:57:40 smithi083 ceph-mon[79930]: pgmap v2684: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:57:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:57:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:57:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:57:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:57:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:57:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:57:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:57:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:57:40 smithi083 ceph-mon[77242]: pgmap v2684: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:57:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:57:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:57:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:57:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:57:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:57:40.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:57:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:57:41.801 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:57:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:57:41] "GET /metrics HTTP/1.1" 200 225179 "" "Prometheus/2.33.4" 2022-09-29T06:57:42.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:57:41 smithi153 ceph-mon[70044]: pgmap v2685: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:42.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:57:41 smithi083 ceph-mon[77242]: pgmap v2685: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:42.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:57:41 smithi083 ceph-mon[79930]: pgmap v2685: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:57:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:57:42.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:57:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:57:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:57:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:57:44.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:57:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:57:43.851Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:57:44.554 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:57:44 smithi083 ceph-mon[79930]: pgmap v2686: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:44.555 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:57:44 smithi083 ceph-mon[77242]: pgmap v2686: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:44.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:57:44 smithi153 ceph-mon[70044]: pgmap v2686: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:44.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:57:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:57:44.553Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:57:46.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:57:46 smithi153 ceph-mon[70044]: pgmap v2687: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:46.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:57:46 smithi083 ceph-mon[77242]: pgmap v2687: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:46.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:57:46 smithi083 ceph-mon[79930]: pgmap v2687: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:48.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:57:48 smithi153 ceph-mon[70044]: pgmap v2688: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:48.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:57:48 smithi083 ceph-mon[77242]: pgmap v2688: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:48.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:57:48 smithi083 ceph-mon[79930]: pgmap v2688: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:57:50 smithi153 ceph-mon[70044]: pgmap v2689: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:50.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:57:50 smithi083 ceph-mon[79930]: pgmap v2689: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:50.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:57:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:57:50.448Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:57:50.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:57:50 smithi083 ceph-mon[77242]: pgmap v2689: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:51.804 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:57:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:57:51] "GET /metrics HTTP/1.1" 200 225179 "" "Prometheus/2.33.4" 2022-09-29T06:57:52.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:57:51 smithi153 ceph-mon[70044]: pgmap v2690: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:52.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:57:51 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:57:52.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:57:51 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:57:52.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:57:51 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:57:52.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:57:51 smithi083 ceph-mon[79930]: pgmap v2690: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:52.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:57:51 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:57:52.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:57:51 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:57:52.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:57:51 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:57:52.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:57:51 smithi083 ceph-mon[77242]: pgmap v2690: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:52.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:57:51 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:57:52.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:57:51 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:57:52.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:57:51 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:57:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:57:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:57:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:57:53.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:57:53 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:57:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:57:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:57:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:57:53.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:57:53 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:57:53.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:57:53 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:57:54.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:57:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:57:53.851Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:57:54.555 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:57:54 smithi083 ceph-mon[79930]: pgmap v2691: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:54.556 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:57:54 smithi083 ceph-mon[77242]: pgmap v2691: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:54.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:57:54 smithi153 ceph-mon[70044]: pgmap v2691: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:54.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:57:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:57:54.554Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:57:56.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:57:56 smithi153 ceph-mon[70044]: pgmap v2692: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:56.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:57:56 smithi083 ceph-mon[77242]: pgmap v2692: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:56.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:57:56 smithi083 ceph-mon[79930]: pgmap v2692: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:57:58 smithi153 ceph-mon[70044]: pgmap v2693: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:58.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:57:58 smithi083 ceph-mon[79930]: pgmap v2693: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:57:58.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:57:58 smithi083 ceph-mon[77242]: pgmap v2693: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:58:00 smithi153 ceph-mon[70044]: pgmap v2694: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:00.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:58:00 smithi083 ceph-mon[77242]: pgmap v2694: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:00.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:58:00 smithi083 ceph-mon[79930]: pgmap v2694: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:00.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:58:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:58:00.449Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:58:01.805 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:58:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:58:01] "GET /metrics HTTP/1.1" 200 225180 "" "Prometheus/2.33.4" 2022-09-29T06:58:02.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:58:01 smithi153 ceph-mon[70044]: pgmap v2695: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:02.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:58:01 smithi083 ceph-mon[79930]: pgmap v2695: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:02.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:58:01 smithi083 ceph-mon[77242]: pgmap v2695: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:58:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:58:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:58:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:58:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:58:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:58:04.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:58:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:58:03.851Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:58:04.556 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:58:04 smithi083 ceph-mon[79930]: pgmap v2696: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:04.556 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:58:04 smithi083 ceph-mon[77242]: pgmap v2696: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:58:04 smithi153 ceph-mon[70044]: pgmap v2696: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:04.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:58:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:58:04.555Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:58:06.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:58:06 smithi153 ceph-mon[70044]: pgmap v2697: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:06.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:58:06 smithi083 ceph-mon[79930]: pgmap v2697: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:06.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:58:06 smithi083 ceph-mon[77242]: pgmap v2697: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:08.202 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:58:08.490 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:58:08.490 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (91m) 3m ago 92m 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:58:08.490 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (92m) 77s ago 92m 62.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:58:08.491 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (93m) 3m ago 93m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:58:08.491 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (93m) 77s ago 93m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:58:08.491 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (88m) 77s ago 95m 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:58:08.491 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (89m) 3m ago 97m 589M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:58:08.492 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (88m) 3m ago 97m 385M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:58:08.492 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (88m) 77s ago 95m 174M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:58:08.492 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (88m) 3m ago 96m 227M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:58:08.492 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (92m) 3m ago 92m 26.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:58:08.492 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (92m) 77s ago 92m 29.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:58:08.493 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (87m) 3m ago 95m 285M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:58:08.493 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (86m) 3m ago 95m 277M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:58:08.493 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (86m) 3m ago 94m 222M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:58:08.493 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (85m) 3m ago 94m 266M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:58:08.494 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (85m) 77s ago 94m 241M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:58:08.494 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (84m) 77s ago 94m 290M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:58:08.494 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (84m) 77s ago 93m 258M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:58:08.494 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (83m) 77s ago 93m 297M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:58:08.495 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (88m) 77s ago 92m 110M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:58:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:58:08 smithi153 ceph-mon[70044]: pgmap v2698: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:08.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:58:08 smithi083 ceph-mon[79930]: pgmap v2698: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:08.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:58:08 smithi083 ceph-mon[77242]: pgmap v2698: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:08.846 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:58:08.846 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:58:08.847 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:58:08.847 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:58:08.847 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:58:08.847 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:58:08.848 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:58:08.848 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:58:08.848 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:58:08.848 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:58:08.849 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:58:08.849 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:58:08.849 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:58:08.849 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:58:08.849 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:58:08.850 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:58:08.850 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:58:08.850 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:58:09.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:58:09 smithi153 ceph-mon[70044]: from='client.37275 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:58:09.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:58:09 smithi153 ceph-mon[70044]: from='client.37281 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:58:09.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:58:09 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4238924840' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:58:09.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:58:09 smithi083 ceph-mon[77242]: from='client.37275 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:58:09.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:58:09 smithi083 ceph-mon[77242]: from='client.37281 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:58:09.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:58:09 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4238924840' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:58:09.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:58:09 smithi083 ceph-mon[79930]: from='client.37275 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:58:09.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:58:09 smithi083 ceph-mon[79930]: from='client.37281 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:58:09.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:58:09 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4238924840' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:58:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:58:10 smithi153 ceph-mon[70044]: pgmap v2699: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:10.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:58:10 smithi083 ceph-mon[79930]: pgmap v2699: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:10.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:58:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:58:10.449Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:58:10.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:58:10 smithi083 ceph-mon[77242]: pgmap v2699: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:11.808 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:58:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:58:11] "GET /metrics HTTP/1.1" 200 225179 "" "Prometheus/2.33.4" 2022-09-29T06:58:12.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:58:11 smithi083 ceph-mon[79930]: pgmap v2700: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:12.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:58:11 smithi083 ceph-mon[77242]: pgmap v2700: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:12.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:58:11 smithi153 ceph-mon[70044]: pgmap v2700: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:58:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:58:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:58:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:58:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:58:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:58:14.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:58:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:58:13.852Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:58:14.556 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:58:14 smithi083 ceph-mon[79930]: pgmap v2701: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:14.557 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:58:14 smithi083 ceph-mon[77242]: pgmap v2701: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:14.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:58:14 smithi153 ceph-mon[70044]: pgmap v2701: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:14.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:58:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:58:14.555Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:58:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:58:16 smithi153 ceph-mon[70044]: pgmap v2702: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:16.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:58:16 smithi083 ceph-mon[79930]: pgmap v2702: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:16.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:58:16 smithi083 ceph-mon[77242]: pgmap v2702: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:58:18 smithi153 ceph-mon[70044]: pgmap v2703: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:18.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:58:18 smithi083 ceph-mon[77242]: pgmap v2703: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:18.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:58:18 smithi083 ceph-mon[79930]: pgmap v2703: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:58:20 smithi153 ceph-mon[70044]: pgmap v2704: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:20.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:58:20 smithi083 ceph-mon[77242]: pgmap v2704: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:20.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:58:20 smithi083 ceph-mon[79930]: pgmap v2704: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:20.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:58:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:58:20.450Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:58:21.810 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:58:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:58:21] "GET /metrics HTTP/1.1" 200 225179 "" "Prometheus/2.33.4" 2022-09-29T06:58:22.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:58:21 smithi083 ceph-mon[79930]: pgmap v2705: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:22.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:58:21 smithi083 ceph-mon[77242]: pgmap v2705: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:22.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:58:21 smithi153 ceph-mon[70044]: pgmap v2705: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:58:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:58:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:58:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:58:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:58:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:58:24.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:58:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:58:23.853Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:58:24.557 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:58:24 smithi083 ceph-mon[79930]: pgmap v2706: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:24.558 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:58:24 smithi083 ceph-mon[77242]: pgmap v2706: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:24.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:58:24 smithi153 ceph-mon[70044]: pgmap v2706: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:24.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:58:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:58:24.556Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:58:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:58:26 smithi153 ceph-mon[70044]: pgmap v2707: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:26.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:58:26 smithi083 ceph-mon[79930]: pgmap v2707: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:26.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:58:26 smithi083 ceph-mon[77242]: pgmap v2707: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:58:28 smithi153 ceph-mon[70044]: pgmap v2708: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:28.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:58:28 smithi083 ceph-mon[77242]: pgmap v2708: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:28.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:58:28 smithi083 ceph-mon[79930]: pgmap v2708: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:58:30 smithi153 ceph-mon[70044]: pgmap v2709: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:30.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:58:30 smithi083 ceph-mon[79930]: pgmap v2709: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:30.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:58:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:58:30.452Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:58:30.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:58:30 smithi083 ceph-mon[77242]: pgmap v2709: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:31.812 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:58:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:58:31] "GET /metrics HTTP/1.1" 200 225177 "" "Prometheus/2.33.4" 2022-09-29T06:58:32.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:58:31 smithi083 ceph-mon[79930]: pgmap v2710: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:32.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:58:31 smithi083 ceph-mon[77242]: pgmap v2710: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:32.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:58:31 smithi153 ceph-mon[70044]: pgmap v2710: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:58:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:58:32.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:58:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:58:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:58:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:58:34.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:58:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:58:33.853Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:58:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:58:34 smithi153 ceph-mon[70044]: pgmap v2711: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:34.558 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:58:34 smithi083 ceph-mon[79930]: pgmap v2711: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:34.559 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:58:34 smithi083 ceph-mon[77242]: pgmap v2711: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:34.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:58:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:58:34.556Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:58:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:58:36 smithi153 ceph-mon[70044]: pgmap v2712: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:36.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:58:36 smithi083 ceph-mon[79930]: pgmap v2712: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:36.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:58:36 smithi083 ceph-mon[77242]: pgmap v2712: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:58:38 smithi153 ceph-mon[70044]: pgmap v2713: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:38.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:58:38 smithi083 ceph-mon[79930]: pgmap v2713: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:38.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:58:38 smithi083 ceph-mon[77242]: pgmap v2713: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:39.205 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:58:39.499 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:58:39.500 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (92m) 3m ago 92m 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:58:39.500 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (92m) 108s ago 92m 62.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:58:39.500 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (93m) 3m ago 93m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:58:39.500 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (93m) 108s ago 93m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:58:39.501 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (89m) 108s ago 96m 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:58:39.501 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (89m) 3m ago 97m 589M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:58:39.501 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (89m) 3m ago 97m 385M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:58:39.501 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (88m) 108s ago 96m 174M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:58:39.501 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (88m) 3m ago 96m 227M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:58:39.502 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (93m) 3m ago 93m 26.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:58:39.502 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (93m) 108s ago 93m 29.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:58:39.502 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (87m) 3m ago 96m 285M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:58:39.502 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (87m) 3m ago 95m 277M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:58:39.503 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (86m) 3m ago 95m 222M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:58:39.503 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (86m) 3m ago 95m 266M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:58:39.503 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (85m) 108s ago 94m 241M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:58:39.503 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (85m) 108s ago 94m 290M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:58:39.503 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (84m) 108s ago 94m 258M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:58:39.504 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (84m) 108s ago 93m 297M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:58:39.504 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (89m) 108s ago 93m 110M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:58:39.855 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:58:39.855 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:58:39.855 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:58:39.856 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:58:39.856 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:58:39.856 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:58:39.856 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:58:39.856 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:58:39.857 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:58:39.857 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:58:39.857 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:58:39.857 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:58:39.858 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:58:39.858 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:58:39.858 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:58:39.858 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:58:39.859 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:58:39.859 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:58:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:58:40 smithi153 ceph-mon[70044]: from='client.37293 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:58:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:58:40 smithi153 ceph-mon[70044]: pgmap v2714: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:58:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:58:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:58:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:58:40.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:58:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:58:40.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:58:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:58:40.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:58:40 smithi153 ceph-mon[70044]: from='client.47233 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:58:40.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:58:40 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3793879807' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:58:40.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:58:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:58:40.451Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:58:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:58:40 smithi083 ceph-mon[79930]: from='client.37293 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:58:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:58:40 smithi083 ceph-mon[79930]: pgmap v2714: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:58:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:58:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:58:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:58:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:58:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:58:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:58:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:58:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:58:40 smithi083 ceph-mon[79930]: from='client.47233 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:58:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:58:40 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3793879807' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:58:40.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:58:40 smithi083 ceph-mon[77242]: from='client.37293 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:58:40.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:58:40 smithi083 ceph-mon[77242]: pgmap v2714: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:40.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:58:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:58:40.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:58:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:58:40.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:58:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:58:40.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:58:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:58:40.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:58:40 smithi083 ceph-mon[77242]: from='client.47233 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:58:40.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:58:40 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3793879807' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:58:41.817 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:58:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:58:41] "GET /metrics HTTP/1.1" 200 225177 "" "Prometheus/2.33.4" 2022-09-29T06:58:42.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:58:41 smithi083 ceph-mon[79930]: pgmap v2715: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:42.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:58:41 smithi083 ceph-mon[77242]: pgmap v2715: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:58:41 smithi153 ceph-mon[70044]: pgmap v2715: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:58:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:58:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:58:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:58:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:58:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:58:44.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:58:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:58:43.854Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:58:44.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:58:44 smithi153 ceph-mon[70044]: pgmap v2716: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:44.558 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:58:44 smithi083 ceph-mon[79930]: pgmap v2716: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:44.559 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:58:44 smithi083 ceph-mon[77242]: pgmap v2716: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:44.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:58:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:58:44.557Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:58:46.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:58:46 smithi153 ceph-mon[70044]: pgmap v2717: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:46.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:58:46 smithi083 ceph-mon[79930]: pgmap v2717: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:46.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:58:46 smithi083 ceph-mon[77242]: pgmap v2717: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:48.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:58:48 smithi153 ceph-mon[70044]: pgmap v2718: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:48.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:58:48 smithi083 ceph-mon[77242]: pgmap v2718: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:48.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:58:48 smithi083 ceph-mon[79930]: pgmap v2718: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:58:50 smithi153 ceph-mon[70044]: pgmap v2719: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:50.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:58:50 smithi083 ceph-mon[79930]: pgmap v2719: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:50.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:58:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:58:50.452Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:58:50.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:58:50 smithi083 ceph-mon[77242]: pgmap v2719: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:51.816 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:58:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:58:51] "GET /metrics HTTP/1.1" 200 225177 "" "Prometheus/2.33.4" 2022-09-29T06:58:51.817 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:58:51 smithi083 ceph-mon[77242]: pgmap v2720: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:52.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:58:51 smithi083 ceph-mon[79930]: pgmap v2720: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:52.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:58:51 smithi153 ceph-mon[70044]: pgmap v2720: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:58:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:58:52.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:58:53.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:58:52 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:58:53.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:58:52 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:58:53.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:58:52 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:58:53.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:58:52 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:58:53.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:58:52 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:58:53.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:58:52 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:58:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:58:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:58:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:58:53.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:58:52 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:58:53.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:58:52 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:58:53.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:58:52 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:58:54.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:58:53 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:58:54.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:58:53 smithi083 ceph-mon[79930]: pgmap v2721: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:54.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:58:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:58:53.854Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:58:54.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:58:53 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:58:54.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:58:53 smithi083 ceph-mon[77242]: pgmap v2721: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:54.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:58:53 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:58:54.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:58:53 smithi153 ceph-mon[70044]: pgmap v2721: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:54.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:58:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:58:54.557Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:58:56.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:58:56 smithi153 ceph-mon[70044]: pgmap v2722: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:56.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:58:56 smithi083 ceph-mon[79930]: pgmap v2722: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:56.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:58:56 smithi083 ceph-mon[77242]: pgmap v2722: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:58:58 smithi153 ceph-mon[70044]: pgmap v2723: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:58.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:58:58 smithi083 ceph-mon[77242]: pgmap v2723: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:58:58.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:58:58 smithi083 ceph-mon[79930]: pgmap v2723: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:59:00 smithi153 ceph-mon[70044]: pgmap v2724: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:00.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:59:00 smithi083 ceph-mon[77242]: pgmap v2724: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:00.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:59:00 smithi083 ceph-mon[79930]: pgmap v2724: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:00.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:59:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:59:00.453Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:59:01.821 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:59:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:59:01] "GET /metrics HTTP/1.1" 200 225175 "" "Prometheus/2.33.4" 2022-09-29T06:59:02.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:59:01 smithi083 ceph-mon[77242]: pgmap v2725: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:02.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:59:01 smithi083 ceph-mon[79930]: pgmap v2725: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:02.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:59:01 smithi153 ceph-mon[70044]: pgmap v2725: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:59:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:59:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:59:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:59:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:59:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:59:04.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:59:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:59:03.855Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:59:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:59:04 smithi153 ceph-mon[70044]: pgmap v2726: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:04.559 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:59:04 smithi083 ceph-mon[79930]: pgmap v2726: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:04.559 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:59:04 smithi083 ceph-mon[77242]: pgmap v2726: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:04.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:59:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:59:04.558Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:59:06.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:59:06 smithi153 ceph-mon[70044]: pgmap v2727: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:06.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:59:06 smithi083 ceph-mon[77242]: pgmap v2727: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:06.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:59:06 smithi083 ceph-mon[79930]: pgmap v2727: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:59:08 smithi153 ceph-mon[70044]: pgmap v2728: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:08.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:59:08 smithi083 ceph-mon[79930]: pgmap v2728: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:08.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:59:08 smithi083 ceph-mon[77242]: pgmap v2728: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:10.182 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:59:10.471 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:59:10.471 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (92m) 4m ago 93m 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:59:10.471 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (93m) 2m ago 93m 62.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:59:10.472 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (94m) 4m ago 94m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:59:10.472 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (94m) 2m ago 94m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:59:10.472 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (89m) 2m ago 96m 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:59:10.472 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (90m) 4m ago 98m 589M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:59:10.473 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (89m) 4m ago 98m 385M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:59:10.473 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (89m) 2m ago 96m 174M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:59:10.473 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (89m) 4m ago 97m 227M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:59:10.473 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (93m) 4m ago 93m 26.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:59:10.474 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (93m) 2m ago 93m 29.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:59:10.474 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (88m) 4m ago 96m 285M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:59:10.474 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (87m) 4m ago 96m 277M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:59:10.474 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (87m) 4m ago 95m 222M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:59:10.474 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (86m) 4m ago 95m 266M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:59:10.475 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (86m) 2m ago 95m 241M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:59:10.475 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (85m) 2m ago 95m 290M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:59:10.475 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (85m) 2m ago 94m 258M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:59:10.475 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (84m) 2m ago 94m 297M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:59:10.476 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (89m) 2m ago 93m 110M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:59:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:59:10 smithi153 ceph-mon[70044]: pgmap v2729: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:10.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:59:10 smithi083 ceph-mon[79930]: pgmap v2729: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:10.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:59:10 smithi083 ceph-mon[77242]: pgmap v2729: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:10.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:59:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:59:10.453Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:59:10.827 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:59:10.828 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:59:10.828 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:59:10.828 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:59:10.829 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:59:10.829 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:59:10.829 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:59:10.829 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:59:10.829 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:59:10.830 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:59:10.830 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:59:10.830 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:59:10.830 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:59:10.831 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:59:10.831 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:59:10.831 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:59:10.831 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:59:10.832 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:59:11.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:59:11 smithi153 ceph-mon[70044]: from='client.37311 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:59:11.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:59:11 smithi153 ceph-mon[70044]: from='client.37317 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:59:11.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:59:11 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/455759661' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:59:11.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:59:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:59:11] "GET /metrics HTTP/1.1" 200 225175 "" "Prometheus/2.33.4" 2022-09-29T06:59:11.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:59:11 smithi083 ceph-mon[77242]: from='client.37311 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:59:11.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:59:11 smithi083 ceph-mon[77242]: from='client.37317 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:59:11.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:59:11 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/455759661' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:59:11.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:59:11 smithi083 ceph-mon[79930]: from='client.37311 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:59:11.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:59:11 smithi083 ceph-mon[79930]: from='client.37317 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:59:11.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:59:11 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/455759661' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:59:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:59:12 smithi153 ceph-mon[70044]: pgmap v2730: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:12.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:59:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:59:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:59:12.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:59:12 smithi083 ceph-mon[79930]: pgmap v2730: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:12.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:59:12 smithi083 ceph-mon[77242]: pgmap v2730: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:59:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:59:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:59:14.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:59:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:59:13.856Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:59:14.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:59:14 smithi153 ceph-mon[70044]: pgmap v2731: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:14.559 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:59:14 smithi083 ceph-mon[79930]: pgmap v2731: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:14.560 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:59:14 smithi083 ceph-mon[77242]: pgmap v2731: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:14.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:59:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:59:14.558Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:59:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:59:16 smithi153 ceph-mon[70044]: pgmap v2732: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:16.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:59:16 smithi083 ceph-mon[79930]: pgmap v2732: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:16.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:59:16 smithi083 ceph-mon[77242]: pgmap v2732: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:59:18 smithi153 ceph-mon[70044]: pgmap v2733: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:18.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:59:18 smithi083 ceph-mon[79930]: pgmap v2733: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:18.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:59:18 smithi083 ceph-mon[77242]: pgmap v2733: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:20.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:59:20 smithi083 ceph-mon[79930]: pgmap v2734: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:20.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:59:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:59:20.454Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:59:20.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:59:20 smithi083 ceph-mon[77242]: pgmap v2734: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:20.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:59:20 smithi153 ceph-mon[70044]: pgmap v2734: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:21.827 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:59:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:59:21] "GET /metrics HTTP/1.1" 200 225175 "" "Prometheus/2.33.4" 2022-09-29T06:59:22.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:59:21 smithi083 ceph-mon[79930]: pgmap v2735: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:22.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:59:21 smithi083 ceph-mon[77242]: pgmap v2735: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:22.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:59:21 smithi153 ceph-mon[70044]: pgmap v2735: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:59:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:59:22.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:59:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:59:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:59:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:59:24.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:59:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:59:23.857Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:59:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:59:24 smithi153 ceph-mon[70044]: pgmap v2736: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:24.560 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:59:24 smithi083 ceph-mon[79930]: pgmap v2736: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:24.561 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:59:24 smithi083 ceph-mon[77242]: pgmap v2736: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:24.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:59:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:59:24.559Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:59:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:59:26 smithi153 ceph-mon[70044]: pgmap v2737: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:26.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:59:26 smithi083 ceph-mon[79930]: pgmap v2737: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:26.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:59:26 smithi083 ceph-mon[77242]: pgmap v2737: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:28.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:59:28 smithi153 ceph-mon[70044]: pgmap v2738: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:28.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:59:28 smithi083 ceph-mon[79930]: pgmap v2738: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:28.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:59:28 smithi083 ceph-mon[77242]: pgmap v2738: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:59:30 smithi153 ceph-mon[70044]: pgmap v2739: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:30.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:59:30 smithi083 ceph-mon[77242]: pgmap v2739: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:30.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:59:30 smithi083 ceph-mon[79930]: pgmap v2739: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:30.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:59:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:59:30.455Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:59:31.827 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:59:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:59:31] "GET /metrics HTTP/1.1" 200 225178 "" "Prometheus/2.33.4" 2022-09-29T06:59:32.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:59:31 smithi083 ceph-mon[79930]: pgmap v2740: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:32.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:59:31 smithi083 ceph-mon[77242]: pgmap v2740: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:32.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:59:31 smithi153 ceph-mon[70044]: pgmap v2740: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:59:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:59:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:59:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:59:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:59:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:59:34.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:59:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:59:33.857Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:59:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:59:34 smithi153 ceph-mon[70044]: pgmap v2741: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:34.561 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:59:34 smithi083 ceph-mon[79930]: pgmap v2741: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:34.561 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:59:34 smithi083 ceph-mon[77242]: pgmap v2741: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:34.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:59:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:59:34.560Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:59:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:59:36 smithi153 ceph-mon[70044]: pgmap v2742: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:36.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:59:36 smithi083 ceph-mon[79930]: pgmap v2742: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:36.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:59:36 smithi083 ceph-mon[77242]: pgmap v2742: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:59:38 smithi153 ceph-mon[70044]: pgmap v2743: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:38.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:59:38 smithi083 ceph-mon[79930]: pgmap v2743: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:38.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:59:38 smithi083 ceph-mon[77242]: pgmap v2743: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:59:40 smithi153 ceph-mon[70044]: pgmap v2744: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:59:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:59:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:59:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:59:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:59:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:59:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:59:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:59:40.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:59:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:59:40.456Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:59:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:59:40 smithi083 ceph-mon[79930]: pgmap v2744: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:59:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:59:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:59:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:59:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:59:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:59:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:59:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:59:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:59:40 smithi083 ceph-mon[77242]: pgmap v2744: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:59:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:59:40.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:59:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T06:59:40.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:59:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:59:40.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:59:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T06:59:41.150 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T06:59:41.436 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T06:59:41.437 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (93m) 4m ago 93m 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T06:59:41.437 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (93m) 2m ago 93m 62.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T06:59:41.437 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (94m) 4m ago 94m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T06:59:41.437 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (94m) 2m ago 94m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T06:59:41.438 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (90m) 2m ago 97m 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T06:59:41.438 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (90m) 4m ago 98m 589M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T06:59:41.438 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (90m) 4m ago 98m 385M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T06:59:41.438 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (89m) 2m ago 97m 174M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T06:59:41.439 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (89m) 4m ago 97m 227M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T06:59:41.439 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (94m) 4m ago 94m 26.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T06:59:41.439 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (94m) 2m ago 94m 29.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T06:59:41.439 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (88m) 4m ago 97m 285M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T06:59:41.440 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (88m) 4m ago 96m 277M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T06:59:41.440 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (88m) 4m ago 96m 222M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T06:59:41.440 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (87m) 4m ago 96m 266M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T06:59:41.440 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (86m) 2m ago 95m 241M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T06:59:41.440 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (86m) 2m ago 95m 290M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T06:59:41.441 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (85m) 2m ago 95m 258M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T06:59:41.441 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (85m) 2m ago 94m 297M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T06:59:41.441 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (90m) 2m ago 94m 110M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T06:59:41.793 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T06:59:41.794 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T06:59:41.794 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T06:59:41.794 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:59:41.794 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T06:59:41.795 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T06:59:41.795 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:59:41.795 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T06:59:41.795 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T06:59:41.796 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:59:41.796 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T06:59:41.796 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T06:59:41.796 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T06:59:41.797 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T06:59:41.797 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T06:59:41.797 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T06:59:41.797 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T06:59:41.797 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T06:59:41.826 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:59:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:59:41] "GET /metrics HTTP/1.1" 200 225173 "" "Prometheus/2.33.4" 2022-09-29T06:59:42.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:59:41 smithi083 ceph-mon[79930]: from='client.37329 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:59:42.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:59:41 smithi083 ceph-mon[79930]: pgmap v2745: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:42.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:59:41 smithi083 ceph-mon[79930]: from='client.47269 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:59:42.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:59:41 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1370537156' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:59:42.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:59:41 smithi083 ceph-mon[77242]: from='client.37329 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:59:42.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:59:41 smithi083 ceph-mon[77242]: pgmap v2745: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:42.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:59:41 smithi083 ceph-mon[77242]: from='client.47269 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:59:42.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:59:41 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1370537156' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:59:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:59:41 smithi153 ceph-mon[70044]: from='client.37329 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:59:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:59:41 smithi153 ceph-mon[70044]: pgmap v2745: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:59:41 smithi153 ceph-mon[70044]: from='client.47269 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T06:59:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:59:41 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1370537156' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T06:59:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:59:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:59:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:59:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:59:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:59:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:59:44.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:59:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:59:43.859Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:59:44.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:59:44 smithi153 ceph-mon[70044]: pgmap v2746: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:44.562 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:59:44 smithi083 ceph-mon[79930]: pgmap v2746: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:44.563 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:59:44 smithi083 ceph-mon[77242]: pgmap v2746: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:44.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:59:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:59:44.561Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:59:46.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:59:46 smithi153 ceph-mon[70044]: pgmap v2747: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:46.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:59:46 smithi083 ceph-mon[79930]: pgmap v2747: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:46.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:59:46 smithi083 ceph-mon[77242]: pgmap v2747: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:48.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:59:48 smithi153 ceph-mon[70044]: pgmap v2748: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:48.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:59:48 smithi083 ceph-mon[79930]: pgmap v2748: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:48.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:59:48 smithi083 ceph-mon[77242]: pgmap v2748: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:59:50 smithi153 ceph-mon[70044]: pgmap v2749: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:50.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:59:50 smithi083 ceph-mon[79930]: pgmap v2749: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:50.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:59:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:59:50.456Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:59:50.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:59:50 smithi083 ceph-mon[77242]: pgmap v2749: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:51.832 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 06:59:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:59:51] "GET /metrics HTTP/1.1" 200 225173 "" "Prometheus/2.33.4" 2022-09-29T06:59:52.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:59:51 smithi083 ceph-mon[79930]: pgmap v2750: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:52.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:59:51 smithi083 ceph-mon[77242]: pgmap v2750: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:52.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:59:51 smithi153 ceph-mon[70044]: pgmap v2750: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 06:59:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T06:59:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T06:59:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 06:59:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:06:59:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T06:59:53.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:59:53 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:59:53.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:59:53 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:59:53.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:59:53 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:59:53.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:59:53 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:59:53.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:59:53 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:59:53.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:59:53 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:59:53.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:59:53 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T06:59:53.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:59:53 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T06:59:53.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:59:53 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T06:59:54.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:59:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:59:53.858Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:59:54.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:59:54 smithi153 ceph-mon[70044]: pgmap v2751: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:54.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:59:54 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:59:54.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:59:54 smithi083 ceph-mon[79930]: pgmap v2751: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:54.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:59:54 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:59:54.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 06:59:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T06:59:54.561Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T06:59:54.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:59:54 smithi083 ceph-mon[77242]: pgmap v2751: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:54.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:59:54 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T06:59:56.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:59:56 smithi153 ceph-mon[70044]: pgmap v2752: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:56.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:59:56 smithi083 ceph-mon[77242]: pgmap v2752: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:56.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:59:56 smithi083 ceph-mon[79930]: pgmap v2752: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:58.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 06:59:58 smithi153 ceph-mon[70044]: pgmap v2753: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:58.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 06:59:58 smithi083 ceph-mon[79930]: pgmap v2753: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T06:59:58.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 06:59:58 smithi083 ceph-mon[77242]: pgmap v2753: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:00.458 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:00:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[77219]: 2022-09-29T06:59:59.999+0000 7f160f665700 -1 log_channel(cluster) log [ERR] : overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T07:00:00.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:00:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:00:00.456Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:00:00.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:00:00 smithi083 ceph-mon[77242]: pgmap v2754: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:00.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:00:00 smithi083 ceph-mon[77242]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T07:00:00.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:00:00 smithi083 ceph-mon[79930]: pgmap v2754: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:00.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:00:00 smithi083 ceph-mon[79930]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T07:00:00.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:00:00 smithi153 ceph-mon[70044]: pgmap v2754: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:00.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:00:00 smithi153 ceph-mon[70044]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T07:00:01.832 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:00:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:00:01] "GET /metrics HTTP/1.1" 200 225178 "" "Prometheus/2.33.4" 2022-09-29T07:00:01.833 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:00:01 smithi083 ceph-mon[77242]: pgmap v2755: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:02.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:00:01 smithi083 ceph-mon[79930]: pgmap v2755: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:02.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:00:01 smithi153 ceph-mon[70044]: pgmap v2755: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:00:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:00:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:00:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:00:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:00:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:00:04.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:00:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:00:03.859Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:00:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:00:04 smithi153 ceph-mon[70044]: pgmap v2756: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:04.563 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:00:04 smithi083 ceph-mon[79930]: pgmap v2756: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:04.563 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:00:04 smithi083 ceph-mon[77242]: pgmap v2756: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:04.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:00:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:00:04.561Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:00:06.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:00:06 smithi153 ceph-mon[70044]: pgmap v2757: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:06.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:00:06 smithi083 ceph-mon[79930]: pgmap v2757: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:06.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:00:06 smithi083 ceph-mon[77242]: pgmap v2757: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:08.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:00:08 smithi083 ceph-mon[79930]: pgmap v2758: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:08.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:00:08 smithi083 ceph-mon[77242]: pgmap v2758: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:08.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:00:08 smithi153 ceph-mon[70044]: pgmap v2758: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:10.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:00:10 smithi083 ceph-mon[79930]: pgmap v2759: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:10.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:00:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:00:10.457Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:00:10.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:00:10 smithi083 ceph-mon[77242]: pgmap v2759: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:10.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:00:10 smithi153 ceph-mon[70044]: pgmap v2759: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:11.836 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:00:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:00:11] "GET /metrics HTTP/1.1" 200 225177 "" "Prometheus/2.33.4" 2022-09-29T07:00:12.121 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:00:12.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:00:11 smithi083 ceph-mon[79930]: pgmap v2760: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:12.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:00:11 smithi083 ceph-mon[77242]: pgmap v2760: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:12.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:00:11 smithi153 ceph-mon[70044]: pgmap v2760: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:12.407 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:00:12.408 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (93m) 5m ago 94m 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:00:12.408 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (94m) 3m ago 94m 62.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:00:12.408 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (95m) 5m ago 95m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:00:12.409 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (95m) 3m ago 95m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:00:12.409 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (90m) 3m ago 97m 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:00:12.409 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (91m) 5m ago 99m 589M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:00:12.409 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (90m) 5m ago 99m 385M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:00:12.410 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (90m) 3m ago 98m 174M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:00:12.410 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (90m) 5m ago 98m 227M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:00:12.410 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (94m) 5m ago 94m 26.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:00:12.410 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (94m) 3m ago 94m 29.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:00:12.411 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (89m) 5m ago 97m 285M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:00:12.411 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (88m) 5m ago 97m 277M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:00:12.411 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (88m) 5m ago 96m 222M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:00:12.411 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (87m) 5m ago 96m 266M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:00:12.412 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (87m) 3m ago 96m 241M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:00:12.412 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (86m) 3m ago 96m 290M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:00:12.412 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (86m) 3m ago 95m 258M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:00:12.412 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (85m) 3m ago 95m 297M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:00:12.413 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (91m) 3m ago 94m 110M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:00:12.765 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:00:12.765 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:00:12.766 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:00:12.766 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:00:12.766 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:00:12.767 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:00:12.767 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:00:12.767 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:00:12.767 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:00:12.767 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:00:12.768 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:00:12.768 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:00:12.768 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:00:12.768 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:00:12.769 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:00:12.769 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:00:12.769 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:00:12.769 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:00:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:00:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:00:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:00:13.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:00:12 smithi083 ceph-mon[79930]: from='client.37347 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:00:13.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:00:12 smithi083 ceph-mon[79930]: from='client.37353 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:00:13.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:00:12 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2074103796' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:00:13.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:00:12 smithi083 ceph-mon[77242]: from='client.37347 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:00:13.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:00:12 smithi083 ceph-mon[77242]: from='client.37353 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:00:13.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:00:12 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2074103796' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:00:13.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:00:12 smithi153 ceph-mon[70044]: from='client.37347 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:00:13.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:00:12 smithi153 ceph-mon[70044]: from='client.37353 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:00:13.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:00:12 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2074103796' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:00:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:00:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:00:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:00:14.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:00:13 smithi083 ceph-mon[77242]: pgmap v2761: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:14.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:00:13 smithi083 ceph-mon[79930]: pgmap v2761: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:14.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:00:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:00:13.860Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:00:14.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:00:13 smithi153 ceph-mon[70044]: pgmap v2761: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:14.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:00:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:00:14.561Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:00:16.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:00:16 smithi083 ceph-mon[79930]: pgmap v2762: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:16.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:00:16 smithi083 ceph-mon[77242]: pgmap v2762: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:16.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:00:16 smithi153 ceph-mon[70044]: pgmap v2762: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:18.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:00:18 smithi083 ceph-mon[79930]: pgmap v2763: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:18.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:00:18 smithi083 ceph-mon[77242]: pgmap v2763: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:18.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:00:18 smithi153 ceph-mon[70044]: pgmap v2763: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:20.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:00:20 smithi083 ceph-mon[77242]: pgmap v2764: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:20.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:00:20 smithi083 ceph-mon[79930]: pgmap v2764: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:20.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:00:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:00:20.458Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:00:20.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:00:20 smithi153 ceph-mon[70044]: pgmap v2764: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:21.837 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:00:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:00:21] "GET /metrics HTTP/1.1" 200 225177 "" "Prometheus/2.33.4" 2022-09-29T07:00:21.837 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:00:21 smithi083 ceph-mon[77242]: pgmap v2765: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:22.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:00:21 smithi083 ceph-mon[79930]: pgmap v2765: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:22.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:00:21 smithi153 ceph-mon[70044]: pgmap v2765: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:00:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:00:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:00:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:00:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:00:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:00:24.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:00:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:00:23.861Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:00:24.563 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:00:24 smithi083 ceph-mon[79930]: pgmap v2766: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:24.564 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:00:24 smithi083 ceph-mon[77242]: pgmap v2766: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:24.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:00:24 smithi153 ceph-mon[70044]: pgmap v2766: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:24.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:00:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:00:24.562Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:00:26.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:00:26 smithi083 ceph-mon[77242]: pgmap v2767: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:26.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:00:26 smithi083 ceph-mon[79930]: pgmap v2767: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:26.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:00:26 smithi153 ceph-mon[70044]: pgmap v2767: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:28.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:00:28 smithi083 ceph-mon[79930]: pgmap v2768: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:28.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:00:28 smithi083 ceph-mon[77242]: pgmap v2768: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:28.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:00:28 smithi153 ceph-mon[70044]: pgmap v2768: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:30.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:00:30 smithi083 ceph-mon[79930]: pgmap v2769: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:30.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:00:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:00:30.459Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:00:30.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:00:30 smithi083 ceph-mon[77242]: pgmap v2769: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:30.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:00:30 smithi153 ceph-mon[70044]: pgmap v2769: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:31.841 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:00:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:00:31] "GET /metrics HTTP/1.1" 200 225180 "" "Prometheus/2.33.4" 2022-09-29T07:00:31.842 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:00:31 smithi083 ceph-mon[77242]: pgmap v2770: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:32.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:00:31 smithi083 ceph-mon[79930]: pgmap v2770: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:32.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:00:31 smithi153 ceph-mon[70044]: pgmap v2770: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:00:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:00:32.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:00:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:00:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:00:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:00:34.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:00:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:00:33.862Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:00:34.564 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:00:34 smithi083 ceph-mon[79930]: pgmap v2771: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:34.564 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:00:34 smithi083 ceph-mon[77242]: pgmap v2771: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:34.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:00:34 smithi153 ceph-mon[70044]: pgmap v2771: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:34.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:00:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:00:34.562Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:00:36.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:00:36 smithi083 ceph-mon[79930]: pgmap v2772: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:36.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:00:36 smithi083 ceph-mon[77242]: pgmap v2772: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:36.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:00:36 smithi153 ceph-mon[70044]: pgmap v2772: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:38.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:00:38 smithi083 ceph-mon[79930]: pgmap v2773: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:38.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:00:38 smithi083 ceph-mon[77242]: pgmap v2773: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:38.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:00:38 smithi153 ceph-mon[70044]: pgmap v2773: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:40.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:00:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:00:40.459Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:00:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:00:40 smithi083 ceph-mon[79930]: pgmap v2774: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:00:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:00:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:00:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:00:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:00:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:00:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:00:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:00:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:00:40 smithi083 ceph-mon[77242]: pgmap v2774: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:40.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:00:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:00:40.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:00:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:00:40.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:00:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:00:40.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:00:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:00:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:00:40 smithi153 ceph-mon[70044]: pgmap v2774: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:00:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:00:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:00:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:00:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:00:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:00:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:00:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:00:41.841 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:00:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:00:41] "GET /metrics HTTP/1.1" 200 225180 "" "Prometheus/2.33.4" 2022-09-29T07:00:42.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:00:41 smithi083 ceph-mon[77242]: pgmap v2775: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:42.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:00:41 smithi083 ceph-mon[79930]: pgmap v2775: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:00:41 smithi153 ceph-mon[70044]: pgmap v2775: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:00:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:00:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:00:43.092 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:00:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:00:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:00:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:00:43.388 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:00:43.388 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (94m) 5m ago 94m 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:00:43.388 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (94m) 3m ago 94m 62.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:00:43.389 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (95m) 5m ago 95m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:00:43.389 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (95m) 3m ago 95m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:00:43.389 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (91m) 3m ago 98m 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:00:43.389 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (91m) 5m ago 99m 589M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:00:43.390 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (91m) 5m ago 99m 385M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:00:43.390 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (90m) 3m ago 98m 174M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:00:43.390 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (90m) 5m ago 98m 227M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:00:43.390 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (95m) 5m ago 95m 26.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:00:43.390 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (95m) 3m ago 95m 29.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:00:43.391 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (89m) 5m ago 98m 285M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:00:43.391 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (89m) 5m ago 97m 277M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:00:43.391 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (89m) 5m ago 97m 222M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:00:43.391 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (88m) 5m ago 97m 266M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:00:43.392 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (87m) 3m ago 96m 241M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:00:43.392 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (87m) 3m ago 96m 290M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:00:43.392 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (86m) 3m ago 96m 258M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:00:43.392 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (86m) 3m ago 96m 297M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:00:43.392 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (91m) 3m ago 95m 110M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:00:43.753 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:00:43.753 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:00:43.754 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:00:43.754 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:00:43.754 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:00:43.754 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:00:43.755 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:00:43.755 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:00:43.755 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:00:43.755 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:00:43.756 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:00:43.756 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:00:43.756 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:00:43.756 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:00:43.757 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:00:43.757 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:00:43.757 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:00:43.757 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:00:44.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:00:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:00:43.862Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:00:44.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:00:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:00:44.562Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:00:44.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:00:44 smithi083 ceph-mon[77242]: from='client.47299 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:00:44.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:00:44 smithi083 ceph-mon[77242]: pgmap v2776: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:44.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:00:44 smithi083 ceph-mon[77242]: from='client.47305 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:00:44.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:00:44 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/921432307' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:00:44.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:00:44 smithi083 ceph-mon[79930]: from='client.47299 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:00:44.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:00:44 smithi083 ceph-mon[79930]: pgmap v2776: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:44.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:00:44 smithi083 ceph-mon[79930]: from='client.47305 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:00:44.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:00:44 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/921432307' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:00:44.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:00:44 smithi153 ceph-mon[70044]: from='client.47299 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:00:44.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:00:44 smithi153 ceph-mon[70044]: pgmap v2776: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:44.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:00:44 smithi153 ceph-mon[70044]: from='client.47305 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:00:44.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:00:44 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/921432307' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:00:46.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:00:46 smithi083 ceph-mon[77242]: pgmap v2777: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:46.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:00:46 smithi083 ceph-mon[79930]: pgmap v2777: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:46.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:00:46 smithi153 ceph-mon[70044]: pgmap v2777: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:48.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:00:48 smithi083 ceph-mon[79930]: pgmap v2778: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:48.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:00:48 smithi083 ceph-mon[77242]: pgmap v2778: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:48.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:00:48 smithi153 ceph-mon[70044]: pgmap v2778: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:50.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:00:50 smithi083 ceph-mon[77242]: pgmap v2779: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:50.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:00:50 smithi083 ceph-mon[79930]: pgmap v2779: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:50.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:00:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:00:50.460Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:00:50.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:00:50 smithi153 ceph-mon[70044]: pgmap v2779: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:51.844 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:00:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:00:51] "GET /metrics HTTP/1.1" 200 225180 "" "Prometheus/2.33.4" 2022-09-29T07:00:52.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:00:51 smithi083 ceph-mon[79930]: pgmap v2780: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:52.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:00:51 smithi083 ceph-mon[77242]: pgmap v2780: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:52.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:00:51 smithi153 ceph-mon[70044]: pgmap v2780: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:00:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:00:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:00:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:00:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:00:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:00:54.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:00:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:00:53.863Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:00:54.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:00:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:00:54.563Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:00:54.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:00:54 smithi083 ceph-mon[77242]: pgmap v2781: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:54.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:00:54 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:00:54.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:00:54 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:00:54.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:00:54 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:00:54.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:00:54 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:00:54.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:00:54 smithi083 ceph-mon[79930]: pgmap v2781: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:54.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:00:54 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:00:54.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:00:54 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:00:54.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:00:54 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:00:54.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:00:54 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:00:54.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:00:54 smithi153 ceph-mon[70044]: pgmap v2781: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:54.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:00:54 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:00:54.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:00:54 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:00:54.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:00:54 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:00:54.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:00:54 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:00:56.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:00:56 smithi083 ceph-mon[77242]: pgmap v2782: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:56.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:00:56 smithi083 ceph-mon[79930]: pgmap v2782: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:56.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:00:56 smithi153 ceph-mon[70044]: pgmap v2782: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:58.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:00:58 smithi083 ceph-mon[77242]: pgmap v2783: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:58.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:00:58 smithi083 ceph-mon[79930]: pgmap v2783: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:00:58.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:00:58 smithi153 ceph-mon[70044]: pgmap v2783: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:00.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:01:00 smithi083 ceph-mon[77242]: pgmap v2784: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:00.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:01:00 smithi083 ceph-mon[79930]: pgmap v2784: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:00.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:01:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:01:00.461Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:01:00.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:01:00 smithi153 ceph-mon[70044]: pgmap v2784: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:01.847 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:01:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:01:01] "GET /metrics HTTP/1.1" 200 225177 "" "Prometheus/2.33.4" 2022-09-29T07:01:02.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:01:01 smithi083 ceph-mon[77242]: pgmap v2785: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:02.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:01:01 smithi083 ceph-mon[79930]: pgmap v2785: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:02.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:01:01 smithi153 ceph-mon[70044]: pgmap v2785: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:01:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:01:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:01:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:01:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:01:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:01:04.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:01:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:01:03.864Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:01:04.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:01:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:01:04.564Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:01:04.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:01:04 smithi083 ceph-mon[77242]: pgmap v2786: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:04.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:01:04 smithi083 ceph-mon[79930]: pgmap v2786: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:04.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:01:04 smithi153 ceph-mon[70044]: pgmap v2786: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:06.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:01:06 smithi083 ceph-mon[77242]: pgmap v2787: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:06.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:01:06 smithi083 ceph-mon[79930]: pgmap v2787: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:06.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:01:06 smithi153 ceph-mon[70044]: pgmap v2787: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:08.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:01:08 smithi083 ceph-mon[79930]: pgmap v2788: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:08.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:01:08 smithi083 ceph-mon[77242]: pgmap v2788: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:08.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:01:08 smithi153 ceph-mon[70044]: pgmap v2788: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:10.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:01:10 smithi083 ceph-mon[79930]: pgmap v2789: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:10.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:01:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:01:10.461Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:01:10.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:01:10 smithi083 ceph-mon[77242]: pgmap v2789: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:10.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:01:10 smithi153 ceph-mon[70044]: pgmap v2789: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:11.848 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:01:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:01:11] "GET /metrics HTTP/1.1" 200 225172 "" "Prometheus/2.33.4" 2022-09-29T07:01:12.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:01:11 smithi083 ceph-mon[77242]: pgmap v2790: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:12.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:01:11 smithi083 ceph-mon[79930]: pgmap v2790: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:12.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:01:11 smithi153 ceph-mon[70044]: pgmap v2790: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:01:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:01:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:01:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:01:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:01:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:01:14.081 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:01:14.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:01:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:01:13.865Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:01:14.370 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:01:14.371 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (94m) 6m ago 95m 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:01:14.371 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (95m) 4m ago 95m 62.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:01:14.371 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (96m) 6m ago 96m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:01:14.371 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (96m) 4m ago 96m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:01:14.372 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (91m) 4m ago 98m 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:01:14.372 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (92m) 6m ago 100m 589M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:01:14.372 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (91m) 6m ago 100m 385M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:01:14.372 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (91m) 4m ago 99m 174M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:01:14.372 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (91m) 6m ago 99m 227M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:01:14.373 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (95m) 6m ago 95m 26.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:01:14.373 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (95m) 4m ago 95m 29.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:01:14.373 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (90m) 6m ago 98m 285M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:01:14.373 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (89m) 6m ago 98m 277M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:01:14.373 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (89m) 6m ago 97m 222M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:01:14.374 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (89m) 6m ago 97m 266M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:01:14.374 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (88m) 4m ago 97m 241M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:01:14.374 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (87m) 4m ago 97m 290M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:01:14.374 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (87m) 4m ago 96m 258M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:01:14.374 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (86m) 4m ago 96m 297M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:01:14.375 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (92m) 4m ago 95m 110M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:01:14.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:01:14 smithi083 ceph-mon[79930]: pgmap v2791: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:14.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:01:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:01:14.564Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:01:14.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:01:14 smithi083 ceph-mon[77242]: pgmap v2791: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:14.730 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:01:14.730 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:01:14.731 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:01:14.731 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:01:14.731 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:01:14.731 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:01:14.732 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:01:14.732 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:01:14.732 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:01:14.732 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:01:14.732 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:01:14.733 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:01:14.733 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:01:14.733 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:01:14.733 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:01:14.734 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:01:14.734 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:01:14.734 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:01:14.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:01:14 smithi153 ceph-mon[70044]: pgmap v2791: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:15.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:01:15 smithi083 ceph-mon[79930]: from='client.37383 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:01:15.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:01:15 smithi083 ceph-mon[79930]: from='client.37389 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:01:15.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:01:15 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4091073200' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:01:15.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:01:15 smithi083 ceph-mon[77242]: from='client.37383 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:01:15.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:01:15 smithi083 ceph-mon[77242]: from='client.37389 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:01:15.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:01:15 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4091073200' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:01:15.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:01:15 smithi153 ceph-mon[70044]: from='client.37383 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:01:15.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:01:15 smithi153 ceph-mon[70044]: from='client.37389 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:01:15.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:01:15 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4091073200' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:01:16.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:01:16 smithi083 ceph-mon[79930]: pgmap v2792: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:16.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:01:16 smithi083 ceph-mon[77242]: pgmap v2792: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:16.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:01:16 smithi153 ceph-mon[70044]: pgmap v2792: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:18.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:01:18 smithi083 ceph-mon[79930]: pgmap v2793: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:18.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:01:18 smithi083 ceph-mon[77242]: pgmap v2793: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:18.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:01:18 smithi153 ceph-mon[70044]: pgmap v2793: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:20.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:01:20 smithi083 ceph-mon[79930]: pgmap v2794: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:20.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:01:20 smithi083 ceph-mon[77242]: pgmap v2794: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:20.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:01:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:01:20.462Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:01:20.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:01:20 smithi153 ceph-mon[70044]: pgmap v2794: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:21.850 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:01:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:01:21] "GET /metrics HTTP/1.1" 200 225172 "" "Prometheus/2.33.4" 2022-09-29T07:01:21.851 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:01:21 smithi083 ceph-mon[77242]: pgmap v2795: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:22.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:01:21 smithi083 ceph-mon[79930]: pgmap v2795: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:22.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:01:21 smithi153 ceph-mon[70044]: pgmap v2795: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:01:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:01:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:01:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:01:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:01:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:01:24.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:01:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:01:23.866Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:01:24.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:01:24 smithi083 ceph-mon[79930]: pgmap v2796: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:24.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:01:24 smithi083 ceph-mon[77242]: pgmap v2796: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:01:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:01:24.565Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:01:24.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:01:24 smithi153 ceph-mon[70044]: pgmap v2796: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:26.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:01:26 smithi083 ceph-mon[79930]: pgmap v2797: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:26.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:01:26 smithi083 ceph-mon[77242]: pgmap v2797: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:26.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:01:26 smithi153 ceph-mon[70044]: pgmap v2797: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:28.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:01:28 smithi083 ceph-mon[79930]: pgmap v2798: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:28.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:01:28 smithi083 ceph-mon[77242]: pgmap v2798: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:28.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:01:28 smithi153 ceph-mon[70044]: pgmap v2798: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:30.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:01:30 smithi083 ceph-mon[79930]: pgmap v2799: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:30.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:01:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:01:30.464Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:01:30.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:01:30 smithi083 ceph-mon[77242]: pgmap v2799: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:30.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:01:30 smithi153 ceph-mon[70044]: pgmap v2799: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:31.854 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:01:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:01:31] "GET /metrics HTTP/1.1" 200 225175 "" "Prometheus/2.33.4" 2022-09-29T07:01:32.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:01:31 smithi083 ceph-mon[79930]: pgmap v2800: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:32.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:01:31 smithi083 ceph-mon[77242]: pgmap v2800: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:32.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:01:31 smithi153 ceph-mon[70044]: pgmap v2800: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:01:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:01:32.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:01:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:01:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:01:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:01:34.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:01:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:01:33.868Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:01:34.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:01:34 smithi083 ceph-mon[77242]: pgmap v2801: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:34.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:01:34 smithi083 ceph-mon[79930]: pgmap v2801: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:34.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:01:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:01:34.565Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:01:34.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:01:34 smithi153 ceph-mon[70044]: pgmap v2801: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:36.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:01:36 smithi083 ceph-mon[79930]: pgmap v2802: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:36.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:01:36 smithi083 ceph-mon[77242]: pgmap v2802: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:36.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:01:36 smithi153 ceph-mon[70044]: pgmap v2802: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:38.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:01:38 smithi083 ceph-mon[79930]: pgmap v2803: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:38.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:01:38 smithi083 ceph-mon[77242]: pgmap v2803: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:38.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:01:38 smithi153 ceph-mon[70044]: pgmap v2803: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:40.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:01:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:01:40.464Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:01:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:01:40 smithi083 ceph-mon[79930]: pgmap v2804: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:01:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:01:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:01:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:01:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:01:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:01:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:01:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:01:40.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:01:40 smithi083 ceph-mon[77242]: pgmap v2804: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:40.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:01:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:01:40.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:01:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:01:40.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:01:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:01:40.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:01:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:01:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:01:40 smithi153 ceph-mon[70044]: pgmap v2804: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:01:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:01:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:01:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:01:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:01:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:01:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:01:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:01:41.857 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:01:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:01:41] "GET /metrics HTTP/1.1" 200 225176 "" "Prometheus/2.33.4" 2022-09-29T07:01:42.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:01:41 smithi083 ceph-mon[79930]: pgmap v2805: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:42.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:01:41 smithi083 ceph-mon[77242]: pgmap v2805: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:01:41 smithi153 ceph-mon[70044]: pgmap v2805: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:01:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:01:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:01:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:01:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:01:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:01:44.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:01:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:01:43.868Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:01:44.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:01:44 smithi083 ceph-mon[79930]: pgmap v2806: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:01:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:01:44.566Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:01:44.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:01:44 smithi083 ceph-mon[77242]: pgmap v2806: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:44.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:01:44 smithi153 ceph-mon[70044]: pgmap v2806: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:45.056 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:01:45.345 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:01:45.346 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (95m) 7m ago 95m 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:01:45.346 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (95m) 4m ago 95m 62.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:01:45.346 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (96m) 7m ago 96m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:01:45.346 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (96m) 4m ago 96m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:01:45.347 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (92m) 4m ago 99m 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:01:45.347 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (92m) 7m ago 100m 589M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:01:45.347 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (92m) 7m ago 101m 385M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:01:45.347 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (91m) 4m ago 99m 174M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:01:45.348 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (91m) 7m ago 99m 227M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:01:45.348 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (96m) 7m ago 96m 26.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:01:45.348 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (96m) 4m ago 96m 29.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:01:45.348 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (90m) 7m ago 99m 285M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:01:45.349 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (90m) 7m ago 98m 277M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:01:45.349 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (90m) 7m ago 98m 222M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:01:45.349 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (89m) 7m ago 98m 266M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:01:45.349 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (88m) 4m ago 97m 241M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:01:45.349 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (88m) 4m ago 97m 290M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:01:45.350 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (87m) 4m ago 97m 258M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:01:45.350 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (87m) 4m ago 97m 297M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:01:45.350 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (92m) 4m ago 96m 110M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:01:45.703 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:01:45.704 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:01:45.704 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:01:45.704 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:01:45.704 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:01:45.705 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:01:45.705 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:01:45.705 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:01:45.705 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:01:45.706 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:01:45.706 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:01:45.706 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:01:45.706 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:01:45.707 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:01:45.707 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:01:45.707 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:01:45.707 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:01:45.707 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:01:46.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:01:46 smithi083 ceph-mon[79930]: from='client.37401 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:01:46.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:01:46 smithi083 ceph-mon[79930]: pgmap v2807: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:46.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:01:46 smithi083 ceph-mon[79930]: from='client.37407 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:01:46.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:01:46 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3525515545' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:01:46.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:01:46 smithi083 ceph-mon[77242]: from='client.37401 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:01:46.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:01:46 smithi083 ceph-mon[77242]: pgmap v2807: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:46.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:01:46 smithi083 ceph-mon[77242]: from='client.37407 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:01:46.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:01:46 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3525515545' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:01:46.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:01:46 smithi153 ceph-mon[70044]: from='client.37401 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:01:46.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:01:46 smithi153 ceph-mon[70044]: pgmap v2807: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:46.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:01:46 smithi153 ceph-mon[70044]: from='client.37407 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:01:46.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:01:46 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3525515545' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:01:48.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:01:48 smithi083 ceph-mon[79930]: pgmap v2808: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:48.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:01:48 smithi083 ceph-mon[77242]: pgmap v2808: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:48.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:01:48 smithi153 ceph-mon[70044]: pgmap v2808: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:50.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:01:50 smithi083 ceph-mon[79930]: pgmap v2809: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:50.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:01:50 smithi083 ceph-mon[77242]: pgmap v2809: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:50.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:01:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:01:50.465Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:01:50.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:01:50 smithi153 ceph-mon[70044]: pgmap v2809: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:51.860 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:01:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:01:51] "GET /metrics HTTP/1.1" 200 225176 "" "Prometheus/2.33.4" 2022-09-29T07:01:52.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:01:51 smithi083 ceph-mon[77242]: pgmap v2810: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:52.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:01:51 smithi083 ceph-mon[79930]: pgmap v2810: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:52.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:01:51 smithi153 ceph-mon[70044]: pgmap v2810: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:01:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:01:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:01:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:01:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:01:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:01:54.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:01:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:01:53.869Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:01:54.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:01:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:01:54.566Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:01:54.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:01:54 smithi083 ceph-mon[79930]: pgmap v2811: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:54.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:01:54 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:01:54.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:01:54 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:01:54.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:01:54 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:01:54.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:01:54 smithi083 ceph-mon[77242]: pgmap v2811: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:54.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:01:54 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:01:54.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:01:54 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:01:54.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:01:54 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:01:54.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:01:54 smithi153 ceph-mon[70044]: pgmap v2811: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:54.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:01:54 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:01:54.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:01:54 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:01:54.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:01:54 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:01:56.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:01:55 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:01:56.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:01:55 smithi083 ceph-mon[79930]: pgmap v2812: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:56.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:01:55 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:01:56.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:01:55 smithi083 ceph-mon[77242]: pgmap v2812: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:56.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:01:55 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:01:56.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:01:55 smithi153 ceph-mon[70044]: pgmap v2812: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:58.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:01:58 smithi083 ceph-mon[79930]: pgmap v2813: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:58.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:01:58 smithi083 ceph-mon[77242]: pgmap v2813: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:01:58.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:01:58 smithi153 ceph-mon[70044]: pgmap v2813: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:00.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:02:00 smithi083 ceph-mon[79930]: pgmap v2814: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:00.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:02:00 smithi083 ceph-mon[77242]: pgmap v2814: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:00.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:02:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:02:00.466Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:02:00.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:02:00 smithi153 ceph-mon[70044]: pgmap v2814: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:01.858 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:02:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:02:01] "GET /metrics HTTP/1.1" 200 225176 "" "Prometheus/2.33.4" 2022-09-29T07:02:01.859 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:02:01 smithi083 ceph-mon[77242]: pgmap v2815: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:02.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:02:01 smithi083 ceph-mon[79930]: pgmap v2815: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:02.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:02:01 smithi153 ceph-mon[70044]: pgmap v2815: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:02:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:02:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:02:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:02:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:02:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:02:04.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:02:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:02:03.870Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:02:04.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:02:04 smithi083 ceph-mon[77242]: pgmap v2816: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:04.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:02:04 smithi083 ceph-mon[79930]: pgmap v2816: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:04.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:02:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:02:04.567Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:02:04.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:02:04 smithi153 ceph-mon[70044]: pgmap v2816: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:06.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:02:06 smithi083 ceph-mon[79930]: pgmap v2817: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:06.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:02:06 smithi083 ceph-mon[77242]: pgmap v2817: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:06.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:02:06 smithi153 ceph-mon[70044]: pgmap v2817: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:08.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:02:08 smithi083 ceph-mon[79930]: pgmap v2818: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:08.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:02:08 smithi083 ceph-mon[77242]: pgmap v2818: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:08.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:02:08 smithi153 ceph-mon[70044]: pgmap v2818: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:10.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:02:10 smithi083 ceph-mon[79930]: pgmap v2819: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:10.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:02:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:02:10.466Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:02:10.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:02:10 smithi083 ceph-mon[77242]: pgmap v2819: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:10.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:02:10 smithi153 ceph-mon[70044]: pgmap v2819: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:11.860 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:02:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:02:11] "GET /metrics HTTP/1.1" 200 225179 "" "Prometheus/2.33.4" 2022-09-29T07:02:12.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:02:11 smithi083 ceph-mon[77242]: pgmap v2820: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:12.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:02:11 smithi083 ceph-mon[79930]: pgmap v2820: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:12.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:02:11 smithi153 ceph-mon[70044]: pgmap v2820: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:02:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:02:12.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:02:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:02:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:02:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:02:14.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:02:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:02:13.870Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:02:14.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:02:14 smithi083 ceph-mon[77242]: pgmap v2821: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:14.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:02:14 smithi083 ceph-mon[79930]: pgmap v2821: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:14.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:02:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:02:14.567Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:02:14.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:02:14 smithi153 ceph-mon[70044]: pgmap v2821: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:16.031 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:02:16.319 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:02:16.319 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (95m) 7m ago 96m 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:02:16.319 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (96m) 5m ago 96m 62.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:02:16.320 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (97m) 7m ago 97m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:02:16.320 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (97m) 5m ago 97m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:02:16.320 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (93m) 5m ago 99m 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:02:16.320 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (93m) 7m ago 101m 589M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:02:16.320 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (92m) 7m ago 101m 385M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:02:16.321 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (92m) 5m ago 100m 174M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:02:16.321 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (92m) 7m ago 100m 227M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:02:16.321 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (96m) 7m ago 96m 26.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:02:16.321 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (96m) 5m ago 96m 29.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:02:16.322 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (91m) 7m ago 99m 285M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:02:16.322 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (90m) 7m ago 99m 277M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:02:16.322 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (90m) 7m ago 99m 222M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:02:16.322 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (90m) 7m ago 98m 266M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:02:16.323 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (89m) 5m ago 98m 241M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:02:16.323 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (88m) 5m ago 98m 290M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:02:16.323 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (88m) 5m ago 97m 258M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:02:16.323 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (87m) 5m ago 97m 297M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:02:16.323 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (93m) 5m ago 96m 110M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:02:16.676 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:02:16.676 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:02:16.676 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:02:16.677 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:02:16.677 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:02:16.677 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:02:16.677 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:02:16.678 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:02:16.678 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:02:16.678 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:02:16.678 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:02:16.679 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:02:16.679 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:02:16.679 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:02:16.679 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:02:16.680 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:02:16.680 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:02:16.680 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:02:16.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:02:16 smithi083 ceph-mon[79930]: pgmap v2822: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:16.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:02:16 smithi083 ceph-mon[77242]: pgmap v2822: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:16.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:02:16 smithi153 ceph-mon[70044]: pgmap v2822: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:17.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:02:17 smithi083 ceph-mon[79930]: from='client.37419 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:02:17.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:02:17 smithi083 ceph-mon[79930]: from='client.37425 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:02:17.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:02:17 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4201965026' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:02:17.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:02:17 smithi083 ceph-mon[77242]: from='client.37419 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:02:17.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:02:17 smithi083 ceph-mon[77242]: from='client.37425 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:02:17.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:02:17 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4201965026' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:02:17.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:02:17 smithi153 ceph-mon[70044]: from='client.37419 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:02:17.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:02:17 smithi153 ceph-mon[70044]: from='client.37425 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:02:17.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:02:17 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4201965026' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:02:18.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:02:18 smithi083 ceph-mon[79930]: pgmap v2823: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:18.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:02:18 smithi083 ceph-mon[77242]: pgmap v2823: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:18.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:02:18 smithi153 ceph-mon[70044]: pgmap v2823: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:20.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:02:20 smithi083 ceph-mon[77242]: pgmap v2824: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:20.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:02:20 smithi083 ceph-mon[79930]: pgmap v2824: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:20.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:02:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:02:20.466Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:02:20.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:02:20 smithi153 ceph-mon[70044]: pgmap v2824: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:21.862 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:02:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:02:21] "GET /metrics HTTP/1.1" 200 225179 "" "Prometheus/2.33.4" 2022-09-29T07:02:21.863 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:02:21 smithi083 ceph-mon[77242]: pgmap v2825: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:22.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:02:21 smithi083 ceph-mon[79930]: pgmap v2825: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:22.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:02:21 smithi153 ceph-mon[70044]: pgmap v2825: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:02:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:02:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:02:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:02:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:02:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:02:24.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:02:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:02:23.870Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:02:24.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:02:24 smithi083 ceph-mon[79930]: pgmap v2826: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:02:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:02:24.568Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:02:24.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:02:24 smithi083 ceph-mon[77242]: pgmap v2826: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:24.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:02:24 smithi153 ceph-mon[70044]: pgmap v2826: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:26.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:02:26 smithi083 ceph-mon[79930]: pgmap v2827: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:26.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:02:26 smithi083 ceph-mon[77242]: pgmap v2827: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:26.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:02:26 smithi153 ceph-mon[70044]: pgmap v2827: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:28.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:02:28 smithi083 ceph-mon[77242]: pgmap v2828: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:28.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:02:28 smithi083 ceph-mon[79930]: pgmap v2828: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:28.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:02:28 smithi153 ceph-mon[70044]: pgmap v2828: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:30.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:02:30 smithi083 ceph-mon[77242]: pgmap v2829: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:30.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:02:30 smithi083 ceph-mon[79930]: pgmap v2829: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:30.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:02:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:02:30.467Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:02:30.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:02:30 smithi153 ceph-mon[70044]: pgmap v2829: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:31.868 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:02:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:02:31] "GET /metrics HTTP/1.1" 200 225177 "" "Prometheus/2.33.4" 2022-09-29T07:02:32.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:02:31 smithi083 ceph-mon[79930]: pgmap v2830: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:32.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:02:31 smithi083 ceph-mon[77242]: pgmap v2830: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:32.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:02:31 smithi153 ceph-mon[70044]: pgmap v2830: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:02:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:02:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:02:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:02:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:02:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:02:34.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:02:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:02:33.871Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:02:34.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:02:34 smithi083 ceph-mon[79930]: pgmap v2831: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:34.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:02:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:02:34.569Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:02:34.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:02:34 smithi083 ceph-mon[77242]: pgmap v2831: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:34.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:02:34 smithi153 ceph-mon[70044]: pgmap v2831: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:36.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:02:36 smithi083 ceph-mon[79930]: pgmap v2832: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:36.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:02:36 smithi083 ceph-mon[77242]: pgmap v2832: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:36.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:02:36 smithi153 ceph-mon[70044]: pgmap v2832: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:38.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:02:38 smithi083 ceph-mon[79930]: pgmap v2833: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:38.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:02:38 smithi083 ceph-mon[77242]: pgmap v2833: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:38.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:02:38 smithi153 ceph-mon[70044]: pgmap v2833: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:40.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:02:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:02:40.467Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:02:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:02:40 smithi083 ceph-mon[79930]: pgmap v2834: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:02:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:02:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:02:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:02:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:02:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:02:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:02:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:02:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:02:40 smithi083 ceph-mon[77242]: pgmap v2834: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:40.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:02:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:02:40.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:02:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:02:40.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:02:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:02:40.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:02:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:02:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:02:40 smithi153 ceph-mon[70044]: pgmap v2834: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:02:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:02:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:02:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:02:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:02:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:02:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:02:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:02:41.870 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:02:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:02:41] "GET /metrics HTTP/1.1" 200 225175 "" "Prometheus/2.33.4" 2022-09-29T07:02:42.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:02:41 smithi083 ceph-mon[79930]: pgmap v2835: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:42.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:02:41 smithi083 ceph-mon[77242]: pgmap v2835: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:02:41 smithi153 ceph-mon[70044]: pgmap v2835: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:02:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:02:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:02:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:02:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:02:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:02:44.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:02:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:02:43.872Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:02:44.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:02:44 smithi083 ceph-mon[79930]: pgmap v2836: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:02:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:02:44.570Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:02:44.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:02:44 smithi083 ceph-mon[77242]: pgmap v2836: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:44.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:02:44 smithi153 ceph-mon[70044]: pgmap v2836: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:46.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:02:46 smithi083 ceph-mon[79930]: pgmap v2837: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:46.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:02:46 smithi083 ceph-mon[77242]: pgmap v2837: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:46.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:02:46 smithi153 ceph-mon[70044]: pgmap v2837: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:46.998 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:02:47.288 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:02:47.288 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (96m) 8m ago 96m 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:02:47.289 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (96m) 5m ago 96m 62.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:02:47.289 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (97m) 8m ago 97m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:02:47.289 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (97m) 5m ago 97m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:02:47.289 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (93m) 5m ago 100m 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:02:47.290 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (93m) 8m ago 101m 589M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:02:47.290 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (93m) 8m ago 102m 385M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:02:47.290 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (92m) 5m ago 100m 174M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:02:47.290 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (92m) 8m ago 100m 227M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:02:47.290 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (97m) 8m ago 97m 26.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:02:47.291 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (97m) 5m ago 97m 29.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:02:47.291 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (91m) 8m ago 100m 285M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:02:47.291 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (91m) 8m ago 99m 277M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:02:47.291 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (91m) 8m ago 99m 222M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:02:47.291 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (90m) 8m ago 99m 266M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:02:47.292 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (90m) 5m ago 98m 241M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:02:47.292 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (89m) 5m ago 98m 290M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:02:47.292 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (88m) 5m ago 98m 258M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:02:47.292 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (88m) 5m ago 98m 297M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:02:47.292 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (93m) 5m ago 97m 110M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:02:47.652 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:02:47.652 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:02:47.653 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:02:47.653 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:02:47.653 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:02:47.653 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:02:47.653 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:02:47.654 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:02:47.654 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:02:47.654 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:02:47.654 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:02:47.655 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:02:47.655 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:02:47.655 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:02:47.655 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:02:47.655 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:02:47.656 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:02:47.656 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:02:48.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:02:48 smithi083 ceph-mon[79930]: from='client.47371 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:02:48.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:02:48 smithi083 ceph-mon[79930]: from='client.37443 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:02:48.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:02:48 smithi083 ceph-mon[79930]: pgmap v2838: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:48.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:02:48 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3671443383' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:02:48.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:02:48 smithi083 ceph-mon[77242]: from='client.47371 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:02:48.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:02:48 smithi083 ceph-mon[77242]: from='client.37443 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:02:48.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:02:48 smithi083 ceph-mon[77242]: pgmap v2838: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:48.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:02:48 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3671443383' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:02:48.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:02:48 smithi153 ceph-mon[70044]: from='client.47371 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:02:48.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:02:48 smithi153 ceph-mon[70044]: from='client.37443 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:02:48.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:02:48 smithi153 ceph-mon[70044]: pgmap v2838: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:48.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:02:48 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3671443383' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:02:50.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:02:50 smithi083 ceph-mon[79930]: pgmap v2839: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:50.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:02:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:02:50.468Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:02:50.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:02:50 smithi083 ceph-mon[77242]: pgmap v2839: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:50.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:02:50 smithi153 ceph-mon[70044]: pgmap v2839: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:51.871 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:02:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:02:51] "GET /metrics HTTP/1.1" 200 225175 "" "Prometheus/2.33.4" 2022-09-29T07:02:51.872 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:02:51 smithi083 ceph-mon[77242]: pgmap v2840: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:52.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:02:51 smithi083 ceph-mon[79930]: pgmap v2840: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:52.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:02:51 smithi153 ceph-mon[70044]: pgmap v2840: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:02:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:02:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:02:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:02:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:02:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:02:54.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:02:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:02:53.872Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:02:54.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:02:54 smithi083 ceph-mon[79930]: pgmap v2841: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:54.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:02:54 smithi083 ceph-mon[77242]: pgmap v2841: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:54.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:02:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:02:54.570Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:02:54.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:02:54 smithi153 ceph-mon[70044]: pgmap v2841: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:55.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:02:55 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:02:55.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:02:55 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:02:55.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:02:55 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:02:55.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:02:55 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:02:55.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:02:55 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:02:55.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:02:55 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:02:55.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:02:55 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:02:55.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:02:55 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:02:55.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:02:55 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:02:56.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:02:56 smithi153 ceph-mon[70044]: pgmap v2842: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:56.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:02:56 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:02:56.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:02:56 smithi083 ceph-mon[79930]: pgmap v2842: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:56.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:02:56 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:02:56.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:02:56 smithi083 ceph-mon[77242]: pgmap v2842: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:56.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:02:56 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:02:58.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:02:58 smithi153 ceph-mon[70044]: pgmap v2843: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:58.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:02:58 smithi083 ceph-mon[79930]: pgmap v2843: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:02:58.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:02:58 smithi083 ceph-mon[77242]: pgmap v2843: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:00.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:03:00 smithi153 ceph-mon[70044]: pgmap v2844: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:00.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:03:00 smithi083 ceph-mon[79930]: pgmap v2844: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:00.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:03:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:03:00.468Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:03:00.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:03:00 smithi083 ceph-mon[77242]: pgmap v2844: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:01.872 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:03:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:03:01] "GET /metrics HTTP/1.1" 200 225177 "" "Prometheus/2.33.4" 2022-09-29T07:03:02.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:03:01 smithi083 ceph-mon[79930]: pgmap v2845: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:02.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:03:01 smithi083 ceph-mon[77242]: pgmap v2845: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:02.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:03:01 smithi153 ceph-mon[70044]: pgmap v2845: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:03:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:03:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:03:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:03:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:03:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:03:04.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:03:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:03:03.873Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:03:04.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:03:04 smithi083 ceph-mon[79930]: pgmap v2846: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:04.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:03:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:03:04.570Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:03:04.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:03:04 smithi083 ceph-mon[77242]: pgmap v2846: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:04.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:03:04 smithi153 ceph-mon[70044]: pgmap v2846: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:06.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:03:06 smithi083 ceph-mon[79930]: pgmap v2847: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:06.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:03:06 smithi083 ceph-mon[77242]: pgmap v2847: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:06.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:03:06 smithi153 ceph-mon[70044]: pgmap v2847: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:08.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:03:08 smithi083 ceph-mon[79930]: pgmap v2848: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:08.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:03:08 smithi083 ceph-mon[77242]: pgmap v2848: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:08.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:03:08 smithi153 ceph-mon[70044]: pgmap v2848: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:10.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:03:10 smithi083 ceph-mon[79930]: pgmap v2849: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:10.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:03:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:03:10.468Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:03:10.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:03:10 smithi083 ceph-mon[77242]: pgmap v2849: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:10.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:03:10 smithi153 ceph-mon[70044]: pgmap v2849: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:11.879 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:03:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:03:11] "GET /metrics HTTP/1.1" 200 225174 "" "Prometheus/2.33.4" 2022-09-29T07:03:12.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:03:11 smithi083 ceph-mon[77242]: pgmap v2850: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:12.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:03:11 smithi083 ceph-mon[79930]: pgmap v2850: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:12.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:03:11 smithi153 ceph-mon[70044]: pgmap v2850: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:03:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:03:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:03:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:03:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:03:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:03:14.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:03:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:03:13.873Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:03:14.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:03:14 smithi083 ceph-mon[79930]: pgmap v2851: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:14.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:03:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:03:14.572Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:03:14.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:03:14 smithi083 ceph-mon[77242]: pgmap v2851: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:14.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:03:14 smithi153 ceph-mon[70044]: pgmap v2851: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:16.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:03:16 smithi083 ceph-mon[77242]: pgmap v2852: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:16.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:03:16 smithi083 ceph-mon[79930]: pgmap v2852: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:16.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:03:16 smithi153 ceph-mon[70044]: pgmap v2852: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:17.981 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:03:18.267 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:03:18.267 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (97m) 8m ago 97m 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:03:18.268 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (97m) 6m ago 97m 62.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:03:18.268 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (98m) 8m ago 98m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:03:18.268 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (98m) 6m ago 98m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:03:18.268 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (94m) 6m ago 100m 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:03:18.269 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (94m) 8m ago 102m 589M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:03:18.269 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (93m) 8m ago 102m 385M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:03:18.269 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (93m) 6m ago 101m 174M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:03:18.269 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (93m) 8m ago 101m 227M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:03:18.269 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (97m) 8m ago 97m 26.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:03:18.270 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (97m) 6m ago 97m 29.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:03:18.270 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (92m) 8m ago 100m 285M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:03:18.270 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (91m) 8m ago 100m 277M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:03:18.270 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (91m) 8m ago 100m 222M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:03:18.270 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (91m) 8m ago 99m 266M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:03:18.271 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (90m) 6m ago 99m 241M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:03:18.271 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (89m) 6m ago 99m 290M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:03:18.271 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (89m) 6m ago 98m 258M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:03:18.271 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (88m) 6m ago 98m 297M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:03:18.272 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (94m) 6m ago 97m 110M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:03:18.633 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:03:18.634 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:03:18.634 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:03:18.634 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:03:18.634 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:03:18.635 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:03:18.635 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:03:18.635 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:03:18.635 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:03:18.635 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:03:18.636 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:03:18.636 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:03:18.636 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:03:18.636 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:03:18.636 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:03:18.637 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:03:18.637 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:03:18.637 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:03:18.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:03:18 smithi083 ceph-mon[79930]: pgmap v2853: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:18.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:03:18 smithi083 ceph-mon[77242]: pgmap v2853: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:18.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:03:18 smithi153 ceph-mon[70044]: pgmap v2853: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:19.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:03:19 smithi083 ceph-mon[79930]: from='client.37455 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:03:19.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:03:19 smithi083 ceph-mon[79930]: from='client.37461 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:03:19.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:03:19 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1095346100' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:03:19.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:03:19 smithi083 ceph-mon[77242]: from='client.37455 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:03:19.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:03:19 smithi083 ceph-mon[77242]: from='client.37461 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:03:19.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:03:19 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1095346100' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:03:19.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:03:19 smithi153 ceph-mon[70044]: from='client.37455 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:03:19.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:03:19 smithi153 ceph-mon[70044]: from='client.37461 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:03:19.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:03:19 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1095346100' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:03:20.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:03:20 smithi083 ceph-mon[77242]: pgmap v2854: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:20.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:03:20 smithi083 ceph-mon[79930]: pgmap v2854: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:20.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:03:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:03:20.469Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:03:20.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:03:20 smithi153 ceph-mon[70044]: pgmap v2854: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:21.877 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:03:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:03:21] "GET /metrics HTTP/1.1" 200 225174 "" "Prometheus/2.33.4" 2022-09-29T07:03:22.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:03:21 smithi083 ceph-mon[79930]: pgmap v2855: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:22.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:03:21 smithi083 ceph-mon[77242]: pgmap v2855: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:22.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:03:21 smithi153 ceph-mon[70044]: pgmap v2855: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:03:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:03:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:03:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:03:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:03:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:03:24.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:03:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:03:23.873Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:03:24.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:03:24 smithi083 ceph-mon[79930]: pgmap v2856: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:24.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:03:24 smithi083 ceph-mon[77242]: pgmap v2856: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:03:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:03:24.573Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:03:24.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:03:24 smithi153 ceph-mon[70044]: pgmap v2856: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:26.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:03:26 smithi083 ceph-mon[77242]: pgmap v2857: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:26.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:03:26 smithi083 ceph-mon[79930]: pgmap v2857: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:26.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:03:26 smithi153 ceph-mon[70044]: pgmap v2857: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:28.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:03:28 smithi083 ceph-mon[79930]: pgmap v2858: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:28.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:03:28 smithi083 ceph-mon[77242]: pgmap v2858: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:28.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:03:28 smithi153 ceph-mon[70044]: pgmap v2858: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:30.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:03:30 smithi083 ceph-mon[79930]: pgmap v2859: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:30.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:03:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:03:30.470Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:03:30.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:03:30 smithi083 ceph-mon[77242]: pgmap v2859: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:30.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:03:30 smithi153 ceph-mon[70044]: pgmap v2859: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:31.878 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:03:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:03:31] "GET /metrics HTTP/1.1" 200 225176 "" "Prometheus/2.33.4" 2022-09-29T07:03:31.879 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:03:31 smithi083 ceph-mon[77242]: pgmap v2860: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:32.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:03:31 smithi083 ceph-mon[79930]: pgmap v2860: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:32.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:03:31 smithi153 ceph-mon[70044]: pgmap v2860: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:03:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:03:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:03:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:03:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:03:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:03:34.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:03:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:03:33.874Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:03:34.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:03:34 smithi083 ceph-mon[79930]: pgmap v2861: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:34.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:03:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:03:34.574Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:03:34.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:03:34 smithi083 ceph-mon[77242]: pgmap v2861: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:34.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:03:34 smithi153 ceph-mon[70044]: pgmap v2861: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:36.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:03:36 smithi083 ceph-mon[79930]: pgmap v2862: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:36.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:03:36 smithi083 ceph-mon[77242]: pgmap v2862: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:36.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:03:36 smithi153 ceph-mon[70044]: pgmap v2862: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:38.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:03:38 smithi083 ceph-mon[79930]: pgmap v2863: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:38.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:03:38 smithi083 ceph-mon[77242]: pgmap v2863: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:38.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:03:38 smithi153 ceph-mon[70044]: pgmap v2863: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:40.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:03:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:03:40.470Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:03:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:03:40 smithi083 ceph-mon[77242]: pgmap v2864: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:03:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:03:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:03:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:03:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:03:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:03:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:03:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:03:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:03:40 smithi083 ceph-mon[79930]: pgmap v2864: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:40.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:03:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:03:40.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:03:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:03:40.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:03:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:03:40.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:03:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:03:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:03:40 smithi153 ceph-mon[70044]: pgmap v2864: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:03:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:03:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:03:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:03:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:03:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:03:40.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:03:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:03:41.880 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:03:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:03:41] "GET /metrics HTTP/1.1" 200 225173 "" "Prometheus/2.33.4" 2022-09-29T07:03:42.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:03:41 smithi083 ceph-mon[79930]: pgmap v2865: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:42.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:03:41 smithi083 ceph-mon[77242]: pgmap v2865: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:03:41 smithi153 ceph-mon[70044]: pgmap v2865: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:03:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:03:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:03:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:03:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:03:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:03:44.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:03:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:03:43.875Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:03:44.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:03:44 smithi083 ceph-mon[79930]: pgmap v2866: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:03:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:03:44.574Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:03:44.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:03:44 smithi083 ceph-mon[77242]: pgmap v2866: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:44.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:03:44 smithi153 ceph-mon[70044]: pgmap v2866: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:46.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:03:46 smithi083 ceph-mon[79930]: pgmap v2867: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:46.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:03:46 smithi083 ceph-mon[77242]: pgmap v2867: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:46.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:03:46 smithi153 ceph-mon[70044]: pgmap v2867: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:48.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:03:48 smithi083 ceph-mon[79930]: pgmap v2868: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:48.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:03:48 smithi083 ceph-mon[77242]: pgmap v2868: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:48.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:03:48 smithi153 ceph-mon[70044]: pgmap v2868: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:48.957 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:03:49.248 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:03:49.248 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (97m) 9m ago 97m 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:03:49.248 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (97m) 6m ago 97m 62.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:03:49.249 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (98m) 9m ago 98m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:03:49.249 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (98m) 6m ago 98m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:03:49.249 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (94m) 6m ago 101m 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:03:49.249 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (95m) 9m ago 102m 589M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:03:49.250 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (94m) 9m ago 103m 385M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:03:49.250 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (93m) 6m ago 101m 174M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:03:49.250 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (93m) 9m ago 101m 227M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:03:49.250 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (98m) 9m ago 98m 26.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:03:49.250 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (98m) 6m ago 98m 29.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:03:49.251 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (93m) 9m ago 101m 285M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:03:49.251 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (92m) 9m ago 100m 277M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:03:49.251 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (92m) 9m ago 100m 222M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:03:49.251 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (91m) 9m ago 100m 266M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:03:49.251 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (91m) 6m ago 99m 241M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:03:49.252 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (90m) 6m ago 99m 290M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:03:49.252 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (89m) 6m ago 99m 258M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:03:49.252 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (89m) 6m ago 99m 297M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:03:49.252 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (94m) 6m ago 98m 110M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:03:49.612 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:03:49.613 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:03:49.613 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:03:49.613 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:03:49.613 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:03:49.614 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:03:49.614 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:03:49.614 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:03:49.614 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:03:49.614 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:03:49.615 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:03:49.615 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:03:49.615 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:03:49.615 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:03:49.615 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:03:49.616 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:03:49.616 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:03:49.616 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:03:50.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:03:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:03:50.470Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:03:50.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:03:50 smithi083 ceph-mon[79930]: from='client.37473 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:03:50.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:03:50 smithi083 ceph-mon[79930]: from='client.37479 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:03:50.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:03:50 smithi083 ceph-mon[79930]: pgmap v2869: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:50.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:03:50 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/823583730' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:03:50.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:03:50 smithi083 ceph-mon[77242]: from='client.37473 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:03:50.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:03:50 smithi083 ceph-mon[77242]: from='client.37479 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:03:50.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:03:50 smithi083 ceph-mon[77242]: pgmap v2869: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:50.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:03:50 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/823583730' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:03:50.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:03:50 smithi153 ceph-mon[70044]: from='client.37473 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:03:50.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:03:50 smithi153 ceph-mon[70044]: from='client.37479 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:03:50.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:03:50 smithi153 ceph-mon[70044]: pgmap v2869: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:50.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:03:50 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/823583730' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:03:51.887 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:03:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:03:51] "GET /metrics HTTP/1.1" 200 225173 "" "Prometheus/2.33.4" 2022-09-29T07:03:52.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:03:51 smithi083 ceph-mon[79930]: pgmap v2870: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:52.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:03:51 smithi083 ceph-mon[77242]: pgmap v2870: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:52.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:03:51 smithi153 ceph-mon[70044]: pgmap v2870: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:03:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:03:52.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:03:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:03:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:03:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:03:54.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:03:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:03:53.875Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:03:54.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:03:54 smithi083 ceph-mon[79930]: pgmap v2871: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:03:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:03:54.576Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:03:54.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:03:54 smithi083 ceph-mon[77242]: pgmap v2871: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:54.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:03:54 smithi153 ceph-mon[70044]: pgmap v2871: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:56.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:03:56 smithi083 ceph-mon[79930]: pgmap v2872: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:56.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:03:56 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:03:56.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:03:56 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:03:56.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:03:56 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:03:56.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:03:56 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:03:56.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:03:56 smithi083 ceph-mon[77242]: pgmap v2872: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:56.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:03:56 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:03:56.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:03:56 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:03:56.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:03:56 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:03:56.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:03:56 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:03:56.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:03:56 smithi153 ceph-mon[70044]: pgmap v2872: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:56.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:03:56 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:03:56.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:03:56 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:03:56.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:03:56 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:03:56.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:03:56 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:03:58.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:03:58 smithi083 ceph-mon[79930]: pgmap v2873: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:58.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:03:58 smithi083 ceph-mon[77242]: pgmap v2873: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:03:58.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:03:58 smithi153 ceph-mon[70044]: pgmap v2873: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:00.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:04:00 smithi083 ceph-mon[79930]: pgmap v2874: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:00.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:04:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:04:00.471Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:04:00.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:04:00 smithi083 ceph-mon[77242]: pgmap v2874: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:00.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:04:00 smithi153 ceph-mon[70044]: pgmap v2874: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:01.889 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:04:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:04:01] "GET /metrics HTTP/1.1" 200 225173 "" "Prometheus/2.33.4" 2022-09-29T07:04:02.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:04:01 smithi083 ceph-mon[79930]: pgmap v2875: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:02.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:04:01 smithi083 ceph-mon[77242]: pgmap v2875: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:02.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:04:01 smithi153 ceph-mon[70044]: pgmap v2875: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:04:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:04:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:04:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:04:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:04:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:04:04.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:04:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:04:03.875Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:04:04.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:04:04 smithi083 ceph-mon[79930]: pgmap v2876: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:04.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:04:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:04:04.576Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:04:04.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:04:04 smithi083 ceph-mon[77242]: pgmap v2876: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:04.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:04:04 smithi153 ceph-mon[70044]: pgmap v2876: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:06.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:04:06 smithi083 ceph-mon[77242]: pgmap v2877: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:06.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:04:06 smithi083 ceph-mon[79930]: pgmap v2877: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:06.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:04:06 smithi153 ceph-mon[70044]: pgmap v2877: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:08.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:04:08 smithi083 ceph-mon[77242]: pgmap v2878: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:08.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:04:08 smithi083 ceph-mon[79930]: pgmap v2878: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:08.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:04:08 smithi153 ceph-mon[70044]: pgmap v2878: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:10.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:04:10 smithi083 ceph-mon[77242]: pgmap v2879: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:10.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:04:10 smithi083 ceph-mon[79930]: pgmap v2879: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:10.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:04:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:04:10.471Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:04:10.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:04:10 smithi153 ceph-mon[70044]: pgmap v2879: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:11.887 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:04:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:04:11] "GET /metrics HTTP/1.1" 200 225168 "" "Prometheus/2.33.4" 2022-09-29T07:04:11.888 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:04:11 smithi083 ceph-mon[77242]: pgmap v2880: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:12.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:04:11 smithi083 ceph-mon[79930]: pgmap v2880: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:12.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:04:11 smithi153 ceph-mon[70044]: pgmap v2880: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:04:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:04:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:04:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:04:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:04:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:04:14.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:04:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:04:13.876Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:04:14.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:04:14 smithi083 ceph-mon[79930]: pgmap v2881: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:14.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:04:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:04:14.576Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:04:14.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:04:14 smithi083 ceph-mon[77242]: pgmap v2881: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:14.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:04:14 smithi153 ceph-mon[70044]: pgmap v2881: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:16.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:04:16 smithi083 ceph-mon[77242]: pgmap v2882: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:16.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:04:16 smithi083 ceph-mon[79930]: pgmap v2882: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:16.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:04:16 smithi153 ceph-mon[70044]: pgmap v2882: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:18.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:04:18 smithi083 ceph-mon[79930]: pgmap v2883: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:18.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:04:18 smithi083 ceph-mon[77242]: pgmap v2883: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:18.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:04:18 smithi153 ceph-mon[70044]: pgmap v2883: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:19.940 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:04:20.232 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:04:20.233 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (98m) 9m ago 98m 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:04:20.233 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (98m) 7m ago 98m 62.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:04:20.233 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (99m) 9m ago 99m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:04:20.233 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (99m) 7m ago 99m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:04:20.234 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (95m) 7m ago 101m 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:04:20.234 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (95m) 9m ago 103m 589M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:04:20.234 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (94m) 9m ago 103m 385M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:04:20.234 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (94m) 7m ago 102m 174M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:04:20.235 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (94m) 9m ago 102m 227M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:04:20.235 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (98m) 9m ago 98m 26.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:04:20.235 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (98m) 7m ago 98m 29.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:04:20.235 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (93m) 9m ago 101m 285M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:04:20.235 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (92m) 9m ago 101m 277M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:04:20.236 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (92m) 9m ago 101m 222M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:04:20.236 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (92m) 9m ago 100m 266M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:04:20.236 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (91m) 7m ago 100m 241M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:04:20.236 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (91m) 7m ago 100m 290M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:04:20.236 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (90m) 7m ago 99m 258M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:04:20.237 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (89m) 7m ago 99m 297M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:04:20.237 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (95m) 7m ago 98m 110M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:04:20.591 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:04:20.591 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:04:20.592 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:04:20.592 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:04:20.592 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:04:20.593 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:04:20.593 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:04:20.593 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:04:20.594 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:04:20.594 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:04:20.594 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:04:20.594 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:04:20.595 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:04:20.595 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:04:20.595 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:04:20.596 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:04:20.596 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:04:20.596 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:04:20.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:04:20 smithi083 ceph-mon[79930]: pgmap v2884: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:20.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:04:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:04:20.471Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:04:20.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:04:20 smithi083 ceph-mon[77242]: pgmap v2884: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:20.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:04:20 smithi153 ceph-mon[70044]: pgmap v2884: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:21.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:04:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:04:21] "GET /metrics HTTP/1.1" 200 225168 "" "Prometheus/2.33.4" 2022-09-29T07:04:21.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:04:21 smithi083 ceph-mon[79930]: from='client.37491 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:04:21.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:04:21 smithi083 ceph-mon[79930]: from='client.37497 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:04:21.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:04:21 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3513298807' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:04:21.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:04:21 smithi083 ceph-mon[77242]: from='client.37491 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:04:21.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:04:21 smithi083 ceph-mon[77242]: from='client.37497 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:04:21.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:04:21 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3513298807' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:04:21.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:04:21 smithi153 ceph-mon[70044]: from='client.37491 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:04:21.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:04:21 smithi153 ceph-mon[70044]: from='client.37497 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:04:21.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:04:21 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3513298807' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:04:22.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:04:22 smithi083 ceph-mon[79930]: pgmap v2885: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:22.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:04:22 smithi083 ceph-mon[77242]: pgmap v2885: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:22.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:04:22 smithi153 ceph-mon[70044]: pgmap v2885: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:04:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:04:22.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:04:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:04:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:04:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:04:24.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:04:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:04:23.877Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:04:24.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:04:24 smithi083 ceph-mon[79930]: pgmap v2886: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:04:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:04:24.576Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:04:24.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:04:24 smithi083 ceph-mon[77242]: pgmap v2886: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:24.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:04:24 smithi153 ceph-mon[70044]: pgmap v2886: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:26.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:04:26 smithi083 ceph-mon[79930]: pgmap v2887: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:26.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:04:26 smithi083 ceph-mon[77242]: pgmap v2887: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:26.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:04:26 smithi153 ceph-mon[70044]: pgmap v2887: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:28.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:04:28 smithi083 ceph-mon[79930]: pgmap v2888: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:28.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:04:28 smithi083 ceph-mon[77242]: pgmap v2888: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:28.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:04:28 smithi153 ceph-mon[70044]: pgmap v2888: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:30.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:04:30 smithi083 ceph-mon[79930]: pgmap v2889: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:30.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:04:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:04:30.473Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:04:30.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:04:30 smithi083 ceph-mon[77242]: pgmap v2889: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:30.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:04:30 smithi153 ceph-mon[70044]: pgmap v2889: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:31.895 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:04:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:04:31] "GET /metrics HTTP/1.1" 200 225173 "" "Prometheus/2.33.4" 2022-09-29T07:04:32.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:04:31 smithi083 ceph-mon[79930]: pgmap v2890: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:32.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:04:31 smithi083 ceph-mon[77242]: pgmap v2890: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:32.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:04:31 smithi153 ceph-mon[70044]: pgmap v2890: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:04:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:04:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:04:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:04:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:04:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:04:34.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:04:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:04:33.878Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:04:34.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:04:34 smithi083 ceph-mon[77242]: pgmap v2891: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:34.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:04:34 smithi083 ceph-mon[79930]: pgmap v2891: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:34.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:04:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:04:34.577Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:04:34.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:04:34 smithi153 ceph-mon[70044]: pgmap v2891: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:36.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:04:36 smithi083 ceph-mon[77242]: pgmap v2892: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:36.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:04:36 smithi083 ceph-mon[79930]: pgmap v2892: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:36.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:04:36 smithi153 ceph-mon[70044]: pgmap v2892: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:38.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:04:38 smithi083 ceph-mon[79930]: pgmap v2893: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:38.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:04:38 smithi083 ceph-mon[77242]: pgmap v2893: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:38.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:04:38 smithi153 ceph-mon[70044]: pgmap v2893: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:39.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:04:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:04:39.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:04:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:04:39.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:04:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:04:39.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:04:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:04:39.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:04:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:04:39.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:04:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:04:40.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:04:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:04:40.472Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:04:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:04:40 smithi083 ceph-mon[77242]: pgmap v2894: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:04:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:04:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:04:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:04:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:04:40 smithi083 ceph-mon[79930]: pgmap v2894: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:04:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:04:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:04:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:04:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:04:40 smithi153 ceph-mon[70044]: pgmap v2894: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:04:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:04:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:04:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:04:41.897 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:04:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:04:41] "GET /metrics HTTP/1.1" 200 225175 "" "Prometheus/2.33.4" 2022-09-29T07:04:42.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:04:41 smithi083 ceph-mon[79930]: pgmap v2895: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:42.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:04:41 smithi083 ceph-mon[77242]: pgmap v2895: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:04:41 smithi153 ceph-mon[70044]: pgmap v2895: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:04:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:04:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:04:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:04:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:04:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:04:44.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:04:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:04:43.878Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:04:44.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:04:44 smithi083 ceph-mon[77242]: pgmap v2896: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:44.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:04:44 smithi083 ceph-mon[79930]: pgmap v2896: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:44.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:04:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:04:44.577Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:04:44.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:04:44 smithi153 ceph-mon[70044]: pgmap v2896: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:46.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:04:46 smithi083 ceph-mon[79930]: pgmap v2897: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:46.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:04:46 smithi083 ceph-mon[77242]: pgmap v2897: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:46.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:04:46 smithi153 ceph-mon[70044]: pgmap v2897: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:48.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:04:48 smithi083 ceph-mon[79930]: pgmap v2898: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:48.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:04:48 smithi083 ceph-mon[77242]: pgmap v2898: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:48.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:04:48 smithi153 ceph-mon[70044]: pgmap v2898: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:50.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:04:50 smithi083 ceph-mon[79930]: pgmap v2899: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:50.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:04:50 smithi083 ceph-mon[77242]: pgmap v2899: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:50.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:04:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:04:50.473Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:04:50.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:04:50 smithi153 ceph-mon[70044]: pgmap v2899: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:50.921 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:04:51.208 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:04:51.209 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (98m) 10m ago 99m 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:04:51.209 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (98m) 8m ago 98m 62.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:04:51.209 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (99m) 10m ago 99m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:04:51.209 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (99m) 8m ago 99m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:04:51.210 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (95m) 8m ago 102m 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:04:51.210 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (96m) 10m ago 103m 589M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:04:51.210 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (95m) 10m ago 104m 385M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:04:51.210 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (94m) 8m ago 102m 174M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:04:51.211 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (94m) 10m ago 102m 227M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:04:51.211 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (99m) 10m ago 99m 26.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:04:51.211 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (99m) 8m ago 99m 29.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:04:51.211 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (94m) 10m ago 102m 285M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:04:51.212 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (93m) 10m ago 101m 277M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:04:51.212 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (93m) 10m ago 101m 222M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:04:51.212 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (92m) 10m ago 101m 266M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:04:51.212 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (92m) 8m ago 101m 241M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:04:51.213 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (91m) 8m ago 100m 290M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:04:51.213 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (90m) 8m ago 100m 258M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:04:51.213 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (90m) 8m ago 100m 297M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:04:51.213 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (95m) 8m ago 99m 110M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:04:51.568 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:04:51.569 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:04:51.569 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:04:51.569 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:04:51.570 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:04:51.570 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:04:51.570 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:04:51.570 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:04:51.571 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:04:51.571 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:04:51.571 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:04:51.571 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:04:51.571 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:04:51.572 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:04:51.572 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:04:51.572 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:04:51.572 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:04:51.573 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:04:51.895 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:04:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:04:51] "GET /metrics HTTP/1.1" 200 225175 "" "Prometheus/2.33.4" 2022-09-29T07:04:52.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:04:51 smithi083 ceph-mon[77242]: from='client.47443 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:04:52.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:04:51 smithi083 ceph-mon[77242]: from='client.47449 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:04:52.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:04:51 smithi083 ceph-mon[77242]: pgmap v2900: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:52.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:04:51 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2831451' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:04:52.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:04:51 smithi083 ceph-mon[79930]: from='client.47443 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:04:52.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:04:51 smithi083 ceph-mon[79930]: from='client.47449 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:04:52.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:04:51 smithi083 ceph-mon[79930]: pgmap v2900: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:52.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:04:51 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2831451' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:04:52.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:04:51 smithi153 ceph-mon[70044]: from='client.47443 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:04:52.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:04:51 smithi153 ceph-mon[70044]: from='client.47449 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:04:52.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:04:51 smithi153 ceph-mon[70044]: pgmap v2900: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:52.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:04:51 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2831451' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:04:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:04:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:04:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:04:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:04:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:04:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:04:54.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:04:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:04:53.879Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:04:54.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:04:54 smithi083 ceph-mon[79930]: pgmap v2901: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:04:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:04:54.578Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:04:54.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:04:54 smithi083 ceph-mon[77242]: pgmap v2901: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:54.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:04:54 smithi153 ceph-mon[70044]: pgmap v2901: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:56.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:04:56 smithi083 ceph-mon[79930]: pgmap v2902: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:56.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:04:56 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:04:56.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:04:56 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:04:56.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:04:56 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:04:56.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:04:56 smithi083 ceph-mon[77242]: pgmap v2902: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:56.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:04:56 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:04:56.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:04:56 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:04:56.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:04:56 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:04:56.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:04:56 smithi153 ceph-mon[70044]: pgmap v2902: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:56.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:04:56 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:04:56.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:04:56 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:04:56.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:04:56 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:04:57.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:04:57 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:04:57.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:04:57 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:04:57.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:04:57 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:04:57.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:04:57 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:04:57.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:04:57 smithi083 ceph-mon[79930]: pgmap v2903: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:57.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:04:57 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:04:57.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:04:57 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:04:57.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:04:57 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:04:57.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:04:57 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:04:57.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:04:57 smithi083 ceph-mon[77242]: pgmap v2903: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:04:58.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:04:57 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:04:58.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:04:57 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:04:58.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:04:57 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:04:58.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:04:57 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:04:58.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:04:57 smithi153 ceph-mon[70044]: pgmap v2903: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:00.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:05:00 smithi083 ceph-mon[79930]: pgmap v2904: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:00.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:05:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:05:00.473Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:05:00.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:05:00 smithi083 ceph-mon[77242]: pgmap v2904: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:00.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:05:00 smithi153 ceph-mon[70044]: pgmap v2904: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:05:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:05:01] "GET /metrics HTTP/1.1" 200 225174 "" "Prometheus/2.33.4" 2022-09-29T07:05:02.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:05:02 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:05:02.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:05:02 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:05:02.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:05:02 smithi083 ceph-mon[77242]: pgmap v2905: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:02.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:05:02 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:05:02.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:05:02 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:05:02.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:05:02 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:05:02.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:05:02 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:05:02.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:05:02 smithi083 ceph-mon[79930]: pgmap v2905: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:02.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:05:02 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:05:02.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:05:02 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:05:02.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:05:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:05:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:05:02.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:05:02 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:05:02.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:05:02 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:05:02.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:05:02 smithi153 ceph-mon[70044]: pgmap v2905: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:02.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:05:02 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:05:02.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:05:02 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:05:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:05:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:05:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:05:04.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:05:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:05:03.880Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:05:04.580 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:05:04 smithi083 ceph-mon[77242]: pgmap v2906: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:04.581 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:05:04 smithi083 ceph-mon[79930]: pgmap v2906: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:04.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:05:04 smithi153 ceph-mon[70044]: pgmap v2906: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:04.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:05:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:05:04.578Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:05:05.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:05:05 smithi083 ceph-mon[79930]: Detected new or changed devices on smithi083 2022-09-29T07:05:05.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:05:05 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:05:05.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:05:05 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:05:05.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:05:05 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:05:05.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:05:05 smithi083 ceph-mon[77242]: Detected new or changed devices on smithi083 2022-09-29T07:05:05.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:05:05 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:05:05.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:05:05 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:05:05.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:05:05 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:05:06.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:05:05 smithi153 ceph-mon[70044]: Detected new or changed devices on smithi083 2022-09-29T07:05:06.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:05:05 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:05:06.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:05:05 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:05:06.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:05:05 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:05:06.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:05:06 smithi083 ceph-mon[79930]: pgmap v2907: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:06.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:05:06 smithi083 ceph-mon[77242]: pgmap v2907: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:07.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:05:06 smithi153 ceph-mon[70044]: pgmap v2907: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:08.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:05:08 smithi083 ceph-mon[79930]: pgmap v2908: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:08.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:05:08 smithi083 ceph-mon[77242]: pgmap v2908: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:09.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:05:08 smithi153 ceph-mon[70044]: pgmap v2908: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:10.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:05:10 smithi083 ceph-mon[79930]: pgmap v2909: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:10.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:05:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:05:10.474Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:05:10.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:05:10 smithi083 ceph-mon[77242]: pgmap v2909: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:11.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:05:10 smithi153 ceph-mon[70044]: pgmap v2909: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:11.899 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:05:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:05:11] "GET /metrics HTTP/1.1" 200 225178 "" "Prometheus/2.33.4" 2022-09-29T07:05:12.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:05:11 smithi083 ceph-mon[79930]: pgmap v2910: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:12.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:05:11 smithi083 ceph-mon[77242]: pgmap v2910: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:12.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:05:11 smithi153 ceph-mon[70044]: pgmap v2910: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:05:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:05:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:05:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:05:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:05:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:05:14.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:05:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:05:13.880Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:05:14.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:05:14 smithi083 ceph-mon[79930]: pgmap v2911: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:14.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:05:14 smithi083 ceph-mon[77242]: pgmap v2911: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:14.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:05:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:05:14.579Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:05:14.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:05:14 smithi153 ceph-mon[70044]: pgmap v2911: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:16.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:05:16 smithi083 ceph-mon[79930]: pgmap v2912: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:16.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:05:16 smithi083 ceph-mon[77242]: pgmap v2912: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:16.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:05:16 smithi153 ceph-mon[70044]: pgmap v2912: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:18.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:05:18 smithi083 ceph-mon[79930]: pgmap v2913: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:18.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:05:18 smithi083 ceph-mon[77242]: pgmap v2913: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:18.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:05:18 smithi153 ceph-mon[70044]: pgmap v2913: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:20.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:05:20 smithi083 ceph-mon[79930]: pgmap v2914: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:20.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:05:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:05:20.474Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:05:20.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:05:20 smithi083 ceph-mon[77242]: pgmap v2914: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:20.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:05:20 smithi153 ceph-mon[70044]: pgmap v2914: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:21.895 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:05:21.901 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:05:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:05:21] "GET /metrics HTTP/1.1" 200 225178 "" "Prometheus/2.33.4" 2022-09-29T07:05:21.902 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:05:21 smithi083 ceph-mon[77242]: pgmap v2915: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:22.186 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:05:22.186 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (99m) 21s ago 99m 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:05:22.186 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (99m) 8m ago 99m 62.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:05:22.187 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (100m) 21s ago 100m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:05:22.187 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (100m) 8m ago 100m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:05:22.187 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (96m) 8m ago 102m 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:05:22.187 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (96m) 21s ago 104m 598M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:05:22.188 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (95m) 21s ago 104m 421M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:05:22.188 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (95m) 8m ago 103m 174M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:05:22.188 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (95m) 21s ago 103m 248M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:05:22.188 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (99m) 21s ago 99m 25.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:05:22.188 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (99m) 8m ago 99m 29.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:05:22.189 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (94m) 21s ago 102m 310M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:05:22.189 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (94m) 21s ago 102m 302M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:05:22.189 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (93m) 21s ago 102m 241M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:05:22.189 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (93m) 21s ago 101m 290M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:05:22.189 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (92m) 8m ago 101m 241M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:05:22.190 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (92m) 8m ago 101m 290M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:05:22.190 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (91m) 8m ago 100m 258M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:05:22.190 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (90m) 8m ago 100m 297M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:05:22.190 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (96m) 8m ago 99m 110M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:05:22.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:05:21 smithi083 ceph-mon[79930]: pgmap v2915: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:22.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:05:21 smithi153 ceph-mon[70044]: pgmap v2915: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:22.541 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:05:22.542 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:05:22.542 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:05:22.542 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:05:22.543 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:05:22.543 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:05:22.543 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:05:22.543 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:05:22.543 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:05:22.544 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:05:22.544 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:05:22.544 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:05:22.544 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:05:22.545 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:05:22.545 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:05:22.545 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:05:22.545 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:05:22.545 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:05:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:05:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:05:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:05:23.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:05:22 smithi083 ceph-mon[77242]: from='client.37527 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:05:23.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:05:22 smithi083 ceph-mon[77242]: from='client.37533 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:05:23.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:05:22 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/650205447' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:05:23.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:05:22 smithi083 ceph-mon[79930]: from='client.37527 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:05:23.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:05:22 smithi083 ceph-mon[79930]: from='client.37533 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:05:23.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:05:22 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/650205447' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:05:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:05:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:05:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:05:23.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:05:22 smithi153 ceph-mon[70044]: from='client.37527 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:05:23.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:05:22 smithi153 ceph-mon[70044]: from='client.37533 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:05:23.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:05:22 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/650205447' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:05:24.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:05:23 smithi083 ceph-mon[79930]: pgmap v2916: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:24.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:05:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:05:23.881Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:05:24.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:05:23 smithi083 ceph-mon[77242]: pgmap v2916: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:24.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:05:23 smithi153 ceph-mon[70044]: pgmap v2916: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:24.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:05:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:05:24.580Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:05:26.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:05:26 smithi083 ceph-mon[79930]: pgmap v2917: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:26.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:05:26 smithi083 ceph-mon[77242]: pgmap v2917: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:26.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:05:26 smithi153 ceph-mon[70044]: pgmap v2917: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:28.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:05:28 smithi083 ceph-mon[79930]: pgmap v2918: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:28.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:05:28 smithi083 ceph-mon[77242]: pgmap v2918: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:28.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:05:28 smithi153 ceph-mon[70044]: pgmap v2918: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:30.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:05:30 smithi083 ceph-mon[79930]: pgmap v2919: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:30.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:05:30 smithi083 ceph-mon[77242]: pgmap v2919: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:30.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:05:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:05:30.475Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:05:30.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:05:30 smithi153 ceph-mon[70044]: pgmap v2919: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:31.910 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:05:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:05:31] "GET /metrics HTTP/1.1" 200 225171 "" "Prometheus/2.33.4" 2022-09-29T07:05:32.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:05:31 smithi083 ceph-mon[77242]: pgmap v2920: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:32.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:05:31 smithi083 ceph-mon[79930]: pgmap v2920: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:32.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:05:31 smithi153 ceph-mon[70044]: pgmap v2920: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:05:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:05:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:05:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:05:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:05:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:05:34.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:05:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:05:33.881Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:05:34.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:05:34 smithi083 ceph-mon[79930]: pgmap v2921: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:34.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:05:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:05:34.581Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:05:34.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:05:34 smithi083 ceph-mon[77242]: pgmap v2921: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:34.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:05:34 smithi153 ceph-mon[70044]: pgmap v2921: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:36.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:05:36 smithi083 ceph-mon[77242]: pgmap v2922: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:36.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:05:36 smithi083 ceph-mon[79930]: pgmap v2922: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:36.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:05:36 smithi153 ceph-mon[70044]: pgmap v2922: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:38.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:05:38 smithi083 ceph-mon[77242]: pgmap v2923: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:38.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:05:38 smithi083 ceph-mon[79930]: pgmap v2923: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:38.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:05:38 smithi153 ceph-mon[70044]: pgmap v2923: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:39.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:05:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:05:39.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:05:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:05:39.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:05:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:05:39.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:05:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:05:39.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:05:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:05:39.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:05:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:05:39.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:05:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:05:39.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:05:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:05:39.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:05:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:05:39.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:05:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:05:39.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:05:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:05:39.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:05:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:05:40.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:05:40 smithi083 ceph-mon[79930]: pgmap v2924: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:05:40 smithi083 ceph-mon[77242]: pgmap v2924: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:40.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:05:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:05:40.476Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:05:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:05:40 smithi153 ceph-mon[70044]: pgmap v2924: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:41.907 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:05:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:05:41] "GET /metrics HTTP/1.1" 200 225176 "" "Prometheus/2.33.4" 2022-09-29T07:05:42.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:05:41 smithi083 ceph-mon[79930]: pgmap v2925: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:42.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:05:41 smithi083 ceph-mon[77242]: pgmap v2925: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:05:41 smithi153 ceph-mon[70044]: pgmap v2925: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:05:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:05:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:05:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:05:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:05:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:05:44.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:05:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:05:43.882Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:05:44.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:05:44 smithi083 ceph-mon[79930]: pgmap v2926: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:05:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:05:44.581Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:05:44.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:05:44 smithi083 ceph-mon[77242]: pgmap v2926: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:44.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:05:44 smithi153 ceph-mon[70044]: pgmap v2926: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:46.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:05:46 smithi083 ceph-mon[79930]: pgmap v2927: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:46.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:05:46 smithi083 ceph-mon[77242]: pgmap v2927: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:46.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:05:46 smithi153 ceph-mon[70044]: pgmap v2927: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:48.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:05:48 smithi083 ceph-mon[79930]: pgmap v2928: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:48.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:05:48 smithi083 ceph-mon[77242]: pgmap v2928: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:48.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:05:48 smithi153 ceph-mon[70044]: pgmap v2928: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:50.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:05:50 smithi083 ceph-mon[79930]: pgmap v2929: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:50.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:05:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:05:50.477Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:05:50.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:05:50 smithi083 ceph-mon[77242]: pgmap v2929: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:50.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:05:50 smithi153 ceph-mon[70044]: pgmap v2929: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:51.911 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:05:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:05:51] "GET /metrics HTTP/1.1" 200 225176 "" "Prometheus/2.33.4" 2022-09-29T07:05:51.911 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:05:51 smithi083 ceph-mon[77242]: pgmap v2930: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:52.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:05:51 smithi083 ceph-mon[79930]: pgmap v2930: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:52.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:05:51 smithi153 ceph-mon[70044]: pgmap v2930: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:05:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:05:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:05:52.863 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:05:53.156 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:05:53.157 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (99m) 52s ago 100m 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:05:53.157 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (99m) 9m ago 99m 62.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:05:53.157 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (100m) 52s ago 100m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:05:53.157 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (100m) 9m ago 100m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:05:53.158 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (96m) 9m ago 103m 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:05:53.158 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (97m) 52s ago 105m 598M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:05:53.158 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (96m) 52s ago 105m 421M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:05:53.158 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (95m) 9m ago 103m 174M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:05:53.158 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (95m) 52s ago 103m 248M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:05:53.159 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (100m) 52s ago 100m 25.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:05:53.159 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (100m) 9m ago 100m 29.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:05:53.159 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (95m) 52s ago 103m 310M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:05:53.159 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (94m) 52s ago 102m 302M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:05:53.160 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (94m) 52s ago 102m 241M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:05:53.160 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (93m) 52s ago 102m 290M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:05:53.160 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (93m) 9m ago 102m 241M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:05:53.160 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (92m) 9m ago 101m 290M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:05:53.160 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (92m) 9m ago 101m 258M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:05:53.161 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (91m) 9m ago 101m 297M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:05:53.161 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (96m) 9m ago 100m 110M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:05:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:05:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:05:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:05:53.522 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:05:53.522 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:05:53.522 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:05:53.522 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:05:53.523 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:05:53.523 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:05:53.523 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:05:53.523 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:05:53.524 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:05:53.524 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:05:53.524 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:05:53.524 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:05:53.524 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:05:53.525 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:05:53.525 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:05:53.525 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:05:53.525 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:05:53.526 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:05:54.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:05:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:05:53.882Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:05:54.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:05:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:05:54.582Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:05:54.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:05:54 smithi083 ceph-mon[77242]: from='client.37545 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:05:54.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:05:54 smithi083 ceph-mon[77242]: from='client.47485 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:05:54.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:05:54 smithi083 ceph-mon[77242]: pgmap v2931: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:54.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:05:54 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/569720226' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:05:54.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:05:54 smithi083 ceph-mon[79930]: from='client.37545 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:05:54.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:05:54 smithi083 ceph-mon[79930]: from='client.47485 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:05:54.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:05:54 smithi083 ceph-mon[79930]: pgmap v2931: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:54.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:05:54 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/569720226' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:05:54.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:05:54 smithi153 ceph-mon[70044]: from='client.37545 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:05:54.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:05:54 smithi153 ceph-mon[70044]: from='client.47485 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:05:54.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:05:54 smithi153 ceph-mon[70044]: pgmap v2931: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:54.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:05:54 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/569720226' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:05:56.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:05:56 smithi083 ceph-mon[79930]: pgmap v2932: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:56.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:05:56 smithi083 ceph-mon[77242]: pgmap v2932: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:56.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:05:56 smithi153 ceph-mon[70044]: pgmap v2932: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:58.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:05:58 smithi083 ceph-mon[79930]: pgmap v2933: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:58.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:05:58 smithi083 ceph-mon[77242]: pgmap v2933: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:05:58.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:05:58 smithi153 ceph-mon[70044]: pgmap v2933: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:00.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:06:00 smithi083 ceph-mon[79930]: pgmap v2934: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:00.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:06:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:06:00.477Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:06:00.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:06:00 smithi083 ceph-mon[77242]: pgmap v2934: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:00.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:06:00 smithi153 ceph-mon[70044]: pgmap v2934: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:01.914 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:06:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:06:01] "GET /metrics HTTP/1.1" 200 225173 "" "Prometheus/2.33.4" 2022-09-29T07:06:02.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:06:01 smithi083 ceph-mon[77242]: pgmap v2935: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:02.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:06:01 smithi083 ceph-mon[79930]: pgmap v2935: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:02.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:06:01 smithi153 ceph-mon[70044]: pgmap v2935: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:06:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:06:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:06:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:06:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:06:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:06:04.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:06:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:06:03.883Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:06:04.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:06:04 smithi083 ceph-mon[79930]: pgmap v2936: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:04.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:06:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:06:04.582Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:06:04.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:06:04 smithi083 ceph-mon[77242]: pgmap v2936: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:04.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:06:04 smithi153 ceph-mon[70044]: pgmap v2936: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:05.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:06:05 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:06:05.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:06:05 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:06:05.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:06:05 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:06:05.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:06:05 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:06:05.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:06:05 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:06:05.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:06:05 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:06:05.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:06:05 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:06:05.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:06:05 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:06:05.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:06:05 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:06:05.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:06:05 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:06:05.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:06:05 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:06:05.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:06:05 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:06:06.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:06:06 smithi083 ceph-mon[79930]: pgmap v2937: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:06.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:06:06 smithi083 ceph-mon[77242]: pgmap v2937: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:06.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:06:06 smithi153 ceph-mon[70044]: pgmap v2937: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:08.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:06:08 smithi083 ceph-mon[79930]: pgmap v2938: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:08.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:06:08 smithi083 ceph-mon[77242]: pgmap v2938: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:08.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:06:08 smithi153 ceph-mon[70044]: pgmap v2938: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:10.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:06:10 smithi083 ceph-mon[77242]: pgmap v2939: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:10.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:06:10 smithi083 ceph-mon[79930]: pgmap v2939: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:10.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:06:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:06:10.477Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:06:10.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:06:10 smithi153 ceph-mon[70044]: pgmap v2939: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:11.914 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:06:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:06:11] "GET /metrics HTTP/1.1" 200 225175 "" "Prometheus/2.33.4" 2022-09-29T07:06:12.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:06:11 smithi083 ceph-mon[79930]: pgmap v2940: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:12.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:06:11 smithi083 ceph-mon[77242]: pgmap v2940: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:12.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:06:11 smithi153 ceph-mon[70044]: pgmap v2940: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:06:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:06:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:06:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:06:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:06:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:06:14.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:06:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:06:13.884Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:06:14.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:06:14 smithi083 ceph-mon[79930]: pgmap v2941: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:14.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:06:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:06:14.583Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:06:14.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:06:14 smithi083 ceph-mon[77242]: pgmap v2941: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:14.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:06:14 smithi153 ceph-mon[70044]: pgmap v2941: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:16.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:06:16 smithi083 ceph-mon[79930]: pgmap v2942: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:16.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:06:16 smithi083 ceph-mon[77242]: pgmap v2942: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:16.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:06:16 smithi153 ceph-mon[70044]: pgmap v2942: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:18.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:06:18 smithi083 ceph-mon[79930]: pgmap v2943: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:18.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:06:18 smithi083 ceph-mon[77242]: pgmap v2943: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:18.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:06:18 smithi153 ceph-mon[70044]: pgmap v2943: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:20.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:06:20 smithi083 ceph-mon[79930]: pgmap v2944: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:20.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:06:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:06:20.478Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:06:20.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:06:20 smithi083 ceph-mon[77242]: pgmap v2944: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:20.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:06:20 smithi153 ceph-mon[70044]: pgmap v2944: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:21.916 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:06:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:06:21] "GET /metrics HTTP/1.1" 200 225175 "" "Prometheus/2.33.4" 2022-09-29T07:06:22.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:06:21 smithi083 ceph-mon[77242]: pgmap v2945: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:22.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:06:21 smithi083 ceph-mon[79930]: pgmap v2945: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:22.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:06:21 smithi153 ceph-mon[70044]: pgmap v2945: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:06:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:06:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:06:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:06:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:06:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:06:23.853 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:06:24.159 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:06:24.160 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (100m) 83s ago 100m 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:06:24.160 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (100m) 9m ago 100m 62.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:06:24.160 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (101m) 83s ago 101m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:06:24.161 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (101m) 9m ago 101m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:06:24.161 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (97m) 9m ago 104m 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:06:24.161 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (97m) 83s ago 105m 598M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:06:24.161 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (96m) 83s ago 105m 421M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:06:24.162 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (96m) 9m ago 104m 174M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:06:24.162 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (96m) 83s ago 104m 248M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:06:24.162 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (100m) 83s ago 100m 25.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:06:24.162 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (100m) 9m ago 100m 29.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:06:24.163 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (95m) 83s ago 103m 310M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:06:24.163 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (95m) 83s ago 103m 302M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:06:24.163 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (94m) 83s ago 103m 241M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:06:24.163 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (94m) 83s ago 102m 290M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:06:24.164 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (93m) 9m ago 102m 241M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:06:24.164 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (93m) 9m ago 102m 290M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:06:24.164 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (92m) 9m ago 101m 258M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:06:24.164 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (91m) 9m ago 101m 297M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:06:24.165 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (97m) 9m ago 100m 110M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:06:24.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:06:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:06:23.884Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:06:24.517 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:06:24.518 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:06:24.518 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:06:24.518 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:06:24.518 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:06:24.519 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:06:24.519 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:06:24.519 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:06:24.519 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:06:24.520 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:06:24.520 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:06:24.520 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:06:24.520 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:06:24.521 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:06:24.521 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:06:24.521 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:06:24.521 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:06:24.521 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:06:24.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:06:24 smithi083 ceph-mon[77242]: pgmap v2946: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:24.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:06:24 smithi083 ceph-mon[79930]: pgmap v2946: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:06:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:06:24.583Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:06:24.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:06:24 smithi153 ceph-mon[70044]: pgmap v2946: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:25.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:06:25 smithi083 ceph-mon[79930]: from='client.37563 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:06:25.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:06:25 smithi083 ceph-mon[79930]: from='client.37569 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:06:25.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:06:25 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/884081662' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:06:25.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:06:25 smithi083 ceph-mon[77242]: from='client.37563 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:06:25.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:06:25 smithi083 ceph-mon[77242]: from='client.37569 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:06:25.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:06:25 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/884081662' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:06:25.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:06:25 smithi153 ceph-mon[70044]: from='client.37563 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:06:25.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:06:25 smithi153 ceph-mon[70044]: from='client.37569 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:06:25.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:06:25 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/884081662' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:06:26.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:06:26 smithi083 ceph-mon[79930]: pgmap v2947: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:26.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:06:26 smithi083 ceph-mon[77242]: pgmap v2947: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:26.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:06:26 smithi153 ceph-mon[70044]: pgmap v2947: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:28.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:06:28 smithi083 ceph-mon[79930]: pgmap v2948: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:28.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:06:28 smithi083 ceph-mon[77242]: pgmap v2948: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:28.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:06:28 smithi153 ceph-mon[70044]: pgmap v2948: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:30.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:06:30 smithi083 ceph-mon[79930]: pgmap v2949: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:30.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:06:30 smithi083 ceph-mon[77242]: pgmap v2949: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:30.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:06:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:06:30.479Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:06:30.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:06:30 smithi153 ceph-mon[70044]: pgmap v2949: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:31.920 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:06:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:06:31] "GET /metrics HTTP/1.1" 200 225170 "" "Prometheus/2.33.4" 2022-09-29T07:06:32.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:06:31 smithi083 ceph-mon[79930]: pgmap v2950: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:32.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:06:31 smithi083 ceph-mon[77242]: pgmap v2950: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:32.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:06:31 smithi153 ceph-mon[70044]: pgmap v2950: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:06:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:06:32.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:06:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:06:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:06:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:06:34.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:06:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:06:33.885Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:06:34.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:06:34 smithi083 ceph-mon[79930]: pgmap v2951: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:34.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:06:34 smithi083 ceph-mon[77242]: pgmap v2951: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:34.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:06:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:06:34.584Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:06:34.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:06:34 smithi153 ceph-mon[70044]: pgmap v2951: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:36.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:06:36 smithi083 ceph-mon[79930]: pgmap v2952: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:36.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:06:36 smithi083 ceph-mon[77242]: pgmap v2952: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:36.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:06:36 smithi153 ceph-mon[70044]: pgmap v2952: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:38.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:06:38 smithi083 ceph-mon[79930]: pgmap v2953: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:38.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:06:38 smithi083 ceph-mon[77242]: pgmap v2953: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:38.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:06:38 smithi153 ceph-mon[70044]: pgmap v2953: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:39.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:06:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:06:39.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:06:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:06:39.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:06:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:06:39.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:06:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:06:39.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:06:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:06:39.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:06:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:06:39.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:06:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:06:39.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:06:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:06:39.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:06:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:06:39.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:06:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:06:39.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:06:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:06:39.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:06:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:06:40.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:06:40 smithi083 ceph-mon[79930]: pgmap v2954: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:40.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:06:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:06:40.480Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:06:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:06:40 smithi083 ceph-mon[77242]: pgmap v2954: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:06:40 smithi153 ceph-mon[70044]: pgmap v2954: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:41.921 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:06:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:06:41] "GET /metrics HTTP/1.1" 200 225176 "" "Prometheus/2.33.4" 2022-09-29T07:06:42.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:06:41 smithi083 ceph-mon[77242]: pgmap v2955: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:42.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:06:41 smithi083 ceph-mon[79930]: pgmap v2955: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:06:41 smithi153 ceph-mon[70044]: pgmap v2955: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:06:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:06:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:06:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:06:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:06:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:06:44.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:06:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:06:43.886Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:06:44.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:06:44 smithi083 ceph-mon[79930]: pgmap v2956: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:06:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:06:44.584Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:06:44.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:06:44 smithi083 ceph-mon[77242]: pgmap v2956: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:44.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:06:44 smithi153 ceph-mon[70044]: pgmap v2956: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:46.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:06:46 smithi083 ceph-mon[79930]: pgmap v2957: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:46.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:06:46 smithi083 ceph-mon[77242]: pgmap v2957: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:46.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:06:46 smithi153 ceph-mon[70044]: pgmap v2957: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:48.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:06:48 smithi083 ceph-mon[77242]: pgmap v2958: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:48.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:06:48 smithi083 ceph-mon[79930]: pgmap v2958: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:48.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:06:48 smithi153 ceph-mon[70044]: pgmap v2958: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:50.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:06:50 smithi083 ceph-mon[79930]: pgmap v2959: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:50.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:06:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:06:50.480Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:06:50.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:06:50 smithi083 ceph-mon[77242]: pgmap v2959: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:50.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:06:50 smithi153 ceph-mon[70044]: pgmap v2959: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:51.923 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:06:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:06:51] "GET /metrics HTTP/1.1" 200 225176 "" "Prometheus/2.33.4" 2022-09-29T07:06:52.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:06:51 smithi083 ceph-mon[79930]: pgmap v2960: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:52.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:06:51 smithi083 ceph-mon[77242]: pgmap v2960: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:52.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:06:51 smithi153 ceph-mon[70044]: pgmap v2960: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:06:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:06:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:06:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:06:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:06:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:06:54.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:06:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:06:53.887Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:06:54.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:06:54 smithi083 ceph-mon[79930]: pgmap v2961: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:06:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:06:54.585Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:06:54.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:06:54 smithi083 ceph-mon[77242]: pgmap v2961: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:54.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:06:54 smithi153 ceph-mon[70044]: pgmap v2961: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:54.843 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:06:55.134 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:06:55.135 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (100m) 114s ago 101m 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:06:55.135 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (100m) 10m ago 100m 62.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:06:55.135 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (101m) 114s ago 101m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:06:55.136 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (101m) 10m ago 101m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:06:55.136 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (97m) 10m ago 104m 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:06:55.136 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (98m) 114s ago 106m 598M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:06:55.136 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (97m) 114s ago 106m 421M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:06:55.137 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (96m) 10m ago 104m 174M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:06:55.137 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (97m) 114s ago 104m 248M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:06:55.137 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (101m) 114s ago 101m 25.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:06:55.137 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (101m) 10m ago 101m 29.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:06:55.137 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (96m) 114s ago 104m 310M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:06:55.138 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (95m) 114s ago 103m 302M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:06:55.138 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (95m) 114s ago 103m 241M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:06:55.138 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (94m) 114s ago 103m 290M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:06:55.138 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (94m) 10m ago 103m 241M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:06:55.139 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (93m) 10m ago 102m 290M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:06:55.139 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (93m) 10m ago 102m 258M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:06:55.139 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (92m) 10m ago 102m 297M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:06:55.139 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (97m) 10m ago 101m 110M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:06:55.493 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:06:55.494 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:06:55.494 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:06:55.494 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:06:55.494 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:06:55.495 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:06:55.495 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:06:55.495 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:06:55.495 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:06:55.496 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:06:55.496 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:06:55.496 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:06:55.496 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:06:55.496 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:06:55.497 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:06:55.497 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:06:55.497 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:06:55.497 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:06:56.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:06:56 smithi083 ceph-mon[79930]: from='client.47515 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:06:56.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:06:56 smithi083 ceph-mon[79930]: from='client.37587 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:06:56.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:06:56 smithi083 ceph-mon[79930]: pgmap v2962: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:56.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:06:56 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2019645742' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:06:56.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:06:56 smithi083 ceph-mon[77242]: from='client.47515 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:06:56.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:06:56 smithi083 ceph-mon[77242]: from='client.37587 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:06:56.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:06:56 smithi083 ceph-mon[77242]: pgmap v2962: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:56.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:06:56 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2019645742' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:06:56.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:06:56 smithi153 ceph-mon[70044]: from='client.47515 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:06:56.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:06:56 smithi153 ceph-mon[70044]: from='client.37587 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:06:56.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:06:56 smithi153 ceph-mon[70044]: pgmap v2962: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:56.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:06:56 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2019645742' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:06:58.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:06:58 smithi083 ceph-mon[79930]: pgmap v2963: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:58.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:06:58 smithi083 ceph-mon[77242]: pgmap v2963: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:06:58.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:06:58 smithi153 ceph-mon[70044]: pgmap v2963: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:00.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:07:00 smithi083 ceph-mon[79930]: pgmap v2964: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:00.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:07:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:07:00.481Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:07:00.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:07:00 smithi083 ceph-mon[77242]: pgmap v2964: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:00.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:07:00 smithi153 ceph-mon[70044]: pgmap v2964: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:01.925 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:07:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:07:01] "GET /metrics HTTP/1.1" 200 225170 "" "Prometheus/2.33.4" 2022-09-29T07:07:02.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:07:01 smithi083 ceph-mon[79930]: pgmap v2965: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:02.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:07:01 smithi083 ceph-mon[77242]: pgmap v2965: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:02.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:07:01 smithi153 ceph-mon[70044]: pgmap v2965: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:07:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:07:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:07:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:07:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:07:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:07:04.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:07:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:07:03.887Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:07:04.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:07:04 smithi083 ceph-mon[79930]: pgmap v2966: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:04.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:07:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:07:04.586Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:07:04.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:07:04 smithi083 ceph-mon[77242]: pgmap v2966: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:04.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:07:04 smithi153 ceph-mon[70044]: pgmap v2966: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:05.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:07:05 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:07:05.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:07:05 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:07:05.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:07:05 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:07:05.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:07:05 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:07:05.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:07:05 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:07:05.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:07:05 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:07:05.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:07:05 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:07:05.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:07:05 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:07:05.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:07:05 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:07:06.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:07:06 smithi083 ceph-mon[79930]: pgmap v2967: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:06.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:07:06 smithi083 ceph-mon[77242]: pgmap v2967: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:06.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:07:06 smithi153 ceph-mon[70044]: pgmap v2967: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:08.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:07:08 smithi083 ceph-mon[79930]: pgmap v2968: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:08.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:07:08 smithi083 ceph-mon[77242]: pgmap v2968: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:08.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:07:08 smithi153 ceph-mon[70044]: pgmap v2968: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:10.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:07:10 smithi083 ceph-mon[79930]: pgmap v2969: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:10.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:07:10 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:07:10.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:07:10 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:07:10.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:07:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:07:10.481Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:07:10.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:07:10 smithi083 ceph-mon[77242]: pgmap v2969: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:10.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:07:10 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:07:10.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:07:10 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:07:10.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:07:10 smithi153 ceph-mon[70044]: pgmap v2969: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:10.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:07:10 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:07:10.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:07:10 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:07:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:07:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:07:11] "GET /metrics HTTP/1.1" 200 225173 "" "Prometheus/2.33.4" 2022-09-29T07:07:12.196 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:07:11 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:07:12.197 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:07:11 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:07:12.197 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:07:11 smithi153 ceph-mon[70044]: pgmap v2970: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:12.449 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:07:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:07:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:07:12.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:07:11 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:07:12.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:07:11 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:07:12.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:07:11 smithi083 ceph-mon[79930]: pgmap v2970: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:12.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:07:11 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:07:12.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:07:11 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:07:12.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:07:11 smithi083 ceph-mon[77242]: pgmap v2970: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:13.296 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:07:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:07:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:07:14.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:07:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:07:13.887Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:07:14.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:07:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:07:14.586Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:07:14.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:07:14 smithi083 ceph-mon[79930]: pgmap v2971: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:14.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:07:14 smithi083 ceph-mon[79930]: Detected new or changed devices on smithi153 2022-09-29T07:07:14.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:07:14 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:07:14.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:07:14 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:07:14.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:07:14 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:07:14.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:07:14 smithi083 ceph-mon[77242]: pgmap v2971: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:14.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:07:14 smithi083 ceph-mon[77242]: Detected new or changed devices on smithi153 2022-09-29T07:07:14.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:07:14 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:07:14.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:07:14 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:07:14.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:07:14 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:07:15.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:07:14 smithi153 ceph-mon[70044]: pgmap v2971: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:15.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:07:14 smithi153 ceph-mon[70044]: Detected new or changed devices on smithi153 2022-09-29T07:07:15.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:07:14 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:07:15.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:07:14 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:07:15.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:07:14 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:07:16.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:07:16 smithi083 ceph-mon[79930]: pgmap v2972: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:16.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:07:16 smithi083 ceph-mon[77242]: pgmap v2972: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:17.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:07:16 smithi153 ceph-mon[70044]: pgmap v2972: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:18.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:07:18 smithi083 ceph-mon[79930]: pgmap v2973: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:18.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:07:18 smithi083 ceph-mon[77242]: pgmap v2973: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:19.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:07:18 smithi153 ceph-mon[70044]: pgmap v2973: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:20.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:07:20 smithi083 ceph-mon[79930]: pgmap v2974: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:20.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:07:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:07:20.482Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:07:20.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:07:20 smithi083 ceph-mon[77242]: pgmap v2974: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:21.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:07:20 smithi153 ceph-mon[70044]: pgmap v2974: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:21.928 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:07:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:07:21] "GET /metrics HTTP/1.1" 200 225173 "" "Prometheus/2.33.4" 2022-09-29T07:07:22.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:07:21 smithi083 ceph-mon[79930]: pgmap v2975: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:22.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:07:21 smithi083 ceph-mon[77242]: pgmap v2975: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:22.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:07:21 smithi153 ceph-mon[70044]: pgmap v2975: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:07:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:07:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:07:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:07:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:07:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:07:24.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:07:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:07:23.888Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:07:24.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:07:24 smithi083 ceph-mon[79930]: pgmap v2976: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:07:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:07:24.587Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:07:24.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:07:24 smithi083 ceph-mon[77242]: pgmap v2976: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:24.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:07:24 smithi153 ceph-mon[70044]: pgmap v2976: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:25.822 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:07:26.113 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:07:26.114 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (101m) 2m ago 101m 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:07:26.114 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (101m) 16s ago 101m 62.7M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:07:26.114 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (102m) 2m ago 102m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:07:26.114 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (102m) 16s ago 102m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:07:26.115 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (98m) 16s ago 105m 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:07:26.115 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (98m) 2m ago 106m 598M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:07:26.115 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (97m) 2m ago 106m 421M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:07:26.115 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (97m) 16s ago 105m 192M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:07:26.116 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (97m) 2m ago 105m 248M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:07:26.116 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (101m) 2m ago 101m 25.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:07:26.116 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (101m) 16s ago 101m 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:07:26.116 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (96m) 2m ago 104m 310M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:07:26.117 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (96m) 2m ago 104m 302M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:07:26.117 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (95m) 2m ago 104m 241M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:07:26.117 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (95m) 2m ago 103m 290M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:07:26.117 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (94m) 16s ago 103m 262M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:07:26.117 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (94m) 16s ago 103m 317M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:07:26.118 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (93m) 16s ago 103m 282M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:07:26.118 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (93m) 16s ago 102m 327M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:07:26.118 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (98m) 16s ago 101m 114M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:07:26.471 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:07:26.471 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:07:26.471 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:07:26.471 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:07:26.472 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:07:26.472 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:07:26.472 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:07:26.472 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:07:26.473 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:07:26.473 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:07:26.473 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:07:26.473 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:07:26.473 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:07:26.474 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:07:26.474 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:07:26.474 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:07:26.474 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:07:26.475 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:07:26.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:07:26 smithi083 ceph-mon[79930]: pgmap v2977: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:26.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:07:26 smithi083 ceph-mon[77242]: pgmap v2977: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:26.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:07:26 smithi153 ceph-mon[70044]: pgmap v2977: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:27.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:07:27 smithi083 ceph-mon[79930]: from='client.37599 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:07:27.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:07:27 smithi083 ceph-mon[79930]: from='client.37605 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:07:27.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:07:27 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/507816849' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:07:27.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:07:27 smithi083 ceph-mon[77242]: from='client.37599 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:07:27.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:07:27 smithi083 ceph-mon[77242]: from='client.37605 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:07:27.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:07:27 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/507816849' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:07:27.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:07:27 smithi153 ceph-mon[70044]: from='client.37599 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:07:27.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:07:27 smithi153 ceph-mon[70044]: from='client.37605 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:07:27.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:07:27 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/507816849' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:07:28.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:07:28 smithi083 ceph-mon[79930]: pgmap v2978: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:28.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:07:28 smithi083 ceph-mon[77242]: pgmap v2978: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:28.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:07:28 smithi153 ceph-mon[70044]: pgmap v2978: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:30.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:07:30 smithi083 ceph-mon[77242]: pgmap v2979: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:30.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:07:30 smithi083 ceph-mon[79930]: pgmap v2979: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:30.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:07:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:07:30.482Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:07:30.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:07:30 smithi153 ceph-mon[70044]: pgmap v2979: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:31.931 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:07:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:07:31] "GET /metrics HTTP/1.1" 200 225176 "" "Prometheus/2.33.4" 2022-09-29T07:07:32.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:07:31 smithi083 ceph-mon[79930]: pgmap v2980: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:32.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:07:31 smithi083 ceph-mon[77242]: pgmap v2980: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:32.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:07:31 smithi153 ceph-mon[70044]: pgmap v2980: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:07:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:07:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:07:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:07:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:07:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:07:34.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:07:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:07:33.889Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:07:34.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:07:34 smithi083 ceph-mon[79930]: pgmap v2981: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:34.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:07:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:07:34.588Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:07:34.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:07:34 smithi083 ceph-mon[77242]: pgmap v2981: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:34.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:07:34 smithi153 ceph-mon[70044]: pgmap v2981: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:36.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:07:36 smithi083 ceph-mon[79930]: pgmap v2982: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:36.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:07:36 smithi083 ceph-mon[77242]: pgmap v2982: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:36.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:07:36 smithi153 ceph-mon[70044]: pgmap v2982: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:38.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:07:38 smithi083 ceph-mon[79930]: pgmap v2983: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:38.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:07:38 smithi083 ceph-mon[77242]: pgmap v2983: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:38.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:07:38 smithi153 ceph-mon[70044]: pgmap v2983: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:39.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:07:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:07:39.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:07:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:07:39.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:07:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:07:39.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:07:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:07:39.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:07:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:07:39.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:07:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:07:39.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:07:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:07:39.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:07:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:07:39.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:07:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:07:39.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:07:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:07:39.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:07:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:07:39.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:07:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:07:40.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:07:40 smithi083 ceph-mon[79930]: pgmap v2984: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:40.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:07:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:07:40.483Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:07:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:07:40 smithi083 ceph-mon[77242]: pgmap v2984: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:07:40 smithi153 ceph-mon[70044]: pgmap v2984: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:41.934 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:07:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:07:41] "GET /metrics HTTP/1.1" 200 225177 "" "Prometheus/2.33.4" 2022-09-29T07:07:42.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:07:41 smithi083 ceph-mon[77242]: pgmap v2985: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:42.235 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:07:41 smithi083 ceph-mon[79930]: pgmap v2985: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:07:41 smithi153 ceph-mon[70044]: pgmap v2985: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:07:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:07:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:07:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:07:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:07:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:07:44.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:07:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:07:43.890Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:07:44.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:07:44 smithi083 ceph-mon[77242]: pgmap v2986: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:44.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:07:44 smithi083 ceph-mon[79930]: pgmap v2986: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:07:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:07:44.589Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:07:44.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:07:44 smithi153 ceph-mon[70044]: pgmap v2986: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:46.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:07:46 smithi083 ceph-mon[79930]: pgmap v2987: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:46.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:07:46 smithi083 ceph-mon[77242]: pgmap v2987: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:46.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:07:46 smithi153 ceph-mon[70044]: pgmap v2987: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:48.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:07:48 smithi083 ceph-mon[79930]: pgmap v2988: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:48.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:07:48 smithi083 ceph-mon[77242]: pgmap v2988: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:48.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:07:48 smithi153 ceph-mon[70044]: pgmap v2988: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:50.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:07:50 smithi083 ceph-mon[79930]: pgmap v2989: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:50.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:07:50 smithi083 ceph-mon[77242]: pgmap v2989: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:50.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:07:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:07:50.484Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:07:50.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:07:50 smithi153 ceph-mon[70044]: pgmap v2989: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:51.935 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:07:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:07:51] "GET /metrics HTTP/1.1" 200 225177 "" "Prometheus/2.33.4" 2022-09-29T07:07:52.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:07:51 smithi083 ceph-mon[79930]: pgmap v2990: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:52.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:07:51 smithi083 ceph-mon[77242]: pgmap v2990: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:52.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:07:51 smithi153 ceph-mon[70044]: pgmap v2990: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:07:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:07:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:07:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:07:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:07:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:07:54.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:07:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:07:53.890Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:07:54.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:07:54 smithi083 ceph-mon[79930]: pgmap v2991: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:54.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:07:54 smithi083 ceph-mon[77242]: pgmap v2991: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:07:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:07:54.590Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:07:54.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:07:54 smithi153 ceph-mon[70044]: pgmap v2991: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:56.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:07:56 smithi083 ceph-mon[79930]: pgmap v2992: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:56.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:07:56 smithi083 ceph-mon[77242]: pgmap v2992: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:56.793 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:07:56.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:07:56 smithi153 ceph-mon[70044]: pgmap v2992: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:57.083 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:07:57.083 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (101m) 2m ago 102m 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:07:57.084 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (101m) 47s ago 101m 62.7M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:07:57.084 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (103m) 2m ago 103m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:07:57.084 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (102m) 47s ago 102m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:07:57.084 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (98m) 47s ago 105m 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:07:57.085 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (99m) 2m ago 107m 598M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:07:57.085 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (98m) 2m ago 107m 421M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:07:57.085 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (97m) 47s ago 105m 192M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:07:57.085 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (98m) 2m ago 105m 248M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:07:57.086 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (102m) 2m ago 102m 25.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:07:57.086 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (102m) 47s ago 102m 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:07:57.086 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (97m) 2m ago 105m 310M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:07:57.086 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (96m) 2m ago 104m 302M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:07:57.087 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (96m) 2m ago 104m 241M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:07:57.087 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (95m) 2m ago 104m 290M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:07:57.087 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (95m) 47s ago 104m 262M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:07:57.087 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (94m) 47s ago 103m 317M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:07:57.087 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (94m) 47s ago 103m 282M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:07:57.088 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (93m) 47s ago 103m 327M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:07:57.088 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (98m) 47s ago 102m 114M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:07:57.436 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:07:57.437 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:07:57.437 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:07:57.437 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:07:57.437 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:07:57.437 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:07:57.438 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:07:57.438 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:07:57.438 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:07:57.438 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:07:57.439 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:07:57.439 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:07:57.439 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:07:57.439 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:07:57.439 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:07:57.440 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:07:57.440 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:07:57.440 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:07:57.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:07:57 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1825170463' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:07:57.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:07:57 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1825170463' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:07:57.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:07:57 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1825170463' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:07:58.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:07:58 smithi083 ceph-mon[79930]: from='client.37617 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:07:58.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:07:58 smithi083 ceph-mon[79930]: from='client.37623 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:07:58.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:07:58 smithi083 ceph-mon[79930]: pgmap v2993: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:58.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:07:58 smithi083 ceph-mon[77242]: from='client.37617 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:07:58.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:07:58 smithi083 ceph-mon[77242]: from='client.37623 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:07:58.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:07:58 smithi083 ceph-mon[77242]: pgmap v2993: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:07:58.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:07:58 smithi153 ceph-mon[70044]: from='client.37617 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:07:58.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:07:58 smithi153 ceph-mon[70044]: from='client.37623 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:07:58.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:07:58 smithi153 ceph-mon[70044]: pgmap v2993: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:00.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:08:00 smithi153 ceph-mon[70044]: pgmap v2994: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:00.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:08:00 smithi083 ceph-mon[79930]: pgmap v2994: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:00.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:08:00 smithi083 ceph-mon[77242]: pgmap v2994: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:00.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:08:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:08:00.484Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:08:01.937 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:08:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:08:01] "GET /metrics HTTP/1.1" 200 225172 "" "Prometheus/2.33.4" 2022-09-29T07:08:02.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:08:01 smithi083 ceph-mon[79930]: pgmap v2995: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:02.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:08:01 smithi083 ceph-mon[77242]: pgmap v2995: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:02.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:08:01 smithi153 ceph-mon[70044]: pgmap v2995: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:08:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:08:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:08:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:08:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:08:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:08:04.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:08:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:08:03.891Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:08:04.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:08:04 smithi083 ceph-mon[79930]: pgmap v2996: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:04.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:08:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:08:04.591Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:08:04.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:08:04 smithi083 ceph-mon[77242]: pgmap v2996: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:04.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:08:04 smithi153 ceph-mon[70044]: pgmap v2996: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:06.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:08:06 smithi083 ceph-mon[77242]: pgmap v2997: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:06.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:08:06 smithi083 ceph-mon[79930]: pgmap v2997: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:06.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:08:06 smithi153 ceph-mon[70044]: pgmap v2997: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:08.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:08:08 smithi083 ceph-mon[77242]: pgmap v2998: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:08.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:08:08 smithi083 ceph-mon[79930]: pgmap v2998: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:08.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:08:08 smithi153 ceph-mon[70044]: pgmap v2998: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:10.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:08:10 smithi153 ceph-mon[70044]: pgmap v2999: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:10.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:08:10 smithi083 ceph-mon[77242]: pgmap v2999: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:10.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:08:10 smithi083 ceph-mon[79930]: pgmap v2999: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:10.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:08:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:08:10.484Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:08:11.939 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:08:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:08:11] "GET /metrics HTTP/1.1" 200 225175 "" "Prometheus/2.33.4" 2022-09-29T07:08:12.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:08:11 smithi083 ceph-mon[79930]: pgmap v3000: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:12.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:08:11 smithi083 ceph-mon[77242]: pgmap v3000: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:12.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:08:11 smithi153 ceph-mon[70044]: pgmap v3000: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:08:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:08:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:08:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:08:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:08:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:08:14.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:08:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:08:13.891Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:08:14.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:08:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:08:14.591Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:08:14.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:08:14 smithi083 ceph-mon[79930]: pgmap v3001: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:14.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:08:14 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:08:14.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:08:14 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:08:14.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:08:14 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:08:14.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:08:14 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:08:14.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:08:14 smithi083 ceph-mon[77242]: pgmap v3001: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:14.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:08:14 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:08:14.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:08:14 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:08:14.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:08:14 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:08:14.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:08:14 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:08:14.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:08:14 smithi153 ceph-mon[70044]: pgmap v3001: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:14.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:08:14 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:08:14.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:08:14 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:08:14.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:08:14 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:08:14.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:08:14 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:08:16.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:08:16 smithi083 ceph-mon[79930]: pgmap v3002: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:16.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:08:16 smithi083 ceph-mon[77242]: pgmap v3002: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:16.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:08:16 smithi153 ceph-mon[70044]: pgmap v3002: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:18.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:08:18 smithi153 ceph-mon[70044]: pgmap v3003: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:18.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:08:18 smithi083 ceph-mon[79930]: pgmap v3003: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:18.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:08:18 smithi083 ceph-mon[77242]: pgmap v3003: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:20.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:08:20 smithi153 ceph-mon[70044]: pgmap v3004: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:20.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:08:20 smithi083 ceph-mon[79930]: pgmap v3004: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:20.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:08:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:08:20.485Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:08:20.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:08:20 smithi083 ceph-mon[77242]: pgmap v3004: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:21.945 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:08:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:08:21] "GET /metrics HTTP/1.1" 200 225175 "" "Prometheus/2.33.4" 2022-09-29T07:08:22.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:08:21 smithi083 ceph-mon[79930]: pgmap v3005: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:22.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:08:21 smithi083 ceph-mon[77242]: pgmap v3005: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:22.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:08:21 smithi153 ceph-mon[70044]: pgmap v3005: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:08:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:08:22.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:08:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:08:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:08:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:08:24.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:08:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:08:23.892Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:08:24.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:08:24 smithi083 ceph-mon[79930]: pgmap v3006: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:08:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:08:24.592Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:08:24.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:08:24 smithi083 ceph-mon[77242]: pgmap v3006: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:24.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:08:24 smithi153 ceph-mon[70044]: pgmap v3006: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:26.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:08:26 smithi153 ceph-mon[70044]: pgmap v3007: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:26.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:08:26 smithi083 ceph-mon[79930]: pgmap v3007: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:26.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:08:26 smithi083 ceph-mon[77242]: pgmap v3007: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:27.762 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:08:28.050 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:08:28.051 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (102m) 3m ago 102m 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:08:28.051 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (102m) 78s ago 102m 62.7M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:08:28.051 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (103m) 3m ago 103m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:08:28.051 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (103m) 78s ago 103m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:08:28.052 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (99m) 78s ago 106m 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:08:28.052 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (99m) 3m ago 107m 598M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:08:28.052 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (98m) 3m ago 107m 421M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:08:28.052 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (98m) 78s ago 106m 192M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:08:28.053 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (98m) 3m ago 106m 248M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:08:28.053 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (102m) 3m ago 102m 25.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:08:28.053 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (102m) 78s ago 102m 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:08:28.053 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (97m) 3m ago 105m 310M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:08:28.053 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (97m) 3m ago 105m 302M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:08:28.054 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (96m) 3m ago 105m 241M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:08:28.054 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (96m) 3m ago 104m 290M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:08:28.054 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (95m) 78s ago 104m 262M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:08:28.054 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (95m) 78s ago 104m 317M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:08:28.055 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (94m) 78s ago 104m 282M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:08:28.055 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (94m) 78s ago 103m 327M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:08:28.055 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (99m) 78s ago 102m 114M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:08:28.407 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:08:28.407 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:08:28.408 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:08:28.408 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:08:28.408 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:08:28.408 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:08:28.409 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:08:28.409 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:08:28.409 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:08:28.409 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:08:28.409 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:08:28.410 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:08:28.410 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:08:28.410 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:08:28.410 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:08:28.411 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:08:28.411 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:08:28.411 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:08:28.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:08:28 smithi153 ceph-mon[70044]: pgmap v3008: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:28.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:08:28 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/243085738' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:08:28.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:08:28 smithi083 ceph-mon[79930]: pgmap v3008: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:28.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:08:28 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/243085738' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:08:28.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:08:28 smithi083 ceph-mon[77242]: pgmap v3008: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:28.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:08:28 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/243085738' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:08:29.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:08:29 smithi153 ceph-mon[70044]: from='client.37635 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:08:29.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:08:29 smithi153 ceph-mon[70044]: from='client.47575 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:08:29.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:08:29 smithi083 ceph-mon[79930]: from='client.37635 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:08:29.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:08:29 smithi083 ceph-mon[79930]: from='client.47575 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:08:29.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:08:29 smithi083 ceph-mon[77242]: from='client.37635 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:08:29.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:08:29 smithi083 ceph-mon[77242]: from='client.47575 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:08:30.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:08:30 smithi153 ceph-mon[70044]: pgmap v3009: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:30.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:08:30 smithi083 ceph-mon[79930]: pgmap v3009: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:30.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:08:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:08:30.486Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:08:30.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:08:30 smithi083 ceph-mon[77242]: pgmap v3009: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:31.946 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:08:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:08:31] "GET /metrics HTTP/1.1" 200 225171 "" "Prometheus/2.33.4" 2022-09-29T07:08:32.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:08:31 smithi083 ceph-mon[77242]: pgmap v3010: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:32.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:08:31 smithi083 ceph-mon[79930]: pgmap v3010: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:32.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:08:31 smithi153 ceph-mon[70044]: pgmap v3010: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:08:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:08:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:08:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:08:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:08:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:08:34.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:08:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:08:33.893Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:08:34.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:08:34 smithi153 ceph-mon[70044]: pgmap v3011: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:34.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:08:34 smithi083 ceph-mon[77242]: pgmap v3011: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:34.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:08:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:08:34.593Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:08:34.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:08:34 smithi083 ceph-mon[79930]: pgmap v3011: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:36.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:08:36 smithi153 ceph-mon[70044]: pgmap v3012: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:36.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:08:36 smithi083 ceph-mon[77242]: pgmap v3012: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:36.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:08:36 smithi083 ceph-mon[79930]: pgmap v3012: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:38.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:08:38 smithi153 ceph-mon[70044]: pgmap v3013: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:38.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:08:38 smithi083 ceph-mon[77242]: pgmap v3013: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:38.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:08:38 smithi083 ceph-mon[79930]: pgmap v3013: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:39.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:08:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:08:39.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:08:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:08:39.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:08:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:08:39.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:08:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:08:39.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:08:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:08:39.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:08:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:08:39.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:08:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:08:39.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:08:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:08:39.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:08:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:08:39.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:08:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:08:39.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:08:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:08:39.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:08:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:08:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:08:40 smithi153 ceph-mon[70044]: pgmap v3014: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:40.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:08:40 smithi083 ceph-mon[79930]: pgmap v3014: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:40.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:08:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:08:40.487Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:08:40.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:08:40 smithi083 ceph-mon[77242]: pgmap v3014: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:41.948 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:08:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:08:41] "GET /metrics HTTP/1.1" 200 225170 "" "Prometheus/2.33.4" 2022-09-29T07:08:42.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:08:41 smithi083 ceph-mon[79930]: pgmap v3015: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:42.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:08:41 smithi083 ceph-mon[77242]: pgmap v3015: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:08:41 smithi153 ceph-mon[70044]: pgmap v3015: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:08:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:08:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:08:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:08:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:08:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:08:44.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:08:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:08:43.894Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:08:44.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:08:44 smithi153 ceph-mon[70044]: pgmap v3016: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:44.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:08:44 smithi083 ceph-mon[79930]: pgmap v3016: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:44.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:08:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:08:44.593Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:08:44.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:08:44 smithi083 ceph-mon[77242]: pgmap v3016: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:46.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:08:46 smithi153 ceph-mon[70044]: pgmap v3017: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:46.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:08:46 smithi083 ceph-mon[79930]: pgmap v3017: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:46.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:08:46 smithi083 ceph-mon[77242]: pgmap v3017: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:48.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:08:48 smithi153 ceph-mon[70044]: pgmap v3018: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:48.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:08:48 smithi083 ceph-mon[79930]: pgmap v3018: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:48.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:08:48 smithi083 ceph-mon[77242]: pgmap v3018: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:50.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:08:50 smithi153 ceph-mon[70044]: pgmap v3019: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:50.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:08:50 smithi083 ceph-mon[79930]: pgmap v3019: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:50.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:08:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:08:50.487Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:08:50.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:08:50 smithi083 ceph-mon[77242]: pgmap v3019: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:51.948 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:08:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:08:51] "GET /metrics HTTP/1.1" 200 225170 "" "Prometheus/2.33.4" 2022-09-29T07:08:52.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:08:51 smithi083 ceph-mon[77242]: pgmap v3020: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:52.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:08:51 smithi083 ceph-mon[79930]: pgmap v3020: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:52.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:08:51 smithi153 ceph-mon[70044]: pgmap v3020: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:08:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:08:52.320Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:08:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:08:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:08:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:08:54.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:08:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:08:53.895Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:08:54.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:08:54 smithi153 ceph-mon[70044]: pgmap v3021: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:54.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:08:54 smithi083 ceph-mon[79930]: pgmap v3021: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:54.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:08:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:08:54.594Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:08:54.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:08:54 smithi083 ceph-mon[77242]: pgmap v3021: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:56.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:08:56 smithi153 ceph-mon[70044]: pgmap v3022: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:56.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:08:56 smithi083 ceph-mon[79930]: pgmap v3022: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:56.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:08:56 smithi083 ceph-mon[77242]: pgmap v3022: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:58.735 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:08:58.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:08:58 smithi153 ceph-mon[70044]: pgmap v3023: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:58.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:08:58 smithi083 ceph-mon[79930]: pgmap v3023: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:58.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:08:58 smithi083 ceph-mon[77242]: pgmap v3023: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:08:59.026 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:08:59.026 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (102m) 3m ago 103m 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:08:59.027 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (102m) 109s ago 102m 62.7M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:08:59.027 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (104m) 3m ago 104m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:08:59.027 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (104m) 109s ago 103m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:08:59.027 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (99m) 109s ago 106m 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:08:59.028 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (100m) 3m ago 108m 598M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:08:59.028 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (99m) 3m ago 108m 421M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:08:59.028 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (98m) 109s ago 106m 192M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:08:59.028 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (99m) 3m ago 106m 248M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:08:59.028 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (103m) 3m ago 103m 25.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:08:59.029 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (103m) 109s ago 103m 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:08:59.029 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (98m) 3m ago 106m 310M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:08:59.029 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (97m) 3m ago 106m 302M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:08:59.029 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (97m) 3m ago 105m 241M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:08:59.030 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (96m) 3m ago 105m 290M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:08:59.030 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (96m) 109s ago 105m 262M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:08:59.030 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (95m) 109s ago 104m 317M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:08:59.030 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (95m) 109s ago 104m 282M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:08:59.031 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (94m) 109s ago 104m 327M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:08:59.031 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (99m) 109s ago 103m 114M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:08:59.382 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:08:59.382 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:08:59.382 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:08:59.383 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:08:59.383 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:08:59.383 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:08:59.383 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:08:59.384 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:08:59.384 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:08:59.384 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:08:59.384 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:08:59.385 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:08:59.385 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:08:59.385 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:08:59.385 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:08:59.386 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:08:59.386 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:08:59.386 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:08:59.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:08:59 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3642435108' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:08:59.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:08:59 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3642435108' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:08:59.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:08:59 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3642435108' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:09:00.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:09:00 smithi153 ceph-mon[70044]: from='client.47587 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:09:00.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:09:00 smithi153 ceph-mon[70044]: from='client.37659 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:09:00.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:09:00 smithi153 ceph-mon[70044]: pgmap v3024: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:09:00.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:09:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:09:00.488Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:09:00.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:09:00 smithi083 ceph-mon[77242]: from='client.47587 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:09:00.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:09:00 smithi083 ceph-mon[77242]: from='client.37659 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:09:00.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:09:00 smithi083 ceph-mon[77242]: pgmap v3024: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:09:00.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:09:00 smithi083 ceph-mon[79930]: from='client.47587 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:09:00.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:09:00 smithi083 ceph-mon[79930]: from='client.37659 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:09:00.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:09:00 smithi083 ceph-mon[79930]: pgmap v3024: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:09:01.955 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:09:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:09:01] "GET /metrics HTTP/1.1" 200 225170 "" "Prometheus/2.33.4" 2022-09-29T07:09:02.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:09:01 smithi083 ceph-mon[77242]: pgmap v3025: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:09:02.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:09:01 smithi083 ceph-mon[79930]: pgmap v3025: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:09:02.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:09:01 smithi153 ceph-mon[70044]: pgmap v3025: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:09:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:09:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:09:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:09:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:09:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:09:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:09:04.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:09:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:09:03.895Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:09:04.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:09:04 smithi153 ceph-mon[70044]: pgmap v3026: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:09:04.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:09:04 smithi083 ceph-mon[79930]: pgmap v3026: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:09:04.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:09:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:09:04.594Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:09:04.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:09:04 smithi083 ceph-mon[77242]: pgmap v3026: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:09:06.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:09:06 smithi153 ceph-mon[70044]: pgmap v3027: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:09:06.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:09:06 smithi083 ceph-mon[79930]: pgmap v3027: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:09:06.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:09:06 smithi083 ceph-mon[77242]: pgmap v3027: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:09:08.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:09:08 smithi153 ceph-mon[70044]: pgmap v3028: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:09:08.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:09:08 smithi083 ceph-mon[79930]: pgmap v3028: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:09:08.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:09:08 smithi083 ceph-mon[77242]: pgmap v3028: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:09:10.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:09:10 smithi153 ceph-mon[70044]: pgmap v3029: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:09:10.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:09:10 smithi083 ceph-mon[79930]: pgmap v3029: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:09:10.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:09:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:09:10.488Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:09:10.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:09:10 smithi083 ceph-mon[77242]: pgmap v3029: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:09:11.951 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:09:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:09:11] "GET /metrics HTTP/1.1" 200 225175 "" "Prometheus/2.33.4" 2022-09-29T07:09:12.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:09:11 smithi083 ceph-mon[77242]: pgmap v3030: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:09:12.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:09:11 smithi083 ceph-mon[79930]: pgmap v3030: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:09:12.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:09:11 smithi153 ceph-mon[70044]: pgmap v3030: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:09:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:09:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:09:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:09:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:09:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:09:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:09:14.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:09:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:09:13.896Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:09:14.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:09:14 smithi153 ceph-mon[70044]: pgmap v3031: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:09:14.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:09:14 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:09:14.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:09:14 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:09:14.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:09:14 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:09:14.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:09:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:09:14.594Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:09:14.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:09:14 smithi083 ceph-mon[77242]: pgmap v3031: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:09:14.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:09:14 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:09:14.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:09:14 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:09:14.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:09:14 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:09:14.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:09:14 smithi083 ceph-mon[79930]: pgmap v3031: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:09:14.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:09:14 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:09:14.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:09:14 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:09:14.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:09:14 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:09:16.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:09:15 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:09:16.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:09:15 smithi153 ceph-mon[70044]: pgmap v3032: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:09:16.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:09:15 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:09:16.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:09:15 smithi083 ceph-mon[79930]: pgmap v3032: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:09:16.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:09:15 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:09:16.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:09:15 smithi083 ceph-mon[77242]: pgmap v3032: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:09:18.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:09:18 smithi153 ceph-mon[70044]: pgmap v3033: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:09:18.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:09:18 smithi083 ceph-mon[77242]: pgmap v3033: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:09:18.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:09:18 smithi083 ceph-mon[79930]: pgmap v3033: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:09:20.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:09:20 smithi153 ceph-mon[70044]: pgmap v3034: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:09:20.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:09:20 smithi083 ceph-mon[77242]: pgmap v3034: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:09:20.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:09:20 smithi083 ceph-mon[79930]: pgmap v3034: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:09:20.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:09:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:09:20.489Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:09:21.955 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:09:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:09:21] "GET /metrics HTTP/1.1" 200 225175 "" "Prometheus/2.33.4" 2022-09-29T07:09:21.956 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:09:21 smithi083 ceph-mon[77242]: pgmap v3035: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:09:22.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:09:21 smithi083 ceph-mon[79930]: pgmap v3035: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:09:22.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:09:21 smithi153 ceph-mon[70044]: pgmap v3035: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:09:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:09:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:09:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:09:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:09:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:09:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:09:24.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:09:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:09:23.896Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:09:24.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:09:24 smithi153 ceph-mon[70044]: pgmap v3036: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:09:24.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:09:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:09:24.596Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:09:24.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:09:24 smithi083 ceph-mon[77242]: pgmap v3036: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:09:24.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:09:24 smithi083 ceph-mon[79930]: pgmap v3036: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:09:26.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:09:26 smithi153 ceph-mon[70044]: pgmap v3037: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:09:26.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:09:26 smithi083 ceph-mon[77242]: pgmap v3037: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:09:26.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:09:26 smithi083 ceph-mon[79930]: pgmap v3037: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:09:28.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:09:28 smithi153 ceph-mon[70044]: pgmap v3038: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:09:28.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:09:28 smithi083 ceph-mon[79930]: pgmap v3038: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:09:28.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:09:28 smithi083 ceph-mon[77242]: pgmap v3038: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:09:29.707 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:09:29.996 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:09:29.997 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (103m) 4m ago 103m 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:09:29.997 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (103m) 2m ago 103m 62.7M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:09:29.997 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (104m) 4m ago 104m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:09:29.998 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (104m) 2m ago 104m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:09:29.998 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (100m) 2m ago 107m 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:09:29.998 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (100m) 4m ago 108m 598M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:09:29.998 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (100m) 4m ago 108m 421M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:09:29.999 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (99m) 2m ago 107m 192M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:09:29.999 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (99m) 4m ago 107m 248M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:09:29.999 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (103m) 4m ago 103m 25.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:09:29.999 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (103m) 2m ago 103m 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:09:30.000 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (98m) 4m ago 106m 310M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:09:30.000 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (98m) 4m ago 106m 302M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:09:30.000 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (97m) 4m ago 106m 241M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:09:30.000 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (97m) 4m ago 105m 290M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:09:30.001 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (96m) 2m ago 105m 262M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:09:30.001 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (96m) 2m ago 105m 317M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:09:30.001 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (95m) 2m ago 105m 282M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:09:30.001 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (95m) 2m ago 104m 327M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:09:30.001 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (100m) 2m ago 103m 114M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:09:30.349 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:09:30.350 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:09:30.350 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:09:30.350 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:09:30.350 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:09:30.351 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:09:30.351 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:09:30.351 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:09:30.351 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:09:30.351 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:09:30.352 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:09:30.352 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:09:30.352 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:09:30.352 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:09:30.353 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:09:30.353 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:09:30.353 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:09:30.353 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:09:30.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:09:30 smithi153 ceph-mon[70044]: pgmap v3039: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:09:30.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:09:30 smithi153 ceph-mon[70044]: from='client.37671 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:09:30.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:09:30 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3650716235' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:09:30.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:09:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:09:30.490Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:09:30.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:09:30 smithi083 ceph-mon[77242]: pgmap v3039: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:09:30.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:09:30 smithi083 ceph-mon[77242]: from='client.37671 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:09:30.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:09:30 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3650716235' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:09:30.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:09:30 smithi083 ceph-mon[79930]: pgmap v3039: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:09:30.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:09:30 smithi083 ceph-mon[79930]: from='client.37671 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:09:30.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:09:30 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3650716235' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:09:31.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:09:31 smithi153 ceph-mon[70044]: from='client.47611 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:09:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:09:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:09:31] "GET /metrics HTTP/1.1" 200 225172 "" "Prometheus/2.33.4" 2022-09-29T07:09:31.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:09:31 smithi083 ceph-mon[79930]: from='client.47611 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:09:31.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:09:31 smithi083 ceph-mon[77242]: from='client.47611 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:09:32.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:09:32 smithi153 ceph-mon[70044]: pgmap v3040: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:09:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:09:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:09:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:09:32.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:09:32 smithi083 ceph-mon[79930]: pgmap v3040: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:09:32.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:09:32 smithi083 ceph-mon[77242]: pgmap v3040: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:09:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:09:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:09:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:09:34.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:09:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:09:33.897Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:09:34.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:09:34 smithi153 ceph-mon[70044]: pgmap v3041: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:09:34.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:09:34 smithi083 ceph-mon[79930]: pgmap v3041: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:09:34.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:09:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:09:34.597Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:09:34.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:09:34 smithi083 ceph-mon[77242]: pgmap v3041: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:09:36.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:09:36 smithi153 ceph-mon[70044]: pgmap v3042: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:09:36.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:09:36 smithi083 ceph-mon[79930]: pgmap v3042: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:09:36.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:09:36 smithi083 ceph-mon[77242]: pgmap v3042: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:09:38.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:09:38 smithi153 ceph-mon[70044]: pgmap v3043: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:09:38.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:09:38 smithi083 ceph-mon[79930]: pgmap v3043: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:09:38.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:09:38 smithi083 ceph-mon[77242]: pgmap v3043: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:09:39.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:09:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:09:39.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:09:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:09:39.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:09:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:09:39.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:09:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:09:39.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:09:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:09:39.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:09:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:09:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:09:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:09:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:09:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:09:39.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:09:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:09:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:09:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:09:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:09:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:09:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:09:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:09:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:09:40 smithi153 ceph-mon[70044]: pgmap v3044: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:09:40.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:09:40 smithi083 ceph-mon[79930]: pgmap v3044: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:09:40.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:09:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:09:40.490Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:09:40.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:09:40 smithi083 ceph-mon[77242]: pgmap v3044: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:09:41.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:09:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:09:41] "GET /metrics HTTP/1.1" 200 225175 "" "Prometheus/2.33.4" 2022-09-29T07:09:42.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:09:41 smithi083 ceph-mon[77242]: pgmap v3045: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:09:42.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:09:41 smithi083 ceph-mon[79930]: pgmap v3045: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:09:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:09:41 smithi153 ceph-mon[70044]: pgmap v3045: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:09:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:09:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:09:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:09:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:09:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:09:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:09:44.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:09:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:09:43.898Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:09:44.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:09:44 smithi153 ceph-mon[70044]: pgmap v3046: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:09:44.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:09:44 smithi083 ceph-mon[77242]: pgmap v3046: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:09:44.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:09:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:09:44.597Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:09:44.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:09:44 smithi083 ceph-mon[79930]: pgmap v3046: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:09:46.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:09:46 smithi153 ceph-mon[70044]: pgmap v3047: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:09:46.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:09:46 smithi083 ceph-mon[79930]: pgmap v3047: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:09:46.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:09:46 smithi083 ceph-mon[77242]: pgmap v3047: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:09:48.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:09:48 smithi153 ceph-mon[70044]: pgmap v3048: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:09:48.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:09:48 smithi083 ceph-mon[79930]: pgmap v3048: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:09:48.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:09:48 smithi083 ceph-mon[77242]: pgmap v3048: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:09:50.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:09:50 smithi153 ceph-mon[70044]: pgmap v3049: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:09:50.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:09:50 smithi083 ceph-mon[77242]: pgmap v3049: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:09:50.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:09:50 smithi083 ceph-mon[79930]: pgmap v3049: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:09:50.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:09:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:09:50.491Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:09:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:09:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:09:51] "GET /metrics HTTP/1.1" 200 225175 "" "Prometheus/2.33.4" 2022-09-29T07:09:51.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:09:51 smithi083 ceph-mon[77242]: pgmap v3050: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:09:52.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:09:51 smithi153 ceph-mon[70044]: pgmap v3050: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:09:52.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:09:51 smithi083 ceph-mon[79930]: pgmap v3050: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:09:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:09:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:09:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:09:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:09:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:09:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:09:54.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:09:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:09:53.898Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:09:54.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:09:54 smithi153 ceph-mon[70044]: pgmap v3051: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:09:54.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:09:54 smithi083 ceph-mon[77242]: pgmap v3051: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:09:54.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:09:54 smithi083 ceph-mon[79930]: pgmap v3051: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:09:54.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:09:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:09:54.598Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:09:56.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:09:56 smithi153 ceph-mon[70044]: pgmap v3052: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:09:56.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:09:56 smithi083 ceph-mon[79930]: pgmap v3052: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:09:56.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:09:56 smithi083 ceph-mon[77242]: pgmap v3052: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:09:58.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:09:58 smithi153 ceph-mon[70044]: pgmap v3053: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:09:58.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:09:58 smithi083 ceph-mon[79930]: pgmap v3053: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:09:58.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:09:58 smithi083 ceph-mon[77242]: pgmap v3053: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:00.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:10:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[77219]: 2022-09-29T07:09:59.999+0000 7f160f665700 -1 log_channel(cluster) log [ERR] : overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T07:10:00.677 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:10:00.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:10:00 smithi153 ceph-mon[70044]: pgmap v3054: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:00.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:10:00 smithi153 ceph-mon[70044]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T07:10:00.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:10:00 smithi083 ceph-mon[79930]: pgmap v3054: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:00.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:10:00 smithi083 ceph-mon[79930]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T07:10:00.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:10:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:10:00.491Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:10:00.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:10:00 smithi083 ceph-mon[77242]: pgmap v3054: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:00.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:10:00 smithi083 ceph-mon[77242]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T07:10:00.965 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:10:00.966 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (103m) 4m ago 104m 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:10:00.966 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (104m) 2m ago 103m 62.7M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:10:00.967 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (105m) 4m ago 105m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:10:00.967 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (105m) 2m ago 105m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:10:00.967 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (100m) 2m ago 107m 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:10:00.968 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (101m) 4m ago 109m 598M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:10:00.968 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (100m) 4m ago 109m 421M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:10:00.969 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (99m) 2m ago 107m 192M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:10:00.969 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (100m) 4m ago 107m 248M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:10:00.969 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (104m) 4m ago 104m 25.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:10:00.970 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (104m) 2m ago 104m 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:10:00.970 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (99m) 4m ago 107m 310M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:10:00.971 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (98m) 4m ago 107m 302M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:10:00.971 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (98m) 4m ago 106m 241M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:10:00.971 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (97m) 4m ago 106m 290M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:10:00.972 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (97m) 2m ago 106m 262M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:10:00.972 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (96m) 2m ago 105m 317M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:10:00.972 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (96m) 2m ago 105m 282M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:10:00.972 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (95m) 2m ago 105m 327M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:10:00.973 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (100m) 2m ago 104m 114M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:10:01.321 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:10:01.321 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:10:01.321 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:10:01.322 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:10:01.322 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:10:01.322 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:10:01.322 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:10:01.323 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:10:01.323 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:10:01.323 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:10:01.323 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:10:01.324 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:10:01.324 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:10:01.324 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:10:01.324 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:10:01.325 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:10:01.325 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:10:01.325 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:10:01.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:10:01 smithi153 ceph-mon[70044]: from='client.47623 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:10:01.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:10:01 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3336612919' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:10:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:10:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:10:01] "GET /metrics HTTP/1.1" 200 225175 "" "Prometheus/2.33.4" 2022-09-29T07:10:01.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:10:01 smithi083 ceph-mon[77242]: from='client.47623 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:10:01.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:10:01 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3336612919' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:10:01.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:10:01 smithi083 ceph-mon[79930]: from='client.47623 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:10:01.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:10:01 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3336612919' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:10:02.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:10:02 smithi153 ceph-mon[70044]: from='client.47629 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:10:02.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:10:02 smithi153 ceph-mon[70044]: pgmap v3055: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:10:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:10:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:10:02.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:10:02 smithi083 ceph-mon[77242]: from='client.47629 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:10:02.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:10:02 smithi083 ceph-mon[77242]: pgmap v3055: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:02.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:10:02 smithi083 ceph-mon[79930]: from='client.47629 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:10:02.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:10:02 smithi083 ceph-mon[79930]: pgmap v3055: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:10:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:10:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:10:04.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:10:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:10:03.899Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:10:04.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:10:04 smithi153 ceph-mon[70044]: pgmap v3056: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:04.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:10:04 smithi083 ceph-mon[79930]: pgmap v3056: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:04.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:10:04 smithi083 ceph-mon[77242]: pgmap v3056: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:04.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:10:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:10:04.598Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:10:06.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:10:06 smithi153 ceph-mon[70044]: pgmap v3057: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:06.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:10:06 smithi083 ceph-mon[79930]: pgmap v3057: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:06.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:10:06 smithi083 ceph-mon[77242]: pgmap v3057: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:08.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:10:08 smithi153 ceph-mon[70044]: pgmap v3058: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:08.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:10:08 smithi083 ceph-mon[79930]: pgmap v3058: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:08.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:10:08 smithi083 ceph-mon[77242]: pgmap v3058: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:10.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:10:10 smithi153 ceph-mon[70044]: pgmap v3059: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:10.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:10:10 smithi083 ceph-mon[79930]: pgmap v3059: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:10.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:10:10 smithi083 ceph-mon[77242]: pgmap v3059: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:10.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:10:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:10:10.492Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:10:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:10:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:10:11] "GET /metrics HTTP/1.1" 200 225176 "" "Prometheus/2.33.4" 2022-09-29T07:10:12.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:10:11 smithi153 ceph-mon[70044]: pgmap v3060: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:12.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:10:11 smithi083 ceph-mon[77242]: pgmap v3060: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:12.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:10:11 smithi083 ceph-mon[79930]: pgmap v3060: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:10:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:10:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:10:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:10:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:10:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:10:14.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:10:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:10:13.900Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:10:14.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:10:14 smithi153 ceph-mon[70044]: pgmap v3061: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:14.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:10:14 smithi083 ceph-mon[77242]: pgmap v3061: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:14.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:10:14 smithi083 ceph-mon[79930]: pgmap v3061: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:14.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:10:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:10:14.598Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:10:15.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:10:15 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:10:15.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:10:15 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:10:15.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:10:15 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:10:15.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:10:15 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:10:15.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:10:15 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:10:15.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:10:15 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:10:15.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:10:15 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:10:15.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:10:15 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:10:15.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:10:15 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:10:16.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:10:16 smithi083 ceph-mon[79930]: pgmap v3062: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:16.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:10:16 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:10:16.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:10:16 smithi083 ceph-mon[77242]: pgmap v3062: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:16.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:10:16 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:10:17.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:10:16 smithi153 ceph-mon[70044]: pgmap v3062: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:17.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:10:16 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:10:18.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:10:18 smithi083 ceph-mon[79930]: pgmap v3063: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:18.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:10:18 smithi083 ceph-mon[77242]: pgmap v3063: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:19.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:10:18 smithi153 ceph-mon[70044]: pgmap v3063: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:20.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:10:20 smithi083 ceph-mon[79930]: pgmap v3064: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:20.973 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:10:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:10:20.493Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:10:20.973 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:10:20 smithi083 ceph-mon[77242]: pgmap v3064: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:21.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:10:20 smithi153 ceph-mon[70044]: pgmap v3064: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:10:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:10:21] "GET /metrics HTTP/1.1" 200 225176 "" "Prometheus/2.33.4" 2022-09-29T07:10:22.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:10:21 smithi153 ceph-mon[70044]: pgmap v3065: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:22.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:10:21 smithi083 ceph-mon[79930]: pgmap v3065: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:22.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:10:21 smithi083 ceph-mon[77242]: pgmap v3065: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:10:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:10:22.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:10:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:10:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:10:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:10:24.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:10:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:10:23.901Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:10:24.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:10:24 smithi153 ceph-mon[70044]: pgmap v3066: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:24.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:10:24 smithi083 ceph-mon[77242]: pgmap v3066: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:24.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:10:24 smithi083 ceph-mon[79930]: pgmap v3066: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:24.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:10:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:10:24.599Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:10:26.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:10:26 smithi153 ceph-mon[70044]: pgmap v3067: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:26.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:10:26 smithi083 ceph-mon[79930]: pgmap v3067: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:26.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:10:26 smithi083 ceph-mon[77242]: pgmap v3067: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:28.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:10:28 smithi153 ceph-mon[70044]: pgmap v3068: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:28.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:10:28 smithi083 ceph-mon[79930]: pgmap v3068: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:28.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:10:28 smithi083 ceph-mon[77242]: pgmap v3068: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:30.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:10:30 smithi153 ceph-mon[70044]: pgmap v3069: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:30.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:10:30 smithi083 ceph-mon[79930]: pgmap v3069: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:30.964 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:10:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:10:30.494Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:10:30.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:10:30 smithi083 ceph-mon[77242]: pgmap v3069: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:31.662 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:10:31.941 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:10:31.942 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (104m) 5m ago 104m 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:10:31.942 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (104m) 3m ago 104m 62.7M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:10:31.943 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (105m) 5m ago 105m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:10:31.943 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (105m) 3m ago 105m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:10:31.943 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (101m) 3m ago 108m 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:10:31.944 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (101m) 5m ago 109m 598M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:10:31.944 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (101m) 5m ago 109m 421M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:10:31.944 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (100m) 3m ago 108m 192M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:10:31.945 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (100m) 5m ago 108m 248M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:10:31.945 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (104m) 5m ago 104m 25.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:10:31.945 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (104m) 3m ago 104m 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:10:31.946 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (99m) 5m ago 107m 310M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:10:31.946 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (99m) 5m ago 107m 302M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:10:31.946 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (98m) 5m ago 107m 241M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:10:31.946 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (98m) 5m ago 106m 290M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:10:31.947 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (97m) 3m ago 106m 262M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:10:31.947 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (97m) 3m ago 106m 317M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:10:31.947 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (96m) 3m ago 106m 282M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:10:31.947 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (96m) 3m ago 105m 327M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:10:31.955 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (101m) 3m ago 105m 114M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:10:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:10:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:10:31] "GET /metrics HTTP/1.1" 200 225177 "" "Prometheus/2.33.4" 2022-09-29T07:10:32.297 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:10:32.298 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:10:32.298 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:10:32.298 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:10:32.298 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:10:32.299 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:10:32.300 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:10:32.301 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:10:32.301 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:10:32.301 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:10:32.301 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:10:32.302 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:10:32.302 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:10:32.302 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:10:32.302 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:10:32.303 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:10:32.303 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:10:32.303 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:10:32.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:10:31 smithi153 ceph-mon[70044]: pgmap v3070: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:32.312 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:10:31 smithi153 ceph-mon[70044]: from='client.47641 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:10:32.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:10:31 smithi083 ceph-mon[77242]: pgmap v3070: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:32.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:10:31 smithi083 ceph-mon[77242]: from='client.47641 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:10:32.477 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:10:31 smithi083 ceph-mon[79930]: pgmap v3070: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:32.478 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:10:31 smithi083 ceph-mon[79930]: from='client.47641 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:10:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:10:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:10:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:10:33.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:10:32 smithi153 ceph-mon[70044]: from='client.47647 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:10:33.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:10:32 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/367427373' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:10:33.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:10:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:10:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:10:33.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:10:32 smithi083 ceph-mon[79930]: from='client.47647 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:10:33.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:10:32 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/367427373' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:10:33.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:10:32 smithi083 ceph-mon[77242]: from='client.47647 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:10:33.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:10:32 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/367427373' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:10:34.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:10:33 smithi083 ceph-mon[79930]: pgmap v3071: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:34.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:10:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:10:33.901Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:10:34.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:10:33 smithi083 ceph-mon[77242]: pgmap v3071: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:34.314 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:10:33 smithi153 ceph-mon[70044]: pgmap v3071: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:34.969 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:10:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:10:34.599Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:10:36.816 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:10:36 smithi153 ceph-mon[70044]: pgmap v3072: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:36.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:10:36 smithi083 ceph-mon[77242]: pgmap v3072: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:36.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:10:36 smithi083 ceph-mon[79930]: pgmap v3072: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:38.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:10:38 smithi153 ceph-mon[70044]: pgmap v3073: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:38.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:10:38 smithi083 ceph-mon[79930]: pgmap v3073: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:38.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:10:38 smithi083 ceph-mon[77242]: pgmap v3073: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:39.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:10:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:10:39.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:10:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:10:39.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:10:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:10:39.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:10:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:10:39.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:10:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:10:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:10:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:10:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:10:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:10:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:10:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:10:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:10:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:10:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:10:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:10:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:10:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:10:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:10:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:10:40.813 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:10:40 smithi153 ceph-mon[70044]: pgmap v3074: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:10:40 smithi083 ceph-mon[79930]: pgmap v3074: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:40.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:10:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:10:40.495Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:10:40.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:10:40 smithi083 ceph-mon[77242]: pgmap v3074: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:10:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:10:41] "GET /metrics HTTP/1.1" 200 225177 "" "Prometheus/2.33.4" 2022-09-29T07:10:42.314 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:10:41 smithi153 ceph-mon[70044]: pgmap v3075: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:10:41 smithi083 ceph-mon[79930]: pgmap v3075: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:42.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:10:41 smithi083 ceph-mon[77242]: pgmap v3075: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:42.815 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:10:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:10:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:10:43.312 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:10:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:10:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:10:44.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:10:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:10:43.902Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:10:44.815 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:10:44 smithi153 ceph-mon[70044]: pgmap v3076: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:44.969 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:10:44 smithi083 ceph-mon[79930]: pgmap v3076: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:44.970 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:10:44 smithi083 ceph-mon[77242]: pgmap v3076: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:44.970 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:10:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:10:44.600Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:10:46.818 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:10:46 smithi153 ceph-mon[70044]: pgmap v3077: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:46.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:10:46 smithi083 ceph-mon[77242]: pgmap v3077: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:46.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:10:46 smithi083 ceph-mon[79930]: pgmap v3077: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:48.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:10:48 smithi153 ceph-mon[70044]: pgmap v3078: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:48.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:10:48 smithi083 ceph-mon[79930]: pgmap v3078: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:48.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:10:48 smithi083 ceph-mon[77242]: pgmap v3078: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:50.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:10:50 smithi153 ceph-mon[70044]: pgmap v3079: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:50.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:10:50 smithi083 ceph-mon[77242]: pgmap v3079: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:50.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:10:50 smithi083 ceph-mon[79930]: pgmap v3079: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:50.965 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:10:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:10:50.495Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:10:51.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:10:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:10:51] "GET /metrics HTTP/1.1" 200 225177 "" "Prometheus/2.33.4" 2022-09-29T07:10:52.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:10:51 smithi153 ceph-mon[70044]: pgmap v3080: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:52.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:10:51 smithi083 ceph-mon[77242]: pgmap v3080: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:52.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:10:51 smithi083 ceph-mon[79930]: pgmap v3080: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:10:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:10:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:10:53.316 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:10:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:10:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:10:54.215 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:10:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:10:53.902Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:10:54.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:10:54 smithi153 ceph-mon[70044]: pgmap v3081: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:54.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:10:54 smithi083 ceph-mon[79930]: pgmap v3081: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:54.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:10:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:10:54.600Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:10:54.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:10:54 smithi083 ceph-mon[77242]: pgmap v3081: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:56.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:10:56 smithi153 ceph-mon[70044]: pgmap v3082: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:56.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:10:56 smithi083 ceph-mon[79930]: pgmap v3082: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:56.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:10:56 smithi083 ceph-mon[77242]: pgmap v3082: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:58.814 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:10:58 smithi153 ceph-mon[70044]: pgmap v3083: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:58.978 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:10:58 smithi083 ceph-mon[79930]: pgmap v3083: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:10:58.985 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:10:58 smithi083 ceph-mon[77242]: pgmap v3083: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:00.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:11:00 smithi153 ceph-mon[70044]: pgmap v3084: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:00.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:11:00 smithi083 ceph-mon[79930]: pgmap v3084: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:00.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:11:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:11:00.495Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:11:00.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:11:00 smithi083 ceph-mon[77242]: pgmap v3084: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:01.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:11:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:11:01] "GET /metrics HTTP/1.1" 200 225176 "" "Prometheus/2.33.4" 2022-09-29T07:11:02.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:11:01 smithi153 ceph-mon[70044]: pgmap v3085: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:02.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:11:01 smithi083 ceph-mon[79930]: pgmap v3085: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:02.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:11:01 smithi083 ceph-mon[77242]: pgmap v3085: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:02.626 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:11:02.822 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:11:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:11:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:11:02.912 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:11:02.912 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (104m) 6m ago 105m 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:11:02.912 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (105m) 3m ago 105m 62.7M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:11:02.913 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (106m) 6m ago 106m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:11:02.913 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (106m) 3m ago 106m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:11:02.913 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (101m) 3m ago 108m 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:11:02.913 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (102m) 6m ago 110m 598M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:11:02.914 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (101m) 6m ago 110m 421M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:11:02.914 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (100m) 3m ago 108m 192M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:11:02.921 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (101m) 6m ago 109m 248M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:11:02.921 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (105m) 6m ago 105m 25.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:11:02.921 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (105m) 3m ago 105m 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:11:02.922 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (100m) 6m ago 108m 310M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:11:02.922 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (99m) 6m ago 108m 302M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:11:02.922 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (99m) 6m ago 107m 241M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:11:02.922 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (98m) 6m ago 107m 290M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:11:02.923 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (98m) 3m ago 107m 262M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:11:02.923 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (97m) 3m ago 106m 317M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:11:02.923 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (97m) 3m ago 106m 282M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:11:02.923 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (96m) 3m ago 106m 327M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:11:02.924 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (101m) 3m ago 105m 114M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:11:03.277 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:11:03.277 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:11:03.277 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:11:03.278 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:11:03.278 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:11:03.278 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:11:03.281 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:11:03.281 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:11:03.281 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:11:03.282 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:11:03.282 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:11:03.282 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:11:03.283 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:11:03.283 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:11:03.283 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:11:03.283 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:11:03.283 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:11:03.284 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:11:03.313 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:11:02 smithi153 ceph-mon[70044]: from='client.37725 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:11:03.318 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:11:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:11:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:11:03.468 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:11:02 smithi083 ceph-mon[79930]: from='client.37725 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:11:03.472 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:11:02 smithi083 ceph-mon[77242]: from='client.37725 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:11:04.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:11:03 smithi083 ceph-mon[79930]: from='client.37731 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:11:04.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:11:03 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1326825089' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:11:04.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:11:03 smithi083 ceph-mon[79930]: pgmap v3086: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:04.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:11:03 smithi083 ceph-mon[77242]: from='client.37731 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:11:04.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:11:03 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1326825089' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:11:04.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:11:03 smithi083 ceph-mon[77242]: pgmap v3086: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:04.218 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:11:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:11:03.903Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:11:04.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:11:03 smithi153 ceph-mon[70044]: from='client.37731 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:11:04.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:11:03 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1326825089' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:11:04.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:11:03 smithi153 ceph-mon[70044]: pgmap v3086: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:04.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:11:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:11:04.601Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:11:06.814 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:11:06 smithi153 ceph-mon[70044]: pgmap v3087: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:06.970 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:11:06 smithi083 ceph-mon[79930]: pgmap v3087: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:06.971 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:11:06 smithi083 ceph-mon[77242]: pgmap v3087: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:08.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:11:08 smithi153 ceph-mon[70044]: pgmap v3088: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:08.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:11:08 smithi083 ceph-mon[79930]: pgmap v3088: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:08.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:11:08 smithi083 ceph-mon[77242]: pgmap v3088: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:10.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:11:10 smithi153 ceph-mon[70044]: pgmap v3089: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:10.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:11:10 smithi083 ceph-mon[79930]: pgmap v3089: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:10.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:11:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:11:10.496Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:11:10.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:11:10 smithi083 ceph-mon[77242]: pgmap v3089: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:11:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:11:11] "GET /metrics HTTP/1.1" 200 225174 "" "Prometheus/2.33.4" 2022-09-29T07:11:12.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:11:11 smithi153 ceph-mon[70044]: pgmap v3090: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:12.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:11:11 smithi083 ceph-mon[77242]: pgmap v3090: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:12.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:11:11 smithi083 ceph-mon[79930]: pgmap v3090: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:11:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:11:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:11:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:11:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:11:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:11:14.214 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:11:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:11:13.905Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:11:14.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:11:14 smithi153 ceph-mon[70044]: pgmap v3091: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:14.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:11:14 smithi083 ceph-mon[79930]: pgmap v3091: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:14.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:11:14 smithi083 ceph-mon[77242]: pgmap v3091: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:14.964 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:11:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:11:14.602Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:11:16.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:11:16 smithi153 ceph-mon[70044]: pgmap v3092: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:16.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:11:16 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:11:16.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:11:16 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:11:16.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:11:16 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:11:16.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:11:16 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:11:16.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:11:16 smithi083 ceph-mon[79930]: pgmap v3092: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:16.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:11:16 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:11:16.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:11:16 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:11:16.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:11:16 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:11:16.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:11:16 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:11:16.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:11:16 smithi083 ceph-mon[77242]: pgmap v3092: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:16.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:11:16 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:11:16.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:11:16 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:11:16.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:11:16 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:11:16.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:11:16 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:11:18.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:11:18 smithi153 ceph-mon[70044]: pgmap v3093: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:18.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:11:18 smithi083 ceph-mon[79930]: pgmap v3093: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:18.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:11:18 smithi083 ceph-mon[77242]: pgmap v3093: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:20.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:11:20 smithi153 ceph-mon[70044]: pgmap v3094: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:20.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:11:20 smithi083 ceph-mon[79930]: pgmap v3094: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:20.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:11:20 smithi083 ceph-mon[77242]: pgmap v3094: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:20.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:11:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:11:20.497Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:11:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:11:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:11:21] "GET /metrics HTTP/1.1" 200 225174 "" "Prometheus/2.33.4" 2022-09-29T07:11:22.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:11:21 smithi153 ceph-mon[70044]: pgmap v3095: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:22.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:11:21 smithi083 ceph-mon[79930]: pgmap v3095: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:22.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:11:21 smithi083 ceph-mon[77242]: pgmap v3095: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:11:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:11:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:11:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:11:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:11:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:11:24.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:11:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:11:23.906Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:11:24.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:11:24 smithi153 ceph-mon[70044]: pgmap v3096: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:24.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:11:24 smithi083 ceph-mon[79930]: pgmap v3096: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:24.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:11:24 smithi083 ceph-mon[77242]: pgmap v3096: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:24.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:11:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:11:24.603Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:11:26.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:11:26 smithi153 ceph-mon[70044]: pgmap v3097: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:26.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:11:26 smithi083 ceph-mon[79930]: pgmap v3097: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:26.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:11:26 smithi083 ceph-mon[77242]: pgmap v3097: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:28.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:11:28 smithi153 ceph-mon[70044]: pgmap v3098: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:28.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:11:28 smithi083 ceph-mon[79930]: pgmap v3098: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:28.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:11:28 smithi083 ceph-mon[77242]: pgmap v3098: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:30.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:11:30 smithi153 ceph-mon[70044]: pgmap v3099: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:30.968 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:11:30 smithi083 ceph-mon[79930]: pgmap v3099: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:30.969 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:11:30 smithi083 ceph-mon[77242]: pgmap v3099: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:30.969 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:11:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:11:30.497Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:11:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:11:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:11:31] "GET /metrics HTTP/1.1" 200 225174 "" "Prometheus/2.33.4" 2022-09-29T07:11:32.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:11:31 smithi153 ceph-mon[70044]: pgmap v3100: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:32.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:11:31 smithi083 ceph-mon[77242]: pgmap v3100: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:32.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:11:31 smithi083 ceph-mon[79930]: pgmap v3100: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:32.810 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:11:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:11:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:11:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:11:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:11:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:11:33.622 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:11:33.907 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:11:33.908 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (105m) 6m ago 105m 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:11:33.909 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (105m) 4m ago 105m 62.7M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:11:33.909 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (106m) 6m ago 106m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:11:33.909 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (106m) 4m ago 106m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:11:33.910 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (102m) 4m ago 109m 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:11:33.910 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (102m) 6m ago 110m 598M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:11:33.910 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (102m) 6m ago 110m 421M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:11:33.910 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (101m) 4m ago 109m 192M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:11:33.911 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (101m) 6m ago 109m 248M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:11:33.911 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (105m) 6m ago 105m 25.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:11:33.911 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (105m) 4m ago 105m 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:11:33.911 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (100m) 6m ago 108m 310M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:11:33.911 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (100m) 6m ago 108m 302M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:11:33.912 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (99m) 6m ago 108m 241M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:11:33.912 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (99m) 6m ago 107m 290M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:11:33.912 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (98m) 4m ago 107m 262M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:11:33.912 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (98m) 4m ago 107m 317M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:11:33.913 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (97m) 4m ago 107m 282M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:11:33.913 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (97m) 4m ago 106m 327M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:11:33.913 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (102m) 4m ago 106m 114M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:11:34.226 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:11:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:11:33.906Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:11:34.260 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:11:34.261 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:11:34.261 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:11:34.267 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:11:34.267 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:11:34.267 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:11:34.267 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:11:34.268 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:11:34.268 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:11:34.268 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:11:34.268 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:11:34.269 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:11:34.269 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:11:34.269 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:11:34.269 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:11:34.269 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:11:34.272 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:11:34.272 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:11:34.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:11:34 smithi153 ceph-mon[70044]: pgmap v3101: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:34.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:11:34 smithi153 ceph-mon[70044]: from='client.37743 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:11:34.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:11:34 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3554771850' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:11:34.967 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:11:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:11:34.603Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:11:34.968 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:11:34 smithi083 ceph-mon[77242]: pgmap v3101: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:34.969 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:11:34 smithi083 ceph-mon[77242]: from='client.37743 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:11:34.969 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:11:34 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3554771850' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:11:35.010 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:11:34 smithi083 ceph-mon[79930]: pgmap v3101: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:35.011 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:11:34 smithi083 ceph-mon[79930]: from='client.37743 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:11:35.011 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:11:34 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3554771850' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:11:35.817 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:11:35 smithi153 ceph-mon[70044]: from='client.37749 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:11:35.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:11:35 smithi083 ceph-mon[77242]: from='client.37749 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:11:35.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:11:35 smithi083 ceph-mon[79930]: from='client.37749 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:11:36.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:11:36 smithi153 ceph-mon[70044]: pgmap v3102: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:36.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:11:36 smithi083 ceph-mon[77242]: pgmap v3102: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:36.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:11:36 smithi083 ceph-mon[79930]: pgmap v3102: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:38.815 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:11:38 smithi153 ceph-mon[70044]: pgmap v3103: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:38.971 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:11:38 smithi083 ceph-mon[79930]: pgmap v3103: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:38.990 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:11:38 smithi083 ceph-mon[77242]: pgmap v3103: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:39.814 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:11:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:11:39.815 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:11:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:11:39.815 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:11:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:11:39.815 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:11:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:11:39.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:11:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:11:39.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:11:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:11:39.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:11:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:11:39.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:11:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:11:39.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:11:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:11:39.968 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:11:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:11:39.968 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:11:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:11:39.968 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:11:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:11:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:11:40 smithi153 ceph-mon[70044]: pgmap v3104: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:40.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:11:40 smithi083 ceph-mon[79930]: pgmap v3104: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:40.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:11:40 smithi083 ceph-mon[77242]: pgmap v3104: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:40.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:11:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:11:40.498Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:11:41.968 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:11:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:11:41] "GET /metrics HTTP/1.1" 200 225176 "" "Prometheus/2.33.4" 2022-09-29T07:11:42.322 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:11:41 smithi153 ceph-mon[70044]: pgmap v3105: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:42.468 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:11:41 smithi083 ceph-mon[79930]: pgmap v3105: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:42.468 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:11:41 smithi083 ceph-mon[77242]: pgmap v3105: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:42.813 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:11:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:11:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:11:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:11:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:11:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:11:44.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:11:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:11:43.907Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:11:44.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:11:44 smithi153 ceph-mon[70044]: pgmap v3106: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:44.981 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:11:44 smithi083 ceph-mon[79930]: pgmap v3106: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:44.982 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:11:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:11:44.604Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:11:44.983 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:11:44 smithi083 ceph-mon[77242]: pgmap v3106: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:46.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:11:46 smithi153 ceph-mon[70044]: pgmap v3107: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:46.972 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:11:46 smithi083 ceph-mon[79930]: pgmap v3107: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:46.973 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:11:46 smithi083 ceph-mon[77242]: pgmap v3107: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:48.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:11:48 smithi153 ceph-mon[70044]: pgmap v3108: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:48.970 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:11:48 smithi083 ceph-mon[79930]: pgmap v3108: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:48.971 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:11:48 smithi083 ceph-mon[77242]: pgmap v3108: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:50.814 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:11:50 smithi153 ceph-mon[70044]: pgmap v3109: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:50.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:11:50 smithi083 ceph-mon[79930]: pgmap v3109: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:50.967 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:11:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:11:50.498Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:11:50.968 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:11:50 smithi083 ceph-mon[77242]: pgmap v3109: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:11:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:11:51] "GET /metrics HTTP/1.1" 200 225176 "" "Prometheus/2.33.4" 2022-09-29T07:11:52.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:11:51 smithi153 ceph-mon[70044]: pgmap v3110: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:52.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:11:51 smithi083 ceph-mon[79930]: pgmap v3110: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:52.485 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:11:51 smithi083 ceph-mon[77242]: pgmap v3110: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:52.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:11:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:11:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:11:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:11:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:11:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:11:54.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:11:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:11:53.908Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:11:54.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:11:54 smithi153 ceph-mon[70044]: pgmap v3111: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:54.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:11:54 smithi083 ceph-mon[79930]: pgmap v3111: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:54.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:11:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:11:54.605Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:11:54.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:11:54 smithi083 ceph-mon[77242]: pgmap v3111: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:56.817 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:11:56 smithi153 ceph-mon[70044]: pgmap v3112: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:56.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:11:56 smithi083 ceph-mon[79930]: pgmap v3112: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:56.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:11:56 smithi083 ceph-mon[77242]: pgmap v3112: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:58.813 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:11:58 smithi153 ceph-mon[70044]: pgmap v3113: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:58.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:11:58 smithi083 ceph-mon[79930]: pgmap v3113: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:11:58.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:11:58 smithi083 ceph-mon[77242]: pgmap v3113: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:00.814 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:12:00 smithi153 ceph-mon[70044]: pgmap v3114: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:00.970 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:12:00 smithi083 ceph-mon[77242]: pgmap v3114: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:00.971 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:12:00 smithi083 ceph-mon[79930]: pgmap v3114: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:00.972 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:12:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:12:00.499Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:12:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:12:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:12:01] "GET /metrics HTTP/1.1" 200 225177 "" "Prometheus/2.33.4" 2022-09-29T07:12:02.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:12:01 smithi153 ceph-mon[70044]: pgmap v3115: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:02.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:12:01 smithi083 ceph-mon[79930]: pgmap v3115: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:02.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:12:01 smithi083 ceph-mon[77242]: pgmap v3115: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:02.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:12:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:12:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:12:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:12:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:12:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:12:04.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:12:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:12:03.908Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:12:04.603 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:12:04.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:12:04 smithi153 ceph-mon[70044]: pgmap v3116: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:04.888 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:12:04.889 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (105m) 7m ago 106m 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:12:04.889 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (106m) 4m ago 106m 62.7M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:12:04.889 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (107m) 7m ago 107m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:12:04.889 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (107m) 4m ago 107m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:12:04.890 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (102m) 4m ago 109m 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:12:04.895 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (103m) 7m ago 111m 598M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:12:04.896 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (102m) 7m ago 111m 421M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:12:04.896 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (101m) 4m ago 109m 192M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:12:04.896 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (102m) 7m ago 110m 248M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:12:04.896 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (106m) 7m ago 106m 25.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:12:04.897 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (106m) 4m ago 106m 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:12:04.897 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (101m) 7m ago 109m 310M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:12:04.897 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (100m) 7m ago 109m 302M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:12:04.897 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (100m) 7m ago 108m 241M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:12:04.918 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (99m) 7m ago 108m 290M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:12:04.919 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (99m) 4m ago 108m 262M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:12:04.919 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (98m) 4m ago 107m 317M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:12:04.920 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (98m) 4m ago 107m 282M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:12:04.920 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (97m) 4m ago 107m 327M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:12:04.921 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (102m) 4m ago 106m 114M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:12:04.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:12:04 smithi083 ceph-mon[79930]: pgmap v3116: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:04.964 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:12:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:12:04.606Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:12:04.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:12:04 smithi083 ceph-mon[77242]: pgmap v3116: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:05.243 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:12:05.243 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:12:05.243 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:12:05.244 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:12:05.244 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:12:05.244 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:12:05.244 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:12:05.245 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:12:05.245 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:12:05.245 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:12:05.245 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:12:05.254 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:12:05.255 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:12:05.255 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:12:05.255 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:12:05.255 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:12:05.256 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:12:05.256 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:12:05.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:12:05 smithi153 ceph-mon[70044]: from='client.37761 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:12:05.814 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:12:05 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/302569610' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:12:05.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:12:05 smithi083 ceph-mon[77242]: from='client.37761 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:12:05.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:12:05 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/302569610' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:12:05.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:12:05 smithi083 ceph-mon[79930]: from='client.37761 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:12:05.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:12:05 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/302569610' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:12:06.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:12:06 smithi153 ceph-mon[70044]: from='client.37767 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:12:06.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:12:06 smithi153 ceph-mon[70044]: pgmap v3117: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:06.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:12:06 smithi083 ceph-mon[79930]: from='client.37767 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:12:06.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:12:06 smithi083 ceph-mon[79930]: pgmap v3117: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:06.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:12:06 smithi083 ceph-mon[77242]: from='client.37767 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:12:06.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:12:06 smithi083 ceph-mon[77242]: pgmap v3117: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:08.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:12:08 smithi153 ceph-mon[70044]: pgmap v3118: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:08.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:12:08 smithi083 ceph-mon[77242]: pgmap v3118: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:08.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:12:08 smithi083 ceph-mon[79930]: pgmap v3118: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:10.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:12:10 smithi153 ceph-mon[70044]: pgmap v3119: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:10.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:12:10 smithi083 ceph-mon[79930]: pgmap v3119: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:10.967 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:12:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:12:10.500Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:12:10.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:12:10 smithi083 ceph-mon[77242]: pgmap v3119: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:11.970 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:12:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:12:11] "GET /metrics HTTP/1.1" 200 225170 "" "Prometheus/2.33.4" 2022-09-29T07:12:12.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:12:11 smithi153 ceph-mon[70044]: pgmap v3120: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:12.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:12:12 smithi083 ceph-mon[79930]: pgmap v3120: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:12.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:12:11 smithi083 ceph-mon[77242]: pgmap v3120: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:12:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:12:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:12:13.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:12:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:12:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:12:14.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:12:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:12:13.909Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:12:14.813 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:12:14 smithi153 ceph-mon[70044]: pgmap v3121: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:14.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:12:14 smithi083 ceph-mon[77242]: pgmap v3121: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:14.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:12:14 smithi083 ceph-mon[79930]: pgmap v3121: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:14.964 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:12:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:12:14.606Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:12:16.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:12:16 smithi153 ceph-mon[70044]: pgmap v3122: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:16.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:12:16 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:12:16.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:12:16 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:12:16.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:12:16 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:12:16.968 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:12:16 smithi083 ceph-mon[79930]: pgmap v3122: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:16.968 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:12:16 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:12:16.969 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:12:16 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:12:16.971 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:12:16 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:12:16.991 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:12:16 smithi083 ceph-mon[77242]: pgmap v3122: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:17.001 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:12:16 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:12:17.001 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:12:16 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:12:17.012 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:12:16 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:12:18.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:12:17 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:12:18.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:12:17 smithi153 ceph-mon[70044]: pgmap v3123: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:18.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:12:17 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:12:18.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:12:17 smithi083 ceph-mon[79930]: pgmap v3123: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:18.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:12:17 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:12:18.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:12:17 smithi083 ceph-mon[77242]: pgmap v3123: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:20.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:12:20 smithi153 ceph-mon[70044]: pgmap v3124: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:20.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:12:20 smithi083 ceph-mon[79930]: pgmap v3124: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:20.989 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:12:20 smithi083 ceph-mon[77242]: pgmap v3124: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:20.990 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:12:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:12:20.501Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:12:21.967 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:12:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:12:21] "GET /metrics HTTP/1.1" 200 225170 "" "Prometheus/2.33.4" 2022-09-29T07:12:22.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:12:21 smithi153 ceph-mon[70044]: pgmap v3125: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:22.468 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:12:21 smithi083 ceph-mon[79930]: pgmap v3125: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:22.469 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:12:21 smithi083 ceph-mon[77242]: pgmap v3125: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:12:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:12:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:12:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:12:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:12:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:12:24.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:12:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:12:23.910Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:12:24.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:12:24 smithi153 ceph-mon[70044]: pgmap v3126: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:24.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:12:24 smithi083 ceph-mon[79930]: pgmap v3126: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:24.964 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:12:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:12:24.606Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:12:24.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:12:24 smithi083 ceph-mon[77242]: pgmap v3126: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:26.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:12:26 smithi153 ceph-mon[70044]: pgmap v3127: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:26.975 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:12:26 smithi083 ceph-mon[79930]: pgmap v3127: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:26.986 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:12:26 smithi083 ceph-mon[77242]: pgmap v3127: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:28.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:12:28 smithi153 ceph-mon[70044]: pgmap v3128: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:28.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:12:28 smithi083 ceph-mon[79930]: pgmap v3128: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:28.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:12:28 smithi083 ceph-mon[77242]: pgmap v3128: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:30.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:12:30 smithi153 ceph-mon[70044]: pgmap v3129: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:30.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:12:30 smithi083 ceph-mon[79930]: pgmap v3129: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:30.964 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:12:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:12:30.502Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:12:30.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:12:30 smithi083 ceph-mon[77242]: pgmap v3129: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:12:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:12:31] "GET /metrics HTTP/1.1" 200 225178 "" "Prometheus/2.33.4" 2022-09-29T07:12:32.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:12:32 smithi153 ceph-mon[70044]: pgmap v3130: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:32.476 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:12:31 smithi083 ceph-mon[79930]: pgmap v3130: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:32.476 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:12:31 smithi083 ceph-mon[77242]: pgmap v3130: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:12:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:12:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:12:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:12:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:12:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:12:34.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:12:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:12:33.910Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:12:34.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:12:34 smithi153 ceph-mon[70044]: pgmap v3131: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:34.969 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:12:34 smithi083 ceph-mon[79930]: pgmap v3131: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:34.970 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:12:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:12:34.606Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:12:34.971 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:12:34 smithi083 ceph-mon[77242]: pgmap v3131: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:35.563 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:12:35.856 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:12:35.857 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (106m) 7m ago 106m 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:12:35.857 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (106m) 5m ago 106m 62.7M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:12:35.857 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (107m) 7m ago 107m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:12:35.857 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (107m) 5m ago 107m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:12:35.858 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (103m) 5m ago 110m 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:12:35.858 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (103m) 7m ago 111m 598M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:12:35.858 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (103m) 7m ago 111m 421M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:12:35.858 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (102m) 5m ago 110m 192M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:12:35.859 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (102m) 7m ago 110m 248M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:12:35.859 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (107m) 7m ago 107m 25.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:12:35.859 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (106m) 5m ago 106m 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:12:35.859 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (101m) 7m ago 109m 310M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:12:35.860 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (101m) 7m ago 109m 302M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:12:35.864 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (100m) 7m ago 109m 241M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:12:35.865 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (100m) 7m ago 109m 290M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:12:35.865 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (99m) 5m ago 108m 262M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:12:35.867 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (99m) 5m ago 108m 317M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:12:35.868 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (98m) 5m ago 108m 282M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:12:35.868 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (98m) 5m ago 107m 327M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:12:35.868 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (103m) 5m ago 107m 114M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:12:36.211 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:12:36.212 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:12:36.212 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:12:36.213 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:12:36.213 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:12:36.221 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:12:36.221 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:12:36.221 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:12:36.222 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:12:36.222 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:12:36.222 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:12:36.222 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:12:36.223 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:12:36.237 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:12:36.237 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:12:36.237 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:12:36.238 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:12:36.247 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:12:36.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:12:36 smithi153 ceph-mon[70044]: pgmap v3132: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:36.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:12:36 smithi153 ceph-mon[70044]: from='client.47713 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:12:36.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:12:36 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/149384461' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:12:36.968 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:12:36 smithi083 ceph-mon[77242]: pgmap v3132: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:36.968 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:12:36 smithi083 ceph-mon[77242]: from='client.47713 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:12:36.969 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:12:36 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/149384461' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:12:36.969 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:12:36 smithi083 ceph-mon[79930]: pgmap v3132: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:36.987 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:12:36 smithi083 ceph-mon[79930]: from='client.47713 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:12:36.987 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:12:36 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/149384461' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:12:37.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:12:37 smithi153 ceph-mon[70044]: from='client.47719 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:12:37.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:12:37 smithi083 ceph-mon[79930]: from='client.47719 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:12:37.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:12:37 smithi083 ceph-mon[77242]: from='client.47719 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:12:38.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:12:38 smithi153 ceph-mon[70044]: pgmap v3133: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:38.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:12:38 smithi083 ceph-mon[79930]: pgmap v3133: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:38.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:12:38 smithi083 ceph-mon[77242]: pgmap v3133: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:39.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:12:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:12:39.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:12:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:12:39.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:12:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:12:39.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:12:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:12:39.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:12:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:12:39.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:12:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:12:39.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:12:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:12:39.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:12:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:12:39.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:12:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:12:39.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:12:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:12:39.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:12:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:12:39.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:12:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:12:40.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:12:40 smithi153 ceph-mon[70044]: pgmap v3134: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:12:40 smithi083 ceph-mon[79930]: pgmap v3134: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:40.964 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:12:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:12:40.503Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:12:40.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:12:40 smithi083 ceph-mon[77242]: pgmap v3134: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:41.966 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:12:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:12:41] "GET /metrics HTTP/1.1" 200 225174 "" "Prometheus/2.33.4" 2022-09-29T07:12:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:12:42 smithi153 ceph-mon[70044]: pgmap v3135: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:42.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:12:42 smithi083 ceph-mon[79930]: pgmap v3135: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:42.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:12:41 smithi083 ceph-mon[77242]: pgmap v3135: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:42.811 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:12:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:12:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:12:43.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:12:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:12:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:12:44.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:12:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:12:43.910Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:12:44.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:12:44 smithi153 ceph-mon[70044]: pgmap v3136: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:44.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:12:44 smithi083 ceph-mon[79930]: pgmap v3136: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:44.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:12:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:12:44.607Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:12:44.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:12:44 smithi083 ceph-mon[77242]: pgmap v3136: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:46.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:12:46 smithi153 ceph-mon[70044]: pgmap v3137: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:46.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:12:46 smithi083 ceph-mon[79930]: pgmap v3137: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:46.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:12:46 smithi083 ceph-mon[77242]: pgmap v3137: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:48.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:12:48 smithi153 ceph-mon[70044]: pgmap v3138: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:48.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:12:48 smithi083 ceph-mon[79930]: pgmap v3138: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:48.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:12:48 smithi083 ceph-mon[77242]: pgmap v3138: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:50.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:12:50 smithi153 ceph-mon[70044]: pgmap v3139: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:50.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:12:50 smithi083 ceph-mon[79930]: pgmap v3139: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:50.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:12:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:12:50.504Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:12:50.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:12:50 smithi083 ceph-mon[77242]: pgmap v3139: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:12:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:12:51] "GET /metrics HTTP/1.1" 200 225174 "" "Prometheus/2.33.4" 2022-09-29T07:12:52.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:12:52 smithi153 ceph-mon[70044]: pgmap v3140: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:52.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:12:52 smithi083 ceph-mon[77242]: pgmap v3140: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:52.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:12:52 smithi083 ceph-mon[79930]: pgmap v3140: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:12:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:12:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:12:53.311 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:12:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:12:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:12:54.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:12:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:12:53.911Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:12:54.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:12:54 smithi153 ceph-mon[70044]: pgmap v3141: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:54.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:12:54 smithi083 ceph-mon[79930]: pgmap v3141: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:54.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:12:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:12:54.608Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:12:54.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:12:54 smithi083 ceph-mon[77242]: pgmap v3141: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:56.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:12:56 smithi153 ceph-mon[70044]: pgmap v3142: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:56.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:12:56 smithi083 ceph-mon[79930]: pgmap v3142: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:56.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:12:56 smithi083 ceph-mon[77242]: pgmap v3142: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:58.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:12:58 smithi153 ceph-mon[70044]: pgmap v3143: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:58.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:12:58 smithi083 ceph-mon[79930]: pgmap v3143: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:12:58.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:12:58 smithi083 ceph-mon[77242]: pgmap v3143: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:00.968 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:13:00 smithi083 ceph-mon[79930]: pgmap v3144: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:00.969 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:13:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:13:00.504Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:13:00.969 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:13:00 smithi083 ceph-mon[77242]: pgmap v3144: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:01.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:13:00 smithi153 ceph-mon[70044]: pgmap v3144: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:13:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:13:01] "GET /metrics HTTP/1.1" 200 225177 "" "Prometheus/2.33.4" 2022-09-29T07:13:02.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:13:02 smithi153 ceph-mon[70044]: pgmap v3145: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:02.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:13:02 smithi083 ceph-mon[79930]: pgmap v3145: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:02.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:13:02 smithi083 ceph-mon[77242]: pgmap v3145: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:13:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:13:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:13:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:13:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:13:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:13:04.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:13:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:13:03.911Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:13:04.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:13:04 smithi153 ceph-mon[70044]: pgmap v3146: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:04.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:13:04 smithi083 ceph-mon[79930]: pgmap v3146: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:04.965 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:13:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:13:04.608Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:13:04.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:13:04 smithi083 ceph-mon[77242]: pgmap v3146: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:06.539 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:13:06.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:13:06 smithi153 ceph-mon[70044]: pgmap v3147: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:06.824 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:13:06.824 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (106m) 8m ago 107m 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:13:06.825 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (107m) 5m ago 107m 62.7M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:13:06.825 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (108m) 8m ago 108m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:13:06.825 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (108m) 5m ago 108m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:13:06.826 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (103m) 5m ago 110m 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:13:06.826 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (104m) 8m ago 112m 598M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:13:06.827 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (103m) 8m ago 112m 421M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:13:06.827 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (103m) 5m ago 110m 192M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:13:06.832 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (103m) 8m ago 111m 248M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:13:06.832 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (107m) 8m ago 107m 25.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:13:06.833 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (107m) 5m ago 107m 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:13:06.833 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (102m) 8m ago 110m 310M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:13:06.834 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (101m) 8m ago 110m 302M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:13:06.841 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (101m) 8m ago 109m 241M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:13:06.841 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (100m) 8m ago 109m 290M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:13:06.842 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (100m) 5m ago 109m 262M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:13:06.842 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (99m) 5m ago 108m 317M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:13:06.843 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (99m) 5m ago 108m 282M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:13:06.843 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (98m) 5m ago 108m 327M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:13:06.843 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (103m) 5m ago 107m 114M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:13:06.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:13:06 smithi083 ceph-mon[77242]: pgmap v3147: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:06.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:13:06 smithi083 ceph-mon[79930]: pgmap v3147: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:07.182 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:13:07.183 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:13:07.183 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:13:07.183 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:13:07.183 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:13:07.184 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:13:07.184 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:13:07.184 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:13:07.184 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:13:07.185 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:13:07.185 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:13:07.185 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:13:07.185 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:13:07.185 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:13:07.186 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:13:07.187 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:13:07.187 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:13:07.187 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:13:07.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:13:07 smithi153 ceph-mon[70044]: from='client.47731 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:13:07.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:13:07 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1130445643' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:13:07.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:13:07 smithi083 ceph-mon[79930]: from='client.47731 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:13:07.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:13:07 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1130445643' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:13:07.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:13:07 smithi083 ceph-mon[77242]: from='client.47731 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:13:07.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:13:07 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1130445643' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:13:08.815 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:13:08 smithi153 ceph-mon[70044]: from='client.47737 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:13:08.815 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:13:08 smithi153 ceph-mon[70044]: pgmap v3148: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:08.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:13:08 smithi083 ceph-mon[79930]: from='client.47737 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:13:08.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:13:08 smithi083 ceph-mon[79930]: pgmap v3148: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:08.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:13:08 smithi083 ceph-mon[77242]: from='client.47737 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:13:08.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:13:08 smithi083 ceph-mon[77242]: pgmap v3148: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:10.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:13:10 smithi083 ceph-mon[79930]: pgmap v3149: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:10.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:13:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:13:10.505Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:13:10.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:13:10 smithi083 ceph-mon[77242]: pgmap v3149: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:11.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:13:10 smithi153 ceph-mon[70044]: pgmap v3149: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:13:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:13:11] "GET /metrics HTTP/1.1" 200 225177 "" "Prometheus/2.33.4" 2022-09-29T07:13:12.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:13:12 smithi153 ceph-mon[70044]: pgmap v3150: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:12.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:13:12 smithi083 ceph-mon[77242]: pgmap v3150: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:12.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:13:12 smithi083 ceph-mon[79930]: pgmap v3150: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:12.818 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:13:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:13:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:13:13.315 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:13:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:13:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:13:14.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:13:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:13:13.911Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:13:14.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:13:14 smithi153 ceph-mon[70044]: pgmap v3151: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:14.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:13:14 smithi083 ceph-mon[77242]: pgmap v3151: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:14.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:13:14 smithi083 ceph-mon[79930]: pgmap v3151: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:14.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:13:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:13:14.609Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:13:16.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:13:16 smithi153 ceph-mon[70044]: pgmap v3152: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:16.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:13:16 smithi083 ceph-mon[79930]: pgmap v3152: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:16.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:13:16 smithi083 ceph-mon[77242]: pgmap v3152: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:17.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:13:17 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:13:17.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:13:17 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:13:17.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:13:17 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:13:17.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:13:17 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:13:17.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:13:17 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:13:17.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:13:17 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:13:18.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:13:17 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:13:18.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:13:17 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:13:18.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:13:17 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:13:18.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:13:18 smithi083 ceph-mon[79930]: pgmap v3153: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:18.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:13:18 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:13:18.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:13:18 smithi083 ceph-mon[77242]: pgmap v3153: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:18.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:13:18 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:13:19.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:13:18 smithi153 ceph-mon[70044]: pgmap v3153: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:19.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:13:18 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:13:20.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:13:20 smithi083 ceph-mon[79930]: pgmap v3154: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:20.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:13:20 smithi083 ceph-mon[77242]: pgmap v3154: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:20.966 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:13:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:13:20.506Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:13:21.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:13:20 smithi153 ceph-mon[70044]: pgmap v3154: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:13:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:13:21] "GET /metrics HTTP/1.1" 200 225177 "" "Prometheus/2.33.4" 2022-09-29T07:13:22.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:13:22 smithi153 ceph-mon[70044]: pgmap v3155: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:22.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:13:22 smithi083 ceph-mon[79930]: pgmap v3155: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:22.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:13:22 smithi083 ceph-mon[77242]: pgmap v3155: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:13:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:13:22.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:13:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:13:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:13:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:13:24.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:13:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:13:23.912Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:13:24.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:13:24 smithi153 ceph-mon[70044]: pgmap v3156: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:24.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:13:24 smithi083 ceph-mon[79930]: pgmap v3156: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:24.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:13:24 smithi083 ceph-mon[77242]: pgmap v3156: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:24.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:13:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:13:24.610Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:13:26.822 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:13:26 smithi153 ceph-mon[70044]: pgmap v3157: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:26.969 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:13:26 smithi083 ceph-mon[79930]: pgmap v3157: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:26.970 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:13:26 smithi083 ceph-mon[77242]: pgmap v3157: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:28.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:13:28 smithi083 ceph-mon[79930]: pgmap v3158: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:28.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:13:28 smithi083 ceph-mon[77242]: pgmap v3158: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:29.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:13:28 smithi153 ceph-mon[70044]: pgmap v3158: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:30.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:13:30 smithi083 ceph-mon[79930]: pgmap v3159: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:30.968 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:13:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:13:30.506Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:13:30.969 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:13:30 smithi083 ceph-mon[77242]: pgmap v3159: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:31.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:13:30 smithi153 ceph-mon[70044]: pgmap v3159: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:13:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:13:31] "GET /metrics HTTP/1.1" 200 225174 "" "Prometheus/2.33.4" 2022-09-29T07:13:32.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:13:32 smithi153 ceph-mon[70044]: pgmap v3160: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:32.469 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:13:32 smithi083 ceph-mon[79930]: pgmap v3160: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:32.469 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:13:32 smithi083 ceph-mon[77242]: pgmap v3160: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:32.816 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:13:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:13:32.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:13:33.312 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:13:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:13:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:13:34.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:13:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:13:33.912Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:13:34.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:13:34 smithi153 ceph-mon[70044]: pgmap v3161: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:34.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:13:34 smithi083 ceph-mon[79930]: pgmap v3161: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:34.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:13:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:13:34.610Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:13:34.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:13:34 smithi083 ceph-mon[77242]: pgmap v3161: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:36.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:13:36 smithi083 ceph-mon[79930]: pgmap v3162: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:36.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:13:36 smithi083 ceph-mon[77242]: pgmap v3162: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:37.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:13:36 smithi153 ceph-mon[70044]: pgmap v3162: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:37.527 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:13:37.803 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:13:37.803 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (107m) 8m ago 107m 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:13:37.803 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (107m) 6m ago 107m 62.7M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:13:37.804 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (108m) 8m ago 108m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:13:37.804 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (108m) 6m ago 108m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:13:37.804 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (104m) 6m ago 111m 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:13:37.804 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (104m) 8m ago 112m 598M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:13:37.804 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (104m) 8m ago 112m 421M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:13:37.805 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (103m) 6m ago 111m 192M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:13:37.805 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (103m) 8m ago 111m 248M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:13:37.805 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (108m) 8m ago 108m 25.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:13:37.805 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (107m) 6m ago 107m 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:13:37.818 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (102m) 8m ago 110m 310M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:13:37.818 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (102m) 8m ago 110m 302M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:13:37.818 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (101m) 8m ago 110m 241M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:13:37.819 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (101m) 8m ago 110m 290M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:13:37.819 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (100m) 6m ago 109m 262M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:13:37.819 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (100m) 6m ago 109m 317M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:13:37.819 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (99m) 6m ago 109m 282M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:13:37.820 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (99m) 6m ago 108m 327M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:13:37.820 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (104m) 6m ago 108m 114M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:13:38.154 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:13:38.155 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:13:38.155 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:13:38.155 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:13:38.156 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:13:38.160 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:13:38.161 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:13:38.161 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:13:38.166 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:13:38.167 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:13:38.170 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:13:38.171 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:13:38.171 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:13:38.172 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:13:38.172 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:13:38.172 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:13:38.173 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:13:38.173 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:13:38.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:13:38 smithi083 ceph-mon[79930]: from='client.47749 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:13:38.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:13:38 smithi083 ceph-mon[79930]: pgmap v3163: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:38.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:13:38 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3881801616' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:13:38.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:13:38 smithi083 ceph-mon[77242]: from='client.47749 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:13:38.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:13:38 smithi083 ceph-mon[77242]: pgmap v3163: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:38.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:13:38 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3881801616' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:13:39.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:13:38 smithi153 ceph-mon[70044]: from='client.47749 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:13:39.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:13:38 smithi153 ceph-mon[70044]: pgmap v3163: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:39.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:13:38 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3881801616' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:13:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:13:39 smithi083 ceph-mon[77242]: from='client.37821 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:13:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:13:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:13:39.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:13:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:13:39.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:13:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:13:39.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:13:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:13:39.974 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:13:39 smithi083 ceph-mon[79930]: from='client.37821 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:13:39.974 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:13:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:13:39.975 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:13:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:13:39.975 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:13:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:13:39.975 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:13:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:13:40.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:13:39 smithi153 ceph-mon[70044]: from='client.37821 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:13:40.063 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:13:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:13:40.063 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:13:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:13:40.064 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:13:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:13:40.064 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:13:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:13:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:13:40 smithi083 ceph-mon[79930]: pgmap v3164: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:13:40 smithi083 ceph-mon[77242]: pgmap v3164: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:40.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:13:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:13:40.507Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:13:41.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:13:40 smithi153 ceph-mon[70044]: pgmap v3164: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:41.970 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:13:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:13:41] "GET /metrics HTTP/1.1" 200 225173 "" "Prometheus/2.33.4" 2022-09-29T07:13:42.318 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:13:42 smithi153 ceph-mon[70044]: pgmap v3165: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:13:42 smithi083 ceph-mon[79930]: pgmap v3165: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:42.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:13:42 smithi083 ceph-mon[77242]: pgmap v3165: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:13:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:13:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:13:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:13:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:13:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:13:44.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:13:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:13:43.913Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:13:44.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:13:44 smithi083 ceph-mon[79930]: pgmap v3166: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:44.964 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:13:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:13:44.611Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:13:44.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:13:44 smithi083 ceph-mon[77242]: pgmap v3166: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:45.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:13:44 smithi153 ceph-mon[70044]: pgmap v3166: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:46.970 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:13:46 smithi083 ceph-mon[79930]: pgmap v3167: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:46.975 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:13:46 smithi083 ceph-mon[77242]: pgmap v3167: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:47.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:13:46 smithi153 ceph-mon[70044]: pgmap v3167: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:48.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:13:48 smithi083 ceph-mon[79930]: pgmap v3168: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:48.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:13:48 smithi083 ceph-mon[77242]: pgmap v3168: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:49.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:13:48 smithi153 ceph-mon[70044]: pgmap v3168: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:50.975 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:13:50 smithi083 ceph-mon[79930]: pgmap v3169: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:50.975 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:13:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:13:50.508Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:13:50.976 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:13:50 smithi083 ceph-mon[77242]: pgmap v3169: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:51.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:13:50 smithi153 ceph-mon[70044]: pgmap v3169: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:51.967 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:13:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:13:51] "GET /metrics HTTP/1.1" 200 225173 "" "Prometheus/2.33.4" 2022-09-29T07:13:52.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:13:52 smithi153 ceph-mon[70044]: pgmap v3170: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:52.467 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:13:52 smithi083 ceph-mon[79930]: pgmap v3170: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:52.468 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:13:52 smithi083 ceph-mon[77242]: pgmap v3170: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:52.815 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:13:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:13:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:13:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:13:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:13:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:13:54.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:13:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:13:53.914Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:13:54.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:13:54 smithi083 ceph-mon[79930]: pgmap v3171: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:54.964 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:13:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:13:54.612Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:13:54.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:13:54 smithi083 ceph-mon[77242]: pgmap v3171: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:55.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:13:54 smithi153 ceph-mon[70044]: pgmap v3171: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:56.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:13:56 smithi083 ceph-mon[79930]: pgmap v3172: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:56.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:13:56 smithi083 ceph-mon[77242]: pgmap v3172: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:57.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:13:56 smithi153 ceph-mon[70044]: pgmap v3172: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:58.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:13:58 smithi083 ceph-mon[79930]: pgmap v3173: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:58.968 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:13:58 smithi083 ceph-mon[77242]: pgmap v3173: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:13:59.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:13:58 smithi153 ceph-mon[70044]: pgmap v3173: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:00.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:14:00 smithi083 ceph-mon[79930]: pgmap v3174: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:00.965 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:14:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:14:00.508Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:14:00.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:14:00 smithi083 ceph-mon[77242]: pgmap v3174: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:01.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:14:00 smithi153 ceph-mon[70044]: pgmap v3174: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:14:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:14:01] "GET /metrics HTTP/1.1" 200 225177 "" "Prometheus/2.33.4" 2022-09-29T07:14:02.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:14:02 smithi153 ceph-mon[70044]: pgmap v3175: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:02.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:14:02 smithi083 ceph-mon[79930]: pgmap v3175: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:02.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:14:02 smithi083 ceph-mon[77242]: pgmap v3175: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:02.819 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:14:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:14:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:14:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:14:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:14:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:14:04.220 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:14:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:14:03.915Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:14:04.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:14:04 smithi083 ceph-mon[79930]: pgmap v3176: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:04.964 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:14:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:14:04.613Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:14:04.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:14:04 smithi083 ceph-mon[77242]: pgmap v3176: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:05.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:14:04 smithi153 ceph-mon[70044]: pgmap v3176: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:06.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:14:06 smithi083 ceph-mon[77242]: pgmap v3177: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:06.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:14:06 smithi083 ceph-mon[79930]: pgmap v3177: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:07.075 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:14:06 smithi153 ceph-mon[70044]: pgmap v3177: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:08.479 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:14:08.771 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:14:08.771 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (107m) 9m ago 108m 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:14:08.771 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (108m) 6m ago 108m 62.7M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:14:08.771 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (109m) 9m ago 109m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:14:08.772 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (109m) 6m ago 109m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:14:08.772 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (104m) 6m ago 111m 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:14:08.772 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (105m) 9m ago 113m 598M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:14:08.772 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (104m) 9m ago 113m 421M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:14:08.773 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (104m) 6m ago 111m 192M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:14:08.773 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (104m) 9m ago 112m 248M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:14:08.773 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (108m) 9m ago 108m 25.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:14:08.773 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (108m) 6m ago 108m 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:14:08.774 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (103m) 9m ago 111m 310M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:14:08.774 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (102m) 9m ago 111m 302M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:14:08.774 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (102m) 9m ago 110m 241M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:14:08.775 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (101m) 9m ago 110m 290M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:14:08.775 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (101m) 6m ago 110m 262M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:14:08.776 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (100m) 6m ago 110m 317M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:14:08.776 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (100m) 6m ago 109m 282M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:14:08.776 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (99m) 6m ago 109m 327M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:14:08.776 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (104m) 6m ago 108m 114M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:14:08.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:14:08 smithi083 ceph-mon[79930]: pgmap v3178: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:08.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:14:08 smithi083 ceph-mon[77242]: pgmap v3178: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:09.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:14:08 smithi153 ceph-mon[70044]: pgmap v3178: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:09.129 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:14:09.129 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:14:09.129 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:14:09.129 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:14:09.130 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:14:09.130 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:14:09.130 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:14:09.130 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:14:09.131 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:14:09.131 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:14:09.131 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:14:09.131 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:14:09.134 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:14:09.134 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:14:09.134 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:14:09.135 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:14:09.135 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:14:09.135 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:14:09.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:14:09 smithi083 ceph-mon[79930]: from='client.47767 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:14:09.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:14:09 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3234135010' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:14:09.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:14:09 smithi083 ceph-mon[77242]: from='client.47767 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:14:09.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:14:09 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3234135010' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:14:10.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:14:09 smithi153 ceph-mon[70044]: from='client.47767 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:14:10.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:14:09 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3234135010' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:14:10.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:14:10 smithi083 ceph-mon[79930]: from='client.37839 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:14:10.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:14:10 smithi083 ceph-mon[79930]: pgmap v3179: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:10.964 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:14:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:14:10.509Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:14:10.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:14:10 smithi083 ceph-mon[77242]: from='client.37839 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:14:10.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:14:10 smithi083 ceph-mon[77242]: pgmap v3179: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:11.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:14:10 smithi153 ceph-mon[70044]: from='client.37839 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:14:11.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:14:10 smithi153 ceph-mon[70044]: pgmap v3179: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:14:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:14:11] "GET /metrics HTTP/1.1" 200 225175 "" "Prometheus/2.33.4" 2022-09-29T07:14:12.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:14:12 smithi153 ceph-mon[70044]: pgmap v3180: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:12.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:14:12 smithi083 ceph-mon[79930]: pgmap v3180: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:12.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:14:12 smithi083 ceph-mon[77242]: pgmap v3180: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:12.811 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:14:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:14:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:14:13.313 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:14:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:14:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:14:14.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:14:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:14:13.915Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:14:14.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:14:14 smithi083 ceph-mon[77242]: pgmap v3181: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:14.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:14:14 smithi083 ceph-mon[79930]: pgmap v3181: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:14.965 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:14:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:14:14.614Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:14:15.063 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:14:14 smithi153 ceph-mon[70044]: pgmap v3181: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:16.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:14:16 smithi083 ceph-mon[77242]: pgmap v3182: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:16.968 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:14:16 smithi083 ceph-mon[79930]: pgmap v3182: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:17.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:14:16 smithi153 ceph-mon[70044]: pgmap v3182: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:18.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:14:18 smithi083 ceph-mon[79930]: pgmap v3183: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:18.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:14:18 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:14:18.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:14:18 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:14:18.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:14:18 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:14:18.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:14:18 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:14:18.968 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:14:18 smithi083 ceph-mon[77242]: pgmap v3183: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:18.970 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:14:18 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:14:18.971 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:14:18 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:14:18.971 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:14:18 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:14:18.971 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:14:18 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:14:19.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:14:18 smithi153 ceph-mon[70044]: pgmap v3183: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:19.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:14:18 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:14:19.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:14:18 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:14:19.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:14:18 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:14:19.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:14:18 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:14:20.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:14:20 smithi083 ceph-mon[79930]: pgmap v3184: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:20.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:14:20 smithi083 ceph-mon[77242]: pgmap v3184: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:20.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:14:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:14:20.510Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:14:21.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:14:20 smithi153 ceph-mon[70044]: pgmap v3184: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:21.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:14:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:14:21] "GET /metrics HTTP/1.1" 200 225175 "" "Prometheus/2.33.4" 2022-09-29T07:14:22.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:14:22 smithi153 ceph-mon[70044]: pgmap v3185: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:22.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:14:22 smithi083 ceph-mon[79930]: pgmap v3185: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:22.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:14:22 smithi083 ceph-mon[77242]: pgmap v3185: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:22.812 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:14:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:14:22.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:14:23.311 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:14:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:14:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:14:24.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:14:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:14:23.916Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:14:24.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:14:24 smithi083 ceph-mon[77242]: pgmap v3186: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:24.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:14:24 smithi083 ceph-mon[79930]: pgmap v3186: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:24.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:14:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:14:24.614Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:14:25.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:14:24 smithi153 ceph-mon[70044]: pgmap v3186: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:26.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:14:26 smithi083 ceph-mon[79930]: pgmap v3187: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:26.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:14:26 smithi083 ceph-mon[77242]: pgmap v3187: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:27.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:14:26 smithi153 ceph-mon[70044]: pgmap v3187: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:28.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:14:28 smithi083 ceph-mon[79930]: pgmap v3188: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:28.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:14:28 smithi083 ceph-mon[77242]: pgmap v3188: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:29.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:14:28 smithi153 ceph-mon[70044]: pgmap v3188: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:30.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:14:30 smithi083 ceph-mon[79930]: pgmap v3189: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:30.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:14:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:14:30.511Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:14:30.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:14:30 smithi083 ceph-mon[77242]: pgmap v3189: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:31.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:14:30 smithi153 ceph-mon[70044]: pgmap v3189: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:14:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:14:31] "GET /metrics HTTP/1.1" 200 225174 "" "Prometheus/2.33.4" 2022-09-29T07:14:32.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:14:32 smithi153 ceph-mon[70044]: pgmap v3190: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:32.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:14:32 smithi083 ceph-mon[79930]: pgmap v3190: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:32.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:14:32 smithi083 ceph-mon[77242]: pgmap v3190: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:14:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:14:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:14:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:14:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:14:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:14:34.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:14:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:14:33.918Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:14:34.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:14:34 smithi083 ceph-mon[79930]: pgmap v3191: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:34.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:14:34 smithi083 ceph-mon[77242]: pgmap v3191: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:34.966 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:14:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:14:34.615Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:14:35.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:14:34 smithi153 ceph-mon[70044]: pgmap v3191: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:36.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:14:36 smithi083 ceph-mon[77242]: pgmap v3192: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:36.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:14:36 smithi083 ceph-mon[79930]: pgmap v3192: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:37.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:14:36 smithi153 ceph-mon[70044]: pgmap v3192: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:38.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:14:38 smithi083 ceph-mon[79930]: pgmap v3193: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:38.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:14:38 smithi083 ceph-mon[77242]: pgmap v3193: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:39.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:14:38 smithi153 ceph-mon[70044]: pgmap v3193: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:39.457 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:14:39.745 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:14:39.746 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (108m) 9m ago 108m 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:14:39.747 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (108m) 7m ago 108m 62.7M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:14:39.747 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (109m) 9m ago 109m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:14:39.748 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (109m) 7m ago 109m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:14:39.748 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (105m) 7m ago 112m 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:14:39.748 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (105m) 9m ago 113m 598M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:14:39.749 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (105m) 9m ago 113m 421M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:14:39.749 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (104m) 7m ago 112m 192M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:14:39.750 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (104m) 9m ago 112m 248M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:14:39.750 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (109m) 9m ago 109m 25.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:14:39.751 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (109m) 7m ago 109m 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:14:39.751 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (103m) 9m ago 112m 310M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:14:39.752 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (103m) 9m ago 111m 302M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:14:39.752 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (102m) 9m ago 111m 241M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:14:39.753 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (102m) 9m ago 111m 290M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:14:39.753 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (101m) 7m ago 110m 262M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:14:39.753 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (101m) 7m ago 110m 317M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:14:39.754 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (100m) 7m ago 110m 282M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:14:39.754 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (100m) 7m ago 109m 327M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:14:39.755 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (105m) 7m ago 109m 114M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:14:39.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:14:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:14:39.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:14:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:14:39.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:14:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:14:39.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:14:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:14:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:14:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:14:39.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:14:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:14:39.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:14:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:14:39.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:14:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:14:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:14:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:14:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:14:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:14:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:14:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:14:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:14:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:14:40.105 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:14:40.105 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:14:40.105 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:14:40.105 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:14:40.106 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:14:40.106 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:14:40.107 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:14:40.107 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:14:40.107 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:14:40.108 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:14:40.108 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:14:40.109 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:14:40.110 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:14:40.110 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:14:40.111 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:14:40.111 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:14:40.112 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:14:40.112 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:14:40.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:14:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:14:40.511Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:14:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:14:40 smithi083 ceph-mon[79930]: from='client.37851 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:14:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:14:40 smithi083 ceph-mon[79930]: pgmap v3194: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:40.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:14:40 smithi083 ceph-mon[79930]: from='client.47791 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:14:40.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:14:40 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3069341161' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:14:40.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:14:40 smithi083 ceph-mon[77242]: from='client.37851 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:14:40.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:14:40 smithi083 ceph-mon[77242]: pgmap v3194: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:40.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:14:40 smithi083 ceph-mon[77242]: from='client.47791 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:14:40.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:14:40 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3069341161' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:14:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:14:40 smithi153 ceph-mon[70044]: from='client.37851 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:14:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:14:40 smithi153 ceph-mon[70044]: pgmap v3194: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:14:40 smithi153 ceph-mon[70044]: from='client.47791 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:14:41.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:14:40 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3069341161' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:14:41.964 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:14:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:14:41] "GET /metrics HTTP/1.1" 200 225174 "" "Prometheus/2.33.4" 2022-09-29T07:14:42.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:14:42 smithi153 ceph-mon[70044]: pgmap v3195: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:42.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:14:42 smithi083 ceph-mon[79930]: pgmap v3195: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:42.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:14:42 smithi083 ceph-mon[77242]: pgmap v3195: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:14:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:14:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:14:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:14:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:14:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:14:44.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:14:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:14:43.918Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:14:44.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:14:44 smithi083 ceph-mon[79930]: pgmap v3196: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:44.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:14:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:14:44.616Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:14:44.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:14:44 smithi083 ceph-mon[77242]: pgmap v3196: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:45.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:14:44 smithi153 ceph-mon[70044]: pgmap v3196: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:46.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:14:46 smithi083 ceph-mon[79930]: pgmap v3197: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:46.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:14:46 smithi083 ceph-mon[77242]: pgmap v3197: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:47.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:14:46 smithi153 ceph-mon[70044]: pgmap v3197: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:48.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:14:48 smithi083 ceph-mon[79930]: pgmap v3198: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:48.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:14:48 smithi083 ceph-mon[77242]: pgmap v3198: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:49.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:14:48 smithi153 ceph-mon[70044]: pgmap v3198: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:50.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:14:50 smithi083 ceph-mon[79930]: pgmap v3199: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:50.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:14:50 smithi083 ceph-mon[77242]: pgmap v3199: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:50.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:14:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:14:50.513Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:14:51.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:14:50 smithi153 ceph-mon[70044]: pgmap v3199: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:14:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:14:51] "GET /metrics HTTP/1.1" 200 225174 "" "Prometheus/2.33.4" 2022-09-29T07:14:52.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:14:52 smithi153 ceph-mon[70044]: pgmap v3200: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:52.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:14:52 smithi083 ceph-mon[79930]: pgmap v3200: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:52.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:14:52 smithi083 ceph-mon[77242]: pgmap v3200: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:52.810 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:14:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:14:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:14:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:14:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:14:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:14:54.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:14:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:14:53.918Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:14:54.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:14:54 smithi083 ceph-mon[79930]: pgmap v3201: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:54.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:14:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:14:54.616Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:14:54.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:14:54 smithi083 ceph-mon[77242]: pgmap v3201: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:55.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:14:54 smithi153 ceph-mon[70044]: pgmap v3201: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:56.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:14:56 smithi083 ceph-mon[79930]: pgmap v3202: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:56.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:14:56 smithi083 ceph-mon[77242]: pgmap v3202: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:57.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:14:56 smithi153 ceph-mon[70044]: pgmap v3202: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:58.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:14:58 smithi083 ceph-mon[79930]: pgmap v3203: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:58.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:14:58 smithi083 ceph-mon[77242]: pgmap v3203: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:14:59.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:14:58 smithi153 ceph-mon[70044]: pgmap v3203: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:00.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:15:00 smithi083 ceph-mon[79930]: pgmap v3204: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:00.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:15:00 smithi083 ceph-mon[77242]: pgmap v3204: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:00.964 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:15:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:15:00.513Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:15:01.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:15:00 smithi153 ceph-mon[70044]: pgmap v3204: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:15:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:15:01] "GET /metrics HTTP/1.1" 200 225178 "" "Prometheus/2.33.4" 2022-09-29T07:15:02.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:15:02 smithi153 ceph-mon[70044]: pgmap v3205: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:02.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:15:02 smithi083 ceph-mon[79930]: pgmap v3205: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:02.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:15:02 smithi083 ceph-mon[77242]: pgmap v3205: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:15:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:15:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:15:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:15:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:15:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:15:04.217 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:15:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:15:03.919Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:15:04.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:15:04 smithi083 ceph-mon[79930]: pgmap v3206: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:04.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:15:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:15:04.617Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:15:04.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:15:04 smithi083 ceph-mon[77242]: pgmap v3206: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:05.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:15:04 smithi153 ceph-mon[70044]: pgmap v3206: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:06.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:15:06 smithi083 ceph-mon[79930]: pgmap v3207: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:06.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:15:06 smithi083 ceph-mon[77242]: pgmap v3207: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:07.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:15:06 smithi153 ceph-mon[70044]: pgmap v3207: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:08.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:15:08 smithi083 ceph-mon[79930]: pgmap v3208: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:08.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:15:08 smithi083 ceph-mon[77242]: pgmap v3208: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:09.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:15:08 smithi153 ceph-mon[70044]: pgmap v3208: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:10.430 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:15:10.730 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:15:10.731 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (108m) 10m ago 109m 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:15:10.731 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (109m) 8m ago 109m 62.7M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:15:10.732 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (110m) 10m ago 110m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:15:10.732 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (110m) 8m ago 110m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:15:10.732 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (105m) 8m ago 112m 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:15:10.733 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (106m) 10m ago 114m 598M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:15:10.733 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (105m) 10m ago 114m 421M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:15:10.734 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (105m) 8m ago 113m 192M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:15:10.734 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (105m) 10m ago 113m 248M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:15:10.735 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (109m) 10m ago 109m 25.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:15:10.735 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (109m) 8m ago 109m 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:15:10.736 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (104m) 10m ago 112m 310M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:15:10.736 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (103m) 10m ago 112m 302M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:15:10.737 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (103m) 10m ago 111m 241M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:15:10.737 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (102m) 10m ago 111m 290M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:15:10.738 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (102m) 8m ago 111m 262M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:15:10.738 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (101m) 8m ago 111m 317M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:15:10.739 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (101m) 8m ago 110m 282M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:15:10.740 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (100m) 8m ago 110m 327M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:15:10.740 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (105m) 8m ago 109m 114M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:15:10.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:15:10 smithi083 ceph-mon[79930]: pgmap v3209: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:10.964 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:15:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:15:10.513Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:15:10.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:15:10 smithi083 ceph-mon[77242]: pgmap v3209: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:11.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:15:10 smithi153 ceph-mon[70044]: pgmap v3209: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:11.082 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:15:11.083 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:15:11.083 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:15:11.083 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:15:11.083 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:15:11.083 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:15:11.084 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:15:11.084 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:15:11.084 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:15:11.084 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:15:11.085 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:15:11.085 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:15:11.085 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:15:11.085 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:15:11.086 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:15:11.086 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:15:11.086 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:15:11.086 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:15:11.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:15:11 smithi083 ceph-mon[77242]: from='client.37869 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:15:11.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:15:11 smithi083 ceph-mon[77242]: from='client.47809 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:15:11.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:15:11 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2178414540' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:15:11.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:15:11 smithi083 ceph-mon[79930]: from='client.37869 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:15:11.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:15:11 smithi083 ceph-mon[79930]: from='client.47809 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:15:11.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:15:11 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2178414540' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:15:11.965 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:15:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:15:11] "GET /metrics HTTP/1.1" 200 225178 "" "Prometheus/2.33.4" 2022-09-29T07:15:12.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:15:11 smithi153 ceph-mon[70044]: from='client.37869 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:15:12.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:15:11 smithi153 ceph-mon[70044]: from='client.47809 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:15:12.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:15:11 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2178414540' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:15:12.596 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:15:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:15:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:15:12.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:15:12 smithi083 ceph-mon[77242]: pgmap v3210: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:12.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:15:12 smithi083 ceph-mon[79930]: pgmap v3210: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:13.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:15:12 smithi153 ceph-mon[70044]: pgmap v3210: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:15:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:15:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:15:14.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:15:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:15:13.919Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:15:14.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:15:14 smithi083 ceph-mon[77242]: pgmap v3211: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:14.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:15:14 smithi083 ceph-mon[79930]: pgmap v3211: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:14.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:15:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:15:14.617Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:15:15.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:15:14 smithi153 ceph-mon[70044]: pgmap v3211: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:16.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:15:16 smithi083 ceph-mon[79930]: pgmap v3212: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:16.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:15:16 smithi083 ceph-mon[77242]: pgmap v3212: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:17.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:15:16 smithi153 ceph-mon[70044]: pgmap v3212: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:18.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:15:18 smithi083 ceph-mon[79930]: pgmap v3213: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:18.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:15:18 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:15:18.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:15:18 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:15:18.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:15:18 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:15:18.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:15:18 smithi083 ceph-mon[77242]: pgmap v3213: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:18.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:15:18 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:15:18.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:15:18 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:15:18.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:15:18 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:15:19.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:15:18 smithi153 ceph-mon[70044]: pgmap v3213: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:19.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:15:18 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:15:19.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:15:18 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:15:19.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:15:18 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:15:20.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:15:19 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:15:20.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:15:19 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:15:20.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:15:19 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:15:20.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:15:19 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:15:20.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:15:19 smithi083 ceph-mon[79930]: pgmap v3214: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:20.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:15:19 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:15:20.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:15:19 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:15:20.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:15:19 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:15:20.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:15:19 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:15:20.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:15:19 smithi083 ceph-mon[77242]: pgmap v3214: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:15:19 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:15:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:15:19 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:15:20.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:15:19 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:15:20.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:15:19 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:15:20.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:15:19 smithi153 ceph-mon[70044]: pgmap v3214: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:20.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:15:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:15:20.514Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:15:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:15:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:15:21] "GET /metrics HTTP/1.1" 200 225178 "" "Prometheus/2.33.4" 2022-09-29T07:15:22.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:15:22 smithi153 ceph-mon[70044]: pgmap v3215: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:22.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:15:22 smithi083 ceph-mon[79930]: pgmap v3215: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:22.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:15:22 smithi083 ceph-mon[77242]: pgmap v3215: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:15:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:15:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:15:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:15:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:15:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:15:24.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:15:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:15:23.920Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:15:24.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:15:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:15:24.618Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:15:24.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:15:24 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:15:24.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:15:24 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:15:24.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:15:24 smithi083 ceph-mon[79930]: pgmap v3216: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:24.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:15:24 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:15:24.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:15:24 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:15:24.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:15:24 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:15:24.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:15:24 smithi083 ceph-mon[77242]: pgmap v3216: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:24.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:15:24 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:15:24.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:15:24 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:15:24.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:15:24 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:15:24.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:15:24 smithi153 ceph-mon[70044]: pgmap v3216: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:24.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:15:24 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:15:26.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:15:26 smithi083 ceph-mon[79930]: pgmap v3217: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:26.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:15:26 smithi083 ceph-mon[77242]: pgmap v3217: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:27.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:15:26 smithi153 ceph-mon[70044]: pgmap v3217: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:28.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:15:28 smithi083 ceph-mon[79930]: pgmap v3218: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:28.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:15:28 smithi083 ceph-mon[77242]: pgmap v3218: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:29.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:15:28 smithi153 ceph-mon[70044]: pgmap v3218: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:30.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:15:30 smithi083 ceph-mon[79930]: pgmap v3219: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:30.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:15:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:15:30.515Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:15:30.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:15:30 smithi083 ceph-mon[77242]: pgmap v3219: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:31.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:15:30 smithi153 ceph-mon[70044]: pgmap v3219: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:15:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:15:31] "GET /metrics HTTP/1.1" 200 225174 "" "Prometheus/2.33.4" 2022-09-29T07:15:32.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:15:32 smithi153 ceph-mon[70044]: pgmap v3220: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:32.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:15:32 smithi083 ceph-mon[79930]: pgmap v3220: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:32.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:15:32 smithi083 ceph-mon[77242]: pgmap v3220: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:15:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:15:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:15:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:15:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:15:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:15:34.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:15:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:15:33.921Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:15:34.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:15:34 smithi083 ceph-mon[79930]: pgmap v3221: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:34.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:15:34 smithi083 ceph-mon[77242]: pgmap v3221: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:34.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:15:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:15:34.619Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:15:35.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:15:34 smithi153 ceph-mon[70044]: pgmap v3221: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:36.968 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:15:36 smithi083 ceph-mon[77242]: pgmap v3222: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:36.969 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:15:36 smithi083 ceph-mon[79930]: pgmap v3222: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:37.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:15:36 smithi153 ceph-mon[70044]: pgmap v3222: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:38.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:15:38 smithi083 ceph-mon[77242]: pgmap v3223: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:38.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:15:38 smithi083 ceph-mon[79930]: pgmap v3223: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:39.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:15:38 smithi153 ceph-mon[70044]: pgmap v3223: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:39.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:15:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:15:39.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:15:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:15:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:15:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:15:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:15:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:15:39.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:15:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:15:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:15:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:15:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:15:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:15:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:15:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:15:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:15:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:15:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:15:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:15:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:15:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:15:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:15:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:15:40.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:15:40 smithi083 ceph-mon[77242]: pgmap v3224: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:15:40 smithi083 ceph-mon[79930]: pgmap v3224: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:40.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:15:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:15:40.516Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:15:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:15:40 smithi153 ceph-mon[70044]: pgmap v3224: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:41.403 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:15:41.692 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:15:41.692 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (109m) 18s ago 109m 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:15:41.693 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (109m) 8m ago 109m 62.7M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:15:41.693 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (110m) 18s ago 110m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:15:41.693 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (110m) 8m ago 110m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:15:41.693 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (106m) 8m ago 113m 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:15:41.694 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (106m) 18s ago 114m 608M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:15:41.694 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (106m) 18s ago 114m 459M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:15:41.694 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (105m) 8m ago 113m 192M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:15:41.694 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (105m) 18s ago 113m 270M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:15:41.694 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (110m) 18s ago 110m 25.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:15:41.695 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (110m) 8m ago 110m 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:15:41.695 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (104m) 18s ago 113m 335M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:15:41.695 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (104m) 18s ago 112m 327M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:15:41.695 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (104m) 18s ago 112m 261M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:15:41.696 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (103m) 18s ago 112m 315M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:15:41.696 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (102m) 8m ago 111m 262M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:15:41.696 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (102m) 8m ago 111m 317M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:15:41.696 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (101m) 8m ago 111m 282M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:15:41.696 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (101m) 8m ago 111m 327M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:15:41.697 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (106m) 8m ago 110m 114M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:15:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:15:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:15:41] "GET /metrics HTTP/1.1" 200 225174 "" "Prometheus/2.33.4" 2022-09-29T07:15:42.047 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:15:42.048 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:15:42.048 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:15:42.048 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:15:42.048 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:15:42.049 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:15:42.049 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:15:42.049 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:15:42.049 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:15:42.050 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:15:42.050 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:15:42.050 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:15:42.051 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:15:42.051 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:15:42.051 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:15:42.051 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:15:42.051 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:15:42.052 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:15:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:15:42 smithi153 ceph-mon[70044]: from='client.47821 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:15:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:15:42 smithi153 ceph-mon[70044]: pgmap v3225: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:15:42 smithi153 ceph-mon[70044]: from='client.37893 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:15:42.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:15:42 smithi083 ceph-mon[79930]: from='client.47821 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:15:42.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:15:42 smithi083 ceph-mon[79930]: pgmap v3225: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:15:42 smithi083 ceph-mon[79930]: from='client.37893 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:15:42.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:15:42 smithi083 ceph-mon[77242]: from='client.47821 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:15:42.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:15:42 smithi083 ceph-mon[77242]: pgmap v3225: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:42.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:15:42 smithi083 ceph-mon[77242]: from='client.37893 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:15:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:15:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:15:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:15:43.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:15:43 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1732028761' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:15:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:15:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:15:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:15:43.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:15:43 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1732028761' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:15:43.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:15:43 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1732028761' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:15:44.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:15:44 smithi083 ceph-mon[79930]: pgmap v3226: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:44.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:15:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:15:43.921Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:15:44.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:15:44 smithi083 ceph-mon[77242]: pgmap v3226: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:44.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:15:44 smithi153 ceph-mon[70044]: pgmap v3226: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:44.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:15:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:15:44.620Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:15:46.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:15:46 smithi083 ceph-mon[79930]: pgmap v3227: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:46.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:15:46 smithi083 ceph-mon[77242]: pgmap v3227: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:47.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:15:46 smithi153 ceph-mon[70044]: pgmap v3227: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:48.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:15:48 smithi083 ceph-mon[79930]: pgmap v3228: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:48.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:15:48 smithi083 ceph-mon[77242]: pgmap v3228: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:49.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:15:48 smithi153 ceph-mon[70044]: pgmap v3228: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:50.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:15:50 smithi083 ceph-mon[79930]: pgmap v3229: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:50.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:15:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:15:50.517Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:15:50.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:15:50 smithi083 ceph-mon[77242]: pgmap v3229: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:51.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:15:50 smithi153 ceph-mon[70044]: pgmap v3229: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:15:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:15:51] "GET /metrics HTTP/1.1" 200 225174 "" "Prometheus/2.33.4" 2022-09-29T07:15:52.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:15:52 smithi153 ceph-mon[70044]: pgmap v3230: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:52.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:15:52 smithi083 ceph-mon[79930]: pgmap v3230: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:52.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:15:52 smithi083 ceph-mon[77242]: pgmap v3230: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:15:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:15:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:15:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:15:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:15:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:15:54.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:15:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:15:53.922Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:15:54.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:15:54 smithi083 ceph-mon[79930]: pgmap v3231: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:54.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:15:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:15:54.620Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:15:54.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:15:54 smithi083 ceph-mon[77242]: pgmap v3231: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:55.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:15:54 smithi153 ceph-mon[70044]: pgmap v3231: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:56.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:15:56 smithi083 ceph-mon[79930]: pgmap v3232: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:56.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:15:56 smithi083 ceph-mon[77242]: pgmap v3232: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:57.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:15:56 smithi153 ceph-mon[70044]: pgmap v3232: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:58.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:15:58 smithi083 ceph-mon[77242]: pgmap v3233: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:58.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:15:58 smithi083 ceph-mon[79930]: pgmap v3233: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:15:59.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:15:58 smithi153 ceph-mon[70044]: pgmap v3233: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:00.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:16:00 smithi083 ceph-mon[79930]: pgmap v3234: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:00.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:16:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:16:00.518Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:16:00.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:16:00 smithi083 ceph-mon[77242]: pgmap v3234: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:01.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:16:00 smithi153 ceph-mon[70044]: pgmap v3234: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:16:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:16:01] "GET /metrics HTTP/1.1" 200 225173 "" "Prometheus/2.33.4" 2022-09-29T07:16:02.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:16:02 smithi153 ceph-mon[70044]: pgmap v3235: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:02.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:16:02 smithi083 ceph-mon[79930]: pgmap v3235: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:02.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:16:02 smithi083 ceph-mon[77242]: pgmap v3235: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:16:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:16:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:16:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:16:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:16:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:16:04.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:16:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:16:03.923Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:16:04.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:16:04 smithi083 ceph-mon[77242]: pgmap v3236: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:04.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:16:04 smithi083 ceph-mon[79930]: pgmap v3236: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:04.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:16:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:16:04.620Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:16:05.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:16:04 smithi153 ceph-mon[70044]: pgmap v3236: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:06.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:16:06 smithi083 ceph-mon[79930]: pgmap v3237: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:06.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:16:06 smithi083 ceph-mon[77242]: pgmap v3237: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:07.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:16:06 smithi153 ceph-mon[70044]: pgmap v3237: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:08.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:16:08 smithi083 ceph-mon[79930]: pgmap v3238: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:08.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:16:08 smithi083 ceph-mon[77242]: pgmap v3238: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:09.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:16:08 smithi153 ceph-mon[70044]: pgmap v3238: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:10.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:16:10 smithi083 ceph-mon[79930]: pgmap v3239: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:10.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:16:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:16:10.518Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:16:10.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:16:10 smithi083 ceph-mon[77242]: pgmap v3239: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:11.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:16:10 smithi153 ceph-mon[70044]: pgmap v3239: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:16:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:16:11] "GET /metrics HTTP/1.1" 200 225174 "" "Prometheus/2.33.4" 2022-09-29T07:16:12.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:16:12 smithi153 ceph-mon[70044]: pgmap v3240: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:12.374 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:16:12.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:16:12 smithi083 ceph-mon[79930]: pgmap v3240: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:12.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:16:12 smithi083 ceph-mon[77242]: pgmap v3240: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:12.661 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:16:12.661 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (109m) 49s ago 110m 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:16:12.661 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (110m) 9m ago 110m 62.7M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:16:12.662 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (111m) 49s ago 111m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:16:12.662 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (111m) 9m ago 111m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:16:12.662 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (106m) 9m ago 113m 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:16:12.662 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (107m) 49s ago 115m 608M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:16:12.663 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (106m) 49s ago 115m 459M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:16:12.663 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (106m) 9m ago 114m 192M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:16:12.663 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (106m) 49s ago 114m 270M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:16:12.663 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (110m) 49s ago 110m 25.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:16:12.664 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (110m) 9m ago 110m 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:16:12.664 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (105m) 49s ago 113m 335M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:16:12.664 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (104m) 49s ago 113m 327M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:16:12.664 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (104m) 49s ago 112m 261M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:16:12.664 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (103m) 49s ago 112m 315M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:16:12.665 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (103m) 9m ago 112m 262M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:16:12.665 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (102m) 9m ago 112m 317M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:16:12.665 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (102m) 9m ago 111m 282M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:16:12.665 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (101m) 9m ago 111m 327M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:16:12.666 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (107m) 9m ago 110m 114M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:16:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:16:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:16:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:16:13.018 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:16:13.018 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:16:13.019 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:16:13.019 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:16:13.019 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:16:13.019 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:16:13.019 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:16:13.020 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:16:13.020 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:16:13.020 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:16:13.020 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:16:13.020 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:16:13.021 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:16:13.021 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:16:13.021 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:16:13.021 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:16:13.022 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:16:13.022 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:16:13.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:16:13 smithi153 ceph-mon[70044]: from='client.47839 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:16:13.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:16:13 smithi153 ceph-mon[70044]: from='client.37911 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:16:13.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:16:13 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/628049733' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:16:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:16:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:16:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:16:13.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:16:13 smithi083 ceph-mon[79930]: from='client.47839 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:16:13.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:16:13 smithi083 ceph-mon[79930]: from='client.37911 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:16:13.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:16:13 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/628049733' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:16:13.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:16:13 smithi083 ceph-mon[77242]: from='client.47839 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:16:13.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:16:13 smithi083 ceph-mon[77242]: from='client.37911 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:16:13.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:16:13 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/628049733' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:16:14.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:16:14 smithi083 ceph-mon[79930]: pgmap v3241: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:14.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:16:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:16:13.923Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:16:14.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:16:14 smithi083 ceph-mon[77242]: pgmap v3241: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:14.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:16:14 smithi153 ceph-mon[70044]: pgmap v3241: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:14.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:16:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:16:14.621Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:16:16.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:16:16 smithi083 ceph-mon[79930]: pgmap v3242: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:16.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:16:16 smithi083 ceph-mon[77242]: pgmap v3242: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:17.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:16:16 smithi153 ceph-mon[70044]: pgmap v3242: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:18.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:16:18 smithi083 ceph-mon[79930]: pgmap v3243: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:18.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:16:18 smithi083 ceph-mon[77242]: pgmap v3243: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:19.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:16:18 smithi153 ceph-mon[70044]: pgmap v3243: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:20.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:16:20 smithi083 ceph-mon[79930]: pgmap v3244: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:20.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:16:20 smithi083 ceph-mon[77242]: pgmap v3244: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:20.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:16:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:16:20.518Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:16:21.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:16:20 smithi153 ceph-mon[70044]: pgmap v3244: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:16:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:16:21] "GET /metrics HTTP/1.1" 200 225174 "" "Prometheus/2.33.4" 2022-09-29T07:16:22.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:16:22 smithi153 ceph-mon[70044]: pgmap v3245: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:22.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:16:22 smithi083 ceph-mon[79930]: pgmap v3245: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:22.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:16:22 smithi083 ceph-mon[77242]: pgmap v3245: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:16:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:16:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:16:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:16:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:16:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:16:24.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:16:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:16:23.924Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:16:24.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:16:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:16:24.629Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:16:24.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:16:24 smithi083 ceph-mon[77242]: pgmap v3246: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:24.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:16:24 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:16:24.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:16:24 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:16:24.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:16:24 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:16:24.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:16:24 smithi083 ceph-mon[79930]: pgmap v3246: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:24.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:16:24 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:16:24.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:16:24 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:16:24.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:16:24 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:16:25.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:16:24 smithi153 ceph-mon[70044]: pgmap v3246: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:25.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:16:24 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:16:25.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:16:24 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:16:25.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:16:24 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:16:25.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:16:25 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:16:25.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:16:25 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:16:26.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:16:25 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:16:26.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:16:26 smithi083 ceph-mon[77242]: pgmap v3247: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:26.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:16:26 smithi083 ceph-mon[79930]: pgmap v3247: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:27.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:16:26 smithi153 ceph-mon[70044]: pgmap v3247: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:28.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:16:28 smithi083 ceph-mon[79930]: pgmap v3248: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:28.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:16:28 smithi083 ceph-mon[77242]: pgmap v3248: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:29.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:16:28 smithi153 ceph-mon[70044]: pgmap v3248: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:30.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:16:30 smithi083 ceph-mon[79930]: pgmap v3249: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:30.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:16:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:16:30.519Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:16:30.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:16:30 smithi083 ceph-mon[77242]: pgmap v3249: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:31.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:16:30 smithi153 ceph-mon[70044]: pgmap v3249: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:16:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:16:31] "GET /metrics HTTP/1.1" 200 225173 "" "Prometheus/2.33.4" 2022-09-29T07:16:32.315 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:16:32 smithi153 ceph-mon[70044]: pgmap v3250: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:32.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:16:32 smithi083 ceph-mon[77242]: pgmap v3250: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:32.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:16:32 smithi083 ceph-mon[79930]: pgmap v3250: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:16:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:16:32.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:16:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:16:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:16:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:16:34.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:16:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:16:33.925Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:16:34.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:16:34 smithi083 ceph-mon[79930]: pgmap v3251: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:34.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:16:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:16:34.622Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:16:34.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:16:34 smithi083 ceph-mon[77242]: pgmap v3251: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:35.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:16:34 smithi153 ceph-mon[70044]: pgmap v3251: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:36.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:16:36 smithi083 ceph-mon[77242]: pgmap v3252: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:36.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:16:36 smithi083 ceph-mon[79930]: pgmap v3252: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:37.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:16:36 smithi153 ceph-mon[70044]: pgmap v3252: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:38.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:16:38 smithi083 ceph-mon[79930]: pgmap v3253: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:38.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:16:38 smithi083 ceph-mon[77242]: pgmap v3253: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:39.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:16:38 smithi153 ceph-mon[70044]: pgmap v3253: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:39.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:16:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:16:39.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:16:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:16:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:16:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:16:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:16:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:16:39.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:16:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:16:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:16:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:16:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:16:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:16:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:16:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:16:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:16:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:16:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:16:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:16:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:16:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:16:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:16:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:16:40.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:16:40 smithi083 ceph-mon[79930]: pgmap v3254: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:40.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:16:40 smithi083 ceph-mon[77242]: pgmap v3254: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:40.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:16:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:16:40.520Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:16:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:16:40 smithi153 ceph-mon[70044]: pgmap v3254: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:16:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:16:41] "GET /metrics HTTP/1.1" 200 225178 "" "Prometheus/2.33.4" 2022-09-29T07:16:42.315 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:16:42 smithi153 ceph-mon[70044]: pgmap v3255: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:42.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:16:42 smithi083 ceph-mon[79930]: pgmap v3255: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:42.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:16:42 smithi083 ceph-mon[77242]: pgmap v3255: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:16:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:16:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:16:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:16:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:16:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:16:43.348 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:16:43.638 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:16:43.639 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (110m) 80s ago 110m 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:16:43.639 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (110m) 9m ago 110m 62.7M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:16:43.639 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (111m) 80s ago 111m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:16:43.640 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (111m) 9m ago 111m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:16:43.640 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (107m) 9m ago 114m 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:16:43.640 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (107m) 80s ago 115m 608M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:16:43.640 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (107m) 80s ago 115m 459M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:16:43.640 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (106m) 9m ago 114m 192M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:16:43.641 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (106m) 80s ago 114m 270M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:16:43.641 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (111m) 80s ago 111m 25.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:16:43.641 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (111m) 9m ago 111m 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:16:43.641 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (105m) 80s ago 114m 335M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:16:43.642 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (105m) 80s ago 113m 327M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:16:43.642 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (105m) 80s ago 113m 261M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:16:43.642 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (104m) 80s ago 113m 315M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:16:43.642 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (103m) 9m ago 112m 262M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:16:43.643 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (103m) 9m ago 112m 317M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:16:43.643 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (102m) 9m ago 112m 282M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:16:43.643 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (102m) 9m ago 112m 327M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:16:43.643 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (107m) 9m ago 111m 114M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:16:43.995 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:16:43.996 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:16:43.996 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:16:43.996 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:16:43.996 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:16:43.997 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:16:43.997 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:16:43.997 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:16:43.997 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:16:43.997 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:16:43.998 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:16:43.998 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:16:43.998 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:16:43.998 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:16:43.999 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:16:43.999 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:16:43.999 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:16:43.999 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:16:44.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:16:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:16:43.924Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:16:44.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:16:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:16:44.622Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:16:44.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:16:44 smithi083 ceph-mon[77242]: from='client.37923 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:16:44.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:16:44 smithi083 ceph-mon[77242]: pgmap v3256: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:44.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:16:44 smithi083 ceph-mon[77242]: from='client.47863 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:16:44.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:16:44 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2252928873' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:16:44.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:16:44 smithi083 ceph-mon[79930]: from='client.37923 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:16:44.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:16:44 smithi083 ceph-mon[79930]: pgmap v3256: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:44.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:16:44 smithi083 ceph-mon[79930]: from='client.47863 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:16:44.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:16:44 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2252928873' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:16:45.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:16:44 smithi153 ceph-mon[70044]: from='client.37923 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:16:45.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:16:44 smithi153 ceph-mon[70044]: pgmap v3256: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:45.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:16:44 smithi153 ceph-mon[70044]: from='client.47863 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:16:45.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:16:44 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2252928873' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:16:46.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:16:46 smithi083 ceph-mon[79930]: pgmap v3257: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:46.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:16:46 smithi083 ceph-mon[77242]: pgmap v3257: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:47.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:16:46 smithi153 ceph-mon[70044]: pgmap v3257: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:48.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:16:48 smithi083 ceph-mon[79930]: pgmap v3258: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:48.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:16:48 smithi083 ceph-mon[77242]: pgmap v3258: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:49.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:16:48 smithi153 ceph-mon[70044]: pgmap v3258: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:50.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:16:50 smithi083 ceph-mon[79930]: pgmap v3259: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:50.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:16:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:16:50.520Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:16:50.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:16:50 smithi083 ceph-mon[77242]: pgmap v3259: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:51.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:16:50 smithi153 ceph-mon[70044]: pgmap v3259: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:16:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:16:51] "GET /metrics HTTP/1.1" 200 225178 "" "Prometheus/2.33.4" 2022-09-29T07:16:52.314 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:16:52 smithi153 ceph-mon[70044]: pgmap v3260: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:52.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:16:52 smithi083 ceph-mon[77242]: pgmap v3260: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:52.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:16:52 smithi083 ceph-mon[79930]: pgmap v3260: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:16:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:16:52.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:16:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:16:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:16:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:16:54.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:16:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:16:53.926Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:16:54.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:16:54 smithi083 ceph-mon[79930]: pgmap v3261: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:54.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:16:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:16:54.623Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:16:54.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:16:54 smithi083 ceph-mon[77242]: pgmap v3261: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:55.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:16:54 smithi153 ceph-mon[70044]: pgmap v3261: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:56.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:16:56 smithi083 ceph-mon[77242]: pgmap v3262: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:56.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:16:56 smithi083 ceph-mon[79930]: pgmap v3262: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:57.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:16:56 smithi153 ceph-mon[70044]: pgmap v3262: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:58.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:16:58 smithi083 ceph-mon[79930]: pgmap v3263: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:58.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:16:58 smithi083 ceph-mon[77242]: pgmap v3263: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:16:59.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:16:58 smithi153 ceph-mon[70044]: pgmap v3263: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:00.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:17:00 smithi083 ceph-mon[79930]: pgmap v3264: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:00.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:17:00 smithi083 ceph-mon[77242]: pgmap v3264: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:00.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:17:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:17:00.521Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:17:01.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:17:00 smithi153 ceph-mon[70044]: pgmap v3264: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:17:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:17:01] "GET /metrics HTTP/1.1" 200 225173 "" "Prometheus/2.33.4" 2022-09-29T07:17:02.315 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:17:02 smithi153 ceph-mon[70044]: pgmap v3265: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:02.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:17:02 smithi083 ceph-mon[79930]: pgmap v3265: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:02.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:17:02 smithi083 ceph-mon[77242]: pgmap v3265: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:17:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:17:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:17:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:17:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:17:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:17:04.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:17:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:17:03.926Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:17:04.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:17:04 smithi083 ceph-mon[79930]: pgmap v3266: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:04.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:17:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:17:04.624Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:17:04.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:17:04 smithi083 ceph-mon[77242]: pgmap v3266: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:05.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:17:04 smithi153 ceph-mon[70044]: pgmap v3266: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:06.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:17:06 smithi083 ceph-mon[79930]: pgmap v3267: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:06.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:17:06 smithi083 ceph-mon[77242]: pgmap v3267: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:07.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:17:06 smithi153 ceph-mon[70044]: pgmap v3267: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:08.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:17:08 smithi083 ceph-mon[79930]: pgmap v3268: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:08.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:17:08 smithi083 ceph-mon[77242]: pgmap v3268: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:09.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:17:08 smithi153 ceph-mon[70044]: pgmap v3268: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:10.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:17:10 smithi083 ceph-mon[79930]: pgmap v3269: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:10.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:17:10 smithi083 ceph-mon[77242]: pgmap v3269: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:10.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:17:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:17:10.522Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:17:11.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:17:10 smithi153 ceph-mon[70044]: pgmap v3269: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:17:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:17:11] "GET /metrics HTTP/1.1" 200 225177 "" "Prometheus/2.33.4" 2022-09-29T07:17:12.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:17:12 smithi083 ceph-mon[79930]: pgmap v3270: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:12.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:17:12 smithi083 ceph-mon[77242]: pgmap v3270: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:17:12 smithi153 ceph-mon[70044]: pgmap v3270: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:12.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:17:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:17:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:17:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:17:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:17:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:17:14.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:17:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:17:13.927Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:17:14.323 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:17:14.611 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:17:14.611 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (110m) 111s ago 111m 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:17:14.612 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (111m) 10m ago 111m 62.7M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:17:14.612 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (112m) 111s ago 112m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:17:14.612 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (112m) 10m ago 112m 27.7M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:17:14.612 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (107m) 10m ago 114m 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:17:14.612 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (108m) 111s ago 116m 608M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:17:14.613 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (107m) 111s ago 116m 459M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:17:14.613 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (107m) 10m ago 115m 192M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:17:14.613 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (107m) 111s ago 115m 270M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:17:14.613 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (111m) 111s ago 111m 25.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:17:14.614 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (111m) 10m ago 111m 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:17:14.614 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (106m) 111s ago 114m 335M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:17:14.614 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (105m) 111s ago 114m 327M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:17:14.614 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (105m) 111s ago 113m 261M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:17:14.615 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (105m) 111s ago 113m 315M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:17:14.615 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (104m) 10m ago 113m 262M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:17:14.615 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (103m) 10m ago 113m 317M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:17:14.615 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (103m) 10m ago 112m 282M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:17:14.615 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (102m) 10m ago 112m 327M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:17:14.616 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (108m) 10m ago 111m 114M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:17:14.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:17:14 smithi083 ceph-mon[77242]: pgmap v3271: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:14.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:17:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:17:14.624Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:17:14.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:17:14 smithi083 ceph-mon[79930]: pgmap v3271: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:14.967 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:17:14.967 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:17:14.968 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:17:14.968 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:17:14.968 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:17:14.968 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:17:14.968 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:17:14.969 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:17:14.969 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:17:14.969 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:17:14.970 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:17:14.970 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:17:14.970 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:17:14.970 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:17:14.971 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:17:14.971 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:17:14.971 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:17:14.971 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:17:15.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:17:14 smithi153 ceph-mon[70044]: pgmap v3271: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:15.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:17:15 smithi083 ceph-mon[79930]: from='client.47875 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:17:15.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:17:15 smithi083 ceph-mon[79930]: from='client.37947 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:17:15.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:17:15 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1183313246' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:17:15.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:17:15 smithi083 ceph-mon[77242]: from='client.47875 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:17:15.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:17:15 smithi083 ceph-mon[77242]: from='client.37947 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:17:15.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:17:15 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1183313246' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:17:16.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:17:15 smithi153 ceph-mon[70044]: from='client.47875 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:17:16.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:17:15 smithi153 ceph-mon[70044]: from='client.37947 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:17:16.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:17:15 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1183313246' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:17:16.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:17:16 smithi083 ceph-mon[79930]: pgmap v3272: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:16.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:17:16 smithi083 ceph-mon[77242]: pgmap v3272: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:17.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:17:16 smithi153 ceph-mon[70044]: pgmap v3272: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:18.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:17:18 smithi083 ceph-mon[79930]: pgmap v3273: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:18.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:17:18 smithi083 ceph-mon[77242]: pgmap v3273: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:19.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:17:18 smithi153 ceph-mon[70044]: pgmap v3273: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:20.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:17:20 smithi083 ceph-mon[79930]: pgmap v3274: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:20.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:17:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:17:20.522Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:17:20.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:17:20 smithi083 ceph-mon[77242]: pgmap v3274: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:21.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:17:20 smithi153 ceph-mon[70044]: pgmap v3274: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:17:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:17:21] "GET /metrics HTTP/1.1" 200 225177 "" "Prometheus/2.33.4" 2022-09-29T07:17:22.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:17:22 smithi083 ceph-mon[79930]: pgmap v3275: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:22.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:17:22 smithi083 ceph-mon[77242]: pgmap v3275: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:17:22 smithi153 ceph-mon[70044]: pgmap v3275: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:22.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:17:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:17:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:17:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:17:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:17:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:17:24.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:17:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:17:23.928Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:17:24.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:17:24 smithi083 ceph-mon[79930]: pgmap v3276: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:24.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:17:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:17:24.625Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:17:24.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:17:24 smithi083 ceph-mon[77242]: pgmap v3276: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:25.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:17:24 smithi153 ceph-mon[70044]: pgmap v3276: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:25.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:17:25 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:17:25.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:17:25 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:17:25.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:17:25 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:17:25.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:17:25 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:17:25.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:17:25 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:17:25.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:17:25 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:17:26.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:17:25 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:17:26.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:17:25 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:17:26.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:17:25 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:17:26.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:17:26 smithi083 ceph-mon[79930]: pgmap v3277: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:26.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:17:26 smithi083 ceph-mon[77242]: pgmap v3277: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:27.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:17:26 smithi153 ceph-mon[70044]: pgmap v3277: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:28.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:17:28 smithi083 ceph-mon[79930]: pgmap v3278: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:28.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:17:28 smithi083 ceph-mon[77242]: pgmap v3278: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:29.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:17:28 smithi153 ceph-mon[70044]: pgmap v3278: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:30.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:17:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:17:30.522Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:17:30.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:17:30 smithi083 ceph-mon[77242]: pgmap v3279: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:30.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:17:30 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:17:30.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:17:30 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:17:30.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:17:30 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:17:30.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:17:30 smithi083 ceph-mon[79930]: pgmap v3279: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:30.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:17:30 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:17:30.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:17:30 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:17:30.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:17:30 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:17:31.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:17:30 smithi153 ceph-mon[70044]: pgmap v3279: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:31.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:17:30 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:17:31.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:17:30 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:17:31.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:17:30 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:17:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:17:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:17:31] "GET /metrics HTTP/1.1" 200 225178 "" "Prometheus/2.33.4" 2022-09-29T07:17:32.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:17:32 smithi083 ceph-mon[79930]: pgmap v3280: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:32.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:17:32 smithi083 ceph-mon[77242]: pgmap v3280: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:17:32 smithi153 ceph-mon[70044]: pgmap v3280: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:32.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:17:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:17:32.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:17:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:17:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:17:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:17:34.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:17:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:17:33.929Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:17:34.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:17:34 smithi083 ceph-mon[79930]: pgmap v3281: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:34.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:17:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:17:34.625Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:17:34.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:17:34 smithi083 ceph-mon[77242]: pgmap v3281: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:35.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:17:34 smithi153 ceph-mon[70044]: pgmap v3281: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:36.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:17:36 smithi083 ceph-mon[79930]: pgmap v3282: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:36.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:17:36 smithi083 ceph-mon[77242]: pgmap v3282: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:37.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:17:36 smithi153 ceph-mon[70044]: pgmap v3282: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:38.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:17:38 smithi083 ceph-mon[79930]: pgmap v3283: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:38.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:17:38 smithi083 ceph-mon[77242]: pgmap v3283: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:39.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:17:38 smithi153 ceph-mon[70044]: pgmap v3283: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:39.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:17:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:17:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:17:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:17:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:17:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:17:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:17:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:17:39.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:17:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:17:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:17:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:17:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:17:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:17:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:17:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:17:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:17:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:17:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:17:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:17:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:17:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:17:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:17:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:17:40.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:17:40 smithi083 ceph-mon[77242]: pgmap v3284: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:17:40 smithi083 ceph-mon[79930]: pgmap v3284: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:40.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:17:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:17:40.523Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:17:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:17:40 smithi153 ceph-mon[70044]: pgmap v3284: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:17:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:17:41] "GET /metrics HTTP/1.1" 200 225173 "" "Prometheus/2.33.4" 2022-09-29T07:17:42.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:17:42 smithi083 ceph-mon[79930]: pgmap v3285: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:42.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:17:42 smithi083 ceph-mon[77242]: pgmap v3285: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:17:42 smithi153 ceph-mon[70044]: pgmap v3285: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:42.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:17:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:17:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:17:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:17:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:17:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:17:44.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:17:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:17:43.930Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:17:44.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:17:44 smithi083 ceph-mon[77242]: pgmap v3286: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:44.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:17:44 smithi083 ceph-mon[79930]: pgmap v3286: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:44.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:17:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:17:44.625Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:17:45.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:17:44 smithi153 ceph-mon[70044]: pgmap v3286: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:45.294 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:17:45.585 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:17:45.586 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (111m) 2m ago 111m 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:17:45.586 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (111m) 15s ago 111m 62.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:17:45.586 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (112m) 2m ago 112m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:17:45.586 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (112m) 15s ago 112m 27.8M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:17:45.587 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (108m) 15s ago 115m 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:17:45.587 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (108m) 2m ago 116m 608M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:17:45.587 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (108m) 2m ago 117m 459M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:17:45.587 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (107m) 15s ago 115m 209M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:17:45.588 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (107m) 2m ago 115m 270M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:17:45.588 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (112m) 2m ago 112m 25.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:17:45.588 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (112m) 15s ago 112m 26.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:17:45.588 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (106m) 2m ago 115m 335M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:17:45.588 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (106m) 2m ago 114m 327M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:17:45.589 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (106m) 2m ago 114m 261M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:17:45.589 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (105m) 2m ago 114m 315M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:17:45.589 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (104m) 15s ago 113m 284M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:17:45.589 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (104m) 15s ago 113m 345M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:17:45.589 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (103m) 15s ago 113m 306M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:17:45.590 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (103m) 15s ago 113m 356M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:17:45.590 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (108m) 15s ago 112m 116M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:17:45.941 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:17:45.941 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:17:45.942 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:17:45.942 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:17:45.942 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:17:45.942 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:17:45.943 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:17:45.943 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:17:45.943 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:17:45.943 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:17:45.943 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:17:45.944 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:17:45.944 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:17:45.944 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:17:45.944 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:17:45.944 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:17:45.945 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:17:45.945 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:17:46.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:17:46 smithi083 ceph-mon[79930]: from='client.37959 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:17:46.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:17:46 smithi083 ceph-mon[79930]: from='client.47899 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:17:46.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:17:46 smithi083 ceph-mon[79930]: pgmap v3287: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:46.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:17:46 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1827643303' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:17:46.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:17:46 smithi083 ceph-mon[77242]: from='client.37959 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:17:46.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:17:46 smithi083 ceph-mon[77242]: from='client.47899 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:17:46.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:17:46 smithi083 ceph-mon[77242]: pgmap v3287: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:46.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:17:46 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1827643303' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:17:47.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:17:46 smithi153 ceph-mon[70044]: from='client.37959 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:17:47.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:17:46 smithi153 ceph-mon[70044]: from='client.47899 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:17:47.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:17:46 smithi153 ceph-mon[70044]: pgmap v3287: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:47.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:17:46 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1827643303' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:17:48.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:17:48 smithi083 ceph-mon[79930]: pgmap v3288: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:48.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:17:48 smithi083 ceph-mon[77242]: pgmap v3288: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:49.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:17:48 smithi153 ceph-mon[70044]: pgmap v3288: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:50.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:17:50 smithi083 ceph-mon[77242]: pgmap v3289: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:50.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:17:50 smithi083 ceph-mon[79930]: pgmap v3289: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:50.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:17:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:17:50.524Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:17:51.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:17:50 smithi153 ceph-mon[70044]: pgmap v3289: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:17:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:17:51] "GET /metrics HTTP/1.1" 200 225173 "" "Prometheus/2.33.4" 2022-09-29T07:17:52.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:17:52 smithi083 ceph-mon[79930]: pgmap v3290: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:52.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:17:52 smithi083 ceph-mon[77242]: pgmap v3290: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:17:52 smithi153 ceph-mon[70044]: pgmap v3290: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:52.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:17:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:17:52.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:17:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:17:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:17:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:17:54.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:17:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:17:53.929Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:17:54.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:17:54 smithi083 ceph-mon[79930]: pgmap v3291: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:54.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:17:54 smithi083 ceph-mon[77242]: pgmap v3291: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:54.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:17:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:17:54.626Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:17:55.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:17:54 smithi153 ceph-mon[70044]: pgmap v3291: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:56.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:17:56 smithi083 ceph-mon[79930]: pgmap v3292: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:56.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:17:56 smithi083 ceph-mon[77242]: pgmap v3292: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:57.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:17:56 smithi153 ceph-mon[70044]: pgmap v3292: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:58.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:17:58 smithi083 ceph-mon[77242]: pgmap v3293: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:58.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:17:58 smithi083 ceph-mon[79930]: pgmap v3293: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:17:59.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:17:58 smithi153 ceph-mon[70044]: pgmap v3293: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:00.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:18:00 smithi083 ceph-mon[77242]: pgmap v3294: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:00.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:18:00 smithi083 ceph-mon[79930]: pgmap v3294: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:00.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:18:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:18:00.525Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:18:01.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:18:00 smithi153 ceph-mon[70044]: pgmap v3294: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:18:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:18:01] "GET /metrics HTTP/1.1" 200 225175 "" "Prometheus/2.33.4" 2022-09-29T07:18:02.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:18:02 smithi083 ceph-mon[79930]: pgmap v3295: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:02.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:18:02 smithi083 ceph-mon[77242]: pgmap v3295: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:18:02 smithi153 ceph-mon[70044]: pgmap v3295: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:02.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:18:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:18:02.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:18:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:18:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:18:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:18:04.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:18:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:18:03.930Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:18:04.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:18:04 smithi083 ceph-mon[79930]: pgmap v3296: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:04.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:18:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:18:04.627Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:18:04.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:18:04 smithi083 ceph-mon[77242]: pgmap v3296: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:05.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:18:04 smithi153 ceph-mon[70044]: pgmap v3296: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:06.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:18:06 smithi083 ceph-mon[79930]: pgmap v3297: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:06.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:18:06 smithi083 ceph-mon[77242]: pgmap v3297: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:07.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:18:06 smithi153 ceph-mon[70044]: pgmap v3297: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:08.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:18:08 smithi083 ceph-mon[79930]: pgmap v3298: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:08.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:18:08 smithi083 ceph-mon[77242]: pgmap v3298: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:09.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:18:08 smithi153 ceph-mon[70044]: pgmap v3298: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:10.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:18:10 smithi083 ceph-mon[79930]: pgmap v3299: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:10.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:18:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:18:10.526Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:18:10.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:18:10 smithi083 ceph-mon[77242]: pgmap v3299: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:11.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:18:10 smithi153 ceph-mon[70044]: pgmap v3299: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:18:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:18:11] "GET /metrics HTTP/1.1" 200 225178 "" "Prometheus/2.33.4" 2022-09-29T07:18:12.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:18:12 smithi083 ceph-mon[79930]: pgmap v3300: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:12.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:18:12 smithi083 ceph-mon[77242]: pgmap v3300: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:18:12 smithi153 ceph-mon[70044]: pgmap v3300: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:12.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:18:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:18:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:18:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:18:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:18:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:18:14.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:18:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:18:13.931Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:18:14.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:18:14 smithi083 ceph-mon[79930]: pgmap v3301: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:14.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:18:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:18:14.627Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:18:14.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:18:14 smithi083 ceph-mon[77242]: pgmap v3301: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:15.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:18:14 smithi153 ceph-mon[70044]: pgmap v3301: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:16.267 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:18:16.553 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:18:16.554 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (111m) 2m ago 112m 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:18:16.554 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (112m) 46s ago 112m 62.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:18:16.554 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (113m) 2m ago 113m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:18:16.554 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (113m) 46s ago 113m 27.8M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:18:16.555 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (109m) 46s ago 115m 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:18:16.555 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (109m) 2m ago 117m 608M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:18:16.555 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (108m) 2m ago 117m 459M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:18:16.555 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (108m) 46s ago 116m 209M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:18:16.555 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (108m) 2m ago 116m 270M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:18:16.556 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (112m) 2m ago 112m 25.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:18:16.556 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (112m) 46s ago 112m 26.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:18:16.556 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (107m) 2m ago 115m 335M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:18:16.556 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (106m) 2m ago 115m 327M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:18:16.557 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (106m) 2m ago 115m 261M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:18:16.557 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (106m) 2m ago 114m 315M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:18:16.557 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (105m) 46s ago 114m 284M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:18:16.557 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (104m) 46s ago 114m 345M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:18:16.557 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (104m) 46s ago 113m 306M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:18:16.558 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (103m) 46s ago 113m 356M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:18:16.558 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (109m) 46s ago 112m 116M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:18:16.909 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:18:16.909 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:18:16.909 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:18:16.909 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:18:16.910 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:18:16.910 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:18:16.910 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:18:16.910 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:18:16.911 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:18:16.911 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:18:16.911 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:18:16.911 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:18:16.912 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:18:16.912 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:18:16.912 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:18:16.912 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:18:16.912 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:18:16.913 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:18:16.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:18:16 smithi083 ceph-mon[79930]: pgmap v3302: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:16.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:18:16 smithi083 ceph-mon[77242]: pgmap v3302: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:17.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:18:16 smithi153 ceph-mon[70044]: pgmap v3302: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:17.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:18:17 smithi083 ceph-mon[79930]: from='client.37977 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:18:17.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:18:17 smithi083 ceph-mon[79930]: from='client.37983 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:18:17.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:18:17 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/634871060' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:18:17.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:18:17 smithi083 ceph-mon[77242]: from='client.37977 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:18:17.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:18:17 smithi083 ceph-mon[77242]: from='client.37983 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:18:17.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:18:17 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/634871060' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:18:18.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:18:17 smithi153 ceph-mon[70044]: from='client.37977 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:18:18.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:18:17 smithi153 ceph-mon[70044]: from='client.37983 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:18:18.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:18:17 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/634871060' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:18:18.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:18:18 smithi083 ceph-mon[79930]: pgmap v3303: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:18.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:18:18 smithi083 ceph-mon[77242]: pgmap v3303: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:19.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:18:18 smithi153 ceph-mon[70044]: pgmap v3303: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:20.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:18:20 smithi083 ceph-mon[79930]: pgmap v3304: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:20.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:18:20 smithi083 ceph-mon[77242]: pgmap v3304: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:20.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:18:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:18:20.527Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:18:21.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:18:20 smithi153 ceph-mon[70044]: pgmap v3304: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:18:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:18:21] "GET /metrics HTTP/1.1" 200 225178 "" "Prometheus/2.33.4" 2022-09-29T07:18:22.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:18:22 smithi083 ceph-mon[79930]: pgmap v3305: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:22.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:18:22 smithi083 ceph-mon[77242]: pgmap v3305: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:18:22 smithi153 ceph-mon[70044]: pgmap v3305: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:22.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:18:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:18:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:18:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:18:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:18:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:18:24.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:18:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:18:23.932Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:18:24.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:18:24 smithi083 ceph-mon[79930]: pgmap v3306: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:24.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:18:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:18:24.627Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:18:24.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:18:24 smithi083 ceph-mon[77242]: pgmap v3306: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:25.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:18:24 smithi153 ceph-mon[70044]: pgmap v3306: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:26.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:18:26 smithi083 ceph-mon[79930]: pgmap v3307: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:26.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:18:26 smithi083 ceph-mon[77242]: pgmap v3307: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:27.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:18:26 smithi153 ceph-mon[70044]: pgmap v3307: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:28.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:18:28 smithi083 ceph-mon[79930]: pgmap v3308: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:28.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:18:28 smithi083 ceph-mon[77242]: pgmap v3308: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:29.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:18:28 smithi153 ceph-mon[70044]: pgmap v3308: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:30.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:18:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:18:30.527Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:18:30.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:18:30 smithi083 ceph-mon[77242]: pgmap v3309: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:30.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:18:30 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:18:30.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:18:30 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:18:30.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:18:30 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:18:30.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:18:30 smithi083 ceph-mon[79930]: pgmap v3309: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:30.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:18:30 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:18:30.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:18:30 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:18:30.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:18:30 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:18:31.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:18:30 smithi153 ceph-mon[70044]: pgmap v3309: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:31.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:18:30 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:18:31.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:18:30 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:18:31.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:18:30 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:18:31.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:18:31 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:18:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:18:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:18:31] "GET /metrics HTTP/1.1" 200 225175 "" "Prometheus/2.33.4" 2022-09-29T07:18:31.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:18:31 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:18:32.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:18:31 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:18:32.691 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:18:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:18:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:18:32.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:18:32 smithi083 ceph-mon[79930]: pgmap v3310: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:32.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:18:32 smithi083 ceph-mon[77242]: pgmap v3310: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:33.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:18:32 smithi153 ceph-mon[70044]: pgmap v3310: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:18:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:18:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:18:34.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:18:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:18:33.933Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:18:34.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:18:34 smithi083 ceph-mon[79930]: pgmap v3311: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:34.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:18:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:18:34.627Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:18:34.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:18:34 smithi083 ceph-mon[77242]: pgmap v3311: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:35.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:18:34 smithi153 ceph-mon[70044]: pgmap v3311: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:36.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:18:36 smithi083 ceph-mon[79930]: pgmap v3312: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:36.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:18:36 smithi083 ceph-mon[77242]: pgmap v3312: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:37.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:18:36 smithi153 ceph-mon[70044]: pgmap v3312: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:38.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:18:38 smithi083 ceph-mon[79930]: pgmap v3313: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:38.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:18:38 smithi083 ceph-mon[77242]: pgmap v3313: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:39.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:18:38 smithi153 ceph-mon[70044]: pgmap v3313: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:39.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:18:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:18:39.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:18:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:18:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:18:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:18:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:18:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:18:39.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:18:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:18:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:18:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:18:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:18:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:18:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:18:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:18:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:18:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:18:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:18:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:18:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:18:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:18:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:18:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:18:40.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:18:40 smithi083 ceph-mon[79930]: pgmap v3314: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:40.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:18:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:18:40.528Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:18:40.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:18:40 smithi083 ceph-mon[77242]: pgmap v3314: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:18:40 smithi153 ceph-mon[70044]: pgmap v3314: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:18:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:18:41] "GET /metrics HTTP/1.1" 200 225175 "" "Prometheus/2.33.4" 2022-09-29T07:18:42.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:18:42 smithi083 ceph-mon[77242]: pgmap v3315: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:18:42 smithi083 ceph-mon[79930]: pgmap v3315: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:18:42 smithi153 ceph-mon[70044]: pgmap v3315: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:42.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:18:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:18:42.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:18:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:18:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:18:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:18:44.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:18:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:18:43.934Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:18:44.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:18:44 smithi083 ceph-mon[77242]: pgmap v3316: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:44.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:18:44 smithi083 ceph-mon[79930]: pgmap v3316: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:44.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:18:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:18:44.628Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:18:45.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:18:44 smithi153 ceph-mon[70044]: pgmap v3316: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:46.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:18:46 smithi083 ceph-mon[77242]: pgmap v3317: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:46.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:18:46 smithi083 ceph-mon[79930]: pgmap v3317: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:47.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:18:46 smithi153 ceph-mon[70044]: pgmap v3317: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:47.234 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:18:47.523 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:18:47.524 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (112m) 3m ago 112m 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:18:47.524 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (112m) 77s ago 112m 62.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:18:47.524 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (113m) 3m ago 113m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:18:47.524 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (113m) 77s ago 113m 27.8M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:18:47.525 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (109m) 77s ago 116m 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:18:47.525 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (109m) 3m ago 117m 608M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:18:47.525 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (109m) 3m ago 118m 459M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:18:47.525 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (108m) 77s ago 116m 209M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:18:47.526 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (108m) 3m ago 116m 270M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:18:47.526 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (113m) 3m ago 113m 25.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:18:47.526 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (113m) 77s ago 113m 26.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:18:47.526 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (107m) 3m ago 116m 335M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:18:47.526 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (107m) 3m ago 115m 327M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:18:47.527 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (107m) 3m ago 115m 261M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:18:47.527 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (106m) 3m ago 115m 315M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:18:47.527 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (106m) 77s ago 114m 284M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:18:47.527 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (105m) 77s ago 114m 345M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:18:47.528 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (104m) 77s ago 114m 306M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:18:47.528 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (104m) 77s ago 114m 356M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:18:47.528 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (109m) 77s ago 113m 116M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:18:47.881 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:18:47.881 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:18:47.881 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:18:47.881 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:18:47.882 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:18:47.882 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:18:47.882 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:18:47.882 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:18:47.883 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:18:47.883 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:18:47.883 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:18:47.883 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:18:47.884 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:18:47.884 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:18:47.884 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:18:47.884 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:18:47.884 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:18:47.885 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:18:48.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:18:48 smithi083 ceph-mon[79930]: from='client.37995 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:18:48.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:18:48 smithi083 ceph-mon[79930]: from='client.38001 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:18:48.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:18:48 smithi083 ceph-mon[79930]: pgmap v3318: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:48.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:18:48 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2794655078' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:18:48.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:18:48 smithi083 ceph-mon[77242]: from='client.37995 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:18:48.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:18:48 smithi083 ceph-mon[77242]: from='client.38001 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:18:48.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:18:48 smithi083 ceph-mon[77242]: pgmap v3318: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:48.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:18:48 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2794655078' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:18:49.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:18:48 smithi153 ceph-mon[70044]: from='client.37995 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:18:49.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:18:48 smithi153 ceph-mon[70044]: from='client.38001 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:18:49.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:18:48 smithi153 ceph-mon[70044]: pgmap v3318: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:49.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:18:48 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2794655078' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:18:50.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:18:50 smithi083 ceph-mon[77242]: pgmap v3319: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:50.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:18:50 smithi083 ceph-mon[79930]: pgmap v3319: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:50.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:18:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:18:50.529Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:18:51.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:18:50 smithi153 ceph-mon[70044]: pgmap v3319: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:18:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:18:51] "GET /metrics HTTP/1.1" 200 225175 "" "Prometheus/2.33.4" 2022-09-29T07:18:52.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:18:52 smithi083 ceph-mon[79930]: pgmap v3320: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:52.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:18:52 smithi083 ceph-mon[77242]: pgmap v3320: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:18:52 smithi153 ceph-mon[70044]: pgmap v3320: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:52.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:18:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:18:52.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:18:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:18:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:18:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:18:54.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:18:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:18:53.934Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:18:54.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:18:54 smithi083 ceph-mon[79930]: pgmap v3321: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:54.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:18:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:18:54.629Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:18:54.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:18:54 smithi083 ceph-mon[77242]: pgmap v3321: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:55.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:18:54 smithi153 ceph-mon[70044]: pgmap v3321: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:56.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:18:56 smithi083 ceph-mon[79930]: pgmap v3322: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:56.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:18:56 smithi083 ceph-mon[77242]: pgmap v3322: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:57.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:18:56 smithi153 ceph-mon[70044]: pgmap v3322: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:58.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:18:58 smithi083 ceph-mon[79930]: pgmap v3323: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:58.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:18:58 smithi083 ceph-mon[77242]: pgmap v3323: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:18:59.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:18:58 smithi153 ceph-mon[70044]: pgmap v3323: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:00.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:19:00 smithi083 ceph-mon[77242]: pgmap v3324: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:00.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:19:00 smithi083 ceph-mon[79930]: pgmap v3324: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:00.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:19:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:19:00.530Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:19:01.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:19:00 smithi153 ceph-mon[70044]: pgmap v3324: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:19:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:19:01] "GET /metrics HTTP/1.1" 200 225173 "" "Prometheus/2.33.4" 2022-09-29T07:19:02.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:19:02 smithi083 ceph-mon[79930]: pgmap v3325: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:02.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:19:02 smithi083 ceph-mon[77242]: pgmap v3325: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:19:02 smithi153 ceph-mon[70044]: pgmap v3325: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:02.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:19:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:19:02.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:19:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:19:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:19:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:19:04.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:19:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:19:03.935Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:19:04.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:19:04 smithi083 ceph-mon[79930]: pgmap v3326: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:04.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:19:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:19:04.630Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:19:04.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:19:04 smithi083 ceph-mon[77242]: pgmap v3326: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:05.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:19:04 smithi153 ceph-mon[70044]: pgmap v3326: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:06.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:19:06 smithi083 ceph-mon[79930]: pgmap v3327: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:06.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:19:06 smithi083 ceph-mon[77242]: pgmap v3327: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:07.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:19:06 smithi153 ceph-mon[70044]: pgmap v3327: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:08.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:19:08 smithi083 ceph-mon[79930]: pgmap v3328: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:08.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:19:08 smithi083 ceph-mon[77242]: pgmap v3328: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:09.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:19:08 smithi153 ceph-mon[70044]: pgmap v3328: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:10.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:19:10 smithi083 ceph-mon[77242]: pgmap v3329: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:10.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:19:10 smithi083 ceph-mon[79930]: pgmap v3329: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:10.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:19:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:19:10.531Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:19:11.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:19:10 smithi153 ceph-mon[70044]: pgmap v3329: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:19:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:19:11] "GET /metrics HTTP/1.1" 200 225173 "" "Prometheus/2.33.4" 2022-09-29T07:19:12.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:19:12 smithi083 ceph-mon[79930]: pgmap v3330: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:12.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:19:12 smithi083 ceph-mon[77242]: pgmap v3330: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:19:12 smithi153 ceph-mon[70044]: pgmap v3330: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:12.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:19:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:19:12.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:19:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:19:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:19:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:19:14.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:19:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:19:13.935Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:19:14.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:19:14 smithi083 ceph-mon[79930]: pgmap v3331: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:14.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:19:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:19:14.631Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:19:14.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:19:14 smithi083 ceph-mon[77242]: pgmap v3331: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:15.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:19:14 smithi153 ceph-mon[70044]: pgmap v3331: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:16.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:19:16 smithi083 ceph-mon[79930]: pgmap v3332: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:16.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:19:16 smithi083 ceph-mon[77242]: pgmap v3332: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:17.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:19:16 smithi153 ceph-mon[70044]: pgmap v3332: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:18.207 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:19:18.495 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:19:18.495 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (113m) 3m ago 113m 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:19:18.496 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (113m) 108s ago 113m 62.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:19:18.496 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (114m) 3m ago 114m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:19:18.496 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (114m) 108s ago 114m 27.8M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:19:18.496 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (110m) 108s ago 116m 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:19:18.497 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (110m) 3m ago 118m 608M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:19:18.497 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (109m) 3m ago 118m 459M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:19:18.497 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (109m) 108s ago 117m 209M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:19:18.497 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (109m) 3m ago 117m 270M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:19:18.498 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (113m) 3m ago 113m 25.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:19:18.498 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (113m) 108s ago 113m 26.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:19:18.498 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (108m) 3m ago 116m 335M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:19:18.498 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (107m) 3m ago 116m 327M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:19:18.499 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (107m) 3m ago 116m 261M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:19:18.499 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (107m) 3m ago 115m 315M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:19:18.499 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (106m) 108s ago 115m 284M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:19:18.499 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (105m) 108s ago 115m 345M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:19:18.500 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (105m) 108s ago 114m 306M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:19:18.500 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (104m) 108s ago 114m 356M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:19:18.500 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (110m) 108s ago 113m 116M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:19:18.851 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:19:18.851 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:19:18.852 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:19:18.852 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:19:18.852 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:19:18.852 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:19:18.852 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:19:18.853 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:19:18.853 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:19:18.853 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:19:18.853 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:19:18.854 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:19:18.854 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:19:18.854 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:19:18.854 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:19:18.854 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:19:18.855 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:19:18.855 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:19:18.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:19:18 smithi083 ceph-mon[79930]: pgmap v3333: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:18.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:19:18 smithi083 ceph-mon[77242]: pgmap v3333: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:19.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:19:18 smithi153 ceph-mon[70044]: pgmap v3333: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:19.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:19:19 smithi083 ceph-mon[79930]: from='client.38013 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:19:19.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:19:19 smithi083 ceph-mon[79930]: from='client.38019 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:19:19.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:19:19 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3829137256' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:19:19.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:19:19 smithi083 ceph-mon[77242]: from='client.38013 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:19:19.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:19:19 smithi083 ceph-mon[77242]: from='client.38019 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:19:19.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:19:19 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3829137256' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:19:20.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:19:19 smithi153 ceph-mon[70044]: from='client.38013 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:19:20.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:19:19 smithi153 ceph-mon[70044]: from='client.38019 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:19:20.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:19:19 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3829137256' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:19:20.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:19:20 smithi083 ceph-mon[77242]: pgmap v3334: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:20.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:19:20 smithi083 ceph-mon[79930]: pgmap v3334: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:20.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:19:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:19:20.531Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:19:21.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:19:20 smithi153 ceph-mon[70044]: pgmap v3334: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:19:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:19:21] "GET /metrics HTTP/1.1" 200 225173 "" "Prometheus/2.33.4" 2022-09-29T07:19:22.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:19:22 smithi083 ceph-mon[79930]: pgmap v3335: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:22.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:19:22 smithi083 ceph-mon[77242]: pgmap v3335: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:19:22 smithi153 ceph-mon[70044]: pgmap v3335: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:22.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:19:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:19:22.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:19:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:19:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:19:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:19:24.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:19:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:19:23.936Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:19:24.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:19:24 smithi083 ceph-mon[79930]: pgmap v3336: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:24.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:19:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:19:24.632Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:19:24.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:19:24 smithi083 ceph-mon[77242]: pgmap v3336: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:25.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:19:24 smithi153 ceph-mon[70044]: pgmap v3336: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:26.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:19:26 smithi083 ceph-mon[79930]: pgmap v3337: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:26.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:19:26 smithi083 ceph-mon[77242]: pgmap v3337: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:27.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:19:26 smithi153 ceph-mon[70044]: pgmap v3337: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:28.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:19:28 smithi083 ceph-mon[79930]: pgmap v3338: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:28.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:19:28 smithi083 ceph-mon[77242]: pgmap v3338: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:29.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:19:28 smithi153 ceph-mon[70044]: pgmap v3338: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:30.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:19:30 smithi083 ceph-mon[79930]: pgmap v3339: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:30.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:19:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:19:30.532Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:19:30.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:19:30 smithi083 ceph-mon[77242]: pgmap v3339: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:31.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:19:30 smithi153 ceph-mon[70044]: pgmap v3339: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:19:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:19:31] "GET /metrics HTTP/1.1" 200 225174 "" "Prometheus/2.33.4" 2022-09-29T07:19:31.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:19:31 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:19:31.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:19:31 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:19:31.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:19:31 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:19:31.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:19:31 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:19:31.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:19:31 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:19:31.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:19:31 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:19:31.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:19:31 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:19:31.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:19:31 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:19:32.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:19:31 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:19:32.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:19:31 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:19:32.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:19:31 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:19:32.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:19:31 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:19:32.680 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:19:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:19:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:19:32.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:19:32 smithi083 ceph-mon[79930]: pgmap v3340: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:32.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:19:32 smithi083 ceph-mon[77242]: pgmap v3340: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:33.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:19:32 smithi153 ceph-mon[70044]: pgmap v3340: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:19:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:19:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:19:34.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:19:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:19:33.937Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:19:34.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:19:34 smithi083 ceph-mon[79930]: pgmap v3341: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:34.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:19:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:19:34.633Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:19:34.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:19:34 smithi083 ceph-mon[77242]: pgmap v3341: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:35.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:19:34 smithi153 ceph-mon[70044]: pgmap v3341: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:36.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:19:36 smithi083 ceph-mon[79930]: pgmap v3342: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:36.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:19:36 smithi083 ceph-mon[77242]: pgmap v3342: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:37.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:19:36 smithi153 ceph-mon[70044]: pgmap v3342: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:38.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:19:38 smithi083 ceph-mon[79930]: pgmap v3343: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:38.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:19:38 smithi083 ceph-mon[77242]: pgmap v3343: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:39.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:19:38 smithi153 ceph-mon[70044]: pgmap v3343: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:39.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:19:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:19:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:19:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:19:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:19:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:19:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:19:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:19:39.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:19:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:19:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:19:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:19:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:19:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:19:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:19:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:19:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:19:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:19:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:19:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:19:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:19:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:19:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:19:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:19:40.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:19:40 smithi083 ceph-mon[79930]: pgmap v3344: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:40.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:19:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:19:40.532Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:19:40.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:19:40 smithi083 ceph-mon[77242]: pgmap v3344: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:19:40 smithi153 ceph-mon[70044]: pgmap v3344: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:19:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:19:41] "GET /metrics HTTP/1.1" 200 225172 "" "Prometheus/2.33.4" 2022-09-29T07:19:42.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:19:42 smithi083 ceph-mon[79930]: pgmap v3345: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:42.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:19:42 smithi083 ceph-mon[77242]: pgmap v3345: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:19:42 smithi153 ceph-mon[70044]: pgmap v3345: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:42.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:19:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:19:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:19:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:19:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:19:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:19:44.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:19:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:19:43.938Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:19:44.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:19:44 smithi083 ceph-mon[79930]: pgmap v3346: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:44.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:19:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:19:44.634Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:19:44.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:19:44 smithi083 ceph-mon[77242]: pgmap v3346: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:45.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:19:44 smithi153 ceph-mon[70044]: pgmap v3346: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:46.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:19:46 smithi083 ceph-mon[77242]: pgmap v3347: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:46.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:19:46 smithi083 ceph-mon[79930]: pgmap v3347: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:47.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:19:46 smithi153 ceph-mon[70044]: pgmap v3347: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:48.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:19:48 smithi083 ceph-mon[79930]: pgmap v3348: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:48.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:19:48 smithi083 ceph-mon[77242]: pgmap v3348: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:49.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:19:48 smithi153 ceph-mon[70044]: pgmap v3348: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:49.176 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:19:49.466 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:19:49.467 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (113m) 4m ago 114m 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:19:49.467 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (113m) 2m ago 113m 62.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:19:49.467 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (114m) 4m ago 114m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:19:49.467 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (114m) 2m ago 114m 27.8M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:19:49.468 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (110m) 2m ago 117m 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:19:49.468 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (111m) 4m ago 118m 608M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:19:49.468 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (110m) 4m ago 119m 459M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:19:49.469 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (109m) 2m ago 117m 209M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:19:49.469 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (109m) 4m ago 117m 270M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:19:49.469 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (114m) 4m ago 114m 25.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:19:49.469 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (114m) 2m ago 114m 26.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:19:49.469 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (109m) 4m ago 117m 335M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:19:49.470 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (108m) 4m ago 116m 327M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:19:49.470 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (108m) 4m ago 116m 261M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:19:49.470 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (107m) 4m ago 116m 315M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:19:49.470 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (107m) 2m ago 115m 284M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:19:49.471 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (106m) 2m ago 115m 345M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:19:49.471 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (105m) 2m ago 115m 306M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:19:49.471 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (105m) 2m ago 115m 356M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:19:49.471 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (110m) 2m ago 114m 116M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:19:49.825 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:19:49.825 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:19:49.826 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:19:49.826 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:19:49.826 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:19:49.826 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:19:49.827 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:19:49.827 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:19:49.827 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:19:49.827 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:19:49.828 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:19:49.828 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:19:49.828 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:19:49.828 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:19:49.829 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:19:49.829 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:19:49.829 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:19:49.829 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:19:50.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:19:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:19:50.532Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:19:50.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:19:50 smithi083 ceph-mon[77242]: from='client.38031 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:19:50.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:19:50 smithi083 ceph-mon[77242]: from='client.38037 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:19:50.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:19:50 smithi083 ceph-mon[77242]: pgmap v3349: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:50.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:19:50 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2527004779' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:19:50.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:19:50 smithi083 ceph-mon[79930]: from='client.38031 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:19:50.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:19:50 smithi083 ceph-mon[79930]: from='client.38037 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:19:50.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:19:50 smithi083 ceph-mon[79930]: pgmap v3349: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:50.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:19:50 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2527004779' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:19:51.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:19:50 smithi153 ceph-mon[70044]: from='client.38031 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:19:51.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:19:50 smithi153 ceph-mon[70044]: from='client.38037 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:19:51.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:19:50 smithi153 ceph-mon[70044]: pgmap v3349: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:51.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:19:50 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2527004779' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:19:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:19:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:19:51] "GET /metrics HTTP/1.1" 200 225172 "" "Prometheus/2.33.4" 2022-09-29T07:19:52.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:19:52 smithi083 ceph-mon[79930]: pgmap v3350: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:52.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:19:52 smithi083 ceph-mon[77242]: pgmap v3350: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:19:52 smithi153 ceph-mon[70044]: pgmap v3350: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:52.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:19:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:19:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:19:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:19:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:19:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:19:54.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:19:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:19:53.938Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:19:54.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:19:54 smithi083 ceph-mon[77242]: pgmap v3351: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:54.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:19:54 smithi083 ceph-mon[79930]: pgmap v3351: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:54.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:19:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:19:54.634Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:19:55.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:19:54 smithi153 ceph-mon[70044]: pgmap v3351: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:56.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:19:56 smithi083 ceph-mon[79930]: pgmap v3352: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:56.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:19:56 smithi083 ceph-mon[77242]: pgmap v3352: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:57.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:19:56 smithi153 ceph-mon[70044]: pgmap v3352: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:58.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:19:58 smithi083 ceph-mon[79930]: pgmap v3353: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:58.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:19:58 smithi083 ceph-mon[77242]: pgmap v3353: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:19:59.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:19:58 smithi153 ceph-mon[70044]: pgmap v3353: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:00.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:20:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[77219]: 2022-09-29T07:20:00.000+0000 7f160f665700 -1 log_channel(cluster) log [ERR] : overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T07:20:00.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:20:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:20:00.533Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:20:00.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:20:00 smithi083 ceph-mon[77242]: pgmap v3354: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:00.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:20:00 smithi083 ceph-mon[77242]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T07:20:00.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:20:00 smithi083 ceph-mon[79930]: pgmap v3354: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:00.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:20:00 smithi083 ceph-mon[79930]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T07:20:01.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:20:00 smithi153 ceph-mon[70044]: pgmap v3354: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:01.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:20:00 smithi153 ceph-mon[70044]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T07:20:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:20:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:20:01] "GET /metrics HTTP/1.1" 200 225176 "" "Prometheus/2.33.4" 2022-09-29T07:20:02.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:20:02 smithi083 ceph-mon[79930]: pgmap v3355: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:02.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:20:02 smithi083 ceph-mon[77242]: pgmap v3355: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:20:02 smithi153 ceph-mon[70044]: pgmap v3355: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:02.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:20:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:20:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:20:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:20:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:20:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:20:04.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:20:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:20:03.939Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:20:04.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:20:04 smithi083 ceph-mon[79930]: pgmap v3356: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:04.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:20:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:20:04.635Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:20:04.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:20:04 smithi083 ceph-mon[77242]: pgmap v3356: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:05.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:20:04 smithi153 ceph-mon[70044]: pgmap v3356: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:06.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:20:06 smithi083 ceph-mon[79930]: pgmap v3357: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:06.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:20:06 smithi083 ceph-mon[77242]: pgmap v3357: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:07.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:20:06 smithi153 ceph-mon[70044]: pgmap v3357: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:08.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:20:08 smithi083 ceph-mon[79930]: pgmap v3358: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:08.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:20:08 smithi083 ceph-mon[77242]: pgmap v3358: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:09.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:20:08 smithi153 ceph-mon[70044]: pgmap v3358: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:10.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:20:10 smithi083 ceph-mon[79930]: pgmap v3359: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:10.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:20:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:20:10.533Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:20:10.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:20:10 smithi083 ceph-mon[77242]: pgmap v3359: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:11.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:20:10 smithi153 ceph-mon[70044]: pgmap v3359: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:20:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:20:11] "GET /metrics HTTP/1.1" 200 225171 "" "Prometheus/2.33.4" 2022-09-29T07:20:12.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:20:12 smithi083 ceph-mon[79930]: pgmap v3360: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:12.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:20:12 smithi083 ceph-mon[77242]: pgmap v3360: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:20:12 smithi153 ceph-mon[70044]: pgmap v3360: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:12.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:20:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:20:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:20:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:20:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:20:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:20:14.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:20:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:20:13.939Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:20:14.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:20:14 smithi083 ceph-mon[79930]: pgmap v3361: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:14.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:20:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:20:14.635Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:20:14.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:20:14 smithi083 ceph-mon[77242]: pgmap v3361: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:15.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:20:14 smithi153 ceph-mon[70044]: pgmap v3361: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:16.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:20:16 smithi083 ceph-mon[77242]: pgmap v3362: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:16.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:20:16 smithi083 ceph-mon[79930]: pgmap v3362: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:17.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:20:16 smithi153 ceph-mon[70044]: pgmap v3362: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:18.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:20:18 smithi083 ceph-mon[79930]: pgmap v3363: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:18.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:20:18 smithi083 ceph-mon[77242]: pgmap v3363: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:19.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:20:18 smithi153 ceph-mon[70044]: pgmap v3363: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:20.150 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:20:20.438 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:20:20.439 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (114m) 4m ago 114m 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:20:20.439 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (114m) 2m ago 114m 62.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:20:20.439 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (115m) 4m ago 115m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:20:20.439 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (115m) 2m ago 115m 27.8M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:20:20.440 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (111m) 2m ago 117m 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:20:20.440 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (111m) 4m ago 119m 608M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:20:20.440 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (110m) 4m ago 119m 459M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:20:20.440 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (110m) 2m ago 118m 209M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:20:20.441 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (110m) 4m ago 118m 270M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:20:20.441 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (114m) 4m ago 114m 25.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:20:20.441 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (114m) 2m ago 114m 26.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:20:20.441 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (109m) 4m ago 117m 335M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:20:20.441 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (108m) 4m ago 117m 327M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:20:20.442 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (108m) 4m ago 117m 261M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:20:20.442 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (108m) 4m ago 116m 315M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:20:20.442 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (107m) 2m ago 116m 284M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:20:20.442 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (107m) 2m ago 116m 345M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:20:20.443 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (106m) 2m ago 115m 306M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:20:20.443 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (105m) 2m ago 115m 356M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:20:20.443 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (111m) 2m ago 114m 116M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:20:20.799 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:20:20.799 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:20:20.799 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:20:20.800 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:20:20.800 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:20:20.800 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:20:20.800 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:20:20.800 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:20:20.801 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:20:20.801 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:20:20.801 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:20:20.801 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:20:20.801 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:20:20.802 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:20:20.802 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:20:20.802 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:20:20.802 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:20:20.803 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:20:20.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:20:20 smithi083 ceph-mon[79930]: pgmap v3364: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:20.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:20:20 smithi083 ceph-mon[77242]: pgmap v3364: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:20.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:20:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:20:20.535Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:20:21.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:20:20 smithi153 ceph-mon[70044]: pgmap v3364: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:20:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:20:21] "GET /metrics HTTP/1.1" 200 225171 "" "Prometheus/2.33.4" 2022-09-29T07:20:21.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:20:21 smithi083 ceph-mon[77242]: from='client.47983 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:20:21.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:20:21 smithi083 ceph-mon[77242]: from='client.38055 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:20:21.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:20:21 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3349302111' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:20:21.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:20:21 smithi083 ceph-mon[79930]: from='client.47983 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:20:21.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:20:21 smithi083 ceph-mon[79930]: from='client.38055 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:20:21.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:20:21 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3349302111' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:20:22.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:20:21 smithi153 ceph-mon[70044]: from='client.47983 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:20:22.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:20:21 smithi153 ceph-mon[70044]: from='client.38055 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:20:22.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:20:21 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3349302111' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:20:22.696 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:20:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:20:22.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:20:22.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:20:22 smithi083 ceph-mon[79930]: pgmap v3365: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:22.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:20:22 smithi083 ceph-mon[77242]: pgmap v3365: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:23.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:20:22 smithi153 ceph-mon[70044]: pgmap v3365: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:20:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:20:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:20:24.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:20:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:20:23.939Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:20:24.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:20:24 smithi083 ceph-mon[77242]: pgmap v3366: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:24.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:20:24 smithi083 ceph-mon[79930]: pgmap v3366: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:24.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:20:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:20:24.635Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:20:25.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:20:24 smithi153 ceph-mon[70044]: pgmap v3366: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:26.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:20:26 smithi083 ceph-mon[79930]: pgmap v3367: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:26.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:20:26 smithi083 ceph-mon[77242]: pgmap v3367: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:27.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:20:26 smithi153 ceph-mon[70044]: pgmap v3367: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:28.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:20:28 smithi083 ceph-mon[77242]: pgmap v3368: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:28.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:20:28 smithi083 ceph-mon[79930]: pgmap v3368: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:29.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:20:28 smithi153 ceph-mon[70044]: pgmap v3368: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:30.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:20:30 smithi083 ceph-mon[79930]: pgmap v3369: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:30.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:20:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:20:30.535Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:20:30.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:20:30 smithi083 ceph-mon[77242]: pgmap v3369: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:31.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:20:30 smithi153 ceph-mon[70044]: pgmap v3369: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:20:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:20:31] "GET /metrics HTTP/1.1" 200 225174 "" "Prometheus/2.33.4" 2022-09-29T07:20:32.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:20:32 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:20:32.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:20:32 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:20:32.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:20:32 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:20:32.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:20:32 smithi083 ceph-mon[79930]: pgmap v3370: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:32.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:20:32 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:20:32.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:20:32 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:20:32.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:20:32 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:20:32.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:20:32 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:20:32.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:20:32 smithi083 ceph-mon[77242]: pgmap v3370: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:32.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:20:32 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:20:32.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:20:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:20:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:20:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:20:32 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:20:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:20:32 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:20:32.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:20:32 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:20:32.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:20:32 smithi153 ceph-mon[70044]: pgmap v3370: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:32.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:20:32 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:20:33.311 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:20:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:20:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:20:34.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:20:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:20:33.940Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:20:34.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:20:34 smithi083 ceph-mon[79930]: pgmap v3371: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:34.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:20:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:20:34.636Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:20:34.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:20:34 smithi083 ceph-mon[77242]: pgmap v3371: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:35.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:20:34 smithi153 ceph-mon[70044]: pgmap v3371: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:36.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:20:36 smithi083 ceph-mon[77242]: pgmap v3372: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:36.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:20:36 smithi083 ceph-mon[79930]: pgmap v3372: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:37.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:20:36 smithi153 ceph-mon[70044]: pgmap v3372: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:38.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:20:38 smithi083 ceph-mon[79930]: pgmap v3373: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:38.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:20:38 smithi083 ceph-mon[77242]: pgmap v3373: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:39.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:20:38 smithi153 ceph-mon[70044]: pgmap v3373: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:39.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:20:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:20:39.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:20:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:20:39.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:20:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:20:39.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:20:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:20:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:20:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:20:39.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:20:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:20:39.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:20:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:20:39.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:20:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:20:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:20:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:20:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:20:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:20:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:20:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:20:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:20:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:20:40.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:20:40 smithi083 ceph-mon[79930]: pgmap v3374: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:40.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:20:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:20:40.536Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:20:40.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:20:40 smithi083 ceph-mon[77242]: pgmap v3374: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:20:40 smithi153 ceph-mon[70044]: pgmap v3374: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:20:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:20:41] "GET /metrics HTTP/1.1" 200 225174 "" "Prometheus/2.33.4" 2022-09-29T07:20:42.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:20:42 smithi083 ceph-mon[79930]: pgmap v3375: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:42.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:20:42 smithi083 ceph-mon[77242]: pgmap v3375: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:20:42 smithi153 ceph-mon[70044]: pgmap v3375: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:42.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:20:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:20:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:20:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:20:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:20:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:20:44.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:20:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:20:43.940Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:20:44.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:20:44 smithi083 ceph-mon[79930]: pgmap v3376: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:44.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:20:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:20:44.636Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:20:44.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:20:44 smithi083 ceph-mon[77242]: pgmap v3376: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:45.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:20:44 smithi153 ceph-mon[70044]: pgmap v3376: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:46.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:20:46 smithi083 ceph-mon[77242]: pgmap v3377: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:46.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:20:46 smithi083 ceph-mon[79930]: pgmap v3377: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:47.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:20:46 smithi153 ceph-mon[70044]: pgmap v3377: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:48.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:20:48 smithi083 ceph-mon[79930]: pgmap v3378: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:48.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:20:48 smithi083 ceph-mon[77242]: pgmap v3378: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:49.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:20:48 smithi153 ceph-mon[70044]: pgmap v3378: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:50.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:20:50 smithi083 ceph-mon[79930]: pgmap v3379: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:50.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:20:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:20:50.537Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:20:50.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:20:50 smithi083 ceph-mon[77242]: pgmap v3379: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:51.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:20:50 smithi153 ceph-mon[70044]: pgmap v3379: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:51.125 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:20:51.413 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:20:51.413 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (114m) 5m ago 115m 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:20:51.413 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (114m) 3m ago 114m 62.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:20:51.413 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (115m) 5m ago 115m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:20:51.414 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (115m) 3m ago 115m 27.8M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:20:51.414 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (111m) 3m ago 118m 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:20:51.414 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (112m) 5m ago 119m 608M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:20:51.414 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (111m) 5m ago 2h 459M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:20:51.415 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (110m) 3m ago 118m 209M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:20:51.415 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (110m) 5m ago 118m 270M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:20:51.415 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (115m) 5m ago 115m 25.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:20:51.415 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (115m) 3m ago 115m 26.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:20:51.416 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (110m) 5m ago 118m 335M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:20:51.416 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (109m) 5m ago 117m 327M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:20:51.416 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (109m) 5m ago 117m 261M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:20:51.416 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (108m) 5m ago 117m 315M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:20:51.416 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (108m) 3m ago 117m 284M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:20:51.417 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (107m) 3m ago 116m 345M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:20:51.417 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (106m) 3m ago 116m 306M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:20:51.417 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (106m) 3m ago 116m 356M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:20:51.417 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (111m) 3m ago 115m 116M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:20:51.766 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:20:51.767 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:20:51.767 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:20:51.767 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:20:51.767 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:20:51.768 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:20:51.768 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:20:51.768 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:20:51.768 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:20:51.768 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:20:51.769 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:20:51.769 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:20:51.769 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:20:51.769 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:20:51.770 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:20:51.770 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:20:51.770 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:20:51.770 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:20:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:20:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:20:51] "GET /metrics HTTP/1.1" 200 225174 "" "Prometheus/2.33.4" 2022-09-29T07:20:52.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:20:52 smithi083 ceph-mon[77242]: from='client.48001 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:20:52.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:20:52 smithi083 ceph-mon[77242]: from='client.48007 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:20:52.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:20:52 smithi083 ceph-mon[77242]: pgmap v3380: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:52.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:20:52 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/634235522' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:20:52.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:20:52 smithi083 ceph-mon[79930]: from='client.48001 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:20:52.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:20:52 smithi083 ceph-mon[79930]: from='client.48007 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:20:52.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:20:52 smithi083 ceph-mon[79930]: pgmap v3380: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:52.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:20:52 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/634235522' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:20:52.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:20:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:20:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:20:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:20:52 smithi153 ceph-mon[70044]: from='client.48001 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:20:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:20:52 smithi153 ceph-mon[70044]: from='client.48007 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:20:52.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:20:52 smithi153 ceph-mon[70044]: pgmap v3380: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:52.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:20:52 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/634235522' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:20:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:20:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:20:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:20:54.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:20:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:20:53.940Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:20:54.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:20:54 smithi083 ceph-mon[77242]: pgmap v3381: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:54.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:20:54 smithi083 ceph-mon[79930]: pgmap v3381: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:54.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:20:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:20:54.636Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:20:55.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:20:54 smithi153 ceph-mon[70044]: pgmap v3381: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:56.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:20:56 smithi083 ceph-mon[77242]: pgmap v3382: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:56.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:20:56 smithi083 ceph-mon[79930]: pgmap v3382: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:57.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:20:56 smithi153 ceph-mon[70044]: pgmap v3382: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:58.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:20:58 smithi083 ceph-mon[79930]: pgmap v3383: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:58.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:20:58 smithi083 ceph-mon[77242]: pgmap v3383: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:20:59.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:20:58 smithi153 ceph-mon[70044]: pgmap v3383: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:00.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:21:00 smithi083 ceph-mon[79930]: pgmap v3384: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:00.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:21:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:21:00.537Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:21:00.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:21:00 smithi083 ceph-mon[77242]: pgmap v3384: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:01.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:21:00 smithi153 ceph-mon[70044]: pgmap v3384: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:21:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:21:01] "GET /metrics HTTP/1.1" 200 225174 "" "Prometheus/2.33.4" 2022-09-29T07:21:02.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:21:02 smithi083 ceph-mon[77242]: pgmap v3385: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:02.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:21:02 smithi083 ceph-mon[79930]: pgmap v3385: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:21:02 smithi153 ceph-mon[70044]: pgmap v3385: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:02.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:21:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:21:02.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:21:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:21:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:21:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:21:04.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:21:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:21:03.941Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:21:04.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:21:04 smithi083 ceph-mon[79930]: pgmap v3386: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:04.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:21:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:21:04.637Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:21:04.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:21:04 smithi083 ceph-mon[77242]: pgmap v3386: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:05.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:21:04 smithi153 ceph-mon[70044]: pgmap v3386: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:06.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:21:06 smithi083 ceph-mon[79930]: pgmap v3387: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:06.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:21:06 smithi083 ceph-mon[77242]: pgmap v3387: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:07.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:21:06 smithi153 ceph-mon[70044]: pgmap v3387: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:08.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:21:08 smithi083 ceph-mon[77242]: pgmap v3388: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:08.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:21:08 smithi083 ceph-mon[79930]: pgmap v3388: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:09.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:21:08 smithi153 ceph-mon[70044]: pgmap v3388: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:10.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:21:10 smithi083 ceph-mon[79930]: pgmap v3389: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:10.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:21:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:21:10.538Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:21:10.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:21:10 smithi083 ceph-mon[77242]: pgmap v3389: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:11.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:21:10 smithi153 ceph-mon[70044]: pgmap v3389: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:21:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:21:11] "GET /metrics HTTP/1.1" 200 225178 "" "Prometheus/2.33.4" 2022-09-29T07:21:12.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:21:12 smithi083 ceph-mon[79930]: pgmap v3390: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:12.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:21:12 smithi083 ceph-mon[77242]: pgmap v3390: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:21:12 smithi153 ceph-mon[70044]: pgmap v3390: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:12.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:21:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:21:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:21:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:21:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:21:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:21:14.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:21:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:21:13.942Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:21:14.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:21:14 smithi083 ceph-mon[79930]: pgmap v3391: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:14.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:21:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:21:14.637Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:21:14.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:21:14 smithi083 ceph-mon[77242]: pgmap v3391: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:15.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:21:14 smithi153 ceph-mon[70044]: pgmap v3391: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:16.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:21:16 smithi083 ceph-mon[79930]: pgmap v3392: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:16.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:21:16 smithi083 ceph-mon[77242]: pgmap v3392: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:17.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:21:16 smithi153 ceph-mon[70044]: pgmap v3392: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:18.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:21:18 smithi083 ceph-mon[79930]: pgmap v3393: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:18.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:21:18 smithi083 ceph-mon[77242]: pgmap v3393: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:19.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:21:18 smithi153 ceph-mon[70044]: pgmap v3393: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:20.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:21:20 smithi083 ceph-mon[79930]: pgmap v3394: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:20.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:21:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:21:20.538Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:21:20.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:21:20 smithi083 ceph-mon[77242]: pgmap v3394: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:21.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:21:20 smithi153 ceph-mon[70044]: pgmap v3394: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:21:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:21:21] "GET /metrics HTTP/1.1" 200 225178 "" "Prometheus/2.33.4" 2022-09-29T07:21:22.095 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:21:22.388 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:21:22.388 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (115m) 5m ago 115m 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:21:22.389 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (115m) 3m ago 115m 62.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:21:22.389 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (116m) 5m ago 116m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:21:22.389 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (116m) 3m ago 116m 27.8M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:21:22.389 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (112m) 3m ago 118m 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:21:22.390 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (112m) 5m ago 2h 608M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:21:22.390 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (111m) 5m ago 2h 459M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:21:22.390 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (111m) 3m ago 119m 209M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:21:22.390 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (111m) 5m ago 119m 270M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:21:22.391 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (115m) 5m ago 115m 25.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:21:22.391 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (115m) 3m ago 115m 26.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:21:22.391 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (110m) 5m ago 118m 335M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:21:22.391 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (110m) 5m ago 118m 327M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:21:22.392 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (109m) 5m ago 118m 261M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:21:22.392 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (109m) 5m ago 117m 315M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:21:22.392 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (108m) 3m ago 117m 284M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:21:22.392 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (108m) 3m ago 117m 345M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:21:22.392 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (107m) 3m ago 116m 306M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:21:22.393 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (106m) 3m ago 116m 356M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:21:22.393 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (112m) 3m ago 115m 116M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:21:22.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:21:22 smithi083 ceph-mon[79930]: pgmap v3395: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:22.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:21:22 smithi083 ceph-mon[77242]: pgmap v3395: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:21:22 smithi153 ceph-mon[70044]: pgmap v3395: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:22.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:21:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:21:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:21:22.748 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:21:22.749 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:21:22.749 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:21:22.749 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:21:22.749 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:21:22.749 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:21:22.750 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:21:22.750 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:21:22.750 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:21:22.750 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:21:22.751 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:21:22.751 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:21:22.751 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:21:22.751 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:21:22.751 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:21:22.752 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:21:22.752 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:21:22.752 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:21:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:21:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:21:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:21:23.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:21:23 smithi153 ceph-mon[70044]: from='client.48019 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:21:23.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:21:23 smithi153 ceph-mon[70044]: from='client.38091 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:21:23.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:21:23 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1415642248' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:21:23.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:21:23 smithi083 ceph-mon[79930]: from='client.48019 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:21:23.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:21:23 smithi083 ceph-mon[79930]: from='client.38091 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:21:23.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:21:23 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1415642248' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:21:23.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:21:23 smithi083 ceph-mon[77242]: from='client.48019 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:21:23.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:21:23 smithi083 ceph-mon[77242]: from='client.38091 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:21:23.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:21:23 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1415642248' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:21:24.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:21:24 smithi083 ceph-mon[79930]: pgmap v3396: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:24.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:21:24 smithi083 ceph-mon[77242]: pgmap v3396: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:24.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:21:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:21:23.942Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:21:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:21:24 smithi153 ceph-mon[70044]: pgmap v3396: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:24.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:21:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:21:24.637Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:21:26.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:21:26 smithi083 ceph-mon[79930]: pgmap v3397: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:26.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:21:26 smithi083 ceph-mon[77242]: pgmap v3397: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:27.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:21:26 smithi153 ceph-mon[70044]: pgmap v3397: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:28.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:21:28 smithi083 ceph-mon[79930]: pgmap v3398: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:28.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:21:28 smithi083 ceph-mon[77242]: pgmap v3398: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:29.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:21:28 smithi153 ceph-mon[70044]: pgmap v3398: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:30.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:21:30 smithi083 ceph-mon[79930]: pgmap v3399: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:30.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:21:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:21:30.539Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:21:30.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:21:30 smithi083 ceph-mon[77242]: pgmap v3399: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:31.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:21:30 smithi153 ceph-mon[70044]: pgmap v3399: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:21:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:21:31] "GET /metrics HTTP/1.1" 200 225175 "" "Prometheus/2.33.4" 2022-09-29T07:21:32.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:21:32 smithi083 ceph-mon[77242]: pgmap v3400: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:32.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:21:32 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:21:32.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:21:32 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:21:32.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:21:32 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:21:32.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:21:32 smithi083 ceph-mon[79930]: pgmap v3400: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:32.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:21:32 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:21:32.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:21:32 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:21:32.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:21:32 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:21:32.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:21:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:21:32.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:21:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:21:32 smithi153 ceph-mon[70044]: pgmap v3400: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:21:32 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:21:32.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:21:32 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:21:32.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:21:32 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:21:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:21:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:21:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:21:33.943 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:21:33 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:21:33.944 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:21:33 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:21:34.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:21:33 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:21:34.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:21:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:21:33.942Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:21:34.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:21:34 smithi083 ceph-mon[77242]: pgmap v3401: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:34.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:21:34 smithi083 ceph-mon[79930]: pgmap v3401: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:34.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:21:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:21:34.638Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:21:35.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:21:34 smithi153 ceph-mon[70044]: pgmap v3401: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:36.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:21:36 smithi083 ceph-mon[79930]: pgmap v3402: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:36.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:21:36 smithi083 ceph-mon[77242]: pgmap v3402: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:37.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:21:36 smithi153 ceph-mon[70044]: pgmap v3402: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:38.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:21:38 smithi083 ceph-mon[79930]: pgmap v3403: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:38.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:21:38 smithi083 ceph-mon[77242]: pgmap v3403: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:39.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:21:38 smithi153 ceph-mon[70044]: pgmap v3403: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:39.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:21:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:21:39.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:21:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:21:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:21:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:21:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:21:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:21:39.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:21:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:21:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:21:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:21:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:21:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:21:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:21:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:21:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:21:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:21:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:21:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:21:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:21:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:21:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:21:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:21:40.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:21:40 smithi083 ceph-mon[79930]: pgmap v3404: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:40.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:21:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:21:40.540Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:21:40.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:21:40 smithi083 ceph-mon[77242]: pgmap v3404: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:21:40 smithi153 ceph-mon[70044]: pgmap v3404: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:21:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:21:41] "GET /metrics HTTP/1.1" 200 225174 "" "Prometheus/2.33.4" 2022-09-29T07:21:42.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:21:42 smithi083 ceph-mon[79930]: pgmap v3405: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:42.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:21:42 smithi083 ceph-mon[77242]: pgmap v3405: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:21:42 smithi153 ceph-mon[70044]: pgmap v3405: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:42.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:21:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:21:42.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:21:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:21:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:21:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:21:44.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:21:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:21:43.943Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:21:44.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:21:44 smithi083 ceph-mon[79930]: pgmap v3406: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:44.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:21:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:21:44.639Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:21:44.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:21:44 smithi083 ceph-mon[77242]: pgmap v3406: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:45.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:21:44 smithi153 ceph-mon[70044]: pgmap v3406: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:47.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:21:46 smithi153 ceph-mon[70044]: pgmap v3407: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:47.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:21:46 smithi083 ceph-mon[77242]: pgmap v3407: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:47.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:21:46 smithi083 ceph-mon[79930]: pgmap v3407: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:49.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:21:48 smithi153 ceph-mon[70044]: pgmap v3408: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:49.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:21:48 smithi083 ceph-mon[79930]: pgmap v3408: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:49.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:21:48 smithi083 ceph-mon[77242]: pgmap v3408: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:50.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:21:50 smithi083 ceph-mon[79930]: pgmap v3409: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:50.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:21:50 smithi083 ceph-mon[77242]: pgmap v3409: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:50.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:21:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:21:50.541Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:21:51.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:21:50 smithi153 ceph-mon[70044]: pgmap v3409: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:21:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:21:51] "GET /metrics HTTP/1.1" 200 225174 "" "Prometheus/2.33.4" 2022-09-29T07:21:52.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:21:52 smithi083 ceph-mon[79930]: pgmap v3410: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:52.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:21:52 smithi083 ceph-mon[77242]: pgmap v3410: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:21:52 smithi153 ceph-mon[70044]: pgmap v3410: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:52.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:21:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:21:52.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:21:53.071 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:21:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:21:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:21:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:21:53.365 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:21:53.365 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (115m) 6m ago 116m 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:21:53.365 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (115m) 4m ago 115m 62.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:21:53.366 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (116m) 6m ago 116m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:21:53.366 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (116m) 4m ago 116m 27.8M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:21:53.366 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (112m) 4m ago 119m 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:21:53.367 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (113m) 6m ago 2h 608M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:21:53.367 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (112m) 6m ago 2h 459M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:21:53.367 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (111m) 4m ago 119m 209M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:21:53.367 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (111m) 6m ago 119m 270M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:21:53.368 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (116m) 6m ago 116m 25.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:21:53.368 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (116m) 4m ago 116m 26.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:21:53.368 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (111m) 6m ago 119m 335M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:21:53.368 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (110m) 6m ago 118m 327M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:21:53.368 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (110m) 6m ago 118m 261M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:21:53.369 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (109m) 6m ago 118m 315M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:21:53.369 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (109m) 4m ago 118m 284M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:21:53.369 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (108m) 4m ago 117m 345M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:21:53.369 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (108m) 4m ago 117m 306M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:21:53.369 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (107m) 4m ago 117m 356M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:21:53.370 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (112m) 4m ago 116m 116M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:21:53.720 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:21:53.720 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:21:53.720 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:21:53.721 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:21:53.721 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:21:53.721 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:21:53.721 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:21:53.722 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:21:53.722 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:21:53.722 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:21:53.722 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:21:53.723 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:21:53.723 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:21:53.723 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:21:53.723 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:21:53.723 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:21:53.724 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:21:53.724 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:21:54.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:21:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:21:53.943Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:21:54.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:21:54 smithi083 ceph-mon[77242]: from='client.38103 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:21:54.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:21:54 smithi083 ceph-mon[77242]: from='client.48043 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:21:54.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:21:54 smithi083 ceph-mon[77242]: pgmap v3411: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:54.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:21:54 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2625682494' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:21:54.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:21:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:21:54.639Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:21:54.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:21:54 smithi083 ceph-mon[79930]: from='client.38103 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:21:54.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:21:54 smithi083 ceph-mon[79930]: from='client.48043 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:21:54.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:21:54 smithi083 ceph-mon[79930]: pgmap v3411: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:54.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:21:54 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2625682494' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:21:55.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:21:54 smithi153 ceph-mon[70044]: from='client.38103 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:21:55.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:21:54 smithi153 ceph-mon[70044]: from='client.48043 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:21:55.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:21:54 smithi153 ceph-mon[70044]: pgmap v3411: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:55.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:21:54 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2625682494' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:21:57.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:21:56 smithi153 ceph-mon[70044]: pgmap v3412: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:57.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:21:56 smithi083 ceph-mon[77242]: pgmap v3412: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:57.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:21:56 smithi083 ceph-mon[79930]: pgmap v3412: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:59.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:21:58 smithi153 ceph-mon[70044]: pgmap v3413: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:59.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:21:58 smithi083 ceph-mon[77242]: pgmap v3413: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:21:59.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:21:58 smithi083 ceph-mon[79930]: pgmap v3413: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:00.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:22:00 smithi083 ceph-mon[79930]: pgmap v3414: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:00.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:22:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:22:00.541Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:22:00.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:22:00 smithi083 ceph-mon[77242]: pgmap v3414: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:01.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:22:00 smithi153 ceph-mon[70044]: pgmap v3414: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:22:01] "GET /metrics HTTP/1.1" 200 225176 "" "Prometheus/2.33.4" 2022-09-29T07:22:02.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:22:02 smithi083 ceph-mon[77242]: pgmap v3415: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:02.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:22:02 smithi083 ceph-mon[79930]: pgmap v3415: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:22:02 smithi153 ceph-mon[70044]: pgmap v3415: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:02.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:22:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:22:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:22:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:22:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:22:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:22:04.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:22:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:22:03.944Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:22:04.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:22:04 smithi083 ceph-mon[79930]: pgmap v3416: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:04.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:22:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:22:04.640Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:22:04.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:22:04 smithi083 ceph-mon[77242]: pgmap v3416: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:05.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:22:04 smithi153 ceph-mon[70044]: pgmap v3416: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:07.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:22:06 smithi153 ceph-mon[70044]: pgmap v3417: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:07.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:22:06 smithi083 ceph-mon[79930]: pgmap v3417: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:07.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:22:06 smithi083 ceph-mon[77242]: pgmap v3417: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:09.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:22:08 smithi153 ceph-mon[70044]: pgmap v3418: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:09.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:22:08 smithi083 ceph-mon[77242]: pgmap v3418: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:09.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:22:08 smithi083 ceph-mon[79930]: pgmap v3418: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:10.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:22:10 smithi083 ceph-mon[79930]: pgmap v3419: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:10.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:22:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:22:10.542Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:22:10.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:22:10 smithi083 ceph-mon[77242]: pgmap v3419: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:11.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:22:10 smithi153 ceph-mon[70044]: pgmap v3419: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:22:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:22:11] "GET /metrics HTTP/1.1" 200 225175 "" "Prometheus/2.33.4" 2022-09-29T07:22:12.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:22:12 smithi083 ceph-mon[79930]: pgmap v3420: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:12.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:22:12 smithi083 ceph-mon[77242]: pgmap v3420: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:22:12 smithi153 ceph-mon[70044]: pgmap v3420: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:12.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:22:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:22:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:22:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:22:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:22:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:22:14.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:22:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:22:13.944Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:22:14.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:22:14 smithi083 ceph-mon[79930]: pgmap v3421: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:14.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:22:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:22:14.642Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:22:14.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:22:14 smithi083 ceph-mon[77242]: pgmap v3421: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:15.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:22:14 smithi153 ceph-mon[70044]: pgmap v3421: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:17.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:22:16 smithi153 ceph-mon[70044]: pgmap v3422: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:17.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:22:16 smithi083 ceph-mon[77242]: pgmap v3422: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:17.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:22:16 smithi083 ceph-mon[79930]: pgmap v3422: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:19.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:22:18 smithi153 ceph-mon[70044]: pgmap v3423: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:19.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:22:18 smithi083 ceph-mon[79930]: pgmap v3423: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:19.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:22:18 smithi083 ceph-mon[77242]: pgmap v3423: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:20.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:22:20 smithi083 ceph-mon[79930]: pgmap v3424: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:20.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:22:20 smithi083 ceph-mon[77242]: pgmap v3424: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:20.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:22:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:22:20.542Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:22:21.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:22:20 smithi153 ceph-mon[70044]: pgmap v3424: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:22:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:22:21] "GET /metrics HTTP/1.1" 200 225175 "" "Prometheus/2.33.4" 2022-09-29T07:22:22.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:22:22 smithi083 ceph-mon[79930]: pgmap v3425: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:22.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:22:22 smithi083 ceph-mon[77242]: pgmap v3425: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:22:22 smithi153 ceph-mon[70044]: pgmap v3425: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:22.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:22:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:22:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:22:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:22:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:22:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:22:24.042 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:22:24.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:22:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:22:23.944Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:22:24.332 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:22:24.332 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (116m) 7m ago 116m 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:22:24.332 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (116m) 4m ago 116m 62.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:22:24.333 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (117m) 7m ago 117m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:22:24.333 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (117m) 4m ago 117m 27.8M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:22:24.333 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (113m) 4m ago 2h 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:22:24.333 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (113m) 7m ago 2h 608M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:22:24.333 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (112m) 7m ago 2h 459M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:22:24.334 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (112m) 4m ago 2h 209M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:22:24.334 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (112m) 7m ago 2h 270M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:22:24.334 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (116m) 7m ago 116m 25.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:22:24.334 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (116m) 4m ago 116m 26.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:22:24.335 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (111m) 7m ago 119m 335M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:22:24.335 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (111m) 7m ago 119m 327M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:22:24.335 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (110m) 7m ago 119m 261M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:22:24.335 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (110m) 7m ago 118m 315M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:22:24.335 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (109m) 4m ago 118m 284M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:22:24.336 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (109m) 4m ago 118m 345M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:22:24.336 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (108m) 4m ago 117m 306M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:22:24.336 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (107m) 4m ago 117m 356M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:22:24.336 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (113m) 4m ago 116m 116M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:22:24.692 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:22:24.692 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:22:24.692 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:22:24.692 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:22:24.693 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:22:24.693 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:22:24.693 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:22:24.693 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:22:24.694 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:22:24.694 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:22:24.694 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:22:24.694 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:22:24.695 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:22:24.695 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:22:24.695 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:22:24.695 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:22:24.695 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:22:24.696 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:22:24.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:22:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:22:24.642Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:22:24.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:22:24 smithi083 ceph-mon[77242]: pgmap v3426: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:24.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:22:24 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1770683659' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:22:24.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:22:24 smithi083 ceph-mon[79930]: pgmap v3426: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:24.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:22:24 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1770683659' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:22:25.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:22:24 smithi153 ceph-mon[70044]: pgmap v3426: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:25.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:22:24 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1770683659' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:22:26.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:22:25 smithi153 ceph-mon[70044]: from='client.48055 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:22:26.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:22:25 smithi153 ceph-mon[70044]: from='client.48061 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:22:26.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:22:25 smithi083 ceph-mon[79930]: from='client.48055 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:22:26.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:22:25 smithi083 ceph-mon[79930]: from='client.48061 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:22:26.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:22:25 smithi083 ceph-mon[77242]: from='client.48055 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:22:26.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:22:25 smithi083 ceph-mon[77242]: from='client.48061 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:22:27.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:22:26 smithi153 ceph-mon[70044]: pgmap v3427: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:27.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:22:26 smithi083 ceph-mon[79930]: pgmap v3427: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:27.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:22:26 smithi083 ceph-mon[77242]: pgmap v3427: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:29.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:22:28 smithi153 ceph-mon[70044]: pgmap v3428: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:29.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:22:28 smithi083 ceph-mon[79930]: pgmap v3428: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:29.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:22:28 smithi083 ceph-mon[77242]: pgmap v3428: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:30.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:22:30 smithi083 ceph-mon[79930]: pgmap v3429: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:30.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:22:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:22:30.543Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:22:30.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:22:30 smithi083 ceph-mon[77242]: pgmap v3429: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:31.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:22:30 smithi153 ceph-mon[70044]: pgmap v3429: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:22:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:22:31] "GET /metrics HTTP/1.1" 200 225176 "" "Prometheus/2.33.4" 2022-09-29T07:22:32.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:22:32 smithi083 ceph-mon[79930]: pgmap v3430: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:32.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:22:32 smithi083 ceph-mon[77242]: pgmap v3430: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:22:32 smithi153 ceph-mon[70044]: pgmap v3430: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:32.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:22:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:22:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:22:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:22:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:22:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:22:33.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:22:33 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:22:33.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:22:33 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:22:33.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:22:33 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:22:33.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:22:33 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:22:33.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:22:33 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:22:33.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:22:33 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:22:33.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:22:33 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:22:33.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:22:33 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:22:33.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:22:33 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:22:34.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:22:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:22:33.945Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:22:34.643 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:22:34 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:22:34.644 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:22:34 smithi083 ceph-mon[79930]: pgmap v3431: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:34.644 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:22:34 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:22:34.644 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:22:34 smithi083 ceph-mon[77242]: pgmap v3431: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:34.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:22:34 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:22:34.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:22:34 smithi153 ceph-mon[70044]: pgmap v3431: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:34.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:22:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:22:34.642Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:22:37.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:22:36 smithi153 ceph-mon[70044]: pgmap v3432: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:37.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:22:36 smithi083 ceph-mon[79930]: pgmap v3432: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:37.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:22:36 smithi083 ceph-mon[77242]: pgmap v3432: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:39.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:22:38 smithi153 ceph-mon[70044]: pgmap v3433: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:39.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:22:38 smithi083 ceph-mon[77242]: pgmap v3433: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:39.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:22:38 smithi083 ceph-mon[79930]: pgmap v3433: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:22:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:22:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:22:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:22:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:22:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:22:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:22:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:22:40.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:22:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:22:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:22:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:22:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:22:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:22:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:22:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:22:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:22:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:22:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:22:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:22:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:22:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:22:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:22:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:22:40.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:22:40 smithi083 ceph-mon[77242]: pgmap v3434: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:22:40 smithi083 ceph-mon[79930]: pgmap v3434: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:40.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:22:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:22:40.544Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:22:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:22:40 smithi153 ceph-mon[70044]: pgmap v3434: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:22:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:22:41] "GET /metrics HTTP/1.1" 200 225175 "" "Prometheus/2.33.4" 2022-09-29T07:22:42.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:22:42 smithi083 ceph-mon[77242]: pgmap v3435: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:22:42 smithi083 ceph-mon[79930]: pgmap v3435: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:22:42 smithi153 ceph-mon[70044]: pgmap v3435: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:42.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:22:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:22:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:22:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:22:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:22:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:22:44.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:22:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:22:43.947Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:22:44.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:22:44 smithi083 ceph-mon[79930]: pgmap v3436: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:44.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:22:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:22:44.643Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:22:44.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:22:44 smithi083 ceph-mon[77242]: pgmap v3436: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:45.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:22:44 smithi153 ceph-mon[70044]: pgmap v3436: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:47.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:22:46 smithi153 ceph-mon[70044]: pgmap v3437: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:47.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:22:46 smithi083 ceph-mon[79930]: pgmap v3437: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:47.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:22:46 smithi083 ceph-mon[77242]: pgmap v3437: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:49.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:22:48 smithi153 ceph-mon[70044]: pgmap v3438: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:49.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:22:48 smithi083 ceph-mon[77242]: pgmap v3438: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:49.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:22:48 smithi083 ceph-mon[79930]: pgmap v3438: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:50.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:22:50 smithi083 ceph-mon[77242]: pgmap v3439: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:50.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:22:50 smithi083 ceph-mon[79930]: pgmap v3439: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:50.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:22:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:22:50.544Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:22:51.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:22:50 smithi153 ceph-mon[70044]: pgmap v3439: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:22:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:22:51] "GET /metrics HTTP/1.1" 200 225175 "" "Prometheus/2.33.4" 2022-09-29T07:22:52.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:22:52 smithi083 ceph-mon[77242]: pgmap v3440: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:52.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:22:52 smithi083 ceph-mon[79930]: pgmap v3440: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:22:52 smithi153 ceph-mon[70044]: pgmap v3440: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:52.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:22:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:22:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:22:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:22:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:22:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:22:54.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:22:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:22:53.947Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:22:54.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:22:54 smithi083 ceph-mon[77242]: pgmap v3441: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:54.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:22:54 smithi083 ceph-mon[79930]: pgmap v3441: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:54.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:22:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:22:54.643Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:22:55.021 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:22:55.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:22:54 smithi153 ceph-mon[70044]: pgmap v3441: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:55.310 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:22:55.310 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (116m) 7m ago 117m 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:22:55.311 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (116m) 5m ago 116m 62.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:22:55.311 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (117m) 7m ago 117m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:22:55.311 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (117m) 5m ago 117m 27.8M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:22:55.311 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (113m) 5m ago 2h 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:22:55.312 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (114m) 7m ago 2h 608M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:22:55.312 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (113m) 7m ago 2h 459M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:22:55.312 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (112m) 5m ago 2h 209M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:22:55.312 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (113m) 7m ago 2h 270M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:22:55.313 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (117m) 7m ago 117m 25.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:22:55.313 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (117m) 5m ago 117m 26.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:22:55.313 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (112m) 7m ago 2h 335M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:22:55.313 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (111m) 7m ago 119m 327M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:22:55.313 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (111m) 7m ago 119m 261M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:22:55.314 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (110m) 7m ago 119m 315M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:22:55.314 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (110m) 5m ago 119m 284M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:22:55.314 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (109m) 5m ago 118m 345M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:22:55.314 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (109m) 5m ago 118m 306M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:22:55.315 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (108m) 5m ago 118m 356M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:22:55.315 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (113m) 5m ago 117m 116M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:22:55.667 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:22:55.668 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:22:55.668 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:22:55.668 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:22:55.668 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:22:55.669 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:22:55.669 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:22:55.669 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:22:55.669 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:22:55.670 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:22:55.670 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:22:55.670 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:22:55.670 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:22:55.671 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:22:55.671 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:22:55.671 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:22:55.671 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:22:55.671 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:22:56.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:22:55 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2894637048' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:22:56.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:22:55 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2894637048' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:22:56.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:22:55 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2894637048' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:22:57.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:22:56 smithi153 ceph-mon[70044]: from='client.48073 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:22:57.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:22:56 smithi153 ceph-mon[70044]: from='client.38145 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:22:57.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:22:56 smithi153 ceph-mon[70044]: pgmap v3442: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:57.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:22:56 smithi083 ceph-mon[79930]: from='client.48073 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:22:57.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:22:56 smithi083 ceph-mon[79930]: from='client.38145 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:22:57.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:22:56 smithi083 ceph-mon[79930]: pgmap v3442: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:57.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:22:56 smithi083 ceph-mon[77242]: from='client.48073 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:22:57.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:22:56 smithi083 ceph-mon[77242]: from='client.38145 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:22:57.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:22:56 smithi083 ceph-mon[77242]: pgmap v3442: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:59.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:22:58 smithi153 ceph-mon[70044]: pgmap v3443: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:59.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:22:58 smithi083 ceph-mon[79930]: pgmap v3443: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:22:59.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:22:58 smithi083 ceph-mon[77242]: pgmap v3443: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:00.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:23:00 smithi083 ceph-mon[77242]: pgmap v3444: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:00.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:23:00 smithi083 ceph-mon[79930]: pgmap v3444: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:00.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:23:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:23:00.545Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:23:01.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:23:00 smithi153 ceph-mon[70044]: pgmap v3444: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:23:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:23:01] "GET /metrics HTTP/1.1" 200 225177 "" "Prometheus/2.33.4" 2022-09-29T07:23:02.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:23:02 smithi083 ceph-mon[79930]: pgmap v3445: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:02.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:23:02 smithi083 ceph-mon[77242]: pgmap v3445: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:23:02 smithi153 ceph-mon[70044]: pgmap v3445: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:02.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:23:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:23:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:23:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:23:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:23:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:23:04.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:23:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:23:03.948Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:23:04.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:23:04 smithi083 ceph-mon[79930]: pgmap v3446: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:04.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:23:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:23:04.644Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:23:04.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:23:04 smithi083 ceph-mon[77242]: pgmap v3446: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:05.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:23:04 smithi153 ceph-mon[70044]: pgmap v3446: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:07.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:23:06 smithi153 ceph-mon[70044]: pgmap v3447: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:07.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:23:06 smithi083 ceph-mon[79930]: pgmap v3447: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:07.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:23:06 smithi083 ceph-mon[77242]: pgmap v3447: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:09.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:23:08 smithi153 ceph-mon[70044]: pgmap v3448: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:09.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:23:08 smithi083 ceph-mon[79930]: pgmap v3448: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:09.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:23:08 smithi083 ceph-mon[77242]: pgmap v3448: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:10.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:23:10 smithi083 ceph-mon[79930]: pgmap v3449: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:10.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:23:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:23:10.546Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:23:10.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:23:10 smithi083 ceph-mon[77242]: pgmap v3449: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:11.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:23:10 smithi153 ceph-mon[70044]: pgmap v3449: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:23:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:23:11] "GET /metrics HTTP/1.1" 200 225174 "" "Prometheus/2.33.4" 2022-09-29T07:23:12.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:23:12 smithi083 ceph-mon[79930]: pgmap v3450: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:12.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:23:12 smithi083 ceph-mon[77242]: pgmap v3450: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:23:12 smithi153 ceph-mon[70044]: pgmap v3450: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:12.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:23:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:23:12.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:23:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:23:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:23:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:23:14.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:23:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:23:13.949Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:23:14.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:23:14 smithi083 ceph-mon[79930]: pgmap v3451: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:14.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:23:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:23:14.645Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:23:14.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:23:14 smithi083 ceph-mon[77242]: pgmap v3451: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:15.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:23:14 smithi153 ceph-mon[70044]: pgmap v3451: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:17.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:23:16 smithi153 ceph-mon[70044]: pgmap v3452: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:17.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:23:16 smithi083 ceph-mon[77242]: pgmap v3452: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:17.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:23:16 smithi083 ceph-mon[79930]: pgmap v3452: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:19.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:23:18 smithi153 ceph-mon[70044]: pgmap v3453: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:19.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:23:18 smithi083 ceph-mon[77242]: pgmap v3453: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:19.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:23:18 smithi083 ceph-mon[79930]: pgmap v3453: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:20.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:23:20 smithi083 ceph-mon[77242]: pgmap v3454: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:20.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:23:20 smithi083 ceph-mon[79930]: pgmap v3454: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:20.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:23:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:23:20.546Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:23:21.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:23:20 smithi153 ceph-mon[70044]: pgmap v3454: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:23:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:23:21] "GET /metrics HTTP/1.1" 200 225174 "" "Prometheus/2.33.4" 2022-09-29T07:23:22.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:23:22 smithi083 ceph-mon[79930]: pgmap v3455: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:22.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:23:22 smithi083 ceph-mon[77242]: pgmap v3455: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:23:22 smithi153 ceph-mon[70044]: pgmap v3455: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:22.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:23:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:23:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:23:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:23:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:23:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:23:24.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:23:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:23:23.950Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:23:24.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:23:24 smithi083 ceph-mon[79930]: pgmap v3456: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:24.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:23:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:23:24.645Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:23:24.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:23:24 smithi083 ceph-mon[77242]: pgmap v3456: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:25.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:23:24 smithi153 ceph-mon[70044]: pgmap v3456: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:25.994 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:23:26.283 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:23:26.283 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (117m) 8m ago 117m 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:23:26.284 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (117m) 5m ago 117m 62.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:23:26.284 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (118m) 8m ago 118m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:23:26.284 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (118m) 5m ago 118m 27.8M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:23:26.285 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (114m) 5m ago 2h 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:23:26.285 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (114m) 8m ago 2h 608M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:23:26.285 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (113m) 8m ago 2h 459M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:23:26.285 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (113m) 5m ago 2h 209M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:23:26.286 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (113m) 8m ago 2h 270M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:23:26.286 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (117m) 8m ago 117m 25.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:23:26.286 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (117m) 5m ago 117m 26.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:23:26.286 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (112m) 8m ago 2h 335M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:23:26.287 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (112m) 8m ago 2h 327M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:23:26.287 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (111m) 8m ago 2h 261M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:23:26.287 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (111m) 8m ago 119m 315M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:23:26.287 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (110m) 5m ago 119m 284M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:23:26.288 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (110m) 5m ago 119m 345M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:23:26.288 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (109m) 5m ago 119m 306M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:23:26.288 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (109m) 5m ago 118m 356M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:23:26.288 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (114m) 5m ago 117m 116M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:23:26.637 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:23:26.637 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:23:26.637 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:23:26.638 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:23:26.638 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:23:26.638 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:23:26.638 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:23:26.639 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:23:26.639 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:23:26.639 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:23:26.640 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:23:26.640 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:23:26.640 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:23:26.640 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:23:26.640 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:23:26.641 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:23:26.641 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:23:26.641 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:23:27.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:23:26 smithi153 ceph-mon[70044]: pgmap v3457: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:27.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:23:26 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3543632130' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:23:27.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:23:26 smithi083 ceph-mon[77242]: pgmap v3457: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:27.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:23:26 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3543632130' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:23:27.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:23:26 smithi083 ceph-mon[79930]: pgmap v3457: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:27.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:23:26 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3543632130' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:23:28.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:23:27 smithi153 ceph-mon[70044]: from='client.38157 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:23:28.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:23:27 smithi153 ceph-mon[70044]: from='client.38163 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:23:28.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:23:27 smithi083 ceph-mon[79930]: from='client.38157 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:23:28.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:23:27 smithi083 ceph-mon[79930]: from='client.38163 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:23:28.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:23:27 smithi083 ceph-mon[77242]: from='client.38157 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:23:28.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:23:27 smithi083 ceph-mon[77242]: from='client.38163 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:23:29.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:23:28 smithi153 ceph-mon[70044]: pgmap v3458: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:29.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:23:28 smithi083 ceph-mon[79930]: pgmap v3458: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:29.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:23:28 smithi083 ceph-mon[77242]: pgmap v3458: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:30.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:23:30 smithi083 ceph-mon[79930]: pgmap v3459: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:30.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:23:30 smithi083 ceph-mon[77242]: pgmap v3459: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:30.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:23:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:23:30.547Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:23:31.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:23:30 smithi153 ceph-mon[70044]: pgmap v3459: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:23:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:23:31] "GET /metrics HTTP/1.1" 200 225175 "" "Prometheus/2.33.4" 2022-09-29T07:23:32.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:23:32 smithi083 ceph-mon[79930]: pgmap v3460: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:32.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:23:32 smithi083 ceph-mon[77242]: pgmap v3460: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:23:32 smithi153 ceph-mon[70044]: pgmap v3460: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:32.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:23:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:23:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:23:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:23:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:23:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:23:33.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:23:33 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:23:33.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:23:33 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:23:33.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:23:33 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:23:33.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:23:33 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:23:33.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:23:33 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:23:33.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:23:33 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:23:33.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:23:33 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:23:33.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:23:33 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:23:33.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:23:33 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:23:34.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:23:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:23:33.964Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:23:34.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:23:34 smithi153 ceph-mon[70044]: pgmap v3461: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:34.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:23:34 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:23:34.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:23:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:23:34.646Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:23:34.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:23:34 smithi083 ceph-mon[77242]: pgmap v3461: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:34.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:23:34 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:23:34.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:23:34 smithi083 ceph-mon[79930]: pgmap v3461: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:34.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:23:34 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:23:37.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:23:36 smithi153 ceph-mon[70044]: pgmap v3462: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:37.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:23:36 smithi083 ceph-mon[79930]: pgmap v3462: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:37.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:23:36 smithi083 ceph-mon[77242]: pgmap v3462: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:39.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:23:38 smithi153 ceph-mon[70044]: pgmap v3463: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:39.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:23:38 smithi083 ceph-mon[79930]: pgmap v3463: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:39.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:23:38 smithi083 ceph-mon[77242]: pgmap v3463: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:23:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:23:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:23:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:23:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:23:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:23:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:23:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:23:40.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:23:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:23:40.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:23:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:23:40.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:23:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:23:40.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:23:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:23:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:23:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:23:40.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:23:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:23:40.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:23:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:23:40.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:23:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:23:40.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:23:40 smithi083 ceph-mon[79930]: pgmap v3464: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:40.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:23:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:23:40.547Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:23:40.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:23:40 smithi083 ceph-mon[77242]: pgmap v3464: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:23:40 smithi153 ceph-mon[70044]: pgmap v3464: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:23:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:23:41] "GET /metrics HTTP/1.1" 200 225176 "" "Prometheus/2.33.4" 2022-09-29T07:23:42.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:23:42 smithi083 ceph-mon[79930]: pgmap v3465: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:42.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:23:42 smithi083 ceph-mon[77242]: pgmap v3465: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:23:42 smithi153 ceph-mon[70044]: pgmap v3465: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:42.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:23:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:23:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:23:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:23:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:23:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:23:44.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:23:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:23:43.951Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:23:44.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:23:44 smithi083 ceph-mon[79930]: pgmap v3466: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:44.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:23:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:23:44.647Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:23:44.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:23:44 smithi083 ceph-mon[77242]: pgmap v3466: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:45.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:23:44 smithi153 ceph-mon[70044]: pgmap v3466: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:47.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:23:46 smithi153 ceph-mon[70044]: pgmap v3467: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:47.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:23:46 smithi083 ceph-mon[79930]: pgmap v3467: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:47.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:23:46 smithi083 ceph-mon[77242]: pgmap v3467: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:49.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:23:48 smithi153 ceph-mon[70044]: pgmap v3468: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:49.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:23:48 smithi083 ceph-mon[79930]: pgmap v3468: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:49.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:23:48 smithi083 ceph-mon[77242]: pgmap v3468: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:50.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:23:50 smithi083 ceph-mon[79930]: pgmap v3469: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:50.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:23:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:23:50.548Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:23:50.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:23:50 smithi083 ceph-mon[77242]: pgmap v3469: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:51.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:23:50 smithi153 ceph-mon[70044]: pgmap v3469: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:23:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:23:51] "GET /metrics HTTP/1.1" 200 225176 "" "Prometheus/2.33.4" 2022-09-29T07:23:52.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:23:52 smithi083 ceph-mon[77242]: pgmap v3470: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:52.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:23:52 smithi083 ceph-mon[79930]: pgmap v3470: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:23:52 smithi153 ceph-mon[70044]: pgmap v3470: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:52.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:23:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:23:52.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:23:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:23:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:23:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:23:54.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:23:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:23:53.951Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:23:54.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:23:54 smithi083 ceph-mon[77242]: pgmap v3471: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:54.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:23:54 smithi083 ceph-mon[79930]: pgmap v3471: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:54.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:23:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:23:54.647Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:23:55.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:23:54 smithi153 ceph-mon[70044]: pgmap v3471: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:56.966 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:23:57.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:23:56 smithi153 ceph-mon[70044]: pgmap v3472: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:57.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:23:56 smithi083 ceph-mon[79930]: pgmap v3472: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:57.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:23:56 smithi083 ceph-mon[77242]: pgmap v3472: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:57.256 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:23:57.256 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (117m) 8m ago 118m 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:23:57.256 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (117m) 6m ago 117m 62.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:23:57.257 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (119m) 8m ago 119m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:23:57.257 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (118m) 6m ago 118m 27.8M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:23:57.257 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (114m) 6m ago 2h 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:23:57.257 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (115m) 8m ago 2h 608M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:23:57.258 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (114m) 8m ago 2h 459M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:23:57.258 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (113m) 6m ago 2h 209M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:23:57.258 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (114m) 8m ago 2h 270M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:23:57.258 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (118m) 8m ago 118m 25.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:23:57.259 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (118m) 6m ago 118m 26.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:23:57.259 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (113m) 8m ago 2h 335M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:23:57.259 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (112m) 8m ago 2h 327M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:23:57.259 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (112m) 8m ago 2h 261M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:23:57.259 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (111m) 8m ago 2h 315M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:23:57.260 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (111m) 6m ago 2h 284M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:23:57.260 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (110m) 6m ago 119m 345M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:23:57.260 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (110m) 6m ago 119m 306M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:23:57.260 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (109m) 6m ago 119m 356M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:23:57.261 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (114m) 6m ago 118m 116M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:23:57.613 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:23:57.614 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:23:57.614 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:23:57.614 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:23:57.615 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:23:57.615 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:23:57.615 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:23:57.615 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:23:57.616 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:23:57.616 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:23:57.616 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:23:57.616 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:23:57.616 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:23:57.617 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:23:57.617 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:23:57.617 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:23:57.617 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:23:57.618 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:23:58.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:23:57 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2636012009' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:23:58.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:23:57 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2636012009' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:23:58.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:23:57 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2636012009' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:23:59.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:23:58 smithi153 ceph-mon[70044]: from='client.48109 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:23:59.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:23:58 smithi153 ceph-mon[70044]: from='client.38181 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:23:59.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:23:58 smithi153 ceph-mon[70044]: pgmap v3473: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:59.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:23:58 smithi083 ceph-mon[79930]: from='client.48109 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:23:59.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:23:58 smithi083 ceph-mon[79930]: from='client.38181 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:23:59.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:23:58 smithi083 ceph-mon[79930]: pgmap v3473: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:23:59.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:23:58 smithi083 ceph-mon[77242]: from='client.48109 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:23:59.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:23:58 smithi083 ceph-mon[77242]: from='client.38181 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:23:59.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:23:58 smithi083 ceph-mon[77242]: pgmap v3473: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:00.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:24:00 smithi083 ceph-mon[79930]: pgmap v3474: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:00.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:24:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:24:00.549Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:24:00.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:24:00 smithi083 ceph-mon[77242]: pgmap v3474: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:01.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:24:00 smithi153 ceph-mon[70044]: pgmap v3474: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:24:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:24:01] "GET /metrics HTTP/1.1" 200 225174 "" "Prometheus/2.33.4" 2022-09-29T07:24:02.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:24:02 smithi083 ceph-mon[79930]: pgmap v3475: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:02.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:24:02 smithi083 ceph-mon[77242]: pgmap v3475: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:24:02 smithi153 ceph-mon[70044]: pgmap v3475: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:02.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:24:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:24:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:24:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:24:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:24:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:24:04.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:24:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:24:03.952Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:24:04.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:24:04 smithi083 ceph-mon[79930]: pgmap v3476: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:04.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:24:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:24:04.648Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:24:04.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:24:04 smithi083 ceph-mon[77242]: pgmap v3476: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:05.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:24:04 smithi153 ceph-mon[70044]: pgmap v3476: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:07.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:24:06 smithi153 ceph-mon[70044]: pgmap v3477: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:07.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:24:06 smithi083 ceph-mon[77242]: pgmap v3477: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:07.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:24:06 smithi083 ceph-mon[79930]: pgmap v3477: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:09.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:24:08 smithi153 ceph-mon[70044]: pgmap v3478: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:09.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:24:08 smithi083 ceph-mon[77242]: pgmap v3478: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:09.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:24:08 smithi083 ceph-mon[79930]: pgmap v3478: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:10.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:24:10 smithi083 ceph-mon[79930]: pgmap v3479: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:10.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:24:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:24:10.550Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:24:10.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:24:10 smithi083 ceph-mon[77242]: pgmap v3479: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:11.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:24:10 smithi153 ceph-mon[70044]: pgmap v3479: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:24:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:24:11] "GET /metrics HTTP/1.1" 200 225177 "" "Prometheus/2.33.4" 2022-09-29T07:24:12.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:24:12 smithi083 ceph-mon[77242]: pgmap v3480: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:12.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:24:12 smithi083 ceph-mon[79930]: pgmap v3480: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:24:12 smithi153 ceph-mon[70044]: pgmap v3480: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:12.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:24:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:24:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:24:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:24:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:24:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:24:14.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:24:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:24:13.953Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:24:14.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:24:14 smithi083 ceph-mon[79930]: pgmap v3481: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:14.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:24:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:24:14.649Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:24:14.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:24:14 smithi083 ceph-mon[77242]: pgmap v3481: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:15.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:24:14 smithi153 ceph-mon[70044]: pgmap v3481: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:17.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:24:16 smithi153 ceph-mon[70044]: pgmap v3482: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:17.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:24:16 smithi083 ceph-mon[79930]: pgmap v3482: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:17.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:24:16 smithi083 ceph-mon[77242]: pgmap v3482: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:19.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:24:18 smithi153 ceph-mon[70044]: pgmap v3483: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:19.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:24:18 smithi083 ceph-mon[79930]: pgmap v3483: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:19.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:24:18 smithi083 ceph-mon[77242]: pgmap v3483: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:20.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:24:20 smithi083 ceph-mon[77242]: pgmap v3484: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:20.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:24:20 smithi083 ceph-mon[79930]: pgmap v3484: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:20.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:24:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:24:20.550Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:24:21.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:24:20 smithi153 ceph-mon[70044]: pgmap v3484: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:24:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:24:21] "GET /metrics HTTP/1.1" 200 225177 "" "Prometheus/2.33.4" 2022-09-29T07:24:22.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:24:22 smithi083 ceph-mon[77242]: pgmap v3485: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:22.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:24:22 smithi083 ceph-mon[79930]: pgmap v3485: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:24:22 smithi153 ceph-mon[70044]: pgmap v3485: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:22.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:24:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:24:22.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:24:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:24:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:24:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:24:24.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:24:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:24:23.952Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:24:24.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:24:24 smithi083 ceph-mon[79930]: pgmap v3486: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:24.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:24:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:24:24.649Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:24:24.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:24:24 smithi083 ceph-mon[77242]: pgmap v3486: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:25.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:24:24 smithi153 ceph-mon[70044]: pgmap v3486: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:27.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:24:26 smithi153 ceph-mon[70044]: pgmap v3487: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:27.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:24:26 smithi083 ceph-mon[79930]: pgmap v3487: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:27.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:24:26 smithi083 ceph-mon[77242]: pgmap v3487: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:27.943 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:24:28.231 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:24:28.231 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (118m) 9m ago 118m 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:24:28.232 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (118m) 6m ago 118m 62.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:24:28.232 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (119m) 9m ago 119m 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:24:28.232 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (119m) 6m ago 119m 27.8M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:24:28.232 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (115m) 6m ago 2h 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:24:28.233 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (115m) 9m ago 2h 608M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:24:28.233 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (114m) 9m ago 2h 459M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:24:28.233 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (114m) 6m ago 2h 209M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:24:28.233 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (114m) 9m ago 2h 270M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:24:28.233 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (118m) 9m ago 118m 25.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:24:28.234 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (118m) 6m ago 118m 26.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:24:28.234 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (113m) 9m ago 2h 335M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:24:28.234 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (113m) 9m ago 2h 327M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:24:28.234 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (112m) 9m ago 2h 261M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:24:28.234 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (112m) 9m ago 2h 315M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:24:28.235 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (111m) 6m ago 2h 284M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:24:28.235 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (111m) 6m ago 2h 345M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:24:28.235 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (110m) 6m ago 2h 306M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:24:28.235 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (110m) 6m ago 119m 356M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:24:28.236 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (115m) 6m ago 118m 116M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:24:28.586 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:24:28.586 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:24:28.587 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:24:28.587 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:24:28.587 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:24:28.587 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:24:28.588 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:24:28.588 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:24:28.588 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:24:28.588 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:24:28.589 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:24:28.589 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:24:28.589 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:24:28.589 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:24:28.589 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:24:28.590 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:24:28.590 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:24:28.590 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:24:29.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:24:28 smithi153 ceph-mon[70044]: pgmap v3488: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:29.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:24:28 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2891469284' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:24:29.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:24:28 smithi083 ceph-mon[77242]: pgmap v3488: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:29.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:24:28 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2891469284' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:24:29.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:24:28 smithi083 ceph-mon[79930]: pgmap v3488: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:29.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:24:28 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2891469284' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:24:30.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:24:29 smithi153 ceph-mon[70044]: from='client.48127 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:24:30.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:24:29 smithi153 ceph-mon[70044]: from='client.48133 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:24:30.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:24:29 smithi083 ceph-mon[79930]: from='client.48127 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:24:30.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:24:29 smithi083 ceph-mon[79930]: from='client.48133 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:24:30.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:24:29 smithi083 ceph-mon[77242]: from='client.48127 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:24:30.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:24:29 smithi083 ceph-mon[77242]: from='client.48133 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:24:30.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:24:30 smithi083 ceph-mon[79930]: pgmap v3489: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:30.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:24:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:24:30.551Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:24:30.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:24:30 smithi083 ceph-mon[77242]: pgmap v3489: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:31.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:24:30 smithi153 ceph-mon[70044]: pgmap v3489: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:24:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:24:31] "GET /metrics HTTP/1.1" 200 225177 "" "Prometheus/2.33.4" 2022-09-29T07:24:32.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:24:32 smithi083 ceph-mon[77242]: pgmap v3490: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:32.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:24:32 smithi083 ceph-mon[79930]: pgmap v3490: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:24:32 smithi153 ceph-mon[70044]: pgmap v3490: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:32.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:24:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:24:32.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:24:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:24:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:24:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:24:34.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:24:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:24:33.953Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:24:34.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:24:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:24:34.652Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:24:34.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:24:34 smithi083 ceph-mon[77242]: pgmap v3491: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:34.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:24:34 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:24:34.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:24:34 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:24:34.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:24:34 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:24:34.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:24:34 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:24:34.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:24:34 smithi083 ceph-mon[79930]: pgmap v3491: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:34.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:24:34 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:24:34.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:24:34 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:24:34.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:24:34 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:24:34.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:24:34 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:24:35.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:24:34 smithi153 ceph-mon[70044]: pgmap v3491: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:35.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:24:34 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:24:35.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:24:34 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:24:35.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:24:34 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:24:35.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:24:34 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:24:37.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:24:36 smithi153 ceph-mon[70044]: pgmap v3492: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:37.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:24:36 smithi083 ceph-mon[77242]: pgmap v3492: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:37.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:24:36 smithi083 ceph-mon[79930]: pgmap v3492: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:39.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:24:38 smithi153 ceph-mon[70044]: pgmap v3493: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:39.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:24:38 smithi083 ceph-mon[77242]: pgmap v3493: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:39.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:24:38 smithi083 ceph-mon[79930]: pgmap v3493: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:24:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:24:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:24:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:24:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:24:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:24:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:24:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:24:40.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:24:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:24:40.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:24:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:24:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:24:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:24:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:24:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:24:40.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:24:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:24:40.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:24:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:24:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:24:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:24:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:24:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:24:40.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:24:40 smithi083 ceph-mon[79930]: pgmap v3494: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:40.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:24:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:24:40.551Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:24:40.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:24:40 smithi083 ceph-mon[77242]: pgmap v3494: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:24:40 smithi153 ceph-mon[70044]: pgmap v3494: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:24:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:24:41] "GET /metrics HTTP/1.1" 200 225175 "" "Prometheus/2.33.4" 2022-09-29T07:24:42.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:24:42 smithi083 ceph-mon[79930]: pgmap v3495: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:42.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:24:42 smithi083 ceph-mon[77242]: pgmap v3495: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:24:42 smithi153 ceph-mon[70044]: pgmap v3495: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:42.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:24:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:24:42.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:24:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:24:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:24:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:24:44.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:24:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:24:43.954Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:24:44.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:24:44 smithi083 ceph-mon[79930]: pgmap v3496: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:44.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:24:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:24:44.651Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:24:44.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:24:44 smithi083 ceph-mon[77242]: pgmap v3496: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:45.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:24:44 smithi153 ceph-mon[70044]: pgmap v3496: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:47.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:24:46 smithi153 ceph-mon[70044]: pgmap v3497: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:47.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:24:46 smithi083 ceph-mon[79930]: pgmap v3497: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:47.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:24:46 smithi083 ceph-mon[77242]: pgmap v3497: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:49.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:24:48 smithi153 ceph-mon[70044]: pgmap v3498: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:49.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:24:48 smithi083 ceph-mon[77242]: pgmap v3498: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:49.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:24:48 smithi083 ceph-mon[79930]: pgmap v3498: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:50.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:24:50 smithi083 ceph-mon[77242]: pgmap v3499: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:50.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:24:50 smithi083 ceph-mon[79930]: pgmap v3499: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:50.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:24:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:24:50.551Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:24:51.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:24:50 smithi153 ceph-mon[70044]: pgmap v3499: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:24:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:24:51] "GET /metrics HTTP/1.1" 200 225175 "" "Prometheus/2.33.4" 2022-09-29T07:24:52.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:24:52 smithi083 ceph-mon[79930]: pgmap v3500: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:52.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:24:52 smithi083 ceph-mon[77242]: pgmap v3500: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:24:52 smithi153 ceph-mon[70044]: pgmap v3500: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:52.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:24:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:24:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:24:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:24:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:24:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:24:54.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:24:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:24:53.954Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:24:54.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:24:54 smithi083 ceph-mon[79930]: pgmap v3501: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:54.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:24:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:24:54.652Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:24:54.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:24:54 smithi083 ceph-mon[77242]: pgmap v3501: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:55.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:24:54 smithi153 ceph-mon[70044]: pgmap v3501: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:57.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:24:56 smithi153 ceph-mon[70044]: pgmap v3502: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:57.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:24:56 smithi083 ceph-mon[77242]: pgmap v3502: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:57.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:24:56 smithi083 ceph-mon[79930]: pgmap v3502: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:58.914 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:24:59.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:24:58 smithi153 ceph-mon[70044]: pgmap v3503: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:59.201 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:24:59.201 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (118m) 9m ago 119m 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:24:59.201 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (118m) 7m ago 118m 62.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:24:59.201 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 9m ago 2h 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:24:59.202 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 7m ago 119m 27.8M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:24:59.202 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (115m) 7m ago 2h 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:24:59.202 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (116m) 9m ago 2h 608M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:24:59.202 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (115m) 9m ago 2h 459M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:24:59.203 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (114m) 7m ago 2h 209M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:24:59.203 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (115m) 9m ago 2h 270M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:24:59.203 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (119m) 9m ago 119m 25.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:24:59.203 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (119m) 7m ago 119m 26.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:24:59.203 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (114m) 9m ago 2h 335M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:24:59.204 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (113m) 9m ago 2h 327M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:24:59.204 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (113m) 9m ago 2h 261M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:24:59.204 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (112m) 9m ago 2h 315M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:24:59.204 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (112m) 7m ago 2h 284M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:24:59.204 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (111m) 7m ago 2h 345M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:24:59.205 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (111m) 7m ago 2h 306M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:24:59.205 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (110m) 7m ago 2h 356M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:24:59.205 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (115m) 7m ago 119m 116M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:24:59.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:24:58 smithi083 ceph-mon[79930]: pgmap v3503: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:59.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:24:58 smithi083 ceph-mon[77242]: pgmap v3503: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:24:59.555 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:24:59.556 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:24:59.556 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:24:59.556 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:24:59.556 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:24:59.557 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:24:59.557 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:24:59.557 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:24:59.557 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:24:59.558 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:24:59.558 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:24:59.558 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:24:59.558 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:24:59.559 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:24:59.559 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:24:59.559 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:24:59.559 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:24:59.559 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:25:00.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:24:59 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1507736571' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:25:00.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:24:59 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1507736571' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:25:00.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:24:59 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1507736571' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:25:00.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:25:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:25:00.552Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:25:00.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:25:00 smithi083 ceph-mon[77242]: from='client.48145 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:25:00.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:25:00 smithi083 ceph-mon[77242]: from='client.48151 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:25:00.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:25:00 smithi083 ceph-mon[77242]: pgmap v3504: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:25:00.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:25:00 smithi083 ceph-mon[79930]: from='client.48145 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:25:00.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:25:00 smithi083 ceph-mon[79930]: from='client.48151 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:25:00.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:25:00 smithi083 ceph-mon[79930]: pgmap v3504: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:25:01.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:25:00 smithi153 ceph-mon[70044]: from='client.48145 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:25:01.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:25:00 smithi153 ceph-mon[70044]: from='client.48151 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:25:01.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:25:00 smithi153 ceph-mon[70044]: pgmap v3504: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:25:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:25:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:25:01] "GET /metrics HTTP/1.1" 200 225174 "" "Prometheus/2.33.4" 2022-09-29T07:25:02.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:25:02 smithi083 ceph-mon[79930]: pgmap v3505: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:25:02.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:25:02 smithi083 ceph-mon[77242]: pgmap v3505: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:25:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:25:02 smithi153 ceph-mon[70044]: pgmap v3505: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:25:02.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:25:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:25:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:25:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:25:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:25:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:25:04.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:25:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:25:03.955Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:25:04.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:25:04 smithi083 ceph-mon[79930]: pgmap v3506: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:25:04.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:25:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:25:04.652Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:25:04.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:25:04 smithi083 ceph-mon[77242]: pgmap v3506: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:25:05.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:25:04 smithi153 ceph-mon[70044]: pgmap v3506: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:25:07.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:25:06 smithi153 ceph-mon[70044]: pgmap v3507: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:25:07.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:25:06 smithi083 ceph-mon[79930]: pgmap v3507: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:25:07.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:25:06 smithi083 ceph-mon[77242]: pgmap v3507: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:25:09.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:25:08 smithi153 ceph-mon[70044]: pgmap v3508: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:25:09.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:25:08 smithi083 ceph-mon[79930]: pgmap v3508: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:25:09.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:25:08 smithi083 ceph-mon[77242]: pgmap v3508: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:25:10.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:25:10 smithi083 ceph-mon[79930]: pgmap v3509: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:25:10.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:25:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:25:10.553Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:25:10.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:25:10 smithi083 ceph-mon[77242]: pgmap v3509: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:25:11.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:25:10 smithi153 ceph-mon[70044]: pgmap v3509: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:25:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:25:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:25:11] "GET /metrics HTTP/1.1" 200 225176 "" "Prometheus/2.33.4" 2022-09-29T07:25:12.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:25:12 smithi083 ceph-mon[79930]: pgmap v3510: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:25:12.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:25:12 smithi083 ceph-mon[77242]: pgmap v3510: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:25:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:25:12 smithi153 ceph-mon[70044]: pgmap v3510: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:25:12.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:25:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:25:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:25:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:25:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:25:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:25:14.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:25:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:25:13.955Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:25:14.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:25:14 smithi083 ceph-mon[77242]: pgmap v3511: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:25:14.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:25:14 smithi083 ceph-mon[79930]: pgmap v3511: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:25:14.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:25:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:25:14.652Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:25:15.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:25:14 smithi153 ceph-mon[70044]: pgmap v3511: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:25:17.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:25:16 smithi153 ceph-mon[70044]: pgmap v3512: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:25:17.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:25:16 smithi083 ceph-mon[79930]: pgmap v3512: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:25:17.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:25:16 smithi083 ceph-mon[77242]: pgmap v3512: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:25:19.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:25:18 smithi153 ceph-mon[70044]: pgmap v3513: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:25:19.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:25:18 smithi083 ceph-mon[79930]: pgmap v3513: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:25:19.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:25:18 smithi083 ceph-mon[77242]: pgmap v3513: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:25:20.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:25:20 smithi083 ceph-mon[79930]: pgmap v3514: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:25:20.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:25:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:25:20.554Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:25:20.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:25:20 smithi083 ceph-mon[77242]: pgmap v3514: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:25:21.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:25:20 smithi153 ceph-mon[70044]: pgmap v3514: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:25:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:25:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:25:21] "GET /metrics HTTP/1.1" 200 225176 "" "Prometheus/2.33.4" 2022-09-29T07:25:22.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:25:22 smithi083 ceph-mon[77242]: pgmap v3515: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:25:22.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:25:22 smithi083 ceph-mon[79930]: pgmap v3515: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:25:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:25:22 smithi153 ceph-mon[70044]: pgmap v3515: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:25:22.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:25:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:25:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:25:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:25:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:25:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:25:24.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:25:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:25:23.956Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:25:24.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:25:24 smithi083 ceph-mon[79930]: pgmap v3516: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:25:24.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:25:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:25:24.653Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:25:24.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:25:24 smithi083 ceph-mon[77242]: pgmap v3516: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:25:25.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:25:24 smithi153 ceph-mon[70044]: pgmap v3516: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:25:27.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:25:26 smithi153 ceph-mon[70044]: pgmap v3517: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:25:27.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:25:26 smithi083 ceph-mon[77242]: pgmap v3517: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:25:27.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:25:26 smithi083 ceph-mon[79930]: pgmap v3517: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:25:29.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:25:28 smithi153 ceph-mon[70044]: pgmap v3518: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:25:29.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:25:28 smithi083 ceph-mon[77242]: pgmap v3518: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:25:29.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:25:28 smithi083 ceph-mon[79930]: pgmap v3518: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:25:29.883 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:25:30.171 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:25:30.171 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (119m) 10m ago 119m 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:25:30.171 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (119m) 8m ago 119m 62.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:25:30.172 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 10m ago 2h 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:25:30.172 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 8m ago 2h 27.8M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:25:30.172 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (116m) 8m ago 2h 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:25:30.172 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (116m) 10m ago 2h 608M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:25:30.173 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (116m) 10m ago 2h 459M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:25:30.173 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (115m) 8m ago 2h 209M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:25:30.173 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (115m) 10m ago 2h 270M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:25:30.173 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (119m) 10m ago 119m 25.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:25:30.174 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (119m) 8m ago 119m 26.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:25:30.174 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (114m) 10m ago 2h 335M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:25:30.174 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (114m) 10m ago 2h 327M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:25:30.175 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (113m) 10m ago 2h 261M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:25:30.175 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (113m) 10m ago 2h 315M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:25:30.175 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (112m) 8m ago 2h 284M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:25:30.175 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (112m) 8m ago 2h 345M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:25:30.176 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (111m) 8m ago 2h 306M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:25:30.176 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (111m) 8m ago 2h 356M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:25:30.176 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (116m) 8m ago 119m 116M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:25:30.522 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:25:30.523 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:25:30.523 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:25:30.523 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:25:30.523 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:25:30.524 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:25:30.524 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:25:30.524 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:25:30.524 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:25:30.525 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:25:30.525 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:25:30.525 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:25:30.525 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:25:30.526 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:25:30.526 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:25:30.526 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:25:30.526 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:25:30.527 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:25:30.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:25:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:25:30.554Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:25:30.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:25:30 smithi083 ceph-mon[77242]: pgmap v3519: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:25:30.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:25:30 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1279194118' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:25:30.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:25:30 smithi083 ceph-mon[79930]: pgmap v3519: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:25:30.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:25:30 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1279194118' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:25:31.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:25:30 smithi153 ceph-mon[70044]: pgmap v3519: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:25:31.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:25:30 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1279194118' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:25:31.788 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:25:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:25:31] "GET /metrics HTTP/1.1" 200 225175 "" "Prometheus/2.33.4" 2022-09-29T07:25:32.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:25:31 smithi153 ceph-mon[70044]: from='client.38229 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:25:32.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:25:31 smithi153 ceph-mon[70044]: from='client.38235 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:25:32.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:25:31 smithi083 ceph-mon[79930]: from='client.38229 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:25:32.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:25:31 smithi083 ceph-mon[79930]: from='client.38235 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:25:32.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:25:31 smithi083 ceph-mon[77242]: from='client.38229 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:25:32.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:25:31 smithi083 ceph-mon[77242]: from='client.38235 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:25:32.789 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:25:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:25:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:25:33.057 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:25:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:25:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:25:33.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:25:32 smithi153 ceph-mon[70044]: pgmap v3520: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:25:33.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:25:32 smithi083 ceph-mon[77242]: pgmap v3520: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:25:33.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:25:32 smithi083 ceph-mon[79930]: pgmap v3520: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:25:34.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:25:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:25:33.957Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:25:34.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:25:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:25:34.654Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:25:34.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:25:34 smithi083 ceph-mon[77242]: pgmap v3521: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:25:34.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:25:34 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:25:34.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:25:34 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:25:34.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:25:34 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:25:34.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:25:34 smithi083 ceph-mon[79930]: pgmap v3521: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:25:34.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:25:34 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:25:34.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:25:34 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:25:34.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:25:34 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:25:35.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:25:34 smithi153 ceph-mon[70044]: pgmap v3521: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:25:35.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:25:34 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:25:35.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:25:34 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:25:35.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:25:34 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:25:36.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:25:36 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:25:36.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:25:36 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:25:36.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:25:36 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:25:36.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:25:36 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:25:36.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:25:36 smithi083 ceph-mon[79930]: pgmap v3522: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:25:36.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:25:36 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:25:36.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:25:36 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:25:36.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:25:36 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:25:36.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:25:36 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:25:36.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:25:36 smithi083 ceph-mon[77242]: pgmap v3522: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:25:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:25:36 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:25:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:25:36 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:25:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:25:36 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:25:36.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:25:36 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:25:36.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:25:36 smithi153 ceph-mon[70044]: pgmap v3522: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:25:39.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:25:38 smithi153 ceph-mon[70044]: pgmap v3523: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:25:39.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:25:38 smithi083 ceph-mon[79930]: pgmap v3523: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:25:39.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:25:38 smithi083 ceph-mon[77242]: pgmap v3523: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:25:40.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:25:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:25:40.556Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:25:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:25:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:25:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:25:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:25:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:25:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:25:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:25:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:25:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:25:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:25:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:25:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:25:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:25:40 smithi083 ceph-mon[77242]: pgmap v3524: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:25:40.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:25:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:25:40.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:25:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:25:40.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:25:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:25:40.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:25:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:25:40.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:25:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:25:40.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:25:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:25:40.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:25:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:25:40.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:25:40 smithi083 ceph-mon[79930]: pgmap v3524: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:25:40.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:25:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:25:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:25:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:25:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:25:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:25:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:25:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:25:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:25:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:25:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:25:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:25:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:25:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:25:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:25:40 smithi153 ceph-mon[70044]: pgmap v3524: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:25:40.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:25:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:25:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:25:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:25:41] "GET /metrics HTTP/1.1" 200 225179 "" "Prometheus/2.33.4" 2022-09-29T07:25:42.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:25:42 smithi083 ceph-mon[77242]: pgmap v3525: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:25:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:25:42 smithi083 ceph-mon[79930]: pgmap v3525: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:25:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:25:42 smithi153 ceph-mon[70044]: pgmap v3525: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:25:42.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:25:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:25:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:25:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:25:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:25:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:25:44.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:25:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:25:43.958Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:25:44.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:25:44 smithi083 ceph-mon[79930]: pgmap v3526: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:25:44.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:25:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:25:44.655Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:25:44.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:25:44 smithi083 ceph-mon[77242]: pgmap v3526: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:25:45.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:25:44 smithi153 ceph-mon[70044]: pgmap v3526: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:25:47.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:25:46 smithi153 ceph-mon[70044]: pgmap v3527: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:25:47.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:25:46 smithi083 ceph-mon[79930]: pgmap v3527: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:25:47.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:25:46 smithi083 ceph-mon[77242]: pgmap v3527: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:25:49.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:25:48 smithi153 ceph-mon[70044]: pgmap v3528: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:25:49.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:25:48 smithi083 ceph-mon[79930]: pgmap v3528: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:25:49.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:25:48 smithi083 ceph-mon[77242]: pgmap v3528: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:25:50.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:25:50 smithi083 ceph-mon[79930]: pgmap v3529: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:25:50.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:25:50 smithi083 ceph-mon[77242]: pgmap v3529: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:25:50.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:25:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:25:50.557Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:25:51.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:25:50 smithi153 ceph-mon[70044]: pgmap v3529: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:25:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:25:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:25:51] "GET /metrics HTTP/1.1" 200 225179 "" "Prometheus/2.33.4" 2022-09-29T07:25:52.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:25:52 smithi083 ceph-mon[79930]: pgmap v3530: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:25:52.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:25:52 smithi083 ceph-mon[77242]: pgmap v3530: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:25:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:25:52 smithi153 ceph-mon[70044]: pgmap v3530: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:25:52.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:25:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:25:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:25:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:25:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:25:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:25:54.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:25:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:25:53.959Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:25:54.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:25:54 smithi083 ceph-mon[79930]: pgmap v3531: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:25:54.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:25:54 smithi083 ceph-mon[77242]: pgmap v3531: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:25:54.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:25:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:25:54.656Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:25:55.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:25:54 smithi153 ceph-mon[70044]: pgmap v3531: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:25:57.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:25:56 smithi153 ceph-mon[70044]: pgmap v3532: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:25:57.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:25:56 smithi083 ceph-mon[79930]: pgmap v3532: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:25:57.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:25:56 smithi083 ceph-mon[77242]: pgmap v3532: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:25:59.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:25:58 smithi153 ceph-mon[70044]: pgmap v3533: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:25:59.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:25:58 smithi083 ceph-mon[79930]: pgmap v3533: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:25:59.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:25:58 smithi083 ceph-mon[77242]: pgmap v3533: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:00.853 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:26:00.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:26:00 smithi083 ceph-mon[79930]: pgmap v3534: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:00.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:26:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:26:00.557Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:26:00.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:26:00 smithi083 ceph-mon[77242]: pgmap v3534: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:01.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:26:00 smithi153 ceph-mon[70044]: pgmap v3534: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:01.143 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:26:01.143 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (119m) 21s ago 2h 26.7M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:26:01.143 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 8m ago 119m 62.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:26:01.143 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 21s ago 2h 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:26:01.144 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 8m ago 2h 27.8M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:26:01.144 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (116m) 8m ago 2h 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:26:01.144 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (117m) 21s ago 2h 617M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:26:01.144 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (116m) 21s ago 2h 494M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:26:01.145 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (115m) 8m ago 2h 209M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:26:01.145 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (116m) 21s ago 2h 291M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:26:01.145 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 21s ago 2h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:26:01.145 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 8m ago 2h 26.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:26:01.145 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (115m) 21s ago 2h 361M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:26:01.146 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (114m) 21s ago 2h 352M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:26:01.146 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (114m) 21s ago 2h 280M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:26:01.146 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (113m) 21s ago 2h 340M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:26:01.146 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (113m) 8m ago 2h 284M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:26:01.147 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (112m) 8m ago 2h 345M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:26:01.147 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (112m) 8m ago 2h 306M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:26:01.147 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (111m) 8m ago 2h 356M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:26:01.147 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (116m) 8m ago 2h 116M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:26:01.499 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:26:01.500 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:26:01.500 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:26:01.500 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:26:01.500 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:26:01.501 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:26:01.501 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:26:01.501 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:26:01.501 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:26:01.501 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:26:01.502 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:26:01.502 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:26:01.502 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:26:01.502 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:26:01.503 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:26:01.503 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:26:01.503 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:26:01.503 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:26:01.796 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:26:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:26:01] "GET /metrics HTTP/1.1" 200 225174 "" "Prometheus/2.33.4" 2022-09-29T07:26:02.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:26:01 smithi153 ceph-mon[70044]: from='client.48181 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:26:02.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:26:01 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1529706605' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:26:02.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:26:01 smithi083 ceph-mon[79930]: from='client.48181 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:26:02.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:26:01 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1529706605' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:26:02.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:26:01 smithi083 ceph-mon[77242]: from='client.48181 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:26:02.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:26:01 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1529706605' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:26:02.797 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:26:02.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:26:03.057 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:26:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:26:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:26:03.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:26:02 smithi153 ceph-mon[70044]: from='client.38253 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:26:03.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:26:02 smithi153 ceph-mon[70044]: pgmap v3535: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:03.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:26:02 smithi083 ceph-mon[79930]: from='client.38253 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:26:03.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:26:02 smithi083 ceph-mon[79930]: pgmap v3535: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:03.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:26:02 smithi083 ceph-mon[77242]: from='client.38253 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:26:03.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:26:02 smithi083 ceph-mon[77242]: pgmap v3535: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:04.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:26:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:26:03.959Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:26:04.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:26:04 smithi083 ceph-mon[77242]: pgmap v3536: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:04.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:26:04 smithi083 ceph-mon[79930]: pgmap v3536: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:04.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:26:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:26:04.656Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:26:05.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:26:04 smithi153 ceph-mon[70044]: pgmap v3536: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:07.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:26:06 smithi153 ceph-mon[70044]: pgmap v3537: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:07.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:26:06 smithi083 ceph-mon[79930]: pgmap v3537: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:07.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:26:06 smithi083 ceph-mon[77242]: pgmap v3537: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:09.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:26:08 smithi153 ceph-mon[70044]: pgmap v3538: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:09.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:26:08 smithi083 ceph-mon[79930]: pgmap v3538: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:09.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:26:08 smithi083 ceph-mon[77242]: pgmap v3538: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:10.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:26:10 smithi083 ceph-mon[79930]: pgmap v3539: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:10.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:26:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:26:10.558Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:26:10.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:26:10 smithi083 ceph-mon[77242]: pgmap v3539: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:11.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:26:10 smithi153 ceph-mon[70044]: pgmap v3539: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:26:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:26:11] "GET /metrics HTTP/1.1" 200 225178 "" "Prometheus/2.33.4" 2022-09-29T07:26:12.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:26:12 smithi083 ceph-mon[77242]: pgmap v3540: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:12.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:26:12 smithi083 ceph-mon[79930]: pgmap v3540: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:26:12 smithi153 ceph-mon[70044]: pgmap v3540: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:12.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:26:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:26:12.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:26:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:26:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:26:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:26:14.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:26:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:26:13.960Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:26:14.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:26:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:26:14.657Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:26:14.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:26:14 smithi083 ceph-mon[77242]: pgmap v3541: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:14.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:26:14 smithi083 ceph-mon[79930]: pgmap v3541: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:15.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:26:14 smithi153 ceph-mon[70044]: pgmap v3541: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:17.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:26:16 smithi153 ceph-mon[70044]: pgmap v3542: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:17.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:26:16 smithi083 ceph-mon[77242]: pgmap v3542: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:17.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:26:16 smithi083 ceph-mon[79930]: pgmap v3542: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:19.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:26:18 smithi153 ceph-mon[70044]: pgmap v3543: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:19.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:26:18 smithi083 ceph-mon[79930]: pgmap v3543: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:19.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:26:18 smithi083 ceph-mon[77242]: pgmap v3543: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:20.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:26:20 smithi083 ceph-mon[79930]: pgmap v3544: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:20.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:26:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:26:20.559Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:26:20.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:26:20 smithi083 ceph-mon[77242]: pgmap v3544: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:21.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:26:20 smithi153 ceph-mon[70044]: pgmap v3544: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:26:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:26:21] "GET /metrics HTTP/1.1" 200 225178 "" "Prometheus/2.33.4" 2022-09-29T07:26:22.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:26:22 smithi083 ceph-mon[79930]: pgmap v3545: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:22.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:26:22 smithi083 ceph-mon[77242]: pgmap v3545: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:26:22 smithi153 ceph-mon[70044]: pgmap v3545: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:22.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:26:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:26:22.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:26:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:26:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:26:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:26:24.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:26:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:26:23.960Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:26:24.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:26:24 smithi083 ceph-mon[79930]: pgmap v3546: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:24.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:26:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:26:24.657Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:26:24.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:26:24 smithi083 ceph-mon[77242]: pgmap v3546: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:25.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:26:24 smithi153 ceph-mon[70044]: pgmap v3546: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:27.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:26:26 smithi153 ceph-mon[70044]: pgmap v3547: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:27.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:26:26 smithi083 ceph-mon[79930]: pgmap v3547: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:27.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:26:26 smithi083 ceph-mon[77242]: pgmap v3547: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:29.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:26:28 smithi153 ceph-mon[70044]: pgmap v3548: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:29.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:26:28 smithi083 ceph-mon[79930]: pgmap v3548: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:29.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:26:28 smithi083 ceph-mon[77242]: pgmap v3548: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:30.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:26:30 smithi083 ceph-mon[79930]: pgmap v3549: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:30.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:26:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:26:30.560Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:26:30.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:26:30 smithi083 ceph-mon[77242]: pgmap v3549: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:31.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:26:30 smithi153 ceph-mon[70044]: pgmap v3549: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:31.824 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:26:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:26:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:26:31] "GET /metrics HTTP/1.1" 200 225178 "" "Prometheus/2.33.4" 2022-09-29T07:26:32.112 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:26:32.113 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 52s ago 2h 26.7M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:26:32.113 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 9m ago 2h 62.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:26:32.113 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 52s ago 2h 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:26:32.113 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 9m ago 2h 27.8M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:26:32.114 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (117m) 9m ago 2h 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:26:32.114 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (117m) 52s ago 2h 617M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:26:32.114 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (117m) 52s ago 2h 494M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:26:32.114 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (116m) 9m ago 2h 209M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:26:32.115 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (116m) 52s ago 2h 291M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:26:32.115 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 52s ago 2h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:26:32.115 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 9m ago 2h 26.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:26:32.115 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (115m) 52s ago 2h 361M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:26:32.116 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (115m) 52s ago 2h 352M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:26:32.116 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (114m) 52s ago 2h 280M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:26:32.116 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (114m) 52s ago 2h 340M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:26:32.116 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (113m) 9m ago 2h 284M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:26:32.116 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (113m) 9m ago 2h 345M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:26:32.117 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (112m) 9m ago 2h 306M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:26:32.117 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (112m) 9m ago 2h 356M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:26:32.117 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (117m) 9m ago 2h 116M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:26:32.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:26:32 smithi083 ceph-mon[77242]: pgmap v3550: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:32.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:26:32 smithi083 ceph-mon[77242]: from='client.48199 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:26:32.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:26:32 smithi083 ceph-mon[79930]: pgmap v3550: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:32.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:26:32 smithi083 ceph-mon[79930]: from='client.48199 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:26:32.471 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:26:32.471 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:26:32.472 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:26:32.472 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:26:32.472 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:26:32.472 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:26:32.472 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:26:32.473 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:26:32.473 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:26:32.473 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:26:32.473 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:26:32.474 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:26:32.474 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:26:32.474 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:26:32.474 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:26:32.475 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:26:32.475 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:26:32.475 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:26:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:26:32 smithi153 ceph-mon[70044]: pgmap v3550: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:26:32 smithi153 ceph-mon[70044]: from='client.48199 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:26:32.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:26:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:26:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:26:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:26:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:26:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:26:33.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:26:33 smithi153 ceph-mon[70044]: from='client.38271 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:26:33.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:26:33 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3393106502' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:26:33.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:26:33 smithi083 ceph-mon[79930]: from='client.38271 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:26:33.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:26:33 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3393106502' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:26:33.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:26:33 smithi083 ceph-mon[77242]: from='client.38271 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:26:33.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:26:33 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3393106502' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:26:34.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:26:34 smithi083 ceph-mon[77242]: pgmap v3551: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:34.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:26:34 smithi083 ceph-mon[79930]: pgmap v3551: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:34.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:26:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:26:33.961Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:26:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:26:34 smithi153 ceph-mon[70044]: pgmap v3551: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:34.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:26:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:26:34.657Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:26:37.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:26:36 smithi153 ceph-mon[70044]: pgmap v3552: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:37.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:26:36 smithi083 ceph-mon[79930]: pgmap v3552: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:37.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:26:36 smithi083 ceph-mon[77242]: pgmap v3552: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:39.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:26:38 smithi153 ceph-mon[70044]: pgmap v3553: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:39.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:26:38 smithi083 ceph-mon[79930]: pgmap v3553: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:39.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:26:38 smithi083 ceph-mon[77242]: pgmap v3553: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:40.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:26:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:26:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:26:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:26:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:26:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:26:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:26:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:26:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:26:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:26:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:26:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:26:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:26:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:26:40.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:26:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:26:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:26:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:26:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:26:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:26:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:26:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:26:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:26:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:26:40.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:26:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:26:40.563Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:26:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:26:40 smithi083 ceph-mon[79930]: pgmap v3554: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:26:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:26:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:26:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:26:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:26:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:26:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:26:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:26:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:26:40 smithi083 ceph-mon[77242]: pgmap v3554: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:26:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:26:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:26:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:26:40.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:26:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:26:40.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:26:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:26:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:26:40 smithi153 ceph-mon[70044]: pgmap v3554: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:26:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:26:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:26:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:26:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:26:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:26:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:26:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:26:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:26:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:26:41] "GET /metrics HTTP/1.1" 200 225177 "" "Prometheus/2.33.4" 2022-09-29T07:26:42.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:26:42 smithi083 ceph-mon[79930]: pgmap v3555: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:42.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:26:42 smithi083 ceph-mon[77242]: pgmap v3555: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:26:42 smithi153 ceph-mon[70044]: pgmap v3555: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:42.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:26:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:26:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:26:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:26:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:26:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:26:44.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:26:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:26:43.961Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:26:44.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:26:44 smithi083 ceph-mon[79930]: pgmap v3556: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:44.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:26:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:26:44.658Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:26:44.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:26:44 smithi083 ceph-mon[77242]: pgmap v3556: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:45.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:26:44 smithi153 ceph-mon[70044]: pgmap v3556: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:47.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:26:46 smithi153 ceph-mon[70044]: pgmap v3557: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:47.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:26:46 smithi083 ceph-mon[79930]: pgmap v3557: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:47.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:26:46 smithi083 ceph-mon[77242]: pgmap v3557: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:49.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:26:48 smithi083 ceph-mon[79930]: pgmap v3558: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:49.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:26:48 smithi083 ceph-mon[77242]: pgmap v3558: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:49.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:26:48 smithi153 ceph-mon[70044]: pgmap v3558: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:50.814 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:26:50 smithi083 ceph-mon[77242]: pgmap v3559: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:50.815 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:26:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:26:50.562Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:26:51.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:26:50 smithi083 ceph-mon[79930]: pgmap v3559: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:51.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:26:50 smithi153 ceph-mon[70044]: pgmap v3559: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:26:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:26:51] "GET /metrics HTTP/1.1" 200 225177 "" "Prometheus/2.33.4" 2022-09-29T07:26:52.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:26:52 smithi083 ceph-mon[79930]: pgmap v3560: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:52.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:26:52 smithi083 ceph-mon[77242]: pgmap v3560: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:26:52 smithi153 ceph-mon[70044]: pgmap v3560: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:52.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:26:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:26:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:26:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:26:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:26:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:26:54.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:26:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:26:53.961Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:26:54.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:26:54 smithi083 ceph-mon[79930]: pgmap v3561: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:54.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:26:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:26:54.659Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:26:54.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:26:54 smithi083 ceph-mon[77242]: pgmap v3561: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:55.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:26:54 smithi153 ceph-mon[70044]: pgmap v3561: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:57.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:26:56 smithi083 ceph-mon[79930]: pgmap v3562: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:57.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:26:56 smithi083 ceph-mon[77242]: pgmap v3562: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:57.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:26:56 smithi153 ceph-mon[70044]: pgmap v3562: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:59.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:26:58 smithi083 ceph-mon[79930]: pgmap v3563: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:59.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:26:58 smithi083 ceph-mon[77242]: pgmap v3563: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:26:59.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:26:58 smithi153 ceph-mon[70044]: pgmap v3563: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:00.814 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:27:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:27:00.563Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:27:01.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:27:00 smithi083 ceph-mon[79930]: pgmap v3564: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:01.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:27:00 smithi083 ceph-mon[77242]: pgmap v3564: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:01.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:27:00 smithi153 ceph-mon[70044]: pgmap v3564: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:27:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:27:01] "GET /metrics HTTP/1.1" 200 225172 "" "Prometheus/2.33.4" 2022-09-29T07:27:02.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:27:02 smithi083 ceph-mon[79930]: pgmap v3565: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:02.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:27:02 smithi083 ceph-mon[77242]: pgmap v3565: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:27:02 smithi153 ceph-mon[70044]: pgmap v3565: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:02.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:27:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:27:02.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:27:02.795 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:27:03.084 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:27:03.084 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 83s ago 2h 26.7M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:27:03.084 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 9m ago 2h 62.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:27:03.085 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 83s ago 2h 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:27:03.085 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 9m ago 2h 27.8M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:27:03.085 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (117m) 9m ago 2h 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:27:03.085 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (118m) 83s ago 2h 617M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:27:03.086 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (117m) 83s ago 2h 494M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:27:03.086 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (116m) 9m ago 2h 209M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:27:03.086 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (117m) 83s ago 2h 291M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:27:03.086 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 83s ago 2h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:27:03.086 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 9m ago 2h 26.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:27:03.087 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (116m) 83s ago 2h 361M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:27:03.087 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (115m) 83s ago 2h 352M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:27:03.087 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (115m) 83s ago 2h 280M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:27:03.087 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (114m) 83s ago 2h 340M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:27:03.087 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (114m) 9m ago 2h 284M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:27:03.088 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (113m) 9m ago 2h 345M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:27:03.088 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (113m) 9m ago 2h 306M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:27:03.088 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (112m) 9m ago 2h 356M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:27:03.088 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (117m) 9m ago 2h 116M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:27:03.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:27:03 smithi153 ceph-mon[70044]: from='client.38283 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:27:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:27:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:27:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:27:03.450 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:27:03.450 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:27:03.451 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:27:03.451 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:27:03.451 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:27:03.451 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:27:03.452 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:27:03.452 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:27:03.452 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:27:03.452 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:27:03.452 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:27:03.453 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:27:03.453 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:27:03.453 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:27:03.453 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:27:03.453 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:27:03.454 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:27:03.454 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:27:03.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:27:03 smithi083 ceph-mon[79930]: from='client.38283 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:27:03.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:27:03 smithi083 ceph-mon[77242]: from='client.38283 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:27:04.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:27:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:27:03.961Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:27:04.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:27:04 smithi083 ceph-mon[77242]: from='client.38289 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:27:04.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:27:04 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/406788725' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:27:04.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:27:04 smithi083 ceph-mon[77242]: pgmap v3566: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:04.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:27:04 smithi083 ceph-mon[79930]: from='client.38289 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:27:04.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:27:04 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/406788725' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:27:04.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:27:04 smithi083 ceph-mon[79930]: pgmap v3566: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:27:04 smithi153 ceph-mon[70044]: from='client.38289 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:27:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:27:04 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/406788725' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:27:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:27:04 smithi153 ceph-mon[70044]: pgmap v3566: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:04.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:27:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:27:04.660Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:27:07.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:27:06 smithi083 ceph-mon[77242]: pgmap v3567: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:07.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:27:06 smithi083 ceph-mon[79930]: pgmap v3567: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:07.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:27:06 smithi153 ceph-mon[70044]: pgmap v3567: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:09.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:27:08 smithi083 ceph-mon[79930]: pgmap v3568: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:09.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:27:08 smithi083 ceph-mon[77242]: pgmap v3568: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:09.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:27:08 smithi153 ceph-mon[70044]: pgmap v3568: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:10.816 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:27:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:27:10.564Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:27:11.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:27:10 smithi083 ceph-mon[79930]: pgmap v3569: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:11.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:27:10 smithi083 ceph-mon[77242]: pgmap v3569: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:11.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:27:10 smithi153 ceph-mon[70044]: pgmap v3569: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:27:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:27:11] "GET /metrics HTTP/1.1" 200 225173 "" "Prometheus/2.33.4" 2022-09-29T07:27:12.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:27:12 smithi083 ceph-mon[77242]: pgmap v3570: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:12.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:27:12 smithi083 ceph-mon[79930]: pgmap v3570: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:27:12 smithi153 ceph-mon[70044]: pgmap v3570: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:12.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:27:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:27:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:27:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:27:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:27:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:27:14.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:27:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:27:13.962Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:27:14.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:27:14 smithi083 ceph-mon[79930]: pgmap v3571: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:14.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:27:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:27:14.660Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:27:14.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:27:14 smithi083 ceph-mon[77242]: pgmap v3571: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:15.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:27:14 smithi153 ceph-mon[70044]: pgmap v3571: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:17.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:27:16 smithi083 ceph-mon[79930]: pgmap v3572: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:17.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:27:16 smithi083 ceph-mon[77242]: pgmap v3572: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:17.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:27:16 smithi153 ceph-mon[70044]: pgmap v3572: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:19.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:27:18 smithi083 ceph-mon[79930]: pgmap v3573: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:19.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:27:18 smithi083 ceph-mon[77242]: pgmap v3573: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:19.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:27:18 smithi153 ceph-mon[70044]: pgmap v3573: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:20.821 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:27:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:27:20.564Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:27:21.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:27:20 smithi083 ceph-mon[79930]: pgmap v3574: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:21.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:27:20 smithi083 ceph-mon[77242]: pgmap v3574: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:21.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:27:20 smithi153 ceph-mon[70044]: pgmap v3574: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:27:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:27:21] "GET /metrics HTTP/1.1" 200 225173 "" "Prometheus/2.33.4" 2022-09-29T07:27:22.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:27:22 smithi083 ceph-mon[77242]: pgmap v3575: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:22.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:27:22 smithi083 ceph-mon[79930]: pgmap v3575: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:22.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:27:22 smithi153 ceph-mon[70044]: pgmap v3575: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:22.560 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:27:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:27:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:27:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:27:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:27:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:27:24.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:27:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:27:23.962Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:27:24.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:27:24 smithi083 ceph-mon[79930]: pgmap v3576: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:24.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:27:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:27:24.661Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:27:24.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:27:24 smithi083 ceph-mon[77242]: pgmap v3576: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:25.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:27:24 smithi153 ceph-mon[70044]: pgmap v3576: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:27.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:27:26 smithi083 ceph-mon[79930]: pgmap v3577: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:27.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:27:26 smithi083 ceph-mon[77242]: pgmap v3577: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:27.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:27:26 smithi153 ceph-mon[70044]: pgmap v3577: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:29.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:27:28 smithi083 ceph-mon[79930]: pgmap v3578: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:29.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:27:28 smithi083 ceph-mon[77242]: pgmap v3578: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:29.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:27:28 smithi153 ceph-mon[70044]: pgmap v3578: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:30.824 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:27:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:27:30.565Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:27:31.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:27:30 smithi083 ceph-mon[79930]: pgmap v3579: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:31.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:27:30 smithi083 ceph-mon[77242]: pgmap v3579: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:31.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:27:30 smithi153 ceph-mon[70044]: pgmap v3579: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:27:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:27:31] "GET /metrics HTTP/1.1" 200 225174 "" "Prometheus/2.33.4" 2022-09-29T07:27:32.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:27:32 smithi083 ceph-mon[79930]: pgmap v3580: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:32.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:27:32 smithi083 ceph-mon[77242]: pgmap v3580: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:27:32 smithi153 ceph-mon[70044]: pgmap v3580: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:32.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:27:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:27:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:27:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:27:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:27:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:27:33.785 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:27:34.078 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:27:34.078 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 114s ago 2h 26.7M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:27:34.078 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 10m ago 2h 62.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:27:34.079 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 114s ago 2h 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:27:34.079 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 10m ago 2h 27.8M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:27:34.079 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (118m) 10m ago 2h 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:27:34.079 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (118m) 114s ago 2h 617M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:27:34.080 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (118m) 114s ago 2h 494M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:27:34.080 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (117m) 10m ago 2h 209M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:27:34.080 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (117m) 114s ago 2h 291M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:27:34.080 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 114s ago 2h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:27:34.080 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 10m ago 2h 26.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:27:34.081 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (116m) 114s ago 2h 361M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:27:34.081 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (116m) 114s ago 2h 352M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:27:34.081 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (115m) 114s ago 2h 280M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:27:34.081 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (115m) 114s ago 2h 340M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:27:34.082 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (114m) 10m ago 2h 284M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:27:34.082 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (114m) 10m ago 2h 345M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:27:34.082 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (113m) 10m ago 2h 306M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:27:34.082 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (113m) 10m ago 2h 356M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:27:34.082 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (118m) 10m ago 2h 116M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:27:34.432 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:27:34.432 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:27:34.432 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:27:34.432 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:27:34.433 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:27:34.433 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:27:34.433 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:27:34.433 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:27:34.433 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:27:34.434 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:27:34.434 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:27:34.434 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:27:34.434 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:27:34.434 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:27:34.435 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:27:34.435 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:27:34.435 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:27:34.435 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:27:34.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:27:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:27:33.962Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:27:34.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:27:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:27:34.661Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:27:34.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:27:34 smithi083 ceph-mon[77242]: pgmap v3581: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:34.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:27:34 smithi083 ceph-mon[77242]: from='client.38301 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:27:34.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:27:34 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/34123620' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:27:34.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:27:34 smithi083 ceph-mon[79930]: pgmap v3581: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:34.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:27:34 smithi083 ceph-mon[79930]: from='client.38301 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:27:34.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:27:34 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/34123620' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:27:35.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:27:34 smithi153 ceph-mon[70044]: pgmap v3581: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:35.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:27:34 smithi153 ceph-mon[70044]: from='client.38301 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:27:35.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:27:34 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/34123620' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:27:36.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:27:35 smithi083 ceph-mon[79930]: from='client.48241 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:27:36.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:27:35 smithi083 ceph-mon[77242]: from='client.48241 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:27:36.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:27:35 smithi153 ceph-mon[70044]: from='client.48241 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:27:37.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:27:36 smithi083 ceph-mon[79930]: pgmap v3582: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:37.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:27:36 smithi083 ceph-mon[77242]: pgmap v3582: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:37.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:27:36 smithi153 ceph-mon[70044]: pgmap v3582: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:39.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:27:38 smithi083 ceph-mon[79930]: pgmap v3583: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:39.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:27:38 smithi083 ceph-mon[77242]: pgmap v3583: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:39.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:27:38 smithi153 ceph-mon[70044]: pgmap v3583: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:40.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:27:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:27:40.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:27:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:27:40.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:27:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:27:40.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:27:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:27:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:27:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:27:40.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:27:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:27:40.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:27:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:27:40.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:27:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:27:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:27:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:27:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:27:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:27:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:27:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:27:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:27:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:27:40.835 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:27:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:27:40.566Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:27:41.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:27:40 smithi083 ceph-mon[79930]: pgmap v3584: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:41.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:27:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:27:41.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:27:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:27:41.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:27:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:27:41.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:27:40 smithi083 ceph-mon[77242]: pgmap v3584: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:27:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:27:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:27:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:27:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:27:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:27:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:27:40 smithi153 ceph-mon[70044]: pgmap v3584: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:27:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:27:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:27:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:27:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:27:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:27:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:27:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:27:41] "GET /metrics HTTP/1.1" 200 225176 "" "Prometheus/2.33.4" 2022-09-29T07:27:42.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:27:42 smithi083 ceph-mon[79930]: pgmap v3585: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:42.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:27:42 smithi083 ceph-mon[77242]: pgmap v3585: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:27:42 smithi153 ceph-mon[70044]: pgmap v3585: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:42.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:27:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:27:42.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:27:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:27:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:27:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:27:44.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:27:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:27:43.964Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:27:44.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:27:44 smithi083 ceph-mon[79930]: pgmap v3586: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:44.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:27:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:27:44.661Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:27:44.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:27:44 smithi083 ceph-mon[77242]: pgmap v3586: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:45.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:27:44 smithi153 ceph-mon[70044]: pgmap v3586: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:46.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:27:46 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:27:46.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:27:46 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:27:46.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:27:46 smithi153 ceph-mon[70044]: pgmap v3587: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:46.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:27:46 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:27:46.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:27:46 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:27:46.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:27:46 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:27:46.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:27:46 smithi083 ceph-mon[79930]: pgmap v3587: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:46.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:27:46 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:27:46.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:27:46 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:27:46.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:27:46 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:27:46.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:27:46 smithi083 ceph-mon[77242]: pgmap v3587: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:46.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:27:46 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:27:49.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:27:48 smithi083 ceph-mon[77242]: pgmap v3588: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:49.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:27:48 smithi083 ceph-mon[79930]: pgmap v3588: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:49.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:27:48 smithi153 ceph-mon[70044]: pgmap v3588: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:50.826 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:27:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:27:50.566Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:27:51.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:27:50 smithi083 ceph-mon[79930]: pgmap v3589: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:51.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:27:50 smithi083 ceph-mon[77242]: pgmap v3589: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:51.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:27:50 smithi153 ceph-mon[70044]: pgmap v3589: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:27:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:27:51] "GET /metrics HTTP/1.1" 200 225176 "" "Prometheus/2.33.4" 2022-09-29T07:27:52.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:27:52 smithi083 ceph-mon[77242]: pgmap v3590: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:52.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:27:52 smithi083 ceph-mon[79930]: pgmap v3590: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:27:52 smithi153 ceph-mon[70044]: pgmap v3590: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:52.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:27:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:27:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:27:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:27:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:27:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:27:54.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:27:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:27:53.964Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:27:54.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:27:54 smithi083 ceph-mon[77242]: pgmap v3591: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:54.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:27:54 smithi083 ceph-mon[79930]: pgmap v3591: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:54.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:27:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:27:54.661Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:27:55.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:27:54 smithi153 ceph-mon[70044]: pgmap v3591: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:57.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:27:56 smithi083 ceph-mon[79930]: pgmap v3592: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:57.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:27:56 smithi083 ceph-mon[77242]: pgmap v3592: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:57.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:27:56 smithi153 ceph-mon[70044]: pgmap v3592: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:59.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:27:58 smithi083 ceph-mon[79930]: pgmap v3593: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:59.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:27:58 smithi083 ceph-mon[77242]: pgmap v3593: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:27:59.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:27:58 smithi153 ceph-mon[70044]: pgmap v3593: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:00.832 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:28:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:28:00.566Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:28:01.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:28:00 smithi083 ceph-mon[79930]: pgmap v3594: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:01.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:28:00 smithi083 ceph-mon[77242]: pgmap v3594: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:01.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:28:00 smithi153 ceph-mon[70044]: pgmap v3594: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:28:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:28:01] "GET /metrics HTTP/1.1" 200 225175 "" "Prometheus/2.33.4" 2022-09-29T07:28:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:28:02 smithi153 ceph-mon[70044]: pgmap v3595: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:02.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:28:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:28:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:28:02.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:28:02 smithi083 ceph-mon[79930]: pgmap v3595: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:02.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:28:02 smithi083 ceph-mon[77242]: pgmap v3595: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:28:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:28:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:28:04.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:28:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:28:03.964Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:28:04.758 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:28:04.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:28:04 smithi083 ceph-mon[79930]: pgmap v3596: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:04.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:28:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:28:04.662Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:28:04.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:28:04 smithi083 ceph-mon[77242]: pgmap v3596: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:05.051 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:28:05.051 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 2m ago 2h 26.7M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:28:05.051 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 19s ago 2h 61.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:28:05.051 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 2m ago 2h 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:28:05.052 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 19s ago 2h 27.8M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:28:05.052 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (118m) 19s ago 2h 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:28:05.052 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (119m) 2m ago 2h 617M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:28:05.052 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (118m) 2m ago 2h 494M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:28:05.053 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (117m) 19s ago 2h 221M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:28:05.053 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (118m) 2m ago 2h 291M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:28:05.053 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 2m ago 2h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:28:05.053 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 19s ago 2h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:28:05.054 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (117m) 2m ago 2h 361M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:28:05.054 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (116m) 2m ago 2h 352M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:28:05.054 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (116m) 2m ago 2h 280M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:28:05.054 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (115m) 2m ago 2h 340M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:28:05.055 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (115m) 19s ago 2h 305M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:28:05.055 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (114m) 19s ago 2h 372M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:28:05.055 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (114m) 19s ago 2h 331M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:28:05.055 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (113m) 19s ago 2h 385M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:28:05.056 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (118m) 19s ago 2h 120M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:28:05.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:28:04 smithi153 ceph-mon[70044]: pgmap v3596: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:05.403 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:28:05.403 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:28:05.404 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:28:05.404 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:28:05.404 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:28:05.404 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:28:05.405 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:28:05.405 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:28:05.405 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:28:05.405 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:28:05.406 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:28:05.406 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:28:05.406 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:28:05.406 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:28:05.407 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:28:05.407 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:28:05.407 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:28:05.407 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:28:06.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:28:05 smithi083 ceph-mon[79930]: from='client.48253 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:28:06.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:28:05 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3578310671' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:28:06.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:28:05 smithi083 ceph-mon[77242]: from='client.48253 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:28:06.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:28:05 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3578310671' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:28:06.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:28:05 smithi153 ceph-mon[70044]: from='client.48253 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:28:06.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:28:05 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3578310671' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:28:07.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:28:06 smithi083 ceph-mon[79930]: from='client.48259 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:28:07.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:28:06 smithi083 ceph-mon[79930]: pgmap v3597: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:07.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:28:06 smithi083 ceph-mon[77242]: from='client.48259 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:28:07.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:28:06 smithi083 ceph-mon[77242]: pgmap v3597: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:07.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:28:06 smithi153 ceph-mon[70044]: from='client.48259 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:28:07.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:28:06 smithi153 ceph-mon[70044]: pgmap v3597: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:09.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:28:08 smithi083 ceph-mon[79930]: pgmap v3598: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:09.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:28:08 smithi083 ceph-mon[77242]: pgmap v3598: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:09.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:28:08 smithi153 ceph-mon[70044]: pgmap v3598: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:10.841 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:28:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:28:10.566Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:28:11.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:28:10 smithi083 ceph-mon[79930]: pgmap v3599: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:11.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:28:10 smithi083 ceph-mon[77242]: pgmap v3599: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:11.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:28:10 smithi153 ceph-mon[70044]: pgmap v3599: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:28:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:28:11] "GET /metrics HTTP/1.1" 200 225174 "" "Prometheus/2.33.4" 2022-09-29T07:28:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:28:12 smithi153 ceph-mon[70044]: pgmap v3600: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:12.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:28:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:28:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:28:12.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:28:12 smithi083 ceph-mon[79930]: pgmap v3600: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:12.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:28:12 smithi083 ceph-mon[77242]: pgmap v3600: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:28:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:28:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:28:14.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:28:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:28:13.965Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:28:14.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:28:14 smithi083 ceph-mon[79930]: pgmap v3601: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:14.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:28:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:28:14.663Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:28:14.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:28:14 smithi083 ceph-mon[77242]: pgmap v3601: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:15.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:28:14 smithi153 ceph-mon[70044]: pgmap v3601: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:17.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:28:16 smithi083 ceph-mon[77242]: pgmap v3602: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:17.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:28:16 smithi083 ceph-mon[79930]: pgmap v3602: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:17.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:28:16 smithi153 ceph-mon[70044]: pgmap v3602: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:19.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:28:18 smithi083 ceph-mon[79930]: pgmap v3603: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:19.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:28:18 smithi083 ceph-mon[77242]: pgmap v3603: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:19.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:28:18 smithi153 ceph-mon[70044]: pgmap v3603: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:20.840 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:28:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:28:20.567Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:28:21.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:28:20 smithi083 ceph-mon[79930]: pgmap v3604: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:21.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:28:20 smithi083 ceph-mon[77242]: pgmap v3604: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:21.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:28:20 smithi153 ceph-mon[70044]: pgmap v3604: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:28:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:28:21] "GET /metrics HTTP/1.1" 200 225174 "" "Prometheus/2.33.4" 2022-09-29T07:28:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:28:22 smithi153 ceph-mon[70044]: pgmap v3605: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:22.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:28:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:28:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:28:22.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:28:22 smithi083 ceph-mon[79930]: pgmap v3605: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:22.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:28:22 smithi083 ceph-mon[77242]: pgmap v3605: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:28:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:28:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:28:24.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:28:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:28:23.965Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:28:24.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:28:24 smithi083 ceph-mon[77242]: pgmap v3606: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:24.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:28:24 smithi083 ceph-mon[79930]: pgmap v3606: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:24.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:28:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:28:24.663Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:28:25.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:28:24 smithi153 ceph-mon[70044]: pgmap v3606: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:27.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:28:26 smithi083 ceph-mon[77242]: pgmap v3607: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:27.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:28:26 smithi083 ceph-mon[79930]: pgmap v3607: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:27.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:28:26 smithi153 ceph-mon[70044]: pgmap v3607: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:29.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:28:28 smithi083 ceph-mon[79930]: pgmap v3608: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:29.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:28:28 smithi083 ceph-mon[77242]: pgmap v3608: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:29.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:28:28 smithi153 ceph-mon[70044]: pgmap v3608: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:30.843 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:28:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:28:30.567Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:28:31.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:28:30 smithi083 ceph-mon[79930]: pgmap v3609: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:31.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:28:30 smithi083 ceph-mon[77242]: pgmap v3609: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:31.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:28:30 smithi153 ceph-mon[70044]: pgmap v3609: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:28:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:28:31] "GET /metrics HTTP/1.1" 200 225171 "" "Prometheus/2.33.4" 2022-09-29T07:28:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:28:32 smithi153 ceph-mon[70044]: pgmap v3610: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:32.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:28:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:28:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:28:32.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:28:32 smithi083 ceph-mon[77242]: pgmap v3610: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:32.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:28:32 smithi083 ceph-mon[79930]: pgmap v3610: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:28:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:28:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:28:34.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:28:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:28:33.966Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:28:34.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:28:34 smithi083 ceph-mon[79930]: pgmap v3611: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:34.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:28:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:28:34.663Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:28:34.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:28:34 smithi083 ceph-mon[77242]: pgmap v3611: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:35.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:28:34 smithi153 ceph-mon[70044]: pgmap v3611: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:35.731 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:28:36.018 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:28:36.019 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 2m ago 2h 26.7M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:28:36.019 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 50s ago 2h 61.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:28:36.019 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 2m ago 2h 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:28:36.020 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 50s ago 2h 27.8M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:28:36.020 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (119m) 50s ago 2h 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:28:36.020 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (119m) 2m ago 2h 617M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:28:36.020 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (119m) 2m ago 2h 494M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:28:36.020 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (118m) 50s ago 2h 221M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:28:36.021 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (118m) 2m ago 2h 291M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:28:36.021 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 2m ago 2h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:28:36.021 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 50s ago 2h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:28:36.021 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (117m) 2m ago 2h 361M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:28:36.021 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (117m) 2m ago 2h 352M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:28:36.022 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (116m) 2m ago 2h 280M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:28:36.022 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (116m) 2m ago 2h 340M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:28:36.022 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (115m) 50s ago 2h 305M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:28:36.022 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (115m) 50s ago 2h 372M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:28:36.022 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (114m) 50s ago 2h 331M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:28:36.023 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (114m) 50s ago 2h 385M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:28:36.023 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (119m) 50s ago 2h 120M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:28:36.377 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:28:36.378 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:28:36.378 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:28:36.378 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:28:36.379 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:28:36.379 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:28:36.379 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:28:36.379 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:28:36.380 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:28:36.380 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:28:36.380 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:28:36.380 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:28:36.381 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:28:36.381 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:28:36.381 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:28:36.381 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:28:36.382 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:28:36.382 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:28:37.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:28:36 smithi083 ceph-mon[79930]: from='client.38337 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:28:37.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:28:36 smithi083 ceph-mon[79930]: pgmap v3612: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:37.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:28:36 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/197686683' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:28:37.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:28:36 smithi083 ceph-mon[77242]: from='client.38337 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:28:37.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:28:36 smithi083 ceph-mon[77242]: pgmap v3612: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:37.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:28:36 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/197686683' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:28:37.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:28:36 smithi153 ceph-mon[70044]: from='client.38337 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:28:37.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:28:36 smithi153 ceph-mon[70044]: pgmap v3612: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:37.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:28:36 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/197686683' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:28:38.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:28:37 smithi083 ceph-mon[79930]: from='client.38343 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:28:38.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:28:37 smithi083 ceph-mon[77242]: from='client.38343 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:28:38.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:28:37 smithi153 ceph-mon[70044]: from='client.38343 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:28:39.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:28:38 smithi083 ceph-mon[79930]: pgmap v3613: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:39.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:28:38 smithi083 ceph-mon[77242]: pgmap v3613: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:39.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:28:38 smithi153 ceph-mon[70044]: pgmap v3613: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:40.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:28:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:28:40.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:28:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:28:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:28:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:28:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:28:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:28:40.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:28:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:28:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:28:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:28:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:28:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:28:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:28:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:28:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:28:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:28:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:28:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:28:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:28:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:28:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:28:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:28:40.852 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:28:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:28:40.568Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:28:41.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:28:40 smithi083 ceph-mon[77242]: pgmap v3614: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:41.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:28:40 smithi083 ceph-mon[79930]: pgmap v3614: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:28:40 smithi153 ceph-mon[70044]: pgmap v3614: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:28:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:28:41] "GET /metrics HTTP/1.1" 200 225178 "" "Prometheus/2.33.4" 2022-09-29T07:28:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:28:42 smithi153 ceph-mon[70044]: pgmap v3615: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:42.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:28:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:28:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:28:42.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:28:42 smithi083 ceph-mon[79930]: pgmap v3615: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:42.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:28:42 smithi083 ceph-mon[77242]: pgmap v3615: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:28:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:28:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:28:44.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:28:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:28:43.966Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:28:44.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:28:44 smithi083 ceph-mon[79930]: pgmap v3616: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:44.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:28:44 smithi083 ceph-mon[77242]: pgmap v3616: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:44.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:28:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:28:44.664Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:28:45.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:28:44 smithi153 ceph-mon[70044]: pgmap v3616: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:47.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:28:46 smithi083 ceph-mon[79930]: pgmap v3617: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:47.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:28:46 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:28:47.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:28:46 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:28:47.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:28:46 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:28:47.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:28:46 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:28:47.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:28:46 smithi083 ceph-mon[77242]: pgmap v3617: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:47.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:28:46 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:28:47.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:28:46 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:28:47.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:28:46 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:28:47.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:28:46 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:28:47.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:28:46 smithi153 ceph-mon[70044]: pgmap v3617: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:47.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:28:46 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:28:47.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:28:46 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:28:47.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:28:46 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:28:47.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:28:46 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:28:49.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:28:48 smithi083 ceph-mon[79930]: pgmap v3618: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:49.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:28:48 smithi083 ceph-mon[77242]: pgmap v3618: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:49.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:28:48 smithi153 ceph-mon[70044]: pgmap v3618: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:50.852 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:28:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:28:50.569Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:28:51.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:28:50 smithi083 ceph-mon[79930]: pgmap v3619: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:51.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:28:50 smithi083 ceph-mon[77242]: pgmap v3619: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:51.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:28:50 smithi153 ceph-mon[70044]: pgmap v3619: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:28:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:28:51] "GET /metrics HTTP/1.1" 200 225178 "" "Prometheus/2.33.4" 2022-09-29T07:28:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:28:52 smithi153 ceph-mon[70044]: pgmap v3620: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:52.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:28:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:28:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:28:52.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:28:52 smithi083 ceph-mon[77242]: pgmap v3620: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:52.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:28:52 smithi083 ceph-mon[79930]: pgmap v3620: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:28:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:28:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:28:54.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:28:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:28:53.967Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:28:54.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:28:54 smithi083 ceph-mon[77242]: pgmap v3621: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:54.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:28:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:28:54.666Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:28:54.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:28:54 smithi083 ceph-mon[79930]: pgmap v3621: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:55.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:28:54 smithi153 ceph-mon[70044]: pgmap v3621: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:57.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:28:56 smithi083 ceph-mon[79930]: pgmap v3622: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:57.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:28:56 smithi083 ceph-mon[77242]: pgmap v3622: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:57.314 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:28:56 smithi153 ceph-mon[70044]: pgmap v3622: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:59.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:28:58 smithi083 ceph-mon[79930]: pgmap v3623: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:59.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:28:58 smithi083 ceph-mon[77242]: pgmap v3623: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:28:59.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:28:58 smithi153 ceph-mon[70044]: pgmap v3623: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:00.854 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:29:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:29:00.569Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:29:01.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:29:00 smithi083 ceph-mon[79930]: pgmap v3624: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:01.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:29:00 smithi083 ceph-mon[77242]: pgmap v3624: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:01.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:29:00 smithi153 ceph-mon[70044]: pgmap v3624: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:29:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:29:01] "GET /metrics HTTP/1.1" 200 225177 "" "Prometheus/2.33.4" 2022-09-29T07:29:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:29:02 smithi153 ceph-mon[70044]: pgmap v3625: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:02.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:29:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:29:02.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:29:02.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:29:02 smithi083 ceph-mon[77242]: pgmap v3625: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:02.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:29:02 smithi083 ceph-mon[79930]: pgmap v3625: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:29:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:29:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:29:04.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:29:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:29:03.968Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:29:04.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:29:04 smithi083 ceph-mon[79930]: pgmap v3626: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:04.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:29:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:29:04.667Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:29:04.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:29:04 smithi083 ceph-mon[77242]: pgmap v3626: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:05.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:29:04 smithi153 ceph-mon[70044]: pgmap v3626: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:06.702 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:29:06.991 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:29:06.991 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 3m ago 2h 26.7M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:29:06.991 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 81s ago 2h 61.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:29:06.992 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 3m ago 2h 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:29:06.992 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 81s ago 2h 27.8M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:29:06.992 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (119m) 81s ago 2h 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:29:06.992 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 3m ago 2h 617M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:29:06.993 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (119m) 3m ago 2h 494M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:29:06.993 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (119m) 81s ago 2h 221M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:29:06.993 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (119m) 3m ago 2h 291M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:29:06.993 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 3m ago 2h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:29:06.993 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 81s ago 2h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:29:06.994 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (118m) 3m ago 2h 361M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:29:06.994 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (117m) 3m ago 2h 352M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:29:06.994 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (117m) 3m ago 2h 280M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:29:06.994 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (116m) 3m ago 2h 340M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:29:06.995 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (116m) 81s ago 2h 305M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:29:06.995 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (115m) 81s ago 2h 372M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:29:06.995 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (115m) 81s ago 2h 331M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:29:06.995 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (114m) 81s ago 2h 385M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:29:06.995 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (119m) 81s ago 2h 120M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:29:07.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:29:06 smithi083 ceph-mon[79930]: pgmap v3627: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:07.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:29:06 smithi083 ceph-mon[77242]: pgmap v3627: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:07.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:29:06 smithi153 ceph-mon[70044]: pgmap v3627: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:07.349 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:29:07.349 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:29:07.350 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:29:07.350 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:29:07.350 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:29:07.350 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:29:07.351 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:29:07.351 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:29:07.351 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:29:07.351 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:29:07.351 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:29:07.352 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:29:07.352 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:29:07.352 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:29:07.352 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:29:07.352 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:29:07.353 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:29:07.353 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:29:08.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:29:07 smithi083 ceph-mon[79930]: from='client.48289 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:29:08.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:29:07 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/923104355' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:29:08.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:29:07 smithi083 ceph-mon[77242]: from='client.48289 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:29:08.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:29:07 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/923104355' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:29:08.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:29:07 smithi153 ceph-mon[70044]: from='client.48289 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:29:08.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:29:07 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/923104355' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:29:09.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:29:08 smithi083 ceph-mon[79930]: from='client.38361 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:29:09.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:29:08 smithi083 ceph-mon[79930]: pgmap v3628: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:09.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:29:08 smithi083 ceph-mon[77242]: from='client.38361 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:29:09.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:29:08 smithi083 ceph-mon[77242]: pgmap v3628: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:09.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:29:08 smithi153 ceph-mon[70044]: from='client.38361 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:29:09.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:29:08 smithi153 ceph-mon[70044]: pgmap v3628: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:10.863 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:29:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:29:10.570Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:29:11.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:29:10 smithi083 ceph-mon[79930]: pgmap v3629: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:11.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:29:10 smithi083 ceph-mon[77242]: pgmap v3629: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:11.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:29:10 smithi153 ceph-mon[70044]: pgmap v3629: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:29:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:29:11] "GET /metrics HTTP/1.1" 200 225179 "" "Prometheus/2.33.4" 2022-09-29T07:29:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:29:12 smithi153 ceph-mon[70044]: pgmap v3630: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:12.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:29:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:29:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:29:12.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:29:12 smithi083 ceph-mon[77242]: pgmap v3630: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:12.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:29:12 smithi083 ceph-mon[79930]: pgmap v3630: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:29:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:29:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:29:14.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:29:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:29:13.968Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:29:14.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:29:14 smithi083 ceph-mon[79930]: pgmap v3631: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:14.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:29:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:29:14.668Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:29:14.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:29:14 smithi083 ceph-mon[77242]: pgmap v3631: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:15.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:29:14 smithi153 ceph-mon[70044]: pgmap v3631: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:17.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:29:16 smithi083 ceph-mon[79930]: pgmap v3632: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:17.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:29:16 smithi083 ceph-mon[77242]: pgmap v3632: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:17.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:29:16 smithi153 ceph-mon[70044]: pgmap v3632: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:19.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:29:18 smithi083 ceph-mon[79930]: pgmap v3633: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:19.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:29:18 smithi083 ceph-mon[77242]: pgmap v3633: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:19.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:29:18 smithi153 ceph-mon[70044]: pgmap v3633: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:20.860 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:29:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:29:20.571Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:29:21.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:29:20 smithi083 ceph-mon[79930]: pgmap v3634: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:21.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:29:20 smithi083 ceph-mon[77242]: pgmap v3634: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:21.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:29:20 smithi153 ceph-mon[70044]: pgmap v3634: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:29:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:29:21] "GET /metrics HTTP/1.1" 200 225179 "" "Prometheus/2.33.4" 2022-09-29T07:29:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:29:22 smithi153 ceph-mon[70044]: pgmap v3635: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:22.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:29:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:29:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:29:22.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:29:22 smithi083 ceph-mon[79930]: pgmap v3635: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:22.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:29:22 smithi083 ceph-mon[77242]: pgmap v3635: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:29:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:29:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:29:24.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:29:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:29:23.968Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:29:24.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:29:24 smithi083 ceph-mon[79930]: pgmap v3636: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:24.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:29:24 smithi083 ceph-mon[77242]: pgmap v3636: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:24.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:29:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:29:24.669Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:29:25.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:29:24 smithi153 ceph-mon[70044]: pgmap v3636: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:27.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:29:26 smithi083 ceph-mon[79930]: pgmap v3637: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:27.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:29:26 smithi083 ceph-mon[77242]: pgmap v3637: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:27.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:29:26 smithi153 ceph-mon[70044]: pgmap v3637: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:29.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:29:28 smithi083 ceph-mon[79930]: pgmap v3638: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:29.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:29:28 smithi083 ceph-mon[77242]: pgmap v3638: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:29.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:29:28 smithi153 ceph-mon[70044]: pgmap v3638: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:30.862 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:29:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:29:30.571Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:29:31.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:29:30 smithi083 ceph-mon[79930]: pgmap v3639: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:31.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:29:30 smithi083 ceph-mon[77242]: pgmap v3639: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:31.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:29:30 smithi153 ceph-mon[70044]: pgmap v3639: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:29:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:29:31] "GET /metrics HTTP/1.1" 200 225180 "" "Prometheus/2.33.4" 2022-09-29T07:29:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:29:32 smithi153 ceph-mon[70044]: pgmap v3640: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:32.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:29:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:29:32.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:29:32.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:29:32 smithi083 ceph-mon[79930]: pgmap v3640: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:32.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:29:32 smithi083 ceph-mon[77242]: pgmap v3640: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:29:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:29:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:29:34.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:29:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:29:33.969Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:29:34.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:29:34 smithi083 ceph-mon[79930]: pgmap v3641: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:34.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:29:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:29:34.669Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:29:34.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:29:34 smithi083 ceph-mon[77242]: pgmap v3641: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:35.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:29:34 smithi153 ceph-mon[70044]: pgmap v3641: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:37.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:29:36 smithi083 ceph-mon[79930]: pgmap v3642: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:37.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:29:36 smithi083 ceph-mon[77242]: pgmap v3642: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:37.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:29:36 smithi153 ceph-mon[70044]: pgmap v3642: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:37.674 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:29:37.963 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:29:37.964 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 3m ago 2h 26.7M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:29:37.964 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 112s ago 2h 61.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:29:37.964 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 3m ago 2h 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:29:37.964 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 112s ago 2h 27.8M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:29:37.965 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 112s ago 2h 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:29:37.965 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 3m ago 2h 617M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:29:37.965 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 3m ago 2h 494M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:29:37.965 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (119m) 112s ago 2h 221M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:29:37.966 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (119m) 3m ago 2h 291M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:29:37.966 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 3m ago 2h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:29:37.966 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 112s ago 2h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:29:37.966 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (118m) 3m ago 2h 361M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:29:37.967 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (118m) 3m ago 2h 352M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:29:37.967 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (117m) 3m ago 2h 280M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:29:37.967 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (117m) 3m ago 2h 340M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:29:37.967 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (116m) 112s ago 2h 305M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:29:37.968 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (116m) 112s ago 2h 372M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:29:37.968 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (115m) 112s ago 2h 331M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:29:37.968 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (115m) 112s ago 2h 385M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:29:37.968 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 112s ago 2h 120M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:29:38.319 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:29:38.319 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:29:38.319 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:29:38.320 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:29:38.320 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:29:38.320 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:29:38.320 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:29:38.320 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:29:38.321 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:29:38.321 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:29:38.321 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:29:38.321 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:29:38.322 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:29:38.322 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:29:38.322 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:29:38.322 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:29:38.323 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:29:38.323 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:29:39.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:29:38 smithi083 ceph-mon[79930]: from='client.38373 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:29:39.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:29:38 smithi083 ceph-mon[79930]: pgmap v3643: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:39.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:29:38 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2456952251' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:29:39.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:29:38 smithi083 ceph-mon[77242]: from='client.38373 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:29:39.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:29:38 smithi083 ceph-mon[77242]: pgmap v3643: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:39.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:29:38 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2456952251' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:29:39.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:29:38 smithi153 ceph-mon[70044]: from='client.38373 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:29:39.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:29:38 smithi153 ceph-mon[70044]: pgmap v3643: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:39.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:29:38 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2456952251' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:29:40.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:29:39 smithi083 ceph-mon[79930]: from='client.48313 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:29:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:29:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:29:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:29:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:29:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:29:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:29:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:29:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:29:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:29:39 smithi083 ceph-mon[77242]: from='client.48313 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:29:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:29:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:29:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:29:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:29:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:29:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:29:40.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:29:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:29:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:29:39 smithi153 ceph-mon[70044]: from='client.48313 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:29:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:29:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:29:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:29:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:29:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:29:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:29:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:29:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:29:40.870 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:29:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:29:40.573Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:29:41.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:29:40 smithi083 ceph-mon[79930]: pgmap v3644: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:41.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:29:40 smithi083 ceph-mon[77242]: pgmap v3644: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:29:40 smithi153 ceph-mon[70044]: pgmap v3644: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:29:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:29:41] "GET /metrics HTTP/1.1" 200 225177 "" "Prometheus/2.33.4" 2022-09-29T07:29:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:29:42 smithi153 ceph-mon[70044]: pgmap v3645: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:42.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:29:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:29:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:29:42.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:29:42 smithi083 ceph-mon[79930]: pgmap v3645: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:42.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:29:42 smithi083 ceph-mon[77242]: pgmap v3645: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:29:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:29:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:29:44.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:29:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:29:43.970Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:29:44.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:29:44 smithi083 ceph-mon[79930]: pgmap v3646: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:44.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:29:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:29:44.669Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:29:44.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:29:44 smithi083 ceph-mon[77242]: pgmap v3646: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:45.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:29:44 smithi153 ceph-mon[70044]: pgmap v3646: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:47.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:29:46 smithi083 ceph-mon[79930]: pgmap v3647: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:47.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:29:46 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:29:47.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:29:46 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:29:47.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:29:46 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:29:47.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:29:46 smithi083 ceph-mon[77242]: pgmap v3647: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:47.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:29:46 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:29:47.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:29:46 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:29:47.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:29:46 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:29:47.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:29:46 smithi153 ceph-mon[70044]: pgmap v3647: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:47.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:29:46 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:29:47.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:29:46 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:29:47.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:29:46 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:29:48.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:29:48 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:29:48.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:29:48 smithi083 ceph-mon[79930]: pgmap v3648: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:48.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:29:48 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:29:48.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:29:48 smithi083 ceph-mon[77242]: pgmap v3648: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:48.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:29:48 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:29:48.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:29:48 smithi153 ceph-mon[70044]: pgmap v3648: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:50.861 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:29:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:29:50.573Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:29:51.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:29:50 smithi083 ceph-mon[79930]: pgmap v3649: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:51.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:29:50 smithi083 ceph-mon[77242]: pgmap v3649: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:51.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:29:50 smithi153 ceph-mon[70044]: pgmap v3649: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:29:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:29:51] "GET /metrics HTTP/1.1" 200 225177 "" "Prometheus/2.33.4" 2022-09-29T07:29:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:29:52 smithi153 ceph-mon[70044]: pgmap v3650: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:52.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:29:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:29:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:29:52.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:29:52 smithi083 ceph-mon[79930]: pgmap v3650: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:52.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:29:52 smithi083 ceph-mon[77242]: pgmap v3650: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:29:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:29:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:29:54.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:29:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:29:53.970Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:29:54.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:29:54 smithi083 ceph-mon[79930]: pgmap v3651: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:54.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:29:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:29:54.670Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:29:54.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:29:54 smithi083 ceph-mon[77242]: pgmap v3651: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:55.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:29:54 smithi153 ceph-mon[70044]: pgmap v3651: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:57.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:29:56 smithi083 ceph-mon[79930]: pgmap v3652: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:57.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:29:56 smithi083 ceph-mon[77242]: pgmap v3652: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:57.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:29:56 smithi153 ceph-mon[70044]: pgmap v3652: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:59.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:29:58 smithi083 ceph-mon[77242]: pgmap v3653: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:59.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:29:58 smithi083 ceph-mon[79930]: pgmap v3653: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:29:59.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:29:58 smithi153 ceph-mon[70044]: pgmap v3653: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:00.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:30:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[77219]: 2022-09-29T07:29:59.999+0000 7f160f665700 -1 log_channel(cluster) log [ERR] : overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T07:30:00.870 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:30:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:30:00.573Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:30:01.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:30:00 smithi083 ceph-mon[77242]: pgmap v3654: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:01.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:30:00 smithi083 ceph-mon[77242]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T07:30:01.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:30:00 smithi083 ceph-mon[79930]: pgmap v3654: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:01.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:30:00 smithi083 ceph-mon[79930]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T07:30:01.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:30:00 smithi153 ceph-mon[70044]: pgmap v3654: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:01.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:30:00 smithi153 ceph-mon[70044]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T07:30:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:30:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:30:01] "GET /metrics HTTP/1.1" 200 225178 "" "Prometheus/2.33.4" 2022-09-29T07:30:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:30:02 smithi153 ceph-mon[70044]: pgmap v3655: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:02.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:30:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:30:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:30:02.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:30:02 smithi083 ceph-mon[77242]: pgmap v3655: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:02.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:30:02 smithi083 ceph-mon[79930]: pgmap v3655: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:30:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:30:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:30:04.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:30:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:30:03.970Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:30:04.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:30:04 smithi083 ceph-mon[77242]: pgmap v3656: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:04.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:30:04 smithi083 ceph-mon[79930]: pgmap v3656: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:04.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:30:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:30:04.671Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:30:05.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:30:04 smithi153 ceph-mon[70044]: pgmap v3656: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:07.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:30:06 smithi083 ceph-mon[79930]: pgmap v3657: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:07.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:30:06 smithi083 ceph-mon[77242]: pgmap v3657: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:07.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:30:06 smithi153 ceph-mon[70044]: pgmap v3657: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:08.640 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:30:08.929 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:30:08.930 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 4m ago 2h 26.7M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:30:08.930 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 2m ago 2h 61.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:30:08.930 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 4m ago 2h 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:30:08.931 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 2m ago 2h 27.8M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:30:08.931 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 2m ago 2h 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:30:08.931 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 4m ago 2h 617M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:30:08.931 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 4m ago 2h 494M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:30:08.931 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 2m ago 2h 221M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:30:08.932 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 4m ago 2h 291M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:30:08.932 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 4m ago 2h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:30:08.932 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 2m ago 2h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:30:08.932 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (119m) 4m ago 2h 361M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:30:08.932 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (118m) 4m ago 2h 352M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:30:08.933 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (118m) 4m ago 2h 280M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:30:08.933 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (117m) 4m ago 2h 340M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:30:08.933 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (117m) 2m ago 2h 305M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:30:08.933 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (116m) 2m ago 2h 372M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:30:08.934 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (116m) 2m ago 2h 331M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:30:08.934 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (115m) 2m ago 2h 385M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:30:08.934 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 2m ago 2h 120M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:30:09.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:30:08 smithi083 ceph-mon[79930]: pgmap v3658: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:09.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:30:08 smithi083 ceph-mon[77242]: pgmap v3658: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:09.290 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:30:09.290 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:30:09.291 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:30:09.291 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:30:09.291 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:30:09.291 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:30:09.292 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:30:09.292 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:30:09.292 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:30:09.292 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:30:09.293 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:30:09.293 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:30:09.293 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:30:09.293 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:30:09.294 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:30:09.294 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:30:09.295 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:30:09.295 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:30:09.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:30:08 smithi153 ceph-mon[70044]: pgmap v3658: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:10.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:30:09 smithi083 ceph-mon[79930]: from='client.48325 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:30:10.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:30:09 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4171698492' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:30:10.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:30:09 smithi083 ceph-mon[77242]: from='client.48325 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:30:10.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:30:09 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4171698492' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:30:10.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:30:09 smithi153 ceph-mon[70044]: from='client.48325 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:30:10.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:30:09 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4171698492' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:30:10.877 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:30:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:30:10.574Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:30:11.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:30:10 smithi083 ceph-mon[79930]: from='client.38397 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:30:11.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:30:10 smithi083 ceph-mon[79930]: pgmap v3659: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:11.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:30:10 smithi083 ceph-mon[77242]: from='client.38397 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:30:11.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:30:10 smithi083 ceph-mon[77242]: pgmap v3659: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:11.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:30:10 smithi153 ceph-mon[70044]: from='client.38397 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:30:11.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:30:10 smithi153 ceph-mon[70044]: pgmap v3659: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:30:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:30:11] "GET /metrics HTTP/1.1" 200 225174 "" "Prometheus/2.33.4" 2022-09-29T07:30:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:30:12 smithi153 ceph-mon[70044]: pgmap v3660: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:12.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:30:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:30:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:30:12.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:30:12 smithi083 ceph-mon[79930]: pgmap v3660: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:12.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:30:12 smithi083 ceph-mon[77242]: pgmap v3660: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:30:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:30:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:30:14.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:30:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:30:13.971Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:30:14.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:30:14 smithi083 ceph-mon[79930]: pgmap v3661: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:14.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:30:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:30:14.671Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:30:14.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:30:14 smithi083 ceph-mon[77242]: pgmap v3661: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:15.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:30:14 smithi153 ceph-mon[70044]: pgmap v3661: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:17.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:30:16 smithi083 ceph-mon[79930]: pgmap v3662: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:17.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:30:16 smithi083 ceph-mon[77242]: pgmap v3662: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:17.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:30:16 smithi153 ceph-mon[70044]: pgmap v3662: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:19.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:30:18 smithi083 ceph-mon[77242]: pgmap v3663: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:19.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:30:18 smithi083 ceph-mon[79930]: pgmap v3663: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:19.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:30:18 smithi153 ceph-mon[70044]: pgmap v3663: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:20.878 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:30:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:30:20.575Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:30:21.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:30:20 smithi083 ceph-mon[77242]: pgmap v3664: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:21.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:30:20 smithi083 ceph-mon[79930]: pgmap v3664: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:21.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:30:20 smithi153 ceph-mon[70044]: pgmap v3664: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:30:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:30:21] "GET /metrics HTTP/1.1" 200 225174 "" "Prometheus/2.33.4" 2022-09-29T07:30:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:30:22 smithi153 ceph-mon[70044]: pgmap v3665: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:22.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:30:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:30:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:30:22.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:30:22 smithi083 ceph-mon[79930]: pgmap v3665: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:22.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:30:22 smithi083 ceph-mon[77242]: pgmap v3665: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:30:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:30:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:30:24.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:30:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:30:23.972Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:30:24.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:30:24 smithi083 ceph-mon[77242]: pgmap v3666: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:24.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:30:24 smithi083 ceph-mon[79930]: pgmap v3666: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:24.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:30:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:30:24.672Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:30:25.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:30:24 smithi153 ceph-mon[70044]: pgmap v3666: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:27.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:30:26 smithi083 ceph-mon[79930]: pgmap v3667: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:27.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:30:26 smithi083 ceph-mon[77242]: pgmap v3667: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:27.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:30:26 smithi153 ceph-mon[70044]: pgmap v3667: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:29.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:30:28 smithi083 ceph-mon[79930]: pgmap v3668: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:29.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:30:28 smithi083 ceph-mon[77242]: pgmap v3668: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:29.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:30:28 smithi153 ceph-mon[70044]: pgmap v3668: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:30.881 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:30:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:30:30.576Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:30:31.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:30:30 smithi083 ceph-mon[77242]: pgmap v3669: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:31.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:30:30 smithi083 ceph-mon[79930]: pgmap v3669: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:31.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:30:30 smithi153 ceph-mon[70044]: pgmap v3669: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:30:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:30:31] "GET /metrics HTTP/1.1" 200 225181 "" "Prometheus/2.33.4" 2022-09-29T07:30:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:30:32 smithi153 ceph-mon[70044]: pgmap v3670: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:32.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:30:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:30:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:30:32.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:30:32 smithi083 ceph-mon[79930]: pgmap v3670: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:32.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:30:32 smithi083 ceph-mon[77242]: pgmap v3670: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:30:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:30:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:30:34.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:30:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:30:33.972Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:30:34.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:30:34 smithi083 ceph-mon[77242]: pgmap v3671: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:34.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:30:34 smithi083 ceph-mon[79930]: pgmap v3671: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:34.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:30:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:30:34.673Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:30:35.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:30:34 smithi153 ceph-mon[70044]: pgmap v3671: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:37.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:30:36 smithi083 ceph-mon[77242]: pgmap v3672: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:37.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:30:36 smithi083 ceph-mon[79930]: pgmap v3672: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:37.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:30:36 smithi153 ceph-mon[70044]: pgmap v3672: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:39.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:30:38 smithi083 ceph-mon[79930]: pgmap v3673: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:39.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:30:38 smithi083 ceph-mon[77242]: pgmap v3673: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:39.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:30:38 smithi153 ceph-mon[70044]: pgmap v3673: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:39.622 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:30:39.920 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:30:39.921 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 5m ago 2h 26.7M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:30:39.921 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 2m ago 2h 61.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:30:39.921 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 5m ago 2h 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:30:39.921 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 2m ago 2h 27.8M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:30:39.922 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 2m ago 2h 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:30:39.922 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 5m ago 2h 617M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:30:39.922 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 5m ago 2h 494M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:30:39.922 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 2m ago 2h 221M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:30:39.923 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 5m ago 2h 291M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:30:39.923 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 5m ago 2h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:30:39.923 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 2m ago 2h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:30:39.923 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (119m) 5m ago 2h 361M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:30:39.923 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (119m) 5m ago 2h 352M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:30:39.924 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (118m) 5m ago 2h 280M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:30:39.924 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (118m) 5m ago 2h 340M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:30:39.924 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (117m) 2m ago 2h 305M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:30:39.924 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (117m) 2m ago 2h 372M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:30:39.925 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (116m) 2m ago 2h 331M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:30:39.925 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (116m) 2m ago 2h 385M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:30:39.925 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 2m ago 2h 120M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:30:40.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:30:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:30:40.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:30:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:30:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:30:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:30:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:30:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:30:40.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:30:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:30:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:30:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:30:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:30:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:30:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:30:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:30:40.278 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:30:40.279 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:30:40.279 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:30:40.279 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:30:40.279 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:30:40.280 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:30:40.280 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:30:40.280 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:30:40.280 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:30:40.280 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:30:40.281 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:30:40.281 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:30:40.281 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:30:40.281 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:30:40.282 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:30:40.282 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:30:40.282 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:30:40.282 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:30:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:30:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:30:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:30:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:30:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:30:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:30:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:30:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:30:40.887 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:30:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:30:40.575Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:30:41.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:30:40 smithi083 ceph-mon[77242]: from='client.48343 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:30:41.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:30:40 smithi083 ceph-mon[77242]: pgmap v3674: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:41.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:30:40 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2042991810' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:30:41.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:30:40 smithi083 ceph-mon[79930]: from='client.48343 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:30:41.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:30:40 smithi083 ceph-mon[79930]: pgmap v3674: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:41.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:30:40 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2042991810' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:30:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:30:40 smithi153 ceph-mon[70044]: from='client.48343 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:30:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:30:40 smithi153 ceph-mon[70044]: pgmap v3674: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:30:40 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2042991810' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:30:41.889 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:30:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:30:41] "GET /metrics HTTP/1.1" 200 225174 "" "Prometheus/2.33.4" 2022-09-29T07:30:41.889 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:30:41 smithi083 ceph-mon[77242]: from='client.48349 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:30:42.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:30:41 smithi083 ceph-mon[79930]: from='client.48349 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:30:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:30:41 smithi153 ceph-mon[70044]: from='client.48349 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:30:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:30:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:30:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:30:43.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:30:42 smithi083 ceph-mon[79930]: pgmap v3675: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:43.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:30:42 smithi083 ceph-mon[77242]: pgmap v3675: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:30:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:30:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:30:43.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:30:42 smithi153 ceph-mon[70044]: pgmap v3675: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:44.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:30:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:30:43.973Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:30:44.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:30:44 smithi083 ceph-mon[79930]: pgmap v3676: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:44.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:30:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:30:44.673Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:30:44.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:30:44 smithi083 ceph-mon[77242]: pgmap v3676: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:45.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:30:44 smithi153 ceph-mon[70044]: pgmap v3676: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:47.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:30:46 smithi083 ceph-mon[79930]: pgmap v3677: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:47.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:30:46 smithi083 ceph-mon[77242]: pgmap v3677: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:47.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:30:46 smithi153 ceph-mon[70044]: pgmap v3677: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:48.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:30:47 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:30:48.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:30:47 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:30:48.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:30:47 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:30:48.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:30:47 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:30:48.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:30:47 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:30:48.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:30:47 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:30:48.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:30:47 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:30:48.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:30:47 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:30:48.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:30:47 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:30:49.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:30:48 smithi083 ceph-mon[79930]: pgmap v3678: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:49.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:30:48 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:30:49.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:30:48 smithi083 ceph-mon[77242]: pgmap v3678: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:49.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:30:48 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:30:49.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:30:48 smithi153 ceph-mon[70044]: pgmap v3678: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:49.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:30:48 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:30:50.903 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:30:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:30:50.577Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:30:51.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:30:50 smithi083 ceph-mon[77242]: pgmap v3679: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:51.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:30:50 smithi083 ceph-mon[79930]: pgmap v3679: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:51.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:30:50 smithi153 ceph-mon[70044]: pgmap v3679: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:30:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:30:51] "GET /metrics HTTP/1.1" 200 225174 "" "Prometheus/2.33.4" 2022-09-29T07:30:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:30:52 smithi153 ceph-mon[70044]: pgmap v3680: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:52.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:30:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:30:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:30:52.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:30:52 smithi083 ceph-mon[79930]: pgmap v3680: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:52.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:30:52 smithi083 ceph-mon[77242]: pgmap v3680: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:30:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:30:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:30:54.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:30:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:30:53.973Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:30:54.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:30:54 smithi083 ceph-mon[79930]: pgmap v3681: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:54.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:30:54 smithi083 ceph-mon[77242]: pgmap v3681: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:54.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:30:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:30:54.675Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:30:55.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:30:54 smithi153 ceph-mon[70044]: pgmap v3681: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:57.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:30:56 smithi083 ceph-mon[79930]: pgmap v3682: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:57.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:30:56 smithi083 ceph-mon[77242]: pgmap v3682: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:57.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:30:56 smithi153 ceph-mon[70044]: pgmap v3682: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:59.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:30:58 smithi083 ceph-mon[79930]: pgmap v3683: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:59.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:30:58 smithi083 ceph-mon[77242]: pgmap v3683: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:30:59.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:30:58 smithi153 ceph-mon[70044]: pgmap v3683: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:00.890 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:31:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:31:00.576Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:31:01.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:31:00 smithi083 ceph-mon[79930]: pgmap v3684: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:01.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:31:00 smithi083 ceph-mon[77242]: pgmap v3684: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:01.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:31:00 smithi153 ceph-mon[70044]: pgmap v3684: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:31:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:31:01] "GET /metrics HTTP/1.1" 200 225175 "" "Prometheus/2.33.4" 2022-09-29T07:31:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:31:02 smithi153 ceph-mon[70044]: pgmap v3685: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:02.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:31:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:31:02.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:31:02.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:31:02 smithi083 ceph-mon[77242]: pgmap v3685: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:02.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:31:02 smithi083 ceph-mon[79930]: pgmap v3685: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:31:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:31:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:31:04.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:31:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:31:03.974Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:31:04.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:31:04 smithi083 ceph-mon[79930]: pgmap v3686: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:04.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:31:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:31:04.675Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:31:04.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:31:04 smithi083 ceph-mon[77242]: pgmap v3686: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:05.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:31:04 smithi153 ceph-mon[70044]: pgmap v3686: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:07.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:31:06 smithi083 ceph-mon[79930]: pgmap v3687: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:07.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:31:06 smithi083 ceph-mon[77242]: pgmap v3687: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:07.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:31:06 smithi153 ceph-mon[70044]: pgmap v3687: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:09.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:31:08 smithi083 ceph-mon[79930]: pgmap v3688: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:09.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:31:08 smithi083 ceph-mon[77242]: pgmap v3688: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:09.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:31:08 smithi153 ceph-mon[70044]: pgmap v3688: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:10.608 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:31:10.892 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:31:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:31:10.578Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:31:10.895 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:31:10.895 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 5m ago 2h 26.7M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:31:10.895 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 3m ago 2h 61.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:31:10.896 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 5m ago 2h 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:31:10.896 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 3m ago 2h 27.8M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:31:10.896 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 3m ago 2h 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:31:10.896 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 5m ago 2h 617M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:31:10.897 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 5m ago 2h 494M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:31:10.897 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 3m ago 2h 221M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:31:10.897 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 5m ago 2h 291M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:31:10.897 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 5m ago 2h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:31:10.898 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 3m ago 2h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:31:10.898 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 5m ago 2h 361M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:31:10.898 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (119m) 5m ago 2h 352M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:31:10.898 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (119m) 5m ago 2h 280M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:31:10.899 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (118m) 5m ago 2h 340M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:31:10.899 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (118m) 3m ago 2h 305M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:31:10.899 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (117m) 3m ago 2h 372M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:31:10.899 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (117m) 3m ago 2h 331M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:31:10.900 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (116m) 3m ago 2h 385M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:31:10.900 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 3m ago 2h 120M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:31:11.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:31:10 smithi083 ceph-mon[79930]: pgmap v3689: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:11.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:31:10 smithi083 ceph-mon[77242]: pgmap v3689: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:11.248 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:31:11.248 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:31:11.248 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:31:11.249 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:31:11.249 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:31:11.249 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:31:11.249 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:31:11.250 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:31:11.250 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:31:11.250 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:31:11.250 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:31:11.251 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:31:11.251 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:31:11.251 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:31:11.251 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:31:11.251 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:31:11.252 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:31:11.252 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:31:11.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:31:10 smithi153 ceph-mon[70044]: pgmap v3689: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:11.893 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:31:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:31:11] "GET /metrics HTTP/1.1" 200 225177 "" "Prometheus/2.33.4" 2022-09-29T07:31:12.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:31:11 smithi083 ceph-mon[77242]: from='client.48361 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:31:12.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:31:11 smithi083 ceph-mon[77242]: from='client.48367 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:31:12.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:31:11 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/627712889' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:31:12.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:31:11 smithi083 ceph-mon[79930]: from='client.48361 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:31:12.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:31:11 smithi083 ceph-mon[79930]: from='client.48367 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:31:12.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:31:11 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/627712889' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:31:12.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:31:11 smithi153 ceph-mon[70044]: from='client.48361 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:31:12.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:31:11 smithi153 ceph-mon[70044]: from='client.48367 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:31:12.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:31:11 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/627712889' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:31:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:31:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:31:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:31:13.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:31:12 smithi083 ceph-mon[77242]: pgmap v3690: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:13.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:31:12 smithi083 ceph-mon[79930]: pgmap v3690: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:13.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:31:12 smithi153 ceph-mon[70044]: pgmap v3690: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:31:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:31:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:31:14.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:31:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:31:13.975Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:31:14.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:31:14 smithi083 ceph-mon[79930]: pgmap v3691: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:14.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:31:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:31:14.675Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:31:14.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:31:14 smithi083 ceph-mon[77242]: pgmap v3691: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:15.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:31:14 smithi153 ceph-mon[70044]: pgmap v3691: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:17.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:31:16 smithi083 ceph-mon[79930]: pgmap v3692: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:17.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:31:16 smithi083 ceph-mon[77242]: pgmap v3692: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:17.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:31:16 smithi153 ceph-mon[70044]: pgmap v3692: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:19.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:31:18 smithi083 ceph-mon[79930]: pgmap v3693: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:19.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:31:18 smithi083 ceph-mon[77242]: pgmap v3693: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:19.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:31:18 smithi153 ceph-mon[70044]: pgmap v3693: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:20.906 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:31:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:31:20.578Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:31:21.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:31:20 smithi083 ceph-mon[79930]: pgmap v3694: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:21.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:31:20 smithi083 ceph-mon[77242]: pgmap v3694: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:21.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:31:20 smithi153 ceph-mon[70044]: pgmap v3694: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:31:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:31:21] "GET /metrics HTTP/1.1" 200 225177 "" "Prometheus/2.33.4" 2022-09-29T07:31:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:31:22 smithi153 ceph-mon[70044]: pgmap v3695: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:22.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:31:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:31:22.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:31:22.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:31:22 smithi083 ceph-mon[79930]: pgmap v3695: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:22.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:31:22 smithi083 ceph-mon[77242]: pgmap v3695: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:31:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:31:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:31:24.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:31:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:31:23.975Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:31:24.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:31:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:31:24.676Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:31:24.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:31:24 smithi083 ceph-mon[77242]: pgmap v3696: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:24.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:31:24 smithi083 ceph-mon[79930]: pgmap v3696: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:25.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:31:24 smithi153 ceph-mon[70044]: pgmap v3696: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:27.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:31:26 smithi083 ceph-mon[79930]: pgmap v3697: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:27.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:31:26 smithi083 ceph-mon[77242]: pgmap v3697: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:27.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:31:26 smithi153 ceph-mon[70044]: pgmap v3697: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:29.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:31:28 smithi083 ceph-mon[77242]: pgmap v3698: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:29.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:31:28 smithi083 ceph-mon[79930]: pgmap v3698: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:29.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:31:28 smithi153 ceph-mon[70044]: pgmap v3698: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:30.898 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:31:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:31:30.579Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:31:31.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:31:30 smithi083 ceph-mon[79930]: pgmap v3699: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:31.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:31:30 smithi083 ceph-mon[77242]: pgmap v3699: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:31.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:31:30 smithi153 ceph-mon[70044]: pgmap v3699: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:31:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:31:31] "GET /metrics HTTP/1.1" 200 225177 "" "Prometheus/2.33.4" 2022-09-29T07:31:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:31:32 smithi153 ceph-mon[70044]: pgmap v3700: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:32.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:31:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:31:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:31:32.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:31:32 smithi083 ceph-mon[77242]: pgmap v3700: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:32.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:31:32 smithi083 ceph-mon[79930]: pgmap v3700: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:31:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:31:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:31:34.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:31:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:31:33.977Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:31:34.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:31:34 smithi083 ceph-mon[79930]: pgmap v3701: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:34.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:31:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:31:34.677Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:31:34.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:31:34 smithi083 ceph-mon[77242]: pgmap v3701: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:35.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:31:34 smithi153 ceph-mon[70044]: pgmap v3701: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:37.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:31:36 smithi083 ceph-mon[79930]: pgmap v3702: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:37.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:31:36 smithi083 ceph-mon[77242]: pgmap v3702: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:37.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:31:36 smithi153 ceph-mon[70044]: pgmap v3702: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:39.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:31:38 smithi083 ceph-mon[79930]: pgmap v3703: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:39.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:31:38 smithi083 ceph-mon[77242]: pgmap v3703: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:39.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:31:38 smithi153 ceph-mon[70044]: pgmap v3703: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:40.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:31:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:31:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:31:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:31:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:31:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:31:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:31:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:31:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:31:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:31:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:31:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:31:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:31:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:31:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:31:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:31:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:31:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:31:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:31:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:31:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:31:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:31:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:31:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:31:40.907 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:31:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:31:40.580Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:31:41.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:31:40 smithi083 ceph-mon[79930]: pgmap v3704: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:41.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:31:40 smithi083 ceph-mon[77242]: pgmap v3704: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:31:40 smithi153 ceph-mon[70044]: pgmap v3704: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:41.576 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:31:41.863 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:31:41.864 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 6m ago 2h 26.7M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:31:41.864 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 3m ago 2h 61.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:31:41.864 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 6m ago 2h 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:31:41.864 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 3m ago 2h 27.8M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:31:41.865 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 3m ago 2h 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:31:41.865 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 6m ago 2h 617M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:31:41.865 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 6m ago 2h 494M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:31:41.865 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 3m ago 2h 221M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:31:41.866 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 6m ago 2h 291M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:31:41.866 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 6m ago 2h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:31:41.866 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 3m ago 2h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:31:41.866 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 6m ago 2h 361M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:31:41.867 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 6m ago 2h 352M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:31:41.867 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 6m ago 2h 280M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:31:41.867 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (119m) 6m ago 2h 340M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:31:41.867 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (118m) 3m ago 2h 305M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:31:41.868 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (118m) 3m ago 2h 372M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:31:41.868 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (117m) 3m ago 2h 331M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:31:41.868 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (117m) 3m ago 2h 385M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:31:41.868 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 3m ago 2h 120M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:31:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:31:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:31:41] "GET /metrics HTTP/1.1" 200 225177 "" "Prometheus/2.33.4" 2022-09-29T07:31:42.221 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:31:42.222 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:31:42.222 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:31:42.222 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:31:42.222 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:31:42.223 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:31:42.223 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:31:42.223 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:31:42.223 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:31:42.224 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:31:42.224 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:31:42.224 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:31:42.224 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:31:42.224 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:31:42.225 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:31:42.225 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:31:42.225 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:31:42.225 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:31:42.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:31:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:31:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:31:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:31:42 smithi153 ceph-mon[70044]: from='client.38445 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:31:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:31:42 smithi153 ceph-mon[70044]: pgmap v3705: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:31:42 smithi153 ceph-mon[70044]: from='client.38451 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:31:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:31:42 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4270551547' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:31:42.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:31:42 smithi083 ceph-mon[79930]: from='client.38445 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:31:42.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:31:42 smithi083 ceph-mon[79930]: pgmap v3705: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:31:42 smithi083 ceph-mon[79930]: from='client.38451 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:31:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:31:42 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4270551547' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:31:42.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:31:42 smithi083 ceph-mon[77242]: from='client.38445 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:31:42.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:31:42 smithi083 ceph-mon[77242]: pgmap v3705: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:31:42 smithi083 ceph-mon[77242]: from='client.38451 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:31:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:31:42 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4270551547' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:31:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:31:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:31:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:31:44.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:31:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:31:43.977Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:31:44.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:31:44 smithi083 ceph-mon[79930]: pgmap v3706: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:44.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:31:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:31:44.677Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:31:44.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:31:44 smithi083 ceph-mon[77242]: pgmap v3706: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:45.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:31:44 smithi153 ceph-mon[70044]: pgmap v3706: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:47.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:31:46 smithi083 ceph-mon[79930]: pgmap v3707: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:47.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:31:46 smithi083 ceph-mon[77242]: pgmap v3707: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:47.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:31:46 smithi153 ceph-mon[70044]: pgmap v3707: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:49.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:31:48 smithi083 ceph-mon[79930]: pgmap v3708: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:49.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:31:48 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:31:49.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:31:48 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:31:49.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:31:48 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:31:49.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:31:48 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:31:49.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:31:48 smithi083 ceph-mon[77242]: pgmap v3708: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:49.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:31:48 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:31:49.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:31:48 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:31:49.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:31:48 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:31:49.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:31:48 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:31:49.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:31:48 smithi153 ceph-mon[70044]: pgmap v3708: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:49.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:31:48 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:31:49.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:31:48 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:31:49.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:31:48 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:31:49.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:31:48 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:31:50.906 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:31:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:31:50.580Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:31:51.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:31:50 smithi083 ceph-mon[77242]: pgmap v3709: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:51.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:31:50 smithi083 ceph-mon[79930]: pgmap v3709: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:51.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:31:50 smithi153 ceph-mon[70044]: pgmap v3709: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:31:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:31:51] "GET /metrics HTTP/1.1" 200 225177 "" "Prometheus/2.33.4" 2022-09-29T07:31:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:31:52 smithi153 ceph-mon[70044]: pgmap v3710: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:52.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:31:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:31:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:31:52.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:31:52 smithi083 ceph-mon[77242]: pgmap v3710: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:52.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:31:52 smithi083 ceph-mon[79930]: pgmap v3710: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:31:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:31:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:31:54.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:31:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:31:53.978Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:31:54.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:31:54 smithi083 ceph-mon[79930]: pgmap v3711: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:54.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:31:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:31:54.678Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:31:54.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:31:54 smithi083 ceph-mon[77242]: pgmap v3711: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:55.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:31:54 smithi153 ceph-mon[70044]: pgmap v3711: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:57.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:31:56 smithi083 ceph-mon[77242]: pgmap v3712: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:57.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:31:56 smithi083 ceph-mon[79930]: pgmap v3712: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:57.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:31:56 smithi153 ceph-mon[70044]: pgmap v3712: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:59.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:31:58 smithi083 ceph-mon[77242]: pgmap v3713: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:59.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:31:58 smithi083 ceph-mon[79930]: pgmap v3713: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:31:59.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:31:58 smithi153 ceph-mon[70044]: pgmap v3713: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:00.908 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:32:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:32:00.581Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:32:01.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:32:00 smithi083 ceph-mon[77242]: pgmap v3714: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:01.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:32:00 smithi083 ceph-mon[79930]: pgmap v3714: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:01.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:32:00 smithi153 ceph-mon[70044]: pgmap v3714: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:32:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:32:01] "GET /metrics HTTP/1.1" 200 225178 "" "Prometheus/2.33.4" 2022-09-29T07:32:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:32:02 smithi153 ceph-mon[70044]: pgmap v3715: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:02.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:32:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:32:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:32:02.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:32:02 smithi083 ceph-mon[79930]: pgmap v3715: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:02.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:32:02 smithi083 ceph-mon[77242]: pgmap v3715: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:32:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:32:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:32:04.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:32:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:32:03.978Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:32:04.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:32:04 smithi083 ceph-mon[79930]: pgmap v3716: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:04.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:32:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:32:04.678Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:32:04.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:32:04 smithi083 ceph-mon[77242]: pgmap v3716: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:05.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:32:04 smithi153 ceph-mon[70044]: pgmap v3716: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:07.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:32:06 smithi083 ceph-mon[79930]: pgmap v3717: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:07.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:32:06 smithi083 ceph-mon[77242]: pgmap v3717: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:07.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:32:06 smithi153 ceph-mon[70044]: pgmap v3717: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:09.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:32:08 smithi083 ceph-mon[79930]: pgmap v3718: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:09.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:32:08 smithi083 ceph-mon[77242]: pgmap v3718: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:09.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:32:08 smithi153 ceph-mon[70044]: pgmap v3718: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:10.911 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:32:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:32:10.581Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:32:11.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:32:10 smithi083 ceph-mon[79930]: pgmap v3719: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:11.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:32:10 smithi083 ceph-mon[77242]: pgmap v3719: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:11.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:32:10 smithi153 ceph-mon[70044]: pgmap v3719: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:32:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:32:11] "GET /metrics HTTP/1.1" 200 225172 "" "Prometheus/2.33.4" 2022-09-29T07:32:12.547 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:32:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:32:12 smithi153 ceph-mon[70044]: pgmap v3720: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:12.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:32:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:32:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:32:12.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:32:12 smithi083 ceph-mon[79930]: pgmap v3720: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:12.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:32:12 smithi083 ceph-mon[77242]: pgmap v3720: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:12.835 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:32:12.835 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 6m ago 2h 26.7M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:32:12.835 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 4m ago 2h 61.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:32:12.836 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 6m ago 2h 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:32:12.836 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 4m ago 2h 27.8M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:32:12.836 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 4m ago 2h 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:32:12.836 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 6m ago 2h 617M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:32:12.837 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 6m ago 2h 494M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:32:12.837 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 4m ago 2h 221M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:32:12.837 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 6m ago 2h 291M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:32:12.837 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 6m ago 2h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:32:12.838 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 4m ago 2h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:32:12.838 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 6m ago 2h 361M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:32:12.838 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 6m ago 2h 352M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:32:12.839 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 6m ago 2h 280M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:32:12.839 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (119m) 6m ago 2h 340M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:32:12.839 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (119m) 4m ago 2h 305M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:32:12.839 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (118m) 4m ago 2h 372M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:32:12.840 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (118m) 4m ago 2h 331M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:32:12.840 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (117m) 4m ago 2h 385M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:32:12.840 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 4m ago 2h 120M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:32:13.188 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:32:13.189 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:32:13.189 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:32:13.189 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:32:13.189 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:32:13.190 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:32:13.190 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:32:13.190 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:32:13.190 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:32:13.191 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:32:13.191 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:32:13.191 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:32:13.191 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:32:13.191 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:32:13.192 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:32:13.192 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:32:13.192 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:32:13.192 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:32:13.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:32:13 smithi153 ceph-mon[70044]: from='client.48397 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:32:13.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:32:13 smithi153 ceph-mon[70044]: from='client.38469 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:32:13.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:32:13 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/647068317' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:32:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:32:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:32:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:32:13.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:32:13 smithi083 ceph-mon[77242]: from='client.48397 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:32:13.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:32:13 smithi083 ceph-mon[77242]: from='client.38469 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:32:13.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:32:13 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/647068317' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:32:13.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:32:13 smithi083 ceph-mon[79930]: from='client.48397 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:32:13.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:32:13 smithi083 ceph-mon[79930]: from='client.38469 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:32:13.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:32:13 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/647068317' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:32:14.274 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:32:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:32:13.978Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:32:14.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:32:14 smithi153 ceph-mon[70044]: pgmap v3721: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:14.679 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:32:14 smithi083 ceph-mon[79930]: pgmap v3721: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:14.680 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:32:14 smithi083 ceph-mon[77242]: pgmap v3721: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:14.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:32:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:32:14.678Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:32:17.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:32:16 smithi083 ceph-mon[79930]: pgmap v3722: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:17.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:32:16 smithi083 ceph-mon[77242]: pgmap v3722: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:17.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:32:16 smithi153 ceph-mon[70044]: pgmap v3722: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:19.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:32:18 smithi083 ceph-mon[79930]: pgmap v3723: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:19.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:32:18 smithi083 ceph-mon[77242]: pgmap v3723: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:19.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:32:18 smithi153 ceph-mon[70044]: pgmap v3723: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:20.911 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:32:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:32:20.582Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:32:21.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:32:20 smithi083 ceph-mon[79930]: pgmap v3724: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:21.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:32:20 smithi083 ceph-mon[77242]: pgmap v3724: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:21.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:32:20 smithi153 ceph-mon[70044]: pgmap v3724: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:32:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:32:21] "GET /metrics HTTP/1.1" 200 225172 "" "Prometheus/2.33.4" 2022-09-29T07:32:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:32:22 smithi153 ceph-mon[70044]: pgmap v3725: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:22.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:32:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:32:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:32:22.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:32:22 smithi083 ceph-mon[79930]: pgmap v3725: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:22.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:32:22 smithi083 ceph-mon[77242]: pgmap v3725: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:32:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:32:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:32:24.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:32:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:32:23.980Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:32:24.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:32:24 smithi083 ceph-mon[79930]: pgmap v3726: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:24.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:32:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:32:24.679Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:32:24.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:32:24 smithi083 ceph-mon[77242]: pgmap v3726: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:25.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:32:24 smithi153 ceph-mon[70044]: pgmap v3726: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:27.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:32:26 smithi083 ceph-mon[79930]: pgmap v3727: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:27.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:32:26 smithi083 ceph-mon[77242]: pgmap v3727: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:27.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:32:26 smithi153 ceph-mon[70044]: pgmap v3727: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:29.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:32:28 smithi083 ceph-mon[79930]: pgmap v3728: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:29.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:32:28 smithi083 ceph-mon[77242]: pgmap v3728: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:29.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:32:28 smithi153 ceph-mon[70044]: pgmap v3728: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:30.917 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:32:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:32:30.582Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:32:31.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:32:30 smithi083 ceph-mon[77242]: pgmap v3729: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:31.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:32:30 smithi083 ceph-mon[79930]: pgmap v3729: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:31.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:32:30 smithi153 ceph-mon[70044]: pgmap v3729: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:32:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:32:31] "GET /metrics HTTP/1.1" 200 225178 "" "Prometheus/2.33.4" 2022-09-29T07:32:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:32:32 smithi153 ceph-mon[70044]: pgmap v3730: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:32.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:32:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:32:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:32:32.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:32:32 smithi083 ceph-mon[77242]: pgmap v3730: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:32.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:32:32 smithi083 ceph-mon[79930]: pgmap v3730: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:32:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:32:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:32:34.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:32:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:32:33.980Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:32:34.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:32:34 smithi083 ceph-mon[79930]: pgmap v3731: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:34.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:32:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:32:34.680Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:32:34.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:32:34 smithi083 ceph-mon[77242]: pgmap v3731: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:35.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:32:34 smithi153 ceph-mon[70044]: pgmap v3731: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:37.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:32:36 smithi083 ceph-mon[79930]: pgmap v3732: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:37.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:32:36 smithi083 ceph-mon[77242]: pgmap v3732: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:37.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:32:36 smithi153 ceph-mon[70044]: pgmap v3732: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:39.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:32:38 smithi083 ceph-mon[77242]: pgmap v3733: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:39.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:32:38 smithi083 ceph-mon[79930]: pgmap v3733: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:39.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:32:38 smithi153 ceph-mon[70044]: pgmap v3733: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:40.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:32:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:32:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:32:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:32:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:32:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:32:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:32:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:32:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:32:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:32:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:32:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:32:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:32:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:32:40.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:32:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:32:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:32:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:32:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:32:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:32:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:32:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:32:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:32:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:32:40.926 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:32:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:32:40.583Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:32:41.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:32:40 smithi083 ceph-mon[77242]: pgmap v3734: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:41.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:32:40 smithi083 ceph-mon[79930]: pgmap v3734: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:32:40 smithi153 ceph-mon[70044]: pgmap v3734: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:32:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:32:41] "GET /metrics HTTP/1.1" 200 225180 "" "Prometheus/2.33.4" 2022-09-29T07:32:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:32:42 smithi153 ceph-mon[70044]: pgmap v3735: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:42.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:32:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:32:42.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:32:42.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:32:42 smithi083 ceph-mon[79930]: pgmap v3735: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:42.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:32:42 smithi083 ceph-mon[77242]: pgmap v3735: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:32:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:32:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:32:43.523 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:32:43.821 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:32:43.821 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 7m ago 2h 26.7M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:32:43.821 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 4m ago 2h 61.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:32:43.821 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 7m ago 2h 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:32:43.822 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 4m ago 2h 27.8M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:32:43.822 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 4m ago 2h 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:32:43.822 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 7m ago 2h 617M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:32:43.822 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 7m ago 2h 494M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:32:43.823 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 4m ago 2h 221M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:32:43.823 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 7m ago 2h 291M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:32:43.823 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 7m ago 2h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:32:43.823 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 4m ago 2h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:32:43.824 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 7m ago 2h 361M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:32:43.824 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 7m ago 2h 352M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:32:43.824 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 7m ago 2h 280M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:32:43.824 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 7m ago 2h 340M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:32:43.825 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (119m) 4m ago 2h 305M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:32:43.825 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (119m) 4m ago 2h 372M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:32:43.825 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (118m) 4m ago 2h 331M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:32:43.825 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (118m) 4m ago 2h 385M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:32:43.826 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 4m ago 2h 120M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:32:44.180 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:32:44.181 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:32:44.181 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:32:44.181 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:32:44.181 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:32:44.182 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:32:44.182 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:32:44.182 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:32:44.182 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:32:44.182 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:32:44.183 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:32:44.183 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:32:44.183 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:32:44.183 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:32:44.183 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:32:44.184 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:32:44.184 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:32:44.184 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:32:44.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:32:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:32:43.981Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:32:44.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:32:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:32:44.680Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:32:44.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:32:44 smithi083 ceph-mon[77242]: from='client.38481 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:32:44.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:32:44 smithi083 ceph-mon[77242]: from='client.38487 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:32:44.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:32:44 smithi083 ceph-mon[77242]: pgmap v3736: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:44.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:32:44 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/717582195' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:32:44.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:32:44 smithi083 ceph-mon[79930]: from='client.38481 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:32:44.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:32:44 smithi083 ceph-mon[79930]: from='client.38487 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:32:44.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:32:44 smithi083 ceph-mon[79930]: pgmap v3736: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:44.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:32:44 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/717582195' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:32:45.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:32:44 smithi153 ceph-mon[70044]: from='client.38481 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:32:45.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:32:44 smithi153 ceph-mon[70044]: from='client.38487 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:32:45.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:32:44 smithi153 ceph-mon[70044]: pgmap v3736: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:45.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:32:44 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/717582195' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:32:47.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:32:46 smithi083 ceph-mon[77242]: pgmap v3737: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:47.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:32:46 smithi083 ceph-mon[79930]: pgmap v3737: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:47.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:32:46 smithi153 ceph-mon[70044]: pgmap v3737: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:49.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:32:48 smithi083 ceph-mon[79930]: pgmap v3738: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:49.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:32:48 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:32:49.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:32:48 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:32:49.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:32:48 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:32:49.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:32:48 smithi083 ceph-mon[77242]: pgmap v3738: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:49.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:32:48 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:32:49.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:32:48 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:32:49.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:32:48 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:32:49.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:32:48 smithi153 ceph-mon[70044]: pgmap v3738: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:49.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:32:48 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:32:49.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:32:48 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:32:49.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:32:48 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:32:50.585 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:32:50 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:32:50.585 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:32:50 smithi083 ceph-mon[79930]: pgmap v3739: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:50.586 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:32:50 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:32:50.586 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:32:50 smithi083 ceph-mon[77242]: pgmap v3739: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:50.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:32:50 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:32:50.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:32:50 smithi153 ceph-mon[70044]: pgmap v3739: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:50.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:32:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:32:50.584Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:32:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:32:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:32:51] "GET /metrics HTTP/1.1" 200 225180 "" "Prometheus/2.33.4" 2022-09-29T07:32:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:32:52 smithi153 ceph-mon[70044]: pgmap v3740: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:52.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:32:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:32:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:32:52.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:32:52 smithi083 ceph-mon[79930]: pgmap v3740: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:52.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:32:52 smithi083 ceph-mon[77242]: pgmap v3740: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:32:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:32:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:32:54.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:32:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:32:53.982Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:32:54.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:32:54 smithi083 ceph-mon[79930]: pgmap v3741: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:54.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:32:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:32:54.681Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:32:54.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:32:54 smithi083 ceph-mon[77242]: pgmap v3741: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:55.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:32:54 smithi153 ceph-mon[70044]: pgmap v3741: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:57.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:32:56 smithi083 ceph-mon[79930]: pgmap v3742: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:57.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:32:56 smithi083 ceph-mon[77242]: pgmap v3742: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:57.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:32:56 smithi153 ceph-mon[70044]: pgmap v3742: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:59.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:32:58 smithi083 ceph-mon[79930]: pgmap v3743: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:59.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:32:58 smithi083 ceph-mon[77242]: pgmap v3743: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:32:59.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:32:58 smithi153 ceph-mon[70044]: pgmap v3743: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:32:59 smithi153 ceph-mon[70044]: pgmap v3744: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:00.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:32:59 smithi083 ceph-mon[79930]: pgmap v3744: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:00.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:32:59 smithi083 ceph-mon[77242]: pgmap v3744: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:00.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:33:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:33:00.585Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:33:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:33:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:33:01] "GET /metrics HTTP/1.1" 200 225174 "" "Prometheus/2.33.4" 2022-09-29T07:33:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:33:02 smithi153 ceph-mon[70044]: pgmap v3745: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:02.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:33:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:33:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:33:02.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:33:02 smithi083 ceph-mon[79930]: pgmap v3745: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:02.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:33:02 smithi083 ceph-mon[77242]: pgmap v3745: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:33:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:33:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:33:04.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:33:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:33:03.982Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:33:04.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:33:04 smithi083 ceph-mon[79930]: pgmap v3746: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:04.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:33:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:33:04.682Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:33:04.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:33:04 smithi083 ceph-mon[77242]: pgmap v3746: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:05.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:33:04 smithi153 ceph-mon[70044]: pgmap v3746: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:07.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:33:06 smithi083 ceph-mon[77242]: pgmap v3747: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:07.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:33:06 smithi083 ceph-mon[79930]: pgmap v3747: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:07.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:33:06 smithi153 ceph-mon[70044]: pgmap v3747: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:08.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:33:07 smithi153 ceph-mon[70044]: pgmap v3748: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:08.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:33:07 smithi083 ceph-mon[79930]: pgmap v3748: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:08.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:33:07 smithi083 ceph-mon[77242]: pgmap v3748: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:10.923 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:33:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:33:10.586Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:33:11.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:33:10 smithi083 ceph-mon[79930]: pgmap v3749: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:11.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:33:10 smithi083 ceph-mon[77242]: pgmap v3749: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:11.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:33:10 smithi153 ceph-mon[70044]: pgmap v3749: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:33:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:33:11] "GET /metrics HTTP/1.1" 200 225177 "" "Prometheus/2.33.4" 2022-09-29T07:33:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:33:12 smithi153 ceph-mon[70044]: pgmap v3750: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:12.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:33:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:33:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:33:12.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:33:12 smithi083 ceph-mon[79930]: pgmap v3750: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:12.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:33:12 smithi083 ceph-mon[77242]: pgmap v3750: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:33:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:33:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:33:14.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:33:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:33:13.983Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:33:14.509 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:33:14.799 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:33:14.800 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 7m ago 2h 26.7M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:33:14.800 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 5m ago 2h 61.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:33:14.800 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 7m ago 2h 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:33:14.800 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 5m ago 2h 27.8M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:33:14.801 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 5m ago 2h 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:33:14.801 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 7m ago 2h 617M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:33:14.801 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 7m ago 2h 494M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:33:14.801 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 5m ago 2h 221M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:33:14.802 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 7m ago 2h 291M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:33:14.802 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 7m ago 2h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:33:14.802 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 5m ago 2h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:33:14.803 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 7m ago 2h 361M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:33:14.803 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 7m ago 2h 352M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:33:14.803 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 7m ago 2h 280M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:33:14.803 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 7m ago 2h 340M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:33:14.803 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 5m ago 2h 305M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:33:14.804 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (119m) 5m ago 2h 372M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:33:14.804 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (119m) 5m ago 2h 331M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:33:14.804 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (118m) 5m ago 2h 385M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:33:14.804 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 5m ago 2h 120M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:33:14.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:33:14 smithi083 ceph-mon[79930]: pgmap v3751: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:14.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:33:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:33:14.683Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:33:14.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:33:14 smithi083 ceph-mon[77242]: pgmap v3751: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:15.156 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:33:15.157 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:33:15.157 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:33:15.157 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:33:15.157 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:33:15.158 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:33:15.158 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:33:15.158 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:33:15.158 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:33:15.158 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:33:15.159 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:33:15.159 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:33:15.159 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:33:15.159 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:33:15.160 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:33:15.160 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:33:15.160 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:33:15.160 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:33:15.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:33:14 smithi153 ceph-mon[70044]: pgmap v3751: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:16.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:33:15 smithi083 ceph-mon[77242]: from='client.38499 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:33:16.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:33:15 smithi083 ceph-mon[77242]: from='client.38505 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:33:16.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:33:15 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2063588997' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:33:16.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:33:15 smithi083 ceph-mon[79930]: from='client.38499 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:33:16.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:33:15 smithi083 ceph-mon[79930]: from='client.38505 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:33:16.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:33:15 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2063588997' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:33:16.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:33:15 smithi153 ceph-mon[70044]: from='client.38499 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:33:16.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:33:15 smithi153 ceph-mon[70044]: from='client.38505 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:33:16.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:33:15 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2063588997' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:33:17.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:33:16 smithi083 ceph-mon[77242]: pgmap v3752: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:17.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:33:16 smithi083 ceph-mon[79930]: pgmap v3752: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:17.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:33:16 smithi153 ceph-mon[70044]: pgmap v3752: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:18.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:33:17 smithi083 ceph-mon[79930]: pgmap v3753: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:18.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:33:17 smithi083 ceph-mon[77242]: pgmap v3753: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:18.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:33:17 smithi153 ceph-mon[70044]: pgmap v3753: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:20.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:33:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:33:20.587Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:33:21.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:33:20 smithi083 ceph-mon[79930]: pgmap v3754: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:21.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:33:20 smithi083 ceph-mon[77242]: pgmap v3754: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:21.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:33:20 smithi153 ceph-mon[70044]: pgmap v3754: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:33:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:33:21] "GET /metrics HTTP/1.1" 200 225177 "" "Prometheus/2.33.4" 2022-09-29T07:33:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:33:22 smithi153 ceph-mon[70044]: pgmap v3755: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:22.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:33:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:33:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:33:22.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:33:22 smithi083 ceph-mon[79930]: pgmap v3755: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:22.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:33:22 smithi083 ceph-mon[77242]: pgmap v3755: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:33:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:33:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:33:24.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:33:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:33:23.984Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:33:24.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:33:24 smithi083 ceph-mon[79930]: pgmap v3756: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:24.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:33:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:33:24.684Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:33:24.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:33:24 smithi083 ceph-mon[77242]: pgmap v3756: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:25.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:33:24 smithi153 ceph-mon[70044]: pgmap v3756: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:26.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:33:25 smithi083 ceph-mon[77242]: pgmap v3757: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:26.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:33:25 smithi083 ceph-mon[79930]: pgmap v3757: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:26.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:33:25 smithi153 ceph-mon[70044]: pgmap v3757: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:29.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:33:28 smithi083 ceph-mon[79930]: pgmap v3758: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:29.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:33:28 smithi083 ceph-mon[77242]: pgmap v3758: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:29.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:33:28 smithi153 ceph-mon[70044]: pgmap v3758: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:30.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:33:29 smithi083 ceph-mon[77242]: pgmap v3759: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:30.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:33:29 smithi083 ceph-mon[79930]: pgmap v3759: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:30.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:33:29 smithi153 ceph-mon[70044]: pgmap v3759: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:30.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:33:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:33:30.587Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:33:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:33:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:33:31] "GET /metrics HTTP/1.1" 200 225173 "" "Prometheus/2.33.4" 2022-09-29T07:33:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:33:32 smithi153 ceph-mon[70044]: pgmap v3760: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:32.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:33:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:33:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:33:32.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:33:32 smithi083 ceph-mon[77242]: pgmap v3760: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:32.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:33:32 smithi083 ceph-mon[79930]: pgmap v3760: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:33:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:33:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:33:34.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:33:33 smithi153 ceph-mon[70044]: pgmap v3761: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:34.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:33:33 smithi083 ceph-mon[79930]: pgmap v3761: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:34.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:33:33 smithi083 ceph-mon[77242]: pgmap v3761: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:34.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:33:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:33:33.984Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:33:34.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:33:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:33:34.684Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:33:36.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:33:35 smithi153 ceph-mon[70044]: pgmap v3762: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:36.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:33:35 smithi083 ceph-mon[79930]: pgmap v3762: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:36.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:33:35 smithi083 ceph-mon[77242]: pgmap v3762: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:38.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:33:37 smithi153 ceph-mon[70044]: pgmap v3763: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:38.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:33:37 smithi083 ceph-mon[79930]: pgmap v3763: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:38.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:33:37 smithi083 ceph-mon[77242]: pgmap v3763: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:39.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:33:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:33:39.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:33:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:33:39.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:33:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:33:39.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:33:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:33:39.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:33:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:33:39.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:33:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:33:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:33:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:33:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:33:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:33:39.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:33:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:33:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:33:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:33:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:33:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:33:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:33:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:33:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:33:40 smithi153 ceph-mon[70044]: pgmap v3764: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:40.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:33:40 smithi083 ceph-mon[79930]: pgmap v3764: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:40.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:33:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:33:40.588Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:33:40.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:33:40 smithi083 ceph-mon[77242]: pgmap v3764: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:33:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:33:41] "GET /metrics HTTP/1.1" 200 225178 "" "Prometheus/2.33.4" 2022-09-29T07:33:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:33:42 smithi153 ceph-mon[70044]: pgmap v3765: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:42.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:33:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:33:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:33:42.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:33:42 smithi083 ceph-mon[79930]: pgmap v3765: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:42.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:33:42 smithi083 ceph-mon[77242]: pgmap v3765: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:33:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:33:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:33:44.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:33:43 smithi153 ceph-mon[70044]: pgmap v3766: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:44.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:33:43 smithi083 ceph-mon[79930]: pgmap v3766: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:44.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:33:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:33:43.985Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:33:44.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:33:43 smithi083 ceph-mon[77242]: pgmap v3766: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:44.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:33:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:33:44.685Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:33:45.483 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:33:45.772 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:33:45.772 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 8m ago 2h 26.7M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:33:45.773 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 6m ago 2h 61.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:33:45.773 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 8m ago 2h 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:33:45.773 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 6m ago 2h 27.8M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:33:45.773 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 6m ago 2h 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:33:45.774 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 8m ago 2h 617M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:33:45.774 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 8m ago 2h 494M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:33:45.774 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 6m ago 2h 221M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:33:45.774 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 8m ago 2h 291M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:33:45.775 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 8m ago 2h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:33:45.775 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 6m ago 2h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:33:45.775 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 8m ago 2h 361M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:33:45.775 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 8m ago 2h 352M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:33:45.776 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 8m ago 2h 280M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:33:45.776 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 8m ago 2h 340M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:33:45.776 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 6m ago 2h 305M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:33:45.776 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 6m ago 2h 372M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:33:45.777 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (119m) 6m ago 2h 331M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:33:45.777 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (119m) 6m ago 2h 385M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:33:45.777 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 6m ago 2h 120M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:33:46.132 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:33:46.132 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:33:46.133 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:33:46.133 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:33:46.133 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:33:46.133 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:33:46.134 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:33:46.134 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:33:46.134 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:33:46.134 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:33:46.135 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:33:46.135 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:33:46.135 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:33:46.135 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:33:46.136 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:33:46.136 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:33:46.136 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:33:46.136 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:33:46.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:33:45 smithi153 ceph-mon[70044]: from='client.38517 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:33:46.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:33:45 smithi153 ceph-mon[70044]: from='client.38523 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:33:46.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:33:45 smithi153 ceph-mon[70044]: pgmap v3767: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:46.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:33:45 smithi083 ceph-mon[79930]: from='client.38517 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:33:46.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:33:45 smithi083 ceph-mon[79930]: from='client.38523 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:33:46.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:33:45 smithi083 ceph-mon[79930]: pgmap v3767: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:46.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:33:45 smithi083 ceph-mon[77242]: from='client.38517 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:33:46.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:33:45 smithi083 ceph-mon[77242]: from='client.38523 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:33:46.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:33:45 smithi083 ceph-mon[77242]: pgmap v3767: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:47.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:33:46 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3679480327' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:33:47.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:33:46 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3679480327' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:33:47.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:33:46 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3679480327' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:33:48.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:33:47 smithi153 ceph-mon[70044]: pgmap v3768: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:48.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:33:47 smithi083 ceph-mon[79930]: pgmap v3768: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:48.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:33:47 smithi083 ceph-mon[77242]: pgmap v3768: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:49.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:33:49 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:33:49.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:33:49 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:33:49.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:33:49 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:33:49.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:33:49 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:33:49.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:33:49 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:33:49.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:33:49 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:33:49.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:33:49 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:33:49.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:33:49 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:33:49.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:33:49 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:33:50.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:33:50 smithi083 ceph-mon[79930]: pgmap v3769: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:50.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:33:50 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:33:50.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:33:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:33:50.588Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:33:50.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:33:50 smithi083 ceph-mon[77242]: pgmap v3769: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:50.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:33:50 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:33:50.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:33:50 smithi153 ceph-mon[70044]: pgmap v3769: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:50.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:33:50 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:33:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:33:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:33:51] "GET /metrics HTTP/1.1" 200 225178 "" "Prometheus/2.33.4" 2022-09-29T07:33:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:33:52 smithi153 ceph-mon[70044]: pgmap v3770: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:52.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:33:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:33:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:33:52.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:33:52 smithi083 ceph-mon[77242]: pgmap v3770: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:52.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:33:52 smithi083 ceph-mon[79930]: pgmap v3770: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:33:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:33:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:33:54.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:33:53 smithi153 ceph-mon[70044]: pgmap v3771: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:54.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:33:53 smithi083 ceph-mon[79930]: pgmap v3771: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:54.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:33:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:33:53.986Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:33:54.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:33:53 smithi083 ceph-mon[77242]: pgmap v3771: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:54.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:33:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:33:54.686Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:33:56.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:33:55 smithi153 ceph-mon[70044]: pgmap v3772: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:56.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:33:55 smithi083 ceph-mon[79930]: pgmap v3772: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:56.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:33:55 smithi083 ceph-mon[77242]: pgmap v3772: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:58.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:33:57 smithi153 ceph-mon[70044]: pgmap v3773: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:58.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:33:57 smithi083 ceph-mon[79930]: pgmap v3773: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:33:58.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:33:57 smithi083 ceph-mon[77242]: pgmap v3773: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:33:59 smithi153 ceph-mon[70044]: pgmap v3774: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:00.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:33:59 smithi083 ceph-mon[79930]: pgmap v3774: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:00.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:33:59 smithi083 ceph-mon[77242]: pgmap v3774: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:00.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:34:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:34:00.590Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:34:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:34:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:34:01] "GET /metrics HTTP/1.1" 200 225176 "" "Prometheus/2.33.4" 2022-09-29T07:34:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:34:02 smithi153 ceph-mon[70044]: pgmap v3775: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:02.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:34:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:34:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:34:02.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:34:02 smithi083 ceph-mon[77242]: pgmap v3775: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:02.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:34:02 smithi083 ceph-mon[79930]: pgmap v3775: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:34:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:34:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:34:04.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:34:03 smithi153 ceph-mon[70044]: pgmap v3776: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:04.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:34:03 smithi083 ceph-mon[79930]: pgmap v3776: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:04.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:34:03 smithi083 ceph-mon[77242]: pgmap v3776: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:04.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:34:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:34:03.987Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:34:04.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:34:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:34:04.687Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:34:06.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:34:05 smithi153 ceph-mon[70044]: pgmap v3777: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:06.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:34:05 smithi083 ceph-mon[79930]: pgmap v3777: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:06.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:34:05 smithi083 ceph-mon[77242]: pgmap v3777: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:08.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:34:07 smithi153 ceph-mon[70044]: pgmap v3778: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:08.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:34:07 smithi083 ceph-mon[79930]: pgmap v3778: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:08.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:34:07 smithi083 ceph-mon[77242]: pgmap v3778: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:10.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:34:09 smithi153 ceph-mon[70044]: pgmap v3779: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:10.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:34:09 smithi083 ceph-mon[77242]: pgmap v3779: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:10.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:34:09 smithi083 ceph-mon[79930]: pgmap v3779: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:10.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:34:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:34:10.590Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:34:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:34:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:34:11] "GET /metrics HTTP/1.1" 200 225172 "" "Prometheus/2.33.4" 2022-09-29T07:34:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:34:12 smithi153 ceph-mon[70044]: pgmap v3780: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:12.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:34:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:34:12.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:34:12.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:34:12 smithi083 ceph-mon[77242]: pgmap v3780: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:12.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:34:12 smithi083 ceph-mon[79930]: pgmap v3780: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:34:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:34:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:34:14.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:34:13 smithi153 ceph-mon[70044]: pgmap v3781: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:14.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:34:13 smithi083 ceph-mon[79930]: pgmap v3781: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:14.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:34:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:34:13.988Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:34:14.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:34:13 smithi083 ceph-mon[77242]: pgmap v3781: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:14.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:34:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:34:14.688Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:34:16.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:34:15 smithi153 ceph-mon[70044]: pgmap v3782: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:16.460 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:34:16.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:34:15 smithi083 ceph-mon[79930]: pgmap v3782: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:16.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:34:15 smithi083 ceph-mon[77242]: pgmap v3782: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:16.749 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:34:16.749 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 8m ago 2h 26.7M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:34:16.749 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 6m ago 2h 61.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:34:16.750 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 8m ago 2h 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:34:16.750 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 6m ago 2h 27.8M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:34:16.750 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 6m ago 2h 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:34:16.750 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 8m ago 2h 617M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:34:16.750 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 8m ago 2h 494M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:34:16.751 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 6m ago 2h 221M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:34:16.751 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 8m ago 2h 291M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:34:16.751 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 8m ago 2h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:34:16.751 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 6m ago 2h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:34:16.751 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 8m ago 2h 361M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:34:16.752 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 8m ago 2h 352M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:34:16.752 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 8m ago 2h 280M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:34:16.752 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 8m ago 2h 340M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:34:16.752 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 6m ago 2h 305M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:34:16.753 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 6m ago 2h 372M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:34:16.753 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 6m ago 2h 331M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:34:16.753 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (119m) 6m ago 2h 385M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:34:16.753 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 6m ago 2h 120M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:34:17.104 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:34:17.104 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:34:17.105 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:34:17.105 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:34:17.105 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:34:17.105 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:34:17.106 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:34:17.106 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:34:17.106 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:34:17.106 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:34:17.106 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:34:17.107 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:34:17.107 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:34:17.107 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:34:17.107 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:34:17.107 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:34:17.108 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:34:17.108 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:34:17.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:34:17 smithi153 ceph-mon[70044]: from='client.38535 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:34:17.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:34:17 smithi153 ceph-mon[70044]: from='client.38541 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:34:17.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:34:17 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2771144035' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:34:17.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:34:17 smithi083 ceph-mon[79930]: from='client.38535 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:34:17.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:34:17 smithi083 ceph-mon[79930]: from='client.38541 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:34:17.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:34:17 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2771144035' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:34:17.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:34:17 smithi083 ceph-mon[77242]: from='client.38535 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:34:17.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:34:17 smithi083 ceph-mon[77242]: from='client.38541 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:34:17.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:34:17 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2771144035' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:34:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:34:18 smithi153 ceph-mon[70044]: pgmap v3783: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:18.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:34:18 smithi083 ceph-mon[77242]: pgmap v3783: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:18.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:34:18 smithi083 ceph-mon[79930]: pgmap v3783: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:34:19 smithi153 ceph-mon[70044]: pgmap v3784: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:20.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:34:19 smithi083 ceph-mon[79930]: pgmap v3784: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:20.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:34:19 smithi083 ceph-mon[77242]: pgmap v3784: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:20.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:34:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:34:20.591Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:34:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:34:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:34:21] "GET /metrics HTTP/1.1" 200 225172 "" "Prometheus/2.33.4" 2022-09-29T07:34:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:34:22 smithi153 ceph-mon[70044]: pgmap v3785: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:22.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:34:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:34:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:34:22.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:34:22 smithi083 ceph-mon[79930]: pgmap v3785: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:22.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:34:22 smithi083 ceph-mon[77242]: pgmap v3785: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:34:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:34:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:34:24.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:34:23 smithi153 ceph-mon[70044]: pgmap v3786: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:24.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:34:23 smithi083 ceph-mon[79930]: pgmap v3786: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:24.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:34:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:34:23.989Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:34:24.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:34:23 smithi083 ceph-mon[77242]: pgmap v3786: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:24.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:34:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:34:24.688Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:34:26.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:34:25 smithi153 ceph-mon[70044]: pgmap v3787: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:26.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:34:25 smithi083 ceph-mon[79930]: pgmap v3787: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:26.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:34:25 smithi083 ceph-mon[77242]: pgmap v3787: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:28.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:34:27 smithi153 ceph-mon[70044]: pgmap v3788: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:28.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:34:27 smithi083 ceph-mon[79930]: pgmap v3788: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:28.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:34:27 smithi083 ceph-mon[77242]: pgmap v3788: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:30.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:34:29 smithi153 ceph-mon[70044]: pgmap v3789: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:30.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:34:29 smithi083 ceph-mon[77242]: pgmap v3789: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:30.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:34:29 smithi083 ceph-mon[79930]: pgmap v3789: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:30.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:34:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:34:30.592Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:34:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:34:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:34:31] "GET /metrics HTTP/1.1" 200 225177 "" "Prometheus/2.33.4" 2022-09-29T07:34:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:34:32 smithi153 ceph-mon[70044]: pgmap v3790: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:32.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:34:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:34:32.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:34:32.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:34:32 smithi083 ceph-mon[77242]: pgmap v3790: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:32.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:34:32 smithi083 ceph-mon[79930]: pgmap v3790: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:34:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:34:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:34:34.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:34:33 smithi153 ceph-mon[70044]: pgmap v3791: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:34.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:34:33 smithi083 ceph-mon[79930]: pgmap v3791: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:34.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:34:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:34:33.989Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:34:34.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:34:33 smithi083 ceph-mon[77242]: pgmap v3791: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:34.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:34:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:34:34.689Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:34:36.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:34:35 smithi153 ceph-mon[70044]: pgmap v3792: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:36.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:34:35 smithi083 ceph-mon[79930]: pgmap v3792: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:36.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:34:35 smithi083 ceph-mon[77242]: pgmap v3792: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:38.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:34:37 smithi153 ceph-mon[70044]: pgmap v3793: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:38.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:34:37 smithi083 ceph-mon[79930]: pgmap v3793: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:38.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:34:37 smithi083 ceph-mon[77242]: pgmap v3793: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:39.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:34:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:34:39.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:34:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:34:39.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:34:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:34:39.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:34:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:34:39.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:34:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:34:39.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:34:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:34:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:34:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:34:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:34:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:34:39.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:34:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:34:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:34:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:34:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:34:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:34:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:34:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:34:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:34:40 smithi153 ceph-mon[70044]: pgmap v3794: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:40.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:34:40 smithi083 ceph-mon[79930]: pgmap v3794: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:40.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:34:40 smithi083 ceph-mon[77242]: pgmap v3794: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:40.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:34:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:34:40.593Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:34:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:34:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:34:41] "GET /metrics HTTP/1.1" 200 225179 "" "Prometheus/2.33.4" 2022-09-29T07:34:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:34:42 smithi153 ceph-mon[70044]: pgmap v3795: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:42.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:34:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:34:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:34:42.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:34:42 smithi083 ceph-mon[79930]: pgmap v3795: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:42.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:34:42 smithi083 ceph-mon[77242]: pgmap v3795: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:34:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:34:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:34:44.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:34:43 smithi153 ceph-mon[70044]: pgmap v3796: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:44.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:34:43 smithi083 ceph-mon[79930]: pgmap v3796: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:44.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:34:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:34:43.989Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:34:44.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:34:43 smithi083 ceph-mon[77242]: pgmap v3796: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:44.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:34:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:34:44.689Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:34:46.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:34:45 smithi153 ceph-mon[70044]: pgmap v3797: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:46.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:34:45 smithi083 ceph-mon[79930]: pgmap v3797: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:46.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:34:45 smithi083 ceph-mon[77242]: pgmap v3797: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:47.430 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:34:47.718 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:34:47.719 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 9m ago 2h 26.7M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:34:47.719 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 7m ago 2h 61.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:34:47.719 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 9m ago 2h 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:34:47.719 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 7m ago 2h 27.8M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:34:47.720 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 7m ago 2h 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:34:47.720 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 9m ago 2h 617M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:34:47.720 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 9m ago 2h 494M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:34:47.720 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 7m ago 2h 221M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:34:47.720 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 9m ago 2h 291M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:34:47.721 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 9m ago 2h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:34:47.721 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 7m ago 2h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:34:47.721 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 9m ago 2h 361M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:34:47.721 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 9m ago 2h 352M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:34:47.721 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 9m ago 2h 280M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:34:47.722 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 9m ago 2h 340M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:34:47.722 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 7m ago 2h 305M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:34:47.722 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 7m ago 2h 372M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:34:47.722 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 7m ago 2h 331M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:34:47.723 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 7m ago 2h 385M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:34:47.723 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 7m ago 2h 120M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:34:48.077 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:34:48.077 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:34:48.077 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:34:48.077 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:34:48.078 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:34:48.078 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:34:48.078 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:34:48.078 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:34:48.079 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:34:48.079 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:34:48.079 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:34:48.079 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:34:48.080 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:34:48.080 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:34:48.080 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:34:48.080 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:34:48.080 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:34:48.081 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:34:48.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:34:47 smithi153 ceph-mon[70044]: from='client.38553 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:34:48.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:34:47 smithi153 ceph-mon[70044]: from='client.38559 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:34:48.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:34:47 smithi153 ceph-mon[70044]: pgmap v3798: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:48.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:34:47 smithi083 ceph-mon[79930]: from='client.38553 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:34:48.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:34:47 smithi083 ceph-mon[79930]: from='client.38559 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:34:48.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:34:47 smithi083 ceph-mon[79930]: pgmap v3798: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:48.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:34:47 smithi083 ceph-mon[77242]: from='client.38553 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:34:48.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:34:47 smithi083 ceph-mon[77242]: from='client.38559 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:34:48.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:34:47 smithi083 ceph-mon[77242]: pgmap v3798: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:49.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:34:48 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2889699473' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:34:49.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:34:48 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2889699473' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:34:49.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:34:48 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2889699473' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:34:50.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:34:49 smithi153 ceph-mon[70044]: pgmap v3799: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:50.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:34:49 smithi083 ceph-mon[79930]: pgmap v3799: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:50.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:34:49 smithi083 ceph-mon[77242]: pgmap v3799: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:50.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:34:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:34:50.602Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:34:51.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:34:51 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:34:51.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:34:51 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:34:51.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:34:51 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:34:51.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:34:51 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:34:51.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:34:50 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:34:51.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:34:50 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:34:51.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:34:50 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:34:51.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:34:50 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:34:51.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:34:50 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:34:51.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:34:50 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:34:51.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:34:50 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:34:51.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:34:50 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:34:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:34:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:34:51] "GET /metrics HTTP/1.1" 200 225179 "" "Prometheus/2.33.4" 2022-09-29T07:34:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:34:52 smithi153 ceph-mon[70044]: pgmap v3800: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:52.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:34:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:34:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:34:52.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:34:52 smithi083 ceph-mon[79930]: pgmap v3800: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:52.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:34:52 smithi083 ceph-mon[77242]: pgmap v3800: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:34:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:34:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:34:54.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:34:53 smithi153 ceph-mon[70044]: pgmap v3801: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:54.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:34:53 smithi083 ceph-mon[79930]: pgmap v3801: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:54.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:34:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:34:53.990Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:34:54.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:34:53 smithi083 ceph-mon[77242]: pgmap v3801: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:54.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:34:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:34:54.690Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:34:56.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:34:55 smithi153 ceph-mon[70044]: pgmap v3802: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:56.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:34:55 smithi083 ceph-mon[79930]: pgmap v3802: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:56.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:34:55 smithi083 ceph-mon[77242]: pgmap v3802: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:58.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:34:57 smithi153 ceph-mon[70044]: pgmap v3803: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:58.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:34:57 smithi083 ceph-mon[79930]: pgmap v3803: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:34:58.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:34:57 smithi083 ceph-mon[77242]: pgmap v3803: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:34:59 smithi153 ceph-mon[70044]: pgmap v3804: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:00.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:34:59 smithi083 ceph-mon[79930]: pgmap v3804: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:00.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:34:59 smithi083 ceph-mon[77242]: pgmap v3804: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:00.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:35:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:35:00.595Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:35:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:35:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:35:01] "GET /metrics HTTP/1.1" 200 225178 "" "Prometheus/2.33.4" 2022-09-29T07:35:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:35:02 smithi153 ceph-mon[70044]: pgmap v3805: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:02.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:35:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:35:02.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:35:02.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:35:02 smithi083 ceph-mon[77242]: pgmap v3805: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:02.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:35:02 smithi083 ceph-mon[79930]: pgmap v3805: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:35:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:35:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:35:04.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:35:03 smithi153 ceph-mon[70044]: pgmap v3806: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:04.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:35:03 smithi083 ceph-mon[79930]: pgmap v3806: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:04.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:35:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:35:03.990Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:35:04.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:35:03 smithi083 ceph-mon[77242]: pgmap v3806: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:04.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:35:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:35:04.691Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:35:06.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:35:05 smithi153 ceph-mon[70044]: pgmap v3807: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:06.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:35:05 smithi083 ceph-mon[77242]: pgmap v3807: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:06.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:35:05 smithi083 ceph-mon[79930]: pgmap v3807: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:08.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:35:07 smithi153 ceph-mon[70044]: pgmap v3808: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:08.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:35:07 smithi083 ceph-mon[79930]: pgmap v3808: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:08.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:35:07 smithi083 ceph-mon[77242]: pgmap v3808: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:10.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:35:09 smithi153 ceph-mon[70044]: pgmap v3809: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:10.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:35:09 smithi083 ceph-mon[79930]: pgmap v3809: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:10.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:35:09 smithi083 ceph-mon[77242]: pgmap v3809: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:10.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:35:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:35:10.595Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:35:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:35:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:35:11] "GET /metrics HTTP/1.1" 200 225177 "" "Prometheus/2.33.4" 2022-09-29T07:35:12.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:35:12 smithi083 ceph-mon[79930]: pgmap v3810: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:12.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:35:12 smithi083 ceph-mon[77242]: pgmap v3810: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:12.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:35:12 smithi153 ceph-mon[70044]: pgmap v3810: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:35:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:35:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:35:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:35:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:35:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:35:14.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:35:13 smithi153 ceph-mon[70044]: pgmap v3811: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:14.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:35:13 smithi083 ceph-mon[79930]: pgmap v3811: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:14.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:35:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:35:13.991Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:35:14.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:35:13 smithi083 ceph-mon[77242]: pgmap v3811: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:14.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:35:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:35:14.691Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:35:16.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:35:15 smithi153 ceph-mon[70044]: pgmap v3812: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:16.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:35:15 smithi083 ceph-mon[77242]: pgmap v3812: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:16.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:35:15 smithi083 ceph-mon[79930]: pgmap v3812: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:18.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:35:17 smithi153 ceph-mon[70044]: pgmap v3813: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:18.401 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:35:18.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:35:17 smithi083 ceph-mon[79930]: pgmap v3813: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:18.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:35:17 smithi083 ceph-mon[77242]: pgmap v3813: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:18.690 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:35:18.691 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 9m ago 2h 26.7M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:35:18.691 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 7m ago 2h 61.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:35:18.691 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 9m ago 2h 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:35:18.691 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 7m ago 2h 27.8M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:35:18.692 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 7m ago 2h 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:35:18.692 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 9m ago 2h 617M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:35:18.692 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 9m ago 2h 494M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:35:18.692 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 7m ago 2h 221M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:35:18.693 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 9m ago 2h 291M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:35:18.693 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 9m ago 2h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:35:18.693 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 7m ago 2h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:35:18.693 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 9m ago 2h 361M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:35:18.693 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 9m ago 2h 352M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:35:18.694 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 9m ago 2h 280M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:35:18.694 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 9m ago 2h 340M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:35:18.694 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 7m ago 2h 305M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:35:18.694 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 7m ago 2h 372M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:35:18.695 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 7m ago 2h 331M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:35:18.695 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 7m ago 2h 385M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:35:18.695 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 7m ago 2h 120M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:35:19.051 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:35:19.052 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:35:19.052 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:35:19.053 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:35:19.053 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:35:19.054 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:35:19.054 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:35:19.055 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:35:19.055 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:35:19.056 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:35:19.056 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:35:19.056 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:35:19.057 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:35:19.057 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:35:19.058 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:35:19.058 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:35:19.058 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:35:19.059 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:35:19.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:35:18 smithi153 ceph-mon[70044]: from='client.48505 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:35:19.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:35:18 smithi153 ceph-mon[70044]: from='client.48511 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:35:19.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:35:18 smithi083 ceph-mon[77242]: from='client.48505 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:35:19.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:35:18 smithi083 ceph-mon[77242]: from='client.48511 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:35:19.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:35:18 smithi083 ceph-mon[79930]: from='client.48505 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:35:19.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:35:18 smithi083 ceph-mon[79930]: from='client.48511 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:35:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:35:20 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2185067193' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:35:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:35:20 smithi153 ceph-mon[70044]: pgmap v3814: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:20.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:35:20 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2185067193' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:35:20.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:35:20 smithi083 ceph-mon[77242]: pgmap v3814: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:20.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:35:20 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2185067193' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:35:20.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:35:20 smithi083 ceph-mon[79930]: pgmap v3814: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:20.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:35:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:35:20.595Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:35:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:35:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:35:21] "GET /metrics HTTP/1.1" 200 225177 "" "Prometheus/2.33.4" 2022-09-29T07:35:22.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:35:22 smithi083 ceph-mon[79930]: pgmap v3815: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:22.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:35:22 smithi083 ceph-mon[77242]: pgmap v3815: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:22.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:35:22 smithi153 ceph-mon[70044]: pgmap v3815: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:35:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:35:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:35:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:35:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:35:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:35:24.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:35:23 smithi153 ceph-mon[70044]: pgmap v3816: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:24.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:35:23 smithi083 ceph-mon[77242]: pgmap v3816: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:24.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:35:24 smithi083 ceph-mon[79930]: pgmap v3816: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:24.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:35:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:35:23.991Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:35:24.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:35:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:35:24.693Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:35:26.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:35:25 smithi153 ceph-mon[70044]: pgmap v3817: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:26.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:35:25 smithi083 ceph-mon[79930]: pgmap v3817: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:26.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:35:25 smithi083 ceph-mon[77242]: pgmap v3817: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:28.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:35:28 smithi153 ceph-mon[70044]: pgmap v3818: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:28.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:35:27 smithi083 ceph-mon[79930]: pgmap v3818: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:28.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:35:27 smithi083 ceph-mon[77242]: pgmap v3818: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:30.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:35:30 smithi153 ceph-mon[70044]: pgmap v3819: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:30.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:35:30 smithi083 ceph-mon[77242]: pgmap v3819: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:30.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:35:30 smithi083 ceph-mon[79930]: pgmap v3819: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:30.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:35:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:35:30.596Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:35:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:35:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:35:31] "GET /metrics HTTP/1.1" 200 225177 "" "Prometheus/2.33.4" 2022-09-29T07:35:32.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:35:32 smithi083 ceph-mon[79930]: pgmap v3820: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:32.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:35:32 smithi083 ceph-mon[77242]: pgmap v3820: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:32.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:35:32 smithi153 ceph-mon[70044]: pgmap v3820: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:35:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:35:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:35:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:35:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:35:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:35:34.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:35:34 smithi153 ceph-mon[70044]: pgmap v3821: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:34.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:35:34 smithi083 ceph-mon[79930]: pgmap v3821: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:34.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:35:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:35:33.992Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:35:34.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:35:34 smithi083 ceph-mon[77242]: pgmap v3821: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:34.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:35:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:35:34.692Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:35:36.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:35:36 smithi153 ceph-mon[70044]: pgmap v3822: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:36.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:35:35 smithi083 ceph-mon[79930]: pgmap v3822: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:36.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:35:35 smithi083 ceph-mon[77242]: pgmap v3822: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:38.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:35:37 smithi153 ceph-mon[70044]: pgmap v3823: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:38.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:35:37 smithi083 ceph-mon[79930]: pgmap v3823: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:38.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:35:37 smithi083 ceph-mon[77242]: pgmap v3823: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:39.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:35:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:35:39.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:35:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:35:39.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:35:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:35:39.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:35:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:35:39.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:35:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:35:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:35:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:35:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:35:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:35:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:35:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:35:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:35:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:35:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:35:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:35:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:35:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:35:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:35:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:35:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:35:40 smithi153 ceph-mon[70044]: pgmap v3824: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:40.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:35:40 smithi083 ceph-mon[79930]: pgmap v3824: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:40.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:35:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:35:40.597Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:35:40.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:35:40 smithi083 ceph-mon[77242]: pgmap v3824: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:35:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:35:41] "GET /metrics HTTP/1.1" 200 225182 "" "Prometheus/2.33.4" 2022-09-29T07:35:42.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:35:42 smithi083 ceph-mon[79930]: pgmap v3825: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:42.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:35:42 smithi083 ceph-mon[77242]: pgmap v3825: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:35:42 smithi153 ceph-mon[70044]: pgmap v3825: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:35:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:35:42.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:35:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:35:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:35:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:35:44.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:35:44 smithi153 ceph-mon[70044]: pgmap v3826: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:44.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:35:44 smithi083 ceph-mon[79930]: pgmap v3826: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:44.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:35:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:35:43.993Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:35:44.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:35:44 smithi083 ceph-mon[77242]: pgmap v3826: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:44.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:35:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:35:44.693Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:35:46.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:35:46 smithi153 ceph-mon[70044]: pgmap v3827: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:46.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:35:46 smithi083 ceph-mon[79930]: pgmap v3827: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:46.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:35:46 smithi083 ceph-mon[77242]: pgmap v3827: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:48.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:35:48 smithi153 ceph-mon[70044]: pgmap v3828: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:48.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:35:48 smithi083 ceph-mon[79930]: pgmap v3828: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:48.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:35:48 smithi083 ceph-mon[77242]: pgmap v3828: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:49.380 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:35:49.669 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:35:49.670 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 10m ago 2h 26.7M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:35:49.670 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 8m ago 2h 61.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:35:49.670 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 10m ago 2h 25.4M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:35:49.671 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 8m ago 2h 27.8M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:35:49.671 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 8m ago 2h 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:35:49.671 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 10m ago 2h 617M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:35:49.671 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 10m ago 2h 494M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:35:49.672 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 8m ago 2h 221M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:35:49.672 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 10m ago 2h 291M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:35:49.672 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 10m ago 2h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:35:49.672 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 8m ago 2h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:35:49.673 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 10m ago 2h 361M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:35:49.673 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 10m ago 2h 352M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:35:49.673 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 10m ago 2h 280M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:35:49.673 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 10m ago 2h 340M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:35:49.674 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 8m ago 2h 305M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:35:49.674 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 8m ago 2h 372M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:35:49.674 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 8m ago 2h 331M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:35:49.674 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 8m ago 2h 385M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:35:49.675 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 8m ago 2h 120M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:35:50.030 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:35:50.030 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:35:50.030 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:35:50.031 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:35:50.031 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:35:50.031 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:35:50.031 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:35:50.032 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:35:50.032 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:35:50.032 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:35:50.032 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:35:50.033 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:35:50.033 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:35:50.033 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:35:50.033 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:35:50.033 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:35:50.034 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:35:50.034 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:35:50.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:35:50 smithi153 ceph-mon[70044]: from='client.48523 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:35:50.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:35:50 smithi153 ceph-mon[70044]: from='client.38595 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:35:50.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:35:50 smithi153 ceph-mon[70044]: pgmap v3829: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:50.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:35:50 smithi083 ceph-mon[79930]: from='client.48523 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:35:50.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:35:50 smithi083 ceph-mon[79930]: from='client.38595 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:35:50.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:35:50 smithi083 ceph-mon[79930]: pgmap v3829: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:50.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:35:50 smithi083 ceph-mon[77242]: from='client.48523 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:35:50.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:35:50 smithi083 ceph-mon[77242]: from='client.38595 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:35:50.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:35:50 smithi083 ceph-mon[77242]: pgmap v3829: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:50.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:35:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:35:50.597Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:35:51.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:35:51 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/130417625' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:35:51.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:35:51 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:35:51.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:35:51 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:35:51.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:35:51 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:35:51.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:35:51 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/130417625' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:35:51.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:35:51 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:35:51.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:35:51 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:35:51.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:35:51 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:35:51.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:35:51 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/130417625' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:35:51.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:35:51 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:35:51.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:35:51 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:35:51.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:35:51 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:35:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:35:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:35:51] "GET /metrics HTTP/1.1" 200 225182 "" "Prometheus/2.33.4" 2022-09-29T07:35:52.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:35:52 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:35:52.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:35:52 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:35:52.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:35:52 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:35:52.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:35:52 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:35:52.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:35:52 smithi083 ceph-mon[77242]: pgmap v3830: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:52.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:35:52 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:35:52.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:35:52 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:35:52.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:35:52 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:35:52.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:35:52 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:35:52.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:35:52 smithi083 ceph-mon[79930]: pgmap v3830: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:52.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:35:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:35:52.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:35:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:35:52 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:35:52.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:35:52 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:35:52.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:35:52 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:35:52.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:35:52 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:35:52.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:35:52 smithi153 ceph-mon[70044]: pgmap v3830: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:35:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:35:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:35:54.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:35:54 smithi153 ceph-mon[70044]: pgmap v3831: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:54.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:35:54 smithi083 ceph-mon[79930]: pgmap v3831: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:54.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:35:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:35:53.993Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:35:54.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:35:54 smithi083 ceph-mon[77242]: pgmap v3831: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:54.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:35:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:35:54.693Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:35:56.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:35:56 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:35:56.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:35:56 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:35:56.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:35:56 smithi153 ceph-mon[70044]: pgmap v3832: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:56.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:35:56 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:35:56.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:35:56 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:35:56.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:35:56 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:35:56.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:35:56 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:35:56.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:35:56 smithi083 ceph-mon[79930]: pgmap v3832: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:56.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:35:56 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:35:56.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:35:56 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:35:56.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:35:56 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:35:56.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:35:56 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:35:56.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:35:56 smithi083 ceph-mon[77242]: pgmap v3832: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:56.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:35:56 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:35:56.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:35:56 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:35:58.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:35:58 smithi083 ceph-mon[77242]: pgmap v3833: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:58.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:35:58 smithi083 ceph-mon[79930]: pgmap v3833: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:35:58.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:35:58 smithi153 ceph-mon[70044]: pgmap v3833: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:00.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:35:59 smithi083 ceph-mon[79930]: Detected new or changed devices on smithi083 2022-09-29T07:36:00.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:35:59 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:36:00.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:35:59 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:36:00.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:35:59 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:36:00.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:35:59 smithi083 ceph-mon[77242]: Detected new or changed devices on smithi083 2022-09-29T07:36:00.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:35:59 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:36:00.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:35:59 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:36:00.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:35:59 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:36:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:35:59 smithi153 ceph-mon[70044]: Detected new or changed devices on smithi083 2022-09-29T07:36:00.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:35:59 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:36:00.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:35:59 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:36:00.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:35:59 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:36:00.896 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:36:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:36:00.598Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:36:01.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:36:00 smithi083 ceph-mon[79930]: pgmap v3834: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:01.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:36:00 smithi083 ceph-mon[77242]: pgmap v3834: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:01.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:36:00 smithi153 ceph-mon[70044]: pgmap v3834: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:36:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:36:01] "GET /metrics HTTP/1.1" 200 225184 "" "Prometheus/2.33.4" 2022-09-29T07:36:02.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:36:02 smithi083 ceph-mon[79930]: pgmap v3835: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:02.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:36:02 smithi083 ceph-mon[77242]: pgmap v3835: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:02.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:36:02 smithi153 ceph-mon[70044]: pgmap v3835: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:36:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:36:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:36:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:36:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:36:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:36:04.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:36:04 smithi153 ceph-mon[70044]: pgmap v3836: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:04.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:36:04 smithi083 ceph-mon[79930]: pgmap v3836: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:04.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:36:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:36:03.994Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:36:04.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:36:04 smithi083 ceph-mon[77242]: pgmap v3836: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:04.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:36:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:36:04.695Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:36:06.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:36:06 smithi153 ceph-mon[70044]: pgmap v3837: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:06.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:36:06 smithi083 ceph-mon[79930]: pgmap v3837: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:06.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:36:06 smithi083 ceph-mon[77242]: pgmap v3837: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:08.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:36:08 smithi153 ceph-mon[70044]: pgmap v3838: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:08.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:36:08 smithi083 ceph-mon[77242]: pgmap v3838: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:08.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:36:08 smithi083 ceph-mon[79930]: pgmap v3838: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:10.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:36:10 smithi153 ceph-mon[70044]: pgmap v3839: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:10.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:36:10 smithi083 ceph-mon[79930]: pgmap v3839: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:10.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:36:10 smithi083 ceph-mon[77242]: pgmap v3839: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:10.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:36:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:36:10.599Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:36:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:36:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:36:11] "GET /metrics HTTP/1.1" 200 225199 "" "Prometheus/2.33.4" 2022-09-29T07:36:12.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:36:12 smithi083 ceph-mon[77242]: pgmap v3840: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:12.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:36:12 smithi083 ceph-mon[79930]: pgmap v3840: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:12.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:36:12 smithi153 ceph-mon[70044]: pgmap v3840: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:36:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:36:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:36:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:36:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:36:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:36:14.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:36:14 smithi153 ceph-mon[70044]: pgmap v3841: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:14.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:36:14 smithi083 ceph-mon[79930]: pgmap v3841: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:14.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:36:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:36:13.995Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:36:14.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:36:14 smithi083 ceph-mon[77242]: pgmap v3841: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:14.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:36:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:36:14.695Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:36:16.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:36:16 smithi153 ceph-mon[70044]: pgmap v3842: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:16.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:36:16 smithi083 ceph-mon[79930]: pgmap v3842: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:16.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:36:16 smithi083 ceph-mon[77242]: pgmap v3842: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:18.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:36:18 smithi153 ceph-mon[70044]: pgmap v3843: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:18.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:36:18 smithi083 ceph-mon[79930]: pgmap v3843: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:18.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:36:18 smithi083 ceph-mon[77242]: pgmap v3843: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:36:20 smithi153 ceph-mon[70044]: pgmap v3844: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:20.356 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:36:20.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:36:20 smithi083 ceph-mon[79930]: pgmap v3844: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:20.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:36:20 smithi083 ceph-mon[77242]: pgmap v3844: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:20.645 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:36:20.645 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 25s ago 2h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:36:20.645 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 8m ago 2h 61.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:36:20.646 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 25s ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:36:20.646 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 8m ago 2h 27.8M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:36:20.646 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 8m ago 2h 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:36:20.646 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 25s ago 2h 626M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:36:20.647 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 25s ago 2h 529M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:36:20.647 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 8m ago 2h 221M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:36:20.647 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 25s ago 2h 307M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:36:20.647 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 25s ago 2h 26.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:36:20.647 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 8m ago 2h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:36:20.648 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 25s ago 2h 386M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:36:20.648 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 25s ago 2h 378M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:36:20.648 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 25s ago 2h 300M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:36:20.648 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 25s ago 2h 365M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:36:20.648 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 8m ago 2h 305M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:36:20.649 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 8m ago 2h 372M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:36:20.649 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 8m ago 2h 331M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:36:20.649 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 8m ago 2h 385M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:36:20.649 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 8m ago 2h 120M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:36:20.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:36:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:36:20.599Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:36:21.001 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:36:21.001 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:36:21.001 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:36:21.001 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:36:21.002 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:36:21.002 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:36:21.002 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:36:21.002 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:36:21.003 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:36:21.003 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:36:21.003 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:36:21.003 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:36:21.003 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:36:21.004 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:36:21.004 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:36:21.004 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:36:21.004 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:36:21.005 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:36:21.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:36:21 smithi153 ceph-mon[70044]: from='client.48541 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:36:21.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:36:21 smithi153 ceph-mon[70044]: from='client.48547 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:36:21.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:36:21 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3919902696' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:36:21.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:36:21 smithi083 ceph-mon[77242]: from='client.48541 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:36:21.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:36:21 smithi083 ceph-mon[77242]: from='client.48547 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:36:21.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:36:21 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3919902696' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:36:21.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:36:21 smithi083 ceph-mon[79930]: from='client.48541 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:36:21.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:36:21 smithi083 ceph-mon[79930]: from='client.48547 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:36:21.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:36:21 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3919902696' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:36:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:36:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:36:21] "GET /metrics HTTP/1.1" 200 225199 "" "Prometheus/2.33.4" 2022-09-29T07:36:22.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:36:22 smithi083 ceph-mon[79930]: pgmap v3845: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:22.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:36:22 smithi083 ceph-mon[77242]: pgmap v3845: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:22.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:36:22 smithi153 ceph-mon[70044]: pgmap v3845: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:36:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:36:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:36:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:36:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:36:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:36:24.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:36:24 smithi153 ceph-mon[70044]: pgmap v3846: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:24.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:36:24 smithi083 ceph-mon[79930]: pgmap v3846: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:24.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:36:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:36:23.995Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:36:24.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:36:24 smithi083 ceph-mon[77242]: pgmap v3846: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:24.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:36:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:36:24.695Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:36:26.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:36:26 smithi153 ceph-mon[70044]: pgmap v3847: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:26.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:36:26 smithi083 ceph-mon[79930]: pgmap v3847: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:26.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:36:26 smithi083 ceph-mon[77242]: pgmap v3847: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:28.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:36:28 smithi153 ceph-mon[70044]: pgmap v3848: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:28.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:36:28 smithi083 ceph-mon[77242]: pgmap v3848: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:28.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:36:28 smithi083 ceph-mon[79930]: pgmap v3848: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:30.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:36:30 smithi153 ceph-mon[70044]: pgmap v3849: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:30.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:36:30 smithi083 ceph-mon[79930]: pgmap v3849: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:30.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:36:30 smithi083 ceph-mon[77242]: pgmap v3849: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:30.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:36:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:36:30.600Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:36:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:36:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:36:31] "GET /metrics HTTP/1.1" 200 225207 "" "Prometheus/2.33.4" 2022-09-29T07:36:32.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:36:32 smithi083 ceph-mon[79930]: pgmap v3850: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:32.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:36:32 smithi083 ceph-mon[77242]: pgmap v3850: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:32.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:36:32 smithi153 ceph-mon[70044]: pgmap v3850: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:32.851 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:36:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:36:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:36:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:36:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:36:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:36:34.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:36:34 smithi153 ceph-mon[70044]: pgmap v3851: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:34.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:36:34 smithi083 ceph-mon[79930]: pgmap v3851: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:34.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:36:33 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:36:33.995Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:36:34.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:36:34 smithi083 ceph-mon[77242]: pgmap v3851: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:34.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:36:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:36:34.696Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:36:36.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:36:36 smithi153 ceph-mon[70044]: pgmap v3852: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:36.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:36:36 smithi083 ceph-mon[79930]: pgmap v3852: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:36.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:36:36 smithi083 ceph-mon[77242]: pgmap v3852: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:38.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:36:38 smithi153 ceph-mon[70044]: pgmap v3853: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:38.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:36:38 smithi083 ceph-mon[79930]: pgmap v3853: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:38.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:36:38 smithi083 ceph-mon[77242]: pgmap v3853: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:39.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:36:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:36:39.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:36:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:36:39.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:36:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:36:39.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:36:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:36:39.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:36:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:36:39.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:36:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:36:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:36:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:36:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:36:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:36:39.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:36:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:36:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:36:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:36:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:36:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:36:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:36:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:36:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:36:40 smithi153 ceph-mon[70044]: pgmap v3854: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:40.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:36:40 smithi083 ceph-mon[79930]: pgmap v3854: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:40.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:36:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:36:40.601Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:36:40.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:36:40 smithi083 ceph-mon[77242]: pgmap v3854: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:36:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:36:41] "GET /metrics HTTP/1.1" 200 225209 "" "Prometheus/2.33.4" 2022-09-29T07:36:42.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:36:42 smithi083 ceph-mon[77242]: pgmap v3855: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:36:42 smithi083 ceph-mon[79930]: pgmap v3855: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:36:42 smithi153 ceph-mon[70044]: pgmap v3855: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:36:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:36:42.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:36:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:36:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:36:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:36:44.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:36:44 smithi153 ceph-mon[70044]: pgmap v3856: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:44.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:36:44 smithi083 ceph-mon[79930]: pgmap v3856: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:44.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:36:43 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:36:43.995Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:36:44.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:36:44 smithi083 ceph-mon[77242]: pgmap v3856: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:44.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:36:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:36:44.696Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:36:46.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:36:46 smithi153 ceph-mon[70044]: pgmap v3857: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:46.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:36:46 smithi083 ceph-mon[79930]: pgmap v3857: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:46.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:36:46 smithi083 ceph-mon[77242]: pgmap v3857: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:48.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:36:48 smithi153 ceph-mon[70044]: pgmap v3858: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:48.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:36:48 smithi083 ceph-mon[79930]: pgmap v3858: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:48.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:36:48 smithi083 ceph-mon[77242]: pgmap v3858: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:50.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:36:50 smithi153 ceph-mon[70044]: pgmap v3859: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:50.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:36:50 smithi083 ceph-mon[77242]: pgmap v3859: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:50.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:36:50 smithi083 ceph-mon[79930]: pgmap v3859: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:50.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:36:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:36:50.601Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:36:51.328 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:36:51.619 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:36:51.619 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 56s ago 2h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:36:51.620 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 9m ago 2h 61.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:36:51.620 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 56s ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:36:51.620 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 9m ago 2h 27.8M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:36:51.620 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 9m ago 2h 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:36:51.621 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 56s ago 2h 626M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:36:51.621 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 56s ago 2h 529M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:36:51.621 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 9m ago 2h 221M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:36:51.621 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 56s ago 2h 307M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:36:51.621 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 56s ago 2h 26.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:36:51.622 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 9m ago 2h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:36:51.622 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 56s ago 2h 386M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:36:51.622 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 56s ago 2h 378M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:36:51.622 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 56s ago 2h 300M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:36:51.623 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 56s ago 2h 365M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:36:51.623 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 9m ago 2h 305M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:36:51.623 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 9m ago 2h 372M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:36:51.623 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 9m ago 2h 331M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:36:51.623 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 9m ago 2h 385M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:36:51.624 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 9m ago 2h 120M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:36:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:36:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:36:51] "GET /metrics HTTP/1.1" 200 225209 "" "Prometheus/2.33.4" 2022-09-29T07:36:51.978 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:36:51.978 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:36:51.978 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:36:51.979 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:36:51.979 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:36:51.979 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:36:51.979 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:36:51.979 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:36:51.980 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:36:51.980 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:36:51.980 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:36:51.980 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:36:51.981 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:36:51.981 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:36:51.981 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:36:51.981 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:36:51.982 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:36:51.982 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:36:52.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:36:52 smithi083 ceph-mon[79930]: from='client.38625 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:36:52.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:36:52 smithi083 ceph-mon[79930]: from='client.48565 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:36:52.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:36:52 smithi083 ceph-mon[79930]: pgmap v3860: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:52.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:36:52 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2120443479' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:36:52.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:36:52 smithi083 ceph-mon[77242]: from='client.38625 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:36:52.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:36:52 smithi083 ceph-mon[77242]: from='client.48565 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:36:52.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:36:52 smithi083 ceph-mon[77242]: pgmap v3860: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:52.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:36:52 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2120443479' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:36:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:36:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:36:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:36:52.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:36:52 smithi153 ceph-mon[70044]: from='client.38625 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:36:52.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:36:52 smithi153 ceph-mon[70044]: from='client.48565 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:36:52.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:36:52 smithi153 ceph-mon[70044]: pgmap v3860: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:52.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:36:52 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2120443479' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:36:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:36:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:36:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:36:54.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:36:54 smithi153 ceph-mon[70044]: pgmap v3861: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:54.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:36:54 smithi083 ceph-mon[79930]: pgmap v3861: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:54.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:36:53 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:36:53.996Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:36:54.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:36:54 smithi083 ceph-mon[77242]: pgmap v3861: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:54.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:36:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:36:54.697Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:36:56.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:36:56 smithi153 ceph-mon[70044]: pgmap v3862: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:56.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:36:56 smithi083 ceph-mon[77242]: pgmap v3862: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:56.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:36:56 smithi083 ceph-mon[79930]: pgmap v3862: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:58.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:36:58 smithi153 ceph-mon[70044]: pgmap v3863: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:58.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:36:58 smithi083 ceph-mon[77242]: pgmap v3863: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:58.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:36:58 smithi083 ceph-mon[79930]: pgmap v3863: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:36:59.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:36:59 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:36:59.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:36:59 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:36:59.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:36:59 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:36:59.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:36:59 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:36:59.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:36:59 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:36:59.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:36:59 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:36:59.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:36:59 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:36:59.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:36:59 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:36:59.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:36:59 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:37:00.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:37:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:37:00.602Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:37:00.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:37:00 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:37:00.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:37:00 smithi083 ceph-mon[77242]: pgmap v3864: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:00.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:37:00 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:37:00.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:37:00 smithi083 ceph-mon[79930]: pgmap v3864: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:01.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:37:00 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:37:01.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:37:00 smithi153 ceph-mon[70044]: pgmap v3864: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:37:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:37:01] "GET /metrics HTTP/1.1" 200 225208 "" "Prometheus/2.33.4" 2022-09-29T07:37:02.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:37:02 smithi083 ceph-mon[77242]: pgmap v3865: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:02.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:37:02 smithi083 ceph-mon[79930]: pgmap v3865: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:02.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:37:02 smithi153 ceph-mon[70044]: pgmap v3865: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:37:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:37:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:37:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:37:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:37:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:37:04.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:37:04 smithi153 ceph-mon[70044]: pgmap v3866: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:04.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:37:04 smithi083 ceph-mon[79930]: pgmap v3866: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:04.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:37:03 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:37:03.997Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:37:04.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:37:04 smithi083 ceph-mon[77242]: pgmap v3866: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:04.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:37:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:37:04.698Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:37:06.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:37:06 smithi153 ceph-mon[70044]: pgmap v3867: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:06.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:37:06 smithi083 ceph-mon[79930]: pgmap v3867: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:06.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:37:06 smithi083 ceph-mon[77242]: pgmap v3867: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:08.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:37:08 smithi153 ceph-mon[70044]: pgmap v3868: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:08.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:37:08 smithi083 ceph-mon[79930]: pgmap v3868: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:08.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:37:08 smithi083 ceph-mon[77242]: pgmap v3868: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:10.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:37:10 smithi153 ceph-mon[70044]: pgmap v3869: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:10.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:37:10 smithi083 ceph-mon[77242]: pgmap v3869: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:10.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:37:10 smithi083 ceph-mon[79930]: pgmap v3869: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:10.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:37:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:37:10.602Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:37:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:37:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:37:11] "GET /metrics HTTP/1.1" 200 225210 "" "Prometheus/2.33.4" 2022-09-29T07:37:12.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:37:12 smithi083 ceph-mon[77242]: pgmap v3870: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:12.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:37:12 smithi083 ceph-mon[79930]: pgmap v3870: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:12.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:37:12 smithi153 ceph-mon[70044]: pgmap v3870: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:37:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:37:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:37:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:37:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:37:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:37:14.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:37:14 smithi153 ceph-mon[70044]: pgmap v3871: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:14.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:37:14 smithi083 ceph-mon[77242]: pgmap v3871: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:14.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:37:14 smithi083 ceph-mon[79930]: pgmap v3871: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:14.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:37:13 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:37:13.998Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:37:14.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:37:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:37:14.699Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:37:16.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:37:16 smithi153 ceph-mon[70044]: pgmap v3872: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:16.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:37:16 smithi083 ceph-mon[79930]: pgmap v3872: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:16.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:37:16 smithi083 ceph-mon[77242]: pgmap v3872: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:18.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:37:18 smithi153 ceph-mon[70044]: pgmap v3873: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:18.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:37:18 smithi083 ceph-mon[79930]: pgmap v3873: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:18.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:37:18 smithi083 ceph-mon[77242]: pgmap v3873: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:37:20 smithi153 ceph-mon[70044]: pgmap v3874: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:20.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:37:20 smithi083 ceph-mon[79930]: pgmap v3874: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:20.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:37:20 smithi083 ceph-mon[77242]: pgmap v3874: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:20.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:37:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:37:20.604Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:37:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:37:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:37:21] "GET /metrics HTTP/1.1" 200 225210 "" "Prometheus/2.33.4" 2022-09-29T07:37:22.305 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:37:22.596 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:37:22.597 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 87s ago 2h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:37:22.597 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 9m ago 2h 61.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:37:22.597 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 87s ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:37:22.597 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 9m ago 2h 27.8M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:37:22.598 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 9m ago 2h 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:37:22.598 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 87s ago 2h 626M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:37:22.598 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 87s ago 2h 529M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:37:22.598 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 9m ago 2h 221M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:37:22.599 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 87s ago 2h 307M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:37:22.599 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 87s ago 2h 26.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:37:22.599 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 9m ago 2h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:37:22.600 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 87s ago 2h 386M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:37:22.600 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 87s ago 2h 378M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:37:22.600 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 87s ago 2h 300M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:37:22.600 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 87s ago 2h 365M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:37:22.600 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 9m ago 2h 305M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:37:22.601 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 9m ago 2h 372M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:37:22.601 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 9m ago 2h 331M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:37:22.601 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 9m ago 2h 385M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:37:22.601 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 9m ago 2h 120M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:37:22.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:37:22 smithi083 ceph-mon[79930]: pgmap v3875: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:22.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:37:22 smithi083 ceph-mon[77242]: pgmap v3875: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:22.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:37:22 smithi153 ceph-mon[70044]: pgmap v3875: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:37:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:37:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:37:22.951 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:37:22.951 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:37:22.951 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:37:22.952 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:37:22.952 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:37:22.952 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:37:22.952 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:37:22.953 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:37:22.953 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:37:22.953 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:37:22.953 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:37:22.953 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:37:22.954 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:37:22.954 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:37:22.954 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:37:22.954 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:37:22.955 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:37:22.955 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:37:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:37:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:37:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:37:23.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:37:23 smithi083 ceph-mon[79930]: from='client.48577 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:37:23.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:37:23 smithi083 ceph-mon[79930]: from='client.48583 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:37:23.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:37:23 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1754275825' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:37:23.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:37:23 smithi083 ceph-mon[77242]: from='client.48577 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:37:23.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:37:23 smithi083 ceph-mon[77242]: from='client.48583 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:37:23.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:37:23 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1754275825' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:37:23.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:37:23 smithi153 ceph-mon[70044]: from='client.48577 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:37:23.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:37:23 smithi153 ceph-mon[70044]: from='client.48583 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:37:23.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:37:23 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1754275825' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:37:24.346 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:37:23 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:37:23.999Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:37:24.700 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:37:24 smithi083 ceph-mon[79930]: pgmap v3876: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:24.701 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:37:24 smithi083 ceph-mon[77242]: pgmap v3876: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:24.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:37:24 smithi153 ceph-mon[70044]: pgmap v3876: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:24.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:37:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:37:24.699Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:37:26.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:37:26 smithi153 ceph-mon[70044]: pgmap v3877: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:26.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:37:26 smithi083 ceph-mon[77242]: pgmap v3877: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:26.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:37:26 smithi083 ceph-mon[79930]: pgmap v3877: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:28.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:37:28 smithi153 ceph-mon[70044]: pgmap v3878: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:28.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:37:28 smithi083 ceph-mon[79930]: pgmap v3878: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:28.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:37:28 smithi083 ceph-mon[77242]: pgmap v3878: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:30.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:37:30 smithi153 ceph-mon[70044]: pgmap v3879: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:30.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:37:30 smithi083 ceph-mon[79930]: pgmap v3879: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:30.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:37:30 smithi083 ceph-mon[77242]: pgmap v3879: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:30.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:37:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:37:30.605Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:37:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:37:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:37:31] "GET /metrics HTTP/1.1" 200 225205 "" "Prometheus/2.33.4" 2022-09-29T07:37:32.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:37:32 smithi083 ceph-mon[77242]: pgmap v3880: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:32.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:37:32 smithi083 ceph-mon[79930]: pgmap v3880: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:32.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:37:32 smithi153 ceph-mon[70044]: pgmap v3880: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:37:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:37:32.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:37:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:37:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:37:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:37:34.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:37:34 smithi153 ceph-mon[70044]: pgmap v3881: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:34.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:37:34 smithi083 ceph-mon[79930]: pgmap v3881: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:34.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:37:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:37:34.000Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:37:34.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:37:34 smithi083 ceph-mon[77242]: pgmap v3881: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:34.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:37:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:37:34.700Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:37:36.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:37:36 smithi153 ceph-mon[70044]: pgmap v3882: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:36.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:37:36 smithi083 ceph-mon[79930]: pgmap v3882: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:36.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:37:36 smithi083 ceph-mon[77242]: pgmap v3882: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:38.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:37:38 smithi153 ceph-mon[70044]: pgmap v3883: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:38.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:37:38 smithi083 ceph-mon[77242]: pgmap v3883: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:38.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:37:38 smithi083 ceph-mon[79930]: pgmap v3883: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:39.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:37:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:37:39.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:37:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:37:39.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:37:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:37:39.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:37:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:37:39.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:37:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:37:39.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:37:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:37:39.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:37:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:37:39.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:37:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:37:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:37:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:37:39.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:37:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:37:39.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:37:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:37:39.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:37:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:37:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:37:40 smithi153 ceph-mon[70044]: pgmap v3884: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:40.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:37:40 smithi083 ceph-mon[79930]: pgmap v3884: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:40.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:37:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:37:40.605Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:37:40.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:37:40 smithi083 ceph-mon[77242]: pgmap v3884: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:37:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:37:41] "GET /metrics HTTP/1.1" 200 225208 "" "Prometheus/2.33.4" 2022-09-29T07:37:42.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:37:42 smithi083 ceph-mon[77242]: pgmap v3885: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:37:42 smithi083 ceph-mon[79930]: pgmap v3885: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:37:42 smithi153 ceph-mon[70044]: pgmap v3885: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:37:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:37:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:37:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:37:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:37:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:37:44.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:37:44 smithi153 ceph-mon[70044]: pgmap v3886: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:44.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:37:44 smithi083 ceph-mon[79930]: pgmap v3886: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:44.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:37:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:37:44.000Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:37:44.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:37:44 smithi083 ceph-mon[77242]: pgmap v3886: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:44.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:37:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:37:44.701Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:37:46.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:37:46 smithi153 ceph-mon[70044]: pgmap v3887: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:46.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:37:46 smithi083 ceph-mon[77242]: pgmap v3887: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:46.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:37:46 smithi083 ceph-mon[79930]: pgmap v3887: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:48.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:37:48 smithi153 ceph-mon[70044]: pgmap v3888: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:48.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:37:48 smithi083 ceph-mon[79930]: pgmap v3888: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:48.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:37:48 smithi083 ceph-mon[77242]: pgmap v3888: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:50.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:37:50 smithi153 ceph-mon[70044]: pgmap v3889: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:50.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:37:50 smithi083 ceph-mon[77242]: pgmap v3889: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:50.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:37:50 smithi083 ceph-mon[79930]: pgmap v3889: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:50.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:37:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:37:50.605Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:37:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:37:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:37:51] "GET /metrics HTTP/1.1" 200 225208 "" "Prometheus/2.33.4" 2022-09-29T07:37:52.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:37:52 smithi083 ceph-mon[79930]: pgmap v3890: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:52.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:37:52 smithi083 ceph-mon[77242]: pgmap v3890: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:52.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:37:52 smithi153 ceph-mon[70044]: pgmap v3890: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:37:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:37:52.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:37:53.277 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:37:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:37:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:37:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:37:53.574 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:37:53.574 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 118s ago 2h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:37:53.575 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 10m ago 2h 61.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:37:53.575 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 118s ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:37:53.575 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 10m ago 2h 27.8M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:37:53.575 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 10m ago 2h 440M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:37:53.575 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 118s ago 2h 626M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:37:53.576 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 118s ago 2h 529M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:37:53.576 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 10m ago 2h 221M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:37:53.576 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 118s ago 2h 307M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:37:53.576 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 118s ago 2h 26.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:37:53.576 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 10m ago 2h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:37:53.577 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 118s ago 2h 386M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:37:53.577 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 118s ago 2h 378M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:37:53.577 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 118s ago 2h 300M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:37:53.577 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 118s ago 2h 365M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:37:53.578 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 10m ago 2h 305M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:37:53.578 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 10m ago 2h 372M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:37:53.578 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 10m ago 2h 331M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:37:53.578 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 10m ago 2h 385M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:37:53.578 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 10m ago 2h 120M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:37:53.935 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:37:53.935 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:37:53.935 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:37:53.935 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:37:53.936 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:37:53.936 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:37:53.936 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:37:53.936 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:37:53.936 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:37:53.937 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:37:53.937 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:37:53.937 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:37:53.937 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:37:53.938 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:37:53.938 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:37:53.938 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:37:53.938 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:37:53.938 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:37:54.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:37:53 smithi153 ceph-mon[70044]: from='client.48595 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:37:54.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:37:53 smithi153 ceph-mon[70044]: from='client.38667 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:37:54.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:37:53 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3223141861' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:37:54.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:37:53 smithi153 ceph-mon[70044]: pgmap v3891: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:54.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:37:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:37:54.000Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:37:54.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:37:53 smithi083 ceph-mon[77242]: from='client.48595 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:37:54.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:37:53 smithi083 ceph-mon[77242]: from='client.38667 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:37:54.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:37:53 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3223141861' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:37:54.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:37:53 smithi083 ceph-mon[77242]: pgmap v3891: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:54.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:37:53 smithi083 ceph-mon[79930]: from='client.48595 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:37:54.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:37:53 smithi083 ceph-mon[79930]: from='client.38667 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:37:54.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:37:53 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3223141861' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:37:54.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:37:53 smithi083 ceph-mon[79930]: pgmap v3891: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:54.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:37:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:37:54.700Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:37:56.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:37:56 smithi153 ceph-mon[70044]: pgmap v3892: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:56.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:37:56 smithi083 ceph-mon[79930]: pgmap v3892: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:56.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:37:56 smithi083 ceph-mon[77242]: pgmap v3892: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:58.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:37:58 smithi153 ceph-mon[70044]: pgmap v3893: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:58.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:37:58 smithi083 ceph-mon[77242]: pgmap v3893: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:58.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:37:58 smithi083 ceph-mon[79930]: pgmap v3893: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:37:59.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:37:59 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:37:59.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:37:59 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:37:59.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:37:59 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:37:59.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:37:59 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:37:59.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:37:59 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:37:59.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:37:59 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:38:00.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:37:59 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:38:00.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:37:59 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:38:00.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:37:59 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:38:00.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:38:00 smithi083 ceph-mon[79930]: pgmap v3894: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:38:00.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:38:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:38:00.605Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:38:00.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:38:00 smithi083 ceph-mon[77242]: pgmap v3894: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:38:01.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:38:00 smithi153 ceph-mon[70044]: pgmap v3894: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:38:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:38:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:38:01] "GET /metrics HTTP/1.1" 200 225210 "" "Prometheus/2.33.4" 2022-09-29T07:38:02.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:38:02 smithi083 ceph-mon[79930]: pgmap v3895: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:38:02.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:38:02 smithi083 ceph-mon[77242]: pgmap v3895: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:38:02.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:38:02 smithi153 ceph-mon[70044]: pgmap v3895: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:38:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:38:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:38:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:38:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:38:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:38:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:38:04.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:38:04 smithi153 ceph-mon[70044]: pgmap v3896: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:38:04.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:38:04 smithi083 ceph-mon[79930]: pgmap v3896: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:38:04.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:38:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:38:04.002Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:38:04.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:38:04 smithi083 ceph-mon[77242]: pgmap v3896: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:38:04.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:38:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:38:04.701Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:38:05.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:38:05 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:38:05.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:38:05 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:38:05.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:38:05 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:38:05.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:38:05 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:38:05.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:38:05 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:38:05.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:38:05 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:38:05.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:38:05 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:38:05.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:38:05 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:38:05.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:38:05 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:38:05.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:38:05 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:38:05.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:38:05 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:38:05.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:38:05 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:38:06.197 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:38:06 smithi153 ceph-mon[70044]: pgmap v3897: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:38:06.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:38:06 smithi083 ceph-mon[79930]: pgmap v3897: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:38:06.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:38:06 smithi083 ceph-mon[77242]: pgmap v3897: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:38:08.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:38:08 smithi083 ceph-mon[77242]: Detected new or changed devices on smithi153 2022-09-29T07:38:08.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:38:08 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:38:08.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:38:08 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:38:08.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:38:08 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:38:08.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:38:08 smithi083 ceph-mon[77242]: pgmap v3898: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:38:08.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:38:08 smithi083 ceph-mon[79930]: Detected new or changed devices on smithi153 2022-09-29T07:38:08.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:38:08 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:38:08.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:38:08 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:38:08.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:38:08 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:38:08.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:38:08 smithi083 ceph-mon[79930]: pgmap v3898: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:38:09.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:38:08 smithi153 ceph-mon[70044]: Detected new or changed devices on smithi153 2022-09-29T07:38:09.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:38:08 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:38:09.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:38:08 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:38:09.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:38:08 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:38:09.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:38:08 smithi153 ceph-mon[70044]: pgmap v3898: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:38:10.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:38:10 smithi153 ceph-mon[70044]: pgmap v3899: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:38:10.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:38:10 smithi083 ceph-mon[79930]: pgmap v3899: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:38:10.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:38:10 smithi083 ceph-mon[77242]: pgmap v3899: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:38:10.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:38:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:38:10.606Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:38:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:38:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:38:11] "GET /metrics HTTP/1.1" 200 225207 "" "Prometheus/2.33.4" 2022-09-29T07:38:12.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:38:12 smithi083 ceph-mon[77242]: pgmap v3900: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:38:12.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:38:12 smithi083 ceph-mon[79930]: pgmap v3900: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:38:12.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:38:12 smithi153 ceph-mon[70044]: pgmap v3900: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:38:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:38:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:38:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:38:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:38:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:38:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:38:14.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:38:14 smithi153 ceph-mon[70044]: pgmap v3901: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:38:14.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:38:14 smithi083 ceph-mon[79930]: pgmap v3901: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:38:14.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:38:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:38:14.002Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:38:14.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:38:14 smithi083 ceph-mon[77242]: pgmap v3901: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:38:14.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:38:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:38:14.702Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:38:16.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:38:16 smithi153 ceph-mon[70044]: pgmap v3902: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:38:16.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:38:16 smithi083 ceph-mon[79930]: pgmap v3902: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:38:16.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:38:16 smithi083 ceph-mon[77242]: pgmap v3902: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:38:18.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:38:18 smithi153 ceph-mon[70044]: pgmap v3903: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:38:18.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:38:18 smithi083 ceph-mon[79930]: pgmap v3903: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:38:18.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:38:18 smithi083 ceph-mon[77242]: pgmap v3903: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:38:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:38:20 smithi153 ceph-mon[70044]: pgmap v3904: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:38:20.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:38:20 smithi083 ceph-mon[79930]: pgmap v3904: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:38:20.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:38:20 smithi083 ceph-mon[77242]: pgmap v3904: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:38:20.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:38:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:38:20.607Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:38:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:38:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:38:21] "GET /metrics HTTP/1.1" 200 225207 "" "Prometheus/2.33.4" 2022-09-29T07:38:22.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:38:22 smithi083 ceph-mon[79930]: pgmap v3905: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:38:22.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:38:22 smithi083 ceph-mon[77242]: pgmap v3905: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:38:22.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:38:22 smithi153 ceph-mon[70044]: pgmap v3905: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:38:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:38:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:38:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:38:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:38:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:38:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:38:24.270 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:38:24.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:38:24 smithi153 ceph-mon[70044]: pgmap v3906: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:38:24.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:38:24 smithi083 ceph-mon[79930]: pgmap v3906: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:38:24.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:38:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:38:24.003Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:38:24.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:38:24 smithi083 ceph-mon[77242]: pgmap v3906: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:38:24.558 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:38:24.559 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 2m ago 2h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:38:24.559 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 20s ago 2h 62.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:38:24.559 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 2m ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:38:24.559 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 20s ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:38:24.560 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 20s ago 2h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:38:24.560 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 2m ago 2h 626M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:38:24.560 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 2m ago 2h 529M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:38:24.560 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 20s ago 2h 235M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:38:24.560 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 2m ago 2h 307M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:38:24.561 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 2m ago 2h 26.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:38:24.561 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 20s ago 2h 27.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:38:24.561 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 2m ago 2h 386M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:38:24.561 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 2m ago 2h 378M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:38:24.561 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 2m ago 2h 300M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:38:24.562 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 2m ago 2h 365M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:38:24.562 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 20s ago 2h 327M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:38:24.562 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 20s ago 2h 400M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:38:24.562 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 20s ago 2h 355M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:38:24.563 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 20s ago 2h 415M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:38:24.563 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 20s ago 2h 121M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:38:24.914 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:38:24.915 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:38:24.915 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:38:24.915 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:38:24.916 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:38:24.916 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:38:24.916 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:38:24.916 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:38:24.916 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:38:24.917 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:38:24.917 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:38:24.917 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:38:24.917 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:38:24.918 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:38:24.918 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:38:24.918 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:38:24.918 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:38:24.918 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:38:24.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:38:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:38:24.701Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:38:25.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:38:25 smithi153 ceph-mon[70044]: from='client.38679 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:38:25.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:38:25 smithi153 ceph-mon[70044]: from='client.38685 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:38:25.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:38:25 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3441999714' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:38:25.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:38:25 smithi083 ceph-mon[77242]: from='client.38679 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:38:25.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:38:25 smithi083 ceph-mon[77242]: from='client.38685 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:38:25.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:38:25 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3441999714' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:38:25.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:38:25 smithi083 ceph-mon[79930]: from='client.38679 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:38:25.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:38:25 smithi083 ceph-mon[79930]: from='client.38685 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:38:25.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:38:25 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3441999714' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:38:26.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:38:26 smithi153 ceph-mon[70044]: pgmap v3907: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:38:26.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:38:26 smithi083 ceph-mon[77242]: pgmap v3907: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:38:26.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:38:26 smithi083 ceph-mon[79930]: pgmap v3907: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:38:28.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:38:28 smithi153 ceph-mon[70044]: pgmap v3908: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:38:28.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:38:28 smithi083 ceph-mon[79930]: pgmap v3908: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:38:28.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:38:28 smithi083 ceph-mon[77242]: pgmap v3908: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:38:30.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:38:30 smithi153 ceph-mon[70044]: pgmap v3909: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:38:30.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:38:30 smithi083 ceph-mon[77242]: pgmap v3909: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:38:30.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:38:30 smithi083 ceph-mon[79930]: pgmap v3909: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:38:30.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:38:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:38:30.607Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:38:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:38:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:38:31] "GET /metrics HTTP/1.1" 200 225209 "" "Prometheus/2.33.4" 2022-09-29T07:38:32.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:38:32 smithi083 ceph-mon[77242]: pgmap v3910: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:38:32.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:38:32 smithi083 ceph-mon[79930]: pgmap v3910: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:38:32.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:38:32 smithi153 ceph-mon[70044]: pgmap v3910: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:38:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:38:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:38:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:38:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:38:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:38:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:38:34.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:38:34 smithi153 ceph-mon[70044]: pgmap v3911: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:38:34.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:38:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:38:34.004Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:38:34.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:38:34 smithi083 ceph-mon[77242]: pgmap v3911: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:38:34.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:38:34 smithi083 ceph-mon[79930]: pgmap v3911: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:38:34.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:38:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:38:34.702Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:38:36.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:38:36 smithi153 ceph-mon[70044]: pgmap v3912: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:38:36.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:38:36 smithi083 ceph-mon[77242]: pgmap v3912: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:38:36.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:38:36 smithi083 ceph-mon[79930]: pgmap v3912: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:38:38.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:38:38 smithi153 ceph-mon[70044]: pgmap v3913: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:38:38.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:38:38 smithi083 ceph-mon[79930]: pgmap v3913: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:38:38.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:38:38 smithi083 ceph-mon[77242]: pgmap v3913: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:38:39.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:38:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:38:39.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:38:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:38:39.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:38:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:38:39.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:38:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:38:39.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:38:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:38:39.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:38:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:38:39.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:38:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:38:39.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:38:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:38:39.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:38:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:38:39.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:38:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:38:39.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:38:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:38:39.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:38:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:38:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:38:40 smithi153 ceph-mon[70044]: pgmap v3914: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:38:40.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:38:40 smithi083 ceph-mon[79930]: pgmap v3914: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:38:40.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:38:40 smithi083 ceph-mon[77242]: pgmap v3914: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:38:40.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:38:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:38:40.607Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:38:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:38:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:38:41] "GET /metrics HTTP/1.1" 200 225207 "" "Prometheus/2.33.4" 2022-09-29T07:38:42.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:38:42 smithi083 ceph-mon[79930]: pgmap v3915: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:38:42.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:38:42 smithi083 ceph-mon[77242]: pgmap v3915: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:38:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:38:42 smithi153 ceph-mon[70044]: pgmap v3915: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:38:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:38:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:38:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:38:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:38:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:38:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:38:44.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:38:44 smithi153 ceph-mon[70044]: pgmap v3916: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:38:44.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:38:44 smithi083 ceph-mon[79930]: pgmap v3916: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:38:44.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:38:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:38:44.004Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:38:44.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:38:44 smithi083 ceph-mon[77242]: pgmap v3916: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:38:44.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:38:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:38:44.703Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:38:46.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:38:46 smithi153 ceph-mon[70044]: pgmap v3917: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:38:46.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:38:46 smithi083 ceph-mon[79930]: pgmap v3917: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:38:46.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:38:46 smithi083 ceph-mon[77242]: pgmap v3917: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:38:48.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:38:48 smithi153 ceph-mon[70044]: pgmap v3918: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:38:48.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:38:48 smithi083 ceph-mon[79930]: pgmap v3918: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:38:48.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:38:48 smithi083 ceph-mon[77242]: pgmap v3918: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:38:50.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:38:50 smithi153 ceph-mon[70044]: pgmap v3919: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:38:50.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:38:50 smithi083 ceph-mon[77242]: pgmap v3919: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:38:50.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:38:50 smithi083 ceph-mon[79930]: pgmap v3919: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:38:50.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:38:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:38:50.608Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:38:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:38:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:38:51] "GET /metrics HTTP/1.1" 200 225207 "" "Prometheus/2.33.4" 2022-09-29T07:38:52.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:38:52 smithi083 ceph-mon[79930]: pgmap v3920: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:38:52.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:38:52 smithi083 ceph-mon[77242]: pgmap v3920: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:38:52.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:38:52 smithi153 ceph-mon[70044]: pgmap v3920: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:38:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:38:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:38:52.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:38:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:38:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:38:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:38:54.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:38:54 smithi153 ceph-mon[70044]: pgmap v3921: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:38:54.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:38:54 smithi083 ceph-mon[79930]: pgmap v3921: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:38:54.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:38:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:38:54.006Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:38:54.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:38:54 smithi083 ceph-mon[77242]: pgmap v3921: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:38:54.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:38:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:38:54.704Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:38:55.241 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:38:55.529 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:38:55.529 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 3m ago 2h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:38:55.530 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 51s ago 2h 62.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:38:55.530 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 3m ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:38:55.530 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 51s ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:38:55.530 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 51s ago 2h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:38:55.531 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 3m ago 2h 626M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:38:55.531 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 3m ago 2h 529M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:38:55.531 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 51s ago 2h 235M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:38:55.531 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 3m ago 2h 307M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:38:55.532 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 3m ago 2h 26.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:38:55.532 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 51s ago 2h 27.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:38:55.532 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 3m ago 2h 386M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:38:55.532 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 3m ago 2h 378M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:38:55.533 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 3m ago 2h 300M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:38:55.533 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 3m ago 2h 365M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:38:55.533 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 51s ago 2h 327M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:38:55.533 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 51s ago 2h 400M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:38:55.533 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 51s ago 2h 355M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:38:55.534 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 51s ago 2h 415M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:38:55.534 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 51s ago 2h 121M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:38:55.883 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:38:55.883 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:38:55.883 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:38:55.884 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:38:55.884 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:38:55.884 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:38:55.884 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:38:55.885 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:38:55.885 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:38:55.885 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:38:55.885 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:38:55.886 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:38:55.886 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:38:55.886 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:38:55.886 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:38:55.887 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:38:55.887 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:38:55.887 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:38:56.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:38:55 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3768599254' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:38:56.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:38:55 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3768599254' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:38:56.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:38:55 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3768599254' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:38:57.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:38:56 smithi083 ceph-mon[79930]: from='client.48631 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:38:57.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:38:56 smithi083 ceph-mon[79930]: from='client.48637 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:38:57.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:38:56 smithi083 ceph-mon[79930]: pgmap v3922: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:38:57.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:38:56 smithi083 ceph-mon[77242]: from='client.48631 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:38:57.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:38:56 smithi083 ceph-mon[77242]: from='client.48637 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:38:57.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:38:56 smithi083 ceph-mon[77242]: pgmap v3922: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:38:57.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:38:56 smithi153 ceph-mon[70044]: from='client.48631 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:38:57.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:38:56 smithi153 ceph-mon[70044]: from='client.48637 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:38:57.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:38:56 smithi153 ceph-mon[70044]: pgmap v3922: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:38:58.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:38:58 smithi153 ceph-mon[70044]: pgmap v3923: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:38:58.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:38:58 smithi083 ceph-mon[79930]: pgmap v3923: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:38:58.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:38:58 smithi083 ceph-mon[77242]: pgmap v3923: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:39:00 smithi153 ceph-mon[70044]: pgmap v3924: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:00.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:39:00 smithi083 ceph-mon[77242]: pgmap v3924: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:00.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:39:00 smithi083 ceph-mon[79930]: pgmap v3924: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:00.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:39:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:39:00.609Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:39:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:39:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:39:01] "GET /metrics HTTP/1.1" 200 225208 "" "Prometheus/2.33.4" 2022-09-29T07:39:02.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:39:02 smithi083 ceph-mon[79930]: pgmap v3925: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:02.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:39:02 smithi083 ceph-mon[77242]: pgmap v3925: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:02.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:39:02 smithi153 ceph-mon[70044]: pgmap v3925: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:39:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:39:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:39:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:39:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:39:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:39:04.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:39:04 smithi153 ceph-mon[70044]: pgmap v3926: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:04.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:39:04 smithi083 ceph-mon[79930]: pgmap v3926: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:04.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:39:04 smithi083 ceph-mon[77242]: pgmap v3926: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:04.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:39:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:39:04.007Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:39:04.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:39:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:39:04.705Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:39:06.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:39:06 smithi153 ceph-mon[70044]: pgmap v3927: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:06.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:39:06 smithi083 ceph-mon[79930]: pgmap v3927: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:06.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:39:06 smithi083 ceph-mon[77242]: pgmap v3927: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:08.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:39:07 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:39:08.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:39:07 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:39:08.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:39:07 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:39:08.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:39:07 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:39:08.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:39:07 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:39:08.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:39:07 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:39:08.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:39:07 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:39:08.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:39:07 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:39:08.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:39:07 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:39:09.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:39:08 smithi153 ceph-mon[70044]: pgmap v3928: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:09.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:39:08 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:39:09.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:39:08 smithi083 ceph-mon[79930]: pgmap v3928: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:09.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:39:08 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:39:09.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:39:08 smithi083 ceph-mon[77242]: pgmap v3928: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:09.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:39:08 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:39:10.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:39:10 smithi153 ceph-mon[70044]: pgmap v3929: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:10.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:39:10 smithi083 ceph-mon[79930]: pgmap v3929: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:10.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:39:10 smithi083 ceph-mon[77242]: pgmap v3929: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:10.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:39:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:39:10.609Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:39:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:39:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:39:11] "GET /metrics HTTP/1.1" 200 225208 "" "Prometheus/2.33.4" 2022-09-29T07:39:12.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:39:12 smithi083 ceph-mon[79930]: pgmap v3930: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:12.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:39:12 smithi083 ceph-mon[77242]: pgmap v3930: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:12.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:39:12 smithi153 ceph-mon[70044]: pgmap v3930: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:39:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:39:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:39:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:39:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:39:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:39:14.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:39:14 smithi153 ceph-mon[70044]: pgmap v3931: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:14.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:39:14 smithi083 ceph-mon[79930]: pgmap v3931: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:14.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:39:14 smithi083 ceph-mon[77242]: pgmap v3931: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:14.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:39:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:39:14.007Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:39:14.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:39:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:39:14.706Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:39:16.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:39:16 smithi153 ceph-mon[70044]: pgmap v3932: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:16.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:39:16 smithi083 ceph-mon[79930]: pgmap v3932: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:16.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:39:16 smithi083 ceph-mon[77242]: pgmap v3932: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:18.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:39:18 smithi153 ceph-mon[70044]: pgmap v3933: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:18.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:39:18 smithi083 ceph-mon[79930]: pgmap v3933: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:18.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:39:18 smithi083 ceph-mon[77242]: pgmap v3933: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:39:20 smithi153 ceph-mon[70044]: pgmap v3934: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:20.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:39:20 smithi083 ceph-mon[79930]: pgmap v3934: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:20.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:39:20 smithi083 ceph-mon[77242]: pgmap v3934: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:20.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:39:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:39:20.610Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:39:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:39:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:39:21] "GET /metrics HTTP/1.1" 200 225208 "" "Prometheus/2.33.4" 2022-09-29T07:39:22.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:39:22 smithi083 ceph-mon[79930]: pgmap v3935: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:22.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:39:22 smithi083 ceph-mon[77242]: pgmap v3935: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:22.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:39:22 smithi153 ceph-mon[70044]: pgmap v3935: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:39:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:39:22.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:39:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:39:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:39:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:39:24.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:39:24 smithi153 ceph-mon[70044]: pgmap v3936: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:24.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:39:24 smithi083 ceph-mon[79930]: pgmap v3936: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:24.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:39:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:39:24.008Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:39:24.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:39:24 smithi083 ceph-mon[77242]: pgmap v3936: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:24.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:39:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:39:24.706Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:39:26.215 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:39:26.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:39:26 smithi153 ceph-mon[70044]: pgmap v3937: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:26.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:39:26 smithi083 ceph-mon[79930]: pgmap v3937: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:26.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:39:26 smithi083 ceph-mon[77242]: pgmap v3937: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:26.506 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:39:26.507 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 3m ago 2h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:39:26.507 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 82s ago 2h 62.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:39:26.507 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 3m ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:39:26.507 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 82s ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:39:26.508 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 82s ago 2h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:39:26.508 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 3m ago 2h 626M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:39:26.508 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 3m ago 2h 529M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:39:26.508 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 82s ago 2h 235M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:39:26.508 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 3m ago 2h 307M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:39:26.509 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 3m ago 2h 26.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:39:26.509 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 82s ago 2h 27.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:39:26.509 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 3m ago 2h 386M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:39:26.509 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 3m ago 2h 378M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:39:26.510 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 3m ago 2h 300M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:39:26.510 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 3m ago 2h 365M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:39:26.510 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 82s ago 2h 327M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:39:26.510 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 82s ago 2h 400M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:39:26.511 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 82s ago 2h 355M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:39:26.511 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 82s ago 2h 415M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:39:26.511 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 82s ago 2h 121M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:39:26.863 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:39:26.864 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:39:26.864 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:39:26.864 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:39:26.864 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:39:26.865 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:39:26.865 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:39:26.865 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:39:26.865 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:39:26.866 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:39:26.866 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:39:26.866 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:39:26.866 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:39:26.866 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:39:26.867 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:39:26.867 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:39:26.867 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:39:26.867 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:39:27.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:39:27 smithi083 ceph-mon[79930]: from='client.38715 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:39:27.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:39:27 smithi083 ceph-mon[79930]: from='client.48655 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:39:27.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:39:27 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/488865583' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:39:27.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:39:27 smithi083 ceph-mon[77242]: from='client.38715 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:39:27.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:39:27 smithi083 ceph-mon[77242]: from='client.48655 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:39:27.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:39:27 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/488865583' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:39:27.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:39:27 smithi153 ceph-mon[70044]: from='client.38715 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:39:27.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:39:27 smithi153 ceph-mon[70044]: from='client.48655 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:39:27.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:39:27 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/488865583' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:39:28.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:39:28 smithi083 ceph-mon[79930]: pgmap v3938: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:28.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:39:28 smithi083 ceph-mon[77242]: pgmap v3938: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:28.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:39:28 smithi153 ceph-mon[70044]: pgmap v3938: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:30.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:39:30 smithi153 ceph-mon[70044]: pgmap v3939: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:30.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:39:30 smithi083 ceph-mon[79930]: pgmap v3939: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:30.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:39:30 smithi083 ceph-mon[77242]: pgmap v3939: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:30.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:39:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:39:30.610Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:39:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:39:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:39:31] "GET /metrics HTTP/1.1" 200 225208 "" "Prometheus/2.33.4" 2022-09-29T07:39:32.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:39:32 smithi083 ceph-mon[77242]: pgmap v3940: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:32.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:39:32 smithi083 ceph-mon[79930]: pgmap v3940: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:32.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:39:32 smithi153 ceph-mon[70044]: pgmap v3940: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:39:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:39:32.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:39:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:39:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:39:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:39:34.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:39:34 smithi153 ceph-mon[70044]: pgmap v3941: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:34.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:39:34 smithi083 ceph-mon[79930]: pgmap v3941: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:34.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:39:34 smithi083 ceph-mon[77242]: pgmap v3941: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:34.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:39:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:39:34.008Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:39:34.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:39:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:39:34.707Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:39:36.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:39:36 smithi153 ceph-mon[70044]: pgmap v3942: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:36.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:39:36 smithi083 ceph-mon[79930]: pgmap v3942: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:36.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:39:36 smithi083 ceph-mon[77242]: pgmap v3942: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:38.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:39:38 smithi153 ceph-mon[70044]: pgmap v3943: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:38.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:39:38 smithi083 ceph-mon[79930]: pgmap v3943: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:38.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:39:38 smithi083 ceph-mon[77242]: pgmap v3943: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:39.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:39:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:39:39.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:39:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:39:39.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:39:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:39:39.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:39:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:39:39.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:39:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:39:39.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:39:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:39:39.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:39:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:39:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:39:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:39:39.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:39:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:39:39.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:39:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:39:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:39:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:39:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:39:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:39:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:39:40 smithi153 ceph-mon[70044]: pgmap v3944: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:40.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:39:40 smithi083 ceph-mon[79930]: pgmap v3944: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:40.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:39:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:39:40.611Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:39:40.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:39:40 smithi083 ceph-mon[77242]: pgmap v3944: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:39:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:39:41] "GET /metrics HTTP/1.1" 200 225205 "" "Prometheus/2.33.4" 2022-09-29T07:39:42.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:39:42 smithi083 ceph-mon[79930]: pgmap v3945: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:42.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:39:42 smithi083 ceph-mon[77242]: pgmap v3945: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:39:42 smithi153 ceph-mon[70044]: pgmap v3945: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:39:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:39:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:39:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:39:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:39:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:39:44.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:39:44 smithi153 ceph-mon[70044]: pgmap v3946: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:44.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:39:44 smithi083 ceph-mon[79930]: pgmap v3946: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:44.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:39:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:39:44.009Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:39:44.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:39:44 smithi083 ceph-mon[77242]: pgmap v3946: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:44.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:39:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:39:44.707Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:39:46.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:39:46 smithi153 ceph-mon[70044]: pgmap v3947: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:46.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:39:46 smithi083 ceph-mon[77242]: pgmap v3947: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:46.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:39:46 smithi083 ceph-mon[79930]: pgmap v3947: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:48.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:39:48 smithi153 ceph-mon[70044]: pgmap v3948: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:48.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:39:48 smithi083 ceph-mon[77242]: pgmap v3948: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:48.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:39:48 smithi083 ceph-mon[79930]: pgmap v3948: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:50.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:39:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:39:50.612Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:39:51.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:39:51 smithi153 ceph-mon[70044]: pgmap v3949: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:51.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:39:51 smithi083 ceph-mon[79930]: pgmap v3949: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:51.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:39:51 smithi083 ceph-mon[77242]: pgmap v3949: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:39:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:39:51] "GET /metrics HTTP/1.1" 200 225205 "" "Prometheus/2.33.4" 2022-09-29T07:39:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:39:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:39:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:39:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:39:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:39:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:39:53.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:39:53 smithi083 ceph-mon[77242]: pgmap v3950: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:53.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:39:53 smithi083 ceph-mon[79930]: pgmap v3950: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:53.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:39:53 smithi153 ceph-mon[70044]: pgmap v3950: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:54.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:39:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:39:54.009Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:39:54.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:39:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:39:54.708Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:39:55.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:39:55 smithi083 ceph-mon[77242]: pgmap v3951: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:55.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:39:55 smithi083 ceph-mon[79930]: pgmap v3951: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:55.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:39:55 smithi153 ceph-mon[70044]: pgmap v3951: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:57.194 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:39:57.486 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:39:57.487 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 4m ago 2h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:39:57.487 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 113s ago 2h 62.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:39:57.487 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 4m ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:39:57.487 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 113s ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:39:57.488 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 113s ago 2h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:39:57.488 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 4m ago 2h 626M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:39:57.488 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 4m ago 2h 529M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:39:57.488 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 113s ago 2h 235M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:39:57.489 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 4m ago 2h 307M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:39:57.489 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 4m ago 2h 26.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:39:57.489 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 113s ago 2h 27.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:39:57.489 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 4m ago 2h 386M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:39:57.489 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 4m ago 2h 378M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:39:57.490 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 4m ago 2h 300M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:39:57.490 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 4m ago 2h 365M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:39:57.490 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 113s ago 2h 327M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:39:57.490 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 113s ago 2h 400M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:39:57.490 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 113s ago 2h 355M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:39:57.491 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 113s ago 2h 415M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:39:57.491 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 113s ago 2h 121M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:39:57.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:39:57 smithi083 ceph-mon[79930]: pgmap v3952: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:57.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:39:57 smithi083 ceph-mon[77242]: pgmap v3952: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:57.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:39:57 smithi153 ceph-mon[70044]: pgmap v3952: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:57.842 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:39:57.857 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:39:57.858 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:39:57.858 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:39:57.858 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:39:57.858 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:39:57.859 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:39:57.859 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:39:57.859 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:39:57.859 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:39:57.860 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:39:57.860 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:39:57.860 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:39:57.860 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:39:57.861 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:39:57.861 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:39:57.861 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:39:57.861 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:39:58.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:39:58 smithi083 ceph-mon[79930]: from='client.48667 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:39:58.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:39:58 smithi083 ceph-mon[79930]: from='client.38739 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:39:58.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:39:58 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3061655999' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:39:58.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:39:58 smithi083 ceph-mon[77242]: from='client.48667 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:39:58.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:39:58 smithi083 ceph-mon[77242]: from='client.38739 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:39:58.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:39:58 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3061655999' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:39:58.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:39:58 smithi153 ceph-mon[70044]: from='client.48667 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:39:58.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:39:58 smithi153 ceph-mon[70044]: from='client.38739 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:39:58.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:39:58 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3061655999' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:39:59.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:39:59 smithi083 ceph-mon[79930]: pgmap v3953: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:59.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:39:59 smithi083 ceph-mon[77242]: pgmap v3953: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:39:59.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:39:59 smithi153 ceph-mon[70044]: pgmap v3953: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:00.387 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:40:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[77219]: 2022-09-29T07:39:59.999+0000 7f160f665700 -1 log_channel(cluster) log [ERR] : overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T07:40:00.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:40:00 smithi083 ceph-mon[79930]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T07:40:00.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:40:00 smithi083 ceph-mon[77242]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T07:40:00.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:40:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:40:00.614Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:40:00.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:40:00 smithi153 ceph-mon[70044]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T07:40:01.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:40:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:40:01] "GET /metrics HTTP/1.1" 200 225208 "" "Prometheus/2.33.4" 2022-09-29T07:40:01.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:40:01 smithi083 ceph-mon[79930]: pgmap v3954: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:01.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:40:01 smithi083 ceph-mon[77242]: pgmap v3954: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:01.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:40:01 smithi153 ceph-mon[70044]: pgmap v3954: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:40:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:40:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:40:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:40:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:40:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:40:03.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:40:03 smithi083 ceph-mon[79930]: pgmap v3955: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:03.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:40:03 smithi083 ceph-mon[77242]: pgmap v3955: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:03.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:40:03 smithi153 ceph-mon[70044]: pgmap v3955: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:04.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:40:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:40:04.010Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:40:04.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:40:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:40:04.709Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:40:05.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:40:05 smithi083 ceph-mon[79930]: pgmap v3956: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:05.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:40:05 smithi083 ceph-mon[77242]: pgmap v3956: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:05.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:40:05 smithi153 ceph-mon[70044]: pgmap v3956: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:07.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:40:07 smithi083 ceph-mon[79930]: pgmap v3957: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:07.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:40:07 smithi083 ceph-mon[77242]: pgmap v3957: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:07.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:40:07 smithi153 ceph-mon[70044]: pgmap v3957: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:08.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:40:08 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:40:08.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:40:08 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:40:08.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:40:08 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:40:08.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:40:08 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:40:08.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:40:08 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:40:08.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:40:08 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:40:08.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:40:08 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:40:08.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:40:08 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:40:08.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:40:08 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:40:10.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:40:09 smithi083 ceph-mon[79930]: pgmap v3958: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:10.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:40:09 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:40:10.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:40:09 smithi083 ceph-mon[77242]: pgmap v3958: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:10.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:40:09 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:40:10.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:40:09 smithi153 ceph-mon[70044]: pgmap v3958: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:10.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:40:09 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:40:10.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:40:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:40:10.614Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:40:11.924 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:40:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:40:11] "GET /metrics HTTP/1.1" 200 225205 "" "Prometheus/2.33.4" 2022-09-29T07:40:12.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:40:11 smithi083 ceph-mon[79930]: pgmap v3959: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:12.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:40:11 smithi083 ceph-mon[77242]: pgmap v3959: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:12.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:40:11 smithi153 ceph-mon[70044]: pgmap v3959: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:40:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:40:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:40:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:40:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:40:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:40:14.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:40:13 smithi083 ceph-mon[79930]: pgmap v3960: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:14.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:40:13 smithi083 ceph-mon[77242]: pgmap v3960: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:14.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:40:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:40:14.011Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:40:14.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:40:13 smithi153 ceph-mon[70044]: pgmap v3960: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:14.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:40:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:40:14.710Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:40:16.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:40:15 smithi083 ceph-mon[79930]: pgmap v3961: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:16.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:40:15 smithi083 ceph-mon[77242]: pgmap v3961: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:16.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:40:15 smithi153 ceph-mon[70044]: pgmap v3961: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:18.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:40:17 smithi083 ceph-mon[79930]: pgmap v3962: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:18.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:40:17 smithi083 ceph-mon[77242]: pgmap v3962: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:18.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:40:17 smithi153 ceph-mon[70044]: pgmap v3962: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:20.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:40:19 smithi083 ceph-mon[79930]: pgmap v3963: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:20.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:40:19 smithi083 ceph-mon[77242]: pgmap v3963: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:40:19 smithi153 ceph-mon[70044]: pgmap v3963: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:20.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:40:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:40:20.614Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:40:21.938 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:40:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:40:21] "GET /metrics HTTP/1.1" 200 225205 "" "Prometheus/2.33.4" 2022-09-29T07:40:22.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:40:21 smithi083 ceph-mon[77242]: pgmap v3964: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:22.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:40:21 smithi083 ceph-mon[79930]: pgmap v3964: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:22.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:40:21 smithi153 ceph-mon[70044]: pgmap v3964: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:40:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:40:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:40:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:40:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:40:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:40:24.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:40:23 smithi083 ceph-mon[77242]: pgmap v3965: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:24.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:40:23 smithi083 ceph-mon[79930]: pgmap v3965: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:24.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:40:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:40:24.012Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:40:24.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:40:23 smithi153 ceph-mon[70044]: pgmap v3965: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:25.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:40:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:40:24.711Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:40:26.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:40:25 smithi083 ceph-mon[79930]: pgmap v3966: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:26.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:40:25 smithi083 ceph-mon[77242]: pgmap v3966: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:26.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:40:25 smithi153 ceph-mon[70044]: pgmap v3966: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:28.170 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:40:28.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:40:27 smithi083 ceph-mon[79930]: pgmap v3967: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:28.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:40:27 smithi083 ceph-mon[77242]: pgmap v3967: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:28.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:40:27 smithi153 ceph-mon[70044]: pgmap v3967: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:28.458 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:40:28.459 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 4m ago 2h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:40:28.459 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 2m ago 2h 62.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:40:28.459 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 4m ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:40:28.459 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 2m ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:40:28.460 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 2m ago 2h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:40:28.460 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 4m ago 2h 626M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:40:28.460 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 4m ago 2h 529M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:40:28.460 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 2m ago 2h 235M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:40:28.461 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 4m ago 2h 307M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:40:28.461 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 4m ago 2h 26.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:40:28.461 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 2m ago 2h 27.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:40:28.461 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 4m ago 2h 386M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:40:28.462 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 4m ago 2h 378M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:40:28.462 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 4m ago 2h 300M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:40:28.462 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 4m ago 2h 365M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:40:28.462 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 2m ago 2h 327M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:40:28.463 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 2m ago 2h 400M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:40:28.463 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 2m ago 2h 355M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:40:28.463 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 2m ago 2h 415M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:40:28.463 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 2m ago 2h 121M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:40:28.812 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:40:28.812 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:40:28.812 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:40:28.812 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:40:28.813 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:40:28.813 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:40:28.813 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:40:28.813 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:40:28.814 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:40:28.814 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:40:28.814 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:40:28.814 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:40:28.815 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:40:28.815 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:40:28.815 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:40:28.815 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:40:28.815 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:40:28.816 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:40:29.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:40:28 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2972955658' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:40:29.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:40:28 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2972955658' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:40:29.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:40:28 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2972955658' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:40:30.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:40:29 smithi083 ceph-mon[79930]: pgmap v3968: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:30.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:40:29 smithi083 ceph-mon[79930]: from='client.48685 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:40:30.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:40:29 smithi083 ceph-mon[79930]: from='client.38757 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:40:30.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:40:29 smithi083 ceph-mon[77242]: pgmap v3968: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:30.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:40:29 smithi083 ceph-mon[77242]: from='client.48685 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:40:30.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:40:29 smithi083 ceph-mon[77242]: from='client.38757 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:40:30.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:40:29 smithi153 ceph-mon[70044]: pgmap v3968: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:30.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:40:29 smithi153 ceph-mon[70044]: from='client.48685 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:40:30.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:40:29 smithi153 ceph-mon[70044]: from='client.38757 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:40:30.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:40:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:40:30.615Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:40:31.958 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:40:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:40:31] "GET /metrics HTTP/1.1" 200 225209 "" "Prometheus/2.33.4" 2022-09-29T07:40:32.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:40:31 smithi083 ceph-mon[79930]: pgmap v3969: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:32.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:40:31 smithi083 ceph-mon[77242]: pgmap v3969: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:32.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:40:31 smithi153 ceph-mon[70044]: pgmap v3969: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:40:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:40:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:40:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:40:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:40:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:40:34.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:40:33 smithi153 ceph-mon[70044]: pgmap v3970: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:34.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:40:33 smithi083 ceph-mon[77242]: pgmap v3970: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:34.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:40:33 smithi083 ceph-mon[79930]: pgmap v3970: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:34.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:40:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:40:34.013Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:40:35.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:40:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:40:34.711Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:40:36.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:40:35 smithi153 ceph-mon[70044]: pgmap v3971: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:36.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:40:35 smithi083 ceph-mon[79930]: pgmap v3971: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:36.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:40:35 smithi083 ceph-mon[77242]: pgmap v3971: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:38.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:40:37 smithi153 ceph-mon[70044]: pgmap v3972: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:38.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:40:37 smithi083 ceph-mon[79930]: pgmap v3972: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:38.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:40:37 smithi083 ceph-mon[77242]: pgmap v3972: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:40:39 smithi153 ceph-mon[70044]: pgmap v3973: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:40:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:40:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:40:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:40:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:40:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:40:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:40:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:40:40.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:40:39 smithi083 ceph-mon[79930]: pgmap v3973: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:40:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:40:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:40:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:40:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:40:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:40:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:40:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:40:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:40:39 smithi083 ceph-mon[77242]: pgmap v3973: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:40:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:40:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:40:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:40:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:40:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:40:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:40:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:40:40.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:40:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:40:40.615Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:40:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:40:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:40:41] "GET /metrics HTTP/1.1" 200 225209 "" "Prometheus/2.33.4" 2022-09-29T07:40:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:40:41 smithi153 ceph-mon[70044]: pgmap v3974: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:42.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:40:41 smithi083 ceph-mon[79930]: pgmap v3974: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:42.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:40:41 smithi083 ceph-mon[77242]: pgmap v3974: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:40:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:40:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:40:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:40:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:40:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:40:44.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:40:43 smithi153 ceph-mon[70044]: pgmap v3975: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:44.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:40:43 smithi083 ceph-mon[79930]: pgmap v3975: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:44.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:40:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:40:44.014Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:40:44.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:40:43 smithi083 ceph-mon[77242]: pgmap v3975: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:45.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:40:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:40:44.712Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:40:46.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:40:45 smithi153 ceph-mon[70044]: pgmap v3976: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:46.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:40:45 smithi083 ceph-mon[79930]: pgmap v3976: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:46.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:40:45 smithi083 ceph-mon[77242]: pgmap v3976: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:48.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:40:47 smithi153 ceph-mon[70044]: pgmap v3977: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:48.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:40:47 smithi083 ceph-mon[79930]: pgmap v3977: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:48.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:40:47 smithi083 ceph-mon[77242]: pgmap v3977: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:50.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:40:49 smithi153 ceph-mon[70044]: pgmap v3978: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:50.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:40:49 smithi083 ceph-mon[79930]: pgmap v3978: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:50.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:40:49 smithi083 ceph-mon[77242]: pgmap v3978: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:50.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:40:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:40:50.616Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:40:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:40:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:40:51] "GET /metrics HTTP/1.1" 200 225209 "" "Prometheus/2.33.4" 2022-09-29T07:40:52.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:40:51 smithi153 ceph-mon[70044]: pgmap v3979: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:52.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:40:51 smithi083 ceph-mon[79930]: pgmap v3979: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:52.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:40:51 smithi083 ceph-mon[77242]: pgmap v3979: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:40:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:40:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:40:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:40:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:40:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:40:54.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:40:53 smithi153 ceph-mon[70044]: pgmap v3980: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:54.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:40:53 smithi083 ceph-mon[79930]: pgmap v3980: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:54.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:40:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:40:54.014Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:40:54.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:40:53 smithi083 ceph-mon[77242]: pgmap v3980: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:55.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:40:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:40:54.712Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:40:56.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:40:55 smithi153 ceph-mon[70044]: pgmap v3981: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:56.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:40:55 smithi083 ceph-mon[79930]: pgmap v3981: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:56.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:40:55 smithi083 ceph-mon[77242]: pgmap v3981: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:58.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:40:57 smithi153 ceph-mon[70044]: pgmap v3982: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:58.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:40:57 smithi083 ceph-mon[79930]: pgmap v3982: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:58.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:40:57 smithi083 ceph-mon[77242]: pgmap v3982: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:59.139 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:40:59.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:40:59 smithi153 ceph-mon[70044]: pgmap v3983: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:59.426 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:40:59.426 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 5m ago 2h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:40:59.427 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 2m ago 2h 62.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:40:59.427 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 5m ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:40:59.427 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 2m ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:40:59.427 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 2m ago 2h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:40:59.428 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 5m ago 2h 626M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:40:59.428 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 5m ago 2h 529M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:40:59.428 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 2m ago 2h 235M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:40:59.428 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 5m ago 2h 307M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:40:59.428 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 5m ago 2h 26.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:40:59.429 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 2m ago 2h 27.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:40:59.429 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 5m ago 2h 386M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:40:59.429 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 5m ago 2h 378M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:40:59.429 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 5m ago 2h 300M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:40:59.429 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 5m ago 2h 365M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:40:59.430 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 2m ago 2h 327M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:40:59.430 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 2m ago 2h 400M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:40:59.430 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 2m ago 2h 355M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:40:59.430 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 2m ago 2h 415M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:40:59.431 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 2m ago 2h 121M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:40:59.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:40:59 smithi083 ceph-mon[77242]: pgmap v3983: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:59.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:40:59 smithi083 ceph-mon[79930]: pgmap v3983: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:40:59.781 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:40:59.782 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:40:59.782 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:40:59.782 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:40:59.783 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:40:59.783 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:40:59.783 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:40:59.783 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:40:59.783 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:40:59.784 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:40:59.784 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:40:59.784 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:40:59.784 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:40:59.785 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:40:59.785 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:40:59.785 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:40:59.785 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:40:59.785 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:41:00.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:41:00 smithi083 ceph-mon[79930]: from='client.48703 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:41:00.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:41:00 smithi083 ceph-mon[79930]: from='client.38775 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:41:00.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:41:00 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/616298694' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:41:00.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:41:00 smithi083 ceph-mon[77242]: from='client.48703 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:41:00.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:41:00 smithi083 ceph-mon[77242]: from='client.38775 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:41:00.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:41:00 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/616298694' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:41:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:41:00 smithi153 ceph-mon[70044]: from='client.48703 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:41:00.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:41:00 smithi153 ceph-mon[70044]: from='client.38775 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:41:00.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:41:00 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/616298694' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:41:00.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:41:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:41:00.616Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:41:01.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:41:01 smithi083 ceph-mon[79930]: pgmap v3984: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:41:01.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:41:01 smithi083 ceph-mon[77242]: pgmap v3984: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:41:01.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:41:01 smithi153 ceph-mon[70044]: pgmap v3984: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:41:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:41:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:41:01] "GET /metrics HTTP/1.1" 200 225207 "" "Prometheus/2.33.4" 2022-09-29T07:41:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:41:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:41:02.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:41:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:41:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:41:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:41:03.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:41:03 smithi083 ceph-mon[79930]: pgmap v3985: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:41:03.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:41:03 smithi083 ceph-mon[77242]: pgmap v3985: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:41:03.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:41:03 smithi153 ceph-mon[70044]: pgmap v3985: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:41:04.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:41:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:41:04.014Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:41:05.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:41:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:41:04.713Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:41:05.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:41:05 smithi083 ceph-mon[77242]: pgmap v3986: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:41:05.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:41:05 smithi083 ceph-mon[79930]: pgmap v3986: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:41:05.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:41:05 smithi153 ceph-mon[70044]: pgmap v3986: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:41:07.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:41:07 smithi083 ceph-mon[79930]: pgmap v3987: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:41:07.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:41:07 smithi083 ceph-mon[77242]: pgmap v3987: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:41:07.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:41:07 smithi153 ceph-mon[70044]: pgmap v3987: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:41:09.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:41:09 smithi083 ceph-mon[79930]: pgmap v3988: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:41:09.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:41:09 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:41:09.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:41:09 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:41:09.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:41:09 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:41:09.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:41:09 smithi083 ceph-mon[77242]: pgmap v3988: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:41:09.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:41:09 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:41:09.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:41:09 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:41:09.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:41:09 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:41:09.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:41:09 smithi153 ceph-mon[70044]: pgmap v3988: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:41:09.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:41:09 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:41:09.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:41:09 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:41:09.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:41:09 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:41:10.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:41:10 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:41:10.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:41:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:41:10.617Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:41:10.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:41:10 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:41:11.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:41:10 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:41:11.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:41:11 smithi083 ceph-mon[77242]: pgmap v3989: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:41:11.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:41:11 smithi083 ceph-mon[79930]: pgmap v3989: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:41:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:41:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:41:11] "GET /metrics HTTP/1.1" 200 225208 "" "Prometheus/2.33.4" 2022-09-29T07:41:12.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:41:11 smithi153 ceph-mon[70044]: pgmap v3989: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:41:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:41:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:41:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:41:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:41:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:41:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:41:13.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:41:13 smithi083 ceph-mon[79930]: pgmap v3990: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:41:13.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:41:13 smithi083 ceph-mon[77242]: pgmap v3990: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:41:14.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:41:13 smithi153 ceph-mon[70044]: pgmap v3990: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:41:14.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:41:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:41:14.015Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:41:15.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:41:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:41:14.714Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:41:15.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:41:15 smithi083 ceph-mon[79930]: pgmap v3991: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:41:15.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:41:15 smithi083 ceph-mon[77242]: pgmap v3991: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:41:16.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:41:15 smithi153 ceph-mon[70044]: pgmap v3991: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:41:17.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:41:17 smithi083 ceph-mon[79930]: pgmap v3992: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:41:17.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:41:17 smithi083 ceph-mon[77242]: pgmap v3992: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:41:18.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:41:17 smithi153 ceph-mon[70044]: pgmap v3992: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:41:19.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:41:19 smithi083 ceph-mon[77242]: pgmap v3993: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:41:19.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:41:19 smithi083 ceph-mon[79930]: pgmap v3993: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:41:20.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:41:19 smithi153 ceph-mon[70044]: pgmap v3993: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:41:20.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:41:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:41:20.618Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:41:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:41:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:41:21] "GET /metrics HTTP/1.1" 200 225208 "" "Prometheus/2.33.4" 2022-09-29T07:41:21.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:41:21 smithi083 ceph-mon[79930]: pgmap v3994: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:41:21.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:41:21 smithi083 ceph-mon[77242]: pgmap v3994: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:41:22.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:41:21 smithi153 ceph-mon[70044]: pgmap v3994: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:41:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:41:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:41:22.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:41:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:41:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:41:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:41:23.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:41:23 smithi083 ceph-mon[79930]: pgmap v3995: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:41:23.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:41:23 smithi083 ceph-mon[77242]: pgmap v3995: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:41:24.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:41:23 smithi153 ceph-mon[70044]: pgmap v3995: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:41:24.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:41:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:41:24.016Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:41:25.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:41:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:41:24.715Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:41:25.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:41:25 smithi083 ceph-mon[77242]: pgmap v3996: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:41:25.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:41:25 smithi083 ceph-mon[79930]: pgmap v3996: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:41:26.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:41:25 smithi153 ceph-mon[70044]: pgmap v3996: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:41:27.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:41:27 smithi083 ceph-mon[79930]: pgmap v3997: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:41:27.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:41:27 smithi083 ceph-mon[77242]: pgmap v3997: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:41:28.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:41:27 smithi153 ceph-mon[70044]: pgmap v3997: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:41:29.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:41:29 smithi083 ceph-mon[77242]: pgmap v3998: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:41:29.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:41:29 smithi083 ceph-mon[79930]: pgmap v3998: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:41:30.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:41:29 smithi153 ceph-mon[70044]: pgmap v3998: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:41:30.112 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:41:30.396 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:41:30.397 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 5m ago 2h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:41:30.397 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 3m ago 2h 62.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:41:30.397 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 5m ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:41:30.398 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 3m ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:41:30.398 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 3m ago 2h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:41:30.398 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 5m ago 2h 626M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:41:30.398 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 5m ago 2h 529M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:41:30.399 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 3m ago 2h 235M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:41:30.399 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 5m ago 2h 307M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:41:30.399 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 5m ago 2h 26.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:41:30.399 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 3m ago 2h 27.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:41:30.400 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 5m ago 2h 386M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:41:30.400 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 5m ago 2h 378M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:41:30.400 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 5m ago 2h 300M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:41:30.400 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 5m ago 2h 365M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:41:30.400 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 3m ago 2h 327M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:41:30.401 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 3m ago 2h 400M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:41:30.401 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 3m ago 2h 355M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:41:30.401 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 3m ago 2h 415M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:41:30.401 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 3m ago 2h 121M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:41:30.754 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:41:30.754 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:41:30.755 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:41:30.755 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:41:30.755 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:41:30.756 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:41:30.756 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:41:30.756 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:41:30.756 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:41:30.756 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:41:30.757 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:41:30.757 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:41:30.757 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:41:30.757 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:41:30.758 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:41:30.758 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:41:30.758 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:41:30.758 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:41:30.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:41:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:41:30.618Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:41:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:41:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:41:31] "GET /metrics HTTP/1.1" 200 225206 "" "Prometheus/2.33.4" 2022-09-29T07:41:31.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:41:31 smithi083 ceph-mon[77242]: pgmap v3999: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:41:31.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:41:31 smithi083 ceph-mon[77242]: from='client.48721 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:41:31.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:41:31 smithi083 ceph-mon[77242]: from='client.38793 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:41:31.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:41:31 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2254037711' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:41:31.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:41:31 smithi083 ceph-mon[79930]: pgmap v3999: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:41:31.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:41:31 smithi083 ceph-mon[79930]: from='client.48721 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:41:31.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:41:31 smithi083 ceph-mon[79930]: from='client.38793 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:41:31.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:41:31 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2254037711' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:41:32.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:41:31 smithi153 ceph-mon[70044]: pgmap v3999: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:41:32.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:41:31 smithi153 ceph-mon[70044]: from='client.48721 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:41:32.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:41:31 smithi153 ceph-mon[70044]: from='client.38793 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:41:32.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:41:31 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2254037711' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:41:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:41:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:41:32.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:41:33.312 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:41:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:41:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:41:33.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:41:33 smithi083 ceph-mon[79930]: pgmap v4000: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:41:33.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:41:33 smithi083 ceph-mon[77242]: pgmap v4000: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:41:34.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:41:33 smithi153 ceph-mon[70044]: pgmap v4000: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:41:34.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:41:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:41:34.017Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:41:35.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:41:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:41:34.715Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:41:35.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:41:35 smithi083 ceph-mon[77242]: pgmap v4001: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:41:35.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:41:35 smithi083 ceph-mon[79930]: pgmap v4001: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:41:36.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:41:35 smithi153 ceph-mon[70044]: pgmap v4001: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:41:37.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:41:37 smithi083 ceph-mon[79930]: pgmap v4002: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:41:37.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:41:37 smithi083 ceph-mon[77242]: pgmap v4002: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:41:38.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:41:37 smithi153 ceph-mon[70044]: pgmap v4002: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:41:39.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:41:39 smithi083 ceph-mon[77242]: pgmap v4003: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:41:39.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:41:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:41:39.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:41:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:41:39.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:41:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:41:39.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:41:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:41:39.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:41:39 smithi083 ceph-mon[79930]: pgmap v4003: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:41:39.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:41:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:41:39.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:41:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:41:39.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:41:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:41:39.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:41:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:41:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:41:39 smithi153 ceph-mon[70044]: pgmap v4003: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:41:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:41:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:41:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:41:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:41:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:41:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:41:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:41:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:41:40.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:41:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:41:40.619Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:41:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:41:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:41:41] "GET /metrics HTTP/1.1" 200 225208 "" "Prometheus/2.33.4" 2022-09-29T07:41:41.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:41:41 smithi083 ceph-mon[79930]: pgmap v4004: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:41:41.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:41:41 smithi083 ceph-mon[77242]: pgmap v4004: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:41:42.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:41:41 smithi153 ceph-mon[70044]: pgmap v4004: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:41:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:41:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:41:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:41:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:41:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:41:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:41:43.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:41:43 smithi083 ceph-mon[79930]: pgmap v4005: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:41:43.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:41:43 smithi083 ceph-mon[77242]: pgmap v4005: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:41:44.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:41:43 smithi153 ceph-mon[70044]: pgmap v4005: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:41:44.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:41:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:41:44.018Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:41:45.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:41:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:41:44.716Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:41:45.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:41:45 smithi083 ceph-mon[79930]: pgmap v4006: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:41:45.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:41:45 smithi083 ceph-mon[77242]: pgmap v4006: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:41:46.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:41:45 smithi153 ceph-mon[70044]: pgmap v4006: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:41:47.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:41:47 smithi083 ceph-mon[79930]: pgmap v4007: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:41:47.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:41:47 smithi083 ceph-mon[77242]: pgmap v4007: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:41:48.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:41:47 smithi153 ceph-mon[70044]: pgmap v4007: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:41:49.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:41:49 smithi083 ceph-mon[79930]: pgmap v4008: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:41:49.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:41:49 smithi083 ceph-mon[77242]: pgmap v4008: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:41:50.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:41:49 smithi153 ceph-mon[70044]: pgmap v4008: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:41:50.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:41:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:41:50.620Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:41:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:41:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:41:51] "GET /metrics HTTP/1.1" 200 225208 "" "Prometheus/2.33.4" 2022-09-29T07:41:51.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:41:51 smithi083 ceph-mon[79930]: pgmap v4009: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:41:51.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:41:51 smithi083 ceph-mon[77242]: pgmap v4009: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:41:52.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:41:51 smithi153 ceph-mon[70044]: pgmap v4009: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:41:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:41:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:41:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:41:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:41:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:41:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:41:53.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:41:53 smithi083 ceph-mon[77242]: pgmap v4010: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:41:53.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:41:53 smithi083 ceph-mon[79930]: pgmap v4010: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:41:54.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:41:53 smithi153 ceph-mon[70044]: pgmap v4010: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:41:54.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:41:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:41:54.018Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:41:55.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:41:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:41:54.717Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:41:55.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:41:55 smithi083 ceph-mon[79930]: pgmap v4011: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:41:55.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:41:55 smithi083 ceph-mon[77242]: pgmap v4011: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:41:56.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:41:55 smithi153 ceph-mon[70044]: pgmap v4011: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:41:57.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:41:57 smithi083 ceph-mon[77242]: pgmap v4012: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:41:57.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:41:57 smithi083 ceph-mon[79930]: pgmap v4012: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:41:58.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:41:57 smithi153 ceph-mon[70044]: pgmap v4012: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:41:59.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:41:59 smithi083 ceph-mon[79930]: pgmap v4013: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:41:59.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:41:59 smithi083 ceph-mon[77242]: pgmap v4013: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:00.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:41:59 smithi153 ceph-mon[70044]: pgmap v4013: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:00.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:42:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:42:00.621Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:42:01.082 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:42:01.372 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:42:01.372 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 6m ago 2h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:42:01.372 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 3m ago 2h 62.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:42:01.373 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 6m ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:42:01.373 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 3m ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:42:01.373 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 3m ago 2h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:42:01.373 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 6m ago 2h 626M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:42:01.373 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 6m ago 2h 529M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:42:01.374 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 3m ago 2h 235M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:42:01.374 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 6m ago 2h 307M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:42:01.374 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 6m ago 2h 26.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:42:01.374 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 3m ago 2h 27.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:42:01.375 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 6m ago 2h 386M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:42:01.375 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 6m ago 2h 378M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:42:01.375 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 6m ago 2h 300M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:42:01.375 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 6m ago 2h 365M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:42:01.376 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 3m ago 2h 327M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:42:01.376 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 3m ago 2h 400M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:42:01.376 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 3m ago 2h 355M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:42:01.376 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 3m ago 2h 415M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:42:01.376 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 3m ago 2h 121M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:42:01.731 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:42:01.731 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:42:01.731 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:42:01.732 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:42:01.732 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:42:01.732 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:42:01.732 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:42:01.732 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:42:01.733 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:42:01.733 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:42:01.733 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:42:01.733 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:42:01.734 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:42:01.734 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:42:01.734 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:42:01.734 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:42:01.735 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:42:01.735 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:42:01.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:42:01 smithi083 ceph-mon[77242]: pgmap v4014: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:42:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:42:01] "GET /metrics HTTP/1.1" 200 225209 "" "Prometheus/2.33.4" 2022-09-29T07:42:01.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:42:01 smithi083 ceph-mon[79930]: pgmap v4014: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:02.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:42:01 smithi153 ceph-mon[70044]: pgmap v4014: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:02.705 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:42:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:42:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:42:02.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:42:02 smithi083 ceph-mon[79930]: from='client.48739 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:42:02.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:42:02 smithi083 ceph-mon[79930]: from='client.48745 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:42:02.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:42:02 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4112490367' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:42:02.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:42:02 smithi083 ceph-mon[77242]: from='client.48739 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:42:02.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:42:02 smithi083 ceph-mon[77242]: from='client.48745 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:42:02.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:42:02 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4112490367' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:42:03.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:42:02 smithi153 ceph-mon[70044]: from='client.48739 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:42:03.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:42:02 smithi153 ceph-mon[70044]: from='client.48745 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:42:03.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:42:02 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4112490367' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:42:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:42:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:42:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:42:03.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:42:03 smithi083 ceph-mon[79930]: pgmap v4015: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:03.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:42:03 smithi083 ceph-mon[77242]: pgmap v4015: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:04.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:42:03 smithi153 ceph-mon[70044]: pgmap v4015: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:04.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:42:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:42:04.018Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:42:05.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:42:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:42:04.717Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:42:05.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:42:05 smithi083 ceph-mon[79930]: pgmap v4016: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:05.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:42:05 smithi083 ceph-mon[77242]: pgmap v4016: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:06.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:42:05 smithi153 ceph-mon[70044]: pgmap v4016: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:07.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:42:07 smithi083 ceph-mon[79930]: pgmap v4017: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:07.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:42:07 smithi083 ceph-mon[77242]: pgmap v4017: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:08.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:42:07 smithi153 ceph-mon[70044]: pgmap v4017: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:10.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:42:09 smithi153 ceph-mon[70044]: pgmap v4018: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:10.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:42:09 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:42:10.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:42:09 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:42:10.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:42:09 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:42:10.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:42:09 smithi083 ceph-mon[77242]: pgmap v4018: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:10.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:42:09 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:42:10.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:42:09 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:42:10.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:42:09 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:42:10.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:42:09 smithi083 ceph-mon[79930]: pgmap v4018: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:10.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:42:09 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:42:10.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:42:09 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:42:10.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:42:09 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:42:10.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:42:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:42:10.621Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:42:11.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:42:11 smithi153 ceph-mon[70044]: pgmap v4019: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:11.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:42:11 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:42:11.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:42:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:42:11] "GET /metrics HTTP/1.1" 200 225204 "" "Prometheus/2.33.4" 2022-09-29T07:42:11.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:42:11 smithi083 ceph-mon[77242]: pgmap v4019: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:11.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:42:11 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:42:11.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:42:11 smithi083 ceph-mon[79930]: pgmap v4019: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:11.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:42:11 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:42:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:42:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:42:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:42:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:42:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:42:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:42:13.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:42:13 smithi083 ceph-mon[79930]: pgmap v4020: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:13.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:42:13 smithi083 ceph-mon[77242]: pgmap v4020: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:13.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:42:13 smithi153 ceph-mon[70044]: pgmap v4020: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:14.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:42:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:42:14.020Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:42:15.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:42:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:42:14.718Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:42:15.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:42:15 smithi083 ceph-mon[77242]: pgmap v4021: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:15.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:42:15 smithi083 ceph-mon[79930]: pgmap v4021: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:15.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:42:15 smithi153 ceph-mon[70044]: pgmap v4021: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:17.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:42:17 smithi083 ceph-mon[77242]: pgmap v4022: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:17.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:42:17 smithi083 ceph-mon[79930]: pgmap v4022: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:17.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:42:17 smithi153 ceph-mon[70044]: pgmap v4022: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:19.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:42:19 smithi083 ceph-mon[77242]: pgmap v4023: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:19.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:42:19 smithi083 ceph-mon[79930]: pgmap v4023: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:19.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:42:19 smithi153 ceph-mon[70044]: pgmap v4023: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:20.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:42:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:42:20.622Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:42:21.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:42:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:42:21] "GET /metrics HTTP/1.1" 200 225204 "" "Prometheus/2.33.4" 2022-09-29T07:42:21.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:42:21 smithi083 ceph-mon[79930]: pgmap v4024: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:21.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:42:21 smithi083 ceph-mon[77242]: pgmap v4024: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:21.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:42:21 smithi153 ceph-mon[70044]: pgmap v4024: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:42:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:42:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:42:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:42:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:42:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:42:23.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:42:23 smithi083 ceph-mon[79930]: pgmap v4025: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:23.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:42:23 smithi083 ceph-mon[77242]: pgmap v4025: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:23.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:42:23 smithi153 ceph-mon[70044]: pgmap v4025: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:24.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:42:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:42:24.020Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:42:25.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:42:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:42:24.719Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:42:25.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:42:25 smithi083 ceph-mon[79930]: pgmap v4026: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:25.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:42:25 smithi083 ceph-mon[77242]: pgmap v4026: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:25.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:42:25 smithi153 ceph-mon[70044]: pgmap v4026: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:27.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:42:27 smithi083 ceph-mon[79930]: pgmap v4027: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:27.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:42:27 smithi083 ceph-mon[77242]: pgmap v4027: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:27.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:42:27 smithi153 ceph-mon[70044]: pgmap v4027: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:29.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:42:29 smithi083 ceph-mon[77242]: pgmap v4028: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:29.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:42:29 smithi083 ceph-mon[79930]: pgmap v4028: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:29.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:42:29 smithi153 ceph-mon[70044]: pgmap v4028: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:30.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:42:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:42:30.623Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:42:31.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:42:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:42:31] "GET /metrics HTTP/1.1" 200 225208 "" "Prometheus/2.33.4" 2022-09-29T07:42:31.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:42:31 smithi083 ceph-mon[79930]: pgmap v4029: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:31.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:42:31 smithi083 ceph-mon[77242]: pgmap v4029: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:31.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:42:31 smithi153 ceph-mon[70044]: pgmap v4029: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:32.055 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:42:32.343 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:42:32.343 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 6m ago 2h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:42:32.343 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 4m ago 2h 62.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:42:32.344 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 6m ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:42:32.344 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 4m ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:42:32.344 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 4m ago 2h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:42:32.344 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 6m ago 2h 626M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:42:32.345 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 6m ago 2h 529M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:42:32.345 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 4m ago 2h 235M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:42:32.345 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 6m ago 2h 307M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:42:32.345 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 6m ago 2h 26.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:42:32.346 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 4m ago 2h 27.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:42:32.346 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 6m ago 2h 386M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:42:32.346 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 6m ago 2h 378M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:42:32.346 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 6m ago 2h 300M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:42:32.346 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 6m ago 2h 365M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:42:32.347 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 4m ago 2h 327M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:42:32.347 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 4m ago 2h 400M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:42:32.347 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 4m ago 2h 355M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:42:32.347 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 4m ago 2h 415M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:42:32.347 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 4m ago 2h 121M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:42:32.699 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:42:32.699 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:42:32.699 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:42:32.700 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:42:32.700 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:42:32.700 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:42:32.700 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:42:32.701 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:42:32.701 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:42:32.701 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:42:32.701 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:42:32.702 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:42:32.702 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:42:32.702 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:42:32.702 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:42:32.703 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:42:32.703 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:42:32.703 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:42:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:42:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:42:32.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:42:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:42:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:42:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:42:33.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:42:33 smithi083 ceph-mon[79930]: pgmap v4030: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:33.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:42:33 smithi083 ceph-mon[79930]: from='client.48757 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:42:33.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:42:33 smithi083 ceph-mon[79930]: from='client.38829 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:42:33.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:42:33 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3005237244' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:42:33.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:42:33 smithi083 ceph-mon[77242]: pgmap v4030: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:33.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:42:33 smithi083 ceph-mon[77242]: from='client.48757 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:42:33.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:42:33 smithi083 ceph-mon[77242]: from='client.38829 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:42:33.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:42:33 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3005237244' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:42:33.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:42:33 smithi153 ceph-mon[70044]: pgmap v4030: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:33.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:42:33 smithi153 ceph-mon[70044]: from='client.48757 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:42:33.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:42:33 smithi153 ceph-mon[70044]: from='client.38829 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:42:33.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:42:33 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3005237244' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:42:34.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:42:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:42:34.020Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:42:35.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:42:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:42:34.719Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:42:35.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:42:35 smithi083 ceph-mon[79930]: pgmap v4031: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:35.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:42:35 smithi083 ceph-mon[77242]: pgmap v4031: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:35.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:42:35 smithi153 ceph-mon[70044]: pgmap v4031: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:37.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:42:37 smithi083 ceph-mon[79930]: pgmap v4032: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:37.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:42:37 smithi083 ceph-mon[77242]: pgmap v4032: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:37.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:42:37 smithi153 ceph-mon[70044]: pgmap v4032: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:39.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:42:39 smithi083 ceph-mon[79930]: pgmap v4033: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:39.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:42:39 smithi083 ceph-mon[77242]: pgmap v4033: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:39.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:42:39 smithi153 ceph-mon[70044]: pgmap v4033: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:40.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:42:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:42:40.624Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:42:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:42:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:42:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:42:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:42:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:42:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:42:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:42:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:42:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:42:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:42:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:42:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:42:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:42:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:42:40.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:42:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:42:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:42:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:42:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:42:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:42:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:42:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:42:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:42:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:42:41.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:42:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:42:41] "GET /metrics HTTP/1.1" 200 225209 "" "Prometheus/2.33.4" 2022-09-29T07:42:41.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:42:41 smithi083 ceph-mon[79930]: pgmap v4034: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:41.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:42:41 smithi083 ceph-mon[77242]: pgmap v4034: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:41.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:42:41 smithi153 ceph-mon[70044]: pgmap v4034: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:42:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:42:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:42:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:42:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:42:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:42:43.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:42:43 smithi083 ceph-mon[77242]: pgmap v4035: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:43.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:42:43 smithi083 ceph-mon[79930]: pgmap v4035: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:43.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:42:43 smithi153 ceph-mon[70044]: pgmap v4035: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:44.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:42:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:42:44.021Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:42:45.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:42:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:42:44.720Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:42:45.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:42:45 smithi083 ceph-mon[79930]: pgmap v4036: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:45.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:42:45 smithi083 ceph-mon[77242]: pgmap v4036: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:45.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:42:45 smithi153 ceph-mon[70044]: pgmap v4036: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:47.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:42:47 smithi083 ceph-mon[79930]: pgmap v4037: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:47.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:42:47 smithi083 ceph-mon[77242]: pgmap v4037: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:47.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:42:47 smithi153 ceph-mon[70044]: pgmap v4037: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:49.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:42:49 smithi153 ceph-mon[70044]: pgmap v4038: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:49.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:42:49 smithi083 ceph-mon[79930]: pgmap v4038: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:49.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:42:49 smithi083 ceph-mon[77242]: pgmap v4038: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:50.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:42:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:42:50.625Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:42:51.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:42:51 smithi153 ceph-mon[70044]: pgmap v4039: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:51.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:42:51 smithi083 ceph-mon[77242]: pgmap v4039: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:42:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:42:51] "GET /metrics HTTP/1.1" 200 225209 "" "Prometheus/2.33.4" 2022-09-29T07:42:51.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:42:51 smithi083 ceph-mon[79930]: pgmap v4039: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:42:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:42:52.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:42:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:42:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:42:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:42:53.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:42:53 smithi153 ceph-mon[70044]: pgmap v4040: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:53.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:42:53 smithi083 ceph-mon[77242]: pgmap v4040: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:53.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:42:53 smithi083 ceph-mon[79930]: pgmap v4040: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:54.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:42:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:42:54.021Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:42:55.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:42:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:42:54.721Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:42:55.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:42:55 smithi153 ceph-mon[70044]: pgmap v4041: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:55.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:42:55 smithi083 ceph-mon[79930]: pgmap v4041: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:55.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:42:55 smithi083 ceph-mon[77242]: pgmap v4041: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:57.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:42:57 smithi153 ceph-mon[70044]: pgmap v4042: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:57.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:42:57 smithi083 ceph-mon[79930]: pgmap v4042: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:57.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:42:57 smithi083 ceph-mon[77242]: pgmap v4042: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:59.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:42:59 smithi153 ceph-mon[70044]: pgmap v4043: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:59.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:42:59 smithi083 ceph-mon[79930]: pgmap v4043: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:42:59.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:42:59 smithi083 ceph-mon[77242]: pgmap v4043: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:43:00.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:43:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:43:00.626Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:43:01.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:43:01 smithi153 ceph-mon[70044]: pgmap v4044: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:43:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:43:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:43:01] "GET /metrics HTTP/1.1" 200 225206 "" "Prometheus/2.33.4" 2022-09-29T07:43:01.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:43:01 smithi083 ceph-mon[79930]: pgmap v4044: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:43:01.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:43:01 smithi083 ceph-mon[77242]: pgmap v4044: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:43:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:43:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:43:02.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:43:03.025 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:43:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:43:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:43:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:43:03.313 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:43:03.313 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 7m ago 2h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:43:03.314 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 4m ago 2h 62.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:43:03.314 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 7m ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:43:03.314 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 4m ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:43:03.315 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 4m ago 2h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:43:03.315 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 7m ago 2h 626M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:43:03.315 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 7m ago 2h 529M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:43:03.315 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 4m ago 2h 235M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:43:03.315 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 7m ago 2h 307M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:43:03.316 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 7m ago 2h 26.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:43:03.316 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 4m ago 2h 27.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:43:03.316 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 7m ago 2h 386M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:43:03.316 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 7m ago 2h 378M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:43:03.317 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 7m ago 2h 300M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:43:03.317 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 7m ago 2h 365M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:43:03.317 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 4m ago 2h 327M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:43:03.317 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 4m ago 2h 400M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:43:03.317 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 4m ago 2h 355M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:43:03.318 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 4m ago 2h 415M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:43:03.318 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 4m ago 2h 121M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:43:03.684 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:43:03.684 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:43:03.685 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:43:03.685 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:43:03.685 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:43:03.685 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:43:03.686 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:43:03.686 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:43:03.686 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:43:03.686 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:43:03.687 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:43:03.687 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:43:03.687 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:43:03.687 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:43:03.688 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:43:03.688 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:43:03.688 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:43:03.688 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:43:03.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:43:03 smithi153 ceph-mon[70044]: pgmap v4045: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:43:03.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:43:03 smithi153 ceph-mon[70044]: from='client.38841 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:43:03.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:43:03 smithi083 ceph-mon[79930]: pgmap v4045: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:43:03.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:43:03 smithi083 ceph-mon[79930]: from='client.38841 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:43:03.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:43:03 smithi083 ceph-mon[77242]: pgmap v4045: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:43:03.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:43:03 smithi083 ceph-mon[77242]: from='client.38841 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:43:04.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:43:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:43:04.021Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:43:04.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:43:04 smithi153 ceph-mon[70044]: from='client.38847 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:43:04.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:43:04 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4293741507' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:43:04.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:43:04 smithi083 ceph-mon[79930]: from='client.38847 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:43:04.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:43:04 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4293741507' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:43:04.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:43:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:43:04.721Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:43:04.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:43:04 smithi083 ceph-mon[77242]: from='client.38847 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:43:04.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:43:04 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4293741507' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:43:05.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:43:05 smithi153 ceph-mon[70044]: pgmap v4046: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:43:05.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:43:05 smithi083 ceph-mon[79930]: pgmap v4046: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:43:05.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:43:05 smithi083 ceph-mon[77242]: pgmap v4046: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:43:07.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:43:07 smithi153 ceph-mon[70044]: pgmap v4047: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:43:07.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:43:07 smithi083 ceph-mon[77242]: pgmap v4047: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:43:07.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:43:07 smithi083 ceph-mon[79930]: pgmap v4047: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:43:09.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:43:09 smithi153 ceph-mon[70044]: pgmap v4048: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:43:09.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:43:09 smithi083 ceph-mon[79930]: pgmap v4048: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:43:09.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:43:09 smithi083 ceph-mon[77242]: pgmap v4048: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:43:10.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:43:10 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:43:10.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:43:10 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:43:10.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:43:10 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:43:10.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:43:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:43:10.626Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:43:10.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:43:10 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:43:10.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:43:10 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:43:10.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:43:10 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:43:10.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:43:10 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:43:10.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:43:10 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:43:10.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:43:10 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:43:11.929 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:43:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:43:11] "GET /metrics HTTP/1.1" 200 225209 "" "Prometheus/2.33.4" 2022-09-29T07:43:12.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:43:11 smithi083 ceph-mon[77242]: pgmap v4049: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:43:12.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:43:11 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:43:12.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:43:11 smithi083 ceph-mon[79930]: pgmap v4049: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:43:12.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:43:11 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:43:12.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:43:11 smithi153 ceph-mon[70044]: pgmap v4049: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:43:12.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:43:11 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:43:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:43:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:43:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:43:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:43:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:43:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:43:14.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:43:13 smithi083 ceph-mon[79930]: pgmap v4050: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:43:14.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:43:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:43:14.022Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:43:14.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:43:13 smithi083 ceph-mon[77242]: pgmap v4050: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:43:14.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:43:13 smithi153 ceph-mon[70044]: pgmap v4050: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:43:15.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:43:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:43:14.721Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:43:16.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:43:15 smithi083 ceph-mon[79930]: pgmap v4051: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:43:16.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:43:15 smithi083 ceph-mon[77242]: pgmap v4051: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:43:16.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:43:15 smithi153 ceph-mon[70044]: pgmap v4051: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:43:18.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:43:17 smithi083 ceph-mon[79930]: pgmap v4052: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:43:18.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:43:17 smithi083 ceph-mon[77242]: pgmap v4052: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:43:18.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:43:17 smithi153 ceph-mon[70044]: pgmap v4052: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:43:20.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:43:19 smithi083 ceph-mon[79930]: pgmap v4053: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:43:20.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:43:19 smithi083 ceph-mon[77242]: pgmap v4053: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:43:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:43:19 smithi153 ceph-mon[70044]: pgmap v4053: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:43:20.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:43:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:43:20.627Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:43:21.942 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:43:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:43:21] "GET /metrics HTTP/1.1" 200 225209 "" "Prometheus/2.33.4" 2022-09-29T07:43:22.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:43:21 smithi083 ceph-mon[79930]: pgmap v4054: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:43:22.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:43:21 smithi083 ceph-mon[77242]: pgmap v4054: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:43:22.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:43:21 smithi153 ceph-mon[70044]: pgmap v4054: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:43:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:43:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:43:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:43:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:43:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:43:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:43:24.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:43:23 smithi083 ceph-mon[77242]: pgmap v4055: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:43:24.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:43:23 smithi083 ceph-mon[79930]: pgmap v4055: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:43:24.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:43:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:43:24.023Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:43:24.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:43:23 smithi153 ceph-mon[70044]: pgmap v4055: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:43:25.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:43:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:43:24.723Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:43:26.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:43:25 smithi083 ceph-mon[79930]: pgmap v4056: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:43:26.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:43:25 smithi083 ceph-mon[77242]: pgmap v4056: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:43:26.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:43:25 smithi153 ceph-mon[70044]: pgmap v4056: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:43:28.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:43:27 smithi083 ceph-mon[79930]: pgmap v4057: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:43:28.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:43:27 smithi083 ceph-mon[77242]: pgmap v4057: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:43:28.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:43:27 smithi153 ceph-mon[70044]: pgmap v4057: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:43:30.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:43:29 smithi083 ceph-mon[79930]: pgmap v4058: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:43:30.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:43:29 smithi083 ceph-mon[77242]: pgmap v4058: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:43:30.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:43:29 smithi153 ceph-mon[70044]: pgmap v4058: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:43:30.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:43:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:43:30.628Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:43:31.957 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:43:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:43:31] "GET /metrics HTTP/1.1" 200 225207 "" "Prometheus/2.33.4" 2022-09-29T07:43:32.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:43:31 smithi083 ceph-mon[77242]: pgmap v4059: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:43:32.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:43:31 smithi083 ceph-mon[79930]: pgmap v4059: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:43:32.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:43:31 smithi153 ceph-mon[70044]: pgmap v4059: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:43:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:43:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:43:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:43:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:43:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:43:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:43:34.013 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:43:34.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:43:33 smithi083 ceph-mon[79930]: pgmap v4060: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:43:34.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:43:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:43:34.023Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:43:34.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:43:33 smithi083 ceph-mon[77242]: pgmap v4060: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:43:34.305 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:43:34.305 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 7m ago 2h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:43:34.305 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 5m ago 2h 62.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:43:34.306 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 7m ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:43:34.306 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 5m ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:43:34.306 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 5m ago 2h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:43:34.306 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 7m ago 2h 626M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:43:34.307 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 7m ago 2h 529M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:43:34.307 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 5m ago 2h 235M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:43:34.307 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 7m ago 2h 307M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:43:34.307 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 7m ago 2h 26.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:43:34.307 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 5m ago 2h 27.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:43:34.308 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 7m ago 2h 386M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:43:34.308 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 7m ago 2h 378M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:43:34.308 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 7m ago 2h 300M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:43:34.308 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 7m ago 2h 365M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:43:34.309 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 5m ago 2h 327M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:43:34.309 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 5m ago 2h 400M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:43:34.309 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 5m ago 2h 355M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:43:34.309 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 5m ago 2h 415M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:43:34.309 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 5m ago 2h 121M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:43:34.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:43:33 smithi153 ceph-mon[70044]: pgmap v4060: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:43:34.665 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:43:34.665 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:43:34.665 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:43:34.665 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:43:34.666 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:43:34.666 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:43:34.666 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:43:34.666 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:43:34.667 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:43:34.667 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:43:34.667 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:43:34.667 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:43:34.667 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:43:34.668 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:43:34.668 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:43:34.668 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:43:34.668 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:43:34.669 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:43:35.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:43:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:43:34.722Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:43:35.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:43:34 smithi083 ceph-mon[77242]: from='client.48793 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:43:35.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:43:34 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2049733106' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:43:35.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:43:34 smithi083 ceph-mon[79930]: from='client.48793 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:43:35.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:43:34 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2049733106' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:43:35.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:43:34 smithi153 ceph-mon[70044]: from='client.48793 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:43:35.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:43:34 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2049733106' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:43:36.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:43:35 smithi153 ceph-mon[70044]: pgmap v4061: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:43:36.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:43:35 smithi153 ceph-mon[70044]: from='client.38865 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:43:36.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:43:35 smithi083 ceph-mon[79930]: pgmap v4061: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:43:36.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:43:35 smithi083 ceph-mon[79930]: from='client.38865 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:43:36.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:43:35 smithi083 ceph-mon[77242]: pgmap v4061: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:43:36.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:43:35 smithi083 ceph-mon[77242]: from='client.38865 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:43:38.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:43:37 smithi153 ceph-mon[70044]: pgmap v4062: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:43:38.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:43:37 smithi083 ceph-mon[79930]: pgmap v4062: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:43:38.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:43:37 smithi083 ceph-mon[77242]: pgmap v4062: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:43:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:43:39 smithi153 ceph-mon[70044]: pgmap v4063: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:43:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:43:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:43:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:43:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:43:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:43:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:43:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:43:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:43:40.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:43:39 smithi083 ceph-mon[79930]: pgmap v4063: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:43:40.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:43:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:43:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:43:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:43:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:43:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:43:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:43:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:43:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:43:39 smithi083 ceph-mon[77242]: pgmap v4063: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:43:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:43:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:43:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:43:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:43:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:43:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:43:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:43:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:43:40.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:43:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:43:40.629Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:43:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:43:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:43:41] "GET /metrics HTTP/1.1" 200 225203 "" "Prometheus/2.33.4" 2022-09-29T07:43:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:43:41 smithi153 ceph-mon[70044]: pgmap v4064: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:43:42.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:43:41 smithi083 ceph-mon[79930]: pgmap v4064: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:43:42.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:43:41 smithi083 ceph-mon[77242]: pgmap v4064: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:43:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:43:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:43:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:43:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:43:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:43:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:43:44.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:43:43 smithi153 ceph-mon[70044]: pgmap v4065: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:43:44.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:43:43 smithi083 ceph-mon[79930]: pgmap v4065: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:43:44.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:43:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:43:44.024Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:43:44.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:43:43 smithi083 ceph-mon[77242]: pgmap v4065: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:43:45.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:43:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:43:44.723Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:43:46.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:43:45 smithi153 ceph-mon[70044]: pgmap v4066: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:43:46.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:43:45 smithi083 ceph-mon[77242]: pgmap v4066: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:43:46.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:43:45 smithi083 ceph-mon[79930]: pgmap v4066: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:43:48.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:43:47 smithi153 ceph-mon[70044]: pgmap v4067: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:43:48.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:43:47 smithi083 ceph-mon[79930]: pgmap v4067: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:43:48.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:43:47 smithi083 ceph-mon[77242]: pgmap v4067: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:43:50.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:43:49 smithi153 ceph-mon[70044]: pgmap v4068: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:43:50.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:43:49 smithi083 ceph-mon[79930]: pgmap v4068: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:43:50.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:43:49 smithi083 ceph-mon[77242]: pgmap v4068: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:43:50.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:43:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:43:50.629Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:43:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:43:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:43:51] "GET /metrics HTTP/1.1" 200 225203 "" "Prometheus/2.33.4" 2022-09-29T07:43:52.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:43:51 smithi153 ceph-mon[70044]: pgmap v4069: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:43:52.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:43:51 smithi083 ceph-mon[79930]: pgmap v4069: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:43:52.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:43:51 smithi083 ceph-mon[77242]: pgmap v4069: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:43:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:43:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:43:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:43:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:43:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:43:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:43:54.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:43:53 smithi153 ceph-mon[70044]: pgmap v4070: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:43:54.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:43:53 smithi083 ceph-mon[79930]: pgmap v4070: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:43:54.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:43:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:43:54.024Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:43:54.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:43:53 smithi083 ceph-mon[77242]: pgmap v4070: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:43:55.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:43:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:43:54.724Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:43:56.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:43:55 smithi153 ceph-mon[70044]: pgmap v4071: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:43:56.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:43:55 smithi083 ceph-mon[79930]: pgmap v4071: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:43:56.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:43:55 smithi083 ceph-mon[77242]: pgmap v4071: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:43:58.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:43:57 smithi153 ceph-mon[70044]: pgmap v4072: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:43:58.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:43:57 smithi083 ceph-mon[79930]: pgmap v4072: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:43:58.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:43:57 smithi083 ceph-mon[77242]: pgmap v4072: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:44:00 smithi153 ceph-mon[70044]: pgmap v4073: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:00.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:44:00 smithi083 ceph-mon[79930]: pgmap v4073: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:00.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:44:00 smithi083 ceph-mon[77242]: pgmap v4073: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:00.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:44:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:44:00.630Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:44:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:44:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:44:01] "GET /metrics HTTP/1.1" 200 225208 "" "Prometheus/2.33.4" 2022-09-29T07:44:02.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:44:02 smithi153 ceph-mon[70044]: pgmap v4074: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:02.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:44:02 smithi083 ceph-mon[79930]: pgmap v4074: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:02.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:44:02 smithi083 ceph-mon[77242]: pgmap v4074: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:44:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:44:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:44:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:44:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:44:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:44:04.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:44:04 smithi153 ceph-mon[70044]: pgmap v4075: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:04.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:44:04 smithi083 ceph-mon[77242]: pgmap v4075: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:04.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:44:04 smithi083 ceph-mon[79930]: pgmap v4075: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:04.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:44:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:44:04.025Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:44:04.994 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:44:05.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:44:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:44:04.724Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:44:05.286 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:44:05.287 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 8m ago 2h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:44:05.287 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 6m ago 2h 62.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:44:05.287 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 8m ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:44:05.288 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 6m ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:44:05.288 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 6m ago 2h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:44:05.288 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 8m ago 2h 626M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:44:05.289 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 8m ago 2h 529M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:44:05.289 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 6m ago 2h 235M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:44:05.289 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 8m ago 2h 307M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:44:05.289 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 8m ago 2h 26.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:44:05.290 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 6m ago 2h 27.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:44:05.290 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 8m ago 2h 386M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:44:05.290 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 8m ago 2h 378M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:44:05.290 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 8m ago 2h 300M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:44:05.291 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 8m ago 2h 365M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:44:05.291 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 6m ago 2h 327M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:44:05.291 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 6m ago 2h 400M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:44:05.291 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 6m ago 2h 355M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:44:05.292 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 6m ago 2h 415M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:44:05.292 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 6m ago 2h 121M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:44:05.642 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:44:05.642 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:44:05.643 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:44:05.643 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:44:05.643 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:44:05.644 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:44:05.644 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:44:05.644 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:44:05.644 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:44:05.645 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:44:05.645 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:44:05.645 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:44:05.645 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:44:05.646 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:44:05.646 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:44:05.646 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:44:05.646 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:44:05.647 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:44:06.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:44:06 smithi153 ceph-mon[70044]: pgmap v4076: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:06.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:44:06 smithi153 ceph-mon[70044]: from='client.38877 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:44:06.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:44:06 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1330989292' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:44:06.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:44:06 smithi083 ceph-mon[79930]: pgmap v4076: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:06.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:44:06 smithi083 ceph-mon[79930]: from='client.38877 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:44:06.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:44:06 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1330989292' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:44:06.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:44:06 smithi083 ceph-mon[77242]: pgmap v4076: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:06.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:44:06 smithi083 ceph-mon[77242]: from='client.38877 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:44:06.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:44:06 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1330989292' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:44:07.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:44:07 smithi153 ceph-mon[70044]: from='client.38883 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:44:07.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:44:07 smithi083 ceph-mon[79930]: from='client.38883 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:44:07.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:44:07 smithi083 ceph-mon[77242]: from='client.38883 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:44:08.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:44:08 smithi153 ceph-mon[70044]: pgmap v4077: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:08.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:44:08 smithi083 ceph-mon[79930]: pgmap v4077: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:08.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:44:08 smithi083 ceph-mon[77242]: pgmap v4077: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:10.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:44:10 smithi153 ceph-mon[70044]: pgmap v4078: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:10.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:44:10 smithi083 ceph-mon[79930]: pgmap v4078: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:10.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:44:10 smithi083 ceph-mon[77242]: pgmap v4078: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:10.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:44:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:44:10.631Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:44:11.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:44:11 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:44:11.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:44:11 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:44:11.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:44:11 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:44:11.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:44:11 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:44:11.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:44:11 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:44:11.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:44:11 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:44:11.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:44:11 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:44:11.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:44:11 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:44:11.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:44:11 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:44:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:44:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:44:11] "GET /metrics HTTP/1.1" 200 225209 "" "Prometheus/2.33.4" 2022-09-29T07:44:12.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:44:12 smithi153 ceph-mon[70044]: pgmap v4079: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:12.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:44:12 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:44:12.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:44:12 smithi083 ceph-mon[77242]: pgmap v4079: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:12.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:44:12 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:44:12.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:44:12 smithi083 ceph-mon[79930]: pgmap v4079: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:12.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:44:12 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:44:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:44:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:44:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:44:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:44:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:44:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:44:14.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:44:14 smithi153 ceph-mon[70044]: pgmap v4080: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:14.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:44:14 smithi083 ceph-mon[79930]: pgmap v4080: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:14.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:44:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:44:14.025Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:44:14.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:44:14 smithi083 ceph-mon[77242]: pgmap v4080: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:15.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:44:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:44:14.726Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:44:16.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:44:16 smithi153 ceph-mon[70044]: pgmap v4081: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:16.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:44:16 smithi083 ceph-mon[79930]: pgmap v4081: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:16.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:44:16 smithi083 ceph-mon[77242]: pgmap v4081: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:18.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:44:18 smithi153 ceph-mon[70044]: pgmap v4082: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:18.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:44:18 smithi083 ceph-mon[79930]: pgmap v4082: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:18.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:44:18 smithi083 ceph-mon[77242]: pgmap v4082: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:19.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:44:19 smithi083 ceph-mon[79930]: pgmap v4083: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:19.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:44:19 smithi083 ceph-mon[77242]: pgmap v4083: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:19.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:44:19 smithi153 ceph-mon[70044]: pgmap v4083: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:20.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:44:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:44:20.631Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:44:21.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:44:21 smithi083 ceph-mon[79930]: pgmap v4084: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:21.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:44:21 smithi083 ceph-mon[77242]: pgmap v4084: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:21.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:44:21 smithi153 ceph-mon[70044]: pgmap v4084: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:44:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:44:21] "GET /metrics HTTP/1.1" 200 225209 "" "Prometheus/2.33.4" 2022-09-29T07:44:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:44:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:44:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:44:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:44:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:44:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:44:23.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:44:23 smithi083 ceph-mon[77242]: pgmap v4085: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:23.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:44:23 smithi083 ceph-mon[79930]: pgmap v4085: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:23.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:44:23 smithi153 ceph-mon[70044]: pgmap v4085: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:24.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:44:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:44:24.026Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:44:25.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:44:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:44:24.726Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:44:25.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:44:25 smithi083 ceph-mon[79930]: pgmap v4086: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:25.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:44:25 smithi083 ceph-mon[77242]: pgmap v4086: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:25.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:44:25 smithi153 ceph-mon[70044]: pgmap v4086: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:27.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:44:27 smithi083 ceph-mon[79930]: pgmap v4087: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:27.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:44:27 smithi083 ceph-mon[77242]: pgmap v4087: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:27.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:44:27 smithi153 ceph-mon[70044]: pgmap v4087: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:29.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:44:29 smithi083 ceph-mon[79930]: pgmap v4088: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:29.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:44:29 smithi083 ceph-mon[77242]: pgmap v4088: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:29.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:44:29 smithi153 ceph-mon[70044]: pgmap v4088: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:30.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:44:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:44:30.632Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:44:31.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:44:31 smithi153 ceph-mon[70044]: pgmap v4089: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:44:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:44:31] "GET /metrics HTTP/1.1" 200 225208 "" "Prometheus/2.33.4" 2022-09-29T07:44:31.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:44:31 smithi083 ceph-mon[79930]: pgmap v4089: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:31.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:44:31 smithi083 ceph-mon[77242]: pgmap v4089: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:44:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:44:32.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:44:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:44:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:44:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:44:33.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:44:33 smithi153 ceph-mon[70044]: pgmap v4090: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:33.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:44:33 smithi083 ceph-mon[77242]: pgmap v4090: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:33.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:44:33 smithi083 ceph-mon[79930]: pgmap v4090: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:34.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:44:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:44:34.027Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:44:35.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:44:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:44:34.727Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:44:35.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:44:35 smithi153 ceph-mon[70044]: pgmap v4091: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:35.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:44:35 smithi083 ceph-mon[79930]: pgmap v4091: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:35.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:44:35 smithi083 ceph-mon[77242]: pgmap v4091: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:35.967 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:44:36.257 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:44:36.258 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 8m ago 2h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:44:36.258 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 6m ago 2h 62.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:44:36.258 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 8m ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:44:36.258 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 6m ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:44:36.259 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 6m ago 2h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:44:36.259 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 8m ago 2h 626M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:44:36.259 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 8m ago 2h 529M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:44:36.259 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 6m ago 2h 235M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:44:36.259 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 8m ago 2h 307M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:44:36.260 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 8m ago 2h 26.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:44:36.260 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 6m ago 2h 27.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:44:36.260 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 8m ago 2h 386M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:44:36.260 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 8m ago 2h 378M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:44:36.260 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 8m ago 2h 300M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:44:36.261 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 8m ago 2h 365M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:44:36.261 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 6m ago 2h 327M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:44:36.261 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 6m ago 2h 400M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:44:36.261 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 6m ago 2h 355M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:44:36.261 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 6m ago 2h 415M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:44:36.262 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 6m ago 2h 121M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:44:36.619 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:44:36.620 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:44:36.620 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:44:36.620 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:44:36.620 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:44:36.621 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:44:36.621 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:44:36.621 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:44:36.621 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:44:36.621 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:44:36.622 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:44:36.622 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:44:36.622 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:44:36.622 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:44:36.623 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:44:36.623 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:44:36.623 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:44:36.623 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:44:36.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:44:36 smithi153 ceph-mon[70044]: from='client.48829 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:44:36.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:44:36 smithi083 ceph-mon[79930]: from='client.48829 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:44:36.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:44:36 smithi083 ceph-mon[77242]: from='client.48829 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:44:37.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:44:37 smithi153 ceph-mon[70044]: pgmap v4092: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:37.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:44:37 smithi153 ceph-mon[70044]: from='client.38901 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:44:37.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:44:37 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3127281293' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:44:37.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:44:37 smithi083 ceph-mon[79930]: pgmap v4092: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:37.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:44:37 smithi083 ceph-mon[79930]: from='client.38901 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:44:37.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:44:37 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3127281293' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:44:37.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:44:37 smithi083 ceph-mon[77242]: pgmap v4092: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:37.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:44:37 smithi083 ceph-mon[77242]: from='client.38901 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:44:37.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:44:37 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3127281293' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:44:39.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:44:39 smithi153 ceph-mon[70044]: pgmap v4093: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:39.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:44:39 smithi083 ceph-mon[79930]: pgmap v4093: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:39.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:44:39 smithi083 ceph-mon[77242]: pgmap v4093: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:44:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:44:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:44:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:44:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:44:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:44:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:44:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:44:40.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:44:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:44:40.633Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:44:40.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:44:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:44:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:44:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:44:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:44:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:44:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:44:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:44:40.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:44:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:44:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:44:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:44:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:44:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:44:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:44:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:44:41.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:44:41 smithi153 ceph-mon[70044]: pgmap v4094: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:44:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:44:41] "GET /metrics HTTP/1.1" 200 225208 "" "Prometheus/2.33.4" 2022-09-29T07:44:41.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:44:41 smithi083 ceph-mon[79930]: pgmap v4094: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:41.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:44:41 smithi083 ceph-mon[77242]: pgmap v4094: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:44:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:44:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:44:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:44:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:44:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:44:43.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:44:43 smithi153 ceph-mon[70044]: pgmap v4095: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:43.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:44:43 smithi083 ceph-mon[79930]: pgmap v4095: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:43.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:44:43 smithi083 ceph-mon[77242]: pgmap v4095: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:44.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:44:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:44:44.027Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:44:45.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:44:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:44:44.727Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:44:45.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:44:45 smithi153 ceph-mon[70044]: pgmap v4096: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:45.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:44:45 smithi083 ceph-mon[77242]: pgmap v4096: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:45.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:44:45 smithi083 ceph-mon[79930]: pgmap v4096: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:47.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:44:47 smithi153 ceph-mon[70044]: pgmap v4097: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:47.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:44:47 smithi083 ceph-mon[77242]: pgmap v4097: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:47.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:44:47 smithi083 ceph-mon[79930]: pgmap v4097: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:49.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:44:49 smithi153 ceph-mon[70044]: pgmap v4098: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:49.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:44:49 smithi083 ceph-mon[77242]: pgmap v4098: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:49.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:44:49 smithi083 ceph-mon[79930]: pgmap v4098: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:50.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:44:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:44:50.633Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:44:51.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:44:51 smithi153 ceph-mon[70044]: pgmap v4099: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:44:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:44:51] "GET /metrics HTTP/1.1" 200 225208 "" "Prometheus/2.33.4" 2022-09-29T07:44:51.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:44:51 smithi083 ceph-mon[77242]: pgmap v4099: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:51.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:44:51 smithi083 ceph-mon[79930]: pgmap v4099: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:44:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:44:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:44:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:44:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:44:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:44:53.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:44:53 smithi153 ceph-mon[70044]: pgmap v4100: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:53.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:44:53 smithi083 ceph-mon[79930]: pgmap v4100: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:53.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:44:53 smithi083 ceph-mon[77242]: pgmap v4100: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:54.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:44:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:44:54.028Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:44:55.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:44:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:44:54.727Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:44:55.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:44:55 smithi153 ceph-mon[70044]: pgmap v4101: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:55.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:44:55 smithi083 ceph-mon[79930]: pgmap v4101: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:55.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:44:55 smithi083 ceph-mon[77242]: pgmap v4101: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:57.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:44:57 smithi153 ceph-mon[70044]: pgmap v4102: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:57.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:44:57 smithi083 ceph-mon[79930]: pgmap v4102: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:57.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:44:57 smithi083 ceph-mon[77242]: pgmap v4102: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:59.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:44:59 smithi153 ceph-mon[70044]: pgmap v4103: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:59.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:44:59 smithi083 ceph-mon[79930]: pgmap v4103: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:44:59.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:44:59 smithi083 ceph-mon[77242]: pgmap v4103: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:45:00.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:45:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:45:00.634Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:45:01.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:45:01 smithi153 ceph-mon[70044]: pgmap v4104: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:45:01.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:45:01 smithi083 ceph-mon[79930]: pgmap v4104: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:45:01.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:45:01 smithi083 ceph-mon[77242]: pgmap v4104: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:45:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:45:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:45:01] "GET /metrics HTTP/1.1" 200 225209 "" "Prometheus/2.33.4" 2022-09-29T07:45:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:45:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:45:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:45:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:45:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:45:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:45:03.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:45:03 smithi153 ceph-mon[70044]: pgmap v4105: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:45:03.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:45:03 smithi083 ceph-mon[79930]: pgmap v4105: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:45:03.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:45:03 smithi083 ceph-mon[77242]: pgmap v4105: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:45:04.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:45:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:45:04.029Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:45:05.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:45:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:45:04.728Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:45:05.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:45:05 smithi153 ceph-mon[70044]: pgmap v4106: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:45:05.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:45:05 smithi083 ceph-mon[79930]: pgmap v4106: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:45:05.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:45:05 smithi083 ceph-mon[77242]: pgmap v4106: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:45:06.946 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:45:07.235 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:45:07.235 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 9m ago 2h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:45:07.235 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 7m ago 2h 62.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:45:07.236 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 9m ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:45:07.236 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 7m ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:45:07.236 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 7m ago 2h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:45:07.236 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 9m ago 2h 626M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:45:07.237 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 9m ago 2h 529M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:45:07.237 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 7m ago 2h 235M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:45:07.237 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 9m ago 2h 307M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:45:07.237 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 9m ago 2h 26.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:45:07.237 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 7m ago 2h 27.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:45:07.238 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 9m ago 2h 386M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:45:07.238 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 9m ago 2h 378M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:45:07.238 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 9m ago 2h 300M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:45:07.238 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 9m ago 2h 365M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:45:07.238 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 7m ago 2h 327M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:45:07.239 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 7m ago 2h 400M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:45:07.239 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 7m ago 2h 355M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:45:07.239 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 7m ago 2h 415M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:45:07.239 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 7m ago 2h 121M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:45:07.593 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:45:07.593 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:45:07.593 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:45:07.594 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:45:07.594 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:45:07.594 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:45:07.594 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:45:07.594 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:45:07.595 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:45:07.595 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:45:07.595 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:45:07.595 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:45:07.595 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:45:07.596 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:45:07.596 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:45:07.596 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:45:07.596 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:45:07.596 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:45:07.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:45:07 smithi153 ceph-mon[70044]: pgmap v4107: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:45:07.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:45:07 smithi153 ceph-mon[70044]: from='client.48847 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:45:07.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:45:07 smithi083 ceph-mon[77242]: pgmap v4107: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:45:07.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:45:07 smithi083 ceph-mon[77242]: from='client.48847 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:45:07.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:45:07 smithi083 ceph-mon[79930]: pgmap v4107: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:45:07.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:45:07 smithi083 ceph-mon[79930]: from='client.48847 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:45:08.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:45:08 smithi153 ceph-mon[70044]: from='client.38919 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:45:08.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:45:08 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2500104999' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:45:08.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:45:08 smithi083 ceph-mon[77242]: from='client.38919 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:45:08.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:45:08 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2500104999' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:45:08.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:45:08 smithi083 ceph-mon[79930]: from='client.38919 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:45:08.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:45:08 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2500104999' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:45:09.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:45:09 smithi153 ceph-mon[70044]: pgmap v4108: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:45:09.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:45:09 smithi083 ceph-mon[79930]: pgmap v4108: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:45:09.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:45:09 smithi083 ceph-mon[77242]: pgmap v4108: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:45:10.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:45:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:45:10.635Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:45:11.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:45:11 smithi153 ceph-mon[70044]: pgmap v4109: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:45:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:45:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:45:11] "GET /metrics HTTP/1.1" 200 225209 "" "Prometheus/2.33.4" 2022-09-29T07:45:11.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:45:11 smithi083 ceph-mon[79930]: pgmap v4109: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:45:11.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:45:11 smithi083 ceph-mon[77242]: pgmap v4109: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:45:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:45:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:45:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:45:12.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:45:12 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:45:12.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:45:12 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:45:12.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:45:12 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:45:12.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:45:12 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:45:12.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:45:12 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:45:12.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:45:12 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:45:12.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:45:12 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:45:12.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:45:12 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:45:12.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:45:12 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:45:12.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:45:12 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:45:12.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:45:12 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:45:12.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:45:12 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:45:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:45:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:45:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:45:13.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:45:13 smithi153 ceph-mon[70044]: pgmap v4110: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:45:13.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:45:13 smithi083 ceph-mon[79930]: pgmap v4110: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:45:13.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:45:13 smithi083 ceph-mon[77242]: pgmap v4110: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:45:14.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:45:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:45:14.030Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:45:15.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:45:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:45:14.728Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:45:15.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:45:15 smithi153 ceph-mon[70044]: pgmap v4111: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:45:15.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:45:15 smithi083 ceph-mon[79930]: pgmap v4111: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:45:15.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:45:15 smithi083 ceph-mon[77242]: pgmap v4111: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:45:17.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:45:17 smithi153 ceph-mon[70044]: pgmap v4112: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:45:17.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:45:17 smithi083 ceph-mon[77242]: pgmap v4112: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:45:17.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:45:17 smithi083 ceph-mon[79930]: pgmap v4112: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:45:19.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:45:19 smithi083 ceph-mon[79930]: pgmap v4113: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:45:19.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:45:19 smithi083 ceph-mon[77242]: pgmap v4113: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:45:20.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:45:19 smithi153 ceph-mon[70044]: pgmap v4113: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:45:20.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:45:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:45:20.636Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:45:21.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:45:21 smithi083 ceph-mon[77242]: pgmap v4114: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:45:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:45:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:45:21] "GET /metrics HTTP/1.1" 200 225209 "" "Prometheus/2.33.4" 2022-09-29T07:45:21.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:45:21 smithi083 ceph-mon[79930]: pgmap v4114: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:45:22.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:45:21 smithi153 ceph-mon[70044]: pgmap v4114: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:45:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:45:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:45:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:45:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:45:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:45:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:45:23.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:45:23 smithi083 ceph-mon[77242]: pgmap v4115: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:45:23.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:45:23 smithi083 ceph-mon[79930]: pgmap v4115: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:45:24.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:45:23 smithi153 ceph-mon[70044]: pgmap v4115: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:45:24.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:45:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:45:24.032Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:45:25.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:45:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:45:24.728Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:45:25.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:45:25 smithi083 ceph-mon[79930]: pgmap v4116: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:45:25.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:45:25 smithi083 ceph-mon[77242]: pgmap v4116: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:45:26.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:45:25 smithi153 ceph-mon[70044]: pgmap v4116: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:45:27.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:45:27 smithi083 ceph-mon[79930]: pgmap v4117: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:45:27.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:45:27 smithi083 ceph-mon[77242]: pgmap v4117: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:45:28.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:45:27 smithi153 ceph-mon[70044]: pgmap v4117: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:45:29.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:45:29 smithi083 ceph-mon[79930]: pgmap v4118: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:45:29.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:45:29 smithi083 ceph-mon[77242]: pgmap v4118: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:45:30.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:45:29 smithi153 ceph-mon[70044]: pgmap v4118: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:45:30.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:45:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:45:30.636Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:45:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:45:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:45:31] "GET /metrics HTTP/1.1" 200 225206 "" "Prometheus/2.33.4" 2022-09-29T07:45:31.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:45:31 smithi083 ceph-mon[79930]: pgmap v4119: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:45:31.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:45:31 smithi083 ceph-mon[77242]: pgmap v4119: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:45:32.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:45:31 smithi153 ceph-mon[70044]: pgmap v4119: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:45:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:45:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:45:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:45:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:45:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:45:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:45:33.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:45:33 smithi083 ceph-mon[79930]: pgmap v4120: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:45:33.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:45:33 smithi083 ceph-mon[77242]: pgmap v4120: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:45:34.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:45:33 smithi153 ceph-mon[70044]: pgmap v4120: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:45:34.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:45:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:45:34.030Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:45:35.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:45:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:45:34.729Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:45:35.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:45:35 smithi083 ceph-mon[79930]: pgmap v4121: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:45:35.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:45:35 smithi083 ceph-mon[77242]: pgmap v4121: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:45:36.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:45:35 smithi153 ceph-mon[70044]: pgmap v4121: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:45:37.914 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:45:37.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:45:37 smithi083 ceph-mon[79930]: pgmap v4122: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:45:37.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:45:37 smithi083 ceph-mon[77242]: pgmap v4122: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:45:38.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:45:37 smithi153 ceph-mon[70044]: pgmap v4122: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:45:38.203 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:45:38.203 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 9m ago 2h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:45:38.204 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 7m ago 2h 62.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:45:38.204 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 9m ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:45:38.204 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 7m ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:45:38.204 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 7m ago 2h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:45:38.204 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 9m ago 2h 626M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:45:38.205 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 9m ago 2h 529M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:45:38.205 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 7m ago 2h 235M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:45:38.205 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 9m ago 2h 307M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:45:38.205 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 9m ago 2h 26.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:45:38.205 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 7m ago 2h 27.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:45:38.206 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 9m ago 2h 386M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:45:38.206 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 9m ago 2h 378M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:45:38.206 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 9m ago 2h 300M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:45:38.206 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 9m ago 2h 365M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:45:38.207 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 7m ago 2h 327M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:45:38.207 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 7m ago 2h 400M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:45:38.207 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 7m ago 2h 355M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:45:38.207 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 7m ago 2h 415M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:45:38.207 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 7m ago 2h 121M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:45:38.558 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:45:38.558 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:45:38.558 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:45:38.559 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:45:38.559 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:45:38.559 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:45:38.559 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:45:38.560 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:45:38.560 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:45:38.560 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:45:38.560 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:45:38.560 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:45:38.561 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:45:38.561 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:45:38.561 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:45:38.561 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:45:38.562 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:45:38.562 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:45:38.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:45:38 smithi083 ceph-mon[79930]: from='client.38931 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:45:38.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:45:38 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/915923669' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:45:38.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:45:38 smithi083 ceph-mon[77242]: from='client.38931 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:45:38.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:45:38 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/915923669' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:45:39.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:45:38 smithi153 ceph-mon[70044]: from='client.38931 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:45:39.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:45:38 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/915923669' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:45:39.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:45:39 smithi083 ceph-mon[79930]: pgmap v4123: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:45:39.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:45:39 smithi083 ceph-mon[79930]: from='client.48871 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:45:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:45:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:45:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:45:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:45:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:45:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:45:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:45:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:45:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:45:39 smithi083 ceph-mon[77242]: pgmap v4123: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:45:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:45:39 smithi083 ceph-mon[77242]: from='client.48871 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:45:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:45:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:45:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:45:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:45:39.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:45:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:45:39.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:45:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:45:40.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:45:39 smithi153 ceph-mon[70044]: pgmap v4123: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:45:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:45:39 smithi153 ceph-mon[70044]: from='client.48871 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:45:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:45:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:45:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:45:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:45:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:45:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:45:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:45:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:45:40.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:45:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:45:40.637Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:45:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:45:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:45:41] "GET /metrics HTTP/1.1" 200 225207 "" "Prometheus/2.33.4" 2022-09-29T07:45:41.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:45:41 smithi083 ceph-mon[79930]: pgmap v4124: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:45:41.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:45:41 smithi083 ceph-mon[77242]: pgmap v4124: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:45:42.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:45:41 smithi153 ceph-mon[70044]: pgmap v4124: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:45:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:45:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:45:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:45:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:45:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:45:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:45:43.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:45:43 smithi083 ceph-mon[79930]: pgmap v4125: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:45:43.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:45:43 smithi083 ceph-mon[77242]: pgmap v4125: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:45:44.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:45:43 smithi153 ceph-mon[70044]: pgmap v4125: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:45:44.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:45:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:45:44.031Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:45:45.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:45:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:45:44.729Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:45:45.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:45:45 smithi083 ceph-mon[79930]: pgmap v4126: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:45:45.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:45:45 smithi083 ceph-mon[77242]: pgmap v4126: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:45:46.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:45:45 smithi153 ceph-mon[70044]: pgmap v4126: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:45:47.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:45:47 smithi083 ceph-mon[79930]: pgmap v4127: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:45:47.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:45:47 smithi083 ceph-mon[77242]: pgmap v4127: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:45:48.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:45:47 smithi153 ceph-mon[70044]: pgmap v4127: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:45:49.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:45:49 smithi083 ceph-mon[79930]: pgmap v4128: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:45:49.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:45:49 smithi083 ceph-mon[77242]: pgmap v4128: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:45:50.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:45:49 smithi153 ceph-mon[70044]: pgmap v4128: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:45:50.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:45:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:45:50.638Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:45:51.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:45:51 smithi083 ceph-mon[79930]: pgmap v4129: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:45:51.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:45:51 smithi083 ceph-mon[77242]: pgmap v4129: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:45:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:45:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:45:51] "GET /metrics HTTP/1.1" 200 225207 "" "Prometheus/2.33.4" 2022-09-29T07:45:52.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:45:51 smithi153 ceph-mon[70044]: pgmap v4129: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:45:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:45:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:45:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:45:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:45:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:45:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:45:53.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:45:53 smithi083 ceph-mon[79930]: pgmap v4130: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:45:53.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:45:53 smithi083 ceph-mon[77242]: pgmap v4130: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:45:54.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:45:53 smithi153 ceph-mon[70044]: pgmap v4130: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:45:54.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:45:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:45:54.034Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:45:55.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:45:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:45:54.730Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:45:55.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:45:55 smithi083 ceph-mon[79930]: pgmap v4131: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:45:55.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:45:55 smithi083 ceph-mon[77242]: pgmap v4131: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:45:56.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:45:55 smithi153 ceph-mon[70044]: pgmap v4131: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:45:57.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:45:57 smithi083 ceph-mon[79930]: pgmap v4132: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:45:57.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:45:57 smithi083 ceph-mon[77242]: pgmap v4132: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:45:58.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:45:57 smithi153 ceph-mon[70044]: pgmap v4132: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:45:59.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:45:59 smithi083 ceph-mon[79930]: pgmap v4133: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:45:59.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:45:59 smithi083 ceph-mon[77242]: pgmap v4133: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:46:00.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:45:59 smithi153 ceph-mon[70044]: pgmap v4133: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:46:00.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:46:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:46:00.639Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:46:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:46:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:46:01] "GET /metrics HTTP/1.1" 200 225206 "" "Prometheus/2.33.4" 2022-09-29T07:46:01.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:46:01 smithi083 ceph-mon[79930]: pgmap v4134: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:46:01.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:46:01 smithi083 ceph-mon[77242]: pgmap v4134: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:46:02.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:46:01 smithi153 ceph-mon[70044]: pgmap v4134: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:46:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:46:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:46:02.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:46:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:46:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:46:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:46:03.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:46:03 smithi083 ceph-mon[79930]: pgmap v4135: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:46:03.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:46:03 smithi083 ceph-mon[77242]: pgmap v4135: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:46:04.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:46:03 smithi153 ceph-mon[70044]: pgmap v4135: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:46:04.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:46:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:46:04.032Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:46:05.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:46:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:46:04.730Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:46:05.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:46:05 smithi083 ceph-mon[77242]: pgmap v4136: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:46:05.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:46:05 smithi083 ceph-mon[79930]: pgmap v4136: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:46:06.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:46:05 smithi153 ceph-mon[70044]: pgmap v4136: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:46:07.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:46:07 smithi083 ceph-mon[79930]: pgmap v4137: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:46:07.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:46:07 smithi083 ceph-mon[77242]: pgmap v4137: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:46:08.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:46:07 smithi153 ceph-mon[70044]: pgmap v4137: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:46:08.888 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:46:09.177 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:46:09.177 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 10m ago 2h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:46:09.177 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 8m ago 2h 62.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:46:09.178 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 10m ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:46:09.178 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 8m ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:46:09.178 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 8m ago 2h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:46:09.178 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 10m ago 2h 626M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:46:09.179 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 10m ago 2h 529M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:46:09.179 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 8m ago 2h 235M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:46:09.179 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 10m ago 2h 307M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:46:09.179 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 10m ago 2h 26.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:46:09.180 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 8m ago 2h 27.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:46:09.180 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 10m ago 2h 386M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:46:09.180 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 10m ago 2h 378M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:46:09.180 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 10m ago 2h 300M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:46:09.180 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 10m ago 2h 365M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:46:09.181 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 8m ago 2h 327M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:46:09.181 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 8m ago 2h 400M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:46:09.181 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 8m ago 2h 355M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:46:09.181 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 8m ago 2h 415M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:46:09.182 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 8m ago 2h 121M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:46:09.532 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:46:09.533 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:46:09.533 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:46:09.533 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:46:09.534 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:46:09.534 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:46:09.534 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:46:09.534 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:46:09.535 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:46:09.535 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:46:09.535 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:46:09.535 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:46:09.536 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:46:09.536 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:46:09.536 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:46:09.536 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:46:09.537 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:46:09.537 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:46:09.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:46:09 smithi083 ceph-mon[79930]: pgmap v4138: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:46:09.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:46:09 smithi083 ceph-mon[79930]: from='client.48883 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:46:09.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:46:09 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1539513212' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:46:09.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:46:09 smithi083 ceph-mon[77242]: pgmap v4138: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:46:09.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:46:09 smithi083 ceph-mon[77242]: from='client.48883 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:46:09.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:46:09 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1539513212' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:46:10.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:46:09 smithi153 ceph-mon[70044]: pgmap v4138: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:46:10.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:46:09 smithi153 ceph-mon[70044]: from='client.48883 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:46:10.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:46:09 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1539513212' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:46:10.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:46:10 smithi083 ceph-mon[79930]: from='client.48889 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:46:10.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:46:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:46:10.639Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:46:10.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:46:10 smithi083 ceph-mon[77242]: from='client.48889 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:46:11.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:46:10 smithi153 ceph-mon[70044]: from='client.48889 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:46:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:46:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:46:11] "GET /metrics HTTP/1.1" 200 225210 "" "Prometheus/2.33.4" 2022-09-29T07:46:11.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:46:11 smithi083 ceph-mon[79930]: pgmap v4139: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:46:11.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:46:11 smithi083 ceph-mon[77242]: pgmap v4139: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:46:12.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:46:11 smithi153 ceph-mon[70044]: pgmap v4139: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:46:12.663 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:46:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:46:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:46:12.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:46:12 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:46:12.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:46:12 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:46:12.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:46:12 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:46:12.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:46:12 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:46:12.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:46:12 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:46:12.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:46:12 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:46:13.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:46:12 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:46:13.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:46:12 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:46:13.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:46:12 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:46:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:46:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:46:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:46:14.033 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:46:13 smithi083 ceph-mon[79930]: pgmap v4140: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:46:14.034 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:46:13 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:46:14.034 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:46:13 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:46:14.034 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:46:13 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:46:14.034 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:46:13 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:46:14.035 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:46:13 smithi083 ceph-mon[77242]: pgmap v4140: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:46:14.035 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:46:13 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:46:14.035 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:46:13 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:46:14.035 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:46:13 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:46:14.036 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:46:13 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:46:14.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:46:13 smithi153 ceph-mon[70044]: pgmap v4140: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:46:14.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:46:13 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:46:14.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:46:13 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:46:14.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:46:13 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:46:14.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:46:13 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:46:14.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:46:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:46:14.032Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:46:15.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:46:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:46:14.730Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:46:16.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:46:15 smithi153 ceph-mon[70044]: pgmap v4141: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:46:16.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:46:15 smithi083 ceph-mon[79930]: pgmap v4141: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:46:16.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:46:15 smithi083 ceph-mon[77242]: pgmap v4141: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:46:18.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:46:17 smithi153 ceph-mon[70044]: pgmap v4142: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:46:18.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:46:17 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:46:18.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:46:17 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:46:18.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:46:17 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:46:18.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:46:17 smithi083 ceph-mon[79930]: pgmap v4142: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:46:18.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:46:17 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:46:18.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:46:17 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:46:18.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:46:17 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:46:18.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:46:17 smithi083 ceph-mon[77242]: pgmap v4142: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:46:18.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:46:17 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:46:18.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:46:17 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:46:18.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:46:17 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:46:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:46:19 smithi153 ceph-mon[70044]: pgmap v4143: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:46:20.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:46:19 smithi083 ceph-mon[79930]: pgmap v4143: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:46:20.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:46:19 smithi083 ceph-mon[77242]: pgmap v4143: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:46:20.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:46:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:46:20.640Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:46:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:46:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:46:21] "GET /metrics HTTP/1.1" 200 225210 "" "Prometheus/2.33.4" 2022-09-29T07:46:22.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:46:21 smithi153 ceph-mon[70044]: pgmap v4144: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:46:22.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:46:21 smithi083 ceph-mon[79930]: pgmap v4144: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:46:22.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:46:21 smithi083 ceph-mon[77242]: pgmap v4144: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:46:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:46:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:46:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:46:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:46:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:46:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:46:24.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:46:23 smithi153 ceph-mon[70044]: pgmap v4145: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:46:24.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:46:23 smithi083 ceph-mon[77242]: pgmap v4145: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:46:24.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:46:23 smithi083 ceph-mon[79930]: pgmap v4145: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:46:24.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:46:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:46:24.034Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:46:25.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:46:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:46:24.731Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:46:26.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:46:25 smithi153 ceph-mon[70044]: pgmap v4146: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:46:26.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:46:25 smithi083 ceph-mon[79930]: pgmap v4146: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:46:26.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:46:25 smithi083 ceph-mon[77242]: pgmap v4146: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:46:28.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:46:27 smithi153 ceph-mon[70044]: pgmap v4147: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:46:28.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:46:27 smithi083 ceph-mon[79930]: pgmap v4147: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:46:28.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:46:27 smithi083 ceph-mon[77242]: pgmap v4147: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:46:30.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:46:29 smithi153 ceph-mon[70044]: pgmap v4148: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:46:30.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:46:29 smithi083 ceph-mon[79930]: pgmap v4148: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:46:30.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:46:29 smithi083 ceph-mon[77242]: pgmap v4148: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:46:30.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:46:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:46:30.641Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:46:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:46:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:46:31] "GET /metrics HTTP/1.1" 200 225206 "" "Prometheus/2.33.4" 2022-09-29T07:46:32.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:46:31 smithi153 ceph-mon[70044]: pgmap v4149: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:46:32.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:46:31 smithi083 ceph-mon[79930]: pgmap v4149: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:46:32.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:46:31 smithi083 ceph-mon[77242]: pgmap v4149: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:46:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:46:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:46:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:46:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:46:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:46:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:46:34.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:46:33 smithi153 ceph-mon[70044]: pgmap v4150: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:46:34.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:46:33 smithi083 ceph-mon[79930]: pgmap v4150: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:46:34.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:46:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:46:34.033Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:46:34.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:46:33 smithi083 ceph-mon[77242]: pgmap v4150: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:46:35.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:46:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:46:34.731Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:46:36.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:46:35 smithi153 ceph-mon[70044]: pgmap v4151: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:46:36.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:46:35 smithi083 ceph-mon[77242]: pgmap v4151: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:46:36.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:46:35 smithi083 ceph-mon[79930]: pgmap v4151: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:46:38.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:46:37 smithi153 ceph-mon[70044]: pgmap v4152: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:46:38.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:46:37 smithi083 ceph-mon[79930]: pgmap v4152: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:46:38.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:46:37 smithi083 ceph-mon[77242]: pgmap v4152: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:46:39.861 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:46:40.151 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:46:40.151 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 23s ago 2h 26.5M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:46:40.152 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 8m ago 2h 62.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:46:40.152 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 23s ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:46:40.152 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 8m ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:46:40.152 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 8m ago 2h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:46:40.153 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 23s ago 2h 635M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:46:40.153 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 23s ago 2h 566M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:46:40.153 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 8m ago 2h 235M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:46:40.153 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 23s ago 2h 330M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:46:40.153 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 23s ago 2h 26.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:46:40.154 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 8m ago 2h 27.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:46:40.154 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 23s ago 2h 412M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:46:40.154 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 23s ago 2h 403M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:46:40.154 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 23s ago 2h 319M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:46:40.154 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 23s ago 2h 390M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:46:40.155 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 8m ago 2h 327M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:46:40.155 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 8m ago 2h 400M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:46:40.155 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 8m ago 2h 355M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:46:40.155 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 8m ago 2h 415M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:46:40.155 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 8m ago 2h 121M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:46:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:46:39 smithi153 ceph-mon[70044]: pgmap v4153: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:46:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:46:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:46:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:46:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:46:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:46:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:46:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:46:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:46:40.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:46:39 smithi083 ceph-mon[79930]: pgmap v4153: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:46:40.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:46:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:46:40.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:46:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:46:40.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:46:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:46:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:46:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:46:40.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:46:39 smithi083 ceph-mon[77242]: pgmap v4153: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:46:40.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:46:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:46:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:46:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:46:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:46:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:46:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:46:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:46:40.508 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:46:40.508 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:46:40.508 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:46:40.508 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:46:40.509 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:46:40.509 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:46:40.509 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:46:40.509 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:46:40.509 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:46:40.510 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:46:40.510 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:46:40.510 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:46:40.510 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:46:40.511 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:46:40.511 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:46:40.511 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:46:40.511 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:46:40.512 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:46:40.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:46:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:46:40.641Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:46:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:46:41 smithi153 ceph-mon[70044]: from='client.38967 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:46:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:46:41 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/678534916' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:46:41.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:46:41 smithi083 ceph-mon[79930]: from='client.38967 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:46:41.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:46:41 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/678534916' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:46:41.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:46:41 smithi083 ceph-mon[77242]: from='client.38967 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:46:41.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:46:41 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/678534916' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:46:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:46:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:46:41] "GET /metrics HTTP/1.1" 200 225209 "" "Prometheus/2.33.4" 2022-09-29T07:46:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:46:42 smithi153 ceph-mon[70044]: pgmap v4154: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:46:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:46:42 smithi153 ceph-mon[70044]: from='client.48907 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:46:42.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:46:42 smithi083 ceph-mon[77242]: pgmap v4154: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:46:42.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:46:42 smithi083 ceph-mon[77242]: from='client.48907 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:46:42.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:46:42 smithi083 ceph-mon[79930]: pgmap v4154: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:46:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:46:42 smithi083 ceph-mon[79930]: from='client.48907 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:46:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:46:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:46:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:46:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:46:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:46:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:46:44.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:46:44 smithi153 ceph-mon[70044]: pgmap v4155: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:46:44.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:46:44 smithi083 ceph-mon[79930]: pgmap v4155: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:46:44.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:46:44 smithi083 ceph-mon[77242]: pgmap v4155: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:46:44.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:46:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:46:44.034Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:46:45.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:46:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:46:44.732Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:46:46.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:46:46 smithi153 ceph-mon[70044]: pgmap v4156: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:46:46.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:46:46 smithi083 ceph-mon[79930]: pgmap v4156: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:46:46.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:46:46 smithi083 ceph-mon[77242]: pgmap v4156: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:46:48.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:46:48 smithi153 ceph-mon[70044]: pgmap v4157: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:46:48.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:46:48 smithi083 ceph-mon[79930]: pgmap v4157: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:46:48.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:46:48 smithi083 ceph-mon[77242]: pgmap v4157: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:46:50.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:46:50 smithi153 ceph-mon[70044]: pgmap v4158: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:46:50.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:46:50 smithi083 ceph-mon[79930]: pgmap v4158: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:46:50.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:46:50 smithi083 ceph-mon[77242]: pgmap v4158: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:46:50.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:46:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:46:50.642Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:46:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:46:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:46:51] "GET /metrics HTTP/1.1" 200 225209 "" "Prometheus/2.33.4" 2022-09-29T07:46:52.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:46:52 smithi153 ceph-mon[70044]: pgmap v4159: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:46:52.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:46:52 smithi083 ceph-mon[79930]: pgmap v4159: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:46:52.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:46:52 smithi083 ceph-mon[77242]: pgmap v4159: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:46:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:46:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:46:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:46:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:46:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:46:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:46:54.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:46:54 smithi153 ceph-mon[70044]: pgmap v4160: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:46:54.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:46:54 smithi083 ceph-mon[79930]: pgmap v4160: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:46:54.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:46:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:46:54.037Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:46:54.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:46:54 smithi083 ceph-mon[77242]: pgmap v4160: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:46:55.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:46:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:46:54.733Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:46:56.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:46:56 smithi153 ceph-mon[70044]: pgmap v4161: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:46:56.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:46:56 smithi083 ceph-mon[79930]: pgmap v4161: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:46:56.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:46:56 smithi083 ceph-mon[77242]: pgmap v4161: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:46:58.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:46:58 smithi153 ceph-mon[70044]: pgmap v4162: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:46:58.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:46:58 smithi083 ceph-mon[79930]: pgmap v4162: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:46:58.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:46:58 smithi083 ceph-mon[77242]: pgmap v4162: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:47:00 smithi153 ceph-mon[70044]: pgmap v4163: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:00.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:47:00 smithi083 ceph-mon[79930]: pgmap v4163: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:00.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:47:00 smithi083 ceph-mon[77242]: pgmap v4163: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:00.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:47:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:47:00.642Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:47:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:47:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:47:01] "GET /metrics HTTP/1.1" 200 225206 "" "Prometheus/2.33.4" 2022-09-29T07:47:02.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:47:02 smithi153 ceph-mon[70044]: pgmap v4164: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:02.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:47:02 smithi083 ceph-mon[79930]: pgmap v4164: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:02.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:47:02 smithi083 ceph-mon[77242]: pgmap v4164: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:47:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:47:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:47:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:47:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:47:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:47:04.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:47:04 smithi153 ceph-mon[70044]: pgmap v4165: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:04.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:47:04 smithi083 ceph-mon[79930]: pgmap v4165: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:04.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:47:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:47:04.036Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:47:04.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:47:04 smithi083 ceph-mon[77242]: pgmap v4165: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:05.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:47:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:47:04.733Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:47:06.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:47:06 smithi153 ceph-mon[70044]: pgmap v4166: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:06.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:47:06 smithi083 ceph-mon[79930]: pgmap v4166: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:06.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:47:06 smithi083 ceph-mon[77242]: pgmap v4166: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:08.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:47:08 smithi153 ceph-mon[70044]: pgmap v4167: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:08.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:47:08 smithi083 ceph-mon[79930]: pgmap v4167: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:08.477 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:47:08 smithi083 ceph-mon[77242]: pgmap v4167: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:10.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:47:10 smithi153 ceph-mon[70044]: pgmap v4168: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:10.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:47:10 smithi083 ceph-mon[79930]: pgmap v4168: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:10.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:47:10 smithi083 ceph-mon[77242]: pgmap v4168: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:10.831 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:47:10.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:47:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:47:10.642Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:47:11.121 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:47:11.122 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 54s ago 2h 26.5M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:47:11.122 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 9m ago 2h 62.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:47:11.122 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 54s ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:47:11.122 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 9m ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:47:11.123 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 9m ago 2h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:47:11.123 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 54s ago 2h 635M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:47:11.123 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 54s ago 2h 566M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:47:11.123 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 9m ago 2h 235M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:47:11.124 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 54s ago 2h 330M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:47:11.124 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 54s ago 2h 26.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:47:11.124 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 9m ago 2h 27.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:47:11.124 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 54s ago 2h 412M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:47:11.124 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 54s ago 2h 403M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:47:11.125 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 54s ago 2h 319M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:47:11.125 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 54s ago 2h 390M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:47:11.125 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 9m ago 2h 327M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:47:11.125 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 9m ago 2h 400M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:47:11.126 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 9m ago 2h 355M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:47:11.126 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 9m ago 2h 415M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:47:11.126 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 9m ago 2h 121M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:47:11.478 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:47:11.479 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:47:11.479 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:47:11.479 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:47:11.480 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:47:11.480 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:47:11.480 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:47:11.480 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:47:11.481 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:47:11.481 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:47:11.481 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:47:11.481 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:47:11.481 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:47:11.482 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:47:11.482 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:47:11.482 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:47:11.482 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:47:11.482 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:47:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:47:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:47:11] "GET /metrics HTTP/1.1" 200 225204 "" "Prometheus/2.33.4" 2022-09-29T07:47:12.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:47:12 smithi153 ceph-mon[70044]: pgmap v4169: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:12.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:47:12 smithi153 ceph-mon[70044]: from='client.48919 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:47:12.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:47:12 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2691401047' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:47:12.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:47:12 smithi083 ceph-mon[79930]: pgmap v4169: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:12.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:47:12 smithi083 ceph-mon[79930]: from='client.48919 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:47:12.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:47:12 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2691401047' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:47:12.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:47:12 smithi083 ceph-mon[77242]: pgmap v4169: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:12.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:47:12 smithi083 ceph-mon[77242]: from='client.48919 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:47:12.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:47:12 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2691401047' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:47:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:47:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:47:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:47:13.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:47:13 smithi153 ceph-mon[70044]: from='client.48925 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:47:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:47:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:47:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:47:13.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:47:13 smithi083 ceph-mon[79930]: from='client.48925 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:47:13.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:47:13 smithi083 ceph-mon[77242]: from='client.48925 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:47:14.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:47:14 smithi153 ceph-mon[70044]: pgmap v4170: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:14.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:47:14 smithi083 ceph-mon[79930]: pgmap v4170: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:14.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:47:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:47:14.036Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:47:14.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:47:14 smithi083 ceph-mon[77242]: pgmap v4170: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:15.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:47:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:47:14.734Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:47:16.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:47:16 smithi153 ceph-mon[70044]: pgmap v4171: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:16.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:47:16 smithi083 ceph-mon[77242]: pgmap v4171: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:16.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:47:16 smithi083 ceph-mon[79930]: pgmap v4171: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:18.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:47:18 smithi083 ceph-mon[79930]: pgmap v4172: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:18.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:47:18 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:47:18.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:47:18 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:47:18.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:47:18 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:47:18.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:47:18 smithi083 ceph-mon[77242]: pgmap v4172: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:18.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:47:18 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:47:18.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:47:18 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:47:18.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:47:18 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:47:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:47:18 smithi153 ceph-mon[70044]: pgmap v4172: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:47:18 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:47:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:47:18 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:47:18.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:47:18 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:47:19.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:47:19 smithi083 ceph-mon[79930]: pgmap v4173: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:19.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:47:19 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:47:19.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:47:19 smithi083 ceph-mon[77242]: pgmap v4173: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:19.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:47:19 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:47:19.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:47:19 smithi153 ceph-mon[70044]: pgmap v4173: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:19.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:47:19 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:47:20.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:47:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:47:20.642Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:47:21.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:47:21 smithi083 ceph-mon[79930]: pgmap v4174: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:21.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:47:21 smithi083 ceph-mon[77242]: pgmap v4174: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:21.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:47:21 smithi153 ceph-mon[70044]: pgmap v4174: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:47:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:47:21] "GET /metrics HTTP/1.1" 200 225204 "" "Prometheus/2.33.4" 2022-09-29T07:47:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:47:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:47:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:47:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:47:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:47:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:47:23.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:47:23 smithi153 ceph-mon[70044]: pgmap v4175: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:23.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:47:23 smithi083 ceph-mon[79930]: pgmap v4175: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:23.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:47:23 smithi083 ceph-mon[77242]: pgmap v4175: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:24.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:47:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:47:24.039Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:47:25.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:47:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:47:24.735Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:47:25.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:47:25 smithi153 ceph-mon[70044]: pgmap v4176: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:25.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:47:25 smithi083 ceph-mon[77242]: pgmap v4176: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:25.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:47:25 smithi083 ceph-mon[79930]: pgmap v4176: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:27.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:47:27 smithi153 ceph-mon[70044]: pgmap v4177: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:27.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:47:27 smithi083 ceph-mon[79930]: pgmap v4177: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:27.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:47:27 smithi083 ceph-mon[77242]: pgmap v4177: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:29.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:47:29 smithi153 ceph-mon[70044]: pgmap v4178: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:29.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:47:29 smithi083 ceph-mon[79930]: pgmap v4178: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:29.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:47:29 smithi083 ceph-mon[77242]: pgmap v4178: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:30.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:47:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:47:30.643Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:47:31.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:47:31 smithi153 ceph-mon[70044]: pgmap v4179: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:47:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:47:31] "GET /metrics HTTP/1.1" 200 225197 "" "Prometheus/2.33.4" 2022-09-29T07:47:31.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:47:31 smithi083 ceph-mon[79930]: pgmap v4179: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:31.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:47:31 smithi083 ceph-mon[77242]: pgmap v4179: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:47:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:47:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:47:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:47:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:47:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:47:33.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:47:33 smithi153 ceph-mon[70044]: pgmap v4180: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:33.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:47:33 smithi083 ceph-mon[79930]: pgmap v4180: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:33.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:47:33 smithi083 ceph-mon[77242]: pgmap v4180: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:34.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:47:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:47:34.037Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:47:35.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:47:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:47:34.736Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:47:35.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:47:35 smithi153 ceph-mon[70044]: pgmap v4181: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:35.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:47:35 smithi083 ceph-mon[77242]: pgmap v4181: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:35.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:47:35 smithi083 ceph-mon[79930]: pgmap v4181: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:37.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:47:37 smithi153 ceph-mon[70044]: pgmap v4182: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:37.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:47:37 smithi083 ceph-mon[79930]: pgmap v4182: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:37.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:47:37 smithi083 ceph-mon[77242]: pgmap v4182: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:39.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:47:39 smithi153 ceph-mon[70044]: pgmap v4183: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:39.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:47:39 smithi083 ceph-mon[79930]: pgmap v4183: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:39.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:47:39 smithi083 ceph-mon[77242]: pgmap v4183: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:47:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:47:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:47:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:47:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:47:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:47:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:47:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:47:40.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:47:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:47:40.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:47:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:47:40.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:47:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:47:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:47:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:47:40.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:47:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:47:40.643Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:47:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:47:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:47:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:47:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:47:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:47:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:47:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:47:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:47:41.804 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:47:41.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:47:41 smithi153 ceph-mon[70044]: pgmap v4184: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:47:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:47:41] "GET /metrics HTTP/1.1" 200 225207 "" "Prometheus/2.33.4" 2022-09-29T07:47:41.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:47:41 smithi083 ceph-mon[79930]: pgmap v4184: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:41.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:47:41 smithi083 ceph-mon[77242]: pgmap v4184: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:42.096 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:47:42.097 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 85s ago 2h 26.5M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:47:42.097 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 9m ago 2h 62.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:47:42.097 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 85s ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:47:42.098 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 9m ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:47:42.098 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 9m ago 2h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:47:42.098 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 85s ago 2h 635M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:47:42.098 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 85s ago 2h 566M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:47:42.099 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 9m ago 2h 235M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:47:42.099 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 85s ago 2h 330M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:47:42.099 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 85s ago 2h 26.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:47:42.099 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 9m ago 2h 27.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:47:42.099 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 85s ago 2h 412M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:47:42.100 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 85s ago 2h 403M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:47:42.100 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 85s ago 2h 319M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:47:42.100 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 85s ago 2h 390M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:47:42.100 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 9m ago 2h 327M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:47:42.101 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 9m ago 2h 400M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:47:42.101 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 9m ago 2h 355M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:47:42.101 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 9m ago 2h 415M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:47:42.101 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 9m ago 2h 121M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:47:42.449 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:47:42.450 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:47:42.450 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:47:42.450 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:47:42.450 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:47:42.451 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:47:42.451 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:47:42.451 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:47:42.451 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:47:42.452 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:47:42.452 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:47:42.452 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:47:42.452 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:47:42.453 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:47:42.453 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:47:42.453 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:47:42.453 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:47:42.453 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:47:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:47:42 smithi153 ceph-mon[70044]: from='client.39003 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:47:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:47:42 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1390359196' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:47:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:47:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:47:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:47:42.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:47:42 smithi083 ceph-mon[79930]: from='client.39003 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:47:42.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:47:42 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1390359196' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:47:42.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:47:42 smithi083 ceph-mon[77242]: from='client.39003 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:47:42.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:47:42 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1390359196' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:47:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:47:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:47:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:47:43.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:47:43 smithi153 ceph-mon[70044]: from='client.48943 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:47:43.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:47:43 smithi153 ceph-mon[70044]: pgmap v4185: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:43.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:47:43 smithi083 ceph-mon[79930]: from='client.48943 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:47:43.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:47:43 smithi083 ceph-mon[79930]: pgmap v4185: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:43.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:47:43 smithi083 ceph-mon[77242]: from='client.48943 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:47:43.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:47:43 smithi083 ceph-mon[77242]: pgmap v4185: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:44.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:47:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:47:44.038Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:47:45.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:47:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:47:44.737Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:47:45.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:47:45 smithi153 ceph-mon[70044]: pgmap v4186: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:45.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:47:45 smithi083 ceph-mon[77242]: pgmap v4186: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:45.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:47:45 smithi083 ceph-mon[79930]: pgmap v4186: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:47.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:47:47 smithi153 ceph-mon[70044]: pgmap v4187: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:47.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:47:47 smithi083 ceph-mon[77242]: pgmap v4187: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:47.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:47:47 smithi083 ceph-mon[79930]: pgmap v4187: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:49.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:47:49 smithi153 ceph-mon[70044]: pgmap v4188: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:49.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:47:49 smithi083 ceph-mon[77242]: pgmap v4188: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:49.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:47:49 smithi083 ceph-mon[79930]: pgmap v4188: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:50.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:47:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:47:50.644Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:47:51.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:47:51 smithi153 ceph-mon[70044]: pgmap v4189: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:51.960 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:47:51 smithi083 ceph-mon[77242]: pgmap v4189: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:47:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:47:51] "GET /metrics HTTP/1.1" 200 225207 "" "Prometheus/2.33.4" 2022-09-29T07:47:51.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:47:51 smithi083 ceph-mon[79930]: pgmap v4189: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:47:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:47:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:47:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:47:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:47:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:47:53.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:47:53 smithi153 ceph-mon[70044]: pgmap v4190: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:53.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:47:53 smithi083 ceph-mon[77242]: pgmap v4190: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:53.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:47:53 smithi083 ceph-mon[79930]: pgmap v4190: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:54.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:47:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:47:54.040Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:47:55.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:47:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:47:54.738Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:47:55.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:47:55 smithi153 ceph-mon[70044]: pgmap v4191: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:55.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:47:55 smithi083 ceph-mon[79930]: pgmap v4191: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:55.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:47:55 smithi083 ceph-mon[77242]: pgmap v4191: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:57.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:47:57 smithi153 ceph-mon[70044]: pgmap v4192: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:57.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:47:57 smithi083 ceph-mon[79930]: pgmap v4192: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:57.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:47:57 smithi083 ceph-mon[77242]: pgmap v4192: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:59.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:47:59 smithi153 ceph-mon[70044]: pgmap v4193: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:59.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:47:59 smithi083 ceph-mon[79930]: pgmap v4193: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:47:59.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:47:59 smithi083 ceph-mon[77242]: pgmap v4193: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:48:00.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:48:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:48:00.645Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:48:01.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:48:01 smithi153 ceph-mon[70044]: pgmap v4194: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:48:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:48:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:48:01] "GET /metrics HTTP/1.1" 200 225204 "" "Prometheus/2.33.4" 2022-09-29T07:48:01.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:48:01 smithi083 ceph-mon[79930]: pgmap v4194: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:48:01.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:48:01 smithi083 ceph-mon[77242]: pgmap v4194: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:48:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:48:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:48:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:48:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:48:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:48:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:48:03.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:48:03 smithi153 ceph-mon[70044]: pgmap v4195: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:48:03.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:48:03 smithi083 ceph-mon[77242]: pgmap v4195: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:48:03.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:48:03 smithi083 ceph-mon[79930]: pgmap v4195: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:48:04.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:48:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:48:04.040Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:48:05.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:48:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:48:04.739Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:48:05.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:48:05 smithi153 ceph-mon[70044]: pgmap v4196: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:48:05.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:48:05 smithi083 ceph-mon[79930]: pgmap v4196: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:48:05.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:48:05 smithi083 ceph-mon[77242]: pgmap v4196: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:48:07.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:48:07 smithi083 ceph-mon[79930]: pgmap v4197: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:48:07.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:48:07 smithi083 ceph-mon[77242]: pgmap v4197: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:48:08.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:48:07 smithi153 ceph-mon[70044]: pgmap v4197: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:48:09.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:48:09 smithi083 ceph-mon[77242]: pgmap v4198: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:48:09.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:48:09 smithi083 ceph-mon[79930]: pgmap v4198: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:48:10.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:48:09 smithi153 ceph-mon[70044]: pgmap v4198: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:48:10.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:48:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:48:10.646Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:48:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:48:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:48:11] "GET /metrics HTTP/1.1" 200 225203 "" "Prometheus/2.33.4" 2022-09-29T07:48:11.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:48:11 smithi083 ceph-mon[79930]: pgmap v4199: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:48:11.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:48:11 smithi083 ceph-mon[77242]: pgmap v4199: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:48:12.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:48:11 smithi153 ceph-mon[70044]: pgmap v4199: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:48:12.770 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:48:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:48:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:48:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:48:13.058 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:48:13.058 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 116s ago 2h 26.5M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:48:13.058 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 10m ago 2h 62.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:48:13.059 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 116s ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:48:13.059 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 10m ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:48:13.059 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 10m ago 2h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:48:13.059 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 116s ago 2h 635M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:48:13.059 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 116s ago 2h 566M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:48:13.060 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 10m ago 2h 235M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:48:13.060 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 116s ago 2h 330M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:48:13.060 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 116s ago 2h 26.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:48:13.060 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 10m ago 2h 27.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:48:13.061 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 116s ago 2h 412M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:48:13.061 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 116s ago 2h 403M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:48:13.061 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 116s ago 2h 319M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:48:13.061 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 116s ago 2h 390M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:48:13.061 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 10m ago 2h 327M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:48:13.062 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 10m ago 2h 400M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:48:13.062 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 10m ago 2h 355M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:48:13.062 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 10m ago 2h 415M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:48:13.063 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 10m ago 2h 121M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:48:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:48:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:48:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:48:13.426 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:48:13.426 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:48:13.427 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:48:13.427 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:48:13.427 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:48:13.427 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:48:13.428 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:48:13.428 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:48:13.428 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:48:13.428 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:48:13.429 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:48:13.429 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:48:13.429 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:48:13.429 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:48:13.429 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:48:13.430 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:48:13.430 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:48:13.430 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:48:13.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:48:13 smithi083 ceph-mon[79930]: pgmap v4200: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:48:13.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:48:13 smithi083 ceph-mon[79930]: from='client.48955 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:48:13.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:48:13 smithi083 ceph-mon[79930]: from='client.39027 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:48:13.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:48:13 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1189512342' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:48:13.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:48:13 smithi083 ceph-mon[77242]: pgmap v4200: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:48:13.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:48:13 smithi083 ceph-mon[77242]: from='client.48955 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:48:13.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:48:13 smithi083 ceph-mon[77242]: from='client.39027 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:48:13.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:48:13 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1189512342' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:48:14.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:48:13 smithi153 ceph-mon[70044]: pgmap v4200: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:48:14.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:48:13 smithi153 ceph-mon[70044]: from='client.48955 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:48:14.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:48:13 smithi153 ceph-mon[70044]: from='client.39027 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:48:14.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:48:13 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1189512342' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:48:14.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:48:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:48:14.039Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:48:15.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:48:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:48:14.740Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:48:15.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:48:15 smithi083 ceph-mon[77242]: pgmap v4201: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:48:15.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:48:15 smithi083 ceph-mon[79930]: pgmap v4201: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:48:16.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:48:15 smithi153 ceph-mon[70044]: pgmap v4201: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:48:17.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:48:17 smithi083 ceph-mon[79930]: pgmap v4202: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:48:17.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:48:17 smithi083 ceph-mon[77242]: pgmap v4202: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:48:18.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:48:17 smithi153 ceph-mon[70044]: pgmap v4202: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:48:18.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:48:18 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:48:18.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:48:18 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:48:18.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:48:18 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:48:18.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:48:18 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:48:18.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:48:18 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:48:18.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:48:18 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:48:19.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:48:18 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:48:19.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:48:18 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:48:19.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:48:18 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:48:19.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:48:19 smithi083 ceph-mon[79930]: pgmap v4203: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:48:19.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:48:19 smithi083 ceph-mon[77242]: pgmap v4203: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:48:20.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:48:19 smithi153 ceph-mon[70044]: pgmap v4203: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:48:20.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:48:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:48:20.646Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:48:21.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:48:21 smithi083 ceph-mon[79930]: pgmap v4204: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:48:21.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:48:21 smithi083 ceph-mon[77242]: pgmap v4204: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:48:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:48:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:48:21] "GET /metrics HTTP/1.1" 200 225203 "" "Prometheus/2.33.4" 2022-09-29T07:48:22.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:48:21 smithi153 ceph-mon[70044]: pgmap v4204: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:48:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:48:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:48:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:48:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:48:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:48:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:48:24.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:48:23 smithi153 ceph-mon[70044]: pgmap v4205: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:48:24.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:48:23 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:48:24.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:48:23 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:48:24.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:48:23 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:48:24.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:48:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:48:24.042Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:48:24.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:48:23 smithi083 ceph-mon[77242]: pgmap v4205: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:48:24.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:48:23 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:48:24.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:48:23 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:48:24.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:48:23 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:48:24.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:48:23 smithi083 ceph-mon[79930]: pgmap v4205: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:48:24.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:48:23 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:48:24.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:48:23 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:48:24.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:48:23 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:48:25.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:48:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:48:24.741Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:48:26.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:48:25 smithi153 ceph-mon[70044]: pgmap v4206: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:48:26.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:48:25 smithi083 ceph-mon[79930]: pgmap v4206: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:48:26.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:48:25 smithi083 ceph-mon[77242]: pgmap v4206: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:48:28.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:48:27 smithi153 ceph-mon[70044]: pgmap v4207: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:48:28.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:48:27 smithi083 ceph-mon[79930]: pgmap v4207: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:48:28.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:48:27 smithi083 ceph-mon[77242]: pgmap v4207: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:48:30.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:48:30 smithi153 ceph-mon[70044]: pgmap v4208: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:48:30.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:48:30 smithi083 ceph-mon[79930]: pgmap v4208: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:48:30.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:48:30 smithi083 ceph-mon[77242]: pgmap v4208: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:48:30.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:48:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:48:30.647Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:48:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:48:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:48:31] "GET /metrics HTTP/1.1" 200 225206 "" "Prometheus/2.33.4" 2022-09-29T07:48:32.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:48:32 smithi153 ceph-mon[70044]: pgmap v4209: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:48:32.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:48:32 smithi083 ceph-mon[79930]: pgmap v4209: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:48:32.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:48:32 smithi083 ceph-mon[77242]: pgmap v4209: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:48:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:48:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:48:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:48:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:48:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:48:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:48:34.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:48:34 smithi153 ceph-mon[70044]: pgmap v4210: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:48:34.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:48:34 smithi083 ceph-mon[79930]: pgmap v4210: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:48:34.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:48:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:48:34.041Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:48:34.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:48:34 smithi083 ceph-mon[77242]: pgmap v4210: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:48:35.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:48:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:48:34.741Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:48:36.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:48:36 smithi153 ceph-mon[70044]: pgmap v4211: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:48:36.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:48:36 smithi083 ceph-mon[79930]: pgmap v4211: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:48:36.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:48:36 smithi083 ceph-mon[77242]: pgmap v4211: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:48:38.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:48:38 smithi153 ceph-mon[70044]: pgmap v4212: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:48:38.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:48:38 smithi083 ceph-mon[79930]: pgmap v4212: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:48:38.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:48:38 smithi083 ceph-mon[77242]: pgmap v4212: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:48:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:48:40 smithi153 ceph-mon[70044]: pgmap v4213: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:48:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:48:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:48:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:48:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:48:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:48:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:48:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:48:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:48:40.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:48:40 smithi083 ceph-mon[77242]: pgmap v4213: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:48:40.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:48:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:48:40.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:48:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:48:40.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:48:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:48:40.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:48:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:48:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:48:40 smithi083 ceph-mon[79930]: pgmap v4213: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:48:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:48:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:48:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:48:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:48:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:48:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:48:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:48:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:48:40.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:48:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:48:40.648Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:48:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:48:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:48:41] "GET /metrics HTTP/1.1" 200 225209 "" "Prometheus/2.33.4" 2022-09-29T07:48:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:48:42 smithi153 ceph-mon[70044]: pgmap v4214: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:48:42.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:48:42 smithi083 ceph-mon[79930]: pgmap v4214: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:48:42.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:48:42 smithi083 ceph-mon[77242]: pgmap v4214: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:48:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:48:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:48:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:48:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:48:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:48:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:48:43.753 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:48:44.042 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:48:44.042 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 2m ago 2h 26.5M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:48:44.042 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 21s ago 2h 62.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:48:44.043 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 2m ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:48:44.043 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 21s ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:48:44.043 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 21s ago 2h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:48:44.043 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 2m ago 2h 635M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:48:44.044 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 2m ago 2h 566M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:48:44.044 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 21s ago 2h 252M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:48:44.044 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 2m ago 2h 330M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:48:44.044 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 2m ago 2h 26.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:48:44.044 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 21s ago 2h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:48:44.045 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 2m ago 2h 412M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:48:44.045 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 2m ago 2h 403M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:48:44.045 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 2m ago 2h 319M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:48:44.045 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 2m ago 2h 390M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:48:44.046 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 21s ago 2h 349M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:48:44.046 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 21s ago 2h 427M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:48:44.046 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 21s ago 2h 380M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:48:44.046 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 21s ago 2h 444M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:48:44.046 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 21s ago 2h 125M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:48:44.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:48:44 smithi153 ceph-mon[70044]: pgmap v4215: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:48:44.399 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:48:44.399 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:48:44.399 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:48:44.400 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:48:44.400 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:48:44.400 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:48:44.400 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:48:44.400 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:48:44.401 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:48:44.401 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:48:44.401 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:48:44.401 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:48:44.401 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:48:44.402 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:48:44.402 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:48:44.402 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:48:44.402 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:48:44.403 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:48:44.460 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:48:44 smithi083 ceph-mon[77242]: pgmap v4215: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:48:44.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:48:44 smithi083 ceph-mon[79930]: pgmap v4215: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:48:44.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:48:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:48:44.044Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:48:45.024 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:48:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:48:44.741Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:48:45.024 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:48:45 smithi083 ceph-mon[77242]: from='client.48973 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:48:45.025 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:48:45 smithi083 ceph-mon[77242]: from='client.48979 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:48:45.025 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:48:45 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2788127286' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:48:45.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:48:45 smithi153 ceph-mon[70044]: from='client.48973 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:48:45.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:48:45 smithi153 ceph-mon[70044]: from='client.48979 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:48:45.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:48:45 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2788127286' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:48:45.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:48:45 smithi083 ceph-mon[79930]: from='client.48973 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:48:45.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:48:45 smithi083 ceph-mon[79930]: from='client.48979 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:48:45.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:48:45 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2788127286' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:48:46.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:48:46 smithi153 ceph-mon[70044]: pgmap v4216: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:48:46.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:48:46 smithi083 ceph-mon[77242]: pgmap v4216: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:48:46.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:48:46 smithi083 ceph-mon[79930]: pgmap v4216: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:48:48.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:48:48 smithi153 ceph-mon[70044]: pgmap v4217: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:48:48.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:48:48 smithi083 ceph-mon[79930]: pgmap v4217: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:48:48.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:48:48 smithi083 ceph-mon[77242]: pgmap v4217: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:48:50.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:48:50 smithi153 ceph-mon[70044]: pgmap v4218: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:48:50.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:48:50 smithi083 ceph-mon[79930]: pgmap v4218: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:48:50.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:48:50 smithi083 ceph-mon[77242]: pgmap v4218: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:48:50.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:48:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:48:50.649Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:48:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:48:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:48:51] "GET /metrics HTTP/1.1" 200 225209 "" "Prometheus/2.33.4" 2022-09-29T07:48:52.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:48:52 smithi153 ceph-mon[70044]: pgmap v4219: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:48:52.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:48:52 smithi083 ceph-mon[79930]: pgmap v4219: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:48:52.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:48:52 smithi083 ceph-mon[77242]: pgmap v4219: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:48:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:48:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:48:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:48:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:48:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:48:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:48:54.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:48:54 smithi153 ceph-mon[70044]: pgmap v4220: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:48:54.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:48:54 smithi083 ceph-mon[79930]: pgmap v4220: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:48:54.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:48:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:48:54.046Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:48:54.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:48:54 smithi083 ceph-mon[77242]: pgmap v4220: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:48:55.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:48:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:48:54.742Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:48:56.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:48:56 smithi153 ceph-mon[70044]: pgmap v4221: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:48:56.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:48:56 smithi083 ceph-mon[79930]: pgmap v4221: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:48:56.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:48:56 smithi083 ceph-mon[77242]: pgmap v4221: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:48:58.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:48:58 smithi153 ceph-mon[70044]: pgmap v4222: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:48:58.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:48:58 smithi083 ceph-mon[79930]: pgmap v4222: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:48:58.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:48:58 smithi083 ceph-mon[77242]: pgmap v4222: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:49:00 smithi153 ceph-mon[70044]: pgmap v4223: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:00.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:49:00 smithi083 ceph-mon[79930]: pgmap v4223: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:00.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:49:00 smithi083 ceph-mon[77242]: pgmap v4223: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:00.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:49:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:49:00.649Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:49:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:49:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:49:01] "GET /metrics HTTP/1.1" 200 225207 "" "Prometheus/2.33.4" 2022-09-29T07:49:02.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:49:02 smithi153 ceph-mon[70044]: pgmap v4224: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:02.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:49:02 smithi083 ceph-mon[77242]: pgmap v4224: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:02.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:49:02 smithi083 ceph-mon[79930]: pgmap v4224: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:49:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:49:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:49:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:49:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:49:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:49:04.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:49:04 smithi153 ceph-mon[70044]: pgmap v4225: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:04.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:49:04 smithi083 ceph-mon[79930]: pgmap v4225: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:04.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:49:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:49:04.043Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:49:04.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:49:04 smithi083 ceph-mon[77242]: pgmap v4225: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:05.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:49:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:49:04.743Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:49:06.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:49:06 smithi153 ceph-mon[70044]: pgmap v4226: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:06.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:49:06 smithi083 ceph-mon[77242]: pgmap v4226: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:06.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:49:06 smithi083 ceph-mon[79930]: pgmap v4226: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:08.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:49:08 smithi153 ceph-mon[70044]: pgmap v4227: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:08.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:49:08 smithi083 ceph-mon[79930]: pgmap v4227: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:08.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:49:08 smithi083 ceph-mon[77242]: pgmap v4227: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:10.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:49:10 smithi083 ceph-mon[79930]: pgmap v4228: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:10.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:49:10 smithi083 ceph-mon[77242]: pgmap v4228: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:49:10 smithi153 ceph-mon[70044]: pgmap v4228: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:10.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:49:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:49:10.650Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:49:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:49:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:49:11] "GET /metrics HTTP/1.1" 200 225208 "" "Prometheus/2.33.4" 2022-09-29T07:49:12.315 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:49:12 smithi153 ceph-mon[70044]: pgmap v4229: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:12.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:49:12 smithi083 ceph-mon[79930]: pgmap v4229: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:12.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:49:12 smithi083 ceph-mon[77242]: pgmap v4229: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:12.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:49:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:49:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:49:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:49:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:49:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:49:14.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:49:14 smithi083 ceph-mon[77242]: pgmap v4230: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:14.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:49:14 smithi083 ceph-mon[79930]: pgmap v4230: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:14.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:49:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:49:14.044Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:49:14.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:49:14 smithi153 ceph-mon[70044]: pgmap v4230: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:14.720 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:49:15.007 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:49:15.008 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 2m ago 2h 26.5M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:49:15.008 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 52s ago 2h 62.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:49:15.008 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 2m ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:49:15.008 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 52s ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:49:15.009 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 52s ago 2h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:49:15.009 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 2m ago 2h 635M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:49:15.009 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 2m ago 2h 566M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:49:15.009 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 52s ago 2h 252M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:49:15.010 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 2m ago 2h 330M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:49:15.010 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 2m ago 2h 26.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:49:15.010 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 52s ago 2h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:49:15.010 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 2m ago 2h 412M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:49:15.010 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 2m ago 2h 403M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:49:15.011 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 2m ago 2h 319M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:49:15.011 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 2m ago 2h 390M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:49:15.011 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 52s ago 2h 349M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:49:15.011 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 52s ago 2h 427M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:49:15.011 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 52s ago 2h 380M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:49:15.012 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 52s ago 2h 444M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:49:15.012 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 52s ago 2h 125M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:49:15.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:49:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:49:14.744Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:49:15.362 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:49:15.362 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:49:15.363 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:49:15.363 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:49:15.363 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:49:15.363 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:49:15.364 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:49:15.364 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:49:15.364 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:49:15.364 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:49:15.365 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:49:15.365 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:49:15.365 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:49:15.365 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:49:15.366 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:49:15.366 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:49:15.366 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:49:15.366 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:49:16.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:49:16 smithi083 ceph-mon[79930]: pgmap v4231: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:16.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:49:16 smithi083 ceph-mon[79930]: from='client.48991 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:49:16.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:49:16 smithi083 ceph-mon[79930]: from='client.48997 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:49:16.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:49:16 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/811724459' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:49:16.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:49:16 smithi083 ceph-mon[77242]: pgmap v4231: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:16.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:49:16 smithi083 ceph-mon[77242]: from='client.48991 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:49:16.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:49:16 smithi083 ceph-mon[77242]: from='client.48997 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:49:16.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:49:16 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/811724459' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:49:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:49:16 smithi153 ceph-mon[70044]: pgmap v4231: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:49:16 smithi153 ceph-mon[70044]: from='client.48991 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:49:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:49:16 smithi153 ceph-mon[70044]: from='client.48997 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:49:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:49:16 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/811724459' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:49:18.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:49:18 smithi083 ceph-mon[79930]: pgmap v4232: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:18.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:49:18 smithi083 ceph-mon[77242]: pgmap v4232: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:49:18 smithi153 ceph-mon[70044]: pgmap v4232: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:20.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:49:20 smithi083 ceph-mon[79930]: pgmap v4233: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:20.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:49:20 smithi083 ceph-mon[77242]: pgmap v4233: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:20.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:49:20 smithi153 ceph-mon[70044]: pgmap v4233: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:20.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:49:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:49:20.651Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:49:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:49:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:49:21] "GET /metrics HTTP/1.1" 200 225208 "" "Prometheus/2.33.4" 2022-09-29T07:49:22.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:49:22 smithi083 ceph-mon[77242]: pgmap v4234: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:22.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:49:22 smithi083 ceph-mon[79930]: pgmap v4234: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:49:22 smithi153 ceph-mon[70044]: pgmap v4234: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:22.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:49:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:49:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:49:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:49:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:49:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:49:24.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:49:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:49:24.046Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:49:24.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:49:24 smithi083 ceph-mon[79930]: pgmap v4235: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:24.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:49:24 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:49:24.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:49:24 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:49:24.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:49:24 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:49:24.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:49:24 smithi083 ceph-mon[77242]: pgmap v4235: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:24.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:49:24 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:49:24.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:49:24 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:49:24.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:49:24 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:49:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:49:24 smithi153 ceph-mon[70044]: pgmap v4235: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:49:24 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:49:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:49:24 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:49:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:49:24 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:49:25.209 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:49:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:49:24.743Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:49:25.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:49:25 smithi083 ceph-mon[77242]: pgmap v4236: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:25.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:49:25 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:49:25.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:49:25 smithi083 ceph-mon[79930]: pgmap v4236: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:25.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:49:25 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:49:25.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:49:25 smithi153 ceph-mon[70044]: pgmap v4236: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:25.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:49:25 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:49:27.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:49:27 smithi153 ceph-mon[70044]: pgmap v4237: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:27.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:49:27 smithi083 ceph-mon[79930]: pgmap v4237: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:27.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:49:27 smithi083 ceph-mon[77242]: pgmap v4237: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:29.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:49:29 smithi153 ceph-mon[70044]: pgmap v4238: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:29.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:49:29 smithi083 ceph-mon[79930]: pgmap v4238: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:29.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:49:29 smithi083 ceph-mon[77242]: pgmap v4238: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:30.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:49:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:49:30.651Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:49:31.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:49:31 smithi153 ceph-mon[70044]: pgmap v4239: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:49:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:49:31] "GET /metrics HTTP/1.1" 200 225208 "" "Prometheus/2.33.4" 2022-09-29T07:49:31.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:49:31 smithi083 ceph-mon[79930]: pgmap v4239: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:31.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:49:31 smithi083 ceph-mon[77242]: pgmap v4239: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:49:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:49:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:49:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:49:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:49:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:49:33.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:49:33 smithi153 ceph-mon[70044]: pgmap v4240: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:33.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:49:33 smithi083 ceph-mon[79930]: pgmap v4240: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:33.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:49:33 smithi083 ceph-mon[77242]: pgmap v4240: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:34.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:49:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:49:34.046Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:49:35.210 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:49:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:49:34.745Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:49:35.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:49:35 smithi153 ceph-mon[70044]: pgmap v4241: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:35.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:49:35 smithi083 ceph-mon[79930]: pgmap v4241: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:35.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:49:35 smithi083 ceph-mon[77242]: pgmap v4241: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:37.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:49:37 smithi153 ceph-mon[70044]: pgmap v4242: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:37.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:49:37 smithi083 ceph-mon[79930]: pgmap v4242: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:37.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:49:37 smithi083 ceph-mon[77242]: pgmap v4242: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:39.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:49:39 smithi153 ceph-mon[70044]: pgmap v4243: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:39.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:49:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:49:39.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:49:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:49:39.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:49:39 smithi083 ceph-mon[79930]: pgmap v4243: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:39.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:49:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:49:39.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:49:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:49:39.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:49:39 smithi083 ceph-mon[77242]: pgmap v4243: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:39.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:49:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:49:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:49:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:49:40.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:49:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:49:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:49:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:49:40.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:49:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:49:40.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:49:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:49:40.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:49:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:49:40.652Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:49:40.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:49:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:49:40.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:49:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:49:41.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:49:41 smithi153 ceph-mon[70044]: pgmap v4244: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:49:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:49:41] "GET /metrics HTTP/1.1" 200 225207 "" "Prometheus/2.33.4" 2022-09-29T07:49:41.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:49:41 smithi083 ceph-mon[79930]: pgmap v4244: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:41.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:49:41 smithi083 ceph-mon[77242]: pgmap v4244: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:49:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:49:42.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:49:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:49:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:49:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:49:43.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:49:43 smithi153 ceph-mon[70044]: pgmap v4245: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:43.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:49:43 smithi083 ceph-mon[79930]: pgmap v4245: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:43.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:49:43 smithi083 ceph-mon[77242]: pgmap v4245: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:44.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:49:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:49:44.046Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:49:45.216 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:49:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:49:44.746Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:49:45.691 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:49:45.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:49:45 smithi153 ceph-mon[70044]: pgmap v4246: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:45.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:49:45 smithi083 ceph-mon[79930]: pgmap v4246: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:45.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:49:45 smithi083 ceph-mon[77242]: pgmap v4246: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:45.980 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:49:45.980 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 3m ago 2h 26.5M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:49:45.980 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 82s ago 2h 62.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:49:45.980 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 3m ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:49:45.981 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 82s ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:49:45.981 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 82s ago 2h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:49:45.981 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 3m ago 2h 635M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:49:45.981 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 3m ago 2h 566M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:49:45.982 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 82s ago 2h 252M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:49:45.982 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 3m ago 2h 330M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:49:45.982 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 3m ago 2h 26.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:49:45.982 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 82s ago 2h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:49:45.983 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 3m ago 2h 412M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:49:45.983 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 3m ago 2h 403M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:49:45.983 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 3m ago 2h 319M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:49:45.983 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 3m ago 2h 390M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:49:45.983 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 82s ago 2h 349M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:49:45.984 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 82s ago 2h 427M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:49:45.984 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 82s ago 2h 380M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:49:45.984 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 82s ago 2h 444M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:49:45.984 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 82s ago 2h 125M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:49:46.340 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:49:46.340 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:49:46.340 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:49:46.341 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:49:46.341 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:49:46.341 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:49:46.341 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:49:46.342 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:49:46.342 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:49:46.342 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:49:46.342 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:49:46.343 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:49:46.343 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:49:46.343 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:49:46.343 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:49:46.344 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:49:46.344 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:49:46.344 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:49:46.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:49:46 smithi153 ceph-mon[70044]: from='client.39075 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:49:46.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:49:46 smithi153 ceph-mon[70044]: from='client.39081 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:49:46.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:49:46 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/671053334' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:49:46.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:49:46 smithi083 ceph-mon[77242]: from='client.39075 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:49:46.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:49:46 smithi083 ceph-mon[77242]: from='client.39081 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:49:46.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:49:46 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/671053334' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:49:46.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:49:46 smithi083 ceph-mon[79930]: from='client.39075 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:49:46.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:49:46 smithi083 ceph-mon[79930]: from='client.39081 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:49:46.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:49:46 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/671053334' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:49:47.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:49:47 smithi153 ceph-mon[70044]: pgmap v4247: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:47.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:49:47 smithi083 ceph-mon[79930]: pgmap v4247: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:47.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:49:47 smithi083 ceph-mon[77242]: pgmap v4247: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:49.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:49:49 smithi153 ceph-mon[70044]: pgmap v4248: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:49.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:49:49 smithi083 ceph-mon[79930]: pgmap v4248: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:49.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:49:49 smithi083 ceph-mon[77242]: pgmap v4248: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:50.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:49:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:49:50.652Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:49:51.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:49:51 smithi153 ceph-mon[70044]: pgmap v4249: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:51.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:49:51 smithi083 ceph-mon[77242]: pgmap v4249: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:49:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:49:51] "GET /metrics HTTP/1.1" 200 225207 "" "Prometheus/2.33.4" 2022-09-29T07:49:51.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:49:51 smithi083 ceph-mon[79930]: pgmap v4249: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:52.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:49:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:49:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:49:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:49:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:49:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:49:53.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:49:53 smithi153 ceph-mon[70044]: pgmap v4250: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:53.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:49:53 smithi083 ceph-mon[79930]: pgmap v4250: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:53.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:49:53 smithi083 ceph-mon[77242]: pgmap v4250: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:54.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:49:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:49:54.048Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:49:55.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:49:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:49:54.747Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:49:55.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:49:55 smithi083 ceph-mon[79930]: pgmap v4251: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:55.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:49:55 smithi083 ceph-mon[77242]: pgmap v4251: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:56.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:49:55 smithi153 ceph-mon[70044]: pgmap v4251: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:57.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:49:57 smithi083 ceph-mon[79930]: pgmap v4252: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:57.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:49:57 smithi083 ceph-mon[77242]: pgmap v4252: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:58.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:49:57 smithi153 ceph-mon[70044]: pgmap v4252: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:59.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:49:59 smithi083 ceph-mon[79930]: pgmap v4253: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:49:59.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:49:59 smithi083 ceph-mon[77242]: pgmap v4253: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:50:00.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:49:59 smithi153 ceph-mon[70044]: pgmap v4253: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:50:00.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:50:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[77219]: 2022-09-29T07:49:59.999+0000 7f160f665700 -1 log_channel(cluster) log [ERR] : overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T07:50:00.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:50:00 smithi083 ceph-mon[77242]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T07:50:00.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:50:00 smithi083 ceph-mon[79930]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T07:50:00.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:50:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:50:00.653Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:50:01.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:50:00 smithi153 ceph-mon[70044]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T07:50:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:50:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:50:01] "GET /metrics HTTP/1.1" 200 225205 "" "Prometheus/2.33.4" 2022-09-29T07:50:01.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:50:01 smithi083 ceph-mon[79930]: pgmap v4254: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:50:01.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:50:01 smithi083 ceph-mon[77242]: pgmap v4254: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:50:02.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:50:01 smithi153 ceph-mon[70044]: pgmap v4254: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:50:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:50:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:50:02.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:50:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:50:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:50:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:50:03.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:50:03 smithi083 ceph-mon[77242]: pgmap v4255: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:50:03.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:50:03 smithi083 ceph-mon[79930]: pgmap v4255: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:50:04.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:50:03 smithi153 ceph-mon[70044]: pgmap v4255: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:50:04.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:50:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:50:04.046Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:50:05.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:50:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:50:04.748Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:50:05.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:50:05 smithi083 ceph-mon[79930]: pgmap v4256: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:50:05.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:50:05 smithi083 ceph-mon[77242]: pgmap v4256: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:50:06.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:50:05 smithi153 ceph-mon[70044]: pgmap v4256: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:50:07.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:50:07 smithi083 ceph-mon[79930]: pgmap v4257: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:50:07.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:50:07 smithi083 ceph-mon[77242]: pgmap v4257: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:50:08.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:50:07 smithi153 ceph-mon[70044]: pgmap v4257: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:50:09.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:50:09 smithi083 ceph-mon[79930]: pgmap v4258: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:50:09.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:50:09 smithi083 ceph-mon[77242]: pgmap v4258: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:50:10.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:50:09 smithi153 ceph-mon[70044]: pgmap v4258: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:50:10.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:50:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:50:10.654Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:50:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:50:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:50:11] "GET /metrics HTTP/1.1" 200 225209 "" "Prometheus/2.33.4" 2022-09-29T07:50:11.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:50:11 smithi083 ceph-mon[79930]: pgmap v4259: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:50:11.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:50:11 smithi083 ceph-mon[77242]: pgmap v4259: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:50:12.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:50:11 smithi153 ceph-mon[70044]: pgmap v4259: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:50:12.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:50:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:50:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:50:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:50:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:50:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:50:13.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:50:13 smithi083 ceph-mon[77242]: pgmap v4260: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:50:13.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:50:13 smithi083 ceph-mon[79930]: pgmap v4260: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:50:14.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:50:13 smithi153 ceph-mon[70044]: pgmap v4260: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:50:14.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:50:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:50:14.048Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:50:15.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:50:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:50:14.749Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:50:15.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:50:15 smithi083 ceph-mon[79930]: pgmap v4261: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:50:15.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:50:15 smithi083 ceph-mon[77242]: pgmap v4261: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:50:16.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:50:15 smithi153 ceph-mon[70044]: pgmap v4261: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:50:16.666 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:50:16.953 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:50:16.954 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 3m ago 2h 26.5M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:50:16.954 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 113s ago 2h 62.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:50:16.954 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 3m ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:50:16.955 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 113s ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:50:16.955 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 113s ago 2h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:50:16.955 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 3m ago 2h 635M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:50:16.955 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 3m ago 2h 566M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:50:16.956 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 113s ago 2h 252M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:50:16.956 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 3m ago 2h 330M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:50:16.956 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 3m ago 2h 26.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:50:16.956 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 113s ago 2h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:50:16.956 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 3m ago 2h 412M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:50:16.957 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 3m ago 2h 403M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:50:16.957 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 3m ago 2h 319M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:50:16.957 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 3m ago 2h 390M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:50:16.958 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 113s ago 2h 349M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:50:16.958 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 113s ago 2h 427M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:50:16.958 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 113s ago 2h 380M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:50:16.958 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 113s ago 2h 444M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:50:16.958 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 113s ago 2h 125M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:50:17.312 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:50:17.312 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:50:17.313 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:50:17.313 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:50:17.313 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:50:17.313 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:50:17.314 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:50:17.314 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:50:17.314 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:50:17.314 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:50:17.315 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:50:17.315 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:50:17.315 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:50:17.315 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:50:17.316 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:50:17.316 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:50:17.316 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:50:17.316 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:50:17.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:50:17 smithi083 ceph-mon[79930]: pgmap v4262: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:50:17.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:50:17 smithi083 ceph-mon[79930]: from='client.49027 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:50:17.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:50:17 smithi083 ceph-mon[79930]: from='client.39099 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:50:17.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:50:17 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4264508304' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:50:17.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:50:17 smithi083 ceph-mon[77242]: pgmap v4262: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:50:17.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:50:17 smithi083 ceph-mon[77242]: from='client.49027 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:50:17.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:50:17 smithi083 ceph-mon[77242]: from='client.39099 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:50:17.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:50:17 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4264508304' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:50:18.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:50:17 smithi153 ceph-mon[70044]: pgmap v4262: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:50:18.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:50:17 smithi153 ceph-mon[70044]: from='client.49027 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:50:18.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:50:17 smithi153 ceph-mon[70044]: from='client.39099 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:50:18.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:50:17 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4264508304' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:50:19.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:50:19 smithi083 ceph-mon[79930]: pgmap v4263: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:50:19.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:50:19 smithi083 ceph-mon[77242]: pgmap v4263: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:50:20.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:50:19 smithi153 ceph-mon[70044]: pgmap v4263: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:50:20.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:50:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:50:20.655Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:50:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:50:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:50:21] "GET /metrics HTTP/1.1" 200 225209 "" "Prometheus/2.33.4" 2022-09-29T07:50:21.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:50:21 smithi083 ceph-mon[77242]: pgmap v4264: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:50:21.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:50:21 smithi083 ceph-mon[79930]: pgmap v4264: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:50:22.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:50:21 smithi153 ceph-mon[70044]: pgmap v4264: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:50:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:50:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:50:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:50:23.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:50:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:50:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:50:23.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:50:23 smithi083 ceph-mon[79930]: pgmap v4265: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:50:23.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:50:23 smithi083 ceph-mon[77242]: pgmap v4265: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:50:24.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:50:23 smithi153 ceph-mon[70044]: pgmap v4265: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:50:24.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:50:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:50:24.049Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:50:24.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:50:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:50:24.749Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:50:24.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:50:24 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:50:24.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:50:24 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:50:24.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:50:24 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:50:24.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:50:24 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:50:24.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:50:24 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:50:24.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:50:24 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:50:25.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:50:24 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:50:25.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:50:24 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:50:25.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:50:24 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:50:26.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:50:25 smithi083 ceph-mon[79930]: pgmap v4266: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:50:26.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:50:25 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:50:26.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:50:25 smithi083 ceph-mon[77242]: pgmap v4266: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:50:26.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:50:25 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:50:26.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:50:25 smithi153 ceph-mon[70044]: pgmap v4266: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:50:26.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:50:25 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:50:28.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:50:27 smithi083 ceph-mon[79930]: pgmap v4267: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:50:28.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:50:27 smithi083 ceph-mon[77242]: pgmap v4267: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:50:28.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:50:27 smithi153 ceph-mon[70044]: pgmap v4267: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:50:30.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:50:29 smithi083 ceph-mon[77242]: pgmap v4268: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:50:30.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:50:29 smithi083 ceph-mon[79930]: pgmap v4268: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:50:30.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:50:29 smithi153 ceph-mon[70044]: pgmap v4268: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:50:30.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:50:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:50:30.656Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:50:31.850 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:50:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:50:31] "GET /metrics HTTP/1.1" 200 225207 "" "Prometheus/2.33.4" 2022-09-29T07:50:32.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:50:31 smithi083 ceph-mon[79930]: pgmap v4269: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:50:32.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:50:31 smithi083 ceph-mon[77242]: pgmap v4269: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:50:32.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:50:31 smithi153 ceph-mon[70044]: pgmap v4269: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:50:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:50:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:50:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:50:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:50:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:50:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:50:34.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:50:33 smithi083 ceph-mon[79930]: pgmap v4270: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:50:34.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:50:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:50:34.048Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:50:34.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:50:33 smithi083 ceph-mon[77242]: pgmap v4270: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:50:34.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:50:33 smithi153 ceph-mon[70044]: pgmap v4270: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:50:35.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:50:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:50:34.750Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:50:36.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:50:35 smithi083 ceph-mon[79930]: pgmap v4271: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:50:36.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:50:35 smithi083 ceph-mon[77242]: pgmap v4271: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:50:36.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:50:35 smithi153 ceph-mon[70044]: pgmap v4271: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:50:38.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:50:37 smithi083 ceph-mon[79930]: pgmap v4272: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:50:38.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:50:37 smithi083 ceph-mon[77242]: pgmap v4272: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:50:38.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:50:37 smithi153 ceph-mon[70044]: pgmap v4272: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:50:40.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:50:39 smithi083 ceph-mon[79930]: pgmap v4273: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:50:40.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:50:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:50:40.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:50:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:50:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:50:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:50:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:50:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:50:40.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:50:39 smithi083 ceph-mon[77242]: pgmap v4273: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:50:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:50:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:50:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:50:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:50:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:50:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:50:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:50:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:50:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:50:39 smithi153 ceph-mon[70044]: pgmap v4273: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:50:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:50:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:50:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:50:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:50:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:50:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:50:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:50:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:50:40.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:50:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:50:40.656Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:50:41.865 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:50:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:50:41] "GET /metrics HTTP/1.1" 200 225205 "" "Prometheus/2.33.4" 2022-09-29T07:50:42.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:50:41 smithi083 ceph-mon[79930]: pgmap v4274: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:50:42.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:50:41 smithi083 ceph-mon[77242]: pgmap v4274: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:50:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:50:41 smithi153 ceph-mon[70044]: pgmap v4274: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:50:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:50:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:50:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:50:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:50:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:50:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:50:44.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:50:43 smithi083 ceph-mon[77242]: pgmap v4275: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:50:44.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:50:43 smithi083 ceph-mon[79930]: pgmap v4275: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:50:44.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:50:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:50:44.048Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:50:44.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:50:43 smithi153 ceph-mon[70044]: pgmap v4275: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:50:45.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:50:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:50:44.751Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:50:45.872 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:50:45 smithi083 ceph-mon[77242]: pgmap v4276: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:50:46.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:50:45 smithi083 ceph-mon[79930]: pgmap v4276: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:50:46.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:50:45 smithi153 ceph-mon[70044]: pgmap v4276: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:50:47.642 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:50:47.932 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:50:47.933 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 4m ago 2h 26.5M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:50:47.933 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 2m ago 2h 62.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:50:47.933 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 4m ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:50:47.934 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 2m ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:50:47.934 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 2m ago 2h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:50:47.934 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 4m ago 2h 635M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:50:47.934 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 4m ago 2h 566M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:50:47.935 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 2m ago 2h 252M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:50:47.935 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 4m ago 2h 330M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:50:47.935 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 4m ago 2h 26.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:50:47.935 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 2m ago 2h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:50:47.935 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 4m ago 2h 412M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:50:47.936 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 4m ago 2h 403M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:50:47.936 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 4m ago 2h 319M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:50:47.936 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 4m ago 2h 390M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:50:47.936 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 2m ago 2h 349M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:50:47.937 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 2m ago 2h 427M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:50:47.937 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 2m ago 2h 380M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:50:47.937 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 2m ago 2h 444M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:50:47.937 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 2m ago 2h 125M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:50:48.210 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:50:47 smithi083 ceph-mon[77242]: pgmap v4277: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:50:48.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:50:47 smithi083 ceph-mon[79930]: pgmap v4277: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:50:48.291 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:50:48.291 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:50:48.292 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:50:48.292 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:50:48.292 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:50:48.292 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:50:48.293 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:50:48.293 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:50:48.293 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:50:48.293 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:50:48.294 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:50:48.294 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:50:48.294 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:50:48.294 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:50:48.294 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:50:48.295 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:50:48.295 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:50:48.295 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:50:48.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:50:47 smithi153 ceph-mon[70044]: pgmap v4277: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:50:49.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:50:48 smithi083 ceph-mon[77242]: from='client.39111 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:50:49.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:50:48 smithi083 ceph-mon[77242]: from='client.49051 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:50:49.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:50:48 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/66526491' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:50:49.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:50:48 smithi083 ceph-mon[79930]: from='client.39111 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:50:49.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:50:48 smithi083 ceph-mon[79930]: from='client.49051 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:50:49.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:50:48 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/66526491' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:50:49.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:50:48 smithi153 ceph-mon[70044]: from='client.39111 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:50:49.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:50:48 smithi153 ceph-mon[70044]: from='client.49051 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:50:49.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:50:48 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/66526491' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:50:50.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:50:49 smithi083 ceph-mon[79930]: pgmap v4278: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:50:50.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:50:49 smithi083 ceph-mon[77242]: pgmap v4278: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:50:50.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:50:49 smithi153 ceph-mon[70044]: pgmap v4278: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:50:50.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:50:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:50:50.658Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:50:51.882 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:50:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:50:51] "GET /metrics HTTP/1.1" 200 225205 "" "Prometheus/2.33.4" 2022-09-29T07:50:52.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:50:51 smithi083 ceph-mon[77242]: pgmap v4279: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:50:52.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:50:51 smithi083 ceph-mon[79930]: pgmap v4279: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:50:52.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:50:51 smithi153 ceph-mon[70044]: pgmap v4279: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:50:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:50:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:50:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:50:53.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:50:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:50:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:50:54.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:50:53 smithi083 ceph-mon[79930]: pgmap v4280: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:50:54.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:50:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:50:54.051Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:50:54.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:50:53 smithi083 ceph-mon[77242]: pgmap v4280: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:50:54.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:50:53 smithi153 ceph-mon[70044]: pgmap v4280: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:50:55.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:50:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:50:54.751Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:50:56.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:50:55 smithi083 ceph-mon[79930]: pgmap v4281: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:50:56.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:50:55 smithi083 ceph-mon[77242]: pgmap v4281: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:50:56.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:50:55 smithi153 ceph-mon[70044]: pgmap v4281: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:50:58.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:50:57 smithi083 ceph-mon[79930]: pgmap v4282: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:50:58.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:50:57 smithi083 ceph-mon[77242]: pgmap v4282: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:50:58.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:50:57 smithi153 ceph-mon[70044]: pgmap v4282: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:00.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:50:59 smithi083 ceph-mon[79930]: pgmap v4283: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:00.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:50:59 smithi083 ceph-mon[77242]: pgmap v4283: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:00.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:50:59 smithi153 ceph-mon[70044]: pgmap v4283: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:00.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:51:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:51:00.658Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:51:01.897 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:51:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:51:01] "GET /metrics HTTP/1.1" 200 225209 "" "Prometheus/2.33.4" 2022-09-29T07:51:02.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:51:01 smithi083 ceph-mon[77242]: pgmap v4284: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:02.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:51:01 smithi083 ceph-mon[79930]: pgmap v4284: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:02.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:51:01 smithi153 ceph-mon[70044]: pgmap v4284: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:51:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:51:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:51:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:51:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:51:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:51:04.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:51:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:51:04.050Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:51:04.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:51:03 smithi083 ceph-mon[79930]: pgmap v4285: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:04.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:51:03 smithi083 ceph-mon[77242]: pgmap v4285: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:04.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:51:03 smithi153 ceph-mon[70044]: pgmap v4285: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:05.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:51:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:51:04.752Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:51:06.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:51:05 smithi083 ceph-mon[77242]: pgmap v4286: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:06.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:51:05 smithi083 ceph-mon[79930]: pgmap v4286: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:06.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:51:05 smithi153 ceph-mon[70044]: pgmap v4286: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:08.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:51:07 smithi083 ceph-mon[79930]: pgmap v4287: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:08.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:51:07 smithi083 ceph-mon[77242]: pgmap v4287: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:08.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:51:07 smithi153 ceph-mon[70044]: pgmap v4287: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:10.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:51:09 smithi083 ceph-mon[79930]: pgmap v4288: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:10.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:51:09 smithi083 ceph-mon[77242]: pgmap v4288: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:10.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:51:09 smithi153 ceph-mon[70044]: pgmap v4288: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:10.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:51:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:51:10.658Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:51:11.914 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:51:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:51:11] "GET /metrics HTTP/1.1" 200 225210 "" "Prometheus/2.33.4" 2022-09-29T07:51:12.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:51:11 smithi083 ceph-mon[77242]: pgmap v4289: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:12.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:51:11 smithi083 ceph-mon[79930]: pgmap v4289: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:12.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:51:11 smithi153 ceph-mon[70044]: pgmap v4289: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:51:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:51:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:51:13.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:51:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:51:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:51:14.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:51:13 smithi083 ceph-mon[77242]: pgmap v4290: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:14.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:51:13 smithi083 ceph-mon[79930]: pgmap v4290: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:14.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:51:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:51:14.051Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:51:14.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:51:13 smithi153 ceph-mon[70044]: pgmap v4290: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:15.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:51:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:51:14.753Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:51:16.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:51:15 smithi083 ceph-mon[79930]: pgmap v4291: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:16.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:51:15 smithi083 ceph-mon[77242]: pgmap v4291: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:16.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:51:15 smithi153 ceph-mon[70044]: pgmap v4291: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:18.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:51:17 smithi083 ceph-mon[79930]: pgmap v4292: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:18.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:51:17 smithi083 ceph-mon[77242]: pgmap v4292: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:18.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:51:17 smithi153 ceph-mon[70044]: pgmap v4292: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:18.617 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:51:18.905 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:51:18.906 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 5m ago 2h 26.5M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:51:18.906 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 2m ago 2h 62.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:51:18.906 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 5m ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:51:18.907 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 2m ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:51:18.907 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 2m ago 2h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:51:18.907 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 5m ago 2h 635M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:51:18.907 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 5m ago 2h 566M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:51:18.908 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 2m ago 2h 252M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:51:18.908 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 5m ago 2h 330M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:51:18.908 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 5m ago 2h 26.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:51:18.908 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 2m ago 2h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:51:18.909 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 5m ago 2h 412M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:51:18.909 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 5m ago 2h 403M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:51:18.909 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 5m ago 2h 319M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:51:18.909 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 5m ago 2h 390M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:51:18.909 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 2m ago 2h 349M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:51:18.910 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 2m ago 2h 427M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:51:18.910 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 2m ago 2h 380M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:51:18.910 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 2m ago 2h 444M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:51:18.911 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 2m ago 2h 125M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:51:19.262 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:51:19.262 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:51:19.263 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:51:19.263 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:51:19.263 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:51:19.263 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:51:19.264 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:51:19.264 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:51:19.264 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:51:19.264 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:51:19.264 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:51:19.265 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:51:19.265 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:51:19.265 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:51:19.265 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:51:19.266 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:51:19.266 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:51:19.266 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:51:20.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:51:19 smithi083 ceph-mon[79930]: pgmap v4293: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:20.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:51:19 smithi083 ceph-mon[79930]: from='client.39129 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:51:20.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:51:19 smithi083 ceph-mon[79930]: from='client.39135 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:51:20.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:51:19 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4045239776' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:51:20.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:51:19 smithi083 ceph-mon[77242]: pgmap v4293: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:20.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:51:19 smithi083 ceph-mon[77242]: from='client.39129 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:51:20.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:51:19 smithi083 ceph-mon[77242]: from='client.39135 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:51:20.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:51:19 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4045239776' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:51:20.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:51:19 smithi153 ceph-mon[70044]: pgmap v4293: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:51:19 smithi153 ceph-mon[70044]: from='client.39129 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:51:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:51:19 smithi153 ceph-mon[70044]: from='client.39135 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:51:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:51:19 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4045239776' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:51:20.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:51:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:51:20.659Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:51:21.928 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:51:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:51:21] "GET /metrics HTTP/1.1" 200 225210 "" "Prometheus/2.33.4" 2022-09-29T07:51:22.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:51:21 smithi083 ceph-mon[77242]: pgmap v4294: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:22.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:51:21 smithi083 ceph-mon[79930]: pgmap v4294: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:22.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:51:21 smithi153 ceph-mon[70044]: pgmap v4294: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:51:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:51:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:51:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:51:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:51:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:51:24.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:51:23 smithi083 ceph-mon[79930]: pgmap v4295: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:24.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:51:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:51:24.053Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:51:24.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:51:23 smithi083 ceph-mon[77242]: pgmap v4295: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:24.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:51:23 smithi153 ceph-mon[70044]: pgmap v4295: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:25.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:51:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:51:24.754Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:51:25.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:51:24 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:51:25.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:51:24 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:51:25.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:51:24 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:51:25.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:51:24 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:51:25.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:51:24 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:51:25.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:51:24 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:51:25.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:51:24 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:51:25.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:51:24 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:51:25.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:51:24 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:51:26.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:51:25 smithi083 ceph-mon[79930]: pgmap v4296: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:26.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:51:25 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:51:26.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:51:25 smithi083 ceph-mon[77242]: pgmap v4296: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:26.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:51:25 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:51:26.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:51:25 smithi153 ceph-mon[70044]: pgmap v4296: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:26.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:51:25 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:51:28.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:51:27 smithi083 ceph-mon[79930]: pgmap v4297: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:28.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:51:27 smithi083 ceph-mon[77242]: pgmap v4297: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:28.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:51:27 smithi153 ceph-mon[70044]: pgmap v4297: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:30.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:51:29 smithi083 ceph-mon[79930]: pgmap v4298: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:30.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:51:29 smithi083 ceph-mon[77242]: pgmap v4298: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:30.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:51:29 smithi153 ceph-mon[70044]: pgmap v4298: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:30.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:51:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:51:30.660Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:51:31.953 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:51:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:51:31] "GET /metrics HTTP/1.1" 200 225209 "" "Prometheus/2.33.4" 2022-09-29T07:51:32.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:51:31 smithi083 ceph-mon[79930]: pgmap v4299: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:32.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:51:31 smithi083 ceph-mon[77242]: pgmap v4299: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:32.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:51:31 smithi153 ceph-mon[70044]: pgmap v4299: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:51:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:51:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:51:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:51:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:51:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:51:34.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:51:33 smithi083 ceph-mon[79930]: pgmap v4300: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:34.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:51:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:51:34.053Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:51:34.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:51:33 smithi083 ceph-mon[77242]: pgmap v4300: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:34.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:51:33 smithi153 ceph-mon[70044]: pgmap v4300: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:35.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:51:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:51:34.754Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:51:36.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:51:35 smithi083 ceph-mon[79930]: pgmap v4301: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:36.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:51:35 smithi083 ceph-mon[77242]: pgmap v4301: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:36.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:51:35 smithi153 ceph-mon[70044]: pgmap v4301: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:38.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:51:37 smithi153 ceph-mon[70044]: pgmap v4302: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:38.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:51:37 smithi083 ceph-mon[79930]: pgmap v4302: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:38.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:51:37 smithi083 ceph-mon[77242]: pgmap v4302: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:51:39 smithi153 ceph-mon[70044]: pgmap v4303: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:51:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:51:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:51:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:51:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:51:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:51:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:51:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:51:40.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:51:39 smithi083 ceph-mon[79930]: pgmap v4303: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:40.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:51:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:51:40.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:51:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:51:40.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:51:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:51:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:51:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:51:40.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:51:39 smithi083 ceph-mon[77242]: pgmap v4303: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:40.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:51:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:51:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:51:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:51:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:51:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:51:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:51:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:51:40.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:51:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:51:40.660Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:51:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:51:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:51:41] "GET /metrics HTTP/1.1" 200 225208 "" "Prometheus/2.33.4" 2022-09-29T07:51:42.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:51:41 smithi153 ceph-mon[70044]: pgmap v4304: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:42.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:51:41 smithi083 ceph-mon[79930]: pgmap v4304: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:42.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:51:41 smithi083 ceph-mon[77242]: pgmap v4304: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:51:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:51:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:51:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:51:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:51:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:51:44.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:51:43 smithi153 ceph-mon[70044]: pgmap v4305: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:44.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:51:43 smithi083 ceph-mon[79930]: pgmap v4305: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:44.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:51:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:51:44.053Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:51:44.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:51:43 smithi083 ceph-mon[77242]: pgmap v4305: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:45.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:51:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:51:44.755Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:51:46.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:51:45 smithi153 ceph-mon[70044]: pgmap v4306: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:46.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:51:45 smithi083 ceph-mon[79930]: pgmap v4306: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:46.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:51:45 smithi083 ceph-mon[77242]: pgmap v4306: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:48.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:51:47 smithi153 ceph-mon[70044]: pgmap v4307: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:48.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:51:47 smithi083 ceph-mon[79930]: pgmap v4307: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:48.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:51:47 smithi083 ceph-mon[77242]: pgmap v4307: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:49.588 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:51:49.875 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:51:49.876 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 5m ago 2h 26.5M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:51:49.876 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 3m ago 2h 62.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:51:49.876 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 5m ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:51:49.876 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 3m ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:51:49.877 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 3m ago 2h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:51:49.877 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 5m ago 2h 635M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:51:49.877 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 5m ago 2h 566M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:51:49.877 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 3m ago 2h 252M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:51:49.877 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 5m ago 2h 330M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:51:49.878 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 5m ago 2h 26.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:51:49.878 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 3m ago 2h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:51:49.878 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 5m ago 2h 412M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:51:49.878 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 5m ago 2h 403M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:51:49.879 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 5m ago 2h 319M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:51:49.879 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 5m ago 2h 390M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:51:49.879 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 3m ago 2h 349M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:51:49.879 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 3m ago 2h 427M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:51:49.880 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 3m ago 2h 380M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:51:49.880 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 3m ago 2h 444M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:51:49.881 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 3m ago 2h 125M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:51:50.233 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:51:50.234 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:51:50.234 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:51:50.234 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:51:50.235 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:51:50.235 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:51:50.235 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:51:50.235 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:51:50.236 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:51:50.236 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:51:50.236 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:51:50.236 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:51:50.236 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:51:50.237 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:51:50.237 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:51:50.237 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:51:50.237 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:51:50.237 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:51:50.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:51:49 smithi153 ceph-mon[70044]: pgmap v4308: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:50.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:51:49 smithi083 ceph-mon[79930]: pgmap v4308: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:50.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:51:49 smithi083 ceph-mon[77242]: pgmap v4308: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:50.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:51:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:51:50.660Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:51:51.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:51:50 smithi153 ceph-mon[70044]: from='client.39147 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:51:51.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:51:50 smithi153 ceph-mon[70044]: from='client.39153 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:51:51.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:51:50 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2601422156' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:51:51.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:51:50 smithi083 ceph-mon[77242]: from='client.39147 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:51:51.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:51:50 smithi083 ceph-mon[77242]: from='client.39153 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:51:51.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:51:50 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2601422156' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:51:51.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:51:50 smithi083 ceph-mon[79930]: from='client.39147 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:51:51.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:51:50 smithi083 ceph-mon[79930]: from='client.39153 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:51:51.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:51:50 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2601422156' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:51:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:51:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:51:51] "GET /metrics HTTP/1.1" 200 225208 "" "Prometheus/2.33.4" 2022-09-29T07:51:52.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:51:51 smithi153 ceph-mon[70044]: pgmap v4309: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:52.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:51:51 smithi083 ceph-mon[79930]: pgmap v4309: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:52.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:51:51 smithi083 ceph-mon[77242]: pgmap v4309: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:51:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:51:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:51:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:51:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:51:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:51:54.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:51:53 smithi153 ceph-mon[70044]: pgmap v4310: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:54.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:51:53 smithi083 ceph-mon[77242]: pgmap v4310: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:54.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:51:53 smithi083 ceph-mon[79930]: pgmap v4310: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:54.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:51:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:51:54.056Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:51:55.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:51:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:51:54.756Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:51:56.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:51:55 smithi153 ceph-mon[70044]: pgmap v4311: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:56.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:51:55 smithi083 ceph-mon[77242]: pgmap v4311: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:56.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:51:55 smithi083 ceph-mon[79930]: pgmap v4311: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:58.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:51:57 smithi153 ceph-mon[70044]: pgmap v4312: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:58.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:51:57 smithi083 ceph-mon[79930]: pgmap v4312: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:51:58.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:51:57 smithi083 ceph-mon[77242]: pgmap v4312: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:52:00 smithi153 ceph-mon[70044]: pgmap v4313: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:00.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:52:00 smithi083 ceph-mon[77242]: pgmap v4313: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:00.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:52:00 smithi083 ceph-mon[79930]: pgmap v4313: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:00.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:52:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:52:00.661Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:52:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:52:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:52:01] "GET /metrics HTTP/1.1" 200 225208 "" "Prometheus/2.33.4" 2022-09-29T07:52:02.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:52:02 smithi153 ceph-mon[70044]: pgmap v4314: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:02.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:52:02 smithi083 ceph-mon[77242]: pgmap v4314: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:02.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:52:02 smithi083 ceph-mon[79930]: pgmap v4314: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:52:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:52:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:52:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:52:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:52:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:52:04.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:52:04 smithi153 ceph-mon[70044]: pgmap v4315: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:04.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:52:04 smithi083 ceph-mon[79930]: pgmap v4315: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:04.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:52:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:52:04.055Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:52:04.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:52:04 smithi083 ceph-mon[77242]: pgmap v4315: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:05.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:52:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:52:04.757Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:52:06.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:52:06 smithi153 ceph-mon[70044]: pgmap v4316: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:06.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:52:06 smithi083 ceph-mon[79930]: pgmap v4316: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:06.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:52:06 smithi083 ceph-mon[77242]: pgmap v4316: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:08.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:52:08 smithi153 ceph-mon[70044]: pgmap v4317: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:08.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:52:08 smithi083 ceph-mon[79930]: pgmap v4317: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:08.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:52:08 smithi083 ceph-mon[77242]: pgmap v4317: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:10.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:52:10 smithi153 ceph-mon[70044]: pgmap v4318: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:10.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:52:10 smithi083 ceph-mon[79930]: pgmap v4318: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:10.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:52:10 smithi083 ceph-mon[77242]: pgmap v4318: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:10.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:52:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:52:10.662Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:52:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:52:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:52:11] "GET /metrics HTTP/1.1" 200 225206 "" "Prometheus/2.33.4" 2022-09-29T07:52:12.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:52:12 smithi153 ceph-mon[70044]: pgmap v4319: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:12.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:52:12 smithi083 ceph-mon[79930]: pgmap v4319: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:12.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:52:12 smithi083 ceph-mon[77242]: pgmap v4319: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:52:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:52:12.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:52:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:52:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:52:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:52:14.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:52:14 smithi153 ceph-mon[70044]: pgmap v4320: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:14.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:52:14 smithi083 ceph-mon[77242]: pgmap v4320: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:14.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:52:14 smithi083 ceph-mon[79930]: pgmap v4320: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:14.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:52:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:52:14.056Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:52:15.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:52:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:52:14.758Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:52:16.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:52:16 smithi153 ceph-mon[70044]: pgmap v4321: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:16.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:52:16 smithi083 ceph-mon[79930]: pgmap v4321: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:16.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:52:16 smithi083 ceph-mon[77242]: pgmap v4321: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:18.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:52:18 smithi153 ceph-mon[70044]: pgmap v4322: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:18.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:52:18 smithi083 ceph-mon[79930]: pgmap v4322: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:18.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:52:18 smithi083 ceph-mon[77242]: pgmap v4322: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:52:20 smithi153 ceph-mon[70044]: pgmap v4323: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:20.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:52:20 smithi083 ceph-mon[77242]: pgmap v4323: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:20.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:52:20 smithi083 ceph-mon[79930]: pgmap v4323: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:20.560 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:52:20.849 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:52:20.849 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 6m ago 2h 26.5M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:52:20.849 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 3m ago 2h 62.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:52:20.850 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 6m ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:52:20.850 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 3m ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:52:20.850 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 3m ago 2h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:52:20.850 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 6m ago 2h 635M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:52:20.851 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 6m ago 2h 566M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:52:20.851 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 3m ago 2h 252M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:52:20.851 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 6m ago 2h 330M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:52:20.851 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 6m ago 2h 26.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:52:20.851 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 3m ago 2h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:52:20.852 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 6m ago 2h 412M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:52:20.852 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 6m ago 2h 403M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:52:20.852 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 6m ago 2h 319M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:52:20.852 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 6m ago 2h 390M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:52:20.852 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 3m ago 2h 349M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:52:20.853 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 3m ago 2h 427M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:52:20.853 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 3m ago 2h 380M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:52:20.853 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 3m ago 2h 444M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:52:20.853 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 3m ago 2h 125M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:52:20.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:52:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:52:20.662Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:52:21.207 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:52:21.207 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:52:21.208 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:52:21.208 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:52:21.208 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:52:21.208 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:52:21.209 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:52:21.209 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:52:21.209 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:52:21.209 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:52:21.209 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:52:21.210 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:52:21.210 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:52:21.210 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:52:21.210 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:52:21.211 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:52:21.211 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:52:21.211 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:52:21.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:52:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:52:21] "GET /metrics HTTP/1.1" 200 225206 "" "Prometheus/2.33.4" 2022-09-29T07:52:22.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:52:22 smithi153 ceph-mon[70044]: pgmap v4324: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:22.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:52:22 smithi153 ceph-mon[70044]: from='client.49099 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:52:22.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:52:22 smithi153 ceph-mon[70044]: from='client.49105 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:52:22.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:52:22 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1063539762' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:52:22.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:52:22 smithi083 ceph-mon[77242]: pgmap v4324: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:22.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:52:22 smithi083 ceph-mon[77242]: from='client.49099 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:52:22.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:52:22 smithi083 ceph-mon[77242]: from='client.49105 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:52:22.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:52:22 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1063539762' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:52:22.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:52:22 smithi083 ceph-mon[79930]: pgmap v4324: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:22.467 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:52:22 smithi083 ceph-mon[79930]: from='client.49099 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:52:22.467 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:52:22 smithi083 ceph-mon[79930]: from='client.49105 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:52:22.467 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:52:22 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1063539762' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:52:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:52:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:52:22.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:52:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:52:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:52:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:52:24.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:52:24 smithi153 ceph-mon[70044]: pgmap v4325: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:24.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:52:24 smithi083 ceph-mon[79930]: pgmap v4325: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:24.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:52:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:52:24.058Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:52:24.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:52:24 smithi083 ceph-mon[77242]: pgmap v4325: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:25.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:52:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:52:24.758Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:52:26.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:52:26 smithi153 ceph-mon[70044]: pgmap v4326: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:26.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:52:26 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:52:26.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:52:26 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:52:26.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:52:26 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:52:26.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:52:26 smithi083 ceph-mon[79930]: pgmap v4326: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:26.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:52:26 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:52:26.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:52:26 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:52:26.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:52:26 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:52:26.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:52:26 smithi083 ceph-mon[77242]: pgmap v4326: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:26.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:52:26 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:52:26.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:52:26 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:52:26.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:52:26 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:52:27.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:52:27 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:52:27.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:52:27 smithi083 ceph-mon[79930]: pgmap v4327: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:27.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:52:27 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:52:27.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:52:27 smithi083 ceph-mon[77242]: pgmap v4327: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:27.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:52:27 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:52:27.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:52:27 smithi153 ceph-mon[70044]: pgmap v4327: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:29.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:52:29 smithi153 ceph-mon[70044]: pgmap v4328: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:29.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:52:29 smithi083 ceph-mon[77242]: pgmap v4328: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:29.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:52:29 smithi083 ceph-mon[79930]: pgmap v4328: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:30.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:52:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:52:30.663Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:52:31.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:52:31 smithi153 ceph-mon[70044]: pgmap v4329: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:31.710 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:52:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:52:31] "GET /metrics HTTP/1.1" 200 225203 "" "Prometheus/2.33.4" 2022-09-29T07:52:31.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:52:31 smithi083 ceph-mon[79930]: pgmap v4329: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:31.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:52:31 smithi083 ceph-mon[77242]: pgmap v4329: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:52:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:52:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:52:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:52:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:52:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:52:33.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:52:33 smithi153 ceph-mon[70044]: pgmap v4330: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:33.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:52:33 smithi083 ceph-mon[79930]: pgmap v4330: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:33.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:52:33 smithi083 ceph-mon[77242]: pgmap v4330: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:34.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:52:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:52:34.057Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:52:35.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:52:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:52:34.759Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:52:35.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:52:35 smithi153 ceph-mon[70044]: pgmap v4331: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:35.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:52:35 smithi083 ceph-mon[79930]: pgmap v4331: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:35.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:52:35 smithi083 ceph-mon[77242]: pgmap v4331: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:37.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:52:37 smithi153 ceph-mon[70044]: pgmap v4332: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:37.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:52:37 smithi083 ceph-mon[79930]: pgmap v4332: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:37.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:52:37 smithi083 ceph-mon[77242]: pgmap v4332: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:39.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:52:39 smithi153 ceph-mon[70044]: pgmap v4333: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:39.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:52:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:52:39.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:52:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:52:39.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:52:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:52:39.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:52:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:52:39.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:52:39 smithi083 ceph-mon[77242]: pgmap v4333: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:39.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:52:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:52:39.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:52:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:52:39.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:52:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:52:39.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:52:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:52:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:52:39 smithi083 ceph-mon[79930]: pgmap v4333: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:39.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:52:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:52:39.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:52:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:52:39.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:52:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:52:39.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:52:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:52:40.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:52:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:52:40.664Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:52:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:52:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:52:41] "GET /metrics HTTP/1.1" 200 225210 "" "Prometheus/2.33.4" 2022-09-29T07:52:41.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:52:41 smithi083 ceph-mon[79930]: pgmap v4334: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:41.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:52:41 smithi083 ceph-mon[77242]: pgmap v4334: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:42.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:52:41 smithi153 ceph-mon[70044]: pgmap v4334: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:52:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:52:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:52:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:52:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:52:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:52:43.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:52:43 smithi083 ceph-mon[77242]: pgmap v4335: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:43.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:52:43 smithi083 ceph-mon[79930]: pgmap v4335: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:44.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:52:43 smithi153 ceph-mon[70044]: pgmap v4335: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:44.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:52:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:52:44.057Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:52:45.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:52:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:52:44.760Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:52:45.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:52:45 smithi083 ceph-mon[79930]: pgmap v4336: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:45.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:52:45 smithi083 ceph-mon[77242]: pgmap v4336: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:46.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:52:45 smithi153 ceph-mon[70044]: pgmap v4336: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:47.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:52:47 smithi083 ceph-mon[79930]: pgmap v4337: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:47.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:52:47 smithi083 ceph-mon[77242]: pgmap v4337: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:48.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:52:47 smithi153 ceph-mon[70044]: pgmap v4337: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:49.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:52:49 smithi083 ceph-mon[79930]: pgmap v4338: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:49.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:52:49 smithi083 ceph-mon[77242]: pgmap v4338: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:50.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:52:49 smithi153 ceph-mon[70044]: pgmap v4338: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:50.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:52:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:52:50.664Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:52:51.533 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:52:51.826 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:52:51.827 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 6m ago 2h 26.5M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:52:51.827 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 4m ago 2h 62.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:52:51.827 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 6m ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:52:51.827 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 4m ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:52:51.827 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 4m ago 2h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:52:51.828 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 6m ago 2h 635M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:52:51.828 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 6m ago 2h 566M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:52:51.828 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 4m ago 2h 252M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:52:51.828 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 6m ago 2h 330M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:52:51.828 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 6m ago 2h 26.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:52:51.829 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 4m ago 2h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:52:51.829 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 6m ago 2h 412M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:52:51.829 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 6m ago 2h 403M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:52:51.829 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 6m ago 2h 319M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:52:51.829 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 6m ago 2h 390M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:52:51.830 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 4m ago 2h 349M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:52:51.830 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 4m ago 2h 427M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:52:51.830 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 4m ago 2h 380M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:52:51.830 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 4m ago 2h 444M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:52:51.831 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 4m ago 2h 125M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:52:51.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:52:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:52:51] "GET /metrics HTTP/1.1" 200 225210 "" "Prometheus/2.33.4" 2022-09-29T07:52:51.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:52:51 smithi083 ceph-mon[79930]: pgmap v4339: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:51.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:52:51 smithi083 ceph-mon[77242]: pgmap v4339: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:52.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:52:51 smithi153 ceph-mon[70044]: pgmap v4339: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:52.181 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:52:52.182 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:52:52.182 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:52:52.182 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:52:52.183 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:52:52.183 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:52:52.183 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:52:52.183 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:52:52.183 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:52:52.184 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:52:52.184 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:52:52.184 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:52:52.184 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:52:52.184 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:52:52.185 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:52:52.185 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:52:52.185 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:52:52.185 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:52:52.582 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:52:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:52:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:52:52.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:52:52 smithi083 ceph-mon[79930]: from='client.39183 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:52:52.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:52:52 smithi083 ceph-mon[79930]: from='client.39189 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:52:52.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:52:52 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/715566042' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:52:52.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:52:52 smithi083 ceph-mon[77242]: from='client.39183 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:52:52.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:52:52 smithi083 ceph-mon[77242]: from='client.39189 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:52:52.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:52:52 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/715566042' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:52:53.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:52:52 smithi153 ceph-mon[70044]: from='client.39183 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:52:53.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:52:52 smithi153 ceph-mon[70044]: from='client.39189 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:52:53.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:52:52 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/715566042' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:52:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:52:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:52:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:52:53.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:52:53 smithi083 ceph-mon[77242]: pgmap v4340: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:53.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:52:53 smithi083 ceph-mon[79930]: pgmap v4340: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:54.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:52:53 smithi153 ceph-mon[70044]: pgmap v4340: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:54.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:52:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:52:54.059Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:52:55.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:52:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:52:54.760Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:52:55.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:52:55 smithi083 ceph-mon[79930]: pgmap v4341: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:55.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:52:55 smithi083 ceph-mon[77242]: pgmap v4341: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:56.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:52:55 smithi153 ceph-mon[70044]: pgmap v4341: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:57.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:52:57 smithi083 ceph-mon[79930]: pgmap v4342: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:57.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:52:57 smithi083 ceph-mon[77242]: pgmap v4342: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:58.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:52:57 smithi153 ceph-mon[70044]: pgmap v4342: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:59.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:52:59 smithi083 ceph-mon[79930]: pgmap v4343: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:52:59.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:52:59 smithi083 ceph-mon[77242]: pgmap v4343: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:53:00.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:52:59 smithi153 ceph-mon[70044]: pgmap v4343: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:53:00.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:53:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:53:00.665Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:53:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:53:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:53:01] "GET /metrics HTTP/1.1" 200 225213 "" "Prometheus/2.33.4" 2022-09-29T07:53:01.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:53:01 smithi083 ceph-mon[79930]: pgmap v4344: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:53:01.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:53:01 smithi083 ceph-mon[77242]: pgmap v4344: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:53:02.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:53:01 smithi153 ceph-mon[70044]: pgmap v4344: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:53:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:53:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:53:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:53:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:53:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:53:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:53:03.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:53:03 smithi083 ceph-mon[77242]: pgmap v4345: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:53:03.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:53:03 smithi083 ceph-mon[79930]: pgmap v4345: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:53:04.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:53:03 smithi153 ceph-mon[70044]: pgmap v4345: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:53:04.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:53:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:53:04.058Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:53:05.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:53:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:53:04.760Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:53:05.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:53:05 smithi083 ceph-mon[79930]: pgmap v4346: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:53:05.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:53:05 smithi083 ceph-mon[77242]: pgmap v4346: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:53:06.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:53:05 smithi153 ceph-mon[70044]: pgmap v4346: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:53:07.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:53:07 smithi083 ceph-mon[79930]: pgmap v4347: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:53:07.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:53:07 smithi083 ceph-mon[77242]: pgmap v4347: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:53:08.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:53:07 smithi153 ceph-mon[70044]: pgmap v4347: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:53:09.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:53:09 smithi083 ceph-mon[79930]: pgmap v4348: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:53:09.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:53:09 smithi083 ceph-mon[77242]: pgmap v4348: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:53:10.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:53:09 smithi153 ceph-mon[70044]: pgmap v4348: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:53:10.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:53:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:53:10.665Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:53:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:53:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:53:11] "GET /metrics HTTP/1.1" 200 225201 "" "Prometheus/2.33.4" 2022-09-29T07:53:11.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:53:11 smithi083 ceph-mon[79930]: pgmap v4349: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:53:11.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:53:11 smithi083 ceph-mon[77242]: pgmap v4349: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:53:12.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:53:11 smithi153 ceph-mon[70044]: pgmap v4349: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:53:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:53:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:53:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:53:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:53:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:53:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:53:13.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:53:13 smithi083 ceph-mon[77242]: pgmap v4350: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:53:13.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:53:13 smithi083 ceph-mon[79930]: pgmap v4350: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:53:14.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:53:13 smithi153 ceph-mon[70044]: pgmap v4350: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:53:14.460 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:53:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:53:14.059Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:53:15.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:53:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:53:14.760Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:53:15.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:53:15 smithi083 ceph-mon[77242]: pgmap v4351: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:53:15.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:53:15 smithi083 ceph-mon[79930]: pgmap v4351: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:53:16.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:53:15 smithi153 ceph-mon[70044]: pgmap v4351: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:53:17.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:53:17 smithi083 ceph-mon[79930]: pgmap v4352: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:53:17.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:53:17 smithi083 ceph-mon[77242]: pgmap v4352: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:53:18.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:53:17 smithi153 ceph-mon[70044]: pgmap v4352: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:53:19.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:53:19 smithi083 ceph-mon[79930]: pgmap v4353: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:53:19.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:53:19 smithi083 ceph-mon[77242]: pgmap v4353: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:53:20.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:53:19 smithi153 ceph-mon[70044]: pgmap v4353: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:53:20.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:53:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:53:20.666Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:53:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:53:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:53:21] "GET /metrics HTTP/1.1" 200 225201 "" "Prometheus/2.33.4" 2022-09-29T07:53:21.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:53:21 smithi083 ceph-mon[79930]: pgmap v4354: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:53:21.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:53:21 smithi083 ceph-mon[77242]: pgmap v4354: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:53:22.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:53:21 smithi153 ceph-mon[70044]: pgmap v4354: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:53:22.505 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:53:22.792 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:53:22.792 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 7m ago 2h 26.5M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:53:22.793 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 4m ago 2h 62.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:53:22.793 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 7m ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:53:22.793 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 4m ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:53:22.793 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 4m ago 2h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:53:22.793 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 7m ago 2h 635M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:53:22.794 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 7m ago 2h 566M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:53:22.794 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 4m ago 2h 252M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:53:22.794 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 7m ago 2h 330M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:53:22.794 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 7m ago 2h 26.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:53:22.795 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 4m ago 2h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:53:22.795 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 7m ago 2h 412M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:53:22.795 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 7m ago 2h 403M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:53:22.795 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 7m ago 2h 319M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:53:22.795 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 7m ago 2h 390M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:53:22.796 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 4m ago 2h 349M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:53:22.796 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 4m ago 2h 427M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:53:22.796 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 4m ago 2h 380M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:53:22.796 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 4m ago 2h 444M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:53:22.796 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 4m ago 2h 125M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:53:22.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:53:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:53:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:53:23.145 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:53:23.145 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:53:23.146 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:53:23.146 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:53:23.146 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:53:23.146 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:53:23.147 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:53:23.147 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:53:23.147 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:53:23.147 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:53:23.148 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:53:23.148 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:53:23.148 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:53:23.148 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:53:23.149 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:53:23.149 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:53:23.149 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:53:23.149 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:53:23.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:53:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:53:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:53:23.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:53:23 smithi083 ceph-mon[79930]: pgmap v4355: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:53:23.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:53:23 smithi083 ceph-mon[79930]: from='client.39201 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:53:23.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:53:23 smithi083 ceph-mon[79930]: from='client.49141 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:53:23.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:53:23 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/164883398' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:53:23.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:53:23 smithi083 ceph-mon[77242]: pgmap v4355: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:53:23.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:53:23 smithi083 ceph-mon[77242]: from='client.39201 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:53:23.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:53:23 smithi083 ceph-mon[77242]: from='client.49141 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:53:23.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:53:23 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/164883398' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:53:24.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:53:23 smithi153 ceph-mon[70044]: pgmap v4355: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:53:24.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:53:23 smithi153 ceph-mon[70044]: from='client.39201 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:53:24.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:53:23 smithi153 ceph-mon[70044]: from='client.49141 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:53:24.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:53:23 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/164883398' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:53:24.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:53:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:53:24.060Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:53:25.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:53:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:53:24.760Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:53:25.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:53:25 smithi083 ceph-mon[79930]: pgmap v4356: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:53:25.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:53:25 smithi083 ceph-mon[77242]: pgmap v4356: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:53:26.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:53:25 smithi153 ceph-mon[70044]: pgmap v4356: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:53:26.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:53:26 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:53:26.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:53:26 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:53:26.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:53:26 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:53:26.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:53:26 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:53:26.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:53:26 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:53:26.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:53:26 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:53:27.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:53:26 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:53:27.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:53:26 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:53:27.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:53:26 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:53:28.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:53:27 smithi083 ceph-mon[79930]: pgmap v4357: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:53:28.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:53:27 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:53:28.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:53:27 smithi083 ceph-mon[77242]: pgmap v4357: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:53:28.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:53:27 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:53:28.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:53:27 smithi153 ceph-mon[70044]: pgmap v4357: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:53:28.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:53:27 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:53:30.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:53:29 smithi083 ceph-mon[79930]: pgmap v4358: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:53:30.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:53:29 smithi083 ceph-mon[77242]: pgmap v4358: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:53:30.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:53:29 smithi153 ceph-mon[70044]: pgmap v4358: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:53:30.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:53:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:53:30.667Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:53:31.836 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:53:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:53:31] "GET /metrics HTTP/1.1" 200 225209 "" "Prometheus/2.33.4" 2022-09-29T07:53:32.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:53:31 smithi083 ceph-mon[79930]: pgmap v4359: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:53:32.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:53:31 smithi083 ceph-mon[77242]: pgmap v4359: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:53:32.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:53:31 smithi153 ceph-mon[70044]: pgmap v4359: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:53:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:53:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:53:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:53:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:53:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:53:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:53:34.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:53:33 smithi083 ceph-mon[77242]: pgmap v4360: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:53:34.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:53:33 smithi083 ceph-mon[79930]: pgmap v4360: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:53:34.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:53:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:53:34.060Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:53:34.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:53:33 smithi153 ceph-mon[70044]: pgmap v4360: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:53:35.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:53:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:53:34.761Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:53:36.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:53:35 smithi083 ceph-mon[79930]: pgmap v4361: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:53:36.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:53:35 smithi083 ceph-mon[77242]: pgmap v4361: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:53:36.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:53:35 smithi153 ceph-mon[70044]: pgmap v4361: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:53:38.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:53:37 smithi083 ceph-mon[79930]: pgmap v4362: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:53:38.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:53:37 smithi083 ceph-mon[77242]: pgmap v4362: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:53:38.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:53:37 smithi153 ceph-mon[70044]: pgmap v4362: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:53:40.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:53:39 smithi083 ceph-mon[79930]: pgmap v4363: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:53:40.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:53:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:53:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:53:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:53:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:53:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:53:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:53:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:53:40.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:53:39 smithi083 ceph-mon[77242]: pgmap v4363: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:53:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:53:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:53:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:53:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:53:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:53:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:53:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:53:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:53:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:53:39 smithi153 ceph-mon[70044]: pgmap v4363: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:53:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:53:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:53:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:53:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:53:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:53:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:53:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:53:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:53:40.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:53:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:53:40.667Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:53:41.850 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:53:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:53:41] "GET /metrics HTTP/1.1" 200 225210 "" "Prometheus/2.33.4" 2022-09-29T07:53:42.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:53:41 smithi083 ceph-mon[77242]: pgmap v4364: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:53:42.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:53:41 smithi083 ceph-mon[79930]: pgmap v4364: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:53:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:53:41 smithi153 ceph-mon[70044]: pgmap v4364: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:53:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:53:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:53:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:53:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:53:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:53:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:53:44.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:53:43 smithi083 ceph-mon[77242]: pgmap v4365: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:53:44.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:53:43 smithi083 ceph-mon[79930]: pgmap v4365: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:53:44.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:53:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:53:44.061Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:53:44.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:53:43 smithi153 ceph-mon[70044]: pgmap v4365: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:53:45.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:53:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:53:44.762Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:53:46.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:53:45 smithi083 ceph-mon[79930]: pgmap v4366: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:53:46.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:53:45 smithi083 ceph-mon[77242]: pgmap v4366: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:53:46.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:53:45 smithi153 ceph-mon[70044]: pgmap v4366: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:53:48.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:53:47 smithi083 ceph-mon[79930]: pgmap v4367: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:53:48.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:53:47 smithi083 ceph-mon[77242]: pgmap v4367: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:53:48.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:53:47 smithi153 ceph-mon[70044]: pgmap v4367: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:53:50.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:53:49 smithi083 ceph-mon[79930]: pgmap v4368: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:53:50.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:53:49 smithi083 ceph-mon[77242]: pgmap v4368: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:53:50.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:53:49 smithi153 ceph-mon[70044]: pgmap v4368: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:53:50.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:53:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:53:50.668Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:53:51.866 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:53:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:53:51] "GET /metrics HTTP/1.1" 200 225210 "" "Prometheus/2.33.4" 2022-09-29T07:53:52.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:53:51 smithi083 ceph-mon[77242]: pgmap v4369: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:53:52.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:53:51 smithi083 ceph-mon[79930]: pgmap v4369: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:53:52.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:53:51 smithi153 ceph-mon[70044]: pgmap v4369: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:53:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:53:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:53:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:53:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:53:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:53:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:53:53.471 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:53:53.759 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:53:53.760 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 7m ago 2h 26.5M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:53:53.760 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 5m ago 2h 62.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:53:53.760 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 7m ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:53:53.760 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 5m ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:53:53.761 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 5m ago 2h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:53:53.761 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 7m ago 2h 635M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:53:53.761 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 7m ago 2h 566M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:53:53.761 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 5m ago 2h 252M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:53:53.762 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 7m ago 2h 330M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:53:53.762 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 7m ago 2h 26.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:53:53.762 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 5m ago 2h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:53:53.762 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 7m ago 2h 412M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:53:53.763 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 7m ago 2h 403M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:53:53.763 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 7m ago 2h 319M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:53:53.763 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 7m ago 2h 390M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:53:53.763 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 5m ago 2h 349M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:53:53.764 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 5m ago 2h 427M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:53:53.764 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 5m ago 2h 380M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:53:53.764 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 5m ago 2h 444M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:53:53.764 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 5m ago 2h 125M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:53:54.138 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:53:54.138 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:53:54.139 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:53:54.139 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:53:54.139 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:53:54.139 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:53:54.140 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:53:54.140 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:53:54.140 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:53:54.140 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:53:54.140 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:53:54.141 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:53:54.141 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:53:54.141 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:53:54.142 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:53:54.142 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:53:54.142 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:53:54.142 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:53:54.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:53:53 smithi083 ceph-mon[79930]: pgmap v4370: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:53:54.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:53:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:53:54.062Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:53:54.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:53:53 smithi083 ceph-mon[77242]: pgmap v4370: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:53:54.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:53:53 smithi153 ceph-mon[70044]: pgmap v4370: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:53:55.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:53:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:53:54.762Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:53:55.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:53:54 smithi083 ceph-mon[77242]: from='client.39219 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:53:55.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:53:54 smithi083 ceph-mon[77242]: from='client.39225 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:53:55.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:53:54 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1885541907' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:53:55.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:53:54 smithi083 ceph-mon[79930]: from='client.39219 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:53:55.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:53:54 smithi083 ceph-mon[79930]: from='client.39225 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:53:55.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:53:54 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1885541907' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:53:55.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:53:54 smithi153 ceph-mon[70044]: from='client.39219 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:53:55.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:53:54 smithi153 ceph-mon[70044]: from='client.39225 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:53:55.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:53:54 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1885541907' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:53:56.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:53:55 smithi083 ceph-mon[79930]: pgmap v4371: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:53:56.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:53:55 smithi083 ceph-mon[77242]: pgmap v4371: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:53:56.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:53:55 smithi153 ceph-mon[70044]: pgmap v4371: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:53:58.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:53:57 smithi083 ceph-mon[79930]: pgmap v4372: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:53:58.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:53:57 smithi083 ceph-mon[77242]: pgmap v4372: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:53:58.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:53:57 smithi153 ceph-mon[70044]: pgmap v4372: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:00.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:53:59 smithi083 ceph-mon[79930]: pgmap v4373: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:00.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:53:59 smithi083 ceph-mon[77242]: pgmap v4373: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:53:59 smithi153 ceph-mon[70044]: pgmap v4373: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:00.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:54:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:54:00.669Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:54:01.888 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:54:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:54:01] "GET /metrics HTTP/1.1" 200 225213 "" "Prometheus/2.33.4" 2022-09-29T07:54:02.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:54:01 smithi083 ceph-mon[79930]: pgmap v4374: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:02.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:54:01 smithi083 ceph-mon[77242]: pgmap v4374: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:02.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:54:01 smithi153 ceph-mon[70044]: pgmap v4374: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:54:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:54:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:54:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:54:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:54:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:54:04.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:54:03 smithi083 ceph-mon[79930]: pgmap v4375: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:04.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:54:03 smithi083 ceph-mon[77242]: pgmap v4375: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:04.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:54:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:54:04.062Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:54:04.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:54:03 smithi153 ceph-mon[70044]: pgmap v4375: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:05.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:54:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:54:04.762Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:54:06.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:54:05 smithi083 ceph-mon[77242]: pgmap v4376: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:06.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:54:05 smithi083 ceph-mon[79930]: pgmap v4376: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:06.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:54:05 smithi153 ceph-mon[70044]: pgmap v4376: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:08.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:54:07 smithi083 ceph-mon[79930]: pgmap v4377: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:08.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:54:07 smithi083 ceph-mon[77242]: pgmap v4377: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:08.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:54:07 smithi153 ceph-mon[70044]: pgmap v4377: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:10.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:54:09 smithi083 ceph-mon[79930]: pgmap v4378: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:10.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:54:09 smithi083 ceph-mon[77242]: pgmap v4378: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:10.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:54:09 smithi153 ceph-mon[70044]: pgmap v4378: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:10.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:54:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:54:10.670Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:54:11.902 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:54:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:54:11] "GET /metrics HTTP/1.1" 200 225211 "" "Prometheus/2.33.4" 2022-09-29T07:54:12.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:54:11 smithi083 ceph-mon[79930]: pgmap v4379: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:12.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:54:11 smithi083 ceph-mon[77242]: pgmap v4379: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:12.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:54:11 smithi153 ceph-mon[70044]: pgmap v4379: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:54:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:54:12.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:54:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:54:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:54:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:54:14.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:54:13 smithi083 ceph-mon[77242]: pgmap v4380: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:14.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:54:13 smithi083 ceph-mon[79930]: pgmap v4380: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:14.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:54:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:54:14.063Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:54:14.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:54:13 smithi153 ceph-mon[70044]: pgmap v4380: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:15.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:54:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:54:14.762Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:54:16.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:54:15 smithi083 ceph-mon[77242]: pgmap v4381: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:16.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:54:15 smithi083 ceph-mon[79930]: pgmap v4381: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:16.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:54:15 smithi153 ceph-mon[70044]: pgmap v4381: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:18.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:54:17 smithi083 ceph-mon[79930]: pgmap v4382: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:18.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:54:17 smithi083 ceph-mon[77242]: pgmap v4382: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:18.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:54:17 smithi153 ceph-mon[70044]: pgmap v4382: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:20.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:54:19 smithi083 ceph-mon[79930]: pgmap v4383: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:20.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:54:19 smithi083 ceph-mon[77242]: pgmap v4383: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:20.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:54:19 smithi153 ceph-mon[70044]: pgmap v4383: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:20.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:54:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:54:20.671Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:54:21.917 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:54:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:54:21] "GET /metrics HTTP/1.1" 200 225211 "" "Prometheus/2.33.4" 2022-09-29T07:54:22.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:54:21 smithi083 ceph-mon[79930]: pgmap v4384: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:22.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:54:21 smithi083 ceph-mon[77242]: pgmap v4384: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:22.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:54:21 smithi153 ceph-mon[70044]: pgmap v4384: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:54:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:54:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:54:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:54:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:54:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:54:24.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:54:23 smithi083 ceph-mon[79930]: pgmap v4385: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:24.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:54:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:54:24.065Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:54:24.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:54:23 smithi083 ceph-mon[77242]: pgmap v4385: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:24.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:54:23 smithi153 ceph-mon[70044]: pgmap v4385: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:24.460 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:54:24.746 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:54:24.746 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 8m ago 2h 26.5M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:54:24.746 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 6m ago 2h 62.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:54:24.747 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 8m ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:54:24.747 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 6m ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:54:24.747 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 6m ago 2h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:54:24.747 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 8m ago 2h 635M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:54:24.748 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 8m ago 2h 566M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:54:24.748 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 6m ago 2h 252M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:54:24.748 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 8m ago 2h 330M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:54:24.748 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 8m ago 2h 26.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:54:24.749 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 6m ago 2h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:54:24.749 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 8m ago 2h 412M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:54:24.749 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 8m ago 2h 403M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:54:24.749 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 8m ago 2h 319M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:54:24.749 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 8m ago 2h 390M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:54:24.750 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 6m ago 2h 349M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:54:24.750 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 6m ago 2h 427M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:54:24.750 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 6m ago 2h 380M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:54:24.750 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 6m ago 2h 444M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:54:24.750 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 6m ago 2h 125M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:54:25.106 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:54:25.106 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:54:25.107 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:54:25.107 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:54:25.107 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:54:25.107 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:54:25.107 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:54:25.108 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:54:25.108 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:54:25.108 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:54:25.108 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:54:25.108 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:54:25.109 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:54:25.109 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:54:25.109 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:54:25.109 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:54:25.110 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:54:25.110 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:54:25.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:54:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:54:24.763Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:54:26.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:54:25 smithi083 ceph-mon[77242]: pgmap v4386: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:26.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:54:25 smithi083 ceph-mon[77242]: from='client.49171 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:54:26.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:54:25 smithi083 ceph-mon[77242]: from='client.49177 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:54:26.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:54:25 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3616894823' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:54:26.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:54:25 smithi083 ceph-mon[79930]: pgmap v4386: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:26.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:54:25 smithi083 ceph-mon[79930]: from='client.49171 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:54:26.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:54:25 smithi083 ceph-mon[79930]: from='client.49177 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:54:26.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:54:25 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3616894823' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:54:26.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:54:25 smithi153 ceph-mon[70044]: pgmap v4386: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:26.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:54:25 smithi153 ceph-mon[70044]: from='client.49171 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:54:26.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:54:25 smithi153 ceph-mon[70044]: from='client.49177 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:54:26.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:54:25 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3616894823' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:54:27.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:54:27 smithi083 ceph-mon[79930]: pgmap v4387: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:27.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:54:27 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:54:27.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:54:27 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:54:27.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:54:27 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:54:27.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:54:27 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:54:27.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:54:27 smithi083 ceph-mon[77242]: pgmap v4387: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:27.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:54:27 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:54:27.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:54:27 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:54:27.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:54:27 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:54:27.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:54:27 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:54:28.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:54:27 smithi153 ceph-mon[70044]: pgmap v4387: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:28.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:54:27 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:54:28.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:54:27 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:54:28.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:54:27 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:54:28.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:54:27 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:54:29.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:54:29 smithi083 ceph-mon[79930]: pgmap v4388: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:29.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:54:29 smithi083 ceph-mon[77242]: pgmap v4388: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:30.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:54:29 smithi153 ceph-mon[70044]: pgmap v4388: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:30.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:54:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:54:30.672Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:54:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:54:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:54:31] "GET /metrics HTTP/1.1" 200 225210 "" "Prometheus/2.33.4" 2022-09-29T07:54:31.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:54:31 smithi083 ceph-mon[79930]: pgmap v4389: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:31.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:54:31 smithi083 ceph-mon[77242]: pgmap v4389: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:32.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:54:31 smithi153 ceph-mon[70044]: pgmap v4389: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:54:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:54:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:54:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:54:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:54:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:54:33.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:54:33 smithi083 ceph-mon[79930]: pgmap v4390: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:33.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:54:33 smithi083 ceph-mon[77242]: pgmap v4390: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:34.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:54:33 smithi153 ceph-mon[70044]: pgmap v4390: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:34.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:54:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:54:34.065Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:54:35.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:54:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:54:34.764Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:54:35.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:54:35 smithi083 ceph-mon[79930]: pgmap v4391: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:35.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:54:35 smithi083 ceph-mon[77242]: pgmap v4391: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:36.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:54:35 smithi153 ceph-mon[70044]: pgmap v4391: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:37.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:54:37 smithi083 ceph-mon[79930]: pgmap v4392: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:37.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:54:37 smithi083 ceph-mon[77242]: pgmap v4392: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:38.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:54:37 smithi153 ceph-mon[70044]: pgmap v4392: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:39.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:54:39 smithi083 ceph-mon[79930]: pgmap v4393: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:39.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:54:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:54:39.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:54:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:54:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:54:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:54:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:54:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:54:39.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:54:39 smithi083 ceph-mon[77242]: pgmap v4393: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:54:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:54:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:54:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:54:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:54:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:54:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:54:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:54:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:54:39 smithi153 ceph-mon[70044]: pgmap v4393: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:54:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:54:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:54:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:54:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:54:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:54:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:54:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:54:40.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:54:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:54:40.672Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:54:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:54:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:54:41] "GET /metrics HTTP/1.1" 200 225207 "" "Prometheus/2.33.4" 2022-09-29T07:54:41.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:54:41 smithi083 ceph-mon[79930]: pgmap v4394: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:41.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:54:41 smithi083 ceph-mon[77242]: pgmap v4394: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:42.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:54:41 smithi153 ceph-mon[70044]: pgmap v4394: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:54:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:54:42.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:54:43.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:54:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:54:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:54:43.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:54:43 smithi083 ceph-mon[79930]: pgmap v4395: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:43.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:54:43 smithi083 ceph-mon[77242]: pgmap v4395: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:44.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:54:43 smithi153 ceph-mon[70044]: pgmap v4395: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:44.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:54:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:54:44.065Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:54:45.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:54:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:54:44.765Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:54:45.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:54:45 smithi083 ceph-mon[79930]: pgmap v4396: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:45.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:54:45 smithi083 ceph-mon[77242]: pgmap v4396: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:46.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:54:45 smithi153 ceph-mon[70044]: pgmap v4396: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:47.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:54:47 smithi083 ceph-mon[79930]: pgmap v4397: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:47.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:54:47 smithi083 ceph-mon[77242]: pgmap v4397: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:48.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:54:47 smithi153 ceph-mon[70044]: pgmap v4397: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:49.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:54:49 smithi083 ceph-mon[79930]: pgmap v4398: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:49.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:54:49 smithi083 ceph-mon[77242]: pgmap v4398: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:50.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:54:49 smithi153 ceph-mon[70044]: pgmap v4398: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:50.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:54:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:54:50.673Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:54:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:54:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:54:51] "GET /metrics HTTP/1.1" 200 225207 "" "Prometheus/2.33.4" 2022-09-29T07:54:51.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:54:51 smithi083 ceph-mon[79930]: pgmap v4399: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:51.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:54:51 smithi083 ceph-mon[77242]: pgmap v4399: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:52.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:54:51 smithi153 ceph-mon[70044]: pgmap v4399: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:54:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:54:52.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:54:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:54:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:54:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:54:53.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:54:53 smithi083 ceph-mon[79930]: pgmap v4400: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:53.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:54:53 smithi083 ceph-mon[77242]: pgmap v4400: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:54.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:54:53 smithi153 ceph-mon[70044]: pgmap v4400: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:54.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:54:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:54:54.073Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:54:55.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:54:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:54:54.765Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:54:55.432 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:54:55.725 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:54:55.725 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 8m ago 2h 26.5M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:54:55.726 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 6m ago 2h 62.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:54:55.726 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 8m ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:54:55.726 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 6m ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:54:55.726 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 6m ago 2h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:54:55.727 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 8m ago 2h 635M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:54:55.727 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 8m ago 2h 566M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:54:55.727 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 6m ago 2h 252M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:54:55.727 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 8m ago 2h 330M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:54:55.728 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 8m ago 2h 26.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:54:55.728 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 6m ago 2h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:54:55.728 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 8m ago 2h 412M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:54:55.728 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 8m ago 2h 403M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:54:55.728 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 8m ago 2h 319M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:54:55.729 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 8m ago 2h 390M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:54:55.729 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 6m ago 2h 349M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:54:55.729 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 6m ago 2h 427M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:54:55.729 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 6m ago 2h 380M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:54:55.730 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 6m ago 2h 444M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:54:55.730 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 6m ago 2h 125M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:54:55.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:54:55 smithi083 ceph-mon[79930]: pgmap v4401: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:55.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:54:55 smithi083 ceph-mon[77242]: pgmap v4401: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:56.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:54:55 smithi153 ceph-mon[70044]: pgmap v4401: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:56.081 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:54:56.081 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:54:56.082 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:54:56.082 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:54:56.082 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:54:56.082 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:54:56.083 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:54:56.083 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:54:56.083 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:54:56.083 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:54:56.083 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:54:56.084 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:54:56.084 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:54:56.084 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:54:56.084 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:54:56.085 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:54:56.085 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:54:56.085 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:54:56.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:54:56 smithi083 ceph-mon[79930]: from='client.39255 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:54:56.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:54:56 smithi083 ceph-mon[79930]: from='client.39261 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:54:56.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:54:56 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2202772638' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:54:56.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:54:56 smithi083 ceph-mon[77242]: from='client.39255 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:54:56.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:54:56 smithi083 ceph-mon[77242]: from='client.39261 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:54:56.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:54:56 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2202772638' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:54:57.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:54:56 smithi153 ceph-mon[70044]: from='client.39255 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:54:57.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:54:56 smithi153 ceph-mon[70044]: from='client.39261 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:54:57.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:54:56 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2202772638' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:54:57.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:54:57 smithi083 ceph-mon[77242]: pgmap v4402: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:57.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:54:57 smithi083 ceph-mon[79930]: pgmap v4402: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:58.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:54:57 smithi153 ceph-mon[70044]: pgmap v4402: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:59.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:54:59 smithi083 ceph-mon[77242]: pgmap v4403: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:54:59.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:54:59 smithi083 ceph-mon[79930]: pgmap v4403: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:00.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:54:59 smithi153 ceph-mon[70044]: pgmap v4403: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:00.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:55:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:55:00.674Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:55:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:55:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:55:01] "GET /metrics HTTP/1.1" 200 225211 "" "Prometheus/2.33.4" 2022-09-29T07:55:01.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:55:01 smithi083 ceph-mon[79930]: pgmap v4404: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:01.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:55:01 smithi083 ceph-mon[77242]: pgmap v4404: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:02.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:55:01 smithi153 ceph-mon[70044]: pgmap v4404: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:55:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:55:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:55:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:55:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:55:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:55:03.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:55:03 smithi083 ceph-mon[79930]: pgmap v4405: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:03.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:55:03 smithi083 ceph-mon[77242]: pgmap v4405: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:04.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:55:03 smithi153 ceph-mon[70044]: pgmap v4405: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:04.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:55:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:55:04.067Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:55:05.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:55:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:55:04.766Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:55:05.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:55:05 smithi083 ceph-mon[77242]: pgmap v4406: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:05.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:55:05 smithi083 ceph-mon[79930]: pgmap v4406: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:06.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:55:05 smithi153 ceph-mon[70044]: pgmap v4406: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:07.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:55:07 smithi083 ceph-mon[79930]: pgmap v4407: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:07.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:55:07 smithi083 ceph-mon[77242]: pgmap v4407: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:08.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:55:07 smithi153 ceph-mon[70044]: pgmap v4407: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:09.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:55:09 smithi083 ceph-mon[79930]: pgmap v4408: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:09.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:55:09 smithi083 ceph-mon[77242]: pgmap v4408: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:10.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:55:09 smithi153 ceph-mon[70044]: pgmap v4408: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:10.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:55:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:55:10.674Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:55:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:55:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:55:11] "GET /metrics HTTP/1.1" 200 225210 "" "Prometheus/2.33.4" 2022-09-29T07:55:11.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:55:11 smithi083 ceph-mon[79930]: pgmap v4409: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:11.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:55:11 smithi083 ceph-mon[77242]: pgmap v4409: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:12.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:55:11 smithi153 ceph-mon[70044]: pgmap v4409: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:55:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:55:12.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:55:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:55:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:55:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:55:13.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:55:13 smithi083 ceph-mon[79930]: pgmap v4410: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:13.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:55:13 smithi083 ceph-mon[77242]: pgmap v4410: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:14.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:55:13 smithi153 ceph-mon[70044]: pgmap v4410: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:14.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:55:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:55:14.067Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:55:15.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:55:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:55:14.766Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:55:15.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:55:15 smithi083 ceph-mon[79930]: pgmap v4411: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:15.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:55:15 smithi083 ceph-mon[77242]: pgmap v4411: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:16.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:55:15 smithi153 ceph-mon[70044]: pgmap v4411: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:17.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:55:17 smithi083 ceph-mon[79930]: pgmap v4412: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:17.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:55:17 smithi083 ceph-mon[77242]: pgmap v4412: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:18.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:55:17 smithi153 ceph-mon[70044]: pgmap v4412: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:19.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:55:19 smithi083 ceph-mon[79930]: pgmap v4413: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:19.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:55:19 smithi083 ceph-mon[77242]: pgmap v4413: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:20.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:55:19 smithi153 ceph-mon[70044]: pgmap v4413: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:20.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:55:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:55:20.674Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:55:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:55:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:55:21] "GET /metrics HTTP/1.1" 200 225210 "" "Prometheus/2.33.4" 2022-09-29T07:55:21.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:55:21 smithi083 ceph-mon[79930]: pgmap v4414: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:21.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:55:21 smithi083 ceph-mon[77242]: pgmap v4414: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:22.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:55:21 smithi153 ceph-mon[70044]: pgmap v4414: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:55:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:55:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:55:23.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:55:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:55:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:55:23.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:55:23 smithi083 ceph-mon[79930]: pgmap v4415: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:23.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:55:23 smithi083 ceph-mon[77242]: pgmap v4415: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:24.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:55:23 smithi153 ceph-mon[70044]: pgmap v4415: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:24.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:55:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:55:24.073Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:55:25.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:55:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:55:24.767Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:55:25.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:55:25 smithi083 ceph-mon[79930]: pgmap v4416: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:25.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:55:25 smithi083 ceph-mon[77242]: pgmap v4416: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:26.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:55:25 smithi153 ceph-mon[70044]: pgmap v4416: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:26.409 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:55:26.701 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:55:26.701 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 9m ago 2h 26.5M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:55:26.701 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 7m ago 2h 62.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:55:26.702 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 9m ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:55:26.702 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 7m ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:55:26.702 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 7m ago 2h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:55:26.703 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 9m ago 2h 635M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:55:26.703 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 9m ago 2h 566M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:55:26.703 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 7m ago 2h 252M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:55:26.703 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 9m ago 2h 330M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:55:26.703 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 9m ago 2h 26.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:55:26.704 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 7m ago 2h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:55:26.704 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 9m ago 2h 412M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:55:26.704 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 9m ago 2h 403M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:55:26.704 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 9m ago 2h 319M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:55:26.705 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 9m ago 2h 390M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:55:26.705 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 7m ago 2h 349M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:55:26.705 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 7m ago 2h 427M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:55:26.705 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 7m ago 2h 380M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:55:26.706 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 7m ago 2h 444M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:55:26.706 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 7m ago 2h 125M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:55:27.056 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:55:27.056 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:55:27.057 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:55:27.057 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:55:27.057 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:55:27.057 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:55:27.058 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:55:27.058 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:55:27.058 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:55:27.058 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:55:27.058 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:55:27.059 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:55:27.059 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:55:27.059 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:55:27.059 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:55:27.059 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:55:27.060 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:55:27.060 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:55:27.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:55:27 smithi083 ceph-mon[79930]: pgmap v4417: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:27.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:55:27 smithi083 ceph-mon[79930]: from='client.49207 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:55:27.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:55:27 smithi083 ceph-mon[79930]: from='client.49213 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:55:27.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:55:27 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1363030312' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:55:27.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:55:27 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:55:27.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:55:27 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:55:27.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:55:27 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:55:27.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:55:27 smithi083 ceph-mon[77242]: pgmap v4417: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:27.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:55:27 smithi083 ceph-mon[77242]: from='client.49207 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:55:27.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:55:27 smithi083 ceph-mon[77242]: from='client.49213 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:55:27.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:55:27 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1363030312' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:55:27.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:55:27 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:55:27.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:55:27 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:55:27.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:55:27 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:55:28.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:55:27 smithi153 ceph-mon[70044]: pgmap v4417: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:28.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:55:27 smithi153 ceph-mon[70044]: from='client.49207 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:55:28.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:55:27 smithi153 ceph-mon[70044]: from='client.49213 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:55:28.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:55:27 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1363030312' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:55:28.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:55:27 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:55:28.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:55:27 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:55:28.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:55:27 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:55:29.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:55:29 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:55:29.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:55:29 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:55:29.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:55:29 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:55:30.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:55:30 smithi083 ceph-mon[79930]: pgmap v4418: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:30.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:55:30 smithi083 ceph-mon[77242]: pgmap v4418: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:30.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:55:30 smithi153 ceph-mon[70044]: pgmap v4418: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:30.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:55:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:55:30.675Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:55:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:55:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:55:31] "GET /metrics HTTP/1.1" 200 225211 "" "Prometheus/2.33.4" 2022-09-29T07:55:32.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:55:32 smithi083 ceph-mon[79930]: pgmap v4419: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:32.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:55:32 smithi083 ceph-mon[77242]: pgmap v4419: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:32.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:55:32 smithi153 ceph-mon[70044]: pgmap v4419: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:32.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:55:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:55:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:55:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:55:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:55:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:55:34.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:55:34 smithi083 ceph-mon[79930]: pgmap v4420: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:34.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:55:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:55:34.069Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:55:34.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:55:34 smithi083 ceph-mon[77242]: pgmap v4420: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:34.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:55:34 smithi153 ceph-mon[70044]: pgmap v4420: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:35.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:55:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:55:34.767Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:55:36.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:55:36 smithi083 ceph-mon[79930]: pgmap v4421: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:36.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:55:36 smithi083 ceph-mon[77242]: pgmap v4421: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:36.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:55:36 smithi153 ceph-mon[70044]: pgmap v4421: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:38.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:55:38 smithi083 ceph-mon[79930]: pgmap v4422: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:38.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:55:38 smithi083 ceph-mon[77242]: pgmap v4422: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:38.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:55:38 smithi153 ceph-mon[70044]: pgmap v4422: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:40.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:55:40 smithi083 ceph-mon[79930]: pgmap v4423: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:40.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:55:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:55:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:55:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:55:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:55:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:55:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:55:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:55:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:55:40 smithi083 ceph-mon[77242]: pgmap v4423: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:55:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:55:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:55:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:55:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:55:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:55:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:55:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:55:40.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:55:40 smithi153 ceph-mon[70044]: pgmap v4423: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:55:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:55:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:55:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:55:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:55:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:55:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:55:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:55:40.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:55:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:55:40.676Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:55:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:55:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:55:41] "GET /metrics HTTP/1.1" 200 225208 "" "Prometheus/2.33.4" 2022-09-29T07:55:42.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:55:42 smithi083 ceph-mon[79930]: pgmap v4424: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:42.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:55:42 smithi083 ceph-mon[77242]: pgmap v4424: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:55:42 smithi153 ceph-mon[70044]: pgmap v4424: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:42.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:55:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:55:42.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:55:43.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:55:43 smithi153 ceph-mon[70044]: pgmap v4425: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:55:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:55:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:55:43.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:55:43 smithi083 ceph-mon[79930]: pgmap v4425: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:43.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:55:43 smithi083 ceph-mon[77242]: pgmap v4425: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:44.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:55:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:55:44.070Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:55:45.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:55:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:55:44.768Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:55:45.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:55:45 smithi083 ceph-mon[79930]: pgmap v4426: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:45.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:55:45 smithi083 ceph-mon[77242]: pgmap v4426: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:45.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:55:45 smithi153 ceph-mon[70044]: pgmap v4426: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:47.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:55:47 smithi083 ceph-mon[79930]: pgmap v4427: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:47.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:55:47 smithi083 ceph-mon[77242]: pgmap v4427: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:48.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:55:47 smithi153 ceph-mon[70044]: pgmap v4427: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:49.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:55:49 smithi083 ceph-mon[79930]: pgmap v4428: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:49.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:55:49 smithi083 ceph-mon[77242]: pgmap v4428: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:50.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:55:49 smithi153 ceph-mon[70044]: pgmap v4428: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:50.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:55:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:55:50.677Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:55:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:55:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:55:51] "GET /metrics HTTP/1.1" 200 225208 "" "Prometheus/2.33.4" 2022-09-29T07:55:51.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:55:51 smithi083 ceph-mon[79930]: pgmap v4429: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:51.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:55:51 smithi083 ceph-mon[77242]: pgmap v4429: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:52.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:55:51 smithi153 ceph-mon[70044]: pgmap v4429: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:55:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:55:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:55:53.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:55:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:55:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:55:53.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:55:53 smithi083 ceph-mon[79930]: pgmap v4430: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:53.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:55:53 smithi083 ceph-mon[77242]: pgmap v4430: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:54.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:55:53 smithi153 ceph-mon[70044]: pgmap v4430: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:54.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:55:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:55:54.072Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:55:55.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:55:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:55:54.768Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:55:55.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:55:55 smithi083 ceph-mon[77242]: pgmap v4431: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:55.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:55:55 smithi083 ceph-mon[79930]: pgmap v4431: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:56.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:55:55 smithi153 ceph-mon[70044]: pgmap v4431: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:57.381 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:55:57.669 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:55:57.670 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 9m ago 2h 26.5M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:55:57.670 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 7m ago 2h 62.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:55:57.670 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 9m ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:55:57.671 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 7m ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:55:57.671 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 7m ago 2h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:55:57.671 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 9m ago 2h 635M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:55:57.671 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 9m ago 2h 566M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:55:57.671 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 7m ago 2h 252M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:55:57.672 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 9m ago 2h 330M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:55:57.672 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 9m ago 2h 26.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:55:57.672 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 7m ago 2h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:55:57.672 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 9m ago 2h 412M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:55:57.672 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 9m ago 2h 403M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:55:57.673 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 9m ago 2h 319M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:55:57.673 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 9m ago 2h 390M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:55:57.673 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 7m ago 2h 349M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:55:57.673 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 7m ago 2h 427M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:55:57.674 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 7m ago 2h 380M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:55:57.674 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 7m ago 2h 444M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:55:57.674 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 7m ago 2h 125M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:55:57.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:55:57 smithi083 ceph-mon[79930]: pgmap v4432: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:57.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:55:57 smithi083 ceph-mon[77242]: pgmap v4432: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:58.025 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:55:58.025 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:55:58.026 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:55:58.026 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:55:58.026 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:55:58.027 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:55:58.027 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:55:58.027 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:55:58.027 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:55:58.028 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:55:58.028 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:55:58.028 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:55:58.028 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:55:58.028 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:55:58.029 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:55:58.029 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:55:58.029 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:55:58.029 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:55:58.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:55:57 smithi153 ceph-mon[70044]: pgmap v4432: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:58.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:55:58 smithi083 ceph-mon[79930]: from='client.39291 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:55:58.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:55:58 smithi083 ceph-mon[79930]: from='client.39297 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:55:58.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:55:58 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4136257247' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:55:58.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:55:58 smithi083 ceph-mon[77242]: from='client.39291 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:55:58.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:55:58 smithi083 ceph-mon[77242]: from='client.39297 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:55:58.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:55:58 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4136257247' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:55:59.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:55:58 smithi153 ceph-mon[70044]: from='client.39291 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:55:59.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:55:58 smithi153 ceph-mon[70044]: from='client.39297 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:55:59.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:55:58 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4136257247' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:55:59.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:55:59 smithi083 ceph-mon[79930]: pgmap v4433: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:55:59.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:55:59 smithi083 ceph-mon[77242]: pgmap v4433: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:00.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:55:59 smithi153 ceph-mon[70044]: pgmap v4433: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:00.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:56:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:56:00.678Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:56:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:56:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:56:01] "GET /metrics HTTP/1.1" 200 225209 "" "Prometheus/2.33.4" 2022-09-29T07:56:01.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:56:01 smithi083 ceph-mon[79930]: pgmap v4434: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:01.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:56:01 smithi083 ceph-mon[77242]: pgmap v4434: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:02.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:56:01 smithi153 ceph-mon[70044]: pgmap v4434: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:56:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:56:02.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:56:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:56:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:56:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:56:03.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:56:03 smithi083 ceph-mon[79930]: pgmap v4435: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:03.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:56:03 smithi083 ceph-mon[77242]: pgmap v4435: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:04.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:56:03 smithi153 ceph-mon[70044]: pgmap v4435: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:04.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:56:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:56:04.070Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:56:05.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:56:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:56:04.769Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:56:05.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:56:05 smithi083 ceph-mon[79930]: pgmap v4436: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:05.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:56:05 smithi083 ceph-mon[77242]: pgmap v4436: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:06.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:56:05 smithi153 ceph-mon[70044]: pgmap v4436: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:07.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:56:07 smithi083 ceph-mon[77242]: pgmap v4437: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:07.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:56:07 smithi083 ceph-mon[79930]: pgmap v4437: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:08.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:56:07 smithi153 ceph-mon[70044]: pgmap v4437: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:09.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:56:09 smithi083 ceph-mon[79930]: pgmap v4438: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:09.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:56:09 smithi083 ceph-mon[77242]: pgmap v4438: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:10.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:56:09 smithi153 ceph-mon[70044]: pgmap v4438: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:10.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:56:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:56:10.679Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:56:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:56:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:56:11] "GET /metrics HTTP/1.1" 200 225210 "" "Prometheus/2.33.4" 2022-09-29T07:56:11.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:56:11 smithi083 ceph-mon[79930]: pgmap v4439: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:11.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:56:11 smithi083 ceph-mon[77242]: pgmap v4439: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:12.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:56:11 smithi153 ceph-mon[70044]: pgmap v4439: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:56:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:56:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:56:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:56:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:56:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:56:13.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:56:13 smithi083 ceph-mon[79930]: pgmap v4440: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:13.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:56:13 smithi083 ceph-mon[77242]: pgmap v4440: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:14.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:56:13 smithi153 ceph-mon[70044]: pgmap v4440: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:14.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:56:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:56:14.071Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:56:15.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:56:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:56:14.770Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:56:15.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:56:15 smithi083 ceph-mon[79930]: pgmap v4441: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:15.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:56:15 smithi083 ceph-mon[77242]: pgmap v4441: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:16.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:56:15 smithi153 ceph-mon[70044]: pgmap v4441: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:17.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:56:17 smithi083 ceph-mon[79930]: pgmap v4442: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:17.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:56:17 smithi083 ceph-mon[77242]: pgmap v4442: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:18.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:56:17 smithi153 ceph-mon[70044]: pgmap v4442: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:19.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:56:19 smithi083 ceph-mon[79930]: pgmap v4443: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:19.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:56:19 smithi083 ceph-mon[77242]: pgmap v4443: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:20.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:56:19 smithi153 ceph-mon[70044]: pgmap v4443: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:20.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:56:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:56:20.680Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:56:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:56:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:56:21] "GET /metrics HTTP/1.1" 200 225210 "" "Prometheus/2.33.4" 2022-09-29T07:56:21.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:56:21 smithi083 ceph-mon[79930]: pgmap v4444: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:21.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:56:21 smithi083 ceph-mon[77242]: pgmap v4444: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:22.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:56:21 smithi153 ceph-mon[70044]: pgmap v4444: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:56:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:56:22.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:56:23.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:56:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:56:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:56:23.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:56:23 smithi083 ceph-mon[79930]: pgmap v4445: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:23.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:56:23 smithi083 ceph-mon[77242]: pgmap v4445: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:24.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:56:23 smithi153 ceph-mon[70044]: pgmap v4445: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:24.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:56:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:56:24.079Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:56:25.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:56:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:56:24.770Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:56:25.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:56:25 smithi083 ceph-mon[79930]: pgmap v4446: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:25.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:56:25 smithi083 ceph-mon[77242]: pgmap v4446: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:26.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:56:25 smithi153 ceph-mon[70044]: pgmap v4446: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:27.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:56:27 smithi083 ceph-mon[79930]: pgmap v4447: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:27.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:56:27 smithi083 ceph-mon[77242]: pgmap v4447: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:28.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:56:27 smithi153 ceph-mon[70044]: pgmap v4447: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:28.360 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:56:28.671 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:56:28.672 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 10m ago 2h 26.5M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:56:28.672 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 8m ago 2h 62.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:56:28.672 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 10m ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:56:28.673 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 8m ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:56:28.673 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 8m ago 2h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:56:28.673 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 10m ago 2h 635M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:56:28.673 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 10m ago 2h 566M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:56:28.673 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 8m ago 2h 252M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:56:28.674 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 10m ago 2h 330M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:56:28.674 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 10m ago 2h 26.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:56:28.674 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 8m ago 2h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:56:28.675 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 10m ago 2h 412M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:56:28.675 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 10m ago 2h 403M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:56:28.675 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 10m ago 2h 319M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:56:28.675 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 10m ago 2h 390M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:56:28.675 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 8m ago 2h 349M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:56:28.676 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 8m ago 2h 427M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:56:28.676 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 8m ago 2h 380M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:56:28.676 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 8m ago 2h 444M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:56:28.677 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 8m ago 2h 125M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:56:28.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:56:28 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:56:28.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:56:28 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:56:28.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:56:28 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:56:28.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:56:28 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:56:28.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:56:28 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:56:28.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:56:28 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:56:28.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:56:28 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:56:28.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:56:28 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:56:28.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:56:28 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:56:28.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:56:28 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:56:29.053 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:56:29.054 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:56:29.054 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:56:29.054 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:56:29.054 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:56:29.055 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:56:29.055 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:56:29.055 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:56:29.055 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:56:29.056 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:56:29.056 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:56:29.056 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:56:29.056 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:56:29.057 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:56:29.057 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:56:29.057 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:56:29.057 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:56:29.057 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:56:29.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:56:28 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:56:29.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:56:28 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:56:29.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:56:28 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:56:29.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:56:28 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:56:29.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:56:28 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:56:29.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:56:29 smithi083 ceph-mon[79930]: pgmap v4448: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:29.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:56:29 smithi083 ceph-mon[79930]: from='client.49243 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:56:29.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:56:29 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:56:29.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:56:29 smithi083 ceph-mon[79930]: from='client.39315 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:56:29.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:56:29 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:56:29.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:56:29 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3828065904' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:56:29.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:56:29 smithi083 ceph-mon[77242]: pgmap v4448: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:29.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:56:29 smithi083 ceph-mon[77242]: from='client.49243 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:56:29.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:56:29 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:56:29.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:56:29 smithi083 ceph-mon[77242]: from='client.39315 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:56:29.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:56:29 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:56:29.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:56:29 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3828065904' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:56:30.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:56:29 smithi153 ceph-mon[70044]: pgmap v4448: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:30.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:56:29 smithi153 ceph-mon[70044]: from='client.49243 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:56:30.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:56:29 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:56:30.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:56:29 smithi153 ceph-mon[70044]: from='client.39315 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:56:30.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:56:29 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:56:30.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:56:29 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3828065904' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:56:30.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:56:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:56:30.680Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:56:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:56:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:56:31] "GET /metrics HTTP/1.1" 200 225212 "" "Prometheus/2.33.4" 2022-09-29T07:56:31.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:56:31 smithi083 ceph-mon[79930]: pgmap v4449: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:31.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:56:31 smithi083 ceph-mon[77242]: pgmap v4449: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:32.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:56:31 smithi153 ceph-mon[70044]: pgmap v4449: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:56:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:56:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:56:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:56:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:56:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:56:33.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:56:33 smithi083 ceph-mon[77242]: pgmap v4450: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:33.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:56:33 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:56:33.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:56:33 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:56:33.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:56:33 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:56:33.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:56:33 smithi083 ceph-mon[79930]: pgmap v4450: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:33.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:56:33 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:56:33.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:56:33 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:56:33.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:56:33 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:56:34.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:56:33 smithi153 ceph-mon[70044]: pgmap v4450: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:34.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:56:33 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:56:34.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:56:33 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:56:34.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:56:33 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:56:34.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:56:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:56:34.072Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:56:35.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:56:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:56:34.771Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:56:35.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:56:35 smithi083 ceph-mon[77242]: pgmap v4451: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:35.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:56:35 smithi083 ceph-mon[79930]: pgmap v4451: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:36.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:56:35 smithi153 ceph-mon[70044]: pgmap v4451: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:37.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:56:37 smithi083 ceph-mon[77242]: pgmap v4452: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:37.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:56:37 smithi083 ceph-mon[79930]: pgmap v4452: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:38.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:56:37 smithi153 ceph-mon[70044]: pgmap v4452: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:39.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:56:39 smithi083 ceph-mon[79930]: pgmap v4453: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:39.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:56:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:56:39.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:56:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:56:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:56:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:56:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:56:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:56:39.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:56:39 smithi083 ceph-mon[77242]: pgmap v4453: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:56:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:56:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:56:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:56:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:56:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:56:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:56:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:56:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:56:39 smithi153 ceph-mon[70044]: pgmap v4453: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:56:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:56:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:56:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:56:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:56:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:56:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:56:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:56:40.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:56:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:56:40.681Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:56:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:56:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:56:41] "GET /metrics HTTP/1.1" 200 225211 "" "Prometheus/2.33.4" 2022-09-29T07:56:41.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:56:41 smithi083 ceph-mon[79930]: pgmap v4454: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:41.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:56:41 smithi083 ceph-mon[77242]: pgmap v4454: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:42.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:56:41 smithi153 ceph-mon[70044]: pgmap v4454: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:56:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:56:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:56:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:56:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:56:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:56:43.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:56:43 smithi083 ceph-mon[79930]: pgmap v4455: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:43.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:56:43 smithi083 ceph-mon[77242]: pgmap v4455: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:44.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:56:43 smithi153 ceph-mon[70044]: pgmap v4455: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:44.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:56:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:56:44.073Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:56:45.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:56:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:56:44.771Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:56:45.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:56:45 smithi083 ceph-mon[79930]: pgmap v4456: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:45.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:56:45 smithi083 ceph-mon[77242]: pgmap v4456: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:46.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:56:45 smithi153 ceph-mon[70044]: pgmap v4456: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:47.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:56:47 smithi083 ceph-mon[79930]: pgmap v4457: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:47.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:56:47 smithi083 ceph-mon[77242]: pgmap v4457: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:48.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:56:47 smithi153 ceph-mon[70044]: pgmap v4457: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:49.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:56:49 smithi083 ceph-mon[79930]: pgmap v4458: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:49.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:56:49 smithi083 ceph-mon[77242]: pgmap v4458: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:50.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:56:49 smithi153 ceph-mon[70044]: pgmap v4458: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:50.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:56:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:56:50.681Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:56:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:56:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:56:51] "GET /metrics HTTP/1.1" 200 225211 "" "Prometheus/2.33.4" 2022-09-29T07:56:51.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:56:51 smithi083 ceph-mon[79930]: pgmap v4459: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:51.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:56:51 smithi083 ceph-mon[77242]: pgmap v4459: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:52.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:56:51 smithi153 ceph-mon[70044]: pgmap v4459: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:56:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:56:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:56:53.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:56:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:56:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:56:53.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:56:53 smithi083 ceph-mon[79930]: pgmap v4460: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:53.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:56:53 smithi083 ceph-mon[77242]: pgmap v4460: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:54.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:56:53 smithi153 ceph-mon[70044]: pgmap v4460: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:54.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:56:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:56:54.081Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:56:55.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:56:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:56:54.773Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:56:55.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:56:55 smithi083 ceph-mon[79930]: pgmap v4461: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:55.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:56:55 smithi083 ceph-mon[77242]: pgmap v4461: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:56.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:56:55 smithi153 ceph-mon[70044]: pgmap v4461: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:57.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:56:57 smithi083 ceph-mon[79930]: pgmap v4462: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:57.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:56:57 smithi083 ceph-mon[77242]: pgmap v4462: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:58.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:56:57 smithi153 ceph-mon[70044]: pgmap v4462: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:59.381 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:56:59.671 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:56:59.671 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 27s ago 2h 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:56:59.671 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 8m ago 2h 62.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:56:59.672 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 27s ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:56:59.672 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 8m ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:56:59.672 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 8m ago 2h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:56:59.672 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 27s ago 2h 644M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:56:59.673 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 27s ago 2h 601M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:56:59.673 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 8m ago 2h 252M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:56:59.673 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 27s ago 2h 351M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:56:59.673 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 27s ago 2h 26.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:56:59.674 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 8m ago 2h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:56:59.674 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 27s ago 2h 437M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:56:59.674 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 27s ago 2h 428M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:56:59.674 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 27s ago 2h 339M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:56:59.675 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 27s ago 2h 415M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:56:59.675 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 8m ago 2h 349M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:56:59.675 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 8m ago 2h 427M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:56:59.675 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 8m ago 2h 380M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:56:59.676 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 8m ago 2h 444M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:56:59.676 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 8m ago 2h 125M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:56:59.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:56:59 smithi083 ceph-mon[79930]: pgmap v4463: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:56:59.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:56:59 smithi083 ceph-mon[77242]: pgmap v4463: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:57:00.027 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:57:00.028 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:57:00.028 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:57:00.028 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:57:00.028 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:57:00.028 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:57:00.029 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:57:00.029 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:57:00.029 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:57:00.029 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:57:00.030 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:57:00.030 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:57:00.030 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:57:00.030 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:57:00.030 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:57:00.031 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:57:00.031 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:57:00.031 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:57:00.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:56:59 smithi153 ceph-mon[70044]: pgmap v4463: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:57:00.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:57:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:57:00.682Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:57:00.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:57:00 smithi083 ceph-mon[77242]: from='client.49261 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:57:00.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:57:00 smithi083 ceph-mon[77242]: from='client.49267 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:57:00.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:57:00 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1068063507' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:57:00.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:57:00 smithi083 ceph-mon[79930]: from='client.49261 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:57:00.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:57:00 smithi083 ceph-mon[79930]: from='client.49267 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:57:00.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:57:00 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1068063507' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:57:01.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:57:00 smithi153 ceph-mon[70044]: from='client.49261 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:57:01.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:57:00 smithi153 ceph-mon[70044]: from='client.49267 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:57:01.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:57:00 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1068063507' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:57:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:57:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:57:01] "GET /metrics HTTP/1.1" 200 225207 "" "Prometheus/2.33.4" 2022-09-29T07:57:01.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:57:01 smithi083 ceph-mon[77242]: pgmap v4464: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:57:01.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:57:01 smithi083 ceph-mon[79930]: pgmap v4464: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:57:02.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:57:01 smithi153 ceph-mon[70044]: pgmap v4464: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:57:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:57:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:57:02.318Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:57:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:57:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:57:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:57:03.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:57:03 smithi083 ceph-mon[79930]: pgmap v4465: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:57:03.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:57:03 smithi083 ceph-mon[77242]: pgmap v4465: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:57:04.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:57:03 smithi153 ceph-mon[70044]: pgmap v4465: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:57:04.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:57:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:57:04.074Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:57:05.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:57:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:57:04.773Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:57:05.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:57:05 smithi083 ceph-mon[77242]: pgmap v4466: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:57:05.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:57:05 smithi083 ceph-mon[79930]: pgmap v4466: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:57:06.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:57:05 smithi153 ceph-mon[70044]: pgmap v4466: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:57:07.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:57:07 smithi083 ceph-mon[77242]: pgmap v4467: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:57:07.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:57:07 smithi083 ceph-mon[79930]: pgmap v4467: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:57:08.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:57:07 smithi153 ceph-mon[70044]: pgmap v4467: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:57:10.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:57:09 smithi153 ceph-mon[70044]: pgmap v4468: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:57:10.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:57:09 smithi083 ceph-mon[77242]: pgmap v4468: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:57:10.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:57:09 smithi083 ceph-mon[79930]: pgmap v4468: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:57:10.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:57:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:57:10.683Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:57:11.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:57:11 smithi083 ceph-mon[79930]: pgmap v4469: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:57:11.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:57:11 smithi083 ceph-mon[77242]: pgmap v4469: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:57:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:57:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:57:11] "GET /metrics HTTP/1.1" 200 225208 "" "Prometheus/2.33.4" 2022-09-29T07:57:12.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:57:11 smithi153 ceph-mon[70044]: pgmap v4469: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:57:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:57:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:57:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:57:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:57:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:57:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:57:14.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:57:13 smithi153 ceph-mon[70044]: pgmap v4470: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:57:14.076 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:57:13 smithi083 ceph-mon[79930]: pgmap v4470: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:57:14.076 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:57:13 smithi083 ceph-mon[77242]: pgmap v4470: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:57:14.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:57:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:57:14.075Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:57:15.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:57:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:57:14.774Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:57:16.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:57:15 smithi153 ceph-mon[70044]: pgmap v4471: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:57:16.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:57:15 smithi083 ceph-mon[79930]: pgmap v4471: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:57:16.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:57:15 smithi083 ceph-mon[77242]: pgmap v4471: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:57:18.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:57:17 smithi153 ceph-mon[70044]: pgmap v4472: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:57:18.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:57:17 smithi083 ceph-mon[79930]: pgmap v4472: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:57:18.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:57:17 smithi083 ceph-mon[77242]: pgmap v4472: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:57:20.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:57:19 smithi153 ceph-mon[70044]: pgmap v4473: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:57:20.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:57:19 smithi083 ceph-mon[77242]: pgmap v4473: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:57:20.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:57:19 smithi083 ceph-mon[79930]: pgmap v4473: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:57:20.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:57:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:57:20.683Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:57:21.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:57:21 smithi083 ceph-mon[79930]: pgmap v4474: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:57:21.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:57:21 smithi083 ceph-mon[77242]: pgmap v4474: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:57:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:57:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:57:21] "GET /metrics HTTP/1.1" 200 225208 "" "Prometheus/2.33.4" 2022-09-29T07:57:22.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:57:21 smithi153 ceph-mon[70044]: pgmap v4474: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:57:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:57:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:57:22.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:57:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:57:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:57:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:57:24.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:57:23 smithi153 ceph-mon[70044]: pgmap v4475: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:57:24.081 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:57:23 smithi083 ceph-mon[79930]: pgmap v4475: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:57:24.082 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:57:23 smithi083 ceph-mon[77242]: pgmap v4475: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:57:24.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:57:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:57:24.081Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:57:25.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:57:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:57:24.775Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:57:26.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:57:25 smithi153 ceph-mon[70044]: pgmap v4476: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:57:26.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:57:25 smithi083 ceph-mon[79930]: pgmap v4476: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:57:26.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:57:25 smithi083 ceph-mon[77242]: pgmap v4476: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:57:28.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:57:27 smithi153 ceph-mon[70044]: pgmap v4477: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:57:28.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:57:27 smithi083 ceph-mon[79930]: pgmap v4477: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:57:28.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:57:27 smithi083 ceph-mon[77242]: pgmap v4477: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:57:30.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:57:29 smithi153 ceph-mon[70044]: pgmap v4478: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:57:30.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:57:29 smithi083 ceph-mon[79930]: pgmap v4478: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:57:30.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:57:29 smithi083 ceph-mon[77242]: pgmap v4478: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:57:30.356 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:57:30.647 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:57:30.647 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 57s ago 2h 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:57:30.648 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 9m ago 2h 62.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:57:30.648 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 57s ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:57:30.648 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 9m ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:57:30.648 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 9m ago 2h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:57:30.648 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 57s ago 2h 644M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:57:30.649 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 57s ago 2h 601M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:57:30.649 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 9m ago 2h 252M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:57:30.649 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 57s ago 2h 351M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:57:30.649 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 57s ago 2h 26.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:57:30.650 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 9m ago 2h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:57:30.650 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 57s ago 2h 437M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:57:30.650 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 57s ago 2h 428M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:57:30.650 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 57s ago 2h 339M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:57:30.651 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 57s ago 2h 415M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:57:30.651 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 9m ago 2h 349M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:57:30.651 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 9m ago 2h 427M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:57:30.651 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 9m ago 2h 380M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:57:30.651 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 9m ago 2h 444M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:57:30.652 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 9m ago 2h 125M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:57:30.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:57:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:57:30.684Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:57:31.003 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:57:31.003 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:57:31.004 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:57:31.004 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:57:31.004 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:57:31.004 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:57:31.005 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:57:31.005 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:57:31.005 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:57:31.005 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:57:31.006 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:57:31.006 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:57:31.006 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:57:31.006 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:57:31.006 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:57:31.007 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:57:31.007 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:57:31.007 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:57:31.759 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:57:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:57:31] "GET /metrics HTTP/1.1" 200 225208 "" "Prometheus/2.33.4" 2022-09-29T07:57:32.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:57:31 smithi153 ceph-mon[70044]: pgmap v4479: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:57:32.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:57:31 smithi153 ceph-mon[70044]: from='client.49279 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:57:32.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:57:31 smithi153 ceph-mon[70044]: from='client.39351 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:57:32.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:57:31 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2059631064' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:57:32.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:57:31 smithi083 ceph-mon[77242]: pgmap v4479: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:57:32.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:57:31 smithi083 ceph-mon[77242]: from='client.49279 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:57:32.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:57:31 smithi083 ceph-mon[77242]: from='client.39351 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:57:32.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:57:31 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2059631064' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:57:32.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:57:31 smithi083 ceph-mon[79930]: pgmap v4479: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:57:32.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:57:31 smithi083 ceph-mon[79930]: from='client.49279 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:57:32.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:57:31 smithi083 ceph-mon[79930]: from='client.39351 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:57:32.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:57:31 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2059631064' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:57:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:57:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:57:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:57:33.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:57:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:57:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:57:34.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:57:33 smithi153 ceph-mon[70044]: pgmap v4480: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:57:34.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:57:33 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:57:34.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:57:33 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:57:34.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:57:33 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:57:34.076 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:57:33 smithi083 ceph-mon[79930]: pgmap v4480: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:57:34.076 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:57:33 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:57:34.076 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:57:33 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:57:34.076 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:57:33 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:57:34.077 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:57:33 smithi083 ceph-mon[77242]: pgmap v4480: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:57:34.077 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:57:33 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:57:34.077 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:57:33 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:57:34.078 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:57:33 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:57:34.460 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:57:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:57:34.075Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:57:35.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:57:34 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:57:35.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:57:34 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:57:35.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:57:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:57:34.775Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:57:35.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:57:34 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:57:36.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:57:35 smithi083 ceph-mon[79930]: pgmap v4481: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:57:36.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:57:35 smithi083 ceph-mon[77242]: pgmap v4481: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:57:36.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:57:35 smithi153 ceph-mon[70044]: pgmap v4481: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:57:38.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:57:37 smithi083 ceph-mon[79930]: pgmap v4482: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:57:38.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:57:37 smithi083 ceph-mon[77242]: pgmap v4482: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:57:38.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:57:37 smithi153 ceph-mon[70044]: pgmap v4482: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:57:40.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:57:39 smithi083 ceph-mon[79930]: pgmap v4483: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:57:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:57:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:57:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:57:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:57:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:57:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:57:40.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:57:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:57:40.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:57:39 smithi083 ceph-mon[77242]: pgmap v4483: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:57:40.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:57:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:57:40.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:57:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:57:40.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:57:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:57:40.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:57:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:57:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:57:39 smithi153 ceph-mon[70044]: pgmap v4483: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:57:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:57:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:57:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:57:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:57:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:57:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:57:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:57:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:57:40.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:57:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:57:40.685Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:57:41.896 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:57:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:57:41] "GET /metrics HTTP/1.1" 200 225214 "" "Prometheus/2.33.4" 2022-09-29T07:57:42.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:57:41 smithi083 ceph-mon[79930]: pgmap v4484: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:57:42.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:57:41 smithi083 ceph-mon[77242]: pgmap v4484: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:57:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:57:41 smithi153 ceph-mon[70044]: pgmap v4484: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:57:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:57:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:57:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:57:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:57:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:57:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:57:44.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:57:43 smithi083 ceph-mon[79930]: pgmap v4485: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:57:44.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:57:43 smithi083 ceph-mon[77242]: pgmap v4485: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:57:44.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:57:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:57:44.077Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:57:44.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:57:43 smithi153 ceph-mon[70044]: pgmap v4485: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:57:45.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:57:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:57:44.777Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:57:46.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:57:45 smithi083 ceph-mon[77242]: pgmap v4486: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:57:46.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:57:45 smithi083 ceph-mon[79930]: pgmap v4486: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:57:46.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:57:45 smithi153 ceph-mon[70044]: pgmap v4486: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:57:48.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:57:47 smithi083 ceph-mon[79930]: pgmap v4487: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:57:48.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:57:47 smithi083 ceph-mon[77242]: pgmap v4487: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:57:48.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:57:47 smithi153 ceph-mon[70044]: pgmap v4487: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:57:50.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:57:49 smithi083 ceph-mon[79930]: pgmap v4488: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:57:50.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:57:49 smithi083 ceph-mon[77242]: pgmap v4488: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:57:50.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:57:49 smithi153 ceph-mon[70044]: pgmap v4488: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:57:50.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:57:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:57:50.686Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:57:51.911 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:57:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:57:51] "GET /metrics HTTP/1.1" 200 225214 "" "Prometheus/2.33.4" 2022-09-29T07:57:52.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:57:51 smithi083 ceph-mon[79930]: pgmap v4489: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:57:52.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:57:51 smithi083 ceph-mon[77242]: pgmap v4489: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:57:52.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:57:51 smithi153 ceph-mon[70044]: pgmap v4489: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:57:52.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:57:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:57:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:57:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:57:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:57:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:57:54.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:57:53 smithi083 ceph-mon[77242]: pgmap v4490: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:57:54.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:57:53 smithi083 ceph-mon[79930]: pgmap v4490: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:57:54.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:57:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:57:54.081Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:57:54.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:57:53 smithi153 ceph-mon[70044]: pgmap v4490: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:57:55.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:57:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:57:54.778Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:57:56.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:57:55 smithi083 ceph-mon[77242]: pgmap v4491: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:57:56.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:57:55 smithi083 ceph-mon[79930]: pgmap v4491: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:57:56.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:57:55 smithi153 ceph-mon[70044]: pgmap v4491: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:57:58.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:57:57 smithi083 ceph-mon[77242]: pgmap v4492: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:57:58.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:57:57 smithi083 ceph-mon[79930]: pgmap v4492: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:57:58.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:57:57 smithi153 ceph-mon[70044]: pgmap v4492: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:00.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:57:59 smithi083 ceph-mon[79930]: pgmap v4493: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:00.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:57:59 smithi083 ceph-mon[77242]: pgmap v4493: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:57:59 smithi153 ceph-mon[70044]: pgmap v4493: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:00.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:58:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:58:00.686Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:58:01.328 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:58:01.617 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:58:01.618 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 88s ago 2h 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:58:01.619 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 9m ago 2h 62.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:58:01.619 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 88s ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:58:01.620 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 9m ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:58:01.620 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 9m ago 2h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:58:01.620 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 88s ago 2h 644M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:58:01.621 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 88s ago 2h 601M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:58:01.621 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 9m ago 2h 252M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:58:01.622 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 88s ago 2h 351M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:58:01.622 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 88s ago 2h 26.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:58:01.623 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 9m ago 2h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:58:01.623 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 88s ago 2h 437M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:58:01.624 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 88s ago 2h 428M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:58:01.624 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 88s ago 2h 339M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:58:01.625 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 88s ago 2h 415M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:58:01.625 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 9m ago 2h 349M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:58:01.625 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 9m ago 2h 427M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:58:01.626 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 9m ago 2h 380M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:58:01.626 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 9m ago 2h 444M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:58:01.626 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 9m ago 2h 125M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:58:01.927 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:58:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:58:01] "GET /metrics HTTP/1.1" 200 225209 "" "Prometheus/2.33.4" 2022-09-29T07:58:01.972 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:58:01.973 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:58:01.973 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:58:01.973 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:58:01.974 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:58:01.974 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:58:01.975 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:58:01.975 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:58:01.976 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:58:01.976 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:58:01.977 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:58:01.977 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:58:01.977 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:58:01.978 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:58:01.978 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:58:01.979 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:58:01.979 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:58:01.980 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:58:02.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:58:01 smithi083 ceph-mon[77242]: pgmap v4494: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:02.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:58:01 smithi083 ceph-mon[79930]: pgmap v4494: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:02.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:58:01 smithi153 ceph-mon[70044]: pgmap v4494: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:02.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:58:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:58:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:58:03.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:58:02 smithi083 ceph-mon[79930]: from='client.49297 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:58:03.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:58:02 smithi083 ceph-mon[79930]: from='client.39369 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:58:03.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:58:02 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1261398973' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:58:03.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:58:02 smithi083 ceph-mon[77242]: from='client.49297 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:58:03.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:58:02 smithi083 ceph-mon[77242]: from='client.39369 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:58:03.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:58:02 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1261398973' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:58:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:58:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:58:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:58:03.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:58:02 smithi153 ceph-mon[70044]: from='client.49297 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:58:03.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:58:02 smithi153 ceph-mon[70044]: from='client.39369 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:58:03.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:58:02 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1261398973' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:58:04.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:58:03 smithi083 ceph-mon[79930]: pgmap v4495: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:04.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:58:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:58:04.077Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:58:04.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:58:03 smithi083 ceph-mon[77242]: pgmap v4495: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:04.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:58:03 smithi153 ceph-mon[70044]: pgmap v4495: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:05.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:58:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:58:04.778Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:58:06.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:58:05 smithi083 ceph-mon[79930]: pgmap v4496: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:06.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:58:05 smithi083 ceph-mon[77242]: pgmap v4496: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:06.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:58:05 smithi153 ceph-mon[70044]: pgmap v4496: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:08.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:58:07 smithi083 ceph-mon[77242]: pgmap v4497: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:08.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:58:07 smithi083 ceph-mon[79930]: pgmap v4497: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:08.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:58:07 smithi153 ceph-mon[70044]: pgmap v4497: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:10.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:58:09 smithi083 ceph-mon[79930]: pgmap v4498: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:10.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:58:09 smithi083 ceph-mon[77242]: pgmap v4498: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:10.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:58:09 smithi153 ceph-mon[70044]: pgmap v4498: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:10.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:58:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:58:10.687Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:58:11.952 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:58:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:58:11] "GET /metrics HTTP/1.1" 200 225210 "" "Prometheus/2.33.4" 2022-09-29T07:58:12.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:58:11 smithi083 ceph-mon[79930]: pgmap v4499: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:12.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:58:11 smithi083 ceph-mon[77242]: pgmap v4499: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:12.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:58:11 smithi153 ceph-mon[70044]: pgmap v4499: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:58:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:58:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:58:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:58:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:58:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:58:14.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:58:13 smithi083 ceph-mon[79930]: pgmap v4500: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:14.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:58:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:58:14.077Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:58:14.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:58:13 smithi083 ceph-mon[77242]: pgmap v4500: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:14.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:58:13 smithi153 ceph-mon[70044]: pgmap v4500: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:15.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:58:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:58:14.778Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:58:16.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:58:15 smithi083 ceph-mon[79930]: pgmap v4501: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:16.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:58:15 smithi083 ceph-mon[77242]: pgmap v4501: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:16.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:58:15 smithi153 ceph-mon[70044]: pgmap v4501: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:18.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:58:17 smithi083 ceph-mon[77242]: pgmap v4502: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:18.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:58:17 smithi083 ceph-mon[79930]: pgmap v4502: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:18.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:58:17 smithi153 ceph-mon[70044]: pgmap v4502: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:58:19 smithi153 ceph-mon[70044]: pgmap v4503: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:20.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:58:19 smithi083 ceph-mon[79930]: pgmap v4503: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:20.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:58:19 smithi083 ceph-mon[77242]: pgmap v4503: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:20.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:58:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:58:20.688Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:58:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:58:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:58:21] "GET /metrics HTTP/1.1" 200 225210 "" "Prometheus/2.33.4" 2022-09-29T07:58:22.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:58:21 smithi153 ceph-mon[70044]: pgmap v4504: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:22.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:58:21 smithi083 ceph-mon[79930]: pgmap v4504: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:22.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:58:21 smithi083 ceph-mon[77242]: pgmap v4504: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:58:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:58:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:58:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:58:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:58:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:58:24.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:58:23 smithi153 ceph-mon[70044]: pgmap v4505: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:24.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:58:23 smithi083 ceph-mon[77242]: pgmap v4505: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:24.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:58:23 smithi083 ceph-mon[79930]: pgmap v4505: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:24.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:58:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:58:24.082Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:58:25.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:58:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:58:24.779Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:58:26.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:58:25 smithi153 ceph-mon[70044]: pgmap v4506: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:26.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:58:25 smithi083 ceph-mon[77242]: pgmap v4506: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:26.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:58:25 smithi083 ceph-mon[79930]: pgmap v4506: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:28.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:58:27 smithi153 ceph-mon[70044]: pgmap v4507: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:28.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:58:27 smithi083 ceph-mon[79930]: pgmap v4507: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:28.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:58:27 smithi083 ceph-mon[77242]: pgmap v4507: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:30.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:58:29 smithi153 ceph-mon[70044]: pgmap v4508: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:30.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:58:29 smithi083 ceph-mon[77242]: pgmap v4508: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:30.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:58:29 smithi083 ceph-mon[79930]: pgmap v4508: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:30.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:58:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:58:30.688Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:58:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:58:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:58:31] "GET /metrics HTTP/1.1" 200 225211 "" "Prometheus/2.33.4" 2022-09-29T07:58:32.297 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:58:32.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:58:31 smithi153 ceph-mon[70044]: pgmap v4509: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:32.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:58:31 smithi083 ceph-mon[79930]: pgmap v4509: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:32.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:58:31 smithi083 ceph-mon[77242]: pgmap v4509: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:32.585 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:58:32.586 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 119s ago 2h 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:58:32.586 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 10m ago 2h 62.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:58:32.586 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 119s ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:58:32.586 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 10m ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:58:32.587 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 10m ago 2h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:58:32.587 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 119s ago 2h 644M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:58:32.587 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 119s ago 2h 601M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:58:32.587 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 10m ago 2h 252M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:58:32.588 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 119s ago 2h 351M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:58:32.588 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 119s ago 2h 26.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:58:32.588 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 10m ago 2h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:58:32.588 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 119s ago 2h 437M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:58:32.588 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 119s ago 2h 428M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:58:32.589 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 119s ago 2h 339M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:58:32.589 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 119s ago 2h 415M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:58:32.589 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 10m ago 2h 349M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:58:32.589 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 10m ago 2h 427M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:58:32.590 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 10m ago 2h 380M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:58:32.590 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 10m ago 2h 444M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:58:32.590 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 10m ago 2h 125M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:58:32.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:58:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:58:32.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:58:32.938 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:58:32.938 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:58:32.938 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:58:32.939 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:58:32.939 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:58:32.939 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:58:32.939 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:58:32.940 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:58:32.940 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:58:32.940 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:58:32.940 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:58:32.941 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:58:32.941 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:58:32.941 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:58:32.941 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:58:32.942 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:58:32.942 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:58:32.942 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:58:33.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:58:32 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2150256391' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:58:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:58:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:58:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:58:33.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:58:32 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2150256391' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:58:33.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:58:32 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2150256391' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:58:34.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:58:33 smithi153 ceph-mon[70044]: from='client.39381 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:58:34.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:58:33 smithi153 ceph-mon[70044]: pgmap v4510: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:34.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:58:33 smithi153 ceph-mon[70044]: from='client.49321 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:58:34.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:58:33 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:58:34.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:58:33 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:58:34.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:58:33 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:58:34.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:58:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:58:34.077Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:58:34.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:58:33 smithi083 ceph-mon[77242]: from='client.39381 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:58:34.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:58:33 smithi083 ceph-mon[77242]: pgmap v4510: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:34.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:58:33 smithi083 ceph-mon[77242]: from='client.49321 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:58:34.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:58:33 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:58:34.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:58:33 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:58:34.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:58:33 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:58:34.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:58:33 smithi083 ceph-mon[79930]: from='client.39381 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:58:34.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:58:33 smithi083 ceph-mon[79930]: pgmap v4510: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:34.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:58:33 smithi083 ceph-mon[79930]: from='client.49321 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:58:34.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:58:33 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:58:34.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:58:33 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:58:34.467 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:58:33 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:58:35.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:58:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:58:34.779Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:58:36.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:58:35 smithi153 ceph-mon[70044]: pgmap v4511: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:36.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:58:35 smithi083 ceph-mon[79930]: pgmap v4511: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:36.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:58:35 smithi083 ceph-mon[77242]: pgmap v4511: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:38.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:58:37 smithi153 ceph-mon[70044]: pgmap v4512: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:38.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:58:37 smithi083 ceph-mon[77242]: pgmap v4512: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:38.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:58:37 smithi083 ceph-mon[79930]: pgmap v4512: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:39.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:58:39 smithi083 ceph-mon[77242]: pgmap v4513: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:39.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:58:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:58:39.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:58:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:58:39.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:58:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:58:39.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:58:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:58:39.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:58:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:58:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:58:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:58:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:58:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:58:39.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:58:39 smithi083 ceph-mon[79930]: pgmap v4513: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:39.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:58:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:58:39.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:58:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:58:39.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:58:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:58:39.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:58:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:58:39.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:58:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:58:39.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:58:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:58:39.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:58:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:58:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:58:39 smithi153 ceph-mon[70044]: pgmap v4513: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:58:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:58:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:58:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:58:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:58:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:58:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:58:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:58:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:58:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:58:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:58:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:58:40.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:58:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:58:40.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:58:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:58:40.689Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:58:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:58:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:58:41] "GET /metrics HTTP/1.1" 200 225208 "" "Prometheus/2.33.4" 2022-09-29T07:58:41.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:58:41 smithi083 ceph-mon[79930]: pgmap v4514: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:41.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:58:41 smithi083 ceph-mon[77242]: pgmap v4514: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:42.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:58:41 smithi153 ceph-mon[70044]: pgmap v4514: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:58:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:58:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:58:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:58:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:58:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:58:43.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:58:43 smithi083 ceph-mon[79930]: pgmap v4515: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:43.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:58:43 smithi083 ceph-mon[77242]: pgmap v4515: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:44.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:58:43 smithi153 ceph-mon[70044]: pgmap v4515: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:44.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:58:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:58:44.079Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:58:45.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:58:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:58:44.781Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:58:45.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:58:45 smithi083 ceph-mon[77242]: pgmap v4516: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:45.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:58:45 smithi083 ceph-mon[79930]: pgmap v4516: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:46.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:58:45 smithi153 ceph-mon[70044]: pgmap v4516: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:48.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:58:47 smithi153 ceph-mon[70044]: pgmap v4517: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:48.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:58:47 smithi083 ceph-mon[77242]: pgmap v4517: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:48.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:58:47 smithi083 ceph-mon[79930]: pgmap v4517: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:50.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:58:49 smithi153 ceph-mon[70044]: pgmap v4518: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:50.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:58:49 smithi083 ceph-mon[79930]: pgmap v4518: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:50.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:58:49 smithi083 ceph-mon[77242]: pgmap v4518: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:50.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:58:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:58:50.690Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:58:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:58:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:58:51] "GET /metrics HTTP/1.1" 200 225208 "" "Prometheus/2.33.4" 2022-09-29T07:58:51.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:58:51 smithi083 ceph-mon[79930]: pgmap v4519: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:51.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:58:51 smithi083 ceph-mon[77242]: pgmap v4519: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:52.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:58:51 smithi153 ceph-mon[70044]: pgmap v4519: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:52.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:58:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:58:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:58:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:58:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:58:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:58:54.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:58:53 smithi153 ceph-mon[70044]: pgmap v4520: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:54.085 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:58:53 smithi083 ceph-mon[79930]: pgmap v4520: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:54.086 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:58:53 smithi083 ceph-mon[77242]: pgmap v4520: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:54.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:58:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:58:54.084Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:58:55.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:58:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:58:54.781Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:58:56.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:58:55 smithi153 ceph-mon[70044]: pgmap v4521: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:56.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:58:55 smithi083 ceph-mon[79930]: pgmap v4521: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:56.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:58:55 smithi083 ceph-mon[77242]: pgmap v4521: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:58.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:58:57 smithi153 ceph-mon[70044]: pgmap v4522: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:58.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:58:57 smithi083 ceph-mon[79930]: pgmap v4522: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:58:58.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:58:57 smithi083 ceph-mon[77242]: pgmap v4522: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:00.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:58:59 smithi153 ceph-mon[70044]: pgmap v4523: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:00.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:58:59 smithi083 ceph-mon[79930]: pgmap v4523: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:00.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:58:59 smithi083 ceph-mon[77242]: pgmap v4523: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:00.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:59:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:59:00.690Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:59:01.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:59:01 smithi083 ceph-mon[77242]: pgmap v4524: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:59:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:59:01] "GET /metrics HTTP/1.1" 200 225210 "" "Prometheus/2.33.4" 2022-09-29T07:59:01.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:59:01 smithi083 ceph-mon[79930]: pgmap v4524: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:02.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:59:01 smithi153 ceph-mon[70044]: pgmap v4524: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:59:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:59:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:59:03.261 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:59:03.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:59:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:59:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:59:03.562 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:59:03.562 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 2m ago 2h 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:59:03.562 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 24s ago 2h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:59:03.563 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 2m ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:59:03.563 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 24s ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:59:03.563 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 24s ago 2h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:59:03.564 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 2m ago 2h 644M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:59:03.564 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 2m ago 2h 601M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:59:03.564 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 24s ago 2h 267M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:59:03.564 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 2m ago 2h 351M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:59:03.565 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 2m ago 2h 26.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:59:03.565 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 24s ago 2h 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:59:03.565 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 2m ago 2h 437M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:59:03.566 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 2m ago 2h 428M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:59:03.566 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 2m ago 2h 339M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:59:03.566 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 2m ago 2h 415M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:59:03.566 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 24s ago 2h 370M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:59:03.567 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 24s ago 2h 455M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:59:03.567 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 24s ago 2h 405M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:59:03.567 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 24s ago 2h 473M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:59:03.567 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 24s ago 2h 127M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:59:03.927 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:59:03.927 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:59:03.928 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:59:03.928 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:59:03.929 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:59:03.929 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:59:03.930 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:59:03.930 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:59:03.931 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:59:03.931 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:59:03.932 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:59:03.932 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:59:03.933 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:59:03.933 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:59:03.933 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:59:03.934 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:59:03.934 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:59:03.935 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:59:04.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:59:03 smithi153 ceph-mon[70044]: pgmap v4525: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:04.081 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:59:03 smithi083 ceph-mon[79930]: pgmap v4525: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:04.081 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:59:03 smithi083 ceph-mon[77242]: pgmap v4525: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:04.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:59:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:59:04.080Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:59:05.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:59:04 smithi153 ceph-mon[70044]: from='client.39399 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:59:05.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:59:04 smithi153 ceph-mon[70044]: from='client.39405 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:59:05.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:59:04 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/622394743' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:59:05.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:59:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:59:04.782Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:59:05.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:59:04 smithi083 ceph-mon[77242]: from='client.39399 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:59:05.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:59:04 smithi083 ceph-mon[77242]: from='client.39405 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:59:05.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:59:04 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/622394743' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:59:05.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:59:04 smithi083 ceph-mon[79930]: from='client.39399 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:59:05.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:59:04 smithi083 ceph-mon[79930]: from='client.39405 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:59:05.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:59:04 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/622394743' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:59:06.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:59:05 smithi153 ceph-mon[70044]: pgmap v4526: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:06.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:59:05 smithi083 ceph-mon[77242]: pgmap v4526: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:06.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:59:05 smithi083 ceph-mon[79930]: pgmap v4526: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:08.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:59:07 smithi153 ceph-mon[70044]: pgmap v4527: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:08.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:59:07 smithi083 ceph-mon[79930]: pgmap v4527: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:08.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:59:07 smithi083 ceph-mon[77242]: pgmap v4527: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:10.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:59:09 smithi153 ceph-mon[70044]: pgmap v4528: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:10.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:59:09 smithi083 ceph-mon[79930]: pgmap v4528: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:10.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:59:09 smithi083 ceph-mon[77242]: pgmap v4528: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:10.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:59:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:59:10.690Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:59:11.757 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:59:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:59:11] "GET /metrics HTTP/1.1" 200 225207 "" "Prometheus/2.33.4" 2022-09-29T07:59:12.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:59:11 smithi153 ceph-mon[70044]: pgmap v4529: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:12.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:59:11 smithi083 ceph-mon[79930]: pgmap v4529: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:12.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:59:11 smithi083 ceph-mon[77242]: pgmap v4529: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:59:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:59:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:59:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:59:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:59:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:59:14.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:59:13 smithi153 ceph-mon[70044]: pgmap v4530: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:14.082 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:59:13 smithi083 ceph-mon[79930]: pgmap v4530: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:14.083 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:59:13 smithi083 ceph-mon[77242]: pgmap v4530: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:14.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:59:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:59:14.082Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:59:15.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:59:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:59:14.782Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:59:16.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:59:15 smithi153 ceph-mon[70044]: pgmap v4531: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:16.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:59:15 smithi083 ceph-mon[77242]: pgmap v4531: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:16.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:59:15 smithi083 ceph-mon[79930]: pgmap v4531: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:18.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:59:17 smithi153 ceph-mon[70044]: pgmap v4532: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:18.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:59:17 smithi083 ceph-mon[77242]: pgmap v4532: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:18.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:59:17 smithi083 ceph-mon[79930]: pgmap v4532: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:20.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:59:19 smithi153 ceph-mon[70044]: pgmap v4533: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:20.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:59:19 smithi083 ceph-mon[79930]: pgmap v4533: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:20.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:59:19 smithi083 ceph-mon[77242]: pgmap v4533: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:20.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:59:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:59:20.691Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:59:21.771 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:59:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:59:21] "GET /metrics HTTP/1.1" 200 225207 "" "Prometheus/2.33.4" 2022-09-29T07:59:22.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:59:21 smithi153 ceph-mon[70044]: pgmap v4534: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:22.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:59:21 smithi083 ceph-mon[77242]: pgmap v4534: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:22.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:59:21 smithi083 ceph-mon[79930]: pgmap v4534: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:22.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:59:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:59:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:59:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:59:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:59:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:59:24.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:59:23 smithi153 ceph-mon[70044]: pgmap v4535: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:24.084 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:59:23 smithi083 ceph-mon[79930]: pgmap v4535: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:24.084 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:59:23 smithi083 ceph-mon[77242]: pgmap v4535: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:24.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:59:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:59:24.083Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:59:25.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:59:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:59:24.783Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:59:26.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:59:25 smithi153 ceph-mon[70044]: pgmap v4536: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:26.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:59:25 smithi083 ceph-mon[77242]: pgmap v4536: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:26.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:59:25 smithi083 ceph-mon[79930]: pgmap v4536: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:28.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:59:27 smithi153 ceph-mon[70044]: pgmap v4537: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:28.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:59:27 smithi083 ceph-mon[79930]: pgmap v4537: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:28.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:59:27 smithi083 ceph-mon[77242]: pgmap v4537: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:30.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:59:29 smithi153 ceph-mon[70044]: pgmap v4538: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:30.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:59:29 smithi083 ceph-mon[79930]: pgmap v4538: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:30.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:59:29 smithi083 ceph-mon[77242]: pgmap v4538: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:30.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:59:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:59:30.692Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:59:31.785 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:59:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:59:31] "GET /metrics HTTP/1.1" 200 225210 "" "Prometheus/2.33.4" 2022-09-29T07:59:32.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:59:31 smithi153 ceph-mon[70044]: pgmap v4539: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:32.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:59:31 smithi083 ceph-mon[79930]: pgmap v4539: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:32.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:59:31 smithi083 ceph-mon[77242]: pgmap v4539: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:59:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:59:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:59:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:59:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:59:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:59:34.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:59:33 smithi153 ceph-mon[70044]: pgmap v4540: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:34.082 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:59:33 smithi083 ceph-mon[79930]: pgmap v4540: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:34.083 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:59:33 smithi083 ceph-mon[77242]: pgmap v4540: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:34.256 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T07:59:34.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:59:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:59:34.081Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:59:34.548 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T07:59:34.549 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 3m ago 2h 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T07:59:34.549 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 55s ago 2h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T07:59:34.549 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 3m ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T07:59:34.549 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 55s ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T07:59:34.550 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 55s ago 2h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T07:59:34.550 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 3m ago 2h 644M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T07:59:34.550 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 3m ago 2h 601M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T07:59:34.550 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 55s ago 2h 267M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T07:59:34.551 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 3m ago 2h 351M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T07:59:34.551 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 3m ago 2h 26.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T07:59:34.551 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 55s ago 2h 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T07:59:34.551 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 3m ago 2h 437M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T07:59:34.551 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 3m ago 2h 428M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T07:59:34.552 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 3m ago 2h 339M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T07:59:34.552 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 3m ago 2h 415M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T07:59:34.552 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 55s ago 2h 370M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T07:59:34.552 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 55s ago 2h 455M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T07:59:34.553 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 55s ago 2h 405M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T07:59:34.553 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 55s ago 2h 473M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T07:59:34.553 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 55s ago 2h 127M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T07:59:34.907 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T07:59:34.907 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T07:59:34.908 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T07:59:34.908 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:59:34.908 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T07:59:34.908 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T07:59:34.909 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:59:34.909 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T07:59:34.909 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T07:59:34.910 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:59:34.910 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T07:59:34.910 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T07:59:34.910 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T07:59:34.911 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T07:59:34.911 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T07:59:34.911 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T07:59:34.911 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T07:59:34.912 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T07:59:35.210 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:59:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:59:34.784Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:59:36.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:59:35 smithi153 ceph-mon[70044]: from='client.39417 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:59:36.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:59:35 smithi153 ceph-mon[70044]: pgmap v4541: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:36.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:59:35 smithi153 ceph-mon[70044]: from='client.49357 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:59:36.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:59:35 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3054434864' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:59:36.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:59:35 smithi083 ceph-mon[79930]: from='client.39417 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:59:36.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:59:35 smithi083 ceph-mon[79930]: pgmap v4541: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:36.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:59:35 smithi083 ceph-mon[79930]: from='client.49357 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:59:36.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:59:35 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3054434864' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:59:36.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:59:35 smithi083 ceph-mon[77242]: from='client.39417 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:59:36.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:59:35 smithi083 ceph-mon[77242]: pgmap v4541: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:36.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:59:35 smithi083 ceph-mon[77242]: from='client.49357 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T07:59:36.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:59:35 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3054434864' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T07:59:38.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:59:37 smithi153 ceph-mon[70044]: pgmap v4542: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:38.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:59:37 smithi083 ceph-mon[79930]: pgmap v4542: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:38.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:59:37 smithi083 ceph-mon[77242]: pgmap v4542: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:59:39 smithi153 ceph-mon[70044]: pgmap v4543: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:59:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:59:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:59:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:59:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:59:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:59:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:59:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:59:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:59:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:59:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:59:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:59:40.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:59:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:59:40.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:59:39 smithi083 ceph-mon[77242]: pgmap v4543: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:40.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:59:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:59:40.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:59:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:59:40.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:59:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:59:40.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:59:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:59:40.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:59:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:59:40.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:59:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:59:40.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:59:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:59:40.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:59:39 smithi083 ceph-mon[79930]: pgmap v4543: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:40.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:59:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T07:59:40.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:59:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T07:59:40.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:59:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T07:59:40.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:59:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:59:40.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:59:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T07:59:40.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:59:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:59:40.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:59:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T07:59:40.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:59:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:59:40.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:59:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:59:40.692Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:59:40.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:59:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:59:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:59:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T07:59:41.897 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:59:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:59:41] "GET /metrics HTTP/1.1" 200 225208 "" "Prometheus/2.33.4" 2022-09-29T07:59:42.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:59:41 smithi083 ceph-mon[77242]: pgmap v4544: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:42.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:59:41 smithi083 ceph-mon[79930]: pgmap v4544: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:59:41 smithi153 ceph-mon[70044]: pgmap v4544: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:59:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:59:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:59:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:59:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:59:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:59:44.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:59:43 smithi083 ceph-mon[79930]: pgmap v4545: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:44.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:59:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:59:44.083Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:59:44.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:59:43 smithi083 ceph-mon[77242]: pgmap v4545: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:44.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:59:43 smithi153 ceph-mon[70044]: pgmap v4545: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:45.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:59:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:59:44.784Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:59:46.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:59:45 smithi083 ceph-mon[77242]: pgmap v4546: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:46.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:59:45 smithi083 ceph-mon[79930]: pgmap v4546: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:46.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:59:45 smithi153 ceph-mon[70044]: pgmap v4546: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:48.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:59:47 smithi083 ceph-mon[77242]: pgmap v4547: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:48.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:59:47 smithi083 ceph-mon[79930]: pgmap v4547: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:48.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:59:47 smithi153 ceph-mon[70044]: pgmap v4547: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:50.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:59:49 smithi083 ceph-mon[77242]: pgmap v4548: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:50.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:59:49 smithi083 ceph-mon[79930]: pgmap v4548: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:50.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:59:49 smithi153 ceph-mon[70044]: pgmap v4548: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:50.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:59:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:59:50.693Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:59:51.916 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 07:59:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:59:51] "GET /metrics HTTP/1.1" 200 225208 "" "Prometheus/2.33.4" 2022-09-29T07:59:52.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:59:51 smithi083 ceph-mon[79930]: pgmap v4549: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:52.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:59:51 smithi083 ceph-mon[77242]: pgmap v4549: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:52.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:59:51 smithi153 ceph-mon[70044]: pgmap v4549: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 07:59:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T07:59:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T07:59:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 07:59:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:07:59:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T07:59:54.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:59:53 smithi083 ceph-mon[77242]: pgmap v4550: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:54.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:59:53 smithi083 ceph-mon[79930]: pgmap v4550: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:54.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:59:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:59:54.084Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:59:54.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:59:53 smithi153 ceph-mon[70044]: pgmap v4550: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:55.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 07:59:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T07:59:54.784Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T07:59:56.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:59:55 smithi083 ceph-mon[79930]: pgmap v4551: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:56.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:59:55 smithi083 ceph-mon[77242]: pgmap v4551: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:56.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:59:55 smithi153 ceph-mon[70044]: pgmap v4551: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:58.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:59:57 smithi083 ceph-mon[79930]: pgmap v4552: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:58.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:59:57 smithi083 ceph-mon[77242]: pgmap v4552: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T07:59:58.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:59:57 smithi153 ceph-mon[70044]: pgmap v4552: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:00.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 07:59:59 smithi083 ceph-mon[79930]: pgmap v4553: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:00.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 07:59:59 smithi083 ceph-mon[77242]: pgmap v4553: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:00.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:00:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[77219]: 2022-09-29T07:59:59.999+0000 7f160f665700 -1 log_channel(cluster) log [ERR] : overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T08:00:00.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 07:59:59 smithi153 ceph-mon[70044]: pgmap v4553: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:00.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:00:00 smithi083 ceph-mon[79930]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T08:00:00.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:00:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:00:00.694Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:00:00.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:00:00 smithi083 ceph-mon[77242]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T08:00:01.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:00:00 smithi153 ceph-mon[70044]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T08:00:01.937 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:00:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:00:01] "GET /metrics HTTP/1.1" 200 225212 "" "Prometheus/2.33.4" 2022-09-29T08:00:02.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:00:01 smithi083 ceph-mon[79930]: pgmap v4554: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:02.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:00:01 smithi083 ceph-mon[77242]: pgmap v4554: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:02.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:00:01 smithi153 ceph-mon[70044]: pgmap v4554: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:02.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:00:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:00:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:00:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:00:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:00:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:00:04.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:00:03 smithi083 ceph-mon[79930]: pgmap v4555: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:04.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:00:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:00:04.085Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:00:04.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:00:03 smithi083 ceph-mon[77242]: pgmap v4555: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:04.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:00:03 smithi153 ceph-mon[70044]: pgmap v4555: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:05.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:00:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:00:04.785Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:00:05.239 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:00:05.529 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:00:05.530 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 3m ago 2h 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:00:05.530 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 86s ago 2h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:00:05.530 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 3m ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:00:05.531 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 86s ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:00:05.531 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 86s ago 2h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:00:05.531 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 3m ago 2h 644M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:00:05.531 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 3m ago 2h 601M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:00:05.532 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 86s ago 2h 267M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:00:05.532 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 3m ago 2h 351M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:00:05.532 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 3m ago 2h 26.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:00:05.532 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 86s ago 2h 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:00:05.533 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 3m ago 2h 437M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:00:05.533 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 3m ago 2h 428M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:00:05.533 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 3m ago 2h 339M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:00:05.533 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 3m ago 2h 415M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:00:05.533 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 86s ago 2h 370M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:00:05.534 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 86s ago 2h 455M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:00:05.534 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 86s ago 2h 405M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:00:05.534 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 86s ago 2h 473M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:00:05.534 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 86s ago 2h 127M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:00:05.885 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:00:05.885 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:00:05.886 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:00:05.886 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:00:05.886 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:00:05.887 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:00:05.887 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:00:05.887 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:00:05.887 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:00:05.887 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:00:05.888 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:00:05.888 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:00:05.888 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:00:05.888 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:00:05.889 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:00:05.889 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:00:05.889 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:00:05.889 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:00:06.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:00:05 smithi083 ceph-mon[77242]: pgmap v4556: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:06.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:00:05 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/80110952' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:00:06.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:00:05 smithi083 ceph-mon[79930]: pgmap v4556: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:06.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:00:05 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/80110952' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:00:06.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:00:05 smithi153 ceph-mon[70044]: pgmap v4556: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:06.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:00:05 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/80110952' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:00:07.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:00:06 smithi083 ceph-mon[77242]: from='client.49369 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:00:07.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:00:06 smithi083 ceph-mon[77242]: from='client.49375 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:00:07.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:00:06 smithi083 ceph-mon[79930]: from='client.49369 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:00:07.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:00:06 smithi083 ceph-mon[79930]: from='client.49375 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:00:07.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:00:06 smithi153 ceph-mon[70044]: from='client.49369 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:00:07.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:00:06 smithi153 ceph-mon[70044]: from='client.49375 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:00:08.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:00:07 smithi083 ceph-mon[77242]: pgmap v4557: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:08.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:00:07 smithi083 ceph-mon[79930]: pgmap v4557: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:08.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:00:07 smithi153 ceph-mon[70044]: pgmap v4557: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:10.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:00:09 smithi083 ceph-mon[77242]: pgmap v4558: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:10.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:00:09 smithi083 ceph-mon[79930]: pgmap v4558: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:10.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:00:09 smithi153 ceph-mon[70044]: pgmap v4558: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:10.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:00:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:00:10.695Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:00:11.953 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:00:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:00:11] "GET /metrics HTTP/1.1" 200 225210 "" "Prometheus/2.33.4" 2022-09-29T08:00:12.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:00:11 smithi083 ceph-mon[77242]: pgmap v4559: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:12.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:00:11 smithi083 ceph-mon[79930]: pgmap v4559: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:12.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:00:11 smithi153 ceph-mon[70044]: pgmap v4559: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:12.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:00:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:00:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:00:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:00:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:00:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:00:14.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:00:13 smithi083 ceph-mon[79930]: pgmap v4560: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:14.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:00:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:00:14.086Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:00:14.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:00:13 smithi083 ceph-mon[77242]: pgmap v4560: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:14.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:00:13 smithi153 ceph-mon[70044]: pgmap v4560: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:15.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:00:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:00:14.786Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:00:16.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:00:15 smithi083 ceph-mon[77242]: pgmap v4561: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:16.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:00:15 smithi083 ceph-mon[79930]: pgmap v4561: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:16.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:00:15 smithi153 ceph-mon[70044]: pgmap v4561: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:18.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:00:17 smithi153 ceph-mon[70044]: pgmap v4562: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:18.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:00:17 smithi083 ceph-mon[77242]: pgmap v4562: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:18.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:00:17 smithi083 ceph-mon[79930]: pgmap v4562: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:00:19 smithi153 ceph-mon[70044]: pgmap v4563: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:20.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:00:19 smithi083 ceph-mon[77242]: pgmap v4563: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:20.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:00:19 smithi083 ceph-mon[79930]: pgmap v4563: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:20.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:00:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:00:20.695Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:00:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:00:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:00:21] "GET /metrics HTTP/1.1" 200 225210 "" "Prometheus/2.33.4" 2022-09-29T08:00:22.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:00:21 smithi153 ceph-mon[70044]: pgmap v4564: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:22.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:00:21 smithi083 ceph-mon[79930]: pgmap v4564: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:22.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:00:21 smithi083 ceph-mon[77242]: pgmap v4564: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:00:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:00:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:00:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:00:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:00:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:00:24.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:00:23 smithi153 ceph-mon[70044]: pgmap v4565: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:24.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:00:23 smithi083 ceph-mon[79930]: pgmap v4565: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:24.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:00:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:00:24.085Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:00:24.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:00:23 smithi083 ceph-mon[77242]: pgmap v4565: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:25.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:00:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:00:24.787Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:00:26.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:00:25 smithi153 ceph-mon[70044]: pgmap v4566: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:26.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:00:25 smithi083 ceph-mon[79930]: pgmap v4566: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:26.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:00:25 smithi083 ceph-mon[77242]: pgmap v4566: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:28.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:00:27 smithi153 ceph-mon[70044]: pgmap v4567: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:28.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:00:27 smithi083 ceph-mon[77242]: pgmap v4567: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:28.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:00:27 smithi083 ceph-mon[79930]: pgmap v4567: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:30.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:00:29 smithi153 ceph-mon[70044]: pgmap v4568: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:30.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:00:29 smithi083 ceph-mon[77242]: pgmap v4568: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:30.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:00:29 smithi083 ceph-mon[79930]: pgmap v4568: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:30.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:00:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:00:30.696Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:00:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:00:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:00:31] "GET /metrics HTTP/1.1" 200 225209 "" "Prometheus/2.33.4" 2022-09-29T08:00:32.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:00:31 smithi153 ceph-mon[70044]: pgmap v4569: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:32.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:00:31 smithi083 ceph-mon[79930]: pgmap v4569: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:32.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:00:31 smithi083 ceph-mon[77242]: pgmap v4569: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:00:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:00:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:00:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:00:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:00:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:00:34.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:00:33 smithi153 ceph-mon[70044]: pgmap v4570: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:34.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:00:33 smithi083 ceph-mon[79930]: pgmap v4570: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:34.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:00:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:00:34.085Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:00:34.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:00:33 smithi083 ceph-mon[77242]: pgmap v4570: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:35.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:00:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:00:34.787Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:00:36.215 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:00:36.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:00:35 smithi153 ceph-mon[70044]: pgmap v4571: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:36.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:00:35 smithi083 ceph-mon[79930]: pgmap v4571: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:36.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:00:35 smithi083 ceph-mon[77242]: pgmap v4571: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:36.504 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:00:36.504 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 4m ago 2h 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:00:36.505 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 117s ago 2h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:00:36.505 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 4m ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:00:36.505 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 117s ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:00:36.505 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 117s ago 2h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:00:36.506 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 4m ago 2h 644M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:00:36.506 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 4m ago 2h 601M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:00:36.506 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 117s ago 2h 267M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:00:36.507 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 4m ago 2h 351M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:00:36.507 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 4m ago 2h 26.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:00:36.507 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 117s ago 2h 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:00:36.507 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 4m ago 2h 437M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:00:36.507 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 4m ago 2h 428M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:00:36.508 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 4m ago 2h 339M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:00:36.508 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 4m ago 2h 415M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:00:36.508 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 117s ago 2h 370M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:00:36.509 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 117s ago 2h 455M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:00:36.509 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 117s ago 2h 405M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:00:36.509 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 117s ago 2h 473M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:00:36.509 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 117s ago 2h 127M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:00:36.861 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:00:36.862 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:00:36.862 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:00:36.862 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:00:36.862 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:00:36.863 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:00:36.863 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:00:36.863 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:00:36.863 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:00:36.864 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:00:36.864 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:00:36.864 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:00:36.864 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:00:36.865 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:00:36.865 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:00:36.865 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:00:36.865 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:00:36.866 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:00:37.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:00:37 smithi083 ceph-mon[77242]: from='client.39453 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:00:37.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:00:37 smithi083 ceph-mon[77242]: pgmap v4572: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:37.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:00:37 smithi083 ceph-mon[77242]: from='client.39459 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:00:37.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:00:37 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/268468160' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:00:37.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:00:37 smithi083 ceph-mon[79930]: from='client.39453 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:00:37.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:00:37 smithi083 ceph-mon[79930]: pgmap v4572: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:37.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:00:37 smithi083 ceph-mon[79930]: from='client.39459 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:00:37.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:00:37 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/268468160' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:00:38.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:00:37 smithi153 ceph-mon[70044]: from='client.39453 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:00:38.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:00:37 smithi153 ceph-mon[70044]: pgmap v4572: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:38.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:00:37 smithi153 ceph-mon[70044]: from='client.39459 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:00:38.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:00:37 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/268468160' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:00:39.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:00:39 smithi083 ceph-mon[77242]: pgmap v4573: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:39.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:00:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:00:39.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:00:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:00:39.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:00:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:00:39.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:00:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:00:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:00:39 smithi083 ceph-mon[79930]: pgmap v4573: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:39.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:00:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:00:39.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:00:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:00:39.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:00:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:00:39.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:00:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:00:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:00:39 smithi153 ceph-mon[70044]: pgmap v4573: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:00:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:00:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:00:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:00:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:00:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:00:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:00:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:00:40.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:00:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:00:40.696Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:00:40.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:00:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:00:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:00:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:00:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:00:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:00:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:00:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:00:40.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:00:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:00:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:00:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:00:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:00:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:00:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:00:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:00:41.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:00:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:00:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:00:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:00:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:00:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:00:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:00:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:00:41.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:00:41 smithi083 ceph-mon[77242]: pgmap v4574: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:00:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:00:41] "GET /metrics HTTP/1.1" 200 225210 "" "Prometheus/2.33.4" 2022-09-29T08:00:41.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:00:41 smithi083 ceph-mon[79930]: pgmap v4574: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:42.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:00:41 smithi153 ceph-mon[70044]: pgmap v4574: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:00:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:00:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:00:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:00:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:00:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:00:43.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:00:43 smithi083 ceph-mon[77242]: pgmap v4575: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:43.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:00:43 smithi083 ceph-mon[79930]: pgmap v4575: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:44.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:00:43 smithi153 ceph-mon[70044]: pgmap v4575: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:44.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:00:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:00:44.086Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:00:45.210 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:00:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:00:44.788Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:00:45.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:00:45 smithi083 ceph-mon[77242]: pgmap v4576: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:45.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:00:45 smithi083 ceph-mon[79930]: pgmap v4576: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:46.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:00:45 smithi153 ceph-mon[70044]: pgmap v4576: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:47.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:00:47 smithi083 ceph-mon[79930]: pgmap v4577: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:47.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:00:47 smithi083 ceph-mon[77242]: pgmap v4577: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:48.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:00:47 smithi153 ceph-mon[70044]: pgmap v4577: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:49.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:00:49 smithi083 ceph-mon[77242]: pgmap v4578: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:49.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:00:49 smithi083 ceph-mon[79930]: pgmap v4578: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:50.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:00:49 smithi153 ceph-mon[70044]: pgmap v4578: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:50.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:00:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:00:50.698Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:00:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:00:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:00:51] "GET /metrics HTTP/1.1" 200 225210 "" "Prometheus/2.33.4" 2022-09-29T08:00:51.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:00:51 smithi083 ceph-mon[79930]: pgmap v4579: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:51.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:00:51 smithi083 ceph-mon[77242]: pgmap v4579: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:52.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:00:51 smithi153 ceph-mon[70044]: pgmap v4579: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:00:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:00:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:00:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:00:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:00:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:00:53.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:00:53 smithi083 ceph-mon[79930]: pgmap v4580: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:53.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:00:53 smithi083 ceph-mon[77242]: pgmap v4580: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:54.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:00:53 smithi153 ceph-mon[70044]: pgmap v4580: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:54.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:00:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:00:54.088Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:00:55.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:00:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:00:54.789Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:00:55.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:00:55 smithi083 ceph-mon[79930]: pgmap v4581: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:55.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:00:55 smithi083 ceph-mon[77242]: pgmap v4581: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:56.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:00:55 smithi153 ceph-mon[70044]: pgmap v4581: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:57.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:00:57 smithi083 ceph-mon[79930]: pgmap v4582: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:57.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:00:57 smithi083 ceph-mon[77242]: pgmap v4582: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:58.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:00:57 smithi153 ceph-mon[70044]: pgmap v4582: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:59.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:00:59 smithi083 ceph-mon[79930]: pgmap v4583: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:00:59.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:00:59 smithi083 ceph-mon[77242]: pgmap v4583: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:01:00.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:00:59 smithi153 ceph-mon[70044]: pgmap v4583: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:01:00.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:01:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:01:00.698Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:01:01.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:01:01 smithi153 ceph-mon[70044]: pgmap v4584: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:01:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:01:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:01:01] "GET /metrics HTTP/1.1" 200 225205 "" "Prometheus/2.33.4" 2022-09-29T08:01:01.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:01:01 smithi083 ceph-mon[79930]: pgmap v4584: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:01:01.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:01:01 smithi083 ceph-mon[77242]: pgmap v4584: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:01:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:01:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:01:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:01:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:01:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:01:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:01:03.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:01:03 smithi083 ceph-mon[79930]: pgmap v4585: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:01:03.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:01:03 smithi083 ceph-mon[77242]: pgmap v4585: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:01:04.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:01:03 smithi153 ceph-mon[70044]: pgmap v4585: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:01:04.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:01:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:01:04.086Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:01:05.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:01:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:01:04.790Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:01:05.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:01:05 smithi083 ceph-mon[79930]: pgmap v4586: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:01:05.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:01:05 smithi083 ceph-mon[77242]: pgmap v4586: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:01:06.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:01:05 smithi153 ceph-mon[70044]: pgmap v4586: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:01:07.187 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:01:07.473 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:01:07.473 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 4m ago 2h 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:01:07.474 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 2m ago 2h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:01:07.474 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 4m ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:01:07.475 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 2m ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:01:07.475 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 2m ago 2h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:01:07.475 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 4m ago 2h 644M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:01:07.476 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 4m ago 2h 601M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:01:07.476 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 2m ago 2h 267M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:01:07.477 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 4m ago 2h 351M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:01:07.477 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 4m ago 2h 26.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:01:07.478 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 2m ago 2h 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:01:07.478 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 4m ago 2h 437M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:01:07.478 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 4m ago 2h 428M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:01:07.479 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 4m ago 2h 339M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:01:07.479 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 4m ago 2h 415M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:01:07.480 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 2m ago 2h 370M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:01:07.480 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 2m ago 2h 455M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:01:07.480 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 2m ago 2h 405M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:01:07.481 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 2m ago 2h 473M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:01:07.481 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 2m ago 2h 127M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:01:07.828 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:01:07.828 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:01:07.829 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:01:07.829 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:01:07.829 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:01:07.829 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:01:07.830 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:01:07.830 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:01:07.830 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:01:07.830 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:01:07.831 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:01:07.831 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:01:07.831 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:01:07.831 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:01:07.832 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:01:07.832 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:01:07.832 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:01:07.832 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:01:07.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:01:07 smithi083 ceph-mon[79930]: pgmap v4587: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:01:07.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:01:07 smithi083 ceph-mon[79930]: from='client.49405 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:01:07.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:01:07 smithi083 ceph-mon[77242]: pgmap v4587: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:01:07.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:01:07 smithi083 ceph-mon[77242]: from='client.49405 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:01:08.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:01:07 smithi153 ceph-mon[70044]: pgmap v4587: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:01:08.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:01:07 smithi153 ceph-mon[70044]: from='client.49405 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:01:08.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:01:08 smithi083 ceph-mon[77242]: from='client.49411 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:01:08.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:01:08 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1491616331' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:01:08.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:01:08 smithi083 ceph-mon[79930]: from='client.49411 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:01:08.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:01:08 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1491616331' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:01:09.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:01:08 smithi153 ceph-mon[70044]: from='client.49411 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:01:09.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:01:08 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1491616331' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:01:09.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:01:09 smithi083 ceph-mon[79930]: pgmap v4588: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:01:09.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:01:09 smithi083 ceph-mon[77242]: pgmap v4588: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:01:10.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:01:09 smithi153 ceph-mon[70044]: pgmap v4588: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:01:10.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:01:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:01:10.699Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:01:11.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:01:11 smithi083 ceph-mon[77242]: pgmap v4589: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:01:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:01:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:01:11] "GET /metrics HTTP/1.1" 200 225210 "" "Prometheus/2.33.4" 2022-09-29T08:01:11.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:01:11 smithi083 ceph-mon[79930]: pgmap v4589: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:01:12.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:01:11 smithi153 ceph-mon[70044]: pgmap v4589: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:01:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:01:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:01:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:01:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:01:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:01:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:01:13.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:01:13 smithi083 ceph-mon[79930]: pgmap v4590: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:01:13.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:01:13 smithi083 ceph-mon[77242]: pgmap v4590: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:01:14.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:01:13 smithi153 ceph-mon[70044]: pgmap v4590: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:01:14.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:01:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:01:14.086Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:01:15.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:01:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:01:14.790Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:01:16.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:01:15 smithi153 ceph-mon[70044]: pgmap v4591: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:01:16.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:01:15 smithi083 ceph-mon[77242]: pgmap v4591: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:01:16.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:01:15 smithi083 ceph-mon[79930]: pgmap v4591: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:01:18.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:01:17 smithi153 ceph-mon[70044]: pgmap v4592: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:01:18.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:01:17 smithi083 ceph-mon[79930]: pgmap v4592: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:01:18.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:01:17 smithi083 ceph-mon[77242]: pgmap v4592: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:01:20.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:01:19 smithi153 ceph-mon[70044]: pgmap v4593: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:01:20.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:01:19 smithi083 ceph-mon[77242]: pgmap v4593: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:01:20.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:01:19 smithi083 ceph-mon[79930]: pgmap v4593: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:01:20.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:01:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:01:20.700Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:01:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:01:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:01:21] "GET /metrics HTTP/1.1" 200 225210 "" "Prometheus/2.33.4" 2022-09-29T08:01:21.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:01:21 smithi083 ceph-mon[79930]: pgmap v4594: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:01:21.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:01:21 smithi083 ceph-mon[77242]: pgmap v4594: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:01:22.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:01:21 smithi153 ceph-mon[70044]: pgmap v4594: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:01:22.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:01:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:01:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:01:23.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:01:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:01:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:01:24.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:01:23 smithi153 ceph-mon[70044]: pgmap v4595: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:01:24.087 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:01:23 smithi083 ceph-mon[79930]: pgmap v4595: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:01:24.088 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:01:23 smithi083 ceph-mon[77242]: pgmap v4595: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:01:24.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:01:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:01:24.087Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:01:25.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:01:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:01:24.791Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:01:26.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:01:25 smithi153 ceph-mon[70044]: pgmap v4596: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:01:26.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:01:25 smithi083 ceph-mon[79930]: pgmap v4596: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:01:26.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:01:25 smithi083 ceph-mon[77242]: pgmap v4596: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:01:28.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:01:27 smithi153 ceph-mon[70044]: pgmap v4597: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:01:28.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:01:27 smithi083 ceph-mon[79930]: pgmap v4597: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:01:28.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:01:27 smithi083 ceph-mon[77242]: pgmap v4597: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:01:30.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:01:29 smithi153 ceph-mon[70044]: pgmap v4598: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:01:30.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:01:29 smithi083 ceph-mon[77242]: pgmap v4598: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:01:30.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:01:29 smithi083 ceph-mon[79930]: pgmap v4598: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:01:30.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:01:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:01:30.700Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:01:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:01:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:01:31] "GET /metrics HTTP/1.1" 200 225211 "" "Prometheus/2.33.4" 2022-09-29T08:01:31.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:01:31 smithi083 ceph-mon[79930]: pgmap v4599: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:01:31.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:01:31 smithi083 ceph-mon[77242]: pgmap v4599: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:01:32.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:01:31 smithi153 ceph-mon[70044]: pgmap v4599: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:01:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:01:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:01:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:01:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:01:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:01:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:01:34.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:01:33 smithi153 ceph-mon[70044]: pgmap v4600: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:01:34.089 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:01:33 smithi083 ceph-mon[79930]: pgmap v4600: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:01:34.090 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:01:33 smithi083 ceph-mon[77242]: pgmap v4600: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:01:34.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:01:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:01:34.088Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:01:35.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:01:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:01:34.792Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:01:36.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:01:35 smithi153 ceph-mon[70044]: pgmap v4601: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:01:36.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:01:35 smithi083 ceph-mon[79930]: pgmap v4601: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:01:36.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:01:35 smithi083 ceph-mon[77242]: pgmap v4601: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:01:38.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:01:37 smithi153 ceph-mon[70044]: pgmap v4602: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:01:38.156 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:01:38.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:01:37 smithi083 ceph-mon[79930]: pgmap v4602: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:01:38.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:01:37 smithi083 ceph-mon[77242]: pgmap v4602: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:01:38.446 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:01:38.446 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 5m ago 2h 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:01:38.446 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 2m ago 2h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:01:38.447 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 5m ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:01:38.447 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 2m ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:01:38.447 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 2m ago 2h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:01:38.447 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 5m ago 2h 644M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:01:38.448 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 5m ago 2h 601M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:01:38.448 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 2m ago 2h 267M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:01:38.448 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 5m ago 2h 351M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:01:38.448 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 5m ago 2h 26.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:01:38.449 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 2m ago 2h 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:01:38.449 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 5m ago 2h 437M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:01:38.449 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 5m ago 2h 428M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:01:38.450 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 5m ago 2h 339M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:01:38.450 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 5m ago 2h 415M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:01:38.450 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 2m ago 2h 370M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:01:38.451 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 2m ago 2h 455M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:01:38.451 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 2m ago 2h 405M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:01:38.451 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 2m ago 2h 473M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:01:38.451 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 2m ago 2h 127M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:01:38.804 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:01:38.805 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:01:38.805 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:01:38.805 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:01:38.806 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:01:38.806 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:01:38.806 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:01:38.807 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:01:38.807 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:01:38.807 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:01:38.807 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:01:38.808 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:01:38.808 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:01:38.808 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:01:38.808 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:01:38.808 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:01:38.809 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:01:38.809 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:01:39.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:01:38 smithi153 ceph-mon[70044]: from='client.39489 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:01:39.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:01:38 smithi083 ceph-mon[79930]: from='client.39489 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:01:39.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:01:38 smithi083 ceph-mon[77242]: from='client.39489 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:01:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:01:39 smithi153 ceph-mon[70044]: pgmap v4603: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:01:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:01:39 smithi153 ceph-mon[70044]: from='client.49429 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:01:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:01:39 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/757253373' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:01:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:01:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:01:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:01:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:01:40.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:01:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:01:40.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:01:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:01:40.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:01:39 smithi083 ceph-mon[77242]: pgmap v4603: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:01:40.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:01:39 smithi083 ceph-mon[77242]: from='client.49429 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:01:40.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:01:39 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/757253373' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:01:40.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:01:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:01:40.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:01:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:01:40.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:01:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:01:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:01:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:01:40.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:01:39 smithi083 ceph-mon[79930]: pgmap v4603: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:01:40.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:01:39 smithi083 ceph-mon[79930]: from='client.49429 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:01:40.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:01:39 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/757253373' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:01:40.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:01:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:01:40.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:01:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:01:40.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:01:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:01:40.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:01:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:01:40.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:01:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:01:40.701Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:01:40.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:01:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:01:40.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:01:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:01:40.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:01:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:01:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:01:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:01:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:01:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:01:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:01:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:01:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:01:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:01:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:01:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:01:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:01:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:01:41.773 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:01:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:01:41] "GET /metrics HTTP/1.1" 200 225205 "" "Prometheus/2.33.4" 2022-09-29T08:01:42.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:01:41 smithi153 ceph-mon[70044]: pgmap v4604: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:01:42.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:01:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:01:42.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:01:41 smithi083 ceph-mon[77242]: pgmap v4604: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:01:42.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:01:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:01:42.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:01:41 smithi083 ceph-mon[79930]: pgmap v4604: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:01:42.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:01:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:01:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:01:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:01:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:01:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:01:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:01:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:01:44.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:01:43 smithi153 ceph-mon[70044]: pgmap v4605: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:01:44.089 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:01:43 smithi083 ceph-mon[79930]: pgmap v4605: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:01:44.089 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:01:43 smithi083 ceph-mon[77242]: pgmap v4605: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:01:44.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:01:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:01:44.088Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:01:45.210 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:01:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:01:44.792Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:01:46.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:01:45 smithi153 ceph-mon[70044]: pgmap v4606: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:01:46.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:01:45 smithi083 ceph-mon[79930]: pgmap v4606: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:01:46.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:01:45 smithi083 ceph-mon[77242]: pgmap v4606: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:01:48.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:01:47 smithi153 ceph-mon[70044]: pgmap v4607: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:01:48.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:01:47 smithi083 ceph-mon[79930]: pgmap v4607: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:01:48.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:01:47 smithi083 ceph-mon[77242]: pgmap v4607: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:01:50.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:01:49 smithi153 ceph-mon[70044]: pgmap v4608: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:01:50.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:01:49 smithi083 ceph-mon[79930]: pgmap v4608: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:01:50.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:01:49 smithi083 ceph-mon[77242]: pgmap v4608: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:01:50.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:01:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:01:50.702Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:01:51.789 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:01:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:01:51] "GET /metrics HTTP/1.1" 200 225205 "" "Prometheus/2.33.4" 2022-09-29T08:01:52.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:01:51 smithi153 ceph-mon[70044]: pgmap v4609: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:01:52.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:01:51 smithi083 ceph-mon[77242]: pgmap v4609: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:01:52.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:01:51 smithi083 ceph-mon[79930]: pgmap v4609: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:01:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:01:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:01:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:01:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:01:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:01:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:01:54.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:01:53 smithi153 ceph-mon[70044]: pgmap v4610: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:01:54.090 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:01:53 smithi083 ceph-mon[79930]: pgmap v4610: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:01:54.090 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:01:53 smithi083 ceph-mon[77242]: pgmap v4610: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:01:54.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:01:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:01:54.089Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:01:55.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:01:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:01:54.794Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:01:56.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:01:55 smithi153 ceph-mon[70044]: pgmap v4611: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:01:56.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:01:55 smithi083 ceph-mon[79930]: pgmap v4611: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:01:56.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:01:55 smithi083 ceph-mon[77242]: pgmap v4611: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:01:58.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:01:57 smithi153 ceph-mon[70044]: pgmap v4612: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:01:58.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:01:57 smithi083 ceph-mon[79930]: pgmap v4612: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:01:58.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:01:57 smithi083 ceph-mon[77242]: pgmap v4612: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:00.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:01:59 smithi153 ceph-mon[70044]: pgmap v4613: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:00.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:01:59 smithi083 ceph-mon[79930]: pgmap v4613: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:00.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:01:59 smithi083 ceph-mon[77242]: pgmap v4613: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:00.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:02:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:02:00.703Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:02:01.808 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:02:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:02:01] "GET /metrics HTTP/1.1" 200 225204 "" "Prometheus/2.33.4" 2022-09-29T08:02:02.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:02:01 smithi083 ceph-mon[77242]: pgmap v4614: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:02.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:02:01 smithi083 ceph-mon[79930]: pgmap v4614: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:02.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:02:01 smithi153 ceph-mon[70044]: pgmap v4614: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:02:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:02:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:02:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:02:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:02:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:02:04.091 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:02:03 smithi083 ceph-mon[79930]: pgmap v4615: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:04.092 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:02:03 smithi083 ceph-mon[77242]: pgmap v4615: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:04.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:02:03 smithi153 ceph-mon[70044]: pgmap v4615: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:04.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:02:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:02:04.090Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:02:05.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:02:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:02:04.794Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:02:06.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:02:05 smithi083 ceph-mon[79930]: pgmap v4616: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:06.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:02:05 smithi083 ceph-mon[77242]: pgmap v4616: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:06.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:02:05 smithi153 ceph-mon[70044]: pgmap v4616: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:08.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:02:07 smithi083 ceph-mon[79930]: pgmap v4617: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:08.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:02:07 smithi083 ceph-mon[77242]: pgmap v4617: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:08.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:02:07 smithi153 ceph-mon[70044]: pgmap v4617: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:09.129 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:02:09.417 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:02:09.417 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 5m ago 2h 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:02:09.417 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 3m ago 2h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:02:09.418 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 5m ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:02:09.418 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 3m ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:02:09.418 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 3m ago 2h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:02:09.419 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 5m ago 2h 644M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:02:09.419 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 5m ago 2h 601M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:02:09.419 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 3m ago 2h 267M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:02:09.419 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 5m ago 2h 351M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:02:09.420 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 5m ago 2h 26.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:02:09.420 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 3m ago 2h 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:02:09.420 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 5m ago 2h 437M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:02:09.420 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 5m ago 2h 428M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:02:09.421 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 5m ago 2h 339M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:02:09.421 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 5m ago 2h 415M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:02:09.421 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 3m ago 2h 370M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:02:09.422 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 3m ago 2h 455M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:02:09.422 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 3m ago 2h 405M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:02:09.422 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 3m ago 2h 473M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:02:09.422 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 3m ago 2h 127M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:02:09.772 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:02:09.772 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:02:09.773 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:02:09.773 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:02:09.773 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:02:09.773 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:02:09.774 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:02:09.774 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:02:09.774 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:02:09.774 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:02:09.775 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:02:09.775 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:02:09.775 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:02:09.775 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:02:09.776 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:02:09.776 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:02:09.776 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:02:09.776 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:02:10.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:02:09 smithi083 ceph-mon[79930]: pgmap v4618: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:10.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:02:09 smithi083 ceph-mon[79930]: from='client.39507 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:02:10.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:02:09 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/76569598' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:02:10.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:02:09 smithi083 ceph-mon[77242]: pgmap v4618: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:10.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:02:09 smithi083 ceph-mon[77242]: from='client.39507 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:02:10.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:02:09 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/76569598' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:02:10.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:02:09 smithi153 ceph-mon[70044]: pgmap v4618: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:10.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:02:09 smithi153 ceph-mon[70044]: from='client.39507 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:02:10.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:02:09 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/76569598' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:02:10.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:02:10 smithi083 ceph-mon[79930]: from='client.49447 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:02:10.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:02:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:02:10.704Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:02:10.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:02:10 smithi083 ceph-mon[77242]: from='client.49447 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:02:11.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:02:10 smithi153 ceph-mon[70044]: from='client.49447 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:02:11.827 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:02:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:02:11] "GET /metrics HTTP/1.1" 200 225206 "" "Prometheus/2.33.4" 2022-09-29T08:02:12.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:02:11 smithi083 ceph-mon[77242]: pgmap v4619: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:12.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:02:11 smithi083 ceph-mon[79930]: pgmap v4619: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:12.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:02:11 smithi153 ceph-mon[70044]: pgmap v4619: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:12.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:02:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:02:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:02:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:02:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:02:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:02:14.092 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:02:13 smithi083 ceph-mon[79930]: pgmap v4620: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:14.093 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:02:13 smithi083 ceph-mon[77242]: pgmap v4620: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:14.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:02:13 smithi153 ceph-mon[70044]: pgmap v4620: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:14.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:02:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:02:14.091Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:02:15.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:02:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:02:14.795Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:02:16.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:02:15 smithi083 ceph-mon[79930]: pgmap v4621: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:16.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:02:15 smithi083 ceph-mon[77242]: pgmap v4621: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:16.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:02:15 smithi153 ceph-mon[70044]: pgmap v4621: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:18.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:02:17 smithi083 ceph-mon[79930]: pgmap v4622: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:18.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:02:17 smithi083 ceph-mon[77242]: pgmap v4622: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:18.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:02:17 smithi153 ceph-mon[70044]: pgmap v4622: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:20.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:02:19 smithi083 ceph-mon[79930]: pgmap v4623: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:20.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:02:19 smithi083 ceph-mon[77242]: pgmap v4623: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:20.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:02:19 smithi153 ceph-mon[70044]: pgmap v4623: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:20.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:02:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:02:20.704Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:02:21.845 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:02:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:02:21] "GET /metrics HTTP/1.1" 200 225206 "" "Prometheus/2.33.4" 2022-09-29T08:02:22.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:02:21 smithi083 ceph-mon[79930]: pgmap v4624: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:22.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:02:21 smithi083 ceph-mon[77242]: pgmap v4624: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:22.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:02:21 smithi153 ceph-mon[70044]: pgmap v4624: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:02:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:02:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:02:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:02:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:02:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:02:24.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:02:23 smithi083 ceph-mon[79930]: pgmap v4625: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:24.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:02:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:02:24.091Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:02:24.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:02:23 smithi083 ceph-mon[77242]: pgmap v4625: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:24.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:02:23 smithi153 ceph-mon[70044]: pgmap v4625: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:25.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:02:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:02:24.796Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:02:26.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:02:25 smithi083 ceph-mon[77242]: pgmap v4626: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:26.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:02:25 smithi083 ceph-mon[79930]: pgmap v4626: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:26.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:02:25 smithi153 ceph-mon[70044]: pgmap v4626: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:28.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:02:27 smithi083 ceph-mon[77242]: pgmap v4627: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:28.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:02:27 smithi083 ceph-mon[79930]: pgmap v4627: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:28.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:02:27 smithi153 ceph-mon[70044]: pgmap v4627: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:30.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:02:29 smithi083 ceph-mon[77242]: pgmap v4628: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:30.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:02:29 smithi083 ceph-mon[79930]: pgmap v4628: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:30.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:02:29 smithi153 ceph-mon[70044]: pgmap v4628: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:30.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:02:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:02:30.705Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:02:31.859 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:02:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:02:31] "GET /metrics HTTP/1.1" 200 225211 "" "Prometheus/2.33.4" 2022-09-29T08:02:32.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:02:31 smithi083 ceph-mon[79930]: pgmap v4629: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:32.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:02:31 smithi083 ceph-mon[77242]: pgmap v4629: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:32.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:02:31 smithi153 ceph-mon[70044]: pgmap v4629: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:02:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:02:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:02:33.311 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:02:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:02:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:02:34.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:02:33 smithi083 ceph-mon[79930]: pgmap v4630: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:34.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:02:33 smithi083 ceph-mon[77242]: pgmap v4630: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:34.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:02:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:02:34.090Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:02:34.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:02:33 smithi153 ceph-mon[70044]: pgmap v4630: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:35.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:02:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:02:34.796Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:02:36.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:02:35 smithi083 ceph-mon[79930]: pgmap v4631: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:36.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:02:35 smithi083 ceph-mon[77242]: pgmap v4631: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:36.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:02:35 smithi153 ceph-mon[70044]: pgmap v4631: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:38.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:02:37 smithi083 ceph-mon[77242]: pgmap v4632: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:38.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:02:37 smithi083 ceph-mon[79930]: pgmap v4632: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:38.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:02:37 smithi153 ceph-mon[70044]: pgmap v4632: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:40.101 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:02:40.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:02:39 smithi083 ceph-mon[77242]: pgmap v4633: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:40.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:02:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:02:40.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:02:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:02:40.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:02:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:02:40.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:02:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:02:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:02:39 smithi083 ceph-mon[79930]: pgmap v4633: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:02:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:02:40.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:02:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:02:40.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:02:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:02:40.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:02:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:02:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:02:39 smithi153 ceph-mon[70044]: pgmap v4633: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:02:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:02:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:02:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:02:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:02:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:02:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:02:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:02:40.390 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:02:40.391 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 6m ago 2h 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:02:40.391 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 4m ago 2h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:02:40.392 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 6m ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:02:40.392 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 4m ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:02:40.393 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 4m ago 2h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:02:40.393 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 6m ago 2h 644M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:02:40.394 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 6m ago 2h 601M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:02:40.395 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 4m ago 2h 267M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:02:40.395 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 6m ago 2h 351M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:02:40.396 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 6m ago 2h 26.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:02:40.396 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 4m ago 2h 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:02:40.397 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 6m ago 2h 437M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:02:40.397 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 6m ago 2h 428M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:02:40.398 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 6m ago 2h 339M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:02:40.398 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 6m ago 2h 415M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:02:40.399 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 4m ago 2h 370M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:02:40.399 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 4m ago 2h 455M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:02:40.400 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 4m ago 2h 405M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:02:40.400 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 4m ago 2h 473M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:02:40.401 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 4m ago 2h 127M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:02:40.746 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:02:40.747 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:02:40.747 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:02:40.747 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:02:40.748 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:02:40.748 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:02:40.749 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:02:40.749 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:02:40.749 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:02:40.750 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:02:40.750 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:02:40.751 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:02:40.751 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:02:40.751 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:02:40.752 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:02:40.752 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:02:40.753 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:02:40.753 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:02:40.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:02:40 smithi083 ceph-mon[79930]: from='client.39525 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:02:40.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:02:40 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3701839961' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:02:40.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:02:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:02:40.704Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:02:40.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:02:40 smithi083 ceph-mon[77242]: from='client.39525 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:02:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:02:40 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3701839961' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:02:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:02:40 smithi153 ceph-mon[70044]: from='client.39525 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:02:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:02:40 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3701839961' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:02:41.878 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:02:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:02:41] "GET /metrics HTTP/1.1" 200 225208 "" "Prometheus/2.33.4" 2022-09-29T08:02:42.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:02:41 smithi083 ceph-mon[79930]: from='client.49465 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:02:42.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:02:41 smithi083 ceph-mon[79930]: pgmap v4634: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:42.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:02:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:02:42.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:02:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:02:42.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:02:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:02:42.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:02:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:02:42.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:02:41 smithi083 ceph-mon[77242]: from='client.49465 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:02:42.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:02:41 smithi083 ceph-mon[77242]: pgmap v4634: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:42.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:02:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:02:42.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:02:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:02:42.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:02:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:02:42.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:02:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:02:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:02:41 smithi153 ceph-mon[70044]: from='client.49465 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:02:42.347 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:02:41 smithi153 ceph-mon[70044]: pgmap v4634: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:42.347 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:02:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:02:42.347 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:02:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:02:42.347 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:02:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:02:42.348 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:02:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:02:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:02:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:02:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:02:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:02:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:02:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:02:44.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:02:43 smithi083 ceph-mon[79930]: pgmap v4635: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:44.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:02:43 smithi083 ceph-mon[77242]: pgmap v4635: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:44.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:02:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:02:44.091Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:02:44.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:02:43 smithi153 ceph-mon[70044]: pgmap v4635: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:45.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:02:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:02:44.796Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:02:46.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:02:45 smithi083 ceph-mon[79930]: pgmap v4636: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:46.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:02:45 smithi083 ceph-mon[77242]: pgmap v4636: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:46.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:02:45 smithi153 ceph-mon[70044]: pgmap v4636: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:48.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:02:47 smithi083 ceph-mon[77242]: pgmap v4637: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:48.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:02:47 smithi083 ceph-mon[79930]: pgmap v4637: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:48.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:02:47 smithi153 ceph-mon[70044]: pgmap v4637: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:50.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:02:49 smithi083 ceph-mon[79930]: pgmap v4638: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:50.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:02:49 smithi083 ceph-mon[77242]: pgmap v4638: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:50.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:02:49 smithi153 ceph-mon[70044]: pgmap v4638: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:50.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:02:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:02:50.706Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:02:51.894 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:02:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:02:51] "GET /metrics HTTP/1.1" 200 225208 "" "Prometheus/2.33.4" 2022-09-29T08:02:52.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:02:51 smithi083 ceph-mon[79930]: pgmap v4639: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:52.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:02:51 smithi083 ceph-mon[77242]: pgmap v4639: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:52.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:02:51 smithi153 ceph-mon[70044]: pgmap v4639: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:02:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:02:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:02:53.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:02:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:02:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:02:54.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:02:53 smithi083 ceph-mon[77242]: pgmap v4640: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:54.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:02:53 smithi083 ceph-mon[79930]: pgmap v4640: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:54.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:02:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:02:54.093Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:02:54.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:02:53 smithi153 ceph-mon[70044]: pgmap v4640: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:55.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:02:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:02:54.798Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:02:56.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:02:55 smithi083 ceph-mon[79930]: pgmap v4641: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:56.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:02:55 smithi083 ceph-mon[77242]: pgmap v4641: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:56.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:02:55 smithi153 ceph-mon[70044]: pgmap v4641: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:58.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:02:57 smithi083 ceph-mon[79930]: pgmap v4642: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:58.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:02:57 smithi083 ceph-mon[77242]: pgmap v4642: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:02:58.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:02:57 smithi153 ceph-mon[70044]: pgmap v4642: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:00.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:02:59 smithi083 ceph-mon[79930]: pgmap v4643: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:00.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:02:59 smithi083 ceph-mon[77242]: pgmap v4643: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:02:59 smithi153 ceph-mon[70044]: pgmap v4643: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:00.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:03:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:03:00.707Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:03:01.909 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:03:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:03:01] "GET /metrics HTTP/1.1" 200 225210 "" "Prometheus/2.33.4" 2022-09-29T08:03:02.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:03:01 smithi083 ceph-mon[77242]: pgmap v4644: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:02.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:03:01 smithi083 ceph-mon[79930]: pgmap v4644: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:02.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:03:01 smithi153 ceph-mon[70044]: pgmap v4644: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:03:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:03:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:03:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:03:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:03:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:03:04.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:03:03 smithi083 ceph-mon[77242]: pgmap v4645: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:04.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:03:03 smithi083 ceph-mon[79930]: pgmap v4645: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:04.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:03:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:03:04.093Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:03:04.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:03:03 smithi153 ceph-mon[70044]: pgmap v4645: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:05.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:03:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:03:04.798Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:03:06.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:03:05 smithi083 ceph-mon[79930]: pgmap v4646: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:06.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:03:05 smithi083 ceph-mon[77242]: pgmap v4646: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:06.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:03:05 smithi153 ceph-mon[70044]: pgmap v4646: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:08.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:03:07 smithi083 ceph-mon[77242]: pgmap v4647: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:08.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:03:07 smithi083 ceph-mon[79930]: pgmap v4647: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:08.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:03:07 smithi153 ceph-mon[70044]: pgmap v4647: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:10.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:03:09 smithi083 ceph-mon[79930]: pgmap v4648: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:10.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:03:09 smithi083 ceph-mon[77242]: pgmap v4648: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:10.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:03:09 smithi153 ceph-mon[70044]: pgmap v4648: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:10.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:03:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:03:10.707Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:03:11.072 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:03:11.360 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:03:11.361 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 6m ago 2h 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:03:11.361 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 4m ago 2h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:03:11.361 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 6m ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:03:11.362 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 4m ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:03:11.362 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 4m ago 2h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:03:11.362 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 6m ago 2h 644M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:03:11.363 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 6m ago 2h 601M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:03:11.363 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 4m ago 2h 267M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:03:11.363 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 6m ago 2h 351M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:03:11.363 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 6m ago 2h 26.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:03:11.363 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 4m ago 2h 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:03:11.364 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 6m ago 2h 437M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:03:11.364 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 6m ago 2h 428M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:03:11.364 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 6m ago 2h 339M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:03:11.364 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 6m ago 2h 415M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:03:11.365 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 4m ago 2h 370M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:03:11.365 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 4m ago 2h 455M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:03:11.365 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 4m ago 2h 405M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:03:11.365 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 4m ago 2h 473M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:03:11.366 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 4m ago 2h 127M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:03:11.722 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:03:11.722 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:03:11.722 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:03:11.722 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:03:11.723 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:03:11.723 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:03:11.723 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:03:11.723 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:03:11.724 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:03:11.724 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:03:11.724 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:03:11.724 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:03:11.725 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:03:11.725 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:03:11.725 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:03:11.725 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:03:11.725 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:03:11.726 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:03:11.926 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:03:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:03:11] "GET /metrics HTTP/1.1" 200 225206 "" "Prometheus/2.33.4" 2022-09-29T08:03:12.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:03:11 smithi083 ceph-mon[77242]: pgmap v4649: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:12.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:03:11 smithi083 ceph-mon[77242]: from='client.49477 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:03:12.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:03:11 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2936796742' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:03:12.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:03:11 smithi083 ceph-mon[79930]: pgmap v4649: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:12.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:03:11 smithi083 ceph-mon[79930]: from='client.49477 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:03:12.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:03:11 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2936796742' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:03:12.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:03:11 smithi153 ceph-mon[70044]: pgmap v4649: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:12.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:03:11 smithi153 ceph-mon[70044]: from='client.49477 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:03:12.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:03:11 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2936796742' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:03:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:03:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:03:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:03:13.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:03:12 smithi083 ceph-mon[77242]: from='client.49483 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:03:13.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:03:12 smithi083 ceph-mon[79930]: from='client.49483 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:03:13.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:03:12 smithi153 ceph-mon[70044]: from='client.49483 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:03:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:03:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:03:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:03:14.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:03:13 smithi083 ceph-mon[79930]: pgmap v4650: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:14.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:03:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:03:14.094Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:03:14.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:03:13 smithi083 ceph-mon[77242]: pgmap v4650: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:14.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:03:13 smithi153 ceph-mon[70044]: pgmap v4650: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:15.074 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:03:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:03:14.799Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:03:16.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:03:15 smithi083 ceph-mon[79930]: pgmap v4651: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:16.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:03:15 smithi083 ceph-mon[77242]: pgmap v4651: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:16.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:03:15 smithi153 ceph-mon[70044]: pgmap v4651: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:18.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:03:17 smithi083 ceph-mon[79930]: pgmap v4652: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:18.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:03:17 smithi083 ceph-mon[77242]: pgmap v4652: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:18.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:03:17 smithi153 ceph-mon[70044]: pgmap v4652: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:20.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:03:19 smithi083 ceph-mon[79930]: pgmap v4653: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:20.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:03:19 smithi083 ceph-mon[77242]: pgmap v4653: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:03:19 smithi153 ceph-mon[70044]: pgmap v4653: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:20.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:03:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:03:20.708Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:03:21.948 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:03:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:03:21] "GET /metrics HTTP/1.1" 200 225206 "" "Prometheus/2.33.4" 2022-09-29T08:03:22.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:03:21 smithi083 ceph-mon[79930]: pgmap v4654: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:22.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:03:21 smithi083 ceph-mon[77242]: pgmap v4654: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:22.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:03:21 smithi153 ceph-mon[70044]: pgmap v4654: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:03:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:03:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:03:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:03:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:03:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:03:24.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:03:23 smithi083 ceph-mon[79930]: pgmap v4655: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:24.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:03:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:03:24.096Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:03:24.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:03:23 smithi083 ceph-mon[77242]: pgmap v4655: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:24.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:03:23 smithi153 ceph-mon[70044]: pgmap v4655: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:25.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:03:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:03:24.800Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:03:26.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:03:25 smithi083 ceph-mon[77242]: pgmap v4656: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:26.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:03:25 smithi083 ceph-mon[79930]: pgmap v4656: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:26.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:03:25 smithi153 ceph-mon[70044]: pgmap v4656: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:28.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:03:27 smithi083 ceph-mon[79930]: pgmap v4657: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:28.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:03:27 smithi083 ceph-mon[77242]: pgmap v4657: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:28.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:03:27 smithi153 ceph-mon[70044]: pgmap v4657: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:30.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:03:29 smithi083 ceph-mon[77242]: pgmap v4658: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:30.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:03:29 smithi083 ceph-mon[79930]: pgmap v4658: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:30.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:03:29 smithi153 ceph-mon[70044]: pgmap v4658: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:30.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:03:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:03:30.708Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:03:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:03:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:03:31] "GET /metrics HTTP/1.1" 200 225210 "" "Prometheus/2.33.4" 2022-09-29T08:03:32.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:03:31 smithi153 ceph-mon[70044]: pgmap v4659: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:32.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:03:31 smithi083 ceph-mon[79930]: pgmap v4659: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:32.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:03:31 smithi083 ceph-mon[77242]: pgmap v4659: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:03:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:03:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:03:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:03:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:03:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:03:34.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:03:33 smithi153 ceph-mon[70044]: pgmap v4660: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:34.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:03:33 smithi083 ceph-mon[79930]: pgmap v4660: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:34.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:03:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:03:34.096Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:03:34.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:03:33 smithi083 ceph-mon[77242]: pgmap v4660: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:35.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:03:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:03:34.800Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:03:36.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:03:35 smithi153 ceph-mon[70044]: pgmap v4661: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:36.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:03:35 smithi083 ceph-mon[79930]: pgmap v4661: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:36.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:03:35 smithi083 ceph-mon[77242]: pgmap v4661: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:38.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:03:37 smithi153 ceph-mon[70044]: pgmap v4662: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:38.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:03:37 smithi083 ceph-mon[79930]: pgmap v4662: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:38.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:03:37 smithi083 ceph-mon[77242]: pgmap v4662: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:03:39 smithi153 ceph-mon[70044]: pgmap v4663: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:03:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:03:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:03:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:03:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:03:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:03:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:03:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:03:40.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:03:39 smithi083 ceph-mon[79930]: pgmap v4663: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:40.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:03:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:03:40.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:03:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:03:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:03:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:03:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:03:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:03:40.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:03:39 smithi083 ceph-mon[77242]: pgmap v4663: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:03:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:03:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:03:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:03:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:03:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:03:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:03:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:03:40.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:03:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:03:40.709Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:03:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:03:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:03:41] "GET /metrics HTTP/1.1" 200 225211 "" "Prometheus/2.33.4" 2022-09-29T08:03:42.060 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:03:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:03:41 smithi153 ceph-mon[70044]: pgmap v4664: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:03:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:03:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:03:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:03:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:03:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:03:42.362 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:03:42.362 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 7m ago 2h 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:03:42.362 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 5m ago 2h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:03:42.363 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 7m ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:03:42.363 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 5m ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:03:42.363 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 5m ago 2h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:03:42.363 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 7m ago 2h 644M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:03:42.363 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 7m ago 2h 601M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:03:42.364 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 5m ago 2h 267M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:03:42.364 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 7m ago 2h 351M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:03:42.364 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 7m ago 2h 26.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:03:42.365 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 5m ago 2h 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:03:42.365 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 7m ago 2h 437M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:03:42.365 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 7m ago 2h 428M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:03:42.365 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 7m ago 2h 339M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:03:42.365 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 7m ago 2h 415M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:03:42.366 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 5m ago 2h 370M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:03:42.366 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 5m ago 2h 455M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:03:42.366 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 5m ago 2h 405M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:03:42.366 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 5m ago 2h 473M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:03:42.367 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 5m ago 2h 127M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:03:42.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:03:41 smithi083 ceph-mon[79930]: pgmap v4664: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:42.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:03:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:03:42.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:03:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:03:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:03:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:03:42.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:03:41 smithi083 ceph-mon[77242]: pgmap v4664: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:42.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:03:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:03:42.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:03:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:03:42.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:03:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:03:42.752 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:03:42.753 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:03:42.753 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:03:42.753 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:03:42.753 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:03:42.754 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:03:42.754 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:03:42.754 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:03:42.754 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:03:42.755 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:03:42.755 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:03:42.755 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:03:42.755 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:03:42.756 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:03:42.756 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:03:42.756 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:03:42.756 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:03:42.757 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:03:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:03:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:03:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:03:43.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:03:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:03:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:03:43.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:03:42 smithi153 ceph-mon[70044]: from='client.39561 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:03:43.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:03:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:03:43.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:03:42 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4233799867' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:03:43.460 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:03:42 smithi083 ceph-mon[77242]: from='client.39561 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:03:43.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:03:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:03:43.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:03:42 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4233799867' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:03:43.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:03:42 smithi083 ceph-mon[79930]: from='client.39561 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:03:43.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:03:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:03:43.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:03:42 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4233799867' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:03:44.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:03:43 smithi153 ceph-mon[70044]: from='client.49501 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:03:44.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:03:43 smithi153 ceph-mon[70044]: pgmap v4665: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:44.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:03:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:03:44.097Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:03:44.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:03:43 smithi083 ceph-mon[77242]: from='client.49501 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:03:44.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:03:43 smithi083 ceph-mon[77242]: pgmap v4665: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:44.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:03:43 smithi083 ceph-mon[79930]: from='client.49501 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:03:44.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:03:43 smithi083 ceph-mon[79930]: pgmap v4665: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:45.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:03:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:03:44.801Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:03:46.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:03:45 smithi153 ceph-mon[70044]: pgmap v4666: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:46.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:03:45 smithi083 ceph-mon[77242]: pgmap v4666: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:46.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:03:45 smithi083 ceph-mon[79930]: pgmap v4666: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:48.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:03:47 smithi153 ceph-mon[70044]: pgmap v4667: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:48.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:03:47 smithi083 ceph-mon[77242]: pgmap v4667: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:48.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:03:47 smithi083 ceph-mon[79930]: pgmap v4667: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:50.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:03:49 smithi153 ceph-mon[70044]: pgmap v4668: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:50.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:03:49 smithi083 ceph-mon[79930]: pgmap v4668: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:50.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:03:49 smithi083 ceph-mon[77242]: pgmap v4668: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:51.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:03:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:03:50.710Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:03:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:03:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:03:51] "GET /metrics HTTP/1.1" 200 225211 "" "Prometheus/2.33.4" 2022-09-29T08:03:52.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:03:51 smithi153 ceph-mon[70044]: pgmap v4669: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:52.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:03:51 smithi083 ceph-mon[79930]: pgmap v4669: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:52.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:03:51 smithi083 ceph-mon[77242]: pgmap v4669: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:03:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:03:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:03:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:03:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:03:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:03:54.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:03:53 smithi153 ceph-mon[70044]: pgmap v4670: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:54.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:03:53 smithi083 ceph-mon[79930]: pgmap v4670: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:54.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:03:53 smithi083 ceph-mon[77242]: pgmap v4670: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:54.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:03:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:03:54.112Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:03:55.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:03:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:03:54.802Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:03:56.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:03:56 smithi153 ceph-mon[70044]: pgmap v4671: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:56.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:03:56 smithi083 ceph-mon[77242]: pgmap v4671: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:56.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:03:56 smithi083 ceph-mon[79930]: pgmap v4671: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:58.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:03:58 smithi153 ceph-mon[70044]: pgmap v4672: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:58.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:03:58 smithi083 ceph-mon[79930]: pgmap v4672: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:03:58.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:03:58 smithi083 ceph-mon[77242]: pgmap v4672: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:04:00 smithi153 ceph-mon[70044]: pgmap v4673: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:00.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:04:00 smithi083 ceph-mon[79930]: pgmap v4673: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:00.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:04:00 smithi083 ceph-mon[77242]: pgmap v4673: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:01.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:04:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:04:00.711Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:04:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:04:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:04:01] "GET /metrics HTTP/1.1" 200 225207 "" "Prometheus/2.33.4" 2022-09-29T08:04:02.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:04:02 smithi153 ceph-mon[70044]: pgmap v4674: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:02.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:04:02 smithi083 ceph-mon[77242]: pgmap v4674: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:02.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:04:02 smithi083 ceph-mon[79930]: pgmap v4674: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:04:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:04:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:04:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:04:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:04:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:04:04.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:04:04 smithi153 ceph-mon[70044]: pgmap v4675: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:04.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:04:04 smithi083 ceph-mon[79930]: pgmap v4675: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:04.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:04:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:04:04.098Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:04:04.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:04:04 smithi083 ceph-mon[77242]: pgmap v4675: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:05.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:04:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:04:04.802Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:04:06.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:04:06 smithi153 ceph-mon[70044]: pgmap v4676: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:06.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:04:06 smithi083 ceph-mon[79930]: pgmap v4676: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:06.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:04:06 smithi083 ceph-mon[77242]: pgmap v4676: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:08.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:04:08 smithi153 ceph-mon[70044]: pgmap v4677: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:08.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:04:08 smithi083 ceph-mon[77242]: pgmap v4677: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:08.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:04:08 smithi083 ceph-mon[79930]: pgmap v4677: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:10.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:04:10 smithi153 ceph-mon[70044]: pgmap v4678: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:10.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:04:10 smithi083 ceph-mon[79930]: pgmap v4678: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:10.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:04:10 smithi083 ceph-mon[77242]: pgmap v4678: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:11.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:04:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:04:10.712Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:04:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:04:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:04:11] "GET /metrics HTTP/1.1" 200 225209 "" "Prometheus/2.33.4" 2022-09-29T08:04:12.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:04:12 smithi153 ceph-mon[70044]: pgmap v4679: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:12.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:04:12 smithi083 ceph-mon[79930]: pgmap v4679: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:12.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:04:12 smithi083 ceph-mon[77242]: pgmap v4679: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:04:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:04:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:04:13.079 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:04:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:04:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:04:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:04:13.375 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:04:13.375 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 7m ago 2h 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:04:13.376 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 5m ago 2h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:04:13.376 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 7m ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:04:13.376 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 5m ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:04:13.376 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 5m ago 2h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:04:13.377 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 7m ago 2h 644M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:04:13.377 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 7m ago 2h 601M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:04:13.377 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 5m ago 2h 267M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:04:13.377 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 7m ago 2h 351M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:04:13.378 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 7m ago 2h 26.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:04:13.378 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 5m ago 2h 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:04:13.378 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 7m ago 2h 437M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:04:13.378 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 7m ago 2h 428M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:04:13.379 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 7m ago 2h 339M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:04:13.379 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 7m ago 2h 415M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:04:13.379 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 5m ago 2h 370M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:04:13.379 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 5m ago 2h 455M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:04:13.380 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 5m ago 2h 405M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:04:13.380 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 5m ago 2h 473M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:04:13.380 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 5m ago 2h 127M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:04:13.737 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:04:13.737 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:04:13.738 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:04:13.742 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:04:13.743 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:04:13.743 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:04:13.744 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:04:13.744 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:04:13.745 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:04:13.745 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:04:13.745 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:04:13.745 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:04:13.746 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:04:13.746 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:04:13.746 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:04:13.746 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:04:13.747 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:04:13.747 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:04:14.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:04:14 smithi153 ceph-mon[70044]: pgmap v4680: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:14.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:04:14 smithi153 ceph-mon[70044]: from='client.39579 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:04:14.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:04:14 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4287587261' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:04:14.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:04:14 smithi083 ceph-mon[77242]: pgmap v4680: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:14.500 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:04:14 smithi083 ceph-mon[77242]: from='client.39579 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:04:14.500 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:04:14 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4287587261' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:04:14.501 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:04:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:04:14.098Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:04:14.501 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:04:14 smithi083 ceph-mon[79930]: pgmap v4680: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:14.502 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:04:14 smithi083 ceph-mon[79930]: from='client.39579 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:04:14.502 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:04:14 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4287587261' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:04:15.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:04:15 smithi083 ceph-mon[77242]: from='client.49519 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:04:15.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:04:15 smithi083 ceph-mon[79930]: from='client.49519 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:04:15.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:04:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:04:14.803Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:04:15.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:04:15 smithi153 ceph-mon[70044]: from='client.49519 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:04:16.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:04:16 smithi153 ceph-mon[70044]: pgmap v4681: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:16.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:04:16 smithi083 ceph-mon[79930]: pgmap v4681: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:16.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:04:16 smithi083 ceph-mon[77242]: pgmap v4681: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:18.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:04:18 smithi153 ceph-mon[70044]: pgmap v4682: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:18.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:04:18 smithi083 ceph-mon[79930]: pgmap v4682: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:18.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:04:18 smithi083 ceph-mon[77242]: pgmap v4682: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:04:20 smithi153 ceph-mon[70044]: pgmap v4683: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:20.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:04:20 smithi083 ceph-mon[79930]: pgmap v4683: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:20.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:04:20 smithi083 ceph-mon[77242]: pgmap v4683: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:21.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:04:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:04:20.712Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:04:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:04:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:04:21] "GET /metrics HTTP/1.1" 200 225209 "" "Prometheus/2.33.4" 2022-09-29T08:04:22.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:04:22 smithi153 ceph-mon[70044]: pgmap v4684: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:22.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:04:22 smithi083 ceph-mon[79930]: pgmap v4684: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:22.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:04:22 smithi083 ceph-mon[77242]: pgmap v4684: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:22.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:04:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:04:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:04:23.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:04:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:04:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:04:24.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:04:24 smithi153 ceph-mon[70044]: pgmap v4685: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:24.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:04:24 smithi083 ceph-mon[79930]: pgmap v4685: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:24.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:04:24 smithi083 ceph-mon[77242]: pgmap v4685: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:24.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:04:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:04:24.113Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:04:25.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:04:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:04:24.804Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:04:26.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:04:26 smithi153 ceph-mon[70044]: pgmap v4686: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:26.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:04:26 smithi083 ceph-mon[77242]: pgmap v4686: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:26.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:04:26 smithi083 ceph-mon[79930]: pgmap v4686: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:28.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:04:28 smithi153 ceph-mon[70044]: pgmap v4687: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:28.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:04:28 smithi083 ceph-mon[79930]: pgmap v4687: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:28.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:04:28 smithi083 ceph-mon[77242]: pgmap v4687: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:30.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:04:30 smithi083 ceph-mon[79930]: pgmap v4688: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:30.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:04:30 smithi083 ceph-mon[77242]: pgmap v4688: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:04:30 smithi153 ceph-mon[70044]: pgmap v4688: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:31.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:04:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:04:30.713Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:04:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:04:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:04:31] "GET /metrics HTTP/1.1" 200 225209 "" "Prometheus/2.33.4" 2022-09-29T08:04:32.315 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:04:32 smithi153 ceph-mon[70044]: pgmap v4689: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:32.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:04:32 smithi083 ceph-mon[77242]: pgmap v4689: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:32.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:04:32 smithi083 ceph-mon[79930]: pgmap v4689: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:04:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:04:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:04:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:04:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:04:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:04:34.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:04:34 smithi083 ceph-mon[79930]: pgmap v4690: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:34.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:04:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:04:34.101Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:04:34.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:04:34 smithi083 ceph-mon[77242]: pgmap v4690: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:04:34 smithi153 ceph-mon[70044]: pgmap v4690: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:35.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:04:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:04:34.805Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:04:36.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:04:36 smithi083 ceph-mon[79930]: pgmap v4691: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:36.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:04:36 smithi083 ceph-mon[77242]: pgmap v4691: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:04:36 smithi153 ceph-mon[70044]: pgmap v4691: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:38.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:04:38 smithi083 ceph-mon[77242]: pgmap v4692: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:38.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:04:38 smithi083 ceph-mon[79930]: pgmap v4692: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:04:38 smithi153 ceph-mon[70044]: pgmap v4692: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:40.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:04:40 smithi083 ceph-mon[79930]: pgmap v4693: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:40.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:04:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:04:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:04:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:04:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:04:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:04:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:04:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:04:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:04:40 smithi083 ceph-mon[77242]: pgmap v4693: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:04:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:04:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:04:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:04:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:04:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:04:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:04:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:04:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:04:40 smithi153 ceph-mon[70044]: pgmap v4693: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:04:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:04:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:04:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:04:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:04:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:04:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:04:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:04:41.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:04:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:04:40.713Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:04:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:04:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:04:41] "GET /metrics HTTP/1.1" 200 225208 "" "Prometheus/2.33.4" 2022-09-29T08:04:42.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:04:42 smithi083 ceph-mon[79930]: pgmap v4694: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:42.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:04:42 smithi083 ceph-mon[77242]: pgmap v4694: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:04:42 smithi153 ceph-mon[70044]: pgmap v4694: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:42.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:04:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:04:42.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:04:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:04:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:04:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:04:43.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:04:43 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:04:43.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:04:43 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:04:43.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:04:43 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:04:43.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:04:43 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:04:43.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:04:43 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:04:43.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:04:43 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:04:43.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:04:43 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:04:43.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:04:43 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:04:43.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:04:43 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:04:44.053 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:04:44.352 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:04:44.352 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 8m ago 2h 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:04:44.352 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 6m ago 2h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:04:44.353 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 8m ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:04:44.353 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 6m ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:04:44.353 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 6m ago 2h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:04:44.353 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 8m ago 2h 644M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:04:44.354 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 8m ago 2h 601M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:04:44.354 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 6m ago 2h 267M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:04:44.354 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 8m ago 2h 351M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:04:44.355 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 8m ago 2h 26.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:04:44.355 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 6m ago 2h 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:04:44.355 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 8m ago 2h 437M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:04:44.355 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 8m ago 2h 428M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:04:44.356 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 8m ago 2h 339M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:04:44.356 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 8m ago 2h 415M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:04:44.356 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 6m ago 2h 370M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:04:44.357 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 6m ago 2h 455M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:04:44.357 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 6m ago 2h 405M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:04:44.357 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 6m ago 2h 473M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:04:44.357 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 6m ago 2h 127M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:04:44.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:04:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:04:44.103Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:04:44.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:04:44 smithi083 ceph-mon[77242]: pgmap v4695: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:44.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:04:44 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:04:44.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:04:44 smithi083 ceph-mon[79930]: pgmap v4695: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:44.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:04:44 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:04:44.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:04:44 smithi153 ceph-mon[70044]: pgmap v4695: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:44.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:04:44 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:04:44.705 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:04:44.706 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:04:44.706 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:04:44.706 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:04:44.707 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:04:44.707 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:04:44.707 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:04:44.707 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:04:44.708 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:04:44.708 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:04:44.708 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:04:44.708 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:04:44.709 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:04:44.709 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:04:44.709 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:04:44.709 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:04:44.710 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:04:44.710 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:04:45.167 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:04:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:04:44.805Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:04:45.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:04:45 smithi083 ceph-mon[79930]: from='client.49531 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:04:45.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:04:45 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2288759344' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:04:45.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:04:45 smithi083 ceph-mon[77242]: from='client.49531 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:04:45.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:04:45 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2288759344' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:04:45.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:04:45 smithi153 ceph-mon[70044]: from='client.49531 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:04:45.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:04:45 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2288759344' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:04:46.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:04:46 smithi083 ceph-mon[79930]: from='client.39603 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:04:46.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:04:46 smithi083 ceph-mon[79930]: pgmap v4696: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:46.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:04:46 smithi083 ceph-mon[77242]: from='client.39603 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:04:46.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:04:46 smithi083 ceph-mon[77242]: pgmap v4696: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:46.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:04:46 smithi153 ceph-mon[70044]: from='client.39603 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:04:46.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:04:46 smithi153 ceph-mon[70044]: pgmap v4696: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:48.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:04:48 smithi083 ceph-mon[79930]: pgmap v4697: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:48.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:04:48 smithi083 ceph-mon[77242]: pgmap v4697: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:48.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:04:48 smithi153 ceph-mon[70044]: pgmap v4697: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:50.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:04:50 smithi083 ceph-mon[79930]: pgmap v4698: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:50.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:04:50 smithi083 ceph-mon[77242]: pgmap v4698: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:04:50 smithi153 ceph-mon[70044]: pgmap v4698: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:51.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:04:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:04:50.714Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:04:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:04:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:04:51] "GET /metrics HTTP/1.1" 200 225208 "" "Prometheus/2.33.4" 2022-09-29T08:04:52.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:04:52 smithi083 ceph-mon[79930]: pgmap v4699: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:52.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:04:52 smithi083 ceph-mon[77242]: pgmap v4699: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:04:52 smithi153 ceph-mon[70044]: pgmap v4699: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:52.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:04:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:04:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:04:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:04:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:04:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:04:54.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:04:54 smithi083 ceph-mon[79930]: pgmap v4700: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:54.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:04:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:04:54.115Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:04:54.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:04:54 smithi083 ceph-mon[77242]: pgmap v4700: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:54.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:04:54 smithi153 ceph-mon[70044]: pgmap v4700: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:55.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:04:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:04:54.807Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:04:56.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:04:56 smithi083 ceph-mon[77242]: pgmap v4701: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:56.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:04:56 smithi083 ceph-mon[79930]: pgmap v4701: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:56.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:04:56 smithi153 ceph-mon[70044]: pgmap v4701: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:58.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:04:58 smithi083 ceph-mon[79930]: pgmap v4702: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:58.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:04:58 smithi083 ceph-mon[77242]: pgmap v4702: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:04:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:04:58 smithi153 ceph-mon[70044]: pgmap v4702: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:00.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:05:00 smithi083 ceph-mon[79930]: pgmap v4703: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:00.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:05:00 smithi083 ceph-mon[77242]: pgmap v4703: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:05:00 smithi153 ceph-mon[70044]: pgmap v4703: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:01.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:05:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:05:00.715Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:05:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:05:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:05:01] "GET /metrics HTTP/1.1" 200 225211 "" "Prometheus/2.33.4" 2022-09-29T08:05:02.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:05:02 smithi083 ceph-mon[77242]: pgmap v4704: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:02.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:05:02 smithi083 ceph-mon[79930]: pgmap v4704: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:05:02 smithi153 ceph-mon[70044]: pgmap v4704: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:02.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:05:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:05:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:05:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:05:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:05:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:05:04.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:05:04 smithi083 ceph-mon[79930]: pgmap v4705: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:04.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:05:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:05:04.103Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:05:04.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:05:04 smithi083 ceph-mon[77242]: pgmap v4705: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:05:04 smithi153 ceph-mon[70044]: pgmap v4705: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:05.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:05:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:05:04.807Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:05:06.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:05:06 smithi083 ceph-mon[79930]: pgmap v4706: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:06.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:05:06 smithi083 ceph-mon[77242]: pgmap v4706: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:06.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:05:06 smithi153 ceph-mon[70044]: pgmap v4706: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:08.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:05:08 smithi083 ceph-mon[79930]: pgmap v4707: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:08.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:05:08 smithi083 ceph-mon[77242]: pgmap v4707: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:05:08 smithi153 ceph-mon[70044]: pgmap v4707: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:10.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:05:10 smithi153 ceph-mon[70044]: pgmap v4708: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:10.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:05:10 smithi083 ceph-mon[79930]: pgmap v4708: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:10.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:05:10 smithi083 ceph-mon[77242]: pgmap v4708: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:11.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:05:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:05:10.716Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:05:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:05:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:05:11] "GET /metrics HTTP/1.1" 200 225213 "" "Prometheus/2.33.4" 2022-09-29T08:05:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:05:12 smithi153 ceph-mon[70044]: pgmap v4709: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:12.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:05:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:05:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:05:12.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:05:12 smithi083 ceph-mon[77242]: pgmap v4709: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:12.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:05:12 smithi083 ceph-mon[79930]: pgmap v4709: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:05:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:05:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:05:14.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:05:14 smithi083 ceph-mon[79930]: pgmap v4710: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:14.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:05:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:05:14.104Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:05:14.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:05:14 smithi083 ceph-mon[77242]: pgmap v4710: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:14.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:05:14 smithi153 ceph-mon[70044]: pgmap v4710: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:15.034 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:05:15.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:05:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:05:14.808Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:05:15.324 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:05:15.324 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 8m ago 2h 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:05:15.324 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 6m ago 2h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:05:15.325 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 8m ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:05:15.325 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 6m ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:05:15.325 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 6m ago 2h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:05:15.325 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 8m ago 2h 644M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:05:15.326 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 8m ago 2h 601M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:05:15.326 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 6m ago 2h 267M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:05:15.326 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 8m ago 2h 351M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:05:15.326 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 8m ago 2h 26.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:05:15.327 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 6m ago 2h 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:05:15.327 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 8m ago 2h 437M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:05:15.327 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 8m ago 2h 428M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:05:15.327 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 8m ago 2h 339M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:05:15.328 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 8m ago 2h 415M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:05:15.328 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 6m ago 2h 370M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:05:15.328 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 6m ago 2h 455M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:05:15.328 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 6m ago 2h 405M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:05:15.329 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 6m ago 2h 473M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:05:15.329 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 6m ago 2h 127M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:05:15.676 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:05:15.677 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:05:15.677 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:05:15.677 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:05:15.678 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:05:15.678 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:05:15.678 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:05:15.679 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:05:15.679 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:05:15.679 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:05:15.679 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:05:15.680 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:05:15.680 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:05:15.680 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:05:15.680 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:05:15.681 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:05:15.681 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:05:15.681 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:05:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:05:16 smithi153 ceph-mon[70044]: pgmap v4711: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:05:16 smithi153 ceph-mon[70044]: from='client.49549 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:05:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:05:16 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1803310060' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:05:16.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:05:16 smithi083 ceph-mon[79930]: pgmap v4711: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:16.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:05:16 smithi083 ceph-mon[79930]: from='client.49549 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:05:16.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:05:16 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1803310060' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:05:16.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:05:16 smithi083 ceph-mon[77242]: pgmap v4711: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:16.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:05:16 smithi083 ceph-mon[77242]: from='client.49549 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:05:16.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:05:16 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1803310060' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:05:17.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:05:17 smithi153 ceph-mon[70044]: from='client.39621 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:05:17.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:05:17 smithi083 ceph-mon[77242]: from='client.39621 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:05:17.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:05:17 smithi083 ceph-mon[79930]: from='client.39621 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:05:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:05:18 smithi153 ceph-mon[70044]: pgmap v4712: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:18.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:05:18 smithi083 ceph-mon[79930]: pgmap v4712: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:18.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:05:18 smithi083 ceph-mon[77242]: pgmap v4712: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:05:20 smithi153 ceph-mon[70044]: pgmap v4713: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:20.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:05:20 smithi083 ceph-mon[79930]: pgmap v4713: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:20.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:05:20 smithi083 ceph-mon[77242]: pgmap v4713: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:21.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:05:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:05:20.716Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:05:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:05:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:05:21] "GET /metrics HTTP/1.1" 200 225213 "" "Prometheus/2.33.4" 2022-09-29T08:05:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:05:22 smithi153 ceph-mon[70044]: pgmap v4714: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:22.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:05:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:05:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:05:22.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:05:22 smithi083 ceph-mon[79930]: pgmap v4714: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:22.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:05:22 smithi083 ceph-mon[77242]: pgmap v4714: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:05:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:05:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:05:24.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:05:24 smithi083 ceph-mon[79930]: pgmap v4715: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:24.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:05:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:05:24.112Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:05:24.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:05:24 smithi083 ceph-mon[77242]: pgmap v4715: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:24.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:05:24 smithi153 ceph-mon[70044]: pgmap v4715: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:25.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:05:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:05:24.808Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:05:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:05:26 smithi153 ceph-mon[70044]: pgmap v4716: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:26.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:05:26 smithi083 ceph-mon[79930]: pgmap v4716: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:26.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:05:26 smithi083 ceph-mon[77242]: pgmap v4716: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:05:28 smithi153 ceph-mon[70044]: pgmap v4717: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:28.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:05:28 smithi083 ceph-mon[77242]: pgmap v4717: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:28.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:05:28 smithi083 ceph-mon[79930]: pgmap v4717: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:05:30 smithi153 ceph-mon[70044]: pgmap v4718: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:30.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:05:30 smithi083 ceph-mon[79930]: pgmap v4718: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:30.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:05:30 smithi083 ceph-mon[77242]: pgmap v4718: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:31.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:05:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:05:30.717Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:05:31.506 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:05:31 smithi083 ceph-mon[79930]: pgmap v4719: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:31.507 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:05:31 smithi083 ceph-mon[77242]: pgmap v4719: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:31.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:05:31 smithi153 ceph-mon[70044]: pgmap v4719: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:05:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:05:31] "GET /metrics HTTP/1.1" 200 225207 "" "Prometheus/2.33.4" 2022-09-29T08:05:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:05:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:05:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:05:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:05:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:05:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:05:34.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:05:33 smithi153 ceph-mon[70044]: pgmap v4720: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:34.106 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:05:33 smithi083 ceph-mon[79930]: pgmap v4720: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:34.107 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:05:33 smithi083 ceph-mon[77242]: pgmap v4720: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:34.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:05:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:05:34.105Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:05:35.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:05:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:05:34.808Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:05:36.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:05:35 smithi153 ceph-mon[70044]: pgmap v4721: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:36.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:05:35 smithi083 ceph-mon[77242]: pgmap v4721: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:36.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:05:35 smithi083 ceph-mon[79930]: pgmap v4721: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:38.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:05:37 smithi153 ceph-mon[70044]: pgmap v4722: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:38.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:05:37 smithi083 ceph-mon[77242]: pgmap v4722: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:38.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:05:37 smithi083 ceph-mon[79930]: pgmap v4722: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:05:39 smithi153 ceph-mon[70044]: pgmap v4723: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:05:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:05:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:05:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:05:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:05:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:05:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:05:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:05:40.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:05:39 smithi083 ceph-mon[79930]: pgmap v4723: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:40.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:05:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:05:40.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:05:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:05:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:05:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:05:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:05:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:05:40.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:05:39 smithi083 ceph-mon[77242]: pgmap v4723: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:05:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:05:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:05:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:05:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:05:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:05:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:05:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:05:41.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:05:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:05:40.718Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:05:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:05:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:05:41] "GET /metrics HTTP/1.1" 200 225211 "" "Prometheus/2.33.4" 2022-09-29T08:05:41.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:05:41 smithi083 ceph-mon[79930]: pgmap v4724: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:41.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:05:41 smithi083 ceph-mon[77242]: pgmap v4724: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:42.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:05:41 smithi153 ceph-mon[70044]: pgmap v4724: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:05:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:05:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:05:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:05:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:05:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:05:44.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:05:43 smithi153 ceph-mon[70044]: pgmap v4725: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:44.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:05:43 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:05:44.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:05:43 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:05:44.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:05:43 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:05:44.107 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:05:43 smithi083 ceph-mon[79930]: pgmap v4725: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:44.107 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:05:43 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:05:44.107 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:05:43 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:05:44.108 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:05:43 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:05:44.108 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:05:43 smithi083 ceph-mon[77242]: pgmap v4725: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:44.108 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:05:43 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:05:44.109 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:05:43 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:05:44.109 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:05:43 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:05:44.460 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:05:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:05:44.105Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:05:45.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:05:44 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:05:45.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:05:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:05:44.809Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:05:45.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:05:44 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:05:45.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:05:44 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:05:46.006 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:05:46.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:05:45 smithi083 ceph-mon[79930]: pgmap v4726: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:46.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:05:45 smithi083 ceph-mon[77242]: pgmap v4726: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:46.293 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:05:46.294 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 9m ago 2h 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:05:46.294 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 7m ago 2h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:05:46.294 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 9m ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:05:46.295 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 7m ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:05:46.295 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 7m ago 2h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:05:46.295 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 9m ago 2h 644M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:05:46.295 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 9m ago 2h 601M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:05:46.296 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 7m ago 2h 267M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:05:46.296 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 9m ago 2h 351M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:05:46.296 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 9m ago 2h 26.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:05:46.296 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 7m ago 2h 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:05:46.297 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 9m ago 2h 437M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:05:46.297 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 9m ago 2h 428M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:05:46.297 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 9m ago 2h 339M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:05:46.297 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 9m ago 2h 415M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:05:46.298 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 7m ago 2h 370M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:05:46.298 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 7m ago 2h 455M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:05:46.298 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 7m ago 2h 405M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:05:46.298 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 7m ago 2h 473M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:05:46.299 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 7m ago 2h 127M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:05:46.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:05:45 smithi153 ceph-mon[70044]: pgmap v4726: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:46.651 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:05:46.669 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:05:46.669 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:05:46.670 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:05:46.670 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:05:46.670 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:05:46.671 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:05:46.671 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:05:46.671 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:05:46.671 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:05:46.672 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:05:46.672 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:05:46.672 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:05:46.672 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:05:46.672 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:05:46.673 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:05:46.673 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:05:46.673 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:05:47.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:05:46 smithi083 ceph-mon[79930]: from='client.39633 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:05:47.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:05:46 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3981165283' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:05:47.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:05:46 smithi083 ceph-mon[77242]: from='client.39633 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:05:47.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:05:46 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3981165283' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:05:47.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:05:46 smithi153 ceph-mon[70044]: from='client.39633 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:05:47.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:05:46 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3981165283' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:05:48.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:05:47 smithi083 ceph-mon[79930]: from='client.49573 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:05:48.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:05:47 smithi083 ceph-mon[79930]: pgmap v4727: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:48.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:05:47 smithi083 ceph-mon[77242]: from='client.49573 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:05:48.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:05:47 smithi083 ceph-mon[77242]: pgmap v4727: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:48.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:05:47 smithi153 ceph-mon[70044]: from='client.49573 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:05:48.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:05:47 smithi153 ceph-mon[70044]: pgmap v4727: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:50.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:05:49 smithi083 ceph-mon[77242]: pgmap v4728: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:50.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:05:49 smithi083 ceph-mon[79930]: pgmap v4728: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:50.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:05:49 smithi153 ceph-mon[70044]: pgmap v4728: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:51.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:05:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:05:50.719Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:05:51.842 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:05:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:05:51] "GET /metrics HTTP/1.1" 200 225211 "" "Prometheus/2.33.4" 2022-09-29T08:05:52.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:05:51 smithi083 ceph-mon[79930]: pgmap v4729: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:52.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:05:51 smithi083 ceph-mon[77242]: pgmap v4729: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:52.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:05:51 smithi153 ceph-mon[70044]: pgmap v4729: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:05:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:05:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:05:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:05:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:05:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:05:54.115 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:05:53 smithi083 ceph-mon[79930]: pgmap v4730: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:54.115 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:05:53 smithi083 ceph-mon[77242]: pgmap v4730: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:54.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:05:53 smithi153 ceph-mon[70044]: pgmap v4730: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:54.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:05:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:05:54.114Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:05:55.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:05:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:05:54.810Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:05:56.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:05:55 smithi083 ceph-mon[79930]: pgmap v4731: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:56.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:05:55 smithi083 ceph-mon[77242]: pgmap v4731: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:56.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:05:55 smithi153 ceph-mon[70044]: pgmap v4731: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:58.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:05:57 smithi083 ceph-mon[79930]: pgmap v4732: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:58.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:05:57 smithi083 ceph-mon[77242]: pgmap v4732: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:05:58.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:05:57 smithi153 ceph-mon[70044]: pgmap v4732: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:00.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:05:59 smithi083 ceph-mon[79930]: pgmap v4733: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:00.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:05:59 smithi083 ceph-mon[77242]: pgmap v4733: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:05:59 smithi153 ceph-mon[70044]: pgmap v4733: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:01.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:06:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:06:00.720Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:06:01.858 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:06:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:06:01] "GET /metrics HTTP/1.1" 200 225212 "" "Prometheus/2.33.4" 2022-09-29T08:06:02.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:06:01 smithi083 ceph-mon[79930]: pgmap v4734: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:02.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:06:01 smithi083 ceph-mon[77242]: pgmap v4734: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:02.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:06:01 smithi153 ceph-mon[70044]: pgmap v4734: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:06:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:06:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:06:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:06:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:06:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:06:04.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:06:03 smithi083 ceph-mon[77242]: pgmap v4735: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:04.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:06:03 smithi083 ceph-mon[79930]: pgmap v4735: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:04.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:06:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:06:04.108Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:06:04.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:06:03 smithi153 ceph-mon[70044]: pgmap v4735: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:05.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:06:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:06:04.810Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:06:06.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:06:05 smithi083 ceph-mon[77242]: pgmap v4736: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:06.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:06:05 smithi083 ceph-mon[79930]: pgmap v4736: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:06.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:06:05 smithi153 ceph-mon[70044]: pgmap v4736: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:08.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:06:07 smithi083 ceph-mon[77242]: pgmap v4737: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:08.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:06:07 smithi083 ceph-mon[79930]: pgmap v4737: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:08.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:06:07 smithi153 ceph-mon[70044]: pgmap v4737: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:10.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:06:09 smithi083 ceph-mon[79930]: pgmap v4738: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:10.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:06:09 smithi083 ceph-mon[77242]: pgmap v4738: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:10.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:06:09 smithi153 ceph-mon[70044]: pgmap v4738: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:11.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:06:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:06:10.721Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:06:11.872 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:06:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:06:11] "GET /metrics HTTP/1.1" 200 225212 "" "Prometheus/2.33.4" 2022-09-29T08:06:12.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:06:11 smithi083 ceph-mon[79930]: pgmap v4739: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:12.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:06:11 smithi083 ceph-mon[77242]: pgmap v4739: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:12.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:06:11 smithi153 ceph-mon[70044]: pgmap v4739: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:06:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:06:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:06:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:06:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:06:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:06:14.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:06:13 smithi083 ceph-mon[79930]: pgmap v4740: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:14.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:06:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:06:14.109Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:06:14.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:06:13 smithi083 ceph-mon[77242]: pgmap v4740: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:14.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:06:13 smithi153 ceph-mon[70044]: pgmap v4740: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:15.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:06:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:06:14.811Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:06:16.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:06:15 smithi083 ceph-mon[79930]: pgmap v4741: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:16.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:06:15 smithi083 ceph-mon[77242]: pgmap v4741: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:16.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:06:15 smithi153 ceph-mon[70044]: pgmap v4741: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:16.978 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:06:17.267 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:06:17.268 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 9m ago 2h 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:06:17.268 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 7m ago 2h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:06:17.268 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 9m ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:06:17.268 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 7m ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:06:17.269 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 7m ago 2h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:06:17.269 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 9m ago 2h 644M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:06:17.269 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 9m ago 2h 601M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:06:17.270 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 7m ago 2h 267M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:06:17.270 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 9m ago 2h 351M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:06:17.270 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 9m ago 2h 26.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:06:17.270 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 7m ago 2h 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:06:17.271 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 9m ago 2h 437M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:06:17.271 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 9m ago 2h 428M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:06:17.271 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 9m ago 2h 339M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:06:17.271 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 9m ago 2h 415M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:06:17.272 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 7m ago 2h 370M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:06:17.272 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 7m ago 2h 455M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:06:17.272 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 7m ago 2h 405M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:06:17.272 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 7m ago 2h 473M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:06:17.272 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 7m ago 2h 127M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:06:17.624 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:06:17.624 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:06:17.625 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:06:17.625 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:06:17.625 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:06:17.625 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:06:17.626 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:06:17.626 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:06:17.626 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:06:17.627 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:06:17.627 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:06:17.627 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:06:17.627 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:06:17.628 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:06:17.628 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:06:17.628 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:06:17.628 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:06:17.629 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:06:18.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:06:17 smithi083 ceph-mon[79930]: pgmap v4742: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:18.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:06:17 smithi083 ceph-mon[79930]: from='client.39651 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:06:18.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:06:17 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2719185308' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:06:18.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:06:17 smithi083 ceph-mon[77242]: pgmap v4742: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:18.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:06:17 smithi083 ceph-mon[77242]: from='client.39651 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:06:18.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:06:17 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2719185308' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:06:18.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:06:17 smithi153 ceph-mon[70044]: pgmap v4742: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:18.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:06:17 smithi153 ceph-mon[70044]: from='client.39651 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:06:18.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:06:17 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2719185308' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:06:19.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:06:18 smithi083 ceph-mon[79930]: from='client.39657 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:06:19.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:06:18 smithi083 ceph-mon[77242]: from='client.39657 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:06:19.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:06:18 smithi153 ceph-mon[70044]: from='client.39657 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:06:20.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:06:19 smithi083 ceph-mon[79930]: pgmap v4743: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:20.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:06:19 smithi083 ceph-mon[77242]: pgmap v4743: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:06:19 smithi153 ceph-mon[70044]: pgmap v4743: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:21.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:06:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:06:20.721Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:06:21.891 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:06:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:06:21] "GET /metrics HTTP/1.1" 200 225212 "" "Prometheus/2.33.4" 2022-09-29T08:06:22.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:06:21 smithi083 ceph-mon[77242]: pgmap v4744: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:22.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:06:21 smithi083 ceph-mon[79930]: pgmap v4744: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:22.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:06:21 smithi153 ceph-mon[70044]: pgmap v4744: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:06:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:06:22.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:06:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:06:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:06:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:06:24.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:06:23 smithi083 ceph-mon[77242]: pgmap v4745: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:24.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:06:23 smithi083 ceph-mon[79930]: pgmap v4745: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:24.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:06:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:06:24.115Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:06:24.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:06:23 smithi153 ceph-mon[70044]: pgmap v4745: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:25.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:06:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:06:24.812Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:06:26.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:06:25 smithi083 ceph-mon[79930]: pgmap v4746: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:26.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:06:25 smithi083 ceph-mon[77242]: pgmap v4746: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:26.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:06:25 smithi153 ceph-mon[70044]: pgmap v4746: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:28.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:06:27 smithi083 ceph-mon[79930]: pgmap v4747: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:28.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:06:27 smithi083 ceph-mon[77242]: pgmap v4747: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:28.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:06:27 smithi153 ceph-mon[70044]: pgmap v4747: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:30.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:06:29 smithi083 ceph-mon[77242]: pgmap v4748: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:30.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:06:29 smithi083 ceph-mon[79930]: pgmap v4748: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:30.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:06:29 smithi153 ceph-mon[70044]: pgmap v4748: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:31.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:06:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:06:30.722Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:06:31.902 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:06:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:06:31] "GET /metrics HTTP/1.1" 200 225214 "" "Prometheus/2.33.4" 2022-09-29T08:06:32.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:06:31 smithi083 ceph-mon[79930]: pgmap v4749: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:32.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:06:31 smithi083 ceph-mon[77242]: pgmap v4749: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:32.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:06:31 smithi153 ceph-mon[70044]: pgmap v4749: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:06:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:06:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:06:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:06:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:06:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:06:34.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:06:33 smithi083 ceph-mon[79930]: pgmap v4750: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:34.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:06:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:06:34.110Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:06:34.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:06:33 smithi083 ceph-mon[77242]: pgmap v4750: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:34.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:06:33 smithi153 ceph-mon[70044]: pgmap v4750: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:35.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:06:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:06:34.813Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:06:36.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:06:35 smithi083 ceph-mon[77242]: pgmap v4751: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:36.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:06:35 smithi083 ceph-mon[79930]: pgmap v4751: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:36.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:06:35 smithi153 ceph-mon[70044]: pgmap v4751: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:38.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:06:37 smithi083 ceph-mon[77242]: pgmap v4752: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:38.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:06:37 smithi083 ceph-mon[79930]: pgmap v4752: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:38.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:06:37 smithi153 ceph-mon[70044]: pgmap v4752: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:40.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:06:39 smithi083 ceph-mon[77242]: pgmap v4753: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:40.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:06:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:06:40.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:06:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:06:40.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:06:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:06:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:06:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:06:40.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:06:39 smithi083 ceph-mon[79930]: pgmap v4753: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:40.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:06:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:06:40.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:06:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:06:40.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:06:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:06:40.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:06:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:06:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:06:39 smithi153 ceph-mon[70044]: pgmap v4753: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:06:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:06:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:06:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:06:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:06:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:06:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:06:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:06:41.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:06:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:06:40.723Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:06:41.915 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:06:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:06:41] "GET /metrics HTTP/1.1" 200 225206 "" "Prometheus/2.33.4" 2022-09-29T08:06:42.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:06:41 smithi083 ceph-mon[79930]: pgmap v4754: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:42.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:06:41 smithi083 ceph-mon[77242]: pgmap v4754: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:06:41 smithi153 ceph-mon[70044]: pgmap v4754: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:06:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:06:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:06:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:06:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:06:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:06:44.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:06:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:06:44.111Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:06:44.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:06:43 smithi083 ceph-mon[77242]: pgmap v4755: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:44.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:06:43 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:06:44.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:06:43 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:06:44.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:06:43 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:06:44.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:06:43 smithi083 ceph-mon[79930]: pgmap v4755: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:44.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:06:43 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:06:44.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:06:43 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:06:44.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:06:43 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:06:44.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:06:43 smithi153 ceph-mon[70044]: pgmap v4755: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:44.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:06:43 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:06:44.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:06:43 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:06:44.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:06:43 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:06:45.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:06:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:06:44.813Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:06:45.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:06:45 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:06:45.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:06:45 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:06:45.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:06:45 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:06:45.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:06:45 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:06:45.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:06:45 smithi083 ceph-mon[79930]: pgmap v4756: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:45.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:06:45 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:06:45.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:06:45 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:06:45.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:06:45 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:06:45.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:06:45 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:06:45.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:06:45 smithi083 ceph-mon[77242]: pgmap v4756: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:45.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:06:45 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:06:45.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:06:45 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:06:45.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:06:45 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:06:45.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:06:45 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:06:45.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:06:45 smithi153 ceph-mon[70044]: pgmap v4756: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:47.965 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:06:48.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:06:47 smithi153 ceph-mon[70044]: pgmap v4757: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:48.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:06:47 smithi083 ceph-mon[79930]: pgmap v4757: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:48.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:06:47 smithi083 ceph-mon[77242]: pgmap v4757: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:48.283 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:06:48.284 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 10m ago 2h 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:06:48.284 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 8m ago 2h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:06:48.284 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 10m ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:06:48.285 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 8m ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:06:48.285 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 8m ago 2h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:06:48.285 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 10m ago 2h 644M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:06:48.285 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 10m ago 2h 601M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:06:48.285 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 8m ago 2h 267M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:06:48.286 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 10m ago 2h 351M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:06:48.286 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 10m ago 2h 26.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:06:48.286 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 8m ago 2h 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:06:48.287 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 10m ago 2h 437M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:06:48.287 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 10m ago 2h 428M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:06:48.287 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 10m ago 2h 339M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:06:48.287 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 10m ago 2h 415M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:06:48.288 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 8m ago 2h 370M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:06:48.288 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 8m ago 2h 455M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:06:48.288 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 8m ago 2h 405M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:06:48.288 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 8m ago 2h 473M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:06:48.289 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 8m ago 2h 127M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:06:48.649 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:06:48.649 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:06:48.650 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:06:48.650 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:06:48.650 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:06:48.650 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:06:48.651 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:06:48.651 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:06:48.652 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:06:48.652 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:06:48.652 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:06:48.653 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:06:48.653 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:06:48.653 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:06:48.653 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:06:48.654 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:06:48.654 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:06:48.654 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:06:49.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:06:48 smithi153 ceph-mon[70044]: from='client.39669 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:06:49.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:06:48 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:06:49.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:06:48 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4029064639' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:06:49.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:06:48 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:06:49.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:06:48 smithi083 ceph-mon[79930]: from='client.39669 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:06:49.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:06:48 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:06:49.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:06:48 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4029064639' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:06:49.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:06:48 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:06:49.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:06:48 smithi083 ceph-mon[77242]: from='client.39669 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:06:49.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:06:48 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:06:49.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:06:48 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4029064639' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:06:49.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:06:48 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:06:50.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:06:49 smithi153 ceph-mon[70044]: from='client.49609 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:06:50.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:06:49 smithi153 ceph-mon[70044]: pgmap v4758: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:50.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:06:49 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:06:50.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:06:49 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:06:50.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:06:49 smithi083 ceph-mon[79930]: from='client.49609 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:06:50.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:06:49 smithi083 ceph-mon[79930]: pgmap v4758: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:50.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:06:49 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:06:50.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:06:49 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:06:50.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:06:49 smithi083 ceph-mon[77242]: from='client.49609 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:06:50.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:06:49 smithi083 ceph-mon[77242]: pgmap v4758: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:50.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:06:49 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:06:50.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:06:49 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:06:51.057 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:06:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:06:50.723Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:06:51.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:06:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:06:51] "GET /metrics HTTP/1.1" 200 225206 "" "Prometheus/2.33.4" 2022-09-29T08:06:52.031 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:06:51 smithi083 ceph-mon[79930]: pgmap v4759: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:52.032 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:06:51 smithi083 ceph-mon[77242]: pgmap v4759: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:52.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:06:51 smithi153 ceph-mon[70044]: pgmap v4759: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:06:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:06:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:06:53.305 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:06:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:06:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:06:53.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:06:53 smithi153 ceph-mon[70044]: Detected new or changed devices on smithi083 2022-09-29T08:06:53.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:06:53 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:06:53.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:06:53 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:06:53.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:06:53 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:06:53.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:06:53 smithi153 ceph-mon[70044]: pgmap v4760: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:53.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:06:53 smithi083 ceph-mon[79930]: Detected new or changed devices on smithi083 2022-09-29T08:06:53.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:06:53 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:06:53.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:06:53 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:06:53.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:06:53 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:06:53.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:06:53 smithi083 ceph-mon[79930]: pgmap v4760: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:53.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:06:53 smithi083 ceph-mon[77242]: Detected new or changed devices on smithi083 2022-09-29T08:06:53.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:06:53 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:06:53.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:06:53 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:06:53.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:06:53 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:06:53.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:06:53 smithi083 ceph-mon[77242]: pgmap v4760: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:54.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:06:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:06:54.118Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:06:55.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:06:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:06:54.814Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:06:55.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:06:55 smithi153 ceph-mon[70044]: pgmap v4761: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:55.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:06:55 smithi083 ceph-mon[79930]: pgmap v4761: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:55.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:06:55 smithi083 ceph-mon[77242]: pgmap v4761: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:58.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:06:57 smithi153 ceph-mon[70044]: pgmap v4762: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:58.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:06:57 smithi083 ceph-mon[79930]: pgmap v4762: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:06:58.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:06:57 smithi083 ceph-mon[77242]: pgmap v4762: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:00.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:06:59 smithi153 ceph-mon[70044]: pgmap v4763: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:00.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:06:59 smithi083 ceph-mon[77242]: pgmap v4763: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:00.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:06:59 smithi083 ceph-mon[79930]: pgmap v4763: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:01.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:07:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:07:00.724Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:07:01.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:07:01 smithi083 ceph-mon[79930]: pgmap v4764: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:01.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:07:01 smithi083 ceph-mon[77242]: pgmap v4764: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:07:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:07:01] "GET /metrics HTTP/1.1" 200 225213 "" "Prometheus/2.33.4" 2022-09-29T08:07:02.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:07:01 smithi153 ceph-mon[70044]: pgmap v4764: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:07:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:07:02.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:07:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:07:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:07:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:07:04.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:07:03 smithi153 ceph-mon[70044]: pgmap v4765: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:04.112 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:07:03 smithi083 ceph-mon[79930]: pgmap v4765: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:04.113 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:07:03 smithi083 ceph-mon[77242]: pgmap v4765: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:04.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:07:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:07:04.111Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:07:05.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:07:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:07:04.815Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:07:06.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:07:05 smithi153 ceph-mon[70044]: pgmap v4766: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:06.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:07:05 smithi083 ceph-mon[79930]: pgmap v4766: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:06.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:07:05 smithi083 ceph-mon[77242]: pgmap v4766: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:08.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:07:07 smithi153 ceph-mon[70044]: pgmap v4767: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:08.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:07:07 smithi083 ceph-mon[77242]: pgmap v4767: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:08.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:07:07 smithi083 ceph-mon[79930]: pgmap v4767: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:10.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:07:09 smithi153 ceph-mon[70044]: pgmap v4768: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:10.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:07:09 smithi083 ceph-mon[79930]: pgmap v4768: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:10.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:07:09 smithi083 ceph-mon[77242]: pgmap v4768: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:11.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:07:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:07:10.724Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:07:11.758 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:07:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:07:11] "GET /metrics HTTP/1.1" 200 225211 "" "Prometheus/2.33.4" 2022-09-29T08:07:12.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:07:11 smithi153 ceph-mon[70044]: pgmap v4769: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:12.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:07:11 smithi083 ceph-mon[79930]: pgmap v4769: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:12.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:07:11 smithi083 ceph-mon[77242]: pgmap v4769: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:07:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:07:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:07:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:07:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:07:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:07:14.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:07:13 smithi153 ceph-mon[70044]: pgmap v4770: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:14.113 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:07:13 smithi083 ceph-mon[79930]: pgmap v4770: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:14.113 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:07:13 smithi083 ceph-mon[77242]: pgmap v4770: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:14.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:07:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:07:14.112Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:07:15.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:07:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:07:14.815Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:07:16.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:07:15 smithi153 ceph-mon[70044]: pgmap v4771: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:16.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:07:15 smithi083 ceph-mon[79930]: pgmap v4771: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:16.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:07:15 smithi083 ceph-mon[77242]: pgmap v4771: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:18.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:07:17 smithi153 ceph-mon[70044]: pgmap v4772: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:18.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:07:17 smithi083 ceph-mon[79930]: pgmap v4772: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:18.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:07:17 smithi083 ceph-mon[77242]: pgmap v4772: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:18.980 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:07:19.267 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:07:19.268 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 30s ago 2h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:07:19.268 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 8m ago 2h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:07:19.268 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 30s ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:07:19.269 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 8m ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:07:19.269 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 8m ago 2h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:07:19.269 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 30s ago 2h 653M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:07:19.269 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 30s ago 2h 638M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:07:19.270 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 8m ago 2h 267M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:07:19.270 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 30s ago 2h 372M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:07:19.270 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 30s ago 2h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:07:19.270 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 8m ago 2h 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:07:19.271 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 30s ago 2h 462M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:07:19.271 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 30s ago 2h 453M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:07:19.271 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 30s ago 2h 358M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:07:19.271 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 30s ago 2h 440M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:07:19.272 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 8m ago 2h 370M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:07:19.272 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 8m ago 2h 455M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:07:19.272 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 8m ago 2h 405M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:07:19.272 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 8m ago 2h 473M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:07:19.273 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 8m ago 2h 127M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:07:19.626 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:07:19.626 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:07:19.627 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:07:19.627 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:07:19.627 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:07:19.627 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:07:19.628 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:07:19.628 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:07:19.628 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:07:19.628 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:07:19.629 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:07:19.629 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:07:19.629 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:07:19.629 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:07:19.629 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:07:19.630 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:07:19.630 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:07:19.630 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:07:20.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:07:19 smithi153 ceph-mon[70044]: pgmap v4773: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:20.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:07:19 smithi153 ceph-mon[70044]: from='client.39687 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:07:20.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:07:19 smithi153 ceph-mon[70044]: from='client.39693 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:07:20.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:07:19 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3605823860' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:07:20.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:07:19 smithi083 ceph-mon[79930]: pgmap v4773: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:20.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:07:19 smithi083 ceph-mon[79930]: from='client.39687 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:07:20.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:07:19 smithi083 ceph-mon[79930]: from='client.39693 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:07:20.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:07:19 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3605823860' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:07:20.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:07:19 smithi083 ceph-mon[77242]: pgmap v4773: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:20.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:07:19 smithi083 ceph-mon[77242]: from='client.39687 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:07:20.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:07:19 smithi083 ceph-mon[77242]: from='client.39693 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:07:20.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:07:19 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3605823860' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:07:21.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:07:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:07:20.725Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:07:21.772 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:07:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:07:21] "GET /metrics HTTP/1.1" 200 225211 "" "Prometheus/2.33.4" 2022-09-29T08:07:22.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:07:21 smithi153 ceph-mon[70044]: pgmap v4774: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:22.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:07:21 smithi083 ceph-mon[79930]: pgmap v4774: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:22.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:07:21 smithi083 ceph-mon[77242]: pgmap v4774: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:07:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:07:22.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:07:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:07:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:07:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:07:24.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:07:23 smithi153 ceph-mon[70044]: pgmap v4775: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:24.118 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:07:23 smithi083 ceph-mon[79930]: pgmap v4775: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:24.119 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:07:23 smithi083 ceph-mon[77242]: pgmap v4775: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:24.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:07:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:07:24.117Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:07:25.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:07:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:07:24.816Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:07:26.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:07:25 smithi153 ceph-mon[70044]: pgmap v4776: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:26.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:07:25 smithi083 ceph-mon[79930]: pgmap v4776: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:26.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:07:25 smithi083 ceph-mon[77242]: pgmap v4776: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:28.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:07:27 smithi153 ceph-mon[70044]: pgmap v4777: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:28.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:07:27 smithi083 ceph-mon[79930]: pgmap v4777: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:28.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:07:27 smithi083 ceph-mon[77242]: pgmap v4777: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:30.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:07:29 smithi153 ceph-mon[70044]: pgmap v4778: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:30.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:07:29 smithi083 ceph-mon[79930]: pgmap v4778: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:30.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:07:29 smithi083 ceph-mon[77242]: pgmap v4778: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:31.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:07:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:07:30.726Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:07:31.789 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:07:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:07:31] "GET /metrics HTTP/1.1" 200 225208 "" "Prometheus/2.33.4" 2022-09-29T08:07:32.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:07:31 smithi153 ceph-mon[70044]: pgmap v4779: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:32.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:07:31 smithi083 ceph-mon[77242]: pgmap v4779: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:32.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:07:31 smithi083 ceph-mon[79930]: pgmap v4779: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:07:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:07:32.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:07:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:07:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:07:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:07:34.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:07:33 smithi153 ceph-mon[70044]: pgmap v4780: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:34.115 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:07:33 smithi083 ceph-mon[79930]: pgmap v4780: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:34.116 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:07:33 smithi083 ceph-mon[77242]: pgmap v4780: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:34.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:07:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:07:34.114Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:07:35.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:07:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:07:34.817Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:07:36.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:07:35 smithi153 ceph-mon[70044]: pgmap v4781: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:36.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:07:35 smithi083 ceph-mon[79930]: pgmap v4781: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:36.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:07:35 smithi083 ceph-mon[77242]: pgmap v4781: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:38.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:07:37 smithi153 ceph-mon[70044]: pgmap v4782: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:38.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:07:37 smithi083 ceph-mon[79930]: pgmap v4782: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:38.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:07:37 smithi083 ceph-mon[77242]: pgmap v4782: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:07:39 smithi153 ceph-mon[70044]: pgmap v4783: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:07:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:07:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:07:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:07:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:07:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:07:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:07:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:07:40.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:07:39 smithi083 ceph-mon[79930]: pgmap v4783: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:40.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:07:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:07:40.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:07:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:07:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:07:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:07:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:07:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:07:40.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:07:39 smithi083 ceph-mon[77242]: pgmap v4783: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:40.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:07:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:07:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:07:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:07:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:07:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:07:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:07:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:07:41.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:07:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:07:40.726Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:07:41.811 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:07:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:07:41] "GET /metrics HTTP/1.1" 200 225210 "" "Prometheus/2.33.4" 2022-09-29T08:07:42.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:07:41 smithi083 ceph-mon[79930]: pgmap v4784: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:42.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:07:41 smithi083 ceph-mon[77242]: pgmap v4784: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:07:41 smithi153 ceph-mon[70044]: pgmap v4784: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:07:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:07:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:07:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:07:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:07:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:07:44.117 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:07:43 smithi083 ceph-mon[79930]: pgmap v4785: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:44.117 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:07:43 smithi083 ceph-mon[77242]: pgmap v4785: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:44.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:07:43 smithi153 ceph-mon[70044]: pgmap v4785: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:44.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:07:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:07:44.115Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:07:45.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:07:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:07:44.817Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:07:46.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:07:45 smithi083 ceph-mon[77242]: pgmap v4786: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:46.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:07:45 smithi083 ceph-mon[79930]: pgmap v4786: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:46.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:07:45 smithi153 ceph-mon[70044]: pgmap v4786: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:48.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:07:47 smithi083 ceph-mon[79930]: pgmap v4787: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:48.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:07:47 smithi083 ceph-mon[77242]: pgmap v4787: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:48.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:07:47 smithi153 ceph-mon[70044]: pgmap v4787: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:49.955 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:07:50.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:07:49 smithi083 ceph-mon[79930]: pgmap v4788: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:50.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:07:49 smithi083 ceph-mon[77242]: pgmap v4788: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:50.247 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:07:50.247 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 61s ago 2h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:07:50.248 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 9m ago 2h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:07:50.248 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 61s ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:07:50.248 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 9m ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:07:50.248 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 9m ago 2h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:07:50.249 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 61s ago 2h 653M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:07:50.249 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 61s ago 2h 638M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:07:50.249 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 9m ago 2h 267M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:07:50.250 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 61s ago 2h 372M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:07:50.250 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 61s ago 2h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:07:50.250 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 9m ago 2h 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:07:50.250 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 61s ago 2h 462M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:07:50.251 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 61s ago 2h 453M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:07:50.251 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 61s ago 2h 358M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:07:50.251 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 61s ago 2h 440M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:07:50.251 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 9m ago 2h 370M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:07:50.252 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 9m ago 2h 455M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:07:50.252 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 9m ago 2h 405M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:07:50.252 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 9m ago 2h 473M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:07:50.252 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 9m ago 2h 127M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:07:50.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:07:49 smithi153 ceph-mon[70044]: pgmap v4788: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:50.605 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:07:50.605 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:07:50.605 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:07:50.606 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:07:50.606 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:07:50.606 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:07:50.606 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:07:50.607 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:07:50.607 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:07:50.607 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:07:50.607 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:07:50.608 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:07:50.608 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:07:50.608 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:07:50.608 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:07:50.608 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:07:50.609 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:07:50.609 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:07:51.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:07:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:07:50.727Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:07:51.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:07:50 smithi083 ceph-mon[77242]: from='client.39705 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:07:51.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:07:50 smithi083 ceph-mon[77242]: from='client.39711 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:07:51.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:07:50 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4049116896' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:07:51.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:07:50 smithi083 ceph-mon[79930]: from='client.39705 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:07:51.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:07:50 smithi083 ceph-mon[79930]: from='client.39711 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:07:51.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:07:50 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4049116896' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:07:51.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:07:50 smithi153 ceph-mon[70044]: from='client.39705 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:07:51.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:07:50 smithi153 ceph-mon[70044]: from='client.39711 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:07:51.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:07:50 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4049116896' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:07:51.829 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:07:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:07:51] "GET /metrics HTTP/1.1" 200 225210 "" "Prometheus/2.33.4" 2022-09-29T08:07:52.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:07:51 smithi083 ceph-mon[79930]: pgmap v4789: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:52.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:07:51 smithi083 ceph-mon[77242]: pgmap v4789: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:52.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:07:51 smithi153 ceph-mon[70044]: pgmap v4789: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:07:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:07:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:07:53.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:07:52 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:07:53.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:07:52 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:07:53.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:07:52 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:07:53.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:07:52 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:07:53.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:07:52 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:07:53.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:07:52 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:07:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:07:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:07:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:07:53.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:07:52 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:07:53.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:07:52 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:07:53.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:07:52 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:07:54.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:07:53 smithi153 ceph-mon[70044]: pgmap v4790: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:54.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:07:53 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:07:54.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:07:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:07:54.115Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:07:54.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:07:53 smithi083 ceph-mon[77242]: pgmap v4790: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:54.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:07:53 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:07:54.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:07:53 smithi083 ceph-mon[79930]: pgmap v4790: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:54.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:07:53 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:07:55.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:07:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:07:54.818Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:07:56.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:07:55 smithi153 ceph-mon[70044]: pgmap v4791: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:56.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:07:55 smithi083 ceph-mon[79930]: pgmap v4791: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:56.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:07:55 smithi083 ceph-mon[77242]: pgmap v4791: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:58.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:07:57 smithi153 ceph-mon[70044]: pgmap v4792: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:58.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:07:57 smithi083 ceph-mon[77242]: pgmap v4792: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:07:58.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:07:57 smithi083 ceph-mon[79930]: pgmap v4792: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:08:00 smithi153 ceph-mon[70044]: pgmap v4793: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:00.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:08:00 smithi083 ceph-mon[77242]: pgmap v4793: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:00.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:08:00 smithi083 ceph-mon[79930]: pgmap v4793: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:01.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:08:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:08:00.728Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:08:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:08:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:08:01] "GET /metrics HTTP/1.1" 200 225212 "" "Prometheus/2.33.4" 2022-09-29T08:08:02.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:08:02 smithi153 ceph-mon[70044]: pgmap v4794: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:02.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:08:02 smithi083 ceph-mon[79930]: pgmap v4794: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:02.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:08:02 smithi083 ceph-mon[77242]: pgmap v4794: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:08:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:08:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:08:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:08:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:08:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:08:04.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:08:04 smithi153 ceph-mon[70044]: pgmap v4795: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:04.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:08:04 smithi083 ceph-mon[77242]: pgmap v4795: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:04.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:08:04 smithi083 ceph-mon[79930]: pgmap v4795: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:04.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:08:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:08:04.115Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:08:05.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:08:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:08:04.818Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:08:06.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:08:06 smithi153 ceph-mon[70044]: pgmap v4796: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:06.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:08:06 smithi083 ceph-mon[79930]: pgmap v4796: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:06.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:08:06 smithi083 ceph-mon[77242]: pgmap v4796: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:08.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:08:08 smithi153 ceph-mon[70044]: pgmap v4797: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:08.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:08:08 smithi083 ceph-mon[79930]: pgmap v4797: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:08.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:08:08 smithi083 ceph-mon[77242]: pgmap v4797: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:10.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:08:10 smithi153 ceph-mon[70044]: pgmap v4798: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:10.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:08:10 smithi083 ceph-mon[79930]: pgmap v4798: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:10.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:08:10 smithi083 ceph-mon[77242]: pgmap v4798: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:11.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:08:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:08:10.729Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:08:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:08:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:08:11] "GET /metrics HTTP/1.1" 200 225211 "" "Prometheus/2.33.4" 2022-09-29T08:08:12.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:08:12 smithi153 ceph-mon[70044]: pgmap v4799: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:12.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:08:12 smithi083 ceph-mon[77242]: pgmap v4799: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:12.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:08:12 smithi083 ceph-mon[79930]: pgmap v4799: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:08:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:08:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:08:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:08:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:08:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:08:14.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:08:14 smithi153 ceph-mon[70044]: pgmap v4800: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:14.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:08:14 smithi083 ceph-mon[79930]: pgmap v4800: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:14.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:08:14 smithi083 ceph-mon[77242]: pgmap v4800: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:14.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:08:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:08:14.116Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:08:15.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:08:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:08:14.819Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:08:16.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:08:16 smithi153 ceph-mon[70044]: pgmap v4801: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:16.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:08:16 smithi083 ceph-mon[79930]: pgmap v4801: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:16.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:08:16 smithi083 ceph-mon[77242]: pgmap v4801: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:18.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:08:18 smithi153 ceph-mon[70044]: pgmap v4802: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:18.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:08:18 smithi083 ceph-mon[77242]: pgmap v4802: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:18.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:08:18 smithi083 ceph-mon[79930]: pgmap v4802: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:08:20 smithi153 ceph-mon[70044]: pgmap v4803: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:20.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:08:20 smithi083 ceph-mon[79930]: pgmap v4803: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:20.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:08:20 smithi083 ceph-mon[77242]: pgmap v4803: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:20.934 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:08:21.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:08:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:08:20.729Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:08:21.225 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:08:21.226 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 92s ago 2h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:08:21.226 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 9m ago 2h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:08:21.226 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 92s ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:08:21.226 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 9m ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:08:21.227 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 9m ago 2h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:08:21.227 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 92s ago 2h 653M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:08:21.227 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 92s ago 2h 638M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:08:21.227 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 9m ago 2h 267M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:08:21.228 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 92s ago 2h 372M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:08:21.228 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 92s ago 2h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:08:21.228 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 9m ago 2h 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:08:21.228 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 92s ago 2h 462M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:08:21.229 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 92s ago 2h 453M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:08:21.229 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 92s ago 2h 358M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:08:21.229 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 92s ago 2h 440M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:08:21.229 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 9m ago 2h 370M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:08:21.229 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 9m ago 2h 455M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:08:21.230 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 9m ago 2h 405M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:08:21.230 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 9m ago 2h 473M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:08:21.230 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 9m ago 2h 127M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:08:21.581 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:08:21.581 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:08:21.582 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:08:21.582 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:08:21.582 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:08:21.583 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:08:21.583 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:08:21.583 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:08:21.583 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:08:21.583 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:08:21.584 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:08:21.584 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:08:21.584 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:08:21.584 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:08:21.585 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:08:21.585 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:08:21.585 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:08:21.585 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:08:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:08:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:08:21] "GET /metrics HTTP/1.1" 200 225211 "" "Prometheus/2.33.4" 2022-09-29T08:08:22.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:08:22 smithi153 ceph-mon[70044]: pgmap v4804: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:22.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:08:22 smithi153 ceph-mon[70044]: from='client.39723 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:08:22.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:08:22 smithi153 ceph-mon[70044]: from='client.49663 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:08:22.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:08:22 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1883590283' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:08:22.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:08:22 smithi083 ceph-mon[77242]: pgmap v4804: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:22.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:08:22 smithi083 ceph-mon[77242]: from='client.39723 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:08:22.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:08:22 smithi083 ceph-mon[77242]: from='client.49663 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:08:22.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:08:22 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1883590283' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:08:22.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:08:22 smithi083 ceph-mon[79930]: pgmap v4804: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:22.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:08:22 smithi083 ceph-mon[79930]: from='client.39723 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:08:22.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:08:22 smithi083 ceph-mon[79930]: from='client.49663 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:08:22.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:08:22 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1883590283' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:08:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:08:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:08:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:08:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:08:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:08:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:08:24.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:08:24 smithi153 ceph-mon[70044]: pgmap v4805: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:24.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:08:24 smithi083 ceph-mon[77242]: pgmap v4805: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:24.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:08:24 smithi083 ceph-mon[79930]: pgmap v4805: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:24.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:08:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:08:24.117Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:08:25.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:08:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:08:24.820Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:08:26.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:08:26 smithi153 ceph-mon[70044]: pgmap v4806: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:26.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:08:26 smithi083 ceph-mon[77242]: pgmap v4806: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:26.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:08:26 smithi083 ceph-mon[79930]: pgmap v4806: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:28.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:08:28 smithi153 ceph-mon[70044]: pgmap v4807: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:28.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:08:28 smithi083 ceph-mon[77242]: pgmap v4807: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:28.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:08:28 smithi083 ceph-mon[79930]: pgmap v4807: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:30.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:08:30 smithi153 ceph-mon[70044]: pgmap v4808: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:30.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:08:30 smithi083 ceph-mon[77242]: pgmap v4808: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:30.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:08:30 smithi083 ceph-mon[79930]: pgmap v4808: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:31.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:08:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:08:30.731Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:08:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:08:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:08:31] "GET /metrics HTTP/1.1" 200 225213 "" "Prometheus/2.33.4" 2022-09-29T08:08:32.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:08:32 smithi153 ceph-mon[70044]: pgmap v4809: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:32.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:08:32 smithi083 ceph-mon[79930]: pgmap v4809: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:32.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:08:32 smithi083 ceph-mon[77242]: pgmap v4809: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:08:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:08:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:08:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:08:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:08:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:08:34.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:08:34 smithi153 ceph-mon[70044]: pgmap v4810: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:34.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:08:34 smithi083 ceph-mon[77242]: pgmap v4810: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:34.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:08:34 smithi083 ceph-mon[79930]: pgmap v4810: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:34.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:08:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:08:34.117Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:08:35.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:08:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:08:34.821Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:08:36.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:08:36 smithi153 ceph-mon[70044]: pgmap v4811: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:36.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:08:36 smithi083 ceph-mon[79930]: pgmap v4811: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:36.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:08:36 smithi083 ceph-mon[77242]: pgmap v4811: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:38.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:08:38 smithi083 ceph-mon[79930]: pgmap v4812: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:38.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:08:38 smithi083 ceph-mon[77242]: pgmap v4812: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:08:38 smithi153 ceph-mon[70044]: pgmap v4812: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:40.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:08:40 smithi083 ceph-mon[79930]: pgmap v4813: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:40.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:08:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:08:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:08:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:08:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:08:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:08:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:08:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:08:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:08:40 smithi083 ceph-mon[77242]: pgmap v4813: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:08:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:08:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:08:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:08:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:08:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:08:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:08:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:08:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:08:40 smithi153 ceph-mon[70044]: pgmap v4813: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:08:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:08:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:08:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:08:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:08:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:08:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:08:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:08:41.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:08:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:08:40.732Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:08:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:08:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:08:41] "GET /metrics HTTP/1.1" 200 225212 "" "Prometheus/2.33.4" 2022-09-29T08:08:42.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:08:42 smithi083 ceph-mon[79930]: pgmap v4814: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:42.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:08:42 smithi083 ceph-mon[77242]: pgmap v4814: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:08:42 smithi153 ceph-mon[70044]: pgmap v4814: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:42.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:08:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:08:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:08:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:08:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:08:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:08:44.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:08:44 smithi083 ceph-mon[79930]: pgmap v4815: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:44.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:08:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:08:44.119Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:08:44.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:08:44 smithi083 ceph-mon[77242]: pgmap v4815: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:44.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:08:44 smithi153 ceph-mon[70044]: pgmap v4815: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:45.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:08:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:08:44.821Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:08:46.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:08:46 smithi083 ceph-mon[79930]: pgmap v4816: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:46.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:08:46 smithi083 ceph-mon[77242]: pgmap v4816: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:46.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:08:46 smithi153 ceph-mon[70044]: pgmap v4816: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:48.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:08:48 smithi083 ceph-mon[79930]: pgmap v4817: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:48.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:08:48 smithi083 ceph-mon[77242]: pgmap v4817: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:48.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:08:48 smithi153 ceph-mon[70044]: pgmap v4817: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:50.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:08:50 smithi083 ceph-mon[79930]: pgmap v4818: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:50.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:08:50 smithi083 ceph-mon[77242]: pgmap v4818: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:08:50 smithi153 ceph-mon[70044]: pgmap v4818: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:51.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:08:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:08:50.732Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:08:51.909 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:08:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:08:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:08:51] "GET /metrics HTTP/1.1" 200 225212 "" "Prometheus/2.33.4" 2022-09-29T08:08:52.198 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:08:52.199 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 2m ago 2h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:08:52.199 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 10m ago 2h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:08:52.199 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 2m ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:08:52.200 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 10m ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:08:52.200 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 10m ago 2h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:08:52.200 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 2m ago 2h 653M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:08:52.201 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 2m ago 2h 638M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:08:52.201 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 10m ago 2h 267M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:08:52.201 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 2m ago 2h 372M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:08:52.201 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 2m ago 2h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:08:52.202 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 10m ago 2h 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:08:52.202 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 2m ago 2h 462M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:08:52.202 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 2m ago 2h 453M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:08:52.202 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 2m ago 2h 358M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:08:52.203 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 2m ago 2h 440M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:08:52.203 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 10m ago 2h 370M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:08:52.203 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 10m ago 2h 455M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:08:52.203 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 10m ago 2h 405M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:08:52.204 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 10m ago 2h 473M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:08:52.204 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 10m ago 2h 127M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:08:52.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:08:52 smithi083 ceph-mon[79930]: pgmap v4819: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:52.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:08:52 smithi083 ceph-mon[77242]: pgmap v4819: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:08:52 smithi153 ceph-mon[70044]: pgmap v4819: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:52.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:08:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:08:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:08:52.558 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:08:52.559 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:08:52.559 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:08:52.559 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:08:52.559 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:08:52.560 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:08:52.560 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:08:52.560 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:08:52.560 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:08:52.561 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:08:52.561 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:08:52.561 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:08:52.561 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:08:52.562 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:08:52.562 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:08:52.562 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:08:52.562 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:08:52.563 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:08:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:08:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:08:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:08:53.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:08:53 smithi153 ceph-mon[70044]: from='client.39741 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:08:53.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:08:53 smithi153 ceph-mon[70044]: from='client.39747 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:08:53.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:08:53 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1547513911' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:08:53.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:08:53 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:08:53.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:08:53 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:08:53.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:08:53 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:08:53.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:08:53 smithi083 ceph-mon[79930]: from='client.39741 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:08:53.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:08:53 smithi083 ceph-mon[79930]: from='client.39747 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:08:53.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:08:53 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1547513911' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:08:53.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:08:53 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:08:53.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:08:53 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:08:53.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:08:53 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:08:53.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:08:53 smithi083 ceph-mon[77242]: from='client.39741 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:08:53.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:08:53 smithi083 ceph-mon[77242]: from='client.39747 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:08:53.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:08:53 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1547513911' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:08:53.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:08:53 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:08:53.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:08:53 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:08:53.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:08:53 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:08:54.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:08:54 smithi083 ceph-mon[79930]: pgmap v4820: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:54.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:08:54 smithi083 ceph-mon[77242]: pgmap v4820: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:54.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:08:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:08:54.119Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:08:54.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:08:54 smithi153 ceph-mon[70044]: pgmap v4820: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:55.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:08:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:08:54.821Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:08:56.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:08:56 smithi083 ceph-mon[79930]: pgmap v4821: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:56.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:08:56 smithi083 ceph-mon[77242]: pgmap v4821: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:56.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:08:56 smithi153 ceph-mon[70044]: pgmap v4821: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:58.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:08:58 smithi083 ceph-mon[77242]: pgmap v4822: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:58.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:08:58 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:08:58.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:08:58 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:08:58.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:08:58 smithi083 ceph-mon[79930]: pgmap v4822: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:58.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:08:58 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:08:58.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:08:58 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:08:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:08:58 smithi153 ceph-mon[70044]: pgmap v4822: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:08:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:08:58 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:08:58.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:08:58 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:09:00.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:08:59 smithi153 ceph-mon[70044]: pgmap v4823: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:09:00.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:08:59 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:09:00.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:08:59 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:09:00.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:08:59 smithi083 ceph-mon[79930]: pgmap v4823: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:09:00.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:08:59 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:09:00.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:08:59 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:09:00.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:08:59 smithi083 ceph-mon[77242]: pgmap v4823: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:09:00.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:08:59 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:09:00.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:08:59 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:09:01.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:09:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:09:00.733Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:09:01.765 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:09:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:09:01] "GET /metrics HTTP/1.1" 200 225214 "" "Prometheus/2.33.4" 2022-09-29T08:09:01.766 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:09:01 smithi083 ceph-mon[77242]: pgmap v4824: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:09:01.766 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:09:01 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:09:01.767 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:09:01 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:09:01.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:09:01 smithi153 ceph-mon[70044]: pgmap v4824: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:09:01.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:09:01 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:09:01.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:09:01 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:09:01.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:09:01 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:09:02.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:09:01 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:09:02.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:09:01 smithi083 ceph-mon[79930]: pgmap v4824: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:09:02.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:09:01 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:09:02.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:09:01 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:09:02.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:09:01 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:09:02.771 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:09:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:09:02.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:09:03.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:09:02 smithi153 ceph-mon[70044]: Detected new or changed devices on smithi153 2022-09-29T08:09:03.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:09:02 smithi083 ceph-mon[79930]: Detected new or changed devices on smithi153 2022-09-29T08:09:03.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:09:02 smithi083 ceph-mon[77242]: Detected new or changed devices on smithi153 2022-09-29T08:09:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:09:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:09:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:09:04.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:09:03 smithi153 ceph-mon[70044]: pgmap v4825: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:09:04.121 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:09:03 smithi083 ceph-mon[79930]: pgmap v4825: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:09:04.121 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:09:03 smithi083 ceph-mon[77242]: pgmap v4825: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:09:04.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:09:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:09:04.120Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:09:05.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:09:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:09:04.821Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:09:06.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:09:05 smithi153 ceph-mon[70044]: pgmap v4826: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:09:06.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:09:05 smithi083 ceph-mon[79930]: pgmap v4826: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:09:06.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:09:05 smithi083 ceph-mon[77242]: pgmap v4826: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:09:08.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:09:07 smithi153 ceph-mon[70044]: pgmap v4827: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:09:08.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:09:07 smithi083 ceph-mon[79930]: pgmap v4827: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:09:08.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:09:07 smithi083 ceph-mon[77242]: pgmap v4827: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:09:10.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:09:09 smithi153 ceph-mon[70044]: pgmap v4828: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:09:10.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:09:09 smithi083 ceph-mon[79930]: pgmap v4828: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:09:10.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:09:09 smithi083 ceph-mon[77242]: pgmap v4828: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:09:11.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:09:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:09:10.733Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:09:11.782 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:09:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:09:11] "GET /metrics HTTP/1.1" 200 225214 "" "Prometheus/2.33.4" 2022-09-29T08:09:12.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:09:11 smithi153 ceph-mon[70044]: pgmap v4829: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:09:12.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:09:11 smithi083 ceph-mon[79930]: pgmap v4829: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:09:12.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:09:11 smithi083 ceph-mon[77242]: pgmap v4829: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:09:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:09:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:09:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:09:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:09:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:09:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:09:14.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:09:13 smithi153 ceph-mon[70044]: pgmap v4830: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:09:14.122 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:09:13 smithi083 ceph-mon[79930]: pgmap v4830: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:09:14.122 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:09:13 smithi083 ceph-mon[77242]: pgmap v4830: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:09:14.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:09:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:09:14.121Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:09:15.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:09:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:09:14.822Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:09:16.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:09:15 smithi153 ceph-mon[70044]: pgmap v4831: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:09:16.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:09:15 smithi083 ceph-mon[79930]: pgmap v4831: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:09:16.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:09:15 smithi083 ceph-mon[77242]: pgmap v4831: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:09:18.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:09:17 smithi153 ceph-mon[70044]: pgmap v4832: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:09:18.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:09:17 smithi083 ceph-mon[79930]: pgmap v4832: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:09:18.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:09:17 smithi083 ceph-mon[77242]: pgmap v4832: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:09:20.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:09:19 smithi153 ceph-mon[70044]: pgmap v4833: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:09:20.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:09:19 smithi083 ceph-mon[79930]: pgmap v4833: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:09:20.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:09:19 smithi083 ceph-mon[77242]: pgmap v4833: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:09:21.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:09:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:09:20.734Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:09:21.797 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:09:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:09:21] "GET /metrics HTTP/1.1" 200 225214 "" "Prometheus/2.33.4" 2022-09-29T08:09:22.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:09:21 smithi153 ceph-mon[70044]: pgmap v4834: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:09:22.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:09:21 smithi083 ceph-mon[77242]: pgmap v4834: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:09:22.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:09:21 smithi083 ceph-mon[79930]: pgmap v4834: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:09:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:09:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:09:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:09:22.886 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:09:23.173 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:09:23.174 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 2m ago 2h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:09:23.174 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 25s ago 2h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:09:23.174 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 2m ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:09:23.174 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 25s ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:09:23.175 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 25s ago 2h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:09:23.175 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 2m ago 2h 653M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:09:23.175 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 2m ago 2h 638M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:09:23.175 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 25s ago 2h 284M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:09:23.176 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 2m ago 2h 372M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:09:23.176 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 2m ago 2h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:09:23.176 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 25s ago 2h 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:09:23.176 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 2m ago 2h 462M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:09:23.177 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 2m ago 2h 453M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:09:23.177 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 2m ago 2h 358M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:09:23.177 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 2m ago 2h 440M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:09:23.177 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 25s ago 2h 392M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:09:23.177 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 25s ago 2h 482M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:09:23.178 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 25s ago 2h 429M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:09:23.178 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 25s ago 2h 503M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:09:23.178 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 25s ago 2h 137M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:09:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:09:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:09:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:09:23.542 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:09:23.542 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:09:23.542 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:09:23.543 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:09:23.543 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:09:23.543 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:09:23.543 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:09:23.544 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:09:23.544 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:09:23.544 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:09:23.544 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:09:23.544 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:09:23.545 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:09:23.545 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:09:23.545 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:09:23.545 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:09:23.546 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:09:23.546 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:09:24.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:09:23 smithi153 ceph-mon[70044]: pgmap v4835: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:09:24.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:09:23 smithi153 ceph-mon[70044]: from='client.49693 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:09:24.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:09:23 smithi153 ceph-mon[70044]: from='client.49699 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:09:24.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:09:23 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3056603778' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:09:24.122 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:09:23 smithi083 ceph-mon[79930]: pgmap v4835: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:09:24.122 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:09:23 smithi083 ceph-mon[79930]: from='client.49693 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:09:24.123 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:09:23 smithi083 ceph-mon[79930]: from='client.49699 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:09:24.123 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:09:23 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3056603778' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:09:24.123 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:09:23 smithi083 ceph-mon[77242]: pgmap v4835: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:09:24.123 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:09:23 smithi083 ceph-mon[77242]: from='client.49693 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:09:24.124 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:09:23 smithi083 ceph-mon[77242]: from='client.49699 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:09:24.124 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:09:23 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3056603778' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:09:24.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:09:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:09:24.121Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:09:25.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:09:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:09:24.822Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:09:26.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:09:25 smithi153 ceph-mon[70044]: pgmap v4836: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:09:26.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:09:25 smithi083 ceph-mon[77242]: pgmap v4836: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:09:26.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:09:25 smithi083 ceph-mon[79930]: pgmap v4836: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:09:28.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:09:27 smithi083 ceph-mon[77242]: pgmap v4837: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:09:28.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:09:27 smithi083 ceph-mon[79930]: pgmap v4837: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:09:28.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:09:27 smithi153 ceph-mon[70044]: pgmap v4837: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:09:30.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:09:29 smithi083 ceph-mon[79930]: pgmap v4838: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:09:30.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:09:29 smithi083 ceph-mon[77242]: pgmap v4838: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:09:30.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:09:29 smithi153 ceph-mon[70044]: pgmap v4838: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:09:31.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:09:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:09:30.735Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:09:31.813 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:09:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:09:31] "GET /metrics HTTP/1.1" 200 225209 "" "Prometheus/2.33.4" 2022-09-29T08:09:31.813 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:09:31 smithi083 ceph-mon[77242]: pgmap v4839: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:09:32.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:09:31 smithi083 ceph-mon[79930]: pgmap v4839: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:09:32.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:09:31 smithi153 ceph-mon[70044]: pgmap v4839: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:09:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:09:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:09:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:09:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:09:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:09:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:09:34.123 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:09:33 smithi083 ceph-mon[79930]: pgmap v4840: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:09:34.124 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:09:33 smithi083 ceph-mon[77242]: pgmap v4840: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:09:34.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:09:33 smithi153 ceph-mon[70044]: pgmap v4840: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:09:34.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:09:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:09:34.122Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:09:35.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:09:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:09:34.822Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:09:36.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:09:35 smithi083 ceph-mon[77242]: pgmap v4841: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:09:36.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:09:35 smithi083 ceph-mon[79930]: pgmap v4841: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:09:36.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:09:35 smithi153 ceph-mon[70044]: pgmap v4841: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:09:38.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:09:37 smithi083 ceph-mon[79930]: pgmap v4842: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:09:38.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:09:37 smithi083 ceph-mon[77242]: pgmap v4842: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:09:38.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:09:37 smithi153 ceph-mon[70044]: pgmap v4842: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:09:40.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:09:39 smithi083 ceph-mon[79930]: pgmap v4843: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:09:40.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:09:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:09:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:09:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:09:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:09:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:09:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:09:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:09:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:09:39 smithi083 ceph-mon[77242]: pgmap v4843: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:09:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:09:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:09:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:09:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:09:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:09:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:09:40.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:09:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:09:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:09:39 smithi153 ceph-mon[70044]: pgmap v4843: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:09:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:09:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:09:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:09:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:09:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:09:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:09:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:09:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:09:41.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:09:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:09:40.736Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:09:41.828 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:09:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:09:41] "GET /metrics HTTP/1.1" 200 225209 "" "Prometheus/2.33.4" 2022-09-29T08:09:42.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:09:41 smithi083 ceph-mon[79930]: pgmap v4844: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:09:42.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:09:41 smithi083 ceph-mon[77242]: pgmap v4844: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:09:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:09:41 smithi153 ceph-mon[70044]: pgmap v4844: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:09:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:09:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:09:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:09:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:09:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:09:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:09:44.124 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:09:43 smithi083 ceph-mon[79930]: pgmap v4845: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:09:44.125 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:09:43 smithi083 ceph-mon[77242]: pgmap v4845: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:09:44.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:09:43 smithi153 ceph-mon[70044]: pgmap v4845: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:09:44.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:09:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:09:44.123Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:09:45.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:09:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:09:44.823Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:09:46.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:09:45 smithi083 ceph-mon[79930]: pgmap v4846: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:09:46.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:09:45 smithi083 ceph-mon[77242]: pgmap v4846: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:09:46.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:09:45 smithi153 ceph-mon[70044]: pgmap v4846: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:09:48.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:09:47 smithi083 ceph-mon[77242]: pgmap v4847: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:09:48.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:09:47 smithi083 ceph-mon[79930]: pgmap v4847: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:09:48.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:09:47 smithi153 ceph-mon[70044]: pgmap v4847: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:09:50.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:09:49 smithi083 ceph-mon[79930]: pgmap v4848: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:09:50.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:09:49 smithi083 ceph-mon[77242]: pgmap v4848: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:09:50.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:09:49 smithi153 ceph-mon[70044]: pgmap v4848: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:09:51.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:09:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:09:50.737Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:09:51.844 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:09:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:09:51] "GET /metrics HTTP/1.1" 200 225209 "" "Prometheus/2.33.4" 2022-09-29T08:09:52.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:09:51 smithi083 ceph-mon[77242]: pgmap v4849: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:09:52.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:09:51 smithi083 ceph-mon[79930]: pgmap v4849: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:09:52.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:09:51 smithi153 ceph-mon[70044]: pgmap v4849: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:09:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:09:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:09:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:09:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:09:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:09:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:09:53.879 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:09:54.124 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:09:53 smithi083 ceph-mon[79930]: pgmap v4850: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:09:54.124 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:09:53 smithi083 ceph-mon[77242]: pgmap v4850: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:09:54.172 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:09:54.172 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 3m ago 2h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:09:54.172 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 56s ago 2h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:09:54.172 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 3m ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:09:54.173 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 56s ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:09:54.173 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 56s ago 2h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:09:54.173 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 3m ago 2h 653M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:09:54.173 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 3m ago 2h 638M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:09:54.174 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 56s ago 2h 284M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:09:54.174 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 3m ago 2h 372M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:09:54.174 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 3m ago 2h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:09:54.175 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 56s ago 2h 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:09:54.175 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 3m ago 2h 462M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:09:54.175 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 3m ago 2h 453M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:09:54.175 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 3m ago 2h 358M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:09:54.176 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 3m ago 2h 440M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:09:54.176 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 56s ago 2h 392M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:09:54.176 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 56s ago 2h 482M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:09:54.176 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 56s ago 2h 429M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:09:54.177 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 56s ago 2h 503M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:09:54.177 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 56s ago 2h 137M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:09:54.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:09:53 smithi153 ceph-mon[70044]: pgmap v4850: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:09:54.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:09:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:09:54.123Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:09:54.531 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:09:54.531 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:09:54.532 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:09:54.532 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:09:54.532 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:09:54.532 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:09:54.533 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:09:54.533 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:09:54.533 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:09:54.533 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:09:54.533 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:09:54.534 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:09:54.534 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:09:54.534 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:09:54.534 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:09:54.535 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:09:54.535 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:09:54.535 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:09:55.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:09:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:09:54.823Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:09:55.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:09:54 smithi083 ceph-mon[77242]: from='client.39777 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:09:55.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:09:54 smithi083 ceph-mon[77242]: from='client.49717 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:09:55.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:09:54 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/867682059' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:09:55.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:09:54 smithi083 ceph-mon[79930]: from='client.39777 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:09:55.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:09:54 smithi083 ceph-mon[79930]: from='client.49717 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:09:55.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:09:54 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/867682059' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:09:55.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:09:54 smithi153 ceph-mon[70044]: from='client.39777 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:09:55.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:09:54 smithi153 ceph-mon[70044]: from='client.49717 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:09:55.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:09:54 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/867682059' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:09:56.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:09:55 smithi083 ceph-mon[77242]: pgmap v4851: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:09:56.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:09:55 smithi083 ceph-mon[79930]: pgmap v4851: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:09:56.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:09:55 smithi153 ceph-mon[70044]: pgmap v4851: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:09:58.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:09:57 smithi083 ceph-mon[79930]: pgmap v4852: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:09:58.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:09:57 smithi083 ceph-mon[77242]: pgmap v4852: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:09:58.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:09:57 smithi153 ceph-mon[70044]: pgmap v4852: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:00.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:09:59 smithi083 ceph-mon[79930]: pgmap v4853: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:00.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:09:59 smithi083 ceph-mon[77242]: pgmap v4853: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:00.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:10:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[77219]: 2022-09-29T08:09:59.999+0000 7f160f665700 -1 log_channel(cluster) log [ERR] : overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T08:10:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:09:59 smithi153 ceph-mon[70044]: pgmap v4853: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:01.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:10:00 smithi083 ceph-mon[79930]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T08:10:01.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:10:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:10:00.738Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:10:01.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:10:00 smithi083 ceph-mon[77242]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T08:10:01.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:10:00 smithi153 ceph-mon[70044]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T08:10:01.863 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:10:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:10:01] "GET /metrics HTTP/1.1" 200 225212 "" "Prometheus/2.33.4" 2022-09-29T08:10:02.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:10:01 smithi083 ceph-mon[79930]: pgmap v4854: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:02.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:10:01 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:10:02.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:10:01 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:10:02.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:10:01 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:10:02.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:10:01 smithi083 ceph-mon[77242]: pgmap v4854: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:02.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:10:01 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:10:02.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:10:01 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:10:02.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:10:01 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:10:02.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:10:01 smithi153 ceph-mon[70044]: pgmap v4854: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:02.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:10:01 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:10:02.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:10:01 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:10:02.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:10:01 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:10:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:10:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:10:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:10:03.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:10:03 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:10:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:10:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:10:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:10:03.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:10:03 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:10:03.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:10:03 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:10:04.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:10:04 smithi083 ceph-mon[77242]: pgmap v4855: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:04.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:10:04 smithi083 ceph-mon[79930]: pgmap v4855: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:04.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:10:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:10:04.125Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:10:04.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:10:04 smithi153 ceph-mon[70044]: pgmap v4855: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:05.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:10:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:10:04.825Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:10:06.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:10:06 smithi153 ceph-mon[70044]: pgmap v4856: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:06.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:10:06 smithi083 ceph-mon[77242]: pgmap v4856: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:06.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:10:06 smithi083 ceph-mon[79930]: pgmap v4856: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:10:08 smithi153 ceph-mon[70044]: pgmap v4857: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:08.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:10:08 smithi083 ceph-mon[77242]: pgmap v4857: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:08.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:10:08 smithi083 ceph-mon[79930]: pgmap v4857: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:10:10 smithi153 ceph-mon[70044]: pgmap v4858: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:10.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:10:10 smithi083 ceph-mon[79930]: pgmap v4858: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:10.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:10:10 smithi083 ceph-mon[77242]: pgmap v4858: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:11.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:10:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:10:10.738Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:10:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:10:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:10:11] "GET /metrics HTTP/1.1" 200 225207 "" "Prometheus/2.33.4" 2022-09-29T08:10:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:10:12 smithi153 ceph-mon[70044]: pgmap v4859: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:12.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:10:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:10:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:10:12.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:10:12 smithi083 ceph-mon[79930]: pgmap v4859: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:12.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:10:12 smithi083 ceph-mon[77242]: pgmap v4859: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:10:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:10:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:10:14.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:10:14 smithi083 ceph-mon[77242]: pgmap v4860: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:14.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:10:14 smithi083 ceph-mon[79930]: pgmap v4860: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:14.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:10:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:10:14.125Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:10:14.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:10:14 smithi153 ceph-mon[70044]: pgmap v4860: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:15.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:10:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:10:14.825Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:10:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:10:16 smithi153 ceph-mon[70044]: pgmap v4861: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:16.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:10:16 smithi083 ceph-mon[79930]: pgmap v4861: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:16.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:10:16 smithi083 ceph-mon[77242]: pgmap v4861: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:10:18 smithi153 ceph-mon[70044]: pgmap v4862: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:18.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:10:18 smithi083 ceph-mon[79930]: pgmap v4862: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:18.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:10:18 smithi083 ceph-mon[77242]: pgmap v4862: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:10:20 smithi153 ceph-mon[70044]: pgmap v4863: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:20.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:10:20 smithi083 ceph-mon[79930]: pgmap v4863: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:20.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:10:20 smithi083 ceph-mon[77242]: pgmap v4863: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:21.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:10:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:10:20.739Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:10:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:10:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:10:21] "GET /metrics HTTP/1.1" 200 225207 "" "Prometheus/2.33.4" 2022-09-29T08:10:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:10:22 smithi153 ceph-mon[70044]: pgmap v4864: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:22.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:10:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:10:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:10:22.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:10:22 smithi083 ceph-mon[79930]: pgmap v4864: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:22.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:10:22 smithi083 ceph-mon[77242]: pgmap v4864: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:10:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:10:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:10:24.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:10:24 smithi083 ceph-mon[77242]: pgmap v4865: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:24.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:10:24 smithi083 ceph-mon[79930]: pgmap v4865: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:24.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:10:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:10:24.125Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:10:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:10:24 smithi153 ceph-mon[70044]: pgmap v4865: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:24.854 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:10:25.141 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:10:25.142 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 3m ago 2h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:10:25.142 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 87s ago 2h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:10:25.143 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 3m ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:10:25.143 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 87s ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:10:25.143 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 87s ago 2h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:10:25.143 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 3m ago 2h 653M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:10:25.144 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 3m ago 2h 638M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:10:25.144 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 87s ago 2h 284M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:10:25.144 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 3m ago 2h 372M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:10:25.144 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 3m ago 2h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:10:25.145 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 87s ago 2h 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:10:25.145 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 3m ago 2h 462M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:10:25.145 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 3m ago 2h 453M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:10:25.145 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 3m ago 2h 358M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:10:25.145 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 3m ago 2h 440M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:10:25.146 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 87s ago 2h 392M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:10:25.146 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 87s ago 2h 482M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:10:25.146 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 87s ago 2h 429M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:10:25.147 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 87s ago 2h 503M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:10:25.147 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 87s ago 2h 137M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:10:25.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:10:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:10:24.825Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:10:25.501 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:10:25.501 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:10:25.502 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:10:25.502 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:10:25.502 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:10:25.503 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:10:25.503 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:10:25.503 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:10:25.503 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:10:25.504 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:10:25.504 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:10:25.504 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:10:25.504 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:10:25.505 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:10:25.505 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:10:25.505 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:10:25.505 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:10:25.506 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:10:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:10:26 smithi153 ceph-mon[70044]: pgmap v4866: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:10:26 smithi153 ceph-mon[70044]: from='client.49729 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:10:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:10:26 smithi153 ceph-mon[70044]: from='client.49735 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:10:26.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:10:26 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2202647904' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:10:26.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:10:26 smithi083 ceph-mon[79930]: pgmap v4866: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:26.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:10:26 smithi083 ceph-mon[79930]: from='client.49729 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:10:26.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:10:26 smithi083 ceph-mon[79930]: from='client.49735 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:10:26.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:10:26 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2202647904' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:10:26.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:10:26 smithi083 ceph-mon[77242]: pgmap v4866: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:26.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:10:26 smithi083 ceph-mon[77242]: from='client.49729 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:10:26.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:10:26 smithi083 ceph-mon[77242]: from='client.49735 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:10:26.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:10:26 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2202647904' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:10:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:10:28 smithi153 ceph-mon[70044]: pgmap v4867: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:28.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:10:28 smithi083 ceph-mon[79930]: pgmap v4867: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:28.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:10:28 smithi083 ceph-mon[77242]: pgmap v4867: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:10:30 smithi153 ceph-mon[70044]: pgmap v4868: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:30.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:10:30 smithi083 ceph-mon[79930]: pgmap v4868: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:30.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:10:30 smithi083 ceph-mon[77242]: pgmap v4868: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:31.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:10:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:10:30.739Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:10:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:10:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:10:31] "GET /metrics HTTP/1.1" 200 225208 "" "Prometheus/2.33.4" 2022-09-29T08:10:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:10:32 smithi153 ceph-mon[70044]: pgmap v4869: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:32.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:10:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:10:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:10:32.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:10:32 smithi083 ceph-mon[79930]: pgmap v4869: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:32.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:10:32 smithi083 ceph-mon[77242]: pgmap v4869: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:10:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:10:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:10:34.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:10:34 smithi083 ceph-mon[79930]: pgmap v4870: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:34.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:10:34 smithi083 ceph-mon[77242]: pgmap v4870: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:34.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:10:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:10:34.126Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:10:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:10:34 smithi153 ceph-mon[70044]: pgmap v4870: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:35.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:10:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:10:34.827Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:10:35.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:10:35 smithi153 ceph-mon[70044]: pgmap v4871: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:35.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:10:35 smithi083 ceph-mon[79930]: pgmap v4871: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:35.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:10:35 smithi083 ceph-mon[77242]: pgmap v4871: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:38.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:10:37 smithi153 ceph-mon[70044]: pgmap v4872: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:38.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:10:37 smithi083 ceph-mon[79930]: pgmap v4872: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:38.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:10:37 smithi083 ceph-mon[77242]: pgmap v4872: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:10:39 smithi153 ceph-mon[70044]: pgmap v4873: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:10:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:10:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:10:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:10:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:10:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:10:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:10:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:10:40.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:10:39 smithi083 ceph-mon[79930]: pgmap v4873: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:40.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:10:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:10:40.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:10:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:10:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:10:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:10:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:10:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:10:40.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:10:39 smithi083 ceph-mon[77242]: pgmap v4873: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:10:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:10:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:10:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:10:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:10:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:10:40.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:10:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:10:41.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:10:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:10:40.740Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:10:41.789 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:10:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:10:41] "GET /metrics HTTP/1.1" 200 225209 "" "Prometheus/2.33.4" 2022-09-29T08:10:42.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:10:41 smithi153 ceph-mon[70044]: pgmap v4874: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:42.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:10:41 smithi083 ceph-mon[77242]: pgmap v4874: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:42.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:10:41 smithi083 ceph-mon[79930]: pgmap v4874: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:10:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:10:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:10:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:10:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:10:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:10:44.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:10:43 smithi153 ceph-mon[70044]: pgmap v4875: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:44.128 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:10:43 smithi083 ceph-mon[79930]: pgmap v4875: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:44.128 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:10:43 smithi083 ceph-mon[77242]: pgmap v4875: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:44.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:10:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:10:44.127Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:10:45.216 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:10:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:10:44.827Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:10:46.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:10:45 smithi153 ceph-mon[70044]: pgmap v4876: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:46.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:10:45 smithi083 ceph-mon[79930]: pgmap v4876: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:46.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:10:45 smithi083 ceph-mon[77242]: pgmap v4876: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:48.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:10:47 smithi153 ceph-mon[70044]: pgmap v4877: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:48.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:10:47 smithi083 ceph-mon[77242]: pgmap v4877: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:48.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:10:47 smithi083 ceph-mon[79930]: pgmap v4877: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:50.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:10:49 smithi083 ceph-mon[79930]: pgmap v4878: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:50.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:10:49 smithi083 ceph-mon[77242]: pgmap v4878: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:50.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:10:49 smithi153 ceph-mon[70044]: pgmap v4878: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:51.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:10:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:10:50.740Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:10:51.813 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:10:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:10:51] "GET /metrics HTTP/1.1" 200 225209 "" "Prometheus/2.33.4" 2022-09-29T08:10:52.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:10:51 smithi083 ceph-mon[79930]: pgmap v4879: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:52.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:10:51 smithi083 ceph-mon[77242]: pgmap v4879: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:52.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:10:51 smithi153 ceph-mon[70044]: pgmap v4879: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:10:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:10:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:10:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:10:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:10:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:10:54.128 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:10:53 smithi083 ceph-mon[79930]: pgmap v4880: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:54.129 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:10:53 smithi083 ceph-mon[77242]: pgmap v4880: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:54.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:10:53 smithi153 ceph-mon[70044]: pgmap v4880: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:54.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:10:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:10:54.128Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:10:55.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:10:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:10:54.827Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:10:55.828 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:10:56.116 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:10:56.117 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 4m ago 2h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:10:56.117 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 118s ago 2h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:10:56.117 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 4m ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:10:56.117 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 118s ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:10:56.118 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 118s ago 2h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:10:56.118 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 4m ago 2h 653M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:10:56.118 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 4m ago 2h 638M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:10:56.118 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 118s ago 2h 284M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:10:56.119 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 4m ago 2h 372M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:10:56.119 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 4m ago 2h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:10:56.119 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 118s ago 2h 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:10:56.119 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 4m ago 2h 462M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:10:56.120 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 4m ago 2h 453M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:10:56.120 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 4m ago 2h 358M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:10:56.120 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 4m ago 2h 440M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:10:56.120 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 118s ago 2h 392M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:10:56.121 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 118s ago 2h 482M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:10:56.121 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 118s ago 2h 429M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:10:56.121 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 118s ago 2h 503M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:10:56.121 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 118s ago 2h 137M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:10:56.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:10:55 smithi083 ceph-mon[79930]: pgmap v4881: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:56.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:10:55 smithi083 ceph-mon[77242]: pgmap v4881: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:56.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:10:55 smithi153 ceph-mon[70044]: pgmap v4881: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:56.471 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:10:56.472 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:10:56.472 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:10:56.472 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:10:56.472 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:10:56.473 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:10:56.473 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:10:56.473 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:10:56.473 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:10:56.474 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:10:56.474 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:10:56.474 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:10:56.474 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:10:56.475 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:10:56.475 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:10:56.475 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:10:56.475 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:10:56.475 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:10:57.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:10:56 smithi083 ceph-mon[79930]: from='client.39813 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:10:57.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:10:56 smithi083 ceph-mon[79930]: from='client.39819 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:10:57.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:10:56 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1134101388' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:10:57.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:10:56 smithi083 ceph-mon[77242]: from='client.39813 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:10:57.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:10:56 smithi083 ceph-mon[77242]: from='client.39819 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:10:57.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:10:56 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1134101388' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:10:57.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:10:56 smithi153 ceph-mon[70044]: from='client.39813 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:10:57.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:10:56 smithi153 ceph-mon[70044]: from='client.39819 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:10:57.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:10:56 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1134101388' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:10:58.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:10:57 smithi083 ceph-mon[79930]: pgmap v4882: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:58.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:10:57 smithi083 ceph-mon[77242]: pgmap v4882: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:10:58.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:10:57 smithi153 ceph-mon[70044]: pgmap v4882: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:00.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:10:59 smithi083 ceph-mon[79930]: pgmap v4883: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:00.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:10:59 smithi083 ceph-mon[77242]: pgmap v4883: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:10:59 smithi153 ceph-mon[70044]: pgmap v4883: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:01.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:11:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:11:00.741Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:11:01.829 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:11:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:11:01] "GET /metrics HTTP/1.1" 200 225211 "" "Prometheus/2.33.4" 2022-09-29T08:11:02.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:11:01 smithi083 ceph-mon[79930]: pgmap v4884: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:02.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:11:01 smithi083 ceph-mon[77242]: pgmap v4884: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:02.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:11:01 smithi153 ceph-mon[70044]: pgmap v4884: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:11:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:11:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:11:03.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:11:02 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:11:03.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:11:02 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:11:03.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:11:02 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:11:03.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:11:02 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:11:03.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:11:02 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:11:03.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:11:02 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:11:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:11:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:11:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:11:03.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:11:02 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:11:03.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:11:02 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:11:03.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:11:02 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:11:04.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:11:03 smithi083 ceph-mon[79930]: pgmap v4885: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:04.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:11:03 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:11:04.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:11:03 smithi083 ceph-mon[77242]: pgmap v4885: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:04.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:11:03 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:11:04.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:11:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:11:04.128Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:11:04.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:11:03 smithi153 ceph-mon[70044]: pgmap v4885: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:04.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:11:03 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:11:05.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:11:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:11:04.827Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:11:06.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:11:05 smithi083 ceph-mon[79930]: pgmap v4886: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:06.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:11:05 smithi083 ceph-mon[77242]: pgmap v4886: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:06.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:11:05 smithi153 ceph-mon[70044]: pgmap v4886: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:08.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:11:07 smithi083 ceph-mon[79930]: pgmap v4887: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:08.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:11:07 smithi083 ceph-mon[77242]: pgmap v4887: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:08.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:11:07 smithi153 ceph-mon[70044]: pgmap v4887: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:10.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:11:09 smithi083 ceph-mon[79930]: pgmap v4888: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:10.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:11:09 smithi083 ceph-mon[77242]: pgmap v4888: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:10.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:11:09 smithi153 ceph-mon[70044]: pgmap v4888: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:11.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:11:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:11:10.742Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:11:11.918 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:11:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:11:11] "GET /metrics HTTP/1.1" 200 225212 "" "Prometheus/2.33.4" 2022-09-29T08:11:12.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:11:11 smithi083 ceph-mon[79930]: pgmap v4889: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:12.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:11:11 smithi083 ceph-mon[77242]: pgmap v4889: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:12.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:11:11 smithi153 ceph-mon[70044]: pgmap v4889: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:11:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:11:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:11:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:11:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:11:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:11:14.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:11:13 smithi083 ceph-mon[79930]: pgmap v4890: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:14.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:11:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:11:14.129Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:11:14.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:11:13 smithi083 ceph-mon[77242]: pgmap v4890: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:14.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:11:13 smithi153 ceph-mon[70044]: pgmap v4890: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:15.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:11:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:11:14.828Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:11:16.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:11:15 smithi083 ceph-mon[79930]: pgmap v4891: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:16.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:11:15 smithi083 ceph-mon[77242]: pgmap v4891: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:16.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:11:15 smithi153 ceph-mon[70044]: pgmap v4891: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:18.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:11:17 smithi083 ceph-mon[79930]: pgmap v4892: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:18.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:11:17 smithi083 ceph-mon[77242]: pgmap v4892: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:18.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:11:17 smithi153 ceph-mon[70044]: pgmap v4892: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:20.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:11:19 smithi083 ceph-mon[77242]: pgmap v4893: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:20.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:11:19 smithi083 ceph-mon[79930]: pgmap v4893: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:11:19 smithi153 ceph-mon[70044]: pgmap v4893: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:21.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:11:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:11:20.742Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:11:21.939 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:11:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:11:21] "GET /metrics HTTP/1.1" 200 225212 "" "Prometheus/2.33.4" 2022-09-29T08:11:22.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:11:21 smithi083 ceph-mon[77242]: pgmap v4894: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:22.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:11:21 smithi083 ceph-mon[79930]: pgmap v4894: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:22.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:11:21 smithi153 ceph-mon[70044]: pgmap v4894: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:11:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:11:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:11:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:11:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:11:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:11:24.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:11:23 smithi083 ceph-mon[79930]: pgmap v4895: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:24.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:11:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:11:24.129Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:11:24.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:11:23 smithi083 ceph-mon[77242]: pgmap v4895: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:24.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:11:23 smithi153 ceph-mon[70044]: pgmap v4895: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:25.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:11:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:11:24.829Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:11:26.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:11:25 smithi083 ceph-mon[79930]: pgmap v4896: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:26.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:11:25 smithi083 ceph-mon[77242]: pgmap v4896: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:26.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:11:25 smithi153 ceph-mon[70044]: pgmap v4896: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:26.799 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:11:27.089 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:11:27.089 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 4m ago 2h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:11:27.089 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 2m ago 2h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:11:27.090 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 4m ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:11:27.090 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 2m ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:11:27.090 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 2m ago 2h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:11:27.090 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 4m ago 2h 653M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:11:27.091 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 4m ago 2h 638M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:11:27.091 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 2m ago 2h 284M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:11:27.091 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 4m ago 2h 372M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:11:27.092 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 4m ago 2h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:11:27.092 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 2m ago 2h 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:11:27.092 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 4m ago 2h 462M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:11:27.092 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 4m ago 2h 453M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:11:27.093 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 4m ago 2h 358M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:11:27.093 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 4m ago 2h 440M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:11:27.093 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 2m ago 2h 392M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:11:27.093 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 2m ago 2h 482M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:11:27.093 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 2m ago 2h 429M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:11:27.094 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 2m ago 2h 503M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:11:27.094 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 2m ago 2h 137M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:11:27.446 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:11:27.447 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:11:27.447 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:11:27.447 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:11:27.448 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:11:27.448 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:11:27.448 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:11:27.448 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:11:27.449 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:11:27.449 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:11:27.449 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:11:27.449 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:11:27.450 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:11:27.450 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:11:27.450 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:11:27.451 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:11:27.451 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:11:27.451 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:11:28.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:11:27 smithi083 ceph-mon[79930]: pgmap v4897: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:28.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:11:27 smithi083 ceph-mon[79930]: from='client.39831 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:11:28.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:11:27 smithi083 ceph-mon[79930]: from='client.39837 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:11:28.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:11:27 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1173712084' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:11:28.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:11:27 smithi083 ceph-mon[77242]: pgmap v4897: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:28.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:11:27 smithi083 ceph-mon[77242]: from='client.39831 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:11:28.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:11:27 smithi083 ceph-mon[77242]: from='client.39837 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:11:28.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:11:27 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1173712084' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:11:28.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:11:27 smithi153 ceph-mon[70044]: pgmap v4897: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:28.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:11:27 smithi153 ceph-mon[70044]: from='client.39831 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:11:28.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:11:27 smithi153 ceph-mon[70044]: from='client.39837 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:11:28.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:11:27 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1173712084' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:11:30.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:11:29 smithi083 ceph-mon[77242]: pgmap v4898: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:30.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:11:29 smithi083 ceph-mon[79930]: pgmap v4898: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:30.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:11:29 smithi153 ceph-mon[70044]: pgmap v4898: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:31.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:11:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:11:30.743Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:11:31.953 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:11:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:11:31] "GET /metrics HTTP/1.1" 200 225213 "" "Prometheus/2.33.4" 2022-09-29T08:11:32.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:11:31 smithi083 ceph-mon[79930]: pgmap v4899: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:32.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:11:31 smithi083 ceph-mon[77242]: pgmap v4899: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:32.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:11:31 smithi153 ceph-mon[70044]: pgmap v4899: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:11:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:11:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:11:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:11:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:11:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:11:34.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:11:33 smithi083 ceph-mon[79930]: pgmap v4900: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:34.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:11:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:11:34.130Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:11:34.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:11:33 smithi083 ceph-mon[77242]: pgmap v4900: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:34.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:11:33 smithi153 ceph-mon[70044]: pgmap v4900: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:35.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:11:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:11:34.830Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:11:36.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:11:35 smithi083 ceph-mon[79930]: pgmap v4901: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:36.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:11:35 smithi083 ceph-mon[77242]: pgmap v4901: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:36.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:11:35 smithi153 ceph-mon[70044]: pgmap v4901: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:38.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:11:37 smithi153 ceph-mon[70044]: pgmap v4902: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:38.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:11:37 smithi083 ceph-mon[79930]: pgmap v4902: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:38.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:11:37 smithi083 ceph-mon[77242]: pgmap v4902: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:11:39 smithi153 ceph-mon[70044]: pgmap v4903: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:11:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:11:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:11:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:11:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:11:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:11:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:11:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:11:40.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:11:39 smithi083 ceph-mon[79930]: pgmap v4903: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:40.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:11:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:11:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:11:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:11:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:11:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:11:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:11:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:11:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:11:39 smithi083 ceph-mon[77242]: pgmap v4903: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:11:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:11:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:11:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:11:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:11:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:11:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:11:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:11:41.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:11:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:11:40.743Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:11:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:11:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:11:41] "GET /metrics HTTP/1.1" 200 225211 "" "Prometheus/2.33.4" 2022-09-29T08:11:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:11:41 smithi153 ceph-mon[70044]: pgmap v4904: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:42.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:11:41 smithi083 ceph-mon[79930]: pgmap v4904: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:42.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:11:41 smithi083 ceph-mon[77242]: pgmap v4904: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:11:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:11:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:11:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:11:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:11:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:11:44.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:11:43 smithi153 ceph-mon[70044]: pgmap v4905: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:44.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:11:43 smithi083 ceph-mon[77242]: pgmap v4905: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:44.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:11:43 smithi083 ceph-mon[79930]: pgmap v4905: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:44.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:11:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:11:44.131Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:11:45.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:11:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:11:44.830Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:11:46.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:11:45 smithi153 ceph-mon[70044]: pgmap v4906: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:46.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:11:45 smithi083 ceph-mon[79930]: pgmap v4906: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:46.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:11:45 smithi083 ceph-mon[77242]: pgmap v4906: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:48.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:11:47 smithi153 ceph-mon[70044]: pgmap v4907: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:48.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:11:47 smithi083 ceph-mon[79930]: pgmap v4907: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:48.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:11:47 smithi083 ceph-mon[77242]: pgmap v4907: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:50.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:11:49 smithi153 ceph-mon[70044]: pgmap v4908: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:50.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:11:49 smithi083 ceph-mon[79930]: pgmap v4908: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:50.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:11:49 smithi083 ceph-mon[77242]: pgmap v4908: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:51.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:11:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:11:50.743Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:11:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:11:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:11:51] "GET /metrics HTTP/1.1" 200 225211 "" "Prometheus/2.33.4" 2022-09-29T08:11:52.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:11:51 smithi153 ceph-mon[70044]: pgmap v4909: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:52.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:11:51 smithi083 ceph-mon[79930]: pgmap v4909: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:52.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:11:51 smithi083 ceph-mon[77242]: pgmap v4909: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:11:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:11:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:11:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:11:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:11:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:11:54.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:11:53 smithi153 ceph-mon[70044]: pgmap v4910: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:54.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:11:53 smithi083 ceph-mon[79930]: pgmap v4910: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:54.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:11:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:11:54.130Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:11:54.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:11:53 smithi083 ceph-mon[77242]: pgmap v4910: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:55.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:11:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:11:54.831Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:11:56.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:11:55 smithi153 ceph-mon[70044]: pgmap v4911: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:56.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:11:55 smithi083 ceph-mon[77242]: pgmap v4911: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:56.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:11:55 smithi083 ceph-mon[79930]: pgmap v4911: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:57.770 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:11:58.059 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:11:58.059 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 5m ago 2h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:11:58.059 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 3m ago 2h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:11:58.060 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 5m ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:11:58.060 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 3m ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:11:58.060 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 3m ago 2h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:11:58.060 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 5m ago 2h 653M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:11:58.060 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 5m ago 2h 638M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:11:58.061 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 3m ago 2h 284M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:11:58.061 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 5m ago 2h 372M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:11:58.061 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 5m ago 2h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:11:58.061 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 3m ago 2h 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:11:58.062 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 5m ago 2h 462M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:11:58.062 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 5m ago 2h 453M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:11:58.062 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 5m ago 2h 358M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:11:58.062 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 5m ago 2h 440M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:11:58.063 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 3m ago 2h 392M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:11:58.063 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 3m ago 2h 482M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:11:58.063 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 3m ago 2h 429M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:11:58.063 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 3m ago 2h 503M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:11:58.063 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 3m ago 2h 137M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:11:58.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:11:57 smithi153 ceph-mon[70044]: pgmap v4912: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:58.417 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:11:58.418 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:11:58.418 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:11:58.418 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:11:58.419 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:11:58.419 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:11:58.419 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:11:58.419 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:11:58.420 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:11:58.420 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:11:58.420 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:11:58.420 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:11:58.421 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:11:58.421 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:11:58.421 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:11:58.421 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:11:58.422 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:11:58.422 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:11:58.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:11:57 smithi083 ceph-mon[79930]: pgmap v4912: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:58.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:11:57 smithi083 ceph-mon[77242]: pgmap v4912: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:11:59.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:11:58 smithi153 ceph-mon[70044]: from='client.39849 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:11:59.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:11:58 smithi153 ceph-mon[70044]: from='client.39855 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:11:59.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:11:58 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2777557122' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:11:59.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:11:58 smithi083 ceph-mon[79930]: from='client.39849 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:11:59.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:11:58 smithi083 ceph-mon[79930]: from='client.39855 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:11:59.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:11:58 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2777557122' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:11:59.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:11:58 smithi083 ceph-mon[77242]: from='client.39849 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:11:59.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:11:58 smithi083 ceph-mon[77242]: from='client.39855 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:11:59.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:11:58 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2777557122' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:12:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:11:59 smithi153 ceph-mon[70044]: pgmap v4913: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:00.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:11:59 smithi083 ceph-mon[79930]: pgmap v4913: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:00.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:11:59 smithi083 ceph-mon[77242]: pgmap v4913: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:01.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:12:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:12:00.744Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:12:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:12:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:12:01] "GET /metrics HTTP/1.1" 200 225210 "" "Prometheus/2.33.4" 2022-09-29T08:12:02.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:12:02 smithi153 ceph-mon[70044]: pgmap v4914: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:02.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:12:02 smithi083 ceph-mon[79930]: pgmap v4914: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:02.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:12:02 smithi083 ceph-mon[77242]: pgmap v4914: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:12:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:12:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:12:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:12:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:12:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:12:03.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:12:03 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:12:03.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:12:03 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:12:03.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:12:03 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:12:03.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:12:03 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:12:03.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:12:03 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:12:03.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:12:03 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:12:03.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:12:03 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:12:03.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:12:03 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:12:03.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:12:03 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:12:04.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:12:04 smithi153 ceph-mon[70044]: pgmap v4915: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:04.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:12:04 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:12:04.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:12:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:12:04.131Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:12:04.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:12:04 smithi083 ceph-mon[77242]: pgmap v4915: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:04.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:12:04 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:12:04.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:12:04 smithi083 ceph-mon[79930]: pgmap v4915: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:04.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:12:04 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:12:05.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:12:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:12:04.832Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:12:06.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:12:06 smithi153 ceph-mon[70044]: pgmap v4916: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:06.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:12:06 smithi083 ceph-mon[79930]: pgmap v4916: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:06.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:12:06 smithi083 ceph-mon[77242]: pgmap v4916: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:08.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:12:08 smithi153 ceph-mon[70044]: pgmap v4917: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:08.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:12:08 smithi083 ceph-mon[79930]: pgmap v4917: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:08.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:12:08 smithi083 ceph-mon[77242]: pgmap v4917: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:10.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:12:10 smithi153 ceph-mon[70044]: pgmap v4918: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:10.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:12:10 smithi083 ceph-mon[79930]: pgmap v4918: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:10.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:12:10 smithi083 ceph-mon[77242]: pgmap v4918: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:11.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:12:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:12:10.745Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:12:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:12:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:12:11] "GET /metrics HTTP/1.1" 200 225212 "" "Prometheus/2.33.4" 2022-09-29T08:12:12.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:12:12 smithi153 ceph-mon[70044]: pgmap v4919: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:12.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:12:12 smithi083 ceph-mon[77242]: pgmap v4919: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:12.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:12:12 smithi083 ceph-mon[79930]: pgmap v4919: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:12.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:12:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:12:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:12:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:12:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:12:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:12:14.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:12:14 smithi153 ceph-mon[70044]: pgmap v4920: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:14.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:12:14 smithi083 ceph-mon[79930]: pgmap v4920: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:14.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:12:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:12:14.132Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:12:14.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:12:14 smithi083 ceph-mon[77242]: pgmap v4920: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:15.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:12:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:12:14.832Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:12:16.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:12:16 smithi153 ceph-mon[70044]: pgmap v4921: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:16.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:12:16 smithi083 ceph-mon[79930]: pgmap v4921: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:16.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:12:16 smithi083 ceph-mon[77242]: pgmap v4921: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:18.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:12:18 smithi153 ceph-mon[70044]: pgmap v4922: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:18.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:12:18 smithi083 ceph-mon[79930]: pgmap v4922: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:18.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:12:18 smithi083 ceph-mon[77242]: pgmap v4922: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:12:20 smithi153 ceph-mon[70044]: pgmap v4923: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:20.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:12:20 smithi083 ceph-mon[77242]: pgmap v4923: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:20.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:12:20 smithi083 ceph-mon[79930]: pgmap v4923: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:21.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:12:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:12:20.746Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:12:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:12:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:12:21] "GET /metrics HTTP/1.1" 200 225212 "" "Prometheus/2.33.4" 2022-09-29T08:12:22.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:12:22 smithi153 ceph-mon[70044]: pgmap v4924: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:22.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:12:22 smithi083 ceph-mon[79930]: pgmap v4924: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:22.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:12:22 smithi083 ceph-mon[77242]: pgmap v4924: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:12:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:12:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:12:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:12:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:12:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:12:24.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:12:24 smithi153 ceph-mon[70044]: pgmap v4925: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:24.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:12:24 smithi083 ceph-mon[79930]: pgmap v4925: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:24.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:12:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:12:24.132Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:12:24.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:12:24 smithi083 ceph-mon[77242]: pgmap v4925: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:25.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:12:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:12:24.833Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:12:26.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:12:26 smithi153 ceph-mon[70044]: pgmap v4926: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:26.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:12:26 smithi083 ceph-mon[79930]: pgmap v4926: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:26.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:12:26 smithi083 ceph-mon[77242]: pgmap v4926: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:28.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:12:28 smithi153 ceph-mon[70044]: pgmap v4927: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:28.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:12:28 smithi083 ceph-mon[79930]: pgmap v4927: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:28.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:12:28 smithi083 ceph-mon[77242]: pgmap v4927: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:28.746 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:12:29.033 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:12:29.034 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 5m ago 2h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:12:29.034 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 3m ago 2h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:12:29.034 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 5m ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:12:29.034 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 3m ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:12:29.035 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 3m ago 2h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:12:29.035 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 5m ago 2h 653M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:12:29.035 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 5m ago 2h 638M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:12:29.035 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 3m ago 2h 284M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:12:29.036 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 5m ago 2h 372M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:12:29.036 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 5m ago 2h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:12:29.036 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 3m ago 2h 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:12:29.036 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 5m ago 2h 462M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:12:29.037 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 5m ago 2h 453M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:12:29.037 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 5m ago 2h 358M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:12:29.037 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 5m ago 2h 440M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:12:29.037 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 3m ago 2h 392M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:12:29.038 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 3m ago 2h 482M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:12:29.038 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 3m ago 2h 429M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:12:29.038 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 3m ago 2h 503M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:12:29.038 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 3m ago 2h 137M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:12:29.389 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:12:29.390 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:12:29.390 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:12:29.390 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:12:29.391 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:12:29.391 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:12:29.391 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:12:29.391 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:12:29.392 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:12:29.392 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:12:29.392 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:12:29.392 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:12:29.393 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:12:29.393 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:12:29.393 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:12:29.393 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:12:29.394 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:12:29.394 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:12:30.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:12:30 smithi153 ceph-mon[70044]: pgmap v4928: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:30.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:12:30 smithi153 ceph-mon[70044]: from='client.49801 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:12:30.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:12:30 smithi153 ceph-mon[70044]: from='client.49807 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:12:30.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:12:30 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/508031818' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:12:30.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:12:30 smithi083 ceph-mon[79930]: pgmap v4928: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:30.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:12:30 smithi083 ceph-mon[79930]: from='client.49801 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:12:30.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:12:30 smithi083 ceph-mon[79930]: from='client.49807 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:12:30.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:12:30 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/508031818' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:12:30.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:12:30 smithi083 ceph-mon[77242]: pgmap v4928: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:30.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:12:30 smithi083 ceph-mon[77242]: from='client.49801 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:12:30.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:12:30 smithi083 ceph-mon[77242]: from='client.49807 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:12:30.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:12:30 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/508031818' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:12:31.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:12:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:12:30.746Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:12:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:12:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:12:31] "GET /metrics HTTP/1.1" 200 225212 "" "Prometheus/2.33.4" 2022-09-29T08:12:32.314 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:12:32 smithi153 ceph-mon[70044]: pgmap v4929: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:32.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:12:32 smithi083 ceph-mon[79930]: pgmap v4929: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:32.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:12:32 smithi083 ceph-mon[77242]: pgmap v4929: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:12:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:12:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:12:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:12:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:12:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:12:34.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:12:34 smithi083 ceph-mon[79930]: pgmap v4930: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:34.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:12:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:12:34.134Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:12:34.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:12:34 smithi083 ceph-mon[77242]: pgmap v4930: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:12:34 smithi153 ceph-mon[70044]: pgmap v4930: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:35.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:12:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:12:34.834Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:12:36.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:12:36 smithi083 ceph-mon[79930]: pgmap v4931: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:36.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:12:36 smithi083 ceph-mon[77242]: pgmap v4931: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:12:36 smithi153 ceph-mon[70044]: pgmap v4931: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:38.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:12:38 smithi083 ceph-mon[79930]: pgmap v4932: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:38.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:12:38 smithi083 ceph-mon[77242]: pgmap v4932: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:12:38 smithi153 ceph-mon[70044]: pgmap v4932: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:40.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:12:40 smithi083 ceph-mon[79930]: pgmap v4933: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:40.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:12:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:12:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:12:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:12:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:12:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:12:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:12:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:12:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:12:40 smithi083 ceph-mon[77242]: pgmap v4933: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:12:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:12:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:12:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:12:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:12:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:12:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:12:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:12:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:12:40 smithi153 ceph-mon[70044]: pgmap v4933: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:12:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:12:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:12:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:12:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:12:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:12:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:12:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:12:41.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:12:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:12:40.747Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:12:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:12:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:12:41] "GET /metrics HTTP/1.1" 200 225204 "" "Prometheus/2.33.4" 2022-09-29T08:12:42.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:12:42 smithi083 ceph-mon[79930]: pgmap v4934: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:42.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:12:42 smithi083 ceph-mon[77242]: pgmap v4934: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:12:42 smithi153 ceph-mon[70044]: pgmap v4934: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:42.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:12:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:12:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:12:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:12:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:12:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:12:44.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:12:44 smithi083 ceph-mon[79930]: pgmap v4935: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:44.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:12:44 smithi083 ceph-mon[77242]: pgmap v4935: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:44.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:12:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:12:44.134Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:12:44.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:12:44 smithi153 ceph-mon[70044]: pgmap v4935: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:45.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:12:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:12:44.834Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:12:46.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:12:46 smithi083 ceph-mon[79930]: pgmap v4936: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:46.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:12:46 smithi083 ceph-mon[77242]: pgmap v4936: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:46.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:12:46 smithi153 ceph-mon[70044]: pgmap v4936: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:48.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:12:48 smithi083 ceph-mon[79930]: pgmap v4937: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:48.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:12:48 smithi083 ceph-mon[77242]: pgmap v4937: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:48.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:12:48 smithi153 ceph-mon[70044]: pgmap v4937: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:50.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:12:50 smithi083 ceph-mon[79930]: pgmap v4938: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:50.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:12:50 smithi083 ceph-mon[77242]: pgmap v4938: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:12:50 smithi153 ceph-mon[70044]: pgmap v4938: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:51.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:12:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:12:50.747Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:12:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:12:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:12:51] "GET /metrics HTTP/1.1" 200 225204 "" "Prometheus/2.33.4" 2022-09-29T08:12:52.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:12:52 smithi083 ceph-mon[79930]: pgmap v4939: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:52.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:12:52 smithi083 ceph-mon[77242]: pgmap v4939: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:12:52 smithi153 ceph-mon[70044]: pgmap v4939: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:52.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:12:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:12:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:12:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:12:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:12:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:12:54.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:12:54 smithi083 ceph-mon[77242]: pgmap v4940: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:54.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:12:54 smithi083 ceph-mon[79930]: pgmap v4940: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:54.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:12:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:12:54.134Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:12:54.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:12:54 smithi153 ceph-mon[70044]: pgmap v4940: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:55.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:12:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:12:54.835Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:12:56.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:12:56 smithi083 ceph-mon[79930]: pgmap v4941: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:56.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:12:56 smithi083 ceph-mon[77242]: pgmap v4941: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:56.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:12:56 smithi153 ceph-mon[70044]: pgmap v4941: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:58.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:12:58 smithi083 ceph-mon[79930]: pgmap v4942: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:58.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:12:58 smithi083 ceph-mon[77242]: pgmap v4942: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:12:58 smithi153 ceph-mon[70044]: pgmap v4942: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:12:59.716 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:13:00.003 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:13:00.004 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 6m ago 2h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:13:00.004 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 4m ago 2h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:13:00.004 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 6m ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:13:00.004 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 4m ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:13:00.005 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 4m ago 2h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:13:00.005 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 6m ago 2h 653M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:13:00.005 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 6m ago 2h 638M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:13:00.005 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 4m ago 2h 284M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:13:00.006 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 6m ago 2h 372M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:13:00.006 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 6m ago 2h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:13:00.006 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 4m ago 2h 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:13:00.006 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 6m ago 2h 462M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:13:00.007 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 6m ago 2h 453M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:13:00.007 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 6m ago 2h 358M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:13:00.007 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 6m ago 2h 440M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:13:00.007 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 4m ago 2h 392M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:13:00.008 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 4m ago 2h 482M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:13:00.008 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 4m ago 2h 429M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:13:00.008 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 4m ago 2h 503M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:13:00.008 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 4m ago 2h 137M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:13:00.361 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:13:00.362 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:13:00.362 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:13:00.362 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:13:00.362 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:13:00.363 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:13:00.363 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:13:00.363 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:13:00.363 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:13:00.364 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:13:00.364 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:13:00.364 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:13:00.364 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:13:00.365 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:13:00.365 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:13:00.365 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:13:00.365 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:13:00.365 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:13:00.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:13:00 smithi083 ceph-mon[79930]: pgmap v4943: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:13:00.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:13:00 smithi083 ceph-mon[77242]: pgmap v4943: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:13:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:13:00 smithi153 ceph-mon[70044]: pgmap v4943: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:13:01.105 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:13:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:13:00.747Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:13:01.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:13:01 smithi083 ceph-mon[79930]: from='client.39885 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:13:01.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:13:01 smithi083 ceph-mon[79930]: from='client.49825 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:13:01.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:13:01 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1059821545' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:13:01.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:13:01 smithi083 ceph-mon[77242]: from='client.39885 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:13:01.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:13:01 smithi083 ceph-mon[77242]: from='client.49825 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:13:01.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:13:01 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1059821545' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:13:01.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:13:01 smithi153 ceph-mon[70044]: from='client.39885 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:13:01.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:13:01 smithi153 ceph-mon[70044]: from='client.49825 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:13:01.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:13:01 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1059821545' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:13:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:13:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:13:01] "GET /metrics HTTP/1.1" 200 225214 "" "Prometheus/2.33.4" 2022-09-29T08:13:02.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:13:02 smithi083 ceph-mon[79930]: pgmap v4944: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:13:02.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:13:02 smithi083 ceph-mon[77242]: pgmap v4944: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:13:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:13:02 smithi153 ceph-mon[70044]: pgmap v4944: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:13:02.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:13:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:13:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:13:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:13:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:13:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:13:04.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:13:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:13:04.142Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:13:04.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:13:04 smithi083 ceph-mon[79930]: pgmap v4945: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:13:04.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:13:04 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:13:04.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:13:04 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:13:04.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:13:04 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:13:04.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:13:04 smithi083 ceph-mon[77242]: pgmap v4945: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:13:04.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:13:04 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:13:04.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:13:04 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:13:04.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:13:04 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:13:04.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:13:04 smithi153 ceph-mon[70044]: pgmap v4945: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:13:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:13:04 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:13:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:13:04 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:13:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:13:04 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:13:05.210 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:13:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:13:04.834Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:13:05.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:13:05 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:13:05.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:13:05 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:13:05.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:13:05 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:13:06.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:13:06 smithi153 ceph-mon[70044]: pgmap v4946: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:13:06.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:13:06 smithi083 ceph-mon[79930]: pgmap v4946: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:13:06.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:13:06 smithi083 ceph-mon[77242]: pgmap v4946: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:13:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:13:08 smithi153 ceph-mon[70044]: pgmap v4947: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:13:08.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:13:08 smithi083 ceph-mon[79930]: pgmap v4947: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:13:08.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:13:08 smithi083 ceph-mon[77242]: pgmap v4947: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:13:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:13:10 smithi153 ceph-mon[70044]: pgmap v4948: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:13:10.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:13:10 smithi083 ceph-mon[79930]: pgmap v4948: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:13:10.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:13:10 smithi083 ceph-mon[77242]: pgmap v4948: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:13:11.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:13:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:13:10.748Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:13:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:13:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:13:11] "GET /metrics HTTP/1.1" 200 225209 "" "Prometheus/2.33.4" 2022-09-29T08:13:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:13:12 smithi153 ceph-mon[70044]: pgmap v4949: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:13:12.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:13:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:13:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:13:12.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:13:12 smithi083 ceph-mon[79930]: pgmap v4949: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:13:12.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:13:12 smithi083 ceph-mon[77242]: pgmap v4949: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:13:13.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:13:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:13:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:13:14.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:13:14 smithi083 ceph-mon[79930]: pgmap v4950: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:13:14.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:13:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:13:14.137Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:13:14.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:13:14 smithi083 ceph-mon[77242]: pgmap v4950: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:13:14.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:13:14 smithi153 ceph-mon[70044]: pgmap v4950: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:13:15.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:13:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:13:14.836Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:13:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:13:16 smithi153 ceph-mon[70044]: pgmap v4951: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:13:16.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:13:16 smithi083 ceph-mon[79930]: pgmap v4951: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:13:16.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:13:16 smithi083 ceph-mon[77242]: pgmap v4951: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:13:18.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:13:18 smithi153 ceph-mon[70044]: pgmap v4952: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:13:18.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:13:18 smithi083 ceph-mon[79930]: pgmap v4952: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:13:18.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:13:18 smithi083 ceph-mon[77242]: pgmap v4952: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:13:20.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:13:20 smithi153 ceph-mon[70044]: pgmap v4953: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:13:20.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:13:20 smithi083 ceph-mon[79930]: pgmap v4953: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:13:20.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:13:20 smithi083 ceph-mon[77242]: pgmap v4953: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:13:21.210 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:13:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:13:20.749Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:13:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:13:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:13:21] "GET /metrics HTTP/1.1" 200 225209 "" "Prometheus/2.33.4" 2022-09-29T08:13:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:13:22 smithi153 ceph-mon[70044]: pgmap v4954: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:13:22.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:13:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:13:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:13:22.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:13:22 smithi083 ceph-mon[79930]: pgmap v4954: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:13:22.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:13:22 smithi083 ceph-mon[77242]: pgmap v4954: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:13:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:13:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:13:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:13:24.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:13:24 smithi083 ceph-mon[79930]: pgmap v4955: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:13:24.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:13:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:13:24.137Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:13:24.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:13:24 smithi083 ceph-mon[77242]: pgmap v4955: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:13:24.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:13:24 smithi153 ceph-mon[70044]: pgmap v4955: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:13:25.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:13:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:13:24.836Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:13:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:13:26 smithi153 ceph-mon[70044]: pgmap v4956: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:13:26.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:13:26 smithi083 ceph-mon[77242]: pgmap v4956: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:13:26.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:13:26 smithi083 ceph-mon[79930]: pgmap v4956: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:13:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:13:28 smithi153 ceph-mon[70044]: pgmap v4957: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:13:28.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:13:28 smithi083 ceph-mon[79930]: pgmap v4957: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:13:28.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:13:28 smithi083 ceph-mon[77242]: pgmap v4957: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:13:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:13:30 smithi153 ceph-mon[70044]: pgmap v4958: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:13:30.688 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:13:30.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:13:30 smithi083 ceph-mon[79930]: pgmap v4958: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:13:30.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:13:30 smithi083 ceph-mon[77242]: pgmap v4958: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:13:30.980 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:13:30.980 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 6m ago 2h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:13:30.980 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 4m ago 2h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:13:30.981 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 6m ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:13:30.981 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 4m ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:13:30.981 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 4m ago 2h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:13:30.981 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 6m ago 2h 653M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:13:30.981 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 6m ago 2h 638M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:13:30.982 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 4m ago 2h 284M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:13:30.982 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 6m ago 2h 372M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:13:30.982 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 6m ago 2h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:13:30.982 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 4m ago 2h 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:13:30.983 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 6m ago 2h 462M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:13:30.983 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 6m ago 2h 453M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:13:30.983 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 6m ago 2h 358M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:13:30.983 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 6m ago 2h 440M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:13:30.984 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 4m ago 2h 392M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:13:30.984 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 4m ago 2h 482M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:13:30.984 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 4m ago 2h 429M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:13:30.984 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 4m ago 2h 503M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:13:30.984 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 4m ago 2h 137M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:13:31.210 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:13:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:13:30.751Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:13:31.339 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:13:31.340 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:13:31.340 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:13:31.340 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:13:31.340 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:13:31.341 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:13:31.341 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:13:31.341 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:13:31.341 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:13:31.342 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:13:31.342 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:13:31.342 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:13:31.342 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:13:31.343 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:13:31.343 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:13:31.343 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:13:31.343 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:13:31.344 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:13:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:13:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:13:31] "GET /metrics HTTP/1.1" 200 225208 "" "Prometheus/2.33.4" 2022-09-29T08:13:32.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:13:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:13:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:13:32.591 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:13:32 smithi153 ceph-mon[70044]: pgmap v4959: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:13:32.592 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:13:32 smithi153 ceph-mon[70044]: from='client.39903 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:13:32.592 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:13:32 smithi153 ceph-mon[70044]: from='client.49843 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:13:32.592 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:13:32 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3114984143' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:13:32.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:13:32 smithi083 ceph-mon[77242]: pgmap v4959: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:13:32.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:13:32 smithi083 ceph-mon[77242]: from='client.39903 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:13:32.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:13:32 smithi083 ceph-mon[77242]: from='client.49843 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:13:32.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:13:32 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3114984143' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:13:32.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:13:32 smithi083 ceph-mon[79930]: pgmap v4959: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:13:32.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:13:32 smithi083 ceph-mon[79930]: from='client.39903 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:13:32.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:13:32 smithi083 ceph-mon[79930]: from='client.49843 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:13:32.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:13:32 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3114984143' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:13:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:13:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:13:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:13:34.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:13:34 smithi083 ceph-mon[79930]: pgmap v4960: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:13:34.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:13:34 smithi083 ceph-mon[77242]: pgmap v4960: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:13:34.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:13:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:13:34.139Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:13:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:13:34 smithi153 ceph-mon[70044]: pgmap v4960: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:13:35.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:13:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:13:34.837Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:13:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:13:36 smithi153 ceph-mon[70044]: pgmap v4961: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:13:36.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:13:36 smithi083 ceph-mon[79930]: pgmap v4961: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:13:36.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:13:36 smithi083 ceph-mon[77242]: pgmap v4961: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:13:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:13:38 smithi153 ceph-mon[70044]: pgmap v4962: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:13:38.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:13:38 smithi083 ceph-mon[79930]: pgmap v4962: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:13:38.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:13:38 smithi083 ceph-mon[77242]: pgmap v4962: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:13:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:13:40 smithi153 ceph-mon[70044]: pgmap v4963: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:13:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:13:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:13:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:13:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:13:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:13:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:13:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:13:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:13:40.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:13:40 smithi083 ceph-mon[77242]: pgmap v4963: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:13:40.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:13:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:13:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:13:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:13:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:13:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:13:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:13:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:13:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:13:40 smithi083 ceph-mon[79930]: pgmap v4963: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:13:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:13:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:13:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:13:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:13:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:13:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:13:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:13:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:13:41.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:13:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:13:40.750Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:13:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:13:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:13:41] "GET /metrics HTTP/1.1" 200 225211 "" "Prometheus/2.33.4" 2022-09-29T08:13:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:13:42 smithi153 ceph-mon[70044]: pgmap v4964: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:13:42.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:13:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:13:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:13:42.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:13:42 smithi083 ceph-mon[79930]: pgmap v4964: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:13:42.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:13:42 smithi083 ceph-mon[77242]: pgmap v4964: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:13:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:13:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:13:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:13:44.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:13:44 smithi083 ceph-mon[79930]: pgmap v4965: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:13:44.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:13:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:13:44.139Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:13:44.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:13:44 smithi083 ceph-mon[77242]: pgmap v4965: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:13:44.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:13:44 smithi153 ceph-mon[70044]: pgmap v4965: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:13:45.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:13:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:13:44.838Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:13:46.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:13:46 smithi153 ceph-mon[70044]: pgmap v4966: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:13:46.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:13:46 smithi083 ceph-mon[79930]: pgmap v4966: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:13:46.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:13:46 smithi083 ceph-mon[77242]: pgmap v4966: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:13:48.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:13:48 smithi153 ceph-mon[70044]: pgmap v4967: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:13:48.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:13:48 smithi083 ceph-mon[79930]: pgmap v4967: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:13:48.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:13:48 smithi083 ceph-mon[77242]: pgmap v4967: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:13:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:13:50 smithi153 ceph-mon[70044]: pgmap v4968: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:13:50.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:13:50 smithi083 ceph-mon[79930]: pgmap v4968: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:13:50.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:13:50 smithi083 ceph-mon[77242]: pgmap v4968: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:13:51.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:13:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:13:50.751Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:13:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:13:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:13:51] "GET /metrics HTTP/1.1" 200 225211 "" "Prometheus/2.33.4" 2022-09-29T08:13:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:13:52 smithi153 ceph-mon[70044]: pgmap v4969: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:13:52.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:13:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:13:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:13:52.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:13:52 smithi083 ceph-mon[79930]: pgmap v4969: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:13:52.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:13:52 smithi083 ceph-mon[77242]: pgmap v4969: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:13:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:13:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:13:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:13:54.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:13:54 smithi083 ceph-mon[79930]: pgmap v4970: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:13:54.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:13:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:13:54.140Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:13:54.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:13:54 smithi083 ceph-mon[77242]: pgmap v4970: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:13:54.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:13:54 smithi153 ceph-mon[70044]: pgmap v4970: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:13:55.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:13:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:13:54.839Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:13:56.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:13:56 smithi153 ceph-mon[70044]: pgmap v4971: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:13:56.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:13:56 smithi083 ceph-mon[79930]: pgmap v4971: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:13:56.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:13:56 smithi083 ceph-mon[77242]: pgmap v4971: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:13:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:13:58 smithi153 ceph-mon[70044]: pgmap v4972: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:13:58.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:13:58 smithi083 ceph-mon[79930]: pgmap v4972: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:13:58.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:13:58 smithi083 ceph-mon[77242]: pgmap v4972: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:14:00 smithi153 ceph-mon[70044]: pgmap v4973: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:00.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:14:00 smithi083 ceph-mon[79930]: pgmap v4973: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:00.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:14:00 smithi083 ceph-mon[77242]: pgmap v4973: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:01.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:14:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:14:00.752Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:14:01.668 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:14:01.958 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:14:01.959 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 7m ago 2h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:14:01.959 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 5m ago 2h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:14:01.959 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 7m ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:14:01.960 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 5m ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:14:01.960 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 5m ago 2h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:14:01.960 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 7m ago 2h 653M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:14:01.960 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 7m ago 2h 638M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:14:01.961 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 5m ago 2h 284M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:14:01.961 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 7m ago 2h 372M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:14:01.961 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 7m ago 2h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:14:01.961 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 5m ago 2h 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:14:01.962 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 7m ago 2h 462M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:14:01.962 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 7m ago 2h 453M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:14:01.962 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 7m ago 2h 358M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:14:01.962 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 7m ago 2h 440M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:14:01.962 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 5m ago 2h 392M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:14:01.963 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 5m ago 2h 482M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:14:01.963 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 5m ago 2h 429M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:14:01.963 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 5m ago 2h 503M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:14:01.964 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 5m ago 2h 137M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:14:01.964 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:14:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:14:01] "GET /metrics HTTP/1.1" 200 225211 "" "Prometheus/2.33.4" 2022-09-29T08:14:02.316 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:14:02.316 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:14:02.316 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:14:02.317 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:14:02.317 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:14:02.317 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:14:02.317 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:14:02.318 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:14:02.318 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:14:02.318 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:14:02.318 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:14:02.319 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:14:02.319 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:14:02.319 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:14:02.319 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:14:02.320 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:14:02.320 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:14:02.320 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:14:02.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:14:02 smithi083 ceph-mon[77242]: pgmap v4974: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:02.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:14:02 smithi083 ceph-mon[79930]: pgmap v4974: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:02.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:14:02 smithi153 ceph-mon[70044]: pgmap v4974: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:14:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:14:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:14:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:14:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:14:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:14:03.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:14:03 smithi083 ceph-mon[77242]: from='client.39921 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:14:03.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:14:03 smithi083 ceph-mon[77242]: from='client.49861 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:14:03.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:14:03 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/42274446' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:14:03.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:14:03 smithi083 ceph-mon[79930]: from='client.39921 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:14:03.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:14:03 smithi083 ceph-mon[79930]: from='client.49861 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:14:03.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:14:03 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/42274446' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:14:03.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:14:03 smithi153 ceph-mon[70044]: from='client.39921 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:14:03.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:14:03 smithi153 ceph-mon[70044]: from='client.49861 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:14:03.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:14:03 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/42274446' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:14:04.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:14:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:14:04.141Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:14:04.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:14:04 smithi083 ceph-mon[77242]: pgmap v4975: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:04.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:14:04 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:14:04.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:14:04 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:14:04.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:14:04 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:14:04.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:14:04 smithi083 ceph-mon[79930]: pgmap v4975: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:04.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:14:04 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:14:04.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:14:04 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:14:04.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:14:04 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:14:04.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:14:04 smithi153 ceph-mon[70044]: pgmap v4975: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:04.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:14:04 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:14:04.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:14:04 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:14:04.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:14:04 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:14:05.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:14:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:14:04.841Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:14:06.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:14:05 smithi083 ceph-mon[79930]: pgmap v4976: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:06.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:14:05 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:14:06.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:14:05 smithi083 ceph-mon[77242]: pgmap v4976: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:06.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:14:05 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:14:06.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:14:05 smithi153 ceph-mon[70044]: pgmap v4976: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:06.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:14:05 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:14:08.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:14:07 smithi083 ceph-mon[77242]: pgmap v4977: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:08.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:14:07 smithi083 ceph-mon[79930]: pgmap v4977: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:08.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:14:07 smithi153 ceph-mon[70044]: pgmap v4977: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:10.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:14:09 smithi083 ceph-mon[79930]: pgmap v4978: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:10.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:14:09 smithi083 ceph-mon[77242]: pgmap v4978: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:10.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:14:09 smithi153 ceph-mon[70044]: pgmap v4978: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:11.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:14:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:14:10.753Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:14:11.893 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:14:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:14:11] "GET /metrics HTTP/1.1" 200 225214 "" "Prometheus/2.33.4" 2022-09-29T08:14:12.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:14:11 smithi083 ceph-mon[79930]: pgmap v4979: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:12.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:14:11 smithi083 ceph-mon[77242]: pgmap v4979: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:12.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:14:11 smithi153 ceph-mon[70044]: pgmap v4979: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:14:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:14:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:14:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:14:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:14:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:14:14.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:14:13 smithi083 ceph-mon[79930]: pgmap v4980: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:14.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:14:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:14:14.142Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:14:14.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:14:13 smithi083 ceph-mon[77242]: pgmap v4980: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:14.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:14:13 smithi153 ceph-mon[70044]: pgmap v4980: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:15.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:14:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:14:14.841Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:14:16.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:14:15 smithi083 ceph-mon[77242]: pgmap v4981: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:16.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:14:15 smithi083 ceph-mon[79930]: pgmap v4981: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:16.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:14:15 smithi153 ceph-mon[70044]: pgmap v4981: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:18.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:14:17 smithi083 ceph-mon[79930]: pgmap v4982: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:18.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:14:17 smithi083 ceph-mon[77242]: pgmap v4982: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:18.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:14:17 smithi153 ceph-mon[70044]: pgmap v4982: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:20.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:14:19 smithi083 ceph-mon[79930]: pgmap v4983: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:20.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:14:19 smithi083 ceph-mon[77242]: pgmap v4983: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:14:19 smithi153 ceph-mon[70044]: pgmap v4983: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:21.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:14:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:14:20.754Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:14:21.910 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:14:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:14:21] "GET /metrics HTTP/1.1" 200 225214 "" "Prometheus/2.33.4" 2022-09-29T08:14:22.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:14:21 smithi083 ceph-mon[79930]: pgmap v4984: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:22.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:14:21 smithi083 ceph-mon[77242]: pgmap v4984: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:22.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:14:21 smithi153 ceph-mon[70044]: pgmap v4984: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:14:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:14:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:14:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:14:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:14:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:14:24.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:14:23 smithi083 ceph-mon[79930]: pgmap v4985: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:24.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:14:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:14:24.142Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:14:24.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:14:23 smithi083 ceph-mon[77242]: pgmap v4985: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:24.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:14:23 smithi153 ceph-mon[70044]: pgmap v4985: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:25.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:14:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:14:24.841Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:14:26.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:14:25 smithi083 ceph-mon[79930]: pgmap v4986: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:26.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:14:25 smithi083 ceph-mon[77242]: pgmap v4986: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:26.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:14:25 smithi153 ceph-mon[70044]: pgmap v4986: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:28.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:14:27 smithi083 ceph-mon[79930]: pgmap v4987: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:28.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:14:27 smithi083 ceph-mon[77242]: pgmap v4987: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:28.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:14:27 smithi153 ceph-mon[70044]: pgmap v4987: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:30.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:14:29 smithi083 ceph-mon[79930]: pgmap v4988: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:30.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:14:29 smithi083 ceph-mon[77242]: pgmap v4988: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:30.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:14:29 smithi153 ceph-mon[70044]: pgmap v4988: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:31.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:14:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:14:30.754Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:14:31.925 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:14:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:14:31] "GET /metrics HTTP/1.1" 200 225213 "" "Prometheus/2.33.4" 2022-09-29T08:14:32.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:14:31 smithi083 ceph-mon[79930]: pgmap v4989: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:32.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:14:31 smithi083 ceph-mon[77242]: pgmap v4989: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:32.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:14:31 smithi153 ceph-mon[70044]: pgmap v4989: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:32.641 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:14:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:14:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:14:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:14:32.926 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:14:32.926 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 7m ago 2h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:14:32.927 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 5m ago 2h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:14:32.927 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 7m ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:14:32.927 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 5m ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:14:32.927 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 5m ago 2h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:14:32.927 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 7m ago 2h 653M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:14:32.928 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 7m ago 2h 638M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:14:32.928 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 5m ago 2h 284M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:14:32.928 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 7m ago 2h 372M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:14:32.928 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 7m ago 2h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:14:32.929 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 5m ago 2h 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:14:32.929 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 7m ago 2h 462M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:14:32.929 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 7m ago 2h 453M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:14:32.929 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 7m ago 2h 358M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:14:32.929 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 7m ago 2h 440M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:14:32.930 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 5m ago 2h 392M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:14:32.930 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 5m ago 2h 482M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:14:32.930 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 5m ago 2h 429M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:14:32.930 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 5m ago 2h 503M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:14:32.931 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 5m ago 2h 137M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:14:33.289 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:14:33.290 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:14:33.290 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:14:33.290 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:14:33.290 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:14:33.291 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:14:33.291 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:14:33.291 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:14:33.291 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:14:33.292 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:14:33.292 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:14:33.292 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:14:33.292 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:14:33.292 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:14:33.293 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:14:33.293 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:14:33.293 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:14:33.293 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:14:33.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:14:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:14:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:14:34.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:14:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:14:34.143Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:14:34.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:14:33 smithi083 ceph-mon[77242]: from='client.49873 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:14:34.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:14:33 smithi083 ceph-mon[77242]: pgmap v4990: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:34.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:14:33 smithi083 ceph-mon[77242]: from='client.49879 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:14:34.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:14:33 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2888641408' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:14:34.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:14:33 smithi083 ceph-mon[79930]: from='client.49873 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:14:34.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:14:33 smithi083 ceph-mon[79930]: pgmap v4990: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:34.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:14:33 smithi083 ceph-mon[79930]: from='client.49879 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:14:34.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:14:33 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2888641408' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:14:34.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:14:33 smithi153 ceph-mon[70044]: from='client.49873 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:14:34.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:14:33 smithi153 ceph-mon[70044]: pgmap v4990: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:34.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:14:33 smithi153 ceph-mon[70044]: from='client.49879 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:14:34.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:14:33 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2888641408' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:14:35.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:14:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:14:34.843Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:14:36.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:14:35 smithi083 ceph-mon[79930]: pgmap v4991: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:36.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:14:35 smithi083 ceph-mon[77242]: pgmap v4991: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:36.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:14:35 smithi153 ceph-mon[70044]: pgmap v4991: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:38.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:14:37 smithi083 ceph-mon[77242]: pgmap v4992: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:38.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:14:37 smithi083 ceph-mon[79930]: pgmap v4992: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:38.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:14:37 smithi153 ceph-mon[70044]: pgmap v4992: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:40.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:14:39 smithi083 ceph-mon[79930]: pgmap v4993: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:40.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:14:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:14:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:14:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:14:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:14:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:14:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:14:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:14:40.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:14:39 smithi083 ceph-mon[77242]: pgmap v4993: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:14:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:14:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:14:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:14:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:14:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:14:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:14:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:14:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:14:39 smithi153 ceph-mon[70044]: pgmap v4993: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:14:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:14:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:14:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:14:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:14:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:14:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:14:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:14:41.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:14:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:14:40.754Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:14:41.950 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:14:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:14:41] "GET /metrics HTTP/1.1" 200 225214 "" "Prometheus/2.33.4" 2022-09-29T08:14:42.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:14:41 smithi083 ceph-mon[79930]: pgmap v4994: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:42.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:14:41 smithi083 ceph-mon[77242]: pgmap v4994: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:14:41 smithi153 ceph-mon[70044]: pgmap v4994: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:14:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:14:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:14:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:14:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:14:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:14:44.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:14:43 smithi083 ceph-mon[79930]: pgmap v4995: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:44.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:14:43 smithi083 ceph-mon[77242]: pgmap v4995: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:44.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:14:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:14:44.144Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:14:44.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:14:43 smithi153 ceph-mon[70044]: pgmap v4995: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:45.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:14:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:14:44.843Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:14:46.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:14:45 smithi083 ceph-mon[79930]: pgmap v4996: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:46.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:14:45 smithi083 ceph-mon[77242]: pgmap v4996: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:46.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:14:45 smithi153 ceph-mon[70044]: pgmap v4996: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:48.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:14:47 smithi083 ceph-mon[79930]: pgmap v4997: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:48.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:14:47 smithi083 ceph-mon[77242]: pgmap v4997: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:48.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:14:47 smithi153 ceph-mon[70044]: pgmap v4997: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:50.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:14:49 smithi153 ceph-mon[70044]: pgmap v4998: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:50.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:14:49 smithi083 ceph-mon[79930]: pgmap v4998: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:50.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:14:49 smithi083 ceph-mon[77242]: pgmap v4998: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:51.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:14:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:14:50.755Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:14:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:14:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:14:51] "GET /metrics HTTP/1.1" 200 225214 "" "Prometheus/2.33.4" 2022-09-29T08:14:52.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:14:51 smithi153 ceph-mon[70044]: pgmap v4999: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:52.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:14:51 smithi083 ceph-mon[79930]: pgmap v4999: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:52.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:14:51 smithi083 ceph-mon[77242]: pgmap v4999: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:14:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:14:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:14:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:14:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:14:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:14:54.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:14:53 smithi153 ceph-mon[70044]: pgmap v5000: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:54.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:14:53 smithi083 ceph-mon[79930]: pgmap v5000: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:54.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:14:53 smithi083 ceph-mon[77242]: pgmap v5000: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:54.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:14:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:14:54.144Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:14:55.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:14:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:14:54.843Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:14:56.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:14:55 smithi153 ceph-mon[70044]: pgmap v5001: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:56.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:14:55 smithi083 ceph-mon[79930]: pgmap v5001: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:56.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:14:55 smithi083 ceph-mon[77242]: pgmap v5001: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:58.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:14:57 smithi153 ceph-mon[70044]: pgmap v5002: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:58.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:14:57 smithi083 ceph-mon[79930]: pgmap v5002: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:14:58.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:14:57 smithi083 ceph-mon[77242]: pgmap v5002: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:14:59 smithi153 ceph-mon[70044]: pgmap v5003: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:00.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:14:59 smithi083 ceph-mon[79930]: pgmap v5003: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:00.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:14:59 smithi083 ceph-mon[77242]: pgmap v5003: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:01.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:15:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:15:00.756Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:15:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:15:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:15:01] "GET /metrics HTTP/1.1" 200 225210 "" "Prometheus/2.33.4" 2022-09-29T08:15:02.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:15:01 smithi153 ceph-mon[70044]: pgmap v5004: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:02.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:15:01 smithi083 ceph-mon[77242]: pgmap v5004: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:02.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:15:01 smithi083 ceph-mon[79930]: pgmap v5004: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:15:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:15:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:15:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:15:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:15:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:15:03.632 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:15:03.920 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:15:03.921 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 8m ago 2h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:15:03.921 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 6m ago 2h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:15:03.921 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 8m ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:15:03.922 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 6m ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:15:03.922 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 6m ago 2h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:15:03.922 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 8m ago 2h 653M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:15:03.922 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 8m ago 2h 638M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:15:03.923 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 6m ago 2h 284M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:15:03.923 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 8m ago 2h 372M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:15:03.923 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 8m ago 2h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:15:03.923 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 6m ago 2h 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:15:03.924 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 8m ago 2h 462M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:15:03.924 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 8m ago 2h 453M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:15:03.924 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 8m ago 2h 358M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:15:03.924 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 8m ago 2h 440M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:15:03.925 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 6m ago 2h 392M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:15:03.925 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 6m ago 2h 482M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:15:03.925 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 6m ago 2h 429M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:15:03.925 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 6m ago 2h 503M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:15:03.925 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 6m ago 2h 137M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:15:04.277 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:15:04.277 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:15:04.278 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:15:04.278 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:15:04.278 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:15:04.278 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:15:04.279 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:15:04.279 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:15:04.279 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:15:04.279 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:15:04.279 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:15:04.280 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:15:04.280 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:15:04.280 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:15:04.280 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:15:04.280 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:15:04.281 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:15:04.281 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:15:04.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:15:03 smithi153 ceph-mon[70044]: pgmap v5005: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:04.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:15:03 smithi083 ceph-mon[79930]: pgmap v5005: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:04.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:15:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:15:04.145Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:15:04.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:15:03 smithi083 ceph-mon[77242]: pgmap v5005: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:05.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:15:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:15:04.845Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:15:05.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:15:04 smithi083 ceph-mon[79930]: from='client.39957 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:15:05.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:15:04 smithi083 ceph-mon[79930]: from='client.49897 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:15:05.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:15:04 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1783292539' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:15:05.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:15:04 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:15:05.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:15:04 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:15:05.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:15:04 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:15:05.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:15:04 smithi083 ceph-mon[77242]: from='client.39957 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:15:05.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:15:04 smithi083 ceph-mon[77242]: from='client.49897 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:15:05.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:15:04 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1783292539' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:15:05.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:15:04 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:15:05.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:15:04 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:15:05.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:15:04 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:15:05.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:15:04 smithi153 ceph-mon[70044]: from='client.39957 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:15:05.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:15:04 smithi153 ceph-mon[70044]: from='client.49897 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:15:05.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:15:04 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1783292539' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:15:05.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:15:04 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:15:05.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:15:04 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:15:05.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:15:04 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:15:06.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:15:05 smithi153 ceph-mon[70044]: pgmap v5006: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:06.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:15:05 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:15:06.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:15:05 smithi083 ceph-mon[79930]: pgmap v5006: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:06.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:15:05 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:15:06.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:15:05 smithi083 ceph-mon[77242]: pgmap v5006: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:06.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:15:05 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:15:08.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:15:07 smithi153 ceph-mon[70044]: pgmap v5007: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:08.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:15:07 smithi083 ceph-mon[79930]: pgmap v5007: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:08.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:15:07 smithi083 ceph-mon[77242]: pgmap v5007: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:10.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:15:09 smithi153 ceph-mon[70044]: pgmap v5008: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:10.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:15:09 smithi083 ceph-mon[79930]: pgmap v5008: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:10.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:15:09 smithi083 ceph-mon[77242]: pgmap v5008: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:11.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:15:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:15:10.757Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:15:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:15:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:15:11] "GET /metrics HTTP/1.1" 200 225217 "" "Prometheus/2.33.4" 2022-09-29T08:15:12.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:15:11 smithi153 ceph-mon[70044]: pgmap v5009: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:12.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:15:11 smithi083 ceph-mon[79930]: pgmap v5009: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:12.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:15:11 smithi083 ceph-mon[77242]: pgmap v5009: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:15:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:15:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:15:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:15:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:15:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:15:14.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:15:14 smithi153 ceph-mon[70044]: pgmap v5010: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:14.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:15:14 smithi083 ceph-mon[79930]: pgmap v5010: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:14.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:15:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:15:14.147Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:15:14.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:15:14 smithi083 ceph-mon[77242]: pgmap v5010: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:15.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:15:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:15:14.846Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:15:16.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:15:16 smithi153 ceph-mon[70044]: pgmap v5011: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:16.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:15:16 smithi083 ceph-mon[79930]: pgmap v5011: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:16.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:15:16 smithi083 ceph-mon[77242]: pgmap v5011: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:18.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:15:18 smithi153 ceph-mon[70044]: pgmap v5012: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:18.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:15:18 smithi083 ceph-mon[77242]: pgmap v5012: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:18.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:15:18 smithi083 ceph-mon[79930]: pgmap v5012: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:15:20 smithi153 ceph-mon[70044]: pgmap v5013: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:20.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:15:20 smithi083 ceph-mon[79930]: pgmap v5013: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:20.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:15:20 smithi083 ceph-mon[77242]: pgmap v5013: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:21.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:15:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:15:20.758Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:15:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:15:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:15:21] "GET /metrics HTTP/1.1" 200 225217 "" "Prometheus/2.33.4" 2022-09-29T08:15:22.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:15:22 smithi153 ceph-mon[70044]: pgmap v5014: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:22.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:15:22 smithi083 ceph-mon[77242]: pgmap v5014: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:22.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:15:22 smithi083 ceph-mon[79930]: pgmap v5014: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:15:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:15:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:15:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:15:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:15:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:15:24.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:15:24 smithi153 ceph-mon[70044]: pgmap v5015: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:24.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:15:24 smithi083 ceph-mon[77242]: pgmap v5015: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:24.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:15:24 smithi083 ceph-mon[79930]: pgmap v5015: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:24.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:15:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:15:24.147Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:15:25.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:15:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:15:24.846Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:15:26.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:15:26 smithi153 ceph-mon[70044]: pgmap v5016: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:26.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:15:26 smithi083 ceph-mon[77242]: pgmap v5016: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:26.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:15:26 smithi083 ceph-mon[79930]: pgmap v5016: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:28.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:15:28 smithi153 ceph-mon[70044]: pgmap v5017: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:28.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:15:28 smithi083 ceph-mon[79930]: pgmap v5017: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:28.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:15:28 smithi083 ceph-mon[77242]: pgmap v5017: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:30.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:15:30 smithi153 ceph-mon[70044]: pgmap v5018: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:30.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:15:30 smithi083 ceph-mon[79930]: pgmap v5018: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:30.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:15:30 smithi083 ceph-mon[77242]: pgmap v5018: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:31.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:15:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:15:30.758Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:15:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:15:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:15:31] "GET /metrics HTTP/1.1" 200 225214 "" "Prometheus/2.33.4" 2022-09-29T08:15:32.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:15:32 smithi153 ceph-mon[70044]: pgmap v5019: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:32.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:15:32 smithi083 ceph-mon[79930]: pgmap v5019: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:32.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:15:32 smithi083 ceph-mon[77242]: pgmap v5019: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:15:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:15:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:15:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:15:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:15:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:15:34.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:15:34 smithi153 ceph-mon[70044]: pgmap v5020: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:34.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:15:34 smithi083 ceph-mon[79930]: pgmap v5020: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:34.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:15:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:15:34.147Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:15:34.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:15:34 smithi083 ceph-mon[77242]: pgmap v5020: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:34.606 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:15:34.896 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:15:34.896 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 8m ago 2h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:15:34.896 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 6m ago 2h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:15:34.896 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 8m ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:15:34.897 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 6m ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:15:34.897 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 6m ago 2h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:15:34.897 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 8m ago 2h 653M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:15:34.897 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 8m ago 2h 638M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:15:34.898 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 6m ago 2h 284M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:15:34.898 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 8m ago 2h 372M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:15:34.898 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 8m ago 2h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:15:34.898 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 6m ago 2h 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:15:34.898 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 8m ago 2h 462M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:15:34.899 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 8m ago 2h 453M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:15:34.899 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 8m ago 2h 358M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:15:34.899 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 8m ago 2h 440M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:15:34.899 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 6m ago 2h 392M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:15:34.900 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 6m ago 2h 482M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:15:34.900 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 6m ago 2h 429M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:15:34.900 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 6m ago 2h 503M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:15:34.900 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 6m ago 2h 137M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:15:35.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:15:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:15:34.847Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:15:35.253 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:15:35.254 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:15:35.254 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:15:35.254 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:15:35.254 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:15:35.255 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:15:35.255 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:15:35.255 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:15:35.255 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:15:35.256 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:15:35.256 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:15:35.256 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:15:35.256 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:15:35.257 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:15:35.257 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:15:35.257 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:15:35.257 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:15:35.257 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:15:36.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:15:36 smithi153 ceph-mon[70044]: from='client.49909 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:15:36.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:15:36 smithi153 ceph-mon[70044]: pgmap v5021: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:36.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:15:36 smithi153 ceph-mon[70044]: from='client.39981 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:15:36.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:15:36 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/951023885' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:15:36.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:15:36 smithi083 ceph-mon[79930]: from='client.49909 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:15:36.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:15:36 smithi083 ceph-mon[79930]: pgmap v5021: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:36.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:15:36 smithi083 ceph-mon[79930]: from='client.39981 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:15:36.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:15:36 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/951023885' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:15:36.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:15:36 smithi083 ceph-mon[77242]: from='client.49909 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:15:36.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:15:36 smithi083 ceph-mon[77242]: pgmap v5021: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:36.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:15:36 smithi083 ceph-mon[77242]: from='client.39981 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:15:36.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:15:36 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/951023885' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:15:38.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:15:38 smithi153 ceph-mon[70044]: pgmap v5022: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:38.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:15:38 smithi083 ceph-mon[77242]: pgmap v5022: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:38.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:15:38 smithi083 ceph-mon[79930]: pgmap v5022: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:15:40 smithi153 ceph-mon[70044]: pgmap v5023: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:15:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:15:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:15:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:15:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:15:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:15:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:15:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:15:40.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:15:40 smithi083 ceph-mon[79930]: pgmap v5023: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:40.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:15:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:15:40.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:15:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:15:40.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:15:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:15:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:15:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:15:40.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:15:40 smithi083 ceph-mon[77242]: pgmap v5023: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:40.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:15:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:15:40.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:15:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:15:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:15:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:15:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:15:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:15:41.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:15:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:15:40.759Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:15:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:15:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:15:41] "GET /metrics HTTP/1.1" 200 225215 "" "Prometheus/2.33.4" 2022-09-29T08:15:42.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:15:42 smithi153 ceph-mon[70044]: pgmap v5024: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:42.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:15:42 smithi083 ceph-mon[77242]: pgmap v5024: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:42.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:15:42 smithi083 ceph-mon[79930]: pgmap v5024: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:15:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:15:42.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:15:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:15:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:15:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:15:44.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:15:44 smithi153 ceph-mon[70044]: pgmap v5025: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:44.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:15:44 smithi083 ceph-mon[79930]: pgmap v5025: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:44.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:15:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:15:44.148Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:15:44.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:15:44 smithi083 ceph-mon[77242]: pgmap v5025: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:45.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:15:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:15:44.847Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:15:46.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:15:46 smithi153 ceph-mon[70044]: pgmap v5026: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:46.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:15:46 smithi083 ceph-mon[79930]: pgmap v5026: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:46.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:15:46 smithi083 ceph-mon[77242]: pgmap v5026: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:48.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:15:48 smithi153 ceph-mon[70044]: pgmap v5027: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:48.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:15:48 smithi083 ceph-mon[79930]: pgmap v5027: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:48.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:15:48 smithi083 ceph-mon[77242]: pgmap v5027: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:50.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:15:50 smithi083 ceph-mon[79930]: pgmap v5028: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:50.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:15:50 smithi083 ceph-mon[77242]: pgmap v5028: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:50.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:15:50 smithi153 ceph-mon[70044]: pgmap v5028: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:51.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:15:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:15:50.760Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:15:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:15:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:15:51] "GET /metrics HTTP/1.1" 200 225215 "" "Prometheus/2.33.4" 2022-09-29T08:15:52.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:15:52 smithi083 ceph-mon[79930]: pgmap v5029: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:52.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:15:52 smithi083 ceph-mon[77242]: pgmap v5029: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:15:52 smithi153 ceph-mon[70044]: pgmap v5029: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:52.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:15:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:15:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:15:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:15:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:15:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:15:54.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:15:54 smithi083 ceph-mon[79930]: pgmap v5030: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:54.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:15:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:15:54.148Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:15:54.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:15:54 smithi083 ceph-mon[77242]: pgmap v5030: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:54.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:15:54 smithi153 ceph-mon[70044]: pgmap v5030: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:55.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:15:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:15:54.848Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:15:56.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:15:56 smithi083 ceph-mon[77242]: pgmap v5031: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:56.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:15:56 smithi083 ceph-mon[79930]: pgmap v5031: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:56.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:15:56 smithi153 ceph-mon[70044]: pgmap v5031: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:58.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:15:58 smithi083 ceph-mon[79930]: pgmap v5032: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:58.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:15:58 smithi083 ceph-mon[77242]: pgmap v5032: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:15:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:15:58 smithi153 ceph-mon[70044]: pgmap v5032: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:00.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:16:00 smithi083 ceph-mon[79930]: pgmap v5033: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:00.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:16:00 smithi083 ceph-mon[77242]: pgmap v5033: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:00.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:16:00 smithi153 ceph-mon[70044]: pgmap v5033: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:01.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:16:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:16:00.761Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:16:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:16:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:16:01] "GET /metrics HTTP/1.1" 200 225206 "" "Prometheus/2.33.4" 2022-09-29T08:16:02.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:16:02 smithi083 ceph-mon[79930]: pgmap v5034: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:02.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:16:02 smithi083 ceph-mon[77242]: pgmap v5034: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:16:02 smithi153 ceph-mon[70044]: pgmap v5034: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:02.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:16:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:16:02.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:16:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:16:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:16:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:16:04.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:16:04 smithi083 ceph-mon[79930]: pgmap v5035: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:04.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:16:04 smithi083 ceph-mon[77242]: pgmap v5035: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:04.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:16:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:16:04.149Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:16:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:16:04 smithi153 ceph-mon[70044]: pgmap v5035: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:05.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:16:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:16:04.848Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:16:05.579 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:16:05.887 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:16:05.888 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 9m ago 2h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:16:05.888 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 7m ago 2h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:16:05.888 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 9m ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:16:05.889 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 7m ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:16:05.889 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 7m ago 2h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:16:05.889 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 9m ago 2h 653M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:16:05.889 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 9m ago 2h 638M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:16:05.889 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 7m ago 2h 284M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:16:05.890 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 9m ago 2h 372M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:16:05.890 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 9m ago 2h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:16:05.890 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 7m ago 2h 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:16:05.890 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 9m ago 2h 462M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:16:05.891 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 9m ago 2h 453M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:16:05.891 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 9m ago 2h 358M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:16:05.891 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 9m ago 2h 440M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:16:05.891 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 7m ago 2h 392M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:16:05.891 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 7m ago 2h 482M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:16:05.892 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 7m ago 2h 429M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:16:05.892 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 7m ago 2h 503M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:16:05.892 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 7m ago 2h 137M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:16:06.279 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:16:06.280 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:16:06.280 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:16:06.280 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:16:06.280 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:16:06.281 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:16:06.281 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:16:06.281 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:16:06.281 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:16:06.281 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:16:06.282 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:16:06.282 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:16:06.282 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:16:06.282 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:16:06.282 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:16:06.283 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:16:06.283 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:16:06.283 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:16:06.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:16:06 smithi083 ceph-mon[77242]: pgmap v5036: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:06.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:16:06 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:16:06.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:16:06 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:16:06.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:16:06 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:16:06.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:16:06 smithi083 ceph-mon[79930]: pgmap v5036: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:06.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:16:06 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:16:06.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:16:06 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:16:06.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:16:06 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:16:06.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:16:06 smithi153 ceph-mon[70044]: pgmap v5036: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:06.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:16:06 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:16:06.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:16:06 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:16:06.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:16:06 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:16:07.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:16:07 smithi083 ceph-mon[77242]: from='client.39993 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:16:07.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:16:07 smithi083 ceph-mon[77242]: from='client.39999 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:16:07.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:16:07 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:16:07.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:16:07 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2695519743' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:16:07.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:16:07 smithi083 ceph-mon[79930]: from='client.39993 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:16:07.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:16:07 smithi083 ceph-mon[79930]: from='client.39999 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:16:07.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:16:07 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:16:07.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:16:07 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2695519743' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:16:07.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:16:07 smithi153 ceph-mon[70044]: from='client.39993 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:16:07.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:16:07 smithi153 ceph-mon[70044]: from='client.39999 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:16:07.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:16:07 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:16:07.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:16:07 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2695519743' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:16:08.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:16:08 smithi083 ceph-mon[79930]: pgmap v5037: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:08.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:16:08 smithi083 ceph-mon[77242]: pgmap v5037: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:16:08 smithi153 ceph-mon[70044]: pgmap v5037: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:10.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:16:10 smithi083 ceph-mon[79930]: pgmap v5038: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:10.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:16:10 smithi083 ceph-mon[77242]: pgmap v5038: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:10.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:16:10 smithi153 ceph-mon[70044]: pgmap v5038: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:11.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:16:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:16:10.761Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:16:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:16:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:16:11] "GET /metrics HTTP/1.1" 200 225215 "" "Prometheus/2.33.4" 2022-09-29T08:16:12.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:16:12 smithi083 ceph-mon[77242]: pgmap v5039: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:12.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:16:12 smithi083 ceph-mon[79930]: pgmap v5039: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:16:12 smithi153 ceph-mon[70044]: pgmap v5039: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:12.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:16:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:16:12.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:16:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:16:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:16:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:16:14.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:16:14 smithi083 ceph-mon[79930]: pgmap v5040: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:14.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:16:14 smithi083 ceph-mon[77242]: pgmap v5040: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:14.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:16:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:16:14.149Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:16:14.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:16:14 smithi153 ceph-mon[70044]: pgmap v5040: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:15.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:16:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:16:14.848Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:16:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:16:16 smithi153 ceph-mon[70044]: pgmap v5041: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:16.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:16:16 smithi083 ceph-mon[79930]: pgmap v5041: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:16.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:16:16 smithi083 ceph-mon[77242]: pgmap v5041: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:16:18 smithi153 ceph-mon[70044]: pgmap v5042: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:18.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:16:18 smithi083 ceph-mon[79930]: pgmap v5042: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:18.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:16:18 smithi083 ceph-mon[77242]: pgmap v5042: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:16:20 smithi153 ceph-mon[70044]: pgmap v5043: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:20.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:16:20 smithi083 ceph-mon[77242]: pgmap v5043: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:20.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:16:20 smithi083 ceph-mon[79930]: pgmap v5043: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:21.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:16:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:16:20.762Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:16:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:16:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:16:21] "GET /metrics HTTP/1.1" 200 225215 "" "Prometheus/2.33.4" 2022-09-29T08:16:22.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:16:22 smithi153 ceph-mon[70044]: pgmap v5044: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:22.559 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:16:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:16:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:16:22.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:16:22 smithi083 ceph-mon[77242]: pgmap v5044: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:22.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:16:22 smithi083 ceph-mon[79930]: pgmap v5044: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:16:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:16:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:16:24.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:16:24 smithi083 ceph-mon[79930]: pgmap v5045: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:24.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:16:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:16:24.150Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:16:24.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:16:24 smithi083 ceph-mon[77242]: pgmap v5045: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:24.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:16:24 smithi153 ceph-mon[70044]: pgmap v5045: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:25.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:16:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:16:24.849Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:16:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:16:26 smithi153 ceph-mon[70044]: pgmap v5046: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:26.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:16:26 smithi083 ceph-mon[79930]: pgmap v5046: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:26.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:16:26 smithi083 ceph-mon[77242]: pgmap v5046: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:16:28 smithi153 ceph-mon[70044]: pgmap v5047: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:28.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:16:28 smithi083 ceph-mon[79930]: pgmap v5047: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:28.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:16:28 smithi083 ceph-mon[77242]: pgmap v5047: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:16:30 smithi153 ceph-mon[70044]: pgmap v5048: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:30.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:16:30 smithi083 ceph-mon[79930]: pgmap v5048: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:30.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:16:30 smithi083 ceph-mon[77242]: pgmap v5048: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:31.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:16:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:16:30.762Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:16:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:16:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:16:31] "GET /metrics HTTP/1.1" 200 225212 "" "Prometheus/2.33.4" 2022-09-29T08:16:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:16:32 smithi153 ceph-mon[70044]: pgmap v5049: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:32.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:16:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:16:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:16:32.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:16:32 smithi083 ceph-mon[79930]: pgmap v5049: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:32.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:16:32 smithi083 ceph-mon[77242]: pgmap v5049: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:16:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:16:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:16:34.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:16:34 smithi083 ceph-mon[79930]: pgmap v5050: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:34.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:16:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:16:34.150Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:16:34.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:16:34 smithi083 ceph-mon[77242]: pgmap v5050: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:16:34 smithi153 ceph-mon[70044]: pgmap v5050: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:35.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:16:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:16:34.849Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:16:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:16:36 smithi153 ceph-mon[70044]: pgmap v5051: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:36.604 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:16:36.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:16:36 smithi083 ceph-mon[79930]: pgmap v5051: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:36.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:16:36 smithi083 ceph-mon[77242]: pgmap v5051: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:36.892 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:16:36.893 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 9m ago 2h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:16:36.893 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 7m ago 2h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:16:36.893 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 9m ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:16:36.893 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 7m ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:16:36.894 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 7m ago 2h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:16:36.894 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 9m ago 2h 653M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:16:36.894 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 9m ago 2h 638M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:16:36.894 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 7m ago 2h 284M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:16:36.895 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 9m ago 2h 372M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:16:36.895 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 9m ago 2h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:16:36.895 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 7m ago 2h 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:16:36.895 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 9m ago 2h 462M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:16:36.895 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 9m ago 2h 453M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:16:36.896 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 9m ago 2h 358M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:16:36.896 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 9m ago 2h 440M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:16:36.896 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 7m ago 2h 392M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:16:36.896 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 7m ago 2h 482M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:16:36.897 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 7m ago 2h 429M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:16:36.897 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 7m ago 2h 503M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:16:36.897 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 7m ago 2h 137M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:16:37.251 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:16:37.251 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:16:37.251 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:16:37.251 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:16:37.252 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:16:37.252 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:16:37.252 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:16:37.252 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:16:37.253 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:16:37.253 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:16:37.253 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:16:37.253 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:16:37.254 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:16:37.254 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:16:37.254 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:16:37.254 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:16:37.254 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:16:37.255 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:16:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:16:38 smithi153 ceph-mon[70044]: from='client.49945 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:16:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:16:38 smithi153 ceph-mon[70044]: pgmap v5052: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:16:38 smithi153 ceph-mon[70044]: from='client.40017 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:16:38.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:16:38 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2488975287' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:16:38.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:16:38 smithi083 ceph-mon[79930]: from='client.49945 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:16:38.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:16:38 smithi083 ceph-mon[79930]: pgmap v5052: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:38.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:16:38 smithi083 ceph-mon[79930]: from='client.40017 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:16:38.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:16:38 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2488975287' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:16:38.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:16:38 smithi083 ceph-mon[77242]: from='client.49945 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:16:38.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:16:38 smithi083 ceph-mon[77242]: pgmap v5052: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:38.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:16:38 smithi083 ceph-mon[77242]: from='client.40017 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:16:38.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:16:38 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2488975287' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:16:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:16:40 smithi153 ceph-mon[70044]: pgmap v5053: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:16:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:16:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:16:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:16:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:16:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:16:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:16:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:16:40.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:16:40 smithi083 ceph-mon[79930]: pgmap v5053: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:40.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:16:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:16:40.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:16:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:16:40.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:16:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:16:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:16:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:16:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:16:40 smithi083 ceph-mon[77242]: pgmap v5053: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:16:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:16:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:16:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:16:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:16:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:16:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:16:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:16:41.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:16:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:16:40.763Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:16:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:16:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:16:41] "GET /metrics HTTP/1.1" 200 225214 "" "Prometheus/2.33.4" 2022-09-29T08:16:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:16:42 smithi153 ceph-mon[70044]: pgmap v5054: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:42.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:16:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:16:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:16:42.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:16:42 smithi083 ceph-mon[79930]: pgmap v5054: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:42.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:16:42 smithi083 ceph-mon[77242]: pgmap v5054: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:16:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:16:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:16:44.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:16:44 smithi083 ceph-mon[79930]: pgmap v5055: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:44.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:16:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:16:44.151Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:16:44.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:16:44 smithi083 ceph-mon[77242]: pgmap v5055: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:44.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:16:44 smithi153 ceph-mon[70044]: pgmap v5055: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:45.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:16:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:16:44.850Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:16:46.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:16:46 smithi153 ceph-mon[70044]: pgmap v5056: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:46.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:16:46 smithi083 ceph-mon[79930]: pgmap v5056: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:46.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:16:46 smithi083 ceph-mon[77242]: pgmap v5056: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:48.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:16:48 smithi153 ceph-mon[70044]: pgmap v5057: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:48.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:16:48 smithi083 ceph-mon[77242]: pgmap v5057: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:48.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:16:48 smithi083 ceph-mon[79930]: pgmap v5057: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:16:50 smithi153 ceph-mon[70044]: pgmap v5058: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:50.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:16:50 smithi083 ceph-mon[77242]: pgmap v5058: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:50.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:16:50 smithi083 ceph-mon[79930]: pgmap v5058: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:51.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:16:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:16:50.763Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:16:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:16:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:16:51] "GET /metrics HTTP/1.1" 200 225214 "" "Prometheus/2.33.4" 2022-09-29T08:16:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:16:52 smithi153 ceph-mon[70044]: pgmap v5059: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:52.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:16:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:16:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:16:52.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:16:52 smithi083 ceph-mon[79930]: pgmap v5059: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:52.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:16:52 smithi083 ceph-mon[77242]: pgmap v5059: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:16:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:16:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:16:54.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:16:54 smithi083 ceph-mon[79930]: pgmap v5060: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:54.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:16:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:16:54.151Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:16:54.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:16:54 smithi083 ceph-mon[77242]: pgmap v5060: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:54.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:16:54 smithi153 ceph-mon[70044]: pgmap v5060: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:55.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:16:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:16:54.850Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:16:56.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:16:56 smithi153 ceph-mon[70044]: pgmap v5061: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:56.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:16:56 smithi083 ceph-mon[79930]: pgmap v5061: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:56.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:16:56 smithi083 ceph-mon[77242]: pgmap v5061: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:58.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:16:58 smithi153 ceph-mon[70044]: pgmap v5062: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:58.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:16:58 smithi083 ceph-mon[79930]: pgmap v5062: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:16:58.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:16:58 smithi083 ceph-mon[77242]: pgmap v5062: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:17:00 smithi153 ceph-mon[70044]: pgmap v5063: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:00.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:17:00 smithi083 ceph-mon[79930]: pgmap v5063: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:00.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:17:00 smithi083 ceph-mon[77242]: pgmap v5063: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:01.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:17:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:17:00.764Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:17:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:17:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:17:01] "GET /metrics HTTP/1.1" 200 225215 "" "Prometheus/2.33.4" 2022-09-29T08:17:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:17:02 smithi153 ceph-mon[70044]: pgmap v5064: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:02.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:17:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:17:02.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:17:02.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:17:02 smithi083 ceph-mon[77242]: pgmap v5064: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:02.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:17:02 smithi083 ceph-mon[79930]: pgmap v5064: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:17:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:17:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:17:04.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:17:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:17:04.152Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:17:04.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:17:04 smithi083 ceph-mon[77242]: pgmap v5065: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:04.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:17:04 smithi083 ceph-mon[79930]: pgmap v5065: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:17:04 smithi153 ceph-mon[70044]: pgmap v5065: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:05.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:17:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:17:04.851Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:17:06.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:17:06 smithi153 ceph-mon[70044]: pgmap v5066: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:06.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:17:06 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:17:06.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:17:06 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:17:06.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:17:06 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:17:06.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:17:06 smithi083 ceph-mon[79930]: pgmap v5066: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:06.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:17:06 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:17:06.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:17:06 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:17:06.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:17:06 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:17:06.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:17:06 smithi083 ceph-mon[77242]: pgmap v5066: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:06.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:17:06 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:17:06.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:17:06 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:17:06.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:17:06 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:17:07.579 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:17:07.887 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:17:07.887 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 10m ago 2h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:17:07.888 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 8m ago 2h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:17:07.888 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 10m ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:17:07.888 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 8m ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:17:07.888 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 8m ago 2h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:17:07.889 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 10m ago 2h 653M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:17:07.889 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 10m ago 2h 638M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:17:07.889 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 8m ago 2h 284M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:17:07.889 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 10m ago 2h 372M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:17:07.889 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 10m ago 2h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:17:07.890 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 8m ago 2h 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:17:07.890 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 10m ago 2h 462M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:17:07.890 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 10m ago 2h 453M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:17:07.890 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 10m ago 2h 358M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:17:07.891 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 10m ago 2h 440M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:17:07.891 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 8m ago 2h 392M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:17:07.891 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 8m ago 2h 482M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:17:07.891 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 8m ago 2h 429M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:17:07.891 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 8m ago 2h 503M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:17:07.892 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 8m ago 2h 137M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:17:08.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:17:07 smithi153 ceph-mon[70044]: pgmap v5067: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:08.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:17:07 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:17:08.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:17:07 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:17:08.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:17:07 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:17:08.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:17:07 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:17:08.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:17:07 smithi083 ceph-mon[79930]: pgmap v5067: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:08.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:17:07 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:17:08.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:17:07 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:17:08.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:17:07 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:17:08.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:17:07 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:17:08.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:17:07 smithi083 ceph-mon[77242]: pgmap v5067: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:08.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:17:07 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:17:08.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:17:07 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:17:08.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:17:07 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:17:08.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:17:07 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:17:08.271 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:17:08.271 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:17:08.271 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:17:08.271 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:17:08.272 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:17:08.272 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:17:08.272 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:17:08.272 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:17:08.272 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:17:08.273 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:17:08.273 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:17:08.273 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:17:08.273 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:17:08.274 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:17:08.274 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:17:08.274 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:17:08.274 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:17:08.274 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:17:09.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:17:08 smithi153 ceph-mon[70044]: from='client.40029 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:17:09.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:17:08 smithi153 ceph-mon[70044]: from='client.40035 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:17:09.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:17:08 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2912085426' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:17:09.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:17:08 smithi083 ceph-mon[79930]: from='client.40029 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:17:09.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:17:08 smithi083 ceph-mon[79930]: from='client.40035 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:17:09.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:17:08 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2912085426' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:17:09.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:17:08 smithi083 ceph-mon[77242]: from='client.40029 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:17:09.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:17:08 smithi083 ceph-mon[77242]: from='client.40035 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:17:09.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:17:08 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2912085426' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:17:10.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:17:09 smithi153 ceph-mon[70044]: pgmap v5068: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:10.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:17:09 smithi083 ceph-mon[79930]: pgmap v5068: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:10.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:17:09 smithi083 ceph-mon[77242]: pgmap v5068: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:11.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:17:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:17:10.765Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:17:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:17:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:17:11] "GET /metrics HTTP/1.1" 200 225215 "" "Prometheus/2.33.4" 2022-09-29T08:17:11.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:17:11 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:17:11.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:17:11 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:17:11.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:17:11 smithi083 ceph-mon[77242]: pgmap v5069: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:11.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:17:11 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:17:11.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:17:11 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:17:11.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:17:11 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:17:11.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:17:11 smithi083 ceph-mon[79930]: pgmap v5069: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:11.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:17:11 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:17:12.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:17:11 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:17:12.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:17:11 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:17:12.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:17:11 smithi153 ceph-mon[70044]: pgmap v5069: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:12.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:17:11 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:17:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:17:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:17:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:17:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:17:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:17:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:17:14.154 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:17:13 smithi083 ceph-mon[79930]: pgmap v5070: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:14.155 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:17:13 smithi083 ceph-mon[77242]: pgmap v5070: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:14.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:17:13 smithi153 ceph-mon[70044]: pgmap v5070: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:14.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:17:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:17:14.153Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:17:15.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:17:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:17:14.851Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:17:16.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:17:15 smithi083 ceph-mon[79930]: pgmap v5071: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:16.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:17:15 smithi083 ceph-mon[77242]: pgmap v5071: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:16.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:17:15 smithi153 ceph-mon[70044]: pgmap v5071: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:18.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:17:17 smithi083 ceph-mon[77242]: pgmap v5072: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:18.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:17:17 smithi083 ceph-mon[79930]: pgmap v5072: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:18.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:17:17 smithi153 ceph-mon[70044]: pgmap v5072: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:20.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:17:19 smithi083 ceph-mon[79930]: pgmap v5073: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:20.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:17:19 smithi083 ceph-mon[77242]: pgmap v5073: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:17:19 smithi153 ceph-mon[70044]: pgmap v5073: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:21.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:17:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:17:20.766Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:17:21.888 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:17:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:17:21] "GET /metrics HTTP/1.1" 200 225215 "" "Prometheus/2.33.4" 2022-09-29T08:17:22.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:17:21 smithi083 ceph-mon[77242]: pgmap v5074: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:22.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:17:21 smithi083 ceph-mon[79930]: pgmap v5074: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:22.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:17:21 smithi153 ceph-mon[70044]: pgmap v5074: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:17:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:17:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:17:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:17:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:17:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:17:24.154 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:17:23 smithi083 ceph-mon[79930]: pgmap v5075: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:24.154 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:17:23 smithi083 ceph-mon[77242]: pgmap v5075: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:24.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:17:23 smithi153 ceph-mon[70044]: pgmap v5075: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:24.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:17:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:17:24.153Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:17:25.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:17:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:17:24.852Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:17:26.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:17:25 smithi083 ceph-mon[77242]: pgmap v5076: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:26.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:17:25 smithi083 ceph-mon[79930]: pgmap v5076: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:26.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:17:25 smithi153 ceph-mon[70044]: pgmap v5076: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:28.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:17:27 smithi083 ceph-mon[79930]: pgmap v5077: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:28.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:17:27 smithi083 ceph-mon[77242]: pgmap v5077: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:28.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:17:27 smithi153 ceph-mon[70044]: pgmap v5077: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:30.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:17:29 smithi083 ceph-mon[77242]: pgmap v5078: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:30.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:17:29 smithi083 ceph-mon[79930]: pgmap v5078: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:30.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:17:29 smithi153 ceph-mon[70044]: pgmap v5078: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:31.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:17:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:17:30.767Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:17:31.905 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:17:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:17:31] "GET /metrics HTTP/1.1" 200 225213 "" "Prometheus/2.33.4" 2022-09-29T08:17:32.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:17:31 smithi083 ceph-mon[77242]: pgmap v5079: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:32.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:17:31 smithi083 ceph-mon[79930]: pgmap v5079: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:32.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:17:31 smithi153 ceph-mon[70044]: pgmap v5079: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:17:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:17:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:17:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:17:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:17:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:17:34.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:17:33 smithi083 ceph-mon[79930]: pgmap v5080: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:34.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:17:33 smithi083 ceph-mon[77242]: pgmap v5080: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:34.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:17:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:17:34.154Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:17:34.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:17:33 smithi153 ceph-mon[70044]: pgmap v5080: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:35.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:17:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:17:34.853Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:17:36.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:17:35 smithi083 ceph-mon[79930]: pgmap v5081: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:36.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:17:35 smithi083 ceph-mon[77242]: pgmap v5081: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:36.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:17:35 smithi153 ceph-mon[70044]: pgmap v5081: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:38.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:17:37 smithi083 ceph-mon[79930]: pgmap v5082: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:38.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:17:37 smithi083 ceph-mon[77242]: pgmap v5082: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:38.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:17:37 smithi153 ceph-mon[70044]: pgmap v5082: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:38.593 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:17:38.886 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:17:38.886 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 28s ago 2h 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:17:38.886 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 8m ago 2h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:17:38.886 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 28s ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:17:38.887 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 8m ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:17:38.887 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 8m ago 2h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:17:38.887 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 28s ago 2h 662M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:17:38.888 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 28s ago 2h 675M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:17:38.888 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 8m ago 2h 284M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:17:38.888 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 28s ago 2h 394M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:17:38.888 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 28s ago 2h 26.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:17:38.888 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 8m ago 2h 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:17:38.889 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 28s ago 2h 488M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:17:38.889 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 28s ago 2h 479M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:17:38.889 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 28s ago 2h 378M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:17:38.889 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 28s ago 2h 465M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:17:38.889 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 8m ago 2h 392M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:17:38.890 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 8m ago 2h 482M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:17:38.890 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 8m ago 2h 429M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:17:38.890 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 8m ago 2h 503M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:17:38.890 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 8m ago 2h 137M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:17:39.247 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:17:39.247 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:17:39.248 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:17:39.248 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:17:39.248 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:17:39.248 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:17:39.249 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:17:39.249 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:17:39.249 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:17:39.249 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:17:39.249 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:17:39.250 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:17:39.250 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:17:39.250 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:17:39.250 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:17:39.251 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:17:39.251 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:17:39.251 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:17:40.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:17:39 smithi083 ceph-mon[79930]: from='client.49981 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:17:40.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:17:39 smithi083 ceph-mon[79930]: pgmap v5083: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:40.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:17:39 smithi083 ceph-mon[79930]: from='client.40053 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:17:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:17:39 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/86831519' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:17:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:17:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:17:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:17:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:17:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:17:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:17:40.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:17:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:17:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:17:39 smithi083 ceph-mon[77242]: from='client.49981 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:17:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:17:39 smithi083 ceph-mon[77242]: pgmap v5083: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:17:39 smithi083 ceph-mon[77242]: from='client.40053 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:17:40.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:17:39 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/86831519' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:17:40.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:17:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:17:40.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:17:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:17:40.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:17:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:17:40.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:17:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:17:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:17:39 smithi153 ceph-mon[70044]: from='client.49981 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:17:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:17:39 smithi153 ceph-mon[70044]: pgmap v5083: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:17:39 smithi153 ceph-mon[70044]: from='client.40053 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:17:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:17:39 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/86831519' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:17:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:17:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:17:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:17:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:17:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:17:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:17:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:17:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:17:41.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:17:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:17:40.767Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:17:41.920 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:17:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:17:41] "GET /metrics HTTP/1.1" 200 225211 "" "Prometheus/2.33.4" 2022-09-29T08:17:42.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:17:41 smithi083 ceph-mon[79930]: pgmap v5084: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:42.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:17:41 smithi083 ceph-mon[77242]: pgmap v5084: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:42.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:17:41 smithi153 ceph-mon[70044]: pgmap v5084: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:17:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:17:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:17:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:17:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:17:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:17:44.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:17:43 smithi083 ceph-mon[79930]: pgmap v5085: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:44.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:17:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:17:44.155Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:17:44.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:17:43 smithi083 ceph-mon[77242]: pgmap v5085: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:44.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:17:43 smithi153 ceph-mon[70044]: pgmap v5085: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:45.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:17:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:17:44.853Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:17:46.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:17:45 smithi083 ceph-mon[79930]: pgmap v5086: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:46.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:17:45 smithi083 ceph-mon[77242]: pgmap v5086: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:46.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:17:45 smithi153 ceph-mon[70044]: pgmap v5086: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:48.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:17:47 smithi083 ceph-mon[79930]: pgmap v5087: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:48.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:17:47 smithi083 ceph-mon[77242]: pgmap v5087: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:48.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:17:47 smithi153 ceph-mon[70044]: pgmap v5087: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:50.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:17:49 smithi083 ceph-mon[79930]: pgmap v5088: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:50.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:17:49 smithi083 ceph-mon[77242]: pgmap v5088: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:50.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:17:49 smithi153 ceph-mon[70044]: pgmap v5088: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:51.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:17:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:17:50.769Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:17:51.945 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:17:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:17:51] "GET /metrics HTTP/1.1" 200 225211 "" "Prometheus/2.33.4" 2022-09-29T08:17:52.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:17:51 smithi083 ceph-mon[79930]: pgmap v5089: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:52.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:17:51 smithi083 ceph-mon[77242]: pgmap v5089: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:52.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:17:51 smithi153 ceph-mon[70044]: pgmap v5089: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:17:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:17:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:17:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:17:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:17:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:17:54.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:17:53 smithi083 ceph-mon[79930]: pgmap v5090: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:54.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:17:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:17:54.154Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:17:54.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:17:53 smithi083 ceph-mon[77242]: pgmap v5090: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:54.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:17:53 smithi153 ceph-mon[70044]: pgmap v5090: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:55.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:17:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:17:54.853Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:17:56.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:17:55 smithi083 ceph-mon[79930]: pgmap v5091: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:56.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:17:55 smithi083 ceph-mon[77242]: pgmap v5091: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:56.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:17:55 smithi153 ceph-mon[70044]: pgmap v5091: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:58.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:17:57 smithi083 ceph-mon[79930]: pgmap v5092: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:58.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:17:57 smithi083 ceph-mon[77242]: pgmap v5092: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:17:58.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:17:57 smithi153 ceph-mon[70044]: pgmap v5092: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:00.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:17:59 smithi083 ceph-mon[79930]: pgmap v5093: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:00.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:17:59 smithi083 ceph-mon[77242]: pgmap v5093: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:00.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:17:59 smithi153 ceph-mon[70044]: pgmap v5093: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:01.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:18:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:18:00.769Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:18:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:18:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:18:01] "GET /metrics HTTP/1.1" 200 225214 "" "Prometheus/2.33.4" 2022-09-29T08:18:02.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:18:01 smithi153 ceph-mon[70044]: pgmap v5094: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:02.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:18:01 smithi083 ceph-mon[77242]: pgmap v5094: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:02.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:18:01 smithi083 ceph-mon[79930]: pgmap v5094: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:18:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:18:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:18:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:18:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:18:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:18:04.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:18:03 smithi153 ceph-mon[70044]: pgmap v5095: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:04.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:18:03 smithi083 ceph-mon[79930]: pgmap v5095: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:04.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:18:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:18:04.155Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:18:04.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:18:03 smithi083 ceph-mon[77242]: pgmap v5095: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:05.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:18:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:18:04.855Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:18:06.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:18:05 smithi153 ceph-mon[70044]: pgmap v5096: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:06.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:18:05 smithi083 ceph-mon[79930]: pgmap v5096: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:06.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:18:05 smithi083 ceph-mon[77242]: pgmap v5096: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:08.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:18:07 smithi153 ceph-mon[70044]: pgmap v5097: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:08.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:18:07 smithi083 ceph-mon[77242]: pgmap v5097: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:08.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:18:07 smithi083 ceph-mon[79930]: pgmap v5097: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:09.570 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:18:09.856 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:18:09.856 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 59s ago 2h 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:18:09.857 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 9m ago 2h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:18:09.857 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 59s ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:18:09.857 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 9m ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:18:09.857 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 9m ago 2h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:18:09.857 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 59s ago 2h 662M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:18:09.858 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 59s ago 2h 675M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:18:09.858 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 9m ago 2h 284M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:18:09.858 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 59s ago 2h 394M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:18:09.858 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 59s ago 2h 26.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:18:09.859 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 9m ago 2h 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:18:09.859 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 59s ago 2h 488M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:18:09.859 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 59s ago 2h 479M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:18:09.859 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 59s ago 2h 378M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:18:09.859 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 59s ago 2h 465M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:18:09.860 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 9m ago 2h 392M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:18:09.860 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 9m ago 2h 482M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:18:09.860 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 9m ago 2h 429M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:18:09.860 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 9m ago 2h 503M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:18:09.860 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 9m ago 2h 137M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:18:10.210 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:18:10.210 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:18:10.210 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:18:10.211 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:18:10.211 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:18:10.211 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:18:10.211 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:18:10.211 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:18:10.212 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:18:10.212 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:18:10.212 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:18:10.212 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:18:10.213 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:18:10.213 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:18:10.213 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:18:10.213 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:18:10.213 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:18:10.214 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:18:10.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:18:09 smithi153 ceph-mon[70044]: pgmap v5098: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:10.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:18:09 smithi083 ceph-mon[79930]: pgmap v5098: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:10.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:18:09 smithi083 ceph-mon[77242]: pgmap v5098: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:11.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:18:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:18:10.769Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:18:11.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:18:10 smithi083 ceph-mon[77242]: from='client.40065 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:18:11.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:18:10 smithi083 ceph-mon[77242]: from='client.50005 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:18:11.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:18:10 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3230621072' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:18:11.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:18:10 smithi083 ceph-mon[79930]: from='client.40065 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:18:11.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:18:10 smithi083 ceph-mon[79930]: from='client.50005 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:18:11.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:18:10 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3230621072' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:18:11.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:18:10 smithi153 ceph-mon[70044]: from='client.40065 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:18:11.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:18:10 smithi153 ceph-mon[70044]: from='client.50005 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:18:11.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:18:10 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3230621072' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:18:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:18:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:18:11] "GET /metrics HTTP/1.1" 200 225213 "" "Prometheus/2.33.4" 2022-09-29T08:18:12.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:18:11 smithi153 ceph-mon[70044]: pgmap v5099: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:12.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:18:11 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:18:12.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:18:11 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:18:12.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:18:11 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:18:12.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:18:11 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:18:12.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:18:11 smithi083 ceph-mon[77242]: pgmap v5099: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:12.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:18:11 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:18:12.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:18:11 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:18:12.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:18:11 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:18:12.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:18:11 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:18:12.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:18:11 smithi083 ceph-mon[79930]: pgmap v5099: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:12.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:18:11 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:18:12.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:18:11 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:18:12.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:18:11 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:18:12.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:18:11 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:18:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:18:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:18:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:18:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:18:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:18:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:18:14.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:18:13 smithi153 ceph-mon[70044]: pgmap v5100: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:14.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:18:13 smithi083 ceph-mon[79930]: pgmap v5100: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:14.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:18:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:18:14.156Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:18:14.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:18:13 smithi083 ceph-mon[77242]: pgmap v5100: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:15.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:18:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:18:14.855Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:18:16.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:18:15 smithi153 ceph-mon[70044]: pgmap v5101: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:16.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:18:15 smithi083 ceph-mon[79930]: pgmap v5101: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:16.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:18:15 smithi083 ceph-mon[77242]: pgmap v5101: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:18.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:18:17 smithi153 ceph-mon[70044]: pgmap v5102: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:18.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:18:17 smithi083 ceph-mon[79930]: pgmap v5102: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:18.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:18:17 smithi083 ceph-mon[77242]: pgmap v5102: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:18:19 smithi153 ceph-mon[70044]: pgmap v5103: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:20.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:18:19 smithi083 ceph-mon[79930]: pgmap v5103: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:20.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:18:19 smithi083 ceph-mon[77242]: pgmap v5103: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:21.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:18:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:18:20.771Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:18:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:18:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:18:21] "GET /metrics HTTP/1.1" 200 225213 "" "Prometheus/2.33.4" 2022-09-29T08:18:22.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:18:21 smithi153 ceph-mon[70044]: pgmap v5104: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:22.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:18:21 smithi083 ceph-mon[79930]: pgmap v5104: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:22.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:18:21 smithi083 ceph-mon[77242]: pgmap v5104: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:18:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:18:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:18:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:18:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:18:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:18:24.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:18:23 smithi153 ceph-mon[70044]: pgmap v5105: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:24.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:18:23 smithi083 ceph-mon[79930]: pgmap v5105: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:24.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:18:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:18:24.156Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:18:24.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:18:23 smithi083 ceph-mon[77242]: pgmap v5105: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:25.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:18:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:18:24.856Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:18:26.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:18:25 smithi153 ceph-mon[70044]: pgmap v5106: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:26.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:18:25 smithi083 ceph-mon[79930]: pgmap v5106: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:26.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:18:25 smithi083 ceph-mon[77242]: pgmap v5106: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:28.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:18:28 smithi153 ceph-mon[70044]: pgmap v5107: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:28.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:18:28 smithi083 ceph-mon[79930]: pgmap v5107: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:28.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:18:28 smithi083 ceph-mon[77242]: pgmap v5107: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:30.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:18:30 smithi153 ceph-mon[70044]: pgmap v5108: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:30.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:18:30 smithi083 ceph-mon[79930]: pgmap v5108: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:30.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:18:30 smithi083 ceph-mon[77242]: pgmap v5108: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:31.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:18:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:18:30.772Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:18:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:18:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:18:31] "GET /metrics HTTP/1.1" 200 225208 "" "Prometheus/2.33.4" 2022-09-29T08:18:32.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:18:32 smithi153 ceph-mon[70044]: pgmap v5109: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:32.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:18:32 smithi083 ceph-mon[77242]: pgmap v5109: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:32.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:18:32 smithi083 ceph-mon[79930]: pgmap v5109: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:18:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:18:32.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:18:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:18:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:18:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:18:34.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:18:34 smithi153 ceph-mon[70044]: pgmap v5110: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:34.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:18:34 smithi083 ceph-mon[79930]: pgmap v5110: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:34.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:18:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:18:34.157Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:18:34.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:18:34 smithi083 ceph-mon[77242]: pgmap v5110: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:35.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:18:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:18:34.857Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:18:36.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:18:36 smithi153 ceph-mon[70044]: pgmap v5111: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:36.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:18:36 smithi083 ceph-mon[77242]: pgmap v5111: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:36.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:18:36 smithi083 ceph-mon[79930]: pgmap v5111: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:38.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:18:38 smithi153 ceph-mon[70044]: pgmap v5112: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:38.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:18:38 smithi083 ceph-mon[79930]: pgmap v5112: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:38.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:18:38 smithi083 ceph-mon[77242]: pgmap v5112: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:18:40 smithi153 ceph-mon[70044]: pgmap v5113: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:18:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:18:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:18:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:18:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:18:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:18:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:18:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:18:40.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:18:40 smithi083 ceph-mon[79930]: pgmap v5113: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:40.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:18:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:18:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:18:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:18:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:18:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:18:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:18:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:18:40.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:18:40 smithi083 ceph-mon[77242]: pgmap v5113: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:18:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:18:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:18:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:18:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:18:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:18:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:18:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:18:40.538 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:18:40.827 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:18:40.828 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 90s ago 2h 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:18:40.828 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 9m ago 2h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:18:40.828 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 90s ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:18:40.828 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 9m ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:18:40.829 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 9m ago 2h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:18:40.829 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 90s ago 2h 662M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:18:40.829 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 90s ago 2h 675M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:18:40.829 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 9m ago 2h 284M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:18:40.830 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 90s ago 2h 394M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:18:40.830 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 90s ago 2h 26.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:18:40.830 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 9m ago 2h 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:18:40.830 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 90s ago 2h 488M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:18:40.831 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 90s ago 2h 479M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:18:40.831 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 90s ago 2h 378M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:18:40.831 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 90s ago 2h 465M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:18:40.831 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 9m ago 2h 392M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:18:40.831 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 9m ago 2h 482M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:18:40.832 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 9m ago 2h 429M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:18:40.832 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 9m ago 2h 503M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:18:40.832 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 9m ago 2h 137M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:18:41.186 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:18:41.186 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:18:41.186 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:18:41.186 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:18:41.187 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:18:41.187 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:18:41.187 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:18:41.187 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:18:41.188 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:18:41.188 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:18:41.188 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:18:41.188 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:18:41.188 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:18:41.189 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:18:41.189 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:18:41.189 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:18:41.189 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:18:41.189 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:18:41.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:18:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:18:40.772Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:18:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:18:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:18:41] "GET /metrics HTTP/1.1" 200 225211 "" "Prometheus/2.33.4" 2022-09-29T08:18:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:18:42 smithi153 ceph-mon[70044]: from='client.50017 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:18:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:18:42 smithi153 ceph-mon[70044]: pgmap v5114: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:18:42 smithi153 ceph-mon[70044]: from='client.50023 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:18:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:18:42 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2803745664' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:18:42.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:18:42 smithi083 ceph-mon[77242]: from='client.50017 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:18:42.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:18:42 smithi083 ceph-mon[77242]: pgmap v5114: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:42.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:18:42 smithi083 ceph-mon[77242]: from='client.50023 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:18:42.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:18:42 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2803745664' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:18:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:18:42 smithi083 ceph-mon[79930]: from='client.50017 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:18:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:18:42 smithi083 ceph-mon[79930]: pgmap v5114: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:42.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:18:42 smithi083 ceph-mon[79930]: from='client.50023 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:18:42.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:18:42 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2803745664' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:18:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:18:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:18:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:18:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:18:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:18:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:18:44.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:18:44 smithi153 ceph-mon[70044]: pgmap v5115: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:44.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:18:44 smithi083 ceph-mon[79930]: pgmap v5115: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:44.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:18:44 smithi083 ceph-mon[77242]: pgmap v5115: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:44.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:18:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:18:44.158Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:18:45.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:18:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:18:44.857Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:18:46.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:18:46 smithi153 ceph-mon[70044]: pgmap v5116: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:46.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:18:46 smithi083 ceph-mon[79930]: pgmap v5116: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:46.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:18:46 smithi083 ceph-mon[77242]: pgmap v5116: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:48.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:18:48 smithi153 ceph-mon[70044]: pgmap v5117: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:48.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:18:48 smithi083 ceph-mon[79930]: pgmap v5117: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:48.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:18:48 smithi083 ceph-mon[77242]: pgmap v5117: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:50.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:18:50 smithi153 ceph-mon[70044]: pgmap v5118: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:50.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:18:50 smithi083 ceph-mon[79930]: pgmap v5118: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:50.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:18:50 smithi083 ceph-mon[77242]: pgmap v5118: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:51.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:18:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:18:50.773Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:18:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:18:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:18:51] "GET /metrics HTTP/1.1" 200 225211 "" "Prometheus/2.33.4" 2022-09-29T08:18:52.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:18:52 smithi153 ceph-mon[70044]: pgmap v5119: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:52.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:18:52 smithi083 ceph-mon[77242]: pgmap v5119: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:52.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:18:52 smithi083 ceph-mon[79930]: pgmap v5119: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:18:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:18:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:18:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:18:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:18:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:18:54.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:18:54 smithi153 ceph-mon[70044]: pgmap v5120: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:54.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:18:54 smithi083 ceph-mon[79930]: pgmap v5120: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:54.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:18:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:18:54.159Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:18:54.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:18:54 smithi083 ceph-mon[77242]: pgmap v5120: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:55.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:18:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:18:54.857Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:18:56.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:18:56 smithi153 ceph-mon[70044]: pgmap v5121: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:56.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:18:56 smithi083 ceph-mon[77242]: pgmap v5121: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:56.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:18:56 smithi083 ceph-mon[79930]: pgmap v5121: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:58.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:18:58 smithi153 ceph-mon[70044]: pgmap v5122: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:58.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:18:58 smithi083 ceph-mon[77242]: pgmap v5122: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:18:58.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:18:58 smithi083 ceph-mon[79930]: pgmap v5122: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:19:00 smithi153 ceph-mon[70044]: pgmap v5123: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:00.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:19:00 smithi083 ceph-mon[79930]: pgmap v5123: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:00.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:19:00 smithi083 ceph-mon[77242]: pgmap v5123: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:01.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:19:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:19:00.774Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:19:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:19:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:19:01] "GET /metrics HTTP/1.1" 200 225212 "" "Prometheus/2.33.4" 2022-09-29T08:19:02.315 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:19:02 smithi153 ceph-mon[70044]: pgmap v5124: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:02.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:19:02 smithi083 ceph-mon[79930]: pgmap v5124: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:02.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:19:02 smithi083 ceph-mon[77242]: pgmap v5124: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:19:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:19:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:19:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:19:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:19:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:19:04.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:19:04 smithi083 ceph-mon[79930]: pgmap v5125: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:04.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:19:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:19:04.159Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:19:04.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:19:04 smithi083 ceph-mon[77242]: pgmap v5125: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:19:04 smithi153 ceph-mon[70044]: pgmap v5125: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:05.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:19:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:19:04.858Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:19:06.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:19:06 smithi083 ceph-mon[79930]: pgmap v5126: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:06.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:19:06 smithi083 ceph-mon[77242]: pgmap v5126: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:06.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:19:06 smithi153 ceph-mon[70044]: pgmap v5126: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:08.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:19:08 smithi083 ceph-mon[79930]: pgmap v5127: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:08.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:19:08 smithi083 ceph-mon[77242]: pgmap v5127: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:08.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:19:08 smithi153 ceph-mon[70044]: pgmap v5127: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:10.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:19:10 smithi083 ceph-mon[79930]: pgmap v5128: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:10.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:19:10 smithi083 ceph-mon[77242]: pgmap v5128: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:19:10 smithi153 ceph-mon[70044]: pgmap v5128: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:11.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:19:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:19:10.775Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:19:11.512 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:19:11.803 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:19:11.803 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 2m ago 2h 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:19:11.803 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 10m ago 2h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:19:11.803 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 2m ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:19:11.804 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 10m ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:19:11.804 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 10m ago 2h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:19:11.804 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 2m ago 2h 662M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:19:11.804 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 2m ago 2h 675M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:19:11.805 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 10m ago 2h 284M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:19:11.805 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 2m ago 2h 394M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:19:11.805 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 2m ago 2h 26.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:19:11.805 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 10m ago 2h 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:19:11.805 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 2m ago 2h 488M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:19:11.806 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 2m ago 2h 479M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:19:11.806 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 2m ago 2h 378M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:19:11.806 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 2m ago 2h 465M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:19:11.806 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 10m ago 2h 392M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:19:11.807 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 10m ago 2h 482M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:19:11.807 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 10m ago 2h 429M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:19:11.807 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 10m ago 2h 503M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:19:11.807 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 10m ago 2h 137M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:19:11.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:19:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:19:11] "GET /metrics HTTP/1.1" 200 225213 "" "Prometheus/2.33.4" 2022-09-29T08:19:12.185 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:19:12.186 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:19:12.186 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:19:12.186 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:19:12.187 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:19:12.187 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:19:12.187 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:19:12.187 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:19:12.188 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:19:12.188 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:19:12.188 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:19:12.188 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:19:12.188 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:19:12.189 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:19:12.189 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:19:12.189 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:19:12.189 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:19:12.190 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:19:12.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:19:12 smithi083 ceph-mon[79930]: pgmap v5129: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:12.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:19:12 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:19:12.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:19:12 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:19:12.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:19:12 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:19:12.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:19:12 smithi083 ceph-mon[77242]: pgmap v5129: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:12.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:19:12 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:19:12.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:19:12 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:19:12.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:19:12 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:19:12.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:19:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:19:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:19:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:19:12 smithi153 ceph-mon[70044]: pgmap v5129: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:19:12 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:19:12.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:19:12 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:19:12.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:19:12 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:19:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:19:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:19:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:19:13.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:19:13 smithi153 ceph-mon[70044]: from='client.40101 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:19:13.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:19:13 smithi153 ceph-mon[70044]: from='client.50041 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:19:13.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:19:13 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/244977279' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:19:13.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:19:13 smithi083 ceph-mon[77242]: from='client.40101 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:19:13.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:19:13 smithi083 ceph-mon[77242]: from='client.50041 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:19:13.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:19:13 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/244977279' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:19:13.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:19:13 smithi083 ceph-mon[79930]: from='client.40101 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:19:13.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:19:13 smithi083 ceph-mon[79930]: from='client.50041 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:19:13.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:19:13 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/244977279' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:19:14.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:19:14 smithi083 ceph-mon[79930]: pgmap v5130: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:14.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:19:14 smithi083 ceph-mon[77242]: pgmap v5130: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:14.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:19:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:19:14.160Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:19:14.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:19:14 smithi153 ceph-mon[70044]: pgmap v5130: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:15.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:19:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:19:14.858Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:19:16.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:19:16 smithi083 ceph-mon[79930]: pgmap v5131: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:16.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:19:16 smithi083 ceph-mon[77242]: pgmap v5131: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:16.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:19:16 smithi153 ceph-mon[70044]: pgmap v5131: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:17.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:19:17 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:19:17.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:19:17 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:19:17.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:19:17 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:19:17.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:19:17 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:19:17.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:19:17 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:19:17.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:19:17 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:19:17.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:19:17 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:19:17.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:19:17 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:19:17.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:19:17 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:19:18.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:19:18 smithi083 ceph-mon[79930]: pgmap v5132: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:18.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:19:18 smithi083 ceph-mon[77242]: pgmap v5132: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:18.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:19:18 smithi153 ceph-mon[70044]: pgmap v5132: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:20.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:19:20 smithi083 ceph-mon[79930]: pgmap v5133: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:20.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:19:20 smithi083 ceph-mon[77242]: pgmap v5133: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:20.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:19:20 smithi153 ceph-mon[70044]: pgmap v5133: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:21.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:19:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:19:20.775Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:19:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:19:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:19:21] "GET /metrics HTTP/1.1" 200 225213 "" "Prometheus/2.33.4" 2022-09-29T08:19:22.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:19:22 smithi083 ceph-mon[77242]: pgmap v5134: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:22.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:19:22 smithi083 ceph-mon[79930]: pgmap v5134: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:22.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:19:22 smithi153 ceph-mon[70044]: pgmap v5134: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:19:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:19:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:19:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:19:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:19:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:19:24.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:19:24 smithi083 ceph-mon[79930]: pgmap v5135: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:24.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:19:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:19:24.160Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:19:24.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:19:24 smithi083 ceph-mon[77242]: pgmap v5135: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:24.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:19:24 smithi153 ceph-mon[70044]: pgmap v5135: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:25.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:19:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:19:24.858Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:19:26.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:19:26 smithi083 ceph-mon[79930]: pgmap v5136: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:26.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:19:26 smithi083 ceph-mon[77242]: pgmap v5136: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:26.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:19:26 smithi153 ceph-mon[70044]: pgmap v5136: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:28.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:19:28 smithi083 ceph-mon[79930]: pgmap v5137: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:28.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:19:28 smithi083 ceph-mon[77242]: pgmap v5137: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:28.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:19:28 smithi153 ceph-mon[70044]: pgmap v5137: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:30.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:19:30 smithi083 ceph-mon[79930]: pgmap v5138: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:30.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:19:30 smithi083 ceph-mon[77242]: pgmap v5138: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:30.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:19:30 smithi153 ceph-mon[70044]: pgmap v5138: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:31.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:19:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:19:30.775Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:19:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:19:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:19:31] "GET /metrics HTTP/1.1" 200 225213 "" "Prometheus/2.33.4" 2022-09-29T08:19:32.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:19:32 smithi083 ceph-mon[79930]: pgmap v5139: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:32.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:19:32 smithi083 ceph-mon[77242]: pgmap v5139: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:32.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:19:32 smithi153 ceph-mon[70044]: pgmap v5139: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:19:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:19:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:19:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:19:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:19:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:19:34.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:19:34 smithi083 ceph-mon[79930]: pgmap v5140: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:34.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:19:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:19:34.161Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:19:34.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:19:34 smithi083 ceph-mon[77242]: pgmap v5140: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:34.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:19:34 smithi153 ceph-mon[70044]: pgmap v5140: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:35.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:19:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:19:34.858Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:19:35.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:19:35 smithi153 ceph-mon[70044]: pgmap v5141: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:35.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:19:35 smithi083 ceph-mon[77242]: pgmap v5141: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:35.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:19:35 smithi083 ceph-mon[79930]: pgmap v5141: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:38.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:19:37 smithi153 ceph-mon[70044]: pgmap v5142: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:38.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:19:37 smithi083 ceph-mon[79930]: pgmap v5142: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:38.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:19:37 smithi083 ceph-mon[77242]: pgmap v5142: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:40.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:19:39 smithi083 ceph-mon[79930]: pgmap v5143: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:40.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:19:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:19:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:19:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:19:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:19:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:19:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:19:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:19:40.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:19:39 smithi083 ceph-mon[77242]: pgmap v5143: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:19:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:19:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:19:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:19:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:19:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:19:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:19:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:19:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:19:39 smithi153 ceph-mon[70044]: pgmap v5143: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:19:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:19:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:19:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:19:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:19:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:19:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:19:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:19:41.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:19:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:19:40.776Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:19:41.908 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:19:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:19:41] "GET /metrics HTTP/1.1" 200 225212 "" "Prometheus/2.33.4" 2022-09-29T08:19:41.908 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:19:41 smithi083 ceph-mon[77242]: pgmap v5144: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:42.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:19:41 smithi083 ceph-mon[79930]: pgmap v5144: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:19:41 smithi153 ceph-mon[70044]: pgmap v5144: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:42.510 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:19:42.798 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:19:42.798 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 2m ago 2h 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:19:42.798 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 26s ago 2h 62.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:19:42.799 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 2m ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:19:42.799 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 26s ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:19:42.799 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 26s ago 2h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:19:42.799 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 2m ago 2h 662M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:19:42.799 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 2m ago 2h 675M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:19:42.800 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 26s ago 2h 298M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:19:42.800 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 2m ago 2h 394M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:19:42.800 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 2m ago 2h 26.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:19:42.800 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 26s ago 2h 25.8M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:19:42.801 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 2m ago 2h 488M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:19:42.801 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 2m ago 2h 479M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:19:42.801 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 2m ago 2h 378M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:19:42.801 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 2m ago 2h 465M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:19:42.802 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 26s ago 2h 414M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:19:42.802 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 26s ago 2h 510M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:19:42.802 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 26s ago 2h 454M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:19:42.802 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 26s ago 2h 532M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:19:42.802 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 26s ago 2h 141M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:19:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:19:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:19:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:19:43.156 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:19:43.156 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:19:43.157 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:19:43.157 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:19:43.157 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:19:43.157 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:19:43.158 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:19:43.158 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:19:43.158 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:19:43.158 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:19:43.159 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:19:43.159 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:19:43.159 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:19:43.159 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:19:43.160 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:19:43.160 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:19:43.160 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:19:43.160 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:19:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:19:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:19:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:19:44.162 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:19:43 smithi083 ceph-mon[79930]: from='client.40119 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:19:44.163 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:19:43 smithi083 ceph-mon[79930]: pgmap v5145: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:44.163 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:19:43 smithi083 ceph-mon[79930]: from='client.40125 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:19:44.163 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:19:43 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/425043017' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:19:44.164 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:19:43 smithi083 ceph-mon[77242]: from='client.40119 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:19:44.164 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:19:43 smithi083 ceph-mon[77242]: pgmap v5145: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:44.164 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:19:43 smithi083 ceph-mon[77242]: from='client.40125 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:19:44.164 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:19:43 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/425043017' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:19:44.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:19:43 smithi153 ceph-mon[70044]: from='client.40119 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:19:44.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:19:43 smithi153 ceph-mon[70044]: pgmap v5145: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:44.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:19:43 smithi153 ceph-mon[70044]: from='client.40125 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:19:44.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:19:43 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/425043017' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:19:44.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:19:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:19:44.161Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:19:45.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:19:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:19:44.858Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:19:46.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:19:45 smithi083 ceph-mon[79930]: pgmap v5146: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:46.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:19:45 smithi083 ceph-mon[77242]: pgmap v5146: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:46.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:19:45 smithi153 ceph-mon[70044]: pgmap v5146: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:48.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:19:47 smithi083 ceph-mon[79930]: pgmap v5147: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:48.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:19:47 smithi083 ceph-mon[77242]: pgmap v5147: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:48.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:19:47 smithi153 ceph-mon[70044]: pgmap v5147: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:50.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:19:49 smithi083 ceph-mon[79930]: pgmap v5148: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:50.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:19:49 smithi083 ceph-mon[77242]: pgmap v5148: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:50.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:19:49 smithi153 ceph-mon[70044]: pgmap v5148: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:51.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:19:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:19:50.777Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:19:51.929 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:19:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:19:51] "GET /metrics HTTP/1.1" 200 225212 "" "Prometheus/2.33.4" 2022-09-29T08:19:52.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:19:51 smithi083 ceph-mon[79930]: pgmap v5149: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:52.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:19:51 smithi083 ceph-mon[77242]: pgmap v5149: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:52.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:19:51 smithi153 ceph-mon[70044]: pgmap v5149: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:19:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:19:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:19:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:19:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:19:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:19:54.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:19:53 smithi083 ceph-mon[77242]: pgmap v5150: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:54.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:19:53 smithi083 ceph-mon[79930]: pgmap v5150: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:54.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:19:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:19:54.163Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:19:54.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:19:53 smithi153 ceph-mon[70044]: pgmap v5150: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:55.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:19:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:19:54.859Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:19:56.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:19:55 smithi083 ceph-mon[79930]: pgmap v5151: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:56.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:19:55 smithi083 ceph-mon[77242]: pgmap v5151: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:56.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:19:55 smithi153 ceph-mon[70044]: pgmap v5151: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:58.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:19:57 smithi083 ceph-mon[79930]: pgmap v5152: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:58.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:19:57 smithi083 ceph-mon[77242]: pgmap v5152: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:19:58.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:19:57 smithi153 ceph-mon[70044]: pgmap v5152: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:00.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:19:59 smithi083 ceph-mon[79930]: pgmap v5153: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:00.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:19:59 smithi083 ceph-mon[77242]: pgmap v5153: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:00.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:20:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[77219]: 2022-09-29T08:19:59.999+0000 7f160f665700 -1 log_channel(cluster) log [ERR] : overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T08:20:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:19:59 smithi153 ceph-mon[70044]: pgmap v5153: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:01.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:20:00 smithi083 ceph-mon[79930]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T08:20:01.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:20:00 smithi083 ceph-mon[77242]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T08:20:01.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:20:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:20:00.778Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:20:01.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:20:00 smithi153 ceph-mon[70044]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T08:20:01.949 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:20:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:20:01] "GET /metrics HTTP/1.1" 200 225211 "" "Prometheus/2.33.4" 2022-09-29T08:20:02.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:20:01 smithi083 ceph-mon[79930]: pgmap v5154: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:02.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:20:01 smithi083 ceph-mon[77242]: pgmap v5154: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:02.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:20:01 smithi153 ceph-mon[70044]: pgmap v5154: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:20:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:20:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:20:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:20:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:20:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:20:04.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:20:03 smithi083 ceph-mon[79930]: pgmap v5155: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:04.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:20:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:20:04.164Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:20:04.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:20:03 smithi083 ceph-mon[77242]: pgmap v5155: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:04.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:20:03 smithi153 ceph-mon[70044]: pgmap v5155: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:05.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:20:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:20:04.859Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:20:06.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:20:05 smithi083 ceph-mon[79930]: pgmap v5156: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:06.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:20:05 smithi083 ceph-mon[77242]: pgmap v5156: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:06.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:20:05 smithi153 ceph-mon[70044]: pgmap v5156: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:08.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:20:07 smithi083 ceph-mon[79930]: pgmap v5157: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:08.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:20:07 smithi083 ceph-mon[77242]: pgmap v5157: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:08.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:20:07 smithi153 ceph-mon[70044]: pgmap v5157: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:10.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:20:09 smithi153 ceph-mon[70044]: pgmap v5158: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:10.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:20:09 smithi083 ceph-mon[79930]: pgmap v5158: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:10.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:20:09 smithi083 ceph-mon[77242]: pgmap v5158: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:11.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:20:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:20:10.778Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:20:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:20:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:20:11] "GET /metrics HTTP/1.1" 200 225212 "" "Prometheus/2.33.4" 2022-09-29T08:20:12.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:20:11 smithi153 ceph-mon[70044]: pgmap v5159: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:12.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:20:11 smithi083 ceph-mon[79930]: pgmap v5159: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:12.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:20:11 smithi083 ceph-mon[77242]: pgmap v5159: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:20:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:20:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:20:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:20:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:20:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:20:13.491 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:20:13.789 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:20:13.789 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 3m ago 2h 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:20:13.789 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 57s ago 2h 62.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:20:13.790 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 3m ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:20:13.790 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 57s ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:20:13.790 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 57s ago 2h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:20:13.790 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 3m ago 2h 662M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:20:13.791 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 3m ago 2h 675M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:20:13.791 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 57s ago 2h 298M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:20:13.791 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 3m ago 2h 394M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:20:13.791 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 3m ago 2h 26.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:20:13.792 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 57s ago 2h 25.8M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:20:13.792 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 3m ago 2h 488M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:20:13.792 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 3m ago 2h 479M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:20:13.792 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 3m ago 2h 378M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:20:13.793 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 3m ago 2h 465M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:20:13.793 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 57s ago 2h 414M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:20:13.793 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 57s ago 2h 510M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:20:13.793 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 57s ago 2h 454M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:20:13.794 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 57s ago 2h 532M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:20:13.794 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 57s ago 2h 141M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:20:14.153 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:20:14.153 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:20:14.153 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:20:14.153 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:20:14.154 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:20:14.154 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:20:14.154 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:20:14.154 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:20:14.155 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:20:14.155 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:20:14.155 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:20:14.155 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:20:14.156 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:20:14.156 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:20:14.156 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:20:14.156 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:20:14.156 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:20:14.157 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:20:14.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:20:13 smithi153 ceph-mon[70044]: pgmap v5160: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:14.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:20:13 smithi083 ceph-mon[79930]: pgmap v5160: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:14.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:20:13 smithi083 ceph-mon[77242]: pgmap v5160: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:14.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:20:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:20:14.163Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:20:15.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:20:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:20:14.861Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:20:15.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:20:14 smithi083 ceph-mon[77242]: from='client.40137 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:20:15.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:20:14 smithi083 ceph-mon[77242]: from='client.40143 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:20:15.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:20:14 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/456625696' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:20:15.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:20:14 smithi083 ceph-mon[79930]: from='client.40137 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:20:15.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:20:14 smithi083 ceph-mon[79930]: from='client.40143 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:20:15.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:20:14 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/456625696' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:20:15.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:20:14 smithi153 ceph-mon[70044]: from='client.40137 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:20:15.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:20:14 smithi153 ceph-mon[70044]: from='client.40143 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:20:15.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:20:14 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/456625696' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:20:16.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:20:15 smithi153 ceph-mon[70044]: pgmap v5161: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:16.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:20:15 smithi083 ceph-mon[79930]: pgmap v5161: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:16.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:20:15 smithi083 ceph-mon[77242]: pgmap v5161: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:18.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:20:17 smithi083 ceph-mon[77242]: pgmap v5162: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:18.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:20:17 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:20:18.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:20:17 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:20:18.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:20:17 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:20:18.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:20:17 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:20:18.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:20:17 smithi083 ceph-mon[79930]: pgmap v5162: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:18.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:20:17 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:20:18.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:20:17 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:20:18.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:20:17 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:20:18.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:20:17 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:20:18.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:20:17 smithi153 ceph-mon[70044]: pgmap v5162: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:18.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:20:17 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:20:18.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:20:17 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:20:18.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:20:17 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:20:18.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:20:17 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:20:20.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:20:19 smithi083 ceph-mon[79930]: pgmap v5163: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:20.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:20:19 smithi083 ceph-mon[77242]: pgmap v5163: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:20:19 smithi153 ceph-mon[70044]: pgmap v5163: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:21.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:20:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:20:20.778Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:20:21.918 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:20:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:20:21] "GET /metrics HTTP/1.1" 200 225212 "" "Prometheus/2.33.4" 2022-09-29T08:20:22.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:20:21 smithi083 ceph-mon[77242]: pgmap v5164: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:22.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:20:21 smithi083 ceph-mon[79930]: pgmap v5164: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:22.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:20:21 smithi153 ceph-mon[70044]: pgmap v5164: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:20:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:20:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:20:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:20:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:20:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:20:24.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:20:23 smithi083 ceph-mon[77242]: pgmap v5165: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:24.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:20:23 smithi083 ceph-mon[79930]: pgmap v5165: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:24.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:20:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:20:24.165Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:20:24.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:20:23 smithi153 ceph-mon[70044]: pgmap v5165: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:25.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:20:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:20:24.861Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:20:26.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:20:25 smithi083 ceph-mon[79930]: pgmap v5166: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:26.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:20:25 smithi083 ceph-mon[77242]: pgmap v5166: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:26.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:20:25 smithi153 ceph-mon[70044]: pgmap v5166: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:28.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:20:27 smithi083 ceph-mon[77242]: pgmap v5167: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:28.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:20:27 smithi083 ceph-mon[79930]: pgmap v5167: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:28.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:20:27 smithi153 ceph-mon[70044]: pgmap v5167: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:30.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:20:29 smithi083 ceph-mon[77242]: pgmap v5168: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:30.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:20:29 smithi083 ceph-mon[79930]: pgmap v5168: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:30.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:20:29 smithi153 ceph-mon[70044]: pgmap v5168: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:31.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:20:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:20:30.779Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:20:31.935 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:20:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:20:31] "GET /metrics HTTP/1.1" 200 225213 "" "Prometheus/2.33.4" 2022-09-29T08:20:32.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:20:31 smithi083 ceph-mon[79930]: pgmap v5169: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:32.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:20:31 smithi083 ceph-mon[77242]: pgmap v5169: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:32.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:20:31 smithi153 ceph-mon[70044]: pgmap v5169: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:20:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:20:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:20:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:20:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:20:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:20:34.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:20:33 smithi083 ceph-mon[79930]: pgmap v5170: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:34.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:20:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:20:34.166Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:20:34.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:20:33 smithi083 ceph-mon[77242]: pgmap v5170: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:34.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:20:33 smithi153 ceph-mon[70044]: pgmap v5170: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:35.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:20:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:20:34.862Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:20:36.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:20:35 smithi083 ceph-mon[79930]: pgmap v5171: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:36.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:20:35 smithi083 ceph-mon[77242]: pgmap v5171: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:36.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:20:35 smithi153 ceph-mon[70044]: pgmap v5171: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:38.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:20:37 smithi083 ceph-mon[79930]: pgmap v5172: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:38.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:20:37 smithi083 ceph-mon[77242]: pgmap v5172: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:38.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:20:37 smithi153 ceph-mon[70044]: pgmap v5172: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:40.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:20:39 smithi083 ceph-mon[79930]: pgmap v5173: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:40.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:20:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:20:40.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:20:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:20:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:20:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:20:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:20:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:20:40.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:20:39 smithi083 ceph-mon[77242]: pgmap v5173: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:20:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:20:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:20:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:20:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:20:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:20:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:20:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:20:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:20:39 smithi153 ceph-mon[70044]: pgmap v5173: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:20:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:20:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:20:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:20:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:20:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:20:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:20:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:20:41.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:20:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:20:40.779Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:20:41.951 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:20:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:20:41] "GET /metrics HTTP/1.1" 200 225214 "" "Prometheus/2.33.4" 2022-09-29T08:20:42.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:20:41 smithi083 ceph-mon[79930]: pgmap v5174: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:42.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:20:41 smithi083 ceph-mon[77242]: pgmap v5174: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:20:41 smithi153 ceph-mon[70044]: pgmap v5174: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:20:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:20:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:20:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:20:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:20:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:20:44.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:20:43 smithi083 ceph-mon[79930]: pgmap v5175: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:44.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:20:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:20:44.167Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:20:44.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:20:43 smithi083 ceph-mon[77242]: pgmap v5175: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:44.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:20:43 smithi153 ceph-mon[70044]: pgmap v5175: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:44.482 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:20:44.767 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:20:44.768 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 3m ago 2h 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:20:44.768 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 88s ago 2h 62.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:20:44.768 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 3m ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:20:44.768 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 88s ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:20:44.769 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 88s ago 2h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:20:44.769 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 3m ago 2h 662M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:20:44.769 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 3m ago 2h 675M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:20:44.770 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 88s ago 2h 298M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:20:44.770 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 3m ago 2h 394M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:20:44.770 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 3m ago 2h 26.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:20:44.770 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 88s ago 2h 25.8M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:20:44.770 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 3m ago 2h 488M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:20:44.771 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 3m ago 2h 479M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:20:44.771 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 3m ago 2h 378M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:20:44.771 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 3m ago 2h 465M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:20:44.771 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 88s ago 2h 414M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:20:44.772 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 88s ago 2h 510M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:20:44.772 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 88s ago 2h 454M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:20:44.772 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 88s ago 2h 532M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:20:44.772 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 88s ago 2h 141M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:20:45.126 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:20:45.126 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:20:45.126 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:20:45.127 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:20:45.127 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:20:45.127 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:20:45.127 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:20:45.128 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:20:45.128 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:20:45.128 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:20:45.128 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:20:45.129 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:20:45.129 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:20:45.129 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:20:45.129 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:20:45.130 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:20:45.130 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:20:45.130 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:20:45.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:20:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:20:44.861Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:20:46.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:20:45 smithi083 ceph-mon[79930]: from='client.40155 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:20:46.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:20:45 smithi083 ceph-mon[79930]: pgmap v5176: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:46.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:20:45 smithi083 ceph-mon[79930]: from='client.40161 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:20:46.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:20:45 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3595598815' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:20:46.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:20:45 smithi083 ceph-mon[77242]: from='client.40155 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:20:46.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:20:45 smithi083 ceph-mon[77242]: pgmap v5176: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:46.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:20:45 smithi083 ceph-mon[77242]: from='client.40161 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:20:46.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:20:45 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3595598815' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:20:46.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:20:45 smithi153 ceph-mon[70044]: from='client.40155 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:20:46.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:20:45 smithi153 ceph-mon[70044]: pgmap v5176: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:46.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:20:45 smithi153 ceph-mon[70044]: from='client.40161 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:20:46.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:20:45 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3595598815' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:20:48.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:20:47 smithi083 ceph-mon[79930]: pgmap v5177: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:48.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:20:47 smithi083 ceph-mon[77242]: pgmap v5177: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:48.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:20:47 smithi153 ceph-mon[70044]: pgmap v5177: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:50.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:20:49 smithi153 ceph-mon[70044]: pgmap v5178: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:50.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:20:49 smithi083 ceph-mon[79930]: pgmap v5178: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:50.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:20:49 smithi083 ceph-mon[77242]: pgmap v5178: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:51.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:20:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:20:50.780Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:20:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:20:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:20:51] "GET /metrics HTTP/1.1" 200 225214 "" "Prometheus/2.33.4" 2022-09-29T08:20:52.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:20:51 smithi153 ceph-mon[70044]: pgmap v5179: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:52.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:20:51 smithi083 ceph-mon[77242]: pgmap v5179: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:52.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:20:51 smithi083 ceph-mon[79930]: pgmap v5179: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:20:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:20:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:20:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:20:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:20:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:20:54.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:20:53 smithi153 ceph-mon[70044]: pgmap v5180: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:54.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:20:53 smithi083 ceph-mon[79930]: pgmap v5180: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:54.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:20:53 smithi083 ceph-mon[77242]: pgmap v5180: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:54.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:20:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:20:54.167Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:20:55.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:20:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:20:54.862Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:20:56.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:20:55 smithi153 ceph-mon[70044]: pgmap v5181: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:56.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:20:55 smithi083 ceph-mon[79930]: pgmap v5181: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:56.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:20:55 smithi083 ceph-mon[77242]: pgmap v5181: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:58.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:20:57 smithi153 ceph-mon[70044]: pgmap v5182: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:58.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:20:57 smithi083 ceph-mon[77242]: pgmap v5182: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:20:58.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:20:57 smithi083 ceph-mon[79930]: pgmap v5182: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:20:59 smithi153 ceph-mon[70044]: pgmap v5183: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:00.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:20:59 smithi083 ceph-mon[77242]: pgmap v5183: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:00.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:20:59 smithi083 ceph-mon[79930]: pgmap v5183: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:01.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:21:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:21:00.781Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:21:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:21:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:21:01] "GET /metrics HTTP/1.1" 200 225210 "" "Prometheus/2.33.4" 2022-09-29T08:21:02.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:21:01 smithi153 ceph-mon[70044]: pgmap v5184: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:02.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:21:01 smithi083 ceph-mon[77242]: pgmap v5184: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:02.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:21:01 smithi083 ceph-mon[79930]: pgmap v5184: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:21:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:21:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:21:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:21:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:21:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:21:04.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:21:03 smithi153 ceph-mon[70044]: pgmap v5185: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:04.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:21:03 smithi083 ceph-mon[79930]: pgmap v5185: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:04.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:21:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:21:04.167Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:21:04.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:21:03 smithi083 ceph-mon[77242]: pgmap v5185: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:05.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:21:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:21:04.863Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:21:06.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:21:05 smithi153 ceph-mon[70044]: pgmap v5186: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:06.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:21:05 smithi083 ceph-mon[79930]: pgmap v5186: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:06.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:21:05 smithi083 ceph-mon[77242]: pgmap v5186: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:08.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:21:07 smithi153 ceph-mon[70044]: pgmap v5187: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:08.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:21:08 smithi083 ceph-mon[79930]: pgmap v5187: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:08.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:21:07 smithi083 ceph-mon[77242]: pgmap v5187: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:10.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:21:10 smithi153 ceph-mon[70044]: pgmap v5188: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:10.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:21:10 smithi083 ceph-mon[79930]: pgmap v5188: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:10.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:21:10 smithi083 ceph-mon[77242]: pgmap v5188: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:11.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:21:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:21:10.782Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:21:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:21:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:21:11] "GET /metrics HTTP/1.1" 200 225210 "" "Prometheus/2.33.4" 2022-09-29T08:21:12.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:21:12 smithi153 ceph-mon[70044]: pgmap v5189: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:12.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:21:12 smithi083 ceph-mon[79930]: pgmap v5189: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:12.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:21:12 smithi083 ceph-mon[77242]: pgmap v5189: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:21:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:21:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:21:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:21:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:21:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:21:14.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:21:14 smithi153 ceph-mon[70044]: pgmap v5190: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:14.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:21:14 smithi083 ceph-mon[79930]: pgmap v5190: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:14.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:21:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:21:14.169Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:21:14.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:21:14 smithi083 ceph-mon[77242]: pgmap v5190: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:15.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:21:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:21:14.864Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:21:15.453 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:21:15.741 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:21:15.742 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 4m ago 2h 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:21:15.742 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 119s ago 2h 62.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:21:15.742 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 4m ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:21:15.743 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 119s ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:21:15.743 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 119s ago 2h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:21:15.743 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 4m ago 3h 662M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:21:15.743 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 4m ago 3h 675M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:21:15.743 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 119s ago 2h 298M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:21:15.744 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 4m ago 2h 394M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:21:15.744 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 4m ago 2h 26.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:21:15.744 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 119s ago 2h 25.8M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:21:15.744 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 4m ago 2h 488M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:21:15.745 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 4m ago 2h 479M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:21:15.745 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 4m ago 2h 378M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:21:15.745 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 4m ago 2h 465M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:21:15.745 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 119s ago 2h 414M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:21:15.746 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 119s ago 2h 510M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:21:15.746 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 119s ago 2h 454M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:21:15.746 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 119s ago 2h 532M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:21:15.746 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 119s ago 2h 141M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:21:16.101 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:21:16.101 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:21:16.102 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:21:16.102 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:21:16.102 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:21:16.102 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:21:16.103 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:21:16.103 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:21:16.103 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:21:16.103 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:21:16.104 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:21:16.104 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:21:16.104 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:21:16.104 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:21:16.104 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:21:16.105 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:21:16.105 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:21:16.105 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:21:16.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:21:16 smithi153 ceph-mon[70044]: pgmap v5191: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:16.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:21:16 smithi083 ceph-mon[77242]: pgmap v5191: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:16.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:21:16 smithi083 ceph-mon[79930]: pgmap v5191: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:17.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:21:17 smithi153 ceph-mon[70044]: from='client.50104 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:21:17.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:21:17 smithi153 ceph-mon[70044]: from='client.40179 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:21:17.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:21:17 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2282900897' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:21:17.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:21:17 smithi083 ceph-mon[79930]: from='client.50104 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:21:17.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:21:17 smithi083 ceph-mon[79930]: from='client.40179 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:21:17.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:21:17 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2282900897' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:21:17.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:21:17 smithi083 ceph-mon[77242]: from='client.50104 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:21:17.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:21:17 smithi083 ceph-mon[77242]: from='client.40179 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:21:17.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:21:17 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2282900897' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:21:18.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:21:18 smithi153 ceph-mon[70044]: pgmap v5192: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:18.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:21:18 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:21:18.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:21:18 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:21:18.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:21:18 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:21:18.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:21:18 smithi083 ceph-mon[79930]: pgmap v5192: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:18.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:21:18 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:21:18.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:21:18 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:21:18.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:21:18 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:21:18.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:21:18 smithi083 ceph-mon[77242]: pgmap v5192: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:18.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:21:18 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:21:18.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:21:18 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:21:18.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:21:18 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:21:19.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:21:19 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:21:19.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:21:19 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:21:19.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:21:19 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:21:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:21:20 smithi153 ceph-mon[70044]: pgmap v5193: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:20.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:21:20 smithi083 ceph-mon[77242]: pgmap v5193: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:20.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:21:20 smithi083 ceph-mon[79930]: pgmap v5193: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:21.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:21:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:21:20.782Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:21:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:21:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:21:21] "GET /metrics HTTP/1.1" 200 225210 "" "Prometheus/2.33.4" 2022-09-29T08:21:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:21:22 smithi153 ceph-mon[70044]: pgmap v5194: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:22.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:21:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:21:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:21:22.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:21:22 smithi083 ceph-mon[79930]: pgmap v5194: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:22.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:21:22 smithi083 ceph-mon[77242]: pgmap v5194: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:21:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:21:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:21:24.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:21:24 smithi083 ceph-mon[79930]: pgmap v5195: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:24.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:21:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:21:24.169Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:21:24.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:21:24 smithi083 ceph-mon[77242]: pgmap v5195: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:21:24 smithi153 ceph-mon[70044]: pgmap v5195: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:25.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:21:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:21:24.865Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:21:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:21:26 smithi153 ceph-mon[70044]: pgmap v5196: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:26.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:21:26 smithi083 ceph-mon[77242]: pgmap v5196: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:26.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:21:26 smithi083 ceph-mon[79930]: pgmap v5196: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:21:28 smithi153 ceph-mon[70044]: pgmap v5197: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:28.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:21:28 smithi083 ceph-mon[77242]: pgmap v5197: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:28.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:21:28 smithi083 ceph-mon[79930]: pgmap v5197: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:21:30 smithi153 ceph-mon[70044]: pgmap v5198: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:30.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:21:30 smithi083 ceph-mon[79930]: pgmap v5198: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:30.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:21:30 smithi083 ceph-mon[77242]: pgmap v5198: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:31.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:21:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:21:30.782Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:21:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:21:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:21:31] "GET /metrics HTTP/1.1" 200 225215 "" "Prometheus/2.33.4" 2022-09-29T08:21:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:21:32 smithi153 ceph-mon[70044]: pgmap v5199: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:32.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:21:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:21:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:21:32.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:21:32 smithi083 ceph-mon[79930]: pgmap v5199: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:32.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:21:32 smithi083 ceph-mon[77242]: pgmap v5199: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:21:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:21:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:21:34.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:21:34 smithi083 ceph-mon[79930]: pgmap v5200: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:34.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:21:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:21:34.170Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:21:34.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:21:34 smithi083 ceph-mon[77242]: pgmap v5200: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:21:34 smithi153 ceph-mon[70044]: pgmap v5200: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:35.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:21:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:21:34.866Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:21:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:21:36 smithi153 ceph-mon[70044]: pgmap v5201: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:36.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:21:36 smithi083 ceph-mon[79930]: pgmap v5201: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:36.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:21:36 smithi083 ceph-mon[77242]: pgmap v5201: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:21:38 smithi153 ceph-mon[70044]: pgmap v5202: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:38.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:21:38 smithi083 ceph-mon[79930]: pgmap v5202: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:38.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:21:38 smithi083 ceph-mon[77242]: pgmap v5202: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:21:40 smithi153 ceph-mon[70044]: pgmap v5203: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:21:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:21:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:21:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:21:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:21:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:21:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:21:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:21:40.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:21:40 smithi083 ceph-mon[79930]: pgmap v5203: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:40.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:21:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:21:40.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:21:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:21:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:21:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:21:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:21:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:21:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:21:40 smithi083 ceph-mon[77242]: pgmap v5203: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:21:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:21:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:21:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:21:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:21:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:21:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:21:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:21:41.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:21:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:21:40.783Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:21:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:21:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:21:41] "GET /metrics HTTP/1.1" 200 225213 "" "Prometheus/2.33.4" 2022-09-29T08:21:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:21:42 smithi153 ceph-mon[70044]: pgmap v5204: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:42.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:21:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:21:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:21:42.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:21:42 smithi083 ceph-mon[79930]: pgmap v5204: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:42.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:21:42 smithi083 ceph-mon[77242]: pgmap v5204: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:21:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:21:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:21:44.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:21:44 smithi083 ceph-mon[79930]: pgmap v5205: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:44.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:21:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:21:44.172Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:21:44.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:21:44 smithi083 ceph-mon[77242]: pgmap v5205: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:44.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:21:44 smithi153 ceph-mon[70044]: pgmap v5205: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:45.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:21:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:21:44.866Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:21:46.433 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:21:46.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:21:46 smithi153 ceph-mon[70044]: pgmap v5206: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:46.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:21:46 smithi083 ceph-mon[79930]: pgmap v5206: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:46.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:21:46 smithi083 ceph-mon[77242]: pgmap v5206: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:46.724 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:21:46.724 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 4m ago 2h 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:21:46.724 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 2m ago 2h 62.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:21:46.725 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 4m ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:21:46.725 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 2m ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:21:46.725 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 2m ago 2h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:21:46.725 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 4m ago 3h 662M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:21:46.726 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 4m ago 3h 675M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:21:46.726 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 2m ago 2h 298M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:21:46.726 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 4m ago 2h 394M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:21:46.726 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 4m ago 2h 26.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:21:46.726 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 2m ago 2h 25.8M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:21:46.727 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 4m ago 2h 488M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:21:46.727 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 4m ago 2h 479M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:21:46.727 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 4m ago 2h 378M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:21:46.727 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 4m ago 2h 465M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:21:46.728 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 2m ago 2h 414M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:21:46.728 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 2m ago 2h 510M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:21:46.728 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 2m ago 2h 454M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:21:46.728 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 2m ago 2h 532M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:21:46.729 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 2m ago 2h 141M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:21:47.082 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:21:47.082 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:21:47.082 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:21:47.083 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:21:47.083 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:21:47.083 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:21:47.083 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:21:47.084 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:21:47.084 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:21:47.084 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:21:47.084 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:21:47.085 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:21:47.085 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:21:47.085 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:21:47.085 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:21:47.086 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:21:47.086 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:21:47.086 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:21:48.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:21:47 smithi083 ceph-mon[79930]: from='client.40191 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:21:48.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:21:47 smithi083 ceph-mon[79930]: from='client.40197 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:21:48.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:21:47 smithi083 ceph-mon[79930]: pgmap v5207: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:48.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:21:47 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2501493202' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:21:48.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:21:47 smithi083 ceph-mon[77242]: from='client.40191 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:21:48.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:21:47 smithi083 ceph-mon[77242]: from='client.40197 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:21:48.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:21:47 smithi083 ceph-mon[77242]: pgmap v5207: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:48.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:21:47 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2501493202' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:21:48.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:21:47 smithi153 ceph-mon[70044]: from='client.40191 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:21:48.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:21:47 smithi153 ceph-mon[70044]: from='client.40197 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:21:48.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:21:47 smithi153 ceph-mon[70044]: pgmap v5207: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:48.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:21:47 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2501493202' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:21:50.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:21:49 smithi083 ceph-mon[79930]: pgmap v5208: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:50.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:21:49 smithi083 ceph-mon[77242]: pgmap v5208: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:50.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:21:49 smithi153 ceph-mon[70044]: pgmap v5208: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:51.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:21:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:21:50.784Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:21:51.936 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:21:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:21:51] "GET /metrics HTTP/1.1" 200 225213 "" "Prometheus/2.33.4" 2022-09-29T08:21:52.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:21:51 smithi083 ceph-mon[79930]: pgmap v5209: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:52.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:21:51 smithi083 ceph-mon[77242]: pgmap v5209: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:52.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:21:51 smithi153 ceph-mon[70044]: pgmap v5209: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:21:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:21:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:21:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:21:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:21:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:21:54.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:21:53 smithi083 ceph-mon[77242]: pgmap v5210: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:54.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:21:53 smithi083 ceph-mon[79930]: pgmap v5210: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:54.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:21:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:21:54.171Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:21:54.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:21:53 smithi153 ceph-mon[70044]: pgmap v5210: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:55.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:21:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:21:54.866Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:21:56.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:21:55 smithi083 ceph-mon[79930]: pgmap v5211: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:56.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:21:55 smithi083 ceph-mon[77242]: pgmap v5211: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:56.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:21:55 smithi153 ceph-mon[70044]: pgmap v5211: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:58.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:21:57 smithi083 ceph-mon[79930]: pgmap v5212: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:58.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:21:57 smithi083 ceph-mon[77242]: pgmap v5212: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:21:58.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:21:57 smithi153 ceph-mon[70044]: pgmap v5212: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:00.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:21:59 smithi083 ceph-mon[79930]: pgmap v5213: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:00.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:21:59 smithi083 ceph-mon[77242]: pgmap v5213: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:21:59 smithi153 ceph-mon[70044]: pgmap v5213: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:01.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:22:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:22:00.785Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:22:01.952 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:22:01] "GET /metrics HTTP/1.1" 200 225210 "" "Prometheus/2.33.4" 2022-09-29T08:22:02.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:22:01 smithi083 ceph-mon[79930]: pgmap v5214: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:02.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:22:01 smithi083 ceph-mon[77242]: pgmap v5214: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:02.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:22:01 smithi153 ceph-mon[70044]: pgmap v5214: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:22:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:22:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:22:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:22:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:22:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:22:04.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:22:03 smithi083 ceph-mon[79930]: pgmap v5215: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:04.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:22:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:22:04.172Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:22:04.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:22:03 smithi083 ceph-mon[77242]: pgmap v5215: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:04.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:22:03 smithi153 ceph-mon[70044]: pgmap v5215: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:05.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:22:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:22:04.867Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:22:06.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:22:05 smithi083 ceph-mon[79930]: pgmap v5216: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:06.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:22:05 smithi083 ceph-mon[77242]: pgmap v5216: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:06.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:22:05 smithi153 ceph-mon[70044]: pgmap v5216: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:08.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:22:07 smithi083 ceph-mon[79930]: pgmap v5217: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:08.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:22:07 smithi083 ceph-mon[77242]: pgmap v5217: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:08.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:22:07 smithi153 ceph-mon[70044]: pgmap v5217: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:10.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:22:09 smithi153 ceph-mon[70044]: pgmap v5218: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:10.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:22:09 smithi083 ceph-mon[77242]: pgmap v5218: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:10.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:22:09 smithi083 ceph-mon[79930]: pgmap v5218: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:11.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:22:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:22:10.786Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:22:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:22:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:22:11] "GET /metrics HTTP/1.1" 200 225217 "" "Prometheus/2.33.4" 2022-09-29T08:22:12.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:22:11 smithi153 ceph-mon[70044]: pgmap v5219: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:12.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:22:11 smithi083 ceph-mon[79930]: pgmap v5219: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:12.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:22:11 smithi083 ceph-mon[77242]: pgmap v5219: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:22:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:22:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:22:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:22:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:22:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:22:14.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:22:13 smithi153 ceph-mon[70044]: pgmap v5220: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:14.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:22:13 smithi083 ceph-mon[79930]: pgmap v5220: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:14.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:22:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:22:14.172Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:22:14.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:22:13 smithi083 ceph-mon[77242]: pgmap v5220: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:15.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:22:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:22:14.868Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:22:16.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:22:15 smithi153 ceph-mon[70044]: pgmap v5221: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:16.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:22:15 smithi083 ceph-mon[79930]: pgmap v5221: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:16.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:22:15 smithi083 ceph-mon[77242]: pgmap v5221: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:17.406 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:22:17.693 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:22:17.694 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 5m ago 2h 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:22:17.694 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 3m ago 2h 62.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:22:17.694 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 5m ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:22:17.694 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 3m ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:22:17.695 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 3m ago 2h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:22:17.695 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 5m ago 3h 662M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:22:17.695 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 5m ago 3h 675M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:22:17.695 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 3m ago 3h 298M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:22:17.696 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 5m ago 3h 394M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:22:17.696 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 5m ago 2h 26.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:22:17.696 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 3m ago 2h 25.8M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:22:17.696 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 5m ago 2h 488M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:22:17.697 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 5m ago 2h 479M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:22:17.697 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 5m ago 2h 378M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:22:17.697 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 5m ago 2h 465M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:22:17.697 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 3m ago 2h 414M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:22:17.698 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 3m ago 2h 510M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:22:17.698 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 3m ago 2h 454M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:22:17.698 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 3m ago 2h 532M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:22:17.698 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 3m ago 2h 141M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:22:18.048 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:22:18.048 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:22:18.049 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:22:18.049 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:22:18.049 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:22:18.049 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:22:18.050 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:22:18.050 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:22:18.050 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:22:18.050 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:22:18.051 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:22:18.051 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:22:18.051 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:22:18.051 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:22:18.052 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:22:18.052 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:22:18.052 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:22:18.052 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:22:18.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:22:17 smithi153 ceph-mon[70044]: pgmap v5222: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:18.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:22:17 smithi153 ceph-mon[70044]: from='client.50140 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:22:18.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:22:17 smithi083 ceph-mon[77242]: pgmap v5222: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:18.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:22:17 smithi083 ceph-mon[77242]: from='client.50140 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:22:18.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:22:17 smithi083 ceph-mon[79930]: pgmap v5222: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:18.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:22:17 smithi083 ceph-mon[79930]: from='client.50140 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:22:19.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:22:18 smithi153 ceph-mon[70044]: from='client.50146 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:22:19.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:22:18 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1558897558' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:22:19.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:22:18 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:22:19.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:22:18 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:22:19.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:22:18 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:22:19.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:22:18 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:22:19.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:22:18 smithi083 ceph-mon[79930]: from='client.50146 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:22:19.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:22:18 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1558897558' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:22:19.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:22:18 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:22:19.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:22:18 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:22:19.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:22:18 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:22:19.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:22:18 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:22:19.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:22:18 smithi083 ceph-mon[77242]: from='client.50146 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:22:19.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:22:18 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1558897558' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:22:19.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:22:18 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:22:19.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:22:18 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:22:19.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:22:18 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:22:19.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:22:18 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:22:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:22:19 smithi153 ceph-mon[70044]: pgmap v5223: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:20.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:22:19 smithi083 ceph-mon[79930]: pgmap v5223: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:20.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:22:19 smithi083 ceph-mon[77242]: pgmap v5223: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:21.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:22:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:22:20.787Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:22:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:22:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:22:21] "GET /metrics HTTP/1.1" 200 225217 "" "Prometheus/2.33.4" 2022-09-29T08:22:22.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:22:21 smithi153 ceph-mon[70044]: pgmap v5224: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:22.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:22:21 smithi083 ceph-mon[79930]: pgmap v5224: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:22.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:22:21 smithi083 ceph-mon[77242]: pgmap v5224: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:22:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:22:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:22:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:22:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:22:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:22:24.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:22:23 smithi153 ceph-mon[70044]: pgmap v5225: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:24.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:22:23 smithi083 ceph-mon[79930]: pgmap v5225: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:24.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:22:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:22:24.172Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:22:24.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:22:23 smithi083 ceph-mon[77242]: pgmap v5225: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:25.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:22:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:22:24.869Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:22:26.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:22:25 smithi153 ceph-mon[70044]: pgmap v5226: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:26.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:22:25 smithi083 ceph-mon[79930]: pgmap v5226: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:26.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:22:25 smithi083 ceph-mon[77242]: pgmap v5226: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:28.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:22:27 smithi153 ceph-mon[70044]: pgmap v5227: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:28.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:22:27 smithi083 ceph-mon[79930]: pgmap v5227: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:28.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:22:27 smithi083 ceph-mon[77242]: pgmap v5227: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:30.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:22:29 smithi153 ceph-mon[70044]: pgmap v5228: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:30.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:22:29 smithi083 ceph-mon[77242]: pgmap v5228: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:30.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:22:29 smithi083 ceph-mon[79930]: pgmap v5228: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:31.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:22:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:22:30.787Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:22:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:22:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:22:31] "GET /metrics HTTP/1.1" 200 225214 "" "Prometheus/2.33.4" 2022-09-29T08:22:32.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:22:32 smithi153 ceph-mon[70044]: pgmap v5229: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:32.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:22:32 smithi083 ceph-mon[79930]: pgmap v5229: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:32.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:22:32 smithi083 ceph-mon[77242]: pgmap v5229: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:22:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:22:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:22:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:22:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:22:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:22:34.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:22:34 smithi153 ceph-mon[70044]: pgmap v5230: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:34.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:22:34 smithi083 ceph-mon[79930]: pgmap v5230: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:34.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:22:34 smithi083 ceph-mon[77242]: pgmap v5230: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:34.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:22:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:22:34.173Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:22:35.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:22:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:22:34.869Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:22:36.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:22:36 smithi153 ceph-mon[70044]: pgmap v5231: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:36.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:22:36 smithi083 ceph-mon[77242]: pgmap v5231: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:36.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:22:36 smithi083 ceph-mon[79930]: pgmap v5231: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:38.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:22:38 smithi153 ceph-mon[70044]: pgmap v5232: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:38.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:22:38 smithi083 ceph-mon[79930]: pgmap v5232: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:38.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:22:38 smithi083 ceph-mon[77242]: pgmap v5232: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:22:40 smithi153 ceph-mon[70044]: pgmap v5233: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:22:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:22:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:22:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:22:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:22:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:22:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:22:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:22:40.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:22:40 smithi083 ceph-mon[77242]: pgmap v5233: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:40.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:22:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:22:40.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:22:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:22:40.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:22:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:22:40.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:22:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:22:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:22:40 smithi083 ceph-mon[79930]: pgmap v5233: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:22:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:22:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:22:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:22:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:22:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:22:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:22:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:22:41.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:22:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:22:40.788Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:22:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:22:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:22:41] "GET /metrics HTTP/1.1" 200 225215 "" "Prometheus/2.33.4" 2022-09-29T08:22:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:22:42 smithi153 ceph-mon[70044]: pgmap v5234: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:42.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:22:42 smithi083 ceph-mon[79930]: pgmap v5234: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:42.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:22:42 smithi083 ceph-mon[77242]: pgmap v5234: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:22:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:22:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:22:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:22:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:22:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:22:44.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:22:44 smithi153 ceph-mon[70044]: pgmap v5235: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:44.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:22:44 smithi083 ceph-mon[79930]: pgmap v5235: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:44.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:22:44 smithi083 ceph-mon[77242]: pgmap v5235: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:44.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:22:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:22:44.174Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:22:45.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:22:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:22:44.870Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:22:46.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:22:46 smithi153 ceph-mon[70044]: pgmap v5236: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:46.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:22:46 smithi083 ceph-mon[77242]: pgmap v5236: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:46.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:22:46 smithi083 ceph-mon[79930]: pgmap v5236: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:48.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:22:48 smithi153 ceph-mon[70044]: pgmap v5237: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:48.376 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:22:48.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:22:48 smithi083 ceph-mon[77242]: pgmap v5237: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:48.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:22:48 smithi083 ceph-mon[79930]: pgmap v5237: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:48.667 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:22:48.668 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 5m ago 2h 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:22:48.668 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 3m ago 2h 62.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:22:48.668 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 5m ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:22:48.668 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 3m ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:22:48.669 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 3m ago 3h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:22:48.669 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 5m ago 3h 662M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:22:48.669 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 5m ago 3h 675M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:22:48.669 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 3m ago 3h 298M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:22:48.670 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 5m ago 3h 394M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:22:48.670 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 5m ago 2h 26.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:22:48.670 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 3m ago 2h 25.8M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:22:48.670 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 5m ago 3h 488M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:22:48.670 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 5m ago 2h 479M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:22:48.671 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 5m ago 2h 378M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:22:48.671 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 5m ago 2h 465M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:22:48.671 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 3m ago 2h 414M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:22:48.671 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 3m ago 2h 510M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:22:48.672 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 3m ago 2h 454M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:22:48.672 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 3m ago 2h 532M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:22:48.672 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 3m ago 2h 141M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:22:49.022 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:22:49.023 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:22:49.023 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:22:49.023 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:22:49.024 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:22:49.024 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:22:49.024 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:22:49.024 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:22:49.025 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:22:49.025 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:22:49.025 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:22:49.025 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:22:49.025 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:22:49.026 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:22:49.026 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:22:49.026 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:22:49.026 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:22:49.027 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:22:49.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:22:49 smithi153 ceph-mon[70044]: from='client.50158 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:22:49.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:22:49 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3663659205' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:22:49.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:22:49 smithi083 ceph-mon[79930]: from='client.50158 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:22:49.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:22:49 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3663659205' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:22:49.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:22:49 smithi083 ceph-mon[77242]: from='client.50158 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:22:49.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:22:49 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3663659205' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:22:50.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:22:50 smithi153 ceph-mon[70044]: from='client.40233 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:22:50.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:22:50 smithi153 ceph-mon[70044]: pgmap v5238: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:50.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:22:50 smithi083 ceph-mon[79930]: from='client.40233 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:22:50.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:22:50 smithi083 ceph-mon[79930]: pgmap v5238: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:50.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:22:50 smithi083 ceph-mon[77242]: from='client.40233 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:22:50.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:22:50 smithi083 ceph-mon[77242]: pgmap v5238: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:51.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:22:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:22:50.789Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:22:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:22:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:22:51] "GET /metrics HTTP/1.1" 200 225215 "" "Prometheus/2.33.4" 2022-09-29T08:22:52.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:22:52 smithi153 ceph-mon[70044]: pgmap v5239: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:52.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:22:52 smithi083 ceph-mon[77242]: pgmap v5239: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:52.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:22:52 smithi083 ceph-mon[79930]: pgmap v5239: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:22:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:22:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:22:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:22:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:22:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:22:54.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:22:54 smithi153 ceph-mon[70044]: pgmap v5240: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:54.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:22:54 smithi083 ceph-mon[79930]: pgmap v5240: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:54.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:22:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:22:54.174Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:22:54.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:22:54 smithi083 ceph-mon[77242]: pgmap v5240: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:55.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:22:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:22:54.871Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:22:56.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:22:56 smithi153 ceph-mon[70044]: pgmap v5241: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:56.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:22:56 smithi083 ceph-mon[79930]: pgmap v5241: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:56.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:22:56 smithi083 ceph-mon[77242]: pgmap v5241: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:58.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:22:58 smithi153 ceph-mon[70044]: pgmap v5242: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:58.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:22:58 smithi083 ceph-mon[79930]: pgmap v5242: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:22:58.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:22:58 smithi083 ceph-mon[77242]: pgmap v5242: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:23:00 smithi153 ceph-mon[70044]: pgmap v5243: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:00.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:23:00 smithi083 ceph-mon[79930]: pgmap v5243: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:00.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:23:00 smithi083 ceph-mon[77242]: pgmap v5243: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:01.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:23:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:23:00.789Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:23:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:23:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:23:01] "GET /metrics HTTP/1.1" 200 225213 "" "Prometheus/2.33.4" 2022-09-29T08:23:02.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:23:02 smithi153 ceph-mon[70044]: pgmap v5244: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:02.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:23:02 smithi083 ceph-mon[79930]: pgmap v5244: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:02.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:23:02 smithi083 ceph-mon[77242]: pgmap v5244: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:23:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:23:02.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:23:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:23:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:23:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:23:04.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:23:04 smithi083 ceph-mon[79930]: pgmap v5245: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:04.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:23:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:23:04.176Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:23:04.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:23:04 smithi083 ceph-mon[77242]: pgmap v5245: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:23:04 smithi153 ceph-mon[70044]: pgmap v5245: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:05.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:23:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:23:04.871Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:23:06.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:23:06 smithi083 ceph-mon[79930]: pgmap v5246: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:06.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:23:06 smithi083 ceph-mon[77242]: pgmap v5246: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:06.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:23:06 smithi153 ceph-mon[70044]: pgmap v5246: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:08.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:23:08 smithi083 ceph-mon[79930]: pgmap v5247: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:08.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:23:08 smithi083 ceph-mon[77242]: pgmap v5247: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:23:08 smithi153 ceph-mon[70044]: pgmap v5247: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:10.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:23:10 smithi083 ceph-mon[79930]: pgmap v5248: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:10.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:23:10 smithi083 ceph-mon[77242]: pgmap v5248: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:23:10 smithi153 ceph-mon[70044]: pgmap v5248: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:11.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:23:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:23:10.790Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:23:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:23:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:23:11] "GET /metrics HTTP/1.1" 200 225215 "" "Prometheus/2.33.4" 2022-09-29T08:23:12.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:23:12 smithi083 ceph-mon[77242]: pgmap v5249: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:12.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:23:12 smithi083 ceph-mon[79930]: pgmap v5249: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:23:12 smithi153 ceph-mon[70044]: pgmap v5249: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:12.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:23:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:23:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:23:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:23:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:23:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:23:14.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:23:14 smithi083 ceph-mon[79930]: pgmap v5250: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:14.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:23:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:23:14.176Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:23:14.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:23:14 smithi083 ceph-mon[77242]: pgmap v5250: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:14.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:23:14 smithi153 ceph-mon[70044]: pgmap v5250: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:15.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:23:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:23:14.871Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:23:16.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:23:16 smithi083 ceph-mon[79930]: pgmap v5251: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:16.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:23:16 smithi083 ceph-mon[77242]: pgmap v5251: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:23:16 smithi153 ceph-mon[70044]: pgmap v5251: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:18.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:23:18 smithi083 ceph-mon[79930]: pgmap v5252: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:18.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:23:18 smithi083 ceph-mon[77242]: pgmap v5252: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:23:18 smithi153 ceph-mon[70044]: pgmap v5252: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:19.366 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:23:19.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:23:19 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:23:19.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:23:19 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:23:19.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:23:19 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:23:19.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:23:19 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:23:19.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:23:19 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:23:19.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:23:19 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:23:19.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:23:19 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:23:19.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:23:19 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:23:19.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:23:19 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:23:19.671 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:23:19.671 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 6m ago 2h 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:23:19.671 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 4m ago 2h 62.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:23:19.672 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 6m ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:23:19.672 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 4m ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:23:19.672 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 4m ago 3h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:23:19.672 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 6m ago 3h 662M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:23:19.672 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 6m ago 3h 675M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:23:19.673 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 4m ago 3h 298M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:23:19.673 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 6m ago 3h 394M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:23:19.673 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 6m ago 2h 26.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:23:19.673 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 4m ago 2h 25.8M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:23:19.674 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 6m ago 3h 488M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:23:19.674 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 6m ago 3h 479M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:23:19.674 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 6m ago 3h 378M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:23:19.674 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 6m ago 2h 465M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:23:19.675 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 4m ago 2h 414M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:23:19.675 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 4m ago 2h 510M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:23:19.675 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 4m ago 2h 454M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:23:19.675 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 4m ago 2h 532M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:23:19.676 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 4m ago 2h 141M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:23:20.030 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:23:20.030 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:23:20.030 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:23:20.031 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:23:20.031 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:23:20.031 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:23:20.031 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:23:20.032 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:23:20.032 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:23:20.032 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:23:20.032 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:23:20.033 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:23:20.033 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:23:20.033 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:23:20.033 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:23:20.033 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:23:20.034 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:23:20.034 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:23:20.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:23:20 smithi083 ceph-mon[79930]: pgmap v5253: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:20.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:23:20 smithi083 ceph-mon[79930]: from='client.50176 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:23:20.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:23:20 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:23:20.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:23:20 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3543774530' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:23:20.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:23:20 smithi083 ceph-mon[77242]: pgmap v5253: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:20.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:23:20 smithi083 ceph-mon[77242]: from='client.50176 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:23:20.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:23:20 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:23:20.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:23:20 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3543774530' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:23:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:23:20 smithi153 ceph-mon[70044]: pgmap v5253: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:23:20 smithi153 ceph-mon[70044]: from='client.50176 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:23:20.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:23:20 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:23:20.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:23:20 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3543774530' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:23:21.093 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:23:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:23:20.790Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:23:21.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:23:21 smithi083 ceph-mon[77242]: from='client.50182 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:23:21.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:23:21 smithi083 ceph-mon[79930]: from='client.50182 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:23:21.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:23:21 smithi153 ceph-mon[70044]: from='client.50182 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:23:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:23:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:23:21] "GET /metrics HTTP/1.1" 200 225215 "" "Prometheus/2.33.4" 2022-09-29T08:23:22.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:23:22 smithi083 ceph-mon[79930]: pgmap v5254: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:22.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:23:22 smithi083 ceph-mon[77242]: pgmap v5254: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:23:22 smithi153 ceph-mon[70044]: pgmap v5254: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:22.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:23:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:23:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:23:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:23:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:23:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:23:24.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:23:24 smithi083 ceph-mon[79930]: pgmap v5255: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:24.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:23:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:23:24.177Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:23:24.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:23:24 smithi083 ceph-mon[77242]: pgmap v5255: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:23:24 smithi153 ceph-mon[70044]: pgmap v5255: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:25.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:23:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:23:24.872Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:23:26.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:23:26 smithi083 ceph-mon[79930]: pgmap v5256: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:26.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:23:26 smithi083 ceph-mon[77242]: pgmap v5256: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:23:26 smithi153 ceph-mon[70044]: pgmap v5256: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:28.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:23:28 smithi083 ceph-mon[79930]: pgmap v5257: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:28.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:23:28 smithi083 ceph-mon[77242]: pgmap v5257: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:23:28 smithi153 ceph-mon[70044]: pgmap v5257: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:30.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:23:30 smithi083 ceph-mon[79930]: pgmap v5258: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:30.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:23:30 smithi083 ceph-mon[77242]: pgmap v5258: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:23:30 smithi153 ceph-mon[70044]: pgmap v5258: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:31.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:23:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:23:30.792Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:23:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:23:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:23:31] "GET /metrics HTTP/1.1" 200 225215 "" "Prometheus/2.33.4" 2022-09-29T08:23:32.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:23:32 smithi083 ceph-mon[77242]: pgmap v5259: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:32.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:23:32 smithi083 ceph-mon[79930]: pgmap v5259: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:23:32 smithi153 ceph-mon[70044]: pgmap v5259: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:32.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:23:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:23:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:23:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:23:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:23:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:23:34.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:23:34 smithi083 ceph-mon[79930]: pgmap v5260: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:34.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:23:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:23:34.177Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:23:34.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:23:34 smithi083 ceph-mon[77242]: pgmap v5260: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:23:34 smithi153 ceph-mon[70044]: pgmap v5260: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:35.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:23:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:23:34.872Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:23:36.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:23:36 smithi083 ceph-mon[79930]: pgmap v5261: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:36.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:23:36 smithi083 ceph-mon[77242]: pgmap v5261: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:23:36 smithi153 ceph-mon[70044]: pgmap v5261: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:38.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:23:38 smithi083 ceph-mon[79930]: pgmap v5262: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:38.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:23:38 smithi083 ceph-mon[77242]: pgmap v5262: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:23:38 smithi153 ceph-mon[70044]: pgmap v5262: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:40.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:23:40 smithi083 ceph-mon[79930]: pgmap v5263: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:40.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:23:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:23:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:23:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:23:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:23:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:23:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:23:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:23:40.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:23:40 smithi083 ceph-mon[77242]: pgmap v5263: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:23:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:23:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:23:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:23:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:23:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:23:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:23:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:23:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:23:40 smithi153 ceph-mon[70044]: pgmap v5263: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:23:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:23:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:23:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:23:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:23:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:23:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:23:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:23:41.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:23:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:23:40.793Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:23:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:23:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:23:41] "GET /metrics HTTP/1.1" 200 225216 "" "Prometheus/2.33.4" 2022-09-29T08:23:42.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:23:42 smithi083 ceph-mon[79930]: pgmap v5264: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:42.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:23:42 smithi083 ceph-mon[77242]: pgmap v5264: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:23:42 smithi153 ceph-mon[70044]: pgmap v5264: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:42.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:23:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:23:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:23:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:23:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:23:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:23:44.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:23:44 smithi083 ceph-mon[79930]: pgmap v5265: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:44.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:23:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:23:44.178Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:23:44.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:23:44 smithi083 ceph-mon[77242]: pgmap v5265: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:44.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:23:44 smithi153 ceph-mon[70044]: pgmap v5265: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:45.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:23:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:23:44.873Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:23:46.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:23:46 smithi083 ceph-mon[79930]: pgmap v5266: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:46.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:23:46 smithi083 ceph-mon[77242]: pgmap v5266: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:46.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:23:46 smithi153 ceph-mon[70044]: pgmap v5266: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:48.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:23:48 smithi083 ceph-mon[79930]: pgmap v5267: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:48.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:23:48 smithi083 ceph-mon[77242]: pgmap v5267: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:48.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:23:48 smithi153 ceph-mon[70044]: pgmap v5267: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:50.357 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:23:50.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:23:50 smithi083 ceph-mon[79930]: pgmap v5268: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:50.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:23:50 smithi083 ceph-mon[77242]: pgmap v5268: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:23:50 smithi153 ceph-mon[70044]: pgmap v5268: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:50.645 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:23:50.646 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 6m ago 2h 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:23:50.646 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 4m ago 2h 62.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:23:50.646 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 6m ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:23:50.646 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 4m ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:23:50.647 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 4m ago 3h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:23:50.647 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 6m ago 3h 662M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:23:50.647 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 6m ago 3h 675M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:23:50.647 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 4m ago 3h 298M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:23:50.647 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 6m ago 3h 394M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:23:50.648 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 6m ago 2h 26.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:23:50.648 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 4m ago 2h 25.8M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:23:50.648 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 6m ago 3h 488M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:23:50.648 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 6m ago 3h 479M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:23:50.649 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 6m ago 3h 378M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:23:50.649 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 6m ago 3h 465M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:23:50.649 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 4m ago 3h 414M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:23:50.649 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 4m ago 2h 510M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:23:50.649 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 4m ago 2h 454M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:23:50.650 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 4m ago 2h 532M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:23:50.650 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 4m ago 2h 141M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:23:51.001 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:23:51.002 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:23:51.002 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:23:51.002 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:23:51.003 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:23:51.003 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:23:51.003 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:23:51.003 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:23:51.003 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:23:51.004 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:23:51.004 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:23:51.004 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:23:51.004 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:23:51.005 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:23:51.005 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:23:51.005 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:23:51.005 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:23:51.006 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:23:51.142 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:23:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:23:50.792Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:23:51.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:23:51 smithi083 ceph-mon[79930]: from='client.50194 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:23:51.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:23:51 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2985734740' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:23:51.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:23:51 smithi083 ceph-mon[77242]: from='client.50194 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:23:51.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:23:51 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2985734740' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:23:51.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:23:51 smithi153 ceph-mon[70044]: from='client.50194 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:23:51.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:23:51 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2985734740' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:23:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:23:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:23:51] "GET /metrics HTTP/1.1" 200 225216 "" "Prometheus/2.33.4" 2022-09-29T08:23:52.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:23:52 smithi083 ceph-mon[79930]: from='client.40269 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:23:52.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:23:52 smithi083 ceph-mon[79930]: pgmap v5269: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:52.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:23:52 smithi083 ceph-mon[77242]: from='client.40269 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:23:52.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:23:52 smithi083 ceph-mon[77242]: pgmap v5269: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:23:52 smithi153 ceph-mon[70044]: from='client.40269 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:23:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:23:52 smithi153 ceph-mon[70044]: pgmap v5269: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:52.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:23:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:23:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:23:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:23:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:23:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:23:54.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:23:54 smithi083 ceph-mon[79930]: pgmap v5270: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:54.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:23:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:23:54.177Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:23:54.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:23:54 smithi083 ceph-mon[77242]: pgmap v5270: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:54.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:23:54 smithi153 ceph-mon[70044]: pgmap v5270: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:55.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:23:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:23:54.873Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:23:56.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:23:56 smithi083 ceph-mon[79930]: pgmap v5271: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:56.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:23:56 smithi083 ceph-mon[77242]: pgmap v5271: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:56.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:23:56 smithi153 ceph-mon[70044]: pgmap v5271: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:58.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:23:58 smithi083 ceph-mon[79930]: pgmap v5272: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:58.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:23:58 smithi083 ceph-mon[77242]: pgmap v5272: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:23:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:23:58 smithi153 ceph-mon[70044]: pgmap v5272: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:00.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:24:00 smithi083 ceph-mon[77242]: pgmap v5273: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:00.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:24:00 smithi083 ceph-mon[79930]: pgmap v5273: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:24:00 smithi153 ceph-mon[70044]: pgmap v5273: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:01.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:24:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:24:00.794Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:24:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:24:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:24:01] "GET /metrics HTTP/1.1" 200 225213 "" "Prometheus/2.33.4" 2022-09-29T08:24:02.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:24:02 smithi083 ceph-mon[79930]: pgmap v5274: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:02.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:24:02 smithi083 ceph-mon[77242]: pgmap v5274: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:24:02 smithi153 ceph-mon[70044]: pgmap v5274: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:02.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:24:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:24:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:24:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:24:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:24:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:24:04.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:24:04 smithi083 ceph-mon[79930]: pgmap v5275: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:04.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:24:04 smithi083 ceph-mon[77242]: pgmap v5275: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:04.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:24:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:24:04.178Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:24:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:24:04 smithi153 ceph-mon[70044]: pgmap v5275: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:05.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:24:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:24:04.874Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:24:06.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:24:06 smithi083 ceph-mon[79930]: pgmap v5276: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:06.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:24:06 smithi083 ceph-mon[77242]: pgmap v5276: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:06.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:24:06 smithi153 ceph-mon[70044]: pgmap v5276: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:08.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:24:08 smithi083 ceph-mon[77242]: pgmap v5277: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:08.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:24:08 smithi083 ceph-mon[79930]: pgmap v5277: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:24:08 smithi153 ceph-mon[70044]: pgmap v5277: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:10.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:24:10 smithi083 ceph-mon[79930]: pgmap v5278: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:10.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:24:10 smithi083 ceph-mon[77242]: pgmap v5278: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:24:10 smithi153 ceph-mon[70044]: pgmap v5278: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:11.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:24:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:24:10.794Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:24:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:24:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:24:11] "GET /metrics HTTP/1.1" 200 225214 "" "Prometheus/2.33.4" 2022-09-29T08:24:12.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:24:12 smithi083 ceph-mon[79930]: pgmap v5279: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:12.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:24:12 smithi083 ceph-mon[77242]: pgmap v5279: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:24:12 smithi153 ceph-mon[70044]: pgmap v5279: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:12.571 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:24:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:24:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:24:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:24:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:24:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:24:14.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:24:14 smithi083 ceph-mon[79930]: pgmap v5280: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:14.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:24:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:24:14.179Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:24:14.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:24:14 smithi083 ceph-mon[77242]: pgmap v5280: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:14.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:24:14 smithi153 ceph-mon[70044]: pgmap v5280: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:15.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:24:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:24:14.875Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:24:16.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:24:16 smithi083 ceph-mon[79930]: pgmap v5281: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:16.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:24:16 smithi083 ceph-mon[77242]: pgmap v5281: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:24:16 smithi153 ceph-mon[70044]: pgmap v5281: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:18.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:24:18 smithi083 ceph-mon[79930]: pgmap v5282: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:18.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:24:18 smithi083 ceph-mon[77242]: pgmap v5282: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:24:18 smithi153 ceph-mon[70044]: pgmap v5282: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:20.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:24:20 smithi083 ceph-mon[77242]: pgmap v5283: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:20.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:24:20 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:24:20.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:24:20 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:24:20.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:24:20 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:24:20.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:24:20 smithi083 ceph-mon[79930]: pgmap v5283: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:20.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:24:20 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:24:20.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:24:20 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:24:20.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:24:20 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:24:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:24:20 smithi153 ceph-mon[70044]: pgmap v5283: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:24:20 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:24:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:24:20 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:24:20.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:24:20 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:24:21.192 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:24:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:24:20.794Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:24:21.325 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:24:21.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:24:21 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:24:21.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:24:21 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:24:21.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:24:21 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:24:21.615 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:24:21.616 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 7m ago 2h 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:24:21.616 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 5m ago 2h 62.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:24:21.616 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 7m ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:24:21.616 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 5m ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:24:21.616 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 5m ago 3h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:24:21.617 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 7m ago 3h 662M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:24:21.617 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 7m ago 3h 675M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:24:21.617 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 5m ago 3h 298M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:24:21.617 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 7m ago 3h 394M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:24:21.617 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 7m ago 2h 26.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:24:21.618 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 5m ago 2h 25.8M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:24:21.618 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 7m ago 3h 488M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:24:21.618 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 7m ago 3h 479M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:24:21.618 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 7m ago 3h 378M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:24:21.619 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 7m ago 3h 465M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:24:21.619 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 5m ago 3h 414M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:24:21.619 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 5m ago 3h 510M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:24:21.619 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 5m ago 2h 454M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:24:21.619 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 5m ago 2h 532M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:24:21.620 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 5m ago 2h 141M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:24:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:24:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:24:21] "GET /metrics HTTP/1.1" 200 225214 "" "Prometheus/2.33.4" 2022-09-29T08:24:21.972 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:24:21.972 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:24:21.973 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:24:21.973 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:24:21.973 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:24:21.974 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:24:21.974 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:24:21.974 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:24:21.975 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:24:21.975 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:24:21.975 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:24:21.975 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:24:21.976 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:24:21.976 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:24:21.976 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:24:21.977 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:24:21.977 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:24:21.977 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:24:22.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:24:22 smithi083 ceph-mon[79930]: pgmap v5284: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:22.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:24:22 smithi083 ceph-mon[79930]: from='client.40281 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:24:22.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:24:22 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2270610073' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:24:22.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:24:22 smithi083 ceph-mon[77242]: pgmap v5284: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:22.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:24:22 smithi083 ceph-mon[77242]: from='client.40281 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:24:22.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:24:22 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2270610073' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:24:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:24:22 smithi153 ceph-mon[70044]: pgmap v5284: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:24:22 smithi153 ceph-mon[70044]: from='client.40281 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:24:22.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:24:22 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2270610073' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:24:22.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:24:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:24:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:24:23.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:24:23 smithi153 ceph-mon[70044]: from='client.40287 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:24:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:24:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:24:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:24:23.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:24:23 smithi083 ceph-mon[79930]: from='client.40287 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:24:23.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:24:23 smithi083 ceph-mon[77242]: from='client.40287 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:24:24.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:24:24 smithi083 ceph-mon[79930]: pgmap v5285: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:24.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:24:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:24:24.180Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:24:24.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:24:24 smithi083 ceph-mon[77242]: pgmap v5285: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:24:24 smithi153 ceph-mon[70044]: pgmap v5285: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:25.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:24:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:24:24.876Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:24:26.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:24:26 smithi083 ceph-mon[79930]: pgmap v5286: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:26.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:24:26 smithi083 ceph-mon[77242]: pgmap v5286: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:24:26 smithi153 ceph-mon[70044]: pgmap v5286: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:24:28 smithi153 ceph-mon[70044]: pgmap v5287: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:28.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:24:28 smithi083 ceph-mon[77242]: pgmap v5287: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:28.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:24:28 smithi083 ceph-mon[79930]: pgmap v5287: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:24:30 smithi153 ceph-mon[70044]: pgmap v5288: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:30.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:24:30 smithi083 ceph-mon[79930]: pgmap v5288: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:30.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:24:30 smithi083 ceph-mon[77242]: pgmap v5288: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:31.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:24:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:24:30.795Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:24:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:24:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:24:31] "GET /metrics HTTP/1.1" 200 225215 "" "Prometheus/2.33.4" 2022-09-29T08:24:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:24:32 smithi153 ceph-mon[70044]: pgmap v5289: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:32.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:24:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:24:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:24:32.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:24:32 smithi083 ceph-mon[77242]: pgmap v5289: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:32.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:24:32 smithi083 ceph-mon[79930]: pgmap v5289: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:24:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:24:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:24:34.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:24:34 smithi083 ceph-mon[79930]: pgmap v5290: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:34.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:24:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:24:34.181Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:24:34.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:24:34 smithi083 ceph-mon[77242]: pgmap v5290: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:24:34 smithi153 ceph-mon[70044]: pgmap v5290: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:35.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:24:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:24:34.876Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:24:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:24:36 smithi153 ceph-mon[70044]: pgmap v5291: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:36.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:24:36 smithi083 ceph-mon[77242]: pgmap v5291: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:36.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:24:36 smithi083 ceph-mon[79930]: pgmap v5291: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:24:38 smithi153 ceph-mon[70044]: pgmap v5292: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:38.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:24:38 smithi083 ceph-mon[79930]: pgmap v5292: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:38.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:24:38 smithi083 ceph-mon[77242]: pgmap v5292: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:24:40 smithi153 ceph-mon[70044]: pgmap v5293: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:24:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:24:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:24:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:24:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:24:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:24:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:24:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:24:40.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:24:40 smithi083 ceph-mon[79930]: pgmap v5293: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:40.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:24:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:24:40.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:24:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:24:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:24:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:24:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:24:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:24:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:24:40 smithi083 ceph-mon[77242]: pgmap v5293: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:24:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:24:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:24:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:24:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:24:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:24:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:24:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:24:41.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:24:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:24:40.796Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:24:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:24:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:24:41] "GET /metrics HTTP/1.1" 200 225215 "" "Prometheus/2.33.4" 2022-09-29T08:24:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:24:42 smithi153 ceph-mon[70044]: pgmap v5294: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:42.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:24:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:24:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:24:42.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:24:42 smithi083 ceph-mon[77242]: pgmap v5294: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:42.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:24:42 smithi083 ceph-mon[79930]: pgmap v5294: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:24:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:24:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:24:44.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:24:44 smithi083 ceph-mon[79930]: pgmap v5295: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:44.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:24:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:24:44.181Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:24:44.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:24:44 smithi083 ceph-mon[77242]: pgmap v5295: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:44.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:24:44 smithi153 ceph-mon[70044]: pgmap v5295: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:45.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:24:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:24:44.877Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:24:46.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:24:46 smithi153 ceph-mon[70044]: pgmap v5296: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:46.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:24:46 smithi083 ceph-mon[77242]: pgmap v5296: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:46.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:24:46 smithi083 ceph-mon[79930]: pgmap v5296: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:48.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:24:48 smithi153 ceph-mon[70044]: pgmap v5297: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:48.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:24:48 smithi083 ceph-mon[79930]: pgmap v5297: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:48.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:24:48 smithi083 ceph-mon[77242]: pgmap v5297: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:24:50 smithi153 ceph-mon[70044]: pgmap v5298: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:50.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:24:50 smithi083 ceph-mon[79930]: pgmap v5298: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:50.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:24:50 smithi083 ceph-mon[77242]: pgmap v5298: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:51.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:24:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:24:50.796Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:24:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:24:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:24:51] "GET /metrics HTTP/1.1" 200 225215 "" "Prometheus/2.33.4" 2022-09-29T08:24:52.301 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:24:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:24:52 smithi153 ceph-mon[70044]: pgmap v5299: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:52.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:24:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:24:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:24:52.588 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:24:52.589 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 7m ago 2h 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:24:52.589 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 5m ago 2h 62.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:24:52.589 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (2h) 7m ago 2h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:24:52.590 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (2h) 5m ago 2h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:24:52.590 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 5m ago 3h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:24:52.590 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 7m ago 3h 662M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:24:52.590 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 7m ago 3h 675M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:24:52.591 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 5m ago 3h 298M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:24:52.591 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 7m ago 3h 394M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:24:52.591 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 7m ago 2h 26.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:24:52.591 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 5m ago 2h 25.8M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:24:52.591 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 7m ago 3h 488M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:24:52.592 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 7m ago 3h 479M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:24:52.592 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 7m ago 3h 378M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:24:52.592 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 7m ago 3h 465M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:24:52.592 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 5m ago 3h 414M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:24:52.593 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 5m ago 3h 510M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:24:52.593 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 5m ago 3h 454M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:24:52.593 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 5m ago 3h 532M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:24:52.593 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 5m ago 2h 141M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:24:52.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:24:52 smithi083 ceph-mon[79930]: pgmap v5299: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:52.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:24:52 smithi083 ceph-mon[77242]: pgmap v5299: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:52.946 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:24:52.946 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:24:52.947 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:24:52.947 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:24:52.947 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:24:52.947 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:24:52.948 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:24:52.948 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:24:52.948 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:24:52.948 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:24:52.949 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:24:52.949 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:24:52.949 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:24:52.949 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:24:52.949 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:24:52.950 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:24:52.950 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:24:52.950 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:24:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:24:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:24:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:24:53.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:24:53 smithi153 ceph-mon[70044]: from='client.50230 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:24:53.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:24:53 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1890455512' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:24:53.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:24:53 smithi083 ceph-mon[79930]: from='client.50230 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:24:53.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:24:53 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1890455512' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:24:53.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:24:53 smithi083 ceph-mon[77242]: from='client.50230 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:24:53.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:24:53 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1890455512' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:24:54.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:24:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:24:54.181Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:24:54.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:24:54 smithi083 ceph-mon[77242]: from='client.50236 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:24:54.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:24:54 smithi083 ceph-mon[77242]: pgmap v5300: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:54.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:24:54 smithi083 ceph-mon[79930]: from='client.50236 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:24:54.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:24:54 smithi083 ceph-mon[79930]: pgmap v5300: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:54.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:24:54 smithi153 ceph-mon[70044]: from='client.50236 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:24:54.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:24:54 smithi153 ceph-mon[70044]: pgmap v5300: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:55.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:24:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:24:54.877Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:24:56.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:24:56 smithi153 ceph-mon[70044]: pgmap v5301: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:56.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:24:56 smithi083 ceph-mon[79930]: pgmap v5301: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:56.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:24:56 smithi083 ceph-mon[77242]: pgmap v5301: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:24:58 smithi153 ceph-mon[70044]: pgmap v5302: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:58.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:24:58 smithi083 ceph-mon[79930]: pgmap v5302: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:24:58.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:24:58 smithi083 ceph-mon[77242]: pgmap v5302: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:25:00 smithi153 ceph-mon[70044]: pgmap v5303: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:00.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:25:00 smithi083 ceph-mon[79930]: pgmap v5303: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:00.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:25:00 smithi083 ceph-mon[77242]: pgmap v5303: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:01.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:25:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:25:00.797Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:25:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:25:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:25:01] "GET /metrics HTTP/1.1" 200 225218 "" "Prometheus/2.33.4" 2022-09-29T08:25:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:25:02 smithi153 ceph-mon[70044]: pgmap v5304: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:02.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:25:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:25:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:25:02.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:25:02 smithi083 ceph-mon[79930]: pgmap v5304: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:02.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:25:02 smithi083 ceph-mon[77242]: pgmap v5304: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:25:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:25:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:25:04.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:25:04 smithi083 ceph-mon[79930]: pgmap v5305: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:04.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:25:04 smithi083 ceph-mon[77242]: pgmap v5305: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:04.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:25:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:25:04.183Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:25:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:25:04 smithi153 ceph-mon[70044]: pgmap v5305: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:05.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:25:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:25:04.878Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:25:06.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:25:06 smithi153 ceph-mon[70044]: pgmap v5306: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:06.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:25:06 smithi083 ceph-mon[79930]: pgmap v5306: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:06.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:25:06 smithi083 ceph-mon[77242]: pgmap v5306: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:25:08 smithi153 ceph-mon[70044]: pgmap v5307: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:08.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:25:08 smithi083 ceph-mon[79930]: pgmap v5307: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:08.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:25:08 smithi083 ceph-mon[77242]: pgmap v5307: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:25:10 smithi153 ceph-mon[70044]: pgmap v5308: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:10.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:25:10 smithi083 ceph-mon[79930]: pgmap v5308: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:10.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:25:10 smithi083 ceph-mon[77242]: pgmap v5308: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:11.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:25:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:25:10.798Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:25:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:25:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:25:11] "GET /metrics HTTP/1.1" 200 225219 "" "Prometheus/2.33.4" 2022-09-29T08:25:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:25:12 smithi153 ceph-mon[70044]: pgmap v5309: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:12.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:25:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:25:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:25:12.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:25:12 smithi083 ceph-mon[77242]: pgmap v5309: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:12.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:25:12 smithi083 ceph-mon[79930]: pgmap v5309: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:25:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:25:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:25:14.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:25:14 smithi083 ceph-mon[79930]: pgmap v5310: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:14.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:25:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:25:14.184Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:25:14.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:25:14 smithi083 ceph-mon[77242]: pgmap v5310: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:14.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:25:14 smithi153 ceph-mon[70044]: pgmap v5310: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:15.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:25:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:25:14.879Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:25:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:25:16 smithi153 ceph-mon[70044]: pgmap v5311: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:16.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:25:16 smithi083 ceph-mon[79930]: pgmap v5311: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:16.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:25:16 smithi083 ceph-mon[77242]: pgmap v5311: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:25:18 smithi153 ceph-mon[70044]: pgmap v5312: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:18.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:25:18 smithi083 ceph-mon[79930]: pgmap v5312: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:18.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:25:18 smithi083 ceph-mon[77242]: pgmap v5312: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:25:20 smithi153 ceph-mon[70044]: pgmap v5313: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:25:20 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:25:20.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:25:20 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:25:20.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:25:20 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:25:20.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:25:20 smithi083 ceph-mon[77242]: pgmap v5313: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:20.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:25:20 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:25:20.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:25:20 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:25:20.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:25:20 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:25:20.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:25:20 smithi083 ceph-mon[79930]: pgmap v5313: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:20.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:25:20 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:25:20.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:25:20 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:25:20.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:25:20 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:25:21.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:25:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:25:20.806Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:25:21.858 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:25:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:25:21] "GET /metrics HTTP/1.1" 200 225219 "" "Prometheus/2.33.4" 2022-09-29T08:25:22.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:25:21 smithi083 ceph-mon[77242]: pgmap v5314: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:22.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:25:21 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:25:22.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:25:21 smithi083 ceph-mon[79930]: pgmap v5314: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:22.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:25:21 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:25:22.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:25:21 smithi153 ceph-mon[70044]: pgmap v5314: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:22.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:25:21 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:25:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:25:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:25:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:25:23.273 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:25:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:25:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:25:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:25:23.573 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:25:23.573 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 8m ago 2h 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:25:23.573 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 6m ago 2h 62.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:25:23.574 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 8m ago 3h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:25:23.574 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 6m ago 3h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:25:23.574 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 6m ago 3h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:25:23.575 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 8m ago 3h 662M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:25:23.575 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 8m ago 3h 675M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:25:23.575 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 6m ago 3h 298M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:25:23.575 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 8m ago 3h 394M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:25:23.575 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (2h) 8m ago 2h 26.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:25:23.576 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (2h) 6m ago 2h 25.8M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:25:23.576 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 8m ago 3h 488M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:25:23.576 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 8m ago 3h 479M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:25:23.576 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 8m ago 3h 378M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:25:23.577 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 8m ago 3h 465M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:25:23.577 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 6m ago 3h 414M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:25:23.577 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 6m ago 3h 510M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:25:23.577 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 6m ago 3h 454M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:25:23.578 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 6m ago 3h 532M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:25:23.578 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 6m ago 2h 141M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:25:23.932 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:25:23.932 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:25:23.933 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:25:23.933 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:25:23.933 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:25:23.933 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:25:23.934 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:25:23.934 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:25:23.934 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:25:23.934 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:25:23.935 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:25:23.935 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:25:23.935 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:25:23.935 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:25:23.935 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:25:23.935 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:25:23.936 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:25:23.936 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:25:24.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:25:23 smithi153 ceph-mon[70044]: pgmap v5315: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:24.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:25:23 smithi153 ceph-mon[70044]: from='client.40317 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:25:24.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:25:23 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3978106047' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:25:24.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:25:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:25:24.183Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:25:24.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:25:23 smithi083 ceph-mon[77242]: pgmap v5315: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:24.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:25:23 smithi083 ceph-mon[77242]: from='client.40317 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:25:24.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:25:23 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3978106047' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:25:24.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:25:23 smithi083 ceph-mon[79930]: pgmap v5315: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:24.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:25:23 smithi083 ceph-mon[79930]: from='client.40317 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:25:24.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:25:23 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3978106047' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:25:25.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:25:24 smithi083 ceph-mon[77242]: from='client.50254 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:25:25.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:25:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:25:24.879Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:25:25.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:25:24 smithi083 ceph-mon[79930]: from='client.50254 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:25:25.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:25:24 smithi153 ceph-mon[70044]: from='client.50254 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:25:26.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:25:25 smithi153 ceph-mon[70044]: pgmap v5316: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:26.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:25:25 smithi083 ceph-mon[79930]: pgmap v5316: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:26.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:25:25 smithi083 ceph-mon[77242]: pgmap v5316: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:28.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:25:27 smithi153 ceph-mon[70044]: pgmap v5317: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:28.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:25:27 smithi083 ceph-mon[79930]: pgmap v5317: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:28.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:25:27 smithi083 ceph-mon[77242]: pgmap v5317: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:30.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:25:29 smithi153 ceph-mon[70044]: pgmap v5318: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:30.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:25:29 smithi083 ceph-mon[79930]: pgmap v5318: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:30.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:25:29 smithi083 ceph-mon[77242]: pgmap v5318: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:31.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:25:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:25:30.799Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:25:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:25:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:25:31] "GET /metrics HTTP/1.1" 200 225213 "" "Prometheus/2.33.4" 2022-09-29T08:25:32.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:25:31 smithi153 ceph-mon[70044]: pgmap v5319: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:32.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:25:31 smithi083 ceph-mon[79930]: pgmap v5319: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:32.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:25:31 smithi083 ceph-mon[77242]: pgmap v5319: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:25:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:25:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:25:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:25:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:25:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:25:34.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:25:33 smithi153 ceph-mon[70044]: pgmap v5320: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:34.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:25:33 smithi083 ceph-mon[79930]: pgmap v5320: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:34.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:25:33 smithi083 ceph-mon[77242]: pgmap v5320: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:34.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:25:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:25:34.185Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:25:35.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:25:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:25:34.880Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:25:36.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:25:36 smithi153 ceph-mon[70044]: pgmap v5321: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:36.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:25:36 smithi083 ceph-mon[79930]: pgmap v5321: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:36.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:25:36 smithi083 ceph-mon[77242]: pgmap v5321: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:38.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:25:38 smithi153 ceph-mon[70044]: pgmap v5322: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:38.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:25:38 smithi083 ceph-mon[77242]: pgmap v5322: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:38.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:25:38 smithi083 ceph-mon[79930]: pgmap v5322: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:25:40 smithi153 ceph-mon[70044]: pgmap v5323: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:25:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:25:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:25:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:25:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:25:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:25:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:25:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:25:40.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:25:40 smithi083 ceph-mon[79930]: pgmap v5323: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:40.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:25:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:25:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:25:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:25:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:25:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:25:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:25:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:25:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:25:40 smithi083 ceph-mon[77242]: pgmap v5323: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:25:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:25:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:25:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:25:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:25:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:25:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:25:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:25:41.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:25:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:25:40.800Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:25:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:25:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:25:41] "GET /metrics HTTP/1.1" 200 225216 "" "Prometheus/2.33.4" 2022-09-29T08:25:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:25:42 smithi153 ceph-mon[70044]: pgmap v5324: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:42.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:25:42 smithi083 ceph-mon[77242]: pgmap v5324: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:25:42 smithi083 ceph-mon[79930]: pgmap v5324: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:25:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:25:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:25:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:25:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:25:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:25:43.308 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:25:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:25:43.076Z caller=compact.go:519 level=info component=tsdb msg="write block" mint=1664429142218 maxt=1664431200000 ulid=01GE45KZVE75F92ZBKVQQ0RAXZ duration=53.953255ms 2022-09-29T08:25:43.308 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:25:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:25:43.084Z caller=head.go:827 level=info component=tsdb msg="Head GC completed" duration=7.481967ms 2022-09-29T08:25:43.308 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:25:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:25:43.085Z caller=checkpoint.go:98 level=info component=tsdb msg="Creating checkpoint" from_segment=0 to_segment=1 mint=1664431200000 2022-09-29T08:25:43.309 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:25:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:25:43.090Z caller=head.go:996 level=info component=tsdb msg="WAL checkpoint complete" first=0 last=1 duration=5.7817ms 2022-09-29T08:25:44.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:25:44 smithi153 ceph-mon[70044]: pgmap v5325: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:44.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:25:44 smithi083 ceph-mon[79930]: pgmap v5325: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:44.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:25:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:25:44.185Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:25:44.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:25:44 smithi083 ceph-mon[77242]: pgmap v5325: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:45.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:25:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:25:44.881Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:25:46.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:25:46 smithi153 ceph-mon[70044]: pgmap v5326: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:46.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:25:46 smithi083 ceph-mon[79930]: pgmap v5326: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:46.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:25:46 smithi083 ceph-mon[77242]: pgmap v5326: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:48.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:25:48 smithi153 ceph-mon[70044]: pgmap v5327: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:48.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:25:48 smithi083 ceph-mon[79930]: pgmap v5327: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:48.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:25:48 smithi083 ceph-mon[77242]: pgmap v5327: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:50.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:25:50 smithi153 ceph-mon[70044]: pgmap v5328: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:50.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:25:50 smithi083 ceph-mon[79930]: pgmap v5328: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:50.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:25:50 smithi083 ceph-mon[77242]: pgmap v5328: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:51.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:25:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:25:50.800Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:25:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:25:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:25:51] "GET /metrics HTTP/1.1" 200 225216 "" "Prometheus/2.33.4" 2022-09-29T08:25:52.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:25:52 smithi153 ceph-mon[70044]: pgmap v5329: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:52.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:25:52 smithi083 ceph-mon[79930]: pgmap v5329: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:52.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:25:52 smithi083 ceph-mon[77242]: pgmap v5329: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:25:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:25:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:25:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:25:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:25:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:25:54.275 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:25:54.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:25:54 smithi153 ceph-mon[70044]: pgmap v5330: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:54.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:25:54 smithi083 ceph-mon[79930]: pgmap v5330: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:54.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:25:54 smithi083 ceph-mon[77242]: pgmap v5330: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:54.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:25:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:25:54.185Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:25:54.564 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:25:54.564 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (2h) 8m ago 3h 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:25:54.565 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (2h) 6m ago 2h 62.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:25:54.565 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 8m ago 3h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:25:54.565 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 6m ago 3h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:25:54.565 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 6m ago 3h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:25:54.566 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 8m ago 3h 662M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:25:54.566 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 8m ago 3h 675M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:25:54.566 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 6m ago 3h 298M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:25:54.566 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 8m ago 3h 394M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:25:54.566 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 8m ago 3h 26.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:25:54.567 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 6m ago 3h 25.8M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:25:54.567 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 8m ago 3h 488M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:25:54.567 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 8m ago 3h 479M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:25:54.567 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 8m ago 3h 378M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:25:54.568 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 8m ago 3h 465M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:25:54.568 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 6m ago 3h 414M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:25:54.568 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 6m ago 3h 510M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:25:54.568 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 6m ago 3h 454M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:25:54.568 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 6m ago 3h 532M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:25:54.569 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 6m ago 3h 141M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:25:54.922 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:25:54.922 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:25:54.922 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:25:54.922 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:25:54.923 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:25:54.923 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:25:54.923 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:25:54.923 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:25:54.924 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:25:54.924 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:25:54.924 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:25:54.924 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:25:54.924 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:25:54.925 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:25:54.925 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:25:54.925 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:25:54.925 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:25:54.925 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:25:55.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:25:55 smithi083 ceph-mon[79930]: from='client.40335 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:25:55.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:25:55 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/503855447' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:25:55.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:25:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:25:54.880Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:25:55.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:25:55 smithi083 ceph-mon[77242]: from='client.40335 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:25:55.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:25:55 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/503855447' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:25:55.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:25:55 smithi153 ceph-mon[70044]: from='client.40335 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:25:55.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:25:55 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/503855447' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:25:56.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:25:56 smithi153 ceph-mon[70044]: from='client.50272 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:25:56.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:25:56 smithi153 ceph-mon[70044]: pgmap v5331: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:56.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:25:56 smithi083 ceph-mon[79930]: from='client.50272 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:25:56.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:25:56 smithi083 ceph-mon[79930]: pgmap v5331: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:56.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:25:56 smithi083 ceph-mon[77242]: from='client.50272 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:25:56.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:25:56 smithi083 ceph-mon[77242]: pgmap v5331: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:58.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:25:58 smithi153 ceph-mon[70044]: pgmap v5332: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:58.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:25:58 smithi083 ceph-mon[77242]: pgmap v5332: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:25:58.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:25:58 smithi083 ceph-mon[79930]: pgmap v5332: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:26:00 smithi153 ceph-mon[70044]: pgmap v5333: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:00.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:26:00 smithi083 ceph-mon[79930]: pgmap v5333: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:00.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:26:00 smithi083 ceph-mon[77242]: pgmap v5333: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:01.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:26:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:26:00.800Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:26:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:26:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:26:01] "GET /metrics HTTP/1.1" 200 225217 "" "Prometheus/2.33.4" 2022-09-29T08:26:02.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:26:02 smithi153 ceph-mon[70044]: pgmap v5334: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:02.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:26:02 smithi083 ceph-mon[79930]: pgmap v5334: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:02.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:26:02 smithi083 ceph-mon[77242]: pgmap v5334: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:26:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:26:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:26:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:26:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:26:04.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:26:04 smithi153 ceph-mon[70044]: pgmap v5335: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:04.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:26:04 smithi083 ceph-mon[79930]: pgmap v5335: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:04.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:26:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:26:04.186Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:26:04.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:26:04 smithi083 ceph-mon[77242]: pgmap v5335: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:05.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:26:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:26:04.882Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:26:06.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:26:06 smithi083 ceph-mon[79930]: pgmap v5336: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:06.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:26:06 smithi083 ceph-mon[77242]: pgmap v5336: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:06.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:26:06 smithi153 ceph-mon[70044]: pgmap v5336: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:08.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:26:08 smithi083 ceph-mon[79930]: pgmap v5337: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:08.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:26:08 smithi083 ceph-mon[77242]: pgmap v5337: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:26:08 smithi153 ceph-mon[70044]: pgmap v5337: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:10.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:26:10 smithi083 ceph-mon[79930]: pgmap v5338: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:10.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:26:10 smithi083 ceph-mon[77242]: pgmap v5338: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:10.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:26:10 smithi153 ceph-mon[70044]: pgmap v5338: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:11.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:26:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:26:10.801Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:26:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:26:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:26:11] "GET /metrics HTTP/1.1" 200 225218 "" "Prometheus/2.33.4" 2022-09-29T08:26:12.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:26:12 smithi083 ceph-mon[79930]: pgmap v5339: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:12.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:26:12 smithi083 ceph-mon[77242]: pgmap v5339: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:26:12 smithi153 ceph-mon[70044]: pgmap v5339: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:12.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:26:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:26:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:26:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:26:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:26:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:26:14.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:26:14 smithi083 ceph-mon[77242]: pgmap v5340: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:14.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:26:14 smithi083 ceph-mon[79930]: pgmap v5340: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:14.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:26:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:26:14.187Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:26:14.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:26:14 smithi153 ceph-mon[70044]: pgmap v5340: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:15.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:26:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:26:14.883Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:26:16.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:26:16 smithi083 ceph-mon[79930]: pgmap v5341: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:16.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:26:16 smithi083 ceph-mon[77242]: pgmap v5341: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:26:16 smithi153 ceph-mon[70044]: pgmap v5341: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:18.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:26:18 smithi083 ceph-mon[79930]: pgmap v5342: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:18.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:26:18 smithi083 ceph-mon[77242]: pgmap v5342: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:26:18 smithi153 ceph-mon[70044]: pgmap v5342: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:20.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:26:20 smithi083 ceph-mon[79930]: pgmap v5343: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:20.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:26:20 smithi083 ceph-mon[77242]: pgmap v5343: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:26:20 smithi153 ceph-mon[70044]: pgmap v5343: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:21.085 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:26:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:26:20.801Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:26:21.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:26:21 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:26:21.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:26:21 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:26:21.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:26:21 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:26:21.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:26:21 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:26:21.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:26:21 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:26:21.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:26:21 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:26:21.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:26:21 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:26:21.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:26:21 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:26:21.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:26:21 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:26:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:26:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:26:21] "GET /metrics HTTP/1.1" 200 225218 "" "Prometheus/2.33.4" 2022-09-29T08:26:22.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:26:22 smithi083 ceph-mon[79930]: pgmap v5344: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:22.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:26:22 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:26:22.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:26:22 smithi083 ceph-mon[77242]: pgmap v5344: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:22.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:26:22 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:26:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:26:22 smithi153 ceph-mon[70044]: pgmap v5344: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:26:22 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:26:22.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:26:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:26:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:26:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:26:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:26:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:26:24.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:26:24 smithi083 ceph-mon[77242]: pgmap v5345: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:24.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:26:24 smithi083 ceph-mon[79930]: pgmap v5345: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:24.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:26:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:26:24.187Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:26:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:26:24 smithi153 ceph-mon[70044]: pgmap v5345: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:25.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:26:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:26:24.883Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:26:25.252 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:26:25.541 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:26:25.541 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 9m ago 3h 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:26:25.541 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 7m ago 3h 62.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:26:25.542 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 9m ago 3h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:26:25.542 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 7m ago 3h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:26:25.542 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 7m ago 3h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:26:25.542 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 9m ago 3h 662M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:26:25.543 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 9m ago 3h 675M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:26:25.543 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 7m ago 3h 298M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:26:25.543 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 9m ago 3h 394M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:26:25.543 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 9m ago 3h 26.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:26:25.543 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 7m ago 3h 25.8M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:26:25.544 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 9m ago 3h 488M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:26:25.544 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 9m ago 3h 479M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:26:25.544 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 9m ago 3h 378M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:26:25.544 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 9m ago 3h 465M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:26:25.544 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 7m ago 3h 414M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:26:25.545 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 7m ago 3h 510M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:26:25.545 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 7m ago 3h 454M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:26:25.545 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 7m ago 3h 532M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:26:25.545 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 7m ago 3h 141M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:26:25.898 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:26:25.898 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:26:25.899 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:26:25.899 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:26:25.899 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:26:25.899 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:26:25.899 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:26:25.900 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:26:25.900 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:26:25.900 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:26:25.900 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:26:25.900 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:26:25.901 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:26:25.901 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:26:25.901 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:26:25.901 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:26:25.902 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:26:25.902 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:26:26.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:26:26 smithi083 ceph-mon[77242]: pgmap v5346: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:26.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:26:26 smithi083 ceph-mon[77242]: from='client.50284 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:26:26.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:26:26 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3140207452' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:26:26.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:26:26 smithi083 ceph-mon[79930]: pgmap v5346: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:26.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:26:26 smithi083 ceph-mon[79930]: from='client.50284 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:26:26.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:26:26 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3140207452' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:26:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:26:26 smithi153 ceph-mon[70044]: pgmap v5346: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:26:26 smithi153 ceph-mon[70044]: from='client.50284 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:26:26.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:26:26 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3140207452' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:26:27.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:26:27 smithi083 ceph-mon[79930]: from='client.50290 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:26:27.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:26:27 smithi083 ceph-mon[77242]: from='client.50290 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:26:27.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:26:27 smithi153 ceph-mon[70044]: from='client.50290 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:26:28.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:26:28 smithi083 ceph-mon[79930]: pgmap v5347: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:28.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:26:28 smithi083 ceph-mon[77242]: pgmap v5347: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:26:28 smithi153 ceph-mon[70044]: pgmap v5347: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:30.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:26:30 smithi083 ceph-mon[77242]: pgmap v5348: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:30.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:26:30 smithi083 ceph-mon[79930]: pgmap v5348: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:26:30 smithi153 ceph-mon[70044]: pgmap v5348: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:31.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:26:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:26:30.802Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:26:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:26:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:26:31] "GET /metrics HTTP/1.1" 200 225218 "" "Prometheus/2.33.4" 2022-09-29T08:26:32.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:26:32 smithi083 ceph-mon[77242]: pgmap v5349: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:32.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:26:32 smithi083 ceph-mon[79930]: pgmap v5349: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:26:32 smithi153 ceph-mon[70044]: pgmap v5349: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:32.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:26:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:26:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:26:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:26:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:26:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:26:34.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:26:34 smithi083 ceph-mon[79930]: pgmap v5350: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:34.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:26:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:26:34.188Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:26:34.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:26:34 smithi083 ceph-mon[77242]: pgmap v5350: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:26:34 smithi153 ceph-mon[70044]: pgmap v5350: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:35.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:26:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:26:34.884Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:26:36.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:26:36 smithi083 ceph-mon[79930]: pgmap v5351: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:36.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:26:36 smithi083 ceph-mon[77242]: pgmap v5351: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:26:36 smithi153 ceph-mon[70044]: pgmap v5351: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:38.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:26:38 smithi083 ceph-mon[79930]: pgmap v5352: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:38.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:26:38 smithi083 ceph-mon[77242]: pgmap v5352: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:26:38 smithi153 ceph-mon[70044]: pgmap v5352: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:40.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:26:40 smithi083 ceph-mon[79930]: pgmap v5353: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:26:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:26:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:26:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:26:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:26:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:26:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:26:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:26:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:26:40 smithi083 ceph-mon[77242]: pgmap v5353: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:26:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:26:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:26:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:26:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:26:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:26:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:26:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:26:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:26:40 smithi153 ceph-mon[70044]: pgmap v5353: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:26:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:26:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:26:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:26:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:26:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:26:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:26:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:26:41.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:26:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:26:40.803Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:26:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:26:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:26:41] "GET /metrics HTTP/1.1" 200 225213 "" "Prometheus/2.33.4" 2022-09-29T08:26:42.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:26:42 smithi083 ceph-mon[79930]: pgmap v5354: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:42.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:26:42 smithi083 ceph-mon[77242]: pgmap v5354: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:26:42 smithi153 ceph-mon[70044]: pgmap v5354: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:42.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:26:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:26:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:26:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:26:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:26:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:26:44.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:26:44 smithi083 ceph-mon[79930]: pgmap v5355: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:44.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:26:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:26:44.189Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:26:44.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:26:44 smithi083 ceph-mon[77242]: pgmap v5355: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:44.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:26:44 smithi153 ceph-mon[70044]: pgmap v5355: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:45.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:26:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:26:44.884Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:26:46.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:26:46 smithi083 ceph-mon[79930]: pgmap v5356: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:46.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:26:46 smithi083 ceph-mon[77242]: pgmap v5356: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:46.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:26:46 smithi153 ceph-mon[70044]: pgmap v5356: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:48.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:26:48 smithi083 ceph-mon[79930]: pgmap v5357: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:48.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:26:48 smithi083 ceph-mon[77242]: pgmap v5357: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:48.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:26:48 smithi153 ceph-mon[70044]: pgmap v5357: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:50.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:26:50 smithi083 ceph-mon[77242]: pgmap v5358: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:50.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:26:50 smithi083 ceph-mon[79930]: pgmap v5358: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:26:50 smithi153 ceph-mon[70044]: pgmap v5358: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:51.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:26:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:26:50.804Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:26:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:26:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:26:51] "GET /metrics HTTP/1.1" 200 225213 "" "Prometheus/2.33.4" 2022-09-29T08:26:52.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:26:52 smithi083 ceph-mon[77242]: pgmap v5359: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:52.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:26:52 smithi083 ceph-mon[79930]: pgmap v5359: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:26:52 smithi153 ceph-mon[70044]: pgmap v5359: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:52.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:26:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:26:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:26:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:26:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:26:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:26:54.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:26:54 smithi083 ceph-mon[79930]: pgmap v5360: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:54.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:26:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:26:54.189Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:26:54.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:26:54 smithi083 ceph-mon[77242]: pgmap v5360: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:54.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:26:54 smithi153 ceph-mon[70044]: pgmap v5360: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:55.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:26:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:26:54.885Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:26:56.224 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:26:56.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:26:56 smithi083 ceph-mon[77242]: pgmap v5361: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:56.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:26:56 smithi083 ceph-mon[79930]: pgmap v5361: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:56.511 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:26:56.512 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 9m ago 3h 26.6M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:26:56.512 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 7m ago 3h 62.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:26:56.512 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 9m ago 3h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:26:56.513 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 7m ago 3h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:26:56.513 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 7m ago 3h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:26:56.513 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 9m ago 3h 662M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:26:56.513 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 9m ago 3h 675M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:26:56.514 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 7m ago 3h 298M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:26:56.514 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 9m ago 3h 394M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:26:56.514 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 9m ago 3h 26.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:26:56.514 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 7m ago 3h 25.8M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:26:56.515 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 9m ago 3h 488M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:26:56.515 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 9m ago 3h 479M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:26:56.515 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 9m ago 3h 378M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:26:56.515 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 9m ago 3h 465M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:26:56.516 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 7m ago 3h 414M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:26:56.516 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 7m ago 3h 510M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:26:56.516 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 7m ago 3h 454M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:26:56.517 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 7m ago 3h 532M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:26:56.517 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 7m ago 3h 141M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:26:56.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:26:56 smithi153 ceph-mon[70044]: pgmap v5361: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:56.865 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:26:56.866 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:26:56.866 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:26:56.866 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:26:56.866 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:26:56.867 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:26:56.867 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:26:56.867 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:26:56.867 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:26:56.867 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:26:56.868 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:26:56.868 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:26:56.868 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:26:56.868 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:26:56.868 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:26:56.869 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:26:56.869 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:26:56.869 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:26:57.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:26:57 smithi083 ceph-mon[77242]: from='client.50302 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:26:57.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:26:57 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/131424663' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:26:57.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:26:57 smithi083 ceph-mon[79930]: from='client.50302 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:26:57.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:26:57 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/131424663' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:26:57.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:26:57 smithi153 ceph-mon[70044]: from='client.50302 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:26:57.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:26:57 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/131424663' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:26:58.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:26:58 smithi083 ceph-mon[79930]: from='client.40377 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:26:58.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:26:58 smithi083 ceph-mon[79930]: pgmap v5362: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:58.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:26:58 smithi083 ceph-mon[77242]: from='client.40377 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:26:58.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:26:58 smithi083 ceph-mon[77242]: pgmap v5362: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:26:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:26:58 smithi153 ceph-mon[70044]: from='client.40377 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:26:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:26:58 smithi153 ceph-mon[70044]: pgmap v5362: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:00.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:27:00 smithi083 ceph-mon[79930]: pgmap v5363: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:00.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:27:00 smithi083 ceph-mon[77242]: pgmap v5363: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:27:00 smithi153 ceph-mon[70044]: pgmap v5363: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:01.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:27:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:27:00.804Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:27:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:27:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:27:01] "GET /metrics HTTP/1.1" 200 225213 "" "Prometheus/2.33.4" 2022-09-29T08:27:02.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:27:02 smithi083 ceph-mon[79930]: pgmap v5364: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:02.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:27:02 smithi083 ceph-mon[77242]: pgmap v5364: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:27:02 smithi153 ceph-mon[70044]: pgmap v5364: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:02.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:27:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:27:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:27:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:27:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:27:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:27:04.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:27:04 smithi083 ceph-mon[79930]: pgmap v5365: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:04.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:27:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:27:04.191Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:27:04.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:27:04 smithi083 ceph-mon[77242]: pgmap v5365: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:27:04 smithi153 ceph-mon[70044]: pgmap v5365: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:05.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:27:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:27:04.885Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:27:06.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:27:06 smithi083 ceph-mon[79930]: pgmap v5366: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:06.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:27:06 smithi083 ceph-mon[77242]: pgmap v5366: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:06.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:27:06 smithi153 ceph-mon[70044]: pgmap v5366: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:08.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:27:08 smithi083 ceph-mon[79930]: pgmap v5367: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:08.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:27:08 smithi083 ceph-mon[77242]: pgmap v5367: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:27:08 smithi153 ceph-mon[70044]: pgmap v5367: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:10.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:27:10 smithi083 ceph-mon[79930]: pgmap v5368: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:10.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:27:10 smithi083 ceph-mon[77242]: pgmap v5368: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:27:10 smithi153 ceph-mon[70044]: pgmap v5368: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:11.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:27:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:27:10.804Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:27:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:27:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:27:11] "GET /metrics HTTP/1.1" 200 225215 "" "Prometheus/2.33.4" 2022-09-29T08:27:12.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:27:12 smithi083 ceph-mon[77242]: pgmap v5369: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:12.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:27:12 smithi083 ceph-mon[79930]: pgmap v5369: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:27:12 smithi153 ceph-mon[70044]: pgmap v5369: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:12.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:27:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:27:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:27:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:27:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:27:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:27:14.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:27:14 smithi083 ceph-mon[79930]: pgmap v5370: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:14.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:27:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:27:14.191Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:27:14.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:27:14 smithi083 ceph-mon[77242]: pgmap v5370: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:14.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:27:14 smithi153 ceph-mon[70044]: pgmap v5370: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:15.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:27:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:27:14.885Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:27:16.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:27:16 smithi083 ceph-mon[79930]: pgmap v5371: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:16.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:27:16 smithi083 ceph-mon[77242]: pgmap v5371: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:27:16 smithi153 ceph-mon[70044]: pgmap v5371: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:18.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:27:18 smithi083 ceph-mon[79930]: pgmap v5372: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:18.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:27:18 smithi083 ceph-mon[77242]: pgmap v5372: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:27:18 smithi153 ceph-mon[70044]: pgmap v5372: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:20.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:27:20 smithi083 ceph-mon[79930]: pgmap v5373: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:20.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:27:20 smithi083 ceph-mon[77242]: pgmap v5373: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:27:20 smithi153 ceph-mon[70044]: pgmap v5373: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:21.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:27:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:27:20.805Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:27:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:27:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:27:21] "GET /metrics HTTP/1.1" 200 225215 "" "Prometheus/2.33.4" 2022-09-29T08:27:22.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:27:22 smithi083 ceph-mon[79930]: pgmap v5374: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:22.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:27:22 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:27:22.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:27:22 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:27:22.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:27:22 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:27:22.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:27:22 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:27:22.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:27:22 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:27:22.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:27:22 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:27:22.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:27:22 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:27:22.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:27:22 smithi083 ceph-mon[77242]: pgmap v5374: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:22.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:27:22 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:27:22.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:27:22 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:27:22.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:27:22 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:27:22.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:27:22 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:27:22.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:27:22 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:27:22.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:27:22 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:27:22.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:27:22 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:27:22.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:27:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:27:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:27:22.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:27:22 smithi153 ceph-mon[70044]: pgmap v5374: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:22.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:27:22 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:27:22.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:27:22 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:27:22.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:27:22 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:27:22.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:27:22 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:27:22.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:27:22 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:27:22.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:27:22 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:27:22.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:27:22 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:27:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:27:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:27:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:27:24.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:27:24 smithi083 ceph-mon[79930]: pgmap v5375: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:24.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:27:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:27:24.191Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:27:24.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:27:24 smithi083 ceph-mon[77242]: pgmap v5375: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:27:24 smithi153 ceph-mon[70044]: pgmap v5375: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:25.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:27:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:27:24.885Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:27:26.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:27:26 smithi083 ceph-mon[79930]: pgmap v5376: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:26.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:27:26 smithi083 ceph-mon[77242]: pgmap v5376: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:27:26 smithi153 ceph-mon[70044]: pgmap v5376: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:27.187 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:27:27.476 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:27:27.477 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 1s ago 3h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:27:27.477 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 8m ago 3h 62.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:27:27.477 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 1s ago 3h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:27:27.477 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 8m ago 3h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:27:27.478 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 8m ago 3h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:27:27.478 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 1s ago 3h 671M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:27:27.478 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 1s ago 3h 710M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:27:27.478 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 8m ago 3h 298M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:27:27.478 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 1s ago 3h 412M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:27:27.479 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 1s ago 3h 26.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:27:27.479 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 8m ago 3h 25.8M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:27:27.479 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 1s ago 3h 513M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:27:27.479 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 1s ago 3h 504M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:27:27.480 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 1s ago 3h 397M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:27:27.480 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 1s ago 3h 490M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:27:27.480 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 8m ago 3h 414M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:27:27.480 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 8m ago 3h 510M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:27:27.480 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 8m ago 3h 454M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:27:27.481 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 8m ago 3h 532M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:27:27.481 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 8m ago 3h 141M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:27:27.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:27:27 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:27:27.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:27:27 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:27:27.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:27:27 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:27:27.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:27:27 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:27:27.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:27:27 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:27:27.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:27:27 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:27:27.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:27:27 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:27:27.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:27:27 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:27:27.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:27:27 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:27:27.833 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:27:27.833 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:27:27.833 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:27:27.834 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:27:27.834 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:27:27.834 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:27:27.834 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:27:27.835 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:27:27.835 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:27:27.835 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:27:27.835 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:27:27.836 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:27:27.836 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:27:27.836 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:27:27.836 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:27:27.836 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:27:27.836 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:27:27.837 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:27:28.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:27:28 smithi083 ceph-mon[79930]: pgmap v5377: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:28.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:27:28 smithi083 ceph-mon[79930]: from='client.50320 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:27:28.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:27:28 smithi083 ceph-mon[79930]: from='client.50326 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:27:28.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:27:28 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2876283621' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:27:28.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:27:28 smithi083 ceph-mon[77242]: pgmap v5377: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:28.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:27:28 smithi083 ceph-mon[77242]: from='client.50320 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:27:28.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:27:28 smithi083 ceph-mon[77242]: from='client.50326 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:27:28.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:27:28 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2876283621' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:27:28.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:27:28 smithi153 ceph-mon[70044]: pgmap v5377: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:28.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:27:28 smithi153 ceph-mon[70044]: from='client.50320 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:27:28.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:27:28 smithi153 ceph-mon[70044]: from='client.50326 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:27:28.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:27:28 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2876283621' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:27:30.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:27:30 smithi083 ceph-mon[79930]: pgmap v5378: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:30.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:27:30 smithi083 ceph-mon[77242]: pgmap v5378: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:30.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:27:30 smithi153 ceph-mon[70044]: pgmap v5378: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:31.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:27:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:27:30.805Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:27:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:27:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:27:31] "GET /metrics HTTP/1.1" 200 225218 "" "Prometheus/2.33.4" 2022-09-29T08:27:32.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:27:32 smithi083 ceph-mon[79930]: pgmap v5379: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:32.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:27:32 smithi083 ceph-mon[77242]: pgmap v5379: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:32.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:27:32 smithi153 ceph-mon[70044]: pgmap v5379: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:27:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:27:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:27:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:27:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:27:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:27:34.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:27:34 smithi083 ceph-mon[79930]: pgmap v5380: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:34.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:27:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:27:34.193Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:27:34.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:27:34 smithi083 ceph-mon[77242]: pgmap v5380: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:34.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:27:34 smithi153 ceph-mon[70044]: pgmap v5380: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:35.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:27:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:27:34.887Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:27:36.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:27:36 smithi083 ceph-mon[79930]: pgmap v5381: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:36.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:27:36 smithi083 ceph-mon[77242]: pgmap v5381: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:36.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:27:36 smithi153 ceph-mon[70044]: pgmap v5381: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:38.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:27:38 smithi083 ceph-mon[79930]: pgmap v5382: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:38.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:27:38 smithi083 ceph-mon[77242]: pgmap v5382: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:38.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:27:38 smithi153 ceph-mon[70044]: pgmap v5382: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:40.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:27:40 smithi083 ceph-mon[79930]: pgmap v5383: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:40.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:27:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:27:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:27:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:27:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:27:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:27:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:27:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:27:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:27:40 smithi083 ceph-mon[77242]: pgmap v5383: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:27:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:27:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:27:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:27:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:27:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:27:40.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:27:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:27:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:27:40 smithi153 ceph-mon[70044]: pgmap v5383: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:27:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:27:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:27:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:27:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:27:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:27:40.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:27:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:27:41.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:27:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:27:40.806Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:27:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:27:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:27:41] "GET /metrics HTTP/1.1" 200 225216 "" "Prometheus/2.33.4" 2022-09-29T08:27:42.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:27:42 smithi083 ceph-mon[79930]: pgmap v5384: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:42.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:27:42 smithi083 ceph-mon[77242]: pgmap v5384: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:27:42 smithi153 ceph-mon[70044]: pgmap v5384: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:27:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:27:42.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:27:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:27:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:27:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:27:44.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:27:44 smithi083 ceph-mon[79930]: pgmap v5385: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:44.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:27:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:27:44.194Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:27:44.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:27:44 smithi083 ceph-mon[77242]: pgmap v5385: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:44.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:27:44 smithi153 ceph-mon[70044]: pgmap v5385: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:45.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:27:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:27:44.887Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:27:46.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:27:46 smithi083 ceph-mon[79930]: pgmap v5386: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:46.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:27:46 smithi083 ceph-mon[77242]: pgmap v5386: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:46.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:27:46 smithi153 ceph-mon[70044]: pgmap v5386: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:48.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:27:48 smithi083 ceph-mon[79930]: pgmap v5387: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:48.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:27:48 smithi083 ceph-mon[77242]: pgmap v5387: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:48.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:27:48 smithi153 ceph-mon[70044]: pgmap v5387: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:50.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:27:50 smithi083 ceph-mon[79930]: pgmap v5388: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:50.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:27:50 smithi083 ceph-mon[77242]: pgmap v5388: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:50.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:27:50 smithi153 ceph-mon[70044]: pgmap v5388: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:51.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:27:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:27:50.807Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:27:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:27:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:27:51] "GET /metrics HTTP/1.1" 200 225216 "" "Prometheus/2.33.4" 2022-09-29T08:27:52.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:27:52 smithi083 ceph-mon[79930]: pgmap v5389: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:52.764 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:27:52 smithi083 ceph-mon[77242]: pgmap v5389: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:52.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:27:52 smithi153 ceph-mon[70044]: pgmap v5389: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:27:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:27:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:27:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:27:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:27:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:27:54.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:27:54 smithi083 ceph-mon[79930]: pgmap v5390: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:54.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:27:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:27:54.194Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:27:54.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:27:54 smithi083 ceph-mon[77242]: pgmap v5390: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:54.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:27:54 smithi153 ceph-mon[70044]: pgmap v5390: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:55.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:27:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:27:54.888Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:27:56.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:27:56 smithi083 ceph-mon[79930]: pgmap v5391: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:56.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:27:56 smithi083 ceph-mon[77242]: pgmap v5391: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:56.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:27:56 smithi153 ceph-mon[70044]: pgmap v5391: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:58.161 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:27:58.450 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:27:58.451 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 32s ago 3h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:27:58.451 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 8m ago 3h 62.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:27:58.451 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 32s ago 3h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:27:58.452 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 8m ago 3h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:27:58.452 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 8m ago 3h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:27:58.452 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 32s ago 3h 671M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:27:58.452 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 32s ago 3h 710M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:27:58.452 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 8m ago 3h 298M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:27:58.453 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 32s ago 3h 412M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:27:58.453 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 32s ago 3h 26.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:27:58.453 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 8m ago 3h 25.8M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:27:58.453 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 32s ago 3h 513M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:27:58.454 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 32s ago 3h 504M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:27:58.454 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 32s ago 3h 397M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:27:58.454 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 32s ago 3h 490M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:27:58.454 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 8m ago 3h 414M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:27:58.455 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 8m ago 3h 510M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:27:58.455 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 8m ago 3h 454M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:27:58.455 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 8m ago 3h 532M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:27:58.455 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 8m ago 3h 141M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:27:58.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:27:58 smithi083 ceph-mon[79930]: pgmap v5392: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:58.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:27:58 smithi083 ceph-mon[77242]: pgmap v5392: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:58.803 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:27:58.804 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:27:58.804 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:27:58.804 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:27:58.804 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:27:58.805 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:27:58.805 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:27:58.805 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:27:58.805 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:27:58.806 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:27:58.806 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:27:58.806 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:27:58.806 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:27:58.807 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:27:58.807 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:27:58.807 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:27:58.807 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:27:58.807 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:27:58.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:27:58 smithi153 ceph-mon[70044]: pgmap v5392: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:27:59.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:27:59 smithi083 ceph-mon[79930]: from='client.50338 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:27:59.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:27:59 smithi083 ceph-mon[79930]: from='client.50344 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:27:59.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:27:59 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1464682766' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:27:59.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:27:59 smithi083 ceph-mon[77242]: from='client.50338 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:27:59.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:27:59 smithi083 ceph-mon[77242]: from='client.50344 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:27:59.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:27:59 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1464682766' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:27:59.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:27:59 smithi153 ceph-mon[70044]: from='client.50338 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:27:59.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:27:59 smithi153 ceph-mon[70044]: from='client.50344 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:27:59.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:27:59 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1464682766' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:28:00.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:28:00 smithi083 ceph-mon[79930]: pgmap v5393: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:28:00.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:28:00 smithi083 ceph-mon[77242]: pgmap v5393: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:28:00.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:28:00 smithi153 ceph-mon[70044]: pgmap v5393: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:28:01.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:28:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:28:00.807Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:28:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:28:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:28:01] "GET /metrics HTTP/1.1" 200 225214 "" "Prometheus/2.33.4" 2022-09-29T08:28:02.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:28:02 smithi083 ceph-mon[79930]: pgmap v5394: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:28:02.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:28:02 smithi083 ceph-mon[77242]: pgmap v5394: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:28:02.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:28:02 smithi153 ceph-mon[70044]: pgmap v5394: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:28:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:28:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:28:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:28:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:28:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:28:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:28:04.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:28:04 smithi083 ceph-mon[79930]: pgmap v5395: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:28:04.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:28:04 smithi083 ceph-mon[77242]: pgmap v5395: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:28:04.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:28:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:28:04.195Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:28:04.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:28:04 smithi153 ceph-mon[70044]: pgmap v5395: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:28:05.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:28:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:28:04.889Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:28:06.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:28:06 smithi083 ceph-mon[79930]: pgmap v5396: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:28:06.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:28:06 smithi083 ceph-mon[77242]: pgmap v5396: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:28:06.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:28:06 smithi153 ceph-mon[70044]: pgmap v5396: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:28:08.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:28:08 smithi083 ceph-mon[79930]: pgmap v5397: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:28:08.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:28:08 smithi083 ceph-mon[77242]: pgmap v5397: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:28:08.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:28:08 smithi153 ceph-mon[70044]: pgmap v5397: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:28:10.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:28:10 smithi083 ceph-mon[77242]: pgmap v5398: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:28:10.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:28:10 smithi083 ceph-mon[79930]: pgmap v5398: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:28:10.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:28:10 smithi153 ceph-mon[70044]: pgmap v5398: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:28:11.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:28:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:28:10.808Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:28:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:28:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:28:11] "GET /metrics HTTP/1.1" 200 225214 "" "Prometheus/2.33.4" 2022-09-29T08:28:12.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:28:12 smithi083 ceph-mon[79930]: pgmap v5399: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:28:12.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:28:12 smithi083 ceph-mon[77242]: pgmap v5399: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:28:12.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:28:12 smithi153 ceph-mon[70044]: pgmap v5399: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:28:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:28:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:28:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:28:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:28:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:28:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:28:14.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:28:14 smithi083 ceph-mon[79930]: pgmap v5400: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:28:14.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:28:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:28:14.196Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:28:14.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:28:14 smithi083 ceph-mon[77242]: pgmap v5400: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:28:14.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:28:14 smithi153 ceph-mon[70044]: pgmap v5400: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:28:15.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:28:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:28:14.889Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:28:16.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:28:16 smithi083 ceph-mon[79930]: pgmap v5401: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:28:16.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:28:16 smithi083 ceph-mon[77242]: pgmap v5401: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:28:16.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:28:16 smithi153 ceph-mon[70044]: pgmap v5401: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:28:18.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:28:18 smithi083 ceph-mon[79930]: pgmap v5402: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:28:18.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:28:18 smithi083 ceph-mon[77242]: pgmap v5402: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:28:18.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:28:18 smithi153 ceph-mon[70044]: pgmap v5402: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:28:20.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:28:20 smithi083 ceph-mon[77242]: pgmap v5403: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:28:20.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:28:20 smithi083 ceph-mon[79930]: pgmap v5403: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:28:20.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:28:20 smithi153 ceph-mon[70044]: pgmap v5403: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:28:21.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:28:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:28:20.808Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:28:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:28:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:28:21] "GET /metrics HTTP/1.1" 200 225214 "" "Prometheus/2.33.4" 2022-09-29T08:28:22.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:28:22 smithi083 ceph-mon[79930]: pgmap v5404: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:28:22.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:28:22 smithi083 ceph-mon[77242]: pgmap v5404: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:28:22.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:28:22 smithi153 ceph-mon[70044]: pgmap v5404: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:28:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:28:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:28:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:28:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:28:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:28:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:28:24.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:28:24 smithi083 ceph-mon[79930]: pgmap v5405: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:28:24.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:28:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:28:24.196Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:28:24.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:28:24 smithi083 ceph-mon[77242]: pgmap v5405: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:28:24.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:28:24 smithi153 ceph-mon[70044]: pgmap v5405: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:28:25.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:28:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:28:24.890Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:28:26.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:28:26 smithi083 ceph-mon[79930]: pgmap v5406: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:28:26.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:28:26 smithi083 ceph-mon[77242]: pgmap v5406: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:28:26.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:28:26 smithi153 ceph-mon[70044]: pgmap v5406: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:28:27.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:28:27 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:28:27.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:28:27 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:28:27.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:28:27 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:28:27.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:28:27 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:28:27.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:28:27 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:28:27.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:28:27 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:28:27.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:28:27 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:28:27.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:28:27 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:28:27.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:28:27 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:28:28.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:28:28 smithi083 ceph-mon[79930]: pgmap v5407: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:28:28.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:28:28 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:28:28.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:28:28 smithi083 ceph-mon[77242]: pgmap v5407: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:28:28.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:28:28 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:28:28.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:28:28 smithi153 ceph-mon[70044]: pgmap v5407: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:28:28.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:28:28 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:28:29.132 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:28:29.423 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:28:29.424 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 63s ago 3h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:28:29.424 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 9m ago 3h 62.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:28:29.424 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 63s ago 3h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:28:29.424 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 9m ago 3h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:28:29.425 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 9m ago 3h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:28:29.425 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (2h) 63s ago 3h 671M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:28:29.425 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 63s ago 3h 710M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:28:29.425 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 9m ago 3h 298M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:28:29.426 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 63s ago 3h 412M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:28:29.426 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 63s ago 3h 26.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:28:29.426 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 9m ago 3h 25.8M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:28:29.426 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 63s ago 3h 513M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:28:29.427 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 63s ago 3h 504M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:28:29.427 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 63s ago 3h 397M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:28:29.427 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 63s ago 3h 490M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:28:29.427 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 9m ago 3h 414M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:28:29.427 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 9m ago 3h 510M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:28:29.428 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 9m ago 3h 454M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:28:29.428 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 9m ago 3h 532M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:28:29.428 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 9m ago 3h 141M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:28:29.782 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:28:29.783 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:28:29.783 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:28:29.783 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:28:29.783 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:28:29.784 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:28:29.784 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:28:29.784 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:28:29.784 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:28:29.784 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:28:29.785 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:28:29.785 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:28:29.785 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:28:29.785 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:28:29.786 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:28:29.786 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:28:29.786 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:28:29.786 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:28:30.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:28:30 smithi083 ceph-mon[79930]: pgmap v5408: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:28:30.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:28:30 smithi083 ceph-mon[79930]: from='client.40425 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:28:30.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:28:30 smithi083 ceph-mon[79930]: from='client.40431 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:28:30.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:28:30 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2669094913' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:28:30.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:28:30 smithi083 ceph-mon[77242]: pgmap v5408: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:28:30.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:28:30 smithi083 ceph-mon[77242]: from='client.40425 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:28:30.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:28:30 smithi083 ceph-mon[77242]: from='client.40431 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:28:30.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:28:30 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2669094913' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:28:30.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:28:30 smithi153 ceph-mon[70044]: pgmap v5408: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:28:30.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:28:30 smithi153 ceph-mon[70044]: from='client.40425 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:28:30.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:28:30 smithi153 ceph-mon[70044]: from='client.40431 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:28:30.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:28:30 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2669094913' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:28:31.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:28:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:28:30.809Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:28:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:28:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:28:31] "GET /metrics HTTP/1.1" 200 225213 "" "Prometheus/2.33.4" 2022-09-29T08:28:32.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:28:32 smithi083 ceph-mon[79930]: pgmap v5409: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:28:32.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:28:32 smithi083 ceph-mon[77242]: pgmap v5409: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:28:32.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:28:32 smithi153 ceph-mon[70044]: pgmap v5409: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:28:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:28:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:28:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:28:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:28:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:28:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:28:34.457 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:28:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:28:34.197Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:28:34.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:28:34 smithi083 ceph-mon[79930]: pgmap v5410: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:28:34.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:28:34 smithi083 ceph-mon[77242]: pgmap v5410: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:28:34.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:28:34 smithi153 ceph-mon[70044]: pgmap v5410: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:28:35.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:28:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:28:34.891Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:28:36.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:28:36 smithi083 ceph-mon[79930]: pgmap v5411: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:28:36.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:28:36 smithi083 ceph-mon[77242]: pgmap v5411: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:28:36.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:28:36 smithi153 ceph-mon[70044]: pgmap v5411: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:28:38.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:28:38 smithi153 ceph-mon[70044]: pgmap v5412: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:28:38.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:28:38 smithi083 ceph-mon[79930]: pgmap v5412: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:28:38.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:28:38 smithi083 ceph-mon[77242]: pgmap v5412: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:28:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:28:40 smithi153 ceph-mon[70044]: pgmap v5413: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:28:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:28:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:28:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:28:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:28:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:28:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:28:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:28:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:28:40.812 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:28:40 smithi083 ceph-mon[77242]: pgmap v5413: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:28:40.812 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:28:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:28:40.812 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:28:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:28:40.812 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:28:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:28:40.813 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:28:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:28:40.813 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:28:40 smithi083 ceph-mon[79930]: pgmap v5413: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:28:40.813 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:28:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:28:40.813 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:28:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:28:40.814 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:28:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:28:40.814 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:28:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:28:41.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:28:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:28:40.811Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:28:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:28:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:28:41] "GET /metrics HTTP/1.1" 200 225220 "" "Prometheus/2.33.4" 2022-09-29T08:28:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:28:42 smithi153 ceph-mon[70044]: pgmap v5414: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:28:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:28:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:28:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:28:42.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:28:42 smithi083 ceph-mon[79930]: pgmap v5414: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:28:42.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:28:42 smithi083 ceph-mon[77242]: pgmap v5414: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:28:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:28:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:28:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:28:44.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:28:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:28:44.199Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:28:44.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:28:44 smithi153 ceph-mon[70044]: pgmap v5415: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:28:44.893 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:28:44 smithi083 ceph-mon[79930]: pgmap v5415: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:28:44.894 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:28:44 smithi083 ceph-mon[77242]: pgmap v5415: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:28:45.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:28:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:28:44.892Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:28:46.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:28:46 smithi153 ceph-mon[70044]: pgmap v5416: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:28:46.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:28:46 smithi083 ceph-mon[79930]: pgmap v5416: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:28:46.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:28:46 smithi083 ceph-mon[77242]: pgmap v5416: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:28:48.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:28:48 smithi153 ceph-mon[70044]: pgmap v5417: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:28:48.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:28:48 smithi083 ceph-mon[79930]: pgmap v5417: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:28:48.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:28:48 smithi083 ceph-mon[77242]: pgmap v5417: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:28:50.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:28:50 smithi153 ceph-mon[70044]: pgmap v5418: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:28:50.813 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:28:50 smithi083 ceph-mon[79930]: pgmap v5418: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:28:50.813 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:28:50 smithi083 ceph-mon[77242]: pgmap v5418: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:28:51.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:28:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:28:50.811Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:28:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:28:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:28:51] "GET /metrics HTTP/1.1" 200 225220 "" "Prometheus/2.33.4" 2022-09-29T08:28:52.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:28:52 smithi153 ceph-mon[70044]: pgmap v5419: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:28:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:28:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:28:52.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:28:52.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:28:52 smithi083 ceph-mon[79930]: pgmap v5419: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:28:52.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:28:52 smithi083 ceph-mon[77242]: pgmap v5419: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:28:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:28:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:28:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:28:54.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:28:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:28:54.199Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:28:54.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:28:54 smithi153 ceph-mon[70044]: pgmap v5420: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:28:54.894 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:28:54 smithi083 ceph-mon[79930]: pgmap v5420: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:28:54.894 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:28:54 smithi083 ceph-mon[77242]: pgmap v5420: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:28:55.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:28:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:28:54.892Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:28:55.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:28:55 smithi153 ceph-mon[70044]: pgmap v5421: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:28:55.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:28:55 smithi083 ceph-mon[79930]: pgmap v5421: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:28:55.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:28:55 smithi083 ceph-mon[77242]: pgmap v5421: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:28:58.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:28:58 smithi153 ceph-mon[70044]: pgmap v5422: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:28:58.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:28:58 smithi083 ceph-mon[79930]: pgmap v5422: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:28:58.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:28:58 smithi083 ceph-mon[77242]: pgmap v5422: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:00.110 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:29:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:29:00 smithi153 ceph-mon[70044]: pgmap v5423: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:00.398 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:29:00.398 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 94s ago 3h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:29:00.398 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 9m ago 3h 62.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:29:00.399 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 94s ago 3h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:29:00.399 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 9m ago 3h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:29:00.399 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (2h) 9m ago 3h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:29:00.399 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 94s ago 3h 671M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:29:00.400 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (2h) 94s ago 3h 710M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:29:00.400 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 9m ago 3h 298M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:29:00.400 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 94s ago 3h 412M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:29:00.400 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 94s ago 3h 26.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:29:00.400 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 9m ago 3h 25.8M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:29:00.401 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 94s ago 3h 513M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:29:00.401 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 94s ago 3h 504M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:29:00.401 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 94s ago 3h 397M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:29:00.401 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 94s ago 3h 490M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:29:00.402 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 9m ago 3h 414M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:29:00.402 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 9m ago 3h 510M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:29:00.402 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 9m ago 3h 454M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:29:00.402 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 9m ago 3h 532M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:29:00.402 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (2h) 9m ago 3h 141M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:29:00.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:29:00 smithi083 ceph-mon[79930]: pgmap v5423: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:00.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:29:00 smithi083 ceph-mon[77242]: pgmap v5423: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:00.756 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:29:00.756 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:29:00.756 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:29:00.757 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:29:00.757 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:29:00.757 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:29:00.757 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:29:00.758 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:29:00.758 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:29:00.758 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:29:00.758 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:29:00.759 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:29:00.759 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:29:00.759 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:29:00.759 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:29:00.759 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:29:00.760 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:29:00.760 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:29:01.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:29:01 smithi083 ceph-mon[77242]: from='client.40443 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:29:01.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:29:01 smithi083 ceph-mon[77242]: from='client.40449 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:29:01.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:29:01 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/38436338' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:29:01.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:29:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:29:00.811Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:29:01.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:29:01 smithi083 ceph-mon[79930]: from='client.40443 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:29:01.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:29:01 smithi083 ceph-mon[79930]: from='client.40449 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:29:01.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:29:01 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/38436338' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:29:01.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:29:01 smithi153 ceph-mon[70044]: from='client.40443 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:29:01.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:29:01 smithi153 ceph-mon[70044]: from='client.40449 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:29:01.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:29:01 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/38436338' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:29:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:29:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:29:01] "GET /metrics HTTP/1.1" 200 225214 "" "Prometheus/2.33.4" 2022-09-29T08:29:02.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:29:02 smithi153 ceph-mon[70044]: pgmap v5424: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:02.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:29:02 smithi083 ceph-mon[77242]: pgmap v5424: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:02.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:29:02 smithi083 ceph-mon[79930]: pgmap v5424: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:29:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:29:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:29:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:29:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:29:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:29:04.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:29:04 smithi153 ceph-mon[70044]: pgmap v5425: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:04.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:29:04 smithi083 ceph-mon[79930]: pgmap v5425: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:04.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:29:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:29:04.201Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:29:04.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:29:04 smithi083 ceph-mon[77242]: pgmap v5425: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:05.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:29:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:29:04.894Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:29:06.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:29:06 smithi153 ceph-mon[70044]: pgmap v5426: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:06.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:29:06 smithi083 ceph-mon[79930]: pgmap v5426: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:06.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:29:06 smithi083 ceph-mon[77242]: pgmap v5426: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:08.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:29:08 smithi153 ceph-mon[70044]: pgmap v5427: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:08.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:29:08 smithi083 ceph-mon[79930]: pgmap v5427: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:08.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:29:08 smithi083 ceph-mon[77242]: pgmap v5427: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:10.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:29:10 smithi083 ceph-mon[79930]: pgmap v5428: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:10.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:29:10 smithi083 ceph-mon[77242]: pgmap v5428: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:29:10 smithi153 ceph-mon[70044]: pgmap v5428: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:11.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:29:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:29:10.813Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:29:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:29:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:29:11] "GET /metrics HTTP/1.1" 200 225217 "" "Prometheus/2.33.4" 2022-09-29T08:29:12.315 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:29:12 smithi153 ceph-mon[70044]: pgmap v5429: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:12.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:29:12 smithi083 ceph-mon[77242]: pgmap v5429: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:12.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:29:12 smithi083 ceph-mon[79930]: pgmap v5429: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:29:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:29:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:29:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:29:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:29:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:29:14.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:29:14 smithi083 ceph-mon[77242]: pgmap v5430: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:14.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:29:14 smithi083 ceph-mon[79930]: pgmap v5430: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:14.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:29:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:29:14.201Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:29:14.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:29:14 smithi153 ceph-mon[70044]: pgmap v5430: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:15.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:29:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:29:14.894Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:29:16.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:29:16 smithi083 ceph-mon[79930]: pgmap v5431: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:16.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:29:16 smithi083 ceph-mon[77242]: pgmap v5431: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:29:16 smithi153 ceph-mon[70044]: pgmap v5431: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:18.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:29:18 smithi083 ceph-mon[77242]: pgmap v5432: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:18.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:29:18 smithi083 ceph-mon[79930]: pgmap v5432: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:29:18 smithi153 ceph-mon[70044]: pgmap v5432: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:20.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:29:20 smithi083 ceph-mon[79930]: pgmap v5433: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:20.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:29:20 smithi083 ceph-mon[77242]: pgmap v5433: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:29:20 smithi153 ceph-mon[70044]: pgmap v5433: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:21.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:29:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:29:20.814Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:29:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:29:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:29:21] "GET /metrics HTTP/1.1" 200 225217 "" "Prometheus/2.33.4" 2022-09-29T08:29:22.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:29:22 smithi083 ceph-mon[77242]: pgmap v5434: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:22.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:29:22 smithi083 ceph-mon[79930]: pgmap v5434: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:29:22 smithi153 ceph-mon[70044]: pgmap v5434: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:22.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:29:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:29:22.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:29:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:29:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:29:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:29:24.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:29:24 smithi083 ceph-mon[79930]: pgmap v5435: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:24.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:29:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:29:24.201Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:29:24.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:29:24 smithi083 ceph-mon[77242]: pgmap v5435: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:29:24 smithi153 ceph-mon[70044]: pgmap v5435: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:25.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:29:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:29:24.895Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:29:26.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:29:26 smithi083 ceph-mon[79930]: pgmap v5436: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:26.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:29:26 smithi083 ceph-mon[77242]: pgmap v5436: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:29:26 smithi153 ceph-mon[70044]: pgmap v5436: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:28.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:29:28 smithi083 ceph-mon[79930]: pgmap v5437: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:28.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:29:28 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:29:28.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:29:28 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:29:28.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:29:28 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:29:28.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:29:28 smithi083 ceph-mon[77242]: pgmap v5437: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:28.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:29:28 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:29:28.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:29:28 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:29:28.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:29:28 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:29:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:29:28 smithi153 ceph-mon[70044]: pgmap v5437: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:28.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:29:28 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:29:28.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:29:28 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:29:28.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:29:28 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:29:30.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:29:30 smithi083 ceph-mon[79930]: pgmap v5438: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:30.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:29:30 smithi083 ceph-mon[77242]: pgmap v5438: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:29:30 smithi153 ceph-mon[70044]: pgmap v5438: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:31.081 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:29:31.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:29:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:29:30.814Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:29:31.371 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:29:31.371 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 2m ago 3h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:29:31.371 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 10m ago 3h 62.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:29:31.371 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 2m ago 3h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:29:31.372 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 10m ago 3h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:29:31.372 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 10m ago 3h 441M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:29:31.372 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 2m ago 3h 671M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:29:31.372 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 2m ago 3h 710M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:29:31.373 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 10m ago 3h 298M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:29:31.373 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (2h) 2m ago 3h 412M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:29:31.373 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 2m ago 3h 26.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:29:31.373 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 10m ago 3h 25.8M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:29:31.373 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 2m ago 3h 513M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:29:31.374 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 2m ago 3h 504M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:29:31.374 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 2m ago 3h 397M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:29:31.374 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 2m ago 3h 490M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:29:31.374 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 10m ago 3h 414M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:29:31.375 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 10m ago 3h 510M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:29:31.375 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 10m ago 3h 454M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:29:31.375 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 10m ago 3h 532M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:29:31.375 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 10m ago 3h 141M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:29:31.728 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:29:31.728 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:29:31.728 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:29:31.729 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:29:31.729 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:29:31.729 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:29:31.729 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:29:31.729 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:29:31.730 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:29:31.730 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:29:31.730 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:29:31.731 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:29:31.731 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:29:31.731 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:29:31.731 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:29:31.731 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:29:31.732 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:29:31.732 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:29:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:29:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:29:31] "GET /metrics HTTP/1.1" 200 225215 "" "Prometheus/2.33.4" 2022-09-29T08:29:32.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:29:32 smithi083 ceph-mon[77242]: pgmap v5439: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:32.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:29:32 smithi083 ceph-mon[77242]: from='client.40461 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:29:32.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:29:32 smithi083 ceph-mon[77242]: from='client.40467 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:29:32.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:29:32 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/548381148' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:29:32.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:29:32 smithi083 ceph-mon[79930]: pgmap v5439: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:32.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:29:32 smithi083 ceph-mon[79930]: from='client.40461 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:29:32.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:29:32 smithi083 ceph-mon[79930]: from='client.40467 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:29:32.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:29:32 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/548381148' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:29:32.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:29:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:29:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:29:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:29:32 smithi153 ceph-mon[70044]: pgmap v5439: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:32.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:29:32 smithi153 ceph-mon[70044]: from='client.40461 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:29:32.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:29:32 smithi153 ceph-mon[70044]: from='client.40467 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:29:32.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:29:32 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/548381148' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:29:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:29:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:29:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:29:33.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:29:33 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:29:33.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:29:33 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:29:33.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:29:33 smithi083 ceph-mon[77242]: pgmap v5440: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:33.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:29:33 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:29:33.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:29:33 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:29:33.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:29:33 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:29:33.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:29:33 smithi083 ceph-mon[79930]: pgmap v5440: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:33.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:29:33 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:29:34.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:29:33 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:29:34.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:29:33 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:29:34.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:29:33 smithi153 ceph-mon[70044]: pgmap v5440: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:34.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:29:33 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:29:34.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:29:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:29:34.203Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:29:35.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:29:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:29:34.897Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:29:36.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:29:35 smithi153 ceph-mon[70044]: pgmap v5441: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:36.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:29:35 smithi083 ceph-mon[79930]: pgmap v5441: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:36.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:29:35 smithi083 ceph-mon[77242]: pgmap v5441: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:38.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:29:38 smithi153 ceph-mon[70044]: pgmap v5442: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:38.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:29:38 smithi083 ceph-mon[77242]: pgmap v5442: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:38.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:29:38 smithi083 ceph-mon[79930]: pgmap v5442: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:29:40 smithi153 ceph-mon[70044]: pgmap v5443: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:29:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:29:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:29:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:29:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:29:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:29:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:29:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:29:40.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:29:40 smithi083 ceph-mon[79930]: pgmap v5443: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:40.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:29:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:29:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:29:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:29:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:29:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:29:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:29:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:29:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:29:40 smithi083 ceph-mon[77242]: pgmap v5443: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:29:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:29:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:29:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:29:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:29:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:29:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:29:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:29:41.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:29:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:29:40.816Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:29:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:29:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:29:41] "GET /metrics HTTP/1.1" 200 225218 "" "Prometheus/2.33.4" 2022-09-29T08:29:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:29:42 smithi153 ceph-mon[70044]: pgmap v5444: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:42.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:29:42 smithi083 ceph-mon[79930]: pgmap v5444: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:42.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:29:42 smithi083 ceph-mon[77242]: pgmap v5444: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:29:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:29:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:29:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:29:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:29:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:29:44.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:29:44 smithi153 ceph-mon[70044]: pgmap v5445: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:44.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:29:44 smithi083 ceph-mon[79930]: pgmap v5445: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:44.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:29:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:29:44.203Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:29:44.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:29:44 smithi083 ceph-mon[77242]: pgmap v5445: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:45.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:29:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:29:44.896Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:29:46.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:29:46 smithi153 ceph-mon[70044]: pgmap v5446: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:46.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:29:46 smithi083 ceph-mon[79930]: pgmap v5446: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:46.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:29:46 smithi083 ceph-mon[77242]: pgmap v5446: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:48.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:29:48 smithi083 ceph-mon[79930]: pgmap v5447: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:48.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:29:48 smithi083 ceph-mon[77242]: pgmap v5447: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:48.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:29:48 smithi153 ceph-mon[70044]: pgmap v5447: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:50.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:29:50 smithi083 ceph-mon[79930]: pgmap v5448: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:50.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:29:50 smithi083 ceph-mon[77242]: pgmap v5448: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:29:50 smithi153 ceph-mon[70044]: pgmap v5448: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:51.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:29:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:29:50.816Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:29:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:29:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:29:51] "GET /metrics HTTP/1.1" 200 225218 "" "Prometheus/2.33.4" 2022-09-29T08:29:52.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:29:52 smithi083 ceph-mon[77242]: pgmap v5449: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:52.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:29:52 smithi083 ceph-mon[79930]: pgmap v5449: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:29:52 smithi153 ceph-mon[70044]: pgmap v5449: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:52.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:29:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:29:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:29:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:29:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:29:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:29:54.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:29:54 smithi083 ceph-mon[79930]: pgmap v5450: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:54.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:29:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:29:54.203Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:29:54.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:29:54 smithi083 ceph-mon[77242]: pgmap v5450: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:54.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:29:54 smithi153 ceph-mon[70044]: pgmap v5450: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:55.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:29:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:29:54.897Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:29:56.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:29:56 smithi083 ceph-mon[79930]: pgmap v5451: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:56.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:29:56 smithi083 ceph-mon[77242]: pgmap v5451: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:56.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:29:56 smithi153 ceph-mon[70044]: pgmap v5451: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:58.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:29:58 smithi083 ceph-mon[79930]: pgmap v5452: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:58.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:29:58 smithi083 ceph-mon[77242]: pgmap v5452: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:29:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:29:58 smithi153 ceph-mon[70044]: pgmap v5452: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:00.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:30:00 smithi083 ceph-mon[79930]: pgmap v5453: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:00.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:30:00 smithi083 ceph-mon[79930]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T08:30:00.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:30:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[77219]: 2022-09-29T08:29:59.999+0000 7f160f665700 -1 log_channel(cluster) log [ERR] : overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T08:30:00.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:30:00 smithi083 ceph-mon[77242]: pgmap v5453: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:00.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:30:00 smithi083 ceph-mon[77242]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T08:30:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:30:00 smithi153 ceph-mon[70044]: pgmap v5453: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:30:00 smithi153 ceph-mon[70044]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T08:30:01.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:30:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:30:00.816Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:30:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:30:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:30:01] "GET /metrics HTTP/1.1" 200 225212 "" "Prometheus/2.33.4" 2022-09-29T08:30:02.055 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:30:02.343 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:30:02.344 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 2m ago 3h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:30:02.344 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 29s ago 3h 63.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:30:02.344 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 2m ago 3h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:30:02.345 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 29s ago 3h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:30:02.345 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 29s ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:30:02.345 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 2m ago 3h 671M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:30:02.345 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 2m ago 3h 710M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:30:02.346 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (2h) 29s ago 3h 313M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:30:02.346 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 2m ago 3h 412M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:30:02.346 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 2m ago 3h 26.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:30:02.346 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 29s ago 3h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:30:02.347 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 2m ago 3h 513M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:30:02.347 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 2m ago 3h 504M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:30:02.347 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 2m ago 3h 397M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:30:02.347 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 2m ago 3h 490M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:30:02.347 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 29s ago 3h 435M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:30:02.348 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 29s ago 3h 537M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:30:02.348 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 29s ago 3h 478M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:30:02.348 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 29s ago 3h 562M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:30:02.348 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 29s ago 3h 153M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:30:02.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:30:02 smithi083 ceph-mon[79930]: pgmap v5454: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:02.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:30:02 smithi083 ceph-mon[77242]: pgmap v5454: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:30:02 smithi153 ceph-mon[70044]: pgmap v5454: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:02.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:30:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:30:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:30:02.697 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:30:02.698 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:30:02.698 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:30:02.698 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:30:02.699 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:30:02.699 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:30:02.699 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:30:02.699 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:30:02.699 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:30:02.700 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:30:02.700 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:30:02.700 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:30:02.700 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:30:02.701 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:30:02.701 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:30:02.701 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:30:02.701 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:30:02.702 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:30:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:30:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:30:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:30:03.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:30:03 smithi153 ceph-mon[70044]: from='client.40479 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:30:03.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:30:03 smithi153 ceph-mon[70044]: from='client.50416 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:30:03.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:30:03 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/406530013' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:30:03.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:30:03 smithi083 ceph-mon[79930]: from='client.40479 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:30:03.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:30:03 smithi083 ceph-mon[79930]: from='client.50416 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:30:03.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:30:03 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/406530013' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:30:03.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:30:03 smithi083 ceph-mon[77242]: from='client.40479 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:30:03.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:30:03 smithi083 ceph-mon[77242]: from='client.50416 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:30:03.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:30:03 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/406530013' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:30:04.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:30:04 smithi083 ceph-mon[77242]: pgmap v5455: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:04.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:30:04 smithi083 ceph-mon[79930]: pgmap v5455: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:04.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:30:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:30:04.204Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:30:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:30:04 smithi153 ceph-mon[70044]: pgmap v5455: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:05.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:30:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:30:04.898Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:30:06.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:30:06 smithi083 ceph-mon[77242]: pgmap v5456: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:06.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:30:06 smithi083 ceph-mon[79930]: pgmap v5456: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:06.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:30:06 smithi153 ceph-mon[70044]: pgmap v5456: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:08.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:30:08 smithi083 ceph-mon[77242]: pgmap v5457: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:08.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:30:08 smithi083 ceph-mon[79930]: pgmap v5457: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:30:08 smithi153 ceph-mon[70044]: pgmap v5457: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:10.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:30:10 smithi083 ceph-mon[79930]: pgmap v5458: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:10.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:30:10 smithi083 ceph-mon[77242]: pgmap v5458: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:30:10 smithi153 ceph-mon[70044]: pgmap v5458: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:11.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:30:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:30:10.816Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:30:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:30:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:30:11] "GET /metrics HTTP/1.1" 200 225213 "" "Prometheus/2.33.4" 2022-09-29T08:30:12.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:30:12 smithi083 ceph-mon[79930]: pgmap v5459: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:12.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:30:12 smithi083 ceph-mon[77242]: pgmap v5459: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:30:12 smithi153 ceph-mon[70044]: pgmap v5459: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:12.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:30:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:30:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:30:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:30:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:30:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:30:14.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:30:14 smithi083 ceph-mon[79930]: pgmap v5460: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:14.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:30:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:30:14.205Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:30:14.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:30:14 smithi083 ceph-mon[77242]: pgmap v5460: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:14.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:30:14 smithi153 ceph-mon[70044]: pgmap v5460: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:15.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:30:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:30:14.898Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:30:16.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:30:16 smithi083 ceph-mon[77242]: pgmap v5461: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:16.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:30:16 smithi083 ceph-mon[79930]: pgmap v5461: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:30:16 smithi153 ceph-mon[70044]: pgmap v5461: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:18.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:30:18 smithi083 ceph-mon[79930]: pgmap v5462: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:18.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:30:18 smithi083 ceph-mon[77242]: pgmap v5462: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:30:18 smithi153 ceph-mon[70044]: pgmap v5462: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:20.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:30:20 smithi083 ceph-mon[79930]: pgmap v5463: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:20.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:30:20 smithi083 ceph-mon[77242]: pgmap v5463: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:30:20 smithi153 ceph-mon[70044]: pgmap v5463: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:21.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:30:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:30:20.817Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:30:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:30:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:30:21] "GET /metrics HTTP/1.1" 200 225213 "" "Prometheus/2.33.4" 2022-09-29T08:30:22.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:30:22 smithi083 ceph-mon[77242]: pgmap v5464: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:22.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:30:22 smithi083 ceph-mon[79930]: pgmap v5464: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:22.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:30:22 smithi153 ceph-mon[70044]: pgmap v5464: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:22.559 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:30:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:30:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:30:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:30:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:30:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:30:24.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:30:24 smithi083 ceph-mon[77242]: pgmap v5465: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:24.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:30:24 smithi083 ceph-mon[79930]: pgmap v5465: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:24.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:30:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:30:24.205Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:30:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:30:24 smithi153 ceph-mon[70044]: pgmap v5465: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:25.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:30:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:30:24.898Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:30:26.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:30:26 smithi083 ceph-mon[79930]: pgmap v5466: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:26.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:30:26 smithi083 ceph-mon[77242]: pgmap v5466: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:30:26 smithi153 ceph-mon[70044]: pgmap v5466: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:28.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:30:28 smithi083 ceph-mon[77242]: pgmap v5467: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:28.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:30:28 smithi083 ceph-mon[79930]: pgmap v5467: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:30:28 smithi153 ceph-mon[70044]: pgmap v5467: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:30.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:30:30 smithi083 ceph-mon[79930]: pgmap v5468: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:30.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:30:30 smithi083 ceph-mon[77242]: pgmap v5468: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:30:30 smithi153 ceph-mon[70044]: pgmap v5468: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:31.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:30:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:30:30.818Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:30:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:30:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:30:31] "GET /metrics HTTP/1.1" 200 225215 "" "Prometheus/2.33.4" 2022-09-29T08:30:32.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:30:32 smithi083 ceph-mon[79930]: pgmap v5469: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:32.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:30:32 smithi083 ceph-mon[77242]: pgmap v5469: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:30:32 smithi153 ceph-mon[70044]: pgmap v5469: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:32.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:30:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:30:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:30:33.025 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:30:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:30:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:30:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:30:33.340 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:30:33.340 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 3m ago 3h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:30:33.340 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 60s ago 3h 63.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:30:33.341 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 3m ago 3h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:30:33.341 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 60s ago 3h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:30:33.341 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 60s ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:30:33.341 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 3m ago 3h 671M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:30:33.342 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 3m ago 3h 710M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:30:33.342 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 60s ago 3h 313M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:30:33.342 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 3m ago 3h 412M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:30:33.342 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 3m ago 3h 26.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:30:33.343 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 60s ago 3h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:30:33.343 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2h) 3m ago 3h 513M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:30:33.343 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 3m ago 3h 504M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:30:33.343 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 3m ago 3h 397M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:30:33.344 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 3m ago 3h 490M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:30:33.344 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 60s ago 3h 435M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:30:33.344 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 60s ago 3h 537M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:30:33.344 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 60s ago 3h 478M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:30:33.344 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 60s ago 3h 562M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:30:33.345 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 60s ago 3h 153M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:30:33.716 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:30:33.716 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:30:33.716 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:30:33.717 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:30:33.717 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:30:33.717 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:30:33.717 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:30:33.718 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:30:33.718 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:30:33.718 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:30:33.718 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:30:33.718 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:30:33.719 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:30:33.719 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:30:33.719 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:30:33.719 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:30:33.720 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:30:33.720 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:30:34.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:30:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:30:34.206Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:30:34.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:30:34 smithi083 ceph-mon[79930]: pgmap v5470: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:34.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:30:34 smithi083 ceph-mon[79930]: from='client.50428 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:30:34.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:30:34 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:30:34.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:30:34 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:30:34.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:30:34 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:30:34.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:30:34 smithi083 ceph-mon[79930]: from='client.40503 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:30:34.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:30:34 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3424442295' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:30:34.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:30:34 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:30:34.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:30:34 smithi083 ceph-mon[77242]: pgmap v5470: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:34.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:30:34 smithi083 ceph-mon[77242]: from='client.50428 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:30:34.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:30:34 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:30:34.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:30:34 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:30:34.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:30:34 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:30:34.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:30:34 smithi083 ceph-mon[77242]: from='client.40503 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:30:34.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:30:34 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3424442295' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:30:34.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:30:34 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:30:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:30:34 smithi153 ceph-mon[70044]: pgmap v5470: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:34.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:30:34 smithi153 ceph-mon[70044]: from='client.50428 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:30:34.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:30:34 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:30:34.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:30:34 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:30:34.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:30:34 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:30:34.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:30:34 smithi153 ceph-mon[70044]: from='client.40503 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:30:34.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:30:34 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3424442295' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:30:34.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:30:34 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:30:35.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:30:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:30:34.899Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:30:36.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:30:36 smithi083 ceph-mon[79930]: pgmap v5471: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:36.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:30:36 smithi083 ceph-mon[77242]: pgmap v5471: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:30:36 smithi153 ceph-mon[70044]: pgmap v5471: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:38.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:30:38 smithi083 ceph-mon[79930]: pgmap v5472: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:38.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:30:38 smithi083 ceph-mon[77242]: pgmap v5472: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:30:38 smithi153 ceph-mon[70044]: pgmap v5472: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:40.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:30:40 smithi083 ceph-mon[79930]: pgmap v5473: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:30:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:30:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:30:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:30:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:30:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:30:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:30:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:30:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:30:40 smithi083 ceph-mon[77242]: pgmap v5473: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:30:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:30:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:30:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:30:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:30:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:30:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:30:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:30:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:30:40 smithi153 ceph-mon[70044]: pgmap v5473: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:30:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:30:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:30:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:30:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:30:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:30:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:30:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:30:41.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:30:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:30:40.819Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:30:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:30:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:30:41] "GET /metrics HTTP/1.1" 200 225213 "" "Prometheus/2.33.4" 2022-09-29T08:30:42.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:30:42 smithi083 ceph-mon[79930]: pgmap v5474: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:42.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:30:42 smithi083 ceph-mon[77242]: pgmap v5474: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:30:42 smithi153 ceph-mon[70044]: pgmap v5474: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:42.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:30:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:30:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:30:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:30:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:30:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:30:44.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:30:44 smithi083 ceph-mon[79930]: pgmap v5475: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:44.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:30:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:30:44.207Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:30:44.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:30:44 smithi083 ceph-mon[77242]: pgmap v5475: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:44.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:30:44 smithi153 ceph-mon[70044]: pgmap v5475: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:45.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:30:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:30:44.900Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:30:46.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:30:46 smithi083 ceph-mon[79930]: pgmap v5476: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:46.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:30:46 smithi083 ceph-mon[77242]: pgmap v5476: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:46.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:30:46 smithi153 ceph-mon[70044]: pgmap v5476: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:48.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:30:48 smithi083 ceph-mon[79930]: pgmap v5477: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:48.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:30:48 smithi083 ceph-mon[77242]: pgmap v5477: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:48.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:30:48 smithi153 ceph-mon[70044]: pgmap v5477: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:50.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:30:50 smithi083 ceph-mon[79930]: pgmap v5478: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:50.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:30:50 smithi083 ceph-mon[77242]: pgmap v5478: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:30:50 smithi153 ceph-mon[70044]: pgmap v5478: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:51.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:30:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:30:50.820Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:30:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:30:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:30:51] "GET /metrics HTTP/1.1" 200 225213 "" "Prometheus/2.33.4" 2022-09-29T08:30:52.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:30:52 smithi083 ceph-mon[79930]: pgmap v5479: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:52.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:30:52 smithi083 ceph-mon[77242]: pgmap v5479: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:30:52 smithi153 ceph-mon[70044]: pgmap v5479: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:52.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:30:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:30:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:30:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:30:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:30:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:30:54.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:30:54 smithi083 ceph-mon[79930]: pgmap v5480: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:54.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:30:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:30:54.207Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:30:54.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:30:54 smithi083 ceph-mon[77242]: pgmap v5480: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:54.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:30:54 smithi153 ceph-mon[70044]: pgmap v5480: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:55.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:30:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:30:54.901Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:30:56.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:30:56 smithi083 ceph-mon[79930]: pgmap v5481: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:56.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:30:56 smithi083 ceph-mon[77242]: pgmap v5481: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:56.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:30:56 smithi153 ceph-mon[70044]: pgmap v5481: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:58.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:30:58 smithi083 ceph-mon[79930]: pgmap v5482: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:58.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:30:58 smithi083 ceph-mon[77242]: pgmap v5482: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:30:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:30:58 smithi153 ceph-mon[70044]: pgmap v5482: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:00.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:31:00 smithi083 ceph-mon[79930]: pgmap v5483: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:00.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:31:00 smithi083 ceph-mon[77242]: pgmap v5483: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:31:00 smithi153 ceph-mon[70044]: pgmap v5483: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:01.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:31:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:31:00.820Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:31:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:31:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:31:01] "GET /metrics HTTP/1.1" 200 225216 "" "Prometheus/2.33.4" 2022-09-29T08:31:02.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:31:02 smithi083 ceph-mon[79930]: pgmap v5484: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:02.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:31:02 smithi083 ceph-mon[77242]: pgmap v5484: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:31:02 smithi153 ceph-mon[70044]: pgmap v5484: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:02.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:31:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:31:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:31:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:31:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:31:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:31:04.044 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:31:04.336 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:31:04.336 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 3m ago 3h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:31:04.336 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 91s ago 3h 63.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:31:04.337 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 3m ago 3h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:31:04.337 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 91s ago 3h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:31:04.337 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 91s ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:31:04.338 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 3m ago 3h 671M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:31:04.338 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 3m ago 3h 710M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:31:04.338 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 91s ago 3h 313M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:31:04.338 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 3m ago 3h 412M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:31:04.339 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 3m ago 3h 26.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:31:04.339 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 91s ago 3h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:31:04.339 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 3m ago 3h 513M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:31:04.339 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2h) 3m ago 3h 504M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:31:04.340 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 3m ago 3h 397M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:31:04.340 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 3m ago 3h 490M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:31:04.340 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 91s ago 3h 435M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:31:04.340 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 91s ago 3h 537M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:31:04.340 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 91s ago 3h 478M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:31:04.341 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 91s ago 3h 562M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:31:04.341 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 91s ago 3h 153M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:31:04.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:31:04 smithi083 ceph-mon[79930]: pgmap v5485: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:04.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:31:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:31:04.207Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:31:04.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:31:04 smithi083 ceph-mon[77242]: pgmap v5485: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:31:04 smithi153 ceph-mon[70044]: pgmap v5485: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:04.693 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:31:04.693 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:31:04.693 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:31:04.693 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:31:04.694 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:31:04.694 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:31:04.694 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:31:04.694 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:31:04.695 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:31:04.695 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:31:04.695 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:31:04.695 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:31:04.695 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:31:04.696 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:31:04.696 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:31:04.696 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:31:04.696 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:31:04.697 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:31:05.188 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:31:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:31:04.901Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:31:05.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:31:05 smithi083 ceph-mon[77242]: from='client.50446 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:31:05.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:31:05 smithi083 ceph-mon[77242]: from='client.40521 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:31:05.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:31:05 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1803033110' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:31:05.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:31:05 smithi083 ceph-mon[79930]: from='client.50446 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:31:05.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:31:05 smithi083 ceph-mon[79930]: from='client.40521 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:31:05.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:31:05 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1803033110' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:31:05.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:31:05 smithi153 ceph-mon[70044]: from='client.50446 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:31:05.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:31:05 smithi153 ceph-mon[70044]: from='client.40521 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:31:05.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:31:05 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1803033110' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:31:06.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:31:06 smithi083 ceph-mon[77242]: pgmap v5486: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:06.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:31:06 smithi083 ceph-mon[79930]: pgmap v5486: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:06.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:31:06 smithi153 ceph-mon[70044]: pgmap v5486: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:08.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:31:08 smithi083 ceph-mon[79930]: pgmap v5487: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:08.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:31:08 smithi083 ceph-mon[77242]: pgmap v5487: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:31:08 smithi153 ceph-mon[70044]: pgmap v5487: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:10.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:31:10 smithi083 ceph-mon[77242]: pgmap v5488: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:10.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:31:10 smithi083 ceph-mon[79930]: pgmap v5488: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:31:10 smithi153 ceph-mon[70044]: pgmap v5488: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:11.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:31:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:31:10.821Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:31:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:31:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:31:11] "GET /metrics HTTP/1.1" 200 225214 "" "Prometheus/2.33.4" 2022-09-29T08:31:12.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:31:12 smithi083 ceph-mon[77242]: pgmap v5489: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:12.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:31:12 smithi083 ceph-mon[79930]: pgmap v5489: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:31:12 smithi153 ceph-mon[70044]: pgmap v5489: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:12.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:31:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:31:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:31:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:31:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:31:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:31:14.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:31:14 smithi083 ceph-mon[79930]: pgmap v5490: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:14.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:31:14 smithi083 ceph-mon[77242]: pgmap v5490: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:14.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:31:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:31:14.208Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:31:14.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:31:14 smithi153 ceph-mon[70044]: pgmap v5490: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:15.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:31:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:31:14.902Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:31:16.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:31:16 smithi083 ceph-mon[79930]: pgmap v5491: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:16.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:31:16 smithi083 ceph-mon[77242]: pgmap v5491: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:31:16 smithi153 ceph-mon[70044]: pgmap v5491: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:18.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:31:18 smithi083 ceph-mon[79930]: pgmap v5492: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:18.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:31:18 smithi083 ceph-mon[77242]: pgmap v5492: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:31:18 smithi153 ceph-mon[70044]: pgmap v5492: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:31:20 smithi153 ceph-mon[70044]: pgmap v5493: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:20.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:31:20 smithi083 ceph-mon[79930]: pgmap v5493: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:20.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:31:20 smithi083 ceph-mon[77242]: pgmap v5493: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:21.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:31:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:31:20.821Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:31:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:31:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:31:21] "GET /metrics HTTP/1.1" 200 225214 "" "Prometheus/2.33.4" 2022-09-29T08:31:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:31:22 smithi153 ceph-mon[70044]: pgmap v5494: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:22.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:31:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:31:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:31:22.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:31:22 smithi083 ceph-mon[79930]: pgmap v5494: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:22.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:31:22 smithi083 ceph-mon[77242]: pgmap v5494: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:31:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:31:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:31:24.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:31:24 smithi083 ceph-mon[77242]: pgmap v5495: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:24.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:31:24 smithi083 ceph-mon[79930]: pgmap v5495: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:24.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:31:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:31:24.208Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:31:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:31:24 smithi153 ceph-mon[70044]: pgmap v5495: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:25.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:31:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:31:24.903Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:31:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:31:26 smithi153 ceph-mon[70044]: pgmap v5496: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:26.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:31:26 smithi083 ceph-mon[77242]: pgmap v5496: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:26.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:31:26 smithi083 ceph-mon[79930]: pgmap v5496: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:31:28 smithi153 ceph-mon[70044]: pgmap v5497: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:28.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:31:28 smithi083 ceph-mon[79930]: pgmap v5497: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:28.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:31:28 smithi083 ceph-mon[77242]: pgmap v5497: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:31:30 smithi153 ceph-mon[70044]: pgmap v5498: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:30.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:31:30 smithi083 ceph-mon[79930]: pgmap v5498: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:30.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:31:30 smithi083 ceph-mon[77242]: pgmap v5498: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:31.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:31:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:31:30.822Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:31:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:31:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:31:31] "GET /metrics HTTP/1.1" 200 225216 "" "Prometheus/2.33.4" 2022-09-29T08:31:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:31:32 smithi153 ceph-mon[70044]: pgmap v5499: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:32.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:31:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:31:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:31:32.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:31:32 smithi083 ceph-mon[77242]: pgmap v5499: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:32.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:31:32 smithi083 ceph-mon[79930]: pgmap v5499: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:31:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:31:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:31:34.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:31:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:31:34.208Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:31:34.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:31:34 smithi083 ceph-mon[77242]: pgmap v5500: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:34.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:31:34 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:31:34.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:31:34 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:31:34.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:31:34 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:31:34.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:31:34 smithi083 ceph-mon[79930]: pgmap v5500: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:34.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:31:34 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:31:34.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:31:34 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:31:34.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:31:34 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:31:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:31:34 smithi153 ceph-mon[70044]: pgmap v5500: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:31:34 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:31:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:31:34 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:31:34.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:31:34 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:31:35.009 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:31:35.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:31:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:31:34.903Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:31:35.296 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:31:35.296 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 4m ago 3h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:31:35.297 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 2m ago 3h 63.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:31:35.297 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 4m ago 3h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:31:35.297 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 2m ago 3h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:31:35.297 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 2m ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:31:35.298 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 4m ago 3h 671M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:31:35.298 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 4m ago 3h 710M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:31:35.298 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 2m ago 3h 313M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:31:35.298 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 4m ago 3h 412M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:31:35.298 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 4m ago 3h 26.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:31:35.299 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 2m ago 3h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:31:35.299 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 4m ago 3h 513M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:31:35.299 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 4m ago 3h 504M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:31:35.299 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2h) 4m ago 3h 397M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:31:35.299 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 4m ago 3h 490M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:31:35.300 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 2m ago 3h 435M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:31:35.300 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 2m ago 3h 537M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:31:35.300 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 2m ago 3h 478M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:31:35.300 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 2m ago 3h 562M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:31:35.301 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 2m ago 3h 153M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:31:35.659 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:31:35.659 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:31:35.659 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:31:35.660 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:31:35.660 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:31:35.660 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:31:35.660 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:31:35.661 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:31:35.661 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:31:35.661 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:31:35.661 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:31:35.661 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:31:35.662 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:31:35.662 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:31:35.662 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:31:35.662 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:31:35.663 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:31:35.663 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:31:35.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:31:35 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:31:35.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:31:35 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:31:35.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:31:35 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:31:36.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:31:36 smithi083 ceph-mon[79930]: pgmap v5501: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:36.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:31:36 smithi083 ceph-mon[79930]: from='client.50464 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:31:36.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:31:36 smithi083 ceph-mon[79930]: from='client.40539 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:31:36.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:31:36 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1470459298' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:31:36.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:31:36 smithi083 ceph-mon[77242]: pgmap v5501: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:36.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:31:36 smithi083 ceph-mon[77242]: from='client.50464 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:31:36.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:31:36 smithi083 ceph-mon[77242]: from='client.40539 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:31:36.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:31:36 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1470459298' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:31:36.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:31:36 smithi153 ceph-mon[70044]: pgmap v5501: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:36.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:31:36 smithi153 ceph-mon[70044]: from='client.50464 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:31:36.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:31:36 smithi153 ceph-mon[70044]: from='client.40539 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:31:36.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:31:36 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1470459298' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:31:38.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:31:38 smithi083 ceph-mon[79930]: pgmap v5502: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:38.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:31:38 smithi083 ceph-mon[77242]: pgmap v5502: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:38.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:31:38 smithi153 ceph-mon[70044]: pgmap v5502: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:40.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:31:40 smithi083 ceph-mon[79930]: pgmap v5503: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:40.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:31:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:31:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:31:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:31:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:31:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:31:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:31:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:31:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:31:40 smithi083 ceph-mon[77242]: pgmap v5503: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:31:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:31:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:31:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:31:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:31:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:31:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:31:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:31:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:31:40 smithi153 ceph-mon[70044]: pgmap v5503: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:31:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:31:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:31:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:31:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:31:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:31:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:31:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:31:41.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:31:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:31:40.822Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:31:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:31:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:31:41] "GET /metrics HTTP/1.1" 200 225212 "" "Prometheus/2.33.4" 2022-09-29T08:31:42.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:31:42 smithi083 ceph-mon[79930]: pgmap v5504: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:42.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:31:42 smithi083 ceph-mon[77242]: pgmap v5504: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:31:42 smithi153 ceph-mon[70044]: pgmap v5504: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:31:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:31:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:31:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:31:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:31:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:31:44.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:31:44 smithi083 ceph-mon[79930]: pgmap v5505: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:31:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:31:44.210Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:31:44.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:31:44 smithi083 ceph-mon[77242]: pgmap v5505: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:44.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:31:44 smithi153 ceph-mon[70044]: pgmap v5505: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:45.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:31:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:31:44.904Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:31:46.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:31:46 smithi083 ceph-mon[79930]: pgmap v5506: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:46.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:31:46 smithi083 ceph-mon[77242]: pgmap v5506: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:46.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:31:46 smithi153 ceph-mon[70044]: pgmap v5506: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:48.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:31:48 smithi153 ceph-mon[70044]: pgmap v5507: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:48.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:31:48 smithi083 ceph-mon[79930]: pgmap v5507: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:48.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:31:48 smithi083 ceph-mon[77242]: pgmap v5507: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:50.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:31:50 smithi153 ceph-mon[70044]: pgmap v5508: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:50.824 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:31:50 smithi083 ceph-mon[79930]: pgmap v5508: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:50.824 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:31:50 smithi083 ceph-mon[77242]: pgmap v5508: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:51.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:31:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:31:50.823Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:31:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:31:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:31:51] "GET /metrics HTTP/1.1" 200 225212 "" "Prometheus/2.33.4" 2022-09-29T08:31:52.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:31:52 smithi153 ceph-mon[70044]: pgmap v5509: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:31:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:31:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:31:52.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:31:52 smithi083 ceph-mon[79930]: pgmap v5509: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:52.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:31:52 smithi083 ceph-mon[77242]: pgmap v5509: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:31:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:31:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:31:54.473 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:31:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:31:54.211Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:31:54.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:31:54 smithi153 ceph-mon[70044]: pgmap v5510: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:54.906 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:31:54 smithi083 ceph-mon[79930]: pgmap v5510: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:54.906 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:31:54 smithi083 ceph-mon[77242]: pgmap v5510: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:55.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:31:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:31:54.904Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:31:56.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:31:56 smithi153 ceph-mon[70044]: pgmap v5511: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:56.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:31:56 smithi083 ceph-mon[77242]: pgmap v5511: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:56.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:31:56 smithi083 ceph-mon[79930]: pgmap v5511: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:58.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:31:58 smithi153 ceph-mon[70044]: pgmap v5512: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:58.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:31:58 smithi083 ceph-mon[77242]: pgmap v5512: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:31:58.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:31:58 smithi083 ceph-mon[79930]: pgmap v5512: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:00.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:32:00 smithi153 ceph-mon[70044]: pgmap v5513: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:00.825 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:32:00 smithi083 ceph-mon[79930]: pgmap v5513: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:00.825 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:32:00 smithi083 ceph-mon[77242]: pgmap v5513: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:01.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:32:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:32:00.824Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:32:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:32:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:32:01] "GET /metrics HTTP/1.1" 200 225217 "" "Prometheus/2.33.4" 2022-09-29T08:32:02.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:32:02 smithi153 ceph-mon[70044]: pgmap v5514: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:32:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:32:02.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:32:02.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:32:02 smithi083 ceph-mon[79930]: pgmap v5514: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:02.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:32:02 smithi083 ceph-mon[77242]: pgmap v5514: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:32:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:32:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:32:04.491 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:32:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:32:04.211Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:32:04.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:32:04 smithi153 ceph-mon[70044]: pgmap v5515: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:04.907 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:32:04 smithi083 ceph-mon[79930]: pgmap v5515: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:04.907 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:32:04 smithi083 ceph-mon[77242]: pgmap v5515: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:05.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:32:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:32:04.905Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:32:05.986 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:32:06.275 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:32:06.275 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 4m ago 3h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:32:06.275 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 2m ago 3h 63.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:32:06.276 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 4m ago 3h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:32:06.276 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 2m ago 3h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:32:06.276 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 2m ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:32:06.277 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 4m ago 3h 671M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:32:06.277 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 4m ago 3h 710M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:32:06.277 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 2m ago 3h 313M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:32:06.277 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 4m ago 3h 412M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:32:06.277 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 4m ago 3h 26.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:32:06.278 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 2m ago 3h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:32:06.278 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 4m ago 3h 513M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:32:06.278 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 4m ago 3h 504M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:32:06.278 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 4m ago 3h 397M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:32:06.279 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2h) 4m ago 3h 490M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:32:06.279 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 2m ago 3h 435M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:32:06.279 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 2m ago 3h 537M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:32:06.279 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 2m ago 3h 478M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:32:06.280 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 2m ago 3h 562M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:32:06.280 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 2m ago 3h 153M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:32:06.634 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:32:06.635 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:32:06.635 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:32:06.635 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:32:06.635 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:32:06.636 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:32:06.636 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:32:06.636 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:32:06.636 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:32:06.636 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:32:06.637 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:32:06.637 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:32:06.637 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:32:06.637 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:32:06.637 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:32:06.638 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:32:06.638 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:32:06.638 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:32:06.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:32:06 smithi153 ceph-mon[70044]: pgmap v5516: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:06.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:32:06 smithi083 ceph-mon[79930]: pgmap v5516: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:06.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:32:06 smithi083 ceph-mon[77242]: pgmap v5516: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:07.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:32:07 smithi153 ceph-mon[70044]: from='client.40551 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:32:07.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:32:07 smithi153 ceph-mon[70044]: from='client.40557 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:32:07.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:32:07 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3554457009' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:32:07.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:32:07 smithi083 ceph-mon[79930]: from='client.40551 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:32:07.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:32:07 smithi083 ceph-mon[79930]: from='client.40557 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:32:07.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:32:07 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3554457009' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:32:07.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:32:07 smithi083 ceph-mon[77242]: from='client.40551 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:32:07.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:32:07 smithi083 ceph-mon[77242]: from='client.40557 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:32:07.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:32:07 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3554457009' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:32:08.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:32:08 smithi153 ceph-mon[70044]: pgmap v5517: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:08.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:32:08 smithi083 ceph-mon[79930]: pgmap v5517: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:08.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:32:08 smithi083 ceph-mon[77242]: pgmap v5517: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:10.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:32:10 smithi153 ceph-mon[70044]: pgmap v5518: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:10.825 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:32:10 smithi083 ceph-mon[79930]: pgmap v5518: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:10.826 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:32:10 smithi083 ceph-mon[77242]: pgmap v5518: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:11.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:32:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:32:10.824Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:32:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:32:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:32:11] "GET /metrics HTTP/1.1" 200 225214 "" "Prometheus/2.33.4" 2022-09-29T08:32:12.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:32:12 smithi153 ceph-mon[70044]: pgmap v5519: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:32:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:32:12.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:32:12.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:32:12 smithi083 ceph-mon[77242]: pgmap v5519: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:12.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:32:12 smithi083 ceph-mon[79930]: pgmap v5519: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:32:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:32:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:32:14.510 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:32:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:32:14.212Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:32:14.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:32:14 smithi153 ceph-mon[70044]: pgmap v5520: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:14.907 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:32:14 smithi083 ceph-mon[79930]: pgmap v5520: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:14.907 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:32:14 smithi083 ceph-mon[77242]: pgmap v5520: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:15.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:32:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:32:14.906Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:32:16.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:32:16 smithi153 ceph-mon[70044]: pgmap v5521: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:16.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:32:16 smithi083 ceph-mon[79930]: pgmap v5521: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:16.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:32:16 smithi083 ceph-mon[77242]: pgmap v5521: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:18.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:32:18 smithi153 ceph-mon[70044]: pgmap v5522: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:18.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:32:18 smithi083 ceph-mon[77242]: pgmap v5522: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:18.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:32:18 smithi083 ceph-mon[79930]: pgmap v5522: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:20.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:32:20 smithi153 ceph-mon[70044]: pgmap v5523: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:20.826 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:32:20 smithi083 ceph-mon[79930]: pgmap v5523: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:20.826 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:32:20 smithi083 ceph-mon[77242]: pgmap v5523: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:21.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:32:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:32:20.825Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:32:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:32:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:32:21] "GET /metrics HTTP/1.1" 200 225214 "" "Prometheus/2.33.4" 2022-09-29T08:32:22.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:32:22 smithi153 ceph-mon[70044]: pgmap v5524: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:32:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:32:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:32:22.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:32:22 smithi083 ceph-mon[79930]: pgmap v5524: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:22.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:32:22 smithi083 ceph-mon[77242]: pgmap v5524: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:32:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:32:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:32:23.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:32:23 smithi153 ceph-mon[70044]: pgmap v5525: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:23.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:32:23 smithi083 ceph-mon[79930]: pgmap v5525: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:23.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:32:23 smithi083 ceph-mon[77242]: pgmap v5525: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:24.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:32:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:32:24.212Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:32:25.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:32:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:32:24.906Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:32:26.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:32:26 smithi153 ceph-mon[70044]: pgmap v5526: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:26.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:32:26 smithi083 ceph-mon[79930]: pgmap v5526: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:26.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:32:26 smithi083 ceph-mon[77242]: pgmap v5526: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:28.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:32:28 smithi083 ceph-mon[79930]: pgmap v5527: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:28.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:32:28 smithi083 ceph-mon[77242]: pgmap v5527: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:32:28 smithi153 ceph-mon[70044]: pgmap v5527: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:30.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:32:30 smithi083 ceph-mon[79930]: pgmap v5528: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:30.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:32:30 smithi083 ceph-mon[77242]: pgmap v5528: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:32:30 smithi153 ceph-mon[70044]: pgmap v5528: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:31.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:32:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:32:30.825Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:32:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:32:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:32:31] "GET /metrics HTTP/1.1" 200 225215 "" "Prometheus/2.33.4" 2022-09-29T08:32:32.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:32:32 smithi083 ceph-mon[79930]: pgmap v5529: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:32.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:32:32 smithi083 ceph-mon[77242]: pgmap v5529: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:32:32 smithi153 ceph-mon[70044]: pgmap v5529: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:32.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:32:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:32:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:32:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:32:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:32:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:32:34.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:32:34 smithi083 ceph-mon[79930]: pgmap v5530: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:34.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:32:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:32:34.214Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:32:34.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:32:34 smithi083 ceph-mon[77242]: pgmap v5530: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:32:34 smithi153 ceph-mon[70044]: pgmap v5530: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:35.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:32:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:32:34.906Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:32:35.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:32:35 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:32:35.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:32:35 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:32:35.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:32:35 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:32:35.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:32:35 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:32:35.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:32:35 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:32:35.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:32:35 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:32:35.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:32:35 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:32:35.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:32:35 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:32:35.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:32:35 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:32:36.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:32:36 smithi083 ceph-mon[79930]: pgmap v5531: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:36.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:32:36 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:32:36.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:32:36 smithi083 ceph-mon[77242]: pgmap v5531: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:36.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:32:36 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:32:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:32:36 smithi153 ceph-mon[70044]: pgmap v5531: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:32:36 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:32:36.964 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:32:37.255 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:32:37.256 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 5m ago 3h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:32:37.256 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 3m ago 3h 63.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:32:37.256 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 5m ago 3h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:32:37.257 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 3m ago 3h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:32:37.257 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 3m ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:32:37.257 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 5m ago 3h 671M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:32:37.257 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 5m ago 3h 710M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:32:37.257 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 3m ago 3h 313M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:32:37.258 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 5m ago 3h 412M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:32:37.258 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 5m ago 3h 26.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:32:37.258 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 3m ago 3h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:32:37.258 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 5m ago 3h 513M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:32:37.259 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 5m ago 3h 504M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:32:37.259 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 5m ago 3h 397M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:32:37.259 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 5m ago 3h 490M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:32:37.259 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (2h) 3m ago 3h 435M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:32:37.259 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 3m ago 3h 537M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:32:37.260 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 3m ago 3h 478M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:32:37.260 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 3m ago 3h 562M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:32:37.260 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 3m ago 3h 153M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:32:37.612 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:32:37.613 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:32:37.613 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:32:37.613 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:32:37.614 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:32:37.614 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:32:37.614 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:32:37.614 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:32:37.615 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:32:37.615 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:32:37.615 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:32:37.615 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:32:37.615 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:32:37.616 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:32:37.616 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:32:37.616 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:32:37.616 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:32:37.617 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:32:38.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:32:38 smithi083 ceph-mon[79930]: from='client.40569 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:32:38.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:32:38 smithi083 ceph-mon[79930]: pgmap v5532: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:38.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:32:38 smithi083 ceph-mon[79930]: from='client.40575 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:32:38.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:32:38 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1599110983' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:32:38.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:32:38 smithi083 ceph-mon[77242]: from='client.40569 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:32:38.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:32:38 smithi083 ceph-mon[77242]: pgmap v5532: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:38.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:32:38 smithi083 ceph-mon[77242]: from='client.40575 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:32:38.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:32:38 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1599110983' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:32:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:32:38 smithi153 ceph-mon[70044]: from='client.40569 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:32:38.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:32:38 smithi153 ceph-mon[70044]: pgmap v5532: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:38.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:32:38 smithi153 ceph-mon[70044]: from='client.40575 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:32:38.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:32:38 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1599110983' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:32:40.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:32:40 smithi083 ceph-mon[79930]: pgmap v5533: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:32:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:32:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:32:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:32:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:32:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:32:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:32:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:32:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:32:40 smithi083 ceph-mon[77242]: pgmap v5533: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:32:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:32:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:32:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:32:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:32:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:32:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:32:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:32:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:32:40 smithi153 ceph-mon[70044]: pgmap v5533: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:32:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:32:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:32:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:32:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:32:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:32:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:32:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:32:41.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:32:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:32:40.825Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:32:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:32:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:32:41] "GET /metrics HTTP/1.1" 200 225217 "" "Prometheus/2.33.4" 2022-09-29T08:32:42.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:32:42 smithi083 ceph-mon[79930]: pgmap v5534: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:42.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:32:42 smithi083 ceph-mon[77242]: pgmap v5534: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:32:42 smithi153 ceph-mon[70044]: pgmap v5534: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:42.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:32:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:32:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:32:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:32:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:32:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:32:44.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:32:44 smithi083 ceph-mon[79930]: pgmap v5535: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:44.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:32:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:32:44.214Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:32:44.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:32:44 smithi083 ceph-mon[77242]: pgmap v5535: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:44.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:32:44 smithi153 ceph-mon[70044]: pgmap v5535: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:45.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:32:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:32:44.907Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:32:46.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:32:46 smithi083 ceph-mon[79930]: pgmap v5536: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:46.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:32:46 smithi083 ceph-mon[77242]: pgmap v5536: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:46.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:32:46 smithi153 ceph-mon[70044]: pgmap v5536: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:48.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:32:48 smithi083 ceph-mon[79930]: pgmap v5537: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:48.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:32:48 smithi083 ceph-mon[77242]: pgmap v5537: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:48.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:32:48 smithi153 ceph-mon[70044]: pgmap v5537: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:50.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:32:50 smithi083 ceph-mon[79930]: pgmap v5538: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:50.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:32:50 smithi083 ceph-mon[77242]: pgmap v5538: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:32:50 smithi153 ceph-mon[70044]: pgmap v5538: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:51.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:32:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:32:50.826Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:32:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:32:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:32:51] "GET /metrics HTTP/1.1" 200 225217 "" "Prometheus/2.33.4" 2022-09-29T08:32:52.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:32:52 smithi083 ceph-mon[79930]: pgmap v5539: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:52.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:32:52 smithi083 ceph-mon[77242]: pgmap v5539: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:32:52 smithi153 ceph-mon[70044]: pgmap v5539: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:52.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:32:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:32:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:32:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:32:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:32:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:32:54.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:32:54 smithi083 ceph-mon[79930]: pgmap v5540: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:54.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:32:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:32:54.214Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:32:54.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:32:54 smithi083 ceph-mon[77242]: pgmap v5540: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:54.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:32:54 smithi153 ceph-mon[70044]: pgmap v5540: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:55.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:32:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:32:54.908Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:32:56.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:32:56 smithi083 ceph-mon[79930]: pgmap v5541: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:56.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:32:56 smithi083 ceph-mon[77242]: pgmap v5541: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:56.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:32:56 smithi153 ceph-mon[70044]: pgmap v5541: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:58.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:32:58 smithi083 ceph-mon[79930]: pgmap v5542: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:58.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:32:58 smithi083 ceph-mon[77242]: pgmap v5542: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:32:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:32:58 smithi153 ceph-mon[70044]: pgmap v5542: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:00.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:33:00 smithi083 ceph-mon[79930]: pgmap v5543: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:00.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:33:00 smithi083 ceph-mon[77242]: pgmap v5543: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:33:00 smithi153 ceph-mon[70044]: pgmap v5543: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:01.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:33:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:33:00.827Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:33:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:33:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:33:01] "GET /metrics HTTP/1.1" 200 225216 "" "Prometheus/2.33.4" 2022-09-29T08:33:02.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:33:02 smithi083 ceph-mon[79930]: pgmap v5544: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:02.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:33:02 smithi083 ceph-mon[77242]: pgmap v5544: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:33:02 smithi153 ceph-mon[70044]: pgmap v5544: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:02.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:33:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:33:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:33:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:33:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:33:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:33:04.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:33:04 smithi083 ceph-mon[77242]: pgmap v5545: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:04.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:33:04 smithi083 ceph-mon[79930]: pgmap v5545: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:04.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:33:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:33:04.215Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:33:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:33:04 smithi153 ceph-mon[70044]: pgmap v5545: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:05.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:33:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:33:04.909Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:33:06.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:33:06 smithi083 ceph-mon[77242]: pgmap v5546: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:06.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:33:06 smithi083 ceph-mon[79930]: pgmap v5546: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:06.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:33:06 smithi153 ceph-mon[70044]: pgmap v5546: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:07.944 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:33:08.232 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:33:08.233 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 5m ago 3h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:33:08.233 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 3m ago 3h 63.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:33:08.233 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 5m ago 3h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:33:08.234 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 3m ago 3h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:33:08.234 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 3m ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:33:08.234 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 5m ago 3h 671M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:33:08.234 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 5m ago 3h 710M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:33:08.235 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 3m ago 3h 313M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:33:08.235 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 5m ago 3h 412M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:33:08.235 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 5m ago 3h 26.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:33:08.235 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 3m ago 3h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:33:08.235 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 5m ago 3h 513M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:33:08.236 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 5m ago 3h 504M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:33:08.236 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 5m ago 3h 397M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:33:08.236 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 5m ago 3h 490M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:33:08.236 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 3m ago 3h 435M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:33:08.236 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (2h) 3m ago 3h 537M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:33:08.237 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 3m ago 3h 478M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:33:08.237 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 3m ago 3h 562M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:33:08.237 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 3m ago 3h 153M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:33:08.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:33:08 smithi083 ceph-mon[79930]: pgmap v5547: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:08.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:33:08 smithi083 ceph-mon[77242]: pgmap v5547: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:33:08 smithi153 ceph-mon[70044]: pgmap v5547: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:08.588 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:33:08.588 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:33:08.588 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:33:08.588 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:33:08.589 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:33:08.589 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:33:08.589 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:33:08.589 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:33:08.589 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:33:08.590 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:33:08.590 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:33:08.590 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:33:08.590 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:33:08.591 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:33:08.591 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:33:08.591 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:33:08.591 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:33:08.591 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:33:09.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:33:09 smithi083 ceph-mon[79930]: from='client.40587 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:33:09.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:33:09 smithi083 ceph-mon[79930]: from='client.40593 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:33:09.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:33:09 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3275016999' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:33:09.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:33:09 smithi083 ceph-mon[77242]: from='client.40587 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:33:09.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:33:09 smithi083 ceph-mon[77242]: from='client.40593 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:33:09.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:33:09 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3275016999' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:33:09.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:33:09 smithi153 ceph-mon[70044]: from='client.40587 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:33:09.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:33:09 smithi153 ceph-mon[70044]: from='client.40593 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:33:09.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:33:09 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3275016999' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:33:10.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:33:10 smithi083 ceph-mon[79930]: pgmap v5548: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:10.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:33:10 smithi083 ceph-mon[77242]: pgmap v5548: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:33:10 smithi153 ceph-mon[70044]: pgmap v5548: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:11.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:33:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:33:10.827Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:33:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:33:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:33:11] "GET /metrics HTTP/1.1" 200 225216 "" "Prometheus/2.33.4" 2022-09-29T08:33:12.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:33:12 smithi083 ceph-mon[79930]: pgmap v5549: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:12.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:33:12 smithi083 ceph-mon[77242]: pgmap v5549: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:33:12 smithi153 ceph-mon[70044]: pgmap v5549: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:12.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:33:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:33:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:33:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:33:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:33:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:33:14.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:33:14 smithi083 ceph-mon[77242]: pgmap v5550: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:14.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:33:14 smithi083 ceph-mon[79930]: pgmap v5550: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:14.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:33:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:33:14.216Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:33:14.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:33:14 smithi153 ceph-mon[70044]: pgmap v5550: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:15.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:33:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:33:14.909Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:33:16.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:33:16 smithi083 ceph-mon[79930]: pgmap v5551: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:16.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:33:16 smithi083 ceph-mon[77242]: pgmap v5551: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:33:16 smithi153 ceph-mon[70044]: pgmap v5551: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:18.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:33:18 smithi083 ceph-mon[79930]: pgmap v5552: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:18.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:33:18 smithi083 ceph-mon[77242]: pgmap v5552: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:33:18 smithi153 ceph-mon[70044]: pgmap v5552: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:20.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:33:20 smithi083 ceph-mon[79930]: pgmap v5553: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:20.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:33:20 smithi083 ceph-mon[77242]: pgmap v5553: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:33:20 smithi153 ceph-mon[70044]: pgmap v5553: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:21.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:33:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:33:20.828Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:33:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:33:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:33:21] "GET /metrics HTTP/1.1" 200 225216 "" "Prometheus/2.33.4" 2022-09-29T08:33:22.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:33:22 smithi083 ceph-mon[79930]: pgmap v5554: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:22.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:33:22 smithi083 ceph-mon[77242]: pgmap v5554: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:33:22 smithi153 ceph-mon[70044]: pgmap v5554: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:22.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:33:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:33:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:33:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:33:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:33:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:33:24.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:33:24 smithi083 ceph-mon[77242]: pgmap v5555: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:24.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:33:24 smithi083 ceph-mon[79930]: pgmap v5555: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:24.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:33:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:33:24.216Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:33:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:33:24 smithi153 ceph-mon[70044]: pgmap v5555: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:25.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:33:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:33:24.910Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:33:26.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:33:26 smithi083 ceph-mon[77242]: pgmap v5556: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:26.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:33:26 smithi083 ceph-mon[79930]: pgmap v5556: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:33:26 smithi153 ceph-mon[70044]: pgmap v5556: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:28.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:33:28 smithi083 ceph-mon[79930]: pgmap v5557: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:28.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:33:28 smithi083 ceph-mon[77242]: pgmap v5557: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:33:28 smithi153 ceph-mon[70044]: pgmap v5557: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:30.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:33:30 smithi083 ceph-mon[79930]: pgmap v5558: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:30.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:33:30 smithi083 ceph-mon[77242]: pgmap v5558: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:33:30 smithi153 ceph-mon[70044]: pgmap v5558: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:31.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:33:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:33:30.829Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:33:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:33:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:33:31] "GET /metrics HTTP/1.1" 200 225216 "" "Prometheus/2.33.4" 2022-09-29T08:33:32.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:33:32 smithi083 ceph-mon[79930]: pgmap v5559: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:32.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:33:32 smithi083 ceph-mon[77242]: pgmap v5559: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:33:32 smithi153 ceph-mon[70044]: pgmap v5559: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:32.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:33:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:33:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:33:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:33:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:33:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:33:34.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:33:34 smithi083 ceph-mon[79930]: pgmap v5560: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:34.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:33:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:33:34.217Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:33:34.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:33:34 smithi083 ceph-mon[77242]: pgmap v5560: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:33:34 smithi153 ceph-mon[70044]: pgmap v5560: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:35.204 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:33:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:33:34.910Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:33:35.205 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:33:35 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:33:35.206 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:33:35 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:33:35.206 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:33:35 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:33:35.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:33:35 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:33:35.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:33:35 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:33:35.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:33:35 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:33:35.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:33:35 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:33:35.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:33:35 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:33:35.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:33:35 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:33:36.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:33:36 smithi083 ceph-mon[79930]: pgmap v5561: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:36.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:33:36 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:33:36.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:33:36 smithi083 ceph-mon[77242]: pgmap v5561: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:36.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:33:36 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:33:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:33:36 smithi153 ceph-mon[70044]: pgmap v5561: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:36.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:33:36 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:33:38.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:33:38 smithi083 ceph-mon[79930]: pgmap v5562: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:38.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:33:38 smithi083 ceph-mon[77242]: pgmap v5562: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:33:38 smithi153 ceph-mon[70044]: pgmap v5562: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:38.912 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:33:39.199 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:33:39.200 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 6m ago 3h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:33:39.200 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 4m ago 3h 63.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:33:39.200 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 6m ago 3h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:33:39.200 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 4m ago 3h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:33:39.201 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 4m ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:33:39.201 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 6m ago 3h 671M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:33:39.201 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 6m ago 3h 710M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:33:39.201 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 4m ago 3h 313M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:33:39.201 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 6m ago 3h 412M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:33:39.202 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 6m ago 3h 26.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:33:39.202 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 4m ago 3h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:33:39.202 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 6m ago 3h 513M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:33:39.203 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 6m ago 3h 504M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:33:39.203 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 6m ago 3h 397M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:33:39.203 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 6m ago 3h 490M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:33:39.203 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 4m ago 3h 435M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:33:39.203 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 4m ago 3h 537M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:33:39.204 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (2h) 4m ago 3h 478M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:33:39.204 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 4m ago 3h 562M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:33:39.204 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 4m ago 3h 153M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:33:39.555 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:33:39.555 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:33:39.556 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:33:39.556 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:33:39.556 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:33:39.556 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:33:39.557 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:33:39.557 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:33:39.557 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:33:39.557 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:33:39.557 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:33:39.558 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:33:39.558 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:33:39.558 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:33:39.558 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:33:39.558 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:33:39.559 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:33:39.559 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:33:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:33:40 smithi153 ceph-mon[70044]: from='client.50536 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:33:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:33:40 smithi153 ceph-mon[70044]: pgmap v5563: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:33:40 smithi153 ceph-mon[70044]: from='client.40611 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:33:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:33:40 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/59158028' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:33:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:33:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:33:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:33:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:33:40.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:33:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:33:40.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:33:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:33:40.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:33:40 smithi083 ceph-mon[79930]: from='client.50536 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:33:40.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:33:40 smithi083 ceph-mon[79930]: pgmap v5563: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:33:40 smithi083 ceph-mon[79930]: from='client.40611 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:33:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:33:40 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/59158028' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:33:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:33:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:33:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:33:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:33:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:33:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:33:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:33:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:33:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:33:40 smithi083 ceph-mon[77242]: from='client.50536 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:33:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:33:40 smithi083 ceph-mon[77242]: pgmap v5563: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:40.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:33:40 smithi083 ceph-mon[77242]: from='client.40611 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:33:40.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:33:40 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/59158028' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:33:40.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:33:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:33:40.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:33:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:33:40.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:33:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:33:40.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:33:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:33:41.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:33:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:33:40.830Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:33:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:33:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:33:41] "GET /metrics HTTP/1.1" 200 225217 "" "Prometheus/2.33.4" 2022-09-29T08:33:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:33:42 smithi153 ceph-mon[70044]: pgmap v5564: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:42.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:33:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:33:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:33:42.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:33:42 smithi083 ceph-mon[77242]: pgmap v5564: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:42.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:33:42 smithi083 ceph-mon[79930]: pgmap v5564: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:33:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:33:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:33:44.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:33:44 smithi153 ceph-mon[70044]: pgmap v5565: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:44.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:33:44 smithi083 ceph-mon[79930]: pgmap v5565: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:44.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:33:44 smithi083 ceph-mon[77242]: pgmap v5565: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:33:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:33:44.218Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:33:45.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:33:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:33:44.911Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:33:46.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:33:46 smithi153 ceph-mon[70044]: pgmap v5566: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:46.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:33:46 smithi083 ceph-mon[79930]: pgmap v5566: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:46.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:33:46 smithi083 ceph-mon[77242]: pgmap v5566: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:48.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:33:48 smithi153 ceph-mon[70044]: pgmap v5567: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:48.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:33:48 smithi083 ceph-mon[79930]: pgmap v5567: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:48.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:33:48 smithi083 ceph-mon[77242]: pgmap v5567: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:33:50 smithi153 ceph-mon[70044]: pgmap v5568: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:50.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:33:50 smithi083 ceph-mon[79930]: pgmap v5568: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:50.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:33:50 smithi083 ceph-mon[77242]: pgmap v5568: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:51.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:33:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:33:50.831Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:33:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:33:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:33:51] "GET /metrics HTTP/1.1" 200 225217 "" "Prometheus/2.33.4" 2022-09-29T08:33:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:33:52 smithi153 ceph-mon[70044]: pgmap v5569: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:52.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:33:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:33:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:33:52.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:33:52 smithi083 ceph-mon[79930]: pgmap v5569: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:52.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:33:52 smithi083 ceph-mon[77242]: pgmap v5569: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:33:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:33:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:33:54.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:33:54 smithi153 ceph-mon[70044]: pgmap v5570: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:54.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:33:54 smithi083 ceph-mon[79930]: pgmap v5570: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:54.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:33:54 smithi083 ceph-mon[77242]: pgmap v5570: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:33:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:33:54.218Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:33:55.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:33:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:33:54.912Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:33:56.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:33:56 smithi153 ceph-mon[70044]: pgmap v5571: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:56.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:33:56 smithi083 ceph-mon[79930]: pgmap v5571: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:56.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:33:56 smithi083 ceph-mon[77242]: pgmap v5571: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:33:58 smithi153 ceph-mon[70044]: pgmap v5572: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:58.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:33:58 smithi083 ceph-mon[79930]: pgmap v5572: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:33:58.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:33:58 smithi083 ceph-mon[77242]: pgmap v5572: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:34:00 smithi153 ceph-mon[70044]: pgmap v5573: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:00.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:34:00 smithi083 ceph-mon[79930]: pgmap v5573: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:00.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:34:00 smithi083 ceph-mon[77242]: pgmap v5573: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:01.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:34:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:34:00.832Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:34:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:34:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:34:01] "GET /metrics HTTP/1.1" 200 225212 "" "Prometheus/2.33.4" 2022-09-29T08:34:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:34:02 smithi153 ceph-mon[70044]: pgmap v5574: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:02.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:34:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:34:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:34:02.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:34:02 smithi083 ceph-mon[79930]: pgmap v5574: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:02.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:34:02 smithi083 ceph-mon[77242]: pgmap v5574: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:34:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:34:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:34:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:34:04 smithi153 ceph-mon[70044]: pgmap v5575: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:04.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:34:04 smithi083 ceph-mon[79930]: pgmap v5575: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:04.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:34:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:34:04.220Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:34:04.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:34:04 smithi083 ceph-mon[77242]: pgmap v5575: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:05.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:34:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:34:04.912Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:34:06.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:34:06 smithi153 ceph-mon[70044]: pgmap v5576: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:06.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:34:06 smithi083 ceph-mon[79930]: pgmap v5576: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:06.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:34:06 smithi083 ceph-mon[77242]: pgmap v5576: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:34:08 smithi153 ceph-mon[70044]: pgmap v5577: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:08.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:34:08 smithi083 ceph-mon[79930]: pgmap v5577: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:08.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:34:08 smithi083 ceph-mon[77242]: pgmap v5577: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:09.884 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:34:10.171 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:34:10.171 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 6m ago 3h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:34:10.171 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 4m ago 3h 63.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:34:10.172 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 6m ago 3h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:34:10.172 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 4m ago 3h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:34:10.172 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 4m ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:34:10.172 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 6m ago 3h 671M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:34:10.173 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 6m ago 3h 710M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:34:10.173 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 4m ago 3h 313M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:34:10.173 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 6m ago 3h 412M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:34:10.173 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 6m ago 3h 26.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:34:10.173 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 4m ago 3h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:34:10.174 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 6m ago 3h 513M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:34:10.174 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 6m ago 3h 504M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:34:10.174 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 6m ago 3h 397M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:34:10.175 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 6m ago 3h 490M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:34:10.175 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 4m ago 3h 435M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:34:10.175 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 4m ago 3h 537M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:34:10.175 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 4m ago 3h 478M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:34:10.175 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (2h) 4m ago 3h 562M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:34:10.176 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 4m ago 3h 153M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:34:10.525 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:34:10.525 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:34:10.525 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:34:10.526 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:34:10.526 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:34:10.526 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:34:10.526 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:34:10.527 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:34:10.527 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:34:10.527 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:34:10.527 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:34:10.527 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:34:10.528 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:34:10.528 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:34:10.528 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:34:10.528 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:34:10.529 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:34:10.529 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:34:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:34:10 smithi153 ceph-mon[70044]: pgmap v5578: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:10.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:34:10 smithi083 ceph-mon[77242]: pgmap v5578: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:10.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:34:10 smithi083 ceph-mon[79930]: pgmap v5578: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:11.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:34:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:34:10.832Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:34:11.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:34:11 smithi153 ceph-mon[70044]: from='client.50554 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:34:11.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:34:11 smithi153 ceph-mon[70044]: from='client.40629 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:34:11.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:34:11 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1110713450' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:34:11.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:34:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:34:11] "GET /metrics HTTP/1.1" 200 225213 "" "Prometheus/2.33.4" 2022-09-29T08:34:11.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:34:11 smithi083 ceph-mon[79930]: from='client.50554 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:34:11.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:34:11 smithi083 ceph-mon[79930]: from='client.40629 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:34:11.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:34:11 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1110713450' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:34:11.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:34:11 smithi083 ceph-mon[77242]: from='client.50554 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:34:11.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:34:11 smithi083 ceph-mon[77242]: from='client.40629 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:34:11.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:34:11 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1110713450' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:34:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:34:12 smithi153 ceph-mon[70044]: pgmap v5579: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:12.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:34:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:34:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:34:12.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:34:12 smithi083 ceph-mon[79930]: pgmap v5579: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:12.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:34:12 smithi083 ceph-mon[77242]: pgmap v5579: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:34:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:34:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:34:14.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:34:14 smithi153 ceph-mon[70044]: pgmap v5580: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:14.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:34:14 smithi083 ceph-mon[79930]: pgmap v5580: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:14.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:34:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:34:14.220Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:34:14.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:34:14 smithi083 ceph-mon[77242]: pgmap v5580: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:15.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:34:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:34:14.913Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:34:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:34:16 smithi153 ceph-mon[70044]: pgmap v5581: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:16.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:34:16 smithi083 ceph-mon[77242]: pgmap v5581: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:16.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:34:16 smithi083 ceph-mon[79930]: pgmap v5581: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:34:18 smithi153 ceph-mon[70044]: pgmap v5582: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:18.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:34:18 smithi083 ceph-mon[79930]: pgmap v5582: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:18.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:34:18 smithi083 ceph-mon[77242]: pgmap v5582: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:34:20 smithi153 ceph-mon[70044]: pgmap v5583: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:20.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:34:20 smithi083 ceph-mon[79930]: pgmap v5583: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:20.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:34:20 smithi083 ceph-mon[77242]: pgmap v5583: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:21.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:34:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:34:20.833Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:34:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:34:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:34:21] "GET /metrics HTTP/1.1" 200 225213 "" "Prometheus/2.33.4" 2022-09-29T08:34:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:34:22 smithi153 ceph-mon[70044]: pgmap v5584: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:22.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:34:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:34:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:34:22.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:34:22 smithi083 ceph-mon[79930]: pgmap v5584: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:22.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:34:22 smithi083 ceph-mon[77242]: pgmap v5584: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:34:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:34:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:34:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:34:24 smithi153 ceph-mon[70044]: pgmap v5585: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:24.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:34:24 smithi083 ceph-mon[79930]: pgmap v5585: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:34:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:34:24.220Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:34:24.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:34:24 smithi083 ceph-mon[77242]: pgmap v5585: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:25.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:34:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:34:24.914Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:34:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:34:26 smithi153 ceph-mon[70044]: pgmap v5586: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:26.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:34:26 smithi083 ceph-mon[79930]: pgmap v5586: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:26.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:34:26 smithi083 ceph-mon[77242]: pgmap v5586: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:34:28 smithi153 ceph-mon[70044]: pgmap v5587: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:28.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:34:28 smithi083 ceph-mon[79930]: pgmap v5587: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:28.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:34:28 smithi083 ceph-mon[77242]: pgmap v5587: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:34:30 smithi153 ceph-mon[70044]: pgmap v5588: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:30.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:34:30 smithi083 ceph-mon[79930]: pgmap v5588: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:30.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:34:30 smithi083 ceph-mon[77242]: pgmap v5588: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:31.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:34:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:34:30.834Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:34:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:34:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:34:31] "GET /metrics HTTP/1.1" 200 225219 "" "Prometheus/2.33.4" 2022-09-29T08:34:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:34:32 smithi153 ceph-mon[70044]: pgmap v5589: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:32.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:34:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:34:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:34:32.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:34:32 smithi083 ceph-mon[79930]: pgmap v5589: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:32.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:34:32 smithi083 ceph-mon[77242]: pgmap v5589: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:34:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:34:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:34:34.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:34:34 smithi083 ceph-mon[79930]: pgmap v5590: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:34.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:34:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:34:34.221Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:34:34.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:34:34 smithi083 ceph-mon[77242]: pgmap v5590: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:34.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:34:34 smithi153 ceph-mon[70044]: pgmap v5590: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:35.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:34:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:34:34.915Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:34:36.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:34:36 smithi083 ceph-mon[79930]: pgmap v5591: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:36.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:34:36 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:34:36.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:34:36 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:34:36.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:34:36 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:34:36.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:34:36 smithi083 ceph-mon[77242]: pgmap v5591: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:36.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:34:36 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:34:36.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:34:36 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:34:36.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:34:36 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:34:36.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:34:36 smithi153 ceph-mon[70044]: pgmap v5591: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:36.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:34:36 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:34:36.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:34:36 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:34:36.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:34:36 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:34:37.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:34:37 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:34:37.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:34:37 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:34:37.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:34:37 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:34:38.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:34:38 smithi083 ceph-mon[79930]: pgmap v5592: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:38.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:34:38 smithi083 ceph-mon[77242]: pgmap v5592: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:38.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:34:38 smithi153 ceph-mon[70044]: pgmap v5592: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:40.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:34:40 smithi083 ceph-mon[79930]: pgmap v5593: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:34:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:34:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:34:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:34:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:34:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:34:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:34:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:34:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:34:40 smithi083 ceph-mon[77242]: pgmap v5593: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:34:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:34:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:34:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:34:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:34:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:34:40.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:34:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:34:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:34:40 smithi153 ceph-mon[70044]: pgmap v5593: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:34:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:34:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:34:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:34:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:34:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:34:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:34:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:34:40.853 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:34:41.143 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:34:41.143 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 7m ago 3h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:34:41.144 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 5m ago 3h 63.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:34:41.144 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 7m ago 3h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:34:41.144 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 5m ago 3h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:34:41.144 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 5m ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:34:41.145 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 7m ago 3h 671M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:34:41.145 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 7m ago 3h 710M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:34:41.145 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 5m ago 3h 313M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:34:41.145 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 7m ago 3h 412M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:34:41.146 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 7m ago 3h 26.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:34:41.146 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 5m ago 3h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:34:41.146 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 7m ago 3h 513M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:34:41.146 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 7m ago 3h 504M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:34:41.147 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 7m ago 3h 397M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:34:41.147 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 7m ago 3h 490M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:34:41.147 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 5m ago 3h 435M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:34:41.147 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 5m ago 3h 537M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:34:41.148 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 5m ago 3h 478M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:34:41.148 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 5m ago 3h 562M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:34:41.148 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 5m ago 3h 153M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:34:41.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:34:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:34:40.834Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:34:41.500 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:34:41.500 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:34:41.500 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:34:41.501 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:34:41.501 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:34:41.501 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:34:41.501 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:34:41.502 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:34:41.502 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:34:41.502 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:34:41.502 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:34:41.503 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:34:41.503 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:34:41.503 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:34:41.503 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:34:41.504 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:34:41.504 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:34:41.504 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:34:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:34:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:34:41] "GET /metrics HTTP/1.1" 200 225212 "" "Prometheus/2.33.4" 2022-09-29T08:34:42.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:34:42 smithi083 ceph-mon[79930]: from='client.40641 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:34:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:34:42 smithi083 ceph-mon[79930]: pgmap v5594: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:34:42 smithi083 ceph-mon[79930]: from='client.50578 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:34:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:34:42 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3392414454' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:34:42.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:34:42 smithi083 ceph-mon[77242]: from='client.40641 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:34:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:34:42 smithi083 ceph-mon[77242]: pgmap v5594: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:34:42 smithi083 ceph-mon[77242]: from='client.50578 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:34:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:34:42 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3392414454' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:34:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:34:42 smithi153 ceph-mon[70044]: from='client.40641 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:34:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:34:42 smithi153 ceph-mon[70044]: pgmap v5594: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:34:42 smithi153 ceph-mon[70044]: from='client.50578 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:34:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:34:42 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3392414454' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:34:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:34:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:34:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:34:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:34:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:34:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:34:44.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:34:44 smithi083 ceph-mon[79930]: pgmap v5595: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:44.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:34:44 smithi083 ceph-mon[77242]: pgmap v5595: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:34:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:34:44.221Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:34:44.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:34:44 smithi153 ceph-mon[70044]: pgmap v5595: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:45.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:34:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:34:44.915Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:34:46.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:34:46 smithi083 ceph-mon[79930]: pgmap v5596: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:46.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:34:46 smithi083 ceph-mon[77242]: pgmap v5596: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:46.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:34:46 smithi153 ceph-mon[70044]: pgmap v5596: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:48.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:34:48 smithi083 ceph-mon[79930]: pgmap v5597: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:48.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:34:48 smithi083 ceph-mon[77242]: pgmap v5597: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:48.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:34:48 smithi153 ceph-mon[70044]: pgmap v5597: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:50.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:34:50 smithi083 ceph-mon[79930]: pgmap v5598: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:50.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:34:50 smithi083 ceph-mon[77242]: pgmap v5598: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:50.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:34:50 smithi153 ceph-mon[70044]: pgmap v5598: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:51.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:34:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:34:50.835Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:34:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:34:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:34:51] "GET /metrics HTTP/1.1" 200 225212 "" "Prometheus/2.33.4" 2022-09-29T08:34:52.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:34:52 smithi083 ceph-mon[79930]: pgmap v5599: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:52.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:34:52 smithi083 ceph-mon[77242]: pgmap v5599: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:52.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:34:52 smithi153 ceph-mon[70044]: pgmap v5599: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:34:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:34:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:34:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:34:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:34:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:34:54.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:34:54 smithi083 ceph-mon[79930]: pgmap v5600: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:54.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:34:54 smithi083 ceph-mon[77242]: pgmap v5600: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:34:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:34:54.222Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:34:54.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:34:54 smithi153 ceph-mon[70044]: pgmap v5600: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:55.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:34:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:34:54.917Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:34:56.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:34:56 smithi083 ceph-mon[77242]: pgmap v5601: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:56.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:34:56 smithi083 ceph-mon[79930]: pgmap v5601: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:56.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:34:56 smithi153 ceph-mon[70044]: pgmap v5601: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:58.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:34:58 smithi083 ceph-mon[77242]: pgmap v5602: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:58.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:34:58 smithi083 ceph-mon[79930]: pgmap v5602: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:34:58.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:34:58 smithi153 ceph-mon[70044]: pgmap v5602: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:00.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:35:00 smithi153 ceph-mon[70044]: pgmap v5603: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:00.837 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:35:00 smithi083 ceph-mon[79930]: pgmap v5603: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:00.837 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:35:00 smithi083 ceph-mon[77242]: pgmap v5603: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:01.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:35:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:35:00.836Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:35:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:35:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:35:01] "GET /metrics HTTP/1.1" 200 225213 "" "Prometheus/2.33.4" 2022-09-29T08:35:02.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:35:02 smithi153 ceph-mon[70044]: pgmap v5604: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:35:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:35:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:35:02.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:35:02 smithi083 ceph-mon[79930]: pgmap v5604: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:02.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:35:02 smithi083 ceph-mon[77242]: pgmap v5604: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:35:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:35:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:35:04.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:35:04 smithi083 ceph-mon[79930]: pgmap v5605: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:04.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:35:04 smithi083 ceph-mon[77242]: pgmap v5605: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:04.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:35:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:35:04.223Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:35:04.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:35:04 smithi153 ceph-mon[70044]: pgmap v5605: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:05.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:35:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:35:04.918Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:35:06.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:35:06 smithi153 ceph-mon[70044]: pgmap v5606: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:06.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:35:06 smithi083 ceph-mon[79930]: pgmap v5606: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:06.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:35:06 smithi083 ceph-mon[77242]: pgmap v5606: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:08.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:35:08 smithi153 ceph-mon[70044]: pgmap v5607: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:08.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:35:08 smithi083 ceph-mon[79930]: pgmap v5607: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:08.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:35:08 smithi083 ceph-mon[77242]: pgmap v5607: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:10.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:35:10 smithi153 ceph-mon[70044]: pgmap v5608: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:10.837 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:35:10 smithi083 ceph-mon[79930]: pgmap v5608: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:10.838 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:35:10 smithi083 ceph-mon[77242]: pgmap v5608: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:11.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:35:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:35:10.836Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:35:11.838 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:35:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:35:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:35:11] "GET /metrics HTTP/1.1" 200 225215 "" "Prometheus/2.33.4" 2022-09-29T08:35:12.129 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:35:12.129 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 7m ago 3h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:35:12.130 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 5m ago 3h 63.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:35:12.130 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 7m ago 3h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:35:12.130 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 5m ago 3h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:35:12.130 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 5m ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:35:12.131 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 7m ago 3h 671M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:35:12.131 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 7m ago 3h 710M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:35:12.131 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 5m ago 3h 313M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:35:12.131 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 7m ago 3h 412M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:35:12.131 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 7m ago 3h 26.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:35:12.132 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 5m ago 3h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:35:12.132 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 7m ago 3h 513M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:35:12.132 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 7m ago 3h 504M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:35:12.132 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 7m ago 3h 397M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:35:12.132 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 7m ago 3h 490M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:35:12.133 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 5m ago 3h 435M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:35:12.133 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 5m ago 3h 537M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:35:12.133 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 5m ago 3h 478M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:35:12.133 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 5m ago 3h 562M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:35:12.134 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 5m ago 3h 153M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:35:12.491 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:35:12.491 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:35:12.492 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:35:12.492 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:35:12.492 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:35:12.492 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:35:12.492 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:35:12.493 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:35:12.493 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:35:12.493 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:35:12.493 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:35:12.493 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:35:12.494 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:35:12.494 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:35:12.494 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:35:12.494 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:35:12.495 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:35:12.495 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:35:12.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:35:12 smithi153 ceph-mon[70044]: pgmap v5609: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:35:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:35:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:35:12.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:35:12 smithi083 ceph-mon[79930]: pgmap v5609: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:12.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:35:12 smithi083 ceph-mon[77242]: pgmap v5609: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:35:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:35:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:35:13.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:35:13 smithi153 ceph-mon[70044]: from='client.50590 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:35:13.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:35:13 smithi153 ceph-mon[70044]: from='client.40665 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:35:13.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:35:13 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2400135664' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:35:13.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:35:13 smithi083 ceph-mon[79930]: from='client.50590 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:35:13.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:35:13 smithi083 ceph-mon[79930]: from='client.40665 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:35:13.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:35:13 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2400135664' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:35:13.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:35:13 smithi083 ceph-mon[77242]: from='client.50590 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:35:13.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:35:13 smithi083 ceph-mon[77242]: from='client.40665 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:35:13.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:35:13 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2400135664' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:35:14.491 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:35:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:35:14.224Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:35:14.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:35:14 smithi153 ceph-mon[70044]: pgmap v5610: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:14.919 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:35:14 smithi083 ceph-mon[79930]: pgmap v5610: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:14.920 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:35:14 smithi083 ceph-mon[77242]: pgmap v5610: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:15.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:35:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:35:14.918Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:35:16.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:35:16 smithi153 ceph-mon[70044]: pgmap v5611: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:16.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:35:16 smithi083 ceph-mon[79930]: pgmap v5611: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:16.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:35:16 smithi083 ceph-mon[77242]: pgmap v5611: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:18.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:35:18 smithi153 ceph-mon[70044]: pgmap v5612: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:18.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:35:18 smithi083 ceph-mon[77242]: pgmap v5612: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:18.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:35:18 smithi083 ceph-mon[79930]: pgmap v5612: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:20.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:35:20 smithi153 ceph-mon[70044]: pgmap v5613: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:20.838 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:35:20 smithi083 ceph-mon[79930]: pgmap v5613: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:20.838 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:35:20 smithi083 ceph-mon[77242]: pgmap v5613: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:21.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:35:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:35:20.837Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:35:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:35:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:35:21] "GET /metrics HTTP/1.1" 200 225215 "" "Prometheus/2.33.4" 2022-09-29T08:35:22.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:35:22 smithi153 ceph-mon[70044]: pgmap v5614: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:35:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:35:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:35:22.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:35:22 smithi083 ceph-mon[77242]: pgmap v5614: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:22.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:35:22 smithi083 ceph-mon[79930]: pgmap v5614: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:35:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:35:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:35:24.505 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:35:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:35:24.224Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:35:24.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:35:24 smithi153 ceph-mon[70044]: pgmap v5615: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:24.922 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:35:24 smithi083 ceph-mon[79930]: pgmap v5615: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:24.922 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:35:24 smithi083 ceph-mon[77242]: pgmap v5615: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:25.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:35:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:35:24.921Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:35:26.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:35:26 smithi153 ceph-mon[70044]: pgmap v5616: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:26.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:35:26 smithi083 ceph-mon[79930]: pgmap v5616: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:26.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:35:26 smithi083 ceph-mon[77242]: pgmap v5616: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:28.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:35:28 smithi153 ceph-mon[70044]: pgmap v5617: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:28.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:35:28 smithi083 ceph-mon[79930]: pgmap v5617: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:28.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:35:28 smithi083 ceph-mon[77242]: pgmap v5617: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:30.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:35:30 smithi153 ceph-mon[70044]: pgmap v5618: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:30.838 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:35:30 smithi083 ceph-mon[79930]: pgmap v5618: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:30.839 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:35:30 smithi083 ceph-mon[77242]: pgmap v5618: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:31.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:35:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:35:30.837Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:35:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:35:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:35:31] "GET /metrics HTTP/1.1" 200 225216 "" "Prometheus/2.33.4" 2022-09-29T08:35:32.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:35:32 smithi153 ceph-mon[70044]: pgmap v5619: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:35:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:35:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:35:32.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:35:32 smithi083 ceph-mon[79930]: pgmap v5619: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:32.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:35:32 smithi083 ceph-mon[77242]: pgmap v5619: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:35:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:35:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:35:34.522 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:35:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:35:34.225Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:35:34.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:35:34 smithi153 ceph-mon[70044]: pgmap v5620: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:34.921 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:35:34 smithi083 ceph-mon[79930]: pgmap v5620: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:34.921 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:35:34 smithi083 ceph-mon[77242]: pgmap v5620: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:35.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:35:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:35:34.919Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:35:36.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:35:36 smithi153 ceph-mon[70044]: pgmap v5621: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:36.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:35:36 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:35:36.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:35:36 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:35:36.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:35:36 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:35:36.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:35:36 smithi083 ceph-mon[79930]: pgmap v5621: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:36.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:35:36 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:35:36.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:35:36 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:35:36.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:35:36 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:35:36.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:35:36 smithi083 ceph-mon[77242]: pgmap v5621: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:36.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:35:36 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:35:36.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:35:36 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:35:36.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:35:36 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:35:38.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:35:38 smithi083 ceph-mon[79930]: pgmap v5622: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:38.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:35:38 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:35:38.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:35:38 smithi083 ceph-mon[77242]: pgmap v5622: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:38.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:35:38 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:35:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:35:38 smithi153 ceph-mon[70044]: pgmap v5622: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:38.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:35:38 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:35:40.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:35:40 smithi083 ceph-mon[79930]: pgmap v5623: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:35:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:35:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:35:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:35:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:35:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:35:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:35:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:35:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:35:40 smithi083 ceph-mon[77242]: pgmap v5623: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:35:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:35:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:35:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:35:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:35:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:35:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:35:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:35:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:35:40 smithi153 ceph-mon[70044]: pgmap v5623: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:35:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:35:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:35:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:35:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:35:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:35:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:35:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:35:41.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:35:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:35:40.838Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:35:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:35:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:35:41] "GET /metrics HTTP/1.1" 200 225214 "" "Prometheus/2.33.4" 2022-09-29T08:35:42.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:35:42 smithi083 ceph-mon[79930]: pgmap v5624: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:42.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:35:42 smithi083 ceph-mon[77242]: pgmap v5624: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:35:42 smithi153 ceph-mon[70044]: pgmap v5624: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:42.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:35:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:35:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:35:42.816 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:35:43.103 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:35:43.103 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 8m ago 3h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:35:43.103 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 6m ago 3h 63.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:35:43.104 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 8m ago 3h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:35:43.104 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 6m ago 3h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:35:43.104 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 6m ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:35:43.104 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 8m ago 3h 671M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:35:43.105 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 8m ago 3h 710M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:35:43.105 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 6m ago 3h 313M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:35:43.105 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 8m ago 3h 412M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:35:43.105 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 8m ago 3h 26.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:35:43.106 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 6m ago 3h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:35:43.106 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 8m ago 3h 513M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:35:43.106 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 8m ago 3h 504M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:35:43.106 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 8m ago 3h 397M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:35:43.107 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 8m ago 3h 490M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:35:43.107 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 6m ago 3h 435M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:35:43.107 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 6m ago 3h 537M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:35:43.107 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 6m ago 3h 478M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:35:43.107 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 6m ago 3h 562M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:35:43.108 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 6m ago 3h 153M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:35:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:35:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:35:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:35:43.465 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:35:43.465 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:35:43.465 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:35:43.466 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:35:43.466 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:35:43.466 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:35:43.466 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:35:43.467 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:35:43.467 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:35:43.467 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:35:43.467 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:35:43.468 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:35:43.468 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:35:43.468 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:35:43.468 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:35:43.468 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:35:43.469 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:35:43.469 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:35:44.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:35:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:35:44.225Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:35:44.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:35:44 smithi083 ceph-mon[77242]: from='client.50608 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:35:44.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:35:44 smithi083 ceph-mon[77242]: pgmap v5625: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:44.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:35:44 smithi083 ceph-mon[77242]: from='client.50614 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:35:44.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:35:44 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2779097965' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:35:44.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:35:44 smithi083 ceph-mon[79930]: from='client.50608 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:35:44.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:35:44 smithi083 ceph-mon[79930]: pgmap v5625: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:44.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:35:44 smithi083 ceph-mon[79930]: from='client.50614 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:35:44.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:35:44 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2779097965' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:35:44.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:35:44 smithi153 ceph-mon[70044]: from='client.50608 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:35:44.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:35:44 smithi153 ceph-mon[70044]: pgmap v5625: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:44.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:35:44 smithi153 ceph-mon[70044]: from='client.50614 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:35:44.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:35:44 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2779097965' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:35:45.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:35:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:35:44.920Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:35:46.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:35:46 smithi083 ceph-mon[79930]: pgmap v5626: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:46.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:35:46 smithi083 ceph-mon[77242]: pgmap v5626: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:46.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:35:46 smithi153 ceph-mon[70044]: pgmap v5626: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:48.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:35:48 smithi083 ceph-mon[79930]: pgmap v5627: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:48.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:35:48 smithi083 ceph-mon[77242]: pgmap v5627: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:48.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:35:48 smithi153 ceph-mon[70044]: pgmap v5627: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:50.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:35:50 smithi083 ceph-mon[79930]: pgmap v5628: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:50.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:35:50 smithi083 ceph-mon[77242]: pgmap v5628: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:35:50 smithi153 ceph-mon[70044]: pgmap v5628: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:51.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:35:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:35:50.838Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:35:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:35:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:35:51] "GET /metrics HTTP/1.1" 200 225214 "" "Prometheus/2.33.4" 2022-09-29T08:35:52.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:35:52 smithi083 ceph-mon[79930]: pgmap v5629: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:52.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:35:52 smithi083 ceph-mon[77242]: pgmap v5629: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:35:52 smithi153 ceph-mon[70044]: pgmap v5629: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:52.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:35:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:35:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:35:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:35:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:35:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:35:54.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:35:54 smithi083 ceph-mon[79930]: pgmap v5630: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:54.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:35:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:35:54.227Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:35:54.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:35:54 smithi083 ceph-mon[77242]: pgmap v5630: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:54.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:35:54 smithi153 ceph-mon[70044]: pgmap v5630: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:55.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:35:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:35:54.921Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:35:56.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:35:56 smithi083 ceph-mon[79930]: pgmap v5631: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:56.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:35:56 smithi083 ceph-mon[77242]: pgmap v5631: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:56.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:35:56 smithi153 ceph-mon[70044]: pgmap v5631: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:58.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:35:58 smithi083 ceph-mon[79930]: pgmap v5632: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:58.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:35:58 smithi083 ceph-mon[77242]: pgmap v5632: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:35:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:35:58 smithi153 ceph-mon[70044]: pgmap v5632: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:00.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:36:00 smithi083 ceph-mon[79930]: pgmap v5633: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:00.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:36:00 smithi083 ceph-mon[77242]: pgmap v5633: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:36:00 smithi153 ceph-mon[70044]: pgmap v5633: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:01.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:36:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:36:00.839Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:36:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:36:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:36:01] "GET /metrics HTTP/1.1" 200 225216 "" "Prometheus/2.33.4" 2022-09-29T08:36:02.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:36:02 smithi083 ceph-mon[79930]: pgmap v5634: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:02.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:36:02 smithi083 ceph-mon[77242]: pgmap v5634: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:36:02 smithi153 ceph-mon[70044]: pgmap v5634: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:02.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:36:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:36:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:36:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:36:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:36:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:36:04.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:36:04 smithi083 ceph-mon[79930]: pgmap v5635: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:04.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:36:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:36:04.227Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:36:04.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:36:04 smithi083 ceph-mon[77242]: pgmap v5635: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:36:04 smithi153 ceph-mon[70044]: pgmap v5635: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:05.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:36:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:36:04.921Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:36:06.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:36:06 smithi083 ceph-mon[79930]: pgmap v5636: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:06.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:36:06 smithi083 ceph-mon[77242]: pgmap v5636: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:06.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:36:06 smithi153 ceph-mon[70044]: pgmap v5636: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:08.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:36:08 smithi083 ceph-mon[79930]: pgmap v5637: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:08.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:36:08 smithi083 ceph-mon[77242]: pgmap v5637: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:36:08 smithi153 ceph-mon[70044]: pgmap v5637: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:10.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:36:10 smithi083 ceph-mon[79930]: pgmap v5638: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:10.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:36:10 smithi083 ceph-mon[77242]: pgmap v5638: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:36:10 smithi153 ceph-mon[70044]: pgmap v5638: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:11.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:36:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:36:10.839Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:36:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:36:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:36:11] "GET /metrics HTTP/1.1" 200 225212 "" "Prometheus/2.33.4" 2022-09-29T08:36:12.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:36:12 smithi083 ceph-mon[77242]: pgmap v5639: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:12.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:36:12 smithi083 ceph-mon[79930]: pgmap v5639: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:36:12 smithi153 ceph-mon[70044]: pgmap v5639: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:12.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:36:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:36:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:36:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:36:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:36:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:36:13.797 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:36:14.087 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:36:14.088 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 8m ago 3h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:36:14.088 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 6m ago 3h 63.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:36:14.088 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 8m ago 3h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:36:14.088 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 6m ago 3h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:36:14.088 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 6m ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:36:14.089 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 8m ago 3h 671M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:36:14.089 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 8m ago 3h 710M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:36:14.089 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 6m ago 3h 313M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:36:14.089 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 8m ago 3h 412M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:36:14.090 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 8m ago 3h 26.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:36:14.090 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 6m ago 3h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:36:14.090 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 8m ago 3h 513M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:36:14.090 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 8m ago 3h 504M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:36:14.090 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 8m ago 3h 397M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:36:14.091 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 8m ago 3h 490M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:36:14.091 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 6m ago 3h 435M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:36:14.091 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 6m ago 3h 537M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:36:14.091 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 6m ago 3h 478M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:36:14.091 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 6m ago 3h 562M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:36:14.092 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 6m ago 3h 153M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:36:14.447 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:36:14.447 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:36:14.448 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:36:14.448 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:36:14.448 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:36:14.448 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:36:14.449 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:36:14.449 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:36:14.449 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:36:14.449 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:36:14.449 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:36:14.450 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:36:14.450 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:36:14.450 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:36:14.450 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:36:14.450 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:36:14.451 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:36:14.451 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:36:14.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:36:14 smithi083 ceph-mon[79930]: pgmap v5640: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:14.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:36:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:36:14.228Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:36:14.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:36:14 smithi083 ceph-mon[77242]: pgmap v5640: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:14.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:36:14 smithi153 ceph-mon[70044]: pgmap v5640: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:15.173 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:36:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:36:14.920Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:36:15.174 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:36:15 smithi083 ceph-mon[77242]: from='client.50626 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:36:15.174 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:36:15 smithi083 ceph-mon[77242]: from='client.50632 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:36:15.174 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:36:15 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2043413781' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:36:15.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:36:15 smithi083 ceph-mon[79930]: from='client.50626 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:36:15.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:36:15 smithi083 ceph-mon[79930]: from='client.50632 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:36:15.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:36:15 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2043413781' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:36:15.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:36:15 smithi153 ceph-mon[70044]: from='client.50626 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:36:15.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:36:15 smithi153 ceph-mon[70044]: from='client.50632 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:36:15.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:36:15 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2043413781' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:36:16.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:36:16 smithi083 ceph-mon[79930]: pgmap v5641: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:16.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:36:16 smithi083 ceph-mon[77242]: pgmap v5641: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:36:16 smithi153 ceph-mon[70044]: pgmap v5641: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:18.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:36:18 smithi083 ceph-mon[79930]: pgmap v5642: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:18.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:36:18 smithi083 ceph-mon[77242]: pgmap v5642: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:36:18 smithi153 ceph-mon[70044]: pgmap v5642: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:20.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:36:20 smithi083 ceph-mon[77242]: pgmap v5643: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:20.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:36:20 smithi083 ceph-mon[79930]: pgmap v5643: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:36:20 smithi153 ceph-mon[70044]: pgmap v5643: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:21.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:36:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:36:20.839Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:36:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:36:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:36:21] "GET /metrics HTTP/1.1" 200 225212 "" "Prometheus/2.33.4" 2022-09-29T08:36:22.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:36:22 smithi083 ceph-mon[79930]: pgmap v5644: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:22.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:36:22 smithi083 ceph-mon[77242]: pgmap v5644: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:36:22 smithi153 ceph-mon[70044]: pgmap v5644: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:22.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:36:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:36:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:36:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:36:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:36:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:36:24.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:36:24 smithi083 ceph-mon[79930]: pgmap v5645: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:24.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:36:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:36:24.229Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:36:24.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:36:24 smithi083 ceph-mon[77242]: pgmap v5645: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:36:24 smithi153 ceph-mon[70044]: pgmap v5645: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:25.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:36:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:36:24.922Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:36:26.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:36:26 smithi083 ceph-mon[79930]: pgmap v5646: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:26.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:36:26 smithi083 ceph-mon[77242]: pgmap v5646: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:36:26 smithi153 ceph-mon[70044]: pgmap v5646: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:28.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:36:28 smithi083 ceph-mon[79930]: pgmap v5647: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:28.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:36:28 smithi083 ceph-mon[77242]: pgmap v5647: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:36:28 smithi153 ceph-mon[70044]: pgmap v5647: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:30.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:36:30 smithi083 ceph-mon[77242]: pgmap v5648: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:30.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:36:30 smithi083 ceph-mon[79930]: pgmap v5648: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:36:30 smithi153 ceph-mon[70044]: pgmap v5648: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:31.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:36:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:36:30.840Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:36:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:36:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:36:31] "GET /metrics HTTP/1.1" 200 225215 "" "Prometheus/2.33.4" 2022-09-29T08:36:32.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:36:32 smithi083 ceph-mon[79930]: pgmap v5649: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:32.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:36:32 smithi083 ceph-mon[77242]: pgmap v5649: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:36:32 smithi153 ceph-mon[70044]: pgmap v5649: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:32.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:36:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:36:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:36:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:36:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:36:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:36:34.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:36:34 smithi083 ceph-mon[79930]: pgmap v5650: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:34.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:36:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:36:34.230Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:36:34.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:36:34 smithi083 ceph-mon[77242]: pgmap v5650: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:36:34 smithi153 ceph-mon[70044]: pgmap v5650: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:35.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:36:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:36:34.923Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:36:36.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:36:36 smithi083 ceph-mon[79930]: pgmap v5651: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:36.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:36:36 smithi083 ceph-mon[77242]: pgmap v5651: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:36:36 smithi153 ceph-mon[70044]: pgmap v5651: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:38.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:36:38 smithi083 ceph-mon[79930]: pgmap v5652: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:38.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:36:38 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:36:38.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:36:38 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:36:38.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:36:38 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:36:38.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:36:38 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:36:38.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:36:38 smithi083 ceph-mon[77242]: pgmap v5652: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:38.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:36:38 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:36:38.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:36:38 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:36:38.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:36:38 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:36:38.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:36:38 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:36:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:36:38 smithi153 ceph-mon[70044]: pgmap v5652: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:36:38 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:36:38.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:36:38 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:36:38.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:36:38 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:36:38.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:36:38 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:36:40.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:36:40 smithi083 ceph-mon[79930]: pgmap v5653: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:36:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:36:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:36:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:36:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:36:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:36:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:36:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:36:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:36:40 smithi083 ceph-mon[77242]: pgmap v5653: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:36:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:36:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:36:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:36:40.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:36:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:36:40.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:36:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:36:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:36:40 smithi153 ceph-mon[70044]: pgmap v5653: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:36:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:36:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:36:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:36:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:36:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:36:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:36:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:36:41.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:36:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:36:40.841Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:36:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:36:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:36:41] "GET /metrics HTTP/1.1" 200 225216 "" "Prometheus/2.33.4" 2022-09-29T08:36:42.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:36:42 smithi083 ceph-mon[79930]: pgmap v5654: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:42.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:36:42 smithi083 ceph-mon[77242]: pgmap v5654: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:36:42 smithi153 ceph-mon[70044]: pgmap v5654: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:42.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:36:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:36:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:36:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:36:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:36:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:36:44.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:36:44 smithi083 ceph-mon[79930]: pgmap v5655: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:44.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:36:44 smithi083 ceph-mon[77242]: pgmap v5655: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:44.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:36:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:36:44.230Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:36:44.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:36:44 smithi153 ceph-mon[70044]: pgmap v5655: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:44.774 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:36:45.067 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:36:45.067 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 9m ago 3h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:36:45.067 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 7m ago 3h 63.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:36:45.068 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 9m ago 3h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:36:45.068 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 7m ago 3h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:36:45.068 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 7m ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:36:45.068 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 9m ago 3h 671M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:36:45.069 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 9m ago 3h 710M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:36:45.069 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 7m ago 3h 313M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:36:45.069 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 9m ago 3h 412M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:36:45.069 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 9m ago 3h 26.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:36:45.069 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 7m ago 3h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:36:45.070 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 9m ago 3h 513M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:36:45.070 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 9m ago 3h 504M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:36:45.070 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 9m ago 3h 397M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:36:45.070 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 9m ago 3h 490M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:36:45.071 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 7m ago 3h 435M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:36:45.071 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 7m ago 3h 537M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:36:45.071 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 7m ago 3h 478M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:36:45.071 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 7m ago 3h 562M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:36:45.071 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 7m ago 3h 153M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:36:45.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:36:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:36:44.923Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:36:45.426 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:36:45.426 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:36:45.426 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:36:45.427 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:36:45.427 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:36:45.427 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:36:45.427 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:36:45.427 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:36:45.428 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:36:45.428 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:36:45.428 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:36:45.428 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:36:45.429 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:36:45.429 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:36:45.429 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:36:45.429 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:36:45.429 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:36:45.430 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:36:46.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:36:46 smithi083 ceph-mon[79930]: from='client.50644 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:36:46.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:36:46 smithi083 ceph-mon[79930]: pgmap v5656: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:46.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:36:46 smithi083 ceph-mon[79930]: from='client.40719 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:36:46.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:36:46 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/417915689' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:36:46.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:36:46 smithi083 ceph-mon[77242]: from='client.50644 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:36:46.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:36:46 smithi083 ceph-mon[77242]: pgmap v5656: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:46.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:36:46 smithi083 ceph-mon[77242]: from='client.40719 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:36:46.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:36:46 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/417915689' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:36:46.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:36:46 smithi153 ceph-mon[70044]: from='client.50644 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:36:46.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:36:46 smithi153 ceph-mon[70044]: pgmap v5656: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:46.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:36:46 smithi153 ceph-mon[70044]: from='client.40719 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:36:46.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:36:46 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/417915689' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:36:48.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:36:48 smithi083 ceph-mon[79930]: pgmap v5657: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:48.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:36:48 smithi083 ceph-mon[77242]: pgmap v5657: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:48.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:36:48 smithi153 ceph-mon[70044]: pgmap v5657: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:50.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:36:50 smithi083 ceph-mon[77242]: pgmap v5658: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:50.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:36:50 smithi083 ceph-mon[79930]: pgmap v5658: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:36:50 smithi153 ceph-mon[70044]: pgmap v5658: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:51.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:36:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:36:50.842Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:36:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:36:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:36:51] "GET /metrics HTTP/1.1" 200 225216 "" "Prometheus/2.33.4" 2022-09-29T08:36:52.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:36:52 smithi083 ceph-mon[77242]: pgmap v5659: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:52.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:36:52 smithi083 ceph-mon[79930]: pgmap v5659: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:36:52 smithi153 ceph-mon[70044]: pgmap v5659: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:52.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:36:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:36:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:36:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:36:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:36:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:36:54.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:36:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:36:54.230Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:36:54.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:36:54 smithi083 ceph-mon[77242]: pgmap v5660: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:54.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:36:54 smithi083 ceph-mon[79930]: pgmap v5660: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:54.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:36:54 smithi153 ceph-mon[70044]: pgmap v5660: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:55.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:36:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:36:54.924Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:36:56.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:36:56 smithi083 ceph-mon[79930]: pgmap v5661: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:56.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:36:56 smithi083 ceph-mon[77242]: pgmap v5661: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:56.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:36:56 smithi153 ceph-mon[70044]: pgmap v5661: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:58.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:36:58 smithi083 ceph-mon[79930]: pgmap v5662: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:58.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:36:58 smithi083 ceph-mon[77242]: pgmap v5662: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:36:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:36:58 smithi153 ceph-mon[70044]: pgmap v5662: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:00.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:37:00 smithi083 ceph-mon[79930]: pgmap v5663: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:00.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:37:00 smithi083 ceph-mon[77242]: pgmap v5663: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:37:00 smithi153 ceph-mon[70044]: pgmap v5663: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:01.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:37:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:37:00.843Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:37:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:37:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:37:01] "GET /metrics HTTP/1.1" 200 225217 "" "Prometheus/2.33.4" 2022-09-29T08:37:02.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:37:02 smithi083 ceph-mon[77242]: pgmap v5664: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:02.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:37:02 smithi083 ceph-mon[79930]: pgmap v5664: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:37:02 smithi153 ceph-mon[70044]: pgmap v5664: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:02.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:37:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:37:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:37:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:37:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:37:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:37:04.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:37:04 smithi083 ceph-mon[79930]: pgmap v5665: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:04.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:37:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:37:04.232Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:37:04.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:37:04 smithi083 ceph-mon[77242]: pgmap v5665: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:37:04 smithi153 ceph-mon[70044]: pgmap v5665: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:05.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:37:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:37:04.924Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:37:06.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:37:06 smithi083 ceph-mon[79930]: pgmap v5666: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:06.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:37:06 smithi083 ceph-mon[77242]: pgmap v5666: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:06.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:37:06 smithi153 ceph-mon[70044]: pgmap v5666: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:08.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:37:08 smithi083 ceph-mon[79930]: pgmap v5667: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:08.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:37:08 smithi083 ceph-mon[77242]: pgmap v5667: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:37:08 smithi153 ceph-mon[70044]: pgmap v5667: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:10.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:37:10 smithi083 ceph-mon[79930]: pgmap v5668: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:10.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:37:10 smithi083 ceph-mon[77242]: pgmap v5668: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:37:10 smithi153 ceph-mon[70044]: pgmap v5668: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:11.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:37:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:37:10.844Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:37:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:37:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:37:11] "GET /metrics HTTP/1.1" 200 225213 "" "Prometheus/2.33.4" 2022-09-29T08:37:12.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:37:12 smithi083 ceph-mon[77242]: pgmap v5669: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:12.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:37:12 smithi083 ceph-mon[79930]: pgmap v5669: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:37:12 smithi153 ceph-mon[70044]: pgmap v5669: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:12.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:37:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:37:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:37:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:37:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:37:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:37:14.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:37:14 smithi083 ceph-mon[79930]: pgmap v5670: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:14.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:37:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:37:14.233Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:37:14.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:37:14 smithi083 ceph-mon[77242]: pgmap v5670: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:14.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:37:14 smithi153 ceph-mon[70044]: pgmap v5670: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:15.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:37:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:37:14.924Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:37:15.755 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:37:16.043 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:37:16.043 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 9m ago 3h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:37:16.043 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 7m ago 3h 63.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:37:16.044 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 9m ago 3h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:37:16.044 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 7m ago 3h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:37:16.044 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 7m ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:37:16.044 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 9m ago 3h 671M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:37:16.044 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 9m ago 3h 710M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:37:16.045 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 7m ago 3h 313M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:37:16.045 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 9m ago 3h 412M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:37:16.045 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 9m ago 3h 26.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:37:16.045 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 7m ago 3h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:37:16.045 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 9m ago 3h 513M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:37:16.046 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 9m ago 3h 504M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:37:16.046 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 9m ago 3h 397M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:37:16.046 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 9m ago 3h 490M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:37:16.046 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 7m ago 3h 435M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:37:16.047 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 7m ago 3h 537M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:37:16.047 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 7m ago 3h 478M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:37:16.047 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 7m ago 3h 562M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:37:16.047 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 7m ago 3h 153M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:37:16.397 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:37:16.397 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:37:16.398 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:37:16.398 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:37:16.398 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:37:16.398 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:37:16.399 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:37:16.399 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:37:16.399 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:37:16.399 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:37:16.399 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:37:16.400 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:37:16.400 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:37:16.400 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:37:16.400 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:37:16.400 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:37:16.401 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:37:16.401 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:37:16.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:37:16 smithi083 ceph-mon[79930]: pgmap v5671: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:16.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:37:16 smithi083 ceph-mon[77242]: pgmap v5671: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:37:16 smithi153 ceph-mon[70044]: pgmap v5671: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:17.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:37:17 smithi083 ceph-mon[79930]: from='client.50662 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:37:17.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:37:17 smithi083 ceph-mon[79930]: from='client.40737 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:37:17.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:37:17 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3898717522' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:37:17.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:37:17 smithi083 ceph-mon[77242]: from='client.50662 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:37:17.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:37:17 smithi083 ceph-mon[77242]: from='client.40737 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:37:17.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:37:17 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3898717522' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:37:17.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:37:17 smithi153 ceph-mon[70044]: from='client.50662 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:37:17.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:37:17 smithi153 ceph-mon[70044]: from='client.40737 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:37:17.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:37:17 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3898717522' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:37:18.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:37:18 smithi083 ceph-mon[79930]: pgmap v5672: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:18.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:37:18 smithi083 ceph-mon[77242]: pgmap v5672: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:37:18 smithi153 ceph-mon[70044]: pgmap v5672: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:20.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:37:20 smithi083 ceph-mon[79930]: pgmap v5673: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:20.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:37:20 smithi083 ceph-mon[77242]: pgmap v5673: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:37:20 smithi153 ceph-mon[70044]: pgmap v5673: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:21.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:37:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:37:20.845Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:37:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:37:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:37:21] "GET /metrics HTTP/1.1" 200 225213 "" "Prometheus/2.33.4" 2022-09-29T08:37:22.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:37:22 smithi083 ceph-mon[79930]: pgmap v5674: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:22.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:37:22 smithi083 ceph-mon[77242]: pgmap v5674: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:37:22 smithi153 ceph-mon[70044]: pgmap v5674: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:22.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:37:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:37:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:37:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:37:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:37:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:37:24.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:37:24 smithi083 ceph-mon[79930]: pgmap v5675: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:24.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:37:24 smithi083 ceph-mon[77242]: pgmap v5675: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:24.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:37:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:37:24.233Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:37:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:37:24 smithi153 ceph-mon[70044]: pgmap v5675: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:25.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:37:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:37:24.925Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:37:26.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:37:26 smithi083 ceph-mon[77242]: pgmap v5676: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:26.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:37:26 smithi083 ceph-mon[79930]: pgmap v5676: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:37:26 smithi153 ceph-mon[70044]: pgmap v5676: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:37:28 smithi153 ceph-mon[70044]: pgmap v5677: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:28.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:37:28 smithi083 ceph-mon[77242]: pgmap v5677: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:28.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:37:28 smithi083 ceph-mon[79930]: pgmap v5677: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:37:30 smithi153 ceph-mon[70044]: pgmap v5678: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:30.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:37:30 smithi083 ceph-mon[77242]: pgmap v5678: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:30.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:37:30 smithi083 ceph-mon[79930]: pgmap v5678: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:31.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:37:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:37:30.845Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:37:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:37:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:37:31] "GET /metrics HTTP/1.1" 200 225219 "" "Prometheus/2.33.4" 2022-09-29T08:37:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:37:32 smithi153 ceph-mon[70044]: pgmap v5679: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:32.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:37:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:37:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:37:32.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:37:32 smithi083 ceph-mon[77242]: pgmap v5679: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:32.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:37:32 smithi083 ceph-mon[79930]: pgmap v5679: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:37:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:37:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:37:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:37:34 smithi153 ceph-mon[70044]: pgmap v5680: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:34.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:37:34 smithi083 ceph-mon[79930]: pgmap v5680: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:34.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:37:34 smithi083 ceph-mon[77242]: pgmap v5680: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:34.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:37:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:37:34.234Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:37:35.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:37:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:37:34.926Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:37:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:37:36 smithi153 ceph-mon[70044]: pgmap v5681: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:36.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:37:36 smithi083 ceph-mon[77242]: pgmap v5681: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:36.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:37:36 smithi083 ceph-mon[79930]: pgmap v5681: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:37:38 smithi153 ceph-mon[70044]: pgmap v5682: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:38.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:37:38 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:37:38.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:37:38 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:37:38.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:37:38 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:37:38.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:37:38 smithi083 ceph-mon[79930]: pgmap v5682: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:38.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:37:38 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:37:38.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:37:38 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:37:38.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:37:38 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:37:38.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:37:38 smithi083 ceph-mon[77242]: pgmap v5682: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:38.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:37:38 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:37:38.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:37:38 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:37:38.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:37:38 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:37:39.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:37:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:37:39.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:37:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:37:39.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:37:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:37:39.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:37:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:37:39.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:37:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:37:39.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:37:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:37:39.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:37:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:37:39.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:37:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:37:39.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:37:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:37:39.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:37:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:37:39.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:37:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:37:39.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:37:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:37:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:37:40 smithi153 ceph-mon[70044]: pgmap v5683: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:37:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:37:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:37:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:37:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:37:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:37:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:37:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:37:40.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:37:40 smithi083 ceph-mon[79930]: pgmap v5683: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:40.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:37:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:37:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:37:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:37:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:37:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:37:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:37:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:37:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:37:40 smithi083 ceph-mon[77242]: pgmap v5683: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:37:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:37:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:37:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:37:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:37:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:37:40.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:37:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:37:41.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:37:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:37:40.847Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:37:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:37:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:37:41] "GET /metrics HTTP/1.1" 200 225216 "" "Prometheus/2.33.4" 2022-09-29T08:37:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:37:42 smithi153 ceph-mon[70044]: pgmap v5684: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:37:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:37:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:37:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:37:42.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:37:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:37:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:37:42.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:37:42 smithi083 ceph-mon[79930]: pgmap v5684: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:42.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:37:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:37:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:37:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:37:42.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:37:42 smithi083 ceph-mon[77242]: pgmap v5684: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:42.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:37:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:37:42.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:37:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:37:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:37:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:37:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:37:44.236 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:37:44 smithi083 ceph-mon[79930]: pgmap v5685: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:44.237 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:37:44 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:37:44.237 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:37:44 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:37:44.237 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:37:44 smithi083 ceph-mon[77242]: pgmap v5685: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:44.238 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:37:44 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:37:44.238 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:37:44 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:37:44.547 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:37:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:37:44.234Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:37:44.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:37:44 smithi153 ceph-mon[70044]: pgmap v5685: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:44.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:37:44 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:37:44.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:37:44 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:37:45.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:37:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:37:44.926Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:37:46.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:37:46 smithi083 ceph-mon[79930]: pgmap v5686: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:46.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:37:46 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:37:46.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:37:46 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:37:46.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:37:46 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:37:46.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:37:46 smithi083 ceph-mon[77242]: pgmap v5686: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:46.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:37:46 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:37:46.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:37:46 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:37:46.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:37:46 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:37:46.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:37:46 smithi153 ceph-mon[70044]: pgmap v5686: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:46.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:37:46 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:37:46.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:37:46 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:37:46.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:37:46 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:37:46.713 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:37:47.002 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:37:47.003 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 4s ago 3h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:37:47.003 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 8m ago 3h 63.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:37:47.003 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 4s ago 3h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:37:47.003 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 8m ago 3h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:37:47.003 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 8m ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:37:47.004 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 4s ago 3h 680M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:37:47.004 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 4s ago 3h 743M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:37:47.004 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 8m ago 3h 313M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:37:47.004 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 4s ago 3h 436M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:37:47.004 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 4s ago 3h 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:37:47.005 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 8m ago 3h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:37:47.005 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 4s ago 3h 538M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:37:47.005 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 4s ago 3h 529M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:37:47.005 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 4s ago 3h 417M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:37:47.005 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 4s ago 3h 515M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:37:47.006 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 8m ago 3h 435M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:37:47.006 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 8m ago 3h 537M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:37:47.006 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 8m ago 3h 478M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:37:47.006 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 8m ago 3h 562M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:37:47.007 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 8m ago 3h 153M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:37:47.359 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:37:47.360 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:37:47.360 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:37:47.360 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:37:47.360 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:37:47.361 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:37:47.361 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:37:47.361 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:37:47.361 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:37:47.361 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:37:47.362 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:37:47.362 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:37:47.362 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:37:47.362 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:37:47.362 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:37:47.363 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:37:47.363 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:37:47.363 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:37:47.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:37:47 smithi083 ceph-mon[79930]: Detected new or changed devices on smithi083 2022-09-29T08:37:47.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:37:47 smithi083 ceph-mon[77242]: Detected new or changed devices on smithi083 2022-09-29T08:37:47.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:37:47 smithi153 ceph-mon[70044]: Detected new or changed devices on smithi083 2022-09-29T08:37:48.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:37:48 smithi083 ceph-mon[79930]: from='client.50680 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:37:48.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:37:48 smithi083 ceph-mon[79930]: from='client.50686 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:37:48.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:37:48 smithi083 ceph-mon[79930]: pgmap v5687: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:48.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:37:48 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2144761255' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:37:48.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:37:48 smithi083 ceph-mon[77242]: from='client.50680 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:37:48.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:37:48 smithi083 ceph-mon[77242]: from='client.50686 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:37:48.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:37:48 smithi083 ceph-mon[77242]: pgmap v5687: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:48.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:37:48 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2144761255' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:37:48.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:37:48 smithi153 ceph-mon[70044]: from='client.50680 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:37:48.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:37:48 smithi153 ceph-mon[70044]: from='client.50686 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:37:48.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:37:48 smithi153 ceph-mon[70044]: pgmap v5687: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:48.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:37:48 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2144761255' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:37:50.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:37:50 smithi083 ceph-mon[79930]: pgmap v5688: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:50.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:37:50 smithi083 ceph-mon[77242]: pgmap v5688: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:37:50 smithi153 ceph-mon[70044]: pgmap v5688: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:51.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:37:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:37:50.847Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:37:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:37:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:37:51] "GET /metrics HTTP/1.1" 200 225216 "" "Prometheus/2.33.4" 2022-09-29T08:37:52.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:37:52 smithi083 ceph-mon[79930]: pgmap v5689: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:52.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:37:52 smithi083 ceph-mon[77242]: pgmap v5689: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:52.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:37:52 smithi153 ceph-mon[70044]: pgmap v5689: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:52.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:37:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:37:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:37:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:37:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:37:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:37:54.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:37:54 smithi083 ceph-mon[79930]: pgmap v5690: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:54.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:37:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:37:54.235Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:37:54.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:37:54 smithi083 ceph-mon[77242]: pgmap v5690: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:54.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:37:54 smithi153 ceph-mon[70044]: pgmap v5690: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:55.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:37:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:37:54.928Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:37:56.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:37:56 smithi083 ceph-mon[77242]: pgmap v5691: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:56.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:37:56 smithi083 ceph-mon[79930]: pgmap v5691: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:56.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:37:56 smithi153 ceph-mon[70044]: pgmap v5691: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:58.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:37:58 smithi083 ceph-mon[77242]: pgmap v5692: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:58.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:37:58 smithi083 ceph-mon[79930]: pgmap v5692: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:37:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:37:58 smithi153 ceph-mon[70044]: pgmap v5692: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:00.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:38:00 smithi083 ceph-mon[79930]: pgmap v5693: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:00.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:38:00 smithi083 ceph-mon[77242]: pgmap v5693: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:38:00 smithi153 ceph-mon[70044]: pgmap v5693: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:01.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:38:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:38:00.848Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:38:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:38:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:38:01] "GET /metrics HTTP/1.1" 200 225217 "" "Prometheus/2.33.4" 2022-09-29T08:38:02.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:38:02 smithi083 ceph-mon[79930]: pgmap v5694: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:02.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:38:02 smithi083 ceph-mon[77242]: pgmap v5694: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:02.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:38:02 smithi153 ceph-mon[70044]: pgmap v5694: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:02.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:38:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:38:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:38:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:38:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:38:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:38:04.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:38:04 smithi083 ceph-mon[77242]: pgmap v5695: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:04.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:38:04 smithi083 ceph-mon[79930]: pgmap v5695: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:04.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:38:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:38:04.236Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:38:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:38:04 smithi153 ceph-mon[70044]: pgmap v5695: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:05.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:38:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:38:04.927Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:38:06.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:38:06 smithi083 ceph-mon[79930]: pgmap v5696: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:06.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:38:06 smithi083 ceph-mon[77242]: pgmap v5696: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:06.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:38:06 smithi153 ceph-mon[70044]: pgmap v5696: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:08.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:38:08 smithi083 ceph-mon[79930]: pgmap v5697: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:08.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:38:08 smithi083 ceph-mon[77242]: pgmap v5697: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:38:08 smithi153 ceph-mon[70044]: pgmap v5697: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:10.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:38:10 smithi083 ceph-mon[79930]: pgmap v5698: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:10.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:38:10 smithi083 ceph-mon[77242]: pgmap v5698: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:38:10 smithi153 ceph-mon[70044]: pgmap v5698: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:11.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:38:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:38:10.848Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:38:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:38:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:38:11] "GET /metrics HTTP/1.1" 200 225214 "" "Prometheus/2.33.4" 2022-09-29T08:38:12.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:38:12 smithi083 ceph-mon[77242]: pgmap v5699: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:12.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:38:12 smithi083 ceph-mon[79930]: pgmap v5699: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:38:12 smithi153 ceph-mon[70044]: pgmap v5699: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:12.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:38:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:38:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:38:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:38:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:38:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:38:14.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:38:14 smithi083 ceph-mon[79930]: pgmap v5700: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:14.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:38:14 smithi083 ceph-mon[77242]: pgmap v5700: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:14.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:38:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:38:14.236Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:38:14.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:38:14 smithi153 ceph-mon[70044]: pgmap v5700: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:15.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:38:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:38:14.929Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:38:16.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:38:16 smithi083 ceph-mon[79930]: pgmap v5701: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:16.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:38:16 smithi083 ceph-mon[77242]: pgmap v5701: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:38:16 smithi153 ceph-mon[70044]: pgmap v5701: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:17.688 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:38:17.976 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:38:17.977 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 35s ago 3h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:38:17.977 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 8m ago 3h 63.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:38:17.977 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 35s ago 3h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:38:17.977 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 8m ago 3h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:38:17.978 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 8m ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:38:17.978 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 35s ago 3h 680M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:38:17.978 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 35s ago 3h 743M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:38:17.978 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 8m ago 3h 313M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:38:17.978 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 35s ago 3h 436M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:38:17.979 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 35s ago 3h 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:38:17.979 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 8m ago 3h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:38:17.979 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 35s ago 3h 538M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:38:17.979 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 35s ago 3h 529M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:38:17.979 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 35s ago 3h 417M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:38:17.980 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 35s ago 3h 515M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:38:17.980 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 8m ago 3h 435M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:38:17.980 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 8m ago 3h 537M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:38:17.980 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 8m ago 3h 478M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:38:17.981 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 8m ago 3h 562M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:38:17.981 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 8m ago 3h 153M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:38:18.335 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:38:18.336 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:38:18.336 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:38:18.336 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:38:18.337 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:38:18.337 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:38:18.337 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:38:18.337 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:38:18.338 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:38:18.338 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:38:18.338 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:38:18.338 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:38:18.339 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:38:18.339 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:38:18.339 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:38:18.339 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:38:18.340 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:38:18.340 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:38:18.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:38:18 smithi083 ceph-mon[79930]: pgmap v5702: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:18.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:38:18 smithi083 ceph-mon[77242]: pgmap v5702: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:38:18 smithi153 ceph-mon[70044]: pgmap v5702: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:19.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:38:19 smithi083 ceph-mon[79930]: from='client.50698 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:38:19.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:38:19 smithi083 ceph-mon[79930]: from='client.50704 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:38:19.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:38:19 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4280879665' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:38:19.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:38:19 smithi083 ceph-mon[77242]: from='client.50698 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:38:19.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:38:19 smithi083 ceph-mon[77242]: from='client.50704 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:38:19.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:38:19 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4280879665' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:38:19.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:38:19 smithi153 ceph-mon[70044]: from='client.50698 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:38:19.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:38:19 smithi153 ceph-mon[70044]: from='client.50704 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:38:19.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:38:19 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4280879665' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:38:20.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:38:20 smithi083 ceph-mon[79930]: pgmap v5703: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:20.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:38:20 smithi083 ceph-mon[77242]: pgmap v5703: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:38:20 smithi153 ceph-mon[70044]: pgmap v5703: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:21.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:38:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:38:20.848Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:38:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:38:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:38:21] "GET /metrics HTTP/1.1" 200 225214 "" "Prometheus/2.33.4" 2022-09-29T08:38:22.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:38:22 smithi083 ceph-mon[77242]: pgmap v5704: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:22.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:38:22 smithi083 ceph-mon[79930]: pgmap v5704: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:38:22 smithi153 ceph-mon[70044]: pgmap v5704: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:22.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:38:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:38:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:38:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:38:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:38:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:38:24.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:38:24 smithi083 ceph-mon[79930]: pgmap v5705: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:24.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:38:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:38:24.237Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:38:24.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:38:24 smithi083 ceph-mon[77242]: pgmap v5705: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:38:24 smithi153 ceph-mon[70044]: pgmap v5705: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:25.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:38:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:38:24.930Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:38:26.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:38:26 smithi083 ceph-mon[77242]: pgmap v5706: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:26.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:38:26 smithi083 ceph-mon[79930]: pgmap v5706: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:38:26 smithi153 ceph-mon[70044]: pgmap v5706: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:28.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:38:28 smithi083 ceph-mon[77242]: pgmap v5707: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:28.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:38:28 smithi083 ceph-mon[79930]: pgmap v5707: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:38:28 smithi153 ceph-mon[70044]: pgmap v5707: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:38:30 smithi153 ceph-mon[70044]: pgmap v5708: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:30.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:38:30 smithi083 ceph-mon[79930]: pgmap v5708: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:30.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:38:30 smithi083 ceph-mon[77242]: pgmap v5708: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:31.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:38:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:38:30.849Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:38:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:38:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:38:31] "GET /metrics HTTP/1.1" 200 225218 "" "Prometheus/2.33.4" 2022-09-29T08:38:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:38:32 smithi153 ceph-mon[70044]: pgmap v5709: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:32.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:38:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:38:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:38:32.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:38:32 smithi083 ceph-mon[79930]: pgmap v5709: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:32.753 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:38:32 smithi083 ceph-mon[77242]: pgmap v5709: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:38:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:38:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:38:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:38:34 smithi153 ceph-mon[70044]: pgmap v5710: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:34.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:38:34 smithi083 ceph-mon[79930]: pgmap v5710: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:34.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:38:34 smithi083 ceph-mon[77242]: pgmap v5710: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:34.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:38:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:38:34.238Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:38:35.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:38:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:38:34.930Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:38:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:38:36 smithi153 ceph-mon[70044]: pgmap v5711: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:36.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:38:36 smithi083 ceph-mon[79930]: pgmap v5711: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:36.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:38:36 smithi083 ceph-mon[77242]: pgmap v5711: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:38:38 smithi153 ceph-mon[70044]: pgmap v5712: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:38.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:38:38 smithi083 ceph-mon[79930]: pgmap v5712: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:38.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:38:38 smithi083 ceph-mon[77242]: pgmap v5712: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:38:40 smithi153 ceph-mon[70044]: pgmap v5713: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:38:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:38:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:38:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:38:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:38:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:38:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:38:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:38:40.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:38:40 smithi083 ceph-mon[79930]: pgmap v5713: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:38:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:38:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:38:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:38:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:38:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:38:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:38:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:38:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:38:40 smithi083 ceph-mon[77242]: pgmap v5713: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:38:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:38:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:38:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:38:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:38:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:38:40.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:38:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:38:41.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:38:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:38:40.849Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:38:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:38:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:38:41] "GET /metrics HTTP/1.1" 200 225214 "" "Prometheus/2.33.4" 2022-09-29T08:38:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:38:42 smithi153 ceph-mon[70044]: pgmap v5714: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:42.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:38:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:38:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:38:42.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:38:42 smithi083 ceph-mon[79930]: pgmap v5714: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:42.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:38:42 smithi083 ceph-mon[77242]: pgmap v5714: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:38:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:38:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:38:44.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:38:44 smithi153 ceph-mon[70044]: pgmap v5715: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:44.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:38:44 smithi083 ceph-mon[79930]: pgmap v5715: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:38:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:38:44.239Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:38:44.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:38:44 smithi083 ceph-mon[77242]: pgmap v5715: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:45.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:38:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:38:44.931Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:38:46.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:38:46 smithi153 ceph-mon[70044]: pgmap v5716: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:46.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:38:46 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:38:46.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:38:46 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:38:46.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:38:46 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:38:46.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:38:46 smithi083 ceph-mon[77242]: pgmap v5716: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:46.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:38:46 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:38:46.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:38:46 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:38:46.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:38:46 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:38:46.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:38:46 smithi083 ceph-mon[79930]: pgmap v5716: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:46.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:38:46 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:38:46.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:38:46 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:38:46.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:38:46 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:38:47.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:38:47 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:38:47.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:38:47 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:38:47.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:38:47 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:38:48.660 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:38:48.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:38:48 smithi083 ceph-mon[79930]: pgmap v5717: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:48.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:38:48 smithi083 ceph-mon[77242]: pgmap v5717: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:48.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:38:48 smithi153 ceph-mon[70044]: pgmap v5717: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:48.948 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:38:48.948 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 66s ago 3h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:38:48.949 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 9m ago 3h 63.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:38:48.949 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 66s ago 3h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:38:48.949 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 9m ago 3h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:38:48.949 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 9m ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:38:48.950 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 66s ago 3h 680M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:38:48.950 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 66s ago 3h 743M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:38:48.950 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 9m ago 3h 313M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:38:48.950 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 66s ago 3h 436M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:38:48.951 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 66s ago 3h 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:38:48.951 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 9m ago 3h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:38:48.951 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 66s ago 3h 538M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:38:48.951 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 66s ago 3h 529M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:38:48.951 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 66s ago 3h 417M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:38:48.952 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 66s ago 3h 515M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:38:48.952 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 9m ago 3h 435M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:38:48.952 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 9m ago 3h 537M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:38:48.952 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 9m ago 3h 478M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:38:48.952 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 9m ago 3h 562M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:38:48.953 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 9m ago 3h 153M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:38:49.303 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:38:49.303 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:38:49.303 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:38:49.304 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:38:49.304 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:38:49.304 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:38:49.304 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:38:49.305 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:38:49.305 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:38:49.305 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:38:49.305 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:38:49.306 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:38:49.306 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:38:49.306 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:38:49.306 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:38:49.307 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:38:49.307 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:38:49.307 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:38:49.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:38:49 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1050325210' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:38:49.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:38:49 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1050325210' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:38:49.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:38:49 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1050325210' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:38:50.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:38:50 smithi083 ceph-mon[79930]: from='client.40785 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:38:50.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:38:50 smithi083 ceph-mon[79930]: from='client.40791 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:38:50.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:38:50 smithi083 ceph-mon[79930]: pgmap v5718: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:50.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:38:50 smithi083 ceph-mon[77242]: from='client.40785 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:38:50.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:38:50 smithi083 ceph-mon[77242]: from='client.40791 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:38:50.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:38:50 smithi083 ceph-mon[77242]: pgmap v5718: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:50.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:38:50 smithi153 ceph-mon[70044]: from='client.40785 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:38:50.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:38:50 smithi153 ceph-mon[70044]: from='client.40791 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:38:50.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:38:50 smithi153 ceph-mon[70044]: pgmap v5718: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:51.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:38:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:38:50.849Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:38:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:38:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:38:51] "GET /metrics HTTP/1.1" 200 225214 "" "Prometheus/2.33.4" 2022-09-29T08:38:52.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:38:52 smithi083 ceph-mon[79930]: pgmap v5719: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:52.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:38:52 smithi083 ceph-mon[77242]: pgmap v5719: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:52.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:38:52 smithi153 ceph-mon[70044]: pgmap v5719: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:38:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:38:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:38:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:38:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:38:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:38:54.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:38:54 smithi083 ceph-mon[79930]: pgmap v5720: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:38:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:38:54.240Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:38:54.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:38:54 smithi083 ceph-mon[77242]: pgmap v5720: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:54.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:38:54 smithi153 ceph-mon[70044]: pgmap v5720: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:55.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:38:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:38:54.931Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:38:56.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:38:56 smithi083 ceph-mon[79930]: pgmap v5721: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:56.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:38:56 smithi083 ceph-mon[77242]: pgmap v5721: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:56.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:38:56 smithi153 ceph-mon[70044]: pgmap v5721: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:58.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:38:58 smithi083 ceph-mon[79930]: pgmap v5722: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:58.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:38:58 smithi083 ceph-mon[77242]: pgmap v5722: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:38:58.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:38:58 smithi153 ceph-mon[70044]: pgmap v5722: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:00.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:39:00 smithi083 ceph-mon[79930]: pgmap v5723: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:00.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:39:00 smithi083 ceph-mon[77242]: pgmap v5723: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:00.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:39:00 smithi153 ceph-mon[70044]: pgmap v5723: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:01.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:39:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:39:00.850Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:39:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:39:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:39:01] "GET /metrics HTTP/1.1" 200 225214 "" "Prometheus/2.33.4" 2022-09-29T08:39:02.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:39:02 smithi083 ceph-mon[79930]: pgmap v5724: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:02.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:39:02 smithi083 ceph-mon[77242]: pgmap v5724: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:02.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:39:02 smithi153 ceph-mon[70044]: pgmap v5724: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:39:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:39:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:39:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:39:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:39:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:39:04.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:39:04 smithi083 ceph-mon[79930]: pgmap v5725: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:04.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:39:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:39:04.240Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:39:04.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:39:04 smithi083 ceph-mon[77242]: pgmap v5725: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:04.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:39:04 smithi153 ceph-mon[70044]: pgmap v5725: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:05.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:39:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:39:04.932Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:39:06.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:39:06 smithi083 ceph-mon[77242]: pgmap v5726: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:06.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:39:06 smithi083 ceph-mon[79930]: pgmap v5726: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:06.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:39:06 smithi153 ceph-mon[70044]: pgmap v5726: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:08.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:39:08 smithi083 ceph-mon[79930]: pgmap v5727: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:08.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:39:08 smithi083 ceph-mon[77242]: pgmap v5727: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:08.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:39:08 smithi153 ceph-mon[70044]: pgmap v5727: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:10.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:39:10 smithi083 ceph-mon[79930]: pgmap v5728: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:10.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:39:10 smithi083 ceph-mon[77242]: pgmap v5728: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:10.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:39:10 smithi153 ceph-mon[70044]: pgmap v5728: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:11.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:39:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:39:10.851Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:39:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:39:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:39:11] "GET /metrics HTTP/1.1" 200 225216 "" "Prometheus/2.33.4" 2022-09-29T08:39:12.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:39:12 smithi083 ceph-mon[79930]: pgmap v5729: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:12.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:39:12 smithi083 ceph-mon[77242]: pgmap v5729: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:12.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:39:12 smithi153 ceph-mon[70044]: pgmap v5729: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:39:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:39:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:39:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:39:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:39:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:39:14.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:39:14 smithi083 ceph-mon[79930]: pgmap v5730: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:14.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:39:14 smithi083 ceph-mon[77242]: pgmap v5730: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:14.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:39:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:39:14.241Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:39:14.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:39:14 smithi153 ceph-mon[70044]: pgmap v5730: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:15.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:39:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:39:14.932Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:39:16.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:39:16 smithi083 ceph-mon[77242]: pgmap v5731: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:16.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:39:16 smithi083 ceph-mon[79930]: pgmap v5731: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:16.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:39:16 smithi153 ceph-mon[70044]: pgmap v5731: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:18.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:39:18 smithi083 ceph-mon[79930]: pgmap v5732: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:18.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:39:18 smithi083 ceph-mon[77242]: pgmap v5732: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:18.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:39:18 smithi153 ceph-mon[70044]: pgmap v5732: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:19.627 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:39:19.918 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:39:19.919 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 97s ago 3h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:39:19.919 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 9m ago 3h 63.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:39:19.919 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 97s ago 3h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:39:19.920 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 9m ago 3h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:39:19.920 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 9m ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:39:19.920 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 97s ago 3h 680M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:39:19.920 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 97s ago 3h 743M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:39:19.921 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 9m ago 3h 313M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:39:19.921 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 97s ago 3h 436M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:39:19.921 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 97s ago 3h 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:39:19.921 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 9m ago 3h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:39:19.922 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 97s ago 3h 538M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:39:19.922 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 97s ago 3h 529M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:39:19.922 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 97s ago 3h 417M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:39:19.923 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 97s ago 3h 515M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:39:19.923 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 9m ago 3h 435M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:39:19.923 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 9m ago 3h 537M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:39:19.923 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 9m ago 3h 478M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:39:19.924 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 9m ago 3h 562M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:39:19.924 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 9m ago 3h 153M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:39:20.272 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:39:20.272 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:39:20.272 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:39:20.272 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:39:20.273 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:39:20.273 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:39:20.273 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:39:20.273 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:39:20.274 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:39:20.274 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:39:20.274 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:39:20.274 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:39:20.275 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:39:20.275 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:39:20.275 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:39:20.275 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:39:20.275 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:39:20.276 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:39:20.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:39:20 smithi083 ceph-mon[79930]: pgmap v5733: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:20.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:39:20 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4028045687' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:39:20.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:39:20 smithi083 ceph-mon[77242]: pgmap v5733: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:20.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:39:20 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4028045687' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:39:20.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:39:20 smithi153 ceph-mon[70044]: pgmap v5733: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:20.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:39:20 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4028045687' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:39:21.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:39:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:39:20.851Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:39:21.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:39:21 smithi153 ceph-mon[70044]: from='client.50734 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:39:21.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:39:21 smithi153 ceph-mon[70044]: from='client.40809 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:39:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:39:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:39:21] "GET /metrics HTTP/1.1" 200 225216 "" "Prometheus/2.33.4" 2022-09-29T08:39:21.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:39:21 smithi083 ceph-mon[77242]: from='client.50734 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:39:21.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:39:21 smithi083 ceph-mon[77242]: from='client.40809 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:39:21.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:39:21 smithi083 ceph-mon[79930]: from='client.50734 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:39:21.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:39:21 smithi083 ceph-mon[79930]: from='client.40809 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:39:22.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:39:22 smithi153 ceph-mon[70044]: pgmap v5734: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:39:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:39:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:39:22.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:39:22 smithi083 ceph-mon[79930]: pgmap v5734: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:22.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:39:22 smithi083 ceph-mon[77242]: pgmap v5734: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:39:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:39:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:39:24.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:39:24 smithi083 ceph-mon[77242]: pgmap v5735: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:24.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:39:24 smithi083 ceph-mon[79930]: pgmap v5735: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:39:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:39:24.241Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:39:24.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:39:24 smithi153 ceph-mon[70044]: pgmap v5735: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:25.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:39:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:39:24.932Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:39:26.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:39:26 smithi153 ceph-mon[70044]: pgmap v5736: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:26.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:39:26 smithi083 ceph-mon[79930]: pgmap v5736: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:26.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:39:26 smithi083 ceph-mon[77242]: pgmap v5736: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:28.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:39:28 smithi153 ceph-mon[70044]: pgmap v5737: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:28.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:39:28 smithi083 ceph-mon[77242]: pgmap v5737: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:28.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:39:28 smithi083 ceph-mon[79930]: pgmap v5737: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:30.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:39:30 smithi153 ceph-mon[70044]: pgmap v5738: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:30.854 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:39:30 smithi083 ceph-mon[79930]: pgmap v5738: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:30.854 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:39:30 smithi083 ceph-mon[77242]: pgmap v5738: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:31.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:39:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:39:30.852Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:39:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:39:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:39:31] "GET /metrics HTTP/1.1" 200 225216 "" "Prometheus/2.33.4" 2022-09-29T08:39:32.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:39:32 smithi153 ceph-mon[70044]: pgmap v5739: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:39:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:39:32.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:39:32.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:39:32 smithi083 ceph-mon[79930]: pgmap v5739: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:32.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:39:32 smithi083 ceph-mon[77242]: pgmap v5739: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:39:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:39:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:39:34.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:39:34 smithi083 ceph-mon[79930]: pgmap v5740: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:34.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:39:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:39:34.242Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:39:34.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:39:34 smithi083 ceph-mon[77242]: pgmap v5740: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:34.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:39:34 smithi153 ceph-mon[70044]: pgmap v5740: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:35.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:39:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:39:34.933Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:39:36.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:39:36 smithi153 ceph-mon[70044]: pgmap v5741: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:36.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:39:36 smithi083 ceph-mon[79930]: pgmap v5741: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:36.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:39:36 smithi083 ceph-mon[77242]: pgmap v5741: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:38.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:39:38 smithi153 ceph-mon[70044]: pgmap v5742: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:38.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:39:38 smithi083 ceph-mon[79930]: pgmap v5742: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:38.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:39:38 smithi083 ceph-mon[77242]: pgmap v5742: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:39:40 smithi153 ceph-mon[70044]: pgmap v5743: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:39:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:39:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:39:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:39:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:39:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:39:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:39:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:39:40.855 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:39:40 smithi083 ceph-mon[79930]: pgmap v5743: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:40.855 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:39:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:39:40.855 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:39:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:39:40.856 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:39:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:39:40.856 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:39:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:39:40.856 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:39:40 smithi083 ceph-mon[77242]: pgmap v5743: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:40.857 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:39:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:39:40.857 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:39:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:39:40.857 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:39:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:39:40.857 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:39:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:39:41.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:39:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:39:40.853Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:39:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:39:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:39:41] "GET /metrics HTTP/1.1" 200 225216 "" "Prometheus/2.33.4" 2022-09-29T08:39:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:39:42 smithi153 ceph-mon[70044]: pgmap v5744: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:39:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:39:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:39:42.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:39:42 smithi083 ceph-mon[79930]: pgmap v5744: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:42.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:39:42 smithi083 ceph-mon[77242]: pgmap v5744: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:39:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:39:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:39:44.502 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:39:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:39:44.243Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:39:44.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:39:44 smithi153 ceph-mon[70044]: pgmap v5745: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:44.935 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:39:44 smithi083 ceph-mon[79930]: pgmap v5745: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:44.936 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:39:44 smithi083 ceph-mon[77242]: pgmap v5745: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:45.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:39:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:39:44.934Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:39:46.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:39:46 smithi153 ceph-mon[70044]: pgmap v5746: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:46.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:39:46 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:39:46.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:39:46 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:39:46.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:39:46 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:39:46.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:39:46 smithi083 ceph-mon[79930]: pgmap v5746: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:46.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:39:46 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:39:46.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:39:46 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:39:46.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:39:46 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:39:46.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:39:46 smithi083 ceph-mon[77242]: pgmap v5746: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:46.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:39:46 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:39:46.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:39:46 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:39:46.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:39:46 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:39:48.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:39:48 smithi153 ceph-mon[70044]: pgmap v5747: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:48.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:39:48 smithi083 ceph-mon[79930]: pgmap v5747: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:48.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:39:48 smithi083 ceph-mon[77242]: pgmap v5747: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:50.601 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:39:50.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:39:50 smithi153 ceph-mon[70044]: pgmap v5748: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:50.855 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:39:50 smithi083 ceph-mon[79930]: pgmap v5748: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:50.855 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:39:50 smithi083 ceph-mon[77242]: pgmap v5748: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:50.888 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:39:50.888 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 2m ago 3h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:39:50.888 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 10m ago 3h 63.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:39:50.889 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 2m ago 3h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:39:50.889 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 10m ago 3h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:39:50.889 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 10m ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:39:50.889 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 2m ago 3h 680M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:39:50.890 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 2m ago 3h 743M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:39:50.890 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 10m ago 3h 313M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:39:50.890 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 2m ago 3h 436M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:39:50.890 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 2m ago 3h 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:39:50.891 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 10m ago 3h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:39:50.891 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 2m ago 3h 538M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:39:50.891 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 2m ago 3h 529M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:39:50.891 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 2m ago 3h 417M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:39:50.892 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 2m ago 3h 515M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:39:50.892 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 10m ago 3h 435M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:39:50.892 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 10m ago 3h 537M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:39:50.892 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 10m ago 3h 478M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:39:50.892 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 10m ago 3h 562M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:39:50.893 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 10m ago 3h 153M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:39:51.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:39:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:39:50.854Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:39:51.247 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:39:51.247 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:39:51.247 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:39:51.247 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:39:51.247 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:39:51.248 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:39:51.248 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:39:51.248 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:39:51.248 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:39:51.249 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:39:51.249 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:39:51.249 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:39:51.249 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:39:51.249 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:39:51.250 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:39:51.250 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:39:51.250 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:39:51.250 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:39:51.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:39:51 smithi153 ceph-mon[70044]: from='client.50752 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:39:51.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:39:51 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:39:51.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:39:51 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:39:51.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:39:51 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1492284157' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:39:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:39:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:39:51] "GET /metrics HTTP/1.1" 200 225216 "" "Prometheus/2.33.4" 2022-09-29T08:39:51.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:39:51 smithi083 ceph-mon[77242]: from='client.50752 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:39:51.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:39:51 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:39:51.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:39:51 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:39:51.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:39:51 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1492284157' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:39:51.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:39:51 smithi083 ceph-mon[79930]: from='client.50752 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:39:51.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:39:51 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:39:51.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:39:51 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:39:51.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:39:51 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1492284157' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:39:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:39:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:39:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:39:52.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:39:52 smithi153 ceph-mon[70044]: from='client.50758 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:39:52.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:39:52 smithi153 ceph-mon[70044]: pgmap v5749: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:52.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:39:52 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:39:52.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:39:52 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:39:52.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:39:52 smithi083 ceph-mon[79930]: from='client.50758 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:39:52.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:39:52 smithi083 ceph-mon[79930]: pgmap v5749: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:52.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:39:52 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:39:52.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:39:52 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:39:52.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:39:52 smithi083 ceph-mon[77242]: from='client.50758 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:39:52.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:39:52 smithi083 ceph-mon[77242]: pgmap v5749: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:52.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:39:52 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:39:52.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:39:52 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:39:53.224 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:39:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:39:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:39:54.520 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:39:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:39:54.243Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:39:54.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:39:54 smithi153 ceph-mon[70044]: pgmap v5750: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:54.936 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:39:54 smithi083 ceph-mon[79930]: pgmap v5750: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:54.936 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:39:54 smithi083 ceph-mon[77242]: pgmap v5750: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:55.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:39:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:39:54.935Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:39:55.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:39:55 smithi083 ceph-mon[77242]: Detected new or changed devices on smithi153 2022-09-29T08:39:55.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:39:55 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:39:55.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:39:55 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:39:55.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:39:55 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:39:55.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:39:55 smithi083 ceph-mon[77242]: pgmap v5751: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:55.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:39:55 smithi083 ceph-mon[79930]: Detected new or changed devices on smithi153 2022-09-29T08:39:55.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:39:55 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:39:55.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:39:55 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:39:55.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:39:55 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:39:55.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:39:55 smithi083 ceph-mon[79930]: pgmap v5751: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:56.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:39:55 smithi153 ceph-mon[70044]: Detected new or changed devices on smithi153 2022-09-29T08:39:56.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:39:55 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:39:56.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:39:55 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:39:56.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:39:55 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:39:56.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:39:55 smithi153 ceph-mon[70044]: pgmap v5751: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:58.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:39:58 smithi083 ceph-mon[77242]: pgmap v5752: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:58.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:39:58 smithi083 ceph-mon[79930]: pgmap v5752: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:39:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:39:58 smithi153 ceph-mon[70044]: pgmap v5752: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:00.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:40:00 smithi083 ceph-mon[79930]: pgmap v5753: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:00.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:40:00 smithi083 ceph-mon[79930]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T08:40:00.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:40:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[77219]: 2022-09-29T08:39:59.999+0000 7f160f665700 -1 log_channel(cluster) log [ERR] : overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T08:40:00.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:40:00 smithi083 ceph-mon[77242]: pgmap v5753: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:00.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:40:00 smithi083 ceph-mon[77242]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T08:40:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:40:00 smithi153 ceph-mon[70044]: pgmap v5753: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:00.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:40:00 smithi153 ceph-mon[70044]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T08:40:01.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:40:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:40:00.856Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:40:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:40:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:40:01] "GET /metrics HTTP/1.1" 200 225216 "" "Prometheus/2.33.4" 2022-09-29T08:40:02.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:40:02 smithi083 ceph-mon[77242]: pgmap v5754: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:02.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:40:02 smithi083 ceph-mon[79930]: pgmap v5754: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:02.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:40:02 smithi153 ceph-mon[70044]: pgmap v5754: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:02.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:40:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:40:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:40:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:40:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:40:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:40:04.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:40:04 smithi083 ceph-mon[79930]: pgmap v5755: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:04.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:40:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:40:04.244Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:40:04.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:40:04 smithi083 ceph-mon[77242]: pgmap v5755: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:40:04 smithi153 ceph-mon[70044]: pgmap v5755: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:05.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:40:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:40:04.936Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:40:06.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:40:06 smithi083 ceph-mon[79930]: pgmap v5756: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:06.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:40:06 smithi083 ceph-mon[77242]: pgmap v5756: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:06.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:40:06 smithi153 ceph-mon[70044]: pgmap v5756: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:08.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:40:08 smithi083 ceph-mon[79930]: pgmap v5757: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:08.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:40:08 smithi083 ceph-mon[77242]: pgmap v5757: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:40:08 smithi153 ceph-mon[70044]: pgmap v5757: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:10.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:40:10 smithi083 ceph-mon[77242]: pgmap v5758: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:10.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:40:10 smithi083 ceph-mon[79930]: pgmap v5758: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:40:10 smithi153 ceph-mon[70044]: pgmap v5758: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:11.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:40:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:40:10.855Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:40:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:40:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:40:11] "GET /metrics HTTP/1.1" 200 225217 "" "Prometheus/2.33.4" 2022-09-29T08:40:12.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:40:12 smithi083 ceph-mon[77242]: pgmap v5759: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:12.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:40:12 smithi083 ceph-mon[79930]: pgmap v5759: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:40:12 smithi153 ceph-mon[70044]: pgmap v5759: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:12.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:40:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:40:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:40:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:40:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:40:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:40:14.206 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:40:14 smithi083 ceph-mon[79930]: pgmap v5760: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:14.206 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:40:14 smithi083 ceph-mon[77242]: pgmap v5760: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:14.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:40:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:40:14.244Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:40:14.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:40:14 smithi153 ceph-mon[70044]: pgmap v5760: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:15.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:40:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:40:14.936Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:40:16.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:40:16 smithi083 ceph-mon[77242]: pgmap v5761: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:16.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:40:16 smithi083 ceph-mon[79930]: pgmap v5761: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:40:16 smithi153 ceph-mon[70044]: pgmap v5761: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:18.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:40:18 smithi083 ceph-mon[79930]: pgmap v5762: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:18.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:40:18 smithi083 ceph-mon[77242]: pgmap v5762: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:40:18 smithi153 ceph-mon[70044]: pgmap v5762: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:40:20 smithi153 ceph-mon[70044]: pgmap v5763: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:20.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:40:20 smithi083 ceph-mon[79930]: pgmap v5763: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:20.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:40:20 smithi083 ceph-mon[77242]: pgmap v5763: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:21.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:40:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:40:20.856Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:40:21.573 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:40:21.862 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:40:21.862 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 2m ago 3h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:40:21.862 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 30s ago 3h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:40:21.863 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 2m ago 3h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:40:21.863 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 30s ago 3h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:40:21.863 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 30s ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:40:21.863 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 2m ago 3h 680M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:40:21.864 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 2m ago 3h 743M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:40:21.864 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 30s ago 3h 329M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:40:21.864 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 2m ago 3h 436M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:40:21.864 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 2m ago 3h 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:40:21.865 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 30s ago 3h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:40:21.865 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 2m ago 3h 538M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:40:21.865 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 2m ago 3h 529M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:40:21.865 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 2m ago 3h 417M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:40:21.865 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 2m ago 3h 515M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:40:21.866 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 30s ago 3h 457M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:40:21.866 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 30s ago 3h 565M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:40:21.866 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 30s ago 3h 503M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:40:21.866 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 30s ago 3h 591M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:40:21.867 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 30s ago 3h 163M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:40:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:40:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:40:21] "GET /metrics HTTP/1.1" 200 225217 "" "Prometheus/2.33.4" 2022-09-29T08:40:22.220 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:40:22.220 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:40:22.221 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:40:22.221 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:40:22.221 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:40:22.221 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:40:22.222 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:40:22.222 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:40:22.222 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:40:22.222 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:40:22.222 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:40:22.223 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:40:22.223 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:40:22.223 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:40:22.223 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:40:22.224 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:40:22.224 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:40:22.224 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:40:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:40:22 smithi153 ceph-mon[70044]: pgmap v5764: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:22.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:40:22 smithi153 ceph-mon[70044]: from='client.40839 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:40:22.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:40:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:40:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:40:22.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:40:22 smithi083 ceph-mon[77242]: pgmap v5764: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:22.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:40:22 smithi083 ceph-mon[77242]: from='client.40839 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:40:22.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:40:22 smithi083 ceph-mon[79930]: pgmap v5764: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:22.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:40:22 smithi083 ceph-mon[79930]: from='client.40839 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:40:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:40:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:40:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:40:23.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:40:23 smithi153 ceph-mon[70044]: from='client.40845 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:40:23.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:40:23 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2311184730' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:40:23.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:40:23 smithi083 ceph-mon[79930]: from='client.40845 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:40:23.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:40:23 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2311184730' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:40:23.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:40:23 smithi083 ceph-mon[77242]: from='client.40845 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:40:23.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:40:23 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2311184730' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:40:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:40:24 smithi153 ceph-mon[70044]: pgmap v5765: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:24.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:40:24 smithi083 ceph-mon[79930]: pgmap v5765: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:24.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:40:24 smithi083 ceph-mon[77242]: pgmap v5765: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:40:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:40:24.245Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:40:25.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:40:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:40:24.937Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:40:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:40:26 smithi153 ceph-mon[70044]: pgmap v5766: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:26.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:40:26 smithi083 ceph-mon[79930]: pgmap v5766: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:26.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:40:26 smithi083 ceph-mon[77242]: pgmap v5766: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:40:28 smithi153 ceph-mon[70044]: pgmap v5767: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:28.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:40:28 smithi083 ceph-mon[79930]: pgmap v5767: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:28.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:40:28 smithi083 ceph-mon[77242]: pgmap v5767: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:40:30 smithi153 ceph-mon[70044]: pgmap v5768: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:30.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:40:30 smithi083 ceph-mon[79930]: pgmap v5768: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:30.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:40:30 smithi083 ceph-mon[77242]: pgmap v5768: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:31.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:40:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:40:30.856Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:40:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:40:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:40:31] "GET /metrics HTTP/1.1" 200 225218 "" "Prometheus/2.33.4" 2022-09-29T08:40:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:40:32 smithi153 ceph-mon[70044]: pgmap v5769: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:32.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:40:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:40:32.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:40:32.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:40:32 smithi083 ceph-mon[77242]: pgmap v5769: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:32.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:40:32 smithi083 ceph-mon[79930]: pgmap v5769: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:40:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:40:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:40:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:40:34 smithi153 ceph-mon[70044]: pgmap v5770: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:34.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:40:34 smithi083 ceph-mon[77242]: pgmap v5770: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:34.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:40:34 smithi083 ceph-mon[79930]: pgmap v5770: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:34.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:40:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:40:34.246Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:40:35.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:40:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:40:34.937Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:40:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:40:36 smithi153 ceph-mon[70044]: pgmap v5771: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:36.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:40:36 smithi083 ceph-mon[79930]: pgmap v5771: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:36.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:40:36 smithi083 ceph-mon[77242]: pgmap v5771: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:40:38 smithi153 ceph-mon[70044]: pgmap v5772: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:38.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:40:38 smithi083 ceph-mon[79930]: pgmap v5772: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:38.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:40:38 smithi083 ceph-mon[77242]: pgmap v5772: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:40:40 smithi153 ceph-mon[70044]: pgmap v5773: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:40:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:40:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:40:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:40:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:40:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:40:40.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:40:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:40:40.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:40:40 smithi083 ceph-mon[79930]: pgmap v5773: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:40.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:40:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:40:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:40:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:40:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:40:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:40:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:40:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:40:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:40:40 smithi083 ceph-mon[77242]: pgmap v5773: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:40:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:40:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:40:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:40:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:40:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:40:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:40:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:40:41.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:40:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:40:40.856Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:40:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:40:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:40:41] "GET /metrics HTTP/1.1" 200 225215 "" "Prometheus/2.33.4" 2022-09-29T08:40:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:40:42 smithi153 ceph-mon[70044]: pgmap v5774: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:42.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:40:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:40:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:40:42.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:40:42 smithi083 ceph-mon[77242]: pgmap v5774: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:40:42 smithi083 ceph-mon[79930]: pgmap v5774: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:40:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:40:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:40:44.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:40:44 smithi153 ceph-mon[70044]: pgmap v5775: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:44.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:40:44 smithi083 ceph-mon[79930]: pgmap v5775: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:40:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:40:44.247Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:40:44.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:40:44 smithi083 ceph-mon[77242]: pgmap v5775: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:45.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:40:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:40:44.937Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:40:46.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:40:46 smithi153 ceph-mon[70044]: pgmap v5776: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:46.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:40:46 smithi083 ceph-mon[77242]: pgmap v5776: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:46.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:40:46 smithi083 ceph-mon[79930]: pgmap v5776: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:48.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:40:48 smithi153 ceph-mon[70044]: pgmap v5777: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:48.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:40:48 smithi083 ceph-mon[77242]: pgmap v5777: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:48.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:40:48 smithi083 ceph-mon[79930]: pgmap v5777: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:40:50 smithi153 ceph-mon[70044]: pgmap v5778: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:50.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:40:50 smithi083 ceph-mon[79930]: pgmap v5778: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:50.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:40:50 smithi083 ceph-mon[77242]: pgmap v5778: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:51.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:40:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:40:50.857Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:40:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:40:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:40:51] "GET /metrics HTTP/1.1" 200 225215 "" "Prometheus/2.33.4" 2022-09-29T08:40:52.550 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:40:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:40:52 smithi153 ceph-mon[70044]: pgmap v5779: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:52.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:40:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:40:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:40:52.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:40:52 smithi083 ceph-mon[79930]: pgmap v5779: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:52.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:40:52 smithi083 ceph-mon[77242]: pgmap v5779: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:52.840 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:40:52.840 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 3m ago 3h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:40:52.840 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 61s ago 3h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:40:52.841 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 3m ago 3h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:40:52.841 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 61s ago 3h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:40:52.841 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 61s ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:40:52.841 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 3m ago 3h 680M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:40:52.842 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 3m ago 3h 743M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:40:52.842 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 61s ago 3h 329M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:40:52.842 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 3m ago 3h 436M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:40:52.842 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 3m ago 3h 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:40:52.843 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 61s ago 3h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:40:52.843 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 3m ago 3h 538M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:40:52.843 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 3m ago 3h 529M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:40:52.843 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 3m ago 3h 417M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:40:52.843 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 3m ago 3h 515M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:40:52.844 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 61s ago 3h 457M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:40:52.844 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 61s ago 3h 565M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:40:52.844 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 61s ago 3h 503M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:40:52.844 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 61s ago 3h 591M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:40:52.845 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 61s ago 3h 163M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:40:53.196 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:40:53.196 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:40:53.196 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:40:53.197 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:40:53.197 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:40:53.197 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:40:53.197 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:40:53.197 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:40:53.198 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:40:53.198 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:40:53.198 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:40:53.198 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:40:53.199 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:40:53.199 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:40:53.199 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:40:53.199 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:40:53.200 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:40:53.200 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:40:53.279 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:40:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:40:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:40:53.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:40:53 smithi153 ceph-mon[70044]: from='client.40857 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:40:53.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:40:53 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3982881423' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:40:53.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:40:53 smithi083 ceph-mon[79930]: from='client.40857 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:40:53.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:40:53 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3982881423' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:40:53.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:40:53 smithi083 ceph-mon[77242]: from='client.40857 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:40:53.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:40:53 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3982881423' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:40:54.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:40:54 smithi153 ceph-mon[70044]: from='client.50794 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:40:54.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:40:54 smithi153 ceph-mon[70044]: pgmap v5780: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:54.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:40:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:40:54.247Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:40:54.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:40:54 smithi083 ceph-mon[77242]: from='client.50794 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:40:54.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:40:54 smithi083 ceph-mon[77242]: pgmap v5780: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:54.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:40:54 smithi083 ceph-mon[79930]: from='client.50794 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:40:54.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:40:54 smithi083 ceph-mon[79930]: pgmap v5780: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:55.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:40:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:40:54.937Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:40:55.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:40:55 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:40:55.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:40:55 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:40:55.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:40:55 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:40:55.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:40:55 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:40:55.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:40:55 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:40:55.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:40:55 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:40:55.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:40:55 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:40:55.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:40:55 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:40:55.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:40:55 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:40:56.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:40:56 smithi083 ceph-mon[79930]: pgmap v5781: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:56.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:40:56 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:40:56.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:40:56 smithi083 ceph-mon[77242]: pgmap v5781: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:56.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:40:56 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:40:56.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:40:56 smithi153 ceph-mon[70044]: pgmap v5781: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:56.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:40:56 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:40:58.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:40:58 smithi083 ceph-mon[77242]: pgmap v5782: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:58.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:40:58 smithi083 ceph-mon[79930]: pgmap v5782: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:40:58.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:40:58 smithi153 ceph-mon[70044]: pgmap v5782: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:00.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:41:00 smithi083 ceph-mon[79930]: pgmap v5783: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:00.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:41:00 smithi083 ceph-mon[77242]: pgmap v5783: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:00.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:41:00 smithi153 ceph-mon[70044]: pgmap v5783: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:01.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:41:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:41:00.858Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:41:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:41:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:41:01] "GET /metrics HTTP/1.1" 200 225217 "" "Prometheus/2.33.4" 2022-09-29T08:41:02.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:41:02 smithi083 ceph-mon[79930]: pgmap v5784: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:02.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:41:02 smithi083 ceph-mon[77242]: pgmap v5784: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:02.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:41:02 smithi153 ceph-mon[70044]: pgmap v5784: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:41:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:41:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:41:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:41:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:41:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:41:04.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:41:04 smithi083 ceph-mon[79930]: pgmap v5785: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:04.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:41:04 smithi083 ceph-mon[77242]: pgmap v5785: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:04.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:41:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:41:04.249Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:41:04.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:41:04 smithi153 ceph-mon[70044]: pgmap v5785: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:05.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:41:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:41:04.939Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:41:06.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:41:06 smithi083 ceph-mon[77242]: pgmap v5786: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:06.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:41:06 smithi083 ceph-mon[79930]: pgmap v5786: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:06.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:41:06 smithi153 ceph-mon[70044]: pgmap v5786: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:08.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:41:08 smithi083 ceph-mon[79930]: pgmap v5787: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:08.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:41:08 smithi083 ceph-mon[77242]: pgmap v5787: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:08.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:41:08 smithi153 ceph-mon[70044]: pgmap v5787: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:10.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:41:10 smithi083 ceph-mon[79930]: pgmap v5788: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:10.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:41:10 smithi083 ceph-mon[77242]: pgmap v5788: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:10.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:41:10 smithi153 ceph-mon[70044]: pgmap v5788: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:11.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:41:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:41:10.858Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:41:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:41:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:41:11] "GET /metrics HTTP/1.1" 200 225216 "" "Prometheus/2.33.4" 2022-09-29T08:41:12.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:41:12 smithi083 ceph-mon[77242]: pgmap v5789: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:12.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:41:12 smithi083 ceph-mon[79930]: pgmap v5789: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:12.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:41:12 smithi153 ceph-mon[70044]: pgmap v5789: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:41:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:41:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:41:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:41:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:41:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:41:14.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:41:14 smithi083 ceph-mon[79930]: pgmap v5790: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:14.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:41:14 smithi083 ceph-mon[77242]: pgmap v5790: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:14.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:41:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:41:14.249Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:41:14.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:41:14 smithi153 ceph-mon[70044]: pgmap v5790: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:15.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:41:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:41:14.939Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:41:16.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:41:16 smithi083 ceph-mon[79930]: pgmap v5791: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:16.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:41:16 smithi083 ceph-mon[77242]: pgmap v5791: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:16.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:41:16 smithi153 ceph-mon[70044]: pgmap v5791: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:18.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:41:18 smithi083 ceph-mon[79930]: pgmap v5792: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:18.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:41:18 smithi083 ceph-mon[77242]: pgmap v5792: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:18.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:41:18 smithi153 ceph-mon[70044]: pgmap v5792: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:20.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:41:20 smithi083 ceph-mon[79930]: pgmap v5793: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:20.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:41:20 smithi083 ceph-mon[77242]: pgmap v5793: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:20.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:41:20 smithi153 ceph-mon[70044]: pgmap v5793: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:21.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:41:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:41:20.858Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:41:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:41:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:41:21] "GET /metrics HTTP/1.1" 200 225216 "" "Prometheus/2.33.4" 2022-09-29T08:41:22.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:41:22 smithi083 ceph-mon[79930]: pgmap v5794: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:22.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:41:22 smithi083 ceph-mon[77242]: pgmap v5794: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:22.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:41:22 smithi153 ceph-mon[70044]: pgmap v5794: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:41:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:41:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:41:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:41:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:41:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:41:23.535 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:41:23.836 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:41:23.836 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 3m ago 3h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:41:23.836 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 92s ago 3h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:41:23.837 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 3m ago 3h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:41:23.837 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 92s ago 3h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:41:23.837 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 92s ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:41:23.837 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 3m ago 3h 680M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:41:23.838 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 3m ago 3h 743M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:41:23.838 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 92s ago 3h 329M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:41:23.838 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 3m ago 3h 436M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:41:23.838 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 3m ago 3h 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:41:23.838 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 92s ago 3h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:41:23.839 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 3m ago 3h 538M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:41:23.839 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 3m ago 3h 529M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:41:23.839 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 3m ago 3h 417M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:41:23.839 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 3m ago 3h 515M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:41:23.840 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 92s ago 3h 457M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:41:23.840 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 92s ago 3h 565M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:41:23.840 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 92s ago 3h 503M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:41:23.840 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 92s ago 3h 591M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:41:23.840 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 92s ago 3h 163M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:41:24.194 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:41:24.194 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:41:24.194 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:41:24.195 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:41:24.195 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:41:24.195 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:41:24.195 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:41:24.196 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:41:24.196 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:41:24.196 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:41:24.196 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:41:24.196 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:41:24.197 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:41:24.197 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:41:24.197 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:41:24.197 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:41:24.197 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:41:24.198 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:41:24.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:41:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:41:24.248Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:41:24.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:41:24 smithi083 ceph-mon[77242]: pgmap v5795: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:24.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:41:24 smithi083 ceph-mon[77242]: from='client.40875 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:41:24.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:41:24 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3936648915' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:41:24.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:41:24 smithi083 ceph-mon[79930]: pgmap v5795: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:24.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:41:24 smithi083 ceph-mon[79930]: from='client.40875 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:41:24.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:41:24 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3936648915' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:41:24.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:41:24 smithi153 ceph-mon[70044]: pgmap v5795: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:24.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:41:24 smithi153 ceph-mon[70044]: from='client.40875 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:41:24.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:41:24 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3936648915' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:41:25.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:41:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:41:24.939Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:41:25.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:41:25 smithi083 ceph-mon[77242]: from='client.40881 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:41:25.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:41:25 smithi083 ceph-mon[79930]: from='client.40881 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:41:25.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:41:25 smithi153 ceph-mon[70044]: from='client.40881 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:41:26.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:41:26 smithi083 ceph-mon[79930]: pgmap v5796: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:26.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:41:26 smithi083 ceph-mon[77242]: pgmap v5796: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:26.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:41:26 smithi153 ceph-mon[70044]: pgmap v5796: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:28.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:41:28 smithi083 ceph-mon[79930]: pgmap v5797: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:28.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:41:28 smithi083 ceph-mon[77242]: pgmap v5797: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:28.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:41:28 smithi153 ceph-mon[70044]: pgmap v5797: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:30.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:41:30 smithi083 ceph-mon[79930]: pgmap v5798: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:30.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:41:30 smithi083 ceph-mon[77242]: pgmap v5798: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:30.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:41:30 smithi153 ceph-mon[70044]: pgmap v5798: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:31.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:41:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:41:30.860Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:41:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:41:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:41:31] "GET /metrics HTTP/1.1" 200 225215 "" "Prometheus/2.33.4" 2022-09-29T08:41:32.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:41:32 smithi083 ceph-mon[79930]: pgmap v5799: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:32.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:41:32 smithi083 ceph-mon[77242]: pgmap v5799: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:32.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:41:32 smithi153 ceph-mon[70044]: pgmap v5799: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:41:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:41:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:41:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:41:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:41:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:41:34.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:41:34 smithi083 ceph-mon[79930]: pgmap v5800: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:34.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:41:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:41:34.250Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:41:34.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:41:34 smithi083 ceph-mon[77242]: pgmap v5800: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:34.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:41:34 smithi153 ceph-mon[70044]: pgmap v5800: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:35.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:41:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:41:34.939Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:41:36.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:41:36 smithi083 ceph-mon[79930]: pgmap v5801: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:36.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:41:36 smithi083 ceph-mon[77242]: pgmap v5801: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:36.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:41:36 smithi153 ceph-mon[70044]: pgmap v5801: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:38.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:41:38 smithi083 ceph-mon[79930]: pgmap v5802: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:38.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:41:38 smithi083 ceph-mon[77242]: pgmap v5802: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:38.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:41:38 smithi153 ceph-mon[70044]: pgmap v5802: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:40.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:41:40 smithi083 ceph-mon[79930]: pgmap v5803: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:41:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:41:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:41:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:41:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:41:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:41:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:41:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:41:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:41:40 smithi083 ceph-mon[77242]: pgmap v5803: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:41:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:41:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:41:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:41:40.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:41:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:41:40.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:41:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:41:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:41:40 smithi153 ceph-mon[70044]: pgmap v5803: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:41:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:41:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:41:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:41:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:41:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:41:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:41:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:41:41.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:41:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:41:40.860Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:41:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:41:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:41:41] "GET /metrics HTTP/1.1" 200 225216 "" "Prometheus/2.33.4" 2022-09-29T08:41:42.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:41:42 smithi083 ceph-mon[77242]: pgmap v5804: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:41:42 smithi083 ceph-mon[79930]: pgmap v5804: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:41:42 smithi153 ceph-mon[70044]: pgmap v5804: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:41:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:41:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:41:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:41:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:41:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:41:44.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:41:44 smithi083 ceph-mon[77242]: pgmap v5805: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:44.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:41:44 smithi083 ceph-mon[79930]: pgmap v5805: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:41:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:41:44.251Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:41:44.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:41:44 smithi153 ceph-mon[70044]: pgmap v5805: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:45.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:41:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:41:44.940Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:41:46.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:41:46 smithi083 ceph-mon[77242]: pgmap v5806: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:46.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:41:46 smithi083 ceph-mon[79930]: pgmap v5806: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:46.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:41:46 smithi153 ceph-mon[70044]: pgmap v5806: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:48.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:41:48 smithi083 ceph-mon[77242]: pgmap v5807: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:48.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:41:48 smithi083 ceph-mon[79930]: pgmap v5807: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:48.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:41:48 smithi153 ceph-mon[70044]: pgmap v5807: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:50.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:41:50 smithi083 ceph-mon[79930]: pgmap v5808: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:50.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:41:50 smithi083 ceph-mon[77242]: pgmap v5808: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:50.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:41:50 smithi153 ceph-mon[70044]: pgmap v5808: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:51.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:41:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:41:50.861Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:41:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:41:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:41:51] "GET /metrics HTTP/1.1" 200 225216 "" "Prometheus/2.33.4" 2022-09-29T08:41:52.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:41:52 smithi083 ceph-mon[79930]: pgmap v5809: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:52.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:41:52 smithi083 ceph-mon[77242]: pgmap v5809: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:52.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:41:52 smithi153 ceph-mon[70044]: pgmap v5809: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:41:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:41:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:41:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:41:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:41:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:41:54.515 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:41:54.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:41:54 smithi083 ceph-mon[77242]: pgmap v5810: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:54.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:41:54 smithi083 ceph-mon[79930]: pgmap v5810: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:41:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:41:54.250Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:41:54.803 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:41:54.803 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 4m ago 3h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:41:54.804 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 2m ago 3h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:41:54.804 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 4m ago 3h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:41:54.804 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 2m ago 3h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:41:54.804 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 2m ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:41:54.805 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 4m ago 3h 680M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:41:54.805 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 4m ago 3h 743M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:41:54.805 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 2m ago 3h 329M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:41:54.805 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 4m ago 3h 436M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:41:54.806 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 4m ago 3h 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:41:54.806 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 2m ago 3h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:41:54.806 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 4m ago 3h 538M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:41:54.806 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 4m ago 3h 529M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:41:54.807 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 4m ago 3h 417M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:41:54.807 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 4m ago 3h 515M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:41:54.807 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 2m ago 3h 457M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:41:54.807 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 2m ago 3h 565M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:41:54.807 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 2m ago 3h 503M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:41:54.808 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 2m ago 3h 591M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:41:54.808 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 2m ago 3h 163M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:41:54.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:41:54 smithi153 ceph-mon[70044]: pgmap v5810: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:55.162 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:41:55.163 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:41:55.163 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:41:55.163 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:41:55.163 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:41:55.164 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:41:55.164 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:41:55.164 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:41:55.164 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:41:55.165 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:41:55.165 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:41:55.165 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:41:55.165 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:41:55.165 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:41:55.166 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:41:55.166 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:41:55.166 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:41:55.167 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:41:55.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:41:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:41:54.940Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:41:55.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:41:55 smithi083 ceph-mon[79930]: from='client.50824 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:41:55.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:41:55 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2838226480' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:41:55.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:41:55 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:41:55.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:41:55 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:41:55.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:41:55 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:41:55.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:41:55 smithi083 ceph-mon[77242]: from='client.50824 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:41:55.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:41:55 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2838226480' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:41:55.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:41:55 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:41:55.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:41:55 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:41:55.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:41:55 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:41:55.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:41:55 smithi153 ceph-mon[70044]: from='client.50824 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:41:55.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:41:55 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2838226480' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:41:55.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:41:55 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:41:55.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:41:55 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:41:55.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:41:55 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:41:56.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:41:56 smithi083 ceph-mon[79930]: from='client.50830 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:41:56.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:41:56 smithi083 ceph-mon[79930]: pgmap v5811: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:56.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:41:56 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:41:56.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:41:56 smithi083 ceph-mon[77242]: from='client.50830 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:41:56.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:41:56 smithi083 ceph-mon[77242]: pgmap v5811: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:56.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:41:56 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:41:56.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:41:56 smithi153 ceph-mon[70044]: from='client.50830 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:41:56.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:41:56 smithi153 ceph-mon[70044]: pgmap v5811: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:56.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:41:56 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:41:58.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:41:58 smithi153 ceph-mon[70044]: pgmap v5812: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:58.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:41:58 smithi083 ceph-mon[79930]: pgmap v5812: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:41:58.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:41:58 smithi083 ceph-mon[77242]: pgmap v5812: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:00.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:42:00 smithi153 ceph-mon[70044]: pgmap v5813: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:00.863 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:42:00 smithi083 ceph-mon[79930]: pgmap v5813: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:00.863 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:42:00 smithi083 ceph-mon[77242]: pgmap v5813: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:01.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:42:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:42:00.862Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:42:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:42:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:42:01] "GET /metrics HTTP/1.1" 200 225212 "" "Prometheus/2.33.4" 2022-09-29T08:42:02.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:42:02 smithi153 ceph-mon[70044]: pgmap v5814: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:42:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:42:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:42:02.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:42:02 smithi083 ceph-mon[77242]: pgmap v5814: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:02.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:42:02 smithi083 ceph-mon[79930]: pgmap v5814: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:42:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:42:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:42:04.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:42:04 smithi083 ceph-mon[79930]: pgmap v5815: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:04.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:42:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:42:04.252Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:42:04.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:42:04 smithi083 ceph-mon[77242]: pgmap v5815: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:04.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:42:04 smithi153 ceph-mon[70044]: pgmap v5815: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:05.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:42:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:42:04.941Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:42:06.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:42:06 smithi153 ceph-mon[70044]: pgmap v5816: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:06.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:42:06 smithi083 ceph-mon[77242]: pgmap v5816: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:06.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:42:06 smithi083 ceph-mon[79930]: pgmap v5816: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:08.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:42:08 smithi153 ceph-mon[70044]: pgmap v5817: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:08.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:42:08 smithi083 ceph-mon[79930]: pgmap v5817: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:08.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:42:08 smithi083 ceph-mon[77242]: pgmap v5817: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:10.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:42:10 smithi153 ceph-mon[70044]: pgmap v5818: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:10.863 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:42:10 smithi083 ceph-mon[79930]: pgmap v5818: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:10.863 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:42:10 smithi083 ceph-mon[77242]: pgmap v5818: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:11.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:42:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:42:10.862Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:42:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:42:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:42:11] "GET /metrics HTTP/1.1" 200 225217 "" "Prometheus/2.33.4" 2022-09-29T08:42:12.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:42:12 smithi153 ceph-mon[70044]: pgmap v5819: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:42:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:42:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:42:12.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:42:12 smithi083 ceph-mon[79930]: pgmap v5819: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:12.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:42:12 smithi083 ceph-mon[77242]: pgmap v5819: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:42:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:42:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:42:14.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:42:14 smithi083 ceph-mon[79930]: pgmap v5820: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:14.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:42:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:42:14.252Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:42:14.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:42:14 smithi083 ceph-mon[77242]: pgmap v5820: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:14.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:42:14 smithi153 ceph-mon[70044]: pgmap v5820: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:15.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:42:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:42:14.941Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:42:16.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:42:16 smithi153 ceph-mon[70044]: pgmap v5821: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:16.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:42:16 smithi083 ceph-mon[79930]: pgmap v5821: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:16.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:42:16 smithi083 ceph-mon[77242]: pgmap v5821: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:18.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:42:18 smithi153 ceph-mon[70044]: pgmap v5822: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:18.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:42:18 smithi083 ceph-mon[79930]: pgmap v5822: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:18.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:42:18 smithi083 ceph-mon[77242]: pgmap v5822: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:20.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:42:20 smithi153 ceph-mon[70044]: pgmap v5823: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:20.864 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:42:20 smithi083 ceph-mon[79930]: pgmap v5823: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:20.864 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:42:20 smithi083 ceph-mon[77242]: pgmap v5823: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:21.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:42:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:42:20.863Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:42:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:42:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:42:21] "GET /metrics HTTP/1.1" 200 225217 "" "Prometheus/2.33.4" 2022-09-29T08:42:22.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:42:22 smithi153 ceph-mon[70044]: pgmap v5824: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:42:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:42:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:42:22.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:42:22 smithi083 ceph-mon[77242]: pgmap v5824: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:22.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:42:22 smithi083 ceph-mon[79930]: pgmap v5824: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:42:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:42:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:42:24.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:42:24 smithi083 ceph-mon[79930]: pgmap v5825: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:42:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:42:24.252Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:42:24.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:42:24 smithi083 ceph-mon[77242]: pgmap v5825: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:24.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:42:24 smithi153 ceph-mon[70044]: pgmap v5825: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:25.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:42:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:42:24.943Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:42:25.490 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:42:25.781 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:42:25.781 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 4m ago 3h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:42:25.781 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 2m ago 3h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:42:25.782 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 4m ago 3h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:42:25.782 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 2m ago 3h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:42:25.782 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 2m ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:42:25.782 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 4m ago 3h 680M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:42:25.783 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 4m ago 3h 743M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:42:25.783 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 2m ago 3h 329M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:42:25.783 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 4m ago 3h 436M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:42:25.783 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 4m ago 3h 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:42:25.783 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 2m ago 3h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:42:25.784 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 4m ago 3h 538M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:42:25.784 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 4m ago 3h 529M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:42:25.784 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 4m ago 3h 417M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:42:25.784 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 4m ago 3h 515M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:42:25.784 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 2m ago 3h 457M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:42:25.785 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 2m ago 3h 565M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:42:25.785 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 2m ago 3h 503M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:42:25.785 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 2m ago 3h 591M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:42:25.785 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 2m ago 3h 163M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:42:26.136 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:42:26.136 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:42:26.137 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:42:26.137 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:42:26.137 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:42:26.137 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:42:26.137 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:42:26.138 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:42:26.138 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:42:26.138 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:42:26.138 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:42:26.139 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:42:26.139 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:42:26.139 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:42:26.139 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:42:26.139 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:42:26.140 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:42:26.140 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:42:26.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:42:26 smithi153 ceph-mon[70044]: pgmap v5826: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:26.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:42:26 smithi153 ceph-mon[70044]: from='client.50842 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:42:26.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:42:26 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1741701895' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:42:26.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:42:26 smithi083 ceph-mon[79930]: pgmap v5826: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:26.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:42:26 smithi083 ceph-mon[79930]: from='client.50842 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:42:26.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:42:26 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1741701895' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:42:26.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:42:26 smithi083 ceph-mon[77242]: pgmap v5826: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:26.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:42:26 smithi083 ceph-mon[77242]: from='client.50842 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:42:26.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:42:26 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1741701895' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:42:27.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:42:27 smithi153 ceph-mon[70044]: from='client.40917 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:42:27.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:42:27 smithi083 ceph-mon[79930]: from='client.40917 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:42:27.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:42:27 smithi083 ceph-mon[77242]: from='client.40917 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:42:28.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:42:28 smithi153 ceph-mon[70044]: pgmap v5827: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:28.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:42:28 smithi083 ceph-mon[79930]: pgmap v5827: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:28.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:42:28 smithi083 ceph-mon[77242]: pgmap v5827: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:30.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:42:30 smithi153 ceph-mon[70044]: pgmap v5828: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:30.865 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:42:30 smithi083 ceph-mon[79930]: pgmap v5828: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:30.865 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:42:30 smithi083 ceph-mon[77242]: pgmap v5828: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:31.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:42:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:42:30.864Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:42:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:42:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:42:31] "GET /metrics HTTP/1.1" 200 225215 "" "Prometheus/2.33.4" 2022-09-29T08:42:32.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:42:32 smithi153 ceph-mon[70044]: pgmap v5829: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:42:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:42:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:42:32.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:42:32 smithi083 ceph-mon[79930]: pgmap v5829: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:32.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:42:32 smithi083 ceph-mon[77242]: pgmap v5829: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:42:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:42:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:42:34.516 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:42:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:42:34.253Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:42:34.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:42:34 smithi153 ceph-mon[70044]: pgmap v5830: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:34.945 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:42:34 smithi083 ceph-mon[79930]: pgmap v5830: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:34.945 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:42:34 smithi083 ceph-mon[77242]: pgmap v5830: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:35.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:42:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:42:34.944Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:42:36.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:42:36 smithi153 ceph-mon[70044]: pgmap v5831: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:36.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:42:36 smithi083 ceph-mon[79930]: pgmap v5831: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:36.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:42:36 smithi083 ceph-mon[77242]: pgmap v5831: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:38.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:42:38 smithi153 ceph-mon[70044]: pgmap v5832: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:38.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:42:38 smithi083 ceph-mon[79930]: pgmap v5832: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:38.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:42:38 smithi083 ceph-mon[77242]: pgmap v5832: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:42:40 smithi153 ceph-mon[70044]: pgmap v5833: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:42:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:42:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:42:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:42:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:42:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:42:40.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:42:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:42:40.865 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:42:40 smithi083 ceph-mon[77242]: pgmap v5833: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:40.866 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:42:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:42:40.866 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:42:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:42:40.866 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:42:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:42:40.866 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:42:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:42:40.867 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:42:40 smithi083 ceph-mon[79930]: pgmap v5833: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:40.867 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:42:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:42:40.867 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:42:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:42:40.867 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:42:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:42:40.867 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:42:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:42:41.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:42:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:42:40.864Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:42:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:42:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:42:41] "GET /metrics HTTP/1.1" 200 225214 "" "Prometheus/2.33.4" 2022-09-29T08:42:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:42:42 smithi153 ceph-mon[70044]: pgmap v5834: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:42:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:42:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:42:42.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:42:42 smithi083 ceph-mon[79930]: pgmap v5834: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:42.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:42:42 smithi083 ceph-mon[77242]: pgmap v5834: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:42:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:42:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:42:44.530 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:42:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:42:44.254Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:42:44.531 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:42:44 smithi083 ceph-mon[77242]: pgmap v5835: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:44.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:42:44 smithi153 ceph-mon[70044]: pgmap v5835: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:44.945 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:42:44 smithi083 ceph-mon[79930]: pgmap v5835: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:45.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:42:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:42:44.944Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:42:46.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:42:46 smithi153 ceph-mon[70044]: pgmap v5836: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:46.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:42:46 smithi083 ceph-mon[79930]: pgmap v5836: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:46.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:42:46 smithi083 ceph-mon[77242]: pgmap v5836: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:48.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:42:48 smithi153 ceph-mon[70044]: pgmap v5837: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:48.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:42:48 smithi083 ceph-mon[77242]: pgmap v5837: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:48.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:42:48 smithi083 ceph-mon[79930]: pgmap v5837: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:50.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:42:50 smithi153 ceph-mon[70044]: pgmap v5838: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:50.867 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:42:50 smithi083 ceph-mon[79930]: pgmap v5838: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:50.867 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:42:50 smithi083 ceph-mon[77242]: pgmap v5838: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:51.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:42:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:42:50.865Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:42:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:42:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:42:51] "GET /metrics HTTP/1.1" 200 225214 "" "Prometheus/2.33.4" 2022-09-29T08:42:52.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:42:52 smithi153 ceph-mon[70044]: pgmap v5839: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:42:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:42:52.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:42:52.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:42:52 smithi083 ceph-mon[79930]: pgmap v5839: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:52.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:42:52 smithi083 ceph-mon[77242]: pgmap v5839: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:42:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:42:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:42:54.548 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:42:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:42:54.254Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:42:54.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:42:54 smithi153 ceph-mon[70044]: pgmap v5840: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:54.946 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:42:54 smithi083 ceph-mon[79930]: pgmap v5840: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:54.946 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:42:54 smithi083 ceph-mon[77242]: pgmap v5840: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:55.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:42:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:42:54.944Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:42:56.466 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:42:56.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:42:56 smithi153 ceph-mon[70044]: pgmap v5841: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:56.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:42:56 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:42:56.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:42:56 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:42:56.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:42:56 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:42:56.814 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:42:56.814 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 5m ago 3h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:42:56.814 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 3m ago 3h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:42:56.815 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 5m ago 3h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:42:56.815 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 3m ago 3h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:42:56.815 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 3m ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:42:56.815 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 5m ago 3h 680M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:42:56.815 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 5m ago 3h 743M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:42:56.816 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 3m ago 3h 329M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:42:56.816 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 5m ago 3h 436M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:42:56.816 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 5m ago 3h 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:42:56.817 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 3m ago 3h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:42:56.817 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 5m ago 3h 538M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:42:56.817 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 5m ago 3h 529M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:42:56.817 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 5m ago 3h 417M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:42:56.817 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 5m ago 3h 515M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:42:56.818 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 3m ago 3h 457M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:42:56.818 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 3m ago 3h 565M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:42:56.818 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 3m ago 3h 503M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:42:56.818 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 3m ago 3h 591M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:42:56.819 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 3m ago 3h 163M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:42:56.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:42:56 smithi083 ceph-mon[77242]: pgmap v5841: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:56.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:42:56 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:42:56.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:42:56 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:42:56.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:42:56 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:42:56.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:42:56 smithi083 ceph-mon[79930]: pgmap v5841: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:56.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:42:56 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:42:56.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:42:56 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:42:56.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:42:56 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:42:57.169 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:42:57.170 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:42:57.170 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:42:57.170 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:42:57.171 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:42:57.171 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:42:57.171 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:42:57.171 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:42:57.171 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:42:57.172 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:42:57.172 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:42:57.172 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:42:57.172 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:42:57.172 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:42:57.173 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:42:57.173 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:42:57.173 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:42:57.173 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:42:57.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:42:57 smithi153 ceph-mon[70044]: from='client.50860 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:42:57.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:42:57 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:42:57.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:42:57 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2055558361' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:42:57.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:42:57 smithi083 ceph-mon[79930]: from='client.50860 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:42:57.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:42:57 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:42:57.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:42:57 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2055558361' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:42:57.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:42:57 smithi083 ceph-mon[77242]: from='client.50860 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:42:57.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:42:57 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:42:57.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:42:57 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2055558361' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:42:58.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:42:58 smithi153 ceph-mon[70044]: from='client.50866 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:42:58.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:42:58 smithi153 ceph-mon[70044]: pgmap v5842: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:58.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:42:58 smithi083 ceph-mon[79930]: from='client.50866 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:42:58.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:42:58 smithi083 ceph-mon[79930]: pgmap v5842: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:42:58.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:42:58 smithi083 ceph-mon[77242]: from='client.50866 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:42:58.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:42:58 smithi083 ceph-mon[77242]: pgmap v5842: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:00.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:43:00 smithi153 ceph-mon[70044]: pgmap v5843: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:00.868 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:43:00 smithi083 ceph-mon[79930]: pgmap v5843: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:00.868 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:43:00 smithi083 ceph-mon[77242]: pgmap v5843: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:01.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:43:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:43:00.866Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:43:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:43:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:43:01] "GET /metrics HTTP/1.1" 200 225217 "" "Prometheus/2.33.4" 2022-09-29T08:43:02.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:43:02 smithi153 ceph-mon[70044]: pgmap v5844: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:43:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:43:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:43:02.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:43:02 smithi083 ceph-mon[79930]: pgmap v5844: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:02.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:43:02 smithi083 ceph-mon[77242]: pgmap v5844: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:43:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:43:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:43:04.561 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:43:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:43:04.254Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:43:04.946 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:43:04 smithi083 ceph-mon[79930]: pgmap v5845: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:04.947 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:43:04 smithi083 ceph-mon[77242]: pgmap v5845: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:05.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:43:04 smithi153 ceph-mon[70044]: pgmap v5845: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:05.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:43:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:43:04.945Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:43:06.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:43:06 smithi083 ceph-mon[79930]: pgmap v5846: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:06.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:43:06 smithi083 ceph-mon[77242]: pgmap v5846: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:07.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:43:06 smithi153 ceph-mon[70044]: pgmap v5846: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:08.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:43:08 smithi083 ceph-mon[79930]: pgmap v5847: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:08.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:43:08 smithi083 ceph-mon[77242]: pgmap v5847: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:09.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:43:08 smithi153 ceph-mon[70044]: pgmap v5847: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:10.868 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:43:10 smithi083 ceph-mon[79930]: pgmap v5848: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:10.869 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:43:10 smithi083 ceph-mon[77242]: pgmap v5848: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:11.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:43:10 smithi153 ceph-mon[70044]: pgmap v5848: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:11.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:43:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:43:10.867Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:43:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:43:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:43:11] "GET /metrics HTTP/1.1" 200 225211 "" "Prometheus/2.33.4" 2022-09-29T08:43:12.578 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:43:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:43:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:43:12.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:43:12 smithi083 ceph-mon[77242]: pgmap v5849: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:12.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:43:12 smithi083 ceph-mon[79930]: pgmap v5849: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:13.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:43:12 smithi153 ceph-mon[70044]: pgmap v5849: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:43:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:43:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:43:14.578 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:43:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:43:14.255Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:43:14.947 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:43:14 smithi083 ceph-mon[79930]: pgmap v5850: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:14.948 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:43:14 smithi083 ceph-mon[77242]: pgmap v5850: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:15.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:43:14 smithi153 ceph-mon[70044]: pgmap v5850: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:15.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:43:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:43:14.946Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:43:16.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:43:16 smithi083 ceph-mon[77242]: pgmap v5851: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:16.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:43:16 smithi083 ceph-mon[79930]: pgmap v5851: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:17.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:43:16 smithi153 ceph-mon[70044]: pgmap v5851: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:18.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:43:18 smithi083 ceph-mon[79930]: pgmap v5852: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:18.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:43:18 smithi083 ceph-mon[77242]: pgmap v5852: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:19.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:43:18 smithi153 ceph-mon[70044]: pgmap v5852: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:20.868 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:43:20 smithi083 ceph-mon[79930]: pgmap v5853: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:20.869 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:43:20 smithi083 ceph-mon[77242]: pgmap v5853: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:21.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:43:20 smithi153 ceph-mon[70044]: pgmap v5853: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:21.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:43:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:43:20.867Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:43:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:43:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:43:21] "GET /metrics HTTP/1.1" 200 225211 "" "Prometheus/2.33.4" 2022-09-29T08:43:22.592 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:43:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:43:22.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:43:22.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:43:22 smithi083 ceph-mon[79930]: pgmap v5854: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:22.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:43:22 smithi083 ceph-mon[77242]: pgmap v5854: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:23.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:43:22 smithi153 ceph-mon[70044]: pgmap v5854: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:43:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:43:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:43:24.593 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:43:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:43:24.257Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:43:24.947 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:43:24 smithi083 ceph-mon[79930]: pgmap v5855: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:24.947 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:43:24 smithi083 ceph-mon[77242]: pgmap v5855: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:25.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:43:24 smithi153 ceph-mon[70044]: pgmap v5855: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:25.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:43:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:43:24.946Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:43:26.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:43:26 smithi083 ceph-mon[79930]: pgmap v5856: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:26.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:43:26 smithi083 ceph-mon[77242]: pgmap v5856: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:27.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:43:26 smithi153 ceph-mon[70044]: pgmap v5856: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:27.493 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:43:27.784 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:43:27.784 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 5m ago 3h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:43:27.785 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 3m ago 3h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:43:27.785 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 5m ago 3h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:43:27.785 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 3m ago 3h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:43:27.785 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 3m ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:43:27.786 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 5m ago 3h 680M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:43:27.786 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 5m ago 3h 743M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:43:27.786 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 3m ago 3h 329M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:43:27.786 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 5m ago 3h 436M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:43:27.786 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 5m ago 3h 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:43:27.787 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 3m ago 3h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:43:27.787 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 5m ago 3h 538M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:43:27.787 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 5m ago 3h 529M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:43:27.787 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 5m ago 3h 417M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:43:27.788 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 5m ago 3h 515M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:43:27.788 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 3m ago 3h 457M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:43:27.788 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 3m ago 3h 565M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:43:27.788 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 3m ago 3h 503M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:43:27.789 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 3m ago 3h 591M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:43:27.789 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 3m ago 3h 163M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:43:28.139 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:43:28.139 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:43:28.140 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:43:28.140 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:43:28.140 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:43:28.140 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:43:28.140 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:43:28.141 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:43:28.141 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:43:28.141 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:43:28.141 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:43:28.141 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:43:28.142 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:43:28.142 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:43:28.142 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:43:28.142 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:43:28.143 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:43:28.143 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:43:28.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:43:28 smithi083 ceph-mon[79930]: pgmap v5857: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:28.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:43:28 smithi083 ceph-mon[79930]: from='client.40947 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:43:28.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:43:28 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2692128002' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:43:28.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:43:28 smithi083 ceph-mon[77242]: pgmap v5857: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:28.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:43:28 smithi083 ceph-mon[77242]: from='client.40947 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:43:28.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:43:28 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2692128002' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:43:29.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:43:28 smithi153 ceph-mon[70044]: pgmap v5857: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:29.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:43:28 smithi153 ceph-mon[70044]: from='client.40947 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:43:29.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:43:28 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2692128002' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:43:29.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:43:29 smithi083 ceph-mon[79930]: from='client.40953 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:43:29.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:43:29 smithi083 ceph-mon[77242]: from='client.40953 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:43:30.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:43:29 smithi153 ceph-mon[70044]: from='client.40953 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:43:30.869 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:43:30 smithi083 ceph-mon[79930]: pgmap v5858: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:30.870 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:43:30 smithi083 ceph-mon[77242]: pgmap v5858: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:31.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:43:30 smithi153 ceph-mon[70044]: pgmap v5858: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:31.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:43:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:43:30.868Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:43:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:43:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:43:31] "GET /metrics HTTP/1.1" 200 225216 "" "Prometheus/2.33.4" 2022-09-29T08:43:32.609 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:43:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:43:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:43:32.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:43:32 smithi083 ceph-mon[79930]: pgmap v5859: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:32.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:43:32 smithi083 ceph-mon[77242]: pgmap v5859: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:33.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:43:32 smithi153 ceph-mon[70044]: pgmap v5859: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:43:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:43:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:43:34.611 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:43:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:43:34.257Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:43:34.948 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:43:34 smithi083 ceph-mon[79930]: pgmap v5860: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:34.949 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:43:34 smithi083 ceph-mon[77242]: pgmap v5860: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:35.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:43:34 smithi153 ceph-mon[70044]: pgmap v5860: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:35.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:43:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:43:34.947Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:43:36.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:43:36 smithi083 ceph-mon[79930]: pgmap v5861: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:36.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:43:36 smithi083 ceph-mon[77242]: pgmap v5861: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:37.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:43:36 smithi153 ceph-mon[70044]: pgmap v5861: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:38.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:43:38 smithi083 ceph-mon[79930]: pgmap v5862: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:38.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:43:38 smithi083 ceph-mon[77242]: pgmap v5862: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:39.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:43:38 smithi153 ceph-mon[70044]: pgmap v5862: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:40.870 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:43:40 smithi083 ceph-mon[77242]: pgmap v5863: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:40.871 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:43:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:43:40.871 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:43:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:43:40.871 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:43:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:43:40.871 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:43:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:43:40.872 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:43:40 smithi083 ceph-mon[79930]: pgmap v5863: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:40.872 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:43:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:43:40.872 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:43:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:43:40.873 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:43:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:43:40.873 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:43:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:43:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:43:40 smithi153 ceph-mon[70044]: pgmap v5863: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:43:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:43:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:43:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:43:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:43:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:43:41.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:43:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:43:41.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:43:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:43:40.869Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:43:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:43:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:43:41] "GET /metrics HTTP/1.1" 200 225215 "" "Prometheus/2.33.4" 2022-09-29T08:43:42.626 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:43:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:43:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:43:42.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:43:42 smithi083 ceph-mon[79930]: pgmap v5864: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:42.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:43:42 smithi083 ceph-mon[77242]: pgmap v5864: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:43.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:43:42 smithi153 ceph-mon[70044]: pgmap v5864: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:43:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:43:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:43:44.628 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:43:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:43:44.258Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:43:44.948 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:43:44 smithi083 ceph-mon[79930]: pgmap v5865: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:44.949 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:43:44 smithi083 ceph-mon[77242]: pgmap v5865: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:45.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:43:44 smithi153 ceph-mon[70044]: pgmap v5865: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:45.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:43:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:43:44.947Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:43:46.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:43:46 smithi083 ceph-mon[77242]: pgmap v5866: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:46.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:43:46 smithi083 ceph-mon[79930]: pgmap v5866: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:47.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:43:46 smithi153 ceph-mon[70044]: pgmap v5866: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:48.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:43:48 smithi083 ceph-mon[79930]: pgmap v5867: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:48.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:43:48 smithi083 ceph-mon[77242]: pgmap v5867: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:49.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:43:48 smithi153 ceph-mon[70044]: pgmap v5867: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:50.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:43:50 smithi083 ceph-mon[77242]: pgmap v5868: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:50.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:43:50 smithi083 ceph-mon[79930]: pgmap v5868: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:50.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:43:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:43:50.870Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:43:51.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:43:50 smithi153 ceph-mon[70044]: pgmap v5868: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:43:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:43:51] "GET /metrics HTTP/1.1" 200 225215 "" "Prometheus/2.33.4" 2022-09-29T08:43:52.640 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:43:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:43:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:43:52.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:43:52 smithi083 ceph-mon[79930]: pgmap v5869: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:52.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:43:52 smithi083 ceph-mon[77242]: pgmap v5869: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:53.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:43:52 smithi153 ceph-mon[70044]: pgmap v5869: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:43:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:43:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:43:54.643 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:43:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:43:54.257Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:43:54.950 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:43:54 smithi083 ceph-mon[79930]: pgmap v5870: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:54.950 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:43:54 smithi083 ceph-mon[77242]: pgmap v5870: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:55.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:43:54 smithi153 ceph-mon[70044]: pgmap v5870: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:55.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:43:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:43:54.948Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:43:55.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:43:55 smithi083 ceph-mon[79930]: pgmap v5871: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:55.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:43:55 smithi083 ceph-mon[77242]: pgmap v5871: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:56.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:43:55 smithi153 ceph-mon[70044]: pgmap v5871: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:57.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:43:56 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:43:57.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:43:56 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:43:57.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:43:56 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:43:57.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:43:56 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:43:57.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:43:56 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:43:57.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:43:56 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:43:57.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:43:56 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:43:57.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:43:56 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:43:57.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:43:56 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:43:58.460 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:43:58.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:43:58 smithi083 ceph-mon[79930]: pgmap v5872: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:58.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:43:58 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:43:58.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:43:58 smithi083 ceph-mon[77242]: pgmap v5872: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:58.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:43:58 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:43:58.751 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:43:58.751 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 6m ago 3h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:43:58.752 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 4m ago 3h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:43:58.752 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 6m ago 3h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:43:58.752 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 4m ago 3h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:43:58.752 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 4m ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:43:58.753 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 6m ago 3h 680M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:43:58.753 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 6m ago 3h 743M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:43:58.753 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 4m ago 3h 329M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:43:58.753 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 6m ago 3h 436M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:43:58.754 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 6m ago 3h 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:43:58.754 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 4m ago 3h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:43:58.754 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 6m ago 3h 538M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:43:58.754 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 6m ago 3h 529M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:43:58.754 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 6m ago 3h 417M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:43:58.755 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 6m ago 3h 515M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:43:58.755 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 4m ago 3h 457M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:43:58.755 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 4m ago 3h 565M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:43:58.755 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 4m ago 3h 503M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:43:58.756 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 4m ago 3h 591M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:43:58.756 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 4m ago 3h 163M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:43:58.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:43:58 smithi153 ceph-mon[70044]: pgmap v5872: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:43:58.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:43:58 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:43:59.108 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:43:59.109 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:43:59.109 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:43:59.109 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:43:59.110 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:43:59.110 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:43:59.110 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:43:59.111 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:43:59.111 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:43:59.111 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:43:59.111 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:43:59.111 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:43:59.112 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:43:59.112 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:43:59.112 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:43:59.112 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:43:59.113 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:43:59.113 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:43:59.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:43:59 smithi083 ceph-mon[77242]: from='client.40965 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:43:59.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:43:59 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/901422845' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:43:59.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:43:59 smithi083 ceph-mon[79930]: from='client.40965 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:43:59.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:43:59 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/901422845' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:43:59.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:43:59 smithi153 ceph-mon[70044]: from='client.40965 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:43:59.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:43:59 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/901422845' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:44:00.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:44:00 smithi083 ceph-mon[79930]: from='client.40971 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:44:00.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:44:00 smithi083 ceph-mon[79930]: pgmap v5873: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:00.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:44:00 smithi083 ceph-mon[77242]: from='client.40971 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:44:00.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:44:00 smithi083 ceph-mon[77242]: pgmap v5873: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:00.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:44:00 smithi153 ceph-mon[70044]: from='client.40971 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:44:00.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:44:00 smithi153 ceph-mon[70044]: pgmap v5873: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:01.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:44:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:44:00.870Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:44:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:44:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:44:01] "GET /metrics HTTP/1.1" 200 225211 "" "Prometheus/2.33.4" 2022-09-29T08:44:02.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:44:02 smithi083 ceph-mon[79930]: pgmap v5874: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:02.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:44:02 smithi083 ceph-mon[77242]: pgmap v5874: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:02.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:44:02 smithi153 ceph-mon[70044]: pgmap v5874: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:44:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:44:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:44:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:44:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:44:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:44:04.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:44:04 smithi083 ceph-mon[79930]: pgmap v5875: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:04.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:44:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:44:04.258Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:44:04.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:44:04 smithi083 ceph-mon[77242]: pgmap v5875: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:04.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:44:04 smithi153 ceph-mon[70044]: pgmap v5875: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:05.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:44:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:44:04.948Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:44:06.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:44:06 smithi083 ceph-mon[79930]: pgmap v5876: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:06.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:44:06 smithi083 ceph-mon[77242]: pgmap v5876: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:06.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:44:06 smithi153 ceph-mon[70044]: pgmap v5876: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:08.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:44:08 smithi083 ceph-mon[79930]: pgmap v5877: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:08.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:44:08 smithi083 ceph-mon[77242]: pgmap v5877: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:08.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:44:08 smithi153 ceph-mon[70044]: pgmap v5877: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:10.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:44:10 smithi083 ceph-mon[79930]: pgmap v5878: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:10.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:44:10 smithi083 ceph-mon[77242]: pgmap v5878: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:10.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:44:10 smithi153 ceph-mon[70044]: pgmap v5878: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:11.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:44:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:44:10.870Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:44:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:44:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:44:11] "GET /metrics HTTP/1.1" 200 225216 "" "Prometheus/2.33.4" 2022-09-29T08:44:12.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:44:12 smithi083 ceph-mon[77242]: pgmap v5879: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:12.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:44:12 smithi083 ceph-mon[79930]: pgmap v5879: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:12.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:44:12 smithi153 ceph-mon[70044]: pgmap v5879: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:44:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:44:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:44:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:44:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:44:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:44:14.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:44:14 smithi083 ceph-mon[77242]: pgmap v5880: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:14.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:44:14 smithi083 ceph-mon[79930]: pgmap v5880: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:14.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:44:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:44:14.259Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:44:14.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:44:14 smithi153 ceph-mon[70044]: pgmap v5880: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:15.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:44:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:44:14.948Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:44:16.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:44:16 smithi083 ceph-mon[77242]: pgmap v5881: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:16.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:44:16 smithi083 ceph-mon[79930]: pgmap v5881: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:16.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:44:16 smithi153 ceph-mon[70044]: pgmap v5881: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:18.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:44:18 smithi083 ceph-mon[79930]: pgmap v5882: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:18.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:44:18 smithi083 ceph-mon[77242]: pgmap v5882: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:18.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:44:18 smithi153 ceph-mon[70044]: pgmap v5882: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:20.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:44:20 smithi083 ceph-mon[79930]: pgmap v5883: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:20.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:44:20 smithi083 ceph-mon[77242]: pgmap v5883: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:20.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:44:20 smithi153 ceph-mon[70044]: pgmap v5883: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:21.215 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:44:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:44:20.870Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:44:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:44:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:44:21] "GET /metrics HTTP/1.1" 200 225216 "" "Prometheus/2.33.4" 2022-09-29T08:44:22.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:44:22 smithi083 ceph-mon[79930]: pgmap v5884: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:22.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:44:22 smithi083 ceph-mon[77242]: pgmap v5884: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:22.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:44:22 smithi153 ceph-mon[70044]: pgmap v5884: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:44:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:44:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:44:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:44:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:44:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:44:24.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:44:24 smithi083 ceph-mon[79930]: pgmap v5885: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:44:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:44:24.260Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:44:24.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:44:24 smithi083 ceph-mon[77242]: pgmap v5885: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:24.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:44:24 smithi153 ceph-mon[70044]: pgmap v5885: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:25.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:44:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:44:24.949Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:44:26.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:44:26 smithi083 ceph-mon[79930]: pgmap v5886: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:26.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:44:26 smithi083 ceph-mon[77242]: pgmap v5886: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:26.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:44:26 smithi153 ceph-mon[70044]: pgmap v5886: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:28.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:44:28 smithi083 ceph-mon[77242]: pgmap v5887: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:28.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:44:28 smithi083 ceph-mon[79930]: pgmap v5887: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:28.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:44:28 smithi153 ceph-mon[70044]: pgmap v5887: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:29.438 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:44:29.728 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:44:29.729 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 6m ago 3h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:44:29.729 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 4m ago 3h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:44:29.729 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 6m ago 3h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:44:29.730 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 4m ago 3h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:44:29.730 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 4m ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:44:29.730 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 6m ago 3h 680M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:44:29.731 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 6m ago 3h 743M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:44:29.731 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 4m ago 3h 329M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:44:29.731 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 6m ago 3h 436M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:44:29.732 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 6m ago 3h 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:44:29.732 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 4m ago 3h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:44:29.732 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 6m ago 3h 538M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:44:29.733 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 6m ago 3h 529M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:44:29.733 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 6m ago 3h 417M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:44:29.733 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 6m ago 3h 515M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:44:29.733 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 4m ago 3h 457M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:44:29.734 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 4m ago 3h 565M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:44:29.734 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 4m ago 3h 503M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:44:29.734 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 4m ago 3h 591M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:44:29.734 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 4m ago 3h 163M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:44:30.082 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:44:30.083 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:44:30.083 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:44:30.083 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:44:30.083 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:44:30.084 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:44:30.084 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:44:30.084 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:44:30.084 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:44:30.085 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:44:30.085 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:44:30.085 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:44:30.085 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:44:30.085 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:44:30.086 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:44:30.086 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:44:30.086 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:44:30.086 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:44:30.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:44:30 smithi083 ceph-mon[77242]: pgmap v5888: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:30.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:44:30 smithi083 ceph-mon[77242]: from='client.50914 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:44:30.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:44:30 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4197490108' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:44:30.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:44:30 smithi083 ceph-mon[79930]: pgmap v5888: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:30.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:44:30 smithi083 ceph-mon[79930]: from='client.50914 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:44:30.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:44:30 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4197490108' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:44:30.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:44:30 smithi153 ceph-mon[70044]: pgmap v5888: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:30.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:44:30 smithi153 ceph-mon[70044]: from='client.50914 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:44:30.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:44:30 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4197490108' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:44:31.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:44:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:44:30.870Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:44:31.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:44:31 smithi083 ceph-mon[77242]: from='client.50920 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:44:31.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:44:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:44:31] "GET /metrics HTTP/1.1" 200 225211 "" "Prometheus/2.33.4" 2022-09-29T08:44:31.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:44:31 smithi083 ceph-mon[79930]: from='client.50920 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:44:31.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:44:31 smithi153 ceph-mon[70044]: from='client.50920 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:44:32.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:44:32 smithi083 ceph-mon[79930]: pgmap v5889: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:32.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:44:32 smithi083 ceph-mon[77242]: pgmap v5889: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:32.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:44:32 smithi153 ceph-mon[70044]: pgmap v5889: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:44:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:44:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:44:33.311 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:44:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:44:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:44:34.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:44:34 smithi083 ceph-mon[79930]: pgmap v5890: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:34.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:44:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:44:34.260Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:44:34.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:44:34 smithi083 ceph-mon[77242]: pgmap v5890: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:34.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:44:34 smithi153 ceph-mon[70044]: pgmap v5890: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:35.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:44:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:44:34.950Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:44:36.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:44:36 smithi083 ceph-mon[79930]: pgmap v5891: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:36.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:44:36 smithi083 ceph-mon[77242]: pgmap v5891: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:36.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:44:36 smithi153 ceph-mon[70044]: pgmap v5891: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:38.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:44:38 smithi083 ceph-mon[77242]: pgmap v5892: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:38.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:44:38 smithi083 ceph-mon[79930]: pgmap v5892: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:38.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:44:38 smithi153 ceph-mon[70044]: pgmap v5892: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:40.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:44:40 smithi083 ceph-mon[79930]: pgmap v5893: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:44:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:44:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:44:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:44:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:44:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:44:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:44:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:44:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:44:40 smithi083 ceph-mon[77242]: pgmap v5893: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:44:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:44:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:44:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:44:40.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:44:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:44:40.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:44:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:44:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:44:40 smithi153 ceph-mon[70044]: pgmap v5893: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:44:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:44:40.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:44:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:44:40.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:44:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:44:40.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:44:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:44:41.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:44:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:44:40.871Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:44:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:44:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:44:41] "GET /metrics HTTP/1.1" 200 225213 "" "Prometheus/2.33.4" 2022-09-29T08:44:42.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:44:42 smithi083 ceph-mon[79930]: pgmap v5894: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:42.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:44:42 smithi083 ceph-mon[77242]: pgmap v5894: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:44:42 smithi153 ceph-mon[70044]: pgmap v5894: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:44:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:44:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:44:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:44:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:44:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:44:44.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:44:44 smithi083 ceph-mon[79930]: pgmap v5895: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:44:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:44:44.261Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:44:44.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:44:44 smithi083 ceph-mon[77242]: pgmap v5895: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:44.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:44:44 smithi153 ceph-mon[70044]: pgmap v5895: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:45.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:44:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:44:44.951Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:44:46.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:44:46 smithi083 ceph-mon[77242]: pgmap v5896: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:46.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:44:46 smithi083 ceph-mon[79930]: pgmap v5896: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:46.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:44:46 smithi153 ceph-mon[70044]: pgmap v5896: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:48.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:44:48 smithi083 ceph-mon[79930]: pgmap v5897: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:48.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:44:48 smithi083 ceph-mon[77242]: pgmap v5897: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:48.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:44:48 smithi153 ceph-mon[70044]: pgmap v5897: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:50.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:44:50 smithi083 ceph-mon[79930]: pgmap v5898: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:50.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:44:50 smithi083 ceph-mon[77242]: pgmap v5898: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:50.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:44:50 smithi153 ceph-mon[70044]: pgmap v5898: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:51.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:44:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:44:50.872Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:44:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:44:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:44:51] "GET /metrics HTTP/1.1" 200 225213 "" "Prometheus/2.33.4" 2022-09-29T08:44:52.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:44:52 smithi083 ceph-mon[79930]: pgmap v5899: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:52.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:44:52 smithi083 ceph-mon[77242]: pgmap v5899: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:52.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:44:52 smithi153 ceph-mon[70044]: pgmap v5899: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:44:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:44:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:44:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:44:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:44:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:44:54.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:44:54 smithi083 ceph-mon[79930]: pgmap v5900: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:44:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:44:54.261Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:44:54.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:44:54 smithi083 ceph-mon[77242]: pgmap v5900: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:54.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:44:54 smithi153 ceph-mon[70044]: pgmap v5900: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:55.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:44:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:44:54.951Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:44:56.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:44:56 smithi083 ceph-mon[79930]: pgmap v5901: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:56.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:44:56 smithi083 ceph-mon[77242]: pgmap v5901: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:56.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:44:56 smithi153 ceph-mon[70044]: pgmap v5901: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:44:58 smithi153 ceph-mon[70044]: pgmap v5902: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:58.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:44:58 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:44:58.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:44:58 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:44:58.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:44:58 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:44:58.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:44:58 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:44:58.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:44:58 smithi083 ceph-mon[79930]: pgmap v5902: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:58.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:44:58 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:44:58.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:44:58 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:44:58.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:44:58 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:44:58.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:44:58 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:44:58.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:44:58 smithi083 ceph-mon[77242]: pgmap v5902: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:44:58.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:44:58 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:44:58.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:44:58 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:44:58.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:44:58 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:44:58.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:44:58 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:45:00.409 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:45:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:45:00 smithi153 ceph-mon[70044]: pgmap v5903: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:00.696 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:45:00.696 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 7m ago 3h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:45:00.696 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 5m ago 3h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:45:00.697 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 7m ago 3h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:45:00.697 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 5m ago 3h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:45:00.697 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 5m ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:45:00.697 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 7m ago 3h 680M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:45:00.698 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 7m ago 3h 743M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:45:00.698 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 5m ago 3h 329M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:45:00.698 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 7m ago 3h 436M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:45:00.698 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 7m ago 3h 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:45:00.699 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 5m ago 3h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:45:00.699 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 7m ago 3h 538M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:45:00.699 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 7m ago 3h 529M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:45:00.699 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 7m ago 3h 417M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:45:00.699 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 7m ago 3h 515M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:45:00.700 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 5m ago 3h 457M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:45:00.700 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 5m ago 3h 565M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:45:00.700 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 5m ago 3h 503M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:45:00.700 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 5m ago 3h 591M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:45:00.700 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 5m ago 3h 163M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:45:00.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:45:00 smithi083 ceph-mon[79930]: pgmap v5903: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:00.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:45:00 smithi083 ceph-mon[77242]: pgmap v5903: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:01.052 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:45:01.052 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:45:01.052 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:45:01.053 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:45:01.053 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:45:01.053 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:45:01.053 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:45:01.053 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:45:01.054 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:45:01.054 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:45:01.054 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:45:01.054 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:45:01.055 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:45:01.055 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:45:01.055 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:45:01.055 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:45:01.055 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:45:01.056 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:45:01.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:45:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:45:00.872Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:45:01.507 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:45:01 smithi083 ceph-mon[79930]: from='client.50932 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:45:01.508 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:45:01 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2197975834' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:45:01.508 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:45:01 smithi083 ceph-mon[77242]: from='client.50932 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:45:01.508 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:45:01 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2197975834' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:45:01.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:45:01 smithi153 ceph-mon[70044]: from='client.50932 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:45:01.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:45:01 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2197975834' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:45:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:45:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:45:01] "GET /metrics HTTP/1.1" 200 225215 "" "Prometheus/2.33.4" 2022-09-29T08:45:02.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:45:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:45:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:45:02.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:45:02 smithi153 ceph-mon[70044]: from='client.50938 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:45:02.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:45:02 smithi153 ceph-mon[70044]: pgmap v5904: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:02.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:45:02 smithi083 ceph-mon[77242]: from='client.50938 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:45:02.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:45:02 smithi083 ceph-mon[77242]: pgmap v5904: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:02.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:45:02 smithi083 ceph-mon[79930]: from='client.50938 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:45:02.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:45:02 smithi083 ceph-mon[79930]: pgmap v5904: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:45:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:45:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:45:04.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:45:03 smithi083 ceph-mon[79930]: pgmap v5905: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:04.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:45:03 smithi083 ceph-mon[77242]: pgmap v5905: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:04.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:45:03 smithi153 ceph-mon[70044]: pgmap v5905: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:04.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:45:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:45:04.262Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:45:05.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:45:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:45:04.952Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:45:06.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:45:06 smithi153 ceph-mon[70044]: pgmap v5906: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:06.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:45:06 smithi083 ceph-mon[77242]: pgmap v5906: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:06.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:45:06 smithi083 ceph-mon[79930]: pgmap v5906: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:08.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:45:08 smithi153 ceph-mon[70044]: pgmap v5907: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:08.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:45:08 smithi083 ceph-mon[77242]: pgmap v5907: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:08.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:45:08 smithi083 ceph-mon[79930]: pgmap v5907: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:45:10 smithi153 ceph-mon[70044]: pgmap v5908: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:10.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:45:10 smithi083 ceph-mon[77242]: pgmap v5908: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:10.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:45:10 smithi083 ceph-mon[79930]: pgmap v5908: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:11.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:45:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:45:10.873Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:45:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:45:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:45:11] "GET /metrics HTTP/1.1" 200 225217 "" "Prometheus/2.33.4" 2022-09-29T08:45:12.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:45:12 smithi153 ceph-mon[70044]: pgmap v5909: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:12.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:45:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:45:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:45:12.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:45:12 smithi083 ceph-mon[77242]: pgmap v5909: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:12.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:45:12 smithi083 ceph-mon[79930]: pgmap v5909: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:45:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:45:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:45:14.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:45:14 smithi153 ceph-mon[70044]: pgmap v5910: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:14.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:45:14 smithi083 ceph-mon[79930]: pgmap v5910: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:14.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:45:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:45:14.262Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:45:14.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:45:14 smithi083 ceph-mon[77242]: pgmap v5910: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:15.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:45:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:45:14.953Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:45:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:45:16 smithi153 ceph-mon[70044]: pgmap v5911: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:16.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:45:16 smithi083 ceph-mon[79930]: pgmap v5911: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:16.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:45:16 smithi083 ceph-mon[77242]: pgmap v5911: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:18.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:45:18 smithi153 ceph-mon[70044]: pgmap v5912: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:18.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:45:18 smithi083 ceph-mon[79930]: pgmap v5912: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:18.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:45:18 smithi083 ceph-mon[77242]: pgmap v5912: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:20.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:45:20 smithi153 ceph-mon[70044]: pgmap v5913: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:20.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:45:20 smithi083 ceph-mon[77242]: pgmap v5913: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:20.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:45:20 smithi083 ceph-mon[79930]: pgmap v5913: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:21.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:45:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:45:20.874Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:45:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:45:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:45:21] "GET /metrics HTTP/1.1" 200 225217 "" "Prometheus/2.33.4" 2022-09-29T08:45:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:45:22 smithi153 ceph-mon[70044]: pgmap v5914: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:22.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:45:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:45:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:45:22.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:45:22 smithi083 ceph-mon[79930]: pgmap v5914: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:22.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:45:22 smithi083 ceph-mon[77242]: pgmap v5914: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:45:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:45:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:45:24.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:45:24 smithi153 ceph-mon[70044]: pgmap v5915: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:24.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:45:24 smithi083 ceph-mon[79930]: pgmap v5915: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:45:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:45:24.262Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:45:24.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:45:24 smithi083 ceph-mon[77242]: pgmap v5915: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:25.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:45:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:45:24.954Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:45:26.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:45:26 smithi153 ceph-mon[70044]: pgmap v5916: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:26.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:45:26 smithi083 ceph-mon[77242]: pgmap v5916: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:26.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:45:26 smithi083 ceph-mon[79930]: pgmap v5916: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:45:28 smithi153 ceph-mon[70044]: pgmap v5917: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:28.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:45:28 smithi083 ceph-mon[79930]: pgmap v5917: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:28.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:45:28 smithi083 ceph-mon[77242]: pgmap v5917: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:30.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:45:30 smithi153 ceph-mon[70044]: pgmap v5918: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:30.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:45:30 smithi083 ceph-mon[79930]: pgmap v5918: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:30.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:45:30 smithi083 ceph-mon[77242]: pgmap v5918: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:31.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:45:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:45:30.874Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:45:31.382 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:45:31.672 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:45:31.672 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 7m ago 3h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:45:31.672 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 5m ago 3h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:45:31.672 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 7m ago 3h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:45:31.673 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 5m ago 3h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:45:31.673 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 5m ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:45:31.673 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 7m ago 3h 680M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:45:31.673 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 7m ago 3h 743M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:45:31.673 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 5m ago 3h 329M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:45:31.674 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 7m ago 3h 436M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:45:31.674 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 7m ago 3h 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:45:31.674 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 5m ago 3h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:45:31.674 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 7m ago 3h 538M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:45:31.675 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 7m ago 3h 529M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:45:31.675 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 7m ago 3h 417M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:45:31.675 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 7m ago 3h 515M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:45:31.675 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 5m ago 3h 457M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:45:31.676 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 5m ago 3h 565M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:45:31.676 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 5m ago 3h 503M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:45:31.676 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 5m ago 3h 591M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:45:31.676 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 5m ago 3h 163M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:45:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:45:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:45:31] "GET /metrics HTTP/1.1" 200 225215 "" "Prometheus/2.33.4" 2022-09-29T08:45:32.027 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:45:32.028 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:45:32.028 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:45:32.028 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:45:32.029 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:45:32.029 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:45:32.029 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:45:32.029 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:45:32.029 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:45:32.030 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:45:32.030 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:45:32.030 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:45:32.030 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:45:32.031 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:45:32.031 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:45:32.031 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:45:32.031 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:45:32.032 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:45:32.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:45:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:45:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:45:32.614 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:45:32 smithi153 ceph-mon[70044]: pgmap v5919: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:32.614 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:45:32 smithi153 ceph-mon[70044]: from='client.41019 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:45:32.615 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:45:32 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3005054608' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:45:32.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:45:32 smithi083 ceph-mon[77242]: pgmap v5919: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:32.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:45:32 smithi083 ceph-mon[77242]: from='client.41019 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:45:32.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:45:32 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3005054608' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:45:32.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:45:32 smithi083 ceph-mon[79930]: pgmap v5919: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:32.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:45:32 smithi083 ceph-mon[79930]: from='client.41019 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:45:32.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:45:32 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3005054608' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:45:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:45:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:45:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:45:33.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:45:33 smithi153 ceph-mon[70044]: from='client.50956 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:45:33.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:45:33 smithi083 ceph-mon[77242]: from='client.50956 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:45:33.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:45:33 smithi083 ceph-mon[79930]: from='client.50956 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:45:34.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:45:34 smithi083 ceph-mon[79930]: pgmap v5920: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:34.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:45:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:45:34.264Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:45:34.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:45:34 smithi083 ceph-mon[77242]: pgmap v5920: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:34.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:45:34 smithi153 ceph-mon[70044]: pgmap v5920: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:35.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:45:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:45:34.955Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:45:36.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:45:36 smithi083 ceph-mon[77242]: pgmap v5921: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:36.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:45:36 smithi083 ceph-mon[79930]: pgmap v5921: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:36.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:45:36 smithi153 ceph-mon[70044]: pgmap v5921: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:38.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:45:38 smithi083 ceph-mon[77242]: pgmap v5922: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:38.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:45:38 smithi083 ceph-mon[79930]: pgmap v5922: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:38.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:45:38 smithi153 ceph-mon[70044]: pgmap v5922: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:40.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:45:40 smithi083 ceph-mon[77242]: pgmap v5923: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:45:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:45:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:45:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:45:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:45:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:45:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:45:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:45:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:45:40 smithi083 ceph-mon[79930]: pgmap v5923: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:45:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:45:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:45:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:45:40.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:45:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:45:40.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:45:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:45:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:45:40 smithi153 ceph-mon[70044]: pgmap v5923: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:45:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:45:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:45:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:45:40.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:45:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:45:40.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:45:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:45:41.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:45:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:45:40.874Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:45:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:45:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:45:41] "GET /metrics HTTP/1.1" 200 225214 "" "Prometheus/2.33.4" 2022-09-29T08:45:42.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:45:42 smithi083 ceph-mon[77242]: pgmap v5924: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:45:42 smithi083 ceph-mon[79930]: pgmap v5924: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:45:42 smithi153 ceph-mon[70044]: pgmap v5924: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:45:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:45:42.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:45:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:45:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:45:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:45:44.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:45:44 smithi083 ceph-mon[77242]: pgmap v5925: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:44.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:45:44 smithi083 ceph-mon[79930]: pgmap v5925: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:45:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:45:44.264Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:45:44.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:45:44 smithi153 ceph-mon[70044]: pgmap v5925: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:45.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:45:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:45:44.956Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:45:46.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:45:46 smithi083 ceph-mon[77242]: pgmap v5926: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:46.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:45:46 smithi083 ceph-mon[79930]: pgmap v5926: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:46.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:45:46 smithi153 ceph-mon[70044]: pgmap v5926: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:48.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:45:48 smithi083 ceph-mon[79930]: pgmap v5927: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:48.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:45:48 smithi083 ceph-mon[77242]: pgmap v5927: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:48.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:45:48 smithi153 ceph-mon[70044]: pgmap v5927: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:50.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:45:50 smithi083 ceph-mon[77242]: pgmap v5928: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:50.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:45:50 smithi083 ceph-mon[79930]: pgmap v5928: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:50.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:45:50 smithi153 ceph-mon[70044]: pgmap v5928: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:51.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:45:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:45:50.875Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:45:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:45:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:45:51] "GET /metrics HTTP/1.1" 200 225214 "" "Prometheus/2.33.4" 2022-09-29T08:45:52.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:45:52 smithi083 ceph-mon[77242]: pgmap v5929: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:52.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:45:52 smithi083 ceph-mon[79930]: pgmap v5929: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:52.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:45:52 smithi153 ceph-mon[70044]: pgmap v5929: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:45:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:45:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:45:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:45:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:45:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:45:54.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:45:54 smithi083 ceph-mon[77242]: pgmap v5930: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:54.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:45:54 smithi083 ceph-mon[79930]: pgmap v5930: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:45:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:45:54.265Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:45:54.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:45:54 smithi153 ceph-mon[70044]: pgmap v5930: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:55.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:45:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:45:54.956Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:45:56.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:45:56 smithi083 ceph-mon[79930]: pgmap v5931: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:56.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:45:56 smithi083 ceph-mon[77242]: pgmap v5931: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:56.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:45:56 smithi153 ceph-mon[70044]: pgmap v5931: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:58.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:45:58 smithi083 ceph-mon[79930]: pgmap v5932: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:58.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:45:58 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:45:58.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:45:58 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:45:58.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:45:58 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:45:58.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:45:58 smithi083 ceph-mon[77242]: pgmap v5932: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:58.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:45:58 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:45:58.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:45:58 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:45:58.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:45:58 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:45:58.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:45:58 smithi153 ceph-mon[70044]: pgmap v5932: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:58.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:45:58 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:45:58.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:45:58 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:45:58.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:45:58 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:45:59.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:45:59 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:45:59.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:45:59 smithi083 ceph-mon[77242]: pgmap v5933: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:45:59.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:45:59 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:45:59.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:45:59 smithi083 ceph-mon[79930]: pgmap v5933: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:46:00.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:45:59 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:46:00.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:45:59 smithi153 ceph-mon[70044]: pgmap v5933: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:46:01.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:46:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:46:00.875Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:46:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:46:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:46:01] "GET /metrics HTTP/1.1" 200 225213 "" "Prometheus/2.33.4" 2022-09-29T08:46:02.359 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:46:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:46:02 smithi153 ceph-mon[70044]: pgmap v5934: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:46:02.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:46:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:46:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:46:02.648 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:46:02.649 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 8m ago 3h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:46:02.649 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 6m ago 3h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:46:02.649 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 8m ago 3h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:46:02.649 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 6m ago 3h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:46:02.650 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 6m ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:46:02.650 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 8m ago 3h 680M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:46:02.650 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 8m ago 3h 743M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:46:02.650 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 6m ago 3h 329M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:46:02.651 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 8m ago 3h 436M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:46:02.651 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 8m ago 3h 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:46:02.651 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 6m ago 3h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:46:02.651 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 8m ago 3h 538M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:46:02.651 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 8m ago 3h 529M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:46:02.652 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 8m ago 3h 417M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:46:02.652 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 8m ago 3h 515M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:46:02.652 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 6m ago 3h 457M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:46:02.652 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 6m ago 3h 565M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:46:02.652 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 6m ago 3h 503M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:46:02.653 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 6m ago 3h 591M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:46:02.653 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 6m ago 3h 163M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:46:02.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:46:02 smithi083 ceph-mon[77242]: pgmap v5934: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:46:02.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:46:02 smithi083 ceph-mon[79930]: pgmap v5934: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:46:03.007 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:46:03.008 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:46:03.008 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:46:03.008 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:46:03.009 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:46:03.009 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:46:03.009 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:46:03.009 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:46:03.009 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:46:03.010 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:46:03.010 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:46:03.010 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:46:03.010 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:46:03.011 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:46:03.011 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:46:03.011 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:46:03.011 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:46:03.012 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:46:03.279 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:46:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:46:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:46:03.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:46:03 smithi153 ceph-mon[70044]: from='client.41037 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:46:03.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:46:03 smithi153 ceph-mon[70044]: from='client.41043 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:46:03.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:46:03 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/939859608' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:46:03.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:46:03 smithi083 ceph-mon[77242]: from='client.41037 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:46:03.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:46:03 smithi083 ceph-mon[77242]: from='client.41043 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:46:03.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:46:03 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/939859608' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:46:03.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:46:03 smithi083 ceph-mon[79930]: from='client.41037 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:46:03.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:46:03 smithi083 ceph-mon[79930]: from='client.41043 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:46:03.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:46:03 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/939859608' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:46:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:46:04 smithi153 ceph-mon[70044]: pgmap v5935: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:46:04.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:46:04 smithi083 ceph-mon[79930]: pgmap v5935: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:46:04.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:46:04 smithi083 ceph-mon[77242]: pgmap v5935: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:46:04.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:46:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:46:04.265Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:46:05.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:46:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:46:04.957Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:46:06.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:46:06 smithi153 ceph-mon[70044]: pgmap v5936: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:46:06.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:46:06 smithi083 ceph-mon[79930]: pgmap v5936: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:46:06.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:46:06 smithi083 ceph-mon[77242]: pgmap v5936: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:46:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:46:08 smithi153 ceph-mon[70044]: pgmap v5937: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:46:08.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:46:08 smithi083 ceph-mon[79930]: pgmap v5937: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:46:08.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:46:08 smithi083 ceph-mon[77242]: pgmap v5937: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:46:10.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:46:10 smithi153 ceph-mon[70044]: pgmap v5938: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:46:10.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:46:10 smithi083 ceph-mon[79930]: pgmap v5938: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:46:10.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:46:10 smithi083 ceph-mon[77242]: pgmap v5938: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:46:11.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:46:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:46:10.876Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:46:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:46:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:46:11] "GET /metrics HTTP/1.1" 200 225213 "" "Prometheus/2.33.4" 2022-09-29T08:46:12.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:46:12 smithi153 ceph-mon[70044]: pgmap v5939: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:46:12.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:46:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:46:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:46:12.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:46:12 smithi083 ceph-mon[77242]: pgmap v5939: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:46:12.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:46:12 smithi083 ceph-mon[79930]: pgmap v5939: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:46:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:46:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:46:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:46:14.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:46:14 smithi153 ceph-mon[70044]: pgmap v5940: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:46:14.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:46:14 smithi083 ceph-mon[79930]: pgmap v5940: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:46:14.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:46:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:46:14.266Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:46:14.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:46:14 smithi083 ceph-mon[77242]: pgmap v5940: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:46:15.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:46:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:46:14.958Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:46:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:46:16 smithi153 ceph-mon[70044]: pgmap v5941: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:46:16.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:46:16 smithi083 ceph-mon[79930]: pgmap v5941: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:46:16.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:46:16 smithi083 ceph-mon[77242]: pgmap v5941: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:46:18.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:46:18 smithi153 ceph-mon[70044]: pgmap v5942: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:46:18.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:46:18 smithi083 ceph-mon[79930]: pgmap v5942: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:46:18.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:46:18 smithi083 ceph-mon[77242]: pgmap v5942: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:46:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:46:20 smithi153 ceph-mon[70044]: pgmap v5943: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:46:20.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:46:20 smithi083 ceph-mon[79930]: pgmap v5943: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:46:20.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:46:20 smithi083 ceph-mon[77242]: pgmap v5943: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:46:21.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:46:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:46:20.877Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:46:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:46:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:46:21] "GET /metrics HTTP/1.1" 200 225213 "" "Prometheus/2.33.4" 2022-09-29T08:46:22.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:46:22 smithi153 ceph-mon[70044]: pgmap v5944: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:46:22.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:46:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:46:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:46:22.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:46:22 smithi083 ceph-mon[79930]: pgmap v5944: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:46:22.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:46:22 smithi083 ceph-mon[77242]: pgmap v5944: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:46:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:46:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:46:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:46:24.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:46:24 smithi083 ceph-mon[79930]: pgmap v5945: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:46:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:46:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:46:24.369Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:46:24.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:46:24 smithi083 ceph-mon[77242]: pgmap v5945: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:46:24.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:46:24 smithi153 ceph-mon[70044]: pgmap v5945: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:46:25.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:46:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:46:24.958Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:46:26.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:46:26 smithi083 ceph-mon[79930]: pgmap v5946: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:46:26.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:46:26 smithi083 ceph-mon[77242]: pgmap v5946: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:46:26.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:46:26 smithi153 ceph-mon[70044]: pgmap v5946: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:46:28.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:46:28 smithi083 ceph-mon[79930]: pgmap v5947: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:46:28.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:46:28 smithi083 ceph-mon[77242]: pgmap v5947: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:46:28.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:46:28 smithi153 ceph-mon[70044]: pgmap v5947: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:46:30.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:46:30 smithi083 ceph-mon[79930]: pgmap v5948: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:46:30.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:46:30 smithi083 ceph-mon[77242]: pgmap v5948: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:46:30.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:46:30 smithi153 ceph-mon[70044]: pgmap v5948: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:46:31.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:46:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:46:30.878Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:46:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:46:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:46:31] "GET /metrics HTTP/1.1" 200 225216 "" "Prometheus/2.33.4" 2022-09-29T08:46:32.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:46:32 smithi083 ceph-mon[79930]: pgmap v5949: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:46:32.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:46:32 smithi083 ceph-mon[77242]: pgmap v5949: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:46:32.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:46:32 smithi153 ceph-mon[70044]: pgmap v5949: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:46:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:46:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:46:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:46:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:46:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:46:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:46:33.337 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:46:33.633 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:46:33.634 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 8m ago 3h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:46:33.634 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 6m ago 3h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:46:33.634 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 8m ago 3h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:46:33.634 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 6m ago 3h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:46:33.635 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 6m ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:46:33.635 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 8m ago 3h 680M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:46:33.635 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 8m ago 3h 743M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:46:33.635 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 6m ago 3h 329M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:46:33.636 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 8m ago 3h 436M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:46:33.636 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 8m ago 3h 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:46:33.636 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 6m ago 3h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:46:33.636 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 8m ago 3h 538M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:46:33.637 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 8m ago 3h 529M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:46:33.637 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 8m ago 3h 417M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:46:33.637 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 8m ago 3h 515M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:46:33.637 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 6m ago 3h 457M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:46:33.638 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 6m ago 3h 565M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:46:33.638 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 6m ago 3h 503M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:46:33.638 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 6m ago 3h 591M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:46:33.638 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 6m ago 3h 163M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:46:33.993 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:46:34.021 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:46:34.021 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:46:34.021 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:46:34.022 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:46:34.022 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:46:34.022 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:46:34.022 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:46:34.023 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:46:34.023 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:46:34.023 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:46:34.023 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:46:34.023 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:46:34.024 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:46:34.024 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:46:34.024 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:46:34.024 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:46:34.024 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:46:34.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:46:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:46:34.266Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:46:34.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:46:34 smithi083 ceph-mon[79930]: pgmap v5950: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:46:34.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:46:34 smithi083 ceph-mon[79930]: from='client.41055 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:46:34.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:46:34 smithi083 ceph-mon[79930]: from='client.41061 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:46:34.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:46:34 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/582433620' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:46:34.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:46:34 smithi083 ceph-mon[77242]: pgmap v5950: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:46:34.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:46:34 smithi083 ceph-mon[77242]: from='client.41055 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:46:34.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:46:34 smithi083 ceph-mon[77242]: from='client.41061 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:46:34.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:46:34 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/582433620' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:46:34.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:46:34 smithi153 ceph-mon[70044]: pgmap v5950: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:46:34.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:46:34 smithi153 ceph-mon[70044]: from='client.41055 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:46:34.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:46:34 smithi153 ceph-mon[70044]: from='client.41061 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:46:34.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:46:34 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/582433620' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:46:35.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:46:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:46:34.959Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:46:36.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:46:36 smithi083 ceph-mon[77242]: pgmap v5951: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:46:36.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:46:36 smithi083 ceph-mon[79930]: pgmap v5951: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:46:36.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:46:36 smithi153 ceph-mon[70044]: pgmap v5951: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:46:38.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:46:38 smithi083 ceph-mon[77242]: pgmap v5952: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:46:38.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:46:38 smithi083 ceph-mon[79930]: pgmap v5952: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:46:38.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:46:38 smithi153 ceph-mon[70044]: pgmap v5952: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:46:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:46:40 smithi083 ceph-mon[77242]: pgmap v5953: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:46:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:46:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:46:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:46:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:46:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:46:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:46:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:46:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:46:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:46:40 smithi083 ceph-mon[79930]: pgmap v5953: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:46:40.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:46:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:46:40.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:46:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:46:40.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:46:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:46:40.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:46:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:46:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:46:40 smithi153 ceph-mon[70044]: pgmap v5953: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:46:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:46:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:46:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:46:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:46:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:46:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:46:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:46:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:46:41.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:46:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:46:40.879Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:46:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:46:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:46:41] "GET /metrics HTTP/1.1" 200 225218 "" "Prometheus/2.33.4" 2022-09-29T08:46:42.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:46:42 smithi083 ceph-mon[77242]: pgmap v5954: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:46:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:46:42 smithi083 ceph-mon[79930]: pgmap v5954: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:46:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:46:42 smithi153 ceph-mon[70044]: pgmap v5954: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:46:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:46:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:46:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:46:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:46:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:46:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:46:44.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:46:44 smithi083 ceph-mon[79930]: pgmap v5955: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:46:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:46:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:46:44.268Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:46:44.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:46:44 smithi083 ceph-mon[77242]: pgmap v5955: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:46:44.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:46:44 smithi153 ceph-mon[70044]: pgmap v5955: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:46:45.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:46:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:46:44.960Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:46:46.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:46:46 smithi083 ceph-mon[77242]: pgmap v5956: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:46:46.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:46:46 smithi083 ceph-mon[79930]: pgmap v5956: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:46:46.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:46:46 smithi153 ceph-mon[70044]: pgmap v5956: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:46:48.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:46:48 smithi083 ceph-mon[77242]: pgmap v5957: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:46:48.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:46:48 smithi083 ceph-mon[79930]: pgmap v5957: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:46:48.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:46:48 smithi153 ceph-mon[70044]: pgmap v5957: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:46:50.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:46:50 smithi083 ceph-mon[79930]: pgmap v5958: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:46:50.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:46:50 smithi083 ceph-mon[77242]: pgmap v5958: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:46:50.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:46:50 smithi153 ceph-mon[70044]: pgmap v5958: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:46:51.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:46:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:46:50.880Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:46:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:46:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:46:51] "GET /metrics HTTP/1.1" 200 225218 "" "Prometheus/2.33.4" 2022-09-29T08:46:52.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:46:52 smithi083 ceph-mon[77242]: pgmap v5959: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:46:52.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:46:52 smithi083 ceph-mon[79930]: pgmap v5959: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:46:52.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:46:52 smithi153 ceph-mon[70044]: pgmap v5959: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:46:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:46:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:46:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:46:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:46:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:46:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:46:54.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:46:54 smithi083 ceph-mon[79930]: pgmap v5960: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:46:54.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:46:54 smithi083 ceph-mon[77242]: pgmap v5960: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:46:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:46:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:46:54.269Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:46:54.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:46:54 smithi153 ceph-mon[70044]: pgmap v5960: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:46:55.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:46:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:46:54.960Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:46:56.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:46:56 smithi083 ceph-mon[77242]: pgmap v5961: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:46:56.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:46:56 smithi083 ceph-mon[79930]: pgmap v5961: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:46:56.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:46:56 smithi153 ceph-mon[70044]: pgmap v5961: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:46:58.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:46:58 smithi083 ceph-mon[79930]: pgmap v5962: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:46:58.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:46:58 smithi083 ceph-mon[77242]: pgmap v5962: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:46:58.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:46:58 smithi153 ceph-mon[70044]: pgmap v5962: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:46:59.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:46:59 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:46:59.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:46:59 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:46:59.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:46:59 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:46:59.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:46:59 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:46:59.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:46:59 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:46:59.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:46:59 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:46:59.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:46:59 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:46:59.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:46:59 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:46:59.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:46:59 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:46:59.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:46:59 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:46:59.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:46:59 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:46:59.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:46:59 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:47:00.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:47:00 smithi083 ceph-mon[79930]: pgmap v5963: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:00.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:47:00 smithi083 ceph-mon[77242]: pgmap v5963: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:00.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:47:00 smithi153 ceph-mon[70044]: pgmap v5963: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:01.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:47:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:47:00.881Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:47:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:47:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:47:01] "GET /metrics HTTP/1.1" 200 225215 "" "Prometheus/2.33.4" 2022-09-29T08:47:02.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:47:02 smithi083 ceph-mon[79930]: pgmap v5964: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:02.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:47:02 smithi083 ceph-mon[77242]: pgmap v5964: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:02.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:47:02 smithi153 ceph-mon[70044]: pgmap v5964: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:47:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:47:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:47:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:47:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:47:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:47:04.322 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:47:04.611 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:47:04.611 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 9m ago 3h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:47:04.611 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 7m ago 3h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:47:04.612 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 9m ago 3h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:47:04.612 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 7m ago 3h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:47:04.612 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 7m ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:47:04.612 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 9m ago 3h 680M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:47:04.613 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 9m ago 3h 743M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:47:04.613 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 7m ago 3h 329M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:47:04.613 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 9m ago 3h 436M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:47:04.613 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 9m ago 3h 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:47:04.614 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 7m ago 3h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:47:04.614 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 9m ago 3h 538M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:47:04.614 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 9m ago 3h 529M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:47:04.614 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 9m ago 3h 417M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:47:04.614 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 9m ago 3h 515M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:47:04.615 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 7m ago 3h 457M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:47:04.615 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 7m ago 3h 565M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:47:04.615 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 7m ago 3h 503M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:47:04.615 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 7m ago 3h 591M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:47:04.615 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 7m ago 3h 163M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:47:04.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:47:04 smithi083 ceph-mon[79930]: pgmap v5965: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:04.741 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:47:04 smithi083 ceph-mon[77242]: pgmap v5965: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:04.742 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:47:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:47:04.268Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:47:04.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:47:04 smithi153 ceph-mon[70044]: pgmap v5965: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:04.967 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:47:04.967 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:47:04.968 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:47:04.968 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:47:04.968 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:47:04.968 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:47:04.968 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:47:04.969 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:47:04.969 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:47:04.969 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:47:04.969 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:47:04.969 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:47:04.970 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:47:04.970 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:47:04.970 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:47:04.970 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:47:04.971 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:47:04.971 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:47:05.379 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:47:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:47:04.960Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:47:05.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:47:05 smithi083 ceph-mon[77242]: from='client.51004 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:47:05.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:47:05 smithi083 ceph-mon[77242]: from='client.51010 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:47:05.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:47:05 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3034621942' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:47:05.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:47:05 smithi083 ceph-mon[79930]: from='client.51004 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:47:05.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:47:05 smithi083 ceph-mon[79930]: from='client.51010 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:47:05.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:47:05 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3034621942' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:47:05.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:47:05 smithi153 ceph-mon[70044]: from='client.51004 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:47:05.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:47:05 smithi153 ceph-mon[70044]: from='client.51010 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:47:05.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:47:05 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3034621942' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:47:06.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:47:06 smithi083 ceph-mon[79930]: pgmap v5966: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:06.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:47:06 smithi083 ceph-mon[77242]: pgmap v5966: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:06.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:47:06 smithi153 ceph-mon[70044]: pgmap v5966: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:08.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:47:08 smithi083 ceph-mon[79930]: pgmap v5967: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:08.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:47:08 smithi083 ceph-mon[77242]: pgmap v5967: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:08.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:47:08 smithi153 ceph-mon[70044]: pgmap v5967: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:10.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:47:10 smithi083 ceph-mon[79930]: pgmap v5968: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:10.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:47:10 smithi083 ceph-mon[77242]: pgmap v5968: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:10.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:47:10 smithi153 ceph-mon[70044]: pgmap v5968: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:11.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:47:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:47:10.881Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:47:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:47:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:47:11] "GET /metrics HTTP/1.1" 200 225214 "" "Prometheus/2.33.4" 2022-09-29T08:47:12.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:47:12 smithi083 ceph-mon[79930]: pgmap v5969: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:12.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:47:12 smithi083 ceph-mon[77242]: pgmap v5969: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:12.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:47:12 smithi153 ceph-mon[70044]: pgmap v5969: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:47:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:47:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:47:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:47:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:47:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:47:14.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:47:14 smithi083 ceph-mon[77242]: pgmap v5970: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:14.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:47:14 smithi083 ceph-mon[79930]: pgmap v5970: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:14.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:47:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:47:14.270Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:47:14.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:47:14 smithi153 ceph-mon[70044]: pgmap v5970: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:15.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:47:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:47:14.961Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:47:16.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:47:16 smithi083 ceph-mon[79930]: pgmap v5971: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:16.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:47:16 smithi083 ceph-mon[77242]: pgmap v5971: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:16.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:47:16 smithi153 ceph-mon[70044]: pgmap v5971: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:18.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:47:18 smithi083 ceph-mon[79930]: pgmap v5972: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:18.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:47:18 smithi083 ceph-mon[77242]: pgmap v5972: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:18.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:47:18 smithi153 ceph-mon[70044]: pgmap v5972: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:20.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:47:20 smithi083 ceph-mon[79930]: pgmap v5973: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:20.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:47:20 smithi083 ceph-mon[77242]: pgmap v5973: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:20.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:47:20 smithi153 ceph-mon[70044]: pgmap v5973: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:21.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:47:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:47:20.882Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:47:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:47:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:47:21] "GET /metrics HTTP/1.1" 200 225214 "" "Prometheus/2.33.4" 2022-09-29T08:47:22.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:47:22 smithi083 ceph-mon[79930]: pgmap v5974: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:22.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:47:22 smithi083 ceph-mon[77242]: pgmap v5974: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:22.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:47:22 smithi153 ceph-mon[70044]: pgmap v5974: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:47:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:47:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:47:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:47:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:47:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:47:24.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:47:24 smithi083 ceph-mon[79930]: pgmap v5975: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:47:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:47:24.270Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:47:24.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:47:24 smithi083 ceph-mon[77242]: pgmap v5975: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:24.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:47:24 smithi153 ceph-mon[70044]: pgmap v5975: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:25.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:47:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:47:24.962Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:47:26.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:47:26 smithi083 ceph-mon[79930]: pgmap v5976: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:26.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:47:26 smithi083 ceph-mon[77242]: pgmap v5976: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:26.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:47:26 smithi153 ceph-mon[70044]: pgmap v5976: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:28.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:47:28 smithi083 ceph-mon[79930]: pgmap v5977: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:28.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:47:28 smithi083 ceph-mon[77242]: pgmap v5977: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:28.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:47:28 smithi153 ceph-mon[70044]: pgmap v5977: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:30.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:47:30 smithi083 ceph-mon[79930]: pgmap v5978: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:30.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:47:30 smithi083 ceph-mon[77242]: pgmap v5978: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:30.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:47:30 smithi153 ceph-mon[70044]: pgmap v5978: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:31.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:47:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:47:30.883Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:47:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:47:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:47:31] "GET /metrics HTTP/1.1" 200 225213 "" "Prometheus/2.33.4" 2022-09-29T08:47:32.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:47:32 smithi083 ceph-mon[79930]: pgmap v5979: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:32.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:47:32 smithi083 ceph-mon[77242]: pgmap v5979: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:32.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:47:32 smithi153 ceph-mon[70044]: pgmap v5979: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:47:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:47:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:47:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:47:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:47:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:47:34.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:47:34 smithi083 ceph-mon[79930]: pgmap v5980: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:34.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:47:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:47:34.271Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:47:34.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:47:34 smithi083 ceph-mon[77242]: pgmap v5980: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:34.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:47:34 smithi153 ceph-mon[70044]: pgmap v5980: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:35.290 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:47:35.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:47:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:47:34.964Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:47:35.580 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:47:35.580 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 9m ago 3h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:47:35.581 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 7m ago 3h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:47:35.581 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 9m ago 3h 25.5M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:47:35.581 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 7m ago 3h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:47:35.581 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 7m ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:47:35.582 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 9m ago 3h 680M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:47:35.582 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 9m ago 3h 743M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:47:35.582 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 7m ago 3h 329M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:47:35.582 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 9m ago 3h 436M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:47:35.583 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 9m ago 3h 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:47:35.583 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 7m ago 3h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:47:35.583 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 9m ago 3h 538M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:47:35.583 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 9m ago 3h 529M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:47:35.583 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 9m ago 3h 417M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:47:35.584 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 9m ago 3h 515M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:47:35.584 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 7m ago 3h 457M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:47:35.584 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 7m ago 3h 565M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:47:35.584 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 7m ago 3h 503M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:47:35.584 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 7m ago 3h 591M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:47:35.585 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 7m ago 3h 163M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:47:35.938 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:47:35.939 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:47:35.939 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:47:35.939 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:47:35.939 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:47:35.940 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:47:35.940 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:47:35.940 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:47:35.940 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:47:35.940 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:47:35.941 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:47:35.941 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:47:35.941 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:47:35.941 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:47:35.942 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:47:35.942 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:47:35.942 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:47:35.942 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:47:36.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:47:36 smithi083 ceph-mon[79930]: pgmap v5981: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:36.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:47:36 smithi083 ceph-mon[79930]: from='client.51022 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:47:36.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:47:36 smithi083 ceph-mon[79930]: from='client.41097 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:47:36.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:47:36 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/762389325' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:47:36.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:47:36 smithi083 ceph-mon[77242]: pgmap v5981: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:36.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:47:36 smithi083 ceph-mon[77242]: from='client.51022 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:47:36.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:47:36 smithi083 ceph-mon[77242]: from='client.41097 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:47:36.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:47:36 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/762389325' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:47:36.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:47:36 smithi153 ceph-mon[70044]: pgmap v5981: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:36.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:47:36 smithi153 ceph-mon[70044]: from='client.51022 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:47:36.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:47:36 smithi153 ceph-mon[70044]: from='client.41097 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:47:36.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:47:36 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/762389325' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:47:38.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:47:38 smithi083 ceph-mon[77242]: pgmap v5982: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:38.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:47:38 smithi083 ceph-mon[79930]: pgmap v5982: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:38.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:47:38 smithi153 ceph-mon[70044]: pgmap v5982: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:40.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:47:40 smithi083 ceph-mon[77242]: pgmap v5983: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:47:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:47:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:47:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:47:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:47:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:47:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:47:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:47:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:47:40 smithi083 ceph-mon[79930]: pgmap v5983: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:47:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:47:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:47:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:47:40.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:47:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:47:40.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:47:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:47:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:47:40 smithi153 ceph-mon[70044]: pgmap v5983: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:47:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:47:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:47:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:47:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:47:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:47:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:47:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:47:41.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:47:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:47:40.884Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:47:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:47:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:47:41] "GET /metrics HTTP/1.1" 200 225215 "" "Prometheus/2.33.4" 2022-09-29T08:47:42.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:47:42 smithi083 ceph-mon[79930]: pgmap v5984: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:42.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:47:42 smithi083 ceph-mon[77242]: pgmap v5984: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:47:42 smithi153 ceph-mon[70044]: pgmap v5984: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:47:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:47:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:47:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:47:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:47:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:47:44.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:47:44 smithi083 ceph-mon[77242]: pgmap v5985: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:44.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:47:44 smithi083 ceph-mon[79930]: pgmap v5985: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:47:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:47:44.272Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:47:44.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:47:44 smithi153 ceph-mon[70044]: pgmap v5985: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:45.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:47:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:47:44.964Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:47:46.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:47:46 smithi083 ceph-mon[77242]: pgmap v5986: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:46.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:47:46 smithi083 ceph-mon[79930]: pgmap v5986: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:46.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:47:46 smithi153 ceph-mon[70044]: pgmap v5986: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:48.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:47:48 smithi083 ceph-mon[77242]: pgmap v5987: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:48.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:47:48 smithi083 ceph-mon[79930]: pgmap v5987: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:48.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:47:48 smithi153 ceph-mon[70044]: pgmap v5987: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:50.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:47:50 smithi083 ceph-mon[79930]: pgmap v5988: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:50.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:47:50 smithi083 ceph-mon[77242]: pgmap v5988: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:50.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:47:50 smithi153 ceph-mon[70044]: pgmap v5988: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:51.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:47:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:47:50.884Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:47:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:47:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:47:51] "GET /metrics HTTP/1.1" 200 225215 "" "Prometheus/2.33.4" 2022-09-29T08:47:52.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:47:52 smithi083 ceph-mon[77242]: pgmap v5989: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:52.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:47:52 smithi083 ceph-mon[79930]: pgmap v5989: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:52.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:47:52 smithi153 ceph-mon[70044]: pgmap v5989: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:47:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:47:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:47:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:47:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:47:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:47:54.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:47:54 smithi083 ceph-mon[79930]: pgmap v5990: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:47:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:47:54.272Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:47:54.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:47:54 smithi083 ceph-mon[77242]: pgmap v5990: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:54.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:47:54 smithi153 ceph-mon[70044]: pgmap v5990: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:55.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:47:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:47:54.965Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:47:56.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:47:56 smithi153 ceph-mon[70044]: pgmap v5991: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:56.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:47:56 smithi083 ceph-mon[79930]: pgmap v5991: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:56.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:47:56 smithi083 ceph-mon[77242]: pgmap v5991: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:58.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:47:58 smithi153 ceph-mon[70044]: pgmap v5992: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:58.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:47:58 smithi083 ceph-mon[77242]: pgmap v5992: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:58.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:47:58 smithi083 ceph-mon[79930]: pgmap v5992: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:47:59.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:47:59 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:47:59.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:47:59 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:47:59.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:47:59 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:47:59.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:47:59 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:47:59.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:47:59 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:47:59.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:47:59 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:47:59.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:47:59 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:47:59.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:47:59 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:47:59.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:47:59 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:48:00.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:48:00 smithi153 ceph-mon[70044]: pgmap v5993: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:48:00.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:48:00 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:48:00.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:48:00 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:48:00.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:48:00 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:48:00.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:48:00 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:48:00.885 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:48:00 smithi083 ceph-mon[79930]: pgmap v5993: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:48:00.886 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:48:00 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:48:00.886 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:48:00 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:48:00.886 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:48:00 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:48:00.887 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:48:00 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:48:00.887 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:48:00 smithi083 ceph-mon[77242]: pgmap v5993: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:48:00.887 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:48:00 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:48:00.888 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:48:00 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:48:00.888 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:48:00 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:48:00.888 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:48:00 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:48:01.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:48:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:48:00.884Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:48:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:48:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:48:01] "GET /metrics HTTP/1.1" 200 225214 "" "Prometheus/2.33.4" 2022-09-29T08:48:02.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:48:02 smithi153 ceph-mon[70044]: pgmap v5994: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:48:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:48:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:48:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:48:02.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:48:02 smithi083 ceph-mon[79930]: pgmap v5994: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:48:02.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:48:02 smithi083 ceph-mon[77242]: pgmap v5994: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:48:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:48:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:48:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:48:04.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:48:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:48:04.272Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:48:04.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:48:04 smithi083 ceph-mon[79930]: pgmap v5995: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:48:04.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:48:04 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:48:04.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:48:04 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:48:04.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:48:04 smithi083 ceph-mon[77242]: pgmap v5995: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:48:04.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:48:04 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:48:04.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:48:04 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:48:04.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:48:04 smithi153 ceph-mon[70044]: pgmap v5995: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:48:04.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:48:04 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:48:04.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:48:04 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:48:05.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:48:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:48:04.964Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:48:05.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:48:05 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:48:05.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:48:05 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:48:06.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:48:05 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:48:06.265 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:48:06.554 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:48:06.554 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 2s ago 3h 26.7M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:48:06.555 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 8m ago 3h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:48:06.555 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 2s ago 3h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:48:06.555 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 8m ago 3h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:48:06.555 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 8m ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:48:06.555 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 2s ago 3h 689M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:48:06.556 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 2s ago 3h 781M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:48:06.556 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 8m ago 3h 329M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:48:06.556 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 2s ago 3h 455M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:48:06.556 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 2s ago 3h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:48:06.556 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 8m ago 3h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:48:06.557 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 2s ago 3h 564M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:48:06.557 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 2s ago 3h 555M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:48:06.557 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 2s ago 3h 437M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:48:06.557 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 2s ago 3h 540M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:48:06.558 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 8m ago 3h 457M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:48:06.558 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 8m ago 3h 565M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:48:06.558 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 8m ago 3h 503M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:48:06.558 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 8m ago 3h 591M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:48:06.558 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 8m ago 3h 163M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:48:06.914 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:48:06.914 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:48:06.914 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:48:06.915 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:48:06.915 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:48:06.915 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:48:06.915 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:48:06.916 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:48:06.916 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:48:06.916 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:48:06.916 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:48:06.916 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:48:06.917 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:48:06.917 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:48:06.917 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:48:06.917 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:48:06.918 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:48:06.918 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:48:06.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:48:06 smithi083 ceph-mon[79930]: pgmap v5996: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:48:06.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:48:06 smithi083 ceph-mon[77242]: pgmap v5996: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:48:07.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:48:06 smithi153 ceph-mon[70044]: pgmap v5996: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:48:07.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:48:07 smithi083 ceph-mon[77242]: from='client.41109 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:48:07.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:48:07 smithi083 ceph-mon[77242]: from='client.51046 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:48:07.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:48:07 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1525448046' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:48:07.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:48:07 smithi083 ceph-mon[79930]: from='client.41109 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:48:07.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:48:07 smithi083 ceph-mon[79930]: from='client.51046 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:48:07.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:48:07 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1525448046' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:48:08.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:48:07 smithi153 ceph-mon[70044]: from='client.41109 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:48:08.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:48:07 smithi153 ceph-mon[70044]: from='client.51046 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:48:08.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:48:07 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1525448046' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:48:08.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:48:08 smithi083 ceph-mon[79930]: pgmap v5997: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:48:08.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:48:08 smithi083 ceph-mon[77242]: pgmap v5997: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:48:09.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:48:08 smithi153 ceph-mon[70044]: pgmap v5997: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:48:10.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:48:10 smithi083 ceph-mon[79930]: pgmap v5998: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:48:10.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:48:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:48:10.885Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:48:10.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:48:10 smithi083 ceph-mon[77242]: pgmap v5998: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:48:11.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:48:10 smithi153 ceph-mon[70044]: pgmap v5998: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:48:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:48:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:48:11] "GET /metrics HTTP/1.1" 200 225213 "" "Prometheus/2.33.4" 2022-09-29T08:48:12.676 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:48:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:48:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:48:12.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:48:12 smithi083 ceph-mon[79930]: pgmap v5999: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:48:12.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:48:12 smithi083 ceph-mon[77242]: pgmap v5999: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:48:13.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:48:12 smithi153 ceph-mon[70044]: pgmap v5999: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:48:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:48:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:48:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:48:14.679 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:48:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:48:14.274Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:48:14.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:48:14 smithi083 ceph-mon[79930]: pgmap v6000: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:48:14.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:48:14 smithi083 ceph-mon[77242]: pgmap v6000: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:48:15.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:48:14 smithi153 ceph-mon[70044]: pgmap v6000: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:48:15.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:48:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:48:14.966Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:48:16.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:48:16 smithi083 ceph-mon[77242]: pgmap v6001: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:48:16.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:48:16 smithi083 ceph-mon[79930]: pgmap v6001: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:48:17.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:48:16 smithi153 ceph-mon[70044]: pgmap v6001: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:48:18.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:48:18 smithi083 ceph-mon[79930]: pgmap v6002: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:48:18.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:48:18 smithi083 ceph-mon[77242]: pgmap v6002: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:48:19.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:48:18 smithi153 ceph-mon[70044]: pgmap v6002: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:48:20.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:48:19 smithi153 ceph-mon[70044]: pgmap v6003: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:48:20.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:48:19 smithi083 ceph-mon[79930]: pgmap v6003: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:48:20.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:48:19 smithi083 ceph-mon[77242]: pgmap v6003: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:48:21.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:48:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:48:20.885Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:48:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:48:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:48:21] "GET /metrics HTTP/1.1" 200 225213 "" "Prometheus/2.33.4" 2022-09-29T08:48:22.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:48:22 smithi083 ceph-mon[79930]: pgmap v6004: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:48:22.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:48:22 smithi083 ceph-mon[77242]: pgmap v6004: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:48:22.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:48:22 smithi153 ceph-mon[70044]: pgmap v6004: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:48:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:48:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:48:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:48:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:48:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:48:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:48:24.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:48:24 smithi083 ceph-mon[79930]: pgmap v6005: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:48:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:48:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:48:24.275Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:48:24.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:48:24 smithi083 ceph-mon[77242]: pgmap v6005: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:48:24.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:48:24 smithi153 ceph-mon[70044]: pgmap v6005: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:48:25.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:48:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:48:24.967Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:48:26.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:48:26 smithi083 ceph-mon[79930]: pgmap v6006: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:48:26.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:48:26 smithi083 ceph-mon[77242]: pgmap v6006: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:48:26.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:48:26 smithi153 ceph-mon[70044]: pgmap v6006: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:48:28.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:48:28 smithi083 ceph-mon[79930]: pgmap v6007: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:48:28.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:48:28 smithi083 ceph-mon[77242]: pgmap v6007: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:48:28.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:48:28 smithi153 ceph-mon[70044]: pgmap v6007: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:48:30.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:48:30 smithi083 ceph-mon[79930]: pgmap v6008: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:48:30.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:48:30 smithi083 ceph-mon[77242]: pgmap v6008: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:48:30.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:48:30 smithi153 ceph-mon[70044]: pgmap v6008: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:48:31.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:48:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:48:30.886Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:48:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:48:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:48:31] "GET /metrics HTTP/1.1" 200 225212 "" "Prometheus/2.33.4" 2022-09-29T08:48:32.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:48:32 smithi083 ceph-mon[79930]: pgmap v6009: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:48:32.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:48:32 smithi083 ceph-mon[77242]: pgmap v6009: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:48:32.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:48:32 smithi153 ceph-mon[70044]: pgmap v6009: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:48:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:48:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:48:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:48:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:48:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:48:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:48:34.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:48:34 smithi083 ceph-mon[79930]: pgmap v6010: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:48:34.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:48:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:48:34.275Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:48:34.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:48:34 smithi083 ceph-mon[77242]: pgmap v6010: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:48:34.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:48:34 smithi153 ceph-mon[70044]: pgmap v6010: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:48:35.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:48:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:48:34.967Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:48:36.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:48:36 smithi083 ceph-mon[79930]: pgmap v6011: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:48:36.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:48:36 smithi083 ceph-mon[77242]: pgmap v6011: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:48:36.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:48:36 smithi153 ceph-mon[70044]: pgmap v6011: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:48:37.242 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:48:37.531 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:48:37.532 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 33s ago 3h 26.7M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:48:37.532 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 8m ago 3h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:48:37.532 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 33s ago 3h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:48:37.533 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 8m ago 3h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:48:37.533 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 8m ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:48:37.533 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 33s ago 3h 689M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:48:37.533 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 33s ago 3h 781M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:48:37.533 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 8m ago 3h 329M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:48:37.534 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 33s ago 3h 455M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:48:37.534 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 33s ago 3h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:48:37.534 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 8m ago 3h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:48:37.534 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 33s ago 3h 564M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:48:37.535 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 33s ago 3h 555M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:48:37.535 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 33s ago 3h 437M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:48:37.535 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 33s ago 3h 540M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:48:37.535 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 8m ago 3h 457M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:48:37.535 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 8m ago 3h 565M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:48:37.536 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 8m ago 3h 503M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:48:37.536 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 8m ago 3h 591M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:48:37.536 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 8m ago 3h 163M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:48:37.887 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:48:37.888 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:48:37.888 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:48:37.888 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:48:37.889 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:48:37.889 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:48:37.889 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:48:37.889 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:48:37.889 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:48:37.890 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:48:37.890 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:48:37.890 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:48:37.890 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:48:37.891 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:48:37.891 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:48:37.891 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:48:37.891 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:48:37.892 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:48:38.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:48:38 smithi083 ceph-mon[79930]: from='client.41127 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:48:38.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:48:38 smithi083 ceph-mon[79930]: pgmap v6012: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:48:38.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:48:38 smithi083 ceph-mon[79930]: from='client.41133 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:48:38.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:48:38 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3478039966' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:48:38.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:48:38 smithi083 ceph-mon[77242]: from='client.41127 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:48:38.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:48:38 smithi083 ceph-mon[77242]: pgmap v6012: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:48:38.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:48:38 smithi083 ceph-mon[77242]: from='client.41133 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:48:38.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:48:38 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3478039966' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:48:38.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:48:38 smithi153 ceph-mon[70044]: from='client.41127 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:48:38.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:48:38 smithi153 ceph-mon[70044]: pgmap v6012: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:48:38.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:48:38 smithi153 ceph-mon[70044]: from='client.41133 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:48:38.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:48:38 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3478039966' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:48:40.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:48:40 smithi083 ceph-mon[79930]: pgmap v6013: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:48:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:48:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:48:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:48:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:48:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:48:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:48:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:48:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:48:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:48:40 smithi083 ceph-mon[77242]: pgmap v6013: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:48:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:48:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:48:40.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:48:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:48:40.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:48:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:48:40.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:48:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:48:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:48:40 smithi153 ceph-mon[70044]: pgmap v6013: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:48:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:48:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:48:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:48:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:48:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:48:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:48:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:48:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:48:41.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:48:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:48:40.887Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:48:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:48:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:48:41] "GET /metrics HTTP/1.1" 200 225212 "" "Prometheus/2.33.4" 2022-09-29T08:48:42.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:48:42 smithi083 ceph-mon[79930]: pgmap v6014: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:48:42.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:48:42 smithi083 ceph-mon[77242]: pgmap v6014: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:48:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:48:42 smithi153 ceph-mon[70044]: pgmap v6014: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:48:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:48:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:48:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:48:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:48:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:48:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:48:44.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:48:44 smithi083 ceph-mon[79930]: pgmap v6015: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:48:44.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:48:44 smithi083 ceph-mon[77242]: pgmap v6015: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:48:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:48:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:48:44.276Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:48:44.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:48:44 smithi153 ceph-mon[70044]: pgmap v6015: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:48:45.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:48:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:48:44.968Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:48:46.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:48:46 smithi083 ceph-mon[79930]: pgmap v6016: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:48:46.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:48:46 smithi083 ceph-mon[77242]: pgmap v6016: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:48:46.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:48:46 smithi153 ceph-mon[70044]: pgmap v6016: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:48:48.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:48:48 smithi083 ceph-mon[79930]: pgmap v6017: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:48:48.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:48:48 smithi083 ceph-mon[77242]: pgmap v6017: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:48:48.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:48:48 smithi153 ceph-mon[70044]: pgmap v6017: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:48:50.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:48:50 smithi083 ceph-mon[79930]: pgmap v6018: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:48:50.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:48:50 smithi083 ceph-mon[77242]: pgmap v6018: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:48:50.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:48:50 smithi153 ceph-mon[70044]: pgmap v6018: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:48:51.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:48:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:48:50.887Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:48:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:48:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:48:51] "GET /metrics HTTP/1.1" 200 225212 "" "Prometheus/2.33.4" 2022-09-29T08:48:52.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:48:52 smithi083 ceph-mon[77242]: pgmap v6019: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:48:52.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:48:52 smithi083 ceph-mon[79930]: pgmap v6019: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:48:52.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:48:52 smithi153 ceph-mon[70044]: pgmap v6019: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:48:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:48:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:48:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:48:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:48:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:48:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:48:54.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:48:54 smithi083 ceph-mon[77242]: pgmap v6020: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:48:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:48:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:48:54.275Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:48:54.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:48:54 smithi083 ceph-mon[79930]: pgmap v6020: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:48:54.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:48:54 smithi153 ceph-mon[70044]: pgmap v6020: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:48:55.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:48:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:48:54.968Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:48:56.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:48:56 smithi083 ceph-mon[79930]: pgmap v6021: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:48:56.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:48:56 smithi083 ceph-mon[77242]: pgmap v6021: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:48:56.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:48:56 smithi153 ceph-mon[70044]: pgmap v6021: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:48:58.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:48:58 smithi083 ceph-mon[77242]: pgmap v6022: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:48:58.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:48:58 smithi083 ceph-mon[79930]: pgmap v6022: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:48:58.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:48:58 smithi153 ceph-mon[70044]: pgmap v6022: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:00.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:49:00 smithi083 ceph-mon[79930]: pgmap v6023: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:00.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:49:00 smithi083 ceph-mon[77242]: pgmap v6023: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:00.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:49:00 smithi153 ceph-mon[70044]: pgmap v6023: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:01.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:49:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:49:00.888Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:49:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:49:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:49:01] "GET /metrics HTTP/1.1" 200 225215 "" "Prometheus/2.33.4" 2022-09-29T08:49:02.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:49:02 smithi083 ceph-mon[79930]: pgmap v6024: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:02.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:49:02 smithi083 ceph-mon[77242]: pgmap v6024: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:02.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:49:02 smithi153 ceph-mon[70044]: pgmap v6024: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:49:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:49:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:49:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:49:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:49:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:49:04.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:49:04 smithi083 ceph-mon[77242]: pgmap v6025: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:04.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:49:04 smithi083 ceph-mon[79930]: pgmap v6025: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:04.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:49:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:49:04.276Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:49:04.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:49:04 smithi153 ceph-mon[70044]: pgmap v6025: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:05.387 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:49:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:49:04.968Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:49:05.387 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:49:05 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:49:05.388 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:49:05 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:49:05.388 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:49:05 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:49:05.388 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:49:05 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:49:05.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:49:05 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:49:05.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:49:05 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:49:05.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:49:05 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:49:05.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:49:05 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:49:05.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:49:05 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:49:05.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:49:05 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:49:05.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:49:05 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:49:05.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:49:05 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:49:06.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:49:06 smithi083 ceph-mon[77242]: pgmap v6026: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:06.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:49:06 smithi083 ceph-mon[79930]: pgmap v6026: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:06.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:49:06 smithi153 ceph-mon[70044]: pgmap v6026: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:08.214 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:49:08.502 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:49:08.502 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 64s ago 3h 26.7M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:49:08.502 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 9m ago 3h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:49:08.503 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 64s ago 3h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:49:08.503 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 9m ago 3h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:49:08.503 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 9m ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:49:08.504 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 64s ago 3h 689M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:49:08.504 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 64s ago 3h 781M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:49:08.504 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 9m ago 3h 329M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:49:08.504 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 64s ago 3h 455M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:49:08.504 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 64s ago 3h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:49:08.505 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 9m ago 3h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:49:08.505 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 64s ago 3h 564M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:49:08.505 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 64s ago 3h 555M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:49:08.505 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 64s ago 3h 437M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:49:08.506 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 64s ago 3h 540M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:49:08.506 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 9m ago 3h 457M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:49:08.506 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 9m ago 3h 565M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:49:08.506 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 9m ago 3h 503M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:49:08.507 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 9m ago 3h 591M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:49:08.507 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 9m ago 3h 163M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:49:08.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:49:08 smithi083 ceph-mon[77242]: pgmap v6027: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:08.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:49:08 smithi083 ceph-mon[79930]: pgmap v6027: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:08.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:49:08 smithi153 ceph-mon[70044]: pgmap v6027: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:08.856 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:49:08.856 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:49:08.856 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:49:08.857 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:49:08.857 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:49:08.857 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:49:08.857 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:49:08.858 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:49:08.858 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:49:08.858 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:49:08.858 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:49:08.859 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:49:08.859 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:49:08.859 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:49:08.859 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:49:08.860 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:49:08.860 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:49:08.860 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:49:09.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:49:09 smithi083 ceph-mon[79930]: from='client.41145 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:49:09.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:49:09 smithi083 ceph-mon[79930]: from='client.41151 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:49:09.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:49:09 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4181663616' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:49:09.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:49:09 smithi083 ceph-mon[77242]: from='client.41145 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:49:09.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:49:09 smithi083 ceph-mon[77242]: from='client.41151 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:49:09.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:49:09 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4181663616' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:49:09.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:49:09 smithi153 ceph-mon[70044]: from='client.41145 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:49:09.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:49:09 smithi153 ceph-mon[70044]: from='client.41151 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:49:09.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:49:09 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4181663616' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:49:10.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:49:10 smithi083 ceph-mon[79930]: pgmap v6028: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:10.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:49:10 smithi083 ceph-mon[77242]: pgmap v6028: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:10.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:49:10 smithi153 ceph-mon[70044]: pgmap v6028: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:11.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:49:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:49:10.888Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:49:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:49:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:49:11] "GET /metrics HTTP/1.1" 200 225215 "" "Prometheus/2.33.4" 2022-09-29T08:49:12.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:49:12 smithi083 ceph-mon[79930]: pgmap v6029: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:12.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:49:12 smithi083 ceph-mon[77242]: pgmap v6029: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:12.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:49:12 smithi153 ceph-mon[70044]: pgmap v6029: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:49:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:49:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:49:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:49:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:49:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:49:14.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:49:14 smithi083 ceph-mon[79930]: pgmap v6030: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:14.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:49:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:49:14.277Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:49:14.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:49:14 smithi083 ceph-mon[77242]: pgmap v6030: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:14.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:49:14 smithi153 ceph-mon[70044]: pgmap v6030: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:15.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:49:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:49:14.969Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:49:16.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:49:16 smithi083 ceph-mon[77242]: pgmap v6031: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:16.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:49:16 smithi083 ceph-mon[79930]: pgmap v6031: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:16.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:49:16 smithi153 ceph-mon[70044]: pgmap v6031: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:18.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:49:18 smithi083 ceph-mon[79930]: pgmap v6032: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:18.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:49:18 smithi083 ceph-mon[77242]: pgmap v6032: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:18.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:49:18 smithi153 ceph-mon[70044]: pgmap v6032: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:20.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:49:20 smithi083 ceph-mon[79930]: pgmap v6033: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:20.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:49:20 smithi083 ceph-mon[77242]: pgmap v6033: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:20.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:49:20 smithi153 ceph-mon[70044]: pgmap v6033: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:21.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:49:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:49:20.889Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:49:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:49:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:49:21] "GET /metrics HTTP/1.1" 200 225215 "" "Prometheus/2.33.4" 2022-09-29T08:49:22.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:49:22 smithi083 ceph-mon[79930]: pgmap v6034: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:22.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:49:22 smithi083 ceph-mon[77242]: pgmap v6034: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:22.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:49:22 smithi153 ceph-mon[70044]: pgmap v6034: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:49:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:49:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:49:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:49:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:49:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:49:24.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:49:24 smithi083 ceph-mon[79930]: pgmap v6035: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:49:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:49:24.278Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:49:24.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:49:24 smithi083 ceph-mon[77242]: pgmap v6035: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:24.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:49:24 smithi153 ceph-mon[70044]: pgmap v6035: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:25.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:49:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:49:24.970Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:49:26.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:49:26 smithi083 ceph-mon[77242]: pgmap v6036: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:26.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:49:26 smithi083 ceph-mon[79930]: pgmap v6036: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:26.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:49:26 smithi153 ceph-mon[70044]: pgmap v6036: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:28.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:49:28 smithi083 ceph-mon[77242]: pgmap v6037: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:28.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:49:28 smithi083 ceph-mon[79930]: pgmap v6037: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:28.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:49:28 smithi153 ceph-mon[70044]: pgmap v6037: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:30.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:49:30 smithi083 ceph-mon[79930]: pgmap v6038: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:30.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:49:30 smithi083 ceph-mon[77242]: pgmap v6038: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:30.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:49:30 smithi153 ceph-mon[70044]: pgmap v6038: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:31.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:49:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:49:30.890Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:49:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:49:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:49:31] "GET /metrics HTTP/1.1" 200 225213 "" "Prometheus/2.33.4" 2022-09-29T08:49:32.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:49:32 smithi083 ceph-mon[77242]: pgmap v6039: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:32.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:49:32 smithi083 ceph-mon[79930]: pgmap v6039: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:32.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:49:32 smithi153 ceph-mon[70044]: pgmap v6039: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:49:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:49:32.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:49:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:49:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:49:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:49:34.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:49:34 smithi083 ceph-mon[79930]: pgmap v6040: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:34.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:49:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:49:34.280Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:49:34.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:49:34 smithi083 ceph-mon[77242]: pgmap v6040: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:34.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:49:34 smithi153 ceph-mon[70044]: pgmap v6040: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:35.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:49:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:49:34.970Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:49:36.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:49:36 smithi083 ceph-mon[79930]: pgmap v6041: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:36.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:49:36 smithi083 ceph-mon[77242]: pgmap v6041: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:36.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:49:36 smithi153 ceph-mon[70044]: pgmap v6041: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:38.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:49:38 smithi083 ceph-mon[77242]: pgmap v6042: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:38.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:49:38 smithi083 ceph-mon[79930]: pgmap v6042: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:38.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:49:38 smithi153 ceph-mon[70044]: pgmap v6042: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:39.186 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:49:39.481 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:49:39.482 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 95s ago 3h 26.7M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:49:39.482 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 9m ago 3h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:49:39.482 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 95s ago 3h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:49:39.483 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 9m ago 3h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:49:39.483 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 9m ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:49:39.483 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 95s ago 3h 689M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:49:39.483 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 95s ago 3h 781M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:49:39.484 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 9m ago 3h 329M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:49:39.484 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 95s ago 3h 455M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:49:39.484 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 95s ago 3h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:49:39.484 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 9m ago 3h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:49:39.485 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 95s ago 3h 564M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:49:39.485 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 95s ago 3h 555M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:49:39.485 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 95s ago 3h 437M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:49:39.485 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 95s ago 3h 540M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:49:39.485 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 9m ago 3h 457M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:49:39.486 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 9m ago 3h 565M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:49:39.486 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 9m ago 3h 503M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:49:39.486 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 9m ago 3h 591M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:49:39.486 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 9m ago 3h 163M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:49:39.840 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:49:39.841 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:49:39.841 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:49:39.841 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:49:39.841 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:49:39.842 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:49:39.842 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:49:39.842 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:49:39.843 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:49:39.843 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:49:39.843 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:49:39.843 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:49:39.844 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:49:39.844 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:49:39.844 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:49:39.844 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:49:39.845 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:49:39.845 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:49:40.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:49:40 smithi083 ceph-mon[77242]: from='client.51094 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:49:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:49:40 smithi083 ceph-mon[77242]: pgmap v6043: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:49:40 smithi083 ceph-mon[77242]: from='client.41169 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:49:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:49:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:49:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:49:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:49:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:49:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:49:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:49:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:49:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:49:40 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4172883155' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:49:40.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:49:40 smithi083 ceph-mon[79930]: from='client.51094 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:49:40.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:49:40 smithi083 ceph-mon[79930]: pgmap v6043: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:40.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:49:40 smithi083 ceph-mon[79930]: from='client.41169 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:49:40.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:49:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:49:40.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:49:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:49:40.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:49:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:49:40.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:49:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:49:40.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:49:40 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4172883155' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:49:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:49:40 smithi153 ceph-mon[70044]: from='client.51094 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:49:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:49:40 smithi153 ceph-mon[70044]: pgmap v6043: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:49:40 smithi153 ceph-mon[70044]: from='client.41169 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:49:40.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:49:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:49:40.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:49:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:49:40.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:49:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:49:40.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:49:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:49:40.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:49:40 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4172883155' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:49:41.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:49:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:49:40.890Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:49:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:49:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:49:41] "GET /metrics HTTP/1.1" 200 225213 "" "Prometheus/2.33.4" 2022-09-29T08:49:42.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:49:42 smithi083 ceph-mon[79930]: pgmap v6044: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:42.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:49:42 smithi083 ceph-mon[77242]: pgmap v6044: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:49:42 smithi153 ceph-mon[70044]: pgmap v6044: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:49:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:49:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:49:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:49:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:49:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:49:44.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:49:44 smithi083 ceph-mon[79930]: pgmap v6045: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:44.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:49:44 smithi083 ceph-mon[77242]: pgmap v6045: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:49:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:49:44.280Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:49:44.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:49:44 smithi153 ceph-mon[70044]: pgmap v6045: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:45.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:49:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:49:44.971Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:49:46.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:49:46 smithi083 ceph-mon[77242]: pgmap v6046: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:46.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:49:46 smithi083 ceph-mon[79930]: pgmap v6046: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:46.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:49:46 smithi153 ceph-mon[70044]: pgmap v6046: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:48.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:49:48 smithi083 ceph-mon[79930]: pgmap v6047: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:48.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:49:48 smithi083 ceph-mon[77242]: pgmap v6047: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:48.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:49:48 smithi153 ceph-mon[70044]: pgmap v6047: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:50.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:49:50 smithi083 ceph-mon[79930]: pgmap v6048: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:50.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:49:50 smithi083 ceph-mon[77242]: pgmap v6048: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:50.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:49:50 smithi153 ceph-mon[70044]: pgmap v6048: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:51.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:49:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:49:50.892Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:49:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:49:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:49:51] "GET /metrics HTTP/1.1" 200 225213 "" "Prometheus/2.33.4" 2022-09-29T08:49:52.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:49:52 smithi153 ceph-mon[70044]: pgmap v6049: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:49:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:49:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:49:52.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:49:52 smithi083 ceph-mon[79930]: pgmap v6049: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:52.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:49:52 smithi083 ceph-mon[77242]: pgmap v6049: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:49:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:49:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:49:54.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:49:54 smithi083 ceph-mon[79930]: pgmap v6050: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:49:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:49:54.280Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:49:54.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:49:54 smithi083 ceph-mon[77242]: pgmap v6050: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:54.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:49:54 smithi153 ceph-mon[70044]: pgmap v6050: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:55.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:49:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:49:54.972Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:49:56.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:49:56 smithi153 ceph-mon[70044]: pgmap v6051: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:56.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:49:56 smithi083 ceph-mon[77242]: pgmap v6051: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:56.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:49:56 smithi083 ceph-mon[79930]: pgmap v6051: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:58.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:49:58 smithi153 ceph-mon[70044]: pgmap v6052: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:58.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:49:58 smithi083 ceph-mon[79930]: pgmap v6052: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:49:58.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:49:58 smithi083 ceph-mon[77242]: pgmap v6052: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:00.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:50:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[77219]: 2022-09-29T08:49:59.999+0000 7f160f665700 -1 log_channel(cluster) log [ERR] : overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T08:50:00.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:50:00 smithi153 ceph-mon[70044]: pgmap v6053: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:00.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:50:00 smithi153 ceph-mon[70044]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T08:50:00.894 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:50:00 smithi083 ceph-mon[79930]: pgmap v6053: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:00.895 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:50:00 smithi083 ceph-mon[79930]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T08:50:00.895 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:50:00 smithi083 ceph-mon[77242]: pgmap v6053: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:00.896 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:50:00 smithi083 ceph-mon[77242]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T08:50:01.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:50:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:50:00.893Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:50:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:50:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:50:01] "GET /metrics HTTP/1.1" 200 225216 "" "Prometheus/2.33.4" 2022-09-29T08:50:02.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:50:02 smithi153 ceph-mon[70044]: pgmap v6054: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:02.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:50:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:50:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:50:02.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:50:02 smithi083 ceph-mon[79930]: pgmap v6054: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:02.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:50:02 smithi083 ceph-mon[77242]: pgmap v6054: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:50:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:50:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:50:04.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:50:04 smithi083 ceph-mon[79930]: pgmap v6055: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:04.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:50:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:50:04.281Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:50:04.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:50:04 smithi083 ceph-mon[77242]: pgmap v6055: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:04.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:50:04 smithi153 ceph-mon[70044]: pgmap v6055: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:05.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:50:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:50:04.973Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:50:05.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:50:05 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:50:05.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:50:05 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:50:05.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:50:05 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:50:05.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:50:05 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:50:05.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:50:05 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:50:05.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:50:05 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:50:05.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:50:05 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:50:05.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:50:05 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:50:05.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:50:05 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:50:06.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:50:06 smithi153 ceph-mon[70044]: pgmap v6056: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:06.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:50:06 smithi083 ceph-mon[77242]: pgmap v6056: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:06.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:50:06 smithi083 ceph-mon[79930]: pgmap v6056: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:08.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:50:08 smithi153 ceph-mon[70044]: pgmap v6057: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:08.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:50:08 smithi083 ceph-mon[79930]: pgmap v6057: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:08.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:50:08 smithi083 ceph-mon[77242]: pgmap v6057: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:10.174 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:50:10.461 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:50:10.462 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 2m ago 3h 26.7M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:50:10.462 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 0s ago 3h 63.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:50:10.463 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 2m ago 3h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:50:10.463 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 0s ago 3h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:50:10.464 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 0s ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:50:10.464 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 2m ago 3h 689M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:50:10.467 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 2m ago 3h 781M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:50:10.467 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 0s ago 3h 345M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:50:10.467 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 2m ago 3h 455M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:50:10.468 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 2m ago 3h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:50:10.468 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 0s ago 3h 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:50:10.468 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 2m ago 3h 564M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:50:10.468 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 2m ago 3h 555M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:50:10.468 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 2m ago 3h 437M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:50:10.469 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 2m ago 3h 540M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:50:10.469 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 0s ago 3h 479M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:50:10.469 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 0s ago 3h 592M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:50:10.469 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 0s ago 3h 528M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:50:10.470 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 0s ago 3h 621M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:50:10.470 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 0s ago 3h 156M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:50:10.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:50:10 smithi153 ceph-mon[70044]: pgmap v6058: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:10.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:50:10 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:50:10.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:50:10 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:50:10.825 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:50:10.826 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:50:10.827 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:50:10.827 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:50:10.827 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:50:10.828 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:50:10.828 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:50:10.829 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:50:10.829 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:50:10.829 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:50:10.830 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:50:10.830 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:50:10.831 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:50:10.831 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:50:10.832 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:50:10.832 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:50:10.832 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:50:10.833 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:50:10.893 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:50:10 smithi083 ceph-mon[79930]: pgmap v6058: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:10.894 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:50:10 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:50:10.894 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:50:10 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:50:10.894 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:50:10 smithi083 ceph-mon[77242]: pgmap v6058: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:10.895 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:50:10 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:50:10.895 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:50:10 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:50:11.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:50:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:50:10.892Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:50:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:50:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:50:11] "GET /metrics HTTP/1.1" 200 225216 "" "Prometheus/2.33.4" 2022-09-29T08:50:11.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:50:11 smithi083 ceph-mon[77242]: from='client.41181 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:50:11.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:50:11 smithi083 ceph-mon[77242]: from='client.41187 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:50:11.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:50:11 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:50:11.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:50:11 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/995611848' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:50:11.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:50:11 smithi083 ceph-mon[79930]: from='client.41181 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:50:11.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:50:11 smithi083 ceph-mon[79930]: from='client.41187 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:50:11.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:50:11 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:50:11.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:50:11 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/995611848' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:50:12.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:50:11 smithi153 ceph-mon[70044]: from='client.41181 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:50:12.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:50:11 smithi153 ceph-mon[70044]: from='client.41187 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:50:12.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:50:11 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:50:12.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:50:11 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/995611848' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:50:12.699 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:50:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:50:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:50:12.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:50:12 smithi083 ceph-mon[77242]: pgmap v6059: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:12.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:50:12 smithi083 ceph-mon[79930]: pgmap v6059: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:13.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:50:12 smithi153 ceph-mon[70044]: pgmap v6059: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:50:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:50:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:50:14.699 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:50:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:50:14.281Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:50:14.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:50:14 smithi083 ceph-mon[77242]: pgmap v6060: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:14.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:50:14 smithi083 ceph-mon[79930]: pgmap v6060: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:15.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:50:14 smithi153 ceph-mon[70044]: pgmap v6060: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:15.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:50:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:50:14.973Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:50:16.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:50:16 smithi083 ceph-mon[79930]: pgmap v6061: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:16.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:50:16 smithi083 ceph-mon[77242]: pgmap v6061: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:17.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:50:16 smithi153 ceph-mon[70044]: pgmap v6061: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:18.969 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:50:18 smithi083 ceph-mon[79930]: pgmap v6062: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:18.970 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:50:18 smithi083 ceph-mon[77242]: pgmap v6062: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:19.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:50:18 smithi153 ceph-mon[70044]: pgmap v6062: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:20.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:50:20 smithi083 ceph-mon[79930]: pgmap v6063: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:20.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:50:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:50:20.893Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:50:20.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:50:20 smithi083 ceph-mon[77242]: pgmap v6063: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:21.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:50:20 smithi153 ceph-mon[70044]: pgmap v6063: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:21.967 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:50:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:50:21] "GET /metrics HTTP/1.1" 200 225216 "" "Prometheus/2.33.4" 2022-09-29T08:50:21.969 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:50:21 smithi083 ceph-mon[79930]: pgmap v6064: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:21.970 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:50:21 smithi083 ceph-mon[77242]: pgmap v6064: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:22.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:50:21 smithi153 ceph-mon[70044]: pgmap v6064: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:22.815 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:50:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:50:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:50:23.313 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:50:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:50:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:50:24.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:50:24 smithi083 ceph-mon[79930]: pgmap v6065: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:24.714 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:50:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:50:24.281Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:50:24.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:50:24 smithi083 ceph-mon[77242]: pgmap v6065: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:24.823 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:50:24 smithi153 ceph-mon[70044]: pgmap v6065: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:25.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:50:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:50:24.973Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:50:26.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:50:26 smithi083 ceph-mon[77242]: pgmap v6066: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:26.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:50:26 smithi083 ceph-mon[79930]: pgmap v6066: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:26.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:50:26 smithi153 ceph-mon[70044]: pgmap v6066: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:28.723 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:50:28 smithi083 ceph-mon[77242]: pgmap v6067: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:28.724 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:50:28 smithi083 ceph-mon[79930]: pgmap v6067: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:28.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:50:28 smithi153 ceph-mon[70044]: pgmap v6067: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:30.718 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:50:30 smithi083 ceph-mon[79930]: pgmap v6068: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:30.719 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:50:30 smithi083 ceph-mon[77242]: pgmap v6068: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:30.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:50:30 smithi153 ceph-mon[70044]: pgmap v6068: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:31.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:50:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:50:30.895Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:50:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:50:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:50:31] "GET /metrics HTTP/1.1" 200 225215 "" "Prometheus/2.33.4" 2022-09-29T08:50:32.718 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:50:32 smithi083 ceph-mon[79930]: pgmap v6069: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:32.719 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:50:32 smithi083 ceph-mon[77242]: pgmap v6069: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:32.819 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:50:32 smithi153 ceph-mon[70044]: pgmap v6069: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:32.819 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:50:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:50:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:50:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:50:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:50:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:50:34.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:50:34 smithi083 ceph-mon[79930]: pgmap v6070: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:34.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:50:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:50:34.282Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:50:34.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:50:34 smithi083 ceph-mon[77242]: pgmap v6070: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:34.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:50:34 smithi153 ceph-mon[70044]: pgmap v6070: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:35.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:50:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:50:34.974Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:50:36.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:50:36 smithi083 ceph-mon[79930]: pgmap v6071: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:36.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:50:36 smithi083 ceph-mon[77242]: pgmap v6071: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:36.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:50:36 smithi153 ceph-mon[70044]: pgmap v6071: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:38.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:50:38 smithi083 ceph-mon[79930]: pgmap v6072: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:38.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:50:38 smithi083 ceph-mon[77242]: pgmap v6072: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:38.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:50:38 smithi153 ceph-mon[70044]: pgmap v6072: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:50:40 smithi083 ceph-mon[77242]: pgmap v6073: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:40.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:50:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:50:40.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:50:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:50:40.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:50:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:50:40.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:50:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:50:40.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:50:40 smithi083 ceph-mon[79930]: pgmap v6073: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:40.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:50:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:50:40.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:50:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:50:40.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:50:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:50:40.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:50:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:50:40.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:50:40 smithi153 ceph-mon[70044]: pgmap v6073: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:40.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:50:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:50:40.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:50:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:50:40.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:50:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:50:40.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:50:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:50:41.155 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:50:41.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:50:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:50:40.895Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:50:41.444 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:50:41.444 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 2m ago 3h 26.7M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:50:41.445 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 31s ago 3h 63.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:50:41.445 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 2m ago 3h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:50:41.446 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 31s ago 3h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:50:41.447 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 31s ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:50:41.447 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 2m ago 3h 689M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:50:41.449 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 2m ago 3h 781M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:50:41.450 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 31s ago 3h 345M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:50:41.450 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 2m ago 3h 455M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:50:41.451 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 2m ago 3h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:50:41.451 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 31s ago 3h 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:50:41.452 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 2m ago 3h 564M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:50:41.453 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 2m ago 3h 555M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:50:41.454 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 2m ago 3h 437M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:50:41.455 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 2m ago 3h 540M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:50:41.455 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 31s ago 3h 479M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:50:41.456 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 31s ago 3h 592M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:50:41.456 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 31s ago 3h 528M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:50:41.457 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 31s ago 3h 621M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:50:41.459 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 31s ago 3h 156M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:50:41.803 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:50:41.803 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:50:41.803 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:50:41.803 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:50:41.804 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:50:41.804 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:50:41.804 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:50:41.804 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:50:41.812 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:50:41.813 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:50:41.813 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:50:41.814 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:50:41.814 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:50:41.814 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:50:41.815 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:50:41.815 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:50:41.816 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:50:41.816 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:50:41.967 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:50:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:50:41] "GET /metrics HTTP/1.1" 200 225218 "" "Prometheus/2.33.4" 2022-09-29T08:50:42.719 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:50:42 smithi083 ceph-mon[79930]: from='client.41199 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:50:42.719 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:50:42 smithi083 ceph-mon[79930]: pgmap v6074: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:42.720 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:50:42 smithi083 ceph-mon[79930]: from='client.51136 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:50:42.720 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:50:42 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2092859221' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:50:42.721 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:50:42 smithi083 ceph-mon[77242]: from='client.41199 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:50:42.721 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:50:42 smithi083 ceph-mon[77242]: pgmap v6074: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:42.722 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:50:42 smithi083 ceph-mon[77242]: from='client.51136 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:50:42.722 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:50:42 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2092859221' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:50:42.812 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:50:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:50:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:50:42.813 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:50:42 smithi153 ceph-mon[70044]: from='client.41199 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:50:42.813 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:50:42 smithi153 ceph-mon[70044]: pgmap v6074: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:42.814 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:50:42 smithi153 ceph-mon[70044]: from='client.51136 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:50:42.822 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:50:42 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2092859221' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:50:43.318 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:50:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:50:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:50:44.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:50:44 smithi083 ceph-mon[79930]: pgmap v6075: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:44.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:50:44 smithi083 ceph-mon[77242]: pgmap v6075: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:44.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:50:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:50:44.282Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:50:44.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:50:44 smithi153 ceph-mon[70044]: pgmap v6075: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:45.472 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:50:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:50:44.975Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:50:46.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:50:46 smithi083 ceph-mon[79930]: pgmap v6076: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:46.718 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:50:46 smithi083 ceph-mon[77242]: pgmap v6076: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:46.814 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:50:46 smithi153 ceph-mon[70044]: pgmap v6076: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:48.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:50:48 smithi083 ceph-mon[77242]: pgmap v6077: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:48.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:50:48 smithi083 ceph-mon[79930]: pgmap v6077: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:48.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:50:48 smithi153 ceph-mon[70044]: pgmap v6077: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:50.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:50:50 smithi083 ceph-mon[79930]: pgmap v6078: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:50.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:50:50 smithi083 ceph-mon[77242]: pgmap v6078: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:50.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:50:50 smithi153 ceph-mon[70044]: pgmap v6078: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:51.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:50:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:50:50.896Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:50:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:50:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:50:51] "GET /metrics HTTP/1.1" 200 225218 "" "Prometheus/2.33.4" 2022-09-29T08:50:52.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:50:52 smithi083 ceph-mon[77242]: pgmap v6079: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:52.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:50:52 smithi083 ceph-mon[79930]: pgmap v6079: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:52.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:50:52 smithi153 ceph-mon[70044]: pgmap v6079: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:52.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:50:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:50:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:50:53.310 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:50:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:50:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:50:54.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:50:54 smithi083 ceph-mon[79930]: pgmap v6080: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:50:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:50:54.283Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:50:54.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:50:54 smithi083 ceph-mon[77242]: pgmap v6080: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:54.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:50:54 smithi153 ceph-mon[70044]: pgmap v6080: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:55.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:50:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:50:54.975Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:50:56.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:50:56 smithi083 ceph-mon[77242]: pgmap v6081: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:56.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:50:56 smithi083 ceph-mon[79930]: pgmap v6081: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:56.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:50:56 smithi153 ceph-mon[70044]: pgmap v6081: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:58.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:50:58 smithi083 ceph-mon[79930]: pgmap v6082: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:58.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:50:58 smithi083 ceph-mon[77242]: pgmap v6082: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:50:58.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:50:58 smithi153 ceph-mon[70044]: pgmap v6082: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:00.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:51:00 smithi083 ceph-mon[79930]: pgmap v6083: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:00.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:51:00 smithi083 ceph-mon[77242]: pgmap v6083: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:00.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:51:00 smithi153 ceph-mon[70044]: pgmap v6083: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:01.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:51:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:51:00.896Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:51:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:51:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:51:01] "GET /metrics HTTP/1.1" 200 225218 "" "Prometheus/2.33.4" 2022-09-29T08:51:02.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:51:02 smithi083 ceph-mon[79930]: pgmap v6084: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:02.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:51:02 smithi083 ceph-mon[77242]: pgmap v6084: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:02.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:51:02 smithi153 ceph-mon[70044]: pgmap v6084: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:02.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:51:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:51:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:51:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:51:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:51:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:51:04.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:51:04 smithi083 ceph-mon[79930]: pgmap v6085: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:04.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:51:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:51:04.284Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:51:04.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:51:04 smithi083 ceph-mon[77242]: pgmap v6085: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:04.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:51:04 smithi153 ceph-mon[70044]: pgmap v6085: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:05.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:51:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:51:04.976Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:51:06.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:51:06 smithi083 ceph-mon[79930]: pgmap v6086: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:06.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:51:06 smithi083 ceph-mon[77242]: pgmap v6086: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:06.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:51:06 smithi153 ceph-mon[70044]: pgmap v6086: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:08.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:51:08 smithi083 ceph-mon[79930]: pgmap v6087: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:08.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:51:08 smithi083 ceph-mon[77242]: pgmap v6087: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:08.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:51:08 smithi153 ceph-mon[70044]: pgmap v6087: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:10.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:51:10 smithi083 ceph-mon[77242]: pgmap v6088: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:10.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:51:10 smithi083 ceph-mon[79930]: pgmap v6088: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:10.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:51:10 smithi153 ceph-mon[70044]: pgmap v6088: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:11.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:51:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:51:10.897Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:51:11.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:51:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:51:11] "GET /metrics HTTP/1.1" 200 225220 "" "Prometheus/2.33.4" 2022-09-29T08:51:11.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:51:11 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:51:11.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:51:11 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:51:11.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:51:11 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:51:11.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:51:11 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:51:11.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:51:11 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:51:11.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:51:11 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:51:11.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:51:11 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:51:11.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:51:11 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:51:11.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:51:11 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:51:11.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:51:11 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:51:11.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:51:11 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:51:11.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:51:11 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:51:12.120 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:51:12.407 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:51:12.407 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 3m ago 3h 26.7M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:51:12.407 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 62s ago 3h 63.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:51:12.407 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 3m ago 3h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:51:12.408 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 62s ago 3h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:51:12.408 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 62s ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:51:12.408 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 3m ago 3h 689M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:51:12.409 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 3m ago 3h 781M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:51:12.409 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 62s ago 3h 345M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:51:12.409 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 3m ago 3h 455M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:51:12.409 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 3m ago 3h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:51:12.410 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 62s ago 3h 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:51:12.410 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 3m ago 3h 564M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:51:12.410 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 3m ago 3h 555M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:51:12.410 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 3m ago 3h 437M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:51:12.411 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 3m ago 3h 540M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:51:12.411 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 62s ago 3h 479M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:51:12.411 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 62s ago 3h 592M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:51:12.412 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 62s ago 3h 528M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:51:12.412 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 62s ago 3h 621M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:51:12.412 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 62s ago 3h 156M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:51:12.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:51:12 smithi083 ceph-mon[79930]: pgmap v6089: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:12.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:51:12 smithi083 ceph-mon[77242]: pgmap v6089: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:12.761 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:51:12.762 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:51:12.777 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:51:12.777 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:51:12.777 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:51:12.778 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:51:12.778 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:51:12.779 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:51:12.779 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:51:12.779 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:51:12.780 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:51:12.780 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:51:12.781 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:51:12.781 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:51:12.781 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:51:12.782 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:51:12.782 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:51:12.783 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:51:12.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:51:12 smithi153 ceph-mon[70044]: pgmap v6089: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:12.811 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:51:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:51:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:51:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:51:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:51:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:51:13.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:51:13 smithi083 ceph-mon[77242]: from='client.41217 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:51:13.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:51:13 smithi083 ceph-mon[77242]: from='client.51154 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:51:13.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:51:13 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/208302668' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:51:13.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:51:13 smithi083 ceph-mon[79930]: from='client.41217 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:51:13.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:51:13 smithi083 ceph-mon[79930]: from='client.51154 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:51:13.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:51:13 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/208302668' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:51:13.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:51:13 smithi153 ceph-mon[70044]: from='client.41217 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:51:13.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:51:13 smithi153 ceph-mon[70044]: from='client.51154 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:51:13.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:51:13 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/208302668' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:51:14.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:51:14 smithi083 ceph-mon[79930]: pgmap v6090: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:14.717 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:51:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:51:14.284Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:51:14.718 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:51:14 smithi083 ceph-mon[77242]: pgmap v6090: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:14.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:51:14 smithi153 ceph-mon[70044]: pgmap v6090: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:15.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:51:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:51:14.977Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:51:16.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:51:16 smithi083 ceph-mon[79930]: pgmap v6091: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:16.725 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:51:16 smithi083 ceph-mon[77242]: pgmap v6091: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:16.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:51:16 smithi153 ceph-mon[70044]: pgmap v6091: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:18.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:51:18 smithi083 ceph-mon[79930]: pgmap v6092: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:18.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:51:18 smithi083 ceph-mon[77242]: pgmap v6092: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:18.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:51:18 smithi153 ceph-mon[70044]: pgmap v6092: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:20.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:51:20 smithi083 ceph-mon[79930]: pgmap v6093: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:20.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:51:20 smithi083 ceph-mon[77242]: pgmap v6093: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:20.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:51:20 smithi153 ceph-mon[70044]: pgmap v6093: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:21.216 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:51:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:51:20.898Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:51:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:51:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:51:21] "GET /metrics HTTP/1.1" 200 225220 "" "Prometheus/2.33.4" 2022-09-29T08:51:22.718 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:51:22 smithi083 ceph-mon[77242]: pgmap v6094: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:22.719 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:51:22 smithi083 ceph-mon[79930]: pgmap v6094: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:22.814 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:51:22 smithi153 ceph-mon[70044]: pgmap v6094: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:22.816 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:51:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:51:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:51:23.312 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:51:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:51:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:51:24.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:51:24 smithi083 ceph-mon[79930]: pgmap v6095: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:24.715 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:51:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:51:24.284Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:51:24.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:51:24 smithi083 ceph-mon[77242]: pgmap v6095: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:24.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:51:24 smithi153 ceph-mon[70044]: pgmap v6095: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:25.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:51:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:51:24.978Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:51:26.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:51:26 smithi083 ceph-mon[79930]: pgmap v6096: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:26.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:51:26 smithi083 ceph-mon[77242]: pgmap v6096: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:26.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:51:26 smithi153 ceph-mon[70044]: pgmap v6096: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:28.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:51:28 smithi083 ceph-mon[79930]: pgmap v6097: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:28.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:51:28 smithi083 ceph-mon[77242]: pgmap v6097: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:28.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:51:28 smithi153 ceph-mon[70044]: pgmap v6097: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:30.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:51:30 smithi083 ceph-mon[79930]: pgmap v6098: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:30.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:51:30 smithi083 ceph-mon[77242]: pgmap v6098: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:30.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:51:30 smithi153 ceph-mon[70044]: pgmap v6098: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:31.219 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:51:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:51:30.898Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:51:31.969 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:51:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:51:31] "GET /metrics HTTP/1.1" 200 225221 "" "Prometheus/2.33.4" 2022-09-29T08:51:32.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:51:32 smithi083 ceph-mon[79930]: pgmap v6099: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:32.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:51:32 smithi083 ceph-mon[77242]: pgmap v6099: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:32.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:51:32 smithi153 ceph-mon[70044]: pgmap v6099: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:32.811 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:51:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:51:32.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:51:33.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:51:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:51:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:51:34.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:51:34 smithi083 ceph-mon[79930]: pgmap v6100: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:34.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:51:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:51:34.285Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:51:34.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:51:34 smithi083 ceph-mon[77242]: pgmap v6100: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:34.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:51:34 smithi153 ceph-mon[70044]: pgmap v6100: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:35.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:51:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:51:34.979Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:51:36.722 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:51:36 smithi083 ceph-mon[79930]: pgmap v6101: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:36.723 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:51:36 smithi083 ceph-mon[77242]: pgmap v6101: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:36.818 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:51:36 smithi153 ceph-mon[70044]: pgmap v6101: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:38.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:51:38 smithi083 ceph-mon[79930]: pgmap v6102: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:38.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:51:38 smithi083 ceph-mon[77242]: pgmap v6102: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:38.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:51:38 smithi153 ceph-mon[70044]: pgmap v6102: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:51:40 smithi083 ceph-mon[79930]: pgmap v6103: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:51:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:51:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:51:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:51:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:51:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:51:40.722 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:51:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:51:40.723 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:51:40 smithi083 ceph-mon[77242]: pgmap v6103: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:40.724 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:51:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:51:40.724 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:51:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:51:40.724 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:51:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:51:40.725 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:51:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:51:40.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:51:40 smithi153 ceph-mon[70044]: pgmap v6103: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:40.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:51:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:51:40.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:51:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:51:40.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:51:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:51:40.813 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:51:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:51:41.227 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:51:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:51:40.899Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:51:41.966 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:51:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:51:41] "GET /metrics HTTP/1.1" 200 225222 "" "Prometheus/2.33.4" 2022-09-29T08:51:42.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:51:42 smithi083 ceph-mon[79930]: pgmap v6104: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:42.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:51:42 smithi083 ceph-mon[77242]: pgmap v6104: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:42.814 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:51:42 smithi153 ceph-mon[70044]: pgmap v6104: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:42.815 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:51:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:51:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:51:43.087 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:51:43.310 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:51:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:51:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:51:43.383 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:51:43.383 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 3m ago 3h 26.7M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:51:43.384 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 93s ago 3h 63.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:51:43.384 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 3m ago 3h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:51:43.385 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 93s ago 3h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:51:43.385 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 93s ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:51:43.407 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 3m ago 3h 689M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:51:43.408 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 3m ago 3h 781M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:51:43.408 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 93s ago 3h 345M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:51:43.408 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 3m ago 3h 455M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:51:43.409 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 3m ago 3h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:51:43.409 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 93s ago 3h 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:51:43.443 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 3m ago 3h 564M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:51:43.444 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 3m ago 3h 555M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:51:43.444 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 3m ago 3h 437M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:51:43.445 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 3m ago 3h 540M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:51:43.445 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 93s ago 3h 479M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:51:43.454 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 93s ago 3h 592M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:51:43.454 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 93s ago 3h 528M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:51:43.455 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 93s ago 3h 621M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:51:43.455 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 93s ago 3h 156M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:51:43.755 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:51:43.756 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:51:43.757 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:51:43.757 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:51:43.757 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:51:43.758 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:51:43.758 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:51:43.759 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:51:43.759 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:51:43.760 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:51:43.760 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:51:43.761 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:51:43.761 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:51:43.761 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:51:43.768 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:51:43.769 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:51:43.780 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:51:43.783 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:51:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:51:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:51:44.285Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:51:44.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:51:44 smithi083 ceph-mon[79930]: from='client.51166 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:51:44.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:51:44 smithi083 ceph-mon[79930]: pgmap v6105: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:44.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:51:44 smithi083 ceph-mon[79930]: from='client.51172 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:51:44.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:51:44 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1386432932' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:51:44.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:51:44 smithi083 ceph-mon[77242]: from='client.51166 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:51:44.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:51:44 smithi083 ceph-mon[77242]: pgmap v6105: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:44.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:51:44 smithi083 ceph-mon[77242]: from='client.51172 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:51:44.740 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:51:44 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1386432932' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:51:44.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:51:44 smithi153 ceph-mon[70044]: from='client.51166 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:51:44.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:51:44 smithi153 ceph-mon[70044]: pgmap v6105: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:44.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:51:44 smithi153 ceph-mon[70044]: from='client.51172 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:51:44.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:51:44 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1386432932' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:51:45.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:51:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:51:44.979Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:51:46.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:51:46 smithi083 ceph-mon[79930]: pgmap v6106: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:46.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:51:46 smithi083 ceph-mon[77242]: pgmap v6106: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:46.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:51:46 smithi153 ceph-mon[70044]: pgmap v6106: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:48.719 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:51:48 smithi083 ceph-mon[79930]: pgmap v6107: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:48.720 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:51:48 smithi083 ceph-mon[77242]: pgmap v6107: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:48.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:51:48 smithi153 ceph-mon[70044]: pgmap v6107: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:50.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:51:50 smithi153 ceph-mon[70044]: pgmap v6108: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:50.905 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:51:50 smithi083 ceph-mon[79930]: pgmap v6108: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:50.906 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:51:50 smithi083 ceph-mon[77242]: pgmap v6108: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:51.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:51:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:51:50.900Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:51:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:51:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:51:51] "GET /metrics HTTP/1.1" 200 225222 "" "Prometheus/2.33.4" 2022-09-29T08:51:52.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:51:52 smithi153 ceph-mon[70044]: pgmap v6109: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:52.811 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:51:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:51:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:51:52.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:51:52 smithi083 ceph-mon[77242]: pgmap v6109: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:52.974 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:51:52 smithi083 ceph-mon[79930]: pgmap v6109: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:53.311 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:51:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:51:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:51:54.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:51:54 smithi083 ceph-mon[79930]: pgmap v6110: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:54.715 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:51:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:51:54.286Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:51:54.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:51:54 smithi083 ceph-mon[77242]: pgmap v6110: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:54.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:51:54 smithi153 ceph-mon[70044]: pgmap v6110: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:55.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:51:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:51:54.980Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:51:56.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:51:56 smithi153 ceph-mon[70044]: pgmap v6111: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:56.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:51:56 smithi083 ceph-mon[77242]: pgmap v6111: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:56.968 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:51:56 smithi083 ceph-mon[79930]: pgmap v6111: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:58.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:51:58 smithi153 ceph-mon[70044]: pgmap v6112: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:58.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:51:58 smithi083 ceph-mon[77242]: pgmap v6112: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:51:58.968 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:51:58 smithi083 ceph-mon[79930]: pgmap v6112: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:00.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:52:00 smithi153 ceph-mon[70044]: pgmap v6113: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:00.902 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:52:00 smithi083 ceph-mon[79930]: pgmap v6113: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:00.903 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:52:00 smithi083 ceph-mon[77242]: pgmap v6113: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:01.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:52:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:52:00.900Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:52:01.971 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:52:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:52:01] "GET /metrics HTTP/1.1" 200 225218 "" "Prometheus/2.33.4" 2022-09-29T08:52:02.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:52:02 smithi153 ceph-mon[70044]: pgmap v6114: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:02.813 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:52:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:52:02.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:52:02.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:52:02 smithi083 ceph-mon[77242]: pgmap v6114: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:02.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:52:02 smithi083 ceph-mon[79930]: pgmap v6114: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:03.310 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:52:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:52:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:52:04.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:52:04 smithi083 ceph-mon[79930]: pgmap v6115: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:04.714 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:52:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:52:04.286Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:52:04.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:52:04 smithi083 ceph-mon[77242]: pgmap v6115: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:04.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:52:04 smithi153 ceph-mon[70044]: pgmap v6115: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:05.472 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:52:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:52:04.981Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:52:06.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:52:06 smithi153 ceph-mon[70044]: pgmap v6116: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:06.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:52:06 smithi083 ceph-mon[79930]: pgmap v6116: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:06.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:52:06 smithi083 ceph-mon[77242]: pgmap v6116: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:08.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:52:08 smithi153 ceph-mon[70044]: pgmap v6117: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:08.970 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:52:08 smithi083 ceph-mon[79930]: pgmap v6117: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:08.971 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:52:08 smithi083 ceph-mon[77242]: pgmap v6117: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:10.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:52:10 smithi153 ceph-mon[70044]: pgmap v6118: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:10.904 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:52:10 smithi083 ceph-mon[79930]: pgmap v6118: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:10.905 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:52:10 smithi083 ceph-mon[77242]: pgmap v6118: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:11.215 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:52:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:52:10.901Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:52:11.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:52:11 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:52:11.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:52:11 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:52:11.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:52:11 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:52:11.964 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:52:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:52:11] "GET /metrics HTTP/1.1" 200 225218 "" "Prometheus/2.33.4" 2022-09-29T08:52:11.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:52:11 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:52:11.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:52:11 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:52:11.989 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:52:11 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:52:11.989 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:52:11 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:52:11.990 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:52:11 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:52:11.990 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:52:11 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:52:12.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:52:12 smithi153 ceph-mon[70044]: pgmap v6119: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:12.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:52:12 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:52:12.813 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:52:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:52:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:52:12.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:52:12 smithi083 ceph-mon[79930]: pgmap v6119: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:12.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:52:12 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:52:12.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:52:12 smithi083 ceph-mon[77242]: pgmap v6119: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:12.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:52:12 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:52:13.312 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:52:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:52:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:52:14.089 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:52:14.375 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:52:14.375 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 4m ago 3h 26.7M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:52:14.376 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 2m ago 3h 63.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:52:14.376 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 4m ago 3h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:52:14.376 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 2m ago 3h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:52:14.376 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 2m ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:52:14.377 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 4m ago 3h 689M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:52:14.377 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 4m ago 3h 781M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:52:14.377 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 2m ago 3h 345M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:52:14.377 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 4m ago 3h 455M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:52:14.384 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 4m ago 3h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:52:14.384 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 2m ago 3h 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:52:14.385 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 4m ago 3h 564M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:52:14.385 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 4m ago 3h 555M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:52:14.385 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 4m ago 3h 437M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:52:14.385 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 4m ago 3h 540M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:52:14.386 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 2m ago 3h 479M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:52:14.386 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 2m ago 3h 592M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:52:14.386 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 2m ago 3h 528M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:52:14.386 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 2m ago 3h 621M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:52:14.387 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 2m ago 3h 156M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:52:14.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:52:14 smithi083 ceph-mon[79930]: pgmap v6120: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:14.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:52:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:52:14.286Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:52:14.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:52:14 smithi083 ceph-mon[77242]: pgmap v6120: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:14.732 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:52:14.732 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:52:14.732 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:52:14.733 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:52:14.733 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:52:14.733 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:52:14.733 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:52:14.735 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:52:14.736 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:52:14.736 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:52:14.736 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:52:14.736 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:52:14.737 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:52:14.737 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:52:14.737 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:52:14.737 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:52:14.738 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:52:14.738 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:52:14.821 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:52:14 smithi153 ceph-mon[70044]: pgmap v6120: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:15.472 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:52:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:52:14.981Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:52:15.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:52:15 smithi153 ceph-mon[70044]: from='client.51184 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:52:15.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:52:15 smithi153 ceph-mon[70044]: from='client.51190 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:52:15.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:52:15 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/370076587' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:52:15.972 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:52:15 smithi083 ceph-mon[79930]: from='client.51184 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:52:15.973 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:52:15 smithi083 ceph-mon[79930]: from='client.51190 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:52:15.973 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:52:15 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/370076587' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:52:15.974 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:52:15 smithi083 ceph-mon[77242]: from='client.51184 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:52:15.974 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:52:15 smithi083 ceph-mon[77242]: from='client.51190 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:52:15.975 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:52:15 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/370076587' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:52:16.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:52:16 smithi153 ceph-mon[70044]: pgmap v6121: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:16.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:52:16 smithi083 ceph-mon[79930]: pgmap v6121: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:16.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:52:16 smithi083 ceph-mon[77242]: pgmap v6121: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:18.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:52:18 smithi153 ceph-mon[70044]: pgmap v6122: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:18.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:52:18 smithi083 ceph-mon[79930]: pgmap v6122: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:18.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:52:18 smithi083 ceph-mon[77242]: pgmap v6122: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:20.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:52:20 smithi153 ceph-mon[70044]: pgmap v6123: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:20.903 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:52:20 smithi083 ceph-mon[79930]: pgmap v6123: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:20.904 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:52:20 smithi083 ceph-mon[77242]: pgmap v6123: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:21.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:52:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:52:20.901Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:52:21.967 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:52:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:52:21] "GET /metrics HTTP/1.1" 200 225218 "" "Prometheus/2.33.4" 2022-09-29T08:52:22.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:52:22 smithi153 ceph-mon[70044]: pgmap v6124: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:22.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:52:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:52:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:52:22.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:52:22 smithi083 ceph-mon[77242]: pgmap v6124: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:22.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:52:22 smithi083 ceph-mon[79930]: pgmap v6124: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:52:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:52:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:52:24.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:52:24 smithi083 ceph-mon[79930]: pgmap v6125: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:24.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:52:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:52:24.287Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:52:24.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:52:24 smithi083 ceph-mon[77242]: pgmap v6125: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:24.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:52:24 smithi153 ceph-mon[70044]: pgmap v6125: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:25.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:52:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:52:24.982Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:52:26.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:52:26 smithi153 ceph-mon[70044]: pgmap v6126: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:26.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:52:26 smithi083 ceph-mon[79930]: pgmap v6126: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:26.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:52:26 smithi083 ceph-mon[77242]: pgmap v6126: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:28.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:52:28 smithi153 ceph-mon[70044]: pgmap v6127: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:28.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:52:28 smithi083 ceph-mon[79930]: pgmap v6127: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:28.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:52:28 smithi083 ceph-mon[77242]: pgmap v6127: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:30.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:52:30 smithi153 ceph-mon[70044]: pgmap v6128: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:30.904 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:52:30 smithi083 ceph-mon[79930]: pgmap v6128: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:30.905 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:52:30 smithi083 ceph-mon[77242]: pgmap v6128: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:31.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:52:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:52:30.902Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:52:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:52:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:52:31] "GET /metrics HTTP/1.1" 200 225218 "" "Prometheus/2.33.4" 2022-09-29T08:52:32.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:52:32 smithi153 ceph-mon[70044]: pgmap v6129: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:32.810 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:52:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:52:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:52:32.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:52:32 smithi083 ceph-mon[77242]: pgmap v6129: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:32.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:52:32 smithi083 ceph-mon[79930]: pgmap v6129: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:52:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:52:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:52:34.544 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:52:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:52:34.289Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:52:34.813 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:52:34 smithi153 ceph-mon[70044]: pgmap v6130: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:34.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:52:34 smithi083 ceph-mon[79930]: pgmap v6130: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:34.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:52:34 smithi083 ceph-mon[77242]: pgmap v6130: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:35.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:52:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:52:34.983Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:52:36.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:52:36 smithi153 ceph-mon[70044]: pgmap v6131: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:36.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:52:36 smithi083 ceph-mon[77242]: pgmap v6131: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:36.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:52:36 smithi083 ceph-mon[79930]: pgmap v6131: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:38.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:52:38 smithi153 ceph-mon[70044]: pgmap v6132: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:38.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:52:38 smithi083 ceph-mon[79930]: pgmap v6132: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:38.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:52:38 smithi083 ceph-mon[77242]: pgmap v6132: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:52:40 smithi153 ceph-mon[70044]: pgmap v6133: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:52:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:52:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:52:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:52:40.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:52:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:52:40.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:52:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:52:40.904 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:52:40 smithi083 ceph-mon[79930]: pgmap v6133: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:40.905 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:52:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:52:40.905 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:52:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:52:40.905 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:52:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:52:40.905 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:52:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:52:40.906 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:52:40 smithi083 ceph-mon[77242]: pgmap v6133: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:40.906 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:52:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:52:40.910 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:52:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:52:40.911 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:52:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:52:40.911 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:52:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:52:41.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:52:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:52:40.902Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:52:41.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:52:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:52:41] "GET /metrics HTTP/1.1" 200 225219 "" "Prometheus/2.33.4" 2022-09-29T08:52:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:52:42 smithi153 ceph-mon[70044]: pgmap v6134: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:52:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:52:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:52:42.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:52:42 smithi083 ceph-mon[79930]: pgmap v6134: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:42.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:52:42 smithi083 ceph-mon[77242]: pgmap v6134: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:52:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:52:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:52:44.561 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:52:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:52:44.289Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:52:44.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:52:44 smithi083 ceph-mon[79930]: pgmap v6135: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:44.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:52:44 smithi083 ceph-mon[77242]: pgmap v6135: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:45.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:52:44 smithi153 ceph-mon[70044]: pgmap v6135: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:45.061 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:52:45.352 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:52:45.352 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 4m ago 3h 26.7M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:52:45.352 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 2m ago 3h 63.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:52:45.353 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 4m ago 3h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:52:45.354 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 2m ago 3h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:52:45.354 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 2m ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:52:45.354 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 4m ago 3h 689M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:52:45.354 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 4m ago 3h 781M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:52:45.355 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 2m ago 3h 345M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:52:45.355 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 4m ago 3h 455M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:52:45.355 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 4m ago 3h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:52:45.355 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 2m ago 3h 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:52:45.355 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 4m ago 3h 564M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:52:45.356 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 4m ago 3h 555M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:52:45.356 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 4m ago 3h 437M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:52:45.357 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 4m ago 3h 540M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:52:45.357 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 2m ago 3h 479M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:52:45.357 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 2m ago 3h 592M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:52:45.357 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 2m ago 3h 528M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:52:45.358 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 2m ago 3h 621M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:52:45.358 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 2m ago 3h 156M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:52:45.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:52:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:52:44.985Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:52:45.708 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:52:45.708 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:52:45.708 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:52:45.708 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:52:45.709 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:52:45.709 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:52:45.709 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:52:45.709 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:52:45.710 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:52:45.710 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:52:45.710 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:52:45.711 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:52:45.711 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:52:45.711 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:52:45.712 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:52:45.712 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:52:45.712 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:52:45.712 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:52:46.972 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:52:46 smithi083 ceph-mon[79930]: from='client.41271 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:52:46.973 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:52:46 smithi083 ceph-mon[79930]: from='client.51208 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:52:46.973 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:52:46 smithi083 ceph-mon[79930]: pgmap v6136: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:46.973 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:52:46 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2848427747' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:52:46.974 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:52:46 smithi083 ceph-mon[77242]: from='client.41271 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:52:46.975 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:52:46 smithi083 ceph-mon[77242]: from='client.51208 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:52:46.975 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:52:46 smithi083 ceph-mon[77242]: pgmap v6136: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:46.976 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:52:46 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2848427747' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:52:47.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:52:46 smithi153 ceph-mon[70044]: from='client.41271 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:52:47.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:52:46 smithi153 ceph-mon[70044]: from='client.51208 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:52:47.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:52:46 smithi153 ceph-mon[70044]: pgmap v6136: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:47.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:52:46 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2848427747' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:52:48.718 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:52:48 smithi083 ceph-mon[77242]: pgmap v6137: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:48.719 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:52:48 smithi083 ceph-mon[79930]: pgmap v6137: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:48.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:52:48 smithi153 ceph-mon[70044]: pgmap v6137: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:50.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:52:50 smithi083 ceph-mon[79930]: pgmap v6138: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:50.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:52:50 smithi083 ceph-mon[77242]: pgmap v6138: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:50.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:52:50 smithi153 ceph-mon[70044]: pgmap v6138: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:51.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:52:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:52:50.903Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:52:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:52:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:52:51] "GET /metrics HTTP/1.1" 200 225219 "" "Prometheus/2.33.4" 2022-09-29T08:52:52.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:52:52 smithi083 ceph-mon[79930]: pgmap v6139: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:52.719 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:52:52 smithi083 ceph-mon[77242]: pgmap v6139: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:52.813 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:52:52 smithi153 ceph-mon[70044]: pgmap v6139: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:52.814 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:52:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:52:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:52:53.313 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:52:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:52:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:52:54.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:52:54 smithi083 ceph-mon[79930]: pgmap v6140: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:54.715 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:52:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:52:54.289Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:52:54.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:52:54 smithi083 ceph-mon[77242]: pgmap v6140: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:54.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:52:54 smithi153 ceph-mon[70044]: pgmap v6140: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:52:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:52:54.985Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:52:56.719 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:52:56 smithi083 ceph-mon[79930]: pgmap v6141: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:56.721 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:52:56 smithi083 ceph-mon[77242]: pgmap v6141: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:56.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:52:56 smithi153 ceph-mon[70044]: pgmap v6141: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:58.719 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:52:58 smithi083 ceph-mon[79930]: pgmap v6142: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:58.721 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:52:58 smithi083 ceph-mon[77242]: pgmap v6142: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:52:58.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:52:58 smithi153 ceph-mon[70044]: pgmap v6142: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:00.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:53:00 smithi083 ceph-mon[79930]: pgmap v6143: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:00.727 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:53:00 smithi083 ceph-mon[77242]: pgmap v6143: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:00.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:53:00 smithi153 ceph-mon[70044]: pgmap v6143: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:01.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:53:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:53:00.903Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:53:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:53:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:53:01] "GET /metrics HTTP/1.1" 200 225217 "" "Prometheus/2.33.4" 2022-09-29T08:53:02.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:53:02 smithi083 ceph-mon[79930]: pgmap v6144: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:02.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:53:02 smithi083 ceph-mon[77242]: pgmap v6144: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:02.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:53:02 smithi153 ceph-mon[70044]: pgmap v6144: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:53:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:53:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:53:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:53:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:53:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:53:04.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:53:04 smithi083 ceph-mon[79930]: pgmap v6145: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:04.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:53:04 smithi083 ceph-mon[77242]: pgmap v6145: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:04.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:53:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:53:04.290Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:53:04.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:53:04 smithi153 ceph-mon[70044]: pgmap v6145: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:05.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:53:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:53:04.986Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:53:06.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:53:06 smithi083 ceph-mon[79930]: pgmap v6146: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:06.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:53:06 smithi083 ceph-mon[77242]: pgmap v6146: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:06.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:53:06 smithi153 ceph-mon[70044]: pgmap v6146: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:08.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:53:08 smithi083 ceph-mon[79930]: pgmap v6147: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:08.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:53:08 smithi083 ceph-mon[77242]: pgmap v6147: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:08.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:53:08 smithi153 ceph-mon[70044]: pgmap v6147: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:10.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:53:10 smithi083 ceph-mon[79930]: pgmap v6148: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:10.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:53:10 smithi083 ceph-mon[77242]: pgmap v6148: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:10.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:53:10 smithi153 ceph-mon[70044]: pgmap v6148: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:11.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:53:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:53:10.904Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:53:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:53:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:53:11] "GET /metrics HTTP/1.1" 200 225219 "" "Prometheus/2.33.4" 2022-09-29T08:53:12.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:53:12 smithi083 ceph-mon[77242]: pgmap v6149: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:12.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:53:12 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:53:12.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:53:12 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:53:12.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:53:12 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:53:12.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:53:12 smithi083 ceph-mon[79930]: pgmap v6149: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:12.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:53:12 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:53:12.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:53:12 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:53:12.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:53:12 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:53:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:53:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:53:12.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:53:12.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:53:12 smithi153 ceph-mon[70044]: pgmap v6149: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:12.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:53:12 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:53:12.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:53:12 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:53:12.818 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:53:12 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:53:13.313 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:53:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:53:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:53:13.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:53:13 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:53:13.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:53:13 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:53:14.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:53:13 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:53:14.692 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:53:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:53:14.290Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:53:14.972 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:53:14 smithi083 ceph-mon[79930]: pgmap v6150: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:14.973 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:53:14 smithi083 ceph-mon[77242]: pgmap v6150: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:15.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:53:14 smithi153 ceph-mon[70044]: pgmap v6150: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:15.470 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:53:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:53:14.987Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:53:16.031 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:53:16.327 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:53:16.328 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 5m ago 3h 26.7M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:53:16.328 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 3m ago 3h 63.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:53:16.329 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 5m ago 3h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:53:16.329 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 3m ago 3h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:53:16.329 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 3m ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:53:16.343 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 5m ago 3h 689M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:53:16.344 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 5m ago 3h 781M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:53:16.344 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 3m ago 3h 345M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:53:16.345 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 5m ago 3h 455M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:53:16.345 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 5m ago 3h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:53:16.349 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 3m ago 3h 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:53:16.350 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 5m ago 3h 564M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:53:16.362 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 5m ago 3h 555M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:53:16.362 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 5m ago 3h 437M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:53:16.370 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 5m ago 3h 540M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:53:16.373 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 3m ago 3h 479M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:53:16.374 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 3m ago 3h 592M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:53:16.374 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 3m ago 3h 528M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:53:16.375 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 3m ago 3h 621M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:53:16.375 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 3m ago 3h 156M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:53:16.716 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:53:16.717 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:53:16.718 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:53:16.718 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:53:16.719 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:53:16.720 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:53:16.720 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:53:16.721 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:53:16.721 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:53:16.722 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:53:16.736 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:53:16.737 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:53:16.737 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:53:16.750 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:53:16.751 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:53:16.751 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:53:16.752 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:53:16.752 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:53:16.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:53:16 smithi083 ceph-mon[79930]: pgmap v6151: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:16.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:53:16 smithi083 ceph-mon[77242]: pgmap v6151: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:17.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:53:16 smithi153 ceph-mon[70044]: pgmap v6151: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:17.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:53:17 smithi083 ceph-mon[79930]: from='client.51220 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:53:17.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:53:17 smithi083 ceph-mon[79930]: from='client.51226 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:53:17.970 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:53:17 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2088383224' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:53:17.971 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:53:17 smithi083 ceph-mon[77242]: from='client.51220 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:53:17.971 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:53:17 smithi083 ceph-mon[77242]: from='client.51226 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:53:17.971 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:53:17 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2088383224' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:53:18.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:53:17 smithi153 ceph-mon[70044]: from='client.51220 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:53:18.063 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:53:17 smithi153 ceph-mon[70044]: from='client.51226 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:53:18.063 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:53:17 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2088383224' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:53:18.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:53:18 smithi083 ceph-mon[79930]: pgmap v6152: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:18.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:53:18 smithi083 ceph-mon[77242]: pgmap v6152: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:19.063 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:53:18 smithi153 ceph-mon[70044]: pgmap v6152: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:20.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:53:20 smithi083 ceph-mon[79930]: pgmap v6153: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:20.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:53:20 smithi083 ceph-mon[77242]: pgmap v6153: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:20.969 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:53:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:53:20.904Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:53:21.070 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:53:20 smithi153 ceph-mon[70044]: pgmap v6153: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:21.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:53:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:53:21] "GET /metrics HTTP/1.1" 200 225219 "" "Prometheus/2.33.4" 2022-09-29T08:53:22.686 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:53:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:53:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:53:22.970 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:53:22 smithi083 ceph-mon[79930]: pgmap v6154: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:22.971 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:53:22 smithi083 ceph-mon[77242]: pgmap v6154: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:23.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:53:22 smithi153 ceph-mon[70044]: pgmap v6154: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:23.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:53:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:53:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:53:24.704 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:53:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:53:24.291Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:53:24.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:53:24 smithi083 ceph-mon[79930]: pgmap v6155: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:24.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:53:24 smithi083 ceph-mon[77242]: pgmap v6155: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:25.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:53:24 smithi153 ceph-mon[70044]: pgmap v6155: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:25.465 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:53:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:53:24.987Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:53:26.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:53:26 smithi083 ceph-mon[79930]: pgmap v6156: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:26.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:53:26 smithi083 ceph-mon[77242]: pgmap v6156: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:27.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:53:26 smithi153 ceph-mon[70044]: pgmap v6156: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:28.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:53:28 smithi083 ceph-mon[79930]: pgmap v6157: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:28.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:53:28 smithi083 ceph-mon[77242]: pgmap v6157: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:28.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:53:28 smithi153 ceph-mon[70044]: pgmap v6157: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:30.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:53:30 smithi083 ceph-mon[79930]: pgmap v6158: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:30.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:53:30 smithi083 ceph-mon[77242]: pgmap v6158: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:30.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:53:30 smithi153 ceph-mon[70044]: pgmap v6158: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:31.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:53:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:53:30.904Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:53:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:53:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:53:31] "GET /metrics HTTP/1.1" 200 225219 "" "Prometheus/2.33.4" 2022-09-29T08:53:32.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:53:32 smithi083 ceph-mon[79930]: pgmap v6159: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:32.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:53:32 smithi083 ceph-mon[77242]: pgmap v6159: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:32.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:53:32 smithi153 ceph-mon[70044]: pgmap v6159: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:32.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:53:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:53:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:53:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:53:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:53:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:53:34.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:53:34 smithi083 ceph-mon[79930]: pgmap v6160: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:34.714 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:53:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:53:34.292Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:53:34.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:53:34 smithi083 ceph-mon[77242]: pgmap v6160: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:34.824 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:53:34 smithi153 ceph-mon[70044]: pgmap v6160: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:35.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:53:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:53:34.988Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:53:36.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:53:36 smithi083 ceph-mon[77242]: pgmap v6161: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:36.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:53:36 smithi083 ceph-mon[79930]: pgmap v6161: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:36.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:53:36 smithi153 ceph-mon[70044]: pgmap v6161: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:38.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:53:38 smithi083 ceph-mon[79930]: pgmap v6162: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:38.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:53:38 smithi083 ceph-mon[77242]: pgmap v6162: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:38.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:53:38 smithi153 ceph-mon[70044]: pgmap v6162: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:53:40 smithi083 ceph-mon[79930]: pgmap v6163: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:53:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:53:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:53:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:53:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:53:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:53:40.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:53:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:53:40.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:53:40 smithi083 ceph-mon[77242]: pgmap v6163: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:40.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:53:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:53:40.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:53:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:53:40.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:53:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:53:40.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:53:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:53:40.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:53:40 smithi153 ceph-mon[70044]: pgmap v6163: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:40.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:53:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:53:40.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:53:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:53:40.813 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:53:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:53:40.813 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:53:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:53:41.217 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:53:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:53:40.905Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:53:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:53:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:53:41] "GET /metrics HTTP/1.1" 200 225213 "" "Prometheus/2.33.4" 2022-09-29T08:53:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:53:42 smithi083 ceph-mon[79930]: pgmap v6164: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:53:42 smithi083 ceph-mon[77242]: pgmap v6164: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:42.815 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:53:42 smithi153 ceph-mon[70044]: pgmap v6164: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:42.816 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:53:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:53:42.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:53:43.310 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:53:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:53:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:53:44.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:53:44 smithi083 ceph-mon[79930]: pgmap v6165: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:44.714 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:53:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:53:44.292Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:53:44.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:53:44 smithi083 ceph-mon[77242]: pgmap v6165: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:44.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:53:44 smithi153 ceph-mon[70044]: pgmap v6165: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:45.465 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:53:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:53:44.989Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:53:46.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:53:46 smithi083 ceph-mon[79930]: pgmap v6166: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:46.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:53:46 smithi083 ceph-mon[77242]: pgmap v6166: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:46.820 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:53:46 smithi153 ceph-mon[70044]: pgmap v6166: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:47.005 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:53:47.294 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:53:47.295 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 5m ago 3h 26.7M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:53:47.295 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 3m ago 3h 63.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:53:47.295 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 5m ago 3h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:53:47.296 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 3m ago 3h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:53:47.296 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 3m ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:53:47.296 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 5m ago 3h 689M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:53:47.296 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 5m ago 3h 781M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:53:47.297 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 3m ago 3h 345M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:53:47.297 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 5m ago 3h 455M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:53:47.297 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 5m ago 3h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:53:47.297 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 3m ago 3h 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:53:47.297 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 5m ago 3h 564M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:53:47.299 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 5m ago 3h 555M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:53:47.299 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 5m ago 3h 437M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:53:47.299 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 5m ago 3h 540M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:53:47.300 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 3m ago 3h 479M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:53:47.300 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 3m ago 3h 592M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:53:47.300 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 3m ago 3h 528M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:53:47.300 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 3m ago 3h 621M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:53:47.301 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 3m ago 3h 156M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:53:47.664 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:53:47.665 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:53:47.665 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:53:47.665 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:53:47.666 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:53:47.666 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:53:47.666 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:53:47.666 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:53:47.667 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:53:47.667 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:53:47.667 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:53:47.667 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:53:47.668 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:53:47.668 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:53:47.668 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:53:47.668 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:53:47.668 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:53:47.669 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:53:48.720 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:53:48 smithi083 ceph-mon[79930]: from='client.41307 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:53:48.721 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:53:48 smithi083 ceph-mon[79930]: from='client.51244 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:53:48.721 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:53:48 smithi083 ceph-mon[79930]: pgmap v6167: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:48.722 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:53:48 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1788686276' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:53:48.722 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:53:48 smithi083 ceph-mon[77242]: from='client.41307 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:53:48.723 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:53:48 smithi083 ceph-mon[77242]: from='client.51244 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:53:48.723 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:53:48 smithi083 ceph-mon[77242]: pgmap v6167: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:48.724 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:53:48 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1788686276' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:53:48.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:53:48 smithi153 ceph-mon[70044]: from='client.41307 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:53:48.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:53:48 smithi153 ceph-mon[70044]: from='client.51244 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:53:48.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:53:48 smithi153 ceph-mon[70044]: pgmap v6167: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:48.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:53:48 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1788686276' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:53:50.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:53:50 smithi083 ceph-mon[79930]: pgmap v6168: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:50.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:53:50 smithi083 ceph-mon[77242]: pgmap v6168: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:50.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:53:50 smithi153 ceph-mon[70044]: pgmap v6168: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:51.215 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:53:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:53:50.905Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:53:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:53:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:53:51] "GET /metrics HTTP/1.1" 200 225213 "" "Prometheus/2.33.4" 2022-09-29T08:53:52.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:53:52 smithi083 ceph-mon[79930]: pgmap v6169: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:52.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:53:52 smithi083 ceph-mon[77242]: pgmap v6169: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:52.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:53:52 smithi153 ceph-mon[70044]: pgmap v6169: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:52.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:53:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:53:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:53:53.311 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:53:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:53:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:53:54.718 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:53:54 smithi083 ceph-mon[79930]: pgmap v6170: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:54.719 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:53:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:53:54.292Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:53:54.720 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:53:54 smithi083 ceph-mon[77242]: pgmap v6170: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:54.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:53:54 smithi153 ceph-mon[70044]: pgmap v6170: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:53:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:53:54.989Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:53:56.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:53:56 smithi083 ceph-mon[77242]: pgmap v6171: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:56.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:53:56 smithi083 ceph-mon[79930]: pgmap v6171: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:56.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:53:56 smithi153 ceph-mon[70044]: pgmap v6171: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:58.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:53:58 smithi083 ceph-mon[79930]: pgmap v6172: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:58.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:53:58 smithi083 ceph-mon[77242]: pgmap v6172: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:53:58.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:53:58 smithi153 ceph-mon[70044]: pgmap v6172: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:00.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:54:00 smithi083 ceph-mon[77242]: pgmap v6173: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:00.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:54:00 smithi083 ceph-mon[79930]: pgmap v6173: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:00.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:54:00 smithi153 ceph-mon[70044]: pgmap v6173: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:01.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:54:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:54:00.906Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:54:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:54:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:54:01] "GET /metrics HTTP/1.1" 200 225218 "" "Prometheus/2.33.4" 2022-09-29T08:54:02.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:54:02 smithi083 ceph-mon[79930]: pgmap v6174: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:02.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:54:02 smithi083 ceph-mon[77242]: pgmap v6174: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:02.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:54:02 smithi153 ceph-mon[70044]: pgmap v6174: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:54:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:54:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:54:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:54:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:54:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:54:04.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:54:04 smithi083 ceph-mon[79930]: pgmap v6175: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:04.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:54:04 smithi083 ceph-mon[77242]: pgmap v6175: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:04.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:54:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:54:04.293Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:54:04.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:54:04 smithi153 ceph-mon[70044]: pgmap v6175: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:05.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:54:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:54:04.990Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:54:06.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:54:06 smithi083 ceph-mon[79930]: pgmap v6176: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:06.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:54:06 smithi083 ceph-mon[77242]: pgmap v6176: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:06.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:54:06 smithi153 ceph-mon[70044]: pgmap v6176: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:08.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:54:08 smithi083 ceph-mon[79930]: pgmap v6177: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:08.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:54:08 smithi083 ceph-mon[77242]: pgmap v6177: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:08.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:54:08 smithi153 ceph-mon[70044]: pgmap v6177: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:10.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:54:10 smithi083 ceph-mon[79930]: pgmap v6178: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:10.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:54:10 smithi083 ceph-mon[77242]: pgmap v6178: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:10.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:54:10 smithi153 ceph-mon[70044]: pgmap v6178: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:11.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:54:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:54:10.906Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:54:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:54:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:54:11] "GET /metrics HTTP/1.1" 200 225217 "" "Prometheus/2.33.4" 2022-09-29T08:54:12.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:54:12 smithi083 ceph-mon[79930]: pgmap v6179: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:12.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:54:12 smithi083 ceph-mon[77242]: pgmap v6179: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:12.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:54:12 smithi153 ceph-mon[70044]: pgmap v6179: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:54:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:54:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:54:13.313 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:54:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:54:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:54:13.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:54:13 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:54:13.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:54:13 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:54:13.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:54:13 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:54:13.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:54:13 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:54:13.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:54:13 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:54:13.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:54:13 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:54:13.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:54:13 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:54:13.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:54:13 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:54:13.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:54:13 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:54:13.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:54:13 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:54:13.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:54:13 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:54:13.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:54:13 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:54:14.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:54:14 smithi083 ceph-mon[79930]: pgmap v6180: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:14.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:54:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:54:14.293Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:54:14.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:54:14 smithi083 ceph-mon[77242]: pgmap v6180: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:14.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:54:14 smithi153 ceph-mon[70044]: pgmap v6180: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:15.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:54:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:54:14.991Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:54:16.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:54:16 smithi083 ceph-mon[79930]: pgmap v6181: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:16.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:54:16 smithi083 ceph-mon[77242]: pgmap v6181: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:16.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:54:16 smithi153 ceph-mon[70044]: pgmap v6181: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:17.974 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:54:18.262 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:54:18.263 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 6m ago 3h 26.7M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:54:18.263 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 4m ago 3h 63.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:54:18.264 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 6m ago 3h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:54:18.264 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 4m ago 3h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:54:18.265 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 4m ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:54:18.265 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 6m ago 3h 689M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:54:18.267 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 6m ago 3h 781M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:54:18.268 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 4m ago 3h 345M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:54:18.268 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 6m ago 3h 455M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:54:18.268 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 6m ago 3h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:54:18.269 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 4m ago 3h 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:54:18.269 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 6m ago 3h 564M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:54:18.270 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 6m ago 3h 555M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:54:18.270 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 6m ago 3h 437M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:54:18.271 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 6m ago 3h 540M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:54:18.271 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 4m ago 3h 479M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:54:18.271 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 4m ago 3h 592M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:54:18.274 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 4m ago 3h 528M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:54:18.274 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 4m ago 3h 621M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:54:18.275 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 4m ago 3h 156M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:54:18.619 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:54:18.620 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:54:18.620 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:54:18.620 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:54:18.621 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:54:18.621 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:54:18.621 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:54:18.621 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:54:18.621 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:54:18.622 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:54:18.622 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:54:18.622 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:54:18.622 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:54:18.623 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:54:18.623 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:54:18.624 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:54:18.624 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:54:18.625 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:54:18.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:54:18 smithi083 ceph-mon[79930]: pgmap v6182: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:18.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:54:18 smithi083 ceph-mon[77242]: pgmap v6182: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:18.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:54:18 smithi153 ceph-mon[70044]: pgmap v6182: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:19.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:54:19 smithi083 ceph-mon[79930]: from='client.41325 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:54:19.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:54:19 smithi083 ceph-mon[79930]: from='client.41331 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:54:19.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:54:19 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/159618079' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:54:19.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:54:19 smithi083 ceph-mon[77242]: from='client.41325 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:54:19.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:54:19 smithi083 ceph-mon[77242]: from='client.41331 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:54:19.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:54:19 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/159618079' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:54:19.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:54:19 smithi153 ceph-mon[70044]: from='client.41325 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:54:19.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:54:19 smithi153 ceph-mon[70044]: from='client.41331 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:54:19.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:54:19 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/159618079' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:54:20.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:54:20 smithi083 ceph-mon[79930]: pgmap v6183: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:20.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:54:20 smithi083 ceph-mon[77242]: pgmap v6183: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:20.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:54:20 smithi153 ceph-mon[70044]: pgmap v6183: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:21.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:54:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:54:20.907Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:54:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:54:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:54:21] "GET /metrics HTTP/1.1" 200 225217 "" "Prometheus/2.33.4" 2022-09-29T08:54:22.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:54:22 smithi083 ceph-mon[77242]: pgmap v6184: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:22.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:54:22 smithi083 ceph-mon[79930]: pgmap v6184: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:22.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:54:22 smithi153 ceph-mon[70044]: pgmap v6184: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:22.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:54:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:54:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:54:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:54:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:54:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:54:24.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:54:24 smithi083 ceph-mon[77242]: pgmap v6185: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:24.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:54:24 smithi083 ceph-mon[79930]: pgmap v6185: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:24.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:54:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:54:24.294Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:54:24.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:54:24 smithi153 ceph-mon[70044]: pgmap v6185: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:25.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:54:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:54:24.992Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:54:26.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:54:26 smithi083 ceph-mon[79930]: pgmap v6186: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:26.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:54:26 smithi083 ceph-mon[77242]: pgmap v6186: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:26.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:54:26 smithi153 ceph-mon[70044]: pgmap v6186: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:28.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:54:28 smithi083 ceph-mon[79930]: pgmap v6187: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:28.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:54:28 smithi083 ceph-mon[77242]: pgmap v6187: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:28.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:54:28 smithi153 ceph-mon[70044]: pgmap v6187: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:30.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:54:30 smithi083 ceph-mon[77242]: pgmap v6188: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:30.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:54:30 smithi083 ceph-mon[79930]: pgmap v6188: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:30.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:54:30 smithi153 ceph-mon[70044]: pgmap v6188: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:31.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:54:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:54:30.908Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:54:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:54:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:54:31] "GET /metrics HTTP/1.1" 200 225220 "" "Prometheus/2.33.4" 2022-09-29T08:54:32.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:54:32 smithi083 ceph-mon[79930]: pgmap v6189: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:32.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:54:32 smithi083 ceph-mon[77242]: pgmap v6189: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:32.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:54:32 smithi153 ceph-mon[70044]: pgmap v6189: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:32.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:54:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:54:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:54:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:54:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:54:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:54:34.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:54:34 smithi083 ceph-mon[77242]: pgmap v6190: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:34.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:54:34 smithi083 ceph-mon[79930]: pgmap v6190: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:34.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:54:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:54:34.296Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:54:34.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:54:34 smithi153 ceph-mon[70044]: pgmap v6190: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:35.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:54:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:54:34.993Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:54:36.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:54:36 smithi083 ceph-mon[79930]: pgmap v6191: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:36.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:54:36 smithi083 ceph-mon[77242]: pgmap v6191: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:36.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:54:36 smithi153 ceph-mon[70044]: pgmap v6191: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:38.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:54:38 smithi083 ceph-mon[79930]: pgmap v6192: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:38.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:54:38 smithi083 ceph-mon[77242]: pgmap v6192: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:38.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:54:38 smithi153 ceph-mon[70044]: pgmap v6192: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:54:40 smithi083 ceph-mon[79930]: pgmap v6193: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:54:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:54:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:54:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:54:40.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:54:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:54:40.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:54:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:54:40.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:54:40 smithi083 ceph-mon[77242]: pgmap v6193: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:40.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:54:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:54:40.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:54:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:54:40.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:54:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:54:40.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:54:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:54:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:54:40 smithi153 ceph-mon[70044]: pgmap v6193: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:54:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:54:40.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:54:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:54:40.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:54:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:54:40.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:54:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:54:41.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:54:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:54:40.909Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:54:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:54:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:54:41] "GET /metrics HTTP/1.1" 200 225213 "" "Prometheus/2.33.4" 2022-09-29T08:54:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:54:42 smithi083 ceph-mon[79930]: pgmap v6194: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:42.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:54:42 smithi083 ceph-mon[77242]: pgmap v6194: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:54:42 smithi153 ceph-mon[70044]: pgmap v6194: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:42.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:54:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:54:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:54:43.314 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:54:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:54:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:54:44.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:54:44 smithi083 ceph-mon[79930]: pgmap v6195: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:54:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:54:44.296Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:54:44.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:54:44 smithi083 ceph-mon[77242]: pgmap v6195: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:44.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:54:44 smithi153 ceph-mon[70044]: pgmap v6195: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:45.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:54:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:54:44.994Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:54:46.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:54:46 smithi083 ceph-mon[79930]: pgmap v6196: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:46.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:54:46 smithi083 ceph-mon[77242]: pgmap v6196: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:46.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:54:46 smithi153 ceph-mon[70044]: pgmap v6196: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:48.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:54:48 smithi083 ceph-mon[79930]: pgmap v6197: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:48.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:54:48 smithi083 ceph-mon[77242]: pgmap v6197: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:48.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:54:48 smithi153 ceph-mon[70044]: pgmap v6197: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:48.947 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:54:49.237 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:54:49.237 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 6m ago 3h 26.7M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:54:49.237 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 4m ago 3h 63.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:54:49.237 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 6m ago 3h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:54:49.238 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 4m ago 3h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:54:49.238 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 4m ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:54:49.238 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 6m ago 3h 689M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:54:49.239 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 6m ago 3h 781M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:54:49.239 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 4m ago 3h 345M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:54:49.239 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 6m ago 3h 455M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:54:49.239 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 6m ago 3h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:54:49.239 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 4m ago 3h 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:54:49.240 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 6m ago 3h 564M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:54:49.240 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 6m ago 3h 555M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:54:49.240 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 6m ago 3h 437M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:54:49.240 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 6m ago 3h 540M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:54:49.241 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 4m ago 3h 479M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:54:49.241 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 4m ago 3h 592M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:54:49.241 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 4m ago 3h 528M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:54:49.241 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 4m ago 3h 621M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:54:49.242 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 4m ago 3h 156M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:54:49.598 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:54:49.598 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:54:49.599 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:54:49.599 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:54:49.600 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:54:49.600 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:54:49.601 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:54:49.601 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:54:49.601 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:54:49.602 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:54:49.602 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:54:49.602 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:54:49.603 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:54:49.603 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:54:49.603 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:54:49.604 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:54:49.604 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:54:49.604 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:54:50.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:54:50 smithi083 ceph-mon[79930]: from='client.51274 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:54:50.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:54:50 smithi083 ceph-mon[79930]: from='client.41349 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:54:50.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:54:50 smithi083 ceph-mon[79930]: pgmap v6198: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:50.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:54:50 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2206280528' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:54:50.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:54:50 smithi083 ceph-mon[77242]: from='client.51274 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:54:50.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:54:50 smithi083 ceph-mon[77242]: from='client.41349 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:54:50.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:54:50 smithi083 ceph-mon[77242]: pgmap v6198: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:50.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:54:50 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2206280528' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:54:50.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:54:50 smithi153 ceph-mon[70044]: from='client.51274 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:54:50.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:54:50 smithi153 ceph-mon[70044]: from='client.41349 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:54:50.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:54:50 smithi153 ceph-mon[70044]: pgmap v6198: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:50.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:54:50 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2206280528' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:54:51.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:54:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:54:50.909Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:54:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:54:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:54:51] "GET /metrics HTTP/1.1" 200 225213 "" "Prometheus/2.33.4" 2022-09-29T08:54:52.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:54:52 smithi083 ceph-mon[79930]: pgmap v6199: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:52.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:54:52 smithi083 ceph-mon[77242]: pgmap v6199: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:52.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:54:52 smithi153 ceph-mon[70044]: pgmap v6199: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:54:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:54:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:54:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:54:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:54:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:54:54.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:54:54 smithi083 ceph-mon[77242]: pgmap v6200: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:54.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:54:54 smithi083 ceph-mon[79930]: pgmap v6200: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:54.715 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:54:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:54:54.297Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:54:54.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:54:54 smithi153 ceph-mon[70044]: pgmap v6200: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:54:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:54:54.994Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:54:56.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:54:56 smithi083 ceph-mon[79930]: pgmap v6201: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:56.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:54:56 smithi083 ceph-mon[77242]: pgmap v6201: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:56.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:54:56 smithi153 ceph-mon[70044]: pgmap v6201: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:58.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:54:58 smithi083 ceph-mon[79930]: pgmap v6202: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:58.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:54:58 smithi083 ceph-mon[77242]: pgmap v6202: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:54:58.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:54:58 smithi153 ceph-mon[70044]: pgmap v6202: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:00.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:55:00 smithi083 ceph-mon[79930]: pgmap v6203: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:00.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:55:00 smithi083 ceph-mon[77242]: pgmap v6203: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:00.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:55:00 smithi153 ceph-mon[70044]: pgmap v6203: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:01.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:55:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:55:00.910Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:55:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:55:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:55:01] "GET /metrics HTTP/1.1" 200 225220 "" "Prometheus/2.33.4" 2022-09-29T08:55:02.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:55:02 smithi083 ceph-mon[77242]: pgmap v6204: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:02.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:55:02 smithi083 ceph-mon[79930]: pgmap v6204: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:02.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:55:02 smithi153 ceph-mon[70044]: pgmap v6204: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:55:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:55:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:55:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:55:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:55:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:55:04.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:55:04 smithi083 ceph-mon[77242]: pgmap v6205: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:04.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:55:04 smithi083 ceph-mon[79930]: pgmap v6205: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:04.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:55:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:55:04.298Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:55:04.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:55:04 smithi153 ceph-mon[70044]: pgmap v6205: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:05.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:55:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:55:04.995Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:55:06.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:55:06 smithi083 ceph-mon[79930]: pgmap v6206: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:06.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:55:06 smithi083 ceph-mon[77242]: pgmap v6206: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:06.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:55:06 smithi153 ceph-mon[70044]: pgmap v6206: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:08.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:55:08 smithi083 ceph-mon[79930]: pgmap v6207: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:08.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:55:08 smithi083 ceph-mon[77242]: pgmap v6207: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:08.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:55:08 smithi153 ceph-mon[70044]: pgmap v6207: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:10.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:55:10 smithi083 ceph-mon[79930]: pgmap v6208: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:10.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:55:10 smithi083 ceph-mon[77242]: pgmap v6208: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:10.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:55:10 smithi153 ceph-mon[70044]: pgmap v6208: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:11.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:55:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:55:10.911Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:55:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:55:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:55:11] "GET /metrics HTTP/1.1" 200 225218 "" "Prometheus/2.33.4" 2022-09-29T08:55:12.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:55:12 smithi083 ceph-mon[79930]: pgmap v6209: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:12.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:55:12 smithi083 ceph-mon[77242]: pgmap v6209: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:12.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:55:12 smithi153 ceph-mon[70044]: pgmap v6209: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:55:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:55:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:55:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:55:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:55:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:55:13.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:55:13 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:55:13.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:55:13 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:55:13.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:55:13 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:55:13.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:55:13 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:55:13.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:55:13 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:55:13.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:55:13 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:55:13.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:55:13 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:55:13.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:55:13 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:55:13.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:55:13 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:55:14.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:55:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:55:14.297Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:55:14.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:55:14 smithi083 ceph-mon[77242]: pgmap v6210: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:14.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:55:14 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:55:14.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:55:14 smithi083 ceph-mon[79930]: pgmap v6210: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:14.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:55:14 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:55:14.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:55:14 smithi153 ceph-mon[70044]: pgmap v6210: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:14.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:55:14 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:55:15.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:55:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:55:14.995Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:55:16.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:55:16 smithi083 ceph-mon[79930]: pgmap v6211: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:16.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:55:16 smithi083 ceph-mon[77242]: pgmap v6211: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:16.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:55:16 smithi153 ceph-mon[70044]: pgmap v6211: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:18.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:55:18 smithi083 ceph-mon[79930]: pgmap v6212: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:18.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:55:18 smithi083 ceph-mon[77242]: pgmap v6212: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:18.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:55:18 smithi153 ceph-mon[70044]: pgmap v6212: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:19.927 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:55:20.217 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:55:20.218 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 7m ago 3h 26.7M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:55:20.218 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 5m ago 3h 63.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:55:20.218 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 7m ago 3h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:55:20.219 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 5m ago 3h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:55:20.219 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 5m ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:55:20.219 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 7m ago 3h 689M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:55:20.219 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 7m ago 3h 781M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:55:20.219 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 5m ago 3h 345M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:55:20.220 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 7m ago 3h 455M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:55:20.220 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 7m ago 3h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:55:20.220 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 5m ago 3h 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:55:20.220 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 7m ago 3h 564M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:55:20.221 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 7m ago 3h 555M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:55:20.221 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 7m ago 3h 437M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:55:20.221 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 7m ago 3h 540M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:55:20.221 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 5m ago 3h 479M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:55:20.222 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 5m ago 3h 592M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:55:20.222 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 5m ago 3h 528M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:55:20.222 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 5m ago 3h 621M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:55:20.222 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 5m ago 3h 156M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:55:20.574 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:55:20.574 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:55:20.574 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:55:20.574 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:55:20.575 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:55:20.575 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:55:20.575 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:55:20.575 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:55:20.576 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:55:20.576 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:55:20.576 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:55:20.576 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:55:20.577 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:55:20.577 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:55:20.577 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:55:20.577 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:55:20.577 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:55:20.578 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:55:20.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:55:20 smithi083 ceph-mon[77242]: pgmap v6213: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:20.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:55:20 smithi083 ceph-mon[79930]: pgmap v6213: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:20.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:55:20 smithi153 ceph-mon[70044]: pgmap v6213: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:21.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:55:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:55:20.911Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:55:21.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:55:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:55:21] "GET /metrics HTTP/1.1" 200 225218 "" "Prometheus/2.33.4" 2022-09-29T08:55:21.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:55:21 smithi083 ceph-mon[77242]: from='client.51292 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:55:21.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:55:21 smithi083 ceph-mon[77242]: from='client.51298 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:55:21.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:55:21 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1698743523' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:55:21.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:55:21 smithi083 ceph-mon[79930]: from='client.51292 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:55:21.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:55:21 smithi083 ceph-mon[79930]: from='client.51298 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:55:21.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:55:21 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1698743523' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:55:21.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:55:21 smithi153 ceph-mon[70044]: from='client.51292 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:55:21.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:55:21 smithi153 ceph-mon[70044]: from='client.51298 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:55:21.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:55:21 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1698743523' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:55:22.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:55:22 smithi083 ceph-mon[79930]: pgmap v6214: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:22.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:55:22 smithi083 ceph-mon[77242]: pgmap v6214: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:22.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:55:22 smithi153 ceph-mon[70044]: pgmap v6214: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:55:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:55:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:55:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:55:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:55:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:55:24.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:55:24 smithi083 ceph-mon[79930]: pgmap v6215: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:55:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:55:24.298Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:55:24.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:55:24 smithi083 ceph-mon[77242]: pgmap v6215: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:24.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:55:24 smithi153 ceph-mon[70044]: pgmap v6215: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:25.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:55:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:55:24.996Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:55:26.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:55:26 smithi083 ceph-mon[79930]: pgmap v6216: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:26.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:55:26 smithi083 ceph-mon[77242]: pgmap v6216: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:26.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:55:26 smithi153 ceph-mon[70044]: pgmap v6216: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:28.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:55:28 smithi083 ceph-mon[79930]: pgmap v6217: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:28.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:55:28 smithi083 ceph-mon[77242]: pgmap v6217: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:28.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:55:28 smithi153 ceph-mon[70044]: pgmap v6217: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:30.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:55:30 smithi083 ceph-mon[79930]: pgmap v6218: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:30.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:55:30 smithi083 ceph-mon[77242]: pgmap v6218: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:30.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:55:30 smithi153 ceph-mon[70044]: pgmap v6218: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:31.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:55:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:55:30.913Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:55:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:55:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:55:31] "GET /metrics HTTP/1.1" 200 225217 "" "Prometheus/2.33.4" 2022-09-29T08:55:32.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:55:32 smithi083 ceph-mon[79930]: pgmap v6219: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:32.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:55:32 smithi083 ceph-mon[77242]: pgmap v6219: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:32.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:55:32 smithi153 ceph-mon[70044]: pgmap v6219: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:55:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:55:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:55:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:55:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:55:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:55:34.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:55:34 smithi083 ceph-mon[79930]: pgmap v6220: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:34.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:55:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:55:34.298Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:55:34.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:55:34 smithi083 ceph-mon[77242]: pgmap v6220: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:34.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:55:34 smithi153 ceph-mon[70044]: pgmap v6220: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:35.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:55:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:55:34.997Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:55:36.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:55:36 smithi083 ceph-mon[79930]: pgmap v6221: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:36.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:55:36 smithi083 ceph-mon[77242]: pgmap v6221: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:36.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:55:36 smithi153 ceph-mon[70044]: pgmap v6221: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:38.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:55:38 smithi083 ceph-mon[79930]: pgmap v6222: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:38.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:55:38 smithi083 ceph-mon[77242]: pgmap v6222: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:38.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:55:38 smithi153 ceph-mon[70044]: pgmap v6222: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:55:40 smithi083 ceph-mon[79930]: pgmap v6223: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:55:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:55:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:55:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:55:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:55:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:55:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:55:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:55:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:55:40 smithi083 ceph-mon[77242]: pgmap v6223: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:40.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:55:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:55:40.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:55:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:55:40.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:55:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:55:40.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:55:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:55:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:55:40 smithi153 ceph-mon[70044]: pgmap v6223: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:55:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:55:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:55:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:55:40.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:55:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:55:40.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:55:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:55:41.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:55:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:55:40.914Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:55:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:55:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:55:41] "GET /metrics HTTP/1.1" 200 225219 "" "Prometheus/2.33.4" 2022-09-29T08:55:42.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:55:42 smithi083 ceph-mon[77242]: pgmap v6224: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:55:42 smithi083 ceph-mon[79930]: pgmap v6224: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:55:42 smithi153 ceph-mon[70044]: pgmap v6224: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:55:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:55:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:55:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:55:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:55:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:55:44.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:55:44 smithi083 ceph-mon[79930]: pgmap v6225: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:55:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:55:44.299Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:55:44.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:55:44 smithi083 ceph-mon[77242]: pgmap v6225: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:44.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:55:44 smithi153 ceph-mon[70044]: pgmap v6225: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:45.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:55:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:55:44.998Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:55:46.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:55:46 smithi083 ceph-mon[79930]: pgmap v6226: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:46.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:55:46 smithi083 ceph-mon[77242]: pgmap v6226: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:46.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:55:46 smithi153 ceph-mon[70044]: pgmap v6226: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:48.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:55:48 smithi083 ceph-mon[77242]: pgmap v6227: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:48.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:55:48 smithi083 ceph-mon[79930]: pgmap v6227: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:48.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:55:48 smithi153 ceph-mon[70044]: pgmap v6227: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:50.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:55:50 smithi083 ceph-mon[79930]: pgmap v6228: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:50.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:55:50 smithi083 ceph-mon[77242]: pgmap v6228: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:50.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:55:50 smithi153 ceph-mon[70044]: pgmap v6228: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:50.902 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:55:51.191 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:55:51.191 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 7m ago 3h 26.7M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:55:51.191 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 5m ago 3h 63.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:55:51.191 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 7m ago 3h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:55:51.192 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 5m ago 3h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:55:51.192 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 5m ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:55:51.192 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 7m ago 3h 689M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:55:51.192 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 7m ago 3h 781M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:55:51.193 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 5m ago 3h 345M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:55:51.193 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 7m ago 3h 455M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:55:51.193 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 7m ago 3h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:55:51.193 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 5m ago 3h 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:55:51.193 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 7m ago 3h 564M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:55:51.194 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 7m ago 3h 555M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:55:51.194 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 7m ago 3h 437M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:55:51.194 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 7m ago 3h 540M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:55:51.194 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 5m ago 3h 479M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:55:51.195 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 5m ago 3h 592M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:55:51.195 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 5m ago 3h 528M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:55:51.195 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 5m ago 3h 621M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:55:51.195 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 5m ago 3h 156M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:55:51.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:55:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:55:50.914Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:55:51.548 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:55:51.548 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:55:51.549 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:55:51.549 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:55:51.549 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:55:51.549 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:55:51.550 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:55:51.550 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:55:51.550 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:55:51.550 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:55:51.551 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:55:51.551 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:55:51.551 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:55:51.551 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:55:51.551 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:55:51.552 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:55:51.552 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:55:51.552 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:55:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:55:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:55:51] "GET /metrics HTTP/1.1" 200 225219 "" "Prometheus/2.33.4" 2022-09-29T08:55:52.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:55:52 smithi083 ceph-mon[79930]: from='client.51310 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:55:52.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:55:52 smithi083 ceph-mon[79930]: from='client.41385 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:55:52.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:55:52 smithi083 ceph-mon[79930]: pgmap v6229: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:52.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:55:52 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2757928172' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:55:52.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:55:52 smithi083 ceph-mon[77242]: from='client.51310 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:55:52.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:55:52 smithi083 ceph-mon[77242]: from='client.41385 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:55:52.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:55:52 smithi083 ceph-mon[77242]: pgmap v6229: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:52.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:55:52 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2757928172' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:55:52.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:55:52 smithi153 ceph-mon[70044]: from='client.51310 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:55:52.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:55:52 smithi153 ceph-mon[70044]: from='client.41385 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:55:52.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:55:52 smithi153 ceph-mon[70044]: pgmap v6229: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:52.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:55:52 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2757928172' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:55:52.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:55:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:55:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:55:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:55:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:55:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:55:54.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:55:54 smithi083 ceph-mon[79930]: pgmap v6230: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:55:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:55:54.299Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:55:54.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:55:54 smithi083 ceph-mon[77242]: pgmap v6230: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:54.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:55:54 smithi153 ceph-mon[70044]: pgmap v6230: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:55.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:55:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:55:54.998Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:55:56.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:55:56 smithi153 ceph-mon[70044]: pgmap v6231: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:56.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:55:56 smithi083 ceph-mon[79930]: pgmap v6231: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:56.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:55:56 smithi083 ceph-mon[77242]: pgmap v6231: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:58.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:55:58 smithi083 ceph-mon[79930]: pgmap v6232: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:58.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:55:58 smithi083 ceph-mon[77242]: pgmap v6232: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:55:58.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:55:58 smithi153 ceph-mon[70044]: pgmap v6232: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:00.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:56:00 smithi083 ceph-mon[79930]: pgmap v6233: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:00.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:56:00 smithi083 ceph-mon[77242]: pgmap v6233: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:00.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:56:00 smithi153 ceph-mon[70044]: pgmap v6233: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:01.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:56:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:56:00.915Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:56:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:56:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:56:01] "GET /metrics HTTP/1.1" 200 225218 "" "Prometheus/2.33.4" 2022-09-29T08:56:02.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:56:02 smithi083 ceph-mon[77242]: pgmap v6234: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:02.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:56:02 smithi083 ceph-mon[79930]: pgmap v6234: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:02.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:56:02 smithi153 ceph-mon[70044]: pgmap v6234: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:56:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:56:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:56:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:56:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:56:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:56:04.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:56:04 smithi083 ceph-mon[79930]: pgmap v6235: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:04.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:56:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:56:04.300Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:56:04.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:56:04 smithi083 ceph-mon[77242]: pgmap v6235: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:04.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:56:04 smithi153 ceph-mon[70044]: pgmap v6235: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:05.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:56:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:56:04.999Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:56:06.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:56:06 smithi153 ceph-mon[70044]: pgmap v6236: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:06.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:56:06 smithi083 ceph-mon[79930]: pgmap v6236: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:06.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:56:06 smithi083 ceph-mon[77242]: pgmap v6236: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:08.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:56:08 smithi083 ceph-mon[77242]: pgmap v6237: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:08.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:56:08 smithi083 ceph-mon[79930]: pgmap v6237: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:08.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:56:08 smithi153 ceph-mon[70044]: pgmap v6237: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:10.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:56:10 smithi083 ceph-mon[79930]: pgmap v6238: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:10.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:56:10 smithi083 ceph-mon[77242]: pgmap v6238: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:10.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:56:10 smithi153 ceph-mon[70044]: pgmap v6238: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:11.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:56:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:56:10.915Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:56:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:56:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:56:11] "GET /metrics HTTP/1.1" 200 225214 "" "Prometheus/2.33.4" 2022-09-29T08:56:12.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:56:12 smithi153 ceph-mon[70044]: pgmap v6239: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:12.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:56:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:56:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:56:12.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:56:12 smithi083 ceph-mon[79930]: pgmap v6239: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:12.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:56:12 smithi083 ceph-mon[77242]: pgmap v6239: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:56:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:56:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:56:14.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:56:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:56:14.301Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:56:14.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:56:14 smithi083 ceph-mon[79930]: pgmap v6240: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:14.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:56:14 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:56:14.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:56:14 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:56:14.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:56:14 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:56:14.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:56:14 smithi083 ceph-mon[77242]: pgmap v6240: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:14.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:56:14 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:56:14.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:56:14 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:56:14.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:56:14 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:56:14.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:56:14 smithi153 ceph-mon[70044]: pgmap v6240: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:14.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:56:14 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:56:14.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:56:14 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:56:14.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:56:14 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:56:15.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:56:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:56:14.999Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:56:15.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:56:15 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:56:15.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:56:15 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:56:16.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:56:15 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:56:16.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:56:16 smithi083 ceph-mon[79930]: pgmap v6241: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:16.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:56:16 smithi083 ceph-mon[77242]: pgmap v6241: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:17.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:56:16 smithi153 ceph-mon[70044]: pgmap v6241: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:18.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:56:18 smithi083 ceph-mon[79930]: pgmap v6242: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:18.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:56:18 smithi083 ceph-mon[77242]: pgmap v6242: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:18.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:56:18 smithi153 ceph-mon[70044]: pgmap v6242: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:20.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:56:20 smithi153 ceph-mon[70044]: pgmap v6243: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:20.918 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:56:20 smithi083 ceph-mon[79930]: pgmap v6243: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:20.919 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:56:20 smithi083 ceph-mon[77242]: pgmap v6243: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:21.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:56:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:56:20.917Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:56:21.873 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:56:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:56:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:56:21] "GET /metrics HTTP/1.1" 200 225214 "" "Prometheus/2.33.4" 2022-09-29T08:56:22.159 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:56:22.159 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 8m ago 3h 26.7M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:56:22.160 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 6m ago 3h 63.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:56:22.160 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 8m ago 3h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:56:22.160 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 6m ago 3h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:56:22.160 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 6m ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:56:22.161 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 8m ago 3h 689M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:56:22.161 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 8m ago 3h 781M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:56:22.161 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 6m ago 3h 345M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:56:22.161 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 8m ago 3h 455M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:56:22.161 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 8m ago 3h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:56:22.162 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 6m ago 3h 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:56:22.162 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 8m ago 3h 564M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:56:22.162 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 8m ago 3h 555M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:56:22.162 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 8m ago 3h 437M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:56:22.162 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 8m ago 3h 540M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:56:22.163 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 6m ago 3h 479M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:56:22.163 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 6m ago 3h 592M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:56:22.163 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 6m ago 3h 528M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:56:22.163 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 6m ago 3h 621M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:56:22.164 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 6m ago 3h 156M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:56:22.519 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:56:22.520 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:56:22.520 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:56:22.520 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:56:22.520 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:56:22.521 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:56:22.521 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:56:22.521 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:56:22.521 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:56:22.521 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:56:22.522 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:56:22.522 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:56:22.522 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:56:22.523 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:56:22.523 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:56:22.523 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:56:22.523 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:56:22.524 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:56:22.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:56:22 smithi153 ceph-mon[70044]: pgmap v6244: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:22.839 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:56:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:56:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:56:22.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:56:22 smithi083 ceph-mon[79930]: pgmap v6244: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:22.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:56:22 smithi083 ceph-mon[77242]: pgmap v6244: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:56:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:56:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:56:23.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:56:23 smithi153 ceph-mon[70044]: from='client.51328 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:56:23.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:56:23 smithi153 ceph-mon[70044]: from='client.51334 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:56:23.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:56:23 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/393668572' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:56:23.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:56:23 smithi083 ceph-mon[79930]: from='client.51328 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:56:23.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:56:23 smithi083 ceph-mon[79930]: from='client.51334 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:56:23.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:56:23 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/393668572' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:56:23.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:56:23 smithi083 ceph-mon[77242]: from='client.51328 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:56:23.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:56:23 smithi083 ceph-mon[77242]: from='client.51334 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:56:23.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:56:23 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/393668572' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:56:24.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:56:24 smithi083 ceph-mon[79930]: pgmap v6245: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:56:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:56:24.301Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:56:24.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:56:24 smithi083 ceph-mon[77242]: pgmap v6245: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:24.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:56:24 smithi153 ceph-mon[70044]: pgmap v6245: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:25.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:56:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:56:25.000Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:56:26.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:56:26 smithi153 ceph-mon[70044]: pgmap v6246: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:26.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:56:26 smithi083 ceph-mon[79930]: pgmap v6246: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:26.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:56:26 smithi083 ceph-mon[77242]: pgmap v6246: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:28.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:56:28 smithi083 ceph-mon[77242]: pgmap v6247: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:28.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:56:28 smithi083 ceph-mon[79930]: pgmap v6247: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:28.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:56:28 smithi153 ceph-mon[70044]: pgmap v6247: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:30.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:56:30 smithi153 ceph-mon[70044]: pgmap v6248: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:30.918 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:56:30 smithi083 ceph-mon[79930]: pgmap v6248: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:30.919 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:56:30 smithi083 ceph-mon[77242]: pgmap v6248: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:31.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:56:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:56:30.917Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:56:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:56:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:56:31] "GET /metrics HTTP/1.1" 200 225217 "" "Prometheus/2.33.4" 2022-09-29T08:56:32.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:56:32 smithi153 ceph-mon[70044]: pgmap v6249: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:56:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:56:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:56:32.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:56:32 smithi083 ceph-mon[79930]: pgmap v6249: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:32.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:56:32 smithi083 ceph-mon[77242]: pgmap v6249: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:56:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:56:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:56:34.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:56:34 smithi083 ceph-mon[77242]: pgmap v6250: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:34.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:56:34 smithi083 ceph-mon[79930]: pgmap v6250: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:34.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:56:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:56:34.302Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:56:34.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:56:34 smithi153 ceph-mon[70044]: pgmap v6250: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:35.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:56:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:56:35.000Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:56:36.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:56:36 smithi153 ceph-mon[70044]: pgmap v6251: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:36.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:56:36 smithi083 ceph-mon[77242]: pgmap v6251: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:36.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:56:36 smithi083 ceph-mon[79930]: pgmap v6251: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:38.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:56:38 smithi083 ceph-mon[79930]: pgmap v6252: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:38.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:56:38 smithi083 ceph-mon[77242]: pgmap v6252: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:38.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:56:38 smithi153 ceph-mon[70044]: pgmap v6252: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:56:40 smithi153 ceph-mon[70044]: pgmap v6253: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:56:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:56:40.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:56:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:56:40.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:56:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:56:40.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:56:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:56:40.919 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:56:40 smithi083 ceph-mon[79930]: pgmap v6253: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:40.920 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:56:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:56:40.920 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:56:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:56:40.920 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:56:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:56:40.920 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:56:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:56:40.921 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:56:40 smithi083 ceph-mon[77242]: pgmap v6253: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:40.921 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:56:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:56:40.921 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:56:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:56:40.921 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:56:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:56:40.922 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:56:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:56:41.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:56:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:56:40.918Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:56:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:56:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:56:41] "GET /metrics HTTP/1.1" 200 225220 "" "Prometheus/2.33.4" 2022-09-29T08:56:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:56:42 smithi153 ceph-mon[70044]: pgmap v6254: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:42.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:56:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:56:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:56:42.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:56:42 smithi083 ceph-mon[79930]: pgmap v6254: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:42.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:56:42 smithi083 ceph-mon[77242]: pgmap v6254: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:56:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:56:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:56:44.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:56:44 smithi083 ceph-mon[79930]: pgmap v6255: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:56:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:56:44.303Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:56:44.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:56:44 smithi083 ceph-mon[77242]: pgmap v6255: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:44.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:56:44 smithi153 ceph-mon[70044]: pgmap v6255: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:45.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:56:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:56:45.001Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:56:46.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:56:46 smithi153 ceph-mon[70044]: pgmap v6256: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:46.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:56:46 smithi083 ceph-mon[79930]: pgmap v6256: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:46.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:56:46 smithi083 ceph-mon[77242]: pgmap v6256: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:48.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:56:48 smithi083 ceph-mon[79930]: pgmap v6257: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:48.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:56:48 smithi083 ceph-mon[77242]: pgmap v6257: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:48.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:56:48 smithi153 ceph-mon[70044]: pgmap v6257: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:50.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:56:50 smithi153 ceph-mon[70044]: pgmap v6258: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:50.920 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:56:50 smithi083 ceph-mon[79930]: pgmap v6258: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:50.920 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:56:50 smithi083 ceph-mon[77242]: pgmap v6258: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:51.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:56:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:56:50.918Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:56:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:56:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:56:51] "GET /metrics HTTP/1.1" 200 225220 "" "Prometheus/2.33.4" 2022-09-29T08:56:52.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:56:52 smithi153 ceph-mon[70044]: pgmap v6259: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:56:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:56:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:56:52.845 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:56:52.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:56:52 smithi083 ceph-mon[79930]: pgmap v6259: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:52.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:56:52 smithi083 ceph-mon[77242]: pgmap v6259: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:53.133 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:56:53.134 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 8m ago 3h 26.7M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:56:53.134 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 6m ago 3h 63.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:56:53.134 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 8m ago 3h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:56:53.135 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 6m ago 3h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:56:53.135 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 6m ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:56:53.135 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 8m ago 3h 689M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:56:53.135 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 8m ago 3h 781M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:56:53.135 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 6m ago 3h 345M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:56:53.136 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 8m ago 3h 455M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:56:53.136 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 8m ago 3h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:56:53.136 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 6m ago 3h 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:56:53.136 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 8m ago 3h 564M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:56:53.136 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 8m ago 3h 555M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:56:53.137 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 8m ago 3h 437M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:56:53.137 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 8m ago 3h 540M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:56:53.137 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 6m ago 3h 479M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:56:53.137 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 6m ago 3h 592M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:56:53.138 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 6m ago 3h 528M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:56:53.138 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 6m ago 3h 621M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:56:53.138 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 6m ago 3h 156M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:56:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:56:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:56:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:56:53.500 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:56:53.559 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:56:53.559 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:56:53.559 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:56:53.559 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:56:53.560 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:56:53.560 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:56:53.560 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:56:53.560 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:56:53.560 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:56:53.561 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:56:53.561 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:56:53.561 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:56:53.561 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:56:53.562 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:56:53.562 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:56:53.562 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:56:53.562 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:56:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:56:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:56:54.302Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:56:54.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:56:54 smithi083 ceph-mon[77242]: from='client.41415 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:56:54.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:56:54 smithi083 ceph-mon[77242]: from='client.41421 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:56:54.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:56:54 smithi083 ceph-mon[77242]: pgmap v6260: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:54.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:56:54 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3700533149' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:56:54.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:56:54 smithi083 ceph-mon[79930]: from='client.41415 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:56:54.720 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:56:54 smithi083 ceph-mon[79930]: from='client.41421 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:56:54.720 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:56:54 smithi083 ceph-mon[79930]: pgmap v6260: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:54.721 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:56:54 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3700533149' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:56:54.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:56:54 smithi153 ceph-mon[70044]: from='client.41415 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:56:54.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:56:54 smithi153 ceph-mon[70044]: from='client.41421 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:56:54.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:56:54 smithi153 ceph-mon[70044]: pgmap v6260: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:54.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:56:54 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3700533149' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:56:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:56:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:56:55.002Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:56:56.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:56:56 smithi153 ceph-mon[70044]: pgmap v6261: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:56.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:56:56 smithi083 ceph-mon[79930]: pgmap v6261: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:56.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:56:56 smithi083 ceph-mon[77242]: pgmap v6261: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:58.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:56:58 smithi083 ceph-mon[79930]: pgmap v6262: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:58.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:56:58 smithi083 ceph-mon[77242]: pgmap v6262: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:56:58.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:56:58 smithi153 ceph-mon[70044]: pgmap v6262: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:00.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:57:00 smithi153 ceph-mon[70044]: pgmap v6263: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:00.920 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:57:00 smithi083 ceph-mon[79930]: pgmap v6263: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:00.920 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:57:00 smithi083 ceph-mon[77242]: pgmap v6263: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:01.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:57:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:57:00.918Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:57:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:57:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:57:01] "GET /metrics HTTP/1.1" 200 225218 "" "Prometheus/2.33.4" 2022-09-29T08:57:02.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:57:02 smithi153 ceph-mon[70044]: pgmap v6264: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:57:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:57:02.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:57:02.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:57:02 smithi083 ceph-mon[79930]: pgmap v6264: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:02.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:57:02 smithi083 ceph-mon[77242]: pgmap v6264: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:57:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:57:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:57:04.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:57:04 smithi083 ceph-mon[77242]: pgmap v6265: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:04.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:57:04 smithi083 ceph-mon[79930]: pgmap v6265: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:04.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:57:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:57:04.303Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:57:04.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:57:04 smithi153 ceph-mon[70044]: pgmap v6265: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:05.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:57:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:57:05.002Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:57:06.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:57:06 smithi153 ceph-mon[70044]: pgmap v6266: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:06.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:57:06 smithi083 ceph-mon[79930]: pgmap v6266: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:06.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:57:06 smithi083 ceph-mon[77242]: pgmap v6266: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:08.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:57:08 smithi083 ceph-mon[79930]: pgmap v6267: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:08.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:57:08 smithi083 ceph-mon[77242]: pgmap v6267: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:08.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:57:08 smithi153 ceph-mon[70044]: pgmap v6267: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:10.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:57:10 smithi153 ceph-mon[70044]: pgmap v6268: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:10.921 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:57:10 smithi083 ceph-mon[79930]: pgmap v6268: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:10.922 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:57:10 smithi083 ceph-mon[77242]: pgmap v6268: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:11.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:57:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:57:10.919Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:57:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:57:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:57:11] "GET /metrics HTTP/1.1" 200 225219 "" "Prometheus/2.33.4" 2022-09-29T08:57:12.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:57:12 smithi153 ceph-mon[70044]: pgmap v6269: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:57:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:57:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:57:12.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:57:12 smithi083 ceph-mon[79930]: pgmap v6269: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:12.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:57:12 smithi083 ceph-mon[77242]: pgmap v6269: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:57:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:57:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:57:14.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:57:14 smithi083 ceph-mon[79930]: pgmap v6270: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:14.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:57:14 smithi083 ceph-mon[77242]: pgmap v6270: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:14.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:57:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:57:14.303Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:57:14.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:57:14 smithi153 ceph-mon[70044]: pgmap v6270: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:15.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:57:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:57:15.002Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:57:15.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:57:15 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:57:15.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:57:15 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:57:15.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:57:15 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:57:15.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:57:15 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:57:15.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:57:15 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:57:15.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:57:15 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:57:15.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:57:15 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:57:15.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:57:15 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:57:15.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:57:15 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:57:15.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:57:15 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:57:15.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:57:15 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:57:15.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:57:15 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:57:16.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:57:16 smithi153 ceph-mon[70044]: pgmap v6271: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:16.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:57:16 smithi083 ceph-mon[79930]: pgmap v6271: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:16.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:57:16 smithi083 ceph-mon[77242]: pgmap v6271: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:18.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:57:18 smithi083 ceph-mon[79930]: pgmap v6272: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:18.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:57:18 smithi083 ceph-mon[77242]: pgmap v6272: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:18.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:57:18 smithi153 ceph-mon[70044]: pgmap v6272: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:20.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:57:20 smithi153 ceph-mon[70044]: pgmap v6273: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:20.922 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:57:20 smithi083 ceph-mon[79930]: pgmap v6273: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:20.922 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:57:20 smithi083 ceph-mon[77242]: pgmap v6273: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:21.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:57:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:57:20.920Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:57:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:57:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:57:21] "GET /metrics HTTP/1.1" 200 225219 "" "Prometheus/2.33.4" 2022-09-29T08:57:22.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:57:22 smithi153 ceph-mon[70044]: pgmap v6274: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:57:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:57:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:57:22.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:57:22 smithi083 ceph-mon[77242]: pgmap v6274: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:22.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:57:22 smithi083 ceph-mon[79930]: pgmap v6274: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:57:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:57:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:57:23.829 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:57:24.118 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:57:24.118 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 9m ago 3h 26.7M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:57:24.118 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 7m ago 3h 63.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:57:24.119 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 9m ago 3h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:57:24.119 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 7m ago 3h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:57:24.119 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 7m ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:57:24.119 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 9m ago 3h 689M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:57:24.119 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 9m ago 3h 781M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:57:24.120 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 7m ago 3h 345M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:57:24.120 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 9m ago 3h 455M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:57:24.120 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 9m ago 3h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:57:24.120 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 7m ago 3h 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:57:24.120 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 9m ago 3h 564M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:57:24.121 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 9m ago 3h 555M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:57:24.121 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 9m ago 3h 437M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:57:24.121 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 9m ago 3h 540M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:57:24.121 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 7m ago 3h 479M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:57:24.121 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 7m ago 3h 592M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:57:24.122 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 7m ago 3h 528M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:57:24.122 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 7m ago 3h 621M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:57:24.122 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 7m ago 3h 156M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:57:24.476 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:57:24.477 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:57:24.477 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:57:24.477 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:57:24.477 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:57:24.477 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:57:24.478 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:57:24.478 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:57:24.478 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:57:24.478 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:57:24.479 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:57:24.479 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:57:24.479 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:57:24.479 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:57:24.480 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:57:24.480 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:57:24.480 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:57:24.480 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:57:24.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:57:24 smithi083 ceph-mon[79930]: pgmap v6275: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:24.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:57:24 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3515199742' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:57:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:57:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:57:24.303Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:57:24.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:57:24 smithi083 ceph-mon[77242]: pgmap v6275: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:24.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:57:24 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3515199742' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:57:24.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:57:24 smithi153 ceph-mon[70044]: pgmap v6275: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:24.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:57:24 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3515199742' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:57:25.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:57:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:57:25.003Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:57:25.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:57:25 smithi153 ceph-mon[70044]: from='client.51364 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:57:25.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:57:25 smithi153 ceph-mon[70044]: from='client.41439 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:57:25.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:57:25 smithi083 ceph-mon[77242]: from='client.51364 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:57:25.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:57:25 smithi083 ceph-mon[77242]: from='client.41439 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:57:25.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:57:25 smithi083 ceph-mon[79930]: from='client.51364 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:57:25.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:57:25 smithi083 ceph-mon[79930]: from='client.41439 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:57:26.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:57:26 smithi153 ceph-mon[70044]: pgmap v6276: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:26.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:57:26 smithi083 ceph-mon[77242]: pgmap v6276: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:26.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:57:26 smithi083 ceph-mon[79930]: pgmap v6276: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:28.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:57:28 smithi083 ceph-mon[79930]: pgmap v6277: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:28.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:57:28 smithi083 ceph-mon[77242]: pgmap v6277: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:28.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:57:28 smithi153 ceph-mon[70044]: pgmap v6277: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:30.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:57:30 smithi153 ceph-mon[70044]: pgmap v6278: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:30.923 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:57:30 smithi083 ceph-mon[79930]: pgmap v6278: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:30.923 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:57:30 smithi083 ceph-mon[77242]: pgmap v6278: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:31.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:57:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:57:30.921Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:57:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:57:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:57:31] "GET /metrics HTTP/1.1" 200 225214 "" "Prometheus/2.33.4" 2022-09-29T08:57:32.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:57:32 smithi153 ceph-mon[70044]: pgmap v6279: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:57:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:57:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:57:32.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:57:32 smithi083 ceph-mon[79930]: pgmap v6279: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:32.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:57:32 smithi083 ceph-mon[77242]: pgmap v6279: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:57:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:57:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:57:34.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:57:34 smithi083 ceph-mon[79930]: pgmap v6280: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:34.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:57:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:57:34.305Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:57:34.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:57:34 smithi083 ceph-mon[77242]: pgmap v6280: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:34.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:57:34 smithi153 ceph-mon[70044]: pgmap v6280: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:35.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:57:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:57:35.004Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:57:36.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:57:36 smithi153 ceph-mon[70044]: pgmap v6281: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:36.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:57:36 smithi083 ceph-mon[79930]: pgmap v6281: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:36.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:57:36 smithi083 ceph-mon[77242]: pgmap v6281: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:38.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:57:38 smithi083 ceph-mon[79930]: pgmap v6282: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:38.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:57:38 smithi083 ceph-mon[77242]: pgmap v6282: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:38.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:57:38 smithi153 ceph-mon[70044]: pgmap v6282: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:57:40 smithi153 ceph-mon[70044]: pgmap v6283: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:57:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:57:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:57:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:57:40.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:57:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:57:40.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:57:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:57:40.923 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:57:40 smithi083 ceph-mon[79930]: pgmap v6283: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:40.924 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:57:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:57:40.924 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:57:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:57:40.924 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:57:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:57:40.924 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:57:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:57:40.925 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:57:40 smithi083 ceph-mon[77242]: pgmap v6283: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:40.925 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:57:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:57:40.925 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:57:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:57:40.925 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:57:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:57:40.925 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:57:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:57:41.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:57:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:57:40.921Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:57:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:57:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:57:41] "GET /metrics HTTP/1.1" 200 225220 "" "Prometheus/2.33.4" 2022-09-29T08:57:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:57:42 smithi153 ceph-mon[70044]: pgmap v6284: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:57:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:57:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:57:42.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:57:42 smithi083 ceph-mon[79930]: pgmap v6284: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:42.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:57:42 smithi083 ceph-mon[77242]: pgmap v6284: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:57:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:57:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:57:44.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:57:44 smithi083 ceph-mon[79930]: pgmap v6285: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:57:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:57:44.305Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:57:44.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:57:44 smithi083 ceph-mon[77242]: pgmap v6285: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:44.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:57:44 smithi153 ceph-mon[70044]: pgmap v6285: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:45.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:57:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:57:45.004Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:57:46.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:57:46 smithi153 ceph-mon[70044]: pgmap v6286: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:46.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:57:46 smithi083 ceph-mon[77242]: pgmap v6286: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:46.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:57:46 smithi083 ceph-mon[79930]: pgmap v6286: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:48.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:57:48 smithi083 ceph-mon[79930]: pgmap v6287: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:48.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:57:48 smithi083 ceph-mon[77242]: pgmap v6287: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:48.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:57:48 smithi153 ceph-mon[70044]: pgmap v6287: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:50.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:57:50 smithi153 ceph-mon[70044]: pgmap v6288: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:50.924 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:57:50 smithi083 ceph-mon[79930]: pgmap v6288: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:50.925 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:57:50 smithi083 ceph-mon[77242]: pgmap v6288: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:51.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:57:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:57:50.922Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:57:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:57:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:57:51] "GET /metrics HTTP/1.1" 200 225220 "" "Prometheus/2.33.4" 2022-09-29T08:57:52.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:57:52 smithi153 ceph-mon[70044]: pgmap v6289: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:57:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:57:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:57:52.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:57:52 smithi083 ceph-mon[79930]: pgmap v6289: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:52.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:57:52 smithi083 ceph-mon[77242]: pgmap v6289: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:57:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:57:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:57:54.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:57:54 smithi083 ceph-mon[79930]: pgmap v6290: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:57:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:57:54.306Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:57:54.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:57:54 smithi083 ceph-mon[77242]: pgmap v6290: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:54.804 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:57:54.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:57:54 smithi153 ceph-mon[70044]: pgmap v6290: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:55.093 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:57:55.094 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 9m ago 3h 26.7M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:57:55.094 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 7m ago 3h 63.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:57:55.094 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 9m ago 3h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:57:55.094 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 7m ago 3h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:57:55.094 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 7m ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:57:55.095 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 9m ago 3h 689M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:57:55.095 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 9m ago 3h 781M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:57:55.095 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 7m ago 3h 345M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:57:55.095 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 9m ago 3h 455M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:57:55.096 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 9m ago 3h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:57:55.096 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 7m ago 3h 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:57:55.096 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 9m ago 3h 564M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:57:55.096 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 9m ago 3h 555M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:57:55.096 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 9m ago 3h 437M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:57:55.097 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 9m ago 3h 540M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:57:55.097 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 7m ago 3h 479M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:57:55.097 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 7m ago 3h 592M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:57:55.097 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 7m ago 3h 528M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:57:55.098 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 7m ago 3h 621M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:57:55.098 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 7m ago 3h 156M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:57:55.447 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:57:55.448 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:57:55.448 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:57:55.448 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:57:55.449 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:57:55.449 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:57:55.449 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:57:55.449 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:57:55.450 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:57:55.450 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:57:55.450 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:57:55.450 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:57:55.450 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:57:55.451 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:57:55.451 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:57:55.451 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:57:55.451 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:57:55.452 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:57:55.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:57:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:57:55.005Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:57:55.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:57:55 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2394617653' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:57:55.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:57:55 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2394617653' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:57:55.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:57:55 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2394617653' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:57:56.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:57:56 smithi153 ceph-mon[70044]: from='client.51382 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:57:56.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:57:56 smithi153 ceph-mon[70044]: from='client.41457 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:57:56.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:57:56 smithi153 ceph-mon[70044]: pgmap v6291: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:56.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:57:56 smithi083 ceph-mon[79930]: from='client.51382 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:57:56.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:57:56 smithi083 ceph-mon[79930]: from='client.41457 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:57:56.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:57:56 smithi083 ceph-mon[79930]: pgmap v6291: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:56.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:57:56 smithi083 ceph-mon[77242]: from='client.51382 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:57:56.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:57:56 smithi083 ceph-mon[77242]: from='client.41457 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:57:56.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:57:56 smithi083 ceph-mon[77242]: pgmap v6291: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:58.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:57:58 smithi083 ceph-mon[79930]: pgmap v6292: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:58.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:57:58 smithi083 ceph-mon[77242]: pgmap v6292: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:57:58.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:57:58 smithi153 ceph-mon[70044]: pgmap v6292: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:00.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:58:00 smithi153 ceph-mon[70044]: pgmap v6293: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:00.925 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:58:00 smithi083 ceph-mon[79930]: pgmap v6293: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:00.925 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:58:00 smithi083 ceph-mon[77242]: pgmap v6293: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:01.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:58:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:58:00.923Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:58:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:58:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:58:01] "GET /metrics HTTP/1.1" 200 225217 "" "Prometheus/2.33.4" 2022-09-29T08:58:02.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:58:02 smithi153 ceph-mon[70044]: pgmap v6294: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:58:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:58:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:58:02.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:58:02 smithi083 ceph-mon[79930]: pgmap v6294: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:02.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:58:02 smithi083 ceph-mon[77242]: pgmap v6294: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:58:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:58:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:58:04.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:58:04 smithi083 ceph-mon[79930]: pgmap v6295: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:04.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:58:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:58:04.307Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:58:04.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:58:04 smithi083 ceph-mon[77242]: pgmap v6295: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:04.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:58:04 smithi153 ceph-mon[70044]: pgmap v6295: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:05.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:58:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:58:05.007Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:58:06.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:58:06 smithi153 ceph-mon[70044]: pgmap v6296: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:06.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:58:06 smithi083 ceph-mon[77242]: pgmap v6296: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:06.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:58:06 smithi083 ceph-mon[79930]: pgmap v6296: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:08.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:58:08 smithi083 ceph-mon[79930]: pgmap v6297: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:08.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:58:08 smithi083 ceph-mon[77242]: pgmap v6297: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:08.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:58:08 smithi153 ceph-mon[70044]: pgmap v6297: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:10.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:58:10 smithi153 ceph-mon[70044]: pgmap v6298: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:10.925 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:58:10 smithi083 ceph-mon[79930]: pgmap v6298: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:10.926 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:58:10 smithi083 ceph-mon[77242]: pgmap v6298: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:11.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:58:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:58:10.923Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:58:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:58:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:58:11] "GET /metrics HTTP/1.1" 200 225219 "" "Prometheus/2.33.4" 2022-09-29T08:58:12.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:58:12 smithi153 ceph-mon[70044]: pgmap v6299: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:58:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:58:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:58:12.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:58:12 smithi083 ceph-mon[79930]: pgmap v6299: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:12.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:58:12 smithi083 ceph-mon[77242]: pgmap v6299: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:58:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:58:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:58:14.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:58:14 smithi083 ceph-mon[79930]: pgmap v6300: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:14.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:58:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:58:14.307Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:58:14.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:58:14 smithi083 ceph-mon[77242]: pgmap v6300: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:14.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:58:14 smithi153 ceph-mon[70044]: pgmap v6300: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:15.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:58:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:58:15.007Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:58:15.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:58:15 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:58:15.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:58:15 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:58:15.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:58:15 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:58:15.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:58:15 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:58:15.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:58:15 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:58:15.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:58:15 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:58:15.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:58:15 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:58:15.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:58:15 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:58:15.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:58:15 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:58:16.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:58:16 smithi153 ceph-mon[70044]: pgmap v6301: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:16.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:58:16 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:58:16.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:58:16 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:58:16.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:58:16 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:58:16.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:58:16 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:58:16.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:58:16 smithi083 ceph-mon[79930]: pgmap v6301: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:16.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:58:16 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:58:16.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:58:16 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:58:16.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:58:16 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:58:16.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:58:16 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:58:16.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:58:16 smithi083 ceph-mon[77242]: pgmap v6301: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:16.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:58:16 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:58:16.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:58:16 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:58:16.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:58:16 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:58:16.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:58:16 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:58:18.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:58:18 smithi083 ceph-mon[79930]: pgmap v6302: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:18.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:58:18 smithi083 ceph-mon[77242]: pgmap v6302: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:18.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:58:18 smithi153 ceph-mon[70044]: pgmap v6302: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:20.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:58:20 smithi153 ceph-mon[70044]: pgmap v6303: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:20.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:58:20 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:58:20.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:58:20 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:58:20.925 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:58:20 smithi083 ceph-mon[77242]: pgmap v6303: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:20.926 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:58:20 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:58:20.926 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:58:20 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:58:20.926 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:58:20 smithi083 ceph-mon[79930]: pgmap v6303: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:20.927 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:58:20 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:58:20.927 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:58:20 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:58:21.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:58:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:58:20.923Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:58:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:58:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:58:21] "GET /metrics HTTP/1.1" 200 225219 "" "Prometheus/2.33.4" 2022-09-29T08:58:21.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:58:21 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:58:21.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:58:21 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:58:22.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:58:21 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:58:22.570 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:58:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:58:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:58:22.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:58:22 smithi083 ceph-mon[79930]: pgmap v6304: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:22.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:58:22 smithi083 ceph-mon[77242]: pgmap v6304: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:23.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:58:22 smithi153 ceph-mon[70044]: pgmap v6304: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:58:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:58:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:58:24.572 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:58:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:58:24.308Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:58:24.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:58:24 smithi083 ceph-mon[77242]: pgmap v6305: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:24.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:58:24 smithi083 ceph-mon[79930]: pgmap v6305: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:25.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:58:24 smithi153 ceph-mon[70044]: pgmap v6305: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:25.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:58:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:58:25.008Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:58:25.777 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:58:26.067 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:58:26.067 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 6s ago 3h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:58:26.068 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 8m ago 3h 63.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:58:26.068 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 6s ago 3h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:58:26.068 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 8m ago 3h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:58:26.068 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 8m ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:58:26.068 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 6s ago 3h 697M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:58:26.069 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 6s ago 3h 817M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:58:26.069 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 8m ago 3h 345M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:58:26.069 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 6s ago 3h 477M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:58:26.069 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 6s ago 3h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:58:26.069 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 8m ago 3h 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:58:26.070 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 6s ago 3h 590M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:58:26.070 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 6s ago 3h 581M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:58:26.070 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 6s ago 3h 456M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:58:26.070 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 6s ago 3h 565M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:58:26.070 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 8m ago 3h 479M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:58:26.071 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 8m ago 3h 592M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:58:26.071 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 8m ago 3h 528M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:58:26.071 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 8m ago 3h 621M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:58:26.071 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 8m ago 3h 156M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:58:26.420 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:58:26.420 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:58:26.421 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:58:26.421 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:58:26.421 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:58:26.421 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:58:26.422 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:58:26.422 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:58:26.422 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:58:26.422 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:58:26.423 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:58:26.423 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:58:26.423 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:58:26.423 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:58:26.423 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:58:26.424 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:58:26.424 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:58:26.424 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:58:26.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:58:26 smithi083 ceph-mon[79930]: pgmap v6306: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:26.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:58:26 smithi083 ceph-mon[79930]: from='client.51400 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:58:26.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:58:26 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2129374552' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:58:26.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:58:26 smithi083 ceph-mon[77242]: pgmap v6306: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:26.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:58:26 smithi083 ceph-mon[77242]: from='client.51400 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:58:26.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:58:26 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2129374552' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:58:27.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:58:26 smithi153 ceph-mon[70044]: pgmap v6306: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:27.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:58:26 smithi153 ceph-mon[70044]: from='client.51400 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:58:27.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:58:26 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2129374552' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:58:27.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:58:27 smithi083 ceph-mon[79930]: from='client.41475 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:58:27.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:58:27 smithi083 ceph-mon[77242]: from='client.41475 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:58:28.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:58:27 smithi153 ceph-mon[70044]: from='client.41475 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:58:28.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:58:28 smithi083 ceph-mon[79930]: pgmap v6307: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:28.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:58:28 smithi083 ceph-mon[77242]: pgmap v6307: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:29.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:58:28 smithi153 ceph-mon[70044]: pgmap v6307: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:30.926 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:58:30 smithi083 ceph-mon[79930]: pgmap v6308: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:30.926 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:58:30 smithi083 ceph-mon[77242]: pgmap v6308: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:31.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:58:30 smithi153 ceph-mon[70044]: pgmap v6308: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:31.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:58:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:58:30.924Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:58:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:58:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:58:31] "GET /metrics HTTP/1.1" 200 225219 "" "Prometheus/2.33.4" 2022-09-29T08:58:32.583 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:58:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:58:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:58:32.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:58:32 smithi083 ceph-mon[79930]: pgmap v6309: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:32.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:58:32 smithi083 ceph-mon[77242]: pgmap v6309: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:33.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:58:32 smithi153 ceph-mon[70044]: pgmap v6309: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:58:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:58:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:58:34.586 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:58:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:58:34.309Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:58:34.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:58:34 smithi083 ceph-mon[79930]: pgmap v6310: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:34.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:58:34 smithi083 ceph-mon[77242]: pgmap v6310: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:35.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:58:34 smithi153 ceph-mon[70044]: pgmap v6310: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:35.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:58:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:58:35.009Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:58:36.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:58:36 smithi083 ceph-mon[77242]: pgmap v6311: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:36.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:58:36 smithi083 ceph-mon[79930]: pgmap v6311: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:37.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:58:36 smithi153 ceph-mon[70044]: pgmap v6311: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:38.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:58:38 smithi083 ceph-mon[79930]: pgmap v6312: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:38.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:58:38 smithi083 ceph-mon[77242]: pgmap v6312: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:38.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:58:38 smithi153 ceph-mon[70044]: pgmap v6312: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:58:40 smithi153 ceph-mon[70044]: pgmap v6313: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:58:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:58:40.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:58:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:58:40.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:58:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:58:40.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:58:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:58:40.926 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:58:40 smithi083 ceph-mon[79930]: pgmap v6313: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:40.927 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:58:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:58:40.927 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:58:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:58:40.927 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:58:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:58:40.927 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:58:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:58:40.928 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:58:40 smithi083 ceph-mon[77242]: pgmap v6313: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:40.928 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:58:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:58:40.928 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:58:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:58:40.929 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:58:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:58:40.929 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:58:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:58:41.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:58:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:58:40.925Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:58:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:58:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:58:41] "GET /metrics HTTP/1.1" 200 225218 "" "Prometheus/2.33.4" 2022-09-29T08:58:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:58:42 smithi153 ceph-mon[70044]: pgmap v6314: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:42.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:58:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:58:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:58:42.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:58:42 smithi083 ceph-mon[79930]: pgmap v6314: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:42.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:58:42 smithi083 ceph-mon[77242]: pgmap v6314: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:58:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:58:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:58:44.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:58:44 smithi083 ceph-mon[79930]: pgmap v6315: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:58:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:58:44.310Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:58:44.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:58:44 smithi083 ceph-mon[77242]: pgmap v6315: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:44.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:58:44 smithi153 ceph-mon[70044]: pgmap v6315: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:45.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:58:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:58:45.009Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:58:46.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:58:46 smithi153 ceph-mon[70044]: pgmap v6316: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:46.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:58:46 smithi083 ceph-mon[79930]: pgmap v6316: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:46.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:58:46 smithi083 ceph-mon[77242]: pgmap v6316: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:48.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:58:48 smithi083 ceph-mon[79930]: pgmap v6317: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:48.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:58:48 smithi083 ceph-mon[77242]: pgmap v6317: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:48.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:58:48 smithi153 ceph-mon[70044]: pgmap v6317: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:50.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:58:50 smithi153 ceph-mon[70044]: pgmap v6318: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:50.928 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:58:50 smithi083 ceph-mon[79930]: pgmap v6318: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:50.928 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:58:50 smithi083 ceph-mon[77242]: pgmap v6318: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:51.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:58:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:58:50.926Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:58:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:58:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:58:51] "GET /metrics HTTP/1.1" 200 225218 "" "Prometheus/2.33.4" 2022-09-29T08:58:52.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:58:52 smithi153 ceph-mon[70044]: pgmap v6319: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:58:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:58:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:58:52.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:58:52 smithi083 ceph-mon[79930]: pgmap v6319: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:52.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:58:52 smithi083 ceph-mon[77242]: pgmap v6319: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:58:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:58:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:58:54.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:58:54 smithi083 ceph-mon[79930]: pgmap v6320: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:58:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:58:54.312Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:58:54.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:58:54 smithi083 ceph-mon[77242]: pgmap v6320: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:54.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:58:54 smithi153 ceph-mon[70044]: pgmap v6320: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:58:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:58:55.010Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:58:56.749 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:58:56.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:58:56 smithi153 ceph-mon[70044]: pgmap v6321: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:56.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:58:56 smithi083 ceph-mon[79930]: pgmap v6321: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:56.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:58:56 smithi083 ceph-mon[77242]: pgmap v6321: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:57.041 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:58:57.041 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 37s ago 3h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:58:57.041 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 8m ago 3h 63.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:58:57.042 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 37s ago 3h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:58:57.042 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 8m ago 3h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:58:57.042 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 8m ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:58:57.042 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 37s ago 3h 697M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:58:57.043 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 37s ago 3h 817M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:58:57.043 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 8m ago 3h 345M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:58:57.043 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 37s ago 3h 477M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:58:57.043 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 37s ago 3h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:58:57.044 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 8m ago 3h 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:58:57.044 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 37s ago 3h 590M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:58:57.044 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 37s ago 3h 581M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:58:57.044 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 37s ago 3h 456M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:58:57.045 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 37s ago 3h 565M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:58:57.045 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 8m ago 3h 479M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:58:57.045 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 8m ago 3h 592M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:58:57.045 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 8m ago 3h 528M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:58:57.046 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 8m ago 3h 621M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:58:57.046 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 8m ago 3h 156M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:58:57.395 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:58:57.395 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:58:57.396 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:58:57.396 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:58:57.396 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:58:57.396 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:58:57.396 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:58:57.397 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:58:57.397 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:58:57.397 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:58:57.397 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:58:57.398 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:58:57.398 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:58:57.398 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:58:57.398 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:58:57.399 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:58:57.399 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:58:57.400 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:58:57.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:58:57 smithi153 ceph-mon[70044]: from='client.51418 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:58:57.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:58:57 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/8029685' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:58:57.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:58:57 smithi083 ceph-mon[79930]: from='client.51418 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:58:57.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:58:57 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/8029685' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:58:57.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:58:57 smithi083 ceph-mon[77242]: from='client.51418 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:58:57.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:58:57 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/8029685' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:58:58.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:58:58 smithi153 ceph-mon[70044]: from='client.51424 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:58:58.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:58:58 smithi153 ceph-mon[70044]: pgmap v6322: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:58.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:58:58 smithi083 ceph-mon[77242]: from='client.51424 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:58:58.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:58:58 smithi083 ceph-mon[77242]: pgmap v6322: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:58:58.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:58:58 smithi083 ceph-mon[79930]: from='client.51424 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:58:58.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:58:58 smithi083 ceph-mon[79930]: pgmap v6322: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:00.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:59:00 smithi153 ceph-mon[70044]: pgmap v6323: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:00.929 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:59:00 smithi083 ceph-mon[79930]: pgmap v6323: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:00.929 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:59:00 smithi083 ceph-mon[77242]: pgmap v6323: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:01.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:59:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:59:00.926Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:59:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:59:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:59:01] "GET /metrics HTTP/1.1" 200 225218 "" "Prometheus/2.33.4" 2022-09-29T08:59:02.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:59:02 smithi153 ceph-mon[70044]: pgmap v6324: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:02.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:59:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:59:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:59:02.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:59:02 smithi083 ceph-mon[79930]: pgmap v6324: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:02.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:59:02 smithi083 ceph-mon[77242]: pgmap v6324: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:59:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:59:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:59:04.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:59:04 smithi083 ceph-mon[77242]: pgmap v6325: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:04.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:59:04 smithi083 ceph-mon[79930]: pgmap v6325: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:04.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:59:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:59:04.312Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:59:04.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:59:04 smithi153 ceph-mon[70044]: pgmap v6325: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:05.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:59:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:59:05.011Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:59:06.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:59:06 smithi153 ceph-mon[70044]: pgmap v6326: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:06.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:59:06 smithi083 ceph-mon[79930]: pgmap v6326: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:06.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:59:06 smithi083 ceph-mon[77242]: pgmap v6326: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:08.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:59:08 smithi083 ceph-mon[77242]: pgmap v6327: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:08.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:59:08 smithi083 ceph-mon[79930]: pgmap v6327: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:08.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:59:08 smithi153 ceph-mon[70044]: pgmap v6327: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:10.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:59:10 smithi153 ceph-mon[70044]: pgmap v6328: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:10.929 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:59:10 smithi083 ceph-mon[79930]: pgmap v6328: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:10.929 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:59:10 smithi083 ceph-mon[77242]: pgmap v6328: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:11.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:59:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:59:10.927Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:59:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:59:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:59:11] "GET /metrics HTTP/1.1" 200 225220 "" "Prometheus/2.33.4" 2022-09-29T08:59:12.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:59:12 smithi153 ceph-mon[70044]: pgmap v6329: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:59:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:59:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:59:12.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:59:12 smithi083 ceph-mon[77242]: pgmap v6329: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:12.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:59:12 smithi083 ceph-mon[79930]: pgmap v6329: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:59:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:59:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:59:14.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:59:14 smithi083 ceph-mon[77242]: pgmap v6330: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:14.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:59:14 smithi083 ceph-mon[79930]: pgmap v6330: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:14.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:59:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:59:14.313Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:59:14.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:59:14 smithi153 ceph-mon[70044]: pgmap v6330: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:15.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:59:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:59:15.011Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:59:16.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:59:16 smithi153 ceph-mon[70044]: pgmap v6331: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:16.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:59:16 smithi083 ceph-mon[77242]: pgmap v6331: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:16.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:59:16 smithi083 ceph-mon[79930]: pgmap v6331: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:18.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:59:18 smithi083 ceph-mon[77242]: pgmap v6332: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:18.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:59:18 smithi083 ceph-mon[79930]: pgmap v6332: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:18.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:59:18 smithi153 ceph-mon[70044]: pgmap v6332: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:20.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:59:20 smithi153 ceph-mon[70044]: pgmap v6333: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:20.928 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:59:20 smithi083 ceph-mon[79930]: pgmap v6333: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:20.929 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:59:20 smithi083 ceph-mon[77242]: pgmap v6333: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:21.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:59:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:59:20.927Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:59:21.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:59:21 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:59:21.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:59:21 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:59:21.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:59:21 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:59:21.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:59:21 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:59:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:59:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:59:21] "GET /metrics HTTP/1.1" 200 225220 "" "Prometheus/2.33.4" 2022-09-29T08:59:21.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:59:21 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:59:21.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:59:21 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:59:21.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:59:21 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:59:21.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:59:21 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:59:21.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:59:21 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T08:59:21.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:59:21 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T08:59:21.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:59:21 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T08:59:21.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:59:21 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T08:59:22.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:59:22 smithi153 ceph-mon[70044]: pgmap v6334: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:59:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:59:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:59:22.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:59:22 smithi083 ceph-mon[79930]: pgmap v6334: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:22.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:59:22 smithi083 ceph-mon[77242]: pgmap v6334: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:59:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:59:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:59:24.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:59:24 smithi083 ceph-mon[79930]: pgmap v6335: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:24.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:59:24 smithi083 ceph-mon[77242]: pgmap v6335: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:24.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:59:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:59:24.313Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:59:24.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:59:24 smithi153 ceph-mon[70044]: pgmap v6335: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:25.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:59:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:59:25.012Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:59:26.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:59:26 smithi153 ceph-mon[70044]: pgmap v6336: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:26.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:59:26 smithi083 ceph-mon[79930]: pgmap v6336: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:26.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:59:26 smithi083 ceph-mon[77242]: pgmap v6336: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:27.725 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:59:28.014 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:59:28.015 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 67s ago 3h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:59:28.015 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 9m ago 3h 63.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:59:28.015 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 67s ago 3h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:59:28.016 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 9m ago 3h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:59:28.016 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 9m ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:59:28.016 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 67s ago 3h 697M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:59:28.016 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 67s ago 3h 817M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:59:28.016 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 9m ago 3h 345M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:59:28.017 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 67s ago 3h 477M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:59:28.017 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 67s ago 3h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:59:28.017 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 9m ago 3h 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:59:28.017 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 67s ago 3h 590M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:59:28.018 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 67s ago 3h 581M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:59:28.018 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 67s ago 3h 456M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:59:28.018 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 67s ago 3h 565M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:59:28.018 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 9m ago 3h 479M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:59:28.019 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 9m ago 3h 592M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:59:28.019 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 9m ago 3h 528M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:59:28.019 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 9m ago 3h 621M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:59:28.019 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 9m ago 3h 156M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:59:28.368 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:59:28.368 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:59:28.369 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:59:28.369 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:59:28.369 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:59:28.370 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:59:28.370 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:59:28.370 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:59:28.370 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:59:28.370 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:59:28.371 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:59:28.371 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:59:28.371 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:59:28.371 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:59:28.372 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:59:28.372 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:59:28.372 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:59:28.372 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:59:28.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:59:28 smithi083 ceph-mon[79930]: pgmap v6337: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:28.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:59:28 smithi083 ceph-mon[79930]: from='client.41505 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:59:28.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:59:28 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2396464259' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:59:28.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:59:28 smithi083 ceph-mon[77242]: pgmap v6337: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:28.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:59:28 smithi083 ceph-mon[77242]: from='client.41505 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:59:28.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:59:28 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2396464259' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:59:28.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:59:28 smithi153 ceph-mon[70044]: pgmap v6337: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:28.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:59:28 smithi153 ceph-mon[70044]: from='client.41505 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:59:28.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:59:28 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2396464259' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:59:29.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:59:29 smithi083 ceph-mon[79930]: from='client.51442 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:59:29.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:59:29 smithi083 ceph-mon[77242]: from='client.51442 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:59:29.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:59:29 smithi153 ceph-mon[70044]: from='client.51442 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:59:30.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:59:30 smithi083 ceph-mon[79930]: pgmap v6338: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:30.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:59:30 smithi083 ceph-mon[77242]: pgmap v6338: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:30.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:59:30 smithi153 ceph-mon[70044]: pgmap v6338: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:31.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:59:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:59:30.928Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:59:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:59:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:59:31] "GET /metrics HTTP/1.1" 200 225216 "" "Prometheus/2.33.4" 2022-09-29T08:59:32.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:59:32 smithi153 ceph-mon[70044]: pgmap v6339: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:59:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:59:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:59:32.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:59:32 smithi083 ceph-mon[79930]: pgmap v6339: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:32.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:59:32 smithi083 ceph-mon[77242]: pgmap v6339: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:59:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:59:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:59:34.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:59:34 smithi083 ceph-mon[79930]: pgmap v6340: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:34.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:59:34 smithi083 ceph-mon[77242]: pgmap v6340: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:34.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:59:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:59:34.313Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:59:34.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:59:34 smithi153 ceph-mon[70044]: pgmap v6340: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:35.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:59:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:59:35.013Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:59:36.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:59:36 smithi153 ceph-mon[70044]: pgmap v6341: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:36.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:59:36 smithi083 ceph-mon[77242]: pgmap v6341: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:36.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:59:36 smithi083 ceph-mon[79930]: pgmap v6341: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:38.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:59:38 smithi083 ceph-mon[79930]: pgmap v6342: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:38.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:59:38 smithi083 ceph-mon[77242]: pgmap v6342: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:38.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:59:38 smithi153 ceph-mon[70044]: pgmap v6342: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:59:40 smithi153 ceph-mon[70044]: pgmap v6343: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:59:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:59:40.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:59:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:59:40.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:59:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:59:40.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:59:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:59:40.931 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:59:40 smithi083 ceph-mon[79930]: pgmap v6343: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:40.931 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:59:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:59:40.931 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:59:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:59:40.932 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:59:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:59:40.932 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:59:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:59:40.932 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:59:40 smithi083 ceph-mon[77242]: pgmap v6343: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:40.932 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:59:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:59:40.933 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:59:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:59:40.933 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:59:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T08:59:40.933 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:59:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T08:59:41.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:59:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:59:40.929Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:59:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:59:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:59:41] "GET /metrics HTTP/1.1" 200 225218 "" "Prometheus/2.33.4" 2022-09-29T08:59:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:59:42 smithi153 ceph-mon[70044]: pgmap v6344: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:59:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:59:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:59:42.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:59:42 smithi083 ceph-mon[79930]: pgmap v6344: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:42.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:59:42 smithi083 ceph-mon[77242]: pgmap v6344: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:59:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:59:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:59:44.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:59:44 smithi083 ceph-mon[77242]: pgmap v6345: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:44.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:59:44 smithi083 ceph-mon[79930]: pgmap v6345: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:44.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:59:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:59:44.315Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:59:44.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:59:44 smithi153 ceph-mon[70044]: pgmap v6345: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:45.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:59:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:59:45.014Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:59:46.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:59:46 smithi153 ceph-mon[70044]: pgmap v6346: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:46.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:59:46 smithi083 ceph-mon[79930]: pgmap v6346: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:46.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:59:46 smithi083 ceph-mon[77242]: pgmap v6346: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:48.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:59:48 smithi083 ceph-mon[79930]: pgmap v6347: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:48.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:59:48 smithi083 ceph-mon[77242]: pgmap v6347: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:48.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:59:48 smithi153 ceph-mon[70044]: pgmap v6347: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:50.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:59:50 smithi153 ceph-mon[70044]: pgmap v6348: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:50.932 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:59:50 smithi083 ceph-mon[79930]: pgmap v6348: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:50.933 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:59:50 smithi083 ceph-mon[77242]: pgmap v6348: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:51.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:59:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:59:50.930Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:59:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 08:59:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:59:51] "GET /metrics HTTP/1.1" 200 225218 "" "Prometheus/2.33.4" 2022-09-29T08:59:52.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:59:52 smithi153 ceph-mon[70044]: pgmap v6349: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 08:59:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T08:59:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T08:59:52.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:59:52 smithi083 ceph-mon[79930]: pgmap v6349: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:52.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:59:52 smithi083 ceph-mon[77242]: pgmap v6349: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 08:59:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:08:59:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T08:59:54.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:59:54 smithi083 ceph-mon[79930]: pgmap v6350: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:59:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:59:54.314Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:59:54.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:59:54 smithi083 ceph-mon[77242]: pgmap v6350: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:54.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:59:54 smithi153 ceph-mon[70044]: pgmap v6350: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 08:59:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T08:59:55.014Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T08:59:56.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:59:56 smithi153 ceph-mon[70044]: pgmap v6351: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:56.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:59:56 smithi083 ceph-mon[77242]: pgmap v6351: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:56.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:59:56 smithi083 ceph-mon[79930]: pgmap v6351: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:58.695 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T08:59:58.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:59:58 smithi083 ceph-mon[79930]: pgmap v6352: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:58.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:59:58 smithi083 ceph-mon[77242]: pgmap v6352: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:58.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:59:58 smithi153 ceph-mon[70044]: pgmap v6352: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T08:59:58.984 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T08:59:58.984 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 98s ago 3h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T08:59:58.984 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 9m ago 3h 63.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T08:59:58.984 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 98s ago 3h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T08:59:58.985 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 9m ago 3h 27.9M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T08:59:58.985 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 9m ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T08:59:58.985 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 98s ago 3h 697M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T08:59:58.985 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 98s ago 3h 817M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T08:59:58.986 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 9m ago 3h 345M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T08:59:58.986 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 98s ago 3h 477M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T08:59:58.986 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 98s ago 3h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T08:59:58.986 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 9m ago 3h 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T08:59:58.987 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 98s ago 3h 590M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T08:59:58.987 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 98s ago 3h 581M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T08:59:58.987 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 98s ago 3h 456M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T08:59:58.987 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 98s ago 3h 565M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T08:59:58.988 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 9m ago 3h 479M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T08:59:58.988 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 9m ago 3h 592M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T08:59:58.988 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 9m ago 3h 528M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T08:59:58.988 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 9m ago 3h 621M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T08:59:58.989 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 9m ago 3h 156M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T08:59:59.337 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T08:59:59.337 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T08:59:59.337 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T08:59:59.338 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:59:59.338 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T08:59:59.338 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T08:59:59.338 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:59:59.338 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T08:59:59.339 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T08:59:59.339 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:59:59.339 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T08:59:59.339 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T08:59:59.340 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T08:59:59.340 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T08:59:59.340 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T08:59:59.341 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T08:59:59.341 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T08:59:59.341 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T08:59:59.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:59:59 smithi083 ceph-mon[79930]: from='client.41523 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:59:59.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 08:59:59 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3453366262' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:59:59.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:59:59 smithi083 ceph-mon[77242]: from='client.41523 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:59:59.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 08:59:59 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3453366262' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T08:59:59.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:59:59 smithi153 ceph-mon[70044]: from='client.41523 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T08:59:59.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 08:59:59 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3453366262' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:00:00.453 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:00:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[77219]: 2022-09-29T08:59:59.999+0000 7f160f665700 -1 log_channel(cluster) log [ERR] : overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T09:00:00.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:00:00 smithi083 ceph-mon[79930]: from='client.41529 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:00:00.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:00:00 smithi083 ceph-mon[79930]: pgmap v6353: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:00:00.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:00:00 smithi083 ceph-mon[79930]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T09:00:00.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:00:00 smithi083 ceph-mon[77242]: from='client.41529 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:00:00.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:00:00 smithi083 ceph-mon[77242]: pgmap v6353: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:00:00.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:00:00 smithi083 ceph-mon[77242]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T09:00:00.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:00:00 smithi153 ceph-mon[70044]: from='client.41529 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:00:00.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:00:00 smithi153 ceph-mon[70044]: pgmap v6353: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:00:00.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:00:00 smithi153 ceph-mon[70044]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T09:00:01.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:00:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:00:00.931Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:00:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:00:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:00:01] "GET /metrics HTTP/1.1" 200 225221 "" "Prometheus/2.33.4" 2022-09-29T09:00:02.058 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:00:01 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:00:01.567Z caller=compact.go:519 level=info component=tsdb msg="write block" mint=1664431200138 maxt=1664438400000 ulid=01GE47JT3BCDM70Z9EJNTEMVGZ duration=52.105276ms 2022-09-29T09:00:02.058 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:00:01 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:00:01.570Z caller=head.go:827 level=info component=tsdb msg="Head GC completed" duration=3.008921ms 2022-09-29T09:00:02.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:00:02 smithi153 ceph-mon[70044]: pgmap v6354: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:00:02.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:00:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:00:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:00:02.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:00:02 smithi083 ceph-mon[79930]: pgmap v6354: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:00:02.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:00:02 smithi083 ceph-mon[77242]: pgmap v6354: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:00:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:00:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:00:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:00:04.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:00:04 smithi083 ceph-mon[79930]: pgmap v6355: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:00:04.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:00:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:00:04.315Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:00:04.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:00:04 smithi083 ceph-mon[77242]: pgmap v6355: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:00:04.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:00:04 smithi153 ceph-mon[70044]: pgmap v6355: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:00:05.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:00:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:00:05.016Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:00:06.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:00:06 smithi153 ceph-mon[70044]: pgmap v6356: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:00:06.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:00:06 smithi083 ceph-mon[79930]: pgmap v6356: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:00:06.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:00:06 smithi083 ceph-mon[77242]: pgmap v6356: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:00:08.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:00:08 smithi083 ceph-mon[79930]: pgmap v6357: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:00:08.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:00:08 smithi083 ceph-mon[77242]: pgmap v6357: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:00:08.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:00:08 smithi153 ceph-mon[70044]: pgmap v6357: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:00:10.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:00:10 smithi153 ceph-mon[70044]: pgmap v6358: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:00:10.933 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:00:10 smithi083 ceph-mon[79930]: pgmap v6358: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:00:10.934 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:00:10 smithi083 ceph-mon[77242]: pgmap v6358: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:00:11.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:00:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:00:10.931Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:00:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:00:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:00:11] "GET /metrics HTTP/1.1" 200 225218 "" "Prometheus/2.33.4" 2022-09-29T09:00:12.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:00:12 smithi153 ceph-mon[70044]: pgmap v6359: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:00:12.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:00:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:00:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:00:12.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:00:12 smithi083 ceph-mon[79930]: pgmap v6359: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:00:12.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:00:12 smithi083 ceph-mon[77242]: pgmap v6359: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:00:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:00:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:00:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:00:14.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:00:14 smithi083 ceph-mon[79930]: pgmap v6360: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:00:14.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:00:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:00:14.315Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:00:14.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:00:14 smithi083 ceph-mon[77242]: pgmap v6360: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:00:14.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:00:14 smithi153 ceph-mon[70044]: pgmap v6360: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:00:15.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:00:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:00:15.015Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:00:16.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:00:16 smithi153 ceph-mon[70044]: pgmap v6361: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:00:16.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:00:16 smithi083 ceph-mon[79930]: pgmap v6361: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:00:16.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:00:16 smithi083 ceph-mon[77242]: pgmap v6361: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:00:18.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:00:18 smithi083 ceph-mon[79930]: pgmap v6362: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:00:18.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:00:18 smithi083 ceph-mon[77242]: pgmap v6362: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:00:18.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:00:18 smithi153 ceph-mon[70044]: pgmap v6362: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:00:20.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:00:20 smithi153 ceph-mon[70044]: pgmap v6363: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:00:20.934 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:00:20 smithi083 ceph-mon[79930]: pgmap v6363: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:00:20.935 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:00:20 smithi083 ceph-mon[77242]: pgmap v6363: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:00:21.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:00:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:00:20.932Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:00:21.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:00:21 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:00:21.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:00:21 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:00:21.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:00:21 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:00:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:00:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:00:21] "GET /metrics HTTP/1.1" 200 225218 "" "Prometheus/2.33.4" 2022-09-29T09:00:21.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:00:21 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:00:21.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:00:21 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:00:21.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:00:21 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:00:21.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:00:21 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:00:21.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:00:21 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:00:21.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:00:21 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:00:22.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:00:22 smithi153 ceph-mon[70044]: pgmap v6364: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:00:22.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:00:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:00:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:00:22.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:00:22 smithi083 ceph-mon[79930]: pgmap v6364: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:00:22.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:00:22 smithi083 ceph-mon[77242]: pgmap v6364: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:00:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:00:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:00:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:00:24.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:00:24 smithi083 ceph-mon[79930]: pgmap v6365: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:00:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:00:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:00:24.317Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:00:24.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:00:24 smithi083 ceph-mon[77242]: pgmap v6365: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:00:24.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:00:24 smithi153 ceph-mon[70044]: pgmap v6365: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:00:25.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:00:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:00:25.016Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:00:26.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:00:26 smithi153 ceph-mon[70044]: pgmap v6366: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:00:26.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:00:26 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:00:26.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:00:26 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:00:26.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:00:26 smithi083 ceph-mon[79930]: pgmap v6366: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:00:26.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:00:26 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:00:26.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:00:26 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:00:26.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:00:26 smithi083 ceph-mon[77242]: pgmap v6366: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:00:26.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:00:26 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:00:26.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:00:26 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:00:27.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:00:27 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:00:27.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:00:27 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:00:28.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:00:27 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:00:28.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:00:28 smithi083 ceph-mon[79930]: pgmap v6367: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:00:28.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:00:28 smithi083 ceph-mon[77242]: pgmap v6367: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:00:29.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:00:28 smithi153 ceph-mon[70044]: pgmap v6367: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:00:29.665 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:00:29.955 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:00:29.955 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 2m ago 3h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:00:29.956 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 3s ago 3h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:00:29.956 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 2m ago 3h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:00:29.956 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 3s ago 3h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:00:29.956 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 3s ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:00:29.957 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 2m ago 3h 697M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:00:29.957 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 2m ago 3h 817M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:00:29.957 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 3s ago 3h 359M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:00:29.957 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 2m ago 3h 477M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:00:29.958 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 2m ago 3h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:00:29.958 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 3s ago 3h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:00:29.958 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 2m ago 3h 590M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:00:29.958 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 2m ago 3h 581M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:00:29.959 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 2m ago 3h 456M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:00:29.959 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 2m ago 3h 565M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:00:29.959 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 3s ago 3h 500M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:00:29.959 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 3s ago 3h 620M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:00:29.959 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 3s ago 3h 552M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:00:29.960 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 3s ago 3h 650M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:00:29.960 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 3s ago 3h 192M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:00:30.316 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:00:30.317 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:00:30.317 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:00:30.317 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:00:30.317 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:00:30.318 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:00:30.318 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:00:30.318 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:00:30.318 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:00:30.319 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:00:30.319 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:00:30.319 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:00:30.319 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:00:30.319 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:00:30.320 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:00:30.320 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:00:30.320 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:00:30.320 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:00:30.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:00:30 smithi083 ceph-mon[79930]: pgmap v6368: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:00:30.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:00:30 smithi083 ceph-mon[79930]: from='client.51472 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:00:30.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:00:30 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4231176272' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:00:30.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:00:30 smithi083 ceph-mon[77242]: pgmap v6368: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:00:30.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:00:30 smithi083 ceph-mon[77242]: from='client.51472 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:00:30.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:00:30 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4231176272' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:00:30.964 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:00:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:00:30.932Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:00:31.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:00:30 smithi153 ceph-mon[70044]: pgmap v6368: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:00:31.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:00:30 smithi153 ceph-mon[70044]: from='client.51472 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:00:31.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:00:30 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4231176272' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:00:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:00:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:00:31] "GET /metrics HTTP/1.1" 200 225220 "" "Prometheus/2.33.4" 2022-09-29T09:00:31.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:00:31 smithi083 ceph-mon[79930]: from='client.41547 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:00:31.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:00:31 smithi083 ceph-mon[77242]: from='client.41547 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:00:32.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:00:31 smithi153 ceph-mon[70044]: from='client.41547 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:00:32.716 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:00:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:00:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:00:33.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:00:32 smithi153 ceph-mon[70044]: pgmap v6369: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:00:33.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:00:32 smithi083 ceph-mon[79930]: pgmap v6369: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:00:33.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:00:32 smithi083 ceph-mon[77242]: pgmap v6369: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:00:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:00:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:00:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:00:34.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:00:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:00:34.317Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:00:35.017 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:00:34 smithi083 ceph-mon[79930]: pgmap v6370: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:00:35.018 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:00:34 smithi083 ceph-mon[77242]: pgmap v6370: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:00:35.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:00:34 smithi153 ceph-mon[70044]: pgmap v6370: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:00:35.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:00:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:00:35.016Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:00:37.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:00:36 smithi153 ceph-mon[70044]: pgmap v6371: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:00:37.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:00:36 smithi083 ceph-mon[79930]: pgmap v6371: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:00:37.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:00:36 smithi083 ceph-mon[77242]: pgmap v6371: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:00:38.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:00:38 smithi083 ceph-mon[79930]: pgmap v6372: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:00:38.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:00:38 smithi083 ceph-mon[77242]: pgmap v6372: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:00:38.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:00:38 smithi153 ceph-mon[70044]: pgmap v6372: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:00:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:00:40 smithi153 ceph-mon[70044]: pgmap v6373: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:00:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:00:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:00:40.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:00:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:00:40.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:00:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:00:40.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:00:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:00:40.935 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:00:40 smithi083 ceph-mon[79930]: pgmap v6373: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:00:40.936 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:00:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:00:40.936 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:00:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:00:40.936 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:00:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:00:40.936 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:00:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:00:40.937 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:00:40 smithi083 ceph-mon[77242]: pgmap v6373: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:00:40.937 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:00:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:00:40.937 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:00:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:00:40.937 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:00:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:00:40.937 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:00:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:00:41.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:00:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:00:40.933Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:00:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:00:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:00:41] "GET /metrics HTTP/1.1" 200 225220 "" "Prometheus/2.33.4" 2022-09-29T09:00:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:00:42 smithi153 ceph-mon[70044]: pgmap v6374: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:00:42.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:00:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:00:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:00:42.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:00:42 smithi083 ceph-mon[79930]: pgmap v6374: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:00:42.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:00:42 smithi083 ceph-mon[77242]: pgmap v6374: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:00:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:00:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:00:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:00:44.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:00:44 smithi083 ceph-mon[79930]: pgmap v6375: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:00:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:00:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:00:44.318Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:00:44.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:00:44 smithi083 ceph-mon[77242]: pgmap v6375: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:00:44.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:00:44 smithi153 ceph-mon[70044]: pgmap v6375: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:00:45.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:00:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:00:45.016Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:00:46.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:00:46 smithi153 ceph-mon[70044]: pgmap v6376: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:00:46.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:00:46 smithi083 ceph-mon[79930]: pgmap v6376: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:00:46.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:00:46 smithi083 ceph-mon[77242]: pgmap v6376: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:00:48.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:00:48 smithi083 ceph-mon[79930]: pgmap v6377: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:00:48.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:00:48 smithi083 ceph-mon[77242]: pgmap v6377: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:00:48.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:00:48 smithi153 ceph-mon[70044]: pgmap v6377: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:00:50.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:00:50 smithi153 ceph-mon[70044]: pgmap v6378: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:00:50.936 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:00:50 smithi083 ceph-mon[79930]: pgmap v6378: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:00:50.937 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:00:50 smithi083 ceph-mon[77242]: pgmap v6378: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:00:51.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:00:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:00:50.934Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:00:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:00:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:00:51] "GET /metrics HTTP/1.1" 200 225220 "" "Prometheus/2.33.4" 2022-09-29T09:00:52.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:00:52 smithi153 ceph-mon[70044]: pgmap v6379: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:00:52.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:00:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:00:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:00:52.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:00:52 smithi083 ceph-mon[79930]: pgmap v6379: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:00:52.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:00:52 smithi083 ceph-mon[77242]: pgmap v6379: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:00:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:00:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:00:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:00:54.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:00:54 smithi083 ceph-mon[79930]: pgmap v6380: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:00:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:00:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:00:54.318Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:00:54.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:00:54 smithi083 ceph-mon[77242]: pgmap v6380: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:00:54.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:00:54 smithi153 ceph-mon[70044]: pgmap v6380: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:00:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:00:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:00:55.017Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:00:56.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:00:56 smithi153 ceph-mon[70044]: pgmap v6381: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:00:56.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:00:56 smithi083 ceph-mon[79930]: pgmap v6381: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:00:56.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:00:56 smithi083 ceph-mon[77242]: pgmap v6381: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:00:58.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:00:58 smithi083 ceph-mon[79930]: pgmap v6382: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:00:58.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:00:58 smithi083 ceph-mon[77242]: pgmap v6382: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:00:58.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:00:58 smithi153 ceph-mon[70044]: pgmap v6382: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:00.642 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:01:00.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:01:00 smithi153 ceph-mon[70044]: pgmap v6383: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:00.931 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:01:00.931 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 2m ago 3h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:01:00.932 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 34s ago 3h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:01:00.932 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 2m ago 3h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:01:00.932 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 34s ago 3h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:01:00.932 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 34s ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:01:00.933 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 2m ago 3h 697M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:01:00.933 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 2m ago 3h 817M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:01:00.933 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 34s ago 3h 359M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:01:00.933 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 2m ago 3h 477M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:01:00.933 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 2m ago 3h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:01:00.934 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 34s ago 3h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:01:00.934 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 2m ago 3h 590M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:01:00.934 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 2m ago 3h 581M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:01:00.934 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 2m ago 3h 456M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:01:00.935 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 2m ago 3h 565M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:01:00.935 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 34s ago 3h 500M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:01:00.935 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 34s ago 3h 620M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:01:00.935 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 34s ago 3h 552M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:01:00.936 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 34s ago 3h 650M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:01:00.936 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 34s ago 3h 192M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:01:00.937 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:01:00 smithi083 ceph-mon[79930]: pgmap v6383: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:00.937 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:01:00 smithi083 ceph-mon[77242]: pgmap v6383: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:01.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:01:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:01:00.934Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:01:01.288 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:01:01.289 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:01:01.289 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:01:01.289 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:01:01.289 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:01:01.290 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:01:01.290 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:01:01.290 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:01:01.290 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:01:01.291 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:01:01.291 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:01:01.291 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:01:01.291 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:01:01.291 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:01:01.292 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:01:01.292 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:01:01.292 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:01:01.292 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:01:01.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:01:01 smithi153 ceph-mon[70044]: from='client.51490 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:01:01.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:01:01 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2738108293' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:01:01.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:01:01 smithi083 ceph-mon[77242]: from='client.51490 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:01:01.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:01:01 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2738108293' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:01:01.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:01:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:01:01] "GET /metrics HTTP/1.1" 200 225219 "" "Prometheus/2.33.4" 2022-09-29T09:01:01.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:01:01 smithi083 ceph-mon[79930]: from='client.51490 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:01:01.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:01:01 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2738108293' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:01:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:01:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:01:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:01:02.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:01:02 smithi153 ceph-mon[70044]: from='client.51496 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:01:02.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:01:02 smithi153 ceph-mon[70044]: pgmap v6384: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:02.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:01:02 smithi083 ceph-mon[79930]: from='client.51496 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:01:02.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:01:02 smithi083 ceph-mon[79930]: pgmap v6384: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:02.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:01:02 smithi083 ceph-mon[77242]: from='client.51496 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:01:02.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:01:02 smithi083 ceph-mon[77242]: pgmap v6384: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:01:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:01:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:01:04.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:01:04 smithi083 ceph-mon[77242]: pgmap v6385: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:04.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:01:04 smithi083 ceph-mon[79930]: pgmap v6385: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:04.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:01:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:01:04.319Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:01:05.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:01:04 smithi153 ceph-mon[70044]: pgmap v6385: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:05.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:01:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:01:05.018Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:01:06.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:01:06 smithi083 ceph-mon[77242]: pgmap v6386: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:06.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:01:06 smithi083 ceph-mon[79930]: pgmap v6386: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:07.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:01:06 smithi153 ceph-mon[70044]: pgmap v6386: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:08.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:01:08 smithi083 ceph-mon[79930]: pgmap v6387: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:08.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:01:08 smithi083 ceph-mon[77242]: pgmap v6387: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:08.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:01:08 smithi153 ceph-mon[70044]: pgmap v6387: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:10.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:01:10 smithi153 ceph-mon[70044]: pgmap v6388: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:10.936 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:01:10 smithi083 ceph-mon[79930]: pgmap v6388: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:10.937 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:01:10 smithi083 ceph-mon[77242]: pgmap v6388: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:11.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:01:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:01:10.934Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:01:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:01:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:01:11] "GET /metrics HTTP/1.1" 200 225214 "" "Prometheus/2.33.4" 2022-09-29T09:01:12.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:01:12 smithi153 ceph-mon[70044]: pgmap v6389: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:01:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:01:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:01:12.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:01:12 smithi083 ceph-mon[79930]: pgmap v6389: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:12.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:01:12 smithi083 ceph-mon[77242]: pgmap v6389: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:01:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:01:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:01:14.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:01:14 smithi083 ceph-mon[79930]: pgmap v6390: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:14.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:01:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:01:14.320Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:01:14.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:01:14 smithi083 ceph-mon[77242]: pgmap v6390: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:15.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:01:14 smithi153 ceph-mon[70044]: pgmap v6390: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:15.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:01:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:01:15.018Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:01:16.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:01:16 smithi083 ceph-mon[77242]: pgmap v6391: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:16.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:01:16 smithi083 ceph-mon[79930]: pgmap v6391: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:17.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:01:16 smithi153 ceph-mon[70044]: pgmap v6391: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:18.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:01:18 smithi153 ceph-mon[70044]: pgmap v6392: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:18.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:01:18 smithi083 ceph-mon[79930]: pgmap v6392: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:18.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:01:18 smithi083 ceph-mon[77242]: pgmap v6392: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:20.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:01:20 smithi153 ceph-mon[70044]: pgmap v6393: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:20.937 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:01:20 smithi083 ceph-mon[79930]: pgmap v6393: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:20.937 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:01:20 smithi083 ceph-mon[77242]: pgmap v6393: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:21.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:01:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:01:20.935Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:01:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:01:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:01:21] "GET /metrics HTTP/1.1" 200 225214 "" "Prometheus/2.33.4" 2022-09-29T09:01:22.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:01:22 smithi153 ceph-mon[70044]: pgmap v6394: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:22.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:01:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:01:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:01:22.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:01:22 smithi083 ceph-mon[79930]: pgmap v6394: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:22.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:01:22 smithi083 ceph-mon[77242]: pgmap v6394: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:01:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:01:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:01:24.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:01:24 smithi083 ceph-mon[77242]: pgmap v6395: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:24.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:01:24 smithi083 ceph-mon[79930]: pgmap v6395: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:24.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:01:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:01:24.320Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:01:25.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:01:24 smithi153 ceph-mon[70044]: pgmap v6395: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:25.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:01:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:01:25.018Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:01:26.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:01:26 smithi083 ceph-mon[79930]: pgmap v6396: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:26.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:01:26 smithi083 ceph-mon[77242]: pgmap v6396: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:27.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:01:26 smithi153 ceph-mon[70044]: pgmap v6396: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:27.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:01:27 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:01:27.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:01:27 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:01:27.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:01:27 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:01:27.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:01:27 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:01:27.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:01:27 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:01:27.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:01:27 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:01:27.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:01:27 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:01:27.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:01:27 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:01:28.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:01:27 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:01:28.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:01:27 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:01:28.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:01:27 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:01:28.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:01:27 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:01:28.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:01:28 smithi083 ceph-mon[79930]: pgmap v6397: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:28.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:01:28 smithi083 ceph-mon[77242]: pgmap v6397: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:29.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:01:28 smithi153 ceph-mon[70044]: pgmap v6397: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:30.938 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:01:30 smithi083 ceph-mon[79930]: pgmap v6398: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:30.939 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:01:30 smithi083 ceph-mon[77242]: pgmap v6398: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:31.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:01:30 smithi153 ceph-mon[70044]: pgmap v6398: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:31.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:01:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:01:30.936Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:01:31.618 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:01:31.902 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:01:31.902 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 3m ago 3h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:01:31.903 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 65s ago 3h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:01:31.903 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 3m ago 3h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:01:31.903 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 65s ago 3h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:01:31.903 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 65s ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:01:31.904 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 3m ago 3h 697M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:01:31.904 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 3m ago 3h 817M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:01:31.904 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 65s ago 3h 359M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:01:31.904 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 3m ago 3h 477M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:01:31.905 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 3m ago 3h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:01:31.905 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 65s ago 3h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:01:31.905 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 3m ago 3h 590M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:01:31.905 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 3m ago 3h 581M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:01:31.905 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 3m ago 3h 456M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:01:31.906 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 3m ago 3h 565M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:01:31.906 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 65s ago 3h 500M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:01:31.906 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 65s ago 3h 620M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:01:31.906 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 65s ago 3h 552M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:01:31.907 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 65s ago 3h 650M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:01:31.907 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 65s ago 3h 192M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:01:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:01:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:01:31] "GET /metrics HTTP/1.1" 200 225218 "" "Prometheus/2.33.4" 2022-09-29T09:01:32.259 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:01:32.259 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:01:32.260 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:01:32.260 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:01:32.260 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:01:32.260 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:01:32.261 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:01:32.261 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:01:32.261 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:01:32.261 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:01:32.261 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:01:32.262 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:01:32.262 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:01:32.262 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:01:32.262 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:01:32.263 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:01:32.263 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:01:32.263 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:01:32.572 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:01:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:01:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:01:32.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:01:32 smithi083 ceph-mon[79930]: pgmap v6399: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:32.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:01:32 smithi083 ceph-mon[79930]: from='client.41577 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:01:32.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:01:32 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1784635680' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:01:32.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:01:32 smithi083 ceph-mon[77242]: pgmap v6399: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:32.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:01:32 smithi083 ceph-mon[77242]: from='client.41577 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:01:32.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:01:32 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1784635680' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:01:33.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:01:32 smithi153 ceph-mon[70044]: pgmap v6399: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:33.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:01:32 smithi153 ceph-mon[70044]: from='client.41577 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:01:33.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:01:32 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1784635680' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:01:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:01:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:01:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:01:33.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:01:33 smithi083 ceph-mon[79930]: from='client.51514 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:01:33.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:01:33 smithi083 ceph-mon[77242]: from='client.51514 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:01:34.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:01:33 smithi153 ceph-mon[70044]: from='client.51514 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:01:34.578 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:01:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:01:34.321Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:01:34.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:01:34 smithi083 ceph-mon[79930]: pgmap v6400: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:34.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:01:34 smithi083 ceph-mon[77242]: pgmap v6400: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:35.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:01:34 smithi153 ceph-mon[70044]: pgmap v6400: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:35.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:01:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:01:35.019Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:01:36.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:01:36 smithi083 ceph-mon[79930]: pgmap v6401: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:36.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:01:36 smithi083 ceph-mon[77242]: pgmap v6401: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:37.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:01:36 smithi153 ceph-mon[70044]: pgmap v6401: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:38.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:01:38 smithi153 ceph-mon[70044]: pgmap v6402: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:38.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:01:38 smithi083 ceph-mon[79930]: pgmap v6402: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:38.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:01:38 smithi083 ceph-mon[77242]: pgmap v6402: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:40.937 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:01:40 smithi083 ceph-mon[77242]: pgmap v6403: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:40.938 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:01:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:01:40.938 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:01:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:01:40.938 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:01:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:01:40.939 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:01:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:01:40.939 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:01:40 smithi083 ceph-mon[79930]: pgmap v6403: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:40.939 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:01:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:01:40.939 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:01:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:01:40.940 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:01:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:01:40.940 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:01:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:01:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:01:40 smithi153 ceph-mon[70044]: pgmap v6403: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:01:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:01:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:01:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:01:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:01:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:01:41.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:01:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:01:41.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:01:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:01:40.935Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:01:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:01:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:01:41] "GET /metrics HTTP/1.1" 200 225219 "" "Prometheus/2.33.4" 2022-09-29T09:01:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:01:42 smithi153 ceph-mon[70044]: pgmap v6404: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:42.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:01:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:01:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:01:42.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:01:42 smithi083 ceph-mon[79930]: pgmap v6404: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:42.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:01:42 smithi083 ceph-mon[77242]: pgmap v6404: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:01:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:01:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:01:44.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:01:44 smithi083 ceph-mon[77242]: pgmap v6405: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:44.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:01:44 smithi083 ceph-mon[79930]: pgmap v6405: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:44.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:01:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:01:44.322Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:01:45.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:01:44 smithi153 ceph-mon[70044]: pgmap v6405: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:45.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:01:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:01:45.020Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:01:46.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:01:46 smithi083 ceph-mon[79930]: pgmap v6406: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:46.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:01:46 smithi083 ceph-mon[77242]: pgmap v6406: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:47.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:01:46 smithi153 ceph-mon[70044]: pgmap v6406: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:48.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:01:48 smithi153 ceph-mon[70044]: pgmap v6407: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:48.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:01:48 smithi083 ceph-mon[79930]: pgmap v6407: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:48.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:01:48 smithi083 ceph-mon[77242]: pgmap v6407: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:50.938 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:01:50 smithi083 ceph-mon[79930]: pgmap v6408: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:50.939 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:01:50 smithi083 ceph-mon[77242]: pgmap v6408: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:51.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:01:50 smithi153 ceph-mon[70044]: pgmap v6408: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:51.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:01:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:01:50.936Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:01:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:01:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:01:51] "GET /metrics HTTP/1.1" 200 225219 "" "Prometheus/2.33.4" 2022-09-29T09:01:52.569 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:01:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:01:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:01:52.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:01:52 smithi083 ceph-mon[79930]: pgmap v6409: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:52.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:01:52 smithi083 ceph-mon[77242]: pgmap v6409: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:53.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:01:52 smithi153 ceph-mon[70044]: pgmap v6409: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:01:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:01:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:01:54.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:01:54 smithi083 ceph-mon[79930]: pgmap v6410: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:01:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:01:54.322Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:01:54.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:01:54 smithi083 ceph-mon[77242]: pgmap v6410: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:55.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:01:54 smithi153 ceph-mon[70044]: pgmap v6410: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:01:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:01:55.021Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:01:56.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:01:56 smithi083 ceph-mon[79930]: pgmap v6411: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:56.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:01:56 smithi083 ceph-mon[77242]: pgmap v6411: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:57.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:01:56 smithi153 ceph-mon[70044]: pgmap v6411: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:58.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:01:58 smithi153 ceph-mon[70044]: pgmap v6412: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:58.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:01:58 smithi083 ceph-mon[79930]: pgmap v6412: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:01:58.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:01:58 smithi083 ceph-mon[77242]: pgmap v6412: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:00.940 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:02:00 smithi083 ceph-mon[79930]: pgmap v6413: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:00.941 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:02:00 smithi083 ceph-mon[77242]: pgmap v6413: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:01.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:02:00 smithi153 ceph-mon[70044]: pgmap v6413: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:01.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:02:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:02:00.937Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:02:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:02:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:02:01] "GET /metrics HTTP/1.1" 200 225217 "" "Prometheus/2.33.4" 2022-09-29T09:02:02.573 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:02:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:02:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:02:02.587 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:02:02.873 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:02:02.874 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 3m ago 3h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:02:02.874 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 96s ago 3h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:02:02.874 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 3m ago 3h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:02:02.875 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 96s ago 3h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:02:02.875 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 96s ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:02:02.875 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 3m ago 3h 697M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:02:02.875 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 3m ago 3h 817M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:02:02.876 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 96s ago 3h 359M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:02:02.876 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 3m ago 3h 477M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:02:02.876 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 3m ago 3h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:02:02.876 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 96s ago 3h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:02:02.876 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 3m ago 3h 590M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:02:02.877 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 3m ago 3h 581M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:02:02.877 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 3m ago 3h 456M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:02:02.877 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 3m ago 3h 565M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:02:02.877 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 96s ago 3h 500M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:02:02.878 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 96s ago 3h 620M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:02:02.878 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 96s ago 3h 552M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:02:02.878 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 96s ago 3h 650M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:02:02.878 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 96s ago 3h 192M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:02:02.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:02:02 smithi083 ceph-mon[79930]: pgmap v6414: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:02.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:02:02 smithi083 ceph-mon[77242]: pgmap v6414: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:03.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:02:02 smithi153 ceph-mon[70044]: pgmap v6414: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:03.229 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:02:03.229 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:02:03.230 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:02:03.230 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:02:03.230 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:02:03.230 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:02:03.231 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:02:03.231 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:02:03.231 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:02:03.231 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:02:03.231 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:02:03.232 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:02:03.232 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:02:03.232 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:02:03.232 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:02:03.233 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:02:03.233 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:02:03.233 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:02:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:02:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:02:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:02:03.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:02:03 smithi083 ceph-mon[79930]: from='client.51526 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:02:03.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:02:03 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/595839357' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:02:03.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:02:03 smithi083 ceph-mon[77242]: from='client.51526 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:02:03.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:02:03 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/595839357' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:02:04.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:02:03 smithi153 ceph-mon[70044]: from='client.51526 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:02:04.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:02:03 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/595839357' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:02:04.579 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:02:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:02:04.322Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:02:04.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:02:04 smithi083 ceph-mon[79930]: from='client.51532 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:02:04.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:02:04 smithi083 ceph-mon[79930]: pgmap v6415: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:04.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:02:04 smithi083 ceph-mon[77242]: from='client.51532 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:02:04.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:02:04 smithi083 ceph-mon[77242]: pgmap v6415: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:05.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:02:04 smithi153 ceph-mon[70044]: from='client.51532 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:02:05.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:02:04 smithi153 ceph-mon[70044]: pgmap v6415: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:05.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:02:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:02:05.022Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:02:06.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:02:06 smithi083 ceph-mon[79930]: pgmap v6416: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:06.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:02:06 smithi083 ceph-mon[77242]: pgmap v6416: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:07.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:02:06 smithi153 ceph-mon[70044]: pgmap v6416: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:08.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:02:08 smithi153 ceph-mon[70044]: pgmap v6417: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:08.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:02:08 smithi083 ceph-mon[79930]: pgmap v6417: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:08.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:02:08 smithi083 ceph-mon[77242]: pgmap v6417: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:10.940 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:02:10 smithi083 ceph-mon[79930]: pgmap v6418: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:10.940 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:02:10 smithi083 ceph-mon[77242]: pgmap v6418: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:11.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:02:10 smithi153 ceph-mon[70044]: pgmap v6418: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:11.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:02:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:02:10.937Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:02:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:02:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:02:11] "GET /metrics HTTP/1.1" 200 225215 "" "Prometheus/2.33.4" 2022-09-29T09:02:12.583 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:02:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:02:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:02:12.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:02:12 smithi083 ceph-mon[79930]: pgmap v6419: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:12.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:02:12 smithi083 ceph-mon[77242]: pgmap v6419: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:13.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:02:12 smithi153 ceph-mon[70044]: pgmap v6419: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:02:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:02:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:02:14.585 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:02:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:02:14.323Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:02:14.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:02:14 smithi083 ceph-mon[79930]: pgmap v6420: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:14.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:02:14 smithi083 ceph-mon[77242]: pgmap v6420: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:15.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:02:14 smithi153 ceph-mon[70044]: pgmap v6420: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:15.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:02:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:02:15.023Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:02:16.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:02:16 smithi083 ceph-mon[79930]: pgmap v6421: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:16.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:02:16 smithi083 ceph-mon[77242]: pgmap v6421: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:17.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:02:16 smithi153 ceph-mon[70044]: pgmap v6421: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:18.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:02:18 smithi153 ceph-mon[70044]: pgmap v6422: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:18.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:02:18 smithi083 ceph-mon[77242]: pgmap v6422: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:18.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:02:18 smithi083 ceph-mon[79930]: pgmap v6422: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:20.940 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:02:20 smithi083 ceph-mon[79930]: pgmap v6423: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:20.941 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:02:20 smithi083 ceph-mon[77242]: pgmap v6423: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:21.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:02:20 smithi153 ceph-mon[70044]: pgmap v6423: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:21.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:02:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:02:20.938Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:02:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:02:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:02:21] "GET /metrics HTTP/1.1" 200 225215 "" "Prometheus/2.33.4" 2022-09-29T09:02:22.578 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:02:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:02:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:02:22.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:02:22 smithi083 ceph-mon[77242]: pgmap v6424: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:22.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:02:22 smithi083 ceph-mon[79930]: pgmap v6424: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:23.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:02:22 smithi153 ceph-mon[70044]: pgmap v6424: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:02:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:02:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:02:24.581 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:02:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:02:24.323Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:02:24.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:02:24 smithi083 ceph-mon[77242]: pgmap v6425: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:24.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:02:24 smithi083 ceph-mon[79930]: pgmap v6425: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:25.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:02:24 smithi153 ceph-mon[70044]: pgmap v6425: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:25.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:02:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:02:25.024Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:02:26.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:02:26 smithi083 ceph-mon[79930]: pgmap v6426: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:26.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:02:26 smithi083 ceph-mon[77242]: pgmap v6426: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:27.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:02:26 smithi153 ceph-mon[70044]: pgmap v6426: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:27.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:02:27 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:02:27.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:02:27 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:02:27.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:02:27 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:02:27.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:02:27 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:02:27.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:02:27 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:02:27.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:02:27 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:02:28.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:02:27 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:02:28.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:02:27 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:02:28.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:02:27 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:02:28.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:02:28 smithi083 ceph-mon[79930]: pgmap v6427: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:28.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:02:28 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:02:28.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:02:28 smithi083 ceph-mon[77242]: pgmap v6427: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:28.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:02:28 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:02:29.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:02:28 smithi153 ceph-mon[70044]: pgmap v6427: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:29.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:02:28 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:02:30.941 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:02:30 smithi083 ceph-mon[79930]: pgmap v6428: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:30.941 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:02:30 smithi083 ceph-mon[77242]: pgmap v6428: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:31.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:02:30 smithi153 ceph-mon[70044]: pgmap v6428: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:31.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:02:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:02:30.938Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:02:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:02:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:02:31] "GET /metrics HTTP/1.1" 200 225218 "" "Prometheus/2.33.4" 2022-09-29T09:02:32.594 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:02:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:02:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:02:32.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:02:32 smithi083 ceph-mon[79930]: pgmap v6429: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:32.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:02:32 smithi083 ceph-mon[77242]: pgmap v6429: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:33.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:02:32 smithi153 ceph-mon[70044]: pgmap v6429: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:02:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:02:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:02:33.565 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:02:33.862 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:02:33.863 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 4m ago 3h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:02:33.863 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 2m ago 3h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:02:33.863 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 4m ago 3h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:02:33.863 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 2m ago 3h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:02:33.864 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 2m ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:02:33.864 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 4m ago 3h 697M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:02:33.864 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 4m ago 3h 817M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:02:33.864 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 2m ago 3h 359M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:02:33.864 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 4m ago 3h 477M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:02:33.865 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 4m ago 3h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:02:33.865 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 2m ago 3h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:02:33.865 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 4m ago 3h 590M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:02:33.865 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 4m ago 3h 581M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:02:33.866 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 4m ago 3h 456M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:02:33.866 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 4m ago 3h 565M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:02:33.866 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 2m ago 3h 500M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:02:33.866 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 2m ago 3h 620M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:02:33.866 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 2m ago 3h 552M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:02:33.867 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 2m ago 3h 650M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:02:33.867 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 2m ago 3h 192M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:02:34.219 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:02:34.219 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:02:34.219 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:02:34.219 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:02:34.220 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:02:34.220 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:02:34.220 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:02:34.220 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:02:34.221 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:02:34.221 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:02:34.221 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:02:34.221 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:02:34.221 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:02:34.222 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:02:34.222 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:02:34.222 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:02:34.222 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:02:34.223 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:02:34.596 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:02:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:02:34.324Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:02:34.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:02:34 smithi083 ceph-mon[79930]: pgmap v6430: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:34.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:02:34 smithi083 ceph-mon[79930]: from='client.51544 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:02:34.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:02:34 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/542404228' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:02:34.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:02:34 smithi083 ceph-mon[77242]: pgmap v6430: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:34.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:02:34 smithi083 ceph-mon[77242]: from='client.51544 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:02:34.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:02:34 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/542404228' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:02:35.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:02:34 smithi153 ceph-mon[70044]: pgmap v6430: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:35.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:02:34 smithi153 ceph-mon[70044]: from='client.51544 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:02:35.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:02:34 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/542404228' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:02:35.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:02:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:02:35.023Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:02:35.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:02:35 smithi083 ceph-mon[77242]: from='client.51550 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:02:35.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:02:35 smithi083 ceph-mon[79930]: from='client.51550 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:02:36.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:02:35 smithi153 ceph-mon[70044]: from='client.51550 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:02:36.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:02:36 smithi083 ceph-mon[79930]: pgmap v6431: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:36.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:02:36 smithi083 ceph-mon[77242]: pgmap v6431: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:37.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:02:36 smithi153 ceph-mon[70044]: pgmap v6431: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:38.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:02:38 smithi153 ceph-mon[70044]: pgmap v6432: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:38.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:02:38 smithi083 ceph-mon[79930]: pgmap v6432: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:38.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:02:38 smithi083 ceph-mon[77242]: pgmap v6432: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:40.941 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:02:40 smithi083 ceph-mon[79930]: pgmap v6433: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:40.942 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:02:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:02:40.942 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:02:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:02:40.942 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:02:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:02:40.943 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:02:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:02:40.943 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:02:40 smithi083 ceph-mon[77242]: pgmap v6433: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:40.943 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:02:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:02:40.944 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:02:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:02:40.944 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:02:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:02:40.944 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:02:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:02:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:02:40 smithi153 ceph-mon[70044]: pgmap v6433: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:02:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:02:41.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:02:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:02:41.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:02:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:02:41.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:02:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:02:41.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:02:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:02:40.939Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:02:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:02:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:02:41] "GET /metrics HTTP/1.1" 200 225222 "" "Prometheus/2.33.4" 2022-09-29T09:02:42.587 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:02:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:02:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:02:42.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:02:42 smithi083 ceph-mon[77242]: pgmap v6434: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:42.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:02:42 smithi083 ceph-mon[79930]: pgmap v6434: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:43.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:02:42 smithi153 ceph-mon[70044]: pgmap v6434: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:02:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:02:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:02:44.590 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:02:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:02:44.325Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:02:44.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:02:44 smithi083 ceph-mon[79930]: pgmap v6435: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:44.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:02:44 smithi083 ceph-mon[77242]: pgmap v6435: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:45.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:02:44 smithi153 ceph-mon[70044]: pgmap v6435: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:45.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:02:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:02:45.024Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:02:46.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:02:46 smithi083 ceph-mon[77242]: pgmap v6436: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:46.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:02:46 smithi083 ceph-mon[79930]: pgmap v6436: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:47.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:02:46 smithi153 ceph-mon[70044]: pgmap v6436: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:48.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:02:48 smithi153 ceph-mon[70044]: pgmap v6437: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:48.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:02:48 smithi083 ceph-mon[77242]: pgmap v6437: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:48.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:02:48 smithi083 ceph-mon[79930]: pgmap v6437: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:50.942 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:02:50 smithi083 ceph-mon[79930]: pgmap v6438: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:50.942 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:02:50 smithi083 ceph-mon[77242]: pgmap v6438: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:51.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:02:50 smithi153 ceph-mon[70044]: pgmap v6438: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:51.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:02:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:02:50.940Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:02:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:02:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:02:51] "GET /metrics HTTP/1.1" 200 225222 "" "Prometheus/2.33.4" 2022-09-29T09:02:52.589 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:02:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:02:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:02:52.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:02:52 smithi083 ceph-mon[77242]: pgmap v6439: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:52.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:02:52 smithi083 ceph-mon[79930]: pgmap v6439: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:53.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:02:52 smithi153 ceph-mon[70044]: pgmap v6439: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:02:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:02:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:02:54.592 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:02:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:02:54.325Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:02:54.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:02:54 smithi083 ceph-mon[79930]: pgmap v6440: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:54.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:02:54 smithi083 ceph-mon[77242]: pgmap v6440: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:55.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:02:54 smithi153 ceph-mon[70044]: pgmap v6440: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:02:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:02:55.025Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:02:56.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:02:56 smithi083 ceph-mon[77242]: pgmap v6441: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:56.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:02:56 smithi083 ceph-mon[79930]: pgmap v6441: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:57.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:02:56 smithi153 ceph-mon[70044]: pgmap v6441: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:58.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:02:58 smithi153 ceph-mon[70044]: pgmap v6442: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:58.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:02:58 smithi083 ceph-mon[77242]: pgmap v6442: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:02:58.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:02:58 smithi083 ceph-mon[79930]: pgmap v6442: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:00.942 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:03:00 smithi083 ceph-mon[79930]: pgmap v6443: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:00.943 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:03:00 smithi083 ceph-mon[77242]: pgmap v6443: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:01.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:03:00 smithi153 ceph-mon[70044]: pgmap v6443: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:01.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:03:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:03:00.940Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:03:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:03:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:03:01] "GET /metrics HTTP/1.1" 200 225216 "" "Prometheus/2.33.4" 2022-09-29T09:03:02.591 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:03:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:03:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:03:02.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:03:02 smithi083 ceph-mon[79930]: pgmap v6444: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:02.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:03:02 smithi083 ceph-mon[77242]: pgmap v6444: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:03.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:03:02 smithi153 ceph-mon[70044]: pgmap v6444: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:03:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:03:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:03:04.546 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:03:04.593 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:03:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:03:04.325Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:03:04.834 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:03:04.835 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 4m ago 3h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:03:04.835 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 2m ago 3h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:03:04.835 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 4m ago 3h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:03:04.836 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 2m ago 3h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:03:04.836 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 2m ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:03:04.836 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 4m ago 3h 697M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:03:04.836 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 4m ago 3h 817M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:03:04.836 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 2m ago 3h 359M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:03:04.837 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 4m ago 3h 477M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:03:04.837 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 4m ago 3h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:03:04.837 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 2m ago 3h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:03:04.837 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 4m ago 3h 590M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:03:04.838 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 4m ago 3h 581M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:03:04.838 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 4m ago 3h 456M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:03:04.838 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 4m ago 3h 565M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:03:04.838 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 2m ago 3h 500M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:03:04.839 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 2m ago 3h 620M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:03:04.839 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 2m ago 3h 552M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:03:04.839 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 2m ago 3h 650M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:03:04.839 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 2m ago 3h 192M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:03:04.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:03:04 smithi083 ceph-mon[79930]: pgmap v6445: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:04.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:03:04 smithi083 ceph-mon[77242]: pgmap v6445: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:05.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:03:04 smithi153 ceph-mon[70044]: pgmap v6445: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:05.191 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:03:05.191 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:03:05.192 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:03:05.192 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:03:05.192 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:03:05.192 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:03:05.192 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:03:05.193 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:03:05.193 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:03:05.193 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:03:05.193 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:03:05.193 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:03:05.194 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:03:05.194 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:03:05.194 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:03:05.194 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:03:05.195 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:03:05.195 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:03:05.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:03:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:03:05.026Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:03:05.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:03:05 smithi083 ceph-mon[77242]: from='client.51562 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:03:05.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:03:05 smithi083 ceph-mon[77242]: from='client.41637 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:03:05.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:03:05 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1757162474' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:03:05.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:03:05 smithi083 ceph-mon[79930]: from='client.51562 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:03:05.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:03:05 smithi083 ceph-mon[79930]: from='client.41637 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:03:05.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:03:05 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1757162474' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:03:06.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:03:05 smithi153 ceph-mon[70044]: from='client.51562 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:03:06.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:03:05 smithi153 ceph-mon[70044]: from='client.41637 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:03:06.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:03:05 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1757162474' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:03:06.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:03:06 smithi083 ceph-mon[77242]: pgmap v6446: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:06.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:03:06 smithi083 ceph-mon[79930]: pgmap v6446: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:07.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:03:06 smithi153 ceph-mon[70044]: pgmap v6446: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:08.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:03:08 smithi153 ceph-mon[70044]: pgmap v6447: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:08.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:03:08 smithi083 ceph-mon[77242]: pgmap v6447: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:08.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:03:08 smithi083 ceph-mon[79930]: pgmap v6447: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:10.944 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:03:10 smithi083 ceph-mon[79930]: pgmap v6448: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:10.944 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:03:10 smithi083 ceph-mon[77242]: pgmap v6448: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:11.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:03:10 smithi153 ceph-mon[70044]: pgmap v6448: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:11.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:03:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:03:10.942Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:03:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:03:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:03:11] "GET /metrics HTTP/1.1" 200 225218 "" "Prometheus/2.33.4" 2022-09-29T09:03:12.595 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:03:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:03:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:03:12.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:03:12 smithi083 ceph-mon[79930]: pgmap v6449: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:12.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:03:12 smithi083 ceph-mon[77242]: pgmap v6449: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:13.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:03:12 smithi153 ceph-mon[70044]: pgmap v6449: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:03:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:03:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:03:14.598 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:03:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:03:14.325Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:03:14.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:03:14 smithi083 ceph-mon[79930]: pgmap v6450: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:14.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:03:14 smithi083 ceph-mon[77242]: pgmap v6450: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:15.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:03:14 smithi153 ceph-mon[70044]: pgmap v6450: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:15.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:03:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:03:15.027Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:03:16.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:03:16 smithi083 ceph-mon[79930]: pgmap v6451: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:16.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:03:16 smithi083 ceph-mon[77242]: pgmap v6451: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:17.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:03:16 smithi153 ceph-mon[70044]: pgmap v6451: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:18.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:03:18 smithi153 ceph-mon[70044]: pgmap v6452: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:18.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:03:18 smithi083 ceph-mon[77242]: pgmap v6452: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:18.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:03:18 smithi083 ceph-mon[79930]: pgmap v6452: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:20.944 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:03:20 smithi083 ceph-mon[79930]: pgmap v6453: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:20.945 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:03:20 smithi083 ceph-mon[77242]: pgmap v6453: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:21.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:03:20 smithi153 ceph-mon[70044]: pgmap v6453: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:21.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:03:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:03:20.942Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:03:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:03:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:03:21] "GET /metrics HTTP/1.1" 200 225218 "" "Prometheus/2.33.4" 2022-09-29T09:03:22.600 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:03:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:03:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:03:22.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:03:22 smithi083 ceph-mon[77242]: pgmap v6454: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:22.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:03:22 smithi083 ceph-mon[79930]: pgmap v6454: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:23.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:03:22 smithi153 ceph-mon[70044]: pgmap v6454: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:03:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:03:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:03:24.602 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:03:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:03:24.326Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:03:24.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:03:24 smithi083 ceph-mon[79930]: pgmap v6455: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:24.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:03:24 smithi083 ceph-mon[77242]: pgmap v6455: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:25.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:03:24 smithi153 ceph-mon[70044]: pgmap v6455: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:25.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:03:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:03:25.028Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:03:26.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:03:26 smithi083 ceph-mon[77242]: pgmap v6456: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:26.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:03:26 smithi083 ceph-mon[79930]: pgmap v6456: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:27.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:03:26 smithi153 ceph-mon[70044]: pgmap v6456: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:28.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:03:28 smithi153 ceph-mon[70044]: pgmap v6457: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:28.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:03:28 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:03:28.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:03:28 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:03:28.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:03:28 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:03:28.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:03:28 smithi083 ceph-mon[77242]: pgmap v6457: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:28.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:03:28 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:03:28.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:03:28 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:03:28.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:03:28 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:03:28.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:03:28 smithi083 ceph-mon[79930]: pgmap v6457: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:28.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:03:28 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:03:28.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:03:28 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:03:28.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:03:28 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:03:29.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:03:29 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:03:29.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:03:29 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:03:30.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:03:29 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:03:30.945 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:03:30 smithi083 ceph-mon[79930]: pgmap v6458: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:30.945 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:03:30 smithi083 ceph-mon[77242]: pgmap v6458: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:31.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:03:30 smithi153 ceph-mon[70044]: pgmap v6458: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:31.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:03:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:03:30.942Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:03:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:03:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:03:31] "GET /metrics HTTP/1.1" 200 225220 "" "Prometheus/2.33.4" 2022-09-29T09:03:32.687 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:03:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:03:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:03:32.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:03:32 smithi083 ceph-mon[77242]: pgmap v6459: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:32.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:03:32 smithi083 ceph-mon[79930]: pgmap v6459: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:33.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:03:32 smithi153 ceph-mon[70044]: pgmap v6459: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:03:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:03:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:03:34.689 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:03:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:03:34.326Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:03:34.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:03:34 smithi083 ceph-mon[79930]: pgmap v6460: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:34.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:03:34 smithi083 ceph-mon[77242]: pgmap v6460: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:35.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:03:34 smithi153 ceph-mon[70044]: pgmap v6460: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:35.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:03:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:03:35.029Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:03:35.510 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:03:35.803 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:03:35.803 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 5m ago 3h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:03:35.803 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 3m ago 3h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:03:35.804 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 5m ago 3h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:03:35.804 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 3m ago 3h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:03:35.804 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 3m ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:03:35.804 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 5m ago 3h 697M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:03:35.804 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 5m ago 3h 817M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:03:35.805 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 3m ago 3h 359M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:03:35.805 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 5m ago 3h 477M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:03:35.805 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 5m ago 3h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:03:35.805 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 3m ago 3h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:03:35.805 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 5m ago 3h 590M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:03:35.806 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 5m ago 3h 581M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:03:35.806 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 5m ago 3h 456M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:03:35.806 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 5m ago 3h 565M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:03:35.806 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 3m ago 3h 500M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:03:35.807 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 3m ago 3h 620M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:03:35.807 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 3m ago 3h 552M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:03:35.807 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 3m ago 3h 650M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:03:35.807 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 3m ago 3h 192M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:03:36.156 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:03:36.157 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:03:36.157 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:03:36.157 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:03:36.157 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:03:36.158 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:03:36.158 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:03:36.158 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:03:36.158 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:03:36.159 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:03:36.159 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:03:36.159 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:03:36.159 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:03:36.160 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:03:36.160 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:03:36.160 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:03:36.160 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:03:36.160 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:03:36.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:03:36 smithi083 ceph-mon[77242]: from='client.41649 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:03:36.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:03:36 smithi083 ceph-mon[77242]: pgmap v6461: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:36.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:03:36 smithi083 ceph-mon[77242]: from='client.41655 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:03:36.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:03:36 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/451801326' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:03:36.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:03:36 smithi083 ceph-mon[79930]: from='client.41649 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:03:36.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:03:36 smithi083 ceph-mon[79930]: pgmap v6461: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:36.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:03:36 smithi083 ceph-mon[79930]: from='client.41655 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:03:36.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:03:36 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/451801326' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:03:37.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:03:36 smithi153 ceph-mon[70044]: from='client.41649 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:03:37.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:03:36 smithi153 ceph-mon[70044]: pgmap v6461: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:37.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:03:36 smithi153 ceph-mon[70044]: from='client.41655 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:03:37.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:03:36 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/451801326' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:03:38.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:03:38 smithi153 ceph-mon[70044]: pgmap v6462: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:38.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:03:38 smithi083 ceph-mon[79930]: pgmap v6462: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:38.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:03:38 smithi083 ceph-mon[77242]: pgmap v6462: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:40.945 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:03:40 smithi083 ceph-mon[77242]: pgmap v6463: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:40.945 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:03:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:03:40.946 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:03:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:03:40.946 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:03:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:03:40.946 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:03:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:03:40.947 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:03:40 smithi083 ceph-mon[79930]: pgmap v6463: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:40.947 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:03:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:03:40.947 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:03:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:03:40.947 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:03:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:03:40.948 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:03:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:03:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:03:40 smithi153 ceph-mon[70044]: pgmap v6463: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:03:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:03:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:03:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:03:41.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:03:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:03:41.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:03:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:03:41.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:03:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:03:40.943Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:03:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:03:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:03:41] "GET /metrics HTTP/1.1" 200 225219 "" "Prometheus/2.33.4" 2022-09-29T09:03:42.606 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:03:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:03:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:03:42.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:03:42 smithi083 ceph-mon[79930]: pgmap v6464: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:42.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:03:42 smithi083 ceph-mon[77242]: pgmap v6464: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:43.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:03:42 smithi153 ceph-mon[70044]: pgmap v6464: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:03:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:03:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:03:44.608 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:03:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:03:44.327Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:03:44.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:03:44 smithi083 ceph-mon[77242]: pgmap v6465: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:44.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:03:44 smithi083 ceph-mon[79930]: pgmap v6465: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:45.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:03:44 smithi153 ceph-mon[70044]: pgmap v6465: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:45.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:03:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:03:45.030Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:03:46.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:03:46 smithi083 ceph-mon[79930]: pgmap v6466: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:46.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:03:46 smithi083 ceph-mon[77242]: pgmap v6466: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:47.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:03:46 smithi153 ceph-mon[70044]: pgmap v6466: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:48.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:03:48 smithi153 ceph-mon[70044]: pgmap v6467: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:48.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:03:48 smithi083 ceph-mon[79930]: pgmap v6467: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:48.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:03:48 smithi083 ceph-mon[77242]: pgmap v6467: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:50.946 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:03:50 smithi083 ceph-mon[79930]: pgmap v6468: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:50.946 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:03:50 smithi083 ceph-mon[77242]: pgmap v6468: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:51.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:03:50 smithi153 ceph-mon[70044]: pgmap v6468: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:51.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:03:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:03:50.944Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:03:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:03:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:03:51] "GET /metrics HTTP/1.1" 200 225219 "" "Prometheus/2.33.4" 2022-09-29T09:03:52.607 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:03:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:03:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:03:52.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:03:52 smithi083 ceph-mon[79930]: pgmap v6469: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:52.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:03:52 smithi083 ceph-mon[77242]: pgmap v6469: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:53.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:03:52 smithi153 ceph-mon[70044]: pgmap v6469: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:03:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:03:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:03:54.610 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:03:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:03:54.327Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:03:54.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:03:54 smithi083 ceph-mon[77242]: pgmap v6470: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:54.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:03:54 smithi083 ceph-mon[79930]: pgmap v6470: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:55.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:03:54 smithi153 ceph-mon[70044]: pgmap v6470: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:03:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:03:55.030Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:03:56.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:03:56 smithi083 ceph-mon[79930]: pgmap v6471: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:56.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:03:56 smithi083 ceph-mon[77242]: pgmap v6471: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:57.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:03:56 smithi153 ceph-mon[70044]: pgmap v6471: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:58.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:03:58 smithi153 ceph-mon[70044]: pgmap v6472: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:58.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:03:58 smithi083 ceph-mon[77242]: pgmap v6472: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:03:58.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:03:58 smithi083 ceph-mon[79930]: pgmap v6472: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:00.947 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:04:00 smithi083 ceph-mon[79930]: pgmap v6473: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:00.948 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:04:00 smithi083 ceph-mon[77242]: pgmap v6473: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:01.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:04:00 smithi153 ceph-mon[70044]: pgmap v6473: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:01.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:04:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:04:00.944Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:04:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:04:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:04:01] "GET /metrics HTTP/1.1" 200 225219 "" "Prometheus/2.33.4" 2022-09-29T09:04:02.610 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:04:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:04:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:04:02.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:04:02 smithi083 ceph-mon[79930]: pgmap v6474: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:02.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:04:02 smithi083 ceph-mon[77242]: pgmap v6474: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:03.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:04:02 smithi153 ceph-mon[70044]: pgmap v6474: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:04:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:04:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:04:04.612 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:04:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:04:04.328Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:04:04.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:04:04 smithi083 ceph-mon[77242]: pgmap v6475: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:04.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:04:04 smithi083 ceph-mon[79930]: pgmap v6475: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:05.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:04:04 smithi153 ceph-mon[70044]: pgmap v6475: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:05.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:04:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:04:05.031Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:04:06.483 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:04:06.775 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:04:06.775 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 5m ago 3h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:04:06.776 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 3m ago 3h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:04:06.776 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 5m ago 3h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:04:06.776 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 3m ago 3h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:04:06.776 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 3m ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:04:06.776 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 5m ago 3h 697M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:04:06.777 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 5m ago 3h 817M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:04:06.777 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 3m ago 3h 359M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:04:06.777 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 5m ago 3h 477M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:04:06.777 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 5m ago 3h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:04:06.778 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 3m ago 3h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:04:06.778 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 5m ago 3h 590M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:04:06.778 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 5m ago 3h 581M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:04:06.778 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 5m ago 3h 456M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:04:06.778 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 5m ago 3h 565M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:04:06.779 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 3m ago 3h 500M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:04:06.779 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 3m ago 3h 620M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:04:06.779 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 3m ago 3h 552M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:04:06.779 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 3m ago 3h 650M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:04:06.780 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 3m ago 3h 192M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:04:06.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:04:06 smithi083 ceph-mon[79930]: pgmap v6476: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:06.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:04:06 smithi083 ceph-mon[77242]: pgmap v6476: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:07.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:04:06 smithi153 ceph-mon[70044]: pgmap v6476: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:07.132 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:04:07.132 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:04:07.132 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:04:07.132 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:04:07.133 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:04:07.133 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:04:07.133 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:04:07.133 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:04:07.134 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:04:07.134 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:04:07.134 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:04:07.134 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:04:07.134 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:04:07.135 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:04:07.135 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:04:07.135 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:04:07.135 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:04:07.136 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:04:07.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:04:07 smithi083 ceph-mon[79930]: from='client.41667 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:04:07.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:04:07 smithi083 ceph-mon[79930]: from='client.41673 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:04:07.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:04:07 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2684659642' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:04:07.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:04:07 smithi083 ceph-mon[77242]: from='client.41667 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:04:07.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:04:07 smithi083 ceph-mon[77242]: from='client.41673 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:04:07.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:04:07 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2684659642' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:04:08.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:04:07 smithi153 ceph-mon[70044]: from='client.41667 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:04:08.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:04:07 smithi153 ceph-mon[70044]: from='client.41673 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:04:08.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:04:07 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2684659642' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:04:08.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:04:08 smithi083 ceph-mon[79930]: pgmap v6477: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:08.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:04:08 smithi083 ceph-mon[77242]: pgmap v6477: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:09.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:04:08 smithi153 ceph-mon[70044]: pgmap v6477: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:10.948 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:04:10 smithi083 ceph-mon[79930]: pgmap v6478: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:10.949 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:04:10 smithi083 ceph-mon[77242]: pgmap v6478: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:11.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:04:10 smithi153 ceph-mon[70044]: pgmap v6478: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:11.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:04:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:04:10.946Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:04:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:04:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:04:11] "GET /metrics HTTP/1.1" 200 225215 "" "Prometheus/2.33.4" 2022-09-29T09:04:12.626 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:04:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:04:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:04:12.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:04:12 smithi083 ceph-mon[79930]: pgmap v6479: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:12.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:04:12 smithi083 ceph-mon[77242]: pgmap v6479: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:13.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:04:12 smithi153 ceph-mon[70044]: pgmap v6479: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:04:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:04:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:04:14.628 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:04:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:04:14.329Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:04:14.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:04:14 smithi083 ceph-mon[79930]: pgmap v6480: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:14.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:04:14 smithi083 ceph-mon[77242]: pgmap v6480: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:15.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:04:14 smithi153 ceph-mon[70044]: pgmap v6480: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:15.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:04:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:04:15.032Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:04:16.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:04:16 smithi083 ceph-mon[79930]: pgmap v6481: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:16.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:04:16 smithi083 ceph-mon[77242]: pgmap v6481: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:17.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:04:16 smithi153 ceph-mon[70044]: pgmap v6481: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:18.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:04:18 smithi153 ceph-mon[70044]: pgmap v6482: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:18.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:04:18 smithi083 ceph-mon[79930]: pgmap v6482: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:18.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:04:18 smithi083 ceph-mon[77242]: pgmap v6482: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:20.948 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:04:20 smithi083 ceph-mon[79930]: pgmap v6483: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:20.949 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:04:20 smithi083 ceph-mon[77242]: pgmap v6483: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:21.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:04:20 smithi153 ceph-mon[70044]: pgmap v6483: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:21.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:04:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:04:20.946Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:04:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:04:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:04:21] "GET /metrics HTTP/1.1" 200 225215 "" "Prometheus/2.33.4" 2022-09-29T09:04:22.619 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:04:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:04:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:04:22.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:04:22 smithi083 ceph-mon[77242]: pgmap v6484: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:22.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:04:22 smithi083 ceph-mon[79930]: pgmap v6484: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:23.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:04:22 smithi153 ceph-mon[70044]: pgmap v6484: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:04:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:04:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:04:24.622 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:04:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:04:24.328Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:04:24.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:04:24 smithi083 ceph-mon[79930]: pgmap v6485: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:24.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:04:24 smithi083 ceph-mon[77242]: pgmap v6485: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:25.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:04:24 smithi153 ceph-mon[70044]: pgmap v6485: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:25.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:04:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:04:25.033Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:04:26.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:04:26 smithi083 ceph-mon[79930]: pgmap v6486: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:26.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:04:26 smithi083 ceph-mon[77242]: pgmap v6486: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:27.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:04:26 smithi153 ceph-mon[70044]: pgmap v6486: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:28.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:04:28 smithi153 ceph-mon[70044]: pgmap v6487: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:28.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:04:28 smithi083 ceph-mon[79930]: pgmap v6487: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:28.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:04:28 smithi083 ceph-mon[77242]: pgmap v6487: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:29.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:04:29 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:04:29.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:04:29 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:04:29.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:04:29 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:04:29.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:04:29 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:04:29.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:04:29 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:04:29.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:04:29 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:04:29.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:04:29 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:04:29.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:04:29 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:04:29.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:04:29 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:04:29.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:04:29 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:04:29.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:04:29 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:04:29.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:04:29 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:04:30.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:04:30 smithi153 ceph-mon[70044]: pgmap v6488: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:30.948 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:04:30 smithi083 ceph-mon[79930]: pgmap v6488: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:30.949 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:04:30 smithi083 ceph-mon[77242]: pgmap v6488: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:31.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:04:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:04:30.946Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:04:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:04:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:04:31] "GET /metrics HTTP/1.1" 200 225216 "" "Prometheus/2.33.4" 2022-09-29T09:04:32.618 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:04:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:04:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:04:32.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:04:32 smithi083 ceph-mon[79930]: pgmap v6489: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:32.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:04:32 smithi083 ceph-mon[77242]: pgmap v6489: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:33.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:04:32 smithi153 ceph-mon[70044]: pgmap v6489: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:04:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:04:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:04:34.620 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:04:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:04:34.330Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:04:34.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:04:34 smithi083 ceph-mon[79930]: pgmap v6490: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:34.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:04:34 smithi083 ceph-mon[77242]: pgmap v6490: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:35.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:04:34 smithi153 ceph-mon[70044]: pgmap v6490: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:35.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:04:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:04:35.034Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:04:36.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:04:36 smithi083 ceph-mon[77242]: pgmap v6491: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:36.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:04:36 smithi083 ceph-mon[79930]: pgmap v6491: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:37.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:04:36 smithi153 ceph-mon[70044]: pgmap v6491: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:37.459 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:04:37.750 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:04:37.750 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 6m ago 3h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:04:37.750 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 4m ago 3h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:04:37.751 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 6m ago 3h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:04:37.751 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 4m ago 3h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:04:37.751 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 4m ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:04:37.752 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 6m ago 3h 697M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:04:37.752 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 6m ago 3h 817M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:04:37.752 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 4m ago 3h 359M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:04:37.752 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 6m ago 3h 477M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:04:37.752 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 6m ago 3h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:04:37.753 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 4m ago 3h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:04:37.753 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 6m ago 3h 590M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:04:37.753 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 6m ago 3h 581M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:04:37.753 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 6m ago 3h 456M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:04:37.754 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 6m ago 3h 565M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:04:37.754 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 4m ago 3h 500M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:04:37.754 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 4m ago 3h 620M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:04:37.755 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 4m ago 3h 552M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:04:37.755 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 4m ago 3h 650M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:04:37.755 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 4m ago 3h 192M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:04:38.109 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:04:38.110 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:04:38.110 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:04:38.110 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:04:38.110 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:04:38.111 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:04:38.111 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:04:38.111 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:04:38.111 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:04:38.111 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:04:38.112 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:04:38.112 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:04:38.112 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:04:38.112 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:04:38.113 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:04:38.113 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:04:38.113 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:04:38.113 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:04:38.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:04:38 smithi153 ceph-mon[70044]: from='client.51616 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:04:38.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:04:38 smithi153 ceph-mon[70044]: pgmap v6492: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:38.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:04:38 smithi153 ceph-mon[70044]: from='client.41691 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:04:38.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:04:38 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2642061322' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:04:38.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:04:38 smithi083 ceph-mon[79930]: from='client.51616 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:04:38.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:04:38 smithi083 ceph-mon[79930]: pgmap v6492: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:38.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:04:38 smithi083 ceph-mon[79930]: from='client.41691 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:04:38.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:04:38 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2642061322' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:04:38.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:04:38 smithi083 ceph-mon[77242]: from='client.51616 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:04:38.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:04:38 smithi083 ceph-mon[77242]: pgmap v6492: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:38.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:04:38 smithi083 ceph-mon[77242]: from='client.41691 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:04:38.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:04:38 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2642061322' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:04:40.950 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:04:40 smithi083 ceph-mon[79930]: pgmap v6493: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:40.950 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:04:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:04:40.950 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:04:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:04:40.951 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:04:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:04:40.951 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:04:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:04:40.951 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:04:40 smithi083 ceph-mon[77242]: pgmap v6493: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:40.952 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:04:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:04:40.952 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:04:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:04:40.952 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:04:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:04:40.952 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:04:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:04:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:04:40 smithi153 ceph-mon[70044]: pgmap v6493: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:04:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:04:41.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:04:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:04:41.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:04:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:04:41.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:04:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:04:41.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:04:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:04:40.947Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:04:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:04:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:04:41] "GET /metrics HTTP/1.1" 200 225221 "" "Prometheus/2.33.4" 2022-09-29T09:04:42.625 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:04:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:04:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:04:42.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:04:42 smithi083 ceph-mon[79930]: pgmap v6494: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:42.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:04:42 smithi083 ceph-mon[77242]: pgmap v6494: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:43.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:04:42 smithi153 ceph-mon[70044]: pgmap v6494: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:04:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:04:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:04:44.626 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:04:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:04:44.330Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:04:44.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:04:44 smithi083 ceph-mon[77242]: pgmap v6495: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:44.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:04:44 smithi083 ceph-mon[79930]: pgmap v6495: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:45.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:04:44 smithi153 ceph-mon[70044]: pgmap v6495: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:45.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:04:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:04:45.035Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:04:46.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:04:46 smithi083 ceph-mon[79930]: pgmap v6496: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:46.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:04:46 smithi083 ceph-mon[77242]: pgmap v6496: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:47.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:04:46 smithi153 ceph-mon[70044]: pgmap v6496: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:48.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:04:48 smithi153 ceph-mon[70044]: pgmap v6497: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:48.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:04:48 smithi083 ceph-mon[79930]: pgmap v6497: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:48.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:04:48 smithi083 ceph-mon[77242]: pgmap v6497: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:50.950 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:04:50 smithi083 ceph-mon[79930]: pgmap v6498: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:50.950 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:04:50 smithi083 ceph-mon[77242]: pgmap v6498: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:51.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:04:50 smithi153 ceph-mon[70044]: pgmap v6498: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:51.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:04:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:04:50.947Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:04:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:04:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:04:51] "GET /metrics HTTP/1.1" 200 225221 "" "Prometheus/2.33.4" 2022-09-29T09:04:52.626 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:04:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:04:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:04:52.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:04:52 smithi083 ceph-mon[79930]: pgmap v6499: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:52.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:04:52 smithi083 ceph-mon[77242]: pgmap v6499: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:53.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:04:52 smithi153 ceph-mon[70044]: pgmap v6499: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:04:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:04:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:04:54.628 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:04:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:04:54.331Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:04:54.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:04:54 smithi083 ceph-mon[79930]: pgmap v6500: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:54.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:04:54 smithi083 ceph-mon[77242]: pgmap v6500: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:55.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:04:54 smithi153 ceph-mon[70044]: pgmap v6500: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:04:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:04:55.035Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:04:56.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:04:56 smithi083 ceph-mon[79930]: pgmap v6501: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:56.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:04:56 smithi083 ceph-mon[77242]: pgmap v6501: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:57.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:04:56 smithi153 ceph-mon[70044]: pgmap v6501: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:58.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:04:58 smithi153 ceph-mon[70044]: pgmap v6502: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:58.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:04:58 smithi083 ceph-mon[79930]: pgmap v6502: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:04:58.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:04:58 smithi083 ceph-mon[77242]: pgmap v6502: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:00.951 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:05:00 smithi083 ceph-mon[79930]: pgmap v6503: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:00.951 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:05:00 smithi083 ceph-mon[77242]: pgmap v6503: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:01.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:05:00 smithi153 ceph-mon[70044]: pgmap v6503: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:01.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:05:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:05:00.949Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:05:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:05:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:05:01] "GET /metrics HTTP/1.1" 200 225219 "" "Prometheus/2.33.4" 2022-09-29T09:05:02.629 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:05:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:05:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:05:02.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:05:02 smithi083 ceph-mon[79930]: pgmap v6504: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:02.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:05:02 smithi083 ceph-mon[77242]: pgmap v6504: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:03.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:05:02 smithi153 ceph-mon[70044]: pgmap v6504: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:05:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:05:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:05:04.631 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:05:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:05:04.332Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:05:04.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:05:04 smithi083 ceph-mon[79930]: pgmap v6505: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:04.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:05:04 smithi083 ceph-mon[77242]: pgmap v6505: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:05.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:05:04 smithi153 ceph-mon[70044]: pgmap v6505: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:05.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:05:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:05:05.036Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:05:06.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:05:06 smithi083 ceph-mon[79930]: pgmap v6506: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:06.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:05:06 smithi083 ceph-mon[77242]: pgmap v6506: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:07.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:05:06 smithi153 ceph-mon[70044]: pgmap v6506: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:08.434 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:05:08.725 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:05:08.725 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 6m ago 3h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:05:08.726 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 4m ago 3h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:05:08.726 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 6m ago 3h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:05:08.726 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 4m ago 3h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:05:08.726 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 4m ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:05:08.727 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 6m ago 3h 697M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:05:08.727 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 6m ago 3h 817M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:05:08.727 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 4m ago 3h 359M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:05:08.727 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 6m ago 3h 477M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:05:08.727 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 6m ago 3h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:05:08.728 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 4m ago 3h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:05:08.728 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 6m ago 3h 590M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:05:08.728 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 6m ago 3h 581M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:05:08.728 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 6m ago 3h 456M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:05:08.728 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 6m ago 3h 565M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:05:08.729 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 4m ago 3h 500M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:05:08.729 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 4m ago 3h 620M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:05:08.729 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 4m ago 3h 552M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:05:08.729 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 4m ago 3h 650M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:05:08.729 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 4m ago 3h 192M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:05:08.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:05:08 smithi153 ceph-mon[70044]: pgmap v6507: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:08.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:05:08 smithi083 ceph-mon[79930]: pgmap v6507: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:08.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:05:08 smithi083 ceph-mon[77242]: pgmap v6507: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:09.079 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:05:09.080 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:05:09.080 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:05:09.080 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:05:09.080 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:05:09.081 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:05:09.081 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:05:09.081 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:05:09.081 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:05:09.081 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:05:09.082 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:05:09.082 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:05:09.082 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:05:09.082 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:05:09.082 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:05:09.083 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:05:09.083 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:05:09.083 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:05:09.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:05:09 smithi153 ceph-mon[70044]: from='client.51634 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:05:09.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:05:09 smithi153 ceph-mon[70044]: from='client.51640 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:05:09.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:05:09 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3495717525' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:05:09.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:05:09 smithi083 ceph-mon[79930]: from='client.51634 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:05:09.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:05:09 smithi083 ceph-mon[79930]: from='client.51640 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:05:09.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:05:09 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3495717525' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:05:09.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:05:09 smithi083 ceph-mon[77242]: from='client.51634 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:05:09.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:05:09 smithi083 ceph-mon[77242]: from='client.51640 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:05:09.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:05:09 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3495717525' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:05:10.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:05:10 smithi153 ceph-mon[70044]: pgmap v6508: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:10.951 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:05:10 smithi083 ceph-mon[79930]: pgmap v6508: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:10.952 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:05:10 smithi083 ceph-mon[77242]: pgmap v6508: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:11.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:05:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:05:10.949Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:05:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:05:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:05:11] "GET /metrics HTTP/1.1" 200 225219 "" "Prometheus/2.33.4" 2022-09-29T09:05:12.630 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:05:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:05:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:05:12.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:05:12 smithi083 ceph-mon[79930]: pgmap v6509: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:12.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:05:12 smithi083 ceph-mon[77242]: pgmap v6509: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:13.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:05:12 smithi153 ceph-mon[70044]: pgmap v6509: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:05:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:05:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:05:14.633 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:05:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:05:14.333Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:05:14.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:05:14 smithi083 ceph-mon[79930]: pgmap v6510: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:14.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:05:14 smithi083 ceph-mon[77242]: pgmap v6510: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:15.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:05:14 smithi153 ceph-mon[70044]: pgmap v6510: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:15.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:05:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:05:15.036Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:05:16.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:05:16 smithi083 ceph-mon[79930]: pgmap v6511: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:16.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:05:16 smithi083 ceph-mon[77242]: pgmap v6511: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:17.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:05:16 smithi153 ceph-mon[70044]: pgmap v6511: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:18.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:05:18 smithi153 ceph-mon[70044]: pgmap v6512: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:18.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:05:18 smithi083 ceph-mon[79930]: pgmap v6512: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:18.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:05:18 smithi083 ceph-mon[77242]: pgmap v6512: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:20.952 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:05:20 smithi083 ceph-mon[79930]: pgmap v6513: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:20.953 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:05:20 smithi083 ceph-mon[77242]: pgmap v6513: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:21.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:05:20 smithi153 ceph-mon[70044]: pgmap v6513: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:21.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:05:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:05:20.950Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:05:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:05:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:05:21] "GET /metrics HTTP/1.1" 200 225219 "" "Prometheus/2.33.4" 2022-09-29T09:05:22.636 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:05:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:05:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:05:22.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:05:22 smithi083 ceph-mon[79930]: pgmap v6514: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:22.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:05:22 smithi083 ceph-mon[77242]: pgmap v6514: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:23.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:05:22 smithi153 ceph-mon[70044]: pgmap v6514: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:05:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:05:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:05:24.638 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:05:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:05:24.333Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:05:24.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:05:24 smithi083 ceph-mon[79930]: pgmap v6515: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:24.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:05:24 smithi083 ceph-mon[77242]: pgmap v6515: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:25.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:05:24 smithi153 ceph-mon[70044]: pgmap v6515: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:25.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:05:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:05:25.037Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:05:26.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:05:26 smithi083 ceph-mon[79930]: pgmap v6516: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:26.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:05:26 smithi083 ceph-mon[77242]: pgmap v6516: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:27.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:05:26 smithi153 ceph-mon[70044]: pgmap v6516: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:28.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:05:28 smithi153 ceph-mon[70044]: pgmap v6517: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:28.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:05:28 smithi083 ceph-mon[79930]: pgmap v6517: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:28.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:05:28 smithi083 ceph-mon[77242]: pgmap v6517: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:29.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:05:29 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:05:29.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:05:29 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:05:29.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:05:29 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:05:29.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:05:29 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:05:29.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:05:29 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:05:29.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:05:29 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:05:29.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:05:29 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:05:29.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:05:29 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:05:29.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:05:29 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:05:30.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:05:30 smithi153 ceph-mon[70044]: pgmap v6518: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:30.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:05:30 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:05:30.953 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:05:30 smithi083 ceph-mon[79930]: pgmap v6518: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:30.953 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:05:30 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:05:30.953 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:05:30 smithi083 ceph-mon[77242]: pgmap v6518: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:30.954 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:05:30 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:05:31.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:05:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:05:30.951Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:05:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:05:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:05:31] "GET /metrics HTTP/1.1" 200 225216 "" "Prometheus/2.33.4" 2022-09-29T09:05:32.637 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:05:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:05:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:05:32.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:05:32 smithi083 ceph-mon[79930]: pgmap v6519: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:32.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:05:32 smithi083 ceph-mon[77242]: pgmap v6519: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:33.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:05:32 smithi153 ceph-mon[70044]: pgmap v6519: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:05:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:05:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:05:34.640 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:05:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:05:34.333Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:05:34.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:05:34 smithi083 ceph-mon[79930]: pgmap v6520: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:34.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:05:34 smithi083 ceph-mon[77242]: pgmap v6520: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:35.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:05:34 smithi153 ceph-mon[70044]: pgmap v6520: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:35.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:05:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:05:35.037Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:05:36.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:05:36 smithi083 ceph-mon[79930]: pgmap v6521: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:36.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:05:36 smithi083 ceph-mon[77242]: pgmap v6521: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:37.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:05:36 smithi153 ceph-mon[70044]: pgmap v6521: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:38.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:05:38 smithi153 ceph-mon[70044]: pgmap v6522: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:38.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:05:38 smithi083 ceph-mon[77242]: pgmap v6522: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:38.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:05:38 smithi083 ceph-mon[79930]: pgmap v6522: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:39.408 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:05:39.697 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:05:39.698 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 7m ago 3h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:05:39.698 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 5m ago 3h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:05:39.698 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 7m ago 3h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:05:39.698 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 5m ago 3h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:05:39.699 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 5m ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:05:39.699 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 7m ago 3h 697M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:05:39.699 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 7m ago 3h 817M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:05:39.699 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 5m ago 3h 359M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:05:39.700 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 7m ago 3h 477M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:05:39.700 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 7m ago 3h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:05:39.700 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 5m ago 3h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:05:39.700 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 7m ago 3h 590M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:05:39.701 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 7m ago 3h 581M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:05:39.701 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 7m ago 3h 456M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:05:39.701 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 7m ago 3h 565M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:05:39.701 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 5m ago 3h 500M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:05:39.701 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 5m ago 3h 620M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:05:39.702 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 5m ago 3h 552M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:05:39.702 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 5m ago 3h 650M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:05:39.702 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 5m ago 3h 192M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:05:40.055 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:05:40.055 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:05:40.055 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:05:40.056 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:05:40.056 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:05:40.056 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:05:40.056 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:05:40.057 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:05:40.057 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:05:40.057 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:05:40.057 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:05:40.058 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:05:40.058 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:05:40.058 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:05:40.058 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:05:40.059 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:05:40.059 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:05:40.059 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:05:40.953 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:05:40 smithi083 ceph-mon[79930]: from='client.41721 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:05:40.954 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:05:40 smithi083 ceph-mon[79930]: pgmap v6523: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:40.954 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:05:40 smithi083 ceph-mon[79930]: from='client.51658 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:05:40.954 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:05:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:05:40.954 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:05:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:05:40.954 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:05:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:05:40.955 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:05:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:05:40.955 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:05:40 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1221944680' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:05:40.955 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:05:40 smithi083 ceph-mon[77242]: from='client.41721 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:05:40.955 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:05:40 smithi083 ceph-mon[77242]: pgmap v6523: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:40.956 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:05:40 smithi083 ceph-mon[77242]: from='client.51658 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:05:40.956 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:05:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:05:40.956 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:05:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:05:40.956 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:05:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:05:40.957 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:05:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:05:40.957 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:05:40 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1221944680' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:05:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:05:40 smithi153 ceph-mon[70044]: from='client.41721 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:05:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:05:40 smithi153 ceph-mon[70044]: pgmap v6523: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:41.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:05:40 smithi153 ceph-mon[70044]: from='client.51658 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:05:41.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:05:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:05:41.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:05:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:05:41.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:05:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:05:41.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:05:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:05:41.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:05:40 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1221944680' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:05:41.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:05:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:05:40.951Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:05:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:05:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:05:41] "GET /metrics HTTP/1.1" 200 225220 "" "Prometheus/2.33.4" 2022-09-29T09:05:42.643 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:05:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:05:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:05:42.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:05:42 smithi083 ceph-mon[77242]: pgmap v6524: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:42.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:05:42 smithi083 ceph-mon[79930]: pgmap v6524: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:43.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:05:42 smithi153 ceph-mon[70044]: pgmap v6524: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:05:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:05:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:05:44.645 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:05:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:05:44.334Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:05:44.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:05:44 smithi083 ceph-mon[79930]: pgmap v6525: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:44.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:05:44 smithi083 ceph-mon[77242]: pgmap v6525: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:45.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:05:44 smithi153 ceph-mon[70044]: pgmap v6525: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:45.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:05:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:05:45.038Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:05:46.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:05:46 smithi083 ceph-mon[79930]: pgmap v6526: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:46.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:05:46 smithi083 ceph-mon[77242]: pgmap v6526: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:47.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:05:46 smithi153 ceph-mon[70044]: pgmap v6526: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:48.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:05:48 smithi153 ceph-mon[70044]: pgmap v6527: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:48.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:05:48 smithi083 ceph-mon[77242]: pgmap v6527: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:48.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:05:48 smithi083 ceph-mon[79930]: pgmap v6527: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:50.953 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:05:50 smithi083 ceph-mon[79930]: pgmap v6528: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:50.954 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:05:50 smithi083 ceph-mon[77242]: pgmap v6528: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:51.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:05:50 smithi153 ceph-mon[70044]: pgmap v6528: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:51.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:05:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:05:50.951Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:05:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:05:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:05:51] "GET /metrics HTTP/1.1" 200 225220 "" "Prometheus/2.33.4" 2022-09-29T09:05:52.645 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:05:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:05:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:05:52.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:05:52 smithi083 ceph-mon[77242]: pgmap v6529: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:52.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:05:52 smithi083 ceph-mon[79930]: pgmap v6529: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:53.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:05:52 smithi153 ceph-mon[70044]: pgmap v6529: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:05:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:05:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:05:54.647 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:05:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:05:54.334Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:05:54.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:05:54 smithi083 ceph-mon[79930]: pgmap v6530: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:54.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:05:54 smithi083 ceph-mon[77242]: pgmap v6530: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:55.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:05:54 smithi153 ceph-mon[70044]: pgmap v6530: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:05:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:05:55.038Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:05:56.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:05:56 smithi083 ceph-mon[79930]: pgmap v6531: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:56.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:05:56 smithi083 ceph-mon[77242]: pgmap v6531: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:57.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:05:56 smithi153 ceph-mon[70044]: pgmap v6531: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:58.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:05:58 smithi153 ceph-mon[70044]: pgmap v6532: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:58.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:05:58 smithi083 ceph-mon[79930]: pgmap v6532: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:05:58.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:05:58 smithi083 ceph-mon[77242]: pgmap v6532: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:00.955 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:06:00 smithi083 ceph-mon[79930]: pgmap v6533: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:00.955 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:06:00 smithi083 ceph-mon[77242]: pgmap v6533: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:01.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:06:00 smithi153 ceph-mon[70044]: pgmap v6533: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:01.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:06:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:06:00.952Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:06:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:06:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:06:01] "GET /metrics HTTP/1.1" 200 225221 "" "Prometheus/2.33.4" 2022-09-29T09:06:02.648 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:06:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:06:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:06:02.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:06:02 smithi083 ceph-mon[79930]: pgmap v6534: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:02.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:06:02 smithi083 ceph-mon[77242]: pgmap v6534: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:03.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:06:02 smithi153 ceph-mon[70044]: pgmap v6534: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:06:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:06:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:06:04.651 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:06:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:06:04.334Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:06:04.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:06:04 smithi083 ceph-mon[77242]: pgmap v6535: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:04.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:06:04 smithi083 ceph-mon[79930]: pgmap v6535: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:05.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:06:04 smithi153 ceph-mon[70044]: pgmap v6535: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:05.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:06:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:06:05.039Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:06:06.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:06:06 smithi083 ceph-mon[79930]: pgmap v6536: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:06.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:06:06 smithi083 ceph-mon[77242]: pgmap v6536: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:07.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:06:06 smithi153 ceph-mon[70044]: pgmap v6536: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:08.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:06:08 smithi153 ceph-mon[70044]: pgmap v6537: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:08.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:06:08 smithi083 ceph-mon[79930]: pgmap v6537: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:08.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:06:08 smithi083 ceph-mon[77242]: pgmap v6537: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:10.384 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:06:10.675 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:06:10.675 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 7m ago 3h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:06:10.675 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 5m ago 3h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:06:10.676 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 7m ago 3h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:06:10.676 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 5m ago 3h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:06:10.676 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 5m ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:06:10.676 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 7m ago 3h 697M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:06:10.677 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 7m ago 3h 817M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:06:10.677 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 5m ago 3h 359M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:06:10.677 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 7m ago 3h 477M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:06:10.677 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 7m ago 3h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:06:10.678 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 5m ago 3h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:06:10.678 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 7m ago 3h 590M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:06:10.678 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 7m ago 3h 581M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:06:10.679 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 7m ago 3h 456M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:06:10.679 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 7m ago 3h 565M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:06:10.679 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 5m ago 3h 500M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:06:10.679 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 5m ago 3h 620M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:06:10.680 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 5m ago 3h 552M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:06:10.680 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 5m ago 3h 650M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:06:10.680 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 5m ago 3h 192M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:06:10.955 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:06:10 smithi083 ceph-mon[79930]: pgmap v6538: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:10.955 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:06:10 smithi083 ceph-mon[77242]: pgmap v6538: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:11.030 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:06:11.031 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:06:11.031 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:06:11.032 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:06:11.032 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:06:11.032 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:06:11.032 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:06:11.033 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:06:11.033 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:06:11.033 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:06:11.034 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:06:11.034 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:06:11.034 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:06:11.035 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:06:11.035 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:06:11.035 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:06:11.035 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:06:11.036 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:06:11.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:06:10 smithi153 ceph-mon[70044]: pgmap v6538: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:11.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:06:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:06:10.952Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:06:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:06:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:06:11] "GET /metrics HTTP/1.1" 200 225219 "" "Prometheus/2.33.4" 2022-09-29T09:06:11.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:06:11 smithi083 ceph-mon[79930]: from='client.41739 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:06:11.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:06:11 smithi083 ceph-mon[79930]: from='client.41745 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:06:11.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:06:11 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2237517795' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:06:11.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:06:11 smithi083 ceph-mon[77242]: from='client.41739 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:06:11.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:06:11 smithi083 ceph-mon[77242]: from='client.41745 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:06:11.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:06:11 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2237517795' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:06:12.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:06:11 smithi153 ceph-mon[70044]: from='client.41739 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:06:12.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:06:11 smithi153 ceph-mon[70044]: from='client.41745 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:06:12.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:06:11 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2237517795' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:06:12.654 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:06:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:06:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:06:12.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:06:12 smithi083 ceph-mon[79930]: pgmap v6539: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:12.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:06:12 smithi083 ceph-mon[77242]: pgmap v6539: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:13.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:06:12 smithi153 ceph-mon[70044]: pgmap v6539: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:06:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:06:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:06:14.657 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:06:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:06:14.336Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:06:14.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:06:14 smithi083 ceph-mon[79930]: pgmap v6540: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:14.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:06:14 smithi083 ceph-mon[77242]: pgmap v6540: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:15.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:06:14 smithi153 ceph-mon[70044]: pgmap v6540: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:15.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:06:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:06:15.040Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:06:16.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:06:16 smithi083 ceph-mon[79930]: pgmap v6541: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:16.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:06:16 smithi083 ceph-mon[77242]: pgmap v6541: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:17.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:06:16 smithi153 ceph-mon[70044]: pgmap v6541: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:18.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:06:18 smithi153 ceph-mon[70044]: pgmap v6542: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:18.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:06:18 smithi083 ceph-mon[77242]: pgmap v6542: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:18.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:06:18 smithi083 ceph-mon[79930]: pgmap v6542: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:20.956 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:06:20 smithi083 ceph-mon[79930]: pgmap v6543: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:20.956 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:06:20 smithi083 ceph-mon[77242]: pgmap v6543: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:21.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:06:20 smithi153 ceph-mon[70044]: pgmap v6543: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:21.215 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:06:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:06:20.953Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:06:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:06:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:06:21] "GET /metrics HTTP/1.1" 200 225219 "" "Prometheus/2.33.4" 2022-09-29T09:06:22.654 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:06:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:06:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:06:22.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:06:22 smithi083 ceph-mon[79930]: pgmap v6544: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:22.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:06:22 smithi083 ceph-mon[77242]: pgmap v6544: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:23.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:06:22 smithi153 ceph-mon[70044]: pgmap v6544: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:06:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:06:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:06:24.657 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:06:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:06:24.336Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:06:24.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:06:24 smithi083 ceph-mon[79930]: pgmap v6545: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:24.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:06:24 smithi083 ceph-mon[77242]: pgmap v6545: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:25.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:06:24 smithi153 ceph-mon[70044]: pgmap v6545: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:25.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:06:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:06:25.041Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:06:26.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:06:26 smithi083 ceph-mon[79930]: pgmap v6546: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:26.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:06:26 smithi083 ceph-mon[77242]: pgmap v6546: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:27.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:06:26 smithi153 ceph-mon[70044]: pgmap v6546: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:28.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:06:28 smithi153 ceph-mon[70044]: pgmap v6547: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:28.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:06:28 smithi083 ceph-mon[77242]: pgmap v6547: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:28.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:06:28 smithi083 ceph-mon[79930]: pgmap v6547: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:30.955 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:06:30 smithi083 ceph-mon[79930]: pgmap v6548: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:30.956 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:06:30 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:06:30.956 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:06:30 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:06:30.956 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:06:30 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:06:30.957 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:06:30 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:06:30.957 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:06:30 smithi083 ceph-mon[77242]: pgmap v6548: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:30.957 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:06:30 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:06:30.957 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:06:30 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:06:30.958 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:06:30 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:06:30.958 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:06:30 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:06:31.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:06:30 smithi153 ceph-mon[70044]: pgmap v6548: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:31.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:06:30 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:06:31.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:06:30 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:06:31.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:06:30 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:06:31.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:06:30 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:06:31.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:06:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:06:30.953Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:06:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:06:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:06:31] "GET /metrics HTTP/1.1" 200 225222 "" "Prometheus/2.33.4" 2022-09-29T09:06:32.658 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:06:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:06:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:06:32.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:06:32 smithi083 ceph-mon[79930]: pgmap v6549: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:32.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:06:32 smithi083 ceph-mon[77242]: pgmap v6549: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:33.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:06:32 smithi153 ceph-mon[70044]: pgmap v6549: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:33.311 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:06:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:06:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:06:34.661 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:06:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:06:34.337Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:06:34.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:06:34 smithi083 ceph-mon[79930]: pgmap v6550: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:34.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:06:34 smithi083 ceph-mon[77242]: pgmap v6550: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:35.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:06:34 smithi153 ceph-mon[70044]: pgmap v6550: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:35.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:06:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:06:35.041Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:06:36.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:06:36 smithi083 ceph-mon[79930]: pgmap v6551: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:36.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:06:36 smithi083 ceph-mon[77242]: pgmap v6551: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:37.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:06:36 smithi153 ceph-mon[70044]: pgmap v6551: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:38.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:06:38 smithi153 ceph-mon[70044]: pgmap v6552: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:38.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:06:38 smithi083 ceph-mon[77242]: pgmap v6552: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:38.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:06:38 smithi083 ceph-mon[79930]: pgmap v6552: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:40.957 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:06:40 smithi083 ceph-mon[79930]: pgmap v6553: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:40.957 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:06:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:06:40.958 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:06:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:06:40.958 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:06:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:06:40.958 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:06:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:06:40.958 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:06:40 smithi083 ceph-mon[77242]: pgmap v6553: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:40.959 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:06:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:06:40.959 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:06:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:06:40.959 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:06:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:06:40.959 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:06:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:06:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:06:40 smithi153 ceph-mon[70044]: pgmap v6553: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:41.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:06:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:06:41.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:06:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:06:41.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:06:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:06:41.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:06:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:06:41.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:06:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:06:40.955Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:06:41.357 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:06:41.648 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:06:41.649 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 8m ago 3h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:06:41.649 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 6m ago 3h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:06:41.649 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 8m ago 3h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:06:41.649 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 6m ago 3h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:06:41.650 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 6m ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:06:41.650 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 8m ago 3h 697M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:06:41.650 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 8m ago 3h 817M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:06:41.650 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 6m ago 3h 359M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:06:41.650 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 8m ago 3h 477M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:06:41.651 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 8m ago 3h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:06:41.651 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 6m ago 3h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:06:41.651 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 8m ago 3h 590M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:06:41.651 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 8m ago 3h 581M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:06:41.651 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 8m ago 3h 456M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:06:41.652 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 8m ago 3h 565M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:06:41.652 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 6m ago 3h 500M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:06:41.652 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 6m ago 3h 620M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:06:41.652 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 6m ago 3h 552M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:06:41.652 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 6m ago 3h 650M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:06:41.653 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 6m ago 3h 192M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:06:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:06:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:06:41] "GET /metrics HTTP/1.1" 200 225215 "" "Prometheus/2.33.4" 2022-09-29T09:06:42.004 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:06:42.004 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:06:42.004 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:06:42.005 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:06:42.005 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:06:42.005 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:06:42.005 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:06:42.005 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:06:42.006 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:06:42.006 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:06:42.006 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:06:42.006 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:06:42.006 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:06:42.007 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:06:42.007 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:06:42.007 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:06:42.007 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:06:42.008 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:06:42.662 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:06:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:06:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:06:42.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:06:42 smithi083 ceph-mon[79930]: from='client.51688 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:06:42.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:06:42 smithi083 ceph-mon[79930]: pgmap v6554: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:42.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:06:42 smithi083 ceph-mon[79930]: from='client.41763 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:06:42.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:06:42 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4049589981' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:06:42.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:06:42 smithi083 ceph-mon[77242]: from='client.51688 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:06:42.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:06:42 smithi083 ceph-mon[77242]: pgmap v6554: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:42.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:06:42 smithi083 ceph-mon[77242]: from='client.41763 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:06:42.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:06:42 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4049589981' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:06:43.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:06:42 smithi153 ceph-mon[70044]: from='client.51688 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:06:43.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:06:42 smithi153 ceph-mon[70044]: pgmap v6554: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:43.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:06:42 smithi153 ceph-mon[70044]: from='client.41763 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:06:43.026 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:06:42 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4049589981' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:06:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:06:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:06:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:06:44.664 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:06:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:06:44.338Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:06:44.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:06:44 smithi083 ceph-mon[77242]: pgmap v6555: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:44.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:06:44 smithi083 ceph-mon[79930]: pgmap v6555: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:45.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:06:44 smithi153 ceph-mon[70044]: pgmap v6555: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:45.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:06:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:06:45.041Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:06:46.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:06:46 smithi083 ceph-mon[79930]: pgmap v6556: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:46.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:06:46 smithi083 ceph-mon[77242]: pgmap v6556: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:47.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:06:46 smithi153 ceph-mon[70044]: pgmap v6556: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:48.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:06:48 smithi153 ceph-mon[70044]: pgmap v6557: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:48.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:06:48 smithi083 ceph-mon[79930]: pgmap v6557: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:48.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:06:48 smithi083 ceph-mon[77242]: pgmap v6557: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:50.958 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:06:50 smithi083 ceph-mon[79930]: pgmap v6558: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:50.959 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:06:50 smithi083 ceph-mon[77242]: pgmap v6558: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:51.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:06:50 smithi153 ceph-mon[70044]: pgmap v6558: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:51.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:06:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:06:50.956Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:06:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:06:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:06:51] "GET /metrics HTTP/1.1" 200 225215 "" "Prometheus/2.33.4" 2022-09-29T09:06:52.663 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:06:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:06:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:06:52.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:06:52 smithi083 ceph-mon[77242]: pgmap v6559: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:52.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:06:52 smithi083 ceph-mon[79930]: pgmap v6559: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:53.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:06:52 smithi153 ceph-mon[70044]: pgmap v6559: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:06:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:06:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:06:54.665 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:06:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:06:54.337Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:06:54.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:06:54 smithi083 ceph-mon[77242]: pgmap v6560: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:54.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:06:54 smithi083 ceph-mon[79930]: pgmap v6560: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:55.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:06:54 smithi153 ceph-mon[70044]: pgmap v6560: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:06:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:06:55.042Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:06:56.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:06:56 smithi083 ceph-mon[77242]: pgmap v6561: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:56.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:06:56 smithi083 ceph-mon[79930]: pgmap v6561: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:57.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:06:56 smithi153 ceph-mon[70044]: pgmap v6561: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:58.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:06:58 smithi153 ceph-mon[70044]: pgmap v6562: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:58.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:06:58 smithi083 ceph-mon[79930]: pgmap v6562: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:06:58.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:06:58 smithi083 ceph-mon[77242]: pgmap v6562: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:00.958 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:07:00 smithi083 ceph-mon[77242]: pgmap v6563: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:00.959 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:07:00 smithi083 ceph-mon[79930]: pgmap v6563: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:01.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:07:00 smithi153 ceph-mon[70044]: pgmap v6563: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:01.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:07:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:07:00.956Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:07:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:07:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:07:01] "GET /metrics HTTP/1.1" 200 225219 "" "Prometheus/2.33.4" 2022-09-29T09:07:02.669 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:07:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:07:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:07:02.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:07:02 smithi083 ceph-mon[79930]: pgmap v6564: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:02.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:07:02 smithi083 ceph-mon[77242]: pgmap v6564: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:03.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:07:02 smithi153 ceph-mon[70044]: pgmap v6564: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:07:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:07:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:07:04.670 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:07:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:07:04.338Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:07:04.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:07:04 smithi083 ceph-mon[77242]: pgmap v6565: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:04.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:07:04 smithi083 ceph-mon[79930]: pgmap v6565: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:05.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:07:04 smithi153 ceph-mon[70044]: pgmap v6565: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:05.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:07:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:07:05.042Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:07:06.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:07:06 smithi083 ceph-mon[77242]: pgmap v6566: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:06.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:07:06 smithi083 ceph-mon[79930]: pgmap v6566: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:07.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:07:06 smithi153 ceph-mon[70044]: pgmap v6566: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:08.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:07:08 smithi153 ceph-mon[70044]: pgmap v6567: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:08.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:07:08 smithi083 ceph-mon[77242]: pgmap v6567: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:08.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:07:08 smithi083 ceph-mon[79930]: pgmap v6567: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:10.959 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:07:10 smithi083 ceph-mon[79930]: pgmap v6568: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:10.959 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:07:10 smithi083 ceph-mon[77242]: pgmap v6568: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:11.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:07:10 smithi153 ceph-mon[70044]: pgmap v6568: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:11.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:07:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:07:10.957Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:07:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:07:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:07:11] "GET /metrics HTTP/1.1" 200 225217 "" "Prometheus/2.33.4" 2022-09-29T09:07:12.330 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:07:12.618 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:07:12.619 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 8m ago 3h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:07:12.619 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 6m ago 3h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:07:12.619 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 8m ago 3h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:07:12.619 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 6m ago 3h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:07:12.620 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 6m ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:07:12.620 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 8m ago 3h 697M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:07:12.620 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 8m ago 3h 817M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:07:12.620 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 6m ago 3h 359M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:07:12.621 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 8m ago 3h 477M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:07:12.621 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 8m ago 3h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:07:12.621 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 6m ago 3h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:07:12.621 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 8m ago 3h 590M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:07:12.622 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 8m ago 3h 581M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:07:12.622 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 8m ago 3h 456M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:07:12.622 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 8m ago 3h 565M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:07:12.622 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 6m ago 3h 500M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:07:12.623 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 6m ago 3h 620M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:07:12.623 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 6m ago 3h 552M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:07:12.623 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 6m ago 3h 650M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:07:12.623 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 6m ago 3h 192M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:07:12.671 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:07:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:07:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:07:12.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:07:12 smithi083 ceph-mon[79930]: pgmap v6569: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:12.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:07:12 smithi083 ceph-mon[77242]: pgmap v6569: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:12.977 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:07:12.977 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:07:12.978 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:07:12.978 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:07:12.978 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:07:12.978 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:07:12.979 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:07:12.979 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:07:12.979 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:07:12.980 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:07:12.980 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:07:12.980 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:07:12.980 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:07:12.981 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:07:12.981 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:07:12.981 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:07:12.981 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:07:12.982 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:07:13.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:07:12 smithi153 ceph-mon[70044]: pgmap v6569: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:07:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:07:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:07:13.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:07:13 smithi083 ceph-mon[77242]: from='client.41775 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:07:13.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:07:13 smithi083 ceph-mon[77242]: from='client.51712 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:07:13.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:07:13 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1540815667' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:07:13.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:07:13 smithi083 ceph-mon[79930]: from='client.41775 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:07:13.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:07:13 smithi083 ceph-mon[79930]: from='client.51712 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:07:13.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:07:13 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1540815667' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:07:14.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:07:13 smithi153 ceph-mon[70044]: from='client.41775 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:07:14.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:07:13 smithi153 ceph-mon[70044]: from='client.51712 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:07:14.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:07:13 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1540815667' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:07:14.679 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:07:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:07:14.339Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:07:14.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:07:14 smithi083 ceph-mon[77242]: pgmap v6570: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:14.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:07:14 smithi083 ceph-mon[79930]: pgmap v6570: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:15.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:07:14 smithi153 ceph-mon[70044]: pgmap v6570: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:15.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:07:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:07:15.043Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:07:16.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:07:16 smithi083 ceph-mon[77242]: pgmap v6571: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:16.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:07:16 smithi083 ceph-mon[79930]: pgmap v6571: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:17.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:07:16 smithi153 ceph-mon[70044]: pgmap v6571: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:18.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:07:18 smithi153 ceph-mon[70044]: pgmap v6572: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:18.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:07:18 smithi083 ceph-mon[77242]: pgmap v6572: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:18.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:07:18 smithi083 ceph-mon[79930]: pgmap v6572: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:20.959 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:07:20 smithi083 ceph-mon[79930]: pgmap v6573: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:20.960 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:07:20 smithi083 ceph-mon[77242]: pgmap v6573: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:21.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:07:20 smithi153 ceph-mon[70044]: pgmap v6573: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:21.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:07:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:07:20.957Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:07:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:07:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:07:21] "GET /metrics HTTP/1.1" 200 225217 "" "Prometheus/2.33.4" 2022-09-29T09:07:22.673 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:07:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:07:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:07:22.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:07:22 smithi083 ceph-mon[79930]: pgmap v6574: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:22.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:07:22 smithi083 ceph-mon[77242]: pgmap v6574: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:23.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:07:22 smithi153 ceph-mon[70044]: pgmap v6574: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:07:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:07:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:07:24.675 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:07:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:07:24.339Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:07:24.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:07:24 smithi083 ceph-mon[79930]: pgmap v6575: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:24.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:07:24 smithi083 ceph-mon[77242]: pgmap v6575: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:25.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:07:24 smithi153 ceph-mon[70044]: pgmap v6575: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:25.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:07:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:07:25.044Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:07:26.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:07:26 smithi083 ceph-mon[77242]: pgmap v6576: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:26.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:07:26 smithi083 ceph-mon[79930]: pgmap v6576: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:27.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:07:26 smithi153 ceph-mon[70044]: pgmap v6576: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:28.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:07:28 smithi153 ceph-mon[70044]: pgmap v6577: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:28.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:07:28 smithi083 ceph-mon[79930]: pgmap v6577: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:28.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:07:28 smithi083 ceph-mon[77242]: pgmap v6577: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:30.959 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:07:30 smithi083 ceph-mon[79930]: pgmap v6578: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:30.960 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:07:30 smithi083 ceph-mon[77242]: pgmap v6578: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:31.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:07:30 smithi153 ceph-mon[70044]: pgmap v6578: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:31.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:07:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:07:30.957Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:07:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:07:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:07:31] "GET /metrics HTTP/1.1" 200 225215 "" "Prometheus/2.33.4" 2022-09-29T09:07:31.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:07:31 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:07:31.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:07:31 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:07:31.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:07:31 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:07:31.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:07:31 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:07:31.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:07:31 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:07:31.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:07:31 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:07:31.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:07:31 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:07:31.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:07:31 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:07:32.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:07:31 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:07:32.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:07:31 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:07:32.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:07:31 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:07:32.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:07:31 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:07:32.676 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:07:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:07:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:07:32.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:07:32 smithi083 ceph-mon[79930]: pgmap v6579: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:32.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:07:32 smithi083 ceph-mon[77242]: pgmap v6579: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:33.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:07:32 smithi153 ceph-mon[70044]: pgmap v6579: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:07:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:07:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:07:34.679 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:07:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:07:34.340Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:07:34.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:07:34 smithi083 ceph-mon[79930]: pgmap v6580: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:34.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:07:34 smithi083 ceph-mon[77242]: pgmap v6580: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:35.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:07:34 smithi153 ceph-mon[70044]: pgmap v6580: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:35.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:07:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:07:35.044Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:07:36.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:07:36 smithi083 ceph-mon[79930]: pgmap v6581: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:36.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:07:36 smithi083 ceph-mon[77242]: pgmap v6581: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:37.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:07:36 smithi153 ceph-mon[70044]: pgmap v6581: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:38.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:07:38 smithi153 ceph-mon[70044]: pgmap v6582: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:38.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:07:38 smithi083 ceph-mon[79930]: pgmap v6582: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:38.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:07:38 smithi083 ceph-mon[77242]: pgmap v6582: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:40.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:07:40 smithi083 ceph-mon[79930]: pgmap v6583: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:40.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:07:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:07:40.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:07:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:07:40.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:07:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:07:40.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:07:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:07:40.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:07:40 smithi083 ceph-mon[77242]: pgmap v6583: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:40.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:07:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:07:40.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:07:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:07:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:07:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:07:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:07:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:07:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:07:40 smithi153 ceph-mon[70044]: pgmap v6583: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:07:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:07:41.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:07:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:07:41.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:07:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:07:41.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:07:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:07:41.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:07:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:07:40.958Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:07:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:07:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:07:41] "GET /metrics HTTP/1.1" 200 225214 "" "Prometheus/2.33.4" 2022-09-29T09:07:42.679 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:07:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:07:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:07:42.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:07:42 smithi083 ceph-mon[79930]: pgmap v6584: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:42.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:07:42 smithi083 ceph-mon[77242]: pgmap v6584: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:43.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:07:42 smithi153 ceph-mon[70044]: pgmap v6584: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:43.306 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:07:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:07:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:07:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:07:43.597 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:07:43.598 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 9m ago 3h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:07:43.598 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 7m ago 3h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:07:43.598 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 9m ago 3h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:07:43.599 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 7m ago 3h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:07:43.599 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 7m ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:07:43.599 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 9m ago 3h 697M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:07:43.600 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 9m ago 3h 817M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:07:43.600 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 7m ago 3h 359M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:07:43.601 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 9m ago 3h 477M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:07:43.601 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 9m ago 3h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:07:43.601 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 7m ago 3h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:07:43.602 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 9m ago 3h 590M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:07:43.602 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 9m ago 3h 581M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:07:43.603 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 9m ago 3h 456M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:07:43.603 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 9m ago 3h 565M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:07:43.603 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 7m ago 3h 500M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:07:43.604 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 7m ago 3h 620M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:07:43.604 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 7m ago 3h 552M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:07:43.605 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 7m ago 3h 650M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:07:43.605 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 7m ago 3h 192M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:07:43.957 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:07:43.957 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:07:43.957 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:07:43.958 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:07:43.958 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:07:43.958 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:07:43.958 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:07:43.959 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:07:43.959 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:07:43.959 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:07:43.960 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:07:43.960 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:07:43.960 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:07:43.960 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:07:43.961 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:07:43.961 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:07:43.961 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:07:43.961 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:07:44.680 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:07:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:07:44.340Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:07:44.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:07:44 smithi083 ceph-mon[79930]: from='client.41793 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:07:44.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:07:44 smithi083 ceph-mon[79930]: from='client.41799 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:07:44.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:07:44 smithi083 ceph-mon[79930]: pgmap v6585: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:44.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:07:44 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/455222643' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:07:44.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:07:44 smithi083 ceph-mon[77242]: from='client.41793 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:07:44.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:07:44 smithi083 ceph-mon[77242]: from='client.41799 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:07:44.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:07:44 smithi083 ceph-mon[77242]: pgmap v6585: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:44.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:07:44 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/455222643' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:07:45.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:07:44 smithi153 ceph-mon[70044]: from='client.41793 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:07:45.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:07:44 smithi153 ceph-mon[70044]: from='client.41799 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:07:45.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:07:44 smithi153 ceph-mon[70044]: pgmap v6585: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:45.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:07:44 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/455222643' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:07:45.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:07:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:07:45.045Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:07:46.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:07:46 smithi083 ceph-mon[79930]: pgmap v6586: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:46.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:07:46 smithi083 ceph-mon[77242]: pgmap v6586: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:47.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:07:46 smithi153 ceph-mon[70044]: pgmap v6586: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:48.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:07:48 smithi153 ceph-mon[70044]: pgmap v6587: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:48.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:07:48 smithi083 ceph-mon[79930]: pgmap v6587: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:48.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:07:48 smithi083 ceph-mon[77242]: pgmap v6587: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:50.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:07:50 smithi083 ceph-mon[79930]: pgmap v6588: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:50.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:07:50 smithi083 ceph-mon[77242]: pgmap v6588: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:51.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:07:50 smithi153 ceph-mon[70044]: pgmap v6588: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:51.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:07:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:07:50.959Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:07:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:07:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:07:51] "GET /metrics HTTP/1.1" 200 225214 "" "Prometheus/2.33.4" 2022-09-29T09:07:52.684 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:07:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:07:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:07:52.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:07:52 smithi083 ceph-mon[79930]: pgmap v6589: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:52.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:07:52 smithi083 ceph-mon[77242]: pgmap v6589: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:53.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:07:52 smithi153 ceph-mon[70044]: pgmap v6589: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:07:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:07:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:07:54.685 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:07:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:07:54.340Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:07:54.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:07:54 smithi083 ceph-mon[79930]: pgmap v6590: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:54.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:07:54 smithi083 ceph-mon[77242]: pgmap v6590: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:55.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:07:54 smithi153 ceph-mon[70044]: pgmap v6590: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:07:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:07:55.046Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:07:56.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:07:56 smithi083 ceph-mon[79930]: pgmap v6591: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:56.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:07:56 smithi083 ceph-mon[77242]: pgmap v6591: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:57.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:07:56 smithi153 ceph-mon[70044]: pgmap v6591: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:58.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:07:58 smithi153 ceph-mon[70044]: pgmap v6592: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:58.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:07:58 smithi083 ceph-mon[79930]: pgmap v6592: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:07:58.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:07:58 smithi083 ceph-mon[77242]: pgmap v6592: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:00.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:08:00 smithi083 ceph-mon[79930]: pgmap v6593: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:00.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:08:00 smithi083 ceph-mon[77242]: pgmap v6593: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:01.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:08:00 smithi153 ceph-mon[70044]: pgmap v6593: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:01.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:08:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:08:00.959Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:08:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:08:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:08:01] "GET /metrics HTTP/1.1" 200 225220 "" "Prometheus/2.33.4" 2022-09-29T09:08:02.685 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:08:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:08:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:08:02.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:08:02 smithi083 ceph-mon[79930]: pgmap v6594: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:02.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:08:02 smithi083 ceph-mon[77242]: pgmap v6594: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:03.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:08:02 smithi153 ceph-mon[70044]: pgmap v6594: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:08:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:08:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:08:04.688 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:08:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:08:04.342Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:08:04.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:08:04 smithi083 ceph-mon[79930]: pgmap v6595: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:04.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:08:04 smithi083 ceph-mon[77242]: pgmap v6595: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:05.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:08:04 smithi153 ceph-mon[70044]: pgmap v6595: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:05.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:08:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:08:05.046Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:08:06.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:08:06 smithi083 ceph-mon[79930]: pgmap v6596: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:06.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:08:06 smithi083 ceph-mon[77242]: pgmap v6596: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:07.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:08:06 smithi153 ceph-mon[70044]: pgmap v6596: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:08.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:08:08 smithi153 ceph-mon[70044]: pgmap v6597: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:08.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:08:08 smithi083 ceph-mon[77242]: pgmap v6597: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:08.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:08:08 smithi083 ceph-mon[79930]: pgmap v6597: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:10.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:08:10 smithi083 ceph-mon[79930]: pgmap v6598: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:10.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:08:10 smithi083 ceph-mon[77242]: pgmap v6598: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:11.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:08:10 smithi153 ceph-mon[70044]: pgmap v6598: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:11.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:08:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:08:10.959Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:08:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:08:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:08:11] "GET /metrics HTTP/1.1" 200 225219 "" "Prometheus/2.33.4" 2022-09-29T09:08:12.689 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:08:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:08:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:08:12.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:08:12 smithi083 ceph-mon[79930]: pgmap v6599: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:12.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:08:12 smithi083 ceph-mon[77242]: pgmap v6599: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:13.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:08:12 smithi153 ceph-mon[70044]: pgmap v6599: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:08:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:08:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:08:14.284 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:08:14.572 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:08:14.572 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 9m ago 3h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:08:14.573 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 7m ago 3h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:08:14.573 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 9m ago 3h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:08:14.573 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 7m ago 3h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:08:14.574 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 7m ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:08:14.574 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 9m ago 3h 697M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:08:14.574 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 9m ago 3h 817M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:08:14.574 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 7m ago 3h 359M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:08:14.575 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 9m ago 3h 477M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:08:14.575 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 9m ago 3h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:08:14.575 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 7m ago 3h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:08:14.575 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 9m ago 3h 590M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:08:14.575 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 9m ago 3h 581M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:08:14.576 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 9m ago 3h 456M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:08:14.576 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 9m ago 3h 565M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:08:14.576 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 7m ago 3h 500M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:08:14.576 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 7m ago 3h 620M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:08:14.577 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 7m ago 3h 552M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:08:14.577 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 7m ago 3h 650M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:08:14.577 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 7m ago 3h 192M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:08:14.691 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:08:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:08:14.342Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:08:14.928 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:08:14.928 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:08:14.928 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:08:14.929 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:08:14.929 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:08:14.929 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:08:14.929 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:08:14.930 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:08:14.930 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:08:14.930 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:08:14.930 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:08:14.931 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:08:14.931 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:08:14.931 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:08:14.931 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:08:14.931 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:08:14.932 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:08:14.932 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:08:14.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:08:14 smithi083 ceph-mon[79930]: pgmap v6600: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:14.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:08:14 smithi083 ceph-mon[77242]: pgmap v6600: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:15.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:08:14 smithi153 ceph-mon[70044]: pgmap v6600: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:15.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:08:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:08:15.046Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:08:15.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:08:15 smithi083 ceph-mon[79930]: from='client.41811 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:08:15.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:08:15 smithi083 ceph-mon[79930]: from='client.41817 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:08:15.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:08:15 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/699341927' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:08:15.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:08:15 smithi083 ceph-mon[77242]: from='client.41811 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:08:15.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:08:15 smithi083 ceph-mon[77242]: from='client.41817 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:08:15.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:08:15 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/699341927' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:08:16.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:08:15 smithi153 ceph-mon[70044]: from='client.41811 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:08:16.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:08:15 smithi153 ceph-mon[70044]: from='client.41817 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:08:16.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:08:15 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/699341927' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:08:16.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:08:16 smithi083 ceph-mon[79930]: pgmap v6601: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:16.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:08:16 smithi083 ceph-mon[77242]: pgmap v6601: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:17.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:08:16 smithi153 ceph-mon[70044]: pgmap v6601: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:18.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:08:18 smithi153 ceph-mon[70044]: pgmap v6602: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:18.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:08:18 smithi083 ceph-mon[79930]: pgmap v6602: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:18.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:08:18 smithi083 ceph-mon[77242]: pgmap v6602: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:20.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:08:20 smithi083 ceph-mon[79930]: pgmap v6603: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:20.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:08:20 smithi083 ceph-mon[77242]: pgmap v6603: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:21.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:08:20 smithi153 ceph-mon[70044]: pgmap v6603: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:21.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:08:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:08:20.960Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:08:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:08:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:08:21] "GET /metrics HTTP/1.1" 200 225219 "" "Prometheus/2.33.4" 2022-09-29T09:08:22.693 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:08:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:08:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:08:22.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:08:22 smithi083 ceph-mon[79930]: pgmap v6604: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:22.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:08:22 smithi083 ceph-mon[77242]: pgmap v6604: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:23.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:08:22 smithi153 ceph-mon[70044]: pgmap v6604: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:08:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:08:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:08:24.694 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:08:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:08:24.343Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:08:24.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:08:24 smithi083 ceph-mon[79930]: pgmap v6605: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:24.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:08:24 smithi083 ceph-mon[77242]: pgmap v6605: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:25.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:08:24 smithi153 ceph-mon[70044]: pgmap v6605: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:25.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:08:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:08:25.047Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:08:26.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:08:26 smithi083 ceph-mon[77242]: pgmap v6606: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:26.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:08:26 smithi083 ceph-mon[79930]: pgmap v6606: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:27.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:08:26 smithi153 ceph-mon[70044]: pgmap v6606: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:28.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:08:28 smithi153 ceph-mon[70044]: pgmap v6607: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:28.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:08:28 smithi083 ceph-mon[79930]: pgmap v6607: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:28.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:08:28 smithi083 ceph-mon[77242]: pgmap v6607: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:30.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:08:30 smithi083 ceph-mon[79930]: pgmap v6608: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:30.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:08:30 smithi083 ceph-mon[77242]: pgmap v6608: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:31.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:08:30 smithi153 ceph-mon[70044]: pgmap v6608: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:31.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:08:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:08:30.960Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:08:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:08:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:08:31] "GET /metrics HTTP/1.1" 200 225220 "" "Prometheus/2.33.4" 2022-09-29T09:08:31.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:08:31 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:08:31.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:08:31 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:08:31.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:08:31 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:08:31.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:08:31 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:08:31.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:08:31 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:08:31.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:08:31 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:08:32.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:08:31 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:08:32.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:08:31 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:08:32.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:08:31 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:08:32.801 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:08:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:08:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:08:33.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:08:32 smithi153 ceph-mon[70044]: pgmap v6609: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:33.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:08:32 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:08:33.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:08:32 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:08:33.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:08:32 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:08:33.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:08:32 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:08:33.060 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:08:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:08:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:08:33.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:08:32 smithi083 ceph-mon[77242]: pgmap v6609: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:33.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:08:32 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:08:33.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:08:32 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:08:33.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:08:32 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:08:33.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:08:32 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:08:33.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:08:32 smithi083 ceph-mon[79930]: pgmap v6609: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:33.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:08:32 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:08:33.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:08:32 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:08:33.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:08:32 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:08:33.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:08:32 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:08:34.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:08:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:08:34.343Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:08:35.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:08:34 smithi153 ceph-mon[70044]: pgmap v6610: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:35.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:08:34 smithi083 ceph-mon[79930]: pgmap v6610: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:35.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:08:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:08:35.047Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:08:35.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:08:34 smithi083 ceph-mon[77242]: pgmap v6610: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:37.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:08:36 smithi083 ceph-mon[79930]: pgmap v6611: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:37.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:08:36 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:08:37.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:08:36 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:08:37.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:08:36 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:08:37.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:08:36 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:08:37.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:08:36 smithi083 ceph-mon[77242]: pgmap v6611: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:37.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:08:36 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:08:37.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:08:36 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:08:37.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:08:36 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:08:37.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:08:36 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:08:37.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:08:36 smithi153 ceph-mon[70044]: pgmap v6611: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:37.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:08:36 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:08:37.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:08:36 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:08:37.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:08:36 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:08:37.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:08:36 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:08:38.557 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:08:38 smithi083 ceph-mon[77242]: pgmap v6612: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:38.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:08:38 smithi153 ceph-mon[70044]: pgmap v6612: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:38.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:08:38 smithi083 ceph-mon[79930]: pgmap v6612: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:08:40 smithi153 ceph-mon[70044]: Detected new or changed devices on smithi083 2022-09-29T09:08:40.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:08:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:08:40.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:08:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:08:40.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:08:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:08:40.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:08:40 smithi153 ceph-mon[70044]: pgmap v6613: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:40.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:08:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:08:40.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:08:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:08:40.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:08:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:08:40.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:08:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:08:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:08:40 smithi083 ceph-mon[79930]: Detected new or changed devices on smithi083 2022-09-29T09:08:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:08:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:08:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:08:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:08:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:08:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:08:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:08:40 smithi083 ceph-mon[79930]: pgmap v6613: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:40.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:08:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:08:40.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:08:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:08:40.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:08:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:08:40.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:08:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:08:40.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:08:40 smithi083 ceph-mon[77242]: Detected new or changed devices on smithi083 2022-09-29T09:08:40.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:08:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:08:40.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:08:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:08:40.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:08:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:08:40.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:08:40 smithi083 ceph-mon[77242]: pgmap v6613: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:40.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:08:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:08:40.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:08:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:08:40.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:08:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:08:40.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:08:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:08:41.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:08:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:08:40.961Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:08:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:08:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:08:41] "GET /metrics HTTP/1.1" 200 225220 "" "Prometheus/2.33.4" 2022-09-29T09:08:42.695 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:08:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:08:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:08:42.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:08:42 smithi083 ceph-mon[79930]: pgmap v6614: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:42.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:08:42 smithi083 ceph-mon[77242]: pgmap v6614: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:43.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:08:42 smithi153 ceph-mon[70044]: pgmap v6614: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:08:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:08:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:08:44.698 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:08:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:08:44.344Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:08:44.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:08:44 smithi083 ceph-mon[77242]: pgmap v6615: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:44.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:08:44 smithi083 ceph-mon[79930]: pgmap v6615: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:45.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:08:44 smithi153 ceph-mon[70044]: pgmap v6615: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:45.256 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:08:45.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:08:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:08:45.048Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:08:45.548 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:08:45.548 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 9s ago 3h 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:08:45.548 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 8m ago 3h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:08:45.549 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 9s ago 3h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:08:45.549 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 8m ago 3h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:08:45.549 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 8m ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:08:45.549 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 9s ago 3h 706M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:08:45.550 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 9s ago 3h 851M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:08:45.550 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 8m ago 3h 359M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:08:45.550 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 9s ago 3h 497M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:08:45.550 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 9s ago 3h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:08:45.551 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 8m ago 3h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:08:45.551 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 9s ago 3h 615M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:08:45.551 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 9s ago 3h 606M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:08:45.551 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 9s ago 3h 475M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:08:45.551 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 9s ago 3h 589M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:08:45.552 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 8m ago 3h 500M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:08:45.552 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 8m ago 3h 620M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:08:45.552 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 8m ago 3h 552M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:08:45.552 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 8m ago 3h 650M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:08:45.553 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 8m ago 3h 192M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:08:45.905 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:08:45.906 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:08:45.906 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:08:45.906 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:08:45.907 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:08:45.907 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:08:45.907 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:08:45.907 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:08:45.908 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:08:45.908 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:08:45.908 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:08:45.908 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:08:45.908 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:08:45.909 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:08:45.909 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:08:45.909 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:08:45.909 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:08:45.910 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:08:46.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:08:46 smithi083 ceph-mon[77242]: from='client.41829 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:08:46.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:08:46 smithi083 ceph-mon[77242]: from='client.51766 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:08:46.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:08:46 smithi083 ceph-mon[77242]: pgmap v6616: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:46.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:08:46 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1018621517' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:08:46.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:08:46 smithi083 ceph-mon[79930]: from='client.41829 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:08:46.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:08:46 smithi083 ceph-mon[79930]: from='client.51766 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:08:46.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:08:46 smithi083 ceph-mon[79930]: pgmap v6616: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:46.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:08:46 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1018621517' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:08:47.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:08:46 smithi153 ceph-mon[70044]: from='client.41829 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:08:47.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:08:46 smithi153 ceph-mon[70044]: from='client.51766 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:08:47.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:08:46 smithi153 ceph-mon[70044]: pgmap v6616: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:47.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:08:46 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1018621517' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:08:48.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:08:48 smithi083 ceph-mon[79930]: pgmap v6617: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:48.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:08:48 smithi083 ceph-mon[77242]: pgmap v6617: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:49.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:08:48 smithi153 ceph-mon[70044]: pgmap v6617: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:50.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:08:50 smithi083 ceph-mon[79930]: pgmap v6618: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:50.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:08:50 smithi083 ceph-mon[77242]: pgmap v6618: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:51.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:08:50 smithi153 ceph-mon[70044]: pgmap v6618: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:51.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:08:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:08:50.962Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:08:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:08:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:08:51] "GET /metrics HTTP/1.1" 200 225220 "" "Prometheus/2.33.4" 2022-09-29T09:08:52.703 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:08:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:08:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:08:52.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:08:52 smithi083 ceph-mon[77242]: pgmap v6619: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:52.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:08:52 smithi083 ceph-mon[79930]: pgmap v6619: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:53.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:08:52 smithi153 ceph-mon[70044]: pgmap v6619: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:08:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:08:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:08:54.705 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:08:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:08:54.344Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:08:54.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:08:54 smithi083 ceph-mon[77242]: pgmap v6620: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:54.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:08:54 smithi083 ceph-mon[79930]: pgmap v6620: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:55.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:08:54 smithi153 ceph-mon[70044]: pgmap v6620: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:08:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:08:55.048Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:08:56.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:08:56 smithi083 ceph-mon[77242]: pgmap v6621: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:56.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:08:56 smithi083 ceph-mon[79930]: pgmap v6621: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:57.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:08:56 smithi153 ceph-mon[70044]: pgmap v6621: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:58.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:08:58 smithi083 ceph-mon[79930]: pgmap v6622: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:58.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:08:58 smithi083 ceph-mon[77242]: pgmap v6622: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:08:59.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:08:58 smithi153 ceph-mon[70044]: pgmap v6622: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:00.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:09:00 smithi083 ceph-mon[79930]: pgmap v6623: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:00.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:09:00 smithi083 ceph-mon[77242]: pgmap v6623: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:01.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:09:00 smithi153 ceph-mon[70044]: pgmap v6623: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:01.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:09:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:09:00.963Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:09:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:09:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:09:01] "GET /metrics HTTP/1.1" 200 225214 "" "Prometheus/2.33.4" 2022-09-29T09:09:02.705 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:09:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:09:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:09:02.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:09:02 smithi083 ceph-mon[79930]: pgmap v6624: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:02.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:09:02 smithi083 ceph-mon[77242]: pgmap v6624: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:03.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:09:02 smithi153 ceph-mon[70044]: pgmap v6624: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:09:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:09:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:09:04.708 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:09:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:09:04.346Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:09:04.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:09:04 smithi083 ceph-mon[79930]: pgmap v6625: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:04.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:09:04 smithi083 ceph-mon[77242]: pgmap v6625: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:05.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:09:04 smithi153 ceph-mon[70044]: pgmap v6625: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:05.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:09:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:09:05.050Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:09:06.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:09:06 smithi083 ceph-mon[77242]: pgmap v6626: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:06.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:09:06 smithi083 ceph-mon[79930]: pgmap v6626: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:07.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:09:06 smithi153 ceph-mon[70044]: pgmap v6626: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:08.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:09:08 smithi083 ceph-mon[79930]: pgmap v6627: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:08.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:09:08 smithi083 ceph-mon[77242]: pgmap v6627: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:09.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:09:08 smithi153 ceph-mon[70044]: pgmap v6627: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:10.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:09:10 smithi083 ceph-mon[79930]: pgmap v6628: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:10.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:09:10 smithi083 ceph-mon[77242]: pgmap v6628: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:11.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:09:10 smithi153 ceph-mon[70044]: pgmap v6628: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:11.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:09:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:09:10.962Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:09:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:09:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:09:11] "GET /metrics HTTP/1.1" 200 225216 "" "Prometheus/2.33.4" 2022-09-29T09:09:12.707 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:09:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:09:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:09:12.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:09:12 smithi083 ceph-mon[79930]: pgmap v6629: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:12.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:09:12 smithi083 ceph-mon[77242]: pgmap v6629: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:13.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:09:12 smithi153 ceph-mon[70044]: pgmap v6629: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:09:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:09:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:09:14.709 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:09:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:09:14.345Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:09:14.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:09:14 smithi083 ceph-mon[79930]: pgmap v6630: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:14.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:09:14 smithi083 ceph-mon[77242]: pgmap v6630: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:15.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:09:14 smithi153 ceph-mon[70044]: pgmap v6630: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:15.466 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:09:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:09:15.050Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:09:16.231 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:09:16.518 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:09:16.519 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 40s ago 3h 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:09:16.519 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 8m ago 3h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:09:16.519 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 40s ago 3h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:09:16.520 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 8m ago 3h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:09:16.520 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 8m ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:09:16.520 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 40s ago 3h 706M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:09:16.520 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 40s ago 3h 851M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:09:16.520 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 8m ago 3h 359M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:09:16.521 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 40s ago 3h 497M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:09:16.521 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 40s ago 3h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:09:16.521 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 8m ago 3h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:09:16.521 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 40s ago 3h 615M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:09:16.521 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 40s ago 3h 606M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:09:16.522 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 40s ago 3h 475M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:09:16.522 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 40s ago 3h 589M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:09:16.522 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 8m ago 3h 500M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:09:16.522 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 8m ago 3h 620M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:09:16.523 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 8m ago 3h 552M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:09:16.523 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 8m ago 3h 650M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:09:16.523 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 8m ago 3h 192M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:09:16.872 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:09:16.873 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:09:16.873 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:09:16.873 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:09:16.873 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:09:16.874 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:09:16.874 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:09:16.874 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:09:16.874 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:09:16.875 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:09:16.875 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:09:16.875 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:09:16.875 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:09:16.875 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:09:16.876 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:09:16.876 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:09:16.876 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:09:16.876 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:09:16.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:09:16 smithi083 ceph-mon[77242]: pgmap v6631: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:16.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:09:16 smithi083 ceph-mon[79930]: pgmap v6631: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:17.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:09:16 smithi153 ceph-mon[70044]: pgmap v6631: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:18.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:09:17 smithi153 ceph-mon[70044]: from='client.41847 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:09:18.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:09:17 smithi153 ceph-mon[70044]: from='client.51784 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:09:18.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:09:17 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2991172185' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:09:18.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:09:17 smithi083 ceph-mon[79930]: from='client.41847 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:09:18.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:09:17 smithi083 ceph-mon[79930]: from='client.51784 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:09:18.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:09:17 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2991172185' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:09:18.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:09:17 smithi083 ceph-mon[77242]: from='client.41847 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:09:18.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:09:17 smithi083 ceph-mon[77242]: from='client.51784 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:09:18.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:09:17 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2991172185' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:09:19.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:09:18 smithi153 ceph-mon[70044]: pgmap v6632: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:19.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:09:18 smithi083 ceph-mon[79930]: pgmap v6632: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:19.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:09:18 smithi083 ceph-mon[77242]: pgmap v6632: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:21.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:09:20 smithi153 ceph-mon[70044]: pgmap v6633: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:21.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:09:20 smithi083 ceph-mon[79930]: pgmap v6633: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:21.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:09:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:09:20.963Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:09:21.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:09:20 smithi083 ceph-mon[77242]: pgmap v6633: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:09:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:09:21] "GET /metrics HTTP/1.1" 200 225216 "" "Prometheus/2.33.4" 2022-09-29T09:09:22.723 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:09:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:09:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:09:23.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:09:22 smithi153 ceph-mon[70044]: pgmap v6634: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:23.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:09:22 smithi083 ceph-mon[79930]: pgmap v6634: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:23.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:09:22 smithi083 ceph-mon[77242]: pgmap v6634: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:09:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:09:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:09:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:09:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:09:24.346Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:09:25.053 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:09:24 smithi083 ceph-mon[79930]: pgmap v6635: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:25.054 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:09:24 smithi083 ceph-mon[77242]: pgmap v6635: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:25.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:09:24 smithi153 ceph-mon[70044]: pgmap v6635: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:25.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:09:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:09:25.051Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:09:27.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:09:26 smithi153 ceph-mon[70044]: pgmap v6636: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:27.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:09:26 smithi083 ceph-mon[77242]: pgmap v6636: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:27.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:09:26 smithi083 ceph-mon[79930]: pgmap v6636: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:28.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:09:28 smithi083 ceph-mon[79930]: pgmap v6637: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:28.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:09:28 smithi083 ceph-mon[77242]: pgmap v6637: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:29.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:09:28 smithi153 ceph-mon[70044]: pgmap v6637: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:30.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:09:30 smithi083 ceph-mon[79930]: pgmap v6638: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:30.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:09:30 smithi083 ceph-mon[77242]: pgmap v6638: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:31.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:09:30 smithi153 ceph-mon[70044]: pgmap v6638: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:31.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:09:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:09:30.964Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:09:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:09:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:09:31] "GET /metrics HTTP/1.1" 200 225217 "" "Prometheus/2.33.4" 2022-09-29T09:09:32.715 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:09:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:09:32.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:09:33.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:09:32 smithi153 ceph-mon[70044]: pgmap v6639: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:33.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:09:32 smithi083 ceph-mon[77242]: pgmap v6639: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:33.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:09:32 smithi083 ceph-mon[79930]: pgmap v6639: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:09:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:09:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:09:34.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:09:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:09:34.348Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:09:35.054 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:09:34 smithi083 ceph-mon[79930]: pgmap v6640: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:35.055 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:09:34 smithi083 ceph-mon[77242]: pgmap v6640: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:35.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:09:34 smithi153 ceph-mon[70044]: pgmap v6640: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:35.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:09:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:09:35.052Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:09:37.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:09:36 smithi153 ceph-mon[70044]: pgmap v6641: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:37.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:09:36 smithi083 ceph-mon[77242]: pgmap v6641: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:37.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:09:36 smithi083 ceph-mon[79930]: pgmap v6641: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:38.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:09:38 smithi083 ceph-mon[79930]: pgmap v6642: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:38.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:09:38 smithi083 ceph-mon[77242]: pgmap v6642: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:39.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:09:38 smithi153 ceph-mon[70044]: pgmap v6642: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:39.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:09:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:09:39.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:09:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:09:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:09:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:09:39.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:09:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:09:39.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:09:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:09:39.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:09:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:09:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:09:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:09:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:09:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:09:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:09:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:09:40.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:09:40 smithi083 ceph-mon[79930]: pgmap v6643: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:40.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:09:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:09:40.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:09:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:09:40.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:09:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:09:40.968 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:09:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:09:40.968 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:09:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:09:40.968 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:09:40 smithi083 ceph-mon[77242]: pgmap v6643: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:40.968 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:09:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:09:40.969 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:09:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:09:40.969 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:09:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:09:40.969 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:09:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:09:40.969 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:09:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:09:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:09:40 smithi153 ceph-mon[70044]: pgmap v6643: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:09:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:09:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:09:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:09:41.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:09:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:09:41.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:09:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:09:41.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:09:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:09:41.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:09:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:09:40.963Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:09:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:09:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:09:41] "GET /metrics HTTP/1.1" 200 225218 "" "Prometheus/2.33.4" 2022-09-29T09:09:42.718 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:09:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:09:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:09:43.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:09:42 smithi153 ceph-mon[70044]: pgmap v6644: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:43.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:09:42 smithi083 ceph-mon[79930]: pgmap v6644: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:43.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:09:42 smithi083 ceph-mon[77242]: pgmap v6644: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:09:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:09:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:09:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:09:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:09:44.348Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:09:45.055 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:09:44 smithi083 ceph-mon[79930]: pgmap v6645: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:45.055 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:09:44 smithi083 ceph-mon[77242]: pgmap v6645: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:45.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:09:44 smithi153 ceph-mon[70044]: pgmap v6645: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:45.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:09:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:09:45.052Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:09:47.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:09:46 smithi153 ceph-mon[70044]: pgmap v6646: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:47.200 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:09:47.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:09:46 smithi083 ceph-mon[79930]: pgmap v6646: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:47.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:09:46 smithi083 ceph-mon[77242]: pgmap v6646: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:47.489 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:09:47.490 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 71s ago 3h 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:09:47.490 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 9m ago 3h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:09:47.491 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 71s ago 3h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:09:47.491 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 9m ago 3h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:09:47.492 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 9m ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:09:47.492 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 71s ago 3h 706M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:09:47.492 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 71s ago 3h 851M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:09:47.493 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 9m ago 3h 359M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:09:47.493 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 71s ago 3h 497M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:09:47.493 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 71s ago 3h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:09:47.493 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 9m ago 3h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:09:47.494 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 71s ago 3h 615M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:09:47.494 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 71s ago 3h 606M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:09:47.494 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 71s ago 3h 475M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:09:47.495 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 71s ago 3h 589M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:09:47.495 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 9m ago 3h 500M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:09:47.495 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 9m ago 3h 620M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:09:47.495 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 9m ago 3h 552M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:09:47.496 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 9m ago 3h 650M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:09:47.496 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 9m ago 3h 192M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:09:47.846 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:09:47.847 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:09:47.847 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:09:47.847 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:09:47.848 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:09:47.848 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:09:47.848 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:09:47.848 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:09:47.849 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:09:47.849 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:09:47.849 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:09:47.850 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:09:47.850 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:09:47.850 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:09:47.850 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:09:47.851 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:09:47.851 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:09:47.851 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:09:48.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:09:48 smithi083 ceph-mon[79930]: from='client.41865 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:09:48.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:09:48 smithi083 ceph-mon[79930]: from='client.51802 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:09:48.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:09:48 smithi083 ceph-mon[79930]: pgmap v6647: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:48.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:09:48 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3230288863' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:09:48.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:09:48 smithi083 ceph-mon[77242]: from='client.41865 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:09:48.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:09:48 smithi083 ceph-mon[77242]: from='client.51802 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:09:48.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:09:48 smithi083 ceph-mon[77242]: pgmap v6647: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:48.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:09:48 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3230288863' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:09:49.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:09:48 smithi153 ceph-mon[70044]: from='client.41865 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:09:49.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:09:48 smithi153 ceph-mon[70044]: from='client.51802 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:09:49.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:09:48 smithi153 ceph-mon[70044]: pgmap v6647: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:49.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:09:48 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3230288863' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:09:50.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:09:50 smithi083 ceph-mon[79930]: pgmap v6648: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:50.968 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:09:50 smithi083 ceph-mon[77242]: pgmap v6648: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:51.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:09:50 smithi153 ceph-mon[70044]: pgmap v6648: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:51.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:09:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:09:50.965Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:09:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:09:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:09:51] "GET /metrics HTTP/1.1" 200 225218 "" "Prometheus/2.33.4" 2022-09-29T09:09:52.722 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:09:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:09:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:09:53.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:09:52 smithi153 ceph-mon[70044]: pgmap v6649: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:53.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:09:52 smithi083 ceph-mon[77242]: pgmap v6649: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:53.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:09:52 smithi083 ceph-mon[79930]: pgmap v6649: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:09:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:09:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:09:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:09:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:09:54.348Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:09:55.055 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:09:54 smithi083 ceph-mon[79930]: pgmap v6650: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:55.056 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:09:54 smithi083 ceph-mon[77242]: pgmap v6650: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:55.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:09:54 smithi153 ceph-mon[70044]: pgmap v6650: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:09:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:09:55.053Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:09:57.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:09:56 smithi153 ceph-mon[70044]: pgmap v6651: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:57.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:09:56 smithi083 ceph-mon[77242]: pgmap v6651: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:57.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:09:56 smithi083 ceph-mon[79930]: pgmap v6651: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:58.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:09:58 smithi083 ceph-mon[77242]: pgmap v6652: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:58.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:09:58 smithi083 ceph-mon[79930]: pgmap v6652: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:09:59.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:09:58 smithi153 ceph-mon[70044]: pgmap v6652: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:00.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:10:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[77219]: 2022-09-29T09:09:59.999+0000 7f160f665700 -1 log_channel(cluster) log [ERR] : overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T09:10:01.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:10:00 smithi153 ceph-mon[70044]: pgmap v6653: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:01.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:10:00 smithi153 ceph-mon[70044]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T09:10:01.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:10:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:10:00.966Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:10:01.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:10:00 smithi083 ceph-mon[77242]: pgmap v6653: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:01.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:10:00 smithi083 ceph-mon[77242]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T09:10:01.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:10:00 smithi083 ceph-mon[79930]: pgmap v6653: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:01.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:10:00 smithi083 ceph-mon[79930]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T09:10:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:10:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:10:01] "GET /metrics HTTP/1.1" 200 225219 "" "Prometheus/2.33.4" 2022-09-29T09:10:02.722 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:10:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:10:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:10:03.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:10:02 smithi153 ceph-mon[70044]: pgmap v6654: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:03.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:10:02 smithi083 ceph-mon[79930]: pgmap v6654: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:03.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:10:02 smithi083 ceph-mon[77242]: pgmap v6654: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:10:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:10:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:10:04.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:10:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:10:04.350Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:10:05.056 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:10:04 smithi083 ceph-mon[79930]: pgmap v6655: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:05.056 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:10:04 smithi083 ceph-mon[77242]: pgmap v6655: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:05.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:10:04 smithi153 ceph-mon[70044]: pgmap v6655: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:05.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:10:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:10:05.054Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:10:07.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:10:06 smithi153 ceph-mon[70044]: pgmap v6656: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:07.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:10:06 smithi083 ceph-mon[77242]: pgmap v6656: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:07.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:10:06 smithi083 ceph-mon[79930]: pgmap v6656: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:08.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:10:08 smithi083 ceph-mon[77242]: pgmap v6657: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:08.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:10:08 smithi083 ceph-mon[79930]: pgmap v6657: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:09.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:10:08 smithi153 ceph-mon[70044]: pgmap v6657: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:11.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:10:10 smithi153 ceph-mon[70044]: pgmap v6658: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:11.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:10:10 smithi083 ceph-mon[79930]: pgmap v6658: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:11.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:10:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:10:10.966Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:10:11.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:10:10 smithi083 ceph-mon[77242]: pgmap v6658: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:10:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:10:11] "GET /metrics HTTP/1.1" 200 225220 "" "Prometheus/2.33.4" 2022-09-29T09:10:12.727 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:10:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:10:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:10:13.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:10:12 smithi153 ceph-mon[70044]: pgmap v6659: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:13.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:10:12 smithi083 ceph-mon[77242]: pgmap v6659: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:13.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:10:12 smithi083 ceph-mon[79930]: pgmap v6659: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:10:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:10:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:10:14.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:10:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:10:14.350Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:10:15.056 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:10:14 smithi083 ceph-mon[79930]: pgmap v6660: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:15.057 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:10:14 smithi083 ceph-mon[77242]: pgmap v6660: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:15.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:10:14 smithi153 ceph-mon[70044]: pgmap v6660: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:15.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:10:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:10:15.054Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:10:17.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:10:16 smithi153 ceph-mon[70044]: pgmap v6661: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:17.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:10:16 smithi083 ceph-mon[79930]: pgmap v6661: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:17.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:10:16 smithi083 ceph-mon[77242]: pgmap v6661: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:18.174 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:10:18.465 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:10:18.465 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 102s ago 3h 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:10:18.465 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 9m ago 3h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:10:18.465 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 102s ago 3h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:10:18.466 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 9m ago 3h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:10:18.466 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 9m ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:10:18.466 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 102s ago 3h 706M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:10:18.467 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 102s ago 3h 851M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:10:18.467 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 9m ago 3h 359M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:10:18.467 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 102s ago 3h 497M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:10:18.467 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 102s ago 3h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:10:18.467 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 9m ago 3h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:10:18.468 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 102s ago 3h 615M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:10:18.468 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 102s ago 3h 606M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:10:18.468 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 102s ago 3h 475M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:10:18.468 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 102s ago 3h 589M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:10:18.469 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 9m ago 3h 500M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:10:18.469 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 9m ago 3h 620M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:10:18.469 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 9m ago 3h 552M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:10:18.469 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 9m ago 3h 650M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:10:18.470 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 9m ago 3h 192M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:10:18.823 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:10:18.824 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:10:18.824 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:10:18.824 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:10:18.824 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:10:18.825 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:10:18.825 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:10:18.825 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:10:18.825 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:10:18.825 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:10:18.826 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:10:18.826 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:10:18.826 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:10:18.826 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:10:18.827 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:10:18.827 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:10:18.827 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:10:18.827 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:10:18.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:10:18 smithi083 ceph-mon[79930]: pgmap v6662: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:18.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:10:18 smithi083 ceph-mon[77242]: pgmap v6662: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:19.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:10:18 smithi153 ceph-mon[70044]: pgmap v6662: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:19.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:10:19 smithi083 ceph-mon[79930]: from='client.41883 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:10:19.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:10:19 smithi083 ceph-mon[79930]: from='client.51820 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:10:19.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:10:19 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1383686061' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:10:19.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:10:19 smithi083 ceph-mon[77242]: from='client.41883 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:10:19.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:10:19 smithi083 ceph-mon[77242]: from='client.51820 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:10:19.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:10:19 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1383686061' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:10:20.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:10:19 smithi153 ceph-mon[70044]: from='client.41883 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:10:20.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:10:19 smithi153 ceph-mon[70044]: from='client.51820 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:10:20.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:10:19 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1383686061' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:10:20.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:10:20 smithi083 ceph-mon[79930]: pgmap v6663: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:20.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:10:20 smithi083 ceph-mon[77242]: pgmap v6663: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:21.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:10:20 smithi153 ceph-mon[70044]: pgmap v6663: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:21.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:10:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:10:20.967Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:10:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:10:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:10:21] "GET /metrics HTTP/1.1" 200 225220 "" "Prometheus/2.33.4" 2022-09-29T09:10:22.728 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:10:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:10:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:10:23.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:10:22 smithi153 ceph-mon[70044]: pgmap v6664: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:23.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:10:22 smithi083 ceph-mon[77242]: pgmap v6664: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:23.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:10:22 smithi083 ceph-mon[79930]: pgmap v6664: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:10:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:10:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:10:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:10:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:10:24.350Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:10:25.058 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:10:24 smithi083 ceph-mon[79930]: pgmap v6665: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:25.058 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:10:24 smithi083 ceph-mon[77242]: pgmap v6665: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:25.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:10:24 smithi153 ceph-mon[70044]: pgmap v6665: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:25.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:10:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:10:25.055Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:10:27.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:10:26 smithi153 ceph-mon[70044]: pgmap v6666: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:27.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:10:26 smithi083 ceph-mon[79930]: pgmap v6666: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:27.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:10:26 smithi083 ceph-mon[77242]: pgmap v6666: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:28.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:10:28 smithi083 ceph-mon[79930]: pgmap v6667: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:28.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:10:28 smithi083 ceph-mon[77242]: pgmap v6667: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:29.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:10:28 smithi153 ceph-mon[70044]: pgmap v6667: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:31.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:10:30 smithi153 ceph-mon[70044]: pgmap v6668: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:31.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:10:30 smithi083 ceph-mon[79930]: pgmap v6668: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:31.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:10:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:10:30.968Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:10:31.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:10:30 smithi083 ceph-mon[77242]: pgmap v6668: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:10:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:10:31] "GET /metrics HTTP/1.1" 200 225214 "" "Prometheus/2.33.4" 2022-09-29T09:10:32.733 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:10:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:10:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:10:33.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:10:32 smithi153 ceph-mon[70044]: pgmap v6669: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:33.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:10:32 smithi083 ceph-mon[79930]: pgmap v6669: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:33.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:10:32 smithi083 ceph-mon[77242]: pgmap v6669: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:10:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:10:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:10:34.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:10:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:10:34.351Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:10:35.058 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:10:34 smithi083 ceph-mon[79930]: pgmap v6670: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:35.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:10:34 smithi153 ceph-mon[70044]: pgmap v6670: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:35.060 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:10:34 smithi083 ceph-mon[77242]: pgmap v6670: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:35.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:10:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:10:35.056Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:10:37.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:10:36 smithi153 ceph-mon[70044]: pgmap v6671: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:37.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:10:36 smithi083 ceph-mon[79930]: pgmap v6671: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:37.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:10:36 smithi083 ceph-mon[77242]: pgmap v6671: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:38.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:10:38 smithi083 ceph-mon[79930]: pgmap v6672: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:38.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:10:38 smithi083 ceph-mon[77242]: pgmap v6672: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:39.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:10:38 smithi153 ceph-mon[70044]: pgmap v6672: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:10:40 smithi153 ceph-mon[70044]: pgmap v6673: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:41.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:10:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:10:41.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:10:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:10:41.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:10:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:10:41.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:10:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:10:41.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:10:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:10:41.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:10:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:10:41.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:10:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:10:41.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:10:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:10:40.968Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:10:41.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:10:40 smithi083 ceph-mon[79930]: pgmap v6673: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:41.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:10:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:10:41.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:10:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:10:41.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:10:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:10:41.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:10:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:10:41.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:10:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:10:41.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:10:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:10:41.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:10:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:10:41.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:10:40 smithi083 ceph-mon[77242]: pgmap v6673: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:41.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:10:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:10:41.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:10:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:10:41.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:10:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:10:41.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:10:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:10:41.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:10:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:10:41.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:10:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:10:41.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:10:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:10:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:10:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:10:41] "GET /metrics HTTP/1.1" 200 225216 "" "Prometheus/2.33.4" 2022-09-29T09:10:42.738 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:10:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:10:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:10:43.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:10:42 smithi153 ceph-mon[70044]: pgmap v6674: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:43.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:10:42 smithi083 ceph-mon[79930]: pgmap v6674: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:43.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:10:42 smithi083 ceph-mon[77242]: pgmap v6674: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:10:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:10:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:10:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:10:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:10:44.351Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:10:45.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:10:44 smithi153 ceph-mon[70044]: pgmap v6675: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:45.059 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:10:44 smithi083 ceph-mon[79930]: pgmap v6675: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:45.059 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:10:44 smithi083 ceph-mon[77242]: pgmap v6675: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:45.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:10:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:10:45.056Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:10:46.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:10:46 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:10:46.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:10:46 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:10:46.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:10:46 smithi083 ceph-mon[79930]: pgmap v6676: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:46.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:10:46 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:10:46.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:10:46 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:10:46.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:10:46 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:10:46.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:10:46 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:10:46.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:10:46 smithi083 ceph-mon[77242]: pgmap v6676: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:46.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:10:46 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:10:46.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:10:46 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:10:46.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:10:46 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:10:46.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:10:46 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:10:46.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:10:46 smithi153 ceph-mon[70044]: pgmap v6676: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:46.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:10:46 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:10:46.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:10:46 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:10:48.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:10:48 smithi083 ceph-mon[79930]: pgmap v6677: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:48.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:10:48 smithi083 ceph-mon[77242]: pgmap v6677: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:49.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:10:48 smithi153 ceph-mon[70044]: pgmap v6677: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:49.145 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:10:49.434 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:10:49.434 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 2m ago 3h 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:10:49.435 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 4s ago 3h 63.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:10:49.435 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 2m ago 3h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:10:49.435 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 4s ago 3h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:10:49.435 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 4s ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:10:49.435 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 2m ago 3h 706M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:10:49.436 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 2m ago 3h 851M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:10:49.436 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 4s ago 3h 375M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:10:49.436 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 2m ago 3h 497M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:10:49.436 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 2m ago 3h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:10:49.436 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 4s ago 3h 25.7M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:10:49.437 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 2m ago 3h 615M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:10:49.437 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 2m ago 3h 606M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:10:49.437 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 2m ago 3h 475M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:10:49.437 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 2m ago 3h 589M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:10:49.437 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 4s ago 3h 522M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:10:49.438 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 4s ago 3h 648M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:10:49.438 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 4s ago 3h 577M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:10:49.438 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 4s ago 3h 680M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:10:49.438 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 4s ago 3h 167M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:10:49.790 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:10:49.790 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:10:49.790 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:10:49.791 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:10:49.791 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:10:49.791 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:10:49.791 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:10:49.792 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:10:49.792 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:10:49.792 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:10:49.792 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:10:49.792 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:10:49.793 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:10:49.793 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:10:49.793 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:10:49.793 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:10:49.794 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:10:49.794 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:10:50.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:10:49 smithi153 ceph-mon[70044]: Detected new or changed devices on smithi153 2022-09-29T09:10:50.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:10:49 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:10:50.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:10:49 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:10:50.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:10:49 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:10:50.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:10:49 smithi083 ceph-mon[79930]: Detected new or changed devices on smithi153 2022-09-29T09:10:50.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:10:49 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:10:50.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:10:49 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:10:50.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:10:49 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:10:50.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:10:49 smithi083 ceph-mon[77242]: Detected new or changed devices on smithi153 2022-09-29T09:10:50.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:10:49 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:10:50.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:10:49 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:10:50.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:10:49 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:10:51.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:10:50 smithi153 ceph-mon[70044]: from='client.51832 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:10:51.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:10:50 smithi153 ceph-mon[70044]: from='client.51838 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:10:51.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:10:50 smithi153 ceph-mon[70044]: pgmap v6678: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:51.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:10:50 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/466181881' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:10:51.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:10:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:10:50.970Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:10:51.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:10:50 smithi083 ceph-mon[77242]: from='client.51832 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:10:51.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:10:50 smithi083 ceph-mon[77242]: from='client.51838 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:10:51.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:10:50 smithi083 ceph-mon[77242]: pgmap v6678: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:51.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:10:50 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/466181881' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:10:51.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:10:50 smithi083 ceph-mon[79930]: from='client.51832 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:10:51.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:10:50 smithi083 ceph-mon[79930]: from='client.51838 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:10:51.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:10:50 smithi083 ceph-mon[79930]: pgmap v6678: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:51.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:10:50 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/466181881' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:10:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:10:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:10:51] "GET /metrics HTTP/1.1" 200 225216 "" "Prometheus/2.33.4" 2022-09-29T09:10:52.733 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:10:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:10:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:10:53.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:10:52 smithi153 ceph-mon[70044]: pgmap v6679: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:53.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:10:52 smithi083 ceph-mon[77242]: pgmap v6679: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:53.228 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:10:52 smithi083 ceph-mon[79930]: pgmap v6679: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:10:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:10:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:10:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:10:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:10:54.351Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:10:55.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:10:54 smithi153 ceph-mon[70044]: pgmap v6680: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:55.060 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:10:54 smithi083 ceph-mon[79930]: pgmap v6680: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:55.060 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:10:54 smithi083 ceph-mon[77242]: pgmap v6680: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:10:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:10:55.058Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:10:57.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:10:56 smithi153 ceph-mon[70044]: pgmap v6681: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:57.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:10:56 smithi083 ceph-mon[77242]: pgmap v6681: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:57.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:10:56 smithi083 ceph-mon[79930]: pgmap v6681: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:58.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:10:58 smithi083 ceph-mon[77242]: pgmap v6682: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:58.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:10:58 smithi083 ceph-mon[79930]: pgmap v6682: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:10:59.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:10:58 smithi153 ceph-mon[70044]: pgmap v6682: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:01.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:11:00 smithi153 ceph-mon[70044]: pgmap v6683: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:01.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:11:00 smithi083 ceph-mon[79930]: pgmap v6683: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:01.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:11:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:11:00.971Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:11:01.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:11:00 smithi083 ceph-mon[77242]: pgmap v6683: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:11:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:11:01] "GET /metrics HTTP/1.1" 200 225216 "" "Prometheus/2.33.4" 2022-09-29T09:11:02.743 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:11:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:11:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:11:03.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:11:02 smithi153 ceph-mon[70044]: pgmap v6684: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:03.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:11:02 smithi083 ceph-mon[79930]: pgmap v6684: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:03.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:11:02 smithi083 ceph-mon[77242]: pgmap v6684: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:11:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:11:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:11:04.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:11:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:11:04.352Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:11:05.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:11:04 smithi153 ceph-mon[70044]: pgmap v6685: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:05.061 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:11:04 smithi083 ceph-mon[79930]: pgmap v6685: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:05.061 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:11:04 smithi083 ceph-mon[77242]: pgmap v6685: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:05.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:11:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:11:05.058Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:11:07.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:11:06 smithi153 ceph-mon[70044]: pgmap v6686: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:07.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:11:06 smithi083 ceph-mon[79930]: pgmap v6686: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:07.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:11:06 smithi083 ceph-mon[77242]: pgmap v6686: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:08.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:11:08 smithi083 ceph-mon[79930]: pgmap v6687: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:08.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:11:08 smithi083 ceph-mon[77242]: pgmap v6687: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:09.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:11:08 smithi153 ceph-mon[70044]: pgmap v6687: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:11.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:11:10 smithi153 ceph-mon[70044]: pgmap v6688: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:11.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:11:10 smithi083 ceph-mon[79930]: pgmap v6688: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:11.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:11:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:11:10.971Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:11:11.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:11:10 smithi083 ceph-mon[77242]: pgmap v6688: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:11:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:11:11] "GET /metrics HTTP/1.1" 200 225219 "" "Prometheus/2.33.4" 2022-09-29T09:11:12.744 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:11:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:11:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:11:13.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:11:12 smithi153 ceph-mon[70044]: pgmap v6689: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:13.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:11:12 smithi083 ceph-mon[77242]: pgmap v6689: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:13.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:11:12 smithi083 ceph-mon[79930]: pgmap v6689: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:11:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:11:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:11:14.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:11:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:11:14.353Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:11:15.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:11:14 smithi153 ceph-mon[70044]: pgmap v6690: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:15.061 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:11:14 smithi083 ceph-mon[79930]: pgmap v6690: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:15.062 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:11:14 smithi083 ceph-mon[77242]: pgmap v6690: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:15.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:11:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:11:15.059Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:11:17.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:11:16 smithi153 ceph-mon[70044]: pgmap v6691: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:17.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:11:16 smithi083 ceph-mon[79930]: pgmap v6691: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:17.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:11:16 smithi083 ceph-mon[77242]: pgmap v6691: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:18.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:11:18 smithi083 ceph-mon[79930]: pgmap v6692: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:18.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:11:18 smithi083 ceph-mon[77242]: pgmap v6692: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:19.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:11:18 smithi153 ceph-mon[70044]: pgmap v6692: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:20.118 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:11:20.406 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:11:20.406 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 2m ago 3h 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:11:20.406 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 35s ago 3h 63.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:11:20.406 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 2m ago 3h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:11:20.407 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 35s ago 3h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:11:20.407 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 35s ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:11:20.407 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 2m ago 3h 706M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:11:20.407 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 2m ago 3h 851M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:11:20.407 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 35s ago 3h 375M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:11:20.408 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 2m ago 3h 497M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:11:20.408 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 2m ago 3h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:11:20.408 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 35s ago 3h 25.7M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:11:20.408 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 2m ago 3h 615M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:11:20.408 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 2m ago 3h 606M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:11:20.409 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 2m ago 3h 475M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:11:20.409 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 2m ago 3h 589M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:11:20.409 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 35s ago 3h 522M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:11:20.409 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 35s ago 3h 648M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:11:20.409 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 35s ago 3h 577M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:11:20.410 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 35s ago 3h 680M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:11:20.410 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 35s ago 3h 167M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:11:20.761 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:11:20.761 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:11:20.762 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:11:20.762 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:11:20.762 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:11:20.762 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:11:20.763 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:11:20.763 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:11:20.763 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:11:20.763 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:11:20.764 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:11:20.764 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:11:20.764 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:11:20.764 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:11:20.765 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:11:20.765 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:11:20.765 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:11:20.765 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:11:21.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:11:20 smithi153 ceph-mon[70044]: pgmap v6693: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:21.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:11:20 smithi083 ceph-mon[79930]: pgmap v6693: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:21.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:11:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:11:20.970Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:11:21.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:11:20 smithi083 ceph-mon[77242]: pgmap v6693: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:11:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:11:21] "GET /metrics HTTP/1.1" 200 225219 "" "Prometheus/2.33.4" 2022-09-29T09:11:21.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:11:21 smithi083 ceph-mon[79930]: from='client.51850 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:11:21.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:11:21 smithi083 ceph-mon[79930]: from='client.51856 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:11:21.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:11:21 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/288103479' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:11:21.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:11:21 smithi083 ceph-mon[77242]: from='client.51850 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:11:21.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:11:21 smithi083 ceph-mon[77242]: from='client.51856 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:11:21.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:11:21 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/288103479' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:11:22.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:11:21 smithi153 ceph-mon[70044]: from='client.51850 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:11:22.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:11:21 smithi153 ceph-mon[70044]: from='client.51856 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:11:22.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:11:21 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/288103479' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:11:22.751 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:11:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:11:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:11:23.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:11:22 smithi153 ceph-mon[70044]: pgmap v6694: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:23.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:11:22 smithi083 ceph-mon[79930]: pgmap v6694: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:23.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:11:22 smithi083 ceph-mon[77242]: pgmap v6694: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:11:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:11:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:11:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:11:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:11:24.354Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:11:25.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:11:24 smithi153 ceph-mon[70044]: pgmap v6695: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:25.062 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:11:24 smithi083 ceph-mon[77242]: pgmap v6695: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:25.063 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:11:24 smithi083 ceph-mon[79930]: pgmap v6695: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:25.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:11:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:11:25.060Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:11:27.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:11:26 smithi153 ceph-mon[70044]: pgmap v6696: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:27.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:11:26 smithi083 ceph-mon[79930]: pgmap v6696: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:27.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:11:26 smithi083 ceph-mon[77242]: pgmap v6696: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:28.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:11:28 smithi083 ceph-mon[79930]: pgmap v6697: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:28.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:11:28 smithi083 ceph-mon[77242]: pgmap v6697: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:29.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:11:28 smithi153 ceph-mon[70044]: pgmap v6697: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:31.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:11:30 smithi153 ceph-mon[70044]: pgmap v6698: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:31.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:11:30 smithi083 ceph-mon[79930]: pgmap v6698: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:31.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:11:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:11:30.972Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:11:31.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:11:30 smithi083 ceph-mon[77242]: pgmap v6698: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:11:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:11:31] "GET /metrics HTTP/1.1" 200 225212 "" "Prometheus/2.33.4" 2022-09-29T09:11:32.750 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:11:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:11:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:11:33.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:11:32 smithi153 ceph-mon[70044]: pgmap v6699: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:33.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:11:32 smithi083 ceph-mon[79930]: pgmap v6699: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:33.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:11:32 smithi083 ceph-mon[77242]: pgmap v6699: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:11:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:11:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:11:34.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:11:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:11:34.354Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:11:35.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:11:34 smithi153 ceph-mon[70044]: pgmap v6700: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:35.063 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:11:34 smithi083 ceph-mon[77242]: pgmap v6700: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:35.063 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:11:34 smithi083 ceph-mon[79930]: pgmap v6700: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:35.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:11:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:11:35.061Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:11:37.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:11:36 smithi153 ceph-mon[70044]: pgmap v6701: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:37.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:11:36 smithi083 ceph-mon[77242]: pgmap v6701: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:37.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:11:36 smithi083 ceph-mon[79930]: pgmap v6701: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:38.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:11:38 smithi083 ceph-mon[79930]: pgmap v6702: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:38.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:11:38 smithi083 ceph-mon[77242]: pgmap v6702: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:39.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:11:38 smithi153 ceph-mon[70044]: pgmap v6702: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:11:40 smithi153 ceph-mon[70044]: pgmap v6703: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:41.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:11:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:11:41.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:11:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:11:41.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:11:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:11:41.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:11:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:11:41.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:11:40 smithi083 ceph-mon[79930]: pgmap v6703: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:41.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:11:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:11:41.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:11:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:11:41.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:11:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:11:41.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:11:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:11:41.214 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:11:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:11:40.972Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:11:41.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:11:40 smithi083 ceph-mon[77242]: pgmap v6703: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:41.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:11:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:11:41.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:11:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:11:41.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:11:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:11:41.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:11:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:11:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:11:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:11:41] "GET /metrics HTTP/1.1" 200 225215 "" "Prometheus/2.33.4" 2022-09-29T09:11:42.755 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:11:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:11:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:11:43.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:11:42 smithi153 ceph-mon[70044]: pgmap v6704: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:43.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:11:42 smithi083 ceph-mon[79930]: pgmap v6704: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:43.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:11:42 smithi083 ceph-mon[77242]: pgmap v6704: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:11:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:11:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:11:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:11:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:11:44.354Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:11:45.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:11:44 smithi153 ceph-mon[70044]: pgmap v6705: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:45.064 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:11:44 smithi083 ceph-mon[79930]: pgmap v6705: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:45.064 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:11:44 smithi083 ceph-mon[77242]: pgmap v6705: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:45.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:11:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:11:45.062Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:11:47.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:11:46 smithi153 ceph-mon[70044]: pgmap v6706: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:47.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:11:46 smithi083 ceph-mon[79930]: pgmap v6706: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:47.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:11:46 smithi083 ceph-mon[77242]: pgmap v6706: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:48.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:11:48 smithi083 ceph-mon[77242]: pgmap v6707: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:48.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:11:48 smithi083 ceph-mon[79930]: pgmap v6707: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:49.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:11:48 smithi153 ceph-mon[70044]: pgmap v6707: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:49.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:11:49 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:11:49.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:11:49 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:11:49.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:11:49 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:11:49.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:11:49 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:11:49.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:11:49 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:11:49.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:11:49 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:11:49.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:11:49 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:11:49.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:11:49 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:11:50.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:11:49 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:11:50.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:11:49 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:11:50.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:11:49 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:11:50.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:11:49 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:11:50.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:11:50 smithi083 ceph-mon[79930]: pgmap v6708: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:50.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:11:50 smithi083 ceph-mon[77242]: pgmap v6708: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:51.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:11:50 smithi153 ceph-mon[70044]: pgmap v6708: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:51.085 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:11:51.372 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:11:51.373 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 3m ago 3h 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:11:51.373 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 66s ago 3h 63.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:11:51.373 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 3m ago 3h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:11:51.374 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 66s ago 3h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:11:51.374 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 66s ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:11:51.374 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 3m ago 3h 706M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:11:51.374 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 3m ago 3h 851M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:11:51.375 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 66s ago 3h 375M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:11:51.375 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 3m ago 3h 497M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:11:51.375 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 3m ago 3h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:11:51.375 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 66s ago 3h 25.7M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:11:51.375 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 3m ago 3h 615M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:11:51.376 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 3m ago 3h 606M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:11:51.376 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 3m ago 3h 475M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:11:51.376 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 3m ago 3h 589M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:11:51.376 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 66s ago 3h 522M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:11:51.377 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 66s ago 3h 648M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:11:51.377 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 66s ago 3h 577M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:11:51.377 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 66s ago 3h 680M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:11:51.377 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 66s ago 3h 167M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:11:51.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:11:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:11:50.972Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:11:51.728 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:11:51.728 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:11:51.728 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:11:51.728 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:11:51.729 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:11:51.729 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:11:51.729 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:11:51.729 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:11:51.729 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:11:51.730 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:11:51.730 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:11:51.730 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:11:51.730 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:11:51.731 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:11:51.731 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:11:51.731 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:11:51.731 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:11:51.731 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:11:51.779 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:11:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:11:51] "GET /metrics HTTP/1.1" 200 225215 "" "Prometheus/2.33.4" 2022-09-29T09:11:52.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:11:51 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3924873147' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:11:52.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:11:51 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3924873147' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:11:52.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:11:51 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3924873147' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:11:52.783 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:11:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:11:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:11:53.058 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:11:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:11:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:11:53.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:11:52 smithi153 ceph-mon[70044]: from='client.51868 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:11:53.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:11:52 smithi153 ceph-mon[70044]: from='client.51874 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:11:53.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:11:52 smithi153 ceph-mon[70044]: pgmap v6709: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:53.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:11:52 smithi083 ceph-mon[77242]: from='client.51868 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:11:53.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:11:52 smithi083 ceph-mon[77242]: from='client.51874 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:11:53.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:11:52 smithi083 ceph-mon[77242]: pgmap v6709: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:53.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:11:52 smithi083 ceph-mon[79930]: from='client.51868 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:11:53.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:11:52 smithi083 ceph-mon[79930]: from='client.51874 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:11:53.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:11:52 smithi083 ceph-mon[79930]: pgmap v6709: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:11:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:11:54.354Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:11:55.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:11:54 smithi153 ceph-mon[70044]: pgmap v6710: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:55.065 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:11:54 smithi083 ceph-mon[79930]: pgmap v6710: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:55.065 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:11:54 smithi083 ceph-mon[77242]: pgmap v6710: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:11:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:11:55.063Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:11:57.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:11:56 smithi153 ceph-mon[70044]: pgmap v6711: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:57.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:11:56 smithi083 ceph-mon[77242]: pgmap v6711: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:57.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:11:56 smithi083 ceph-mon[79930]: pgmap v6711: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:58.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:11:58 smithi083 ceph-mon[79930]: pgmap v6712: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:58.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:11:58 smithi083 ceph-mon[77242]: pgmap v6712: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:11:59.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:11:58 smithi153 ceph-mon[70044]: pgmap v6712: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:01.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:12:00 smithi153 ceph-mon[70044]: pgmap v6713: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:01.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:12:00 smithi083 ceph-mon[79930]: pgmap v6713: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:01.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:12:00 smithi083 ceph-mon[77242]: pgmap v6713: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:01.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:12:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:12:00.973Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:12:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:12:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:12:01] "GET /metrics HTTP/1.1" 200 225217 "" "Prometheus/2.33.4" 2022-09-29T09:12:02.762 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:12:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:12:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:12:03.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:12:02 smithi153 ceph-mon[70044]: pgmap v6714: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:03.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:12:02 smithi083 ceph-mon[79930]: pgmap v6714: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:03.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:12:02 smithi083 ceph-mon[77242]: pgmap v6714: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:12:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:12:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:12:04.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:12:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:12:04.355Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:12:05.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:12:04 smithi153 ceph-mon[70044]: pgmap v6715: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:05.065 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:12:04 smithi083 ceph-mon[79930]: pgmap v6715: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:05.066 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:12:04 smithi083 ceph-mon[77242]: pgmap v6715: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:05.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:12:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:12:05.063Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:12:07.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:12:06 smithi153 ceph-mon[70044]: pgmap v6716: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:07.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:12:06 smithi083 ceph-mon[79930]: pgmap v6716: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:07.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:12:06 smithi083 ceph-mon[77242]: pgmap v6716: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:08.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:12:08 smithi083 ceph-mon[77242]: pgmap v6717: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:08.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:12:08 smithi083 ceph-mon[79930]: pgmap v6717: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:09.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:12:08 smithi153 ceph-mon[70044]: pgmap v6717: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:11.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:12:10 smithi153 ceph-mon[70044]: pgmap v6718: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:11.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:12:10 smithi083 ceph-mon[79930]: pgmap v6718: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:11.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:12:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:12:10.974Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:12:11.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:12:10 smithi083 ceph-mon[77242]: pgmap v6718: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:12:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:12:11] "GET /metrics HTTP/1.1" 200 225217 "" "Prometheus/2.33.4" 2022-09-29T09:12:12.762 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:12:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:12:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:12:13.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:12:12 smithi153 ceph-mon[70044]: pgmap v6719: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:13.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:12:12 smithi083 ceph-mon[77242]: pgmap v6719: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:13.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:12:12 smithi083 ceph-mon[79930]: pgmap v6719: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:12:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:12:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:12:14.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:12:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:12:14.355Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:12:15.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:12:14 smithi153 ceph-mon[70044]: pgmap v6720: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:15.066 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:12:14 smithi083 ceph-mon[79930]: pgmap v6720: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:15.066 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:12:14 smithi083 ceph-mon[77242]: pgmap v6720: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:15.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:12:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:12:15.064Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:12:17.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:12:16 smithi153 ceph-mon[70044]: pgmap v6721: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:17.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:12:16 smithi083 ceph-mon[79930]: pgmap v6721: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:17.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:12:16 smithi083 ceph-mon[77242]: pgmap v6721: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:18.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:12:18 smithi083 ceph-mon[79930]: pgmap v6722: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:18.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:12:18 smithi083 ceph-mon[77242]: pgmap v6722: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:19.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:12:18 smithi153 ceph-mon[70044]: pgmap v6722: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:21.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:12:20 smithi153 ceph-mon[70044]: pgmap v6723: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:21.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:12:20 smithi083 ceph-mon[79930]: pgmap v6723: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:21.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:12:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:12:20.975Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:12:21.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:12:20 smithi083 ceph-mon[77242]: pgmap v6723: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:12:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:12:21] "GET /metrics HTTP/1.1" 200 225217 "" "Prometheus/2.33.4" 2022-09-29T09:12:22.055 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:12:22.344 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:12:22.344 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 3m ago 3h 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:12:22.344 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 97s ago 3h 63.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:12:22.344 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 3m ago 3h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:12:22.345 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 97s ago 3h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:12:22.345 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 97s ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:12:22.345 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 3m ago 3h 706M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:12:22.345 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 3m ago 3h 851M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:12:22.346 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 97s ago 3h 375M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:12:22.346 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 3m ago 3h 497M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:12:22.346 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 3m ago 3h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:12:22.346 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 97s ago 3h 25.7M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:12:22.347 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 3m ago 3h 615M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:12:22.347 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 3m ago 3h 606M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:12:22.347 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 3m ago 3h 475M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:12:22.347 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 3m ago 3h 589M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:12:22.347 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 97s ago 3h 522M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:12:22.348 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 97s ago 3h 648M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:12:22.348 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 97s ago 3h 577M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:12:22.348 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 97s ago 3h 680M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:12:22.348 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 97s ago 3h 167M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:12:22.698 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:12:22.699 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:12:22.699 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:12:22.700 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:12:22.700 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:12:22.700 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:12:22.700 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:12:22.701 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:12:22.701 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:12:22.701 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:12:22.701 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:12:22.701 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:12:22.702 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:12:22.702 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:12:22.702 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:12:22.702 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:12:22.703 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:12:22.703 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:12:22.765 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:12:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:12:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:12:23.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:12:22 smithi153 ceph-mon[70044]: pgmap v6724: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:23.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:12:22 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1037657556' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:12:23.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:12:22 smithi083 ceph-mon[77242]: pgmap v6724: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:23.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:12:22 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1037657556' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:12:23.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:12:22 smithi083 ceph-mon[79930]: pgmap v6724: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:23.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:12:22 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1037657556' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:12:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:12:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:12:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:12:24.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:12:23 smithi153 ceph-mon[70044]: from='client.51886 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:12:24.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:12:23 smithi153 ceph-mon[70044]: from='client.41961 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:12:24.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:12:23 smithi083 ceph-mon[77242]: from='client.51886 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:12:24.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:12:23 smithi083 ceph-mon[77242]: from='client.41961 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:12:24.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:12:23 smithi083 ceph-mon[79930]: from='client.51886 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:12:24.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:12:23 smithi083 ceph-mon[79930]: from='client.41961 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:12:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:12:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:12:24.356Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:12:25.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:12:24 smithi153 ceph-mon[70044]: pgmap v6725: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:25.067 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:12:24 smithi083 ceph-mon[79930]: pgmap v6725: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:25.067 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:12:24 smithi083 ceph-mon[77242]: pgmap v6725: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:25.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:12:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:12:25.065Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:12:27.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:12:26 smithi153 ceph-mon[70044]: pgmap v6726: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:27.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:12:26 smithi083 ceph-mon[77242]: pgmap v6726: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:27.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:12:26 smithi083 ceph-mon[79930]: pgmap v6726: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:28.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:12:28 smithi083 ceph-mon[77242]: pgmap v6727: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:28.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:12:28 smithi083 ceph-mon[79930]: pgmap v6727: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:29.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:12:28 smithi153 ceph-mon[70044]: pgmap v6727: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:31.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:12:30 smithi153 ceph-mon[70044]: pgmap v6728: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:31.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:12:30 smithi083 ceph-mon[77242]: pgmap v6728: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:31.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:12:30 smithi083 ceph-mon[79930]: pgmap v6728: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:31.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:12:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:12:30.976Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:12:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:12:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:12:31] "GET /metrics HTTP/1.1" 200 225213 "" "Prometheus/2.33.4" 2022-09-29T09:12:32.769 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:12:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:12:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:12:33.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:12:32 smithi153 ceph-mon[70044]: pgmap v6729: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:33.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:12:32 smithi083 ceph-mon[77242]: pgmap v6729: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:33.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:12:32 smithi083 ceph-mon[79930]: pgmap v6729: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:12:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:12:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:12:34.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:12:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:12:34.357Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:12:35.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:12:34 smithi153 ceph-mon[70044]: pgmap v6730: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:35.067 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:12:34 smithi083 ceph-mon[79930]: pgmap v6730: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:35.068 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:12:34 smithi083 ceph-mon[77242]: pgmap v6730: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:35.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:12:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:12:35.065Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:12:37.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:12:36 smithi153 ceph-mon[70044]: pgmap v6731: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:37.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:12:36 smithi083 ceph-mon[79930]: pgmap v6731: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:37.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:12:36 smithi083 ceph-mon[77242]: pgmap v6731: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:38.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:12:38 smithi083 ceph-mon[77242]: pgmap v6732: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:38.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:12:38 smithi083 ceph-mon[79930]: pgmap v6732: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:39.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:12:38 smithi153 ceph-mon[70044]: pgmap v6732: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:12:40 smithi153 ceph-mon[70044]: pgmap v6733: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:41.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:12:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:12:41.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:12:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:12:41.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:12:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:12:41.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:12:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:12:41.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:12:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:12:40.976Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:12:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:12:40 smithi083 ceph-mon[77242]: pgmap v6733: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:12:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:12:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:12:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:12:41.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:12:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:12:41.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:12:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:12:41.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:12:40 smithi083 ceph-mon[79930]: pgmap v6733: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:41.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:12:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:12:41.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:12:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:12:41.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:12:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:12:41.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:12:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:12:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:12:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:12:41] "GET /metrics HTTP/1.1" 200 225220 "" "Prometheus/2.33.4" 2022-09-29T09:12:42.772 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:12:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:12:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:12:43.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:12:42 smithi153 ceph-mon[70044]: pgmap v6734: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:43.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:12:42 smithi083 ceph-mon[79930]: pgmap v6734: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:43.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:12:42 smithi083 ceph-mon[77242]: pgmap v6734: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:12:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:12:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:12:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:12:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:12:44.358Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:12:45.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:12:44 smithi153 ceph-mon[70044]: pgmap v6735: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:45.068 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:12:44 smithi083 ceph-mon[79930]: pgmap v6735: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:45.068 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:12:44 smithi083 ceph-mon[77242]: pgmap v6735: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:45.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:12:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:12:45.066Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:12:47.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:12:46 smithi153 ceph-mon[70044]: pgmap v6736: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:47.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:12:46 smithi083 ceph-mon[77242]: pgmap v6736: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:47.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:12:46 smithi083 ceph-mon[79930]: pgmap v6736: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:48.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:12:48 smithi083 ceph-mon[77242]: pgmap v6737: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:48.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:12:48 smithi083 ceph-mon[79930]: pgmap v6737: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:49.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:12:48 smithi153 ceph-mon[70044]: pgmap v6737: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:49.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:12:49 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:12:49.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:12:49 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:12:49.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:12:49 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:12:49.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:12:49 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:12:49.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:12:49 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:12:49.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:12:49 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:12:50.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:12:49 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:12:50.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:12:49 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:12:50.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:12:49 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:12:50.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:12:50 smithi083 ceph-mon[77242]: pgmap v6738: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:50.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:12:50 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:12:50.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:12:50 smithi083 ceph-mon[79930]: pgmap v6738: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:50.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:12:50 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:12:51.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:12:50 smithi153 ceph-mon[70044]: pgmap v6738: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:51.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:12:50 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:12:51.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:12:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:12:50.977Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:12:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:12:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:12:51] "GET /metrics HTTP/1.1" 200 225220 "" "Prometheus/2.33.4" 2022-09-29T09:12:52.775 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:12:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:12:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:12:53.028 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:12:53.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:12:52 smithi153 ceph-mon[70044]: pgmap v6739: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:53.059 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:12:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:12:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:12:53.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:12:52 smithi083 ceph-mon[77242]: pgmap v6739: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:53.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:12:52 smithi083 ceph-mon[79930]: pgmap v6739: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:53.317 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:12:53.317 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 4m ago 3h 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:12:53.318 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 2m ago 3h 63.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:12:53.318 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 4m ago 3h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:12:53.318 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 2m ago 3h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:12:53.318 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 2m ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:12:53.318 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 4m ago 3h 706M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:12:53.319 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 4m ago 3h 851M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:12:53.319 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 2m ago 3h 375M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:12:53.319 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 4m ago 3h 497M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:12:53.319 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 4m ago 3h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:12:53.319 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 2m ago 3h 25.7M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:12:53.320 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 4m ago 3h 615M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:12:53.320 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 4m ago 3h 606M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:12:53.320 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 4m ago 3h 475M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:12:53.320 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 4m ago 3h 589M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:12:53.320 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 2m ago 3h 522M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:12:53.321 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 2m ago 3h 648M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:12:53.321 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 2m ago 3h 577M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:12:53.321 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 2m ago 3h 680M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:12:53.321 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 2m ago 3h 167M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:12:53.681 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:12:53.681 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:12:53.681 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:12:53.681 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:12:53.682 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:12:53.682 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:12:53.682 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:12:53.682 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:12:53.683 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:12:53.683 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:12:53.683 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:12:53.683 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:12:53.684 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:12:53.684 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:12:53.684 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:12:53.684 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:12:53.684 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:12:53.685 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:12:54.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:12:53 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3046340654' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:12:54.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:12:53 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3046340654' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:12:54.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:12:53 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3046340654' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:12:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:12:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:12:54.358Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:12:55.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:12:54 smithi153 ceph-mon[70044]: from='client.51904 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:12:55.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:12:54 smithi153 ceph-mon[70044]: from='client.51910 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:12:55.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:12:54 smithi153 ceph-mon[70044]: pgmap v6740: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:55.068 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:12:54 smithi083 ceph-mon[77242]: from='client.51904 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:12:55.069 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:12:54 smithi083 ceph-mon[77242]: from='client.51910 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:12:55.069 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:12:54 smithi083 ceph-mon[77242]: pgmap v6740: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:55.069 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:12:54 smithi083 ceph-mon[79930]: from='client.51904 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:12:55.069 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:12:54 smithi083 ceph-mon[79930]: from='client.51910 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:12:55.070 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:12:54 smithi083 ceph-mon[79930]: pgmap v6740: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:12:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:12:55.066Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:12:57.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:12:56 smithi153 ceph-mon[70044]: pgmap v6741: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:57.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:12:56 smithi083 ceph-mon[79930]: pgmap v6741: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:57.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:12:56 smithi083 ceph-mon[77242]: pgmap v6741: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:58.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:12:58 smithi083 ceph-mon[79930]: pgmap v6742: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:58.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:12:58 smithi083 ceph-mon[77242]: pgmap v6742: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:12:59.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:12:58 smithi153 ceph-mon[70044]: pgmap v6742: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:01.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:13:00 smithi153 ceph-mon[70044]: pgmap v6743: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:01.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:13:00 smithi083 ceph-mon[77242]: pgmap v6743: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:01.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:13:00 smithi083 ceph-mon[79930]: pgmap v6743: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:01.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:13:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:13:00.978Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:13:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:13:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:13:01] "GET /metrics HTTP/1.1" 200 225217 "" "Prometheus/2.33.4" 2022-09-29T09:13:02.780 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:13:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:13:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:13:03.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:13:02 smithi153 ceph-mon[70044]: pgmap v6744: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:03.059 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:13:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:13:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:13:03.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:13:02 smithi083 ceph-mon[79930]: pgmap v6744: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:03.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:13:02 smithi083 ceph-mon[77242]: pgmap v6744: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:04.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:13:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:13:04.360Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:13:05.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:13:04 smithi153 ceph-mon[70044]: pgmap v6745: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:05.069 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:13:04 smithi083 ceph-mon[79930]: pgmap v6745: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:05.069 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:13:04 smithi083 ceph-mon[77242]: pgmap v6745: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:05.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:13:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:13:05.067Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:13:07.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:13:06 smithi153 ceph-mon[70044]: pgmap v6746: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:07.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:13:06 smithi083 ceph-mon[79930]: pgmap v6746: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:07.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:13:06 smithi083 ceph-mon[77242]: pgmap v6746: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:08.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:13:08 smithi083 ceph-mon[79930]: pgmap v6747: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:08.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:13:08 smithi083 ceph-mon[77242]: pgmap v6747: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:09.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:13:08 smithi153 ceph-mon[70044]: pgmap v6747: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:11.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:13:10 smithi153 ceph-mon[70044]: pgmap v6748: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:11.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:13:10 smithi083 ceph-mon[77242]: pgmap v6748: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:11.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:13:10 smithi083 ceph-mon[79930]: pgmap v6748: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:11.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:13:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:13:10.979Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:13:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:13:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:13:11] "GET /metrics HTTP/1.1" 200 225220 "" "Prometheus/2.33.4" 2022-09-29T09:13:12.789 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:13:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:13:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:13:13.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:13:12 smithi153 ceph-mon[70044]: pgmap v6749: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:13.059 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:13:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:13:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:13:13.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:13:12 smithi083 ceph-mon[77242]: pgmap v6749: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:13.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:13:12 smithi083 ceph-mon[79930]: pgmap v6749: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:14.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:13:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:13:14.361Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:13:15.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:13:14 smithi153 ceph-mon[70044]: pgmap v6750: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:15.070 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:13:14 smithi083 ceph-mon[79930]: pgmap v6750: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:15.070 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:13:14 smithi083 ceph-mon[77242]: pgmap v6750: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:15.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:13:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:13:15.068Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:13:17.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:13:16 smithi153 ceph-mon[70044]: pgmap v6751: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:17.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:13:16 smithi083 ceph-mon[79930]: pgmap v6751: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:17.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:13:16 smithi083 ceph-mon[77242]: pgmap v6751: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:18.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:13:18 smithi083 ceph-mon[79930]: pgmap v6752: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:18.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:13:18 smithi083 ceph-mon[77242]: pgmap v6752: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:19.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:13:18 smithi153 ceph-mon[70044]: pgmap v6752: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:21.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:13:20 smithi153 ceph-mon[70044]: pgmap v6753: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:21.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:13:20 smithi083 ceph-mon[79930]: pgmap v6753: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:21.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:13:20 smithi083 ceph-mon[77242]: pgmap v6753: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:21.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:13:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:13:20.979Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:13:21.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:13:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:13:21] "GET /metrics HTTP/1.1" 200 225220 "" "Prometheus/2.33.4" 2022-09-29T09:13:22.786 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:13:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:13:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:13:23.058 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:13:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:13:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:13:23.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:13:22 smithi153 ceph-mon[70044]: pgmap v6754: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:23.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:13:22 smithi083 ceph-mon[77242]: pgmap v6754: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:23.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:13:22 smithi083 ceph-mon[79930]: pgmap v6754: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:24.010 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:13:24.301 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:13:24.302 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 4m ago 3h 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:13:24.302 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 2m ago 3h 63.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:13:24.303 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 4m ago 3h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:13:24.303 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 2m ago 3h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:13:24.303 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 2m ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:13:24.304 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 4m ago 3h 706M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:13:24.304 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 4m ago 3h 851M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:13:24.304 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 2m ago 3h 375M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:13:24.304 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 4m ago 3h 497M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:13:24.305 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 4m ago 3h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:13:24.305 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 2m ago 3h 25.7M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:13:24.305 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 4m ago 3h 615M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:13:24.305 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 4m ago 3h 606M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:13:24.306 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 4m ago 3h 475M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:13:24.306 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 4m ago 3h 589M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:13:24.306 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 2m ago 3h 522M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:13:24.307 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 2m ago 3h 648M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:13:24.307 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 2m ago 3h 577M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:13:24.307 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 2m ago 3h 680M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:13:24.308 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 2m ago 3h 167M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:13:24.656 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:13:24.657 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:13:24.657 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:13:24.657 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:13:24.657 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:13:24.658 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:13:24.658 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:13:24.658 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:13:24.658 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:13:24.659 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:13:24.659 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:13:24.659 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:13:24.659 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:13:24.659 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:13:24.660 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:13:24.660 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:13:24.660 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:13:24.660 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:13:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:13:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:13:24.360Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:13:25.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:13:24 smithi153 ceph-mon[70044]: pgmap v6755: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:25.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:13:24 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/810355386' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:13:25.070 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:13:24 smithi083 ceph-mon[79930]: pgmap v6755: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:25.070 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:13:24 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/810355386' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:13:25.071 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:13:24 smithi083 ceph-mon[77242]: pgmap v6755: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:25.071 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:13:24 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/810355386' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:13:25.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:13:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:13:25.068Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:13:26.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:13:25 smithi153 ceph-mon[70044]: from='client.51922 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:13:26.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:13:25 smithi153 ceph-mon[70044]: from='client.41997 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:13:26.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:13:25 smithi083 ceph-mon[79930]: from='client.51922 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:13:26.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:13:25 smithi083 ceph-mon[79930]: from='client.41997 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:13:26.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:13:25 smithi083 ceph-mon[77242]: from='client.51922 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:13:26.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:13:25 smithi083 ceph-mon[77242]: from='client.41997 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:13:27.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:13:26 smithi153 ceph-mon[70044]: pgmap v6756: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:27.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:13:26 smithi083 ceph-mon[79930]: pgmap v6756: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:27.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:13:26 smithi083 ceph-mon[77242]: pgmap v6756: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:28.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:13:28 smithi083 ceph-mon[79930]: pgmap v6757: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:28.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:13:28 smithi083 ceph-mon[77242]: pgmap v6757: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:29.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:13:28 smithi153 ceph-mon[70044]: pgmap v6757: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:31.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:13:30 smithi153 ceph-mon[70044]: pgmap v6758: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:31.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:13:30 smithi083 ceph-mon[79930]: pgmap v6758: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:31.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:13:30 smithi083 ceph-mon[77242]: pgmap v6758: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:31.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:13:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:13:30.980Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:13:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:13:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:13:31] "GET /metrics HTTP/1.1" 200 225217 "" "Prometheus/2.33.4" 2022-09-29T09:13:32.787 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:13:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:13:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:13:33.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:13:32 smithi153 ceph-mon[70044]: pgmap v6759: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:33.059 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:13:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:13:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:13:33.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:13:32 smithi083 ceph-mon[79930]: pgmap v6759: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:33.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:13:32 smithi083 ceph-mon[77242]: pgmap v6759: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:34.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:13:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:13:34.362Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:13:35.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:13:34 smithi153 ceph-mon[70044]: pgmap v6760: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:35.071 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:13:34 smithi083 ceph-mon[79930]: pgmap v6760: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:35.072 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:13:34 smithi083 ceph-mon[77242]: pgmap v6760: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:35.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:13:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:13:35.069Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:13:37.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:13:36 smithi153 ceph-mon[70044]: pgmap v6761: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:37.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:13:36 smithi083 ceph-mon[77242]: pgmap v6761: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:37.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:13:36 smithi083 ceph-mon[79930]: pgmap v6761: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:38.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:13:38 smithi083 ceph-mon[79930]: pgmap v6762: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:38.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:13:38 smithi083 ceph-mon[77242]: pgmap v6762: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:39.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:13:38 smithi153 ceph-mon[70044]: pgmap v6762: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:13:40 smithi153 ceph-mon[70044]: pgmap v6763: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:41.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:13:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:13:41.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:13:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:13:41.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:13:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:13:41.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:13:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:13:41.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:13:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:13:40.981Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:13:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:13:40 smithi083 ceph-mon[77242]: pgmap v6763: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:13:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:13:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:13:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:13:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:13:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:13:41.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:13:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:13:41.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:13:40 smithi083 ceph-mon[79930]: pgmap v6763: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:41.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:13:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:13:41.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:13:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:13:41.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:13:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:13:41.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:13:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:13:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:13:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:13:41] "GET /metrics HTTP/1.1" 200 225218 "" "Prometheus/2.33.4" 2022-09-29T09:13:42.792 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:13:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:13:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:13:43.058 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:13:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:13:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:13:43.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:13:42 smithi153 ceph-mon[70044]: pgmap v6764: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:43.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:13:42 smithi083 ceph-mon[79930]: pgmap v6764: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:43.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:13:42 smithi083 ceph-mon[77242]: pgmap v6764: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:13:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:13:44.362Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:13:45.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:13:44 smithi153 ceph-mon[70044]: pgmap v6765: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:45.072 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:13:44 smithi083 ceph-mon[79930]: pgmap v6765: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:45.073 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:13:44 smithi083 ceph-mon[77242]: pgmap v6765: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:45.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:13:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:13:45.070Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:13:47.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:13:46 smithi153 ceph-mon[70044]: pgmap v6766: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:47.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:13:46 smithi083 ceph-mon[79930]: pgmap v6766: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:47.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:13:46 smithi083 ceph-mon[77242]: pgmap v6766: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:48.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:13:48 smithi083 ceph-mon[77242]: pgmap v6767: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:48.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:13:48 smithi083 ceph-mon[79930]: pgmap v6767: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:49.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:13:48 smithi153 ceph-mon[70044]: pgmap v6767: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:51.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:13:50 smithi153 ceph-mon[70044]: pgmap v6768: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:51.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:13:50 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:13:51.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:13:50 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:13:51.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:13:50 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:13:51.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:13:50 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:13:51.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:13:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:13:50.981Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:13:51.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:13:50 smithi083 ceph-mon[77242]: pgmap v6768: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:51.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:13:50 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:13:51.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:13:50 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:13:51.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:13:50 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:13:51.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:13:50 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:13:51.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:13:50 smithi083 ceph-mon[79930]: pgmap v6768: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:51.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:13:50 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:13:51.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:13:50 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:13:51.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:13:50 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:13:51.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:13:50 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:13:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:13:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:13:51] "GET /metrics HTTP/1.1" 200 225218 "" "Prometheus/2.33.4" 2022-09-29T09:13:52.799 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:13:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:13:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:13:53.058 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:13:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:13:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:13:53.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:13:52 smithi153 ceph-mon[70044]: pgmap v6769: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:53.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:13:52 smithi083 ceph-mon[79930]: pgmap v6769: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:53.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:13:52 smithi083 ceph-mon[77242]: pgmap v6769: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:13:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:13:54.363Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:13:54.985 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:13:55.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:13:54 smithi153 ceph-mon[70044]: pgmap v6770: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:55.072 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:13:54 smithi083 ceph-mon[79930]: pgmap v6770: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:55.072 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:13:54 smithi083 ceph-mon[77242]: pgmap v6770: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:55.273 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:13:55.274 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 5m ago 3h 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:13:55.274 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 3m ago 3h 63.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:13:55.274 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 5m ago 3h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:13:55.274 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 3m ago 3h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:13:55.275 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 3m ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:13:55.275 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 5m ago 3h 706M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:13:55.275 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 5m ago 3h 851M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:13:55.275 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 3m ago 3h 375M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:13:55.275 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 5m ago 3h 497M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:13:55.276 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 5m ago 3h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:13:55.276 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 3m ago 3h 25.7M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:13:55.276 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 5m ago 3h 615M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:13:55.276 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 5m ago 3h 606M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:13:55.277 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 5m ago 3h 475M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:13:55.277 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 5m ago 3h 589M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:13:55.277 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 3m ago 3h 522M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:13:55.277 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 3m ago 3h 648M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:13:55.277 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 3m ago 3h 577M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:13:55.278 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 3m ago 3h 680M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:13:55.278 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 3m ago 3h 167M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:13:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:13:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:13:55.070Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:13:55.628 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:13:55.628 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:13:55.629 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:13:55.629 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:13:55.629 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:13:55.629 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:13:55.630 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:13:55.630 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:13:55.630 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:13:55.630 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:13:55.631 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:13:55.631 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:13:55.631 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:13:55.631 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:13:55.631 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:13:55.632 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:13:55.632 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:13:55.632 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:13:56.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:13:55 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2297979404' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:13:56.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:13:55 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2297979404' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:13:56.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:13:55 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2297979404' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:13:57.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:13:56 smithi153 ceph-mon[70044]: from='client.42009 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:13:57.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:13:56 smithi153 ceph-mon[70044]: from='client.51946 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:13:57.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:13:56 smithi153 ceph-mon[70044]: pgmap v6771: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:57.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:13:56 smithi083 ceph-mon[79930]: from='client.42009 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:13:57.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:13:56 smithi083 ceph-mon[79930]: from='client.51946 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:13:57.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:13:56 smithi083 ceph-mon[79930]: pgmap v6771: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:57.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:13:56 smithi083 ceph-mon[77242]: from='client.42009 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:13:57.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:13:56 smithi083 ceph-mon[77242]: from='client.51946 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:13:57.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:13:56 smithi083 ceph-mon[77242]: pgmap v6771: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:58.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:13:58 smithi083 ceph-mon[79930]: pgmap v6772: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:58.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:13:58 smithi083 ceph-mon[77242]: pgmap v6772: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:13:59.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:13:58 smithi153 ceph-mon[70044]: pgmap v6772: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:01.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:14:00 smithi153 ceph-mon[70044]: pgmap v6773: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:01.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:14:00 smithi083 ceph-mon[79930]: pgmap v6773: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:01.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:14:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:14:00.982Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:14:01.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:14:00 smithi083 ceph-mon[77242]: pgmap v6773: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:14:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:14:01] "GET /metrics HTTP/1.1" 200 225217 "" "Prometheus/2.33.4" 2022-09-29T09:14:02.796 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:14:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:14:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:14:03.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:14:02 smithi153 ceph-mon[70044]: pgmap v6774: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:03.059 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:14:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:14:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:14:03.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:14:02 smithi083 ceph-mon[79930]: pgmap v6774: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:03.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:14:02 smithi083 ceph-mon[77242]: pgmap v6774: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:04.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:14:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:14:04.364Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:14:05.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:14:04 smithi153 ceph-mon[70044]: pgmap v6775: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:05.074 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:14:04 smithi083 ceph-mon[79930]: pgmap v6775: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:05.075 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:14:04 smithi083 ceph-mon[77242]: pgmap v6775: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:05.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:14:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:14:05.072Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:14:07.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:14:06 smithi153 ceph-mon[70044]: pgmap v6776: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:07.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:14:06 smithi083 ceph-mon[79930]: pgmap v6776: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:07.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:14:06 smithi083 ceph-mon[77242]: pgmap v6776: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:08.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:14:08 smithi083 ceph-mon[79930]: pgmap v6777: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:08.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:14:08 smithi083 ceph-mon[77242]: pgmap v6777: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:09.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:14:08 smithi153 ceph-mon[70044]: pgmap v6777: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:11.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:14:10 smithi153 ceph-mon[70044]: pgmap v6778: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:11.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:14:10 smithi083 ceph-mon[79930]: pgmap v6778: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:11.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:14:10 smithi083 ceph-mon[77242]: pgmap v6778: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:11.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:14:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:14:10.983Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:14:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:14:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:14:11] "GET /metrics HTTP/1.1" 200 225217 "" "Prometheus/2.33.4" 2022-09-29T09:14:12.800 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:14:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:14:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:14:13.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:14:12 smithi153 ceph-mon[70044]: pgmap v6779: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:13.059 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:14:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:14:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:14:13.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:14:12 smithi083 ceph-mon[79930]: pgmap v6779: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:13.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:14:12 smithi083 ceph-mon[77242]: pgmap v6779: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:14.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:14:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:14:14.365Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:14:15.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:14:14 smithi153 ceph-mon[70044]: pgmap v6780: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:15.074 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:14:14 smithi083 ceph-mon[79930]: pgmap v6780: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:15.074 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:14:14 smithi083 ceph-mon[77242]: pgmap v6780: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:15.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:14:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:14:15.072Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:14:17.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:14:16 smithi153 ceph-mon[70044]: pgmap v6781: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:17.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:14:16 smithi083 ceph-mon[79930]: pgmap v6781: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:17.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:14:16 smithi083 ceph-mon[77242]: pgmap v6781: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:18.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:14:18 smithi083 ceph-mon[79930]: pgmap v6782: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:18.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:14:18 smithi083 ceph-mon[77242]: pgmap v6782: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:19.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:14:18 smithi153 ceph-mon[70044]: pgmap v6782: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:21.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:14:20 smithi153 ceph-mon[70044]: pgmap v6783: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:21.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:14:20 smithi083 ceph-mon[79930]: pgmap v6783: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:21.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:14:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:14:20.983Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:14:21.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:14:20 smithi083 ceph-mon[77242]: pgmap v6783: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:14:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:14:21] "GET /metrics HTTP/1.1" 200 225217 "" "Prometheus/2.33.4" 2022-09-29T09:14:22.803 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:14:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:14:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:14:23.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:14:22 smithi153 ceph-mon[70044]: pgmap v6784: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:23.059 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:14:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:14:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:14:23.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:14:22 smithi083 ceph-mon[79930]: pgmap v6784: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:23.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:14:22 smithi083 ceph-mon[77242]: pgmap v6784: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:14:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:14:24.365Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:14:25.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:14:24 smithi153 ceph-mon[70044]: pgmap v6785: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:25.075 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:14:24 smithi083 ceph-mon[79930]: pgmap v6785: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:25.075 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:14:24 smithi083 ceph-mon[77242]: pgmap v6785: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:25.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:14:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:14:25.073Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:14:25.955 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:14:26.245 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:14:26.245 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 5m ago 3h 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:14:26.246 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 3m ago 3h 63.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:14:26.246 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 5m ago 3h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:14:26.246 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 3m ago 3h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:14:26.246 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 3m ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:14:26.246 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 5m ago 3h 706M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:14:26.247 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 5m ago 3h 851M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:14:26.247 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 3m ago 3h 375M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:14:26.247 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 5m ago 3h 497M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:14:26.247 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 5m ago 3h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:14:26.248 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 3m ago 3h 25.7M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:14:26.248 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 5m ago 3h 615M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:14:26.248 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 5m ago 3h 606M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:14:26.248 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 5m ago 3h 475M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:14:26.249 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 5m ago 3h 589M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:14:26.249 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 3m ago 3h 522M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:14:26.249 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 3m ago 3h 648M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:14:26.249 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 3m ago 3h 577M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:14:26.249 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 3m ago 3h 680M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:14:26.250 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 3m ago 3h 167M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:14:26.598 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:14:26.599 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:14:26.599 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:14:26.599 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:14:26.600 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:14:26.600 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:14:26.600 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:14:26.600 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:14:26.600 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:14:26.601 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:14:26.601 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:14:26.601 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:14:26.601 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:14:26.602 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:14:26.602 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:14:26.602 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:14:26.602 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:14:26.602 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:14:27.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:14:26 smithi083 ceph-mon[77242]: pgmap v6786: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:27.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:14:26 smithi083 ceph-mon[77242]: from='client.42027 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:14:27.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:14:26 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/824595676' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:14:27.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:14:26 smithi083 ceph-mon[79930]: pgmap v6786: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:27.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:14:26 smithi083 ceph-mon[79930]: from='client.42027 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:14:27.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:14:26 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/824595676' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:14:27.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:14:26 smithi153 ceph-mon[70044]: pgmap v6786: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:27.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:14:26 smithi153 ceph-mon[70044]: from='client.42027 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:14:27.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:14:26 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/824595676' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:14:28.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:14:27 smithi083 ceph-mon[79930]: from='client.42033 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:14:28.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:14:27 smithi083 ceph-mon[77242]: from='client.42033 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:14:28.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:14:27 smithi153 ceph-mon[70044]: from='client.42033 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:14:29.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:14:28 smithi083 ceph-mon[79930]: pgmap v6787: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:29.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:14:28 smithi083 ceph-mon[77242]: pgmap v6787: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:29.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:14:28 smithi153 ceph-mon[70044]: pgmap v6787: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:31.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:14:30 smithi083 ceph-mon[79930]: pgmap v6788: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:31.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:14:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:14:30.984Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:14:31.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:14:30 smithi083 ceph-mon[77242]: pgmap v6788: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:31.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:14:30 smithi153 ceph-mon[70044]: pgmap v6788: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:14:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:14:31] "GET /metrics HTTP/1.1" 200 225223 "" "Prometheus/2.33.4" 2022-09-29T09:14:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:14:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:14:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:14:33.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:14:32 smithi083 ceph-mon[79930]: pgmap v6789: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:33.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:14:32 smithi083 ceph-mon[77242]: pgmap v6789: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:33.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:14:32 smithi153 ceph-mon[70044]: pgmap v6789: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:33.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:14:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:14:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:14:34.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:14:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:14:34.366Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:14:35.075 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:14:34 smithi083 ceph-mon[79930]: pgmap v6790: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:35.076 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:14:34 smithi083 ceph-mon[77242]: pgmap v6790: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:35.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:14:34 smithi153 ceph-mon[70044]: pgmap v6790: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:35.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:14:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:14:35.073Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:14:37.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:14:36 smithi083 ceph-mon[79930]: pgmap v6791: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:37.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:14:36 smithi083 ceph-mon[77242]: pgmap v6791: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:37.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:14:36 smithi153 ceph-mon[70044]: pgmap v6791: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:38.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:14:38 smithi083 ceph-mon[77242]: pgmap v6792: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:38.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:14:38 smithi083 ceph-mon[79930]: pgmap v6792: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:39.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:14:38 smithi153 ceph-mon[70044]: pgmap v6792: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:14:40 smithi153 ceph-mon[70044]: pgmap v6793: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:41.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:14:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:14:41.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:14:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:14:41.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:14:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:14:41.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:14:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:14:41.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:14:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:14:40.984Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:14:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:14:40 smithi083 ceph-mon[77242]: pgmap v6793: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:14:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:14:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:14:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:14:41.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:14:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:14:41.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:14:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:14:41.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:14:40 smithi083 ceph-mon[79930]: pgmap v6793: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:41.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:14:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:14:41.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:14:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:14:41.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:14:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:14:41.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:14:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:14:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:14:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:14:41] "GET /metrics HTTP/1.1" 200 225218 "" "Prometheus/2.33.4" 2022-09-29T09:14:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:14:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:14:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:14:43.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:14:42 smithi083 ceph-mon[79930]: pgmap v6794: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:43.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:14:42 smithi083 ceph-mon[77242]: pgmap v6794: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:43.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:14:42 smithi153 ceph-mon[70044]: pgmap v6794: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:43.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:14:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:14:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:14:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:14:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:14:44.366Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:14:45.076 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:14:44 smithi083 ceph-mon[79930]: pgmap v6795: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:45.076 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:14:44 smithi083 ceph-mon[77242]: pgmap v6795: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:45.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:14:44 smithi153 ceph-mon[70044]: pgmap v6795: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:45.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:14:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:14:45.073Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:14:47.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:14:46 smithi083 ceph-mon[77242]: pgmap v6796: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:47.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:14:46 smithi083 ceph-mon[79930]: pgmap v6796: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:47.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:14:46 smithi153 ceph-mon[70044]: pgmap v6796: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:48.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:14:48 smithi083 ceph-mon[77242]: pgmap v6797: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:48.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:14:48 smithi083 ceph-mon[79930]: pgmap v6797: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:49.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:14:48 smithi153 ceph-mon[70044]: pgmap v6797: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:51.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:14:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:14:50.984Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:14:51.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:14:50 smithi083 ceph-mon[77242]: pgmap v6798: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:51.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:14:50 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:14:51.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:14:50 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:14:51.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:14:50 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:14:51.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:14:50 smithi083 ceph-mon[79930]: pgmap v6798: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:51.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:14:50 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:14:51.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:14:50 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:14:51.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:14:50 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:14:51.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:14:50 smithi153 ceph-mon[70044]: pgmap v6798: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:51.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:14:50 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:14:51.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:14:50 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:14:51.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:14:50 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:14:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:14:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:14:51] "GET /metrics HTTP/1.1" 200 225218 "" "Prometheus/2.33.4" 2022-09-29T09:14:52.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:14:52 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:14:52.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:14:52 smithi083 ceph-mon[79930]: pgmap v6799: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:52.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:14:52 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:14:52.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:14:52 smithi083 ceph-mon[77242]: pgmap v6799: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:52.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:14:52 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:14:52.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:14:52 smithi153 ceph-mon[70044]: pgmap v6799: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:52.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:14:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:14:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:14:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:14:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:14:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:14:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:14:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:14:54.366Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:14:55.077 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:14:54 smithi083 ceph-mon[79930]: pgmap v6800: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:55.078 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:14:54 smithi083 ceph-mon[77242]: pgmap v6800: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:55.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:14:54 smithi153 ceph-mon[70044]: pgmap v6800: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:14:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:14:55.075Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:14:56.924 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:14:57.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:14:56 smithi083 ceph-mon[77242]: pgmap v6801: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:57.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:14:56 smithi083 ceph-mon[79930]: pgmap v6801: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:57.213 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:14:57.213 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 6m ago 3h 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:14:57.214 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 4m ago 3h 63.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:14:57.214 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 6m ago 3h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:14:57.214 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 4m ago 3h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:14:57.214 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 4m ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:14:57.215 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 6m ago 3h 706M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:14:57.215 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 6m ago 3h 851M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:14:57.215 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 4m ago 3h 375M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:14:57.215 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 6m ago 3h 497M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:14:57.216 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 6m ago 3h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:14:57.216 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 4m ago 3h 25.7M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:14:57.216 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 6m ago 3h 615M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:14:57.217 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 6m ago 3h 606M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:14:57.217 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 6m ago 3h 475M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:14:57.217 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 6m ago 3h 589M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:14:57.217 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 4m ago 3h 522M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:14:57.217 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 4m ago 3h 648M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:14:57.218 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 4m ago 3h 577M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:14:57.218 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 4m ago 3h 680M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:14:57.218 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 4m ago 3h 167M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:14:57.314 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:14:56 smithi153 ceph-mon[70044]: pgmap v6801: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:57.572 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:14:57.572 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:14:57.573 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:14:57.573 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:14:57.573 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:14:57.574 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:14:57.574 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:14:57.574 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:14:57.574 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:14:57.575 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:14:57.575 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:14:57.575 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:14:57.575 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:14:57.576 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:14:57.576 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:14:57.576 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:14:57.576 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:14:57.577 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:14:58.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:14:57 smithi083 ceph-mon[79930]: from='client.42045 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:14:58.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:14:57 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/514158497' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:14:58.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:14:57 smithi083 ceph-mon[77242]: from='client.42045 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:14:58.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:14:57 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/514158497' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:14:58.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:14:57 smithi153 ceph-mon[70044]: from='client.42045 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:14:58.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:14:57 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/514158497' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:14:59.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:14:58 smithi083 ceph-mon[79930]: from='client.42051 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:14:59.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:14:58 smithi083 ceph-mon[79930]: pgmap v6802: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:59.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:14:58 smithi083 ceph-mon[77242]: from='client.42051 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:14:59.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:14:58 smithi083 ceph-mon[77242]: pgmap v6802: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:14:59.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:14:58 smithi153 ceph-mon[70044]: from='client.42051 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:14:59.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:14:58 smithi153 ceph-mon[70044]: pgmap v6802: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:01.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:15:00 smithi083 ceph-mon[79930]: pgmap v6803: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:01.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:15:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:15:00.986Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:15:01.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:15:00 smithi083 ceph-mon[77242]: pgmap v6803: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:01.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:15:00 smithi153 ceph-mon[70044]: pgmap v6803: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:15:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:15:01] "GET /metrics HTTP/1.1" 200 225220 "" "Prometheus/2.33.4" 2022-09-29T09:15:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:15:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:15:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:15:03.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:15:02 smithi083 ceph-mon[79930]: pgmap v6804: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:03.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:15:02 smithi083 ceph-mon[77242]: pgmap v6804: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:03.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:15:02 smithi153 ceph-mon[70044]: pgmap v6804: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:15:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:15:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:15:04.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:15:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:15:04.367Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:15:05.079 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:15:04 smithi083 ceph-mon[79930]: pgmap v6805: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:05.079 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:15:04 smithi083 ceph-mon[77242]: pgmap v6805: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:05.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:15:04 smithi153 ceph-mon[70044]: pgmap v6805: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:05.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:15:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:15:05.076Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:15:07.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:15:06 smithi083 ceph-mon[79930]: pgmap v6806: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:07.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:15:06 smithi083 ceph-mon[77242]: pgmap v6806: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:07.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:15:06 smithi153 ceph-mon[70044]: pgmap v6806: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:08.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:15:08 smithi083 ceph-mon[79930]: pgmap v6807: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:08.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:15:08 smithi083 ceph-mon[77242]: pgmap v6807: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:09.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:15:08 smithi153 ceph-mon[70044]: pgmap v6807: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:11.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:15:10 smithi083 ceph-mon[77242]: pgmap v6808: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:11.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:15:10 smithi083 ceph-mon[79930]: pgmap v6808: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:11.214 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:15:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:15:10.986Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:15:11.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:15:10 smithi153 ceph-mon[70044]: pgmap v6808: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:15:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:15:11] "GET /metrics HTTP/1.1" 200 225219 "" "Prometheus/2.33.4" 2022-09-29T09:15:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:15:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:15:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:15:13.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:15:12 smithi083 ceph-mon[79930]: pgmap v6809: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:13.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:15:12 smithi083 ceph-mon[77242]: pgmap v6809: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:13.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:15:12 smithi153 ceph-mon[70044]: pgmap v6809: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:13.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:15:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:15:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:15:14.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:15:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:15:14.367Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:15:15.079 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:15:14 smithi083 ceph-mon[79930]: pgmap v6810: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:15.080 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:15:14 smithi083 ceph-mon[77242]: pgmap v6810: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:15.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:15:14 smithi153 ceph-mon[70044]: pgmap v6810: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:15.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:15:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:15:15.077Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:15:17.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:15:16 smithi083 ceph-mon[79930]: pgmap v6811: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:17.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:15:16 smithi083 ceph-mon[77242]: pgmap v6811: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:17.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:15:16 smithi153 ceph-mon[70044]: pgmap v6811: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:18.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:15:18 smithi083 ceph-mon[79930]: pgmap v6812: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:18.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:15:18 smithi083 ceph-mon[77242]: pgmap v6812: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:19.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:15:18 smithi153 ceph-mon[70044]: pgmap v6812: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:21.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:15:20 smithi083 ceph-mon[79930]: pgmap v6813: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:21.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:15:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:15:20.987Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:15:21.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:15:20 smithi083 ceph-mon[77242]: pgmap v6813: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:21.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:15:20 smithi153 ceph-mon[70044]: pgmap v6813: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:15:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:15:21] "GET /metrics HTTP/1.1" 200 225219 "" "Prometheus/2.33.4" 2022-09-29T09:15:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:15:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:15:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:15:23.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:15:22 smithi083 ceph-mon[79930]: pgmap v6814: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:23.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:15:22 smithi083 ceph-mon[77242]: pgmap v6814: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:15:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:15:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:15:23.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:15:22 smithi153 ceph-mon[70044]: pgmap v6814: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:15:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:15:24.367Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:15:25.080 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:15:24 smithi083 ceph-mon[79930]: pgmap v6815: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:25.080 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:15:24 smithi083 ceph-mon[77242]: pgmap v6815: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:25.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:15:24 smithi153 ceph-mon[70044]: pgmap v6815: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:25.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:15:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:15:25.077Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:15:27.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:15:26 smithi083 ceph-mon[79930]: pgmap v6816: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:27.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:15:26 smithi083 ceph-mon[77242]: pgmap v6816: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:27.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:15:26 smithi153 ceph-mon[70044]: pgmap v6816: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:27.904 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:15:28.191 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:15:28.191 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 6m ago 3h 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:15:28.191 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 4m ago 3h 63.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:15:28.192 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 6m ago 3h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:15:28.192 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 4m ago 3h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:15:28.192 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 4m ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:15:28.192 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 6m ago 3h 706M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:15:28.192 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 6m ago 3h 851M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:15:28.193 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 4m ago 3h 375M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:15:28.193 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 6m ago 3h 497M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:15:28.193 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 6m ago 3h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:15:28.193 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 4m ago 3h 25.7M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:15:28.193 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 6m ago 3h 615M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:15:28.194 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 6m ago 3h 606M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:15:28.194 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 6m ago 3h 475M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:15:28.194 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 6m ago 3h 589M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:15:28.194 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 4m ago 3h 522M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:15:28.195 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 4m ago 3h 648M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:15:28.195 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 4m ago 3h 577M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:15:28.195 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 4m ago 3h 680M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:15:28.195 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 4m ago 3h 167M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:15:28.542 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:15:28.542 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:15:28.543 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:15:28.543 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:15:28.543 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:15:28.543 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:15:28.544 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:15:28.544 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:15:28.544 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:15:28.544 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:15:28.544 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:15:28.545 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:15:28.545 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:15:28.545 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:15:28.545 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:15:28.546 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:15:28.546 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:15:28.546 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:15:28.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:15:28 smithi083 ceph-mon[79930]: pgmap v6817: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:28.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:15:28 smithi083 ceph-mon[79930]: from='client.51994 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:15:28.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:15:28 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1853505540' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:15:28.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:15:28 smithi083 ceph-mon[77242]: pgmap v6817: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:28.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:15:28 smithi083 ceph-mon[77242]: from='client.51994 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:15:28.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:15:28 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1853505540' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:15:29.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:15:28 smithi153 ceph-mon[70044]: pgmap v6817: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:29.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:15:28 smithi153 ceph-mon[70044]: from='client.51994 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:15:29.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:15:28 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1853505540' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:15:29.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:15:29 smithi083 ceph-mon[79930]: from='client.42069 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:15:29.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:15:29 smithi083 ceph-mon[77242]: from='client.42069 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:15:30.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:15:29 smithi153 ceph-mon[70044]: from='client.42069 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:15:30.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:15:30 smithi083 ceph-mon[79930]: pgmap v6818: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:30.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:15:30 smithi083 ceph-mon[77242]: pgmap v6818: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:31.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:15:30 smithi153 ceph-mon[70044]: pgmap v6818: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:31.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:15:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:15:30.987Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:15:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:15:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:15:31] "GET /metrics HTTP/1.1" 200 225218 "" "Prometheus/2.33.4" 2022-09-29T09:15:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:15:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:15:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:15:33.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:15:32 smithi083 ceph-mon[79930]: pgmap v6819: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:33.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:15:32 smithi083 ceph-mon[77242]: pgmap v6819: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:33.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:15:32 smithi153 ceph-mon[70044]: pgmap v6819: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:33.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:15:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:15:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:15:34.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:15:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:15:34.368Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:15:35.081 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:15:34 smithi083 ceph-mon[79930]: pgmap v6820: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:35.081 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:15:34 smithi083 ceph-mon[77242]: pgmap v6820: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:35.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:15:34 smithi153 ceph-mon[70044]: pgmap v6820: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:35.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:15:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:15:35.078Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:15:37.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:15:36 smithi083 ceph-mon[79930]: pgmap v6821: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:37.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:15:36 smithi083 ceph-mon[77242]: pgmap v6821: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:37.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:15:36 smithi153 ceph-mon[70044]: pgmap v6821: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:38.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:15:38 smithi083 ceph-mon[79930]: pgmap v6822: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:38.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:15:38 smithi083 ceph-mon[77242]: pgmap v6822: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:39.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:15:38 smithi153 ceph-mon[70044]: pgmap v6822: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:41.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:15:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:15:40.988Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:15:41.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:15:40 smithi083 ceph-mon[79930]: pgmap v6823: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:41.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:15:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:15:41.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:15:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:15:41.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:15:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:15:41.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:15:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:15:41.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:15:40 smithi083 ceph-mon[77242]: pgmap v6823: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:41.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:15:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:15:41.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:15:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:15:41.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:15:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:15:41.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:15:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:15:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:15:40 smithi153 ceph-mon[70044]: pgmap v6823: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:15:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:15:41.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:15:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:15:41.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:15:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:15:41.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:15:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:15:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:15:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:15:41] "GET /metrics HTTP/1.1" 200 225220 "" "Prometheus/2.33.4" 2022-09-29T09:15:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:15:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:15:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:15:43.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:15:42 smithi083 ceph-mon[77242]: pgmap v6824: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:43.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:15:42 smithi083 ceph-mon[79930]: pgmap v6824: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:43.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:15:42 smithi153 ceph-mon[70044]: pgmap v6824: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:43.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:15:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:15:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:15:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:15:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:15:44.369Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:15:45.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:15:44 smithi083 ceph-mon[79930]: pgmap v6825: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:45.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:15:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:15:45.079Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:15:45.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:15:44 smithi083 ceph-mon[77242]: pgmap v6825: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:45.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:15:44 smithi153 ceph-mon[70044]: pgmap v6825: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:47.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:15:46 smithi083 ceph-mon[79930]: pgmap v6826: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:47.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:15:46 smithi083 ceph-mon[77242]: pgmap v6826: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:47.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:15:46 smithi153 ceph-mon[70044]: pgmap v6826: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:48.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:15:48 smithi083 ceph-mon[79930]: pgmap v6827: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:48.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:15:48 smithi083 ceph-mon[77242]: pgmap v6827: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:49.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:15:48 smithi153 ceph-mon[70044]: pgmap v6827: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:51.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:15:50 smithi083 ceph-mon[79930]: pgmap v6828: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:51.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:15:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:15:50.988Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:15:51.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:15:50 smithi083 ceph-mon[77242]: pgmap v6828: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:51.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:15:50 smithi153 ceph-mon[70044]: pgmap v6828: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:51.833 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:15:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:15:51] "GET /metrics HTTP/1.1" 200 225220 "" "Prometheus/2.33.4" 2022-09-29T09:15:52.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:15:51 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:15:52.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:15:51 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:15:52.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:15:51 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:15:52.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:15:51 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:15:52.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:15:51 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:15:52.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:15:51 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:15:52.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:15:51 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:15:52.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:15:51 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:15:52.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:15:51 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:15:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:15:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:15:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:15:53.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:15:52 smithi083 ceph-mon[79930]: pgmap v6829: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:53.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:15:52 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:15:53.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:15:52 smithi083 ceph-mon[77242]: pgmap v6829: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:53.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:15:52 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:15:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:15:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:15:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:15:53.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:15:52 smithi153 ceph-mon[70044]: pgmap v6829: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:53.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:15:52 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:15:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:15:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:15:54.369Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:15:55.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:15:54 smithi083 ceph-mon[79930]: pgmap v6830: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:55.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:15:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:15:55.080Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:15:55.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:15:54 smithi083 ceph-mon[77242]: pgmap v6830: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:55.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:15:54 smithi153 ceph-mon[70044]: pgmap v6830: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:57.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:15:56 smithi083 ceph-mon[79930]: pgmap v6831: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:57.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:15:56 smithi083 ceph-mon[77242]: pgmap v6831: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:57.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:15:56 smithi153 ceph-mon[70044]: pgmap v6831: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:58.866 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:15:58.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:15:58 smithi083 ceph-mon[79930]: pgmap v6832: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:58.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:15:58 smithi083 ceph-mon[77242]: pgmap v6832: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:59.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:15:58 smithi153 ceph-mon[70044]: pgmap v6832: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:15:59.159 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:15:59.159 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 7m ago 3h 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:15:59.159 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 5m ago 3h 63.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:15:59.160 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 7m ago 3h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:15:59.160 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 5m ago 3h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:15:59.160 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 5m ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:15:59.160 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 7m ago 3h 706M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:15:59.160 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 7m ago 3h 851M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:15:59.161 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 5m ago 3h 375M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:15:59.161 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 7m ago 3h 497M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:15:59.161 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 7m ago 3h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:15:59.161 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 5m ago 3h 25.7M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:15:59.162 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 7m ago 3h 615M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:15:59.162 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 7m ago 3h 606M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:15:59.162 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 7m ago 3h 475M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:15:59.162 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 7m ago 3h 589M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:15:59.163 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 5m ago 3h 522M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:15:59.163 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 5m ago 3h 648M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:15:59.163 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 5m ago 3h 577M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:15:59.163 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 5m ago 3h 680M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:15:59.164 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 5m ago 3h 167M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:15:59.514 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:15:59.515 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:15:59.515 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:15:59.515 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:15:59.516 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:15:59.516 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:15:59.516 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:15:59.516 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:15:59.517 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:15:59.517 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:15:59.517 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:15:59.517 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:15:59.518 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:15:59.518 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:15:59.518 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:15:59.518 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:15:59.519 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:15:59.519 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:15:59.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:15:59 smithi083 ceph-mon[79930]: from='client.52012 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:15:59.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:15:59 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1769198459' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:15:59.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:15:59 smithi083 ceph-mon[77242]: from='client.52012 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:15:59.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:15:59 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1769198459' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:16:00.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:15:59 smithi153 ceph-mon[70044]: from='client.52012 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:16:00.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:15:59 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1769198459' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:16:00.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:16:00 smithi083 ceph-mon[79930]: from='client.42087 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:16:00.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:16:00 smithi083 ceph-mon[79930]: pgmap v6833: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:16:00.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:16:00 smithi083 ceph-mon[77242]: from='client.42087 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:16:00.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:16:00 smithi083 ceph-mon[77242]: pgmap v6833: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:16:01.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:16:00 smithi153 ceph-mon[70044]: from='client.42087 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:16:01.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:16:00 smithi153 ceph-mon[70044]: pgmap v6833: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:16:01.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:16:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:16:00.989Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:16:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:16:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:16:01] "GET /metrics HTTP/1.1" 200 225218 "" "Prometheus/2.33.4" 2022-09-29T09:16:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:16:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:16:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:16:03.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:16:02 smithi083 ceph-mon[79930]: pgmap v6834: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:16:03.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:16:02 smithi083 ceph-mon[77242]: pgmap v6834: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:16:03.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:16:02 smithi153 ceph-mon[70044]: pgmap v6834: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:16:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:16:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:16:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:16:04.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:16:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:16:04.370Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:16:05.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:16:04 smithi083 ceph-mon[79930]: pgmap v6835: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:16:05.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:16:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:16:05.081Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:16:05.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:16:04 smithi083 ceph-mon[77242]: pgmap v6835: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:16:05.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:16:04 smithi153 ceph-mon[70044]: pgmap v6835: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:16:07.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:16:06 smithi083 ceph-mon[79930]: pgmap v6836: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:16:07.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:16:06 smithi083 ceph-mon[77242]: pgmap v6836: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:16:07.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:16:06 smithi153 ceph-mon[70044]: pgmap v6836: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:16:08.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:16:08 smithi083 ceph-mon[77242]: pgmap v6837: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:16:08.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:16:08 smithi083 ceph-mon[79930]: pgmap v6837: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:16:09.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:16:08 smithi153 ceph-mon[70044]: pgmap v6837: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:16:11.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:16:10 smithi083 ceph-mon[79930]: pgmap v6838: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:16:11.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:16:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:16:10.989Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:16:11.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:16:10 smithi083 ceph-mon[77242]: pgmap v6838: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:16:11.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:16:10 smithi153 ceph-mon[70044]: pgmap v6838: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:16:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:16:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:16:11] "GET /metrics HTTP/1.1" 200 225217 "" "Prometheus/2.33.4" 2022-09-29T09:16:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:16:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:16:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:16:13.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:16:12 smithi083 ceph-mon[79930]: pgmap v6839: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:16:13.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:16:12 smithi083 ceph-mon[77242]: pgmap v6839: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:16:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:16:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:16:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:16:13.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:16:12 smithi153 ceph-mon[70044]: pgmap v6839: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:16:14.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:16:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:16:14.371Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:16:15.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:16:14 smithi083 ceph-mon[79930]: pgmap v6840: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:16:15.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:16:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:16:15.082Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:16:15.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:16:14 smithi083 ceph-mon[77242]: pgmap v6840: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:16:15.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:16:14 smithi153 ceph-mon[70044]: pgmap v6840: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:16:17.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:16:16 smithi083 ceph-mon[79930]: pgmap v6841: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:16:17.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:16:16 smithi083 ceph-mon[77242]: pgmap v6841: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:16:17.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:16:16 smithi153 ceph-mon[70044]: pgmap v6841: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:16:18.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:16:18 smithi083 ceph-mon[79930]: pgmap v6842: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:16:18.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:16:18 smithi083 ceph-mon[77242]: pgmap v6842: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:16:19.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:16:18 smithi153 ceph-mon[70044]: pgmap v6842: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:16:21.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:16:20 smithi083 ceph-mon[79930]: pgmap v6843: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:16:21.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:16:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:16:20.990Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:16:21.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:16:20 smithi083 ceph-mon[77242]: pgmap v6843: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:16:21.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:16:20 smithi153 ceph-mon[70044]: pgmap v6843: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:16:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:16:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:16:21] "GET /metrics HTTP/1.1" 200 225217 "" "Prometheus/2.33.4" 2022-09-29T09:16:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:16:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:16:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:16:23.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:16:22 smithi083 ceph-mon[77242]: pgmap v6844: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:16:23.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:16:22 smithi083 ceph-mon[79930]: pgmap v6844: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:16:23.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:16:22 smithi153 ceph-mon[70044]: pgmap v6844: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:16:23.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:16:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:16:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:16:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:16:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:16:24.372Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:16:25.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:16:24 smithi083 ceph-mon[79930]: pgmap v6845: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:16:25.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:16:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:16:25.082Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:16:25.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:16:24 smithi083 ceph-mon[77242]: pgmap v6845: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:16:25.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:16:24 smithi153 ceph-mon[70044]: pgmap v6845: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:16:27.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:16:26 smithi083 ceph-mon[79930]: pgmap v6846: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:16:27.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:16:26 smithi083 ceph-mon[77242]: pgmap v6846: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:16:27.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:16:26 smithi153 ceph-mon[70044]: pgmap v6846: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:16:28.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:16:28 smithi083 ceph-mon[79930]: pgmap v6847: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:16:28.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:16:28 smithi083 ceph-mon[77242]: pgmap v6847: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:16:29.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:16:28 smithi153 ceph-mon[70044]: pgmap v6847: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:16:29.843 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:16:30.135 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:16:30.135 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 7m ago 3h 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:16:30.135 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 5m ago 3h 63.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:16:30.136 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 7m ago 3h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:16:30.136 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 5m ago 3h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:16:30.136 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 5m ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:16:30.136 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 7m ago 3h 706M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:16:30.137 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 7m ago 3h 851M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:16:30.137 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 5m ago 3h 375M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:16:30.137 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 7m ago 3h 497M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:16:30.137 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 7m ago 3h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:16:30.137 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 5m ago 3h 25.7M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:16:30.138 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 7m ago 3h 615M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:16:30.138 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 7m ago 3h 606M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:16:30.138 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 7m ago 3h 475M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:16:30.138 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 7m ago 3h 589M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:16:30.139 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 5m ago 3h 522M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:16:30.139 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 5m ago 3h 648M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:16:30.139 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 5m ago 3h 577M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:16:30.139 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 5m ago 3h 680M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:16:30.139 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 5m ago 3h 167M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:16:30.491 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:16:30.491 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:16:30.491 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:16:30.492 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:16:30.492 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:16:30.492 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:16:30.492 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:16:30.492 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:16:30.493 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:16:30.493 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:16:30.493 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:16:30.493 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:16:30.494 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:16:30.494 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:16:30.494 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:16:30.494 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:16:30.495 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:16:30.495 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:16:31.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:16:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:16:30.990Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:16:31.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:16:30 smithi083 ceph-mon[77242]: pgmap v6848: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:16:31.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:16:30 smithi083 ceph-mon[77242]: from='client.42099 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:16:31.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:16:30 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/63877426' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:16:31.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:16:30 smithi083 ceph-mon[79930]: pgmap v6848: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:16:31.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:16:30 smithi083 ceph-mon[79930]: from='client.42099 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:16:31.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:16:30 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/63877426' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:16:31.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:16:30 smithi153 ceph-mon[70044]: pgmap v6848: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:16:31.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:16:30 smithi153 ceph-mon[70044]: from='client.42099 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:16:31.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:16:30 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/63877426' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:16:31.851 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:16:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:16:31] "GET /metrics HTTP/1.1" 200 225224 "" "Prometheus/2.33.4" 2022-09-29T09:16:32.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:16:31 smithi083 ceph-mon[77242]: from='client.42105 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:16:32.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:16:31 smithi083 ceph-mon[79930]: from='client.42105 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:16:32.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:16:31 smithi153 ceph-mon[70044]: from='client.42105 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:16:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:16:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:16:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:16:33.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:16:32 smithi083 ceph-mon[79930]: pgmap v6849: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:16:33.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:16:32 smithi083 ceph-mon[77242]: pgmap v6849: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:16:33.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:16:32 smithi153 ceph-mon[70044]: pgmap v6849: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:16:33.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:16:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:16:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:16:34.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:16:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:16:34.372Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:16:35.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:16:34 smithi083 ceph-mon[79930]: pgmap v6850: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:16:35.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:16:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:16:35.083Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:16:35.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:16:34 smithi083 ceph-mon[77242]: pgmap v6850: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:16:35.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:16:34 smithi153 ceph-mon[70044]: pgmap v6850: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:16:37.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:16:36 smithi083 ceph-mon[79930]: pgmap v6851: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:16:37.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:16:36 smithi083 ceph-mon[77242]: pgmap v6851: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:16:37.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:16:36 smithi153 ceph-mon[70044]: pgmap v6851: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:16:38.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:16:38 smithi083 ceph-mon[79930]: pgmap v6852: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:16:38.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:16:38 smithi083 ceph-mon[77242]: pgmap v6852: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:16:39.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:16:38 smithi153 ceph-mon[70044]: pgmap v6852: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:16:41.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:16:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:16:40.991Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:16:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:16:40 smithi083 ceph-mon[77242]: pgmap v6853: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:16:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:16:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:16:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:16:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:16:41.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:16:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:16:41.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:16:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:16:41.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:16:40 smithi083 ceph-mon[79930]: pgmap v6853: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:16:41.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:16:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:16:41.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:16:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:16:41.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:16:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:16:41.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:16:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:16:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:16:40 smithi153 ceph-mon[70044]: pgmap v6853: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:16:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:16:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:16:41.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:16:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:16:41.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:16:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:16:41.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:16:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:16:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:16:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:16:41] "GET /metrics HTTP/1.1" 200 225220 "" "Prometheus/2.33.4" 2022-09-29T09:16:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:16:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:16:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:16:43.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:16:42 smithi083 ceph-mon[79930]: pgmap v6854: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:16:43.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:16:42 smithi083 ceph-mon[77242]: pgmap v6854: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:16:43.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:16:42 smithi153 ceph-mon[70044]: pgmap v6854: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:16:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:16:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:16:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:16:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:16:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:16:44.372Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:16:45.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:16:44 smithi083 ceph-mon[79930]: pgmap v6855: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:16:45.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:16:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:16:45.082Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:16:45.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:16:44 smithi083 ceph-mon[77242]: pgmap v6855: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:16:45.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:16:44 smithi153 ceph-mon[70044]: pgmap v6855: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:16:47.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:16:46 smithi083 ceph-mon[79930]: pgmap v6856: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:16:47.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:16:46 smithi083 ceph-mon[77242]: pgmap v6856: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:16:47.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:16:46 smithi153 ceph-mon[70044]: pgmap v6856: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:16:48.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:16:48 smithi083 ceph-mon[79930]: pgmap v6857: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:16:48.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:16:48 smithi083 ceph-mon[77242]: pgmap v6857: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:16:49.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:16:48 smithi153 ceph-mon[70044]: pgmap v6857: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:16:51.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:16:50 smithi083 ceph-mon[79930]: pgmap v6858: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:16:51.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:16:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:16:50.993Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:16:51.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:16:50 smithi083 ceph-mon[77242]: pgmap v6858: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:16:51.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:16:50 smithi153 ceph-mon[70044]: pgmap v6858: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:16:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:16:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:16:51] "GET /metrics HTTP/1.1" 200 225220 "" "Prometheus/2.33.4" 2022-09-29T09:16:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:16:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:16:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:16:53.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:16:52 smithi083 ceph-mon[77242]: pgmap v6859: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:16:53.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:16:52 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:16:53.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:16:52 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:16:53.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:16:52 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:16:53.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:16:52 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:16:53.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:16:52 smithi083 ceph-mon[79930]: pgmap v6859: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:16:53.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:16:52 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:16:53.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:16:52 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:16:53.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:16:52 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:16:53.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:16:52 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:16:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:16:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:16:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:16:53.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:16:52 smithi153 ceph-mon[70044]: pgmap v6859: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:16:53.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:16:52 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:16:53.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:16:52 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:16:53.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:16:52 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:16:53.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:16:52 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:16:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:16:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:16:54.373Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:16:55.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:16:54 smithi083 ceph-mon[77242]: pgmap v6860: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:16:55.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:16:54 smithi083 ceph-mon[79930]: pgmap v6860: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:16:55.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:16:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:16:55.083Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:16:55.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:16:54 smithi153 ceph-mon[70044]: pgmap v6860: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:16:57.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:16:56 smithi083 ceph-mon[79930]: pgmap v6861: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:16:57.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:16:56 smithi083 ceph-mon[77242]: pgmap v6861: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:16:57.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:16:56 smithi153 ceph-mon[70044]: pgmap v6861: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:16:58.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:16:58 smithi083 ceph-mon[79930]: pgmap v6862: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:16:58.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:16:58 smithi083 ceph-mon[77242]: pgmap v6862: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:16:59.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:16:58 smithi153 ceph-mon[70044]: pgmap v6862: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:00.818 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:17:01.110 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:17:01.110 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 8m ago 3h 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:17:01.110 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 6m ago 3h 63.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:17:01.111 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 8m ago 3h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:17:01.111 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 6m ago 3h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:17:01.111 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 6m ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:17:01.111 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 8m ago 3h 706M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:17:01.112 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 8m ago 3h 851M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:17:01.112 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 6m ago 3h 375M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:17:01.112 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 8m ago 3h 497M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:17:01.112 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 8m ago 3h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:17:01.112 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 6m ago 3h 25.7M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:17:01.113 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 8m ago 3h 615M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:17:01.113 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 8m ago 3h 606M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:17:01.113 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 8m ago 3h 475M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:17:01.113 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 8m ago 3h 589M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:17:01.114 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 6m ago 3h 522M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:17:01.114 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 6m ago 3h 648M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:17:01.114 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 6m ago 3h 577M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:17:01.114 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 6m ago 3h 680M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:17:01.115 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 6m ago 3h 167M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:17:01.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:17:00 smithi083 ceph-mon[79930]: pgmap v6863: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:01.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:17:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:17:00.992Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:17:01.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:17:00 smithi083 ceph-mon[77242]: pgmap v6863: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:01.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:17:00 smithi153 ceph-mon[70044]: pgmap v6863: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:01.465 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:17:01.465 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:17:01.465 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:17:01.466 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:17:01.466 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:17:01.466 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:17:01.466 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:17:01.467 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:17:01.467 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:17:01.467 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:17:01.468 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:17:01.468 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:17:01.468 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:17:01.468 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:17:01.468 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:17:01.469 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:17:01.469 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:17:01.469 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:17:01.856 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:17:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:17:01] "GET /metrics HTTP/1.1" 200 225219 "" "Prometheus/2.33.4" 2022-09-29T09:17:01.878 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:17:01 smithi083 ceph-mon[77242]: from='client.52048 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:17:01.879 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:17:01 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/631835435' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:17:02.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:17:01 smithi083 ceph-mon[79930]: from='client.52048 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:17:02.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:17:01 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/631835435' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:17:02.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:17:01 smithi153 ceph-mon[70044]: from='client.52048 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:17:02.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:17:01 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/631835435' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:17:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:17:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:17:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:17:03.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:17:02 smithi083 ceph-mon[79930]: from='client.42123 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:17:03.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:17:02 smithi083 ceph-mon[79930]: pgmap v6864: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:03.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:17:02 smithi083 ceph-mon[77242]: from='client.42123 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:17:03.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:17:02 smithi083 ceph-mon[77242]: pgmap v6864: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:03.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:17:02 smithi153 ceph-mon[70044]: from='client.42123 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:17:03.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:17:02 smithi153 ceph-mon[70044]: pgmap v6864: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:17:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:17:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:17:04.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:17:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:17:04.373Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:17:05.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:17:04 smithi083 ceph-mon[79930]: pgmap v6865: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:05.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:17:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:17:05.084Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:17:05.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:17:04 smithi083 ceph-mon[77242]: pgmap v6865: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:05.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:17:04 smithi153 ceph-mon[70044]: pgmap v6865: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:07.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:17:06 smithi083 ceph-mon[79930]: pgmap v6866: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:07.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:17:06 smithi083 ceph-mon[77242]: pgmap v6866: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:07.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:17:06 smithi153 ceph-mon[70044]: pgmap v6866: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:08.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:17:08 smithi083 ceph-mon[77242]: pgmap v6867: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:08.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:17:08 smithi083 ceph-mon[79930]: pgmap v6867: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:09.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:17:08 smithi153 ceph-mon[70044]: pgmap v6867: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:11.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:17:10 smithi083 ceph-mon[79930]: pgmap v6868: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:11.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:17:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:17:10.993Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:17:11.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:17:10 smithi083 ceph-mon[77242]: pgmap v6868: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:11.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:17:10 smithi153 ceph-mon[70044]: pgmap v6868: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:17:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:17:11] "GET /metrics HTTP/1.1" 200 225220 "" "Prometheus/2.33.4" 2022-09-29T09:17:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:17:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:17:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:17:13.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:17:12 smithi083 ceph-mon[77242]: pgmap v6869: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:13.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:17:12 smithi083 ceph-mon[79930]: pgmap v6869: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:13.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:17:12 smithi153 ceph-mon[70044]: pgmap v6869: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:17:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:17:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:17:14.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:17:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:17:14.374Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:17:15.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:17:14 smithi083 ceph-mon[79930]: pgmap v6870: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:15.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:17:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:17:15.084Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:17:15.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:17:14 smithi083 ceph-mon[77242]: pgmap v6870: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:15.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:17:14 smithi153 ceph-mon[70044]: pgmap v6870: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:17.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:17:16 smithi083 ceph-mon[77242]: pgmap v6871: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:17.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:17:16 smithi083 ceph-mon[79930]: pgmap v6871: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:17.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:17:16 smithi153 ceph-mon[70044]: pgmap v6871: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:18.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:17:18 smithi083 ceph-mon[79930]: pgmap v6872: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:18.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:17:18 smithi083 ceph-mon[77242]: pgmap v6872: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:19.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:17:18 smithi153 ceph-mon[70044]: pgmap v6872: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:21.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:17:20 smithi083 ceph-mon[79930]: pgmap v6873: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:21.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:17:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:17:20.994Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:17:21.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:17:20 smithi083 ceph-mon[77242]: pgmap v6873: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:21.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:17:20 smithi153 ceph-mon[70044]: pgmap v6873: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:17:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:17:21] "GET /metrics HTTP/1.1" 200 225220 "" "Prometheus/2.33.4" 2022-09-29T09:17:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:17:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:17:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:17:23.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:17:22 smithi083 ceph-mon[79930]: pgmap v6874: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:23.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:17:22 smithi083 ceph-mon[77242]: pgmap v6874: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:23.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:17:22 smithi153 ceph-mon[70044]: pgmap v6874: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:17:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:17:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:17:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:17:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:17:24.374Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:17:25.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:17:24 smithi083 ceph-mon[79930]: pgmap v6875: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:25.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:17:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:17:25.085Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:17:25.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:17:24 smithi083 ceph-mon[77242]: pgmap v6875: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:25.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:17:24 smithi153 ceph-mon[70044]: pgmap v6875: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:27.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:17:26 smithi083 ceph-mon[79930]: pgmap v6876: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:27.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:17:26 smithi083 ceph-mon[77242]: pgmap v6876: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:27.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:17:26 smithi153 ceph-mon[70044]: pgmap v6876: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:28.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:17:28 smithi083 ceph-mon[79930]: pgmap v6877: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:28.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:17:28 smithi083 ceph-mon[77242]: pgmap v6877: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:29.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:17:28 smithi153 ceph-mon[70044]: pgmap v6877: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:31.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:17:30 smithi083 ceph-mon[79930]: pgmap v6878: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:31.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:17:30 smithi083 ceph-mon[77242]: pgmap v6878: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:31.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:17:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:17:30.995Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:17:31.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:17:30 smithi153 ceph-mon[70044]: pgmap v6878: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:31.791 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:17:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:17:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:17:31] "GET /metrics HTTP/1.1" 200 225221 "" "Prometheus/2.33.4" 2022-09-29T09:17:32.081 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:17:32.082 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 8m ago 3h 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:17:32.082 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 6m ago 3h 63.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:17:32.082 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 8m ago 3h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:17:32.083 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 6m ago 3h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:17:32.083 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 6m ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:17:32.083 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 8m ago 3h 706M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:17:32.083 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 8m ago 3h 851M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:17:32.084 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 6m ago 3h 375M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:17:32.084 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 8m ago 3h 497M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:17:32.084 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 8m ago 3h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:17:32.084 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 6m ago 3h 25.7M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:17:32.084 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 8m ago 3h 615M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:17:32.085 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 8m ago 3h 606M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:17:32.085 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 8m ago 3h 475M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:17:32.085 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 8m ago 3h 589M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:17:32.085 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 6m ago 3h 522M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:17:32.086 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 6m ago 3h 648M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:17:32.086 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 6m ago 3h 577M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:17:32.086 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 6m ago 3h 680M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:17:32.086 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 6m ago 3h 167M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:17:32.435 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:17:32.436 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:17:32.436 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:17:32.436 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:17:32.436 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:17:32.437 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:17:32.437 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:17:32.437 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:17:32.437 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:17:32.438 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:17:32.438 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:17:32.438 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:17:32.438 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:17:32.439 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:17:32.439 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:17:32.439 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:17:32.439 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:17:32.440 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:17:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:17:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:17:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:17:33.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:17:32 smithi083 ceph-mon[77242]: from='client.52066 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:17:33.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:17:32 smithi083 ceph-mon[77242]: pgmap v6879: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:33.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:17:32 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/15670718' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:17:33.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:17:32 smithi083 ceph-mon[79930]: from='client.52066 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:17:33.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:17:32 smithi083 ceph-mon[79930]: pgmap v6879: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:33.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:17:32 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/15670718' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:17:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:17:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:17:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:17:33.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:17:32 smithi153 ceph-mon[70044]: from='client.52066 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:17:33.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:17:32 smithi153 ceph-mon[70044]: pgmap v6879: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:33.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:17:32 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/15670718' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:17:34.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:17:33 smithi083 ceph-mon[79930]: from='client.52072 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:17:34.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:17:33 smithi083 ceph-mon[77242]: from='client.52072 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:17:34.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:17:33 smithi153 ceph-mon[70044]: from='client.52072 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:17:34.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:17:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:17:34.375Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:17:35.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:17:34 smithi083 ceph-mon[79930]: pgmap v6880: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:35.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:17:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:17:35.085Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:17:35.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:17:34 smithi083 ceph-mon[77242]: pgmap v6880: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:35.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:17:34 smithi153 ceph-mon[70044]: pgmap v6880: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:37.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:17:36 smithi083 ceph-mon[79930]: pgmap v6881: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:37.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:17:36 smithi083 ceph-mon[77242]: pgmap v6881: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:37.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:17:36 smithi153 ceph-mon[70044]: pgmap v6881: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:38.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:17:38 smithi083 ceph-mon[77242]: pgmap v6882: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:38.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:17:38 smithi083 ceph-mon[79930]: pgmap v6882: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:39.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:17:38 smithi153 ceph-mon[70044]: pgmap v6882: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:41.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:17:40 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:17:40.996Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:17:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:17:40 smithi083 ceph-mon[77242]: pgmap v6883: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:17:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:17:41.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:17:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:17:41.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:17:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:17:41.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:17:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:17:41.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:17:40 smithi083 ceph-mon[79930]: pgmap v6883: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:41.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:17:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:17:41.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:17:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:17:41.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:17:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:17:41.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:17:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:17:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:17:40 smithi153 ceph-mon[70044]: pgmap v6883: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:17:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:17:41.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:17:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:17:41.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:17:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:17:41.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:17:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:17:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:17:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:17:41] "GET /metrics HTTP/1.1" 200 225211 "" "Prometheus/2.33.4" 2022-09-29T09:17:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:17:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:17:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:17:43.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:17:42 smithi083 ceph-mon[79930]: pgmap v6884: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:43.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:17:42 smithi083 ceph-mon[77242]: pgmap v6884: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:43.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:17:42 smithi153 ceph-mon[70044]: pgmap v6884: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:43.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:17:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:17:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:17:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:17:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:17:44.376Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:17:45.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:17:44 smithi083 ceph-mon[79930]: pgmap v6885: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:45.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:17:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:17:45.086Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:17:45.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:17:44 smithi083 ceph-mon[77242]: pgmap v6885: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:45.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:17:44 smithi153 ceph-mon[70044]: pgmap v6885: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:47.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:17:46 smithi083 ceph-mon[77242]: pgmap v6886: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:47.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:17:46 smithi083 ceph-mon[79930]: pgmap v6886: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:47.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:17:46 smithi153 ceph-mon[70044]: pgmap v6886: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:48.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:17:48 smithi083 ceph-mon[79930]: pgmap v6887: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:48.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:17:48 smithi083 ceph-mon[77242]: pgmap v6887: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:49.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:17:48 smithi153 ceph-mon[70044]: pgmap v6887: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:51.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:17:50 smithi083 ceph-mon[79930]: pgmap v6888: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:51.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:17:50 smithi083 ceph-mon[77242]: pgmap v6888: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:51.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:17:50 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:17:50.996Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:17:51.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:17:50 smithi153 ceph-mon[70044]: pgmap v6888: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:17:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:17:51] "GET /metrics HTTP/1.1" 200 225211 "" "Prometheus/2.33.4" 2022-09-29T09:17:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:17:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:17:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:17:53.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:17:52 smithi083 ceph-mon[77242]: pgmap v6889: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:53.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:17:52 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:17:53.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:17:52 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:17:53.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:17:52 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:17:53.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:17:52 smithi083 ceph-mon[79930]: pgmap v6889: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:53.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:17:52 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:17:53.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:17:52 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:17:53.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:17:52 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:17:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:17:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:17:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:17:53.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:17:52 smithi153 ceph-mon[70044]: pgmap v6889: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:53.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:17:52 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:17:53.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:17:52 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:17:53.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:17:52 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:17:54.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:17:54 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:17:54.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:17:54 smithi153 ceph-mon[70044]: pgmap v6890: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:54.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:17:54 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:17:54.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:17:54 smithi083 ceph-mon[79930]: pgmap v6890: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:54.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:17:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:17:54.375Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:17:54.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:17:54 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:17:54.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:17:54 smithi083 ceph-mon[77242]: pgmap v6890: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:17:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:17:55.086Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:17:57.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:17:56 smithi083 ceph-mon[79930]: pgmap v6891: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:57.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:17:56 smithi083 ceph-mon[77242]: pgmap v6891: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:57.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:17:56 smithi153 ceph-mon[70044]: pgmap v6891: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:58.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:17:58 smithi083 ceph-mon[79930]: pgmap v6892: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:58.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:17:58 smithi083 ceph-mon[77242]: pgmap v6892: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:17:59.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:17:58 smithi153 ceph-mon[70044]: pgmap v6892: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:01.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:18:00 smithi083 ceph-mon[79930]: pgmap v6893: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:01.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:18:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:18:00.996Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:18:01.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:18:00 smithi083 ceph-mon[77242]: pgmap v6893: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:01.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:18:00 smithi153 ceph-mon[70044]: pgmap v6893: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:18:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:18:01] "GET /metrics HTTP/1.1" 200 225218 "" "Prometheus/2.33.4" 2022-09-29T09:18:02.762 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:18:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:18:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:18:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:18:03.053 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:18:03.053 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 9m ago 3h 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:18:03.054 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 7m ago 3h 63.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:18:03.054 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 9m ago 3h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:18:03.054 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 7m ago 3h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:18:03.055 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 7m ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:18:03.055 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 9m ago 3h 706M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:18:03.055 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 9m ago 3h 851M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:18:03.055 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 7m ago 3h 375M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:18:03.056 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 9m ago 3h 497M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:18:03.056 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 9m ago 3h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:18:03.056 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 7m ago 3h 25.7M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:18:03.056 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 9m ago 3h 615M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:18:03.057 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 9m ago 3h 606M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:18:03.057 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 9m ago 3h 475M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:18:03.057 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 9m ago 3h 589M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:18:03.057 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 7m ago 3h 522M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:18:03.058 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 7m ago 3h 648M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:18:03.058 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 7m ago 3h 577M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:18:03.058 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 7m ago 3h 680M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:18:03.058 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 7m ago 3h 167M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:18:03.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:18:02 smithi083 ceph-mon[77242]: pgmap v6894: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:03.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:18:02 smithi083 ceph-mon[79930]: pgmap v6894: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:03.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:18:02 smithi153 ceph-mon[70044]: pgmap v6894: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:18:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:18:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:18:03.419 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:18:03.420 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:18:03.420 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:18:03.420 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:18:03.420 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:18:03.421 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:18:03.421 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:18:03.421 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:18:03.421 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:18:03.422 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:18:03.422 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:18:03.422 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:18:03.422 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:18:03.423 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:18:03.423 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:18:03.423 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:18:03.423 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:18:03.424 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:18:04.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:18:03 smithi083 ceph-mon[79930]: from='client.42153 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:18:04.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:18:03 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3338952880' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:18:04.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:18:03 smithi083 ceph-mon[77242]: from='client.42153 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:18:04.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:18:03 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3338952880' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:18:04.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:18:03 smithi153 ceph-mon[70044]: from='client.42153 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:18:04.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:18:03 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3338952880' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:18:04.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:18:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:18:04.376Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:18:05.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:18:04 smithi083 ceph-mon[79930]: from='client.42159 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:18:05.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:18:04 smithi083 ceph-mon[79930]: pgmap v6895: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:05.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:18:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:18:05.088Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:18:05.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:18:04 smithi083 ceph-mon[77242]: from='client.42159 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:18:05.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:18:04 smithi083 ceph-mon[77242]: pgmap v6895: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:05.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:18:04 smithi153 ceph-mon[70044]: from='client.42159 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:18:05.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:18:04 smithi153 ceph-mon[70044]: pgmap v6895: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:07.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:18:06 smithi083 ceph-mon[79930]: pgmap v6896: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:07.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:18:06 smithi083 ceph-mon[77242]: pgmap v6896: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:07.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:18:06 smithi153 ceph-mon[70044]: pgmap v6896: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:08.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:18:08 smithi083 ceph-mon[79930]: pgmap v6897: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:08.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:18:08 smithi083 ceph-mon[77242]: pgmap v6897: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:09.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:18:08 smithi153 ceph-mon[70044]: pgmap v6897: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:11.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:18:10 smithi083 ceph-mon[79930]: pgmap v6898: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:11.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:18:10 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:18:10.997Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:18:11.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:18:10 smithi083 ceph-mon[77242]: pgmap v6898: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:11.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:18:10 smithi153 ceph-mon[70044]: pgmap v6898: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:18:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:18:11] "GET /metrics HTTP/1.1" 200 225219 "" "Prometheus/2.33.4" 2022-09-29T09:18:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:18:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:18:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:18:13.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:18:12 smithi083 ceph-mon[77242]: pgmap v6899: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:13.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:18:12 smithi083 ceph-mon[79930]: pgmap v6899: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:13.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:18:12 smithi153 ceph-mon[70044]: pgmap v6899: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:18:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:18:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:18:14.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:18:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:18:14.376Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:18:15.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:18:14 smithi083 ceph-mon[77242]: pgmap v6900: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:15.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:18:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:18:15.088Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:18:15.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:18:14 smithi083 ceph-mon[79930]: pgmap v6900: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:15.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:18:14 smithi153 ceph-mon[70044]: pgmap v6900: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:17.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:18:16 smithi083 ceph-mon[79930]: pgmap v6901: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:17.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:18:16 smithi083 ceph-mon[77242]: pgmap v6901: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:17.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:18:16 smithi153 ceph-mon[70044]: pgmap v6901: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:18.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:18:18 smithi083 ceph-mon[77242]: pgmap v6902: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:18.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:18:18 smithi083 ceph-mon[79930]: pgmap v6902: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:19.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:18:18 smithi153 ceph-mon[70044]: pgmap v6902: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:21.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:18:20 smithi083 ceph-mon[79930]: pgmap v6903: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:21.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:18:20 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:18:20.998Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:18:21.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:18:20 smithi083 ceph-mon[77242]: pgmap v6903: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:21.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:18:20 smithi153 ceph-mon[70044]: pgmap v6903: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:18:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:18:21] "GET /metrics HTTP/1.1" 200 225219 "" "Prometheus/2.33.4" 2022-09-29T09:18:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:18:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:18:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:18:23.147 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:18:22 smithi083 ceph-mon[77242]: pgmap v6904: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:23.148 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:18:22 smithi083 ceph-mon[79930]: pgmap v6904: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:23.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:18:22 smithi153 ceph-mon[70044]: pgmap v6904: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:23.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:18:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:18:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:18:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:18:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:18:24.377Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:18:25.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:18:24 smithi083 ceph-mon[77242]: pgmap v6905: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:25.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:18:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:18:25.088Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:18:25.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:18:24 smithi083 ceph-mon[79930]: pgmap v6905: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:25.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:18:24 smithi153 ceph-mon[70044]: pgmap v6905: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:27.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:18:26 smithi083 ceph-mon[79930]: pgmap v6906: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:27.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:18:26 smithi083 ceph-mon[77242]: pgmap v6906: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:27.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:18:26 smithi153 ceph-mon[70044]: pgmap v6906: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:28.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:18:28 smithi083 ceph-mon[79930]: pgmap v6907: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:28.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:18:28 smithi083 ceph-mon[77242]: pgmap v6907: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:29.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:18:28 smithi153 ceph-mon[70044]: pgmap v6907: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:31.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:18:30 smithi083 ceph-mon[79930]: pgmap v6908: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:31.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:18:30 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:18:30.999Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:18:31.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:18:30 smithi083 ceph-mon[77242]: pgmap v6908: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:31.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:18:30 smithi153 ceph-mon[70044]: pgmap v6908: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:18:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:18:31] "GET /metrics HTTP/1.1" 200 225215 "" "Prometheus/2.33.4" 2022-09-29T09:18:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:18:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:18:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:18:33.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:18:32 smithi083 ceph-mon[77242]: pgmap v6909: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:33.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:18:32 smithi083 ceph-mon[79930]: pgmap v6909: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:33.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:18:32 smithi153 ceph-mon[70044]: pgmap v6909: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:18:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:18:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:18:33.743 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:18:34.029 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:18:34.029 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 9m ago 3h 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:18:34.029 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 7m ago 3h 63.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:18:34.030 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 9m ago 3h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:18:34.030 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 7m ago 3h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:18:34.030 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 7m ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:18:34.030 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 9m ago 3h 706M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:18:34.031 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 9m ago 3h 851M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:18:34.031 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 7m ago 3h 375M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:18:34.031 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 9m ago 3h 497M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:18:34.031 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 9m ago 3h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:18:34.032 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 7m ago 3h 25.7M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:18:34.032 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 9m ago 3h 615M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:18:34.032 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 9m ago 3h 606M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:18:34.032 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 9m ago 3h 475M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:18:34.032 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 9m ago 3h 589M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:18:34.033 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 7m ago 3h 522M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:18:34.033 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 7m ago 3h 648M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:18:34.033 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 7m ago 3h 577M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:18:34.033 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 7m ago 3h 680M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:18:34.033 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 7m ago 3h 167M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:18:34.391 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:18:34.391 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:18:34.392 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:18:34.392 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:18:34.392 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:18:34.392 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:18:34.392 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:18:34.393 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:18:34.393 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:18:34.393 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:18:34.393 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:18:34.394 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:18:34.394 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:18:34.394 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:18:34.394 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:18:34.395 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:18:34.395 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:18:34.395 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:18:34.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:18:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:18:34.379Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:18:35.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:18:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:18:35.088Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:18:35.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:18:34 smithi083 ceph-mon[77242]: from='client.42171 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:18:35.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:18:34 smithi083 ceph-mon[77242]: pgmap v6910: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:35.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:18:34 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1620066741' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:18:35.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:18:34 smithi083 ceph-mon[79930]: from='client.42171 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:18:35.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:18:34 smithi083 ceph-mon[79930]: pgmap v6910: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:35.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:18:34 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1620066741' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:18:35.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:18:34 smithi153 ceph-mon[70044]: from='client.42171 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:18:35.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:18:34 smithi153 ceph-mon[70044]: pgmap v6910: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:35.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:18:34 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1620066741' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:18:36.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:18:35 smithi083 ceph-mon[79930]: from='client.42177 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:18:36.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:18:35 smithi083 ceph-mon[77242]: from='client.42177 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:18:36.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:18:35 smithi153 ceph-mon[70044]: from='client.42177 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:18:37.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:18:36 smithi083 ceph-mon[79930]: pgmap v6911: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:37.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:18:36 smithi083 ceph-mon[77242]: pgmap v6911: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:37.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:18:36 smithi153 ceph-mon[70044]: pgmap v6911: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:38.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:18:38 smithi083 ceph-mon[79930]: pgmap v6912: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:38.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:18:38 smithi083 ceph-mon[77242]: pgmap v6912: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:39.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:18:38 smithi153 ceph-mon[70044]: pgmap v6912: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:18:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:18:40.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:18:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:18:40.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:18:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:18:40.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:18:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:18:40.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:18:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:18:40.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:18:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:18:40.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:18:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:18:40.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:18:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:18:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:18:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:18:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:18:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:18:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:18:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:18:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:18:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:18:41.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:18:40 smithi083 ceph-mon[79930]: pgmap v6913: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:41.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:18:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:18:40.999Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:18:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:18:40 smithi083 ceph-mon[77242]: pgmap v6913: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:18:40 smithi153 ceph-mon[70044]: pgmap v6913: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:18:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:18:41] "GET /metrics HTTP/1.1" 200 225218 "" "Prometheus/2.33.4" 2022-09-29T09:18:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:18:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:18:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:18:43.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:18:42 smithi083 ceph-mon[79930]: pgmap v6914: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:43.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:18:42 smithi083 ceph-mon[77242]: pgmap v6914: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:43.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:18:42 smithi153 ceph-mon[70044]: pgmap v6914: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:43.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:18:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:18:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:18:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:18:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:18:44.379Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:18:45.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:18:44 smithi083 ceph-mon[79930]: pgmap v6915: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:45.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:18:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:18:45.089Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:18:45.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:18:44 smithi083 ceph-mon[77242]: pgmap v6915: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:45.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:18:44 smithi153 ceph-mon[70044]: pgmap v6915: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:47.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:18:46 smithi083 ceph-mon[79930]: pgmap v6916: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:47.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:18:46 smithi083 ceph-mon[77242]: pgmap v6916: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:47.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:18:46 smithi153 ceph-mon[70044]: pgmap v6916: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:48.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:18:48 smithi083 ceph-mon[79930]: pgmap v6917: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:48.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:18:48 smithi083 ceph-mon[77242]: pgmap v6917: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:49.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:18:48 smithi153 ceph-mon[70044]: pgmap v6917: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:51.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:18:50 smithi083 ceph-mon[79930]: pgmap v6918: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:51.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:18:50 smithi083 ceph-mon[77242]: pgmap v6918: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:51.214 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:18:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:18:51.000Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:18:51.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:18:50 smithi153 ceph-mon[70044]: pgmap v6918: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:18:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:18:51] "GET /metrics HTTP/1.1" 200 225218 "" "Prometheus/2.33.4" 2022-09-29T09:18:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:18:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:18:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:18:53.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:18:52 smithi083 ceph-mon[79930]: pgmap v6919: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:53.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:18:52 smithi083 ceph-mon[77242]: pgmap v6919: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:53.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:18:52 smithi153 ceph-mon[70044]: pgmap v6919: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:53.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:18:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:18:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:18:54.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:18:53 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:18:54.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:18:53 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:18:54.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:18:53 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:18:54.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:18:53 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:18:54.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:18:53 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:18:54.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:18:53 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:18:54.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:18:53 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:18:54.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:18:53 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:18:54.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:18:53 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:18:54.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:18:53 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:18:54.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:18:53 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:18:54.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:18:53 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:18:54.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:18:53 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:18:54.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:18:53 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:18:54.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:18:53 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:18:54.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:18:53 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:18:54.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:18:53 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:18:54.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:18:53 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:18:54.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:18:53 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:18:54.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:18:53 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:18:54.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:18:53 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:18:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:18:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:18:54.379Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:18:55.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:18:54 smithi083 ceph-mon[79930]: pgmap v6920: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:55.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:18:54 smithi083 ceph-mon[77242]: pgmap v6920: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:55.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:18:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:18:55.090Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:18:55.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:18:54 smithi153 ceph-mon[70044]: pgmap v6920: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:57.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:18:56 smithi083 ceph-mon[79930]: pgmap v6921: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:57.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:18:56 smithi083 ceph-mon[77242]: pgmap v6921: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:57.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:18:56 smithi153 ceph-mon[70044]: pgmap v6921: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:59.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:18:58 smithi083 ceph-mon[79930]: pgmap v6922: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:59.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:18:58 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:18:59.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:18:58 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:18:59.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:18:58 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:18:59.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:18:58 smithi083 ceph-mon[77242]: pgmap v6922: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:59.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:18:58 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:18:59.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:18:58 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:18:59.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:18:58 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:18:59.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:18:58 smithi153 ceph-mon[70044]: pgmap v6922: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:18:59.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:18:58 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:18:59.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:18:58 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:18:59.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:18:58 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:19:01.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:19:00 smithi083 ceph-mon[79930]: pgmap v6923: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:01.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:19:00 smithi083 ceph-mon[77242]: pgmap v6923: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:01.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:19:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:19:01.000Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:19:01.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:19:00 smithi153 ceph-mon[70044]: pgmap v6923: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:19:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:19:01] "GET /metrics HTTP/1.1" 200 225216 "" "Prometheus/2.33.4" 2022-09-29T09:19:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:19:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:19:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:19:03.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:19:02 smithi083 ceph-mon[79930]: pgmap v6924: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:03.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:19:02 smithi083 ceph-mon[77242]: pgmap v6924: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:03.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:19:02 smithi153 ceph-mon[70044]: pgmap v6924: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:19:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:19:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:19:04.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:19:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:19:04.381Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:19:04.748 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:19:05.008 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:19:05.009 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 7s ago 3h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:19:05.009 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 8m ago 3h 63.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:19:05.009 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 7s ago 3h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:19:05.009 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 8m ago 3h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:19:05.009 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 8m ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:19:05.010 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 7s ago 3h 714M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:19:05.010 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 7s ago 3h 888M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:19:05.010 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 8m ago 3h 375M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:19:05.010 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 7s ago 3h 519M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:19:05.011 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 7s ago 3h 26.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:19:05.011 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 8m ago 3h 25.7M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:19:05.011 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 7s ago 3h 641M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:19:05.011 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 7s ago 3h 631M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:19:05.012 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 7s ago 3h 495M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:19:05.012 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 7s ago 3h 615M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:19:05.012 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 8m ago 3h 522M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:19:05.012 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 8m ago 3h 648M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:19:05.012 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 8m ago 3h 577M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:19:05.013 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 8m ago 3h 680M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:19:05.013 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 8m ago 3h 167M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:19:05.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:19:04 smithi083 ceph-mon[77242]: pgmap v6925: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:05.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:19:04 smithi083 ceph-mon[79930]: pgmap v6925: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:05.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:19:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:19:05.090Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:19:05.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:19:04 smithi153 ceph-mon[70044]: pgmap v6925: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:05.366 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:19:05.367 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:19:05.367 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:19:05.367 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:19:05.367 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:19:05.368 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:19:05.368 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:19:05.368 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:19:05.368 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:19:05.369 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:19:05.369 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:19:05.369 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:19:05.369 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:19:05.370 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:19:05.370 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:19:05.370 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:19:05.370 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:19:05.371 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:19:06.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:19:05 smithi083 ceph-mon[79930]: from='client.42189 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:19:06.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:19:05 smithi083 ceph-mon[79930]: from='client.42195 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:19:06.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:19:05 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3821664346' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:19:06.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:19:05 smithi083 ceph-mon[77242]: from='client.42189 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:19:06.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:19:05 smithi083 ceph-mon[77242]: from='client.42195 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:19:06.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:19:05 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3821664346' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:19:06.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:19:05 smithi153 ceph-mon[70044]: from='client.42189 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:19:06.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:19:05 smithi153 ceph-mon[70044]: from='client.42195 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:19:06.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:19:05 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3821664346' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:19:07.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:19:06 smithi083 ceph-mon[77242]: pgmap v6926: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:07.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:19:06 smithi083 ceph-mon[79930]: pgmap v6926: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:07.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:19:06 smithi153 ceph-mon[70044]: pgmap v6926: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:08.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:19:08 smithi083 ceph-mon[79930]: pgmap v6927: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:08.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:19:08 smithi083 ceph-mon[77242]: pgmap v6927: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:09.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:19:08 smithi153 ceph-mon[70044]: pgmap v6927: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:11.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:19:10 smithi083 ceph-mon[79930]: pgmap v6928: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:11.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:19:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:19:11.001Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:19:11.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:19:10 smithi083 ceph-mon[77242]: pgmap v6928: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:11.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:19:10 smithi153 ceph-mon[70044]: pgmap v6928: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:19:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:19:11] "GET /metrics HTTP/1.1" 200 225216 "" "Prometheus/2.33.4" 2022-09-29T09:19:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:19:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:19:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:19:13.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:19:12 smithi083 ceph-mon[79930]: pgmap v6929: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:13.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:19:12 smithi083 ceph-mon[77242]: pgmap v6929: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:13.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:19:12 smithi153 ceph-mon[70044]: pgmap v6929: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:13.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:19:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:19:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:19:14.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:19:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:19:14.381Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:19:15.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:19:14 smithi083 ceph-mon[79930]: pgmap v6930: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:15.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:19:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:19:15.091Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:19:15.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:19:14 smithi083 ceph-mon[77242]: pgmap v6930: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:15.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:19:14 smithi153 ceph-mon[70044]: pgmap v6930: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:17.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:19:16 smithi083 ceph-mon[79930]: pgmap v6931: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:17.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:19:16 smithi083 ceph-mon[77242]: pgmap v6931: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:17.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:19:16 smithi153 ceph-mon[70044]: pgmap v6931: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:18.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:19:18 smithi083 ceph-mon[77242]: pgmap v6932: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:18.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:19:18 smithi083 ceph-mon[79930]: pgmap v6932: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:19.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:19:18 smithi153 ceph-mon[70044]: pgmap v6932: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:21.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:19:20 smithi083 ceph-mon[79930]: pgmap v6933: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:21.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:19:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:19:21.002Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:19:21.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:19:20 smithi083 ceph-mon[77242]: pgmap v6933: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:21.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:19:20 smithi153 ceph-mon[70044]: pgmap v6933: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:19:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:19:21] "GET /metrics HTTP/1.1" 200 225216 "" "Prometheus/2.33.4" 2022-09-29T09:19:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:19:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:19:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:19:23.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:19:22 smithi083 ceph-mon[79930]: pgmap v6934: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:23.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:19:22 smithi083 ceph-mon[77242]: pgmap v6934: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:23.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:19:22 smithi153 ceph-mon[70044]: pgmap v6934: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:23.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:19:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:19:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:19:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:19:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:19:24.382Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:19:25.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:19:24 smithi083 ceph-mon[79930]: pgmap v6935: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:25.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:19:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:19:25.091Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:19:25.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:19:24 smithi083 ceph-mon[77242]: pgmap v6935: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:25.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:19:24 smithi153 ceph-mon[70044]: pgmap v6935: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:27.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:19:26 smithi083 ceph-mon[79930]: pgmap v6936: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:27.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:19:26 smithi083 ceph-mon[77242]: pgmap v6936: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:27.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:19:26 smithi153 ceph-mon[70044]: pgmap v6936: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:28.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:19:28 smithi083 ceph-mon[79930]: pgmap v6937: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:28.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:19:28 smithi083 ceph-mon[77242]: pgmap v6937: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:29.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:19:28 smithi153 ceph-mon[70044]: pgmap v6937: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:31.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:19:30 smithi083 ceph-mon[79930]: pgmap v6938: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:31.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:19:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:19:31.001Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:19:31.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:19:30 smithi083 ceph-mon[77242]: pgmap v6938: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:31.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:19:30 smithi153 ceph-mon[70044]: pgmap v6938: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:19:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:19:31] "GET /metrics HTTP/1.1" 200 225206 "" "Prometheus/2.33.4" 2022-09-29T09:19:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:19:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:19:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:19:33.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:19:32 smithi083 ceph-mon[79930]: pgmap v6939: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:33.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:19:32 smithi083 ceph-mon[77242]: pgmap v6939: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:33.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:19:32 smithi153 ceph-mon[70044]: pgmap v6939: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:19:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:19:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:19:34.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:19:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:19:34.383Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:19:35.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:19:34 smithi083 ceph-mon[77242]: pgmap v6940: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:35.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:19:34 smithi083 ceph-mon[79930]: pgmap v6940: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:35.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:19:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:19:35.092Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:19:35.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:19:34 smithi153 ceph-mon[70044]: pgmap v6940: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:35.691 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:19:35.983 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:19:35.984 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 38s ago 3h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:19:35.984 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 8m ago 3h 63.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:19:35.984 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 38s ago 3h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:19:35.984 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 8m ago 3h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:19:35.985 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 8m ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:19:35.985 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 38s ago 3h 714M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:19:35.985 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 38s ago 3h 888M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:19:35.985 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 8m ago 3h 375M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:19:35.986 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 38s ago 3h 519M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:19:35.986 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 38s ago 3h 26.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:19:35.986 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 8m ago 3h 25.7M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:19:35.986 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 38s ago 3h 641M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:19:35.987 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 38s ago 3h 631M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:19:35.987 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 38s ago 3h 495M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:19:35.987 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 38s ago 3h 615M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:19:35.987 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 8m ago 3h 522M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:19:35.988 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 8m ago 3h 648M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:19:35.988 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 8m ago 3h 577M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:19:35.988 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 8m ago 3h 680M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:19:35.988 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 8m ago 3h 167M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:19:36.338 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:19:36.339 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:19:36.339 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:19:36.339 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:19:36.339 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:19:36.340 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:19:36.340 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:19:36.340 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:19:36.340 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:19:36.341 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:19:36.341 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:19:36.341 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:19:36.341 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:19:36.342 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:19:36.342 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:19:36.342 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:19:36.342 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:19:36.342 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:19:37.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:19:36 smithi083 ceph-mon[79930]: from='client.42207 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:19:37.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:19:36 smithi083 ceph-mon[79930]: pgmap v6941: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:37.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:19:36 smithi083 ceph-mon[79930]: from='client.42213 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:19:37.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:19:36 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/730837007' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:19:37.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:19:36 smithi083 ceph-mon[77242]: from='client.42207 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:19:37.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:19:36 smithi083 ceph-mon[77242]: pgmap v6941: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:37.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:19:36 smithi083 ceph-mon[77242]: from='client.42213 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:19:37.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:19:36 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/730837007' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:19:37.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:19:36 smithi153 ceph-mon[70044]: from='client.42207 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:19:37.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:19:36 smithi153 ceph-mon[70044]: pgmap v6941: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:37.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:19:36 smithi153 ceph-mon[70044]: from='client.42213 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:19:37.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:19:36 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/730837007' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:19:38.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:19:38 smithi083 ceph-mon[79930]: pgmap v6942: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:38.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:19:38 smithi083 ceph-mon[77242]: pgmap v6942: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:39.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:19:38 smithi153 ceph-mon[70044]: pgmap v6942: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:19:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:19:40.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:19:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:19:40.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:19:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:19:40.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:19:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:19:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:19:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:19:40.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:19:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:19:40.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:19:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:19:40.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:19:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:19:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:19:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:19:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:19:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:19:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:19:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:19:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:19:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:19:41.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:19:40 smithi083 ceph-mon[79930]: pgmap v6943: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:19:40 smithi083 ceph-mon[77242]: pgmap v6943: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:41.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:19:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:19:41.003Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:19:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:19:40 smithi153 ceph-mon[70044]: pgmap v6943: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:19:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:19:41] "GET /metrics HTTP/1.1" 200 225213 "" "Prometheus/2.33.4" 2022-09-29T09:19:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:19:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:19:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:19:43.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:19:42 smithi083 ceph-mon[79930]: pgmap v6944: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:43.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:19:42 smithi083 ceph-mon[77242]: pgmap v6944: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:19:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:19:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:19:43.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:19:42 smithi153 ceph-mon[70044]: pgmap v6944: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:19:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:19:44.383Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:19:45.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:19:44 smithi083 ceph-mon[79930]: pgmap v6945: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:45.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:19:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:19:45.093Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:19:45.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:19:44 smithi083 ceph-mon[77242]: pgmap v6945: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:45.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:19:44 smithi153 ceph-mon[70044]: pgmap v6945: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:47.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:19:46 smithi083 ceph-mon[79930]: pgmap v6946: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:47.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:19:46 smithi083 ceph-mon[77242]: pgmap v6946: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:47.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:19:46 smithi153 ceph-mon[70044]: pgmap v6946: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:48.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:19:48 smithi083 ceph-mon[77242]: pgmap v6947: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:48.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:19:48 smithi083 ceph-mon[79930]: pgmap v6947: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:49.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:19:48 smithi153 ceph-mon[70044]: pgmap v6947: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:51.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:19:50 smithi083 ceph-mon[79930]: pgmap v6948: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:51.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:19:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:19:51.003Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:19:51.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:19:50 smithi083 ceph-mon[77242]: pgmap v6948: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:51.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:19:50 smithi153 ceph-mon[70044]: pgmap v6948: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:19:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:19:51] "GET /metrics HTTP/1.1" 200 225213 "" "Prometheus/2.33.4" 2022-09-29T09:19:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:19:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:19:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:19:53.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:19:52 smithi083 ceph-mon[77242]: pgmap v6949: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:53.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:19:52 smithi083 ceph-mon[79930]: pgmap v6949: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:53.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:19:52 smithi153 ceph-mon[70044]: pgmap v6949: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:53.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:19:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:19:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:19:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:19:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:19:54.384Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:19:55.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:19:54 smithi083 ceph-mon[79930]: pgmap v6950: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:55.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:19:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:19:55.094Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:19:55.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:19:54 smithi083 ceph-mon[77242]: pgmap v6950: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:55.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:19:54 smithi153 ceph-mon[70044]: pgmap v6950: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:57.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:19:56 smithi083 ceph-mon[79930]: pgmap v6951: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:57.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:19:56 smithi083 ceph-mon[77242]: pgmap v6951: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:57.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:19:56 smithi153 ceph-mon[70044]: pgmap v6951: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:58.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:19:58 smithi083 ceph-mon[79930]: pgmap v6952: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:58.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:19:58 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:19:58.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:19:58 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:19:58.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:19:58 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:19:58.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:19:58 smithi083 ceph-mon[77242]: pgmap v6952: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:58.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:19:58 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:19:58.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:19:58 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:19:58.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:19:58 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:19:59.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:19:58 smithi153 ceph-mon[70044]: pgmap v6952: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:19:59.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:19:58 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:19:59.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:19:58 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:19:59.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:19:58 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:20:00.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:20:00 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:20:00.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:20:00 smithi083 ceph-mon[79930]: pgmap v6953: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:20:00.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:20:00 smithi083 ceph-mon[79930]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T09:20:00.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:20:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[77219]: 2022-09-29T09:19:59.999+0000 7f160f665700 -1 log_channel(cluster) log [ERR] : overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T09:20:00.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:20:00 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:20:00.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:20:00 smithi083 ceph-mon[77242]: pgmap v6953: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:20:00.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:20:00 smithi083 ceph-mon[77242]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T09:20:00.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:20:00 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:20:00.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:20:00 smithi153 ceph-mon[70044]: pgmap v6953: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:20:00.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:20:00 smithi153 ceph-mon[70044]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T09:20:01.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:20:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:20:01.004Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:20:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:20:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:20:01] "GET /metrics HTTP/1.1" 200 225216 "" "Prometheus/2.33.4" 2022-09-29T09:20:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:20:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:20:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:20:03.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:20:02 smithi083 ceph-mon[79930]: pgmap v6954: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:20:03.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:20:02 smithi083 ceph-mon[77242]: pgmap v6954: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:20:03.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:20:02 smithi153 ceph-mon[70044]: pgmap v6954: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:20:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:20:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:20:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:20:04.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:20:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:20:04.384Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:20:05.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:20:04 smithi083 ceph-mon[79930]: pgmap v6955: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:20:05.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:20:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:20:05.094Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:20:05.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:20:04 smithi083 ceph-mon[77242]: pgmap v6955: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:20:05.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:20:04 smithi153 ceph-mon[70044]: pgmap v6955: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:20:06.662 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:20:06.954 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:20:06.955 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 69s ago 3h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:20:06.955 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 9m ago 3h 63.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:20:06.955 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 69s ago 3h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:20:06.955 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 9m ago 3h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:20:06.956 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 9m ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:20:06.956 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 69s ago 3h 714M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:20:06.956 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 69s ago 3h 888M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:20:06.956 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 9m ago 3h 375M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:20:06.957 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 69s ago 3h 519M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:20:06.957 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 69s ago 3h 26.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:20:06.957 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 9m ago 3h 25.7M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:20:06.957 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 69s ago 3h 641M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:20:06.958 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 69s ago 3h 631M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:20:06.958 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 69s ago 3h 495M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:20:06.958 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 69s ago 3h 615M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:20:06.958 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 9m ago 3h 522M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:20:06.959 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 9m ago 3h 648M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:20:06.959 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 9m ago 3h 577M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:20:06.959 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 9m ago 3h 680M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:20:06.959 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 9m ago 3h 167M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:20:07.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:20:06 smithi083 ceph-mon[79930]: pgmap v6956: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:20:07.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:20:06 smithi083 ceph-mon[77242]: pgmap v6956: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:20:07.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:20:06 smithi153 ceph-mon[70044]: pgmap v6956: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:20:07.310 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:20:07.310 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:20:07.310 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:20:07.311 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:20:07.311 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:20:07.311 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:20:07.311 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:20:07.312 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:20:07.312 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:20:07.312 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:20:07.312 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:20:07.313 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:20:07.313 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:20:07.313 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:20:07.313 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:20:07.313 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:20:07.314 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:20:07.314 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:20:08.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:20:07 smithi083 ceph-mon[79930]: from='client.52156 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:20:08.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:20:07 smithi083 ceph-mon[79930]: from='client.42231 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:20:08.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:20:07 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1624077099' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:20:08.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:20:07 smithi083 ceph-mon[77242]: from='client.52156 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:20:08.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:20:07 smithi083 ceph-mon[77242]: from='client.42231 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:20:08.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:20:07 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1624077099' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:20:08.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:20:07 smithi153 ceph-mon[70044]: from='client.52156 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:20:08.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:20:07 smithi153 ceph-mon[70044]: from='client.42231 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:20:08.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:20:07 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1624077099' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:20:09.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:20:08 smithi083 ceph-mon[79930]: pgmap v6957: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:20:09.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:20:08 smithi083 ceph-mon[77242]: pgmap v6957: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:20:09.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:20:08 smithi153 ceph-mon[70044]: pgmap v6957: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:20:11.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:20:10 smithi083 ceph-mon[79930]: pgmap v6958: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:20:11.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:20:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:20:11.005Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:20:11.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:20:10 smithi083 ceph-mon[77242]: pgmap v6958: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:20:11.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:20:10 smithi153 ceph-mon[70044]: pgmap v6958: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:20:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:20:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:20:11] "GET /metrics HTTP/1.1" 200 225221 "" "Prometheus/2.33.4" 2022-09-29T09:20:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:20:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:20:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:20:13.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:20:12 smithi083 ceph-mon[77242]: pgmap v6959: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:20:13.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:20:12 smithi083 ceph-mon[79930]: pgmap v6959: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:20:13.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:20:12 smithi153 ceph-mon[70044]: pgmap v6959: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:20:13.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:20:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:20:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:20:14.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:20:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:20:14.385Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:20:15.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:20:14 smithi083 ceph-mon[79930]: pgmap v6960: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:20:15.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:20:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:20:15.095Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:20:15.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:20:14 smithi083 ceph-mon[77242]: pgmap v6960: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:20:15.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:20:14 smithi153 ceph-mon[70044]: pgmap v6960: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:20:17.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:20:16 smithi083 ceph-mon[77242]: pgmap v6961: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:20:17.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:20:16 smithi083 ceph-mon[79930]: pgmap v6961: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:20:17.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:20:16 smithi153 ceph-mon[70044]: pgmap v6961: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:20:19.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:20:18 smithi153 ceph-mon[70044]: pgmap v6962: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:20:19.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:20:18 smithi083 ceph-mon[79930]: pgmap v6962: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:20:19.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:20:18 smithi083 ceph-mon[77242]: pgmap v6962: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:20:21.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:20:20 smithi083 ceph-mon[79930]: pgmap v6963: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:20:21.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:20:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:20:21.005Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:20:21.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:20:20 smithi083 ceph-mon[77242]: pgmap v6963: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:20:21.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:20:20 smithi153 ceph-mon[70044]: pgmap v6963: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:20:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:20:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:20:21] "GET /metrics HTTP/1.1" 200 225221 "" "Prometheus/2.33.4" 2022-09-29T09:20:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:20:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:20:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:20:23.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:20:22 smithi083 ceph-mon[79930]: pgmap v6964: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:20:23.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:20:22 smithi083 ceph-mon[77242]: pgmap v6964: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:20:23.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:20:22 smithi153 ceph-mon[70044]: pgmap v6964: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:20:23.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:20:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:20:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:20:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:20:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:20:24.385Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:20:25.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:20:24 smithi083 ceph-mon[79930]: pgmap v6965: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:20:25.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:20:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:20:25.095Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:20:25.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:20:24 smithi083 ceph-mon[77242]: pgmap v6965: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:20:25.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:20:24 smithi153 ceph-mon[70044]: pgmap v6965: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:20:27.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:20:26 smithi083 ceph-mon[79930]: pgmap v6966: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:20:27.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:20:26 smithi083 ceph-mon[77242]: pgmap v6966: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:20:27.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:20:26 smithi153 ceph-mon[70044]: pgmap v6966: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:20:29.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:20:28 smithi153 ceph-mon[70044]: pgmap v6967: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:20:29.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:20:28 smithi083 ceph-mon[77242]: pgmap v6967: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:20:29.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:20:28 smithi083 ceph-mon[79930]: pgmap v6967: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:20:31.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:20:30 smithi083 ceph-mon[77242]: pgmap v6968: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:20:31.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:20:30 smithi083 ceph-mon[79930]: pgmap v6968: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:20:31.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:20:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:20:31.006Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:20:31.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:20:30 smithi153 ceph-mon[70044]: pgmap v6968: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:20:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:20:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:20:31] "GET /metrics HTTP/1.1" 200 225218 "" "Prometheus/2.33.4" 2022-09-29T09:20:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:20:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:20:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:20:33.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:20:32 smithi083 ceph-mon[77242]: pgmap v6969: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:20:33.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:20:32 smithi083 ceph-mon[79930]: pgmap v6969: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:20:33.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:20:32 smithi153 ceph-mon[70044]: pgmap v6969: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:20:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:20:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:20:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:20:34.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:20:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:20:34.386Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:20:35.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:20:34 smithi083 ceph-mon[79930]: pgmap v6970: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:20:35.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:20:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:20:35.096Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:20:35.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:20:34 smithi083 ceph-mon[77242]: pgmap v6970: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:20:35.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:20:34 smithi153 ceph-mon[70044]: pgmap v6970: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:20:37.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:20:36 smithi083 ceph-mon[79930]: pgmap v6971: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:20:37.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:20:36 smithi083 ceph-mon[77242]: pgmap v6971: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:20:37.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:20:36 smithi153 ceph-mon[70044]: pgmap v6971: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:20:37.634 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:20:37.923 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:20:37.924 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 100s ago 3h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:20:37.924 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 9m ago 3h 63.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:20:37.924 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 100s ago 3h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:20:37.925 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 9m ago 3h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:20:37.925 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 9m ago 3h 442M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:20:37.925 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 100s ago 3h 714M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:20:37.925 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 100s ago 3h 888M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:20:37.925 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 9m ago 3h 375M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:20:37.926 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 100s ago 3h 519M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:20:37.926 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 100s ago 3h 26.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:20:37.926 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 9m ago 3h 25.7M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:20:37.926 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 100s ago 3h 641M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:20:37.927 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 100s ago 3h 631M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:20:37.927 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 100s ago 3h 495M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:20:37.927 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 100s ago 3h 615M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:20:37.927 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 9m ago 3h 522M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:20:37.927 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 9m ago 3h 648M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:20:37.928 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 9m ago 3h 577M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:20:37.928 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 9m ago 3h 680M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:20:37.928 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 9m ago 3h 167M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:20:38.278 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:20:38.279 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:20:38.279 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:20:38.279 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:20:38.279 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:20:38.280 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:20:38.280 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:20:38.280 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:20:38.280 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:20:38.281 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:20:38.281 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:20:38.281 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:20:38.281 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:20:38.281 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:20:38.282 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:20:38.282 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:20:38.282 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:20:38.282 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:20:39.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:20:38 smithi153 ceph-mon[70044]: from='client.42243 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:20:39.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:20:38 smithi153 ceph-mon[70044]: pgmap v6972: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:20:39.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:20:38 smithi153 ceph-mon[70044]: from='client.52180 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:20:39.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:20:38 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1394699398' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:20:39.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:20:38 smithi083 ceph-mon[77242]: from='client.42243 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:20:39.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:20:38 smithi083 ceph-mon[77242]: pgmap v6972: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:20:39.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:20:38 smithi083 ceph-mon[77242]: from='client.52180 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:20:39.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:20:38 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1394699398' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:20:39.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:20:38 smithi083 ceph-mon[79930]: from='client.42243 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:20:39.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:20:38 smithi083 ceph-mon[79930]: pgmap v6972: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:20:39.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:20:38 smithi083 ceph-mon[79930]: from='client.52180 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:20:39.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:20:38 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1394699398' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:20:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:20:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:20:40.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:20:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:20:40.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:20:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:20:40.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:20:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:20:40.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:20:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:20:40.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:20:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:20:40.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:20:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:20:40.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:20:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:20:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:20:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:20:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:20:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:20:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:20:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:20:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:20:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:20:41.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:20:40 smithi083 ceph-mon[79930]: pgmap v6973: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:20:41.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:20:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:20:41.006Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:20:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:20:40 smithi083 ceph-mon[77242]: pgmap v6973: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:20:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:20:40 smithi153 ceph-mon[70044]: pgmap v6973: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:20:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:20:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:20:41] "GET /metrics HTTP/1.1" 200 225219 "" "Prometheus/2.33.4" 2022-09-29T09:20:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:20:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:20:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:20:43.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:20:42 smithi083 ceph-mon[79930]: pgmap v6974: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:20:43.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:20:42 smithi083 ceph-mon[77242]: pgmap v6974: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:20:43.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:20:42 smithi153 ceph-mon[70044]: pgmap v6974: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:20:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:20:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:20:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:20:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:20:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:20:44.386Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:20:45.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:20:44 smithi083 ceph-mon[79930]: pgmap v6975: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:20:45.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:20:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:20:45.097Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:20:45.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:20:44 smithi083 ceph-mon[77242]: pgmap v6975: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:20:45.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:20:44 smithi153 ceph-mon[70044]: pgmap v6975: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:20:47.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:20:46 smithi083 ceph-mon[79930]: pgmap v6976: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:20:47.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:20:46 smithi083 ceph-mon[77242]: pgmap v6976: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:20:47.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:20:46 smithi153 ceph-mon[70044]: pgmap v6976: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:20:49.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:20:48 smithi153 ceph-mon[70044]: pgmap v6977: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:20:49.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:20:48 smithi083 ceph-mon[79930]: pgmap v6977: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:20:49.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:20:48 smithi083 ceph-mon[77242]: pgmap v6977: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:20:51.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:20:50 smithi083 ceph-mon[77242]: pgmap v6978: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:20:51.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:20:50 smithi083 ceph-mon[79930]: pgmap v6978: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:20:51.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:20:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:20:51.006Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:20:51.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:20:50 smithi153 ceph-mon[70044]: pgmap v6978: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:20:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:20:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:20:51] "GET /metrics HTTP/1.1" 200 225219 "" "Prometheus/2.33.4" 2022-09-29T09:20:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:20:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:20:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:20:53.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:20:52 smithi083 ceph-mon[77242]: pgmap v6979: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:20:53.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:20:52 smithi083 ceph-mon[79930]: pgmap v6979: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:20:53.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:20:52 smithi153 ceph-mon[70044]: pgmap v6979: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:20:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:20:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:20:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:20:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:20:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:20:54.387Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:20:55.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:20:54 smithi083 ceph-mon[77242]: pgmap v6980: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:20:55.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:20:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:20:55.098Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:20:55.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:20:54 smithi083 ceph-mon[79930]: pgmap v6980: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:20:55.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:20:54 smithi153 ceph-mon[70044]: pgmap v6980: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:20:57.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:20:56 smithi083 ceph-mon[77242]: pgmap v6981: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:20:57.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:20:56 smithi083 ceph-mon[79930]: pgmap v6981: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:20:57.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:20:56 smithi153 ceph-mon[70044]: pgmap v6981: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:20:59.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:20:58 smithi153 ceph-mon[70044]: pgmap v6982: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:20:59.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:20:58 smithi083 ceph-mon[79930]: pgmap v6982: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:20:59.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:20:58 smithi083 ceph-mon[77242]: pgmap v6982: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:00.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:20:59 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:21:00.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:20:59 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:21:00.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:20:59 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:21:00.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:20:59 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:21:00.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:20:59 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:21:00.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:20:59 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:21:00.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:20:59 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:21:00.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:20:59 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:21:00.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:20:59 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:21:01.010 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:21:00 smithi083 ceph-mon[79930]: pgmap v6983: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:01.010 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:21:00 smithi083 ceph-mon[77242]: pgmap v6983: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:01.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:21:00 smithi153 ceph-mon[70044]: pgmap v6983: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:01.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:21:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:21:01.007Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:21:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:21:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:21:01] "GET /metrics HTTP/1.1" 200 225220 "" "Prometheus/2.33.4" 2022-09-29T09:21:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:21:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:21:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:21:03.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:21:02 smithi083 ceph-mon[77242]: pgmap v6984: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:03.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:21:02 smithi083 ceph-mon[79930]: pgmap v6984: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:03.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:21:02 smithi153 ceph-mon[70044]: pgmap v6984: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:21:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:21:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:21:04.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:21:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:21:04.389Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:21:05.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:21:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:21:05.099Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:21:05.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:21:04 smithi083 ceph-mon[77242]: pgmap v6985: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:05.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:21:04 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:21:05.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:21:04 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:21:05.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:21:04 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:21:05.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:21:04 smithi083 ceph-mon[79930]: pgmap v6985: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:05.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:21:04 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:21:05.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:21:04 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:21:05.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:21:04 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:21:05.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:21:04 smithi153 ceph-mon[70044]: pgmap v6985: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:05.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:21:04 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:21:05.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:21:04 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:21:05.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:21:04 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:21:07.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:21:06 smithi083 ceph-mon[79930]: pgmap v6986: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:07.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:21:06 smithi083 ceph-mon[77242]: pgmap v6986: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:07.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:21:06 smithi153 ceph-mon[70044]: pgmap v6986: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:08.605 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:21:08.895 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:21:08.896 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 2m ago 3h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:21:08.896 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 4s ago 3h 62.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:21:08.896 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 2m ago 3h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:21:08.896 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 4s ago 3h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:21:08.897 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 4s ago 3h 443M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:21:08.897 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 2m ago 4h 714M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:21:08.897 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 2m ago 4h 888M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:21:08.897 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 4s ago 3h 391M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:21:08.898 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 2m ago 3h 519M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:21:08.898 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 2m ago 3h 26.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:21:08.898 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 4s ago 3h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:21:08.898 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 2m ago 3h 641M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:21:08.898 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 2m ago 3h 631M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:21:08.899 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 2m ago 3h 495M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:21:08.899 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 2m ago 3h 615M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:21:08.899 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 4s ago 3h 544M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:21:08.899 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 4s ago 3h 675M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:21:08.900 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 4s ago 3h 602M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:21:08.900 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 4s ago 3h 709M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:21:08.900 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 4s ago 3h 168M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:21:09.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:21:08 smithi153 ceph-mon[70044]: pgmap v6987: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:09.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:21:08 smithi083 ceph-mon[77242]: pgmap v6987: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:09.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:21:08 smithi083 ceph-mon[79930]: pgmap v6987: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:09.250 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:21:09.250 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:21:09.250 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:21:09.250 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:21:09.251 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:21:09.251 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:21:09.251 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:21:09.251 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:21:09.251 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:21:09.252 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:21:09.252 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:21:09.252 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:21:09.252 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:21:09.253 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:21:09.253 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:21:09.253 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:21:09.253 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:21:09.253 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:21:10.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:21:09 smithi153 ceph-mon[70044]: from='client.52192 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:21:10.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:21:09 smithi153 ceph-mon[70044]: from='client.52198 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:21:10.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:21:09 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/158945366' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:21:10.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:21:09 smithi083 ceph-mon[79930]: from='client.52192 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:21:10.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:21:09 smithi083 ceph-mon[79930]: from='client.52198 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:21:10.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:21:09 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/158945366' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:21:10.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:21:09 smithi083 ceph-mon[77242]: from='client.52192 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:21:10.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:21:09 smithi083 ceph-mon[77242]: from='client.52198 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:21:10.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:21:09 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/158945366' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:21:11.011 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:21:10 smithi083 ceph-mon[77242]: pgmap v6988: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:11.011 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:21:10 smithi083 ceph-mon[79930]: pgmap v6988: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:11.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:21:10 smithi153 ceph-mon[70044]: pgmap v6988: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:11.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:21:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:21:11.008Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:21:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:21:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:21:11] "GET /metrics HTTP/1.1" 200 225222 "" "Prometheus/2.33.4" 2022-09-29T09:21:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:21:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:21:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:21:13.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:21:12 smithi083 ceph-mon[79930]: pgmap v6989: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:13.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:21:12 smithi083 ceph-mon[77242]: pgmap v6989: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:13.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:21:12 smithi153 ceph-mon[70044]: pgmap v6989: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:13.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:21:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:21:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:21:14.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:21:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:21:14.389Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:21:15.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:21:14 smithi083 ceph-mon[77242]: pgmap v6990: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:15.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:21:14 smithi083 ceph-mon[79930]: pgmap v6990: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:15.214 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:21:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:21:15.100Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:21:15.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:21:14 smithi153 ceph-mon[70044]: pgmap v6990: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:17.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:21:16 smithi083 ceph-mon[79930]: pgmap v6991: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:17.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:21:16 smithi083 ceph-mon[77242]: pgmap v6991: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:17.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:21:16 smithi153 ceph-mon[70044]: pgmap v6991: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:19.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:21:18 smithi153 ceph-mon[70044]: pgmap v6992: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:19.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:21:18 smithi083 ceph-mon[79930]: pgmap v6992: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:19.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:21:18 smithi083 ceph-mon[77242]: pgmap v6992: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:21.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:21:20 smithi083 ceph-mon[79930]: pgmap v6993: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:21.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:21:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:21:21.008Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:21:21.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:21:20 smithi083 ceph-mon[77242]: pgmap v6993: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:21.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:21:20 smithi153 ceph-mon[70044]: pgmap v6993: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:21:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:21:21] "GET /metrics HTTP/1.1" 200 225222 "" "Prometheus/2.33.4" 2022-09-29T09:21:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:21:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:21:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:21:23.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:21:22 smithi083 ceph-mon[79930]: pgmap v6994: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:23.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:21:22 smithi083 ceph-mon[77242]: pgmap v6994: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:21:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:21:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:21:23.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:21:22 smithi153 ceph-mon[70044]: pgmap v6994: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:21:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:21:24.390Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:21:25.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:21:24 smithi083 ceph-mon[79930]: pgmap v6995: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:25.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:21:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:21:25.100Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:21:25.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:21:24 smithi083 ceph-mon[77242]: pgmap v6995: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:25.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:21:24 smithi153 ceph-mon[70044]: pgmap v6995: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:27.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:21:26 smithi083 ceph-mon[79930]: pgmap v6996: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:27.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:21:26 smithi083 ceph-mon[77242]: pgmap v6996: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:27.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:21:26 smithi153 ceph-mon[70044]: pgmap v6996: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:29.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:21:28 smithi153 ceph-mon[70044]: pgmap v6997: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:29.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:21:28 smithi083 ceph-mon[79930]: pgmap v6997: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:29.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:21:28 smithi083 ceph-mon[77242]: pgmap v6997: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:31.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:21:30 smithi083 ceph-mon[79930]: pgmap v6998: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:31.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:21:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:21:31.009Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:21:31.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:21:30 smithi083 ceph-mon[77242]: pgmap v6998: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:31.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:21:30 smithi153 ceph-mon[70044]: pgmap v6998: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:21:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:21:31] "GET /metrics HTTP/1.1" 200 225218 "" "Prometheus/2.33.4" 2022-09-29T09:21:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:21:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:21:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:21:33.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:21:32 smithi083 ceph-mon[77242]: pgmap v6999: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:33.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:21:32 smithi083 ceph-mon[79930]: pgmap v6999: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:33.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:21:32 smithi153 ceph-mon[70044]: pgmap v6999: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:33.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:21:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:21:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:21:34.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:21:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:21:34.390Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:21:35.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:21:34 smithi083 ceph-mon[77242]: pgmap v7000: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:35.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:21:34 smithi083 ceph-mon[79930]: pgmap v7000: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:35.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:21:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:21:35.100Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:21:35.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:21:34 smithi153 ceph-mon[70044]: pgmap v7000: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:37.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:21:36 smithi083 ceph-mon[79930]: pgmap v7001: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:37.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:21:36 smithi083 ceph-mon[77242]: pgmap v7001: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:37.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:21:36 smithi153 ceph-mon[70044]: pgmap v7001: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:39.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:21:38 smithi153 ceph-mon[70044]: pgmap v7002: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:39.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:21:38 smithi083 ceph-mon[79930]: pgmap v7002: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:39.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:21:38 smithi083 ceph-mon[77242]: pgmap v7002: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:39.571 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:21:39.858 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:21:39.859 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 2m ago 3h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:21:39.859 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 35s ago 3h 62.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:21:39.859 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 2m ago 3h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:21:39.860 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 35s ago 3h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:21:39.860 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 35s ago 3h 443M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:21:39.860 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 2m ago 4h 714M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:21:39.860 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 2m ago 4h 888M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:21:39.860 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 35s ago 3h 391M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:21:39.861 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 2m ago 3h 519M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:21:39.861 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 2m ago 3h 26.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:21:39.861 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 35s ago 3h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:21:39.861 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 2m ago 3h 641M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:21:39.862 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 2m ago 3h 631M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:21:39.862 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 2m ago 3h 495M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:21:39.862 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 2m ago 3h 615M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:21:39.862 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 35s ago 3h 544M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:21:39.863 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 35s ago 3h 675M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:21:39.863 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 35s ago 3h 602M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:21:39.863 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 35s ago 3h 709M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:21:39.863 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 35s ago 3h 168M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:21:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:21:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:21:40.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:21:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:21:40.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:21:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:21:40.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:21:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:21:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:21:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:21:40.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:21:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:21:40.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:21:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:21:40.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:21:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:21:40.218 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:21:40.219 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:21:40.219 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:21:40.219 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:21:40.219 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:21:40.219 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:21:40.220 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:21:40.220 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:21:40.220 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:21:40.220 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:21:40.221 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:21:40.221 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:21:40.221 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:21:40.222 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:21:40.222 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:21:40.222 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:21:40.222 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:21:40.223 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:21:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:21:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:21:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:21:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:21:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:21:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:21:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:21:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:21:41.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:21:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:21:41.009Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:21:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:21:40 smithi083 ceph-mon[77242]: from='client.52210 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:21:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:21:40 smithi083 ceph-mon[77242]: from='client.52216 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:21:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:21:40 smithi083 ceph-mon[77242]: pgmap v7003: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:41.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:21:40 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3488473206' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:21:41.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:21:40 smithi083 ceph-mon[79930]: from='client.52210 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:21:41.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:21:40 smithi083 ceph-mon[79930]: from='client.52216 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:21:41.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:21:40 smithi083 ceph-mon[79930]: pgmap v7003: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:41.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:21:40 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3488473206' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:21:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:21:40 smithi153 ceph-mon[70044]: from='client.52210 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:21:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:21:40 smithi153 ceph-mon[70044]: from='client.52216 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:21:41.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:21:40 smithi153 ceph-mon[70044]: pgmap v7003: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:41.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:21:40 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3488473206' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:21:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:21:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:21:41] "GET /metrics HTTP/1.1" 200 225216 "" "Prometheus/2.33.4" 2022-09-29T09:21:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:21:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:21:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:21:43.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:21:42 smithi083 ceph-mon[79930]: pgmap v7004: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:43.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:21:42 smithi083 ceph-mon[77242]: pgmap v7004: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:43.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:21:42 smithi153 ceph-mon[70044]: pgmap v7004: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:21:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:21:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:21:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:21:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:21:44.391Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:21:45.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:21:44 smithi083 ceph-mon[79930]: pgmap v7005: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:45.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:21:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:21:45.101Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:21:45.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:21:44 smithi083 ceph-mon[77242]: pgmap v7005: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:45.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:21:44 smithi153 ceph-mon[70044]: pgmap v7005: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:47.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:21:46 smithi083 ceph-mon[79930]: pgmap v7006: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:47.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:21:46 smithi083 ceph-mon[77242]: pgmap v7006: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:47.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:21:46 smithi153 ceph-mon[70044]: pgmap v7006: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:49.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:21:48 smithi153 ceph-mon[70044]: pgmap v7007: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:49.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:21:48 smithi083 ceph-mon[79930]: pgmap v7007: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:49.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:21:48 smithi083 ceph-mon[77242]: pgmap v7007: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:51.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:21:50 smithi083 ceph-mon[79930]: pgmap v7008: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:51.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:21:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:21:51.011Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:21:51.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:21:50 smithi083 ceph-mon[77242]: pgmap v7008: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:51.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:21:50 smithi153 ceph-mon[70044]: pgmap v7008: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:21:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:21:51] "GET /metrics HTTP/1.1" 200 225216 "" "Prometheus/2.33.4" 2022-09-29T09:21:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:21:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:21:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:21:53.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:21:52 smithi083 ceph-mon[79930]: pgmap v7009: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:53.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:21:52 smithi083 ceph-mon[77242]: pgmap v7009: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:53.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:21:52 smithi153 ceph-mon[70044]: pgmap v7009: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:21:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:21:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:21:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:21:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:21:54.391Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:21:55.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:21:54 smithi083 ceph-mon[77242]: pgmap v7010: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:55.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:21:54 smithi083 ceph-mon[79930]: pgmap v7010: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:55.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:21:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:21:55.101Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:21:55.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:21:54 smithi153 ceph-mon[70044]: pgmap v7010: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:57.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:21:56 smithi083 ceph-mon[77242]: pgmap v7011: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:57.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:21:56 smithi083 ceph-mon[79930]: pgmap v7011: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:57.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:21:56 smithi153 ceph-mon[70044]: pgmap v7011: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:59.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:21:58 smithi153 ceph-mon[70044]: pgmap v7012: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:59.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:21:58 smithi083 ceph-mon[79930]: pgmap v7012: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:21:59.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:21:58 smithi083 ceph-mon[77242]: pgmap v7012: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:01.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:22:00 smithi083 ceph-mon[77242]: pgmap v7013: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:01.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:22:00 smithi083 ceph-mon[79930]: pgmap v7013: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:01.214 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:22:01.011Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:22:01.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:22:00 smithi153 ceph-mon[70044]: pgmap v7013: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:22:01] "GET /metrics HTTP/1.1" 200 225220 "" "Prometheus/2.33.4" 2022-09-29T09:22:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:22:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:22:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:22:03.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:22:02 smithi083 ceph-mon[79930]: pgmap v7014: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:03.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:22:02 smithi083 ceph-mon[77242]: pgmap v7014: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:03.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:22:02 smithi153 ceph-mon[70044]: pgmap v7014: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:22:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:22:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:22:04.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:22:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:22:04.392Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:22:05.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:22:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:22:05.105Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:22:05.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:22:04 smithi083 ceph-mon[79930]: pgmap v7015: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:05.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:22:04 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:22:05.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:22:04 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:22:05.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:22:04 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:22:05.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:22:04 smithi083 ceph-mon[77242]: pgmap v7015: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:05.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:22:04 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:22:05.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:22:04 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:22:05.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:22:04 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:22:05.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:22:04 smithi153 ceph-mon[70044]: pgmap v7015: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:05.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:22:04 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:22:05.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:22:04 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:22:05.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:22:04 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:22:06.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:22:06 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:22:06.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:22:06 smithi083 ceph-mon[79930]: pgmap v7016: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:06.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:22:06 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:22:06.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:22:06 smithi083 ceph-mon[77242]: pgmap v7016: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:06.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:22:06 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:22:06.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:22:06 smithi153 ceph-mon[70044]: pgmap v7016: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:09.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:22:08 smithi153 ceph-mon[70044]: pgmap v7017: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:09.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:22:08 smithi083 ceph-mon[79930]: pgmap v7017: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:09.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:22:08 smithi083 ceph-mon[77242]: pgmap v7017: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:10.537 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:22:10.826 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:22:10.826 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 3m ago 3h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:22:10.827 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 66s ago 3h 62.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:22:10.827 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 3m ago 3h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:22:10.827 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 66s ago 3h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:22:10.827 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 66s ago 3h 443M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:22:10.828 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 3m ago 4h 714M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:22:10.828 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 3m ago 4h 888M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:22:10.828 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 66s ago 4h 391M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:22:10.828 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 3m ago 4h 519M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:22:10.828 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 3m ago 3h 26.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:22:10.829 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 66s ago 3h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:22:10.829 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 3m ago 3h 641M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:22:10.829 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 3m ago 3h 631M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:22:10.829 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 3m ago 3h 495M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:22:10.829 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 3m ago 3h 615M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:22:10.830 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 66s ago 3h 544M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:22:10.830 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 66s ago 3h 675M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:22:10.830 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 66s ago 3h 602M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:22:10.831 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 66s ago 3h 709M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:22:10.831 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 66s ago 3h 168M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:22:11.186 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:22:11.186 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:22:11.186 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:22:11.187 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:22:11.187 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:22:11.187 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:22:11.187 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:22:11.188 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:22:11.188 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:22:11.188 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:22:11.188 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:22:11.188 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:22:11.189 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:22:11.189 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:22:11.189 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:22:11.189 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:22:11.189 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:22:11.190 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:22:11.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:22:10 smithi083 ceph-mon[79930]: pgmap v7018: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:11.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:22:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:22:11.012Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:22:11.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:22:10 smithi083 ceph-mon[77242]: pgmap v7018: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:11.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:22:10 smithi153 ceph-mon[70044]: pgmap v7018: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:11.952 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:22:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:22:11] "GET /metrics HTTP/1.1" 200 225217 "" "Prometheus/2.33.4" 2022-09-29T09:22:11.952 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:22:11 smithi083 ceph-mon[77242]: from='client.52228 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:22:11.953 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:22:11 smithi083 ceph-mon[77242]: from='client.42303 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:22:11.953 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:22:11 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2854941306' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:22:12.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:22:11 smithi083 ceph-mon[79930]: from='client.52228 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:22:12.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:22:11 smithi083 ceph-mon[79930]: from='client.42303 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:22:12.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:22:11 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2854941306' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:22:12.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:22:11 smithi153 ceph-mon[70044]: from='client.52228 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:22:12.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:22:11 smithi153 ceph-mon[70044]: from='client.42303 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:22:12.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:22:11 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2854941306' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:22:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:22:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:22:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:22:13.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:22:12 smithi083 ceph-mon[79930]: pgmap v7019: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:13.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:22:12 smithi083 ceph-mon[77242]: pgmap v7019: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:13.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:22:12 smithi153 ceph-mon[70044]: pgmap v7019: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:13.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:22:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:22:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:22:14.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:22:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:22:14.393Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:22:15.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:22:14 smithi083 ceph-mon[79930]: pgmap v7020: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:15.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:22:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:22:15.103Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:22:15.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:22:14 smithi083 ceph-mon[77242]: pgmap v7020: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:15.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:22:14 smithi153 ceph-mon[70044]: pgmap v7020: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:17.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:22:16 smithi083 ceph-mon[79930]: pgmap v7021: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:17.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:22:16 smithi083 ceph-mon[77242]: pgmap v7021: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:17.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:22:16 smithi153 ceph-mon[70044]: pgmap v7021: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:19.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:22:18 smithi153 ceph-mon[70044]: pgmap v7022: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:19.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:22:18 smithi083 ceph-mon[79930]: pgmap v7022: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:19.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:22:18 smithi083 ceph-mon[77242]: pgmap v7022: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:21.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:22:20 smithi083 ceph-mon[79930]: pgmap v7023: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:21.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:22:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:22:21.012Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:22:21.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:22:20 smithi083 ceph-mon[77242]: pgmap v7023: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:21.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:22:20 smithi153 ceph-mon[70044]: pgmap v7023: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:22:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:22:21] "GET /metrics HTTP/1.1" 200 225217 "" "Prometheus/2.33.4" 2022-09-29T09:22:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:22:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:22:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:22:23.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:22:22 smithi083 ceph-mon[77242]: pgmap v7024: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:23.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:22:22 smithi083 ceph-mon[79930]: pgmap v7024: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:23.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:22:22 smithi153 ceph-mon[70044]: pgmap v7024: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:22:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:22:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:22:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:22:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:22:24.393Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:22:25.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:22:24 smithi083 ceph-mon[79930]: pgmap v7025: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:25.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:22:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:22:25.104Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:22:25.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:22:24 smithi083 ceph-mon[77242]: pgmap v7025: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:25.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:22:24 smithi153 ceph-mon[70044]: pgmap v7025: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:27.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:22:26 smithi083 ceph-mon[79930]: pgmap v7026: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:27.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:22:26 smithi083 ceph-mon[77242]: pgmap v7026: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:27.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:22:26 smithi153 ceph-mon[70044]: pgmap v7026: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:29.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:22:28 smithi153 ceph-mon[70044]: pgmap v7027: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:29.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:22:28 smithi083 ceph-mon[79930]: pgmap v7027: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:29.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:22:28 smithi083 ceph-mon[77242]: pgmap v7027: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:31.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:22:30 smithi083 ceph-mon[79930]: pgmap v7028: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:31.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:22:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:22:31.013Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:22:31.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:22:30 smithi083 ceph-mon[77242]: pgmap v7028: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:31.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:22:30 smithi153 ceph-mon[70044]: pgmap v7028: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:22:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:22:31] "GET /metrics HTTP/1.1" 200 225216 "" "Prometheus/2.33.4" 2022-09-29T09:22:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:22:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:22:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:22:33.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:22:32 smithi083 ceph-mon[79930]: pgmap v7029: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:33.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:22:32 smithi083 ceph-mon[77242]: pgmap v7029: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:33.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:22:32 smithi153 ceph-mon[70044]: pgmap v7029: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:22:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:22:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:22:34.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:22:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:22:34.394Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:22:35.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:22:34 smithi083 ceph-mon[79930]: pgmap v7030: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:35.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:22:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:22:35.104Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:22:35.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:22:34 smithi083 ceph-mon[77242]: pgmap v7030: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:35.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:22:34 smithi153 ceph-mon[70044]: pgmap v7030: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:37.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:22:36 smithi153 ceph-mon[70044]: pgmap v7031: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:37.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:22:36 smithi083 ceph-mon[79930]: pgmap v7031: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:37.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:22:36 smithi083 ceph-mon[77242]: pgmap v7031: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:39.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:22:38 smithi153 ceph-mon[70044]: pgmap v7032: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:39.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:22:38 smithi083 ceph-mon[79930]: pgmap v7032: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:39.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:22:38 smithi083 ceph-mon[77242]: pgmap v7032: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:22:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:22:40.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:22:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:22:40.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:22:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:22:40.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:22:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:22:40.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:22:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:22:40.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:22:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:22:40.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:22:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:22:40.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:22:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:22:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:22:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:22:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:22:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:22:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:22:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:22:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:22:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:22:41.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:22:40 smithi083 ceph-mon[79930]: pgmap v7033: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:22:40 smithi083 ceph-mon[77242]: pgmap v7033: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:41.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:22:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:22:41.013Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:22:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:22:40 smithi153 ceph-mon[70044]: pgmap v7033: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:41.510 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:22:41.798 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:22:41.798 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 3m ago 3h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:22:41.798 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 97s ago 3h 62.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:22:41.799 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 3m ago 3h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:22:41.799 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 97s ago 3h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:22:41.799 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 97s ago 4h 443M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:22:41.799 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 3m ago 4h 714M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:22:41.799 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 3m ago 4h 888M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:22:41.800 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 97s ago 4h 391M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:22:41.800 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 3m ago 4h 519M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:22:41.800 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 3m ago 3h 26.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:22:41.800 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 97s ago 3h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:22:41.800 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 3m ago 4h 641M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:22:41.801 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 3m ago 3h 631M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:22:41.801 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 3m ago 3h 495M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:22:41.801 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 3m ago 3h 615M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:22:41.801 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 97s ago 3h 544M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:22:41.802 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 97s ago 3h 675M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:22:41.802 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 97s ago 3h 602M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:22:41.802 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 97s ago 3h 709M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:22:41.802 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 97s ago 3h 168M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:22:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:22:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:22:41] "GET /metrics HTTP/1.1" 200 225213 "" "Prometheus/2.33.4" 2022-09-29T09:22:42.158 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:22:42.158 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:22:42.159 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:22:42.159 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:22:42.159 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:22:42.159 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:22:42.160 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:22:42.160 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:22:42.160 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:22:42.160 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:22:42.161 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:22:42.161 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:22:42.161 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:22:42.161 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:22:42.161 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:22:42.162 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:22:42.162 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:22:42.162 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:22:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:22:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:22:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:22:43.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:22:42 smithi083 ceph-mon[77242]: from='client.52246 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:22:43.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:22:42 smithi083 ceph-mon[77242]: from='client.52252 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:22:43.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:22:42 smithi083 ceph-mon[77242]: pgmap v7034: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:43.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:22:42 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2521747033' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:22:43.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:22:42 smithi083 ceph-mon[79930]: from='client.52246 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:22:43.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:22:42 smithi083 ceph-mon[79930]: from='client.52252 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:22:43.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:22:42 smithi083 ceph-mon[79930]: pgmap v7034: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:43.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:22:42 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2521747033' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:22:43.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:22:42 smithi153 ceph-mon[70044]: from='client.52246 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:22:43.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:22:42 smithi153 ceph-mon[70044]: from='client.52252 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:22:43.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:22:42 smithi153 ceph-mon[70044]: pgmap v7034: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:43.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:22:42 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2521747033' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:22:43.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:22:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:22:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:22:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:22:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:22:44.395Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:22:45.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:22:44 smithi153 ceph-mon[70044]: pgmap v7035: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:45.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:22:44 smithi083 ceph-mon[79930]: pgmap v7035: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:45.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:22:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:22:45.105Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:22:45.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:22:44 smithi083 ceph-mon[77242]: pgmap v7035: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:47.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:22:46 smithi153 ceph-mon[70044]: pgmap v7036: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:47.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:22:46 smithi083 ceph-mon[79930]: pgmap v7036: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:47.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:22:46 smithi083 ceph-mon[77242]: pgmap v7036: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:49.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:22:48 smithi153 ceph-mon[70044]: pgmap v7037: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:49.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:22:48 smithi083 ceph-mon[79930]: pgmap v7037: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:49.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:22:48 smithi083 ceph-mon[77242]: pgmap v7037: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:51.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:22:50 smithi083 ceph-mon[79930]: pgmap v7038: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:51.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:22:50 smithi083 ceph-mon[77242]: pgmap v7038: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:51.214 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:22:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:22:51.014Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:22:51.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:22:50 smithi153 ceph-mon[70044]: pgmap v7038: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:22:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:22:51] "GET /metrics HTTP/1.1" 200 225213 "" "Prometheus/2.33.4" 2022-09-29T09:22:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:22:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:22:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:22:53.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:22:52 smithi153 ceph-mon[70044]: pgmap v7039: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:53.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:22:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:22:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:22:53.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:22:52 smithi083 ceph-mon[79930]: pgmap v7039: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:53.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:22:52 smithi083 ceph-mon[77242]: pgmap v7039: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:22:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:22:54.395Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:22:55.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:22:54 smithi153 ceph-mon[70044]: pgmap v7040: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:55.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:22:54 smithi083 ceph-mon[79930]: pgmap v7040: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:55.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:22:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:22:55.104Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:22:55.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:22:54 smithi083 ceph-mon[77242]: pgmap v7040: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:57.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:22:56 smithi153 ceph-mon[70044]: pgmap v7041: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:57.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:22:56 smithi083 ceph-mon[79930]: pgmap v7041: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:57.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:22:56 smithi083 ceph-mon[77242]: pgmap v7041: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:59.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:22:58 smithi153 ceph-mon[70044]: pgmap v7042: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:59.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:22:58 smithi083 ceph-mon[79930]: pgmap v7042: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:22:59.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:22:58 smithi083 ceph-mon[77242]: pgmap v7042: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:01.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:23:00 smithi083 ceph-mon[79930]: pgmap v7043: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:01.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:23:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:23:01.015Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:23:01.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:23:00 smithi083 ceph-mon[77242]: pgmap v7043: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:01.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:23:00 smithi153 ceph-mon[70044]: pgmap v7043: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:23:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:23:01] "GET /metrics HTTP/1.1" 200 225218 "" "Prometheus/2.33.4" 2022-09-29T09:23:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:23:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:23:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:23:03.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:23:02 smithi153 ceph-mon[70044]: pgmap v7044: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:23:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:23:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:23:03.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:23:02 smithi083 ceph-mon[79930]: pgmap v7044: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:03.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:23:02 smithi083 ceph-mon[77242]: pgmap v7044: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:04.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:23:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:23:04.395Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:23:05.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:23:04 smithi153 ceph-mon[70044]: pgmap v7045: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:05.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:23:04 smithi083 ceph-mon[79930]: pgmap v7045: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:05.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:23:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:23:05.106Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:23:05.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:23:04 smithi083 ceph-mon[77242]: pgmap v7045: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:06.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:23:05 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:23:06.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:23:05 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:23:06.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:23:05 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:23:06.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:23:05 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:23:06.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:23:05 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:23:06.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:23:05 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:23:06.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:23:05 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:23:06.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:23:05 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:23:06.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:23:05 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:23:06.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:23:05 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:23:06.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:23:05 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:23:06.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:23:05 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:23:07.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:23:06 smithi153 ceph-mon[70044]: pgmap v7046: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:07.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:23:06 smithi083 ceph-mon[79930]: pgmap v7046: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:07.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:23:06 smithi083 ceph-mon[77242]: pgmap v7046: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:09.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:23:08 smithi153 ceph-mon[70044]: pgmap v7047: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:09.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:23:08 smithi083 ceph-mon[79930]: pgmap v7047: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:09.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:23:08 smithi083 ceph-mon[77242]: pgmap v7047: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:11.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:23:10 smithi153 ceph-mon[70044]: pgmap v7048: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:11.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:23:10 smithi083 ceph-mon[79930]: pgmap v7048: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:11.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:23:10 smithi083 ceph-mon[77242]: pgmap v7048: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:11.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:23:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:23:11.016Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:23:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:23:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:23:11] "GET /metrics HTTP/1.1" 200 225216 "" "Prometheus/2.33.4" 2022-09-29T09:23:12.480 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:23:12.770 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:23:12.771 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 4m ago 3h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:23:12.771 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 2m ago 3h 62.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:23:12.771 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 4m ago 3h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:23:12.771 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 2m ago 3h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:23:12.772 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 2m ago 4h 443M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:23:12.772 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 4m ago 4h 714M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:23:12.772 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 4m ago 4h 888M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:23:12.772 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 2m ago 4h 391M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:23:12.773 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 4m ago 4h 519M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:23:12.773 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 4m ago 3h 26.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:23:12.773 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 2m ago 3h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:23:12.773 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 4m ago 4h 641M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:23:12.774 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 4m ago 4h 631M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:23:12.774 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 4m ago 3h 495M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:23:12.774 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 4m ago 3h 615M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:23:12.775 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 2m ago 3h 544M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:23:12.775 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 2m ago 3h 675M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:23:12.775 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 2m ago 3h 602M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:23:12.775 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 2m ago 3h 709M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:23:12.776 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 2m ago 3h 168M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:23:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:23:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:23:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:23:13.126 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:23:13.126 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:23:13.126 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:23:13.127 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:23:13.127 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:23:13.127 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:23:13.127 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:23:13.128 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:23:13.128 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:23:13.128 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:23:13.128 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:23:13.129 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:23:13.129 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:23:13.129 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:23:13.129 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:23:13.130 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:23:13.130 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:23:13.130 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:23:13.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:23:12 smithi153 ceph-mon[70044]: pgmap v7049: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:13.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:23:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:23:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:23:13.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:23:12 smithi083 ceph-mon[79930]: pgmap v7049: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:13.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:23:12 smithi083 ceph-mon[77242]: pgmap v7049: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:14.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:23:13 smithi153 ceph-mon[70044]: from='client.42333 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:23:14.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:23:13 smithi153 ceph-mon[70044]: from='client.42339 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:23:14.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:23:13 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2761184687' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:23:14.398 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:23:13 smithi083 ceph-mon[79930]: from='client.42333 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:23:14.399 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:23:13 smithi083 ceph-mon[79930]: from='client.42339 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:23:14.399 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:23:13 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2761184687' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:23:14.399 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:23:13 smithi083 ceph-mon[77242]: from='client.42333 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:23:14.399 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:23:13 smithi083 ceph-mon[77242]: from='client.42339 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:23:14.400 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:23:13 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2761184687' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:23:14.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:23:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:23:14.396Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:23:15.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:23:14 smithi153 ceph-mon[70044]: pgmap v7050: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:15.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:23:14 smithi083 ceph-mon[79930]: pgmap v7050: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:15.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:23:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:23:15.106Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:23:15.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:23:14 smithi083 ceph-mon[77242]: pgmap v7050: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:17.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:23:16 smithi153 ceph-mon[70044]: pgmap v7051: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:17.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:23:16 smithi083 ceph-mon[79930]: pgmap v7051: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:17.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:23:16 smithi083 ceph-mon[77242]: pgmap v7051: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:19.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:23:18 smithi153 ceph-mon[70044]: pgmap v7052: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:19.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:23:18 smithi083 ceph-mon[77242]: pgmap v7052: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:19.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:23:18 smithi083 ceph-mon[79930]: pgmap v7052: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:21.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:23:20 smithi153 ceph-mon[70044]: pgmap v7053: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:21.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:23:20 smithi083 ceph-mon[79930]: pgmap v7053: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:21.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:23:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:23:21.017Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:23:21.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:23:20 smithi083 ceph-mon[77242]: pgmap v7053: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:23:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:23:21] "GET /metrics HTTP/1.1" 200 225216 "" "Prometheus/2.33.4" 2022-09-29T09:23:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:23:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:23:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:23:23.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:23:22 smithi153 ceph-mon[70044]: pgmap v7054: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:23.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:23:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:23:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:23:23.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:23:22 smithi083 ceph-mon[79930]: pgmap v7054: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:23.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:23:22 smithi083 ceph-mon[77242]: pgmap v7054: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:23:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:23:24.397Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:23:25.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:23:24 smithi153 ceph-mon[70044]: pgmap v7055: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:25.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:23:24 smithi083 ceph-mon[77242]: pgmap v7055: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:25.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:23:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:23:25.106Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:23:25.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:23:24 smithi083 ceph-mon[79930]: pgmap v7055: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:27.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:23:26 smithi153 ceph-mon[70044]: pgmap v7056: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:27.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:23:26 smithi083 ceph-mon[79930]: pgmap v7056: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:27.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:23:26 smithi083 ceph-mon[77242]: pgmap v7056: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:29.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:23:28 smithi153 ceph-mon[70044]: pgmap v7057: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:29.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:23:28 smithi083 ceph-mon[79930]: pgmap v7057: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:29.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:23:28 smithi083 ceph-mon[77242]: pgmap v7057: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:31.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:23:30 smithi153 ceph-mon[70044]: pgmap v7058: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:31.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:23:30 smithi083 ceph-mon[79930]: pgmap v7058: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:31.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:23:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:23:31.018Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:23:31.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:23:30 smithi083 ceph-mon[77242]: pgmap v7058: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:23:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:23:31] "GET /metrics HTTP/1.1" 200 225213 "" "Prometheus/2.33.4" 2022-09-29T09:23:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:23:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:23:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:23:33.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:23:32 smithi153 ceph-mon[70044]: pgmap v7059: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:33.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:23:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:23:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:23:33.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:23:32 smithi083 ceph-mon[79930]: pgmap v7059: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:33.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:23:32 smithi083 ceph-mon[77242]: pgmap v7059: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:34.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:23:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:23:34.397Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:23:35.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:23:34 smithi153 ceph-mon[70044]: pgmap v7060: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:35.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:23:34 smithi083 ceph-mon[77242]: pgmap v7060: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:35.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:23:34 smithi083 ceph-mon[79930]: pgmap v7060: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:35.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:23:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:23:35.106Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:23:37.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:23:36 smithi153 ceph-mon[70044]: pgmap v7061: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:37.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:23:36 smithi083 ceph-mon[79930]: pgmap v7061: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:37.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:23:36 smithi083 ceph-mon[77242]: pgmap v7061: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:39.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:23:38 smithi153 ceph-mon[70044]: pgmap v7062: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:39.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:23:38 smithi083 ceph-mon[79930]: pgmap v7062: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:39.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:23:38 smithi083 ceph-mon[77242]: pgmap v7062: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:40.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:23:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:23:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:23:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:23:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:23:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:23:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:23:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:23:40.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:23:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:23:40.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:23:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:23:40.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:23:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:23:40.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:23:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:23:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:23:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:23:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:23:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:23:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:23:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:23:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:23:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:23:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:23:40 smithi153 ceph-mon[70044]: pgmap v7063: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:41.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:23:40 smithi083 ceph-mon[79930]: pgmap v7063: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:41.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:23:40 smithi083 ceph-mon[77242]: pgmap v7063: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:41.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:23:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:23:41.019Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:23:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:23:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:23:41] "GET /metrics HTTP/1.1" 200 225219 "" "Prometheus/2.33.4" 2022-09-29T09:23:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:23:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:23:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:23:43.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:23:42 smithi153 ceph-mon[70044]: pgmap v7064: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:43.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:23:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:23:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:23:43.461 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:23:43.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:23:42 smithi083 ceph-mon[79930]: pgmap v7064: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:43.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:23:42 smithi083 ceph-mon[77242]: pgmap v7064: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:43.757 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:23:43.757 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 4m ago 3h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:23:43.757 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 2m ago 3h 62.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:23:43.758 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 4m ago 3h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:23:43.758 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 2m ago 3h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:23:43.758 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 2m ago 4h 443M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:23:43.758 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 4m ago 4h 714M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:23:43.758 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 4m ago 4h 888M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:23:43.759 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 2m ago 4h 391M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:23:43.759 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 4m ago 4h 519M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:23:43.759 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 4m ago 3h 26.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:23:43.759 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 2m ago 3h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:23:43.760 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 4m ago 4h 641M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:23:43.760 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 4m ago 4h 631M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:23:43.760 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 4m ago 4h 495M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:23:43.760 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 4m ago 4h 615M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:23:43.760 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 2m ago 3h 544M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:23:43.761 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 2m ago 3h 675M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:23:43.761 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 2m ago 3h 602M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:23:43.761 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 2m ago 3h 709M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:23:43.761 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 2m ago 3h 168M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:23:44.119 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:23:44.119 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:23:44.119 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:23:44.120 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:23:44.120 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:23:44.120 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:23:44.120 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:23:44.120 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:23:44.121 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:23:44.121 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:23:44.121 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:23:44.121 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:23:44.121 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:23:44.122 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:23:44.122 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:23:44.122 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:23:44.122 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:23:44.122 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:23:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:23:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:23:44.397Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:23:45.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:23:44 smithi153 ceph-mon[70044]: from='client.42351 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:23:45.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:23:44 smithi153 ceph-mon[70044]: from='client.42357 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:23:45.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:23:44 smithi153 ceph-mon[70044]: pgmap v7065: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:45.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:23:44 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/999612768' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:23:45.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:23:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:23:45.107Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:23:45.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:23:44 smithi083 ceph-mon[79930]: from='client.42351 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:23:45.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:23:44 smithi083 ceph-mon[79930]: from='client.42357 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:23:45.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:23:44 smithi083 ceph-mon[79930]: pgmap v7065: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:45.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:23:44 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/999612768' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:23:45.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:23:44 smithi083 ceph-mon[77242]: from='client.42351 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:23:45.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:23:44 smithi083 ceph-mon[77242]: from='client.42357 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:23:45.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:23:44 smithi083 ceph-mon[77242]: pgmap v7065: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:45.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:23:44 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/999612768' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:23:47.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:23:46 smithi153 ceph-mon[70044]: pgmap v7066: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:47.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:23:46 smithi083 ceph-mon[79930]: pgmap v7066: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:47.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:23:46 smithi083 ceph-mon[77242]: pgmap v7066: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:49.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:23:48 smithi153 ceph-mon[70044]: pgmap v7067: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:49.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:23:48 smithi083 ceph-mon[79930]: pgmap v7067: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:49.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:23:48 smithi083 ceph-mon[77242]: pgmap v7067: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:51.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:23:50 smithi153 ceph-mon[70044]: pgmap v7068: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:51.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:23:50 smithi083 ceph-mon[79930]: pgmap v7068: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:51.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:23:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:23:51.019Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:23:51.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:23:50 smithi083 ceph-mon[77242]: pgmap v7068: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:23:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:23:51] "GET /metrics HTTP/1.1" 200 225219 "" "Prometheus/2.33.4" 2022-09-29T09:23:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:23:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:23:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:23:53.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:23:52 smithi153 ceph-mon[70044]: pgmap v7069: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:53.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:23:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:23:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:23:53.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:23:52 smithi083 ceph-mon[79930]: pgmap v7069: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:53.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:23:52 smithi083 ceph-mon[77242]: pgmap v7069: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:23:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:23:54.398Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:23:55.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:23:54 smithi153 ceph-mon[70044]: pgmap v7070: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:55.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:23:54 smithi083 ceph-mon[79930]: pgmap v7070: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:55.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:23:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:23:55.108Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:23:55.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:23:54 smithi083 ceph-mon[77242]: pgmap v7070: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:57.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:23:56 smithi153 ceph-mon[70044]: pgmap v7071: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:57.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:23:56 smithi083 ceph-mon[77242]: pgmap v7071: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:57.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:23:56 smithi083 ceph-mon[79930]: pgmap v7071: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:59.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:23:58 smithi153 ceph-mon[70044]: pgmap v7072: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:59.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:23:58 smithi083 ceph-mon[77242]: pgmap v7072: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:23:59.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:23:58 smithi083 ceph-mon[79930]: pgmap v7072: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:01.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:24:00 smithi153 ceph-mon[70044]: pgmap v7073: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:01.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:24:00 smithi083 ceph-mon[79930]: pgmap v7073: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:01.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:24:00 smithi083 ceph-mon[77242]: pgmap v7073: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:01.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:24:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:24:01.020Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:24:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:24:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:24:01] "GET /metrics HTTP/1.1" 200 225221 "" "Prometheus/2.33.4" 2022-09-29T09:24:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:24:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:24:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:24:03.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:24:02 smithi153 ceph-mon[70044]: pgmap v7074: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:24:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:24:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:24:03.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:24:02 smithi083 ceph-mon[79930]: pgmap v7074: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:03.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:24:02 smithi083 ceph-mon[77242]: pgmap v7074: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:04.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:24:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:24:04.398Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:24:05.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:24:04 smithi153 ceph-mon[70044]: pgmap v7075: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:05.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:24:04 smithi083 ceph-mon[79930]: pgmap v7075: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:05.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:24:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:24:05.108Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:24:05.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:24:04 smithi083 ceph-mon[77242]: pgmap v7075: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:06.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:24:05 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:24:06.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:24:05 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:24:06.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:24:05 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:24:06.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:24:05 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:24:06.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:24:05 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:24:06.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:24:05 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:24:06.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:24:05 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:24:06.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:24:05 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:24:06.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:24:05 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:24:07.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:24:06 smithi153 ceph-mon[70044]: pgmap v7076: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:07.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:24:06 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:24:07.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:24:06 smithi083 ceph-mon[79930]: pgmap v7076: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:07.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:24:06 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:24:07.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:24:06 smithi083 ceph-mon[77242]: pgmap v7076: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:07.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:24:06 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:24:09.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:24:08 smithi153 ceph-mon[70044]: pgmap v7077: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:09.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:24:08 smithi083 ceph-mon[79930]: pgmap v7077: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:09.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:24:08 smithi083 ceph-mon[77242]: pgmap v7077: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:11.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:24:10 smithi153 ceph-mon[70044]: pgmap v7078: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:11.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:24:10 smithi083 ceph-mon[79930]: pgmap v7078: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:11.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:24:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:24:11.020Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:24:11.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:24:10 smithi083 ceph-mon[77242]: pgmap v7078: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:24:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:24:11] "GET /metrics HTTP/1.1" 200 225220 "" "Prometheus/2.33.4" 2022-09-29T09:24:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:24:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:24:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:24:13.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:24:12 smithi153 ceph-mon[70044]: pgmap v7079: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:13.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:24:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:24:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:24:13.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:24:12 smithi083 ceph-mon[79930]: pgmap v7079: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:13.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:24:12 smithi083 ceph-mon[77242]: pgmap v7079: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:14.448 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:24:14.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:24:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:24:14.398Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:24:14.735 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:24:14.735 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 5m ago 3h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:24:14.735 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 3m ago 3h 62.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:24:14.735 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 5m ago 3h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:24:14.736 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 3m ago 3h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:24:14.736 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 3m ago 4h 443M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:24:14.736 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 5m ago 4h 714M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:24:14.736 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 5m ago 4h 888M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:24:14.737 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 3m ago 4h 391M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:24:14.737 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 5m ago 4h 519M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:24:14.737 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 5m ago 3h 26.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:24:14.737 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 3m ago 3h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:24:14.738 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 5m ago 4h 641M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:24:14.738 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 5m ago 4h 631M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:24:14.738 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 5m ago 4h 495M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:24:14.738 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 5m ago 4h 615M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:24:14.739 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 3m ago 4h 544M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:24:14.739 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 3m ago 4h 675M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:24:14.739 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 3m ago 3h 602M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:24:14.739 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 3m ago 3h 709M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:24:14.739 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 3m ago 3h 168M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:24:15.096 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:24:15.097 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:24:15.097 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:24:15.097 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:24:15.097 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:24:15.097 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:24:15.098 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:24:15.098 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:24:15.098 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:24:15.098 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:24:15.099 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:24:15.099 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:24:15.099 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:24:15.099 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:24:15.099 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:24:15.100 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:24:15.100 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:24:15.100 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:24:15.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:24:14 smithi153 ceph-mon[70044]: pgmap v7080: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:15.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:24:14 smithi083 ceph-mon[77242]: pgmap v7080: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:15.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:24:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:24:15.108Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:24:15.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:24:14 smithi083 ceph-mon[79930]: pgmap v7080: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:16.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:24:15 smithi153 ceph-mon[70044]: from='client.42369 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:24:16.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:24:15 smithi153 ceph-mon[70044]: from='client.52306 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:24:16.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:24:15 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2985071886' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:24:16.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:24:15 smithi083 ceph-mon[79930]: from='client.42369 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:24:16.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:24:15 smithi083 ceph-mon[79930]: from='client.52306 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:24:16.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:24:15 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2985071886' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:24:16.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:24:15 smithi083 ceph-mon[77242]: from='client.42369 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:24:16.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:24:15 smithi083 ceph-mon[77242]: from='client.52306 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:24:16.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:24:15 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2985071886' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:24:17.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:24:16 smithi153 ceph-mon[70044]: pgmap v7081: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:17.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:24:16 smithi083 ceph-mon[77242]: pgmap v7081: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:17.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:24:16 smithi083 ceph-mon[79930]: pgmap v7081: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:19.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:24:18 smithi153 ceph-mon[70044]: pgmap v7082: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:19.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:24:18 smithi083 ceph-mon[79930]: pgmap v7082: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:19.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:24:18 smithi083 ceph-mon[77242]: pgmap v7082: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:21.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:24:20 smithi153 ceph-mon[70044]: pgmap v7083: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:21.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:24:20 smithi083 ceph-mon[79930]: pgmap v7083: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:21.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:24:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:24:21.021Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:24:21.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:24:20 smithi083 ceph-mon[77242]: pgmap v7083: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:24:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:24:21] "GET /metrics HTTP/1.1" 200 225220 "" "Prometheus/2.33.4" 2022-09-29T09:24:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:24:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:24:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:24:23.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:24:22 smithi153 ceph-mon[70044]: pgmap v7084: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:23.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:24:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:24:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:24:23.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:24:22 smithi083 ceph-mon[79930]: pgmap v7084: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:23.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:24:22 smithi083 ceph-mon[77242]: pgmap v7084: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:24:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:24:24.399Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:24:25.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:24:24 smithi153 ceph-mon[70044]: pgmap v7085: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:25.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:24:24 smithi083 ceph-mon[79930]: pgmap v7085: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:25.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:24:24 smithi083 ceph-mon[77242]: pgmap v7085: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:25.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:24:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:24:25.109Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:24:27.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:24:26 smithi153 ceph-mon[70044]: pgmap v7086: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:27.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:24:26 smithi083 ceph-mon[79930]: pgmap v7086: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:27.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:24:26 smithi083 ceph-mon[77242]: pgmap v7086: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:29.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:24:28 smithi153 ceph-mon[70044]: pgmap v7087: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:29.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:24:28 smithi083 ceph-mon[79930]: pgmap v7087: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:29.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:24:28 smithi083 ceph-mon[77242]: pgmap v7087: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:31.313 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:24:30 smithi153 ceph-mon[70044]: pgmap v7088: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:31.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:24:30 smithi083 ceph-mon[77242]: pgmap v7088: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:31.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:24:30 smithi083 ceph-mon[79930]: pgmap v7088: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:31.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:24:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:24:31.021Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:24:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:24:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:24:31] "GET /metrics HTTP/1.1" 200 225218 "" "Prometheus/2.33.4" 2022-09-29T09:24:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:24:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:24:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:24:33.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:24:32 smithi153 ceph-mon[70044]: pgmap v7089: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:33.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:24:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:24:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:24:33.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:24:32 smithi083 ceph-mon[79930]: pgmap v7089: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:33.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:24:32 smithi083 ceph-mon[77242]: pgmap v7089: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:34.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:24:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:24:34.401Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:24:35.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:24:34 smithi153 ceph-mon[70044]: pgmap v7090: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:35.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:24:34 smithi083 ceph-mon[79930]: pgmap v7090: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:35.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:24:34 smithi083 ceph-mon[77242]: pgmap v7090: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:35.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:24:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:24:35.110Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:24:37.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:24:36 smithi153 ceph-mon[70044]: pgmap v7091: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:37.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:24:36 smithi083 ceph-mon[77242]: pgmap v7091: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:37.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:24:36 smithi083 ceph-mon[79930]: pgmap v7091: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:39.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:24:38 smithi153 ceph-mon[70044]: pgmap v7092: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:39.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:24:38 smithi083 ceph-mon[79930]: pgmap v7092: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:39.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:24:38 smithi083 ceph-mon[77242]: pgmap v7092: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:24:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:24:40.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:24:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:24:40.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:24:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:24:40.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:24:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:24:40.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:24:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:24:40.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:24:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:24:40.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:24:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:24:40.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:24:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:24:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:24:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:24:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:24:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:24:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:24:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:24:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:24:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:24:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:24:40 smithi153 ceph-mon[70044]: pgmap v7093: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:41.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:24:40 smithi083 ceph-mon[79930]: pgmap v7093: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:41.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:24:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:24:41.022Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:24:41.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:24:40 smithi083 ceph-mon[77242]: pgmap v7093: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:24:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:24:41] "GET /metrics HTTP/1.1" 200 225215 "" "Prometheus/2.33.4" 2022-09-29T09:24:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:24:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:24:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:24:43.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:24:42 smithi153 ceph-mon[70044]: pgmap v7094: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:43.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:24:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:24:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:24:43.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:24:42 smithi083 ceph-mon[79930]: pgmap v7094: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:43.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:24:42 smithi083 ceph-mon[77242]: pgmap v7094: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:24:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:24:44.401Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:24:45.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:24:44 smithi153 ceph-mon[70044]: pgmap v7095: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:45.424 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:24:45.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:24:44 smithi083 ceph-mon[77242]: pgmap v7095: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:45.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:24:44 smithi083 ceph-mon[79930]: pgmap v7095: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:45.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:24:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:24:45.111Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:24:45.711 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:24:45.712 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 5m ago 3h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:24:45.712 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 3m ago 3h 62.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:24:45.712 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (3h) 5m ago 3h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:24:45.712 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (3h) 3m ago 3h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:24:45.713 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 3m ago 4h 443M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:24:45.713 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 5m ago 4h 714M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:24:45.713 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 5m ago 4h 888M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:24:45.713 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 3m ago 4h 391M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:24:45.713 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 5m ago 4h 519M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:24:45.714 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 5m ago 3h 26.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:24:45.714 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 3m ago 3h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:24:45.714 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 5m ago 4h 641M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:24:45.715 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 5m ago 4h 631M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:24:45.715 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 5m ago 4h 495M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:24:45.715 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 5m ago 4h 615M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:24:45.715 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 3m ago 4h 544M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:24:45.716 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 3m ago 4h 675M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:24:45.716 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 3m ago 4h 602M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:24:45.716 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 3m ago 4h 709M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:24:45.716 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 3m ago 3h 168M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:24:46.066 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:24:46.067 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:24:46.067 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:24:46.067 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:24:46.067 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:24:46.067 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:24:46.068 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:24:46.068 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:24:46.068 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:24:46.068 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:24:46.069 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:24:46.069 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:24:46.069 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:24:46.069 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:24:46.069 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:24:46.070 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:24:46.070 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:24:46.070 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:24:47.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:24:47 smithi153 ceph-mon[70044]: from='client.52318 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:24:47.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:24:47 smithi153 ceph-mon[70044]: from='client.52324 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:24:47.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:24:47 smithi153 ceph-mon[70044]: pgmap v7096: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:47.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:24:47 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4050284292' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:24:47.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:24:47 smithi083 ceph-mon[79930]: from='client.52318 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:24:47.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:24:47 smithi083 ceph-mon[79930]: from='client.52324 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:24:47.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:24:47 smithi083 ceph-mon[79930]: pgmap v7096: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:47.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:24:47 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4050284292' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:24:47.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:24:47 smithi083 ceph-mon[77242]: from='client.52318 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:24:47.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:24:47 smithi083 ceph-mon[77242]: from='client.52324 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:24:47.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:24:47 smithi083 ceph-mon[77242]: pgmap v7096: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:47.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:24:47 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4050284292' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:24:49.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:24:48 smithi153 ceph-mon[70044]: pgmap v7097: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:49.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:24:48 smithi083 ceph-mon[79930]: pgmap v7097: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:49.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:24:48 smithi083 ceph-mon[77242]: pgmap v7097: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:51.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:24:50 smithi153 ceph-mon[70044]: pgmap v7098: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:51.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:24:50 smithi083 ceph-mon[79930]: pgmap v7098: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:51.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:24:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:24:51.023Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:24:51.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:24:50 smithi083 ceph-mon[77242]: pgmap v7098: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:24:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:24:51] "GET /metrics HTTP/1.1" 200 225215 "" "Prometheus/2.33.4" 2022-09-29T09:24:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:24:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:24:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:24:53.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:24:52 smithi153 ceph-mon[70044]: pgmap v7099: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:53.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:24:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:24:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:24:53.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:24:52 smithi083 ceph-mon[77242]: pgmap v7099: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:53.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:24:52 smithi083 ceph-mon[79930]: pgmap v7099: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:24:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:24:54.402Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:24:55.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:24:55 smithi153 ceph-mon[70044]: pgmap v7100: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:55.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:24:55 smithi083 ceph-mon[79930]: pgmap v7100: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:55.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:24:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:24:55.111Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:24:55.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:24:54 smithi083 ceph-mon[77242]: pgmap v7100: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:57.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:24:57 smithi153 ceph-mon[70044]: pgmap v7101: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:57.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:24:57 smithi083 ceph-mon[79930]: pgmap v7101: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:57.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:24:57 smithi083 ceph-mon[77242]: pgmap v7101: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:59.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:24:58 smithi153 ceph-mon[70044]: pgmap v7102: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:59.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:24:58 smithi083 ceph-mon[77242]: pgmap v7102: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:24:59.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:24:58 smithi083 ceph-mon[79930]: pgmap v7102: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:01.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:25:00 smithi153 ceph-mon[70044]: pgmap v7103: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:01.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:25:00 smithi083 ceph-mon[77242]: pgmap v7103: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:01.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:25:00 smithi083 ceph-mon[79930]: pgmap v7103: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:01.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:25:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:25:01.024Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:25:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:25:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:25:01] "GET /metrics HTTP/1.1" 200 225218 "" "Prometheus/2.33.4" 2022-09-29T09:25:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:25:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:25:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:25:03.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:25:03 smithi153 ceph-mon[70044]: pgmap v7104: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:25:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:25:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:25:03.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:25:02 smithi083 ceph-mon[79930]: pgmap v7104: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:03.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:25:02 smithi083 ceph-mon[77242]: pgmap v7104: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:04.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:25:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:25:04.402Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:25:05.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:25:05 smithi153 ceph-mon[70044]: pgmap v7105: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:05.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:25:05 smithi083 ceph-mon[77242]: pgmap v7105: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:05.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:25:05 smithi083 ceph-mon[79930]: pgmap v7105: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:05.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:25:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:25:05.112Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:25:07.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:25:07 smithi153 ceph-mon[70044]: pgmap v7106: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:07.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:25:07 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:25:07.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:25:07 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:25:07.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:25:07 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:25:07.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:25:07 smithi083 ceph-mon[79930]: pgmap v7106: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:07.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:25:07 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:25:07.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:25:07 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:25:07.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:25:07 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:25:07.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:25:07 smithi083 ceph-mon[77242]: pgmap v7106: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:07.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:25:07 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:25:07.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:25:07 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:25:07.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:25:07 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:25:08.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:25:08 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:25:08.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:25:08 smithi083 ceph-mon[77242]: pgmap v7107: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:08.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:25:08 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:25:08.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:25:08 smithi083 ceph-mon[79930]: pgmap v7107: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:08.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:25:08 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:25:08.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:25:08 smithi153 ceph-mon[70044]: pgmap v7107: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:11.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:25:11 smithi153 ceph-mon[70044]: pgmap v7108: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:11.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:25:11 smithi083 ceph-mon[77242]: pgmap v7108: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:11.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:25:11 smithi083 ceph-mon[79930]: pgmap v7108: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:11.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:25:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:25:11.024Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:25:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:25:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:25:11] "GET /metrics HTTP/1.1" 200 225219 "" "Prometheus/2.33.4" 2022-09-29T09:25:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:25:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:25:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:25:13.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:25:13 smithi153 ceph-mon[70044]: pgmap v7109: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:13.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:25:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:25:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:25:13.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:25:13 smithi083 ceph-mon[79930]: pgmap v7109: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:13.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:25:13 smithi083 ceph-mon[77242]: pgmap v7109: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:14.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:25:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:25:14.402Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:25:15.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:25:15 smithi153 ceph-mon[70044]: pgmap v7110: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:15.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:25:15 smithi083 ceph-mon[79930]: pgmap v7110: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:15.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:25:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:25:15.113Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:25:15.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:25:15 smithi083 ceph-mon[77242]: pgmap v7110: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:16.392 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:25:16.681 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:25:16.682 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 6m ago 3h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:25:16.682 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 4m ago 3h 62.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:25:16.682 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 6m ago 4h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:25:16.682 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 4m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:25:16.683 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 4m ago 4h 443M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:25:16.683 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 6m ago 4h 714M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:25:16.683 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 6m ago 4h 888M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:25:16.684 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 4m ago 4h 391M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:25:16.684 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 6m ago 4h 519M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:25:16.684 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (3h) 6m ago 3h 26.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:25:16.684 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (3h) 4m ago 3h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:25:16.684 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 6m ago 4h 641M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:25:16.685 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 6m ago 4h 631M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:25:16.685 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 6m ago 4h 495M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:25:16.685 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 6m ago 4h 615M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:25:16.685 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 4m ago 4h 544M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:25:16.686 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 4m ago 4h 675M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:25:16.686 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 4m ago 4h 602M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:25:16.686 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 4m ago 4h 709M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:25:16.686 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 4m ago 3h 168M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:25:17.037 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:25:17.037 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:25:17.037 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:25:17.037 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:25:17.038 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:25:17.038 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:25:17.038 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:25:17.038 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:25:17.039 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:25:17.039 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:25:17.039 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:25:17.040 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:25:17.040 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:25:17.040 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:25:17.040 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:25:17.041 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:25:17.041 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:25:17.041 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:25:17.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:25:17 smithi153 ceph-mon[70044]: pgmap v7111: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:17.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:25:17 smithi083 ceph-mon[77242]: pgmap v7111: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:17.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:25:17 smithi083 ceph-mon[79930]: pgmap v7111: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:18.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:25:18 smithi153 ceph-mon[70044]: from='client.52336 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:25:18.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:25:18 smithi153 ceph-mon[70044]: from='client.42411 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:25:18.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:25:18 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/591802280' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:25:18.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:25:18 smithi083 ceph-mon[79930]: from='client.52336 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:25:18.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:25:18 smithi083 ceph-mon[79930]: from='client.42411 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:25:18.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:25:18 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/591802280' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:25:18.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:25:18 smithi083 ceph-mon[77242]: from='client.52336 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:25:18.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:25:18 smithi083 ceph-mon[77242]: from='client.42411 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:25:18.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:25:18 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/591802280' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:25:19.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:25:19 smithi153 ceph-mon[70044]: pgmap v7112: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:19.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:25:19 smithi083 ceph-mon[77242]: pgmap v7112: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:19.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:25:19 smithi083 ceph-mon[79930]: pgmap v7112: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:21.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:25:21 smithi153 ceph-mon[70044]: pgmap v7113: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:21.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:25:21 smithi083 ceph-mon[79930]: pgmap v7113: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:21.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:25:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:25:21.024Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:25:21.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:25:21 smithi083 ceph-mon[77242]: pgmap v7113: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:25:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:25:21] "GET /metrics HTTP/1.1" 200 225219 "" "Prometheus/2.33.4" 2022-09-29T09:25:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:25:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:25:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:25:23.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:25:23 smithi153 ceph-mon[70044]: pgmap v7114: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:23.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:25:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:25:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:25:23.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:25:23 smithi083 ceph-mon[79930]: pgmap v7114: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:23.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:25:23 smithi083 ceph-mon[77242]: pgmap v7114: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:25:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:25:24.403Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:25:25.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:25:25 smithi153 ceph-mon[70044]: pgmap v7115: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:25.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:25:25 smithi083 ceph-mon[79930]: pgmap v7115: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:25.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:25:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:25:25.114Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:25:25.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:25:25 smithi083 ceph-mon[77242]: pgmap v7115: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:27.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:25:27 smithi153 ceph-mon[70044]: pgmap v7116: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:27.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:25:27 smithi083 ceph-mon[79930]: pgmap v7116: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:27.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:25:27 smithi083 ceph-mon[77242]: pgmap v7116: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:29.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:25:28 smithi153 ceph-mon[70044]: pgmap v7117: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:29.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:25:28 smithi083 ceph-mon[79930]: pgmap v7117: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:29.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:25:28 smithi083 ceph-mon[77242]: pgmap v7117: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:31.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:25:31 smithi153 ceph-mon[70044]: pgmap v7118: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:31.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:25:31 smithi083 ceph-mon[79930]: pgmap v7118: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:31.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:25:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:25:31.025Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:25:31.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:25:31 smithi083 ceph-mon[77242]: pgmap v7118: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:25:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:25:31] "GET /metrics HTTP/1.1" 200 225212 "" "Prometheus/2.33.4" 2022-09-29T09:25:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:25:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:25:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:25:33.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:25:33 smithi153 ceph-mon[70044]: pgmap v7119: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:33.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:25:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:25:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:25:33.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:25:33 smithi083 ceph-mon[79930]: pgmap v7119: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:33.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:25:33 smithi083 ceph-mon[77242]: pgmap v7119: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:34.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:25:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:25:34.404Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:25:35.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:25:35 smithi153 ceph-mon[70044]: pgmap v7120: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:35.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:25:35 smithi083 ceph-mon[79930]: pgmap v7120: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:35.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:25:35 smithi083 ceph-mon[77242]: pgmap v7120: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:35.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:25:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:25:35.114Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:25:37.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:25:37 smithi153 ceph-mon[70044]: pgmap v7121: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:37.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:25:37 smithi083 ceph-mon[79930]: pgmap v7121: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:37.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:25:37 smithi083 ceph-mon[77242]: pgmap v7121: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:39.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:25:38 smithi153 ceph-mon[70044]: pgmap v7122: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:39.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:25:38 smithi083 ceph-mon[79930]: pgmap v7122: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:39.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:25:38 smithi083 ceph-mon[77242]: pgmap v7122: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:25:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:25:40.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:25:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:25:40.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:25:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:25:40.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:25:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:25:40.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:25:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:25:40.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:25:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:25:40.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:25:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:25:40.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:25:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:25:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:25:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:25:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:25:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:25:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:25:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:25:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:25:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:25:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:25:41 smithi153 ceph-mon[70044]: pgmap v7123: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:41.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:25:41 smithi083 ceph-mon[79930]: pgmap v7123: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:41.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:25:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:25:41.025Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:25:41.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:25:41 smithi083 ceph-mon[77242]: pgmap v7123: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:25:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:25:41] "GET /metrics HTTP/1.1" 200 225216 "" "Prometheus/2.33.4" 2022-09-29T09:25:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:25:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:25:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:25:43.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:25:43 smithi153 ceph-mon[70044]: pgmap v7124: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:43.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:25:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:25:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:25:43.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:25:43 smithi083 ceph-mon[79930]: pgmap v7124: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:43.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:25:43 smithi083 ceph-mon[77242]: pgmap v7124: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:25:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:25:44.405Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:25:45.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:25:45 smithi153 ceph-mon[70044]: pgmap v7125: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:45.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:25:45 smithi083 ceph-mon[79930]: pgmap v7125: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:45.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:25:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:25:45.115Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:25:45.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:25:45 smithi083 ceph-mon[77242]: pgmap v7125: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:47.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:25:47 smithi153 ceph-mon[70044]: pgmap v7126: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:47.361 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:25:47.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:25:47 smithi083 ceph-mon[79930]: pgmap v7126: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:47.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:25:47 smithi083 ceph-mon[77242]: pgmap v7126: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:47.649 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:25:47.649 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3h) 6m ago 3h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:25:47.650 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (3h) 4m ago 3h 62.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:25:47.650 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 6m ago 4h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:25:47.650 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 4m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:25:47.651 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 4m ago 4h 443M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:25:47.651 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 6m ago 4h 714M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:25:47.651 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 6m ago 4h 888M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:25:47.651 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 4m ago 4h 391M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:25:47.651 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 6m ago 4h 519M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:25:47.652 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 6m ago 4h 26.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:25:47.652 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 4m ago 4h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:25:47.652 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 6m ago 4h 641M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:25:47.652 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 6m ago 4h 631M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:25:47.653 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 6m ago 4h 495M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:25:47.653 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 6m ago 4h 615M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:25:47.653 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 4m ago 4h 544M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:25:47.653 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 4m ago 4h 675M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:25:47.654 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 4m ago 4h 602M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:25:47.654 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 4m ago 4h 709M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:25:47.654 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 4m ago 4h 168M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:25:48.006 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:25:48.006 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:25:48.007 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:25:48.007 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:25:48.007 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:25:48.008 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:25:48.008 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:25:48.008 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:25:48.008 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:25:48.008 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:25:48.009 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:25:48.009 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:25:48.009 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:25:48.009 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:25:48.010 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:25:48.010 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:25:48.010 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:25:48.010 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:25:49.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:25:48 smithi153 ceph-mon[70044]: from='client.52354 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:25:49.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:25:48 smithi153 ceph-mon[70044]: from='client.42429 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:25:49.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:25:48 smithi153 ceph-mon[70044]: pgmap v7127: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:49.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:25:48 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3851337483' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:25:49.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:25:48 smithi083 ceph-mon[77242]: from='client.52354 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:25:49.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:25:48 smithi083 ceph-mon[77242]: from='client.42429 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:25:49.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:25:48 smithi083 ceph-mon[77242]: pgmap v7127: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:49.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:25:48 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3851337483' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:25:49.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:25:48 smithi083 ceph-mon[79930]: from='client.52354 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:25:49.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:25:48 smithi083 ceph-mon[79930]: from='client.42429 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:25:49.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:25:48 smithi083 ceph-mon[79930]: pgmap v7127: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:49.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:25:48 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3851337483' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:25:51.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:25:51 smithi153 ceph-mon[70044]: pgmap v7128: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:51.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:25:51 smithi083 ceph-mon[79930]: pgmap v7128: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:51.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:25:51 smithi083 ceph-mon[77242]: pgmap v7128: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:51.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:25:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:25:51.025Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:25:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:25:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:25:51] "GET /metrics HTTP/1.1" 200 225216 "" "Prometheus/2.33.4" 2022-09-29T09:25:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:25:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:25:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:25:53.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:25:53 smithi153 ceph-mon[70044]: pgmap v7129: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:53.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:25:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:25:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:25:53.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:25:53 smithi083 ceph-mon[79930]: pgmap v7129: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:53.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:25:53 smithi083 ceph-mon[77242]: pgmap v7129: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:25:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:25:54.405Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:25:55.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:25:55 smithi153 ceph-mon[70044]: pgmap v7130: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:55.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:25:55 smithi083 ceph-mon[77242]: pgmap v7130: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:55.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:25:55 smithi083 ceph-mon[79930]: pgmap v7130: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:55.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:25:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:25:55.116Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:25:57.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:25:57 smithi153 ceph-mon[70044]: pgmap v7131: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:57.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:25:57 smithi083 ceph-mon[79930]: pgmap v7131: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:57.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:25:57 smithi083 ceph-mon[77242]: pgmap v7131: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:59.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:25:58 smithi153 ceph-mon[70044]: pgmap v7132: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:59.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:25:58 smithi083 ceph-mon[79930]: pgmap v7132: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:25:59.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:25:58 smithi083 ceph-mon[77242]: pgmap v7132: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:01.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:26:01 smithi153 ceph-mon[70044]: pgmap v7133: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:01.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:26:01 smithi083 ceph-mon[79930]: pgmap v7133: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:01.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:26:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:26:01.026Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:26:01.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:26:01 smithi083 ceph-mon[77242]: pgmap v7133: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:26:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:26:01] "GET /metrics HTTP/1.1" 200 225217 "" "Prometheus/2.33.4" 2022-09-29T09:26:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:26:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:26:03.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:26:03 smithi153 ceph-mon[70044]: pgmap v7134: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:26:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:26:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:26:03.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:26:03 smithi083 ceph-mon[79930]: pgmap v7134: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:03.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:26:03 smithi083 ceph-mon[77242]: pgmap v7134: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:04.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:26:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:26:04.405Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:26:05.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:26:05 smithi153 ceph-mon[70044]: pgmap v7135: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:05.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:26:05 smithi083 ceph-mon[77242]: pgmap v7135: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:05.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:26:05 smithi083 ceph-mon[79930]: pgmap v7135: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:05.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:26:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:26:05.117Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:26:07.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:26:07 smithi153 ceph-mon[70044]: pgmap v7136: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:07.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:26:07 smithi083 ceph-mon[79930]: pgmap v7136: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:07.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:26:07 smithi083 ceph-mon[77242]: pgmap v7136: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:08.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:26:08 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:26:08.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:26:08 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:26:08.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:26:08 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:26:08.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:26:08 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:26:08.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:26:08 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:26:08.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:26:08 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:26:08.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:26:08 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:26:08.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:26:08 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:26:08.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:26:08 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:26:08.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:26:08 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:26:08.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:26:08 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:26:08.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:26:08 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:26:09.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:26:09 smithi153 ceph-mon[70044]: pgmap v7137: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:09.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:26:09 smithi083 ceph-mon[79930]: pgmap v7137: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:09.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:26:09 smithi083 ceph-mon[77242]: pgmap v7137: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:11.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:26:11 smithi153 ceph-mon[70044]: pgmap v7138: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:11.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:26:11 smithi083 ceph-mon[77242]: pgmap v7138: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:11.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:26:11 smithi083 ceph-mon[79930]: pgmap v7138: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:11.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:26:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:26:11.027Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:26:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:26:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:26:11] "GET /metrics HTTP/1.1" 200 225220 "" "Prometheus/2.33.4" 2022-09-29T09:26:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:26:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:26:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:26:13.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:26:13 smithi153 ceph-mon[70044]: pgmap v7139: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:13.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:26:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:26:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:26:13.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:26:13 smithi083 ceph-mon[79930]: pgmap v7139: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:13.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:26:13 smithi083 ceph-mon[77242]: pgmap v7139: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:14.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:26:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:26:14.406Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:26:15.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:26:15 smithi153 ceph-mon[70044]: pgmap v7140: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:15.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:26:15 smithi083 ceph-mon[79930]: pgmap v7140: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:15.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:26:15 smithi083 ceph-mon[77242]: pgmap v7140: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:15.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:26:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:26:15.118Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:26:17.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:26:17 smithi153 ceph-mon[70044]: pgmap v7141: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:17.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:26:17 smithi083 ceph-mon[79930]: pgmap v7141: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:17.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:26:17 smithi083 ceph-mon[77242]: pgmap v7141: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:18.332 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:26:18.618 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:26:18.618 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 7m ago 4h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:26:18.618 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 5m ago 4h 62.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:26:18.619 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 7m ago 4h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:26:18.619 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 5m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:26:18.619 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 5m ago 4h 443M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:26:18.619 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 7m ago 4h 714M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:26:18.620 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 7m ago 4h 888M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:26:18.620 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 5m ago 4h 391M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:26:18.620 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 7m ago 4h 519M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:26:18.620 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 7m ago 4h 26.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:26:18.621 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 5m ago 4h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:26:18.621 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 7m ago 4h 641M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:26:18.621 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 7m ago 4h 631M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:26:18.621 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 7m ago 4h 495M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:26:18.621 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 7m ago 4h 615M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:26:18.622 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 5m ago 4h 544M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:26:18.622 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 5m ago 4h 675M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:26:18.622 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 5m ago 4h 602M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:26:18.622 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 5m ago 4h 709M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:26:18.623 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 5m ago 4h 168M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:26:18.978 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:26:18.978 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:26:18.978 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:26:18.978 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:26:18.979 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:26:18.979 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:26:18.979 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:26:18.979 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:26:18.980 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:26:18.980 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:26:18.980 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:26:18.980 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:26:18.981 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:26:18.981 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:26:18.981 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:26:18.981 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:26:18.981 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:26:18.982 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:26:19.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:26:18 smithi153 ceph-mon[70044]: pgmap v7142: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:19.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:26:18 smithi083 ceph-mon[77242]: pgmap v7142: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:19.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:26:18 smithi083 ceph-mon[79930]: pgmap v7142: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:20.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:26:19 smithi153 ceph-mon[70044]: from='client.52372 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:26:20.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:26:19 smithi153 ceph-mon[70044]: from='client.52378 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:26:20.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:26:19 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3806127349' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:26:20.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:26:19 smithi083 ceph-mon[79930]: from='client.52372 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:26:20.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:26:19 smithi083 ceph-mon[79930]: from='client.52378 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:26:20.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:26:19 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3806127349' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:26:20.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:26:19 smithi083 ceph-mon[77242]: from='client.52372 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:26:20.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:26:19 smithi083 ceph-mon[77242]: from='client.52378 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:26:20.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:26:19 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3806127349' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:26:21.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:26:20 smithi083 ceph-mon[79930]: pgmap v7143: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:21.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:26:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:26:21.027Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:26:21.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:26:20 smithi083 ceph-mon[77242]: pgmap v7143: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:21.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:26:20 smithi153 ceph-mon[70044]: pgmap v7143: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:26:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:26:21] "GET /metrics HTTP/1.1" 200 225220 "" "Prometheus/2.33.4" 2022-09-29T09:26:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:26:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:26:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:26:23.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:26:23 smithi153 ceph-mon[70044]: pgmap v7144: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:23.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:26:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:26:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:26:23.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:26:23 smithi083 ceph-mon[77242]: pgmap v7144: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:23.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:26:23 smithi083 ceph-mon[79930]: pgmap v7144: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:26:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:26:24.407Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:26:25.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:26:25 smithi153 ceph-mon[70044]: pgmap v7145: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:25.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:26:25 smithi083 ceph-mon[79930]: pgmap v7145: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:25.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:26:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:26:25.119Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:26:25.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:26:25 smithi083 ceph-mon[77242]: pgmap v7145: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:27.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:26:27 smithi153 ceph-mon[70044]: pgmap v7146: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:27.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:26:27 smithi083 ceph-mon[79930]: pgmap v7146: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:27.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:26:27 smithi083 ceph-mon[77242]: pgmap v7146: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:29.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:26:28 smithi153 ceph-mon[70044]: pgmap v7147: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:29.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:26:28 smithi083 ceph-mon[79930]: pgmap v7147: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:29.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:26:28 smithi083 ceph-mon[77242]: pgmap v7147: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:31.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:26:31 smithi153 ceph-mon[70044]: pgmap v7148: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:31.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:26:31 smithi083 ceph-mon[79930]: pgmap v7148: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:31.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:26:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:26:31.027Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:26:31.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:26:31 smithi083 ceph-mon[77242]: pgmap v7148: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:26:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:26:31] "GET /metrics HTTP/1.1" 200 225219 "" "Prometheus/2.33.4" 2022-09-29T09:26:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:26:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:26:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:26:33.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:26:33 smithi153 ceph-mon[70044]: pgmap v7149: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:33.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:26:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:26:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:26:33.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:26:33 smithi083 ceph-mon[79930]: pgmap v7149: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:33.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:26:33 smithi083 ceph-mon[77242]: pgmap v7149: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:34.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:26:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:26:34.408Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:26:35.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:26:35 smithi153 ceph-mon[70044]: pgmap v7150: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:35.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:26:35 smithi083 ceph-mon[79930]: pgmap v7150: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:35.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:26:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:26:35.119Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:26:35.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:26:35 smithi083 ceph-mon[77242]: pgmap v7150: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:37.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:26:37 smithi153 ceph-mon[70044]: pgmap v7151: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:37.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:26:37 smithi083 ceph-mon[79930]: pgmap v7151: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:37.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:26:37 smithi083 ceph-mon[77242]: pgmap v7151: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:39.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:26:38 smithi153 ceph-mon[70044]: pgmap v7152: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:39.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:26:38 smithi083 ceph-mon[79930]: pgmap v7152: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:39.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:26:38 smithi083 ceph-mon[77242]: pgmap v7152: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:26:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:26:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:26:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:26:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:26:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:26:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:26:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:26:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:26:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:26:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:26:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:26:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:26:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:26:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:26:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:26:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:26:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:26:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:26:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:26:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:26:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:26:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:26:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:26:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:26:40 smithi153 ceph-mon[70044]: pgmap v7153: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:41.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:26:40 smithi083 ceph-mon[79930]: pgmap v7153: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:41.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:26:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:26:41.028Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:26:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:26:40 smithi083 ceph-mon[77242]: pgmap v7153: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:26:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:26:41] "GET /metrics HTTP/1.1" 200 225218 "" "Prometheus/2.33.4" 2022-09-29T09:26:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:26:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:26:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:26:43.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:26:43 smithi153 ceph-mon[70044]: pgmap v7154: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:43.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:26:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:26:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:26:43.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:26:43 smithi083 ceph-mon[79930]: pgmap v7154: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:43.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:26:43 smithi083 ceph-mon[77242]: pgmap v7154: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:26:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:26:44.409Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:26:45.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:26:45 smithi153 ceph-mon[70044]: pgmap v7155: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:45.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:26:45 smithi083 ceph-mon[79930]: pgmap v7155: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:45.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:26:45 smithi083 ceph-mon[77242]: pgmap v7155: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:45.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:26:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:26:45.120Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:26:47.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:26:47 smithi153 ceph-mon[70044]: pgmap v7156: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:47.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:26:47 smithi083 ceph-mon[77242]: pgmap v7156: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:47.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:26:47 smithi083 ceph-mon[79930]: pgmap v7156: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:49.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:26:48 smithi083 ceph-mon[79930]: pgmap v7157: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:49.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:26:48 smithi083 ceph-mon[77242]: pgmap v7157: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:49.306 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:26:49.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:26:48 smithi153 ceph-mon[70044]: pgmap v7157: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:49.593 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:26:49.593 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 7m ago 4h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:26:49.594 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 5m ago 4h 62.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:26:49.594 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 7m ago 4h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:26:49.594 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 5m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:26:49.595 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 5m ago 4h 443M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:26:49.595 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 7m ago 4h 714M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:26:49.595 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 7m ago 4h 888M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:26:49.596 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 5m ago 4h 391M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:26:49.596 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 7m ago 4h 519M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:26:49.596 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 7m ago 4h 26.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:26:49.596 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 5m ago 4h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:26:49.597 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 7m ago 4h 641M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:26:49.597 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 7m ago 4h 631M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:26:49.597 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 7m ago 4h 495M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:26:49.597 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 7m ago 4h 615M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:26:49.598 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 5m ago 4h 544M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:26:49.598 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 5m ago 4h 675M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:26:49.598 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 5m ago 4h 602M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:26:49.598 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 5m ago 4h 709M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:26:49.598 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 5m ago 4h 168M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:26:49.947 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:26:49.947 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:26:49.948 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:26:49.948 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:26:49.948 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:26:49.948 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:26:49.949 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:26:49.949 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:26:49.949 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:26:49.949 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:26:49.949 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:26:49.950 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:26:49.950 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:26:49.950 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:26:49.950 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:26:49.951 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:26:49.951 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:26:49.951 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:26:50.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:26:49 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3189891974' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:26:50.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:26:49 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3189891974' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:26:50.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:26:49 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3189891974' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:26:51.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:26:51 smithi153 ceph-mon[70044]: from='client.42459 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:26:51.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:26:51 smithi153 ceph-mon[70044]: from='client.52396 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:26:51.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:26:51 smithi153 ceph-mon[70044]: pgmap v7158: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:51.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:26:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:26:51.028Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:26:51.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:26:51 smithi083 ceph-mon[77242]: from='client.42459 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:26:51.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:26:51 smithi083 ceph-mon[77242]: from='client.52396 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:26:51.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:26:51 smithi083 ceph-mon[77242]: pgmap v7158: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:51.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:26:51 smithi083 ceph-mon[79930]: from='client.42459 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:26:51.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:26:51 smithi083 ceph-mon[79930]: from='client.52396 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:26:51.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:26:51 smithi083 ceph-mon[79930]: pgmap v7158: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:26:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:26:51] "GET /metrics HTTP/1.1" 200 225218 "" "Prometheus/2.33.4" 2022-09-29T09:26:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:26:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:26:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:26:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:26:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:26:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:26:53.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:26:53 smithi153 ceph-mon[70044]: pgmap v7159: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:53.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:26:53 smithi083 ceph-mon[79930]: pgmap v7159: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:53.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:26:53 smithi083 ceph-mon[77242]: pgmap v7159: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:26:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:26:54.409Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:26:55.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:26:55 smithi153 ceph-mon[70044]: pgmap v7160: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:55.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:26:55 smithi083 ceph-mon[79930]: pgmap v7160: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:55.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:26:55 smithi083 ceph-mon[77242]: pgmap v7160: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:55.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:26:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:26:55.121Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:26:57.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:26:57 smithi153 ceph-mon[70044]: pgmap v7161: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:57.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:26:57 smithi083 ceph-mon[79930]: pgmap v7161: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:57.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:26:57 smithi083 ceph-mon[77242]: pgmap v7161: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:59.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:26:58 smithi083 ceph-mon[79930]: pgmap v7162: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:59.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:26:58 smithi083 ceph-mon[77242]: pgmap v7162: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:26:59.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:26:58 smithi153 ceph-mon[70044]: pgmap v7162: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:01.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:27:01 smithi153 ceph-mon[70044]: pgmap v7163: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:01.470 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:27:01 smithi083 ceph-mon[79930]: pgmap v7163: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:01.471 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:27:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:27:01.029Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:27:01.471 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:27:01 smithi083 ceph-mon[77242]: pgmap v7163: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:27:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:27:01] "GET /metrics HTTP/1.1" 200 225221 "" "Prometheus/2.33.4" 2022-09-29T09:27:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:27:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:27:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:27:03.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:27:03 smithi153 ceph-mon[70044]: pgmap v7164: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:27:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:27:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:27:03.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:27:03 smithi083 ceph-mon[79930]: pgmap v7164: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:03.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:27:03 smithi083 ceph-mon[77242]: pgmap v7164: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:04.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:27:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:27:04.410Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:27:05.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:27:05 smithi153 ceph-mon[70044]: pgmap v7165: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:05.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:27:05 smithi083 ceph-mon[79930]: pgmap v7165: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:05.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:27:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:27:05.121Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:27:05.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:27:05 smithi083 ceph-mon[77242]: pgmap v7165: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:07.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:27:07 smithi153 ceph-mon[70044]: pgmap v7166: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:07.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:27:07 smithi083 ceph-mon[79930]: pgmap v7166: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:07.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:27:07 smithi083 ceph-mon[77242]: pgmap v7166: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:08.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:27:08 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:27:08.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:27:08 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:27:08.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:27:08 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:27:08.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:27:08 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:27:08.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:27:08 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:27:08.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:27:08 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:27:08.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:27:08 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:27:08.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:27:08 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:27:08.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:27:08 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:27:09.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:27:09 smithi153 ceph-mon[70044]: pgmap v7167: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:09.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:27:09 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:27:09.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:27:09 smithi083 ceph-mon[79930]: pgmap v7167: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:09.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:27:09 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:27:09.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:27:09 smithi083 ceph-mon[77242]: pgmap v7167: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:09.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:27:09 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:27:11.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:27:11 smithi153 ceph-mon[70044]: pgmap v7168: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:11.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:27:11 smithi083 ceph-mon[79930]: pgmap v7168: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:11.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:27:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:27:11.029Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:27:11.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:27:11 smithi083 ceph-mon[77242]: pgmap v7168: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:27:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:27:11] "GET /metrics HTTP/1.1" 200 225218 "" "Prometheus/2.33.4" 2022-09-29T09:27:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:27:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:27:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:27:13.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:27:13 smithi153 ceph-mon[70044]: pgmap v7169: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:13.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:27:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:27:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:27:13.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:27:13 smithi083 ceph-mon[79930]: pgmap v7169: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:13.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:27:13 smithi083 ceph-mon[77242]: pgmap v7169: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:14.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:27:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:27:14.412Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:27:15.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:27:15 smithi083 ceph-mon[79930]: pgmap v7170: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:15.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:27:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:27:15.122Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:27:15.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:27:15 smithi083 ceph-mon[77242]: pgmap v7170: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:15.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:27:15 smithi153 ceph-mon[70044]: pgmap v7170: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:17.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:27:17 smithi083 ceph-mon[77242]: pgmap v7171: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:17.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:27:17 smithi083 ceph-mon[79930]: pgmap v7171: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:17.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:27:17 smithi153 ceph-mon[70044]: pgmap v7171: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:19.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:27:18 smithi083 ceph-mon[77242]: pgmap v7172: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:19.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:27:18 smithi083 ceph-mon[79930]: pgmap v7172: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:19.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:27:18 smithi153 ceph-mon[70044]: pgmap v7172: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:20.274 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:27:20.561 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:27:20.561 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 8m ago 4h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:27:20.561 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 6m ago 4h 62.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:27:20.562 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 8m ago 4h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:27:20.562 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 6m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:27:20.562 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 6m ago 4h 443M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:27:20.562 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 8m ago 4h 714M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:27:20.563 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 8m ago 4h 888M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:27:20.563 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 6m ago 4h 391M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:27:20.563 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 8m ago 4h 519M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:27:20.563 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 8m ago 4h 26.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:27:20.563 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 6m ago 4h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:27:20.564 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 8m ago 4h 641M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:27:20.564 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 8m ago 4h 631M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:27:20.564 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 8m ago 4h 495M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:27:20.564 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 8m ago 4h 615M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:27:20.564 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 6m ago 4h 544M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:27:20.565 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 6m ago 4h 675M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:27:20.565 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 6m ago 4h 602M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:27:20.565 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 6m ago 4h 709M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:27:20.565 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 6m ago 4h 168M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:27:20.918 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:27:20.918 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:27:20.919 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:27:20.919 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:27:20.919 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:27:20.919 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:27:20.919 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:27:20.920 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:27:20.920 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:27:20.920 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:27:20.920 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:27:20.920 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:27:20.921 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:27:20.921 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:27:20.921 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:27:20.921 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:27:20.921 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:27:20.922 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:27:21.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:27:21 smithi153 ceph-mon[70044]: pgmap v7173: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:21.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:27:21 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2376934211' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:27:21.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:27:21 smithi083 ceph-mon[77242]: pgmap v7173: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:21.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:27:21 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2376934211' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:27:21.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:27:21 smithi083 ceph-mon[79930]: pgmap v7173: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:21.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:27:21 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2376934211' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:27:21.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:27:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:27:21.029Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:27:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:27:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:27:21] "GET /metrics HTTP/1.1" 200 225218 "" "Prometheus/2.33.4" 2022-09-29T09:27:22.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:27:22 smithi153 ceph-mon[70044]: from='client.42477 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:27:22.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:27:22 smithi153 ceph-mon[70044]: from='client.52414 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:27:22.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:27:22 smithi083 ceph-mon[77242]: from='client.42477 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:27:22.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:27:22 smithi083 ceph-mon[77242]: from='client.52414 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:27:22.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:27:22 smithi083 ceph-mon[79930]: from='client.42477 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:27:22.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:27:22 smithi083 ceph-mon[79930]: from='client.52414 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:27:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:27:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:27:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:27:23.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:27:23 smithi153 ceph-mon[70044]: pgmap v7174: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:23.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:27:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:27:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:27:23.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:27:23 smithi083 ceph-mon[79930]: pgmap v7174: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:23.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:27:23 smithi083 ceph-mon[77242]: pgmap v7174: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:27:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:27:24.412Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:27:25.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:27:25 smithi153 ceph-mon[70044]: pgmap v7175: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:25.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:27:25 smithi083 ceph-mon[79930]: pgmap v7175: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:25.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:27:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:27:25.122Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:27:25.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:27:25 smithi083 ceph-mon[77242]: pgmap v7175: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:27.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:27:27 smithi153 ceph-mon[70044]: pgmap v7176: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:27.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:27:27 smithi083 ceph-mon[79930]: pgmap v7176: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:27.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:27:27 smithi083 ceph-mon[77242]: pgmap v7176: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:29.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:27:28 smithi083 ceph-mon[79930]: pgmap v7177: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:29.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:27:28 smithi083 ceph-mon[77242]: pgmap v7177: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:29.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:27:28 smithi153 ceph-mon[70044]: pgmap v7177: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:31.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:27:31 smithi153 ceph-mon[70044]: pgmap v7178: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:31.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:27:31 smithi083 ceph-mon[79930]: pgmap v7178: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:31.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:27:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:27:31.030Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:27:31.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:27:31 smithi083 ceph-mon[77242]: pgmap v7178: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:27:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:27:31] "GET /metrics HTTP/1.1" 200 225220 "" "Prometheus/2.33.4" 2022-09-29T09:27:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:27:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:27:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:27:33.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:27:33 smithi153 ceph-mon[70044]: pgmap v7179: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:33.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:27:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:27:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:27:33.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:27:33 smithi083 ceph-mon[79930]: pgmap v7179: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:33.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:27:33 smithi083 ceph-mon[77242]: pgmap v7179: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:34.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:27:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:27:34.413Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:27:35.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:27:35 smithi153 ceph-mon[70044]: pgmap v7180: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:35.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:27:35 smithi083 ceph-mon[79930]: pgmap v7180: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:35.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:27:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:27:35.123Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:27:35.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:27:35 smithi083 ceph-mon[77242]: pgmap v7180: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:37.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:27:37 smithi083 ceph-mon[79930]: pgmap v7181: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:37.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:27:37 smithi083 ceph-mon[77242]: pgmap v7181: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:37.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:27:37 smithi153 ceph-mon[70044]: pgmap v7181: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:39.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:27:38 smithi083 ceph-mon[79930]: pgmap v7182: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:39.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:27:38 smithi083 ceph-mon[77242]: pgmap v7182: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:39.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:27:38 smithi153 ceph-mon[70044]: pgmap v7182: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:27:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:27:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:27:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:27:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:27:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:27:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:27:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:27:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:27:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:27:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:27:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:27:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:27:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:27:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:27:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:27:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:27:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:27:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:27:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:27:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:27:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:27:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:27:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:27:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:27:40 smithi153 ceph-mon[70044]: pgmap v7183: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:41.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:27:40 smithi083 ceph-mon[79930]: pgmap v7183: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:41.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:27:40 smithi083 ceph-mon[77242]: pgmap v7183: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:41.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:27:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:27:41.031Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:27:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:27:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:27:41] "GET /metrics HTTP/1.1" 200 225222 "" "Prometheus/2.33.4" 2022-09-29T09:27:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:27:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:27:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:27:43.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:27:43 smithi153 ceph-mon[70044]: pgmap v7184: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:43.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:27:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:27:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:27:43.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:27:43 smithi083 ceph-mon[79930]: pgmap v7184: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:43.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:27:43 smithi083 ceph-mon[77242]: pgmap v7184: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:27:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:27:44.414Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:27:45.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:27:45 smithi153 ceph-mon[70044]: pgmap v7185: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:45.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:27:45 smithi083 ceph-mon[79930]: pgmap v7185: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:45.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:27:45 smithi083 ceph-mon[77242]: pgmap v7185: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:45.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:27:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:27:45.123Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:27:47.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:27:47 smithi083 ceph-mon[79930]: pgmap v7186: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:47.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:27:47 smithi083 ceph-mon[77242]: pgmap v7186: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:47.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:27:47 smithi153 ceph-mon[70044]: pgmap v7186: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:49.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:27:48 smithi083 ceph-mon[77242]: pgmap v7187: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:49.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:27:48 smithi083 ceph-mon[79930]: pgmap v7187: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:49.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:27:48 smithi153 ceph-mon[70044]: pgmap v7187: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:51.248 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:27:51.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:27:51 smithi153 ceph-mon[70044]: pgmap v7188: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:51.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:27:51 smithi083 ceph-mon[79930]: pgmap v7188: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:51.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:27:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:27:51.032Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:27:51.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:27:51 smithi083 ceph-mon[77242]: pgmap v7188: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:51.537 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:27:51.538 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 8m ago 4h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:27:51.538 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 6m ago 4h 62.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:27:51.538 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 8m ago 4h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:27:51.539 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 6m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:27:51.539 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 6m ago 4h 443M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:27:51.539 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 8m ago 4h 714M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:27:51.539 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 8m ago 4h 888M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:27:51.540 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 6m ago 4h 391M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:27:51.540 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 8m ago 4h 519M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:27:51.540 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 8m ago 4h 26.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:27:51.540 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 6m ago 4h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:27:51.540 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 8m ago 4h 641M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:27:51.541 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 8m ago 4h 631M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:27:51.541 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 8m ago 4h 495M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:27:51.541 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 8m ago 4h 615M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:27:51.541 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 6m ago 4h 544M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:27:51.542 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 6m ago 4h 675M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:27:51.542 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 6m ago 4h 602M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:27:51.542 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 6m ago 4h 709M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:27:51.542 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 6m ago 4h 168M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:27:51.892 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:27:51.892 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:27:51.893 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:27:51.893 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:27:51.893 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:27:51.893 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:27:51.894 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:27:51.894 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:27:51.894 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:27:51.894 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:27:51.895 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:27:51.895 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:27:51.895 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:27:51.895 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:27:51.896 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:27:51.896 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:27:51.896 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:27:51.896 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:27:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:27:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:27:51] "GET /metrics HTTP/1.1" 200 225222 "" "Prometheus/2.33.4" 2022-09-29T09:27:52.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:27:52 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3104669122' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:27:52.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:27:52 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3104669122' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:27:52.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:27:52 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3104669122' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:27:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:27:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:27:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:27:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:27:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:27:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:27:53.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:27:53 smithi153 ceph-mon[70044]: from='client.52426 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:27:53.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:27:53 smithi153 ceph-mon[70044]: from='client.52432 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:27:53.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:27:53 smithi153 ceph-mon[70044]: pgmap v7189: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:53.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:27:53 smithi083 ceph-mon[79930]: from='client.52426 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:27:53.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:27:53 smithi083 ceph-mon[79930]: from='client.52432 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:27:53.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:27:53 smithi083 ceph-mon[79930]: pgmap v7189: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:53.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:27:53 smithi083 ceph-mon[77242]: from='client.52426 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:27:53.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:27:53 smithi083 ceph-mon[77242]: from='client.52432 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:27:53.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:27:53 smithi083 ceph-mon[77242]: pgmap v7189: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:27:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:27:54.414Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:27:55.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:27:55 smithi083 ceph-mon[79930]: pgmap v7190: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:55.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:27:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:27:55.124Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:27:55.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:27:55 smithi083 ceph-mon[77242]: pgmap v7190: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:55.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:27:55 smithi153 ceph-mon[70044]: pgmap v7190: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:57.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:27:57 smithi083 ceph-mon[79930]: pgmap v7191: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:57.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:27:57 smithi083 ceph-mon[77242]: pgmap v7191: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:57.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:27:57 smithi153 ceph-mon[70044]: pgmap v7191: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:59.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:27:58 smithi083 ceph-mon[79930]: pgmap v7192: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:59.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:27:58 smithi083 ceph-mon[77242]: pgmap v7192: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:27:59.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:27:58 smithi153 ceph-mon[70044]: pgmap v7192: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:01.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:28:01 smithi153 ceph-mon[70044]: pgmap v7193: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:01.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:28:01 smithi083 ceph-mon[79930]: pgmap v7193: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:01.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:28:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:28:01.032Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:28:01.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:28:01 smithi083 ceph-mon[77242]: pgmap v7193: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:28:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:28:01] "GET /metrics HTTP/1.1" 200 225221 "" "Prometheus/2.33.4" 2022-09-29T09:28:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:28:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:28:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:28:03.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:28:03 smithi153 ceph-mon[70044]: pgmap v7194: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:28:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:28:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:28:03.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:28:03 smithi083 ceph-mon[77242]: pgmap v7194: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:03.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:28:03 smithi083 ceph-mon[79930]: pgmap v7194: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:04.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:28:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:28:04.415Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:28:05.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:28:05 smithi083 ceph-mon[77242]: pgmap v7195: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:05.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:28:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:28:05.125Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:28:05.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:28:05 smithi083 ceph-mon[79930]: pgmap v7195: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:05.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:28:05 smithi153 ceph-mon[70044]: pgmap v7195: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:07.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:28:07 smithi083 ceph-mon[79930]: pgmap v7196: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:07.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:28:07 smithi083 ceph-mon[77242]: pgmap v7196: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:07.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:28:07 smithi153 ceph-mon[70044]: pgmap v7196: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:09.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:28:08 smithi083 ceph-mon[79930]: pgmap v7197: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:09.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:28:08 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:28:09.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:28:08 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:28:09.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:28:08 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:28:09.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:28:08 smithi083 ceph-mon[77242]: pgmap v7197: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:09.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:28:08 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:28:09.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:28:08 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:28:09.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:28:08 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:28:09.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:28:08 smithi153 ceph-mon[70044]: pgmap v7197: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:09.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:28:08 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:28:09.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:28:08 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:28:09.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:28:08 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:28:10.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:28:10 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:28:10.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:28:10 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:28:10.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:28:10 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:28:11.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:28:11 smithi153 ceph-mon[70044]: pgmap v7198: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:11.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:28:11 smithi083 ceph-mon[77242]: pgmap v7198: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:11.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:28:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:28:11.033Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:28:11.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:28:11 smithi083 ceph-mon[79930]: pgmap v7198: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:28:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:28:11] "GET /metrics HTTP/1.1" 200 225218 "" "Prometheus/2.33.4" 2022-09-29T09:28:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:28:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:28:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:28:13.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:28:13 smithi153 ceph-mon[70044]: pgmap v7199: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:13.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:28:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:28:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:28:13.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:28:13 smithi083 ceph-mon[79930]: pgmap v7199: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:13.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:28:13 smithi083 ceph-mon[77242]: pgmap v7199: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:14.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:28:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:28:14.416Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:28:15.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:28:15 smithi153 ceph-mon[70044]: pgmap v7200: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:15.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:28:15 smithi083 ceph-mon[79930]: pgmap v7200: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:15.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:28:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:28:15.126Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:28:15.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:28:15 smithi083 ceph-mon[77242]: pgmap v7200: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:17.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:28:17 smithi153 ceph-mon[70044]: pgmap v7201: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:17.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:28:17 smithi083 ceph-mon[79930]: pgmap v7201: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:17.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:28:17 smithi083 ceph-mon[77242]: pgmap v7201: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:19.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:28:18 smithi083 ceph-mon[79930]: pgmap v7202: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:19.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:28:18 smithi083 ceph-mon[77242]: pgmap v7202: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:19.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:28:18 smithi153 ceph-mon[70044]: pgmap v7202: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:21.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:28:21 smithi083 ceph-mon[77242]: pgmap v7203: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:21.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:28:21 smithi083 ceph-mon[79930]: pgmap v7203: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:21.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:28:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:28:21.034Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:28:21.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:28:21 smithi153 ceph-mon[70044]: pgmap v7203: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:28:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:28:21] "GET /metrics HTTP/1.1" 200 225218 "" "Prometheus/2.33.4" 2022-09-29T09:28:22.222 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:28:22.513 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:28:22.514 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 9m ago 4h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:28:22.514 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 7m ago 4h 62.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:28:22.514 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 9m ago 4h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:28:22.514 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 7m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:28:22.515 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 7m ago 4h 443M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:28:22.515 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (3h) 9m ago 4h 714M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:28:22.515 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 9m ago 4h 888M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:28:22.515 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 7m ago 4h 391M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:28:22.516 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 9m ago 4h 519M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:28:22.516 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 9m ago 4h 26.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:28:22.516 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 7m ago 4h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:28:22.516 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 9m ago 4h 641M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:28:22.516 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 9m ago 4h 631M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:28:22.517 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 9m ago 4h 495M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:28:22.517 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 9m ago 4h 615M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:28:22.517 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 7m ago 4h 544M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:28:22.517 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 7m ago 4h 675M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:28:22.517 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 7m ago 4h 602M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:28:22.518 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 7m ago 4h 709M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:28:22.518 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 7m ago 4h 168M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:28:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:28:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:28:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:28:22.921 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:28:22.922 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:28:22.922 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:28:22.922 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:28:22.923 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:28:22.923 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:28:22.923 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:28:22.923 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:28:22.924 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:28:22.924 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:28:22.924 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:28:22.924 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:28:22.925 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:28:22.925 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:28:22.925 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:28:22.925 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:28:22.925 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:28:22.926 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:28:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:28:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:28:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:28:23.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:28:23 smithi153 ceph-mon[70044]: pgmap v7204: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:23.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:28:23 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1354398403' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:28:23.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:28:23 smithi083 ceph-mon[77242]: pgmap v7204: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:23.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:28:23 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1354398403' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:28:23.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:28:23 smithi083 ceph-mon[79930]: pgmap v7204: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:23.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:28:23 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1354398403' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:28:24.419 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:28:24 smithi083 ceph-mon[79930]: from='client.42513 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:28:24.419 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:28:24 smithi083 ceph-mon[79930]: from='client.52450 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:28:24.420 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:28:24 smithi083 ceph-mon[77242]: from='client.42513 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:28:24.420 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:28:24 smithi083 ceph-mon[77242]: from='client.52450 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:28:24.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:28:24 smithi153 ceph-mon[70044]: from='client.42513 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:28:24.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:28:24 smithi153 ceph-mon[70044]: from='client.52450 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:28:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:28:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:28:24.416Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:28:25.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:28:25 smithi083 ceph-mon[77242]: pgmap v7205: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:25.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:28:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:28:25.126Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:28:25.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:28:25 smithi083 ceph-mon[79930]: pgmap v7205: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:25.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:28:25 smithi153 ceph-mon[70044]: pgmap v7205: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:27.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:28:27 smithi083 ceph-mon[77242]: pgmap v7206: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:27.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:28:27 smithi083 ceph-mon[79930]: pgmap v7206: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:27.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:28:27 smithi153 ceph-mon[70044]: pgmap v7206: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:29.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:28:28 smithi083 ceph-mon[79930]: pgmap v7207: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:29.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:28:28 smithi083 ceph-mon[77242]: pgmap v7207: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:29.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:28:28 smithi153 ceph-mon[70044]: pgmap v7207: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:31.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:28:31 smithi083 ceph-mon[79930]: pgmap v7208: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:31.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:28:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:28:31.035Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:28:31.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:28:31 smithi083 ceph-mon[77242]: pgmap v7208: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:31.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:28:31 smithi153 ceph-mon[70044]: pgmap v7208: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:28:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:28:31] "GET /metrics HTTP/1.1" 200 225216 "" "Prometheus/2.33.4" 2022-09-29T09:28:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:28:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:28:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:28:33.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:28:33 smithi153 ceph-mon[70044]: pgmap v7209: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:33.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:28:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:28:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:28:33.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:28:33 smithi083 ceph-mon[77242]: pgmap v7209: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:33.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:28:33 smithi083 ceph-mon[79930]: pgmap v7209: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:34.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:28:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:28:34.417Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:28:35.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:28:35 smithi083 ceph-mon[77242]: pgmap v7210: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:35.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:28:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:28:35.127Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:28:35.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:28:35 smithi083 ceph-mon[79930]: pgmap v7210: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:35.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:28:35 smithi153 ceph-mon[70044]: pgmap v7210: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:37.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:28:37 smithi083 ceph-mon[77242]: pgmap v7211: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:37.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:28:37 smithi083 ceph-mon[79930]: pgmap v7211: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:37.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:28:37 smithi153 ceph-mon[70044]: pgmap v7211: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:39.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:28:38 smithi083 ceph-mon[79930]: pgmap v7212: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:39.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:28:38 smithi083 ceph-mon[77242]: pgmap v7212: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:39.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:28:38 smithi153 ceph-mon[70044]: pgmap v7212: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:28:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:28:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:28:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:28:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:28:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:28:40.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:28:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:28:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:28:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:28:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:28:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:28:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:28:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:28:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:28:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:28:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:28:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:28:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:28:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:28:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:28:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:28:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:28:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:28:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:28:40 smithi153 ceph-mon[70044]: pgmap v7213: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:41.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:28:40 smithi083 ceph-mon[79930]: pgmap v7213: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:41.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:28:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:28:41.035Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:28:41.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:28:40 smithi083 ceph-mon[77242]: pgmap v7213: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:28:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:28:41] "GET /metrics HTTP/1.1" 200 225219 "" "Prometheus/2.33.4" 2022-09-29T09:28:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:28:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:28:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:28:43.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:28:43 smithi153 ceph-mon[70044]: pgmap v7214: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:43.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:28:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:28:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:28:43.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:28:43 smithi083 ceph-mon[79930]: pgmap v7214: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:43.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:28:43 smithi083 ceph-mon[77242]: pgmap v7214: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:28:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:28:44.418Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:28:45.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:28:45 smithi083 ceph-mon[79930]: pgmap v7215: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:45.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:28:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:28:45.128Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:28:45.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:28:45 smithi083 ceph-mon[77242]: pgmap v7215: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:45.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:28:45 smithi153 ceph-mon[70044]: pgmap v7215: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:47.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:28:47 smithi083 ceph-mon[77242]: pgmap v7216: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:47.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:28:47 smithi083 ceph-mon[79930]: pgmap v7216: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:47.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:28:47 smithi153 ceph-mon[70044]: pgmap v7216: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:49.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:28:48 smithi083 ceph-mon[79930]: pgmap v7217: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:49.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:28:48 smithi083 ceph-mon[77242]: pgmap v7217: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:49.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:28:48 smithi153 ceph-mon[70044]: pgmap v7217: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:51.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:28:51 smithi083 ceph-mon[79930]: pgmap v7218: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:51.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:28:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:28:51.035Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:28:51.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:28:51 smithi083 ceph-mon[77242]: pgmap v7218: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:51.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:28:51 smithi153 ceph-mon[70044]: pgmap v7218: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:51.966 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:28:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:28:51] "GET /metrics HTTP/1.1" 200 225219 "" "Prometheus/2.33.4" 2022-09-29T09:28:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:28:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:28:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:28:53.194 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:28:53.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:28:53 smithi153 ceph-mon[70044]: pgmap v7219: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:53.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:28:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:28:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:28:53.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:28:53 smithi083 ceph-mon[77242]: pgmap v7219: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:53.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:28:53 smithi083 ceph-mon[79930]: pgmap v7219: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:53.494 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:28:53.494 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 9m ago 4h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:28:53.495 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 7m ago 4h 62.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:28:53.495 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 9m ago 4h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:28:53.495 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 7m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:28:53.496 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (3h) 7m ago 4h 443M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:28:53.496 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 9m ago 4h 714M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:28:53.497 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 9m ago 4h 888M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:28:53.497 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 7m ago 4h 391M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:28:53.497 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 9m ago 4h 519M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:28:53.566 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 9m ago 4h 26.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:28:53.566 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 7m ago 4h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:28:53.567 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 9m ago 4h 641M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:28:53.567 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 9m ago 4h 631M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:28:53.567 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 9m ago 4h 495M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:28:53.567 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 9m ago 4h 615M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:28:53.568 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 7m ago 4h 544M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:28:53.568 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 7m ago 4h 675M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:28:53.568 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 7m ago 4h 602M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:28:53.568 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 7m ago 4h 709M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:28:53.568 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (3h) 7m ago 4h 168M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:28:53.856 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:28:53.857 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:28:53.857 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:28:53.857 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:28:53.858 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:28:53.858 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:28:53.858 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:28:53.858 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:28:53.859 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:28:53.859 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:28:53.859 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:28:53.859 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:28:53.859 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:28:53.860 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:28:53.860 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:28:53.860 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:28:53.860 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:28:53.861 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:28:54.419 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:28:54 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1731543041' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:28:54.420 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:28:54 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1731543041' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:28:54.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:28:54 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1731543041' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:28:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:28:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:28:54.417Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:28:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:28:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:28:55.129Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:28:55.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:28:55 smithi083 ceph-mon[79930]: from='client.52462 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:28:55.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:28:55 smithi083 ceph-mon[79930]: from='client.52468 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:28:55.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:28:55 smithi083 ceph-mon[79930]: pgmap v7220: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:55.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:28:55 smithi083 ceph-mon[77242]: from='client.52462 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:28:55.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:28:55 smithi083 ceph-mon[77242]: from='client.52468 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:28:55.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:28:55 smithi083 ceph-mon[77242]: pgmap v7220: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:55.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:28:55 smithi153 ceph-mon[70044]: from='client.52462 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:28:55.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:28:55 smithi153 ceph-mon[70044]: from='client.52468 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:28:55.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:28:55 smithi153 ceph-mon[70044]: pgmap v7220: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:57.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:28:57 smithi083 ceph-mon[79930]: pgmap v7221: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:57.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:28:57 smithi083 ceph-mon[77242]: pgmap v7221: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:57.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:28:57 smithi153 ceph-mon[70044]: pgmap v7221: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:59.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:28:58 smithi083 ceph-mon[79930]: pgmap v7222: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:59.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:28:58 smithi083 ceph-mon[77242]: pgmap v7222: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:28:59.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:28:58 smithi153 ceph-mon[70044]: pgmap v7222: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:01.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:29:01 smithi083 ceph-mon[79930]: pgmap v7223: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:01.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:29:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:29:01.036Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:29:01.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:29:01 smithi083 ceph-mon[77242]: pgmap v7223: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:01.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:29:01 smithi153 ceph-mon[70044]: pgmap v7223: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:29:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:29:01] "GET /metrics HTTP/1.1" 200 225211 "" "Prometheus/2.33.4" 2022-09-29T09:29:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:29:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:29:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:29:03.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:29:03 smithi153 ceph-mon[70044]: pgmap v7224: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:29:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:29:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:29:03.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:29:03 smithi083 ceph-mon[79930]: pgmap v7224: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:03.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:29:03 smithi083 ceph-mon[77242]: pgmap v7224: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:04.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:29:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:29:04.419Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:29:05.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:29:05 smithi083 ceph-mon[79930]: pgmap v7225: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:05.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:29:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:29:05.129Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:29:05.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:29:05 smithi083 ceph-mon[77242]: pgmap v7225: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:05.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:29:05 smithi153 ceph-mon[70044]: pgmap v7225: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:07.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:29:07 smithi083 ceph-mon[79930]: pgmap v7226: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:07.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:29:07 smithi083 ceph-mon[77242]: pgmap v7226: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:07.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:29:07 smithi153 ceph-mon[70044]: pgmap v7226: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:09.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:29:08 smithi083 ceph-mon[79930]: pgmap v7227: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:09.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:29:08 smithi083 ceph-mon[77242]: pgmap v7227: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:09.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:29:08 smithi153 ceph-mon[70044]: pgmap v7227: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:10.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:29:09 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:29:10.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:29:09 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:29:10.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:29:09 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:29:10.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:29:09 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:29:10.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:29:09 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:29:10.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:29:09 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:29:10.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:29:09 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:29:10.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:29:09 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:29:10.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:29:09 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:29:10.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:29:09 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:29:10.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:29:09 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:29:10.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:29:09 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:29:10.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:29:09 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:29:10.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:29:09 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:29:10.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:29:09 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:29:10.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:29:09 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:29:10.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:29:09 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:29:10.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:29:09 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:29:10.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:29:09 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:29:10.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:29:09 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:29:10.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:29:09 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:29:11.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:29:10 smithi083 ceph-mon[79930]: pgmap v7228: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:11.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:29:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:29:11.036Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:29:11.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:29:10 smithi083 ceph-mon[77242]: pgmap v7228: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:11.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:29:10 smithi153 ceph-mon[70044]: pgmap v7228: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:29:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:29:11] "GET /metrics HTTP/1.1" 200 225220 "" "Prometheus/2.33.4" 2022-09-29T09:29:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:29:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:29:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:29:13.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:29:13 smithi153 ceph-mon[70044]: pgmap v7229: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:13.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:29:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:29:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:29:13.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:29:13 smithi083 ceph-mon[79930]: pgmap v7229: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:13.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:29:13 smithi083 ceph-mon[77242]: pgmap v7229: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:14.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:29:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:29:14.419Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:29:15.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:29:14 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:29:15.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:29:14 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:29:15.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:29:14 smithi153 ceph-mon[70044]: pgmap v7230: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:15.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:29:14 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:29:15.131 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:29:14 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:29:15.132 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:29:14 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:29:15.132 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:29:14 smithi083 ceph-mon[79930]: pgmap v7230: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:15.132 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:29:14 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:29:15.133 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:29:14 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:29:15.133 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:29:14 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:29:15.133 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:29:14 smithi083 ceph-mon[77242]: pgmap v7230: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:15.133 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:29:14 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:29:15.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:29:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:29:15.129Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:29:17.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:29:17 smithi083 ceph-mon[79930]: pgmap v7231: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:17.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:29:17 smithi083 ceph-mon[77242]: pgmap v7231: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:17.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:29:17 smithi153 ceph-mon[70044]: pgmap v7231: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:19.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:29:18 smithi083 ceph-mon[79930]: pgmap v7232: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:19.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:29:18 smithi083 ceph-mon[77242]: pgmap v7232: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:19.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:29:18 smithi153 ceph-mon[70044]: pgmap v7232: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:21.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:29:21 smithi083 ceph-mon[79930]: pgmap v7233: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:21.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:29:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:29:21.037Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:29:21.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:29:21 smithi083 ceph-mon[77242]: pgmap v7233: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:21.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:29:21 smithi153 ceph-mon[70044]: pgmap v7233: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:29:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:29:21] "GET /metrics HTTP/1.1" 200 225220 "" "Prometheus/2.33.4" 2022-09-29T09:29:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:29:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:29:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:29:23.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:29:23 smithi153 ceph-mon[70044]: pgmap v7234: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:23.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:29:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:29:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:29:23.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:29:23 smithi083 ceph-mon[79930]: pgmap v7234: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:23.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:29:23 smithi083 ceph-mon[77242]: pgmap v7234: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:24.194 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:29:24.486 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:29:24.486 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 10s ago 4h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:29:24.486 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 8m ago 4h 62.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:29:24.487 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 10s ago 4h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:29:24.487 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 8m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:29:24.487 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 8m ago 4h 443M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:29:24.487 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 10s ago 4h 723M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:29:24.488 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (3h) 10s ago 4h 924M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:29:24.488 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 8m ago 4h 391M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:29:24.488 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (3h) 10s ago 4h 540M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:29:24.488 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 10s ago 4h 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:29:24.488 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 8m ago 4h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:29:24.489 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 10s ago 4h 666M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:29:24.489 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 10s ago 4h 657M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:29:24.489 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 10s ago 4h 515M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:29:24.489 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 10s ago 4h 639M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:29:24.489 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 8m ago 4h 544M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:29:24.490 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 8m ago 4h 675M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:29:24.490 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 8m ago 4h 602M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:29:24.490 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 8m ago 4h 709M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:29:24.490 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 8m ago 4h 168M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:29:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:29:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:29:24.420Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:29:24.840 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:29:24.841 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:29:24.841 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:29:24.841 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:29:24.841 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:29:24.842 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:29:24.842 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:29:24.842 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:29:24.842 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:29:24.843 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:29:24.843 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:29:24.843 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:29:24.843 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:29:24.843 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:29:24.844 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:29:24.844 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:29:24.844 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:29:24.844 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:29:25.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:29:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:29:25.129Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:29:25.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:29:25 smithi083 ceph-mon[77242]: pgmap v7235: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:25.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:29:25 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3122126450' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:29:25.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:29:25 smithi083 ceph-mon[79930]: pgmap v7235: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:25.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:29:25 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3122126450' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:29:25.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:29:25 smithi153 ceph-mon[70044]: pgmap v7235: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:25.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:29:25 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3122126450' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:29:26.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:29:26 smithi083 ceph-mon[77242]: from='client.42549 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:29:26.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:29:26 smithi083 ceph-mon[77242]: from='client.42555 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:29:26.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:29:26 smithi083 ceph-mon[79930]: from='client.42549 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:29:26.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:29:26 smithi083 ceph-mon[79930]: from='client.42555 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:29:26.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:29:26 smithi153 ceph-mon[70044]: from='client.42549 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:29:26.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:29:26 smithi153 ceph-mon[70044]: from='client.42555 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:29:27.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:29:27 smithi083 ceph-mon[77242]: pgmap v7236: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:27.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:29:27 smithi083 ceph-mon[79930]: pgmap v7236: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:27.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:29:27 smithi153 ceph-mon[70044]: pgmap v7236: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:29.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:29:28 smithi083 ceph-mon[79930]: pgmap v7237: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:29.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:29:28 smithi083 ceph-mon[77242]: pgmap v7237: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:29.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:29:28 smithi153 ceph-mon[70044]: pgmap v7237: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:31.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:29:31 smithi083 ceph-mon[77242]: pgmap v7238: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:31.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:29:31 smithi083 ceph-mon[79930]: pgmap v7238: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:31.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:29:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:29:31.038Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:29:31.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:29:31 smithi153 ceph-mon[70044]: pgmap v7238: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:29:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:29:31] "GET /metrics HTTP/1.1" 200 225216 "" "Prometheus/2.33.4" 2022-09-29T09:29:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:29:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:29:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:29:33.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:29:33 smithi153 ceph-mon[70044]: pgmap v7239: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:33.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:29:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:29:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:29:33.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:29:33 smithi083 ceph-mon[79930]: pgmap v7239: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:33.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:29:33 smithi083 ceph-mon[77242]: pgmap v7239: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:34.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:29:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:29:34.421Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:29:35.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:29:35 smithi083 ceph-mon[79930]: pgmap v7240: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:35.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:29:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:29:35.130Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:29:35.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:29:35 smithi083 ceph-mon[77242]: pgmap v7240: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:35.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:29:35 smithi153 ceph-mon[70044]: pgmap v7240: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:37.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:29:37 smithi083 ceph-mon[79930]: pgmap v7241: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:37.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:29:37 smithi083 ceph-mon[77242]: pgmap v7241: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:37.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:29:37 smithi153 ceph-mon[70044]: pgmap v7241: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:39.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:29:38 smithi083 ceph-mon[79930]: pgmap v7242: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:39.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:29:38 smithi083 ceph-mon[77242]: pgmap v7242: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:39.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:29:38 smithi153 ceph-mon[70044]: pgmap v7242: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:29:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:29:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:29:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:29:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:29:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:29:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:29:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:29:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:29:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:29:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:29:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:29:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:29:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:29:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:29:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:29:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:29:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:29:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:29:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:29:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:29:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:29:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:29:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:29:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:29:40 smithi153 ceph-mon[70044]: pgmap v7243: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:41.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:29:40 smithi083 ceph-mon[79930]: pgmap v7243: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:41.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:29:40 smithi083 ceph-mon[77242]: pgmap v7243: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:41.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:29:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:29:41.038Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:29:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:29:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:29:41] "GET /metrics HTTP/1.1" 200 225220 "" "Prometheus/2.33.4" 2022-09-29T09:29:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:29:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:29:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:29:43.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:29:43 smithi153 ceph-mon[70044]: pgmap v7244: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:43.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:29:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:29:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:29:43.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:29:43 smithi083 ceph-mon[77242]: pgmap v7244: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:43.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:29:43 smithi083 ceph-mon[79930]: pgmap v7244: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:29:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:29:44.421Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:29:45.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:29:45 smithi083 ceph-mon[79930]: pgmap v7245: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:45.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:29:45 smithi083 ceph-mon[77242]: pgmap v7245: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:45.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:29:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:29:45.131Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:29:45.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:29:45 smithi153 ceph-mon[70044]: pgmap v7245: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:47.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:29:47 smithi083 ceph-mon[79930]: pgmap v7246: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:47.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:29:47 smithi083 ceph-mon[77242]: pgmap v7246: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:47.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:29:47 smithi153 ceph-mon[70044]: pgmap v7246: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:49.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:29:48 smithi083 ceph-mon[79930]: pgmap v7247: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:49.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:29:48 smithi083 ceph-mon[77242]: pgmap v7247: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:49.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:29:48 smithi153 ceph-mon[70044]: pgmap v7247: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:51.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:29:51 smithi083 ceph-mon[79930]: pgmap v7248: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:51.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:29:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:29:51.038Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:29:51.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:29:51 smithi083 ceph-mon[77242]: pgmap v7248: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:51.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:29:51 smithi153 ceph-mon[70044]: pgmap v7248: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:29:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:29:51] "GET /metrics HTTP/1.1" 200 225220 "" "Prometheus/2.33.4" 2022-09-29T09:29:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:29:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:29:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:29:53.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:29:53 smithi153 ceph-mon[70044]: pgmap v7249: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:53.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:29:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:29:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:29:53.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:29:53 smithi083 ceph-mon[77242]: pgmap v7249: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:53.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:29:53 smithi083 ceph-mon[79930]: pgmap v7249: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:29:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:29:54.422Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:29:55.171 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:29:55.460 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:29:55.461 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 41s ago 4h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:29:55.461 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 8m ago 4h 62.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:29:55.461 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 41s ago 4h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:29:55.461 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 8m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:29:55.462 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 8m ago 4h 443M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:29:55.462 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 41s ago 4h 723M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:29:55.462 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 41s ago 4h 924M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:29:55.462 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (3h) 8m ago 4h 391M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:29:55.462 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 41s ago 4h 540M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:29:55.463 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 41s ago 4h 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:29:55.463 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 8m ago 4h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:29:55.463 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 41s ago 4h 666M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:29:55.463 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 41s ago 4h 657M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:29:55.464 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 41s ago 4h 515M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:29:55.464 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 41s ago 4h 639M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:29:55.464 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 8m ago 4h 544M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:29:55.464 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 8m ago 4h 675M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:29:55.464 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 8m ago 4h 602M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:29:55.465 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 8m ago 4h 709M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:29:55.465 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 8m ago 4h 168M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:29:55.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:29:55 smithi083 ceph-mon[79930]: pgmap v7250: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:55.466 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:29:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:29:55.131Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:29:55.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:29:55 smithi083 ceph-mon[77242]: pgmap v7250: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:55.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:29:55 smithi153 ceph-mon[70044]: pgmap v7250: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:55.812 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:29:55.812 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:29:55.812 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:29:55.813 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:29:55.813 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:29:55.813 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:29:55.813 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:29:55.814 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:29:55.814 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:29:55.814 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:29:55.814 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:29:55.814 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:29:55.815 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:29:55.815 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:29:55.815 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:29:55.815 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:29:55.816 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:29:55.816 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:29:56.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:29:56 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/197714293' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:29:56.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:29:56 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/197714293' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:29:56.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:29:56 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/197714293' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:29:57.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:29:57 smithi083 ceph-mon[79930]: from='client.52498 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:29:57.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:29:57 smithi083 ceph-mon[79930]: from='client.42573 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:29:57.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:29:57 smithi083 ceph-mon[79930]: pgmap v7251: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:57.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:29:57 smithi083 ceph-mon[77242]: from='client.52498 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:29:57.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:29:57 smithi083 ceph-mon[77242]: from='client.42573 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:29:57.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:29:57 smithi083 ceph-mon[77242]: pgmap v7251: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:57.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:29:57 smithi153 ceph-mon[70044]: from='client.52498 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:29:57.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:29:57 smithi153 ceph-mon[70044]: from='client.42573 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:29:57.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:29:57 smithi153 ceph-mon[70044]: pgmap v7251: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:59.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:29:58 smithi083 ceph-mon[79930]: pgmap v7252: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:59.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:29:58 smithi083 ceph-mon[77242]: pgmap v7252: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:29:59.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:29:58 smithi153 ceph-mon[70044]: pgmap v7252: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:00.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:30:00 smithi153 ceph-mon[70044]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T09:30:00.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:30:00 smithi083 ceph-mon[79930]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T09:30:00.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:30:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[77219]: 2022-09-29T09:29:59.999+0000 7f160f665700 -1 log_channel(cluster) log [ERR] : overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T09:30:00.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:30:00 smithi083 ceph-mon[77242]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T09:30:01.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:30:01 smithi083 ceph-mon[79930]: pgmap v7253: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:01.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:30:01 smithi083 ceph-mon[77242]: pgmap v7253: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:01.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:30:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:30:01.039Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:30:01.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:30:01 smithi153 ceph-mon[70044]: pgmap v7253: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:30:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:30:01] "GET /metrics HTTP/1.1" 200 225215 "" "Prometheus/2.33.4" 2022-09-29T09:30:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:30:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:30:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:30:03.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:30:03 smithi153 ceph-mon[70044]: pgmap v7254: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:30:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:30:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:30:03.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:30:03 smithi083 ceph-mon[79930]: pgmap v7254: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:03.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:30:03 smithi083 ceph-mon[77242]: pgmap v7254: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:04.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:30:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:30:04.422Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:30:05.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:30:05 smithi083 ceph-mon[79930]: pgmap v7255: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:05.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:30:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:30:05.132Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:30:05.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:30:05 smithi083 ceph-mon[77242]: pgmap v7255: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:05.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:30:05 smithi153 ceph-mon[70044]: pgmap v7255: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:07.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:30:07 smithi083 ceph-mon[79930]: pgmap v7256: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:07.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:30:07 smithi083 ceph-mon[77242]: pgmap v7256: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:07.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:30:07 smithi153 ceph-mon[70044]: pgmap v7256: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:09.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:30:08 smithi083 ceph-mon[79930]: pgmap v7257: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:09.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:30:08 smithi083 ceph-mon[77242]: pgmap v7257: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:09.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:30:08 smithi153 ceph-mon[70044]: pgmap v7257: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:11.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:30:11 smithi083 ceph-mon[79930]: pgmap v7258: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:11.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:30:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:30:11.041Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:30:11.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:30:11 smithi083 ceph-mon[77242]: pgmap v7258: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:11.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:30:11 smithi153 ceph-mon[70044]: pgmap v7258: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:30:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:30:11] "GET /metrics HTTP/1.1" 200 225216 "" "Prometheus/2.33.4" 2022-09-29T09:30:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:30:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:30:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:30:13.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:30:13 smithi153 ceph-mon[70044]: pgmap v7259: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:13.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:30:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:30:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:30:13.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:30:13 smithi083 ceph-mon[79930]: pgmap v7259: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:13.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:30:13 smithi083 ceph-mon[77242]: pgmap v7259: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:14.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:30:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:30:14.423Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:30:15.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:30:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:30:15.132Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:30:15.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:30:15 smithi083 ceph-mon[77242]: pgmap v7260: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:15.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:30:15 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:30:15.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:30:15 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:30:15.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:30:15 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:30:15.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:30:15 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:30:15.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:30:15 smithi083 ceph-mon[79930]: pgmap v7260: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:15.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:30:15 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:30:15.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:30:15 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:30:15.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:30:15 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:30:15.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:30:15 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:30:15.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:30:15 smithi153 ceph-mon[70044]: pgmap v7260: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:15.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:30:15 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:30:15.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:30:15 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:30:15.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:30:15 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:30:15.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:30:15 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:30:17.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:30:17 smithi083 ceph-mon[79930]: pgmap v7261: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:17.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:30:17 smithi083 ceph-mon[77242]: pgmap v7261: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:17.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:30:17 smithi153 ceph-mon[70044]: pgmap v7261: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:19.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:30:18 smithi083 ceph-mon[79930]: pgmap v7262: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:19.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:30:18 smithi083 ceph-mon[77242]: pgmap v7262: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:19.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:30:18 smithi153 ceph-mon[70044]: pgmap v7262: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:21.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:30:21 smithi083 ceph-mon[79930]: pgmap v7263: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:21.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:30:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:30:21.041Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:30:21.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:30:21 smithi083 ceph-mon[77242]: pgmap v7263: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:21.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:30:21 smithi153 ceph-mon[70044]: pgmap v7263: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:30:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:30:21] "GET /metrics HTTP/1.1" 200 225216 "" "Prometheus/2.33.4" 2022-09-29T09:30:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:30:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:30:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:30:23.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:30:23 smithi153 ceph-mon[70044]: pgmap v7264: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:23.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:30:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:30:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:30:23.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:30:23 smithi083 ceph-mon[79930]: pgmap v7264: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:23.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:30:23 smithi083 ceph-mon[77242]: pgmap v7264: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:30:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:30:24.423Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:30:25.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:30:25 smithi083 ceph-mon[79930]: pgmap v7265: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:25.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:30:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:30:25.133Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:30:25.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:30:25 smithi083 ceph-mon[77242]: pgmap v7265: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:25.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:30:25 smithi153 ceph-mon[70044]: pgmap v7265: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:26.140 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:30:26.426 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:30:26.426 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 72s ago 4h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:30:26.427 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 9m ago 4h 62.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:30:26.427 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 72s ago 4h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:30:26.427 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 9m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:30:26.427 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 9m ago 4h 443M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:30:26.427 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 72s ago 4h 723M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:30:26.428 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 72s ago 4h 924M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:30:26.428 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 9m ago 4h 391M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:30:26.428 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 72s ago 4h 540M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:30:26.428 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 72s ago 4h 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:30:26.428 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 9m ago 4h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:30:26.429 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3h) 72s ago 4h 666M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:30:26.429 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 72s ago 4h 657M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:30:26.429 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 72s ago 4h 515M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:30:26.429 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 72s ago 4h 639M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:30:26.429 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 9m ago 4h 544M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:30:26.430 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 9m ago 4h 675M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:30:26.430 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 9m ago 4h 602M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:30:26.430 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 9m ago 4h 709M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:30:26.430 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 9m ago 4h 168M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:30:26.784 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:30:26.784 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:30:26.785 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:30:26.785 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:30:26.785 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:30:26.786 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:30:26.786 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:30:26.786 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:30:26.786 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:30:26.786 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:30:26.787 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:30:26.787 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:30:26.787 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:30:26.787 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:30:26.788 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:30:26.788 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:30:26.788 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:30:26.788 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:30:27.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:30:27 smithi083 ceph-mon[79930]: pgmap v7266: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:27.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:30:27 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3989778278' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:30:27.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:30:27 smithi083 ceph-mon[77242]: pgmap v7266: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:27.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:30:27 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3989778278' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:30:27.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:30:27 smithi153 ceph-mon[70044]: pgmap v7266: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:27.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:30:27 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3989778278' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:30:28.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:30:28 smithi083 ceph-mon[79930]: from='client.52516 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:30:28.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:30:28 smithi083 ceph-mon[79930]: from='client.52522 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:30:28.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:30:28 smithi083 ceph-mon[77242]: from='client.52516 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:30:28.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:30:28 smithi083 ceph-mon[77242]: from='client.52522 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:30:28.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:30:28 smithi153 ceph-mon[70044]: from='client.52516 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:30:28.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:30:28 smithi153 ceph-mon[70044]: from='client.52522 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:30:29.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:30:29 smithi083 ceph-mon[79930]: pgmap v7267: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:29.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:30:29 smithi083 ceph-mon[77242]: pgmap v7267: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:29.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:30:29 smithi153 ceph-mon[70044]: pgmap v7267: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:31.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:30:31 smithi083 ceph-mon[79930]: pgmap v7268: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:31.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:30:31 smithi083 ceph-mon[77242]: pgmap v7268: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:31.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:30:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:30:31.041Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:30:31.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:30:31 smithi153 ceph-mon[70044]: pgmap v7268: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:30:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:30:31] "GET /metrics HTTP/1.1" 200 225215 "" "Prometheus/2.33.4" 2022-09-29T09:30:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:30:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:30:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:30:33.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:30:33 smithi153 ceph-mon[70044]: pgmap v7269: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:30:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:30:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:30:33.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:30:33 smithi083 ceph-mon[79930]: pgmap v7269: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:33.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:30:33 smithi083 ceph-mon[77242]: pgmap v7269: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:34.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:30:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:30:34.424Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:30:35.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:30:35 smithi083 ceph-mon[79930]: pgmap v7270: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:35.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:30:35 smithi083 ceph-mon[77242]: pgmap v7270: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:35.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:30:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:30:35.134Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:30:35.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:30:35 smithi153 ceph-mon[70044]: pgmap v7270: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:36.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:30:36 smithi083 ceph-mon[79930]: pgmap v7271: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:36.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:30:36 smithi083 ceph-mon[77242]: pgmap v7271: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:36.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:30:36 smithi153 ceph-mon[70044]: pgmap v7271: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:39.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:30:38 smithi083 ceph-mon[77242]: pgmap v7272: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:39.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:30:38 smithi083 ceph-mon[79930]: pgmap v7272: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:39.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:30:38 smithi153 ceph-mon[70044]: pgmap v7272: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:30:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:30:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:30:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:30:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:30:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:30:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:30:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:30:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:30:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:30:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:30:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:30:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:30:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:30:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:30:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:30:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:30:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:30:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:30:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:30:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:30:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:30:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:30:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:30:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:30:40 smithi153 ceph-mon[70044]: pgmap v7273: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:41.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:30:40 smithi083 ceph-mon[79930]: pgmap v7273: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:41.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:30:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:30:41.042Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:30:41.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:30:40 smithi083 ceph-mon[77242]: pgmap v7273: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:30:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:30:41] "GET /metrics HTTP/1.1" 200 225219 "" "Prometheus/2.33.4" 2022-09-29T09:30:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:30:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:30:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:30:43.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:30:43 smithi153 ceph-mon[70044]: pgmap v7274: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:43.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:30:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:30:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:30:43.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:30:43 smithi083 ceph-mon[79930]: pgmap v7274: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:43.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:30:43 smithi083 ceph-mon[77242]: pgmap v7274: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:30:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:30:44.424Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:30:45.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:30:45 smithi083 ceph-mon[77242]: pgmap v7275: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:45.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:30:45 smithi083 ceph-mon[79930]: pgmap v7275: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:45.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:30:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:30:45.135Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:30:45.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:30:45 smithi153 ceph-mon[70044]: pgmap v7275: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:47.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:30:47 smithi083 ceph-mon[77242]: pgmap v7276: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:47.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:30:47 smithi083 ceph-mon[79930]: pgmap v7276: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:47.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:30:47 smithi153 ceph-mon[70044]: pgmap v7276: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:49.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:30:48 smithi083 ceph-mon[79930]: pgmap v7277: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:49.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:30:48 smithi083 ceph-mon[77242]: pgmap v7277: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:49.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:30:48 smithi153 ceph-mon[70044]: pgmap v7277: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:51.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:30:51 smithi083 ceph-mon[77242]: pgmap v7278: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:51.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:30:51 smithi083 ceph-mon[79930]: pgmap v7278: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:51.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:30:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:30:51.042Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:30:51.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:30:51 smithi153 ceph-mon[70044]: pgmap v7278: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:30:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:30:51] "GET /metrics HTTP/1.1" 200 225219 "" "Prometheus/2.33.4" 2022-09-29T09:30:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:30:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:30:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:30:53.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:30:53 smithi153 ceph-mon[70044]: pgmap v7279: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:53.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:30:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:30:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:30:53.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:30:53 smithi083 ceph-mon[77242]: pgmap v7279: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:53.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:30:53 smithi083 ceph-mon[79930]: pgmap v7279: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:30:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:30:54.424Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:30:55.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:30:55 smithi083 ceph-mon[79930]: pgmap v7280: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:55.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:30:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:30:55.135Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:30:55.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:30:55 smithi083 ceph-mon[77242]: pgmap v7280: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:55.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:30:55 smithi153 ceph-mon[70044]: pgmap v7280: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:57.109 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:30:57.397 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:30:57.397 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 103s ago 4h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:30:57.398 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 9m ago 4h 62.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:30:57.398 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 103s ago 4h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:30:57.398 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 9m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:30:57.398 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 9m ago 4h 443M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:30:57.399 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 103s ago 4h 723M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:30:57.399 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 103s ago 4h 924M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:30:57.399 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 9m ago 4h 391M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:30:57.399 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 103s ago 4h 540M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:30:57.400 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 103s ago 4h 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:30:57.400 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 9m ago 4h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:30:57.400 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 103s ago 4h 666M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:30:57.400 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3h) 103s ago 4h 657M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:30:57.400 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 103s ago 4h 515M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:30:57.401 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 103s ago 4h 639M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:30:57.401 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 9m ago 4h 544M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:30:57.401 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 9m ago 4h 675M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:30:57.401 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 9m ago 4h 602M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:30:57.401 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 9m ago 4h 709M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:30:57.402 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 9m ago 4h 168M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:30:57.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:30:57 smithi083 ceph-mon[79930]: pgmap v7281: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:57.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:30:57 smithi083 ceph-mon[77242]: pgmap v7281: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:57.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:30:57 smithi153 ceph-mon[70044]: pgmap v7281: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:57.748 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:30:57.748 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:30:57.748 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:30:57.749 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:30:57.749 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:30:57.749 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:30:57.749 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:30:57.749 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:30:57.750 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:30:57.750 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:30:57.750 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:30:57.750 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:30:57.751 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:30:57.751 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:30:57.751 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:30:57.751 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:30:57.751 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:30:57.752 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:30:58.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:30:58 smithi083 ceph-mon[79930]: from='client.42603 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:30:58.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:30:58 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3565963484' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:30:58.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:30:58 smithi083 ceph-mon[77242]: from='client.42603 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:30:58.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:30:58 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3565963484' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:30:58.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:30:58 smithi153 ceph-mon[70044]: from='client.42603 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:30:58.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:30:58 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3565963484' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:30:59.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:30:59 smithi083 ceph-mon[79930]: from='client.52540 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:30:59.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:30:59 smithi083 ceph-mon[79930]: pgmap v7282: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:59.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:30:59 smithi083 ceph-mon[77242]: from='client.52540 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:30:59.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:30:59 smithi083 ceph-mon[77242]: pgmap v7282: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:30:59.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:30:59 smithi153 ceph-mon[70044]: from='client.52540 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:30:59.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:30:59 smithi153 ceph-mon[70044]: pgmap v7282: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:00.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:31:00 smithi083 ceph-mon[79930]: pgmap v7283: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:00.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:31:00 smithi083 ceph-mon[77242]: pgmap v7283: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:00.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:31:00 smithi153 ceph-mon[70044]: pgmap v7283: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:01.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:31:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:31:01.043Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:31:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:31:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:31:01] "GET /metrics HTTP/1.1" 200 225223 "" "Prometheus/2.33.4" 2022-09-29T09:31:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:31:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:31:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:31:03.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:31:03 smithi153 ceph-mon[70044]: pgmap v7284: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:31:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:31:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:31:03.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:31:03 smithi083 ceph-mon[79930]: pgmap v7284: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:03.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:31:03 smithi083 ceph-mon[77242]: pgmap v7284: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:04.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:31:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:31:04.425Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:31:05.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:31:05 smithi083 ceph-mon[77242]: pgmap v7285: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:05.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:31:05 smithi083 ceph-mon[79930]: pgmap v7285: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:05.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:31:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:31:05.136Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:31:05.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:31:05 smithi153 ceph-mon[70044]: pgmap v7285: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:07.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:31:07 smithi083 ceph-mon[79930]: pgmap v7286: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:07.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:31:07 smithi083 ceph-mon[77242]: pgmap v7286: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:07.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:31:07 smithi153 ceph-mon[70044]: pgmap v7286: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:09.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:31:08 smithi083 ceph-mon[77242]: pgmap v7287: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:09.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:31:08 smithi083 ceph-mon[79930]: pgmap v7287: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:09.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:31:08 smithi153 ceph-mon[70044]: pgmap v7287: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:11.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:31:11 smithi083 ceph-mon[79930]: pgmap v7288: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:11.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:31:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:31:11.043Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:31:11.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:31:11 smithi083 ceph-mon[77242]: pgmap v7288: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:11.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:31:11 smithi153 ceph-mon[70044]: pgmap v7288: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:31:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:31:11] "GET /metrics HTTP/1.1" 200 225219 "" "Prometheus/2.33.4" 2022-09-29T09:31:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:31:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:31:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:31:13.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:31:13 smithi153 ceph-mon[70044]: pgmap v7289: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:13.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:31:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:31:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:31:13.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:31:13 smithi083 ceph-mon[79930]: pgmap v7289: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:13.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:31:13 smithi083 ceph-mon[77242]: pgmap v7289: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:14.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:31:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:31:14.425Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:31:15.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:31:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:31:15.135Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:31:15.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:31:15 smithi083 ceph-mon[77242]: pgmap v7290: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:15.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:31:15 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:31:15.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:31:15 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:31:15.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:31:15 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:31:15.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:31:15 smithi083 ceph-mon[79930]: pgmap v7290: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:15.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:31:15 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:31:15.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:31:15 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:31:15.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:31:15 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:31:15.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:31:15 smithi153 ceph-mon[70044]: pgmap v7290: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:15.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:31:15 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:31:15.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:31:15 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:31:15.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:31:15 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:31:17.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:31:17 smithi083 ceph-mon[79930]: pgmap v7291: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:17.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:31:17 smithi083 ceph-mon[77242]: pgmap v7291: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:17.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:31:17 smithi153 ceph-mon[70044]: pgmap v7291: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:19.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:31:18 smithi083 ceph-mon[79930]: pgmap v7292: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:19.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:31:18 smithi083 ceph-mon[77242]: pgmap v7292: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:19.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:31:18 smithi153 ceph-mon[70044]: pgmap v7292: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:20.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:31:20 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:31:20.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:31:20 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:31:20.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:31:20 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:31:20.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:31:20 smithi083 ceph-mon[79930]: pgmap v7293: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:20.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:31:20 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:31:20.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:31:20 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:31:20.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:31:20 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:31:20.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:31:20 smithi083 ceph-mon[77242]: pgmap v7293: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:20.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:31:20 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:31:20.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:31:20 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:31:20.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:31:20 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:31:20.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:31:20 smithi153 ceph-mon[70044]: pgmap v7293: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:21.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:31:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:31:21.044Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:31:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:31:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:31:21] "GET /metrics HTTP/1.1" 200 225219 "" "Prometheus/2.33.4" 2022-09-29T09:31:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:31:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:31:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:31:23.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:31:23 smithi153 ceph-mon[70044]: pgmap v7294: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:23.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:31:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:31:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:31:23.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:31:23 smithi083 ceph-mon[79930]: pgmap v7294: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:23.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:31:23 smithi083 ceph-mon[77242]: pgmap v7294: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:31:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:31:24.426Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:31:25.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:31:25 smithi083 ceph-mon[77242]: pgmap v7295: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:25.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:31:25 smithi083 ceph-mon[79930]: pgmap v7295: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:25.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:31:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:31:25.137Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:31:25.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:31:25 smithi153 ceph-mon[70044]: pgmap v7295: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:27.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:31:27 smithi083 ceph-mon[79930]: pgmap v7296: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:27.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:31:27 smithi083 ceph-mon[77242]: pgmap v7296: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:27.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:31:27 smithi153 ceph-mon[70044]: pgmap v7296: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:28.076 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:31:28.363 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:31:28.364 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 2m ago 4h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:31:28.364 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 8s ago 4h 62.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:31:28.364 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 2m ago 4h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:31:28.365 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 8s ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:31:28.365 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 8s ago 4h 443M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:31:28.365 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 2m ago 4h 723M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:31:28.365 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 2m ago 4h 924M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:31:28.366 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 8s ago 4h 407M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:31:28.366 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 2m ago 4h 540M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:31:28.366 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 2m ago 4h 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:31:28.366 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 8s ago 4h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:31:28.367 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 2m ago 4h 666M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:31:28.367 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 2m ago 4h 657M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:31:28.367 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3h) 2m ago 4h 515M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:31:28.367 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 2m ago 4h 639M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:31:28.368 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 8s ago 4h 566M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:31:28.368 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 8s ago 4h 702M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:31:28.368 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 8s ago 4h 626M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:31:28.368 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 8s ago 4h 739M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:31:28.369 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 8s ago 4h 171M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:31:28.714 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:31:28.715 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:31:28.715 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:31:28.715 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:31:28.715 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:31:28.716 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:31:28.716 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:31:28.716 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:31:28.716 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:31:28.716 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:31:28.717 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:31:28.717 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:31:28.717 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:31:28.717 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:31:28.718 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:31:28.718 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:31:28.718 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:31:28.718 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:31:29.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:31:28 smithi083 ceph-mon[77242]: from='client.42621 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:31:29.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:31:28 smithi083 ceph-mon[77242]: pgmap v7297: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:29.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:31:28 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1828492938' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:31:29.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:31:28 smithi083 ceph-mon[79930]: from='client.42621 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:31:29.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:31:28 smithi083 ceph-mon[79930]: pgmap v7297: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:29.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:31:28 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1828492938' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:31:29.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:31:28 smithi153 ceph-mon[70044]: from='client.42621 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:31:29.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:31:28 smithi153 ceph-mon[70044]: pgmap v7297: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:29.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:31:28 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1828492938' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:31:30.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:31:29 smithi083 ceph-mon[79930]: from='client.52558 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:31:30.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:31:29 smithi083 ceph-mon[77242]: from='client.52558 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:31:30.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:31:29 smithi153 ceph-mon[70044]: from='client.52558 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:31:31.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:31:30 smithi083 ceph-mon[79930]: pgmap v7298: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:31.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:31:30 smithi083 ceph-mon[77242]: pgmap v7298: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:31.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:31:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:31:31.045Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:31:31.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:31:30 smithi153 ceph-mon[70044]: pgmap v7298: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:31:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:31:31] "GET /metrics HTTP/1.1" 200 225216 "" "Prometheus/2.33.4" 2022-09-29T09:31:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:31:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:31:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:31:33.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:31:33 smithi153 ceph-mon[70044]: pgmap v7299: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:33.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:31:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:31:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:31:33.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:31:33 smithi083 ceph-mon[79930]: pgmap v7299: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:33.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:31:33 smithi083 ceph-mon[77242]: pgmap v7299: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:34.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:31:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:31:34.427Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:31:35.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:31:35 smithi083 ceph-mon[79930]: pgmap v7300: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:35.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:31:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:31:35.137Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:31:35.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:31:35 smithi083 ceph-mon[77242]: pgmap v7300: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:35.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:31:35 smithi153 ceph-mon[70044]: pgmap v7300: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:37.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:31:37 smithi083 ceph-mon[79930]: pgmap v7301: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:37.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:31:37 smithi083 ceph-mon[77242]: pgmap v7301: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:37.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:31:37 smithi153 ceph-mon[70044]: pgmap v7301: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:39.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:31:38 smithi083 ceph-mon[79930]: pgmap v7302: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:39.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:31:38 smithi083 ceph-mon[77242]: pgmap v7302: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:39.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:31:38 smithi153 ceph-mon[70044]: pgmap v7302: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:31:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:31:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:31:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:31:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:31:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:31:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:31:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:31:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:31:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:31:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:31:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:31:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:31:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:31:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:31:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:31:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:31:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:31:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:31:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:31:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:31:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:31:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:31:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:31:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:31:40 smithi153 ceph-mon[70044]: pgmap v7303: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:41.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:31:40 smithi083 ceph-mon[79930]: pgmap v7303: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:41.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:31:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:31:41.045Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:31:41.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:31:40 smithi083 ceph-mon[77242]: pgmap v7303: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:31:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:31:41] "GET /metrics HTTP/1.1" 200 225214 "" "Prometheus/2.33.4" 2022-09-29T09:31:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:31:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:31:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:31:43.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:31:43 smithi153 ceph-mon[70044]: pgmap v7304: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:43.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:31:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:31:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:31:43.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:31:43 smithi083 ceph-mon[79930]: pgmap v7304: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:43.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:31:43 smithi083 ceph-mon[77242]: pgmap v7304: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:31:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:31:44.428Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:31:45.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:31:45 smithi083 ceph-mon[79930]: pgmap v7305: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:45.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:31:45 smithi083 ceph-mon[77242]: pgmap v7305: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:45.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:31:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:31:45.138Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:31:45.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:31:45 smithi153 ceph-mon[70044]: pgmap v7305: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:46.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:31:46 smithi083 ceph-mon[77242]: pgmap v7306: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:46.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:31:46 smithi083 ceph-mon[79930]: pgmap v7306: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:46.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:31:46 smithi153 ceph-mon[70044]: pgmap v7306: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:49.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:31:48 smithi083 ceph-mon[79930]: pgmap v7307: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:49.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:31:48 smithi083 ceph-mon[77242]: pgmap v7307: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:49.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:31:48 smithi153 ceph-mon[70044]: pgmap v7307: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:51.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:31:51 smithi083 ceph-mon[79930]: pgmap v7308: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:51.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:31:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:31:51.046Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:31:51.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:31:51 smithi083 ceph-mon[77242]: pgmap v7308: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:51.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:31:51 smithi153 ceph-mon[70044]: pgmap v7308: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:31:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:31:51] "GET /metrics HTTP/1.1" 200 225214 "" "Prometheus/2.33.4" 2022-09-29T09:31:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:31:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:31:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:31:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:31:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:31:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:31:53.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:31:53 smithi153 ceph-mon[70044]: pgmap v7309: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:53.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:31:53 smithi083 ceph-mon[79930]: pgmap v7309: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:53.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:31:53 smithi083 ceph-mon[77242]: pgmap v7309: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:31:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:31:54.428Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:31:55.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:31:55 smithi083 ceph-mon[79930]: pgmap v7310: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:55.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:31:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:31:55.139Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:31:55.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:31:55 smithi083 ceph-mon[77242]: pgmap v7310: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:55.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:31:55 smithi153 ceph-mon[70044]: pgmap v7310: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:56.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:31:56 smithi083 ceph-mon[79930]: pgmap v7311: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:56.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:31:56 smithi083 ceph-mon[77242]: pgmap v7311: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:56.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:31:56 smithi153 ceph-mon[70044]: pgmap v7311: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:59.040 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:31:59.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:31:58 smithi083 ceph-mon[77242]: pgmap v7312: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:59.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:31:58 smithi083 ceph-mon[79930]: pgmap v7312: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:59.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:31:58 smithi153 ceph-mon[70044]: pgmap v7312: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:31:59.330 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:31:59.330 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 2m ago 4h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:31:59.331 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 39s ago 4h 62.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:31:59.331 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 2m ago 4h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:31:59.331 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 39s ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:31:59.332 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 39s ago 4h 443M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:31:59.332 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 2m ago 4h 723M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:31:59.332 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 2m ago 4h 924M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:31:59.332 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 39s ago 4h 407M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:31:59.333 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 2m ago 4h 540M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:31:59.333 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 2m ago 4h 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:31:59.333 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 39s ago 4h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:31:59.334 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 2m ago 4h 666M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:31:59.334 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 2m ago 4h 657M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:31:59.334 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 2m ago 4h 515M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:31:59.334 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3h) 2m ago 4h 639M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:31:59.335 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 39s ago 4h 566M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:31:59.335 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 39s ago 4h 702M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:31:59.335 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 39s ago 4h 626M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:31:59.336 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 39s ago 4h 739M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:31:59.336 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 39s ago 4h 171M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:31:59.684 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:31:59.684 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:31:59.684 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:31:59.685 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:31:59.685 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:31:59.685 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:31:59.685 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:31:59.686 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:31:59.686 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:31:59.686 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:31:59.686 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:31:59.687 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:31:59.687 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:31:59.687 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:31:59.687 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:31:59.687 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:31:59.688 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:31:59.688 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:32:00.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:31:59 smithi083 ceph-mon[77242]: from='client.42639 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:32:00.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:31:59 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/287683557' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:32:00.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:31:59 smithi083 ceph-mon[79930]: from='client.42639 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:32:00.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:31:59 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/287683557' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:32:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:31:59 smithi153 ceph-mon[70044]: from='client.42639 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:32:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:31:59 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/287683557' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:32:01.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:32:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:32:01.046Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:32:01.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:32:00 smithi083 ceph-mon[77242]: from='client.42645 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:32:01.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:32:00 smithi083 ceph-mon[77242]: pgmap v7313: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:32:01.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:32:00 smithi083 ceph-mon[79930]: from='client.42645 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:32:01.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:32:00 smithi083 ceph-mon[79930]: pgmap v7313: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:32:01.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:32:00 smithi153 ceph-mon[70044]: from='client.42645 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:32:01.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:32:00 smithi153 ceph-mon[70044]: pgmap v7313: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:32:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:32:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:32:01] "GET /metrics HTTP/1.1" 200 225217 "" "Prometheus/2.33.4" 2022-09-29T09:32:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:32:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:32:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:32:03.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:32:03 smithi153 ceph-mon[70044]: pgmap v7314: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:32:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:32:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:32:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:32:03.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:32:03 smithi083 ceph-mon[79930]: pgmap v7314: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:32:03.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:32:03 smithi083 ceph-mon[77242]: pgmap v7314: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:32:04.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:32:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:32:04.429Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:32:05.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:32:05 smithi083 ceph-mon[79930]: pgmap v7315: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:32:05.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:32:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:32:05.140Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:32:05.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:32:05 smithi083 ceph-mon[77242]: pgmap v7315: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:32:05.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:32:05 smithi153 ceph-mon[70044]: pgmap v7315: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:32:06.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:32:06 smithi083 ceph-mon[77242]: pgmap v7316: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:32:06.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:32:06 smithi083 ceph-mon[79930]: pgmap v7316: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:32:06.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:32:06 smithi153 ceph-mon[70044]: pgmap v7316: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:32:09.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:32:08 smithi083 ceph-mon[79930]: pgmap v7317: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:32:09.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:32:08 smithi083 ceph-mon[77242]: pgmap v7317: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:32:09.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:32:08 smithi153 ceph-mon[70044]: pgmap v7317: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:32:11.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:32:11 smithi083 ceph-mon[77242]: pgmap v7318: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:32:11.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:32:11 smithi083 ceph-mon[79930]: pgmap v7318: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:32:11.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:32:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:32:11.047Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:32:11.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:32:11 smithi153 ceph-mon[70044]: pgmap v7318: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:32:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:32:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:32:11] "GET /metrics HTTP/1.1" 200 225219 "" "Prometheus/2.33.4" 2022-09-29T09:32:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:32:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:32:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:32:13.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:32:13 smithi153 ceph-mon[70044]: pgmap v7319: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:32:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:32:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:32:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:32:13.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:32:13 smithi083 ceph-mon[77242]: pgmap v7319: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:32:13.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:32:13 smithi083 ceph-mon[79930]: pgmap v7319: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:32:14.432 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:32:14 smithi083 ceph-mon[79930]: pgmap v7320: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:32:14.432 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:32:14 smithi083 ceph-mon[77242]: pgmap v7320: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:32:14.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:32:14 smithi153 ceph-mon[70044]: pgmap v7320: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:32:14.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:32:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:32:14.429Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:32:15.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:32:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:32:15.140Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:32:17.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:32:17 smithi083 ceph-mon[79930]: pgmap v7321: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:32:17.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:32:17 smithi083 ceph-mon[77242]: pgmap v7321: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:32:17.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:32:17 smithi153 ceph-mon[70044]: pgmap v7321: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:32:19.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:32:18 smithi083 ceph-mon[79930]: pgmap v7322: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:32:19.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:32:18 smithi083 ceph-mon[77242]: pgmap v7322: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:32:19.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:32:18 smithi153 ceph-mon[70044]: pgmap v7322: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:32:20.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:32:20 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:32:20.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:32:20 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:32:20.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:32:20 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:32:20.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:32:20 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:32:20.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:32:20 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:32:20.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:32:20 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:32:20.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:32:20 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:32:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:32:20 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:32:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:32:20 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:32:21.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:32:21 smithi083 ceph-mon[79930]: pgmap v7323: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:32:21.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:32:21 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:32:21.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:32:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:32:21.047Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:32:21.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:32:21 smithi083 ceph-mon[77242]: pgmap v7323: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:32:21.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:32:21 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:32:21.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:32:21 smithi153 ceph-mon[70044]: pgmap v7323: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:32:21.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:32:21 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:32:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:32:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:32:21] "GET /metrics HTTP/1.1" 200 225219 "" "Prometheus/2.33.4" 2022-09-29T09:32:22.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:32:22 smithi083 ceph-mon[77242]: pgmap v7324: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:32:22.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:32:22 smithi083 ceph-mon[79930]: pgmap v7324: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:32:22.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:32:22 smithi153 ceph-mon[70044]: pgmap v7324: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:32:22.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:32:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:32:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:32:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:32:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:32:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:32:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:32:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:32:24.430Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:32:25.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:32:25 smithi083 ceph-mon[77242]: pgmap v7325: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:32:25.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:32:25 smithi083 ceph-mon[79930]: pgmap v7325: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:32:25.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:32:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:32:25.140Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:32:25.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:32:25 smithi153 ceph-mon[70044]: pgmap v7325: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:32:26.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:32:26 smithi083 ceph-mon[79930]: pgmap v7326: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:32:26.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:32:26 smithi083 ceph-mon[77242]: pgmap v7326: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:32:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:32:26 smithi153 ceph-mon[70044]: pgmap v7326: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:32:29.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:32:28 smithi083 ceph-mon[79930]: pgmap v7327: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:32:29.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:32:28 smithi083 ceph-mon[77242]: pgmap v7327: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:32:29.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:32:28 smithi153 ceph-mon[70044]: pgmap v7327: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:32:30.013 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:32:30.302 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:32:30.303 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 3m ago 4h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:32:30.303 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 70s ago 4h 62.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:32:30.303 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 3m ago 4h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:32:30.303 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 70s ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:32:30.304 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 70s ago 4h 443M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:32:30.304 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 3m ago 4h 723M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:32:30.304 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 3m ago 4h 924M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:32:30.305 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 70s ago 4h 407M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:32:30.305 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 3m ago 4h 540M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:32:30.305 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 3m ago 4h 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:32:30.305 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 70s ago 4h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:32:30.306 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 3m ago 4h 666M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:32:30.306 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 3m ago 4h 657M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:32:30.306 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 3m ago 4h 515M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:32:30.306 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 3m ago 4h 639M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:32:30.307 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (3h) 70s ago 4h 566M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:32:30.307 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 70s ago 4h 702M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:32:30.307 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 70s ago 4h 626M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:32:30.307 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 70s ago 4h 739M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:32:30.308 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 70s ago 4h 171M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:32:30.657 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:32:30.657 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:32:30.657 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:32:30.658 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:32:30.658 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:32:30.658 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:32:30.658 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:32:30.659 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:32:30.659 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:32:30.659 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:32:30.659 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:32:30.660 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:32:30.660 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:32:30.660 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:32:30.660 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:32:30.660 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:32:30.661 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:32:30.661 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:32:31.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:32:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:32:31.048Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:32:31.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:32:31 smithi083 ceph-mon[79930]: from='client.52588 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:32:31.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:32:31 smithi083 ceph-mon[79930]: pgmap v7328: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:32:31.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:32:31 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2740204617' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:32:31.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:32:31 smithi083 ceph-mon[77242]: from='client.52588 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:32:31.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:32:31 smithi083 ceph-mon[77242]: pgmap v7328: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:32:31.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:32:31 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2740204617' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:32:31.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:32:31 smithi153 ceph-mon[70044]: from='client.52588 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:32:31.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:32:31 smithi153 ceph-mon[70044]: pgmap v7328: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:32:31.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:32:31 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2740204617' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:32:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:32:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:32:31] "GET /metrics HTTP/1.1" 200 225219 "" "Prometheus/2.33.4" 2022-09-29T09:32:32.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:32:32 smithi083 ceph-mon[77242]: from='client.42663 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:32:32.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:32:32 smithi083 ceph-mon[77242]: pgmap v7329: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:32:32.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:32:32 smithi083 ceph-mon[79930]: from='client.42663 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:32:32.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:32:32 smithi083 ceph-mon[79930]: pgmap v7329: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:32:32.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:32:32 smithi153 ceph-mon[70044]: from='client.42663 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:32:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:32:32 smithi153 ceph-mon[70044]: pgmap v7329: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:32:32.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:32:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:32:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:32:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:32:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:32:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:32:34.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:32:34 smithi083 ceph-mon[77242]: pgmap v7330: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:32:34.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:32:34 smithi083 ceph-mon[79930]: pgmap v7330: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:32:34.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:32:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:32:34.431Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:32:34.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:32:34 smithi153 ceph-mon[70044]: pgmap v7330: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:32:35.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:32:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:32:35.141Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:32:36.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:32:36 smithi083 ceph-mon[79930]: pgmap v7331: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:32:36.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:32:36 smithi083 ceph-mon[77242]: pgmap v7331: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:32:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:32:36 smithi153 ceph-mon[70044]: pgmap v7331: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:32:39.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:32:38 smithi083 ceph-mon[79930]: pgmap v7332: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:32:39.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:32:38 smithi083 ceph-mon[77242]: pgmap v7332: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:32:39.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:32:38 smithi153 ceph-mon[70044]: pgmap v7332: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:32:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:32:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:32:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:32:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:32:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:32:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:32:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:32:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:32:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:32:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:32:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:32:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:32:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:32:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:32:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:32:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:32:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:32:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:32:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:32:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:32:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:32:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:32:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:32:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:32:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:32:40 smithi153 ceph-mon[70044]: pgmap v7333: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:32:41.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:32:40 smithi083 ceph-mon[79930]: pgmap v7333: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:32:41.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:32:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:32:41.049Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:32:41.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:32:40 smithi083 ceph-mon[77242]: pgmap v7333: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:32:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:32:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:32:41] "GET /metrics HTTP/1.1" 200 225219 "" "Prometheus/2.33.4" 2022-09-29T09:32:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:32:42 smithi083 ceph-mon[79930]: pgmap v7334: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:32:42.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:32:42 smithi083 ceph-mon[77242]: pgmap v7334: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:32:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:32:42 smithi153 ceph-mon[70044]: pgmap v7334: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:32:42.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:32:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:32:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:32:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:32:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:32:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:32:44.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:32:44 smithi083 ceph-mon[79930]: pgmap v7335: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:32:44.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:32:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:32:44.431Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:32:44.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:32:44 smithi083 ceph-mon[77242]: pgmap v7335: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:32:44.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:32:44 smithi153 ceph-mon[70044]: pgmap v7335: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:32:45.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:32:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:32:45.141Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:32:46.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:32:46 smithi083 ceph-mon[77242]: pgmap v7336: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:32:46.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:32:46 smithi083 ceph-mon[79930]: pgmap v7336: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:32:46.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:32:46 smithi153 ceph-mon[70044]: pgmap v7336: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:32:49.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:32:48 smithi083 ceph-mon[79930]: pgmap v7337: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:32:49.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:32:48 smithi083 ceph-mon[77242]: pgmap v7337: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:32:49.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:32:48 smithi153 ceph-mon[70044]: pgmap v7337: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:32:50.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:32:50 smithi083 ceph-mon[79930]: pgmap v7338: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:32:50.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:32:50 smithi083 ceph-mon[77242]: pgmap v7338: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:32:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:32:50 smithi153 ceph-mon[70044]: pgmap v7338: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:32:51.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:32:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:32:51.050Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:32:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:32:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:32:51] "GET /metrics HTTP/1.1" 200 225219 "" "Prometheus/2.33.4" 2022-09-29T09:32:52.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:32:52 smithi083 ceph-mon[79930]: pgmap v7339: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:32:52.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:32:52 smithi083 ceph-mon[77242]: pgmap v7339: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:32:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:32:52 smithi153 ceph-mon[70044]: pgmap v7339: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:32:52.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:32:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:32:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:32:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:32:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:32:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:32:54.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:32:54 smithi083 ceph-mon[79930]: pgmap v7340: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:32:54.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:32:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:32:54.432Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:32:54.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:32:54 smithi083 ceph-mon[77242]: pgmap v7340: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:32:54.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:32:54 smithi153 ceph-mon[70044]: pgmap v7340: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:32:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:32:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:32:55.142Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:32:56.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:32:56 smithi083 ceph-mon[79930]: pgmap v7341: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:32:56.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:32:56 smithi083 ceph-mon[77242]: pgmap v7341: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:32:56.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:32:56 smithi153 ceph-mon[70044]: pgmap v7341: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:32:59.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:32:58 smithi083 ceph-mon[79930]: pgmap v7342: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:32:59.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:32:58 smithi083 ceph-mon[77242]: pgmap v7342: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:32:59.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:32:58 smithi153 ceph-mon[70044]: pgmap v7342: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:00.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:33:00 smithi083 ceph-mon[77242]: pgmap v7343: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:00.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:33:00 smithi083 ceph-mon[79930]: pgmap v7343: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:00.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:33:00 smithi153 ceph-mon[70044]: pgmap v7343: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:00.978 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:33:01.271 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:33:01.271 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 3m ago 4h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:33:01.271 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 101s ago 4h 62.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:33:01.271 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 3m ago 4h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:33:01.272 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 101s ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:33:01.272 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 101s ago 4h 443M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:33:01.272 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 3m ago 4h 723M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:33:01.272 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 3m ago 4h 924M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:33:01.273 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 101s ago 4h 407M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:33:01.273 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 3m ago 4h 540M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:33:01.273 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 3m ago 4h 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:33:01.273 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 101s ago 4h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:33:01.273 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 3m ago 4h 666M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:33:01.274 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 3m ago 4h 657M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:33:01.274 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 3m ago 4h 515M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:33:01.274 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 3m ago 4h 639M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:33:01.274 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 101s ago 4h 566M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:33:01.275 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (3h) 101s ago 4h 702M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:33:01.275 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 101s ago 4h 626M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:33:01.275 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 101s ago 4h 739M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:33:01.275 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 101s ago 4h 171M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:33:01.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:33:01 smithi083 ceph-mon[79930]: from='client.42675 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:33:01.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:33:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:33:01.050Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:33:01.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:33:01 smithi083 ceph-mon[77242]: from='client.42675 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:33:01.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:33:01 smithi153 ceph-mon[70044]: from='client.42675 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:33:01.630 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:33:01.630 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:33:01.630 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:33:01.631 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:33:01.631 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:33:01.632 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:33:01.632 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:33:01.632 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:33:01.632 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:33:01.633 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:33:01.633 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:33:01.633 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:33:01.634 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:33:01.634 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:33:01.634 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:33:01.634 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:33:01.634 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:33:01.635 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:33:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:33:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:33:01] "GET /metrics HTTP/1.1" 200 225218 "" "Prometheus/2.33.4" 2022-09-29T09:33:02.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:33:02 smithi083 ceph-mon[79930]: from='client.42681 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:33:02.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:33:02 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2544393860' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:33:02.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:33:02 smithi083 ceph-mon[79930]: pgmap v7344: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:02.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:33:02 smithi083 ceph-mon[77242]: from='client.42681 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:33:02.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:33:02 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2544393860' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:33:02.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:33:02 smithi083 ceph-mon[77242]: pgmap v7344: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:02.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:33:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:33:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:33:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:33:02 smithi153 ceph-mon[70044]: from='client.42681 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:33:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:33:02 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2544393860' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:33:02.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:33:02 smithi153 ceph-mon[70044]: pgmap v7344: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:33:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:33:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:33:04.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:33:04 smithi083 ceph-mon[79930]: pgmap v7345: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:04.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:33:04 smithi083 ceph-mon[77242]: pgmap v7345: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:04.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:33:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:33:04.433Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:33:04.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:33:04 smithi153 ceph-mon[70044]: pgmap v7345: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:05.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:33:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:33:05.143Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:33:06.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:33:06 smithi083 ceph-mon[79930]: pgmap v7346: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:06.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:33:06 smithi083 ceph-mon[77242]: pgmap v7346: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:06.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:33:06 smithi153 ceph-mon[70044]: pgmap v7346: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:09.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:33:08 smithi083 ceph-mon[79930]: pgmap v7347: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:09.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:33:08 smithi083 ceph-mon[77242]: pgmap v7347: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:09.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:33:08 smithi153 ceph-mon[70044]: pgmap v7347: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:10.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:33:10 smithi083 ceph-mon[77242]: pgmap v7348: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:10.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:33:10 smithi083 ceph-mon[79930]: pgmap v7348: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:10.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:33:10 smithi153 ceph-mon[70044]: pgmap v7348: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:11.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:33:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:33:11.051Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:33:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:33:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:33:11] "GET /metrics HTTP/1.1" 200 225220 "" "Prometheus/2.33.4" 2022-09-29T09:33:12.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:33:12 smithi083 ceph-mon[79930]: pgmap v7349: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:12.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:33:12 smithi083 ceph-mon[77242]: pgmap v7349: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:33:12 smithi153 ceph-mon[70044]: pgmap v7349: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:12.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:33:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:33:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:33:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:33:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:33:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:33:14.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:33:14 smithi083 ceph-mon[79930]: pgmap v7350: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:14.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:33:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:33:14.434Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:33:14.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:33:14 smithi083 ceph-mon[77242]: pgmap v7350: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:14.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:33:14 smithi153 ceph-mon[70044]: pgmap v7350: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:15.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:33:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:33:15.143Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:33:16.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:33:16 smithi083 ceph-mon[79930]: pgmap v7351: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:16.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:33:16 smithi083 ceph-mon[77242]: pgmap v7351: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:16.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:33:16 smithi153 ceph-mon[70044]: pgmap v7351: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:19.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:33:18 smithi083 ceph-mon[79930]: pgmap v7352: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:19.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:33:18 smithi083 ceph-mon[77242]: pgmap v7352: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:19.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:33:18 smithi153 ceph-mon[70044]: pgmap v7352: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:20.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:33:20 smithi083 ceph-mon[79930]: pgmap v7353: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:20.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:33:20 smithi083 ceph-mon[77242]: pgmap v7353: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:20.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:33:20 smithi153 ceph-mon[70044]: pgmap v7353: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:21.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:33:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:33:21.051Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:33:21.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:33:21 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:33:21.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:33:21 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:33:21.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:33:21 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:33:21.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:33:21 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:33:21.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:33:21 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:33:21.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:33:21 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:33:21.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:33:21 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:33:21.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:33:21 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:33:21.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:33:21 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:33:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:33:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:33:21] "GET /metrics HTTP/1.1" 200 225220 "" "Prometheus/2.33.4" 2022-09-29T09:33:22.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:33:22 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:33:22.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:33:22 smithi083 ceph-mon[79930]: pgmap v7354: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:22.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:33:22 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:33:22.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:33:22 smithi083 ceph-mon[77242]: pgmap v7354: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:22.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:33:22 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:33:22.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:33:22 smithi153 ceph-mon[70044]: pgmap v7354: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:33:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:33:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:33:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:33:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:33:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:33:24.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:33:24 smithi083 ceph-mon[79930]: pgmap v7355: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:24.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:33:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:33:24.434Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:33:24.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:33:24 smithi083 ceph-mon[77242]: pgmap v7355: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:33:24 smithi153 ceph-mon[70044]: pgmap v7355: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:25.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:33:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:33:25.144Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:33:26.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:33:26 smithi083 ceph-mon[79930]: pgmap v7356: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:26.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:33:26 smithi083 ceph-mon[77242]: pgmap v7356: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:33:26 smithi153 ceph-mon[70044]: pgmap v7356: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:29.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:33:28 smithi083 ceph-mon[79930]: pgmap v7357: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:29.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:33:28 smithi083 ceph-mon[77242]: pgmap v7357: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:29.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:33:28 smithi153 ceph-mon[70044]: pgmap v7357: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:30.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:33:30 smithi083 ceph-mon[77242]: pgmap v7358: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:30.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:33:30 smithi083 ceph-mon[79930]: pgmap v7358: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:33:30 smithi153 ceph-mon[70044]: pgmap v7358: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:31.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:33:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:33:31.053Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:33:31.950 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:33:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:33:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:33:31] "GET /metrics HTTP/1.1" 200 225219 "" "Prometheus/2.33.4" 2022-09-29T09:33:32.242 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:33:32.243 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 4m ago 4h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:33:32.243 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 2m ago 4h 62.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:33:32.243 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 4m ago 4h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:33:32.244 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 2m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:33:32.244 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 2m ago 4h 443M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:33:32.244 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 4m ago 4h 723M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:33:32.244 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 4m ago 4h 924M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:33:32.245 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 2m ago 4h 407M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:33:32.245 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 4m ago 4h 540M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:33:32.245 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 4m ago 4h 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:33:32.245 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 2m ago 4h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:33:32.246 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 4m ago 4h 666M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:33:32.246 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 4m ago 4h 657M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:33:32.246 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 4m ago 4h 515M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:33:32.246 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 4m ago 4h 639M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:33:32.247 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 2m ago 4h 566M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:33:32.247 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 2m ago 4h 702M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:33:32.247 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (3h) 2m ago 4h 626M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:33:32.247 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 2m ago 4h 739M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:33:32.248 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 2m ago 4h 171M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:33:32.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:33:32 smithi083 ceph-mon[77242]: from='client.42693 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:33:32.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:33:32 smithi083 ceph-mon[77242]: pgmap v7359: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:32.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:33:32 smithi083 ceph-mon[79930]: from='client.42693 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:33:32.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:33:32 smithi083 ceph-mon[79930]: pgmap v7359: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:33:32 smithi153 ceph-mon[70044]: from='client.42693 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:33:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:33:32 smithi153 ceph-mon[70044]: pgmap v7359: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:32.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:33:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:33:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:33:32.599 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:33:32.599 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:33:32.599 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:33:32.600 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:33:32.600 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:33:32.600 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:33:32.600 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:33:32.601 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:33:32.601 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:33:32.601 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:33:32.601 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:33:32.602 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:33:32.602 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:33:32.602 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:33:32.602 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:33:32.602 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:33:32.603 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:33:32.603 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:33:33.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:33:33 smithi153 ceph-mon[70044]: from='client.42699 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:33:33.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:33:33 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3111326348' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:33:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:33:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:33:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:33:33.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:33:33 smithi083 ceph-mon[79930]: from='client.42699 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:33:33.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:33:33 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3111326348' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:33:33.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:33:33 smithi083 ceph-mon[77242]: from='client.42699 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:33:33.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:33:33 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3111326348' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:33:34.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:33:34 smithi083 ceph-mon[79930]: pgmap v7360: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:34.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:33:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:33:34.435Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:33:34.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:33:34 smithi083 ceph-mon[77242]: pgmap v7360: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:33:34 smithi153 ceph-mon[70044]: pgmap v7360: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:35.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:33:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:33:35.144Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:33:36.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:33:36 smithi083 ceph-mon[79930]: pgmap v7361: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:36.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:33:36 smithi083 ceph-mon[77242]: pgmap v7361: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:33:36 smithi153 ceph-mon[70044]: pgmap v7361: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:39.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:33:38 smithi083 ceph-mon[77242]: pgmap v7362: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:39.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:33:38 smithi083 ceph-mon[79930]: pgmap v7362: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:39.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:33:38 smithi153 ceph-mon[70044]: pgmap v7362: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:33:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:33:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:33:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:33:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:33:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:33:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:33:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:33:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:33:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:33:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:33:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:33:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:33:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:33:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:33:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:33:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:33:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:33:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:33:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:33:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:33:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:33:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:33:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:33:41.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:33:40 smithi153 ceph-mon[70044]: pgmap v7363: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:41.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:33:40 smithi083 ceph-mon[79930]: pgmap v7363: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:41.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:33:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:33:41.053Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:33:41.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:33:40 smithi083 ceph-mon[77242]: pgmap v7363: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:33:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:33:41] "GET /metrics HTTP/1.1" 200 225216 "" "Prometheus/2.33.4" 2022-09-29T09:33:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:33:42 smithi083 ceph-mon[79930]: pgmap v7364: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:42.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:33:42 smithi083 ceph-mon[77242]: pgmap v7364: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:33:42 smithi153 ceph-mon[70044]: pgmap v7364: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:42.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:33:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:33:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:33:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:33:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:33:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:33:44.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:33:44 smithi083 ceph-mon[79930]: pgmap v7365: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:44.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:33:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:33:44.436Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:33:44.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:33:44 smithi083 ceph-mon[77242]: pgmap v7365: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:44.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:33:44 smithi153 ceph-mon[70044]: pgmap v7365: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:45.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:33:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:33:45.145Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:33:46.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:33:46 smithi083 ceph-mon[79930]: pgmap v7366: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:46.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:33:46 smithi083 ceph-mon[77242]: pgmap v7366: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:46.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:33:46 smithi153 ceph-mon[70044]: pgmap v7366: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:49.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:33:48 smithi083 ceph-mon[79930]: pgmap v7367: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:49.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:33:48 smithi083 ceph-mon[77242]: pgmap v7367: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:49.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:33:48 smithi153 ceph-mon[70044]: pgmap v7367: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:50.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:33:50 smithi083 ceph-mon[79930]: pgmap v7368: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:50.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:33:50 smithi083 ceph-mon[77242]: pgmap v7368: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:50.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:33:50 smithi153 ceph-mon[70044]: pgmap v7368: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:51.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:33:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:33:51.054Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:33:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:33:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:33:51] "GET /metrics HTTP/1.1" 200 225216 "" "Prometheus/2.33.4" 2022-09-29T09:33:52.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:33:52 smithi083 ceph-mon[79930]: pgmap v7369: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:52.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:33:52 smithi083 ceph-mon[77242]: pgmap v7369: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:33:52 smithi153 ceph-mon[70044]: pgmap v7369: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:52.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:33:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:33:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:33:53.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:33:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:33:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:33:54.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:33:54 smithi083 ceph-mon[79930]: pgmap v7370: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:54.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:33:54 smithi083 ceph-mon[77242]: pgmap v7370: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:54.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:33:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:33:54.436Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:33:54.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:33:54 smithi153 ceph-mon[70044]: pgmap v7370: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:33:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:33:55.146Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:33:56.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:33:56 smithi083 ceph-mon[79930]: pgmap v7371: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:56.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:33:56 smithi083 ceph-mon[77242]: pgmap v7371: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:56.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:33:56 smithi153 ceph-mon[70044]: pgmap v7371: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:59.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:33:58 smithi083 ceph-mon[79930]: pgmap v7372: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:59.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:33:58 smithi083 ceph-mon[77242]: pgmap v7372: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:33:59.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:33:58 smithi153 ceph-mon[70044]: pgmap v7372: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:00.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:34:00 smithi083 ceph-mon[79930]: pgmap v7373: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:00.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:34:00 smithi083 ceph-mon[77242]: pgmap v7373: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:00.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:34:00 smithi153 ceph-mon[70044]: pgmap v7373: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:01.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:34:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:34:01.055Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:34:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:34:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:34:01] "GET /metrics HTTP/1.1" 200 225219 "" "Prometheus/2.33.4" 2022-09-29T09:34:02.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:34:02 smithi083 ceph-mon[79930]: pgmap v7374: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:02.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:34:02 smithi083 ceph-mon[77242]: pgmap v7374: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:34:02 smithi153 ceph-mon[70044]: pgmap v7374: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:02.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:34:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:34:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:34:02.925 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:34:03.213 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:34:03.214 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 4m ago 4h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:34:03.214 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 2m ago 4h 62.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:34:03.214 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 4m ago 4h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:34:03.214 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 2m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:34:03.215 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 2m ago 4h 443M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:34:03.215 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 4m ago 4h 723M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:34:03.215 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 4m ago 4h 924M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:34:03.215 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 2m ago 4h 407M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:34:03.215 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 4m ago 4h 540M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:34:03.216 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 4m ago 4h 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:34:03.216 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 2m ago 4h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:34:03.216 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 4m ago 4h 666M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:34:03.216 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 4m ago 4h 657M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:34:03.216 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 4m ago 4h 515M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:34:03.217 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 4m ago 4h 639M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:34:03.217 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 2m ago 4h 566M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:34:03.217 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 2m ago 4h 702M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:34:03.217 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 2m ago 4h 626M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:34:03.217 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (3h) 2m ago 4h 739M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:34:03.218 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 2m ago 4h 171M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:34:03.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:34:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:34:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:34:03.582 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:34:03.582 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:34:03.582 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:34:03.583 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:34:03.583 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:34:03.583 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:34:03.583 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:34:03.584 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:34:03.584 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:34:03.584 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:34:03.584 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:34:03.584 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:34:03.585 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:34:03.585 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:34:03.585 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:34:03.585 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:34:03.585 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:34:03.586 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:34:04.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:34:03 smithi083 ceph-mon[79930]: from='client.52642 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:34:04.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:34:03 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2873068679' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:34:04.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:34:03 smithi083 ceph-mon[77242]: from='client.52642 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:34:04.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:34:03 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2873068679' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:34:04.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:34:03 smithi153 ceph-mon[70044]: from='client.52642 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:34:04.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:34:03 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2873068679' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:34:04.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:34:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:34:04.436Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:34:05.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:34:04 smithi083 ceph-mon[79930]: from='client.42717 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:34:05.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:34:04 smithi083 ceph-mon[79930]: pgmap v7375: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:05.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:34:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:34:05.147Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:34:05.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:34:04 smithi083 ceph-mon[77242]: from='client.42717 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:34:05.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:34:04 smithi083 ceph-mon[77242]: pgmap v7375: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:05.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:34:04 smithi153 ceph-mon[70044]: from='client.42717 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:34:05.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:34:04 smithi153 ceph-mon[70044]: pgmap v7375: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:06.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:34:06 smithi083 ceph-mon[79930]: pgmap v7376: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:06.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:34:06 smithi083 ceph-mon[77242]: pgmap v7376: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:06.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:34:06 smithi153 ceph-mon[70044]: pgmap v7376: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:09.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:34:08 smithi083 ceph-mon[77242]: pgmap v7377: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:09.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:34:08 smithi083 ceph-mon[79930]: pgmap v7377: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:09.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:34:08 smithi153 ceph-mon[70044]: pgmap v7377: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:10.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:34:10 smithi083 ceph-mon[77242]: pgmap v7378: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:10.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:34:10 smithi083 ceph-mon[79930]: pgmap v7378: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:34:10 smithi153 ceph-mon[70044]: pgmap v7378: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:11.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:34:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:34:11.055Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:34:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:34:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:34:11] "GET /metrics HTTP/1.1" 200 225221 "" "Prometheus/2.33.4" 2022-09-29T09:34:12.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:34:12 smithi083 ceph-mon[79930]: pgmap v7379: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:12.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:34:12 smithi083 ceph-mon[77242]: pgmap v7379: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:34:12 smithi153 ceph-mon[70044]: pgmap v7379: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:12.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:34:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:34:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:34:13.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:34:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:34:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:34:14.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:34:14 smithi083 ceph-mon[79930]: pgmap v7380: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:14.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:34:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:34:14.438Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:34:14.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:34:14 smithi083 ceph-mon[77242]: pgmap v7380: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:14.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:34:14 smithi153 ceph-mon[70044]: pgmap v7380: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:15.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:34:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:34:15.147Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:34:16.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:34:16 smithi083 ceph-mon[79930]: pgmap v7381: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:16.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:34:16 smithi083 ceph-mon[77242]: pgmap v7381: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:34:16 smithi153 ceph-mon[70044]: pgmap v7381: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:19.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:34:18 smithi083 ceph-mon[79930]: pgmap v7382: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:19.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:34:18 smithi083 ceph-mon[77242]: pgmap v7382: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:19.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:34:18 smithi153 ceph-mon[70044]: pgmap v7382: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:20.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:34:20 smithi083 ceph-mon[79930]: pgmap v7383: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:20.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:34:20 smithi083 ceph-mon[77242]: pgmap v7383: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:34:20 smithi153 ceph-mon[70044]: pgmap v7383: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:21.441 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:34:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:34:21.056Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:34:21.442 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:34:21 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:34:21.442 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:34:21 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:34:21.443 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:34:21 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:34:21.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:34:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:34:21] "GET /metrics HTTP/1.1" 200 225221 "" "Prometheus/2.33.4" 2022-09-29T09:34:21.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:34:21 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:34:21.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:34:21 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:34:21.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:34:21 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:34:21.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:34:21 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:34:21.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:34:21 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:34:21.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:34:21 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:34:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:34:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:34:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:34:23.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:34:22 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:34:23.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:34:22 smithi153 ceph-mon[70044]: pgmap v7384: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:34:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:34:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:34:23.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:34:22 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:34:23.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:34:22 smithi083 ceph-mon[79930]: pgmap v7384: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:23.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:34:22 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:34:23.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:34:22 smithi083 ceph-mon[77242]: pgmap v7384: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:24.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:34:24 smithi083 ceph-mon[79930]: pgmap v7385: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:24.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:34:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:34:24.437Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:34:24.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:34:24 smithi083 ceph-mon[77242]: pgmap v7385: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:24.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:34:24 smithi153 ceph-mon[70044]: pgmap v7385: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:25.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:34:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:34:25.148Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:34:26.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:34:26 smithi083 ceph-mon[79930]: pgmap v7386: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:26.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:34:26 smithi083 ceph-mon[77242]: pgmap v7386: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:26.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:34:26 smithi153 ceph-mon[70044]: pgmap v7386: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:29.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:34:28 smithi083 ceph-mon[79930]: pgmap v7387: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:29.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:34:28 smithi083 ceph-mon[77242]: pgmap v7387: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:29.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:34:28 smithi153 ceph-mon[70044]: pgmap v7387: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:30.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:34:30 smithi083 ceph-mon[77242]: pgmap v7388: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:30.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:34:30 smithi083 ceph-mon[79930]: pgmap v7388: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:30.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:34:30 smithi153 ceph-mon[70044]: pgmap v7388: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:31.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:34:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:34:31.057Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:34:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:34:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:34:31] "GET /metrics HTTP/1.1" 200 225216 "" "Prometheus/2.33.4" 2022-09-29T09:34:32.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:34:32 smithi083 ceph-mon[77242]: pgmap v7389: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:32.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:34:32 smithi083 ceph-mon[79930]: pgmap v7389: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:32.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:34:32 smithi153 ceph-mon[70044]: pgmap v7389: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:32.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:34:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:34:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:34:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:34:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:34:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:34:33.908 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:34:34.198 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:34:34.198 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 5m ago 4h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:34:34.199 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 3m ago 4h 62.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:34:34.199 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 5m ago 4h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:34:34.199 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 3m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:34:34.199 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 3m ago 4h 443M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:34:34.200 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 5m ago 4h 723M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:34:34.200 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 5m ago 4h 924M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:34:34.200 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 3m ago 4h 407M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:34:34.200 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 5m ago 4h 540M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:34:34.200 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 5m ago 4h 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:34:34.201 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 3m ago 4h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:34:34.201 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 5m ago 4h 666M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:34:34.201 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 5m ago 4h 657M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:34:34.201 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 5m ago 4h 515M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:34:34.202 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 5m ago 4h 639M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:34:34.202 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 3m ago 4h 566M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:34:34.202 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 3m ago 4h 702M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:34:34.202 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 3m ago 4h 626M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:34:34.203 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 3m ago 4h 739M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:34:34.203 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 3m ago 4h 171M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:34:34.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:34:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:34:34.438Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:34:34.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:34:34 smithi083 ceph-mon[79930]: from='client.52660 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:34:34.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:34:34 smithi083 ceph-mon[79930]: pgmap v7390: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:34.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:34:34 smithi083 ceph-mon[77242]: from='client.52660 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:34:34.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:34:34 smithi083 ceph-mon[77242]: pgmap v7390: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:34.556 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:34:34.557 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:34:34.557 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:34:34.557 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:34:34.557 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:34:34.558 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:34:34.558 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:34:34.558 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:34:34.558 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:34:34.559 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:34:34.559 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:34:34.559 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:34:34.559 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:34:34.559 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:34:34.560 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:34:34.560 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:34:34.560 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:34:34.560 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:34:34.561 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:34:34 smithi153 ceph-mon[70044]: from='client.52660 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:34:34.561 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:34:34 smithi153 ceph-mon[70044]: pgmap v7390: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:35.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:34:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:34:35.148Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:34:35.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:34:35 smithi083 ceph-mon[77242]: from='client.42735 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:34:35.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:34:35 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1299266180' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:34:35.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:34:35 smithi083 ceph-mon[79930]: from='client.42735 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:34:35.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:34:35 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1299266180' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:34:35.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:34:35 smithi153 ceph-mon[70044]: from='client.42735 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:34:35.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:34:35 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1299266180' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:34:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:34:36 smithi153 ceph-mon[70044]: pgmap v7391: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:36.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:34:36 smithi083 ceph-mon[79930]: pgmap v7391: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:36.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:34:36 smithi083 ceph-mon[77242]: pgmap v7391: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:39.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:34:38 smithi083 ceph-mon[79930]: pgmap v7392: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:39.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:34:38 smithi083 ceph-mon[77242]: pgmap v7392: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:39.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:34:38 smithi153 ceph-mon[70044]: pgmap v7392: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:34:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:34:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:34:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:34:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:34:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:34:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:34:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:34:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:34:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:34:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:34:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:34:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:34:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:34:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:34:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:34:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:34:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:34:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:34:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:34:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:34:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:34:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:34:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:34:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:34:41 smithi153 ceph-mon[70044]: pgmap v7393: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:41.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:34:41 smithi083 ceph-mon[79930]: pgmap v7393: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:41.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:34:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:34:41.058Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:34:41.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:34:41 smithi083 ceph-mon[77242]: pgmap v7393: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:34:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:34:41] "GET /metrics HTTP/1.1" 200 225218 "" "Prometheus/2.33.4" 2022-09-29T09:34:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:34:42 smithi083 ceph-mon[79930]: pgmap v7394: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:42.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:34:42 smithi083 ceph-mon[77242]: pgmap v7394: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:34:42 smithi153 ceph-mon[70044]: pgmap v7394: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:42.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:34:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:34:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:34:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:34:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:34:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:34:44.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:34:44 smithi083 ceph-mon[77242]: pgmap v7395: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:44.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:34:44 smithi083 ceph-mon[79930]: pgmap v7395: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:44.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:34:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:34:44.440Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:34:44.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:34:44 smithi153 ceph-mon[70044]: pgmap v7395: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:45.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:34:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:34:45.149Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:34:46.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:34:46 smithi083 ceph-mon[79930]: pgmap v7396: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:46.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:34:46 smithi083 ceph-mon[77242]: pgmap v7396: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:46.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:34:46 smithi153 ceph-mon[70044]: pgmap v7396: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:49.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:34:48 smithi083 ceph-mon[79930]: pgmap v7397: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:49.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:34:48 smithi083 ceph-mon[77242]: pgmap v7397: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:49.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:34:48 smithi153 ceph-mon[70044]: pgmap v7397: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:50.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:34:50 smithi083 ceph-mon[79930]: pgmap v7398: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:50.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:34:50 smithi083 ceph-mon[77242]: pgmap v7398: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:34:50 smithi153 ceph-mon[70044]: pgmap v7398: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:51.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:34:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:34:51.058Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:34:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:34:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:34:51] "GET /metrics HTTP/1.1" 200 225218 "" "Prometheus/2.33.4" 2022-09-29T09:34:52.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:34:52 smithi083 ceph-mon[77242]: pgmap v7399: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:52.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:34:52 smithi083 ceph-mon[79930]: pgmap v7399: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:34:52 smithi153 ceph-mon[70044]: pgmap v7399: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:52.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:34:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:34:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:34:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:34:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:34:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:34:54.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:34:54 smithi153 ceph-mon[70044]: pgmap v7400: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:54.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:34:54 smithi083 ceph-mon[77242]: pgmap v7400: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:54.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:34:54 smithi083 ceph-mon[79930]: pgmap v7400: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:54.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:34:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:34:54.440Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:34:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:34:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:34:55.149Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:34:56.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:34:56 smithi153 ceph-mon[70044]: pgmap v7401: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:56.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:34:56 smithi083 ceph-mon[77242]: pgmap v7401: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:56.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:34:56 smithi083 ceph-mon[79930]: pgmap v7401: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:59.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:34:58 smithi083 ceph-mon[77242]: pgmap v7402: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:59.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:34:58 smithi083 ceph-mon[79930]: pgmap v7402: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:34:59.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:34:58 smithi153 ceph-mon[70044]: pgmap v7402: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:35:00 smithi153 ceph-mon[70044]: pgmap v7403: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:00.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:35:00 smithi083 ceph-mon[79930]: pgmap v7403: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:00.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:35:00 smithi083 ceph-mon[77242]: pgmap v7403: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:01.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:35:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:35:01.059Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:35:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:35:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:35:01] "GET /metrics HTTP/1.1" 200 225221 "" "Prometheus/2.33.4" 2022-09-29T09:35:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:35:02 smithi153 ceph-mon[70044]: pgmap v7404: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:02.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:35:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:35:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:35:02.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:35:02 smithi083 ceph-mon[79930]: pgmap v7404: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:02.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:35:02 smithi083 ceph-mon[77242]: pgmap v7404: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:35:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:35:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:35:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:35:04 smithi153 ceph-mon[70044]: pgmap v7405: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:04.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:35:04 smithi083 ceph-mon[79930]: pgmap v7405: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:04.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:35:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:35:04.442Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:35:04.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:35:04 smithi083 ceph-mon[77242]: pgmap v7405: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:04.883 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:35:05.174 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:35:05.175 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 5m ago 4h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:35:05.175 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 3m ago 4h 62.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:35:05.175 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 5m ago 4h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:35:05.176 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 3m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:35:05.176 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 3m ago 4h 443M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:35:05.176 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 5m ago 4h 723M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:35:05.176 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 5m ago 4h 924M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:35:05.177 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 3m ago 4h 407M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:35:05.177 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 5m ago 4h 540M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:35:05.177 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 5m ago 4h 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:35:05.177 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 3m ago 4h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:35:05.177 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 5m ago 4h 666M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:35:05.178 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 5m ago 4h 657M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:35:05.178 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 5m ago 4h 515M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:35:05.178 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 5m ago 4h 639M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:35:05.178 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 3m ago 4h 566M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:35:05.179 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 3m ago 4h 702M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:35:05.179 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 3m ago 4h 626M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:35:05.179 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 3m ago 4h 739M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:35:05.179 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 3m ago 4h 171M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:35:05.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:35:05 smithi083 ceph-mon[79930]: from='client.52678 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:35:05.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:35:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:35:05.151Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:35:05.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:35:05 smithi083 ceph-mon[77242]: from='client.52678 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:35:05.531 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:35:05.531 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:35:05.531 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:35:05.532 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:35:05.532 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:35:05.532 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:35:05.533 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:35:05.533 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:35:05.533 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:35:05.533 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:35:05.533 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:35:05.534 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:35:05.534 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:35:05.534 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:35:05.534 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:35:05.535 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:35:05.535 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:35:05.535 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:35:05.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:35:05 smithi153 ceph-mon[70044]: from='client.52678 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:35:06.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:35:06 smithi153 ceph-mon[70044]: from='client.52684 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:35:06.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:35:06 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2733556573' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:35:06.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:35:06 smithi153 ceph-mon[70044]: pgmap v7406: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:06.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:35:06 smithi083 ceph-mon[79930]: from='client.52684 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:35:06.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:35:06 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2733556573' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:35:06.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:35:06 smithi083 ceph-mon[79930]: pgmap v7406: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:06.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:35:06 smithi083 ceph-mon[77242]: from='client.52684 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:35:06.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:35:06 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2733556573' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:35:06.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:35:06 smithi083 ceph-mon[77242]: pgmap v7406: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:09.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:35:08 smithi083 ceph-mon[79930]: pgmap v7407: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:09.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:35:08 smithi083 ceph-mon[77242]: pgmap v7407: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:09.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:35:08 smithi153 ceph-mon[70044]: pgmap v7407: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:10.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:35:10 smithi153 ceph-mon[70044]: pgmap v7408: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:10.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:35:10 smithi083 ceph-mon[77242]: pgmap v7408: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:10.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:35:10 smithi083 ceph-mon[79930]: pgmap v7408: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:11.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:35:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:35:11.060Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:35:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:35:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:35:11] "GET /metrics HTTP/1.1" 200 225218 "" "Prometheus/2.33.4" 2022-09-29T09:35:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:35:12 smithi153 ceph-mon[70044]: pgmap v7409: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:12.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:35:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:35:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:35:12.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:35:12 smithi083 ceph-mon[79930]: pgmap v7409: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:12.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:35:12 smithi083 ceph-mon[77242]: pgmap v7409: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:35:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:35:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:35:14.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:35:14 smithi153 ceph-mon[70044]: pgmap v7410: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:14.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:35:14 smithi083 ceph-mon[79930]: pgmap v7410: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:14.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:35:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:35:14.442Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:35:14.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:35:14 smithi083 ceph-mon[77242]: pgmap v7410: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:15.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:35:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:35:15.151Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:35:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:35:16 smithi153 ceph-mon[70044]: pgmap v7411: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:16.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:35:16 smithi083 ceph-mon[79930]: pgmap v7411: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:16.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:35:16 smithi083 ceph-mon[77242]: pgmap v7411: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:19.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:35:18 smithi083 ceph-mon[79930]: pgmap v7412: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:19.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:35:18 smithi083 ceph-mon[77242]: pgmap v7412: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:19.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:35:18 smithi153 ceph-mon[70044]: pgmap v7412: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:20.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:35:20 smithi153 ceph-mon[70044]: pgmap v7413: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:20.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:35:20 smithi083 ceph-mon[79930]: pgmap v7413: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:20.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:35:20 smithi083 ceph-mon[77242]: pgmap v7413: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:21.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:35:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:35:21.061Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:35:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:35:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:35:21] "GET /metrics HTTP/1.1" 200 225218 "" "Prometheus/2.33.4" 2022-09-29T09:35:22.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:35:22 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:35:22.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:35:22 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:35:22.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:35:22 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:35:22.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:35:22 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:35:22.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:35:22 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:35:22.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:35:22 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:35:22.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:35:22 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:35:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:35:22 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:35:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:35:22 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:35:22.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:35:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:35:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:35:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:35:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:35:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:35:23.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:35:23 smithi153 ceph-mon[70044]: pgmap v7414: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:23.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:35:23 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:35:23.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:35:23 smithi083 ceph-mon[77242]: pgmap v7414: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:23.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:35:23 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:35:23.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:35:23 smithi083 ceph-mon[79930]: pgmap v7414: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:23.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:35:23 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:35:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:35:24 smithi153 ceph-mon[70044]: pgmap v7415: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:24.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:35:24 smithi083 ceph-mon[79930]: pgmap v7415: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:24.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:35:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:35:24.443Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:35:24.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:35:24 smithi083 ceph-mon[77242]: pgmap v7415: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:25.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:35:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:35:25.153Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:35:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:35:26 smithi153 ceph-mon[70044]: pgmap v7416: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:26.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:35:26 smithi083 ceph-mon[79930]: pgmap v7416: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:26.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:35:26 smithi083 ceph-mon[77242]: pgmap v7416: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:29.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:35:28 smithi083 ceph-mon[79930]: pgmap v7417: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:29.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:35:28 smithi083 ceph-mon[77242]: pgmap v7417: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:29.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:35:28 smithi153 ceph-mon[70044]: pgmap v7417: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:35:30 smithi153 ceph-mon[70044]: pgmap v7418: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:30.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:35:30 smithi083 ceph-mon[79930]: pgmap v7418: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:30.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:35:30 smithi083 ceph-mon[77242]: pgmap v7418: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:31.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:35:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:35:31.061Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:35:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:35:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:35:31] "GET /metrics HTTP/1.1" 200 225222 "" "Prometheus/2.33.4" 2022-09-29T09:35:32.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:35:32 smithi153 ceph-mon[70044]: pgmap v7419: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:32.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:35:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:35:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:35:32.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:35:32 smithi083 ceph-mon[79930]: pgmap v7419: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:32.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:35:32 smithi083 ceph-mon[77242]: pgmap v7419: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:35:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:35:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:35:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:35:34 smithi153 ceph-mon[70044]: pgmap v7420: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:34.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:35:34 smithi083 ceph-mon[79930]: pgmap v7420: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:34.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:35:34 smithi083 ceph-mon[77242]: pgmap v7420: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:34.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:35:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:35:34.444Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:35:35.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:35:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:35:35.152Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:35:35.855 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:35:36.143 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:35:36.143 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 6m ago 4h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:35:36.143 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 4m ago 4h 62.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:35:36.144 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 6m ago 4h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:35:36.144 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 4m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:35:36.144 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 4m ago 4h 443M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:35:36.144 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 6m ago 4h 723M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:35:36.145 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 6m ago 4h 924M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:35:36.145 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 4m ago 4h 407M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:35:36.145 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 6m ago 4h 540M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:35:36.145 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 6m ago 4h 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:35:36.145 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 4m ago 4h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:35:36.146 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 6m ago 4h 666M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:35:36.146 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 6m ago 4h 657M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:35:36.146 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 6m ago 4h 515M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:35:36.147 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 6m ago 4h 639M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:35:36.147 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 4m ago 4h 566M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:35:36.147 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 4m ago 4h 702M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:35:36.147 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 4m ago 4h 626M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:35:36.148 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 4m ago 4h 739M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:35:36.148 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 4m ago 4h 171M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:35:36.505 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:35:36.506 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:35:36.506 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:35:36.506 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:35:36.507 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:35:36.507 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:35:36.507 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:35:36.507 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:35:36.508 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:35:36.508 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:35:36.508 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:35:36.508 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:35:36.509 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:35:36.509 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:35:36.509 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:35:36.509 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:35:36.510 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:35:36.510 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:35:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:35:36 smithi153 ceph-mon[70044]: from='client.42765 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:35:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:35:36 smithi153 ceph-mon[70044]: from='client.42771 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:35:36.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:35:36 smithi153 ceph-mon[70044]: pgmap v7421: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:36.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:35:36 smithi083 ceph-mon[79930]: from='client.42765 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:35:36.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:35:36 smithi083 ceph-mon[79930]: from='client.42771 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:35:36.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:35:36 smithi083 ceph-mon[79930]: pgmap v7421: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:36.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:35:36 smithi083 ceph-mon[77242]: from='client.42765 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:35:36.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:35:36 smithi083 ceph-mon[77242]: from='client.42771 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:35:36.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:35:36 smithi083 ceph-mon[77242]: pgmap v7421: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:37.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:35:37 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/239682911' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:35:37.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:35:37 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/239682911' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:35:37.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:35:37 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/239682911' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:35:39.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:35:38 smithi083 ceph-mon[79930]: pgmap v7422: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:39.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:35:38 smithi083 ceph-mon[77242]: pgmap v7422: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:39.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:35:38 smithi153 ceph-mon[70044]: pgmap v7422: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:35:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:35:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:35:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:35:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:35:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:35:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:35:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:35:40.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:35:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:35:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:35:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:35:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:35:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:35:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:35:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:35:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:35:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:35:40.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:35:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:35:40.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:35:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:35:40.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:35:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:35:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:35:41 smithi153 ceph-mon[70044]: pgmap v7423: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:41.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:35:41 smithi083 ceph-mon[79930]: pgmap v7423: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:41.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:35:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:35:41.061Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:35:41.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:35:41 smithi083 ceph-mon[77242]: pgmap v7423: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:35:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:35:41] "GET /metrics HTTP/1.1" 200 225218 "" "Prometheus/2.33.4" 2022-09-29T09:35:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:35:42 smithi153 ceph-mon[70044]: pgmap v7424: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:42.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:35:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:35:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:35:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:35:42 smithi083 ceph-mon[79930]: pgmap v7424: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:35:42 smithi083 ceph-mon[77242]: pgmap v7424: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:35:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:35:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:35:44.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:35:44 smithi153 ceph-mon[70044]: pgmap v7425: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:44.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:35:44 smithi083 ceph-mon[79930]: pgmap v7425: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:44.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:35:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:35:44.445Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:35:44.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:35:44 smithi083 ceph-mon[77242]: pgmap v7425: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:45.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:35:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:35:45.152Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:35:46.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:35:46 smithi153 ceph-mon[70044]: pgmap v7426: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:46.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:35:46 smithi083 ceph-mon[77242]: pgmap v7426: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:46.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:35:46 smithi083 ceph-mon[79930]: pgmap v7426: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:49.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:35:48 smithi083 ceph-mon[79930]: pgmap v7427: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:49.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:35:48 smithi083 ceph-mon[77242]: pgmap v7427: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:49.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:35:48 smithi153 ceph-mon[70044]: pgmap v7427: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:35:50 smithi153 ceph-mon[70044]: pgmap v7428: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:50.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:35:50 smithi083 ceph-mon[79930]: pgmap v7428: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:50.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:35:50 smithi083 ceph-mon[77242]: pgmap v7428: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:51.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:35:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:35:51.062Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:35:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:35:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:35:51] "GET /metrics HTTP/1.1" 200 225218 "" "Prometheus/2.33.4" 2022-09-29T09:35:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:35:52 smithi153 ceph-mon[70044]: pgmap v7429: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:52.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:35:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:35:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:35:52.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:35:52 smithi083 ceph-mon[79930]: pgmap v7429: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:52.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:35:52 smithi083 ceph-mon[77242]: pgmap v7429: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:53.311 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:35:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:35:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:35:54.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:35:54 smithi153 ceph-mon[70044]: pgmap v7430: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:54.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:35:54 smithi083 ceph-mon[79930]: pgmap v7430: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:54.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:35:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:35:54.445Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:35:54.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:35:54 smithi083 ceph-mon[77242]: pgmap v7430: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:35:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:35:55.152Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:35:56.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:35:56 smithi153 ceph-mon[70044]: pgmap v7431: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:56.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:35:56 smithi083 ceph-mon[79930]: pgmap v7431: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:56.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:35:56 smithi083 ceph-mon[77242]: pgmap v7431: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:59.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:35:58 smithi083 ceph-mon[79930]: pgmap v7432: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:59.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:35:58 smithi083 ceph-mon[77242]: pgmap v7432: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:35:59.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:35:58 smithi153 ceph-mon[70044]: pgmap v7432: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:36:00 smithi153 ceph-mon[70044]: pgmap v7433: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:00.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:36:00 smithi083 ceph-mon[79930]: pgmap v7433: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:00.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:36:00 smithi083 ceph-mon[77242]: pgmap v7433: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:01.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:36:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:36:01.063Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:36:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:36:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:36:01] "GET /metrics HTTP/1.1" 200 225221 "" "Prometheus/2.33.4" 2022-09-29T09:36:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:36:02 smithi153 ceph-mon[70044]: pgmap v7434: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:02.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:36:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:36:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:36:02.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:36:02 smithi083 ceph-mon[79930]: pgmap v7434: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:02.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:36:02 smithi083 ceph-mon[77242]: pgmap v7434: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:36:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:36:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:36:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:36:04 smithi153 ceph-mon[70044]: pgmap v7435: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:04.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:36:04 smithi083 ceph-mon[79930]: pgmap v7435: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:04.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:36:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:36:04.446Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:36:04.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:36:04 smithi083 ceph-mon[77242]: pgmap v7435: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:05.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:36:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:36:05.153Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:36:06.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:36:06 smithi153 ceph-mon[70044]: pgmap v7436: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:06.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:36:06 smithi083 ceph-mon[79930]: pgmap v7436: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:06.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:36:06 smithi083 ceph-mon[77242]: pgmap v7436: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:06.831 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:36:07.118 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:36:07.118 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 6m ago 4h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:36:07.118 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 4m ago 4h 62.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:36:07.119 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 6m ago 4h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:36:07.119 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 4m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:36:07.119 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 4m ago 4h 443M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:36:07.119 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 6m ago 4h 723M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:36:07.119 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 6m ago 4h 924M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:36:07.120 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 4m ago 4h 407M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:36:07.120 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 6m ago 4h 540M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:36:07.120 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 6m ago 4h 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:36:07.120 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 4m ago 4h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:36:07.120 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 6m ago 4h 666M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:36:07.121 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 6m ago 4h 657M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:36:07.121 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 6m ago 4h 515M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:36:07.121 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 6m ago 4h 639M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:36:07.121 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 4m ago 4h 566M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:36:07.122 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 4m ago 4h 702M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:36:07.122 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 4m ago 4h 626M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:36:07.122 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 4m ago 4h 739M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:36:07.122 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 4m ago 4h 171M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:36:07.474 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:36:07.474 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:36:07.474 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:36:07.475 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:36:07.475 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:36:07.475 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:36:07.475 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:36:07.475 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:36:07.476 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:36:07.476 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:36:07.476 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:36:07.476 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:36:07.477 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:36:07.477 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:36:07.477 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:36:07.477 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:36:07.477 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:36:07.478 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:36:07.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:36:07 smithi153 ceph-mon[70044]: from='client.42783 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:36:07.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:36:07 smithi153 ceph-mon[70044]: from='client.52720 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:36:07.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:36:07 smithi083 ceph-mon[79930]: from='client.42783 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:36:07.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:36:07 smithi083 ceph-mon[79930]: from='client.52720 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:36:07.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:36:07 smithi083 ceph-mon[77242]: from='client.42783 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:36:07.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:36:07 smithi083 ceph-mon[77242]: from='client.52720 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:36:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:36:08 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/393037284' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:36:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:36:08 smithi153 ceph-mon[70044]: pgmap v7437: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:08.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:36:08 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/393037284' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:36:08.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:36:08 smithi083 ceph-mon[79930]: pgmap v7437: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:08.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:36:08 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/393037284' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:36:08.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:36:08 smithi083 ceph-mon[77242]: pgmap v7437: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:10.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:36:10 smithi153 ceph-mon[70044]: pgmap v7438: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:10.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:36:10 smithi083 ceph-mon[79930]: pgmap v7438: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:10.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:36:10 smithi083 ceph-mon[77242]: pgmap v7438: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:11.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:36:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:36:11.062Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:36:11.965 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:36:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:36:11] "GET /metrics HTTP/1.1" 200 225219 "" "Prometheus/2.33.4" 2022-09-29T09:36:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:36:12 smithi153 ceph-mon[70044]: pgmap v7439: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:12.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:36:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:36:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:36:12.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:36:12 smithi083 ceph-mon[79930]: pgmap v7439: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:12.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:36:12 smithi083 ceph-mon[77242]: pgmap v7439: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:36:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:36:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:36:14.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:36:14 smithi153 ceph-mon[70044]: pgmap v7440: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:14.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:36:14 smithi083 ceph-mon[79930]: pgmap v7440: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:14.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:36:14 smithi083 ceph-mon[77242]: pgmap v7440: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:14.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:36:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:36:14.446Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:36:15.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:36:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:36:15.154Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:36:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:36:16 smithi153 ceph-mon[70044]: pgmap v7441: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:16.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:36:16 smithi083 ceph-mon[79930]: pgmap v7441: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:16.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:36:16 smithi083 ceph-mon[77242]: pgmap v7441: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:19.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:36:18 smithi083 ceph-mon[77242]: pgmap v7442: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:19.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:36:18 smithi083 ceph-mon[79930]: pgmap v7442: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:19.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:36:18 smithi153 ceph-mon[70044]: pgmap v7442: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:36:20 smithi153 ceph-mon[70044]: pgmap v7443: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:20.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:36:20 smithi083 ceph-mon[77242]: pgmap v7443: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:20.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:36:20 smithi083 ceph-mon[79930]: pgmap v7443: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:21.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:36:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:36:21.063Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:36:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:36:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:36:21] "GET /metrics HTTP/1.1" 200 225219 "" "Prometheus/2.33.4" 2022-09-29T09:36:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:36:22 smithi153 ceph-mon[70044]: pgmap v7444: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:22.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:36:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:36:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:36:22.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:36:22 smithi083 ceph-mon[79930]: pgmap v7444: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:22.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:36:22 smithi083 ceph-mon[77242]: pgmap v7444: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:36:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:36:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:36:23.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:36:23 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:36:23.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:36:23 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:36:23.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:36:23 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:36:23.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:36:23 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:36:23.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:36:23 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:36:23.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:36:23 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:36:23.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:36:23 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:36:23.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:36:23 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:36:23.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:36:23 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:36:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:36:24 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:36:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:36:24 smithi153 ceph-mon[70044]: pgmap v7445: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:24.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:36:24 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:36:24.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:36:24 smithi083 ceph-mon[79930]: pgmap v7445: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:24.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:36:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:36:24.447Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:36:24.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:36:24 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:36:24.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:36:24 smithi083 ceph-mon[77242]: pgmap v7445: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:25.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:36:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:36:25.154Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:36:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:36:26 smithi153 ceph-mon[70044]: pgmap v7446: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:26.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:36:26 smithi083 ceph-mon[79930]: pgmap v7446: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:26.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:36:26 smithi083 ceph-mon[77242]: pgmap v7446: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:29.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:36:28 smithi083 ceph-mon[79930]: pgmap v7447: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:29.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:36:28 smithi083 ceph-mon[77242]: pgmap v7447: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:29.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:36:28 smithi153 ceph-mon[70044]: pgmap v7447: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:36:30 smithi153 ceph-mon[70044]: pgmap v7448: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:30.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:36:30 smithi083 ceph-mon[77242]: pgmap v7448: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:30.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:36:30 smithi083 ceph-mon[79930]: pgmap v7448: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:31.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:36:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:36:31.063Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:36:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:36:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:36:31] "GET /metrics HTTP/1.1" 200 225219 "" "Prometheus/2.33.4" 2022-09-29T09:36:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:36:32 smithi153 ceph-mon[70044]: pgmap v7449: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:32.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:36:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:36:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:36:32.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:36:32 smithi083 ceph-mon[77242]: pgmap v7449: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:32.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:36:32 smithi083 ceph-mon[79930]: pgmap v7449: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:36:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:36:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:36:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:36:34 smithi153 ceph-mon[70044]: pgmap v7450: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:34.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:36:34 smithi083 ceph-mon[79930]: pgmap v7450: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:34.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:36:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:36:34.448Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:36:34.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:36:34 smithi083 ceph-mon[77242]: pgmap v7450: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:35.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:36:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:36:35.154Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:36:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:36:36 smithi153 ceph-mon[70044]: pgmap v7451: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:36.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:36:36 smithi083 ceph-mon[79930]: pgmap v7451: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:36.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:36:36 smithi083 ceph-mon[77242]: pgmap v7451: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:37.800 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:36:38.091 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:36:38.091 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 7m ago 4h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:36:38.092 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 5m ago 4h 62.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:36:38.092 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 7m ago 4h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:36:38.092 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 5m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:36:38.092 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 5m ago 4h 443M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:36:38.093 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 7m ago 4h 723M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:36:38.093 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 7m ago 4h 924M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:36:38.093 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 5m ago 4h 407M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:36:38.093 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 7m ago 4h 540M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:36:38.094 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 7m ago 4h 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:36:38.094 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 5m ago 4h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:36:38.094 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 7m ago 4h 666M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:36:38.094 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 7m ago 4h 657M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:36:38.095 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 7m ago 4h 515M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:36:38.095 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 7m ago 4h 639M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:36:38.095 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 5m ago 4h 566M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:36:38.095 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 5m ago 4h 702M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:36:38.096 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 5m ago 4h 626M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:36:38.096 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 5m ago 4h 739M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:36:38.096 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 5m ago 4h 171M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:36:38.448 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:36:38.448 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:36:38.449 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:36:38.449 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:36:38.449 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:36:38.449 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:36:38.450 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:36:38.450 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:36:38.450 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:36:38.450 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:36:38.450 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:36:38.451 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:36:38.451 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:36:38.451 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:36:38.451 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:36:38.452 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:36:38.452 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:36:38.452 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:36:39.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:36:38 smithi083 ceph-mon[79930]: from='client.42801 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:36:39.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:36:38 smithi083 ceph-mon[79930]: from='client.52738 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:36:39.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:36:38 smithi083 ceph-mon[79930]: pgmap v7452: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:39.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:36:38 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2794018181' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:36:39.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:36:38 smithi083 ceph-mon[77242]: from='client.42801 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:36:39.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:36:38 smithi083 ceph-mon[77242]: from='client.52738 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:36:39.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:36:38 smithi083 ceph-mon[77242]: pgmap v7452: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:39.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:36:38 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2794018181' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:36:39.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:36:38 smithi153 ceph-mon[70044]: from='client.42801 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:36:39.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:36:38 smithi153 ceph-mon[70044]: from='client.52738 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:36:39.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:36:38 smithi153 ceph-mon[70044]: pgmap v7452: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:39.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:36:38 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2794018181' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:36:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:36:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:36:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:36:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:36:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:36:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:36:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:36:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:36:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:36:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:36:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:36:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:36:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:36:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:36:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:36:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:36:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:36:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:36:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:36:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:36:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:36:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:36:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:36:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:36:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:36:41 smithi153 ceph-mon[70044]: pgmap v7453: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:41.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:36:41 smithi083 ceph-mon[79930]: pgmap v7453: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:41.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:36:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:36:41.064Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:36:41.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:36:41 smithi083 ceph-mon[77242]: pgmap v7453: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:36:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:36:41] "GET /metrics HTTP/1.1" 200 225221 "" "Prometheus/2.33.4" 2022-09-29T09:36:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:36:42 smithi153 ceph-mon[70044]: pgmap v7454: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:42.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:36:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:36:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:36:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:36:42 smithi083 ceph-mon[79930]: pgmap v7454: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:36:42 smithi083 ceph-mon[77242]: pgmap v7454: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:43.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:36:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:36:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:36:44.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:36:44 smithi153 ceph-mon[70044]: pgmap v7455: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:44.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:36:44 smithi083 ceph-mon[79930]: pgmap v7455: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:44.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:36:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:36:44.448Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:36:44.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:36:44 smithi083 ceph-mon[77242]: pgmap v7455: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:45.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:36:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:36:45.155Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:36:46.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:36:46 smithi153 ceph-mon[70044]: pgmap v7456: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:46.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:36:46 smithi083 ceph-mon[77242]: pgmap v7456: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:46.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:36:46 smithi083 ceph-mon[79930]: pgmap v7456: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:49.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:36:48 smithi083 ceph-mon[79930]: pgmap v7457: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:49.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:36:48 smithi083 ceph-mon[77242]: pgmap v7457: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:49.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:36:48 smithi153 ceph-mon[70044]: pgmap v7457: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:36:50 smithi153 ceph-mon[70044]: pgmap v7458: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:50.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:36:50 smithi083 ceph-mon[79930]: pgmap v7458: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:50.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:36:50 smithi083 ceph-mon[77242]: pgmap v7458: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:51.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:36:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:36:51.064Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:36:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:36:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:36:51] "GET /metrics HTTP/1.1" 200 225221 "" "Prometheus/2.33.4" 2022-09-29T09:36:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:36:52 smithi153 ceph-mon[70044]: pgmap v7459: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:52.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:36:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:36:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:36:52.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:36:52 smithi083 ceph-mon[77242]: pgmap v7459: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:52.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:36:52 smithi083 ceph-mon[79930]: pgmap v7459: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:53.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:36:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:36:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:36:54.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:36:54 smithi153 ceph-mon[70044]: pgmap v7460: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:54.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:36:54 smithi083 ceph-mon[79930]: pgmap v7460: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:54.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:36:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:36:54.449Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:36:54.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:36:54 smithi083 ceph-mon[77242]: pgmap v7460: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:36:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:36:55.155Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:36:56.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:36:56 smithi153 ceph-mon[70044]: pgmap v7461: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:56.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:36:56 smithi083 ceph-mon[79930]: pgmap v7461: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:56.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:36:56 smithi083 ceph-mon[77242]: pgmap v7461: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:59.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:36:58 smithi083 ceph-mon[79930]: pgmap v7462: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:59.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:36:58 smithi083 ceph-mon[77242]: pgmap v7462: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:36:59.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:36:58 smithi153 ceph-mon[70044]: pgmap v7462: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:37:00 smithi153 ceph-mon[70044]: pgmap v7463: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:00.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:37:00 smithi083 ceph-mon[79930]: pgmap v7463: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:00.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:37:00 smithi083 ceph-mon[77242]: pgmap v7463: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:01.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:37:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:37:01.065Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:37:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:37:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:37:01] "GET /metrics HTTP/1.1" 200 225221 "" "Prometheus/2.33.4" 2022-09-29T09:37:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:37:02 smithi153 ceph-mon[70044]: pgmap v7464: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:02.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:37:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:37:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:37:02.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:37:02 smithi083 ceph-mon[79930]: pgmap v7464: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:02.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:37:02 smithi083 ceph-mon[77242]: pgmap v7464: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:37:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:37:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:37:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:37:04 smithi153 ceph-mon[70044]: pgmap v7465: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:04.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:37:04 smithi083 ceph-mon[79930]: pgmap v7465: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:04.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:37:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:37:04.451Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:37:04.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:37:04 smithi083 ceph-mon[77242]: pgmap v7465: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:05.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:37:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:37:05.155Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:37:06.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:37:06 smithi153 ceph-mon[70044]: pgmap v7466: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:06.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:37:06 smithi083 ceph-mon[79930]: pgmap v7466: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:06.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:37:06 smithi083 ceph-mon[77242]: pgmap v7466: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:08.772 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:37:09.062 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:37:09.062 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 7m ago 4h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:37:09.063 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 5m ago 4h 62.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:37:09.063 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 7m ago 4h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:37:09.063 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 5m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:37:09.063 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 5m ago 4h 443M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:37:09.064 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 7m ago 4h 723M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:37:09.064 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 7m ago 4h 924M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:37:09.064 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 5m ago 4h 407M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:37:09.064 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 7m ago 4h 540M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:37:09.064 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 7m ago 4h 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:37:09.065 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 5m ago 4h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:37:09.065 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 7m ago 4h 666M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:37:09.065 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 7m ago 4h 657M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:37:09.065 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 7m ago 4h 515M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:37:09.065 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 7m ago 4h 639M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:37:09.066 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 5m ago 4h 566M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:37:09.066 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 5m ago 4h 702M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:37:09.066 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 5m ago 4h 626M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:37:09.066 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 5m ago 4h 739M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:37:09.066 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 5m ago 4h 171M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:37:09.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:37:08 smithi083 ceph-mon[79930]: pgmap v7467: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:09.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:37:08 smithi083 ceph-mon[77242]: pgmap v7467: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:09.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:37:08 smithi153 ceph-mon[70044]: pgmap v7467: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:09.420 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:37:09.420 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:37:09.421 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:37:09.421 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:37:09.421 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:37:09.421 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:37:09.422 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:37:09.422 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:37:09.422 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:37:09.422 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:37:09.423 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:37:09.423 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:37:09.423 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:37:09.423 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:37:09.423 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:37:09.424 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:37:09.424 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:37:09.424 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:37:10.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:37:09 smithi083 ceph-mon[77242]: from='client.42819 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:37:10.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:37:09 smithi083 ceph-mon[77242]: from='client.52756 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:37:10.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:37:09 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/697215220' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:37:10.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:37:09 smithi083 ceph-mon[79930]: from='client.42819 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:37:10.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:37:09 smithi083 ceph-mon[79930]: from='client.52756 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:37:10.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:37:09 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/697215220' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:37:10.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:37:09 smithi153 ceph-mon[70044]: from='client.42819 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:37:10.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:37:09 smithi153 ceph-mon[70044]: from='client.52756 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:37:10.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:37:09 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/697215220' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:37:11.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:37:10 smithi083 ceph-mon[79930]: pgmap v7468: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:11.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:37:10 smithi083 ceph-mon[77242]: pgmap v7468: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:11.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:37:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:37:11.065Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:37:11.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:37:10 smithi153 ceph-mon[70044]: pgmap v7468: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:37:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:37:11] "GET /metrics HTTP/1.1" 200 225215 "" "Prometheus/2.33.4" 2022-09-29T09:37:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:37:12 smithi153 ceph-mon[70044]: pgmap v7469: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:12.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:37:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:37:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:37:12.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:37:12 smithi083 ceph-mon[79930]: pgmap v7469: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:12.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:37:12 smithi083 ceph-mon[77242]: pgmap v7469: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:13.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:37:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:37:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:37:14.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:37:14 smithi153 ceph-mon[70044]: pgmap v7470: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:14.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:37:14 smithi083 ceph-mon[79930]: pgmap v7470: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:14.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:37:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:37:14.451Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:37:14.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:37:14 smithi083 ceph-mon[77242]: pgmap v7470: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:15.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:37:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:37:15.156Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:37:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:37:16 smithi153 ceph-mon[70044]: pgmap v7471: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:16.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:37:16 smithi083 ceph-mon[79930]: pgmap v7471: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:16.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:37:16 smithi083 ceph-mon[77242]: pgmap v7471: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:19.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:37:18 smithi083 ceph-mon[79930]: pgmap v7472: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:19.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:37:18 smithi083 ceph-mon[77242]: pgmap v7472: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:19.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:37:18 smithi153 ceph-mon[70044]: pgmap v7472: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:37:20 smithi153 ceph-mon[70044]: pgmap v7473: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:20.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:37:20 smithi083 ceph-mon[79930]: pgmap v7473: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:20.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:37:20 smithi083 ceph-mon[77242]: pgmap v7473: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:21.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:37:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:37:21.066Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:37:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:37:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:37:21] "GET /metrics HTTP/1.1" 200 225215 "" "Prometheus/2.33.4" 2022-09-29T09:37:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:37:22 smithi153 ceph-mon[70044]: pgmap v7474: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:22.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:37:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:37:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:37:22.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:37:22 smithi083 ceph-mon[77242]: pgmap v7474: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:22.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:37:22 smithi083 ceph-mon[79930]: pgmap v7474: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:23.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:37:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:37:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:37:24.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:37:23 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:37:24.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:37:23 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:37:24.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:37:23 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:37:24.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:37:23 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:37:24.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:37:23 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:37:24.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:37:23 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:37:24.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:37:23 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:37:24.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:37:23 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:37:24.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:37:23 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:37:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:37:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:37:24.451Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:37:25.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:37:24 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:37:25.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:37:24 smithi083 ceph-mon[79930]: pgmap v7475: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:25.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:37:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:37:25.157Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:37:25.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:37:24 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:37:25.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:37:24 smithi083 ceph-mon[77242]: pgmap v7475: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:25.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:37:24 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:37:25.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:37:24 smithi153 ceph-mon[70044]: pgmap v7475: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:37:26 smithi153 ceph-mon[70044]: pgmap v7476: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:26.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:37:26 smithi083 ceph-mon[79930]: pgmap v7476: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:26.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:37:26 smithi083 ceph-mon[77242]: pgmap v7476: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:29.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:37:28 smithi083 ceph-mon[77242]: pgmap v7477: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:29.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:37:28 smithi083 ceph-mon[79930]: pgmap v7477: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:29.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:37:28 smithi153 ceph-mon[70044]: pgmap v7477: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:37:30 smithi153 ceph-mon[70044]: pgmap v7478: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:30.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:37:30 smithi083 ceph-mon[79930]: pgmap v7478: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:30.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:37:30 smithi083 ceph-mon[77242]: pgmap v7478: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:31.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:37:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:37:31.066Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:37:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:37:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:37:31] "GET /metrics HTTP/1.1" 200 225219 "" "Prometheus/2.33.4" 2022-09-29T09:37:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:37:32 smithi153 ceph-mon[70044]: pgmap v7479: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:32.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:37:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:37:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:37:32.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:37:32 smithi083 ceph-mon[79930]: pgmap v7479: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:32.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:37:32 smithi083 ceph-mon[77242]: pgmap v7479: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:37:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:37:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:37:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:37:34 smithi153 ceph-mon[70044]: pgmap v7480: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:34.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:37:34 smithi083 ceph-mon[77242]: pgmap v7480: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:34.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:37:34 smithi083 ceph-mon[79930]: pgmap v7480: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:34.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:37:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:37:34.452Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:37:35.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:37:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:37:35.158Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:37:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:37:36 smithi153 ceph-mon[70044]: pgmap v7481: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:36.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:37:36 smithi083 ceph-mon[79930]: pgmap v7481: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:36.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:37:36 smithi083 ceph-mon[77242]: pgmap v7481: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:39.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:37:38 smithi083 ceph-mon[77242]: pgmap v7482: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:39.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:37:38 smithi083 ceph-mon[79930]: pgmap v7482: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:39.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:37:38 smithi153 ceph-mon[70044]: pgmap v7482: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:39.747 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:37:40.038 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:37:40.038 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 8m ago 4h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:37:40.039 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 6m ago 4h 62.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:37:40.039 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 8m ago 4h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:37:40.039 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 6m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:37:40.039 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 6m ago 4h 443M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:37:40.040 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 8m ago 4h 723M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:37:40.040 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 8m ago 4h 924M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:37:40.040 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 6m ago 4h 407M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:37:40.040 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 8m ago 4h 540M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:37:40.041 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 8m ago 4h 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:37:40.041 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 6m ago 4h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:37:40.041 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 8m ago 4h 666M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:37:40.041 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 8m ago 4h 657M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:37:40.042 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 8m ago 4h 515M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:37:40.042 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 8m ago 4h 639M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:37:40.042 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 6m ago 4h 566M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:37:40.042 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 6m ago 4h 702M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:37:40.042 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 6m ago 4h 626M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:37:40.043 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 6m ago 4h 739M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:37:40.043 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 6m ago 4h 171M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:37:40.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:37:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:37:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:37:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:37:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:37:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:37:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:37:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:37:40.394 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:37:40.394 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:37:40.395 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:37:40.395 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:37:40.395 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:37:40.395 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:37:40.396 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:37:40.396 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:37:40.396 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:37:40.396 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:37:40.396 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:37:40.397 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:37:40.397 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:37:40.397 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:37:40.397 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:37:40.397 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:37:40.398 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:37:40.398 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:37:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:37:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:37:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:37:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:37:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:37:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:37:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:37:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:37:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:37:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:37:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:37:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:37:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:37:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:37:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:37:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:37:41.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:37:41 smithi153 ceph-mon[70044]: from='client.42837 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:37:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:37:41 smithi153 ceph-mon[70044]: from='client.42843 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:37:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:37:41 smithi153 ceph-mon[70044]: pgmap v7483: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:37:41 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2244183399' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:37:41.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:37:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:37:41.067Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:37:41.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:37:41 smithi083 ceph-mon[79930]: from='client.42837 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:37:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:37:41 smithi083 ceph-mon[79930]: from='client.42843 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:37:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:37:41 smithi083 ceph-mon[79930]: pgmap v7483: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:37:41 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2244183399' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:37:41.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:37:41 smithi083 ceph-mon[77242]: from='client.42837 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:37:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:37:41 smithi083 ceph-mon[77242]: from='client.42843 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:37:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:37:41 smithi083 ceph-mon[77242]: pgmap v7483: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:37:41 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2244183399' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:37:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:37:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:37:41] "GET /metrics HTTP/1.1" 200 225222 "" "Prometheus/2.33.4" 2022-09-29T09:37:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:37:42 smithi153 ceph-mon[70044]: pgmap v7484: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:42.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:37:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:37:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:37:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:37:42 smithi083 ceph-mon[79930]: pgmap v7484: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:37:42 smithi083 ceph-mon[77242]: pgmap v7484: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:43.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:37:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:37:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:37:44.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:37:44 smithi153 ceph-mon[70044]: pgmap v7485: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:44.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:37:44 smithi083 ceph-mon[79930]: pgmap v7485: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:44.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:37:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:37:44.453Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:37:44.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:37:44 smithi083 ceph-mon[77242]: pgmap v7485: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:45.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:37:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:37:45.158Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:37:46.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:37:46 smithi153 ceph-mon[70044]: pgmap v7486: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:46.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:37:46 smithi083 ceph-mon[79930]: pgmap v7486: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:46.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:37:46 smithi083 ceph-mon[77242]: pgmap v7486: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:49.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:37:48 smithi083 ceph-mon[79930]: pgmap v7487: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:49.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:37:48 smithi083 ceph-mon[77242]: pgmap v7487: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:49.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:37:48 smithi153 ceph-mon[70044]: pgmap v7487: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:37:50 smithi153 ceph-mon[70044]: pgmap v7488: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:50.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:37:50 smithi083 ceph-mon[79930]: pgmap v7488: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:50.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:37:50 smithi083 ceph-mon[77242]: pgmap v7488: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:51.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:37:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:37:51.068Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:37:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:37:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:37:51] "GET /metrics HTTP/1.1" 200 225222 "" "Prometheus/2.33.4" 2022-09-29T09:37:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:37:52 smithi153 ceph-mon[70044]: pgmap v7489: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:52.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:37:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:37:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:37:52.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:37:52 smithi083 ceph-mon[79930]: pgmap v7489: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:52.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:37:52 smithi083 ceph-mon[77242]: pgmap v7489: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:53.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:37:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:37:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:37:54.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:37:54 smithi153 ceph-mon[70044]: pgmap v7490: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:54.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:37:54 smithi083 ceph-mon[79930]: pgmap v7490: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:54.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:37:54 smithi083 ceph-mon[77242]: pgmap v7490: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:54.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:37:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:37:54.453Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:37:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:37:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:37:55.158Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:37:56.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:37:56 smithi153 ceph-mon[70044]: pgmap v7491: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:56.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:37:56 smithi083 ceph-mon[79930]: pgmap v7491: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:56.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:37:56 smithi083 ceph-mon[77242]: pgmap v7491: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:59.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:37:58 smithi083 ceph-mon[79930]: pgmap v7492: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:59.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:37:58 smithi083 ceph-mon[77242]: pgmap v7492: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:37:59.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:37:58 smithi153 ceph-mon[70044]: pgmap v7492: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:38:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:38:00 smithi153 ceph-mon[70044]: pgmap v7493: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:38:00.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:38:00 smithi083 ceph-mon[79930]: pgmap v7493: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:38:00.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:38:00 smithi083 ceph-mon[77242]: pgmap v7493: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:38:01.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:38:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:38:01.069Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:38:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:38:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:38:01] "GET /metrics HTTP/1.1" 200 225222 "" "Prometheus/2.33.4" 2022-09-29T09:38:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:38:02 smithi153 ceph-mon[70044]: pgmap v7494: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:38:02.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:38:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:38:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:38:02.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:38:02 smithi083 ceph-mon[79930]: pgmap v7494: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:38:02.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:38:02 smithi083 ceph-mon[77242]: pgmap v7494: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:38:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:38:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:38:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:38:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:38:04 smithi153 ceph-mon[70044]: pgmap v7495: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:38:04.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:38:04 smithi083 ceph-mon[79930]: pgmap v7495: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:38:04.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:38:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:38:04.453Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:38:04.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:38:04 smithi083 ceph-mon[77242]: pgmap v7495: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:38:05.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:38:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:38:05.159Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:38:06.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:38:06 smithi153 ceph-mon[70044]: pgmap v7496: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:38:06.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:38:06 smithi083 ceph-mon[79930]: pgmap v7496: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:38:06.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:38:06 smithi083 ceph-mon[77242]: pgmap v7496: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:38:09.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:38:08 smithi083 ceph-mon[79930]: pgmap v7497: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:38:09.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:38:08 smithi083 ceph-mon[77242]: pgmap v7497: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:38:09.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:38:08 smithi153 ceph-mon[70044]: pgmap v7497: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:38:10.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:38:10 smithi153 ceph-mon[70044]: pgmap v7498: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:38:10.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:38:10 smithi083 ceph-mon[79930]: pgmap v7498: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:38:10.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:38:10 smithi083 ceph-mon[77242]: pgmap v7498: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:38:10.720 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:38:11.009 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:38:11.010 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 8m ago 4h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:38:11.010 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 6m ago 4h 62.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:38:11.010 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 8m ago 4h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:38:11.010 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 6m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:38:11.011 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 6m ago 4h 443M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:38:11.011 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 8m ago 4h 723M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:38:11.011 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 8m ago 4h 924M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:38:11.011 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 6m ago 4h 407M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:38:11.012 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 8m ago 4h 540M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:38:11.012 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 8m ago 4h 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:38:11.012 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 6m ago 4h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:38:11.012 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 8m ago 4h 666M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:38:11.012 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 8m ago 4h 657M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:38:11.013 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 8m ago 4h 515M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:38:11.013 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 8m ago 4h 639M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:38:11.013 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 6m ago 4h 566M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:38:11.013 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 6m ago 4h 702M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:38:11.013 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 6m ago 4h 626M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:38:11.014 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 6m ago 4h 739M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:38:11.014 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 6m ago 4h 171M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:38:11.368 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:38:11.368 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:38:11.368 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:38:11.369 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:38:11.369 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:38:11.369 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:38:11.369 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:38:11.369 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:38:11.370 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:38:11.370 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:38:11.370 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:38:11.370 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:38:11.371 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:38:11.371 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:38:11.371 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:38:11.371 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:38:11.371 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:38:11.372 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:38:11.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:38:11 smithi083 ceph-mon[79930]: from='client.42855 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:38:11.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:38:11 smithi083 ceph-mon[79930]: from='client.42861 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:38:11.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:38:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:38:11.069Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:38:11.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:38:11 smithi083 ceph-mon[77242]: from='client.42855 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:38:11.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:38:11 smithi083 ceph-mon[77242]: from='client.42861 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:38:11.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:38:11 smithi153 ceph-mon[70044]: from='client.42855 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:38:11.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:38:11 smithi153 ceph-mon[70044]: from='client.42861 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:38:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:38:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:38:11] "GET /metrics HTTP/1.1" 200 225219 "" "Prometheus/2.33.4" 2022-09-29T09:38:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:38:12 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3992494980' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:38:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:38:12 smithi153 ceph-mon[70044]: pgmap v7499: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:38:12.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:38:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:38:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:38:12.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:38:12 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3992494980' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:38:12.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:38:12 smithi083 ceph-mon[79930]: pgmap v7499: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:38:12.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:38:12 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3992494980' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:38:12.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:38:12 smithi083 ceph-mon[77242]: pgmap v7499: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:38:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:38:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:38:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:38:14.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:38:14 smithi153 ceph-mon[70044]: pgmap v7500: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:38:14.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:38:14 smithi083 ceph-mon[79930]: pgmap v7500: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:38:14.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:38:14 smithi083 ceph-mon[77242]: pgmap v7500: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:38:14.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:38:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:38:14.454Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:38:15.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:38:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:38:15.159Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:38:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:38:16 smithi153 ceph-mon[70044]: pgmap v7501: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:38:16.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:38:16 smithi083 ceph-mon[79930]: pgmap v7501: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:38:16.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:38:16 smithi083 ceph-mon[77242]: pgmap v7501: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:38:19.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:38:18 smithi083 ceph-mon[79930]: pgmap v7502: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:38:19.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:38:18 smithi083 ceph-mon[77242]: pgmap v7502: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:38:19.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:38:18 smithi153 ceph-mon[70044]: pgmap v7502: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:38:20.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:38:20 smithi153 ceph-mon[70044]: pgmap v7503: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:38:20.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:38:20 smithi083 ceph-mon[79930]: pgmap v7503: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:38:20.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:38:20 smithi083 ceph-mon[77242]: pgmap v7503: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:38:21.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:38:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:38:21.071Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:38:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:38:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:38:21] "GET /metrics HTTP/1.1" 200 225219 "" "Prometheus/2.33.4" 2022-09-29T09:38:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:38:22 smithi153 ceph-mon[70044]: pgmap v7504: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:38:22.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:38:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:38:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:38:22.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:38:22 smithi083 ceph-mon[79930]: pgmap v7504: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:38:22.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:38:22 smithi083 ceph-mon[77242]: pgmap v7504: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:38:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:38:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:38:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:38:24.456 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:38:24 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:38:24.457 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:38:24 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:38:24.457 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:38:24 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:38:24.457 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:38:24 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:38:24.457 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:38:24 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:38:24.458 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:38:24 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:38:24.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:38:24 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:38:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:38:24 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:38:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:38:24 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:38:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:38:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:38:24.454Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:38:25.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:38:25 smithi083 ceph-mon[77242]: pgmap v7505: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:38:25.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:38:25 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:38:25.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:38:25 smithi083 ceph-mon[79930]: pgmap v7505: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:38:25.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:38:25 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:38:25.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:38:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:38:25.160Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:38:25.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:38:25 smithi153 ceph-mon[70044]: pgmap v7505: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:38:25.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:38:25 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:38:26.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:38:26 smithi153 ceph-mon[70044]: pgmap v7506: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:38:26.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:38:26 smithi083 ceph-mon[79930]: pgmap v7506: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:38:26.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:38:26 smithi083 ceph-mon[77242]: pgmap v7506: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:38:29.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:38:28 smithi083 ceph-mon[79930]: pgmap v7507: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:38:29.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:38:28 smithi083 ceph-mon[77242]: pgmap v7507: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:38:29.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:38:28 smithi153 ceph-mon[70044]: pgmap v7507: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:38:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:38:30 smithi153 ceph-mon[70044]: pgmap v7508: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:38:30.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:38:30 smithi083 ceph-mon[79930]: pgmap v7508: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:38:30.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:38:30 smithi083 ceph-mon[77242]: pgmap v7508: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:38:31.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:38:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:38:31.071Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:38:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:38:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:38:31] "GET /metrics HTTP/1.1" 200 225216 "" "Prometheus/2.33.4" 2022-09-29T09:38:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:38:32 smithi153 ceph-mon[70044]: pgmap v7509: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:38:32.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:38:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:38:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:38:32.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:38:32 smithi083 ceph-mon[77242]: pgmap v7509: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:38:32.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:38:32 smithi083 ceph-mon[79930]: pgmap v7509: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:38:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:38:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:38:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:38:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:38:34 smithi153 ceph-mon[70044]: pgmap v7510: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:38:34.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:38:34 smithi083 ceph-mon[79930]: pgmap v7510: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:38:34.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:38:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:38:34.456Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:38:34.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:38:34 smithi083 ceph-mon[77242]: pgmap v7510: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:38:35.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:38:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:38:35.160Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:38:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:38:36 smithi153 ceph-mon[70044]: pgmap v7511: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:38:36.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:38:36 smithi083 ceph-mon[79930]: pgmap v7511: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:38:36.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:38:36 smithi083 ceph-mon[77242]: pgmap v7511: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:38:39.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:38:38 smithi153 ceph-mon[70044]: pgmap v7512: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:38:39.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:38:38 smithi083 ceph-mon[79930]: pgmap v7512: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:38:39.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:38:38 smithi083 ceph-mon[77242]: pgmap v7512: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:38:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:38:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:38:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:38:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:38:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:38:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:38:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:38:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:38:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:38:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:38:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:38:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:38:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:38:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:38:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:38:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:38:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:38:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:38:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:38:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:38:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:38:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:38:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:38:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:38:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:38:41 smithi153 ceph-mon[70044]: pgmap v7513: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:38:41.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:38:41 smithi083 ceph-mon[79930]: pgmap v7513: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:38:41.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:38:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:38:41.072Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:38:41.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:38:41 smithi083 ceph-mon[77242]: pgmap v7513: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:38:41.693 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:38:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:38:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:38:41] "GET /metrics HTTP/1.1" 200 225218 "" "Prometheus/2.33.4" 2022-09-29T09:38:41.980 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:38:41.980 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 9m ago 4h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:38:41.980 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 7m ago 4h 62.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:38:41.980 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 9m ago 4h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:38:41.981 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 7m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:38:41.981 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 7m ago 4h 443M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:38:41.981 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 9m ago 4h 723M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:38:41.981 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 9m ago 4h 924M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:38:41.982 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 7m ago 4h 407M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:38:41.982 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 9m ago 4h 540M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:38:41.982 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 9m ago 4h 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:38:41.982 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 7m ago 4h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:38:41.982 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 9m ago 4h 666M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:38:41.983 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 9m ago 4h 657M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:38:41.983 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 9m ago 4h 515M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:38:41.983 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 9m ago 4h 639M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:38:41.983 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 7m ago 4h 566M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:38:41.984 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 7m ago 4h 702M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:38:41.984 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 7m ago 4h 626M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:38:41.984 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 7m ago 4h 739M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:38:41.984 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 7m ago 4h 171M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:38:42.336 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:38:42.336 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:38:42.337 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:38:42.337 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:38:42.337 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:38:42.337 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:38:42.338 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:38:42.338 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:38:42.338 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:38:42.338 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:38:42.338 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:38:42.339 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:38:42.339 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:38:42.339 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:38:42.339 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:38:42.340 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:38:42.340 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:38:42.340 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:38:42.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:38:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:38:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:38:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:38:42 smithi153 ceph-mon[70044]: from='client.42873 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:38:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:38:42 smithi153 ceph-mon[70044]: from='client.42879 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:38:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:38:42 smithi153 ceph-mon[70044]: pgmap v7514: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:38:42.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:38:42 smithi083 ceph-mon[77242]: from='client.42873 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:38:42.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:38:42 smithi083 ceph-mon[77242]: from='client.42879 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:38:42.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:38:42 smithi083 ceph-mon[77242]: pgmap v7514: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:38:42.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:38:42 smithi083 ceph-mon[79930]: from='client.42873 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:38:42.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:38:42 smithi083 ceph-mon[79930]: from='client.42879 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:38:42.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:38:42 smithi083 ceph-mon[79930]: pgmap v7514: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:38:43.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:38:43 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/714493951' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:38:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:38:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:38:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:38:43.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:38:43 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/714493951' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:38:43.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:38:43 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/714493951' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:38:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:38:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:38:44.457Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:38:45.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:38:45 smithi083 ceph-mon[79930]: pgmap v7515: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:38:45.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:38:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:38:45.161Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:38:45.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:38:45 smithi083 ceph-mon[77242]: pgmap v7515: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:38:45.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:38:45 smithi153 ceph-mon[70044]: pgmap v7515: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:38:47.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:38:47 smithi153 ceph-mon[70044]: pgmap v7516: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:38:47.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:38:47 smithi083 ceph-mon[79930]: pgmap v7516: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:38:47.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:38:47 smithi083 ceph-mon[77242]: pgmap v7516: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:38:50.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:38:49 smithi083 ceph-mon[79930]: pgmap v7517: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:38:50.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:38:49 smithi083 ceph-mon[77242]: pgmap v7517: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:38:50.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:38:49 smithi153 ceph-mon[70044]: pgmap v7517: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:38:51.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:38:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:38:51.072Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:38:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:38:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:38:51] "GET /metrics HTTP/1.1" 200 225218 "" "Prometheus/2.33.4" 2022-09-29T09:38:52.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:38:51 smithi153 ceph-mon[70044]: pgmap v7518: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:38:52.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:38:51 smithi083 ceph-mon[77242]: pgmap v7518: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:38:52.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:38:51 smithi083 ceph-mon[79930]: pgmap v7518: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:38:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:38:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:38:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:38:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:38:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:38:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:38:54.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:38:53 smithi153 ceph-mon[70044]: pgmap v7519: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:38:54.459 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:38:53 smithi083 ceph-mon[79930]: pgmap v7519: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:38:54.459 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:38:53 smithi083 ceph-mon[77242]: pgmap v7519: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:38:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:38:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:38:54.457Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:38:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:38:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:38:55.162Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:38:56.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:38:55 smithi153 ceph-mon[70044]: pgmap v7520: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:38:56.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:38:55 smithi083 ceph-mon[79930]: pgmap v7520: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:38:56.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:38:55 smithi083 ceph-mon[77242]: pgmap v7520: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:38:58.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:38:57 smithi153 ceph-mon[70044]: pgmap v7521: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:38:58.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:38:57 smithi083 ceph-mon[79930]: pgmap v7521: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:38:58.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:38:57 smithi083 ceph-mon[77242]: pgmap v7521: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:38:59 smithi153 ceph-mon[70044]: pgmap v7522: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:00.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:38:59 smithi083 ceph-mon[79930]: pgmap v7522: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:00.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:38:59 smithi083 ceph-mon[77242]: pgmap v7522: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:01.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:39:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:39:01.073Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:39:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:39:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:39:01] "GET /metrics HTTP/1.1" 200 225217 "" "Prometheus/2.33.4" 2022-09-29T09:39:02.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:39:01 smithi153 ceph-mon[70044]: pgmap v7523: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:02.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:39:01 smithi083 ceph-mon[79930]: pgmap v7523: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:02.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:39:01 smithi083 ceph-mon[77242]: pgmap v7523: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:02.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:39:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:39:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:39:03.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:39:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:39:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:39:04.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:39:03 smithi153 ceph-mon[70044]: pgmap v7524: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:04.459 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:39:03 smithi083 ceph-mon[79930]: pgmap v7524: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:04.460 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:39:03 smithi083 ceph-mon[77242]: pgmap v7524: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:04.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:39:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:39:04.458Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:39:05.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:39:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:39:05.163Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:39:06.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:39:05 smithi153 ceph-mon[70044]: pgmap v7525: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:06.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:39:05 smithi083 ceph-mon[77242]: pgmap v7525: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:06.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:39:05 smithi083 ceph-mon[79930]: pgmap v7525: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:08.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:39:07 smithi153 ceph-mon[70044]: pgmap v7526: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:08.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:39:07 smithi083 ceph-mon[79930]: pgmap v7526: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:08.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:39:07 smithi083 ceph-mon[77242]: pgmap v7526: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:10.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:39:09 smithi153 ceph-mon[70044]: pgmap v7527: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:10.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:39:09 smithi083 ceph-mon[77242]: pgmap v7527: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:10.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:39:09 smithi083 ceph-mon[79930]: pgmap v7527: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:11.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:39:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:39:11.073Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:39:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:39:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:39:11] "GET /metrics HTTP/1.1" 200 225215 "" "Prometheus/2.33.4" 2022-09-29T09:39:12.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:39:12 smithi153 ceph-mon[70044]: pgmap v7528: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:12.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:39:11 smithi083 ceph-mon[79930]: pgmap v7528: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:12.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:39:11 smithi083 ceph-mon[77242]: pgmap v7528: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:12.660 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:39:12.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:39:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:39:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:39:12.947 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:39:12.948 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 9m ago 4h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:39:12.948 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 7m ago 4h 62.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:39:12.948 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 9m ago 4h 25.6M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:39:12.948 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 7m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:39:12.948 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 7m ago 4h 443M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:39:12.949 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 9m ago 4h 723M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:39:12.949 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 9m ago 4h 924M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:39:12.949 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 7m ago 4h 407M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:39:12.949 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 9m ago 4h 540M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:39:12.950 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 9m ago 4h 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:39:12.950 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 7m ago 4h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:39:12.950 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 9m ago 4h 666M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:39:12.950 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 9m ago 4h 657M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:39:12.951 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 9m ago 4h 515M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:39:12.951 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 9m ago 4h 639M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:39:12.951 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 7m ago 4h 566M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:39:12.951 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 7m ago 4h 702M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:39:12.952 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 7m ago 4h 626M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:39:12.952 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 7m ago 4h 739M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:39:12.952 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 7m ago 4h 171M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:39:13.302 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:39:13.302 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:39:13.303 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:39:13.303 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:39:13.303 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:39:13.303 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:39:13.304 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:39:13.304 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:39:13.304 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:39:13.304 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:39:13.304 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:39:13.305 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:39:13.305 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:39:13.305 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:39:13.305 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:39:13.305 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:39:13.306 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:39:13.306 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:39:13.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:39:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:39:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:39:14.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:39:14 smithi153 ceph-mon[70044]: pgmap v7529: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:14.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:39:14 smithi153 ceph-mon[70044]: from='client.52822 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:39:14.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:39:14 smithi153 ceph-mon[70044]: from='client.42897 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:39:14.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:39:14 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1039041888' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:39:14.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:39:14 smithi083 ceph-mon[79930]: pgmap v7529: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:14.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:39:14 smithi083 ceph-mon[79930]: from='client.52822 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:39:14.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:39:14 smithi083 ceph-mon[79930]: from='client.42897 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:39:14.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:39:14 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1039041888' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:39:14.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:39:14 smithi083 ceph-mon[77242]: pgmap v7529: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:14.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:39:14 smithi083 ceph-mon[77242]: from='client.52822 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:39:14.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:39:14 smithi083 ceph-mon[77242]: from='client.42897 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:39:14.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:39:14 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1039041888' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:39:14.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:39:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:39:14.459Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:39:15.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:39:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:39:15.164Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:39:16.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:39:16 smithi153 ceph-mon[70044]: pgmap v7530: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:16.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:39:16 smithi083 ceph-mon[79930]: pgmap v7530: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:16.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:39:16 smithi083 ceph-mon[77242]: pgmap v7530: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:18.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:39:18 smithi153 ceph-mon[70044]: pgmap v7531: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:18.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:39:18 smithi083 ceph-mon[79930]: pgmap v7531: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:18.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:39:18 smithi083 ceph-mon[77242]: pgmap v7531: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:20.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:39:20 smithi153 ceph-mon[70044]: pgmap v7532: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:20.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:39:20 smithi083 ceph-mon[77242]: pgmap v7532: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:20.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:39:20 smithi083 ceph-mon[79930]: pgmap v7532: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:21.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:39:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:39:21.074Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:39:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:39:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:39:21] "GET /metrics HTTP/1.1" 200 225215 "" "Prometheus/2.33.4" 2022-09-29T09:39:22.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:39:22 smithi153 ceph-mon[70044]: pgmap v7533: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:22.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:39:22 smithi083 ceph-mon[79930]: pgmap v7533: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:22.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:39:22 smithi083 ceph-mon[77242]: pgmap v7533: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:39:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:39:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:39:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:39:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:39:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:39:24.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:39:24 smithi153 ceph-mon[70044]: pgmap v7534: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:24.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:39:24 smithi083 ceph-mon[79930]: pgmap v7534: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:24.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:39:24 smithi083 ceph-mon[77242]: pgmap v7534: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:39:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:39:24.460Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:39:25.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:39:25 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:39:25.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:39:25 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:39:25.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:39:25 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:39:25.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:39:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:39:25.164Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:39:25.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:39:25 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:39:25.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:39:25 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:39:25.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:39:25 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:39:25.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:39:25 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:39:25.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:39:25 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:39:25.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:39:25 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:39:26.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:39:26 smithi083 ceph-mon[77242]: pgmap v7535: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:26.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:39:26 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:39:26.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:39:26 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:39:26.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:39:26 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:39:26.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:39:26 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:39:26.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:39:26 smithi083 ceph-mon[79930]: pgmap v7535: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:26.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:39:26 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:39:26.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:39:26 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:39:26.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:39:26 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:39:26.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:39:26 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:39:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:39:26 smithi153 ceph-mon[70044]: pgmap v7535: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:39:26 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:39:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:39:26 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:39:26.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:39:26 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:39:26.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:39:26 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:39:28.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:39:28 smithi083 ceph-mon[77242]: pgmap v7536: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:28.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:39:28 smithi083 ceph-mon[79930]: pgmap v7536: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:39:28 smithi153 ceph-mon[70044]: pgmap v7536: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:30.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:39:30 smithi083 ceph-mon[79930]: pgmap v7537: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:30.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:39:30 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:39:30.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:39:30 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:39:30.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:39:30 smithi083 ceph-mon[77242]: pgmap v7537: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:30.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:39:30 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:39:30.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:39:30 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:39:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:39:30 smithi153 ceph-mon[70044]: pgmap v7537: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:39:30 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:39:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:39:30 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:39:31.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:39:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:39:31.074Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:39:31.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:39:31 smithi083 ceph-mon[77242]: pgmap v7538: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:31.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:39:31 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:39:31.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:39:31 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:39:31.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:39:31 smithi083 ceph-mon[79930]: pgmap v7538: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:31.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:39:31 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:39:31.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:39:31 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:39:31.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:39:31 smithi153 ceph-mon[70044]: pgmap v7538: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:31.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:39:31 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:39:31.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:39:31 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:39:31.728 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:39:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:39:31] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T09:39:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:39:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:39:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:39:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:39:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:39:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:39:34.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:39:33 smithi083 ceph-mon[79930]: pgmap v7539: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:34.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:39:33 smithi083 ceph-mon[79930]: Detected new or changed devices on smithi083 2022-09-29T09:39:34.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:39:33 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:39:34.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:39:33 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:39:34.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:39:33 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:39:34.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:39:33 smithi083 ceph-mon[77242]: pgmap v7539: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:34.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:39:33 smithi083 ceph-mon[77242]: Detected new or changed devices on smithi083 2022-09-29T09:39:34.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:39:33 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:39:34.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:39:33 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:39:34.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:39:33 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:39:34.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:39:33 smithi153 ceph-mon[70044]: pgmap v7539: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:34.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:39:33 smithi153 ceph-mon[70044]: Detected new or changed devices on smithi083 2022-09-29T09:39:34.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:39:33 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:39:34.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:39:33 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:39:34.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:39:33 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:39:34.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:39:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:39:34.460Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:39:35.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:39:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:39:35.166Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:39:36.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:39:35 smithi083 ceph-mon[79930]: pgmap v7540: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:36.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:39:35 smithi083 ceph-mon[77242]: pgmap v7540: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:36.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:39:35 smithi153 ceph-mon[70044]: pgmap v7540: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:38.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:39:37 smithi083 ceph-mon[77242]: pgmap v7541: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:38.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:39:37 smithi083 ceph-mon[79930]: pgmap v7541: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:38.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:39:37 smithi153 ceph-mon[70044]: pgmap v7541: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:39:39 smithi153 ceph-mon[70044]: pgmap v7542: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:39:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:39:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:39:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:39:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:39:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:39:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:39:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:39:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:39:39 smithi083 ceph-mon[79930]: pgmap v7542: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:39:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:39:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:39:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:39:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:39:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:39:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:39:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:39:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:39:39 smithi083 ceph-mon[77242]: pgmap v7542: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:39:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:39:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:39:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:39:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:39:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:39:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:39:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:39:41.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:39:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:39:41.076Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:39:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:39:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:39:41] "GET /metrics HTTP/1.1" 200 225227 "" "Prometheus/2.33.4" 2022-09-29T09:39:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:39:41 smithi153 ceph-mon[70044]: pgmap v7543: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:42.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:39:41 smithi083 ceph-mon[77242]: pgmap v7543: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:39:41 smithi083 ceph-mon[79930]: pgmap v7543: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:39:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:39:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:39:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:39:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:39:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:39:43.635 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:39:43.927 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:39:43.928 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 14s ago 4h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:39:43.928 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 8m ago 4h 62.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:39:43.928 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 14s ago 4h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:39:43.928 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 8m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:39:43.929 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 8m ago 4h 443M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:39:43.929 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 14s ago 4h 731M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:39:43.929 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 14s ago 4h 959M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:39:43.929 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 8m ago 4h 407M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:39:43.930 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 14s ago 4h 562M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:39:43.930 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 14s ago 4h 26.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:39:43.930 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 8m ago 4h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:39:43.930 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 14s ago 4h 691M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:39:43.930 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 14s ago 4h 682M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:39:43.931 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 14s ago 4h 534M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:39:43.931 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 14s ago 4h 664M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:39:43.931 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 8m ago 4h 566M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:39:43.931 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 8m ago 4h 702M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:39:43.931 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 8m ago 4h 626M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:39:43.932 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 8m ago 4h 739M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:39:43.932 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 8m ago 4h 171M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:39:44.285 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:39:44.285 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:39:44.286 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:39:44.286 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:39:44.286 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:39:44.286 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:39:44.286 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:39:44.287 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:39:44.287 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:39:44.287 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:39:44.287 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:39:44.288 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:39:44.288 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:39:44.288 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:39:44.288 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:39:44.288 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:39:44.289 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:39:44.289 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:39:44.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:39:43 smithi153 ceph-mon[70044]: pgmap v7544: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:44.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:39:43 smithi083 ceph-mon[79930]: pgmap v7544: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:44.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:39:43 smithi083 ceph-mon[77242]: pgmap v7544: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:44.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:39:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:39:44.460Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:39:45.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:39:44 smithi153 ceph-mon[70044]: from='client.42909 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:39:45.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:39:44 smithi153 ceph-mon[70044]: from='client.42915 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:39:45.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:39:44 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/875557290' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:39:45.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:39:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:39:45.166Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:39:45.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:39:44 smithi083 ceph-mon[77242]: from='client.42909 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:39:45.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:39:44 smithi083 ceph-mon[77242]: from='client.42915 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:39:45.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:39:44 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/875557290' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:39:45.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:39:44 smithi083 ceph-mon[79930]: from='client.42909 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:39:45.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:39:44 smithi083 ceph-mon[79930]: from='client.42915 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:39:45.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:39:44 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/875557290' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:39:46.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:39:45 smithi153 ceph-mon[70044]: pgmap v7545: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:46.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:39:45 smithi083 ceph-mon[77242]: pgmap v7545: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:46.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:39:45 smithi083 ceph-mon[79930]: pgmap v7545: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:48.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:39:47 smithi153 ceph-mon[70044]: pgmap v7546: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:48.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:39:47 smithi083 ceph-mon[77242]: pgmap v7546: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:48.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:39:47 smithi083 ceph-mon[79930]: pgmap v7546: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:50.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:39:49 smithi153 ceph-mon[70044]: pgmap v7547: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:50.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:39:49 smithi083 ceph-mon[79930]: pgmap v7547: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:50.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:39:49 smithi083 ceph-mon[77242]: pgmap v7547: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:51.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:39:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:39:51.076Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:39:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:39:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:39:51] "GET /metrics HTTP/1.1" 200 225227 "" "Prometheus/2.33.4" 2022-09-29T09:39:52.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:39:51 smithi153 ceph-mon[70044]: pgmap v7548: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:52.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:39:51 smithi083 ceph-mon[77242]: pgmap v7548: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:52.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:39:51 smithi083 ceph-mon[79930]: pgmap v7548: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:52.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:39:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:39:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:39:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:39:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:39:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:39:54.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:39:54 smithi153 ceph-mon[70044]: pgmap v7549: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:54.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:39:53 smithi083 ceph-mon[79930]: pgmap v7549: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:54.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:39:53 smithi083 ceph-mon[77242]: pgmap v7549: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:54.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:39:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:39:54.461Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:39:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:39:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:39:55.167Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:39:56.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:39:56 smithi153 ceph-mon[70044]: pgmap v7550: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:56.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:39:56 smithi083 ceph-mon[77242]: pgmap v7550: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:56.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:39:56 smithi083 ceph-mon[79930]: pgmap v7550: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:58.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:39:58 smithi153 ceph-mon[70044]: pgmap v7551: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:58.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:39:58 smithi083 ceph-mon[79930]: pgmap v7551: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:39:58.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:39:58 smithi083 ceph-mon[77242]: pgmap v7551: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:40:00 smithi153 ceph-mon[70044]: pgmap v7552: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:40:00 smithi153 ceph-mon[70044]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T09:40:00.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:40:00 smithi083 ceph-mon[79930]: pgmap v7552: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:00.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:40:00 smithi083 ceph-mon[79930]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T09:40:00.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:40:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[77219]: 2022-09-29T09:39:59.999+0000 7f160f665700 -1 log_channel(cluster) log [ERR] : overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T09:40:00.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:40:00 smithi083 ceph-mon[77242]: pgmap v7552: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:00.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:40:00 smithi083 ceph-mon[77242]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T09:40:01.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:40:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:40:01.078Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:40:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:40:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:40:01] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T09:40:02.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:40:02 smithi153 ceph-mon[70044]: pgmap v7553: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:02.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:40:02 smithi083 ceph-mon[79930]: pgmap v7553: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:02.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:40:02 smithi083 ceph-mon[77242]: pgmap v7553: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:40:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:40:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:40:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:40:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:40:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:40:04.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:40:04 smithi153 ceph-mon[70044]: pgmap v7554: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:04.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:40:04 smithi083 ceph-mon[79930]: pgmap v7554: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:04.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:40:04 smithi083 ceph-mon[77242]: pgmap v7554: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:04.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:40:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:40:04.462Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:40:05.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:40:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:40:05.168Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:40:06.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:40:06 smithi153 ceph-mon[70044]: pgmap v7555: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:06.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:40:06 smithi083 ceph-mon[77242]: pgmap v7555: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:06.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:40:06 smithi083 ceph-mon[79930]: pgmap v7555: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:08.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:40:08 smithi153 ceph-mon[70044]: pgmap v7556: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:08.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:40:08 smithi083 ceph-mon[77242]: pgmap v7556: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:08.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:40:08 smithi083 ceph-mon[79930]: pgmap v7556: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:10.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:40:10 smithi153 ceph-mon[70044]: pgmap v7557: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:10.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:40:10 smithi083 ceph-mon[79930]: pgmap v7557: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:10.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:40:10 smithi083 ceph-mon[77242]: pgmap v7557: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:11.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:40:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:40:11.078Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:40:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:40:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:40:11] "GET /metrics HTTP/1.1" 200 225229 "" "Prometheus/2.33.4" 2022-09-29T09:40:12.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:40:12 smithi153 ceph-mon[70044]: pgmap v7558: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:12.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:40:12 smithi083 ceph-mon[79930]: pgmap v7558: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:12.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:40:12 smithi083 ceph-mon[77242]: pgmap v7558: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:12.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:40:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:40:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:40:13.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:40:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:40:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:40:14.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:40:14 smithi153 ceph-mon[70044]: pgmap v7559: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:14.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:40:14 smithi083 ceph-mon[79930]: pgmap v7559: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:14.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:40:14 smithi083 ceph-mon[77242]: pgmap v7559: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:14.615 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:40:14.905 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:40:14.906 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 45s ago 4h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:40:14.906 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 8m ago 4h 62.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:40:14.906 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 45s ago 4h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:40:14.906 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 8m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:40:14.907 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 8m ago 4h 443M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:40:14.907 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 45s ago 4h 731M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:40:14.907 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 45s ago 4h 959M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:40:14.907 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 8m ago 4h 407M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:40:14.908 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 45s ago 4h 562M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:40:14.908 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 45s ago 4h 26.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:40:14.908 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 8m ago 4h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:40:14.908 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 45s ago 4h 691M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:40:14.908 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 45s ago 4h 682M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:40:14.909 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 45s ago 4h 534M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:40:14.909 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 45s ago 4h 664M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:40:14.909 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 8m ago 4h 566M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:40:14.909 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 8m ago 4h 702M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:40:14.909 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 8m ago 4h 626M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:40:14.910 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 8m ago 4h 739M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:40:14.910 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 8m ago 4h 171M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:40:14.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:40:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:40:14.462Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:40:15.264 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:40:15.264 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:40:15.264 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:40:15.265 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:40:15.265 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:40:15.265 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:40:15.265 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:40:15.266 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:40:15.266 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:40:15.266 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:40:15.266 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:40:15.267 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:40:15.267 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:40:15.267 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:40:15.267 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:40:15.267 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:40:15.268 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:40:15.268 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:40:15.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:40:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:40:15.167Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:40:16.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:40:16 smithi153 ceph-mon[70044]: pgmap v7560: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:16.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:40:16 smithi153 ceph-mon[70044]: from='client.42927 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:40:16.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:40:16 smithi153 ceph-mon[70044]: from='client.42933 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:40:16.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:40:16 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2806201708' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:40:16.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:40:16 smithi083 ceph-mon[79930]: pgmap v7560: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:16.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:40:16 smithi083 ceph-mon[79930]: from='client.42927 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:40:16.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:40:16 smithi083 ceph-mon[79930]: from='client.42933 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:40:16.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:40:16 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2806201708' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:40:16.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:40:16 smithi083 ceph-mon[77242]: pgmap v7560: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:16.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:40:16 smithi083 ceph-mon[77242]: from='client.42927 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:40:16.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:40:16 smithi083 ceph-mon[77242]: from='client.42933 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:40:16.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:40:16 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2806201708' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:40:18.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:40:18 smithi153 ceph-mon[70044]: pgmap v7561: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:18.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:40:18 smithi083 ceph-mon[79930]: pgmap v7561: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:18.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:40:18 smithi083 ceph-mon[77242]: pgmap v7561: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:40:20 smithi153 ceph-mon[70044]: pgmap v7562: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:20.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:40:20 smithi083 ceph-mon[79930]: pgmap v7562: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:20.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:40:20 smithi083 ceph-mon[77242]: pgmap v7562: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:21.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:40:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:40:21.078Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:40:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:40:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:40:21] "GET /metrics HTTP/1.1" 200 225229 "" "Prometheus/2.33.4" 2022-09-29T09:40:22.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:40:22 smithi153 ceph-mon[70044]: pgmap v7563: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:22.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:40:22 smithi083 ceph-mon[77242]: pgmap v7563: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:22.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:40:22 smithi083 ceph-mon[79930]: pgmap v7563: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:40:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:40:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:40:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:40:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:40:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:40:24.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:40:24 smithi153 ceph-mon[70044]: pgmap v7564: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:24.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:40:24 smithi083 ceph-mon[79930]: pgmap v7564: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:24.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:40:24 smithi083 ceph-mon[77242]: pgmap v7564: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:24.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:40:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:40:24.464Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:40:25.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:40:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:40:25.169Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:40:26.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:40:26 smithi153 ceph-mon[70044]: pgmap v7565: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:26.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:40:26 smithi083 ceph-mon[79930]: pgmap v7565: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:26.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:40:26 smithi083 ceph-mon[77242]: pgmap v7565: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:28.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:40:28 smithi153 ceph-mon[70044]: pgmap v7566: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:28.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:40:28 smithi083 ceph-mon[79930]: pgmap v7566: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:28.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:40:28 smithi083 ceph-mon[77242]: pgmap v7566: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:30.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:40:30 smithi153 ceph-mon[70044]: pgmap v7567: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:30.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:40:30 smithi083 ceph-mon[77242]: pgmap v7567: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:30.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:40:30 smithi083 ceph-mon[79930]: pgmap v7567: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:31.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:40:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:40:31.079Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:40:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:40:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:40:31] "GET /metrics HTTP/1.1" 200 225229 "" "Prometheus/2.33.4" 2022-09-29T09:40:32.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:40:32 smithi153 ceph-mon[70044]: pgmap v7568: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:32.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:40:32 smithi083 ceph-mon[79930]: pgmap v7568: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:32.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:40:32 smithi083 ceph-mon[77242]: pgmap v7568: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:40:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:40:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:40:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:40:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:40:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:40:34.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:40:33 smithi153 ceph-mon[70044]: pgmap v7569: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:34.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:40:33 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:40:34.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:40:33 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:40:34.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:40:33 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:40:34.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:40:33 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:40:34.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:40:33 smithi083 ceph-mon[79930]: pgmap v7569: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:34.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:40:33 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:40:34.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:40:33 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:40:34.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:40:33 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:40:34.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:40:33 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:40:34.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:40:33 smithi083 ceph-mon[77242]: pgmap v7569: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:34.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:40:33 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:40:34.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:40:33 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:40:34.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:40:33 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:40:34.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:40:33 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:40:34.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:40:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:40:34.464Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:40:35.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:40:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:40:35.170Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:40:36.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:40:35 smithi153 ceph-mon[70044]: pgmap v7570: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:36.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:40:35 smithi083 ceph-mon[77242]: pgmap v7570: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:36.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:40:35 smithi083 ceph-mon[79930]: pgmap v7570: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:38.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:40:37 smithi153 ceph-mon[70044]: pgmap v7571: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:38.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:40:37 smithi083 ceph-mon[77242]: pgmap v7571: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:38.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:40:37 smithi083 ceph-mon[79930]: pgmap v7571: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:40:39 smithi153 ceph-mon[70044]: pgmap v7572: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:40:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:40:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:40:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:40:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:40:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:40:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:40:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:40:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:40:39 smithi083 ceph-mon[79930]: pgmap v7572: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:40:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:40:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:40:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:40:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:40:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:40:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:40:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:40:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:40:39 smithi083 ceph-mon[77242]: pgmap v7572: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:40:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:40:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:40:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:40:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:40:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:40:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:40:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:40:41.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:40:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:40:41.079Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:40:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:40:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:40:41] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T09:40:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:40:41 smithi153 ceph-mon[70044]: pgmap v7573: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:42.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:40:41 smithi083 ceph-mon[77242]: pgmap v7573: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:40:41 smithi083 ceph-mon[79930]: pgmap v7573: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:40:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:40:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:40:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:40:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:40:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:40:44.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:40:43 smithi153 ceph-mon[70044]: pgmap v7574: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:44.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:40:43 smithi083 ceph-mon[79930]: pgmap v7574: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:44.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:40:43 smithi083 ceph-mon[77242]: pgmap v7574: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:44.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:40:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:40:44.465Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:40:45.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:40:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:40:45.170Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:40:45.593 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:40:45.884 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:40:45.884 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 76s ago 4h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:40:45.885 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 9m ago 4h 62.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:40:45.885 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 76s ago 4h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:40:45.885 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 9m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:40:45.885 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 9m ago 4h 443M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:40:45.886 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 76s ago 4h 731M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:40:45.886 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 76s ago 4h 959M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:40:45.886 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 9m ago 4h 407M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:40:45.886 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 76s ago 4h 562M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:40:45.886 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 76s ago 4h 26.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:40:45.887 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 9m ago 4h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:40:45.887 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 76s ago 4h 691M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:40:45.887 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 76s ago 4h 682M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:40:45.887 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 76s ago 4h 534M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:40:45.887 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 76s ago 4h 664M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:40:45.888 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 9m ago 4h 566M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:40:45.888 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 9m ago 4h 702M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:40:45.888 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 9m ago 4h 626M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:40:45.888 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 9m ago 4h 739M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:40:45.888 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 9m ago 4h 171M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:40:46.240 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:40:46.241 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:40:46.241 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:40:46.241 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:40:46.241 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:40:46.242 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:40:46.242 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:40:46.242 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:40:46.242 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:40:46.243 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:40:46.243 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:40:46.243 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:40:46.243 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:40:46.243 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:40:46.244 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:40:46.244 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:40:46.244 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:40:46.244 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:40:46.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:40:46 smithi153 ceph-mon[70044]: pgmap v7575: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:46.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:40:46 smithi083 ceph-mon[79930]: pgmap v7575: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:46.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:40:46 smithi083 ceph-mon[77242]: pgmap v7575: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:47.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:40:47 smithi153 ceph-mon[70044]: from='client.52876 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:40:47.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:40:47 smithi153 ceph-mon[70044]: from='client.42951 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:40:47.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:40:47 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1452547182' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:40:47.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:40:47 smithi083 ceph-mon[79930]: from='client.52876 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:40:47.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:40:47 smithi083 ceph-mon[79930]: from='client.42951 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:40:47.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:40:47 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1452547182' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:40:47.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:40:47 smithi083 ceph-mon[77242]: from='client.52876 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:40:47.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:40:47 smithi083 ceph-mon[77242]: from='client.42951 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:40:47.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:40:47 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1452547182' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:40:48.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:40:48 smithi153 ceph-mon[70044]: pgmap v7576: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:48.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:40:48 smithi083 ceph-mon[79930]: pgmap v7576: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:48.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:40:48 smithi083 ceph-mon[77242]: pgmap v7576: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:50.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:40:50 smithi153 ceph-mon[70044]: pgmap v7577: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:50.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:40:50 smithi083 ceph-mon[79930]: pgmap v7577: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:50.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:40:50 smithi083 ceph-mon[77242]: pgmap v7577: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:51.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:40:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:40:51.079Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:40:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:40:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:40:51] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T09:40:52.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:40:52 smithi153 ceph-mon[70044]: pgmap v7578: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:52.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:40:52 smithi083 ceph-mon[77242]: pgmap v7578: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:52.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:40:52 smithi083 ceph-mon[79930]: pgmap v7578: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:52.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:40:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:40:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:40:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:40:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:40:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:40:54.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:40:54 smithi153 ceph-mon[70044]: pgmap v7579: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:54.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:40:54 smithi083 ceph-mon[79930]: pgmap v7579: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:54.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:40:54 smithi083 ceph-mon[77242]: pgmap v7579: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:54.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:40:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:40:54.465Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:40:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:40:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:40:55.171Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:40:56.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:40:56 smithi153 ceph-mon[70044]: pgmap v7580: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:56.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:40:56 smithi083 ceph-mon[79930]: pgmap v7580: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:56.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:40:56 smithi083 ceph-mon[77242]: pgmap v7580: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:58.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:40:58 smithi153 ceph-mon[70044]: pgmap v7581: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:58.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:40:58 smithi083 ceph-mon[79930]: pgmap v7581: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:40:58.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:40:58 smithi083 ceph-mon[77242]: pgmap v7581: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:41:00 smithi153 ceph-mon[70044]: pgmap v7582: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:00.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:41:00 smithi083 ceph-mon[77242]: pgmap v7582: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:00.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:41:00 smithi083 ceph-mon[79930]: pgmap v7582: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:01.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:41:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:41:01.080Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:41:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:41:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:41:01] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T09:41:02.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:41:02 smithi153 ceph-mon[70044]: pgmap v7583: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:02.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:41:02 smithi083 ceph-mon[79930]: pgmap v7583: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:02.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:41:02 smithi083 ceph-mon[77242]: pgmap v7583: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:41:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:41:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:41:03.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:41:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:41:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:41:04.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:41:04 smithi153 ceph-mon[70044]: pgmap v7584: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:04.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:41:04 smithi083 ceph-mon[77242]: pgmap v7584: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:04.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:41:04 smithi083 ceph-mon[79930]: pgmap v7584: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:04.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:41:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:41:04.466Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:41:05.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:41:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:41:05.171Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:41:06.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:41:06 smithi153 ceph-mon[70044]: pgmap v7585: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:06.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:41:06 smithi083 ceph-mon[79930]: pgmap v7585: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:06.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:41:06 smithi083 ceph-mon[77242]: pgmap v7585: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:08.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:41:08 smithi153 ceph-mon[70044]: pgmap v7586: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:08.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:41:08 smithi083 ceph-mon[79930]: pgmap v7586: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:08.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:41:08 smithi083 ceph-mon[77242]: pgmap v7586: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:10.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:41:10 smithi153 ceph-mon[70044]: pgmap v7587: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:10.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:41:10 smithi083 ceph-mon[79930]: pgmap v7587: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:10.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:41:10 smithi083 ceph-mon[77242]: pgmap v7587: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:11.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:41:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:41:11.080Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:41:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:41:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:41:11] "GET /metrics HTTP/1.1" 200 225228 "" "Prometheus/2.33.4" 2022-09-29T09:41:12.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:41:12 smithi153 ceph-mon[70044]: pgmap v7588: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:12.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:41:12 smithi083 ceph-mon[79930]: pgmap v7588: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:12.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:41:12 smithi083 ceph-mon[77242]: pgmap v7588: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:41:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:41:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:41:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:41:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:41:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:41:14.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:41:14 smithi153 ceph-mon[70044]: pgmap v7589: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:14.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:41:14 smithi083 ceph-mon[79930]: pgmap v7589: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:14.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:41:14 smithi083 ceph-mon[77242]: pgmap v7589: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:14.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:41:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:41:14.467Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:41:15.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:41:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:41:15.172Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:41:16.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:41:16 smithi153 ceph-mon[70044]: pgmap v7590: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:16.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:41:16 smithi083 ceph-mon[79930]: pgmap v7590: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:16.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:41:16 smithi083 ceph-mon[77242]: pgmap v7590: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:16.568 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:41:16.854 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:41:16.854 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 107s ago 4h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:41:16.854 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 9m ago 4h 62.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:41:16.854 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 107s ago 4h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:41:16.855 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 9m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:41:16.855 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 9m ago 4h 443M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:41:16.855 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 107s ago 4h 731M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:41:16.855 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 107s ago 4h 959M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:41:16.856 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 9m ago 4h 407M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:41:16.856 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 107s ago 4h 562M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:41:16.856 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 107s ago 4h 26.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:41:16.856 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 9m ago 4h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:41:16.857 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 107s ago 4h 691M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:41:16.857 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 107s ago 4h 682M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:41:16.857 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 107s ago 4h 534M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:41:16.857 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 107s ago 4h 664M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:41:16.857 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 9m ago 4h 566M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:41:16.858 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 9m ago 4h 702M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:41:16.858 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 9m ago 4h 626M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:41:16.858 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 9m ago 4h 739M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:41:16.858 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 9m ago 4h 171M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:41:17.208 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:41:17.209 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:41:17.209 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:41:17.209 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:41:17.209 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:41:17.210 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:41:17.210 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:41:17.210 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:41:17.210 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:41:17.211 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:41:17.211 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:41:17.211 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:41:17.211 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:41:17.212 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:41:17.212 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:41:17.212 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:41:17.212 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:41:17.212 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:41:18.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:41:18 smithi153 ceph-mon[70044]: pgmap v7591: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:18.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:41:18 smithi153 ceph-mon[70044]: from='client.52894 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:41:18.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:41:18 smithi153 ceph-mon[70044]: from='client.52900 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:41:18.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:41:18 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1655898949' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:41:18.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:41:18 smithi083 ceph-mon[79930]: pgmap v7591: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:18.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:41:18 smithi083 ceph-mon[79930]: from='client.52894 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:41:18.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:41:18 smithi083 ceph-mon[79930]: from='client.52900 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:41:18.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:41:18 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1655898949' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:41:18.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:41:18 smithi083 ceph-mon[77242]: pgmap v7591: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:18.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:41:18 smithi083 ceph-mon[77242]: from='client.52894 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:41:18.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:41:18 smithi083 ceph-mon[77242]: from='client.52900 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:41:18.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:41:18 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1655898949' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:41:20.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:41:20 smithi083 ceph-mon[79930]: pgmap v7592: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:20.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:41:20 smithi083 ceph-mon[77242]: pgmap v7592: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:20.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:41:20 smithi153 ceph-mon[70044]: pgmap v7592: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:21.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:41:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:41:21.081Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:41:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:41:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:41:21] "GET /metrics HTTP/1.1" 200 225228 "" "Prometheus/2.33.4" 2022-09-29T09:41:22.315 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:41:22 smithi153 ceph-mon[70044]: pgmap v7593: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:22.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:41:22 smithi083 ceph-mon[79930]: pgmap v7593: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:22.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:41:22 smithi083 ceph-mon[77242]: pgmap v7593: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:41:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:41:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:41:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:41:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:41:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:41:24.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:41:24 smithi083 ceph-mon[79930]: pgmap v7594: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:24.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:41:24 smithi083 ceph-mon[77242]: pgmap v7594: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:41:24 smithi153 ceph-mon[70044]: pgmap v7594: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:24.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:41:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:41:24.467Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:41:25.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:41:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:41:25.172Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:41:26.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:41:26 smithi083 ceph-mon[79930]: pgmap v7595: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:26.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:41:26 smithi083 ceph-mon[77242]: pgmap v7595: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:26.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:41:26 smithi153 ceph-mon[70044]: pgmap v7595: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:28.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:41:28 smithi083 ceph-mon[79930]: pgmap v7596: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:28.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:41:28 smithi083 ceph-mon[77242]: pgmap v7596: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:41:28 smithi153 ceph-mon[70044]: pgmap v7596: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:30.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:41:30 smithi083 ceph-mon[79930]: pgmap v7597: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:30.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:41:30 smithi083 ceph-mon[77242]: pgmap v7597: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:41:30 smithi153 ceph-mon[70044]: pgmap v7597: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:31.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:41:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:41:31.081Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:41:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:41:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:41:31] "GET /metrics HTTP/1.1" 200 225229 "" "Prometheus/2.33.4" 2022-09-29T09:41:32.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:41:32 smithi083 ceph-mon[79930]: pgmap v7598: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:32.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:41:32 smithi083 ceph-mon[77242]: pgmap v7598: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:41:32 smithi153 ceph-mon[70044]: pgmap v7598: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:32.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:41:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:41:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:41:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:41:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:41:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:41:34.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:41:34 smithi083 ceph-mon[79930]: pgmap v7599: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:34.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:41:34 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:41:34.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:41:34 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:41:34.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:41:34 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:41:34.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:41:34 smithi083 ceph-mon[77242]: pgmap v7599: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:34.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:41:34 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:41:34.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:41:34 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:41:34.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:41:34 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:41:34.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:41:34 smithi153 ceph-mon[70044]: pgmap v7599: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:41:34 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:41:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:41:34 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:41:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:41:34 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:41:34.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:41:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:41:34.467Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:41:35.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:41:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:41:35.173Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:41:36.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:41:36 smithi083 ceph-mon[79930]: pgmap v7600: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:36.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:41:36 smithi083 ceph-mon[77242]: pgmap v7600: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:41:36 smithi153 ceph-mon[70044]: pgmap v7600: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:38.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:41:38 smithi083 ceph-mon[79930]: pgmap v7601: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:38.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:41:38 smithi083 ceph-mon[77242]: pgmap v7601: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:38.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:41:38 smithi153 ceph-mon[70044]: pgmap v7601: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:39.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:41:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:41:39.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:41:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:41:39.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:41:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:41:39.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:41:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:41:39.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:41:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:41:39.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:41:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:41:39.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:41:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:41:39.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:41:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:41:39.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:41:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:41:39.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:41:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:41:39.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:41:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:41:39.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:41:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:41:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:41:40 smithi153 ceph-mon[70044]: pgmap v7602: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:41:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:41:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:41:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:41:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:41:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:41:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:41:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:41:40.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:41:40 smithi083 ceph-mon[79930]: pgmap v7602: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:41:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:41:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:41:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:41:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:41:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:41:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:41:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:41:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:41:40 smithi083 ceph-mon[77242]: pgmap v7602: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:41:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:41:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:41:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:41:40.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:41:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:41:40.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:41:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:41:41.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:41:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:41:41.081Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:41:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:41:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:41:41] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T09:41:42.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:41:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:41:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:41:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:41:42 smithi153 ceph-mon[70044]: pgmap v7603: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:41:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:41:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:41:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:41:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:41:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:41:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:41:42 smithi083 ceph-mon[79930]: pgmap v7603: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:41:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:41:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:41:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:41:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:41:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:41:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:41:42 smithi083 ceph-mon[77242]: pgmap v7603: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:41:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:41:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:41:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:41:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:41:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:41:43.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:41:43 smithi153 ceph-mon[70044]: Detected new or changed devices on smithi153 2022-09-29T09:41:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:41:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:41:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:41:43.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:41:43 smithi083 ceph-mon[79930]: Detected new or changed devices on smithi153 2022-09-29T09:41:43.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:41:43 smithi083 ceph-mon[77242]: Detected new or changed devices on smithi153 2022-09-29T09:41:44.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:41:44 smithi153 ceph-mon[70044]: pgmap v7604: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:44.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:41:44 smithi083 ceph-mon[79930]: pgmap v7604: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:44.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:41:44 smithi083 ceph-mon[77242]: pgmap v7604: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:44.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:41:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:41:44.468Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:41:45.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:41:45 smithi083 ceph-mon[79930]: pgmap v7605: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:45.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:41:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:41:45.173Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:41:45.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:41:45 smithi083 ceph-mon[77242]: pgmap v7605: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:45.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:41:45 smithi153 ceph-mon[70044]: pgmap v7605: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:47.534 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:41:47.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:41:47 smithi083 ceph-mon[79930]: pgmap v7606: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:47.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:41:47 smithi083 ceph-mon[77242]: pgmap v7606: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:47.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:41:47 smithi153 ceph-mon[70044]: pgmap v7606: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:47.820 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:41:47.821 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 2m ago 4h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:41:47.821 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 9s ago 4h 62.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:41:47.821 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 2m ago 4h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:41:47.822 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 9s ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:41:47.822 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 9s ago 4h 443M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:41:47.822 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 2m ago 4h 731M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:41:47.822 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 2m ago 4h 959M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:41:47.823 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 9s ago 4h 422M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:41:47.823 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 2m ago 4h 562M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:41:47.823 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 2m ago 4h 26.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:41:47.823 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 9s ago 4h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:41:47.824 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 2m ago 4h 691M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:41:47.824 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 2m ago 4h 682M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:41:47.824 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 2m ago 4h 534M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:41:47.824 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 2m ago 4h 664M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:41:47.825 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 9s ago 4h 587M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:41:47.825 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 9s ago 4h 730M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:41:47.825 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 9s ago 4h 651M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:41:47.826 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 9s ago 4h 768M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:41:47.826 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 9s ago 4h 173M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:41:48.178 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:41:48.179 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:41:48.179 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:41:48.179 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:41:48.179 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:41:48.179 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:41:48.180 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:41:48.180 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:41:48.180 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:41:48.180 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:41:48.181 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:41:48.181 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:41:48.181 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:41:48.181 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:41:48.182 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:41:48.182 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:41:48.182 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:41:48.183 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:41:49.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:41:49 smithi153 ceph-mon[70044]: from='client.52912 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:41:49.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:41:49 smithi153 ceph-mon[70044]: from='client.52918 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:41:49.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:41:49 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3792230607' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:41:49.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:41:49 smithi083 ceph-mon[79930]: from='client.52912 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:41:49.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:41:49 smithi083 ceph-mon[79930]: from='client.52918 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:41:49.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:41:49 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3792230607' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:41:49.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:41:49 smithi083 ceph-mon[77242]: from='client.52912 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:41:49.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:41:49 smithi083 ceph-mon[77242]: from='client.52918 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:41:49.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:41:49 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3792230607' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:41:50.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:41:50 smithi153 ceph-mon[70044]: pgmap v7607: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:50.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:41:50 smithi083 ceph-mon[79930]: pgmap v7607: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:50.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:41:50 smithi083 ceph-mon[77242]: pgmap v7607: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:51.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:41:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:41:51.082Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:41:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:41:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:41:51] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T09:41:52.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:41:52 smithi153 ceph-mon[70044]: pgmap v7608: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:52.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:41:52 smithi083 ceph-mon[79930]: pgmap v7608: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:52.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:41:52 smithi083 ceph-mon[77242]: pgmap v7608: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:52.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:41:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:41:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:41:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:41:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:41:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:41:54.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:41:54 smithi153 ceph-mon[70044]: pgmap v7609: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:54.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:41:54 smithi083 ceph-mon[79930]: pgmap v7609: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:54.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:41:54 smithi083 ceph-mon[77242]: pgmap v7609: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:54.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:41:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:41:54.468Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:41:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:41:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:41:55.175Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:41:56.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:41:56 smithi153 ceph-mon[70044]: pgmap v7610: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:56.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:41:56 smithi083 ceph-mon[79930]: pgmap v7610: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:56.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:41:56 smithi083 ceph-mon[77242]: pgmap v7610: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:58.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:41:58 smithi153 ceph-mon[70044]: pgmap v7611: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:58.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:41:58 smithi083 ceph-mon[79930]: pgmap v7611: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:41:58.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:41:58 smithi083 ceph-mon[77242]: pgmap v7611: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:42:00 smithi153 ceph-mon[70044]: pgmap v7612: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:00.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:42:00 smithi083 ceph-mon[79930]: pgmap v7612: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:00.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:42:00 smithi083 ceph-mon[77242]: pgmap v7612: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:01.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:42:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:42:01.082Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:42:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:42:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:42:01] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T09:42:02.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:42:02 smithi153 ceph-mon[70044]: pgmap v7613: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:02.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:42:02 smithi083 ceph-mon[77242]: pgmap v7613: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:02.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:42:02 smithi083 ceph-mon[79930]: pgmap v7613: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:02.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:42:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:42:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:42:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:42:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:42:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:42:04.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:42:04 smithi153 ceph-mon[70044]: pgmap v7614: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:04.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:42:04 smithi083 ceph-mon[79930]: pgmap v7614: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:04.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:42:04 smithi083 ceph-mon[77242]: pgmap v7614: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:04.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:42:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:42:04.469Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:42:05.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:42:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:42:05.175Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:42:06.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:42:06 smithi153 ceph-mon[70044]: pgmap v7615: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:06.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:42:06 smithi083 ceph-mon[77242]: pgmap v7615: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:06.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:42:06 smithi083 ceph-mon[79930]: pgmap v7615: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:08.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:42:08 smithi153 ceph-mon[70044]: pgmap v7616: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:08.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:42:08 smithi083 ceph-mon[79930]: pgmap v7616: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:08.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:42:08 smithi083 ceph-mon[77242]: pgmap v7616: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:10.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:42:10 smithi153 ceph-mon[70044]: pgmap v7617: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:10.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:42:10 smithi083 ceph-mon[79930]: pgmap v7617: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:10.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:42:10 smithi083 ceph-mon[77242]: pgmap v7617: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:11.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:42:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:42:11.084Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:42:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:42:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:42:11] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T09:42:12.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:42:12 smithi153 ceph-mon[70044]: pgmap v7618: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:12.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:42:12 smithi083 ceph-mon[79930]: pgmap v7618: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:12.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:42:12 smithi083 ceph-mon[77242]: pgmap v7618: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:12.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:42:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:42:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:42:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:42:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:42:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:42:14.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:42:14 smithi153 ceph-mon[70044]: pgmap v7619: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:14.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:42:14 smithi083 ceph-mon[77242]: pgmap v7619: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:14.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:42:14 smithi083 ceph-mon[79930]: pgmap v7619: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:14.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:42:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:42:14.470Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:42:15.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:42:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:42:15.175Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:42:16.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:42:16 smithi153 ceph-mon[70044]: pgmap v7620: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:16.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:42:16 smithi083 ceph-mon[79930]: pgmap v7620: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:16.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:42:16 smithi083 ceph-mon[77242]: pgmap v7620: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:18.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:42:18 smithi153 ceph-mon[70044]: pgmap v7621: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:18.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:42:18 smithi083 ceph-mon[77242]: pgmap v7621: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:18.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:42:18 smithi083 ceph-mon[79930]: pgmap v7621: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:18.507 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:42:18.798 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:42:18.798 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 2m ago 4h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:42:18.799 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 40s ago 4h 62.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:42:18.799 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 2m ago 4h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:42:18.799 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 40s ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:42:18.799 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 40s ago 4h 443M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:42:18.799 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 2m ago 4h 731M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:42:18.800 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 2m ago 4h 959M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:42:18.800 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 40s ago 4h 422M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:42:18.800 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 2m ago 4h 562M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:42:18.800 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 2m ago 4h 26.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:42:18.800 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 40s ago 4h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:42:18.801 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 2m ago 4h 691M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:42:18.801 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 2m ago 4h 682M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:42:18.801 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 2m ago 4h 534M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:42:18.801 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 2m ago 4h 664M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:42:18.802 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 40s ago 4h 587M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:42:18.802 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 40s ago 4h 730M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:42:18.802 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 40s ago 4h 651M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:42:18.802 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 40s ago 4h 768M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:42:18.802 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 40s ago 4h 173M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:42:19.151 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:42:19.151 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:42:19.151 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:42:19.152 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:42:19.152 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:42:19.152 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:42:19.153 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:42:19.153 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:42:19.153 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:42:19.153 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:42:19.154 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:42:19.154 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:42:19.154 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:42:19.154 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:42:19.155 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:42:19.155 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:42:19.155 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:42:19.155 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:42:20.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:42:20 smithi083 ceph-mon[79930]: pgmap v7622: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:20.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:42:20 smithi083 ceph-mon[79930]: from='client.42999 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:42:20.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:42:20 smithi083 ceph-mon[79930]: from='client.43005 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:42:20.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:42:20 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3760646298' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:42:20.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:42:20 smithi083 ceph-mon[77242]: pgmap v7622: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:20.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:42:20 smithi083 ceph-mon[77242]: from='client.42999 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:42:20.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:42:20 smithi083 ceph-mon[77242]: from='client.43005 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:42:20.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:42:20 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3760646298' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:42:20.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:42:20 smithi153 ceph-mon[70044]: pgmap v7622: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:42:20 smithi153 ceph-mon[70044]: from='client.42999 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:42:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:42:20 smithi153 ceph-mon[70044]: from='client.43005 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:42:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:42:20 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3760646298' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:42:21.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:42:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:42:21.085Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:42:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:42:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:42:21] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T09:42:22.316 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:42:22 smithi153 ceph-mon[70044]: pgmap v7623: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:22.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:42:22 smithi083 ceph-mon[77242]: pgmap v7623: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:22.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:42:22 smithi083 ceph-mon[79930]: pgmap v7623: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:22.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:42:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:42:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:42:23.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:42:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:42:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:42:24.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:42:24 smithi083 ceph-mon[77242]: pgmap v7624: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:24.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:42:24 smithi083 ceph-mon[79930]: pgmap v7624: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:42:24 smithi153 ceph-mon[70044]: pgmap v7624: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:24.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:42:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:42:24.471Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:42:25.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:42:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:42:25.176Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:42:26.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:42:26 smithi083 ceph-mon[77242]: pgmap v7625: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:26.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:42:26 smithi083 ceph-mon[79930]: pgmap v7625: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:26.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:42:26 smithi153 ceph-mon[70044]: pgmap v7625: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:28.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:42:28 smithi083 ceph-mon[79930]: pgmap v7626: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:28.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:42:28 smithi083 ceph-mon[77242]: pgmap v7626: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:42:28 smithi153 ceph-mon[70044]: pgmap v7626: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:30.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:42:30 smithi083 ceph-mon[79930]: pgmap v7627: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:30.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:42:30 smithi083 ceph-mon[77242]: pgmap v7627: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:42:30 smithi153 ceph-mon[70044]: pgmap v7627: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:31.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:42:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:42:31.085Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:42:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:42:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:42:31] "GET /metrics HTTP/1.1" 200 225229 "" "Prometheus/2.33.4" 2022-09-29T09:42:32.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:42:32 smithi083 ceph-mon[79930]: pgmap v7628: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:32.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:42:32 smithi083 ceph-mon[77242]: pgmap v7628: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:42:32 smithi153 ceph-mon[70044]: pgmap v7628: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:32.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:42:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:42:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:42:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:42:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:42:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:42:34.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:42:34 smithi083 ceph-mon[79930]: pgmap v7629: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:34.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:42:34 smithi083 ceph-mon[77242]: pgmap v7629: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:42:34 smithi153 ceph-mon[70044]: pgmap v7629: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:34.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:42:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:42:34.472Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:42:35.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:42:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:42:35.177Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:42:36.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:42:36 smithi083 ceph-mon[77242]: pgmap v7630: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:36.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:42:36 smithi083 ceph-mon[79930]: pgmap v7630: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:42:36 smithi153 ceph-mon[70044]: pgmap v7630: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:38.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:42:38 smithi083 ceph-mon[77242]: pgmap v7631: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:38.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:42:38 smithi083 ceph-mon[79930]: pgmap v7631: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:42:38 smithi153 ceph-mon[70044]: pgmap v7631: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:40.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:42:40 smithi083 ceph-mon[79930]: pgmap v7632: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:42:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:42:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:42:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:42:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:42:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:42:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:42:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:42:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:42:40 smithi083 ceph-mon[77242]: pgmap v7632: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:42:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:42:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:42:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:42:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:42:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:42:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:42:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:42:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:42:40 smithi153 ceph-mon[70044]: pgmap v7632: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:42:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:42:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:42:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:42:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:42:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:42:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:42:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:42:41.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:42:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:42:41.086Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:42:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:42:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:42:41] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T09:42:42.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:42:42 smithi083 ceph-mon[79930]: pgmap v7633: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:42:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:42:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:42:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:42:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:42:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:42:42.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:42:42 smithi083 ceph-mon[77242]: pgmap v7633: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:42.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:42:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:42:42.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:42:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:42:42.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:42:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:42:42.559 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:42:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:42:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:42:42.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:42:42 smithi153 ceph-mon[70044]: pgmap v7633: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:42.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:42:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:42:42.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:42:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:42:42.561 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:42:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:42:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:42:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:42:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:42:43.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:42:43 smithi083 ceph-mon[79930]: pgmap v7634: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:43.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:42:43 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:42:43.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:42:43 smithi083 ceph-mon[77242]: pgmap v7634: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:43.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:42:43 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:42:43.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:42:43 smithi153 ceph-mon[70044]: pgmap v7634: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:43.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:42:43 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:42:44.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:42:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:42:44.472Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:42:45.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:42:45 smithi083 ceph-mon[79930]: pgmap v7635: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:45.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:42:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:42:45.178Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:42:45.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:42:45 smithi083 ceph-mon[77242]: pgmap v7635: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:45.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:42:45 smithi153 ceph-mon[70044]: pgmap v7635: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:47.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:42:47 smithi083 ceph-mon[77242]: pgmap v7636: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:47.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:42:47 smithi083 ceph-mon[79930]: pgmap v7636: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:47.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:42:47 smithi153 ceph-mon[70044]: pgmap v7636: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:49.476 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:42:49.764 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:42:49.764 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 3m ago 4h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:42:49.764 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 71s ago 4h 62.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:42:49.765 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 3m ago 4h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:42:49.765 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 71s ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:42:49.765 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 71s ago 4h 443M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:42:49.765 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 3m ago 4h 731M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:42:49.766 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 3m ago 4h 959M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:42:49.766 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 71s ago 4h 422M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:42:49.766 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 3m ago 4h 562M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:42:49.766 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 3m ago 4h 26.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:42:49.767 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 71s ago 4h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:42:49.767 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 3m ago 4h 691M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:42:49.767 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 3m ago 4h 682M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:42:49.767 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 3m ago 4h 534M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:42:49.768 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 3m ago 4h 664M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:42:49.768 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 71s ago 4h 587M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:42:49.768 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 71s ago 4h 730M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:42:49.768 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 71s ago 4h 651M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:42:49.768 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 71s ago 4h 768M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:42:49.769 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 71s ago 4h 173M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:42:50.120 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:42:50.121 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:42:50.121 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:42:50.121 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:42:50.121 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:42:50.122 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:42:50.122 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:42:50.122 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:42:50.122 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:42:50.122 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:42:50.123 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:42:50.123 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:42:50.123 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:42:50.123 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:42:50.124 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:42:50.124 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:42:50.124 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:42:50.124 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:42:50.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:42:50 smithi153 ceph-mon[70044]: pgmap v7637: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:50.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:42:50 smithi083 ceph-mon[77242]: pgmap v7637: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:50.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:42:50 smithi083 ceph-mon[79930]: pgmap v7637: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:51.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:42:51 smithi153 ceph-mon[70044]: from='client.43017 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:42:51.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:42:51 smithi153 ceph-mon[70044]: from='client.43023 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:42:51.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:42:51 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4124822723' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:42:51.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:42:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:42:51.086Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:42:51.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:42:51 smithi083 ceph-mon[77242]: from='client.43017 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:42:51.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:42:51 smithi083 ceph-mon[77242]: from='client.43023 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:42:51.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:42:51 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4124822723' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:42:51.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:42:51 smithi083 ceph-mon[79930]: from='client.43017 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:42:51.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:42:51 smithi083 ceph-mon[79930]: from='client.43023 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:42:51.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:42:51 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4124822723' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:42:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:42:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:42:51] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T09:42:52.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:42:52 smithi153 ceph-mon[70044]: pgmap v7638: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:52.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:42:52 smithi083 ceph-mon[77242]: pgmap v7638: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:52.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:42:52 smithi083 ceph-mon[79930]: pgmap v7638: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:42:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:42:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:42:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:42:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:42:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:42:54.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:42:54 smithi153 ceph-mon[70044]: pgmap v7639: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:54.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:42:54 smithi083 ceph-mon[79930]: pgmap v7639: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:54.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:42:54 smithi083 ceph-mon[77242]: pgmap v7639: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:54.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:42:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:42:54.472Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:42:55.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:42:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:42:55.179Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:42:56.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:42:56 smithi153 ceph-mon[70044]: pgmap v7640: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:56.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:42:56 smithi083 ceph-mon[79930]: pgmap v7640: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:56.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:42:56 smithi083 ceph-mon[77242]: pgmap v7640: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:58.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:42:58 smithi153 ceph-mon[70044]: pgmap v7641: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:58.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:42:58 smithi083 ceph-mon[77242]: pgmap v7641: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:42:58.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:42:58 smithi083 ceph-mon[79930]: pgmap v7641: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:00.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:43:00 smithi153 ceph-mon[70044]: pgmap v7642: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:00.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:43:00 smithi083 ceph-mon[79930]: pgmap v7642: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:00.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:43:00 smithi083 ceph-mon[77242]: pgmap v7642: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:01.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:43:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:43:01.086Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:43:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:43:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:43:01] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T09:43:02.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:43:02 smithi153 ceph-mon[70044]: pgmap v7643: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:02.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:43:02 smithi083 ceph-mon[77242]: pgmap v7643: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:02.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:43:02 smithi083 ceph-mon[79930]: pgmap v7643: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:43:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:43:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:43:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:43:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:43:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:43:04.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:43:04 smithi153 ceph-mon[70044]: pgmap v7644: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:04.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:43:04 smithi083 ceph-mon[79930]: pgmap v7644: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:04.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:43:04 smithi083 ceph-mon[77242]: pgmap v7644: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:04.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:43:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:43:04.473Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:43:05.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:43:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:43:05.179Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:43:06.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:43:06 smithi153 ceph-mon[70044]: pgmap v7645: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:06.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:43:06 smithi083 ceph-mon[79930]: pgmap v7645: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:06.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:43:06 smithi083 ceph-mon[77242]: pgmap v7645: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:08.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:43:08 smithi153 ceph-mon[70044]: pgmap v7646: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:08.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:43:08 smithi083 ceph-mon[77242]: pgmap v7646: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:08.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:43:08 smithi083 ceph-mon[79930]: pgmap v7646: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:10.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:43:10 smithi153 ceph-mon[70044]: pgmap v7647: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:10.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:43:10 smithi083 ceph-mon[79930]: pgmap v7647: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:10.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:43:10 smithi083 ceph-mon[77242]: pgmap v7647: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:11.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:43:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:43:11.087Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:43:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:43:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:43:11] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T09:43:12.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:43:12 smithi153 ceph-mon[70044]: pgmap v7648: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:12.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:43:12 smithi083 ceph-mon[79930]: pgmap v7648: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:12.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:43:12 smithi083 ceph-mon[77242]: pgmap v7648: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:43:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:43:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:43:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:43:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:43:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:43:14.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:43:14 smithi083 ceph-mon[79930]: pgmap v7649: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:14.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:43:14 smithi083 ceph-mon[77242]: pgmap v7649: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:14.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:43:14 smithi153 ceph-mon[70044]: pgmap v7649: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:14.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:43:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:43:14.473Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:43:15.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:43:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:43:15.179Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:43:16.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:43:16 smithi083 ceph-mon[79930]: pgmap v7650: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:16.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:43:16 smithi083 ceph-mon[77242]: pgmap v7650: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:43:16 smithi153 ceph-mon[70044]: pgmap v7650: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:18.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:43:18 smithi083 ceph-mon[79930]: pgmap v7651: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:18.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:43:18 smithi083 ceph-mon[77242]: pgmap v7651: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:18.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:43:18 smithi153 ceph-mon[70044]: pgmap v7651: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:20.443 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:43:20.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:43:20 smithi083 ceph-mon[79930]: pgmap v7652: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:20.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:43:20 smithi083 ceph-mon[77242]: pgmap v7652: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:20.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:43:20 smithi153 ceph-mon[70044]: pgmap v7652: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:20.732 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:43:20.732 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 3m ago 4h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:43:20.732 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 102s ago 4h 62.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:43:20.733 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 3m ago 4h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:43:20.733 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 102s ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:43:20.733 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 102s ago 4h 443M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:43:20.733 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 3m ago 4h 731M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:43:20.734 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 3m ago 4h 959M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:43:20.734 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 102s ago 4h 422M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:43:20.734 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 3m ago 4h 562M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:43:20.734 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 3m ago 4h 26.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:43:20.735 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 102s ago 4h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:43:20.736 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 3m ago 4h 691M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:43:20.736 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 3m ago 4h 682M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:43:20.736 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 3m ago 4h 534M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:43:20.736 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 3m ago 4h 664M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:43:20.737 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 102s ago 4h 587M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:43:20.737 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 102s ago 4h 730M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:43:20.737 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 102s ago 4h 651M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:43:20.737 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 102s ago 4h 768M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:43:20.738 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 102s ago 4h 173M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:43:21.090 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:43:21.090 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:43:21.091 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:43:21.091 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:43:21.091 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:43:21.091 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:43:21.092 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:43:21.092 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:43:21.092 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:43:21.092 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:43:21.093 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:43:21.093 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:43:21.093 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:43:21.093 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:43:21.093 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:43:21.094 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:43:21.094 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:43:21.094 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:43:21.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:43:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:43:21.087Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:43:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:43:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:43:21] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T09:43:22.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:43:22 smithi083 ceph-mon[79930]: pgmap v7653: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:22.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:43:22 smithi083 ceph-mon[79930]: from='client.52966 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:43:22.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:43:22 smithi083 ceph-mon[79930]: from='client.43041 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:43:22.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:43:22 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2104649268' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:43:22.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:43:22 smithi083 ceph-mon[77242]: pgmap v7653: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:22.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:43:22 smithi083 ceph-mon[77242]: from='client.52966 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:43:22.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:43:22 smithi083 ceph-mon[77242]: from='client.43041 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:43:22.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:43:22 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2104649268' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:43:22.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:43:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:43:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:43:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:43:22 smithi153 ceph-mon[70044]: pgmap v7653: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:43:22 smithi153 ceph-mon[70044]: from='client.52966 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:43:22.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:43:22 smithi153 ceph-mon[70044]: from='client.43041 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:43:22.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:43:22 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2104649268' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:43:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:43:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:43:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:43:24.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:43:24 smithi083 ceph-mon[77242]: pgmap v7654: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:24.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:43:24 smithi083 ceph-mon[79930]: pgmap v7654: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:24.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:43:24 smithi153 ceph-mon[70044]: pgmap v7654: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:24.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:43:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:43:24.473Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:43:25.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:43:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:43:25.180Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:43:26.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:43:26 smithi083 ceph-mon[77242]: pgmap v7655: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:26.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:43:26 smithi083 ceph-mon[79930]: pgmap v7655: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:43:26 smithi153 ceph-mon[70044]: pgmap v7655: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:28.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:43:28 smithi083 ceph-mon[79930]: pgmap v7656: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:28.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:43:28 smithi083 ceph-mon[77242]: pgmap v7656: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:28.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:43:28 smithi153 ceph-mon[70044]: pgmap v7656: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:30.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:43:30 smithi083 ceph-mon[79930]: pgmap v7657: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:30.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:43:30 smithi083 ceph-mon[77242]: pgmap v7657: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:43:30 smithi153 ceph-mon[70044]: pgmap v7657: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:31.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:43:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:43:31.089Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:43:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:43:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:43:31] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T09:43:32.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:43:32 smithi083 ceph-mon[79930]: pgmap v7658: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:32.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:43:32 smithi083 ceph-mon[77242]: pgmap v7658: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:43:32 smithi153 ceph-mon[70044]: pgmap v7658: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:32.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:43:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:43:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:43:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:43:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:43:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:43:34.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:43:34 smithi083 ceph-mon[79930]: pgmap v7659: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:34.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:43:34 smithi083 ceph-mon[77242]: pgmap v7659: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:34.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:43:34 smithi153 ceph-mon[70044]: pgmap v7659: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:34.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:43:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:43:34.475Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:43:35.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:43:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:43:35.181Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:43:36.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:43:36 smithi083 ceph-mon[77242]: pgmap v7660: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:36.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:43:36 smithi083 ceph-mon[79930]: pgmap v7660: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:43:36 smithi153 ceph-mon[70044]: pgmap v7660: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:38.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:43:38 smithi083 ceph-mon[79930]: pgmap v7661: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:38.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:43:38 smithi083 ceph-mon[77242]: pgmap v7661: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:38.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:43:38 smithi153 ceph-mon[70044]: pgmap v7661: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:43:40 smithi083 ceph-mon[79930]: pgmap v7662: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:43:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:43:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:43:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:43:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:43:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:43:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:43:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:43:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:43:40 smithi083 ceph-mon[77242]: pgmap v7662: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:43:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:43:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:43:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:43:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:43:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:43:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:43:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:43:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:43:40 smithi153 ceph-mon[70044]: pgmap v7662: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:43:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:43:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:43:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:43:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:43:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:43:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:43:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:43:41.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:43:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:43:41.089Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:43:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:43:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:43:41] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T09:43:42.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:43:42 smithi083 ceph-mon[79930]: pgmap v7663: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:42.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:43:42 smithi083 ceph-mon[77242]: pgmap v7663: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:42.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:43:42 smithi153 ceph-mon[70044]: pgmap v7663: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:42.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:43:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:43:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:43:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:43:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:43:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:43:43.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:43:43 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:43:43.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:43:43 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:43:43.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:43:43 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:43:43.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:43:43 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:43:43.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:43:43 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:43:43.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:43:43 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:43:43.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:43:43 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:43:43.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:43:43 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:43:43.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:43:43 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:43:43.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:43:43 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:43:43.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:43:43 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:43:43.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:43:43 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:43:44.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:43:44 smithi083 ceph-mon[79930]: pgmap v7664: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:44.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:43:44 smithi083 ceph-mon[77242]: pgmap v7664: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:44.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:43:44 smithi153 ceph-mon[70044]: pgmap v7664: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:44.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:43:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:43:44.476Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:43:45.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:43:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:43:45.182Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:43:46.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:43:46 smithi083 ceph-mon[77242]: pgmap v7665: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:46.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:43:46 smithi083 ceph-mon[79930]: pgmap v7665: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:46.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:43:46 smithi153 ceph-mon[70044]: pgmap v7665: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:48.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:43:48 smithi083 ceph-mon[79930]: pgmap v7666: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:48.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:43:48 smithi083 ceph-mon[77242]: pgmap v7666: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:48.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:43:48 smithi153 ceph-mon[70044]: pgmap v7666: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:50.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:43:50 smithi083 ceph-mon[79930]: pgmap v7667: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:50.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:43:50 smithi083 ceph-mon[77242]: pgmap v7667: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:50.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:43:50 smithi153 ceph-mon[70044]: pgmap v7667: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:51.418 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:43:51.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:43:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:43:51.090Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:43:51.704 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:43:51.704 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 4m ago 4h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:43:51.704 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 2m ago 4h 62.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:43:51.705 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 4m ago 4h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:43:51.705 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 2m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:43:51.705 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 2m ago 4h 443M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:43:51.705 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 4m ago 4h 731M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:43:51.705 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 4m ago 4h 959M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:43:51.706 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 2m ago 4h 422M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:43:51.706 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 4m ago 4h 562M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:43:51.706 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 4m ago 4h 26.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:43:51.706 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 2m ago 4h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:43:51.707 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 4m ago 4h 691M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:43:51.707 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 4m ago 4h 682M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:43:51.707 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 4m ago 4h 534M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:43:51.707 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 4m ago 4h 664M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:43:51.707 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 2m ago 4h 587M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:43:51.708 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 2m ago 4h 730M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:43:51.708 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 2m ago 4h 651M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:43:51.708 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 2m ago 4h 768M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:43:51.708 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 2m ago 4h 173M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:43:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:43:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:43:51] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T09:43:52.064 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:43:52.064 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:43:52.065 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:43:52.065 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:43:52.065 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:43:52.065 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:43:52.066 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:43:52.066 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:43:52.066 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:43:52.066 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:43:52.066 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:43:52.067 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:43:52.067 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:43:52.067 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:43:52.067 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:43:52.068 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:43:52.068 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:43:52.068 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:43:52.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:43:52 smithi083 ceph-mon[79930]: pgmap v7668: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:52.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:43:52 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/37394564' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:43:52.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:43:52 smithi083 ceph-mon[77242]: pgmap v7668: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:52.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:43:52 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/37394564' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:43:52.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:43:52 smithi153 ceph-mon[70044]: pgmap v7668: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:43:52 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/37394564' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:43:52.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:43:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:43:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:43:53.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:43:53 smithi153 ceph-mon[70044]: from='client.43053 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:43:53.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:43:53 smithi153 ceph-mon[70044]: from='client.52990 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:43:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:43:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:43:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:43:53.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:43:53 smithi083 ceph-mon[77242]: from='client.43053 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:43:53.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:43:53 smithi083 ceph-mon[77242]: from='client.52990 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:43:53.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:43:53 smithi083 ceph-mon[79930]: from='client.43053 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:43:53.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:43:53 smithi083 ceph-mon[79930]: from='client.52990 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:43:54.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:43:54 smithi083 ceph-mon[79930]: pgmap v7669: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:54.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:43:54 smithi083 ceph-mon[77242]: pgmap v7669: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:54.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:43:54 smithi153 ceph-mon[70044]: pgmap v7669: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:54.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:43:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:43:54.476Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:43:55.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:43:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:43:55.182Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:43:56.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:43:56 smithi083 ceph-mon[77242]: pgmap v7670: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:56.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:43:56 smithi083 ceph-mon[79930]: pgmap v7670: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:56.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:43:56 smithi153 ceph-mon[70044]: pgmap v7670: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:58.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:43:58 smithi083 ceph-mon[77242]: pgmap v7671: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:58.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:43:58 smithi083 ceph-mon[79930]: pgmap v7671: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:43:58.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:43:58 smithi153 ceph-mon[70044]: pgmap v7671: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:00.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:44:00 smithi083 ceph-mon[77242]: pgmap v7672: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:00.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:44:00 smithi083 ceph-mon[79930]: pgmap v7672: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:00.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:44:00 smithi153 ceph-mon[70044]: pgmap v7672: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:01.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:44:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:44:01.090Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:44:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:44:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:44:01] "GET /metrics HTTP/1.1" 200 225229 "" "Prometheus/2.33.4" 2022-09-29T09:44:02.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:44:02 smithi083 ceph-mon[79930]: pgmap v7673: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:02.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:44:02 smithi083 ceph-mon[77242]: pgmap v7673: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:44:02 smithi153 ceph-mon[70044]: pgmap v7673: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:02.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:44:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:44:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:44:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:44:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:44:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:44:04.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:44:04 smithi083 ceph-mon[79930]: pgmap v7674: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:04.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:44:04 smithi083 ceph-mon[77242]: pgmap v7674: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:04.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:44:04 smithi153 ceph-mon[70044]: pgmap v7674: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:04.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:44:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:44:04.476Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:44:05.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:44:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:44:05.183Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:44:06.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:44:06 smithi083 ceph-mon[79930]: pgmap v7675: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:06.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:44:06 smithi083 ceph-mon[77242]: pgmap v7675: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:06.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:44:06 smithi153 ceph-mon[70044]: pgmap v7675: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:08.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:44:08 smithi083 ceph-mon[79930]: pgmap v7676: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:08.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:44:08 smithi083 ceph-mon[77242]: pgmap v7676: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:08.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:44:08 smithi153 ceph-mon[70044]: pgmap v7676: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:10.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:44:10 smithi083 ceph-mon[79930]: pgmap v7677: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:10.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:44:10 smithi083 ceph-mon[77242]: pgmap v7677: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:10.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:44:10 smithi153 ceph-mon[70044]: pgmap v7677: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:11.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:44:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:44:11.091Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:44:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:44:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:44:11] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T09:44:12.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:44:12 smithi083 ceph-mon[77242]: pgmap v7678: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:12.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:44:12 smithi083 ceph-mon[79930]: pgmap v7678: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:12.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:44:12 smithi153 ceph-mon[70044]: pgmap v7678: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:12.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:44:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:44:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:44:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:44:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:44:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:44:14.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:44:14 smithi083 ceph-mon[79930]: pgmap v7679: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:14.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:44:14 smithi083 ceph-mon[77242]: pgmap v7679: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:14.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:44:14 smithi153 ceph-mon[70044]: pgmap v7679: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:14.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:44:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:44:14.477Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:44:15.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:44:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:44:15.184Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:44:16.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:44:16 smithi083 ceph-mon[79930]: pgmap v7680: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:16.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:44:16 smithi083 ceph-mon[77242]: pgmap v7680: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:16.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:44:16 smithi153 ceph-mon[70044]: pgmap v7680: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:18.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:44:18 smithi083 ceph-mon[77242]: pgmap v7681: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:18.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:44:18 smithi083 ceph-mon[79930]: pgmap v7681: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:44:18 smithi153 ceph-mon[70044]: pgmap v7681: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:20.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:44:20 smithi083 ceph-mon[79930]: pgmap v7682: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:20.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:44:20 smithi083 ceph-mon[77242]: pgmap v7682: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:44:20 smithi153 ceph-mon[70044]: pgmap v7682: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:21.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:44:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:44:21.091Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:44:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:44:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:44:21] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T09:44:22.395 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:44:22.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:44:22 smithi083 ceph-mon[79930]: pgmap v7683: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:22.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:44:22 smithi083 ceph-mon[77242]: pgmap v7683: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:44:22 smithi153 ceph-mon[70044]: pgmap v7683: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:22.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:44:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:44:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:44:22.683 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:44:22.683 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 4m ago 4h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:44:22.683 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 2m ago 4h 62.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:44:22.683 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 4m ago 4h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:44:22.684 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 2m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:44:22.684 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 2m ago 4h 443M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:44:22.684 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 4m ago 4h 731M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:44:22.684 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 4m ago 4h 959M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:44:22.685 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 2m ago 4h 422M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:44:22.685 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 4m ago 4h 562M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:44:22.685 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 4m ago 4h 26.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:44:22.685 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 2m ago 4h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:44:22.686 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 4m ago 4h 691M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:44:22.686 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 4m ago 4h 682M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:44:22.686 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 4m ago 4h 534M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:44:22.686 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 4m ago 4h 664M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:44:22.686 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 2m ago 4h 587M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:44:22.687 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 2m ago 4h 730M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:44:22.687 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 2m ago 4h 651M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:44:22.688 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 2m ago 4h 768M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:44:22.688 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 2m ago 4h 173M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:44:23.039 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:44:23.040 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:44:23.040 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:44:23.040 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:44:23.040 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:44:23.041 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:44:23.041 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:44:23.041 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:44:23.041 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:44:23.041 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:44:23.042 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:44:23.042 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:44:23.042 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:44:23.042 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:44:23.043 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:44:23.043 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:44:23.044 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:44:23.044 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:44:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:44:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:44:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:44:24.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:44:24 smithi153 ceph-mon[70044]: pgmap v7684: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:24.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:44:24 smithi153 ceph-mon[70044]: from='client.43071 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:44:24.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:44:24 smithi153 ceph-mon[70044]: from='client.43077 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:44:24.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:44:24 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2959118918' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:44:24.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:44:24 smithi083 ceph-mon[79930]: pgmap v7684: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:24.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:44:24 smithi083 ceph-mon[79930]: from='client.43071 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:44:24.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:44:24 smithi083 ceph-mon[79930]: from='client.43077 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:44:24.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:44:24 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2959118918' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:44:24.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:44:24 smithi083 ceph-mon[77242]: pgmap v7684: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:24.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:44:24 smithi083 ceph-mon[77242]: from='client.43071 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:44:24.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:44:24 smithi083 ceph-mon[77242]: from='client.43077 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:44:24.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:44:24 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2959118918' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:44:24.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:44:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:44:24.478Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:44:25.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:44:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:44:25.185Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:44:26.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:44:26 smithi153 ceph-mon[70044]: pgmap v7685: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:26.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:44:26 smithi083 ceph-mon[79930]: pgmap v7685: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:26.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:44:26 smithi083 ceph-mon[77242]: pgmap v7685: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:28.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:44:28 smithi153 ceph-mon[70044]: pgmap v7686: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:28.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:44:28 smithi083 ceph-mon[77242]: pgmap v7686: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:28.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:44:28 smithi083 ceph-mon[79930]: pgmap v7686: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:30.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:44:30 smithi153 ceph-mon[70044]: pgmap v7687: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:30.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:44:30 smithi083 ceph-mon[79930]: pgmap v7687: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:30.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:44:30 smithi083 ceph-mon[77242]: pgmap v7687: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:31.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:44:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:44:31.093Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:44:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:44:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:44:31] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T09:44:32.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:44:32 smithi153 ceph-mon[70044]: pgmap v7688: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:32.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:44:32 smithi083 ceph-mon[79930]: pgmap v7688: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:32.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:44:32 smithi083 ceph-mon[77242]: pgmap v7688: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:44:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:44:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:44:33.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:44:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:44:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:44:34.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:44:34 smithi153 ceph-mon[70044]: pgmap v7689: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:34.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:44:34 smithi083 ceph-mon[79930]: pgmap v7689: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:34.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:44:34 smithi083 ceph-mon[77242]: pgmap v7689: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:34.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:44:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:44:34.479Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:44:35.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:44:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:44:35.185Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:44:36.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:44:36 smithi153 ceph-mon[70044]: pgmap v7690: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:36.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:44:36 smithi083 ceph-mon[79930]: pgmap v7690: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:36.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:44:36 smithi083 ceph-mon[77242]: pgmap v7690: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:38.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:44:38 smithi083 ceph-mon[79930]: pgmap v7691: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:38.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:44:38 smithi083 ceph-mon[77242]: pgmap v7691: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:38.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:44:38 smithi153 ceph-mon[70044]: pgmap v7691: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:40.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:44:40 smithi083 ceph-mon[79930]: pgmap v7692: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:44:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:44:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:44:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:44:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:44:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:44:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:44:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:44:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:44:40 smithi083 ceph-mon[77242]: pgmap v7692: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:44:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:44:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:44:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:44:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:44:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:44:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:44:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:44:40.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:44:40 smithi153 ceph-mon[70044]: pgmap v7692: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:44:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:44:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:44:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:44:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:44:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:44:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:44:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:44:41.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:44:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:44:41.093Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:44:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:44:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:44:41] "GET /metrics HTTP/1.1" 200 225225 "" "Prometheus/2.33.4" 2022-09-29T09:44:42.315 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:44:42 smithi153 ceph-mon[70044]: pgmap v7693: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:42.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:44:42 smithi083 ceph-mon[79930]: pgmap v7693: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:42.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:44:42 smithi083 ceph-mon[77242]: pgmap v7693: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:42.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:44:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:44:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:44:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:44:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:44:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:44:44.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:44:44 smithi153 ceph-mon[70044]: pgmap v7694: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:44.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:44:44 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:44:44.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:44:44 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:44:44.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:44:44 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:44:44.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:44:44 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:44:44.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:44:44 smithi083 ceph-mon[79930]: pgmap v7694: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:44.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:44:44 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:44:44.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:44:44 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:44:44.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:44:44 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:44:44.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:44:44 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:44:44.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:44:44 smithi083 ceph-mon[77242]: pgmap v7694: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:44.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:44:44 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:44:44.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:44:44 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:44:44.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:44:44 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:44:44.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:44:44 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:44:44.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:44:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:44:44.480Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:44:45.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:44:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:44:45.185Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:44:46.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:44:46 smithi153 ceph-mon[70044]: pgmap v7695: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:46.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:44:46 smithi083 ceph-mon[79930]: pgmap v7695: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:46.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:44:46 smithi083 ceph-mon[77242]: pgmap v7695: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:48.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:44:48 smithi153 ceph-mon[70044]: pgmap v7696: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:48.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:44:48 smithi083 ceph-mon[79930]: pgmap v7696: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:48.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:44:48 smithi083 ceph-mon[77242]: pgmap v7696: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:50.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:44:50 smithi153 ceph-mon[70044]: pgmap v7697: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:50.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:44:50 smithi083 ceph-mon[79930]: pgmap v7697: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:50.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:44:50 smithi083 ceph-mon[77242]: pgmap v7697: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:51.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:44:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:44:51.094Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:44:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:44:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:44:51] "GET /metrics HTTP/1.1" 200 225225 "" "Prometheus/2.33.4" 2022-09-29T09:44:52.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:44:52 smithi153 ceph-mon[70044]: pgmap v7698: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:52.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:44:52 smithi083 ceph-mon[79930]: pgmap v7698: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:52.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:44:52 smithi083 ceph-mon[77242]: pgmap v7698: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:44:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:44:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:44:53.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:44:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:44:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:44:53.369 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:44:53.660 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:44:53.660 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 5m ago 4h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:44:53.661 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 3m ago 4h 62.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:44:53.661 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 5m ago 4h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:44:53.662 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 3m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:44:53.662 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 3m ago 4h 443M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:44:53.662 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 5m ago 4h 731M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:44:53.662 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 5m ago 4h 959M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:44:53.663 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 3m ago 4h 422M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:44:53.663 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 5m ago 4h 562M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:44:53.663 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 5m ago 4h 26.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:44:53.663 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 3m ago 4h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:44:53.663 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 5m ago 4h 691M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:44:53.664 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 5m ago 4h 682M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:44:53.664 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 5m ago 4h 534M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:44:53.664 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 5m ago 4h 664M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:44:53.664 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 3m ago 4h 587M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:44:53.664 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 3m ago 4h 730M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:44:53.665 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 3m ago 4h 651M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:44:53.665 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 3m ago 4h 768M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:44:53.665 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 3m ago 4h 173M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:44:54.015 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:44:54.016 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:44:54.016 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:44:54.016 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:44:54.016 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:44:54.017 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:44:54.017 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:44:54.017 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:44:54.017 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:44:54.018 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:44:54.018 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:44:54.018 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:44:54.018 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:44:54.018 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:44:54.019 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:44:54.019 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:44:54.019 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:44:54.019 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:44:54.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:44:54 smithi153 ceph-mon[70044]: pgmap v7699: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:54.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:44:54 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1947248743' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:44:54.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:44:54 smithi083 ceph-mon[79930]: pgmap v7699: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:54.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:44:54 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1947248743' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:44:54.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:44:54 smithi083 ceph-mon[77242]: pgmap v7699: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:54.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:44:54 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1947248743' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:44:54.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:44:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:44:54.480Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:44:55.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:44:55 smithi083 ceph-mon[79930]: from='client.43089 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:44:55.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:44:55 smithi083 ceph-mon[79930]: from='client.43095 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:44:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:44:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:44:55.186Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:44:55.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:44:55 smithi083 ceph-mon[77242]: from='client.43089 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:44:55.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:44:55 smithi083 ceph-mon[77242]: from='client.43095 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:44:55.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:44:55 smithi153 ceph-mon[70044]: from='client.43089 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:44:55.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:44:55 smithi153 ceph-mon[70044]: from='client.43095 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:44:56.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:44:56 smithi083 ceph-mon[77242]: pgmap v7700: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:56.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:44:56 smithi083 ceph-mon[79930]: pgmap v7700: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:56.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:44:56 smithi153 ceph-mon[70044]: pgmap v7700: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:58.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:44:58 smithi083 ceph-mon[79930]: pgmap v7701: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:58.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:44:58 smithi083 ceph-mon[77242]: pgmap v7701: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:44:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:44:58 smithi153 ceph-mon[70044]: pgmap v7701: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:00.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:45:00 smithi083 ceph-mon[77242]: pgmap v7702: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:00.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:45:00 smithi083 ceph-mon[79930]: pgmap v7702: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:00.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:45:00 smithi153 ceph-mon[70044]: pgmap v7702: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:01.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:45:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:45:01.095Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:45:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:45:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:45:01] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T09:45:02.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:45:02 smithi083 ceph-mon[79930]: pgmap v7703: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:02.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:45:02 smithi083 ceph-mon[77242]: pgmap v7703: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:45:02 smithi153 ceph-mon[70044]: pgmap v7703: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:02.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:45:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:45:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:45:03.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:45:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:45:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:45:04.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:45:04 smithi083 ceph-mon[79930]: pgmap v7704: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:04.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:45:04 smithi083 ceph-mon[77242]: pgmap v7704: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:04.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:45:04 smithi153 ceph-mon[70044]: pgmap v7704: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:04.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:45:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:45:04.482Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:45:05.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:45:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:45:05.187Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:45:06.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:45:06 smithi083 ceph-mon[79930]: pgmap v7705: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:06.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:45:06 smithi083 ceph-mon[77242]: pgmap v7705: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:06.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:45:06 smithi153 ceph-mon[70044]: pgmap v7705: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:08.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:45:08 smithi083 ceph-mon[79930]: pgmap v7706: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:08.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:45:08 smithi083 ceph-mon[77242]: pgmap v7706: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:08.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:45:08 smithi153 ceph-mon[70044]: pgmap v7706: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:10.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:45:10 smithi083 ceph-mon[79930]: pgmap v7707: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:10.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:45:10 smithi083 ceph-mon[77242]: pgmap v7707: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:10.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:45:10 smithi153 ceph-mon[70044]: pgmap v7707: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:11.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:45:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:45:11.096Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:45:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:45:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:45:11] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T09:45:12.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:45:12 smithi083 ceph-mon[77242]: pgmap v7708: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:12.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:45:12 smithi083 ceph-mon[79930]: pgmap v7708: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:45:12 smithi153 ceph-mon[70044]: pgmap v7708: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:12.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:45:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:45:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:45:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:45:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:45:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:45:14.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:45:14 smithi083 ceph-mon[79930]: pgmap v7709: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:14.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:45:14 smithi083 ceph-mon[77242]: pgmap v7709: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:14.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:45:14 smithi153 ceph-mon[70044]: pgmap v7709: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:14.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:45:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:45:14.483Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:45:15.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:45:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:45:15.188Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:45:16.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:45:16 smithi083 ceph-mon[79930]: pgmap v7710: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:16.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:45:16 smithi083 ceph-mon[77242]: pgmap v7710: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:45:16 smithi153 ceph-mon[70044]: pgmap v7710: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:18.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:45:18 smithi083 ceph-mon[79930]: pgmap v7711: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:18.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:45:18 smithi083 ceph-mon[77242]: pgmap v7711: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:45:18 smithi153 ceph-mon[70044]: pgmap v7711: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:20.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:45:20 smithi083 ceph-mon[79930]: pgmap v7712: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:20.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:45:20 smithi083 ceph-mon[77242]: pgmap v7712: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:20.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:45:20 smithi153 ceph-mon[70044]: pgmap v7712: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:21.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:45:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:45:21.097Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:45:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:45:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:45:21] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T09:45:22.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:45:22 smithi083 ceph-mon[77242]: pgmap v7713: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:22.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:45:22 smithi083 ceph-mon[79930]: pgmap v7713: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:45:22 smithi153 ceph-mon[70044]: pgmap v7713: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:22.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:45:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:45:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:45:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:45:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:45:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:45:24.346 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:45:24.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:45:24 smithi083 ceph-mon[79930]: pgmap v7714: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:24.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:45:24 smithi083 ceph-mon[77242]: pgmap v7714: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:24.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:45:24 smithi153 ceph-mon[70044]: pgmap v7714: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:24.635 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:45:24.636 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 5m ago 4h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:45:24.636 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 3m ago 4h 62.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:45:24.636 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 5m ago 4h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:45:24.636 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 3m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:45:24.637 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 3m ago 4h 443M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:45:24.637 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 5m ago 4h 731M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:45:24.637 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 5m ago 4h 959M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:45:24.637 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 3m ago 4h 422M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:45:24.638 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 5m ago 4h 562M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:45:24.638 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 5m ago 4h 26.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:45:24.638 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 3m ago 4h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:45:24.638 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 5m ago 4h 691M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:45:24.638 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 5m ago 4h 682M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:45:24.639 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 5m ago 4h 534M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:45:24.639 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 5m ago 4h 664M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:45:24.639 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 3m ago 4h 587M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:45:24.639 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 3m ago 4h 730M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:45:24.639 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 3m ago 4h 651M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:45:24.640 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 3m ago 4h 768M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:45:24.640 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 3m ago 4h 173M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:45:24.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:45:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:45:24.483Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:45:24.993 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:45:24.993 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:45:24.993 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:45:24.993 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:45:24.994 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:45:24.994 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:45:24.994 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:45:24.994 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:45:24.995 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:45:24.995 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:45:24.995 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:45:24.995 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:45:24.996 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:45:24.996 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:45:24.996 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:45:24.996 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:45:24.997 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:45:24.997 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:45:25.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:45:25 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/895121671' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:45:25.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:45:25 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/895121671' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:45:25.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:45:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:45:25.188Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:45:25.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:45:25 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/895121671' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:45:26.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:45:26 smithi083 ceph-mon[77242]: pgmap v7715: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:26.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:45:26 smithi083 ceph-mon[77242]: from='client.43107 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:45:26.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:45:26 smithi083 ceph-mon[77242]: from='client.53044 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:45:26.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:45:26 smithi083 ceph-mon[79930]: pgmap v7715: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:26.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:45:26 smithi083 ceph-mon[79930]: from='client.43107 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:45:26.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:45:26 smithi083 ceph-mon[79930]: from='client.53044 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:45:26.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:45:26 smithi153 ceph-mon[70044]: pgmap v7715: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:45:26 smithi153 ceph-mon[70044]: from='client.43107 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:45:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:45:26 smithi153 ceph-mon[70044]: from='client.53044 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:45:28.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:45:28 smithi083 ceph-mon[79930]: pgmap v7716: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:28.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:45:28 smithi083 ceph-mon[77242]: pgmap v7716: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:45:28 smithi153 ceph-mon[70044]: pgmap v7716: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:30.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:45:30 smithi083 ceph-mon[79930]: pgmap v7717: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:30.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:45:30 smithi083 ceph-mon[77242]: pgmap v7717: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:30.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:45:30 smithi153 ceph-mon[70044]: pgmap v7717: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:31.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:45:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:45:31.097Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:45:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:45:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:45:31] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T09:45:32.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:45:32 smithi083 ceph-mon[79930]: pgmap v7718: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:32.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:45:32 smithi083 ceph-mon[77242]: pgmap v7718: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:32.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:45:32 smithi153 ceph-mon[70044]: pgmap v7718: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:32.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:45:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:45:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:45:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:45:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:45:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:45:34.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:45:34 smithi083 ceph-mon[79930]: pgmap v7719: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:34.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:45:34 smithi083 ceph-mon[77242]: pgmap v7719: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:34.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:45:34 smithi153 ceph-mon[70044]: pgmap v7719: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:34.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:45:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:45:34.485Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:45:35.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:45:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:45:35.190Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:45:36.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:45:36 smithi083 ceph-mon[79930]: pgmap v7720: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:36.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:45:36 smithi083 ceph-mon[77242]: pgmap v7720: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:36.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:45:36 smithi153 ceph-mon[70044]: pgmap v7720: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:38.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:45:38 smithi083 ceph-mon[77242]: pgmap v7721: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:38.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:45:38 smithi083 ceph-mon[79930]: pgmap v7721: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:38.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:45:38 smithi153 ceph-mon[70044]: pgmap v7721: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:40.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:45:40 smithi083 ceph-mon[79930]: pgmap v7722: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:45:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:45:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:45:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:45:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:45:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:45:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:45:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:45:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:45:40 smithi083 ceph-mon[77242]: pgmap v7722: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:45:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:45:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:45:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:45:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:45:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:45:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:45:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:45:40.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:45:40 smithi153 ceph-mon[70044]: pgmap v7722: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:45:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:45:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:45:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:45:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:45:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:45:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:45:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:45:41.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:45:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:45:41.098Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:45:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:45:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:45:41] "GET /metrics HTTP/1.1" 200 225229 "" "Prometheus/2.33.4" 2022-09-29T09:45:42.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:45:42 smithi083 ceph-mon[77242]: pgmap v7723: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:45:42 smithi083 ceph-mon[79930]: pgmap v7723: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:42.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:45:42 smithi153 ceph-mon[70044]: pgmap v7723: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:42.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:45:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:45:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:45:43.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:45:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:45:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:45:44.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:45:44 smithi083 ceph-mon[79930]: pgmap v7724: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:44.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:45:44 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:45:44.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:45:44 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:45:44.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:45:44 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:45:44.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:45:44 smithi083 ceph-mon[77242]: pgmap v7724: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:44.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:45:44 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:45:44.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:45:44 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:45:44.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:45:44 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:45:44.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:45:44 smithi153 ceph-mon[70044]: pgmap v7724: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:44.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:45:44 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:45:44.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:45:44 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:45:44.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:45:44 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:45:44.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:45:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:45:44.484Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:45:45.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:45:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:45:45.190Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:45:45.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:45:45 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:45:45.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:45:45 smithi083 ceph-mon[77242]: pgmap v7725: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:45.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:45:45 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:45:45.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:45:45 smithi083 ceph-mon[79930]: pgmap v7725: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:45.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:45:45 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:45:45.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:45:45 smithi153 ceph-mon[70044]: pgmap v7725: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:47.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:45:47 smithi083 ceph-mon[79930]: pgmap v7726: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:47.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:45:47 smithi083 ceph-mon[77242]: pgmap v7726: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:47.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:45:47 smithi153 ceph-mon[70044]: pgmap v7726: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:50.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:45:50 smithi153 ceph-mon[70044]: pgmap v7727: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:50.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:45:50 smithi083 ceph-mon[77242]: pgmap v7727: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:50.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:45:50 smithi083 ceph-mon[79930]: pgmap v7727: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:51.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:45:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:45:51.099Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:45:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:45:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:45:51] "GET /metrics HTTP/1.1" 200 225229 "" "Prometheus/2.33.4" 2022-09-29T09:45:52.315 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:45:52 smithi153 ceph-mon[70044]: pgmap v7728: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:52.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:45:52 smithi083 ceph-mon[79930]: pgmap v7728: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:52.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:45:52 smithi083 ceph-mon[77242]: pgmap v7728: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:52.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:45:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:45:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:45:53.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:45:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:45:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:45:54.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:45:54 smithi083 ceph-mon[79930]: pgmap v7729: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:54.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:45:54 smithi083 ceph-mon[77242]: pgmap v7729: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:54.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:45:54 smithi153 ceph-mon[70044]: pgmap v7729: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:54.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:45:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:45:54.486Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:45:55.325 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:45:55.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:45:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:45:55.191Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:45:55.614 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:45:55.614 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 6m ago 4h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:45:55.614 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 4m ago 4h 62.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:45:55.615 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 6m ago 4h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:45:55.615 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 4m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:45:55.615 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 4m ago 4h 443M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:45:55.615 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 6m ago 4h 731M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:45:55.616 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 6m ago 4h 959M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:45:55.616 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 4m ago 4h 422M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:45:55.616 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 6m ago 4h 562M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:45:55.616 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 6m ago 4h 26.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:45:55.616 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 4m ago 4h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:45:55.617 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 6m ago 4h 691M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:45:55.617 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 6m ago 4h 682M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:45:55.617 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 6m ago 4h 534M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:45:55.617 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 6m ago 4h 664M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:45:55.617 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 4m ago 4h 587M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:45:55.618 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 4m ago 4h 730M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:45:55.618 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 4m ago 4h 651M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:45:55.618 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 4m ago 4h 768M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:45:55.618 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 4m ago 4h 173M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:45:55.967 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:45:55.967 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:45:55.968 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:45:55.968 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:45:55.968 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:45:55.968 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:45:55.969 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:45:55.969 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:45:55.969 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:45:55.969 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:45:55.970 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:45:55.970 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:45:55.970 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:45:55.970 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:45:55.971 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:45:55.971 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:45:55.971 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:45:55.971 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:45:56.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:45:56 smithi083 ceph-mon[79930]: pgmap v7730: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:56.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:45:56 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/964076089' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:45:56.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:45:56 smithi083 ceph-mon[77242]: pgmap v7730: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:56.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:45:56 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/964076089' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:45:56.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:45:56 smithi153 ceph-mon[70044]: pgmap v7730: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:56.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:45:56 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/964076089' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:45:57.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:45:57 smithi083 ceph-mon[79930]: from='client.53056 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:45:57.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:45:57 smithi083 ceph-mon[79930]: from='client.53062 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:45:57.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:45:57 smithi083 ceph-mon[77242]: from='client.53056 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:45:57.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:45:57 smithi083 ceph-mon[77242]: from='client.53062 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:45:57.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:45:57 smithi153 ceph-mon[70044]: from='client.53056 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:45:57.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:45:57 smithi153 ceph-mon[70044]: from='client.53062 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:45:58.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:45:58 smithi083 ceph-mon[79930]: pgmap v7731: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:58.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:45:58 smithi083 ceph-mon[77242]: pgmap v7731: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:45:58.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:45:58 smithi153 ceph-mon[70044]: pgmap v7731: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:00.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:46:00 smithi083 ceph-mon[79930]: pgmap v7732: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:00.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:46:00 smithi083 ceph-mon[77242]: pgmap v7732: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:46:00 smithi153 ceph-mon[70044]: pgmap v7732: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:01.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:46:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:46:01.099Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:46:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:46:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:46:01] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T09:46:02.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:46:02 smithi083 ceph-mon[79930]: pgmap v7733: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:02.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:46:02 smithi083 ceph-mon[77242]: pgmap v7733: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:46:02 smithi153 ceph-mon[70044]: pgmap v7733: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:02.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:46:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:46:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:46:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:46:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:46:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:46:04.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:46:04 smithi083 ceph-mon[79930]: pgmap v7734: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:04.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:46:04 smithi083 ceph-mon[77242]: pgmap v7734: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:04.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:46:04 smithi153 ceph-mon[70044]: pgmap v7734: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:04.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:46:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:46:04.487Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:46:05.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:46:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:46:05.193Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:46:06.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:46:06 smithi083 ceph-mon[79930]: pgmap v7735: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:06.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:46:06 smithi083 ceph-mon[77242]: pgmap v7735: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:06.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:46:06 smithi153 ceph-mon[70044]: pgmap v7735: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:08.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:46:08 smithi083 ceph-mon[79930]: pgmap v7736: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:08.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:46:08 smithi083 ceph-mon[77242]: pgmap v7736: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:08.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:46:08 smithi153 ceph-mon[70044]: pgmap v7736: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:10.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:46:10 smithi083 ceph-mon[79930]: pgmap v7737: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:10.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:46:10 smithi083 ceph-mon[77242]: pgmap v7737: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:10.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:46:10 smithi153 ceph-mon[70044]: pgmap v7737: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:11.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:46:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:46:11.099Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:46:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:46:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:46:11] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T09:46:12.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:46:12 smithi083 ceph-mon[79930]: pgmap v7738: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:12.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:46:12 smithi083 ceph-mon[77242]: pgmap v7738: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:46:12 smithi153 ceph-mon[70044]: pgmap v7738: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:12.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:46:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:46:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:46:13.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:46:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:46:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:46:14.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:46:14 smithi083 ceph-mon[79930]: pgmap v7739: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:14.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:46:14 smithi083 ceph-mon[77242]: pgmap v7739: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:14.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:46:14 smithi153 ceph-mon[70044]: pgmap v7739: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:14.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:46:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:46:14.487Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:46:15.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:46:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:46:15.194Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:46:16.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:46:16 smithi083 ceph-mon[79930]: pgmap v7740: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:16.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:46:16 smithi083 ceph-mon[77242]: pgmap v7740: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:46:16 smithi153 ceph-mon[70044]: pgmap v7740: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:18.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:46:18 smithi083 ceph-mon[77242]: pgmap v7741: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:18.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:46:18 smithi083 ceph-mon[79930]: pgmap v7741: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:46:18 smithi153 ceph-mon[70044]: pgmap v7741: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:20.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:46:20 smithi083 ceph-mon[79930]: pgmap v7742: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:20.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:46:20 smithi083 ceph-mon[77242]: pgmap v7742: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:46:20 smithi153 ceph-mon[70044]: pgmap v7742: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:21.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:46:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:46:21.100Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:46:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:46:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:46:21] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T09:46:22.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:46:22 smithi083 ceph-mon[79930]: pgmap v7743: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:22.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:46:22 smithi083 ceph-mon[77242]: pgmap v7743: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:46:22 smithi153 ceph-mon[70044]: pgmap v7743: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:22.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:46:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:46:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:46:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:46:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:46:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:46:24.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:46:24 smithi083 ceph-mon[79930]: pgmap v7744: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:24.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:46:24 smithi083 ceph-mon[77242]: pgmap v7744: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:46:24 smithi153 ceph-mon[70044]: pgmap v7744: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:24.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:46:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:46:24.487Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:46:25.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:46:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:46:25.194Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:46:26.297 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:46:26.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:46:26 smithi083 ceph-mon[79930]: pgmap v7745: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:26.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:46:26 smithi083 ceph-mon[77242]: pgmap v7745: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:26.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:46:26 smithi153 ceph-mon[70044]: pgmap v7745: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:26.585 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:46:26.585 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 6m ago 4h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:46:26.586 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 4m ago 4h 62.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:46:26.586 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 6m ago 4h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:46:26.586 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 4m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:46:26.586 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 4m ago 4h 443M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:46:26.587 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 6m ago 4h 731M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:46:26.587 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 6m ago 4h 959M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:46:26.587 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 4m ago 4h 422M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:46:26.587 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 6m ago 4h 562M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:46:26.588 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 6m ago 4h 26.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:46:26.588 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 4m ago 4h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:46:26.588 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 6m ago 4h 691M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:46:26.588 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 6m ago 4h 682M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:46:26.588 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 6m ago 4h 534M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:46:26.589 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 6m ago 4h 664M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:46:26.589 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 4m ago 4h 587M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:46:26.589 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 4m ago 4h 730M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:46:26.589 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 4m ago 4h 651M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:46:26.590 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 4m ago 4h 768M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:46:26.590 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 4m ago 4h 173M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:46:26.938 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:46:26.939 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:46:26.939 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:46:26.939 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:46:26.939 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:46:26.940 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:46:26.940 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:46:26.940 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:46:26.940 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:46:26.941 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:46:26.941 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:46:26.941 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:46:26.941 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:46:26.941 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:46:26.942 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:46:26.942 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:46:26.942 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:46:26.942 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:46:27.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:46:27 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1895249185' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:46:27.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:46:27 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1895249185' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:46:27.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:46:27 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1895249185' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:46:28.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:46:28 smithi083 ceph-mon[79930]: from='client.43143 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:46:28.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:46:28 smithi083 ceph-mon[79930]: pgmap v7746: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:28.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:46:28 smithi083 ceph-mon[79930]: from='client.53080 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:46:28.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:46:28 smithi083 ceph-mon[77242]: from='client.43143 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:46:28.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:46:28 smithi083 ceph-mon[77242]: pgmap v7746: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:28.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:46:28 smithi083 ceph-mon[77242]: from='client.53080 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:46:28.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:46:28 smithi153 ceph-mon[70044]: from='client.43143 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:46:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:46:28 smithi153 ceph-mon[70044]: pgmap v7746: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:46:28 smithi153 ceph-mon[70044]: from='client.53080 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:46:30.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:46:30 smithi083 ceph-mon[79930]: pgmap v7747: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:30.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:46:30 smithi083 ceph-mon[77242]: pgmap v7747: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:30.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:46:30 smithi153 ceph-mon[70044]: pgmap v7747: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:31.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:46:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:46:31.101Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:46:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:46:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:46:31] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T09:46:32.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:46:32 smithi083 ceph-mon[77242]: pgmap v7748: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:32.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:46:32 smithi083 ceph-mon[79930]: pgmap v7748: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:46:32 smithi153 ceph-mon[70044]: pgmap v7748: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:32.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:46:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:46:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:46:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:46:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:46:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:46:34.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:46:34 smithi083 ceph-mon[79930]: pgmap v7749: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:34.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:46:34 smithi083 ceph-mon[77242]: pgmap v7749: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:46:34 smithi153 ceph-mon[70044]: pgmap v7749: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:34.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:46:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:46:34.488Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:46:35.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:46:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:46:35.195Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:46:36.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:46:36 smithi083 ceph-mon[79930]: pgmap v7750: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:36.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:46:36 smithi083 ceph-mon[77242]: pgmap v7750: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:46:36 smithi153 ceph-mon[70044]: pgmap v7750: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:38.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:46:38 smithi083 ceph-mon[79930]: pgmap v7751: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:38.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:46:38 smithi083 ceph-mon[77242]: pgmap v7751: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:38.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:46:38 smithi153 ceph-mon[70044]: pgmap v7751: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:40.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:46:40 smithi083 ceph-mon[77242]: pgmap v7752: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:40.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:46:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:46:40.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:46:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:46:40.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:46:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:46:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:46:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:46:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:46:40 smithi083 ceph-mon[79930]: pgmap v7752: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:46:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:46:40.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:46:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:46:40.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:46:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:46:40.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:46:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:46:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:46:40 smithi153 ceph-mon[70044]: pgmap v7752: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:46:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:46:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:46:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:46:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:46:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:46:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:46:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:46:41.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:46:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:46:41.102Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:46:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:46:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:46:41] "GET /metrics HTTP/1.1" 200 225228 "" "Prometheus/2.33.4" 2022-09-29T09:46:42.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:46:42 smithi083 ceph-mon[79930]: pgmap v7753: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:42.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:46:42 smithi083 ceph-mon[77242]: pgmap v7753: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:42.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:46:42 smithi153 ceph-mon[70044]: pgmap v7753: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:42.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:46:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:46:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:46:43.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:46:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:46:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:46:44.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:46:44 smithi083 ceph-mon[79930]: pgmap v7754: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:44.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:46:44 smithi083 ceph-mon[77242]: pgmap v7754: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:44.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:46:44 smithi153 ceph-mon[70044]: pgmap v7754: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:44.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:46:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:46:44.487Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:46:45.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:46:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:46:45.194Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:46:45.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:46:45 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:46:45.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:46:45 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:46:45.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:46:45 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:46:45.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:46:45 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:46:45.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:46:45 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:46:45.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:46:45 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:46:45.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:46:45 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:46:45.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:46:45 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:46:45.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:46:45 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:46:45.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:46:45 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:46:45.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:46:45 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:46:45.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:46:45 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:46:46.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:46:46 smithi083 ceph-mon[77242]: pgmap v7755: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:46.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:46:46 smithi083 ceph-mon[79930]: pgmap v7755: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:46.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:46:46 smithi153 ceph-mon[70044]: pgmap v7755: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:48.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:46:48 smithi083 ceph-mon[77242]: pgmap v7756: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:48.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:46:48 smithi083 ceph-mon[79930]: pgmap v7756: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:48.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:46:48 smithi153 ceph-mon[70044]: pgmap v7756: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:50.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:46:50 smithi083 ceph-mon[79930]: pgmap v7757: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:50.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:46:50 smithi083 ceph-mon[77242]: pgmap v7757: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:46:50 smithi153 ceph-mon[70044]: pgmap v7757: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:51.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:46:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:46:51.103Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:46:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:46:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:46:51] "GET /metrics HTTP/1.1" 200 225228 "" "Prometheus/2.33.4" 2022-09-29T09:46:52.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:46:52 smithi083 ceph-mon[79930]: pgmap v7758: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:52.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:46:52 smithi083 ceph-mon[77242]: pgmap v7758: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:46:52 smithi153 ceph-mon[70044]: pgmap v7758: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:52.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:46:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:46:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:46:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:46:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:46:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:46:54.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:46:54 smithi083 ceph-mon[79930]: pgmap v7759: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:54.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:46:54 smithi083 ceph-mon[77242]: pgmap v7759: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:54.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:46:54 smithi153 ceph-mon[70044]: pgmap v7759: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:54.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:46:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:46:54.489Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:46:55.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:46:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:46:55.195Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:46:56.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:46:56 smithi083 ceph-mon[79930]: pgmap v7760: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:56.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:46:56 smithi083 ceph-mon[77242]: pgmap v7760: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:56.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:46:56 smithi153 ceph-mon[70044]: pgmap v7760: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:57.267 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:46:57.556 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:46:57.557 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 7m ago 4h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:46:57.557 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 5m ago 4h 62.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:46:57.557 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 7m ago 4h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:46:57.558 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 5m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:46:57.558 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 5m ago 4h 443M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:46:57.558 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 7m ago 4h 731M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:46:57.558 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 7m ago 4h 959M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:46:57.558 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 5m ago 4h 422M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:46:57.559 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 7m ago 4h 562M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:46:57.559 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 7m ago 4h 26.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:46:57.559 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 5m ago 4h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:46:57.559 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 7m ago 4h 691M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:46:57.559 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 7m ago 4h 682M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:46:57.560 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 7m ago 4h 534M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:46:57.560 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 7m ago 4h 664M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:46:57.560 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 5m ago 4h 587M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:46:57.560 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 5m ago 4h 730M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:46:57.560 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 5m ago 4h 651M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:46:57.561 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 5m ago 4h 768M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:46:57.561 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 5m ago 4h 173M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:46:57.910 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:46:57.911 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:46:57.911 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:46:57.911 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:46:57.911 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:46:57.912 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:46:57.912 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:46:57.912 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:46:57.912 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:46:57.912 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:46:57.913 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:46:57.913 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:46:57.913 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:46:57.913 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:46:57.913 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:46:57.914 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:46:57.914 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:46:57.914 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:46:58.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:46:58 smithi083 ceph-mon[79930]: pgmap v7761: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:58.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:46:58 smithi083 ceph-mon[79930]: from='client.43161 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:46:58.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:46:58 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3056271549' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:46:58.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:46:58 smithi083 ceph-mon[77242]: pgmap v7761: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:58.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:46:58 smithi083 ceph-mon[77242]: from='client.43161 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:46:58.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:46:58 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3056271549' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:46:58.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:46:58 smithi153 ceph-mon[70044]: pgmap v7761: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:46:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:46:58 smithi153 ceph-mon[70044]: from='client.43161 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:46:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:46:58 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3056271549' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:46:59.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:46:59 smithi083 ceph-mon[77242]: from='client.53098 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:46:59.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:46:59 smithi083 ceph-mon[79930]: from='client.53098 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:46:59.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:46:59 smithi153 ceph-mon[70044]: from='client.53098 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:47:00.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:47:00 smithi083 ceph-mon[79930]: pgmap v7762: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:00.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:47:00 smithi083 ceph-mon[77242]: pgmap v7762: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:47:00 smithi153 ceph-mon[70044]: pgmap v7762: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:01.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:47:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:47:01.103Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:47:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:47:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:47:01] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T09:47:02.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:47:02 smithi083 ceph-mon[79930]: pgmap v7763: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:02.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:47:02 smithi083 ceph-mon[77242]: pgmap v7763: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:47:02 smithi153 ceph-mon[70044]: pgmap v7763: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:02.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:47:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:47:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:47:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:47:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:47:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:47:04.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:47:04 smithi083 ceph-mon[79930]: pgmap v7764: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:04.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:47:04 smithi083 ceph-mon[77242]: pgmap v7764: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:04.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:47:04 smithi153 ceph-mon[70044]: pgmap v7764: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:04.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:47:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:47:04.490Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:47:05.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:47:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:47:05.196Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:47:06.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:47:06 smithi083 ceph-mon[77242]: pgmap v7765: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:06.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:47:06 smithi083 ceph-mon[79930]: pgmap v7765: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:06.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:47:06 smithi153 ceph-mon[70044]: pgmap v7765: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:08.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:47:08 smithi083 ceph-mon[79930]: pgmap v7766: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:08.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:47:08 smithi083 ceph-mon[77242]: pgmap v7766: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:47:08 smithi153 ceph-mon[70044]: pgmap v7766: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:10.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:47:10 smithi083 ceph-mon[79930]: pgmap v7767: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:10.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:47:10 smithi083 ceph-mon[77242]: pgmap v7767: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:10.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:47:10 smithi153 ceph-mon[70044]: pgmap v7767: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:11.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:47:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:47:11.104Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:47:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:47:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:47:11] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T09:47:12.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:47:12 smithi083 ceph-mon[79930]: pgmap v7768: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:12.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:47:12 smithi083 ceph-mon[77242]: pgmap v7768: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:47:12 smithi153 ceph-mon[70044]: pgmap v7768: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:12.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:47:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:47:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:47:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:47:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:47:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:47:14.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:47:14 smithi083 ceph-mon[79930]: pgmap v7769: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:14.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:47:14 smithi083 ceph-mon[77242]: pgmap v7769: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:14.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:47:14 smithi153 ceph-mon[70044]: pgmap v7769: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:14.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:47:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:47:14.490Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:47:15.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:47:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:47:15.196Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:47:16.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:47:16 smithi083 ceph-mon[79930]: pgmap v7770: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:16.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:47:16 smithi083 ceph-mon[77242]: pgmap v7770: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:16.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:47:16 smithi153 ceph-mon[70044]: pgmap v7770: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:18.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:47:18 smithi083 ceph-mon[77242]: pgmap v7771: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:18.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:47:18 smithi083 ceph-mon[79930]: pgmap v7771: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:47:18 smithi153 ceph-mon[70044]: pgmap v7771: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:20.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:47:20 smithi083 ceph-mon[79930]: pgmap v7772: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:20.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:47:20 smithi083 ceph-mon[77242]: pgmap v7772: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:47:20 smithi153 ceph-mon[70044]: pgmap v7772: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:21.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:47:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:47:21.105Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:47:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:47:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:47:21] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T09:47:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:47:22 smithi153 ceph-mon[70044]: pgmap v7773: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:22.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:47:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:47:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:47:22.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:47:22 smithi083 ceph-mon[79930]: pgmap v7773: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:22.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:47:22 smithi083 ceph-mon[77242]: pgmap v7773: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:47:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:47:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:47:24.492 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:47:24 smithi083 ceph-mon[79930]: pgmap v7774: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:24.493 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:47:24 smithi083 ceph-mon[77242]: pgmap v7774: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:24.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:47:24 smithi153 ceph-mon[70044]: pgmap v7774: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:24.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:47:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:47:24.491Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:47:25.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:47:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:47:25.197Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:47:26.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:47:26 smithi153 ceph-mon[70044]: pgmap v7775: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:26.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:47:26 smithi083 ceph-mon[79930]: pgmap v7775: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:26.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:47:26 smithi083 ceph-mon[77242]: pgmap v7775: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:28.239 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:47:28.530 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:47:28.531 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 7m ago 4h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:47:28.531 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 5m ago 4h 62.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:47:28.531 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 7m ago 4h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:47:28.531 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 5m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:47:28.532 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 5m ago 4h 443M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:47:28.532 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 7m ago 4h 731M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:47:28.532 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 7m ago 4h 959M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:47:28.532 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 5m ago 4h 422M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:47:28.533 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 7m ago 4h 562M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:47:28.533 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 7m ago 4h 26.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:47:28.533 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 5m ago 4h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:47:28.533 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 7m ago 4h 691M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:47:28.533 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 7m ago 4h 682M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:47:28.534 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 7m ago 4h 534M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:47:28.534 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 7m ago 4h 664M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:47:28.534 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 5m ago 4h 587M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:47:28.534 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 5m ago 4h 730M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:47:28.535 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 5m ago 4h 651M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:47:28.535 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 5m ago 4h 768M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:47:28.535 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 5m ago 4h 173M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:47:28.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:47:28 smithi153 ceph-mon[70044]: pgmap v7776: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:28.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:47:28 smithi083 ceph-mon[79930]: pgmap v7776: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:28.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:47:28 smithi083 ceph-mon[77242]: pgmap v7776: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:28.883 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:47:28.884 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:47:28.884 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:47:28.884 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:47:28.885 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:47:28.885 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:47:28.885 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:47:28.885 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:47:28.885 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:47:28.886 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:47:28.886 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:47:28.886 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:47:28.886 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:47:28.887 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:47:28.887 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:47:28.887 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:47:28.887 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:47:28.888 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:47:29.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:47:29 smithi153 ceph-mon[70044]: from='client.53110 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:47:29.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:47:29 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/299017424' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:47:29.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:47:29 smithi083 ceph-mon[79930]: from='client.53110 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:47:29.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:47:29 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/299017424' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:47:29.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:47:29 smithi083 ceph-mon[77242]: from='client.53110 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:47:29.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:47:29 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/299017424' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:47:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:47:30 smithi153 ceph-mon[70044]: pgmap v7777: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:47:30 smithi153 ceph-mon[70044]: from='client.43185 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:47:30.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:47:30 smithi083 ceph-mon[79930]: pgmap v7777: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:30.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:47:30 smithi083 ceph-mon[79930]: from='client.43185 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:47:30.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:47:30 smithi083 ceph-mon[77242]: pgmap v7777: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:30.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:47:30 smithi083 ceph-mon[77242]: from='client.43185 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:47:31.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:47:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:47:31.106Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:47:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:47:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:47:31] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T09:47:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:47:32 smithi153 ceph-mon[70044]: pgmap v7778: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:32.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:47:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:47:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:47:32.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:47:32 smithi083 ceph-mon[79930]: pgmap v7778: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:32.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:47:32 smithi083 ceph-mon[77242]: pgmap v7778: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:33.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:47:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:47:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:47:34.493 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:47:34 smithi083 ceph-mon[79930]: pgmap v7779: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:34.493 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:47:34 smithi083 ceph-mon[77242]: pgmap v7779: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:47:34 smithi153 ceph-mon[70044]: pgmap v7779: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:34.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:47:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:47:34.492Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:47:35.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:47:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:47:35.199Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:47:36.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:47:36 smithi153 ceph-mon[70044]: pgmap v7780: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:36.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:47:36 smithi083 ceph-mon[79930]: pgmap v7780: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:36.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:47:36 smithi083 ceph-mon[77242]: pgmap v7780: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:47:38 smithi153 ceph-mon[70044]: pgmap v7781: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:38.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:47:38 smithi083 ceph-mon[79930]: pgmap v7781: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:38.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:47:38 smithi083 ceph-mon[77242]: pgmap v7781: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:40.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:47:40 smithi153 ceph-mon[70044]: pgmap v7782: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:47:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:47:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:47:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:47:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:47:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:47:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:47:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:47:40.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:47:40 smithi083 ceph-mon[77242]: pgmap v7782: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:40.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:47:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:47:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:47:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:47:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:47:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:47:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:47:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:47:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:47:40 smithi083 ceph-mon[79930]: pgmap v7782: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:47:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:47:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:47:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:47:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:47:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:47:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:47:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:47:41.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:47:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:47:41.105Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:47:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:47:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:47:41] "GET /metrics HTTP/1.1" 200 225226 "" "Prometheus/2.33.4" 2022-09-29T09:47:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:47:42 smithi153 ceph-mon[70044]: pgmap v7783: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:42.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:47:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:47:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:47:42.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:47:42 smithi083 ceph-mon[77242]: pgmap v7783: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:47:42 smithi083 ceph-mon[79930]: pgmap v7783: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:47:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:47:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:47:44.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:47:44 smithi153 ceph-mon[70044]: pgmap v7784: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:44.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:47:44 smithi083 ceph-mon[79930]: pgmap v7784: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:47:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:47:44.493Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:47:44.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:47:44 smithi083 ceph-mon[77242]: pgmap v7784: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:45.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:47:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:47:45.199Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:47:45.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:47:45 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:47:45.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:47:45 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:47:45.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:47:45 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:47:45.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:47:45 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:47:45.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:47:45 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:47:45.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:47:45 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:47:45.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:47:45 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:47:45.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:47:45 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:47:45.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:47:45 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:47:46.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:47:46 smithi153 ceph-mon[70044]: pgmap v7785: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:46.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:47:46 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:47:46.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:47:46 smithi083 ceph-mon[79930]: pgmap v7785: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:46.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:47:46 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:47:46.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:47:46 smithi083 ceph-mon[77242]: pgmap v7785: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:46.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:47:46 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:47:48.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:47:48 smithi153 ceph-mon[70044]: pgmap v7786: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:48.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:47:48 smithi083 ceph-mon[79930]: pgmap v7786: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:48.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:47:48 smithi083 ceph-mon[77242]: pgmap v7786: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:47:50 smithi153 ceph-mon[70044]: pgmap v7787: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:50.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:47:50 smithi083 ceph-mon[79930]: pgmap v7787: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:50.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:47:50 smithi083 ceph-mon[77242]: pgmap v7787: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:51.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:47:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:47:51.106Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:47:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:47:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:47:51] "GET /metrics HTTP/1.1" 200 225226 "" "Prometheus/2.33.4" 2022-09-29T09:47:52.562 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:47:52 smithi153 ceph-mon[70044]: pgmap v7788: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:52.563 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:47:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:47:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:47:52.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:47:52 smithi083 ceph-mon[77242]: pgmap v7788: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:52.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:47:52 smithi083 ceph-mon[79930]: pgmap v7788: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:53.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:47:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:47:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:47:54.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:47:54 smithi153 ceph-mon[70044]: pgmap v7789: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:54.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:47:54 smithi083 ceph-mon[79930]: pgmap v7789: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:54.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:47:54 smithi083 ceph-mon[77242]: pgmap v7789: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:47:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:47:54.492Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:47:55.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:47:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:47:55.201Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:47:56.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:47:56 smithi153 ceph-mon[70044]: pgmap v7790: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:56.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:47:56 smithi083 ceph-mon[79930]: pgmap v7790: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:56.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:47:56 smithi083 ceph-mon[77242]: pgmap v7790: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:58.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:47:58 smithi153 ceph-mon[70044]: pgmap v7791: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:58.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:47:58 smithi083 ceph-mon[79930]: pgmap v7791: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:58.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:47:58 smithi083 ceph-mon[77242]: pgmap v7791: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:47:59.211 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:47:59.502 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:47:59.502 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 8m ago 4h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:47:59.503 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 6m ago 4h 62.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:47:59.503 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 8m ago 4h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:47:59.503 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 6m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:47:59.503 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 6m ago 4h 443M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:47:59.504 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 8m ago 4h 731M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:47:59.504 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 8m ago 4h 959M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:47:59.504 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 6m ago 4h 422M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:47:59.504 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 8m ago 4h 562M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:47:59.505 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 8m ago 4h 26.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:47:59.505 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 6m ago 4h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:47:59.505 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 8m ago 4h 691M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:47:59.505 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 8m ago 4h 682M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:47:59.505 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 8m ago 4h 534M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:47:59.506 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 8m ago 4h 664M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:47:59.506 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 6m ago 4h 587M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:47:59.506 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 6m ago 4h 730M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:47:59.506 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 6m ago 4h 651M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:47:59.507 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 6m ago 4h 768M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:47:59.507 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 6m ago 4h 173M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:47:59.859 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:47:59.859 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:47:59.860 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:47:59.860 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:47:59.860 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:47:59.861 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:47:59.861 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:47:59.861 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:47:59.861 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:47:59.861 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:47:59.862 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:47:59.862 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:47:59.862 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:47:59.862 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:47:59.863 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:47:59.863 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:47:59.863 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:47:59.863 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:48:00.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:48:00 smithi153 ceph-mon[70044]: pgmap v7792: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:48:00 smithi153 ceph-mon[70044]: from='client.43197 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:48:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:48:00 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/430781360' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:48:00.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:48:00 smithi083 ceph-mon[79930]: pgmap v7792: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:00.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:48:00 smithi083 ceph-mon[79930]: from='client.43197 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:48:00.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:48:00 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/430781360' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:48:00.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:48:00 smithi083 ceph-mon[77242]: pgmap v7792: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:00.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:48:00 smithi083 ceph-mon[77242]: from='client.43197 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:48:00.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:48:00 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/430781360' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:48:01.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:48:01 smithi083 ceph-mon[79930]: from='client.53134 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:48:01.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:48:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:48:01.106Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:48:01.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:48:01 smithi083 ceph-mon[77242]: from='client.53134 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:48:01.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:48:01 smithi153 ceph-mon[70044]: from='client.53134 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:48:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:48:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:48:01] "GET /metrics HTTP/1.1" 200 225227 "" "Prometheus/2.33.4" 2022-09-29T09:48:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:48:02 smithi153 ceph-mon[70044]: pgmap v7793: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:02.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:48:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:48:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:48:02.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:48:02 smithi083 ceph-mon[79930]: pgmap v7793: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:02.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:48:02 smithi083 ceph-mon[77242]: pgmap v7793: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:48:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:48:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:48:04.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:48:04 smithi083 ceph-mon[79930]: pgmap v7794: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:04.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:48:04 smithi083 ceph-mon[77242]: pgmap v7794: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:04.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:48:04 smithi153 ceph-mon[70044]: pgmap v7794: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:04.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:48:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:48:04.494Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:48:05.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:48:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:48:05.201Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:48:06.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:48:06 smithi083 ceph-mon[77242]: pgmap v7795: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:06.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:48:06 smithi083 ceph-mon[79930]: pgmap v7795: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:06.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:48:06 smithi153 ceph-mon[70044]: pgmap v7795: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:08.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:48:08 smithi083 ceph-mon[77242]: pgmap v7796: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:08.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:48:08 smithi083 ceph-mon[79930]: pgmap v7796: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:48:08 smithi153 ceph-mon[70044]: pgmap v7796: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:10.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:48:10 smithi083 ceph-mon[77242]: pgmap v7797: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:10.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:48:10 smithi083 ceph-mon[79930]: pgmap v7797: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:10.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:48:10 smithi153 ceph-mon[70044]: pgmap v7797: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:11.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:48:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:48:11.108Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:48:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:48:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:48:11] "GET /metrics HTTP/1.1" 200 225229 "" "Prometheus/2.33.4" 2022-09-29T09:48:12.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:48:12 smithi083 ceph-mon[79930]: pgmap v7798: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:12.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:48:12 smithi083 ceph-mon[77242]: pgmap v7798: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:48:12 smithi153 ceph-mon[70044]: pgmap v7798: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:12.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:48:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:48:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:48:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:48:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:48:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:48:14.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:48:14 smithi083 ceph-mon[79930]: pgmap v7799: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:14.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:48:14 smithi083 ceph-mon[77242]: pgmap v7799: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:14.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:48:14 smithi153 ceph-mon[70044]: pgmap v7799: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:14.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:48:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:48:14.494Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:48:15.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:48:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:48:15.202Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:48:16.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:48:16 smithi083 ceph-mon[79930]: pgmap v7800: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:16.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:48:16 smithi083 ceph-mon[77242]: pgmap v7800: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:16.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:48:16 smithi153 ceph-mon[70044]: pgmap v7800: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:18.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:48:18 smithi083 ceph-mon[79930]: pgmap v7801: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:18.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:48:18 smithi083 ceph-mon[77242]: pgmap v7801: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:48:18 smithi153 ceph-mon[70044]: pgmap v7801: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:20.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:48:20 smithi083 ceph-mon[79930]: pgmap v7802: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:20.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:48:20 smithi083 ceph-mon[77242]: pgmap v7802: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:20.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:48:20 smithi153 ceph-mon[70044]: pgmap v7802: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:21.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:48:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:48:21.108Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:48:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:48:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:48:21] "GET /metrics HTTP/1.1" 200 225229 "" "Prometheus/2.33.4" 2022-09-29T09:48:22.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:48:22 smithi083 ceph-mon[79930]: pgmap v7803: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:22.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:48:22 smithi083 ceph-mon[77242]: pgmap v7803: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:22.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:48:22 smithi153 ceph-mon[70044]: pgmap v7803: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:22.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:48:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:48:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:48:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:48:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:48:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:48:24.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:48:24 smithi083 ceph-mon[79930]: pgmap v7804: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:24.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:48:24 smithi083 ceph-mon[77242]: pgmap v7804: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:24.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:48:24 smithi153 ceph-mon[70044]: pgmap v7804: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:24.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:48:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:48:24.495Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:48:25.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:48:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:48:25.203Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:48:26.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:48:26 smithi083 ceph-mon[79930]: pgmap v7805: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:26.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:48:26 smithi083 ceph-mon[77242]: pgmap v7805: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:26.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:48:26 smithi153 ceph-mon[70044]: pgmap v7805: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:28.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:48:28 smithi083 ceph-mon[79930]: pgmap v7806: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:28.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:48:28 smithi083 ceph-mon[77242]: pgmap v7806: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:28.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:48:28 smithi153 ceph-mon[70044]: pgmap v7806: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:30.187 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:48:30.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:48:30 smithi083 ceph-mon[77242]: pgmap v7807: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:30.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:48:30 smithi083 ceph-mon[79930]: pgmap v7807: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:30.481 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:48:30.481 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 9m ago 4h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:48:30.481 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 6m ago 4h 62.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:48:30.481 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 9m ago 4h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:48:30.481 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 6m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:48:30.482 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 6m ago 4h 443M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:48:30.482 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 9m ago 4h 731M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:48:30.482 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 9m ago 4h 959M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:48:30.482 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 6m ago 4h 422M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:48:30.483 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 9m ago 4h 562M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:48:30.483 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 9m ago 4h 26.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:48:30.483 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 6m ago 4h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:48:30.483 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 9m ago 4h 691M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:48:30.483 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 9m ago 4h 682M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:48:30.484 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 9m ago 4h 534M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:48:30.484 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 9m ago 4h 664M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:48:30.484 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 6m ago 4h 587M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:48:30.484 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 6m ago 4h 730M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:48:30.485 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 6m ago 4h 651M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:48:30.485 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 6m ago 4h 768M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:48:30.485 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 6m ago 4h 173M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:48:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:48:30 smithi153 ceph-mon[70044]: pgmap v7807: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:30.834 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:48:30.835 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:48:30.835 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:48:30.835 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:48:30.835 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:48:30.835 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:48:30.836 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:48:30.836 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:48:30.836 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:48:30.837 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:48:30.837 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:48:30.837 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:48:30.837 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:48:30.837 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:48:30.838 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:48:30.838 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:48:30.838 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:48:30.838 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:48:31.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:48:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:48:31.108Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:48:31.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:48:31 smithi083 ceph-mon[77242]: from='client.53146 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:48:31.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:48:31 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3835830760' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:48:31.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:48:31 smithi083 ceph-mon[79930]: from='client.53146 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:48:31.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:48:31 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3835830760' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:48:31.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:48:31 smithi153 ceph-mon[70044]: from='client.53146 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:48:31.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:48:31 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3835830760' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:48:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:48:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:48:31] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T09:48:32.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:48:32 smithi083 ceph-mon[79930]: pgmap v7808: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:32.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:48:32 smithi083 ceph-mon[79930]: from='client.53152 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:48:32.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:48:32 smithi083 ceph-mon[77242]: pgmap v7808: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:32.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:48:32 smithi083 ceph-mon[77242]: from='client.53152 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:48:32.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:48:32 smithi153 ceph-mon[70044]: pgmap v7808: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:48:32 smithi153 ceph-mon[70044]: from='client.53152 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:48:32.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:48:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:48:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:48:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:48:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:48:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:48:34.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:48:34 smithi083 ceph-mon[79930]: pgmap v7809: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:34.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:48:34 smithi083 ceph-mon[77242]: pgmap v7809: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:48:34 smithi153 ceph-mon[70044]: pgmap v7809: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:34.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:48:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:48:34.496Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:48:35.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:48:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:48:35.203Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:48:36.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:48:36 smithi083 ceph-mon[79930]: pgmap v7810: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:36.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:48:36 smithi083 ceph-mon[77242]: pgmap v7810: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:36.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:48:36 smithi153 ceph-mon[70044]: pgmap v7810: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:38.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:48:38 smithi083 ceph-mon[79930]: pgmap v7811: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:38.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:48:38 smithi083 ceph-mon[77242]: pgmap v7811: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:48:38 smithi153 ceph-mon[70044]: pgmap v7811: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:40.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:48:40 smithi083 ceph-mon[79930]: pgmap v7812: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:48:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:48:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:48:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:48:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:48:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:48:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:48:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:48:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:48:40 smithi083 ceph-mon[77242]: pgmap v7812: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:48:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:48:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:48:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:48:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:48:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:48:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:48:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:48:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:48:40 smithi153 ceph-mon[70044]: pgmap v7812: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:48:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:48:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:48:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:48:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:48:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:48:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:48:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:48:41.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:48:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:48:41.110Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:48:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:48:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:48:41] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T09:48:42.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:48:42 smithi083 ceph-mon[79930]: pgmap v7813: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:42.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:48:42 smithi083 ceph-mon[77242]: pgmap v7813: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:48:42 smithi153 ceph-mon[70044]: pgmap v7813: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:42.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:48:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:48:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:48:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:48:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:48:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:48:44.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:48:44 smithi083 ceph-mon[79930]: pgmap v7814: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:44.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:48:44 smithi083 ceph-mon[77242]: pgmap v7814: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:44.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:48:44 smithi153 ceph-mon[70044]: pgmap v7814: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:44.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:48:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:48:44.496Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:48:45.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:48:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:48:45.204Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:48:46.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:48:46 smithi083 ceph-mon[79930]: pgmap v7815: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:46.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:48:46 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:48:46.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:48:46 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:48:46.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:48:46 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:48:46.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:48:46 smithi083 ceph-mon[77242]: pgmap v7815: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:46.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:48:46 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:48:46.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:48:46 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:48:46.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:48:46 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:48:46.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:48:46 smithi153 ceph-mon[70044]: pgmap v7815: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:46.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:48:46 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:48:46.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:48:46 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:48:46.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:48:46 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:48:47.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:48:47 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:48:47.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:48:47 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:48:47.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:48:47 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:48:48.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:48:48 smithi153 ceph-mon[70044]: pgmap v7816: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:48.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:48:48 smithi083 ceph-mon[79930]: pgmap v7816: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:48.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:48:48 smithi083 ceph-mon[77242]: pgmap v7816: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:50.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:48:50 smithi153 ceph-mon[70044]: pgmap v7817: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:50.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:48:50 smithi083 ceph-mon[79930]: pgmap v7817: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:50.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:48:50 smithi083 ceph-mon[77242]: pgmap v7817: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:51.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:48:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:48:51.110Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:48:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:48:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:48:51] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T09:48:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:48:52 smithi153 ceph-mon[70044]: pgmap v7818: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:52.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:48:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:48:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:48:52.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:48:52 smithi083 ceph-mon[79930]: pgmap v7818: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:52.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:48:52 smithi083 ceph-mon[77242]: pgmap v7818: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:48:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:48:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:48:54.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:48:54 smithi153 ceph-mon[70044]: pgmap v7819: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:54.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:48:54 smithi083 ceph-mon[79930]: pgmap v7819: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:48:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:48:54.497Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:48:54.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:48:54 smithi083 ceph-mon[77242]: pgmap v7819: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:55.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:48:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:48:55.204Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:48:56.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:48:56 smithi153 ceph-mon[70044]: pgmap v7820: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:56.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:48:56 smithi083 ceph-mon[79930]: pgmap v7820: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:56.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:48:56 smithi083 ceph-mon[77242]: pgmap v7820: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:48:58 smithi153 ceph-mon[70044]: pgmap v7821: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:58.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:48:58 smithi083 ceph-mon[79930]: pgmap v7821: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:58.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:48:58 smithi083 ceph-mon[77242]: pgmap v7821: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:59.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:48:59 smithi083 ceph-mon[79930]: pgmap v7822: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:59.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:48:59 smithi083 ceph-mon[77242]: pgmap v7822: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:48:59.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:48:59 smithi153 ceph-mon[70044]: pgmap v7822: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:49:01.161 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:49:01.438 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:49:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:49:01.111Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:49:01.451 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:49:01.451 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 9m ago 4h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:49:01.451 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 7m ago 4h 62.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:49:01.452 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 9m ago 4h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:49:01.452 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 7m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:49:01.452 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 7m ago 4h 443M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:49:01.452 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 9m ago 4h 731M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:49:01.453 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 9m ago 4h 959M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:49:01.453 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 7m ago 4h 422M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:49:01.453 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 9m ago 4h 562M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:49:01.453 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 9m ago 4h 26.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:49:01.454 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 7m ago 4h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:49:01.454 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 9m ago 4h 691M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:49:01.454 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 9m ago 4h 682M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:49:01.454 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 9m ago 4h 534M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:49:01.455 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 9m ago 4h 664M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:49:01.455 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 7m ago 4h 587M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:49:01.455 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 7m ago 4h 730M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:49:01.455 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 7m ago 4h 651M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:49:01.456 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 7m ago 4h 768M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:49:01.456 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 7m ago 4h 173M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:49:01.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:49:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:49:01] "GET /metrics HTTP/1.1" 200 225227 "" "Prometheus/2.33.4" 2022-09-29T09:49:01.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:49:01 smithi083 ceph-mon[79930]: pgmap v7823: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:49:01.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:49:01 smithi083 ceph-mon[79930]: from='client.53164 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:49:01.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:49:01 smithi083 ceph-mon[77242]: pgmap v7823: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:49:01.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:49:01 smithi083 ceph-mon[77242]: from='client.53164 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:49:01.805 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:49:01.805 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:49:01.806 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:49:01.806 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:49:01.806 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:49:01.807 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:49:01.807 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:49:01.807 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:49:01.807 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:49:01.807 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:49:01.808 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:49:01.808 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:49:01.808 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:49:01.808 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:49:01.809 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:49:01.809 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:49:01.809 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:49:01.809 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:49:01.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:49:01 smithi153 ceph-mon[70044]: pgmap v7823: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:49:01.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:49:01 smithi153 ceph-mon[70044]: from='client.53164 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:49:02.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:49:02 smithi083 ceph-mon[79930]: from='client.53170 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:49:02.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:49:02 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3091260273' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:49:02.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:49:02 smithi083 ceph-mon[77242]: from='client.53170 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:49:02.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:49:02 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3091260273' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:49:02.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:49:02 smithi153 ceph-mon[70044]: from='client.53170 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:49:02.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:49:02 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3091260273' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:49:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:49:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:49:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:49:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:49:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:49:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:49:04.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:49:04 smithi083 ceph-mon[77242]: pgmap v7824: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:49:04.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:49:04 smithi083 ceph-mon[79930]: pgmap v7824: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:49:04.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:49:04 smithi153 ceph-mon[70044]: pgmap v7824: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:49:04.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:49:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:49:04.497Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:49:05.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:49:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:49:05.205Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:49:06.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:49:06 smithi083 ceph-mon[79930]: pgmap v7825: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:49:06.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:49:06 smithi083 ceph-mon[77242]: pgmap v7825: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:49:06.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:49:06 smithi153 ceph-mon[70044]: pgmap v7825: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:49:08.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:49:08 smithi083 ceph-mon[77242]: pgmap v7826: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:49:08.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:49:08 smithi083 ceph-mon[79930]: pgmap v7826: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:49:08.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:49:08 smithi153 ceph-mon[70044]: pgmap v7826: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:49:10.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:49:10 smithi083 ceph-mon[79930]: pgmap v7827: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:49:10.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:49:10 smithi083 ceph-mon[77242]: pgmap v7827: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:49:10.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:49:10 smithi153 ceph-mon[70044]: pgmap v7827: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:49:11.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:49:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:49:11.111Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:49:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:49:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:49:11] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T09:49:12.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:49:12 smithi083 ceph-mon[79930]: pgmap v7828: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:49:12.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:49:12 smithi083 ceph-mon[77242]: pgmap v7828: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:49:12.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:49:12 smithi153 ceph-mon[70044]: pgmap v7828: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:49:12.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:49:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:49:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:49:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:49:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:49:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:49:14.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:49:14 smithi083 ceph-mon[79930]: pgmap v7829: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:49:14.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:49:14 smithi083 ceph-mon[77242]: pgmap v7829: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:49:14.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:49:14 smithi153 ceph-mon[70044]: pgmap v7829: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:49:14.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:49:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:49:14.497Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:49:15.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:49:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:49:15.205Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:49:16.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:49:16 smithi083 ceph-mon[79930]: pgmap v7830: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:49:16.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:49:16 smithi083 ceph-mon[77242]: pgmap v7830: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:49:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:49:16 smithi153 ceph-mon[70044]: pgmap v7830: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:49:18.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:49:18 smithi083 ceph-mon[79930]: pgmap v7831: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:49:18.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:49:18 smithi083 ceph-mon[77242]: pgmap v7831: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:49:18.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:49:18 smithi153 ceph-mon[70044]: pgmap v7831: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:49:20.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:49:20 smithi083 ceph-mon[79930]: pgmap v7832: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:49:20.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:49:20 smithi083 ceph-mon[77242]: pgmap v7832: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:49:20.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:49:20 smithi153 ceph-mon[70044]: pgmap v7832: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:49:21.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:49:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:49:21.111Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:49:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:49:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:49:21] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T09:49:22.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:49:22 smithi083 ceph-mon[79930]: pgmap v7833: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:49:22.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:49:22 smithi083 ceph-mon[77242]: pgmap v7833: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:49:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:49:22 smithi153 ceph-mon[70044]: pgmap v7833: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:49:22.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:49:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:49:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:49:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:49:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:49:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:49:24.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:49:24 smithi083 ceph-mon[79930]: pgmap v7834: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:49:24.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:49:24 smithi083 ceph-mon[77242]: pgmap v7834: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:49:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:49:24 smithi153 ceph-mon[70044]: pgmap v7834: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:49:24.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:49:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:49:24.498Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:49:25.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:49:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:49:25.206Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:49:26.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:49:26 smithi083 ceph-mon[79930]: pgmap v7835: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:49:26.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:49:26 smithi083 ceph-mon[77242]: pgmap v7835: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:49:26.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:49:26 smithi153 ceph-mon[70044]: pgmap v7835: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:49:28.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:49:28 smithi083 ceph-mon[79930]: pgmap v7836: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:49:28.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:49:28 smithi083 ceph-mon[77242]: pgmap v7836: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:49:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:49:28 smithi153 ceph-mon[70044]: pgmap v7836: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:49:30.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:49:30 smithi083 ceph-mon[79930]: pgmap v7837: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:49:30.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:49:30 smithi083 ceph-mon[77242]: pgmap v7837: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:49:30.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:49:30 smithi153 ceph-mon[70044]: pgmap v7837: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:49:31.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:49:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:49:31.112Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:49:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:49:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:49:31] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T09:49:32.133 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:49:32.424 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:49:32.424 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 10m ago 4h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:49:32.424 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 7m ago 4h 62.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:49:32.425 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 10m ago 4h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:49:32.425 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 7m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:49:32.425 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 7m ago 4h 443M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:49:32.425 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 10m ago 4h 731M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:49:32.425 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 10m ago 4h 959M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:49:32.426 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 7m ago 4h 422M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:49:32.426 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 10m ago 4h 562M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:49:32.426 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 10m ago 4h 26.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:49:32.426 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 7m ago 4h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:49:32.427 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 10m ago 4h 691M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:49:32.427 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 10m ago 4h 682M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:49:32.427 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 10m ago 4h 534M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:49:32.427 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 10m ago 4h 664M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:49:32.428 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 7m ago 4h 587M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:49:32.428 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 7m ago 4h 730M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:49:32.428 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 7m ago 4h 651M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:49:32.428 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 7m ago 4h 768M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:49:32.428 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 7m ago 4h 173M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:49:32.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:49:32 smithi083 ceph-mon[77242]: pgmap v7838: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:49:32.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:49:32 smithi083 ceph-mon[79930]: pgmap v7838: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:49:32.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:49:32 smithi153 ceph-mon[70044]: pgmap v7838: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:49:32.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:49:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:49:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:49:32.778 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:49:32.779 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:49:32.779 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:49:32.779 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:49:32.779 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:49:32.780 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:49:32.780 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:49:32.780 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:49:32.780 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:49:32.781 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:49:32.781 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:49:32.781 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:49:32.781 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:49:32.782 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:49:32.782 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:49:32.782 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:49:32.782 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:49:32.783 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:49:33.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:49:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:49:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:49:33.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:49:33 smithi153 ceph-mon[70044]: from='client.43251 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:49:33.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:49:33 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3239192179' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:49:33.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:49:33 smithi083 ceph-mon[79930]: from='client.43251 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:49:33.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:49:33 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3239192179' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:49:33.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:49:33 smithi083 ceph-mon[77242]: from='client.43251 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:49:33.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:49:33 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3239192179' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:49:34.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:49:34 smithi083 ceph-mon[77242]: pgmap v7839: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:49:34.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:49:34 smithi083 ceph-mon[77242]: from='client.53188 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:49:34.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:49:34 smithi083 ceph-mon[79930]: pgmap v7839: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:49:34.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:49:34 smithi083 ceph-mon[79930]: from='client.53188 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:49:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:49:34 smithi153 ceph-mon[70044]: pgmap v7839: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:49:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:49:34 smithi153 ceph-mon[70044]: from='client.53188 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:49:34.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:49:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:49:34.498Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:49:35.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:49:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:49:35.207Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:49:36.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:49:36 smithi083 ceph-mon[79930]: pgmap v7840: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:49:36.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:49:36 smithi083 ceph-mon[77242]: pgmap v7840: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:49:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:49:36 smithi153 ceph-mon[70044]: pgmap v7840: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:49:38.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:49:38 smithi083 ceph-mon[79930]: pgmap v7841: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:49:38.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:49:38 smithi083 ceph-mon[77242]: pgmap v7841: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:49:38.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:49:38 smithi153 ceph-mon[70044]: pgmap v7841: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:49:40.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:49:40 smithi083 ceph-mon[79930]: pgmap v7842: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:49:40.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:49:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:49:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:49:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:49:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:49:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:49:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:49:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:49:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:49:40 smithi083 ceph-mon[77242]: pgmap v7842: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:49:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:49:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:49:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:49:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:49:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:49:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:49:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:49:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:49:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:49:40 smithi153 ceph-mon[70044]: pgmap v7842: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:49:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:49:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:49:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:49:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:49:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:49:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:49:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:49:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:49:41.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:49:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:49:41.113Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:49:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:49:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:49:41] "GET /metrics HTTP/1.1" 200 225229 "" "Prometheus/2.33.4" 2022-09-29T09:49:42.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:49:42 smithi083 ceph-mon[77242]: pgmap v7843: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:49:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:49:42 smithi083 ceph-mon[79930]: pgmap v7843: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:49:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:49:42 smithi153 ceph-mon[70044]: pgmap v7843: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:49:42.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:49:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:49:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:49:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:49:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:49:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:49:44.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:49:44 smithi083 ceph-mon[79930]: pgmap v7844: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:49:44.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:49:44 smithi083 ceph-mon[77242]: pgmap v7844: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:49:44.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:49:44 smithi153 ceph-mon[70044]: pgmap v7844: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:49:44.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:49:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:49:44.500Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:49:45.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:49:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:49:45.207Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:49:46.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:49:46 smithi083 ceph-mon[79930]: pgmap v7845: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:49:46.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:49:46 smithi083 ceph-mon[77242]: pgmap v7845: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:49:46.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:49:46 smithi153 ceph-mon[70044]: pgmap v7845: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:49:47.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:49:47 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:49:47.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:49:47 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:49:47.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:49:47 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:49:47.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:49:47 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:49:47.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:49:47 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:49:47.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:49:47 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:49:47.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:49:47 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:49:47.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:49:47 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:49:47.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:49:47 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:49:47.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:49:47 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:49:47.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:49:47 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:49:47.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:49:47 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:49:47.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:49:47 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:49:47.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:49:47 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:49:47.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:49:47 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:49:47.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:49:47 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:49:47.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:49:47 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:49:47.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:49:47 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:49:47.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:49:47 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:49:47.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:49:47 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:49:47.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:49:47 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:49:48.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:49:48 smithi083 ceph-mon[79930]: pgmap v7846: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:49:48.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:49:48 smithi083 ceph-mon[77242]: pgmap v7846: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:49:48.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:49:48 smithi153 ceph-mon[70044]: pgmap v7846: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:49:50.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:49:50 smithi083 ceph-mon[79930]: pgmap v7847: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:49:50.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:49:50 smithi083 ceph-mon[77242]: pgmap v7847: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:49:50.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:49:50 smithi153 ceph-mon[70044]: pgmap v7847: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:49:51.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:49:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:49:51.113Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:49:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:49:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:49:51] "GET /metrics HTTP/1.1" 200 225229 "" "Prometheus/2.33.4" 2022-09-29T09:49:52.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:49:51 smithi153 ceph-mon[70044]: pgmap v7848: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:49:52.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:49:51 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:49:52.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:49:51 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:49:52.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:49:51 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:49:52.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:49:51 smithi083 ceph-mon[77242]: pgmap v7848: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:49:52.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:49:51 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:49:52.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:49:51 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:49:52.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:49:51 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:49:52.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:49:51 smithi083 ceph-mon[79930]: pgmap v7848: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:49:52.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:49:51 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:49:52.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:49:51 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:49:52.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:49:51 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:49:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:49:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:49:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:49:53.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:49:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:49:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:49:54.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:49:54 smithi083 ceph-mon[79930]: pgmap v7849: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:49:54.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:49:54 smithi083 ceph-mon[77242]: pgmap v7849: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:49:54.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:49:54 smithi153 ceph-mon[70044]: pgmap v7849: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:49:54.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:49:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:49:54.500Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:49:55.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:49:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:49:55.208Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:49:56.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:49:56 smithi083 ceph-mon[79930]: pgmap v7850: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:49:56.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:49:56 smithi083 ceph-mon[77242]: pgmap v7850: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:49:56.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:49:56 smithi153 ceph-mon[70044]: pgmap v7850: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:49:58.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:49:58 smithi083 ceph-mon[79930]: pgmap v7851: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:49:58.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:49:58 smithi083 ceph-mon[77242]: pgmap v7851: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:49:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:49:58 smithi153 ceph-mon[70044]: pgmap v7851: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:00.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:50:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[77219]: 2022-09-29T09:49:59.999+0000 7f160f665700 -1 log_channel(cluster) log [ERR] : overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T09:50:00.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:50:00 smithi083 ceph-mon[77242]: pgmap v7852: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:00.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:50:00 smithi083 ceph-mon[77242]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T09:50:00.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:50:00 smithi083 ceph-mon[79930]: pgmap v7852: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:00.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:50:00 smithi083 ceph-mon[79930]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T09:50:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:50:00 smithi153 ceph-mon[70044]: pgmap v7852: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:50:00 smithi153 ceph-mon[70044]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T09:50:01.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:50:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:50:01.114Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:50:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:50:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:50:01] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T09:50:02.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:50:02 smithi083 ceph-mon[79930]: pgmap v7853: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:02.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:50:02 smithi083 ceph-mon[77242]: pgmap v7853: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:50:02 smithi153 ceph-mon[70044]: pgmap v7853: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:02.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:50:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:50:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:50:03.107 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:50:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:50:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:50:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:50:03.404 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:50:03.405 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 12s ago 4h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:50:03.405 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 8m ago 4h 62.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:50:03.405 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 12s ago 4h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:50:03.405 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 8m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:50:03.406 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 8m ago 4h 443M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:50:03.406 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 12s ago 4h 740M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:50:03.406 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 12s ago 4h 994M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:50:03.406 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 8m ago 4h 422M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:50:03.407 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 12s ago 4h 581M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:50:03.407 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 12s ago 4h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:50:03.407 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 8m ago 4h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:50:03.407 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 12s ago 4h 717M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:50:03.408 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 12s ago 4h 708M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:50:03.408 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 12s ago 4h 554M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:50:03.408 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 12s ago 4h 689M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:50:03.408 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 8m ago 4h 587M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:50:03.408 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 8m ago 4h 730M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:50:03.409 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 8m ago 4h 651M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:50:03.409 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 8m ago 4h 768M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:50:03.409 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 8m ago 4h 173M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:50:03.770 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:50:03.771 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:50:03.771 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:50:03.771 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:50:03.772 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:50:03.772 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:50:03.772 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:50:03.772 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:50:03.773 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:50:03.773 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:50:03.773 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:50:03.773 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:50:03.774 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:50:03.774 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:50:03.774 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:50:03.774 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:50:03.775 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:50:03.775 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:50:04.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:50:04 smithi083 ceph-mon[79930]: pgmap v7854: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:04.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:50:04 smithi083 ceph-mon[79930]: from='client.43269 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:50:04.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:50:04 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2847894768' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:50:04.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:50:04 smithi083 ceph-mon[77242]: pgmap v7854: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:04.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:50:04 smithi083 ceph-mon[77242]: from='client.43269 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:50:04.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:50:04 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2847894768' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:50:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:50:04 smithi153 ceph-mon[70044]: pgmap v7854: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:50:04 smithi153 ceph-mon[70044]: from='client.43269 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:50:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:50:04 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2847894768' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:50:04.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:50:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:50:04.500Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:50:05.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:50:05 smithi083 ceph-mon[79930]: from='client.43275 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:50:05.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:50:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:50:05.209Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:50:05.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:50:05 smithi083 ceph-mon[77242]: from='client.43275 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:50:05.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:50:05 smithi153 ceph-mon[70044]: from='client.43275 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:50:06.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:50:06 smithi083 ceph-mon[79930]: pgmap v7855: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:06.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:50:06 smithi083 ceph-mon[77242]: pgmap v7855: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:06.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:50:06 smithi153 ceph-mon[70044]: pgmap v7855: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:08.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:50:08 smithi083 ceph-mon[79930]: pgmap v7856: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:08.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:50:08 smithi083 ceph-mon[77242]: pgmap v7856: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:50:08 smithi153 ceph-mon[70044]: pgmap v7856: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:10.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:50:10 smithi083 ceph-mon[79930]: pgmap v7857: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:10.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:50:10 smithi083 ceph-mon[77242]: pgmap v7857: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:10.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:50:10 smithi153 ceph-mon[70044]: pgmap v7857: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:11.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:50:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:50:11.115Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:50:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:50:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:50:11] "GET /metrics HTTP/1.1" 200 225223 "" "Prometheus/2.33.4" 2022-09-29T09:50:12.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:50:12 smithi083 ceph-mon[79930]: pgmap v7858: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:12.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:50:12 smithi083 ceph-mon[77242]: pgmap v7858: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:50:12 smithi153 ceph-mon[70044]: pgmap v7858: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:12.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:50:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:50:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:50:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:50:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:50:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:50:14.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:50:14 smithi083 ceph-mon[77242]: pgmap v7859: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:14.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:50:14 smithi083 ceph-mon[79930]: pgmap v7859: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:14.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:50:14 smithi153 ceph-mon[70044]: pgmap v7859: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:14.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:50:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:50:14.501Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:50:15.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:50:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:50:15.210Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:50:16.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:50:16 smithi083 ceph-mon[79930]: pgmap v7860: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:16.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:50:16 smithi083 ceph-mon[77242]: pgmap v7860: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:50:16 smithi153 ceph-mon[70044]: pgmap v7860: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:18.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:50:18 smithi083 ceph-mon[79930]: pgmap v7861: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:18.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:50:18 smithi083 ceph-mon[77242]: pgmap v7861: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:50:18 smithi153 ceph-mon[70044]: pgmap v7861: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:20.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:50:20 smithi083 ceph-mon[77242]: pgmap v7862: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:20.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:50:20 smithi083 ceph-mon[79930]: pgmap v7862: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:20.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:50:20 smithi153 ceph-mon[70044]: pgmap v7862: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:21.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:50:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:50:21.116Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:50:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:50:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:50:21] "GET /metrics HTTP/1.1" 200 225223 "" "Prometheus/2.33.4" 2022-09-29T09:50:22.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:50:22 smithi083 ceph-mon[79930]: pgmap v7863: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:22.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:50:22 smithi083 ceph-mon[77242]: pgmap v7863: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:50:22 smithi153 ceph-mon[70044]: pgmap v7863: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:22.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:50:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:50:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:50:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:50:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:50:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:50:24.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:50:24 smithi083 ceph-mon[79930]: pgmap v7864: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:24.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:50:24 smithi083 ceph-mon[77242]: pgmap v7864: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:50:24 smithi153 ceph-mon[70044]: pgmap v7864: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:24.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:50:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:50:24.501Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:50:25.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:50:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:50:25.210Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:50:26.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:50:26 smithi083 ceph-mon[79930]: pgmap v7865: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:26.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:50:26 smithi083 ceph-mon[77242]: pgmap v7865: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:50:26 smithi153 ceph-mon[70044]: pgmap v7865: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:28.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:50:28 smithi083 ceph-mon[79930]: pgmap v7866: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:28.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:50:28 smithi083 ceph-mon[77242]: pgmap v7866: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:28.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:50:28 smithi153 ceph-mon[70044]: pgmap v7866: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:30.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:50:30 smithi083 ceph-mon[79930]: pgmap v7867: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:30.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:50:30 smithi083 ceph-mon[77242]: pgmap v7867: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:50:30 smithi153 ceph-mon[70044]: pgmap v7867: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:31.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:50:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:50:31.117Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:50:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:50:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:50:31] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T09:50:32.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:50:32 smithi083 ceph-mon[77242]: pgmap v7868: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:32.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:50:32 smithi083 ceph-mon[79930]: pgmap v7868: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:50:32 smithi153 ceph-mon[70044]: pgmap v7868: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:32.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:50:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:50:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:50:33.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:50:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:50:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:50:34.096 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:50:34.382 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:50:34.383 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 43s ago 4h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:50:34.383 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 8m ago 4h 62.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:50:34.383 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 43s ago 4h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:50:34.384 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 8m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:50:34.384 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 8m ago 4h 443M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:50:34.384 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 43s ago 4h 740M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:50:34.384 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 43s ago 4h 994M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:50:34.384 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 8m ago 4h 422M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:50:34.385 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 43s ago 4h 581M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:50:34.385 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 43s ago 4h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:50:34.385 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 8m ago 4h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:50:34.385 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 43s ago 4h 717M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:50:34.385 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 43s ago 4h 708M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:50:34.386 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 43s ago 4h 554M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:50:34.386 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 43s ago 4h 689M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:50:34.386 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 8m ago 4h 587M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:50:34.386 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 8m ago 4h 730M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:50:34.387 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 8m ago 4h 651M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:50:34.387 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 8m ago 4h 768M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:50:34.387 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 8m ago 4h 173M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:50:34.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:50:34 smithi083 ceph-mon[79930]: pgmap v7869: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:34.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:50:34 smithi083 ceph-mon[77242]: pgmap v7869: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:50:34 smithi153 ceph-mon[70044]: pgmap v7869: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:34.738 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:50:34.738 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:50:34.739 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:50:34.739 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:50:34.739 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:50:34.739 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:50:34.740 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:50:34.740 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:50:34.740 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:50:34.740 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:50:34.741 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:50:34.741 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:50:34.741 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:50:34.741 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:50:34.741 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:50:34.742 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:50:34.742 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:50:34.742 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:50:34.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:50:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:50:34.501Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:50:35.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:50:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:50:35.211Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:50:35.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:50:35 smithi083 ceph-mon[77242]: from='client.53218 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:50:35.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:50:35 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1274640585' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:50:35.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:50:35 smithi083 ceph-mon[79930]: from='client.53218 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:50:35.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:50:35 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1274640585' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:50:35.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:50:35 smithi153 ceph-mon[70044]: from='client.53218 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:50:35.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:50:35 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1274640585' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:50:36.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:50:36 smithi083 ceph-mon[77242]: from='client.43293 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:50:36.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:50:36 smithi083 ceph-mon[77242]: pgmap v7870: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:36.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:50:36 smithi083 ceph-mon[79930]: from='client.43293 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:50:36.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:50:36 smithi083 ceph-mon[79930]: pgmap v7870: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:36.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:50:36 smithi153 ceph-mon[70044]: from='client.43293 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:50:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:50:36 smithi153 ceph-mon[70044]: pgmap v7870: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:38.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:50:38 smithi083 ceph-mon[77242]: pgmap v7871: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:38.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:50:38 smithi083 ceph-mon[79930]: pgmap v7871: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:38.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:50:38 smithi153 ceph-mon[70044]: pgmap v7871: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:40.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:50:40 smithi083 ceph-mon[77242]: pgmap v7872: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:40.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:50:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:50:40.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:50:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:50:40.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:50:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:50:40.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:50:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:50:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:50:40 smithi083 ceph-mon[79930]: pgmap v7872: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:50:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:50:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:50:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:50:40.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:50:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:50:40.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:50:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:50:40.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:50:40 smithi153 ceph-mon[70044]: pgmap v7872: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:50:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:50:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:50:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:50:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:50:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:50:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:50:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:50:41.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:50:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:50:41.117Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:50:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:50:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:50:41] "GET /metrics HTTP/1.1" 200 225228 "" "Prometheus/2.33.4" 2022-09-29T09:50:42.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:50:42 smithi083 ceph-mon[77242]: pgmap v7873: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:50:42 smithi083 ceph-mon[79930]: pgmap v7873: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:50:42 smithi153 ceph-mon[70044]: pgmap v7873: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:42.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:50:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:50:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:50:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:50:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:50:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:50:44.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:50:44 smithi083 ceph-mon[77242]: pgmap v7874: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:44.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:50:44 smithi083 ceph-mon[79930]: pgmap v7874: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:44.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:50:44 smithi153 ceph-mon[70044]: pgmap v7874: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:44.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:50:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:50:44.502Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:50:45.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:50:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:50:45.211Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:50:46.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:50:46 smithi083 ceph-mon[79930]: pgmap v7875: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:46.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:50:46 smithi083 ceph-mon[77242]: pgmap v7875: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:46.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:50:46 smithi153 ceph-mon[70044]: pgmap v7875: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:48.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:50:48 smithi083 ceph-mon[79930]: pgmap v7876: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:48.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:50:48 smithi083 ceph-mon[77242]: pgmap v7876: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:48.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:50:48 smithi153 ceph-mon[70044]: pgmap v7876: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:50.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:50:50 smithi083 ceph-mon[79930]: pgmap v7877: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:50.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:50:50 smithi083 ceph-mon[77242]: pgmap v7877: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:50.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:50:50 smithi153 ceph-mon[70044]: pgmap v7877: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:51.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:50:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:50:51.118Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:50:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:50:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:50:51] "GET /metrics HTTP/1.1" 200 225228 "" "Prometheus/2.33.4" 2022-09-29T09:50:52.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:50:52 smithi083 ceph-mon[79930]: pgmap v7878: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:52.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:50:52 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:50:52.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:50:52 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:50:52.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:50:52 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:50:52.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:50:52 smithi083 ceph-mon[77242]: pgmap v7878: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:52.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:50:52 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:50:52.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:50:52 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:50:52.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:50:52 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:50:52.556 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:50:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:50:52.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:50:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:50:52 smithi153 ceph-mon[70044]: pgmap v7878: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:50:52 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:50:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:50:52 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:50:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:50:52 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:50:53.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:50:53 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:50:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:50:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:50:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:50:53.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:50:53 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:50:53.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:50:53 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:50:54.503 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:50:54 smithi083 ceph-mon[79930]: pgmap v7879: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:54.509 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:50:54 smithi083 ceph-mon[77242]: pgmap v7879: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:54.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:50:54 smithi153 ceph-mon[70044]: pgmap v7879: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:54.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:50:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:50:54.502Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:50:55.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:50:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:50:55.212Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:50:56.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:50:56 smithi153 ceph-mon[70044]: pgmap v7880: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:56.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:50:56 smithi083 ceph-mon[79930]: pgmap v7880: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:56.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:50:56 smithi083 ceph-mon[77242]: pgmap v7880: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:50:58 smithi153 ceph-mon[70044]: pgmap v7881: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:58.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:50:58 smithi083 ceph-mon[79930]: pgmap v7881: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:50:58.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:50:58 smithi083 ceph-mon[77242]: pgmap v7881: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:51:00 smithi153 ceph-mon[70044]: pgmap v7882: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:00.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:51:00 smithi083 ceph-mon[79930]: pgmap v7882: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:00.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:51:00 smithi083 ceph-mon[77242]: pgmap v7882: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:01.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:51:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:51:01.118Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:51:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:51:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:51:01] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T09:51:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:51:02 smithi153 ceph-mon[70044]: pgmap v7883: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:02.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:51:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:51:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:51:02.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:51:02 smithi083 ceph-mon[79930]: pgmap v7883: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:02.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:51:02 smithi083 ceph-mon[77242]: pgmap v7883: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:51:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:51:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:51:04.503 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:51:04 smithi083 ceph-mon[79930]: pgmap v7884: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:04.504 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:51:04 smithi083 ceph-mon[77242]: pgmap v7884: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:04.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:51:04 smithi153 ceph-mon[70044]: pgmap v7884: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:04.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:51:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:51:04.502Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:51:05.068 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:51:05.361 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:51:05.361 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 74s ago 4h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:51:05.362 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 9m ago 4h 62.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:51:05.362 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 74s ago 4h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:51:05.362 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 9m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:51:05.362 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 9m ago 4h 443M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:51:05.362 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 74s ago 4h 740M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:51:05.363 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 74s ago 4h 994M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:51:05.363 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 9m ago 4h 422M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:51:05.363 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 74s ago 4h 581M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:51:05.363 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 74s ago 4h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:51:05.364 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 9m ago 4h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:51:05.364 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 74s ago 4h 717M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:51:05.364 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 74s ago 4h 708M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:51:05.364 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 74s ago 4h 554M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:51:05.365 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 74s ago 4h 689M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:51:05.365 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 9m ago 4h 587M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:51:05.365 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 9m ago 4h 730M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:51:05.365 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 9m ago 4h 651M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:51:05.365 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 9m ago 4h 768M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:51:05.366 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 9m ago 4h 173M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:51:05.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:51:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:51:05.214Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:51:05.719 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:51:05.719 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:51:05.719 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:51:05.719 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:51:05.720 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:51:05.720 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:51:05.720 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:51:05.720 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:51:05.721 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:51:05.721 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:51:05.721 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:51:05.721 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:51:05.722 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:51:05.722 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:51:05.722 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:51:05.722 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:51:05.722 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:51:05.723 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:51:06.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:51:06 smithi153 ceph-mon[70044]: pgmap v7885: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:06.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:51:06 smithi153 ceph-mon[70044]: from='client.43305 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:51:06.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:51:06 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1441104661' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:51:06.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:51:06 smithi083 ceph-mon[79930]: pgmap v7885: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:06.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:51:06 smithi083 ceph-mon[79930]: from='client.43305 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:51:06.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:51:06 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1441104661' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:51:06.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:51:06 smithi083 ceph-mon[77242]: pgmap v7885: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:06.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:51:06 smithi083 ceph-mon[77242]: from='client.43305 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:51:06.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:51:06 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1441104661' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:51:07.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:51:07 smithi153 ceph-mon[70044]: from='client.53242 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:51:07.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:51:07 smithi083 ceph-mon[79930]: from='client.53242 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:51:07.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:51:07 smithi083 ceph-mon[77242]: from='client.53242 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:51:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:51:08 smithi153 ceph-mon[70044]: pgmap v7886: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:08.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:51:08 smithi083 ceph-mon[79930]: pgmap v7886: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:08.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:51:08 smithi083 ceph-mon[77242]: pgmap v7886: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:51:10 smithi153 ceph-mon[70044]: pgmap v7887: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:10.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:51:10 smithi083 ceph-mon[79930]: pgmap v7887: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:10.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:51:10 smithi083 ceph-mon[77242]: pgmap v7887: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:11.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:51:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:51:11.118Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:51:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:51:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:51:11] "GET /metrics HTTP/1.1" 200 225228 "" "Prometheus/2.33.4" 2022-09-29T09:51:12.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:51:12 smithi153 ceph-mon[70044]: pgmap v7888: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:12.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:51:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:51:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:51:12.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:51:12 smithi083 ceph-mon[79930]: pgmap v7888: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:12.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:51:12 smithi083 ceph-mon[77242]: pgmap v7888: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:51:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:51:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:51:14.505 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:51:14 smithi083 ceph-mon[79930]: pgmap v7889: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:14.505 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:51:14 smithi083 ceph-mon[77242]: pgmap v7889: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:14.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:51:14 smithi153 ceph-mon[70044]: pgmap v7889: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:14.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:51:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:51:14.503Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:51:15.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:51:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:51:15.214Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:51:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:51:16 smithi153 ceph-mon[70044]: pgmap v7890: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:16.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:51:16 smithi083 ceph-mon[77242]: pgmap v7890: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:16.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:51:16 smithi083 ceph-mon[79930]: pgmap v7890: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:51:18 smithi153 ceph-mon[70044]: pgmap v7891: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:18.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:51:18 smithi083 ceph-mon[79930]: pgmap v7891: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:18.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:51:18 smithi083 ceph-mon[77242]: pgmap v7891: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:20.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:51:20 smithi153 ceph-mon[70044]: pgmap v7892: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:20.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:51:20 smithi083 ceph-mon[79930]: pgmap v7892: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:20.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:51:20 smithi083 ceph-mon[77242]: pgmap v7892: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:21.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:51:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:51:21.119Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:51:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:51:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:51:21] "GET /metrics HTTP/1.1" 200 225228 "" "Prometheus/2.33.4" 2022-09-29T09:51:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:51:22 smithi153 ceph-mon[70044]: pgmap v7893: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:22.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:51:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:51:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:51:22.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:51:22 smithi083 ceph-mon[79930]: pgmap v7893: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:22.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:51:22 smithi083 ceph-mon[77242]: pgmap v7893: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:51:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:51:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:51:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:51:24 smithi153 ceph-mon[70044]: pgmap v7894: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:24.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:51:24 smithi083 ceph-mon[79930]: pgmap v7894: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:51:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:51:24.504Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:51:24.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:51:24 smithi083 ceph-mon[77242]: pgmap v7894: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:25.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:51:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:51:25.214Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:51:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:51:26 smithi153 ceph-mon[70044]: pgmap v7895: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:26.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:51:26 smithi083 ceph-mon[77242]: pgmap v7895: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:26.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:51:26 smithi083 ceph-mon[79930]: pgmap v7895: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:51:28 smithi153 ceph-mon[70044]: pgmap v7896: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:28.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:51:28 smithi083 ceph-mon[79930]: pgmap v7896: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:28.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:51:28 smithi083 ceph-mon[77242]: pgmap v7896: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:51:30 smithi153 ceph-mon[70044]: pgmap v7897: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:30.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:51:30 smithi083 ceph-mon[79930]: pgmap v7897: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:30.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:51:30 smithi083 ceph-mon[77242]: pgmap v7897: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:31.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:51:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:51:31.120Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:51:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:51:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:51:31] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T09:51:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:51:32 smithi153 ceph-mon[70044]: pgmap v7898: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:32.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:51:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:51:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:51:32.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:51:32 smithi083 ceph-mon[79930]: pgmap v7898: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:32.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:51:32 smithi083 ceph-mon[77242]: pgmap v7898: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:51:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:51:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:51:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:51:34 smithi153 ceph-mon[70044]: pgmap v7899: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:34.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:51:34 smithi083 ceph-mon[79930]: pgmap v7899: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:34.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:51:34 smithi083 ceph-mon[77242]: pgmap v7899: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:34.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:51:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:51:34.505Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:51:35.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:51:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:51:35.215Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:51:36.050 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:51:36.335 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:51:36.335 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 105s ago 4h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:51:36.336 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 9m ago 4h 62.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:51:36.336 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 105s ago 4h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:51:36.336 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 9m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:51:36.336 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 9m ago 4h 443M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:51:36.337 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 105s ago 4h 740M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:51:36.337 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 105s ago 4h 994M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:51:36.337 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 9m ago 4h 422M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:51:36.337 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 105s ago 4h 581M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:51:36.338 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 105s ago 4h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:51:36.338 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 9m ago 4h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:51:36.338 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 105s ago 4h 717M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:51:36.338 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 105s ago 4h 708M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:51:36.339 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 105s ago 4h 554M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:51:36.339 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 105s ago 4h 689M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:51:36.339 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 9m ago 4h 587M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:51:36.339 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 9m ago 4h 730M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:51:36.339 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 9m ago 4h 651M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:51:36.340 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 9m ago 4h 768M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:51:36.340 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 9m ago 4h 173M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:51:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:51:36 smithi153 ceph-mon[70044]: pgmap v7900: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:36.690 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:51:36.690 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:51:36.690 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:51:36.691 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:51:36.691 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:51:36.691 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:51:36.691 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:51:36.691 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:51:36.692 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:51:36.692 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:51:36.692 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:51:36.692 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:51:36.692 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:51:36.693 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:51:36.693 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:51:36.693 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:51:36.693 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:51:36.693 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:51:36.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:51:36 smithi083 ceph-mon[79930]: pgmap v7900: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:36.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:51:36 smithi083 ceph-mon[77242]: pgmap v7900: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:37.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:51:37 smithi153 ceph-mon[70044]: from='client.43323 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:51:37.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:51:37 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/459646409' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:51:37.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:51:37 smithi083 ceph-mon[79930]: from='client.43323 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:51:37.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:51:37 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/459646409' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:51:37.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:51:37 smithi083 ceph-mon[77242]: from='client.43323 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:51:37.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:51:37 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/459646409' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:51:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:51:38 smithi153 ceph-mon[70044]: from='client.53260 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:51:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:51:38 smithi153 ceph-mon[70044]: pgmap v7901: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:38.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:51:38 smithi083 ceph-mon[79930]: from='client.53260 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:51:38.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:51:38 smithi083 ceph-mon[79930]: pgmap v7901: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:38.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:51:38 smithi083 ceph-mon[77242]: from='client.53260 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:51:38.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:51:38 smithi083 ceph-mon[77242]: pgmap v7901: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:51:40 smithi153 ceph-mon[70044]: pgmap v7902: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:51:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:51:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:51:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:51:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:51:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:51:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:51:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:51:40.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:51:40 smithi083 ceph-mon[79930]: pgmap v7902: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:51:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:51:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:51:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:51:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:51:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:51:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:51:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:51:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:51:40 smithi083 ceph-mon[77242]: pgmap v7902: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:51:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:51:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:51:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:51:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:51:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:51:40.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:51:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:51:41.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:51:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:51:41.121Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:51:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:51:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:51:41] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T09:51:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:51:42 smithi153 ceph-mon[70044]: pgmap v7903: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:42.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:51:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:51:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:51:42.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:51:42 smithi083 ceph-mon[79930]: pgmap v7903: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:42.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:51:42 smithi083 ceph-mon[77242]: pgmap v7903: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:51:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:51:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:51:44.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:51:44 smithi153 ceph-mon[70044]: pgmap v7904: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:44.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:51:44 smithi083 ceph-mon[79930]: pgmap v7904: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:44.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:51:44 smithi083 ceph-mon[77242]: pgmap v7904: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:51:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:51:44.505Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:51:45.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:51:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:51:45.216Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:51:46.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:51:46 smithi153 ceph-mon[70044]: pgmap v7905: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:46.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:51:46 smithi083 ceph-mon[79930]: pgmap v7905: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:46.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:51:46 smithi083 ceph-mon[77242]: pgmap v7905: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:48.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:51:48 smithi153 ceph-mon[70044]: pgmap v7906: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:48.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:51:48 smithi083 ceph-mon[79930]: pgmap v7906: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:48.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:51:48 smithi083 ceph-mon[77242]: pgmap v7906: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:50.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:51:50 smithi083 ceph-mon[79930]: pgmap v7907: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:50.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:51:50 smithi083 ceph-mon[77242]: pgmap v7907: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:50.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:51:50 smithi153 ceph-mon[70044]: pgmap v7907: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:51.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:51:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:51:51.122Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:51:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:51:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:51:51] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T09:51:52.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:51:52 smithi083 ceph-mon[79930]: pgmap v7908: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:52.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:51:52 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:51:52.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:51:52 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:51:52.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:51:52 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:51:52.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:51:52 smithi083 ceph-mon[77242]: pgmap v7908: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:52.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:51:52 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:51:52.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:51:52 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:51:52.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:51:52 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:51:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:51:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:51:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:51:52.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:51:52 smithi153 ceph-mon[70044]: pgmap v7908: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:52.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:51:52 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:51:52.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:51:52 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:51:52.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:51:52 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:51:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:51:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:51:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:51:54.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:51:54 smithi083 ceph-mon[79930]: pgmap v7909: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:51:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:51:54.505Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:51:54.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:51:54 smithi083 ceph-mon[77242]: pgmap v7909: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:54.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:51:54 smithi153 ceph-mon[70044]: pgmap v7909: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:55.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:51:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:51:55.216Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:51:56.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:51:56 smithi083 ceph-mon[77242]: pgmap v7910: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:56.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:51:56 smithi083 ceph-mon[79930]: pgmap v7910: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:56.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:51:56 smithi153 ceph-mon[70044]: pgmap v7910: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:58.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:51:57 smithi083 ceph-mon[79930]: pgmap v7911: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:58.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:51:57 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:51:58.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:51:57 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:51:58.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:51:57 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:51:58.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:51:57 smithi083 ceph-mon[77242]: pgmap v7911: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:58.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:51:57 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:51:58.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:51:57 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:51:58.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:51:57 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:51:58.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:51:57 smithi153 ceph-mon[70044]: pgmap v7911: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:51:58.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:51:57 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:51:58.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:51:57 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:51:58.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:51:57 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:52:00.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:52:00 smithi083 ceph-mon[79930]: pgmap v7912: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:00.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:52:00 smithi083 ceph-mon[77242]: pgmap v7912: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:00.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:52:00 smithi153 ceph-mon[70044]: pgmap v7912: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:01.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:52:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:52:01.122Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:52:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:52:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:52:01] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T09:52:02.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:52:02 smithi083 ceph-mon[79930]: pgmap v7913: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:02.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:52:02 smithi083 ceph-mon[77242]: pgmap v7913: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:52:02 smithi153 ceph-mon[70044]: pgmap v7913: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:02.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:52:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:52:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:52:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:52:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:52:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:52:04.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:52:04 smithi083 ceph-mon[79930]: pgmap v7914: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:04.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:52:04 smithi083 ceph-mon[77242]: pgmap v7914: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:52:04 smithi153 ceph-mon[70044]: pgmap v7914: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:04.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:52:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:52:04.505Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:52:05.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:52:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:52:05.217Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:52:06.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:52:06 smithi083 ceph-mon[79930]: pgmap v7915: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:06.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:52:06 smithi083 ceph-mon[77242]: pgmap v7915: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:06.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:52:06 smithi153 ceph-mon[70044]: pgmap v7915: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:07.019 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:52:07.311 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:52:07.312 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 2m ago 4h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:52:07.312 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 10s ago 4h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:52:07.312 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 2m ago 4h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:52:07.313 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 10s ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:52:07.313 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 10s ago 4h 443M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:52:07.313 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 2m ago 4h 740M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:52:07.313 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 2m ago 4h 994M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:52:07.313 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 10s ago 4h 438M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:52:07.314 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 2m ago 4h 581M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:52:07.314 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 2m ago 4h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:52:07.314 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 10s ago 4h 25.8M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:52:07.315 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 2m ago 4h 717M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:52:07.315 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 2m ago 4h 708M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:52:07.315 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 2m ago 4h 554M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:52:07.315 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 2m ago 4h 689M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:52:07.315 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 10s ago 4h 609M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:52:07.316 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 10s ago 4h 757M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:52:07.316 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 10s ago 4h 675M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:52:07.316 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 10s ago 4h 798M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:52:07.316 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 10s ago 4h 175M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:52:07.666 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:52:07.667 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:52:07.667 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:52:07.667 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:52:07.668 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:52:07.668 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:52:07.668 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:52:07.668 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:52:07.668 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:52:07.669 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:52:07.669 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:52:07.669 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:52:07.669 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:52:07.670 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:52:07.670 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:52:07.670 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:52:07.670 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:52:07.671 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:52:08.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:52:08 smithi083 ceph-mon[79930]: pgmap v7916: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:08.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:52:08 smithi083 ceph-mon[79930]: from='client.43341 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:52:08.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:52:08 smithi083 ceph-mon[79930]: from='client.43347 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:52:08.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:52:08 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2314630166' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:52:08.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:52:08 smithi083 ceph-mon[77242]: pgmap v7916: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:08.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:52:08 smithi083 ceph-mon[77242]: from='client.43341 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:52:08.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:52:08 smithi083 ceph-mon[77242]: from='client.43347 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:52:08.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:52:08 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2314630166' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:52:08.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:52:08 smithi153 ceph-mon[70044]: pgmap v7916: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:52:08 smithi153 ceph-mon[70044]: from='client.43341 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:52:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:52:08 smithi153 ceph-mon[70044]: from='client.43347 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:52:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:52:08 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2314630166' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:52:10.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:52:10 smithi083 ceph-mon[79930]: pgmap v7917: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:10.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:52:10 smithi083 ceph-mon[77242]: pgmap v7917: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:10.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:52:10 smithi153 ceph-mon[70044]: pgmap v7917: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:11.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:52:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:52:11.124Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:52:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:52:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:52:11] "GET /metrics HTTP/1.1" 200 225228 "" "Prometheus/2.33.4" 2022-09-29T09:52:12.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:52:12 smithi083 ceph-mon[79930]: pgmap v7918: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:12.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:52:12 smithi083 ceph-mon[77242]: pgmap v7918: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:52:12 smithi153 ceph-mon[70044]: pgmap v7918: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:12.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:52:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:52:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:52:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:52:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:52:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:52:14.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:52:14 smithi083 ceph-mon[79930]: pgmap v7919: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:14.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:52:14 smithi083 ceph-mon[77242]: pgmap v7919: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:14.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:52:14 smithi153 ceph-mon[70044]: pgmap v7919: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:14.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:52:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:52:14.506Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:52:15.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:52:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:52:15.218Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:52:16.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:52:16 smithi083 ceph-mon[79930]: pgmap v7920: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:16.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:52:16 smithi083 ceph-mon[77242]: pgmap v7920: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:16.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:52:16 smithi153 ceph-mon[70044]: pgmap v7920: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:18.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:52:18 smithi083 ceph-mon[79930]: pgmap v7921: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:18.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:52:18 smithi083 ceph-mon[77242]: pgmap v7921: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:18.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:52:18 smithi153 ceph-mon[70044]: pgmap v7921: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:20.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:52:20 smithi083 ceph-mon[79930]: pgmap v7922: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:20.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:52:20 smithi083 ceph-mon[77242]: pgmap v7922: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:20.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:52:20 smithi153 ceph-mon[70044]: pgmap v7922: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:21.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:52:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:52:21.124Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:52:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:52:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:52:21] "GET /metrics HTTP/1.1" 200 225228 "" "Prometheus/2.33.4" 2022-09-29T09:52:22.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:52:22 smithi083 ceph-mon[79930]: pgmap v7923: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:22.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:52:22 smithi083 ceph-mon[77242]: pgmap v7923: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:52:22 smithi153 ceph-mon[70044]: pgmap v7923: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:22.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:52:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:52:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:52:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:52:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:52:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:52:24.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:52:24 smithi083 ceph-mon[79930]: pgmap v7924: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:24.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:52:24 smithi083 ceph-mon[77242]: pgmap v7924: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:52:24 smithi153 ceph-mon[70044]: pgmap v7924: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:24.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:52:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:52:24.507Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:52:25.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:52:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:52:25.218Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:52:26.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:52:26 smithi083 ceph-mon[79930]: pgmap v7925: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:26.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:52:26 smithi083 ceph-mon[77242]: pgmap v7925: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:26.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:52:26 smithi153 ceph-mon[70044]: pgmap v7925: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:28.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:52:28 smithi083 ceph-mon[79930]: pgmap v7926: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:28.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:52:28 smithi083 ceph-mon[77242]: pgmap v7926: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:28.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:52:28 smithi153 ceph-mon[70044]: pgmap v7926: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:30.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:52:30 smithi083 ceph-mon[79930]: pgmap v7927: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:30.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:52:30 smithi083 ceph-mon[77242]: pgmap v7927: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:30.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:52:30 smithi153 ceph-mon[70044]: pgmap v7927: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:31.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:52:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:52:31.125Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:52:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:52:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:52:31] "GET /metrics HTTP/1.1" 200 225227 "" "Prometheus/2.33.4" 2022-09-29T09:52:32.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:52:32 smithi083 ceph-mon[79930]: pgmap v7928: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:32.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:52:32 smithi083 ceph-mon[77242]: pgmap v7928: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:52:32 smithi153 ceph-mon[70044]: pgmap v7928: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:32.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:52:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:52:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:52:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:52:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:52:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:52:34.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:52:34 smithi083 ceph-mon[79930]: pgmap v7929: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:34.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:52:34 smithi083 ceph-mon[77242]: pgmap v7929: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:34.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:52:34 smithi153 ceph-mon[70044]: pgmap v7929: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:34.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:52:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:52:34.508Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:52:35.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:52:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:52:35.219Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:52:36.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:52:36 smithi083 ceph-mon[77242]: pgmap v7930: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:36.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:52:36 smithi083 ceph-mon[79930]: pgmap v7930: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:52:36 smithi153 ceph-mon[70044]: pgmap v7930: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:37.992 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:52:38.283 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:52:38.283 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 2m ago 4h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:52:38.283 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 41s ago 4h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:52:38.283 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 2m ago 4h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:52:38.284 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 41s ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:52:38.284 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 41s ago 4h 443M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:52:38.284 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 2m ago 4h 740M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:52:38.284 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 2m ago 4h 994M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:52:38.285 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 41s ago 4h 438M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:52:38.285 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 2m ago 4h 581M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:52:38.285 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 2m ago 4h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:52:38.285 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 41s ago 4h 25.8M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:52:38.285 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 2m ago 4h 717M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:52:38.286 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 2m ago 4h 708M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:52:38.286 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 2m ago 4h 554M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:52:38.286 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 2m ago 4h 689M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:52:38.286 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 41s ago 4h 609M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:52:38.287 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 41s ago 4h 757M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:52:38.287 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 41s ago 4h 675M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:52:38.287 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 41s ago 4h 798M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:52:38.287 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 41s ago 4h 175M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:52:38.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:52:38 smithi083 ceph-mon[79930]: pgmap v7931: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:38.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:52:38 smithi083 ceph-mon[77242]: pgmap v7931: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:38.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:52:38 smithi153 ceph-mon[70044]: pgmap v7931: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:38.640 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:52:38.640 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:52:38.640 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:52:38.640 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:52:38.641 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:52:38.641 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:52:38.641 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:52:38.641 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:52:38.642 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:52:38.642 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:52:38.642 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:52:38.642 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:52:38.642 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:52:38.643 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:52:38.643 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:52:38.643 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:52:38.643 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:52:38.644 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:52:39.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:52:39 smithi083 ceph-mon[77242]: from='client.43359 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:52:39.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:52:39 smithi083 ceph-mon[77242]: from='client.43365 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:52:39.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:52:39 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/817767174' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:52:39.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:52:39 smithi083 ceph-mon[79930]: from='client.43359 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:52:39.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:52:39 smithi083 ceph-mon[79930]: from='client.43365 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:52:39.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:52:39 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/817767174' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:52:39.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:52:39 smithi153 ceph-mon[70044]: from='client.43359 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:52:39.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:52:39 smithi153 ceph-mon[70044]: from='client.43365 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:52:39.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:52:39 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/817767174' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:52:40.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:52:40 smithi083 ceph-mon[77242]: pgmap v7932: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:40.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:52:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:52:40.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:52:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:52:40.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:52:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:52:40.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:52:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:52:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:52:40 smithi083 ceph-mon[79930]: pgmap v7932: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:52:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:52:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:52:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:52:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:52:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:52:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:52:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:52:40.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:52:40 smithi153 ceph-mon[70044]: pgmap v7932: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:52:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:52:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:52:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:52:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:52:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:52:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:52:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:52:41.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:52:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:52:41.125Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:52:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:52:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:52:41] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T09:52:42.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:52:42 smithi083 ceph-mon[79930]: pgmap v7933: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:42.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:52:42 smithi083 ceph-mon[77242]: pgmap v7933: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:52:42 smithi153 ceph-mon[70044]: pgmap v7933: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:42.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:52:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:52:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:52:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:52:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:52:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:52:44.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:52:44 smithi083 ceph-mon[79930]: pgmap v7934: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:44.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:52:44 smithi083 ceph-mon[77242]: pgmap v7934: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:44.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:52:44 smithi153 ceph-mon[70044]: pgmap v7934: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:44.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:52:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:52:44.509Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:52:45.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:52:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:52:45.218Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:52:46.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:52:46 smithi153 ceph-mon[70044]: pgmap v7935: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:46.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:52:46 smithi083 ceph-mon[79930]: pgmap v7935: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:46.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:52:46 smithi083 ceph-mon[77242]: pgmap v7935: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:48.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:52:48 smithi153 ceph-mon[70044]: pgmap v7936: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:48.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:52:48 smithi083 ceph-mon[79930]: pgmap v7936: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:48.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:52:48 smithi083 ceph-mon[77242]: pgmap v7936: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:52:50 smithi153 ceph-mon[70044]: pgmap v7937: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:50.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:52:50 smithi083 ceph-mon[79930]: pgmap v7937: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:50.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:52:50 smithi083 ceph-mon[77242]: pgmap v7937: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:51.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:52:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:52:51.126Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:52:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:52:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:52:51] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T09:52:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:52:52 smithi153 ceph-mon[70044]: pgmap v7938: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:52.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:52:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:52:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:52:52.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:52:52 smithi083 ceph-mon[79930]: pgmap v7938: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:52.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:52:52 smithi083 ceph-mon[77242]: pgmap v7938: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:52:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:52:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:52:54.510 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:52:54 smithi083 ceph-mon[79930]: pgmap v7939: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:54.511 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:52:54 smithi083 ceph-mon[77242]: pgmap v7939: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:54.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:52:54 smithi153 ceph-mon[70044]: pgmap v7939: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:54.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:52:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:52:54.509Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:52:55.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:52:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:52:55.219Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:52:56.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:52:56 smithi153 ceph-mon[70044]: pgmap v7940: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:56.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:52:56 smithi083 ceph-mon[79930]: pgmap v7940: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:56.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:52:56 smithi083 ceph-mon[77242]: pgmap v7940: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:58.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:52:58 smithi153 ceph-mon[70044]: pgmap v7941: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:52:58 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:52:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:52:58 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:52:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:52:58 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:52:58.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:52:58 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:52:58.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:52:58 smithi083 ceph-mon[77242]: pgmap v7941: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:58.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:52:58 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:52:58.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:52:58 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:52:58.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:52:58 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:52:58.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:52:58 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:52:58.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:52:58 smithi083 ceph-mon[79930]: pgmap v7941: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:52:58.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:52:58 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:52:58.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:52:58 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:52:58.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:52:58 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:52:58.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:52:58 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:53:00.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:53:00 smithi153 ceph-mon[70044]: pgmap v7942: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:00.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:53:00 smithi083 ceph-mon[77242]: pgmap v7942: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:00.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:53:00 smithi083 ceph-mon[79930]: pgmap v7942: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:01.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:53:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:53:01.126Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:53:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:53:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:53:01] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T09:53:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:53:02 smithi153 ceph-mon[70044]: pgmap v7943: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:02.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:53:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:53:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:53:02.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:53:02 smithi083 ceph-mon[79930]: pgmap v7943: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:02.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:53:02 smithi083 ceph-mon[77242]: pgmap v7943: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:53:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:53:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:53:04.511 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:53:04 smithi083 ceph-mon[79930]: pgmap v7944: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:04.511 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:53:04 smithi083 ceph-mon[77242]: pgmap v7944: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:04.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:53:04 smithi153 ceph-mon[70044]: pgmap v7944: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:04.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:53:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:53:04.509Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:53:05.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:53:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:53:05.221Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:53:06.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:53:06 smithi153 ceph-mon[70044]: pgmap v7945: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:06.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:53:06 smithi083 ceph-mon[77242]: pgmap v7945: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:06.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:53:06 smithi083 ceph-mon[79930]: pgmap v7945: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:08.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:53:08 smithi153 ceph-mon[70044]: pgmap v7946: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:08.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:53:08 smithi083 ceph-mon[79930]: pgmap v7946: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:08.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:53:08 smithi083 ceph-mon[77242]: pgmap v7946: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:08.964 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:53:09.260 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:53:09.261 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 3m ago 4h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:53:09.261 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 72s ago 4h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:53:09.261 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 3m ago 4h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:53:09.261 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 72s ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:53:09.262 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 72s ago 4h 443M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:53:09.262 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 3m ago 4h 740M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:53:09.262 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 3m ago 4h 994M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:53:09.262 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 72s ago 4h 438M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:53:09.263 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 3m ago 4h 581M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:53:09.263 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 3m ago 4h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:53:09.263 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 72s ago 4h 25.8M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:53:09.263 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 3m ago 4h 717M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:53:09.263 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 3m ago 4h 708M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:53:09.264 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 3m ago 4h 554M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:53:09.264 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 3m ago 4h 689M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:53:09.264 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 72s ago 4h 609M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:53:09.264 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 72s ago 4h 757M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:53:09.264 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 72s ago 4h 675M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:53:09.265 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 72s ago 4h 798M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:53:09.265 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 72s ago 4h 175M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:53:09.618 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:53:09.618 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:53:09.618 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:53:09.619 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:53:09.619 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:53:09.619 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:53:09.619 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:53:09.619 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:53:09.620 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:53:09.620 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:53:09.620 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:53:09.620 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:53:09.620 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:53:09.621 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:53:09.621 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:53:09.621 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:53:09.621 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:53:09.622 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:53:10.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:53:10 smithi153 ceph-mon[70044]: pgmap v7947: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:53:10 smithi153 ceph-mon[70044]: from='client.43377 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:53:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:53:10 smithi153 ceph-mon[70044]: from='client.43383 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:53:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:53:10 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/107773167' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:53:10.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:53:10 smithi083 ceph-mon[77242]: pgmap v7947: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:10.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:53:10 smithi083 ceph-mon[77242]: from='client.43377 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:53:10.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:53:10 smithi083 ceph-mon[77242]: from='client.43383 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:53:10.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:53:10 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/107773167' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:53:10.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:53:10 smithi083 ceph-mon[79930]: pgmap v7947: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:10.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:53:10 smithi083 ceph-mon[79930]: from='client.43377 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:53:10.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:53:10 smithi083 ceph-mon[79930]: from='client.43383 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:53:10.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:53:10 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/107773167' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:53:11.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:53:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:53:11.127Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:53:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:53:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:53:11] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T09:53:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:53:12 smithi153 ceph-mon[70044]: pgmap v7948: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:12.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:53:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:53:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:53:12.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:53:12 smithi083 ceph-mon[79930]: pgmap v7948: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:12.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:53:12 smithi083 ceph-mon[77242]: pgmap v7948: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:53:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:53:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:53:14.512 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:53:14 smithi083 ceph-mon[79930]: pgmap v7949: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:14.512 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:53:14 smithi083 ceph-mon[77242]: pgmap v7949: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:14.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:53:14 smithi153 ceph-mon[70044]: pgmap v7949: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:14.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:53:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:53:14.511Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:53:15.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:53:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:53:15.221Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:53:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:53:16 smithi153 ceph-mon[70044]: pgmap v7950: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:16.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:53:16 smithi083 ceph-mon[77242]: pgmap v7950: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:16.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:53:16 smithi083 ceph-mon[79930]: pgmap v7950: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:18.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:53:18 smithi153 ceph-mon[70044]: pgmap v7951: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:18.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:53:18 smithi083 ceph-mon[79930]: pgmap v7951: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:18.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:53:18 smithi083 ceph-mon[77242]: pgmap v7951: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:20.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:53:20 smithi153 ceph-mon[70044]: pgmap v7952: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:20.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:53:20 smithi083 ceph-mon[79930]: pgmap v7952: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:20.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:53:20 smithi083 ceph-mon[77242]: pgmap v7952: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:21.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:53:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:53:21.128Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:53:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:53:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:53:21] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T09:53:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:53:22 smithi153 ceph-mon[70044]: pgmap v7953: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:22.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:53:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:53:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:53:22.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:53:22 smithi083 ceph-mon[77242]: pgmap v7953: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:22.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:53:22 smithi083 ceph-mon[79930]: pgmap v7953: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:53:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:53:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:53:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:53:24 smithi153 ceph-mon[70044]: pgmap v7954: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:24.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:53:24 smithi083 ceph-mon[79930]: pgmap v7954: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:24.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:53:24 smithi083 ceph-mon[77242]: pgmap v7954: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:53:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:53:24.511Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:53:25.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:53:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:53:25.221Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:53:26.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:53:26 smithi153 ceph-mon[70044]: pgmap v7955: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:26.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:53:26 smithi083 ceph-mon[79930]: pgmap v7955: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:26.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:53:26 smithi083 ceph-mon[77242]: pgmap v7955: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:28.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:53:28 smithi153 ceph-mon[70044]: pgmap v7956: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:28.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:53:28 smithi083 ceph-mon[79930]: pgmap v7956: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:28.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:53:28 smithi083 ceph-mon[77242]: pgmap v7956: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:53:30 smithi153 ceph-mon[70044]: pgmap v7957: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:30.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:53:30 smithi083 ceph-mon[79930]: pgmap v7957: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:30.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:53:30 smithi083 ceph-mon[77242]: pgmap v7957: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:31.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:53:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:53:31.128Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:53:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:53:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:53:31] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T09:53:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:53:32 smithi153 ceph-mon[70044]: pgmap v7958: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:32.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:53:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:53:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:53:32.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:53:32 smithi083 ceph-mon[79930]: pgmap v7958: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:32.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:53:32 smithi083 ceph-mon[77242]: pgmap v7958: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:53:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:53:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:53:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:53:34 smithi153 ceph-mon[70044]: pgmap v7959: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:34.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:53:34 smithi083 ceph-mon[79930]: pgmap v7959: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:34.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:53:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:53:34.512Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:53:34.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:53:34 smithi083 ceph-mon[77242]: pgmap v7959: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:35.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:53:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:53:35.222Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:53:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:53:36 smithi153 ceph-mon[70044]: pgmap v7960: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:36.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:53:36 smithi083 ceph-mon[77242]: pgmap v7960: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:36.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:53:36 smithi083 ceph-mon[79930]: pgmap v7960: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:38.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:53:38 smithi153 ceph-mon[70044]: pgmap v7961: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:38.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:53:38 smithi083 ceph-mon[77242]: pgmap v7961: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:38.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:53:38 smithi083 ceph-mon[79930]: pgmap v7961: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:39.940 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:53:40.229 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:53:40.230 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 3m ago 4h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:53:40.230 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 103s ago 4h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:53:40.230 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 3m ago 4h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:53:40.231 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 103s ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:53:40.231 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 103s ago 4h 443M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:53:40.231 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 3m ago 4h 740M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:53:40.231 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 3m ago 4h 994M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:53:40.232 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 103s ago 4h 438M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:53:40.232 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 3m ago 4h 581M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:53:40.232 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 3m ago 4h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:53:40.232 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 103s ago 4h 25.8M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:53:40.232 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 3m ago 4h 717M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:53:40.233 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 3m ago 4h 708M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:53:40.233 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 3m ago 4h 554M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:53:40.233 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 3m ago 4h 689M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:53:40.233 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 103s ago 4h 609M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:53:40.234 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 103s ago 4h 757M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:53:40.234 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 103s ago 4h 675M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:53:40.234 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 103s ago 4h 798M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:53:40.234 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 103s ago 4h 175M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:53:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:53:40 smithi153 ceph-mon[70044]: pgmap v7962: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:53:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:53:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:53:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:53:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:53:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:53:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:53:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:53:40.589 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:53:40.590 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:53:40.590 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:53:40.590 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:53:40.590 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:53:40.591 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:53:40.591 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:53:40.591 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:53:40.591 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:53:40.592 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:53:40.592 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:53:40.592 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:53:40.592 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:53:40.592 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:53:40.593 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:53:40.593 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:53:40.593 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:53:40.593 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:53:40.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:53:40 smithi083 ceph-mon[77242]: pgmap v7962: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:40.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:53:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:53:40.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:53:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:53:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:53:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:53:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:53:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:53:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:53:40 smithi083 ceph-mon[79930]: pgmap v7962: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:53:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:53:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:53:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:53:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:53:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:53:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:53:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:53:41.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:53:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:53:41.128Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:53:41.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:53:41 smithi083 ceph-mon[77242]: from='client.53326 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:53:41.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:53:41 smithi083 ceph-mon[77242]: from='client.53332 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:53:41.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:53:41 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3176167736' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:53:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:53:41 smithi083 ceph-mon[79930]: from='client.53326 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:53:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:53:41 smithi083 ceph-mon[79930]: from='client.53332 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:53:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:53:41 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3176167736' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:53:41.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:53:41 smithi153 ceph-mon[70044]: from='client.53326 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:53:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:53:41 smithi153 ceph-mon[70044]: from='client.53332 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:53:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:53:41 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3176167736' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:53:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:53:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:53:41] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T09:53:42.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:53:42 smithi083 ceph-mon[77242]: pgmap v7963: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:53:42 smithi083 ceph-mon[79930]: pgmap v7963: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:53:42 smithi153 ceph-mon[70044]: pgmap v7963: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:53:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:53:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:53:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:53:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:53:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:53:44.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:53:44 smithi083 ceph-mon[79930]: pgmap v7964: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:44.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:53:44 smithi083 ceph-mon[77242]: pgmap v7964: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:53:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:53:44.512Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:53:44.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:53:44 smithi153 ceph-mon[70044]: pgmap v7964: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:45.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:53:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:53:45.222Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:53:46.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:53:46 smithi083 ceph-mon[79930]: pgmap v7965: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:46.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:53:46 smithi083 ceph-mon[77242]: pgmap v7965: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:46.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:53:46 smithi153 ceph-mon[70044]: pgmap v7965: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:48.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:53:48 smithi083 ceph-mon[79930]: pgmap v7966: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:48.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:53:48 smithi083 ceph-mon[77242]: pgmap v7966: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:48.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:53:48 smithi153 ceph-mon[70044]: pgmap v7966: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:50.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:53:50 smithi083 ceph-mon[79930]: pgmap v7967: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:50.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:53:50 smithi083 ceph-mon[77242]: pgmap v7967: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:50.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:53:50 smithi153 ceph-mon[70044]: pgmap v7967: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:51.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:53:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:53:51.129Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:53:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:53:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:53:51] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T09:53:52.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:53:52 smithi083 ceph-mon[77242]: pgmap v7968: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:52.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:53:52 smithi083 ceph-mon[79930]: pgmap v7968: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:52.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:53:52 smithi153 ceph-mon[70044]: pgmap v7968: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:53:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:53:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:53:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:53:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:53:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:53:54.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:53:54 smithi083 ceph-mon[79930]: pgmap v7969: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:53:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:53:54.513Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:53:54.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:53:54 smithi083 ceph-mon[77242]: pgmap v7969: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:54.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:53:54 smithi153 ceph-mon[70044]: pgmap v7969: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:55.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:53:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:53:55.223Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:53:56.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:53:56 smithi083 ceph-mon[79930]: pgmap v7970: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:56.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:53:56 smithi083 ceph-mon[77242]: pgmap v7970: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:56.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:53:56 smithi153 ceph-mon[70044]: pgmap v7970: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:58.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:53:58 smithi083 ceph-mon[79930]: pgmap v7971: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:58.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:53:58 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:53:58.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:53:58 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:53:58.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:53:58 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:53:58.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:53:58 smithi083 ceph-mon[77242]: pgmap v7971: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:58.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:53:58 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:53:58.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:53:58 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:53:58.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:53:58 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:53:58.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:53:58 smithi153 ceph-mon[70044]: pgmap v7971: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:53:58.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:53:58 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:53:58.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:53:58 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:53:58.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:53:58 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:54:00.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:53:59 smithi153 ceph-mon[70044]: pgmap v7972: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:00.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:53:59 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:54:00.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:53:59 smithi083 ceph-mon[77242]: pgmap v7972: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:00.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:53:59 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:54:00.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:53:59 smithi083 ceph-mon[79930]: pgmap v7972: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:00.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:53:59 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:54:01.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:54:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:54:01.130Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:54:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:54:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:54:01] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T09:54:01.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:54:01 smithi083 ceph-mon[79930]: pgmap v7973: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:01.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:54:01 smithi083 ceph-mon[77242]: pgmap v7973: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:02.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:54:01 smithi153 ceph-mon[70044]: pgmap v7973: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:54:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:54:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:54:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:54:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:54:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:54:04.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:54:04 smithi083 ceph-mon[79930]: pgmap v7974: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:04.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:54:04 smithi083 ceph-mon[77242]: pgmap v7974: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:04.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:54:04 smithi153 ceph-mon[70044]: pgmap v7974: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:04.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:54:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:54:04.513Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:54:05.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:54:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:54:05.223Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:54:06.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:54:06 smithi083 ceph-mon[79930]: pgmap v7975: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:06.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:54:06 smithi083 ceph-mon[77242]: pgmap v7975: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:06.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:54:06 smithi153 ceph-mon[70044]: pgmap v7975: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:08.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:54:08 smithi083 ceph-mon[79930]: pgmap v7976: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:08.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:54:08 smithi083 ceph-mon[77242]: pgmap v7976: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:08.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:54:08 smithi153 ceph-mon[70044]: pgmap v7976: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:10.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:54:10 smithi083 ceph-mon[79930]: pgmap v7977: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:10.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:54:10 smithi083 ceph-mon[77242]: pgmap v7977: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:54:10 smithi153 ceph-mon[70044]: pgmap v7977: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:10.917 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:54:11.207 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:54:11.208 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 4m ago 4h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:54:11.208 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 2m ago 4h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:54:11.208 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 4m ago 4h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:54:11.209 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 2m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:54:11.209 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 2m ago 4h 443M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:54:11.209 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 4m ago 4h 740M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:54:11.209 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 4m ago 4h 994M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:54:11.209 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 2m ago 4h 438M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:54:11.210 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 4m ago 4h 581M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:54:11.210 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 4m ago 4h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:54:11.210 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 2m ago 4h 25.8M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:54:11.210 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 4m ago 4h 717M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:54:11.211 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 4m ago 4h 708M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:54:11.211 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 4m ago 4h 554M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:54:11.211 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 4m ago 4h 689M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:54:11.211 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 2m ago 4h 609M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:54:11.212 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 2m ago 4h 757M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:54:11.212 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 2m ago 4h 675M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:54:11.212 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 2m ago 4h 798M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:54:11.212 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 2m ago 4h 175M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:54:11.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:54:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:54:11.130Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:54:11.566 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:54:11.567 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:54:11.567 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:54:11.567 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:54:11.567 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:54:11.568 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:54:11.568 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:54:11.568 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:54:11.568 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:54:11.569 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:54:11.569 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:54:11.569 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:54:11.569 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:54:11.570 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:54:11.570 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:54:11.570 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:54:11.570 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:54:11.571 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:54:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:54:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:54:11] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T09:54:12.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:54:12 smithi083 ceph-mon[79930]: pgmap v7978: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:12.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:54:12 smithi083 ceph-mon[79930]: from='client.53344 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:54:12.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:54:12 smithi083 ceph-mon[79930]: from='client.53350 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:54:12.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:54:12 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/813351810' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:54:12.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:54:12 smithi083 ceph-mon[77242]: pgmap v7978: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:12.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:54:12 smithi083 ceph-mon[77242]: from='client.53344 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:54:12.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:54:12 smithi083 ceph-mon[77242]: from='client.53350 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:54:12.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:54:12 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/813351810' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:54:12.556 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:54:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:54:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:54:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:54:12 smithi153 ceph-mon[70044]: pgmap v7978: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:54:12 smithi153 ceph-mon[70044]: from='client.53344 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:54:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:54:12 smithi153 ceph-mon[70044]: from='client.53350 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:54:12.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:54:12 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/813351810' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:54:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:54:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:54:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:54:14.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:54:14 smithi083 ceph-mon[77242]: pgmap v7979: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:14.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:54:14 smithi083 ceph-mon[79930]: pgmap v7979: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:14.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:54:14 smithi153 ceph-mon[70044]: pgmap v7979: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:14.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:54:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:54:14.513Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:54:15.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:54:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:54:15.224Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:54:16.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:54:16 smithi083 ceph-mon[79930]: pgmap v7980: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:16.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:54:16 smithi083 ceph-mon[77242]: pgmap v7980: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:16.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:54:16 smithi153 ceph-mon[70044]: pgmap v7980: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:18.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:54:18 smithi083 ceph-mon[77242]: pgmap v7981: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:18.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:54:18 smithi083 ceph-mon[79930]: pgmap v7981: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:18.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:54:18 smithi153 ceph-mon[70044]: pgmap v7981: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:20.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:54:20 smithi083 ceph-mon[79930]: pgmap v7982: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:20.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:54:20 smithi083 ceph-mon[77242]: pgmap v7982: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:20.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:54:20 smithi153 ceph-mon[70044]: pgmap v7982: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:21.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:54:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:54:21.131Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:54:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:54:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:54:21] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T09:54:22.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:54:22 smithi083 ceph-mon[79930]: pgmap v7983: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:22.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:54:22 smithi083 ceph-mon[77242]: pgmap v7983: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:22.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:54:22 smithi153 ceph-mon[70044]: pgmap v7983: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:22.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:54:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:54:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:54:23.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:54:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:54:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:54:24.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:54:24 smithi083 ceph-mon[77242]: pgmap v7984: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:24.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:54:24 smithi083 ceph-mon[79930]: pgmap v7984: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:24.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:54:24 smithi153 ceph-mon[70044]: pgmap v7984: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:24.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:54:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:54:24.515Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:54:25.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:54:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:54:25.225Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:54:26.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:54:26 smithi083 ceph-mon[77242]: pgmap v7985: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:26.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:54:26 smithi083 ceph-mon[79930]: pgmap v7985: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:26.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:54:26 smithi153 ceph-mon[70044]: pgmap v7985: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:28.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:54:28 smithi083 ceph-mon[79930]: pgmap v7986: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:28.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:54:28 smithi083 ceph-mon[77242]: pgmap v7986: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:54:28 smithi153 ceph-mon[70044]: pgmap v7986: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:30.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:54:30 smithi083 ceph-mon[79930]: pgmap v7987: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:30.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:54:30 smithi083 ceph-mon[77242]: pgmap v7987: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:30.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:54:30 smithi153 ceph-mon[70044]: pgmap v7987: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:31.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:54:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:54:31.132Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:54:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:54:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:54:31] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T09:54:32.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:54:32 smithi083 ceph-mon[79930]: pgmap v7988: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:32.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:54:32 smithi083 ceph-mon[77242]: pgmap v7988: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:54:32 smithi153 ceph-mon[70044]: pgmap v7988: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:32.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:54:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:54:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:54:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:54:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:54:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:54:34.516 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:54:34 smithi083 ceph-mon[79930]: pgmap v7989: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:34.516 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:54:34 smithi083 ceph-mon[77242]: pgmap v7989: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:34.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:54:34 smithi153 ceph-mon[70044]: pgmap v7989: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:34.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:54:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:54:34.515Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:54:35.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:54:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:54:35.226Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:54:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:54:36 smithi153 ceph-mon[70044]: pgmap v7990: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:36.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:54:36 smithi083 ceph-mon[79930]: pgmap v7990: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:36.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:54:36 smithi083 ceph-mon[77242]: pgmap v7990: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:54:38 smithi153 ceph-mon[70044]: pgmap v7991: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:38.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:54:38 smithi083 ceph-mon[77242]: pgmap v7991: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:38.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:54:38 smithi083 ceph-mon[79930]: pgmap v7991: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:54:40 smithi153 ceph-mon[70044]: pgmap v7992: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:54:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:54:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:54:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:54:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:54:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:54:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:54:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:54:40.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:54:40 smithi083 ceph-mon[79930]: pgmap v7992: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:40.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:54:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:54:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:54:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:54:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:54:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:54:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:54:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:54:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:54:40 smithi083 ceph-mon[77242]: pgmap v7992: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:54:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:54:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:54:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:54:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:54:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:54:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:54:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:54:41.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:54:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:54:41.133Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:54:41.889 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:54:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:54:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:54:41] "GET /metrics HTTP/1.1" 200 225227 "" "Prometheus/2.33.4" 2022-09-29T09:54:42.182 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:54:42.182 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 4m ago 4h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:54:42.182 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 2m ago 4h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:54:42.183 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 4m ago 4h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:54:42.183 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 2m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:54:42.183 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 2m ago 4h 443M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:54:42.183 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 4m ago 4h 740M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:54:42.183 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 4m ago 4h 994M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:54:42.184 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 2m ago 4h 438M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:54:42.184 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 4m ago 4h 581M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:54:42.184 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 4m ago 4h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:54:42.184 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 2m ago 4h 25.8M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:54:42.184 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 4m ago 4h 717M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:54:42.185 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 4m ago 4h 708M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:54:42.185 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 4m ago 4h 554M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:54:42.185 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 4m ago 4h 689M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:54:42.185 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 2m ago 4h 609M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:54:42.186 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 2m ago 4h 757M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:54:42.186 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 2m ago 4h 675M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:54:42.186 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 2m ago 4h 798M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:54:42.186 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 2m ago 4h 175M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:54:42.539 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:54:42.539 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:54:42.539 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:54:42.540 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:54:42.540 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:54:42.540 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:54:42.540 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:54:42.541 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:54:42.541 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:54:42.541 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:54:42.541 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:54:42.541 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:54:42.542 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:54:42.542 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:54:42.542 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:54:42.542 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:54:42.543 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:54:42.543 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:54:42.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:54:42 smithi153 ceph-mon[70044]: pgmap v7993: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:42.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:54:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:54:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:54:42.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:54:42 smithi083 ceph-mon[77242]: pgmap v7993: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:42.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:54:42 smithi083 ceph-mon[79930]: pgmap v7993: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:54:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:54:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:54:43.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:54:43 smithi153 ceph-mon[70044]: from='client.53362 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:54:43.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:54:43 smithi153 ceph-mon[70044]: from='client.43437 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:54:43.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:54:43 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/940556269' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:54:43.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:54:43 smithi083 ceph-mon[79930]: from='client.53362 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:54:43.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:54:43 smithi083 ceph-mon[79930]: from='client.43437 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:54:43.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:54:43 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/940556269' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:54:43.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:54:43 smithi083 ceph-mon[77242]: from='client.53362 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:54:43.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:54:43 smithi083 ceph-mon[77242]: from='client.43437 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:54:43.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:54:43 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/940556269' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:54:44.517 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:54:44 smithi083 ceph-mon[79930]: pgmap v7994: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:44.518 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:54:44 smithi083 ceph-mon[77242]: pgmap v7994: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:44.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:54:44 smithi153 ceph-mon[70044]: pgmap v7994: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:44.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:54:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:54:44.516Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:54:45.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:54:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:54:45.226Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:54:46.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:54:46 smithi153 ceph-mon[70044]: pgmap v7995: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:46.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:54:46 smithi083 ceph-mon[77242]: pgmap v7995: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:46.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:54:46 smithi083 ceph-mon[79930]: pgmap v7995: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:48.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:54:48 smithi153 ceph-mon[70044]: pgmap v7996: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:48.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:54:48 smithi083 ceph-mon[79930]: pgmap v7996: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:48.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:54:48 smithi083 ceph-mon[77242]: pgmap v7996: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:50.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:54:50 smithi153 ceph-mon[70044]: pgmap v7997: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:50.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:54:50 smithi083 ceph-mon[79930]: pgmap v7997: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:50.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:54:50 smithi083 ceph-mon[77242]: pgmap v7997: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:51.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:54:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:54:51.133Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:54:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:54:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:54:51] "GET /metrics HTTP/1.1" 200 225227 "" "Prometheus/2.33.4" 2022-09-29T09:54:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:54:52 smithi153 ceph-mon[70044]: pgmap v7998: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:52.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:54:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:54:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:54:52.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:54:52 smithi083 ceph-mon[79930]: pgmap v7998: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:52.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:54:52 smithi083 ceph-mon[77242]: pgmap v7998: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:54:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:54:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:54:54.518 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:54:54 smithi083 ceph-mon[79930]: pgmap v7999: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:54.518 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:54:54 smithi083 ceph-mon[77242]: pgmap v7999: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:54.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:54:54 smithi153 ceph-mon[70044]: pgmap v7999: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:54.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:54:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:54:54.517Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:54:55.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:54:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:54:55.227Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:54:56.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:54:56 smithi153 ceph-mon[70044]: pgmap v8000: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:56.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:54:56 smithi083 ceph-mon[79930]: pgmap v8000: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:56.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:54:56 smithi083 ceph-mon[77242]: pgmap v8000: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:54:58 smithi153 ceph-mon[70044]: pgmap v8001: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:58.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:54:58 smithi083 ceph-mon[79930]: pgmap v8001: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:58.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:54:58 smithi083 ceph-mon[77242]: pgmap v8001: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:54:59.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:54:59 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:54:59.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:54:59 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:54:59.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:54:59 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:54:59.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:54:59 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:54:59.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:54:59 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:54:59.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:54:59 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:54:59.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:54:59 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:54:59.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:54:59 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:54:59.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:54:59 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:55:00.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:55:00 smithi153 ceph-mon[70044]: pgmap v8002: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:55:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:55:00 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:55:00.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:55:00 smithi083 ceph-mon[77242]: pgmap v8002: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:55:00.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:55:00 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:55:00.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:55:00 smithi083 ceph-mon[79930]: pgmap v8002: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:55:00.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:55:00 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:55:01.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:55:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:55:01.135Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:55:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:55:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:55:01] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T09:55:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:55:02 smithi153 ceph-mon[70044]: pgmap v8003: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:55:02.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:55:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:55:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:55:02.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:55:02 smithi083 ceph-mon[79930]: pgmap v8003: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:55:02.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:55:02 smithi083 ceph-mon[77242]: pgmap v8003: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:55:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:55:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:55:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:55:04.519 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:55:04 smithi083 ceph-mon[79930]: pgmap v8004: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:55:04.519 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:55:04 smithi083 ceph-mon[77242]: pgmap v8004: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:55:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:55:04 smithi153 ceph-mon[70044]: pgmap v8004: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:55:04.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:55:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:55:04.518Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:55:05.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:55:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:55:05.227Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:55:06.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:55:06 smithi153 ceph-mon[70044]: pgmap v8005: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:55:06.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:55:06 smithi083 ceph-mon[79930]: pgmap v8005: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:55:06.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:55:06 smithi083 ceph-mon[77242]: pgmap v8005: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:55:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:55:08 smithi153 ceph-mon[70044]: pgmap v8006: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:55:08.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:55:08 smithi083 ceph-mon[79930]: pgmap v8006: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:55:08.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:55:08 smithi083 ceph-mon[77242]: pgmap v8006: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:55:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:55:10 smithi153 ceph-mon[70044]: pgmap v8007: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:55:10.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:55:10 smithi083 ceph-mon[79930]: pgmap v8007: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:55:10.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:55:10 smithi083 ceph-mon[77242]: pgmap v8007: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:55:11.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:55:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:55:11.135Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:55:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:55:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:55:11] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T09:55:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:55:12 smithi153 ceph-mon[70044]: pgmap v8008: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:55:12.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:55:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:55:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:55:12.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:55:12 smithi083 ceph-mon[79930]: pgmap v8008: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:55:12.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:55:12 smithi083 ceph-mon[77242]: pgmap v8008: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:55:12.864 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:55:13.152 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:55:13.152 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 5m ago 4h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:55:13.153 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 3m ago 4h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:55:13.153 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 5m ago 4h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:55:13.153 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 3m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:55:13.153 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 3m ago 4h 443M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:55:13.154 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 5m ago 4h 740M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:55:13.154 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 5m ago 4h 994M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:55:13.154 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 3m ago 4h 438M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:55:13.155 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 5m ago 4h 581M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:55:13.155 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 5m ago 4h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:55:13.155 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 3m ago 4h 25.8M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:55:13.155 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 5m ago 4h 717M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:55:13.156 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 5m ago 4h 708M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:55:13.156 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 5m ago 4h 554M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:55:13.156 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 5m ago 4h 689M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:55:13.156 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 3m ago 4h 609M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:55:13.157 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 3m ago 4h 757M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:55:13.157 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 3m ago 4h 675M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:55:13.157 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 3m ago 4h 798M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:55:13.157 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 3m ago 4h 175M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:55:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:55:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:55:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:55:13.520 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:55:13.520 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:55:13.520 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:55:13.521 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:55:13.521 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:55:13.521 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:55:13.521 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:55:13.522 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:55:13.522 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:55:13.522 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:55:13.522 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:55:13.523 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:55:13.523 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:55:13.523 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:55:13.523 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:55:13.523 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:55:13.524 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:55:13.524 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:55:14.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:55:14 smithi153 ceph-mon[70044]: pgmap v8009: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:55:14.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:55:14 smithi153 ceph-mon[70044]: from='client.53380 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:55:14.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:55:14 smithi153 ceph-mon[70044]: from='client.53386 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:55:14.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:55:14 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1689407927' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:55:14.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:55:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:55:14.518Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:55:14.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:55:14 smithi083 ceph-mon[77242]: pgmap v8009: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:55:14.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:55:14 smithi083 ceph-mon[77242]: from='client.53380 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:55:14.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:55:14 smithi083 ceph-mon[77242]: from='client.53386 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:55:14.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:55:14 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1689407927' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:55:14.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:55:14 smithi083 ceph-mon[79930]: pgmap v8009: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:55:14.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:55:14 smithi083 ceph-mon[79930]: from='client.53380 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:55:14.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:55:14 smithi083 ceph-mon[79930]: from='client.53386 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:55:14.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:55:14 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1689407927' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:55:15.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:55:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:55:15.228Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:55:16.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:55:16 smithi153 ceph-mon[70044]: pgmap v8010: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:55:16.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:55:16 smithi083 ceph-mon[79930]: pgmap v8010: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:55:16.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:55:16 smithi083 ceph-mon[77242]: pgmap v8010: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:55:18.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:55:18 smithi153 ceph-mon[70044]: pgmap v8011: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:55:18.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:55:18 smithi083 ceph-mon[79930]: pgmap v8011: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:55:18.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:55:18 smithi083 ceph-mon[77242]: pgmap v8011: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:55:20.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:55:20 smithi153 ceph-mon[70044]: pgmap v8012: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:55:20.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:55:20 smithi083 ceph-mon[79930]: pgmap v8012: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:55:20.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:55:20 smithi083 ceph-mon[77242]: pgmap v8012: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:55:21.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:55:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:55:21.136Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:55:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:55:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:55:21] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T09:55:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:55:22 smithi153 ceph-mon[70044]: pgmap v8013: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:55:22.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:55:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:55:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:55:22.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:55:22 smithi083 ceph-mon[79930]: pgmap v8013: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:55:22.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:55:22 smithi083 ceph-mon[77242]: pgmap v8013: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:55:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:55:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:55:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:55:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:55:24 smithi153 ceph-mon[70044]: pgmap v8014: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:55:24.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:55:24 smithi083 ceph-mon[79930]: pgmap v8014: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:55:24.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:55:24 smithi083 ceph-mon[77242]: pgmap v8014: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:55:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:55:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:55:24.519Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:55:25.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:55:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:55:25.228Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:55:26.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:55:26 smithi153 ceph-mon[70044]: pgmap v8015: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:55:26.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:55:26 smithi083 ceph-mon[79930]: pgmap v8015: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:55:26.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:55:26 smithi083 ceph-mon[77242]: pgmap v8015: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:55:28.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:55:28 smithi153 ceph-mon[70044]: pgmap v8016: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:55:28.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:55:28 smithi083 ceph-mon[79930]: pgmap v8016: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:55:28.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:55:28 smithi083 ceph-mon[77242]: pgmap v8016: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:55:30.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:55:30 smithi153 ceph-mon[70044]: pgmap v8017: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:55:30.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:55:30 smithi083 ceph-mon[77242]: pgmap v8017: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:55:30.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:55:30 smithi083 ceph-mon[79930]: pgmap v8017: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:55:31.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:55:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:55:31.137Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:55:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:55:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:55:31] "GET /metrics HTTP/1.1" 200 225229 "" "Prometheus/2.33.4" 2022-09-29T09:55:32.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:55:32 smithi083 ceph-mon[79930]: pgmap v8018: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:55:32.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:55:32 smithi083 ceph-mon[77242]: pgmap v8018: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:55:32.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:55:32 smithi153 ceph-mon[70044]: pgmap v8018: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:55:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:55:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:55:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:55:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:55:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:55:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:55:34.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:55:34 smithi083 ceph-mon[79930]: pgmap v8019: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:55:34.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:55:34 smithi083 ceph-mon[77242]: pgmap v8019: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:55:34.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:55:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:55:34.519Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:55:34.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:55:34 smithi153 ceph-mon[70044]: pgmap v8019: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:55:35.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:55:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:55:35.228Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:55:36.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:55:36 smithi083 ceph-mon[79930]: pgmap v8020: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:55:36.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:55:36 smithi083 ceph-mon[77242]: pgmap v8020: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:55:36.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:55:36 smithi153 ceph-mon[70044]: pgmap v8020: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:55:38.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:55:38 smithi083 ceph-mon[77242]: pgmap v8021: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:55:38.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:55:38 smithi083 ceph-mon[79930]: pgmap v8021: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:55:38.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:55:38 smithi153 ceph-mon[70044]: pgmap v8021: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:55:40.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:55:40 smithi083 ceph-mon[79930]: pgmap v8022: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:55:40.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:55:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:55:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:55:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:55:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:55:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:55:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:55:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:55:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:55:40 smithi083 ceph-mon[77242]: pgmap v8022: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:55:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:55:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:55:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:55:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:55:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:55:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:55:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:55:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:55:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:55:40 smithi153 ceph-mon[70044]: pgmap v8022: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:55:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:55:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:55:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:55:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:55:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:55:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:55:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:55:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:55:41.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:55:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:55:41.138Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:55:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:55:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:55:41] "GET /metrics HTTP/1.1" 200 225227 "" "Prometheus/2.33.4" 2022-09-29T09:55:42.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:55:42 smithi083 ceph-mon[79930]: pgmap v8023: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:55:42.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:55:42 smithi083 ceph-mon[77242]: pgmap v8023: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:55:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:55:42 smithi153 ceph-mon[70044]: pgmap v8023: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:55:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:55:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:55:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:55:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:55:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:55:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:55:43.848 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:55:44.140 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:55:44.140 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 5m ago 4h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:55:44.140 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 3m ago 4h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:55:44.141 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 5m ago 4h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:55:44.141 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 3m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:55:44.141 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 3m ago 4h 443M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:55:44.141 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 5m ago 4h 740M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:55:44.142 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 5m ago 4h 994M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:55:44.142 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 3m ago 4h 438M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:55:44.142 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 5m ago 4h 581M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:55:44.142 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 5m ago 4h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:55:44.143 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 3m ago 4h 25.8M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:55:44.143 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 5m ago 4h 717M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:55:44.143 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 5m ago 4h 708M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:55:44.143 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 5m ago 4h 554M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:55:44.144 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 5m ago 4h 689M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:55:44.144 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 3m ago 4h 609M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:55:44.144 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 3m ago 4h 757M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:55:44.144 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 3m ago 4h 675M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:55:44.145 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 3m ago 4h 798M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:55:44.145 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 3m ago 4h 175M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:55:44.498 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:55:44.498 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:55:44.498 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:55:44.499 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:55:44.499 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:55:44.499 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:55:44.499 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:55:44.500 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:55:44.500 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:55:44.500 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:55:44.500 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:55:44.501 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:55:44.501 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:55:44.501 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:55:44.501 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:55:44.502 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:55:44.502 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:55:44.502 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:55:44.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:55:44 smithi083 ceph-mon[79930]: pgmap v8024: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:55:44.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:55:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:55:44.520Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:55:44.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:55:44 smithi083 ceph-mon[77242]: pgmap v8024: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:55:44.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:55:44 smithi153 ceph-mon[70044]: pgmap v8024: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:55:45.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:55:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:55:45.229Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:55:45.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:55:45 smithi083 ceph-mon[77242]: from='client.43467 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:55:45.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:55:45 smithi083 ceph-mon[77242]: from='client.43473 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:55:45.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:55:45 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2676283061' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:55:45.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:55:45 smithi083 ceph-mon[79930]: from='client.43467 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:55:45.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:55:45 smithi083 ceph-mon[79930]: from='client.43473 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:55:45.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:55:45 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2676283061' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:55:45.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:55:45 smithi153 ceph-mon[70044]: from='client.43467 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:55:45.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:55:45 smithi153 ceph-mon[70044]: from='client.43473 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:55:45.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:55:45 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2676283061' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:55:46.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:55:46 smithi083 ceph-mon[77242]: pgmap v8025: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:55:46.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:55:46 smithi083 ceph-mon[79930]: pgmap v8025: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:55:46.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:55:46 smithi153 ceph-mon[70044]: pgmap v8025: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:55:48.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:55:48 smithi083 ceph-mon[79930]: pgmap v8026: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:55:48.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:55:48 smithi083 ceph-mon[77242]: pgmap v8026: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:55:48.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:55:48 smithi153 ceph-mon[70044]: pgmap v8026: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:55:50.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:55:50 smithi083 ceph-mon[79930]: pgmap v8027: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:55:50.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:55:50 smithi083 ceph-mon[77242]: pgmap v8027: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:55:50.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:55:50 smithi153 ceph-mon[70044]: pgmap v8027: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:55:51.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:55:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:55:51.139Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:55:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:55:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:55:51] "GET /metrics HTTP/1.1" 200 225227 "" "Prometheus/2.33.4" 2022-09-29T09:55:52.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:55:52 smithi083 ceph-mon[79930]: pgmap v8028: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:55:52.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:55:52 smithi083 ceph-mon[77242]: pgmap v8028: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:55:52.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:55:52 smithi153 ceph-mon[70044]: pgmap v8028: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:55:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:55:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:55:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:55:53.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:55:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:55:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:55:54.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:55:54 smithi083 ceph-mon[79930]: pgmap v8029: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:55:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:55:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:55:54.521Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:55:54.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:55:54 smithi083 ceph-mon[77242]: pgmap v8029: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:55:54.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:55:54 smithi153 ceph-mon[70044]: pgmap v8029: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:55:55.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:55:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:55:55.230Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:55:56.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:55:56 smithi083 ceph-mon[79930]: pgmap v8030: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:55:56.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:55:56 smithi083 ceph-mon[77242]: pgmap v8030: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:55:56.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:55:56 smithi153 ceph-mon[70044]: pgmap v8030: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:55:58.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:55:58 smithi083 ceph-mon[79930]: pgmap v8031: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:55:58.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:55:58 smithi083 ceph-mon[77242]: pgmap v8031: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:55:58.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:55:58 smithi153 ceph-mon[70044]: pgmap v8031: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:00.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:56:00 smithi083 ceph-mon[79930]: pgmap v8032: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:00.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:56:00 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:56:00.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:56:00 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:56:00.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:56:00 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:56:00.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:56:00 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:56:00.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:56:00 smithi083 ceph-mon[77242]: pgmap v8032: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:00.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:56:00 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:56:00.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:56:00 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:56:00.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:56:00 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:56:00.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:56:00 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:56:00.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:56:00 smithi153 ceph-mon[70044]: pgmap v8032: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:00.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:56:00 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:56:00.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:56:00 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:56:00.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:56:00 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:56:00.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:56:00 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:56:01.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:56:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:56:01.139Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:56:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:56:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:56:01] "GET /metrics HTTP/1.1" 200 225227 "" "Prometheus/2.33.4" 2022-09-29T09:56:02.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:56:02 smithi083 ceph-mon[79930]: pgmap v8033: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:02.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:56:02 smithi083 ceph-mon[77242]: pgmap v8033: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:02.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:56:02 smithi153 ceph-mon[70044]: pgmap v8033: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:56:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:56:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:56:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:56:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:56:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:56:04.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:56:04 smithi083 ceph-mon[79930]: pgmap v8034: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:04.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:56:04 smithi083 ceph-mon[77242]: pgmap v8034: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:04.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:56:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:56:04.521Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:56:04.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:56:04 smithi153 ceph-mon[70044]: pgmap v8034: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:05.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:56:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:56:05.230Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:56:06.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:56:06 smithi083 ceph-mon[79930]: pgmap v8035: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:06.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:56:06 smithi083 ceph-mon[77242]: pgmap v8035: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:06.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:56:06 smithi153 ceph-mon[70044]: pgmap v8035: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:08.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:56:08 smithi083 ceph-mon[79930]: pgmap v8036: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:08.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:56:08 smithi083 ceph-mon[77242]: pgmap v8036: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:08.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:56:08 smithi153 ceph-mon[70044]: pgmap v8036: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:10.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:56:10 smithi083 ceph-mon[77242]: pgmap v8037: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:10.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:56:10 smithi083 ceph-mon[79930]: pgmap v8037: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:10.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:56:10 smithi153 ceph-mon[70044]: pgmap v8037: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:11.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:56:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:56:11.140Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:56:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:56:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:56:11] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T09:56:12.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:56:12 smithi083 ceph-mon[79930]: pgmap v8038: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:12.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:56:12 smithi083 ceph-mon[77242]: pgmap v8038: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:12.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:56:12 smithi153 ceph-mon[70044]: pgmap v8038: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:56:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:56:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:56:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:56:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:56:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:56:14.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:56:14 smithi083 ceph-mon[79930]: pgmap v8039: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:14.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:56:14 smithi083 ceph-mon[77242]: pgmap v8039: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:14.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:56:14 smithi153 ceph-mon[70044]: pgmap v8039: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:14.826 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:56:14.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:56:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:56:14.524Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:56:15.113 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:56:15.114 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 6m ago 4h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:56:15.114 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 4m ago 4h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:56:15.114 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 6m ago 4h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:56:15.115 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 4m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:56:15.115 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 4m ago 4h 443M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:56:15.115 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 6m ago 4h 740M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:56:15.115 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 6m ago 4h 994M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:56:15.116 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 4m ago 4h 438M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:56:15.116 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 6m ago 4h 581M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:56:15.116 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 6m ago 4h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:56:15.116 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 4m ago 4h 25.8M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:56:15.117 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 6m ago 4h 717M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:56:15.117 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 6m ago 4h 708M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:56:15.117 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 6m ago 4h 554M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:56:15.117 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 6m ago 4h 689M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:56:15.117 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 4m ago 4h 609M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:56:15.118 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 4m ago 4h 757M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:56:15.118 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 4m ago 4h 675M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:56:15.118 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 4m ago 4h 798M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:56:15.118 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 4m ago 4h 175M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:56:15.473 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:56:15.473 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:56:15.473 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:56:15.473 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:56:15.474 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:56:15.474 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:56:15.474 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:56:15.474 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:56:15.475 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:56:15.475 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:56:15.475 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:56:15.475 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:56:15.476 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:56:15.476 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:56:15.476 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:56:15.476 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:56:15.476 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:56:15.477 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:56:15.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:56:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:56:15.231Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:56:16.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:56:16 smithi083 ceph-mon[79930]: pgmap v8040: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:16.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:56:16 smithi083 ceph-mon[79930]: from='client.43485 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:56:16.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:56:16 smithi083 ceph-mon[79930]: from='client.43491 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:56:16.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:56:16 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1917579966' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:56:16.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:56:16 smithi083 ceph-mon[77242]: pgmap v8040: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:16.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:56:16 smithi083 ceph-mon[77242]: from='client.43485 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:56:16.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:56:16 smithi083 ceph-mon[77242]: from='client.43491 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:56:16.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:56:16 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1917579966' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:56:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:56:16 smithi153 ceph-mon[70044]: pgmap v8040: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:56:16 smithi153 ceph-mon[70044]: from='client.43485 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:56:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:56:16 smithi153 ceph-mon[70044]: from='client.43491 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:56:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:56:16 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1917579966' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:56:18.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:56:18 smithi083 ceph-mon[79930]: pgmap v8041: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:18.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:56:18 smithi083 ceph-mon[77242]: pgmap v8041: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:56:18 smithi153 ceph-mon[70044]: pgmap v8041: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:20.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:56:20 smithi083 ceph-mon[79930]: pgmap v8042: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:20.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:56:20 smithi083 ceph-mon[77242]: pgmap v8042: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:20.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:56:20 smithi153 ceph-mon[70044]: pgmap v8042: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:21.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:56:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:56:21.141Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:56:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:56:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:56:21] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T09:56:22.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:56:22 smithi083 ceph-mon[79930]: pgmap v8043: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:22.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:56:22 smithi083 ceph-mon[77242]: pgmap v8043: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:56:22 smithi153 ceph-mon[70044]: pgmap v8043: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:22.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:56:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:56:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:56:23.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:56:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:56:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:56:24.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:56:24 smithi083 ceph-mon[79930]: pgmap v8044: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:24.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:56:24 smithi083 ceph-mon[77242]: pgmap v8044: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:24.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:56:24 smithi153 ceph-mon[70044]: pgmap v8044: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:24.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:56:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:56:24.522Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:56:25.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:56:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:56:25.233Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:56:26.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:56:26 smithi083 ceph-mon[77242]: pgmap v8045: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:26.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:56:26 smithi083 ceph-mon[79930]: pgmap v8045: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:56:26 smithi153 ceph-mon[70044]: pgmap v8045: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:56:28 smithi153 ceph-mon[70044]: pgmap v8046: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:28.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:56:28 smithi083 ceph-mon[79930]: pgmap v8046: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:28.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:56:28 smithi083 ceph-mon[77242]: pgmap v8046: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:56:30 smithi153 ceph-mon[70044]: pgmap v8047: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:30.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:56:30 smithi083 ceph-mon[79930]: pgmap v8047: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:30.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:56:30 smithi083 ceph-mon[77242]: pgmap v8047: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:31.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:56:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:56:31.141Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:56:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:56:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:56:31] "GET /metrics HTTP/1.1" 200 225228 "" "Prometheus/2.33.4" 2022-09-29T09:56:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:56:32 smithi153 ceph-mon[70044]: pgmap v8048: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:32.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:56:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:56:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:56:32.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:56:32 smithi083 ceph-mon[79930]: pgmap v8048: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:32.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:56:32 smithi083 ceph-mon[77242]: pgmap v8048: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:56:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:56:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:56:34.525 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:56:34 smithi083 ceph-mon[79930]: pgmap v8049: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:34.525 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:56:34 smithi083 ceph-mon[77242]: pgmap v8049: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:34.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:56:34 smithi153 ceph-mon[70044]: pgmap v8049: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:34.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:56:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:56:34.524Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:56:35.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:56:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:56:35.233Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:56:36.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:56:36 smithi153 ceph-mon[70044]: pgmap v8050: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:36.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:56:36 smithi083 ceph-mon[79930]: pgmap v8050: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:36.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:56:36 smithi083 ceph-mon[77242]: pgmap v8050: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:38.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:56:38 smithi153 ceph-mon[70044]: pgmap v8051: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:38.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:56:38 smithi083 ceph-mon[79930]: pgmap v8051: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:38.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:56:38 smithi083 ceph-mon[77242]: pgmap v8051: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:56:40 smithi153 ceph-mon[70044]: pgmap v8052: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:56:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:56:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:56:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:56:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:56:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:56:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:56:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:56:40.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:56:40 smithi083 ceph-mon[79930]: pgmap v8052: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:40.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:56:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:56:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:56:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:56:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:56:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:56:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:56:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:56:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:56:40 smithi083 ceph-mon[77242]: pgmap v8052: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:56:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:56:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:56:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:56:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:56:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:56:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:56:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:56:41.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:56:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:56:41.142Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:56:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:56:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:56:41] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T09:56:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:56:42 smithi153 ceph-mon[70044]: pgmap v8053: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:42.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:56:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:56:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:56:42.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:56:42 smithi083 ceph-mon[79930]: pgmap v8053: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:42.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:56:42 smithi083 ceph-mon[77242]: pgmap v8053: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:56:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:56:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:56:44.525 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:56:44 smithi083 ceph-mon[79930]: pgmap v8054: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:44.526 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:56:44 smithi083 ceph-mon[77242]: pgmap v8054: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:44.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:56:44 smithi153 ceph-mon[70044]: pgmap v8054: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:44.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:56:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:56:44.524Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:56:45.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:56:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:56:45.233Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:56:45.800 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:56:46.087 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:56:46.088 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 6m ago 4h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:56:46.088 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 4m ago 4h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:56:46.088 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 6m ago 4h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:56:46.089 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 4m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:56:46.089 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 4m ago 4h 443M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:56:46.089 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 6m ago 4h 740M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:56:46.089 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 6m ago 4h 994M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:56:46.089 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 4m ago 4h 438M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:56:46.090 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 6m ago 4h 581M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:56:46.090 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 6m ago 4h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:56:46.090 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 4m ago 4h 25.8M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:56:46.090 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 6m ago 4h 717M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:56:46.090 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 6m ago 4h 708M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:56:46.091 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 6m ago 4h 554M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:56:46.091 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 6m ago 4h 689M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:56:46.091 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 4m ago 4h 609M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:56:46.091 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 4m ago 4h 757M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:56:46.092 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 4m ago 4h 675M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:56:46.092 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 4m ago 4h 798M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:56:46.092 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 4m ago 4h 175M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:56:46.445 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:56:46.446 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:56:46.446 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:56:46.446 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:56:46.446 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:56:46.447 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:56:46.447 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:56:46.447 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:56:46.447 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:56:46.447 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:56:46.448 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:56:46.448 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:56:46.448 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:56:46.448 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:56:46.448 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:56:46.449 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:56:46.449 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:56:46.449 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:56:46.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:56:46 smithi153 ceph-mon[70044]: pgmap v8055: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:46.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:56:46 smithi083 ceph-mon[79930]: pgmap v8055: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:46.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:56:46 smithi083 ceph-mon[77242]: pgmap v8055: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:47.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:56:47 smithi153 ceph-mon[70044]: from='client.43503 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:56:47.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:56:47 smithi153 ceph-mon[70044]: from='client.53440 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:56:47.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:56:47 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2472524173' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:56:47.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:56:47 smithi083 ceph-mon[79930]: from='client.43503 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:56:47.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:56:47 smithi083 ceph-mon[79930]: from='client.53440 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:56:47.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:56:47 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2472524173' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:56:47.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:56:47 smithi083 ceph-mon[77242]: from='client.43503 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:56:47.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:56:47 smithi083 ceph-mon[77242]: from='client.53440 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:56:47.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:56:47 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2472524173' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:56:48.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:56:48 smithi153 ceph-mon[70044]: pgmap v8056: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:48.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:56:48 smithi083 ceph-mon[79930]: pgmap v8056: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:48.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:56:48 smithi083 ceph-mon[77242]: pgmap v8056: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:56:50 smithi153 ceph-mon[70044]: pgmap v8057: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:50.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:56:50 smithi083 ceph-mon[79930]: pgmap v8057: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:50.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:56:50 smithi083 ceph-mon[77242]: pgmap v8057: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:51.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:56:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:56:51.142Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:56:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:56:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:56:51] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T09:56:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:56:52 smithi153 ceph-mon[70044]: pgmap v8058: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:52.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:56:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:56:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:56:52.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:56:52 smithi083 ceph-mon[79930]: pgmap v8058: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:52.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:56:52 smithi083 ceph-mon[77242]: pgmap v8058: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:53.313 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:56:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:56:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:56:54.525 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:56:54 smithi083 ceph-mon[79930]: pgmap v8059: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:54.526 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:56:54 smithi083 ceph-mon[77242]: pgmap v8059: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:54.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:56:54 smithi153 ceph-mon[70044]: pgmap v8059: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:54.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:56:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:56:54.524Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:56:55.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:56:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:56:55.233Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:56:56.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:56:56 smithi153 ceph-mon[70044]: pgmap v8060: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:56.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:56:56 smithi083 ceph-mon[77242]: pgmap v8060: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:56.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:56:56 smithi083 ceph-mon[79930]: pgmap v8060: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:56:58 smithi153 ceph-mon[70044]: pgmap v8061: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:58.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:56:58 smithi083 ceph-mon[77242]: pgmap v8061: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:56:58.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:56:58 smithi083 ceph-mon[79930]: pgmap v8061: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:57:00 smithi153 ceph-mon[70044]: pgmap v8062: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:57:00 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:57:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:57:00 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:57:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:57:00 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:57:00.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:57:00 smithi083 ceph-mon[77242]: pgmap v8062: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:00.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:57:00 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:57:00.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:57:00 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:57:00.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:57:00 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:57:00.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:57:00 smithi083 ceph-mon[79930]: pgmap v8062: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:00.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:57:00 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:57:00.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:57:00 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:57:00.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:57:00 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:57:01.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:57:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:57:01.142Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:57:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:57:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:57:01] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T09:57:01.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:57:01 smithi083 ceph-mon[79930]: pgmap v8063: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:01.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:57:01 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:57:01.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:57:01 smithi083 ceph-mon[77242]: pgmap v8063: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:01.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:57:01 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:57:02.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:57:01 smithi153 ceph-mon[70044]: pgmap v8063: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:02.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:57:01 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:57:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:57:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:57:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:57:03.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:57:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:57:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:57:04.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:57:04 smithi083 ceph-mon[79930]: pgmap v8064: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:04.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:57:04 smithi083 ceph-mon[77242]: pgmap v8064: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:04.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:57:04 smithi153 ceph-mon[70044]: pgmap v8064: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:04.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:57:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:57:04.525Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:57:05.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:57:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:57:05.235Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:57:06.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:57:06 smithi083 ceph-mon[79930]: pgmap v8065: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:06.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:57:06 smithi083 ceph-mon[77242]: pgmap v8065: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:06.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:57:06 smithi153 ceph-mon[70044]: pgmap v8065: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:57:08 smithi153 ceph-mon[70044]: pgmap v8066: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:08.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:57:08 smithi083 ceph-mon[77242]: pgmap v8066: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:08.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:57:08 smithi083 ceph-mon[79930]: pgmap v8066: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:57:10 smithi153 ceph-mon[70044]: pgmap v8067: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:10.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:57:10 smithi083 ceph-mon[79930]: pgmap v8067: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:10.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:57:10 smithi083 ceph-mon[77242]: pgmap v8067: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:11.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:57:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:57:11.144Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:57:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:57:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:57:11] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T09:57:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:57:12 smithi153 ceph-mon[70044]: pgmap v8068: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:12.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:57:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:57:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:57:12.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:57:12 smithi083 ceph-mon[79930]: pgmap v8068: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:12.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:57:12 smithi083 ceph-mon[77242]: pgmap v8068: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:57:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:57:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:57:14.526 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:57:14 smithi083 ceph-mon[79930]: pgmap v8069: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:14.527 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:57:14 smithi083 ceph-mon[77242]: pgmap v8069: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:14.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:57:14 smithi153 ceph-mon[70044]: pgmap v8069: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:14.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:57:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:57:14.525Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:57:15.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:57:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:57:15.236Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:57:16.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:57:16 smithi153 ceph-mon[70044]: pgmap v8070: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:16.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:57:16 smithi083 ceph-mon[79930]: pgmap v8070: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:16.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:57:16 smithi083 ceph-mon[77242]: pgmap v8070: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:16.773 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:57:17.061 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:57:17.061 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 7m ago 4h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:57:17.061 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 5m ago 4h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:57:17.061 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 7m ago 4h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:57:17.062 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 5m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:57:17.062 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 5m ago 4h 443M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:57:17.062 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 7m ago 4h 740M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:57:17.062 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 7m ago 4h 994M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:57:17.063 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 5m ago 4h 438M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:57:17.063 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 7m ago 4h 581M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:57:17.063 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 7m ago 4h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:57:17.063 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 5m ago 4h 25.8M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:57:17.064 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 7m ago 4h 717M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:57:17.064 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 7m ago 4h 708M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:57:17.064 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 7m ago 4h 554M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:57:17.064 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 7m ago 4h 689M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:57:17.065 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 5m ago 4h 609M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:57:17.065 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 5m ago 4h 757M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:57:17.065 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 5m ago 4h 675M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:57:17.065 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 5m ago 4h 798M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:57:17.065 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 5m ago 4h 175M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:57:17.421 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:57:17.421 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:57:17.421 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:57:17.422 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:57:17.422 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:57:17.422 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:57:17.423 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:57:17.423 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:57:17.423 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:57:17.423 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:57:17.424 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:57:17.424 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:57:17.424 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:57:17.424 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:57:17.425 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:57:17.425 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:57:17.425 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:57:17.425 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:57:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:57:18 smithi153 ceph-mon[70044]: pgmap v8071: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:57:18 smithi153 ceph-mon[70044]: from='client.43521 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:57:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:57:18 smithi153 ceph-mon[70044]: from='client.43527 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:57:18.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:57:18 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3485959049' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:57:18.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:57:18 smithi083 ceph-mon[79930]: pgmap v8071: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:18.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:57:18 smithi083 ceph-mon[79930]: from='client.43521 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:57:18.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:57:18 smithi083 ceph-mon[79930]: from='client.43527 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:57:18.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:57:18 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3485959049' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:57:18.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:57:18 smithi083 ceph-mon[77242]: pgmap v8071: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:18.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:57:18 smithi083 ceph-mon[77242]: from='client.43521 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:57:18.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:57:18 smithi083 ceph-mon[77242]: from='client.43527 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:57:18.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:57:18 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3485959049' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:57:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:57:20 smithi153 ceph-mon[70044]: pgmap v8072: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:20.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:57:20 smithi083 ceph-mon[79930]: pgmap v8072: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:20.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:57:20 smithi083 ceph-mon[77242]: pgmap v8072: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:21.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:57:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:57:21.144Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:57:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:57:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:57:21] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T09:57:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:57:22 smithi153 ceph-mon[70044]: pgmap v8073: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:22.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:57:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:57:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:57:22.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:57:22 smithi083 ceph-mon[79930]: pgmap v8073: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:22.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:57:22 smithi083 ceph-mon[77242]: pgmap v8073: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:57:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:57:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:57:24.526 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:57:24 smithi083 ceph-mon[79930]: pgmap v8074: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:24.527 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:57:24 smithi083 ceph-mon[77242]: pgmap v8074: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:24.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:57:24 smithi153 ceph-mon[70044]: pgmap v8074: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:24.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:57:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:57:24.525Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:57:25.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:57:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:57:25.236Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:57:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:57:26 smithi153 ceph-mon[70044]: pgmap v8075: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:26.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:57:26 smithi083 ceph-mon[77242]: pgmap v8075: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:26.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:57:26 smithi083 ceph-mon[79930]: pgmap v8075: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:57:28 smithi153 ceph-mon[70044]: pgmap v8076: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:28.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:57:28 smithi083 ceph-mon[79930]: pgmap v8076: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:28.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:57:28 smithi083 ceph-mon[77242]: pgmap v8076: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:30.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:57:30 smithi153 ceph-mon[70044]: pgmap v8077: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:30.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:57:30 smithi083 ceph-mon[79930]: pgmap v8077: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:30.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:57:30 smithi083 ceph-mon[77242]: pgmap v8077: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:31.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:57:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:57:31.145Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:57:31.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:57:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:57:31] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T09:57:32.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:57:32 smithi153 ceph-mon[70044]: pgmap v8078: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:32.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:57:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:57:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:57:32.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:57:32 smithi083 ceph-mon[79930]: pgmap v8078: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:32.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:57:32 smithi083 ceph-mon[77242]: pgmap v8078: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:57:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:57:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:57:34.527 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:57:34 smithi083 ceph-mon[79930]: pgmap v8079: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:34.528 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:57:34 smithi083 ceph-mon[77242]: pgmap v8079: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:34.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:57:34 smithi153 ceph-mon[70044]: pgmap v8079: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:34.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:57:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:57:34.526Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:57:35.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:57:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:57:35.237Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:57:36.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:57:36 smithi153 ceph-mon[70044]: pgmap v8080: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:36.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:57:36 smithi083 ceph-mon[77242]: pgmap v8080: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:36.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:57:36 smithi083 ceph-mon[79930]: pgmap v8080: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:57:38 smithi153 ceph-mon[70044]: pgmap v8081: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:38.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:57:38 smithi083 ceph-mon[79930]: pgmap v8081: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:38.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:57:38 smithi083 ceph-mon[77242]: pgmap v8081: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:57:40 smithi153 ceph-mon[70044]: pgmap v8082: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:57:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:57:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:57:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:57:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:57:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:57:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:57:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:57:40.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:57:40 smithi083 ceph-mon[79930]: pgmap v8082: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:40.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:57:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:57:40.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:57:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:57:40.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:57:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:57:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:57:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:57:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:57:40 smithi083 ceph-mon[77242]: pgmap v8082: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:57:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:57:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:57:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:57:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:57:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:57:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:57:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:57:41.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:57:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:57:41.146Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:57:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:57:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:57:41] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T09:57:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:57:42 smithi153 ceph-mon[70044]: pgmap v8083: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:42.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:57:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:57:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:57:42.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:57:42 smithi083 ceph-mon[79930]: pgmap v8083: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:42.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:57:42 smithi083 ceph-mon[77242]: pgmap v8083: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:43.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:57:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:57:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:57:44.528 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:57:44 smithi083 ceph-mon[79930]: pgmap v8084: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:44.529 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:57:44 smithi083 ceph-mon[77242]: pgmap v8084: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:44.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:57:44 smithi153 ceph-mon[70044]: pgmap v8084: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:44.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:57:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:57:44.527Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:57:45.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:57:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:57:45.237Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:57:46.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:57:46 smithi153 ceph-mon[70044]: pgmap v8085: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:46.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:57:46 smithi083 ceph-mon[79930]: pgmap v8085: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:46.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:57:46 smithi083 ceph-mon[77242]: pgmap v8085: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:47.747 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:57:48.035 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:57:48.035 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 7m ago 4h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:57:48.035 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 5m ago 4h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:57:48.036 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 7m ago 4h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:57:48.036 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 5m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:57:48.036 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 5m ago 4h 443M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:57:48.037 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 7m ago 4h 740M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:57:48.037 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 7m ago 4h 994M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:57:48.037 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 5m ago 4h 438M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:57:48.037 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 7m ago 4h 581M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:57:48.037 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 7m ago 4h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:57:48.038 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 5m ago 4h 25.8M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:57:48.038 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 7m ago 4h 717M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:57:48.038 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 7m ago 4h 708M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:57:48.039 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 7m ago 4h 554M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:57:48.039 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 7m ago 4h 689M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:57:48.039 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 5m ago 4h 609M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:57:48.039 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 5m ago 4h 757M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:57:48.040 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 5m ago 4h 675M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:57:48.040 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 5m ago 4h 798M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:57:48.040 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 5m ago 4h 175M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:57:48.396 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:57:48.397 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:57:48.397 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:57:48.397 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:57:48.397 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:57:48.398 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:57:48.398 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:57:48.398 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:57:48.399 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:57:48.399 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:57:48.399 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:57:48.399 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:57:48.400 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:57:48.400 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:57:48.400 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:57:48.400 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:57:48.401 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:57:48.401 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:57:48.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:57:48 smithi153 ceph-mon[70044]: pgmap v8086: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:48.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:57:48 smithi083 ceph-mon[79930]: pgmap v8086: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:48.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:57:48 smithi083 ceph-mon[77242]: pgmap v8086: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:49.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:57:49 smithi153 ceph-mon[70044]: from='client.43539 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:57:49.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:57:49 smithi153 ceph-mon[70044]: from='client.43545 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:57:49.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:57:49 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/108032770' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:57:49.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:57:49 smithi083 ceph-mon[79930]: from='client.43539 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:57:49.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:57:49 smithi083 ceph-mon[79930]: from='client.43545 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:57:49.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:57:49 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/108032770' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:57:49.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:57:49 smithi083 ceph-mon[77242]: from='client.43539 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:57:49.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:57:49 smithi083 ceph-mon[77242]: from='client.43545 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:57:49.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:57:49 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/108032770' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:57:50.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:57:50 smithi153 ceph-mon[70044]: pgmap v8087: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:50.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:57:50 smithi083 ceph-mon[79930]: pgmap v8087: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:50.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:57:50 smithi083 ceph-mon[77242]: pgmap v8087: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:51.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:57:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:57:51.146Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:57:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:57:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:57:51] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T09:57:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:57:52 smithi153 ceph-mon[70044]: pgmap v8088: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:52.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:57:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:57:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:57:52.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:57:52 smithi083 ceph-mon[79930]: pgmap v8088: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:52.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:57:52 smithi083 ceph-mon[77242]: pgmap v8088: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:57:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:57:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:57:54.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:57:54 smithi153 ceph-mon[70044]: pgmap v8089: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:54.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:57:54 smithi083 ceph-mon[79930]: pgmap v8089: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:54.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:57:54 smithi083 ceph-mon[77242]: pgmap v8089: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:57:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:57:54.527Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:57:55.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:57:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:57:55.237Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:57:56.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:57:56 smithi153 ceph-mon[70044]: pgmap v8090: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:56.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:57:56 smithi083 ceph-mon[79930]: pgmap v8090: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:56.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:57:56 smithi083 ceph-mon[77242]: pgmap v8090: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:57:58 smithi153 ceph-mon[70044]: pgmap v8091: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:58.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:57:58 smithi083 ceph-mon[79930]: pgmap v8091: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:57:58.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:57:58 smithi083 ceph-mon[77242]: pgmap v8091: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:58:00 smithi153 ceph-mon[70044]: pgmap v8092: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:00.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:58:00 smithi083 ceph-mon[79930]: pgmap v8092: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:00.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:58:00 smithi083 ceph-mon[77242]: pgmap v8092: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:01.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:58:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:58:01.147Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:58:01.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:58:01 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:58:01.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:58:01 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:58:01.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:58:01 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:58:01.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:58:01 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:58:01.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:58:01 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:58:01.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:58:01 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:58:01.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:58:01 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:58:01.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:58:01 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:58:01.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:58:01 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:58:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:58:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:58:01] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T09:58:02.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:58:02 smithi083 ceph-mon[79930]: pgmap v8093: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:02.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:58:02 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:58:02.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:58:02 smithi083 ceph-mon[77242]: pgmap v8093: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:02.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:58:02 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:58:02.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:58:02 smithi153 ceph-mon[70044]: pgmap v8093: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:02.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:58:02 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:58:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:58:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:58:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:58:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:58:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:58:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:58:04.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:58:04 smithi083 ceph-mon[79930]: pgmap v8094: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:04.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:58:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:58:04.529Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:58:04.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:58:04 smithi083 ceph-mon[77242]: pgmap v8094: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:04.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:58:04 smithi153 ceph-mon[70044]: pgmap v8094: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:05.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:58:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:58:05.239Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:58:06.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:58:06 smithi083 ceph-mon[77242]: pgmap v8095: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:06.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:58:06 smithi083 ceph-mon[79930]: pgmap v8095: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:06.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:58:06 smithi153 ceph-mon[70044]: pgmap v8095: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:08.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:58:08 smithi083 ceph-mon[79930]: pgmap v8096: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:08.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:58:08 smithi083 ceph-mon[77242]: pgmap v8096: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:08.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:58:08 smithi153 ceph-mon[70044]: pgmap v8096: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:10.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:58:10 smithi083 ceph-mon[77242]: pgmap v8097: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:10.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:58:10 smithi083 ceph-mon[79930]: pgmap v8097: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:10.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:58:10 smithi153 ceph-mon[70044]: pgmap v8097: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:11.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:58:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:58:11.148Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:58:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:58:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:58:11] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T09:58:12.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:58:12 smithi083 ceph-mon[79930]: pgmap v8098: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:12.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:58:12 smithi083 ceph-mon[77242]: pgmap v8098: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:12.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:58:12 smithi153 ceph-mon[70044]: pgmap v8098: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:58:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:58:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:58:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:58:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:58:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:58:14.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:58:14 smithi083 ceph-mon[79930]: pgmap v8099: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:14.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:58:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:58:14.529Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:58:14.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:58:14 smithi083 ceph-mon[77242]: pgmap v8099: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:14.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:58:14 smithi153 ceph-mon[70044]: pgmap v8099: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:15.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:58:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:58:15.239Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:58:16.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:58:16 smithi083 ceph-mon[79930]: pgmap v8100: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:16.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:58:16 smithi083 ceph-mon[77242]: pgmap v8100: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:16.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:58:16 smithi153 ceph-mon[70044]: pgmap v8100: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:18.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:58:18 smithi083 ceph-mon[79930]: pgmap v8101: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:18.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:58:18 smithi083 ceph-mon[77242]: pgmap v8101: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:18.723 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:58:18.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:58:18 smithi153 ceph-mon[70044]: pgmap v8101: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:19.011 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:58:19.011 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 8m ago 4h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:58:19.012 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 6m ago 4h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:58:19.012 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 8m ago 4h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:58:19.012 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 6m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:58:19.012 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 6m ago 4h 443M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:58:19.013 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 8m ago 4h 740M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:58:19.013 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 8m ago 4h 994M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:58:19.013 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 6m ago 4h 438M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:58:19.013 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 8m ago 4h 581M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:58:19.014 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 8m ago 4h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:58:19.014 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 6m ago 4h 25.8M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:58:19.014 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 8m ago 4h 717M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:58:19.014 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 8m ago 4h 708M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:58:19.014 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 8m ago 4h 554M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:58:19.015 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 8m ago 4h 689M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:58:19.015 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 6m ago 4h 609M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:58:19.015 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 6m ago 4h 757M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:58:19.016 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 6m ago 4h 675M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:58:19.016 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 6m ago 4h 798M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:58:19.017 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 6m ago 4h 175M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:58:19.367 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:58:19.368 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:58:19.368 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:58:19.368 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:58:19.368 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:58:19.369 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:58:19.369 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:58:19.369 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:58:19.369 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:58:19.370 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:58:19.370 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:58:19.370 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:58:19.370 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:58:19.371 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:58:19.371 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:58:19.371 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:58:19.371 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:58:19.371 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:58:19.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:58:19 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2253033675' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:58:19.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:58:19 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2253033675' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:58:19.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:58:19 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2253033675' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:58:20.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:58:20 smithi083 ceph-mon[79930]: pgmap v8102: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:20.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:58:20 smithi083 ceph-mon[79930]: from='client.43557 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:58:20.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:58:20 smithi083 ceph-mon[79930]: from='client.53494 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:58:20.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:58:20 smithi083 ceph-mon[77242]: pgmap v8102: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:20.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:58:20 smithi083 ceph-mon[77242]: from='client.43557 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:58:20.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:58:20 smithi083 ceph-mon[77242]: from='client.53494 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:58:20.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:58:20 smithi153 ceph-mon[70044]: pgmap v8102: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:20.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:58:20 smithi153 ceph-mon[70044]: from='client.43557 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:58:20.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:58:20 smithi153 ceph-mon[70044]: from='client.53494 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:58:21.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:58:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:58:21.149Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:58:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:58:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:58:21] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T09:58:22.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:58:22 smithi083 ceph-mon[77242]: pgmap v8103: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:22.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:58:22 smithi083 ceph-mon[79930]: pgmap v8103: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:22.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:58:22 smithi153 ceph-mon[70044]: pgmap v8103: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:58:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:58:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:58:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:58:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:58:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:58:24.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:58:24 smithi083 ceph-mon[79930]: pgmap v8104: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:58:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:58:24.530Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:58:24.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:58:24 smithi083 ceph-mon[77242]: pgmap v8104: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:24.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:58:24 smithi153 ceph-mon[70044]: pgmap v8104: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:25.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:58:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:58:25.240Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:58:26.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:58:26 smithi083 ceph-mon[79930]: pgmap v8105: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:26.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:58:26 smithi083 ceph-mon[77242]: pgmap v8105: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:26.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:58:26 smithi153 ceph-mon[70044]: pgmap v8105: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:28.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:58:28 smithi083 ceph-mon[79930]: pgmap v8106: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:28.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:58:28 smithi083 ceph-mon[77242]: pgmap v8106: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:28.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:58:28 smithi153 ceph-mon[70044]: pgmap v8106: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:29.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:58:29 smithi083 ceph-mon[79930]: pgmap v8107: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:29.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:58:29 smithi083 ceph-mon[77242]: pgmap v8107: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:29.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:58:29 smithi153 ceph-mon[70044]: pgmap v8107: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:31.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:58:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:58:31.149Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:58:31.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:58:31 smithi083 ceph-mon[77242]: pgmap v8108: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:58:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:58:31] "GET /metrics HTTP/1.1" 200 225228 "" "Prometheus/2.33.4" 2022-09-29T09:58:31.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:58:31 smithi083 ceph-mon[79930]: pgmap v8108: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:32.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:58:31 smithi153 ceph-mon[70044]: pgmap v8108: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:58:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:58:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:58:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:58:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:58:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:58:34.532 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:58:34 smithi083 ceph-mon[79930]: pgmap v8109: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:34.532 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:58:34 smithi083 ceph-mon[77242]: pgmap v8109: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:34.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:58:34 smithi153 ceph-mon[70044]: pgmap v8109: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:34.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:58:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:58:34.531Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:58:35.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:58:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:58:35.242Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:58:36.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:58:36 smithi153 ceph-mon[70044]: pgmap v8110: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:36.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:58:36 smithi083 ceph-mon[79930]: pgmap v8110: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:36.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:58:36 smithi083 ceph-mon[77242]: pgmap v8110: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:58:38 smithi153 ceph-mon[70044]: pgmap v8111: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:38.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:58:38 smithi083 ceph-mon[79930]: pgmap v8111: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:38.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:58:38 smithi083 ceph-mon[77242]: pgmap v8111: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:58:40 smithi153 ceph-mon[70044]: pgmap v8112: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:58:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:58:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:58:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:58:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:58:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:58:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:58:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:58:40.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:58:40 smithi083 ceph-mon[79930]: pgmap v8112: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:40.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:58:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:58:40.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:58:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:58:40.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:58:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:58:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:58:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:58:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:58:40 smithi083 ceph-mon[77242]: pgmap v8112: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:58:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:58:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:58:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:58:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:58:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:58:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:58:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:58:41.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:58:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:58:41.150Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:58:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:58:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:58:41] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T09:58:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:58:42 smithi153 ceph-mon[70044]: pgmap v8113: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:42.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:58:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:58:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:58:42.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:58:42 smithi083 ceph-mon[79930]: pgmap v8113: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:42.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:58:42 smithi083 ceph-mon[77242]: pgmap v8113: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:58:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:58:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:58:44.532 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:58:44 smithi083 ceph-mon[79930]: pgmap v8114: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:44.533 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:58:44 smithi083 ceph-mon[77242]: pgmap v8114: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:44.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:58:44 smithi153 ceph-mon[70044]: pgmap v8114: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:44.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:58:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:58:44.532Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:58:45.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:58:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:58:45.243Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:58:46.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:58:46 smithi153 ceph-mon[70044]: pgmap v8115: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:46.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:58:46 smithi083 ceph-mon[79930]: pgmap v8115: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:46.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:58:46 smithi083 ceph-mon[77242]: pgmap v8115: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:48.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:58:48 smithi153 ceph-mon[70044]: pgmap v8116: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:48.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:58:48 smithi083 ceph-mon[79930]: pgmap v8116: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:48.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:58:48 smithi083 ceph-mon[77242]: pgmap v8116: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:49.689 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:58:49.977 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:58:49.978 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 8m ago 4h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:58:49.978 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 6m ago 4h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:58:49.978 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 8m ago 4h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:58:49.979 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 6m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:58:49.979 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 6m ago 4h 443M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:58:49.979 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 8m ago 4h 740M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:58:49.979 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 8m ago 4h 994M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:58:49.980 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 6m ago 4h 438M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:58:49.980 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 8m ago 4h 581M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:58:49.980 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 8m ago 4h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:58:49.980 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 6m ago 4h 25.8M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:58:49.980 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 8m ago 4h 717M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:58:49.981 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 8m ago 4h 708M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:58:49.981 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 8m ago 4h 554M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:58:49.981 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 8m ago 4h 689M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:58:49.981 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 6m ago 4h 609M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:58:49.982 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 6m ago 4h 757M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:58:49.982 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 6m ago 4h 675M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:58:49.982 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 6m ago 4h 798M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:58:49.982 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 6m ago 4h 175M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:58:50.334 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:58:50.335 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:58:50.335 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:58:50.335 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:58:50.335 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:58:50.336 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:58:50.336 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:58:50.336 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:58:50.336 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:58:50.336 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:58:50.337 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:58:50.337 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:58:50.337 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:58:50.337 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:58:50.338 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:58:50.338 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:58:50.338 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:58:50.338 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:58:50.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:58:50 smithi153 ceph-mon[70044]: pgmap v8117: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:50.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:58:50 smithi083 ceph-mon[77242]: pgmap v8117: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:50.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:58:50 smithi083 ceph-mon[79930]: pgmap v8117: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:51.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:58:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:58:51.150Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:58:51.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:58:51 smithi083 ceph-mon[77242]: from='client.43575 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:58:51.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:58:51 smithi083 ceph-mon[77242]: from='client.43581 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:58:51.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:58:51 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1291780171' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:58:51.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:58:51 smithi083 ceph-mon[79930]: from='client.43575 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:58:51.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:58:51 smithi083 ceph-mon[79930]: from='client.43581 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:58:51.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:58:51 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1291780171' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:58:51.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:58:51 smithi153 ceph-mon[70044]: from='client.43575 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:58:51.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:58:51 smithi153 ceph-mon[70044]: from='client.43581 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:58:51.563 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:58:51 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1291780171' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:58:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:58:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:58:51] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T09:58:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:58:52 smithi153 ceph-mon[70044]: pgmap v8118: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:52.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:58:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:58:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:58:52.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:58:52 smithi083 ceph-mon[77242]: pgmap v8118: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:52.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:58:52 smithi083 ceph-mon[79930]: pgmap v8118: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:58:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:58:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:58:54.533 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:58:54 smithi083 ceph-mon[79930]: pgmap v8119: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:54.534 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:58:54 smithi083 ceph-mon[77242]: pgmap v8119: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:54.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:58:54 smithi153 ceph-mon[70044]: pgmap v8119: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:54.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:58:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:58:54.532Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:58:55.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:58:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:58:55.243Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:58:56.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:58:56 smithi153 ceph-mon[70044]: pgmap v8120: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:56.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:58:56 smithi083 ceph-mon[79930]: pgmap v8120: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:56.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:58:56 smithi083 ceph-mon[77242]: pgmap v8120: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:58:58 smithi153 ceph-mon[70044]: pgmap v8121: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:58.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:58:58 smithi083 ceph-mon[79930]: pgmap v8121: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:58:58.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:58:58 smithi083 ceph-mon[77242]: pgmap v8121: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:59:00 smithi153 ceph-mon[70044]: pgmap v8122: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:00.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:59:00 smithi083 ceph-mon[79930]: pgmap v8122: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:00.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:59:00 smithi083 ceph-mon[77242]: pgmap v8122: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:01.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:59:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:59:01.152Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:59:01.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:59:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:59:01] "GET /metrics HTTP/1.1" 200 225228 "" "Prometheus/2.33.4" 2022-09-29T09:59:02.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:59:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:59:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:59:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:59:02 smithi153 ceph-mon[70044]: pgmap v8123: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:59:02 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:59:02.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:59:02 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:59:02.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:59:02 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:59:02.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:59:02 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:59:02.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:59:02 smithi083 ceph-mon[77242]: pgmap v8123: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:02.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:59:02 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:59:02.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:59:02 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:59:02.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:59:02 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:59:02.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:59:02 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:59:02.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:59:02 smithi083 ceph-mon[79930]: pgmap v8123: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:02.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:59:02 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T09:59:02.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:59:02 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T09:59:02.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:59:02 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T09:59:02.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:59:02 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T09:59:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:59:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:59:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:59:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:59:04 smithi153 ceph-mon[70044]: pgmap v8124: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:04.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:59:04 smithi083 ceph-mon[79930]: pgmap v8124: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:04.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:59:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:59:04.533Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:59:04.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:59:04 smithi083 ceph-mon[77242]: pgmap v8124: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:05.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:59:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:59:05.243Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:59:06.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:59:06 smithi153 ceph-mon[70044]: pgmap v8125: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:06.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:59:06 smithi083 ceph-mon[79930]: pgmap v8125: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:06.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:59:06 smithi083 ceph-mon[77242]: pgmap v8125: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:08.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:59:08 smithi153 ceph-mon[70044]: pgmap v8126: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:08.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:59:08 smithi083 ceph-mon[79930]: pgmap v8126: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:08.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:59:08 smithi083 ceph-mon[77242]: pgmap v8126: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:59:10 smithi153 ceph-mon[70044]: pgmap v8127: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:10.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:59:10 smithi083 ceph-mon[79930]: pgmap v8127: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:10.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:59:10 smithi083 ceph-mon[77242]: pgmap v8127: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:11.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:59:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:59:11.151Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:59:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:59:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:59:11] "GET /metrics HTTP/1.1" 200 225228 "" "Prometheus/2.33.4" 2022-09-29T09:59:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:59:12 smithi153 ceph-mon[70044]: pgmap v8128: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:12.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:59:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:59:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:59:12.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:59:12 smithi083 ceph-mon[77242]: pgmap v8128: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:12.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:59:12 smithi083 ceph-mon[79930]: pgmap v8128: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:59:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:59:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:59:14.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:59:14 smithi083 ceph-mon[77242]: pgmap v8129: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:14.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:59:14 smithi083 ceph-mon[79930]: pgmap v8129: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:14.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:59:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:59:14.533Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:59:14.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:59:14 smithi153 ceph-mon[70044]: pgmap v8129: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:15.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:59:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:59:15.244Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:59:16.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:59:16 smithi083 ceph-mon[79930]: pgmap v8130: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:16.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:59:16 smithi083 ceph-mon[77242]: pgmap v8130: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:16.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:59:16 smithi153 ceph-mon[70044]: pgmap v8130: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:18.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:59:18 smithi083 ceph-mon[77242]: pgmap v8131: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:18.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:59:18 smithi083 ceph-mon[79930]: pgmap v8131: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:18.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:59:18 smithi153 ceph-mon[70044]: pgmap v8131: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:20.654 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:59:20.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:59:20 smithi083 ceph-mon[79930]: pgmap v8132: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:20.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:59:20 smithi083 ceph-mon[77242]: pgmap v8132: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:20.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:59:20 smithi153 ceph-mon[70044]: pgmap v8132: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:20.942 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:59:20.943 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 9m ago 4h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:59:20.943 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 7m ago 4h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:59:20.943 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 9m ago 4h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:59:20.944 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 7m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:59:20.944 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 7m ago 4h 443M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:59:20.944 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 9m ago 4h 740M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:59:20.945 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 9m ago 4h 994M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:59:20.945 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 7m ago 4h 438M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:59:20.945 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 9m ago 4h 581M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:59:20.946 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 9m ago 4h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:59:20.946 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 7m ago 4h 25.8M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:59:20.946 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 9m ago 4h 717M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:59:20.946 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 9m ago 4h 708M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:59:20.947 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 9m ago 4h 554M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:59:20.947 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 9m ago 4h 689M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:59:20.947 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 7m ago 4h 609M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:59:20.947 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 7m ago 4h 757M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:59:20.948 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 7m ago 4h 675M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:59:20.948 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 7m ago 4h 798M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:59:20.948 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 7m ago 4h 175M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:59:21.298 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:59:21.298 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:59:21.299 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:59:21.299 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:59:21.299 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:59:21.299 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:59:21.299 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:59:21.300 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:59:21.300 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:59:21.300 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:59:21.300 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:59:21.300 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:59:21.301 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:59:21.301 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:59:21.301 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:59:21.301 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:59:21.302 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:59:21.302 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:59:21.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:59:21 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/345969205' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:59:21.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:59:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:59:21.151Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:59:21.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:59:21 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/345969205' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:59:21.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:59:21 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/345969205' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:59:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:59:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:59:21] "GET /metrics HTTP/1.1" 200 225228 "" "Prometheus/2.33.4" 2022-09-29T09:59:22.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:59:22 smithi083 ceph-mon[79930]: pgmap v8133: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:22.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:59:22 smithi083 ceph-mon[79930]: from='client.43593 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:59:22.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:59:22 smithi083 ceph-mon[79930]: from='client.43599 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:59:22.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:59:22 smithi083 ceph-mon[77242]: pgmap v8133: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:22.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:59:22 smithi083 ceph-mon[77242]: from='client.43593 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:59:22.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:59:22 smithi083 ceph-mon[77242]: from='client.43599 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:59:22.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:59:22 smithi153 ceph-mon[70044]: pgmap v8133: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:22.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:59:22 smithi153 ceph-mon[70044]: from='client.43593 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:59:22.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:59:22 smithi153 ceph-mon[70044]: from='client.43599 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:59:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:59:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:59:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:59:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:59:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:59:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:59:24.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:59:24 smithi083 ceph-mon[79930]: pgmap v8134: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:24.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:59:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:59:24.534Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:59:24.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:59:24 smithi083 ceph-mon[77242]: pgmap v8134: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:24.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:59:24 smithi153 ceph-mon[70044]: pgmap v8134: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:25.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:59:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:59:25.245Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:59:26.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:59:26 smithi083 ceph-mon[79930]: pgmap v8135: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:26.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:59:26 smithi083 ceph-mon[77242]: pgmap v8135: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:26.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:59:26 smithi153 ceph-mon[70044]: pgmap v8135: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:28.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:59:28 smithi083 ceph-mon[77242]: pgmap v8136: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:28.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:59:28 smithi083 ceph-mon[79930]: pgmap v8136: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:28.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:59:28 smithi153 ceph-mon[70044]: pgmap v8136: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:30.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:59:30 smithi083 ceph-mon[77242]: pgmap v8137: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:30.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:59:30 smithi083 ceph-mon[79930]: pgmap v8137: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:30.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:59:30 smithi153 ceph-mon[70044]: pgmap v8137: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:31.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:59:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:59:31.153Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:59:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:59:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:59:31] "GET /metrics HTTP/1.1" 200 225226 "" "Prometheus/2.33.4" 2022-09-29T09:59:32.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:59:32 smithi083 ceph-mon[77242]: pgmap v8138: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:32.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:59:32 smithi083 ceph-mon[79930]: pgmap v8138: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:32.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:59:32 smithi153 ceph-mon[70044]: pgmap v8138: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:59:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:59:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:59:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:59:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:59:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:59:34.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:59:34 smithi083 ceph-mon[77242]: pgmap v8139: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:34.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:59:34 smithi083 ceph-mon[79930]: pgmap v8139: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:34.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:59:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:59:34.535Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:59:34.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:59:34 smithi153 ceph-mon[70044]: pgmap v8139: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:35.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:59:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:59:35.244Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:59:36.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:59:36 smithi083 ceph-mon[79930]: pgmap v8140: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:36.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:59:36 smithi083 ceph-mon[77242]: pgmap v8140: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:36.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:59:36 smithi153 ceph-mon[70044]: pgmap v8140: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:38.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:59:38 smithi083 ceph-mon[79930]: pgmap v8141: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:38.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:59:38 smithi083 ceph-mon[77242]: pgmap v8141: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:38.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:59:38 smithi153 ceph-mon[70044]: pgmap v8141: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:40.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:59:40 smithi083 ceph-mon[77242]: pgmap v8142: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:59:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:59:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:59:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:59:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:59:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:59:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:59:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:59:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:59:40 smithi083 ceph-mon[79930]: pgmap v8142: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:59:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:59:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:59:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:59:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:59:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:59:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:59:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:59:40.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:59:40 smithi153 ceph-mon[70044]: pgmap v8142: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:59:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:59:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:59:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:59:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:59:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T09:59:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:59:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T09:59:41.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:59:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:59:41.153Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:59:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:59:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:59:41] "GET /metrics HTTP/1.1" 200 225224 "" "Prometheus/2.33.4" 2022-09-29T09:59:42.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:59:42 smithi083 ceph-mon[79930]: pgmap v8143: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:42.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:59:42 smithi083 ceph-mon[77242]: pgmap v8143: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:42.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:59:42 smithi153 ceph-mon[70044]: pgmap v8143: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:59:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:59:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:59:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:59:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:59:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:59:44.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:59:44 smithi083 ceph-mon[79930]: pgmap v8144: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:44.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:59:44 smithi083 ceph-mon[77242]: pgmap v8144: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:59:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:59:44.536Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:59:44.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:59:44 smithi153 ceph-mon[70044]: pgmap v8144: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:45.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:59:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:59:45.245Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:59:46.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:59:46 smithi083 ceph-mon[79930]: pgmap v8145: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:46.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:59:46 smithi083 ceph-mon[77242]: pgmap v8145: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:46.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:59:46 smithi153 ceph-mon[70044]: pgmap v8145: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:48.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:59:48 smithi083 ceph-mon[79930]: pgmap v8146: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:48.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:59:48 smithi083 ceph-mon[77242]: pgmap v8146: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:48.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:59:48 smithi153 ceph-mon[70044]: pgmap v8146: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:50.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:59:50 smithi083 ceph-mon[79930]: pgmap v8147: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:50.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:59:50 smithi083 ceph-mon[77242]: pgmap v8147: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:50.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:59:50 smithi153 ceph-mon[70044]: pgmap v8147: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:51.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:59:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:59:51.154Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:59:51.627 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T09:59:51.916 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T09:59:51.917 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 10m ago 4h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T09:59:51.917 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 7m ago 4h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T09:59:51.917 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 10m ago 4h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T09:59:51.917 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 7m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T09:59:51.918 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 7m ago 4h 443M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T09:59:51.918 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 10m ago 4h 740M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T09:59:51.918 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 10m ago 4h 994M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T09:59:51.918 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 7m ago 4h 438M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T09:59:51.919 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 10m ago 4h 581M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T09:59:51.919 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 10m ago 4h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T09:59:51.919 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 7m ago 4h 25.8M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T09:59:51.919 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 10m ago 4h 717M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T09:59:51.919 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 10m ago 4h 708M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T09:59:51.920 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 10m ago 4h 554M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T09:59:51.920 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 10m ago 4h 689M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T09:59:51.920 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 7m ago 4h 609M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T09:59:51.920 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 7m ago 4h 757M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T09:59:51.921 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 7m ago 4h 675M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T09:59:51.921 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 7m ago 4h 798M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T09:59:51.921 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 7m ago 4h 175M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T09:59:51.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 09:59:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:59:51] "GET /metrics HTTP/1.1" 200 225224 "" "Prometheus/2.33.4" 2022-09-29T09:59:52.273 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T09:59:52.274 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T09:59:52.274 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T09:59:52.274 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:59:52.274 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T09:59:52.275 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T09:59:52.275 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:59:52.275 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T09:59:52.275 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T09:59:52.276 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:59:52.276 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T09:59:52.276 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T09:59:52.276 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T09:59:52.276 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T09:59:52.277 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T09:59:52.277 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T09:59:52.277 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T09:59:52.277 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T09:59:52.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:59:52 smithi083 ceph-mon[77242]: pgmap v8148: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:52.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:59:52 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/757896234' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:59:52.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:59:52 smithi083 ceph-mon[79930]: pgmap v8148: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:52.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:59:52 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/757896234' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:59:52.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:59:52 smithi153 ceph-mon[70044]: pgmap v8148: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:52.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:59:52 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/757896234' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T09:59:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 09:59:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T09:59:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T09:59:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 09:59:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:09:59:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T09:59:53.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:59:53 smithi083 ceph-mon[79930]: from='client.53542 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:59:53.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:59:53 smithi083 ceph-mon[79930]: from='client.53548 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:59:53.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:59:53 smithi083 ceph-mon[77242]: from='client.53542 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:59:53.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:59:53 smithi083 ceph-mon[77242]: from='client.53548 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:59:53.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:59:53 smithi153 ceph-mon[70044]: from='client.53542 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:59:53.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:59:53 smithi153 ceph-mon[70044]: from='client.53548 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T09:59:54.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:59:54 smithi083 ceph-mon[79930]: pgmap v8149: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:54.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:59:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:59:54.535Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:59:54.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:59:54 smithi083 ceph-mon[77242]: pgmap v8149: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:54.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:59:54 smithi153 ceph-mon[70044]: pgmap v8149: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:55.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 09:59:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T09:59:55.246Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T09:59:56.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:59:56 smithi083 ceph-mon[79930]: pgmap v8150: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:56.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:59:56 smithi083 ceph-mon[77242]: pgmap v8150: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:56.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:59:56 smithi153 ceph-mon[70044]: pgmap v8150: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:58.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 09:59:58 smithi083 ceph-mon[79930]: pgmap v8151: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:58.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 09:59:58 smithi083 ceph-mon[77242]: pgmap v8151: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T09:59:58.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 09:59:58 smithi153 ceph-mon[70044]: pgmap v8151: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:00.383 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:00:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[77219]: 2022-09-29T09:59:59.999+0000 7f160f665700 -1 log_channel(cluster) log [ERR] : overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T10:00:00.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:00:00 smithi083 ceph-mon[79930]: pgmap v8152: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:00.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:00:00 smithi083 ceph-mon[79930]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T10:00:00.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:00:00 smithi083 ceph-mon[77242]: pgmap v8152: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:00.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:00:00 smithi083 ceph-mon[77242]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T10:00:00.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:00:00 smithi153 ceph-mon[70044]: pgmap v8152: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:00.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:00:00 smithi153 ceph-mon[70044]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T10:00:01.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:00:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:00:01.155Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:00:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:00:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:00:01] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T10:00:02.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:00:02 smithi083 ceph-mon[79930]: pgmap v8153: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:02.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:00:02 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:00:02.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:00:02 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:00:02.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:00:02 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:00:02.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:00:02 smithi083 ceph-mon[77242]: pgmap v8153: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:02.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:00:02 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:00:02.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:00:02 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:00:02.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:00:02 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:00:02.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:00:02 smithi153 ceph-mon[70044]: pgmap v8153: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:02.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:00:02 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:00:02.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:00:02 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:00:02.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:00:02 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:00:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:00:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:00:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:00:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:00:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:00:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:00:03.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:00:03 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:00:03.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:00:03 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:00:03.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:00:03 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:00:03.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:00:03 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:00:03.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:00:03 smithi083 ceph-mon[79930]: pgmap v8154: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:03.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:00:03 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:00:03.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:00:03 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:00:03.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:00:03 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:00:03.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:00:03 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:00:03.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:00:03 smithi083 ceph-mon[77242]: pgmap v8154: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:03.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:00:03 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:00:03.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:00:03 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:00:03.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:00:03 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:00:03.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:00:03 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:00:03.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:00:03 smithi153 ceph-mon[70044]: pgmap v8154: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:04.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:00:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:00:04.535Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:00:05.645 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:00:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:00:05.246Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:00:05.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:00:05 smithi083 ceph-mon[79930]: pgmap v8155: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:05.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:00:05 smithi083 ceph-mon[77242]: pgmap v8155: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:06.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:00:05 smithi153 ceph-mon[70044]: pgmap v8155: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:07.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:00:07 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:00:07.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:00:07 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:00:07.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:00:07 smithi083 ceph-mon[79930]: pgmap v8156: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:07.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:00:07 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:00:07.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:00:07 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:00:07.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:00:07 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:00:07.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:00:07 smithi083 ceph-mon[77242]: pgmap v8156: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:07.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:00:07 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:00:07.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:00:07 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:00:07.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:00:07 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:00:07.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:00:07 smithi153 ceph-mon[70044]: pgmap v8156: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:07.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:00:07 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:00:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:00:10 smithi153 ceph-mon[70044]: pgmap v8157: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:10.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:00:10 smithi083 ceph-mon[79930]: pgmap v8157: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:10.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:00:10 smithi083 ceph-mon[77242]: pgmap v8157: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:11.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:00:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:00:11.155Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:00:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:00:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:00:11] "GET /metrics HTTP/1.1" 200 225223 "" "Prometheus/2.33.4" 2022-09-29T10:00:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:00:12 smithi153 ceph-mon[70044]: pgmap v8158: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:12.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:00:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:00:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:00:12.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:00:12 smithi083 ceph-mon[79930]: pgmap v8158: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:12.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:00:12 smithi083 ceph-mon[77242]: pgmap v8158: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:00:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:00:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:00:14.538 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:00:14 smithi083 ceph-mon[79930]: pgmap v8159: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:14.539 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:00:14 smithi083 ceph-mon[77242]: pgmap v8159: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:14.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:00:14 smithi153 ceph-mon[70044]: pgmap v8159: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:14.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:00:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:00:14.537Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:00:15.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:00:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:00:15.248Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:00:16.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:00:16 smithi153 ceph-mon[70044]: pgmap v8160: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:16.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:00:16 smithi083 ceph-mon[79930]: pgmap v8160: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:16.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:00:16 smithi083 ceph-mon[77242]: pgmap v8160: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:18.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:00:18 smithi153 ceph-mon[70044]: pgmap v8161: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:18.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:00:18 smithi083 ceph-mon[79930]: pgmap v8161: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:18.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:00:18 smithi083 ceph-mon[77242]: pgmap v8161: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:20.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:00:20 smithi153 ceph-mon[70044]: pgmap v8162: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:20.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:00:20 smithi083 ceph-mon[79930]: pgmap v8162: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:20.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:00:20 smithi083 ceph-mon[77242]: pgmap v8162: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:21.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:00:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:00:21.155Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:00:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:00:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:00:21] "GET /metrics HTTP/1.1" 200 225223 "" "Prometheus/2.33.4" 2022-09-29T10:00:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:00:22 smithi153 ceph-mon[70044]: pgmap v8163: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:22.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:00:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:00:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:00:22.603 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:00:22.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:00:22 smithi083 ceph-mon[79930]: pgmap v8163: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:22.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:00:22 smithi083 ceph-mon[77242]: pgmap v8163: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:22.890 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:00:22.890 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 16s ago 4h 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:00:22.890 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 8m ago 4h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:00:22.891 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 16s ago 4h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:00:22.891 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 8m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:00:22.891 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 8m ago 4h 443M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:00:22.891 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 16s ago 4h 749M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:00:22.892 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 16s ago 4h 1054M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:00:22.892 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 8m ago 4h 438M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:00:22.892 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 16s ago 4h 604M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:00:22.892 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 16s ago 4h 25.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:00:22.892 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 8m ago 4h 25.8M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:00:22.893 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 16s ago 4h 742M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:00:22.893 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 16s ago 4h 733M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:00:22.893 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 16s ago 4h 573M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:00:22.893 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 16s ago 4h 714M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:00:22.893 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 8m ago 4h 609M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:00:22.894 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 8m ago 4h 757M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:00:22.894 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 8m ago 4h 675M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:00:22.894 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 8m ago 4h 798M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:00:22.894 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 8m ago 4h 175M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:00:23.243 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:00:23.244 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:00:23.244 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:00:23.244 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:00:23.244 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:00:23.245 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:00:23.245 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:00:23.245 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:00:23.245 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:00:23.246 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:00:23.246 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:00:23.246 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:00:23.246 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:00:23.247 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:00:23.247 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:00:23.247 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:00:23.247 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:00:23.247 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:00:23.274 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:00:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:00:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:00:23.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:00:23 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2523144991' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:00:23.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:00:23 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2523144991' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:00:23.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:00:23 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2523144991' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:00:24.538 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:00:24 smithi083 ceph-mon[79930]: from='client.53560 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:00:24.539 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:00:24 smithi083 ceph-mon[79930]: pgmap v8164: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:24.539 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:00:24 smithi083 ceph-mon[79930]: from='client.43635 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:00:24.539 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:00:24 smithi083 ceph-mon[77242]: from='client.53560 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:00:24.540 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:00:24 smithi083 ceph-mon[77242]: pgmap v8164: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:24.540 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:00:24 smithi083 ceph-mon[77242]: from='client.43635 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:00:24.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:00:24 smithi153 ceph-mon[70044]: from='client.53560 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:00:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:00:24 smithi153 ceph-mon[70044]: pgmap v8164: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:00:24 smithi153 ceph-mon[70044]: from='client.43635 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:00:24.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:00:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:00:24.537Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:00:25.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:00:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:00:25.248Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:00:26.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:00:26 smithi153 ceph-mon[70044]: pgmap v8165: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:26.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:00:26 smithi083 ceph-mon[79930]: pgmap v8165: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:26.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:00:26 smithi083 ceph-mon[77242]: pgmap v8165: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:00:28 smithi153 ceph-mon[70044]: pgmap v8166: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:28.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:00:28 smithi083 ceph-mon[79930]: pgmap v8166: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:28.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:00:28 smithi083 ceph-mon[77242]: pgmap v8166: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:00:30 smithi153 ceph-mon[70044]: pgmap v8167: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:30.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:00:30 smithi083 ceph-mon[77242]: pgmap v8167: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:30.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:00:30 smithi083 ceph-mon[79930]: pgmap v8167: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:31.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:00:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:00:31.156Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:00:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:00:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:00:31] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T10:00:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:00:32 smithi153 ceph-mon[70044]: pgmap v8168: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:32.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:00:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:00:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:00:32.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:00:32 smithi083 ceph-mon[79930]: pgmap v8168: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:32.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:00:32 smithi083 ceph-mon[77242]: pgmap v8168: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:00:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:00:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:00:34.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:00:34 smithi153 ceph-mon[70044]: pgmap v8169: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:34.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:00:34 smithi083 ceph-mon[79930]: pgmap v8169: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:34.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:00:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:00:34.538Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:00:34.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:00:34 smithi083 ceph-mon[77242]: pgmap v8169: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:35.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:00:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:00:35.249Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:00:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:00:36 smithi153 ceph-mon[70044]: pgmap v8170: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:36.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:00:36 smithi083 ceph-mon[79930]: pgmap v8170: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:36.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:00:36 smithi083 ceph-mon[77242]: pgmap v8170: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:00:38 smithi153 ceph-mon[70044]: pgmap v8171: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:38.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:00:38 smithi083 ceph-mon[77242]: pgmap v8171: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:38.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:00:38 smithi083 ceph-mon[79930]: pgmap v8171: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:00:40 smithi153 ceph-mon[70044]: pgmap v8172: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:00:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:00:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:00:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:00:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:00:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:00:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:00:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:00:40.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:00:40 smithi083 ceph-mon[77242]: pgmap v8172: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:40.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:00:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:00:40.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:00:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:00:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:00:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:00:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:00:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:00:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:00:40 smithi083 ceph-mon[79930]: pgmap v8172: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:00:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:00:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:00:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:00:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:00:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:00:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:00:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:00:41.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:00:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:00:41.157Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:00:41.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:00:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:00:41] "GET /metrics HTTP/1.1" 200 225228 "" "Prometheus/2.33.4" 2022-09-29T10:00:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:00:42 smithi153 ceph-mon[70044]: pgmap v8173: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:42.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:00:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:00:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:00:42.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:00:42 smithi083 ceph-mon[79930]: pgmap v8173: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:42.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:00:42 smithi083 ceph-mon[77242]: pgmap v8173: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:00:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:00:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:00:44.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:00:44 smithi153 ceph-mon[70044]: pgmap v8174: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:44.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:00:44 smithi083 ceph-mon[79930]: pgmap v8174: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:44.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:00:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:00:44.539Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:00:44.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:00:44 smithi083 ceph-mon[77242]: pgmap v8174: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:45.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:00:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:00:45.250Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:00:46.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:00:46 smithi083 ceph-mon[79930]: pgmap v8175: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:46.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:00:46 smithi083 ceph-mon[77242]: pgmap v8175: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:46.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:00:46 smithi153 ceph-mon[70044]: pgmap v8175: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:48.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:00:48 smithi083 ceph-mon[79930]: pgmap v8176: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:48.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:00:48 smithi083 ceph-mon[77242]: pgmap v8176: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:48.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:00:48 smithi153 ceph-mon[70044]: pgmap v8176: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:50.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:00:50 smithi083 ceph-mon[79930]: pgmap v8177: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:50.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:00:50 smithi083 ceph-mon[77242]: pgmap v8177: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:50.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:00:50 smithi153 ceph-mon[70044]: pgmap v8177: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:51.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:00:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:00:51.157Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:00:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:00:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:00:51] "GET /metrics HTTP/1.1" 200 225228 "" "Prometheus/2.33.4" 2022-09-29T10:00:52.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:00:52 smithi083 ceph-mon[79930]: pgmap v8178: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:52.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:00:52 smithi083 ceph-mon[77242]: pgmap v8178: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:52.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:00:52 smithi153 ceph-mon[70044]: pgmap v8178: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:00:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:00:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:00:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:00:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:00:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:00:53.580 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:00:53.878 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:00:53.878 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 47s ago 4h 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:00:53.878 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 8m ago 4h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:00:53.879 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 47s ago 4h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:00:53.879 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 8m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:00:53.879 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 8m ago 4h 443M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:00:53.879 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 47s ago 4h 749M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:00:53.880 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 47s ago 4h 1054M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:00:53.880 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 8m ago 4h 438M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:00:53.880 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 47s ago 4h 604M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:00:53.880 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 47s ago 4h 25.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:00:53.880 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 8m ago 4h 25.8M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:00:53.881 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 47s ago 4h 742M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:00:53.881 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 47s ago 4h 733M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:00:53.881 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 47s ago 4h 573M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:00:53.881 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 47s ago 4h 714M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:00:53.882 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 8m ago 4h 609M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:00:53.882 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 8m ago 4h 757M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:00:53.882 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 8m ago 4h 675M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:00:53.882 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 8m ago 4h 798M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:00:53.883 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 8m ago 4h 175M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:00:54.234 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:00:54.235 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:00:54.235 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:00:54.235 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:00:54.236 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:00:54.236 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:00:54.236 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:00:54.236 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:00:54.236 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:00:54.237 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:00:54.237 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:00:54.237 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:00:54.237 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:00:54.238 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:00:54.238 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:00:54.238 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:00:54.238 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:00:54.239 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:00:54.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:00:54 smithi083 ceph-mon[79930]: pgmap v8179: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:54.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:00:54 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/199263958' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:00:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:00:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:00:54.538Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:00:54.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:00:54 smithi083 ceph-mon[77242]: pgmap v8179: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:54.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:00:54 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/199263958' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:00:54.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:00:54 smithi153 ceph-mon[70044]: pgmap v8179: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:54.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:00:54 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/199263958' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:00:55.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:00:55 smithi083 ceph-mon[79930]: from='client.43647 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:00:55.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:00:55 smithi083 ceph-mon[79930]: from='client.43653 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:00:55.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:00:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:00:55.250Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:00:55.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:00:55 smithi083 ceph-mon[77242]: from='client.43647 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:00:55.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:00:55 smithi083 ceph-mon[77242]: from='client.43653 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:00:55.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:00:55 smithi153 ceph-mon[70044]: from='client.43647 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:00:55.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:00:55 smithi153 ceph-mon[70044]: from='client.43653 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:00:56.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:00:56 smithi083 ceph-mon[79930]: pgmap v8180: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:56.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:00:56 smithi083 ceph-mon[77242]: pgmap v8180: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:56.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:00:56 smithi153 ceph-mon[70044]: pgmap v8180: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:58.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:00:58 smithi083 ceph-mon[79930]: pgmap v8181: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:58.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:00:58 smithi083 ceph-mon[77242]: pgmap v8181: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:00:58.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:00:58 smithi153 ceph-mon[70044]: pgmap v8181: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:00.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:01:00 smithi083 ceph-mon[77242]: pgmap v8182: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:00.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:01:00 smithi083 ceph-mon[79930]: pgmap v8182: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:00.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:01:00 smithi153 ceph-mon[70044]: pgmap v8182: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:01.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:01:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:01:01.158Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:01:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:01:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:01:01] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T10:01:02.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:01:02 smithi083 ceph-mon[79930]: pgmap v8183: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:02.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:01:02 smithi083 ceph-mon[77242]: pgmap v8183: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:02.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:01:02 smithi153 ceph-mon[70044]: pgmap v8183: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:01:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:01:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:01:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:01:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:01:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:01:04.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:01:04 smithi083 ceph-mon[79930]: pgmap v8184: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:04.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:01:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:01:04.540Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:01:04.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:01:04 smithi083 ceph-mon[77242]: pgmap v8184: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:04.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:01:04 smithi153 ceph-mon[70044]: pgmap v8184: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:05.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:01:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:01:05.251Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:01:06.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:01:06 smithi083 ceph-mon[79930]: pgmap v8185: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:06.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:01:06 smithi083 ceph-mon[77242]: pgmap v8185: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:06.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:01:06 smithi153 ceph-mon[70044]: pgmap v8185: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:07.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:01:07 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:01:07.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:01:07 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:01:07.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:01:07 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:01:07.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:01:07 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:01:07.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:01:07 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:01:07.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:01:07 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:01:07.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:01:07 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:01:07.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:01:07 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:01:07.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:01:07 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:01:08.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:01:08 smithi083 ceph-mon[79930]: pgmap v8186: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:08.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:01:08 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:01:08.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:01:08 smithi083 ceph-mon[77242]: pgmap v8186: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:08.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:01:08 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:01:08.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:01:08 smithi153 ceph-mon[70044]: pgmap v8186: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:08.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:01:08 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:01:10.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:01:10 smithi083 ceph-mon[79930]: pgmap v8187: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:10.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:01:10 smithi083 ceph-mon[77242]: pgmap v8187: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:10.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:01:10 smithi153 ceph-mon[70044]: pgmap v8187: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:11.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:01:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:01:11.159Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:01:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:01:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:01:11] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T10:01:12.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:01:12 smithi083 ceph-mon[79930]: pgmap v8188: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:12.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:01:12 smithi083 ceph-mon[77242]: pgmap v8188: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:12.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:01:12 smithi153 ceph-mon[70044]: pgmap v8188: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:01:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:01:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:01:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:01:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:01:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:01:14.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:01:14 smithi083 ceph-mon[79930]: pgmap v8189: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:14.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:01:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:01:14.540Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:01:14.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:01:14 smithi083 ceph-mon[77242]: pgmap v8189: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:14.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:01:14 smithi153 ceph-mon[70044]: pgmap v8189: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:15.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:01:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:01:15.251Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:01:16.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:01:16 smithi083 ceph-mon[79930]: pgmap v8190: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:16.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:01:16 smithi083 ceph-mon[77242]: pgmap v8190: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:16.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:01:16 smithi153 ceph-mon[70044]: pgmap v8190: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:18.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:01:18 smithi083 ceph-mon[77242]: pgmap v8191: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:18.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:01:18 smithi083 ceph-mon[79930]: pgmap v8191: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:18.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:01:18 smithi153 ceph-mon[70044]: pgmap v8191: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:20.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:01:20 smithi083 ceph-mon[79930]: pgmap v8192: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:20.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:01:20 smithi083 ceph-mon[77242]: pgmap v8192: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:20.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:01:20 smithi153 ceph-mon[70044]: pgmap v8192: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:21.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:01:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:01:21.160Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:01:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:01:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:01:21] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T10:01:22.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:01:22 smithi083 ceph-mon[77242]: pgmap v8193: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:22.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:01:22 smithi083 ceph-mon[79930]: pgmap v8193: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:22.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:01:22 smithi153 ceph-mon[70044]: pgmap v8193: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:01:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:01:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:01:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:01:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:01:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:01:24.559 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:01:24.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:01:24 smithi083 ceph-mon[79930]: pgmap v8194: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:24.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:01:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:01:24.540Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:01:24.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:01:24 smithi083 ceph-mon[77242]: pgmap v8194: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:24.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:01:24 smithi153 ceph-mon[70044]: pgmap v8194: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:24.846 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:01:24.846 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 78s ago 4h 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:01:24.847 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 9m ago 4h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:01:24.847 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 78s ago 4h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:01:24.847 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 9m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:01:24.847 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 9m ago 4h 443M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:01:24.847 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 78s ago 4h 749M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:01:24.848 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 78s ago 4h 1054M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:01:24.848 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 9m ago 4h 438M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:01:24.848 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 78s ago 4h 604M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:01:24.848 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 78s ago 4h 25.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:01:24.849 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 9m ago 4h 25.8M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:01:24.849 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 78s ago 4h 742M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:01:24.849 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 78s ago 4h 733M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:01:24.849 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 78s ago 4h 573M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:01:24.850 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 78s ago 4h 714M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:01:24.850 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 9m ago 4h 609M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:01:24.850 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 9m ago 4h 757M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:01:24.850 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 9m ago 4h 675M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:01:24.851 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 9m ago 4h 798M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:01:24.851 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 9m ago 4h 175M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:01:25.202 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:01:25.202 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:01:25.203 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:01:25.203 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:01:25.203 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:01:25.203 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:01:25.203 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:01:25.204 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:01:25.204 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:01:25.204 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:01:25.204 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:01:25.205 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:01:25.205 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:01:25.205 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:01:25.205 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:01:25.205 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:01:25.206 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:01:25.206 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:01:25.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:01:25 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1898277936' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:01:25.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:01:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:01:25.251Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:01:25.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:01:25 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1898277936' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:01:25.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:01:25 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1898277936' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:01:26.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:01:26 smithi083 ceph-mon[79930]: from='client.43665 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:01:26.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:01:26 smithi083 ceph-mon[79930]: pgmap v8195: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:26.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:01:26 smithi083 ceph-mon[79930]: from='client.43671 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:01:26.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:01:26 smithi083 ceph-mon[77242]: from='client.43665 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:01:26.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:01:26 smithi083 ceph-mon[77242]: pgmap v8195: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:26.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:01:26 smithi083 ceph-mon[77242]: from='client.43671 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:01:26.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:01:26 smithi153 ceph-mon[70044]: from='client.43665 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:01:26.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:01:26 smithi153 ceph-mon[70044]: pgmap v8195: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:26.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:01:26 smithi153 ceph-mon[70044]: from='client.43671 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:01:28.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:01:28 smithi083 ceph-mon[79930]: pgmap v8196: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:28.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:01:28 smithi083 ceph-mon[77242]: pgmap v8196: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:28.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:01:28 smithi153 ceph-mon[70044]: pgmap v8196: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:30.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:01:30 smithi083 ceph-mon[79930]: pgmap v8197: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:30.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:01:30 smithi083 ceph-mon[77242]: pgmap v8197: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:30.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:01:30 smithi153 ceph-mon[70044]: pgmap v8197: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:31.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:01:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:01:31.160Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:01:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:01:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:01:31] "GET /metrics HTTP/1.1" 200 225228 "" "Prometheus/2.33.4" 2022-09-29T10:01:32.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:01:32 smithi083 ceph-mon[79930]: pgmap v8198: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:32.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:01:32 smithi083 ceph-mon[77242]: pgmap v8198: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:32.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:01:32 smithi153 ceph-mon[70044]: pgmap v8198: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:01:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:01:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:01:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:01:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:01:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:01:34.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:01:34 smithi083 ceph-mon[79930]: pgmap v8199: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:34.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:01:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:01:34.541Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:01:34.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:01:34 smithi083 ceph-mon[77242]: pgmap v8199: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:34.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:01:34 smithi153 ceph-mon[70044]: pgmap v8199: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:35.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:01:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:01:35.253Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:01:36.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:01:36 smithi083 ceph-mon[79930]: pgmap v8200: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:36.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:01:36 smithi083 ceph-mon[77242]: pgmap v8200: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:36.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:01:36 smithi153 ceph-mon[70044]: pgmap v8200: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:38.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:01:38 smithi083 ceph-mon[79930]: pgmap v8201: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:38.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:01:38 smithi083 ceph-mon[77242]: pgmap v8201: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:38.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:01:38 smithi153 ceph-mon[70044]: pgmap v8201: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:40.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:01:40 smithi083 ceph-mon[77242]: pgmap v8202: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:40.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:01:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:01:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:01:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:01:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:01:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:01:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:01:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:01:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:01:40 smithi083 ceph-mon[79930]: pgmap v8202: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:01:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:01:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:01:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:01:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:01:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:01:40.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:01:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:01:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:01:40 smithi153 ceph-mon[70044]: pgmap v8202: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:01:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:01:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:01:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:01:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:01:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:01:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:01:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:01:41.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:01:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:01:41.161Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:01:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:01:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:01:41] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T10:01:42.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:01:42 smithi083 ceph-mon[79930]: pgmap v8203: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:42.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:01:42 smithi083 ceph-mon[77242]: pgmap v8203: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:01:42 smithi153 ceph-mon[70044]: pgmap v8203: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:01:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:01:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:01:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:01:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:01:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:01:44.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:01:44 smithi083 ceph-mon[79930]: pgmap v8204: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:44.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:01:44 smithi083 ceph-mon[77242]: pgmap v8204: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:01:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:01:44.542Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:01:44.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:01:44 smithi153 ceph-mon[70044]: pgmap v8204: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:45.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:01:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:01:45.253Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:01:46.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:01:46 smithi083 ceph-mon[77242]: pgmap v8205: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:46.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:01:46 smithi083 ceph-mon[79930]: pgmap v8205: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:46.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:01:46 smithi153 ceph-mon[70044]: pgmap v8205: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:48.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:01:48 smithi083 ceph-mon[79930]: pgmap v8206: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:48.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:01:48 smithi083 ceph-mon[77242]: pgmap v8206: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:48.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:01:48 smithi153 ceph-mon[70044]: pgmap v8206: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:50.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:01:50 smithi083 ceph-mon[79930]: pgmap v8207: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:50.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:01:50 smithi083 ceph-mon[77242]: pgmap v8207: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:50.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:01:50 smithi153 ceph-mon[70044]: pgmap v8207: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:51.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:01:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:01:51.162Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:01:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:01:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:01:51] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T10:01:52.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:01:52 smithi083 ceph-mon[77242]: pgmap v8208: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:52.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:01:52 smithi083 ceph-mon[79930]: pgmap v8208: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:52.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:01:52 smithi153 ceph-mon[70044]: pgmap v8208: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:01:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:01:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:01:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:01:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:01:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:01:54.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:01:54 smithi083 ceph-mon[79930]: pgmap v8209: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:54.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:01:54 smithi083 ceph-mon[77242]: pgmap v8209: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:01:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:01:54.542Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:01:54.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:01:54 smithi153 ceph-mon[70044]: pgmap v8209: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:55.547 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:01:55.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:01:55 smithi083 ceph-mon[79930]: pgmap v8210: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:55.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:01:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:01:55.253Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:01:55.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:01:55 smithi083 ceph-mon[77242]: pgmap v8210: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:55.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:01:55 smithi153 ceph-mon[70044]: pgmap v8210: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:55.836 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:01:55.837 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 109s ago 4h 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:01:55.837 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 9m ago 4h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:01:55.837 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 109s ago 4h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:01:55.837 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 9m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:01:55.838 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 9m ago 4h 443M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:01:55.838 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 109s ago 4h 749M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:01:55.838 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 109s ago 4h 1054M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:01:55.838 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 9m ago 4h 438M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:01:55.839 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 109s ago 4h 604M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:01:55.839 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 109s ago 4h 25.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:01:55.839 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 9m ago 4h 25.8M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:01:55.839 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 109s ago 4h 742M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:01:55.839 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 109s ago 4h 733M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:01:55.840 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 109s ago 4h 573M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:01:55.840 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 109s ago 4h 714M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:01:55.840 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 9m ago 4h 609M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:01:55.840 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 9m ago 4h 757M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:01:55.841 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 9m ago 4h 675M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:01:55.841 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 9m ago 4h 798M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:01:55.841 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 9m ago 4h 175M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:01:56.191 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:01:56.191 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:01:56.192 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:01:56.192 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:01:56.192 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:01:56.192 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:01:56.192 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:01:56.193 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:01:56.193 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:01:56.193 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:01:56.193 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:01:56.194 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:01:56.194 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:01:56.194 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:01:56.194 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:01:56.195 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:01:56.195 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:01:56.195 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:01:56.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:01:56 smithi153 ceph-mon[70044]: from='client.53614 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:01:56.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:01:56 smithi153 ceph-mon[70044]: from='client.43689 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:01:56.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:01:56 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2789961578' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:01:56.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:01:56 smithi083 ceph-mon[79930]: from='client.53614 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:01:56.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:01:56 smithi083 ceph-mon[79930]: from='client.43689 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:01:56.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:01:56 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2789961578' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:01:56.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:01:56 smithi083 ceph-mon[77242]: from='client.53614 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:01:56.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:01:56 smithi083 ceph-mon[77242]: from='client.43689 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:01:56.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:01:56 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2789961578' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:01:57.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:01:57 smithi153 ceph-mon[70044]: pgmap v8211: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:57.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:01:57 smithi083 ceph-mon[79930]: pgmap v8211: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:01:57.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:01:57 smithi083 ceph-mon[77242]: pgmap v8211: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:02:00 smithi153 ceph-mon[70044]: pgmap v8212: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:00.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:02:00 smithi083 ceph-mon[77242]: pgmap v8212: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:00.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:02:00 smithi083 ceph-mon[79930]: pgmap v8212: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:01.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:02:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:02:01.163Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:02:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:02:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:02:01] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T10:02:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:02:02 smithi153 ceph-mon[70044]: pgmap v8213: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:02.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:02:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:02:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:02:02.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:02:02 smithi083 ceph-mon[79930]: pgmap v8213: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:02.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:02:02 smithi083 ceph-mon[77242]: pgmap v8213: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:02:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:02:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:02:04.544 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:02:04 smithi083 ceph-mon[79930]: pgmap v8214: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:04.545 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:02:04 smithi083 ceph-mon[77242]: pgmap v8214: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:04.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:02:04 smithi153 ceph-mon[70044]: pgmap v8214: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:04.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:02:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:02:04.543Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:02:05.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:02:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:02:05.254Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:02:06.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:02:06 smithi153 ceph-mon[70044]: pgmap v8215: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:06.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:02:06 smithi083 ceph-mon[79930]: pgmap v8215: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:06.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:02:06 smithi083 ceph-mon[77242]: pgmap v8215: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:02:08 smithi153 ceph-mon[70044]: pgmap v8216: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:02:08 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:02:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:02:08 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:02:08.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:02:08 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:02:08.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:02:08 smithi083 ceph-mon[79930]: pgmap v8216: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:08.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:02:08 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:02:08.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:02:08 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:02:08.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:02:08 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:02:08.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:02:08 smithi083 ceph-mon[77242]: pgmap v8216: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:08.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:02:08 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:02:08.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:02:08 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:02:08.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:02:08 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:02:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:02:10 smithi153 ceph-mon[70044]: pgmap v8217: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:10.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:02:10 smithi083 ceph-mon[79930]: pgmap v8217: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:10.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:02:10 smithi083 ceph-mon[77242]: pgmap v8217: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:11.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:02:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:02:11.164Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:02:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:02:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:02:11] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T10:02:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:02:12 smithi153 ceph-mon[70044]: pgmap v8218: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:12.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:02:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:02:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:02:12.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:02:12 smithi083 ceph-mon[79930]: pgmap v8218: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:12.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:02:12 smithi083 ceph-mon[77242]: pgmap v8218: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:02:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:02:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:02:13.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:02:13 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:02:13.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:02:13 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:02:13.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:02:13 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:02:13.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:02:13 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:02:13.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:02:13 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:02:13.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:02:13 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:02:13.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:02:13 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:02:13.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:02:13 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:02:13.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:02:13 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:02:14.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:02:14 smithi083 ceph-mon[79930]: pgmap v8219: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:14.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:02:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:02:14.543Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:02:14.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:02:14 smithi083 ceph-mon[77242]: pgmap v8219: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:14.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:02:14 smithi153 ceph-mon[70044]: pgmap v8219: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:15.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:02:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:02:15.254Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:02:16.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:02:16 smithi083 ceph-mon[79930]: pgmap v8220: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:16.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:02:16 smithi083 ceph-mon[77242]: pgmap v8220: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:16.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:02:16 smithi153 ceph-mon[70044]: pgmap v8220: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:18.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:02:18 smithi083 ceph-mon[79930]: pgmap v8221: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:18.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:02:18 smithi083 ceph-mon[77242]: pgmap v8221: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:18.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:02:18 smithi153 ceph-mon[70044]: pgmap v8221: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:20.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:02:20 smithi083 ceph-mon[79930]: pgmap v8222: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:20.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:02:20 smithi083 ceph-mon[77242]: pgmap v8222: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:20.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:02:20 smithi153 ceph-mon[70044]: pgmap v8222: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:21.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:02:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:02:21.165Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:02:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:02:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:02:21] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T10:02:22.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:02:22 smithi083 ceph-mon[79930]: pgmap v8223: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:22.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:02:22 smithi083 ceph-mon[77242]: pgmap v8223: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:22.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:02:22 smithi153 ceph-mon[70044]: pgmap v8223: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:02:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:02:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:02:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:02:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:02:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:02:24.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:02:24 smithi083 ceph-mon[79930]: pgmap v8224: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:24.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:02:24 smithi083 ceph-mon[77242]: pgmap v8224: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:02:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:02:24.543Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:02:24.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:02:24 smithi153 ceph-mon[70044]: pgmap v8224: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:25.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:02:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:02:25.255Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:02:26.522 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:02:26.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:02:26 smithi083 ceph-mon[79930]: pgmap v8225: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:26.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:02:26 smithi083 ceph-mon[77242]: pgmap v8225: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:26.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:02:26 smithi153 ceph-mon[70044]: pgmap v8225: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:26.815 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:02:26.815 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 2m ago 4h 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:02:26.816 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 14s ago 4h 63.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:02:26.816 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 2m ago 4h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:02:26.816 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 14s ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:02:26.816 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 14s ago 4h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:02:26.817 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 2m ago 4h 749M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:02:26.817 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 2m ago 4h 1054M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:02:26.817 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 14s ago 4h 454M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:02:26.817 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 2m ago 4h 604M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:02:26.818 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 2m ago 4h 25.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:02:26.818 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 14s ago 4h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:02:26.818 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 2m ago 4h 742M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:02:26.818 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 2m ago 4h 733M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:02:26.819 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 2m ago 4h 573M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:02:26.819 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 2m ago 4h 714M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:02:26.819 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 14s ago 4h 631M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:02:26.819 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 14s ago 4h 785M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:02:26.819 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 14s ago 4h 700M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:02:26.820 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 14s ago 4h 827M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:02:26.820 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 14s ago 4h 177M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:02:27.170 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:02:27.170 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:02:27.170 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:02:27.170 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:02:27.171 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:02:27.171 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:02:27.171 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:02:27.171 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:02:27.172 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:02:27.172 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:02:27.172 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:02:27.172 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:02:27.173 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:02:27.173 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:02:27.173 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:02:27.173 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:02:27.173 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:02:27.174 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:02:27.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:02:27 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/851865979' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:02:27.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:02:27 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/851865979' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:02:27.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:02:27 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/851865979' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:02:28.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:02:28 smithi083 ceph-mon[79930]: from='client.43701 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:02:28.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:02:28 smithi083 ceph-mon[79930]: pgmap v8226: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:28.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:02:28 smithi083 ceph-mon[79930]: from='client.53638 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:02:28.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:02:28 smithi083 ceph-mon[77242]: from='client.43701 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:02:28.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:02:28 smithi083 ceph-mon[77242]: pgmap v8226: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:28.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:02:28 smithi083 ceph-mon[77242]: from='client.53638 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:02:28.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:02:28 smithi153 ceph-mon[70044]: from='client.43701 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:02:28.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:02:28 smithi153 ceph-mon[70044]: pgmap v8226: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:28.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:02:28 smithi153 ceph-mon[70044]: from='client.53638 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:02:30.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:02:30 smithi083 ceph-mon[79930]: pgmap v8227: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:30.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:02:30 smithi083 ceph-mon[77242]: pgmap v8227: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:30.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:02:30 smithi153 ceph-mon[70044]: pgmap v8227: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:31.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:02:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:02:31.166Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:02:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:02:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:02:31] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T10:02:32.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:02:32 smithi083 ceph-mon[77242]: pgmap v8228: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:32.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:02:32 smithi083 ceph-mon[79930]: pgmap v8228: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:32.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:02:32 smithi153 ceph-mon[70044]: pgmap v8228: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:02:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:02:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:02:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:02:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:02:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:02:34.546 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:02:34 smithi083 ceph-mon[79930]: pgmap v8229: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:34.546 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:02:34 smithi083 ceph-mon[77242]: pgmap v8229: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:34.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:02:34 smithi153 ceph-mon[70044]: pgmap v8229: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:34.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:02:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:02:34.545Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:02:35.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:02:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:02:35.256Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:02:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:02:36 smithi153 ceph-mon[70044]: pgmap v8230: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:36.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:02:36 smithi083 ceph-mon[77242]: pgmap v8230: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:36.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:02:36 smithi083 ceph-mon[79930]: pgmap v8230: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:02:38 smithi153 ceph-mon[70044]: pgmap v8231: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:38.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:02:38 smithi083 ceph-mon[77242]: pgmap v8231: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:38.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:02:38 smithi083 ceph-mon[79930]: pgmap v8231: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:02:40 smithi153 ceph-mon[70044]: pgmap v8232: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:02:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:02:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:02:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:02:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:02:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:02:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:02:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:02:40.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:02:40 smithi083 ceph-mon[79930]: pgmap v8232: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:40.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:02:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:02:40.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:02:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:02:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:02:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:02:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:02:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:02:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:02:40 smithi083 ceph-mon[77242]: pgmap v8232: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:02:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:02:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:02:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:02:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:02:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:02:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:02:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:02:41.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:02:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:02:41.166Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:02:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:02:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:02:41] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T10:02:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:02:42 smithi153 ceph-mon[70044]: pgmap v8233: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:42.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:02:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:02:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:02:42.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:02:42 smithi083 ceph-mon[77242]: pgmap v8233: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:42.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:02:42 smithi083 ceph-mon[79930]: pgmap v8233: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:43.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:02:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:02:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:02:44.547 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:02:44 smithi083 ceph-mon[79930]: pgmap v8234: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:44.547 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:02:44 smithi083 ceph-mon[77242]: pgmap v8234: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:44.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:02:44 smithi153 ceph-mon[70044]: pgmap v8234: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:44.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:02:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:02:44.546Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:02:45.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:02:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:02:45.257Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:02:46.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:02:46 smithi153 ceph-mon[70044]: pgmap v8235: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:46.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:02:46 smithi083 ceph-mon[77242]: pgmap v8235: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:46.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:02:46 smithi083 ceph-mon[79930]: pgmap v8235: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:48.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:02:48 smithi153 ceph-mon[70044]: pgmap v8236: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:48.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:02:48 smithi083 ceph-mon[77242]: pgmap v8236: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:48.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:02:48 smithi083 ceph-mon[79930]: pgmap v8236: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:02:50 smithi153 ceph-mon[70044]: pgmap v8237: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:50.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:02:50 smithi083 ceph-mon[77242]: pgmap v8237: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:50.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:02:50 smithi083 ceph-mon[79930]: pgmap v8237: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:51.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:02:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:02:51.167Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:02:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:02:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:02:51] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T10:02:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:02:52 smithi153 ceph-mon[70044]: pgmap v8238: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:52.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:02:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:02:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:02:52.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:02:52 smithi083 ceph-mon[77242]: pgmap v8238: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:52.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:02:52 smithi083 ceph-mon[79930]: pgmap v8238: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:53.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:02:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:02:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:02:54.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:02:54 smithi083 ceph-mon[77242]: pgmap v8239: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:54.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:02:54 smithi083 ceph-mon[79930]: pgmap v8239: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:02:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:02:54.545Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:02:54.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:02:54 smithi153 ceph-mon[70044]: pgmap v8239: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:55.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:02:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:02:55.257Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:02:56.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:02:56 smithi083 ceph-mon[79930]: pgmap v8240: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:56.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:02:56 smithi083 ceph-mon[77242]: pgmap v8240: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:56.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:02:56 smithi153 ceph-mon[70044]: pgmap v8240: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:57.496 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:02:57.785 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:02:57.785 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 2m ago 4h 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:02:57.785 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 45s ago 4h 63.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:02:57.786 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 2m ago 4h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:02:57.786 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 45s ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:02:57.786 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 45s ago 4h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:02:57.786 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 2m ago 4h 749M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:02:57.787 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 2m ago 4h 1054M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:02:57.787 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 45s ago 4h 454M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:02:57.787 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 2m ago 4h 604M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:02:57.787 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 2m ago 4h 25.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:02:57.788 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 45s ago 4h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:02:57.788 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 2m ago 4h 742M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:02:57.788 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 2m ago 4h 733M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:02:57.788 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 2m ago 4h 573M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:02:57.789 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 2m ago 4h 714M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:02:57.789 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 45s ago 4h 631M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:02:57.789 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 45s ago 4h 785M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:02:57.789 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 45s ago 4h 700M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:02:57.789 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 45s ago 4h 827M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:02:57.790 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 45s ago 4h 177M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:02:58.141 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:02:58.142 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:02:58.142 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:02:58.142 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:02:58.143 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:02:58.143 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:02:58.143 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:02:58.143 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:02:58.144 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:02:58.144 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:02:58.144 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:02:58.144 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:02:58.145 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:02:58.145 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:02:58.145 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:02:58.145 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:02:58.145 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:02:58.146 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:02:58.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:02:58 smithi083 ceph-mon[77242]: pgmap v8241: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:58.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:02:58 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3350861908' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:02:58.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:02:58 smithi083 ceph-mon[79930]: pgmap v8241: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:58.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:02:58 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3350861908' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:02:58.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:02:58 smithi153 ceph-mon[70044]: pgmap v8241: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:02:58.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:02:58 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3350861908' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:02:59.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:02:59 smithi083 ceph-mon[77242]: from='client.43719 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:02:59.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:02:59 smithi083 ceph-mon[77242]: from='client.43725 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:02:59.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:02:59 smithi083 ceph-mon[79930]: from='client.43719 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:02:59.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:02:59 smithi083 ceph-mon[79930]: from='client.43725 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:02:59.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:02:59 smithi153 ceph-mon[70044]: from='client.43719 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:02:59.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:02:59 smithi153 ceph-mon[70044]: from='client.43725 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:03:00.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:03:00 smithi083 ceph-mon[79930]: pgmap v8242: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:00.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:03:00 smithi083 ceph-mon[77242]: pgmap v8242: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:00.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:03:00 smithi153 ceph-mon[70044]: pgmap v8242: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:01.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:03:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:03:01.167Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:03:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:03:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:03:01] "GET /metrics HTTP/1.1" 200 225229 "" "Prometheus/2.33.4" 2022-09-29T10:03:02.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:03:02 smithi083 ceph-mon[77242]: pgmap v8243: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:02.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:03:02 smithi083 ceph-mon[79930]: pgmap v8243: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:02.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:03:02 smithi153 ceph-mon[70044]: pgmap v8243: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:03:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:03:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:03:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:03:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:03:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:03:04.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:03:04 smithi083 ceph-mon[79930]: pgmap v8244: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:04.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:03:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:03:04.546Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:03:04.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:03:04 smithi083 ceph-mon[77242]: pgmap v8244: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:04.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:03:04 smithi153 ceph-mon[70044]: pgmap v8244: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:05.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:03:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:03:05.259Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:03:06.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:03:06 smithi083 ceph-mon[77242]: pgmap v8245: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:06.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:03:06 smithi083 ceph-mon[79930]: pgmap v8245: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:06.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:03:06 smithi153 ceph-mon[70044]: pgmap v8245: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:08.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:03:08 smithi083 ceph-mon[79930]: pgmap v8246: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:08.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:03:08 smithi083 ceph-mon[77242]: pgmap v8246: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:08.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:03:08 smithi153 ceph-mon[70044]: pgmap v8246: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:10.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:03:10 smithi083 ceph-mon[77242]: pgmap v8247: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:10.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:03:10 smithi083 ceph-mon[79930]: pgmap v8247: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:10.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:03:10 smithi153 ceph-mon[70044]: pgmap v8247: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:11.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:03:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:03:11.169Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:03:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:03:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:03:11] "GET /metrics HTTP/1.1" 200 225228 "" "Prometheus/2.33.4" 2022-09-29T10:03:12.635 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:03:12 smithi083 ceph-mon[77242]: pgmap v8248: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:12.636 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:03:12 smithi083 ceph-mon[79930]: pgmap v8248: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:12.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:03:12 smithi153 ceph-mon[70044]: pgmap v8248: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:03:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:03:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:03:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:03:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:03:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:03:13.607 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:03:13 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:03:13.607 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:03:13 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:03:13.607 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:03:13 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:03:13.608 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:03:13 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:03:13.608 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:03:13 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:03:13.608 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:03:13 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:03:13.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:03:13 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:03:13.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:03:13 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:03:13.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:03:13 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:03:14.621 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:03:14 smithi083 ceph-mon[79930]: pgmap v8249: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:14.622 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:03:14 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:03:14.622 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:03:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:03:14.547Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:03:14.623 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:03:14 smithi083 ceph-mon[77242]: pgmap v8249: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:14.623 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:03:14 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:03:14.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:03:14 smithi153 ceph-mon[70044]: pgmap v8249: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:14.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:03:14 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:03:15.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:03:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:03:15.259Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:03:16.666 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:03:16 smithi083 ceph-mon[79930]: pgmap v8250: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:16.666 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:03:16 smithi083 ceph-mon[77242]: pgmap v8250: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:16.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:03:16 smithi153 ceph-mon[70044]: pgmap v8250: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:18.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:03:18 smithi083 ceph-mon[79930]: pgmap v8251: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:18.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:03:18 smithi083 ceph-mon[77242]: pgmap v8251: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:18.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:03:18 smithi153 ceph-mon[70044]: pgmap v8251: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:20.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:03:20 smithi083 ceph-mon[77242]: pgmap v8252: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:20.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:03:20 smithi083 ceph-mon[79930]: pgmap v8252: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:20.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:03:20 smithi153 ceph-mon[70044]: pgmap v8252: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:21.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:03:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:03:21.169Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:03:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:03:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:03:21] "GET /metrics HTTP/1.1" 200 225228 "" "Prometheus/2.33.4" 2022-09-29T10:03:22.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:03:22 smithi083 ceph-mon[77242]: pgmap v8253: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:22.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:03:22 smithi083 ceph-mon[79930]: pgmap v8253: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:22.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:03:22 smithi153 ceph-mon[70044]: pgmap v8253: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:03:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:03:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:03:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:03:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:03:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:03:24.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:03:24 smithi083 ceph-mon[77242]: pgmap v8254: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:24.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:03:24 smithi083 ceph-mon[79930]: pgmap v8254: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:03:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:03:24.549Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:03:24.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:03:24 smithi153 ceph-mon[70044]: pgmap v8254: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:25.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:03:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:03:25.259Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:03:26.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:03:26 smithi083 ceph-mon[79930]: pgmap v8255: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:26.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:03:26 smithi083 ceph-mon[77242]: pgmap v8255: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:26.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:03:26 smithi153 ceph-mon[70044]: pgmap v8255: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:28.473 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:03:28.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:03:28 smithi083 ceph-mon[79930]: pgmap v8256: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:28.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:03:28 smithi083 ceph-mon[77242]: pgmap v8256: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:28.763 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:03:28.763 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 3m ago 4h 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:03:28.763 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 76s ago 4h 63.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:03:28.763 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 3m ago 4h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:03:28.764 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 76s ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:03:28.764 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 76s ago 4h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:03:28.764 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 3m ago 4h 749M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:03:28.764 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 3m ago 4h 1054M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:03:28.765 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 76s ago 4h 454M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:03:28.765 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 3m ago 4h 604M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:03:28.765 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 3m ago 4h 25.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:03:28.765 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 76s ago 4h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:03:28.765 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 3m ago 4h 742M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:03:28.766 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 3m ago 4h 733M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:03:28.766 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 3m ago 4h 573M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:03:28.766 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 3m ago 4h 714M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:03:28.766 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 76s ago 4h 631M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:03:28.767 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 76s ago 4h 785M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:03:28.767 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 76s ago 4h 700M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:03:28.767 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 76s ago 4h 827M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:03:28.767 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 76s ago 4h 177M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:03:28.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:03:28 smithi153 ceph-mon[70044]: pgmap v8256: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:29.118 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:03:29.118 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:03:29.118 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:03:29.119 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:03:29.119 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:03:29.119 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:03:29.119 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:03:29.120 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:03:29.120 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:03:29.120 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:03:29.120 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:03:29.120 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:03:29.121 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:03:29.121 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:03:29.121 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:03:29.121 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:03:29.122 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:03:29.122 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:03:29.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:03:29 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3079214052' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:03:29.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:03:29 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3079214052' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:03:29.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:03:29 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3079214052' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:03:30.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:03:30 smithi083 ceph-mon[79930]: from='client.43737 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:03:30.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:03:30 smithi083 ceph-mon[79930]: pgmap v8257: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:30.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:03:30 smithi083 ceph-mon[79930]: from='client.53674 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:03:30.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:03:30 smithi083 ceph-mon[77242]: from='client.43737 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:03:30.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:03:30 smithi083 ceph-mon[77242]: pgmap v8257: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:30.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:03:30 smithi083 ceph-mon[77242]: from='client.53674 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:03:30.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:03:30 smithi153 ceph-mon[70044]: from='client.43737 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:03:30.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:03:30 smithi153 ceph-mon[70044]: pgmap v8257: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:30.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:03:30 smithi153 ceph-mon[70044]: from='client.53674 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:03:31.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:03:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:03:31.169Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:03:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:03:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:03:31] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T10:03:32.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:03:32 smithi083 ceph-mon[79930]: pgmap v8258: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:32.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:03:32 smithi083 ceph-mon[77242]: pgmap v8258: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:32.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:03:32 smithi153 ceph-mon[70044]: pgmap v8258: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:03:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:03:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:03:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:03:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:03:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:03:34.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:03:34 smithi083 ceph-mon[79930]: pgmap v8259: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:34.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:03:34 smithi083 ceph-mon[77242]: pgmap v8259: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:34.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:03:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:03:34.550Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:03:34.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:03:34 smithi153 ceph-mon[70044]: pgmap v8259: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:35.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:03:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:03:35.261Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:03:36.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:03:36 smithi083 ceph-mon[77242]: pgmap v8260: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:36.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:03:36 smithi083 ceph-mon[79930]: pgmap v8260: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:36.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:03:36 smithi153 ceph-mon[70044]: pgmap v8260: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:38.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:03:38 smithi083 ceph-mon[79930]: pgmap v8261: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:38.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:03:38 smithi083 ceph-mon[77242]: pgmap v8261: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:38.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:03:38 smithi153 ceph-mon[70044]: pgmap v8261: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:40.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:03:40 smithi083 ceph-mon[77242]: pgmap v8262: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:40.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:03:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:03:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:03:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:03:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:03:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:03:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:03:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:03:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:03:40 smithi083 ceph-mon[79930]: pgmap v8262: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:03:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:03:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:03:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:03:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:03:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:03:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:03:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:03:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:03:40 smithi153 ceph-mon[70044]: pgmap v8262: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:03:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:03:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:03:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:03:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:03:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:03:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:03:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:03:41.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:03:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:03:41.170Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:03:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:03:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:03:41] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T10:03:42.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:03:42 smithi083 ceph-mon[79930]: pgmap v8263: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:42.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:03:42 smithi083 ceph-mon[77242]: pgmap v8263: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:03:42 smithi153 ceph-mon[70044]: pgmap v8263: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:03:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:03:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:03:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:03:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:03:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:03:44.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:03:44 smithi083 ceph-mon[79930]: pgmap v8264: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:03:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:03:44.550Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:03:44.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:03:44 smithi083 ceph-mon[77242]: pgmap v8264: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:44.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:03:44 smithi153 ceph-mon[70044]: pgmap v8264: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:45.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:03:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:03:45.261Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:03:46.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:03:46 smithi083 ceph-mon[79930]: pgmap v8265: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:46.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:03:46 smithi083 ceph-mon[77242]: pgmap v8265: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:46.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:03:46 smithi153 ceph-mon[70044]: pgmap v8265: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:48.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:03:48 smithi083 ceph-mon[79930]: pgmap v8266: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:48.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:03:48 smithi083 ceph-mon[77242]: pgmap v8266: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:48.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:03:48 smithi153 ceph-mon[70044]: pgmap v8266: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:50.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:03:50 smithi083 ceph-mon[79930]: pgmap v8267: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:50.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:03:50 smithi083 ceph-mon[77242]: pgmap v8267: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:50.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:03:50 smithi153 ceph-mon[70044]: pgmap v8267: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:51.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:03:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:03:51.171Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:03:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:03:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:03:51] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T10:03:52.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:03:52 smithi083 ceph-mon[79930]: pgmap v8268: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:52.737 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:03:52 smithi083 ceph-mon[77242]: pgmap v8268: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:52.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:03:52 smithi153 ceph-mon[70044]: pgmap v8268: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:03:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:03:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:03:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:03:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:03:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:03:54.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:03:54 smithi083 ceph-mon[79930]: pgmap v8269: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:03:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:03:54.550Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:03:54.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:03:54 smithi083 ceph-mon[77242]: pgmap v8269: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:54.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:03:54 smithi153 ceph-mon[70044]: pgmap v8269: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:55.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:03:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:03:55.262Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:03:56.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:03:56 smithi083 ceph-mon[79930]: pgmap v8270: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:56.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:03:56 smithi083 ceph-mon[77242]: pgmap v8270: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:56.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:03:56 smithi153 ceph-mon[70044]: pgmap v8270: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:58.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:03:58 smithi083 ceph-mon[79930]: pgmap v8271: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:58.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:03:58 smithi083 ceph-mon[77242]: pgmap v8271: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:58.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:03:58 smithi153 ceph-mon[70044]: pgmap v8271: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:03:59.446 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:03:59.734 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:03:59.735 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 3m ago 4h 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:03:59.735 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 107s ago 4h 63.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:03:59.735 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 3m ago 4h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:03:59.735 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 107s ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:03:59.736 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 107s ago 4h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:03:59.736 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 3m ago 4h 749M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:03:59.736 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 3m ago 4h 1054M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:03:59.736 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 107s ago 4h 454M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:03:59.737 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 3m ago 4h 604M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:03:59.737 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 3m ago 4h 25.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:03:59.737 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 107s ago 4h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:03:59.737 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 3m ago 4h 742M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:03:59.737 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 3m ago 4h 733M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:03:59.738 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 3m ago 4h 573M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:03:59.738 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 3m ago 4h 714M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:03:59.738 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 107s ago 4h 631M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:03:59.738 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 107s ago 4h 785M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:03:59.739 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 107s ago 4h 700M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:03:59.739 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 107s ago 4h 827M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:03:59.739 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 107s ago 4h 177M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:04:00.090 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:04:00.090 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:04:00.091 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:04:00.091 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:04:00.091 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:04:00.091 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:04:00.092 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:04:00.092 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:04:00.092 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:04:00.092 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:04:00.093 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:04:00.093 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:04:00.093 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:04:00.093 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:04:00.094 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:04:00.094 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:04:00.094 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:04:00.094 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:04:00.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:04:00 smithi083 ceph-mon[79930]: pgmap v8272: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:04:00.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:04:00 smithi083 ceph-mon[79930]: from='client.53686 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:04:00.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:04:00 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4078709364' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:04:00.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:04:00 smithi083 ceph-mon[77242]: pgmap v8272: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:04:00.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:04:00 smithi083 ceph-mon[77242]: from='client.53686 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:04:00.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:04:00 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4078709364' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:04:00.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:04:00 smithi153 ceph-mon[70044]: pgmap v8272: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:04:00.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:04:00 smithi153 ceph-mon[70044]: from='client.53686 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:04:00.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:04:00 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4078709364' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:04:01.427 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:04:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:04:01.171Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:04:01.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:04:01 smithi083 ceph-mon[77242]: from='client.43761 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:04:01.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:04:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:04:01] "GET /metrics HTTP/1.1" 200 225229 "" "Prometheus/2.33.4" 2022-09-29T10:04:01.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:04:01 smithi083 ceph-mon[79930]: from='client.43761 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:04:01.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:04:01 smithi153 ceph-mon[70044]: from='client.43761 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:04:02.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:04:02 smithi083 ceph-mon[79930]: pgmap v8273: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:04:02.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:04:02 smithi083 ceph-mon[77242]: pgmap v8273: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:04:02.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:04:02 smithi153 ceph-mon[70044]: pgmap v8273: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:04:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:04:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:04:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:04:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:04:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:04:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:04:04.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:04:04 smithi083 ceph-mon[79930]: pgmap v8274: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:04:04.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:04:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:04:04.551Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:04:04.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:04:04 smithi083 ceph-mon[77242]: pgmap v8274: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:04:04.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:04:04 smithi153 ceph-mon[70044]: pgmap v8274: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:04:05.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:04:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:04:05.263Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:04:06.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:04:06 smithi083 ceph-mon[79930]: pgmap v8275: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:04:06.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:04:06 smithi083 ceph-mon[77242]: pgmap v8275: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:04:06.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:04:06 smithi153 ceph-mon[70044]: pgmap v8275: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:04:08.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:04:08 smithi083 ceph-mon[79930]: pgmap v8276: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:04:08.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:04:08 smithi083 ceph-mon[77242]: pgmap v8276: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:04:08.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:04:08 smithi153 ceph-mon[70044]: pgmap v8276: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:04:10.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:04:10 smithi083 ceph-mon[77242]: pgmap v8277: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:04:10.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:04:10 smithi083 ceph-mon[79930]: pgmap v8277: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:04:10.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:04:10 smithi153 ceph-mon[70044]: pgmap v8277: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:04:11.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:04:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:04:11.172Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:04:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:04:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:04:11] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T10:04:12.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:04:12 smithi083 ceph-mon[79930]: pgmap v8278: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:04:12.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:04:12 smithi083 ceph-mon[77242]: pgmap v8278: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:04:12.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:04:12 smithi153 ceph-mon[70044]: pgmap v8278: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:04:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:04:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:04:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:04:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:04:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:04:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:04:14.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:04:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:04:14.551Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:04:14.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:04:14 smithi083 ceph-mon[79930]: pgmap v8279: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:04:14.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:04:14 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:04:14.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:04:14 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:04:14.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:04:14 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:04:14.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:04:14 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:04:14.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:04:14 smithi083 ceph-mon[77242]: pgmap v8279: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:04:14.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:04:14 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:04:14.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:04:14 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:04:14.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:04:14 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:04:14.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:04:14 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:04:14.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:04:14 smithi153 ceph-mon[70044]: pgmap v8279: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:04:14.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:04:14 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:04:14.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:04:14 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:04:14.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:04:14 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:04:14.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:04:14 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:04:15.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:04:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:04:15.264Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:04:16.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:04:16 smithi083 ceph-mon[79930]: pgmap v8280: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:04:16.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:04:16 smithi083 ceph-mon[77242]: pgmap v8280: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:04:16.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:04:16 smithi153 ceph-mon[70044]: pgmap v8280: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:04:18.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:04:18 smithi083 ceph-mon[79930]: pgmap v8281: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:04:18.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:04:18 smithi083 ceph-mon[77242]: pgmap v8281: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:04:18.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:04:18 smithi153 ceph-mon[70044]: pgmap v8281: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:04:20.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:04:20 smithi083 ceph-mon[79930]: pgmap v8282: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:04:20.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:04:20 smithi083 ceph-mon[77242]: pgmap v8282: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:04:20.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:04:20 smithi153 ceph-mon[70044]: pgmap v8282: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:04:21.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:04:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:04:21.173Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:04:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:04:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:04:21] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T10:04:22.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:04:22 smithi083 ceph-mon[79930]: pgmap v8283: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:04:22.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:04:22 smithi083 ceph-mon[77242]: pgmap v8283: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:04:22.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:04:22 smithi153 ceph-mon[70044]: pgmap v8283: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:04:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:04:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:04:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:04:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:04:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:04:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:04:24.553 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:04:24 smithi083 ceph-mon[79930]: pgmap v8284: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:04:24.554 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:04:24 smithi083 ceph-mon[77242]: pgmap v8284: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:04:24.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:04:24 smithi153 ceph-mon[70044]: pgmap v8284: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:04:24.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:04:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:04:24.552Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:04:25.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:04:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:04:25.264Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:04:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:04:26 smithi153 ceph-mon[70044]: pgmap v8285: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:04:26.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:04:26 smithi083 ceph-mon[79930]: pgmap v8285: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:04:26.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:04:26 smithi083 ceph-mon[77242]: pgmap v8285: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:04:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:04:28 smithi153 ceph-mon[70044]: pgmap v8286: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:04:28.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:04:28 smithi083 ceph-mon[79930]: pgmap v8286: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:04:28.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:04:28 smithi083 ceph-mon[77242]: pgmap v8286: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:04:30.419 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:04:30.707 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:04:30.708 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 4m ago 4h 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:04:30.708 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 2m ago 4h 63.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:04:30.708 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 4m ago 4h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:04:30.709 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 2m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:04:30.709 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 2m ago 4h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:04:30.709 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 4m ago 4h 749M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:04:30.709 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 4m ago 4h 1054M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:04:30.710 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 2m ago 4h 454M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:04:30.710 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 4m ago 4h 604M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:04:30.710 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 4m ago 4h 25.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:04:30.710 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 2m ago 4h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:04:30.711 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 4m ago 4h 742M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:04:30.711 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 4m ago 4h 733M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:04:30.711 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 4m ago 4h 573M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:04:30.711 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 4m ago 4h 714M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:04:30.712 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 2m ago 4h 631M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:04:30.712 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 2m ago 4h 785M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:04:30.712 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 2m ago 4h 700M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:04:30.712 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 2m ago 4h 827M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:04:30.713 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 2m ago 4h 177M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:04:30.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:04:30 smithi083 ceph-mon[79930]: pgmap v8287: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:04:30.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:04:30 smithi083 ceph-mon[77242]: pgmap v8287: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:04:30.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:04:30 smithi153 ceph-mon[70044]: pgmap v8287: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:04:31.059 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:04:31.060 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:04:31.060 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:04:31.060 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:04:31.060 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:04:31.061 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:04:31.061 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:04:31.061 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:04:31.061 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:04:31.062 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:04:31.062 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:04:31.062 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:04:31.063 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:04:31.063 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:04:31.063 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:04:31.063 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:04:31.064 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:04:31.064 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:04:31.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:04:31 smithi083 ceph-mon[77242]: from='client.43773 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:04:31.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:04:31 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2726279655' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:04:31.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:04:31 smithi083 ceph-mon[79930]: from='client.43773 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:04:31.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:04:31 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2726279655' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:04:31.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:04:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:04:31.173Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:04:31.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:04:31 smithi153 ceph-mon[70044]: from='client.43773 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:04:31.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:04:31 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2726279655' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:04:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:04:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:04:31] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T10:04:32.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:04:32 smithi083 ceph-mon[77242]: pgmap v8288: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:04:32.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:04:32 smithi083 ceph-mon[77242]: from='client.43779 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:04:32.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:04:32 smithi083 ceph-mon[79930]: pgmap v8288: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:04:32.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:04:32 smithi083 ceph-mon[79930]: from='client.43779 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:04:32.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:04:32 smithi153 ceph-mon[70044]: pgmap v8288: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:04:32.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:04:32 smithi153 ceph-mon[70044]: from='client.43779 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:04:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:04:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:04:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:04:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:04:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:04:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:04:34.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:04:34 smithi083 ceph-mon[77242]: pgmap v8289: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:04:34.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:04:34 smithi083 ceph-mon[79930]: pgmap v8289: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:04:34.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:04:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:04:34.553Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:04:34.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:04:34 smithi153 ceph-mon[70044]: pgmap v8289: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:04:35.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:04:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:04:35.266Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:04:36.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:04:36 smithi083 ceph-mon[79930]: pgmap v8290: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:04:36.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:04:36 smithi083 ceph-mon[77242]: pgmap v8290: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:04:36.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:04:36 smithi153 ceph-mon[70044]: pgmap v8290: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:04:38.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:04:38 smithi083 ceph-mon[79930]: pgmap v8291: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:04:38.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:04:38 smithi083 ceph-mon[77242]: pgmap v8291: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:04:38.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:04:38 smithi153 ceph-mon[70044]: pgmap v8291: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:04:40.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:04:40 smithi083 ceph-mon[79930]: pgmap v8292: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:04:40.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:04:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:04:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:04:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:04:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:04:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:04:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:04:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:04:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:04:40 smithi083 ceph-mon[77242]: pgmap v8292: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:04:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:04:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:04:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:04:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:04:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:04:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:04:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:04:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:04:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:04:40 smithi153 ceph-mon[70044]: pgmap v8292: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:04:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:04:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:04:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:04:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:04:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:04:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:04:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:04:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:04:41.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:04:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:04:41.174Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:04:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:04:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:04:41] "GET /metrics HTTP/1.1" 200 225226 "" "Prometheus/2.33.4" 2022-09-29T10:04:42.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:04:42 smithi083 ceph-mon[77242]: pgmap v8293: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:04:42.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:04:42 smithi083 ceph-mon[79930]: pgmap v8293: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:04:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:04:42 smithi153 ceph-mon[70044]: pgmap v8293: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:04:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:04:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:04:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:04:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:04:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:04:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:04:44.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:04:44 smithi083 ceph-mon[77242]: pgmap v8294: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:04:44.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:04:44 smithi083 ceph-mon[79930]: pgmap v8294: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:04:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:04:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:04:44.554Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:04:44.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:04:44 smithi153 ceph-mon[70044]: pgmap v8294: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:04:45.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:04:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:04:45.266Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:04:46.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:04:46 smithi083 ceph-mon[79930]: pgmap v8295: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:04:46.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:04:46 smithi083 ceph-mon[77242]: pgmap v8295: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:04:46.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:04:46 smithi153 ceph-mon[70044]: pgmap v8295: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:04:48.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:04:48 smithi083 ceph-mon[79930]: pgmap v8296: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:04:48.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:04:48 smithi083 ceph-mon[77242]: pgmap v8296: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:04:48.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:04:48 smithi153 ceph-mon[70044]: pgmap v8296: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:04:50.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:04:50 smithi083 ceph-mon[79930]: pgmap v8297: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:04:50.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:04:50 smithi083 ceph-mon[77242]: pgmap v8297: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:04:50.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:04:50 smithi153 ceph-mon[70044]: pgmap v8297: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:04:51.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:04:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:04:51.175Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:04:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:04:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:04:51] "GET /metrics HTTP/1.1" 200 225226 "" "Prometheus/2.33.4" 2022-09-29T10:04:52.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:04:52 smithi083 ceph-mon[77242]: pgmap v8298: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:04:52.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:04:52 smithi083 ceph-mon[79930]: pgmap v8298: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:04:52.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:04:52 smithi153 ceph-mon[70044]: pgmap v8298: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:04:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:04:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:04:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:04:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:04:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:04:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:04:54.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:04:54 smithi083 ceph-mon[79930]: pgmap v8299: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:04:54.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:04:54 smithi083 ceph-mon[77242]: pgmap v8299: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:04:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:04:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:04:54.554Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:04:54.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:04:54 smithi153 ceph-mon[70044]: pgmap v8299: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:04:55.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:04:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:04:55.266Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:04:56.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:04:56 smithi083 ceph-mon[79930]: pgmap v8300: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:04:56.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:04:56 smithi083 ceph-mon[77242]: pgmap v8300: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:04:56.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:04:56 smithi153 ceph-mon[70044]: pgmap v8300: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:04:58.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:04:58 smithi083 ceph-mon[79930]: pgmap v8301: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:04:58.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:04:58 smithi083 ceph-mon[77242]: pgmap v8301: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:04:58.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:04:58 smithi153 ceph-mon[70044]: pgmap v8301: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:00.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:05:00 smithi083 ceph-mon[79930]: pgmap v8302: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:00.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:05:00 smithi083 ceph-mon[77242]: pgmap v8302: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:00.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:05:00 smithi153 ceph-mon[70044]: pgmap v8302: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:01.386 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:05:01.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:05:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:05:01.176Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:05:01.676 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:05:01.677 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 4m ago 4h 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:05:01.677 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 2m ago 4h 63.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:05:01.677 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 4m ago 4h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:05:01.677 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 2m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:05:01.678 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 2m ago 4h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:05:01.678 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 4m ago 4h 749M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:05:01.678 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 4m ago 4h 1054M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:05:01.678 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 2m ago 4h 454M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:05:01.679 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 4m ago 4h 604M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:05:01.679 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 4m ago 4h 25.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:05:01.679 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 2m ago 4h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:05:01.679 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 4m ago 4h 742M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:05:01.680 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 4m ago 4h 733M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:05:01.680 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 4m ago 4h 573M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:05:01.680 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 4m ago 4h 714M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:05:01.680 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 2m ago 4h 631M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:05:01.680 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 2m ago 4h 785M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:05:01.681 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 2m ago 4h 700M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:05:01.681 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 2m ago 4h 827M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:05:01.681 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 2m ago 4h 177M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:05:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:05:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:05:01] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T10:05:02.030 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:05:02.031 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:05:02.031 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:05:02.031 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:05:02.032 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:05:02.032 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:05:02.032 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:05:02.032 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:05:02.033 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:05:02.033 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:05:02.033 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:05:02.033 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:05:02.033 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:05:02.034 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:05:02.034 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:05:02.034 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:05:02.034 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:05:02.035 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:05:02.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:05:02 smithi083 ceph-mon[79930]: pgmap v8303: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:02.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:05:02 smithi083 ceph-mon[79930]: from='client.53722 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:05:02.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:05:02 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1263213442' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:05:02.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:05:02 smithi083 ceph-mon[77242]: pgmap v8303: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:02.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:05:02 smithi083 ceph-mon[77242]: from='client.53722 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:05:02.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:05:02 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1263213442' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:05:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:05:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:05:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:05:02.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:05:02 smithi153 ceph-mon[70044]: pgmap v8303: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:02.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:05:02 smithi153 ceph-mon[70044]: from='client.53722 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:05:02.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:05:02 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1263213442' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:05:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:05:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:05:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:05:03.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:05:03 smithi083 ceph-mon[79930]: from='client.43797 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:05:03.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:05:03 smithi083 ceph-mon[77242]: from='client.43797 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:05:03.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:05:03 smithi153 ceph-mon[70044]: from='client.43797 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:05:04.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:05:04 smithi083 ceph-mon[79930]: pgmap v8304: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:04.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:05:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:05:04.555Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:05:04.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:05:04 smithi083 ceph-mon[77242]: pgmap v8304: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:04.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:05:04 smithi153 ceph-mon[70044]: pgmap v8304: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:05.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:05:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:05:05.268Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:05:06.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:05:06 smithi083 ceph-mon[79930]: pgmap v8305: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:06.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:05:06 smithi083 ceph-mon[77242]: pgmap v8305: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:06.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:05:06 smithi153 ceph-mon[70044]: pgmap v8305: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:08.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:05:08 smithi083 ceph-mon[79930]: pgmap v8306: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:08.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:05:08 smithi083 ceph-mon[77242]: pgmap v8306: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:08.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:05:08 smithi153 ceph-mon[70044]: pgmap v8306: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:10.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:05:10 smithi083 ceph-mon[79930]: pgmap v8307: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:10.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:05:10 smithi083 ceph-mon[77242]: pgmap v8307: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:10.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:05:10 smithi153 ceph-mon[70044]: pgmap v8307: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:11.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:05:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:05:11.177Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:05:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:05:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:05:11] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T10:05:12.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:05:12 smithi083 ceph-mon[77242]: pgmap v8308: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:12.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:05:12 smithi083 ceph-mon[79930]: pgmap v8308: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:12.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:05:12 smithi153 ceph-mon[70044]: pgmap v8308: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:05:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:05:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:05:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:05:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:05:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:05:14.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:05:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:05:14.556Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:05:14.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:05:14 smithi083 ceph-mon[77242]: pgmap v8309: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:14.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:05:14 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:05:14.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:05:14 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:05:14.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:05:14 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:05:14.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:05:14 smithi083 ceph-mon[79930]: pgmap v8309: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:14.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:05:14 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:05:14.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:05:14 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:05:14.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:05:14 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:05:14.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:05:14 smithi153 ceph-mon[70044]: pgmap v8309: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:14.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:05:14 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:05:14.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:05:14 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:05:14.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:05:14 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:05:15.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:05:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:05:15.267Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:05:16.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:05:15 smithi083 ceph-mon[79930]: pgmap v8310: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:16.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:05:15 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:05:16.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:05:15 smithi083 ceph-mon[77242]: pgmap v8310: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:16.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:05:15 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:05:16.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:05:15 smithi153 ceph-mon[70044]: pgmap v8310: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:16.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:05:15 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:05:18.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:05:17 smithi083 ceph-mon[79930]: pgmap v8311: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:18.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:05:17 smithi083 ceph-mon[77242]: pgmap v8311: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:18.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:05:17 smithi153 ceph-mon[70044]: pgmap v8311: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:20.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:05:20 smithi083 ceph-mon[77242]: pgmap v8312: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:20.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:05:20 smithi083 ceph-mon[79930]: pgmap v8312: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:20.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:05:20 smithi153 ceph-mon[70044]: pgmap v8312: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:21.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:05:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:05:21.178Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:05:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:05:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:05:21] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T10:05:22.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:05:22 smithi083 ceph-mon[77242]: pgmap v8313: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:22.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:05:22 smithi083 ceph-mon[79930]: pgmap v8313: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:22.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:05:22 smithi153 ceph-mon[70044]: pgmap v8313: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:05:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:05:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:05:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:05:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:05:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:05:24.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:05:24 smithi083 ceph-mon[79930]: pgmap v8314: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:05:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:05:24.557Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:05:24.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:05:24 smithi083 ceph-mon[77242]: pgmap v8314: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:24.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:05:24 smithi153 ceph-mon[70044]: pgmap v8314: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:25.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:05:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:05:25.268Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:05:26.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:05:26 smithi083 ceph-mon[79930]: pgmap v8315: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:26.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:05:26 smithi083 ceph-mon[77242]: pgmap v8315: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:26.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:05:26 smithi153 ceph-mon[70044]: pgmap v8315: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:28.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:05:28 smithi083 ceph-mon[79930]: pgmap v8316: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:28.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:05:28 smithi083 ceph-mon[77242]: pgmap v8316: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:28.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:05:28 smithi153 ceph-mon[70044]: pgmap v8316: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:30.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:05:30 smithi083 ceph-mon[79930]: pgmap v8317: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:30.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:05:30 smithi083 ceph-mon[77242]: pgmap v8317: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:30.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:05:30 smithi153 ceph-mon[70044]: pgmap v8317: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:31.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:05:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:05:31.178Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:05:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:05:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:05:31] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T10:05:32.357 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:05:32.645 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:05:32.645 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 5m ago 4h 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:05:32.645 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 3m ago 4h 63.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:05:32.646 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 5m ago 4h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:05:32.646 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 3m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:05:32.646 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 3m ago 4h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:05:32.647 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 5m ago 4h 749M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:05:32.647 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 5m ago 4h 1054M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:05:32.647 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 3m ago 4h 454M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:05:32.647 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 5m ago 4h 604M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:05:32.648 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 5m ago 4h 25.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:05:32.648 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 3m ago 4h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:05:32.648 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 5m ago 4h 742M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:05:32.648 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 5m ago 4h 733M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:05:32.649 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 5m ago 4h 573M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:05:32.649 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 5m ago 4h 714M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:05:32.649 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 3m ago 4h 631M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:05:32.649 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 3m ago 4h 785M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:05:32.650 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 3m ago 4h 700M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:05:32.650 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 3m ago 4h 827M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:05:32.650 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 3m ago 4h 177M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:05:32.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:05:32 smithi083 ceph-mon[77242]: pgmap v8318: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:32.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:05:32 smithi083 ceph-mon[79930]: pgmap v8318: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:32.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:05:32 smithi153 ceph-mon[70044]: pgmap v8318: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:05:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:05:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:05:33.002 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:05:33.003 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:05:33.003 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:05:33.003 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:05:33.004 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:05:33.004 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:05:33.004 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:05:33.004 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:05:33.004 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:05:33.005 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:05:33.005 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:05:33.005 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:05:33.005 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:05:33.006 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:05:33.006 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:05:33.006 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:05:33.006 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:05:33.007 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:05:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:05:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:05:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:05:33.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:05:33 smithi083 ceph-mon[79930]: from='client.43809 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:05:33.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:05:33 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1580549069' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:05:33.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:05:33 smithi083 ceph-mon[77242]: from='client.43809 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:05:33.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:05:33 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1580549069' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:05:33.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:05:33 smithi153 ceph-mon[70044]: from='client.43809 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:05:33.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:05:33 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1580549069' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:05:34.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:05:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:05:34.558Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:05:34.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:05:34 smithi083 ceph-mon[77242]: from='client.43815 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:05:34.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:05:34 smithi083 ceph-mon[77242]: pgmap v8319: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:34.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:05:34 smithi083 ceph-mon[79930]: from='client.43815 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:05:34.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:05:34 smithi083 ceph-mon[79930]: pgmap v8319: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:34.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:05:34 smithi153 ceph-mon[70044]: from='client.43815 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:05:34.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:05:34 smithi153 ceph-mon[70044]: pgmap v8319: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:35.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:05:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:05:35.269Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:05:36.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:05:36 smithi083 ceph-mon[77242]: pgmap v8320: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:36.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:05:36 smithi083 ceph-mon[79930]: pgmap v8320: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:36.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:05:36 smithi153 ceph-mon[70044]: pgmap v8320: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:38.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:05:38 smithi083 ceph-mon[77242]: pgmap v8321: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:38.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:05:38 smithi083 ceph-mon[79930]: pgmap v8321: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:38.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:05:38 smithi153 ceph-mon[70044]: pgmap v8321: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:40.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:05:40 smithi083 ceph-mon[79930]: pgmap v8322: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:40.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:05:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:05:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:05:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:05:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:05:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:05:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:05:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:05:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:05:40 smithi083 ceph-mon[77242]: pgmap v8322: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:05:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:05:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:05:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:05:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:05:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:05:40.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:05:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:05:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:05:40 smithi153 ceph-mon[70044]: pgmap v8322: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:05:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:05:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:05:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:05:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:05:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:05:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:05:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:05:41.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:05:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:05:41.178Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:05:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:05:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:05:41] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T10:05:42.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:05:42 smithi083 ceph-mon[79930]: pgmap v8323: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:42.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:05:42 smithi083 ceph-mon[77242]: pgmap v8323: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:05:42 smithi153 ceph-mon[70044]: pgmap v8323: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:05:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:05:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:05:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:05:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:05:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:05:44.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:05:44 smithi083 ceph-mon[77242]: pgmap v8324: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:44.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:05:44 smithi083 ceph-mon[79930]: pgmap v8324: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:05:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:05:44.559Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:05:44.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:05:44 smithi153 ceph-mon[70044]: pgmap v8324: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:45.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:05:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:05:45.270Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:05:46.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:05:46 smithi083 ceph-mon[79930]: pgmap v8325: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:46.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:05:46 smithi083 ceph-mon[77242]: pgmap v8325: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:46.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:05:46 smithi153 ceph-mon[70044]: pgmap v8325: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:48.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:05:48 smithi083 ceph-mon[77242]: pgmap v8326: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:48.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:05:48 smithi083 ceph-mon[79930]: pgmap v8326: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:48.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:05:48 smithi153 ceph-mon[70044]: pgmap v8326: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:50.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:05:50 smithi083 ceph-mon[79930]: pgmap v8327: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:50.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:05:50 smithi083 ceph-mon[77242]: pgmap v8327: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:50.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:05:50 smithi153 ceph-mon[70044]: pgmap v8327: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:51.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:05:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:05:51.179Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:05:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:05:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:05:51] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T10:05:52.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:05:52 smithi083 ceph-mon[79930]: pgmap v8328: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:52.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:05:52 smithi083 ceph-mon[77242]: pgmap v8328: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:52.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:05:52 smithi153 ceph-mon[70044]: pgmap v8328: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:05:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:05:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:05:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:05:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:05:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:05:54.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:05:54 smithi083 ceph-mon[79930]: pgmap v8329: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:54.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:05:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:05:54.559Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:05:54.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:05:54 smithi083 ceph-mon[77242]: pgmap v8329: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:54.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:05:54 smithi153 ceph-mon[70044]: pgmap v8329: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:55.714 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:05:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:05:55.270Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:05:56.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:05:56 smithi083 ceph-mon[79930]: pgmap v8330: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:56.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:05:56 smithi083 ceph-mon[77242]: pgmap v8330: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:56.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:05:56 smithi153 ceph-mon[70044]: pgmap v8330: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:58.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:05:58 smithi083 ceph-mon[79930]: pgmap v8331: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:58.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:05:58 smithi083 ceph-mon[77242]: pgmap v8331: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:05:58.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:05:58 smithi153 ceph-mon[70044]: pgmap v8331: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:00.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:06:00 smithi083 ceph-mon[79930]: pgmap v8332: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:00.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:06:00 smithi083 ceph-mon[77242]: pgmap v8332: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:00.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:06:00 smithi153 ceph-mon[70044]: pgmap v8332: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:01.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:06:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:06:01.180Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:06:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:06:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:06:01] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T10:06:02.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:06:02 smithi083 ceph-mon[79930]: pgmap v8333: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:02.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:06:02 smithi083 ceph-mon[77242]: pgmap v8333: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:02.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:06:02 smithi153 ceph-mon[70044]: pgmap v8333: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:06:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:06:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:06:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:06:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:06:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:06:03.331 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:06:03.626 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:06:03.627 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 5m ago 4h 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:06:03.627 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 3m ago 4h 63.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:06:03.627 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 5m ago 4h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:06:03.628 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 3m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:06:03.628 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 3m ago 4h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:06:03.628 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 5m ago 4h 749M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:06:03.628 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 5m ago 4h 1054M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:06:03.629 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 3m ago 4h 454M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:06:03.629 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 5m ago 4h 604M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:06:03.629 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 5m ago 4h 25.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:06:03.629 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 3m ago 4h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:06:03.630 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 5m ago 4h 742M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:06:03.630 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 5m ago 4h 733M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:06:03.630 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 5m ago 4h 573M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:06:03.630 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 5m ago 4h 714M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:06:03.630 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 3m ago 4h 631M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:06:03.631 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 3m ago 4h 785M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:06:03.631 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 3m ago 4h 700M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:06:03.631 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 3m ago 4h 827M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:06:03.631 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 3m ago 4h 177M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:06:03.990 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:06:03.991 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:06:03.991 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:06:03.991 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:06:03.991 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:06:03.992 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:06:03.992 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:06:03.992 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:06:03.992 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:06:03.993 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:06:03.993 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:06:03.993 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:06:03.993 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:06:03.993 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:06:03.994 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:06:03.994 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:06:03.994 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:06:03.995 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:06:04.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:06:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:06:04.559Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:06:04.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:06:04 smithi083 ceph-mon[79930]: pgmap v8334: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:04.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:06:04 smithi083 ceph-mon[79930]: from='client.43827 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:06:04.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:06:04 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1328613154' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:06:04.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:06:04 smithi083 ceph-mon[77242]: pgmap v8334: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:04.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:06:04 smithi083 ceph-mon[77242]: from='client.43827 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:06:04.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:06:04 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1328613154' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:06:04.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:06:04 smithi153 ceph-mon[70044]: pgmap v8334: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:04.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:06:04 smithi153 ceph-mon[70044]: from='client.43827 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:06:04.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:06:04 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1328613154' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:06:05.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:06:05 smithi083 ceph-mon[79930]: from='client.53764 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:06:05.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:06:05 smithi083 ceph-mon[77242]: from='client.53764 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:06:05.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:06:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:06:05.271Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:06:05.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:06:05 smithi153 ceph-mon[70044]: from='client.53764 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:06:06.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:06:06 smithi083 ceph-mon[79930]: pgmap v8335: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:06.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:06:06 smithi083 ceph-mon[77242]: pgmap v8335: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:06.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:06:06 smithi153 ceph-mon[70044]: pgmap v8335: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:08.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:06:08 smithi083 ceph-mon[79930]: pgmap v8336: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:08.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:06:08 smithi083 ceph-mon[77242]: pgmap v8336: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:08.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:06:08 smithi153 ceph-mon[70044]: pgmap v8336: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:10.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:06:10 smithi083 ceph-mon[79930]: pgmap v8337: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:10.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:06:10 smithi083 ceph-mon[77242]: pgmap v8337: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:10.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:06:10 smithi153 ceph-mon[70044]: pgmap v8337: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:11.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:06:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:06:11.180Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:06:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:06:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:06:11] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T10:06:12.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:06:12 smithi083 ceph-mon[79930]: pgmap v8338: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:12.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:06:12 smithi083 ceph-mon[77242]: pgmap v8338: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:12.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:06:12 smithi153 ceph-mon[70044]: pgmap v8338: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:06:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:06:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:06:13.317 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:06:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:06:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:06:14.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:06:14 smithi083 ceph-mon[79930]: pgmap v8339: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:14.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:06:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:06:14.561Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:06:14.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:06:14 smithi083 ceph-mon[77242]: pgmap v8339: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:14.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:06:14 smithi153 ceph-mon[70044]: pgmap v8339: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:15.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:06:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:06:15.271Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:06:15.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:06:15 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:06:15.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:06:15 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:06:15.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:06:15 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:06:15.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:06:15 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:06:15.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:06:15 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:06:15.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:06:15 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:06:15.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:06:15 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:06:15.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:06:15 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:06:15.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:06:15 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:06:16.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:06:16 smithi083 ceph-mon[77242]: pgmap v8340: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:16.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:06:16 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:06:16.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:06:16 smithi083 ceph-mon[79930]: pgmap v8340: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:16.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:06:16 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:06:16.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:06:16 smithi153 ceph-mon[70044]: pgmap v8340: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:16.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:06:16 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:06:18.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:06:18 smithi083 ceph-mon[79930]: pgmap v8341: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:18.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:06:18 smithi083 ceph-mon[77242]: pgmap v8341: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:18.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:06:18 smithi153 ceph-mon[70044]: pgmap v8341: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:20.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:06:20 smithi083 ceph-mon[79930]: pgmap v8342: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:20.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:06:20 smithi083 ceph-mon[77242]: pgmap v8342: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:20.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:06:20 smithi153 ceph-mon[70044]: pgmap v8342: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:21.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:06:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:06:21.180Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:06:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:06:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:06:21] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T10:06:22.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:06:22 smithi083 ceph-mon[79930]: pgmap v8343: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:22.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:06:22 smithi083 ceph-mon[77242]: pgmap v8343: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:22.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:06:22 smithi153 ceph-mon[70044]: pgmap v8343: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:06:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:06:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:06:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:06:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:06:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:06:24.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:06:24 smithi083 ceph-mon[79930]: pgmap v8344: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:06:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:06:24.561Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:06:24.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:06:24 smithi083 ceph-mon[77242]: pgmap v8344: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:24.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:06:24 smithi153 ceph-mon[70044]: pgmap v8344: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:25.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:06:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:06:25.271Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:06:26.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:06:26 smithi083 ceph-mon[79930]: pgmap v8345: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:26.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:06:26 smithi083 ceph-mon[77242]: pgmap v8345: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:26.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:06:26 smithi153 ceph-mon[70044]: pgmap v8345: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:28.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:06:28 smithi083 ceph-mon[77242]: pgmap v8346: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:28.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:06:28 smithi083 ceph-mon[79930]: pgmap v8346: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:28.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:06:28 smithi153 ceph-mon[70044]: pgmap v8346: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:30.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:06:30 smithi083 ceph-mon[79930]: pgmap v8347: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:30.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:06:30 smithi083 ceph-mon[77242]: pgmap v8347: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:30.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:06:30 smithi153 ceph-mon[70044]: pgmap v8347: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:31.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:06:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:06:31.181Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:06:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:06:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:06:31] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T10:06:32.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:06:32 smithi083 ceph-mon[79930]: pgmap v8348: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:32.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:06:32 smithi083 ceph-mon[77242]: pgmap v8348: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:32.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:06:32 smithi153 ceph-mon[70044]: pgmap v8348: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:06:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:06:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:06:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:06:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:06:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:06:34.316 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:06:34.606 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:06:34.606 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 6m ago 4h 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:06:34.607 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 4m ago 4h 63.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:06:34.607 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 6m ago 4h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:06:34.607 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 4m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:06:34.607 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 4m ago 4h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:06:34.608 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 6m ago 4h 749M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:06:34.608 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 6m ago 4h 1054M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:06:34.608 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 4m ago 4h 454M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:06:34.608 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 6m ago 4h 604M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:06:34.609 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 6m ago 4h 25.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:06:34.609 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 4m ago 4h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:06:34.609 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 6m ago 4h 742M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:06:34.609 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 6m ago 4h 733M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:06:34.610 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 6m ago 4h 573M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:06:34.610 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 6m ago 4h 714M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:06:34.610 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 4m ago 4h 631M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:06:34.610 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 4m ago 4h 785M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:06:34.611 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 4m ago 4h 700M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:06:34.611 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 4m ago 4h 827M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:06:34.611 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 4m ago 4h 177M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:06:34.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:06:34 smithi083 ceph-mon[79930]: pgmap v8349: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:34.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:06:34 smithi083 ceph-mon[77242]: pgmap v8349: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:34.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:06:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:06:34.561Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:06:34.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:06:34 smithi153 ceph-mon[70044]: pgmap v8349: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:34.966 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:06:34.967 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:06:34.967 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:06:34.967 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:06:34.968 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:06:34.968 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:06:34.968 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:06:34.968 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:06:34.968 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:06:34.969 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:06:34.969 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:06:34.969 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:06:34.969 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:06:34.970 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:06:34.970 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:06:34.970 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:06:34.970 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:06:34.971 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:06:35.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:06:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:06:35.271Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:06:35.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:06:35 smithi083 ceph-mon[77242]: from='client.43845 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:06:35.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:06:35 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3427368355' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:06:35.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:06:35 smithi083 ceph-mon[79930]: from='client.43845 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:06:35.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:06:35 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3427368355' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:06:35.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:06:35 smithi153 ceph-mon[70044]: from='client.43845 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:06:35.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:06:35 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3427368355' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:06:36.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:06:36 smithi083 ceph-mon[79930]: from='client.43851 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:06:36.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:06:36 smithi083 ceph-mon[79930]: pgmap v8350: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:36.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:06:36 smithi083 ceph-mon[77242]: from='client.43851 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:06:36.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:06:36 smithi083 ceph-mon[77242]: pgmap v8350: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:36.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:06:36 smithi153 ceph-mon[70044]: from='client.43851 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:06:36.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:06:36 smithi153 ceph-mon[70044]: pgmap v8350: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:38.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:06:38 smithi083 ceph-mon[79930]: pgmap v8351: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:38.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:06:38 smithi083 ceph-mon[77242]: pgmap v8351: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:38.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:06:38 smithi153 ceph-mon[70044]: pgmap v8351: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:40.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:06:40 smithi083 ceph-mon[79930]: pgmap v8352: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:40.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:06:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:06:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:06:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:06:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:06:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:06:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:06:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:06:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:06:40 smithi083 ceph-mon[77242]: pgmap v8352: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:06:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:06:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:06:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:06:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:06:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:06:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:06:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:06:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:06:40 smithi153 ceph-mon[70044]: pgmap v8352: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:06:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:06:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:06:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:06:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:06:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:06:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:06:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:06:41.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:06:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:06:41.182Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:06:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:06:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:06:41] "GET /metrics HTTP/1.1" 200 225229 "" "Prometheus/2.33.4" 2022-09-29T10:06:42.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:06:42 smithi083 ceph-mon[79930]: pgmap v8353: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:42.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:06:42 smithi083 ceph-mon[77242]: pgmap v8353: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:06:42 smithi153 ceph-mon[70044]: pgmap v8353: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:06:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:06:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:06:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:06:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:06:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:06:44.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:06:44 smithi083 ceph-mon[79930]: pgmap v8354: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:44.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:06:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:06:44.563Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:06:44.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:06:44 smithi083 ceph-mon[77242]: pgmap v8354: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:44.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:06:44 smithi153 ceph-mon[70044]: pgmap v8354: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:45.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:06:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:06:45.272Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:06:46.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:06:46 smithi153 ceph-mon[70044]: pgmap v8355: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:46.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:06:46 smithi083 ceph-mon[79930]: pgmap v8355: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:46.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:06:46 smithi083 ceph-mon[77242]: pgmap v8355: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:48.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:06:48 smithi153 ceph-mon[70044]: pgmap v8356: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:48.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:06:48 smithi083 ceph-mon[79930]: pgmap v8356: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:48.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:06:48 smithi083 ceph-mon[77242]: pgmap v8356: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:50.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:06:50 smithi153 ceph-mon[70044]: pgmap v8357: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:50.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:06:50 smithi083 ceph-mon[77242]: pgmap v8357: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:50.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:06:50 smithi083 ceph-mon[79930]: pgmap v8357: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:51.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:06:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:06:51.182Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:06:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:06:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:06:51] "GET /metrics HTTP/1.1" 200 225229 "" "Prometheus/2.33.4" 2022-09-29T10:06:52.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:06:52 smithi153 ceph-mon[70044]: pgmap v8358: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:06:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:06:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:06:52.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:06:52 smithi083 ceph-mon[79930]: pgmap v8358: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:52.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:06:52 smithi083 ceph-mon[77242]: pgmap v8358: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:06:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:06:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:06:54.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:06:54 smithi153 ceph-mon[70044]: pgmap v8359: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:54.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:06:54 smithi083 ceph-mon[79930]: pgmap v8359: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:54.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:06:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:06:54.563Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:06:54.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:06:54 smithi083 ceph-mon[77242]: pgmap v8359: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:55.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:06:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:06:55.273Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:06:56.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:06:56 smithi153 ceph-mon[70044]: pgmap v8360: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:56.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:06:56 smithi083 ceph-mon[79930]: pgmap v8360: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:56.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:06:56 smithi083 ceph-mon[77242]: pgmap v8360: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:58.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:06:58 smithi153 ceph-mon[70044]: pgmap v8361: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:58.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:06:58 smithi083 ceph-mon[79930]: pgmap v8361: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:06:58.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:06:58 smithi083 ceph-mon[77242]: pgmap v8361: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:00.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:07:00 smithi153 ceph-mon[70044]: pgmap v8362: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:00.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:07:00 smithi083 ceph-mon[77242]: pgmap v8362: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:00.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:07:00 smithi083 ceph-mon[79930]: pgmap v8362: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:01.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:07:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:07:01.183Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:07:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:07:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:07:01] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T10:07:02.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:07:02 smithi153 ceph-mon[70044]: pgmap v8363: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:07:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:07:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:07:02.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:07:02 smithi083 ceph-mon[77242]: pgmap v8363: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:02.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:07:02 smithi083 ceph-mon[79930]: pgmap v8363: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:03.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:07:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:07:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:07:04.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:07:04 smithi083 ceph-mon[79930]: pgmap v8364: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:04.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:07:04 smithi083 ceph-mon[77242]: pgmap v8364: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:04.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:07:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:07:04.564Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:07:04.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:07:04 smithi153 ceph-mon[70044]: pgmap v8364: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:05.293 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:07:05.586 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:07:05.586 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 6m ago 4h 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:07:05.586 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 4m ago 4h 63.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:07:05.587 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 6m ago 4h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:07:05.587 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 4m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:07:05.587 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 4m ago 4h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:07:05.587 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 6m ago 4h 749M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:07:05.587 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 6m ago 4h 1054M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:07:05.588 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 4m ago 4h 454M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:07:05.588 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 6m ago 4h 604M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:07:05.588 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 6m ago 4h 25.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:07:05.588 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 4m ago 4h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:07:05.589 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 6m ago 4h 742M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:07:05.589 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 6m ago 4h 733M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:07:05.589 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 6m ago 4h 573M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:07:05.589 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 6m ago 4h 714M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:07:05.590 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 4m ago 4h 631M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:07:05.590 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 4m ago 4h 785M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:07:05.590 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 4m ago 4h 700M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:07:05.590 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 4m ago 4h 827M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:07:05.591 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 4m ago 4h 177M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:07:05.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:07:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:07:05.273Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:07:05.941 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:07:05.941 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:07:05.941 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:07:05.942 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:07:05.942 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:07:05.942 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:07:05.942 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:07:05.943 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:07:05.943 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:07:05.943 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:07:05.943 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:07:05.943 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:07:05.944 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:07:05.944 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:07:05.944 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:07:05.944 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:07:05.945 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:07:05.945 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:07:06.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:07:06 smithi083 ceph-mon[79930]: pgmap v8365: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:06.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:07:06 smithi083 ceph-mon[79930]: from='client.43863 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:07:06.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:07:06 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2373782040' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:07:06.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:07:06 smithi083 ceph-mon[77242]: pgmap v8365: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:06.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:07:06 smithi083 ceph-mon[77242]: from='client.43863 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:07:06.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:07:06 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2373782040' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:07:06.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:07:06 smithi153 ceph-mon[70044]: pgmap v8365: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:06.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:07:06 smithi153 ceph-mon[70044]: from='client.43863 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:07:06.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:07:06 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2373782040' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:07:07.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:07:07 smithi083 ceph-mon[79930]: from='client.53800 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:07:07.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:07:07 smithi083 ceph-mon[77242]: from='client.53800 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:07:07.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:07:07 smithi153 ceph-mon[70044]: from='client.53800 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:07:08.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:07:08 smithi083 ceph-mon[79930]: pgmap v8366: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:08.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:07:08 smithi083 ceph-mon[77242]: pgmap v8366: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:08.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:07:08 smithi153 ceph-mon[70044]: pgmap v8366: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:10.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:07:10 smithi083 ceph-mon[79930]: pgmap v8367: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:10.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:07:10 smithi083 ceph-mon[77242]: pgmap v8367: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:10.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:07:10 smithi153 ceph-mon[70044]: pgmap v8367: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:11.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:07:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:07:11.184Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:07:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:07:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:07:11] "GET /metrics HTTP/1.1" 200 225229 "" "Prometheus/2.33.4" 2022-09-29T10:07:12.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:07:12 smithi083 ceph-mon[79930]: pgmap v8368: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:12.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:07:12 smithi083 ceph-mon[77242]: pgmap v8368: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:12.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:07:12 smithi153 ceph-mon[70044]: pgmap v8368: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:07:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:07:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:07:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:07:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:07:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:07:14.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:07:14 smithi083 ceph-mon[79930]: pgmap v8369: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:14.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:07:14 smithi083 ceph-mon[77242]: pgmap v8369: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:14.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:07:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:07:14.564Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:07:14.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:07:14 smithi153 ceph-mon[70044]: pgmap v8369: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:15.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:07:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:07:15.275Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:07:16.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:07:16 smithi083 ceph-mon[79930]: pgmap v8370: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:16.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:07:16 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:07:16.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:07:16 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:07:16.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:07:16 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:07:16.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:07:16 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:07:16.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:07:16 smithi083 ceph-mon[77242]: pgmap v8370: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:16.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:07:16 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:07:16.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:07:16 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:07:16.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:07:16 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:07:16.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:07:16 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:07:16.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:07:16 smithi153 ceph-mon[70044]: pgmap v8370: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:16.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:07:16 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:07:16.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:07:16 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:07:16.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:07:16 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:07:16.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:07:16 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:07:18.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:07:18 smithi083 ceph-mon[79930]: pgmap v8371: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:18.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:07:18 smithi083 ceph-mon[77242]: pgmap v8371: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:18.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:07:18 smithi153 ceph-mon[70044]: pgmap v8371: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:20.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:07:20 smithi083 ceph-mon[79930]: pgmap v8372: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:20.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:07:20 smithi083 ceph-mon[77242]: pgmap v8372: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:20.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:07:20 smithi153 ceph-mon[70044]: pgmap v8372: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:21.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:07:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:07:21.184Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:07:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:07:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:07:21] "GET /metrics HTTP/1.1" 200 225229 "" "Prometheus/2.33.4" 2022-09-29T10:07:22.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:07:22 smithi083 ceph-mon[79930]: pgmap v8373: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:22.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:07:22 smithi083 ceph-mon[77242]: pgmap v8373: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:22.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:07:22 smithi153 ceph-mon[70044]: pgmap v8373: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:07:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:07:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:07:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:07:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:07:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:07:24.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:07:24 smithi083 ceph-mon[79930]: pgmap v8374: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:07:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:07:24.565Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:07:24.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:07:24 smithi083 ceph-mon[77242]: pgmap v8374: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:24.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:07:24 smithi153 ceph-mon[70044]: pgmap v8374: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:25.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:07:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:07:25.276Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:07:26.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:07:26 smithi083 ceph-mon[79930]: pgmap v8375: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:26.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:07:26 smithi083 ceph-mon[77242]: pgmap v8375: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:26.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:07:26 smithi153 ceph-mon[70044]: pgmap v8375: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:28.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:07:28 smithi083 ceph-mon[79930]: pgmap v8376: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:28.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:07:28 smithi083 ceph-mon[77242]: pgmap v8376: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:28.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:07:28 smithi153 ceph-mon[70044]: pgmap v8376: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:30.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:07:30 smithi083 ceph-mon[79930]: pgmap v8377: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:30.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:07:30 smithi083 ceph-mon[77242]: pgmap v8377: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:30.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:07:30 smithi153 ceph-mon[70044]: pgmap v8377: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:31.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:07:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:07:31.185Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:07:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:07:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:07:31] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T10:07:32.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:07:32 smithi083 ceph-mon[77242]: pgmap v8378: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:32.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:07:32 smithi083 ceph-mon[79930]: pgmap v8378: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:32.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:07:32 smithi153 ceph-mon[70044]: pgmap v8378: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:07:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:07:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:07:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:07:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:07:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:07:34.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:07:34 smithi083 ceph-mon[79930]: pgmap v8379: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:34.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:07:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:07:34.565Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:07:34.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:07:34 smithi083 ceph-mon[77242]: pgmap v8379: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:34.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:07:34 smithi153 ceph-mon[70044]: pgmap v8379: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:35.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:07:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:07:35.277Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:07:36.264 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:07:36.552 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:07:36.552 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 7m ago 4h 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:07:36.552 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 5m ago 4h 63.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:07:36.553 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 7m ago 4h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:07:36.553 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 5m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:07:36.553 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 5m ago 4h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:07:36.553 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 7m ago 4h 749M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:07:36.554 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 7m ago 4h 1054M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:07:36.554 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 5m ago 4h 454M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:07:36.554 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 7m ago 4h 604M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:07:36.554 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 7m ago 4h 25.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:07:36.555 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 5m ago 4h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:07:36.555 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 7m ago 4h 742M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:07:36.555 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 7m ago 4h 733M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:07:36.555 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 7m ago 4h 573M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:07:36.556 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 7m ago 4h 714M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:07:36.556 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 5m ago 4h 631M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:07:36.556 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 5m ago 4h 785M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:07:36.556 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 5m ago 4h 700M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:07:36.557 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 5m ago 4h 827M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:07:36.557 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 5m ago 4h 177M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:07:36.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:07:36 smithi083 ceph-mon[79930]: pgmap v8380: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:36.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:07:36 smithi083 ceph-mon[77242]: pgmap v8380: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:36.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:07:36 smithi153 ceph-mon[70044]: pgmap v8380: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:36.905 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:07:36.905 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:07:36.906 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:07:36.906 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:07:36.906 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:07:36.906 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:07:36.907 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:07:36.907 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:07:36.907 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:07:36.907 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:07:36.908 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:07:36.908 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:07:36.908 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:07:36.908 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:07:36.909 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:07:36.909 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:07:36.909 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:07:36.909 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:07:37.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:07:37 smithi083 ceph-mon[79930]: from='client.43881 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:07:37.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:07:37 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2731558960' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:07:37.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:07:37 smithi083 ceph-mon[77242]: from='client.43881 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:07:37.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:07:37 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2731558960' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:07:37.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:07:37 smithi153 ceph-mon[70044]: from='client.43881 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:07:37.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:07:37 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2731558960' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:07:38.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:07:38 smithi083 ceph-mon[79930]: from='client.43887 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:07:38.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:07:38 smithi083 ceph-mon[79930]: pgmap v8381: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:38.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:07:38 smithi083 ceph-mon[77242]: from='client.43887 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:07:38.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:07:38 smithi083 ceph-mon[77242]: pgmap v8381: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:38.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:07:38 smithi153 ceph-mon[70044]: from='client.43887 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:07:38.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:07:38 smithi153 ceph-mon[70044]: pgmap v8381: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:40.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:07:40 smithi083 ceph-mon[79930]: pgmap v8382: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:07:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:07:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:07:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:07:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:07:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:07:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:07:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:07:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:07:40 smithi083 ceph-mon[77242]: pgmap v8382: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:07:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:07:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:07:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:07:40.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:07:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:07:40.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:07:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:07:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:07:40 smithi153 ceph-mon[70044]: pgmap v8382: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:07:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:07:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:07:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:07:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:07:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:07:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:07:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:07:41.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:07:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:07:41.186Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:07:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:07:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:07:41] "GET /metrics HTTP/1.1" 200 225229 "" "Prometheus/2.33.4" 2022-09-29T10:07:42.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:07:42 smithi083 ceph-mon[79930]: pgmap v8383: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:42.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:07:42 smithi083 ceph-mon[77242]: pgmap v8383: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:07:42 smithi153 ceph-mon[70044]: pgmap v8383: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:07:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:07:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:07:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:07:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:07:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:07:44.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:07:44 smithi083 ceph-mon[79930]: pgmap v8384: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:07:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:07:44.566Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:07:44.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:07:44 smithi083 ceph-mon[77242]: pgmap v8384: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:44.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:07:44 smithi153 ceph-mon[70044]: pgmap v8384: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:45.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:07:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:07:45.278Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:07:46.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:07:46 smithi083 ceph-mon[77242]: pgmap v8385: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:46.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:07:46 smithi083 ceph-mon[79930]: pgmap v8385: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:46.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:07:46 smithi153 ceph-mon[70044]: pgmap v8385: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:48.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:07:48 smithi083 ceph-mon[77242]: pgmap v8386: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:48.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:07:48 smithi083 ceph-mon[79930]: pgmap v8386: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:48.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:07:48 smithi153 ceph-mon[70044]: pgmap v8386: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:50.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:07:50 smithi083 ceph-mon[79930]: pgmap v8387: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:50.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:07:50 smithi083 ceph-mon[77242]: pgmap v8387: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:50.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:07:50 smithi153 ceph-mon[70044]: pgmap v8387: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:51.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:07:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:07:51.186Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:07:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:07:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:07:51] "GET /metrics HTTP/1.1" 200 225229 "" "Prometheus/2.33.4" 2022-09-29T10:07:52.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:07:52 smithi083 ceph-mon[79930]: pgmap v8388: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:52.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:07:52 smithi083 ceph-mon[77242]: pgmap v8388: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:52.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:07:52 smithi153 ceph-mon[70044]: pgmap v8388: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:07:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:07:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:07:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:07:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:07:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:07:54.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:07:54 smithi083 ceph-mon[79930]: pgmap v8389: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:54.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:07:54 smithi083 ceph-mon[77242]: pgmap v8389: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:07:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:07:54.566Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:07:54.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:07:54 smithi153 ceph-mon[70044]: pgmap v8389: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:55.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:07:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:07:55.278Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:07:56.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:07:56 smithi083 ceph-mon[79930]: pgmap v8390: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:56.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:07:56 smithi083 ceph-mon[77242]: pgmap v8390: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:56.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:07:56 smithi153 ceph-mon[70044]: pgmap v8390: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:58.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:07:58 smithi083 ceph-mon[79930]: pgmap v8391: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:58.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:07:58 smithi083 ceph-mon[77242]: pgmap v8391: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:07:58.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:07:58 smithi153 ceph-mon[70044]: pgmap v8391: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:00.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:08:00 smithi083 ceph-mon[79930]: pgmap v8392: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:00.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:08:00 smithi083 ceph-mon[77242]: pgmap v8392: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:00.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:08:00 smithi153 ceph-mon[70044]: pgmap v8392: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:01.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:08:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:08:01.187Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:08:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:08:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:08:01] "GET /metrics HTTP/1.1" 200 225228 "" "Prometheus/2.33.4" 2022-09-29T10:08:02.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:08:02 smithi083 ceph-mon[77242]: pgmap v8393: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:02.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:08:02 smithi083 ceph-mon[79930]: pgmap v8393: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:02.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:08:02 smithi153 ceph-mon[70044]: pgmap v8393: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:08:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:08:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:08:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:08:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:08:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:08:04.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:08:04 smithi083 ceph-mon[77242]: pgmap v8394: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:04.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:08:04 smithi083 ceph-mon[79930]: pgmap v8394: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:04.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:08:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:08:04.567Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:08:04.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:08:04 smithi153 ceph-mon[70044]: pgmap v8394: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:05.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:08:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:08:05.279Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:08:06.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:08:06 smithi083 ceph-mon[79930]: pgmap v8395: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:06.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:08:06 smithi083 ceph-mon[77242]: pgmap v8395: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:06.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:08:06 smithi153 ceph-mon[70044]: pgmap v8395: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:07.232 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:08:07.520 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:08:07.521 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 8m ago 4h 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:08:07.521 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 5m ago 4h 63.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:08:07.521 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 8m ago 4h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:08:07.521 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 5m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:08:07.522 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 5m ago 4h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:08:07.522 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 8m ago 4h 749M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:08:07.522 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 8m ago 4h 1054M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:08:07.522 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 5m ago 4h 454M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:08:07.523 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 8m ago 4h 604M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:08:07.523 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 8m ago 4h 25.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:08:07.523 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 5m ago 4h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:08:07.523 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 8m ago 4h 742M 2500M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:08:07.524 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 8m ago 4h 733M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:08:07.524 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 8m ago 4h 573M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:08:07.524 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 8m ago 4h 714M 2500M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:08:07.524 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 5m ago 4h 631M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:08:07.524 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 5m ago 4h 785M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:08:07.525 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 5m ago 4h 700M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:08:07.525 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 5m ago 4h 827M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:08:07.525 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 5m ago 4h 177M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:08:07.875 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:08:07.875 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:08:07.876 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:08:07.876 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:08:07.876 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:08:07.876 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:08:07.877 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:08:07.877 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:08:07.877 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:08:07.877 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:08:07.878 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:08:07.878 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:08:07.878 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:08:07.878 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:08:07.879 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:08:07.879 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:08:07.879 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:08:07.879 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:08:08.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:08:08 smithi083 ceph-mon[79930]: pgmap v8396: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:08.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:08:08 smithi083 ceph-mon[79930]: from='client.53830 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:08:08.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:08:08 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1408044458' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:08:08.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:08:08 smithi083 ceph-mon[77242]: pgmap v8396: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:08.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:08:08 smithi083 ceph-mon[77242]: from='client.53830 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:08:08.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:08:08 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1408044458' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:08:08.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:08:08 smithi153 ceph-mon[70044]: pgmap v8396: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:08.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:08:08 smithi153 ceph-mon[70044]: from='client.53830 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:08:08.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:08:08 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1408044458' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:08:09.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:08:09 smithi083 ceph-mon[77242]: from='client.43905 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:08:09.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:08:09 smithi083 ceph-mon[79930]: from='client.43905 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:08:09.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:08:09 smithi153 ceph-mon[70044]: from='client.43905 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:08:10.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:08:10 smithi083 ceph-mon[79930]: pgmap v8397: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:10.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:08:10 smithi083 ceph-mon[77242]: pgmap v8397: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:10.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:08:10 smithi153 ceph-mon[70044]: pgmap v8397: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:11.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:08:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:08:11.187Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:08:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:08:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:08:11] "GET /metrics HTTP/1.1" 200 225228 "" "Prometheus/2.33.4" 2022-09-29T10:08:12.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:08:12 smithi083 ceph-mon[79930]: pgmap v8398: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:12.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:08:12 smithi083 ceph-mon[77242]: pgmap v8398: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:12.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:08:12 smithi153 ceph-mon[70044]: pgmap v8398: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:08:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:08:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:08:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:08:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:08:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:08:14.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:08:14 smithi153 ceph-mon[70044]: pgmap v8399: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:14.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:08:14 smithi083 ceph-mon[79930]: pgmap v8399: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:14.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:08:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:08:14.567Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:08:14.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:08:14 smithi083 ceph-mon[77242]: pgmap v8399: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:15.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:08:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:08:15.279Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:08:16.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:08:16 smithi153 ceph-mon[70044]: pgmap v8400: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:16.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:08:16 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:08:16.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:08:16 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:08:16.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:08:16 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:08:16.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:08:16 smithi083 ceph-mon[79930]: pgmap v8400: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:16.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:08:16 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:08:16.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:08:16 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:08:16.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:08:16 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:08:16.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:08:16 smithi083 ceph-mon[77242]: pgmap v8400: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:16.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:08:16 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:08:16.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:08:16 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:08:16.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:08:16 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:08:17.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:08:17 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:08:17.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:08:17 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:08:17.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:08:17 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:08:17.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:08:17 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:08:17.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:08:17 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:08:17.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:08:17 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:08:17.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:08:17 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:08:17.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:08:17 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:08:17.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:08:17 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:08:17.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:08:17 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:08:17.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:08:17 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:08:17.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:08:17 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:08:17.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:08:17 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:08:17.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:08:17 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:08:17.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:08:17 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:08:17.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:08:17 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:08:17.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:08:17 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:08:17.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:08:17 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:08:17.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:08:17 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:08:17.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:08:17 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:08:17.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:08:17 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:08:17.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:08:17 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:08:17.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:08:17 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:08:17.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:08:17 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:08:17.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:08:17 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:08:17.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:08:17 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:08:17.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:08:17 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:08:17.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:08:17 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:08:17.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:08:17 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:08:17.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:08:17 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:08:18.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:08:18 smithi153 ceph-mon[70044]: pgmap v8401: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:18.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:08:18 smithi153 ceph-mon[70044]: Adjusting osd_memory_target on smithi083 to 2492M 2022-09-29T10:08:18.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:08:18 smithi083 ceph-mon[79930]: pgmap v8401: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:18.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:08:18 smithi083 ceph-mon[79930]: Adjusting osd_memory_target on smithi083 to 2492M 2022-09-29T10:08:18.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:08:18 smithi083 ceph-mon[77242]: pgmap v8401: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:18.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:08:18 smithi083 ceph-mon[77242]: Adjusting osd_memory_target on smithi083 to 2492M 2022-09-29T10:08:20.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:08:20 smithi153 ceph-mon[70044]: pgmap v8402: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:20.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:08:20 smithi083 ceph-mon[79930]: pgmap v8402: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:20.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:08:20 smithi083 ceph-mon[77242]: pgmap v8402: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:21.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:08:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:08:21.188Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:08:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:08:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:08:21] "GET /metrics HTTP/1.1" 200 225228 "" "Prometheus/2.33.4" 2022-09-29T10:08:22.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:08:22 smithi153 ceph-mon[70044]: pgmap v8403: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:08:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:08:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:08:22.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:08:22 smithi083 ceph-mon[79930]: pgmap v8403: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:22.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:08:22 smithi083 ceph-mon[77242]: pgmap v8403: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:08:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:08:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:08:24.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:08:24 smithi153 ceph-mon[70044]: pgmap v8404: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:24.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:08:24 smithi083 ceph-mon[79930]: pgmap v8404: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:24.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:08:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:08:24.567Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:08:24.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:08:24 smithi083 ceph-mon[77242]: pgmap v8404: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:25.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:08:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:08:25.279Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:08:26.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:08:26 smithi153 ceph-mon[70044]: pgmap v8405: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:26.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:08:26 smithi083 ceph-mon[79930]: pgmap v8405: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:26.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:08:26 smithi083 ceph-mon[77242]: pgmap v8405: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:28.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:08:28 smithi153 ceph-mon[70044]: pgmap v8406: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:28.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:08:28 smithi083 ceph-mon[77242]: pgmap v8406: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:28.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:08:28 smithi083 ceph-mon[79930]: pgmap v8406: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:30.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:08:30 smithi153 ceph-mon[70044]: pgmap v8407: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:30.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:08:30 smithi083 ceph-mon[79930]: pgmap v8407: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:30.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:08:30 smithi083 ceph-mon[77242]: pgmap v8407: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:31.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:08:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:08:31.188Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:08:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:08:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:08:31] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T10:08:32.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:08:32 smithi153 ceph-mon[70044]: pgmap v8408: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:08:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:08:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:08:32.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:08:32 smithi083 ceph-mon[79930]: pgmap v8408: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:32.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:08:32 smithi083 ceph-mon[77242]: pgmap v8408: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:08:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:08:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:08:34.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:08:34 smithi083 ceph-mon[79930]: pgmap v8409: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:34.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:08:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:08:34.568Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:08:34.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:08:34 smithi083 ceph-mon[77242]: pgmap v8409: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:34.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:08:34 smithi153 ceph-mon[70044]: pgmap v8409: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:35.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:08:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:08:35.279Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:08:36.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:08:36 smithi083 ceph-mon[77242]: pgmap v8410: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:36.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:08:36 smithi083 ceph-mon[79930]: pgmap v8410: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:36.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:08:36 smithi153 ceph-mon[70044]: pgmap v8410: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:38.201 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:08:38.498 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:08:38.499 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 8m ago 4h 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:08:38.499 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 6m ago 4h 63.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:08:38.499 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 8m ago 4h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:08:38.500 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 6m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:08:38.500 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 6m ago 4h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:08:38.500 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 8m ago 4h 749M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:08:38.500 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 8m ago 4h 1054M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:08:38.501 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 6m ago 4h 454M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:08:38.501 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 8m ago 4h 604M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:08:38.501 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 8m ago 4h 25.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:08:38.501 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 6m ago 4h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:08:38.502 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 8m ago 4h 742M 2492M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:08:38.502 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 8m ago 4h 733M 2492M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:08:38.502 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 8m ago 4h 573M 2492M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:08:38.503 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 8m ago 4h 714M 2492M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:08:38.503 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 6m ago 4h 631M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:08:38.503 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 6m ago 4h 785M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:08:38.503 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 6m ago 4h 700M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:08:38.504 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 6m ago 4h 827M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:08:38.504 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 6m ago 4h 177M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:08:38.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:08:38 smithi083 ceph-mon[79930]: pgmap v8411: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:38.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:08:38 smithi083 ceph-mon[77242]: pgmap v8411: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:38.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:08:38 smithi153 ceph-mon[70044]: pgmap v8411: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:38.858 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:08:38.858 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:08:38.858 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:08:38.859 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:08:38.859 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:08:38.859 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:08:38.859 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:08:38.860 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:08:38.860 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:08:38.860 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:08:38.860 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:08:38.861 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:08:38.861 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:08:38.861 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:08:38.861 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:08:38.862 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:08:38.862 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:08:38.862 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:08:39.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:08:39 smithi083 ceph-mon[79930]: from='client.53848 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:08:39.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:08:39 smithi083 ceph-mon[79930]: from='client.53854 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:08:39.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:08:39 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4090853370' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:08:39.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:08:39 smithi083 ceph-mon[77242]: from='client.53848 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:08:39.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:08:39 smithi083 ceph-mon[77242]: from='client.53854 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:08:39.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:08:39 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4090853370' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:08:39.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:08:39 smithi153 ceph-mon[70044]: from='client.53848 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:08:39.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:08:39 smithi153 ceph-mon[70044]: from='client.53854 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:08:39.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:08:39 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4090853370' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:08:40.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:08:40 smithi083 ceph-mon[79930]: pgmap v8412: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:40.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:08:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:08:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:08:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:08:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:08:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:08:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:08:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:08:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:08:40 smithi083 ceph-mon[77242]: pgmap v8412: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:08:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:08:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:08:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:08:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:08:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:08:40.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:08:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:08:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:08:40 smithi153 ceph-mon[70044]: pgmap v8412: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:08:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:08:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:08:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:08:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:08:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:08:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:08:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:08:41.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:08:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:08:41.189Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:08:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:08:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:08:41] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T10:08:42.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:08:42 smithi083 ceph-mon[77242]: pgmap v8413: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:42.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:08:42 smithi083 ceph-mon[79930]: pgmap v8413: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:08:42 smithi153 ceph-mon[70044]: pgmap v8413: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:08:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:08:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:08:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:08:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:08:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:08:44.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:08:44 smithi083 ceph-mon[77242]: pgmap v8414: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:44.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:08:44 smithi083 ceph-mon[79930]: pgmap v8414: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:08:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:08:44.569Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:08:44.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:08:44 smithi153 ceph-mon[70044]: pgmap v8414: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:45.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:08:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:08:45.280Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:08:46.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:08:46 smithi083 ceph-mon[79930]: pgmap v8415: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:46.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:08:46 smithi083 ceph-mon[77242]: pgmap v8415: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:46.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:08:46 smithi153 ceph-mon[70044]: pgmap v8415: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:48.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:08:48 smithi083 ceph-mon[79930]: pgmap v8416: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:48.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:08:48 smithi083 ceph-mon[77242]: pgmap v8416: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:48.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:08:48 smithi153 ceph-mon[70044]: pgmap v8416: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:50.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:08:50 smithi083 ceph-mon[79930]: pgmap v8417: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:50.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:08:50 smithi083 ceph-mon[77242]: pgmap v8417: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:50.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:08:50 smithi153 ceph-mon[70044]: pgmap v8417: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:51.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:08:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:08:51.189Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:08:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:08:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:08:51] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T10:08:52.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:08:52 smithi083 ceph-mon[79930]: pgmap v8418: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:52.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:08:52 smithi083 ceph-mon[77242]: pgmap v8418: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:52.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:08:52 smithi153 ceph-mon[70044]: pgmap v8418: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:08:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:08:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:08:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:08:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:08:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:08:54.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:08:54 smithi083 ceph-mon[79930]: pgmap v8419: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:08:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:08:54.569Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:08:54.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:08:54 smithi083 ceph-mon[77242]: pgmap v8419: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:54.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:08:54 smithi153 ceph-mon[70044]: pgmap v8419: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:55.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:08:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:08:55.281Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:08:56.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:08:56 smithi083 ceph-mon[79930]: pgmap v8420: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:56.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:08:56 smithi083 ceph-mon[77242]: pgmap v8420: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:56.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:08:56 smithi153 ceph-mon[70044]: pgmap v8420: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:58.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:08:58 smithi083 ceph-mon[79930]: pgmap v8421: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:58.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:08:58 smithi083 ceph-mon[77242]: pgmap v8421: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:08:58.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:08:58 smithi153 ceph-mon[70044]: pgmap v8421: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:00.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:09:00 smithi083 ceph-mon[79930]: pgmap v8422: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:00.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:09:00 smithi083 ceph-mon[77242]: pgmap v8422: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:00.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:09:00 smithi153 ceph-mon[70044]: pgmap v8422: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:01.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:09:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:09:01.190Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:09:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:09:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:09:01] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T10:09:02.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:09:02 smithi083 ceph-mon[77242]: pgmap v8423: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:02.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:09:02 smithi083 ceph-mon[79930]: pgmap v8423: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:02.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:09:02 smithi153 ceph-mon[70044]: pgmap v8423: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:09:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:09:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:09:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:09:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:09:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:09:04.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:09:04 smithi083 ceph-mon[79930]: pgmap v8424: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:04.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:09:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:09:04.570Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:09:04.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:09:04 smithi083 ceph-mon[77242]: pgmap v8424: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:04.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:09:04 smithi153 ceph-mon[70044]: pgmap v8424: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:05.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:09:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:09:05.281Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:09:06.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:09:06 smithi083 ceph-mon[77242]: pgmap v8425: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:06.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:09:06 smithi083 ceph-mon[79930]: pgmap v8425: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:06.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:09:06 smithi153 ceph-mon[70044]: pgmap v8425: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:08.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:09:08 smithi083 ceph-mon[79930]: pgmap v8426: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:08.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:09:08 smithi083 ceph-mon[77242]: pgmap v8426: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:08.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:09:08 smithi153 ceph-mon[70044]: pgmap v8426: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:09.182 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:09:09.476 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:09:09.476 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 9m ago 4h 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:09:09.477 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 6m ago 4h 63.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:09:09.477 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 9m ago 4h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:09:09.477 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 6m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:09:09.477 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 6m ago 4h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:09:09.478 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 9m ago 4h 749M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:09:09.478 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 9m ago 4h 1054M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:09:09.478 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 6m ago 4h 454M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:09:09.478 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 9m ago 4h 604M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:09:09.479 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 9m ago 4h 25.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:09:09.479 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 6m ago 4h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:09:09.479 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 9m ago 4h 742M 2492M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:09:09.479 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 9m ago 4h 733M 2492M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:09:09.479 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 9m ago 4h 573M 2492M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:09:09.480 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 9m ago 4h 714M 2492M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:09:09.480 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 6m ago 4h 631M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:09:09.480 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 6m ago 4h 785M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:09:09.480 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 6m ago 4h 700M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:09:09.481 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 6m ago 4h 827M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:09:09.481 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 6m ago 4h 177M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:09:09.830 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:09:09.831 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:09:09.831 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:09:09.831 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:09:09.832 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:09:09.832 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:09:09.832 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:09:09.832 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:09:09.832 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:09:09.833 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:09:09.833 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:09:09.833 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:09:09.833 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:09:09.834 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:09:09.834 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:09:09.834 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:09:09.834 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:09:09.834 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:09:10.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:09:10 smithi083 ceph-mon[79930]: pgmap v8427: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:10.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:09:10 smithi083 ceph-mon[79930]: from='client.43935 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:09:10.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:09:10 smithi083 ceph-mon[79930]: from='client.43941 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:09:10.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:09:10 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1179785917' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:09:10.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:09:10 smithi083 ceph-mon[77242]: pgmap v8427: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:10.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:09:10 smithi083 ceph-mon[77242]: from='client.43935 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:09:10.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:09:10 smithi083 ceph-mon[77242]: from='client.43941 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:09:10.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:09:10 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1179785917' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:09:10.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:09:10 smithi153 ceph-mon[70044]: pgmap v8427: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:10.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:09:10 smithi153 ceph-mon[70044]: from='client.43935 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:09:10.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:09:10 smithi153 ceph-mon[70044]: from='client.43941 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:09:10.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:09:10 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1179785917' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:09:11.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:09:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:09:11.190Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:09:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:09:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:09:11] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T10:09:12.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:09:12 smithi083 ceph-mon[79930]: pgmap v8428: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:12.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:09:12 smithi083 ceph-mon[77242]: pgmap v8428: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:12.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:09:12 smithi153 ceph-mon[70044]: pgmap v8428: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:09:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:09:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:09:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:09:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:09:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:09:14.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:09:14 smithi083 ceph-mon[79930]: pgmap v8429: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:14.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:09:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:09:14.570Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:09:14.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:09:14 smithi083 ceph-mon[77242]: pgmap v8429: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:14.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:09:14 smithi153 ceph-mon[70044]: pgmap v8429: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:15.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:09:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:09:15.282Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:09:16.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:09:16 smithi083 ceph-mon[77242]: pgmap v8430: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:16.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:09:16 smithi083 ceph-mon[79930]: pgmap v8430: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:16.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:09:16 smithi153 ceph-mon[70044]: pgmap v8430: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:17.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:09:17 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:09:17.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:09:17 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:09:17.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:09:17 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:09:17.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:09:17 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:09:17.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:09:17 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:09:17.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:09:17 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:09:17.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:09:17 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:09:17.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:09:17 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:09:17.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:09:17 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:09:18.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:09:18 smithi083 ceph-mon[77242]: pgmap v8431: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:18.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:09:18 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:09:18.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:09:18 smithi083 ceph-mon[79930]: pgmap v8431: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:18.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:09:18 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:09:18.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:09:18 smithi153 ceph-mon[70044]: pgmap v8431: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:18.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:09:18 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:09:20.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:09:20 smithi083 ceph-mon[77242]: pgmap v8432: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:20.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:09:20 smithi083 ceph-mon[79930]: pgmap v8432: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:20.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:09:20 smithi153 ceph-mon[70044]: pgmap v8432: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:21.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:09:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:09:21.191Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:09:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:09:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:09:21] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T10:09:22.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:09:22 smithi083 ceph-mon[79930]: pgmap v8433: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:22.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:09:22 smithi083 ceph-mon[77242]: pgmap v8433: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:22.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:09:22 smithi153 ceph-mon[70044]: pgmap v8433: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:09:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:09:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:09:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:09:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:09:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:09:24.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:09:24 smithi083 ceph-mon[79930]: pgmap v8434: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:09:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:09:24.571Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:09:24.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:09:24 smithi083 ceph-mon[77242]: pgmap v8434: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:24.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:09:24 smithi153 ceph-mon[70044]: pgmap v8434: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:25.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:09:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:09:25.282Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:09:26.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:09:26 smithi083 ceph-mon[79930]: pgmap v8435: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:26.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:09:26 smithi083 ceph-mon[77242]: pgmap v8435: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:26.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:09:26 smithi153 ceph-mon[70044]: pgmap v8435: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:28.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:09:28 smithi083 ceph-mon[79930]: pgmap v8436: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:28.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:09:28 smithi083 ceph-mon[77242]: pgmap v8436: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:28.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:09:28 smithi153 ceph-mon[70044]: pgmap v8436: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:30.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:09:30 smithi083 ceph-mon[79930]: pgmap v8437: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:30.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:09:30 smithi083 ceph-mon[77242]: pgmap v8437: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:30.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:09:30 smithi153 ceph-mon[70044]: pgmap v8437: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:31.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:09:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:09:31.191Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:09:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:09:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:09:31] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T10:09:32.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:09:32 smithi083 ceph-mon[77242]: pgmap v8438: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:32.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:09:32 smithi083 ceph-mon[79930]: pgmap v8438: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:32.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:09:32 smithi153 ceph-mon[70044]: pgmap v8438: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:09:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:09:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:09:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:09:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:09:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:09:34.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:09:34 smithi083 ceph-mon[79930]: pgmap v8439: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:34.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:09:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:09:34.572Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:09:34.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:09:34 smithi083 ceph-mon[77242]: pgmap v8439: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:34.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:09:34 smithi153 ceph-mon[70044]: pgmap v8439: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:35.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:09:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:09:35.282Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:09:36.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:09:36 smithi153 ceph-mon[70044]: pgmap v8440: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:36.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:09:36 smithi083 ceph-mon[79930]: pgmap v8440: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:36.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:09:36 smithi083 ceph-mon[77242]: pgmap v8440: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:38.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:09:38 smithi153 ceph-mon[70044]: pgmap v8441: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:38.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:09:38 smithi083 ceph-mon[77242]: pgmap v8441: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:38.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:09:38 smithi083 ceph-mon[79930]: pgmap v8441: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:40.159 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:09:40.451 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:09:40.451 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 9m ago 4h 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:09:40.452 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 7m ago 4h 63.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:09:40.452 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 9m ago 4h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:09:40.452 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 7m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:09:40.452 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 7m ago 4h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:09:40.453 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 9m ago 4h 749M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:09:40.453 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 9m ago 4h 1054M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:09:40.453 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 7m ago 4h 454M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:09:40.453 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 9m ago 4h 604M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:09:40.454 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 9m ago 4h 25.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:09:40.454 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 7m ago 4h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:09:40.454 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 9m ago 4h 742M 2492M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:09:40.454 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 9m ago 4h 733M 2492M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:09:40.455 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 9m ago 4h 573M 2492M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:09:40.455 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 9m ago 4h 714M 2492M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:09:40.455 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 7m ago 4h 631M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:09:40.455 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 7m ago 4h 785M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:09:40.456 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 7m ago 4h 700M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:09:40.456 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 7m ago 4h 827M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:09:40.456 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 7m ago 4h 177M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:09:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:09:40 smithi153 ceph-mon[70044]: pgmap v8442: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:09:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:09:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:09:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:09:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:09:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:09:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:09:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:09:40.809 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:09:40.810 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:09:40.810 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:09:40.810 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:09:40.811 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:09:40.811 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:09:40.811 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:09:40.811 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:09:40.811 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:09:40.812 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:09:40.812 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:09:40.812 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:09:40.812 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:09:40.813 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:09:40.813 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:09:40.813 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:09:40.813 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:09:40.814 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:09:40.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:09:40 smithi083 ceph-mon[79930]: pgmap v8442: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:40.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:09:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:09:40.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:09:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:09:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:09:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:09:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:09:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:09:40.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:09:40 smithi083 ceph-mon[77242]: pgmap v8442: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:09:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:09:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:09:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:09:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:09:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:09:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:09:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:09:41.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:09:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:09:41.191Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:09:41.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:09:41 smithi153 ceph-mon[70044]: from='client.43953 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:09:41.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:09:41 smithi153 ceph-mon[70044]: from='client.53890 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:09:41.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:09:41 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1528674645' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:09:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:09:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:09:41] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T10:09:41.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:09:41 smithi083 ceph-mon[77242]: from='client.43953 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:09:41.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:09:41 smithi083 ceph-mon[77242]: from='client.53890 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:09:41.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:09:41 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1528674645' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:09:41.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:09:41 smithi083 ceph-mon[79930]: from='client.43953 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:09:41.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:09:41 smithi083 ceph-mon[79930]: from='client.53890 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:09:41.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:09:41 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1528674645' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:09:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:09:42 smithi153 ceph-mon[70044]: pgmap v8443: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:09:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:09:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:09:42.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:09:42 smithi083 ceph-mon[77242]: pgmap v8443: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:42.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:09:42 smithi083 ceph-mon[79930]: pgmap v8443: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:09:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:09:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:09:44.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:09:44 smithi153 ceph-mon[70044]: pgmap v8444: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:44.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:09:44 smithi083 ceph-mon[79930]: pgmap v8444: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:44.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:09:44 smithi083 ceph-mon[77242]: pgmap v8444: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:44.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:09:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:09:44.573Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:09:45.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:09:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:09:45.283Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:09:46.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:09:46 smithi153 ceph-mon[70044]: pgmap v8445: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:46.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:09:46 smithi083 ceph-mon[79930]: pgmap v8445: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:46.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:09:46 smithi083 ceph-mon[77242]: pgmap v8445: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:48.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:09:48 smithi153 ceph-mon[70044]: pgmap v8446: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:48.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:09:48 smithi083 ceph-mon[77242]: pgmap v8446: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:48.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:09:48 smithi083 ceph-mon[79930]: pgmap v8446: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:50.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:09:50 smithi153 ceph-mon[70044]: pgmap v8447: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:50.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:09:50 smithi083 ceph-mon[79930]: pgmap v8447: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:50.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:09:50 smithi083 ceph-mon[77242]: pgmap v8447: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:51.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:09:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:09:51.193Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:09:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:09:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:09:51] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T10:09:52.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:09:52 smithi153 ceph-mon[70044]: pgmap v8448: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:09:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:09:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:09:52.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:09:52 smithi083 ceph-mon[79930]: pgmap v8448: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:52.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:09:52 smithi083 ceph-mon[77242]: pgmap v8448: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:53.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:09:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:09:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:09:54.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:09:54 smithi153 ceph-mon[70044]: pgmap v8449: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:54.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:09:54 smithi083 ceph-mon[79930]: pgmap v8449: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:54.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:09:54 smithi083 ceph-mon[77242]: pgmap v8449: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:54.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:09:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:09:54.574Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:09:55.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:09:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:09:55.282Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:09:56.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:09:56 smithi153 ceph-mon[70044]: pgmap v8450: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:56.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:09:56 smithi083 ceph-mon[79930]: pgmap v8450: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:56.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:09:56 smithi083 ceph-mon[77242]: pgmap v8450: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:58.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:09:58 smithi153 ceph-mon[70044]: pgmap v8451: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:58.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:09:58 smithi083 ceph-mon[79930]: pgmap v8451: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:09:58.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:09:58 smithi083 ceph-mon[77242]: pgmap v8451: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:00.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:10:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[77219]: 2022-09-29T10:09:59.999+0000 7f160f665700 -1 log_channel(cluster) log [ERR] : overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T10:10:00.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:10:00 smithi153 ceph-mon[70044]: pgmap v8452: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:00.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:10:00 smithi153 ceph-mon[70044]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T10:10:00.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:10:00 smithi083 ceph-mon[79930]: pgmap v8452: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:00.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:10:00 smithi083 ceph-mon[79930]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T10:10:00.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:10:00 smithi083 ceph-mon[77242]: pgmap v8452: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:00.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:10:00 smithi083 ceph-mon[77242]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T10:10:01.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:10:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:10:01.194Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:10:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:10:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:10:01] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T10:10:02.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:10:02 smithi153 ceph-mon[70044]: pgmap v8453: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:10:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:10:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:10:02.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:10:02 smithi083 ceph-mon[79930]: pgmap v8453: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:02.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:10:02 smithi083 ceph-mon[77242]: pgmap v8453: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:10:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:10:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:10:04.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:10:04 smithi153 ceph-mon[70044]: pgmap v8454: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:04.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:10:04 smithi083 ceph-mon[79930]: pgmap v8454: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:04.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:10:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:10:04.574Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:10:04.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:10:04 smithi083 ceph-mon[77242]: pgmap v8454: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:05.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:10:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:10:05.283Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:10:06.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:10:06 smithi153 ceph-mon[70044]: pgmap v8455: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:06.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:10:06 smithi083 ceph-mon[79930]: pgmap v8455: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:06.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:10:06 smithi083 ceph-mon[77242]: pgmap v8455: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:08.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:10:08 smithi153 ceph-mon[70044]: pgmap v8456: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:08.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:10:08 smithi083 ceph-mon[79930]: pgmap v8456: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:08.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:10:08 smithi083 ceph-mon[77242]: pgmap v8456: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:10.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:10:10 smithi153 ceph-mon[70044]: pgmap v8457: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:10.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:10:10 smithi083 ceph-mon[79930]: pgmap v8457: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:10.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:10:10 smithi083 ceph-mon[77242]: pgmap v8457: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:11.137 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:10:11.427 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:10:11.428 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 10m ago 4h 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:10:11.428 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 7m ago 4h 63.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:10:11.428 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 10m ago 4h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:10:11.428 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 7m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:10:11.429 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 7m ago 4h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:10:11.429 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 10m ago 4h 749M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:10:11.429 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 10m ago 4h 1054M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:10:11.429 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 7m ago 4h 454M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:10:11.430 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 10m ago 4h 604M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:10:11.430 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 10m ago 4h 25.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:10:11.430 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 7m ago 4h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:10:11.430 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 10m ago 4h 742M 2492M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:10:11.431 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 10m ago 4h 733M 2492M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:10:11.431 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 10m ago 4h 573M 2492M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:10:11.431 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 10m ago 4h 714M 2492M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:10:11.431 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 7m ago 4h 631M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:10:11.432 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 7m ago 4h 785M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:10:11.432 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 7m ago 4h 700M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:10:11.432 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 7m ago 4h 827M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:10:11.432 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 7m ago 4h 177M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:10:11.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:10:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:10:11.195Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:10:11.785 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:10:11.785 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:10:11.786 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:10:11.786 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:10:11.786 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:10:11.787 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:10:11.787 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:10:11.787 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:10:11.787 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:10:11.787 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:10:11.788 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:10:11.788 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:10:11.788 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:10:11.788 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:10:11.789 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:10:11.789 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:10:11.789 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:10:11.790 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:10:11.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:10:11 smithi153 ceph-mon[70044]: pgmap v8458: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:11.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:10:11 smithi153 ceph-mon[70044]: from='client.43971 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:10:11.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:10:11 smithi153 ceph-mon[70044]: from='client.43977 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:10:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:10:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:10:11] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T10:10:11.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:10:11 smithi083 ceph-mon[77242]: pgmap v8458: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:11.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:10:11 smithi083 ceph-mon[77242]: from='client.43971 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:10:11.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:10:11 smithi083 ceph-mon[77242]: from='client.43977 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:10:11.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:10:11 smithi083 ceph-mon[79930]: pgmap v8458: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:11.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:10:11 smithi083 ceph-mon[79930]: from='client.43971 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:10:11.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:10:11 smithi083 ceph-mon[79930]: from='client.43977 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:10:12.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:10:12 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4086499355' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:10:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:10:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:10:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:10:12.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:10:12 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4086499355' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:10:12.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:10:12 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4086499355' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:10:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:10:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:10:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:10:14.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:10:14 smithi083 ceph-mon[79930]: pgmap v8459: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:14.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:10:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:10:14.575Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:10:14.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:10:14 smithi083 ceph-mon[77242]: pgmap v8459: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:14.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:10:14 smithi153 ceph-mon[70044]: pgmap v8459: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:15.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:10:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:10:15.283Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:10:16.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:10:16 smithi083 ceph-mon[79930]: pgmap v8460: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:16.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:10:16 smithi083 ceph-mon[77242]: pgmap v8460: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:16.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:10:16 smithi153 ceph-mon[70044]: pgmap v8460: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:18.383 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:10:18 smithi083 ceph-mon[77242]: pgmap v8461: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:18.383 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:10:18 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:10:18.383 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:10:18 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:10:18.384 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:10:18 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:10:18.384 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:10:18 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:10:18.384 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:10:18 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:10:18.384 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:10:18 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:10:18.385 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:10:18 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:10:18.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:10:18 smithi083 ceph-mon[79930]: pgmap v8461: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:18.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:10:18 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:10:18.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:10:18 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:10:18.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:10:18 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:10:18.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:10:18 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:10:18.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:10:18 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:10:18.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:10:18 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:10:18.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:10:18 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:10:18.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:10:18 smithi153 ceph-mon[70044]: pgmap v8461: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:18.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:10:18 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:10:18.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:10:18 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:10:18.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:10:18 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:10:18.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:10:18 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:10:18.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:10:18 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:10:18.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:10:18 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:10:18.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:10:18 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:10:20.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:10:20 smithi083 ceph-mon[79930]: pgmap v8462: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:20.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:10:20 smithi083 ceph-mon[77242]: pgmap v8462: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:20.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:10:20 smithi153 ceph-mon[70044]: pgmap v8462: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:21.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:10:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:10:21.195Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:10:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:10:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:10:21] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T10:10:22.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:10:22 smithi083 ceph-mon[79930]: pgmap v8463: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:22.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:10:22 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:10:22.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:10:22 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:10:22.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:10:22 smithi083 ceph-mon[77242]: pgmap v8463: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:22.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:10:22 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:10:22.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:10:22 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:10:22.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:10:22 smithi153 ceph-mon[70044]: pgmap v8463: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:22.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:10:22 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:10:22.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:10:22 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:10:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:10:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:10:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:10:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:10:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:10:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:10:24.192 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:10:24 smithi083 ceph-mon[77242]: pgmap v8464: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:24.192 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:10:24 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:10:24.192 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:10:24 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:10:24.193 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:10:24 smithi083 ceph-mon[79930]: pgmap v8464: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:24.193 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:10:24 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:10:24.193 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:10:24 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:10:24.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:10:24 smithi153 ceph-mon[70044]: pgmap v8464: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:24.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:10:24 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:10:24.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:10:24 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:10:24.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:10:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:10:24.575Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:10:25.590 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:10:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:10:25.284Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:10:26.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:10:26 smithi153 ceph-mon[70044]: pgmap v8465: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:26.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:10:26 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:10:26.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:10:26 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:10:26.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:10:26 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:10:26.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:10:26 smithi083 ceph-mon[79930]: pgmap v8465: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:26.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:10:26 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:10:26.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:10:26 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:10:26.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:10:26 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:10:26.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:10:26 smithi083 ceph-mon[77242]: pgmap v8465: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:26.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:10:26 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:10:26.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:10:26 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:10:26.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:10:26 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:10:27.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:10:27 smithi153 ceph-mon[70044]: Detected new or changed devices on smithi083 2022-09-29T10:10:27.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:10:27 smithi083 ceph-mon[79930]: Detected new or changed devices on smithi083 2022-09-29T10:10:27.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:10:27 smithi083 ceph-mon[77242]: Detected new or changed devices on smithi083 2022-09-29T10:10:28.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:10:28 smithi153 ceph-mon[70044]: pgmap v8466: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:28.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:10:28 smithi083 ceph-mon[79930]: pgmap v8466: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:28.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:10:28 smithi083 ceph-mon[77242]: pgmap v8466: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:30.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:10:30 smithi083 ceph-mon[79930]: pgmap v8467: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:30.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:10:30 smithi083 ceph-mon[77242]: pgmap v8467: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:30.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:10:30 smithi153 ceph-mon[70044]: pgmap v8467: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:31.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:10:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:10:31.196Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:10:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:10:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:10:31] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T10:10:32.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:10:32 smithi083 ceph-mon[79930]: pgmap v8468: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:32.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:10:32 smithi083 ceph-mon[77242]: pgmap v8468: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:32.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:10:32 smithi153 ceph-mon[70044]: pgmap v8468: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:10:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:10:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:10:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:10:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:10:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:10:34.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:10:34 smithi083 ceph-mon[77242]: pgmap v8469: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:34.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:10:34 smithi083 ceph-mon[79930]: pgmap v8469: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:34.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:10:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:10:34.577Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:10:34.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:10:34 smithi153 ceph-mon[70044]: pgmap v8469: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:35.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:10:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:10:35.285Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:10:36.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:10:36 smithi083 ceph-mon[77242]: pgmap v8470: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:36.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:10:36 smithi083 ceph-mon[79930]: pgmap v8470: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:36.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:10:36 smithi153 ceph-mon[70044]: pgmap v8470: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:38.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:10:38 smithi083 ceph-mon[77242]: pgmap v8471: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:38.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:10:38 smithi083 ceph-mon[79930]: pgmap v8471: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:38.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:10:38 smithi153 ceph-mon[70044]: pgmap v8471: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:40.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:10:40 smithi083 ceph-mon[79930]: pgmap v8472: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:40.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:10:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:10:40.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:10:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:10:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:10:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:10:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:10:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:10:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:10:40 smithi083 ceph-mon[77242]: pgmap v8472: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:10:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:10:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:10:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:10:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:10:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:10:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:10:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:10:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:10:40 smithi153 ceph-mon[70044]: pgmap v8472: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:10:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:10:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:10:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:10:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:10:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:10:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:10:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:10:41.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:10:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:10:41.196Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:10:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:10:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:10:41] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T10:10:42.114 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:10:42.402 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:10:42.402 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 20s ago 4h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:10:42.402 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 8m ago 4h 63.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:10:42.403 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 20s ago 4h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:10:42.403 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 8m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:10:42.403 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 8m ago 4h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:10:42.403 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 20s ago 4h 757M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:10:42.404 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 20s ago 4h 1091M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:10:42.404 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 8m ago 4h 454M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:10:42.404 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 20s ago 4h 625M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:10:42.404 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 20s ago 4h 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:10:42.405 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 8m ago 4h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:10:42.405 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 20s ago 4h 767M 2492M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:10:42.405 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 20s ago 4h 758M 2492M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:10:42.405 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 20s ago 4h 593M 2492M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:10:42.405 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 20s ago 4h 739M 2492M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:10:42.406 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 8m ago 4h 631M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:10:42.406 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 8m ago 4h 785M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:10:42.406 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 8m ago 4h 700M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:10:42.406 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 8m ago 4h 827M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:10:42.407 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 8m ago 4h 177M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:10:42.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:10:42 smithi083 ceph-mon[79930]: pgmap v8473: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:42.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:10:42 smithi083 ceph-mon[77242]: pgmap v8473: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:42.760 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:10:42.760 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:10:42.760 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:10:42.761 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:10:42.761 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:10:42.761 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:10:42.761 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:10:42.762 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:10:42.762 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:10:42.762 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:10:42.762 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:10:42.763 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:10:42.763 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:10:42.763 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:10:42.763 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:10:42.764 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:10:42.764 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:10:42.764 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:10:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:10:42 smithi153 ceph-mon[70044]: pgmap v8473: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:10:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:10:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:10:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:10:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:10:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:10:43.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:10:43 smithi083 ceph-mon[77242]: from='client.53920 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:10:43.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:10:43 smithi083 ceph-mon[77242]: from='client.53926 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:10:43.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:10:43 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/258313647' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:10:43.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:10:43 smithi083 ceph-mon[79930]: from='client.53920 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:10:43.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:10:43 smithi083 ceph-mon[79930]: from='client.53926 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:10:43.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:10:43 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/258313647' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:10:43.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:10:43 smithi153 ceph-mon[70044]: from='client.53920 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:10:43.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:10:43 smithi153 ceph-mon[70044]: from='client.53926 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:10:43.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:10:43 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/258313647' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:10:44.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:10:44 smithi083 ceph-mon[79930]: pgmap v8474: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:10:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:10:44.578Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:10:44.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:10:44 smithi083 ceph-mon[77242]: pgmap v8474: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:44.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:10:44 smithi153 ceph-mon[70044]: pgmap v8474: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:45.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:10:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:10:45.285Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:10:46.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:10:46 smithi083 ceph-mon[79930]: pgmap v8475: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:46.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:10:46 smithi083 ceph-mon[77242]: pgmap v8475: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:46.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:10:46 smithi153 ceph-mon[70044]: pgmap v8475: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:48.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:10:48 smithi083 ceph-mon[79930]: pgmap v8476: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:48.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:10:48 smithi083 ceph-mon[77242]: pgmap v8476: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:48.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:10:48 smithi153 ceph-mon[70044]: pgmap v8476: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:50.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:10:50 smithi083 ceph-mon[79930]: pgmap v8477: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:50.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:10:50 smithi083 ceph-mon[77242]: pgmap v8477: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:50.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:10:50 smithi153 ceph-mon[70044]: pgmap v8477: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:51.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:10:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:10:51.197Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:10:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:10:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:10:51] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T10:10:52.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:10:52 smithi083 ceph-mon[77242]: pgmap v8478: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:52.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:10:52 smithi083 ceph-mon[79930]: pgmap v8478: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:52.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:10:52 smithi153 ceph-mon[70044]: pgmap v8478: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:10:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:10:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:10:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:10:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:10:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:10:54.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:10:54 smithi083 ceph-mon[79930]: pgmap v8479: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:10:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:10:54.579Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:10:54.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:10:54 smithi083 ceph-mon[77242]: pgmap v8479: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:54.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:10:54 smithi153 ceph-mon[70044]: pgmap v8479: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:55.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:10:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:10:55.287Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:10:56.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:10:56 smithi083 ceph-mon[79930]: pgmap v8480: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:56.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:10:56 smithi083 ceph-mon[77242]: pgmap v8480: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:56.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:10:56 smithi153 ceph-mon[70044]: pgmap v8480: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:58.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:10:58 smithi083 ceph-mon[79930]: pgmap v8481: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:58.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:10:58 smithi083 ceph-mon[77242]: pgmap v8481: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:10:58.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:10:58 smithi153 ceph-mon[70044]: pgmap v8481: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:00.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:11:00 smithi083 ceph-mon[77242]: pgmap v8482: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:00.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:11:00 smithi083 ceph-mon[79930]: pgmap v8482: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:00.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:11:00 smithi153 ceph-mon[70044]: pgmap v8482: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:01.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:11:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:11:01.198Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:11:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:11:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:11:01] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T10:11:02.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:11:02 smithi083 ceph-mon[79930]: pgmap v8483: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:02.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:11:02 smithi083 ceph-mon[77242]: pgmap v8483: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:02.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:11:02 smithi153 ceph-mon[70044]: pgmap v8483: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:11:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:11:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:11:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:11:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:11:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:11:04.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:11:04 smithi083 ceph-mon[79930]: pgmap v8484: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:04.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:11:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:11:04.580Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:11:04.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:11:04 smithi083 ceph-mon[77242]: pgmap v8484: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:04.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:11:04 smithi153 ceph-mon[70044]: pgmap v8484: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:05.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:11:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:11:05.287Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:11:06.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:11:06 smithi083 ceph-mon[79930]: pgmap v8485: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:06.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:11:06 smithi083 ceph-mon[77242]: pgmap v8485: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:06.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:11:06 smithi153 ceph-mon[70044]: pgmap v8485: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:08.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:11:08 smithi083 ceph-mon[79930]: pgmap v8486: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:08.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:11:08 smithi083 ceph-mon[77242]: pgmap v8486: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:08.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:11:08 smithi153 ceph-mon[70044]: pgmap v8486: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:10.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:11:10 smithi083 ceph-mon[79930]: pgmap v8487: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:10.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:11:10 smithi083 ceph-mon[77242]: pgmap v8487: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:10.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:11:10 smithi153 ceph-mon[70044]: pgmap v8487: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:11.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:11:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:11:11.199Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:11:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:11:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:11:11] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T10:11:12.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:11:12 smithi083 ceph-mon[79930]: pgmap v8488: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:12.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:11:12 smithi083 ceph-mon[77242]: pgmap v8488: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:12.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:11:12 smithi153 ceph-mon[70044]: pgmap v8488: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:11:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:11:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:11:13.086 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:11:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:11:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:11:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:11:13.376 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:11:13.376 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 51s ago 4h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:11:13.377 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 9m ago 4h 63.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:11:13.377 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 51s ago 4h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:11:13.377 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 9m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:11:13.377 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 9m ago 4h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:11:13.378 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 51s ago 4h 757M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:11:13.378 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 51s ago 4h 1091M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:11:13.378 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 9m ago 4h 454M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:11:13.378 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 51s ago 4h 625M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:11:13.379 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 51s ago 4h 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:11:13.379 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 9m ago 4h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:11:13.379 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 51s ago 4h 767M 2492M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:11:13.379 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 51s ago 4h 758M 2492M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:11:13.380 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 51s ago 4h 593M 2492M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:11:13.380 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 51s ago 4h 739M 2492M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:11:13.380 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 9m ago 4h 631M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:11:13.380 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 9m ago 4h 785M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:11:13.380 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 9m ago 4h 700M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:11:13.381 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 9m ago 4h 827M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:11:13.381 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 9m ago 4h 177M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:11:13.742 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:11:13.743 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:11:13.743 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:11:13.743 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:11:13.744 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:11:13.744 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:11:13.744 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:11:13.744 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:11:13.745 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:11:13.745 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:11:13.745 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:11:13.745 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:11:13.746 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:11:13.746 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:11:13.746 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:11:13.746 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:11:13.747 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:11:13.747 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:11:14.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:11:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:11:14.580Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:11:14.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:11:14 smithi083 ceph-mon[77242]: pgmap v8489: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:14.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:11:14 smithi083 ceph-mon[77242]: from='client.44007 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:11:14.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:11:14 smithi083 ceph-mon[77242]: from='client.44013 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:11:14.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:11:14 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/317442247' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:11:14.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:11:14 smithi083 ceph-mon[79930]: pgmap v8489: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:14.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:11:14 smithi083 ceph-mon[79930]: from='client.44007 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:11:14.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:11:14 smithi083 ceph-mon[79930]: from='client.44013 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:11:14.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:11:14 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/317442247' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:11:14.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:11:14 smithi153 ceph-mon[70044]: pgmap v8489: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:14.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:11:14 smithi153 ceph-mon[70044]: from='client.44007 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:11:14.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:11:14 smithi153 ceph-mon[70044]: from='client.44013 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:11:14.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:11:14 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/317442247' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:11:15.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:11:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:11:15.287Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:11:16.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:11:16 smithi083 ceph-mon[77242]: pgmap v8490: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:16.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:11:16 smithi083 ceph-mon[79930]: pgmap v8490: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:16.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:11:16 smithi153 ceph-mon[70044]: pgmap v8490: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:18.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:11:18 smithi083 ceph-mon[77242]: pgmap v8491: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:18.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:11:18 smithi083 ceph-mon[79930]: pgmap v8491: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:18.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:11:18 smithi153 ceph-mon[70044]: pgmap v8491: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:20.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:11:20 smithi153 ceph-mon[70044]: pgmap v8492: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:20.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:11:20 smithi083 ceph-mon[77242]: pgmap v8492: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:20.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:11:20 smithi083 ceph-mon[79930]: pgmap v8492: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:21.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:11:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:11:21.199Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:11:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:11:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:11:21] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T10:11:22.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:11:22 smithi153 ceph-mon[70044]: pgmap v8493: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:11:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:11:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:11:22.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:11:22 smithi083 ceph-mon[79930]: pgmap v8493: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:22.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:11:22 smithi083 ceph-mon[77242]: pgmap v8493: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:11:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:11:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:11:24.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:11:24 smithi153 ceph-mon[70044]: pgmap v8494: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:24.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:11:24 smithi083 ceph-mon[79930]: pgmap v8494: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:24.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:11:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:11:24.581Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:11:24.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:11:24 smithi083 ceph-mon[77242]: pgmap v8494: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:25.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:11:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:11:25.288Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:11:26.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:11:26 smithi153 ceph-mon[70044]: pgmap v8495: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:26.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:11:26 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:11:26.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:11:26 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:11:26.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:11:26 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:11:26.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:11:26 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:11:26.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:11:26 smithi083 ceph-mon[77242]: pgmap v8495: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:26.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:11:26 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:11:26.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:11:26 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:11:26.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:11:26 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:11:26.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:11:26 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:11:26.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:11:26 smithi083 ceph-mon[79930]: pgmap v8495: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:26.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:11:26 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:11:26.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:11:26 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:11:26.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:11:26 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:11:26.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:11:26 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:11:28.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:11:28 smithi153 ceph-mon[70044]: pgmap v8496: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:28.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:11:28 smithi083 ceph-mon[79930]: pgmap v8496: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:28.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:11:28 smithi083 ceph-mon[77242]: pgmap v8496: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:30.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:11:30 smithi153 ceph-mon[70044]: pgmap v8497: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:30.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:11:30 smithi083 ceph-mon[79930]: pgmap v8497: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:30.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:11:30 smithi083 ceph-mon[77242]: pgmap v8497: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:31.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:11:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:11:31.200Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:11:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:11:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:11:31] "GET /metrics HTTP/1.1" 200 225227 "" "Prometheus/2.33.4" 2022-09-29T10:11:32.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:11:32 smithi153 ceph-mon[70044]: pgmap v8498: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:11:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:11:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:11:32.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:11:32 smithi083 ceph-mon[79930]: pgmap v8498: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:32.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:11:32 smithi083 ceph-mon[77242]: pgmap v8498: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:11:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:11:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:11:34.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:11:34 smithi153 ceph-mon[70044]: pgmap v8499: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:34.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:11:34 smithi083 ceph-mon[79930]: pgmap v8499: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:34.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:11:34 smithi083 ceph-mon[77242]: pgmap v8499: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:34.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:11:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:11:34.582Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:11:35.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:11:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:11:35.289Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:11:36.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:11:36 smithi153 ceph-mon[70044]: pgmap v8500: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:36.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:11:36 smithi083 ceph-mon[79930]: pgmap v8500: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:36.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:11:36 smithi083 ceph-mon[77242]: pgmap v8500: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:38.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:11:38 smithi153 ceph-mon[70044]: pgmap v8501: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:38.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:11:38 smithi083 ceph-mon[79930]: pgmap v8501: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:38.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:11:38 smithi083 ceph-mon[77242]: pgmap v8501: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:11:40 smithi153 ceph-mon[70044]: pgmap v8502: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:11:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:11:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:11:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:11:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:11:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:11:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:11:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:11:40.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:11:40 smithi083 ceph-mon[79930]: pgmap v8502: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:40.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:11:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:11:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:11:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:11:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:11:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:11:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:11:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:11:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:11:40 smithi083 ceph-mon[77242]: pgmap v8502: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:11:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:11:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:11:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:11:40.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:11:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:11:40.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:11:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:11:41.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:11:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:11:41.200Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:11:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:11:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:11:41] "GET /metrics HTTP/1.1" 200 225229 "" "Prometheus/2.33.4" 2022-09-29T10:11:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:11:42 smithi153 ceph-mon[70044]: pgmap v8503: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:11:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:11:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:11:42.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:11:42 smithi083 ceph-mon[77242]: pgmap v8503: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:42.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:11:42 smithi083 ceph-mon[79930]: pgmap v8503: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:11:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:11:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:11:44.071 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:11:44.359 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:11:44.359 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 82s ago 4h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:11:44.360 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 9m ago 4h 63.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:11:44.360 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 82s ago 4h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:11:44.360 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 9m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:11:44.360 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 9m ago 4h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:11:44.361 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 82s ago 4h 757M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:11:44.361 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 82s ago 4h 1091M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:11:44.361 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 9m ago 4h 454M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:11:44.361 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 82s ago 4h 625M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:11:44.362 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 82s ago 4h 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:11:44.362 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 9m ago 4h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:11:44.362 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 82s ago 4h 767M 2492M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:11:44.362 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 82s ago 4h 758M 2492M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:11:44.363 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 82s ago 4h 593M 2492M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:11:44.363 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 82s ago 4h 739M 2492M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:11:44.363 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 9m ago 4h 631M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:11:44.363 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 9m ago 4h 785M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:11:44.364 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 9m ago 4h 700M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:11:44.364 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 9m ago 4h 827M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:11:44.364 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 9m ago 4h 177M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:11:44.716 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:11:44.716 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:11:44.717 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:11:44.717 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:11:44.717 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:11:44.717 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:11:44.718 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:11:44.718 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:11:44.718 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:11:44.718 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:11:44.718 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:11:44.719 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:11:44.719 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:11:44.719 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:11:44.719 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:11:44.720 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:11:44.720 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:11:44.720 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:11:44.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:11:44 smithi153 ceph-mon[70044]: pgmap v8504: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:44.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:11:44 smithi083 ceph-mon[77242]: pgmap v8504: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:44.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:11:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:11:44.582Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:11:44.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:11:44 smithi083 ceph-mon[79930]: pgmap v8504: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:45.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:11:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:11:45.289Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:11:45.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:11:45 smithi083 ceph-mon[77242]: from='client.44025 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:11:45.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:11:45 smithi083 ceph-mon[77242]: from='client.44031 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:11:45.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:11:45 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/442501149' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:11:45.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:11:45 smithi083 ceph-mon[79930]: from='client.44025 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:11:45.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:11:45 smithi083 ceph-mon[79930]: from='client.44031 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:11:45.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:11:45 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/442501149' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:11:45.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:11:45 smithi153 ceph-mon[70044]: from='client.44025 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:11:45.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:11:45 smithi153 ceph-mon[70044]: from='client.44031 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:11:45.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:11:45 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/442501149' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:11:46.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:11:46 smithi153 ceph-mon[70044]: pgmap v8505: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:46.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:11:46 smithi083 ceph-mon[77242]: pgmap v8505: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:46.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:11:46 smithi083 ceph-mon[79930]: pgmap v8505: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:48.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:11:48 smithi153 ceph-mon[70044]: pgmap v8506: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:48.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:11:48 smithi083 ceph-mon[77242]: pgmap v8506: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:48.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:11:48 smithi083 ceph-mon[79930]: pgmap v8506: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:50.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:11:50 smithi153 ceph-mon[70044]: pgmap v8507: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:50.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:11:50 smithi083 ceph-mon[77242]: pgmap v8507: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:50.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:11:50 smithi083 ceph-mon[79930]: pgmap v8507: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:51.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:11:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:11:51.201Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:11:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:11:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:11:51] "GET /metrics HTTP/1.1" 200 225229 "" "Prometheus/2.33.4" 2022-09-29T10:11:52.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:11:52 smithi153 ceph-mon[70044]: pgmap v8508: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:11:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:11:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:11:52.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:11:52 smithi083 ceph-mon[77242]: pgmap v8508: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:52.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:11:52 smithi083 ceph-mon[79930]: pgmap v8508: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:11:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:11:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:11:54.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:11:54 smithi153 ceph-mon[70044]: pgmap v8509: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:54.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:11:54 smithi083 ceph-mon[79930]: pgmap v8509: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:54.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:11:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:11:54.583Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:11:54.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:11:54 smithi083 ceph-mon[77242]: pgmap v8509: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:55.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:11:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:11:55.290Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:11:56.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:11:56 smithi153 ceph-mon[70044]: pgmap v8510: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:56.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:11:56 smithi083 ceph-mon[79930]: pgmap v8510: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:56.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:11:56 smithi083 ceph-mon[77242]: pgmap v8510: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:58.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:11:58 smithi153 ceph-mon[70044]: pgmap v8511: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:58.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:11:58 smithi083 ceph-mon[79930]: pgmap v8511: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:11:58.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:11:58 smithi083 ceph-mon[77242]: pgmap v8511: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:00.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:12:00 smithi153 ceph-mon[70044]: pgmap v8512: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:00.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:12:00 smithi083 ceph-mon[79930]: pgmap v8512: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:00.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:12:00 smithi083 ceph-mon[77242]: pgmap v8512: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:01.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:12:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:12:01.202Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:12:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:12:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:12:01] "GET /metrics HTTP/1.1" 200 225225 "" "Prometheus/2.33.4" 2022-09-29T10:12:02.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:12:02 smithi153 ceph-mon[70044]: pgmap v8513: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:12:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:12:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:12:02.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:12:02 smithi083 ceph-mon[79930]: pgmap v8513: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:02.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:12:02 smithi083 ceph-mon[77242]: pgmap v8513: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:12:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:12:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:12:04.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:12:04 smithi153 ceph-mon[70044]: pgmap v8514: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:04.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:12:04 smithi083 ceph-mon[77242]: pgmap v8514: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:04.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:12:04 smithi083 ceph-mon[79930]: pgmap v8514: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:04.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:12:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:12:04.584Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:12:05.586 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:12:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:12:05.291Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:12:05.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:12:05 smithi083 ceph-mon[79930]: pgmap v8515: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:05.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:12:05 smithi083 ceph-mon[77242]: pgmap v8515: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:06.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:12:05 smithi153 ceph-mon[70044]: pgmap v8515: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:08.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:12:07 smithi083 ceph-mon[79930]: pgmap v8516: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:08.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:12:07 smithi083 ceph-mon[77242]: pgmap v8516: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:08.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:12:07 smithi153 ceph-mon[70044]: pgmap v8516: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:10.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:12:10 smithi083 ceph-mon[79930]: pgmap v8517: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:10.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:12:10 smithi083 ceph-mon[77242]: pgmap v8517: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:10.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:12:10 smithi153 ceph-mon[70044]: pgmap v8517: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:11.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:12:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:12:11.202Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:12:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:12:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:12:11] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T10:12:12.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:12:12 smithi083 ceph-mon[79930]: pgmap v8518: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:12.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:12:12 smithi083 ceph-mon[77242]: pgmap v8518: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:12.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:12:12 smithi153 ceph-mon[70044]: pgmap v8518: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:12:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:12:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:12:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:12:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:12:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:12:14.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:12:14 smithi083 ceph-mon[79930]: pgmap v8519: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:14.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:12:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:12:14.584Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:12:14.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:12:14 smithi083 ceph-mon[77242]: pgmap v8519: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:14.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:12:14 smithi153 ceph-mon[70044]: pgmap v8519: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:15.041 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:12:15.329 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:12:15.329 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 113s ago 4h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:12:15.330 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 10m ago 4h 63.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:12:15.330 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 113s ago 4h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:12:15.330 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 10m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:12:15.330 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 10m ago 4h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:12:15.331 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 113s ago 4h 757M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:12:15.331 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 113s ago 4h 1091M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:12:15.331 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 10m ago 4h 454M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:12:15.331 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 113s ago 4h 625M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:12:15.332 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 113s ago 4h 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:12:15.332 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 10m ago 4h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:12:15.332 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 113s ago 4h 767M 2492M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:12:15.332 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 113s ago 4h 758M 2492M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:12:15.333 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 113s ago 4h 593M 2492M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:12:15.333 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 113s ago 4h 739M 2492M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:12:15.333 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 10m ago 4h 631M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:12:15.333 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 10m ago 4h 785M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:12:15.334 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 10m ago 4h 700M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:12:15.334 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 10m ago 4h 827M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:12:15.334 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 10m ago 4h 177M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:12:15.684 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:12:15.685 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:12:15.685 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:12:15.685 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:12:15.685 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:12:15.686 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:12:15.686 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:12:15.686 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:12:15.686 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:12:15.686 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:12:15.687 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:12:15.687 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:12:15.687 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:12:15.687 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:12:15.688 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:12:15.688 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:12:15.688 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:12:15.688 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:12:15.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:12:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:12:15.291Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:12:16.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:12:16 smithi083 ceph-mon[77242]: pgmap v8520: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:16.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:12:16 smithi083 ceph-mon[77242]: from='client.53974 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:12:16.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:12:16 smithi083 ceph-mon[77242]: from='client.44049 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:12:16.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:12:16 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3305938142' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:12:16.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:12:16 smithi083 ceph-mon[79930]: pgmap v8520: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:16.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:12:16 smithi083 ceph-mon[79930]: from='client.53974 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:12:16.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:12:16 smithi083 ceph-mon[79930]: from='client.44049 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:12:16.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:12:16 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3305938142' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:12:16.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:12:16 smithi153 ceph-mon[70044]: pgmap v8520: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:16.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:12:16 smithi153 ceph-mon[70044]: from='client.53974 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:12:16.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:12:16 smithi153 ceph-mon[70044]: from='client.44049 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:12:16.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:12:16 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3305938142' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:12:18.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:12:18 smithi083 ceph-mon[79930]: pgmap v8521: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:18.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:12:18 smithi083 ceph-mon[77242]: pgmap v8521: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:18.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:12:18 smithi153 ceph-mon[70044]: pgmap v8521: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:20.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:12:20 smithi083 ceph-mon[79930]: pgmap v8522: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:20.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:12:20 smithi083 ceph-mon[77242]: pgmap v8522: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:20.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:12:20 smithi153 ceph-mon[70044]: pgmap v8522: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:21.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:12:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:12:21.203Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:12:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:12:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:12:21] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T10:12:22.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:12:22 smithi083 ceph-mon[79930]: pgmap v8523: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:22.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:12:22 smithi083 ceph-mon[77242]: pgmap v8523: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:22.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:12:22 smithi153 ceph-mon[70044]: pgmap v8523: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:12:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:12:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:12:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:12:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:12:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:12:24.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:12:24 smithi083 ceph-mon[79930]: pgmap v8524: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:12:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:12:24.585Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:12:24.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:12:24 smithi083 ceph-mon[77242]: pgmap v8524: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:24.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:12:24 smithi153 ceph-mon[70044]: pgmap v8524: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:25.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:12:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:12:25.293Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:12:26.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:12:26 smithi083 ceph-mon[79930]: pgmap v8525: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:26.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:12:26 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:12:26.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:12:26 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:12:26.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:12:26 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:12:26.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:12:26 smithi083 ceph-mon[77242]: pgmap v8525: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:26.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:12:26 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:12:26.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:12:26 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:12:26.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:12:26 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:12:26.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:12:26 smithi153 ceph-mon[70044]: pgmap v8525: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:26.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:12:26 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:12:26.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:12:26 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:12:26.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:12:26 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:12:28.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:12:28 smithi083 ceph-mon[79930]: pgmap v8526: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:28.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:12:28 smithi083 ceph-mon[77242]: pgmap v8526: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:28.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:12:28 smithi153 ceph-mon[70044]: pgmap v8526: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:30.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:12:30 smithi083 ceph-mon[79930]: pgmap v8527: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:30.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:12:30 smithi083 ceph-mon[77242]: pgmap v8527: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:30.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:12:30 smithi153 ceph-mon[70044]: pgmap v8527: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:31.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:12:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:12:31.205Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:12:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:12:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:12:31] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T10:12:32.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:12:32 smithi083 ceph-mon[79930]: pgmap v8528: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:32.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:12:32 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:12:32.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:12:32 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:12:32.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:12:32 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:12:32.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:12:32 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:12:32.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:12:32 smithi083 ceph-mon[77242]: pgmap v8528: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:32.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:12:32 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:12:32.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:12:32 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:12:32.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:12:32 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:12:32.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:12:32 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:12:32.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:12:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:12:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:12:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:12:32 smithi153 ceph-mon[70044]: pgmap v8528: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:32.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:12:32 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:12:32.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:12:32 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:12:32.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:12:32 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:12:32.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:12:32 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:12:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:12:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:12:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:12:34.613 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:12:34 smithi153 ceph-mon[70044]: pgmap v8529: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:34.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:12:34 smithi083 ceph-mon[79930]: pgmap v8529: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:34.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:12:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:12:34.586Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:12:34.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:12:34 smithi083 ceph-mon[77242]: pgmap v8529: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:35.682 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:12:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:12:35.294Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:12:35.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:12:35 smithi083 ceph-mon[77242]: Detected new or changed devices on smithi153 2022-09-29T10:12:35.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:12:35 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:12:35.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:12:35 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:12:35.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:12:35 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:12:35.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:12:35 smithi083 ceph-mon[77242]: pgmap v8530: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:35.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:12:35 smithi083 ceph-mon[79930]: Detected new or changed devices on smithi153 2022-09-29T10:12:35.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:12:35 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:12:35.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:12:35 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:12:35.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:12:35 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:12:35.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:12:35 smithi083 ceph-mon[79930]: pgmap v8530: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:36.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:12:35 smithi153 ceph-mon[70044]: Detected new or changed devices on smithi153 2022-09-29T10:12:36.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:12:35 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:12:36.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:12:35 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:12:36.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:12:35 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:12:36.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:12:35 smithi153 ceph-mon[70044]: pgmap v8530: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:38.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:12:37 smithi083 ceph-mon[77242]: pgmap v8531: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:38.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:12:37 smithi083 ceph-mon[79930]: pgmap v8531: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:38.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:12:37 smithi153 ceph-mon[70044]: pgmap v8531: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:40.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:12:40 smithi083 ceph-mon[79930]: pgmap v8532: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:40.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:12:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:12:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:12:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:12:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:12:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:12:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:12:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:12:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:12:40 smithi083 ceph-mon[77242]: pgmap v8532: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:12:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:12:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:12:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:12:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:12:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:12:40.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:12:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:12:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:12:40 smithi153 ceph-mon[70044]: pgmap v8532: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:12:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:12:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:12:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:12:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:12:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:12:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:12:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:12:41.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:12:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:12:41.205Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:12:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:12:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:12:41] "GET /metrics HTTP/1.1" 200 225228 "" "Prometheus/2.33.4" 2022-09-29T10:12:42.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:12:42 smithi083 ceph-mon[79930]: pgmap v8533: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:42.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:12:42 smithi083 ceph-mon[77242]: pgmap v8533: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:12:42 smithi153 ceph-mon[70044]: pgmap v8533: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:12:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:12:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:12:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:12:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:12:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:12:44.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:12:44 smithi083 ceph-mon[79930]: pgmap v8534: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:44.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:12:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:12:44.587Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:12:44.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:12:44 smithi083 ceph-mon[77242]: pgmap v8534: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:44.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:12:44 smithi153 ceph-mon[70044]: pgmap v8534: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:45.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:12:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:12:45.294Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:12:46.012 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:12:46.299 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:12:46.300 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 2m ago 4h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:12:46.300 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 15s ago 4h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:12:46.300 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 2m ago 4h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:12:46.300 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 15s ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:12:46.301 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 15s ago 4h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:12:46.301 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 2m ago 4h 757M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:12:46.301 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 2m ago 4h 1091M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:12:46.301 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 15s ago 4h 470M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:12:46.302 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 2m ago 4h 625M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:12:46.302 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 2m ago 4h 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:12:46.302 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 15s ago 4h 25.8M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:12:46.302 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 2m ago 4h 767M 2492M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:12:46.303 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 2m ago 4h 758M 2492M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:12:46.303 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 2m ago 4h 593M 2492M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:12:46.303 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 2m ago 4h 739M 2492M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:12:46.303 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 15s ago 4h 653M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:12:46.303 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 15s ago 4h 812M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:12:46.304 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 15s ago 4h 725M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:12:46.304 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 15s ago 4h 857M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:12:46.304 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 15s ago 4h 181M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:12:46.657 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:12:46.657 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:12:46.657 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:12:46.658 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:12:46.658 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:12:46.658 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:12:46.658 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:12:46.658 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:12:46.659 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:12:46.659 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:12:46.659 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:12:46.659 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:12:46.660 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:12:46.660 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:12:46.660 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:12:46.660 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:12:46.660 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:12:46.661 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:12:46.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:12:46 smithi083 ceph-mon[77242]: pgmap v8535: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:46.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:12:46 smithi083 ceph-mon[79930]: pgmap v8535: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:46.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:12:46 smithi153 ceph-mon[70044]: pgmap v8535: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:47.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:12:47 smithi083 ceph-mon[79930]: from='client.53992 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:12:47.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:12:47 smithi083 ceph-mon[79930]: from='client.44067 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:12:47.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:12:47 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/862825086' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:12:47.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:12:47 smithi083 ceph-mon[77242]: from='client.53992 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:12:47.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:12:47 smithi083 ceph-mon[77242]: from='client.44067 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:12:47.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:12:47 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/862825086' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:12:47.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:12:47 smithi153 ceph-mon[70044]: from='client.53992 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:12:47.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:12:47 smithi153 ceph-mon[70044]: from='client.44067 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:12:47.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:12:47 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/862825086' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:12:48.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:12:48 smithi083 ceph-mon[79930]: pgmap v8536: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:48.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:12:48 smithi083 ceph-mon[77242]: pgmap v8536: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:48.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:12:48 smithi153 ceph-mon[70044]: pgmap v8536: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:50.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:12:50 smithi083 ceph-mon[77242]: pgmap v8537: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:50.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:12:50 smithi083 ceph-mon[79930]: pgmap v8537: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:50.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:12:50 smithi153 ceph-mon[70044]: pgmap v8537: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:51.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:12:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:12:51.206Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:12:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:12:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:12:51] "GET /metrics HTTP/1.1" 200 225228 "" "Prometheus/2.33.4" 2022-09-29T10:12:52.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:12:52 smithi083 ceph-mon[77242]: pgmap v8538: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:52.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:12:52 smithi083 ceph-mon[79930]: pgmap v8538: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:52.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:12:52 smithi153 ceph-mon[70044]: pgmap v8538: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:12:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:12:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:12:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:12:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:12:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:12:54.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:12:54 smithi083 ceph-mon[77242]: pgmap v8539: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:54.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:12:54 smithi083 ceph-mon[79930]: pgmap v8539: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:54.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:12:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:12:54.587Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:12:54.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:12:54 smithi153 ceph-mon[70044]: pgmap v8539: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:55.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:12:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:12:55.294Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:12:56.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:12:56 smithi083 ceph-mon[77242]: pgmap v8540: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:56.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:12:56 smithi083 ceph-mon[79930]: pgmap v8540: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:56.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:12:56 smithi153 ceph-mon[70044]: pgmap v8540: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:58.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:12:58 smithi083 ceph-mon[77242]: pgmap v8541: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:58.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:12:58 smithi083 ceph-mon[79930]: pgmap v8541: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:12:58.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:12:58 smithi153 ceph-mon[70044]: pgmap v8541: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:00.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:13:00 smithi083 ceph-mon[79930]: pgmap v8542: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:00.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:13:00 smithi083 ceph-mon[77242]: pgmap v8542: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:00.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:13:00 smithi153 ceph-mon[70044]: pgmap v8542: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:01.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:13:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:13:01.206Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:13:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:13:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:13:01] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T10:13:02.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:13:02 smithi083 ceph-mon[77242]: pgmap v8543: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:02.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:13:02 smithi083 ceph-mon[79930]: pgmap v8543: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:02.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:13:02 smithi153 ceph-mon[70044]: pgmap v8543: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:13:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:13:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:13:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:13:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:13:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:13:04.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:13:04 smithi083 ceph-mon[79930]: pgmap v8544: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:04.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:13:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:13:04.588Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:13:04.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:13:04 smithi083 ceph-mon[77242]: pgmap v8544: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:04.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:13:04 smithi153 ceph-mon[70044]: pgmap v8544: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:05.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:13:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:13:05.294Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:13:06.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:13:06 smithi083 ceph-mon[79930]: pgmap v8545: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:06.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:13:06 smithi083 ceph-mon[77242]: pgmap v8545: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:06.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:13:06 smithi153 ceph-mon[70044]: pgmap v8545: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:08.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:13:08 smithi083 ceph-mon[79930]: pgmap v8546: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:08.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:13:08 smithi083 ceph-mon[77242]: pgmap v8546: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:08.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:13:08 smithi153 ceph-mon[70044]: pgmap v8546: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:10.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:13:10 smithi153 ceph-mon[70044]: pgmap v8547: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:10.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:13:10 smithi083 ceph-mon[79930]: pgmap v8547: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:10.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:13:10 smithi083 ceph-mon[77242]: pgmap v8547: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:11.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:13:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:13:11.207Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:13:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:13:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:13:11] "GET /metrics HTTP/1.1" 200 225229 "" "Prometheus/2.33.4" 2022-09-29T10:13:12.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:13:12 smithi153 ceph-mon[70044]: pgmap v8548: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:13:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:13:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:13:12.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:13:12 smithi083 ceph-mon[77242]: pgmap v8548: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:12.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:13:12 smithi083 ceph-mon[79930]: pgmap v8548: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:13:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:13:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:13:14.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:13:14 smithi153 ceph-mon[70044]: pgmap v8549: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:14.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:13:14 smithi083 ceph-mon[79930]: pgmap v8549: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:14.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:13:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:13:14.588Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:13:14.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:13:14 smithi083 ceph-mon[77242]: pgmap v8549: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:15.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:13:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:13:15.294Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:13:16.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:13:16 smithi153 ceph-mon[70044]: pgmap v8550: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:16.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:13:16 smithi083 ceph-mon[77242]: pgmap v8550: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:16.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:13:16 smithi083 ceph-mon[79930]: pgmap v8550: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:16.980 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:13:17.268 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:13:17.268 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 2m ago 4h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:13:17.269 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 46s ago 4h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:13:17.269 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 2m ago 4h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:13:17.269 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 46s ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:13:17.270 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 46s ago 4h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:13:17.270 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 2m ago 4h 757M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:13:17.270 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 2m ago 4h 1091M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:13:17.270 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 46s ago 4h 470M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:13:17.271 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 2m ago 4h 625M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:13:17.271 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 2m ago 4h 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:13:17.271 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 46s ago 4h 25.8M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:13:17.271 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 2m ago 4h 767M 2492M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:13:17.272 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 2m ago 4h 758M 2492M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:13:17.272 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 2m ago 4h 593M 2492M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:13:17.272 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 2m ago 4h 739M 2492M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:13:17.272 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 46s ago 4h 653M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:13:17.273 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 46s ago 4h 812M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:13:17.273 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 46s ago 4h 725M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:13:17.273 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 46s ago 4h 857M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:13:17.273 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 46s ago 4h 181M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:13:17.627 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:13:17.627 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:13:17.627 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:13:17.628 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:13:17.628 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:13:17.628 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:13:17.628 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:13:17.628 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:13:17.629 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:13:17.629 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:13:17.629 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:13:17.629 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:13:17.630 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:13:17.630 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:13:17.630 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:13:17.630 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:13:17.631 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:13:17.631 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:13:18.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:13:18 smithi153 ceph-mon[70044]: pgmap v8551: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:18.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:13:18 smithi153 ceph-mon[70044]: from='client.44079 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:13:18.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:13:18 smithi153 ceph-mon[70044]: from='client.44085 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:13:18.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:13:18 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/329780123' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:13:18.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:13:18 smithi083 ceph-mon[79930]: pgmap v8551: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:18.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:13:18 smithi083 ceph-mon[79930]: from='client.44079 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:13:18.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:13:18 smithi083 ceph-mon[79930]: from='client.44085 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:13:18.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:13:18 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/329780123' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:13:18.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:13:18 smithi083 ceph-mon[77242]: pgmap v8551: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:18.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:13:18 smithi083 ceph-mon[77242]: from='client.44079 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:13:18.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:13:18 smithi083 ceph-mon[77242]: from='client.44085 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:13:18.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:13:18 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/329780123' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:13:20.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:13:20 smithi153 ceph-mon[70044]: pgmap v8552: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:20.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:13:20 smithi083 ceph-mon[79930]: pgmap v8552: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:20.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:13:20 smithi083 ceph-mon[77242]: pgmap v8552: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:21.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:13:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:13:21.208Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:13:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:13:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:13:21] "GET /metrics HTTP/1.1" 200 225229 "" "Prometheus/2.33.4" 2022-09-29T10:13:22.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:13:22 smithi153 ceph-mon[70044]: pgmap v8553: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:13:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:13:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:13:22.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:13:22 smithi083 ceph-mon[77242]: pgmap v8553: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:22.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:13:22 smithi083 ceph-mon[79930]: pgmap v8553: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:13:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:13:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:13:24.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:13:24 smithi153 ceph-mon[70044]: pgmap v8554: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:24.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:13:24 smithi083 ceph-mon[79930]: pgmap v8554: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:24.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:13:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:13:24.589Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:13:24.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:13:24 smithi083 ceph-mon[77242]: pgmap v8554: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:25.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:13:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:13:25.294Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:13:26.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:13:26 smithi153 ceph-mon[70044]: pgmap v8555: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:26.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:13:26 smithi083 ceph-mon[77242]: pgmap v8555: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:26.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:13:26 smithi083 ceph-mon[79930]: pgmap v8555: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:28.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:13:28 smithi153 ceph-mon[70044]: pgmap v8556: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:28.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:13:28 smithi083 ceph-mon[79930]: pgmap v8556: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:28.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:13:28 smithi083 ceph-mon[77242]: pgmap v8556: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:30.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:13:30 smithi153 ceph-mon[70044]: pgmap v8557: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:30.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:13:30 smithi083 ceph-mon[79930]: pgmap v8557: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:30.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:13:30 smithi083 ceph-mon[77242]: pgmap v8557: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:31.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:13:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:13:31.208Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:13:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:13:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:13:31] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T10:13:32.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:13:32 smithi153 ceph-mon[70044]: pgmap v8558: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:13:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:13:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:13:32.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:13:32 smithi083 ceph-mon[77242]: pgmap v8558: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:32.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:13:32 smithi083 ceph-mon[79930]: pgmap v8558: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:13:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:13:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:13:34.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:13:34 smithi153 ceph-mon[70044]: pgmap v8559: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:34.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:13:34 smithi083 ceph-mon[79930]: pgmap v8559: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:34.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:13:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:13:34.589Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:13:34.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:13:34 smithi083 ceph-mon[77242]: pgmap v8559: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:35.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:13:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:13:35.300Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:13:35.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:13:35 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:13:35.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:13:35 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:13:35.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:13:35 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:13:35.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:13:35 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:13:35.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:13:35 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:13:35.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:13:35 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:13:35.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:13:35 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:13:35.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:13:35 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:13:35.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:13:35 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:13:35.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:13:35 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:13:35.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:13:35 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:13:35.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:13:35 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:13:36.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:13:36 smithi153 ceph-mon[70044]: pgmap v8560: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:36.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:13:36 smithi083 ceph-mon[77242]: pgmap v8560: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:36.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:13:36 smithi083 ceph-mon[79930]: pgmap v8560: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:38.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:13:38 smithi153 ceph-mon[70044]: pgmap v8561: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:38.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:13:38 smithi083 ceph-mon[77242]: pgmap v8561: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:38.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:13:38 smithi083 ceph-mon[79930]: pgmap v8561: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:13:40 smithi153 ceph-mon[70044]: pgmap v8562: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:13:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:13:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:13:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:13:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:13:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:13:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:13:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:13:40.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:13:40 smithi083 ceph-mon[77242]: pgmap v8562: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:40.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:13:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:13:40.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:13:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:13:40.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:13:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:13:40.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:13:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:13:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:13:40 smithi083 ceph-mon[79930]: pgmap v8562: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:13:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:13:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:13:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:13:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:13:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:13:40.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:13:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:13:41.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:13:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:13:41.209Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:13:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:13:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:13:41] "GET /metrics HTTP/1.1" 200 225224 "" "Prometheus/2.33.4" 2022-09-29T10:13:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:13:42 smithi153 ceph-mon[70044]: pgmap v8563: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:13:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:13:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:13:42.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:13:42 smithi083 ceph-mon[77242]: pgmap v8563: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:42.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:13:42 smithi083 ceph-mon[79930]: pgmap v8563: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:13:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:13:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:13:44.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:13:44 smithi153 ceph-mon[70044]: pgmap v8564: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:44.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:13:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:13:44.590Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:13:44.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:13:44 smithi083 ceph-mon[77242]: pgmap v8564: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:44.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:13:44 smithi083 ceph-mon[79930]: pgmap v8564: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:45.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:13:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:13:45.296Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:13:46.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:13:46 smithi153 ceph-mon[70044]: pgmap v8565: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:46.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:13:46 smithi083 ceph-mon[77242]: pgmap v8565: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:46.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:13:46 smithi083 ceph-mon[79930]: pgmap v8565: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:47.951 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:13:48.238 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:13:48.238 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 3m ago 4h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:13:48.238 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 77s ago 4h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:13:48.238 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 3m ago 4h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:13:48.239 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 77s ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:13:48.239 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 77s ago 4h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:13:48.239 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 3m ago 4h 757M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:13:48.239 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 3m ago 4h 1091M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:13:48.240 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 77s ago 4h 470M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:13:48.240 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 3m ago 4h 625M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:13:48.240 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 3m ago 4h 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:13:48.240 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 77s ago 4h 25.8M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:13:48.241 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 3m ago 4h 767M 2492M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:13:48.241 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 3m ago 4h 758M 2492M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:13:48.241 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 3m ago 4h 593M 2492M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:13:48.241 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 3m ago 4h 739M 2492M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:13:48.242 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 77s ago 4h 653M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:13:48.242 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 77s ago 4h 812M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:13:48.242 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 77s ago 4h 725M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:13:48.242 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 77s ago 4h 857M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:13:48.243 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 77s ago 4h 181M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:13:48.598 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:13:48.598 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:13:48.598 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:13:48.599 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:13:48.599 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:13:48.599 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:13:48.599 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:13:48.600 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:13:48.600 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:13:48.600 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:13:48.600 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:13:48.601 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:13:48.601 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:13:48.601 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:13:48.601 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:13:48.602 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:13:48.602 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:13:48.602 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:13:48.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:13:48 smithi153 ceph-mon[70044]: pgmap v8566: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:48.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:13:48 smithi083 ceph-mon[77242]: pgmap v8566: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:48.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:13:48 smithi083 ceph-mon[79930]: pgmap v8566: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:49.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:13:49 smithi153 ceph-mon[70044]: from='client.54028 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:13:49.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:13:49 smithi153 ceph-mon[70044]: from='client.54034 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:13:49.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:13:49 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1789381625' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:13:49.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:13:49 smithi083 ceph-mon[79930]: from='client.54028 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:13:49.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:13:49 smithi083 ceph-mon[79930]: from='client.54034 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:13:49.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:13:49 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1789381625' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:13:49.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:13:49 smithi083 ceph-mon[77242]: from='client.54028 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:13:49.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:13:49 smithi083 ceph-mon[77242]: from='client.54034 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:13:49.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:13:49 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1789381625' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:13:50.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:13:50 smithi153 ceph-mon[70044]: pgmap v8567: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:50.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:13:50 smithi083 ceph-mon[79930]: pgmap v8567: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:50.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:13:50 smithi083 ceph-mon[77242]: pgmap v8567: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:51.506 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:13:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:13:51.210Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:13:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:13:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:13:51] "GET /metrics HTTP/1.1" 200 225224 "" "Prometheus/2.33.4" 2022-09-29T10:13:52.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:13:52 smithi153 ceph-mon[70044]: pgmap v8568: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:13:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:13:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:13:52.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:13:52 smithi083 ceph-mon[79930]: pgmap v8568: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:52.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:13:52 smithi083 ceph-mon[77242]: pgmap v8568: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:13:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:13:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:13:54.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:13:54 smithi153 ceph-mon[70044]: pgmap v8569: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:54.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:13:54 smithi083 ceph-mon[77242]: pgmap v8569: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:54.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:13:54 smithi083 ceph-mon[79930]: pgmap v8569: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:54.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:13:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:13:54.591Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:13:55.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:13:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:13:55.297Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:13:56.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:13:56 smithi153 ceph-mon[70044]: pgmap v8570: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:56.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:13:56 smithi083 ceph-mon[77242]: pgmap v8570: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:56.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:13:56 smithi083 ceph-mon[79930]: pgmap v8570: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:58.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:13:58 smithi153 ceph-mon[70044]: pgmap v8571: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:58.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:13:58 smithi083 ceph-mon[77242]: pgmap v8571: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:13:58.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:13:58 smithi083 ceph-mon[79930]: pgmap v8571: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:00.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:14:00 smithi153 ceph-mon[70044]: pgmap v8572: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:00.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:14:00 smithi083 ceph-mon[79930]: pgmap v8572: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:00.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:14:00 smithi083 ceph-mon[77242]: pgmap v8572: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:01.506 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:14:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:14:01.211Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:14:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:14:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:14:01] "GET /metrics HTTP/1.1" 200 225228 "" "Prometheus/2.33.4" 2022-09-29T10:14:02.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:14:02 smithi153 ceph-mon[70044]: pgmap v8573: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:14:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:14:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:14:02.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:14:02 smithi083 ceph-mon[79930]: pgmap v8573: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:02.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:14:02 smithi083 ceph-mon[77242]: pgmap v8573: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:14:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:14:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:14:04.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:14:04 smithi153 ceph-mon[70044]: pgmap v8574: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:04.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:14:04 smithi083 ceph-mon[77242]: pgmap v8574: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:04.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:14:04 smithi083 ceph-mon[79930]: pgmap v8574: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:04.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:14:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:14:04.591Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:14:05.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:14:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:14:05.298Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:14:06.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:14:06 smithi153 ceph-mon[70044]: pgmap v8575: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:06.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:14:06 smithi083 ceph-mon[79930]: pgmap v8575: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:06.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:14:06 smithi083 ceph-mon[77242]: pgmap v8575: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:07.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:14:07 smithi083 ceph-mon[79930]: pgmap v8576: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:07.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:14:07 smithi083 ceph-mon[77242]: pgmap v8576: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:08.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:14:07 smithi153 ceph-mon[70044]: pgmap v8576: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:10.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:14:10 smithi083 ceph-mon[77242]: pgmap v8577: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:10.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:14:10 smithi083 ceph-mon[79930]: pgmap v8577: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:10.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:14:10 smithi153 ceph-mon[70044]: pgmap v8577: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:11.507 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:14:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:14:11.212Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:14:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:14:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:14:11] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T10:14:12.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:14:12 smithi083 ceph-mon[79930]: pgmap v8578: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:12.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:14:12 smithi083 ceph-mon[77242]: pgmap v8578: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:12.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:14:12 smithi153 ceph-mon[70044]: pgmap v8578: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:14:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:14:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:14:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:14:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:14:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:14:14.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:14:14 smithi083 ceph-mon[77242]: pgmap v8579: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:14.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:14:14 smithi083 ceph-mon[79930]: pgmap v8579: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:14.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:14:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:14:14.592Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:14:14.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:14:14 smithi153 ceph-mon[70044]: pgmap v8579: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:15.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:14:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:14:15.298Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:14:16.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:14:16 smithi083 ceph-mon[79930]: pgmap v8580: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:16.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:14:16 smithi083 ceph-mon[77242]: pgmap v8580: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:16.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:14:16 smithi153 ceph-mon[70044]: pgmap v8580: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:18.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:14:18 smithi083 ceph-mon[79930]: pgmap v8581: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:18.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:14:18 smithi083 ceph-mon[77242]: pgmap v8581: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:18.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:14:18 smithi153 ceph-mon[70044]: pgmap v8581: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:18.922 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:14:19.211 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:14:19.211 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 3m ago 4h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:14:19.211 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 108s ago 4h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:14:19.212 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 3m ago 4h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:14:19.212 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 108s ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:14:19.212 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 108s ago 4h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:14:19.212 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 3m ago 4h 757M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:14:19.213 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 3m ago 4h 1091M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:14:19.213 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 108s ago 4h 470M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:14:19.213 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 3m ago 4h 625M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:14:19.213 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 3m ago 4h 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:14:19.213 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 108s ago 4h 25.8M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:14:19.214 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 3m ago 4h 767M 2492M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:14:19.214 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 3m ago 4h 758M 2492M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:14:19.214 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 3m ago 4h 593M 2492M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:14:19.214 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 3m ago 4h 739M 2492M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:14:19.214 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 108s ago 4h 653M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:14:19.215 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 108s ago 4h 812M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:14:19.215 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 108s ago 4h 725M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:14:19.215 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 108s ago 4h 857M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:14:19.215 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 108s ago 4h 181M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:14:19.566 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:14:19.566 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:14:19.567 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:14:19.567 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:14:19.567 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:14:19.567 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:14:19.568 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:14:19.568 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:14:19.568 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:14:19.568 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:14:19.568 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:14:19.569 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:14:19.569 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:14:19.569 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:14:19.569 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:14:19.570 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:14:19.570 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:14:19.570 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:14:20.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:14:20 smithi083 ceph-mon[79930]: pgmap v8582: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:20.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:14:20 smithi083 ceph-mon[79930]: from='client.54046 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:14:20.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:14:20 smithi083 ceph-mon[79930]: from='client.54052 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:14:20.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:14:20 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1467998516' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:14:20.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:14:20 smithi083 ceph-mon[77242]: pgmap v8582: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:20.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:14:20 smithi083 ceph-mon[77242]: from='client.54046 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:14:20.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:14:20 smithi083 ceph-mon[77242]: from='client.54052 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:14:20.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:14:20 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1467998516' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:14:20.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:14:20 smithi153 ceph-mon[70044]: pgmap v8582: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:20.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:14:20 smithi153 ceph-mon[70044]: from='client.54046 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:14:20.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:14:20 smithi153 ceph-mon[70044]: from='client.54052 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:14:20.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:14:20 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1467998516' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:14:21.507 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:14:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:14:21.212Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:14:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:14:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:14:21] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T10:14:22.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:14:22 smithi083 ceph-mon[79930]: pgmap v8583: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:22.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:14:22 smithi083 ceph-mon[77242]: pgmap v8583: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:22.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:14:22 smithi153 ceph-mon[70044]: pgmap v8583: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:14:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:14:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:14:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:14:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:14:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:14:24.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:14:24 smithi083 ceph-mon[79930]: pgmap v8584: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:14:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:14:24.593Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:14:24.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:14:24 smithi083 ceph-mon[77242]: pgmap v8584: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:24.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:14:24 smithi153 ceph-mon[70044]: pgmap v8584: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:25.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:14:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:14:25.298Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:14:26.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:14:26 smithi083 ceph-mon[79930]: pgmap v8585: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:26.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:14:26 smithi083 ceph-mon[77242]: pgmap v8585: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:26.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:14:26 smithi153 ceph-mon[70044]: pgmap v8585: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:28.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:14:28 smithi153 ceph-mon[70044]: pgmap v8586: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:28.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:14:28 smithi083 ceph-mon[79930]: pgmap v8586: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:28.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:14:28 smithi083 ceph-mon[77242]: pgmap v8586: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:30.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:14:30 smithi153 ceph-mon[70044]: pgmap v8587: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:30.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:14:30 smithi083 ceph-mon[79930]: pgmap v8587: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:30.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:14:30 smithi083 ceph-mon[77242]: pgmap v8587: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:31.507 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:14:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:14:31.212Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:14:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:14:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:14:31] "GET /metrics HTTP/1.1" 200 225229 "" "Prometheus/2.33.4" 2022-09-29T10:14:32.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:14:32 smithi153 ceph-mon[70044]: pgmap v8588: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:14:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:14:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:14:32.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:14:32 smithi083 ceph-mon[77242]: pgmap v8588: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:32.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:14:32 smithi083 ceph-mon[79930]: pgmap v8588: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:14:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:14:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:14:34.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:14:34 smithi153 ceph-mon[70044]: pgmap v8589: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:34.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:14:34 smithi083 ceph-mon[79930]: pgmap v8589: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:34.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:14:34 smithi083 ceph-mon[77242]: pgmap v8589: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:34.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:14:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:14:34.594Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:14:35.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:14:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:14:35.298Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:14:35.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:14:35 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:14:35.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:14:35 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:14:35.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:14:35 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:14:35.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:14:35 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:14:35.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:14:35 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:14:35.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:14:35 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:14:35.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:14:35 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:14:35.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:14:35 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:14:35.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:14:35 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:14:36.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:14:36 smithi153 ceph-mon[70044]: pgmap v8590: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:36.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:14:36 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:14:36.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:14:36 smithi083 ceph-mon[79930]: pgmap v8590: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:36.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:14:36 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:14:36.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:14:36 smithi083 ceph-mon[77242]: pgmap v8590: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:36.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:14:36 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:14:38.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:14:38 smithi153 ceph-mon[70044]: pgmap v8591: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:38.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:14:38 smithi083 ceph-mon[79930]: pgmap v8591: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:38.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:14:38 smithi083 ceph-mon[77242]: pgmap v8591: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:14:40 smithi153 ceph-mon[70044]: pgmap v8592: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:14:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:14:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:14:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:14:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:14:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:14:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:14:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:14:40.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:14:40 smithi083 ceph-mon[79930]: pgmap v8592: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:40.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:14:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:14:40.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:14:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:14:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:14:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:14:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:14:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:14:40.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:14:40 smithi083 ceph-mon[77242]: pgmap v8592: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:14:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:14:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:14:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:14:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:14:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:14:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:14:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:14:41.507 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:14:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:14:41.213Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:14:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:14:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:14:41] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T10:14:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:14:42 smithi153 ceph-mon[70044]: pgmap v8593: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:14:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:14:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:14:42.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:14:42 smithi083 ceph-mon[79930]: pgmap v8593: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:42.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:14:42 smithi083 ceph-mon[77242]: pgmap v8593: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:14:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:14:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:14:44.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:14:44 smithi153 ceph-mon[70044]: pgmap v8594: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:44.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:14:44 smithi083 ceph-mon[79930]: pgmap v8594: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:44.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:14:44 smithi083 ceph-mon[77242]: pgmap v8594: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:44.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:14:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:14:44.594Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:14:45.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:14:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:14:45.300Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:14:46.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:14:46 smithi153 ceph-mon[70044]: pgmap v8595: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:46.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:14:46 smithi083 ceph-mon[79930]: pgmap v8595: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:46.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:14:46 smithi083 ceph-mon[77242]: pgmap v8595: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:48.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:14:48 smithi153 ceph-mon[70044]: pgmap v8596: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:48.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:14:48 smithi083 ceph-mon[79930]: pgmap v8596: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:48.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:14:48 smithi083 ceph-mon[77242]: pgmap v8596: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:49.894 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:14:50.180 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:14:50.181 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 4m ago 4h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:14:50.181 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 2m ago 4h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:14:50.181 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 4m ago 4h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:14:50.182 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 2m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:14:50.182 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 2m ago 4h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:14:50.182 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 4m ago 4h 757M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:14:50.183 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 4m ago 4h 1091M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:14:50.183 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 2m ago 4h 470M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:14:50.183 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 4m ago 4h 625M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:14:50.183 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 4m ago 4h 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:14:50.183 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 2m ago 4h 25.8M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:14:50.184 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 4m ago 4h 767M 2492M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:14:50.184 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 4m ago 4h 758M 2492M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:14:50.184 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 4m ago 4h 593M 2492M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:14:50.184 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 4m ago 4h 739M 2492M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:14:50.185 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 2m ago 4h 653M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:14:50.185 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 2m ago 4h 812M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:14:50.185 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 2m ago 4h 725M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:14:50.185 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 2m ago 4h 857M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:14:50.186 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 2m ago 4h 181M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:14:50.538 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:14:50.538 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:14:50.538 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:14:50.539 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:14:50.539 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:14:50.539 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:14:50.539 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:14:50.540 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:14:50.540 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:14:50.540 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:14:50.540 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:14:50.541 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:14:50.541 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:14:50.541 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:14:50.541 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:14:50.541 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:14:50.542 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:14:50.542 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:14:50.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:14:50 smithi153 ceph-mon[70044]: pgmap v8597: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:50.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:14:50 smithi083 ceph-mon[79930]: pgmap v8597: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:50.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:14:50 smithi083 ceph-mon[77242]: pgmap v8597: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:51.507 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:14:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:14:51.213Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:14:51.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:14:51 smithi153 ceph-mon[70044]: from='client.44133 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:14:51.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:14:51 smithi153 ceph-mon[70044]: from='client.44139 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:14:51.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:14:51 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/631280412' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:14:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:14:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:14:51] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T10:14:51.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:14:51 smithi083 ceph-mon[77242]: from='client.44133 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:14:51.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:14:51 smithi083 ceph-mon[77242]: from='client.44139 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:14:51.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:14:51 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/631280412' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:14:51.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:14:51 smithi083 ceph-mon[79930]: from='client.44133 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:14:51.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:14:51 smithi083 ceph-mon[79930]: from='client.44139 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:14:51.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:14:51 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/631280412' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:14:52.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:14:52 smithi153 ceph-mon[70044]: pgmap v8598: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:14:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:14:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:14:52.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:14:52 smithi083 ceph-mon[79930]: pgmap v8598: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:52.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:14:52 smithi083 ceph-mon[77242]: pgmap v8598: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:14:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:14:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:14:54.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:14:54 smithi153 ceph-mon[70044]: pgmap v8599: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:54.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:14:54 smithi083 ceph-mon[79930]: pgmap v8599: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:54.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:14:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:14:54.593Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:14:54.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:14:54 smithi083 ceph-mon[77242]: pgmap v8599: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:55.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:14:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:14:55.300Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:14:56.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:14:56 smithi153 ceph-mon[70044]: pgmap v8600: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:56.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:14:56 smithi083 ceph-mon[79930]: pgmap v8600: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:56.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:14:56 smithi083 ceph-mon[77242]: pgmap v8600: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:58.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:14:58 smithi153 ceph-mon[70044]: pgmap v8601: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:58.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:14:58 smithi083 ceph-mon[77242]: pgmap v8601: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:14:58.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:14:58 smithi083 ceph-mon[79930]: pgmap v8601: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:00.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:15:00 smithi153 ceph-mon[70044]: pgmap v8602: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:00.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:15:00 smithi083 ceph-mon[79930]: pgmap v8602: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:00.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:15:00 smithi083 ceph-mon[77242]: pgmap v8602: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:01.506 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:15:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:15:01.215Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:15:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:15:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:15:01] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T10:15:02.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:15:02 smithi153 ceph-mon[70044]: pgmap v8603: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:15:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:15:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:15:02.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:15:02 smithi083 ceph-mon[79930]: pgmap v8603: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:02.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:15:02 smithi083 ceph-mon[77242]: pgmap v8603: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:15:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:15:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:15:04.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:15:04 smithi153 ceph-mon[70044]: pgmap v8604: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:04.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:15:04 smithi083 ceph-mon[79930]: pgmap v8604: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:04.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:15:04 smithi083 ceph-mon[77242]: pgmap v8604: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:04.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:15:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:15:04.595Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:15:05.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:15:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:15:05.301Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:15:06.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:15:06 smithi153 ceph-mon[70044]: pgmap v8605: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:06.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:15:06 smithi083 ceph-mon[79930]: pgmap v8605: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:06.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:15:06 smithi083 ceph-mon[77242]: pgmap v8605: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:08.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:15:08 smithi153 ceph-mon[70044]: pgmap v8606: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:08.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:15:08 smithi083 ceph-mon[79930]: pgmap v8606: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:08.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:15:08 smithi083 ceph-mon[77242]: pgmap v8606: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:10.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:15:10 smithi083 ceph-mon[79930]: pgmap v8607: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:10.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:15:10 smithi083 ceph-mon[77242]: pgmap v8607: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:11.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:15:10 smithi153 ceph-mon[70044]: pgmap v8607: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:11.507 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:15:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:15:11.216Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:15:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:15:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:15:11] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T10:15:12.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:15:12 smithi153 ceph-mon[70044]: pgmap v8608: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:15:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:15:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:15:12.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:15:12 smithi083 ceph-mon[79930]: pgmap v8608: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:12.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:15:12 smithi083 ceph-mon[77242]: pgmap v8608: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:15:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:15:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:15:14.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:15:14 smithi083 ceph-mon[79930]: pgmap v8609: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:14.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:15:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:15:14.596Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:15:14.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:15:14 smithi083 ceph-mon[77242]: pgmap v8609: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:15.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:15:14 smithi153 ceph-mon[70044]: pgmap v8609: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:15.572 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:15:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:15:15.301Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:15:15.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:15:15 smithi083 ceph-mon[77242]: pgmap v8610: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:15.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:15:15 smithi083 ceph-mon[79930]: pgmap v8610: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:16.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:15:15 smithi153 ceph-mon[70044]: pgmap v8610: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:18.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:15:17 smithi083 ceph-mon[79930]: pgmap v8611: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:18.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:15:17 smithi083 ceph-mon[77242]: pgmap v8611: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:18.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:15:17 smithi153 ceph-mon[70044]: pgmap v8611: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:20.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:15:20 smithi083 ceph-mon[79930]: pgmap v8612: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:20.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:15:20 smithi083 ceph-mon[77242]: pgmap v8612: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:20.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:15:20 smithi153 ceph-mon[70044]: pgmap v8612: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:20.866 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:15:21.154 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:15:21.155 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 4m ago 4h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:15:21.155 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 2m ago 4h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:15:21.155 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 4m ago 4h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:15:21.155 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 2m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:15:21.156 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 2m ago 4h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:15:21.156 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 4m ago 4h 757M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:15:21.156 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 4m ago 4h 1091M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:15:21.156 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 2m ago 4h 470M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:15:21.157 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 4m ago 4h 625M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:15:21.157 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 4m ago 4h 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:15:21.157 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 2m ago 4h 25.8M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:15:21.157 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 4m ago 4h 767M 2492M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:15:21.157 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 4m ago 4h 758M 2492M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:15:21.158 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 4m ago 4h 593M 2492M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:15:21.158 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 4m ago 4h 739M 2492M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:15:21.158 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 2m ago 4h 653M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:15:21.158 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 2m ago 4h 812M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:15:21.159 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 2m ago 4h 725M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:15:21.159 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 2m ago 4h 857M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:15:21.159 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 2m ago 4h 181M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:15:21.506 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:15:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:15:21.217Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:15:21.512 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:15:21.512 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:15:21.513 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:15:21.513 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:15:21.513 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:15:21.513 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:15:21.513 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:15:21.514 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:15:21.514 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:15:21.514 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:15:21.514 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:15:21.515 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:15:21.515 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:15:21.515 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:15:21.515 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:15:21.515 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:15:21.516 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:15:21.516 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:15:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:15:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:15:21] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T10:15:22.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:15:22 smithi083 ceph-mon[77242]: from='client.44151 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:15:22.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:15:22 smithi083 ceph-mon[77242]: pgmap v8613: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:22.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:15:22 smithi083 ceph-mon[77242]: from='client.44157 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:15:22.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:15:22 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/776764296' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:15:22.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:15:22 smithi083 ceph-mon[79930]: from='client.44151 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:15:22.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:15:22 smithi083 ceph-mon[79930]: pgmap v8613: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:22.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:15:22 smithi083 ceph-mon[79930]: from='client.44157 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:15:22.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:15:22 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/776764296' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:15:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:15:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:15:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:15:22.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:15:22 smithi153 ceph-mon[70044]: from='client.44151 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:15:22.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:15:22 smithi153 ceph-mon[70044]: pgmap v8613: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:22.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:15:22 smithi153 ceph-mon[70044]: from='client.44157 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:15:22.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:15:22 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/776764296' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:15:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:15:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:15:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:15:24.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:15:24 smithi083 ceph-mon[79930]: pgmap v8614: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:24.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:15:24 smithi083 ceph-mon[77242]: pgmap v8614: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:15:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:15:24.596Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:15:24.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:15:24 smithi153 ceph-mon[70044]: pgmap v8614: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:25.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:15:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:15:25.301Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:15:26.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:15:26 smithi083 ceph-mon[79930]: pgmap v8615: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:26.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:15:26 smithi083 ceph-mon[77242]: pgmap v8615: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:26.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:15:26 smithi153 ceph-mon[70044]: pgmap v8615: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:28.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:15:28 smithi153 ceph-mon[70044]: pgmap v8616: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:28.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:15:28 smithi083 ceph-mon[79930]: pgmap v8616: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:28.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:15:28 smithi083 ceph-mon[77242]: pgmap v8616: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:30.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:15:30 smithi153 ceph-mon[70044]: pgmap v8617: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:30.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:15:30 smithi083 ceph-mon[79930]: pgmap v8617: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:30.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:15:30 smithi083 ceph-mon[77242]: pgmap v8617: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:31.506 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:15:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:15:31.216Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:15:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:15:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:15:31] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T10:15:32.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:15:32 smithi153 ceph-mon[70044]: pgmap v8618: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:15:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:15:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:15:32.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:15:32 smithi083 ceph-mon[77242]: pgmap v8618: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:32.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:15:32 smithi083 ceph-mon[79930]: pgmap v8618: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:15:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:15:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:15:34.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:15:34 smithi153 ceph-mon[70044]: pgmap v8619: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:34.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:15:34 smithi083 ceph-mon[79930]: pgmap v8619: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:34.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:15:34 smithi083 ceph-mon[77242]: pgmap v8619: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:34.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:15:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:15:34.597Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:15:35.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:15:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:15:35.302Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:15:36.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:15:36 smithi153 ceph-mon[70044]: pgmap v8620: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:36.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:15:36 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:15:36.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:15:36 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:15:36.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:15:36 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:15:36.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:15:36 smithi083 ceph-mon[77242]: pgmap v8620: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:36.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:15:36 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:15:36.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:15:36 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:15:36.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:15:36 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:15:36.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:15:36 smithi083 ceph-mon[79930]: pgmap v8620: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:36.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:15:36 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:15:36.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:15:36 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:15:36.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:15:36 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:15:37.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:15:37 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:15:37.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:15:37 smithi083 ceph-mon[79930]: pgmap v8621: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:37.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:15:37 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:15:37.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:15:37 smithi083 ceph-mon[77242]: pgmap v8621: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:38.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:15:37 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:15:38.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:15:37 smithi153 ceph-mon[70044]: pgmap v8621: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:40.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:15:40 smithi083 ceph-mon[79930]: pgmap v8622: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:40.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:15:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:15:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:15:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:15:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:15:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:15:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:15:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:15:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:15:40 smithi083 ceph-mon[77242]: pgmap v8622: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:15:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:15:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:15:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:15:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:15:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:15:40.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:15:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:15:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:15:40 smithi153 ceph-mon[70044]: pgmap v8622: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:15:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:15:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:15:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:15:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:15:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:15:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:15:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:15:41.507 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:15:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:15:41.217Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:15:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:15:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:15:41] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T10:15:42.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:15:42 smithi083 ceph-mon[79930]: pgmap v8623: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:42.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:15:42 smithi083 ceph-mon[77242]: pgmap v8623: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:15:42 smithi153 ceph-mon[70044]: pgmap v8623: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:15:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:15:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:15:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:15:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:15:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:15:44.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:15:44 smithi153 ceph-mon[70044]: pgmap v8624: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:44.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:15:44 smithi083 ceph-mon[79930]: pgmap v8624: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:44.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:15:44 smithi083 ceph-mon[77242]: pgmap v8624: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:44.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:15:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:15:44.598Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:15:45.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:15:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:15:45.303Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:15:46.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:15:46 smithi153 ceph-mon[70044]: pgmap v8625: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:46.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:15:46 smithi083 ceph-mon[79930]: pgmap v8625: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:46.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:15:46 smithi083 ceph-mon[77242]: pgmap v8625: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:48.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:15:48 smithi153 ceph-mon[70044]: pgmap v8626: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:48.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:15:48 smithi083 ceph-mon[79930]: pgmap v8626: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:48.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:15:48 smithi083 ceph-mon[77242]: pgmap v8626: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:50.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:15:50 smithi153 ceph-mon[70044]: pgmap v8627: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:50.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:15:50 smithi083 ceph-mon[77242]: pgmap v8627: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:50.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:15:50 smithi083 ceph-mon[79930]: pgmap v8627: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:51.506 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:15:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:15:51.218Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:15:51.837 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:15:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:15:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:15:51] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T10:15:52.128 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:15:52.129 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 5m ago 4h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:15:52.129 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 3m ago 4h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:15:52.129 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 5m ago 4h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:15:52.129 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 3m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:15:52.130 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 3m ago 4h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:15:52.130 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 5m ago 4h 757M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:15:52.130 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 5m ago 4h 1091M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:15:52.130 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 3m ago 4h 470M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:15:52.131 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 5m ago 4h 625M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:15:52.131 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 5m ago 4h 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:15:52.131 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 3m ago 4h 25.8M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:15:52.131 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 5m ago 4h 767M 2492M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:15:52.131 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 5m ago 4h 758M 2492M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:15:52.132 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 5m ago 4h 593M 2492M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:15:52.132 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 5m ago 4h 739M 2492M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:15:52.132 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 3m ago 4h 653M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:15:52.132 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 3m ago 4h 812M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:15:52.132 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 3m ago 4h 725M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:15:52.133 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 3m ago 4h 857M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:15:52.133 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 3m ago 4h 181M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:15:52.483 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:15:52.483 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:15:52.484 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:15:52.484 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:15:52.484 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:15:52.485 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:15:52.485 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:15:52.485 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:15:52.485 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:15:52.486 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:15:52.486 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:15:52.486 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:15:52.486 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:15:52.486 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:15:52.487 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:15:52.487 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:15:52.487 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:15:52.487 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:15:52.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:15:52 smithi153 ceph-mon[70044]: pgmap v8628: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:15:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:15:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:15:52.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:15:52 smithi083 ceph-mon[79930]: pgmap v8628: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:52.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:15:52 smithi083 ceph-mon[77242]: pgmap v8628: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:15:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:15:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:15:53.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:15:53 smithi153 ceph-mon[70044]: from='client.54100 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:15:53.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:15:53 smithi153 ceph-mon[70044]: from='client.54106 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:15:53.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:15:53 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/301555929' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:15:53.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:15:53 smithi083 ceph-mon[77242]: from='client.54100 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:15:53.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:15:53 smithi083 ceph-mon[77242]: from='client.54106 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:15:53.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:15:53 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/301555929' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:15:53.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:15:53 smithi083 ceph-mon[79930]: from='client.54100 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:15:53.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:15:53 smithi083 ceph-mon[79930]: from='client.54106 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:15:53.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:15:53 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/301555929' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:15:54.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:15:54 smithi153 ceph-mon[70044]: pgmap v8629: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:54.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:15:54 smithi083 ceph-mon[77242]: pgmap v8629: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:54.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:15:54 smithi083 ceph-mon[79930]: pgmap v8629: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:54.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:15:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:15:54.598Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:15:55.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:15:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:15:55.304Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:15:56.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:15:56 smithi153 ceph-mon[70044]: pgmap v8630: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:56.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:15:56 smithi083 ceph-mon[79930]: pgmap v8630: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:56.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:15:56 smithi083 ceph-mon[77242]: pgmap v8630: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:58.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:15:58 smithi153 ceph-mon[70044]: pgmap v8631: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:58.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:15:58 smithi083 ceph-mon[79930]: pgmap v8631: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:15:58.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:15:58 smithi083 ceph-mon[77242]: pgmap v8631: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:00.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:16:00 smithi153 ceph-mon[70044]: pgmap v8632: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:00.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:16:00 smithi083 ceph-mon[79930]: pgmap v8632: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:00.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:16:00 smithi083 ceph-mon[77242]: pgmap v8632: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:01.507 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:16:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:16:01.219Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:16:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:16:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:16:01] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T10:16:02.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:16:02 smithi153 ceph-mon[70044]: pgmap v8633: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:16:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:16:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:16:02.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:16:02 smithi083 ceph-mon[79930]: pgmap v8633: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:02.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:16:02 smithi083 ceph-mon[77242]: pgmap v8633: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:16:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:16:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:16:04.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:16:04 smithi153 ceph-mon[70044]: pgmap v8634: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:04.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:16:04 smithi083 ceph-mon[79930]: pgmap v8634: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:04.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:16:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:16:04.599Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:16:04.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:16:04 smithi083 ceph-mon[77242]: pgmap v8634: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:05.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:16:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:16:05.305Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:16:06.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:16:06 smithi153 ceph-mon[70044]: pgmap v8635: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:06.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:16:06 smithi083 ceph-mon[79930]: pgmap v8635: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:06.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:16:06 smithi083 ceph-mon[77242]: pgmap v8635: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:08.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:16:08 smithi153 ceph-mon[70044]: pgmap v8636: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:08.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:16:08 smithi083 ceph-mon[79930]: pgmap v8636: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:08.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:16:08 smithi083 ceph-mon[77242]: pgmap v8636: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:10.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:16:10 smithi153 ceph-mon[70044]: pgmap v8637: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:10.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:16:10 smithi083 ceph-mon[79930]: pgmap v8637: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:10.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:16:10 smithi083 ceph-mon[77242]: pgmap v8637: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:11.506 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:16:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:16:11.220Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:16:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:16:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:16:11] "GET /metrics HTTP/1.1" 200 225229 "" "Prometheus/2.33.4" 2022-09-29T10:16:12.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:16:12 smithi153 ceph-mon[70044]: pgmap v8638: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:16:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:16:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:16:12.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:16:12 smithi083 ceph-mon[77242]: pgmap v8638: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:12.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:16:12 smithi083 ceph-mon[79930]: pgmap v8638: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:16:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:16:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:16:14.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:16:14 smithi153 ceph-mon[70044]: pgmap v8639: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:14.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:16:14 smithi083 ceph-mon[79930]: pgmap v8639: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:14.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:16:14 smithi083 ceph-mon[77242]: pgmap v8639: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:14.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:16:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:16:14.599Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:16:15.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:16:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:16:15.305Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:16:16.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:16:16 smithi153 ceph-mon[70044]: pgmap v8640: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:16.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:16:16 smithi083 ceph-mon[77242]: pgmap v8640: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:16.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:16:16 smithi083 ceph-mon[79930]: pgmap v8640: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:18.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:16:18 smithi153 ceph-mon[70044]: pgmap v8641: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:18.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:16:18 smithi083 ceph-mon[79930]: pgmap v8641: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:18.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:16:18 smithi083 ceph-mon[77242]: pgmap v8641: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:20.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:16:20 smithi153 ceph-mon[70044]: pgmap v8642: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:20.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:16:20 smithi083 ceph-mon[79930]: pgmap v8642: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:20.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:16:20 smithi083 ceph-mon[77242]: pgmap v8642: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:21.506 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:16:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:16:21.220Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:16:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:16:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:16:21] "GET /metrics HTTP/1.1" 200 225229 "" "Prometheus/2.33.4" 2022-09-29T10:16:22.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:16:22 smithi153 ceph-mon[70044]: pgmap v8643: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:16:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:16:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:16:22.808 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:16:22.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:16:22 smithi083 ceph-mon[77242]: pgmap v8643: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:22.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:16:22 smithi083 ceph-mon[79930]: pgmap v8643: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:23.095 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:16:23.095 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 6m ago 4h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:16:23.096 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 3m ago 4h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:16:23.096 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 6m ago 4h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:16:23.096 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 3m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:16:23.096 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 3m ago 4h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:16:23.097 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 6m ago 4h 757M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:16:23.097 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 6m ago 4h 1091M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:16:23.097 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 3m ago 4h 470M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:16:23.097 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 6m ago 4h 625M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:16:23.097 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 6m ago 4h 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:16:23.098 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 3m ago 4h 25.8M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:16:23.098 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 6m ago 4h 767M 2492M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:16:23.098 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 6m ago 4h 758M 2492M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:16:23.098 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 6m ago 4h 593M 2492M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:16:23.099 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 6m ago 4h 739M 2492M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:16:23.099 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 3m ago 4h 653M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:16:23.099 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 3m ago 4h 812M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:16:23.099 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 3m ago 4h 725M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:16:23.100 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 3m ago 4h 857M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:16:23.100 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 3m ago 4h 181M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:16:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:16:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:16:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:16:23.457 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:16:23.457 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:16:23.457 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:16:23.457 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:16:23.458 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:16:23.458 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:16:23.458 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:16:23.458 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:16:23.459 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:16:23.459 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:16:23.459 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:16:23.459 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:16:23.460 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:16:23.460 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:16:23.460 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:16:23.460 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:16:23.461 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:16:23.461 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:16:23.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:16:23 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3545000045' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:16:23.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:16:23 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3545000045' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:16:23.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:16:23 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3545000045' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:16:24.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:16:24 smithi153 ceph-mon[70044]: from='client.54118 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:16:24.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:16:24 smithi153 ceph-mon[70044]: pgmap v8644: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:24.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:16:24 smithi153 ceph-mon[70044]: from='client.54124 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:16:24.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:16:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:16:24.600Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:16:24.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:16:24 smithi083 ceph-mon[77242]: from='client.54118 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:16:24.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:16:24 smithi083 ceph-mon[77242]: pgmap v8644: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:24.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:16:24 smithi083 ceph-mon[77242]: from='client.54124 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:16:24.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:16:24 smithi083 ceph-mon[79930]: from='client.54118 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:16:24.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:16:24 smithi083 ceph-mon[79930]: pgmap v8644: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:24.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:16:24 smithi083 ceph-mon[79930]: from='client.54124 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:16:25.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:16:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:16:25.306Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:16:26.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:16:26 smithi153 ceph-mon[70044]: pgmap v8645: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:26.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:16:26 smithi083 ceph-mon[79930]: pgmap v8645: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:26.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:16:26 smithi083 ceph-mon[77242]: pgmap v8645: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:28.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:16:28 smithi153 ceph-mon[70044]: pgmap v8646: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:28.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:16:28 smithi083 ceph-mon[79930]: pgmap v8646: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:28.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:16:28 smithi083 ceph-mon[77242]: pgmap v8646: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:30.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:16:30 smithi153 ceph-mon[70044]: pgmap v8647: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:30.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:16:30 smithi083 ceph-mon[79930]: pgmap v8647: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:30.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:16:30 smithi083 ceph-mon[77242]: pgmap v8647: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:31.506 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:16:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:16:31.221Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:16:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:16:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:16:31] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T10:16:32.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:16:32 smithi153 ceph-mon[70044]: pgmap v8648: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:16:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:16:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:16:32.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:16:32 smithi083 ceph-mon[79930]: pgmap v8648: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:32.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:16:32 smithi083 ceph-mon[77242]: pgmap v8648: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:16:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:16:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:16:34.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:16:34 smithi153 ceph-mon[70044]: pgmap v8649: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:34.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:16:34 smithi083 ceph-mon[79930]: pgmap v8649: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:34.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:16:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:16:34.601Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:16:34.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:16:34 smithi083 ceph-mon[77242]: pgmap v8649: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:35.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:16:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:16:35.307Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:16:36.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:16:36 smithi153 ceph-mon[70044]: pgmap v8650: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:36.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:16:36 smithi083 ceph-mon[79930]: pgmap v8650: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:36.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:16:36 smithi083 ceph-mon[77242]: pgmap v8650: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:37.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:16:37 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:16:37.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:16:37 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:16:37.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:16:37 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:16:37.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:16:37 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:16:37.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:16:37 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:16:37.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:16:37 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:16:37.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:16:37 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:16:37.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:16:37 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:16:37.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:16:37 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:16:37.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:16:37 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:16:37.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:16:37 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:16:37.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:16:37 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:16:38.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:16:38 smithi153 ceph-mon[70044]: pgmap v8651: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:38.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:16:38 smithi083 ceph-mon[79930]: pgmap v8651: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:38.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:16:38 smithi083 ceph-mon[77242]: pgmap v8651: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:40.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:16:40 smithi083 ceph-mon[79930]: pgmap v8652: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:40.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:16:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:16:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:16:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:16:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:16:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:16:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:16:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:16:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:16:40 smithi083 ceph-mon[77242]: pgmap v8652: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:16:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:16:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:16:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:16:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:16:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:16:40.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:16:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:16:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:16:40 smithi153 ceph-mon[70044]: pgmap v8652: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:16:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:16:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:16:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:16:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:16:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:16:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:16:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:16:41.507 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:16:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:16:41.222Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:16:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:16:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:16:41] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T10:16:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:16:42 smithi153 ceph-mon[70044]: pgmap v8653: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:16:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:16:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:16:42.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:16:42 smithi083 ceph-mon[79930]: pgmap v8653: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:42.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:16:42 smithi083 ceph-mon[77242]: pgmap v8653: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:16:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:16:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:16:44.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:16:44 smithi083 ceph-mon[79930]: pgmap v8654: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:44.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:16:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:16:44.602Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:16:44.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:16:44 smithi083 ceph-mon[77242]: pgmap v8654: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:45.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:16:44 smithi153 ceph-mon[70044]: pgmap v8654: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:45.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:16:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:16:45.307Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:16:46.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:16:46 smithi083 ceph-mon[79930]: pgmap v8655: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:46.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:16:46 smithi083 ceph-mon[77242]: pgmap v8655: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:47.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:16:46 smithi153 ceph-mon[70044]: pgmap v8655: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:48.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:16:48 smithi083 ceph-mon[79930]: pgmap v8656: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:48.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:16:48 smithi083 ceph-mon[77242]: pgmap v8656: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:49.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:16:48 smithi153 ceph-mon[70044]: pgmap v8656: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:50.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:16:50 smithi083 ceph-mon[77242]: pgmap v8657: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:50.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:16:50 smithi083 ceph-mon[79930]: pgmap v8657: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:51.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:16:50 smithi153 ceph-mon[70044]: pgmap v8657: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:51.507 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:16:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:16:51.222Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:16:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:16:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:16:51] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T10:16:52.578 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:16:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:16:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:16:52.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:16:52 smithi083 ceph-mon[77242]: pgmap v8658: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:52.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:16:52 smithi083 ceph-mon[79930]: pgmap v8658: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:53.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:16:52 smithi153 ceph-mon[70044]: pgmap v8658: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:16:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:16:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:16:53.785 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:16:54.074 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:16:54.075 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 6m ago 4h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:16:54.075 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 4m ago 4h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:16:54.075 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 6m ago 4h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:16:54.075 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 4m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:16:54.076 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 4m ago 4h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:16:54.076 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 6m ago 4h 757M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:16:54.076 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 6m ago 4h 1091M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:16:54.076 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 4m ago 4h 470M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:16:54.076 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 6m ago 4h 625M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:16:54.077 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 6m ago 4h 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:16:54.077 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 4m ago 4h 25.8M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:16:54.077 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 6m ago 4h 767M 2492M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:16:54.077 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 6m ago 4h 758M 2492M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:16:54.078 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 6m ago 4h 593M 2492M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:16:54.078 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 6m ago 4h 739M 2492M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:16:54.078 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 4m ago 4h 653M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:16:54.078 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 4m ago 4h 812M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:16:54.079 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 4m ago 4h 725M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:16:54.079 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 4m ago 4h 857M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:16:54.079 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 4m ago 4h 181M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:16:54.432 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:16:54.432 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:16:54.433 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:16:54.433 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:16:54.433 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:16:54.433 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:16:54.434 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:16:54.434 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:16:54.434 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:16:54.434 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:16:54.434 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:16:54.435 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:16:54.435 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:16:54.435 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:16:54.435 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:16:54.436 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:16:54.436 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:16:54.436 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:16:54.582 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:16:54 smithi153 ceph-mon[70044]: pgmap v8659: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:54.582 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:16:54 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/823208913' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:16:54.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:16:54 smithi083 ceph-mon[79930]: pgmap v8659: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:54.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:16:54 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/823208913' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:16:54.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:16:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:16:54.602Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:16:54.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:16:54 smithi083 ceph-mon[77242]: pgmap v8659: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:54.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:16:54 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/823208913' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:16:55.585 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:16:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:16:55.307Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:16:55.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:16:55 smithi083 ceph-mon[77242]: from='client.54136 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:16:55.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:16:55 smithi083 ceph-mon[77242]: from='client.44211 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:16:55.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:16:55 smithi083 ceph-mon[77242]: pgmap v8660: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:55.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:16:55 smithi083 ceph-mon[79930]: from='client.54136 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:16:55.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:16:55 smithi083 ceph-mon[79930]: from='client.44211 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:16:55.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:16:55 smithi083 ceph-mon[79930]: pgmap v8660: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:56.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:16:55 smithi153 ceph-mon[70044]: from='client.54136 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:16:56.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:16:55 smithi153 ceph-mon[70044]: from='client.44211 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:16:56.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:16:55 smithi153 ceph-mon[70044]: pgmap v8660: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:58.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:16:57 smithi153 ceph-mon[70044]: pgmap v8661: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:58.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:16:57 smithi083 ceph-mon[77242]: pgmap v8661: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:16:58.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:16:57 smithi083 ceph-mon[79930]: pgmap v8661: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:00.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:17:00 smithi153 ceph-mon[70044]: pgmap v8662: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:00.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:17:00 smithi083 ceph-mon[77242]: pgmap v8662: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:00.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:17:00 smithi083 ceph-mon[79930]: pgmap v8662: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:01.506 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:17:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:17:01.222Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:17:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:17:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:17:01] "GET /metrics HTTP/1.1" 200 225223 "" "Prometheus/2.33.4" 2022-09-29T10:17:02.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:17:02 smithi153 ceph-mon[70044]: pgmap v8663: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:17:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:17:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:17:02.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:17:02 smithi083 ceph-mon[79930]: pgmap v8663: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:02.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:17:02 smithi083 ceph-mon[77242]: pgmap v8663: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:17:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:17:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:17:04.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:17:04 smithi153 ceph-mon[70044]: pgmap v8664: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:04.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:17:04 smithi083 ceph-mon[79930]: pgmap v8664: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:04.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:17:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:17:04.603Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:17:04.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:17:04 smithi083 ceph-mon[77242]: pgmap v8664: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:05.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:17:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:17:05.308Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:17:06.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:17:06 smithi153 ceph-mon[70044]: pgmap v8665: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:06.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:17:06 smithi083 ceph-mon[79930]: pgmap v8665: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:06.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:17:06 smithi083 ceph-mon[77242]: pgmap v8665: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:08.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:17:08 smithi153 ceph-mon[70044]: pgmap v8666: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:08.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:17:08 smithi083 ceph-mon[79930]: pgmap v8666: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:08.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:17:08 smithi083 ceph-mon[77242]: pgmap v8666: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:10.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:17:10 smithi153 ceph-mon[70044]: pgmap v8667: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:10.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:17:10 smithi083 ceph-mon[79930]: pgmap v8667: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:10.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:17:10 smithi083 ceph-mon[77242]: pgmap v8667: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:11.506 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:17:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:17:11.222Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:17:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:17:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:17:11] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T10:17:12.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:17:12 smithi153 ceph-mon[70044]: pgmap v8668: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:17:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:17:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:17:12.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:17:12 smithi083 ceph-mon[77242]: pgmap v8668: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:12.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:17:12 smithi083 ceph-mon[79930]: pgmap v8668: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:17:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:17:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:17:14.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:17:14 smithi153 ceph-mon[70044]: pgmap v8669: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:14.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:17:14 smithi083 ceph-mon[79930]: pgmap v8669: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:14.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:17:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:17:14.603Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:17:14.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:17:14 smithi083 ceph-mon[77242]: pgmap v8669: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:15.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:17:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:17:15.308Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:17:16.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:17:16 smithi153 ceph-mon[70044]: pgmap v8670: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:16.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:17:16 smithi083 ceph-mon[77242]: pgmap v8670: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:16.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:17:16 smithi083 ceph-mon[79930]: pgmap v8670: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:18.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:17:18 smithi153 ceph-mon[70044]: pgmap v8671: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:18.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:17:18 smithi083 ceph-mon[79930]: pgmap v8671: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:18.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:17:18 smithi083 ceph-mon[77242]: pgmap v8671: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:20.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:17:20 smithi153 ceph-mon[70044]: pgmap v8672: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:20.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:17:20 smithi083 ceph-mon[79930]: pgmap v8672: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:20.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:17:20 smithi083 ceph-mon[77242]: pgmap v8672: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:21.507 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:17:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:17:21.223Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:17:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:17:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:17:21] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T10:17:22.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:17:22 smithi153 ceph-mon[70044]: pgmap v8673: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:17:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:17:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:17:22.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:17:22 smithi083 ceph-mon[79930]: pgmap v8673: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:22.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:17:22 smithi083 ceph-mon[77242]: pgmap v8673: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:17:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:17:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:17:24.761 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:17:24.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:17:24 smithi153 ceph-mon[70044]: pgmap v8674: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:24.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:17:24 smithi083 ceph-mon[77242]: pgmap v8674: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:24.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:17:24 smithi083 ceph-mon[79930]: pgmap v8674: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:24.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:17:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:17:24.605Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:17:25.048 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:17:25.049 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 7m ago 4h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:17:25.049 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 4m ago 4h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:17:25.049 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 7m ago 4h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:17:25.049 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 4m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:17:25.050 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 4m ago 4h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:17:25.050 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 7m ago 4h 757M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:17:25.050 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 7m ago 4h 1091M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:17:25.050 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 4m ago 4h 470M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:17:25.050 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 7m ago 4h 625M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:17:25.051 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 7m ago 4h 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:17:25.051 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 4m ago 4h 25.8M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:17:25.051 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 7m ago 4h 767M 2492M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:17:25.051 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 7m ago 4h 758M 2492M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:17:25.052 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 7m ago 4h 593M 2492M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:17:25.052 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 7m ago 4h 739M 2492M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:17:25.052 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 4m ago 4h 653M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:17:25.052 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 4m ago 4h 812M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:17:25.052 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 4m ago 4h 725M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:17:25.053 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 4m ago 4h 857M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:17:25.053 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 4m ago 4h 181M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:17:25.405 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:17:25.406 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:17:25.406 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:17:25.406 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:17:25.407 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:17:25.407 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:17:25.407 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:17:25.407 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:17:25.408 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:17:25.408 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:17:25.408 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:17:25.408 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:17:25.409 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:17:25.409 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:17:25.409 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:17:25.409 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:17:25.409 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:17:25.410 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:17:25.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:17:25 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4001977076' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:17:25.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:17:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:17:25.308Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:17:25.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:17:25 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4001977076' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:17:25.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:17:25 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4001977076' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:17:26.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:17:26 smithi153 ceph-mon[70044]: from='client.54154 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:17:26.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:17:26 smithi153 ceph-mon[70044]: pgmap v8675: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:26.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:17:26 smithi153 ceph-mon[70044]: from='client.44229 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:17:26.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:17:26 smithi083 ceph-mon[77242]: from='client.54154 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:17:26.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:17:26 smithi083 ceph-mon[77242]: pgmap v8675: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:26.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:17:26 smithi083 ceph-mon[77242]: from='client.44229 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:17:26.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:17:26 smithi083 ceph-mon[79930]: from='client.54154 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:17:26.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:17:26 smithi083 ceph-mon[79930]: pgmap v8675: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:26.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:17:26 smithi083 ceph-mon[79930]: from='client.44229 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:17:28.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:17:28 smithi153 ceph-mon[70044]: pgmap v8676: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:28.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:17:28 smithi083 ceph-mon[79930]: pgmap v8676: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:28.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:17:28 smithi083 ceph-mon[77242]: pgmap v8676: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:30.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:17:30 smithi153 ceph-mon[70044]: pgmap v8677: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:30.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:17:30 smithi083 ceph-mon[79930]: pgmap v8677: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:30.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:17:30 smithi083 ceph-mon[77242]: pgmap v8677: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:31.506 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:17:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:17:31.224Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:17:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:17:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:17:31] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T10:17:32.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:17:32 smithi153 ceph-mon[70044]: pgmap v8678: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:17:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:17:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:17:32.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:17:32 smithi083 ceph-mon[77242]: pgmap v8678: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:32.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:17:32 smithi083 ceph-mon[79930]: pgmap v8678: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:17:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:17:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:17:34.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:17:34 smithi153 ceph-mon[70044]: pgmap v8679: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:34.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:17:34 smithi083 ceph-mon[79930]: pgmap v8679: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:34.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:17:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:17:34.604Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:17:34.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:17:34 smithi083 ceph-mon[77242]: pgmap v8679: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:35.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:17:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:17:35.309Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:17:36.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:17:36 smithi153 ceph-mon[70044]: pgmap v8680: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:36.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:17:36 smithi083 ceph-mon[77242]: pgmap v8680: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:36.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:17:36 smithi083 ceph-mon[79930]: pgmap v8680: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:37.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:17:37 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:17:37.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:17:37 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:17:37.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:17:37 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:17:37.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:17:37 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:17:37.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:17:37 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:17:37.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:17:37 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:17:37.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:17:37 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:17:37.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:17:37 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:17:37.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:17:37 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:17:38.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:17:38 smithi153 ceph-mon[70044]: pgmap v8681: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:38.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:17:38 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:17:38.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:17:38 smithi083 ceph-mon[79930]: pgmap v8681: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:38.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:17:38 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:17:38.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:17:38 smithi083 ceph-mon[77242]: pgmap v8681: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:38.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:17:38 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:17:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:17:40 smithi153 ceph-mon[70044]: pgmap v8682: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:17:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:17:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:17:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:17:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:17:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:17:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:17:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:17:40.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:17:40 smithi083 ceph-mon[79930]: pgmap v8682: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:40.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:17:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:17:40.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:17:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:17:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:17:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:17:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:17:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:17:40.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:17:40 smithi083 ceph-mon[77242]: pgmap v8682: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:17:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:17:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:17:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:17:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:17:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:17:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:17:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:17:41.506 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:17:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:17:41.224Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:17:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:17:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:17:41] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T10:17:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:17:42 smithi153 ceph-mon[70044]: pgmap v8683: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:17:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:17:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:17:42.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:17:42 smithi083 ceph-mon[77242]: pgmap v8683: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:42.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:17:42 smithi083 ceph-mon[79930]: pgmap v8683: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:17:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:17:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:17:44.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:17:44 smithi153 ceph-mon[70044]: pgmap v8684: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:44.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:17:44 smithi083 ceph-mon[79930]: pgmap v8684: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:44.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:17:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:17:44.606Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:17:44.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:17:44 smithi083 ceph-mon[77242]: pgmap v8684: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:45.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:17:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:17:45.309Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:17:46.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:17:46 smithi083 ceph-mon[79930]: pgmap v8685: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:46.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:17:46 smithi083 ceph-mon[77242]: pgmap v8685: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:47.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:17:46 smithi153 ceph-mon[70044]: pgmap v8685: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:48.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:17:48 smithi083 ceph-mon[77242]: pgmap v8686: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:48.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:17:48 smithi083 ceph-mon[79930]: pgmap v8686: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:49.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:17:48 smithi153 ceph-mon[70044]: pgmap v8686: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:50.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:17:50 smithi083 ceph-mon[79930]: pgmap v8687: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:50.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:17:50 smithi083 ceph-mon[77242]: pgmap v8687: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:51.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:17:50 smithi153 ceph-mon[70044]: pgmap v8687: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:51.506 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:17:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:17:51.225Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:17:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:17:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:17:51] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T10:17:52.569 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:17:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:17:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:17:52.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:17:52 smithi083 ceph-mon[77242]: pgmap v8688: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:52.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:17:52 smithi083 ceph-mon[79930]: pgmap v8688: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:53.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:17:52 smithi153 ceph-mon[70044]: pgmap v8688: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:17:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:17:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:17:54.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:17:54 smithi083 ceph-mon[79930]: pgmap v8689: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:54.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:17:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:17:54.606Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:17:54.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:17:54 smithi083 ceph-mon[77242]: pgmap v8689: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:55.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:17:54 smithi153 ceph-mon[70044]: pgmap v8689: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:55.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:17:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:17:55.310Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:17:55.733 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:17:56.021 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:17:56.022 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 7m ago 4h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:17:56.022 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 5m ago 4h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:17:56.022 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 7m ago 4h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:17:56.022 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 5m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:17:56.023 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 5m ago 4h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:17:56.023 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 7m ago 4h 757M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:17:56.023 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 7m ago 4h 1091M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:17:56.023 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 5m ago 4h 470M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:17:56.024 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 7m ago 4h 625M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:17:56.024 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 7m ago 4h 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:17:56.024 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 5m ago 4h 25.8M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:17:56.024 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 7m ago 4h 767M 2492M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:17:56.025 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 7m ago 4h 758M 2492M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:17:56.025 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 7m ago 4h 593M 2492M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:17:56.025 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 7m ago 4h 739M 2492M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:17:56.025 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 5m ago 4h 653M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:17:56.026 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 5m ago 4h 812M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:17:56.026 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 5m ago 4h 725M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:17:56.026 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 5m ago 4h 857M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:17:56.026 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 5m ago 4h 181M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:17:56.373 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:17:56.374 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:17:56.374 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:17:56.374 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:17:56.374 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:17:56.375 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:17:56.375 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:17:56.375 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:17:56.375 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:17:56.376 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:17:56.376 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:17:56.376 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:17:56.376 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:17:56.376 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:17:56.377 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:17:56.377 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:17:56.377 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:17:56.377 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:17:56.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:17:56 smithi083 ceph-mon[79930]: pgmap v8690: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:56.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:17:56 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1145254835' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:17:56.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:17:56 smithi083 ceph-mon[77242]: pgmap v8690: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:56.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:17:56 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1145254835' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:17:57.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:17:56 smithi153 ceph-mon[70044]: pgmap v8690: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:57.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:17:56 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1145254835' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:17:57.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:17:57 smithi083 ceph-mon[79930]: from='client.54172 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:17:57.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:17:57 smithi083 ceph-mon[79930]: from='client.54178 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:17:57.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:17:57 smithi083 ceph-mon[77242]: from='client.54172 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:17:57.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:17:57 smithi083 ceph-mon[77242]: from='client.54178 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:17:58.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:17:57 smithi153 ceph-mon[70044]: from='client.54172 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:17:58.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:17:57 smithi153 ceph-mon[70044]: from='client.54178 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:17:58.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:17:58 smithi083 ceph-mon[79930]: pgmap v8691: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:58.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:17:58 smithi083 ceph-mon[77242]: pgmap v8691: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:17:59.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:17:58 smithi153 ceph-mon[70044]: pgmap v8691: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:00.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:18:00 smithi083 ceph-mon[79930]: pgmap v8692: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:00.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:18:00 smithi083 ceph-mon[77242]: pgmap v8692: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:01.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:18:00 smithi153 ceph-mon[70044]: pgmap v8692: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:01.507 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:18:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:18:01.227Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:18:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:18:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:18:01] "GET /metrics HTTP/1.1" 200 225220 "" "Prometheus/2.33.4" 2022-09-29T10:18:02.589 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:18:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:18:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:18:02.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:18:02 smithi083 ceph-mon[79930]: pgmap v8693: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:02.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:18:02 smithi083 ceph-mon[77242]: pgmap v8693: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:03.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:18:02 smithi153 ceph-mon[70044]: pgmap v8693: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:18:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:18:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:18:04.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:18:04 smithi153 ceph-mon[70044]: pgmap v8694: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:04.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:18:04 smithi083 ceph-mon[79930]: pgmap v8694: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:04.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:18:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:18:04.607Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:18:04.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:18:04 smithi083 ceph-mon[77242]: pgmap v8694: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:05.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:18:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:18:05.311Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:18:06.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:18:06 smithi153 ceph-mon[70044]: pgmap v8695: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:06.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:18:06 smithi083 ceph-mon[77242]: pgmap v8695: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:06.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:18:06 smithi083 ceph-mon[79930]: pgmap v8695: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:08.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:18:08 smithi153 ceph-mon[70044]: pgmap v8696: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:08.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:18:08 smithi083 ceph-mon[77242]: pgmap v8696: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:08.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:18:08 smithi083 ceph-mon[79930]: pgmap v8696: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:10.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:18:10 smithi153 ceph-mon[70044]: pgmap v8697: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:10.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:18:10 smithi083 ceph-mon[77242]: pgmap v8697: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:10.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:18:10 smithi083 ceph-mon[79930]: pgmap v8697: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:11.507 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:18:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:18:11.227Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:18:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:18:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:18:11] "GET /metrics HTTP/1.1" 200 225229 "" "Prometheus/2.33.4" 2022-09-29T10:18:12.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:18:12 smithi153 ceph-mon[70044]: pgmap v8698: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:18:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:18:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:18:12.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:18:12 smithi083 ceph-mon[77242]: pgmap v8698: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:12.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:18:12 smithi083 ceph-mon[79930]: pgmap v8698: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:18:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:18:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:18:14.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:18:14 smithi153 ceph-mon[70044]: pgmap v8699: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:14.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:18:14 smithi083 ceph-mon[79930]: pgmap v8699: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:14.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:18:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:18:14.608Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:18:14.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:18:14 smithi083 ceph-mon[77242]: pgmap v8699: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:15.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:18:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:18:15.311Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:18:16.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:18:16 smithi153 ceph-mon[70044]: pgmap v8700: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:16.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:18:16 smithi083 ceph-mon[79930]: pgmap v8700: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:16.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:18:16 smithi083 ceph-mon[77242]: pgmap v8700: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:18.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:18:18 smithi153 ceph-mon[70044]: pgmap v8701: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:18.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:18:18 smithi083 ceph-mon[79930]: pgmap v8701: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:18.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:18:18 smithi083 ceph-mon[77242]: pgmap v8701: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:20.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:18:20 smithi153 ceph-mon[70044]: pgmap v8702: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:20.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:18:20 smithi083 ceph-mon[79930]: pgmap v8702: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:20.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:18:20 smithi083 ceph-mon[77242]: pgmap v8702: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:21.507 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:18:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:18:21.227Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:18:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:18:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:18:21] "GET /metrics HTTP/1.1" 200 225229 "" "Prometheus/2.33.4" 2022-09-29T10:18:22.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:18:22 smithi153 ceph-mon[70044]: pgmap v8703: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:18:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:18:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:18:22.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:18:22 smithi083 ceph-mon[79930]: pgmap v8703: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:22.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:18:22 smithi083 ceph-mon[77242]: pgmap v8703: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:18:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:18:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:18:24.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:18:24 smithi153 ceph-mon[70044]: pgmap v8704: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:24.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:18:24 smithi083 ceph-mon[79930]: pgmap v8704: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:24.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:18:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:18:24.608Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:18:24.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:18:24 smithi083 ceph-mon[77242]: pgmap v8704: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:25.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:18:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:18:25.312Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:18:26.700 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:18:26.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:18:26 smithi153 ceph-mon[70044]: pgmap v8705: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:26.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:18:26 smithi083 ceph-mon[79930]: pgmap v8705: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:26.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:18:26 smithi083 ceph-mon[77242]: pgmap v8705: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:26.988 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:18:26.989 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 8m ago 4h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:18:26.989 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 5m ago 4h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:18:26.989 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 8m ago 4h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:18:26.989 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 5m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:18:26.990 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 5m ago 4h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:18:26.990 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 8m ago 4h 757M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:18:26.990 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 8m ago 4h 1091M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:18:26.990 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 5m ago 4h 470M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:18:26.991 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 8m ago 4h 625M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:18:26.991 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 8m ago 4h 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:18:26.991 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 5m ago 4h 25.8M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:18:26.991 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 8m ago 4h 767M 2492M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:18:26.991 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 8m ago 4h 758M 2492M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:18:26.992 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 8m ago 4h 593M 2492M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:18:26.992 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 8m ago 4h 739M 2492M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:18:26.992 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 5m ago 4h 653M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:18:26.992 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 5m ago 4h 812M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:18:26.992 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 5m ago 4h 725M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:18:26.993 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 5m ago 4h 857M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:18:26.993 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 5m ago 4h 181M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:18:27.343 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:18:27.343 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:18:27.344 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:18:27.344 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:18:27.344 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:18:27.344 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:18:27.345 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:18:27.345 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:18:27.345 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:18:27.345 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:18:27.346 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:18:27.346 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:18:27.346 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:18:27.346 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:18:27.347 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:18:27.347 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:18:27.347 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:18:27.347 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:18:27.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:18:27 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/597978357' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:18:27.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:18:27 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/597978357' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:18:27.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:18:27 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/597978357' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:18:28.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:18:28 smithi153 ceph-mon[70044]: from='client.44259 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:18:28.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:18:28 smithi153 ceph-mon[70044]: pgmap v8706: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:28.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:18:28 smithi153 ceph-mon[70044]: from='client.44265 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:18:28.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:18:28 smithi083 ceph-mon[79930]: from='client.44259 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:18:28.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:18:28 smithi083 ceph-mon[79930]: pgmap v8706: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:28.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:18:28 smithi083 ceph-mon[79930]: from='client.44265 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:18:28.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:18:28 smithi083 ceph-mon[77242]: from='client.44259 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:18:28.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:18:28 smithi083 ceph-mon[77242]: pgmap v8706: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:28.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:18:28 smithi083 ceph-mon[77242]: from='client.44265 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:18:30.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:18:30 smithi153 ceph-mon[70044]: pgmap v8707: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:30.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:18:30 smithi083 ceph-mon[79930]: pgmap v8707: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:30.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:18:30 smithi083 ceph-mon[77242]: pgmap v8707: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:31.506 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:18:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:18:31.228Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:18:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:18:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:18:31] "GET /metrics HTTP/1.1" 200 225229 "" "Prometheus/2.33.4" 2022-09-29T10:18:32.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:18:32 smithi153 ceph-mon[70044]: pgmap v8708: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:18:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:18:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:18:32.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:18:32 smithi083 ceph-mon[79930]: pgmap v8708: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:32.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:18:32 smithi083 ceph-mon[77242]: pgmap v8708: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:18:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:18:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:18:34.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:18:34 smithi153 ceph-mon[70044]: pgmap v8709: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:34.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:18:34 smithi083 ceph-mon[79930]: pgmap v8709: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:34.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:18:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:18:34.608Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:18:34.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:18:34 smithi083 ceph-mon[77242]: pgmap v8709: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:35.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:18:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:18:35.313Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:18:36.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:18:36 smithi153 ceph-mon[70044]: pgmap v8710: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:36.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:18:36 smithi083 ceph-mon[79930]: pgmap v8710: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:36.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:18:36 smithi083 ceph-mon[77242]: pgmap v8710: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:38.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:18:38 smithi153 ceph-mon[70044]: pgmap v8711: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:38.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:18:38 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:18:38.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:18:38 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:18:38.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:18:38 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:18:38.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:18:38 smithi083 ceph-mon[77242]: pgmap v8711: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:38.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:18:38 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:18:38.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:18:38 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:18:38.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:18:38 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:18:38.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:18:38 smithi083 ceph-mon[79930]: pgmap v8711: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:38.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:18:38 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:18:38.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:18:38 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:18:38.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:18:38 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:18:39.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:18:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:18:39.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:18:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:18:39.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:18:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:18:39.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:18:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:18:39.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:18:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:18:39.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:18:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:18:39.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:18:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:18:39.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:18:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:18:39.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:18:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:18:39.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:18:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:18:39.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:18:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:18:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:18:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:18:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:18:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:18:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:18:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:18:39.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:18:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:18:39.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:18:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:18:39.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:18:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:18:39.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:18:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:18:39.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:18:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:18:39.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:18:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:18:39.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:18:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:18:39.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:18:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:18:39.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:18:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:18:39.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:18:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:18:39.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:18:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:18:39.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:18:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:18:39.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:18:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:18:39.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:18:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:18:39.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:18:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:18:39.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:18:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:18:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:18:40 smithi153 ceph-mon[70044]: Adjusting osd_memory_target on smithi083 to 2483M 2022-09-29T10:18:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:18:40 smithi153 ceph-mon[70044]: pgmap v8712: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:18:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:18:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:18:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:18:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:18:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:18:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:18:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:18:40.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:18:40 smithi083 ceph-mon[79930]: Adjusting osd_memory_target on smithi083 to 2483M 2022-09-29T10:18:40.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:18:40 smithi083 ceph-mon[79930]: pgmap v8712: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:18:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:18:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:18:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:18:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:18:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:18:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:18:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:18:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:18:40 smithi083 ceph-mon[77242]: Adjusting osd_memory_target on smithi083 to 2483M 2022-09-29T10:18:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:18:40 smithi083 ceph-mon[77242]: pgmap v8712: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:18:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:18:40.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:18:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:18:40.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:18:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:18:40.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:18:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:18:41.507 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:18:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:18:41.229Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:18:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:18:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:18:41] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T10:18:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:18:42 smithi153 ceph-mon[70044]: pgmap v8713: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:18:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:18:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:18:42.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:18:42 smithi083 ceph-mon[79930]: pgmap v8713: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:42.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:18:42 smithi083 ceph-mon[77242]: pgmap v8713: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:18:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:18:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:18:44.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:18:44 smithi083 ceph-mon[79930]: pgmap v8714: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:44.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:18:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:18:44.608Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:18:44.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:18:44 smithi083 ceph-mon[77242]: pgmap v8714: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:45.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:18:44 smithi153 ceph-mon[70044]: pgmap v8714: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:45.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:18:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:18:45.313Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:18:46.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:18:46 smithi083 ceph-mon[79930]: pgmap v8715: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:46.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:18:46 smithi083 ceph-mon[77242]: pgmap v8715: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:47.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:18:46 smithi153 ceph-mon[70044]: pgmap v8715: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:48.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:18:48 smithi083 ceph-mon[79930]: pgmap v8716: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:48.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:18:48 smithi083 ceph-mon[77242]: pgmap v8716: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:49.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:18:48 smithi153 ceph-mon[70044]: pgmap v8716: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:50.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:18:50 smithi083 ceph-mon[79930]: pgmap v8717: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:50.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:18:50 smithi083 ceph-mon[77242]: pgmap v8717: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:51.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:18:50 smithi153 ceph-mon[70044]: pgmap v8717: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:51.507 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:18:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:18:51.229Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:18:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:18:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:18:51] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T10:18:52.568 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:18:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:18:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:18:52.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:18:52 smithi083 ceph-mon[77242]: pgmap v8718: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:52.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:18:52 smithi083 ceph-mon[79930]: pgmap v8718: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:53.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:18:52 smithi153 ceph-mon[70044]: pgmap v8718: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:18:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:18:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:18:54.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:18:54 smithi083 ceph-mon[79930]: pgmap v8719: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:54.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:18:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:18:54.610Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:18:54.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:18:54 smithi083 ceph-mon[77242]: pgmap v8719: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:55.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:18:54 smithi153 ceph-mon[70044]: pgmap v8719: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:55.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:18:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:18:55.313Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:18:56.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:18:56 smithi083 ceph-mon[77242]: pgmap v8720: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:56.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:18:56 smithi083 ceph-mon[79930]: pgmap v8720: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:57.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:18:56 smithi153 ceph-mon[70044]: pgmap v8720: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:57.670 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:18:57.959 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:18:57.960 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 8m ago 4h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:18:57.960 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 6m ago 4h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:18:57.960 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 8m ago 4h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:18:57.961 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 6m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:18:57.961 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 6m ago 4h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:18:57.961 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 8m ago 4h 757M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:18:57.961 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 8m ago 4h 1091M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:18:57.962 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 6m ago 4h 470M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:18:57.962 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 8m ago 4h 625M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:18:57.962 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 8m ago 4h 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:18:57.962 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 6m ago 4h 25.8M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:18:57.963 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 8m ago 4h 767M 2483M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:18:57.963 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 8m ago 4h 758M 2483M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:18:57.963 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 8m ago 4h 593M 2483M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:18:57.963 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 8m ago 4h 739M 2483M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:18:57.963 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 6m ago 4h 653M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:18:57.964 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 6m ago 4h 812M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:18:57.964 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 6m ago 4h 725M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:18:57.964 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 6m ago 4h 857M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:18:57.964 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 6m ago 4h 181M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:18:58.312 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:18:58.312 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:18:58.312 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:18:58.312 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:18:58.313 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:18:58.313 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:18:58.313 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:18:58.313 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:18:58.314 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:18:58.314 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:18:58.314 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:18:58.314 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:18:58.315 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:18:58.315 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:18:58.315 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:18:58.315 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:18:58.316 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:18:58.316 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:18:58.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:18:58 smithi083 ceph-mon[79930]: pgmap v8721: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:59.003 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:18:58 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/791890636' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:18:59.003 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:18:58 smithi083 ceph-mon[77242]: pgmap v8721: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:59.004 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:18:58 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/791890636' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:18:59.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:18:58 smithi153 ceph-mon[70044]: pgmap v8721: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:18:59.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:18:58 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/791890636' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:18:59.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:18:59 smithi083 ceph-mon[79930]: from='client.54208 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:18:59.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:18:59 smithi083 ceph-mon[79930]: from='client.44283 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:18:59.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:18:59 smithi083 ceph-mon[77242]: from='client.54208 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:18:59.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:18:59 smithi083 ceph-mon[77242]: from='client.44283 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:19:00.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:18:59 smithi153 ceph-mon[70044]: from='client.54208 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:19:00.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:18:59 smithi153 ceph-mon[70044]: from='client.44283 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:19:00.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:19:00 smithi083 ceph-mon[79930]: pgmap v8722: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:00.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:19:00 smithi083 ceph-mon[77242]: pgmap v8722: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:01.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:19:00 smithi153 ceph-mon[70044]: pgmap v8722: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:01.506 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:19:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:19:01.230Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:19:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:19:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:19:01] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T10:19:02.590 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:19:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:19:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:19:02.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:19:02 smithi083 ceph-mon[79930]: pgmap v8723: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:02.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:19:02 smithi083 ceph-mon[77242]: pgmap v8723: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:03.022 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:19:02 smithi153 ceph-mon[70044]: pgmap v8723: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:19:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:19:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:19:04.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:19:04 smithi083 ceph-mon[79930]: pgmap v8724: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:04.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:19:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:19:04.610Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:19:04.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:19:04 smithi083 ceph-mon[77242]: pgmap v8724: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:05.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:19:04 smithi153 ceph-mon[70044]: pgmap v8724: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:05.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:19:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:19:05.313Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:19:06.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:19:06 smithi083 ceph-mon[77242]: pgmap v8725: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:06.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:19:06 smithi083 ceph-mon[79930]: pgmap v8725: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:07.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:19:06 smithi153 ceph-mon[70044]: pgmap v8725: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:08.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:19:08 smithi083 ceph-mon[79930]: pgmap v8726: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:08.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:19:08 smithi083 ceph-mon[77242]: pgmap v8726: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:09.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:19:08 smithi153 ceph-mon[70044]: pgmap v8726: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:09.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:19:09 smithi083 ceph-mon[77242]: pgmap v8727: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:09.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:19:09 smithi083 ceph-mon[79930]: pgmap v8727: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:10.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:19:09 smithi153 ceph-mon[70044]: pgmap v8727: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:11.506 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:19:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:19:11.231Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:19:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:19:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:19:11] "GET /metrics HTTP/1.1" 200 225228 "" "Prometheus/2.33.4" 2022-09-29T10:19:12.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:19:12 smithi153 ceph-mon[70044]: pgmap v8728: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:12.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:19:12 smithi083 ceph-mon[79930]: pgmap v8728: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:12.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:19:12 smithi083 ceph-mon[77242]: pgmap v8728: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:19:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:19:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:19:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:19:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:19:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:19:14.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:19:14 smithi153 ceph-mon[70044]: pgmap v8729: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:14.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:19:14 smithi083 ceph-mon[77242]: pgmap v8729: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:14.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:19:14 smithi083 ceph-mon[79930]: pgmap v8729: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:14.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:19:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:19:14.610Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:19:15.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:19:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:19:15.314Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:19:16.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:19:16 smithi153 ceph-mon[70044]: pgmap v8730: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:16.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:19:16 smithi083 ceph-mon[77242]: pgmap v8730: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:16.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:19:16 smithi083 ceph-mon[79930]: pgmap v8730: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:18.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:19:18 smithi153 ceph-mon[70044]: pgmap v8731: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:18.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:19:18 smithi083 ceph-mon[79930]: pgmap v8731: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:18.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:19:18 smithi083 ceph-mon[77242]: pgmap v8731: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:20.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:19:20 smithi153 ceph-mon[70044]: pgmap v8732: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:20.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:19:20 smithi083 ceph-mon[77242]: pgmap v8732: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:20.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:19:20 smithi083 ceph-mon[79930]: pgmap v8732: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:21.507 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:19:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:19:21.232Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:19:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:19:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:19:21] "GET /metrics HTTP/1.1" 200 225228 "" "Prometheus/2.33.4" 2022-09-29T10:19:22.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:19:22 smithi153 ceph-mon[70044]: pgmap v8733: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:19:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:19:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:19:22.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:19:22 smithi083 ceph-mon[79930]: pgmap v8733: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:22.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:19:22 smithi083 ceph-mon[77242]: pgmap v8733: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:19:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:19:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:19:24.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:19:24 smithi153 ceph-mon[70044]: pgmap v8734: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:24.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:19:24 smithi083 ceph-mon[79930]: pgmap v8734: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:24.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:19:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:19:24.611Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:19:24.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:19:24 smithi083 ceph-mon[77242]: pgmap v8734: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:25.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:19:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:19:25.315Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:19:26.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:19:26 smithi153 ceph-mon[70044]: pgmap v8735: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:26.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:19:26 smithi083 ceph-mon[79930]: pgmap v8735: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:26.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:19:26 smithi083 ceph-mon[77242]: pgmap v8735: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:28.639 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:19:28.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:19:28 smithi153 ceph-mon[70044]: pgmap v8736: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:28.928 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:19:28.929 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 9m ago 4h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:19:28.929 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 6m ago 4h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:19:28.929 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 9m ago 4h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:19:28.929 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 6m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:19:28.930 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 6m ago 4h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:19:28.930 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 9m ago 4h 757M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:19:28.930 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 9m ago 4h 1091M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:19:28.930 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 6m ago 4h 470M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:19:28.931 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 9m ago 4h 625M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:19:28.931 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 9m ago 4h 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:19:28.931 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 6m ago 4h 25.8M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:19:28.931 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 9m ago 4h 767M 2483M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:19:28.931 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 9m ago 4h 758M 2483M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:19:28.932 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 9m ago 4h 593M 2483M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:19:28.932 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 9m ago 4h 739M 2483M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:19:28.932 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 6m ago 4h 653M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:19:28.932 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 6m ago 4h 812M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:19:28.933 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 6m ago 4h 725M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:19:28.933 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 6m ago 4h 857M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:19:28.933 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 6m ago 4h 181M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:19:28.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:19:28 smithi083 ceph-mon[77242]: pgmap v8736: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:28.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:19:28 smithi083 ceph-mon[79930]: pgmap v8736: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:29.284 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:19:29.285 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:19:29.285 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:19:29.285 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:19:29.286 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:19:29.286 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:19:29.286 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:19:29.286 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:19:29.287 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:19:29.287 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:19:29.287 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:19:29.287 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:19:29.288 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:19:29.288 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:19:29.288 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:19:29.288 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:19:29.288 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:19:29.289 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:19:29.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:19:29 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2919703149' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:19:29.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:19:29 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2919703149' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:19:29.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:19:29 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2919703149' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:19:30.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:19:30 smithi153 ceph-mon[70044]: from='client.54226 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:19:30.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:19:30 smithi153 ceph-mon[70044]: from='client.54232 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:19:30.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:19:30 smithi153 ceph-mon[70044]: pgmap v8737: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:30.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:19:30 smithi083 ceph-mon[79930]: from='client.54226 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:19:30.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:19:30 smithi083 ceph-mon[79930]: from='client.54232 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:19:30.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:19:30 smithi083 ceph-mon[79930]: pgmap v8737: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:30.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:19:30 smithi083 ceph-mon[77242]: from='client.54226 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:19:30.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:19:30 smithi083 ceph-mon[77242]: from='client.54232 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:19:30.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:19:30 smithi083 ceph-mon[77242]: pgmap v8737: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:31.507 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:19:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:19:31.232Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:19:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:19:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:19:31] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T10:19:32.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:19:32 smithi153 ceph-mon[70044]: pgmap v8738: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:19:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:19:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:19:32.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:19:32 smithi083 ceph-mon[79930]: pgmap v8738: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:32.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:19:32 smithi083 ceph-mon[77242]: pgmap v8738: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:19:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:19:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:19:34.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:19:34 smithi153 ceph-mon[70044]: pgmap v8739: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:34.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:19:34 smithi083 ceph-mon[79930]: pgmap v8739: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:34.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:19:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:19:34.611Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:19:34.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:19:34 smithi083 ceph-mon[77242]: pgmap v8739: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:35.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:19:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:19:35.316Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:19:36.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:19:36 smithi153 ceph-mon[70044]: pgmap v8740: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:36.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:19:36 smithi083 ceph-mon[77242]: pgmap v8740: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:36.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:19:36 smithi083 ceph-mon[79930]: pgmap v8740: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:38.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:19:38 smithi153 ceph-mon[70044]: pgmap v8741: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:38.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:19:38 smithi083 ceph-mon[79930]: pgmap v8741: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:38.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:19:38 smithi083 ceph-mon[77242]: pgmap v8741: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:39.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:19:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:19:39.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:19:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:19:39.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:19:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:19:39.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:19:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:19:39.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:19:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:19:39.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:19:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:19:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:19:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:19:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:19:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:19:39.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:19:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:19:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:19:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:19:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:19:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:19:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:19:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:19:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:19:40 smithi153 ceph-mon[70044]: pgmap v8742: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:19:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:19:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:19:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:19:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:19:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:19:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:19:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:19:40.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:19:40 smithi083 ceph-mon[79930]: pgmap v8742: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:40.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:19:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:19:40.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:19:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:19:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:19:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:19:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:19:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:19:40.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:19:40 smithi083 ceph-mon[77242]: pgmap v8742: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:19:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:19:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:19:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:19:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:19:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:19:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:19:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:19:41.506 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:19:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:19:41.233Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:19:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:19:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:19:41] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T10:19:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:19:42 smithi153 ceph-mon[70044]: pgmap v8743: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:19:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:19:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:19:42.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:19:42 smithi083 ceph-mon[79930]: pgmap v8743: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:42.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:19:42 smithi083 ceph-mon[77242]: pgmap v8743: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:19:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:19:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:19:44.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:19:44 smithi153 ceph-mon[70044]: pgmap v8744: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:44.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:19:44 smithi083 ceph-mon[79930]: pgmap v8744: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:44.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:19:44 smithi083 ceph-mon[77242]: pgmap v8744: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:44.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:19:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:19:44.612Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:19:45.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:19:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:19:45.316Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:19:46.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:19:46 smithi083 ceph-mon[79930]: pgmap v8745: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:46.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:19:46 smithi083 ceph-mon[77242]: pgmap v8745: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:47.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:19:46 smithi153 ceph-mon[70044]: pgmap v8745: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:48.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:19:48 smithi083 ceph-mon[79930]: pgmap v8746: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:48.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:19:48 smithi083 ceph-mon[77242]: pgmap v8746: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:49.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:19:48 smithi153 ceph-mon[70044]: pgmap v8746: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:50.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:19:50 smithi083 ceph-mon[77242]: pgmap v8747: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:50.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:19:50 smithi083 ceph-mon[79930]: pgmap v8747: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:51.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:19:50 smithi153 ceph-mon[70044]: pgmap v8747: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:51.507 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:19:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:19:51.234Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:19:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:19:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:19:51] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T10:19:52.569 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:19:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:19:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:19:52.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:19:52 smithi083 ceph-mon[79930]: pgmap v8748: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:52.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:19:52 smithi083 ceph-mon[77242]: pgmap v8748: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:53.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:19:52 smithi153 ceph-mon[70044]: pgmap v8748: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:19:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:19:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:19:54.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:19:54 smithi083 ceph-mon[79930]: pgmap v8749: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:54.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:19:54 smithi083 ceph-mon[77242]: pgmap v8749: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:54.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:19:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:19:54.613Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:19:55.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:19:54 smithi153 ceph-mon[70044]: pgmap v8749: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:55.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:19:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:19:55.317Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:19:56.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:19:56 smithi083 ceph-mon[79930]: pgmap v8750: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:56.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:19:56 smithi083 ceph-mon[77242]: pgmap v8750: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:57.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:19:56 smithi153 ceph-mon[70044]: pgmap v8750: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:58.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:19:58 smithi083 ceph-mon[79930]: pgmap v8751: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:58.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:19:58 smithi083 ceph-mon[77242]: pgmap v8751: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:59.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:19:58 smithi153 ceph-mon[70044]: pgmap v8751: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:19:59.612 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:19:59.897 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:19:59.897 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 9m ago 4h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:19:59.898 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 7m ago 4h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:19:59.898 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 9m ago 4h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:19:59.898 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 7m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:19:59.898 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 7m ago 4h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:19:59.899 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 9m ago 4h 757M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:19:59.899 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 9m ago 4h 1091M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:19:59.899 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 7m ago 4h 470M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:19:59.899 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 9m ago 4h 625M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:19:59.900 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 9m ago 4h 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:19:59.900 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 7m ago 4h 25.8M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:19:59.900 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 9m ago 4h 767M 2483M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:19:59.900 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 9m ago 4h 758M 2483M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:19:59.900 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 9m ago 4h 593M 2483M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:19:59.901 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 9m ago 4h 739M 2483M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:19:59.901 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 7m ago 4h 653M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:19:59.901 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 7m ago 4h 812M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:19:59.901 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 7m ago 4h 725M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:19:59.901 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 7m ago 4h 857M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:19:59.902 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 7m ago 4h 181M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:20:00.255 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:20:00.256 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:20:00.256 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:20:00.256 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:20:00.256 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:20:00.257 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:20:00.257 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:20:00.257 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:20:00.257 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:20:00.257 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:20:00.258 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:20:00.258 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:20:00.258 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:20:00.258 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:20:00.259 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:20:00.259 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:20:00.259 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:20:00.259 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:20:00.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:20:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[77219]: 2022-09-29T10:20:00.002+0000 7f160f665700 -1 log_channel(cluster) log [ERR] : overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T10:20:00.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:20:00 smithi083 ceph-mon[77242]: pgmap v8752: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:20:00.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:20:00 smithi083 ceph-mon[77242]: from='client.44313 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:20:00.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:20:00 smithi083 ceph-mon[77242]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T10:20:00.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:20:00 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4255576451' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:20:00.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:20:00 smithi083 ceph-mon[79930]: pgmap v8752: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:20:00.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:20:00 smithi083 ceph-mon[79930]: from='client.44313 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:20:00.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:20:00 smithi083 ceph-mon[79930]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T10:20:00.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:20:00 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4255576451' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:20:01.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:20:00 smithi153 ceph-mon[70044]: pgmap v8752: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:20:01.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:20:00 smithi153 ceph-mon[70044]: from='client.44313 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:20:01.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:20:00 smithi153 ceph-mon[70044]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T10:20:01.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:20:00 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4255576451' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:20:01.507 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:20:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:20:01.234Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:20:01.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:20:01 smithi083 ceph-mon[79930]: from='client.54250 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:20:01.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:20:01 smithi083 ceph-mon[77242]: from='client.54250 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:20:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:20:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:20:01] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T10:20:02.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:20:01 smithi153 ceph-mon[70044]: from='client.54250 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:20:02.589 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:20:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:20:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:20:02.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:20:02 smithi083 ceph-mon[79930]: pgmap v8753: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:20:02.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:20:02 smithi083 ceph-mon[77242]: pgmap v8753: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:20:03.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:20:02 smithi153 ceph-mon[70044]: pgmap v8753: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:20:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:20:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:20:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:20:04.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:20:04 smithi083 ceph-mon[79930]: pgmap v8754: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:20:04.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:20:04 smithi083 ceph-mon[77242]: pgmap v8754: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:20:04.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:20:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:20:04.613Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:20:05.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:20:04 smithi153 ceph-mon[70044]: pgmap v8754: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:20:05.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:20:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:20:05.318Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:20:06.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:20:06 smithi083 ceph-mon[79930]: pgmap v8755: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:20:06.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:20:06 smithi083 ceph-mon[77242]: pgmap v8755: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:20:07.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:20:06 smithi153 ceph-mon[70044]: pgmap v8755: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:20:08.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:20:08 smithi083 ceph-mon[79930]: pgmap v8756: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:20:08.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:20:08 smithi083 ceph-mon[77242]: pgmap v8756: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:20:09.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:20:08 smithi153 ceph-mon[70044]: pgmap v8756: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:20:10.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:20:10 smithi083 ceph-mon[79930]: pgmap v8757: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:20:10.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:20:10 smithi083 ceph-mon[77242]: pgmap v8757: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:20:11.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:20:10 smithi153 ceph-mon[70044]: pgmap v8757: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:20:11.506 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:20:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:20:11.234Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:20:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:20:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:20:11] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T10:20:12.604 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:20:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:20:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:20:12.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:20:12 smithi083 ceph-mon[79930]: pgmap v8758: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:20:12.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:20:12 smithi083 ceph-mon[77242]: pgmap v8758: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:20:13.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:20:12 smithi153 ceph-mon[70044]: pgmap v8758: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:20:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:20:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:20:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:20:14.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:20:14 smithi083 ceph-mon[79930]: pgmap v8759: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:20:14.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:20:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:20:14.614Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:20:14.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:20:14 smithi083 ceph-mon[77242]: pgmap v8759: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:20:15.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:20:14 smithi153 ceph-mon[70044]: pgmap v8759: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:20:15.659 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:20:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:20:15.318Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:20:15.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:20:15 smithi083 ceph-mon[77242]: pgmap v8760: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:20:15.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:20:15 smithi083 ceph-mon[79930]: pgmap v8760: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:20:16.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:20:15 smithi153 ceph-mon[70044]: pgmap v8760: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:20:18.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:20:18 smithi153 ceph-mon[70044]: pgmap v8761: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:20:18.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:20:18 smithi083 ceph-mon[79930]: pgmap v8761: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:20:18.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:20:18 smithi083 ceph-mon[77242]: pgmap v8761: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:20:20.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:20:20 smithi153 ceph-mon[70044]: pgmap v8762: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:20:20.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:20:20 smithi083 ceph-mon[77242]: pgmap v8762: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:20:20.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:20:20 smithi083 ceph-mon[79930]: pgmap v8762: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:20:21.506 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:20:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:20:21.234Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:20:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:20:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:20:21] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T10:20:22.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:20:22 smithi153 ceph-mon[70044]: pgmap v8763: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:20:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:20:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:20:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:20:22.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:20:22 smithi083 ceph-mon[79930]: pgmap v8763: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:20:22.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:20:22 smithi083 ceph-mon[77242]: pgmap v8763: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:20:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:20:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:20:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:20:24.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:20:24 smithi153 ceph-mon[70044]: pgmap v8764: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:20:24.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:20:24 smithi083 ceph-mon[79930]: pgmap v8764: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:20:24.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:20:24 smithi083 ceph-mon[77242]: pgmap v8764: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:20:24.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:20:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:20:24.615Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:20:25.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:20:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:20:25.319Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:20:26.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:20:26 smithi153 ceph-mon[70044]: pgmap v8765: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:20:26.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:20:26 smithi083 ceph-mon[77242]: pgmap v8765: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:20:26.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:20:26 smithi083 ceph-mon[79930]: pgmap v8765: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:20:28.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:20:28 smithi153 ceph-mon[70044]: pgmap v8766: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:20:28.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:20:28 smithi083 ceph-mon[77242]: pgmap v8766: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:20:28.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:20:28 smithi083 ceph-mon[79930]: pgmap v8766: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:20:30.582 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:20:30.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:20:30 smithi153 ceph-mon[70044]: pgmap v8767: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:20:30.870 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:20:30.870 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 10m ago 4h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:20:30.870 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 7m ago 4h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:20:30.871 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 10m ago 4h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:20:30.871 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 7m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:20:30.871 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 7m ago 4h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:20:30.871 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 10m ago 4h 757M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:20:30.871 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 10m ago 4h 1091M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:20:30.872 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 7m ago 4h 470M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:20:30.872 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 10m ago 4h 625M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:20:30.872 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 10m ago 4h 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:20:30.872 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 7m ago 4h 25.8M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:20:30.873 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 10m ago 4h 767M 2483M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:20:30.873 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 10m ago 4h 758M 2483M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:20:30.873 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 10m ago 4h 593M 2483M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:20:30.873 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 10m ago 4h 739M 2483M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:20:30.874 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 7m ago 4h 653M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:20:30.874 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 7m ago 4h 812M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:20:30.874 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 7m ago 4h 725M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:20:30.874 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 7m ago 4h 857M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:20:30.874 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 7m ago 4h 181M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:20:30.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:20:30 smithi083 ceph-mon[79930]: pgmap v8767: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:20:30.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:20:30 smithi083 ceph-mon[77242]: pgmap v8767: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:20:31.229 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:20:31.230 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:20:31.230 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:20:31.230 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:20:31.230 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:20:31.231 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:20:31.231 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:20:31.231 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:20:31.231 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:20:31.232 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:20:31.232 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:20:31.232 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:20:31.232 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:20:31.233 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:20:31.233 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:20:31.233 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:20:31.233 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:20:31.234 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:20:31.506 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:20:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:20:31.234Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:20:31.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:20:31 smithi153 ceph-mon[70044]: from='client.44331 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:20:31.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:20:31 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2661307564' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:20:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:20:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:20:31] "GET /metrics HTTP/1.1" 200 225227 "" "Prometheus/2.33.4" 2022-09-29T10:20:31.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:20:31 smithi083 ceph-mon[77242]: from='client.44331 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:20:31.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:20:31 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2661307564' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:20:31.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:20:31 smithi083 ceph-mon[79930]: from='client.44331 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:20:31.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:20:31 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2661307564' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:20:32.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:20:32 smithi153 ceph-mon[70044]: from='client.44337 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:20:32.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:20:32 smithi153 ceph-mon[70044]: pgmap v8768: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:20:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:20:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:20:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:20:32.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:20:32 smithi083 ceph-mon[77242]: from='client.44337 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:20:32.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:20:32 smithi083 ceph-mon[77242]: pgmap v8768: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:20:32.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:20:32 smithi083 ceph-mon[79930]: from='client.44337 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:20:32.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:20:32 smithi083 ceph-mon[79930]: pgmap v8768: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:20:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:20:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:20:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:20:34.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:20:34 smithi153 ceph-mon[70044]: pgmap v8769: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:20:34.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:20:34 smithi083 ceph-mon[79930]: pgmap v8769: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:20:34.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:20:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:20:34.615Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:20:34.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:20:34 smithi083 ceph-mon[77242]: pgmap v8769: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:20:35.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:20:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:20:35.320Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:20:36.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:20:36 smithi153 ceph-mon[70044]: pgmap v8770: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:20:36.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:20:36 smithi083 ceph-mon[77242]: pgmap v8770: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:20:36.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:20:36 smithi083 ceph-mon[79930]: pgmap v8770: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:20:38.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:20:38 smithi153 ceph-mon[70044]: pgmap v8771: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:20:38.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:20:38 smithi083 ceph-mon[77242]: pgmap v8771: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:20:38.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:20:38 smithi083 ceph-mon[79930]: pgmap v8771: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:20:39.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:20:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:20:39.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:20:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:20:39.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:20:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:20:39.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:20:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:20:39.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:20:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:20:39.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:20:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:20:39.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:20:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:20:39.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:20:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:20:39.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:20:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:20:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:20:40 smithi153 ceph-mon[70044]: pgmap v8772: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:20:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:20:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:20:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:20:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:20:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:20:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:20:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:20:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:20:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:20:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:20:40.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:20:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:20:40.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:20:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:20:40.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:20:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:20:40.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:20:40 smithi083 ceph-mon[79930]: pgmap v8772: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:20:40.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:20:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:20:40.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:20:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:20:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:20:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:20:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:20:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:20:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:20:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:20:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:20:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:20:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:20:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:20:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:20:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:20:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:20:40 smithi083 ceph-mon[77242]: pgmap v8772: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:20:40.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:20:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:20:40.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:20:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:20:40.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:20:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:20:40.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:20:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:20:40.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:20:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:20:40.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:20:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:20:40.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:20:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:20:40.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:20:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:20:41.507 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:20:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:20:41.235Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:20:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:20:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:20:41] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T10:20:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:20:42 smithi153 ceph-mon[70044]: pgmap v8773: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:20:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:20:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:20:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:20:42.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:20:42 smithi083 ceph-mon[79930]: pgmap v8773: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:20:42.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:20:42 smithi083 ceph-mon[77242]: pgmap v8773: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:20:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:20:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:20:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:20:44.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:20:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:20:44.617Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:20:44.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:20:44 smithi083 ceph-mon[77242]: pgmap v8774: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:20:44.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:20:44 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:20:44.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:20:44 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:20:44.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:20:44 smithi083 ceph-mon[79930]: pgmap v8774: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:20:44.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:20:44 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:20:44.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:20:44 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:20:45.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:20:44 smithi153 ceph-mon[70044]: pgmap v8774: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:20:45.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:20:44 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:20:45.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:20:44 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:20:45.615 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:20:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:20:45.319Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:20:45.615 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:20:45 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:20:45.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:20:45 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:20:46.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:20:45 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:20:46.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:20:46 smithi083 ceph-mon[79930]: pgmap v8775: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:20:46.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:20:46 smithi083 ceph-mon[77242]: pgmap v8775: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:20:47.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:20:46 smithi153 ceph-mon[70044]: pgmap v8775: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:20:47.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:20:47 smithi083 ceph-mon[79930]: pgmap v8776: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:20:47.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:20:47 smithi083 ceph-mon[77242]: pgmap v8776: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:20:48.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:20:47 smithi153 ceph-mon[70044]: pgmap v8776: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:20:50.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:20:50 smithi153 ceph-mon[70044]: pgmap v8777: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:20:50.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:20:50 smithi083 ceph-mon[77242]: pgmap v8777: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:20:50.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:20:50 smithi083 ceph-mon[79930]: pgmap v8777: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:20:51.507 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:20:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:20:51.236Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:20:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:20:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:20:51] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T10:20:52.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:20:52 smithi153 ceph-mon[70044]: pgmap v8778: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:20:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:20:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:20:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:20:52.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:20:52 smithi083 ceph-mon[77242]: pgmap v8778: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:20:52.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:20:52 smithi083 ceph-mon[79930]: pgmap v8778: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:20:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:20:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:20:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:20:54.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:20:54 smithi153 ceph-mon[70044]: pgmap v8779: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:20:54.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:20:54 smithi083 ceph-mon[77242]: pgmap v8779: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:20:54.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:20:54 smithi083 ceph-mon[79930]: pgmap v8779: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:20:54.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:20:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:20:54.616Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:20:55.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:20:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:20:55.320Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:20:56.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:20:56 smithi153 ceph-mon[70044]: pgmap v8780: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:20:56.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:20:56 smithi083 ceph-mon[79930]: pgmap v8780: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:20:56.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:20:56 smithi083 ceph-mon[77242]: pgmap v8780: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:20:58.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:20:58 smithi153 ceph-mon[70044]: pgmap v8781: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:20:58.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:20:58 smithi083 ceph-mon[79930]: pgmap v8781: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:20:58.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:20:58 smithi083 ceph-mon[77242]: pgmap v8781: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:00.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:21:00 smithi153 ceph-mon[70044]: pgmap v8782: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:00.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:21:00 smithi083 ceph-mon[79930]: pgmap v8782: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:00.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:21:00 smithi083 ceph-mon[77242]: pgmap v8782: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:01.507 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:21:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:21:01.237Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:21:01.558 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:21:01.848 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:21:01.849 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 17s ago 4h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:21:01.849 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 8m ago 4h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:21:01.849 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 17s ago 4h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:21:01.849 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 8m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:21:01.849 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 8m ago 4h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:21:01.850 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 17s ago 5h 766M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:21:01.850 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 17s ago 5h 1128M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:21:01.850 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 8m ago 4h 470M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:21:01.851 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 17s ago 4h 647M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:21:01.851 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 17s ago 4h 26.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:21:01.851 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 8m ago 4h 25.8M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:21:01.851 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 17s ago 4h 793M 2483M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:21:01.851 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 17s ago 4h 783M 2483M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:21:01.852 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 17s ago 4h 612M 2483M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:21:01.852 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 17s ago 4h 764M 2483M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:21:01.852 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 8m ago 4h 653M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:21:01.852 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 8m ago 4h 812M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:21:01.853 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 8m ago 4h 725M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:21:01.853 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 8m ago 4h 857M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:21:01.853 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 8m ago 4h 181M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:21:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:21:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:21:01] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T10:21:02.201 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:21:02.202 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:21:02.202 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:21:02.202 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:21:02.202 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:21:02.203 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:21:02.203 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:21:02.203 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:21:02.203 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:21:02.204 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:21:02.204 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:21:02.204 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:21:02.204 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:21:02.205 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:21:02.205 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:21:02.205 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:21:02.205 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:21:02.205 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:21:02.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:21:02 smithi153 ceph-mon[70044]: pgmap v8783: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:02.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:21:02 smithi153 ceph-mon[70044]: from='client.44349 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:21:02.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:21:02 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3450900681' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:21:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:21:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:21:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:21:02.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:21:02 smithi083 ceph-mon[79930]: pgmap v8783: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:02.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:21:02 smithi083 ceph-mon[79930]: from='client.44349 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:21:02.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:21:02 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3450900681' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:21:02.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:21:02 smithi083 ceph-mon[77242]: pgmap v8783: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:02.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:21:02 smithi083 ceph-mon[77242]: from='client.44349 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:21:02.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:21:02 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3450900681' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:21:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:21:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:21:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:21:03.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:21:03 smithi153 ceph-mon[70044]: from='client.54286 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:21:03.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:21:03 smithi083 ceph-mon[79930]: from='client.54286 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:21:03.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:21:03 smithi083 ceph-mon[77242]: from='client.54286 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:21:04.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:21:04 smithi153 ceph-mon[70044]: pgmap v8784: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:04.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:21:04 smithi083 ceph-mon[79930]: pgmap v8784: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:04.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:21:04 smithi083 ceph-mon[77242]: pgmap v8784: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:04.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:21:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:21:04.617Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:21:05.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:21:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:21:05.322Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:21:06.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:21:06 smithi153 ceph-mon[70044]: pgmap v8785: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:06.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:21:06 smithi083 ceph-mon[79930]: pgmap v8785: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:06.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:21:06 smithi083 ceph-mon[77242]: pgmap v8785: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:08.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:21:08 smithi153 ceph-mon[70044]: pgmap v8786: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:08.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:21:08 smithi083 ceph-mon[79930]: pgmap v8786: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:08.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:21:08 smithi083 ceph-mon[77242]: pgmap v8786: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:10.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:21:10 smithi153 ceph-mon[70044]: pgmap v8787: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:10.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:21:10 smithi083 ceph-mon[79930]: pgmap v8787: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:10.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:21:10 smithi083 ceph-mon[77242]: pgmap v8787: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:11.507 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:21:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:21:11.237Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:21:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:21:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:21:11] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T10:21:12.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:21:12 smithi153 ceph-mon[70044]: pgmap v8788: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:21:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:21:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:21:12.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:21:12 smithi083 ceph-mon[79930]: pgmap v8788: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:12.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:21:12 smithi083 ceph-mon[77242]: pgmap v8788: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:21:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:21:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:21:14.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:21:14 smithi083 ceph-mon[79930]: pgmap v8789: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:14.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:21:14 smithi083 ceph-mon[77242]: pgmap v8789: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:14.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:21:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:21:14.618Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:21:15.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:21:14 smithi153 ceph-mon[70044]: pgmap v8789: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:15.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:21:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:21:15.323Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:21:16.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:21:16 smithi083 ceph-mon[79930]: pgmap v8790: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:16.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:21:16 smithi083 ceph-mon[77242]: pgmap v8790: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:17.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:21:16 smithi153 ceph-mon[70044]: pgmap v8790: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:18.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:21:18 smithi083 ceph-mon[79930]: pgmap v8791: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:18.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:21:18 smithi083 ceph-mon[77242]: pgmap v8791: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:19.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:21:18 smithi153 ceph-mon[70044]: pgmap v8791: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:20.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:21:20 smithi083 ceph-mon[79930]: pgmap v8792: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:20.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:21:20 smithi083 ceph-mon[77242]: pgmap v8792: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:21.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:21:20 smithi153 ceph-mon[70044]: pgmap v8792: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:21.507 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:21:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:21:21.238Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:21:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:21:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:21:21] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T10:21:22.578 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:21:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:21:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:21:22.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:21:22 smithi083 ceph-mon[79930]: pgmap v8793: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:22.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:21:22 smithi083 ceph-mon[77242]: pgmap v8793: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:23.022 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:21:22 smithi153 ceph-mon[70044]: pgmap v8793: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:21:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:21:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:21:24.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:21:24 smithi083 ceph-mon[79930]: pgmap v8794: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:24.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:21:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:21:24.619Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:21:24.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:21:24 smithi083 ceph-mon[77242]: pgmap v8794: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:25.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:21:24 smithi153 ceph-mon[70044]: pgmap v8794: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:25.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:21:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:21:25.323Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:21:26.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:21:26 smithi083 ceph-mon[79930]: pgmap v8795: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:26.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:21:26 smithi083 ceph-mon[77242]: pgmap v8795: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:27.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:21:26 smithi153 ceph-mon[70044]: pgmap v8795: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:28.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:21:28 smithi083 ceph-mon[79930]: pgmap v8796: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:28.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:21:28 smithi083 ceph-mon[77242]: pgmap v8796: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:29.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:21:28 smithi153 ceph-mon[70044]: pgmap v8796: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:30.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:21:30 smithi083 ceph-mon[79930]: pgmap v8797: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:30.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:21:30 smithi083 ceph-mon[77242]: pgmap v8797: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:31.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:21:30 smithi153 ceph-mon[70044]: pgmap v8797: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:31.506 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:21:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:21:31.239Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:21:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:21:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:21:31] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T10:21:32.528 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:21:32.593 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:21:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:21:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:21:32.817 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:21:32.817 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 48s ago 4h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:21:32.817 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 9m ago 4h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:21:32.817 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 48s ago 4h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:21:32.818 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 9m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:21:32.818 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 9m ago 4h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:21:32.818 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 48s ago 5h 766M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:21:32.818 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 48s ago 5h 1128M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:21:32.819 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 9m ago 4h 470M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:21:32.819 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 48s ago 4h 647M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:21:32.819 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 48s ago 4h 26.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:21:32.819 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 9m ago 4h 25.8M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:21:32.820 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 48s ago 4h 793M 2483M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:21:32.820 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 48s ago 4h 783M 2483M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:21:32.820 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 48s ago 4h 612M 2483M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:21:32.820 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 48s ago 4h 764M 2483M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:21:32.820 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 9m ago 4h 653M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:21:32.821 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 9m ago 4h 812M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:21:32.821 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 9m ago 4h 725M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:21:32.821 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 9m ago 4h 857M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:21:32.821 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 9m ago 4h 181M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:21:32.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:21:32 smithi083 ceph-mon[79930]: pgmap v8798: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:32.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:21:32 smithi083 ceph-mon[77242]: pgmap v8798: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:33.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:21:32 smithi153 ceph-mon[70044]: pgmap v8798: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:33.170 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:21:33.171 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:21:33.171 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:21:33.171 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:21:33.171 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:21:33.172 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:21:33.172 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:21:33.172 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:21:33.172 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:21:33.173 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:21:33.173 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:21:33.173 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:21:33.173 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:21:33.173 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:21:33.174 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:21:33.174 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:21:33.174 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:21:33.174 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:21:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:21:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:21:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:21:33.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:21:33 smithi083 ceph-mon[79930]: from='client.54298 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:21:33.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:21:33 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/629399663' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:21:33.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:21:33 smithi083 ceph-mon[77242]: from='client.54298 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:21:33.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:21:33 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/629399663' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:21:34.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:21:33 smithi153 ceph-mon[70044]: from='client.54298 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:21:34.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:21:33 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/629399663' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:21:34.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:21:34 smithi083 ceph-mon[77242]: from='client.44373 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:21:34.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:21:34 smithi083 ceph-mon[77242]: pgmap v8799: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:34.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:21:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:21:34.619Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:21:34.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:21:34 smithi083 ceph-mon[79930]: from='client.44373 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:21:34.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:21:34 smithi083 ceph-mon[79930]: pgmap v8799: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:35.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:21:34 smithi153 ceph-mon[70044]: from='client.44373 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:21:35.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:21:34 smithi153 ceph-mon[70044]: pgmap v8799: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:35.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:21:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:21:35.324Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:21:36.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:21:36 smithi083 ceph-mon[77242]: pgmap v8800: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:36.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:21:36 smithi083 ceph-mon[79930]: pgmap v8800: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:37.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:21:36 smithi153 ceph-mon[70044]: pgmap v8800: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:38.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:21:38 smithi083 ceph-mon[79930]: pgmap v8801: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:38.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:21:38 smithi083 ceph-mon[77242]: pgmap v8801: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:39.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:21:38 smithi153 ceph-mon[70044]: pgmap v8801: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:40.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:21:40 smithi083 ceph-mon[79930]: pgmap v8802: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:40.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:21:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:21:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:21:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:21:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:21:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:21:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:21:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:21:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:21:40 smithi083 ceph-mon[77242]: pgmap v8802: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:21:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:21:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:21:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:21:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:21:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:21:40.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:21:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:21:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:21:40 smithi153 ceph-mon[70044]: pgmap v8802: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:21:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:21:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:21:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:21:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:21:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:21:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:21:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:21:41.507 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:21:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:21:41.240Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:21:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:21:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:21:41] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T10:21:42.616 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:21:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:21:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:21:42.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:21:42 smithi083 ceph-mon[79930]: pgmap v8803: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:42.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:21:42 smithi083 ceph-mon[77242]: pgmap v8803: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:43.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:21:42 smithi153 ceph-mon[70044]: pgmap v8803: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:21:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:21:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:21:44.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:21:44 smithi083 ceph-mon[79930]: pgmap v8804: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:44.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:21:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:21:44.621Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:21:44.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:21:44 smithi083 ceph-mon[77242]: pgmap v8804: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:45.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:21:44 smithi153 ceph-mon[70044]: pgmap v8804: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:45.626 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:21:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:21:45.324Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:21:45.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:21:45 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:21:45.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:21:45 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:21:45.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:21:45 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:21:45.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:21:45 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:21:45.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:21:45 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:21:45.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:21:45 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:21:45.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:21:45 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:21:45.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:21:45 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:21:46.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:21:45 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:21:46.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:21:45 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:21:46.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:21:45 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:21:46.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:21:45 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:21:46.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:21:46 smithi083 ceph-mon[79930]: pgmap v8805: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:46.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:21:46 smithi083 ceph-mon[77242]: pgmap v8805: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:47.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:21:46 smithi153 ceph-mon[70044]: pgmap v8805: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:47.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:21:47 smithi083 ceph-mon[79930]: pgmap v8806: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:47.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:21:47 smithi083 ceph-mon[77242]: pgmap v8806: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:48.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:21:47 smithi153 ceph-mon[70044]: pgmap v8806: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:50.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:21:50 smithi153 ceph-mon[70044]: pgmap v8807: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:50.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:21:50 smithi083 ceph-mon[79930]: pgmap v8807: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:50.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:21:50 smithi083 ceph-mon[77242]: pgmap v8807: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:51.507 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:21:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:21:51.241Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:21:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:21:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:21:51] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T10:21:52.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:21:52 smithi153 ceph-mon[70044]: pgmap v8808: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:21:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:21:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:21:52.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:21:52 smithi083 ceph-mon[79930]: pgmap v8808: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:52.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:21:52 smithi083 ceph-mon[77242]: pgmap v8808: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:53.312 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:21:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:21:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:21:54.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:21:54 smithi153 ceph-mon[70044]: pgmap v8809: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:54.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:21:54 smithi083 ceph-mon[79930]: pgmap v8809: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:54.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:21:54 smithi083 ceph-mon[77242]: pgmap v8809: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:54.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:21:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:21:54.620Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:21:55.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:21:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:21:55.325Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:21:56.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:21:56 smithi153 ceph-mon[70044]: pgmap v8810: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:56.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:21:56 smithi083 ceph-mon[77242]: pgmap v8810: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:56.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:21:56 smithi083 ceph-mon[79930]: pgmap v8810: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:58.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:21:58 smithi153 ceph-mon[70044]: pgmap v8811: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:58.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:21:58 smithi083 ceph-mon[79930]: pgmap v8811: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:21:58.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:21:58 smithi083 ceph-mon[77242]: pgmap v8811: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:00.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:22:00 smithi153 ceph-mon[70044]: pgmap v8812: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:00.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:22:00 smithi083 ceph-mon[79930]: pgmap v8812: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:00.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:22:00 smithi083 ceph-mon[77242]: pgmap v8812: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:01.507 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:22:01.241Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:22:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:22:01] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T10:22:02.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:22:02 smithi153 ceph-mon[70044]: pgmap v8813: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:22:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:22:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:22:02.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:22:02 smithi083 ceph-mon[79930]: pgmap v8813: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:02.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:22:02 smithi083 ceph-mon[77242]: pgmap v8813: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:22:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:22:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:22:03.504 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:22:03.804 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:22:03.804 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 79s ago 4h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:22:03.804 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 9m ago 4h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:22:03.804 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 79s ago 4h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:22:03.805 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 9m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:22:03.805 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 9m ago 4h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:22:03.805 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 79s ago 5h 766M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:22:03.805 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 79s ago 5h 1128M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:22:03.806 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 9m ago 4h 470M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:22:03.806 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 79s ago 5h 647M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:22:03.806 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 79s ago 4h 26.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:22:03.806 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 9m ago 4h 25.8M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:22:03.806 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 79s ago 4h 793M 2483M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:22:03.807 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 79s ago 4h 783M 2483M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:22:03.807 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 79s ago 4h 612M 2483M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:22:03.807 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 79s ago 4h 764M 2483M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:22:03.807 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 9m ago 4h 653M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:22:03.807 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 9m ago 4h 812M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:22:03.808 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 9m ago 4h 725M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:22:03.808 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 9m ago 4h 857M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:22:03.808 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 9m ago 4h 181M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:22:04.158 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:22:04.159 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:22:04.159 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:22:04.159 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:22:04.160 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:22:04.160 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:22:04.160 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:22:04.160 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:22:04.160 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:22:04.161 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:22:04.161 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:22:04.161 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:22:04.161 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:22:04.162 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:22:04.162 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:22:04.162 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:22:04.162 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:22:04.163 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:22:04.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:22:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:22:04.620Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:22:04.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:22:04 smithi083 ceph-mon[77242]: pgmap v8814: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:04.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:22:04 smithi083 ceph-mon[77242]: from='client.54316 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:22:04.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:22:04 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4048552057' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:22:04.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:22:04 smithi083 ceph-mon[79930]: pgmap v8814: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:04.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:22:04 smithi083 ceph-mon[79930]: from='client.54316 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:22:04.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:22:04 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4048552057' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:22:05.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:22:04 smithi153 ceph-mon[70044]: pgmap v8814: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:05.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:22:04 smithi153 ceph-mon[70044]: from='client.54316 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:22:05.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:22:04 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4048552057' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:22:05.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:22:05 smithi083 ceph-mon[79930]: from='client.44391 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:22:05.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:22:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:22:05.326Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:22:05.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:22:05 smithi083 ceph-mon[77242]: from='client.44391 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:22:06.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:22:05 smithi153 ceph-mon[70044]: from='client.44391 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:22:06.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:22:06 smithi083 ceph-mon[79930]: pgmap v8815: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:06.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:22:06 smithi083 ceph-mon[77242]: pgmap v8815: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:07.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:22:06 smithi153 ceph-mon[70044]: pgmap v8815: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:08.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:22:08 smithi083 ceph-mon[79930]: pgmap v8816: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:08.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:22:08 smithi083 ceph-mon[77242]: pgmap v8816: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:09.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:22:08 smithi153 ceph-mon[70044]: pgmap v8816: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:10.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:22:10 smithi083 ceph-mon[79930]: pgmap v8817: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:10.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:22:10 smithi083 ceph-mon[77242]: pgmap v8817: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:11.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:22:10 smithi153 ceph-mon[70044]: pgmap v8817: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:11.507 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:22:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:22:11.242Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:22:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:22:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:22:11] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T10:22:12.578 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:22:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:22:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:22:12.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:22:12 smithi083 ceph-mon[77242]: pgmap v8818: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:12.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:22:12 smithi083 ceph-mon[79930]: pgmap v8818: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:13.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:22:12 smithi153 ceph-mon[70044]: pgmap v8818: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:22:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:22:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:22:14.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:22:14 smithi083 ceph-mon[79930]: pgmap v8819: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:14.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:22:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:22:14.621Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:22:14.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:22:14 smithi083 ceph-mon[77242]: pgmap v8819: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:15.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:22:14 smithi153 ceph-mon[70044]: pgmap v8819: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:15.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:22:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:22:15.327Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:22:16.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:22:16 smithi083 ceph-mon[79930]: pgmap v8820: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:16.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:22:16 smithi083 ceph-mon[77242]: pgmap v8820: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:17.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:22:16 smithi153 ceph-mon[70044]: pgmap v8820: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:18.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:22:18 smithi083 ceph-mon[79930]: pgmap v8821: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:18.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:22:18 smithi083 ceph-mon[77242]: pgmap v8821: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:19.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:22:18 smithi153 ceph-mon[70044]: pgmap v8821: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:20.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:22:20 smithi083 ceph-mon[79930]: pgmap v8822: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:20.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:22:20 smithi083 ceph-mon[77242]: pgmap v8822: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:21.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:22:20 smithi153 ceph-mon[70044]: pgmap v8822: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:21.506 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:22:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:22:21.243Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:22:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:22:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:22:21] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T10:22:22.596 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:22:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:22:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:22:22.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:22:22 smithi083 ceph-mon[77242]: pgmap v8823: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:22.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:22:22 smithi083 ceph-mon[79930]: pgmap v8823: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:23.022 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:22:22 smithi153 ceph-mon[70044]: pgmap v8823: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:22:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:22:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:22:24.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:22:24 smithi083 ceph-mon[77242]: pgmap v8824: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:24.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:22:24 smithi083 ceph-mon[79930]: pgmap v8824: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:24.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:22:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:22:24.622Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:22:25.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:22:24 smithi153 ceph-mon[70044]: pgmap v8824: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:25.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:22:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:22:25.328Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:22:26.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:22:26 smithi083 ceph-mon[77242]: pgmap v8825: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:26.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:22:26 smithi083 ceph-mon[79930]: pgmap v8825: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:27.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:22:26 smithi153 ceph-mon[70044]: pgmap v8825: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:28.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:22:28 smithi083 ceph-mon[77242]: pgmap v8826: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:28.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:22:28 smithi083 ceph-mon[79930]: pgmap v8826: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:29.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:22:28 smithi153 ceph-mon[70044]: pgmap v8826: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:30.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:22:30 smithi083 ceph-mon[77242]: pgmap v8827: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:30.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:22:30 smithi083 ceph-mon[79930]: pgmap v8827: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:31.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:22:30 smithi153 ceph-mon[70044]: pgmap v8827: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:31.506 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:22:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:22:31.243Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:22:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:22:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:22:31] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T10:22:32.610 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:22:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:22:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:22:32.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:22:32 smithi083 ceph-mon[77242]: pgmap v8828: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:32.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:22:32 smithi083 ceph-mon[79930]: pgmap v8828: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:33.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:22:32 smithi153 ceph-mon[70044]: pgmap v8828: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:22:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:22:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:22:34.484 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:22:34.774 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:22:34.774 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 110s ago 4h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:22:34.774 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 10m ago 4h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:22:34.775 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 110s ago 4h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:22:34.775 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 10m ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:22:34.775 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 10m ago 5h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:22:34.775 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 110s ago 5h 766M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:22:34.776 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 110s ago 5h 1128M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:22:34.776 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 10m ago 5h 470M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:22:34.776 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 110s ago 5h 647M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:22:34.776 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 110s ago 4h 26.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:22:34.776 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 10m ago 4h 25.8M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:22:34.777 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 110s ago 4h 793M 2483M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:22:34.777 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 110s ago 4h 783M 2483M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:22:34.777 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 110s ago 4h 612M 2483M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:22:34.777 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 110s ago 4h 764M 2483M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:22:34.778 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 10m ago 4h 653M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:22:34.778 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 10m ago 4h 812M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:22:34.778 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 10m ago 4h 725M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:22:34.778 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 10m ago 4h 857M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:22:34.779 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 10m ago 4h 181M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:22:34.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:22:34 smithi083 ceph-mon[77242]: pgmap v8829: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:34.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:22:34 smithi083 ceph-mon[79930]: pgmap v8829: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:34.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:22:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:22:34.622Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:22:35.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:22:34 smithi153 ceph-mon[70044]: pgmap v8829: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:35.130 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:22:35.131 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:22:35.131 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:22:35.131 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:22:35.131 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:22:35.132 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:22:35.132 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:22:35.132 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:22:35.132 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:22:35.133 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:22:35.133 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:22:35.133 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:22:35.133 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:22:35.134 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:22:35.134 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:22:35.134 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:22:35.134 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:22:35.134 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:22:35.617 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:22:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:22:35.328Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:22:35.617 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:22:35 smithi083 ceph-mon[77242]: from='client.44403 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:22:35.618 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:22:35 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1245526069' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:22:35.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:22:35 smithi083 ceph-mon[79930]: from='client.44403 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:22:35.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:22:35 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1245526069' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:22:36.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:22:35 smithi153 ceph-mon[70044]: from='client.44403 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:22:36.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:22:35 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1245526069' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:22:36.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:22:36 smithi083 ceph-mon[77242]: from='client.44409 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:22:36.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:22:36 smithi083 ceph-mon[77242]: pgmap v8830: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:36.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:22:36 smithi083 ceph-mon[79930]: from='client.44409 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:22:36.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:22:36 smithi083 ceph-mon[79930]: pgmap v8830: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:37.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:22:36 smithi153 ceph-mon[70044]: from='client.44409 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:22:37.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:22:36 smithi153 ceph-mon[70044]: pgmap v8830: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:38.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:22:38 smithi083 ceph-mon[79930]: pgmap v8831: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:38.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:22:38 smithi083 ceph-mon[77242]: pgmap v8831: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:39.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:22:38 smithi153 ceph-mon[70044]: pgmap v8831: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:40.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:22:40 smithi083 ceph-mon[77242]: pgmap v8832: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:40.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:22:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:22:40.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:22:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:22:40.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:22:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:22:40.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:22:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:22:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:22:40 smithi083 ceph-mon[79930]: pgmap v8832: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:22:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:22:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:22:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:22:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:22:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:22:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:22:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:22:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:22:40 smithi153 ceph-mon[70044]: pgmap v8832: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:22:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:22:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:22:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:22:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:22:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:22:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:22:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:22:41.507 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:22:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:22:41.244Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:22:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:22:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:22:41] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T10:22:42.628 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:22:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:22:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:22:42.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:22:42 smithi083 ceph-mon[79930]: pgmap v8833: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:42.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:22:42 smithi083 ceph-mon[77242]: pgmap v8833: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:43.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:22:42 smithi153 ceph-mon[70044]: pgmap v8833: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:22:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:22:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:22:44.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:22:44 smithi083 ceph-mon[77242]: pgmap v8834: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:44.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:22:44 smithi083 ceph-mon[79930]: pgmap v8834: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:44.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:22:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:22:44.623Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:22:45.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:22:44 smithi153 ceph-mon[70044]: pgmap v8834: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:45.634 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:22:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:22:45.330Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:22:45.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:22:45 smithi083 ceph-mon[79930]: pgmap v8835: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:45.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:22:45 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:22:45.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:22:45 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:22:45.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:22:45 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:22:45.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:22:45 smithi083 ceph-mon[77242]: pgmap v8835: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:45.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:22:45 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:22:45.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:22:45 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:22:45.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:22:45 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:22:46.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:22:45 smithi153 ceph-mon[70044]: pgmap v8835: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:46.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:22:45 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:22:46.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:22:45 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:22:46.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:22:45 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:22:48.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:22:48 smithi083 ceph-mon[79930]: pgmap v8836: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:48.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:22:48 smithi083 ceph-mon[77242]: pgmap v8836: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:48.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:22:48 smithi153 ceph-mon[70044]: pgmap v8836: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:50.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:22:50 smithi083 ceph-mon[79930]: pgmap v8837: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:50.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:22:50 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:22:50.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:22:50 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:22:50.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:22:50 smithi083 ceph-mon[77242]: pgmap v8837: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:50.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:22:50 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:22:50.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:22:50 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:22:51.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:22:50 smithi153 ceph-mon[70044]: pgmap v8837: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:51.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:22:50 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:22:51.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:22:50 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:22:51.507 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:22:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:22:51.244Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:22:51.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:22:51 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:22:51.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:22:51 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:22:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:22:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:22:51] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T10:22:52.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:22:51 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:22:52.627 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:22:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:22:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:22:52.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:22:52 smithi083 ceph-mon[79930]: pgmap v8838: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:52.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:22:52 smithi083 ceph-mon[77242]: pgmap v8838: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:53.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:22:52 smithi153 ceph-mon[70044]: pgmap v8838: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:22:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:22:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:22:54.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:22:54 smithi083 ceph-mon[79930]: pgmap v8839: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:54.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:22:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:22:54.624Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:22:54.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:22:54 smithi083 ceph-mon[77242]: pgmap v8839: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:55.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:22:54 smithi153 ceph-mon[70044]: pgmap v8839: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:55.684 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:22:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:22:55.330Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:22:55.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:22:55 smithi083 ceph-mon[77242]: pgmap v8840: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:55.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:22:55 smithi083 ceph-mon[79930]: pgmap v8840: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:56.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:22:55 smithi153 ceph-mon[70044]: pgmap v8840: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:58.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:22:58 smithi083 ceph-mon[79930]: pgmap v8841: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:58.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:22:58 smithi083 ceph-mon[77242]: pgmap v8841: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:22:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:22:58 smithi153 ceph-mon[70044]: pgmap v8841: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:00.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:23:00 smithi153 ceph-mon[70044]: pgmap v8842: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:00.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:23:00 smithi083 ceph-mon[79930]: pgmap v8842: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:00.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:23:00 smithi083 ceph-mon[77242]: pgmap v8842: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:01.507 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:23:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:23:01.245Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:23:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:23:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:23:01] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T10:23:02.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:23:02 smithi153 ceph-mon[70044]: pgmap v8843: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:23:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:23:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:23:02.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:23:02 smithi083 ceph-mon[77242]: pgmap v8843: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:02.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:23:02 smithi083 ceph-mon[79930]: pgmap v8843: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:23:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:23:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:23:04.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:23:04 smithi083 ceph-mon[77242]: pgmap v8844: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:04.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:23:04 smithi083 ceph-mon[79930]: pgmap v8844: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:04.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:23:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:23:04.624Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:23:05.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:23:04 smithi153 ceph-mon[70044]: pgmap v8844: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:05.463 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:23:05.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:23:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:23:05.331Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:23:05.749 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:23:05.749 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 2m ago 4h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:23:05.750 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 15s ago 4h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:23:05.750 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 2m ago 4h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:23:05.750 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 15s ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:23:05.750 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 15s ago 5h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:23:05.751 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 2m ago 5h 766M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:23:05.751 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 2m ago 5h 1128M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:23:05.751 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 15s ago 5h 485M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:23:05.751 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 2m ago 5h 647M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:23:05.752 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 2m ago 4h 26.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:23:05.752 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 15s ago 4h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:23:05.752 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 2m ago 5h 793M 2483M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:23:05.752 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 2m ago 5h 783M 2483M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:23:05.752 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 2m ago 4h 612M 2483M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:23:05.753 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 2m ago 4h 764M 2483M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:23:05.753 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 15s ago 4h 675M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:23:05.753 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 15s ago 4h 839M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:23:05.753 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 15s ago 4h 749M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:23:05.754 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 15s ago 4h 886M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:23:05.754 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 15s ago 4h 184M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:23:06.105 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:23:06.105 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:23:06.105 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:23:06.106 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:23:06.106 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:23:06.106 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:23:06.106 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:23:06.107 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:23:06.107 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:23:06.107 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:23:06.107 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:23:06.107 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:23:06.108 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:23:06.108 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:23:06.108 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:23:06.108 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:23:06.109 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:23:06.109 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:23:06.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:23:06 smithi083 ceph-mon[77242]: pgmap v8845: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:06.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:23:06 smithi083 ceph-mon[77242]: from='client.54352 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:23:06.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:23:06 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3690426066' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:23:06.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:23:06 smithi083 ceph-mon[79930]: pgmap v8845: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:06.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:23:06 smithi083 ceph-mon[79930]: from='client.54352 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:23:06.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:23:06 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3690426066' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:23:07.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:23:06 smithi153 ceph-mon[70044]: pgmap v8845: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:07.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:23:06 smithi153 ceph-mon[70044]: from='client.54352 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:23:07.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:23:06 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3690426066' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:23:07.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:23:07 smithi083 ceph-mon[79930]: from='client.44427 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:23:07.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:23:07 smithi083 ceph-mon[77242]: from='client.44427 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:23:08.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:23:07 smithi153 ceph-mon[70044]: from='client.44427 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:23:08.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:23:08 smithi083 ceph-mon[79930]: pgmap v8846: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:08.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:23:08 smithi083 ceph-mon[77242]: pgmap v8846: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:09.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:23:08 smithi153 ceph-mon[70044]: pgmap v8846: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:10.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:23:10 smithi083 ceph-mon[79930]: pgmap v8847: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:10.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:23:10 smithi083 ceph-mon[77242]: pgmap v8847: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:11.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:23:10 smithi153 ceph-mon[70044]: pgmap v8847: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:11.507 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:23:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:23:11.245Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:23:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:23:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:23:11] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T10:23:12.577 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:23:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:23:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:23:12.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:23:12 smithi083 ceph-mon[77242]: pgmap v8848: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:12.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:23:12 smithi083 ceph-mon[79930]: pgmap v8848: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:13.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:23:12 smithi153 ceph-mon[70044]: pgmap v8848: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:23:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:23:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:23:14.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:23:14 smithi083 ceph-mon[79930]: pgmap v8849: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:14.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:23:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:23:14.624Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:23:14.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:23:14 smithi083 ceph-mon[77242]: pgmap v8849: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:15.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:23:14 smithi153 ceph-mon[70044]: pgmap v8849: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:15.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:23:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:23:15.332Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:23:16.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:23:16 smithi083 ceph-mon[79930]: pgmap v8850: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:16.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:23:16 smithi083 ceph-mon[77242]: pgmap v8850: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:17.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:23:16 smithi153 ceph-mon[70044]: pgmap v8850: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:18.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:23:18 smithi083 ceph-mon[79930]: pgmap v8851: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:18.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:23:18 smithi083 ceph-mon[77242]: pgmap v8851: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:19.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:23:18 smithi153 ceph-mon[70044]: pgmap v8851: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:20.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:23:20 smithi083 ceph-mon[79930]: pgmap v8852: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:20.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:23:20 smithi083 ceph-mon[77242]: pgmap v8852: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:21.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:23:20 smithi153 ceph-mon[70044]: pgmap v8852: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:21.506 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:23:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:23:21.246Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:23:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:23:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:23:21] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T10:23:22.593 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:23:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:23:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:23:22.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:23:22 smithi083 ceph-mon[77242]: pgmap v8853: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:22.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:23:22 smithi083 ceph-mon[79930]: pgmap v8853: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:23.022 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:23:22 smithi153 ceph-mon[70044]: pgmap v8853: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:23:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:23:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:23:24.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:23:24 smithi083 ceph-mon[79930]: pgmap v8854: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:24.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:23:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:23:24.625Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:23:24.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:23:24 smithi083 ceph-mon[77242]: pgmap v8854: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:25.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:23:24 smithi153 ceph-mon[70044]: pgmap v8854: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:25.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:23:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:23:25.332Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:23:26.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:23:26 smithi083 ceph-mon[79930]: pgmap v8855: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:26.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:23:26 smithi083 ceph-mon[77242]: pgmap v8855: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:27.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:23:26 smithi153 ceph-mon[70044]: pgmap v8855: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:28.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:23:28 smithi083 ceph-mon[79930]: pgmap v8856: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:28.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:23:28 smithi083 ceph-mon[77242]: pgmap v8856: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:29.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:23:28 smithi153 ceph-mon[70044]: pgmap v8856: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:30.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:23:30 smithi083 ceph-mon[79930]: pgmap v8857: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:30.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:23:30 smithi083 ceph-mon[77242]: pgmap v8857: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:31.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:23:30 smithi153 ceph-mon[70044]: pgmap v8857: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:31.507 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:23:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:23:31.247Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:23:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:23:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:23:31] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T10:23:32.610 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:23:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:23:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:23:32.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:23:32 smithi083 ceph-mon[77242]: pgmap v8858: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:32.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:23:32 smithi083 ceph-mon[79930]: pgmap v8858: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:33.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:23:32 smithi153 ceph-mon[70044]: pgmap v8858: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:23:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:23:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:23:34.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:23:34 smithi083 ceph-mon[79930]: pgmap v8859: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:34.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:23:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:23:34.625Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:23:34.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:23:34 smithi083 ceph-mon[77242]: pgmap v8859: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:35.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:23:34 smithi153 ceph-mon[70044]: pgmap v8859: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:35.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:23:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:23:35.332Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:23:36.429 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:23:36.719 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:23:36.720 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 2m ago 4h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:23:36.720 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 46s ago 4h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:23:36.720 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 2m ago 4h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:23:36.721 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 46s ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:23:36.721 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 46s ago 5h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:23:36.721 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 2m ago 5h 766M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:23:36.721 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 2m ago 5h 1128M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:23:36.721 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 46s ago 5h 485M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:23:36.722 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 2m ago 5h 647M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:23:36.722 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 2m ago 4h 26.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:23:36.722 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 46s ago 4h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:23:36.722 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 2m ago 5h 793M 2483M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:23:36.722 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 2m ago 5h 783M 2483M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:23:36.723 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 2m ago 5h 612M 2483M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:23:36.723 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 2m ago 5h 764M 2483M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:23:36.723 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 46s ago 4h 675M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:23:36.723 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 46s ago 4h 839M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:23:36.724 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 46s ago 4h 749M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:23:36.724 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 46s ago 4h 886M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:23:36.724 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 46s ago 4h 184M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:23:36.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:23:36 smithi083 ceph-mon[79930]: pgmap v8860: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:36.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:23:36 smithi083 ceph-mon[77242]: pgmap v8860: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:37.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:23:36 smithi153 ceph-mon[70044]: pgmap v8860: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:37.077 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:23:37.078 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:23:37.078 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:23:37.078 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:23:37.078 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:23:37.079 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:23:37.079 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:23:37.079 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:23:37.079 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:23:37.079 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:23:37.080 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:23:37.080 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:23:37.080 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:23:37.080 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:23:37.081 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:23:37.081 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:23:37.081 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:23:37.081 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:23:37.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:23:37 smithi083 ceph-mon[79930]: from='client.54370 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:23:37.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:23:37 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/398261476' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:23:37.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:23:37 smithi083 ceph-mon[77242]: from='client.54370 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:23:37.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:23:37 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/398261476' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:23:38.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:23:37 smithi153 ceph-mon[70044]: from='client.54370 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:23:38.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:23:37 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/398261476' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:23:38.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:23:38 smithi083 ceph-mon[79930]: from='client.54376 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:23:38.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:23:38 smithi083 ceph-mon[79930]: pgmap v8861: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:38.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:23:38 smithi083 ceph-mon[77242]: from='client.54376 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:23:38.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:23:38 smithi083 ceph-mon[77242]: pgmap v8861: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:39.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:23:38 smithi153 ceph-mon[70044]: from='client.54376 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:23:39.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:23:38 smithi153 ceph-mon[70044]: pgmap v8861: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:40.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:23:40 smithi083 ceph-mon[79930]: pgmap v8862: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:23:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:23:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:23:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:23:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:23:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:23:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:23:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:23:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:23:40 smithi083 ceph-mon[77242]: pgmap v8862: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:23:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:23:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:23:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:23:40.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:23:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:23:40.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:23:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:23:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:23:40 smithi153 ceph-mon[70044]: pgmap v8862: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:23:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:23:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:23:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:23:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:23:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:23:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:23:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:23:41.506 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:23:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:23:41.248Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:23:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:23:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:23:41] "GET /metrics HTTP/1.1" 200 225227 "" "Prometheus/2.33.4" 2022-09-29T10:23:42.629 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:23:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:23:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:23:42.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:23:42 smithi083 ceph-mon[77242]: pgmap v8863: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:42.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:23:42 smithi083 ceph-mon[79930]: pgmap v8863: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:43.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:23:42 smithi153 ceph-mon[70044]: pgmap v8863: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:23:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:23:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:23:44.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:23:44 smithi083 ceph-mon[79930]: pgmap v8864: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:44.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:23:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:23:44.626Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:23:44.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:23:44 smithi083 ceph-mon[77242]: pgmap v8864: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:45.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:23:44 smithi153 ceph-mon[70044]: pgmap v8864: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:45.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:23:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:23:45.333Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:23:46.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:23:46 smithi083 ceph-mon[77242]: pgmap v8865: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:46.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:23:46 smithi083 ceph-mon[79930]: pgmap v8865: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:47.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:23:46 smithi153 ceph-mon[70044]: pgmap v8865: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:48.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:23:48 smithi083 ceph-mon[79930]: pgmap v8866: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:48.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:23:48 smithi083 ceph-mon[77242]: pgmap v8866: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:49.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:23:48 smithi153 ceph-mon[70044]: pgmap v8866: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:50.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:23:50 smithi083 ceph-mon[79930]: pgmap v8867: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:50.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:23:50 smithi083 ceph-mon[77242]: pgmap v8867: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:51.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:23:50 smithi153 ceph-mon[70044]: pgmap v8867: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:51.645 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:23:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:23:51] "GET /metrics HTTP/1.1" 200 225227 "" "Prometheus/2.33.4" 2022-09-29T10:23:51.646 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:23:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:23:51.257Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:23:51.646 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:23:51 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:23:51.647 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:23:51 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:23:51.647 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:23:51 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:23:51.647 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:23:51 smithi083 ceph-mon[77242]: pgmap v8868: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:51.647 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:23:51 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:23:51.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:23:51 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:23:51.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:23:51 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:23:51.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:23:51 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:23:51.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:23:51 smithi083 ceph-mon[79930]: pgmap v8868: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:51.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:23:51 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:23:52.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:23:51 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:23:52.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:23:51 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:23:52.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:23:51 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:23:52.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:23:51 smithi153 ceph-mon[70044]: pgmap v8868: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:52.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:23:51 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:23:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:23:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:23:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:23:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:23:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:23:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:23:54.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:23:54 smithi083 ceph-mon[79930]: pgmap v8869: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:54.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:23:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:23:54.627Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:23:54.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:23:54 smithi083 ceph-mon[77242]: pgmap v8869: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:55.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:23:54 smithi153 ceph-mon[70044]: pgmap v8869: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:55.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:23:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:23:55.334Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:23:56.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:23:56 smithi083 ceph-mon[79930]: pgmap v8870: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:56.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:23:56 smithi083 ceph-mon[77242]: pgmap v8870: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:57.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:23:56 smithi153 ceph-mon[70044]: pgmap v8870: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:58.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:23:58 smithi083 ceph-mon[79930]: pgmap v8871: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:58.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:23:58 smithi083 ceph-mon[77242]: pgmap v8871: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:23:59.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:23:58 smithi153 ceph-mon[70044]: pgmap v8871: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:00.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:24:00 smithi083 ceph-mon[79930]: pgmap v8872: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:00.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:24:00 smithi083 ceph-mon[77242]: pgmap v8872: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:01.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:24:00 smithi153 ceph-mon[70044]: pgmap v8872: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:01.506 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:24:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:24:01.249Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:24:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:24:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:24:01] "GET /metrics HTTP/1.1" 200 225227 "" "Prometheus/2.33.4" 2022-09-29T10:24:02.574 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:24:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:24:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:24:02.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:24:02 smithi083 ceph-mon[77242]: pgmap v8873: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:02.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:24:02 smithi083 ceph-mon[79930]: pgmap v8873: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:03.022 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:24:02 smithi153 ceph-mon[70044]: pgmap v8873: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:24:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:24:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:24:04.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:24:04 smithi083 ceph-mon[77242]: pgmap v8874: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:04.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:24:04 smithi083 ceph-mon[79930]: pgmap v8874: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:04.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:24:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:24:04.628Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:24:05.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:24:04 smithi153 ceph-mon[70044]: pgmap v8874: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:05.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:24:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:24:05.334Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:24:06.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:24:06 smithi083 ceph-mon[77242]: pgmap v8875: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:06.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:24:06 smithi083 ceph-mon[79930]: pgmap v8875: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:07.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:24:06 smithi153 ceph-mon[70044]: pgmap v8875: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:07.403 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:24:07.691 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:24:07.692 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 3m ago 4h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:24:07.692 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 77s ago 4h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:24:07.692 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 3m ago 4h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:24:07.693 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 77s ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:24:07.693 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 77s ago 5h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:24:07.693 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 3m ago 5h 766M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:24:07.693 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 3m ago 5h 1128M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:24:07.694 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 77s ago 5h 485M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:24:07.694 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 3m ago 5h 647M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:24:07.694 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 3m ago 4h 26.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:24:07.694 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 77s ago 4h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:24:07.694 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 3m ago 5h 793M 2483M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:24:07.695 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 3m ago 5h 783M 2483M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:24:07.695 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 3m ago 5h 612M 2483M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:24:07.695 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 3m ago 5h 764M 2483M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:24:07.695 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 77s ago 5h 675M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:24:07.696 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 77s ago 5h 839M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:24:07.696 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 77s ago 4h 749M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:24:07.696 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 77s ago 4h 886M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:24:07.696 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 77s ago 4h 184M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:24:08.052 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:24:08.052 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:24:08.053 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:24:08.053 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:24:08.053 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:24:08.053 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:24:08.054 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:24:08.054 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:24:08.054 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:24:08.054 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:24:08.055 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:24:08.055 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:24:08.055 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:24:08.055 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:24:08.056 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:24:08.056 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:24:08.056 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:24:08.056 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:24:08.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:24:08 smithi083 ceph-mon[77242]: pgmap v8876: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:08.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:24:08 smithi083 ceph-mon[77242]: from='client.54388 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:24:08.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:24:08 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1471111867' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:24:08.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:24:08 smithi083 ceph-mon[79930]: pgmap v8876: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:08.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:24:08 smithi083 ceph-mon[79930]: from='client.54388 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:24:08.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:24:08 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1471111867' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:24:09.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:24:08 smithi153 ceph-mon[70044]: pgmap v8876: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:09.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:24:08 smithi153 ceph-mon[70044]: from='client.54388 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:24:09.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:24:08 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1471111867' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:24:09.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:24:09 smithi083 ceph-mon[77242]: from='client.54394 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:24:09.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:24:09 smithi083 ceph-mon[79930]: from='client.54394 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:24:10.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:24:09 smithi153 ceph-mon[70044]: from='client.54394 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:24:10.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:24:10 smithi083 ceph-mon[79930]: pgmap v8877: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:10.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:24:10 smithi083 ceph-mon[77242]: pgmap v8877: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:11.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:24:10 smithi153 ceph-mon[70044]: pgmap v8877: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:11.507 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:24:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:24:11.250Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:24:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:24:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:24:11] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T10:24:12.593 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:24:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:24:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:24:12.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:24:12 smithi083 ceph-mon[79930]: pgmap v8878: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:12.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:24:12 smithi083 ceph-mon[77242]: pgmap v8878: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:13.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:24:12 smithi153 ceph-mon[70044]: pgmap v8878: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:24:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:24:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:24:14.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:24:14 smithi083 ceph-mon[79930]: pgmap v8879: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:14.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:24:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:24:14.629Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:24:14.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:24:14 smithi083 ceph-mon[77242]: pgmap v8879: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:15.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:24:14 smithi153 ceph-mon[70044]: pgmap v8879: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:15.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:24:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:24:15.335Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:24:16.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:24:16 smithi083 ceph-mon[77242]: pgmap v8880: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:16.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:24:16 smithi083 ceph-mon[79930]: pgmap v8880: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:17.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:24:16 smithi153 ceph-mon[70044]: pgmap v8880: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:18.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:24:18 smithi083 ceph-mon[79930]: pgmap v8881: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:18.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:24:18 smithi083 ceph-mon[77242]: pgmap v8881: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:19.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:24:18 smithi153 ceph-mon[70044]: pgmap v8881: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:20.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:24:20 smithi083 ceph-mon[77242]: pgmap v8882: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:20.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:24:20 smithi083 ceph-mon[79930]: pgmap v8882: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:21.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:24:20 smithi153 ceph-mon[70044]: pgmap v8882: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:21.507 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:24:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:24:21.250Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:24:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:24:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:24:21] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T10:24:22.606 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:24:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:24:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:24:22.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:24:22 smithi083 ceph-mon[77242]: pgmap v8883: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:22.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:24:22 smithi083 ceph-mon[79930]: pgmap v8883: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:23.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:24:22 smithi153 ceph-mon[70044]: pgmap v8883: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:24:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:24:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:24:24.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:24:24 smithi083 ceph-mon[77242]: pgmap v8884: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:24.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:24:24 smithi083 ceph-mon[79930]: pgmap v8884: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:24.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:24:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:24:24.629Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:24:25.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:24:24 smithi153 ceph-mon[70044]: pgmap v8884: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:25.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:24:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:24:25.335Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:24:26.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:24:26 smithi083 ceph-mon[77242]: pgmap v8885: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:26.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:24:26 smithi083 ceph-mon[79930]: pgmap v8885: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:27.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:24:26 smithi153 ceph-mon[70044]: pgmap v8885: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:28.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:24:28 smithi083 ceph-mon[77242]: pgmap v8886: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:28.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:24:28 smithi083 ceph-mon[79930]: pgmap v8886: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:29.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:24:28 smithi153 ceph-mon[70044]: pgmap v8886: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:30.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:24:30 smithi083 ceph-mon[79930]: pgmap v8887: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:30.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:24:30 smithi083 ceph-mon[77242]: pgmap v8887: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:31.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:24:30 smithi153 ceph-mon[70044]: pgmap v8887: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:31.507 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:24:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:24:31.251Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:24:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:24:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:24:31] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T10:24:32.622 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:24:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:24:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:24:32.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:24:32 smithi083 ceph-mon[79930]: pgmap v8888: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:32.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:24:32 smithi083 ceph-mon[77242]: pgmap v8888: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:33.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:24:32 smithi153 ceph-mon[70044]: pgmap v8888: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:24:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:24:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:24:34.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:24:34 smithi083 ceph-mon[77242]: pgmap v8889: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:34.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:24:34 smithi083 ceph-mon[79930]: pgmap v8889: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:34.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:24:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:24:34.630Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:24:35.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:24:34 smithi153 ceph-mon[70044]: pgmap v8889: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:35.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:24:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:24:35.336Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:24:36.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:24:36 smithi083 ceph-mon[77242]: pgmap v8890: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:36.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:24:36 smithi083 ceph-mon[79930]: pgmap v8890: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:37.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:24:36 smithi153 ceph-mon[70044]: pgmap v8890: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:38.379 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:24:38.684 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:24:38.684 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 3m ago 4h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:24:38.684 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 108s ago 4h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:24:38.685 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (4h) 3m ago 4h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:24:38.685 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (4h) 108s ago 4h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:24:38.685 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 108s ago 5h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:24:38.685 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 3m ago 5h 766M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:24:38.685 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 3m ago 5h 1128M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:24:38.686 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 108s ago 5h 485M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:24:38.686 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 3m ago 5h 647M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:24:38.686 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 3m ago 4h 26.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:24:38.687 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 108s ago 4h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:24:38.687 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 3m ago 5h 793M 2483M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:24:38.687 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 3m ago 5h 783M 2483M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:24:38.687 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 3m ago 5h 612M 2483M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:24:38.687 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 3m ago 5h 764M 2483M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:24:38.688 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 108s ago 5h 675M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:24:38.688 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 108s ago 5h 839M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:24:38.688 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 108s ago 5h 749M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:24:38.688 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 108s ago 4h 886M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:24:38.689 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 108s ago 4h 184M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:24:38.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:24:38 smithi083 ceph-mon[77242]: pgmap v8891: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:38.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:24:38 smithi083 ceph-mon[79930]: pgmap v8891: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:39.040 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:24:39.040 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:24:39.041 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:24:39.041 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:24:39.041 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:24:39.041 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:24:39.042 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:24:39.042 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:24:39.042 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:24:39.042 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:24:39.042 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:24:39.043 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:24:39.043 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:24:39.043 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:24:39.043 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:24:39.044 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:24:39.044 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:24:39.044 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:24:39.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:24:38 smithi153 ceph-mon[70044]: pgmap v8891: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:39.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:24:39 smithi083 ceph-mon[79930]: from='client.44475 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:24:39.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:24:39 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2858812817' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:24:39.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:24:39 smithi083 ceph-mon[77242]: from='client.44475 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:24:39.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:24:39 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2858812817' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:24:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:24:39 smithi153 ceph-mon[70044]: from='client.44475 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:24:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:24:39 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2858812817' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:24:40.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:24:40 smithi083 ceph-mon[77242]: from='client.54412 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:24:40.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:24:40 smithi083 ceph-mon[77242]: pgmap v8892: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:40.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:24:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:24:40.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:24:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:24:40.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:24:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:24:40.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:24:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:24:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:24:40 smithi083 ceph-mon[79930]: from='client.54412 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:24:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:24:40 smithi083 ceph-mon[79930]: pgmap v8892: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:24:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:24:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:24:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:24:40.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:24:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:24:40.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:24:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:24:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:24:40 smithi153 ceph-mon[70044]: from='client.54412 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:24:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:24:40 smithi153 ceph-mon[70044]: pgmap v8892: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:24:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:24:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:24:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:24:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:24:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:24:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:24:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:24:41.506 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:24:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:24:41.251Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:24:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:24:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:24:41] "GET /metrics HTTP/1.1" 200 225228 "" "Prometheus/2.33.4" 2022-09-29T10:24:42.639 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:24:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:24:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:24:42.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:24:42 smithi083 ceph-mon[79930]: pgmap v8893: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:42.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:24:42 smithi083 ceph-mon[77242]: pgmap v8893: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:43.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:24:42 smithi153 ceph-mon[70044]: pgmap v8893: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:24:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:24:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:24:44.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:24:44 smithi083 ceph-mon[77242]: pgmap v8894: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:44.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:24:44 smithi083 ceph-mon[79930]: pgmap v8894: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:44.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:24:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:24:44.631Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:24:45.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:24:44 smithi153 ceph-mon[70044]: pgmap v8894: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:45.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:24:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:24:45.336Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:24:46.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:24:46 smithi083 ceph-mon[77242]: pgmap v8895: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:46.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:24:46 smithi083 ceph-mon[79930]: pgmap v8895: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:47.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:24:46 smithi153 ceph-mon[70044]: pgmap v8895: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:48.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:24:48 smithi083 ceph-mon[77242]: pgmap v8896: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:48.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:24:48 smithi083 ceph-mon[79930]: pgmap v8896: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:49.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:24:48 smithi153 ceph-mon[70044]: pgmap v8896: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:49.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:24:49 smithi083 ceph-mon[79930]: pgmap v8897: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:49.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:24:49 smithi083 ceph-mon[77242]: pgmap v8897: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:50.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:24:49 smithi153 ceph-mon[70044]: pgmap v8897: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:51.506 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:24:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:24:51.252Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:24:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:24:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:24:51] "GET /metrics HTTP/1.1" 200 225228 "" "Prometheus/2.33.4" 2022-09-29T10:24:52.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:24:52 smithi083 ceph-mon[77242]: pgmap v8898: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:52.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:24:52 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:24:52.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:24:52 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:24:52.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:24:52 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:24:52.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:24:52 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:24:52.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:24:52 smithi083 ceph-mon[79930]: pgmap v8898: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:52.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:24:52 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:24:52.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:24:52 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:24:52.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:24:52 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:24:52.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:24:52 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:24:52.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:24:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:24:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:24:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:24:52 smithi153 ceph-mon[70044]: pgmap v8898: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:24:52 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:24:52.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:24:52 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:24:52.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:24:52 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:24:52.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:24:52 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:24:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:24:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:24:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:24:54.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:24:54 smithi083 ceph-mon[79930]: pgmap v8899: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:54.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:24:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:24:54.632Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:24:54.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:24:54 smithi083 ceph-mon[77242]: pgmap v8899: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:55.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:24:54 smithi153 ceph-mon[70044]: pgmap v8899: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:55.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:24:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:24:55.337Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:24:56.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:24:56 smithi083 ceph-mon[77242]: pgmap v8900: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:56.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:24:56 smithi083 ceph-mon[79930]: pgmap v8900: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:57.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:24:56 smithi153 ceph-mon[70044]: pgmap v8900: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:58.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:24:58 smithi083 ceph-mon[77242]: pgmap v8901: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:58.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:24:58 smithi083 ceph-mon[79930]: pgmap v8901: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:24:59.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:24:58 smithi153 ceph-mon[70044]: pgmap v8901: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:00.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:25:00 smithi083 ceph-mon[77242]: pgmap v8902: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:00.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:25:00 smithi083 ceph-mon[79930]: pgmap v8902: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:01.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:25:00 smithi153 ceph-mon[70044]: pgmap v8902: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:01.506 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:25:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:25:01.253Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:25:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:25:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:25:01] "GET /metrics HTTP/1.1" 200 225229 "" "Prometheus/2.33.4" 2022-09-29T10:25:02.591 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:25:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:25:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:25:02.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:25:02 smithi083 ceph-mon[77242]: pgmap v8903: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:02.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:25:02 smithi083 ceph-mon[79930]: pgmap v8903: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:03.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:25:02 smithi153 ceph-mon[70044]: pgmap v8903: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:25:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:25:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:25:04.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:25:04 smithi083 ceph-mon[77242]: pgmap v8904: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:04.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:25:04 smithi083 ceph-mon[79930]: pgmap v8904: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:04.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:25:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:25:04.633Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:25:05.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:25:04 smithi153 ceph-mon[70044]: pgmap v8904: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:05.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:25:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:25:05.338Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:25:06.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:25:06 smithi083 ceph-mon[79930]: pgmap v8905: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:06.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:25:06 smithi083 ceph-mon[77242]: pgmap v8905: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:07.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:25:06 smithi153 ceph-mon[70044]: pgmap v8905: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:08.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:25:08 smithi083 ceph-mon[79930]: pgmap v8906: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:08.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:25:08 smithi083 ceph-mon[77242]: pgmap v8906: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:09.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:25:08 smithi153 ceph-mon[70044]: pgmap v8906: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:09.369 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:25:09.662 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:25:09.662 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 4m ago 4h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:25:09.663 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 2m ago 4h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:25:09.663 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 4m ago 5h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:25:09.663 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 2m ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:25:09.663 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 2m ago 5h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:25:09.663 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 4m ago 5h 766M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:25:09.664 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 4m ago 5h 1128M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:25:09.664 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 2m ago 5h 485M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:25:09.664 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 4m ago 5h 647M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:25:09.664 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (4h) 4m ago 4h 26.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:25:09.665 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (4h) 2m ago 4h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:25:09.665 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 4m ago 5h 793M 2483M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:25:09.665 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 4m ago 5h 783M 2483M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:25:09.665 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 4m ago 5h 612M 2483M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:25:09.666 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 4m ago 5h 764M 2483M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:25:09.666 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 2m ago 5h 675M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:25:09.666 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 2m ago 5h 839M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:25:09.666 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 2m ago 5h 749M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:25:09.667 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 2m ago 5h 886M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:25:09.667 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 2m ago 4h 184M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:25:10.022 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:25:10.022 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:25:10.022 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:25:10.023 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:25:10.023 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:25:10.023 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:25:10.024 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:25:10.024 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:25:10.024 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:25:10.024 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:25:10.025 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:25:10.025 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:25:10.025 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:25:10.025 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:25:10.025 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:25:10.026 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:25:10.026 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:25:10.026 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:25:10.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:25:10 smithi083 ceph-mon[79930]: pgmap v8907: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:10.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:25:10 smithi083 ceph-mon[79930]: from='client.54424 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:25:10.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:25:10 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3679058606' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:25:10.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:25:10 smithi083 ceph-mon[77242]: pgmap v8907: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:10.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:25:10 smithi083 ceph-mon[77242]: from='client.54424 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:25:10.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:25:10 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3679058606' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:25:11.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:25:10 smithi153 ceph-mon[70044]: pgmap v8907: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:11.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:25:10 smithi153 ceph-mon[70044]: from='client.54424 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:25:11.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:25:10 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3679058606' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:25:11.507 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:25:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:25:11.253Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:25:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:25:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:25:11] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T10:25:11.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:25:11 smithi083 ceph-mon[79930]: from='client.44499 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:25:11.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:25:11 smithi083 ceph-mon[77242]: from='client.44499 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:25:12.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:25:11 smithi153 ceph-mon[70044]: from='client.44499 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:25:12.609 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:25:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:25:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:25:12.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:25:12 smithi083 ceph-mon[77242]: pgmap v8908: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:12.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:25:12 smithi083 ceph-mon[79930]: pgmap v8908: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:13.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:25:12 smithi153 ceph-mon[70044]: pgmap v8908: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:25:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:25:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:25:14.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:25:14 smithi083 ceph-mon[77242]: pgmap v8909: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:14.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:25:14 smithi083 ceph-mon[79930]: pgmap v8909: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:14.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:25:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:25:14.633Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:25:15.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:25:14 smithi153 ceph-mon[70044]: pgmap v8909: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:15.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:25:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:25:15.338Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:25:16.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:25:16 smithi083 ceph-mon[79930]: pgmap v8910: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:16.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:25:16 smithi083 ceph-mon[77242]: pgmap v8910: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:17.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:25:16 smithi153 ceph-mon[70044]: pgmap v8910: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:18.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:25:18 smithi083 ceph-mon[79930]: pgmap v8911: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:18.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:25:18 smithi083 ceph-mon[77242]: pgmap v8911: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:19.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:25:18 smithi153 ceph-mon[70044]: pgmap v8911: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:20.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:25:20 smithi083 ceph-mon[79930]: pgmap v8912: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:20.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:25:20 smithi083 ceph-mon[77242]: pgmap v8912: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:21.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:25:20 smithi153 ceph-mon[70044]: pgmap v8912: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:21.507 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:25:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:25:21.254Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:25:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:25:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:25:21] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T10:25:22.623 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:25:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:25:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:25:22.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:25:22 smithi083 ceph-mon[79930]: pgmap v8913: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:22.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:25:22 smithi083 ceph-mon[77242]: pgmap v8913: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:23.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:25:22 smithi153 ceph-mon[70044]: pgmap v8913: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:25:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:25:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:25:24.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:25:24 smithi083 ceph-mon[79930]: pgmap v8914: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:24.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:25:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:25:24.633Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:25:24.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:25:24 smithi083 ceph-mon[77242]: pgmap v8914: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:25.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:25:24 smithi153 ceph-mon[70044]: pgmap v8914: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:25.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:25:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:25:25.338Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:25:26.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:25:26 smithi083 ceph-mon[79930]: pgmap v8915: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:26.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:25:26 smithi083 ceph-mon[77242]: pgmap v8915: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:27.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:25:26 smithi153 ceph-mon[70044]: pgmap v8915: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:28.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:25:28 smithi083 ceph-mon[79930]: pgmap v8916: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:28.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:25:28 smithi083 ceph-mon[77242]: pgmap v8916: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:29.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:25:28 smithi153 ceph-mon[70044]: pgmap v8916: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:30.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:25:30 smithi083 ceph-mon[77242]: pgmap v8917: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:30.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:25:30 smithi083 ceph-mon[79930]: pgmap v8917: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:31.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:25:30 smithi153 ceph-mon[70044]: pgmap v8917: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:31.506 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:25:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:25:31.255Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:25:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:25:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:25:31] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T10:25:32.637 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:25:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:25:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:25:32.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:25:32 smithi083 ceph-mon[79930]: pgmap v8918: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:32.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:25:32 smithi083 ceph-mon[77242]: pgmap v8918: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:33.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:25:32 smithi153 ceph-mon[70044]: pgmap v8918: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:25:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:25:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:25:34.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:25:34 smithi083 ceph-mon[79930]: pgmap v8919: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:34.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:25:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:25:34.634Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:25:34.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:25:34 smithi083 ceph-mon[77242]: pgmap v8919: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:35.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:25:34 smithi153 ceph-mon[70044]: pgmap v8919: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:35.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:25:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:25:35.339Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:25:36.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:25:36 smithi083 ceph-mon[79930]: pgmap v8920: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:36.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:25:36 smithi083 ceph-mon[77242]: pgmap v8920: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:37.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:25:36 smithi153 ceph-mon[70044]: pgmap v8920: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:38.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:25:38 smithi083 ceph-mon[77242]: pgmap v8921: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:38.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:25:38 smithi083 ceph-mon[79930]: pgmap v8921: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:39.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:25:38 smithi153 ceph-mon[70044]: pgmap v8921: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:40.350 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:25:40.637 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:25:40.637 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 4m ago 4h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:25:40.638 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (4h) 2m ago 4h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:25:40.638 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 4m ago 5h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:25:40.638 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 2m ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:25:40.638 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 2m ago 5h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:25:40.639 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 4m ago 5h 766M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:25:40.639 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 4m ago 5h 1128M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:25:40.639 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 2m ago 5h 485M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:25:40.640 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 4m ago 5h 647M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:25:40.640 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 4m ago 5h 26.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:25:40.640 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 2m ago 5h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:25:40.641 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 4m ago 5h 793M 2483M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:25:40.641 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 4m ago 5h 783M 2483M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:25:40.641 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 4m ago 5h 612M 2483M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:25:40.641 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 4m ago 5h 764M 2483M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:25:40.642 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 2m ago 5h 675M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:25:40.642 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 2m ago 5h 839M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:25:40.642 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 2m ago 5h 749M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:25:40.642 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 2m ago 5h 886M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:25:40.643 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 2m ago 5h 184M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:25:40.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:25:40 smithi083 ceph-mon[79930]: pgmap v8922: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:40.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:25:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:25:40.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:25:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:25:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:25:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:25:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:25:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:25:40.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:25:40 smithi083 ceph-mon[77242]: pgmap v8922: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:25:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:25:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:25:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:25:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:25:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:25:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:25:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:25:40.993 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:25:40.993 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:25:40.993 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:25:40.993 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:25:40.994 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:25:40.994 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:25:40.994 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:25:40.994 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:25:40.995 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:25:40.995 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:25:40.995 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:25:40.995 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:25:40.996 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:25:40.996 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:25:40.996 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:25:40.996 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:25:40.996 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:25:40.997 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:25:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:25:40 smithi153 ceph-mon[70044]: pgmap v8922: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:25:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:25:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:25:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:25:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:25:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:25:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:25:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:25:41.506 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:25:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:25:41.254Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:25:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:25:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:25:41] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T10:25:41.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:25:41 smithi083 ceph-mon[77242]: from='client.44511 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:25:41.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:25:41 smithi083 ceph-mon[77242]: from='client.44517 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:25:41.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:25:41 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4105409791' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:25:41.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:25:41 smithi083 ceph-mon[79930]: from='client.44511 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:25:41.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:25:41 smithi083 ceph-mon[79930]: from='client.44517 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:25:41.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:25:41 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4105409791' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:25:42.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:25:41 smithi153 ceph-mon[70044]: from='client.44511 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:25:42.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:25:41 smithi153 ceph-mon[70044]: from='client.44517 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:25:42.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:25:41 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4105409791' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:25:42.656 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:25:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:25:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:25:42.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:25:42 smithi083 ceph-mon[79930]: pgmap v8923: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:42.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:25:42 smithi083 ceph-mon[77242]: pgmap v8923: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:43.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:25:42 smithi153 ceph-mon[70044]: pgmap v8923: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:25:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:25:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:25:44.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:25:44 smithi083 ceph-mon[79930]: pgmap v8924: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:44.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:25:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:25:44.635Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:25:44.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:25:44 smithi083 ceph-mon[77242]: pgmap v8924: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:45.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:25:44 smithi153 ceph-mon[70044]: pgmap v8924: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:45.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:25:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:25:45.340Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:25:46.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:25:46 smithi083 ceph-mon[79930]: pgmap v8925: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:46.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:25:46 smithi083 ceph-mon[77242]: pgmap v8925: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:47.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:25:46 smithi153 ceph-mon[70044]: pgmap v8925: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:48.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:25:48 smithi083 ceph-mon[79930]: pgmap v8926: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:48.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:25:48 smithi083 ceph-mon[77242]: pgmap v8926: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:49.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:25:48 smithi153 ceph-mon[70044]: pgmap v8926: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:50.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:25:50 smithi083 ceph-mon[79930]: pgmap v8927: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:50.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:25:50 smithi083 ceph-mon[77242]: pgmap v8927: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:51.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:25:50 smithi153 ceph-mon[70044]: pgmap v8927: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:51.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:25:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:25:51] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T10:25:51.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:25:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:25:51.256Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:25:52.597 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:25:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:25:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:25:52.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:25:52 smithi083 ceph-mon[79930]: pgmap v8928: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:52.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:25:52 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:25:52.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:25:52 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:25:52.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:25:52 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:25:52.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:25:52 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:25:52.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:25:52 smithi083 ceph-mon[77242]: pgmap v8928: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:52.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:25:52 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:25:52.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:25:52 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:25:52.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:25:52 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:25:52.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:25:52 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:25:53.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:25:52 smithi153 ceph-mon[70044]: pgmap v8928: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:53.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:25:52 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:25:53.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:25:52 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:25:53.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:25:52 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:25:53.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:25:52 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:25:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:25:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:25:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:25:54.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:25:54 smithi083 ceph-mon[79930]: pgmap v8929: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:54.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:25:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:25:54.634Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:25:54.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:25:54 smithi083 ceph-mon[77242]: pgmap v8929: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:55.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:25:54 smithi153 ceph-mon[70044]: pgmap v8929: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:55.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:25:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:25:55.340Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:25:56.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:25:56 smithi083 ceph-mon[79930]: pgmap v8930: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:56.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:25:56 smithi083 ceph-mon[77242]: pgmap v8930: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:57.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:25:56 smithi153 ceph-mon[70044]: pgmap v8930: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:58.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:25:58 smithi083 ceph-mon[79930]: pgmap v8931: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:58.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:25:58 smithi083 ceph-mon[77242]: pgmap v8931: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:25:59.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:25:58 smithi153 ceph-mon[70044]: pgmap v8931: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:00.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:26:00 smithi083 ceph-mon[79930]: pgmap v8932: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:00.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:26:00 smithi083 ceph-mon[77242]: pgmap v8932: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:01.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:26:00 smithi153 ceph-mon[70044]: pgmap v8932: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:01.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:26:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:26:01] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T10:26:01.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:26:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:26:01.256Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:26:02.612 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:26:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:26:02.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:26:02 smithi083 ceph-mon[79930]: pgmap v8933: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:02.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:26:02 smithi083 ceph-mon[77242]: pgmap v8933: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:03.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:26:02 smithi153 ceph-mon[70044]: pgmap v8933: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:26:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:26:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:26:04.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:26:04 smithi083 ceph-mon[79930]: pgmap v8934: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:04.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:26:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:26:04.635Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:26:04.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:26:04 smithi083 ceph-mon[77242]: pgmap v8934: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:05.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:26:04 smithi153 ceph-mon[70044]: pgmap v8934: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:05.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:26:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:26:05.341Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:26:06.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:26:06 smithi083 ceph-mon[77242]: pgmap v8935: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:06.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:26:06 smithi083 ceph-mon[79930]: pgmap v8935: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:07.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:26:06 smithi153 ceph-mon[70044]: pgmap v8935: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:08.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:26:08 smithi083 ceph-mon[79930]: pgmap v8936: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:08.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:26:08 smithi083 ceph-mon[77242]: pgmap v8936: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:09.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:26:08 smithi153 ceph-mon[70044]: pgmap v8936: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:10.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:26:10 smithi083 ceph-mon[79930]: pgmap v8937: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:10.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:26:10 smithi083 ceph-mon[77242]: pgmap v8937: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:11.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:26:10 smithi153 ceph-mon[70044]: pgmap v8937: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:11.320 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:26:11.607 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:26:11.608 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4h) 5m ago 5h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:26:11.608 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 3m ago 5h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:26:11.608 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 5m ago 5h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:26:11.609 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 3m ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:26:11.609 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 3m ago 5h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:26:11.609 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 5m ago 5h 766M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:26:11.609 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 5m ago 5h 1128M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:26:11.609 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 3m ago 5h 485M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:26:11.610 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 5m ago 5h 647M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:26:11.610 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 5m ago 5h 26.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:26:11.610 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 3m ago 5h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:26:11.610 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 5m ago 5h 793M 2483M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:26:11.611 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 5m ago 5h 783M 2483M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:26:11.611 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 5m ago 5h 612M 2483M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:26:11.611 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 5m ago 5h 764M 2483M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:26:11.611 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 3m ago 5h 675M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:26:11.611 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 3m ago 5h 839M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:26:11.612 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 3m ago 5h 749M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:26:11.612 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 3m ago 5h 886M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:26:11.612 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 3m ago 5h 184M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:26:11.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:26:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:26:11] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T10:26:11.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:26:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:26:11.256Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:26:11.963 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:26:11.963 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:26:11.963 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:26:11.964 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:26:11.964 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:26:11.964 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:26:11.964 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:26:11.965 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:26:11.965 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:26:11.965 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:26:11.965 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:26:11.965 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:26:11.966 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:26:11.966 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:26:11.966 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:26:11.966 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:26:11.967 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:26:11.967 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:26:12.629 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:26:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:26:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:26:12.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:26:12 smithi083 ceph-mon[77242]: pgmap v8938: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:12.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:26:12 smithi083 ceph-mon[77242]: from='client.54460 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:26:12.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:26:12 smithi083 ceph-mon[77242]: from='client.54466 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:26:12.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:26:12 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3219239355' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:26:12.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:26:12 smithi083 ceph-mon[79930]: pgmap v8938: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:12.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:26:12 smithi083 ceph-mon[79930]: from='client.54460 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:26:12.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:26:12 smithi083 ceph-mon[79930]: from='client.54466 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:26:12.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:26:12 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3219239355' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:26:13.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:26:12 smithi153 ceph-mon[70044]: pgmap v8938: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:13.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:26:12 smithi153 ceph-mon[70044]: from='client.54460 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:26:13.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:26:12 smithi153 ceph-mon[70044]: from='client.54466 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:26:13.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:26:12 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3219239355' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:26:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:26:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:26:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:26:14.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:26:14 smithi083 ceph-mon[79930]: pgmap v8939: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:14.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:26:14 smithi083 ceph-mon[77242]: pgmap v8939: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:14.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:26:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:26:14.636Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:26:15.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:26:14 smithi153 ceph-mon[70044]: pgmap v8939: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:15.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:26:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:26:15.342Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:26:16.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:26:16 smithi083 ceph-mon[77242]: pgmap v8940: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:16.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:26:16 smithi083 ceph-mon[79930]: pgmap v8940: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:17.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:26:16 smithi153 ceph-mon[70044]: pgmap v8940: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:18.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:26:18 smithi083 ceph-mon[77242]: pgmap v8941: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:18.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:26:18 smithi083 ceph-mon[79930]: pgmap v8941: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:19.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:26:18 smithi153 ceph-mon[70044]: pgmap v8941: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:20.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:26:20 smithi083 ceph-mon[79930]: pgmap v8942: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:20.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:26:20 smithi083 ceph-mon[77242]: pgmap v8942: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:21.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:26:20 smithi153 ceph-mon[70044]: pgmap v8942: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:21.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:26:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:26:21] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T10:26:21.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:26:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:26:21.257Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:26:22.646 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:26:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:26:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:26:22.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:26:22 smithi083 ceph-mon[79930]: pgmap v8943: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:22.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:26:22 smithi083 ceph-mon[77242]: pgmap v8943: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:23.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:26:22 smithi153 ceph-mon[70044]: pgmap v8943: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:26:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:26:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:26:24.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:26:24 smithi083 ceph-mon[79930]: pgmap v8944: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:24.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:26:24 smithi083 ceph-mon[77242]: pgmap v8944: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:24.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:26:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:26:24.636Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:26:25.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:26:24 smithi153 ceph-mon[70044]: pgmap v8944: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:25.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:26:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:26:25.343Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:26:26.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:26:26 smithi083 ceph-mon[79930]: pgmap v8945: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:26.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:26:26 smithi083 ceph-mon[77242]: pgmap v8945: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:27.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:26:26 smithi153 ceph-mon[70044]: pgmap v8945: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:28.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:26:28 smithi083 ceph-mon[79930]: pgmap v8946: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:28.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:26:28 smithi083 ceph-mon[77242]: pgmap v8946: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:29.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:26:28 smithi153 ceph-mon[70044]: pgmap v8946: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:30.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:26:30 smithi083 ceph-mon[77242]: pgmap v8947: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:30.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:26:30 smithi083 ceph-mon[79930]: pgmap v8947: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:31.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:26:30 smithi153 ceph-mon[70044]: pgmap v8947: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:31.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:26:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:26:31] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T10:26:31.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:26:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:26:31.258Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:26:32.660 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:26:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:26:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:26:32.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:26:32 smithi083 ceph-mon[77242]: pgmap v8948: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:32.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:26:32 smithi083 ceph-mon[79930]: pgmap v8948: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:33.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:26:32 smithi153 ceph-mon[70044]: pgmap v8948: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:26:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:26:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:26:34.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:26:34 smithi083 ceph-mon[79930]: pgmap v8949: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:34.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:26:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:26:34.638Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:26:34.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:26:34 smithi083 ceph-mon[77242]: pgmap v8949: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:35.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:26:34 smithi153 ceph-mon[70044]: pgmap v8949: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:35.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:26:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:26:35.343Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:26:36.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:26:35 smithi153 ceph-mon[70044]: pgmap v8950: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:36.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:26:35 smithi083 ceph-mon[79930]: pgmap v8950: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:36.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:26:35 smithi083 ceph-mon[77242]: pgmap v8950: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:38.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:26:38 smithi083 ceph-mon[79930]: pgmap v8951: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:38.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:26:38 smithi083 ceph-mon[77242]: pgmap v8951: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:26:38 smithi153 ceph-mon[70044]: pgmap v8951: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:40.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:26:40 smithi083 ceph-mon[77242]: pgmap v8952: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:40.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:26:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:26:40.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:26:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:26:40.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:26:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:26:40.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:26:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:26:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:26:40 smithi083 ceph-mon[79930]: pgmap v8952: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:26:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:26:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:26:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:26:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:26:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:26:40.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:26:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:26:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:26:40 smithi153 ceph-mon[70044]: pgmap v8952: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:26:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:26:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:26:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:26:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:26:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:26:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:26:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:26:41.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:26:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:26:41] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T10:26:41.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:26:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:26:41.260Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:26:42.288 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:26:42.576 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:26:42.577 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 5m ago 5h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:26:42.577 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 3m ago 5h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:26:42.577 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 5m ago 5h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:26:42.578 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 3m ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:26:42.578 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 3m ago 5h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:26:42.578 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 5m ago 5h 766M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:26:42.578 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 5m ago 5h 1128M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:26:42.579 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 3m ago 5h 485M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:26:42.579 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 5m ago 5h 647M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:26:42.579 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 5m ago 5h 26.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:26:42.579 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 3m ago 5h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:26:42.579 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 5m ago 5h 793M 2483M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:26:42.580 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 5m ago 5h 783M 2483M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:26:42.580 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 5m ago 5h 612M 2483M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:26:42.580 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 5m ago 5h 764M 2483M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:26:42.580 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 3m ago 5h 675M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:26:42.581 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 3m ago 5h 839M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:26:42.581 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 3m ago 5h 749M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:26:42.581 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 3m ago 5h 886M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:26:42.581 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 3m ago 5h 184M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:26:42.604 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:26:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:26:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:26:42.932 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:26:42.932 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:26:42.932 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:26:42.933 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:26:42.933 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:26:42.933 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:26:42.933 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:26:42.933 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:26:42.934 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:26:42.934 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:26:42.934 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:26:42.934 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:26:42.935 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:26:42.935 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:26:42.935 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:26:42.935 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:26:42.936 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:26:42.936 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:26:42.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:26:42 smithi083 ceph-mon[79930]: pgmap v8953: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:42.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:26:42 smithi083 ceph-mon[77242]: pgmap v8953: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:43.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:26:42 smithi153 ceph-mon[70044]: pgmap v8953: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:26:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:26:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:26:43.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:26:43 smithi083 ceph-mon[79930]: from='client.54478 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:26:43.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:26:43 smithi083 ceph-mon[79930]: from='client.44553 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:26:43.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:26:43 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4245028715' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:26:43.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:26:43 smithi083 ceph-mon[77242]: from='client.54478 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:26:43.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:26:43 smithi083 ceph-mon[77242]: from='client.44553 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:26:43.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:26:43 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4245028715' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:26:44.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:26:43 smithi153 ceph-mon[70044]: from='client.54478 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:26:44.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:26:43 smithi153 ceph-mon[70044]: from='client.44553 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:26:44.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:26:43 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4245028715' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:26:44.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:26:44 smithi083 ceph-mon[79930]: pgmap v8954: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:44.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:26:44 smithi083 ceph-mon[77242]: pgmap v8954: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:44.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:26:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:26:44.638Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:26:45.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:26:44 smithi153 ceph-mon[70044]: pgmap v8954: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:45.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:26:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:26:45.344Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:26:46.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:26:46 smithi083 ceph-mon[79930]: pgmap v8955: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:46.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:26:46 smithi083 ceph-mon[77242]: pgmap v8955: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:47.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:26:46 smithi153 ceph-mon[70044]: pgmap v8955: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:48.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:26:48 smithi083 ceph-mon[79930]: pgmap v8956: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:48.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:26:48 smithi083 ceph-mon[77242]: pgmap v8956: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:49.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:26:48 smithi153 ceph-mon[70044]: pgmap v8956: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:50.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:26:50 smithi083 ceph-mon[79930]: pgmap v8957: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:50.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:26:50 smithi083 ceph-mon[77242]: pgmap v8957: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:51.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:26:50 smithi153 ceph-mon[70044]: pgmap v8957: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:51.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:26:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:26:51] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T10:26:51.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:26:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:26:51.260Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:26:52.626 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:26:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:26:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:26:52.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:26:52 smithi083 ceph-mon[79930]: pgmap v8958: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:52.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:26:52 smithi083 ceph-mon[77242]: pgmap v8958: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:53.022 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:26:52 smithi153 ceph-mon[70044]: pgmap v8958: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:26:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:26:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:26:53.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:26:53 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:26:53.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:26:53 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:26:53.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:26:53 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:26:53.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:26:53 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:26:53.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:26:53 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:26:53.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:26:53 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:26:53.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:26:53 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:26:53.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:26:53 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:26:54.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:26:53 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:26:54.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:26:53 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:26:54.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:26:53 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:26:54.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:26:53 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:26:54.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:26:54 smithi083 ceph-mon[79930]: pgmap v8959: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:54.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:26:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:26:54.639Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:26:54.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:26:54 smithi083 ceph-mon[77242]: pgmap v8959: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:55.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:26:54 smithi153 ceph-mon[70044]: pgmap v8959: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:55.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:26:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:26:55.345Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:26:56.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:26:56 smithi083 ceph-mon[79930]: pgmap v8960: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:56.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:26:56 smithi083 ceph-mon[77242]: pgmap v8960: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:57.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:26:56 smithi153 ceph-mon[70044]: pgmap v8960: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:58.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:26:58 smithi083 ceph-mon[77242]: pgmap v8961: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:58.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:26:58 smithi083 ceph-mon[79930]: pgmap v8961: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:26:59.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:26:58 smithi153 ceph-mon[70044]: pgmap v8961: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:00.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:27:00 smithi083 ceph-mon[77242]: pgmap v8962: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:00.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:27:00 smithi083 ceph-mon[79930]: pgmap v8962: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:01.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:27:00 smithi153 ceph-mon[70044]: pgmap v8962: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:01.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:27:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:27:01] "GET /metrics HTTP/1.1" 200 225226 "" "Prometheus/2.33.4" 2022-09-29T10:27:01.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:27:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:27:01.262Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:27:02.650 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:27:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:27:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:27:02.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:27:02 smithi083 ceph-mon[79930]: pgmap v8963: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:02.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:27:02 smithi083 ceph-mon[77242]: pgmap v8963: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:03.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:27:02 smithi153 ceph-mon[70044]: pgmap v8963: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:27:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:27:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:27:04.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:27:04 smithi083 ceph-mon[79930]: pgmap v8964: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:04.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:27:04 smithi083 ceph-mon[77242]: pgmap v8964: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:04.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:27:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:27:04.641Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:27:05.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:27:04 smithi153 ceph-mon[70044]: pgmap v8964: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:05.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:27:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:27:05.345Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:27:06.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:27:06 smithi083 ceph-mon[77242]: pgmap v8965: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:06.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:27:06 smithi083 ceph-mon[79930]: pgmap v8965: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:07.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:27:06 smithi153 ceph-mon[70044]: pgmap v8965: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:08.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:27:08 smithi083 ceph-mon[79930]: pgmap v8966: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:08.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:27:08 smithi083 ceph-mon[77242]: pgmap v8966: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:09.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:27:08 smithi153 ceph-mon[70044]: pgmap v8966: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:10.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:27:10 smithi083 ceph-mon[79930]: pgmap v8967: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:10.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:27:10 smithi083 ceph-mon[77242]: pgmap v8967: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:11.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:27:10 smithi153 ceph-mon[70044]: pgmap v8967: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:11.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:27:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:27:11] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T10:27:11.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:27:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:27:11.262Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:27:12.663 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:27:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:27:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:27:12.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:27:12 smithi083 ceph-mon[77242]: pgmap v8968: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:12.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:27:12 smithi083 ceph-mon[79930]: pgmap v8968: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:13.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:27:12 smithi153 ceph-mon[70044]: pgmap v8968: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:13.258 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:27:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:27:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:27:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:27:13.555 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:27:13.555 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 6m ago 5h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:27:13.555 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 4m ago 5h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:27:13.556 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 6m ago 5h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:27:13.556 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 4m ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:27:13.556 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 4m ago 5h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:27:13.556 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 6m ago 5h 766M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:27:13.557 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 6m ago 5h 1128M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:27:13.557 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 4m ago 5h 485M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:27:13.557 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 6m ago 5h 647M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:27:13.557 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 6m ago 5h 26.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:27:13.558 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 4m ago 5h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:27:13.558 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 6m ago 5h 793M 2483M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:27:13.558 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 6m ago 5h 783M 2483M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:27:13.558 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 6m ago 5h 612M 2483M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:27:13.559 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 6m ago 5h 764M 2483M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:27:13.559 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 4m ago 5h 675M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:27:13.559 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 4m ago 5h 839M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:27:13.559 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 4m ago 5h 749M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:27:13.559 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 4m ago 5h 886M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:27:13.560 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 4m ago 5h 184M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:27:13.923 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:27:13.923 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:27:13.923 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:27:13.924 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:27:13.924 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:27:13.924 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:27:13.924 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:27:13.925 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:27:13.925 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:27:13.925 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:27:13.925 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:27:13.926 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:27:13.926 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:27:13.926 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:27:13.926 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:27:13.926 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:27:13.927 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:27:13.927 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:27:14.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:27:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:27:14.641Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:27:14.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:27:14 smithi083 ceph-mon[79930]: pgmap v8969: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:14.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:27:14 smithi083 ceph-mon[79930]: from='client.54496 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:27:14.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:27:14 smithi083 ceph-mon[79930]: from='client.54502 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:27:14.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:27:14 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3225418607' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:27:14.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:27:14 smithi083 ceph-mon[77242]: pgmap v8969: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:14.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:27:14 smithi083 ceph-mon[77242]: from='client.54496 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:27:14.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:27:14 smithi083 ceph-mon[77242]: from='client.54502 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:27:14.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:27:14 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3225418607' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:27:15.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:27:14 smithi153 ceph-mon[70044]: pgmap v8969: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:15.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:27:14 smithi153 ceph-mon[70044]: from='client.54496 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:27:15.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:27:14 smithi153 ceph-mon[70044]: from='client.54502 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:27:15.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:27:14 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3225418607' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:27:15.674 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:27:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:27:15.345Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:27:15.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:27:15 smithi083 ceph-mon[77242]: pgmap v8970: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:15.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:27:15 smithi083 ceph-mon[79930]: pgmap v8970: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:16.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:27:15 smithi153 ceph-mon[70044]: pgmap v8970: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:18.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:27:18 smithi083 ceph-mon[79930]: pgmap v8971: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:18.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:27:18 smithi083 ceph-mon[77242]: pgmap v8971: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:27:18 smithi153 ceph-mon[70044]: pgmap v8971: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:20.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:27:20 smithi083 ceph-mon[79930]: pgmap v8972: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:20.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:27:20 smithi083 ceph-mon[77242]: pgmap v8972: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:21.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:27:20 smithi153 ceph-mon[70044]: pgmap v8972: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:21.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:27:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:27:21] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T10:27:21.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:27:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:27:21.263Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:27:22.613 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:27:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:27:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:27:22.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:27:22 smithi083 ceph-mon[79930]: pgmap v8973: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:22.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:27:22 smithi083 ceph-mon[77242]: pgmap v8973: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:23.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:27:22 smithi153 ceph-mon[70044]: pgmap v8973: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:27:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:27:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:27:24.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:27:24 smithi083 ceph-mon[79930]: pgmap v8974: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:24.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:27:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:27:24.642Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:27:24.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:27:24 smithi083 ceph-mon[77242]: pgmap v8974: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:25.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:27:24 smithi153 ceph-mon[70044]: pgmap v8974: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:25.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:27:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:27:25.346Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:27:26.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:27:26 smithi083 ceph-mon[77242]: pgmap v8975: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:26.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:27:26 smithi083 ceph-mon[79930]: pgmap v8975: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:27.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:27:26 smithi153 ceph-mon[70044]: pgmap v8975: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:28.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:27:28 smithi083 ceph-mon[79930]: pgmap v8976: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:28.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:27:28 smithi083 ceph-mon[77242]: pgmap v8976: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:29.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:27:28 smithi153 ceph-mon[70044]: pgmap v8976: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:30.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:27:30 smithi083 ceph-mon[79930]: pgmap v8977: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:30.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:27:30 smithi083 ceph-mon[77242]: pgmap v8977: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:31.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:27:30 smithi153 ceph-mon[70044]: pgmap v8977: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:31.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:27:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:27:31] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T10:27:31.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:27:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:27:31.263Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:27:32.631 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:27:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:27:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:27:32.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:27:32 smithi083 ceph-mon[77242]: pgmap v8978: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:32.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:27:32 smithi083 ceph-mon[79930]: pgmap v8978: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:33.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:27:32 smithi153 ceph-mon[70044]: pgmap v8978: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:27:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:27:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:27:34.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:27:34 smithi083 ceph-mon[79930]: pgmap v8979: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:34.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:27:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:27:34.643Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:27:34.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:27:34 smithi083 ceph-mon[77242]: pgmap v8979: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:35.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:27:34 smithi153 ceph-mon[70044]: pgmap v8979: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:35.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:27:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:27:35.347Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:27:36.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:27:36 smithi083 ceph-mon[79930]: pgmap v8980: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:36.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:27:36 smithi083 ceph-mon[77242]: pgmap v8980: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:37.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:27:36 smithi153 ceph-mon[70044]: pgmap v8980: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:38.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:27:38 smithi083 ceph-mon[79930]: pgmap v8981: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:38.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:27:38 smithi083 ceph-mon[77242]: pgmap v8981: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:39.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:27:38 smithi153 ceph-mon[70044]: pgmap v8981: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:40.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:27:40 smithi083 ceph-mon[79930]: pgmap v8982: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:40.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:27:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:27:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:27:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:27:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:27:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:27:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:27:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:27:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:27:40 smithi083 ceph-mon[77242]: pgmap v8982: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:27:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:27:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:27:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:27:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:27:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:27:40.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:27:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:27:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:27:40 smithi153 ceph-mon[70044]: pgmap v8982: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:27:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:27:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:27:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:27:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:27:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:27:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:27:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:27:41.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:27:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:27:41] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T10:27:41.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:27:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:27:41.264Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:27:42.646 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:27:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:27:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:27:42.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:27:42 smithi083 ceph-mon[79930]: pgmap v8983: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:42.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:27:42 smithi083 ceph-mon[77242]: pgmap v8983: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:43.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:27:42 smithi153 ceph-mon[70044]: pgmap v8983: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:27:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:27:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:27:44.246 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:27:44.532 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:27:44.532 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 7m ago 5h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:27:44.532 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 4m ago 5h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:27:44.533 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 7m ago 5h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:27:44.533 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 4m ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:27:44.533 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 4m ago 5h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:27:44.533 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 7m ago 5h 766M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:27:44.533 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 7m ago 5h 1128M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:27:44.534 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 4m ago 5h 485M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:27:44.534 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 7m ago 5h 647M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:27:44.534 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 7m ago 5h 26.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:27:44.534 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 4m ago 5h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:27:44.535 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 7m ago 5h 793M 2483M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:27:44.535 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 7m ago 5h 783M 2483M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:27:44.535 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 7m ago 5h 612M 2483M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:27:44.535 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 7m ago 5h 764M 2483M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:27:44.536 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 4m ago 5h 675M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:27:44.536 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 4m ago 5h 839M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:27:44.536 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 4m ago 5h 749M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:27:44.536 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 4m ago 5h 886M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:27:44.536 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 4m ago 5h 184M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:27:44.891 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:27:44.891 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:27:44.892 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:27:44.892 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:27:44.892 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:27:44.892 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:27:44.893 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:27:44.893 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:27:44.893 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:27:44.893 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:27:44.894 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:27:44.894 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:27:44.894 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:27:44.894 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:27:44.895 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:27:44.895 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:27:44.895 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:27:44.895 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:27:44.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:27:44 smithi083 ceph-mon[79930]: pgmap v8984: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:44.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:27:44 smithi083 ceph-mon[77242]: pgmap v8984: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:44.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:27:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:27:44.644Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:27:45.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:27:44 smithi153 ceph-mon[70044]: pgmap v8984: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:45.653 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:27:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:27:45.347Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:27:45.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:27:45 smithi083 ceph-mon[79930]: from='client.44583 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:27:45.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:27:45 smithi083 ceph-mon[79930]: from='client.54520 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:27:45.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:27:45 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3296518834' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:27:45.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:27:45 smithi083 ceph-mon[77242]: from='client.44583 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:27:45.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:27:45 smithi083 ceph-mon[77242]: from='client.54520 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:27:45.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:27:45 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3296518834' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:27:46.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:27:45 smithi153 ceph-mon[70044]: from='client.44583 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:27:46.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:27:45 smithi153 ceph-mon[70044]: from='client.54520 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:27:46.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:27:45 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3296518834' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:27:46.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:27:46 smithi083 ceph-mon[79930]: pgmap v8985: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:46.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:27:46 smithi083 ceph-mon[77242]: pgmap v8985: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:47.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:27:46 smithi153 ceph-mon[70044]: pgmap v8985: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:48.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:27:48 smithi083 ceph-mon[79930]: pgmap v8986: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:48.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:27:48 smithi083 ceph-mon[77242]: pgmap v8986: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:49.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:27:48 smithi153 ceph-mon[70044]: pgmap v8986: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:50.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:27:50 smithi083 ceph-mon[79930]: pgmap v8987: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:50.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:27:50 smithi083 ceph-mon[77242]: pgmap v8987: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:51.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:27:50 smithi153 ceph-mon[70044]: pgmap v8987: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:51.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:27:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:27:51] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T10:27:51.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:27:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:27:51.264Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:27:52.663 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:27:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:27:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:27:52.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:27:52 smithi083 ceph-mon[77242]: pgmap v8988: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:52.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:27:52 smithi083 ceph-mon[79930]: pgmap v8988: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:53.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:27:52 smithi153 ceph-mon[70044]: pgmap v8988: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:27:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:27:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:27:53.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:27:53 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:27:53.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:27:53 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:27:53.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:27:53 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:27:53.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:27:53 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:27:53.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:27:53 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:27:53.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:27:53 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:27:54.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:27:53 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:27:54.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:27:53 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:27:54.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:27:53 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:27:54.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:27:54 smithi083 ceph-mon[79930]: pgmap v8989: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:54.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:27:54 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:27:54.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:27:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:27:54.643Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:27:54.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:27:54 smithi083 ceph-mon[77242]: pgmap v8989: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:54.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:27:54 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:27:55.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:27:54 smithi153 ceph-mon[70044]: pgmap v8989: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:55.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:27:54 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:27:55.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:27:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:27:55.348Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:27:56.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:27:56 smithi083 ceph-mon[79930]: pgmap v8990: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:56.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:27:56 smithi083 ceph-mon[77242]: pgmap v8990: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:57.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:27:56 smithi153 ceph-mon[70044]: pgmap v8990: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:58.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:27:57 smithi153 ceph-mon[70044]: pgmap v8991: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:58.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:27:57 smithi083 ceph-mon[79930]: pgmap v8991: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:27:58.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:27:57 smithi083 ceph-mon[77242]: pgmap v8991: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:00.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:28:00 smithi083 ceph-mon[79930]: pgmap v8992: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:00.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:28:00 smithi083 ceph-mon[77242]: pgmap v8992: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:01.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:28:00 smithi153 ceph-mon[70044]: pgmap v8992: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:01.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:28:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:28:01] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T10:28:01.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:28:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:28:01.265Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:28:02.623 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:28:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:28:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:28:02.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:28:02 smithi083 ceph-mon[79930]: pgmap v8993: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:02.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:28:02 smithi083 ceph-mon[77242]: pgmap v8993: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:03.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:28:02 smithi153 ceph-mon[70044]: pgmap v8993: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:28:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:28:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:28:04.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:28:04 smithi083 ceph-mon[77242]: pgmap v8994: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:04.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:28:04 smithi083 ceph-mon[79930]: pgmap v8994: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:04.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:28:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:28:04.643Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:28:05.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:28:04 smithi153 ceph-mon[70044]: pgmap v8994: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:05.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:28:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:28:05.348Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:28:06.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:28:06 smithi083 ceph-mon[79930]: pgmap v8995: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:06.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:28:06 smithi083 ceph-mon[77242]: pgmap v8995: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:07.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:28:06 smithi153 ceph-mon[70044]: pgmap v8995: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:08.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:28:08 smithi083 ceph-mon[79930]: pgmap v8996: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:08.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:28:08 smithi083 ceph-mon[77242]: pgmap v8996: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:09.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:28:08 smithi153 ceph-mon[70044]: pgmap v8996: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:10.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:28:10 smithi083 ceph-mon[79930]: pgmap v8997: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:10.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:28:10 smithi083 ceph-mon[77242]: pgmap v8997: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:11.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:28:10 smithi153 ceph-mon[70044]: pgmap v8997: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:11.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:28:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:28:11] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T10:28:11.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:28:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:28:11.266Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:28:12.636 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:28:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:28:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:28:12.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:28:12 smithi083 ceph-mon[79930]: pgmap v8998: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:12.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:28:12 smithi083 ceph-mon[77242]: pgmap v8998: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:13.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:28:12 smithi153 ceph-mon[70044]: pgmap v8998: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:28:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:28:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:28:14.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:28:14 smithi083 ceph-mon[79930]: pgmap v8999: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:14.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:28:14 smithi083 ceph-mon[77242]: pgmap v8999: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:14.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:28:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:28:14.645Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:28:15.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:28:14 smithi153 ceph-mon[70044]: pgmap v8999: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:15.216 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:28:15.505 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:28:15.506 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 7m ago 5h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:28:15.506 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 5m ago 5h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:28:15.506 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 7m ago 5h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:28:15.506 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 5m ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:28:15.506 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 5m ago 5h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:28:15.507 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 7m ago 5h 766M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:28:15.507 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 7m ago 5h 1128M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:28:15.507 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 5m ago 5h 485M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:28:15.507 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 7m ago 5h 647M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:28:15.508 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 7m ago 5h 26.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:28:15.508 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 5m ago 5h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:28:15.508 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 7m ago 5h 793M 2483M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:28:15.508 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 7m ago 5h 783M 2483M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:28:15.508 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 7m ago 5h 612M 2483M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:28:15.509 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 7m ago 5h 764M 2483M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:28:15.509 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 5m ago 5h 675M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:28:15.509 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 5m ago 5h 839M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:28:15.509 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 5m ago 5h 749M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:28:15.510 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 5m ago 5h 886M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:28:15.510 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 5m ago 5h 184M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:28:15.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:28:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:28:15.348Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:28:15.862 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:28:15.863 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:28:15.863 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:28:15.863 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:28:15.863 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:28:15.864 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:28:15.864 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:28:15.864 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:28:15.864 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:28:15.865 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:28:15.865 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:28:15.865 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:28:15.865 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:28:15.865 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:28:15.866 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:28:15.866 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:28:15.866 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:28:15.866 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:28:16.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:28:16 smithi083 ceph-mon[77242]: pgmap v9000: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:16.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:28:16 smithi083 ceph-mon[77242]: from='client.54532 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:28:16.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:28:16 smithi083 ceph-mon[77242]: from='client.44607 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:28:16.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:28:16 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4045512347' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:28:16.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:28:16 smithi083 ceph-mon[79930]: pgmap v9000: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:16.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:28:16 smithi083 ceph-mon[79930]: from='client.54532 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:28:16.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:28:16 smithi083 ceph-mon[79930]: from='client.44607 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:28:16.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:28:16 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4045512347' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:28:17.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:28:16 smithi153 ceph-mon[70044]: pgmap v9000: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:17.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:28:16 smithi153 ceph-mon[70044]: from='client.54532 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:28:17.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:28:16 smithi153 ceph-mon[70044]: from='client.44607 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:28:17.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:28:16 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4045512347' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:28:18.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:28:18 smithi083 ceph-mon[79930]: pgmap v9001: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:18.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:28:18 smithi083 ceph-mon[77242]: pgmap v9001: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:19.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:28:18 smithi153 ceph-mon[70044]: pgmap v9001: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:20.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:28:20 smithi083 ceph-mon[79930]: pgmap v9002: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:20.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:28:20 smithi083 ceph-mon[77242]: pgmap v9002: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:21.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:28:20 smithi153 ceph-mon[70044]: pgmap v9002: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:21.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:28:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:28:21] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T10:28:21.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:28:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:28:21.266Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:28:22.651 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:28:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:28:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:28:22.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:28:22 smithi083 ceph-mon[79930]: pgmap v9003: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:22.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:28:22 smithi083 ceph-mon[77242]: pgmap v9003: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:23.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:28:22 smithi153 ceph-mon[70044]: pgmap v9003: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:28:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:28:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:28:24.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:28:24 smithi083 ceph-mon[79930]: pgmap v9004: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:24.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:28:24 smithi083 ceph-mon[77242]: pgmap v9004: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:24.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:28:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:28:24.645Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:28:25.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:28:24 smithi153 ceph-mon[70044]: pgmap v9004: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:25.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:28:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:28:25.350Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:28:26.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:28:26 smithi083 ceph-mon[79930]: pgmap v9005: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:26.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:28:26 smithi083 ceph-mon[77242]: pgmap v9005: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:27.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:28:26 smithi153 ceph-mon[70044]: pgmap v9005: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:28.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:28:28 smithi083 ceph-mon[79930]: pgmap v9006: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:28.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:28:28 smithi083 ceph-mon[77242]: pgmap v9006: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:29.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:28:28 smithi153 ceph-mon[70044]: pgmap v9006: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:30.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:28:30 smithi083 ceph-mon[79930]: pgmap v9007: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:30.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:28:30 smithi083 ceph-mon[77242]: pgmap v9007: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:31.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:28:30 smithi153 ceph-mon[70044]: pgmap v9007: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:31.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:28:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:28:31] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T10:28:31.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:28:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:28:31.267Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:28:32.666 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:28:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:28:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:28:32.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:28:32 smithi083 ceph-mon[79930]: pgmap v9008: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:32.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:28:32 smithi083 ceph-mon[77242]: pgmap v9008: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:33.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:28:32 smithi153 ceph-mon[70044]: pgmap v9008: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:28:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:28:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:28:34.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:28:34 smithi083 ceph-mon[79930]: pgmap v9009: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:34.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:28:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:28:34.645Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:28:34.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:28:34 smithi083 ceph-mon[77242]: pgmap v9009: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:35.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:28:34 smithi153 ceph-mon[70044]: pgmap v9009: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:35.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:28:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:28:35.351Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:28:36.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:28:36 smithi083 ceph-mon[79930]: pgmap v9010: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:36.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:28:36 smithi083 ceph-mon[77242]: pgmap v9010: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:37.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:28:36 smithi153 ceph-mon[70044]: pgmap v9010: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:38.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:28:38 smithi083 ceph-mon[77242]: pgmap v9011: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:38.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:28:38 smithi083 ceph-mon[79930]: pgmap v9011: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:39.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:28:38 smithi153 ceph-mon[70044]: pgmap v9011: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:40.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:28:40 smithi083 ceph-mon[77242]: pgmap v9012: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:40.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:28:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:28:40.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:28:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:28:40.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:28:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:28:40.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:28:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:28:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:28:40 smithi083 ceph-mon[79930]: pgmap v9012: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:28:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:28:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:28:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:28:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:28:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:28:40.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:28:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:28:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:28:40 smithi153 ceph-mon[70044]: pgmap v9012: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:28:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:28:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:28:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:28:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:28:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:28:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:28:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:28:41.686 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:28:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:28:41] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T10:28:41.687 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:28:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:28:41.268Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:28:41.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:28:41 smithi083 ceph-mon[79930]: pgmap v9013: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:41.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:28:41 smithi083 ceph-mon[77242]: pgmap v9013: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:42.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:28:41 smithi153 ceph-mon[70044]: pgmap v9013: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:28:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:28:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:28:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:28:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:28:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:28:44.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:28:44 smithi083 ceph-mon[79930]: pgmap v9014: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:44.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:28:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:28:44.646Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:28:44.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:28:44 smithi083 ceph-mon[77242]: pgmap v9014: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:45.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:28:44 smithi153 ceph-mon[70044]: pgmap v9014: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:45.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:28:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:28:45.351Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:28:46.188 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:28:46.478 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:28:46.479 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 8m ago 5h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:28:46.479 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 5m ago 5h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:28:46.479 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 8m ago 5h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:28:46.479 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 5m ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:28:46.480 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (4h) 5m ago 5h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:28:46.480 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (4h) 8m ago 5h 766M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:28:46.480 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 8m ago 5h 1128M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:28:46.481 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 5m ago 5h 485M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:28:46.481 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 8m ago 5h 647M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:28:46.481 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 8m ago 5h 26.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:28:46.481 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 5m ago 5h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:28:46.481 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 8m ago 5h 793M 2483M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:28:46.482 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 8m ago 5h 783M 2483M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:28:46.482 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 8m ago 5h 612M 2483M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:28:46.482 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 8m ago 5h 764M 2483M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:28:46.482 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 5m ago 5h 675M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:28:46.483 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 5m ago 5h 839M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:28:46.483 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 5m ago 5h 749M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:28:46.483 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 5m ago 5h 886M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:28:46.483 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (4h) 5m ago 5h 184M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:28:46.834 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:28:46.834 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:28:46.835 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:28:46.835 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:28:46.835 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:28:46.835 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:28:46.836 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:28:46.836 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:28:46.836 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:28:46.836 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:28:46.836 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:28:46.837 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:28:46.837 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:28:46.837 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:28:46.837 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:28:46.838 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:28:46.838 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:28:46.838 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:28:46.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:28:46 smithi083 ceph-mon[79930]: pgmap v9015: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:46.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:28:46 smithi083 ceph-mon[77242]: pgmap v9015: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:47.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:28:46 smithi153 ceph-mon[70044]: pgmap v9015: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:47.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:28:47 smithi083 ceph-mon[79930]: from='client.44619 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:28:47.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:28:47 smithi083 ceph-mon[79930]: from='client.44625 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:28:47.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:28:47 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/630144178' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:28:47.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:28:47 smithi083 ceph-mon[77242]: from='client.44619 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:28:47.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:28:47 smithi083 ceph-mon[77242]: from='client.44625 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:28:47.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:28:47 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/630144178' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:28:48.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:28:47 smithi153 ceph-mon[70044]: from='client.44619 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:28:48.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:28:47 smithi153 ceph-mon[70044]: from='client.44625 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:28:48.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:28:47 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/630144178' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:28:48.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:28:48 smithi083 ceph-mon[79930]: pgmap v9016: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:48.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:28:48 smithi083 ceph-mon[77242]: pgmap v9016: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:49.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:28:48 smithi153 ceph-mon[70044]: pgmap v9016: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:50.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:28:50 smithi083 ceph-mon[79930]: pgmap v9017: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:50.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:28:50 smithi083 ceph-mon[77242]: pgmap v9017: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:51.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:28:50 smithi153 ceph-mon[70044]: pgmap v9017: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:51.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:28:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:28:51] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T10:28:51.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:28:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:28:51.269Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:28:52.643 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:28:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:28:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:28:52.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:28:52 smithi083 ceph-mon[79930]: pgmap v9018: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:52.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:28:52 smithi083 ceph-mon[77242]: pgmap v9018: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:53.022 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:28:52 smithi153 ceph-mon[70044]: pgmap v9018: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:28:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:28:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:28:54.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:28:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:28:54.646Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:28:54.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:28:54 smithi083 ceph-mon[79930]: pgmap v9019: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:54.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:28:54 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:28:54.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:28:54 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:28:54.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:28:54 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:28:54.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:28:54 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:28:54.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:28:54 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:28:54.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:28:54 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:28:54.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:28:54 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:28:54.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:28:54 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:28:54.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:28:54 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:28:54.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:28:54 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:28:54.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:28:54 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:28:54.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:28:54 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:28:54.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:28:54 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:28:54.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:28:54 smithi083 ceph-mon[77242]: pgmap v9019: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:54.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:28:54 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:28:54.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:28:54 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:28:54.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:28:54 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:28:54.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:28:54 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:28:54.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:28:54 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:28:54.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:28:54 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:28:54.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:28:54 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:28:54.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:28:54 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:28:54.968 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:28:54 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:28:54.968 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:28:54 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:28:54.968 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:28:54 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:28:54.968 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:28:54 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:28:54.968 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:28:54 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:28:55.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:28:54 smithi153 ceph-mon[70044]: pgmap v9019: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:55.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:28:54 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:28:55.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:28:54 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:28:55.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:28:54 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:28:55.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:28:54 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:28:55.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:28:54 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:28:55.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:28:54 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:28:55.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:28:54 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:28:55.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:28:54 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:28:55.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:28:54 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:28:55.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:28:54 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:28:55.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:28:54 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:28:55.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:28:54 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:28:55.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:28:54 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:28:55.650 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:28:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:28:55.352Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:28:55.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:28:55 smithi083 ceph-mon[77242]: Adjusting osd_memory_target on smithi083 to 2474M 2022-09-29T10:28:55.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:28:55 smithi083 ceph-mon[79930]: Adjusting osd_memory_target on smithi083 to 2474M 2022-09-29T10:28:56.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:28:55 smithi153 ceph-mon[70044]: Adjusting osd_memory_target on smithi083 to 2474M 2022-09-29T10:28:56.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:28:56 smithi083 ceph-mon[77242]: pgmap v9020: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:56.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:28:56 smithi083 ceph-mon[79930]: pgmap v9020: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:57.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:28:56 smithi153 ceph-mon[70044]: pgmap v9020: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:58.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:28:58 smithi083 ceph-mon[79930]: pgmap v9021: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:58.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:28:58 smithi083 ceph-mon[77242]: pgmap v9021: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:28:59.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:28:58 smithi153 ceph-mon[70044]: pgmap v9021: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:00.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:29:00 smithi083 ceph-mon[79930]: pgmap v9022: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:00.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:29:00 smithi083 ceph-mon[77242]: pgmap v9022: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:01.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:29:00 smithi153 ceph-mon[70044]: pgmap v9022: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:01.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:29:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:29:01] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T10:29:01.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:29:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:29:01.269Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:29:02.661 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:29:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:29:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:29:02.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:29:02 smithi083 ceph-mon[79930]: pgmap v9023: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:02.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:29:02 smithi083 ceph-mon[77242]: pgmap v9023: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:03.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:29:02 smithi153 ceph-mon[70044]: pgmap v9023: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:29:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:29:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:29:04.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:29:04 smithi083 ceph-mon[79930]: pgmap v9024: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:04.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:29:04 smithi083 ceph-mon[77242]: pgmap v9024: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:04.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:29:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:29:04.647Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:29:05.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:29:04 smithi153 ceph-mon[70044]: pgmap v9024: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:05.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:29:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:29:05.352Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:29:06.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:29:06 smithi083 ceph-mon[77242]: pgmap v9025: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:06.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:29:06 smithi083 ceph-mon[79930]: pgmap v9025: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:07.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:29:06 smithi153 ceph-mon[70044]: pgmap v9025: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:08.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:29:08 smithi083 ceph-mon[77242]: pgmap v9026: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:08.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:29:08 smithi083 ceph-mon[79930]: pgmap v9026: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:09.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:29:08 smithi153 ceph-mon[70044]: pgmap v9026: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:10.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:29:10 smithi083 ceph-mon[79930]: pgmap v9027: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:10.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:29:10 smithi083 ceph-mon[77242]: pgmap v9027: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:11.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:29:10 smithi153 ceph-mon[70044]: pgmap v9027: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:11.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:29:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:29:11] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T10:29:11.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:29:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:29:11.270Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:29:12.676 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:29:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:29:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:29:12.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:29:12 smithi083 ceph-mon[77242]: pgmap v9028: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:12.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:29:12 smithi083 ceph-mon[79930]: pgmap v9028: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:13.022 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:29:12 smithi153 ceph-mon[70044]: pgmap v9028: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:29:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:29:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:29:14.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:29:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:29:14.647Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:29:14.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:29:14 smithi083 ceph-mon[77242]: pgmap v9029: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:14.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:29:14 smithi083 ceph-mon[79930]: pgmap v9029: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:15.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:29:14 smithi153 ceph-mon[70044]: pgmap v9029: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:15.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:29:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:29:15.353Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:29:16.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:29:16 smithi083 ceph-mon[79930]: pgmap v9030: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:16.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:29:16 smithi083 ceph-mon[77242]: pgmap v9030: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:17.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:29:16 smithi153 ceph-mon[70044]: pgmap v9030: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:17.160 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:29:17.450 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:29:17.450 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 8m ago 5h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:29:17.450 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 6m ago 5h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:29:17.451 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 8m ago 5h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:29:17.451 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 6m ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:29:17.451 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 6m ago 5h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:29:17.451 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 8m ago 5h 766M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:29:17.452 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (4h) 8m ago 5h 1128M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:29:17.452 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 6m ago 5h 485M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:29:17.452 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 8m ago 5h 647M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:29:17.452 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 8m ago 5h 26.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:29:17.452 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 6m ago 5h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:29:17.453 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 8m ago 5h 793M 2474M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:29:17.453 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 8m ago 5h 783M 2474M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:29:17.453 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 8m ago 5h 612M 2474M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:29:17.453 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 8m ago 5h 764M 2474M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:29:17.454 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 6m ago 5h 675M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:29:17.454 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 6m ago 5h 839M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:29:17.454 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 6m ago 5h 749M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:29:17.454 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 6m ago 5h 886M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:29:17.454 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 6m ago 5h 184M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:29:17.807 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:29:17.808 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:29:17.808 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:29:17.808 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:29:17.808 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:29:17.809 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:29:17.809 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:29:17.809 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:29:17.809 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:29:17.810 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:29:17.810 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:29:17.810 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:29:17.810 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:29:17.811 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:29:17.811 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:29:17.811 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:29:17.811 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:29:17.811 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:29:18.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:29:18 smithi083 ceph-mon[79930]: pgmap v9031: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:18.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:29:18 smithi083 ceph-mon[79930]: from='client.44637 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:29:18.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:29:18 smithi083 ceph-mon[79930]: from='client.44643 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:29:18.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:29:18 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4100343527' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:29:18.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:29:18 smithi083 ceph-mon[77242]: pgmap v9031: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:18.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:29:18 smithi083 ceph-mon[77242]: from='client.44637 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:29:18.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:29:18 smithi083 ceph-mon[77242]: from='client.44643 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:29:18.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:29:18 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4100343527' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:29:19.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:29:18 smithi153 ceph-mon[70044]: pgmap v9031: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:19.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:29:18 smithi153 ceph-mon[70044]: from='client.44637 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:29:19.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:29:18 smithi153 ceph-mon[70044]: from='client.44643 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:29:19.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:29:18 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4100343527' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:29:20.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:29:20 smithi083 ceph-mon[79930]: pgmap v9032: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:20.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:29:20 smithi083 ceph-mon[77242]: pgmap v9032: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:21.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:29:20 smithi153 ceph-mon[70044]: pgmap v9032: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:21.690 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:29:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:29:21] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T10:29:21.691 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:29:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:29:21.271Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:29:21.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:29:21 smithi083 ceph-mon[79930]: pgmap v9033: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:21.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:29:21 smithi083 ceph-mon[77242]: pgmap v9033: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:22.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:29:21 smithi153 ceph-mon[70044]: pgmap v9033: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:29:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:29:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:29:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:29:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:29:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:29:24.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:29:24 smithi083 ceph-mon[79930]: pgmap v9034: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:24.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:29:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:29:24.647Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:29:24.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:29:24 smithi083 ceph-mon[77242]: pgmap v9034: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:25.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:29:24 smithi153 ceph-mon[70044]: pgmap v9034: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:25.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:29:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:29:25.353Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:29:26.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:29:26 smithi083 ceph-mon[77242]: pgmap v9035: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:26.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:29:26 smithi083 ceph-mon[79930]: pgmap v9035: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:27.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:29:26 smithi153 ceph-mon[70044]: pgmap v9035: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:28.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:29:28 smithi083 ceph-mon[79930]: pgmap v9036: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:28.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:29:28 smithi083 ceph-mon[77242]: pgmap v9036: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:29.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:29:28 smithi153 ceph-mon[70044]: pgmap v9036: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:30.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:29:30 smithi083 ceph-mon[77242]: pgmap v9037: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:30.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:29:30 smithi083 ceph-mon[79930]: pgmap v9037: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:31.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:29:30 smithi153 ceph-mon[70044]: pgmap v9037: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:31.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:29:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:29:31] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T10:29:31.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:29:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:29:31.272Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:29:32.649 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:29:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:29:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:29:32.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:29:32 smithi083 ceph-mon[79930]: pgmap v9038: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:32.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:29:32 smithi083 ceph-mon[77242]: pgmap v9038: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:33.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:29:32 smithi153 ceph-mon[70044]: pgmap v9038: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:29:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:29:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:29:34.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:29:34 smithi083 ceph-mon[77242]: pgmap v9039: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:34.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:29:34 smithi083 ceph-mon[79930]: pgmap v9039: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:34.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:29:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:29:34.648Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:29:35.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:29:34 smithi153 ceph-mon[70044]: pgmap v9039: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:35.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:29:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:29:35.354Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:29:36.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:29:36 smithi083 ceph-mon[79930]: pgmap v9040: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:36.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:29:36 smithi083 ceph-mon[77242]: pgmap v9040: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:37.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:29:36 smithi153 ceph-mon[70044]: pgmap v9040: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:38.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:29:38 smithi083 ceph-mon[79930]: pgmap v9041: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:38.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:29:38 smithi083 ceph-mon[77242]: pgmap v9041: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:39.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:29:38 smithi153 ceph-mon[70044]: pgmap v9041: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:40.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:29:40 smithi083 ceph-mon[79930]: pgmap v9042: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:29:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:29:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:29:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:29:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:29:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:29:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:29:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:29:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:29:40 smithi083 ceph-mon[77242]: pgmap v9042: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:29:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:29:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:29:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:29:40.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:29:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:29:40.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:29:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:29:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:29:40 smithi153 ceph-mon[70044]: pgmap v9042: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:29:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:29:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:29:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:29:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:29:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:29:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:29:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:29:41.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:29:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:29:41] "GET /metrics HTTP/1.1" 200 225229 "" "Prometheus/2.33.4" 2022-09-29T10:29:41.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:29:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:29:41.273Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:29:42.668 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:29:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:29:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:29:42.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:29:42 smithi083 ceph-mon[79930]: pgmap v9043: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:42.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:29:42 smithi083 ceph-mon[77242]: pgmap v9043: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:43.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:29:42 smithi153 ceph-mon[70044]: pgmap v9043: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:29:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:29:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:29:44.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:29:44 smithi083 ceph-mon[79930]: pgmap v9044: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:44.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:29:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:29:44.649Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:29:44.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:29:44 smithi083 ceph-mon[77242]: pgmap v9044: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:45.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:29:44 smithi153 ceph-mon[70044]: pgmap v9044: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:45.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:29:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:29:45.355Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:29:46.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:29:46 smithi083 ceph-mon[79930]: pgmap v9045: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:46.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:29:46 smithi083 ceph-mon[77242]: pgmap v9045: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:47.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:29:46 smithi153 ceph-mon[70044]: pgmap v9045: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:48.131 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:29:48.421 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:29:48.421 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 9m ago 5h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:29:48.422 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 6m ago 5h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:29:48.422 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 9m ago 5h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:29:48.422 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 6m ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:29:48.422 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 6m ago 5h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:29:48.423 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 9m ago 5h 766M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:29:48.423 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 9m ago 5h 1128M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:29:48.423 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (4h) 6m ago 5h 485M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:29:48.423 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (4h) 9m ago 5h 647M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:29:48.423 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 9m ago 5h 26.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:29:48.424 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 6m ago 5h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:29:48.424 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 9m ago 5h 793M 2474M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:29:48.424 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 9m ago 5h 783M 2474M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:29:48.424 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 9m ago 5h 612M 2474M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:29:48.424 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 9m ago 5h 764M 2474M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:29:48.425 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 6m ago 5h 675M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:29:48.425 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 6m ago 5h 839M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:29:48.425 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 6m ago 5h 749M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:29:48.425 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 6m ago 5h 886M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:29:48.425 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 6m ago 5h 184M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:29:48.779 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:29:48.779 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:29:48.780 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:29:48.780 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:29:48.780 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:29:48.780 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:29:48.781 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:29:48.781 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:29:48.781 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:29:48.781 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:29:48.782 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:29:48.782 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:29:48.782 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:29:48.782 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:29:48.783 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:29:48.783 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:29:48.783 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:29:48.783 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:29:48.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:29:48 smithi083 ceph-mon[77242]: pgmap v9046: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:48.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:29:48 smithi083 ceph-mon[79930]: pgmap v9046: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:49.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:29:48 smithi153 ceph-mon[70044]: pgmap v9046: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:49.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:29:49 smithi083 ceph-mon[79930]: from='client.54586 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:29:49.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:29:49 smithi083 ceph-mon[79930]: from='client.44661 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:29:49.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:29:49 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/725901839' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:29:49.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:29:49 smithi083 ceph-mon[77242]: from='client.54586 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:29:49.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:29:49 smithi083 ceph-mon[77242]: from='client.44661 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:29:49.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:29:49 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/725901839' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:29:50.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:29:49 smithi153 ceph-mon[70044]: from='client.54586 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:29:50.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:29:49 smithi153 ceph-mon[70044]: from='client.44661 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:29:50.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:29:49 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/725901839' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:29:50.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:29:50 smithi083 ceph-mon[79930]: pgmap v9047: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:50.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:29:50 smithi083 ceph-mon[77242]: pgmap v9047: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:51.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:29:50 smithi153 ceph-mon[70044]: pgmap v9047: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:51.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:29:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:29:51] "GET /metrics HTTP/1.1" 200 225229 "" "Prometheus/2.33.4" 2022-09-29T10:29:51.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:29:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:29:51.273Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:29:52.689 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:29:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:29:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:29:52.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:29:52 smithi083 ceph-mon[79930]: pgmap v9048: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:52.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:29:52 smithi083 ceph-mon[77242]: pgmap v9048: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:53.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:29:52 smithi153 ceph-mon[70044]: pgmap v9048: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:29:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:29:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:29:54.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:29:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:29:54.649Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:29:54.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:29:54 smithi083 ceph-mon[79930]: pgmap v9049: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:54.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:29:54 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:29:54.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:29:54 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:29:54.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:29:54 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:29:54.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:29:54 smithi083 ceph-mon[77242]: pgmap v9049: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:54.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:29:54 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:29:54.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:29:54 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:29:54.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:29:54 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:29:55.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:29:54 smithi153 ceph-mon[70044]: pgmap v9049: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:55.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:29:54 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:29:55.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:29:54 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:29:55.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:29:54 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:29:55.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:29:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:29:55.354Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:29:56.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:29:56 smithi083 ceph-mon[77242]: pgmap v9050: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:56.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:29:56 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:29:56.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:29:56 smithi083 ceph-mon[79930]: pgmap v9050: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:56.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:29:56 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:29:56.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:29:56 smithi153 ceph-mon[70044]: pgmap v9050: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:56.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:29:56 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:29:58.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:29:58 smithi083 ceph-mon[79930]: pgmap v9051: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:58.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:29:58 smithi083 ceph-mon[77242]: pgmap v9051: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:29:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:29:58 smithi153 ceph-mon[70044]: pgmap v9051: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:00.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:30:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[77219]: 2022-09-29T10:29:59.999+0000 7f160f665700 -1 log_channel(cluster) log [ERR] : overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T10:30:00.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:30:00 smithi083 ceph-mon[79930]: pgmap v9052: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:00.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:30:00 smithi083 ceph-mon[79930]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T10:30:00.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:30:00 smithi083 ceph-mon[77242]: pgmap v9052: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:00.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:30:00 smithi083 ceph-mon[77242]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T10:30:01.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:30:00 smithi153 ceph-mon[70044]: pgmap v9052: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:01.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:30:00 smithi153 ceph-mon[70044]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T10:30:01.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:30:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:30:01] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T10:30:01.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:30:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:30:01.274Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:30:02.650 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:30:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:30:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:30:02.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:30:02 smithi083 ceph-mon[79930]: pgmap v9053: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:02.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:30:02 smithi083 ceph-mon[77242]: pgmap v9053: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:03.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:30:02 smithi153 ceph-mon[70044]: pgmap v9053: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:30:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:30:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:30:04.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:30:04 smithi083 ceph-mon[77242]: pgmap v9054: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:04.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:30:04 smithi083 ceph-mon[79930]: pgmap v9054: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:04.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:30:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:30:04.650Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:30:05.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:30:04 smithi153 ceph-mon[70044]: pgmap v9054: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:05.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:30:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:30:05.355Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:30:06.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:30:06 smithi083 ceph-mon[79930]: pgmap v9055: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:06.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:30:06 smithi083 ceph-mon[77242]: pgmap v9055: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:07.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:30:06 smithi153 ceph-mon[70044]: pgmap v9055: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:08.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:30:08 smithi083 ceph-mon[77242]: pgmap v9056: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:08.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:30:08 smithi083 ceph-mon[79930]: pgmap v9056: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:09.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:30:08 smithi153 ceph-mon[70044]: pgmap v9056: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:10.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:30:10 smithi083 ceph-mon[79930]: pgmap v9057: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:10.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:30:10 smithi083 ceph-mon[77242]: pgmap v9057: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:11.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:30:10 smithi153 ceph-mon[70044]: pgmap v9057: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:11.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:30:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:30:11] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T10:30:11.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:30:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:30:11.274Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:30:12.665 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:30:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:30:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:30:12.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:30:12 smithi083 ceph-mon[79930]: pgmap v9058: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:12.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:30:12 smithi083 ceph-mon[77242]: pgmap v9058: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:13.022 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:30:12 smithi153 ceph-mon[70044]: pgmap v9058: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:30:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:30:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:30:14.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:30:14 smithi083 ceph-mon[79930]: pgmap v9059: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:14.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:30:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:30:14.651Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:30:14.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:30:14 smithi083 ceph-mon[77242]: pgmap v9059: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:15.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:30:14 smithi153 ceph-mon[70044]: pgmap v9059: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:15.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:30:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:30:15.356Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:30:16.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:30:16 smithi083 ceph-mon[79930]: pgmap v9060: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:16.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:30:16 smithi083 ceph-mon[77242]: pgmap v9060: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:17.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:30:16 smithi153 ceph-mon[70044]: pgmap v9060: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:18.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:30:18 smithi083 ceph-mon[79930]: pgmap v9061: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:18.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:30:18 smithi083 ceph-mon[77242]: pgmap v9061: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:19.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:30:18 smithi153 ceph-mon[70044]: pgmap v9061: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:19.106 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:30:19.395 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:30:19.396 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 9m ago 5h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:30:19.396 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 7m ago 5h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:30:19.396 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 9m ago 5h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:30:19.397 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 7m ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:30:19.397 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 7m ago 5h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:30:19.397 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 9m ago 5h 766M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:30:19.397 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 9m ago 5h 1128M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:30:19.398 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 7m ago 5h 485M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:30:19.398 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 9m ago 5h 647M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:30:19.398 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 9m ago 5h 26.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:30:19.398 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 7m ago 5h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:30:19.399 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4h) 9m ago 5h 793M 2474M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:30:19.399 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 9m ago 5h 783M 2474M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:30:19.399 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 9m ago 5h 612M 2474M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:30:19.399 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 9m ago 5h 764M 2474M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:30:19.400 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 7m ago 5h 675M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:30:19.400 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 7m ago 5h 839M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:30:19.400 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 7m ago 5h 749M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:30:19.400 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 7m ago 5h 886M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:30:19.400 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 7m ago 5h 184M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:30:19.754 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:30:19.754 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:30:19.754 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:30:19.755 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:30:19.755 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:30:19.755 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:30:19.755 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:30:19.756 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:30:19.756 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:30:19.756 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:30:19.756 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:30:19.757 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:30:19.757 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:30:19.757 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:30:19.757 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:30:19.758 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:30:19.758 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:30:19.758 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:30:20.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:30:20 smithi083 ceph-mon[79930]: from='client.54604 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:30:20.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:30:20 smithi083 ceph-mon[79930]: pgmap v9062: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:20.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:30:20 smithi083 ceph-mon[79930]: from='client.54610 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:30:20.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:30:20 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1841739863' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:30:20.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:30:20 smithi083 ceph-mon[77242]: from='client.54604 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:30:20.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:30:20 smithi083 ceph-mon[77242]: pgmap v9062: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:20.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:30:20 smithi083 ceph-mon[77242]: from='client.54610 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:30:20.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:30:20 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1841739863' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:30:21.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:30:20 smithi153 ceph-mon[70044]: from='client.54604 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:30:21.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:30:20 smithi153 ceph-mon[70044]: pgmap v9062: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:21.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:30:20 smithi153 ceph-mon[70044]: from='client.54610 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:30:21.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:30:20 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1841739863' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:30:21.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:30:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:30:21] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T10:30:21.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:30:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:30:21.275Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:30:22.683 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:30:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:30:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:30:22.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:30:22 smithi083 ceph-mon[77242]: pgmap v9063: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:22.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:30:22 smithi083 ceph-mon[79930]: pgmap v9063: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:23.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:30:22 smithi153 ceph-mon[70044]: pgmap v9063: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:30:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:30:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:30:24.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:30:24 smithi083 ceph-mon[79930]: pgmap v9064: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:24.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:30:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:30:24.651Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:30:24.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:30:24 smithi083 ceph-mon[77242]: pgmap v9064: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:25.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:30:24 smithi153 ceph-mon[70044]: pgmap v9064: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:25.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:30:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:30:25.356Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:30:26.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:30:26 smithi083 ceph-mon[77242]: pgmap v9065: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:26.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:30:26 smithi083 ceph-mon[79930]: pgmap v9065: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:27.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:30:26 smithi153 ceph-mon[70044]: pgmap v9065: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:28.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:30:28 smithi083 ceph-mon[79930]: pgmap v9066: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:28.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:30:28 smithi083 ceph-mon[77242]: pgmap v9066: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:29.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:30:28 smithi153 ceph-mon[70044]: pgmap v9066: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:30.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:30:30 smithi083 ceph-mon[79930]: pgmap v9067: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:30.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:30:30 smithi083 ceph-mon[77242]: pgmap v9067: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:31.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:30:30 smithi153 ceph-mon[70044]: pgmap v9067: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:31.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:30:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:30:31] "GET /metrics HTTP/1.1" 200 225227 "" "Prometheus/2.33.4" 2022-09-29T10:30:31.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:30:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:30:31.276Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:30:32.697 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:30:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:30:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:30:32.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:30:32 smithi083 ceph-mon[79930]: pgmap v9068: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:32.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:30:32 smithi083 ceph-mon[77242]: pgmap v9068: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:33.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:30:32 smithi153 ceph-mon[70044]: pgmap v9068: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:30:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:30:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:30:34.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:30:34 smithi083 ceph-mon[77242]: pgmap v9069: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:34.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:30:34 smithi083 ceph-mon[79930]: pgmap v9069: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:34.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:30:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:30:34.652Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:30:35.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:30:34 smithi153 ceph-mon[70044]: pgmap v9069: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:35.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:30:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:30:35.358Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:30:36.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:30:36 smithi083 ceph-mon[79930]: pgmap v9070: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:36.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:30:36 smithi083 ceph-mon[77242]: pgmap v9070: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:37.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:30:36 smithi153 ceph-mon[70044]: pgmap v9070: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:38.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:30:38 smithi083 ceph-mon[79930]: pgmap v9071: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:38.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:30:38 smithi083 ceph-mon[77242]: pgmap v9071: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:39.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:30:38 smithi153 ceph-mon[70044]: pgmap v9071: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:40.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:30:40 smithi083 ceph-mon[79930]: pgmap v9072: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:40.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:30:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:30:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:30:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:30:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:30:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:30:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:30:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:30:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:30:40 smithi083 ceph-mon[77242]: pgmap v9072: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:30:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:30:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:30:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:30:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:30:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:30:40.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:30:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:30:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:30:40 smithi153 ceph-mon[70044]: pgmap v9072: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:30:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:30:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:30:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:30:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:30:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:30:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:30:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:30:41.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:30:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:30:41] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T10:30:41.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:30:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:30:41.276Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:30:42.668 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:30:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:30:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:30:42.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:30:42 smithi083 ceph-mon[77242]: pgmap v9073: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:42.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:30:42 smithi083 ceph-mon[79930]: pgmap v9073: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:43.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:30:42 smithi153 ceph-mon[70044]: pgmap v9073: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:30:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:30:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:30:44.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:30:44 smithi083 ceph-mon[79930]: pgmap v9074: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:44.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:30:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:30:44.652Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:30:44.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:30:44 smithi083 ceph-mon[77242]: pgmap v9074: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:45.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:30:44 smithi153 ceph-mon[70044]: pgmap v9074: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:45.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:30:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:30:45.358Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:30:46.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:30:46 smithi083 ceph-mon[79930]: pgmap v9075: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:46.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:30:46 smithi083 ceph-mon[77242]: pgmap v9075: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:47.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:30:46 smithi153 ceph-mon[70044]: pgmap v9075: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:48.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:30:48 smithi083 ceph-mon[79930]: pgmap v9076: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:48.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:30:48 smithi083 ceph-mon[77242]: pgmap v9076: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:49.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:30:48 smithi153 ceph-mon[70044]: pgmap v9076: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:50.075 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:30:50.363 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:30:50.363 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 10m ago 5h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:30:50.363 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 8m ago 5h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:30:50.364 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 10m ago 5h 25.7M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:30:50.364 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 8m ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:30:50.364 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 8m ago 5h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:30:50.364 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 10m ago 5h 766M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:30:50.365 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 10m ago 5h 1128M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:30:50.365 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 8m ago 5h 485M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:30:50.365 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 10m ago 5h 647M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:30:50.365 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 10m ago 5h 26.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:30:50.366 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 8m ago 5h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:30:50.366 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 10m ago 5h 793M 2474M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:30:50.366 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 10m ago 5h 783M 2474M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:30:50.366 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 10m ago 5h 612M 2474M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:30:50.367 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 10m ago 5h 764M 2474M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:30:50.367 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 8m ago 5h 675M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:30:50.367 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 8m ago 5h 839M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:30:50.367 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 8m ago 5h 749M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:30:50.367 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 8m ago 5h 886M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:30:50.368 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 8m ago 5h 184M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:30:50.720 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:30:50.720 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:30:50.720 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:30:50.720 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:30:50.721 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:30:50.721 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:30:50.721 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:30:50.721 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:30:50.721 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:30:50.722 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:30:50.722 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:30:50.722 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:30:50.722 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:30:50.723 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:30:50.723 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:30:50.723 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:30:50.723 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:30:50.723 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:30:50.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:30:50 smithi083 ceph-mon[77242]: pgmap v9077: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:50.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:30:50 smithi083 ceph-mon[79930]: pgmap v9077: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:51.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:30:50 smithi153 ceph-mon[70044]: pgmap v9077: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:51.688 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:30:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:30:51] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T10:30:51.688 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:30:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:30:51.276Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:30:51.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:30:51 smithi083 ceph-mon[79930]: from='client.54622 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:30:51.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:30:51 smithi083 ceph-mon[79930]: from='client.44697 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:30:51.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:30:51 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/505948055' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:30:51.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:30:51 smithi083 ceph-mon[77242]: from='client.54622 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:30:51.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:30:51 smithi083 ceph-mon[77242]: from='client.44697 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:30:51.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:30:51 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/505948055' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:30:52.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:30:51 smithi153 ceph-mon[70044]: from='client.54622 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:30:52.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:30:51 smithi153 ceph-mon[70044]: from='client.44697 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:30:52.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:30:51 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/505948055' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:30:52.690 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:30:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:30:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:30:52.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:30:52 smithi083 ceph-mon[79930]: pgmap v9078: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:52.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:30:52 smithi083 ceph-mon[77242]: pgmap v9078: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:53.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:30:52 smithi153 ceph-mon[70044]: pgmap v9078: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:30:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:30:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:30:54.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:30:54 smithi083 ceph-mon[79930]: pgmap v9079: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:54.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:30:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:30:54.653Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:30:54.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:30:54 smithi083 ceph-mon[77242]: pgmap v9079: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:55.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:30:54 smithi153 ceph-mon[70044]: pgmap v9079: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:55.700 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:30:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:30:55.359Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:30:55.700 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:30:55 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:30:55.701 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:30:55 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:30:55.701 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:30:55 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:30:55.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:30:55 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:30:55.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:30:55 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:30:55.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:30:55 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:30:56.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:30:55 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:30:56.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:30:55 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:30:56.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:30:55 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:30:56.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:30:56 smithi083 ceph-mon[79930]: pgmap v9080: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:56.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:30:56 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:30:56.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:30:56 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:30:56.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:30:56 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:30:56.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:30:56 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:30:56.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:30:56 smithi083 ceph-mon[77242]: pgmap v9080: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:56.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:30:56 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:30:56.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:30:56 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:30:56.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:30:56 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:30:56.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:30:56 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:30:57.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:30:56 smithi153 ceph-mon[70044]: pgmap v9080: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:57.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:30:56 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:30:57.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:30:56 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:30:57.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:30:56 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:30:57.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:30:56 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:30:57.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:30:57 smithi083 ceph-mon[79930]: pgmap v9081: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:57.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:30:57 smithi083 ceph-mon[77242]: pgmap v9081: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:30:58.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:30:57 smithi153 ceph-mon[70044]: pgmap v9081: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:00.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:31:00 smithi083 ceph-mon[79930]: pgmap v9082: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:00.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:31:00 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:31:00.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:31:00 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:31:00.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:31:00 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:31:00.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:31:00 smithi083 ceph-mon[77242]: pgmap v9082: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:00.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:31:00 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:31:00.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:31:00 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:31:00.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:31:00 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:31:01.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:31:00 smithi153 ceph-mon[70044]: pgmap v9082: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:01.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:31:00 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:31:01.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:31:00 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:31:01.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:31:00 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:31:01.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:31:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:31:01] "GET /metrics HTTP/1.1" 200 225227 "" "Prometheus/2.33.4" 2022-09-29T10:31:01.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:31:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:31:01.276Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:31:02.664 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:31:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:31:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:31:02.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:31:02 smithi083 ceph-mon[79930]: pgmap v9083: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:02.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:31:02 smithi083 ceph-mon[77242]: pgmap v9083: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:03.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:31:02 smithi153 ceph-mon[70044]: pgmap v9083: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:31:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:31:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:31:04.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:31:04 smithi083 ceph-mon[79930]: pgmap v9084: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:04.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:31:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:31:04.654Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:31:04.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:31:04 smithi083 ceph-mon[77242]: pgmap v9084: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:05.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:31:04 smithi153 ceph-mon[70044]: pgmap v9084: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:05.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:31:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:31:05.359Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:31:06.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:31:06 smithi083 ceph-mon[77242]: pgmap v9085: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:06.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:31:06 smithi083 ceph-mon[79930]: pgmap v9085: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:07.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:31:06 smithi153 ceph-mon[70044]: pgmap v9085: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:08.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:31:08 smithi083 ceph-mon[79930]: pgmap v9086: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:08.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:31:08 smithi083 ceph-mon[77242]: pgmap v9086: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:09.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:31:08 smithi153 ceph-mon[70044]: pgmap v9086: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:10.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:31:10 smithi083 ceph-mon[77242]: pgmap v9087: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:10.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:31:10 smithi083 ceph-mon[79930]: pgmap v9087: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:11.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:31:10 smithi153 ceph-mon[70044]: pgmap v9087: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:11.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:31:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:31:11] "GET /metrics HTTP/1.1" 200 225227 "" "Prometheus/2.33.4" 2022-09-29T10:31:11.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:31:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:31:11.277Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:31:12.688 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:31:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:31:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:31:12.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:31:12 smithi083 ceph-mon[77242]: pgmap v9088: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:12.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:31:12 smithi083 ceph-mon[79930]: pgmap v9088: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:13.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:31:12 smithi153 ceph-mon[70044]: pgmap v9088: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:31:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:31:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:31:14.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:31:14 smithi083 ceph-mon[77242]: pgmap v9089: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:14.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:31:14 smithi083 ceph-mon[79930]: pgmap v9089: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:14.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:31:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:31:14.653Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:31:15.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:31:14 smithi153 ceph-mon[70044]: pgmap v9089: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:15.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:31:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:31:15.359Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:31:16.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:31:16 smithi083 ceph-mon[77242]: pgmap v9090: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:16.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:31:16 smithi083 ceph-mon[79930]: pgmap v9090: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:17.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:31:16 smithi153 ceph-mon[70044]: pgmap v9090: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:18.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:31:18 smithi083 ceph-mon[79930]: pgmap v9091: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:18.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:31:18 smithi083 ceph-mon[77242]: pgmap v9091: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:19.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:31:18 smithi153 ceph-mon[70044]: pgmap v9091: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:20.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:31:20 smithi083 ceph-mon[77242]: pgmap v9092: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:20.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:31:20 smithi083 ceph-mon[79930]: pgmap v9092: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:21.051 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:31:21.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:31:20 smithi153 ceph-mon[70044]: pgmap v9092: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:21.338 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:31:21.338 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 21s ago 5h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:31:21.338 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 8m ago 5h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:31:21.339 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 21s ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:31:21.339 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 8m ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:31:21.339 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 8m ago 5h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:31:21.340 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 21s ago 5h 774M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:31:21.340 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 21s ago 5h 1165M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:31:21.340 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 8m ago 5h 485M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:31:21.340 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 21s ago 5h 669M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:31:21.341 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 21s ago 5h 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:31:21.341 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 8m ago 5h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:31:21.341 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 21s ago 5h 819M 2474M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:31:21.342 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4h) 21s ago 5h 809M 2474M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:31:21.342 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4h) 21s ago 5h 632M 2474M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:31:21.342 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 21s ago 5h 789M 2474M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:31:21.342 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 8m ago 5h 675M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:31:21.343 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 8m ago 5h 839M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:31:21.343 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 8m ago 5h 749M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:31:21.343 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 8m ago 5h 886M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:31:21.343 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 8m ago 5h 184M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:31:21.696 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:31:21.696 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:31:21.696 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:31:21.696 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:31:21.697 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:31:21.697 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:31:21.697 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:31:21.697 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:31:21.697 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:31:21.698 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:31:21.698 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:31:21.698 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:31:21.698 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:31:21.699 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:31:21.699 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:31:21.699 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:31:21.699 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:31:21.699 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:31:21.707 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:31:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:31:21] "GET /metrics HTTP/1.1" 200 225227 "" "Prometheus/2.33.4" 2022-09-29T10:31:21.707 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:31:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:31:21.277Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:31:21.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:31:21 smithi083 ceph-mon[77242]: from='client.44709 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:31:21.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:31:21 smithi083 ceph-mon[77242]: pgmap v9093: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:21.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:31:21 smithi083 ceph-mon[77242]: from='client.54646 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:31:21.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:31:21 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2859665122' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:31:21.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:31:21 smithi083 ceph-mon[79930]: from='client.44709 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:31:21.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:31:21 smithi083 ceph-mon[79930]: pgmap v9093: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:21.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:31:21 smithi083 ceph-mon[79930]: from='client.54646 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:31:21.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:31:21 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2859665122' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:31:22.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:31:21 smithi153 ceph-mon[70044]: from='client.44709 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:31:22.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:31:21 smithi153 ceph-mon[70044]: pgmap v9093: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:22.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:31:21 smithi153 ceph-mon[70044]: from='client.54646 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:31:22.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:31:21 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2859665122' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:31:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:31:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:31:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:31:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:31:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:31:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:31:24.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:31:24 smithi083 ceph-mon[79930]: pgmap v9094: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:24.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:31:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:31:24.654Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:31:24.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:31:24 smithi083 ceph-mon[77242]: pgmap v9094: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:25.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:31:24 smithi153 ceph-mon[70044]: pgmap v9094: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:25.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:31:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:31:25.359Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:31:26.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:31:26 smithi083 ceph-mon[79930]: pgmap v9095: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:26.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:31:26 smithi083 ceph-mon[77242]: pgmap v9095: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:27.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:31:26 smithi153 ceph-mon[70044]: pgmap v9095: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:28.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:31:28 smithi083 ceph-mon[79930]: pgmap v9096: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:28.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:31:28 smithi083 ceph-mon[77242]: pgmap v9096: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:29.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:31:28 smithi153 ceph-mon[70044]: pgmap v9096: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:30.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:31:30 smithi083 ceph-mon[79930]: pgmap v9097: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:30.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:31:30 smithi083 ceph-mon[77242]: pgmap v9097: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:31.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:31:30 smithi153 ceph-mon[70044]: pgmap v9097: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:31.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:31:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:31:31] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T10:31:31.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:31:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:31:31.278Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:31:32.679 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:31:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:31:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:31:32.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:31:32 smithi083 ceph-mon[79930]: pgmap v9098: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:32.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:31:32 smithi083 ceph-mon[77242]: pgmap v9098: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:33.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:31:32 smithi153 ceph-mon[70044]: pgmap v9098: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:31:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:31:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:31:34.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:31:34 smithi083 ceph-mon[79930]: pgmap v9099: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:34.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:31:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:31:34.654Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:31:34.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:31:34 smithi083 ceph-mon[77242]: pgmap v9099: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:35.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:31:34 smithi153 ceph-mon[70044]: pgmap v9099: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:35.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:31:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:31:35.360Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:31:36.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:31:36 smithi083 ceph-mon[79930]: pgmap v9100: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:36.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:31:36 smithi083 ceph-mon[77242]: pgmap v9100: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:37.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:31:36 smithi153 ceph-mon[70044]: pgmap v9100: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:38.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:31:38 smithi083 ceph-mon[79930]: pgmap v9101: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:38.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:31:38 smithi083 ceph-mon[77242]: pgmap v9101: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:39.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:31:38 smithi153 ceph-mon[70044]: pgmap v9101: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:40.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:31:40 smithi083 ceph-mon[79930]: pgmap v9102: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:40.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:31:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:31:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:31:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:31:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:31:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:31:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:31:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:31:40.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:31:40 smithi083 ceph-mon[77242]: pgmap v9102: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:31:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:31:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:31:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:31:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:31:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:31:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:31:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:31:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:31:40 smithi153 ceph-mon[70044]: pgmap v9102: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:31:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:31:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:31:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:31:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:31:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:31:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:31:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:31:41.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:31:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:31:41] "GET /metrics HTTP/1.1" 200 225229 "" "Prometheus/2.33.4" 2022-09-29T10:31:41.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:31:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:31:41.279Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:31:42.693 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:31:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:31:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:31:42.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:31:42 smithi083 ceph-mon[79930]: pgmap v9103: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:42.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:31:42 smithi083 ceph-mon[77242]: pgmap v9103: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:43.022 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:31:42 smithi153 ceph-mon[70044]: pgmap v9103: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:31:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:31:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:31:44.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:31:44 smithi083 ceph-mon[79930]: pgmap v9104: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:44.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:31:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:31:44.655Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:31:44.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:31:44 smithi083 ceph-mon[77242]: pgmap v9104: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:45.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:31:44 smithi153 ceph-mon[70044]: pgmap v9104: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:45.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:31:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:31:45.361Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:31:46.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:31:46 smithi083 ceph-mon[79930]: pgmap v9105: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:46.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:31:46 smithi083 ceph-mon[77242]: pgmap v9105: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:47.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:31:46 smithi153 ceph-mon[70044]: pgmap v9105: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:48.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:31:48 smithi083 ceph-mon[79930]: pgmap v9106: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:48.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:31:48 smithi083 ceph-mon[77242]: pgmap v9106: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:49.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:31:48 smithi153 ceph-mon[70044]: pgmap v9106: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:50.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:31:50 smithi083 ceph-mon[79930]: pgmap v9107: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:50.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:31:50 smithi083 ceph-mon[77242]: pgmap v9107: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:51.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:31:50 smithi153 ceph-mon[70044]: pgmap v9107: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:51.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:31:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:31:51] "GET /metrics HTTP/1.1" 200 225229 "" "Prometheus/2.33.4" 2022-09-29T10:31:51.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:31:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:31:51.280Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:31:52.021 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:31:52.310 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:31:52.310 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 52s ago 5h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:31:52.310 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 9m ago 5h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:31:52.311 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 52s ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:31:52.311 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 9m ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:31:52.311 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 9m ago 5h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:31:52.311 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 52s ago 5h 774M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:31:52.312 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 52s ago 5h 1165M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:31:52.312 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 9m ago 5h 485M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:31:52.312 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 52s ago 5h 669M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:31:52.312 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 52s ago 5h 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:31:52.312 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 9m ago 5h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:31:52.313 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 52s ago 5h 819M 2474M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:31:52.313 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 52s ago 5h 809M 2474M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:31:52.313 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 52s ago 5h 632M 2474M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:31:52.313 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4h) 52s ago 5h 789M 2474M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:31:52.314 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 9m ago 5h 675M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:31:52.314 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 9m ago 5h 839M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:31:52.314 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 9m ago 5h 749M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:31:52.314 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 9m ago 5h 886M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:31:52.314 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 9m ago 5h 184M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:31:52.666 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:31:52.666 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:31:52.666 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:31:52.667 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:31:52.667 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:31:52.667 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:31:52.667 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:31:52.667 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:31:52.668 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:31:52.668 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:31:52.668 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:31:52.668 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:31:52.669 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:31:52.669 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:31:52.669 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:31:52.669 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:31:52.669 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:31:52.670 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:31:52.706 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:31:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:31:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:31:52.706 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:31:52 smithi083 ceph-mon[77242]: pgmap v9108: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:52.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:31:52 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1460159718' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:31:52.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:31:52 smithi083 ceph-mon[79930]: pgmap v9108: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:52.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:31:52 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1460159718' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:31:53.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:31:52 smithi153 ceph-mon[70044]: pgmap v9108: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:53.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:31:52 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1460159718' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:31:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:31:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:31:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:31:54.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:31:53 smithi153 ceph-mon[70044]: from='client.44727 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:31:54.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:31:53 smithi153 ceph-mon[70044]: from='client.44733 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:31:54.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:31:53 smithi153 ceph-mon[70044]: pgmap v9109: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:54.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:31:53 smithi083 ceph-mon[79930]: from='client.44727 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:31:54.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:31:53 smithi083 ceph-mon[79930]: from='client.44733 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:31:54.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:31:53 smithi083 ceph-mon[79930]: pgmap v9109: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:54.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:31:53 smithi083 ceph-mon[77242]: from='client.44727 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:31:54.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:31:53 smithi083 ceph-mon[77242]: from='client.44733 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:31:54.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:31:53 smithi083 ceph-mon[77242]: pgmap v9109: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:54.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:31:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:31:54.655Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:31:55.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:31:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:31:55.362Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:31:56.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:31:56 smithi153 ceph-mon[70044]: pgmap v9110: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:56.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:31:56 smithi083 ceph-mon[77242]: pgmap v9110: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:56.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:31:56 smithi083 ceph-mon[79930]: pgmap v9110: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:31:58 smithi153 ceph-mon[70044]: pgmap v9111: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:58.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:31:58 smithi083 ceph-mon[79930]: pgmap v9111: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:31:58.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:31:58 smithi083 ceph-mon[77242]: pgmap v9111: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:00.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:32:00 smithi083 ceph-mon[79930]: pgmap v9112: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:00.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:32:00 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:32:00.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:32:00 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:32:00.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:32:00 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:32:00.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:32:00 smithi083 ceph-mon[77242]: pgmap v9112: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:00.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:32:00 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:32:00.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:32:00 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:32:00.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:32:00 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:32:01.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:32:00 smithi153 ceph-mon[70044]: pgmap v9112: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:01.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:32:00 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:32:01.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:32:00 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:32:01.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:32:00 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:32:01.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:32:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:32:01] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T10:32:01.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:32:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:32:01.279Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:32:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:32:02 smithi153 ceph-mon[70044]: pgmap v9113: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:02.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:32:02 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:32:02.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:32:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:32:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:32:02.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:32:02 smithi083 ceph-mon[79930]: pgmap v9113: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:02.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:32:02 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:32:02.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:32:02 smithi083 ceph-mon[77242]: pgmap v9113: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:02.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:32:02 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:32:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:32:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:32:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:32:04.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:32:04 smithi083 ceph-mon[79930]: pgmap v9114: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:04.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:32:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:32:04.655Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:32:04.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:32:04 smithi083 ceph-mon[77242]: pgmap v9114: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:05.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:32:04 smithi153 ceph-mon[70044]: pgmap v9114: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:05.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:32:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:32:05.363Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:32:06.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:32:06 smithi083 ceph-mon[77242]: pgmap v9115: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:06.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:32:06 smithi083 ceph-mon[79930]: pgmap v9115: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:07.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:32:06 smithi153 ceph-mon[70044]: pgmap v9115: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:08.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:32:08 smithi083 ceph-mon[79930]: pgmap v9116: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:08.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:32:08 smithi083 ceph-mon[77242]: pgmap v9116: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:09.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:32:08 smithi153 ceph-mon[70044]: pgmap v9116: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:10.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:32:10 smithi083 ceph-mon[79930]: pgmap v9117: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:10.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:32:10 smithi083 ceph-mon[77242]: pgmap v9117: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:11.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:32:10 smithi153 ceph-mon[70044]: pgmap v9117: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:11.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:32:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:32:11] "GET /metrics HTTP/1.1" 200 225223 "" "Prometheus/2.33.4" 2022-09-29T10:32:11.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:32:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:32:11.280Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:32:12.692 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:32:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:32:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:32:12.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:32:12 smithi083 ceph-mon[79930]: pgmap v9118: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:12.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:32:12 smithi083 ceph-mon[77242]: pgmap v9118: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:13.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:32:12 smithi153 ceph-mon[70044]: pgmap v9118: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:32:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:32:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:32:14.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:32:14 smithi083 ceph-mon[79930]: pgmap v9119: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:14.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:32:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:32:14.657Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:32:14.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:32:14 smithi083 ceph-mon[77242]: pgmap v9119: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:15.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:32:14 smithi153 ceph-mon[70044]: pgmap v9119: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:15.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:32:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:32:15.363Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:32:16.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:32:16 smithi083 ceph-mon[79930]: pgmap v9120: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:16.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:32:16 smithi083 ceph-mon[77242]: pgmap v9120: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:17.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:32:16 smithi153 ceph-mon[70044]: pgmap v9120: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:18.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:32:18 smithi083 ceph-mon[77242]: pgmap v9121: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:18.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:32:18 smithi083 ceph-mon[79930]: pgmap v9121: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:19.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:32:18 smithi153 ceph-mon[70044]: pgmap v9121: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:20.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:32:20 smithi083 ceph-mon[77242]: pgmap v9122: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:20.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:32:20 smithi083 ceph-mon[79930]: pgmap v9122: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:21.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:32:20 smithi153 ceph-mon[70044]: pgmap v9122: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:21.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:32:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:32:21] "GET /metrics HTTP/1.1" 200 225223 "" "Prometheus/2.33.4" 2022-09-29T10:32:21.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:32:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:32:21.282Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:32:22.709 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:32:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:32:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:32:22.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:32:22 smithi083 ceph-mon[77242]: pgmap v9123: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:22.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:32:22 smithi083 ceph-mon[79930]: pgmap v9123: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:22.995 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:32:23.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:32:22 smithi153 ceph-mon[70044]: pgmap v9123: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:23.283 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:32:23.283 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 83s ago 5h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:32:23.283 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 9m ago 5h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:32:23.284 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 83s ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:32:23.284 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 9m ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:32:23.284 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 9m ago 5h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:32:23.284 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 83s ago 5h 774M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:32:23.284 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 83s ago 5h 1165M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:32:23.285 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 9m ago 5h 485M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:32:23.285 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 83s ago 5h 669M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:32:23.285 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 83s ago 5h 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:32:23.285 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 9m ago 5h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:32:23.285 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 83s ago 5h 819M 2474M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:32:23.286 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 83s ago 5h 809M 2474M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:32:23.286 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 83s ago 5h 632M 2474M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:32:23.286 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 83s ago 5h 789M 2474M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:32:23.287 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (4h) 9m ago 5h 675M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:32:23.287 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 9m ago 5h 839M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:32:23.287 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 9m ago 5h 749M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:32:23.287 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 9m ago 5h 886M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:32:23.287 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 9m ago 5h 184M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:32:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:32:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:32:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:32:23.648 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:32:23.648 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:32:23.648 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:32:23.649 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:32:23.649 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:32:23.649 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:32:23.649 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:32:23.649 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:32:23.650 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:32:23.650 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:32:23.650 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:32:23.650 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:32:23.650 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:32:23.651 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:32:23.651 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:32:23.651 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:32:23.651 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:32:23.652 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:32:24.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:32:23 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4178367565' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:32:24.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:32:23 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4178367565' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:32:24.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:32:23 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4178367565' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:32:24.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:32:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:32:24.657Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:32:24.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:32:24 smithi083 ceph-mon[77242]: from='client.54676 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:32:24.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:32:24 smithi083 ceph-mon[77242]: pgmap v9124: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:24.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:32:24 smithi083 ceph-mon[77242]: from='client.44751 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:32:24.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:32:24 smithi083 ceph-mon[79930]: from='client.54676 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:32:24.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:32:24 smithi083 ceph-mon[79930]: pgmap v9124: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:24.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:32:24 smithi083 ceph-mon[79930]: from='client.44751 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:32:25.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:32:24 smithi153 ceph-mon[70044]: from='client.54676 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:32:25.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:32:24 smithi153 ceph-mon[70044]: pgmap v9124: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:25.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:32:24 smithi153 ceph-mon[70044]: from='client.44751 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:32:25.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:32:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:32:25.364Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:32:26.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:32:25 smithi153 ceph-mon[70044]: pgmap v9125: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:26.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:32:25 smithi083 ceph-mon[77242]: pgmap v9125: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:26.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:32:25 smithi083 ceph-mon[79930]: pgmap v9125: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:32:28 smithi153 ceph-mon[70044]: pgmap v9126: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:28.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:32:28 smithi083 ceph-mon[79930]: pgmap v9126: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:28.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:32:28 smithi083 ceph-mon[77242]: pgmap v9126: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:30.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:32:30 smithi083 ceph-mon[79930]: pgmap v9127: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:30.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:32:30 smithi083 ceph-mon[77242]: pgmap v9127: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:31.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:32:30 smithi153 ceph-mon[70044]: pgmap v9127: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:31.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:32:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:32:31] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T10:32:31.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:32:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:32:31.282Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:32:32.686 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:32:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:32:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:32:32.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:32:32 smithi083 ceph-mon[77242]: pgmap v9128: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:32.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:32:32 smithi083 ceph-mon[79930]: pgmap v9128: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:33.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:32:32 smithi153 ceph-mon[70044]: pgmap v9128: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:32:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:32:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:32:34.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:32:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:32:34.657Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:32:34.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:32:34 smithi083 ceph-mon[77242]: pgmap v9129: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:34.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:32:34 smithi083 ceph-mon[79930]: pgmap v9129: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:35.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:32:34 smithi153 ceph-mon[70044]: pgmap v9129: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:35.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:32:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:32:35.365Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:32:36.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:32:36 smithi083 ceph-mon[77242]: pgmap v9130: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:36.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:32:36 smithi083 ceph-mon[79930]: pgmap v9130: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:37.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:32:36 smithi153 ceph-mon[70044]: pgmap v9130: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:38.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:32:38 smithi083 ceph-mon[79930]: pgmap v9131: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:38.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:32:38 smithi083 ceph-mon[77242]: pgmap v9131: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:39.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:32:38 smithi153 ceph-mon[70044]: pgmap v9131: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:40.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:32:40 smithi083 ceph-mon[79930]: pgmap v9132: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:32:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:32:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:32:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:32:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:32:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:32:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:32:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:32:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:32:40 smithi083 ceph-mon[77242]: pgmap v9132: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:32:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:32:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:32:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:32:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:32:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:32:40.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:32:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:32:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:32:40 smithi153 ceph-mon[70044]: pgmap v9132: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:32:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:32:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:32:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:32:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:32:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:32:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:32:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:32:41.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:32:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:32:41] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T10:32:41.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:32:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:32:41.282Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:32:42.701 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:32:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:32:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:32:42.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:32:42 smithi083 ceph-mon[79930]: pgmap v9133: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:42.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:32:42 smithi083 ceph-mon[77242]: pgmap v9133: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:43.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:32:42 smithi153 ceph-mon[70044]: pgmap v9133: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:32:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:32:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:32:44.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:32:44 smithi083 ceph-mon[79930]: pgmap v9134: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:44.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:32:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:32:44.658Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:32:44.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:32:44 smithi083 ceph-mon[77242]: pgmap v9134: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:45.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:32:44 smithi153 ceph-mon[70044]: pgmap v9134: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:45.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:32:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:32:45.365Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:32:46.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:32:46 smithi083 ceph-mon[79930]: pgmap v9135: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:46.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:32:46 smithi083 ceph-mon[77242]: pgmap v9135: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:47.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:32:46 smithi153 ceph-mon[70044]: pgmap v9135: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:48.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:32:48 smithi083 ceph-mon[79930]: pgmap v9136: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:48.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:32:48 smithi083 ceph-mon[77242]: pgmap v9136: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:49.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:32:48 smithi153 ceph-mon[70044]: pgmap v9136: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:51.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:32:50 smithi153 ceph-mon[70044]: pgmap v9137: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:51.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:32:50 smithi083 ceph-mon[79930]: pgmap v9137: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:51.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:32:50 smithi083 ceph-mon[77242]: pgmap v9137: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:51.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:32:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:32:51] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T10:32:51.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:32:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:32:51.283Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:32:52.715 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:32:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:32:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:32:53.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:32:52 smithi153 ceph-mon[70044]: pgmap v9138: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:53.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:32:52 smithi083 ceph-mon[77242]: pgmap v9138: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:53.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:32:52 smithi083 ceph-mon[79930]: pgmap v9138: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:32:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:32:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:32:53.978 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:32:54.264 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:32:54.264 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 114s ago 5h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:32:54.265 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 10m ago 5h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:32:54.265 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 114s ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:32:54.265 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 10m ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:32:54.265 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 10m ago 5h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:32:54.265 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 114s ago 5h 774M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:32:54.266 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 114s ago 5h 1165M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:32:54.266 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 10m ago 5h 485M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:32:54.266 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 114s ago 5h 669M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:32:54.266 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 114s ago 5h 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:32:54.267 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 10m ago 5h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:32:54.267 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 114s ago 5h 819M 2474M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:32:54.267 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 114s ago 5h 809M 2474M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:32:54.267 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 114s ago 5h 632M 2474M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:32:54.267 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 114s ago 5h 789M 2474M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:32:54.268 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 10m ago 5h 675M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:32:54.268 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (4h) 10m ago 5h 839M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:32:54.268 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 10m ago 5h 749M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:32:54.268 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 10m ago 5h 886M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:32:54.268 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 10m ago 5h 184M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:32:54.624 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:32:54.624 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:32:54.625 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:32:54.625 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:32:54.625 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:32:54.625 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:32:54.625 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:32:54.626 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:32:54.626 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:32:54.626 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:32:54.626 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:32:54.627 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:32:54.627 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:32:54.627 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:32:54.627 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:32:54.627 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:32:54.628 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:32:54.628 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:32:54.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:32:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:32:54.658Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:32:54.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:32:54 smithi083 ceph-mon[77242]: pgmap v9139: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:54.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:32:54 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2402250651' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:32:54.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:32:54 smithi083 ceph-mon[79930]: pgmap v9139: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:54.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:32:54 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2402250651' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:32:55.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:32:54 smithi153 ceph-mon[70044]: pgmap v9139: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:55.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:32:54 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2402250651' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:32:55.693 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:32:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:32:55.366Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:32:55.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:32:55 smithi083 ceph-mon[77242]: from='client.54694 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:32:55.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:32:55 smithi083 ceph-mon[77242]: from='client.44769 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:32:55.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:32:55 smithi083 ceph-mon[79930]: from='client.54694 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:32:55.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:32:55 smithi083 ceph-mon[79930]: from='client.44769 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:32:56.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:32:55 smithi153 ceph-mon[70044]: from='client.54694 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:32:56.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:32:55 smithi153 ceph-mon[70044]: from='client.44769 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:32:56.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:32:56 smithi083 ceph-mon[77242]: pgmap v9140: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:56.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:32:56 smithi083 ceph-mon[79930]: pgmap v9140: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:57.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:32:56 smithi153 ceph-mon[70044]: pgmap v9140: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:58.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:32:58 smithi083 ceph-mon[77242]: pgmap v9141: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:58.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:32:58 smithi083 ceph-mon[79930]: pgmap v9141: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:32:59.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:32:58 smithi153 ceph-mon[70044]: pgmap v9141: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:00.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:33:00 smithi083 ceph-mon[77242]: pgmap v9142: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:00.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:33:00 smithi083 ceph-mon[79930]: pgmap v9142: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:01.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:33:00 smithi153 ceph-mon[70044]: pgmap v9142: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:01.706 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:33:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:33:01] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T10:33:01.707 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:33:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:33:01.283Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:33:01.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:33:01 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:33:01.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:33:01 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:33:01.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:33:01 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:33:01.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:33:01 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:33:01.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:33:01 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:33:01.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:33:01 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:33:02.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:33:01 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:33:02.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:33:01 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:33:02.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:33:01 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:33:02.708 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:33:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:33:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:33:02.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:33:02 smithi083 ceph-mon[77242]: pgmap v9143: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:02.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:33:02 smithi083 ceph-mon[79930]: pgmap v9143: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:03.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:33:02 smithi153 ceph-mon[70044]: pgmap v9143: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:33:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:33:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:33:04.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:33:04 smithi083 ceph-mon[79930]: pgmap v9144: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:04.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:33:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:33:04.660Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:33:04.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:33:04 smithi083 ceph-mon[77242]: pgmap v9144: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:05.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:33:04 smithi153 ceph-mon[70044]: pgmap v9144: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:05.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:33:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:33:05.367Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:33:07.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:33:06 smithi153 ceph-mon[70044]: pgmap v9145: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:07.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:33:06 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:33:07.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:33:06 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:33:07.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:33:06 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:33:07.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:33:06 smithi083 ceph-mon[79930]: pgmap v9145: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:07.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:33:06 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:33:07.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:33:06 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:33:07.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:33:06 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:33:07.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:33:06 smithi083 ceph-mon[77242]: pgmap v9145: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:07.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:33:06 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:33:07.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:33:06 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:33:07.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:33:06 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:33:09.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:33:08 smithi153 ceph-mon[70044]: pgmap v9146: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:09.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:33:08 smithi083 ceph-mon[79930]: pgmap v9146: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:09.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:33:08 smithi083 ceph-mon[77242]: pgmap v9146: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:10.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:33:09 smithi153 ceph-mon[70044]: pgmap v9147: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:10.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:33:09 smithi083 ceph-mon[79930]: pgmap v9147: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:10.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:33:09 smithi083 ceph-mon[77242]: pgmap v9147: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:11.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:33:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:33:11] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T10:33:11.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:33:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:33:11.284Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:33:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:33:12 smithi153 ceph-mon[70044]: pgmap v9148: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:12.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:33:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:33:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:33:12.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:33:12 smithi083 ceph-mon[79930]: pgmap v9148: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:12.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:33:12 smithi083 ceph-mon[77242]: pgmap v9148: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:33:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:33:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:33:14.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:33:14 smithi083 ceph-mon[79930]: pgmap v9149: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:14.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:33:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:33:14.661Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:33:14.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:33:14 smithi083 ceph-mon[77242]: pgmap v9149: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:15.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:33:14 smithi153 ceph-mon[70044]: pgmap v9149: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:15.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:33:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:33:15.368Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:33:16.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:33:16 smithi083 ceph-mon[77242]: pgmap v9150: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:16.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:33:16 smithi083 ceph-mon[79930]: pgmap v9150: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:17.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:33:16 smithi153 ceph-mon[70044]: pgmap v9150: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:18.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:33:18 smithi083 ceph-mon[77242]: pgmap v9151: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:18.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:33:18 smithi083 ceph-mon[79930]: pgmap v9151: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:19.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:33:18 smithi153 ceph-mon[70044]: pgmap v9151: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:20.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:33:20 smithi083 ceph-mon[79930]: pgmap v9152: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:20.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:33:20 smithi083 ceph-mon[77242]: pgmap v9152: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:21.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:33:20 smithi153 ceph-mon[70044]: pgmap v9152: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:21.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:33:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:33:21] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T10:33:21.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:33:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:33:21.284Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:33:22.705 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:33:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:33:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:33:22.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:33:22 smithi083 ceph-mon[79930]: pgmap v9153: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:22.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:33:22 smithi083 ceph-mon[77242]: pgmap v9153: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:23.022 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:33:22 smithi153 ceph-mon[70044]: pgmap v9153: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:33:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:33:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:33:24.952 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:33:24.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:33:24 smithi083 ceph-mon[79930]: pgmap v9154: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:24.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:33:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:33:24.661Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:33:24.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:33:24 smithi083 ceph-mon[77242]: pgmap v9154: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:25.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:33:24 smithi153 ceph-mon[70044]: pgmap v9154: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:25.243 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:33:25.243 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 2m ago 5h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:33:25.243 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 19s ago 5h 62.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:33:25.244 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 2m ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:33:25.244 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 19s ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:33:25.244 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 19s ago 5h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:33:25.244 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 2m ago 5h 774M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:33:25.244 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 2m ago 5h 1165M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:33:25.245 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 19s ago 5h 501M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:33:25.245 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 2m ago 5h 669M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:33:25.245 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 2m ago 5h 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:33:25.245 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 19s ago 5h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:33:25.246 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 2m ago 5h 819M 2474M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:33:25.246 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 2m ago 5h 809M 2474M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:33:25.246 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 2m ago 5h 632M 2474M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:33:25.246 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 2m ago 5h 789M 2474M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:33:25.247 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 19s ago 5h 696M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:33:25.247 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 19s ago 5h 867M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:33:25.247 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (4h) 19s ago 5h 774M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:33:25.247 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 19s ago 5h 916M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:33:25.248 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 19s ago 5h 187M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:33:25.601 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:33:25.601 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:33:25.602 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:33:25.602 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:33:25.602 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:33:25.602 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:33:25.603 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:33:25.603 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:33:25.603 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:33:25.603 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:33:25.604 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:33:25.604 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:33:25.604 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:33:25.604 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:33:25.604 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:33:25.605 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:33:25.605 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:33:25.605 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:33:25.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:33:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:33:25.368Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:33:26.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:33:25 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3102316932' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:33:26.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:33:25 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3102316932' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:33:26.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:33:25 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3102316932' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:33:27.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:33:26 smithi153 ceph-mon[70044]: from='client.54712 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:33:27.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:33:26 smithi153 ceph-mon[70044]: pgmap v9155: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:27.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:33:26 smithi153 ceph-mon[70044]: from='client.44787 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:33:27.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:33:26 smithi083 ceph-mon[79930]: from='client.54712 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:33:27.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:33:26 smithi083 ceph-mon[79930]: pgmap v9155: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:27.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:33:26 smithi083 ceph-mon[79930]: from='client.44787 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:33:27.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:33:26 smithi083 ceph-mon[77242]: from='client.54712 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:33:27.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:33:26 smithi083 ceph-mon[77242]: pgmap v9155: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:27.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:33:26 smithi083 ceph-mon[77242]: from='client.44787 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:33:29.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:33:28 smithi153 ceph-mon[70044]: pgmap v9156: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:29.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:33:28 smithi083 ceph-mon[79930]: pgmap v9156: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:29.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:33:28 smithi083 ceph-mon[77242]: pgmap v9156: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:30.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:33:29 smithi153 ceph-mon[70044]: pgmap v9157: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:30.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:33:29 smithi083 ceph-mon[77242]: pgmap v9157: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:30.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:33:29 smithi083 ceph-mon[79930]: pgmap v9157: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:31.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:33:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:33:31] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T10:33:31.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:33:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:33:31.284Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:33:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:33:32 smithi153 ceph-mon[70044]: pgmap v9158: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:32.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:33:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:33:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:33:32.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:33:32 smithi083 ceph-mon[77242]: pgmap v9158: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:32.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:33:32 smithi083 ceph-mon[79930]: pgmap v9158: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:33:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:33:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:33:34.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:33:34 smithi083 ceph-mon[79930]: pgmap v9159: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:34.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:33:34 smithi083 ceph-mon[77242]: pgmap v9159: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:34.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:33:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:33:34.662Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:33:35.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:33:34 smithi153 ceph-mon[70044]: pgmap v9159: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:35.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:33:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:33:35.369Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:33:36.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:33:36 smithi083 ceph-mon[79930]: pgmap v9160: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:36.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:33:36 smithi083 ceph-mon[77242]: pgmap v9160: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:37.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:33:36 smithi153 ceph-mon[70044]: pgmap v9160: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:38.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:33:38 smithi083 ceph-mon[77242]: pgmap v9161: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:38.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:33:38 smithi083 ceph-mon[79930]: pgmap v9161: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:39.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:33:38 smithi153 ceph-mon[70044]: pgmap v9161: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:40.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:33:40 smithi083 ceph-mon[77242]: pgmap v9162: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:40.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:33:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:33:40.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:33:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:33:40.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:33:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:33:40.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:33:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:33:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:33:40 smithi083 ceph-mon[79930]: pgmap v9162: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:33:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:33:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:33:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:33:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:33:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:33:40.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:33:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:33:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:33:40 smithi153 ceph-mon[70044]: pgmap v9162: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:33:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:33:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:33:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:33:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:33:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:33:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:33:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:33:41.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:33:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:33:41] "GET /metrics HTTP/1.1" 200 225229 "" "Prometheus/2.33.4" 2022-09-29T10:33:41.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:33:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:33:41.285Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:33:42.707 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:33:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:33:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:33:42.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:33:42 smithi083 ceph-mon[77242]: pgmap v9163: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:42.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:33:42 smithi083 ceph-mon[79930]: pgmap v9163: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:43.022 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:33:42 smithi153 ceph-mon[70044]: pgmap v9163: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:33:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:33:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:33:44.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:33:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:33:44.663Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:33:44.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:33:44 smithi083 ceph-mon[77242]: pgmap v9164: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:44.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:33:44 smithi083 ceph-mon[79930]: pgmap v9164: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:45.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:33:44 smithi153 ceph-mon[70044]: pgmap v9164: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:45.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:33:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:33:45.369Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:33:47.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:33:46 smithi153 ceph-mon[70044]: pgmap v9165: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:47.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:33:46 smithi083 ceph-mon[79930]: pgmap v9165: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:47.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:33:46 smithi083 ceph-mon[77242]: pgmap v9165: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:49.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:33:48 smithi153 ceph-mon[70044]: pgmap v9166: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:49.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:33:48 smithi083 ceph-mon[79930]: pgmap v9166: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:49.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:33:48 smithi083 ceph-mon[77242]: pgmap v9166: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:51.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:33:50 smithi153 ceph-mon[70044]: pgmap v9167: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:51.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:33:50 smithi083 ceph-mon[79930]: pgmap v9167: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:51.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:33:50 smithi083 ceph-mon[77242]: pgmap v9167: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:51.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:33:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:33:51] "GET /metrics HTTP/1.1" 200 225229 "" "Prometheus/2.33.4" 2022-09-29T10:33:51.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:33:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:33:51.285Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:33:52.722 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:33:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:33:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:33:53.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:33:52 smithi153 ceph-mon[70044]: pgmap v9168: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:53.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:33:52 smithi083 ceph-mon[79930]: pgmap v9168: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:53.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:33:52 smithi083 ceph-mon[77242]: pgmap v9168: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:33:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:33:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:33:54.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:33:54 smithi083 ceph-mon[79930]: pgmap v9169: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:54.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:33:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:33:54.664Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:33:54.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:33:54 smithi083 ceph-mon[77242]: pgmap v9169: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:55.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:33:54 smithi153 ceph-mon[70044]: pgmap v9169: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:55.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:33:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:33:55.370Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:33:55.930 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:33:56.217 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:33:56.217 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 2m ago 5h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:33:56.218 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 50s ago 5h 62.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:33:56.218 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 2m ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:33:56.218 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 50s ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:33:56.218 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 50s ago 5h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:33:56.218 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 2m ago 5h 774M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:33:56.219 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 2m ago 5h 1165M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:33:56.219 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 50s ago 5h 501M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:33:56.219 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 2m ago 5h 669M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:33:56.219 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 2m ago 5h 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:33:56.219 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 50s ago 5h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:33:56.220 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 2m ago 5h 819M 2474M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:33:56.220 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 2m ago 5h 809M 2474M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:33:56.220 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 2m ago 5h 632M 2474M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:33:56.220 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 2m ago 5h 789M 2474M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:33:56.221 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 50s ago 5h 696M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:33:56.221 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 50s ago 5h 867M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:33:56.221 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 50s ago 5h 774M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:33:56.221 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (4h) 50s ago 5h 916M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:33:56.221 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 50s ago 5h 187M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:33:56.575 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:33:56.575 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:33:56.576 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:33:56.576 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:33:56.576 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:33:56.577 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:33:56.577 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:33:56.577 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:33:56.577 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:33:56.578 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:33:56.578 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:33:56.578 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:33:56.578 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:33:56.579 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:33:56.579 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:33:56.579 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:33:56.579 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:33:56.580 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:33:56.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:33:56 smithi083 ceph-mon[77242]: pgmap v9170: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:56.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:33:56 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2667542539' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:33:56.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:33:56 smithi083 ceph-mon[79930]: pgmap v9170: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:56.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:33:56 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2667542539' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:33:57.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:33:56 smithi153 ceph-mon[70044]: pgmap v9170: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:57.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:33:56 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2667542539' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:33:57.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:33:57 smithi083 ceph-mon[79930]: from='client.44799 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:33:57.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:33:57 smithi083 ceph-mon[79930]: from='client.54736 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:33:57.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:33:57 smithi083 ceph-mon[77242]: from='client.44799 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:33:57.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:33:57 smithi083 ceph-mon[77242]: from='client.54736 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:33:58.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:33:57 smithi153 ceph-mon[70044]: from='client.44799 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:33:58.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:33:57 smithi153 ceph-mon[70044]: from='client.54736 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:33:59.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:33:58 smithi153 ceph-mon[70044]: pgmap v9171: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:59.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:33:58 smithi083 ceph-mon[79930]: pgmap v9171: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:33:59.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:33:58 smithi083 ceph-mon[77242]: pgmap v9171: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:01.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:34:00 smithi153 ceph-mon[70044]: pgmap v9172: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:01.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:34:00 smithi083 ceph-mon[79930]: pgmap v9172: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:01.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:34:00 smithi083 ceph-mon[77242]: pgmap v9172: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:01.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:34:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:34:01] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T10:34:01.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:34:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:34:01.286Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:34:02.720 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:34:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:34:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:34:03.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:34:02 smithi153 ceph-mon[70044]: pgmap v9173: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:03.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:34:02 smithi083 ceph-mon[77242]: pgmap v9173: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:03.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:34:02 smithi083 ceph-mon[79930]: pgmap v9173: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:34:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:34:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:34:04.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:34:04 smithi083 ceph-mon[79930]: pgmap v9174: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:04.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:34:04 smithi083 ceph-mon[77242]: pgmap v9174: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:04.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:34:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:34:04.664Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:34:05.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:34:04 smithi153 ceph-mon[70044]: pgmap v9174: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:05.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:34:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:34:05.371Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:34:06.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:34:05 smithi153 ceph-mon[70044]: pgmap v9175: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:06.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:34:05 smithi083 ceph-mon[79930]: pgmap v9175: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:06.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:34:05 smithi083 ceph-mon[77242]: pgmap v9175: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:07.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:34:06 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:34:07.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:34:06 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:34:07.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:34:06 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:34:07.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:34:06 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:34:07.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:34:06 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:34:07.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:34:06 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:34:07.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:34:06 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:34:07.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:34:06 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:34:07.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:34:06 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:34:08.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:34:08 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:34:08.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:34:08 smithi153 ceph-mon[70044]: pgmap v9176: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:08.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:34:08 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:34:08.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:34:08 smithi083 ceph-mon[79930]: pgmap v9176: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:08.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:34:08 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:34:08.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:34:08 smithi083 ceph-mon[77242]: pgmap v9176: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:10.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:34:10 smithi083 ceph-mon[79930]: pgmap v9177: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:10.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:34:10 smithi083 ceph-mon[77242]: pgmap v9177: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:11.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:34:10 smithi153 ceph-mon[70044]: pgmap v9177: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:11.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:34:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:34:11] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T10:34:11.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:34:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:34:11.286Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:34:12.714 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:34:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:34:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:34:13.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:34:12 smithi153 ceph-mon[70044]: pgmap v9178: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:13.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:34:12 smithi083 ceph-mon[79930]: pgmap v9178: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:13.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:34:12 smithi083 ceph-mon[77242]: pgmap v9178: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:34:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:34:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:34:14.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:34:14 smithi083 ceph-mon[79930]: pgmap v9179: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:14.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:34:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:34:14.665Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:34:14.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:34:14 smithi083 ceph-mon[77242]: pgmap v9179: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:15.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:34:14 smithi153 ceph-mon[70044]: pgmap v9179: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:15.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:34:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:34:15.372Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:34:17.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:34:16 smithi153 ceph-mon[70044]: pgmap v9180: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:17.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:34:16 smithi083 ceph-mon[77242]: pgmap v9180: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:17.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:34:16 smithi083 ceph-mon[79930]: pgmap v9180: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:19.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:34:18 smithi153 ceph-mon[70044]: pgmap v9181: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:19.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:34:18 smithi083 ceph-mon[79930]: pgmap v9181: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:19.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:34:18 smithi083 ceph-mon[77242]: pgmap v9181: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:20.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:34:19 smithi153 ceph-mon[70044]: pgmap v9182: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:20.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:34:19 smithi083 ceph-mon[79930]: pgmap v9182: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:20.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:34:19 smithi083 ceph-mon[77242]: pgmap v9182: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:21.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:34:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:34:21] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T10:34:21.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:34:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:34:21.287Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:34:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:34:22 smithi153 ceph-mon[70044]: pgmap v9183: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:22.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:34:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:34:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:34:22.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:34:22 smithi083 ceph-mon[79930]: pgmap v9183: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:22.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:34:22 smithi083 ceph-mon[77242]: pgmap v9183: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:34:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:34:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:34:24.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:34:24 smithi083 ceph-mon[79930]: pgmap v9184: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:24.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:34:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:34:24.666Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:34:24.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:34:24 smithi083 ceph-mon[77242]: pgmap v9184: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:25.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:34:24 smithi153 ceph-mon[70044]: pgmap v9184: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:25.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:34:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:34:25.373Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:34:26.906 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:34:27.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:34:26 smithi153 ceph-mon[70044]: pgmap v9185: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:27.196 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:34:27.196 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 3m ago 5h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:34:27.196 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 81s ago 5h 62.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:34:27.196 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 3m ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:34:27.197 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 81s ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:34:27.197 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 81s ago 5h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:34:27.197 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 3m ago 5h 774M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:34:27.197 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 3m ago 5h 1165M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:34:27.198 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 81s ago 5h 501M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:34:27.198 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 3m ago 5h 669M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:34:27.198 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 3m ago 5h 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:34:27.198 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 81s ago 5h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:34:27.198 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 3m ago 5h 819M 2474M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:34:27.199 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 3m ago 5h 809M 2474M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:34:27.199 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 3m ago 5h 632M 2474M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:34:27.199 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 3m ago 5h 789M 2474M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:34:27.199 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 81s ago 5h 696M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:34:27.200 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 81s ago 5h 867M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:34:27.200 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 81s ago 5h 774M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:34:27.200 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 81s ago 5h 916M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:34:27.200 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 81s ago 5h 187M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:34:27.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:34:26 smithi083 ceph-mon[79930]: pgmap v9185: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:27.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:34:26 smithi083 ceph-mon[77242]: pgmap v9185: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:27.549 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:34:27.550 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:34:27.550 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:34:27.550 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:34:27.551 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:34:27.551 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:34:27.551 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:34:27.551 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:34:27.552 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:34:27.552 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:34:27.552 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:34:27.552 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:34:27.553 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:34:27.553 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:34:27.553 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:34:27.553 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:34:27.554 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:34:27.554 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:34:28.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:34:27 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3700330156' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:34:28.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:34:27 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3700330156' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:34:28.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:34:27 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3700330156' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:34:29.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:34:28 smithi153 ceph-mon[70044]: from='client.54748 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:34:29.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:34:28 smithi153 ceph-mon[70044]: from='client.44823 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:34:29.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:34:28 smithi153 ceph-mon[70044]: pgmap v9186: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:29.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:34:28 smithi083 ceph-mon[79930]: from='client.54748 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:34:29.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:34:28 smithi083 ceph-mon[79930]: from='client.44823 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:34:29.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:34:28 smithi083 ceph-mon[79930]: pgmap v9186: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:29.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:34:28 smithi083 ceph-mon[77242]: from='client.54748 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:34:29.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:34:28 smithi083 ceph-mon[77242]: from='client.44823 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:34:29.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:34:28 smithi083 ceph-mon[77242]: pgmap v9186: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:30.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:34:29 smithi153 ceph-mon[70044]: pgmap v9187: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:30.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:34:29 smithi083 ceph-mon[79930]: pgmap v9187: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:30.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:34:29 smithi083 ceph-mon[77242]: pgmap v9187: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:31.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:34:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:34:31] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T10:34:31.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:34:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:34:31.288Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:34:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:34:32 smithi153 ceph-mon[70044]: pgmap v9188: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:32.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:34:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:34:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:34:32.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:34:32 smithi083 ceph-mon[77242]: pgmap v9188: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:32.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:34:32 smithi083 ceph-mon[79930]: pgmap v9188: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:34:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:34:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:34:34.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:34:34 smithi083 ceph-mon[77242]: pgmap v9189: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:34.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:34:34 smithi083 ceph-mon[79930]: pgmap v9189: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:34.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:34:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:34:34.667Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:34:35.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:34:34 smithi153 ceph-mon[70044]: pgmap v9189: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:35.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:34:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:34:35.374Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:34:37.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:34:36 smithi153 ceph-mon[70044]: pgmap v9190: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:37.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:34:36 smithi083 ceph-mon[79930]: pgmap v9190: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:37.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:34:36 smithi083 ceph-mon[77242]: pgmap v9190: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:39.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:34:38 smithi153 ceph-mon[70044]: pgmap v9191: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:39.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:34:38 smithi083 ceph-mon[77242]: pgmap v9191: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:39.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:34:38 smithi083 ceph-mon[79930]: pgmap v9191: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:34:40 smithi153 ceph-mon[70044]: pgmap v9192: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:34:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:34:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:34:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:34:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:34:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:34:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:34:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:34:41.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:34:40 smithi083 ceph-mon[79930]: pgmap v9192: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:41.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:34:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:34:41.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:34:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:34:41.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:34:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:34:41.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:34:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:34:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:34:40 smithi083 ceph-mon[77242]: pgmap v9192: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:34:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:34:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:34:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:34:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:34:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:34:41.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:34:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:34:41.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:34:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:34:41] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T10:34:41.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:34:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:34:41.288Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:34:42.722 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:34:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:34:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:34:43.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:34:42 smithi153 ceph-mon[70044]: pgmap v9193: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:43.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:34:42 smithi083 ceph-mon[79930]: pgmap v9193: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:43.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:34:42 smithi083 ceph-mon[77242]: pgmap v9193: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:34:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:34:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:34:44.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:34:44 smithi083 ceph-mon[79930]: pgmap v9194: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:44.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:34:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:34:44.667Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:34:44.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:34:44 smithi083 ceph-mon[77242]: pgmap v9194: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:45.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:34:44 smithi153 ceph-mon[70044]: pgmap v9194: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:45.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:34:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:34:45.373Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:34:47.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:34:46 smithi153 ceph-mon[70044]: pgmap v9195: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:47.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:34:46 smithi083 ceph-mon[79930]: pgmap v9195: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:47.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:34:46 smithi083 ceph-mon[77242]: pgmap v9195: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:48.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:34:47 smithi153 ceph-mon[70044]: pgmap v9196: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:48.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:34:47 smithi083 ceph-mon[77242]: pgmap v9196: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:48.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:34:47 smithi083 ceph-mon[79930]: pgmap v9196: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:51.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:34:50 smithi153 ceph-mon[70044]: pgmap v9197: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:51.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:34:50 smithi083 ceph-mon[77242]: pgmap v9197: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:51.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:34:50 smithi083 ceph-mon[79930]: pgmap v9197: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:51.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:34:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:34:51] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T10:34:51.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:34:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:34:51.289Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:34:52.725 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:34:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:34:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:34:53.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:34:52 smithi153 ceph-mon[70044]: pgmap v9198: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:53.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:34:52 smithi083 ceph-mon[79930]: pgmap v9198: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:53.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:34:52 smithi083 ceph-mon[77242]: pgmap v9198: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:34:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:34:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:34:54.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:34:54 smithi083 ceph-mon[79930]: pgmap v9199: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:54.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:34:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:34:54.668Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:34:54.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:34:54 smithi083 ceph-mon[77242]: pgmap v9199: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:55.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:34:54 smithi153 ceph-mon[70044]: pgmap v9199: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:55.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:34:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:34:55.374Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:34:57.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:34:56 smithi153 ceph-mon[70044]: pgmap v9200: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:57.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:34:56 smithi083 ceph-mon[79930]: pgmap v9200: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:57.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:34:56 smithi083 ceph-mon[77242]: pgmap v9200: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:57.875 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:34:58.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:34:57 smithi153 ceph-mon[70044]: pgmap v9201: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:58.163 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:34:58.163 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 3m ago 5h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:34:58.164 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 112s ago 5h 62.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:34:58.164 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 3m ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:34:58.164 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 112s ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:34:58.164 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 112s ago 5h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:34:58.165 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 3m ago 5h 774M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:34:58.165 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 3m ago 5h 1165M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:34:58.165 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 112s ago 5h 501M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:34:58.165 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 3m ago 5h 669M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:34:58.165 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 3m ago 5h 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:34:58.166 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 112s ago 5h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:34:58.166 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 3m ago 5h 819M 2474M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:34:58.166 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 3m ago 5h 809M 2474M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:34:58.166 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 3m ago 5h 632M 2474M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:34:58.166 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 3m ago 5h 789M 2474M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:34:58.167 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 112s ago 5h 696M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:34:58.167 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 112s ago 5h 867M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:34:58.167 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 112s ago 5h 774M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:34:58.167 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 112s ago 5h 916M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:34:58.167 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 112s ago 5h 187M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:34:58.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:34:57 smithi083 ceph-mon[77242]: pgmap v9201: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:58.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:34:57 smithi083 ceph-mon[79930]: pgmap v9201: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:34:58.518 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:34:58.519 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:34:58.519 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:34:58.519 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:34:58.519 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:34:58.519 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:34:58.520 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:34:58.520 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:34:58.520 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:34:58.520 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:34:58.520 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:34:58.521 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:34:58.521 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:34:58.521 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:34:58.521 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:34:58.521 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:34:58.522 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:34:58.522 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:34:59.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:34:58 smithi153 ceph-mon[70044]: from='client.54766 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:34:59.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:34:58 smithi153 ceph-mon[70044]: from='client.54772 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:34:59.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:34:58 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1362062499' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:34:59.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:34:58 smithi083 ceph-mon[79930]: from='client.54766 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:34:59.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:34:58 smithi083 ceph-mon[79930]: from='client.54772 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:34:59.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:34:58 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1362062499' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:34:59.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:34:58 smithi083 ceph-mon[77242]: from='client.54766 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:34:59.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:34:58 smithi083 ceph-mon[77242]: from='client.54772 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:34:59.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:34:58 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1362062499' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:35:00.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:34:59 smithi153 ceph-mon[70044]: pgmap v9202: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:00.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:34:59 smithi083 ceph-mon[79930]: pgmap v9202: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:00.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:34:59 smithi083 ceph-mon[77242]: pgmap v9202: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:01.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:35:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:35:01] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T10:35:01.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:35:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:35:01.290Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:35:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:35:02 smithi153 ceph-mon[70044]: pgmap v9203: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:02.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:35:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:35:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:35:02.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:35:02 smithi083 ceph-mon[79930]: pgmap v9203: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:02.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:35:02 smithi083 ceph-mon[77242]: pgmap v9203: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:35:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:35:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:35:04.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:35:04 smithi083 ceph-mon[79930]: pgmap v9204: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:04.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:35:04 smithi083 ceph-mon[77242]: pgmap v9204: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:04.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:35:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:35:04.669Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:35:05.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:35:04 smithi153 ceph-mon[70044]: pgmap v9204: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:05.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:35:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:35:05.374Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:35:07.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:35:06 smithi153 ceph-mon[70044]: pgmap v9205: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:07.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:35:06 smithi083 ceph-mon[79930]: pgmap v9205: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:07.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:35:06 smithi083 ceph-mon[77242]: pgmap v9205: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:08.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:35:07 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:35:08.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:35:07 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:35:08.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:35:07 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:35:08.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:35:07 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:35:08.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:35:07 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:35:08.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:35:07 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:35:08.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:35:07 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:35:08.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:35:07 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:35:08.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:35:07 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:35:08.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:35:07 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:35:08.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:35:07 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:35:08.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:35:07 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:35:09.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:35:08 smithi153 ceph-mon[70044]: pgmap v9206: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:09.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:35:08 smithi083 ceph-mon[79930]: pgmap v9206: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:09.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:35:08 smithi083 ceph-mon[77242]: pgmap v9206: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:10.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:35:09 smithi153 ceph-mon[70044]: pgmap v9207: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:10.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:35:09 smithi083 ceph-mon[79930]: pgmap v9207: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:10.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:35:09 smithi083 ceph-mon[77242]: pgmap v9207: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:11.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:35:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:35:11] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T10:35:11.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:35:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:35:11.291Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:35:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:35:12 smithi153 ceph-mon[70044]: pgmap v9208: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:12.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:35:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:35:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:35:12.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:35:12 smithi083 ceph-mon[79930]: pgmap v9208: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:12.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:35:12 smithi083 ceph-mon[77242]: pgmap v9208: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:35:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:35:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:35:14.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:35:14 smithi083 ceph-mon[79930]: pgmap v9209: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:14.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:35:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:35:14.670Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:35:14.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:35:14 smithi083 ceph-mon[77242]: pgmap v9209: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:15.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:35:14 smithi153 ceph-mon[70044]: pgmap v9209: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:15.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:35:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:35:15.374Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:35:17.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:35:16 smithi153 ceph-mon[70044]: pgmap v9210: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:17.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:35:16 smithi083 ceph-mon[79930]: pgmap v9210: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:17.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:35:16 smithi083 ceph-mon[77242]: pgmap v9210: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:19.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:35:18 smithi153 ceph-mon[70044]: pgmap v9211: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:19.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:35:18 smithi083 ceph-mon[79930]: pgmap v9211: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:19.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:35:18 smithi083 ceph-mon[77242]: pgmap v9211: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:21.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:35:20 smithi153 ceph-mon[70044]: pgmap v9212: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:21.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:35:20 smithi083 ceph-mon[79930]: pgmap v9212: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:21.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:35:20 smithi083 ceph-mon[77242]: pgmap v9212: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:21.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:35:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:35:21] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T10:35:21.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:35:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:35:21.292Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:35:22.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:35:21 smithi153 ceph-mon[70044]: pgmap v9213: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:22.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:35:21 smithi083 ceph-mon[77242]: pgmap v9213: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:22.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:35:21 smithi083 ceph-mon[79930]: pgmap v9213: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:35:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:35:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:35:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:35:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:35:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:35:24.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:35:24 smithi083 ceph-mon[79930]: pgmap v9214: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:24.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:35:24 smithi083 ceph-mon[77242]: pgmap v9214: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:24.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:35:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:35:24.670Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:35:25.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:35:24 smithi153 ceph-mon[70044]: pgmap v9214: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:25.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:35:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:35:25.376Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:35:27.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:35:26 smithi153 ceph-mon[70044]: pgmap v9215: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:27.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:35:26 smithi083 ceph-mon[79930]: pgmap v9215: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:27.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:35:26 smithi083 ceph-mon[77242]: pgmap v9215: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:28.846 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:35:29.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:35:28 smithi153 ceph-mon[70044]: pgmap v9216: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:29.132 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:35:29.132 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 4m ago 5h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:35:29.133 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 2m ago 5h 62.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:35:29.133 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 4m ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:35:29.133 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 2m ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:35:29.133 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 2m ago 5h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:35:29.134 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 4m ago 5h 774M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:35:29.134 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 4m ago 5h 1165M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:35:29.134 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 2m ago 5h 501M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:35:29.134 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 4m ago 5h 669M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:35:29.135 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 4m ago 5h 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:35:29.135 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 2m ago 5h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:35:29.135 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 4m ago 5h 819M 2474M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:35:29.135 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 4m ago 5h 809M 2474M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:35:29.135 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 4m ago 5h 632M 2474M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:35:29.136 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 4m ago 5h 789M 2474M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:35:29.136 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 2m ago 5h 696M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:35:29.136 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 2m ago 5h 867M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:35:29.136 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 2m ago 5h 774M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:35:29.137 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 2m ago 5h 916M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:35:29.137 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 2m ago 5h 187M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:35:29.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:35:28 smithi083 ceph-mon[77242]: pgmap v9216: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:29.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:35:28 smithi083 ceph-mon[79930]: pgmap v9216: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:29.488 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:35:29.489 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:35:29.489 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:35:29.489 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:35:29.489 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:35:29.490 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:35:29.490 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:35:29.490 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:35:29.490 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:35:29.491 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:35:29.491 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:35:29.491 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:35:29.491 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:35:29.491 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:35:29.492 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:35:29.492 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:35:29.492 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:35:29.492 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:35:30.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:35:29 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2323759765' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:35:30.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:35:29 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2323759765' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:35:30.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:35:29 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2323759765' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:35:31.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:35:30 smithi153 ceph-mon[70044]: from='client.54784 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:35:31.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:35:30 smithi153 ceph-mon[70044]: from='client.44859 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:35:31.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:35:30 smithi153 ceph-mon[70044]: pgmap v9217: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:31.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:35:30 smithi083 ceph-mon[79930]: from='client.54784 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:35:31.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:35:30 smithi083 ceph-mon[79930]: from='client.44859 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:35:31.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:35:30 smithi083 ceph-mon[79930]: pgmap v9217: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:31.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:35:30 smithi083 ceph-mon[77242]: from='client.54784 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:35:31.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:35:30 smithi083 ceph-mon[77242]: from='client.44859 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:35:31.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:35:30 smithi083 ceph-mon[77242]: pgmap v9217: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:31.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:35:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:35:31] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T10:35:31.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:35:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:35:31.292Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:35:32.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:35:31 smithi153 ceph-mon[70044]: pgmap v9218: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:32.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:35:31 smithi083 ceph-mon[77242]: pgmap v9218: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:32.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:35:31 smithi083 ceph-mon[79930]: pgmap v9218: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:35:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:35:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:35:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:35:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:35:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:35:34.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:35:34 smithi083 ceph-mon[79930]: pgmap v9219: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:34.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:35:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:35:34.670Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:35:34.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:35:34 smithi083 ceph-mon[77242]: pgmap v9219: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:35.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:35:34 smithi153 ceph-mon[70044]: pgmap v9219: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:35.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:35:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:35:35.375Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:35:37.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:35:36 smithi153 ceph-mon[70044]: pgmap v9220: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:37.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:35:36 smithi083 ceph-mon[79930]: pgmap v9220: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:37.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:35:36 smithi083 ceph-mon[77242]: pgmap v9220: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:39.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:35:38 smithi153 ceph-mon[70044]: pgmap v9221: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:39.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:35:38 smithi083 ceph-mon[79930]: pgmap v9221: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:39.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:35:38 smithi083 ceph-mon[77242]: pgmap v9221: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:35:40 smithi153 ceph-mon[70044]: pgmap v9222: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:35:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:35:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:35:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:35:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:35:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:35:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:35:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:35:41.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:35:40 smithi083 ceph-mon[77242]: pgmap v9222: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:41.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:35:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:35:41.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:35:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:35:41.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:35:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:35:41.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:35:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:35:41.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:35:40 smithi083 ceph-mon[79930]: pgmap v9222: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:41.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:35:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:35:41.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:35:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:35:41.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:35:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:35:41.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:35:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:35:41.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:35:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:35:41] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T10:35:41.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:35:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:35:41.293Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:35:42.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:35:41 smithi153 ceph-mon[70044]: pgmap v9223: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:42.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:35:41 smithi083 ceph-mon[79930]: pgmap v9223: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:42.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:35:41 smithi083 ceph-mon[77242]: pgmap v9223: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:35:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:35:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:35:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:35:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:35:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:35:44.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:35:44 smithi083 ceph-mon[79930]: pgmap v9224: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:44.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:35:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:35:44.671Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:35:44.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:35:44 smithi083 ceph-mon[77242]: pgmap v9224: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:45.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:35:44 smithi153 ceph-mon[70044]: pgmap v9224: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:45.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:35:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:35:45.375Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:35:47.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:35:46 smithi153 ceph-mon[70044]: pgmap v9225: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:47.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:35:46 smithi083 ceph-mon[79930]: pgmap v9225: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:47.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:35:46 smithi083 ceph-mon[77242]: pgmap v9225: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:49.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:35:48 smithi153 ceph-mon[70044]: pgmap v9226: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:49.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:35:48 smithi083 ceph-mon[79930]: pgmap v9226: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:49.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:35:48 smithi083 ceph-mon[77242]: pgmap v9226: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:51.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:35:50 smithi153 ceph-mon[70044]: pgmap v9227: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:51.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:35:50 smithi083 ceph-mon[79930]: pgmap v9227: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:51.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:35:50 smithi083 ceph-mon[77242]: pgmap v9227: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:51.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:35:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:35:51] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T10:35:51.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:35:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:35:51.293Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:35:52.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:35:51 smithi153 ceph-mon[70044]: pgmap v9228: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:52.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:35:51 smithi083 ceph-mon[79930]: pgmap v9228: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:52.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:35:51 smithi083 ceph-mon[77242]: pgmap v9228: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:35:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:35:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:35:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:35:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:35:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:35:54.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:35:54 smithi083 ceph-mon[79930]: pgmap v9229: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:54.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:35:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:35:54.671Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:35:54.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:35:54 smithi083 ceph-mon[77242]: pgmap v9229: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:55.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:35:54 smithi153 ceph-mon[70044]: pgmap v9229: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:55.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:35:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:35:55.376Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:35:57.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:35:56 smithi153 ceph-mon[70044]: pgmap v9230: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:57.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:35:56 smithi083 ceph-mon[79930]: pgmap v9230: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:57.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:35:56 smithi083 ceph-mon[77242]: pgmap v9230: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:59.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:35:58 smithi153 ceph-mon[70044]: pgmap v9231: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:59.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:35:58 smithi083 ceph-mon[79930]: pgmap v9231: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:59.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:35:58 smithi083 ceph-mon[77242]: pgmap v9231: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:35:59.810 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:36:00.101 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:36:00.101 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 5m ago 5h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:36:00.102 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 2m ago 5h 62.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:36:00.102 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 5m ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:36:00.102 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 2m ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:36:00.102 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 2m ago 5h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:36:00.102 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 5m ago 5h 774M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:36:00.103 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 5m ago 5h 1165M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:36:00.103 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 2m ago 5h 501M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:36:00.103 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 5m ago 5h 669M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:36:00.103 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 5m ago 5h 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:36:00.103 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 2m ago 5h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:36:00.104 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 5m ago 5h 819M 2474M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:36:00.104 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 5m ago 5h 809M 2474M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:36:00.104 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 5m ago 5h 632M 2474M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:36:00.104 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 5m ago 5h 789M 2474M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:36:00.105 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 2m ago 5h 696M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:36:00.105 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 2m ago 5h 867M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:36:00.105 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 2m ago 5h 774M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:36:00.105 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 2m ago 5h 916M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:36:00.106 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 2m ago 5h 187M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:36:00.453 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:36:00.453 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:36:00.454 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:36:00.454 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:36:00.454 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:36:00.454 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:36:00.455 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:36:00.455 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:36:00.455 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:36:00.455 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:36:00.455 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:36:00.456 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:36:00.456 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:36:00.456 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:36:00.456 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:36:00.456 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:36:00.457 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:36:00.457 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:36:01.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:36:00 smithi153 ceph-mon[70044]: pgmap v9232: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:36:01.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:36:00 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1982912889' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:36:01.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:36:00 smithi083 ceph-mon[79930]: pgmap v9232: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:36:01.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:36:00 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1982912889' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:36:01.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:36:00 smithi083 ceph-mon[77242]: pgmap v9232: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:36:01.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:36:00 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1982912889' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:36:01.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:36:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:36:01] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T10:36:01.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:36:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:36:01.293Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:36:02.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:36:01 smithi153 ceph-mon[70044]: from='client.44871 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:36:02.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:36:01 smithi153 ceph-mon[70044]: from='client.44877 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:36:02.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:36:01 smithi153 ceph-mon[70044]: pgmap v9233: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:36:02.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:36:01 smithi083 ceph-mon[79930]: from='client.44871 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:36:02.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:36:01 smithi083 ceph-mon[79930]: from='client.44877 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:36:02.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:36:01 smithi083 ceph-mon[79930]: pgmap v9233: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:36:02.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:36:01 smithi083 ceph-mon[77242]: from='client.44871 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:36:02.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:36:01 smithi083 ceph-mon[77242]: from='client.44877 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:36:02.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:36:01 smithi083 ceph-mon[77242]: pgmap v9233: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:36:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:36:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:36:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:36:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:36:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:36:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:36:04.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:36:04 smithi083 ceph-mon[77242]: pgmap v9234: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:36:04.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:36:04 smithi083 ceph-mon[79930]: pgmap v9234: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:36:04.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:36:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:36:04.672Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:36:05.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:36:04 smithi153 ceph-mon[70044]: pgmap v9234: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:36:05.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:36:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:36:05.377Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:36:07.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:36:06 smithi153 ceph-mon[70044]: pgmap v9235: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:36:07.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:36:06 smithi083 ceph-mon[77242]: pgmap v9235: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:36:07.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:36:06 smithi083 ceph-mon[79930]: pgmap v9235: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:36:09.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:36:08 smithi153 ceph-mon[70044]: pgmap v9236: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:36:09.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:36:08 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:36:09.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:36:08 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:36:09.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:36:08 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:36:09.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:36:08 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:36:09.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:36:08 smithi083 ceph-mon[79930]: pgmap v9236: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:36:09.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:36:08 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:36:09.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:36:08 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:36:09.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:36:08 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:36:09.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:36:08 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:36:09.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:36:08 smithi083 ceph-mon[77242]: pgmap v9236: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:36:09.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:36:08 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:36:09.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:36:08 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:36:09.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:36:08 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:36:09.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:36:08 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:36:10.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:36:09 smithi153 ceph-mon[70044]: pgmap v9237: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:36:10.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:36:09 smithi083 ceph-mon[79930]: pgmap v9237: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:36:10.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:36:09 smithi083 ceph-mon[77242]: pgmap v9237: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:36:11.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:36:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:36:11] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T10:36:11.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:36:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:36:11.293Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:36:12.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:36:12 smithi083 ceph-mon[79930]: pgmap v9238: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:36:12.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:36:12 smithi083 ceph-mon[77242]: pgmap v9238: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:36:12.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:36:12 smithi153 ceph-mon[70044]: pgmap v9238: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:36:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:36:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:36:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:36:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:36:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:36:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:36:14.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:36:14 smithi083 ceph-mon[79930]: pgmap v9239: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:36:14.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:36:14 smithi083 ceph-mon[77242]: pgmap v9239: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:36:14.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:36:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:36:14.673Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:36:15.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:36:14 smithi153 ceph-mon[70044]: pgmap v9239: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:36:15.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:36:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:36:15.378Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:36:17.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:36:16 smithi153 ceph-mon[70044]: pgmap v9240: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:36:17.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:36:16 smithi083 ceph-mon[79930]: pgmap v9240: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:36:17.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:36:16 smithi083 ceph-mon[77242]: pgmap v9240: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:36:19.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:36:18 smithi153 ceph-mon[70044]: pgmap v9241: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:36:19.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:36:18 smithi083 ceph-mon[79930]: pgmap v9241: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:36:19.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:36:18 smithi083 ceph-mon[77242]: pgmap v9241: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:36:20.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:36:19 smithi153 ceph-mon[70044]: pgmap v9242: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:36:20.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:36:19 smithi083 ceph-mon[77242]: pgmap v9242: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:36:20.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:36:19 smithi083 ceph-mon[79930]: pgmap v9242: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:36:21.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:36:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:36:21.294Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:36:21.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:36:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:36:21] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T10:36:22.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:36:22 smithi083 ceph-mon[79930]: pgmap v9243: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:36:22.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:36:22 smithi083 ceph-mon[77242]: pgmap v9243: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:36:22.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:36:22 smithi153 ceph-mon[70044]: pgmap v9243: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:36:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:36:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:36:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:36:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:36:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:36:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:36:24.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:36:24 smithi083 ceph-mon[79930]: pgmap v9244: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:36:24.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:36:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:36:24.673Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:36:24.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:36:24 smithi083 ceph-mon[77242]: pgmap v9244: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:36:25.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:36:24 smithi153 ceph-mon[70044]: pgmap v9244: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:36:25.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:36:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:36:25.378Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:36:27.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:36:26 smithi153 ceph-mon[70044]: pgmap v9245: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:36:27.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:36:26 smithi083 ceph-mon[79930]: pgmap v9245: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:36:27.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:36:26 smithi083 ceph-mon[77242]: pgmap v9245: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:36:29.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:36:28 smithi153 ceph-mon[70044]: pgmap v9246: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:36:29.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:36:28 smithi083 ceph-mon[79930]: pgmap v9246: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:36:29.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:36:28 smithi083 ceph-mon[77242]: pgmap v9246: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:36:30.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:36:29 smithi153 ceph-mon[70044]: pgmap v9247: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:36:30.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:36:29 smithi083 ceph-mon[79930]: pgmap v9247: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:36:30.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:36:29 smithi083 ceph-mon[77242]: pgmap v9247: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:36:30.774 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:36:31.061 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:36:31.061 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 5m ago 5h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:36:31.061 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 3m ago 5h 62.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:36:31.062 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 5m ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:36:31.062 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 3m ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:36:31.062 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 3m ago 5h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:36:31.062 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 5m ago 5h 774M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:36:31.062 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 5m ago 5h 1165M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:36:31.063 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 3m ago 5h 501M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:36:31.063 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 5m ago 5h 669M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:36:31.063 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 5m ago 5h 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:36:31.063 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 3m ago 5h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:36:31.064 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 5m ago 5h 819M 2474M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:36:31.064 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 5m ago 5h 809M 2474M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:36:31.064 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 5m ago 5h 632M 2474M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:36:31.064 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 5m ago 5h 789M 2474M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:36:31.064 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 3m ago 5h 696M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:36:31.065 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 3m ago 5h 867M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:36:31.065 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 3m ago 5h 774M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:36:31.065 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 3m ago 5h 916M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:36:31.065 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 3m ago 5h 187M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:36:31.416 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:36:31.416 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:36:31.416 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:36:31.417 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:36:31.417 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:36:31.417 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:36:31.418 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:36:31.418 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:36:31.418 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:36:31.418 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:36:31.418 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:36:31.419 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:36:31.419 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:36:31.419 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:36:31.419 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:36:31.420 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:36:31.420 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:36:31.420 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:36:31.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:36:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:36:31] "GET /metrics HTTP/1.1" 200 225229 "" "Prometheus/2.33.4" 2022-09-29T10:36:31.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:36:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:36:31.294Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:36:32.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:36:32 smithi083 ceph-mon[79930]: from='client.44889 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:36:32.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:36:32 smithi083 ceph-mon[79930]: from='client.54826 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:36:32.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:36:32 smithi083 ceph-mon[79930]: pgmap v9248: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:36:32.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:36:32 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2128639325' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:36:32.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:36:32 smithi083 ceph-mon[77242]: from='client.44889 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:36:32.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:36:32 smithi083 ceph-mon[77242]: from='client.54826 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:36:32.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:36:32 smithi083 ceph-mon[77242]: pgmap v9248: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:36:32.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:36:32 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2128639325' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:36:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:36:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:36:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:36:32.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:36:32 smithi153 ceph-mon[70044]: from='client.44889 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:36:32.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:36:32 smithi153 ceph-mon[70044]: from='client.54826 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:36:32.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:36:32 smithi153 ceph-mon[70044]: pgmap v9248: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:36:32.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:36:32 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2128639325' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:36:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:36:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:36:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:36:34.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:36:34 smithi083 ceph-mon[79930]: pgmap v9249: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:36:34.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:36:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:36:34.673Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:36:34.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:36:34 smithi083 ceph-mon[77242]: pgmap v9249: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:36:35.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:36:34 smithi153 ceph-mon[70044]: pgmap v9249: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:36:35.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:36:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:36:35.379Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:36:37.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:36:36 smithi153 ceph-mon[70044]: pgmap v9250: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:36:37.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:36:36 smithi083 ceph-mon[77242]: pgmap v9250: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:36:37.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:36:36 smithi083 ceph-mon[79930]: pgmap v9250: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:36:38.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:36:37 smithi153 ceph-mon[70044]: pgmap v9251: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:36:38.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:36:37 smithi083 ceph-mon[79930]: pgmap v9251: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:36:38.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:36:37 smithi083 ceph-mon[77242]: pgmap v9251: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:36:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:36:40 smithi153 ceph-mon[70044]: pgmap v9252: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:36:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:36:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:36:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:36:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:36:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:36:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:36:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:36:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:36:41.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:36:40 smithi083 ceph-mon[79930]: pgmap v9252: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:36:41.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:36:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:36:41.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:36:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:36:41.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:36:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:36:41.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:36:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:36:41.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:36:40 smithi083 ceph-mon[77242]: pgmap v9252: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:36:41.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:36:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:36:41.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:36:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:36:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:36:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:36:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:36:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:36:41.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:36:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:36:41] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T10:36:41.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:36:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:36:41.295Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:36:42.744 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:36:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:36:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:36:43.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:36:42 smithi153 ceph-mon[70044]: pgmap v9253: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:36:43.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:36:42 smithi083 ceph-mon[79930]: pgmap v9253: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:36:43.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:36:42 smithi083 ceph-mon[77242]: pgmap v9253: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:36:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:36:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:36:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:36:44.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:36:44 smithi083 ceph-mon[79930]: pgmap v9254: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:36:44.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:36:44 smithi083 ceph-mon[77242]: pgmap v9254: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:36:44.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:36:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:36:44.674Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:36:45.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:36:44 smithi153 ceph-mon[70044]: pgmap v9254: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:36:45.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:36:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:36:45.379Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:36:47.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:36:46 smithi153 ceph-mon[70044]: pgmap v9255: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:36:47.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:36:46 smithi083 ceph-mon[77242]: pgmap v9255: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:36:47.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:36:46 smithi083 ceph-mon[79930]: pgmap v9255: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:36:48.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:36:47 smithi153 ceph-mon[70044]: pgmap v9256: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:36:48.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:36:47 smithi083 ceph-mon[77242]: pgmap v9256: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:36:48.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:36:47 smithi083 ceph-mon[79930]: pgmap v9256: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:36:51.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:36:50 smithi153 ceph-mon[70044]: pgmap v9257: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:36:51.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:36:50 smithi083 ceph-mon[77242]: pgmap v9257: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:36:51.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:36:50 smithi083 ceph-mon[79930]: pgmap v9257: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:36:51.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:36:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:36:51] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T10:36:51.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:36:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:36:51.297Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:36:52.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:36:51 smithi153 ceph-mon[70044]: pgmap v9258: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:36:52.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:36:51 smithi083 ceph-mon[79930]: pgmap v9258: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:36:52.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:36:51 smithi083 ceph-mon[77242]: pgmap v9258: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:36:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:36:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:36:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:36:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:36:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:36:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:36:54.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:36:54 smithi083 ceph-mon[79930]: pgmap v9259: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:36:54.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:36:54 smithi083 ceph-mon[77242]: pgmap v9259: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:36:54.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:36:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:36:54.675Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:36:55.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:36:54 smithi153 ceph-mon[70044]: pgmap v9259: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:36:55.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:36:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:36:55.380Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:36:56.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:36:55 smithi153 ceph-mon[70044]: pgmap v9260: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:36:56.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:36:55 smithi083 ceph-mon[79930]: pgmap v9260: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:36:56.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:36:55 smithi083 ceph-mon[77242]: pgmap v9260: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:36:58.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:36:58 smithi083 ceph-mon[79930]: pgmap v9261: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:36:58.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:36:58 smithi083 ceph-mon[77242]: pgmap v9261: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:36:58.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:36:58 smithi153 ceph-mon[70044]: pgmap v9261: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:00.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:36:59 smithi153 ceph-mon[70044]: pgmap v9262: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:00.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:36:59 smithi083 ceph-mon[79930]: pgmap v9262: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:00.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:36:59 smithi083 ceph-mon[77242]: pgmap v9262: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:01.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:37:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:37:01] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T10:37:01.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:37:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:37:01.297Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:37:01.738 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:37:02.026 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:37:02.027 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 6m ago 5h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:37:02.027 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 3m ago 5h 62.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:37:02.027 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 6m ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:37:02.027 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 3m ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:37:02.027 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 3m ago 5h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:37:02.028 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 6m ago 5h 774M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:37:02.028 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 6m ago 5h 1165M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:37:02.028 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 3m ago 5h 501M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:37:02.028 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 6m ago 5h 669M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:37:02.028 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 6m ago 5h 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:37:02.029 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 3m ago 5h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:37:02.029 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 6m ago 5h 819M 2474M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:37:02.029 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 6m ago 5h 809M 2474M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:37:02.029 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 6m ago 5h 632M 2474M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:37:02.029 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 6m ago 5h 789M 2474M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:37:02.030 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 3m ago 5h 696M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:37:02.030 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 3m ago 5h 867M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:37:02.030 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 3m ago 5h 774M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:37:02.030 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 3m ago 5h 916M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:37:02.031 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 3m ago 5h 187M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:37:02.384 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:37:02.384 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:37:02.385 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:37:02.385 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:37:02.385 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:37:02.385 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:37:02.386 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:37:02.386 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:37:02.386 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:37:02.386 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:37:02.387 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:37:02.387 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:37:02.387 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:37:02.387 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:37:02.388 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:37:02.388 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:37:02.388 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:37:02.388 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:37:02.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:37:02 smithi083 ceph-mon[79930]: pgmap v9263: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:02.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:37:02 smithi083 ceph-mon[79930]: from='client.44907 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:37:02.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:37:02 smithi083 ceph-mon[77242]: pgmap v9263: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:02.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:37:02 smithi083 ceph-mon[77242]: from='client.44907 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:37:02.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:37:02 smithi153 ceph-mon[70044]: pgmap v9263: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:02.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:37:02 smithi153 ceph-mon[70044]: from='client.44907 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:37:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:37:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:37:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:37:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:37:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:37:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:37:03.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:37:03 smithi083 ceph-mon[79930]: from='client.44913 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:37:03.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:37:03 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/322247432' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:37:03.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:37:03 smithi083 ceph-mon[77242]: from='client.44913 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:37:03.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:37:03 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/322247432' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:37:03.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:37:03 smithi153 ceph-mon[70044]: from='client.44913 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:37:03.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:37:03 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/322247432' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:37:04.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:37:04 smithi083 ceph-mon[77242]: pgmap v9264: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:04.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:37:04 smithi083 ceph-mon[79930]: pgmap v9264: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:04.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:37:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:37:04.676Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:37:05.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:37:04 smithi153 ceph-mon[70044]: pgmap v9264: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:05.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:37:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:37:05.380Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:37:06.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:37:05 smithi153 ceph-mon[70044]: pgmap v9265: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:06.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:37:05 smithi083 ceph-mon[79930]: pgmap v9265: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:06.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:37:05 smithi083 ceph-mon[77242]: pgmap v9265: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:08.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:37:08 smithi083 ceph-mon[79930]: pgmap v9266: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:08.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:37:08 smithi083 ceph-mon[77242]: pgmap v9266: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:08.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:37:08 smithi153 ceph-mon[70044]: pgmap v9266: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:09.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:37:09 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:37:09.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:37:09 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:37:09.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:37:09 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:37:09.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:37:09 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:37:09.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:37:09 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:37:09.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:37:09 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:37:09.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:37:09 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:37:09.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:37:09 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:37:09.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:37:09 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:37:09.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:37:09 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:37:09.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:37:09 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:37:09.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:37:09 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:37:10.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:37:10 smithi083 ceph-mon[79930]: pgmap v9267: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:10.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:37:10 smithi083 ceph-mon[77242]: pgmap v9267: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:10.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:37:10 smithi153 ceph-mon[70044]: pgmap v9267: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:11.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:37:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:37:11] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T10:37:11.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:37:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:37:11.298Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:37:12.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:37:12 smithi083 ceph-mon[79930]: pgmap v9268: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:12.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:37:12 smithi083 ceph-mon[77242]: pgmap v9268: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:12.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:37:12 smithi153 ceph-mon[70044]: pgmap v9268: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:37:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:37:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:37:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:37:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:37:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:37:14.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:37:14 smithi083 ceph-mon[79930]: pgmap v9269: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:14.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:37:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:37:14.677Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:37:14.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:37:14 smithi083 ceph-mon[77242]: pgmap v9269: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:15.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:37:14 smithi153 ceph-mon[70044]: pgmap v9269: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:15.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:37:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:37:15.381Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:37:16.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:37:15 smithi153 ceph-mon[70044]: pgmap v9270: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:16.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:37:15 smithi083 ceph-mon[79930]: pgmap v9270: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:16.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:37:15 smithi083 ceph-mon[77242]: pgmap v9270: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:18.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:37:18 smithi083 ceph-mon[79930]: pgmap v9271: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:18.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:37:18 smithi083 ceph-mon[77242]: pgmap v9271: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:18.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:37:18 smithi153 ceph-mon[70044]: pgmap v9271: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:20.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:37:19 smithi153 ceph-mon[70044]: pgmap v9272: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:20.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:37:19 smithi083 ceph-mon[79930]: pgmap v9272: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:20.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:37:19 smithi083 ceph-mon[77242]: pgmap v9272: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:21.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:37:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:37:21] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T10:37:21.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:37:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:37:21.297Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:37:22.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:37:22 smithi083 ceph-mon[77242]: pgmap v9273: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:22.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:37:22 smithi083 ceph-mon[79930]: pgmap v9273: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:22.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:37:22 smithi153 ceph-mon[70044]: pgmap v9273: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:37:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:37:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:37:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:37:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:37:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:37:24.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:37:24 smithi083 ceph-mon[77242]: pgmap v9274: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:24.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:37:24 smithi083 ceph-mon[79930]: pgmap v9274: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:24.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:37:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:37:24.678Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:37:25.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:37:24 smithi153 ceph-mon[70044]: pgmap v9274: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:25.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:37:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:37:25.382Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:37:26.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:37:25 smithi153 ceph-mon[70044]: pgmap v9275: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:26.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:37:25 smithi083 ceph-mon[79930]: pgmap v9275: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:26.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:37:25 smithi083 ceph-mon[77242]: pgmap v9275: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:28.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:37:28 smithi083 ceph-mon[79930]: pgmap v9276: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:28.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:37:28 smithi083 ceph-mon[77242]: pgmap v9276: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:28.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:37:28 smithi153 ceph-mon[70044]: pgmap v9276: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:30.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:37:29 smithi153 ceph-mon[70044]: pgmap v9277: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:30.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:37:29 smithi083 ceph-mon[79930]: pgmap v9277: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:30.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:37:29 smithi083 ceph-mon[77242]: pgmap v9277: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:31.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:37:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:37:31] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T10:37:31.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:37:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:37:31.298Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:37:32.703 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:37:32.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:37:32 smithi083 ceph-mon[79930]: pgmap v9278: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:32.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:37:32 smithi083 ceph-mon[77242]: pgmap v9278: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:32.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:37:32 smithi153 ceph-mon[70044]: pgmap v9278: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:37:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:37:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:37:32.992 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:37:32.992 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 6m ago 5h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:37:32.993 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 4m ago 5h 62.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:37:32.993 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 6m ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:37:32.993 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 4m ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:37:32.993 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 4m ago 5h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:37:32.994 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 6m ago 5h 774M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:37:32.994 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 6m ago 5h 1165M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:37:32.994 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 4m ago 5h 501M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:37:32.994 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 6m ago 5h 669M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:37:32.995 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 6m ago 5h 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:37:32.995 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 4m ago 5h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:37:32.995 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 6m ago 5h 819M 2474M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:37:32.995 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 6m ago 5h 809M 2474M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:37:32.995 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 6m ago 5h 632M 2474M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:37:32.996 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 6m ago 5h 789M 2474M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:37:32.996 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 4m ago 5h 696M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:37:32.996 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 4m ago 5h 867M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:37:32.996 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 4m ago 5h 774M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:37:32.996 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 4m ago 5h 916M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:37:32.997 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 4m ago 5h 187M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:37:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:37:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:37:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:37:33.353 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:37:33.354 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:37:33.354 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:37:33.354 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:37:33.354 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:37:33.355 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:37:33.355 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:37:33.355 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:37:33.355 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:37:33.355 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:37:33.356 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:37:33.356 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:37:33.356 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:37:33.356 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:37:33.357 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:37:33.357 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:37:33.357 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:37:33.357 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:37:33.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:37:33 smithi083 ceph-mon[77242]: from='client.54856 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:37:33.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:37:33 smithi083 ceph-mon[79930]: from='client.54856 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:37:33.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:37:33 smithi153 ceph-mon[70044]: from='client.54856 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:37:34.680 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:37:34 smithi083 ceph-mon[79930]: from='client.54862 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:37:34.680 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:37:34 smithi083 ceph-mon[79930]: pgmap v9279: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:34.680 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:37:34 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1328089991' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:37:34.681 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:37:34 smithi083 ceph-mon[77242]: from='client.54862 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:37:34.681 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:37:34 smithi083 ceph-mon[77242]: pgmap v9279: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:34.681 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:37:34 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1328089991' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:37:34.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:37:34 smithi153 ceph-mon[70044]: from='client.54862 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:37:34.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:37:34 smithi153 ceph-mon[70044]: pgmap v9279: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:34.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:37:34 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1328089991' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:37:34.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:37:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:37:34.679Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:37:35.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:37:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:37:35.383Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:37:36.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:37:36 smithi083 ceph-mon[79930]: pgmap v9280: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:36.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:37:36 smithi083 ceph-mon[77242]: pgmap v9280: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:36.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:37:36 smithi153 ceph-mon[70044]: pgmap v9280: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:38.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:37:38 smithi083 ceph-mon[79930]: pgmap v9281: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:38.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:37:38 smithi083 ceph-mon[77242]: pgmap v9281: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:38.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:37:38 smithi153 ceph-mon[70044]: pgmap v9281: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:37:39 smithi153 ceph-mon[70044]: pgmap v9282: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:40.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:37:39 smithi083 ceph-mon[79930]: pgmap v9282: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:40.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:37:39 smithi083 ceph-mon[77242]: pgmap v9282: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:37:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:37:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:37:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:37:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:37:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:37:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:37:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:37:41.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:37:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:37:41.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:37:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:37:41.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:37:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:37:41.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:37:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:37:41.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:37:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:37:41.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:37:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:37:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:37:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:37:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:37:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:37:41.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:37:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:37:41] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T10:37:41.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:37:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:37:41.298Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:37:42.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:37:41 smithi153 ceph-mon[70044]: pgmap v9283: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:42.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:37:41 smithi083 ceph-mon[77242]: pgmap v9283: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:42.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:37:41 smithi083 ceph-mon[79930]: pgmap v9283: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:37:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:37:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:37:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:37:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:37:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:37:44.680 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:37:44 smithi083 ceph-mon[79930]: pgmap v9284: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:44.681 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:37:44 smithi083 ceph-mon[77242]: pgmap v9284: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:44.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:37:44 smithi153 ceph-mon[70044]: pgmap v9284: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:44.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:37:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:37:44.679Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:37:45.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:37:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:37:45.384Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:37:46.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:37:46 smithi083 ceph-mon[77242]: pgmap v9285: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:46.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:37:46 smithi083 ceph-mon[79930]: pgmap v9285: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:46.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:37:46 smithi153 ceph-mon[70044]: pgmap v9285: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:48.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:37:48 smithi083 ceph-mon[77242]: pgmap v9286: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:48.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:37:48 smithi083 ceph-mon[79930]: pgmap v9286: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:48.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:37:48 smithi153 ceph-mon[70044]: pgmap v9286: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:50.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:37:49 smithi153 ceph-mon[70044]: pgmap v9287: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:50.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:37:49 smithi083 ceph-mon[77242]: pgmap v9287: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:50.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:37:49 smithi083 ceph-mon[79930]: pgmap v9287: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:51.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:37:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:37:51] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T10:37:51.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:37:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:37:51.300Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:37:52.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:37:52 smithi083 ceph-mon[79930]: pgmap v9288: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:52.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:37:52 smithi083 ceph-mon[77242]: pgmap v9288: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:52.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:37:52 smithi153 ceph-mon[70044]: pgmap v9288: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:37:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:37:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:37:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:37:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:37:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:37:54.681 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:37:54 smithi083 ceph-mon[79930]: pgmap v9289: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:54.681 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:37:54 smithi083 ceph-mon[77242]: pgmap v9289: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:54.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:37:54 smithi153 ceph-mon[70044]: pgmap v9289: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:54.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:37:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:37:54.680Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:37:55.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:37:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:37:55.384Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:37:56.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:37:56 smithi083 ceph-mon[79930]: pgmap v9290: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:56.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:37:56 smithi083 ceph-mon[77242]: pgmap v9290: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:56.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:37:56 smithi153 ceph-mon[70044]: pgmap v9290: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:58.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:37:58 smithi083 ceph-mon[79930]: pgmap v9291: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:58.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:37:58 smithi083 ceph-mon[77242]: pgmap v9291: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:37:58.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:37:58 smithi153 ceph-mon[70044]: pgmap v9291: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:00.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:37:59 smithi153 ceph-mon[70044]: pgmap v9292: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:00.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:37:59 smithi083 ceph-mon[79930]: pgmap v9292: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:00.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:37:59 smithi083 ceph-mon[77242]: pgmap v9292: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:01.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:38:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:38:01] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T10:38:01.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:38:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:38:01.300Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:38:02.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:38:02 smithi083 ceph-mon[77242]: pgmap v9293: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:02.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:38:02 smithi083 ceph-mon[79930]: pgmap v9293: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:02.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:38:02 smithi153 ceph-mon[70044]: pgmap v9293: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:38:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:38:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:38:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:38:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:38:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:38:03.686 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:38:03.974 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:38:03.975 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 7m ago 5h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:38:03.975 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 4m ago 5h 62.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:38:03.975 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 7m ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:38:03.976 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 4m ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:38:03.976 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 4m ago 5h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:38:03.976 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 7m ago 5h 774M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:38:03.976 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 7m ago 5h 1165M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:38:03.977 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 4m ago 5h 501M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:38:03.977 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 7m ago 5h 669M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:38:03.977 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 7m ago 5h 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:38:03.977 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 4m ago 5h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:38:03.978 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 7m ago 5h 819M 2474M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:38:03.978 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 7m ago 5h 809M 2474M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:38:03.978 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 7m ago 5h 632M 2474M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:38:03.978 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 7m ago 5h 789M 2474M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:38:03.979 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 4m ago 5h 696M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:38:03.979 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 4m ago 5h 867M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:38:03.979 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 4m ago 5h 774M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:38:03.979 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 4m ago 5h 916M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:38:03.979 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 4m ago 5h 187M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:38:04.331 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:38:04.331 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:38:04.331 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:38:04.331 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:38:04.332 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:38:04.332 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:38:04.332 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:38:04.332 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:38:04.333 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:38:04.333 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:38:04.333 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:38:04.333 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:38:04.334 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:38:04.334 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:38:04.334 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:38:04.334 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:38:04.335 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:38:04.335 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:38:04.681 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:38:04 smithi083 ceph-mon[79930]: pgmap v9294: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:04.681 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:38:04 smithi083 ceph-mon[79930]: from='client.44943 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:38:04.681 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:38:04 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3972554004' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:38:04.682 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:38:04 smithi083 ceph-mon[77242]: pgmap v9294: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:04.682 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:38:04 smithi083 ceph-mon[77242]: from='client.44943 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:38:04.682 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:38:04 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3972554004' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:38:04.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:38:04 smithi153 ceph-mon[70044]: pgmap v9294: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:04.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:38:04 smithi153 ceph-mon[70044]: from='client.44943 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:38:04.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:38:04 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3972554004' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:38:04.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:38:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:38:04.680Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:38:05.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:38:05 smithi083 ceph-mon[77242]: from='client.54880 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:38:05.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:38:05 smithi083 ceph-mon[79930]: from='client.54880 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:38:05.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:38:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:38:05.385Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:38:05.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:38:05 smithi153 ceph-mon[70044]: from='client.54880 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:38:06.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:38:06 smithi083 ceph-mon[77242]: pgmap v9295: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:06.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:38:06 smithi083 ceph-mon[79930]: pgmap v9295: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:06.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:38:06 smithi153 ceph-mon[70044]: pgmap v9295: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:08.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:38:08 smithi083 ceph-mon[77242]: pgmap v9296: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:08.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:38:08 smithi083 ceph-mon[79930]: pgmap v9296: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:08.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:38:08 smithi153 ceph-mon[70044]: pgmap v9296: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:10.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:38:09 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:38:10.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:38:09 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:38:10.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:38:09 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:38:10.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:38:09 smithi153 ceph-mon[70044]: pgmap v9297: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:10.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:38:09 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:38:10.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:38:09 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:38:10.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:38:09 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:38:10.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:38:09 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:38:10.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:38:09 smithi083 ceph-mon[79930]: pgmap v9297: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:10.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:38:09 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:38:10.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:38:09 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:38:10.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:38:09 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:38:10.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:38:09 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:38:10.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:38:09 smithi083 ceph-mon[77242]: pgmap v9297: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:10.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:38:09 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:38:11.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:38:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:38:11] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T10:38:11.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:38:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:38:11.300Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:38:12.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:38:12 smithi083 ceph-mon[79930]: pgmap v9298: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:12.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:38:12 smithi083 ceph-mon[77242]: pgmap v9298: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:12.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:38:12 smithi153 ceph-mon[70044]: pgmap v9298: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:38:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:38:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:38:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:38:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:38:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:38:14.681 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:38:14 smithi083 ceph-mon[79930]: pgmap v9299: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:14.682 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:38:14 smithi083 ceph-mon[77242]: pgmap v9299: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:14.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:38:14 smithi153 ceph-mon[70044]: pgmap v9299: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:14.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:38:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:38:14.681Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:38:15.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:38:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:38:15.386Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:38:16.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:38:16 smithi083 ceph-mon[79930]: pgmap v9300: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:16.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:38:16 smithi083 ceph-mon[77242]: pgmap v9300: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:16.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:38:16 smithi153 ceph-mon[70044]: pgmap v9300: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:18.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:38:18 smithi083 ceph-mon[79930]: pgmap v9301: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:18.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:38:18 smithi083 ceph-mon[77242]: pgmap v9301: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:18.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:38:18 smithi153 ceph-mon[70044]: pgmap v9301: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:20.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:38:19 smithi153 ceph-mon[70044]: pgmap v9302: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:20.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:38:19 smithi083 ceph-mon[77242]: pgmap v9302: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:20.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:38:19 smithi083 ceph-mon[79930]: pgmap v9302: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:21.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:38:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:38:21] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T10:38:21.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:38:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:38:21.301Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:38:22.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:38:22 smithi083 ceph-mon[79930]: pgmap v9303: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:22.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:38:22 smithi083 ceph-mon[77242]: pgmap v9303: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:22.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:38:22 smithi153 ceph-mon[70044]: pgmap v9303: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:38:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:38:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:38:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:38:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:38:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:38:24.683 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:38:24 smithi083 ceph-mon[79930]: pgmap v9304: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:24.683 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:38:24 smithi083 ceph-mon[77242]: pgmap v9304: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:24.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:38:24 smithi153 ceph-mon[70044]: pgmap v9304: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:24.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:38:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:38:24.682Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:38:25.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:38:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:38:25.386Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:38:26.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:38:26 smithi083 ceph-mon[79930]: pgmap v9305: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:26.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:38:26 smithi083 ceph-mon[77242]: pgmap v9305: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:26.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:38:26 smithi153 ceph-mon[70044]: pgmap v9305: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:28.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:38:28 smithi083 ceph-mon[79930]: pgmap v9306: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:28.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:38:28 smithi083 ceph-mon[77242]: pgmap v9306: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:28.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:38:28 smithi153 ceph-mon[70044]: pgmap v9306: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:30.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:38:29 smithi153 ceph-mon[70044]: pgmap v9307: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:30.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:38:29 smithi083 ceph-mon[79930]: pgmap v9307: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:30.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:38:29 smithi083 ceph-mon[77242]: pgmap v9307: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:31.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:38:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:38:31.301Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:38:31.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:38:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:38:31] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T10:38:32.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:38:32 smithi083 ceph-mon[77242]: pgmap v9308: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:32.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:38:32 smithi083 ceph-mon[79930]: pgmap v9308: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:32.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:38:32 smithi153 ceph-mon[70044]: pgmap v9308: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:38:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:38:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:38:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:38:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:38:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:38:34.661 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:38:34.682 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:38:34 smithi083 ceph-mon[79930]: pgmap v9309: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:34.683 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:38:34 smithi083 ceph-mon[77242]: pgmap v9309: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:34.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:38:34 smithi153 ceph-mon[70044]: pgmap v9309: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:34.950 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:38:34.950 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 7m ago 5h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:38:34.950 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 5m ago 5h 62.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:38:34.950 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 7m ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:38:34.951 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 5m ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:38:34.951 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 5m ago 5h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:38:34.951 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 7m ago 5h 774M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:38:34.951 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 7m ago 5h 1165M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:38:34.952 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 5m ago 5h 501M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:38:34.952 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 7m ago 5h 669M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:38:34.952 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 7m ago 5h 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:38:34.952 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 5m ago 5h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:38:34.953 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 7m ago 5h 819M 2474M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:38:34.953 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 7m ago 5h 809M 2474M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:38:34.953 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 7m ago 5h 632M 2474M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:38:34.953 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 7m ago 5h 789M 2474M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:38:34.953 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 5m ago 5h 696M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:38:34.954 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 5m ago 5h 867M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:38:34.954 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 5m ago 5h 774M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:38:34.954 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 5m ago 5h 916M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:38:34.954 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 5m ago 5h 187M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:38:34.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:38:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:38:34.682Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:38:35.304 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:38:35.304 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:38:35.305 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:38:35.305 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:38:35.305 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:38:35.305 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:38:35.305 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:38:35.306 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:38:35.306 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:38:35.306 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:38:35.307 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:38:35.307 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:38:35.307 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:38:35.307 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:38:35.307 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:38:35.308 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:38:35.308 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:38:35.308 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:38:35.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:38:35 smithi083 ceph-mon[79930]: from='client.54892 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:38:35.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:38:35 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/822359044' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:38:35.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:38:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:38:35.386Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:38:35.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:38:35 smithi083 ceph-mon[77242]: from='client.54892 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:38:35.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:38:35 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/822359044' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:38:35.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:38:35 smithi153 ceph-mon[70044]: from='client.54892 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:38:35.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:38:35 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/822359044' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:38:36.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:38:36 smithi083 ceph-mon[79930]: from='client.54898 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:38:36.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:38:36 smithi083 ceph-mon[79930]: pgmap v9310: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:36.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:38:36 smithi083 ceph-mon[77242]: from='client.54898 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:38:36.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:38:36 smithi083 ceph-mon[77242]: pgmap v9310: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:36.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:38:36 smithi153 ceph-mon[70044]: from='client.54898 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:38:36.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:38:36 smithi153 ceph-mon[70044]: pgmap v9310: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:38.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:38:38 smithi083 ceph-mon[77242]: pgmap v9311: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:38.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:38:38 smithi083 ceph-mon[79930]: pgmap v9311: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:38.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:38:38 smithi153 ceph-mon[70044]: pgmap v9311: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:38:39 smithi153 ceph-mon[70044]: pgmap v9312: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:40.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:38:39 smithi083 ceph-mon[79930]: pgmap v9312: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:40.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:38:39 smithi083 ceph-mon[77242]: pgmap v9312: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:38:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:38:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:38:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:38:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:38:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:38:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:38:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:38:41.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:38:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:38:41.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:38:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:38:41.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:38:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:38:41.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:38:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:38:41.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:38:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:38:41.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:38:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:38:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:38:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:38:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:38:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:38:41.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:38:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:38:41] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T10:38:41.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:38:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:38:41.303Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:38:42.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:38:41 smithi153 ceph-mon[70044]: pgmap v9313: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:42.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:38:41 smithi083 ceph-mon[79930]: pgmap v9313: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:42.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:38:41 smithi083 ceph-mon[77242]: pgmap v9313: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:38:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:38:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:38:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:38:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:38:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:38:44.684 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:38:44 smithi083 ceph-mon[79930]: pgmap v9314: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:44.685 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:38:44 smithi083 ceph-mon[77242]: pgmap v9314: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:44.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:38:44 smithi153 ceph-mon[70044]: pgmap v9314: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:44.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:38:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:38:44.683Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:38:45.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:38:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:38:45.388Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:38:46.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:38:46 smithi083 ceph-mon[79930]: pgmap v9315: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:46.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:38:46 smithi083 ceph-mon[77242]: pgmap v9315: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:46.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:38:46 smithi153 ceph-mon[70044]: pgmap v9315: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:48.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:38:48 smithi083 ceph-mon[79930]: pgmap v9316: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:48.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:38:48 smithi083 ceph-mon[77242]: pgmap v9316: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:48.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:38:48 smithi153 ceph-mon[70044]: pgmap v9316: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:50.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:38:49 smithi153 ceph-mon[70044]: pgmap v9317: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:50.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:38:49 smithi083 ceph-mon[79930]: pgmap v9317: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:50.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:38:49 smithi083 ceph-mon[77242]: pgmap v9317: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:51.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:38:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:38:51] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T10:38:51.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:38:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:38:51.303Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:38:52.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:38:52 smithi083 ceph-mon[79930]: pgmap v9318: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:52.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:38:52 smithi083 ceph-mon[77242]: pgmap v9318: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:52.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:38:52 smithi153 ceph-mon[70044]: pgmap v9318: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:38:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:38:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:38:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:38:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:38:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:38:54.685 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:38:54 smithi083 ceph-mon[79930]: pgmap v9319: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:54.686 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:38:54 smithi083 ceph-mon[77242]: pgmap v9319: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:54.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:38:54 smithi153 ceph-mon[70044]: pgmap v9319: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:54.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:38:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:38:54.684Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:38:55.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:38:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:38:55.389Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:38:56.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:38:56 smithi083 ceph-mon[79930]: pgmap v9320: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:56.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:38:56 smithi083 ceph-mon[77242]: pgmap v9320: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:56.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:38:56 smithi153 ceph-mon[70044]: pgmap v9320: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:58.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:38:58 smithi083 ceph-mon[79930]: pgmap v9321: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:58.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:38:58 smithi083 ceph-mon[77242]: pgmap v9321: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:38:58.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:38:58 smithi153 ceph-mon[70044]: pgmap v9321: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:00.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:38:59 smithi153 ceph-mon[70044]: pgmap v9322: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:00.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:38:59 smithi083 ceph-mon[79930]: pgmap v9322: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:00.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:38:59 smithi083 ceph-mon[77242]: pgmap v9322: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:01.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:39:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:39:01] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T10:39:01.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:39:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:39:01.304Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:39:02.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:39:02 smithi083 ceph-mon[77242]: pgmap v9323: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:02.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:39:02 smithi083 ceph-mon[79930]: pgmap v9323: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:02.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:39:02 smithi153 ceph-mon[70044]: pgmap v9323: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:39:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:39:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:39:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:39:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:39:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:39:04.686 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:39:04 smithi083 ceph-mon[79930]: pgmap v9324: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:04.687 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:39:04 smithi083 ceph-mon[77242]: pgmap v9324: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:04.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:39:04 smithi153 ceph-mon[70044]: pgmap v9324: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:04.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:39:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:39:04.685Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:39:05.631 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:39:05.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:39:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:39:05.389Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:39:05.919 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:39:05.920 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 8m ago 5h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:39:05.920 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 6m ago 5h 62.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:39:05.920 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 8m ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:39:05.920 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 6m ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:39:05.921 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 6m ago 5h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:39:05.921 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 8m ago 5h 774M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:39:05.921 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 8m ago 5h 1165M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:39:05.921 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 6m ago 5h 501M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:39:05.921 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 8m ago 5h 669M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:39:05.922 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 8m ago 5h 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:39:05.922 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 6m ago 5h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:39:05.922 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 8m ago 5h 819M 2474M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:39:05.922 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 8m ago 5h 809M 2474M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:39:05.922 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 8m ago 5h 632M 2474M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:39:05.923 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 8m ago 5h 789M 2474M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:39:05.923 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 6m ago 5h 696M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:39:05.923 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 6m ago 5h 867M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:39:05.923 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 6m ago 5h 774M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:39:05.924 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 6m ago 5h 916M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:39:05.924 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 6m ago 5h 187M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:39:06.275 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:39:06.275 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:39:06.275 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:39:06.276 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:39:06.276 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:39:06.276 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:39:06.276 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:39:06.276 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:39:06.277 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:39:06.277 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:39:06.277 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:39:06.277 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:39:06.278 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:39:06.278 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:39:06.278 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:39:06.278 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:39:06.278 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:39:06.279 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:39:06.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:39:06 smithi083 ceph-mon[77242]: pgmap v9325: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:06.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:39:06 smithi083 ceph-mon[77242]: from='client.54910 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:39:06.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:39:06 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3413734496' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:39:06.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:39:06 smithi083 ceph-mon[79930]: pgmap v9325: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:06.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:39:06 smithi083 ceph-mon[79930]: from='client.54910 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:39:06.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:39:06 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3413734496' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:39:06.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:39:06 smithi153 ceph-mon[70044]: pgmap v9325: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:06.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:39:06 smithi153 ceph-mon[70044]: from='client.54910 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:39:06.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:39:06 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3413734496' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:39:07.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:39:07 smithi083 ceph-mon[79930]: from='client.54916 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:39:07.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:39:07 smithi083 ceph-mon[77242]: from='client.54916 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:39:07.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:39:07 smithi153 ceph-mon[70044]: from='client.54916 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:39:08.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:39:08 smithi083 ceph-mon[77242]: pgmap v9326: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:08.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:39:08 smithi083 ceph-mon[79930]: pgmap v9326: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:08.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:39:08 smithi153 ceph-mon[70044]: pgmap v9326: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:10.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:39:09 smithi153 ceph-mon[70044]: pgmap v9327: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:10.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:39:09 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:39:10.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:39:09 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:39:10.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:39:09 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:39:10.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:39:09 smithi083 ceph-mon[77242]: pgmap v9327: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:10.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:39:09 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:39:10.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:39:09 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:39:10.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:39:09 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:39:10.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:39:09 smithi083 ceph-mon[79930]: pgmap v9327: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:10.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:39:09 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:39:10.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:39:09 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:39:10.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:39:09 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:39:11.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:39:10 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:39:11.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:39:10 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:39:11.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:39:10 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:39:11.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:39:10 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:39:11.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:39:10 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:39:11.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:39:10 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:39:11.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:39:10 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:39:11.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:39:10 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:39:11.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:39:10 smithi153 ceph-mon[70044]: Adjusting osd_memory_target on smithi083 to 2464M 2022-09-29T10:39:11.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:39:10 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:39:11.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:39:10 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:39:11.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:39:10 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:39:11.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:39:10 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:39:11.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:39:10 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:39:11.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:39:10 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:39:11.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:39:10 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:39:11.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:39:10 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:39:11.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:39:10 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:39:11.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:39:10 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:39:11.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:39:10 smithi083 ceph-mon[79930]: Adjusting osd_memory_target on smithi083 to 2464M 2022-09-29T10:39:11.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:39:10 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:39:11.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:39:10 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:39:11.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:39:10 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:39:11.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:39:10 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:39:11.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:39:10 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:39:11.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:39:10 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:39:11.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:39:10 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:39:11.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:39:10 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:39:11.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:39:10 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:39:11.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:39:10 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:39:11.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:39:10 smithi083 ceph-mon[77242]: Adjusting osd_memory_target on smithi083 to 2464M 2022-09-29T10:39:11.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:39:10 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:39:11.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:39:10 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:39:11.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:39:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:39:11] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T10:39:11.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:39:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:39:11.304Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:39:12.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:39:11 smithi153 ceph-mon[70044]: pgmap v9328: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:12.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:39:11 smithi083 ceph-mon[79930]: pgmap v9328: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:12.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:39:11 smithi083 ceph-mon[77242]: pgmap v9328: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:39:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:39:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:39:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:39:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:39:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:39:14.687 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:39:14 smithi083 ceph-mon[79930]: pgmap v9329: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:14.687 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:39:14 smithi083 ceph-mon[77242]: pgmap v9329: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:14.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:39:14 smithi153 ceph-mon[70044]: pgmap v9329: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:14.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:39:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:39:14.685Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:39:15.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:39:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:39:15.390Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:39:16.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:39:16 smithi083 ceph-mon[77242]: pgmap v9330: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:16.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:39:16 smithi083 ceph-mon[79930]: pgmap v9330: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:16.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:39:16 smithi153 ceph-mon[70044]: pgmap v9330: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:18.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:39:18 smithi083 ceph-mon[79930]: pgmap v9331: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:18.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:39:18 smithi083 ceph-mon[77242]: pgmap v9331: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:18.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:39:18 smithi153 ceph-mon[70044]: pgmap v9331: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:20.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:39:19 smithi153 ceph-mon[70044]: pgmap v9332: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:20.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:39:19 smithi083 ceph-mon[79930]: pgmap v9332: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:20.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:39:19 smithi083 ceph-mon[77242]: pgmap v9332: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:21.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:39:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:39:21.306Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:39:21.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:39:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:39:21] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T10:39:22.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:39:22 smithi083 ceph-mon[79930]: pgmap v9333: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:22.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:39:22 smithi083 ceph-mon[77242]: pgmap v9333: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:22.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:39:22 smithi153 ceph-mon[70044]: pgmap v9333: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:39:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:39:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:39:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:39:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:39:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:39:24.687 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:39:24 smithi083 ceph-mon[79930]: pgmap v9334: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:24.688 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:39:24 smithi083 ceph-mon[77242]: pgmap v9334: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:24.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:39:24 smithi153 ceph-mon[70044]: pgmap v9334: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:24.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:39:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:39:24.686Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:39:25.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:39:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:39:25.390Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:39:26.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:39:26 smithi083 ceph-mon[79930]: pgmap v9335: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:26.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:39:26 smithi083 ceph-mon[77242]: pgmap v9335: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:26.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:39:26 smithi153 ceph-mon[70044]: pgmap v9335: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:28.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:39:28 smithi083 ceph-mon[79930]: pgmap v9336: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:28.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:39:28 smithi083 ceph-mon[77242]: pgmap v9336: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:28.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:39:28 smithi153 ceph-mon[70044]: pgmap v9336: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:30.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:39:29 smithi153 ceph-mon[70044]: pgmap v9337: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:30.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:39:29 smithi083 ceph-mon[77242]: pgmap v9337: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:30.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:39:29 smithi083 ceph-mon[79930]: pgmap v9337: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:31.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:39:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:39:31] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T10:39:31.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:39:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:39:31.306Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:39:32.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:39:32 smithi083 ceph-mon[79930]: pgmap v9338: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:32.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:39:32 smithi083 ceph-mon[77242]: pgmap v9338: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:32.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:39:32 smithi153 ceph-mon[70044]: pgmap v9338: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:39:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:39:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:39:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:39:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:39:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:39:34.689 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:39:34 smithi083 ceph-mon[79930]: pgmap v9339: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:34.689 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:39:34 smithi083 ceph-mon[77242]: pgmap v9339: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:34.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:39:34 smithi153 ceph-mon[70044]: pgmap v9339: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:34.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:39:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:39:34.688Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:39:35.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:39:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:39:35.391Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:39:36.604 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:39:36.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:39:36 smithi083 ceph-mon[79930]: pgmap v9340: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:36.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:39:36 smithi083 ceph-mon[77242]: pgmap v9340: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:36.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:39:36 smithi153 ceph-mon[70044]: pgmap v9340: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:36.889 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:39:36.890 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 8m ago 5h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:39:36.890 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 6m ago 5h 62.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:39:36.890 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 8m ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:39:36.890 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 6m ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:39:36.891 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 6m ago 5h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:39:36.891 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 8m ago 5h 774M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:39:36.891 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 8m ago 5h 1165M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:39:36.891 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 6m ago 5h 501M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:39:36.891 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 8m ago 5h 669M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:39:36.892 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 8m ago 5h 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:39:36.892 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 6m ago 5h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:39:36.892 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 8m ago 5h 819M 2464M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:39:36.892 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 8m ago 5h 809M 2464M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:39:36.892 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 8m ago 5h 632M 2464M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:39:36.893 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 8m ago 5h 789M 2464M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:39:36.893 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 6m ago 5h 696M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:39:36.893 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 6m ago 5h 867M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:39:36.893 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 6m ago 5h 774M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:39:36.893 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 6m ago 5h 916M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:39:36.894 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 6m ago 5h 187M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:39:37.249 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:39:37.249 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:39:37.249 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:39:37.250 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:39:37.250 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:39:37.250 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:39:37.250 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:39:37.251 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:39:37.251 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:39:37.251 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:39:37.251 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:39:37.252 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:39:37.252 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:39:37.252 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:39:37.252 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:39:37.253 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:39:37.253 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:39:37.253 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:39:37.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:39:37 smithi083 ceph-mon[79930]: from='client.54928 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:39:37.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:39:37 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/263884393' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:39:37.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:39:37 smithi083 ceph-mon[77242]: from='client.54928 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:39:37.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:39:37 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/263884393' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:39:37.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:39:37 smithi153 ceph-mon[70044]: from='client.54928 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:39:37.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:39:37 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/263884393' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:39:38.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:39:38 smithi083 ceph-mon[77242]: from='client.54934 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:39:38.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:39:38 smithi083 ceph-mon[77242]: pgmap v9341: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:38.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:39:38 smithi083 ceph-mon[79930]: from='client.54934 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:39:38.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:39:38 smithi083 ceph-mon[79930]: pgmap v9341: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:38.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:39:38 smithi153 ceph-mon[70044]: from='client.54934 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:39:38.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:39:38 smithi153 ceph-mon[70044]: pgmap v9341: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:39:39 smithi153 ceph-mon[70044]: pgmap v9342: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:40.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:39:39 smithi083 ceph-mon[79930]: pgmap v9342: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:40.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:39:39 smithi083 ceph-mon[77242]: pgmap v9342: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:39:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:39:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:39:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:39:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:39:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:39:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:39:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:39:41.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:39:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:39:41.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:39:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:39:41.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:39:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:39:41.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:39:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:39:41.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:39:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:39:41.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:39:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:39:41.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:39:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:39:41.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:39:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:39:41.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:39:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:39:41] "GET /metrics HTTP/1.1" 200 225227 "" "Prometheus/2.33.4" 2022-09-29T10:39:41.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:39:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:39:41.307Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:39:42.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:39:41 smithi153 ceph-mon[70044]: pgmap v9343: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:42.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:39:41 smithi083 ceph-mon[77242]: pgmap v9343: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:42.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:39:41 smithi083 ceph-mon[79930]: pgmap v9343: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:39:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:39:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:39:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:39:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:39:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:39:44.689 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:39:44 smithi083 ceph-mon[79930]: pgmap v9344: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:44.690 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:39:44 smithi083 ceph-mon[77242]: pgmap v9344: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:44.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:39:44 smithi153 ceph-mon[70044]: pgmap v9344: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:44.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:39:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:39:44.688Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:39:45.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:39:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:39:45.392Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:39:46.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:39:46 smithi083 ceph-mon[79930]: pgmap v9345: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:46.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:39:46 smithi083 ceph-mon[77242]: pgmap v9345: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:46.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:39:46 smithi153 ceph-mon[70044]: pgmap v9345: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:48.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:39:48 smithi083 ceph-mon[79930]: pgmap v9346: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:48.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:39:48 smithi083 ceph-mon[77242]: pgmap v9346: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:48.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:39:48 smithi153 ceph-mon[70044]: pgmap v9346: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:50.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:39:49 smithi153 ceph-mon[70044]: pgmap v9347: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:50.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:39:49 smithi083 ceph-mon[79930]: pgmap v9347: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:50.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:39:49 smithi083 ceph-mon[77242]: pgmap v9347: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:51.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:39:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:39:51] "GET /metrics HTTP/1.1" 200 225227 "" "Prometheus/2.33.4" 2022-09-29T10:39:51.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:39:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:39:51.307Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:39:52.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:39:52 smithi083 ceph-mon[79930]: pgmap v9348: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:52.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:39:52 smithi083 ceph-mon[77242]: pgmap v9348: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:52.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:39:52 smithi153 ceph-mon[70044]: pgmap v9348: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:39:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:39:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:39:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:39:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:39:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:39:54.690 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:39:54 smithi083 ceph-mon[79930]: pgmap v9349: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:54.690 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:39:54 smithi083 ceph-mon[77242]: pgmap v9349: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:54.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:39:54 smithi153 ceph-mon[70044]: pgmap v9349: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:54.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:39:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:39:54.689Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:39:55.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:39:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:39:55.392Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:39:56.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:39:56 smithi083 ceph-mon[77242]: pgmap v9350: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:56.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:39:56 smithi083 ceph-mon[79930]: pgmap v9350: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:56.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:39:56 smithi153 ceph-mon[70044]: pgmap v9350: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:58.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:39:58 smithi083 ceph-mon[77242]: pgmap v9351: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:58.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:39:58 smithi083 ceph-mon[79930]: pgmap v9351: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:39:58.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:39:58 smithi153 ceph-mon[70044]: pgmap v9351: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:00.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:39:59 smithi153 ceph-mon[70044]: pgmap v9352: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:00.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:39:59 smithi083 ceph-mon[79930]: pgmap v9352: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:00.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:39:59 smithi083 ceph-mon[77242]: pgmap v9352: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:00.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:40:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[77219]: 2022-09-29T10:39:59.999+0000 7f160f665700 -1 log_channel(cluster) log [ERR] : overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T10:40:01.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:40:00 smithi153 ceph-mon[70044]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T10:40:01.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:40:00 smithi083 ceph-mon[79930]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T10:40:01.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:40:00 smithi083 ceph-mon[77242]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T10:40:01.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:40:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:40:01.310Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:40:01.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:40:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:40:01] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T10:40:02.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:40:01 smithi153 ceph-mon[70044]: pgmap v9353: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:02.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:40:01 smithi083 ceph-mon[77242]: pgmap v9353: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:02.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:40:01 smithi083 ceph-mon[79930]: pgmap v9353: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:40:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:40:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:40:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:40:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:40:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:40:04.690 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:40:04 smithi083 ceph-mon[79930]: pgmap v9354: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:04.691 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:40:04 smithi083 ceph-mon[77242]: pgmap v9354: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:04.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:40:04 smithi153 ceph-mon[70044]: pgmap v9354: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:04.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:40:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:40:04.689Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:40:05.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:40:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:40:05.392Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:40:06.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:40:06 smithi083 ceph-mon[77242]: pgmap v9355: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:06.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:40:06 smithi083 ceph-mon[79930]: pgmap v9355: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:06.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:40:06 smithi153 ceph-mon[70044]: pgmap v9355: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:07.577 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:40:07.866 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:40:07.866 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 9m ago 5h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:40:07.866 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 7m ago 5h 62.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:40:07.866 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 9m ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:40:07.867 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 7m ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:40:07.867 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 7m ago 5h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:40:07.867 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 9m ago 5h 774M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:40:07.868 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 9m ago 5h 1165M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:40:07.868 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 7m ago 5h 501M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:40:07.868 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 9m ago 5h 669M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:40:07.868 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 9m ago 5h 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:40:07.868 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 7m ago 5h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:40:07.869 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 9m ago 5h 819M 2464M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:40:07.869 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 9m ago 5h 809M 2464M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:40:07.869 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 9m ago 5h 632M 2464M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:40:07.870 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 9m ago 5h 789M 2464M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:40:07.870 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 7m ago 5h 696M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:40:07.870 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 7m ago 5h 867M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:40:07.870 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 7m ago 5h 774M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:40:07.870 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 7m ago 5h 916M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:40:07.871 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 7m ago 5h 187M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:40:08.223 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:40:08.223 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:40:08.224 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:40:08.224 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:40:08.224 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:40:08.225 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:40:08.225 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:40:08.225 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:40:08.225 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:40:08.226 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:40:08.226 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:40:08.226 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:40:08.226 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:40:08.227 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:40:08.227 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:40:08.227 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:40:08.227 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:40:08.227 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:40:08.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:40:08 smithi083 ceph-mon[77242]: pgmap v9356: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:08.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:40:08 smithi083 ceph-mon[77242]: from='client.45015 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:40:08.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:40:08 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2586215384' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:40:08.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:40:08 smithi083 ceph-mon[79930]: pgmap v9356: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:08.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:40:08 smithi083 ceph-mon[79930]: from='client.45015 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:40:08.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:40:08 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2586215384' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:40:08.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:40:08 smithi153 ceph-mon[70044]: pgmap v9356: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:08.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:40:08 smithi153 ceph-mon[70044]: from='client.45015 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:40:08.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:40:08 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2586215384' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:40:09.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:40:09 smithi083 ceph-mon[77242]: from='client.45021 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:40:09.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:40:09 smithi083 ceph-mon[79930]: from='client.45021 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:40:09.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:40:09 smithi153 ceph-mon[70044]: from='client.45021 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:40:10.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:40:10 smithi083 ceph-mon[79930]: pgmap v9357: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:10.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:40:10 smithi083 ceph-mon[77242]: pgmap v9357: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:10.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:40:10 smithi153 ceph-mon[70044]: pgmap v9357: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:11.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:40:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:40:11] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T10:40:11.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:40:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:40:11.308Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:40:11.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:40:11 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:40:11.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:40:11 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:40:11.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:40:11 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:40:11.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:40:11 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:40:11.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:40:11 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:40:11.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:40:11 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:40:11.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:40:11 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:40:11.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:40:11 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:40:11.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:40:11 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:40:11.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:40:11 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:40:11.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:40:11 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:40:11.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:40:11 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:40:12.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:40:12 smithi083 ceph-mon[79930]: pgmap v9358: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:12.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:40:12 smithi083 ceph-mon[77242]: pgmap v9358: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:12.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:40:12 smithi153 ceph-mon[70044]: pgmap v9358: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:40:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:40:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:40:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:40:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:40:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:40:14.690 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:40:14 smithi083 ceph-mon[79930]: pgmap v9359: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:14.691 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:40:14 smithi083 ceph-mon[77242]: pgmap v9359: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:14.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:40:14 smithi153 ceph-mon[70044]: pgmap v9359: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:14.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:40:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:40:14.689Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:40:15.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:40:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:40:15.393Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:40:16.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:40:16 smithi083 ceph-mon[79930]: pgmap v9360: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:16.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:40:16 smithi083 ceph-mon[77242]: pgmap v9360: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:16.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:40:16 smithi153 ceph-mon[70044]: pgmap v9360: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:18.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:40:18 smithi083 ceph-mon[79930]: pgmap v9361: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:18.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:40:18 smithi083 ceph-mon[77242]: pgmap v9361: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:18.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:40:18 smithi153 ceph-mon[70044]: pgmap v9361: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:20.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:40:19 smithi153 ceph-mon[70044]: pgmap v9362: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:20.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:40:19 smithi083 ceph-mon[79930]: pgmap v9362: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:20.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:40:19 smithi083 ceph-mon[77242]: pgmap v9362: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:21.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:40:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:40:21] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T10:40:21.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:40:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:40:21.310Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:40:22.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:40:22 smithi083 ceph-mon[79930]: pgmap v9363: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:22.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:40:22 smithi083 ceph-mon[77242]: pgmap v9363: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:22.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:40:22 smithi153 ceph-mon[70044]: pgmap v9363: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:40:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:40:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:40:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:40:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:40:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:40:24.690 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:40:24 smithi083 ceph-mon[79930]: pgmap v9364: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:24.691 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:40:24 smithi083 ceph-mon[77242]: pgmap v9364: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:24.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:40:24 smithi153 ceph-mon[70044]: pgmap v9364: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:24.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:40:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:40:24.689Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:40:25.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:40:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:40:25.394Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:40:26.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:40:26 smithi083 ceph-mon[79930]: pgmap v9365: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:26.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:40:26 smithi083 ceph-mon[77242]: pgmap v9365: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:26.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:40:26 smithi153 ceph-mon[70044]: pgmap v9365: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:28.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:40:28 smithi083 ceph-mon[79930]: pgmap v9366: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:28.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:40:28 smithi083 ceph-mon[77242]: pgmap v9366: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:28.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:40:28 smithi153 ceph-mon[70044]: pgmap v9366: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:30.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:40:29 smithi153 ceph-mon[70044]: pgmap v9367: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:30.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:40:29 smithi083 ceph-mon[79930]: pgmap v9367: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:30.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:40:29 smithi083 ceph-mon[77242]: pgmap v9367: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:31.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:40:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:40:31] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T10:40:31.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:40:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:40:31.310Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:40:32.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:40:32 smithi083 ceph-mon[77242]: pgmap v9368: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:32.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:40:32 smithi083 ceph-mon[79930]: pgmap v9368: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:32.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:40:32 smithi153 ceph-mon[70044]: pgmap v9368: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:40:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:40:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:40:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:40:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:40:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:40:34.691 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:40:34 smithi083 ceph-mon[79930]: pgmap v9369: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:34.692 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:40:34 smithi083 ceph-mon[77242]: pgmap v9369: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:34.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:40:34 smithi153 ceph-mon[70044]: pgmap v9369: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:34.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:40:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:40:34.690Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:40:35.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:40:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:40:35.395Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:40:36.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:40:36 smithi083 ceph-mon[79930]: pgmap v9370: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:36.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:40:36 smithi083 ceph-mon[77242]: pgmap v9370: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:36.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:40:36 smithi153 ceph-mon[70044]: pgmap v9370: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:38.552 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:40:38.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:40:38 smithi083 ceph-mon[77242]: pgmap v9371: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:38.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:40:38 smithi083 ceph-mon[79930]: pgmap v9371: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:38.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:40:38 smithi153 ceph-mon[70044]: pgmap v9371: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:38.844 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:40:38.845 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 9m ago 5h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:40:38.845 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 7m ago 5h 62.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:40:38.845 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 9m ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:40:38.845 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 7m ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:40:38.846 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 7m ago 5h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:40:38.846 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 9m ago 5h 774M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:40:38.846 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 9m ago 5h 1165M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:40:38.846 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 7m ago 5h 501M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:40:38.847 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 9m ago 5h 669M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:40:38.847 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 9m ago 5h 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:40:38.847 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 7m ago 5h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:40:38.847 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 9m ago 5h 819M 2464M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:40:38.848 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 9m ago 5h 809M 2464M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:40:38.848 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 9m ago 5h 632M 2464M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:40:38.848 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 9m ago 5h 789M 2464M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:40:38.848 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 7m ago 5h 696M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:40:38.848 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 7m ago 5h 867M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:40:38.849 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 7m ago 5h 774M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:40:38.849 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 7m ago 5h 916M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:40:38.849 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 7m ago 5h 187M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:40:39.202 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:40:39.203 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:40:39.203 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:40:39.203 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:40:39.204 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:40:39.204 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:40:39.204 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:40:39.204 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:40:39.204 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:40:39.205 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:40:39.205 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:40:39.205 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:40:39.205 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:40:39.205 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:40:39.206 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:40:39.206 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:40:39.206 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:40:39.207 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:40:39.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:40:39 smithi083 ceph-mon[77242]: from='client.54964 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:40:39.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:40:39 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3204716822' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:40:39.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:40:39 smithi083 ceph-mon[79930]: from='client.54964 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:40:39.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:40:39 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3204716822' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:40:39.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:40:39 smithi153 ceph-mon[70044]: from='client.54964 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:40:39.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:40:39 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3204716822' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:40:40.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:40:40 smithi083 ceph-mon[79930]: from='client.45039 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:40:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:40:40 smithi083 ceph-mon[79930]: pgmap v9372: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:40:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:40:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:40:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:40:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:40:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:40:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:40:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:40:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:40:40 smithi083 ceph-mon[77242]: from='client.45039 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:40:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:40:40 smithi083 ceph-mon[77242]: pgmap v9372: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:40:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:40:40.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:40:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:40:40.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:40:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:40:40.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:40:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:40:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:40:40 smithi153 ceph-mon[70044]: from='client.45039 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:40:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:40:40 smithi153 ceph-mon[70044]: pgmap v9372: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:40:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:40:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:40:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:40:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:40:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:40:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:40:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:40:41.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:40:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:40:41] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T10:40:41.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:40:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:40:41.310Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:40:42.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:40:42 smithi083 ceph-mon[79930]: pgmap v9373: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:42.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:40:42 smithi083 ceph-mon[77242]: pgmap v9373: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:40:42 smithi153 ceph-mon[70044]: pgmap v9373: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:40:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:40:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:40:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:40:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:40:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:40:44.692 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:40:44 smithi083 ceph-mon[79930]: pgmap v9374: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:44.693 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:40:44 smithi083 ceph-mon[77242]: pgmap v9374: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:44.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:40:44 smithi153 ceph-mon[70044]: pgmap v9374: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:44.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:40:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:40:44.691Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:40:45.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:40:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:40:45.395Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:40:46.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:40:46 smithi083 ceph-mon[79930]: pgmap v9375: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:46.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:40:46 smithi083 ceph-mon[77242]: pgmap v9375: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:46.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:40:46 smithi153 ceph-mon[70044]: pgmap v9375: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:48.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:40:48 smithi083 ceph-mon[79930]: pgmap v9376: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:48.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:40:48 smithi083 ceph-mon[77242]: pgmap v9376: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:48.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:40:48 smithi153 ceph-mon[70044]: pgmap v9376: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:50.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:40:49 smithi153 ceph-mon[70044]: pgmap v9377: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:50.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:40:49 smithi083 ceph-mon[79930]: pgmap v9377: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:50.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:40:49 smithi083 ceph-mon[77242]: pgmap v9377: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:51.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:40:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:40:51] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T10:40:51.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:40:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:40:51.311Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:40:52.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:40:52 smithi083 ceph-mon[79930]: pgmap v9378: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:52.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:40:52 smithi083 ceph-mon[77242]: pgmap v9378: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:52.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:40:52 smithi153 ceph-mon[70044]: pgmap v9378: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:40:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:40:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:40:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:40:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:40:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:40:54.692 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:40:54 smithi083 ceph-mon[79930]: pgmap v9379: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:54.692 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:40:54 smithi083 ceph-mon[77242]: pgmap v9379: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:54.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:40:54 smithi153 ceph-mon[70044]: pgmap v9379: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:54.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:40:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:40:54.691Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:40:55.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:40:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:40:55.396Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:40:56.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:40:56 smithi083 ceph-mon[79930]: pgmap v9380: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:56.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:40:56 smithi083 ceph-mon[77242]: pgmap v9380: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:56.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:40:56 smithi153 ceph-mon[70044]: pgmap v9380: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:58.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:40:58 smithi083 ceph-mon[79930]: pgmap v9381: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:58.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:40:58 smithi083 ceph-mon[77242]: pgmap v9381: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:40:58.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:40:58 smithi153 ceph-mon[70044]: pgmap v9381: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:00.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:40:59 smithi153 ceph-mon[70044]: pgmap v9382: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:00.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:40:59 smithi083 ceph-mon[79930]: pgmap v9382: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:00.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:40:59 smithi083 ceph-mon[77242]: pgmap v9382: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:01.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:41:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:41:01] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T10:41:01.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:41:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:41:01.312Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:41:02.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:41:02 smithi083 ceph-mon[77242]: pgmap v9383: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:02.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:41:02 smithi083 ceph-mon[79930]: pgmap v9383: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:02.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:41:02 smithi153 ceph-mon[70044]: pgmap v9383: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:41:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:41:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:41:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:41:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:41:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:41:04.692 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:41:04 smithi083 ceph-mon[79930]: pgmap v9384: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:04.693 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:41:04 smithi083 ceph-mon[77242]: pgmap v9384: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:04.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:41:04 smithi153 ceph-mon[70044]: pgmap v9384: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:04.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:41:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:41:04.691Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:41:05.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:41:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:41:05.397Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:41:06.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:41:06 smithi083 ceph-mon[79930]: pgmap v9385: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:06.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:41:06 smithi083 ceph-mon[77242]: pgmap v9385: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:06.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:41:06 smithi153 ceph-mon[70044]: pgmap v9385: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:08.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:41:08 smithi083 ceph-mon[79930]: pgmap v9386: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:08.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:41:08 smithi083 ceph-mon[77242]: pgmap v9386: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:08.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:41:08 smithi153 ceph-mon[70044]: pgmap v9386: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:09.531 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:41:09.823 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:41:09.824 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 10m ago 5h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:41:09.824 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 8m ago 5h 62.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:41:09.824 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 10m ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:41:09.824 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 8m ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:41:09.825 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 8m ago 5h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:41:09.825 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 10m ago 5h 774M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:41:09.825 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 10m ago 5h 1165M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:41:09.825 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 8m ago 5h 501M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:41:09.826 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 10m ago 5h 669M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:41:09.826 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 10m ago 5h 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:41:09.826 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 8m ago 5h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:41:09.826 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 10m ago 5h 819M 2464M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:41:09.827 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 10m ago 5h 809M 2464M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:41:09.827 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 10m ago 5h 632M 2464M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:41:09.827 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 10m ago 5h 789M 2464M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:41:09.827 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 8m ago 5h 696M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:41:09.828 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 8m ago 5h 867M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:41:09.828 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 8m ago 5h 774M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:41:09.828 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 8m ago 5h 916M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:41:09.828 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 8m ago 5h 187M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:41:10.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:41:09 smithi153 ceph-mon[70044]: pgmap v9387: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:10.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:41:09 smithi153 ceph-mon[70044]: from='client.54982 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:41:10.181 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:41:10.182 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:41:10.182 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:41:10.182 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:41:10.182 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:41:10.183 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:41:10.183 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:41:10.183 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:41:10.183 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:41:10.184 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:41:10.184 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:41:10.184 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:41:10.184 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:41:10.184 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:41:10.185 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:41:10.185 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:41:10.185 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:41:10.185 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:41:10.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:41:09 smithi083 ceph-mon[79930]: pgmap v9387: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:10.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:41:09 smithi083 ceph-mon[79930]: from='client.54982 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:41:10.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:41:09 smithi083 ceph-mon[77242]: pgmap v9387: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:10.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:41:09 smithi083 ceph-mon[77242]: from='client.54982 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:41:11.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:41:10 smithi153 ceph-mon[70044]: from='client.45057 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:41:11.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:41:10 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2079624614' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:41:11.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:41:10 smithi083 ceph-mon[79930]: from='client.45057 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:41:11.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:41:10 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2079624614' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:41:11.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:41:10 smithi083 ceph-mon[77242]: from='client.45057 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:41:11.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:41:10 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2079624614' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:41:11.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:41:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:41:11] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T10:41:11.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:41:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:41:11.312Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:41:12.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:41:11 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:41:12.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:41:11 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:41:12.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:41:11 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:41:12.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:41:11 smithi083 ceph-mon[79930]: pgmap v9388: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:12.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:41:11 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:41:12.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:41:11 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:41:12.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:41:11 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:41:12.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:41:11 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:41:12.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:41:11 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:41:12.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:41:11 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:41:12.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:41:11 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:41:12.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:41:11 smithi083 ceph-mon[77242]: pgmap v9388: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:12.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:41:11 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:41:12.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:41:11 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:41:12.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:41:11 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:41:12.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:41:11 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:41:12.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:41:11 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:41:12.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:41:11 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:41:12.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:41:11 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:41:12.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:41:11 smithi153 ceph-mon[70044]: pgmap v9388: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:12.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:41:11 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:41:12.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:41:11 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:41:12.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:41:11 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:41:12.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:41:11 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:41:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:41:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:41:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:41:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:41:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:41:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:41:14.692 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:41:14 smithi083 ceph-mon[79930]: pgmap v9389: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:14.693 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:41:14 smithi083 ceph-mon[77242]: pgmap v9389: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:14.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:41:14 smithi153 ceph-mon[70044]: pgmap v9389: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:14.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:41:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:41:14.691Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:41:15.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:41:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:41:15.396Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:41:16.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:41:16 smithi083 ceph-mon[79930]: pgmap v9390: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:16.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:41:16 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:41:16.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:41:16 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:41:16.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:41:16 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:41:16.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:41:16 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:41:16.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:41:16 smithi083 ceph-mon[77242]: pgmap v9390: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:16.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:41:16 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:41:16.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:41:16 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:41:16.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:41:16 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:41:16.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:41:16 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:41:17.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:41:16 smithi153 ceph-mon[70044]: pgmap v9390: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:17.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:41:16 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:41:17.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:41:16 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:41:17.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:41:16 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:41:17.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:41:16 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:41:18.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:41:18 smithi083 ceph-mon[79930]: pgmap v9391: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:18.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:41:18 smithi083 ceph-mon[77242]: pgmap v9391: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:19.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:41:18 smithi153 ceph-mon[70044]: pgmap v9391: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:41:20 smithi153 ceph-mon[70044]: Detected new or changed devices on smithi083 2022-09-29T10:41:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:41:20 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:41:20.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:41:20 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:41:20.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:41:20 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:41:20.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:41:20 smithi153 ceph-mon[70044]: pgmap v9392: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:20.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:41:20 smithi083 ceph-mon[79930]: Detected new or changed devices on smithi083 2022-09-29T10:41:20.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:41:20 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:41:20.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:41:20 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:41:20.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:41:20 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:41:20.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:41:20 smithi083 ceph-mon[79930]: pgmap v9392: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:20.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:41:20 smithi083 ceph-mon[77242]: Detected new or changed devices on smithi083 2022-09-29T10:41:20.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:41:20 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:41:20.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:41:20 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:41:20.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:41:20 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:41:20.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:41:20 smithi083 ceph-mon[77242]: pgmap v9392: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:21.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:41:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:41:21] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T10:41:21.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:41:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:41:21.314Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:41:22.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:41:22 smithi083 ceph-mon[79930]: pgmap v9393: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:22.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:41:22 smithi083 ceph-mon[77242]: pgmap v9393: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:22.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:41:22 smithi153 ceph-mon[70044]: pgmap v9393: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:41:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:41:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:41:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:41:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:41:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:41:24.694 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:41:24 smithi083 ceph-mon[79930]: pgmap v9394: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:24.695 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:41:24 smithi083 ceph-mon[77242]: pgmap v9394: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:24.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:41:24 smithi153 ceph-mon[70044]: pgmap v9394: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:24.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:41:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:41:24.692Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:41:25.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:41:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:41:25.397Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:41:26.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:41:26 smithi083 ceph-mon[79930]: pgmap v9395: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:26.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:41:26 smithi083 ceph-mon[77242]: pgmap v9395: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:26.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:41:26 smithi153 ceph-mon[70044]: pgmap v9395: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:28.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:41:28 smithi083 ceph-mon[79930]: pgmap v9396: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:28.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:41:28 smithi083 ceph-mon[77242]: pgmap v9396: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:28.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:41:28 smithi153 ceph-mon[70044]: pgmap v9396: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:30.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:41:29 smithi083 ceph-mon[79930]: pgmap v9397: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:30.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:41:29 smithi083 ceph-mon[77242]: pgmap v9397: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:30.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:41:29 smithi153 ceph-mon[70044]: pgmap v9397: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:31.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:41:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:41:31] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T10:41:31.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:41:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:41:31.315Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:41:32.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:41:32 smithi083 ceph-mon[79930]: pgmap v9398: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:32.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:41:32 smithi083 ceph-mon[77242]: pgmap v9398: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:32.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:41:32 smithi153 ceph-mon[70044]: pgmap v9398: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:41:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:41:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:41:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:41:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:41:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:41:34.694 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:41:34 smithi083 ceph-mon[79930]: pgmap v9399: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:34.695 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:41:34 smithi083 ceph-mon[77242]: pgmap v9399: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:34.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:41:34 smithi153 ceph-mon[70044]: pgmap v9399: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:34.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:41:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:41:34.693Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:41:35.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:41:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:41:35.398Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:41:36.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:41:36 smithi083 ceph-mon[77242]: pgmap v9400: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:36.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:41:36 smithi083 ceph-mon[79930]: pgmap v9400: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:36.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:41:36 smithi153 ceph-mon[70044]: pgmap v9400: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:38.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:41:38 smithi083 ceph-mon[79930]: pgmap v9401: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:38.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:41:38 smithi083 ceph-mon[77242]: pgmap v9401: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:38.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:41:38 smithi153 ceph-mon[70044]: pgmap v9401: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:40.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:41:39 smithi083 ceph-mon[79930]: pgmap v9402: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:40.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:41:39 smithi083 ceph-mon[77242]: pgmap v9402: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:41:39 smithi153 ceph-mon[70044]: pgmap v9402: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:40.513 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:41:40.801 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:41:40.801 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 25s ago 5h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:41:40.802 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 8m ago 5h 62.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:41:40.802 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 25s ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:41:40.802 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 8m ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:41:40.802 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 8m ago 5h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:41:40.803 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 25s ago 5h 783M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:41:40.803 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 25s ago 5h 1198M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:41:40.803 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 8m ago 5h 501M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:41:40.803 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 25s ago 5h 688M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:41:40.804 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 25s ago 5h 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:41:40.804 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 8m ago 5h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:41:40.804 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 25s ago 5h 844M 2464M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:41:40.804 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 25s ago 5h 834M 2464M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:41:40.804 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 25s ago 5h 651M 2464M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:41:40.805 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 25s ago 5h 814M 2464M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:41:40.805 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 8m ago 5h 696M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:41:40.805 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 8m ago 5h 867M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:41:40.805 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 8m ago 5h 774M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:41:40.806 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 8m ago 5h 916M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:41:40.806 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 8m ago 5h 187M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:41:41.159 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:41:41.163 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:41:41.164 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:41:41.164 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:41:41.164 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:41:41.164 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:41:41.165 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:41:41.165 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:41:41.165 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:41:41.165 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:41:41.166 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:41:41.166 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:41:41.166 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:41:41.166 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:41:41.167 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:41:41.167 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:41:41.167 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:41:41.167 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:41:41.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:41:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:41:41.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:41:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:41:41.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:41:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:41:41.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:41:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:41:41.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:41:40 smithi083 ceph-mon[79930]: from='client.55000 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:41:41.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:41:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:41:41.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:41:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:41:41.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:41:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:41:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:41:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:41:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:41:40 smithi083 ceph-mon[77242]: from='client.55000 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:41:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:41:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:41:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:41:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:41:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:41:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:41:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:41:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:41:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:41:40 smithi153 ceph-mon[70044]: from='client.55000 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:41:41.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:41:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:41:41] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T10:41:41.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:41:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:41:41.315Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:41:42.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:41:41 smithi083 ceph-mon[79930]: from='client.45075 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:41:42.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:41:41 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1257501164' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:41:42.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:41:41 smithi083 ceph-mon[79930]: pgmap v9403: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:42.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:41:41 smithi083 ceph-mon[77242]: from='client.45075 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:41:42.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:41:41 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1257501164' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:41:42.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:41:41 smithi083 ceph-mon[77242]: pgmap v9403: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:41:41 smithi153 ceph-mon[70044]: from='client.45075 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:41:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:41:41 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1257501164' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:41:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:41:41 smithi153 ceph-mon[70044]: pgmap v9403: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:41:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:41:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:41:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:41:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:41:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:41:44.695 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:41:44 smithi083 ceph-mon[79930]: pgmap v9404: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:44.695 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:41:44 smithi083 ceph-mon[77242]: pgmap v9404: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:44.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:41:44 smithi153 ceph-mon[70044]: pgmap v9404: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:44.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:41:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:41:44.694Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:41:45.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:41:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:41:45.398Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:41:46.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:41:46 smithi083 ceph-mon[79930]: pgmap v9405: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:46.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:41:46 smithi083 ceph-mon[77242]: pgmap v9405: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:46.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:41:46 smithi153 ceph-mon[70044]: pgmap v9405: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:48.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:41:48 smithi083 ceph-mon[79930]: pgmap v9406: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:48.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:41:48 smithi083 ceph-mon[77242]: pgmap v9406: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:48.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:41:48 smithi153 ceph-mon[70044]: pgmap v9406: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:50.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:41:49 smithi083 ceph-mon[79930]: pgmap v9407: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:50.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:41:49 smithi083 ceph-mon[77242]: pgmap v9407: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:50.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:41:49 smithi153 ceph-mon[70044]: pgmap v9407: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:51.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:41:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:41:51] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T10:41:51.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:41:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:41:51.316Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:41:52.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:41:52 smithi083 ceph-mon[79930]: pgmap v9408: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:52.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:41:52 smithi083 ceph-mon[77242]: pgmap v9408: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:52.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:41:52 smithi153 ceph-mon[70044]: pgmap v9408: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:41:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:41:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:41:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:41:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:41:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:41:54.694 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:41:54 smithi083 ceph-mon[79930]: pgmap v9409: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:54.695 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:41:54 smithi083 ceph-mon[77242]: pgmap v9409: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:54.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:41:54 smithi153 ceph-mon[70044]: pgmap v9409: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:54.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:41:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:41:54.693Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:41:55.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:41:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:41:55.399Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:41:56.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:41:56 smithi083 ceph-mon[79930]: pgmap v9410: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:56.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:41:56 smithi083 ceph-mon[77242]: pgmap v9410: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:56.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:41:56 smithi153 ceph-mon[70044]: pgmap v9410: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:58.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:41:58 smithi083 ceph-mon[79930]: pgmap v9411: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:58.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:41:58 smithi083 ceph-mon[77242]: pgmap v9411: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:41:58.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:41:58 smithi153 ceph-mon[70044]: pgmap v9411: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:00.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:41:59 smithi083 ceph-mon[77242]: pgmap v9412: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:00.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:41:59 smithi083 ceph-mon[79930]: pgmap v9412: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:41:59 smithi153 ceph-mon[70044]: pgmap v9412: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:01.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:42:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:42:01] "GET /metrics HTTP/1.1" 200 225229 "" "Prometheus/2.33.4" 2022-09-29T10:42:01.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:42:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:42:01.317Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:42:02.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:42:02 smithi083 ceph-mon[77242]: pgmap v9413: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:02.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:42:02 smithi083 ceph-mon[79930]: pgmap v9413: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:02.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:42:02 smithi153 ceph-mon[70044]: pgmap v9413: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:42:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:42:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:42:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:42:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:42:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:42:04.695 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:42:04 smithi083 ceph-mon[79930]: pgmap v9414: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:04.696 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:42:04 smithi083 ceph-mon[77242]: pgmap v9414: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:04.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:42:04 smithi153 ceph-mon[70044]: pgmap v9414: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:04.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:42:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:42:04.694Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:42:05.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:42:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:42:05.400Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:42:06.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:42:06 smithi083 ceph-mon[79930]: pgmap v9415: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:06.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:42:06 smithi083 ceph-mon[77242]: pgmap v9415: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:06.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:42:06 smithi153 ceph-mon[70044]: pgmap v9415: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:08.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:42:08 smithi083 ceph-mon[77242]: pgmap v9416: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:08.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:42:08 smithi083 ceph-mon[79930]: pgmap v9416: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:08.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:42:08 smithi153 ceph-mon[70044]: pgmap v9416: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:10.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:42:09 smithi083 ceph-mon[79930]: pgmap v9417: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:10.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:42:09 smithi083 ceph-mon[77242]: pgmap v9417: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:10.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:42:09 smithi153 ceph-mon[70044]: pgmap v9417: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:11.489 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:42:11.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:42:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:42:11] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T10:42:11.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:42:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:42:11.317Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:42:11.781 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:42:11.781 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 56s ago 5h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:42:11.781 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 9m ago 5h 62.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:42:11.782 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 56s ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:42:11.782 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 9m ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:42:11.782 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 9m ago 5h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:42:11.782 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 56s ago 5h 783M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:42:11.783 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 56s ago 5h 1198M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:42:11.783 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 9m ago 5h 501M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:42:11.783 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 56s ago 5h 688M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:42:11.783 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 56s ago 5h 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:42:11.783 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 9m ago 5h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:42:11.784 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 56s ago 5h 844M 2464M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:42:11.784 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 56s ago 5h 834M 2464M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:42:11.784 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 56s ago 5h 651M 2464M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:42:11.784 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 56s ago 5h 814M 2464M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:42:11.785 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 9m ago 5h 696M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:42:11.785 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 9m ago 5h 867M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:42:11.785 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 9m ago 5h 774M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:42:11.785 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 9m ago 5h 916M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:42:11.785 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 9m ago 5h 187M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:42:12.137 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:42:12.138 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:42:12.138 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:42:12.138 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:42:12.138 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:42:12.139 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:42:12.139 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:42:12.139 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:42:12.139 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:42:12.139 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:42:12.140 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:42:12.140 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:42:12.140 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:42:12.140 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:42:12.141 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:42:12.141 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:42:12.141 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:42:12.141 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:42:12.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:42:12 smithi083 ceph-mon[77242]: pgmap v9418: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:12.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:42:12 smithi083 ceph-mon[77242]: from='client.45087 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:42:12.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:42:12 smithi083 ceph-mon[77242]: from='client.55024 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:42:12.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:42:12 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2806762863' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:42:12.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:42:12 smithi083 ceph-mon[79930]: pgmap v9418: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:12.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:42:12 smithi083 ceph-mon[79930]: from='client.45087 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:42:12.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:42:12 smithi083 ceph-mon[79930]: from='client.55024 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:42:12.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:42:12 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2806762863' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:42:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:42:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:42:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:42:12.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:42:12 smithi153 ceph-mon[70044]: pgmap v9418: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:12.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:42:12 smithi153 ceph-mon[70044]: from='client.45087 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:42:12.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:42:12 smithi153 ceph-mon[70044]: from='client.55024 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:42:12.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:42:12 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2806762863' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:42:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:42:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:42:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:42:14.696 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:42:14 smithi083 ceph-mon[79930]: pgmap v9419: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:14.696 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:42:14 smithi083 ceph-mon[77242]: pgmap v9419: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:14.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:42:14 smithi153 ceph-mon[70044]: pgmap v9419: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:14.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:42:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:42:14.695Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:42:15.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:42:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:42:15.400Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:42:16.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:42:16 smithi083 ceph-mon[79930]: pgmap v9420: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:16.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:42:16 smithi083 ceph-mon[77242]: pgmap v9420: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:16.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:42:16 smithi153 ceph-mon[70044]: pgmap v9420: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:18.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:42:18 smithi083 ceph-mon[79930]: pgmap v9421: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:18.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:42:18 smithi083 ceph-mon[77242]: pgmap v9421: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:18.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:42:18 smithi153 ceph-mon[70044]: pgmap v9421: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:20.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:42:19 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:42:20.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:42:19 smithi083 ceph-mon[79930]: pgmap v9422: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:20.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:42:19 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:42:20.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:42:19 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:42:20.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:42:19 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:42:20.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:42:19 smithi083 ceph-mon[77242]: pgmap v9422: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:20.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:42:19 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:42:20.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:42:19 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:42:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:42:19 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:42:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:42:19 smithi153 ceph-mon[70044]: pgmap v9422: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:42:19 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:42:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:42:19 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:42:21.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:42:20 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:42:21.319 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:42:20 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:42:21.320 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:42:20 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:42:21.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:42:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:42:21] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T10:42:21.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:42:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:42:21.319Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:42:22.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:42:21 smithi153 ceph-mon[70044]: pgmap v9423: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:22.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:42:21 smithi083 ceph-mon[79930]: pgmap v9423: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:22.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:42:21 smithi083 ceph-mon[77242]: pgmap v9423: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:42:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:42:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:42:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:42:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:42:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:42:24.697 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:42:24 smithi083 ceph-mon[79930]: pgmap v9424: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:24.697 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:42:24 smithi083 ceph-mon[77242]: pgmap v9424: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:24.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:42:24 smithi153 ceph-mon[70044]: pgmap v9424: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:24.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:42:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:42:24.696Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:42:25.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:42:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:42:25.401Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:42:26.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:42:26 smithi083 ceph-mon[79930]: pgmap v9425: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:26.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:42:26 smithi083 ceph-mon[77242]: pgmap v9425: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:26.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:42:26 smithi153 ceph-mon[70044]: pgmap v9425: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:28.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:42:28 smithi083 ceph-mon[79930]: pgmap v9426: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:28.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:42:28 smithi083 ceph-mon[77242]: pgmap v9426: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:28.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:42:28 smithi153 ceph-mon[70044]: pgmap v9426: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:30.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:42:29 smithi083 ceph-mon[79930]: pgmap v9427: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:30.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:42:29 smithi083 ceph-mon[77242]: pgmap v9427: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:30.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:42:29 smithi153 ceph-mon[70044]: pgmap v9427: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:31.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:42:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:42:31] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T10:42:31.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:42:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:42:31.319Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:42:32.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:42:32 smithi083 ceph-mon[79930]: pgmap v9428: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:32.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:42:32 smithi083 ceph-mon[77242]: pgmap v9428: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:32.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:42:32 smithi153 ceph-mon[70044]: pgmap v9428: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:42:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:42:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:42:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:42:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:42:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:42:34.698 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:42:34 smithi083 ceph-mon[79930]: pgmap v9429: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:34.698 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:42:34 smithi083 ceph-mon[77242]: pgmap v9429: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:34.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:42:34 smithi153 ceph-mon[70044]: pgmap v9429: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:34.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:42:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:42:34.697Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:42:35.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:42:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:42:35.402Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:42:36.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:42:36 smithi083 ceph-mon[79930]: pgmap v9430: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:36.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:42:36 smithi083 ceph-mon[77242]: pgmap v9430: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:36.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:42:36 smithi153 ceph-mon[70044]: pgmap v9430: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:38.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:42:38 smithi083 ceph-mon[79930]: pgmap v9431: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:38.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:42:38 smithi083 ceph-mon[77242]: pgmap v9431: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:38.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:42:38 smithi153 ceph-mon[70044]: pgmap v9431: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:40.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:42:39 smithi083 ceph-mon[79930]: pgmap v9432: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:40.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:42:39 smithi083 ceph-mon[77242]: pgmap v9432: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:42:39 smithi153 ceph-mon[70044]: pgmap v9432: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:41.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:42:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:42:41.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:42:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:42:41.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:42:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:42:41.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:42:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:42:41.218 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:42:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:42:41.218 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:42:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:42:41.218 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:42:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:42:41.219 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:42:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:42:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:42:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:42:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:42:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:42:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:42:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:42:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:42:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:42:41.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:42:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:42:41] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T10:42:41.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:42:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:42:41.320Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:42:42.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:42:41 smithi083 ceph-mon[77242]: pgmap v9433: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:42.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:42:41 smithi083 ceph-mon[79930]: pgmap v9433: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:42:41 smithi153 ceph-mon[70044]: pgmap v9433: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:42.462 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:42:42.750 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:42:42.751 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 87s ago 5h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:42:42.751 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 9m ago 5h 62.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:42:42.751 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 87s ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:42:42.752 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 9m ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:42:42.752 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 9m ago 5h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:42:42.752 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 87s ago 5h 783M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:42:42.752 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 87s ago 5h 1198M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:42:42.752 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 9m ago 5h 501M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:42:42.753 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 87s ago 5h 688M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:42:42.753 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 87s ago 5h 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:42:42.753 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 9m ago 5h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:42:42.753 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 87s ago 5h 844M 2464M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:42:42.753 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 87s ago 5h 834M 2464M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:42:42.754 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 87s ago 5h 651M 2464M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:42:42.754 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 87s ago 5h 814M 2464M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:42:42.754 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 9m ago 5h 696M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:42:42.754 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 9m ago 5h 867M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:42:42.755 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 9m ago 5h 774M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:42:42.755 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 9m ago 5h 916M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:42:42.755 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 9m ago 5h 187M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:42:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:42:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:42:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:42:43.110 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:42:43.110 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:42:43.110 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:42:43.111 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:42:43.111 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:42:43.111 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:42:43.111 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:42:43.111 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:42:43.112 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:42:43.112 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:42:43.112 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:42:43.112 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:42:43.113 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:42:43.113 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:42:43.113 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:42:43.113 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:42:43.113 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:42:43.114 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:42:43.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:42:42 smithi083 ceph-mon[77242]: from='client.45105 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:42:43.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:42:42 smithi083 ceph-mon[77242]: from='client.55042 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:42:43.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:42:42 smithi083 ceph-mon[79930]: from='client.45105 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:42:43.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:42:42 smithi083 ceph-mon[79930]: from='client.55042 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:42:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:42:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:42:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:42:43.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:42:42 smithi153 ceph-mon[70044]: from='client.45105 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:42:43.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:42:42 smithi153 ceph-mon[70044]: from='client.55042 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:42:44.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:42:43 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2227209161' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:42:44.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:42:43 smithi083 ceph-mon[79930]: pgmap v9434: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:44.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:42:43 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2227209161' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:42:44.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:42:43 smithi083 ceph-mon[77242]: pgmap v9434: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:44.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:42:43 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2227209161' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:42:44.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:42:43 smithi153 ceph-mon[70044]: pgmap v9434: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:44.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:42:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:42:44.697Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:42:45.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:42:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:42:45.403Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:42:46.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:42:46 smithi083 ceph-mon[79930]: pgmap v9435: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:46.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:42:46 smithi083 ceph-mon[77242]: pgmap v9435: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:46.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:42:46 smithi153 ceph-mon[70044]: pgmap v9435: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:48.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:42:48 smithi083 ceph-mon[79930]: pgmap v9436: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:48.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:42:48 smithi083 ceph-mon[77242]: pgmap v9436: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:48.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:42:48 smithi153 ceph-mon[70044]: pgmap v9436: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:50.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:42:49 smithi083 ceph-mon[79930]: pgmap v9437: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:50.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:42:49 smithi083 ceph-mon[77242]: pgmap v9437: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:50.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:42:49 smithi153 ceph-mon[70044]: pgmap v9437: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:51.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:42:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:42:51] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T10:42:51.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:42:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:42:51.320Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:42:52.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:42:52 smithi083 ceph-mon[79930]: pgmap v9438: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:52.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:42:52 smithi083 ceph-mon[77242]: pgmap v9438: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:52.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:42:52 smithi153 ceph-mon[70044]: pgmap v9438: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:42:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:42:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:42:53.311 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:42:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:42:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:42:54.699 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:42:54 smithi083 ceph-mon[79930]: pgmap v9439: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:54.699 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:42:54 smithi083 ceph-mon[77242]: pgmap v9439: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:54.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:42:54 smithi153 ceph-mon[70044]: pgmap v9439: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:54.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:42:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:42:54.698Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:42:55.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:42:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:42:55.403Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:42:56.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:42:56 smithi083 ceph-mon[79930]: pgmap v9440: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:56.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:42:56 smithi083 ceph-mon[77242]: pgmap v9440: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:56.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:42:56 smithi153 ceph-mon[70044]: pgmap v9440: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:58.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:42:58 smithi083 ceph-mon[79930]: pgmap v9441: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:58.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:42:58 smithi083 ceph-mon[77242]: pgmap v9441: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:42:58.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:42:58 smithi153 ceph-mon[70044]: pgmap v9441: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:00.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:42:59 smithi083 ceph-mon[79930]: pgmap v9442: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:00.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:42:59 smithi083 ceph-mon[77242]: pgmap v9442: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:42:59 smithi153 ceph-mon[70044]: pgmap v9442: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:01.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:43:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:43:01] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T10:43:01.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:43:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:43:01.321Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:43:02.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:43:02 smithi083 ceph-mon[79930]: pgmap v9443: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:02.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:43:02 smithi083 ceph-mon[77242]: pgmap v9443: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:02.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:43:02 smithi153 ceph-mon[70044]: pgmap v9443: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:43:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:43:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:43:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:43:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:43:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:43:04.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:43:04 smithi083 ceph-mon[79930]: pgmap v9444: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:04.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:43:04 smithi083 ceph-mon[77242]: pgmap v9444: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:04.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:43:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:43:04.698Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:43:04.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:43:04 smithi153 ceph-mon[70044]: pgmap v9444: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:05.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:43:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:43:05.404Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:43:06.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:43:06 smithi083 ceph-mon[79930]: pgmap v9445: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:06.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:43:06 smithi083 ceph-mon[77242]: pgmap v9445: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:06.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:43:06 smithi153 ceph-mon[70044]: pgmap v9445: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:08.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:43:08 smithi083 ceph-mon[79930]: pgmap v9446: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:08.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:43:08 smithi083 ceph-mon[77242]: pgmap v9446: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:08.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:43:08 smithi153 ceph-mon[70044]: pgmap v9446: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:10.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:43:09 smithi083 ceph-mon[79930]: pgmap v9447: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:10.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:43:09 smithi083 ceph-mon[77242]: pgmap v9447: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:10.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:43:09 smithi153 ceph-mon[70044]: pgmap v9447: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:11.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:43:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:43:11] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T10:43:11.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:43:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:43:11.322Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:43:12.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:43:12 smithi083 ceph-mon[79930]: pgmap v9448: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:12.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:43:12 smithi083 ceph-mon[77242]: pgmap v9448: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:12.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:43:12 smithi153 ceph-mon[70044]: pgmap v9448: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:43:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:43:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:43:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:43:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:43:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:43:13.440 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:43:13.739 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:43:13.740 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 118s ago 5h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:43:13.740 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 10m ago 5h 62.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:43:13.740 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 118s ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:43:13.741 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 10m ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:43:13.741 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 10m ago 5h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:43:13.741 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 118s ago 5h 783M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:43:13.741 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 118s ago 5h 1198M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:43:13.742 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 10m ago 5h 501M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:43:13.742 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 118s ago 5h 688M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:43:13.742 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 118s ago 5h 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:43:13.742 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 10m ago 5h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:43:13.742 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 118s ago 5h 844M 2464M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:43:13.743 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 118s ago 5h 834M 2464M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:43:13.743 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 118s ago 5h 651M 2464M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:43:13.743 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 118s ago 5h 814M 2464M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:43:13.743 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 10m ago 5h 696M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:43:13.744 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 10m ago 5h 867M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:43:13.744 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 10m ago 5h 774M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:43:13.744 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 10m ago 5h 916M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:43:13.744 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 10m ago 5h 187M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:43:14.103 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:43:14.103 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:43:14.103 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:43:14.103 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:43:14.104 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:43:14.104 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:43:14.104 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:43:14.104 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:43:14.104 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:43:14.105 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:43:14.105 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:43:14.105 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:43:14.105 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:43:14.105 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:43:14.106 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:43:14.106 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:43:14.106 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:43:14.106 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:43:14.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:43:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:43:14.698Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:43:14.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:43:14 smithi083 ceph-mon[79930]: pgmap v9449: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:14.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:43:14 smithi083 ceph-mon[79930]: from='client.55054 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:43:14.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:43:14 smithi083 ceph-mon[79930]: from='client.45129 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:43:14.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:43:14 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2564950958' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:43:14.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:43:14 smithi083 ceph-mon[77242]: pgmap v9449: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:14.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:43:14 smithi083 ceph-mon[77242]: from='client.55054 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:43:14.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:43:14 smithi083 ceph-mon[77242]: from='client.45129 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:43:14.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:43:14 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2564950958' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:43:14.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:43:14 smithi153 ceph-mon[70044]: pgmap v9449: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:14.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:43:14 smithi153 ceph-mon[70044]: from='client.55054 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:43:14.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:43:14 smithi153 ceph-mon[70044]: from='client.45129 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:43:14.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:43:14 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2564950958' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:43:15.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:43:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:43:15.405Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:43:16.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:43:16 smithi083 ceph-mon[79930]: pgmap v9450: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:16.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:43:16 smithi083 ceph-mon[77242]: pgmap v9450: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:16.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:43:16 smithi153 ceph-mon[70044]: pgmap v9450: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:18.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:43:18 smithi083 ceph-mon[79930]: pgmap v9451: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:18.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:43:18 smithi083 ceph-mon[77242]: pgmap v9451: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:18.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:43:18 smithi153 ceph-mon[70044]: pgmap v9451: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:20.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:43:19 smithi083 ceph-mon[79930]: pgmap v9452: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:20.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:43:19 smithi083 ceph-mon[77242]: pgmap v9452: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:43:19 smithi153 ceph-mon[70044]: pgmap v9452: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:21.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:43:20 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:43:21.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:43:20 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:43:21.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:43:20 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:43:21.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:43:20 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:43:21.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:43:20 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:43:21.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:43:20 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:43:21.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:43:20 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:43:21.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:43:20 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:43:21.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:43:20 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:43:21.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:43:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:43:21] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T10:43:21.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:43:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:43:21.322Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:43:22.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:43:21 smithi083 ceph-mon[77242]: pgmap v9453: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:22.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:43:21 smithi083 ceph-mon[79930]: pgmap v9453: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:22.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:43:21 smithi153 ceph-mon[70044]: pgmap v9453: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:43:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:43:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:43:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:43:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:43:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:43:24.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:43:24 smithi083 ceph-mon[77242]: pgmap v9454: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:24.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:43:24 smithi083 ceph-mon[79930]: pgmap v9454: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:24.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:43:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:43:24.699Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:43:24.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:43:24 smithi153 ceph-mon[70044]: pgmap v9454: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:25.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:43:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:43:25.406Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:43:25.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:43:25 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:43:25.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:43:25 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:43:25.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:43:25 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:43:25.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:43:25 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:43:25.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:43:25 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:43:25.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:43:25 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:43:25.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:43:25 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:43:25.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:43:25 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:43:25.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:43:25 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:43:25.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:43:25 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:43:25.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:43:25 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:43:25.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:43:25 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:43:26.614 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:43:26 smithi153 ceph-mon[70044]: pgmap v9455: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:26.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:43:26 smithi083 ceph-mon[77242]: pgmap v9455: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:26.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:43:26 smithi083 ceph-mon[79930]: pgmap v9455: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:28.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:43:28 smithi153 ceph-mon[70044]: pgmap v9456: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:28.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:43:28 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:43:28.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:43:28 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:43:28.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:43:28 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:43:28.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:43:28 smithi083 ceph-mon[79930]: pgmap v9456: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:28.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:43:28 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:43:28.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:43:28 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:43:28.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:43:28 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:43:28.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:43:28 smithi083 ceph-mon[77242]: pgmap v9456: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:28.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:43:28 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:43:28.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:43:28 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:43:28.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:43:28 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:43:29.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:43:29 smithi153 ceph-mon[70044]: Detected new or changed devices on smithi153 2022-09-29T10:43:29.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:43:29 smithi083 ceph-mon[79930]: Detected new or changed devices on smithi153 2022-09-29T10:43:29.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:43:29 smithi083 ceph-mon[77242]: Detected new or changed devices on smithi153 2022-09-29T10:43:30.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:43:30 smithi153 ceph-mon[70044]: pgmap v9457: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:30.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:43:30 smithi083 ceph-mon[79930]: pgmap v9457: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:30.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:43:30 smithi083 ceph-mon[77242]: pgmap v9457: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:31.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:43:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:43:31] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T10:43:31.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:43:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:43:31.323Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:43:32.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:43:32 smithi153 ceph-mon[70044]: pgmap v9458: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:43:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:43:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:43:32.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:43:32 smithi083 ceph-mon[79930]: pgmap v9458: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:32.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:43:32 smithi083 ceph-mon[77242]: pgmap v9458: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:43:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:43:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:43:34.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:43:34 smithi153 ceph-mon[70044]: pgmap v9459: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:34.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:43:34 smithi083 ceph-mon[79930]: pgmap v9459: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:34.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:43:34 smithi083 ceph-mon[77242]: pgmap v9459: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:34.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:43:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:43:34.700Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:43:35.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:43:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:43:35.407Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:43:36.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:43:36 smithi153 ceph-mon[70044]: pgmap v9460: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:36.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:43:36 smithi083 ceph-mon[79930]: pgmap v9460: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:36.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:43:36 smithi083 ceph-mon[77242]: pgmap v9460: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:38.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:43:38 smithi153 ceph-mon[70044]: pgmap v9461: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:38.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:43:38 smithi083 ceph-mon[79930]: pgmap v9461: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:38.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:43:38 smithi083 ceph-mon[77242]: pgmap v9461: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:40.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:43:39 smithi083 ceph-mon[79930]: pgmap v9462: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:40.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:43:39 smithi083 ceph-mon[77242]: pgmap v9462: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:43:39 smithi153 ceph-mon[70044]: pgmap v9462: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:41.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:43:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:43:41.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:43:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:43:41.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:43:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:43:41.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:43:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:43:41.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:43:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:43:41.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:43:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:43:41.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:43:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:43:41.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:43:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:43:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:43:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:43:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:43:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:43:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:43:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:43:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:43:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:43:41.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:43:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:43:41.324Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:43:41.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:43:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:43:41] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T10:43:42.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:43:41 smithi083 ceph-mon[79930]: pgmap v9463: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:42.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:43:41 smithi083 ceph-mon[77242]: pgmap v9463: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:43:41 smithi153 ceph-mon[70044]: pgmap v9463: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:43:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:43:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:43:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:43:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:43:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:43:44.423 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:43:44.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:43:44 smithi083 ceph-mon[79930]: pgmap v9464: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:44.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:43:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:43:44.700Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:43:44.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:43:44 smithi083 ceph-mon[77242]: pgmap v9464: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:44.716 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:43:44.716 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 2m ago 5h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:43:44.716 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 20s ago 5h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:43:44.716 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 2m ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:43:44.717 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 20s ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:43:44.717 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 20s ago 5h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:43:44.717 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 2m ago 5h 783M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:43:44.717 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 2m ago 5h 1198M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:43:44.718 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 20s ago 5h 516M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:43:44.718 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 2m ago 5h 688M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:43:44.718 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 2m ago 5h 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:43:44.718 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 20s ago 5h 26.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:43:44.719 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 2m ago 5h 844M 2464M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:43:44.719 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 2m ago 5h 834M 2464M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:43:44.719 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 2m ago 5h 651M 2464M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:43:44.719 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 2m ago 5h 814M 2464M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:43:44.719 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 20s ago 5h 718M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:43:44.720 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 20s ago 5h 895M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:43:44.720 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 20s ago 5h 799M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:43:44.720 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 20s ago 5h 945M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:43:44.720 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 20s ago 5h 189M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:43:44.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:43:44 smithi153 ceph-mon[70044]: pgmap v9464: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:45.075 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:43:45.075 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:43:45.076 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:43:45.076 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:43:45.076 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:43:45.076 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:43:45.077 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:43:45.077 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:43:45.077 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:43:45.077 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:43:45.077 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:43:45.078 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:43:45.078 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:43:45.078 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:43:45.078 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:43:45.079 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:43:45.079 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:43:45.079 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:43:45.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:43:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:43:45.407Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:43:45.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:43:45 smithi083 ceph-mon[79930]: from='client.55072 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:43:45.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:43:45 smithi083 ceph-mon[79930]: from='client.45147 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:43:45.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:43:45 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4106684640' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:43:45.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:43:45 smithi083 ceph-mon[77242]: from='client.55072 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:43:45.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:43:45 smithi083 ceph-mon[77242]: from='client.45147 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:43:45.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:43:45 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4106684640' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:43:45.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:43:45 smithi153 ceph-mon[70044]: from='client.55072 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:43:45.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:43:45 smithi153 ceph-mon[70044]: from='client.45147 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:43:45.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:43:45 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4106684640' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:43:46.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:43:46 smithi153 ceph-mon[70044]: pgmap v9465: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:46.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:43:46 smithi083 ceph-mon[79930]: pgmap v9465: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:46.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:43:46 smithi083 ceph-mon[77242]: pgmap v9465: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:48.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:43:48 smithi153 ceph-mon[70044]: pgmap v9466: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:48.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:43:48 smithi083 ceph-mon[79930]: pgmap v9466: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:48.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:43:48 smithi083 ceph-mon[77242]: pgmap v9466: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:50.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:43:49 smithi083 ceph-mon[79930]: pgmap v9467: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:50.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:43:49 smithi083 ceph-mon[77242]: pgmap v9467: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:50.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:43:49 smithi153 ceph-mon[70044]: pgmap v9467: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:51.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:43:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:43:51] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T10:43:51.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:43:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:43:51.325Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:43:52.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:43:52 smithi153 ceph-mon[70044]: pgmap v9468: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:43:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:43:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:43:52.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:43:52 smithi083 ceph-mon[79930]: pgmap v9468: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:52.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:43:52 smithi083 ceph-mon[77242]: pgmap v9468: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:43:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:43:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:43:54.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:43:54 smithi153 ceph-mon[70044]: pgmap v9469: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:54.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:43:54 smithi083 ceph-mon[79930]: pgmap v9469: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:54.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:43:54 smithi083 ceph-mon[77242]: pgmap v9469: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:54.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:43:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:43:54.701Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:43:55.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:43:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:43:55.409Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:43:56.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:43:56 smithi153 ceph-mon[70044]: pgmap v9470: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:56.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:43:56 smithi083 ceph-mon[79930]: pgmap v9470: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:56.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:43:56 smithi083 ceph-mon[77242]: pgmap v9470: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:58.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:43:58 smithi153 ceph-mon[70044]: pgmap v9471: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:58.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:43:58 smithi083 ceph-mon[79930]: pgmap v9471: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:43:58.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:43:58 smithi083 ceph-mon[77242]: pgmap v9471: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:00.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:43:59 smithi083 ceph-mon[79930]: pgmap v9472: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:00.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:43:59 smithi083 ceph-mon[77242]: pgmap v9472: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:43:59 smithi153 ceph-mon[70044]: pgmap v9472: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:01.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:44:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:44:01] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T10:44:01.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:44:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:44:01.326Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:44:02.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:44:02 smithi153 ceph-mon[70044]: pgmap v9473: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:44:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:44:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:44:02.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:44:02 smithi083 ceph-mon[79930]: pgmap v9473: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:02.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:44:02 smithi083 ceph-mon[77242]: pgmap v9473: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:44:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:44:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:44:04.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:44:04 smithi153 ceph-mon[70044]: pgmap v9474: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:04.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:44:04 smithi083 ceph-mon[79930]: pgmap v9474: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:04.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:44:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:44:04.702Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:44:04.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:44:04 smithi083 ceph-mon[77242]: pgmap v9474: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:05.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:44:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:44:05.409Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:44:06.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:44:06 smithi153 ceph-mon[70044]: pgmap v9475: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:06.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:44:06 smithi083 ceph-mon[79930]: pgmap v9475: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:06.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:44:06 smithi083 ceph-mon[77242]: pgmap v9475: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:08.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:44:08 smithi153 ceph-mon[70044]: pgmap v9476: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:08.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:44:08 smithi083 ceph-mon[79930]: pgmap v9476: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:08.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:44:08 smithi083 ceph-mon[77242]: pgmap v9476: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:10.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:44:09 smithi083 ceph-mon[77242]: pgmap v9477: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:10.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:44:09 smithi083 ceph-mon[79930]: pgmap v9477: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:10.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:44:09 smithi153 ceph-mon[70044]: pgmap v9477: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:11.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:44:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:44:11] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T10:44:11.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:44:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:44:11.326Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:44:12.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:44:12 smithi153 ceph-mon[70044]: pgmap v9478: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:44:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:44:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:44:12.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:44:12 smithi083 ceph-mon[79930]: pgmap v9478: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:12.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:44:12 smithi083 ceph-mon[77242]: pgmap v9478: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:44:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:44:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:44:14.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:44:14 smithi153 ceph-mon[70044]: pgmap v9479: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:14.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:44:14 smithi083 ceph-mon[79930]: pgmap v9479: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:14.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:44:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:44:14.703Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:44:14.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:44:14 smithi083 ceph-mon[77242]: pgmap v9479: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:15.400 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:44:15.691 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:44:15.691 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 3m ago 5h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:44:15.692 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 51s ago 5h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:44:15.692 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 3m ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:44:15.692 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 51s ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:44:15.692 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 51s ago 5h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:44:15.693 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 3m ago 5h 783M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:44:15.693 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 3m ago 5h 1198M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:44:15.693 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 51s ago 5h 516M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:44:15.693 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 3m ago 5h 688M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:44:15.693 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 3m ago 5h 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:44:15.694 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 51s ago 5h 26.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:44:15.694 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 3m ago 5h 844M 2464M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:44:15.694 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 3m ago 5h 834M 2464M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:44:15.694 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 3m ago 5h 651M 2464M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:44:15.695 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 3m ago 5h 814M 2464M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:44:15.695 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 51s ago 5h 718M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:44:15.695 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 51s ago 5h 895M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:44:15.695 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 51s ago 5h 799M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:44:15.696 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 51s ago 5h 945M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:44:15.696 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 51s ago 5h 189M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:44:15.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:44:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:44:15.409Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:44:16.050 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:44:16.051 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:44:16.051 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:44:16.051 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:44:16.052 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:44:16.052 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:44:16.052 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:44:16.052 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:44:16.053 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:44:16.053 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:44:16.053 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:44:16.053 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:44:16.054 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:44:16.054 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:44:16.054 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:44:16.054 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:44:16.055 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:44:16.055 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:44:16.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:44:16 smithi153 ceph-mon[70044]: from='client.45159 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:44:16.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:44:16 smithi153 ceph-mon[70044]: pgmap v9480: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:16.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:44:16 smithi153 ceph-mon[70044]: from='client.45165 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:44:16.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:44:16 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/682135379' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:44:16.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:44:16 smithi083 ceph-mon[79930]: from='client.45159 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:44:16.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:44:16 smithi083 ceph-mon[79930]: pgmap v9480: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:16.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:44:16 smithi083 ceph-mon[79930]: from='client.45165 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:44:16.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:44:16 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/682135379' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:44:16.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:44:16 smithi083 ceph-mon[77242]: from='client.45159 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:44:16.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:44:16 smithi083 ceph-mon[77242]: pgmap v9480: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:16.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:44:16 smithi083 ceph-mon[77242]: from='client.45165 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:44:16.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:44:16 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/682135379' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:44:18.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:44:18 smithi153 ceph-mon[70044]: pgmap v9481: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:18.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:44:18 smithi083 ceph-mon[79930]: pgmap v9481: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:18.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:44:18 smithi083 ceph-mon[77242]: pgmap v9481: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:20.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:44:19 smithi083 ceph-mon[79930]: pgmap v9482: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:20.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:44:19 smithi083 ceph-mon[77242]: pgmap v9482: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:44:19 smithi153 ceph-mon[70044]: pgmap v9482: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:21.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:44:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:44:21] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T10:44:21.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:44:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:44:21.327Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:44:22.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:44:22 smithi153 ceph-mon[70044]: pgmap v9483: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:44:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:44:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:44:22.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:44:22 smithi083 ceph-mon[79930]: pgmap v9483: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:22.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:44:22 smithi083 ceph-mon[77242]: pgmap v9483: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:44:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:44:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:44:24.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:44:24 smithi153 ceph-mon[70044]: pgmap v9484: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:24.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:44:24 smithi083 ceph-mon[79930]: pgmap v9484: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:24.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:44:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:44:24.704Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:44:24.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:44:24 smithi083 ceph-mon[77242]: pgmap v9484: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:25.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:44:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:44:25.411Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:44:26.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:44:26 smithi153 ceph-mon[70044]: pgmap v9485: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:26.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:44:26 smithi083 ceph-mon[79930]: pgmap v9485: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:26.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:44:26 smithi083 ceph-mon[77242]: pgmap v9485: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:28.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:44:28 smithi153 ceph-mon[70044]: pgmap v9486: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:28.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:44:28 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:44:28.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:44:28 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:44:28.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:44:28 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:44:28.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:44:28 smithi083 ceph-mon[79930]: pgmap v9486: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:28.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:44:28 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:44:28.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:44:28 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:44:28.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:44:28 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:44:28.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:44:28 smithi083 ceph-mon[77242]: pgmap v9486: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:28.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:44:28 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:44:28.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:44:28 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:44:28.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:44:28 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:44:30.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:44:29 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:44:30.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:44:29 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:44:30.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:44:29 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:44:31.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:44:30 smithi153 ceph-mon[70044]: pgmap v9487: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:31.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:44:30 smithi083 ceph-mon[79930]: pgmap v9487: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:31.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:44:30 smithi083 ceph-mon[77242]: pgmap v9487: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:31.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:44:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:44:31] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T10:44:31.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:44:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:44:31.328Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:44:32.753 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:44:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:44:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:44:33.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:44:32 smithi153 ceph-mon[70044]: pgmap v9488: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:33.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:44:32 smithi083 ceph-mon[79930]: pgmap v9488: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:33.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:44:32 smithi083 ceph-mon[77242]: pgmap v9488: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:44:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:44:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:44:34.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:44:34 smithi083 ceph-mon[79930]: pgmap v9489: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:34.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:44:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:44:34.704Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:44:34.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:44:34 smithi083 ceph-mon[77242]: pgmap v9489: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:35.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:44:34 smithi153 ceph-mon[70044]: pgmap v9489: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:35.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:44:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:44:35.411Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:44:37.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:44:36 smithi153 ceph-mon[70044]: pgmap v9490: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:37.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:44:36 smithi083 ceph-mon[77242]: pgmap v9490: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:37.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:44:36 smithi083 ceph-mon[79930]: pgmap v9490: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:39.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:44:38 smithi153 ceph-mon[70044]: pgmap v9491: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:39.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:44:38 smithi083 ceph-mon[79930]: pgmap v9491: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:39.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:44:38 smithi083 ceph-mon[77242]: pgmap v9491: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:40.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:44:39 smithi083 ceph-mon[79930]: pgmap v9492: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:40.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:44:39 smithi083 ceph-mon[77242]: pgmap v9492: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:44:39 smithi153 ceph-mon[70044]: pgmap v9492: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:41.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:44:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:44:41.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:44:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:44:41.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:44:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:44:41.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:44:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:44:41.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:44:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:44:41.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:44:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:44:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:44:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:44:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:44:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:44:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:44:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:44:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:44:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:44:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:44:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:44:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:44:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:44:41.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:44:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:44:41.328Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:44:41.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:44:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:44:41] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T10:44:42.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:44:41 smithi083 ceph-mon[79930]: pgmap v9493: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:42.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:44:41 smithi083 ceph-mon[77242]: pgmap v9493: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:44:41 smithi153 ceph-mon[70044]: pgmap v9493: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:44:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:44:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:44:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:44:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:44:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:44:44.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:44:44 smithi153 ceph-mon[70044]: pgmap v9494: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:44.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:44:44 smithi083 ceph-mon[79930]: pgmap v9494: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:44.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:44:44 smithi083 ceph-mon[77242]: pgmap v9494: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:44.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:44:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:44:44.704Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:44:45.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:44:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:44:45.411Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:44:46.372 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:44:46.665 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:44:46.666 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 3m ago 5h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:44:46.666 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 82s ago 5h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:44:46.666 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 3m ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:44:46.667 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 82s ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:44:46.667 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 82s ago 5h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:44:46.667 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 3m ago 5h 783M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:44:46.667 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 3m ago 5h 1198M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:44:46.668 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 82s ago 5h 516M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:44:46.668 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 3m ago 5h 688M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:44:46.668 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 3m ago 5h 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:44:46.669 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 82s ago 5h 26.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:44:46.669 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 3m ago 5h 844M 2464M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:44:46.669 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 3m ago 5h 834M 2464M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:44:46.670 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 3m ago 5h 651M 2464M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:44:46.670 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 3m ago 5h 814M 2464M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:44:46.670 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 82s ago 5h 718M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:44:46.670 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 82s ago 5h 895M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:44:46.671 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 82s ago 5h 799M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:44:46.671 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 82s ago 5h 945M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:44:46.671 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 82s ago 5h 189M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:44:46.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:44:46 smithi153 ceph-mon[70044]: pgmap v9495: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:46.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:44:46 smithi083 ceph-mon[79930]: pgmap v9495: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:46.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:44:46 smithi083 ceph-mon[77242]: pgmap v9495: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:47.022 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:44:47.023 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:44:47.023 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:44:47.023 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:44:47.023 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:44:47.024 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:44:47.024 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:44:47.024 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:44:47.024 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:44:47.025 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:44:47.025 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:44:47.025 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:44:47.025 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:44:47.026 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:44:47.026 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:44:47.026 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:44:47.026 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:44:47.027 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:44:47.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:44:47 smithi153 ceph-mon[70044]: from='client.55108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:44:47.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:44:47 smithi153 ceph-mon[70044]: from='client.55114 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:44:47.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:44:47 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2646052935' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:44:47.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:44:47 smithi083 ceph-mon[79930]: from='client.55108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:44:47.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:44:47 smithi083 ceph-mon[79930]: from='client.55114 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:44:47.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:44:47 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2646052935' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:44:47.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:44:47 smithi083 ceph-mon[77242]: from='client.55108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:44:47.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:44:47 smithi083 ceph-mon[77242]: from='client.55114 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:44:47.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:44:47 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2646052935' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:44:48.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:44:48 smithi153 ceph-mon[70044]: pgmap v9496: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:48.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:44:48 smithi083 ceph-mon[79930]: pgmap v9496: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:48.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:44:48 smithi083 ceph-mon[77242]: pgmap v9496: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:50.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:44:49 smithi083 ceph-mon[79930]: pgmap v9497: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:50.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:44:49 smithi083 ceph-mon[77242]: pgmap v9497: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:50.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:44:49 smithi153 ceph-mon[70044]: pgmap v9497: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:51.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:44:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:44:51] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T10:44:51.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:44:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:44:51.330Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:44:52.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:44:52 smithi153 ceph-mon[70044]: pgmap v9498: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:44:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:44:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:44:52.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:44:52 smithi083 ceph-mon[79930]: pgmap v9498: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:52.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:44:52 smithi083 ceph-mon[77242]: pgmap v9498: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:44:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:44:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:44:54.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:44:54 smithi153 ceph-mon[70044]: pgmap v9499: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:54.970 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:44:54 smithi083 ceph-mon[77242]: pgmap v9499: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:54.971 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:44:54 smithi083 ceph-mon[79930]: pgmap v9499: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:54.971 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:44:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:44:54.704Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:44:55.723 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:44:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:44:55.412Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:44:56.813 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:44:56 smithi153 ceph-mon[70044]: pgmap v9500: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:56.970 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:44:56 smithi083 ceph-mon[79930]: pgmap v9500: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:56.971 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:44:56 smithi083 ceph-mon[77242]: pgmap v9500: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:58.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:44:58 smithi153 ceph-mon[70044]: pgmap v9501: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:58.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:44:58 smithi083 ceph-mon[79930]: pgmap v9501: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:44:58.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:44:58 smithi083 ceph-mon[77242]: pgmap v9501: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:00.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:44:59 smithi083 ceph-mon[79930]: pgmap v9502: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:00.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:44:59 smithi083 ceph-mon[77242]: pgmap v9502: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:00.316 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:44:59 smithi153 ceph-mon[70044]: pgmap v9502: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:01.714 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:45:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:45:01] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T10:45:01.715 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:45:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:45:01.331Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:45:02.814 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:45:02 smithi153 ceph-mon[70044]: pgmap v9503: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:02.828 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:45:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:45:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:45:02.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:45:02 smithi083 ceph-mon[77242]: pgmap v9503: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:02.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:45:02 smithi083 ceph-mon[79930]: pgmap v9503: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:03.318 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:45:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:45:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:45:04.813 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:45:04 smithi153 ceph-mon[70044]: pgmap v9504: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:04.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:45:04 smithi083 ceph-mon[79930]: pgmap v9504: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:04.972 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:45:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:45:04.705Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:45:04.973 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:45:04 smithi083 ceph-mon[77242]: pgmap v9504: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:05.722 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:45:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:45:05.412Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:45:06.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:45:06 smithi153 ceph-mon[70044]: pgmap v9505: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:06.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:45:06 smithi083 ceph-mon[77242]: pgmap v9505: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:06.976 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:45:06 smithi083 ceph-mon[79930]: pgmap v9505: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:08.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:45:08 smithi153 ceph-mon[70044]: pgmap v9506: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:08.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:45:08 smithi083 ceph-mon[79930]: pgmap v9506: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:08.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:45:08 smithi083 ceph-mon[77242]: pgmap v9506: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:10.222 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:45:09 smithi083 ceph-mon[79930]: pgmap v9507: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:10.223 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:45:09 smithi083 ceph-mon[77242]: pgmap v9507: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:10.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:45:09 smithi153 ceph-mon[70044]: pgmap v9507: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:11.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:45:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:45:11] "GET /metrics HTTP/1.1" 200 225229 "" "Prometheus/2.33.4" 2022-09-29T10:45:11.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:45:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:45:11.331Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:45:12.818 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:45:12 smithi153 ceph-mon[70044]: pgmap v9508: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:12.819 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:45:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:45:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:45:12.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:45:12 smithi083 ceph-mon[77242]: pgmap v9508: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:12.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:45:12 smithi083 ceph-mon[79930]: pgmap v9508: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:13.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:45:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:45:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:45:14.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:45:14 smithi153 ceph-mon[70044]: pgmap v9509: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:14.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:45:14 smithi083 ceph-mon[79930]: pgmap v9509: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:14.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:45:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:45:14.706Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:45:14.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:45:14 smithi083 ceph-mon[77242]: pgmap v9509: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:15.717 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:45:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:45:15.413Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:45:16.818 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:45:16 smithi153 ceph-mon[70044]: pgmap v9510: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:16.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:45:16 smithi083 ceph-mon[79930]: pgmap v9510: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:16.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:45:16 smithi083 ceph-mon[77242]: pgmap v9510: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:17.347 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:45:17.644 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:45:17.645 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 4m ago 5h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:45:17.645 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 113s ago 5h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:45:17.646 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 4m ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:45:17.646 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 113s ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:45:17.647 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 113s ago 5h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:45:17.647 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 4m ago 5h 783M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:45:17.647 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 4m ago 5h 1198M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:45:17.666 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 113s ago 5h 516M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:45:17.666 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 4m ago 5h 688M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:45:17.667 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 4m ago 5h 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:45:17.667 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 113s ago 5h 26.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:45:17.668 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 4m ago 5h 844M 2464M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:45:17.668 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 4m ago 5h 834M 2464M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:45:17.669 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 4m ago 5h 651M 2464M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:45:17.701 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 4m ago 5h 814M 2464M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:45:17.702 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 113s ago 5h 718M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:45:17.702 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 113s ago 5h 895M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:45:17.703 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 113s ago 5h 799M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:45:17.703 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 113s ago 5h 945M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:45:17.704 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 113s ago 5h 189M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:45:17.993 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:45:17.993 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:45:18.004 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:45:18.004 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:45:18.005 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:45:18.005 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:45:18.006 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:45:18.032 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:45:18.033 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:45:18.034 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:45:18.037 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:45:18.037 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:45:18.038 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:45:18.038 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:45:18.038 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:45:18.051 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:45:18.052 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:45:18.052 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:45:18.814 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:45:18 smithi153 ceph-mon[70044]: from='client.55126 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:45:18.815 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:45:18 smithi153 ceph-mon[70044]: pgmap v9511: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:18.815 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:45:18 smithi153 ceph-mon[70044]: from='client.55132 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:45:18.815 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:45:18 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2097241481' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:45:18.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:45:18 smithi083 ceph-mon[79930]: from='client.55126 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:45:18.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:45:18 smithi083 ceph-mon[79930]: pgmap v9511: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:18.968 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:45:18 smithi083 ceph-mon[79930]: from='client.55132 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:45:18.984 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:45:18 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2097241481' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:45:18.985 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:45:18 smithi083 ceph-mon[77242]: from='client.55126 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:45:18.986 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:45:18 smithi083 ceph-mon[77242]: pgmap v9511: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:18.986 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:45:18 smithi083 ceph-mon[77242]: from='client.55132 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:45:18.986 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:45:18 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2097241481' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:45:20.224 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:45:19 smithi083 ceph-mon[79930]: pgmap v9512: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:20.235 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:45:19 smithi083 ceph-mon[77242]: pgmap v9512: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:20.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:45:19 smithi153 ceph-mon[70044]: pgmap v9512: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:21.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:45:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:45:21] "GET /metrics HTTP/1.1" 200 225229 "" "Prometheus/2.33.4" 2022-09-29T10:45:21.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:45:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:45:21.332Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:45:22.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:45:22 smithi153 ceph-mon[70044]: pgmap v9513: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:45:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:45:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:45:22.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:45:22 smithi083 ceph-mon[77242]: pgmap v9513: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:22.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:45:22 smithi083 ceph-mon[79930]: pgmap v9513: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:45:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:45:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:45:24.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:45:24 smithi153 ceph-mon[70044]: pgmap v9514: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:24.970 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:45:24 smithi083 ceph-mon[79930]: pgmap v9514: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:24.972 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:45:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:45:24.707Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:45:24.973 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:45:24 smithi083 ceph-mon[77242]: pgmap v9514: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:25.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:45:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:45:25.414Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:45:26.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:45:26 smithi153 ceph-mon[70044]: pgmap v9515: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:26.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:45:26 smithi083 ceph-mon[79930]: pgmap v9515: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:26.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:45:26 smithi083 ceph-mon[77242]: pgmap v9515: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:28.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:45:28 smithi153 ceph-mon[70044]: pgmap v9516: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:28.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:45:28 smithi083 ceph-mon[79930]: pgmap v9516: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:28.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:45:28 smithi083 ceph-mon[77242]: pgmap v9516: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:29.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:45:29 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:45:29.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:45:29 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:45:29.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:45:29 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:45:29.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:45:29 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:45:29.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:45:29 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:45:29.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:45:29 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:45:29.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:45:29 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:45:29.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:45:29 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:45:29.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:45:29 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:45:29.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:45:29 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:45:29.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:45:29 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:45:29.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:45:29 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:45:30.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:45:30 smithi153 ceph-mon[70044]: pgmap v9517: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:30.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:45:30 smithi083 ceph-mon[79930]: pgmap v9517: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:30.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:45:30 smithi083 ceph-mon[77242]: pgmap v9517: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:31.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:45:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:45:31] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T10:45:31.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:45:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:45:31.332Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:45:32.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:45:32 smithi153 ceph-mon[70044]: pgmap v9518: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:45:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:45:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:45:32.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:45:32 smithi083 ceph-mon[77242]: pgmap v9518: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:32.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:45:32 smithi083 ceph-mon[79930]: pgmap v9518: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:45:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:45:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:45:34.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:45:34 smithi153 ceph-mon[70044]: pgmap v9519: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:34.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:45:34 smithi083 ceph-mon[79930]: pgmap v9519: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:34.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:45:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:45:34.707Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:45:34.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:45:34 smithi083 ceph-mon[77242]: pgmap v9519: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:35.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:45:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:45:35.415Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:45:36.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:45:36 smithi153 ceph-mon[70044]: pgmap v9520: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:36.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:45:36 smithi083 ceph-mon[79930]: pgmap v9520: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:36.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:45:36 smithi083 ceph-mon[77242]: pgmap v9520: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:38.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:45:38 smithi153 ceph-mon[70044]: pgmap v9521: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:38.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:45:38 smithi083 ceph-mon[79930]: pgmap v9521: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:38.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:45:38 smithi083 ceph-mon[77242]: pgmap v9521: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:40.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:45:39 smithi083 ceph-mon[77242]: pgmap v9522: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:45:39 smithi083 ceph-mon[79930]: pgmap v9522: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:45:39 smithi153 ceph-mon[70044]: pgmap v9522: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:41.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:45:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:45:41.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:45:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:45:41.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:45:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:45:41.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:45:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:45:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:45:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:45:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:45:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:45:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:45:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:45:41.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:45:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:45:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:45:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:45:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:45:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:45:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:45:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:45:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:45:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:45:41.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:45:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:45:41] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T10:45:41.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:45:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:45:41.332Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:45:42.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:45:41 smithi083 ceph-mon[79930]: pgmap v9523: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:42.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:45:41 smithi083 ceph-mon[77242]: pgmap v9523: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:42.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:45:41 smithi153 ceph-mon[70044]: pgmap v9523: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:45:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:45:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:45:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:45:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:45:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:45:44.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:45:44 smithi153 ceph-mon[70044]: pgmap v9524: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:44.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:45:44 smithi083 ceph-mon[77242]: pgmap v9524: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:44.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:45:44 smithi083 ceph-mon[79930]: pgmap v9524: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:44.966 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:45:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:45:44.707Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:45:45.716 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:45:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:45:45.416Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:45:46.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:45:46 smithi153 ceph-mon[70044]: pgmap v9525: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:46.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:45:46 smithi083 ceph-mon[79930]: pgmap v9525: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:46.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:45:46 smithi083 ceph-mon[77242]: pgmap v9525: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:48.320 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:45:48.612 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:45:48.613 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 4m ago 5h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:45:48.613 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 2m ago 5h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:45:48.614 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 4m ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:45:48.614 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 2m ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:45:48.615 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 2m ago 5h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:45:48.615 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 4m ago 5h 783M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:45:48.615 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 4m ago 5h 1198M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:45:48.616 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 2m ago 5h 516M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:45:48.616 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 4m ago 5h 688M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:45:48.617 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 4m ago 5h 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:45:48.617 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 2m ago 5h 26.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:45:48.618 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 4m ago 5h 844M 2464M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:45:48.619 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 4m ago 5h 834M 2464M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:45:48.619 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 4m ago 5h 651M 2464M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:45:48.619 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 4m ago 5h 814M 2464M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:45:48.619 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 2m ago 5h 718M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:45:48.620 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 2m ago 5h 895M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:45:48.620 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 2m ago 5h 799M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:45:48.620 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 2m ago 5h 945M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:45:48.620 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 2m ago 5h 189M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:45:48.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:45:48 smithi153 ceph-mon[70044]: pgmap v9526: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:48.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:45:48 smithi083 ceph-mon[79930]: pgmap v9526: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:48.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:45:48 smithi083 ceph-mon[77242]: pgmap v9526: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:48.965 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:45:48.967 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:45:48.967 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:45:48.969 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:45:48.969 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:45:48.970 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:45:48.973 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:45:48.974 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:45:48.974 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:45:48.974 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:45:48.975 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:45:48.975 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:45:48.976 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:45:48.978 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:45:48.978 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:45:48.979 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:45:48.979 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:45:48.980 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:45:49.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:45:49 smithi153 ceph-mon[70044]: from='client.45213 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:45:49.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:45:49 smithi153 ceph-mon[70044]: from='client.45219 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:45:49.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:45:49 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2577466670' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:45:49.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:45:49 smithi083 ceph-mon[79930]: from='client.45213 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:45:49.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:45:49 smithi083 ceph-mon[79930]: from='client.45219 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:45:49.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:45:49 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2577466670' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:45:49.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:45:49 smithi083 ceph-mon[77242]: from='client.45213 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:45:49.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:45:49 smithi083 ceph-mon[77242]: from='client.45219 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:45:49.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:45:49 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2577466670' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:45:50.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:45:50 smithi153 ceph-mon[70044]: pgmap v9527: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:50.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:45:50 smithi083 ceph-mon[79930]: pgmap v9527: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:50.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:45:50 smithi083 ceph-mon[77242]: pgmap v9527: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:51.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:45:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:45:51] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T10:45:51.715 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:45:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:45:51.333Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:45:52.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:45:52 smithi153 ceph-mon[70044]: pgmap v9528: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:52.810 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:45:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:45:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:45:52.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:45:52 smithi083 ceph-mon[79930]: pgmap v9528: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:52.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:45:52 smithi083 ceph-mon[77242]: pgmap v9528: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:45:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:45:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:45:54.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:45:54 smithi153 ceph-mon[70044]: pgmap v9529: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:54.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:45:54 smithi083 ceph-mon[79930]: pgmap v9529: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:54.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:45:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:45:54.709Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:45:54.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:45:54 smithi083 ceph-mon[77242]: pgmap v9529: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:55.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:45:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:45:55.417Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:45:56.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:45:56 smithi153 ceph-mon[70044]: pgmap v9530: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:56.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:45:56 smithi083 ceph-mon[77242]: pgmap v9530: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:56.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:45:56 smithi083 ceph-mon[79930]: pgmap v9530: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:58.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:45:58 smithi153 ceph-mon[70044]: pgmap v9531: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:58.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:45:58 smithi083 ceph-mon[77242]: pgmap v9531: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:45:58.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:45:58 smithi083 ceph-mon[79930]: pgmap v9531: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:00.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:45:59 smithi083 ceph-mon[79930]: pgmap v9532: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:00.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:45:59 smithi083 ceph-mon[77242]: pgmap v9532: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:00.319 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:45:59 smithi153 ceph-mon[70044]: pgmap v9532: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:01.715 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:46:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:46:01] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T10:46:01.716 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:46:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:46:01.333Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:46:02.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:46:02 smithi153 ceph-mon[70044]: pgmap v9533: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:02.810 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:46:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:46:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:46:02.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:46:02 smithi083 ceph-mon[79930]: pgmap v9533: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:02.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:46:02 smithi083 ceph-mon[77242]: pgmap v9533: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:46:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:46:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:46:04.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:46:04 smithi153 ceph-mon[70044]: pgmap v9534: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:04.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:46:04 smithi083 ceph-mon[79930]: pgmap v9534: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:04.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:46:04 smithi083 ceph-mon[77242]: pgmap v9534: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:04.967 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:46:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:46:04.709Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:46:05.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:46:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:46:05.417Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:46:06.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:46:06 smithi153 ceph-mon[70044]: pgmap v9535: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:06.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:46:06 smithi083 ceph-mon[77242]: pgmap v9535: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:06.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:46:06 smithi083 ceph-mon[79930]: pgmap v9535: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:08.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:46:08 smithi153 ceph-mon[70044]: pgmap v9536: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:08.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:46:08 smithi083 ceph-mon[77242]: pgmap v9536: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:08.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:46:08 smithi083 ceph-mon[79930]: pgmap v9536: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:10.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:46:09 smithi083 ceph-mon[77242]: pgmap v9537: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:10.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:46:09 smithi083 ceph-mon[79930]: pgmap v9537: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:10.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:46:09 smithi153 ceph-mon[70044]: pgmap v9537: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:11.714 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:46:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:46:11] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T10:46:11.715 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:46:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:46:11.334Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:46:12.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:46:12 smithi153 ceph-mon[70044]: pgmap v9538: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:12.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:46:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:46:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:46:12.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:46:12 smithi083 ceph-mon[77242]: pgmap v9538: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:12.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:46:12 smithi083 ceph-mon[79930]: pgmap v9538: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:13.310 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:46:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:46:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:46:14.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:46:14 smithi153 ceph-mon[70044]: pgmap v9539: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:14.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:46:14 smithi083 ceph-mon[77242]: pgmap v9539: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:14.975 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:46:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:46:14.710Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:46:14.976 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:46:14 smithi083 ceph-mon[79930]: pgmap v9539: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:15.714 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:46:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:46:15.418Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:46:16.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:46:16 smithi153 ceph-mon[70044]: pgmap v9540: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:16.971 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:46:16 smithi083 ceph-mon[77242]: pgmap v9540: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:16.977 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:46:16 smithi083 ceph-mon[79930]: pgmap v9540: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:18.813 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:46:18 smithi153 ceph-mon[70044]: pgmap v9541: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:18.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:46:18 smithi083 ceph-mon[77242]: pgmap v9541: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:18.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:46:18 smithi083 ceph-mon[79930]: pgmap v9541: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:19.302 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:46:19.578 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:46:19.587 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 5m ago 5h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:46:19.587 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 2m ago 5h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:46:19.588 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 5m ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:46:19.599 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 2m ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:46:19.600 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 2m ago 5h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:46:19.600 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 5m ago 5h 783M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:46:19.600 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 5m ago 5h 1198M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:46:19.601 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 2m ago 5h 516M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:46:19.601 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 5m ago 5h 688M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:46:19.602 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 5m ago 5h 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:46:19.602 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 2m ago 5h 26.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:46:19.603 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 5m ago 5h 844M 2464M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:46:19.603 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 5m ago 5h 834M 2464M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:46:19.603 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 5m ago 5h 651M 2464M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:46:19.604 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 5m ago 5h 814M 2464M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:46:19.604 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 2m ago 5h 718M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:46:19.605 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 2m ago 5h 895M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:46:19.605 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 2m ago 5h 799M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:46:19.606 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 2m ago 5h 945M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:46:19.606 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 2m ago 5h 189M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:46:19.962 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:46:19.963 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:46:19.963 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:46:19.963 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:46:19.964 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:46:19.964 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:46:19.964 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:46:19.964 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:46:19.965 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:46:19.965 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:46:19.965 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:46:19.965 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:46:19.965 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:46:19.980 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:46:19.980 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:46:19.981 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:46:19.981 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:46:19.981 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:46:20.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:46:19 smithi083 ceph-mon[77242]: from='client.45231 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:46:20.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:46:19 smithi083 ceph-mon[77242]: pgmap v9542: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:20.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:46:19 smithi083 ceph-mon[77242]: from='client.45237 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:46:20.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:46:19 smithi083 ceph-mon[79930]: from='client.45231 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:46:20.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:46:19 smithi083 ceph-mon[79930]: pgmap v9542: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:20.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:46:19 smithi083 ceph-mon[79930]: from='client.45237 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:46:20.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:46:19 smithi153 ceph-mon[70044]: from='client.45231 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:46:20.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:46:19 smithi153 ceph-mon[70044]: pgmap v9542: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:20.312 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:46:19 smithi153 ceph-mon[70044]: from='client.45237 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:46:21.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:46:20 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/57832742' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:46:21.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:46:20 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/57832742' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:46:21.313 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:46:20 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/57832742' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:46:21.714 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:46:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:46:21] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T10:46:21.714 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:46:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:46:21.335Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:46:22.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:46:21 smithi083 ceph-mon[79930]: pgmap v9543: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:22.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:46:21 smithi083 ceph-mon[77242]: pgmap v9543: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:22.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:46:21 smithi153 ceph-mon[70044]: pgmap v9543: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:22.815 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:46:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:46:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:46:23.321 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:46:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:46:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:46:24.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:46:24 smithi153 ceph-mon[70044]: pgmap v9544: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:24.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:46:24 smithi083 ceph-mon[79930]: pgmap v9544: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:24.965 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:46:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:46:24.710Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:46:24.977 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:46:24 smithi083 ceph-mon[77242]: pgmap v9544: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:25.726 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:46:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:46:25.419Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:46:26.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:46:26 smithi153 ceph-mon[70044]: pgmap v9545: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:26.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:46:26 smithi083 ceph-mon[77242]: pgmap v9545: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:26.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:46:26 smithi083 ceph-mon[79930]: pgmap v9545: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:28.829 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:46:28 smithi153 ceph-mon[70044]: pgmap v9546: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:28.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:46:28 smithi083 ceph-mon[79930]: pgmap v9546: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:28.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:46:28 smithi083 ceph-mon[77242]: pgmap v9546: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:30.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:46:29 smithi083 ceph-mon[79930]: pgmap v9547: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:30.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:46:29 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:46:30.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:46:29 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:46:30.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:46:29 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:46:30.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:46:29 smithi083 ceph-mon[77242]: pgmap v9547: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:30.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:46:29 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:46:30.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:46:29 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:46:30.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:46:29 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:46:30.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:46:29 smithi153 ceph-mon[70044]: pgmap v9547: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:30.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:46:29 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:46:30.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:46:29 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:46:30.312 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:46:29 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:46:31.337 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:46:31 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:46:31.338 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:46:31 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:46:31.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:46:31 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:46:31.715 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:46:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:46:31] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T10:46:31.716 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:46:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:46:31.335Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:46:32.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:46:32 smithi083 ceph-mon[79930]: pgmap v9548: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:32.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:46:32 smithi083 ceph-mon[77242]: pgmap v9548: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:32.564 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:46:32 smithi153 ceph-mon[70044]: pgmap v9548: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:32.570 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:46:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:46:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:46:33.311 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:46:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:46:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:46:34.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:46:34 smithi153 ceph-mon[70044]: pgmap v9549: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:34.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:46:34 smithi083 ceph-mon[77242]: pgmap v9549: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:34.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:46:34 smithi083 ceph-mon[79930]: pgmap v9549: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:34.965 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:46:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:46:34.710Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:46:35.714 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:46:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:46:35.419Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:46:36.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:46:36 smithi153 ceph-mon[70044]: pgmap v9550: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:36.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:46:36 smithi083 ceph-mon[79930]: pgmap v9550: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:36.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:46:36 smithi083 ceph-mon[77242]: pgmap v9550: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:38.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:46:38 smithi153 ceph-mon[70044]: pgmap v9551: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:38.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:46:38 smithi083 ceph-mon[79930]: pgmap v9551: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:38.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:46:38 smithi083 ceph-mon[77242]: pgmap v9551: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:40.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:46:39 smithi083 ceph-mon[79930]: pgmap v9552: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:40.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:46:39 smithi083 ceph-mon[77242]: pgmap v9552: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:40.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:46:39 smithi153 ceph-mon[70044]: pgmap v9552: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:41.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:46:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:46:41.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:46:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:46:41.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:46:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:46:41.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:46:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:46:41.220 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:46:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:46:41.220 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:46:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:46:41.241 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:46:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:46:41.272 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:46:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:46:41.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:46:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:46:41.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:46:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:46:41.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:46:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:46:41.346 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:46:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:46:41.722 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:46:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:46:41.336Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:46:41.723 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:46:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:46:41] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T10:46:42.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:46:41 smithi083 ceph-mon[77242]: pgmap v9553: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:42.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:46:41 smithi083 ceph-mon[79930]: pgmap v9553: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:42.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:46:41 smithi153 ceph-mon[70044]: pgmap v9553: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:46:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:46:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:46:43.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:46:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:46:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:46:44.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:46:44 smithi153 ceph-mon[70044]: pgmap v9554: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:44.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:46:44 smithi083 ceph-mon[79930]: pgmap v9554: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:44.967 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:46:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:46:44.711Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:46:44.968 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:46:44 smithi083 ceph-mon[77242]: pgmap v9554: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:45.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:46:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:46:45.420Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:46:46.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:46:46 smithi153 ceph-mon[70044]: pgmap v9555: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:46.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:46:46 smithi083 ceph-mon[77242]: pgmap v9555: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:46.975 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:46:46 smithi083 ceph-mon[79930]: pgmap v9555: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:48.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:46:48 smithi153 ceph-mon[70044]: pgmap v9556: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:48.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:46:48 smithi083 ceph-mon[79930]: pgmap v9556: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:48.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:46:48 smithi083 ceph-mon[77242]: pgmap v9556: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:50.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:46:49 smithi083 ceph-mon[79930]: pgmap v9557: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:50.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:46:49 smithi083 ceph-mon[77242]: pgmap v9557: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:50.261 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:46:50.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:46:49 smithi153 ceph-mon[70044]: pgmap v9557: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:50.548 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:46:50.548 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 5m ago 5h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:46:50.549 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 3m ago 5h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:46:50.549 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 5m ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:46:50.550 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 3m ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:46:50.550 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 3m ago 5h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:46:50.551 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 5m ago 5h 783M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:46:50.551 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 5m ago 5h 1198M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:46:50.551 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 3m ago 5h 516M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:46:50.552 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 5m ago 5h 688M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:46:50.552 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 5m ago 5h 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:46:50.552 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 3m ago 5h 26.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:46:50.553 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 5m ago 5h 844M 2464M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:46:50.553 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 5m ago 5h 834M 2464M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:46:50.562 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 5m ago 5h 651M 2464M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:46:50.563 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 5m ago 5h 814M 2464M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:46:50.564 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 3m ago 5h 718M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:46:50.564 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 3m ago 5h 895M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:46:50.564 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 3m ago 5h 799M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:46:50.565 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 3m ago 5h 945M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:46:50.565 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 3m ago 5h 189M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:46:50.907 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:46:50.909 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:46:50.909 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:46:50.910 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:46:50.910 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:46:50.911 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:46:50.911 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:46:50.912 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:46:50.912 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:46:50.913 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:46:50.913 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:46:50.914 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:46:50.914 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:46:50.915 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:46:50.915 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:46:50.937 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:46:50.938 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:46:50.938 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:46:51.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:46:50 smithi083 ceph-mon[79930]: from='client.55180 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:46:51.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:46:50 smithi083 ceph-mon[79930]: from='client.45255 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:46:51.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:46:50 smithi083 ceph-mon[77242]: from='client.55180 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:46:51.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:46:50 smithi083 ceph-mon[77242]: from='client.45255 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:46:51.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:46:50 smithi153 ceph-mon[70044]: from='client.55180 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:46:51.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:46:50 smithi153 ceph-mon[70044]: from='client.45255 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:46:51.718 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:46:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:46:51] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T10:46:51.719 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:46:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:46:51.337Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:46:52.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:46:51 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2150175513' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:46:52.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:46:51 smithi083 ceph-mon[79930]: pgmap v9558: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:52.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:46:51 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2150175513' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:46:52.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:46:51 smithi083 ceph-mon[77242]: pgmap v9558: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:52.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:46:51 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2150175513' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:46:52.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:46:51 smithi153 ceph-mon[70044]: pgmap v9558: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:52.810 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:46:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:46:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:46:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:46:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:46:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:46:54.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:46:54 smithi153 ceph-mon[70044]: pgmap v9559: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:54.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:46:54 smithi083 ceph-mon[79930]: pgmap v9559: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:54.971 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:46:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:46:54.712Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:46:54.989 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:46:54 smithi083 ceph-mon[77242]: pgmap v9559: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:55.714 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:46:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:46:55.420Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:46:56.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:46:56 smithi153 ceph-mon[70044]: pgmap v9560: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:56.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:46:56 smithi083 ceph-mon[79930]: pgmap v9560: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:56.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:46:56 smithi083 ceph-mon[77242]: pgmap v9560: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:58.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:46:58 smithi153 ceph-mon[70044]: pgmap v9561: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:58.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:46:58 smithi083 ceph-mon[79930]: pgmap v9561: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:46:58.983 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:46:58 smithi083 ceph-mon[77242]: pgmap v9561: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:00.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:46:59 smithi083 ceph-mon[79930]: pgmap v9562: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:00.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:46:59 smithi083 ceph-mon[77242]: pgmap v9562: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:00.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:46:59 smithi153 ceph-mon[70044]: pgmap v9562: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:01.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:47:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:47:01] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T10:47:01.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:47:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:47:01.338Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:47:02.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:47:02 smithi153 ceph-mon[70044]: pgmap v9563: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:02.810 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:47:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:47:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:47:02.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:47:02 smithi083 ceph-mon[77242]: pgmap v9563: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:02.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:47:02 smithi083 ceph-mon[79930]: pgmap v9563: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:47:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:47:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:47:04.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:47:04 smithi153 ceph-mon[70044]: pgmap v9564: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:04.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:47:04 smithi083 ceph-mon[79930]: pgmap v9564: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:04.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:47:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:47:04.712Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:47:04.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:47:04 smithi083 ceph-mon[77242]: pgmap v9564: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:05.720 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:47:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:47:05.420Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:47:06.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:47:06 smithi153 ceph-mon[70044]: pgmap v9565: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:06.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:47:06 smithi083 ceph-mon[77242]: pgmap v9565: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:06.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:47:06 smithi083 ceph-mon[79930]: pgmap v9565: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:08.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:47:08 smithi153 ceph-mon[70044]: pgmap v9566: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:08.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:47:08 smithi083 ceph-mon[79930]: pgmap v9566: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:08.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:47:08 smithi083 ceph-mon[77242]: pgmap v9566: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:10.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:47:09 smithi083 ceph-mon[79930]: pgmap v9567: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:10.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:47:09 smithi083 ceph-mon[77242]: pgmap v9567: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:10.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:47:09 smithi153 ceph-mon[70044]: pgmap v9567: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:11.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:47:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:47:11] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T10:47:11.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:47:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:47:11.339Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:47:12.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:47:12 smithi153 ceph-mon[70044]: pgmap v9568: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:47:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:47:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:47:12.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:47:12 smithi083 ceph-mon[79930]: pgmap v9568: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:12.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:47:12 smithi083 ceph-mon[77242]: pgmap v9568: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:47:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:47:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:47:14.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:47:14 smithi153 ceph-mon[70044]: pgmap v9569: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:14.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:47:14 smithi083 ceph-mon[79930]: pgmap v9569: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:14.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:47:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:47:14.713Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:47:14.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:47:14 smithi083 ceph-mon[77242]: pgmap v9569: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:15.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:47:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:47:15.421Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:47:16.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:47:16 smithi153 ceph-mon[70044]: pgmap v9570: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:16.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:47:16 smithi083 ceph-mon[79930]: pgmap v9570: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:16.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:47:16 smithi083 ceph-mon[77242]: pgmap v9570: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:18.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:47:18 smithi153 ceph-mon[70044]: pgmap v9571: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:18.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:47:18 smithi083 ceph-mon[79930]: pgmap v9571: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:18.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:47:18 smithi083 ceph-mon[77242]: pgmap v9571: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:20.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:47:19 smithi083 ceph-mon[77242]: pgmap v9572: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:20.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:47:19 smithi083 ceph-mon[79930]: pgmap v9572: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:47:19 smithi153 ceph-mon[70044]: pgmap v9572: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:21.231 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:47:21.525 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:47:21.527 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 6m ago 5h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:47:21.527 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 3m ago 5h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:47:21.528 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 6m ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:47:21.540 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 3m ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:47:21.540 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 3m ago 5h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:47:21.541 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 6m ago 5h 783M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:47:21.541 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 6m ago 5h 1198M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:47:21.542 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 3m ago 5h 516M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:47:21.542 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 6m ago 5h 688M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:47:21.543 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 6m ago 5h 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:47:21.543 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 3m ago 5h 26.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:47:21.543 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 6m ago 5h 844M 2464M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:47:21.544 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 6m ago 5h 834M 2464M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:47:21.544 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 6m ago 5h 651M 2464M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:47:21.545 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 6m ago 5h 814M 2464M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:47:21.545 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 3m ago 5h 718M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:47:21.548 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 3m ago 5h 895M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:47:21.549 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 3m ago 5h 799M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:47:21.549 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 3m ago 5h 945M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:47:21.550 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 3m ago 5h 189M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:47:21.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:47:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:47:21] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T10:47:21.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:47:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:47:21.338Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:47:21.881 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:47:21.881 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:47:21.882 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:47:21.882 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:47:21.883 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:47:21.883 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:47:21.883 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:47:21.883 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:47:21.884 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:47:21.885 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:47:21.885 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:47:21.885 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:47:21.886 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:47:21.886 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:47:21.886 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:47:21.886 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:47:21.887 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:47:21.888 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:47:22.810 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:47:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:47:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:47:22.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:47:22 smithi153 ceph-mon[70044]: from='client.55198 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:47:22.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:47:22 smithi153 ceph-mon[70044]: pgmap v9573: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:22.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:47:22 smithi153 ceph-mon[70044]: from='client.55204 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:47:22.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:47:22 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2708686548' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:47:22.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:47:22 smithi083 ceph-mon[77242]: from='client.55198 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:47:22.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:47:22 smithi083 ceph-mon[77242]: pgmap v9573: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:22.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:47:22 smithi083 ceph-mon[77242]: from='client.55204 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:47:22.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:47:22 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2708686548' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:47:22.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:47:22 smithi083 ceph-mon[79930]: from='client.55198 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:47:22.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:47:22 smithi083 ceph-mon[79930]: pgmap v9573: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:22.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:47:22 smithi083 ceph-mon[79930]: from='client.55204 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:47:22.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:47:22 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2708686548' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:47:23.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:47:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:47:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:47:24.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:47:24 smithi153 ceph-mon[70044]: pgmap v9574: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:24.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:47:24 smithi083 ceph-mon[79930]: pgmap v9574: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:24.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:47:24 smithi083 ceph-mon[77242]: pgmap v9574: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:24.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:47:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:47:24.713Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:47:25.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:47:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:47:25.422Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:47:26.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:47:26 smithi153 ceph-mon[70044]: pgmap v9575: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:26.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:47:26 smithi083 ceph-mon[79930]: pgmap v9575: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:26.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:47:26 smithi083 ceph-mon[77242]: pgmap v9575: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:28.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:47:28 smithi153 ceph-mon[70044]: pgmap v9576: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:28.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:47:28 smithi083 ceph-mon[79930]: pgmap v9576: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:28.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:47:28 smithi083 ceph-mon[77242]: pgmap v9576: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:30.220 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:47:29 smithi083 ceph-mon[79930]: pgmap v9577: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:30.221 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:47:29 smithi083 ceph-mon[77242]: pgmap v9577: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:30.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:47:29 smithi153 ceph-mon[70044]: pgmap v9577: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:31.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:47:30 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:47:31.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:47:30 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:47:31.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:47:30 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:47:31.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:47:30 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:47:31.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:47:30 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:47:31.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:47:30 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:47:31.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:47:30 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:47:31.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:47:30 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:47:31.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:47:30 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:47:31.312 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:47:30 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:47:31.312 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:47:30 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:47:31.313 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:47:30 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:47:31.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:47:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:47:31] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T10:47:31.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:47:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:47:31.340Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:47:32.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:47:31 smithi083 ceph-mon[79930]: pgmap v9578: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:32.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:47:31 smithi083 ceph-mon[77242]: pgmap v9578: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:32.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:47:31 smithi153 ceph-mon[70044]: pgmap v9578: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:32.817 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:47:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:47:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:47:33.326 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:47:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:47:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:47:34.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:47:34 smithi153 ceph-mon[70044]: pgmap v9579: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:34.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:47:34 smithi083 ceph-mon[77242]: pgmap v9579: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:34.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:47:34 smithi083 ceph-mon[79930]: pgmap v9579: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:34.967 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:47:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:47:34.714Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:47:35.719 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:47:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:47:35.422Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:47:36.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:47:36 smithi153 ceph-mon[70044]: pgmap v9580: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:36.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:47:36 smithi083 ceph-mon[77242]: pgmap v9580: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:36.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:47:36 smithi083 ceph-mon[79930]: pgmap v9580: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:38.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:47:38 smithi153 ceph-mon[70044]: pgmap v9581: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:38.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:47:38 smithi083 ceph-mon[79930]: pgmap v9581: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:38.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:47:38 smithi083 ceph-mon[77242]: pgmap v9581: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:47:39 smithi083 ceph-mon[79930]: pgmap v9582: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:47:39 smithi083 ceph-mon[77242]: pgmap v9582: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:40.316 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:47:39 smithi153 ceph-mon[70044]: pgmap v9582: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:41.218 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:47:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:47:41.219 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:47:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:47:41.219 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:47:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:47:41.219 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:47:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:47:41.220 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:47:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:47:41.220 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:47:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:47:41.220 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:47:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:47:41.221 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:47:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:47:41.315 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:47:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:47:41.315 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:47:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:47:41.315 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:47:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:47:41.316 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:47:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:47:41.715 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:47:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:47:41] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T10:47:41.716 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:47:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:47:41.341Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:47:42.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:47:41 smithi083 ceph-mon[79930]: pgmap v9583: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:42.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:47:41 smithi083 ceph-mon[77242]: pgmap v9583: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:47:41 smithi153 ceph-mon[70044]: pgmap v9583: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:47:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:47:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:47:43.318 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:47:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:47:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:47:44.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:47:44 smithi153 ceph-mon[70044]: pgmap v9584: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:44.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:47:44 smithi083 ceph-mon[79930]: pgmap v9584: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:44.964 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:47:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:47:44.716Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:47:44.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:47:44 smithi083 ceph-mon[77242]: pgmap v9584: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:45.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:47:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:47:45.423Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:47:46.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:47:46 smithi153 ceph-mon[70044]: pgmap v9585: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:46.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:47:46 smithi083 ceph-mon[79930]: pgmap v9585: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:46.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:47:46 smithi083 ceph-mon[77242]: pgmap v9585: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:48.815 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:47:48 smithi153 ceph-mon[70044]: pgmap v9586: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:48.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:47:48 smithi083 ceph-mon[79930]: pgmap v9586: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:48.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:47:48 smithi083 ceph-mon[77242]: pgmap v9586: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:50.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:47:49 smithi083 ceph-mon[79930]: pgmap v9587: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:50.219 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:47:49 smithi083 ceph-mon[77242]: pgmap v9587: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:50.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:47:49 smithi153 ceph-mon[70044]: pgmap v9587: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:51.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:47:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:47:51] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T10:47:51.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:47:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:47:51.342Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:47:52.209 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:47:52.497 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:47:52.497 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 6m ago 5h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:47:52.497 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 4m ago 5h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:47:52.497 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 6m ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:47:52.498 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 4m ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:47:52.498 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 4m ago 5h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:47:52.498 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 6m ago 5h 783M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:47:52.499 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 6m ago 5h 1198M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:47:52.499 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 4m ago 5h 516M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:47:52.499 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 6m ago 5h 688M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:47:52.499 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 6m ago 5h 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:47:52.499 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 4m ago 5h 26.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:47:52.500 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 6m ago 5h 844M 2464M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:47:52.500 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 6m ago 5h 834M 2464M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:47:52.501 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 6m ago 5h 651M 2464M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:47:52.501 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 6m ago 5h 814M 2464M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:47:52.502 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 4m ago 5h 718M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:47:52.512 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 4m ago 5h 895M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:47:52.512 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 4m ago 5h 799M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:47:52.513 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 4m ago 5h 945M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:47:52.513 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 4m ago 5h 189M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:47:52.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:47:52 smithi153 ceph-mon[70044]: pgmap v9588: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:52.810 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:47:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:47:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:47:52.873 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:47:52.874 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:47:52.874 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:47:52.875 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:47:52.875 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:47:52.876 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:47:52.876 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:47:52.876 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:47:52.877 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:47:52.883 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:47:52.883 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:47:52.884 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:47:52.884 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:47:52.884 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:47:52.884 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:47:52.884 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:47:52.885 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:47:52.885 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:47:52.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:47:52 smithi083 ceph-mon[79930]: pgmap v9588: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:52.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:47:52 smithi083 ceph-mon[77242]: pgmap v9588: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:53.323 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:47:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:47:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:47:53.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:47:53 smithi153 ceph-mon[70044]: from='client.55216 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:47:53.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:47:53 smithi153 ceph-mon[70044]: from='client.45291 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:47:53.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:47:53 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/642770939' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:47:53.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:47:53 smithi083 ceph-mon[79930]: from='client.55216 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:47:53.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:47:53 smithi083 ceph-mon[79930]: from='client.45291 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:47:53.968 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:47:53 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/642770939' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:47:53.968 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:47:53 smithi083 ceph-mon[77242]: from='client.55216 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:47:53.968 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:47:53 smithi083 ceph-mon[77242]: from='client.45291 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:47:53.968 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:47:53 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/642770939' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:47:54.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:47:54 smithi153 ceph-mon[70044]: pgmap v9589: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:54.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:47:54 smithi083 ceph-mon[79930]: pgmap v9589: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:54.966 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:47:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:47:54.716Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:47:54.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:47:54 smithi083 ceph-mon[77242]: pgmap v9589: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:55.727 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:47:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:47:55.424Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:47:56.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:47:56 smithi153 ceph-mon[70044]: pgmap v9590: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:56.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:47:56 smithi083 ceph-mon[79930]: pgmap v9590: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:56.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:47:56 smithi083 ceph-mon[77242]: pgmap v9590: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:58.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:47:58 smithi153 ceph-mon[70044]: pgmap v9591: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:58.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:47:58 smithi083 ceph-mon[77242]: pgmap v9591: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:47:58.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:47:58 smithi083 ceph-mon[79930]: pgmap v9591: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:00.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:47:59 smithi083 ceph-mon[79930]: pgmap v9592: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:00.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:47:59 smithi083 ceph-mon[77242]: pgmap v9592: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:00.318 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:47:59 smithi153 ceph-mon[70044]: pgmap v9592: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:01.715 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:48:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:48:01] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T10:48:01.716 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:48:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:48:01.342Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:48:02.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:48:02 smithi153 ceph-mon[70044]: pgmap v9593: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:02.811 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:48:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:48:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:48:02.999 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:48:02 smithi083 ceph-mon[79930]: pgmap v9593: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:03.000 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:48:02 smithi083 ceph-mon[77242]: pgmap v9593: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:03.350 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:48:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:48:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:48:04.818 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:48:04 smithi153 ceph-mon[70044]: pgmap v9594: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:04.977 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:48:04 smithi083 ceph-mon[79930]: pgmap v9594: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:04.977 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:48:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:48:04.715Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:48:04.978 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:48:04 smithi083 ceph-mon[77242]: pgmap v9594: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:05.720 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:48:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:48:05.424Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:48:06.815 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:48:06 smithi153 ceph-mon[70044]: pgmap v9595: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:06.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:48:06 smithi083 ceph-mon[79930]: pgmap v9595: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:06.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:48:06 smithi083 ceph-mon[77242]: pgmap v9595: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:08.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:48:08 smithi153 ceph-mon[70044]: pgmap v9596: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:08.968 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:48:08 smithi083 ceph-mon[79930]: pgmap v9596: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:08.970 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:48:08 smithi083 ceph-mon[77242]: pgmap v9596: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:10.219 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:48:09 smithi083 ceph-mon[77242]: pgmap v9597: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:10.220 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:48:09 smithi083 ceph-mon[79930]: pgmap v9597: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:10.314 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:48:09 smithi153 ceph-mon[70044]: pgmap v9597: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:11.721 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:48:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:48:11] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T10:48:11.722 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:48:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:48:11.344Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:48:12.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:48:12 smithi153 ceph-mon[70044]: pgmap v9598: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:48:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:48:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:48:12.981 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:48:12 smithi083 ceph-mon[77242]: pgmap v9598: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:12.982 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:48:12 smithi083 ceph-mon[79930]: pgmap v9598: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:48:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:48:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:48:14.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:48:14 smithi153 ceph-mon[70044]: pgmap v9599: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:14.971 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:48:14 smithi083 ceph-mon[79930]: pgmap v9599: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:14.980 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:48:14 smithi083 ceph-mon[77242]: pgmap v9599: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:14.981 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:48:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:48:14.717Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:48:15.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:48:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:48:15.425Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:48:16.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:48:16 smithi153 ceph-mon[70044]: pgmap v9600: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:16.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:48:16 smithi083 ceph-mon[77242]: pgmap v9600: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:16.968 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:48:16 smithi083 ceph-mon[79930]: pgmap v9600: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:18.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:48:18 smithi153 ceph-mon[70044]: pgmap v9601: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:18.968 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:48:18 smithi083 ceph-mon[79930]: pgmap v9601: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:18.969 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:48:18 smithi083 ceph-mon[77242]: pgmap v9601: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:20.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:48:19 smithi083 ceph-mon[79930]: pgmap v9602: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:20.234 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:48:19 smithi083 ceph-mon[77242]: pgmap v9602: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:48:19 smithi153 ceph-mon[70044]: pgmap v9602: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:21.715 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:48:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:48:21] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T10:48:21.716 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:48:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:48:21.344Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:48:22.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:48:22 smithi153 ceph-mon[70044]: pgmap v9603: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:22.810 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:48:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:48:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:48:22.971 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:48:22 smithi083 ceph-mon[77242]: pgmap v9603: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:22.972 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:48:22 smithi083 ceph-mon[79930]: pgmap v9603: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:23.183 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:48:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:48:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:48:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:48:23.504 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:48:23.505 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 7m ago 5h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:48:23.505 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 4m ago 5h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:48:23.516 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 7m ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:48:23.516 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 4m ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:48:23.517 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 4m ago 5h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:48:23.517 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 7m ago 5h 783M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:48:23.537 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 7m ago 5h 1198M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:48:23.537 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 4m ago 5h 516M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:48:23.537 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 7m ago 5h 688M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:48:23.538 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 7m ago 5h 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:48:23.538 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 4m ago 5h 26.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:48:23.538 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 7m ago 5h 844M 2464M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:48:23.538 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 7m ago 5h 834M 2464M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:48:23.539 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 7m ago 5h 651M 2464M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:48:23.539 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 7m ago 5h 814M 2464M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:48:23.539 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 4m ago 5h 718M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:48:23.539 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 4m ago 5h 895M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:48:23.540 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 4m ago 5h 799M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:48:23.540 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 4m ago 5h 945M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:48:23.540 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 4m ago 5h 189M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:48:23.854 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:48:23.855 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:48:23.855 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:48:23.856 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:48:23.856 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:48:23.857 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:48:23.857 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:48:23.862 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:48:23.863 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:48:23.863 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:48:23.864 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:48:23.864 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:48:23.864 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:48:23.865 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:48:23.865 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:48:23.866 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:48:23.867 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:48:23.867 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:48:24.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:48:24 smithi153 ceph-mon[70044]: from='client.55234 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:48:24.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:48:24 smithi153 ceph-mon[70044]: from='client.55240 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:48:24.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:48:24 smithi153 ceph-mon[70044]: pgmap v9604: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:24.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:48:24 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3732650287' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:48:24.976 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:48:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:48:24.717Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:48:24.977 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:48:24 smithi083 ceph-mon[77242]: from='client.55234 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:48:24.977 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:48:24 smithi083 ceph-mon[77242]: from='client.55240 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:48:24.979 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:48:24 smithi083 ceph-mon[77242]: pgmap v9604: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:24.979 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:48:24 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3732650287' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:48:24.980 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:48:24 smithi083 ceph-mon[79930]: from='client.55234 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:48:24.980 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:48:24 smithi083 ceph-mon[79930]: from='client.55240 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:48:24.980 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:48:24 smithi083 ceph-mon[79930]: pgmap v9604: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:24.981 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:48:24 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3732650287' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:48:25.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:48:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:48:25.426Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:48:26.819 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:48:26 smithi153 ceph-mon[70044]: pgmap v9605: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:26.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:48:26 smithi083 ceph-mon[79930]: pgmap v9605: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:26.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:48:26 smithi083 ceph-mon[77242]: pgmap v9605: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:28.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:48:28 smithi153 ceph-mon[70044]: pgmap v9606: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:28.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:48:28 smithi083 ceph-mon[79930]: pgmap v9606: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:28.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:48:28 smithi083 ceph-mon[77242]: pgmap v9606: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:30.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:48:29 smithi083 ceph-mon[77242]: pgmap v9607: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:30.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:48:29 smithi083 ceph-mon[79930]: pgmap v9607: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:30.317 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:48:29 smithi153 ceph-mon[70044]: pgmap v9607: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:31.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:48:30 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:48:31.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:48:30 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:48:31.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:48:30 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:48:31.268 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:48:30 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:48:31.268 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:48:30 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:48:31.269 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:48:30 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:48:31.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:48:30 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:48:31.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:48:30 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:48:31.312 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:48:30 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:48:31.715 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:48:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:48:31] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T10:48:31.716 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:48:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:48:31.350Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:48:32.723 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:48:32 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:48:32.723 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:48:32 smithi083 ceph-mon[77242]: pgmap v9608: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:32.724 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:48:32 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:48:32.724 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:48:32 smithi083 ceph-mon[79930]: pgmap v9608: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:32.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:48:32 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:48:32.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:48:32 smithi153 ceph-mon[70044]: pgmap v9608: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:32.812 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:48:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:48:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:48:33.323 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:48:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:48:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:48:34.826 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:48:34 smithi153 ceph-mon[70044]: pgmap v9609: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:34.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:48:34 smithi083 ceph-mon[79930]: pgmap v9609: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:34.969 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:48:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:48:34.718Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:48:34.969 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:48:34 smithi083 ceph-mon[77242]: pgmap v9609: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:35.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:48:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:48:35.427Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:48:36.830 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:48:36 smithi153 ceph-mon[70044]: pgmap v9610: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:36.970 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:48:36 smithi083 ceph-mon[77242]: pgmap v9610: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:36.971 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:48:36 smithi083 ceph-mon[79930]: pgmap v9610: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:38.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:48:38 smithi153 ceph-mon[70044]: pgmap v9611: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:38.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:48:38 smithi083 ceph-mon[79930]: pgmap v9611: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:38.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:48:38 smithi083 ceph-mon[77242]: pgmap v9611: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:48:39 smithi083 ceph-mon[79930]: pgmap v9612: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:40.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:48:39 smithi083 ceph-mon[77242]: pgmap v9612: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:48:39 smithi153 ceph-mon[70044]: pgmap v9612: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:41.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:48:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:48:41.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:48:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:48:41.218 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:48:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:48:41.219 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:48:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:48:41.220 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:48:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:48:41.220 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:48:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:48:41.220 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:48:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:48:41.220 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:48:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:48:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:48:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:48:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:48:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:48:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:48:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:48:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:48:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:48:41.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:48:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:48:41] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T10:48:41.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:48:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:48:41.345Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:48:42.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:48:41 smithi083 ceph-mon[79930]: pgmap v9613: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:42.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:48:41 smithi083 ceph-mon[77242]: pgmap v9613: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:42.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:48:41 smithi153 ceph-mon[70044]: pgmap v9613: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:48:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:48:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:48:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:48:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:48:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:48:44.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:48:44 smithi153 ceph-mon[70044]: pgmap v9614: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:44.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:48:44 smithi083 ceph-mon[79930]: pgmap v9614: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:44.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:48:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:48:44.718Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:48:44.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:48:44 smithi083 ceph-mon[77242]: pgmap v9614: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:45.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:48:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:48:45.428Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:48:46.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:48:46 smithi153 ceph-mon[70044]: pgmap v9615: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:46.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:48:46 smithi083 ceph-mon[79930]: pgmap v9615: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:46.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:48:46 smithi083 ceph-mon[77242]: pgmap v9615: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:48.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:48:48 smithi083 ceph-mon[79930]: pgmap v9616: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:48.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:48:48 smithi083 ceph-mon[77242]: pgmap v9616: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:49.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:48:48 smithi153 ceph-mon[70044]: pgmap v9616: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:50.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:48:49 smithi083 ceph-mon[79930]: pgmap v9617: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:50.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:48:49 smithi083 ceph-mon[77242]: pgmap v9617: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:50.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:48:49 smithi153 ceph-mon[70044]: pgmap v9617: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:51.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:48:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:48:51] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T10:48:51.715 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:48:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:48:51.345Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:48:52.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:48:52 smithi153 ceph-mon[70044]: pgmap v9618: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:48:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:48:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:48:52.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:48:52 smithi083 ceph-mon[79930]: pgmap v9618: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:52.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:48:52 smithi083 ceph-mon[77242]: pgmap v9618: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:48:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:48:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:48:54.180 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:48:54.475 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:48:54.476 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 7m ago 5h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:48:54.476 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 5m ago 5h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:48:54.477 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 7m ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:48:54.477 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 5m ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:48:54.478 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 5m ago 5h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:48:54.479 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 7m ago 5h 783M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:48:54.480 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 7m ago 5h 1198M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:48:54.481 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 5m ago 5h 516M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:48:54.481 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 7m ago 5h 688M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:48:54.481 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 7m ago 5h 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:48:54.482 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 5m ago 5h 26.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:48:54.482 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 7m ago 5h 844M 2464M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:48:54.482 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 7m ago 5h 834M 2464M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:48:54.483 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 7m ago 5h 651M 2464M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:48:54.483 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 7m ago 5h 814M 2464M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:48:54.484 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 5m ago 5h 718M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:48:54.484 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 5m ago 5h 895M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:48:54.485 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 5m ago 5h 799M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:48:54.485 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 5m ago 5h 945M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:48:54.486 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 5m ago 5h 189M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:48:54.828 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:48:54.829 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:48:54.829 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:48:54.829 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:48:54.829 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:48:54.830 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:48:54.830 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:48:54.830 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:48:54.831 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:48:54.831 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:48:54.831 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:48:54.831 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:48:54.832 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:48:54.832 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:48:54.832 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:48:54.832 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:48:54.833 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:48:54.833 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:48:54.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:48:54 smithi083 ceph-mon[79930]: pgmap v9619: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:54.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:48:54 smithi083 ceph-mon[77242]: pgmap v9619: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:54.965 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:48:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:48:54.718Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:48:55.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:48:54 smithi153 ceph-mon[70044]: pgmap v9619: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:55.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:48:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:48:55.428Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:48:55.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:48:55 smithi083 ceph-mon[79930]: from='client.45321 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:48:55.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:48:55 smithi083 ceph-mon[79930]: from='client.55258 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:48:55.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:48:55 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4044047686' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:48:55.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:48:55 smithi083 ceph-mon[77242]: from='client.45321 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:48:55.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:48:55 smithi083 ceph-mon[77242]: from='client.55258 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:48:55.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:48:55 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4044047686' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:48:56.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:48:55 smithi153 ceph-mon[70044]: from='client.45321 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:48:56.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:48:55 smithi153 ceph-mon[70044]: from='client.55258 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:48:56.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:48:55 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4044047686' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:48:56.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:48:56 smithi083 ceph-mon[79930]: pgmap v9620: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:56.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:48:56 smithi083 ceph-mon[77242]: pgmap v9620: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:57.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:48:56 smithi153 ceph-mon[70044]: pgmap v9620: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:58.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:48:58 smithi083 ceph-mon[79930]: pgmap v9621: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:58.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:48:58 smithi083 ceph-mon[77242]: pgmap v9621: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:48:59.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:48:58 smithi153 ceph-mon[70044]: pgmap v9621: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:00.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:48:59 smithi083 ceph-mon[79930]: pgmap v9622: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:00.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:48:59 smithi083 ceph-mon[77242]: pgmap v9622: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:48:59 smithi153 ceph-mon[70044]: pgmap v9622: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:01.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:49:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:49:01] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T10:49:01.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:49:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:49:01.346Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:49:02.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:49:02 smithi153 ceph-mon[70044]: pgmap v9623: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:49:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:49:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:49:02.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:49:02 smithi083 ceph-mon[79930]: pgmap v9623: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:02.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:49:02 smithi083 ceph-mon[77242]: pgmap v9623: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:49:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:49:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:49:04.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:49:04 smithi083 ceph-mon[79930]: pgmap v9624: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:04.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:49:04 smithi083 ceph-mon[77242]: pgmap v9624: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:04.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:49:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:49:04.720Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:49:05.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:49:04 smithi153 ceph-mon[70044]: pgmap v9624: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:05.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:49:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:49:05.429Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:49:06.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:49:06 smithi083 ceph-mon[79930]: pgmap v9625: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:06.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:49:06 smithi083 ceph-mon[77242]: pgmap v9625: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:07.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:49:06 smithi153 ceph-mon[70044]: pgmap v9625: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:08.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:49:08 smithi083 ceph-mon[79930]: pgmap v9626: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:08.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:49:08 smithi083 ceph-mon[77242]: pgmap v9626: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:09.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:49:08 smithi153 ceph-mon[70044]: pgmap v9626: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:10.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:49:09 smithi083 ceph-mon[77242]: pgmap v9627: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:10.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:49:09 smithi083 ceph-mon[79930]: pgmap v9627: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:10.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:49:09 smithi153 ceph-mon[70044]: pgmap v9627: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:11.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:49:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:49:11] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T10:49:11.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:49:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:49:11.346Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:49:12.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:49:12 smithi153 ceph-mon[70044]: pgmap v9628: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:49:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:49:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:49:12.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:49:12 smithi083 ceph-mon[77242]: pgmap v9628: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:12.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:49:12 smithi083 ceph-mon[79930]: pgmap v9628: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:49:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:49:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:49:14.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:49:14 smithi083 ceph-mon[79930]: pgmap v9629: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:14.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:49:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:49:14.721Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:49:14.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:49:14 smithi083 ceph-mon[77242]: pgmap v9629: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:15.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:49:14 smithi153 ceph-mon[70044]: pgmap v9629: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:15.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:49:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:49:15.429Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:49:16.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:49:16 smithi083 ceph-mon[79930]: pgmap v9630: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:16.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:49:16 smithi083 ceph-mon[77242]: pgmap v9630: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:17.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:49:16 smithi153 ceph-mon[70044]: pgmap v9630: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:18.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:49:18 smithi083 ceph-mon[79930]: pgmap v9631: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:18.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:49:18 smithi083 ceph-mon[77242]: pgmap v9631: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:19.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:49:18 smithi153 ceph-mon[70044]: pgmap v9631: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:20.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:49:19 smithi083 ceph-mon[79930]: pgmap v9632: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:20.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:49:19 smithi083 ceph-mon[77242]: pgmap v9632: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:49:19 smithi153 ceph-mon[70044]: pgmap v9632: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:21.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:49:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:49:21] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T10:49:21.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:49:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:49:21.347Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:49:22.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:49:22 smithi153 ceph-mon[70044]: pgmap v9633: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:22.810 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:49:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:49:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:49:22.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:49:22 smithi083 ceph-mon[79930]: pgmap v9633: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:22.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:49:22 smithi083 ceph-mon[77242]: pgmap v9633: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:49:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:49:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:49:24.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:49:24 smithi083 ceph-mon[79930]: pgmap v9634: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:24.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:49:24 smithi083 ceph-mon[77242]: pgmap v9634: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:24.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:49:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:49:24.721Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:49:25.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:49:24 smithi153 ceph-mon[70044]: pgmap v9634: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:25.152 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:49:25.441 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:49:25.442 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 8m ago 5h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:49:25.442 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 6m ago 5h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:49:25.442 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 8m ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:49:25.443 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 6m ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:49:25.443 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 6m ago 5h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:49:25.443 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 8m ago 5h 783M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:49:25.444 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 8m ago 5h 1198M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:49:25.445 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 6m ago 5h 516M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:49:25.447 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 8m ago 5h 688M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:49:25.447 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 8m ago 5h 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:49:25.448 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 6m ago 5h 26.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:49:25.448 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 8m ago 5h 844M 2464M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:49:25.448 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 8m ago 5h 834M 2464M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:49:25.448 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 8m ago 5h 651M 2464M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:49:25.449 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 8m ago 5h 814M 2464M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:49:25.449 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 6m ago 5h 718M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:49:25.449 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 6m ago 5h 895M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:49:25.449 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 6m ago 5h 799M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:49:25.449 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 6m ago 5h 945M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:49:25.450 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 6m ago 5h 189M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:49:25.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:49:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:49:25.430Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:49:25.802 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:49:25.802 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:49:25.803 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:49:25.803 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:49:25.803 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:49:25.803 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:49:25.804 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:49:25.804 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:49:25.804 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:49:25.804 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:49:25.805 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:49:25.805 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:49:25.805 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:49:25.805 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:49:25.806 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:49:25.806 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:49:25.806 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:49:25.806 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:49:26.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:49:26 smithi083 ceph-mon[79930]: from='client.45339 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:49:26.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:49:26 smithi083 ceph-mon[79930]: from='client.45345 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:49:26.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:49:26 smithi083 ceph-mon[79930]: pgmap v9635: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:26.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:49:26 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2694735921' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:49:26.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:49:26 smithi083 ceph-mon[77242]: from='client.45339 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:49:26.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:49:26 smithi083 ceph-mon[77242]: from='client.45345 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:49:26.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:49:26 smithi083 ceph-mon[77242]: pgmap v9635: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:26.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:49:26 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2694735921' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:49:27.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:49:26 smithi153 ceph-mon[70044]: from='client.45339 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:49:27.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:49:26 smithi153 ceph-mon[70044]: from='client.45345 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:49:27.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:49:26 smithi153 ceph-mon[70044]: pgmap v9635: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:27.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:49:26 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2694735921' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:49:28.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:49:28 smithi083 ceph-mon[79930]: pgmap v9636: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:28.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:49:28 smithi083 ceph-mon[77242]: pgmap v9636: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:29.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:49:28 smithi153 ceph-mon[70044]: pgmap v9636: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:30.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:49:29 smithi083 ceph-mon[79930]: pgmap v9637: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:30.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:49:29 smithi083 ceph-mon[77242]: pgmap v9637: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:30.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:49:29 smithi153 ceph-mon[70044]: pgmap v9637: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:31.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:49:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:49:31] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T10:49:31.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:49:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:49:31.348Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:49:31.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:49:31 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:49:31.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:49:31 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:49:31.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:49:31 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:49:31.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:49:31 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:49:31.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:49:31 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:49:31.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:49:31 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:49:31.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:49:31 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:49:31.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:49:31 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:49:31.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:49:31 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:49:32.568 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:49:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:49:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:49:32.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:49:32 smithi083 ceph-mon[79930]: pgmap v9638: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:32.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:49:32 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:49:32.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:49:32 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:49:32.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:49:32 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:49:32.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:49:32 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:49:32.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:49:32 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:49:32.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:49:32 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:49:32.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:49:32 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:49:32.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:49:32 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:49:32.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:49:32 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:49:32.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:49:32 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:49:32.968 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:49:32 smithi083 ceph-mon[77242]: pgmap v9638: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:32.969 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:49:32 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:49:32.969 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:49:32 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:49:32.969 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:49:32 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:49:32.970 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:49:32 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:49:32.970 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:49:32 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:49:32.971 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:49:32 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:49:32.971 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:49:32 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:49:32.971 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:49:32 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:49:32.972 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:49:32 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:49:32.972 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:49:32 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:49:33.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:49:32 smithi153 ceph-mon[70044]: pgmap v9638: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:33.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:49:32 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:49:33.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:49:32 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:49:33.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:49:32 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:49:33.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:49:32 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:49:33.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:49:32 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:49:33.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:49:32 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:49:33.026 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:49:32 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:49:33.026 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:49:32 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:49:33.026 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:49:32 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:49:33.027 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:49:32 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:49:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:49:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:49:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:49:33.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:49:33 smithi083 ceph-mon[79930]: Adjusting osd_memory_target on smithi083 to 2456M 2022-09-29T10:49:33.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:49:33 smithi083 ceph-mon[77242]: Adjusting osd_memory_target on smithi083 to 2456M 2022-09-29T10:49:34.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:49:33 smithi153 ceph-mon[70044]: Adjusting osd_memory_target on smithi083 to 2456M 2022-09-29T10:49:34.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:49:34 smithi083 ceph-mon[79930]: pgmap v9639: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:34.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:49:34 smithi083 ceph-mon[77242]: pgmap v9639: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:34.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:49:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:49:34.722Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:49:35.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:49:34 smithi153 ceph-mon[70044]: pgmap v9639: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:35.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:49:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:49:35.431Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:49:36.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:49:36 smithi083 ceph-mon[77242]: pgmap v9640: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:36.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:49:36 smithi083 ceph-mon[79930]: pgmap v9640: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:37.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:49:36 smithi153 ceph-mon[70044]: pgmap v9640: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:38.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:49:38 smithi083 ceph-mon[77242]: pgmap v9641: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:38.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:49:38 smithi083 ceph-mon[79930]: pgmap v9641: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:39.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:49:38 smithi153 ceph-mon[70044]: pgmap v9641: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:40.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:49:39 smithi083 ceph-mon[79930]: pgmap v9642: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:40.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:49:39 smithi083 ceph-mon[77242]: pgmap v9642: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:49:39 smithi153 ceph-mon[70044]: pgmap v9642: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:41.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:49:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:49:41.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:49:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:49:41.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:49:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:49:41.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:49:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:49:41.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:49:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:49:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:49:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:49:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:49:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:49:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:49:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:49:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:49:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:49:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:49:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:49:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:49:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:49:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:49:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:49:41.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:49:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:49:41] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T10:49:41.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:49:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:49:41.347Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:49:42.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:49:41 smithi083 ceph-mon[79930]: pgmap v9643: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:42.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:49:41 smithi083 ceph-mon[77242]: pgmap v9643: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:49:41 smithi153 ceph-mon[70044]: pgmap v9643: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:49:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:49:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:49:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:49:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:49:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:49:44.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:49:44 smithi083 ceph-mon[79930]: pgmap v9644: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:44.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:49:44 smithi083 ceph-mon[77242]: pgmap v9644: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:44.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:49:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:49:44.723Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:49:45.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:49:44 smithi153 ceph-mon[70044]: pgmap v9644: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:45.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:49:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:49:45.431Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:49:46.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:49:46 smithi083 ceph-mon[79930]: pgmap v9645: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:46.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:49:46 smithi083 ceph-mon[77242]: pgmap v9645: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:47.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:49:46 smithi153 ceph-mon[70044]: pgmap v9645: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:48.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:49:48 smithi083 ceph-mon[79930]: pgmap v9646: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:48.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:49:48 smithi083 ceph-mon[77242]: pgmap v9646: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:49.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:49:48 smithi153 ceph-mon[70044]: pgmap v9646: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:50.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:49:49 smithi083 ceph-mon[79930]: pgmap v9647: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:50.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:49:49 smithi083 ceph-mon[77242]: pgmap v9647: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:50.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:49:49 smithi153 ceph-mon[70044]: pgmap v9647: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:51.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:49:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:49:51] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T10:49:51.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:49:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:49:51.348Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:49:52.574 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:49:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:49:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:49:52.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:49:52 smithi083 ceph-mon[79930]: pgmap v9648: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:52.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:49:52 smithi083 ceph-mon[77242]: pgmap v9648: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:53.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:49:52 smithi153 ceph-mon[70044]: pgmap v9648: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:49:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:49:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:49:54.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:49:54 smithi083 ceph-mon[79930]: pgmap v9649: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:54.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:49:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:49:54.723Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:49:54.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:49:54 smithi083 ceph-mon[77242]: pgmap v9649: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:55.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:49:54 smithi153 ceph-mon[70044]: pgmap v9649: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:55.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:49:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:49:55.432Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:49:56.128 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:49:56.420 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:49:56.421 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 8m ago 5h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:49:56.421 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 6m ago 5h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:49:56.421 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 8m ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:49:56.421 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 6m ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:49:56.422 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 6m ago 5h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:49:56.422 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 8m ago 5h 783M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:49:56.422 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 8m ago 5h 1198M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:49:56.422 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 6m ago 5h 516M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:49:56.423 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 8m ago 5h 688M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:49:56.423 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 8m ago 5h 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:49:56.423 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 6m ago 5h 26.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:49:56.423 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 8m ago 5h 844M 2456M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:49:56.424 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 8m ago 5h 834M 2456M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:49:56.424 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 8m ago 5h 651M 2456M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:49:56.424 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 8m ago 5h 814M 2456M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:49:56.424 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 6m ago 5h 718M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:49:56.424 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 6m ago 5h 895M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:49:56.425 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 6m ago 5h 799M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:49:56.425 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 6m ago 5h 945M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:49:56.425 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 6m ago 5h 189M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:49:56.777 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:49:56.777 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:49:56.778 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:49:56.778 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:49:56.778 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:49:56.778 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:49:56.779 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:49:56.779 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:49:56.779 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:49:56.779 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:49:56.780 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:49:56.780 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:49:56.780 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:49:56.780 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:49:56.781 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:49:56.781 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:49:56.781 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:49:56.781 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:49:56.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:49:56 smithi083 ceph-mon[77242]: pgmap v9650: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:56.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:49:56 smithi083 ceph-mon[79930]: pgmap v9650: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:57.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:49:56 smithi153 ceph-mon[70044]: pgmap v9650: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:57.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:49:57 smithi083 ceph-mon[79930]: from='client.45357 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:49:57.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:49:57 smithi083 ceph-mon[79930]: from='client.45363 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:49:57.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:49:57 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/93058943' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:49:57.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:49:57 smithi083 ceph-mon[77242]: from='client.45357 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:49:57.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:49:57 smithi083 ceph-mon[77242]: from='client.45363 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:49:57.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:49:57 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/93058943' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:49:58.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:49:57 smithi153 ceph-mon[70044]: from='client.45357 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:49:58.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:49:57 smithi153 ceph-mon[70044]: from='client.45363 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:49:58.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:49:57 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/93058943' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:49:58.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:49:58 smithi083 ceph-mon[79930]: pgmap v9651: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:58.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:49:58 smithi083 ceph-mon[77242]: pgmap v9651: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:49:59.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:49:58 smithi153 ceph-mon[70044]: pgmap v9651: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:00.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:49:59 smithi083 ceph-mon[79930]: pgmap v9652: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:00.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:49:59 smithi083 ceph-mon[77242]: pgmap v9652: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:00.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:50:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[77219]: 2022-09-29T10:49:59.999+0000 7f160f665700 -1 log_channel(cluster) log [ERR] : overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T10:50:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:49:59 smithi153 ceph-mon[70044]: pgmap v9652: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:01.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:50:00 smithi083 ceph-mon[79930]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T10:50:01.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:50:00 smithi083 ceph-mon[77242]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T10:50:01.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:50:00 smithi153 ceph-mon[70044]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T10:50:01.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:50:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:50:01] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T10:50:01.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:50:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:50:01.348Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:50:02.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:50:01 smithi083 ceph-mon[79930]: pgmap v9653: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:02.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:50:01 smithi083 ceph-mon[77242]: pgmap v9653: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:02.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:50:01 smithi153 ceph-mon[70044]: pgmap v9653: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:50:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:50:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:50:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:50:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:50:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:50:04.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:50:04 smithi083 ceph-mon[79930]: pgmap v9654: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:04.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:50:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:50:04.724Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:50:04.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:50:04 smithi083 ceph-mon[77242]: pgmap v9654: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:05.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:50:04 smithi153 ceph-mon[70044]: pgmap v9654: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:05.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:50:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:50:05.434Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:50:06.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:50:06 smithi083 ceph-mon[79930]: pgmap v9655: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:06.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:50:06 smithi083 ceph-mon[77242]: pgmap v9655: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:07.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:50:06 smithi153 ceph-mon[70044]: pgmap v9655: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:08.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:50:08 smithi083 ceph-mon[79930]: pgmap v9656: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:08.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:50:08 smithi083 ceph-mon[77242]: pgmap v9656: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:09.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:50:08 smithi153 ceph-mon[70044]: pgmap v9656: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:10.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:50:09 smithi083 ceph-mon[77242]: pgmap v9657: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:10.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:50:09 smithi083 ceph-mon[79930]: pgmap v9657: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:10.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:50:09 smithi153 ceph-mon[70044]: pgmap v9657: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:11.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:50:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:50:11] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T10:50:11.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:50:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:50:11.349Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:50:12.580 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:50:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:50:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:50:12.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:50:12 smithi083 ceph-mon[79930]: pgmap v9658: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:12.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:50:12 smithi083 ceph-mon[77242]: pgmap v9658: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:13.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:50:12 smithi153 ceph-mon[70044]: pgmap v9658: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:50:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:50:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:50:14.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:50:14 smithi083 ceph-mon[79930]: pgmap v9659: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:14.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:50:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:50:14.725Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:50:14.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:50:14 smithi083 ceph-mon[77242]: pgmap v9659: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:15.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:50:14 smithi153 ceph-mon[70044]: pgmap v9659: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:15.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:50:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:50:15.434Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:50:16.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:50:16 smithi083 ceph-mon[79930]: pgmap v9660: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:16.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:50:16 smithi083 ceph-mon[77242]: pgmap v9660: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:17.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:50:16 smithi153 ceph-mon[70044]: pgmap v9660: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:18.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:50:18 smithi083 ceph-mon[77242]: pgmap v9661: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:18.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:50:18 smithi083 ceph-mon[79930]: pgmap v9661: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:19.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:50:18 smithi153 ceph-mon[70044]: pgmap v9661: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:20.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:50:19 smithi083 ceph-mon[79930]: pgmap v9662: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:20.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:50:19 smithi083 ceph-mon[77242]: pgmap v9662: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:50:19 smithi153 ceph-mon[70044]: pgmap v9662: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:21.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:50:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:50:21] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T10:50:21.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:50:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:50:21.350Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:50:22.583 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:50:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:50:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:50:22.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:50:22 smithi083 ceph-mon[79930]: pgmap v9663: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:22.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:50:22 smithi083 ceph-mon[77242]: pgmap v9663: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:23.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:50:22 smithi153 ceph-mon[70044]: pgmap v9663: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:50:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:50:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:50:24.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:50:24 smithi083 ceph-mon[79930]: pgmap v9664: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:24.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:50:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:50:24.726Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:50:24.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:50:24 smithi083 ceph-mon[77242]: pgmap v9664: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:25.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:50:24 smithi153 ceph-mon[70044]: pgmap v9664: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:25.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:50:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:50:25.434Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:50:26.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:50:26 smithi083 ceph-mon[79930]: pgmap v9665: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:26.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:50:26 smithi083 ceph-mon[77242]: pgmap v9665: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:27.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:50:26 smithi153 ceph-mon[70044]: pgmap v9665: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:27.107 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:50:27.394 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:50:27.394 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 9m ago 5h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:50:27.394 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 7m ago 5h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:50:27.395 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 9m ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:50:27.395 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 7m ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:50:27.395 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 7m ago 5h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:50:27.395 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 9m ago 5h 783M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:50:27.396 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 9m ago 5h 1198M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:50:27.396 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 7m ago 5h 516M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:50:27.396 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 9m ago 5h 688M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:50:27.396 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 9m ago 5h 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:50:27.397 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 7m ago 5h 26.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:50:27.397 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 9m ago 5h 844M 2456M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:50:27.397 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 9m ago 5h 834M 2456M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:50:27.397 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 9m ago 5h 651M 2456M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:50:27.397 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 9m ago 5h 814M 2456M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:50:27.398 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 7m ago 5h 718M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:50:27.398 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 7m ago 5h 895M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:50:27.398 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 7m ago 5h 799M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:50:27.398 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 7m ago 5h 945M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:50:27.399 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 7m ago 5h 189M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:50:27.750 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:50:27.751 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:50:27.751 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:50:27.751 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:50:27.751 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:50:27.752 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:50:27.752 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:50:27.752 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:50:27.752 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:50:27.753 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:50:27.753 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:50:27.753 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:50:27.753 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:50:27.754 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:50:27.754 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:50:27.754 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:50:27.754 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:50:27.755 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:50:28.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:50:28 smithi083 ceph-mon[79930]: from='client.45375 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:50:28.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:50:28 smithi083 ceph-mon[79930]: from='client.55312 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:50:28.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:50:28 smithi083 ceph-mon[79930]: pgmap v9666: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:28.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:50:28 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2551492546' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:50:28.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:50:28 smithi083 ceph-mon[77242]: from='client.45375 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:50:28.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:50:28 smithi083 ceph-mon[77242]: from='client.55312 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:50:28.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:50:28 smithi083 ceph-mon[77242]: pgmap v9666: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:28.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:50:28 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2551492546' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:50:29.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:50:28 smithi153 ceph-mon[70044]: from='client.45375 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:50:29.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:50:28 smithi153 ceph-mon[70044]: from='client.55312 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:50:29.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:50:28 smithi153 ceph-mon[70044]: pgmap v9666: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:29.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:50:28 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2551492546' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:50:30.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:50:29 smithi083 ceph-mon[79930]: pgmap v9667: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:30.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:50:29 smithi083 ceph-mon[77242]: pgmap v9667: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:30.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:50:29 smithi153 ceph-mon[70044]: pgmap v9667: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:31.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:50:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:50:31] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T10:50:31.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:50:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:50:31.351Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:50:32.586 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:50:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:50:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:50:32.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:50:32 smithi083 ceph-mon[79930]: pgmap v9668: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:32.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:50:32 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:50:32.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:50:32 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:50:32.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:50:32 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:50:32.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:50:32 smithi083 ceph-mon[77242]: pgmap v9668: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:32.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:50:32 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:50:32.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:50:32 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:50:32.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:50:32 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:50:33.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:50:32 smithi153 ceph-mon[70044]: pgmap v9668: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:33.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:50:32 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:50:33.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:50:32 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:50:33.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:50:32 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:50:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:50:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:50:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:50:34.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:50:33 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:50:34.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:50:33 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:50:34.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:50:33 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:50:35.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:50:34 smithi153 ceph-mon[70044]: pgmap v9669: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:35.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:50:34 smithi083 ceph-mon[79930]: pgmap v9669: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:35.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:50:34 smithi083 ceph-mon[77242]: pgmap v9669: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:35.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:50:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:50:34.726Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:50:35.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:50:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:50:35.435Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:50:37.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:50:36 smithi153 ceph-mon[70044]: pgmap v9670: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:37.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:50:36 smithi083 ceph-mon[79930]: pgmap v9670: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:37.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:50:36 smithi083 ceph-mon[77242]: pgmap v9670: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:39.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:50:38 smithi153 ceph-mon[70044]: pgmap v9671: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:39.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:50:38 smithi083 ceph-mon[79930]: pgmap v9671: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:39.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:50:38 smithi083 ceph-mon[77242]: pgmap v9671: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:40.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:50:39 smithi083 ceph-mon[79930]: pgmap v9672: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:40.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:50:39 smithi083 ceph-mon[77242]: pgmap v9672: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:50:39 smithi153 ceph-mon[70044]: pgmap v9672: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:41.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:50:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:50:41.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:50:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:50:41.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:50:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:50:41.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:50:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:50:41.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:50:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:50:41.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:50:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:50:41.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:50:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:50:41.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:50:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:50:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:50:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:50:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:50:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:50:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:50:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:50:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:50:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:50:41.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:50:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:50:41] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T10:50:41.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:50:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:50:41.352Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:50:42.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:50:41 smithi083 ceph-mon[77242]: pgmap v9673: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:42.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:50:41 smithi083 ceph-mon[79930]: pgmap v9673: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:50:41 smithi153 ceph-mon[70044]: pgmap v9673: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:50:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:50:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:50:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:50:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:50:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:50:44.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:50:44 smithi083 ceph-mon[79930]: pgmap v9674: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:44.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:50:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:50:44.727Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:50:44.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:50:44 smithi083 ceph-mon[77242]: pgmap v9674: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:45.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:50:44 smithi153 ceph-mon[70044]: pgmap v9674: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:45.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:50:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:50:45.436Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:50:46.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:50:46 smithi083 ceph-mon[79930]: pgmap v9675: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:46.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:50:46 smithi083 ceph-mon[77242]: pgmap v9675: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:47.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:50:46 smithi153 ceph-mon[70044]: pgmap v9675: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:48.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:50:48 smithi083 ceph-mon[79930]: pgmap v9676: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:48.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:50:48 smithi083 ceph-mon[77242]: pgmap v9676: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:49.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:50:48 smithi153 ceph-mon[70044]: pgmap v9676: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:50.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:50:49 smithi083 ceph-mon[79930]: pgmap v9677: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:50.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:50:49 smithi083 ceph-mon[77242]: pgmap v9677: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:50.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:50:49 smithi153 ceph-mon[70044]: pgmap v9677: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:51.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:50:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:50:51] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T10:50:51.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:50:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:50:51.352Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:50:52.592 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:50:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:50:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:50:52.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:50:52 smithi083 ceph-mon[79930]: pgmap v9678: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:52.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:50:52 smithi083 ceph-mon[77242]: pgmap v9678: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:53.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:50:52 smithi153 ceph-mon[70044]: pgmap v9678: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:50:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:50:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:50:54.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:50:54 smithi083 ceph-mon[79930]: pgmap v9679: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:54.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:50:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:50:54.727Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:50:54.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:50:54 smithi083 ceph-mon[77242]: pgmap v9679: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:55.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:50:54 smithi153 ceph-mon[70044]: pgmap v9679: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:55.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:50:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:50:55.436Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:50:56.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:50:56 smithi083 ceph-mon[79930]: pgmap v9680: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:56.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:50:56 smithi083 ceph-mon[77242]: pgmap v9680: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:57.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:50:56 smithi153 ceph-mon[70044]: pgmap v9680: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:58.076 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:50:58.364 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:50:58.364 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 9m ago 5h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:50:58.364 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 7m ago 5h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:50:58.365 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 9m ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:50:58.365 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 7m ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:50:58.365 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 7m ago 5h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:50:58.365 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 9m ago 5h 783M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:50:58.366 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 9m ago 5h 1198M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:50:58.366 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 7m ago 5h 516M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:50:58.366 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 9m ago 5h 688M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:50:58.366 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 9m ago 5h 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:50:58.367 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 7m ago 5h 26.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:50:58.367 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 9m ago 5h 844M 2456M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:50:58.367 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 9m ago 5h 834M 2456M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:50:58.367 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 9m ago 5h 651M 2456M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:50:58.368 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 9m ago 5h 814M 2456M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:50:58.368 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 7m ago 5h 718M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:50:58.368 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 7m ago 5h 895M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:50:58.368 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 7m ago 5h 799M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:50:58.368 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 7m ago 5h 945M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:50:58.369 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 7m ago 5h 189M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:50:58.719 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:50:58.719 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:50:58.720 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:50:58.720 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:50:58.720 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:50:58.720 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:50:58.720 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:50:58.721 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:50:58.721 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:50:58.721 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:50:58.721 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:50:58.722 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:50:58.722 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:50:58.722 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:50:58.722 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:50:58.723 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:50:58.723 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:50:58.723 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:50:58.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:50:58 smithi083 ceph-mon[79930]: pgmap v9681: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:58.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:50:58 smithi083 ceph-mon[77242]: pgmap v9681: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:59.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:50:58 smithi153 ceph-mon[70044]: pgmap v9681: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:50:59.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:50:59 smithi083 ceph-mon[79930]: from='client.55324 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:50:59.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:50:59 smithi083 ceph-mon[79930]: from='client.55330 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:50:59.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:50:59 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2221243331' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:50:59.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:50:59 smithi083 ceph-mon[77242]: from='client.55324 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:50:59.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:50:59 smithi083 ceph-mon[77242]: from='client.55330 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:50:59.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:50:59 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2221243331' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:51:00.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:50:59 smithi153 ceph-mon[70044]: from='client.55324 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:51:00.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:50:59 smithi153 ceph-mon[70044]: from='client.55330 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:51:00.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:50:59 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2221243331' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:51:00.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:51:00 smithi083 ceph-mon[79930]: pgmap v9682: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:51:00.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:51:00 smithi083 ceph-mon[77242]: pgmap v9682: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:51:01.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:51:00 smithi153 ceph-mon[70044]: pgmap v9682: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:51:01.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:51:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:51:01] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T10:51:01.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:51:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:51:01.353Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:51:02.606 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:51:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:51:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:51:02.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:51:02 smithi083 ceph-mon[79930]: pgmap v9683: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:51:02.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:51:02 smithi083 ceph-mon[77242]: pgmap v9683: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:51:03.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:51:02 smithi153 ceph-mon[70044]: pgmap v9683: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:51:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:51:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:51:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:51:04.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:51:04 smithi083 ceph-mon[79930]: pgmap v9684: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:51:04.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:51:04 smithi083 ceph-mon[77242]: pgmap v9684: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:51:04.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:51:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:51:04.728Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:51:05.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:51:04 smithi153 ceph-mon[70044]: pgmap v9684: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:51:05.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:51:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:51:05.437Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:51:06.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:51:06 smithi083 ceph-mon[77242]: pgmap v9685: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:51:06.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:51:06 smithi083 ceph-mon[79930]: pgmap v9685: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:51:07.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:51:06 smithi153 ceph-mon[70044]: pgmap v9685: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:51:08.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:51:08 smithi083 ceph-mon[77242]: pgmap v9686: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:51:08.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:51:08 smithi083 ceph-mon[79930]: pgmap v9686: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:51:09.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:51:08 smithi153 ceph-mon[70044]: pgmap v9686: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:51:10.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:51:09 smithi083 ceph-mon[79930]: pgmap v9687: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:51:10.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:51:09 smithi083 ceph-mon[77242]: pgmap v9687: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:51:10.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:51:09 smithi153 ceph-mon[70044]: pgmap v9687: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:51:11.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:51:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:51:11] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T10:51:11.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:51:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:51:11.354Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:51:12.598 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:51:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:51:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:51:12.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:51:12 smithi083 ceph-mon[79930]: pgmap v9688: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:51:12.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:51:12 smithi083 ceph-mon[77242]: pgmap v9688: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:51:13.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:51:12 smithi153 ceph-mon[70044]: pgmap v9688: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:51:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:51:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:51:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:51:14.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:51:14 smithi083 ceph-mon[79930]: pgmap v9689: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:51:14.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:51:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:51:14.728Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:51:14.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:51:14 smithi083 ceph-mon[77242]: pgmap v9689: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:51:15.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:51:14 smithi153 ceph-mon[70044]: pgmap v9689: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:51:15.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:51:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:51:15.437Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:51:16.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:51:16 smithi083 ceph-mon[79930]: pgmap v9690: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:51:16.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:51:16 smithi083 ceph-mon[77242]: pgmap v9690: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:51:17.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:51:16 smithi153 ceph-mon[70044]: pgmap v9690: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:51:18.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:51:18 smithi083 ceph-mon[79930]: pgmap v9691: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:51:18.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:51:18 smithi083 ceph-mon[77242]: pgmap v9691: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:51:19.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:51:18 smithi153 ceph-mon[70044]: pgmap v9691: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:51:20.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:51:19 smithi083 ceph-mon[79930]: pgmap v9692: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:51:20.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:51:19 smithi083 ceph-mon[77242]: pgmap v9692: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:51:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:51:19 smithi153 ceph-mon[70044]: pgmap v9692: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:51:21.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:51:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:51:21] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T10:51:21.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:51:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:51:21.354Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:51:22.600 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:51:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:51:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:51:22.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:51:22 smithi083 ceph-mon[77242]: pgmap v9693: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:51:22.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:51:22 smithi083 ceph-mon[79930]: pgmap v9693: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:51:23.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:51:22 smithi153 ceph-mon[70044]: pgmap v9693: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:51:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:51:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:51:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:51:24.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:51:24 smithi083 ceph-mon[77242]: pgmap v9694: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:51:24.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:51:24 smithi083 ceph-mon[79930]: pgmap v9694: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:51:24.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:51:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:51:24.729Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:51:25.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:51:24 smithi153 ceph-mon[70044]: pgmap v9694: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:51:25.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:51:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:51:25.438Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:51:26.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:51:26 smithi083 ceph-mon[79930]: pgmap v9695: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:51:26.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:51:26 smithi083 ceph-mon[77242]: pgmap v9695: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:51:27.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:51:26 smithi153 ceph-mon[70044]: pgmap v9695: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:51:28.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:51:28 smithi083 ceph-mon[77242]: pgmap v9696: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:51:28.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:51:28 smithi083 ceph-mon[79930]: pgmap v9696: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:51:29.047 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:51:29.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:51:28 smithi153 ceph-mon[70044]: pgmap v9696: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:51:29.336 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:51:29.336 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 10m ago 5h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:51:29.336 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 8m ago 5h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:51:29.337 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 10m ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:51:29.337 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 8m ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:51:29.337 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 8m ago 5h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:51:29.337 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 10m ago 5h 783M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:51:29.337 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 10m ago 5h 1198M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:51:29.338 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 8m ago 5h 516M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:51:29.338 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 10m ago 5h 688M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:51:29.338 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 10m ago 5h 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:51:29.338 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 8m ago 5h 26.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:51:29.339 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 10m ago 5h 844M 2456M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:51:29.339 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 10m ago 5h 834M 2456M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:51:29.339 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 10m ago 5h 651M 2456M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:51:29.339 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 10m ago 5h 814M 2456M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:51:29.340 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 8m ago 5h 718M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:51:29.340 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 8m ago 5h 895M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:51:29.340 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 8m ago 5h 799M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:51:29.340 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 8m ago 5h 945M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:51:29.341 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 8m ago 5h 189M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:51:29.694 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:51:29.694 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:51:29.694 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:51:29.695 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:51:29.695 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:51:29.695 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:51:29.695 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:51:29.696 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:51:29.696 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:51:29.696 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:51:29.696 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:51:29.696 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:51:29.697 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:51:29.697 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:51:29.697 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:51:29.697 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:51:29.698 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:51:29.698 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:51:30.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:51:29 smithi083 ceph-mon[79930]: from='client.45411 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:51:30.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:51:29 smithi083 ceph-mon[79930]: from='client.55348 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:51:30.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:51:29 smithi083 ceph-mon[79930]: pgmap v9697: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:51:30.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:51:29 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3476399208' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:51:30.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:51:29 smithi083 ceph-mon[77242]: from='client.45411 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:51:30.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:51:29 smithi083 ceph-mon[77242]: from='client.55348 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:51:30.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:51:29 smithi083 ceph-mon[77242]: pgmap v9697: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:51:30.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:51:29 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3476399208' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:51:30.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:51:29 smithi153 ceph-mon[70044]: from='client.45411 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:51:30.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:51:29 smithi153 ceph-mon[70044]: from='client.55348 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:51:30.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:51:29 smithi153 ceph-mon[70044]: pgmap v9697: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:51:30.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:51:29 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3476399208' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:51:31.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:51:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:51:31] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T10:51:31.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:51:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:51:31.355Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:51:32.604 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:51:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:51:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:51:32.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:51:32 smithi083 ceph-mon[79930]: pgmap v9698: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:51:32.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:51:32 smithi083 ceph-mon[77242]: pgmap v9698: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:51:33.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:51:32 smithi153 ceph-mon[70044]: pgmap v9698: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:51:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:51:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:51:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:51:33.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:51:33 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:51:33.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:51:33 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:51:33.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:51:33 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:51:33.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:51:33 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:51:33.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:51:33 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:51:33.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:51:33 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:51:33.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:51:33 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:51:33.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:51:33 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:51:33.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:51:33 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:51:33.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:51:33 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:51:33.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:51:33 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:51:33.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:51:33 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:51:33.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:51:33 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:51:33.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:51:33 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:51:34.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:51:33 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:51:34.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:51:33 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:51:34.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:51:33 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:51:34.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:51:33 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:51:34.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:51:33 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:51:34.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:51:33 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:51:34.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:51:33 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:51:34.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:51:34 smithi083 ceph-mon[79930]: pgmap v9699: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:51:34.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:51:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:51:34.729Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:51:34.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:51:34 smithi083 ceph-mon[77242]: pgmap v9699: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:51:35.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:51:34 smithi153 ceph-mon[70044]: pgmap v9699: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:51:35.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:51:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:51:35.439Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:51:36.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:51:36 smithi083 ceph-mon[77242]: pgmap v9700: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:51:36.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:51:36 smithi083 ceph-mon[79930]: pgmap v9700: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:51:37.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:51:36 smithi153 ceph-mon[70044]: pgmap v9700: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:51:38.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:51:38 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:51:38.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:51:38 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:51:38.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:51:38 smithi083 ceph-mon[79930]: pgmap v9701: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:51:38.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:51:38 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:51:38.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:51:38 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:51:38.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:51:38 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:51:38.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:51:38 smithi083 ceph-mon[77242]: pgmap v9701: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:51:38.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:51:38 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:51:38.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:51:38 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:51:38.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:51:38 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:51:38.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:51:38 smithi153 ceph-mon[70044]: pgmap v9701: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:51:38.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:51:38 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:51:40.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:51:39 smithi083 ceph-mon[79930]: pgmap v9702: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:51:40.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:51:39 smithi083 ceph-mon[77242]: pgmap v9702: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:51:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:51:39 smithi153 ceph-mon[70044]: pgmap v9702: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:51:41.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:51:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:51:41.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:51:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:51:41.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:51:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:51:41.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:51:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:51:41.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:51:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:51:41.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:51:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:51:41.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:51:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:51:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:51:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:51:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:51:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:51:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:51:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:51:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:51:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:51:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:51:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:51:41.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:51:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:51:41] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T10:51:41.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:51:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:51:41.356Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:51:42.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:51:41 smithi083 ceph-mon[79930]: pgmap v9703: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:51:42.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:51:41 smithi083 ceph-mon[77242]: pgmap v9703: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:51:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:51:41 smithi153 ceph-mon[70044]: pgmap v9703: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:51:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:51:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:51:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:51:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:51:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:51:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:51:44.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:51:44 smithi083 ceph-mon[79930]: pgmap v9704: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:51:44.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:51:44 smithi083 ceph-mon[77242]: pgmap v9704: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:51:44.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:51:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:51:44.731Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:51:45.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:51:44 smithi153 ceph-mon[70044]: pgmap v9704: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:51:45.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:51:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:51:45.440Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:51:46.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:51:46 smithi083 ceph-mon[79930]: pgmap v9705: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:51:46.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:51:46 smithi083 ceph-mon[77242]: pgmap v9705: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:51:47.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:51:46 smithi153 ceph-mon[70044]: pgmap v9705: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:51:48.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:51:48 smithi083 ceph-mon[77242]: pgmap v9706: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:51:48.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:51:48 smithi083 ceph-mon[79930]: pgmap v9706: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:51:49.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:51:48 smithi153 ceph-mon[70044]: pgmap v9706: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:51:50.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:51:49 smithi083 ceph-mon[79930]: pgmap v9707: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:51:50.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:51:49 smithi083 ceph-mon[77242]: pgmap v9707: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:51:50.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:51:49 smithi153 ceph-mon[70044]: pgmap v9707: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:51:51.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:51:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:51:51] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T10:51:51.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:51:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:51:51.357Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:51:52.610 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:51:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:51:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:51:52.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:51:52 smithi083 ceph-mon[77242]: pgmap v9708: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:51:52.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:51:52 smithi083 ceph-mon[79930]: pgmap v9708: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:51:53.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:51:52 smithi153 ceph-mon[70044]: pgmap v9708: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:51:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:51:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:51:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:51:54.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:51:54 smithi083 ceph-mon[79930]: pgmap v9709: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:51:54.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:51:54 smithi083 ceph-mon[77242]: pgmap v9709: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:51:54.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:51:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:51:54.731Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:51:55.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:51:54 smithi153 ceph-mon[70044]: pgmap v9709: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:51:55.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:51:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:51:55.441Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:51:56.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:51:56 smithi083 ceph-mon[79930]: pgmap v9710: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:51:56.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:51:56 smithi083 ceph-mon[77242]: pgmap v9710: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:51:57.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:51:56 smithi153 ceph-mon[70044]: pgmap v9710: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:51:58.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:51:58 smithi083 ceph-mon[79930]: pgmap v9711: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:51:58.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:51:58 smithi083 ceph-mon[77242]: pgmap v9711: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:51:59.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:51:58 smithi153 ceph-mon[70044]: pgmap v9711: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:00.027 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:52:00.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:51:59 smithi083 ceph-mon[77242]: pgmap v9712: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:00.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:51:59 smithi083 ceph-mon[79930]: pgmap v9712: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:51:59 smithi153 ceph-mon[70044]: pgmap v9712: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:00.318 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:52:00.318 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 22s ago 5h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:52:00.318 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 8m ago 5h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:52:00.319 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 22s ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:52:00.319 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 8m ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:52:00.319 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 8m ago 5h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:52:00.319 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 22s ago 5h 791M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:52:00.320 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 22s ago 5h 1235M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:52:00.320 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 8m ago 5h 516M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:52:00.320 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 22s ago 5h 709M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:52:00.320 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 22s ago 5h 26.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:52:00.321 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 8m ago 5h 26.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:52:00.321 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 22s ago 5h 870M 2456M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:52:00.321 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 22s ago 5h 860M 2456M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:52:00.321 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 22s ago 5h 671M 2456M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:52:00.321 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 22s ago 5h 839M 2456M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:52:00.322 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 8m ago 5h 718M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:52:00.322 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 8m ago 5h 895M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:52:00.322 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 8m ago 5h 799M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:52:00.322 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 8m ago 5h 945M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:52:00.323 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 8m ago 5h 189M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:52:00.674 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:52:00.674 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:52:00.674 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:52:00.674 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:52:00.675 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:52:00.675 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:52:00.675 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:52:00.675 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:52:00.676 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:52:00.676 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:52:00.676 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:52:00.676 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:52:00.676 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:52:00.677 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:52:00.677 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:52:00.677 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:52:00.677 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:52:00.678 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:52:01.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:52:00 smithi083 ceph-mon[79930]: from='client.55360 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:52:01.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:52:00 smithi083 ceph-mon[79930]: from='client.45435 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:52:01.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:52:00 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3403864568' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:52:01.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:52:00 smithi083 ceph-mon[77242]: from='client.55360 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:52:01.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:52:00 smithi083 ceph-mon[77242]: from='client.45435 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:52:01.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:52:00 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3403864568' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:52:01.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:52:00 smithi153 ceph-mon[70044]: from='client.55360 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:52:01.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:52:00 smithi153 ceph-mon[70044]: from='client.45435 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:52:01.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:52:00 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3403864568' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:52:01.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:52:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:52:01.358Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:52:01.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:52:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:52:01] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T10:52:02.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:52:01 smithi083 ceph-mon[79930]: pgmap v9713: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:02.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:52:01 smithi083 ceph-mon[77242]: pgmap v9713: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:02.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:52:01 smithi153 ceph-mon[70044]: pgmap v9713: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:52:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:52:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:52:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:52:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:52:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:52:04.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:52:04 smithi083 ceph-mon[79930]: pgmap v9714: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:04.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:52:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:52:04.731Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:52:04.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:52:04 smithi083 ceph-mon[77242]: pgmap v9714: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:05.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:52:04 smithi153 ceph-mon[70044]: pgmap v9714: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:05.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:52:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:52:05.441Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:52:06.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:52:06 smithi083 ceph-mon[79930]: pgmap v9715: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:06.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:52:06 smithi083 ceph-mon[77242]: pgmap v9715: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:07.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:52:06 smithi153 ceph-mon[70044]: pgmap v9715: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:08.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:52:08 smithi083 ceph-mon[79930]: pgmap v9716: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:08.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:52:08 smithi083 ceph-mon[77242]: pgmap v9716: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:09.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:52:08 smithi153 ceph-mon[70044]: pgmap v9716: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:10.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:52:09 smithi083 ceph-mon[79930]: pgmap v9717: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:10.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:52:09 smithi083 ceph-mon[77242]: pgmap v9717: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:10.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:52:09 smithi153 ceph-mon[70044]: pgmap v9717: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:11.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:52:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:52:11.358Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:52:11.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:52:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:52:11] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T10:52:12.616 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:52:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:52:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:52:12.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:52:12 smithi083 ceph-mon[77242]: pgmap v9718: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:12.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:52:12 smithi083 ceph-mon[79930]: pgmap v9718: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:13.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:52:12 smithi153 ceph-mon[70044]: pgmap v9718: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:52:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:52:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:52:14.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:52:14 smithi083 ceph-mon[77242]: pgmap v9719: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:14.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:52:14 smithi083 ceph-mon[79930]: pgmap v9719: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:14.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:52:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:52:14.732Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:52:15.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:52:14 smithi153 ceph-mon[70044]: pgmap v9719: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:15.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:52:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:52:15.441Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:52:16.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:52:16 smithi083 ceph-mon[79930]: pgmap v9720: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:16.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:52:16 smithi083 ceph-mon[77242]: pgmap v9720: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:17.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:52:16 smithi153 ceph-mon[70044]: pgmap v9720: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:18.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:52:18 smithi083 ceph-mon[79930]: pgmap v9721: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:18.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:52:18 smithi083 ceph-mon[77242]: pgmap v9721: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:19.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:52:18 smithi153 ceph-mon[70044]: pgmap v9721: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:20.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:52:19 smithi083 ceph-mon[79930]: pgmap v9722: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:20.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:52:19 smithi083 ceph-mon[77242]: pgmap v9722: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:52:19 smithi153 ceph-mon[70044]: pgmap v9722: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:21.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:52:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:52:21] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T10:52:21.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:52:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:52:21.359Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:52:22.619 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:52:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:52:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:52:22.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:52:22 smithi083 ceph-mon[79930]: pgmap v9723: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:22.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:52:22 smithi083 ceph-mon[77242]: pgmap v9723: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:23.022 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:52:22 smithi153 ceph-mon[70044]: pgmap v9723: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:52:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:52:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:52:24.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:52:24 smithi083 ceph-mon[79930]: pgmap v9724: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:24.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:52:24 smithi083 ceph-mon[77242]: pgmap v9724: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:24.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:52:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:52:24.733Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:52:25.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:52:24 smithi153 ceph-mon[70044]: pgmap v9724: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:25.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:52:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:52:25.442Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:52:26.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:52:26 smithi083 ceph-mon[79930]: pgmap v9725: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:26.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:52:26 smithi083 ceph-mon[77242]: pgmap v9725: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:27.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:52:26 smithi153 ceph-mon[70044]: pgmap v9725: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:28.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:52:28 smithi083 ceph-mon[79930]: pgmap v9726: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:28.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:52:28 smithi083 ceph-mon[77242]: pgmap v9726: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:29.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:52:28 smithi153 ceph-mon[70044]: pgmap v9726: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:30.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:52:29 smithi083 ceph-mon[79930]: pgmap v9727: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:30.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:52:29 smithi083 ceph-mon[77242]: pgmap v9727: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:30.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:52:29 smithi153 ceph-mon[70044]: pgmap v9727: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:30.999 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:52:31.288 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:52:31.288 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 53s ago 5h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:52:31.288 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 9m ago 5h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:52:31.288 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 53s ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:52:31.289 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 9m ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:52:31.289 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 9m ago 5h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:52:31.289 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 53s ago 5h 791M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:52:31.289 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 53s ago 5h 1235M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:52:31.289 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 9m ago 5h 516M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:52:31.290 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 53s ago 5h 709M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:52:31.290 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 53s ago 5h 26.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:52:31.290 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 9m ago 5h 26.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:52:31.290 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 53s ago 5h 870M 2456M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:52:31.291 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 53s ago 5h 860M 2456M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:52:31.291 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 53s ago 5h 671M 2456M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:52:31.291 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 53s ago 5h 839M 2456M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:52:31.291 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 9m ago 5h 718M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:52:31.292 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 9m ago 5h 895M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:52:31.292 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 9m ago 5h 799M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:52:31.292 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 9m ago 5h 945M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:52:31.292 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 9m ago 5h 189M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:52:31.648 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:52:31.648 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:52:31.648 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:52:31.649 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:52:31.649 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:52:31.649 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:52:31.649 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:52:31.650 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:52:31.650 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:52:31.650 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:52:31.650 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:52:31.650 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:52:31.651 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:52:31.651 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:52:31.651 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:52:31.651 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:52:31.652 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:52:31.652 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:52:31.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:52:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:52:31] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T10:52:31.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:52:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:52:31.360Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:52:32.621 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:52:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:52:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:52:32.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:52:32 smithi083 ceph-mon[79930]: from='client.55378 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:52:32.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:52:32 smithi083 ceph-mon[79930]: from='client.55384 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:52:32.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:52:32 smithi083 ceph-mon[79930]: pgmap v9728: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:32.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:52:32 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3333614295' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:52:32.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:52:32 smithi083 ceph-mon[77242]: from='client.55378 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:52:32.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:52:32 smithi083 ceph-mon[77242]: from='client.55384 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:52:32.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:52:32 smithi083 ceph-mon[77242]: pgmap v9728: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:32.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:52:32 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3333614295' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:52:33.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:52:32 smithi153 ceph-mon[70044]: from='client.55378 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:52:33.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:52:32 smithi153 ceph-mon[70044]: from='client.55384 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:52:33.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:52:32 smithi153 ceph-mon[70044]: pgmap v9728: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:33.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:52:32 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3333614295' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:52:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:52:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:52:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:52:34.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:52:34 smithi083 ceph-mon[77242]: pgmap v9729: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:34.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:52:34 smithi083 ceph-mon[79930]: pgmap v9729: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:34.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:52:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:52:34.733Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:52:35.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:52:34 smithi153 ceph-mon[70044]: pgmap v9729: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:35.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:52:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:52:35.443Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:52:36.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:52:36 smithi083 ceph-mon[79930]: pgmap v9730: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:36.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:52:36 smithi083 ceph-mon[77242]: pgmap v9730: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:37.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:52:36 smithi153 ceph-mon[70044]: pgmap v9730: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:38.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:52:38 smithi083 ceph-mon[79930]: pgmap v9731: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:38.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:52:38 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:52:38.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:52:38 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:52:38.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:52:38 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:52:38.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:52:38 smithi083 ceph-mon[77242]: pgmap v9731: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:38.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:52:38 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:52:38.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:52:38 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:52:38.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:52:38 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:52:39.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:52:38 smithi153 ceph-mon[70044]: pgmap v9731: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:39.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:52:38 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:52:39.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:52:38 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:52:39.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:52:38 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:52:39.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:52:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:52:39.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:52:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:52:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:52:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:52:40.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:52:40 smithi083 ceph-mon[79930]: pgmap v9732: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:52:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:52:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:52:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:52:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:52:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:52:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:52:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:52:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:52:40 smithi083 ceph-mon[77242]: pgmap v9732: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:52:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:52:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:52:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:52:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:52:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:52:40.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:52:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:52:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:52:40 smithi153 ceph-mon[70044]: pgmap v9732: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:52:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:52:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:52:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:52:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:52:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:52:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:52:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:52:41.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:52:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:52:41] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T10:52:41.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:52:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:52:41.360Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:52:42.649 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:52:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:52:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:52:42.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:52:42 smithi083 ceph-mon[79930]: pgmap v9733: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:42.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:52:42 smithi083 ceph-mon[77242]: pgmap v9733: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:43.022 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:52:42 smithi153 ceph-mon[70044]: pgmap v9733: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:52:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:52:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:52:44.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:52:44 smithi083 ceph-mon[79930]: pgmap v9734: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:44.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:52:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:52:44.734Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:52:44.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:52:44 smithi083 ceph-mon[77242]: pgmap v9734: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:45.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:52:44 smithi153 ceph-mon[70044]: pgmap v9734: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:45.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:52:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:52:45.442Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:52:46.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:52:46 smithi083 ceph-mon[79930]: pgmap v9735: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:46.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:52:46 smithi083 ceph-mon[77242]: pgmap v9735: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:47.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:52:46 smithi153 ceph-mon[70044]: pgmap v9735: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:48.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:52:48 smithi083 ceph-mon[79930]: pgmap v9736: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:48.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:52:48 smithi083 ceph-mon[77242]: pgmap v9736: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:49.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:52:48 smithi153 ceph-mon[70044]: pgmap v9736: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:50.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:52:49 smithi083 ceph-mon[79930]: pgmap v9737: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:50.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:52:49 smithi083 ceph-mon[77242]: pgmap v9737: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:50.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:52:49 smithi153 ceph-mon[70044]: pgmap v9737: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:51.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:52:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:52:51] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T10:52:51.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:52:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:52:51.360Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:52:52.628 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:52:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:52:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:52:52.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:52:52 smithi083 ceph-mon[79930]: pgmap v9738: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:52.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:52:52 smithi083 ceph-mon[77242]: pgmap v9738: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:53.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:52:52 smithi153 ceph-mon[70044]: pgmap v9738: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:52:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:52:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:52:54.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:52:54 smithi083 ceph-mon[77242]: pgmap v9739: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:54.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:52:54 smithi083 ceph-mon[79930]: pgmap v9739: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:54.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:52:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:52:54.734Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:52:55.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:52:54 smithi153 ceph-mon[70044]: pgmap v9739: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:55.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:52:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:52:55.443Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:52:56.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:52:56 smithi083 ceph-mon[79930]: pgmap v9740: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:56.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:52:56 smithi083 ceph-mon[77242]: pgmap v9740: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:57.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:52:56 smithi153 ceph-mon[70044]: pgmap v9740: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:58.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:52:58 smithi083 ceph-mon[79930]: pgmap v9741: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:58.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:52:58 smithi083 ceph-mon[77242]: pgmap v9741: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:52:59.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:52:58 smithi153 ceph-mon[70044]: pgmap v9741: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:52:59 smithi153 ceph-mon[70044]: pgmap v9742: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:00.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:52:59 smithi083 ceph-mon[79930]: pgmap v9742: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:00.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:52:59 smithi083 ceph-mon[77242]: pgmap v9742: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:01.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:53:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:53:01] "GET /metrics HTTP/1.1" 200 225228 "" "Prometheus/2.33.4" 2022-09-29T10:53:01.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:53:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:53:01.360Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:53:01.975 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:53:02.264 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:53:02.264 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 84s ago 5h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:53:02.264 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 9m ago 5h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:53:02.265 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 84s ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:53:02.265 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 9m ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:53:02.265 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 9m ago 5h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:53:02.265 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 84s ago 5h 791M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:53:02.265 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 84s ago 5h 1235M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:53:02.266 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 9m ago 5h 516M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:53:02.266 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 84s ago 5h 709M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:53:02.266 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 84s ago 5h 26.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:53:02.266 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 9m ago 5h 26.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:53:02.267 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 84s ago 5h 870M 2456M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:53:02.267 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 84s ago 5h 860M 2456M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:53:02.267 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 84s ago 5h 671M 2456M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:53:02.267 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 84s ago 5h 839M 2456M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:53:02.268 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 9m ago 5h 718M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:53:02.268 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 9m ago 5h 895M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:53:02.268 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 9m ago 5h 799M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:53:02.268 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 9m ago 5h 945M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:53:02.269 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 9m ago 5h 189M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:53:02.623 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:53:02.623 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:53:02.624 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:53:02.624 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:53:02.624 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:53:02.624 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:53:02.625 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:53:02.625 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:53:02.625 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:53:02.625 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:53:02.626 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:53:02.626 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:53:02.626 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:53:02.626 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:53:02.626 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:53:02.627 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:53:02.627 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:53:02.627 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:53:02.631 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:53:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:53:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:53:02.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:53:02 smithi083 ceph-mon[79930]: pgmap v9743: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:02.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:53:02 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3170622567' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:53:02.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:53:02 smithi083 ceph-mon[77242]: pgmap v9743: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:02.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:53:02 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3170622567' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:53:03.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:53:02 smithi153 ceph-mon[70044]: pgmap v9743: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:03.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:53:02 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3170622567' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:53:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:53:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:53:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:53:03.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:53:03 smithi083 ceph-mon[79930]: from='client.55396 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:53:03.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:53:03 smithi083 ceph-mon[79930]: from='client.55402 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:53:03.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:53:03 smithi083 ceph-mon[77242]: from='client.55396 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:53:03.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:53:03 smithi083 ceph-mon[77242]: from='client.55402 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:53:04.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:53:03 smithi153 ceph-mon[70044]: from='client.55396 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:53:04.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:53:03 smithi153 ceph-mon[70044]: from='client.55402 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:53:04.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:53:04 smithi083 ceph-mon[79930]: pgmap v9744: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:04.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:53:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:53:04.735Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:53:04.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:53:04 smithi083 ceph-mon[77242]: pgmap v9744: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:05.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:53:04 smithi153 ceph-mon[70044]: pgmap v9744: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:05.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:53:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:53:05.444Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:53:06.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:53:06 smithi083 ceph-mon[79930]: pgmap v9745: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:06.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:53:06 smithi083 ceph-mon[77242]: pgmap v9745: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:07.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:53:06 smithi153 ceph-mon[70044]: pgmap v9745: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:08.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:53:08 smithi083 ceph-mon[79930]: pgmap v9746: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:08.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:53:08 smithi083 ceph-mon[77242]: pgmap v9746: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:09.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:53:08 smithi153 ceph-mon[70044]: pgmap v9746: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:10.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:53:09 smithi153 ceph-mon[70044]: pgmap v9747: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:10.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:53:09 smithi083 ceph-mon[79930]: pgmap v9747: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:10.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:53:09 smithi083 ceph-mon[77242]: pgmap v9747: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:11.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:53:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:53:11] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T10:53:11.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:53:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:53:11.361Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:53:12.635 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:53:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:53:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:53:12.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:53:12 smithi083 ceph-mon[79930]: pgmap v9748: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:12.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:53:12 smithi083 ceph-mon[77242]: pgmap v9748: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:13.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:53:12 smithi153 ceph-mon[70044]: pgmap v9748: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:53:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:53:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:53:14.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:53:14 smithi083 ceph-mon[79930]: pgmap v9749: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:14.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:53:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:53:14.735Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:53:14.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:53:14 smithi083 ceph-mon[77242]: pgmap v9749: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:15.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:53:14 smithi153 ceph-mon[70044]: pgmap v9749: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:15.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:53:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:53:15.444Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:53:16.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:53:16 smithi083 ceph-mon[77242]: pgmap v9750: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:16.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:53:16 smithi083 ceph-mon[79930]: pgmap v9750: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:17.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:53:16 smithi153 ceph-mon[70044]: pgmap v9750: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:18.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:53:18 smithi083 ceph-mon[79930]: pgmap v9751: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:18.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:53:18 smithi083 ceph-mon[77242]: pgmap v9751: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:19.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:53:18 smithi153 ceph-mon[70044]: pgmap v9751: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:53:19 smithi153 ceph-mon[70044]: pgmap v9752: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:20.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:53:19 smithi083 ceph-mon[77242]: pgmap v9752: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:20.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:53:19 smithi083 ceph-mon[79930]: pgmap v9752: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:21.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:53:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:53:21] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T10:53:21.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:53:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:53:21.361Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:53:22.638 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:53:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:53:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:53:22.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:53:22 smithi083 ceph-mon[77242]: pgmap v9753: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:22.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:53:22 smithi083 ceph-mon[79930]: pgmap v9753: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:23.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:53:22 smithi153 ceph-mon[70044]: pgmap v9753: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:53:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:53:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:53:24.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:53:24 smithi083 ceph-mon[79930]: pgmap v9754: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:24.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:53:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:53:24.736Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:53:24.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:53:24 smithi083 ceph-mon[77242]: pgmap v9754: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:25.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:53:24 smithi153 ceph-mon[70044]: pgmap v9754: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:25.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:53:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:53:25.444Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:53:26.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:53:26 smithi083 ceph-mon[77242]: pgmap v9755: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:26.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:53:26 smithi083 ceph-mon[79930]: pgmap v9755: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:27.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:53:26 smithi153 ceph-mon[70044]: pgmap v9755: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:28.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:53:28 smithi083 ceph-mon[79930]: pgmap v9756: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:28.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:53:28 smithi083 ceph-mon[77242]: pgmap v9756: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:29.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:53:28 smithi153 ceph-mon[70044]: pgmap v9756: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:30.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:53:29 smithi153 ceph-mon[70044]: pgmap v9757: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:30.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:53:29 smithi083 ceph-mon[77242]: pgmap v9757: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:30.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:53:29 smithi083 ceph-mon[79930]: pgmap v9757: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:31.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:53:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:53:31] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T10:53:31.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:53:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:53:31.363Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:53:32.640 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:53:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:53:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:53:32.951 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:53:32.960 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:53:32 smithi083 ceph-mon[77242]: pgmap v9758: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:32.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:53:32 smithi083 ceph-mon[79930]: pgmap v9758: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:33.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:53:32 smithi153 ceph-mon[70044]: pgmap v9758: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:33.241 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:53:33.241 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 115s ago 5h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:53:33.242 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 10m ago 5h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:53:33.242 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 115s ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:53:33.242 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 10m ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:53:33.242 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 10m ago 5h 444M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:53:33.243 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 115s ago 5h 791M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:53:33.243 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 115s ago 5h 1235M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:53:33.243 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 10m ago 5h 516M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:53:33.243 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 115s ago 5h 709M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:53:33.244 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 115s ago 5h 26.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:53:33.244 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 10m ago 5h 26.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:53:33.244 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 115s ago 5h 870M 2456M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:53:33.244 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 115s ago 5h 860M 2456M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:53:33.244 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 115s ago 5h 671M 2456M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:53:33.245 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 115s ago 5h 839M 2456M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:53:33.245 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 10m ago 5h 718M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:53:33.245 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 10m ago 5h 895M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:53:33.245 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 10m ago 5h 799M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:53:33.246 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 10m ago 5h 945M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:53:33.246 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 10m ago 5h 189M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:53:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:53:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:53:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:53:33.608 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:53:33.608 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:53:33.608 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:53:33.609 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:53:33.609 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:53:33.609 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:53:33.609 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:53:33.609 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:53:33.610 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:53:33.610 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:53:33.610 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:53:33.610 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:53:33.611 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:53:33.611 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:53:33.611 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:53:33.611 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:53:33.612 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:53:33.612 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:53:33.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:53:33 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3173538490' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:53:33.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:53:33 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3173538490' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:53:34.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:53:33 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3173538490' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:53:34.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:53:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:53:34.736Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:53:34.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:53:34 smithi083 ceph-mon[77242]: from='client.45483 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:53:34.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:53:34 smithi083 ceph-mon[77242]: from='client.55420 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:53:34.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:53:34 smithi083 ceph-mon[77242]: pgmap v9759: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:34.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:53:34 smithi083 ceph-mon[79930]: from='client.45483 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:53:34.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:53:34 smithi083 ceph-mon[79930]: from='client.55420 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:53:34.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:53:34 smithi083 ceph-mon[79930]: pgmap v9759: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:35.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:53:34 smithi153 ceph-mon[70044]: from='client.45483 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:53:35.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:53:34 smithi153 ceph-mon[70044]: from='client.55420 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:53:35.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:53:34 smithi153 ceph-mon[70044]: pgmap v9759: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:35.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:53:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:53:35.446Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:53:36.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:53:36 smithi083 ceph-mon[77242]: pgmap v9760: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:36.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:53:36 smithi083 ceph-mon[79930]: pgmap v9760: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:37.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:53:36 smithi153 ceph-mon[70044]: pgmap v9760: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:38.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:53:38 smithi083 ceph-mon[79930]: pgmap v9761: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:38.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:53:38 smithi083 ceph-mon[77242]: pgmap v9761: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:39.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:53:38 smithi153 ceph-mon[70044]: pgmap v9761: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:39.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:53:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:53:39.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:53:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:53:39.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:53:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:53:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:53:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:53:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:53:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:53:39.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:53:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:53:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:53:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:53:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:53:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:53:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:53:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:53:40.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:53:40 smithi083 ceph-mon[77242]: pgmap v9762: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:40.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:53:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:53:40.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:53:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:53:40.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:53:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:53:40.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:53:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:53:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:53:40 smithi083 ceph-mon[79930]: pgmap v9762: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:53:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:53:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:53:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:53:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:53:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:53:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:53:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:53:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:53:40 smithi153 ceph-mon[70044]: pgmap v9762: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:53:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:53:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:53:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:53:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:53:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:53:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:53:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:53:41.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:53:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:53:41] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T10:53:41.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:53:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:53:41.363Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:53:42.663 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:53:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:53:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:53:42.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:53:42 smithi083 ceph-mon[79930]: pgmap v9763: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:42.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:53:42 smithi083 ceph-mon[77242]: pgmap v9763: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:43.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:53:42 smithi153 ceph-mon[70044]: pgmap v9763: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:53:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:53:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:53:44.739 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:53:44 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:53:44.739 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:53:44 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:53:44.739 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:53:44 smithi083 ceph-mon[79930]: pgmap v9764: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:44.740 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:53:44 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:53:44.740 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:53:44 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:53:44.740 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:53:44 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:53:44.740 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:53:44 smithi083 ceph-mon[77242]: pgmap v9764: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:44.741 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:53:44 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:53:44.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:53:44 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:53:44.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:53:44 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:53:44.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:53:44 smithi153 ceph-mon[70044]: pgmap v9764: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:44.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:53:44 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:53:45.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:53:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:53:44.738Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:53:45.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:53:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:53:45.446Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:53:46.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:53:46 smithi083 ceph-mon[79930]: pgmap v9765: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:46.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:53:46 smithi083 ceph-mon[77242]: pgmap v9765: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:47.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:53:46 smithi153 ceph-mon[70044]: pgmap v9765: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:48.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:53:48 smithi083 ceph-mon[77242]: pgmap v9766: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:48.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:53:48 smithi083 ceph-mon[79930]: pgmap v9766: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:49.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:53:48 smithi153 ceph-mon[70044]: pgmap v9766: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:50.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:53:49 smithi153 ceph-mon[70044]: pgmap v9767: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:50.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:53:49 smithi083 ceph-mon[79930]: pgmap v9767: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:50.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:53:49 smithi083 ceph-mon[77242]: pgmap v9767: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:51.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:53:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:53:51] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T10:53:51.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:53:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:53:51.363Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:53:52.647 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:53:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:53:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:53:52.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:53:52 smithi083 ceph-mon[79930]: pgmap v9768: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:52.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:53:52 smithi083 ceph-mon[77242]: pgmap v9768: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:53.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:53:52 smithi153 ceph-mon[70044]: pgmap v9768: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:53:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:53:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:53:54.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:53:54 smithi083 ceph-mon[77242]: pgmap v9769: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:54.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:53:54 smithi083 ceph-mon[79930]: pgmap v9769: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:54.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:53:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:53:54.737Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:53:55.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:53:54 smithi153 ceph-mon[70044]: pgmap v9769: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:55.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:53:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:53:55.447Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:53:56.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:53:56 smithi083 ceph-mon[77242]: pgmap v9770: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:56.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:53:56 smithi083 ceph-mon[79930]: pgmap v9770: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:57.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:53:56 smithi153 ceph-mon[70044]: pgmap v9770: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:58.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:53:58 smithi083 ceph-mon[79930]: pgmap v9771: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:58.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:53:58 smithi083 ceph-mon[77242]: pgmap v9771: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:53:59.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:53:58 smithi153 ceph-mon[70044]: pgmap v9771: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:53:59 smithi153 ceph-mon[70044]: pgmap v9772: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:00.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:53:59 smithi083 ceph-mon[77242]: pgmap v9772: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:00.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:53:59 smithi083 ceph-mon[79930]: pgmap v9772: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:01.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:54:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:54:01.364Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:54:01.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:54:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:54:01] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T10:54:02.651 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:54:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:54:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:54:02.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:54:02 smithi083 ceph-mon[79930]: pgmap v9773: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:02.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:54:02 smithi083 ceph-mon[77242]: pgmap v9773: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:03.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:54:02 smithi153 ceph-mon[70044]: pgmap v9773: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:54:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:54:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:54:03.950 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:54:04.240 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:54:04.240 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 2m ago 5h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:54:04.241 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 20s ago 5h 63.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:54:04.241 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 2m ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:54:04.241 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 20s ago 5h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:54:04.241 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 20s ago 5h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:54:04.242 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 2m ago 5h 791M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:54:04.242 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 2m ago 5h 1235M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:54:04.242 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 20s ago 5h 532M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:54:04.242 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 2m ago 5h 709M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:54:04.242 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 2m ago 5h 26.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:54:04.243 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 20s ago 5h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:54:04.243 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 2m ago 5h 870M 2456M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:54:04.243 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 2m ago 5h 860M 2456M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:54:04.243 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 2m ago 5h 671M 2456M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:54:04.243 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 2m ago 5h 839M 2456M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:54:04.244 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 20s ago 5h 740M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:54:04.244 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 20s ago 5h 922M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:54:04.244 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 20s ago 5h 823M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:54:04.244 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 20s ago 5h 975M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:54:04.245 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 20s ago 5h 191M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:54:04.592 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:54:04.592 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:54:04.593 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:54:04.593 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:54:04.593 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:54:04.594 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:54:04.594 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:54:04.594 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:54:04.594 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:54:04.595 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:54:04.595 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:54:04.595 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:54:04.595 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:54:04.595 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:54:04.596 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:54:04.596 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:54:04.596 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:54:04.596 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:54:04.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:54:04 smithi083 ceph-mon[77242]: pgmap v9774: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:04.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:54:04 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2455031347' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:54:04.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:54:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:54:04.737Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:54:04.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:54:04 smithi083 ceph-mon[79930]: pgmap v9774: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:04.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:54:04 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2455031347' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:54:05.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:54:04 smithi153 ceph-mon[70044]: pgmap v9774: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:05.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:54:04 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2455031347' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:54:05.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:54:05 smithi083 ceph-mon[79930]: from='client.45501 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:54:05.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:54:05 smithi083 ceph-mon[79930]: from='client.55438 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:54:05.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:54:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:54:05.447Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:54:05.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:54:05 smithi083 ceph-mon[77242]: from='client.45501 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:54:05.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:54:05 smithi083 ceph-mon[77242]: from='client.55438 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:54:06.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:54:05 smithi153 ceph-mon[70044]: from='client.45501 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:54:06.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:54:05 smithi153 ceph-mon[70044]: from='client.55438 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:54:06.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:54:06 smithi083 ceph-mon[77242]: pgmap v9775: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:06.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:54:06 smithi083 ceph-mon[79930]: pgmap v9775: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:07.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:54:06 smithi153 ceph-mon[70044]: pgmap v9775: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:08.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:54:08 smithi083 ceph-mon[79930]: pgmap v9776: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:08.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:54:08 smithi083 ceph-mon[77242]: pgmap v9776: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:09.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:54:08 smithi153 ceph-mon[70044]: pgmap v9776: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:10.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:54:09 smithi153 ceph-mon[70044]: pgmap v9777: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:10.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:54:09 smithi083 ceph-mon[79930]: pgmap v9777: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:10.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:54:09 smithi083 ceph-mon[77242]: pgmap v9777: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:11.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:54:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:54:11] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T10:54:11.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:54:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:54:11.365Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:54:12.654 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:54:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:54:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:54:12.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:54:12 smithi083 ceph-mon[77242]: pgmap v9778: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:12.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:54:12 smithi083 ceph-mon[79930]: pgmap v9778: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:13.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:54:12 smithi153 ceph-mon[70044]: pgmap v9778: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:54:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:54:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:54:14.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:54:14 smithi083 ceph-mon[79930]: pgmap v9779: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:14.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:54:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:54:14.739Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:54:14.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:54:14 smithi083 ceph-mon[77242]: pgmap v9779: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:15.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:54:14 smithi153 ceph-mon[70044]: pgmap v9779: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:15.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:54:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:54:15.448Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:54:16.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:54:16 smithi083 ceph-mon[77242]: pgmap v9780: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:16.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:54:16 smithi083 ceph-mon[79930]: pgmap v9780: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:17.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:54:16 smithi153 ceph-mon[70044]: pgmap v9780: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:18.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:54:18 smithi083 ceph-mon[79930]: pgmap v9781: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:18.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:54:18 smithi083 ceph-mon[77242]: pgmap v9781: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:19.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:54:18 smithi153 ceph-mon[70044]: pgmap v9781: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:54:19 smithi153 ceph-mon[70044]: pgmap v9782: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:20.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:54:19 smithi083 ceph-mon[79930]: pgmap v9782: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:20.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:54:19 smithi083 ceph-mon[77242]: pgmap v9782: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:21.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:54:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:54:21] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T10:54:21.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:54:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:54:21.365Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:54:22.657 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:54:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:54:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:54:22.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:54:22 smithi083 ceph-mon[79930]: pgmap v9783: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:22.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:54:22 smithi083 ceph-mon[77242]: pgmap v9783: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:23.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:54:22 smithi153 ceph-mon[70044]: pgmap v9783: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:54:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:54:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:54:24.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:54:24 smithi083 ceph-mon[79930]: pgmap v9784: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:24.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:54:24 smithi083 ceph-mon[77242]: pgmap v9784: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:24.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:54:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:54:24.740Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:54:25.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:54:24 smithi153 ceph-mon[70044]: pgmap v9784: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:25.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:54:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:54:25.449Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:54:26.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:54:26 smithi083 ceph-mon[79930]: pgmap v9785: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:26.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:54:26 smithi083 ceph-mon[77242]: pgmap v9785: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:27.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:54:26 smithi153 ceph-mon[70044]: pgmap v9785: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:28.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:54:28 smithi083 ceph-mon[79930]: pgmap v9786: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:28.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:54:28 smithi083 ceph-mon[77242]: pgmap v9786: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:29.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:54:28 smithi153 ceph-mon[70044]: pgmap v9786: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:30.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:54:29 smithi153 ceph-mon[70044]: pgmap v9787: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:30.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:54:29 smithi083 ceph-mon[79930]: pgmap v9787: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:30.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:54:29 smithi083 ceph-mon[77242]: pgmap v9787: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:31.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:54:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:54:31] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T10:54:31.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:54:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:54:31.365Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:54:32.660 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:54:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:54:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:54:32.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:54:32 smithi083 ceph-mon[77242]: pgmap v9788: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:32.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:54:32 smithi083 ceph-mon[79930]: pgmap v9788: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:33.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:54:32 smithi153 ceph-mon[70044]: pgmap v9788: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:54:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:54:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:54:34.918 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:54:34.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:54:34 smithi083 ceph-mon[79930]: pgmap v9789: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:34.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:54:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:54:34.740Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:54:34.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:54:34 smithi083 ceph-mon[77242]: pgmap v9789: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:35.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:54:34 smithi153 ceph-mon[70044]: pgmap v9789: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:35.210 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:54:35.210 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 2m ago 5h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:54:35.211 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 51s ago 5h 63.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:54:35.211 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 2m ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:54:35.211 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 51s ago 5h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:54:35.212 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 51s ago 5h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:54:35.212 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 2m ago 5h 791M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:54:35.212 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 2m ago 5h 1235M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:54:35.212 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 51s ago 5h 532M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:54:35.212 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 2m ago 5h 709M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:54:35.213 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 2m ago 5h 26.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:54:35.213 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 51s ago 5h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:54:35.213 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 2m ago 5h 870M 2456M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:54:35.213 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 2m ago 5h 860M 2456M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:54:35.214 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 2m ago 5h 671M 2456M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:54:35.214 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 2m ago 5h 839M 2456M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:54:35.214 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 51s ago 5h 740M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:54:35.214 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 51s ago 5h 922M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:54:35.215 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 51s ago 5h 823M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:54:35.215 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 51s ago 5h 975M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:54:35.215 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 51s ago 5h 191M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:54:35.565 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:54:35.565 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:54:35.566 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:54:35.566 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:54:35.566 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:54:35.566 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:54:35.567 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:54:35.567 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:54:35.567 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:54:35.567 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:54:35.567 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:54:35.568 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:54:35.568 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:54:35.568 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:54:35.568 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:54:35.569 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:54:35.569 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:54:35.569 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:54:35.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:54:35 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3338898930' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:54:35.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:54:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:54:35.449Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:54:35.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:54:35 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3338898930' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:54:36.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:54:35 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3338898930' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:54:36.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:54:36 smithi083 ceph-mon[79930]: from='client.55450 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:54:36.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:54:36 smithi083 ceph-mon[79930]: from='client.55456 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:54:36.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:54:36 smithi083 ceph-mon[79930]: pgmap v9790: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:36.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:54:36 smithi083 ceph-mon[77242]: from='client.55450 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:54:36.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:54:36 smithi083 ceph-mon[77242]: from='client.55456 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:54:36.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:54:36 smithi083 ceph-mon[77242]: pgmap v9790: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:37.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:54:36 smithi153 ceph-mon[70044]: from='client.55450 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:54:37.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:54:36 smithi153 ceph-mon[70044]: from='client.55456 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:54:37.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:54:36 smithi153 ceph-mon[70044]: pgmap v9790: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:38.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:54:38 smithi083 ceph-mon[77242]: pgmap v9791: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:38.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:54:38 smithi083 ceph-mon[79930]: pgmap v9791: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:39.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:54:38 smithi153 ceph-mon[70044]: pgmap v9791: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:54:39 smithi153 ceph-mon[70044]: pgmap v9792: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:40.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:54:39 smithi083 ceph-mon[79930]: pgmap v9792: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:40.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:54:39 smithi083 ceph-mon[77242]: pgmap v9792: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:54:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:54:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:54:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:54:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:54:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:54:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:54:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:54:41.367 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:54:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:54:41.368 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:54:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:54:41.368 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:54:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:54:41.369 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:54:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:54:41.369 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:54:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:54:41.370 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:54:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:54:41.370 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:54:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:54:41.370 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:54:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:54:41.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:54:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:54:41] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T10:54:41.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:54:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:54:41.366Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:54:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:54:41 smithi153 ceph-mon[70044]: pgmap v9793: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:42.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:54:41 smithi083 ceph-mon[77242]: pgmap v9793: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:54:41 smithi083 ceph-mon[79930]: pgmap v9793: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:54:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:54:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:54:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:54:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:54:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:54:44.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:54:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:54:44.740Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:54:44.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:54:44 smithi083 ceph-mon[77242]: pgmap v9794: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:44.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:54:44 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:54:44.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:54:44 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:54:44.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:54:44 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:54:44.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:54:44 smithi083 ceph-mon[79930]: pgmap v9794: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:44.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:54:44 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:54:44.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:54:44 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:54:44.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:54:44 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:54:45.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:54:44 smithi153 ceph-mon[70044]: pgmap v9794: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:45.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:54:44 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:54:45.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:54:44 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:54:45.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:54:44 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:54:45.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:54:45 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:54:45.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:54:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:54:45.450Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:54:45.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:54:45 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:54:46.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:54:45 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:54:46.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:54:46 smithi083 ceph-mon[79930]: pgmap v9795: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:46.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:54:46 smithi083 ceph-mon[77242]: pgmap v9795: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:47.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:54:46 smithi153 ceph-mon[70044]: pgmap v9795: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:48.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:54:48 smithi083 ceph-mon[77242]: pgmap v9796: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:48.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:54:48 smithi083 ceph-mon[79930]: pgmap v9796: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:49.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:54:48 smithi153 ceph-mon[70044]: pgmap v9796: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:50.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:54:49 smithi153 ceph-mon[70044]: pgmap v9797: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:50.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:54:49 smithi083 ceph-mon[79930]: pgmap v9797: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:50.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:54:49 smithi083 ceph-mon[77242]: pgmap v9797: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:51.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:54:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:54:51] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T10:54:51.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:54:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:54:51.366Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:54:52.667 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:54:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:54:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:54:52.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:54:52 smithi083 ceph-mon[79930]: pgmap v9798: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:52.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:54:52 smithi083 ceph-mon[77242]: pgmap v9798: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:53.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:54:52 smithi153 ceph-mon[70044]: pgmap v9798: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:54:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:54:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:54:54.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:54:54 smithi083 ceph-mon[79930]: pgmap v9799: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:54.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:54:54 smithi083 ceph-mon[77242]: pgmap v9799: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:54.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:54:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:54:54.741Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:54:55.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:54:54 smithi153 ceph-mon[70044]: pgmap v9799: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:55.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:54:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:54:55.451Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:54:56.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:54:56 smithi083 ceph-mon[79930]: pgmap v9800: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:56.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:54:56 smithi083 ceph-mon[77242]: pgmap v9800: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:57.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:54:56 smithi153 ceph-mon[70044]: pgmap v9800: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:58.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:54:58 smithi083 ceph-mon[77242]: pgmap v9801: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:58.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:54:58 smithi083 ceph-mon[79930]: pgmap v9801: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:54:59.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:54:58 smithi153 ceph-mon[70044]: pgmap v9801: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:54:59 smithi153 ceph-mon[70044]: pgmap v9802: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:00.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:54:59 smithi083 ceph-mon[79930]: pgmap v9802: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:00.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:54:59 smithi083 ceph-mon[77242]: pgmap v9802: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:01.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:55:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:55:01] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T10:55:01.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:55:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:55:01.367Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:55:02.672 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:55:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:55:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:55:02.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:55:02 smithi083 ceph-mon[79930]: pgmap v9803: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:02.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:55:02 smithi083 ceph-mon[77242]: pgmap v9803: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:03.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:55:02 smithi153 ceph-mon[70044]: pgmap v9803: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:55:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:55:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:55:04.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:55:04 smithi083 ceph-mon[79930]: pgmap v9804: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:04.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:55:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:55:04.741Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:55:04.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:55:04 smithi083 ceph-mon[77242]: pgmap v9804: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:05.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:55:04 smithi153 ceph-mon[70044]: pgmap v9804: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:05.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:55:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:55:05.452Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:55:05.893 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:55:06.180 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:55:06.181 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 3m ago 5h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:55:06.181 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 82s ago 5h 63.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:55:06.181 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 3m ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:55:06.182 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 82s ago 5h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:55:06.182 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 82s ago 5h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:55:06.182 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 3m ago 5h 791M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:55:06.182 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 3m ago 5h 1235M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:55:06.183 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 82s ago 5h 532M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:55:06.183 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 3m ago 5h 709M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:55:06.183 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 3m ago 5h 26.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:55:06.183 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 82s ago 5h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:55:06.184 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 3m ago 5h 870M 2456M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:55:06.184 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 3m ago 5h 860M 2456M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:55:06.184 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 3m ago 5h 671M 2456M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:55:06.184 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 3m ago 5h 839M 2456M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:55:06.185 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 82s ago 5h 740M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:55:06.185 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 82s ago 5h 922M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:55:06.185 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 82s ago 5h 823M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:55:06.185 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 82s ago 5h 975M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:55:06.186 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 82s ago 5h 191M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:55:06.536 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:55:06.537 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:55:06.537 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:55:06.537 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:55:06.538 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:55:06.538 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:55:06.538 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:55:06.538 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:55:06.539 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:55:06.539 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:55:06.539 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:55:06.539 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:55:06.539 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:55:06.540 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:55:06.540 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:55:06.540 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:55:06.540 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:55:06.541 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:55:06.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:55:06 smithi083 ceph-mon[79930]: pgmap v9805: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:06.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:55:06 smithi083 ceph-mon[79930]: from='client.45537 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:55:06.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:55:06 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1030739711' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:55:06.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:55:06 smithi083 ceph-mon[77242]: pgmap v9805: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:06.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:55:06 smithi083 ceph-mon[77242]: from='client.45537 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:55:06.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:55:06 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1030739711' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:55:07.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:55:06 smithi153 ceph-mon[70044]: pgmap v9805: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:07.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:55:06 smithi153 ceph-mon[70044]: from='client.45537 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:55:07.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:55:06 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1030739711' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:55:07.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:55:07 smithi083 ceph-mon[79930]: from='client.55474 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:55:07.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:55:07 smithi083 ceph-mon[77242]: from='client.55474 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:55:08.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:55:07 smithi153 ceph-mon[70044]: from='client.55474 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:55:08.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:55:08 smithi083 ceph-mon[79930]: pgmap v9806: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:08.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:55:08 smithi083 ceph-mon[77242]: pgmap v9806: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:09.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:55:08 smithi153 ceph-mon[70044]: pgmap v9806: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:10.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:55:09 smithi153 ceph-mon[70044]: pgmap v9807: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:10.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:55:09 smithi083 ceph-mon[79930]: pgmap v9807: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:10.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:55:09 smithi083 ceph-mon[77242]: pgmap v9807: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:11.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:55:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:55:11] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T10:55:11.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:55:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:55:11.367Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:55:12.673 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:55:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:55:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:55:12.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:55:12 smithi083 ceph-mon[79930]: pgmap v9808: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:12.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:55:12 smithi083 ceph-mon[77242]: pgmap v9808: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:13.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:55:12 smithi153 ceph-mon[70044]: pgmap v9808: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:55:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:55:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:55:14.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:55:14 smithi083 ceph-mon[79930]: pgmap v9809: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:14.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:55:14 smithi083 ceph-mon[77242]: pgmap v9809: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:14.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:55:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:55:14.742Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:55:15.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:55:14 smithi153 ceph-mon[70044]: pgmap v9809: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:15.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:55:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:55:15.453Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:55:16.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:55:16 smithi083 ceph-mon[77242]: pgmap v9810: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:16.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:55:16 smithi083 ceph-mon[79930]: pgmap v9810: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:17.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:55:16 smithi153 ceph-mon[70044]: pgmap v9810: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:18.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:55:18 smithi083 ceph-mon[79930]: pgmap v9811: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:18.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:55:18 smithi083 ceph-mon[77242]: pgmap v9811: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:19.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:55:18 smithi153 ceph-mon[70044]: pgmap v9811: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:55:19 smithi153 ceph-mon[70044]: pgmap v9812: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:20.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:55:19 smithi083 ceph-mon[79930]: pgmap v9812: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:20.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:55:19 smithi083 ceph-mon[77242]: pgmap v9812: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:21.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:55:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:55:21] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T10:55:21.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:55:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:55:21.368Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:55:22.676 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:55:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:55:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:55:22.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:55:22 smithi083 ceph-mon[79930]: pgmap v9813: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:22.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:55:22 smithi083 ceph-mon[77242]: pgmap v9813: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:23.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:55:22 smithi153 ceph-mon[70044]: pgmap v9813: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:55:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:55:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:55:24.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:55:24 smithi083 ceph-mon[79930]: pgmap v9814: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:24.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:55:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:55:24.743Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:55:24.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:55:24 smithi083 ceph-mon[77242]: pgmap v9814: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:25.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:55:24 smithi153 ceph-mon[70044]: pgmap v9814: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:25.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:55:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:55:25.454Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:55:26.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:55:26 smithi083 ceph-mon[79930]: pgmap v9815: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:26.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:55:26 smithi083 ceph-mon[77242]: pgmap v9815: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:27.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:55:26 smithi153 ceph-mon[70044]: pgmap v9815: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:28.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:55:28 smithi083 ceph-mon[79930]: pgmap v9816: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:28.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:55:28 smithi083 ceph-mon[77242]: pgmap v9816: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:29.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:55:28 smithi153 ceph-mon[70044]: pgmap v9816: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:30.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:55:29 smithi153 ceph-mon[70044]: pgmap v9817: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:30.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:55:29 smithi083 ceph-mon[79930]: pgmap v9817: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:30.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:55:29 smithi083 ceph-mon[77242]: pgmap v9817: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:31.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:55:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:55:31] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T10:55:31.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:55:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:55:31.367Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:55:32.679 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:55:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:55:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:55:32.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:55:32 smithi083 ceph-mon[77242]: pgmap v9818: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:32.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:55:32 smithi083 ceph-mon[79930]: pgmap v9818: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:33.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:55:32 smithi153 ceph-mon[70044]: pgmap v9818: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:55:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:55:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:55:34.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:55:34 smithi083 ceph-mon[77242]: pgmap v9819: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:34.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:55:34 smithi083 ceph-mon[79930]: pgmap v9819: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:34.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:55:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:55:34.744Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:55:35.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:55:34 smithi153 ceph-mon[70044]: pgmap v9819: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:35.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:55:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:55:35.454Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:55:36.864 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:55:36.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:55:36 smithi083 ceph-mon[79930]: pgmap v9820: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:36.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:55:36 smithi083 ceph-mon[77242]: pgmap v9820: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:37.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:55:36 smithi153 ceph-mon[70044]: pgmap v9820: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:37.153 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:55:37.153 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 3m ago 5h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:55:37.153 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 113s ago 5h 63.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:55:37.153 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 3m ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:55:37.154 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 113s ago 5h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:55:37.154 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 113s ago 5h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:55:37.154 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 3m ago 5h 791M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:55:37.154 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 3m ago 5h 1235M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:55:37.154 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 113s ago 5h 532M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:55:37.155 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 3m ago 5h 709M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:55:37.155 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 3m ago 5h 26.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:55:37.155 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 113s ago 5h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:55:37.155 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 3m ago 5h 870M 2456M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:55:37.155 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 3m ago 5h 860M 2456M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:55:37.156 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 3m ago 5h 671M 2456M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:55:37.156 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 3m ago 5h 839M 2456M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:55:37.156 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 113s ago 5h 740M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:55:37.156 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 113s ago 5h 922M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:55:37.156 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 113s ago 5h 823M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:55:37.157 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 113s ago 5h 975M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:55:37.157 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 113s ago 5h 191M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:55:37.509 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:55:37.509 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:55:37.510 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:55:37.510 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:55:37.510 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:55:37.510 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:55:37.510 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:55:37.511 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:55:37.511 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:55:37.511 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:55:37.511 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:55:37.512 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:55:37.512 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:55:37.512 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:55:37.512 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:55:37.512 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:55:37.513 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:55:37.513 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:55:37.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:55:37 smithi083 ceph-mon[79930]: from='client.55486 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:55:37.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:55:37 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1563681119' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:55:37.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:55:37 smithi083 ceph-mon[77242]: from='client.55486 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:55:37.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:55:37 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1563681119' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:55:38.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:55:37 smithi153 ceph-mon[70044]: from='client.55486 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:55:38.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:55:37 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1563681119' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:55:38.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:55:38 smithi083 ceph-mon[79930]: from='client.55492 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:55:38.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:55:38 smithi083 ceph-mon[79930]: pgmap v9821: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:38.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:55:38 smithi083 ceph-mon[77242]: from='client.55492 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:55:38.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:55:38 smithi083 ceph-mon[77242]: pgmap v9821: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:39.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:55:38 smithi153 ceph-mon[70044]: from='client.55492 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:55:39.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:55:38 smithi153 ceph-mon[70044]: pgmap v9821: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:55:39 smithi153 ceph-mon[70044]: pgmap v9822: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:40.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:55:39 smithi083 ceph-mon[79930]: pgmap v9822: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:40.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:55:39 smithi083 ceph-mon[77242]: pgmap v9822: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:55:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:55:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:55:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:55:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:55:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:55:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:55:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:55:41.369 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:55:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:55:41.370 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:55:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:55:41.370 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:55:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:55:41.370 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:55:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:55:41.371 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:55:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:55:41.371 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:55:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:55:41.371 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:55:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:55:41.371 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:55:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:55:41.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:55:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:55:41] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T10:55:41.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:55:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:55:41.368Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:55:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:55:42 smithi153 ceph-mon[70044]: pgmap v9823: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:42.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:55:41 smithi083 ceph-mon[77242]: pgmap v9823: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:42.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:55:41 smithi083 ceph-mon[79930]: pgmap v9823: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:55:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:55:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:55:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:55:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:55:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:55:44.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:55:44 smithi083 ceph-mon[77242]: pgmap v9824: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:44.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:55:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:55:44.745Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:55:44.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:55:44 smithi083 ceph-mon[79930]: pgmap v9824: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:45.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:55:44 smithi153 ceph-mon[70044]: pgmap v9824: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:45.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:55:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:55:45.455Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:55:45.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:55:45 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:55:45.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:55:45 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:55:45.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:55:45 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:55:45.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:55:45 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:55:45.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:55:45 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:55:45.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:55:45 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:55:45.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:55:45 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:55:45.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:55:45 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:55:46.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:55:45 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:55:46.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:55:45 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:55:46.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:55:45 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:55:46.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:55:45 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:55:46.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:55:46 smithi083 ceph-mon[79930]: pgmap v9825: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:46.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:55:46 smithi083 ceph-mon[77242]: pgmap v9825: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:47.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:55:46 smithi153 ceph-mon[70044]: pgmap v9825: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:48.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:55:48 smithi083 ceph-mon[79930]: pgmap v9826: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:48.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:55:48 smithi083 ceph-mon[77242]: pgmap v9826: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:49.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:55:48 smithi153 ceph-mon[70044]: pgmap v9826: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:50.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:55:49 smithi153 ceph-mon[70044]: pgmap v9827: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:50.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:55:50 smithi083 ceph-mon[79930]: pgmap v9827: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:50.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:55:49 smithi083 ceph-mon[77242]: pgmap v9827: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:51.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:55:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:55:51] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T10:55:51.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:55:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:55:51.369Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:55:52.685 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:55:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:55:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:55:52.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:55:52 smithi083 ceph-mon[79930]: pgmap v9828: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:52.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:55:52 smithi083 ceph-mon[77242]: pgmap v9828: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:53.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:55:52 smithi153 ceph-mon[70044]: pgmap v9828: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:55:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:55:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:55:54.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:55:54 smithi083 ceph-mon[79930]: pgmap v9829: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:54.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:55:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:55:54.745Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:55:54.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:55:54 smithi083 ceph-mon[77242]: pgmap v9829: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:55.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:55:54 smithi153 ceph-mon[70044]: pgmap v9829: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:55.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:55:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:55:55.456Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:55:56.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:55:56 smithi083 ceph-mon[79930]: pgmap v9830: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:56.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:55:56 smithi083 ceph-mon[77242]: pgmap v9830: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:57.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:55:56 smithi153 ceph-mon[70044]: pgmap v9830: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:58.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:55:58 smithi083 ceph-mon[79930]: pgmap v9831: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:58.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:55:58 smithi083 ceph-mon[77242]: pgmap v9831: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:55:59.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:55:58 smithi153 ceph-mon[70044]: pgmap v9831: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:56:00 smithi153 ceph-mon[70044]: pgmap v9832: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:00.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:56:00 smithi083 ceph-mon[79930]: pgmap v9832: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:00.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:56:00 smithi083 ceph-mon[77242]: pgmap v9832: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:01.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:56:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:56:01] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T10:56:01.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:56:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:56:01.368Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:56:02.688 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:56:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:56:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:56:02.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:56:02 smithi083 ceph-mon[79930]: pgmap v9833: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:02.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:56:02 smithi083 ceph-mon[77242]: pgmap v9833: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:03.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:56:02 smithi153 ceph-mon[70044]: pgmap v9833: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:56:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:56:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:56:04.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:56:04 smithi083 ceph-mon[77242]: pgmap v9834: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:04.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:56:04 smithi083 ceph-mon[79930]: pgmap v9834: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:04.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:56:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:56:04.745Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:56:05.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:56:04 smithi153 ceph-mon[70044]: pgmap v9834: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:05.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:56:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:56:05.457Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:56:06.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:56:06 smithi083 ceph-mon[79930]: pgmap v9835: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:06.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:56:06 smithi083 ceph-mon[77242]: pgmap v9835: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:07.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:56:06 smithi153 ceph-mon[70044]: pgmap v9835: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:07.839 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:56:08.128 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:56:08.129 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 4m ago 5h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:56:08.129 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 2m ago 5h 63.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:56:08.129 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 4m ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:56:08.130 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 2m ago 5h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:56:08.130 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 2m ago 5h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:56:08.130 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 4m ago 5h 791M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:56:08.130 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 4m ago 5h 1235M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:56:08.130 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 2m ago 5h 532M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:56:08.131 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 4m ago 5h 709M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:56:08.131 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 4m ago 5h 26.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:56:08.131 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 2m ago 5h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:56:08.131 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 4m ago 5h 870M 2456M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:56:08.132 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 4m ago 5h 860M 2456M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:56:08.132 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 4m ago 5h 671M 2456M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:56:08.132 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 4m ago 5h 839M 2456M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:56:08.132 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 2m ago 5h 740M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:56:08.132 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 2m ago 5h 922M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:56:08.133 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 2m ago 5h 823M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:56:08.133 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 2m ago 5h 975M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:56:08.133 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 2m ago 5h 191M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:56:08.481 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:56:08.482 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:56:08.482 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:56:08.482 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:56:08.482 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:56:08.483 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:56:08.483 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:56:08.483 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:56:08.483 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:56:08.483 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:56:08.484 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:56:08.484 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:56:08.484 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:56:08.484 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:56:08.485 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:56:08.485 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:56:08.485 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:56:08.485 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:56:08.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:56:08 smithi083 ceph-mon[79930]: pgmap v9836: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:08.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:56:08 smithi083 ceph-mon[79930]: from='client.55504 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:56:08.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:56:08 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3983636812' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:56:08.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:56:08 smithi083 ceph-mon[77242]: pgmap v9836: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:08.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:56:08 smithi083 ceph-mon[77242]: from='client.55504 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:56:08.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:56:08 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3983636812' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:56:09.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:56:08 smithi153 ceph-mon[70044]: pgmap v9836: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:09.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:56:08 smithi153 ceph-mon[70044]: from='client.55504 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:56:09.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:56:08 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3983636812' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:56:09.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:56:09 smithi083 ceph-mon[79930]: from='client.45579 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:56:09.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:56:09 smithi083 ceph-mon[77242]: from='client.45579 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:56:10.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:56:09 smithi153 ceph-mon[70044]: from='client.45579 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:56:10.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:56:10 smithi083 ceph-mon[79930]: pgmap v9837: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:10.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:56:10 smithi083 ceph-mon[77242]: pgmap v9837: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:11.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:56:10 smithi153 ceph-mon[70044]: pgmap v9837: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:11.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:56:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:56:11] "GET /metrics HTTP/1.1" 200 225229 "" "Prometheus/2.33.4" 2022-09-29T10:56:11.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:56:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:56:11.369Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:56:12.702 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:56:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:56:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:56:12.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:56:12 smithi083 ceph-mon[79930]: pgmap v9838: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:12.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:56:12 smithi083 ceph-mon[77242]: pgmap v9838: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:13.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:56:12 smithi153 ceph-mon[70044]: pgmap v9838: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:56:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:56:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:56:14.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:56:14 smithi083 ceph-mon[79930]: pgmap v9839: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:14.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:56:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:56:14.746Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:56:14.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:56:14 smithi083 ceph-mon[77242]: pgmap v9839: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:15.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:56:14 smithi153 ceph-mon[70044]: pgmap v9839: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:15.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:56:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:56:15.458Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:56:16.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:56:16 smithi083 ceph-mon[79930]: pgmap v9840: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:16.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:56:16 smithi083 ceph-mon[77242]: pgmap v9840: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:17.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:56:16 smithi153 ceph-mon[70044]: pgmap v9840: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:18.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:56:18 smithi083 ceph-mon[77242]: pgmap v9841: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:18.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:56:18 smithi083 ceph-mon[79930]: pgmap v9841: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:19.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:56:18 smithi153 ceph-mon[70044]: pgmap v9841: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:56:20 smithi153 ceph-mon[70044]: pgmap v9842: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:20.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:56:20 smithi083 ceph-mon[79930]: pgmap v9842: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:20.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:56:20 smithi083 ceph-mon[77242]: pgmap v9842: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:21.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:56:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:56:21] "GET /metrics HTTP/1.1" 200 225229 "" "Prometheus/2.33.4" 2022-09-29T10:56:21.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:56:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:56:21.370Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:56:22.694 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:56:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:56:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:56:22.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:56:22 smithi083 ceph-mon[77242]: pgmap v9843: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:22.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:56:22 smithi083 ceph-mon[79930]: pgmap v9843: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:23.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:56:22 smithi153 ceph-mon[70044]: pgmap v9843: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:56:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:56:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:56:24.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:56:24 smithi083 ceph-mon[77242]: pgmap v9844: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:24.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:56:24 smithi083 ceph-mon[79930]: pgmap v9844: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:24.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:56:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:56:24.747Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:56:25.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:56:24 smithi153 ceph-mon[70044]: pgmap v9844: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:25.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:56:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:56:25.459Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:56:26.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:56:26 smithi083 ceph-mon[79930]: pgmap v9845: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:26.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:56:26 smithi083 ceph-mon[77242]: pgmap v9845: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:27.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:56:26 smithi153 ceph-mon[70044]: pgmap v9845: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:28.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:56:28 smithi083 ceph-mon[79930]: pgmap v9846: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:28.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:56:28 smithi083 ceph-mon[77242]: pgmap v9846: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:29.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:56:28 smithi153 ceph-mon[70044]: pgmap v9846: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:30.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:56:30 smithi153 ceph-mon[70044]: pgmap v9847: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:30.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:56:30 smithi083 ceph-mon[79930]: pgmap v9847: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:30.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:56:30 smithi083 ceph-mon[77242]: pgmap v9847: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:31.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:56:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:56:31] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T10:56:31.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:56:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:56:31.371Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:56:32.697 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:56:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:56:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:56:32.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:56:32 smithi083 ceph-mon[79930]: pgmap v9848: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:32.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:56:32 smithi083 ceph-mon[77242]: pgmap v9848: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:33.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:56:32 smithi153 ceph-mon[70044]: pgmap v9848: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:33.311 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:56:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:56:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:56:34.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:56:34 smithi083 ceph-mon[79930]: pgmap v9849: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:34.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:56:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:56:34.747Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:56:34.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:56:34 smithi083 ceph-mon[77242]: pgmap v9849: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:35.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:56:34 smithi153 ceph-mon[70044]: pgmap v9849: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:35.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:56:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:56:35.459Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:56:36.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:56:36 smithi083 ceph-mon[79930]: pgmap v9850: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:36.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:56:36 smithi083 ceph-mon[77242]: pgmap v9850: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:37.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:56:36 smithi153 ceph-mon[70044]: pgmap v9850: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:38.809 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:56:38.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:56:38 smithi083 ceph-mon[79930]: pgmap v9851: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:38.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:56:38 smithi083 ceph-mon[77242]: pgmap v9851: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:39.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:56:38 smithi153 ceph-mon[70044]: pgmap v9851: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:39.102 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:56:39.102 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 5m ago 5h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:56:39.102 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 2m ago 5h 63.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:56:39.102 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 5m ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:56:39.103 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 2m ago 5h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:56:39.103 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 2m ago 5h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:56:39.103 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 5m ago 5h 791M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:56:39.103 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 5m ago 5h 1235M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:56:39.103 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 2m ago 5h 532M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:56:39.104 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 5m ago 5h 709M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:56:39.104 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 5m ago 5h 26.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:56:39.104 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 2m ago 5h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:56:39.104 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 5m ago 5h 870M 2456M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:56:39.104 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 5m ago 5h 860M 2456M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:56:39.105 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 5m ago 5h 671M 2456M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:56:39.105 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 5m ago 5h 839M 2456M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:56:39.105 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 2m ago 5h 740M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:56:39.105 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 2m ago 5h 922M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:56:39.105 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 2m ago 5h 823M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:56:39.106 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 2m ago 5h 975M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:56:39.106 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 2m ago 5h 191M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:56:39.456 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:56:39.456 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:56:39.457 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:56:39.457 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:56:39.457 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:56:39.457 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:56:39.457 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:56:39.458 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:56:39.458 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:56:39.458 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:56:39.458 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:56:39.458 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:56:39.459 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:56:39.459 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:56:39.459 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:56:39.459 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:56:39.460 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:56:39.460 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:56:39.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:56:39 smithi083 ceph-mon[79930]: from='client.55522 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:56:39.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:56:39 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1193203075' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:56:39.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:56:39 smithi083 ceph-mon[77242]: from='client.55522 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:56:39.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:56:39 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1193203075' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:56:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:56:39 smithi153 ceph-mon[70044]: from='client.55522 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:56:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:56:39 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1193203075' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:56:40.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:56:40 smithi083 ceph-mon[79930]: from='client.45597 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:56:40.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:56:40 smithi083 ceph-mon[79930]: pgmap v9852: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:56:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:56:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:56:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:56:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:56:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:56:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:56:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:56:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:56:40 smithi083 ceph-mon[77242]: from='client.45597 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:56:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:56:40 smithi083 ceph-mon[77242]: pgmap v9852: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:56:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:56:40.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:56:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:56:40.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:56:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:56:40.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:56:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:56:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:56:40 smithi153 ceph-mon[70044]: from='client.45597 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:56:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:56:40 smithi153 ceph-mon[70044]: pgmap v9852: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:56:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:56:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:56:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:56:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:56:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:56:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:56:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:56:41.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:56:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:56:41] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T10:56:41.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:56:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:56:41.372Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:56:42.711 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:56:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:56:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:56:42.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:56:42 smithi083 ceph-mon[77242]: pgmap v9853: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:42.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:56:42 smithi083 ceph-mon[79930]: pgmap v9853: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:43.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:56:42 smithi153 ceph-mon[70044]: pgmap v9853: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:56:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:56:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:56:45.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:56:44 smithi153 ceph-mon[70044]: pgmap v9854: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:45.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:56:44 smithi083 ceph-mon[79930]: pgmap v9854: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:45.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:56:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:56:44.748Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:56:45.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:56:44 smithi083 ceph-mon[77242]: pgmap v9854: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:45.718 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:56:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:56:45.460Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:56:46.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:56:45 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:56:46.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:56:45 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:56:46.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:56:45 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:56:46.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:56:45 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:56:46.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:56:45 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:56:46.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:56:45 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:56:46.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:56:45 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:56:46.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:56:45 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:56:46.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:56:45 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:56:47.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:56:46 smithi153 ceph-mon[70044]: pgmap v9855: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:47.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:56:46 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:56:47.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:56:46 smithi083 ceph-mon[79930]: pgmap v9855: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:47.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:56:46 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:56:47.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:56:46 smithi083 ceph-mon[77242]: pgmap v9855: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:47.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:56:46 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:56:49.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:56:48 smithi153 ceph-mon[70044]: pgmap v9856: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:49.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:56:48 smithi083 ceph-mon[79930]: pgmap v9856: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:49.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:56:48 smithi083 ceph-mon[77242]: pgmap v9856: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:50.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:56:50 smithi153 ceph-mon[70044]: pgmap v9857: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:50.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:56:50 smithi083 ceph-mon[79930]: pgmap v9857: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:50.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:56:50 smithi083 ceph-mon[77242]: pgmap v9857: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:51.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:56:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:56:51] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T10:56:51.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:56:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:56:51.373Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:56:52.704 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:56:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:56:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:56:52.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:56:52 smithi083 ceph-mon[79930]: pgmap v9858: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:52.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:56:52 smithi083 ceph-mon[77242]: pgmap v9858: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:53.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:56:52 smithi153 ceph-mon[70044]: pgmap v9858: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:56:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:56:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:56:54.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:56:54 smithi083 ceph-mon[79930]: pgmap v9859: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:54.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:56:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:56:54.748Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:56:54.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:56:54 smithi083 ceph-mon[77242]: pgmap v9859: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:55.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:56:54 smithi153 ceph-mon[70044]: pgmap v9859: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:55.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:56:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:56:55.461Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:56:57.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:56:56 smithi153 ceph-mon[70044]: pgmap v9860: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:57.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:56:56 smithi083 ceph-mon[79930]: pgmap v9860: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:57.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:56:56 smithi083 ceph-mon[77242]: pgmap v9860: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:59.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:56:58 smithi153 ceph-mon[70044]: pgmap v9861: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:59.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:56:58 smithi083 ceph-mon[79930]: pgmap v9861: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:56:59.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:56:58 smithi083 ceph-mon[77242]: pgmap v9861: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:57:00 smithi153 ceph-mon[70044]: pgmap v9862: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:00.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:57:00 smithi083 ceph-mon[77242]: pgmap v9862: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:00.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:57:00 smithi083 ceph-mon[79930]: pgmap v9862: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:01.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:57:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:57:01] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T10:57:01.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:57:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:57:01.373Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:57:02.708 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:57:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:57:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:57:02.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:57:02 smithi083 ceph-mon[77242]: pgmap v9863: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:02.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:57:02 smithi083 ceph-mon[79930]: pgmap v9863: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:03.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:57:02 smithi153 ceph-mon[70044]: pgmap v9863: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:57:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:57:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:57:04.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:57:04 smithi083 ceph-mon[79930]: pgmap v9864: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:04.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:57:04 smithi083 ceph-mon[77242]: pgmap v9864: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:04.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:57:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:57:04.749Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:57:05.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:57:04 smithi153 ceph-mon[70044]: pgmap v9864: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:05.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:57:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:57:05.460Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:57:07.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:57:06 smithi153 ceph-mon[70044]: pgmap v9865: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:07.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:57:06 smithi083 ceph-mon[77242]: pgmap v9865: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:07.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:57:06 smithi083 ceph-mon[79930]: pgmap v9865: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:09.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:57:08 smithi153 ceph-mon[70044]: pgmap v9866: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:09.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:57:08 smithi083 ceph-mon[79930]: pgmap v9866: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:09.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:57:08 smithi083 ceph-mon[77242]: pgmap v9866: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:09.779 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:57:10.072 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:57:10.073 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 5m ago 5h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:57:10.073 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 3m ago 5h 63.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:57:10.073 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 5m ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:57:10.073 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 3m ago 5h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:57:10.074 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 3m ago 5h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:57:10.074 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 5m ago 5h 791M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:57:10.074 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 5m ago 5h 1235M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:57:10.075 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 3m ago 5h 532M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:57:10.075 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 5m ago 5h 709M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:57:10.075 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 5m ago 5h 26.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:57:10.075 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 3m ago 5h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:57:10.075 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 5m ago 5h 870M 2456M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:57:10.076 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 5m ago 5h 860M 2456M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:57:10.076 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 5m ago 5h 671M 2456M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:57:10.076 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 5m ago 5h 839M 2456M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:57:10.076 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 3m ago 5h 740M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:57:10.077 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 3m ago 5h 922M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:57:10.077 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 3m ago 5h 823M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:57:10.077 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 3m ago 5h 975M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:57:10.077 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 3m ago 5h 191M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:57:10.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:57:10 smithi153 ceph-mon[70044]: pgmap v9867: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:10.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:57:10 smithi153 ceph-mon[70044]: from='client.55540 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:57:10.428 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:57:10.428 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:57:10.428 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:57:10.429 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:57:10.429 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:57:10.429 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:57:10.429 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:57:10.429 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:57:10.430 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:57:10.430 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:57:10.430 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:57:10.430 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:57:10.431 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:57:10.431 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:57:10.431 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:57:10.431 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:57:10.431 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:57:10.432 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:57:10.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:57:10 smithi083 ceph-mon[77242]: pgmap v9867: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:10.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:57:10 smithi083 ceph-mon[77242]: from='client.55540 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:57:10.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:57:10 smithi083 ceph-mon[79930]: pgmap v9867: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:10.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:57:10 smithi083 ceph-mon[79930]: from='client.55540 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:57:11.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:57:11 smithi153 ceph-mon[70044]: from='client.45615 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:57:11.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:57:11 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3756639281' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:57:11.374 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:57:11 smithi083 ceph-mon[79930]: from='client.45615 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:57:11.374 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:57:11 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3756639281' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:57:11.375 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:57:11 smithi083 ceph-mon[77242]: from='client.45615 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:57:11.375 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:57:11 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3756639281' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:57:11.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:57:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:57:11] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T10:57:11.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:57:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:57:11.373Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:57:12.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:57:12 smithi153 ceph-mon[70044]: pgmap v9868: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:12.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:57:12 smithi083 ceph-mon[77242]: pgmap v9868: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:12.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:57:12 smithi083 ceph-mon[79930]: pgmap v9868: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:57:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:57:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:57:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:57:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:57:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:57:14.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:57:14 smithi083 ceph-mon[77242]: pgmap v9869: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:14.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:57:14 smithi083 ceph-mon[79930]: pgmap v9869: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:14.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:57:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:57:14.750Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:57:15.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:57:14 smithi153 ceph-mon[70044]: pgmap v9869: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:15.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:57:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:57:15.461Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:57:17.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:57:16 smithi153 ceph-mon[70044]: pgmap v9870: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:17.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:57:16 smithi083 ceph-mon[79930]: pgmap v9870: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:17.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:57:16 smithi083 ceph-mon[77242]: pgmap v9870: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:19.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:57:18 smithi153 ceph-mon[70044]: pgmap v9871: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:19.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:57:18 smithi083 ceph-mon[79930]: pgmap v9871: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:19.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:57:18 smithi083 ceph-mon[77242]: pgmap v9871: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:57:20 smithi153 ceph-mon[70044]: pgmap v9872: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:20.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:57:20 smithi083 ceph-mon[79930]: pgmap v9872: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:20.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:57:20 smithi083 ceph-mon[77242]: pgmap v9872: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:21.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:57:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:57:21] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T10:57:21.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:57:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:57:21.374Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:57:22.714 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:57:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:57:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:57:23.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:57:22 smithi153 ceph-mon[70044]: pgmap v9873: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:23.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:57:22 smithi083 ceph-mon[79930]: pgmap v9873: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:23.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:57:22 smithi083 ceph-mon[77242]: pgmap v9873: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:57:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:57:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:57:25.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:57:24 smithi153 ceph-mon[70044]: pgmap v9874: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:25.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:57:24 smithi083 ceph-mon[77242]: pgmap v9874: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:25.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:57:24 smithi083 ceph-mon[79930]: pgmap v9874: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:25.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:57:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:57:24.750Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:57:25.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:57:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:57:25.462Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:57:27.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:57:26 smithi153 ceph-mon[70044]: pgmap v9875: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:27.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:57:26 smithi083 ceph-mon[79930]: pgmap v9875: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:27.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:57:26 smithi083 ceph-mon[77242]: pgmap v9875: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:29.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:57:28 smithi153 ceph-mon[70044]: pgmap v9876: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:29.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:57:28 smithi083 ceph-mon[79930]: pgmap v9876: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:29.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:57:28 smithi083 ceph-mon[77242]: pgmap v9876: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:30.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:57:30 smithi153 ceph-mon[70044]: pgmap v9877: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:30.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:57:30 smithi083 ceph-mon[79930]: pgmap v9877: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:30.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:57:30 smithi083 ceph-mon[77242]: pgmap v9877: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:31.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:57:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:57:31] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T10:57:31.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:57:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:57:31.375Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:57:32.717 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:57:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:57:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:57:33.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:57:32 smithi153 ceph-mon[70044]: pgmap v9878: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:33.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:57:32 smithi083 ceph-mon[77242]: pgmap v9878: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:33.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:57:32 smithi083 ceph-mon[79930]: pgmap v9878: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:57:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:57:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:57:35.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:57:34 smithi153 ceph-mon[70044]: pgmap v9879: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:35.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:57:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:57:34.751Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:57:35.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:57:34 smithi083 ceph-mon[77242]: pgmap v9879: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:35.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:57:34 smithi083 ceph-mon[79930]: pgmap v9879: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:35.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:57:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:57:35.463Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:57:37.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:57:36 smithi153 ceph-mon[70044]: pgmap v9880: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:37.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:57:36 smithi083 ceph-mon[79930]: pgmap v9880: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:37.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:57:36 smithi083 ceph-mon[77242]: pgmap v9880: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:39.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:57:38 smithi153 ceph-mon[70044]: pgmap v9881: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:39.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:57:38 smithi083 ceph-mon[79930]: pgmap v9881: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:39.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:57:38 smithi083 ceph-mon[77242]: pgmap v9881: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:57:40 smithi153 ceph-mon[70044]: pgmap v9882: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:40.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:57:40 smithi083 ceph-mon[79930]: pgmap v9882: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:40.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:57:40 smithi083 ceph-mon[77242]: pgmap v9882: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:40.753 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:57:41.041 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:57:41.041 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 6m ago 5h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:57:41.042 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 3m ago 5h 63.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:57:41.042 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 6m ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:57:41.042 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 3m ago 5h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:57:41.043 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 3m ago 5h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:57:41.043 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 6m ago 5h 791M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:57:41.043 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 6m ago 5h 1235M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:57:41.043 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 3m ago 5h 532M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:57:41.043 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 6m ago 5h 709M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:57:41.044 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 6m ago 5h 26.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:57:41.044 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 3m ago 5h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:57:41.044 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 6m ago 5h 870M 2456M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:57:41.044 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 6m ago 5h 860M 2456M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:57:41.045 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 6m ago 5h 671M 2456M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:57:41.045 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 6m ago 5h 839M 2456M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:57:41.045 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 3m ago 5h 740M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:57:41.045 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 3m ago 5h 922M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:57:41.046 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 3m ago 5h 823M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:57:41.046 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 3m ago 5h 975M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:57:41.046 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 3m ago 5h 191M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:57:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:57:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:57:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:57:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:57:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:57:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:57:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:57:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:57:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:57:41 smithi153 ceph-mon[70044]: from='client.45627 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:57:41.375 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:57:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:57:41.376 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:57:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:57:41.376 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:57:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:57:41.376 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:57:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:57:41.376 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:57:41 smithi083 ceph-mon[79930]: from='client.45627 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:57:41.377 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:57:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:57:41.377 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:57:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:57:41.377 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:57:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:57:41.377 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:57:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:57:41.377 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:57:41 smithi083 ceph-mon[77242]: from='client.45627 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:57:41.401 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:57:41.402 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:57:41.402 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:57:41.402 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:57:41.402 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:57:41.403 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:57:41.403 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:57:41.403 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:57:41.403 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:57:41.403 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:57:41.404 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:57:41.404 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:57:41.404 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:57:41.404 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:57:41.404 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:57:41.405 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:57:41.405 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:57:41.405 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:57:41.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:57:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:57:41] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T10:57:41.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:57:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:57:41.375Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:57:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:57:42 smithi153 ceph-mon[70044]: from='client.45633 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:57:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:57:42 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1333293012' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:57:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:57:42 smithi153 ceph-mon[70044]: pgmap v9883: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:42.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:57:42 smithi083 ceph-mon[77242]: from='client.45633 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:57:42.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:57:42 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1333293012' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:57:42.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:57:42 smithi083 ceph-mon[77242]: pgmap v9883: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:57:42 smithi083 ceph-mon[79930]: from='client.45633 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:57:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:57:42 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1333293012' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:57:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:57:42 smithi083 ceph-mon[79930]: pgmap v9883: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:57:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:57:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:57:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:57:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:57:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:57:45.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:57:44 smithi153 ceph-mon[70044]: pgmap v9884: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:45.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:57:44 smithi083 ceph-mon[79930]: pgmap v9884: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:45.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:57:44 smithi083 ceph-mon[77242]: pgmap v9884: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:45.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:57:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:57:44.752Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:57:45.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:57:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:57:45.464Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:57:47.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:57:46 smithi153 ceph-mon[70044]: pgmap v9885: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:47.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:57:46 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:57:47.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:57:46 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:57:47.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:57:46 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:57:47.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:57:46 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:57:47.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:57:46 smithi083 ceph-mon[79930]: pgmap v9885: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:47.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:57:46 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:57:47.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:57:46 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:57:47.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:57:46 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:57:47.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:57:46 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:57:47.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:57:46 smithi083 ceph-mon[77242]: pgmap v9885: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:47.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:57:46 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:57:47.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:57:46 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:57:47.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:57:46 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:57:47.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:57:46 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:57:49.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:57:48 smithi153 ceph-mon[70044]: pgmap v9886: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:49.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:57:48 smithi083 ceph-mon[77242]: pgmap v9886: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:49.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:57:48 smithi083 ceph-mon[79930]: pgmap v9886: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:50.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:57:50 smithi153 ceph-mon[70044]: pgmap v9887: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:50.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:57:50 smithi083 ceph-mon[79930]: pgmap v9887: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:50.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:57:50 smithi083 ceph-mon[77242]: pgmap v9887: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:51.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:57:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:57:51] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T10:57:51.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:57:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:57:51.376Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:57:52.724 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:57:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:57:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:57:53.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:57:52 smithi153 ceph-mon[70044]: pgmap v9888: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:53.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:57:52 smithi083 ceph-mon[77242]: pgmap v9888: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:53.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:57:52 smithi083 ceph-mon[79930]: pgmap v9888: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:57:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:57:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:57:55.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:57:54 smithi153 ceph-mon[70044]: pgmap v9889: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:55.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:57:54 smithi083 ceph-mon[77242]: pgmap v9889: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:55.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:57:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:57:54.752Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:57:55.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:57:54 smithi083 ceph-mon[79930]: pgmap v9889: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:55.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:57:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:57:55.464Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:57:57.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:57:56 smithi153 ceph-mon[70044]: pgmap v9890: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:57.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:57:56 smithi083 ceph-mon[79930]: pgmap v9890: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:57.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:57:56 smithi083 ceph-mon[77242]: pgmap v9890: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:59.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:57:58 smithi153 ceph-mon[70044]: pgmap v9891: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:59.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:57:58 smithi083 ceph-mon[79930]: pgmap v9891: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:57:59.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:57:58 smithi083 ceph-mon[77242]: pgmap v9891: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:58:00 smithi153 ceph-mon[70044]: pgmap v9892: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:00.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:58:00 smithi083 ceph-mon[79930]: pgmap v9892: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:00.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:58:00 smithi083 ceph-mon[77242]: pgmap v9892: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:01.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:58:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:58:01] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T10:58:01.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:58:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:58:01.376Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:58:02.727 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:58:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:58:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:58:03.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:58:02 smithi153 ceph-mon[70044]: pgmap v9893: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:03.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:58:02 smithi083 ceph-mon[79930]: pgmap v9893: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:03.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:58:02 smithi083 ceph-mon[77242]: pgmap v9893: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:03.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:58:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:58:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:58:05.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:58:04 smithi153 ceph-mon[70044]: pgmap v9894: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:05.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:58:04 smithi083 ceph-mon[79930]: pgmap v9894: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:05.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:58:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:58:04.753Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:58:05.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:58:04 smithi083 ceph-mon[77242]: pgmap v9894: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:05.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:58:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:58:05.464Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:58:07.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:58:06 smithi153 ceph-mon[70044]: pgmap v9895: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:07.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:58:06 smithi083 ceph-mon[77242]: pgmap v9895: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:07.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:58:06 smithi083 ceph-mon[79930]: pgmap v9895: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:09.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:58:08 smithi153 ceph-mon[70044]: pgmap v9896: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:09.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:58:08 smithi083 ceph-mon[79930]: pgmap v9896: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:09.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:58:08 smithi083 ceph-mon[77242]: pgmap v9896: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:10.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:58:10 smithi153 ceph-mon[70044]: pgmap v9897: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:10.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:58:10 smithi083 ceph-mon[79930]: pgmap v9897: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:10.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:58:10 smithi083 ceph-mon[77242]: pgmap v9897: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:11.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:58:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:58:11] "GET /metrics HTTP/1.1" 200 225229 "" "Prometheus/2.33.4" 2022-09-29T10:58:11.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:58:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:58:11.377Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:58:11.733 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:58:12.021 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:58:12.021 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 6m ago 5h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:58:12.021 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 4m ago 5h 63.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:58:12.022 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 6m ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:58:12.022 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 4m ago 5h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:58:12.022 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 4m ago 5h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:58:12.022 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 6m ago 5h 791M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:58:12.023 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 6m ago 5h 1235M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:58:12.023 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 4m ago 5h 532M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:58:12.023 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 6m ago 5h 709M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:58:12.023 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 6m ago 5h 26.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:58:12.023 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 4m ago 5h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:58:12.024 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 6m ago 5h 870M 2456M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:58:12.024 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 6m ago 5h 860M 2456M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:58:12.024 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 6m ago 5h 671M 2456M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:58:12.024 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 6m ago 5h 839M 2456M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:58:12.024 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 4m ago 5h 740M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:58:12.025 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 4m ago 5h 922M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:58:12.025 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 4m ago 5h 823M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:58:12.025 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 4m ago 5h 975M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:58:12.025 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 4m ago 5h 191M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:58:12.376 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:58:12.377 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:58:12.377 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:58:12.377 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:58:12.377 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:58:12.378 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:58:12.378 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:58:12.378 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:58:12.378 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:58:12.379 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:58:12.379 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:58:12.379 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:58:12.379 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:58:12.380 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:58:12.380 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:58:12.380 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:58:12.380 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:58:12.381 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:58:12.730 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:58:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:58:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:58:13.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:58:12 smithi153 ceph-mon[70044]: pgmap v9898: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:13.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:58:12 smithi153 ceph-mon[70044]: from='client.55576 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:58:13.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:58:12 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1578046462' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:58:13.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:58:12 smithi083 ceph-mon[79930]: pgmap v9898: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:13.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:58:12 smithi083 ceph-mon[79930]: from='client.55576 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:58:13.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:58:12 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1578046462' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:58:13.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:58:12 smithi083 ceph-mon[77242]: pgmap v9898: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:13.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:58:12 smithi083 ceph-mon[77242]: from='client.55576 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:58:13.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:58:12 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1578046462' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:58:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:58:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:58:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:58:14.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:58:13 smithi153 ceph-mon[70044]: from='client.45651 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:58:14.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:58:13 smithi083 ceph-mon[79930]: from='client.45651 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:58:14.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:58:13 smithi083 ceph-mon[77242]: from='client.45651 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:58:15.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:58:14 smithi153 ceph-mon[70044]: pgmap v9899: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:15.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:58:14 smithi083 ceph-mon[79930]: pgmap v9899: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:15.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:58:14 smithi083 ceph-mon[77242]: pgmap v9899: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:15.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:58:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:58:14.753Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:58:15.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:58:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:58:15.465Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:58:17.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:58:16 smithi153 ceph-mon[70044]: pgmap v9900: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:17.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:58:16 smithi083 ceph-mon[79930]: pgmap v9900: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:17.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:58:16 smithi083 ceph-mon[77242]: pgmap v9900: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:19.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:58:18 smithi153 ceph-mon[70044]: pgmap v9901: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:19.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:58:18 smithi083 ceph-mon[77242]: pgmap v9901: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:19.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:58:18 smithi083 ceph-mon[79930]: pgmap v9901: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:58:20 smithi153 ceph-mon[70044]: pgmap v9902: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:20.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:58:20 smithi083 ceph-mon[79930]: pgmap v9902: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:20.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:58:20 smithi083 ceph-mon[77242]: pgmap v9902: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:21.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:58:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:58:21] "GET /metrics HTTP/1.1" 200 225229 "" "Prometheus/2.33.4" 2022-09-29T10:58:21.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:58:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:58:21.378Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:58:22.733 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:58:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:58:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:58:23.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:58:22 smithi153 ceph-mon[70044]: pgmap v9903: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:23.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:58:22 smithi083 ceph-mon[77242]: pgmap v9903: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:23.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:58:22 smithi083 ceph-mon[79930]: pgmap v9903: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:58:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:58:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:58:25.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:58:24 smithi153 ceph-mon[70044]: pgmap v9904: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:25.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:58:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:58:24.753Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:58:25.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:58:24 smithi083 ceph-mon[77242]: pgmap v9904: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:25.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:58:24 smithi083 ceph-mon[79930]: pgmap v9904: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:25.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:58:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:58:25.466Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:58:27.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:58:26 smithi153 ceph-mon[70044]: pgmap v9905: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:27.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:58:26 smithi083 ceph-mon[77242]: pgmap v9905: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:27.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:58:26 smithi083 ceph-mon[79930]: pgmap v9905: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:29.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:58:28 smithi153 ceph-mon[70044]: pgmap v9906: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:29.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:58:28 smithi083 ceph-mon[77242]: pgmap v9906: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:29.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:58:28 smithi083 ceph-mon[79930]: pgmap v9906: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:30.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:58:30 smithi153 ceph-mon[70044]: pgmap v9907: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:30.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:58:30 smithi083 ceph-mon[77242]: pgmap v9907: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:30.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:58:30 smithi083 ceph-mon[79930]: pgmap v9907: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:31.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:58:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:58:31] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T10:58:31.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:58:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:58:31.378Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:58:32.736 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:58:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:58:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:58:33.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:58:32 smithi153 ceph-mon[70044]: pgmap v9908: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:33.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:58:32 smithi083 ceph-mon[77242]: pgmap v9908: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:33.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:58:32 smithi083 ceph-mon[79930]: pgmap v9908: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:58:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:58:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:58:35.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:58:34 smithi153 ceph-mon[70044]: pgmap v9909: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:35.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:58:34 smithi083 ceph-mon[79930]: pgmap v9909: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:35.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:58:34 smithi083 ceph-mon[77242]: pgmap v9909: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:35.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:58:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:58:34.754Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:58:35.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:58:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:58:35.466Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:58:37.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:58:36 smithi153 ceph-mon[70044]: pgmap v9910: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:37.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:58:36 smithi083 ceph-mon[77242]: pgmap v9910: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:37.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:58:36 smithi083 ceph-mon[79930]: pgmap v9910: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:39.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:58:38 smithi153 ceph-mon[70044]: pgmap v9911: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:39.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:58:38 smithi083 ceph-mon[79930]: pgmap v9911: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:39.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:58:38 smithi083 ceph-mon[77242]: pgmap v9911: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:58:40 smithi153 ceph-mon[70044]: pgmap v9912: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:40.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:58:40 smithi083 ceph-mon[79930]: pgmap v9912: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:40.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:58:40 smithi083 ceph-mon[77242]: pgmap v9912: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:58:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:58:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:58:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:58:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:58:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:58:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:58:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:58:41.380 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:58:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:58:41.380 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:58:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:58:41.380 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:58:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:58:41.380 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:58:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:58:41.381 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:58:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:58:41.381 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:58:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:58:41.381 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:58:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:58:41.381 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:58:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:58:41.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:58:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:58:41] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T10:58:41.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:58:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:58:41.379Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:58:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:58:42 smithi153 ceph-mon[70044]: pgmap v9913: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:42.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:58:42 smithi083 ceph-mon[79930]: pgmap v9913: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:42.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:58:42 smithi083 ceph-mon[77242]: pgmap v9913: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:42.702 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:58:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:58:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:58:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:58:42.991 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:58:42.992 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 7m ago 5h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:58:42.992 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 4m ago 5h 63.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:58:42.992 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 7m ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:58:42.992 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 4m ago 5h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:58:42.993 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 4m ago 5h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:58:42.993 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 7m ago 5h 791M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:58:42.993 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 7m ago 5h 1235M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:58:42.993 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 4m ago 5h 532M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:58:42.993 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 7m ago 5h 709M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:58:42.994 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 7m ago 5h 26.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:58:42.994 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 4m ago 5h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:58:42.994 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 7m ago 5h 870M 2456M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:58:42.994 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 7m ago 5h 860M 2456M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:58:42.995 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 7m ago 5h 671M 2456M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:58:42.995 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 7m ago 5h 839M 2456M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:58:42.995 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 4m ago 5h 740M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:58:42.995 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 4m ago 5h 922M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:58:42.995 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 4m ago 5h 823M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:58:42.996 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 4m ago 5h 975M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:58:42.996 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 4m ago 5h 191M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:58:43.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:58:43 smithi153 ceph-mon[70044]: from='client.55594 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:58:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:58:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:58:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:58:43.352 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:58:43.352 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:58:43.352 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:58:43.352 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:58:43.353 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:58:43.353 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:58:43.353 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:58:43.353 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:58:43.354 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:58:43.354 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:58:43.354 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:58:43.354 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:58:43.355 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:58:43.355 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:58:43.355 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:58:43.355 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:58:43.355 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:58:43.356 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:58:43.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:58:43 smithi083 ceph-mon[77242]: from='client.55594 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:58:43.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:58:43 smithi083 ceph-mon[79930]: from='client.55594 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:58:44.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:58:44 smithi153 ceph-mon[70044]: from='client.45669 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:58:44.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:58:44 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1025699426' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:58:44.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:58:44 smithi153 ceph-mon[70044]: pgmap v9914: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:44.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:58:44 smithi083 ceph-mon[79930]: from='client.45669 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:58:44.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:58:44 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1025699426' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:58:44.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:58:44 smithi083 ceph-mon[79930]: pgmap v9914: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:44.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:58:44 smithi083 ceph-mon[77242]: from='client.45669 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:58:44.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:58:44 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1025699426' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:58:44.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:58:44 smithi083 ceph-mon[77242]: pgmap v9914: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:45.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:58:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:58:44.754Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:58:45.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:58:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:58:45.468Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:58:47.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:58:46 smithi153 ceph-mon[70044]: pgmap v9915: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:47.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:58:46 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:58:47.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:58:46 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:58:47.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:58:46 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:58:47.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:58:46 smithi083 ceph-mon[79930]: pgmap v9915: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:47.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:58:46 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:58:47.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:58:46 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:58:47.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:58:46 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:58:47.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:58:46 smithi083 ceph-mon[77242]: pgmap v9915: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:47.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:58:46 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:58:47.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:58:46 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:58:47.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:58:46 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:58:48.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:58:48 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:58:48.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:58:48 smithi153 ceph-mon[70044]: pgmap v9916: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:48.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:58:48 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:58:48.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:58:48 smithi083 ceph-mon[79930]: pgmap v9916: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:48.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:58:48 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:58:48.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:58:48 smithi083 ceph-mon[77242]: pgmap v9916: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:50.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:58:50 smithi153 ceph-mon[70044]: pgmap v9917: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:50.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:58:50 smithi083 ceph-mon[77242]: pgmap v9917: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:50.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:58:50 smithi083 ceph-mon[79930]: pgmap v9917: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:51.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:58:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:58:51] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T10:58:51.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:58:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:58:51.379Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:58:52.742 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:58:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:58:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:58:53.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:58:52 smithi153 ceph-mon[70044]: pgmap v9918: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:53.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:58:52 smithi083 ceph-mon[79930]: pgmap v9918: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:53.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:58:52 smithi083 ceph-mon[77242]: pgmap v9918: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:58:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:58:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:58:55.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:58:54 smithi153 ceph-mon[70044]: pgmap v9919: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:55.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:58:54 smithi083 ceph-mon[79930]: pgmap v9919: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:55.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:58:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:58:54.755Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:58:55.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:58:54 smithi083 ceph-mon[77242]: pgmap v9919: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:55.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:58:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:58:55.468Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:58:57.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:58:56 smithi153 ceph-mon[70044]: pgmap v9920: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:57.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:58:56 smithi083 ceph-mon[77242]: pgmap v9920: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:57.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:58:56 smithi083 ceph-mon[79930]: pgmap v9920: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:59.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:58:58 smithi153 ceph-mon[70044]: pgmap v9921: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:59.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:58:58 smithi083 ceph-mon[79930]: pgmap v9921: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:58:59.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:58:58 smithi083 ceph-mon[77242]: pgmap v9921: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:59:00 smithi153 ceph-mon[70044]: pgmap v9922: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:00.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:59:00 smithi083 ceph-mon[79930]: pgmap v9922: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:00.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:59:00 smithi083 ceph-mon[77242]: pgmap v9922: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:01.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:59:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:59:01] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T10:59:01.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:59:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:59:01.380Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:59:02.746 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:59:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:59:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:59:03.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:59:02 smithi153 ceph-mon[70044]: pgmap v9923: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:03.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:59:02 smithi083 ceph-mon[79930]: pgmap v9923: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:03.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:59:02 smithi083 ceph-mon[77242]: pgmap v9923: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:59:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:59:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:59:05.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:59:04 smithi153 ceph-mon[70044]: pgmap v9924: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:05.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:59:04 smithi083 ceph-mon[79930]: pgmap v9924: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:05.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:59:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:59:04.756Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:59:05.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:59:04 smithi083 ceph-mon[77242]: pgmap v9924: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:05.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:59:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:59:05.469Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:59:07.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:59:06 smithi153 ceph-mon[70044]: pgmap v9925: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:07.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:59:06 smithi083 ceph-mon[79930]: pgmap v9925: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:07.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:59:06 smithi083 ceph-mon[77242]: pgmap v9925: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:09.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:59:08 smithi153 ceph-mon[70044]: pgmap v9926: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:09.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:59:08 smithi083 ceph-mon[79930]: pgmap v9926: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:09.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:59:08 smithi083 ceph-mon[77242]: pgmap v9926: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:10.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:59:10 smithi153 ceph-mon[70044]: pgmap v9927: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:10.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:59:10 smithi083 ceph-mon[79930]: pgmap v9927: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:10.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:59:10 smithi083 ceph-mon[77242]: pgmap v9927: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:11.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:59:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:59:11.381Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:59:11.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:59:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:59:11] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T10:59:12.748 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:59:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:59:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:59:13.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:59:12 smithi153 ceph-mon[70044]: pgmap v9928: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:13.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:59:12 smithi083 ceph-mon[79930]: pgmap v9928: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:13.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:59:12 smithi083 ceph-mon[77242]: pgmap v9928: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:59:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:59:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:59:13.683 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:59:13.974 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:59:13.975 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 7m ago 5h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:59:13.975 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 5m ago 5h 63.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:59:13.975 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 7m ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:59:13.976 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 5m ago 5h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:59:13.976 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 5m ago 5h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:59:13.976 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 7m ago 5h 791M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:59:13.976 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 7m ago 5h 1235M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:59:13.976 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 5m ago 5h 532M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:59:13.977 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 7m ago 5h 709M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:59:13.977 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 7m ago 5h 26.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:59:13.977 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 5m ago 5h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:59:13.977 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 7m ago 5h 870M 2456M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:59:13.977 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 7m ago 5h 860M 2456M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:59:13.978 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 7m ago 5h 671M 2456M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:59:13.978 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 7m ago 5h 839M 2456M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:59:13.978 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 5m ago 5h 740M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:59:13.978 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 5m ago 5h 922M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:59:13.979 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 5m ago 5h 823M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:59:13.979 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 5m ago 5h 975M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:59:13.979 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 5m ago 5h 191M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:59:14.331 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:59:14.332 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:59:14.332 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:59:14.332 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:59:14.332 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:59:14.333 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:59:14.333 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:59:14.333 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:59:14.333 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:59:14.334 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:59:14.334 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:59:14.334 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:59:14.334 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:59:14.334 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:59:14.335 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:59:14.335 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:59:14.335 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:59:14.335 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:59:15.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:59:14 smithi153 ceph-mon[70044]: from='client.45681 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:59:15.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:59:14 smithi153 ceph-mon[70044]: pgmap v9929: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:15.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:59:14 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/556456740' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:59:15.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:59:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:59:14.756Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:59:15.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:59:14 smithi083 ceph-mon[77242]: from='client.45681 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:59:15.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:59:14 smithi083 ceph-mon[77242]: pgmap v9929: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:15.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:59:14 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/556456740' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:59:15.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:59:14 smithi083 ceph-mon[79930]: from='client.45681 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:59:15.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:59:14 smithi083 ceph-mon[79930]: pgmap v9929: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:15.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:59:14 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/556456740' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:59:15.756 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:59:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:59:15.469Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:59:16.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:59:15 smithi153 ceph-mon[70044]: from='client.55618 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:59:16.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:59:15 smithi083 ceph-mon[77242]: from='client.55618 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:59:16.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:59:15 smithi083 ceph-mon[79930]: from='client.55618 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:59:17.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:59:16 smithi153 ceph-mon[70044]: pgmap v9930: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:17.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:59:16 smithi083 ceph-mon[77242]: pgmap v9930: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:17.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:59:16 smithi083 ceph-mon[79930]: pgmap v9930: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:19.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:59:18 smithi153 ceph-mon[70044]: pgmap v9931: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:19.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:59:18 smithi083 ceph-mon[77242]: pgmap v9931: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:19.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:59:18 smithi083 ceph-mon[79930]: pgmap v9931: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:59:20 smithi153 ceph-mon[70044]: pgmap v9932: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:20.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:59:20 smithi083 ceph-mon[77242]: pgmap v9932: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:20.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:59:20 smithi083 ceph-mon[79930]: pgmap v9932: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:21.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:59:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:59:21] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T10:59:21.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:59:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:59:21.382Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:59:22.752 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:59:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:59:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:59:23.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:59:22 smithi153 ceph-mon[70044]: pgmap v9933: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:23.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:59:22 smithi083 ceph-mon[77242]: pgmap v9933: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:23.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:59:22 smithi083 ceph-mon[79930]: pgmap v9933: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:59:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:59:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:59:25.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:59:24 smithi153 ceph-mon[70044]: pgmap v9934: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:25.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:59:24 smithi083 ceph-mon[79930]: pgmap v9934: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:25.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:59:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:59:24.759Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:59:25.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:59:24 smithi083 ceph-mon[77242]: pgmap v9934: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:25.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:59:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:59:25.470Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:59:27.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:59:26 smithi153 ceph-mon[70044]: pgmap v9935: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:27.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:59:26 smithi083 ceph-mon[79930]: pgmap v9935: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:27.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:59:26 smithi083 ceph-mon[77242]: pgmap v9935: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:29.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:59:28 smithi153 ceph-mon[70044]: pgmap v9936: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:29.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:59:28 smithi083 ceph-mon[79930]: pgmap v9936: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:29.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:59:28 smithi083 ceph-mon[77242]: pgmap v9936: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:30.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:59:30 smithi153 ceph-mon[70044]: pgmap v9937: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:30.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:59:30 smithi083 ceph-mon[79930]: pgmap v9937: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:30.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:59:30 smithi083 ceph-mon[77242]: pgmap v9937: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:31.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:59:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:59:31] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T10:59:31.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:59:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:59:31.382Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:59:32.755 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:59:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:59:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:59:33.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:59:32 smithi153 ceph-mon[70044]: pgmap v9938: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:33.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:59:32 smithi083 ceph-mon[79930]: pgmap v9938: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:33.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:59:32 smithi083 ceph-mon[77242]: pgmap v9938: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:59:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:59:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:59:35.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:59:34 smithi153 ceph-mon[70044]: pgmap v9939: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:35.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:59:34 smithi083 ceph-mon[79930]: pgmap v9939: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:35.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:59:34 smithi083 ceph-mon[77242]: pgmap v9939: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:35.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:59:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:59:34.758Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:59:35.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:59:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:59:35.470Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:59:37.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:59:36 smithi153 ceph-mon[70044]: pgmap v9940: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:37.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:59:36 smithi083 ceph-mon[79930]: pgmap v9940: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:37.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:59:36 smithi083 ceph-mon[77242]: pgmap v9940: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:39.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:59:38 smithi153 ceph-mon[70044]: pgmap v9941: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:39.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:59:38 smithi083 ceph-mon[79930]: pgmap v9941: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:39.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:59:38 smithi083 ceph-mon[77242]: pgmap v9941: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:59:40 smithi153 ceph-mon[70044]: pgmap v9942: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:40.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:59:40 smithi083 ceph-mon[79930]: pgmap v9942: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:40.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:59:40 smithi083 ceph-mon[77242]: pgmap v9942: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:59:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:59:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:59:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:59:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:59:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:59:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:59:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:59:41.384 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:59:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:59:41.384 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:59:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:59:41.385 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:59:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:59:41.385 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:59:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:59:41.385 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:59:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:59:41.386 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:59:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:59:41.386 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:59:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T10:59:41.386 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:59:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T10:59:41.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:59:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:59:41] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T10:59:41.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:59:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:59:41.383Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:59:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:59:42 smithi153 ceph-mon[70044]: pgmap v9943: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:42.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:59:42 smithi083 ceph-mon[77242]: pgmap v9943: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:42.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:59:42 smithi083 ceph-mon[79930]: pgmap v9943: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:59:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:59:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:59:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:59:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:59:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:59:44.657 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T10:59:44.947 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T10:59:44.947 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 8m ago 5h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T10:59:44.947 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 6m ago 5h 63.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T10:59:44.948 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 8m ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T10:59:44.948 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 6m ago 5h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T10:59:44.948 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 6m ago 5h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T10:59:44.948 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 8m ago 5h 791M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T10:59:44.948 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 8m ago 5h 1235M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T10:59:44.949 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 6m ago 5h 532M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T10:59:44.949 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 8m ago 5h 709M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T10:59:44.949 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 8m ago 5h 26.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T10:59:44.949 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 6m ago 5h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T10:59:44.949 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 8m ago 5h 870M 2456M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T10:59:44.950 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 8m ago 5h 860M 2456M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T10:59:44.950 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 8m ago 5h 671M 2456M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T10:59:44.950 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 8m ago 5h 839M 2456M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T10:59:44.950 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 6m ago 5h 740M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T10:59:44.951 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 6m ago 5h 922M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T10:59:44.951 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 6m ago 5h 823M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T10:59:44.951 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 6m ago 5h 975M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T10:59:44.951 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 6m ago 5h 191M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T10:59:45.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:59:44 smithi153 ceph-mon[70044]: pgmap v9944: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:45.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:59:44 smithi083 ceph-mon[79930]: pgmap v9944: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:45.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:59:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:59:44.758Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:59:45.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:59:44 smithi083 ceph-mon[77242]: pgmap v9944: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:45.306 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T10:59:45.306 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T10:59:45.306 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T10:59:45.307 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:59:45.307 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T10:59:45.307 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T10:59:45.307 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:59:45.307 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T10:59:45.308 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T10:59:45.308 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:59:45.308 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T10:59:45.308 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T10:59:45.308 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T10:59:45.309 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T10:59:45.309 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T10:59:45.309 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T10:59:45.309 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T10:59:45.309 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T10:59:45.762 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:59:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:59:45.470Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:59:46.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:59:45 smithi153 ceph-mon[70044]: from='client.55630 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:59:46.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:59:45 smithi153 ceph-mon[70044]: from='client.45705 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:59:46.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:59:45 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1701898553' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:59:46.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:59:45 smithi083 ceph-mon[79930]: from='client.55630 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:59:46.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:59:45 smithi083 ceph-mon[79930]: from='client.45705 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:59:46.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:59:45 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1701898553' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:59:46.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:59:45 smithi083 ceph-mon[77242]: from='client.55630 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:59:46.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:59:45 smithi083 ceph-mon[77242]: from='client.45705 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T10:59:46.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:59:45 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1701898553' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T10:59:47.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:59:46 smithi153 ceph-mon[70044]: pgmap v9945: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:47.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:59:46 smithi083 ceph-mon[79930]: pgmap v9945: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:47.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:59:46 smithi083 ceph-mon[77242]: pgmap v9945: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:48.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:59:47 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:59:48.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:59:47 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:59:48.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:59:47 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:59:48.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:59:47 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:59:48.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:59:47 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:59:48.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:59:47 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:59:48.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:59:47 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T10:59:48.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:59:47 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T10:59:48.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:59:47 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T10:59:49.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:59:48 smithi153 ceph-mon[70044]: pgmap v9946: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:49.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:59:48 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:59:49.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:59:48 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:59:49.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:59:48 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:59:49.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:59:48 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:59:49.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:59:48 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:59:49.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:59:48 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:59:49.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:59:48 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:59:49.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:59:48 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:59:49.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:59:48 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:59:49.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:59:48 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:59:49.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:59:48 smithi083 ceph-mon[79930]: pgmap v9946: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:49.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:59:48 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:59:49.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:59:48 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:59:49.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:59:48 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:59:49.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:59:48 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:59:49.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:59:48 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:59:49.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:59:48 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:59:49.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:59:48 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:59:49.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:59:48 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:59:49.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:59:48 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:59:49.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:59:48 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:59:49.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:59:48 smithi083 ceph-mon[77242]: pgmap v9946: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:49.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:59:48 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:59:49.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:59:48 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:59:49.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:59:48 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:59:49.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:59:48 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:59:49.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:59:48 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:59:49.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:59:48 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:59:49.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:59:48 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:59:49.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:59:48 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T10:59:49.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:59:48 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:59:49.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:59:48 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T10:59:50.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:59:49 smithi153 ceph-mon[70044]: Adjusting osd_memory_target on smithi083 to 2447M 2022-09-29T10:59:50.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:59:49 smithi083 ceph-mon[79930]: Adjusting osd_memory_target on smithi083 to 2447M 2022-09-29T10:59:50.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:59:49 smithi083 ceph-mon[77242]: Adjusting osd_memory_target on smithi083 to 2447M 2022-09-29T10:59:51.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:59:50 smithi153 ceph-mon[70044]: pgmap v9947: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:51.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:59:50 smithi083 ceph-mon[79930]: pgmap v9947: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:51.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:59:50 smithi083 ceph-mon[77242]: pgmap v9947: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:51.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 10:59:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:59:51] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T10:59:51.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:59:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:59:51.384Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:59:52.775 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 10:59:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T10:59:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T10:59:53.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:59:52 smithi153 ceph-mon[70044]: pgmap v9948: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:53.058 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 10:59:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:10:59:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T10:59:53.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:59:52 smithi083 ceph-mon[79930]: pgmap v9948: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:53.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:59:52 smithi083 ceph-mon[77242]: pgmap v9948: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:55.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:59:54 smithi153 ceph-mon[70044]: pgmap v9949: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:55.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:59:54 smithi083 ceph-mon[79930]: pgmap v9949: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:55.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:59:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:59:54.758Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:59:55.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:59:54 smithi083 ceph-mon[77242]: pgmap v9949: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:55.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 10:59:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T10:59:55.472Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T10:59:57.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:59:56 smithi153 ceph-mon[70044]: pgmap v9950: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:57.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:59:56 smithi083 ceph-mon[79930]: pgmap v9950: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:57.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:59:56 smithi083 ceph-mon[77242]: pgmap v9950: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:59.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 10:59:58 smithi153 ceph-mon[70044]: pgmap v9951: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:59.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 10:59:58 smithi083 ceph-mon[79930]: pgmap v9951: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T10:59:59.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 10:59:58 smithi083 ceph-mon[77242]: pgmap v9951: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:00:00 smithi153 ceph-mon[70044]: pgmap v9952: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:00:00 smithi153 ceph-mon[70044]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T11:00:00.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:00:00 smithi083 ceph-mon[79930]: pgmap v9952: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:00.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:00:00 smithi083 ceph-mon[79930]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T11:00:00.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:00:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[77219]: 2022-09-29T10:59:59.999+0000 7f160f665700 -1 log_channel(cluster) log [ERR] : overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T11:00:00.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:00:00 smithi083 ceph-mon[77242]: pgmap v9952: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:00.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:00:00 smithi083 ceph-mon[77242]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T11:00:01.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:00:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:00:01] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T11:00:01.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:00:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:00:01.385Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:00:02.057 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:00:01 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:00:01.566Z caller=compact.go:519 level=info component=tsdb msg="write block" mint=1664438400138 maxt=1664445600000 ulid=01GE4EEHBAVN0KVW1PZWGGFA8X duration=51.754396ms 2022-09-29T11:00:02.057 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:00:01 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:00:01.570Z caller=head.go:827 level=info component=tsdb msg="Head GC completed" duration=3.949912ms 2022-09-29T11:00:02.058 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:00:01 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:00:01.571Z caller=checkpoint.go:98 level=info component=tsdb msg="Creating checkpoint" from_segment=2 to_segment=3 mint=1664445600000 2022-09-29T11:00:02.058 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:00:01 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:00:01.693Z caller=head.go:996 level=info component=tsdb msg="WAL checkpoint complete" first=2 last=3 duration=122.5697ms 2022-09-29T11:00:02.764 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:00:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:00:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:00:03.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:00:02 smithi153 ceph-mon[70044]: pgmap v9953: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:03.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:00:02 smithi083 ceph-mon[77242]: pgmap v9953: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:03.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:00:02 smithi083 ceph-mon[79930]: pgmap v9953: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:00:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:00:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:00:05.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:00:04 smithi153 ceph-mon[70044]: pgmap v9954: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:05.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:00:04 smithi083 ceph-mon[79930]: pgmap v9954: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:05.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:00:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:00:04.758Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:00:05.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:00:04 smithi083 ceph-mon[77242]: pgmap v9954: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:05.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:00:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:00:05.473Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:00:07.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:00:06 smithi153 ceph-mon[70044]: pgmap v9955: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:07.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:00:06 smithi083 ceph-mon[79930]: pgmap v9955: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:07.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:00:06 smithi083 ceph-mon[77242]: pgmap v9955: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:09.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:00:08 smithi153 ceph-mon[70044]: pgmap v9956: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:09.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:00:08 smithi083 ceph-mon[79930]: pgmap v9956: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:09.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:00:08 smithi083 ceph-mon[77242]: pgmap v9956: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:10.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:00:10 smithi083 ceph-mon[77242]: pgmap v9957: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:10.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:00:10 smithi083 ceph-mon[79930]: pgmap v9957: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:00:10 smithi153 ceph-mon[70044]: pgmap v9957: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:11.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:00:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:00:11] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T11:00:11.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:00:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:00:11.385Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:00:12.767 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:00:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:00:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:00:13.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:00:12 smithi153 ceph-mon[70044]: pgmap v9958: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:13.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:00:12 smithi083 ceph-mon[77242]: pgmap v9958: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:13.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:00:12 smithi083 ceph-mon[79930]: pgmap v9958: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:00:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:00:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:00:15.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:00:14 smithi153 ceph-mon[70044]: pgmap v9959: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:15.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:00:14 smithi083 ceph-mon[79930]: pgmap v9959: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:15.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:00:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:00:14.758Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:00:15.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:00:14 smithi083 ceph-mon[77242]: pgmap v9959: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:15.634 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:00:15.926 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:00:15.926 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 8m ago 5h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:00:15.926 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 6m ago 5h 63.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:00:15.927 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 8m ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:00:15.927 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 6m ago 5h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:00:15.927 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 6m ago 5h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:00:15.927 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 8m ago 5h 791M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:00:15.928 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 8m ago 5h 1235M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:00:15.928 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 6m ago 5h 532M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:00:15.928 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 8m ago 5h 709M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:00:15.928 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 8m ago 5h 26.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:00:15.929 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 6m ago 5h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:00:15.929 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 8m ago 5h 870M 2447M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:00:15.929 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 8m ago 5h 860M 2447M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:00:15.929 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 8m ago 5h 671M 2447M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:00:15.930 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 8m ago 5h 839M 2447M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:00:15.930 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 6m ago 5h 740M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:00:15.930 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 6m ago 5h 922M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:00:15.930 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 6m ago 5h 823M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:00:15.931 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 6m ago 5h 975M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:00:15.931 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 6m ago 5h 191M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:00:15.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:00:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:00:15.473Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:00:16.288 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:00:16.288 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:00:16.289 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:00:16.289 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:00:16.289 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:00:16.289 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:00:16.290 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:00:16.290 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:00:16.290 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:00:16.291 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:00:16.291 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:00:16.291 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:00:16.291 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:00:16.291 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:00:16.292 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:00:16.292 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:00:16.292 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:00:16.292 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:00:17.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:00:16 smithi153 ceph-mon[70044]: from='client.45717 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:00:17.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:00:16 smithi153 ceph-mon[70044]: pgmap v9960: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:17.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:00:16 smithi153 ceph-mon[70044]: from='client.55654 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:00:17.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:00:16 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2805954818' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:00:17.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:00:16 smithi083 ceph-mon[79930]: from='client.45717 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:00:17.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:00:16 smithi083 ceph-mon[79930]: pgmap v9960: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:17.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:00:16 smithi083 ceph-mon[79930]: from='client.55654 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:00:17.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:00:16 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2805954818' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:00:17.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:00:16 smithi083 ceph-mon[77242]: from='client.45717 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:00:17.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:00:16 smithi083 ceph-mon[77242]: pgmap v9960: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:17.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:00:16 smithi083 ceph-mon[77242]: from='client.55654 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:00:17.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:00:16 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2805954818' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:00:19.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:00:18 smithi153 ceph-mon[70044]: pgmap v9961: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:19.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:00:18 smithi083 ceph-mon[79930]: pgmap v9961: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:19.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:00:18 smithi083 ceph-mon[77242]: pgmap v9961: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:00:20 smithi153 ceph-mon[70044]: pgmap v9962: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:20.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:00:20 smithi083 ceph-mon[77242]: pgmap v9962: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:20.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:00:20 smithi083 ceph-mon[79930]: pgmap v9962: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:21.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:00:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:00:21] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T11:00:21.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:00:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:00:21.386Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:00:22.770 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:00:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:00:22.316Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:00:23.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:00:22 smithi153 ceph-mon[70044]: pgmap v9963: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:23.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:00:22 smithi083 ceph-mon[79930]: pgmap v9963: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:23.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:00:22 smithi083 ceph-mon[77242]: pgmap v9963: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:00:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:00:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:00:25.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:00:24 smithi153 ceph-mon[70044]: pgmap v9964: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:25.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:00:24 smithi083 ceph-mon[79930]: pgmap v9964: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:25.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:00:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:00:24.759Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:00:25.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:00:24 smithi083 ceph-mon[77242]: pgmap v9964: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:25.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:00:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:00:25.474Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:00:27.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:00:26 smithi153 ceph-mon[70044]: pgmap v9965: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:27.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:00:26 smithi083 ceph-mon[79930]: pgmap v9965: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:27.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:00:26 smithi083 ceph-mon[77242]: pgmap v9965: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:29.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:00:28 smithi153 ceph-mon[70044]: pgmap v9966: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:29.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:00:28 smithi083 ceph-mon[79930]: pgmap v9966: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:29.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:00:28 smithi083 ceph-mon[77242]: pgmap v9966: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:30.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:00:30 smithi083 ceph-mon[79930]: pgmap v9967: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:30.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:00:30 smithi083 ceph-mon[77242]: pgmap v9967: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:00:30 smithi153 ceph-mon[70044]: pgmap v9967: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:31.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:00:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:00:31] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T11:00:31.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:00:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:00:31.387Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:00:32.772 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:00:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:00:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:00:33.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:00:32 smithi153 ceph-mon[70044]: pgmap v9968: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:33.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:00:32 smithi083 ceph-mon[77242]: pgmap v9968: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:33.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:00:32 smithi083 ceph-mon[79930]: pgmap v9968: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:00:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:00:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:00:35.048 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:00:34 smithi083 ceph-mon[79930]: pgmap v9969: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:35.048 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:00:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:00:34.760Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:00:35.049 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:00:34 smithi083 ceph-mon[77242]: pgmap v9969: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:35.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:00:34 smithi153 ceph-mon[70044]: pgmap v9969: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:35.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:00:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:00:35.474Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:00:37.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:00:36 smithi153 ceph-mon[70044]: pgmap v9970: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:37.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:00:36 smithi083 ceph-mon[79930]: pgmap v9970: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:37.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:00:36 smithi083 ceph-mon[77242]: pgmap v9970: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:39.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:00:38 smithi153 ceph-mon[70044]: pgmap v9971: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:39.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:00:38 smithi083 ceph-mon[79930]: pgmap v9971: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:39.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:00:38 smithi083 ceph-mon[77242]: pgmap v9971: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:40.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:00:40 smithi083 ceph-mon[79930]: pgmap v9972: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:40.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:00:40 smithi083 ceph-mon[77242]: pgmap v9972: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:00:40 smithi153 ceph-mon[70044]: pgmap v9972: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:41.388 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:00:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:00:41.389 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:00:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:00:41.389 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:00:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:00:41.389 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:00:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:00:41.390 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:00:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:00:41.390 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:00:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:00:41.390 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:00:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:00:41.391 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:00:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:00:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:00:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:00:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:00:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:00:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:00:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:00:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:00:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:00:41.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:00:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:00:41] "GET /metrics HTTP/1.1" 200 225228 "" "Prometheus/2.33.4" 2022-09-29T11:00:41.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:00:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:00:41.387Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:00:42.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:00:42 smithi083 ceph-mon[77242]: pgmap v9973: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:00:42 smithi083 ceph-mon[79930]: pgmap v9973: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:00:42 smithi153 ceph-mon[70044]: pgmap v9973: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:42.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:00:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:00:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:00:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:00:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:00:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:00:45.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:00:44 smithi153 ceph-mon[70044]: pgmap v9974: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:45.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:00:44 smithi083 ceph-mon[79930]: pgmap v9974: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:45.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:00:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:00:44.760Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:00:45.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:00:44 smithi083 ceph-mon[77242]: pgmap v9974: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:45.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:00:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:00:45.475Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:00:46.617 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:00:46.906 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:00:46.906 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 9m ago 5h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:00:46.907 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 7m ago 5h 63.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:00:46.907 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 9m ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:00:46.907 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 7m ago 5h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:00:46.907 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 7m ago 5h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:00:46.908 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 9m ago 5h 791M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:00:46.908 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 9m ago 5h 1235M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:00:46.908 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 7m ago 5h 532M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:00:46.908 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 9m ago 5h 709M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:00:46.908 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 9m ago 5h 26.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:00:46.909 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 7m ago 5h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:00:46.909 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 9m ago 5h 870M 2447M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:00:46.909 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 9m ago 5h 860M 2447M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:00:46.909 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 9m ago 5h 671M 2447M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:00:46.910 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 9m ago 5h 839M 2447M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:00:46.910 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 7m ago 5h 740M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:00:46.910 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 7m ago 5h 922M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:00:46.910 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 7m ago 5h 823M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:00:46.911 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 7m ago 5h 975M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:00:46.911 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 7m ago 5h 191M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:00:47.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:00:46 smithi153 ceph-mon[70044]: pgmap v9975: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:47.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:00:46 smithi083 ceph-mon[79930]: pgmap v9975: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:47.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:00:46 smithi083 ceph-mon[77242]: pgmap v9975: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:47.261 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:00:47.261 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:00:47.262 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:00:47.262 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:00:47.262 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:00:47.262 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:00:47.263 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:00:47.263 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:00:47.263 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:00:47.263 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:00:47.263 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:00:47.264 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:00:47.264 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:00:47.264 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:00:47.264 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:00:47.265 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:00:47.265 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:00:47.265 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:00:48.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:00:47 smithi153 ceph-mon[70044]: from='client.45735 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:00:48.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:00:47 smithi153 ceph-mon[70044]: from='client.45741 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:00:48.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:00:47 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/615376959' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:00:48.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:00:47 smithi083 ceph-mon[79930]: from='client.45735 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:00:48.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:00:47 smithi083 ceph-mon[79930]: from='client.45741 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:00:48.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:00:47 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/615376959' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:00:48.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:00:47 smithi083 ceph-mon[77242]: from='client.45735 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:00:48.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:00:47 smithi083 ceph-mon[77242]: from='client.45741 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:00:48.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:00:47 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/615376959' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:00:49.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:00:48 smithi153 ceph-mon[70044]: pgmap v9976: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:49.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:00:48 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:00:49.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:00:48 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:00:49.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:00:48 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:00:49.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:00:48 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:00:49.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:00:48 smithi083 ceph-mon[79930]: pgmap v9976: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:49.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:00:48 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:00:49.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:00:48 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:00:49.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:00:48 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:00:49.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:00:48 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:00:49.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:00:48 smithi083 ceph-mon[77242]: pgmap v9976: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:49.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:00:48 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:00:49.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:00:48 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:00:49.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:00:48 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:00:49.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:00:48 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:00:50.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:00:50 smithi083 ceph-mon[79930]: pgmap v9977: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:50.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:00:50 smithi083 ceph-mon[77242]: pgmap v9977: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:00:50 smithi153 ceph-mon[70044]: pgmap v9977: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:51.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:00:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:00:51] "GET /metrics HTTP/1.1" 200 225228 "" "Prometheus/2.33.4" 2022-09-29T11:00:51.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:00:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:00:51.388Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:00:52.778 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:00:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:00:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:00:53.057 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:00:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:00:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:00:53.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:00:52 smithi153 ceph-mon[70044]: pgmap v9978: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:53.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:00:52 smithi083 ceph-mon[79930]: pgmap v9978: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:53.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:00:52 smithi083 ceph-mon[77242]: pgmap v9978: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:55.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:00:54 smithi153 ceph-mon[70044]: pgmap v9979: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:55.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:00:54 smithi083 ceph-mon[79930]: pgmap v9979: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:55.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:00:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:00:54.760Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:00:55.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:00:54 smithi083 ceph-mon[77242]: pgmap v9979: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:55.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:00:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:00:55.476Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:00:57.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:00:56 smithi153 ceph-mon[70044]: pgmap v9980: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:57.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:00:56 smithi083 ceph-mon[77242]: pgmap v9980: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:57.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:00:56 smithi083 ceph-mon[79930]: pgmap v9980: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:59.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:00:58 smithi153 ceph-mon[70044]: pgmap v9981: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:59.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:00:58 smithi083 ceph-mon[77242]: pgmap v9981: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:00:59.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:00:58 smithi083 ceph-mon[79930]: pgmap v9981: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:00.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:01:00 smithi083 ceph-mon[79930]: pgmap v9982: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:00.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:01:00 smithi083 ceph-mon[77242]: pgmap v9982: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:01:00 smithi153 ceph-mon[70044]: pgmap v9982: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:01.670 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:01:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:01:01] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T11:01:01.671 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:01:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:01:01.389Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:01:02.783 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:01:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:01:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:01:03.057 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:01:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:01:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:01:03.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:01:02 smithi153 ceph-mon[70044]: pgmap v9983: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:03.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:01:02 smithi083 ceph-mon[77242]: pgmap v9983: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:03.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:01:02 smithi083 ceph-mon[79930]: pgmap v9983: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:05.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:01:04 smithi153 ceph-mon[70044]: pgmap v9984: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:05.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:01:04 smithi083 ceph-mon[79930]: pgmap v9984: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:05.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:01:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:01:04.761Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:01:05.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:01:04 smithi083 ceph-mon[77242]: pgmap v9984: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:05.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:01:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:01:05.476Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:01:07.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:01:06 smithi153 ceph-mon[70044]: pgmap v9985: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:07.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:01:06 smithi083 ceph-mon[79930]: pgmap v9985: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:07.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:01:06 smithi083 ceph-mon[77242]: pgmap v9985: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:09.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:01:08 smithi153 ceph-mon[70044]: pgmap v9986: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:09.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:01:08 smithi083 ceph-mon[79930]: pgmap v9986: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:09.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:01:08 smithi083 ceph-mon[77242]: pgmap v9986: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:10.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:01:10 smithi083 ceph-mon[79930]: pgmap v9987: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:10.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:01:10 smithi083 ceph-mon[77242]: pgmap v9987: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:01:10 smithi153 ceph-mon[70044]: pgmap v9987: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:11.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:01:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:01:11] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T11:01:11.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:01:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:01:11.389Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:01:12.786 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:01:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:01:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:01:13.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:01:12 smithi153 ceph-mon[70044]: pgmap v9988: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:13.057 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:01:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:01:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:01:13.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:01:12 smithi083 ceph-mon[79930]: pgmap v9988: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:13.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:01:12 smithi083 ceph-mon[77242]: pgmap v9988: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:15.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:01:14 smithi153 ceph-mon[70044]: pgmap v9989: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:15.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:01:14 smithi083 ceph-mon[79930]: pgmap v9989: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:15.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:01:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:01:14.761Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:01:15.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:01:14 smithi083 ceph-mon[77242]: pgmap v9989: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:15.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:01:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:01:15.477Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:01:17.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:01:16 smithi153 ceph-mon[70044]: pgmap v9990: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:17.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:01:16 smithi083 ceph-mon[79930]: pgmap v9990: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:17.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:01:16 smithi083 ceph-mon[77242]: pgmap v9990: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:17.591 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:01:17.880 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:01:17.881 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 9m ago 5h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:01:17.881 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 7m ago 5h 63.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:01:17.881 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 9m ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:01:17.881 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 7m ago 5h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:01:17.882 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 7m ago 5h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:01:17.882 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 9m ago 5h 791M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:01:17.882 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 9m ago 5h 1235M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:01:17.882 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 7m ago 5h 532M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:01:17.883 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 9m ago 5h 709M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:01:17.883 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 9m ago 5h 26.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:01:17.883 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 7m ago 5h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:01:17.883 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 9m ago 5h 870M 2447M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:01:17.884 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 9m ago 5h 860M 2447M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:01:17.884 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 9m ago 5h 671M 2447M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:01:17.884 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 9m ago 5h 839M 2447M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:01:17.884 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 7m ago 5h 740M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:01:17.884 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 7m ago 5h 922M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:01:17.885 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 7m ago 5h 823M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:01:17.885 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 7m ago 5h 975M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:01:17.885 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 7m ago 5h 191M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:01:18.236 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:01:18.237 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:01:18.237 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:01:18.237 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:01:18.237 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:01:18.238 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:01:18.238 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:01:18.238 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:01:18.238 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:01:18.239 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:01:18.239 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:01:18.239 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:01:18.239 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:01:18.240 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:01:18.240 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:01:18.240 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:01:18.241 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:01:18.241 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:01:19.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:01:18 smithi153 ceph-mon[70044]: from='client.45753 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:01:19.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:01:18 smithi153 ceph-mon[70044]: pgmap v9991: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:19.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:01:18 smithi153 ceph-mon[70044]: from='client.45759 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:01:19.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:01:18 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3994854871' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:01:19.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:01:18 smithi083 ceph-mon[79930]: from='client.45753 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:01:19.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:01:18 smithi083 ceph-mon[79930]: pgmap v9991: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:19.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:01:18 smithi083 ceph-mon[79930]: from='client.45759 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:01:19.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:01:18 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3994854871' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:01:19.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:01:18 smithi083 ceph-mon[77242]: from='client.45753 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:01:19.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:01:18 smithi083 ceph-mon[77242]: pgmap v9991: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:19.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:01:18 smithi083 ceph-mon[77242]: from='client.45759 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:01:19.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:01:18 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3994854871' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:01:20.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:01:20 smithi083 ceph-mon[77242]: pgmap v9992: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:20.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:01:20 smithi083 ceph-mon[79930]: pgmap v9992: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:01:20 smithi153 ceph-mon[70044]: pgmap v9992: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:21.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:01:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:01:21] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T11:01:21.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:01:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:01:21.389Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:01:22.788 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:01:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:01:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:01:23.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:01:22 smithi153 ceph-mon[70044]: pgmap v9993: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:23.057 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:01:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:01:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:01:23.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:01:22 smithi083 ceph-mon[77242]: pgmap v9993: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:23.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:01:22 smithi083 ceph-mon[79930]: pgmap v9993: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:25.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:01:24 smithi153 ceph-mon[70044]: pgmap v9994: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:25.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:01:24 smithi083 ceph-mon[79930]: pgmap v9994: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:25.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:01:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:01:24.762Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:01:25.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:01:24 smithi083 ceph-mon[77242]: pgmap v9994: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:25.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:01:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:01:25.477Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:01:27.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:01:26 smithi153 ceph-mon[70044]: pgmap v9995: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:27.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:01:26 smithi083 ceph-mon[79930]: pgmap v9995: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:27.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:01:26 smithi083 ceph-mon[77242]: pgmap v9995: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:29.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:01:28 smithi153 ceph-mon[70044]: pgmap v9996: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:29.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:01:28 smithi083 ceph-mon[79930]: pgmap v9996: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:29.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:01:28 smithi083 ceph-mon[77242]: pgmap v9996: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:30.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:01:30 smithi083 ceph-mon[79930]: pgmap v9997: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:30.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:01:30 smithi083 ceph-mon[77242]: pgmap v9997: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:01:30 smithi153 ceph-mon[70044]: pgmap v9997: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:31.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:01:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:01:31] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T11:01:31.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:01:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:01:31.390Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:01:32.791 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:01:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:01:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:01:33.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:01:32 smithi153 ceph-mon[70044]: pgmap v9998: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:33.058 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:01:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:01:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:01:33.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:01:32 smithi083 ceph-mon[79930]: pgmap v9998: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:33.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:01:32 smithi083 ceph-mon[77242]: pgmap v9998: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:35.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:01:34 smithi153 ceph-mon[70044]: pgmap v9999: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:35.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:01:34 smithi083 ceph-mon[79930]: pgmap v9999: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:35.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:01:34 smithi083 ceph-mon[77242]: pgmap v9999: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:35.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:01:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:01:34.763Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:01:35.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:01:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:01:35.477Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:01:37.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:01:36 smithi153 ceph-mon[70044]: pgmap v10000: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:37.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:01:36 smithi083 ceph-mon[77242]: pgmap v10000: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:37.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:01:36 smithi083 ceph-mon[79930]: pgmap v10000: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:39.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:01:38 smithi153 ceph-mon[70044]: pgmap v10001: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:39.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:01:38 smithi083 ceph-mon[79930]: pgmap v10001: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:39.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:01:38 smithi083 ceph-mon[77242]: pgmap v10001: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:40.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:01:40 smithi083 ceph-mon[79930]: pgmap v10002: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:40.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:01:40 smithi083 ceph-mon[77242]: pgmap v10002: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:01:40 smithi153 ceph-mon[70044]: pgmap v10002: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:41.392 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:01:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:01:41.393 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:01:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:01:41.393 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:01:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:01:41.393 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:01:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:01:41.393 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:01:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:01:41.394 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:01:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:01:41.394 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:01:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:01:41.394 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:01:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:01:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:01:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:01:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:01:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:01:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:01:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:01:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:01:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:01:41.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:01:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:01:41] "GET /metrics HTTP/1.1" 200 225228 "" "Prometheus/2.33.4" 2022-09-29T11:01:41.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:01:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:01:41.391Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:01:42.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:01:42 smithi083 ceph-mon[79930]: pgmap v10003: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:42.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:01:42 smithi083 ceph-mon[77242]: pgmap v10003: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:01:42 smithi153 ceph-mon[70044]: pgmap v10003: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:42.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:01:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:01:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:01:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:01:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:01:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:01:45.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:01:44 smithi153 ceph-mon[70044]: pgmap v10004: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:45.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:01:44 smithi083 ceph-mon[79930]: pgmap v10004: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:45.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:01:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:01:44.764Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:01:45.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:01:44 smithi083 ceph-mon[77242]: pgmap v10004: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:45.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:01:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:01:45.478Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:01:47.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:01:46 smithi153 ceph-mon[70044]: pgmap v10005: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:47.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:01:46 smithi083 ceph-mon[79930]: pgmap v10005: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:47.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:01:46 smithi083 ceph-mon[77242]: pgmap v10005: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:48.561 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:01:48.873 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:01:48.874 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 10m ago 5h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:01:48.874 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 8m ago 5h 63.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:01:48.875 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 10m ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:01:48.875 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 8m ago 5h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:01:48.875 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 8m ago 5h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:01:48.875 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 10m ago 5h 791M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:01:48.876 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 10m ago 5h 1235M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:01:48.876 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 8m ago 5h 532M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:01:48.876 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 10m ago 5h 709M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:01:48.876 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 10m ago 5h 26.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:01:48.876 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 8m ago 5h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:01:48.877 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 10m ago 5h 870M 2447M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:01:48.877 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 10m ago 5h 860M 2447M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:01:48.877 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 10m ago 5h 671M 2447M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:01:48.877 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 10m ago 5h 839M 2447M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:01:48.878 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 8m ago 5h 740M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:01:48.878 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 8m ago 5h 922M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:01:48.878 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 8m ago 5h 823M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:01:48.878 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 8m ago 5h 975M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:01:48.879 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 8m ago 5h 191M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:01:49.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:01:48 smithi153 ceph-mon[70044]: pgmap v10006: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:49.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:01:48 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:01:49.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:01:48 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:01:49.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:01:48 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:01:49.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:01:48 smithi083 ceph-mon[79930]: pgmap v10006: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:49.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:01:48 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:01:49.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:01:48 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:01:49.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:01:48 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:01:49.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:01:48 smithi083 ceph-mon[77242]: pgmap v10006: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:49.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:01:48 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:01:49.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:01:48 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:01:49.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:01:48 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:01:49.262 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:01:49.263 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:01:49.263 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:01:49.263 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:01:49.263 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:01:49.264 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:01:49.264 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:01:49.264 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:01:49.264 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:01:49.265 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:01:49.265 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:01:49.265 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:01:49.265 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:01:49.266 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:01:49.266 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:01:49.266 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:01:49.266 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:01:49.267 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:01:50.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:01:49 smithi153 ceph-mon[70044]: from='client.45771 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:01:50.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:01:49 smithi153 ceph-mon[70044]: from='client.55708 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:01:50.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:01:49 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:01:50.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:01:49 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:01:50.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:01:49 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:01:50.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:01:49 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:01:50.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:01:49 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2593797265' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:01:50.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:01:49 smithi083 ceph-mon[79930]: from='client.45771 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:01:50.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:01:49 smithi083 ceph-mon[79930]: from='client.55708 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:01:50.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:01:49 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:01:50.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:01:49 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:01:50.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:01:49 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:01:50.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:01:49 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:01:50.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:01:49 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2593797265' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:01:50.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:01:49 smithi083 ceph-mon[77242]: from='client.45771 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:01:50.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:01:49 smithi083 ceph-mon[77242]: from='client.55708 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:01:50.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:01:49 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:01:50.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:01:49 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:01:50.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:01:49 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:01:50.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:01:49 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:01:50.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:01:49 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2593797265' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:01:51.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:01:50 smithi153 ceph-mon[70044]: pgmap v10007: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:51.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:01:50 smithi083 ceph-mon[79930]: pgmap v10007: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:51.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:01:50 smithi083 ceph-mon[77242]: pgmap v10007: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:51.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:01:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:01:51] "GET /metrics HTTP/1.1" 200 225228 "" "Prometheus/2.33.4" 2022-09-29T11:01:51.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:01:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:01:51.391Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:01:52.805 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:01:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:01:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:01:53.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:01:52 smithi153 ceph-mon[70044]: pgmap v10008: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:53.057 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:01:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:01:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:01:53.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:01:52 smithi083 ceph-mon[79930]: pgmap v10008: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:53.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:01:52 smithi083 ceph-mon[77242]: pgmap v10008: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:54.765 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:01:54 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:01:54.765 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:01:54 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:01:54.766 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:01:54 smithi083 ceph-mon[77242]: pgmap v10009: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:54.766 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:01:54 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:01:54.766 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:01:54 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:01:54.767 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:01:54 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:01:54.767 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:01:54 smithi083 ceph-mon[79930]: pgmap v10009: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:54.767 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:01:54 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:01:54.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:01:54 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:01:54.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:01:54 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:01:54.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:01:54 smithi153 ceph-mon[70044]: pgmap v10009: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:54.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:01:54 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:01:55.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:01:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:01:54.763Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:01:55.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:01:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:01:55.479Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:01:57.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:01:56 smithi153 ceph-mon[70044]: pgmap v10010: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:57.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:01:56 smithi083 ceph-mon[77242]: pgmap v10010: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:57.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:01:56 smithi083 ceph-mon[79930]: pgmap v10010: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:59.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:01:58 smithi153 ceph-mon[70044]: pgmap v10011: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:59.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:01:58 smithi083 ceph-mon[77242]: pgmap v10011: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:01:59.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:01:58 smithi083 ceph-mon[79930]: pgmap v10011: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:00.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:02:00 smithi083 ceph-mon[79930]: pgmap v10012: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:00.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:02:00 smithi083 ceph-mon[77242]: pgmap v10012: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:02:00 smithi153 ceph-mon[70044]: pgmap v10012: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:01.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:02:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:02:01.392Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:02:01.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:02:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:02:01] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T11:02:02.800 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:02:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:02:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:02:03.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:02:02 smithi153 ceph-mon[70044]: pgmap v10013: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:03.057 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:02:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:02:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:02:03.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:02:02 smithi083 ceph-mon[79930]: pgmap v10013: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:03.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:02:02 smithi083 ceph-mon[77242]: pgmap v10013: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:05.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:02:04 smithi153 ceph-mon[70044]: pgmap v10014: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:05.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:02:04 smithi083 ceph-mon[79930]: pgmap v10014: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:05.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:02:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:02:04.765Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:02:05.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:02:04 smithi083 ceph-mon[77242]: pgmap v10014: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:05.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:02:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:02:05.479Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:02:07.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:02:06 smithi153 ceph-mon[70044]: pgmap v10015: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:07.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:02:06 smithi083 ceph-mon[79930]: pgmap v10015: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:07.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:02:06 smithi083 ceph-mon[77242]: pgmap v10015: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:09.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:02:08 smithi083 ceph-mon[79930]: pgmap v10016: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:09.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:02:08 smithi083 ceph-mon[77242]: pgmap v10016: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:09.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:02:08 smithi153 ceph-mon[70044]: pgmap v10016: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:10.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:02:10 smithi083 ceph-mon[79930]: pgmap v10017: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:10.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:02:10 smithi083 ceph-mon[77242]: pgmap v10017: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:02:10 smithi153 ceph-mon[70044]: pgmap v10017: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:11.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:02:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:02:11.393Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:02:11.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:02:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:02:11] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T11:02:12.803 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:02:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:02:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:02:13.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:02:12 smithi153 ceph-mon[70044]: pgmap v10018: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:13.057 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:02:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:02:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:02:13.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:02:12 smithi083 ceph-mon[79930]: pgmap v10018: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:13.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:02:12 smithi083 ceph-mon[77242]: pgmap v10018: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:15.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:02:14 smithi153 ceph-mon[70044]: pgmap v10019: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:15.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:02:14 smithi083 ceph-mon[79930]: pgmap v10019: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:15.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:02:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:02:14.766Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:02:15.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:02:14 smithi083 ceph-mon[77242]: pgmap v10019: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:15.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:02:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:02:15.479Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:02:17.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:02:16 smithi083 ceph-mon[79930]: pgmap v10020: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:17.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:02:16 smithi083 ceph-mon[77242]: pgmap v10020: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:17.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:02:16 smithi153 ceph-mon[70044]: pgmap v10020: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:19.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:02:18 smithi083 ceph-mon[77242]: pgmap v10021: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:19.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:02:18 smithi083 ceph-mon[79930]: pgmap v10021: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:19.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:02:18 smithi153 ceph-mon[70044]: pgmap v10021: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:19.592 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:02:19.880 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:02:19.881 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 26s ago 5h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:02:19.881 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 8m ago 5h 63.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:02:19.881 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 26s ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:02:19.881 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 8m ago 5h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:02:19.882 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 8m ago 5h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:02:19.882 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 26s ago 5h 799M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:02:19.882 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 26s ago 5h 1271M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:02:19.882 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 8m ago 5h 532M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:02:19.883 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 26s ago 5h 731M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:02:19.883 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 26s ago 5h 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:02:19.883 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 8m ago 5h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:02:19.883 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 26s ago 5h 895M 2447M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:02:19.883 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 26s ago 5h 885M 2447M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:02:19.884 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 26s ago 5h 690M 2447M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:02:19.884 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 26s ago 5h 864M 2447M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:02:19.884 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 8m ago 5h 740M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:02:19.884 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 8m ago 5h 922M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:02:19.885 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 8m ago 5h 823M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:02:19.885 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 8m ago 5h 975M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:02:19.885 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 8m ago 5h 191M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:02:20.238 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:02:20.238 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:02:20.238 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:02:20.239 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:02:20.239 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:02:20.239 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:02:20.239 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:02:20.240 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:02:20.240 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:02:20.240 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:02:20.240 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:02:20.240 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:02:20.241 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:02:20.241 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:02:20.241 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:02:20.241 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:02:20.242 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:02:20.242 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:02:20.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:02:20 smithi083 ceph-mon[79930]: from='client.55720 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:02:20.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:02:20 smithi083 ceph-mon[79930]: pgmap v10022: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:20.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:02:20 smithi083 ceph-mon[79930]: from='client.45795 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:02:20.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:02:20 smithi083 ceph-mon[77242]: from='client.55720 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:02:20.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:02:20 smithi083 ceph-mon[77242]: pgmap v10022: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:20.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:02:20 smithi083 ceph-mon[77242]: from='client.45795 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:02:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:02:20 smithi153 ceph-mon[70044]: from='client.55720 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:02:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:02:20 smithi153 ceph-mon[70044]: pgmap v10022: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:02:20 smithi153 ceph-mon[70044]: from='client.45795 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:02:21.395 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:02:21 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3981422073' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:02:21.395 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:02:21 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3981422073' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:02:21.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:02:21 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3981422073' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:02:21.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:02:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:02:21] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T11:02:21.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:02:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:02:21.394Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:02:22.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:02:22 smithi083 ceph-mon[79930]: pgmap v10023: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:22.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:02:22 smithi083 ceph-mon[77242]: pgmap v10023: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:02:22 smithi153 ceph-mon[70044]: pgmap v10023: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:22.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:02:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:02:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:02:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:02:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:02:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:02:25.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:02:24 smithi153 ceph-mon[70044]: pgmap v10024: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:25.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:02:24 smithi083 ceph-mon[79930]: pgmap v10024: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:25.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:02:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:02:24.767Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:02:25.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:02:24 smithi083 ceph-mon[77242]: pgmap v10024: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:25.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:02:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:02:25.480Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:02:27.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:02:26 smithi083 ceph-mon[79930]: pgmap v10025: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:27.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:02:26 smithi083 ceph-mon[77242]: pgmap v10025: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:27.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:02:26 smithi153 ceph-mon[70044]: pgmap v10025: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:29.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:02:28 smithi083 ceph-mon[79930]: pgmap v10026: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:29.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:02:28 smithi083 ceph-mon[77242]: pgmap v10026: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:29.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:02:28 smithi153 ceph-mon[70044]: pgmap v10026: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:30.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:02:30 smithi083 ceph-mon[79930]: pgmap v10027: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:30.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:02:30 smithi083 ceph-mon[77242]: pgmap v10027: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:02:30 smithi153 ceph-mon[70044]: pgmap v10027: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:31.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:02:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:02:31] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T11:02:31.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:02:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:02:31.394Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:02:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:02:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:02:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:02:33.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:02:32 smithi083 ceph-mon[79930]: pgmap v10028: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:33.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:02:32 smithi083 ceph-mon[77242]: pgmap v10028: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:33.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:02:32 smithi153 ceph-mon[70044]: pgmap v10028: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:02:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:02:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:02:35.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:02:34 smithi083 ceph-mon[77242]: pgmap v10029: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:35.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:02:34 smithi083 ceph-mon[79930]: pgmap v10029: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:35.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:02:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:02:34.768Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:02:35.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:02:34 smithi153 ceph-mon[70044]: pgmap v10029: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:35.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:02:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:02:35.481Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:02:37.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:02:36 smithi083 ceph-mon[79930]: pgmap v10030: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:37.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:02:36 smithi083 ceph-mon[77242]: pgmap v10030: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:37.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:02:36 smithi153 ceph-mon[70044]: pgmap v10030: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:39.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:02:38 smithi083 ceph-mon[79930]: pgmap v10031: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:39.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:02:38 smithi083 ceph-mon[77242]: pgmap v10031: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:39.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:02:38 smithi153 ceph-mon[70044]: pgmap v10031: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:40.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:02:40 smithi083 ceph-mon[79930]: pgmap v10032: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:40.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:02:40 smithi083 ceph-mon[77242]: pgmap v10032: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:02:40 smithi153 ceph-mon[70044]: pgmap v10032: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:41.396 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:02:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:02:41.396 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:02:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:02:41.396 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:02:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:02:41.397 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:02:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:02:41.397 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:02:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:02:41.397 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:02:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:02:41.397 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:02:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:02:41.398 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:02:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:02:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:02:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:02:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:02:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:02:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:02:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:02:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:02:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:02:41.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:02:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:02:41] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T11:02:41.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:02:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:02:41.395Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:02:42.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:02:42 smithi083 ceph-mon[79930]: pgmap v10033: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:42.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:02:42 smithi083 ceph-mon[77242]: pgmap v10033: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:02:42 smithi153 ceph-mon[70044]: pgmap v10033: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:42.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:02:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:02:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:02:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:02:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:02:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:02:45.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:02:44 smithi083 ceph-mon[79930]: pgmap v10034: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:45.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:02:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:02:44.768Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:02:45.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:02:44 smithi083 ceph-mon[77242]: pgmap v10034: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:45.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:02:44 smithi153 ceph-mon[70044]: pgmap v10034: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:45.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:02:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:02:45.482Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:02:47.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:02:46 smithi083 ceph-mon[79930]: pgmap v10035: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:47.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:02:46 smithi083 ceph-mon[77242]: pgmap v10035: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:47.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:02:46 smithi153 ceph-mon[70044]: pgmap v10035: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:49.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:02:48 smithi083 ceph-mon[79930]: pgmap v10036: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:49.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:02:48 smithi083 ceph-mon[77242]: pgmap v10036: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:49.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:02:48 smithi153 ceph-mon[70044]: pgmap v10036: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:50.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:02:50 smithi083 ceph-mon[79930]: pgmap v10037: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:50.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:02:50 smithi083 ceph-mon[77242]: pgmap v10037: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:02:50 smithi153 ceph-mon[70044]: pgmap v10037: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:50.564 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:02:50.852 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:02:50.852 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 57s ago 5h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:02:50.852 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 9m ago 5h 63.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:02:50.853 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 57s ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:02:50.853 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 9m ago 5h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:02:50.853 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 9m ago 5h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:02:50.853 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 57s ago 5h 799M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:02:50.854 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 57s ago 5h 1271M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:02:50.854 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 9m ago 5h 532M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:02:50.854 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 57s ago 5h 731M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:02:50.854 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 57s ago 5h 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:02:50.855 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 9m ago 5h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:02:50.855 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 57s ago 5h 895M 2447M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:02:50.855 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 57s ago 5h 885M 2447M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:02:50.855 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 57s ago 5h 690M 2447M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:02:50.855 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 57s ago 5h 864M 2447M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:02:50.856 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 9m ago 5h 740M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:02:50.856 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 9m ago 5h 922M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:02:50.856 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 9m ago 5h 823M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:02:50.856 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 9m ago 5h 975M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:02:50.857 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 9m ago 5h 191M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:02:51.208 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:02:51.208 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:02:51.208 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:02:51.209 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:02:51.209 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:02:51.209 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:02:51.209 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:02:51.210 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:02:51.210 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:02:51.210 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:02:51.210 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:02:51.211 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:02:51.211 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:02:51.211 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:02:51.211 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:02:51.211 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:02:51.212 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:02:51.212 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:02:51.396 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:02:51 smithi083 ceph-mon[79930]: from='client.45807 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:02:51.396 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:02:51 smithi083 ceph-mon[79930]: from='client.55744 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:02:51.396 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:02:51 smithi083 ceph-mon[77242]: from='client.45807 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:02:51.396 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:02:51 smithi083 ceph-mon[77242]: from='client.55744 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:02:51.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:02:51 smithi153 ceph-mon[70044]: from='client.45807 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:02:51.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:02:51 smithi153 ceph-mon[70044]: from='client.55744 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:02:51.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:02:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:02:51] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T11:02:51.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:02:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:02:51.395Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:02:52.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:02:52 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/660875527' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:02:52.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:02:52 smithi083 ceph-mon[79930]: pgmap v10038: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:52.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:02:52 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/660875527' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:02:52.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:02:52 smithi083 ceph-mon[77242]: pgmap v10038: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:52.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:02:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:02:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:02:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:02:52 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/660875527' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:02:52.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:02:52 smithi153 ceph-mon[70044]: pgmap v10038: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:02:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:02:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:02:55.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:02:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:02:54.767Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:02:55.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:02:54 smithi083 ceph-mon[79930]: pgmap v10039: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:55.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:02:54 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:02:55.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:02:54 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:02:55.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:02:54 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:02:55.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:02:54 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:02:55.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:02:54 smithi083 ceph-mon[77242]: pgmap v10039: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:55.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:02:54 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:02:55.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:02:54 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:02:55.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:02:54 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:02:55.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:02:54 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:02:55.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:02:54 smithi153 ceph-mon[70044]: pgmap v10039: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:55.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:02:54 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:02:55.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:02:54 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:02:55.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:02:54 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:02:55.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:02:54 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:02:55.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:02:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:02:55.482Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:02:57.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:02:56 smithi083 ceph-mon[77242]: pgmap v10040: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:57.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:02:56 smithi083 ceph-mon[79930]: pgmap v10040: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:57.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:02:56 smithi153 ceph-mon[70044]: pgmap v10040: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:59.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:02:58 smithi083 ceph-mon[77242]: pgmap v10041: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:59.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:02:58 smithi083 ceph-mon[79930]: pgmap v10041: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:02:59.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:02:58 smithi153 ceph-mon[70044]: pgmap v10041: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:00.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:03:00 smithi083 ceph-mon[79930]: pgmap v10042: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:00.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:03:00 smithi083 ceph-mon[77242]: pgmap v10042: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:03:00 smithi153 ceph-mon[70044]: pgmap v10042: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:01.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:03:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:03:01] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T11:03:01.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:03:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:03:01.396Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:03:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:03:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:03:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:03:03.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:03:02 smithi083 ceph-mon[79930]: pgmap v10043: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:03.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:03:02 smithi083 ceph-mon[77242]: pgmap v10043: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:03.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:03:02 smithi153 ceph-mon[70044]: pgmap v10043: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:03:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:03:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:03:05.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:03:04 smithi083 ceph-mon[79930]: pgmap v10044: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:05.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:03:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:03:04.768Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:03:05.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:03:04 smithi083 ceph-mon[77242]: pgmap v10044: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:05.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:03:04 smithi153 ceph-mon[70044]: pgmap v10044: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:05.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:03:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:03:05.484Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:03:07.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:03:06 smithi083 ceph-mon[79930]: pgmap v10045: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:07.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:03:06 smithi083 ceph-mon[77242]: pgmap v10045: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:07.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:03:06 smithi153 ceph-mon[70044]: pgmap v10045: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:09.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:03:08 smithi083 ceph-mon[79930]: pgmap v10046: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:09.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:03:08 smithi083 ceph-mon[77242]: pgmap v10046: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:09.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:03:08 smithi153 ceph-mon[70044]: pgmap v10046: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:10.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:03:10 smithi083 ceph-mon[79930]: pgmap v10047: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:10.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:03:10 smithi083 ceph-mon[77242]: pgmap v10047: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:03:10 smithi153 ceph-mon[70044]: pgmap v10047: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:11.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:03:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:03:11] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T11:03:11.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:03:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:03:11.396Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:03:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:03:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:03:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:03:13.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:03:12 smithi083 ceph-mon[77242]: pgmap v10048: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:13.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:03:12 smithi083 ceph-mon[79930]: pgmap v10048: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:13.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:03:12 smithi153 ceph-mon[70044]: pgmap v10048: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:03:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:03:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:03:15.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:03:14 smithi083 ceph-mon[79930]: pgmap v10049: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:15.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:03:14 smithi083 ceph-mon[77242]: pgmap v10049: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:15.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:03:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:03:14.769Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:03:15.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:03:14 smithi153 ceph-mon[70044]: pgmap v10049: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:15.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:03:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:03:15.484Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:03:17.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:03:16 smithi083 ceph-mon[79930]: pgmap v10050: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:17.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:03:16 smithi083 ceph-mon[77242]: pgmap v10050: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:17.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:03:16 smithi153 ceph-mon[70044]: pgmap v10050: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:19.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:03:18 smithi083 ceph-mon[79930]: pgmap v10051: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:19.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:03:18 smithi083 ceph-mon[77242]: pgmap v10051: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:19.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:03:18 smithi153 ceph-mon[70044]: pgmap v10051: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:20.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:03:20 smithi083 ceph-mon[79930]: pgmap v10052: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:20.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:03:20 smithi083 ceph-mon[77242]: pgmap v10052: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:03:20 smithi153 ceph-mon[70044]: pgmap v10052: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:21.534 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:03:21.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:03:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:03:21] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T11:03:21.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:03:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:03:21.396Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:03:21.827 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:03:21.827 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 88s ago 5h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:03:21.828 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 9m ago 5h 63.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:03:21.828 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 88s ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:03:21.828 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 9m ago 5h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:03:21.828 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 9m ago 5h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:03:21.828 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 88s ago 5h 799M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:03:21.829 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 88s ago 5h 1271M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:03:21.829 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 9m ago 5h 532M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:03:21.829 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 88s ago 5h 731M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:03:21.829 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 88s ago 5h 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:03:21.830 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 9m ago 5h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:03:21.830 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 88s ago 5h 895M 2447M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:03:21.830 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 88s ago 5h 885M 2447M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:03:21.830 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 88s ago 5h 690M 2447M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:03:21.831 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 88s ago 5h 864M 2447M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:03:21.831 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 9m ago 5h 740M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:03:21.831 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 9m ago 5h 922M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:03:21.831 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 9m ago 5h 823M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:03:21.832 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 9m ago 5h 975M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:03:21.832 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 9m ago 5h 191M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:03:22.184 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:03:22.184 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:03:22.184 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:03:22.185 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:03:22.185 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:03:22.185 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:03:22.185 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:03:22.186 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:03:22.186 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:03:22.186 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:03:22.187 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:03:22.187 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:03:22.187 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:03:22.187 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:03:22.188 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:03:22.188 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:03:22.188 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:03:22.188 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:03:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:03:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:03:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:03:23.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:03:22 smithi083 ceph-mon[79930]: from='client.45825 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:03:23.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:03:22 smithi083 ceph-mon[79930]: pgmap v10053: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:23.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:03:22 smithi083 ceph-mon[79930]: from='client.45831 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:03:23.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:03:22 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1597874509' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:03:23.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:03:22 smithi083 ceph-mon[77242]: from='client.45825 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:03:23.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:03:22 smithi083 ceph-mon[77242]: pgmap v10053: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:23.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:03:22 smithi083 ceph-mon[77242]: from='client.45831 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:03:23.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:03:22 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1597874509' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:03:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:03:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:03:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:03:23.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:03:22 smithi153 ceph-mon[70044]: from='client.45825 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:03:23.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:03:22 smithi153 ceph-mon[70044]: pgmap v10053: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:23.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:03:22 smithi153 ceph-mon[70044]: from='client.45831 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:03:23.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:03:22 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1597874509' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:03:25.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:03:24 smithi083 ceph-mon[79930]: pgmap v10054: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:25.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:03:24 smithi083 ceph-mon[77242]: pgmap v10054: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:25.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:03:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:03:24.769Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:03:25.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:03:24 smithi153 ceph-mon[70044]: pgmap v10054: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:25.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:03:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:03:25.484Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:03:27.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:03:26 smithi083 ceph-mon[77242]: pgmap v10055: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:27.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:03:26 smithi083 ceph-mon[79930]: pgmap v10055: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:27.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:03:26 smithi153 ceph-mon[70044]: pgmap v10055: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:29.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:03:28 smithi083 ceph-mon[79930]: pgmap v10056: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:29.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:03:28 smithi083 ceph-mon[77242]: pgmap v10056: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:29.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:03:28 smithi153 ceph-mon[70044]: pgmap v10056: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:30.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:03:30 smithi083 ceph-mon[77242]: pgmap v10057: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:30.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:03:30 smithi083 ceph-mon[79930]: pgmap v10057: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:03:30 smithi153 ceph-mon[70044]: pgmap v10057: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:31.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:03:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:03:31] "GET /metrics HTTP/1.1" 200 225229 "" "Prometheus/2.33.4" 2022-09-29T11:03:31.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:03:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:03:31.398Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:03:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:03:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:03:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:03:33.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:03:32 smithi083 ceph-mon[79930]: pgmap v10058: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:33.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:03:32 smithi083 ceph-mon[77242]: pgmap v10058: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:33.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:03:32 smithi153 ceph-mon[70044]: pgmap v10058: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:03:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:03:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:03:35.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:03:34 smithi083 ceph-mon[79930]: pgmap v10059: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:35.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:03:34 smithi083 ceph-mon[77242]: pgmap v10059: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:35.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:03:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:03:34.770Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:03:35.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:03:34 smithi153 ceph-mon[70044]: pgmap v10059: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:35.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:03:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:03:35.485Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:03:37.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:03:36 smithi083 ceph-mon[79930]: pgmap v10060: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:37.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:03:36 smithi083 ceph-mon[77242]: pgmap v10060: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:37.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:03:36 smithi153 ceph-mon[70044]: pgmap v10060: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:39.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:03:38 smithi083 ceph-mon[79930]: pgmap v10061: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:39.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:03:38 smithi083 ceph-mon[77242]: pgmap v10061: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:39.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:03:38 smithi153 ceph-mon[70044]: pgmap v10061: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:40.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:03:40 smithi083 ceph-mon[79930]: pgmap v10062: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:40.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:03:40 smithi083 ceph-mon[77242]: pgmap v10062: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:03:40 smithi153 ceph-mon[70044]: pgmap v10062: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:41.399 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:03:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:03:41.400 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:03:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:03:41.400 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:03:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:03:41.400 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:03:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:03:41.401 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:03:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:03:41.401 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:03:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:03:41.401 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:03:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:03:41.401 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:03:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:03:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:03:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:03:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:03:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:03:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:03:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:03:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:03:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:03:41.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:03:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:03:41] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T11:03:41.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:03:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:03:41.398Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:03:42.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:03:42 smithi083 ceph-mon[79930]: pgmap v10063: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:42.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:03:42 smithi083 ceph-mon[77242]: pgmap v10063: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:03:42 smithi153 ceph-mon[70044]: pgmap v10063: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:42.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:03:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:03:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:03:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:03:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:03:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:03:45.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:03:44 smithi083 ceph-mon[79930]: pgmap v10064: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:45.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:03:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:03:44.771Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:03:45.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:03:44 smithi083 ceph-mon[77242]: pgmap v10064: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:45.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:03:44 smithi153 ceph-mon[70044]: pgmap v10064: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:45.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:03:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:03:45.486Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:03:47.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:03:46 smithi083 ceph-mon[79930]: pgmap v10065: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:47.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:03:46 smithi083 ceph-mon[77242]: pgmap v10065: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:47.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:03:46 smithi153 ceph-mon[70044]: pgmap v10065: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:49.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:03:48 smithi083 ceph-mon[77242]: pgmap v10066: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:49.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:03:48 smithi083 ceph-mon[79930]: pgmap v10066: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:49.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:03:48 smithi153 ceph-mon[70044]: pgmap v10066: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:50.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:03:50 smithi083 ceph-mon[79930]: pgmap v10067: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:50.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:03:50 smithi083 ceph-mon[77242]: pgmap v10067: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:03:50 smithi153 ceph-mon[70044]: pgmap v10067: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:51.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:03:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:03:51.400Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:03:51.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:03:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:03:51] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T11:03:52.506 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:03:52.796 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:03:52.796 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 119s ago 5h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:03:52.796 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 10m ago 5h 63.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:03:52.797 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 119s ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:03:52.797 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 10m ago 5h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:03:52.797 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 10m ago 5h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:03:52.797 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 119s ago 5h 799M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:03:52.797 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 119s ago 5h 1271M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:03:52.798 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 10m ago 5h 532M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:03:52.798 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 119s ago 5h 731M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:03:52.798 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 119s ago 5h 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:03:52.798 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 10m ago 5h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:03:52.799 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 119s ago 5h 895M 2447M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:03:52.799 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 119s ago 5h 885M 2447M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:03:52.799 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 119s ago 5h 690M 2447M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:03:52.799 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 119s ago 5h 864M 2447M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:03:52.800 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 10m ago 5h 740M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:03:52.800 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 10m ago 5h 922M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:03:52.800 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 10m ago 5h 823M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:03:52.800 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 10m ago 5h 975M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:03:52.800 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 10m ago 5h 191M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:03:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:03:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:03:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:03:53.149 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:03:53.150 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:03:53.150 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:03:53.150 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:03:53.150 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:03:53.151 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:03:53.151 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:03:53.151 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:03:53.151 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:03:53.152 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:03:53.152 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:03:53.152 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:03:53.152 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:03:53.152 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:03:53.153 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:03:53.153 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:03:53.153 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:03:53.153 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:03:53.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:03:52 smithi083 ceph-mon[77242]: pgmap v10068: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:53.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:03:52 smithi083 ceph-mon[79930]: pgmap v10068: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:53.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:03:52 smithi153 ceph-mon[70044]: pgmap v10068: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:03:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:03:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:03:54.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:03:53 smithi083 ceph-mon[79930]: from='client.55774 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:03:54.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:03:53 smithi083 ceph-mon[79930]: from='client.45849 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:03:54.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:03:53 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3662651978' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:03:54.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:03:53 smithi083 ceph-mon[77242]: from='client.55774 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:03:54.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:03:53 smithi083 ceph-mon[77242]: from='client.45849 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:03:54.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:03:53 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3662651978' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:03:54.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:03:53 smithi153 ceph-mon[70044]: from='client.55774 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:03:54.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:03:53 smithi153 ceph-mon[70044]: from='client.45849 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:03:54.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:03:53 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3662651978' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:03:55.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:03:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:03:54.771Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:03:55.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:03:54 smithi083 ceph-mon[79930]: pgmap v10069: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:55.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:03:54 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:03:55.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:03:54 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:03:55.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:03:54 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:03:55.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:03:54 smithi083 ceph-mon[77242]: pgmap v10069: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:55.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:03:54 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:03:55.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:03:54 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:03:55.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:03:54 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:03:55.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:03:54 smithi153 ceph-mon[70044]: pgmap v10069: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:55.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:03:54 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:03:55.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:03:54 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:03:55.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:03:54 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:03:55.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:03:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:03:55.486Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:03:57.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:03:56 smithi083 ceph-mon[77242]: pgmap v10070: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:57.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:03:56 smithi083 ceph-mon[79930]: pgmap v10070: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:57.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:03:56 smithi153 ceph-mon[70044]: pgmap v10070: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:59.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:03:58 smithi083 ceph-mon[79930]: pgmap v10071: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:59.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:03:58 smithi083 ceph-mon[77242]: pgmap v10071: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:03:59.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:03:58 smithi153 ceph-mon[70044]: pgmap v10071: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:00.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:04:00 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:04:00.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:04:00 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:04:00.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:04:00 smithi083 ceph-mon[79930]: pgmap v10072: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:00.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:04:00 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:04:00.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:04:00 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:04:00.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:04:00 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:04:00.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:04:00 smithi083 ceph-mon[77242]: pgmap v10072: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:00.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:04:00 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:04:00.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:04:00 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:04:00.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:04:00 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:04:00.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:04:00 smithi153 ceph-mon[70044]: pgmap v10072: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:00.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:04:00 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:04:01.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:04:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:04:01] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T11:04:01.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:04:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:04:01.400Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:04:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:04:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:04:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:04:03.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:04:02 smithi083 ceph-mon[79930]: pgmap v10073: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:03.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:04:02 smithi083 ceph-mon[77242]: pgmap v10073: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:03.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:04:02 smithi153 ceph-mon[70044]: pgmap v10073: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:04:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:04:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:04:05.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:04:04 smithi083 ceph-mon[79930]: pgmap v10074: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:05.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:04:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:04:04.772Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:04:05.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:04:04 smithi083 ceph-mon[77242]: pgmap v10074: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:05.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:04:04 smithi153 ceph-mon[70044]: pgmap v10074: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:05.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:04:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:04:05.488Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:04:07.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:04:06 smithi083 ceph-mon[79930]: pgmap v10075: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:07.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:04:06 smithi083 ceph-mon[77242]: pgmap v10075: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:07.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:04:06 smithi153 ceph-mon[70044]: pgmap v10075: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:09.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:04:08 smithi083 ceph-mon[79930]: pgmap v10076: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:09.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:04:08 smithi083 ceph-mon[77242]: pgmap v10076: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:09.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:04:08 smithi153 ceph-mon[70044]: pgmap v10076: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:10.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:04:10 smithi083 ceph-mon[79930]: pgmap v10077: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:10.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:04:10 smithi083 ceph-mon[77242]: pgmap v10077: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:04:10 smithi153 ceph-mon[70044]: pgmap v10077: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:11.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:04:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:04:11] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T11:04:11.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:04:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:04:11.401Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:04:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:04:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:04:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:04:13.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:04:12 smithi083 ceph-mon[79930]: pgmap v10078: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:13.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:04:12 smithi083 ceph-mon[77242]: pgmap v10078: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:13.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:04:12 smithi153 ceph-mon[70044]: pgmap v10078: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:04:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:04:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:04:15.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:04:14 smithi083 ceph-mon[79930]: pgmap v10079: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:15.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:04:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:04:14.773Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:04:15.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:04:14 smithi083 ceph-mon[77242]: pgmap v10079: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:15.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:04:14 smithi153 ceph-mon[70044]: pgmap v10079: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:15.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:04:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:04:15.488Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:04:17.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:04:16 smithi083 ceph-mon[79930]: pgmap v10080: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:17.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:04:16 smithi083 ceph-mon[77242]: pgmap v10080: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:17.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:04:16 smithi153 ceph-mon[70044]: pgmap v10080: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:19.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:04:18 smithi083 ceph-mon[79930]: pgmap v10081: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:19.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:04:18 smithi083 ceph-mon[77242]: pgmap v10081: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:19.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:04:18 smithi153 ceph-mon[70044]: pgmap v10081: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:20.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:04:20 smithi083 ceph-mon[79930]: pgmap v10082: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:20.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:04:20 smithi083 ceph-mon[77242]: pgmap v10082: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:04:20 smithi153 ceph-mon[70044]: pgmap v10082: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:21.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:04:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:04:21.402Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:04:21.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:04:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:04:21] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T11:04:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:04:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:04:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:04:23.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:04:22 smithi083 ceph-mon[79930]: pgmap v10083: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:23.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:04:22 smithi083 ceph-mon[77242]: pgmap v10083: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:23.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:04:22 smithi153 ceph-mon[70044]: pgmap v10083: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:04:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:04:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:04:23.489 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:04:23.783 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:04:23.783 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 2m ago 5h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:04:23.783 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 24s ago 5h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:04:23.783 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 2m ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:04:23.784 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 24s ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:04:23.784 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 24s ago 5h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:04:23.784 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 2m ago 5h 799M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:04:23.784 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 2m ago 5h 1271M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:04:23.785 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 24s ago 5h 548M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:04:23.785 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 2m ago 5h 731M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:04:23.785 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 2m ago 5h 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:04:23.785 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 24s ago 5h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:04:23.785 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 2m ago 5h 895M 2447M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:04:23.786 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 2m ago 5h 885M 2447M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:04:23.786 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 2m ago 5h 690M 2447M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:04:23.786 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 2m ago 5h 864M 2447M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:04:23.786 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 24s ago 5h 762M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:04:23.787 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 24s ago 5h 950M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:04:23.787 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 24s ago 5h 848M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:04:23.787 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 24s ago 5h 1029M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:04:23.787 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 24s ago 5h 149M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:04:24.144 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:04:24.145 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:04:24.145 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:04:24.145 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:04:24.146 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:04:24.146 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:04:24.146 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:04:24.146 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:04:24.147 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:04:24.147 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:04:24.147 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:04:24.147 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:04:24.147 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:04:24.148 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:04:24.148 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:04:24.148 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:04:24.149 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:04:24.149 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:04:25.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:04:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:04:24.774Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:04:25.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:04:24 smithi083 ceph-mon[79930]: from='client.45861 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:04:25.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:04:24 smithi083 ceph-mon[79930]: from='client.55798 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:04:25.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:04:24 smithi083 ceph-mon[79930]: pgmap v10084: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:25.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:04:24 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/392906622' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:04:25.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:04:24 smithi083 ceph-mon[77242]: from='client.45861 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:04:25.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:04:24 smithi083 ceph-mon[77242]: from='client.55798 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:04:25.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:04:24 smithi083 ceph-mon[77242]: pgmap v10084: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:25.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:04:24 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/392906622' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:04:25.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:04:24 smithi153 ceph-mon[70044]: from='client.45861 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:04:25.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:04:24 smithi153 ceph-mon[70044]: from='client.55798 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:04:25.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:04:24 smithi153 ceph-mon[70044]: pgmap v10084: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:25.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:04:24 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/392906622' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:04:25.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:04:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:04:25.489Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:04:27.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:04:26 smithi083 ceph-mon[79930]: pgmap v10085: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:27.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:04:26 smithi083 ceph-mon[77242]: pgmap v10085: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:27.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:04:26 smithi153 ceph-mon[70044]: pgmap v10085: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:29.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:04:28 smithi083 ceph-mon[79930]: pgmap v10086: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:29.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:04:28 smithi083 ceph-mon[77242]: pgmap v10086: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:29.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:04:28 smithi153 ceph-mon[70044]: pgmap v10086: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:30.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:04:30 smithi083 ceph-mon[79930]: pgmap v10087: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:30.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:04:30 smithi083 ceph-mon[77242]: pgmap v10087: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:04:30 smithi153 ceph-mon[70044]: pgmap v10087: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:31.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:04:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:04:31] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T11:04:31.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:04:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:04:31.402Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:04:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:04:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:04:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:04:33.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:04:32 smithi083 ceph-mon[77242]: pgmap v10088: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:33.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:04:32 smithi083 ceph-mon[79930]: pgmap v10088: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:33.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:04:32 smithi153 ceph-mon[70044]: pgmap v10088: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:04:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:04:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:04:35.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:04:34 smithi083 ceph-mon[79930]: pgmap v10089: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:35.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:04:34 smithi083 ceph-mon[77242]: pgmap v10089: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:35.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:04:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:04:34.774Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:04:35.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:04:34 smithi153 ceph-mon[70044]: pgmap v10089: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:35.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:04:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:04:35.490Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:04:37.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:04:36 smithi083 ceph-mon[79930]: pgmap v10090: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:37.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:04:36 smithi083 ceph-mon[77242]: pgmap v10090: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:37.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:04:36 smithi153 ceph-mon[70044]: pgmap v10090: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:39.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:04:38 smithi083 ceph-mon[79930]: pgmap v10091: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:39.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:04:38 smithi083 ceph-mon[77242]: pgmap v10091: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:39.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:04:38 smithi153 ceph-mon[70044]: pgmap v10091: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:40.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:04:40 smithi083 ceph-mon[79930]: pgmap v10092: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:40.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:04:40 smithi083 ceph-mon[77242]: pgmap v10092: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:04:40 smithi153 ceph-mon[70044]: pgmap v10092: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:41.404 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:04:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:04:41.404 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:04:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:04:41.404 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:04:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:04:41.404 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:04:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:04:41.405 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:04:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:04:41.405 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:04:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:04:41.405 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:04:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:04:41.406 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:04:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:04:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:04:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:04:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:04:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:04:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:04:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:04:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:04:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:04:41.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:04:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:04:41] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T11:04:41.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:04:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:04:41.403Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:04:42.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:04:42 smithi083 ceph-mon[79930]: pgmap v10093: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:42.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:04:42 smithi083 ceph-mon[77242]: pgmap v10093: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:04:42 smithi153 ceph-mon[70044]: pgmap v10093: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:42.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:04:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:04:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:04:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:04:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:04:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:04:45.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:04:44 smithi083 ceph-mon[79930]: pgmap v10094: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:45.216 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:04:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:04:44.775Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:04:45.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:04:44 smithi083 ceph-mon[77242]: pgmap v10094: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:45.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:04:44 smithi153 ceph-mon[70044]: pgmap v10094: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:45.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:04:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:04:45.490Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:04:47.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:04:46 smithi083 ceph-mon[77242]: pgmap v10095: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:47.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:04:46 smithi083 ceph-mon[79930]: pgmap v10095: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:47.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:04:46 smithi153 ceph-mon[70044]: pgmap v10095: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:49.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:04:48 smithi083 ceph-mon[79930]: pgmap v10096: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:49.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:04:48 smithi083 ceph-mon[77242]: pgmap v10096: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:49.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:04:48 smithi153 ceph-mon[70044]: pgmap v10096: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:50.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:04:50 smithi083 ceph-mon[79930]: pgmap v10097: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:50.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:04:50 smithi083 ceph-mon[77242]: pgmap v10097: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:04:50 smithi153 ceph-mon[70044]: pgmap v10097: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:51.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:04:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:04:51] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T11:04:51.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:04:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:04:51.403Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:04:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:04:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:04:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:04:53.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:04:52 smithi083 ceph-mon[77242]: pgmap v10098: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:53.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:04:52 smithi083 ceph-mon[79930]: pgmap v10098: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:53.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:04:52 smithi153 ceph-mon[70044]: pgmap v10098: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:04:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:04:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:04:54.490 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:04:54.784 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:04:54.784 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 3m ago 5h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:04:54.784 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 55s ago 5h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:04:54.785 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 3m ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:04:54.785 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 55s ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:04:54.785 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 55s ago 5h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:04:54.785 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 3m ago 5h 799M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:04:54.786 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 3m ago 5h 1271M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:04:54.786 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 55s ago 5h 548M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:04:54.786 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 3m ago 5h 731M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:04:54.786 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 3m ago 5h 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:04:54.787 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 55s ago 5h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:04:54.787 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 3m ago 5h 895M 2447M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:04:54.787 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 3m ago 5h 885M 2447M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:04:54.787 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 3m ago 5h 690M 2447M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:04:54.787 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 3m ago 5h 864M 2447M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:04:54.788 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 55s ago 5h 762M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:04:54.788 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 55s ago 5h 950M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:04:54.788 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 55s ago 5h 848M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:04:54.788 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 55s ago 5h 1029M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:04:54.788 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 55s ago 5h 149M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:04:55.141 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:04:55.142 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:04:55.142 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:04:55.143 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:04:55.143 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:04:55.143 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:04:55.143 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:04:55.143 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:04:55.144 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:04:55.144 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:04:55.144 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:04:55.144 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:04:55.145 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:04:55.145 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:04:55.145 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:04:55.145 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:04:55.146 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:04:55.146 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:04:55.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:04:54 smithi083 ceph-mon[79930]: pgmap v10099: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:55.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:04:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:04:54.776Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:04:55.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:04:54 smithi083 ceph-mon[77242]: pgmap v10099: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:55.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:04:54 smithi153 ceph-mon[70044]: pgmap v10099: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:55.860 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:04:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:04:55.490Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:04:55.861 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:04:55 smithi083 ceph-mon[77242]: from='client.55810 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:04:56.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:04:55 smithi083 ceph-mon[79930]: from='client.55810 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:04:56.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:04:55 smithi083 ceph-mon[79930]: from='client.45885 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:04:56.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:04:55 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2260091236' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:04:56.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:04:55 smithi083 ceph-mon[77242]: from='client.45885 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:04:56.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:04:55 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2260091236' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:04:56.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:04:55 smithi153 ceph-mon[70044]: from='client.55810 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:04:56.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:04:55 smithi153 ceph-mon[70044]: from='client.45885 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:04:56.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:04:55 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2260091236' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:04:57.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:04:56 smithi083 ceph-mon[77242]: pgmap v10100: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:57.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:04:56 smithi083 ceph-mon[79930]: pgmap v10100: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:57.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:04:56 smithi153 ceph-mon[70044]: pgmap v10100: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:59.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:04:58 smithi083 ceph-mon[79930]: pgmap v10101: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:59.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:04:58 smithi083 ceph-mon[77242]: pgmap v10101: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:04:59.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:04:58 smithi153 ceph-mon[70044]: pgmap v10101: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:00.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:05:00 smithi083 ceph-mon[79930]: pgmap v10102: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:00.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:05:00 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:05:00.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:05:00 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:05:00.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:05:00 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:05:00.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:05:00 smithi083 ceph-mon[77242]: pgmap v10102: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:00.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:05:00 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:05:00.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:05:00 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:05:00.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:05:00 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:05:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:05:00 smithi153 ceph-mon[70044]: pgmap v10102: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:05:00 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:05:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:05:00 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:05:00.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:05:00 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:05:01.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:05:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:05:01] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T11:05:01.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:05:01 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:05:01.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:05:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:05:01.403Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:05:01.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:05:01 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:05:01.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:05:01 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:05:02.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:05:02 smithi153 ceph-mon[70044]: pgmap v10103: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:05:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:05:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:05:02.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:05:02 smithi083 ceph-mon[79930]: pgmap v10103: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:02.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:05:02 smithi083 ceph-mon[77242]: pgmap v10103: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:05:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:05:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:05:05.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:05:04 smithi083 ceph-mon[79930]: pgmap v10104: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:05.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:05:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:05:04.776Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:05:05.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:05:04 smithi083 ceph-mon[77242]: pgmap v10104: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:05.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:05:04 smithi153 ceph-mon[70044]: pgmap v10104: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:05.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:05:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:05:05.491Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:05:07.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:05:06 smithi083 ceph-mon[77242]: pgmap v10105: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:07.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:05:06 smithi083 ceph-mon[79930]: pgmap v10105: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:07.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:05:06 smithi153 ceph-mon[70044]: pgmap v10105: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:09.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:05:08 smithi083 ceph-mon[79930]: pgmap v10106: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:09.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:05:08 smithi083 ceph-mon[77242]: pgmap v10106: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:09.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:05:08 smithi153 ceph-mon[70044]: pgmap v10106: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:10.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:05:10 smithi083 ceph-mon[79930]: pgmap v10107: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:10.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:05:10 smithi083 ceph-mon[77242]: pgmap v10107: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:05:10 smithi153 ceph-mon[70044]: pgmap v10107: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:11.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:05:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:05:11] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T11:05:11.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:05:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:05:11.404Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:05:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:05:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:05:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:05:13.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:05:12 smithi083 ceph-mon[77242]: pgmap v10108: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:13.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:05:12 smithi083 ceph-mon[79930]: pgmap v10108: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:13.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:05:12 smithi153 ceph-mon[70044]: pgmap v10108: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:05:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:05:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:05:15.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:05:14 smithi083 ceph-mon[79930]: pgmap v10109: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:15.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:05:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:05:14.778Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:05:15.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:05:14 smithi083 ceph-mon[77242]: pgmap v10109: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:15.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:05:14 smithi153 ceph-mon[70044]: pgmap v10109: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:15.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:05:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:05:15.492Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:05:17.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:05:16 smithi083 ceph-mon[79930]: pgmap v10110: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:17.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:05:16 smithi083 ceph-mon[77242]: pgmap v10110: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:17.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:05:16 smithi153 ceph-mon[70044]: pgmap v10110: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:19.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:05:18 smithi083 ceph-mon[79930]: pgmap v10111: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:19.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:05:18 smithi083 ceph-mon[77242]: pgmap v10111: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:19.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:05:18 smithi153 ceph-mon[70044]: pgmap v10111: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:20.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:05:20 smithi083 ceph-mon[79930]: pgmap v10112: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:20.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:05:20 smithi083 ceph-mon[77242]: pgmap v10112: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:05:20 smithi153 ceph-mon[70044]: pgmap v10112: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:21.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:05:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:05:21] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T11:05:21.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:05:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:05:21.404Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:05:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:05:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:05:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:05:23.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:05:22 smithi083 ceph-mon[77242]: pgmap v10113: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:23.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:05:22 smithi083 ceph-mon[79930]: pgmap v10113: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:23.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:05:22 smithi153 ceph-mon[70044]: pgmap v10113: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:05:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:05:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:05:25.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:05:24 smithi083 ceph-mon[79930]: pgmap v10114: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:25.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:05:24 smithi083 ceph-mon[77242]: pgmap v10114: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:25.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:05:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:05:24.778Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:05:25.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:05:24 smithi153 ceph-mon[70044]: pgmap v10114: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:25.468 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:05:25.760 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:05:25.761 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 3m ago 5h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:05:25.761 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 86s ago 5h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:05:25.761 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 3m ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:05:25.762 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 86s ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:05:25.762 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 86s ago 5h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:05:25.762 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 3m ago 5h 799M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:05:25.763 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 3m ago 5h 1271M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:05:25.763 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 86s ago 5h 548M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:05:25.763 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 3m ago 5h 731M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:05:25.763 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 3m ago 5h 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:05:25.764 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 86s ago 5h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:05:25.764 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 3m ago 5h 895M 2447M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:05:25.764 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 3m ago 5h 885M 2447M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:05:25.764 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 3m ago 5h 690M 2447M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:05:25.764 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 3m ago 5h 864M 2447M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:05:25.765 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 86s ago 5h 762M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:05:25.765 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 86s ago 5h 950M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:05:25.765 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 86s ago 5h 848M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:05:25.765 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 86s ago 5h 1029M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:05:25.766 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 86s ago 5h 149M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:05:25.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:05:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:05:25.492Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:05:26.118 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:05:26.119 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:05:26.119 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:05:26.119 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:05:26.119 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:05:26.120 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:05:26.120 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:05:26.120 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:05:26.120 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:05:26.121 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:05:26.121 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:05:26.121 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:05:26.121 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:05:26.122 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:05:26.122 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:05:26.122 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:05:26.122 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:05:26.123 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:05:27.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:05:26 smithi083 ceph-mon[79930]: from='client.55828 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:05:27.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:05:26 smithi083 ceph-mon[79930]: from='client.55834 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:05:27.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:05:26 smithi083 ceph-mon[79930]: pgmap v10115: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:27.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:05:26 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/527484750' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:05:27.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:05:26 smithi083 ceph-mon[77242]: from='client.55828 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:05:27.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:05:26 smithi083 ceph-mon[77242]: from='client.55834 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:05:27.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:05:26 smithi083 ceph-mon[77242]: pgmap v10115: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:27.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:05:26 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/527484750' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:05:27.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:05:26 smithi153 ceph-mon[70044]: from='client.55828 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:05:27.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:05:26 smithi153 ceph-mon[70044]: from='client.55834 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:05:27.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:05:26 smithi153 ceph-mon[70044]: pgmap v10115: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:27.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:05:26 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/527484750' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:05:29.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:05:28 smithi083 ceph-mon[77242]: pgmap v10116: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:29.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:05:28 smithi083 ceph-mon[79930]: pgmap v10116: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:29.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:05:28 smithi153 ceph-mon[70044]: pgmap v10116: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:30.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:05:30 smithi083 ceph-mon[79930]: pgmap v10117: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:30.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:05:30 smithi083 ceph-mon[77242]: pgmap v10117: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:05:30 smithi153 ceph-mon[70044]: pgmap v10117: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:31.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:05:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:05:31] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T11:05:31.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:05:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:05:31.405Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:05:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:05:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:05:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:05:33.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:05:32 smithi083 ceph-mon[79930]: pgmap v10118: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:33.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:05:32 smithi083 ceph-mon[77242]: pgmap v10118: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:33.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:05:32 smithi153 ceph-mon[70044]: pgmap v10118: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:05:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:05:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:05:35.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:05:34 smithi083 ceph-mon[79930]: pgmap v10119: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:35.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:05:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:05:34.779Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:05:35.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:05:34 smithi083 ceph-mon[77242]: pgmap v10119: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:35.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:05:34 smithi153 ceph-mon[70044]: pgmap v10119: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:35.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:05:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:05:35.494Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:05:37.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:05:36 smithi083 ceph-mon[79930]: pgmap v10120: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:37.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:05:36 smithi083 ceph-mon[77242]: pgmap v10120: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:37.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:05:36 smithi153 ceph-mon[70044]: pgmap v10120: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:39.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:05:38 smithi083 ceph-mon[79930]: pgmap v10121: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:39.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:05:38 smithi083 ceph-mon[77242]: pgmap v10121: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:39.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:05:38 smithi153 ceph-mon[70044]: pgmap v10121: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:40.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:05:40 smithi083 ceph-mon[79930]: pgmap v10122: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:40.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:05:40 smithi083 ceph-mon[77242]: pgmap v10122: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:05:40 smithi153 ceph-mon[70044]: pgmap v10122: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:41.407 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:05:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:05:41.408 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:05:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:05:41.408 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:05:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:05:41.408 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:05:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:05:41.409 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:05:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:05:41.409 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:05:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:05:41.409 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:05:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:05:41.409 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:05:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:05:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:05:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:05:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:05:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:05:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:05:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:05:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:05:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:05:41.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:05:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:05:41] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T11:05:41.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:05:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:05:41.406Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:05:42.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:05:42 smithi083 ceph-mon[79930]: pgmap v10123: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:42.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:05:42 smithi083 ceph-mon[77242]: pgmap v10123: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:05:42 smithi153 ceph-mon[70044]: pgmap v10123: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:42.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:05:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:05:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:05:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:05:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:05:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:05:45.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:05:44 smithi083 ceph-mon[79930]: pgmap v10124: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:45.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:05:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:05:44.779Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:05:45.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:05:44 smithi083 ceph-mon[77242]: pgmap v10124: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:45.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:05:44 smithi153 ceph-mon[70044]: pgmap v10124: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:45.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:05:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:05:45.495Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:05:47.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:05:46 smithi083 ceph-mon[79930]: pgmap v10125: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:47.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:05:46 smithi083 ceph-mon[77242]: pgmap v10125: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:47.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:05:46 smithi153 ceph-mon[70044]: pgmap v10125: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:49.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:05:48 smithi083 ceph-mon[77242]: pgmap v10126: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:49.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:05:48 smithi083 ceph-mon[79930]: pgmap v10126: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:49.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:05:48 smithi153 ceph-mon[70044]: pgmap v10126: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:50.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:05:50 smithi083 ceph-mon[79930]: pgmap v10127: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:50.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:05:50 smithi083 ceph-mon[77242]: pgmap v10127: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:05:50 smithi153 ceph-mon[70044]: pgmap v10127: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:51.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:05:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:05:51] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T11:05:51.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:05:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:05:51.407Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:05:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:05:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:05:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:05:53.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:05:52 smithi083 ceph-mon[77242]: pgmap v10128: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:53.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:05:52 smithi083 ceph-mon[79930]: pgmap v10128: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:53.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:05:52 smithi153 ceph-mon[70044]: pgmap v10128: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:05:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:05:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:05:55.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:05:54 smithi083 ceph-mon[79930]: pgmap v10129: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:55.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:05:54 smithi083 ceph-mon[77242]: pgmap v10129: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:55.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:05:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:05:54.779Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:05:55.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:05:54 smithi153 ceph-mon[70044]: pgmap v10129: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:55.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:05:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:05:55.496Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:05:56.442 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:05:56.729 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:05:56.729 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 4m ago 5h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:05:56.729 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 117s ago 5h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:05:56.729 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 4m ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:05:56.730 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 117s ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:05:56.730 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 117s ago 5h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:05:56.730 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 4m ago 5h 799M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:05:56.730 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 4m ago 5h 1271M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:05:56.731 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 117s ago 5h 548M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:05:56.731 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 4m ago 5h 731M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:05:56.731 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 4m ago 5h 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:05:56.731 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 117s ago 5h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:05:56.731 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 4m ago 5h 895M 2447M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:05:56.732 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 4m ago 5h 885M 2447M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:05:56.732 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 4m ago 5h 690M 2447M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:05:56.732 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 4m ago 5h 864M 2447M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:05:56.732 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 117s ago 5h 762M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:05:56.733 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 117s ago 5h 950M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:05:56.733 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 117s ago 5h 848M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:05:56.733 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 117s ago 5h 1029M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:05:56.733 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 117s ago 5h 149M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:05:57.091 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:05:57.091 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:05:57.092 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:05:57.092 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:05:57.092 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:05:57.092 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:05:57.092 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:05:57.093 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:05:57.093 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:05:57.093 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:05:57.093 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:05:57.094 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:05:57.094 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:05:57.094 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:05:57.094 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:05:57.095 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:05:57.095 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:05:57.095 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:05:57.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:05:56 smithi083 ceph-mon[77242]: pgmap v10130: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:57.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:05:56 smithi083 ceph-mon[79930]: pgmap v10130: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:57.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:05:56 smithi153 ceph-mon[70044]: pgmap v10130: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:58.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:05:57 smithi083 ceph-mon[79930]: from='client.55846 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:05:58.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:05:57 smithi083 ceph-mon[79930]: from='client.45921 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:05:58.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:05:57 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/381995876' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:05:58.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:05:57 smithi083 ceph-mon[77242]: from='client.55846 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:05:58.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:05:57 smithi083 ceph-mon[77242]: from='client.45921 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:05:58.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:05:57 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/381995876' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:05:58.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:05:57 smithi153 ceph-mon[70044]: from='client.55846 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:05:58.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:05:57 smithi153 ceph-mon[70044]: from='client.45921 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:05:58.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:05:57 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/381995876' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:05:59.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:05:58 smithi083 ceph-mon[79930]: pgmap v10131: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:59.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:05:58 smithi083 ceph-mon[77242]: pgmap v10131: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:05:59.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:05:58 smithi153 ceph-mon[70044]: pgmap v10131: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:00.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:06:00 smithi083 ceph-mon[77242]: pgmap v10132: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:00.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:06:00 smithi083 ceph-mon[79930]: pgmap v10132: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:06:00 smithi153 ceph-mon[70044]: pgmap v10132: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:01.408 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:06:01 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:06:01.408 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:06:01 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:06:01.408 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:06:01 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:06:01.409 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:06:01 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:06:01.409 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:06:01 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:06:01.409 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:06:01 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:06:01.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:06:01 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:06:01.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:06:01 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:06:01.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:06:01 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:06:01.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:06:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:06:01] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T11:06:01.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:06:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:06:01.407Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:06:02.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:06:02 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:06:02.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:06:02 smithi083 ceph-mon[79930]: pgmap v10133: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:02.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:06:02 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:06:02.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:06:02 smithi083 ceph-mon[77242]: pgmap v10133: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:06:02 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:06:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:06:02 smithi153 ceph-mon[70044]: pgmap v10133: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:02.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:06:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:06:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:06:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:06:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:06:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:06:05.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:06:04 smithi083 ceph-mon[79930]: pgmap v10134: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:05.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:06:04 smithi083 ceph-mon[77242]: pgmap v10134: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:05.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:06:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:06:04.779Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:06:05.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:06:04 smithi153 ceph-mon[70044]: pgmap v10134: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:05.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:06:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:06:05.496Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:06:07.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:06:06 smithi083 ceph-mon[77242]: pgmap v10135: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:07.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:06:06 smithi083 ceph-mon[79930]: pgmap v10135: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:07.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:06:06 smithi153 ceph-mon[70044]: pgmap v10135: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:09.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:06:08 smithi083 ceph-mon[79930]: pgmap v10136: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:09.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:06:08 smithi083 ceph-mon[77242]: pgmap v10136: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:09.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:06:08 smithi153 ceph-mon[70044]: pgmap v10136: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:10.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:06:10 smithi083 ceph-mon[79930]: pgmap v10137: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:10.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:06:10 smithi083 ceph-mon[77242]: pgmap v10137: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:06:10 smithi153 ceph-mon[70044]: pgmap v10137: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:11.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:06:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:06:11] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T11:06:11.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:06:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:06:11.408Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:06:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:06:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:06:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:06:13.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:06:12 smithi083 ceph-mon[79930]: pgmap v10138: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:13.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:06:12 smithi083 ceph-mon[77242]: pgmap v10138: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:13.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:06:12 smithi153 ceph-mon[70044]: pgmap v10138: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:06:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:06:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:06:15.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:06:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:06:14.780Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:06:15.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:06:14 smithi083 ceph-mon[77242]: pgmap v10139: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:15.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:06:14 smithi083 ceph-mon[79930]: pgmap v10139: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:15.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:06:14 smithi153 ceph-mon[70044]: pgmap v10139: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:15.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:06:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:06:15.497Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:06:17.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:06:16 smithi083 ceph-mon[77242]: pgmap v10140: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:17.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:06:16 smithi083 ceph-mon[79930]: pgmap v10140: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:17.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:06:16 smithi153 ceph-mon[70044]: pgmap v10140: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:19.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:06:18 smithi083 ceph-mon[79930]: pgmap v10141: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:19.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:06:18 smithi083 ceph-mon[77242]: pgmap v10141: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:19.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:06:18 smithi153 ceph-mon[70044]: pgmap v10141: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:20.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:06:20 smithi083 ceph-mon[79930]: pgmap v10142: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:20.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:06:20 smithi083 ceph-mon[77242]: pgmap v10142: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:06:20 smithi153 ceph-mon[70044]: pgmap v10142: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:21.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:06:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:06:21] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T11:06:21.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:06:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:06:21.409Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:06:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:06:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:06:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:06:23.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:06:22 smithi083 ceph-mon[77242]: pgmap v10143: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:23.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:06:22 smithi083 ceph-mon[79930]: pgmap v10143: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:23.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:06:22 smithi153 ceph-mon[70044]: pgmap v10143: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:06:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:06:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:06:25.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:06:24 smithi083 ceph-mon[79930]: pgmap v10144: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:25.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:06:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:06:24.781Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:06:25.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:06:24 smithi083 ceph-mon[77242]: pgmap v10144: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:25.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:06:24 smithi153 ceph-mon[70044]: pgmap v10144: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:25.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:06:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:06:25.498Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:06:27.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:06:26 smithi083 ceph-mon[79930]: pgmap v10145: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:27.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:06:26 smithi083 ceph-mon[77242]: pgmap v10145: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:27.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:06:26 smithi153 ceph-mon[70044]: pgmap v10145: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:27.412 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:06:27.699 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:06:27.700 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 4m ago 5h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:06:27.700 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 2m ago 5h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:06:27.700 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 4m ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:06:27.700 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 2m ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:06:27.701 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 2m ago 5h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:06:27.701 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 4m ago 5h 799M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:06:27.701 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 4m ago 5h 1271M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:06:27.701 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 2m ago 5h 548M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:06:27.701 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 4m ago 5h 731M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:06:27.702 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 4m ago 5h 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:06:27.702 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 2m ago 5h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:06:27.702 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 4m ago 5h 895M 2447M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:06:27.702 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 4m ago 5h 885M 2447M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:06:27.703 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 4m ago 5h 690M 2447M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:06:27.703 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 4m ago 5h 864M 2447M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:06:27.703 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 2m ago 5h 762M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:06:27.704 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 2m ago 5h 950M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:06:27.704 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 2m ago 5h 848M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:06:27.704 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 2m ago 5h 1029M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:06:27.705 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 2m ago 5h 149M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:06:28.056 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:06:28.056 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:06:28.057 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:06:28.057 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:06:28.057 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:06:28.057 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:06:28.057 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:06:28.058 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:06:28.058 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:06:28.058 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:06:28.058 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:06:28.059 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:06:28.059 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:06:28.059 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:06:28.059 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:06:28.059 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:06:28.060 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:06:28.060 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:06:29.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:06:28 smithi083 ceph-mon[77242]: from='client.45933 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:06:29.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:06:28 smithi083 ceph-mon[77242]: from='client.55870 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:06:29.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:06:28 smithi083 ceph-mon[77242]: pgmap v10146: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:29.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:06:28 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/222593607' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:06:29.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:06:28 smithi083 ceph-mon[79930]: from='client.45933 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:06:29.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:06:28 smithi083 ceph-mon[79930]: from='client.55870 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:06:29.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:06:28 smithi083 ceph-mon[79930]: pgmap v10146: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:29.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:06:28 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/222593607' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:06:29.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:06:28 smithi153 ceph-mon[70044]: from='client.45933 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:06:29.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:06:28 smithi153 ceph-mon[70044]: from='client.55870 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:06:29.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:06:28 smithi153 ceph-mon[70044]: pgmap v10146: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:29.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:06:28 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/222593607' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:06:30.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:06:30 smithi083 ceph-mon[79930]: pgmap v10147: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:30.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:06:30 smithi083 ceph-mon[77242]: pgmap v10147: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:06:30 smithi153 ceph-mon[70044]: pgmap v10147: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:31.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:06:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:06:31] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T11:06:31.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:06:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:06:31.409Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:06:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:06:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:06:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:06:33.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:06:32 smithi083 ceph-mon[79930]: pgmap v10148: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:33.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:06:32 smithi083 ceph-mon[77242]: pgmap v10148: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:33.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:06:32 smithi153 ceph-mon[70044]: pgmap v10148: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:06:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:06:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:06:35.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:06:34 smithi083 ceph-mon[79930]: pgmap v10149: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:35.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:06:34 smithi083 ceph-mon[77242]: pgmap v10149: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:35.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:06:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:06:34.781Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:06:35.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:06:34 smithi153 ceph-mon[70044]: pgmap v10149: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:35.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:06:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:06:35.499Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:06:37.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:06:36 smithi083 ceph-mon[79930]: pgmap v10150: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:37.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:06:36 smithi083 ceph-mon[77242]: pgmap v10150: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:37.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:06:36 smithi153 ceph-mon[70044]: pgmap v10150: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:39.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:06:38 smithi083 ceph-mon[77242]: pgmap v10151: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:39.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:06:38 smithi083 ceph-mon[79930]: pgmap v10151: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:39.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:06:38 smithi153 ceph-mon[70044]: pgmap v10151: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:40.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:06:40 smithi083 ceph-mon[77242]: pgmap v10152: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:40.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:06:40 smithi083 ceph-mon[79930]: pgmap v10152: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:06:40 smithi153 ceph-mon[70044]: pgmap v10152: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:41.411 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:06:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:06:41.411 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:06:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:06:41.411 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:06:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:06:41.412 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:06:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:06:41.412 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:06:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:06:41.412 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:06:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:06:41.412 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:06:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:06:41.413 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:06:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:06:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:06:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:06:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:06:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:06:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:06:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:06:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:06:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:06:41.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:06:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:06:41] "GET /metrics HTTP/1.1" 200 225229 "" "Prometheus/2.33.4" 2022-09-29T11:06:41.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:06:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:06:41.410Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:06:42.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:06:42 smithi083 ceph-mon[77242]: pgmap v10153: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:42.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:06:42 smithi083 ceph-mon[79930]: pgmap v10153: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:06:42 smithi153 ceph-mon[70044]: pgmap v10153: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:42.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:06:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:06:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:06:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:06:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:06:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:06:45.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:06:44 smithi083 ceph-mon[79930]: pgmap v10154: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:45.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:06:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:06:44.782Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:06:45.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:06:44 smithi083 ceph-mon[77242]: pgmap v10154: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:45.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:06:44 smithi153 ceph-mon[70044]: pgmap v10154: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:45.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:06:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:06:45.500Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:06:47.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:06:46 smithi083 ceph-mon[77242]: pgmap v10155: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:47.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:06:46 smithi083 ceph-mon[79930]: pgmap v10155: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:47.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:06:46 smithi153 ceph-mon[70044]: pgmap v10155: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:49.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:06:48 smithi083 ceph-mon[77242]: pgmap v10156: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:49.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:06:48 smithi083 ceph-mon[79930]: pgmap v10156: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:49.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:06:48 smithi153 ceph-mon[70044]: pgmap v10156: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:50.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:06:50 smithi083 ceph-mon[77242]: pgmap v10157: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:50.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:06:50 smithi083 ceph-mon[79930]: pgmap v10157: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:06:50 smithi153 ceph-mon[70044]: pgmap v10157: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:51.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:06:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:06:51] "GET /metrics HTTP/1.1" 200 225229 "" "Prometheus/2.33.4" 2022-09-29T11:06:51.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:06:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:06:51.411Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:06:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:06:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:06:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:06:53.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:06:52 smithi083 ceph-mon[77242]: pgmap v10158: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:53.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:06:52 smithi083 ceph-mon[79930]: pgmap v10158: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:53.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:06:52 smithi153 ceph-mon[70044]: pgmap v10158: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:06:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:06:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:06:55.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:06:54 smithi083 ceph-mon[77242]: pgmap v10159: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:55.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:06:54 smithi083 ceph-mon[79930]: pgmap v10159: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:55.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:06:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:06:54.783Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:06:55.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:06:54 smithi153 ceph-mon[70044]: pgmap v10159: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:55.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:06:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:06:55.501Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:06:57.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:06:56 smithi083 ceph-mon[79930]: pgmap v10160: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:57.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:06:56 smithi083 ceph-mon[77242]: pgmap v10160: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:57.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:06:56 smithi153 ceph-mon[70044]: pgmap v10160: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:58.382 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:06:58.668 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:06:58.669 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 5m ago 5h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:06:58.669 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 2m ago 5h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:06:58.669 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 5m ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:06:58.670 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 2m ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:06:58.670 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 2m ago 5h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:06:58.670 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 5m ago 5h 799M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:06:58.671 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 5m ago 5h 1271M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:06:58.671 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 2m ago 5h 548M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:06:58.671 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 5m ago 5h 731M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:06:58.671 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 5m ago 5h 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:06:58.672 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 2m ago 5h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:06:58.672 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 5m ago 5h 895M 2447M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:06:58.672 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 5m ago 5h 885M 2447M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:06:58.672 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 5m ago 5h 690M 2447M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:06:58.673 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 5m ago 5h 864M 2447M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:06:58.673 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 2m ago 5h 762M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:06:58.673 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 2m ago 5h 950M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:06:58.673 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 2m ago 5h 848M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:06:58.673 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 2m ago 5h 1029M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:06:58.674 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 2m ago 5h 149M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:06:59.025 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:06:59.026 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:06:59.026 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:06:59.027 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:06:59.027 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:06:59.027 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:06:59.027 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:06:59.027 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:06:59.028 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:06:59.028 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:06:59.028 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:06:59.028 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:06:59.029 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:06:59.029 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:06:59.029 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:06:59.029 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:06:59.029 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:06:59.030 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:06:59.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:06:58 smithi083 ceph-mon[79930]: pgmap v10161: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:59.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:06:58 smithi083 ceph-mon[77242]: pgmap v10161: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:06:59.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:06:58 smithi153 ceph-mon[70044]: pgmap v10161: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:00.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:06:59 smithi083 ceph-mon[77242]: from='client.55882 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:07:00.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:06:59 smithi083 ceph-mon[77242]: from='client.55888 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:07:00.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:06:59 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4121799938' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:07:00.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:06:59 smithi083 ceph-mon[79930]: from='client.55882 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:07:00.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:06:59 smithi083 ceph-mon[79930]: from='client.55888 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:07:00.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:06:59 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4121799938' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:07:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:06:59 smithi153 ceph-mon[70044]: from='client.55882 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:07:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:06:59 smithi153 ceph-mon[70044]: from='client.55888 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:07:00.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:06:59 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4121799938' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:07:01.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:07:00 smithi083 ceph-mon[77242]: pgmap v10162: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:01.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:07:00 smithi083 ceph-mon[79930]: pgmap v10162: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:01.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:07:00 smithi153 ceph-mon[70044]: pgmap v10162: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:01.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:07:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:07:01] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T11:07:01.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:07:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:07:01.410Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:07:02.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:07:01 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:07:02.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:07:01 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:07:02.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:07:01 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:07:02.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:07:01 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:07:02.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:07:01 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:07:02.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:07:01 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:07:02.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:07:01 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:07:02.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:07:01 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:07:02.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:07:01 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:07:02.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:07:01 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:07:02.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:07:01 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:07:02.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:07:01 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:07:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:07:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:07:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:07:03.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:07:02 smithi083 ceph-mon[79930]: pgmap v10163: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:03.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:07:02 smithi083 ceph-mon[77242]: pgmap v10163: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:03.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:07:02 smithi153 ceph-mon[70044]: pgmap v10163: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:07:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:07:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:07:05.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:07:04 smithi083 ceph-mon[79930]: pgmap v10164: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:05.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:07:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:07:04.784Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:07:05.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:07:04 smithi083 ceph-mon[77242]: pgmap v10164: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:05.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:07:04 smithi153 ceph-mon[70044]: pgmap v10164: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:05.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:07:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:07:05.502Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:07:07.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:07:06 smithi083 ceph-mon[79930]: pgmap v10165: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:07.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:07:06 smithi083 ceph-mon[77242]: pgmap v10165: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:07.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:07:06 smithi153 ceph-mon[70044]: pgmap v10165: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:09.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:07:08 smithi083 ceph-mon[79930]: pgmap v10166: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:09.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:07:08 smithi083 ceph-mon[77242]: pgmap v10166: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:09.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:07:08 smithi153 ceph-mon[70044]: pgmap v10166: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:10.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:07:10 smithi083 ceph-mon[79930]: pgmap v10167: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:10.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:07:10 smithi083 ceph-mon[77242]: pgmap v10167: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:07:10 smithi153 ceph-mon[70044]: pgmap v10167: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:11.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:07:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:07:11] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T11:07:11.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:07:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:07:11.411Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:07:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:07:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:07:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:07:13.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:07:12 smithi083 ceph-mon[79930]: pgmap v10168: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:13.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:07:12 smithi083 ceph-mon[77242]: pgmap v10168: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:07:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:07:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:07:13.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:07:12 smithi153 ceph-mon[70044]: pgmap v10168: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:15.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:07:14 smithi083 ceph-mon[79930]: pgmap v10169: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:15.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:07:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:07:14.785Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:07:15.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:07:14 smithi083 ceph-mon[77242]: pgmap v10169: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:15.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:07:14 smithi153 ceph-mon[70044]: pgmap v10169: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:15.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:07:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:07:15.503Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:07:17.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:07:16 smithi083 ceph-mon[79930]: pgmap v10170: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:17.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:07:16 smithi083 ceph-mon[77242]: pgmap v10170: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:17.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:07:16 smithi153 ceph-mon[70044]: pgmap v10170: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:19.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:07:18 smithi083 ceph-mon[77242]: pgmap v10171: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:19.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:07:18 smithi083 ceph-mon[79930]: pgmap v10171: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:19.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:07:18 smithi153 ceph-mon[70044]: pgmap v10171: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:20.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:07:20 smithi083 ceph-mon[79930]: pgmap v10172: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:20.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:07:20 smithi083 ceph-mon[77242]: pgmap v10172: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:07:20 smithi153 ceph-mon[70044]: pgmap v10172: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:21.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:07:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:07:21] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T11:07:21.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:07:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:07:21.412Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:07:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:07:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:07:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:07:23.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:07:22 smithi083 ceph-mon[77242]: pgmap v10173: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:23.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:07:22 smithi083 ceph-mon[79930]: pgmap v10173: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:23.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:07:22 smithi153 ceph-mon[70044]: pgmap v10173: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:07:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:07:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:07:25.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:07:24 smithi083 ceph-mon[79930]: pgmap v10174: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:25.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:07:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:07:24.786Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:07:25.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:07:24 smithi083 ceph-mon[77242]: pgmap v10174: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:25.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:07:24 smithi153 ceph-mon[70044]: pgmap v10174: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:25.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:07:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:07:25.503Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:07:27.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:07:26 smithi083 ceph-mon[79930]: pgmap v10175: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:27.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:07:26 smithi083 ceph-mon[77242]: pgmap v10175: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:27.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:07:26 smithi153 ceph-mon[70044]: pgmap v10175: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:29.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:07:28 smithi083 ceph-mon[77242]: pgmap v10176: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:29.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:07:28 smithi083 ceph-mon[79930]: pgmap v10176: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:29.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:07:28 smithi153 ceph-mon[70044]: pgmap v10176: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:29.354 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:07:29.646 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:07:29.647 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 5m ago 5h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:07:29.647 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 3m ago 5h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:07:29.647 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 5m ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:07:29.647 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 3m ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:07:29.648 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 3m ago 5h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:07:29.648 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 5m ago 5h 799M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:07:29.648 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 5m ago 5h 1271M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:07:29.648 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 3m ago 5h 548M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:07:29.648 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 5m ago 5h 731M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:07:29.649 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 5m ago 5h 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:07:29.649 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 3m ago 5h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:07:29.649 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 5m ago 5h 895M 2447M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:07:29.649 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 5m ago 5h 885M 2447M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:07:29.649 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 5m ago 5h 690M 2447M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:07:29.650 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 5m ago 5h 864M 2447M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:07:29.650 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 3m ago 5h 762M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:07:29.650 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 3m ago 5h 950M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:07:29.650 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 3m ago 5h 848M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:07:29.651 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 3m ago 5h 1029M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:07:29.651 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 3m ago 5h 149M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:07:30.002 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:07:30.002 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:07:30.003 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:07:30.003 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:07:30.003 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:07:30.003 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:07:30.003 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:07:30.004 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:07:30.004 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:07:30.004 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:07:30.004 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:07:30.005 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:07:30.005 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:07:30.005 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:07:30.005 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:07:30.005 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:07:30.006 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:07:30.006 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:07:30.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:07:30 smithi083 ceph-mon[79930]: from='client.55900 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:07:30.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:07:30 smithi083 ceph-mon[79930]: from='client.55906 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:07:30.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:07:30 smithi083 ceph-mon[79930]: pgmap v10177: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:30.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:07:30 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1206447775' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:07:30.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:07:30 smithi083 ceph-mon[77242]: from='client.55900 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:07:30.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:07:30 smithi083 ceph-mon[77242]: from='client.55906 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:07:30.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:07:30 smithi083 ceph-mon[77242]: pgmap v10177: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:30.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:07:30 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1206447775' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:07:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:07:30 smithi153 ceph-mon[70044]: from='client.55900 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:07:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:07:30 smithi153 ceph-mon[70044]: from='client.55906 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:07:30.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:07:30 smithi153 ceph-mon[70044]: pgmap v10177: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:30.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:07:30 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1206447775' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:07:31.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:07:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:07:31.412Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:07:31.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:07:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:07:31] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T11:07:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:07:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:07:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:07:33.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:07:32 smithi083 ceph-mon[79930]: pgmap v10178: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:33.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:07:32 smithi083 ceph-mon[77242]: pgmap v10178: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:33.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:07:32 smithi153 ceph-mon[70044]: pgmap v10178: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:07:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:07:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:07:35.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:07:34 smithi083 ceph-mon[77242]: pgmap v10179: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:35.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:07:34 smithi083 ceph-mon[79930]: pgmap v10179: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:35.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:07:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:07:34.786Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:07:35.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:07:34 smithi153 ceph-mon[70044]: pgmap v10179: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:35.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:07:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:07:35.504Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:07:37.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:07:36 smithi083 ceph-mon[79930]: pgmap v10180: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:37.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:07:36 smithi083 ceph-mon[77242]: pgmap v10180: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:37.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:07:36 smithi153 ceph-mon[70044]: pgmap v10180: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:39.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:07:38 smithi083 ceph-mon[77242]: pgmap v10181: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:39.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:07:38 smithi083 ceph-mon[79930]: pgmap v10181: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:39.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:07:38 smithi153 ceph-mon[70044]: pgmap v10181: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:40.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:07:40 smithi083 ceph-mon[79930]: pgmap v10182: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:40.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:07:40 smithi083 ceph-mon[77242]: pgmap v10182: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:07:40 smithi153 ceph-mon[70044]: pgmap v10182: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:41.413 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:07:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:07:41.414 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:07:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:07:41.414 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:07:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:07:41.414 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:07:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:07:41.415 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:07:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:07:41.415 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:07:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:07:41.415 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:07:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:07:41.415 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:07:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:07:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:07:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:07:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:07:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:07:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:07:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:07:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:07:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:07:41.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:07:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:07:41] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T11:07:41.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:07:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:07:41.412Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:07:42.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:07:42 smithi083 ceph-mon[77242]: pgmap v10183: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:07:42 smithi083 ceph-mon[79930]: pgmap v10183: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:07:42 smithi153 ceph-mon[70044]: pgmap v10183: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:42.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:07:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:07:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:07:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:07:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:07:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:07:45.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:07:44 smithi083 ceph-mon[79930]: pgmap v10184: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:45.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:07:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:07:44.786Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:07:45.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:07:44 smithi083 ceph-mon[77242]: pgmap v10184: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:45.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:07:44 smithi153 ceph-mon[70044]: pgmap v10184: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:45.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:07:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:07:45.504Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:07:47.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:07:46 smithi083 ceph-mon[79930]: pgmap v10185: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:47.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:07:46 smithi083 ceph-mon[77242]: pgmap v10185: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:47.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:07:46 smithi153 ceph-mon[70044]: pgmap v10185: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:49.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:07:48 smithi083 ceph-mon[79930]: pgmap v10186: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:49.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:07:48 smithi083 ceph-mon[77242]: pgmap v10186: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:49.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:07:48 smithi153 ceph-mon[70044]: pgmap v10186: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:50.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:07:50 smithi083 ceph-mon[79930]: pgmap v10187: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:50.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:07:50 smithi083 ceph-mon[77242]: pgmap v10187: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:07:50 smithi153 ceph-mon[70044]: pgmap v10187: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:51.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:07:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:07:51] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T11:07:51.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:07:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:07:51.413Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:07:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:07:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:07:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:07:53.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:07:52 smithi083 ceph-mon[79930]: pgmap v10188: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:53.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:07:52 smithi083 ceph-mon[77242]: pgmap v10188: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:53.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:07:52 smithi153 ceph-mon[70044]: pgmap v10188: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:07:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:07:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:07:55.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:07:54 smithi083 ceph-mon[79930]: pgmap v10189: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:55.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:07:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:07:54.787Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:07:55.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:07:54 smithi083 ceph-mon[77242]: pgmap v10189: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:55.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:07:54 smithi153 ceph-mon[70044]: pgmap v10189: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:55.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:07:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:07:55.505Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:07:57.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:07:56 smithi083 ceph-mon[77242]: pgmap v10190: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:57.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:07:56 smithi083 ceph-mon[79930]: pgmap v10190: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:57.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:07:56 smithi153 ceph-mon[70044]: pgmap v10190: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:59.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:07:58 smithi083 ceph-mon[79930]: pgmap v10191: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:59.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:07:58 smithi083 ceph-mon[77242]: pgmap v10191: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:07:59.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:07:58 smithi153 ceph-mon[70044]: pgmap v10191: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:00.330 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:08:00.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:08:00 smithi083 ceph-mon[79930]: pgmap v10192: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:00.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:08:00 smithi083 ceph-mon[77242]: pgmap v10192: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:08:00 smithi153 ceph-mon[70044]: pgmap v10192: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:00.619 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:08:00.620 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 6m ago 5h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:08:00.620 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 4m ago 5h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:08:00.620 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 6m ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:08:00.620 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 4m ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:08:00.621 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 4m ago 5h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:08:00.621 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 6m ago 5h 799M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:08:00.621 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 6m ago 5h 1271M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:08:00.621 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 4m ago 5h 548M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:08:00.621 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 6m ago 5h 731M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:08:00.622 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 6m ago 5h 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:08:00.622 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 4m ago 5h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:08:00.622 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 6m ago 5h 895M 2447M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:08:00.622 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 6m ago 5h 885M 2447M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:08:00.623 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 6m ago 5h 690M 2447M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:08:00.623 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 6m ago 5h 864M 2447M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:08:00.623 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 4m ago 5h 762M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:08:00.623 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 4m ago 5h 950M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:08:00.624 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 4m ago 5h 848M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:08:00.624 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 4m ago 5h 1029M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:08:00.624 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 4m ago 5h 149M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:08:00.976 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:08:00.977 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:08:00.977 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:08:00.977 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:08:00.978 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:08:00.978 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:08:00.978 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:08:00.978 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:08:00.979 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:08:00.979 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:08:00.979 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:08:00.979 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:08:00.980 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:08:00.980 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:08:00.980 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:08:00.981 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:08:00.981 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:08:00.981 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:08:01.414 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:08:01 smithi083 ceph-mon[79930]: from='client.45987 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:08:01.415 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:08:01 smithi083 ceph-mon[79930]: from='client.55924 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:08:01.415 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:08:01 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3225902313' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:08:01.415 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:08:01 smithi083 ceph-mon[77242]: from='client.45987 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:08:01.416 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:08:01 smithi083 ceph-mon[77242]: from='client.55924 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:08:01.416 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:08:01 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3225902313' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:08:01.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:08:01 smithi153 ceph-mon[70044]: from='client.45987 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:08:01.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:08:01 smithi153 ceph-mon[70044]: from='client.55924 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:08:01.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:08:01 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3225902313' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:08:01.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:08:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:08:01] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T11:08:01.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:08:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:08:01.413Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:08:02.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:08:02 smithi083 ceph-mon[79930]: pgmap v10193: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:02.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:08:02 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:08:02.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:08:02 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:08:02.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:08:02 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:08:02.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:08:02 smithi083 ceph-mon[77242]: pgmap v10193: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:02.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:08:02 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:08:02.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:08:02 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:08:02.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:08:02 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:08:02.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:08:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:08:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:08:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:08:02 smithi153 ceph-mon[70044]: pgmap v10193: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:08:02 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:08:02.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:08:02 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:08:02.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:08:02 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:08:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:08:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:08:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:08:03.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:08:03 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:08:03.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:08:03 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:08:03.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:08:03 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:08:04.788 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:08:04 smithi083 ceph-mon[79930]: pgmap v10194: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:04.789 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:08:04 smithi083 ceph-mon[77242]: pgmap v10194: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:04.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:08:04 smithi153 ceph-mon[70044]: pgmap v10194: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:05.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:08:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:08:04.787Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:08:05.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:08:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:08:05.505Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:08:07.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:08:06 smithi083 ceph-mon[77242]: pgmap v10195: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:07.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:08:06 smithi083 ceph-mon[79930]: pgmap v10195: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:07.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:08:06 smithi153 ceph-mon[70044]: pgmap v10195: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:09.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:08:08 smithi083 ceph-mon[79930]: pgmap v10196: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:09.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:08:08 smithi083 ceph-mon[77242]: pgmap v10196: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:09.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:08:08 smithi153 ceph-mon[70044]: pgmap v10196: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:10.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:08:10 smithi083 ceph-mon[79930]: pgmap v10197: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:10.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:08:10 smithi083 ceph-mon[77242]: pgmap v10197: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:08:10 smithi153 ceph-mon[70044]: pgmap v10197: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:11.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:08:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:08:11] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T11:08:11.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:08:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:08:11.414Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:08:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:08:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:08:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:08:13.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:08:12 smithi083 ceph-mon[79930]: pgmap v10198: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:13.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:08:12 smithi083 ceph-mon[77242]: pgmap v10198: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:13.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:08:12 smithi153 ceph-mon[70044]: pgmap v10198: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:08:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:08:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:08:15.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:08:14 smithi083 ceph-mon[77242]: pgmap v10199: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:15.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:08:14 smithi083 ceph-mon[79930]: pgmap v10199: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:15.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:08:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:08:14.788Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:08:15.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:08:14 smithi153 ceph-mon[70044]: pgmap v10199: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:15.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:08:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:08:15.506Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:08:17.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:08:16 smithi083 ceph-mon[79930]: pgmap v10200: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:17.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:08:16 smithi083 ceph-mon[77242]: pgmap v10200: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:17.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:08:16 smithi153 ceph-mon[70044]: pgmap v10200: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:19.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:08:18 smithi083 ceph-mon[79930]: pgmap v10201: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:19.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:08:18 smithi083 ceph-mon[77242]: pgmap v10201: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:19.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:08:18 smithi153 ceph-mon[70044]: pgmap v10201: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:20.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:08:20 smithi083 ceph-mon[79930]: pgmap v10202: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:20.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:08:20 smithi083 ceph-mon[77242]: pgmap v10202: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:08:20 smithi153 ceph-mon[70044]: pgmap v10202: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:21.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:08:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:08:21] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T11:08:21.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:08:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:08:21.415Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:08:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:08:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:08:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:08:23.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:08:22 smithi083 ceph-mon[77242]: pgmap v10203: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:23.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:08:22 smithi083 ceph-mon[79930]: pgmap v10203: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:23.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:08:22 smithi153 ceph-mon[70044]: pgmap v10203: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:08:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:08:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:08:25.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:08:24 smithi083 ceph-mon[79930]: pgmap v10204: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:25.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:08:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:08:24.789Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:08:25.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:08:24 smithi083 ceph-mon[77242]: pgmap v10204: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:25.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:08:24 smithi153 ceph-mon[70044]: pgmap v10204: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:25.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:08:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:08:25.507Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:08:27.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:08:26 smithi083 ceph-mon[77242]: pgmap v10205: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:27.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:08:26 smithi083 ceph-mon[79930]: pgmap v10205: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:27.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:08:26 smithi153 ceph-mon[70044]: pgmap v10205: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:29.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:08:28 smithi083 ceph-mon[79930]: pgmap v10206: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:29.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:08:28 smithi083 ceph-mon[77242]: pgmap v10206: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:29.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:08:28 smithi153 ceph-mon[70044]: pgmap v10206: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:30.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:08:30 smithi083 ceph-mon[77242]: pgmap v10207: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:30.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:08:30 smithi083 ceph-mon[79930]: pgmap v10207: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:08:30 smithi153 ceph-mon[70044]: pgmap v10207: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:31.304 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:08:31.592 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:08:31.592 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 6m ago 5h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:08:31.592 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 4m ago 5h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:08:31.593 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 6m ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:08:31.593 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 4m ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:08:31.593 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 4m ago 5h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:08:31.593 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 6m ago 5h 799M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:08:31.594 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 6m ago 5h 1271M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:08:31.594 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 4m ago 5h 548M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:08:31.594 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 6m ago 5h 731M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:08:31.594 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 6m ago 5h 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:08:31.594 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 4m ago 5h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:08:31.595 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 6m ago 5h 895M 2447M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:08:31.595 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 6m ago 5h 885M 2447M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:08:31.595 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 6m ago 5h 690M 2447M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:08:31.595 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 6m ago 5h 864M 2447M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:08:31.596 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 4m ago 5h 762M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:08:31.596 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 4m ago 5h 950M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:08:31.596 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 4m ago 5h 848M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:08:31.596 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 4m ago 5h 1029M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:08:31.596 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 4m ago 5h 149M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:08:31.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:08:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:08:31] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T11:08:31.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:08:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:08:31.414Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:08:31.949 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:08:31.949 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:08:31.949 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:08:31.950 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:08:31.950 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:08:31.950 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:08:31.950 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:08:31.951 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:08:31.951 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:08:31.951 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:08:31.951 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:08:31.951 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:08:31.952 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:08:31.952 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:08:31.952 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:08:31.952 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:08:31.953 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:08:31.953 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:08:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:08:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:08:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:08:33.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:08:32 smithi083 ceph-mon[77242]: from='client.46005 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:08:33.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:08:32 smithi083 ceph-mon[77242]: from='client.46011 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:08:33.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:08:32 smithi083 ceph-mon[77242]: pgmap v10208: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:33.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:08:32 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/859581840' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:08:33.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:08:32 smithi083 ceph-mon[79930]: from='client.46005 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:08:33.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:08:32 smithi083 ceph-mon[79930]: from='client.46011 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:08:33.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:08:32 smithi083 ceph-mon[79930]: pgmap v10208: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:33.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:08:32 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/859581840' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:08:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:08:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:08:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:08:33.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:08:32 smithi153 ceph-mon[70044]: from='client.46005 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:08:33.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:08:32 smithi153 ceph-mon[70044]: from='client.46011 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:08:33.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:08:32 smithi153 ceph-mon[70044]: pgmap v10208: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:33.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:08:32 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/859581840' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:08:35.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:08:34 smithi083 ceph-mon[79930]: pgmap v10209: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:35.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:08:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:08:34.789Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:08:35.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:08:34 smithi083 ceph-mon[77242]: pgmap v10209: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:35.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:08:34 smithi153 ceph-mon[70044]: pgmap v10209: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:35.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:08:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:08:35.508Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:08:37.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:08:36 smithi083 ceph-mon[79930]: pgmap v10210: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:37.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:08:36 smithi083 ceph-mon[77242]: pgmap v10210: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:37.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:08:36 smithi153 ceph-mon[70044]: pgmap v10210: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:39.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:08:38 smithi083 ceph-mon[79930]: pgmap v10211: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:39.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:08:38 smithi083 ceph-mon[77242]: pgmap v10211: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:39.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:08:38 smithi153 ceph-mon[70044]: pgmap v10211: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:40.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:08:40 smithi083 ceph-mon[79930]: pgmap v10212: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:40.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:08:40 smithi083 ceph-mon[77242]: pgmap v10212: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:08:40 smithi153 ceph-mon[70044]: pgmap v10212: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:41.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:08:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:08:41.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:08:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:08:41.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:08:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:08:41.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:08:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:08:41.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:08:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:08:41.416Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:08:41.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:08:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:08:41.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:08:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:08:41.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:08:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:08:41.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:08:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:08:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:08:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:08:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:08:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:08:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:08:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:08:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:08:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:08:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:08:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:08:41] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T11:08:42.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:08:42 smithi083 ceph-mon[79930]: pgmap v10213: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:42.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:08:42 smithi083 ceph-mon[77242]: pgmap v10213: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:08:42 smithi153 ceph-mon[70044]: pgmap v10213: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:42.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:08:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:08:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:08:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:08:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:08:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:08:45.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:08:44 smithi083 ceph-mon[79930]: pgmap v10214: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:45.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:08:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:08:44.790Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:08:45.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:08:44 smithi083 ceph-mon[77242]: pgmap v10214: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:45.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:08:44 smithi153 ceph-mon[70044]: pgmap v10214: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:45.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:08:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:08:45.508Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:08:47.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:08:46 smithi083 ceph-mon[79930]: pgmap v10215: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:47.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:08:46 smithi083 ceph-mon[77242]: pgmap v10215: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:47.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:08:46 smithi153 ceph-mon[70044]: pgmap v10215: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:49.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:08:48 smithi083 ceph-mon[79930]: pgmap v10216: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:49.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:08:48 smithi083 ceph-mon[77242]: pgmap v10216: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:49.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:08:48 smithi153 ceph-mon[70044]: pgmap v10216: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:50.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:08:50 smithi083 ceph-mon[79930]: pgmap v10217: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:50.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:08:50 smithi083 ceph-mon[77242]: pgmap v10217: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:08:50 smithi153 ceph-mon[70044]: pgmap v10217: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:51.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:08:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:08:51] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T11:08:51.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:08:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:08:51.416Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:08:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:08:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:08:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:08:53.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:08:52 smithi083 ceph-mon[79930]: pgmap v10218: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:53.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:08:52 smithi083 ceph-mon[77242]: pgmap v10218: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:53.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:08:52 smithi153 ceph-mon[70044]: pgmap v10218: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:08:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:08:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:08:55.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:08:54 smithi083 ceph-mon[79930]: pgmap v10219: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:55.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:08:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:08:54.791Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:08:55.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:08:54 smithi083 ceph-mon[77242]: pgmap v10219: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:55.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:08:54 smithi153 ceph-mon[70044]: pgmap v10219: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:55.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:08:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:08:55.509Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:08:57.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:08:56 smithi083 ceph-mon[79930]: pgmap v10220: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:57.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:08:56 smithi083 ceph-mon[77242]: pgmap v10220: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:57.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:08:56 smithi153 ceph-mon[70044]: pgmap v10220: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:59.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:08:58 smithi083 ceph-mon[79930]: pgmap v10221: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:59.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:08:58 smithi083 ceph-mon[77242]: pgmap v10221: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:08:59.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:08:58 smithi153 ceph-mon[70044]: pgmap v10221: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:00.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:09:00 smithi083 ceph-mon[79930]: pgmap v10222: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:00.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:09:00 smithi083 ceph-mon[77242]: pgmap v10222: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:09:00 smithi153 ceph-mon[70044]: pgmap v10222: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:01.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:09:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:09:01] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T11:09:01.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:09:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:09:01.416Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:09:02.272 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:09:02.566 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:09:02.566 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 7m ago 5h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:09:02.567 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 5m ago 5h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:09:02.567 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 7m ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:09:02.567 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 5m ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:09:02.567 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 5m ago 5h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:09:02.568 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 7m ago 5h 799M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:09:02.568 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 7m ago 5h 1271M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:09:02.568 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 5m ago 5h 548M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:09:02.568 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 7m ago 5h 731M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:09:02.569 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 7m ago 5h 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:09:02.569 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 5m ago 5h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:09:02.569 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 7m ago 5h 895M 2447M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:09:02.570 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 7m ago 5h 885M 2447M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:09:02.570 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 7m ago 5h 690M 2447M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:09:02.570 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 7m ago 5h 864M 2447M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:09:02.570 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 5m ago 5h 762M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:09:02.571 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 5m ago 5h 950M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:09:02.571 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 5m ago 5h 848M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:09:02.571 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 5m ago 5h 1029M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:09:02.571 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 5m ago 5h 149M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:09:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:09:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:09:02.316Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:09:02.942 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:09:02.942 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:09:02.943 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:09:02.943 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:09:02.943 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:09:02.943 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:09:02.943 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:09:02.944 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:09:02.944 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:09:02.944 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:09:02.944 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:09:02.944 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:09:02.945 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:09:02.945 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:09:02.945 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:09:02.945 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:09:02.946 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:09:02.946 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:09:03.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:09:02 smithi083 ceph-mon[79930]: pgmap v10223: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:03.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:09:02 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:09:03.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:09:02 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:09:03.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:09:02 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:09:03.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:09:02 smithi083 ceph-mon[77242]: pgmap v10223: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:03.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:09:02 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:09:03.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:09:02 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:09:03.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:09:02 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:09:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:09:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:09:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:09:03.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:09:02 smithi153 ceph-mon[70044]: pgmap v10223: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:03.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:09:02 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:09:03.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:09:02 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:09:03.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:09:02 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:09:04.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:09:03 smithi083 ceph-mon[79930]: from='client.55954 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:09:04.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:09:03 smithi083 ceph-mon[79930]: from='client.46029 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:09:04.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:09:03 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2088673209' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:09:04.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:09:03 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:09:04.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:09:03 smithi083 ceph-mon[77242]: from='client.55954 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:09:04.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:09:03 smithi083 ceph-mon[77242]: from='client.46029 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:09:04.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:09:03 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2088673209' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:09:04.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:09:03 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:09:04.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:09:03 smithi153 ceph-mon[70044]: from='client.55954 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:09:04.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:09:03 smithi153 ceph-mon[70044]: from='client.46029 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:09:04.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:09:03 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2088673209' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:09:04.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:09:03 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:09:05.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:09:04 smithi083 ceph-mon[79930]: pgmap v10224: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:05.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:09:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:09:04.791Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:09:05.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:09:04 smithi083 ceph-mon[77242]: pgmap v10224: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:05.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:09:04 smithi153 ceph-mon[70044]: pgmap v10224: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:05.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:09:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:09:05.509Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:09:07.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:09:06 smithi083 ceph-mon[79930]: pgmap v10225: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:07.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:09:06 smithi083 ceph-mon[77242]: pgmap v10225: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:07.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:09:06 smithi153 ceph-mon[70044]: pgmap v10225: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:09.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:09:08 smithi083 ceph-mon[79930]: pgmap v10226: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:09.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:09:08 smithi083 ceph-mon[77242]: pgmap v10226: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:09.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:09:08 smithi153 ceph-mon[70044]: pgmap v10226: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:10.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:09:10 smithi083 ceph-mon[79930]: pgmap v10227: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:10.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:09:10 smithi083 ceph-mon[77242]: pgmap v10227: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:09:10 smithi153 ceph-mon[70044]: pgmap v10227: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:11.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:09:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:09:11] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T11:09:11.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:09:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:09:11.417Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:09:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:09:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:09:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:09:13.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:09:12 smithi083 ceph-mon[79930]: pgmap v10228: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:13.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:09:12 smithi083 ceph-mon[77242]: pgmap v10228: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:13.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:09:12 smithi153 ceph-mon[70044]: pgmap v10228: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:09:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:09:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:09:15.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:09:14 smithi083 ceph-mon[79930]: pgmap v10229: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:15.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:09:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:09:14.792Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:09:15.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:09:14 smithi083 ceph-mon[77242]: pgmap v10229: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:15.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:09:14 smithi153 ceph-mon[70044]: pgmap v10229: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:15.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:09:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:09:15.510Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:09:17.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:09:16 smithi083 ceph-mon[79930]: pgmap v10230: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:17.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:09:16 smithi083 ceph-mon[77242]: pgmap v10230: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:17.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:09:16 smithi153 ceph-mon[70044]: pgmap v10230: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:19.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:09:18 smithi083 ceph-mon[77242]: pgmap v10231: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:19.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:09:18 smithi083 ceph-mon[79930]: pgmap v10231: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:19.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:09:18 smithi153 ceph-mon[70044]: pgmap v10231: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:20.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:09:20 smithi083 ceph-mon[77242]: pgmap v10232: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:20.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:09:20 smithi083 ceph-mon[79930]: pgmap v10232: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:09:20 smithi153 ceph-mon[70044]: pgmap v10232: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:21.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:09:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:09:21] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T11:09:21.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:09:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:09:21.417Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:09:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:09:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:09:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:09:23.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:09:22 smithi083 ceph-mon[79930]: pgmap v10233: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:23.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:09:22 smithi083 ceph-mon[77242]: pgmap v10233: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:23.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:09:22 smithi153 ceph-mon[70044]: pgmap v10233: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:09:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:09:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:09:25.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:09:24 smithi083 ceph-mon[79930]: pgmap v10234: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:25.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:09:24 smithi083 ceph-mon[77242]: pgmap v10234: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:25.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:09:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:09:24.792Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:09:25.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:09:24 smithi153 ceph-mon[70044]: pgmap v10234: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:25.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:09:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:09:25.511Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:09:27.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:09:26 smithi083 ceph-mon[79930]: pgmap v10235: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:27.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:09:26 smithi083 ceph-mon[77242]: pgmap v10235: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:27.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:09:26 smithi153 ceph-mon[70044]: pgmap v10235: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:29.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:09:28 smithi083 ceph-mon[79930]: pgmap v10236: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:29.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:09:28 smithi083 ceph-mon[77242]: pgmap v10236: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:29.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:09:28 smithi153 ceph-mon[70044]: pgmap v10236: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:30.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:09:30 smithi083 ceph-mon[79930]: pgmap v10237: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:30.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:09:30 smithi083 ceph-mon[77242]: pgmap v10237: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:09:30 smithi153 ceph-mon[70044]: pgmap v10237: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:31.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:09:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:09:31] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T11:09:31.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:09:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:09:31.418Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:09:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:09:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:09:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:09:33.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:09:32 smithi083 ceph-mon[79930]: pgmap v10238: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:33.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:09:32 smithi083 ceph-mon[77242]: pgmap v10238: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:33.271 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:09:33.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:09:32 smithi153 ceph-mon[70044]: pgmap v10238: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:09:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:09:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:09:33.568 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:09:33.569 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 7m ago 5h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:09:33.569 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 5m ago 5h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:09:33.569 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 7m ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:09:33.570 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 5m ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:09:33.570 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 5m ago 5h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:09:33.570 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 7m ago 5h 799M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:09:33.571 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 7m ago 5h 1271M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:09:33.571 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 5m ago 5h 548M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:09:33.571 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 7m ago 5h 731M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:09:33.571 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 7m ago 5h 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:09:33.572 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 5m ago 5h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:09:33.572 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 7m ago 5h 895M 2447M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:09:33.572 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 7m ago 5h 885M 2447M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:09:33.572 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 7m ago 5h 690M 2447M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:09:33.572 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 7m ago 5h 864M 2447M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:09:33.573 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 5m ago 5h 762M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:09:33.573 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 5m ago 5h 950M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:09:33.573 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 5m ago 5h 848M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:09:33.573 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 5m ago 5h 1029M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:09:33.574 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 5m ago 5h 149M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:09:33.932 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:09:33.932 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:09:33.933 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:09:33.933 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:09:33.933 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:09:33.933 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:09:33.934 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:09:33.934 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:09:33.934 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:09:33.934 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:09:33.935 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:09:33.935 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:09:33.935 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:09:33.935 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:09:33.936 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:09:33.936 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:09:33.936 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:09:33.936 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:09:34.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:09:33 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1694391098' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:09:34.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:09:33 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1694391098' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:09:34.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:09:33 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1694391098' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:09:35.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:09:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:09:34.792Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:09:35.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:09:34 smithi083 ceph-mon[79930]: from='client.55972 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:09:35.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:09:34 smithi083 ceph-mon[79930]: from='client.55978 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:09:35.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:09:34 smithi083 ceph-mon[79930]: pgmap v10239: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:35.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:09:34 smithi083 ceph-mon[77242]: from='client.55972 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:09:35.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:09:34 smithi083 ceph-mon[77242]: from='client.55978 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:09:35.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:09:34 smithi083 ceph-mon[77242]: pgmap v10239: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:35.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:09:34 smithi153 ceph-mon[70044]: from='client.55972 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:09:35.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:09:34 smithi153 ceph-mon[70044]: from='client.55978 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:09:35.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:09:34 smithi153 ceph-mon[70044]: pgmap v10239: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:35.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:09:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:09:35.512Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:09:37.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:09:36 smithi083 ceph-mon[77242]: pgmap v10240: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:37.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:09:36 smithi083 ceph-mon[79930]: pgmap v10240: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:37.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:09:36 smithi153 ceph-mon[70044]: pgmap v10240: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:39.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:09:38 smithi083 ceph-mon[77242]: pgmap v10241: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:39.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:09:38 smithi083 ceph-mon[79930]: pgmap v10241: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:39.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:09:38 smithi153 ceph-mon[70044]: pgmap v10241: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:40.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:09:40 smithi083 ceph-mon[77242]: pgmap v10242: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:09:40 smithi083 ceph-mon[79930]: pgmap v10242: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:09:40 smithi153 ceph-mon[70044]: pgmap v10242: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:41.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:09:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:09:41.419Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:09:41.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:09:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:09:41.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:09:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:09:41.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:09:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:09:41.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:09:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:09:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:09:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:09:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:09:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:09:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:09:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:09:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:09:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:09:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:09:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:09:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:09:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:09:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:09:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:09:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:09:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:09:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:09:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:09:41] "GET /metrics HTTP/1.1" 200 225217 "" "Prometheus/2.33.4" 2022-09-29T11:09:42.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:09:42 smithi083 ceph-mon[79930]: pgmap v10243: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:42.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:09:42 smithi083 ceph-mon[77242]: pgmap v10243: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:09:42 smithi153 ceph-mon[70044]: pgmap v10243: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:42.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:09:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:09:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:09:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:09:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:09:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:09:45.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:09:44 smithi083 ceph-mon[79930]: pgmap v10244: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:45.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:09:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:09:44.793Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:09:45.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:09:44 smithi083 ceph-mon[77242]: pgmap v10244: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:45.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:09:44 smithi153 ceph-mon[70044]: pgmap v10244: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:45.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:09:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:09:45.513Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:09:47.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:09:46 smithi083 ceph-mon[77242]: pgmap v10245: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:47.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:09:46 smithi083 ceph-mon[79930]: pgmap v10245: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:47.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:09:46 smithi153 ceph-mon[70044]: pgmap v10245: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:49.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:09:48 smithi083 ceph-mon[77242]: pgmap v10246: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:49.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:09:48 smithi083 ceph-mon[79930]: pgmap v10246: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:49.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:09:48 smithi153 ceph-mon[70044]: pgmap v10246: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:50.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:09:50 smithi083 ceph-mon[79930]: pgmap v10247: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:50.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:09:50 smithi083 ceph-mon[77242]: pgmap v10247: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:09:50 smithi153 ceph-mon[70044]: pgmap v10247: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:51.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:09:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:09:51] "GET /metrics HTTP/1.1" 200 225217 "" "Prometheus/2.33.4" 2022-09-29T11:09:51.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:09:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:09:51.419Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:09:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:09:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:09:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:09:53.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:09:52 smithi083 ceph-mon[79930]: pgmap v10248: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:53.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:09:52 smithi083 ceph-mon[77242]: pgmap v10248: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:53.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:09:52 smithi153 ceph-mon[70044]: pgmap v10248: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:09:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:09:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:09:55.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:09:54 smithi083 ceph-mon[79930]: pgmap v10249: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:55.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:09:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:09:54.794Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:09:55.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:09:54 smithi083 ceph-mon[77242]: pgmap v10249: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:55.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:09:54 smithi153 ceph-mon[70044]: pgmap v10249: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:55.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:09:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:09:55.513Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:09:57.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:09:56 smithi083 ceph-mon[79930]: pgmap v10250: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:57.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:09:56 smithi083 ceph-mon[77242]: pgmap v10250: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:57.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:09:56 smithi153 ceph-mon[70044]: pgmap v10250: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:59.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:09:58 smithi153 ceph-mon[70044]: pgmap v10251: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:59.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:09:58 smithi083 ceph-mon[79930]: pgmap v10251: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:09:59.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:09:58 smithi083 ceph-mon[77242]: pgmap v10251: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:00.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:10:00 smithi083 ceph-mon[79930]: pgmap v10252: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:00.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:10:00 smithi083 ceph-mon[79930]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T11:10:00.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:10:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[77219]: 2022-09-29T11:09:59.999+0000 7f160f665700 -1 log_channel(cluster) log [ERR] : overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T11:10:00.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:10:00 smithi083 ceph-mon[77242]: pgmap v10252: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:00.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:10:00 smithi083 ceph-mon[77242]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T11:10:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:10:00 smithi153 ceph-mon[70044]: pgmap v10252: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:10:00 smithi153 ceph-mon[70044]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T11:10:01.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:10:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:10:01] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T11:10:01.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:10:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:10:01.420Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:10:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:10:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:10:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:10:03.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:10:02 smithi083 ceph-mon[79930]: pgmap v10253: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:03.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:10:02 smithi083 ceph-mon[77242]: pgmap v10253: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:03.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:10:02 smithi153 ceph-mon[70044]: pgmap v10253: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:10:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:10:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:10:04.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:10:03 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:10:04.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:10:03 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:10:04.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:10:03 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:10:04.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:10:03 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:10:04.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:10:03 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:10:04.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:10:03 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:10:04.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:10:03 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:10:04.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:10:03 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:10:04.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:10:03 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:10:04.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:10:03 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:10:04.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:10:03 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:10:04.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:10:03 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:10:04.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:10:03 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:10:04.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:10:03 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:10:04.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:10:03 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:10:04.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:10:03 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:10:04.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:10:03 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:10:04.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:10:03 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:10:04.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:10:03 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:10:04.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:10:03 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:10:04.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:10:03 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:10:04.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:10:03 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:10:04.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:10:03 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:10:04.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:10:03 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:10:04.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:10:03 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:10:04.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:10:03 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:10:04.250 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:10:04.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:10:03 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:10:04.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:10:03 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:10:04.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:10:03 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:10:04.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:10:03 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:10:04.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:10:03 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:10:04.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:10:03 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:10:04.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:10:03 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:10:04.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:10:03 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:10:04.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:10:03 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:10:04.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:10:03 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:10:04.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:10:03 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:10:04.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:10:03 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:10:04.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:10:03 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:10:04.539 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:10:04.539 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 8m ago 5h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:10:04.539 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 6m ago 5h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:10:04.540 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 8m ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:10:04.540 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 6m ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:10:04.540 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 6m ago 5h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:10:04.540 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 8m ago 5h 799M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:10:04.541 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 8m ago 5h 1271M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:10:04.541 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 6m ago 5h 548M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:10:04.541 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 8m ago 5h 731M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:10:04.541 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 8m ago 5h 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:10:04.541 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 6m ago 5h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:10:04.542 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 8m ago 5h 895M 2438M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:10:04.542 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 8m ago 5h 885M 2438M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:10:04.542 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 8m ago 5h 690M 2438M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:10:04.542 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 8m ago 5h 864M 2438M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:10:04.543 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 6m ago 5h 762M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:10:04.543 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 6m ago 5h 950M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:10:04.543 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 6m ago 5h 848M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:10:04.543 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 6m ago 5h 1029M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:10:04.543 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 6m ago 5h 149M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:10:04.896 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:10:04.896 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:10:04.897 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:10:04.897 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:10:04.897 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:10:04.897 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:10:04.898 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:10:04.898 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:10:04.898 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:10:04.898 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:10:04.898 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:10:04.899 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:10:04.899 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:10:04.899 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:10:04.899 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:10:04.900 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:10:04.900 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:10:04.900 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:10:05.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:10:04 smithi083 ceph-mon[79930]: Adjusting osd_memory_target on smithi083 to 2438M 2022-09-29T11:10:05.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:10:04 smithi083 ceph-mon[79930]: pgmap v10254: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:05.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:10:04 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/136464294' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:10:05.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:10:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:10:04.794Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:10:05.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:10:04 smithi083 ceph-mon[77242]: Adjusting osd_memory_target on smithi083 to 2438M 2022-09-29T11:10:05.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:10:04 smithi083 ceph-mon[77242]: pgmap v10254: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:05.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:10:04 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/136464294' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:10:05.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:10:04 smithi153 ceph-mon[70044]: Adjusting osd_memory_target on smithi083 to 2438M 2022-09-29T11:10:05.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:10:04 smithi153 ceph-mon[70044]: pgmap v10254: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:05.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:10:04 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/136464294' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:10:05.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:10:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:10:05.514Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:10:06.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:10:05 smithi153 ceph-mon[70044]: from='client.46059 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:10:06.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:10:05 smithi153 ceph-mon[70044]: from='client.55996 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:10:06.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:10:05 smithi083 ceph-mon[79930]: from='client.46059 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:10:06.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:10:05 smithi083 ceph-mon[79930]: from='client.55996 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:10:06.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:10:05 smithi083 ceph-mon[77242]: from='client.46059 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:10:06.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:10:05 smithi083 ceph-mon[77242]: from='client.55996 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:10:07.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:10:06 smithi153 ceph-mon[70044]: pgmap v10255: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:07.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:10:06 smithi083 ceph-mon[77242]: pgmap v10255: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:07.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:10:06 smithi083 ceph-mon[79930]: pgmap v10255: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:09.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:10:08 smithi153 ceph-mon[70044]: pgmap v10256: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:09.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:10:08 smithi083 ceph-mon[79930]: pgmap v10256: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:09.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:10:08 smithi083 ceph-mon[77242]: pgmap v10256: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:10.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:10:10 smithi083 ceph-mon[79930]: pgmap v10257: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:10.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:10:10 smithi083 ceph-mon[77242]: pgmap v10257: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:10:10 smithi153 ceph-mon[70044]: pgmap v10257: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:11.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:10:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:10:11] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T11:10:11.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:10:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:10:11.421Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:10:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:10:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:10:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:10:13.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:10:12 smithi083 ceph-mon[79930]: pgmap v10258: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:13.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:10:12 smithi083 ceph-mon[77242]: pgmap v10258: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:13.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:10:12 smithi153 ceph-mon[70044]: pgmap v10258: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:10:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:10:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:10:15.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:10:14 smithi083 ceph-mon[79930]: pgmap v10259: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:15.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:10:14 smithi083 ceph-mon[77242]: pgmap v10259: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:15.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:10:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:10:14.795Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:10:15.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:10:14 smithi153 ceph-mon[70044]: pgmap v10259: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:15.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:10:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:10:15.515Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:10:17.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:10:16 smithi083 ceph-mon[77242]: pgmap v10260: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:17.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:10:16 smithi083 ceph-mon[79930]: pgmap v10260: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:17.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:10:16 smithi153 ceph-mon[70044]: pgmap v10260: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:19.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:10:18 smithi083 ceph-mon[79930]: pgmap v10261: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:19.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:10:18 smithi083 ceph-mon[77242]: pgmap v10261: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:19.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:10:18 smithi153 ceph-mon[70044]: pgmap v10261: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:20.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:10:20 smithi083 ceph-mon[79930]: pgmap v10262: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:20.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:10:20 smithi083 ceph-mon[77242]: pgmap v10262: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:10:20 smithi153 ceph-mon[70044]: pgmap v10262: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:21.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:10:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:10:21] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T11:10:21.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:10:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:10:21.422Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:10:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:10:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:10:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:10:23.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:10:22 smithi083 ceph-mon[79930]: pgmap v10263: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:23.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:10:22 smithi083 ceph-mon[77242]: pgmap v10263: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:23.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:10:22 smithi153 ceph-mon[70044]: pgmap v10263: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:10:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:10:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:10:25.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:10:24 smithi083 ceph-mon[79930]: pgmap v10264: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:25.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:10:24 smithi083 ceph-mon[77242]: pgmap v10264: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:25.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:10:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:10:24.795Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:10:25.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:10:24 smithi153 ceph-mon[70044]: pgmap v10264: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:25.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:10:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:10:25.515Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:10:27.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:10:26 smithi083 ceph-mon[79930]: pgmap v10265: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:27.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:10:26 smithi083 ceph-mon[77242]: pgmap v10265: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:27.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:10:26 smithi153 ceph-mon[70044]: pgmap v10265: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:29.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:10:28 smithi153 ceph-mon[70044]: pgmap v10266: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:29.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:10:28 smithi083 ceph-mon[77242]: pgmap v10266: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:29.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:10:28 smithi083 ceph-mon[79930]: pgmap v10266: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:30.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:10:30 smithi083 ceph-mon[79930]: pgmap v10267: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:30.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:10:30 smithi083 ceph-mon[77242]: pgmap v10267: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:10:30 smithi153 ceph-mon[70044]: pgmap v10267: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:31.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:10:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:10:31] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T11:10:31.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:10:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:10:31.423Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:10:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:10:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:10:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:10:33.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:10:32 smithi083 ceph-mon[77242]: pgmap v10268: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:33.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:10:32 smithi083 ceph-mon[79930]: pgmap v10268: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:33.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:10:32 smithi153 ceph-mon[70044]: pgmap v10268: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:10:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:10:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:10:35.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:10:34 smithi083 ceph-mon[77242]: pgmap v10269: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:35.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:10:34 smithi083 ceph-mon[79930]: pgmap v10269: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:35.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:10:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:10:34.796Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:10:35.223 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:10:35.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:10:34 smithi153 ceph-mon[70044]: pgmap v10269: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:35.514 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:10:35.514 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 8m ago 5h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:10:35.514 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 6m ago 5h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:10:35.514 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 8m ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:10:35.514 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 6m ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:10:35.515 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 6m ago 5h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:10:35.515 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 8m ago 5h 799M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:10:35.515 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 8m ago 5h 1271M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:10:35.515 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 6m ago 5h 548M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:10:35.516 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 8m ago 5h 731M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:10:35.516 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 8m ago 5h 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:10:35.516 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 6m ago 5h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:10:35.516 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 8m ago 5h 895M 2438M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:10:35.516 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 8m ago 5h 885M 2438M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:10:35.517 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 8m ago 5h 690M 2438M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:10:35.517 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 8m ago 5h 864M 2438M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:10:35.517 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 6m ago 5h 762M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:10:35.517 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 6m ago 5h 950M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:10:35.517 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 6m ago 5h 848M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:10:35.518 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 6m ago 5h 1029M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:10:35.518 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 6m ago 5h 149M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:10:35.867 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:10:35.867 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:10:35.867 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:10:35.868 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:10:35.868 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:10:35.868 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:10:35.868 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:10:35.868 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:10:35.869 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:10:35.869 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:10:35.869 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:10:35.869 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:10:35.869 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:10:35.870 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:10:35.870 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:10:35.870 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:10:35.870 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:10:35.871 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:10:35.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:10:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:10:35.516Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:10:36.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:10:35 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3681514814' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:10:36.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:10:35 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3681514814' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:10:36.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:10:35 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3681514814' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:10:37.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:10:36 smithi153 ceph-mon[70044]: from='client.56008 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:10:37.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:10:36 smithi153 ceph-mon[70044]: from='client.46083 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:10:37.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:10:36 smithi153 ceph-mon[70044]: pgmap v10270: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:37.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:10:36 smithi083 ceph-mon[79930]: from='client.56008 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:10:37.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:10:36 smithi083 ceph-mon[79930]: from='client.46083 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:10:37.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:10:36 smithi083 ceph-mon[79930]: pgmap v10270: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:37.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:10:36 smithi083 ceph-mon[77242]: from='client.56008 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:10:37.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:10:36 smithi083 ceph-mon[77242]: from='client.46083 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:10:37.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:10:36 smithi083 ceph-mon[77242]: pgmap v10270: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:39.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:10:38 smithi153 ceph-mon[70044]: pgmap v10271: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:39.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:10:38 smithi083 ceph-mon[79930]: pgmap v10271: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:39.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:10:38 smithi083 ceph-mon[77242]: pgmap v10271: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:40.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:10:40 smithi083 ceph-mon[79930]: pgmap v10272: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:40.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:10:40 smithi083 ceph-mon[77242]: pgmap v10272: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:10:40 smithi153 ceph-mon[70044]: pgmap v10272: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:41.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:10:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:10:41.424Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:10:41.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:10:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:10:41.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:10:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:10:41.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:10:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:10:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:10:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:10:41.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:10:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:10:41.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:10:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:10:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:10:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:10:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:10:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:10:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:10:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:10:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:10:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:10:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:10:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:10:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:10:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:10:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:10:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:10:41] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T11:10:42.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:10:42 smithi083 ceph-mon[79930]: pgmap v10273: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:42.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:10:42 smithi083 ceph-mon[77242]: pgmap v10273: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:10:42 smithi153 ceph-mon[70044]: pgmap v10273: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:42.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:10:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:10:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:10:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:10:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:10:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:10:45.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:10:44 smithi083 ceph-mon[79930]: pgmap v10274: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:45.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:10:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:10:44.797Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:10:45.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:10:44 smithi083 ceph-mon[77242]: pgmap v10274: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:45.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:10:44 smithi153 ceph-mon[70044]: pgmap v10274: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:45.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:10:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:10:45.516Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:10:47.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:10:46 smithi153 ceph-mon[70044]: pgmap v10275: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:47.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:10:46 smithi083 ceph-mon[79930]: pgmap v10275: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:47.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:10:46 smithi083 ceph-mon[77242]: pgmap v10275: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:49.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:10:48 smithi153 ceph-mon[70044]: pgmap v10276: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:49.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:10:48 smithi083 ceph-mon[79930]: pgmap v10276: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:49.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:10:48 smithi083 ceph-mon[77242]: pgmap v10276: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:50.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:10:50 smithi083 ceph-mon[79930]: pgmap v10277: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:50.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:10:50 smithi083 ceph-mon[77242]: pgmap v10277: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:10:50 smithi153 ceph-mon[70044]: pgmap v10277: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:51.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:10:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:10:51] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T11:10:51.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:10:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:10:51.424Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:10:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:10:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:10:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:10:53.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:10:52 smithi153 ceph-mon[70044]: pgmap v10278: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:10:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:10:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:10:53.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:10:52 smithi083 ceph-mon[77242]: pgmap v10278: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:53.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:10:52 smithi083 ceph-mon[79930]: pgmap v10278: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:55.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:10:54 smithi083 ceph-mon[79930]: pgmap v10279: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:55.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:10:54 smithi083 ceph-mon[77242]: pgmap v10279: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:55.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:10:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:10:54.798Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:10:55.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:10:54 smithi153 ceph-mon[70044]: pgmap v10279: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:55.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:10:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:10:55.516Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:10:57.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:10:56 smithi153 ceph-mon[70044]: pgmap v10280: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:57.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:10:56 smithi083 ceph-mon[79930]: pgmap v10280: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:57.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:10:56 smithi083 ceph-mon[77242]: pgmap v10280: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:59.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:10:58 smithi153 ceph-mon[70044]: pgmap v10281: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:59.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:10:58 smithi083 ceph-mon[79930]: pgmap v10281: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:10:59.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:10:58 smithi083 ceph-mon[77242]: pgmap v10281: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:00.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:11:00 smithi083 ceph-mon[79930]: pgmap v10282: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:00.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:11:00 smithi083 ceph-mon[77242]: pgmap v10282: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:11:00 smithi153 ceph-mon[70044]: pgmap v10282: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:01.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:11:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:11:01] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T11:11:01.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:11:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:11:01.425Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:11:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:11:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:11:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:11:03.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:11:02 smithi153 ceph-mon[70044]: pgmap v10283: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:11:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:11:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:11:03.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:11:02 smithi083 ceph-mon[79930]: pgmap v10283: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:03.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:11:02 smithi083 ceph-mon[77242]: pgmap v10283: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:04.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:11:03 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:11:04.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:11:03 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:11:04.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:11:03 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:11:04.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:11:03 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:11:04.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:11:03 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:11:04.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:11:03 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:11:04.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:11:03 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:11:04.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:11:03 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:11:04.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:11:03 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:11:05.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:11:04 smithi083 ceph-mon[79930]: pgmap v10284: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:05.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:11:04 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:11:05.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:11:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:11:04.798Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:11:05.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:11:04 smithi083 ceph-mon[77242]: pgmap v10284: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:05.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:11:04 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:11:05.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:11:04 smithi153 ceph-mon[70044]: pgmap v10284: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:05.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:11:04 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:11:05.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:11:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:11:05.517Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:11:06.195 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:11:06.481 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:11:06.481 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 9m ago 5h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:11:06.482 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 7m ago 5h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:11:06.482 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 9m ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:11:06.482 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 7m ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:11:06.482 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 7m ago 5h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:11:06.483 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 9m ago 5h 799M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:11:06.483 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 9m ago 5h 1271M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:11:06.483 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 7m ago 5h 548M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:11:06.483 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 9m ago 5h 731M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:11:06.484 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 9m ago 5h 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:11:06.484 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 7m ago 5h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:11:06.484 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 9m ago 5h 895M 2438M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:11:06.484 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 9m ago 5h 885M 2438M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:11:06.485 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 9m ago 5h 690M 2438M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:11:06.485 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 9m ago 5h 864M 2438M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:11:06.485 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 7m ago 5h 762M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:11:06.485 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 7m ago 5h 950M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:11:06.486 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 7m ago 5h 848M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:11:06.486 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 7m ago 5h 1029M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:11:06.486 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 7m ago 5h 149M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:11:06.836 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:11:06.836 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:11:06.837 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:11:06.837 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:11:06.837 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:11:06.837 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:11:06.837 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:11:06.838 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:11:06.838 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:11:06.838 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:11:06.838 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:11:06.839 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:11:06.839 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:11:06.839 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:11:06.839 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:11:06.839 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:11:06.840 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:11:06.840 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:11:07.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:11:06 smithi153 ceph-mon[70044]: pgmap v10285: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:07.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:11:06 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3930523927' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:11:07.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:11:06 smithi083 ceph-mon[79930]: pgmap v10285: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:07.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:11:06 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3930523927' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:11:07.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:11:06 smithi083 ceph-mon[77242]: pgmap v10285: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:07.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:11:06 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3930523927' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:11:08.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:11:07 smithi153 ceph-mon[70044]: from='client.46095 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:11:08.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:11:07 smithi153 ceph-mon[70044]: from='client.46101 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:11:08.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:11:07 smithi083 ceph-mon[77242]: from='client.46095 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:11:08.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:11:07 smithi083 ceph-mon[77242]: from='client.46101 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:11:08.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:11:07 smithi083 ceph-mon[79930]: from='client.46095 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:11:08.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:11:07 smithi083 ceph-mon[79930]: from='client.46101 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:11:09.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:11:08 smithi153 ceph-mon[70044]: pgmap v10286: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:09.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:11:08 smithi083 ceph-mon[79930]: pgmap v10286: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:09.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:11:08 smithi083 ceph-mon[77242]: pgmap v10286: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:10.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:11:10 smithi083 ceph-mon[77242]: pgmap v10287: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:10.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:11:10 smithi083 ceph-mon[79930]: pgmap v10287: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:11:10 smithi153 ceph-mon[70044]: pgmap v10287: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:11.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:11:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:11:11] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T11:11:11.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:11:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:11:11.425Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:11:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:11:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:11:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:11:13.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:11:12 smithi153 ceph-mon[70044]: pgmap v10288: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:11:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:11:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:11:13.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:11:12 smithi083 ceph-mon[79930]: pgmap v10288: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:13.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:11:12 smithi083 ceph-mon[77242]: pgmap v10288: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:15.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:11:14 smithi083 ceph-mon[79930]: pgmap v10289: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:15.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:11:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:11:14.799Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:11:15.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:11:14 smithi083 ceph-mon[77242]: pgmap v10289: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:15.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:11:14 smithi153 ceph-mon[70044]: pgmap v10289: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:15.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:11:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:11:15.518Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:11:17.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:11:16 smithi153 ceph-mon[70044]: pgmap v10290: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:17.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:11:16 smithi083 ceph-mon[77242]: pgmap v10290: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:17.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:11:16 smithi083 ceph-mon[79930]: pgmap v10290: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:19.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:11:18 smithi153 ceph-mon[70044]: pgmap v10291: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:19.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:11:18 smithi083 ceph-mon[79930]: pgmap v10291: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:19.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:11:18 smithi083 ceph-mon[77242]: pgmap v10291: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:20.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:11:20 smithi083 ceph-mon[79930]: pgmap v10292: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:20.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:11:20 smithi083 ceph-mon[77242]: pgmap v10292: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:20.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:11:20 smithi153 ceph-mon[70044]: pgmap v10292: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:21.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:11:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:11:21] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T11:11:21.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:11:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:11:21.426Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:11:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:11:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:11:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:11:23.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:11:22 smithi153 ceph-mon[70044]: pgmap v10293: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:11:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:11:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:11:23.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:11:22 smithi083 ceph-mon[79930]: pgmap v10293: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:23.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:11:22 smithi083 ceph-mon[77242]: pgmap v10293: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:25.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:11:24 smithi083 ceph-mon[79930]: pgmap v10294: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:25.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:11:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:11:24.800Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:11:25.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:11:24 smithi083 ceph-mon[77242]: pgmap v10294: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:25.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:11:24 smithi153 ceph-mon[70044]: pgmap v10294: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:25.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:11:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:11:25.518Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:11:27.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:11:26 smithi153 ceph-mon[70044]: pgmap v10295: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:27.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:11:26 smithi083 ceph-mon[79930]: pgmap v10295: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:27.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:11:26 smithi083 ceph-mon[77242]: pgmap v10295: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:29.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:11:28 smithi153 ceph-mon[70044]: pgmap v10296: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:29.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:11:28 smithi083 ceph-mon[79930]: pgmap v10296: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:29.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:11:28 smithi083 ceph-mon[77242]: pgmap v10296: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:30.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:11:30 smithi083 ceph-mon[79930]: pgmap v10297: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:30.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:11:30 smithi083 ceph-mon[77242]: pgmap v10297: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:11:30 smithi153 ceph-mon[70044]: pgmap v10297: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:31.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:11:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:11:31] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T11:11:31.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:11:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:11:31.426Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:11:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:11:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:11:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:11:33.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:11:32 smithi153 ceph-mon[70044]: pgmap v10298: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:11:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:11:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:11:33.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:11:32 smithi083 ceph-mon[79930]: pgmap v10298: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:33.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:11:32 smithi083 ceph-mon[77242]: pgmap v10298: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:35.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:11:34 smithi083 ceph-mon[79930]: pgmap v10299: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:35.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:11:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:11:34.801Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:11:35.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:11:34 smithi083 ceph-mon[77242]: pgmap v10299: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:35.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:11:34 smithi153 ceph-mon[70044]: pgmap v10299: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:35.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:11:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:11:35.519Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:11:37.162 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:11:37.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:11:36 smithi153 ceph-mon[70044]: pgmap v10300: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:37.452 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:11:37.453 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 9m ago 5h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:11:37.453 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 7m ago 5h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:11:37.453 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 9m ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:11:37.453 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 7m ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:11:37.453 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 7m ago 5h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:11:37.454 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 9m ago 5h 799M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:11:37.454 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 9m ago 5h 1271M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:11:37.454 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 7m ago 5h 548M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:11:37.454 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 9m ago 5h 731M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:11:37.455 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 9m ago 5h 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:11:37.455 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 7m ago 5h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:11:37.455 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 9m ago 5h 895M 2438M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:11:37.455 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 9m ago 5h 885M 2438M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:11:37.456 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 9m ago 5h 690M 2438M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:11:37.456 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 9m ago 5h 864M 2438M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:11:37.456 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 7m ago 5h 762M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:11:37.456 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 7m ago 5h 950M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:11:37.456 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 7m ago 5h 848M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:11:37.457 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 7m ago 5h 1029M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:11:37.457 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 7m ago 5h 149M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:11:37.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:11:36 smithi083 ceph-mon[79930]: pgmap v10300: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:37.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:11:36 smithi083 ceph-mon[77242]: pgmap v10300: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:37.804 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:11:37.804 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:11:37.805 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:11:37.805 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:11:37.805 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:11:37.805 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:11:37.805 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:11:37.806 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:11:37.806 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:11:37.806 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:11:37.806 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:11:37.807 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:11:37.807 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:11:37.807 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:11:37.807 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:11:37.807 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:11:37.808 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:11:37.808 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:11:38.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:11:37 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/835945514' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:11:38.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:11:37 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/835945514' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:11:38.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:11:37 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/835945514' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:11:39.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:11:38 smithi153 ceph-mon[70044]: from='client.46113 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:11:39.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:11:38 smithi153 ceph-mon[70044]: from='client.56050 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:11:39.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:11:38 smithi153 ceph-mon[70044]: pgmap v10301: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:39.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:11:38 smithi083 ceph-mon[79930]: from='client.46113 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:11:39.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:11:38 smithi083 ceph-mon[79930]: from='client.56050 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:11:39.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:11:38 smithi083 ceph-mon[79930]: pgmap v10301: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:39.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:11:38 smithi083 ceph-mon[77242]: from='client.46113 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:11:39.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:11:38 smithi083 ceph-mon[77242]: from='client.56050 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:11:39.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:11:38 smithi083 ceph-mon[77242]: pgmap v10301: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:40.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:11:40 smithi083 ceph-mon[79930]: pgmap v10302: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:40.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:11:40 smithi083 ceph-mon[77242]: pgmap v10302: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:11:40 smithi153 ceph-mon[70044]: pgmap v10302: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:41.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:11:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:11:41.427Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:11:41.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:11:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:11:41.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:11:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:11:41.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:11:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:11:41.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:11:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:11:41.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:11:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:11:41.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:11:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:11:41.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:11:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:11:41.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:11:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:11:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:11:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:11:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:11:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:11:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:11:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:11:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:11:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:11:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:11:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:11:41] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T11:11:42.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:11:42 smithi083 ceph-mon[79930]: pgmap v10303: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:42.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:11:42 smithi083 ceph-mon[77242]: pgmap v10303: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:42.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:11:42 smithi153 ceph-mon[70044]: pgmap v10303: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:42.559 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:11:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:11:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:11:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:11:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:11:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:11:45.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:11:44 smithi083 ceph-mon[79930]: pgmap v10304: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:45.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:11:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:11:44.802Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:11:45.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:11:44 smithi083 ceph-mon[77242]: pgmap v10304: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:45.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:11:44 smithi153 ceph-mon[70044]: pgmap v10304: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:45.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:11:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:11:45.520Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:11:47.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:11:46 smithi153 ceph-mon[70044]: pgmap v10305: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:47.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:11:46 smithi083 ceph-mon[79930]: pgmap v10305: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:47.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:11:46 smithi083 ceph-mon[77242]: pgmap v10305: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:49.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:11:48 smithi153 ceph-mon[70044]: pgmap v10306: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:49.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:11:48 smithi083 ceph-mon[79930]: pgmap v10306: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:49.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:11:48 smithi083 ceph-mon[77242]: pgmap v10306: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:50.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:11:50 smithi083 ceph-mon[79930]: pgmap v10307: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:50.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:11:50 smithi083 ceph-mon[77242]: pgmap v10307: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:11:50 smithi153 ceph-mon[70044]: pgmap v10307: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:51.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:11:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:11:51] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T11:11:51.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:11:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:11:51.428Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:11:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:11:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:11:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:11:53.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:11:52 smithi153 ceph-mon[70044]: pgmap v10308: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:11:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:11:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:11:53.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:11:52 smithi083 ceph-mon[77242]: pgmap v10308: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:53.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:11:52 smithi083 ceph-mon[79930]: pgmap v10308: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:55.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:11:54 smithi083 ceph-mon[79930]: pgmap v10309: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:55.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:11:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:11:54.802Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:11:55.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:11:54 smithi083 ceph-mon[77242]: pgmap v10309: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:55.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:11:54 smithi153 ceph-mon[70044]: pgmap v10309: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:55.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:11:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:11:55.520Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:11:57.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:11:56 smithi153 ceph-mon[70044]: pgmap v10310: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:57.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:11:56 smithi083 ceph-mon[77242]: pgmap v10310: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:57.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:11:56 smithi083 ceph-mon[79930]: pgmap v10310: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:59.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:11:58 smithi153 ceph-mon[70044]: pgmap v10311: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:59.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:11:58 smithi083 ceph-mon[77242]: pgmap v10311: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:11:59.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:11:58 smithi083 ceph-mon[79930]: pgmap v10311: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:00.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:12:00 smithi083 ceph-mon[77242]: pgmap v10312: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:00.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:12:00 smithi083 ceph-mon[79930]: pgmap v10312: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:12:00 smithi153 ceph-mon[70044]: pgmap v10312: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:01.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:12:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:12:01] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T11:12:01.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:12:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:12:01.428Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:12:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:12:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:12:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:12:03.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:12:02 smithi153 ceph-mon[70044]: pgmap v10313: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:12:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:12:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:12:03.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:12:02 smithi083 ceph-mon[79930]: pgmap v10313: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:03.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:12:02 smithi083 ceph-mon[77242]: pgmap v10313: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:05.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:12:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:12:04.802Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:12:05.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:12:04 smithi083 ceph-mon[79930]: pgmap v10314: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:05.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:12:04 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:12:05.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:12:04 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:12:05.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:12:04 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:12:05.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:12:04 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:12:05.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:12:04 smithi083 ceph-mon[77242]: pgmap v10314: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:05.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:12:04 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:12:05.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:12:04 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:12:05.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:12:04 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:12:05.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:12:04 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:12:05.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:12:04 smithi153 ceph-mon[70044]: pgmap v10314: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:05.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:12:04 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:12:05.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:12:04 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:12:05.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:12:04 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:12:05.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:12:04 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:12:05.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:12:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:12:05.530Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:12:06.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:12:05 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:12:06.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:12:05 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:12:06.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:12:05 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:12:06.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:12:05 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:12:06.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:12:05 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:12:06.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:12:05 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:12:06.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:12:05 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:12:06.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:12:05 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:12:06.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:12:05 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:12:07.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:12:07 smithi153 ceph-mon[70044]: pgmap v10315: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:07.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:12:07 smithi083 ceph-mon[79930]: pgmap v10315: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:07.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:12:07 smithi083 ceph-mon[77242]: pgmap v10315: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:08.133 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:12:08.445 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:12:08.446 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 10m ago 5h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:12:08.446 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 8m ago 5h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:12:08.446 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 10m ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:12:08.447 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 8m ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:12:08.447 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 8m ago 5h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:12:08.447 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 10m ago 5h 799M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:12:08.447 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 10m ago 5h 1271M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:12:08.448 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 8m ago 5h 548M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:12:08.448 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 10m ago 5h 731M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:12:08.448 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 10m ago 5h 26.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:12:08.448 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 8m ago 5h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:12:08.448 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 10m ago 5h 895M 2438M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:12:08.449 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 10m ago 5h 885M 2438M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:12:08.449 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 10m ago 5h 690M 2438M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:12:08.449 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 10m ago 5h 864M 2438M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:12:08.449 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 8m ago 5h 762M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:12:08.450 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 8m ago 5h 950M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:12:08.450 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 8m ago 5h 848M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:12:08.450 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 8m ago 5h 1029M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:12:08.450 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 8m ago 5h 149M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:12:08.822 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:12:08.823 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:12:08.823 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:12:08.824 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:12:08.824 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:12:08.824 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:12:08.824 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:12:08.825 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:12:08.825 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:12:08.825 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:12:08.825 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:12:08.825 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:12:08.826 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:12:08.826 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:12:08.826 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:12:08.826 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:12:08.827 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:12:08.827 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:12:09.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:12:09 smithi153 ceph-mon[70044]: pgmap v10316: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:09.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:12:09 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2558012704' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:12:09.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:12:09 smithi083 ceph-mon[79930]: pgmap v10316: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:09.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:12:09 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2558012704' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:12:09.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:12:09 smithi083 ceph-mon[77242]: pgmap v10316: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:09.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:12:09 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2558012704' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:12:10.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:12:10 smithi083 ceph-mon[79930]: from='client.46131 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:12:10.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:12:10 smithi083 ceph-mon[79930]: from='client.46137 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:12:10.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:12:10 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:12:10.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:12:10 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:12:10.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:12:10 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:12:10.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:12:10 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:12:10.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:12:10 smithi083 ceph-mon[77242]: from='client.46131 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:12:10.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:12:10 smithi083 ceph-mon[77242]: from='client.46137 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:12:10.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:12:10 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:12:10.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:12:10 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:12:10.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:12:10 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:12:10.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:12:10 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:12:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:12:10 smithi153 ceph-mon[70044]: from='client.46131 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:12:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:12:10 smithi153 ceph-mon[70044]: from='client.46137 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:12:10.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:12:10 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:12:10.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:12:10 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:12:10.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:12:10 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:12:10.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:12:10 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:12:11.205 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:12:11 smithi083 ceph-mon[79930]: pgmap v10317: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:11.206 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:12:11 smithi083 ceph-mon[77242]: pgmap v10317: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:11.458 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:12:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:12:11.429Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:12:11.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:12:11 smithi153 ceph-mon[70044]: pgmap v10317: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:11.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:12:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:12:11] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T11:12:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:12:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:12:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:12:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:12:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:12:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:12:13.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:12:13 smithi153 ceph-mon[70044]: pgmap v10318: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:13.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:12:13 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:12:13.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:12:13 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:12:13.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:12:13 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:12:13.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:12:13 smithi083 ceph-mon[79930]: pgmap v10318: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:13.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:12:13 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:12:13.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:12:13 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:12:13.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:12:13 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:12:13.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:12:13 smithi083 ceph-mon[77242]: pgmap v10318: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:13.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:12:13 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:12:13.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:12:13 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:12:13.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:12:13 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:12:14.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:12:14 smithi083 ceph-mon[79930]: Detected new or changed devices on smithi083 2022-09-29T11:12:14.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:12:14 smithi083 ceph-mon[77242]: Detected new or changed devices on smithi083 2022-09-29T11:12:14.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:12:14 smithi153 ceph-mon[70044]: Detected new or changed devices on smithi083 2022-09-29T11:12:15.081 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:12:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:12:14.803Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:12:15.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:12:15 smithi083 ceph-mon[79930]: pgmap v10319: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:15.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:12:15 smithi083 ceph-mon[77242]: pgmap v10319: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:15.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:12:15 smithi153 ceph-mon[70044]: pgmap v10319: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:15.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:12:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:12:15.522Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:12:16.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:12:16 smithi083 ceph-mon[79930]: pgmap v10320: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:16.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:12:16 smithi083 ceph-mon[77242]: pgmap v10320: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:12:16 smithi153 ceph-mon[70044]: pgmap v10320: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:19.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:12:18 smithi153 ceph-mon[70044]: pgmap v10321: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:19.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:12:18 smithi083 ceph-mon[79930]: pgmap v10321: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:19.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:12:18 smithi083 ceph-mon[77242]: pgmap v10321: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:20.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:12:20 smithi083 ceph-mon[79930]: pgmap v10322: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:20.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:12:20 smithi083 ceph-mon[77242]: pgmap v10322: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:12:20 smithi153 ceph-mon[70044]: pgmap v10322: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:21.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:12:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:12:21] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T11:12:21.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:12:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:12:21.430Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:12:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:12:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:12:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:12:23.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:12:22 smithi153 ceph-mon[70044]: pgmap v10323: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:12:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:12:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:12:23.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:12:22 smithi083 ceph-mon[79930]: pgmap v10323: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:23.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:12:22 smithi083 ceph-mon[77242]: pgmap v10323: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:25.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:12:24 smithi083 ceph-mon[79930]: pgmap v10324: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:25.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:12:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:12:24.805Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:12:25.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:12:24 smithi083 ceph-mon[77242]: pgmap v10324: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:25.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:12:24 smithi153 ceph-mon[70044]: pgmap v10324: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:25.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:12:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:12:25.522Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:12:27.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:12:26 smithi153 ceph-mon[70044]: pgmap v10325: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:27.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:12:26 smithi083 ceph-mon[79930]: pgmap v10325: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:27.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:12:26 smithi083 ceph-mon[77242]: pgmap v10325: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:29.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:12:29 smithi153 ceph-mon[70044]: pgmap v10326: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:29.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:12:29 smithi083 ceph-mon[79930]: pgmap v10326: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:29.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:12:29 smithi083 ceph-mon[77242]: pgmap v10326: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:30.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:12:30 smithi083 ceph-mon[79930]: pgmap v10327: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:30.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:12:30 smithi083 ceph-mon[77242]: pgmap v10327: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:12:30 smithi153 ceph-mon[70044]: pgmap v10327: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:31.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:12:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:12:31] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T11:12:31.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:12:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:12:31.431Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:12:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:12:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:12:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:12:33.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:12:32 smithi153 ceph-mon[70044]: pgmap v10328: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:12:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:12:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:12:33.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:12:32 smithi083 ceph-mon[79930]: pgmap v10328: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:33.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:12:32 smithi083 ceph-mon[77242]: pgmap v10328: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:35.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:12:34 smithi083 ceph-mon[79930]: pgmap v10329: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:35.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:12:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:12:34.805Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:12:35.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:12:34 smithi083 ceph-mon[77242]: pgmap v10329: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:35.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:12:34 smithi153 ceph-mon[70044]: pgmap v10329: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:35.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:12:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:12:35.522Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:12:37.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:12:37 smithi153 ceph-mon[70044]: pgmap v10330: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:37.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:12:37 smithi083 ceph-mon[79930]: pgmap v10330: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:37.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:12:37 smithi083 ceph-mon[77242]: pgmap v10330: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:39.157 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:12:39.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:12:39 smithi153 ceph-mon[70044]: pgmap v10331: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:39.452 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:12:39.453 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 30s ago 5h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:12:39.453 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 8m ago 5h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:12:39.453 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 30s ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:12:39.453 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 8m ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:12:39.454 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 8m ago 5h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:12:39.454 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 30s ago 5h 807M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:12:39.454 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 30s ago 5h 1307M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:12:39.455 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 8m ago 5h 548M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:12:39.455 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 30s ago 5h 753M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:12:39.455 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 30s ago 5h 26.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:12:39.455 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 8m ago 5h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:12:39.456 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 30s ago 5h 921M 2438M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:12:39.456 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 30s ago 5h 910M 2438M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:12:39.456 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 30s ago 5h 710M 2438M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:12:39.456 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 30s ago 5h 889M 2438M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:12:39.456 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 8m ago 5h 762M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:12:39.457 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 8m ago 5h 950M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:12:39.457 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 8m ago 5h 848M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:12:39.457 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 8m ago 5h 1029M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:12:39.457 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 8m ago 5h 149M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:12:39.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:12:39 smithi083 ceph-mon[79930]: pgmap v10331: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:39.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:12:39 smithi083 ceph-mon[77242]: pgmap v10331: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:39.805 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:12:39.806 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:12:39.806 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:12:39.806 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:12:39.807 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:12:39.807 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:12:39.807 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:12:39.807 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:12:39.807 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:12:39.808 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:12:39.808 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:12:39.808 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:12:39.808 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:12:39.809 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:12:39.809 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:12:39.809 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:12:39.809 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:12:39.810 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:12:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:12:40 smithi153 ceph-mon[70044]: from='client.56080 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:12:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:12:40 smithi153 ceph-mon[70044]: from='client.46155 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:12:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:12:40 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3336510063' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:12:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:12:40 smithi153 ceph-mon[70044]: pgmap v10332: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:40.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:12:40 smithi083 ceph-mon[79930]: from='client.56080 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:12:40.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:12:40 smithi083 ceph-mon[79930]: from='client.46155 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:12:40.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:12:40 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3336510063' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:12:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:12:40 smithi083 ceph-mon[79930]: pgmap v10332: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:12:40 smithi083 ceph-mon[77242]: from='client.56080 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:12:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:12:40 smithi083 ceph-mon[77242]: from='client.46155 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:12:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:12:40 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3336510063' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:12:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:12:40 smithi083 ceph-mon[77242]: pgmap v10332: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:41.507 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:12:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:12:41.432Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:12:41.507 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:12:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:12:41.508 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:12:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:12:41.508 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:12:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:12:41.508 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:12:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:12:41.508 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:12:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:12:41.509 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:12:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:12:41.509 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:12:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:12:41.509 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:12:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:12:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:12:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:12:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:12:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:12:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:12:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:12:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:12:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:12:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:12:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:12:41] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T11:12:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:12:42 smithi153 ceph-mon[70044]: pgmap v10333: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:42.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:12:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:12:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:12:42.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:12:42 smithi083 ceph-mon[79930]: pgmap v10333: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:42.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:12:42 smithi083 ceph-mon[77242]: pgmap v10333: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:12:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:12:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:12:45.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:12:44 smithi083 ceph-mon[77242]: pgmap v10334: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:45.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:12:44 smithi083 ceph-mon[79930]: pgmap v10334: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:45.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:12:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:12:44.805Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:12:45.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:12:44 smithi153 ceph-mon[70044]: pgmap v10334: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:45.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:12:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:12:45.523Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:12:47.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:12:47 smithi153 ceph-mon[70044]: pgmap v10335: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:47.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:12:47 smithi083 ceph-mon[79930]: pgmap v10335: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:47.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:12:47 smithi083 ceph-mon[77242]: pgmap v10335: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:49.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:12:49 smithi153 ceph-mon[70044]: pgmap v10336: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:49.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:12:49 smithi083 ceph-mon[79930]: pgmap v10336: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:49.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:12:49 smithi083 ceph-mon[77242]: pgmap v10336: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:12:50 smithi153 ceph-mon[70044]: pgmap v10337: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:50.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:12:50 smithi083 ceph-mon[77242]: pgmap v10337: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:50.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:12:50 smithi083 ceph-mon[79930]: pgmap v10337: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:51.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:12:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:12:51] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T11:12:51.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:12:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:12:51.432Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:12:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:12:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:12:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:12:53.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:12:53 smithi153 ceph-mon[70044]: pgmap v10338: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:12:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:12:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:12:53.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:12:53 smithi083 ceph-mon[79930]: pgmap v10338: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:53.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:12:53 smithi083 ceph-mon[77242]: pgmap v10338: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:55.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:12:55 smithi083 ceph-mon[79930]: pgmap v10339: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:55.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:12:55 smithi083 ceph-mon[77242]: pgmap v10339: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:55.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:12:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:12:54.807Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:12:55.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:12:55 smithi153 ceph-mon[70044]: pgmap v10339: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:55.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:12:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:12:55.523Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:12:57.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:12:57 smithi153 ceph-mon[70044]: pgmap v10340: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:57.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:12:57 smithi083 ceph-mon[77242]: pgmap v10340: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:57.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:12:57 smithi083 ceph-mon[79930]: pgmap v10340: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:59.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:12:59 smithi153 ceph-mon[70044]: pgmap v10341: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:59.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:12:59 smithi083 ceph-mon[79930]: pgmap v10341: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:12:59.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:12:59 smithi083 ceph-mon[77242]: pgmap v10341: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:13:00 smithi153 ceph-mon[70044]: pgmap v10342: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:00.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:13:00 smithi083 ceph-mon[79930]: pgmap v10342: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:00.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:13:00 smithi083 ceph-mon[77242]: pgmap v10342: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:01.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:13:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:13:01] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T11:13:01.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:13:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:13:01.432Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:13:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:13:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:13:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:13:03.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:13:03 smithi153 ceph-mon[70044]: pgmap v10343: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:13:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:13:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:13:03.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:13:03 smithi083 ceph-mon[79930]: pgmap v10343: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:03.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:13:03 smithi083 ceph-mon[77242]: pgmap v10343: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:05.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:13:05 smithi083 ceph-mon[79930]: pgmap v10344: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:05.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:13:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:13:04.807Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:13:05.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:13:05 smithi083 ceph-mon[77242]: pgmap v10344: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:05.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:13:05 smithi153 ceph-mon[70044]: pgmap v10344: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:05.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:13:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:13:05.524Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:13:07.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:13:07 smithi153 ceph-mon[70044]: pgmap v10345: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:07.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:13:07 smithi083 ceph-mon[77242]: pgmap v10345: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:07.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:13:07 smithi083 ceph-mon[79930]: pgmap v10345: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:09.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:13:09 smithi153 ceph-mon[70044]: pgmap v10346: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:09.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:13:09 smithi083 ceph-mon[79930]: pgmap v10346: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:09.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:13:09 smithi083 ceph-mon[77242]: pgmap v10346: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:10.132 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:13:10.421 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:13:10.421 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 61s ago 5h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:13:10.421 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 9m ago 5h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:13:10.422 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 61s ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:13:10.422 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 9m ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:13:10.422 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 9m ago 5h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:13:10.422 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 61s ago 5h 807M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:13:10.423 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 61s ago 5h 1307M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:13:10.423 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 9m ago 5h 548M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:13:10.423 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 61s ago 5h 753M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:13:10.423 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 61s ago 5h 26.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:13:10.424 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 9m ago 5h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:13:10.424 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 61s ago 5h 921M 2438M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:13:10.424 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 61s ago 5h 910M 2438M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:13:10.424 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 61s ago 5h 710M 2438M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:13:10.424 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 61s ago 5h 889M 2438M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:13:10.425 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 9m ago 5h 762M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:13:10.425 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 9m ago 5h 950M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:13:10.425 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 9m ago 5h 848M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:13:10.425 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 9m ago 5h 1029M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:13:10.426 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 9m ago 5h 149M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:13:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:13:10 smithi153 ceph-mon[70044]: pgmap v10347: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:10.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:13:10 smithi083 ceph-mon[79930]: pgmap v10347: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:10.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:13:10 smithi083 ceph-mon[77242]: pgmap v10347: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:10.783 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:13:10.784 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:13:10.784 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:13:10.784 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:13:10.785 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:13:10.785 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:13:10.785 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:13:10.786 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:13:10.786 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:13:10.786 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:13:10.786 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:13:10.787 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:13:10.787 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:13:10.787 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:13:10.787 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:13:10.787 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:13:10.788 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:13:10.788 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:13:11.506 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:13:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:13:11.433Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:13:11.507 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:13:11 smithi083 ceph-mon[79930]: from='client.56098 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:13:11.507 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:13:11 smithi083 ceph-mon[79930]: from='client.46173 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:13:11.508 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:13:11 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/105078401' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:13:11.508 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:13:11 smithi083 ceph-mon[77242]: from='client.56098 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:13:11.508 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:13:11 smithi083 ceph-mon[77242]: from='client.46173 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:13:11.509 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:13:11 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/105078401' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:13:11.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:13:11 smithi153 ceph-mon[70044]: from='client.56098 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:13:11.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:13:11 smithi153 ceph-mon[70044]: from='client.46173 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:13:11.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:13:11 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/105078401' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:13:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:13:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:13:11] "GET /metrics HTTP/1.1" 200 225229 "" "Prometheus/2.33.4" 2022-09-29T11:13:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:13:12 smithi153 ceph-mon[70044]: pgmap v10348: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:12.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:13:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:13:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:13:12.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:13:12 smithi083 ceph-mon[79930]: pgmap v10348: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:12.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:13:12 smithi083 ceph-mon[77242]: pgmap v10348: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:13:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:13:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:13:13.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:13:13 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:13:13.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:13:13 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:13:13.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:13:13 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:13:13.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:13:13 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:13:13.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:13:13 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:13:13.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:13:13 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:13:13.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:13:13 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:13:13.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:13:13 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:13:13.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:13:13 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:13:14.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:13:14 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:13:14.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:13:14 smithi083 ceph-mon[77242]: pgmap v10349: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:14.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:13:14 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:13:14.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:13:14 smithi083 ceph-mon[79930]: pgmap v10349: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:14.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:13:14 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:13:14.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:13:14 smithi153 ceph-mon[70044]: pgmap v10349: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:15.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:13:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:13:14.808Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:13:15.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:13:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:13:15.525Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:13:17.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:13:17 smithi153 ceph-mon[70044]: pgmap v10350: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:17.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:13:17 smithi083 ceph-mon[79930]: pgmap v10350: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:17.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:13:17 smithi083 ceph-mon[77242]: pgmap v10350: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:19.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:13:19 smithi153 ceph-mon[70044]: pgmap v10351: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:19.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:13:19 smithi083 ceph-mon[79930]: pgmap v10351: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:19.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:13:19 smithi083 ceph-mon[77242]: pgmap v10351: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:13:20 smithi153 ceph-mon[70044]: pgmap v10352: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:20.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:13:20 smithi083 ceph-mon[77242]: pgmap v10352: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:20.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:13:20 smithi083 ceph-mon[79930]: pgmap v10352: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:21.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:13:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:13:21] "GET /metrics HTTP/1.1" 200 225229 "" "Prometheus/2.33.4" 2022-09-29T11:13:21.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:13:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:13:21.434Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:13:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:13:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:13:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:13:23.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:13:23 smithi153 ceph-mon[70044]: pgmap v10353: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:13:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:13:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:13:23.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:13:23 smithi083 ceph-mon[79930]: pgmap v10353: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:23.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:13:23 smithi083 ceph-mon[77242]: pgmap v10353: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:25.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:13:25 smithi083 ceph-mon[79930]: pgmap v10354: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:25.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:13:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:13:24.808Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:13:25.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:13:25 smithi083 ceph-mon[77242]: pgmap v10354: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:25.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:13:25 smithi153 ceph-mon[70044]: pgmap v10354: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:25.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:13:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:13:25.525Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:13:27.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:13:27 smithi153 ceph-mon[70044]: pgmap v10355: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:27.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:13:27 smithi083 ceph-mon[77242]: pgmap v10355: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:27.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:13:27 smithi083 ceph-mon[79930]: pgmap v10355: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:29.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:13:29 smithi153 ceph-mon[70044]: pgmap v10356: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:29.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:13:29 smithi083 ceph-mon[77242]: pgmap v10356: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:29.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:13:29 smithi083 ceph-mon[79930]: pgmap v10356: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:13:30 smithi153 ceph-mon[70044]: pgmap v10357: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:30.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:13:30 smithi083 ceph-mon[79930]: pgmap v10357: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:30.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:13:30 smithi083 ceph-mon[77242]: pgmap v10357: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:31.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:13:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:13:31] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T11:13:31.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:13:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:13:31.435Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:13:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:13:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:13:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:13:33.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:13:33 smithi153 ceph-mon[70044]: pgmap v10358: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:13:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:13:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:13:33.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:13:33 smithi083 ceph-mon[77242]: pgmap v10358: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:33.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:13:33 smithi083 ceph-mon[79930]: pgmap v10358: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:35.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:13:35 smithi083 ceph-mon[79930]: pgmap v10359: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:35.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:13:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:13:34.808Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:13:35.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:13:35 smithi083 ceph-mon[77242]: pgmap v10359: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:35.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:13:35 smithi153 ceph-mon[70044]: pgmap v10359: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:35.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:13:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:13:35.526Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:13:37.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:13:37 smithi153 ceph-mon[70044]: pgmap v10360: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:37.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:13:37 smithi083 ceph-mon[79930]: pgmap v10360: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:37.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:13:37 smithi083 ceph-mon[77242]: pgmap v10360: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:39.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:13:39 smithi153 ceph-mon[70044]: pgmap v10361: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:39.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:13:39 smithi083 ceph-mon[77242]: pgmap v10361: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:39.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:13:39 smithi083 ceph-mon[79930]: pgmap v10361: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:13:40 smithi153 ceph-mon[70044]: pgmap v10362: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:40.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:13:40 smithi083 ceph-mon[77242]: pgmap v10362: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:40.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:13:40 smithi083 ceph-mon[79930]: pgmap v10362: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:41.105 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:13:41.393 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:13:41.393 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 92s ago 5h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:13:41.394 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 9m ago 5h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:13:41.394 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 92s ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:13:41.394 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 9m ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:13:41.394 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 9m ago 5h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:13:41.395 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 92s ago 5h 807M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:13:41.395 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 92s ago 5h 1307M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:13:41.395 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 9m ago 5h 548M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:13:41.395 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 92s ago 5h 753M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:13:41.395 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 92s ago 5h 26.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:13:41.396 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 9m ago 5h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:13:41.396 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 92s ago 5h 921M 2438M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:13:41.396 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 92s ago 5h 910M 2438M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:13:41.396 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 92s ago 5h 710M 2438M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:13:41.397 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 92s ago 5h 889M 2438M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:13:41.397 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 9m ago 5h 762M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:13:41.397 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 9m ago 5h 950M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:13:41.397 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 9m ago 5h 848M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:13:41.397 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 9m ago 5h 1029M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:13:41.398 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 9m ago 5h 149M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:13:41.507 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:13:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:13:41.508 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:13:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:13:41.508 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:13:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:13:41.508 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:13:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:13:41.509 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:13:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:13:41.509 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:13:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:13:41.509 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:13:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:13:41.509 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:13:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:13:41.510 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:13:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:13:41.434Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:13:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:13:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:13:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:13:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:13:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:13:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:13:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:13:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:13:41.752 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:13:41.752 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:13:41.752 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:13:41.752 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:13:41.753 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:13:41.753 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:13:41.753 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:13:41.753 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:13:41.754 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:13:41.754 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:13:41.754 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:13:41.754 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:13:41.755 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:13:41.755 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:13:41.755 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:13:41.755 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:13:41.756 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:13:41.756 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:13:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:13:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:13:41] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T11:13:42.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:13:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:13:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:13:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:13:42 smithi153 ceph-mon[70044]: from='client.56116 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:13:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:13:42 smithi153 ceph-mon[70044]: from='client.46191 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:13:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:13:42 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3705396424' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:13:42.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:13:42 smithi153 ceph-mon[70044]: pgmap v10363: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:42.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:13:42 smithi083 ceph-mon[79930]: from='client.56116 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:13:42.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:13:42 smithi083 ceph-mon[79930]: from='client.46191 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:13:42.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:13:42 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3705396424' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:13:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:13:42 smithi083 ceph-mon[79930]: pgmap v10363: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:42.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:13:42 smithi083 ceph-mon[77242]: from='client.56116 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:13:42.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:13:42 smithi083 ceph-mon[77242]: from='client.46191 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:13:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:13:42 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3705396424' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:13:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:13:42 smithi083 ceph-mon[77242]: pgmap v10363: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:13:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:13:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:13:45.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:13:45 smithi083 ceph-mon[79930]: pgmap v10364: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:45.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:13:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:13:44.809Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:13:45.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:13:45 smithi083 ceph-mon[77242]: pgmap v10364: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:45.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:13:45 smithi153 ceph-mon[70044]: pgmap v10364: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:45.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:13:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:13:45.528Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:13:47.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:13:47 smithi153 ceph-mon[70044]: pgmap v10365: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:47.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:13:47 smithi083 ceph-mon[79930]: pgmap v10365: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:47.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:13:47 smithi083 ceph-mon[77242]: pgmap v10365: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:49.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:13:49 smithi153 ceph-mon[70044]: pgmap v10366: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:49.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:13:49 smithi083 ceph-mon[79930]: pgmap v10366: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:49.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:13:49 smithi083 ceph-mon[77242]: pgmap v10366: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:13:50 smithi153 ceph-mon[70044]: pgmap v10367: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:50.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:13:50 smithi083 ceph-mon[79930]: pgmap v10367: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:50.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:13:50 smithi083 ceph-mon[77242]: pgmap v10367: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:51.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:13:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:13:51] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T11:13:51.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:13:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:13:51.436Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:13:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:13:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:13:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:13:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:13:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:13:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:13:53.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:13:53 smithi153 ceph-mon[70044]: pgmap v10368: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:53.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:13:53 smithi083 ceph-mon[79930]: pgmap v10368: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:53.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:13:53 smithi083 ceph-mon[77242]: pgmap v10368: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:55.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:13:55 smithi083 ceph-mon[79930]: pgmap v10369: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:55.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:13:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:13:54.809Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:13:55.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:13:55 smithi083 ceph-mon[77242]: pgmap v10369: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:55.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:13:55 smithi153 ceph-mon[70044]: pgmap v10369: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:55.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:13:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:13:55.529Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:13:57.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:13:57 smithi153 ceph-mon[70044]: pgmap v10370: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:57.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:13:57 smithi083 ceph-mon[79930]: pgmap v10370: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:57.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:13:57 smithi083 ceph-mon[77242]: pgmap v10370: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:59.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:13:59 smithi153 ceph-mon[70044]: pgmap v10371: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:59.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:13:59 smithi083 ceph-mon[79930]: pgmap v10371: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:13:59.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:13:59 smithi083 ceph-mon[77242]: pgmap v10371: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:14:00 smithi153 ceph-mon[70044]: pgmap v10372: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:00.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:14:00 smithi083 ceph-mon[79930]: pgmap v10372: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:00.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:14:00 smithi083 ceph-mon[77242]: pgmap v10372: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:01.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:14:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:14:01] "GET /metrics HTTP/1.1" 200 225228 "" "Prometheus/2.33.4" 2022-09-29T11:14:01.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:14:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:14:01.436Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:14:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:14:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:14:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:14:03.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:14:03 smithi153 ceph-mon[70044]: pgmap v10373: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:14:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:14:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:14:03.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:14:03 smithi083 ceph-mon[79930]: pgmap v10373: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:03.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:14:03 smithi083 ceph-mon[77242]: pgmap v10373: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:05.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:14:05 smithi083 ceph-mon[79930]: pgmap v10374: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:05.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:14:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:14:04.810Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:14:05.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:14:05 smithi083 ceph-mon[77242]: pgmap v10374: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:05.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:14:05 smithi153 ceph-mon[70044]: pgmap v10374: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:05.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:14:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:14:05.529Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:14:07.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:14:07 smithi153 ceph-mon[70044]: pgmap v10375: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:07.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:14:07 smithi083 ceph-mon[77242]: pgmap v10375: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:07.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:14:07 smithi083 ceph-mon[79930]: pgmap v10375: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:09.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:14:09 smithi153 ceph-mon[70044]: pgmap v10376: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:09.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:14:09 smithi083 ceph-mon[79930]: pgmap v10376: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:09.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:14:09 smithi083 ceph-mon[77242]: pgmap v10376: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:14:10 smithi153 ceph-mon[70044]: pgmap v10377: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:10.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:14:10 smithi083 ceph-mon[79930]: pgmap v10377: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:10.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:14:10 smithi083 ceph-mon[77242]: pgmap v10377: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:11.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:14:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:14:11] "GET /metrics HTTP/1.1" 200 225226 "" "Prometheus/2.33.4" 2022-09-29T11:14:11.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:14:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:14:11.437Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:14:12.078 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:14:12.365 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:14:12.365 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 2m ago 5h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:14:12.366 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 10m ago 5h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:14:12.366 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 2m ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:14:12.366 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 10m ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:14:12.366 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 10m ago 5h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:14:12.367 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 2m ago 5h 807M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:14:12.367 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 2m ago 5h 1307M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:14:12.367 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 10m ago 5h 548M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:14:12.367 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 2m ago 5h 753M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:14:12.368 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 2m ago 5h 26.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:14:12.368 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 10m ago 5h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:14:12.368 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 2m ago 5h 921M 2438M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:14:12.368 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 2m ago 5h 910M 2438M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:14:12.369 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 2m ago 5h 710M 2438M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:14:12.369 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 2m ago 5h 889M 2438M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:14:12.369 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 10m ago 5h 762M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:14:12.369 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 10m ago 5h 950M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:14:12.369 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 10m ago 5h 848M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:14:12.370 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 10m ago 5h 1029M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:14:12.370 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 10m ago 5h 149M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:14:12.719 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:14:12.719 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:14:12.719 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:14:12.720 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:14:12.720 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:14:12.720 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:14:12.720 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:14:12.720 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:14:12.721 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:14:12.721 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:14:12.721 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:14:12.721 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:14:12.722 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:14:12.722 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:14:12.722 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:14:12.722 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:14:12.723 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:14:12.723 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:14:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:14:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:14:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:14:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:14:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:14:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:14:13.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:14:13 smithi153 ceph-mon[70044]: pgmap v10378: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:13.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:14:13 smithi153 ceph-mon[70044]: from='client.46203 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:14:13.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:14:13 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4085479180' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:14:13.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:14:13 smithi083 ceph-mon[79930]: pgmap v10378: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:13.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:14:13 smithi083 ceph-mon[79930]: from='client.46203 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:14:13.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:14:13 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4085479180' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:14:13.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:14:13 smithi083 ceph-mon[77242]: pgmap v10378: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:13.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:14:13 smithi083 ceph-mon[77242]: from='client.46203 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:14:13.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:14:13 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4085479180' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:14:14.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:14:14 smithi153 ceph-mon[70044]: from='client.56140 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:14:14.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:14:14 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:14:14.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:14:14 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:14:14.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:14:14 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:14:14.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:14:14 smithi083 ceph-mon[79930]: from='client.56140 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:14:14.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:14:14 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:14:14.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:14:14 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:14:14.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:14:14 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:14:14.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:14:14 smithi083 ceph-mon[77242]: from='client.56140 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:14:14.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:14:14 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:14:14.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:14:14 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:14:14.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:14:14 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:14:15.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:14:15 smithi083 ceph-mon[79930]: pgmap v10379: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:15.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:14:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:14:14.811Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:14:15.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:14:15 smithi083 ceph-mon[77242]: pgmap v10379: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:15.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:14:15 smithi153 ceph-mon[70044]: pgmap v10379: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:15.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:14:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:14:15.530Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:14:17.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:14:17 smithi153 ceph-mon[70044]: pgmap v10380: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:17.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:14:17 smithi083 ceph-mon[77242]: pgmap v10380: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:17.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:14:17 smithi083 ceph-mon[79930]: pgmap v10380: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:19.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:14:19 smithi153 ceph-mon[70044]: pgmap v10381: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:19.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:14:19 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:14:19.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:14:19 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:14:19.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:14:19 smithi083 ceph-mon[79930]: pgmap v10381: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:19.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:14:19 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:14:19.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:14:19 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:14:19.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:14:19 smithi083 ceph-mon[77242]: pgmap v10381: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:19.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:14:19 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:14:19.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:14:19 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:14:20.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:14:20 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:14:20.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:14:20 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:14:20.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:14:20 smithi153 ceph-mon[70044]: pgmap v10382: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:20.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:14:20 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:14:20.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:14:20 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:14:20.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:14:20 smithi083 ceph-mon[79930]: pgmap v10382: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:20.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:14:20 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:14:20.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:14:20 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:14:20.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:14:20 smithi083 ceph-mon[77242]: pgmap v10382: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:21.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:14:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:14:21] "GET /metrics HTTP/1.1" 200 225226 "" "Prometheus/2.33.4" 2022-09-29T11:14:21.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:14:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:14:21.438Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:14:22.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:14:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:14:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:14:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:14:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:14:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:14:23.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:14:23 smithi153 ceph-mon[70044]: pgmap v10383: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:23.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:14:23 smithi153 ceph-mon[70044]: Detected new or changed devices on smithi153 2022-09-29T11:14:23.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:14:23 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:14:23.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:14:23 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:14:23.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:14:23 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:14:23.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:14:23 smithi083 ceph-mon[79930]: pgmap v10383: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:23.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:14:23 smithi083 ceph-mon[79930]: Detected new or changed devices on smithi153 2022-09-29T11:14:23.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:14:23 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:14:23.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:14:23 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:14:23.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:14:23 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:14:23.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:14:23 smithi083 ceph-mon[77242]: pgmap v10383: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:23.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:14:23 smithi083 ceph-mon[77242]: Detected new or changed devices on smithi153 2022-09-29T11:14:23.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:14:23 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:14:23.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:14:23 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:14:23.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:14:23 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:14:25.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:14:25 smithi083 ceph-mon[79930]: pgmap v10384: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:25.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:14:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:14:24.812Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:14:25.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:14:25 smithi083 ceph-mon[77242]: pgmap v10384: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:25.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:14:25 smithi153 ceph-mon[70044]: pgmap v10384: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:25.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:14:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:14:25.530Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:14:27.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:14:27 smithi083 ceph-mon[79930]: pgmap v10385: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:27.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:14:27 smithi083 ceph-mon[77242]: pgmap v10385: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:27.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:14:27 smithi153 ceph-mon[70044]: pgmap v10385: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:29.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:14:29 smithi083 ceph-mon[79930]: pgmap v10386: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:29.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:14:29 smithi083 ceph-mon[77242]: pgmap v10386: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:29.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:14:29 smithi153 ceph-mon[70044]: pgmap v10386: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:14:30 smithi153 ceph-mon[70044]: pgmap v10387: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:30.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:14:30 smithi083 ceph-mon[77242]: pgmap v10387: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:30.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:14:30 smithi083 ceph-mon[79930]: pgmap v10387: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:31.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:14:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:14:31] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T11:14:31.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:14:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:14:31.438Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:14:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:14:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:14:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:14:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:14:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:14:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:14:33.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:14:33 smithi153 ceph-mon[70044]: pgmap v10388: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:33.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:14:33 smithi083 ceph-mon[77242]: pgmap v10388: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:33.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:14:33 smithi083 ceph-mon[79930]: pgmap v10388: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:35.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:14:35 smithi083 ceph-mon[77242]: pgmap v10389: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:35.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:14:35 smithi083 ceph-mon[79930]: pgmap v10389: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:35.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:14:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:14:34.813Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:14:35.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:14:35 smithi153 ceph-mon[70044]: pgmap v10389: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:35.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:14:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:14:35.531Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:14:37.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:14:37 smithi153 ceph-mon[70044]: pgmap v10390: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:37.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:14:37 smithi083 ceph-mon[77242]: pgmap v10390: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:37.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:14:37 smithi083 ceph-mon[79930]: pgmap v10390: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:39.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:14:39 smithi153 ceph-mon[70044]: pgmap v10391: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:39.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:14:39 smithi083 ceph-mon[79930]: pgmap v10391: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:39.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:14:39 smithi083 ceph-mon[77242]: pgmap v10391: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:14:40 smithi153 ceph-mon[70044]: pgmap v10392: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:40.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:14:40 smithi083 ceph-mon[79930]: pgmap v10392: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:40.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:14:40 smithi083 ceph-mon[77242]: pgmap v10392: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:41.507 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:14:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:14:41.439Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:14:41.508 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:14:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:14:41.508 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:14:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:14:41.508 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:14:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:14:41.508 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:14:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:14:41.509 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:14:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:14:41.509 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:14:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:14:41.509 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:14:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:14:41.510 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:14:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:14:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:14:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:14:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:14:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:14:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:14:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:14:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:14:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:14:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:14:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:14:41] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T11:14:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:14:42 smithi153 ceph-mon[70044]: pgmap v10393: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:42.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:14:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:14:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:14:42.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:14:42 smithi083 ceph-mon[77242]: pgmap v10393: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:42.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:14:42 smithi083 ceph-mon[79930]: pgmap v10393: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:43.043 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:14:43.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:14:43 smithi153 ceph-mon[70044]: from='client.56152 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:14:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:14:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:14:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:14:43.336 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:14:43.352 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 2m ago 5h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:14:43.353 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 24s ago 5h 64.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:14:43.353 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 2m ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:14:43.353 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 24s ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:14:43.353 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 24s ago 5h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:14:43.354 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 2m ago 5h 807M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:14:43.354 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 2m ago 5h 1307M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:14:43.354 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 24s ago 5h 563M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:14:43.354 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 2m ago 5h 753M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:14:43.355 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 2m ago 5h 26.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:14:43.355 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 24s ago 5h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:14:43.355 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 2m ago 5h 921M 2438M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:14:43.355 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 2m ago 5h 910M 2438M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:14:43.355 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 2m ago 5h 710M 2438M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:14:43.356 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 2m ago 5h 889M 2438M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:14:43.356 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 24s ago 5h 783M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:14:43.356 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 24s ago 5h 977M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:14:43.356 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 24s ago 5h 873M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:14:43.356 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 24s ago 5h 1058M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:14:43.357 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 24s ago 5h 150M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:14:43.697 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:14:43.698 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:14:43.698 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:14:43.698 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:14:43.699 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:14:43.699 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:14:43.699 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:14:43.699 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:14:43.700 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:14:43.700 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:14:43.700 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:14:43.700 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:14:43.701 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:14:43.701 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:14:43.701 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:14:43.701 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:14:43.701 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:14:43.702 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:14:43.710 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:14:43 smithi083 ceph-mon[77242]: from='client.56152 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:14:43.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:14:43 smithi083 ceph-mon[79930]: from='client.56152 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:14:44.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:14:44 smithi153 ceph-mon[70044]: from='client.46227 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:14:44.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:14:44 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/425145832' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:14:44.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:14:44 smithi153 ceph-mon[70044]: pgmap v10394: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:44.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:14:44 smithi083 ceph-mon[79930]: from='client.46227 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:14:44.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:14:44 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/425145832' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:14:44.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:14:44 smithi083 ceph-mon[79930]: pgmap v10394: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:44.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:14:44 smithi083 ceph-mon[77242]: from='client.46227 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:14:44.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:14:44 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/425145832' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:14:44.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:14:44 smithi083 ceph-mon[77242]: pgmap v10394: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:45.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:14:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:14:44.813Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:14:45.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:14:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:14:45.531Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:14:47.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:14:47 smithi153 ceph-mon[70044]: pgmap v10395: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:47.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:14:47 smithi083 ceph-mon[77242]: pgmap v10395: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:47.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:14:47 smithi083 ceph-mon[79930]: pgmap v10395: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:49.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:14:49 smithi153 ceph-mon[70044]: pgmap v10396: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:49.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:14:49 smithi083 ceph-mon[77242]: pgmap v10396: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:49.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:14:49 smithi083 ceph-mon[79930]: pgmap v10396: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:14:50 smithi153 ceph-mon[70044]: pgmap v10397: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:50.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:14:50 smithi083 ceph-mon[79930]: pgmap v10397: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:50.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:14:50 smithi083 ceph-mon[77242]: pgmap v10397: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:51.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:14:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:14:51] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T11:14:51.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:14:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:14:51.439Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:14:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:14:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:14:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:14:53.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:14:53 smithi153 ceph-mon[70044]: pgmap v10398: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:14:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:14:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:14:53.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:14:53 smithi083 ceph-mon[77242]: pgmap v10398: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:53.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:14:53 smithi083 ceph-mon[79930]: pgmap v10398: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:55.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:14:55 smithi083 ceph-mon[79930]: pgmap v10399: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:55.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:14:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:14:54.814Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:14:55.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:14:55 smithi083 ceph-mon[77242]: pgmap v10399: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:55.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:14:55 smithi153 ceph-mon[70044]: pgmap v10399: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:55.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:14:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:14:55.531Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:14:57.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:14:57 smithi153 ceph-mon[70044]: pgmap v10400: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:57.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:14:57 smithi083 ceph-mon[77242]: pgmap v10400: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:57.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:14:57 smithi083 ceph-mon[79930]: pgmap v10400: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:59.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:14:59 smithi153 ceph-mon[70044]: pgmap v10401: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:59.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:14:59 smithi083 ceph-mon[79930]: pgmap v10401: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:14:59.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:14:59 smithi083 ceph-mon[77242]: pgmap v10401: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:15:00 smithi153 ceph-mon[70044]: pgmap v10402: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:00.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:15:00 smithi083 ceph-mon[79930]: pgmap v10402: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:00.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:15:00 smithi083 ceph-mon[77242]: pgmap v10402: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:01.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:15:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:15:01] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T11:15:01.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:15:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:15:01.441Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:15:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:15:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:15:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:15:03.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:15:03 smithi153 ceph-mon[70044]: pgmap v10403: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:15:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:15:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:15:03.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:15:03 smithi083 ceph-mon[77242]: pgmap v10403: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:03.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:15:03 smithi083 ceph-mon[79930]: pgmap v10403: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:05.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:15:05 smithi083 ceph-mon[77242]: pgmap v10404: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:05.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:15:05 smithi083 ceph-mon[79930]: pgmap v10404: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:05.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:15:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:15:04.815Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:15:05.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:15:05 smithi153 ceph-mon[70044]: pgmap v10404: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:05.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:15:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:15:05.532Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:15:07.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:15:07 smithi153 ceph-mon[70044]: pgmap v10405: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:07.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:15:07 smithi083 ceph-mon[77242]: pgmap v10405: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:07.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:15:07 smithi083 ceph-mon[79930]: pgmap v10405: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:09.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:15:09 smithi153 ceph-mon[70044]: pgmap v10406: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:09.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:15:09 smithi083 ceph-mon[77242]: pgmap v10406: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:09.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:15:09 smithi083 ceph-mon[79930]: pgmap v10406: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:15:10 smithi153 ceph-mon[70044]: pgmap v10407: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:10.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:15:10 smithi083 ceph-mon[77242]: pgmap v10407: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:10.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:15:10 smithi083 ceph-mon[79930]: pgmap v10407: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:11.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:15:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:15:11] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T11:15:11.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:15:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:15:11.440Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:15:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:15:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:15:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:15:13.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:15:13 smithi153 ceph-mon[70044]: pgmap v10408: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:15:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:15:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:15:13.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:15:13 smithi083 ceph-mon[79930]: pgmap v10408: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:13.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:15:13 smithi083 ceph-mon[77242]: pgmap v10408: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:14.022 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:15:14.312 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:15:14.313 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 3m ago 5h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:15:14.313 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 55s ago 5h 64.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:15:14.313 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 3m ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:15:14.314 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 55s ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:15:14.314 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 55s ago 5h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:15:14.314 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 3m ago 5h 807M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:15:14.315 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 3m ago 5h 1307M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:15:14.315 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 55s ago 5h 563M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:15:14.315 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 3m ago 5h 753M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:15:14.315 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 3m ago 5h 26.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:15:14.315 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 55s ago 5h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:15:14.316 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 3m ago 5h 921M 2438M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:15:14.316 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 3m ago 5h 910M 2438M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:15:14.316 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 3m ago 5h 710M 2438M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:15:14.316 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 3m ago 5h 889M 2438M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:15:14.317 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 55s ago 5h 783M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:15:14.317 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 55s ago 5h 977M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:15:14.317 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 55s ago 5h 873M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:15:14.317 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 55s ago 5h 1058M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:15:14.318 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 55s ago 5h 150M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:15:14.669 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:15:14.670 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:15:14.670 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:15:14.670 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:15:14.671 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:15:14.671 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:15:14.671 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:15:14.671 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:15:14.672 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:15:14.672 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:15:14.672 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:15:14.673 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:15:14.673 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:15:14.673 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:15:14.674 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:15:14.674 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:15:14.674 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:15:14.675 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:15:15.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:15:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:15:14.815Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:15:15.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:15:15 smithi083 ceph-mon[77242]: pgmap v10409: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:15.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:15:15 smithi083 ceph-mon[77242]: from='client.46239 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:15:15.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:15:15 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2912820169' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:15:15.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:15:15 smithi083 ceph-mon[79930]: pgmap v10409: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:15.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:15:15 smithi083 ceph-mon[79930]: from='client.46239 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:15:15.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:15:15 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2912820169' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:15:15.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:15:15 smithi153 ceph-mon[70044]: pgmap v10409: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:15.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:15:15 smithi153 ceph-mon[70044]: from='client.46239 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:15:15.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:15:15 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2912820169' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:15:15.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:15:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:15:15.533Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:15:16.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:15:16 smithi153 ceph-mon[70044]: from='client.56176 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:15:16.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:15:16 smithi083 ceph-mon[79930]: from='client.56176 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:15:16.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:15:16 smithi083 ceph-mon[77242]: from='client.56176 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:15:17.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:15:17 smithi153 ceph-mon[70044]: pgmap v10410: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:17.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:15:17 smithi083 ceph-mon[77242]: pgmap v10410: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:17.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:15:17 smithi083 ceph-mon[79930]: pgmap v10410: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:19.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:15:19 smithi083 ceph-mon[79930]: pgmap v10411: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:19.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:15:19 smithi083 ceph-mon[77242]: pgmap v10411: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:19.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:15:19 smithi153 ceph-mon[70044]: pgmap v10411: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:15:20 smithi153 ceph-mon[70044]: pgmap v10412: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:20.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:15:20 smithi083 ceph-mon[79930]: pgmap v10412: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:20.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:15:20 smithi083 ceph-mon[77242]: pgmap v10412: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:21.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:15:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:15:21] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T11:15:21.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:15:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:15:21.442Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:15:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:15:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:15:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:15:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:15:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:15:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:15:23.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:15:23 smithi153 ceph-mon[70044]: pgmap v10413: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:23.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:15:23 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:15:23.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:15:23 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:15:23.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:15:23 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:15:23.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:15:23 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:15:23.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:15:23 smithi083 ceph-mon[77242]: pgmap v10413: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:23.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:15:23 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:15:23.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:15:23 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:15:23.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:15:23 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:15:23.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:15:23 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:15:23.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:15:23 smithi083 ceph-mon[79930]: pgmap v10413: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:23.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:15:23 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:15:23.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:15:23 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:15:23.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:15:23 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:15:23.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:15:23 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:15:25.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:15:25 smithi083 ceph-mon[79930]: pgmap v10414: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:25.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:15:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:15:24.817Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:15:25.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:15:25 smithi083 ceph-mon[77242]: pgmap v10414: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:25.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:15:25 smithi153 ceph-mon[70044]: pgmap v10414: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:25.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:15:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:15:25.534Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:15:27.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:15:27 smithi153 ceph-mon[70044]: pgmap v10415: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:27.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:15:27 smithi083 ceph-mon[79930]: pgmap v10415: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:27.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:15:27 smithi083 ceph-mon[77242]: pgmap v10415: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:29.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:15:29 smithi083 ceph-mon[79930]: pgmap v10416: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:29.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:15:29 smithi083 ceph-mon[77242]: pgmap v10416: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:29.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:15:29 smithi153 ceph-mon[70044]: pgmap v10416: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:15:30 smithi153 ceph-mon[70044]: pgmap v10417: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:30.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:15:30 smithi083 ceph-mon[79930]: pgmap v10417: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:30.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:15:30 smithi083 ceph-mon[77242]: pgmap v10417: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:31.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:15:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:15:31.442Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:15:31.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:15:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:15:31] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T11:15:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:15:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:15:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:15:33.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:15:33 smithi153 ceph-mon[70044]: pgmap v10418: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:15:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:15:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:15:33.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:15:33 smithi083 ceph-mon[79930]: pgmap v10418: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:33.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:15:33 smithi083 ceph-mon[77242]: pgmap v10418: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:35.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:15:35 smithi083 ceph-mon[79930]: pgmap v10419: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:35.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:15:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:15:34.818Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:15:35.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:15:35 smithi083 ceph-mon[77242]: pgmap v10419: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:35.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:15:35 smithi153 ceph-mon[70044]: pgmap v10419: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:35.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:15:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:15:35.534Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:15:37.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:15:37 smithi083 ceph-mon[79930]: pgmap v10420: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:37.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:15:37 smithi083 ceph-mon[77242]: pgmap v10420: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:37.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:15:37 smithi153 ceph-mon[70044]: pgmap v10420: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:39.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:15:39 smithi083 ceph-mon[79930]: pgmap v10421: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:39.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:15:39 smithi083 ceph-mon[77242]: pgmap v10421: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:39.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:15:39 smithi153 ceph-mon[70044]: pgmap v10421: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:15:40 smithi153 ceph-mon[70044]: pgmap v10422: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:40.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:15:40 smithi083 ceph-mon[79930]: pgmap v10422: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:40.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:15:40 smithi083 ceph-mon[77242]: pgmap v10422: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:41.506 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:15:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:15:41.442Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:15:41.507 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:15:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:15:41.507 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:15:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:15:41.508 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:15:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:15:41.508 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:15:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:15:41.508 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:15:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:15:41.508 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:15:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:15:41.509 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:15:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:15:41.509 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:15:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:15:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:15:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:15:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:15:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:15:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:15:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:15:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:15:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:15:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:15:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:15:41] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T11:15:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:15:42 smithi153 ceph-mon[70044]: pgmap v10423: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:42.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:15:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:15:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:15:42.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:15:42 smithi083 ceph-mon[77242]: pgmap v10423: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:42.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:15:42 smithi083 ceph-mon[79930]: pgmap v10423: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:15:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:15:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:15:44.998 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:15:45.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:15:45 smithi083 ceph-mon[79930]: pgmap v10424: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:45.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:15:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:15:44.818Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:15:45.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:15:45 smithi083 ceph-mon[77242]: pgmap v10424: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:45.290 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:15:45.290 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 3m ago 5h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:15:45.290 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 86s ago 5h 64.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:15:45.291 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 3m ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:15:45.291 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 86s ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:15:45.291 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 86s ago 5h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:15:45.292 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 3m ago 5h 807M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:15:45.292 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 3m ago 5h 1307M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:15:45.292 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 86s ago 5h 563M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:15:45.292 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 3m ago 5h 753M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:15:45.293 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 3m ago 5h 26.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:15:45.293 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 86s ago 5h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:15:45.293 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 3m ago 5h 921M 2438M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:15:45.293 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 3m ago 5h 910M 2438M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:15:45.294 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 3m ago 5h 710M 2438M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:15:45.294 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 3m ago 5h 889M 2438M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:15:45.294 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 86s ago 5h 783M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:15:45.294 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 86s ago 5h 977M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:15:45.295 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 86s ago 5h 873M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:15:45.295 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 86s ago 5h 1058M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:15:45.295 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 86s ago 5h 150M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:15:45.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:15:45 smithi153 ceph-mon[70044]: pgmap v10424: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:45.647 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:15:45.648 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:15:45.648 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:15:45.648 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:15:45.649 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:15:45.649 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:15:45.649 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:15:45.649 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:15:45.649 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:15:45.650 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:15:45.650 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:15:45.650 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:15:45.650 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:15:45.651 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:15:45.651 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:15:45.651 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:15:45.651 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:15:45.652 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:15:45.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:15:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:15:45.533Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:15:46.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:15:46 smithi083 ceph-mon[79930]: from='client.56188 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:15:46.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:15:46 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2953822536' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:15:46.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:15:46 smithi083 ceph-mon[77242]: from='client.56188 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:15:46.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:15:46 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2953822536' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:15:46.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:15:46 smithi153 ceph-mon[70044]: from='client.56188 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:15:46.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:15:46 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2953822536' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:15:47.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:15:47 smithi083 ceph-mon[79930]: from='client.46263 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:15:47.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:15:47 smithi083 ceph-mon[79930]: pgmap v10425: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:47.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:15:47 smithi083 ceph-mon[77242]: from='client.46263 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:15:47.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:15:47 smithi083 ceph-mon[77242]: pgmap v10425: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:47.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:15:47 smithi153 ceph-mon[70044]: from='client.46263 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:15:47.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:15:47 smithi153 ceph-mon[70044]: pgmap v10425: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:49.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:15:49 smithi083 ceph-mon[79930]: pgmap v10426: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:49.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:15:49 smithi083 ceph-mon[77242]: pgmap v10426: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:49.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:15:49 smithi153 ceph-mon[70044]: pgmap v10426: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:15:50 smithi153 ceph-mon[70044]: pgmap v10427: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:50.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:15:50 smithi083 ceph-mon[79930]: pgmap v10427: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:50.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:15:50 smithi083 ceph-mon[77242]: pgmap v10427: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:51.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:15:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:15:51] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T11:15:51.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:15:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:15:51.443Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:15:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:15:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:15:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:15:53.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:15:53 smithi153 ceph-mon[70044]: pgmap v10428: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:15:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:15:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:15:53.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:15:53 smithi083 ceph-mon[79930]: pgmap v10428: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:53.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:15:53 smithi083 ceph-mon[77242]: pgmap v10428: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:55.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:15:55 smithi083 ceph-mon[79930]: pgmap v10429: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:55.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:15:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:15:54.819Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:15:55.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:15:55 smithi083 ceph-mon[77242]: pgmap v10429: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:55.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:15:55 smithi153 ceph-mon[70044]: pgmap v10429: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:55.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:15:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:15:55.535Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:15:57.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:15:57 smithi083 ceph-mon[79930]: pgmap v10430: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:57.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:15:57 smithi083 ceph-mon[77242]: pgmap v10430: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:57.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:15:57 smithi153 ceph-mon[70044]: pgmap v10430: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:59.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:15:59 smithi083 ceph-mon[79930]: pgmap v10431: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:59.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:15:59 smithi083 ceph-mon[77242]: pgmap v10431: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:15:59.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:15:59 smithi153 ceph-mon[70044]: pgmap v10431: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:16:00 smithi153 ceph-mon[70044]: pgmap v10432: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:00.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:16:00 smithi083 ceph-mon[79930]: pgmap v10432: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:00.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:16:00 smithi083 ceph-mon[77242]: pgmap v10432: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:01.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:16:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:16:01] "GET /metrics HTTP/1.1" 200 225228 "" "Prometheus/2.33.4" 2022-09-29T11:16:01.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:16:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:16:01.443Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:16:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:16:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:16:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:16:03.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:16:03 smithi153 ceph-mon[70044]: pgmap v10433: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:16:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:16:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:16:03.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:16:03 smithi083 ceph-mon[79930]: pgmap v10433: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:03.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:16:03 smithi083 ceph-mon[77242]: pgmap v10433: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:05.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:16:05 smithi083 ceph-mon[79930]: pgmap v10434: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:05.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:16:05 smithi083 ceph-mon[77242]: pgmap v10434: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:05.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:16:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:16:04.820Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:16:05.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:16:05 smithi153 ceph-mon[70044]: pgmap v10434: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:05.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:16:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:16:05.536Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:16:07.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:16:07 smithi083 ceph-mon[79930]: pgmap v10435: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:07.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:16:07 smithi083 ceph-mon[77242]: pgmap v10435: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:07.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:16:07 smithi153 ceph-mon[70044]: pgmap v10435: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:09.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:16:09 smithi083 ceph-mon[79930]: pgmap v10436: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:09.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:16:09 smithi083 ceph-mon[77242]: pgmap v10436: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:09.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:16:09 smithi153 ceph-mon[70044]: pgmap v10436: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:16:10 smithi153 ceph-mon[70044]: pgmap v10437: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:10.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:16:10 smithi083 ceph-mon[79930]: pgmap v10437: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:10.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:16:10 smithi083 ceph-mon[77242]: pgmap v10437: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:11.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:16:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:16:11] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T11:16:11.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:16:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:16:11.444Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:16:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:16:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:16:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:16:13.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:16:13 smithi153 ceph-mon[70044]: pgmap v10438: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:16:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:16:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:16:13.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:16:13 smithi083 ceph-mon[77242]: pgmap v10438: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:13.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:16:13 smithi083 ceph-mon[79930]: pgmap v10438: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:15.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:16:15 smithi083 ceph-mon[79930]: pgmap v10439: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:15.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:16:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:16:14.821Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:16:15.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:16:15 smithi083 ceph-mon[77242]: pgmap v10439: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:15.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:16:15 smithi153 ceph-mon[70044]: pgmap v10439: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:15.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:16:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:16:15.536Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:16:15.973 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:16:16.263 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:16:16.263 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 4m ago 5h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:16:16.263 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 117s ago 5h 64.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:16:16.264 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 4m ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:16:16.264 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 117s ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:16:16.264 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 117s ago 5h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:16:16.264 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 4m ago 5h 807M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:16:16.264 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 4m ago 5h 1307M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:16:16.265 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 117s ago 5h 563M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:16:16.265 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 4m ago 5h 753M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:16:16.265 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 4m ago 5h 26.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:16:16.265 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 117s ago 5h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:16:16.266 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 4m ago 5h 921M 2438M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:16:16.266 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 4m ago 5h 910M 2438M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:16:16.266 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 4m ago 5h 710M 2438M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:16:16.266 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 4m ago 5h 889M 2438M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:16:16.267 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 117s ago 5h 783M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:16:16.267 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 117s ago 5h 977M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:16:16.267 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 117s ago 5h 873M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:16:16.267 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 117s ago 5h 1058M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:16:16.267 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 117s ago 5h 150M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:16:16.619 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:16:16.619 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:16:16.620 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:16:16.620 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:16:16.620 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:16:16.620 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:16:16.621 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:16:16.621 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:16:16.621 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:16:16.621 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:16:16.622 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:16:16.622 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:16:16.622 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:16:16.622 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:16:16.622 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:16:16.623 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:16:16.623 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:16:16.623 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:16:17.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:16:17 smithi083 ceph-mon[79930]: from='client.46275 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:16:17.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:16:17 smithi083 ceph-mon[79930]: pgmap v10440: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:17.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:16:17 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3947144034' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:16:17.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:16:17 smithi083 ceph-mon[77242]: from='client.46275 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:16:17.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:16:17 smithi083 ceph-mon[77242]: pgmap v10440: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:17.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:16:17 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3947144034' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:16:17.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:16:17 smithi153 ceph-mon[70044]: from='client.46275 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:16:17.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:16:17 smithi153 ceph-mon[70044]: pgmap v10440: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:17.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:16:17 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3947144034' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:16:18.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:16:18 smithi083 ceph-mon[79930]: from='client.46281 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:16:18.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:16:18 smithi083 ceph-mon[77242]: from='client.46281 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:16:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:16:18 smithi153 ceph-mon[70044]: from='client.46281 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:16:19.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:16:19 smithi083 ceph-mon[79930]: pgmap v10441: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:19.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:16:19 smithi083 ceph-mon[77242]: pgmap v10441: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:19.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:16:19 smithi153 ceph-mon[70044]: pgmap v10441: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:16:20 smithi153 ceph-mon[70044]: pgmap v10442: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:20.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:16:20 smithi083 ceph-mon[77242]: pgmap v10442: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:20.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:16:20 smithi083 ceph-mon[79930]: pgmap v10442: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:21.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:16:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:16:21] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T11:16:21.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:16:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:16:21.446Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:16:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:16:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:16:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:16:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:16:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:16:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:16:23.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:16:23 smithi153 ceph-mon[70044]: pgmap v10443: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:23.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:16:23 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:16:23.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:16:23 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:16:23.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:16:23 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:16:23.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:16:23 smithi083 ceph-mon[79930]: pgmap v10443: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:23.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:16:23 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:16:23.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:16:23 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:16:23.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:16:23 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:16:23.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:16:23 smithi083 ceph-mon[77242]: pgmap v10443: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:23.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:16:23 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:16:23.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:16:23 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:16:23.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:16:23 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:16:24.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:16:24 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:16:24.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:16:24 smithi083 ceph-mon[77242]: pgmap v10444: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:24.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:16:24 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:16:24.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:16:24 smithi083 ceph-mon[79930]: pgmap v10444: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:24.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:16:24 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:16:24.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:16:24 smithi153 ceph-mon[70044]: pgmap v10444: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:25.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:16:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:16:24.821Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:16:25.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:16:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:16:25.537Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:16:27.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:16:27 smithi083 ceph-mon[79930]: pgmap v10445: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:27.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:16:27 smithi083 ceph-mon[77242]: pgmap v10445: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:27.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:16:27 smithi153 ceph-mon[70044]: pgmap v10445: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:29.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:16:29 smithi083 ceph-mon[79930]: pgmap v10446: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:29.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:16:29 smithi083 ceph-mon[77242]: pgmap v10446: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:29.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:16:29 smithi153 ceph-mon[70044]: pgmap v10446: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:16:30 smithi153 ceph-mon[70044]: pgmap v10447: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:30.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:16:30 smithi083 ceph-mon[79930]: pgmap v10447: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:30.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:16:30 smithi083 ceph-mon[77242]: pgmap v10447: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:31.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:16:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:16:31] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T11:16:31.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:16:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:16:31.446Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:16:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:16:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:16:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:16:33.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:16:33 smithi153 ceph-mon[70044]: pgmap v10448: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:16:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:16:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:16:33.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:16:33 smithi083 ceph-mon[79930]: pgmap v10448: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:33.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:16:33 smithi083 ceph-mon[77242]: pgmap v10448: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:35.075 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:16:35 smithi083 ceph-mon[77242]: pgmap v10449: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:35.075 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:16:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:16:34.822Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:16:35.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:16:35 smithi083 ceph-mon[79930]: pgmap v10449: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:35.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:16:35 smithi153 ceph-mon[70044]: pgmap v10449: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:35.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:16:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:16:35.537Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:16:37.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:16:37 smithi083 ceph-mon[77242]: pgmap v10450: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:37.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:16:37 smithi083 ceph-mon[79930]: pgmap v10450: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:37.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:16:37 smithi153 ceph-mon[70044]: pgmap v10450: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:39.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:16:39 smithi083 ceph-mon[77242]: pgmap v10451: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:39.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:16:39 smithi083 ceph-mon[79930]: pgmap v10451: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:39.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:16:39 smithi153 ceph-mon[70044]: pgmap v10451: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:16:40 smithi153 ceph-mon[70044]: pgmap v10452: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:40.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:16:40 smithi083 ceph-mon[79930]: pgmap v10452: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:40.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:16:40 smithi083 ceph-mon[77242]: pgmap v10452: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:16:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:16:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:16:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:16:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:16:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:16:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:16:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:16:41.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:16:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:16:41] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T11:16:41.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:16:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:16:41.446Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:16:41.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:16:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:16:41.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:16:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:16:41.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:16:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:16:41.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:16:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:16:41.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:16:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:16:41.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:16:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:16:41.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:16:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:16:41.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:16:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:16:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:16:42 smithi153 ceph-mon[70044]: pgmap v10453: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:42.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:16:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:16:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:16:42.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:16:42 smithi083 ceph-mon[77242]: pgmap v10453: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:42.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:16:42 smithi083 ceph-mon[79930]: pgmap v10453: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:16:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:16:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:16:45.076 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:16:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:16:44.823Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:16:45.076 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:16:45 smithi083 ceph-mon[77242]: pgmap v10454: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:45.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:16:45 smithi083 ceph-mon[79930]: pgmap v10454: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:45.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:16:45 smithi153 ceph-mon[70044]: pgmap v10454: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:45.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:16:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:16:45.538Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:16:46.949 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:16:47.241 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:16:47.242 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 4m ago 5h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:16:47.242 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 2m ago 5h 64.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:16:47.242 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 4m ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:16:47.242 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 2m ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:16:47.243 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 2m ago 5h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:16:47.243 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 4m ago 5h 807M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:16:47.243 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 4m ago 5h 1307M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:16:47.243 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 2m ago 5h 563M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:16:47.243 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 4m ago 5h 753M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:16:47.244 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 4m ago 5h 26.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:16:47.244 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 2m ago 5h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:16:47.244 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 4m ago 5h 921M 2438M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:16:47.244 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 4m ago 5h 910M 2438M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:16:47.244 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 4m ago 5h 710M 2438M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:16:47.245 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 4m ago 5h 889M 2438M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:16:47.245 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 2m ago 5h 783M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:16:47.245 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 2m ago 5h 977M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:16:47.245 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 2m ago 5h 873M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:16:47.246 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 2m ago 5h 1058M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:16:47.246 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 2m ago 5h 150M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:16:47.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:16:47 smithi083 ceph-mon[79930]: pgmap v10455: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:47.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:16:47 smithi083 ceph-mon[77242]: pgmap v10455: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:47.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:16:47 smithi153 ceph-mon[70044]: pgmap v10455: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:47.598 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:16:47.599 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:16:47.599 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:16:47.599 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:16:47.599 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:16:47.600 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:16:47.600 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:16:47.600 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:16:47.600 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:16:47.600 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:16:47.601 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:16:47.601 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:16:47.601 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:16:47.601 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:16:47.602 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:16:47.602 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:16:47.602 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:16:47.602 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:16:48.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:16:48 smithi083 ceph-mon[79930]: from='client.56224 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:16:48.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:16:48 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2421603793' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:16:48.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:16:48 smithi083 ceph-mon[77242]: from='client.56224 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:16:48.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:16:48 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2421603793' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:16:48.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:16:48 smithi153 ceph-mon[70044]: from='client.56224 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:16:48.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:16:48 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2421603793' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:16:49.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:16:49 smithi083 ceph-mon[79930]: from='client.46299 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:16:49.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:16:49 smithi083 ceph-mon[79930]: pgmap v10456: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:49.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:16:49 smithi083 ceph-mon[77242]: from='client.46299 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:16:49.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:16:49 smithi083 ceph-mon[77242]: pgmap v10456: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:49.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:16:49 smithi153 ceph-mon[70044]: from='client.46299 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:16:49.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:16:49 smithi153 ceph-mon[70044]: pgmap v10456: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:16:50 smithi153 ceph-mon[70044]: pgmap v10457: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:50.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:16:50 smithi083 ceph-mon[77242]: pgmap v10457: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:50.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:16:50 smithi083 ceph-mon[79930]: pgmap v10457: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:51.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:16:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:16:51] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T11:16:51.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:16:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:16:51.447Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:16:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:16:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:16:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:16:53.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:16:53 smithi153 ceph-mon[70044]: pgmap v10458: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:16:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:16:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:16:53.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:16:53 smithi083 ceph-mon[79930]: pgmap v10458: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:53.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:16:53 smithi083 ceph-mon[77242]: pgmap v10458: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:55.079 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:16:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:16:54.823Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:16:55.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:16:55 smithi083 ceph-mon[79930]: pgmap v10459: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:55.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:16:55 smithi083 ceph-mon[77242]: pgmap v10459: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:55.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:16:55 smithi153 ceph-mon[70044]: pgmap v10459: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:55.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:16:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:16:55.539Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:16:57.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:16:57 smithi083 ceph-mon[79930]: pgmap v10460: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:57.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:16:57 smithi083 ceph-mon[77242]: pgmap v10460: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:57.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:16:57 smithi153 ceph-mon[70044]: pgmap v10460: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:59.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:16:59 smithi083 ceph-mon[79930]: pgmap v10461: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:59.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:16:59 smithi083 ceph-mon[77242]: pgmap v10461: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:16:59.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:16:59 smithi153 ceph-mon[70044]: pgmap v10461: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:17:00 smithi153 ceph-mon[70044]: pgmap v10462: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:00.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:17:00 smithi083 ceph-mon[79930]: pgmap v10462: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:00.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:17:00 smithi083 ceph-mon[77242]: pgmap v10462: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:01.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:17:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:17:01.448Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:17:01.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:17:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:17:01] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T11:17:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:17:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:17:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:17:03.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:17:03 smithi153 ceph-mon[70044]: pgmap v10463: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:17:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:17:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:17:03.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:17:03 smithi083 ceph-mon[79930]: pgmap v10463: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:03.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:17:03 smithi083 ceph-mon[77242]: pgmap v10463: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:05.082 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:17:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:17:04.824Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:17:05.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:17:05 smithi083 ceph-mon[79930]: pgmap v10464: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:05.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:17:05 smithi083 ceph-mon[77242]: pgmap v10464: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:05.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:17:05 smithi153 ceph-mon[70044]: pgmap v10464: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:05.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:17:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:17:05.539Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:17:07.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:17:07 smithi083 ceph-mon[79930]: pgmap v10465: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:07.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:17:07 smithi083 ceph-mon[77242]: pgmap v10465: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:07.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:17:07 smithi153 ceph-mon[70044]: pgmap v10465: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:09.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:17:09 smithi083 ceph-mon[79930]: pgmap v10466: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:09.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:17:09 smithi083 ceph-mon[77242]: pgmap v10466: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:09.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:17:09 smithi153 ceph-mon[70044]: pgmap v10466: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:17:10 smithi153 ceph-mon[70044]: pgmap v10467: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:10.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:17:10 smithi083 ceph-mon[77242]: pgmap v10467: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:10.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:17:10 smithi083 ceph-mon[79930]: pgmap v10467: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:11.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:17:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:17:11.448Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:17:11.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:17:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:17:11] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T11:17:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:17:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:17:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:17:13.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:17:13 smithi153 ceph-mon[70044]: pgmap v10468: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:17:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:17:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:17:13.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:17:13 smithi083 ceph-mon[79930]: pgmap v10468: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:13.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:17:13 smithi083 ceph-mon[77242]: pgmap v10468: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:15.085 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:17:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:17:14.825Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:17:15.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:17:15 smithi083 ceph-mon[77242]: pgmap v10469: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:15.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:17:15 smithi083 ceph-mon[79930]: pgmap v10469: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:15.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:17:15 smithi153 ceph-mon[70044]: pgmap v10469: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:15.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:17:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:17:15.540Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:17:17.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:17:17 smithi083 ceph-mon[79930]: pgmap v10470: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:17.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:17:17 smithi083 ceph-mon[77242]: pgmap v10470: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:17.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:17:17 smithi153 ceph-mon[70044]: pgmap v10470: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:17.924 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:17:18.212 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:17:18.212 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 5m ago 5h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:17:18.212 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 2m ago 5h 64.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:17:18.213 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 5m ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:17:18.213 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 2m ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:17:18.213 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 2m ago 5h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:17:18.213 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 5m ago 5h 807M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:17:18.214 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 5m ago 5h 1307M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:17:18.214 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 2m ago 5h 563M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:17:18.214 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 5m ago 5h 753M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:17:18.214 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 5m ago 5h 26.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:17:18.215 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 2m ago 5h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:17:18.215 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 5m ago 5h 921M 2438M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:17:18.215 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 5m ago 5h 910M 2438M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:17:18.215 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 5m ago 5h 710M 2438M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:17:18.215 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 5m ago 5h 889M 2438M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:17:18.216 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 2m ago 5h 783M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:17:18.216 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 2m ago 5h 977M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:17:18.216 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 2m ago 5h 873M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:17:18.216 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 2m ago 5h 1058M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:17:18.217 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 2m ago 5h 150M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:17:18.566 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:17:18.567 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:17:18.567 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:17:18.567 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:17:18.567 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:17:18.567 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:17:18.568 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:17:18.568 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:17:18.568 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:17:18.568 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:17:18.569 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:17:18.569 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:17:18.569 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:17:18.569 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:17:18.570 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:17:18.570 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:17:18.570 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:17:18.570 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:17:19.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:17:19 smithi083 ceph-mon[77242]: from='client.56242 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:17:19.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:17:19 smithi083 ceph-mon[77242]: pgmap v10471: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:19.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:17:19 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3494538594' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:17:19.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:17:19 smithi083 ceph-mon[79930]: from='client.56242 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:17:19.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:17:19 smithi083 ceph-mon[79930]: pgmap v10471: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:19.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:17:19 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3494538594' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:17:19.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:17:19 smithi153 ceph-mon[70044]: from='client.56242 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:17:19.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:17:19 smithi153 ceph-mon[70044]: pgmap v10471: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:19.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:17:19 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3494538594' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:17:20.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:17:20 smithi083 ceph-mon[77242]: from='client.46317 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:17:20.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:17:20 smithi083 ceph-mon[79930]: from='client.46317 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:17:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:17:20 smithi153 ceph-mon[70044]: from='client.46317 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:17:21.450 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:17:21 smithi083 ceph-mon[79930]: pgmap v10472: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:21.451 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:17:21 smithi083 ceph-mon[77242]: pgmap v10472: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:21.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:17:21 smithi153 ceph-mon[70044]: pgmap v10472: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:21.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:17:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:17:21.449Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:17:21.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:17:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:17:21] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T11:17:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:17:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:17:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:17:23.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:17:23 smithi153 ceph-mon[70044]: pgmap v10473: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:17:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:17:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:17:23.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:17:23 smithi083 ceph-mon[79930]: pgmap v10473: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:23.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:17:23 smithi083 ceph-mon[77242]: pgmap v10473: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:24.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:17:24 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:17:24.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:17:24 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:17:24.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:17:24 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:17:24.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:17:24 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:17:24.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:17:24 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:17:24.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:17:24 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:17:24.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:17:24 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:17:24.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:17:24 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:17:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:17:24 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:17:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:17:24 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:17:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:17:24 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:17:24.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:17:24 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:17:25.101 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:17:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:17:24.825Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:17:25.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:17:25 smithi083 ceph-mon[79930]: pgmap v10474: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:25.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:17:25 smithi083 ceph-mon[77242]: pgmap v10474: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:25.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:17:25 smithi153 ceph-mon[70044]: pgmap v10474: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:25.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:17:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:17:25.541Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:17:27.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:17:27 smithi083 ceph-mon[79930]: pgmap v10475: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:27.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:17:27 smithi083 ceph-mon[77242]: pgmap v10475: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:27.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:17:27 smithi153 ceph-mon[70044]: pgmap v10475: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:29.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:17:29 smithi083 ceph-mon[77242]: pgmap v10476: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:29.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:17:29 smithi083 ceph-mon[79930]: pgmap v10476: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:29.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:17:29 smithi153 ceph-mon[70044]: pgmap v10476: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:17:30 smithi153 ceph-mon[70044]: pgmap v10477: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:30.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:17:30 smithi083 ceph-mon[79930]: pgmap v10477: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:30.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:17:30 smithi083 ceph-mon[77242]: pgmap v10477: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:31.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:17:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:17:31] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T11:17:31.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:17:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:17:31.449Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:17:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:17:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:17:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:17:33.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:17:33 smithi153 ceph-mon[70044]: pgmap v10478: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:17:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:17:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:17:33.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:17:33 smithi083 ceph-mon[79930]: pgmap v10478: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:33.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:17:33 smithi083 ceph-mon[77242]: pgmap v10478: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:35.090 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:17:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:17:34.825Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:17:35.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:17:35 smithi083 ceph-mon[79930]: pgmap v10479: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:35.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:17:35 smithi083 ceph-mon[77242]: pgmap v10479: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:35.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:17:35 smithi153 ceph-mon[70044]: pgmap v10479: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:35.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:17:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:17:35.541Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:17:37.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:17:37 smithi083 ceph-mon[79930]: pgmap v10480: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:37.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:17:37 smithi083 ceph-mon[77242]: pgmap v10480: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:37.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:17:37 smithi153 ceph-mon[70044]: pgmap v10480: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:39.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:17:39 smithi083 ceph-mon[79930]: pgmap v10481: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:39.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:17:39 smithi083 ceph-mon[77242]: pgmap v10481: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:39.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:17:39 smithi153 ceph-mon[70044]: pgmap v10481: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:17:40 smithi153 ceph-mon[70044]: pgmap v10482: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:40.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:17:40 smithi083 ceph-mon[79930]: pgmap v10482: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:40.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:17:40 smithi083 ceph-mon[77242]: pgmap v10482: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:17:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:17:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:17:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:17:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:17:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:17:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:17:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:17:41.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:17:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:17:41] "GET /metrics HTTP/1.1" 200 225228 "" "Prometheus/2.33.4" 2022-09-29T11:17:41.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:17:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:17:41.450Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:17:41.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:17:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:17:41.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:17:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:17:41.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:17:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:17:41.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:17:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:17:41.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:17:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:17:41.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:17:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:17:41.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:17:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:17:41.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:17:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:17:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:17:42 smithi153 ceph-mon[70044]: pgmap v10483: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:42.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:17:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:17:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:17:42.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:17:42 smithi083 ceph-mon[77242]: pgmap v10483: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:42.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:17:42 smithi083 ceph-mon[79930]: pgmap v10483: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:17:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:17:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:17:45.091 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:17:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:17:44.826Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:17:45.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:17:45 smithi083 ceph-mon[79930]: pgmap v10484: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:45.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:17:45 smithi083 ceph-mon[77242]: pgmap v10484: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:45.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:17:45 smithi153 ceph-mon[70044]: pgmap v10484: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:45.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:17:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:17:45.542Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:17:47.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:17:47 smithi083 ceph-mon[79930]: pgmap v10485: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:47.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:17:47 smithi083 ceph-mon[77242]: pgmap v10485: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:47.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:17:47 smithi153 ceph-mon[70044]: pgmap v10485: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:48.892 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:17:49.183 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:17:49.183 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 5m ago 5h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:17:49.184 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 3m ago 5h 64.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:17:49.184 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 5m ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:17:49.184 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 3m ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:17:49.184 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 3m ago 5h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:17:49.185 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 5m ago 5h 807M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:17:49.185 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 5m ago 5h 1307M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:17:49.185 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 3m ago 5h 563M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:17:49.185 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 5m ago 5h 753M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:17:49.186 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 5m ago 5h 26.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:17:49.186 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 3m ago 5h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:17:49.186 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 5m ago 5h 921M 2438M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:17:49.186 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 5m ago 5h 910M 2438M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:17:49.187 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 5m ago 5h 710M 2438M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:17:49.187 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 5m ago 5h 889M 2438M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:17:49.187 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 3m ago 5h 783M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:17:49.187 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 3m ago 5h 977M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:17:49.187 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 3m ago 5h 873M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:17:49.188 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 3m ago 5h 1058M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:17:49.188 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 3m ago 5h 150M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:17:49.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:17:49 smithi083 ceph-mon[79930]: pgmap v10486: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:49.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:17:49 smithi083 ceph-mon[77242]: pgmap v10486: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:49.542 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:17:49.542 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:17:49.542 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:17:49.543 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:17:49.543 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:17:49.543 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:17:49.543 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:17:49.544 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:17:49.544 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:17:49.544 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:17:49.544 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:17:49.544 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:17:49.545 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:17:49.545 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:17:49.545 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:17:49.545 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:17:49.546 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:17:49.546 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:17:49.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:17:49 smithi153 ceph-mon[70044]: pgmap v10486: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:50.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:17:50 smithi083 ceph-mon[79930]: from='client.46329 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:17:50.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:17:50 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/749921341' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:17:50.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:17:50 smithi083 ceph-mon[77242]: from='client.46329 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:17:50.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:17:50 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/749921341' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:17:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:17:50 smithi153 ceph-mon[70044]: from='client.46329 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:17:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:17:50 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/749921341' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:17:51.452 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:17:51 smithi083 ceph-mon[79930]: from='client.56266 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:17:51.452 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:17:51 smithi083 ceph-mon[79930]: pgmap v10487: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:51.453 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:17:51 smithi083 ceph-mon[77242]: from='client.56266 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:17:51.453 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:17:51 smithi083 ceph-mon[77242]: pgmap v10487: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:51.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:17:51 smithi153 ceph-mon[70044]: from='client.56266 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:17:51.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:17:51 smithi153 ceph-mon[70044]: pgmap v10487: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:51.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:17:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:17:51] "GET /metrics HTTP/1.1" 200 225228 "" "Prometheus/2.33.4" 2022-09-29T11:17:51.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:17:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:17:51.451Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:17:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:17:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:17:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:17:53.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:17:53 smithi153 ceph-mon[70044]: pgmap v10488: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:17:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:17:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:17:53.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:17:53 smithi083 ceph-mon[79930]: pgmap v10488: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:53.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:17:53 smithi083 ceph-mon[77242]: pgmap v10488: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:55.105 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:17:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:17:54.827Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:17:55.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:17:55 smithi083 ceph-mon[79930]: pgmap v10489: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:55.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:17:55 smithi083 ceph-mon[77242]: pgmap v10489: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:55.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:17:55 smithi153 ceph-mon[70044]: pgmap v10489: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:55.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:17:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:17:55.543Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:17:57.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:17:57 smithi083 ceph-mon[79930]: pgmap v10490: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:57.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:17:57 smithi083 ceph-mon[77242]: pgmap v10490: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:57.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:17:57 smithi153 ceph-mon[70044]: pgmap v10490: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:59.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:17:59 smithi083 ceph-mon[79930]: pgmap v10491: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:59.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:17:59 smithi083 ceph-mon[77242]: pgmap v10491: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:17:59.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:17:59 smithi153 ceph-mon[70044]: pgmap v10491: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:18:00 smithi153 ceph-mon[70044]: pgmap v10492: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:00.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:18:00 smithi083 ceph-mon[79930]: pgmap v10492: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:00.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:18:00 smithi083 ceph-mon[77242]: pgmap v10492: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:01.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:18:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:18:01] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T11:18:01.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:18:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:18:01.451Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:18:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:18:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:18:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:18:03.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:18:03 smithi153 ceph-mon[70044]: pgmap v10493: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:18:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:18:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:18:03.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:18:03 smithi083 ceph-mon[79930]: pgmap v10493: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:03.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:18:03 smithi083 ceph-mon[77242]: pgmap v10493: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:05.099 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:18:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:18:04.827Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:18:05.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:18:05 smithi083 ceph-mon[79930]: pgmap v10494: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:05.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:18:05 smithi083 ceph-mon[77242]: pgmap v10494: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:05.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:18:05 smithi153 ceph-mon[70044]: pgmap v10494: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:05.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:18:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:18:05.544Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:18:07.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:18:07 smithi083 ceph-mon[77242]: pgmap v10495: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:07.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:18:07 smithi083 ceph-mon[79930]: pgmap v10495: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:07.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:18:07 smithi153 ceph-mon[70044]: pgmap v10495: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:09.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:18:09 smithi083 ceph-mon[79930]: pgmap v10496: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:09.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:18:09 smithi083 ceph-mon[77242]: pgmap v10496: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:09.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:18:09 smithi153 ceph-mon[70044]: pgmap v10496: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:18:10 smithi153 ceph-mon[70044]: pgmap v10497: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:10.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:18:10 smithi083 ceph-mon[77242]: pgmap v10497: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:10.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:18:10 smithi083 ceph-mon[79930]: pgmap v10497: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:11.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:18:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:18:11] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T11:18:11.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:18:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:18:11.452Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:18:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:18:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:18:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:18:13.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:18:13 smithi153 ceph-mon[70044]: pgmap v10498: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:18:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:18:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:18:13.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:18:13 smithi083 ceph-mon[77242]: pgmap v10498: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:13.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:18:13 smithi083 ceph-mon[79930]: pgmap v10498: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:15.102 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:18:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:18:14.828Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:18:15.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:18:15 smithi083 ceph-mon[79930]: pgmap v10499: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:15.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:18:15 smithi083 ceph-mon[77242]: pgmap v10499: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:15.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:18:15 smithi153 ceph-mon[70044]: pgmap v10499: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:15.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:18:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:18:15.545Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:18:17.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:18:17 smithi083 ceph-mon[77242]: pgmap v10500: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:17.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:18:17 smithi083 ceph-mon[79930]: pgmap v10500: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:17.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:18:17 smithi153 ceph-mon[70044]: pgmap v10500: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:19.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:18:19 smithi083 ceph-mon[79930]: pgmap v10501: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:19.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:18:19 smithi083 ceph-mon[77242]: pgmap v10501: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:19.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:18:19 smithi153 ceph-mon[70044]: pgmap v10501: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:19.869 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:18:20.159 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:18:20.159 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 6m ago 5h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:18:20.160 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 4m ago 5h 64.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:18:20.160 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 6m ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:18:20.160 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 4m ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:18:20.160 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 4m ago 5h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:18:20.160 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 6m ago 5h 807M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:18:20.161 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 6m ago 5h 1307M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:18:20.161 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 4m ago 5h 563M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:18:20.161 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 6m ago 5h 753M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:18:20.161 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 6m ago 5h 26.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:18:20.161 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 4m ago 5h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:18:20.162 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 6m ago 5h 921M 2438M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:18:20.162 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 6m ago 5h 910M 2438M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:18:20.162 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 6m ago 5h 710M 2438M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:18:20.163 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 6m ago 5h 889M 2438M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:18:20.163 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 4m ago 5h 783M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:18:20.163 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 4m ago 5h 977M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:18:20.163 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 4m ago 5h 873M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:18:20.163 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 4m ago 5h 1058M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:18:20.164 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 4m ago 5h 150M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:18:20.517 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:18:20.517 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:18:20.517 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:18:20.518 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:18:20.518 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:18:20.518 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:18:20.519 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:18:20.519 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:18:20.519 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:18:20.519 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:18:20.520 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:18:20.520 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:18:20.520 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:18:20.520 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:18:20.521 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:18:20.521 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:18:20.521 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:18:20.522 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:18:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:18:20 smithi153 ceph-mon[70044]: from='client.46347 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:18:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:18:20 smithi153 ceph-mon[70044]: pgmap v10502: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:20.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:18:20 smithi083 ceph-mon[79930]: from='client.46347 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:18:20.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:18:20 smithi083 ceph-mon[79930]: pgmap v10502: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:20.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:18:20 smithi083 ceph-mon[77242]: from='client.46347 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:18:20.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:18:20 smithi083 ceph-mon[77242]: pgmap v10502: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:21.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:18:21 smithi153 ceph-mon[70044]: from='client.56284 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:18:21.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:18:21 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1192183750' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:18:21.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:18:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:18:21] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T11:18:21.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:18:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:18:21.452Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:18:21.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:18:21 smithi083 ceph-mon[79930]: from='client.56284 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:18:21.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:18:21 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1192183750' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:18:21.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:18:21 smithi083 ceph-mon[77242]: from='client.56284 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:18:21.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:18:21 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1192183750' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:18:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:18:22 smithi153 ceph-mon[70044]: pgmap v10503: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:22.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:18:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:18:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:18:22.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:18:22 smithi083 ceph-mon[79930]: pgmap v10503: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:22.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:18:22 smithi083 ceph-mon[77242]: pgmap v10503: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:18:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:18:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:18:25.103 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:18:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:18:24.828Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:18:25.104 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:18:25 smithi083 ceph-mon[77242]: pgmap v10504: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:25.104 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:18:25 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:18:25.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:18:25 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:18:25.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:18:25 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:18:25.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:18:25 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:18:25.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:18:25 smithi083 ceph-mon[79930]: pgmap v10504: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:25.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:18:25 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:18:25.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:18:25 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:18:25.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:18:25 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:18:25.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:18:25 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:18:25.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:18:25 smithi153 ceph-mon[70044]: pgmap v10504: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:25.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:18:25 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:18:25.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:18:25 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:18:25.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:18:25 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:18:25.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:18:25 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:18:25.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:18:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:18:25.545Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:18:27.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:18:27 smithi083 ceph-mon[79930]: pgmap v10505: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:27.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:18:27 smithi083 ceph-mon[77242]: pgmap v10505: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:27.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:18:27 smithi153 ceph-mon[70044]: pgmap v10505: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:29.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:18:29 smithi083 ceph-mon[79930]: pgmap v10506: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:29.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:18:29 smithi083 ceph-mon[77242]: pgmap v10506: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:29.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:18:29 smithi153 ceph-mon[70044]: pgmap v10506: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:18:30 smithi153 ceph-mon[70044]: pgmap v10507: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:30.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:18:30 smithi083 ceph-mon[79930]: pgmap v10507: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:30.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:18:30 smithi083 ceph-mon[77242]: pgmap v10507: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:31.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:18:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:18:31] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T11:18:31.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:18:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:18:31.454Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:18:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:18:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:18:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:18:33.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:18:33 smithi153 ceph-mon[70044]: pgmap v10508: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:18:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:18:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:18:33.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:18:33 smithi083 ceph-mon[77242]: pgmap v10508: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:33.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:18:33 smithi083 ceph-mon[79930]: pgmap v10508: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:35.109 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:18:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:18:34.830Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:18:35.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:18:35 smithi083 ceph-mon[79930]: pgmap v10509: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:35.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:18:35 smithi083 ceph-mon[77242]: pgmap v10509: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:35.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:18:35 smithi153 ceph-mon[70044]: pgmap v10509: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:35.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:18:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:18:35.546Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:18:37.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:18:37 smithi083 ceph-mon[79930]: pgmap v10510: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:37.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:18:37 smithi083 ceph-mon[77242]: pgmap v10510: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:37.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:18:37 smithi153 ceph-mon[70044]: pgmap v10510: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:39.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:18:39 smithi083 ceph-mon[77242]: pgmap v10511: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:39.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:18:39 smithi083 ceph-mon[79930]: pgmap v10511: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:39.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:18:39 smithi153 ceph-mon[70044]: pgmap v10511: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:18:40 smithi153 ceph-mon[70044]: pgmap v10512: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:40.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:18:40 smithi083 ceph-mon[79930]: pgmap v10512: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:40.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:18:40 smithi083 ceph-mon[77242]: pgmap v10512: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:18:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:18:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:18:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:18:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:18:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:18:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:18:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:18:41.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:18:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:18:41] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T11:18:41.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:18:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:18:41.454Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:18:41.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:18:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:18:41.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:18:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:18:41.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:18:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:18:41.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:18:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:18:41.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:18:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:18:41.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:18:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:18:41.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:18:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:18:41.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:18:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:18:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:18:42 smithi153 ceph-mon[70044]: pgmap v10513: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:42.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:18:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:18:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:18:42.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:18:42 smithi083 ceph-mon[79930]: pgmap v10513: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:42.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:18:42 smithi083 ceph-mon[77242]: pgmap v10513: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:18:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:18:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:18:45.109 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:18:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:18:44.831Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:18:45.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:18:45 smithi083 ceph-mon[79930]: pgmap v10514: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:45.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:18:45 smithi083 ceph-mon[77242]: pgmap v10514: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:45.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:18:45 smithi153 ceph-mon[70044]: pgmap v10514: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:45.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:18:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:18:45.547Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:18:47.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:18:47 smithi083 ceph-mon[77242]: pgmap v10515: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:47.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:18:47 smithi083 ceph-mon[79930]: pgmap v10515: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:47.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:18:47 smithi153 ceph-mon[70044]: pgmap v10515: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:49.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:18:49 smithi083 ceph-mon[79930]: pgmap v10516: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:49.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:18:49 smithi083 ceph-mon[77242]: pgmap v10516: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:49.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:18:49 smithi153 ceph-mon[70044]: pgmap v10516: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:18:50 smithi153 ceph-mon[70044]: pgmap v10517: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:50.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:18:50 smithi083 ceph-mon[79930]: pgmap v10517: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:50.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:18:50 smithi083 ceph-mon[77242]: pgmap v10517: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:50.837 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:18:51.125 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:18:51.125 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 6m ago 5h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:18:51.126 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 4m ago 5h 64.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:18:51.126 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 6m ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:18:51.126 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 4m ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:18:51.126 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 4m ago 5h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:18:51.127 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 6m ago 5h 807M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:18:51.127 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 6m ago 5h 1307M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:18:51.127 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 4m ago 5h 563M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:18:51.127 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 6m ago 5h 753M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:18:51.128 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 6m ago 5h 26.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:18:51.128 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 4m ago 5h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:18:51.128 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 6m ago 5h 921M 2438M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:18:51.128 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 6m ago 5h 910M 2438M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:18:51.128 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 6m ago 5h 710M 2438M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:18:51.129 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 6m ago 5h 889M 2438M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:18:51.129 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 4m ago 5h 783M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:18:51.129 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 4m ago 5h 977M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:18:51.129 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 4m ago 5h 873M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:18:51.130 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 4m ago 5h 1058M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:18:51.130 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 4m ago 5h 150M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:18:51.484 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:18:51.484 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:18:51.485 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:18:51.485 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:18:51.485 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:18:51.485 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:18:51.485 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:18:51.486 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:18:51.486 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:18:51.486 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:18:51.486 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:18:51.486 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:18:51.487 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:18:51.487 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:18:51.487 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:18:51.487 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:18:51.487 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:18:51.488 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:18:51.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:18:51 smithi153 ceph-mon[70044]: from='client.46365 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:18:51.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:18:51 smithi153 ceph-mon[70044]: from='client.56302 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:18:51.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:18:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:18:51] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T11:18:51.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:18:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:18:51.453Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:18:51.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:18:51 smithi083 ceph-mon[77242]: from='client.46365 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:18:51.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:18:51 smithi083 ceph-mon[77242]: from='client.56302 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:18:51.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:18:51 smithi083 ceph-mon[79930]: from='client.46365 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:18:51.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:18:51 smithi083 ceph-mon[79930]: from='client.56302 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:18:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:18:52 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2658882963' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:18:52.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:18:52 smithi153 ceph-mon[70044]: pgmap v10518: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:52.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:18:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:18:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:18:52.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:18:52 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2658882963' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:18:52.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:18:52 smithi083 ceph-mon[79930]: pgmap v10518: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:52.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:18:52 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2658882963' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:18:52.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:18:52 smithi083 ceph-mon[77242]: pgmap v10518: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:18:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:18:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:18:55.112 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:18:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:18:54.831Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:18:55.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:18:55 smithi083 ceph-mon[79930]: pgmap v10519: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:55.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:18:55 smithi083 ceph-mon[77242]: pgmap v10519: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:55.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:18:55 smithi153 ceph-mon[70044]: pgmap v10519: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:55.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:18:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:18:55.547Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:18:57.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:18:57 smithi083 ceph-mon[77242]: pgmap v10520: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:57.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:18:57 smithi083 ceph-mon[79930]: pgmap v10520: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:57.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:18:57 smithi153 ceph-mon[70044]: pgmap v10520: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:59.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:18:59 smithi083 ceph-mon[77242]: pgmap v10521: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:59.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:18:59 smithi083 ceph-mon[79930]: pgmap v10521: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:18:59.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:18:59 smithi153 ceph-mon[70044]: pgmap v10521: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:19:00 smithi153 ceph-mon[70044]: pgmap v10522: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:00.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:19:00 smithi083 ceph-mon[77242]: pgmap v10522: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:00.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:19:00 smithi083 ceph-mon[79930]: pgmap v10522: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:01.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:19:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:19:01] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T11:19:01.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:19:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:19:01.454Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:19:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:19:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:19:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:19:03.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:19:03 smithi153 ceph-mon[70044]: pgmap v10523: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:19:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:19:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:19:03.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:19:03 smithi083 ceph-mon[79930]: pgmap v10523: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:03.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:19:03 smithi083 ceph-mon[77242]: pgmap v10523: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:05.118 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:19:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:19:04.832Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:19:05.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:19:05 smithi083 ceph-mon[79930]: pgmap v10524: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:05.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:19:05 smithi083 ceph-mon[77242]: pgmap v10524: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:05.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:19:05 smithi153 ceph-mon[70044]: pgmap v10524: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:05.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:19:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:19:05.548Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:19:07.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:19:07 smithi083 ceph-mon[79930]: pgmap v10525: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:07.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:19:07 smithi083 ceph-mon[77242]: pgmap v10525: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:07.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:19:07 smithi153 ceph-mon[70044]: pgmap v10525: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:09.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:19:09 smithi083 ceph-mon[79930]: pgmap v10526: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:09.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:19:09 smithi083 ceph-mon[77242]: pgmap v10526: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:09.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:19:09 smithi153 ceph-mon[70044]: pgmap v10526: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:19:10 smithi153 ceph-mon[70044]: pgmap v10527: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:10.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:19:10 smithi083 ceph-mon[79930]: pgmap v10527: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:10.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:19:10 smithi083 ceph-mon[77242]: pgmap v10527: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:11.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:19:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:19:11] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T11:19:11.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:19:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:19:11.455Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:19:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:19:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:19:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:19:13.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:19:13 smithi153 ceph-mon[70044]: pgmap v10528: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:19:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:19:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:19:13.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:19:13 smithi083 ceph-mon[79930]: pgmap v10528: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:13.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:19:13 smithi083 ceph-mon[77242]: pgmap v10528: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:15.121 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:19:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:19:14.832Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:19:15.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:19:15 smithi083 ceph-mon[77242]: pgmap v10529: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:15.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:19:15 smithi083 ceph-mon[79930]: pgmap v10529: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:15.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:19:15 smithi153 ceph-mon[70044]: pgmap v10529: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:15.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:19:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:19:15.548Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:19:17.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:19:17 smithi083 ceph-mon[79930]: pgmap v10530: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:17.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:19:17 smithi083 ceph-mon[77242]: pgmap v10530: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:17.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:19:17 smithi153 ceph-mon[70044]: pgmap v10530: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:19.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:19:19 smithi083 ceph-mon[79930]: pgmap v10531: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:19.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:19:19 smithi083 ceph-mon[77242]: pgmap v10531: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:19.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:19:19 smithi153 ceph-mon[70044]: pgmap v10531: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:19:20 smithi153 ceph-mon[70044]: pgmap v10532: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:20.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:19:20 smithi083 ceph-mon[79930]: pgmap v10532: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:20.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:19:20 smithi083 ceph-mon[77242]: pgmap v10532: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:21.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:19:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:19:21] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T11:19:21.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:19:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:19:21.456Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:19:21.813 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:19:22.099 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:19:22.100 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 7m ago 5h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:19:22.100 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 5m ago 5h 64.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:19:22.100 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 7m ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:19:22.100 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 5m ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:19:22.100 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 5m ago 5h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:19:22.101 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 7m ago 5h 807M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:19:22.101 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 7m ago 5h 1307M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:19:22.101 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 5m ago 5h 563M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:19:22.101 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 7m ago 5h 753M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:19:22.101 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 7m ago 5h 26.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:19:22.102 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 5m ago 5h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:19:22.102 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 7m ago 5h 921M 2438M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:19:22.102 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 7m ago 5h 910M 2438M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:19:22.102 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 7m ago 5h 710M 2438M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:19:22.103 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 7m ago 5h 889M 2438M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:19:22.103 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 5m ago 5h 783M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:19:22.103 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 5m ago 5h 977M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:19:22.103 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 5m ago 5h 873M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:19:22.103 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 5m ago 5h 1058M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:19:22.104 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 5m ago 5h 150M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:19:22.455 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:19:22.455 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:19:22.455 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:19:22.456 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:19:22.456 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:19:22.456 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:19:22.456 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:19:22.456 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:19:22.457 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:19:22.457 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:19:22.457 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:19:22.457 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:19:22.458 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:19:22.458 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:19:22.458 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:19:22.458 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:19:22.459 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:19:22.459 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:19:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:19:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:19:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:19:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:19:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:19:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:19:23.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:19:23 smithi153 ceph-mon[70044]: from='client.46383 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:19:23.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:19:23 smithi153 ceph-mon[70044]: pgmap v10533: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:23.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:19:23 smithi153 ceph-mon[70044]: from='client.46389 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:19:23.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:19:23 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/352993059' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:19:23.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:19:23 smithi083 ceph-mon[77242]: from='client.46383 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:19:23.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:19:23 smithi083 ceph-mon[77242]: pgmap v10533: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:23.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:19:23 smithi083 ceph-mon[77242]: from='client.46389 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:19:23.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:19:23 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/352993059' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:19:23.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:19:23 smithi083 ceph-mon[79930]: from='client.46383 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:19:23.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:19:23 smithi083 ceph-mon[79930]: pgmap v10533: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:23.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:19:23 smithi083 ceph-mon[79930]: from='client.46389 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:19:23.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:19:23 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/352993059' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:19:25.124 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:19:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:19:24.832Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:19:25.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:19:25 smithi083 ceph-mon[79930]: pgmap v10534: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:25.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:19:25 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:19:25.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:19:25 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:19:25.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:19:25 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:19:25.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:19:25 smithi083 ceph-mon[77242]: pgmap v10534: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:25.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:19:25 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:19:25.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:19:25 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:19:25.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:19:25 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:19:25.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:19:25 smithi153 ceph-mon[70044]: pgmap v10534: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:25.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:19:25 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:19:25.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:19:25 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:19:25.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:19:25 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:19:25.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:19:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:19:25.549Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:19:26.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:19:26 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:19:26.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:19:26 smithi083 ceph-mon[79930]: pgmap v10535: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:26.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:19:26 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:19:26.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:19:26 smithi083 ceph-mon[77242]: pgmap v10535: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:26.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:19:26 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:19:26.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:19:26 smithi153 ceph-mon[70044]: pgmap v10535: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:29.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:19:29 smithi083 ceph-mon[77242]: pgmap v10536: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:29.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:19:29 smithi083 ceph-mon[79930]: pgmap v10536: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:29.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:19:29 smithi153 ceph-mon[70044]: pgmap v10536: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:19:30 smithi153 ceph-mon[70044]: pgmap v10537: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:30.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:19:30 smithi083 ceph-mon[79930]: pgmap v10537: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:30.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:19:30 smithi083 ceph-mon[77242]: pgmap v10537: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:31.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:19:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:19:31] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T11:19:31.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:19:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:19:31.456Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:19:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:19:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:19:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:19:33.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:19:33 smithi153 ceph-mon[70044]: pgmap v10538: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:19:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:19:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:19:33.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:19:33 smithi083 ceph-mon[79930]: pgmap v10538: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:33.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:19:33 smithi083 ceph-mon[77242]: pgmap v10538: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:35.127 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:19:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:19:34.833Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:19:35.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:19:35 smithi083 ceph-mon[79930]: pgmap v10539: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:35.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:19:35 smithi083 ceph-mon[77242]: pgmap v10539: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:35.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:19:35 smithi153 ceph-mon[70044]: pgmap v10539: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:35.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:19:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:19:35.550Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:19:37.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:19:37 smithi083 ceph-mon[79930]: pgmap v10540: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:37.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:19:37 smithi083 ceph-mon[77242]: pgmap v10540: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:37.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:19:37 smithi153 ceph-mon[70044]: pgmap v10540: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:39.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:19:39 smithi083 ceph-mon[79930]: pgmap v10541: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:39.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:19:39 smithi083 ceph-mon[77242]: pgmap v10541: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:39.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:19:39 smithi153 ceph-mon[70044]: pgmap v10541: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:19:40 smithi153 ceph-mon[70044]: pgmap v10542: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:40.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:19:40 smithi083 ceph-mon[79930]: pgmap v10542: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:40.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:19:40 smithi083 ceph-mon[77242]: pgmap v10542: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:19:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:19:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:19:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:19:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:19:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:19:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:19:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:19:41.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:19:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:19:41] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T11:19:41.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:19:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:19:41.457Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:19:41.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:19:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:19:41.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:19:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:19:41.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:19:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:19:41.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:19:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:19:41.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:19:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:19:41.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:19:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:19:41.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:19:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:19:41.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:19:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:19:42.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:19:42 smithi083 ceph-mon[79930]: pgmap v10543: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:42.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:19:42 smithi083 ceph-mon[77242]: pgmap v10543: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:19:42 smithi153 ceph-mon[70044]: pgmap v10543: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:19:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:19:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:19:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:19:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:19:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:19:45.128 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:19:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:19:44.834Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:19:45.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:19:45 smithi083 ceph-mon[77242]: pgmap v10544: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:45.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:19:45 smithi083 ceph-mon[79930]: pgmap v10544: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:45.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:19:45 smithi153 ceph-mon[70044]: pgmap v10544: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:45.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:19:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:19:45.551Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:19:47.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:19:47 smithi083 ceph-mon[79930]: pgmap v10545: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:47.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:19:47 smithi083 ceph-mon[77242]: pgmap v10545: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:47.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:19:47 smithi153 ceph-mon[70044]: pgmap v10545: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:49.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:19:49 smithi083 ceph-mon[77242]: pgmap v10546: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:49.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:19:49 smithi083 ceph-mon[79930]: pgmap v10546: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:49.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:19:49 smithi153 ceph-mon[70044]: pgmap v10546: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:50.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:19:50 smithi083 ceph-mon[79930]: pgmap v10547: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:50.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:19:50 smithi083 ceph-mon[77242]: pgmap v10547: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:50.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:19:50 smithi153 ceph-mon[70044]: pgmap v10547: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:51.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:19:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:19:51] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T11:19:51.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:19:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:19:51.458Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:19:52.780 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:19:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:19:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:19:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:19:53.071 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:19:53.071 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 7m ago 5h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:19:53.072 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 5m ago 5h 64.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:19:53.072 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 7m ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:19:53.072 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 5m ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:19:53.072 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 5m ago 5h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:19:53.073 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 7m ago 5h 807M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:19:53.073 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 7m ago 5h 1307M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:19:53.073 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 5m ago 5h 563M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:19:53.073 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 7m ago 5h 753M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:19:53.073 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 7m ago 5h 26.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:19:53.074 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 5m ago 5h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:19:53.074 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 7m ago 5h 921M 2438M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:19:53.074 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 7m ago 5h 910M 2438M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:19:53.074 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 7m ago 5h 710M 2438M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:19:53.075 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 7m ago 5h 889M 2438M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:19:53.075 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 5m ago 5h 783M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:19:53.075 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 5m ago 5h 977M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:19:53.075 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 5m ago 5h 873M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:19:53.076 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 5m ago 5h 1058M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:19:53.076 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 5m ago 5h 150M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:19:53.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:19:53 smithi153 ceph-mon[70044]: pgmap v10548: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:19:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:19:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:19:53.438 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:19:53.438 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:19:53.438 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:19:53.439 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:19:53.439 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:19:53.439 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:19:53.439 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:19:53.440 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:19:53.440 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:19:53.440 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:19:53.440 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:19:53.440 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:19:53.441 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:19:53.441 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:19:53.441 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:19:53.441 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:19:53.442 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:19:53.442 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:19:53.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:19:53 smithi083 ceph-mon[79930]: pgmap v10548: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:53.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:19:53 smithi083 ceph-mon[77242]: pgmap v10548: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:54.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:19:54 smithi083 ceph-mon[79930]: from='client.56332 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:19:54.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:19:54 smithi083 ceph-mon[79930]: from='client.56338 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:19:54.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:19:54 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1435543557' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:19:54.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:19:54 smithi083 ceph-mon[77242]: from='client.56332 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:19:54.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:19:54 smithi083 ceph-mon[77242]: from='client.56338 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:19:54.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:19:54 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1435543557' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:19:54.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:19:54 smithi153 ceph-mon[70044]: from='client.56332 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:19:54.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:19:54 smithi153 ceph-mon[70044]: from='client.56338 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:19:54.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:19:54 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1435543557' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:19:55.137 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:19:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:19:54.834Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:19:55.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:19:55 smithi083 ceph-mon[79930]: pgmap v10549: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:55.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:19:55 smithi083 ceph-mon[77242]: pgmap v10549: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:55.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:19:55 smithi153 ceph-mon[70044]: pgmap v10549: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:55.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:19:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:19:55.552Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:19:57.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:19:57 smithi083 ceph-mon[79930]: pgmap v10550: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:57.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:19:57 smithi083 ceph-mon[77242]: pgmap v10550: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:57.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:19:57 smithi153 ceph-mon[70044]: pgmap v10550: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:59.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:19:59 smithi083 ceph-mon[79930]: pgmap v10551: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:59.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:19:59 smithi083 ceph-mon[77242]: pgmap v10551: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:19:59.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:19:59 smithi153 ceph-mon[70044]: pgmap v10551: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:00.313 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:20:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[77219]: 2022-09-29T11:19:59.999+0000 7f160f665700 -1 log_channel(cluster) log [ERR] : overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T11:20:00.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:20:00 smithi083 ceph-mon[79930]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T11:20:00.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:20:00 smithi083 ceph-mon[79930]: pgmap v10552: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:00.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:20:00 smithi083 ceph-mon[77242]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T11:20:00.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:20:00 smithi083 ceph-mon[77242]: pgmap v10552: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:00.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:20:00 smithi153 ceph-mon[70044]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T11:20:00.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:20:00 smithi153 ceph-mon[70044]: pgmap v10552: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:01.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:20:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:20:01] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T11:20:01.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:20:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:20:01.458Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:20:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:20:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:20:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:20:03.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:20:03 smithi153 ceph-mon[70044]: pgmap v10553: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:20:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:20:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:20:03.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:20:03 smithi083 ceph-mon[79930]: pgmap v10553: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:03.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:20:03 smithi083 ceph-mon[77242]: pgmap v10553: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:05.137 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:20:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:20:04.835Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:20:05.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:20:05 smithi083 ceph-mon[77242]: pgmap v10554: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:05.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:20:05 smithi083 ceph-mon[79930]: pgmap v10554: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:05.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:20:05 smithi153 ceph-mon[70044]: pgmap v10554: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:05.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:20:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:20:05.552Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:20:07.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:20:07 smithi083 ceph-mon[79930]: pgmap v10555: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:07.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:20:07 smithi083 ceph-mon[77242]: pgmap v10555: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:07.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:20:07 smithi153 ceph-mon[70044]: pgmap v10555: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:09.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:20:09 smithi083 ceph-mon[79930]: pgmap v10556: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:09.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:20:09 smithi083 ceph-mon[77242]: pgmap v10556: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:09.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:20:09 smithi153 ceph-mon[70044]: pgmap v10556: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:10.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:20:10 smithi083 ceph-mon[79930]: pgmap v10557: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:10.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:20:10 smithi083 ceph-mon[77242]: pgmap v10557: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:10.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:20:10 smithi153 ceph-mon[70044]: pgmap v10557: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:11.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:20:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:20:11] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T11:20:11.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:20:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:20:11.458Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:20:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:20:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:20:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:20:13.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:20:13 smithi153 ceph-mon[70044]: pgmap v10558: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:20:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:20:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:20:13.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:20:13 smithi083 ceph-mon[79930]: pgmap v10558: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:13.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:20:13 smithi083 ceph-mon[77242]: pgmap v10558: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:15.141 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:20:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:20:14.835Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:20:15.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:20:15 smithi083 ceph-mon[79930]: pgmap v10559: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:15.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:20:15 smithi083 ceph-mon[77242]: pgmap v10559: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:15.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:20:15 smithi153 ceph-mon[70044]: pgmap v10559: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:15.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:20:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:20:15.553Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:20:17.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:20:17 smithi083 ceph-mon[79930]: pgmap v10560: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:17.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:20:17 smithi083 ceph-mon[77242]: pgmap v10560: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:17.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:20:17 smithi153 ceph-mon[70044]: pgmap v10560: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:19.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:20:19 smithi083 ceph-mon[79930]: pgmap v10561: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:19.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:20:19 smithi083 ceph-mon[77242]: pgmap v10561: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:19.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:20:19 smithi153 ceph-mon[70044]: pgmap v10561: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:20.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:20:20 smithi083 ceph-mon[79930]: pgmap v10562: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:20.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:20:20 smithi083 ceph-mon[77242]: pgmap v10562: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:20.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:20:20 smithi153 ceph-mon[70044]: pgmap v10562: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:21.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:20:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:20:21] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T11:20:21.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:20:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:20:21.458Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:20:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:20:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:20:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:20:23.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:20:23 smithi153 ceph-mon[70044]: pgmap v10563: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:20:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:20:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:20:23.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:20:23 smithi083 ceph-mon[79930]: pgmap v10563: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:23.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:20:23 smithi083 ceph-mon[77242]: pgmap v10563: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:23.774 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:20:24.064 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:20:24.064 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 8m ago 5h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:20:24.065 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 6m ago 5h 64.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:20:24.065 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 8m ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:20:24.065 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 6m ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:20:24.065 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 6m ago 5h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:20:24.066 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 8m ago 5h 807M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:20:24.066 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 8m ago 5h 1307M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:20:24.066 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 6m ago 5h 563M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:20:24.067 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 8m ago 5h 753M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:20:24.067 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 8m ago 5h 26.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:20:24.067 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 6m ago 5h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:20:24.067 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 8m ago 5h 921M 2438M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:20:24.067 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 8m ago 5h 910M 2438M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:20:24.068 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 8m ago 5h 710M 2438M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:20:24.068 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 8m ago 5h 889M 2438M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:20:24.068 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 6m ago 5h 783M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:20:24.068 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 6m ago 5h 977M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:20:24.069 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 6m ago 5h 873M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:20:24.069 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 6m ago 5h 1058M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:20:24.069 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 6m ago 5h 150M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:20:24.439 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:20:24.439 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:20:24.439 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:20:24.440 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:20:24.440 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:20:24.440 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:20:24.440 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:20:24.441 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:20:24.441 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:20:24.441 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:20:24.441 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:20:24.441 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:20:24.442 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:20:24.442 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:20:24.442 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:20:24.442 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:20:24.443 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:20:24.443 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:20:25.143 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:20:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:20:24.836Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:20:25.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:20:25 smithi083 ceph-mon[79930]: from='client.46419 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:20:25.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:20:25 smithi083 ceph-mon[79930]: from='client.56356 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:20:25.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:20:25 smithi083 ceph-mon[79930]: pgmap v10564: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:25.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:20:25 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1990890553' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:20:25.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:20:25 smithi083 ceph-mon[77242]: from='client.46419 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:20:25.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:20:25 smithi083 ceph-mon[77242]: from='client.56356 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:20:25.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:20:25 smithi083 ceph-mon[77242]: pgmap v10564: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:25.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:20:25 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1990890553' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:20:25.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:20:25 smithi153 ceph-mon[70044]: from='client.46419 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:20:25.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:20:25 smithi153 ceph-mon[70044]: from='client.56356 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:20:25.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:20:25 smithi153 ceph-mon[70044]: pgmap v10564: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:25.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:20:25 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1990890553' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:20:25.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:20:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:20:25.554Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:20:26.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:20:26 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:20:26.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:20:26 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:20:26.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:20:26 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:20:26.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:20:26 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:20:26.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:20:26 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:20:26.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:20:26 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:20:26.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:20:26 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:20:26.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:20:26 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:20:26.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:20:26 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:20:26.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:20:26 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:20:26.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:20:26 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:20:26.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:20:26 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:20:26.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:20:26 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:20:26.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:20:26 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:20:26.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:20:26 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:20:26.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:20:26 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:20:26.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:20:26 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:20:26.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:20:26 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:20:26.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:20:26 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:20:26.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:20:26 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:20:26.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:20:26 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:20:26.467 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:20:26 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:20:26.467 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:20:26 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:20:26.467 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:20:26 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:20:26.467 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:20:26 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:20:26.467 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:20:26 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:20:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:20:26 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:20:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:20:26 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:20:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:20:26 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:20:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:20:26 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:20:26.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:20:26 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:20:26.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:20:26 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:20:26.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:20:26 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:20:26.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:20:26 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:20:26.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:20:26 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:20:26.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:20:26 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:20:26.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:20:26 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:20:26.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:20:26 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:20:26.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:20:26 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:20:27.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:20:27 smithi083 ceph-mon[79930]: Adjusting osd_memory_target on smithi083 to 2429M 2022-09-29T11:20:27.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:20:27 smithi083 ceph-mon[79930]: pgmap v10565: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:27.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:20:27 smithi083 ceph-mon[77242]: Adjusting osd_memory_target on smithi083 to 2429M 2022-09-29T11:20:27.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:20:27 smithi083 ceph-mon[77242]: pgmap v10565: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:27.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:20:27 smithi153 ceph-mon[70044]: Adjusting osd_memory_target on smithi083 to 2429M 2022-09-29T11:20:27.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:20:27 smithi153 ceph-mon[70044]: pgmap v10565: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:29.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:20:29 smithi083 ceph-mon[79930]: pgmap v10566: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:29.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:20:29 smithi083 ceph-mon[77242]: pgmap v10566: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:29.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:20:29 smithi153 ceph-mon[70044]: pgmap v10566: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:30.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:20:30 smithi083 ceph-mon[79930]: pgmap v10567: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:30.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:20:30 smithi083 ceph-mon[77242]: pgmap v10567: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:30.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:20:30 smithi153 ceph-mon[70044]: pgmap v10567: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:31.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:20:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:20:31] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T11:20:31.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:20:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:20:31.459Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:20:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:20:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:20:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:20:33.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:20:33 smithi153 ceph-mon[70044]: pgmap v10568: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:20:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:20:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:20:33.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:20:33 smithi083 ceph-mon[77242]: pgmap v10568: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:33.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:20:33 smithi083 ceph-mon[79930]: pgmap v10568: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:35.146 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:20:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:20:34.837Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:20:35.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:20:35 smithi083 ceph-mon[79930]: pgmap v10569: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:35.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:20:35 smithi083 ceph-mon[77242]: pgmap v10569: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:35.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:20:35 smithi153 ceph-mon[70044]: pgmap v10569: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:35.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:20:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:20:35.555Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:20:37.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:20:37 smithi083 ceph-mon[79930]: pgmap v10570: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:37.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:20:37 smithi083 ceph-mon[77242]: pgmap v10570: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:37.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:20:37 smithi153 ceph-mon[70044]: pgmap v10570: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:39.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:20:39 smithi083 ceph-mon[79930]: pgmap v10571: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:39.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:20:39 smithi083 ceph-mon[77242]: pgmap v10571: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:39.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:20:39 smithi153 ceph-mon[70044]: pgmap v10571: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:40.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:20:40 smithi083 ceph-mon[79930]: pgmap v10572: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:20:40 smithi083 ceph-mon[77242]: pgmap v10572: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:20:40 smithi153 ceph-mon[70044]: pgmap v10572: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:41.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:20:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:20:41] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T11:20:41.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:20:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:20:41.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:20:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:20:41.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:20:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:20:41.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:20:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:20:41.714 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:20:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:20:41.460Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:20:41.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:20:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:20:41.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:20:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:20:41.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:20:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:20:41.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:20:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:20:41.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:20:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:20:41.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:20:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:20:41.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:20:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:20:41.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:20:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:20:42.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:20:42 smithi083 ceph-mon[79930]: pgmap v10573: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:42.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:20:42 smithi083 ceph-mon[77242]: pgmap v10573: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:20:42 smithi153 ceph-mon[70044]: pgmap v10573: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:20:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:20:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:20:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:20:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:20:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:20:45.147 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:20:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:20:44.838Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:20:45.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:20:45 smithi083 ceph-mon[79930]: pgmap v10574: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:45.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:20:45 smithi083 ceph-mon[77242]: pgmap v10574: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:45.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:20:45 smithi153 ceph-mon[70044]: pgmap v10574: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:45.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:20:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:20:45.556Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:20:47.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:20:47 smithi083 ceph-mon[79930]: pgmap v10575: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:47.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:20:47 smithi083 ceph-mon[77242]: pgmap v10575: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:47.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:20:47 smithi153 ceph-mon[70044]: pgmap v10575: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:49.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:20:49 smithi083 ceph-mon[79930]: pgmap v10576: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:49.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:20:49 smithi083 ceph-mon[77242]: pgmap v10576: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:49.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:20:49 smithi153 ceph-mon[70044]: pgmap v10576: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:50.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:20:50 smithi083 ceph-mon[79930]: pgmap v10577: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:50.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:20:50 smithi083 ceph-mon[77242]: pgmap v10577: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:50.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:20:50 smithi153 ceph-mon[70044]: pgmap v10577: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:20:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:20:51] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T11:20:51.991 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:20:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:20:51.461Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:20:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:20:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:20:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:20:53.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:20:53 smithi153 ceph-mon[70044]: pgmap v10578: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:20:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:20:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:20:53.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:20:53 smithi083 ceph-mon[79930]: pgmap v10578: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:53.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:20:53 smithi083 ceph-mon[77242]: pgmap v10578: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:54.766 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:20:55.053 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:20:55.053 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 8m ago 5h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:20:55.054 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 6m ago 5h 64.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:20:55.054 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 8m ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:20:55.054 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 6m ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:20:55.054 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 6m ago 5h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:20:55.055 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 8m ago 6h 807M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:20:55.055 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 8m ago 6h 1307M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:20:55.055 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 6m ago 5h 563M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:20:55.055 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 8m ago 5h 753M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:20:55.056 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 8m ago 5h 26.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:20:55.056 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 6m ago 5h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:20:55.056 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 8m ago 5h 921M 2429M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:20:55.056 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 8m ago 5h 910M 2429M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:20:55.056 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 8m ago 5h 710M 2429M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:20:55.057 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 8m ago 5h 889M 2429M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:20:55.057 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 6m ago 5h 783M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:20:55.057 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 6m ago 5h 977M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:20:55.057 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 6m ago 5h 873M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:20:55.057 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 6m ago 5h 1058M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:20:55.058 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 6m ago 5h 150M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:20:55.152 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:20:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:20:54.838Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:20:55.410 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:20:55.411 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:20:55.411 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:20:55.411 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:20:55.411 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:20:55.411 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:20:55.412 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:20:55.412 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:20:55.412 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:20:55.412 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:20:55.413 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:20:55.413 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:20:55.413 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:20:55.413 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:20:55.414 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:20:55.414 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:20:55.414 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:20:55.414 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:20:55.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:20:55 smithi083 ceph-mon[77242]: pgmap v10579: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:55.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:20:55 smithi083 ceph-mon[79930]: pgmap v10579: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:55.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:20:55 smithi153 ceph-mon[70044]: pgmap v10579: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:55.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:20:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:20:55.555Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:20:56.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:20:56 smithi083 ceph-mon[79930]: from='client.46437 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:20:56.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:20:56 smithi083 ceph-mon[79930]: from='client.46443 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:20:56.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:20:56 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/594484053' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:20:56.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:20:56 smithi083 ceph-mon[77242]: from='client.46437 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:20:56.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:20:56 smithi083 ceph-mon[77242]: from='client.46443 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:20:56.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:20:56 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/594484053' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:20:56.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:20:56 smithi153 ceph-mon[70044]: from='client.46437 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:20:56.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:20:56 smithi153 ceph-mon[70044]: from='client.46443 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:20:56.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:20:56 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/594484053' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:20:57.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:20:57 smithi083 ceph-mon[79930]: pgmap v10580: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:57.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:20:57 smithi083 ceph-mon[77242]: pgmap v10580: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:57.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:20:57 smithi153 ceph-mon[70044]: pgmap v10580: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:59.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:20:59 smithi083 ceph-mon[79930]: pgmap v10581: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:59.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:20:59 smithi083 ceph-mon[77242]: pgmap v10581: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:20:59.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:20:59 smithi153 ceph-mon[70044]: pgmap v10581: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:00.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:21:00 smithi083 ceph-mon[79930]: pgmap v10582: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:00.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:21:00 smithi083 ceph-mon[77242]: pgmap v10582: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:00.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:21:00 smithi153 ceph-mon[70044]: pgmap v10582: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:21:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:21:01] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T11:21:01.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:21:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:21:01.462Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:21:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:21:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:21:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:21:03.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:21:03 smithi153 ceph-mon[70044]: pgmap v10583: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:21:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:21:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:21:03.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:21:03 smithi083 ceph-mon[79930]: pgmap v10583: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:03.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:21:03 smithi083 ceph-mon[77242]: pgmap v10583: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:05.156 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:21:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:21:04.839Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:21:05.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:21:05 smithi083 ceph-mon[79930]: pgmap v10584: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:05.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:21:05 smithi083 ceph-mon[77242]: pgmap v10584: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:05.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:21:05 smithi153 ceph-mon[70044]: pgmap v10584: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:05.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:21:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:21:05.557Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:21:07.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:21:07 smithi083 ceph-mon[79930]: pgmap v10585: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:07.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:21:07 smithi083 ceph-mon[77242]: pgmap v10585: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:07.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:21:07 smithi153 ceph-mon[70044]: pgmap v10585: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:09.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:21:09 smithi083 ceph-mon[79930]: pgmap v10586: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:09.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:21:09 smithi083 ceph-mon[77242]: pgmap v10586: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:09.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:21:09 smithi153 ceph-mon[70044]: pgmap v10586: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:10.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:21:10 smithi083 ceph-mon[79930]: pgmap v10587: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:10.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:21:10 smithi083 ceph-mon[77242]: pgmap v10587: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:10.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:21:10 smithi153 ceph-mon[70044]: pgmap v10587: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:21:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:21:11] "GET /metrics HTTP/1.1" 200 225228 "" "Prometheus/2.33.4" 2022-09-29T11:21:11.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:21:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:21:11.462Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:21:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:21:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:21:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:21:13.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:21:13 smithi153 ceph-mon[70044]: pgmap v10588: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:21:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:21:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:21:13.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:21:13 smithi083 ceph-mon[79930]: pgmap v10588: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:13.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:21:13 smithi083 ceph-mon[77242]: pgmap v10588: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:15.159 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:21:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:21:14.839Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:21:15.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:21:15 smithi083 ceph-mon[77242]: pgmap v10589: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:15.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:21:15 smithi083 ceph-mon[79930]: pgmap v10589: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:15.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:21:15 smithi153 ceph-mon[70044]: pgmap v10589: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:15.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:21:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:21:15.557Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:21:17.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:21:17 smithi083 ceph-mon[79930]: pgmap v10590: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:17.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:21:17 smithi083 ceph-mon[77242]: pgmap v10590: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:17.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:21:17 smithi153 ceph-mon[70044]: pgmap v10590: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:19.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:21:19 smithi083 ceph-mon[77242]: pgmap v10591: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:19.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:21:19 smithi083 ceph-mon[79930]: pgmap v10591: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:19.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:21:19 smithi153 ceph-mon[70044]: pgmap v10591: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:20.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:21:20 smithi083 ceph-mon[79930]: pgmap v10592: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:20.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:21:20 smithi083 ceph-mon[77242]: pgmap v10592: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:20.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:21:20 smithi153 ceph-mon[70044]: pgmap v10592: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:21:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:21:21] "GET /metrics HTTP/1.1" 200 225228 "" "Prometheus/2.33.4" 2022-09-29T11:21:21.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:21:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:21:21.463Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:21:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:21:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:21:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:21:23.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:21:23 smithi153 ceph-mon[70044]: pgmap v10593: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:21:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:21:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:21:23.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:21:23 smithi083 ceph-mon[77242]: pgmap v10593: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:23.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:21:23 smithi083 ceph-mon[79930]: pgmap v10593: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:25.162 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:21:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:21:24.840Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:21:25.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:21:25 smithi083 ceph-mon[79930]: pgmap v10594: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:25.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:21:25 smithi083 ceph-mon[77242]: pgmap v10594: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:25.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:21:25 smithi153 ceph-mon[70044]: pgmap v10594: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:25.738 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:21:25.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:21:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:21:25.557Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:21:26.029 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:21:26.029 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 9m ago 5h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:21:26.029 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 7m ago 5h 64.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:21:26.030 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 9m ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:21:26.030 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 7m ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:21:26.030 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 7m ago 5h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:21:26.030 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 9m ago 6h 807M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:21:26.031 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 9m ago 6h 1307M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:21:26.031 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 7m ago 5h 563M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:21:26.031 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 9m ago 5h 753M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:21:26.031 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 9m ago 5h 26.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:21:26.032 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 7m ago 5h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:21:26.032 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 9m ago 5h 921M 2429M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:21:26.032 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 9m ago 5h 910M 2429M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:21:26.032 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 9m ago 5h 710M 2429M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:21:26.032 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 9m ago 5h 889M 2429M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:21:26.033 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 7m ago 5h 783M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:21:26.033 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 7m ago 5h 977M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:21:26.033 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 7m ago 5h 873M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:21:26.033 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 7m ago 5h 1058M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:21:26.034 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 7m ago 5h 150M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:21:26.403 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:21:26.403 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:21:26.403 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:21:26.404 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:21:26.404 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:21:26.404 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:21:26.404 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:21:26.405 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:21:26.405 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:21:26.405 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:21:26.405 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:21:26.405 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:21:26.406 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:21:26.406 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:21:26.406 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:21:26.406 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:21:26.407 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:21:26.407 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:21:26.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:21:26 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:21:26.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:21:26 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:21:26.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:21:26 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:21:26.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:21:26 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:21:26.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:21:26 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:21:26.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:21:26 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:21:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:21:26 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:21:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:21:26 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:21:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:21:26 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:21:27.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:21:27 smithi083 ceph-mon[77242]: from='client.46455 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:21:27.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:21:27 smithi083 ceph-mon[77242]: from='client.46461 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:21:27.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:21:27 smithi083 ceph-mon[77242]: pgmap v10595: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:27.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:21:27 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3668229729' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:21:27.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:21:27 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:21:27.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:21:27 smithi083 ceph-mon[79930]: from='client.46455 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:21:27.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:21:27 smithi083 ceph-mon[79930]: from='client.46461 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:21:27.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:21:27 smithi083 ceph-mon[79930]: pgmap v10595: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:27.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:21:27 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3668229729' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:21:27.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:21:27 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:21:27.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:21:27 smithi153 ceph-mon[70044]: from='client.46455 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:21:27.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:21:27 smithi153 ceph-mon[70044]: from='client.46461 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:21:27.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:21:27 smithi153 ceph-mon[70044]: pgmap v10595: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:27.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:21:27 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3668229729' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:21:27.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:21:27 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:21:28.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:21:28 smithi083 ceph-mon[77242]: pgmap v10596: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:28.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:21:28 smithi083 ceph-mon[79930]: pgmap v10596: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:21:28 smithi153 ceph-mon[70044]: pgmap v10596: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:30.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:21:30 smithi083 ceph-mon[77242]: pgmap v10597: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:30.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:21:30 smithi083 ceph-mon[79930]: pgmap v10597: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:30.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:21:30 smithi153 ceph-mon[70044]: pgmap v10597: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:21:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:21:31] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T11:21:31.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:21:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:21:31.463Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:21:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:21:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:21:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:21:33.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:21:33 smithi153 ceph-mon[70044]: pgmap v10598: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:21:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:21:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:21:33.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:21:33 smithi083 ceph-mon[79930]: pgmap v10598: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:33.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:21:33 smithi083 ceph-mon[77242]: pgmap v10598: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:35.165 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:21:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:21:34.841Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:21:35.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:21:35 smithi083 ceph-mon[77242]: pgmap v10599: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:35.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:21:35 smithi083 ceph-mon[79930]: pgmap v10599: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:35.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:21:35 smithi153 ceph-mon[70044]: pgmap v10599: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:35.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:21:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:21:35.559Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:21:37.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:21:37 smithi083 ceph-mon[79930]: pgmap v10600: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:37.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:21:37 smithi083 ceph-mon[77242]: pgmap v10600: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:37.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:21:37 smithi153 ceph-mon[70044]: pgmap v10600: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:38.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:21:38 smithi083 ceph-mon[79930]: pgmap v10601: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:38.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:21:38 smithi083 ceph-mon[77242]: pgmap v10601: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:21:38 smithi153 ceph-mon[70044]: pgmap v10601: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:21:40 smithi083 ceph-mon[79930]: pgmap v10602: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:21:40 smithi083 ceph-mon[77242]: pgmap v10602: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:21:40 smithi153 ceph-mon[70044]: pgmap v10602: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:41.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:21:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:21:41] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T11:21:41.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:21:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:21:41.464Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:21:41.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:21:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:21:41.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:21:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:21:41.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:21:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:21:41.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:21:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:21:41.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:21:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:21:41.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:21:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:21:41.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:21:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:21:41.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:21:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:21:41.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:21:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:21:41.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:21:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:21:41.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:21:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:21:41.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:21:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:21:42.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:21:42 smithi083 ceph-mon[79930]: pgmap v10603: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:42.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:21:42 smithi083 ceph-mon[77242]: pgmap v10603: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:21:42 smithi153 ceph-mon[70044]: pgmap v10603: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:21:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:21:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:21:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:21:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:21:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:21:45.166 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:21:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:21:44.842Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:21:45.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:21:45 smithi083 ceph-mon[79930]: pgmap v10604: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:45.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:21:45 smithi083 ceph-mon[77242]: pgmap v10604: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:45.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:21:45 smithi153 ceph-mon[70044]: pgmap v10604: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:45.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:21:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:21:45.559Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:21:47.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:21:47 smithi083 ceph-mon[79930]: pgmap v10605: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:47.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:21:47 smithi083 ceph-mon[77242]: pgmap v10605: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:47.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:21:47 smithi153 ceph-mon[70044]: pgmap v10605: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:48.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:21:48 smithi153 ceph-mon[70044]: pgmap v10606: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:48.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:21:48 smithi083 ceph-mon[79930]: pgmap v10606: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:48.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:21:48 smithi083 ceph-mon[77242]: pgmap v10606: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:50.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:21:50 smithi083 ceph-mon[79930]: pgmap v10607: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:50.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:21:50 smithi083 ceph-mon[77242]: pgmap v10607: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:50.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:21:50 smithi153 ceph-mon[70044]: pgmap v10607: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:21:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:21:51] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T11:21:51.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:21:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:21:51.465Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:21:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:21:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:21:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:21:53.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:21:53 smithi153 ceph-mon[70044]: pgmap v10608: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:21:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:21:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:21:53.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:21:53 smithi083 ceph-mon[79930]: pgmap v10608: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:53.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:21:53 smithi083 ceph-mon[77242]: pgmap v10608: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:55.171 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:21:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:21:54.842Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:21:55.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:21:55 smithi083 ceph-mon[79930]: pgmap v10609: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:55.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:21:55 smithi083 ceph-mon[77242]: pgmap v10609: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:55.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:21:55 smithi153 ceph-mon[70044]: pgmap v10609: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:55.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:21:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:21:55.560Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:21:56.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:21:56 smithi083 ceph-mon[79930]: pgmap v10610: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:56.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:21:56 smithi083 ceph-mon[77242]: pgmap v10610: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:56.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:21:56 smithi153 ceph-mon[70044]: pgmap v10610: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:56.731 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:21:57.018 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:21:57.019 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 9m ago 5h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:21:57.019 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 7m ago 5h 64.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:21:57.019 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 9m ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:21:57.020 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 7m ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:21:57.020 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 7m ago 5h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:21:57.020 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 9m ago 6h 807M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:21:57.020 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 9m ago 6h 1307M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:21:57.021 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 7m ago 5h 563M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:21:57.021 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 9m ago 5h 753M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:21:57.021 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 9m ago 5h 26.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:21:57.021 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 7m ago 5h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:21:57.021 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 9m ago 5h 921M 2429M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:21:57.022 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 9m ago 5h 910M 2429M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:21:57.022 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 9m ago 5h 710M 2429M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:21:57.022 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 9m ago 5h 889M 2429M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:21:57.023 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 7m ago 5h 783M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:21:57.023 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 7m ago 5h 977M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:21:57.023 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 7m ago 5h 873M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:21:57.023 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 7m ago 5h 1058M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:21:57.024 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 7m ago 5h 150M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:21:57.373 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:21:57.374 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:21:57.374 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:21:57.374 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:21:57.375 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:21:57.375 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:21:57.375 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:21:57.375 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:21:57.375 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:21:57.376 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:21:57.376 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:21:57.376 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:21:57.376 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:21:57.377 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:21:57.377 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:21:57.377 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:21:57.377 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:21:57.378 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:21:57.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:21:57 smithi083 ceph-mon[79930]: from='client.56404 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:21:57.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:21:57 smithi083 ceph-mon[79930]: from='client.46479 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:21:57.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:21:57 smithi083 ceph-mon[77242]: from='client.56404 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:21:57.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:21:57 smithi083 ceph-mon[77242]: from='client.46479 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:21:57.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:21:57 smithi153 ceph-mon[70044]: from='client.56404 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:21:57.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:21:57 smithi153 ceph-mon[70044]: from='client.46479 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:21:58.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:21:58 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4021449730' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:21:58.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:21:58 smithi083 ceph-mon[79930]: pgmap v10611: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:58.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:21:58 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4021449730' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:21:58.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:21:58 smithi083 ceph-mon[77242]: pgmap v10611: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:21:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:21:58 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4021449730' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:21:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:21:58 smithi153 ceph-mon[70044]: pgmap v10611: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:00.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:22:00 smithi083 ceph-mon[79930]: pgmap v10612: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:00.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:22:00 smithi083 ceph-mon[77242]: pgmap v10612: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:00.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:22:00 smithi153 ceph-mon[70044]: pgmap v10612: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:22:01] "GET /metrics HTTP/1.1" 200 225229 "" "Prometheus/2.33.4" 2022-09-29T11:22:01.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:22:01.466Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:22:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:22:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:22:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:22:03.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:22:03 smithi153 ceph-mon[70044]: pgmap v10613: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:22:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:22:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:22:03.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:22:03 smithi083 ceph-mon[79930]: pgmap v10613: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:03.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:22:03 smithi083 ceph-mon[77242]: pgmap v10613: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:04.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:22:04 smithi083 ceph-mon[79930]: pgmap v10614: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:04.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:22:04 smithi083 ceph-mon[77242]: pgmap v10614: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:22:04 smithi153 ceph-mon[70044]: pgmap v10614: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:05.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:22:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:22:04.843Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:22:05.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:22:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:22:05.562Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:22:07.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:22:07 smithi083 ceph-mon[77242]: pgmap v10615: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:07.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:22:07 smithi083 ceph-mon[79930]: pgmap v10615: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:07.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:22:07 smithi153 ceph-mon[70044]: pgmap v10615: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:08.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:22:08 smithi083 ceph-mon[77242]: pgmap v10616: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:08.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:22:08 smithi083 ceph-mon[79930]: pgmap v10616: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:22:08 smithi153 ceph-mon[70044]: pgmap v10616: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:10.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:22:10 smithi083 ceph-mon[79930]: pgmap v10617: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:10.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:22:10 smithi083 ceph-mon[77242]: pgmap v10617: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:10.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:22:10 smithi153 ceph-mon[70044]: pgmap v10617: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:22:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:22:11] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T11:22:11.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:22:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:22:11.466Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:22:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:22:12 smithi153 ceph-mon[70044]: pgmap v10618: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:12.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:22:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:22:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:22:12.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:22:12 smithi083 ceph-mon[77242]: pgmap v10618: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:12.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:22:12 smithi083 ceph-mon[79930]: pgmap v10618: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:22:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:22:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:22:14.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:22:14 smithi153 ceph-mon[70044]: pgmap v10619: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:14.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:22:14 smithi083 ceph-mon[79930]: pgmap v10619: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:14.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:22:14 smithi083 ceph-mon[77242]: pgmap v10619: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:15.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:22:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:22:14.843Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:22:15.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:22:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:22:15.562Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:22:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:22:16 smithi153 ceph-mon[70044]: pgmap v10620: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:16.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:22:16 smithi083 ceph-mon[79930]: pgmap v10620: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:16.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:22:16 smithi083 ceph-mon[77242]: pgmap v10620: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:22:18 smithi153 ceph-mon[70044]: pgmap v10621: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:18.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:22:18 smithi083 ceph-mon[79930]: pgmap v10621: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:18.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:22:18 smithi083 ceph-mon[77242]: pgmap v10621: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:20.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:22:20 smithi083 ceph-mon[79930]: pgmap v10622: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:20.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:22:20 smithi083 ceph-mon[77242]: pgmap v10622: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:20.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:22:20 smithi153 ceph-mon[70044]: pgmap v10622: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:22:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:22:21] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T11:22:21.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:22:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:22:21.467Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:22:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:22:22 smithi153 ceph-mon[70044]: pgmap v10623: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:22.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:22:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:22:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:22:22.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:22:22 smithi083 ceph-mon[77242]: pgmap v10623: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:22.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:22:22 smithi083 ceph-mon[79930]: pgmap v10623: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:22:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:22:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:22:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:22:24 smithi153 ceph-mon[70044]: pgmap v10624: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:24.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:22:24 smithi083 ceph-mon[79930]: pgmap v10624: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:24.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:22:24 smithi083 ceph-mon[77242]: pgmap v10624: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:25.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:22:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:22:24.843Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:22:25.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:22:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:22:25.563Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:22:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:22:26 smithi153 ceph-mon[70044]: pgmap v10625: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:26.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:22:26 smithi083 ceph-mon[77242]: pgmap v10625: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:26.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:22:26 smithi083 ceph-mon[79930]: pgmap v10625: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:27.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:22:27 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:22:27.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:22:27 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:22:27.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:22:27 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:22:27.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:22:27 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:22:27.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:22:27 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:22:27.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:22:27 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:22:27.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:22:27 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:22:27.698 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:22:27.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:22:27 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:22:27.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:22:27 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:22:27.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:22:27 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:22:27.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:22:27 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:22:27.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:22:27 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:22:27.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:22:27 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:22:27.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:22:27 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:22:27.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:22:27 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:22:27.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:22:27 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:22:27.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:22:27 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:22:27.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:22:27 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:22:27.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:22:27 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:22:27.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:22:27 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:22:27.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:22:27 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:22:28.022 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:22:28.023 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 10m ago 5h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:22:28.023 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 8m ago 5h 64.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:22:28.023 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 10m ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:22:28.023 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 8m ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:22:28.024 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 8m ago 6h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:22:28.024 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 10m ago 6h 807M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:22:28.024 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 10m ago 6h 1307M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:22:28.024 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 8m ago 6h 563M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:22:28.025 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 10m ago 6h 753M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:22:28.025 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 10m ago 5h 26.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:22:28.025 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 8m ago 5h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:22:28.025 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 10m ago 5h 921M 2429M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:22:28.025 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 10m ago 5h 910M 2429M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:22:28.026 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 10m ago 5h 710M 2429M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:22:28.026 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 10m ago 5h 889M 2429M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:22:28.026 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 8m ago 5h 783M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:22:28.026 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 8m ago 5h 977M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:22:28.027 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 8m ago 5h 873M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:22:28.027 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 8m ago 5h 1058M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:22:28.027 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 8m ago 5h 150M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:22:28.409 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:22:28.410 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:22:28.410 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:22:28.411 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:22:28.411 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:22:28.411 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:22:28.411 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:22:28.411 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:22:28.412 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:22:28.412 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:22:28.412 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:22:28.412 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:22:28.412 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:22:28.413 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:22:28.413 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:22:28.413 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:22:28.413 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:22:28.414 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:22:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:22:28 smithi153 ceph-mon[70044]: from='client.56422 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:22:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:22:28 smithi153 ceph-mon[70044]: from='client.46497 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:22:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:22:28 smithi153 ceph-mon[70044]: pgmap v10626: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:28.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:22:28 smithi083 ceph-mon[79930]: from='client.56422 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:22:28.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:22:28 smithi083 ceph-mon[79930]: from='client.46497 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:22:28.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:22:28 smithi083 ceph-mon[79930]: pgmap v10626: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:28.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:22:28 smithi083 ceph-mon[77242]: from='client.56422 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:22:28.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:22:28 smithi083 ceph-mon[77242]: from='client.46497 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:22:28.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:22:28 smithi083 ceph-mon[77242]: pgmap v10626: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:29.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:22:29 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1616343615' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:22:29.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:22:29 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1616343615' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:22:29.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:22:29 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1616343615' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:22:30.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:22:30 smithi083 ceph-mon[79930]: pgmap v10627: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:30.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:22:30 smithi083 ceph-mon[77242]: pgmap v10627: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:30.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:22:30 smithi153 ceph-mon[70044]: pgmap v10627: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:22:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:22:31] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T11:22:31.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:22:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:22:31.467Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:22:32.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:22:32 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:22:32.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:22:32 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:22:32.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:22:32 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:22:32.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:22:32 smithi083 ceph-mon[77242]: pgmap v10628: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:32.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:22:32 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:22:32.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:22:32 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:22:32.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:22:32 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:22:32.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:22:32 smithi083 ceph-mon[79930]: pgmap v10628: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:22:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:22:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:22:32.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:22:32 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:22:32.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:22:32 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:22:32.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:22:32 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:22:32.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:22:32 smithi153 ceph-mon[70044]: pgmap v10628: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:22:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:22:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:22:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:22:34 smithi153 ceph-mon[70044]: pgmap v10629: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:34.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:22:34 smithi083 ceph-mon[79930]: pgmap v10629: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:34.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:22:34 smithi083 ceph-mon[77242]: pgmap v10629: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:35.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:22:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:22:34.844Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:22:35.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:22:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:22:35.563Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:22:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:22:36 smithi153 ceph-mon[70044]: pgmap v10630: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:36.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:22:36 smithi083 ceph-mon[77242]: pgmap v10630: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:36.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:22:36 smithi083 ceph-mon[79930]: pgmap v10630: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:22:38 smithi153 ceph-mon[70044]: pgmap v10631: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:38.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:22:38 smithi083 ceph-mon[79930]: pgmap v10631: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:38.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:22:38 smithi083 ceph-mon[77242]: pgmap v10631: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:22:40 smithi083 ceph-mon[79930]: pgmap v10632: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:22:40 smithi083 ceph-mon[77242]: pgmap v10632: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:22:40 smithi153 ceph-mon[70044]: pgmap v10632: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:41.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:22:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:22:41] "GET /metrics HTTP/1.1" 200 225228 "" "Prometheus/2.33.4" 2022-09-29T11:22:41.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:22:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:22:41.469Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:22:41.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:22:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:22:41.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:22:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:22:41.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:22:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:22:41.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:22:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:22:41.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:22:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:22:41.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:22:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:22:41.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:22:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:22:41.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:22:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:22:41.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:22:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:22:41.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:22:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:22:41.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:22:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:22:41.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:22:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:22:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:22:42 smithi083 ceph-mon[79930]: pgmap v10633: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:42.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:22:42 smithi083 ceph-mon[77242]: pgmap v10633: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:22:42 smithi153 ceph-mon[70044]: pgmap v10633: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:22:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:22:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:22:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:22:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:22:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:22:44.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:22:44 smithi153 ceph-mon[70044]: pgmap v10634: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:44.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:22:44 smithi083 ceph-mon[77242]: pgmap v10634: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:44.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:22:44 smithi083 ceph-mon[79930]: pgmap v10634: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:45.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:22:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:22:44.845Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:22:45.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:22:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:22:45.564Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:22:46.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:22:46 smithi153 ceph-mon[70044]: pgmap v10635: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:46.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:22:46 smithi083 ceph-mon[79930]: pgmap v10635: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:46.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:22:46 smithi083 ceph-mon[77242]: pgmap v10635: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:48.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:22:48 smithi153 ceph-mon[70044]: pgmap v10636: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:48.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:22:48 smithi083 ceph-mon[79930]: pgmap v10636: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:48.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:22:48 smithi083 ceph-mon[77242]: pgmap v10636: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:50.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:22:50 smithi083 ceph-mon[79930]: pgmap v10637: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:50.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:22:50 smithi083 ceph-mon[77242]: pgmap v10637: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:50.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:22:50 smithi153 ceph-mon[70044]: pgmap v10637: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:22:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:22:51] "GET /metrics HTTP/1.1" 200 225228 "" "Prometheus/2.33.4" 2022-09-29T11:22:51.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:22:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:22:51.469Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:22:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:22:52 smithi153 ceph-mon[70044]: pgmap v10638: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:52.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:22:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:22:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:22:52.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:22:52 smithi083 ceph-mon[79930]: pgmap v10638: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:52.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:22:52 smithi083 ceph-mon[77242]: pgmap v10638: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:22:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:22:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:22:54.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:22:54 smithi153 ceph-mon[70044]: pgmap v10639: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:54.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:22:54 smithi083 ceph-mon[77242]: pgmap v10639: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:54.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:22:54 smithi083 ceph-mon[79930]: pgmap v10639: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:55.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:22:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:22:54.846Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:22:55.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:22:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:22:55.565Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:22:56.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:22:56 smithi153 ceph-mon[70044]: pgmap v10640: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:56.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:22:56 smithi083 ceph-mon[79930]: pgmap v10640: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:56.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:22:56 smithi083 ceph-mon[77242]: pgmap v10640: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:22:58 smithi153 ceph-mon[70044]: pgmap v10641: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:58.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:22:58 smithi083 ceph-mon[79930]: pgmap v10641: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:58.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:22:58 smithi083 ceph-mon[77242]: pgmap v10641: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:22:58.731 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:22:59.020 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:22:59.021 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 27s ago 5h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:22:59.021 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 8m ago 5h 64.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:22:59.021 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 27s ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:22:59.022 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 8m ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:22:59.022 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 8m ago 6h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:22:59.022 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 27s ago 6h 815M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:22:59.022 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 27s ago 6h 1344M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:22:59.023 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 8m ago 6h 563M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:22:59.023 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 27s ago 6h 774M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:22:59.023 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 27s ago 5h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:22:59.023 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 8m ago 5h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:22:59.024 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 27s ago 6h 947M 2429M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:22:59.024 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 27s ago 6h 936M 2429M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:22:59.024 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 27s ago 5h 730M 2429M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:22:59.024 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 27s ago 5h 914M 2429M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:22:59.025 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 8m ago 5h 783M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:22:59.025 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 8m ago 5h 977M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:22:59.025 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 8m ago 5h 873M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:22:59.025 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 8m ago 5h 1058M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:22:59.026 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 8m ago 5h 150M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:22:59.377 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:22:59.378 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:22:59.378 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:22:59.378 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:22:59.378 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:22:59.379 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:22:59.379 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:22:59.379 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:22:59.379 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:22:59.380 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:22:59.380 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:22:59.380 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:22:59.380 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:22:59.381 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:22:59.381 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:22:59.381 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:22:59.381 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:22:59.382 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:22:59.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:22:59 smithi153 ceph-mon[70044]: from='client.46509 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:22:59.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:22:59 smithi153 ceph-mon[70044]: from='client.46515 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:22:59.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:22:59 smithi083 ceph-mon[79930]: from='client.46509 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:22:59.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:22:59 smithi083 ceph-mon[79930]: from='client.46515 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:22:59.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:22:59 smithi083 ceph-mon[77242]: from='client.46509 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:22:59.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:22:59 smithi083 ceph-mon[77242]: from='client.46515 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:23:00.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:23:00 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1339674260' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:23:00.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:23:00 smithi083 ceph-mon[79930]: pgmap v10642: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:23:00.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:23:00 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1339674260' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:23:00.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:23:00 smithi083 ceph-mon[77242]: pgmap v10642: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:23:00.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:23:00 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1339674260' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:23:00.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:23:00 smithi153 ceph-mon[70044]: pgmap v10642: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:23:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:23:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:23:01] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T11:23:01.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:23:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:23:01.470Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:23:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:23:02 smithi153 ceph-mon[70044]: pgmap v10643: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:23:02.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:23:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:23:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:23:02.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:23:02 smithi083 ceph-mon[77242]: pgmap v10643: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:23:02.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:23:02 smithi083 ceph-mon[79930]: pgmap v10643: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:23:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:23:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:23:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:23:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:23:04 smithi153 ceph-mon[70044]: pgmap v10644: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:23:04.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:23:04 smithi083 ceph-mon[79930]: pgmap v10644: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:23:04.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:23:04 smithi083 ceph-mon[77242]: pgmap v10644: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:23:05.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:23:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:23:04.847Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:23:05.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:23:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:23:05.566Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:23:06.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:23:06 smithi153 ceph-mon[70044]: pgmap v10645: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:23:06.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:23:06 smithi083 ceph-mon[77242]: pgmap v10645: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:23:06.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:23:06 smithi083 ceph-mon[79930]: pgmap v10645: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:23:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:23:08 smithi153 ceph-mon[70044]: pgmap v10646: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:23:08.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:23:08 smithi083 ceph-mon[77242]: pgmap v10646: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:23:08.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:23:08 smithi083 ceph-mon[79930]: pgmap v10646: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:23:10.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:23:10 smithi083 ceph-mon[77242]: pgmap v10647: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:23:10.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:23:10 smithi083 ceph-mon[79930]: pgmap v10647: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:23:10.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:23:10 smithi153 ceph-mon[70044]: pgmap v10647: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:23:11.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:23:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:23:11.471Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:23:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:23:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:23:11] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T11:23:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:23:12 smithi153 ceph-mon[70044]: pgmap v10648: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:23:12.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:23:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:23:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:23:12.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:23:12 smithi083 ceph-mon[77242]: pgmap v10648: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:23:12.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:23:12 smithi083 ceph-mon[79930]: pgmap v10648: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:23:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:23:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:23:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:23:14.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:23:14 smithi153 ceph-mon[70044]: pgmap v10649: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:23:14.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:23:14 smithi083 ceph-mon[77242]: pgmap v10649: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:23:14.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:23:14 smithi083 ceph-mon[79930]: pgmap v10649: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:23:15.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:23:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:23:14.847Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:23:15.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:23:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:23:15.566Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:23:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:23:16 smithi153 ceph-mon[70044]: pgmap v10650: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:23:16.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:23:16 smithi083 ceph-mon[79930]: pgmap v10650: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:23:16.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:23:16 smithi083 ceph-mon[77242]: pgmap v10650: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:23:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:23:18 smithi153 ceph-mon[70044]: pgmap v10651: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:23:18.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:23:18 smithi083 ceph-mon[79930]: pgmap v10651: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:23:18.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:23:18 smithi083 ceph-mon[77242]: pgmap v10651: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:23:20.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:23:20 smithi083 ceph-mon[79930]: pgmap v10652: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:23:20.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:23:20 smithi083 ceph-mon[77242]: pgmap v10652: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:23:20.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:23:20 smithi153 ceph-mon[70044]: pgmap v10652: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:23:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:23:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:23:21] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T11:23:21.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:23:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:23:21.471Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:23:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:23:22 smithi153 ceph-mon[70044]: pgmap v10653: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:23:22.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:23:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:23:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:23:22.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:23:22 smithi083 ceph-mon[79930]: pgmap v10653: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:23:22.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:23:22 smithi083 ceph-mon[77242]: pgmap v10653: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:23:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:23:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:23:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:23:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:23:24 smithi153 ceph-mon[70044]: pgmap v10654: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:23:24.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:23:24 smithi083 ceph-mon[79930]: pgmap v10654: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:23:24.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:23:24 smithi083 ceph-mon[77242]: pgmap v10654: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:23:25.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:23:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:23:24.847Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:23:25.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:23:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:23:25.567Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:23:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:23:26 smithi153 ceph-mon[70044]: pgmap v10655: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:23:26.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:23:26 smithi083 ceph-mon[77242]: pgmap v10655: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:23:26.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:23:26 smithi083 ceph-mon[79930]: pgmap v10655: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:23:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:23:28 smithi153 ceph-mon[70044]: pgmap v10656: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:23:28.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:23:28 smithi083 ceph-mon[79930]: pgmap v10656: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:23:28.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:23:28 smithi083 ceph-mon[77242]: pgmap v10656: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:23:29.703 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:23:29.991 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:23:29.992 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 58s ago 5h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:23:29.992 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 9m ago 5h 64.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:23:29.992 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 58s ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:23:29.993 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 9m ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:23:29.993 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 9m ago 6h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:23:29.993 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 58s ago 6h 815M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:23:29.993 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 58s ago 6h 1344M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:23:29.994 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 9m ago 6h 563M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:23:29.994 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 58s ago 6h 774M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:23:29.994 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 58s ago 5h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:23:29.994 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 9m ago 5h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:23:29.995 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 58s ago 6h 947M 2429M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:23:29.995 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 58s ago 6h 936M 2429M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:23:29.995 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 58s ago 6h 730M 2429M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:23:29.995 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 58s ago 5h 914M 2429M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:23:29.996 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 9m ago 5h 783M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:23:29.996 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 9m ago 5h 977M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:23:29.996 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 9m ago 5h 873M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:23:29.996 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 9m ago 5h 1058M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:23:29.997 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 9m ago 5h 150M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:23:30.346 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:23:30.347 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:23:30.347 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:23:30.347 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:23:30.348 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:23:30.348 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:23:30.348 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:23:30.348 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:23:30.348 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:23:30.349 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:23:30.349 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:23:30.349 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:23:30.349 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:23:30.350 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:23:30.350 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:23:30.350 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:23:30.350 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:23:30.351 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:23:30.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:23:30 smithi083 ceph-mon[79930]: from='client.46527 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:23:30.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:23:30 smithi083 ceph-mon[79930]: from='client.46533 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:23:30.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:23:30 smithi083 ceph-mon[79930]: pgmap v10657: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:23:30.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:23:30 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4262514326' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:23:30.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:23:30 smithi083 ceph-mon[77242]: from='client.46527 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:23:30.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:23:30 smithi083 ceph-mon[77242]: from='client.46533 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:23:30.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:23:30 smithi083 ceph-mon[77242]: pgmap v10657: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:23:30.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:23:30 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4262514326' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:23:30.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:23:30 smithi153 ceph-mon[70044]: from='client.46527 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:23:30.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:23:30 smithi153 ceph-mon[70044]: from='client.46533 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:23:30.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:23:30 smithi153 ceph-mon[70044]: pgmap v10657: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:23:30.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:23:30 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4262514326' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:23:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:23:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:23:31] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T11:23:31.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:23:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:23:31.472Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:23:32.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:23:32 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:23:32.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:23:32 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:23:32.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:23:32 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:23:32.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:23:32 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:23:32.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:23:32 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:23:32.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:23:32 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:23:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:23:32 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:23:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:23:32 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:23:32.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:23:32 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:23:32.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:23:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:23:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:23:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:23:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:23:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:23:33.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:23:33 smithi153 ceph-mon[70044]: pgmap v10658: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:23:33.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:23:33 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:23:33.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:23:33 smithi083 ceph-mon[77242]: pgmap v10658: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:23:33.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:23:33 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:23:33.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:23:33 smithi083 ceph-mon[79930]: pgmap v10658: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:23:33.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:23:33 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:23:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:23:34 smithi153 ceph-mon[70044]: pgmap v10659: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:23:34.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:23:34 smithi083 ceph-mon[79930]: pgmap v10659: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:23:34.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:23:34 smithi083 ceph-mon[77242]: pgmap v10659: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:23:35.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:23:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:23:34.848Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:23:35.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:23:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:23:35.567Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:23:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:23:36 smithi153 ceph-mon[70044]: pgmap v10660: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:23:36.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:23:36 smithi083 ceph-mon[77242]: pgmap v10660: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:23:36.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:23:36 smithi083 ceph-mon[79930]: pgmap v10660: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:23:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:23:38 smithi153 ceph-mon[70044]: pgmap v10661: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:23:38.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:23:38 smithi083 ceph-mon[79930]: pgmap v10661: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:23:38.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:23:38 smithi083 ceph-mon[77242]: pgmap v10661: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:23:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:23:40 smithi083 ceph-mon[77242]: pgmap v10662: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:23:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:23:40 smithi083 ceph-mon[79930]: pgmap v10662: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:23:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:23:40 smithi153 ceph-mon[70044]: pgmap v10662: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:23:41.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:23:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:23:41] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T11:23:41.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:23:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:23:41.473Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:23:41.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:23:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:23:41.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:23:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:23:41.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:23:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:23:41.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:23:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:23:41.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:23:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:23:41.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:23:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:23:41.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:23:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:23:41.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:23:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:23:41.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:23:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:23:41.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:23:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:23:41.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:23:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:23:41.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:23:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:23:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:23:42 smithi083 ceph-mon[79930]: pgmap v10663: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:23:42.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:23:42 smithi083 ceph-mon[77242]: pgmap v10663: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:23:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:23:42 smithi153 ceph-mon[70044]: pgmap v10663: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:23:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:23:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:23:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:23:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:23:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:23:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:23:44.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:23:44 smithi153 ceph-mon[70044]: pgmap v10664: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:23:44.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:23:44 smithi083 ceph-mon[77242]: pgmap v10664: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:23:44.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:23:44 smithi083 ceph-mon[79930]: pgmap v10664: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:23:45.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:23:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:23:44.849Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:23:45.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:23:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:23:45.568Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:23:46.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:23:46 smithi153 ceph-mon[70044]: pgmap v10665: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:23:46.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:23:46 smithi083 ceph-mon[79930]: pgmap v10665: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:23:46.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:23:46 smithi083 ceph-mon[77242]: pgmap v10665: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:23:48.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:23:48 smithi153 ceph-mon[70044]: pgmap v10666: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:23:48.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:23:48 smithi083 ceph-mon[79930]: pgmap v10666: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:23:48.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:23:48 smithi083 ceph-mon[77242]: pgmap v10666: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:23:50.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:23:50 smithi083 ceph-mon[79930]: pgmap v10667: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:23:50.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:23:50 smithi083 ceph-mon[77242]: pgmap v10667: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:23:50.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:23:50 smithi153 ceph-mon[70044]: pgmap v10667: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:23:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:23:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:23:51] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T11:23:51.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:23:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:23:51.474Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:23:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:23:52 smithi153 ceph-mon[70044]: pgmap v10668: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:23:52.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:23:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:23:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:23:52.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:23:52 smithi083 ceph-mon[79930]: pgmap v10668: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:23:52.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:23:52 smithi083 ceph-mon[77242]: pgmap v10668: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:23:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:23:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:23:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:23:54.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:23:54 smithi153 ceph-mon[70044]: pgmap v10669: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:23:54.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:23:54 smithi083 ceph-mon[79930]: pgmap v10669: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:23:54.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:23:54 smithi083 ceph-mon[77242]: pgmap v10669: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:23:55.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:23:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:23:54.848Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:23:55.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:23:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:23:55.569Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:23:56.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:23:56 smithi153 ceph-mon[70044]: pgmap v10670: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:23:56.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:23:56 smithi083 ceph-mon[79930]: pgmap v10670: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:23:56.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:23:56 smithi083 ceph-mon[77242]: pgmap v10670: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:23:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:23:58 smithi153 ceph-mon[70044]: pgmap v10671: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:23:58.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:23:58 smithi083 ceph-mon[79930]: pgmap v10671: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:23:58.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:23:58 smithi083 ceph-mon[77242]: pgmap v10671: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:00.676 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:24:00.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:24:00 smithi083 ceph-mon[79930]: pgmap v10672: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:00.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:24:00 smithi083 ceph-mon[77242]: pgmap v10672: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:00.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:24:00 smithi153 ceph-mon[70044]: pgmap v10672: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:00.965 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:24:00.965 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 89s ago 5h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:24:00.965 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 9m ago 5h 64.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:24:00.966 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 89s ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:24:00.966 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 9m ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:24:00.966 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 9m ago 6h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:24:00.966 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 89s ago 6h 815M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:24:00.967 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 89s ago 6h 1344M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:24:00.967 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 9m ago 6h 563M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:24:00.967 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 89s ago 6h 774M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:24:00.967 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 89s ago 5h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:24:00.968 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 9m ago 5h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:24:00.968 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 89s ago 6h 947M 2429M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:24:00.968 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 89s ago 6h 936M 2429M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:24:00.968 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 89s ago 6h 730M 2429M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:24:00.968 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 89s ago 6h 914M 2429M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:24:00.969 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 9m ago 6h 783M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:24:00.969 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 9m ago 5h 977M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:24:00.969 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 9m ago 5h 873M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:24:00.969 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 9m ago 5h 1058M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:24:00.970 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 9m ago 5h 150M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:24:01.320 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:24:01.320 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:24:01.321 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:24:01.321 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:24:01.321 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:24:01.321 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:24:01.322 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:24:01.322 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:24:01.322 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:24:01.322 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:24:01.322 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:24:01.323 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:24:01.323 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:24:01.323 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:24:01.323 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:24:01.324 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:24:01.324 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:24:01.324 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:24:01.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:24:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:24:01.474Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:24:01.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:24:01 smithi083 ceph-mon[77242]: from='client.56476 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:24:01.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:24:01 smithi083 ceph-mon[77242]: from='client.56482 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:24:01.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:24:01 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2808245548' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:24:01.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:24:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:24:01] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T11:24:01.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:24:01 smithi083 ceph-mon[79930]: from='client.56476 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:24:01.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:24:01 smithi083 ceph-mon[79930]: from='client.56482 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:24:01.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:24:01 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2808245548' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:24:01.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:24:01 smithi153 ceph-mon[70044]: from='client.56476 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:24:01.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:24:01 smithi153 ceph-mon[70044]: from='client.56482 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:24:01.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:24:01 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2808245548' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:24:02.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:24:02 smithi083 ceph-mon[79930]: pgmap v10673: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:02.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:24:02 smithi083 ceph-mon[77242]: pgmap v10673: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:02.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:24:02 smithi153 ceph-mon[70044]: pgmap v10673: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:24:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:24:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:24:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:24:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:24:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:24:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:24:04 smithi153 ceph-mon[70044]: pgmap v10674: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:04.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:24:04 smithi083 ceph-mon[79930]: pgmap v10674: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:04.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:24:04 smithi083 ceph-mon[77242]: pgmap v10674: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:05.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:24:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:24:04.849Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:24:05.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:24:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:24:05.570Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:24:06.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:24:06 smithi153 ceph-mon[70044]: pgmap v10675: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:06.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:24:06 smithi083 ceph-mon[79930]: pgmap v10675: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:06.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:24:06 smithi083 ceph-mon[77242]: pgmap v10675: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:24:08 smithi153 ceph-mon[70044]: pgmap v10676: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:08.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:24:08 smithi083 ceph-mon[77242]: pgmap v10676: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:08.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:24:08 smithi083 ceph-mon[79930]: pgmap v10676: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:10.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:24:10 smithi083 ceph-mon[77242]: pgmap v10677: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:10.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:24:10 smithi083 ceph-mon[79930]: pgmap v10677: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:10.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:24:10 smithi153 ceph-mon[70044]: pgmap v10677: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:24:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:24:11] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T11:24:11.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:24:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:24:11.474Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:24:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:24:12 smithi153 ceph-mon[70044]: pgmap v10678: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:12.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:24:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:24:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:24:12.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:24:12 smithi083 ceph-mon[77242]: pgmap v10678: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:12.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:24:12 smithi083 ceph-mon[79930]: pgmap v10678: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:24:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:24:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:24:14.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:24:14 smithi153 ceph-mon[70044]: pgmap v10679: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:14.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:24:14 smithi083 ceph-mon[77242]: pgmap v10679: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:14.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:24:14 smithi083 ceph-mon[79930]: pgmap v10679: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:15.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:24:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:24:14.851Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:24:15.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:24:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:24:15.571Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:24:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:24:16 smithi153 ceph-mon[70044]: pgmap v10680: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:16.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:24:16 smithi083 ceph-mon[77242]: pgmap v10680: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:16.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:24:16 smithi083 ceph-mon[79930]: pgmap v10680: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:18.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:24:18 smithi153 ceph-mon[70044]: pgmap v10681: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:18.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:24:18 smithi083 ceph-mon[79930]: pgmap v10681: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:18.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:24:18 smithi083 ceph-mon[77242]: pgmap v10681: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:20.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:24:20 smithi083 ceph-mon[77242]: pgmap v10682: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:20.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:24:20 smithi083 ceph-mon[79930]: pgmap v10682: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:20.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:24:20 smithi153 ceph-mon[70044]: pgmap v10682: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:24:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:24:21] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T11:24:21.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:24:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:24:21.475Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:24:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:24:22 smithi153 ceph-mon[70044]: pgmap v10683: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:22.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:24:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:24:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:24:22.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:24:22 smithi083 ceph-mon[79930]: pgmap v10683: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:22.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:24:22 smithi083 ceph-mon[77242]: pgmap v10683: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:24:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:24:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:24:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:24:24 smithi153 ceph-mon[70044]: pgmap v10684: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:24.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:24:24 smithi083 ceph-mon[79930]: pgmap v10684: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:24.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:24:24 smithi083 ceph-mon[77242]: pgmap v10684: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:25.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:24:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:24:24.851Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:24:25.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:24:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:24:25.571Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:24:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:24:26 smithi153 ceph-mon[70044]: pgmap v10685: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:26.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:24:26 smithi083 ceph-mon[77242]: pgmap v10685: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:26.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:24:26 smithi083 ceph-mon[79930]: pgmap v10685: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:24:28 smithi153 ceph-mon[70044]: pgmap v10686: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:28.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:24:28 smithi083 ceph-mon[79930]: pgmap v10686: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:28.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:24:28 smithi083 ceph-mon[77242]: pgmap v10686: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:30.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:24:30 smithi083 ceph-mon[79930]: pgmap v10687: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:30.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:24:30 smithi083 ceph-mon[77242]: pgmap v10687: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:30.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:24:30 smithi153 ceph-mon[70044]: pgmap v10687: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:31.648 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:24:31.934 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:24:31.934 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 2m ago 5h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:24:31.935 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 10m ago 5h 64.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:24:31.935 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (5h) 2m ago 5h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:24:31.935 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (5h) 10m ago 5h 28.0M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:24:31.935 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 10m ago 6h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:24:31.936 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 2m ago 6h 815M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:24:31.936 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 2m ago 6h 1344M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:24:31.936 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 10m ago 6h 563M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:24:31.936 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 2m ago 6h 774M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:24:31.937 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 2m ago 5h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:24:31.937 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 10m ago 5h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:24:31.937 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 2m ago 6h 947M 2429M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:24:31.937 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 2m ago 6h 936M 2429M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:24:31.938 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 2m ago 6h 730M 2429M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:24:31.938 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 2m ago 6h 914M 2429M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:24:31.938 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 10m ago 6h 783M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:24:31.938 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 10m ago 6h 977M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:24:31.939 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 10m ago 6h 873M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:24:31.939 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 10m ago 5h 1058M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:24:31.939 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 10m ago 5h 150M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:24:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:24:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:24:31] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T11:24:31.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:24:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:24:31.477Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:24:32.292 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:24:32.292 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:24:32.292 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:24:32.293 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:24:32.293 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:24:32.293 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:24:32.293 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:24:32.294 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:24:32.294 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:24:32.294 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:24:32.294 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:24:32.295 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:24:32.295 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:24:32.295 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:24:32.295 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:24:32.295 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:24:32.296 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:24:32.296 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:24:32.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:24:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:24:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:24:32.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:24:32 smithi153 ceph-mon[70044]: from='client.56494 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:24:32.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:24:32 smithi153 ceph-mon[70044]: from='client.56500 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:24:32.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:24:32 smithi153 ceph-mon[70044]: pgmap v10688: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:32.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:24:32 smithi083 ceph-mon[77242]: from='client.56494 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:24:32.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:24:32 smithi083 ceph-mon[77242]: from='client.56500 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:24:32.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:24:32 smithi083 ceph-mon[77242]: pgmap v10688: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:32.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:24:32 smithi083 ceph-mon[79930]: from='client.56494 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:24:32.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:24:32 smithi083 ceph-mon[79930]: from='client.56500 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:24:32.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:24:32 smithi083 ceph-mon[79930]: pgmap v10688: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:24:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:24:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:24:33.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:24:33 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3259855206' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:24:33.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:24:33 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:24:33.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:24:33 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:24:33.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:24:33 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:24:33.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:24:33 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3259855206' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:24:33.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:24:33 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:24:33.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:24:33 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:24:33.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:24:33 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:24:33.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:24:33 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3259855206' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:24:33.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:24:33 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:24:33.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:24:33 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:24:33.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:24:33 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:24:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:24:34 smithi153 ceph-mon[70044]: pgmap v10689: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:34.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:24:34 smithi083 ceph-mon[77242]: pgmap v10689: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:34.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:24:34 smithi083 ceph-mon[79930]: pgmap v10689: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:35.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:24:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:24:34.851Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:24:35.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:24:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:24:35.571Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:24:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:24:36 smithi153 ceph-mon[70044]: pgmap v10690: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:36.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:24:36 smithi083 ceph-mon[79930]: pgmap v10690: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:36.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:24:36 smithi083 ceph-mon[77242]: pgmap v10690: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:38.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:24:38 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:24:38.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:24:38 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:24:38.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:24:38 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:24:38.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:24:38 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:24:38.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:24:38 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:24:38.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:24:38 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:24:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:24:38 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:24:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:24:38 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:24:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:24:38 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:24:39.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:24:39 smithi083 ceph-mon[77242]: pgmap v10691: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:39.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:24:39 smithi083 ceph-mon[79930]: pgmap v10691: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:39.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:24:39 smithi153 ceph-mon[70044]: pgmap v10691: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:24:40 smithi083 ceph-mon[79930]: pgmap v10692: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:24:40 smithi083 ceph-mon[77242]: pgmap v10692: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:24:40 smithi153 ceph-mon[70044]: pgmap v10692: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:41.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:24:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:24:41] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T11:24:41.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:24:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:24:41.478Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:24:41.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:24:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:24:41.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:24:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:24:41.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:24:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:24:41.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:24:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:24:41.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:24:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:24:41.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:24:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:24:41.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:24:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:24:41.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:24:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:24:41.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:24:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:24:41.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:24:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:24:41.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:24:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:24:41.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:24:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:24:42.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:24:42 smithi083 ceph-mon[77242]: pgmap v10693: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:42.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:24:42 smithi083 ceph-mon[79930]: pgmap v10693: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:24:42 smithi153 ceph-mon[70044]: pgmap v10693: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:24:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:24:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:24:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:24:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:24:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:24:44.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:24:44 smithi153 ceph-mon[70044]: pgmap v10694: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:44.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:24:44 smithi083 ceph-mon[77242]: pgmap v10694: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:44.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:24:44 smithi083 ceph-mon[79930]: pgmap v10694: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:45.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:24:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:24:44.851Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:24:45.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:24:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:24:45.572Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:24:46.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:24:46 smithi153 ceph-mon[70044]: pgmap v10695: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:46.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:24:46 smithi083 ceph-mon[79930]: pgmap v10695: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:46.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:24:46 smithi083 ceph-mon[77242]: pgmap v10695: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:48.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:24:48 smithi153 ceph-mon[70044]: pgmap v10696: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:48.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:24:48 smithi083 ceph-mon[79930]: pgmap v10696: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:48.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:24:48 smithi083 ceph-mon[77242]: pgmap v10696: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:50.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:24:50 smithi083 ceph-mon[77242]: pgmap v10697: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:50.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:24:50 smithi083 ceph-mon[79930]: pgmap v10697: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:50.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:24:50 smithi153 ceph-mon[70044]: pgmap v10697: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:24:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:24:51] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T11:24:51.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:24:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:24:51.478Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:24:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:24:52 smithi153 ceph-mon[70044]: pgmap v10698: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:52.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:24:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:24:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:24:52.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:24:52 smithi083 ceph-mon[79930]: pgmap v10698: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:52.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:24:52 smithi083 ceph-mon[77242]: pgmap v10698: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:24:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:24:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:24:54.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:24:54 smithi153 ceph-mon[70044]: pgmap v10699: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:54.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:24:54 smithi083 ceph-mon[79930]: pgmap v10699: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:54.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:24:54 smithi083 ceph-mon[77242]: pgmap v10699: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:55.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:24:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:24:54.852Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:24:55.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:24:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:24:55.572Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:24:56.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:24:56 smithi153 ceph-mon[70044]: pgmap v10700: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:56.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:24:56 smithi083 ceph-mon[77242]: pgmap v10700: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:56.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:24:56 smithi083 ceph-mon[79930]: pgmap v10700: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:24:58 smithi153 ceph-mon[70044]: pgmap v10701: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:58.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:24:58 smithi083 ceph-mon[79930]: pgmap v10701: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:24:58.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:24:58 smithi083 ceph-mon[77242]: pgmap v10701: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:00.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:25:00 smithi083 ceph-mon[79930]: pgmap v10702: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:00.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:25:00 smithi083 ceph-mon[77242]: pgmap v10702: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:00.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:25:00 smithi153 ceph-mon[70044]: pgmap v10702: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:25:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:25:01] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T11:25:01.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:25:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:25:01.479Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:25:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:25:02 smithi153 ceph-mon[70044]: pgmap v10703: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:02.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:25:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:25:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:25:02.622 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:25:02.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:25:02 smithi083 ceph-mon[79930]: pgmap v10703: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:02.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:25:02 smithi083 ceph-mon[77242]: pgmap v10703: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:02.911 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:25:02.911 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 2m ago 5h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:25:02.912 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 25s ago 5h 63.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:25:02.912 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 2m ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:25:02.912 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 25s ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:25:02.912 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 25s ago 6h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:25:02.913 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 2m ago 6h 815M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:25:02.913 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 2m ago 6h 1344M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:25:02.913 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 25s ago 6h 578M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:25:02.913 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 2m ago 6h 774M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:25:02.914 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 2m ago 5h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:25:02.914 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 25s ago 5h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:25:02.914 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 2m ago 6h 947M 2429M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:25:02.914 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 2m ago 6h 936M 2429M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:25:02.914 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 2m ago 6h 730M 2429M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:25:02.915 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 2m ago 6h 914M 2429M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:25:02.915 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 25s ago 6h 805M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:25:02.915 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 25s ago 6h 1029M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:25:02.915 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 25s ago 6h 898M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:25:02.915 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 25s ago 6h 1089M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:25:02.916 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 25s ago 5h 152M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:25:03.266 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:25:03.267 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:25:03.267 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:25:03.267 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:25:03.267 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:25:03.268 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:25:03.268 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:25:03.268 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:25:03.268 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:25:03.268 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:25:03.269 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:25:03.269 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:25:03.269 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:25:03.269 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:25:03.270 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:25:03.270 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:25:03.270 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:25:03.270 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:25:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:25:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:25:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:25:03.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:25:03 smithi153 ceph-mon[70044]: from='client.46581 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:25:03.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:25:03 smithi153 ceph-mon[70044]: from='client.56518 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:25:03.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:25:03 smithi083 ceph-mon[79930]: from='client.46581 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:25:03.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:25:03 smithi083 ceph-mon[79930]: from='client.56518 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:25:03.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:25:03 smithi083 ceph-mon[77242]: from='client.46581 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:25:03.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:25:03 smithi083 ceph-mon[77242]: from='client.56518 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:25:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:25:04 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2441202535' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:25:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:25:04 smithi153 ceph-mon[70044]: pgmap v10704: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:04.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:25:04 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2441202535' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:25:04.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:25:04 smithi083 ceph-mon[77242]: pgmap v10704: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:04.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:25:04 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2441202535' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:25:04.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:25:04 smithi083 ceph-mon[79930]: pgmap v10704: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:05.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:25:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:25:04.853Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:25:05.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:25:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:25:05.573Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:25:06.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:25:06 smithi153 ceph-mon[70044]: pgmap v10705: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:06.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:25:06 smithi083 ceph-mon[79930]: pgmap v10705: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:06.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:25:06 smithi083 ceph-mon[77242]: pgmap v10705: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:25:08 smithi153 ceph-mon[70044]: pgmap v10706: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:08.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:25:08 smithi083 ceph-mon[79930]: pgmap v10706: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:08.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:25:08 smithi083 ceph-mon[77242]: pgmap v10706: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:10.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:25:10 smithi083 ceph-mon[77242]: pgmap v10707: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:10.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:25:10 smithi083 ceph-mon[79930]: pgmap v10707: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:10.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:25:10 smithi153 ceph-mon[70044]: pgmap v10707: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:25:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:25:11] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T11:25:11.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:25:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:25:11.479Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:25:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:25:12 smithi153 ceph-mon[70044]: pgmap v10708: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:12.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:25:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:25:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:25:12.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:25:12 smithi083 ceph-mon[79930]: pgmap v10708: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:12.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:25:12 smithi083 ceph-mon[77242]: pgmap v10708: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:25:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:25:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:25:14.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:25:14 smithi153 ceph-mon[70044]: pgmap v10709: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:14.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:25:14 smithi083 ceph-mon[79930]: pgmap v10709: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:14.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:25:14 smithi083 ceph-mon[77242]: pgmap v10709: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:15.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:25:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:25:14.853Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:25:15.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:25:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:25:15.573Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:25:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:25:16 smithi153 ceph-mon[70044]: pgmap v10710: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:16.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:25:16 smithi083 ceph-mon[79930]: pgmap v10710: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:16.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:25:16 smithi083 ceph-mon[77242]: pgmap v10710: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:25:18 smithi153 ceph-mon[70044]: pgmap v10711: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:18.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:25:18 smithi083 ceph-mon[77242]: pgmap v10711: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:18.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:25:18 smithi083 ceph-mon[79930]: pgmap v10711: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:20.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:25:20 smithi083 ceph-mon[79930]: pgmap v10712: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:20.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:25:20 smithi083 ceph-mon[77242]: pgmap v10712: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:20.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:25:20 smithi153 ceph-mon[70044]: pgmap v10712: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:25:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:25:21] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T11:25:21.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:25:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:25:21.480Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:25:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:25:22 smithi153 ceph-mon[70044]: pgmap v10713: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:22.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:25:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:25:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:25:22.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:25:22 smithi083 ceph-mon[77242]: pgmap v10713: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:22.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:25:22 smithi083 ceph-mon[79930]: pgmap v10713: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:25:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:25:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:25:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:25:24 smithi153 ceph-mon[70044]: pgmap v10714: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:24.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:25:24 smithi083 ceph-mon[79930]: pgmap v10714: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:24.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:25:24 smithi083 ceph-mon[77242]: pgmap v10714: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:25.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:25:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:25:24.853Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:25:25.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:25:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:25:25.574Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:25:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:25:26 smithi153 ceph-mon[70044]: pgmap v10715: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:26.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:25:26 smithi083 ceph-mon[79930]: pgmap v10715: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:26.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:25:26 smithi083 ceph-mon[77242]: pgmap v10715: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:25:28 smithi153 ceph-mon[70044]: pgmap v10716: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:28.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:25:28 smithi083 ceph-mon[79930]: pgmap v10716: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:28.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:25:28 smithi083 ceph-mon[77242]: pgmap v10716: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:30.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:25:30 smithi083 ceph-mon[79930]: pgmap v10717: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:30.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:25:30 smithi083 ceph-mon[77242]: pgmap v10717: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:30.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:25:30 smithi153 ceph-mon[70044]: pgmap v10717: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:25:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:25:31] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T11:25:31.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:25:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:25:31.480Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:25:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:25:32 smithi153 ceph-mon[70044]: pgmap v10718: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:32.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:25:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:25:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:25:32.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:25:32 smithi083 ceph-mon[77242]: pgmap v10718: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:32.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:25:32 smithi083 ceph-mon[79930]: pgmap v10718: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:25:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:25:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:25:33.606 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:25:33.901 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:25:33.901 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 3m ago 5h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:25:33.901 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (5h) 56s ago 5h 63.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:25:33.901 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 3m ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:25:33.902 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 56s ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:25:33.902 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 56s ago 6h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:25:33.902 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 3m ago 6h 815M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:25:33.902 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 3m ago 6h 1344M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:25:33.903 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 56s ago 6h 578M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:25:33.903 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 3m ago 6h 774M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:25:33.903 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5h) 3m ago 5h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:25:33.903 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (5h) 56s ago 5h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:25:33.903 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 3m ago 6h 947M 2429M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:25:33.904 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 3m ago 6h 936M 2429M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:25:33.904 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 3m ago 6h 730M 2429M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:25:33.904 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 3m ago 6h 914M 2429M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:25:33.904 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 56s ago 6h 805M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:25:33.905 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 56s ago 6h 1029M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:25:33.905 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 56s ago 6h 898M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:25:33.905 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 56s ago 6h 1089M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:25:33.905 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 56s ago 6h 152M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:25:34.259 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:25:34.259 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:25:34.260 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:25:34.260 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:25:34.260 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:25:34.260 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:25:34.261 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:25:34.261 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:25:34.261 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:25:34.261 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:25:34.261 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:25:34.262 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:25:34.262 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:25:34.262 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:25:34.262 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:25:34.263 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:25:34.263 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:25:34.263 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:25:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:25:34 smithi153 ceph-mon[70044]: from='client.56530 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:25:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:25:34 smithi153 ceph-mon[70044]: from='client.56536 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:25:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:25:34 smithi153 ceph-mon[70044]: pgmap v10719: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:34.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:25:34 smithi083 ceph-mon[79930]: from='client.56530 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:25:34.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:25:34 smithi083 ceph-mon[79930]: from='client.56536 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:25:34.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:25:34 smithi083 ceph-mon[79930]: pgmap v10719: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:34.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:25:34 smithi083 ceph-mon[77242]: from='client.56530 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:25:34.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:25:34 smithi083 ceph-mon[77242]: from='client.56536 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:25:34.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:25:34 smithi083 ceph-mon[77242]: pgmap v10719: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:35.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:25:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:25:34.854Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:25:35.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:25:35 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1787983863' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:25:35.577 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:25:35 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1787983863' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:25:35.578 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:25:35 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1787983863' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:25:35.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:25:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:25:35.575Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:25:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:25:36 smithi153 ceph-mon[70044]: pgmap v10720: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:36.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:25:36 smithi083 ceph-mon[77242]: pgmap v10720: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:36.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:25:36 smithi083 ceph-mon[79930]: pgmap v10720: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:38.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:25:37 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:25:38.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:25:37 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:25:38.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:25:37 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:25:38.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:25:37 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:25:38.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:25:37 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:25:38.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:25:37 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:25:38.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:25:37 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:25:38.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:25:37 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:25:38.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:25:37 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:25:39.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:25:38 smithi153 ceph-mon[70044]: pgmap v10721: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:39.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:25:38 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:25:39.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:25:38 smithi083 ceph-mon[79930]: pgmap v10721: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:39.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:25:38 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:25:39.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:25:38 smithi083 ceph-mon[77242]: pgmap v10721: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:39.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:25:38 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:25:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:25:40 smithi083 ceph-mon[79930]: pgmap v10722: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:25:40 smithi083 ceph-mon[77242]: pgmap v10722: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:25:40 smithi153 ceph-mon[70044]: pgmap v10722: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:41.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:25:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:25:41] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T11:25:41.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:25:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:25:41.481Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:25:41.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:25:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:25:41.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:25:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:25:41.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:25:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:25:41.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:25:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:25:41.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:25:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:25:41.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:25:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:25:41.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:25:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:25:41.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:25:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:25:41.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:25:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:25:41.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:25:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:25:41.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:25:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:25:41.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:25:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:25:42.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:25:42 smithi083 ceph-mon[77242]: pgmap v10723: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:25:42 smithi083 ceph-mon[79930]: pgmap v10723: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:25:42 smithi153 ceph-mon[70044]: pgmap v10723: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:25:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:25:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:25:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:25:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:25:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:25:44.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:25:44 smithi153 ceph-mon[70044]: pgmap v10724: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:44.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:25:44 smithi083 ceph-mon[79930]: pgmap v10724: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:44.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:25:44 smithi083 ceph-mon[77242]: pgmap v10724: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:45.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:25:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:25:44.855Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:25:45.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:25:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:25:45.575Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:25:46.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:25:46 smithi153 ceph-mon[70044]: pgmap v10725: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:46.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:25:46 smithi083 ceph-mon[79930]: pgmap v10725: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:46.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:25:46 smithi083 ceph-mon[77242]: pgmap v10725: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:48.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:25:48 smithi153 ceph-mon[70044]: pgmap v10726: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:48.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:25:48 smithi083 ceph-mon[77242]: pgmap v10726: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:48.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:25:48 smithi083 ceph-mon[79930]: pgmap v10726: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:50.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:25:50 smithi083 ceph-mon[79930]: pgmap v10727: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:50.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:25:50 smithi083 ceph-mon[77242]: pgmap v10727: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:50.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:25:50 smithi153 ceph-mon[70044]: pgmap v10727: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:25:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:25:51] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T11:25:51.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:25:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:25:51.482Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:25:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:25:52 smithi153 ceph-mon[70044]: pgmap v10728: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:52.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:25:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:25:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:25:52.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:25:52 smithi083 ceph-mon[77242]: pgmap v10728: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:52.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:25:52 smithi083 ceph-mon[79930]: pgmap v10728: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:25:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:25:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:25:54.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:25:54 smithi153 ceph-mon[70044]: pgmap v10729: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:54.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:25:54 smithi083 ceph-mon[79930]: pgmap v10729: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:54.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:25:54 smithi083 ceph-mon[77242]: pgmap v10729: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:55.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:25:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:25:54.855Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:25:55.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:25:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:25:55.576Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:25:56.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:25:56 smithi153 ceph-mon[70044]: pgmap v10730: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:56.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:25:56 smithi083 ceph-mon[79930]: pgmap v10730: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:56.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:25:56 smithi083 ceph-mon[77242]: pgmap v10730: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:25:58 smithi153 ceph-mon[70044]: pgmap v10731: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:58.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:25:58 smithi083 ceph-mon[79930]: pgmap v10731: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:25:58.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:25:58 smithi083 ceph-mon[77242]: pgmap v10731: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:00.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:26:00 smithi083 ceph-mon[79930]: pgmap v10732: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:00.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:26:00 smithi083 ceph-mon[77242]: pgmap v10732: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:00.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:26:00 smithi153 ceph-mon[70044]: pgmap v10732: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:26:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:26:01] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T11:26:01.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:26:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:26:01.483Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:26:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:26:02 smithi153 ceph-mon[70044]: pgmap v10733: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:02.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:26:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:26:02.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:26:02 smithi083 ceph-mon[79930]: pgmap v10733: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:02.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:26:02 smithi083 ceph-mon[77242]: pgmap v10733: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:26:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:26:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:26:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:26:04 smithi153 ceph-mon[70044]: pgmap v10734: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:04.584 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:26:04.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:26:04 smithi083 ceph-mon[79930]: pgmap v10734: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:04.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:26:04 smithi083 ceph-mon[77242]: pgmap v10734: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:04.874 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:26:04.874 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (5h) 3m ago 6h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:26:04.875 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 87s ago 6h 63.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:26:04.875 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 3m ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:26:04.875 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 87s ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:26:04.875 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 87s ago 6h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:26:04.876 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 3m ago 6h 815M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:26:04.876 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 3m ago 6h 1344M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:26:04.876 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 87s ago 6h 578M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:26:04.876 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 3m ago 6h 774M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:26:04.877 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 3m ago 6h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:26:04.877 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 87s ago 6h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:26:04.877 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 3m ago 6h 947M 2429M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:26:04.877 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 3m ago 6h 936M 2429M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:26:04.877 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 3m ago 6h 730M 2429M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:26:04.878 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 3m ago 6h 914M 2429M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:26:04.878 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 87s ago 6h 805M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:26:04.878 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 87s ago 6h 1029M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:26:04.879 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 87s ago 6h 898M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:26:04.879 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 87s ago 6h 1089M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:26:04.879 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 87s ago 6h 152M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:26:05.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:26:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:26:04.855Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:26:05.228 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:26:05.228 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:26:05.229 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:26:05.229 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:26:05.229 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:26:05.229 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:26:05.229 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:26:05.230 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:26:05.230 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:26:05.230 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:26:05.231 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:26:05.231 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:26:05.231 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:26:05.231 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:26:05.232 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:26:05.232 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:26:05.232 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:26:05.233 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:26:05.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:26:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:26:05.576Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:26:05.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:26:05 smithi083 ceph-mon[77242]: from='client.56548 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:26:05.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:26:05 smithi083 ceph-mon[77242]: from='client.56554 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:26:05.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:26:05 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/675137694' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:26:05.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:26:05 smithi083 ceph-mon[79930]: from='client.56548 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:26:05.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:26:05 smithi083 ceph-mon[79930]: from='client.56554 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:26:05.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:26:05 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/675137694' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:26:05.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:26:05 smithi153 ceph-mon[70044]: from='client.56548 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:26:05.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:26:05 smithi153 ceph-mon[70044]: from='client.56554 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:26:05.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:26:05 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/675137694' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:26:06.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:26:06 smithi083 ceph-mon[79930]: pgmap v10735: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:06.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:26:06 smithi083 ceph-mon[77242]: pgmap v10735: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:06.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:26:06 smithi153 ceph-mon[70044]: pgmap v10735: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:26:08 smithi153 ceph-mon[70044]: pgmap v10736: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:08.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:26:08 smithi083 ceph-mon[79930]: pgmap v10736: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:08.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:26:08 smithi083 ceph-mon[77242]: pgmap v10736: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:10.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:26:10 smithi083 ceph-mon[79930]: pgmap v10737: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:10.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:26:10 smithi083 ceph-mon[77242]: pgmap v10737: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:10.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:26:10 smithi153 ceph-mon[70044]: pgmap v10737: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:26:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:26:11] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T11:26:11.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:26:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:26:11.484Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:26:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:26:12 smithi153 ceph-mon[70044]: pgmap v10738: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:12.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:26:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:26:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:26:12.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:26:12 smithi083 ceph-mon[79930]: pgmap v10738: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:12.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:26:12 smithi083 ceph-mon[77242]: pgmap v10738: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:26:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:26:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:26:14.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:26:14 smithi153 ceph-mon[70044]: pgmap v10739: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:14.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:26:14 smithi083 ceph-mon[79930]: pgmap v10739: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:14.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:26:14 smithi083 ceph-mon[77242]: pgmap v10739: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:15.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:26:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:26:14.857Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:26:15.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:26:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:26:15.577Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:26:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:26:16 smithi153 ceph-mon[70044]: pgmap v10740: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:16.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:26:16 smithi083 ceph-mon[79930]: pgmap v10740: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:16.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:26:16 smithi083 ceph-mon[77242]: pgmap v10740: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:26:18 smithi153 ceph-mon[70044]: pgmap v10741: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:18.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:26:18 smithi083 ceph-mon[79930]: pgmap v10741: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:18.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:26:18 smithi083 ceph-mon[77242]: pgmap v10741: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:20.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:26:20 smithi083 ceph-mon[79930]: pgmap v10742: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:20.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:26:20 smithi083 ceph-mon[77242]: pgmap v10742: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:20.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:26:20 smithi153 ceph-mon[70044]: pgmap v10742: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:26:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:26:21] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T11:26:21.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:26:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:26:21.484Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:26:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:26:22 smithi153 ceph-mon[70044]: pgmap v10743: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:22.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:26:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:26:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:26:22.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:26:22 smithi083 ceph-mon[77242]: pgmap v10743: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:22.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:26:22 smithi083 ceph-mon[79930]: pgmap v10743: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:26:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:26:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:26:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:26:24 smithi153 ceph-mon[70044]: pgmap v10744: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:24.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:26:24 smithi083 ceph-mon[79930]: pgmap v10744: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:24.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:26:24 smithi083 ceph-mon[77242]: pgmap v10744: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:25.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:26:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:26:24.857Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:26:25.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:26:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:26:25.577Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:26:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:26:26 smithi153 ceph-mon[70044]: pgmap v10745: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:26.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:26:26 smithi083 ceph-mon[79930]: pgmap v10745: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:26.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:26:26 smithi083 ceph-mon[77242]: pgmap v10745: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:26:28 smithi153 ceph-mon[70044]: pgmap v10746: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:28.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:26:28 smithi083 ceph-mon[79930]: pgmap v10746: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:28.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:26:28 smithi083 ceph-mon[77242]: pgmap v10746: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:30.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:26:30 smithi083 ceph-mon[79930]: pgmap v10747: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:30.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:26:30 smithi083 ceph-mon[77242]: pgmap v10747: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:30.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:26:30 smithi153 ceph-mon[70044]: pgmap v10747: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:26:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:26:31] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T11:26:31.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:26:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:26:31.484Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:26:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:26:32 smithi153 ceph-mon[70044]: pgmap v10748: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:32.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:26:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:26:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:26:32.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:26:32 smithi083 ceph-mon[79930]: pgmap v10748: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:32.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:26:32 smithi083 ceph-mon[77242]: pgmap v10748: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:26:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:26:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:26:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:26:34 smithi153 ceph-mon[70044]: pgmap v10749: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:34.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:26:34 smithi083 ceph-mon[79930]: pgmap v10749: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:34.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:26:34 smithi083 ceph-mon[77242]: pgmap v10749: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:35.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:26:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:26:34.858Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:26:35.553 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:26:35.842 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:26:35.843 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 4m ago 6h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:26:35.843 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 118s ago 6h 63.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:26:35.843 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 4m ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:26:35.844 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 118s ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:26:35.844 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 118s ago 6h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:26:35.844 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 4m ago 6h 815M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:26:35.844 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 4m ago 6h 1344M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:26:35.844 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 118s ago 6h 578M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:26:35.845 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 4m ago 6h 774M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:26:35.845 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 4m ago 6h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:26:35.845 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 118s ago 6h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:26:35.845 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 4m ago 6h 947M 2429M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:26:35.846 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 4m ago 6h 936M 2429M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:26:35.846 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 4m ago 6h 730M 2429M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:26:35.846 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 4m ago 6h 914M 2429M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:26:35.846 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 118s ago 6h 805M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:26:35.846 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 118s ago 6h 1029M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:26:35.847 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 118s ago 6h 898M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:26:35.847 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 118s ago 6h 1089M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:26:35.847 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 118s ago 6h 152M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:26:35.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:26:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:26:35.578Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:26:36.197 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:26:36.198 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:26:36.198 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:26:36.198 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:26:36.199 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:26:36.199 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:26:36.199 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:26:36.199 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:26:36.199 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:26:36.200 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:26:36.200 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:26:36.200 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:26:36.200 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:26:36.201 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:26:36.201 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:26:36.201 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:26:36.201 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:26:36.202 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:26:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:26:36 smithi153 ceph-mon[70044]: from='client.46635 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:26:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:26:36 smithi153 ceph-mon[70044]: from='client.56572 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:26:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:26:36 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1625204388' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:26:36.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:26:36 smithi153 ceph-mon[70044]: pgmap v10750: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:36.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:26:36 smithi083 ceph-mon[79930]: from='client.46635 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:26:36.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:26:36 smithi083 ceph-mon[79930]: from='client.56572 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:26:36.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:26:36 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1625204388' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:26:36.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:26:36 smithi083 ceph-mon[79930]: pgmap v10750: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:36.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:26:36 smithi083 ceph-mon[77242]: from='client.46635 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:26:36.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:26:36 smithi083 ceph-mon[77242]: from='client.56572 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:26:36.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:26:36 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1625204388' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:26:36.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:26:36 smithi083 ceph-mon[77242]: pgmap v10750: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:26:38 smithi153 ceph-mon[70044]: pgmap v10751: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:38.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:26:38 smithi083 ceph-mon[77242]: pgmap v10751: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:38.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:26:38 smithi083 ceph-mon[79930]: pgmap v10751: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:39.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:26:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:26:39.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:26:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:26:39.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:26:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:26:39.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:26:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:26:39.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:26:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:26:39.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:26:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:26:39.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:26:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:26:39.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:26:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:26:39.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:26:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:26:39.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:26:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:26:39.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:26:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:26:39.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:26:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:26:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:26:40 smithi083 ceph-mon[77242]: pgmap v10752: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:26:40 smithi083 ceph-mon[79930]: pgmap v10752: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:26:40 smithi153 ceph-mon[70044]: pgmap v10752: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:41.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:26:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:26:41] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T11:26:41.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:26:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:26:41.486Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:26:41.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:26:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:26:41.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:26:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:26:41.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:26:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:26:41.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:26:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:26:41.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:26:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:26:41.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:26:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:26:41.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:26:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:26:41.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:26:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:26:41.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:26:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:26:41.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:26:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:26:41.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:26:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:26:41.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:26:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:26:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:26:42 smithi083 ceph-mon[79930]: pgmap v10753: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:42.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:26:42 smithi083 ceph-mon[77242]: pgmap v10753: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:26:42 smithi153 ceph-mon[70044]: pgmap v10753: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:26:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:26:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:26:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:26:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:26:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:26:44.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:26:44 smithi153 ceph-mon[70044]: pgmap v10754: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:44.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:26:44 smithi083 ceph-mon[79930]: pgmap v10754: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:44.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:26:44 smithi083 ceph-mon[77242]: pgmap v10754: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:45.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:26:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:26:44.858Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:26:45.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:26:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:26:45.579Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:26:46.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:26:46 smithi153 ceph-mon[70044]: pgmap v10755: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:46.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:26:46 smithi083 ceph-mon[79930]: pgmap v10755: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:46.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:26:46 smithi083 ceph-mon[77242]: pgmap v10755: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:48.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:26:48 smithi153 ceph-mon[70044]: pgmap v10756: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:48.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:26:48 smithi083 ceph-mon[79930]: pgmap v10756: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:48.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:26:48 smithi083 ceph-mon[77242]: pgmap v10756: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:50.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:26:50 smithi083 ceph-mon[79930]: pgmap v10757: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:50.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:26:50 smithi083 ceph-mon[77242]: pgmap v10757: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:50.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:26:50 smithi153 ceph-mon[70044]: pgmap v10757: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:26:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:26:51] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T11:26:51.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:26:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:26:51.486Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:26:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:26:52 smithi153 ceph-mon[70044]: pgmap v10758: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:52.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:26:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:26:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:26:52.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:26:52 smithi083 ceph-mon[79930]: pgmap v10758: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:52.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:26:52 smithi083 ceph-mon[77242]: pgmap v10758: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:26:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:26:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:26:54.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:26:54 smithi153 ceph-mon[70044]: pgmap v10759: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:54.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:26:54 smithi083 ceph-mon[77242]: pgmap v10759: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:54.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:26:54 smithi083 ceph-mon[79930]: pgmap v10759: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:55.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:26:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:26:54.860Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:26:55.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:26:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:26:55.580Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:26:56.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:26:56 smithi153 ceph-mon[70044]: pgmap v10760: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:56.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:26:56 smithi083 ceph-mon[79930]: pgmap v10760: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:56.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:26:56 smithi083 ceph-mon[77242]: pgmap v10760: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:26:58 smithi153 ceph-mon[70044]: pgmap v10761: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:58.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:26:58 smithi083 ceph-mon[79930]: pgmap v10761: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:26:58.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:26:58 smithi083 ceph-mon[77242]: pgmap v10761: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:00.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:27:00 smithi083 ceph-mon[77242]: pgmap v10762: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:00.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:27:00 smithi083 ceph-mon[79930]: pgmap v10762: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:00.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:27:00 smithi153 ceph-mon[70044]: pgmap v10762: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:27:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:27:01] "GET /metrics HTTP/1.1" 200 225229 "" "Prometheus/2.33.4" 2022-09-29T11:27:01.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:27:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:27:01.487Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:27:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:27:02 smithi153 ceph-mon[70044]: pgmap v10763: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:02.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:27:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:27:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:27:02.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:27:02 smithi083 ceph-mon[79930]: pgmap v10763: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:02.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:27:02 smithi083 ceph-mon[77242]: pgmap v10763: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:27:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:27:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:27:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:27:04 smithi153 ceph-mon[70044]: pgmap v10764: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:04.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:27:04 smithi083 ceph-mon[79930]: pgmap v10764: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:04.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:27:04 smithi083 ceph-mon[77242]: pgmap v10764: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:05.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:27:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:27:04.860Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:27:05.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:27:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:27:05.581Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:27:06.525 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:27:06.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:27:06 smithi153 ceph-mon[70044]: pgmap v10765: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:06.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:27:06 smithi083 ceph-mon[79930]: pgmap v10765: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:06.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:27:06 smithi083 ceph-mon[77242]: pgmap v10765: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:06.812 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:27:06.812 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 4m ago 6h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:27:06.812 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 2m ago 6h 63.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:27:06.813 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 4m ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:27:06.813 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 2m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:27:06.813 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 2m ago 6h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:27:06.813 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 4m ago 6h 815M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:27:06.814 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 4m ago 6h 1344M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:27:06.814 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 2m ago 6h 578M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:27:06.814 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 4m ago 6h 774M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:27:06.814 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 4m ago 6h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:27:06.815 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 2m ago 6h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:27:06.815 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 4m ago 6h 947M 2429M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:27:06.815 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 4m ago 6h 936M 2429M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:27:06.815 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 4m ago 6h 730M 2429M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:27:06.815 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 4m ago 6h 914M 2429M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:27:06.816 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 2m ago 6h 805M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:27:06.816 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 2m ago 6h 1029M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:27:06.816 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 2m ago 6h 898M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:27:06.816 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 2m ago 6h 1089M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:27:06.817 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 2m ago 6h 152M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:27:07.166 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:27:07.166 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:27:07.167 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:27:07.167 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:27:07.167 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:27:07.167 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:27:07.168 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:27:07.168 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:27:07.168 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:27:07.168 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:27:07.169 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:27:07.169 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:27:07.169 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:27:07.169 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:27:07.170 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:27:07.170 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:27:07.170 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:27:07.170 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:27:07.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:27:07 smithi153 ceph-mon[70044]: from='client.56584 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:27:07.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:27:07 smithi153 ceph-mon[70044]: from='client.46659 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:27:07.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:27:07 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3422301940' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:27:07.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:27:07 smithi083 ceph-mon[79930]: from='client.56584 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:27:07.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:27:07 smithi083 ceph-mon[79930]: from='client.46659 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:27:07.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:27:07 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3422301940' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:27:07.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:27:07 smithi083 ceph-mon[77242]: from='client.56584 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:27:07.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:27:07 smithi083 ceph-mon[77242]: from='client.46659 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:27:07.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:27:07 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3422301940' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:27:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:27:08 smithi153 ceph-mon[70044]: pgmap v10766: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:08.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:27:08 smithi083 ceph-mon[79930]: pgmap v10766: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:08.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:27:08 smithi083 ceph-mon[77242]: pgmap v10766: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:10.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:27:10 smithi083 ceph-mon[77242]: pgmap v10767: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:10.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:27:10 smithi083 ceph-mon[79930]: pgmap v10767: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:10.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:27:10 smithi153 ceph-mon[70044]: pgmap v10767: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:27:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:27:11] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T11:27:11.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:27:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:27:11.487Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:27:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:27:12 smithi153 ceph-mon[70044]: pgmap v10768: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:12.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:27:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:27:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:27:12.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:27:12 smithi083 ceph-mon[77242]: pgmap v10768: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:12.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:27:12 smithi083 ceph-mon[79930]: pgmap v10768: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:27:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:27:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:27:14.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:27:14 smithi153 ceph-mon[70044]: pgmap v10769: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:14.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:27:14 smithi083 ceph-mon[79930]: pgmap v10769: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:14.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:27:14 smithi083 ceph-mon[77242]: pgmap v10769: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:15.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:27:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:27:14.861Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:27:15.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:27:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:27:15.582Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:27:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:27:16 smithi153 ceph-mon[70044]: pgmap v10770: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:16.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:27:16 smithi083 ceph-mon[77242]: pgmap v10770: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:16.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:27:16 smithi083 ceph-mon[79930]: pgmap v10770: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:27:18 smithi153 ceph-mon[70044]: pgmap v10771: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:18.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:27:18 smithi083 ceph-mon[79930]: pgmap v10771: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:18.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:27:18 smithi083 ceph-mon[77242]: pgmap v10771: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:20.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:27:20 smithi083 ceph-mon[79930]: pgmap v10772: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:20.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:27:20 smithi083 ceph-mon[77242]: pgmap v10772: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:20.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:27:20 smithi153 ceph-mon[70044]: pgmap v10772: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:27:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:27:21] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T11:27:21.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:27:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:27:21.488Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:27:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:27:22 smithi153 ceph-mon[70044]: pgmap v10773: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:22.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:27:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:27:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:27:22.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:27:22 smithi083 ceph-mon[79930]: pgmap v10773: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:22.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:27:22 smithi083 ceph-mon[77242]: pgmap v10773: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:27:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:27:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:27:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:27:24 smithi153 ceph-mon[70044]: pgmap v10774: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:24.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:27:24 smithi083 ceph-mon[79930]: pgmap v10774: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:24.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:27:24 smithi083 ceph-mon[77242]: pgmap v10774: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:25.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:27:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:27:24.861Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:27:25.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:27:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:27:25.583Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:27:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:27:26 smithi153 ceph-mon[70044]: pgmap v10775: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:26.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:27:26 smithi083 ceph-mon[77242]: pgmap v10775: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:26.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:27:26 smithi083 ceph-mon[79930]: pgmap v10775: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:27:28 smithi153 ceph-mon[70044]: pgmap v10776: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:28.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:27:28 smithi083 ceph-mon[79930]: pgmap v10776: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:28.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:27:28 smithi083 ceph-mon[77242]: pgmap v10776: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:30.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:27:30 smithi083 ceph-mon[79930]: pgmap v10777: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:30.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:27:30 smithi083 ceph-mon[77242]: pgmap v10777: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:30.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:27:30 smithi153 ceph-mon[70044]: pgmap v10777: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:27:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:27:31] "GET /metrics HTTP/1.1" 200 225229 "" "Prometheus/2.33.4" 2022-09-29T11:27:31.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:27:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:27:31.489Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:27:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:27:32 smithi153 ceph-mon[70044]: pgmap v10778: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:32.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:27:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:27:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:27:32.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:27:32 smithi083 ceph-mon[79930]: pgmap v10778: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:32.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:27:32 smithi083 ceph-mon[77242]: pgmap v10778: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:27:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:27:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:27:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:27:34 smithi153 ceph-mon[70044]: pgmap v10779: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:34.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:27:34 smithi083 ceph-mon[79930]: pgmap v10779: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:34.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:27:34 smithi083 ceph-mon[77242]: pgmap v10779: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:35.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:27:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:27:34.861Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:27:35.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:27:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:27:35.584Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:27:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:27:36 smithi153 ceph-mon[70044]: pgmap v10780: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:36.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:27:36 smithi083 ceph-mon[79930]: pgmap v10780: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:36.732 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:27:36 smithi083 ceph-mon[77242]: pgmap v10780: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:37.490 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:27:37.776 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:27:37.777 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 5m ago 6h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:27:37.777 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 3m ago 6h 63.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:27:37.777 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 5m ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:27:37.778 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 3m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:27:37.778 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 3m ago 6h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:27:37.778 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 5m ago 6h 815M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:27:37.778 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 5m ago 6h 1344M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:27:37.779 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 3m ago 6h 578M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:27:37.779 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 5m ago 6h 774M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:27:37.779 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 5m ago 6h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:27:37.779 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 3m ago 6h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:27:37.779 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 5m ago 6h 947M 2429M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:27:37.780 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 5m ago 6h 936M 2429M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:27:37.780 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 5m ago 6h 730M 2429M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:27:37.780 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 5m ago 6h 914M 2429M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:27:37.780 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 3m ago 6h 805M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:27:37.780 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 3m ago 6h 1029M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:27:37.781 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 3m ago 6h 898M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:27:37.781 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 3m ago 6h 1089M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:27:37.781 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 3m ago 6h 152M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:27:38.133 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:27:38.134 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:27:38.134 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:27:38.134 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:27:38.134 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:27:38.135 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:27:38.135 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:27:38.135 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:27:38.135 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:27:38.136 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:27:38.136 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:27:38.136 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:27:38.136 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:27:38.136 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:27:38.137 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:27:38.137 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:27:38.137 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:27:38.137 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:27:38.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:27:38 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1478350350' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:27:38.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:27:38 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1478350350' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:27:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:27:38 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1478350350' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:27:39.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:27:39 smithi083 ceph-mon[79930]: from='client.46671 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:27:39.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:27:39 smithi083 ceph-mon[79930]: from='client.46677 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:27:39.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:27:39 smithi083 ceph-mon[79930]: pgmap v10781: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:39.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:27:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:27:39.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:27:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:27:39.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:27:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:27:39.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:27:39 smithi083 ceph-mon[77242]: from='client.46671 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:27:39.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:27:39 smithi083 ceph-mon[77242]: from='client.46677 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:27:39.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:27:39 smithi083 ceph-mon[77242]: pgmap v10781: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:39.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:27:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:27:39.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:27:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:27:39.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:27:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:27:39.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:27:39 smithi153 ceph-mon[70044]: from='client.46671 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:27:39.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:27:39 smithi153 ceph-mon[70044]: from='client.46677 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:27:39.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:27:39 smithi153 ceph-mon[70044]: pgmap v10781: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:39.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:27:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:27:39.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:27:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:27:39.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:27:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:27:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:27:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:27:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:27:40 smithi083 ceph-mon[79930]: pgmap v10782: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:27:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:27:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:27:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:27:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:27:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:27:40.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:27:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:27:40.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:27:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:27:40.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:27:40 smithi083 ceph-mon[77242]: pgmap v10782: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:40.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:27:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:27:40.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:27:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:27:40.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:27:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:27:40.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:27:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:27:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:27:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:27:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:27:40 smithi153 ceph-mon[70044]: pgmap v10782: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:27:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:27:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:27:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:27:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:27:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:27:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:27:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:27:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:27:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:27:41] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T11:27:41.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:27:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:27:41.490Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:27:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:27:42 smithi153 ceph-mon[70044]: pgmap v10783: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:42.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:27:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:27:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:27:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:27:42 smithi083 ceph-mon[79930]: pgmap v10783: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:42.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:27:42 smithi083 ceph-mon[77242]: pgmap v10783: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:27:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:27:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:27:44.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:27:44 smithi153 ceph-mon[70044]: pgmap v10784: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:44.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:27:44 smithi083 ceph-mon[79930]: pgmap v10784: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:44.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:27:44 smithi083 ceph-mon[77242]: pgmap v10784: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:45.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:27:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:27:44.862Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:27:45.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:27:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:27:45.585Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:27:47.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:27:47 smithi153 ceph-mon[70044]: pgmap v10785: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:47.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:27:47 smithi083 ceph-mon[79930]: pgmap v10785: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:47.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:27:47 smithi083 ceph-mon[77242]: pgmap v10785: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:49.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:27:49 smithi153 ceph-mon[70044]: pgmap v10786: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:49.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:27:49 smithi083 ceph-mon[77242]: pgmap v10786: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:49.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:27:49 smithi083 ceph-mon[79930]: pgmap v10786: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:51.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:27:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:27:51] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T11:27:51.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:27:51 smithi083 ceph-mon[79930]: pgmap v10787: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:51.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:27:51 smithi083 ceph-mon[77242]: pgmap v10787: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:51.714 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:27:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:27:51.491Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:27:51.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:27:51 smithi153 ceph-mon[70044]: pgmap v10787: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:27:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:27:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:27:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:27:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:27:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:27:53.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:27:53 smithi083 ceph-mon[79930]: pgmap v10788: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:53.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:27:53 smithi083 ceph-mon[77242]: pgmap v10788: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:53.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:27:53 smithi153 ceph-mon[70044]: pgmap v10788: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:55.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:27:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:27:54.862Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:27:55.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:27:55 smithi083 ceph-mon[79930]: pgmap v10789: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:55.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:27:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:27:55.586Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:27:55.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:27:55 smithi083 ceph-mon[77242]: pgmap v10789: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:55.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:27:55 smithi153 ceph-mon[70044]: pgmap v10789: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:57.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:27:57 smithi083 ceph-mon[79930]: pgmap v10790: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:57.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:27:57 smithi083 ceph-mon[77242]: pgmap v10790: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:57.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:27:57 smithi153 ceph-mon[70044]: pgmap v10790: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:59.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:27:59 smithi083 ceph-mon[79930]: pgmap v10791: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:59.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:27:59 smithi083 ceph-mon[77242]: pgmap v10791: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:27:59.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:27:59 smithi153 ceph-mon[70044]: pgmap v10791: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:01.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:28:01 smithi083 ceph-mon[77242]: pgmap v10792: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:01.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:28:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:28:01] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T11:28:01.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:28:01 smithi083 ceph-mon[79930]: pgmap v10792: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:01.714 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:28:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:28:01.491Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:28:01.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:28:01 smithi153 ceph-mon[70044]: pgmap v10792: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:28:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:28:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:28:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:28:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:28:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:28:03.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:28:03 smithi083 ceph-mon[79930]: pgmap v10793: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:03.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:28:03 smithi083 ceph-mon[77242]: pgmap v10793: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:03.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:28:03 smithi153 ceph-mon[70044]: pgmap v10793: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:05.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:28:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:28:04.863Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:28:05.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:28:05 smithi083 ceph-mon[77242]: pgmap v10794: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:05.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:28:05 smithi083 ceph-mon[79930]: pgmap v10794: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:05.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:28:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:28:05.587Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:28:05.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:28:05 smithi153 ceph-mon[70044]: pgmap v10794: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:07.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:28:07 smithi083 ceph-mon[79930]: pgmap v10795: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:07.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:28:07 smithi083 ceph-mon[77242]: pgmap v10795: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:07.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:28:07 smithi153 ceph-mon[70044]: pgmap v10795: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:08.461 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:28:08.750 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:28:08.750 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 5m ago 6h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:28:08.750 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 3m ago 6h 63.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:28:08.751 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 5m ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:28:08.751 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 3m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:28:08.751 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 3m ago 6h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:28:08.751 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 5m ago 6h 815M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:28:08.752 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 5m ago 6h 1344M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:28:08.752 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 3m ago 6h 578M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:28:08.752 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 5m ago 6h 774M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:28:08.752 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 5m ago 6h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:28:08.752 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 3m ago 6h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:28:08.753 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 5m ago 6h 947M 2429M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:28:08.753 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 5m ago 6h 936M 2429M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:28:08.753 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 5m ago 6h 730M 2429M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:28:08.753 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 5m ago 6h 914M 2429M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:28:08.754 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 3m ago 6h 805M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:28:08.754 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 3m ago 6h 1029M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:28:08.754 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 3m ago 6h 898M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:28:08.754 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 3m ago 6h 1089M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:28:08.755 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 3m ago 6h 152M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:28:09.100 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:28:09.101 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:28:09.101 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:28:09.101 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:28:09.101 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:28:09.102 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:28:09.102 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:28:09.102 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:28:09.102 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:28:09.103 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:28:09.103 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:28:09.103 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:28:09.103 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:28:09.104 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:28:09.104 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:28:09.104 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:28:09.104 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:28:09.104 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:28:09.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:28:09 smithi083 ceph-mon[79930]: pgmap v10796: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:09.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:28:09 smithi083 ceph-mon[79930]: from='client.56620 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:28:09.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:28:09 smithi083 ceph-mon[79930]: from='client.56626 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:28:09.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:28:09 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/155242548' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:28:09.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:28:09 smithi083 ceph-mon[77242]: pgmap v10796: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:09.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:28:09 smithi083 ceph-mon[77242]: from='client.56620 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:28:09.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:28:09 smithi083 ceph-mon[77242]: from='client.56626 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:28:09.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:28:09 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/155242548' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:28:09.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:28:09 smithi153 ceph-mon[70044]: pgmap v10796: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:09.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:28:09 smithi153 ceph-mon[70044]: from='client.56620 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:28:09.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:28:09 smithi153 ceph-mon[70044]: from='client.56626 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:28:09.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:28:09 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/155242548' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:28:11.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:28:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:28:11] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T11:28:11.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:28:11 smithi083 ceph-mon[79930]: pgmap v10797: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:11.714 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:28:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:28:11.492Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:28:11.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:28:11 smithi083 ceph-mon[77242]: pgmap v10797: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:11.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:28:11 smithi153 ceph-mon[70044]: pgmap v10797: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:28:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:28:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:28:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:28:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:28:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:28:13.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:28:13 smithi083 ceph-mon[77242]: pgmap v10798: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:13.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:28:13 smithi083 ceph-mon[79930]: pgmap v10798: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:13.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:28:13 smithi153 ceph-mon[70044]: pgmap v10798: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:15.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:28:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:28:14.863Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:28:15.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:28:15 smithi083 ceph-mon[79930]: pgmap v10799: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:15.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:28:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:28:15.586Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:28:15.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:28:15 smithi083 ceph-mon[77242]: pgmap v10799: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:15.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:28:15 smithi153 ceph-mon[70044]: pgmap v10799: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:17.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:28:17 smithi083 ceph-mon[79930]: pgmap v10800: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:17.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:28:17 smithi083 ceph-mon[77242]: pgmap v10800: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:17.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:28:17 smithi153 ceph-mon[70044]: pgmap v10800: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:19.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:28:19 smithi083 ceph-mon[79930]: pgmap v10801: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:19.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:28:19 smithi083 ceph-mon[77242]: pgmap v10801: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:19.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:28:19 smithi153 ceph-mon[70044]: pgmap v10801: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:21.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:28:21 smithi153 ceph-mon[70044]: pgmap v10802: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:28:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:28:21] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T11:28:21.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:28:21 smithi083 ceph-mon[79930]: pgmap v10802: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:21.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:28:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:28:21.492Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:28:21.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:28:21 smithi083 ceph-mon[77242]: pgmap v10802: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:28:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:28:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:28:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:28:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:28:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:28:23.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:28:23 smithi153 ceph-mon[70044]: pgmap v10803: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:23.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:28:23 smithi083 ceph-mon[79930]: pgmap v10803: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:23.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:28:23 smithi083 ceph-mon[77242]: pgmap v10803: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:25.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:28:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:28:24.863Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:28:25.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:28:25 smithi153 ceph-mon[70044]: pgmap v10804: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:25.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:28:25 smithi083 ceph-mon[77242]: pgmap v10804: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:25.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:28:25 smithi083 ceph-mon[79930]: pgmap v10804: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:25.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:28:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:28:25.587Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:28:27.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:28:27 smithi153 ceph-mon[70044]: pgmap v10805: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:27.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:28:27 smithi083 ceph-mon[79930]: pgmap v10805: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:27.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:28:27 smithi083 ceph-mon[77242]: pgmap v10805: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:29.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:28:29 smithi153 ceph-mon[70044]: pgmap v10806: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:29.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:28:29 smithi083 ceph-mon[79930]: pgmap v10806: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:29.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:28:29 smithi083 ceph-mon[77242]: pgmap v10806: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:31.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:28:31 smithi153 ceph-mon[70044]: pgmap v10807: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:28:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:28:31] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T11:28:31.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:28:31 smithi083 ceph-mon[79930]: pgmap v10807: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:31.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:28:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:28:31.494Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:28:31.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:28:31 smithi083 ceph-mon[77242]: pgmap v10807: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:28:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:28:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:28:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:28:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:28:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:28:33.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:28:33 smithi153 ceph-mon[70044]: pgmap v10808: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:33.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:28:33 smithi083 ceph-mon[77242]: pgmap v10808: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:33.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:28:33 smithi083 ceph-mon[79930]: pgmap v10808: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:35.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:28:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:28:34.865Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:28:35.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:28:35 smithi153 ceph-mon[70044]: pgmap v10809: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:35.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:28:35 smithi083 ceph-mon[77242]: pgmap v10809: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:35.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:28:35 smithi083 ceph-mon[79930]: pgmap v10809: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:35.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:28:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:28:35.587Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:28:37.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:28:37 smithi153 ceph-mon[70044]: pgmap v10810: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:37.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:28:37 smithi083 ceph-mon[77242]: pgmap v10810: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:37.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:28:37 smithi083 ceph-mon[79930]: pgmap v10810: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:39.431 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:28:39.724 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:28:39.725 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 6m ago 6h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:28:39.725 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 4m ago 6h 63.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:28:39.725 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 6m ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:28:39.726 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 4m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:28:39.726 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 4m ago 6h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:28:39.726 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (5h) 6m ago 6h 815M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:28:39.726 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 6m ago 6h 1344M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:28:39.727 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 4m ago 6h 578M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:28:39.727 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 6m ago 6h 774M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:28:39.727 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 6m ago 6h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:28:39.727 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 4m ago 6h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:28:39.728 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 6m ago 6h 947M 2429M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:28:39.728 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 6m ago 6h 936M 2429M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:28:39.728 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 6m ago 6h 730M 2429M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:28:39.728 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 6m ago 6h 914M 2429M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:28:39.729 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 4m ago 6h 805M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:28:39.729 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 4m ago 6h 1029M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:28:39.729 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 4m ago 6h 898M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:28:39.729 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 4m ago 6h 1089M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:28:39.730 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 4m ago 6h 152M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:28:39.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:28:39 smithi153 ceph-mon[70044]: pgmap v10811: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:28:39 smithi083 ceph-mon[79930]: pgmap v10811: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:39.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:28:39 smithi083 ceph-mon[77242]: pgmap v10811: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:40.107 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:28:40.108 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:28:40.108 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:28:40.108 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:28:40.108 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:28:40.108 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:28:40.109 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:28:40.109 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:28:40.109 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:28:40.109 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:28:40.110 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:28:40.110 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:28:40.110 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:28:40.110 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:28:40.111 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:28:40.111 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:28:40.111 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:28:40.111 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:28:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:28:40 smithi153 ceph-mon[70044]: from='client.46707 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:28:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:28:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:28:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:28:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:28:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:28:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:28:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:28:40 smithi153 ceph-mon[70044]: from='client.56644 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:28:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:28:40 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2254823562' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:28:40.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:28:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:28:40.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:28:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:28:40.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:28:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:28:40.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:28:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:28:40.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:28:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:28:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:28:40 smithi083 ceph-mon[79930]: from='client.46707 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:28:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:28:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:28:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:28:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:28:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:28:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:28:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:28:40 smithi083 ceph-mon[79930]: from='client.56644 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:28:40.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:28:40 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2254823562' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:28:40.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:28:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:28:40.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:28:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:28:40.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:28:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:28:40.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:28:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:28:40.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:28:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:28:40.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:28:40 smithi083 ceph-mon[77242]: from='client.46707 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:28:40.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:28:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:28:40.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:28:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:28:40.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:28:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:28:40.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:28:40 smithi083 ceph-mon[77242]: from='client.56644 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:28:40.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:28:40 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2254823562' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:28:40.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:28:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:28:40.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:28:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:28:40.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:28:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:28:40.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:28:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:28:40.968 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:28:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:28:41.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:28:41 smithi153 ceph-mon[70044]: pgmap v10812: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:28:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:28:41] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T11:28:41.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:28:41 smithi083 ceph-mon[79930]: pgmap v10812: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:41.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:28:41 smithi083 ceph-mon[77242]: pgmap v10812: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:41.964 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:28:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:28:41.494Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:28:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:28:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:28:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:28:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:28:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:28:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:28:43.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:28:43 smithi153 ceph-mon[70044]: pgmap v10813: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:43.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:28:43 smithi083 ceph-mon[79930]: pgmap v10813: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:43.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:28:43 smithi083 ceph-mon[77242]: pgmap v10813: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:45.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:28:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:28:44.865Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:28:45.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:28:45 smithi153 ceph-mon[70044]: pgmap v10814: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:45.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:28:45 smithi083 ceph-mon[79930]: pgmap v10814: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:45.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:28:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:28:45.588Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:28:45.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:28:45 smithi083 ceph-mon[77242]: pgmap v10814: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:47.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:28:47 smithi153 ceph-mon[70044]: pgmap v10815: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:47.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:28:47 smithi083 ceph-mon[79930]: pgmap v10815: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:47.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:28:47 smithi083 ceph-mon[77242]: pgmap v10815: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:49.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:28:49 smithi153 ceph-mon[70044]: pgmap v10816: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:49.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:28:49 smithi083 ceph-mon[77242]: pgmap v10816: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:49.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:28:49 smithi083 ceph-mon[79930]: pgmap v10816: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:51.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:28:51 smithi153 ceph-mon[70044]: pgmap v10817: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:28:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:28:51] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T11:28:51.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:28:51 smithi083 ceph-mon[79930]: pgmap v10817: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:51.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:28:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:28:51.495Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:28:51.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:28:51 smithi083 ceph-mon[77242]: pgmap v10817: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:28:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:28:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:28:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:28:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:28:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:28:53.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:28:53 smithi153 ceph-mon[70044]: pgmap v10818: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:53.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:28:53 smithi083 ceph-mon[79930]: pgmap v10818: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:53.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:28:53 smithi083 ceph-mon[77242]: pgmap v10818: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:55.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:28:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:28:54.866Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:28:55.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:28:55 smithi153 ceph-mon[70044]: pgmap v10819: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:55.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:28:55 smithi083 ceph-mon[79930]: pgmap v10819: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:55.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:28:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:28:55.588Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:28:55.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:28:55 smithi083 ceph-mon[77242]: pgmap v10819: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:57.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:28:57 smithi153 ceph-mon[70044]: pgmap v10820: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:57.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:28:57 smithi083 ceph-mon[79930]: pgmap v10820: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:57.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:28:57 smithi083 ceph-mon[77242]: pgmap v10820: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:59.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:28:59 smithi153 ceph-mon[70044]: pgmap v10821: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:59.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:28:59 smithi083 ceph-mon[77242]: pgmap v10821: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:28:59.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:28:59 smithi083 ceph-mon[79930]: pgmap v10821: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:29:01.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:29:01 smithi153 ceph-mon[70044]: pgmap v10822: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:29:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:29:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:29:01] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T11:29:01.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:29:01 smithi083 ceph-mon[79930]: pgmap v10822: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:29:01.964 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:29:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:29:01.495Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:29:01.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:29:01 smithi083 ceph-mon[77242]: pgmap v10822: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:29:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:29:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:29:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:29:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:29:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:29:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:29:03.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:29:03 smithi153 ceph-mon[70044]: pgmap v10823: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:29:03.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:29:03 smithi083 ceph-mon[79930]: pgmap v10823: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:29:03.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:29:03 smithi083 ceph-mon[77242]: pgmap v10823: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:29:05.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:29:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:29:04.866Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:29:05.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:29:05 smithi153 ceph-mon[70044]: pgmap v10824: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:29:05.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:29:05 smithi083 ceph-mon[79930]: pgmap v10824: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:29:05.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:29:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:29:05.589Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:29:05.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:29:05 smithi083 ceph-mon[77242]: pgmap v10824: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:29:07.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:29:07 smithi153 ceph-mon[70044]: pgmap v10825: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:29:07.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:29:07 smithi083 ceph-mon[77242]: pgmap v10825: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:29:07.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:29:07 smithi083 ceph-mon[79930]: pgmap v10825: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:29:09.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:29:09 smithi153 ceph-mon[70044]: pgmap v10826: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:29:09.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:29:09 smithi083 ceph-mon[79930]: pgmap v10826: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:29:09.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:29:09 smithi083 ceph-mon[77242]: pgmap v10826: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:29:10.437 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:29:10.733 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:29:10.734 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 6m ago 6h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:29:10.734 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 4m ago 6h 63.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:29:10.734 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 6m ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:29:10.735 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 4m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:29:10.735 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (5h) 4m ago 6h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:29:10.735 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 6m ago 6h 815M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:29:10.735 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (5h) 6m ago 6h 1344M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:29:10.736 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 4m ago 6h 578M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:29:10.736 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 6m ago 6h 774M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:29:10.736 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 6m ago 6h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:29:10.736 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 4m ago 6h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:29:10.736 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 6m ago 6h 947M 2429M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:29:10.737 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 6m ago 6h 936M 2429M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:29:10.737 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 6m ago 6h 730M 2429M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:29:10.737 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 6m ago 6h 914M 2429M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:29:10.737 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 4m ago 6h 805M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:29:10.738 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 4m ago 6h 1029M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:29:10.738 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 4m ago 6h 898M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:29:10.738 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 4m ago 6h 1089M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:29:10.738 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (5h) 4m ago 6h 152M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:29:11.089 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:29:11.089 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:29:11.090 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:29:11.090 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:29:11.090 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:29:11.090 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:29:11.091 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:29:11.091 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:29:11.091 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:29:11.091 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:29:11.092 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:29:11.092 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:29:11.092 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:29:11.092 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:29:11.092 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:29:11.093 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:29:11.093 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:29:11.093 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:29:11.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:29:11 smithi153 ceph-mon[70044]: pgmap v10827: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:29:11.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:29:11 smithi153 ceph-mon[70044]: from='client.56656 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:29:11.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:29:11 smithi153 ceph-mon[70044]: from='client.46731 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:29:11.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:29:11 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1215872626' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:29:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:29:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:29:11] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T11:29:11.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:29:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:29:11.495Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:29:11.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:29:11 smithi083 ceph-mon[77242]: pgmap v10827: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:29:11.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:29:11 smithi083 ceph-mon[77242]: from='client.56656 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:29:11.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:29:11 smithi083 ceph-mon[77242]: from='client.46731 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:29:11.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:29:11 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1215872626' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:29:11.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:29:11 smithi083 ceph-mon[79930]: pgmap v10827: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:29:11.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:29:11 smithi083 ceph-mon[79930]: from='client.56656 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:29:11.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:29:11 smithi083 ceph-mon[79930]: from='client.46731 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:29:11.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:29:11 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1215872626' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:29:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:29:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:29:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:29:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:29:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:29:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:29:13.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:29:13 smithi153 ceph-mon[70044]: pgmap v10828: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:29:13.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:29:13 smithi083 ceph-mon[79930]: pgmap v10828: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:29:13.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:29:13 smithi083 ceph-mon[77242]: pgmap v10828: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:29:15.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:29:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:29:14.867Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:29:15.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:29:15 smithi153 ceph-mon[70044]: pgmap v10829: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:29:15.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:29:15 smithi083 ceph-mon[79930]: pgmap v10829: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:29:15.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:29:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:29:15.589Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:29:15.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:29:15 smithi083 ceph-mon[77242]: pgmap v10829: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:29:17.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:29:17 smithi153 ceph-mon[70044]: pgmap v10830: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:29:17.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:29:17 smithi083 ceph-mon[79930]: pgmap v10830: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:29:17.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:29:17 smithi083 ceph-mon[77242]: pgmap v10830: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:29:19.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:29:19 smithi153 ceph-mon[70044]: pgmap v10831: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:29:19.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:29:19 smithi083 ceph-mon[79930]: pgmap v10831: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:29:19.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:29:19 smithi083 ceph-mon[77242]: pgmap v10831: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:29:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:29:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:29:21] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T11:29:21.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:29:21 smithi083 ceph-mon[79930]: pgmap v10832: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:29:21.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:29:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:29:21.496Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:29:21.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:29:21 smithi083 ceph-mon[77242]: pgmap v10832: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:29:22.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:29:21 smithi153 ceph-mon[70044]: pgmap v10832: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:29:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:29:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:29:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:29:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:29:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:29:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:29:23.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:29:23 smithi083 ceph-mon[79930]: pgmap v10833: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:29:23.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:29:23 smithi083 ceph-mon[77242]: pgmap v10833: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:29:24.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:29:23 smithi153 ceph-mon[70044]: pgmap v10833: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:29:25.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:29:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:29:24.868Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:29:25.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:29:25 smithi083 ceph-mon[79930]: pgmap v10834: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:29:25.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:29:25 smithi083 ceph-mon[77242]: pgmap v10834: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:29:25.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:29:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:29:25.590Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:29:26.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:29:25 smithi153 ceph-mon[70044]: pgmap v10834: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:29:27.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:29:27 smithi083 ceph-mon[79930]: pgmap v10835: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:29:27.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:29:27 smithi083 ceph-mon[77242]: pgmap v10835: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:29:28.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:29:27 smithi153 ceph-mon[70044]: pgmap v10835: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:29:29.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:29:29 smithi083 ceph-mon[79930]: pgmap v10836: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:29:29.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:29:29 smithi083 ceph-mon[77242]: pgmap v10836: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:29:30.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:29:29 smithi153 ceph-mon[70044]: pgmap v10836: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:29:31.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:29:31 smithi083 ceph-mon[79930]: pgmap v10837: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:29:31.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:29:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:29:31] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T11:29:31.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:29:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:29:31.497Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:29:31.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:29:31 smithi083 ceph-mon[77242]: pgmap v10837: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:29:32.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:29:31 smithi153 ceph-mon[70044]: pgmap v10837: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:29:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:29:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:29:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:29:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:29:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:29:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:29:33.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:29:33 smithi083 ceph-mon[79930]: pgmap v10838: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:29:33.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:29:33 smithi083 ceph-mon[77242]: pgmap v10838: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:29:34.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:29:33 smithi153 ceph-mon[70044]: pgmap v10838: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:29:35.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:29:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:29:34.868Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:29:35.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:29:35 smithi083 ceph-mon[79930]: pgmap v10839: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:29:35.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:29:35 smithi083 ceph-mon[77242]: pgmap v10839: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:29:35.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:29:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:29:35.592Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:29:36.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:29:35 smithi153 ceph-mon[70044]: pgmap v10839: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:29:37.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:29:37 smithi083 ceph-mon[79930]: pgmap v10840: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:29:37.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:29:37 smithi083 ceph-mon[77242]: pgmap v10840: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:29:38.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:29:37 smithi153 ceph-mon[70044]: pgmap v10840: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:29:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:29:39 smithi083 ceph-mon[79930]: pgmap v10841: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:29:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:29:39 smithi083 ceph-mon[77242]: pgmap v10841: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:29:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:29:39 smithi153 ceph-mon[70044]: pgmap v10841: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:29:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:29:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:29:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:29:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:29:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:29:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:29:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:29:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:29:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:29:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:29:40.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:29:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:29:40.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:29:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:29:40.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:29:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:29:40.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:29:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:29:40.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:29:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:29:40.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:29:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:29:40.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:29:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:29:40.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:29:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:29:40.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:29:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:29:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:29:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:29:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:29:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:29:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:29:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:29:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:29:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:29:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:29:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:29:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:29:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:29:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:29:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:29:41.404 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:29:41.694 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:29:41.695 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 7m ago 6h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:29:41.695 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 5m ago 6h 63.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:29:41.695 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 7m ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:29:41.696 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 5m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:29:41.696 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 5m ago 6h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:29:41.696 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 7m ago 6h 815M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:29:41.696 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 7m ago 6h 1344M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:29:41.697 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (5h) 5m ago 6h 578M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:29:41.697 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (5h) 7m ago 6h 774M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:29:41.697 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 7m ago 6h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:29:41.697 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 5m ago 6h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:29:41.698 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 7m ago 6h 947M 2429M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:29:41.698 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 7m ago 6h 936M 2429M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:29:41.698 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 7m ago 6h 730M 2429M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:29:41.698 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 7m ago 6h 914M 2429M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:29:41.698 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 5m ago 6h 805M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:29:41.699 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 5m ago 6h 1029M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:29:41.699 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 5m ago 6h 898M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:29:41.699 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 5m ago 6h 1089M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:29:41.699 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 5m ago 6h 152M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:29:41.939 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:29:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:29:41] "GET /metrics HTTP/1.1" 200 225225 "" "Prometheus/2.33.4" 2022-09-29T11:29:41.940 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:29:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:29:41.497Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:29:41.940 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:29:41 smithi083 ceph-mon[77242]: pgmap v10842: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:29:41.941 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:29:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:29:42.051 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:29:42.052 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:29:42.052 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:29:42.052 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:29:42.052 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:29:42.052 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:29:42.053 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:29:42.053 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:29:42.053 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:29:42.053 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:29:42.054 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:29:42.054 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:29:42.054 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:29:42.054 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:29:42.055 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:29:42.055 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:29:42.055 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:29:42.055 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:29:42.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:29:41 smithi083 ceph-mon[79930]: pgmap v10842: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:29:42.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:29:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:29:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:29:41 smithi153 ceph-mon[70044]: pgmap v10842: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:29:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:29:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:29:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:29:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:29:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:29:43.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:29:42 smithi083 ceph-mon[79930]: from='client.56674 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:29:43.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:29:42 smithi083 ceph-mon[79930]: from='client.56680 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:29:43.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:29:42 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2554465061' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:29:43.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:29:42 smithi083 ceph-mon[77242]: from='client.56674 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:29:43.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:29:42 smithi083 ceph-mon[77242]: from='client.56680 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:29:43.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:29:42 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2554465061' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:29:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:29:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:29:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:29:43.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:29:42 smithi153 ceph-mon[70044]: from='client.56674 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:29:43.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:29:42 smithi153 ceph-mon[70044]: from='client.56680 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:29:43.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:29:42 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2554465061' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:29:44.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:29:43 smithi083 ceph-mon[77242]: pgmap v10843: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:29:44.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:29:43 smithi083 ceph-mon[79930]: pgmap v10843: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:29:44.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:29:43 smithi153 ceph-mon[70044]: pgmap v10843: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:29:45.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:29:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:29:44.869Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:29:45.949 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:29:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:29:45.591Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:29:46.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:29:45 smithi083 ceph-mon[79930]: pgmap v10844: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:29:46.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:29:45 smithi083 ceph-mon[77242]: pgmap v10844: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:29:46.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:29:45 smithi153 ceph-mon[70044]: pgmap v10844: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:29:48.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:29:47 smithi083 ceph-mon[79930]: pgmap v10845: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:29:48.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:29:47 smithi083 ceph-mon[77242]: pgmap v10845: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:29:48.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:29:47 smithi153 ceph-mon[70044]: pgmap v10845: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:29:50.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:29:49 smithi083 ceph-mon[79930]: pgmap v10846: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:29:50.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:29:49 smithi083 ceph-mon[77242]: pgmap v10846: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:29:50.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:29:49 smithi153 ceph-mon[70044]: pgmap v10846: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:29:51.958 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:29:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:29:51] "GET /metrics HTTP/1.1" 200 225225 "" "Prometheus/2.33.4" 2022-09-29T11:29:51.958 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:29:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:29:51.498Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:29:52.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:29:51 smithi083 ceph-mon[77242]: pgmap v10847: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:29:52.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:29:51 smithi083 ceph-mon[79930]: pgmap v10847: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:29:52.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:29:51 smithi153 ceph-mon[70044]: pgmap v10847: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:29:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:29:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:29:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:29:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:29:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:29:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:29:54.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:29:53 smithi083 ceph-mon[79930]: pgmap v10848: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:29:54.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:29:53 smithi083 ceph-mon[77242]: pgmap v10848: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:29:54.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:29:53 smithi153 ceph-mon[70044]: pgmap v10848: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:29:55.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:29:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:29:54.870Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:29:55.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:29:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:29:55.592Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:29:56.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:29:55 smithi153 ceph-mon[70044]: pgmap v10849: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:29:56.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:29:55 smithi083 ceph-mon[79930]: pgmap v10849: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:29:56.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:29:55 smithi083 ceph-mon[77242]: pgmap v10849: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:29:58.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:29:57 smithi153 ceph-mon[70044]: pgmap v10850: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:29:58.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:29:57 smithi083 ceph-mon[77242]: pgmap v10850: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:29:58.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:29:57 smithi083 ceph-mon[79930]: pgmap v10850: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:29:59 smithi153 ceph-mon[70044]: pgmap v10851: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:00.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:29:59 smithi083 ceph-mon[79930]: pgmap v10851: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:00.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:29:59 smithi083 ceph-mon[77242]: pgmap v10851: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:00.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:30:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[77219]: 2022-09-29T11:29:59.999+0000 7f160f665700 -1 log_channel(cluster) log [ERR] : overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T11:30:01.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:30:00 smithi153 ceph-mon[70044]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T11:30:01.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:30:00 smithi083 ceph-mon[77242]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T11:30:01.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:30:00 smithi083 ceph-mon[79930]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T11:30:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:30:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:30:01] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T11:30:01.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:30:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:30:01.498Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:30:02.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:30:01 smithi153 ceph-mon[70044]: pgmap v10852: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:02.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:30:01 smithi083 ceph-mon[79930]: pgmap v10852: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:02.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:30:01 smithi083 ceph-mon[77242]: pgmap v10852: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:30:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:30:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:30:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:30:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:30:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:30:04.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:30:03 smithi153 ceph-mon[70044]: pgmap v10853: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:04.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:30:03 smithi083 ceph-mon[77242]: pgmap v10853: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:04.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:30:03 smithi083 ceph-mon[79930]: pgmap v10853: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:05.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:30:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:30:04.870Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:30:05.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:30:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:30:05.593Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:30:06.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:30:05 smithi153 ceph-mon[70044]: pgmap v10854: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:06.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:30:05 smithi083 ceph-mon[79930]: pgmap v10854: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:06.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:30:05 smithi083 ceph-mon[77242]: pgmap v10854: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:08.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:30:07 smithi153 ceph-mon[70044]: pgmap v10855: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:08.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:30:07 smithi083 ceph-mon[79930]: pgmap v10855: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:08.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:30:07 smithi083 ceph-mon[77242]: pgmap v10855: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:10.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:30:09 smithi153 ceph-mon[70044]: pgmap v10856: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:10.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:30:09 smithi083 ceph-mon[79930]: pgmap v10856: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:10.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:30:09 smithi083 ceph-mon[77242]: pgmap v10856: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:30:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:30:11] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T11:30:11.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:30:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:30:11.499Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:30:12.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:30:11 smithi153 ceph-mon[70044]: pgmap v10857: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:12.379 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:30:12.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:30:11 smithi083 ceph-mon[79930]: pgmap v10857: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:12.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:30:11 smithi083 ceph-mon[77242]: pgmap v10857: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:12.668 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:30:12.669 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 7m ago 6h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:30:12.669 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 5m ago 6h 63.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:30:12.669 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 7m ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:30:12.669 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 5m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:30:12.670 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 5m ago 6h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:30:12.670 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 7m ago 6h 815M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:30:12.670 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 7m ago 6h 1344M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:30:12.670 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 5m ago 6h 578M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:30:12.671 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 7m ago 6h 774M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:30:12.671 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 7m ago 6h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:30:12.671 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 5m ago 6h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:30:12.671 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 7m ago 6h 947M 2429M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:30:12.672 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 7m ago 6h 936M 2429M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:30:12.672 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 7m ago 6h 730M 2429M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:30:12.672 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 7m ago 6h 914M 2429M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:30:12.672 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 5m ago 6h 805M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:30:12.672 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 5m ago 6h 1029M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:30:12.673 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 5m ago 6h 898M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:30:12.673 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 5m ago 6h 1089M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:30:12.673 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 5m ago 6h 152M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:30:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:30:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:30:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:30:13.023 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:30:13.023 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:30:13.023 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:30:13.024 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:30:13.024 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:30:13.024 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:30:13.024 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:30:13.025 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:30:13.025 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:30:13.025 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:30:13.026 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:30:13.026 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:30:13.026 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:30:13.026 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:30:13.027 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:30:13.027 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:30:13.027 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:30:13.027 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:30:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:30:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:30:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:30:14.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:30:13 smithi153 ceph-mon[70044]: pgmap v10858: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:14.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:30:13 smithi153 ceph-mon[70044]: from='client.46761 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:30:14.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:30:13 smithi153 ceph-mon[70044]: from='client.56698 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:30:14.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:30:13 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1859502471' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:30:14.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:30:13 smithi083 ceph-mon[79930]: pgmap v10858: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:14.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:30:13 smithi083 ceph-mon[79930]: from='client.46761 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:30:14.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:30:13 smithi083 ceph-mon[79930]: from='client.56698 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:30:14.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:30:13 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1859502471' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:30:14.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:30:13 smithi083 ceph-mon[77242]: pgmap v10858: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:14.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:30:13 smithi083 ceph-mon[77242]: from='client.46761 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:30:14.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:30:13 smithi083 ceph-mon[77242]: from='client.56698 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:30:14.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:30:13 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1859502471' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:30:15.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:30:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:30:14.871Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:30:15.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:30:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:30:15.594Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:30:16.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:30:15 smithi153 ceph-mon[70044]: pgmap v10859: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:16.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:30:15 smithi083 ceph-mon[77242]: pgmap v10859: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:16.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:30:15 smithi083 ceph-mon[79930]: pgmap v10859: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:18.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:30:17 smithi153 ceph-mon[70044]: pgmap v10860: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:18.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:30:17 smithi083 ceph-mon[79930]: pgmap v10860: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:18.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:30:17 smithi083 ceph-mon[77242]: pgmap v10860: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:30:20 smithi153 ceph-mon[70044]: pgmap v10861: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:20.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:30:20 smithi083 ceph-mon[77242]: pgmap v10861: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:20.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:30:20 smithi083 ceph-mon[79930]: pgmap v10861: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:30:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:30:21] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T11:30:21.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:30:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:30:21.499Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:30:22.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:30:22 smithi153 ceph-mon[70044]: pgmap v10862: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:22.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:30:22 smithi083 ceph-mon[77242]: pgmap v10862: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:22.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:30:22 smithi083 ceph-mon[79930]: pgmap v10862: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:30:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:30:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:30:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:30:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:30:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:30:24.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:30:24 smithi153 ceph-mon[70044]: pgmap v10863: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:24.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:30:24 smithi083 ceph-mon[77242]: pgmap v10863: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:24.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:30:24 smithi083 ceph-mon[79930]: pgmap v10863: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:25.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:30:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:30:24.872Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:30:25.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:30:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:30:25.595Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:30:26.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:30:26 smithi153 ceph-mon[70044]: pgmap v10864: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:26.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:30:26 smithi083 ceph-mon[79930]: pgmap v10864: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:26.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:30:26 smithi083 ceph-mon[77242]: pgmap v10864: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:28.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:30:28 smithi153 ceph-mon[70044]: pgmap v10865: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:28.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:30:28 smithi083 ceph-mon[79930]: pgmap v10865: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:28.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:30:28 smithi083 ceph-mon[77242]: pgmap v10865: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:30.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:30:30 smithi153 ceph-mon[70044]: pgmap v10866: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:30.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:30:30 smithi083 ceph-mon[77242]: pgmap v10866: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:30.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:30:30 smithi083 ceph-mon[79930]: pgmap v10866: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:30:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:30:31] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T11:30:31.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:30:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:30:31.499Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:30:32.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:30:32 smithi153 ceph-mon[70044]: pgmap v10867: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:32.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:30:32 smithi083 ceph-mon[79930]: pgmap v10867: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:32.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:30:32 smithi083 ceph-mon[77242]: pgmap v10867: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:30:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:30:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:30:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:30:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:30:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:30:34.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:30:34 smithi153 ceph-mon[70044]: pgmap v10868: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:34.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:30:34 smithi083 ceph-mon[79930]: pgmap v10868: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:34.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:30:34 smithi083 ceph-mon[77242]: pgmap v10868: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:35.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:30:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:30:34.872Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:30:35.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:30:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:30:35.596Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:30:36.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:30:36 smithi153 ceph-mon[70044]: pgmap v10869: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:36.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:30:36 smithi083 ceph-mon[77242]: pgmap v10869: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:36.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:30:36 smithi083 ceph-mon[79930]: pgmap v10869: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:38.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:30:38 smithi153 ceph-mon[70044]: pgmap v10870: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:38.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:30:38 smithi083 ceph-mon[79930]: pgmap v10870: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:38.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:30:38 smithi083 ceph-mon[77242]: pgmap v10870: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:30:40 smithi153 ceph-mon[70044]: pgmap v10871: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:30:40 smithi083 ceph-mon[79930]: pgmap v10871: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:30:40 smithi083 ceph-mon[77242]: pgmap v10871: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:30:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:30:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:30:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:30:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:30:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:30:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:30:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:30:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:30:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:30:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:30:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:30:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:30:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:30:41.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:30:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:30:41.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:30:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:30:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:30:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:30:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:30:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:30:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:30:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:30:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:30:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:30:41.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:30:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:30:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:30:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:30:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:30:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:30:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:30:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:30:41.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:30:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:30:41.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:30:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:30:41.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:30:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:30:41.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:30:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:30:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:30:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:30:41] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T11:30:41.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:30:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:30:41.500Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:30:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:30:42 smithi153 ceph-mon[70044]: pgmap v10872: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:30:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:30:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:30:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:30:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:30:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:30:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:30:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:30:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:30:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:30:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:30:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:30:42.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:30:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:30:42.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:30:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:30:42.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:30:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:30:42.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:30:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:30:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:30:42 smithi083 ceph-mon[79930]: pgmap v10872: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:42.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:30:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:30:42.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:30:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:30:42.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:30:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:30:42.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:30:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:30:42.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:30:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:30:42.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:30:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:30:42.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:30:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:30:42.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:30:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:30:42.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:30:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:30:42.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:30:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:30:42.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:30:42 smithi083 ceph-mon[77242]: pgmap v10872: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:42.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:30:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:30:42.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:30:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:30:42.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:30:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:30:42.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:30:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:30:42.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:30:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:30:42.467 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:30:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:30:42.467 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:30:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:30:42.467 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:30:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:30:42.467 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:30:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:30:42.467 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:30:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:30:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:30:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:30:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:30:43.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:30:43 smithi153 ceph-mon[70044]: Adjusting osd_memory_target on smithi083 to 2420M 2022-09-29T11:30:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:30:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:30:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:30:43.358 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:30:43.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:30:43 smithi083 ceph-mon[79930]: Adjusting osd_memory_target on smithi083 to 2420M 2022-09-29T11:30:43.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:30:43 smithi083 ceph-mon[77242]: Adjusting osd_memory_target on smithi083 to 2420M 2022-09-29T11:30:43.656 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:30:43.656 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 8m ago 6h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:30:43.656 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 6m ago 6h 63.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:30:43.657 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 8m ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:30:43.657 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 6m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:30:43.657 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 6m ago 6h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:30:43.657 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 8m ago 6h 815M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:30:43.658 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 8m ago 6h 1344M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:30:43.658 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 6m ago 6h 578M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:30:43.658 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 8m ago 6h 774M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:30:43.658 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 8m ago 6h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:30:43.659 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 6m ago 6h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:30:43.659 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5h) 8m ago 6h 947M 2420M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:30:43.659 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 8m ago 6h 936M 2420M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:30:43.659 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 8m ago 6h 730M 2420M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:30:43.660 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 8m ago 6h 914M 2420M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:30:43.660 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 6m ago 6h 805M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:30:43.660 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 6m ago 6h 1029M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:30:43.660 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 6m ago 6h 898M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:30:43.660 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 6m ago 6h 1089M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:30:43.661 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 6m ago 6h 152M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:30:44.015 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:30:44.015 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:30:44.015 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:30:44.015 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:30:44.016 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:30:44.016 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:30:44.016 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:30:44.016 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:30:44.017 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:30:44.017 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:30:44.017 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:30:44.017 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:30:44.018 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:30:44.018 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:30:44.018 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:30:44.018 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:30:44.019 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:30:44.019 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:30:44.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:30:44 smithi153 ceph-mon[70044]: pgmap v10873: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:44.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:30:44 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2304666629' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:30:44.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:30:44 smithi083 ceph-mon[79930]: pgmap v10873: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:44.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:30:44 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2304666629' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:30:44.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:30:44 smithi083 ceph-mon[77242]: pgmap v10873: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:44.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:30:44 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2304666629' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:30:45.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:30:45 smithi083 ceph-mon[79930]: from='client.46779 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:30:45.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:30:45 smithi083 ceph-mon[79930]: from='client.56716 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:30:45.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:30:45 smithi083 ceph-mon[77242]: from='client.46779 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:30:45.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:30:45 smithi083 ceph-mon[77242]: from='client.56716 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:30:45.214 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:30:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:30:44.871Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:30:45.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:30:45 smithi153 ceph-mon[70044]: from='client.46779 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:30:45.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:30:45 smithi153 ceph-mon[70044]: from='client.56716 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:30:45.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:30:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:30:45.596Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:30:46.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:30:46 smithi153 ceph-mon[70044]: pgmap v10874: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:46.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:30:46 smithi083 ceph-mon[79930]: pgmap v10874: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:46.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:30:46 smithi083 ceph-mon[77242]: pgmap v10874: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:48.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:30:48 smithi153 ceph-mon[70044]: pgmap v10875: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:48.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:30:48 smithi083 ceph-mon[79930]: pgmap v10875: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:48.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:30:48 smithi083 ceph-mon[77242]: pgmap v10875: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:50.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:30:50 smithi083 ceph-mon[79930]: pgmap v10876: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:50.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:30:50 smithi083 ceph-mon[77242]: pgmap v10876: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:30:50 smithi153 ceph-mon[70044]: pgmap v10876: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:30:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:30:51] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T11:30:51.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:30:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:30:51.501Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:30:52.316 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:30:52 smithi153 ceph-mon[70044]: pgmap v10877: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:52.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:30:52 smithi083 ceph-mon[79930]: pgmap v10877: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:52.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:30:52 smithi083 ceph-mon[77242]: pgmap v10877: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:30:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:30:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:30:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:30:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:30:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:30:54.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:30:54 smithi083 ceph-mon[79930]: pgmap v10878: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:54.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:30:54 smithi083 ceph-mon[77242]: pgmap v10878: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:54.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:30:54 smithi153 ceph-mon[70044]: pgmap v10878: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:55.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:30:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:30:54.873Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:30:55.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:30:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:30:55.597Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:30:56.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:30:56 smithi083 ceph-mon[79930]: pgmap v10879: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:56.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:30:56 smithi083 ceph-mon[77242]: pgmap v10879: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:56.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:30:56 smithi153 ceph-mon[70044]: pgmap v10879: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:58.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:30:58 smithi083 ceph-mon[77242]: pgmap v10880: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:58.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:30:58 smithi083 ceph-mon[79930]: pgmap v10880: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:30:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:30:58 smithi153 ceph-mon[70044]: pgmap v10880: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:00.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:31:00 smithi083 ceph-mon[79930]: pgmap v10881: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:00.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:31:00 smithi083 ceph-mon[77242]: pgmap v10881: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:31:00 smithi153 ceph-mon[70044]: pgmap v10881: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:31:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:31:01] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T11:31:01.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:31:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:31:01.502Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:31:02.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:31:02 smithi083 ceph-mon[79930]: pgmap v10882: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:02.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:31:02 smithi083 ceph-mon[77242]: pgmap v10882: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:31:02 smithi153 ceph-mon[70044]: pgmap v10882: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:02.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:31:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:31:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:31:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:31:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:31:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:31:04.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:31:04 smithi083 ceph-mon[79930]: pgmap v10883: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:04.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:31:04 smithi083 ceph-mon[77242]: pgmap v10883: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:04.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:31:04 smithi153 ceph-mon[70044]: pgmap v10883: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:05.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:31:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:31:04.874Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:31:05.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:31:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:31:05.598Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:31:06.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:31:06 smithi083 ceph-mon[79930]: pgmap v10884: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:06.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:31:06 smithi083 ceph-mon[77242]: pgmap v10884: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:06.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:31:06 smithi153 ceph-mon[70044]: pgmap v10884: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:08.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:31:08 smithi083 ceph-mon[79930]: pgmap v10885: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:08.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:31:08 smithi083 ceph-mon[77242]: pgmap v10885: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:31:08 smithi153 ceph-mon[70044]: pgmap v10885: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:10.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:31:10 smithi083 ceph-mon[79930]: pgmap v10886: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:10.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:31:10 smithi083 ceph-mon[77242]: pgmap v10886: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:10.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:31:10 smithi153 ceph-mon[70044]: pgmap v10886: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:31:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:31:11] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T11:31:11.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:31:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:31:11.502Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:31:12.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:31:12 smithi083 ceph-mon[77242]: pgmap v10887: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:12.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:31:12 smithi083 ceph-mon[79930]: pgmap v10887: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:31:12 smithi153 ceph-mon[70044]: pgmap v10887: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:12.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:31:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:31:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:31:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:31:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:31:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:31:14.344 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:31:14.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:31:14 smithi083 ceph-mon[79930]: pgmap v10888: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:14.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:31:14 smithi083 ceph-mon[77242]: pgmap v10888: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:14.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:31:14 smithi153 ceph-mon[70044]: pgmap v10888: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:14.632 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:31:14.632 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 8m ago 6h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:31:14.632 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 6m ago 6h 63.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:31:14.633 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 8m ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:31:14.633 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 6m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:31:14.633 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 6m ago 6h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:31:14.633 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 8m ago 6h 815M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:31:14.634 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 8m ago 6h 1344M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:31:14.634 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 6m ago 6h 578M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:31:14.634 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 8m ago 6h 774M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:31:14.634 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 8m ago 6h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:31:14.634 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 6m ago 6h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:31:14.635 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 8m ago 6h 947M 2420M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:31:14.635 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5h) 8m ago 6h 936M 2420M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:31:14.635 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5h) 8m ago 6h 730M 2420M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:31:14.635 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 8m ago 6h 914M 2420M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:31:14.636 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 6m ago 6h 805M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:31:14.636 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 6m ago 6h 1029M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:31:14.636 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 6m ago 6h 898M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:31:14.636 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 6m ago 6h 1089M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:31:14.636 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 6m ago 6h 152M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:31:14.990 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:31:14.991 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:31:14.991 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:31:14.991 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:31:14.991 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:31:14.992 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:31:14.992 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:31:14.992 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:31:14.992 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:31:14.993 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:31:14.993 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:31:14.993 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:31:14.993 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:31:14.993 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:31:14.994 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:31:14.994 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:31:14.994 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:31:14.994 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:31:15.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:31:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:31:14.874Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:31:15.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:31:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:31:15.598Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:31:15.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:31:15 smithi083 ceph-mon[77242]: pgmap v10889: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:15.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:31:15 smithi083 ceph-mon[77242]: from='client.56728 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:31:15.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:31:15 smithi083 ceph-mon[77242]: from='client.56734 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:31:15.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:31:15 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3777323251' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:31:15.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:31:15 smithi083 ceph-mon[79930]: pgmap v10889: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:15.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:31:15 smithi083 ceph-mon[79930]: from='client.56728 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:31:15.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:31:15 smithi083 ceph-mon[79930]: from='client.56734 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:31:15.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:31:15 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3777323251' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:31:15.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:31:15 smithi153 ceph-mon[70044]: pgmap v10889: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:15.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:31:15 smithi153 ceph-mon[70044]: from='client.56728 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:31:15.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:31:15 smithi153 ceph-mon[70044]: from='client.56734 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:31:15.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:31:15 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3777323251' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:31:17.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:31:17 smithi083 ceph-mon[77242]: pgmap v10890: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:17.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:31:17 smithi083 ceph-mon[79930]: pgmap v10890: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:17.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:31:17 smithi153 ceph-mon[70044]: pgmap v10890: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:19.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:31:19 smithi083 ceph-mon[77242]: pgmap v10891: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:19.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:31:19 smithi083 ceph-mon[79930]: pgmap v10891: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:19.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:31:19 smithi153 ceph-mon[70044]: pgmap v10891: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:21.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:31:21 smithi153 ceph-mon[70044]: pgmap v10892: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:21.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:31:21 smithi083 ceph-mon[79930]: pgmap v10892: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:21.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:31:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:31:21.503Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:31:21.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:31:21 smithi083 ceph-mon[77242]: pgmap v10892: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:21.964 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:31:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:31:21] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T11:31:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:31:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:31:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:31:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:31:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:31:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:31:23.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:31:23 smithi153 ceph-mon[70044]: pgmap v10893: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:23.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:31:23 smithi083 ceph-mon[79930]: pgmap v10893: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:23.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:31:23 smithi083 ceph-mon[77242]: pgmap v10893: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:25.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:31:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:31:24.875Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:31:25.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:31:25 smithi153 ceph-mon[70044]: pgmap v10894: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:25.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:31:25 smithi083 ceph-mon[79930]: pgmap v10894: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:25.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:31:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:31:25.599Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:31:25.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:31:25 smithi083 ceph-mon[77242]: pgmap v10894: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:27.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:31:27 smithi153 ceph-mon[70044]: pgmap v10895: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:27.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:31:27 smithi083 ceph-mon[79930]: pgmap v10895: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:27.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:31:27 smithi083 ceph-mon[77242]: pgmap v10895: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:29.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:31:29 smithi153 ceph-mon[70044]: pgmap v10896: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:29.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:31:29 smithi083 ceph-mon[79930]: pgmap v10896: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:29.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:31:29 smithi083 ceph-mon[77242]: pgmap v10896: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:31.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:31:31 smithi153 ceph-mon[70044]: pgmap v10897: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:31.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:31:31 smithi083 ceph-mon[79930]: pgmap v10897: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:31.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:31:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:31:31.504Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:31:31.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:31:31 smithi083 ceph-mon[77242]: pgmap v10897: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:31.964 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:31:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:31:31] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T11:31:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:31:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:31:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:31:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:31:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:31:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:31:33.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:31:33 smithi153 ceph-mon[70044]: pgmap v10898: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:33.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:31:33 smithi083 ceph-mon[79930]: pgmap v10898: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:33.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:31:33 smithi083 ceph-mon[77242]: pgmap v10898: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:35.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:31:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:31:34.876Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:31:35.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:31:35 smithi153 ceph-mon[70044]: pgmap v10899: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:35.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:31:35 smithi083 ceph-mon[79930]: pgmap v10899: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:35.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:31:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:31:35.600Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:31:35.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:31:35 smithi083 ceph-mon[77242]: pgmap v10899: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:37.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:31:37 smithi153 ceph-mon[70044]: pgmap v10900: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:37.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:31:37 smithi083 ceph-mon[79930]: pgmap v10900: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:37.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:31:37 smithi083 ceph-mon[77242]: pgmap v10900: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:39.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:31:39 smithi153 ceph-mon[70044]: pgmap v10901: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:31:39 smithi083 ceph-mon[79930]: pgmap v10901: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:31:39 smithi083 ceph-mon[77242]: pgmap v10901: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:31:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:31:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:31:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:31:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:31:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:31:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:31:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:31:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:31:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:31:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:31:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:31:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:31:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:31:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:31:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:31:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:31:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:31:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:31:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:31:40.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:31:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:31:40.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:31:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:31:41.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:31:41 smithi153 ceph-mon[70044]: pgmap v10902: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:41.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:31:41 smithi083 ceph-mon[79930]: pgmap v10902: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:41.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:31:41 smithi083 ceph-mon[77242]: pgmap v10902: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:41.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:31:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:31:41] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T11:31:41.964 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:31:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:31:41.505Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:31:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:31:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:31:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:31:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:31:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:31:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:31:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:31:42.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:31:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:31:42.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:31:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:31:42.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:31:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:31:42.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:31:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:31:42.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:31:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:31:42.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:31:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:31:42.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:31:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:31:42.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:31:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:31:42.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:31:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:31:42.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:31:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:31:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:31:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:31:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:31:44.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:31:44 smithi153 ceph-mon[70044]: pgmap v10903: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:44.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:31:44 smithi083 ceph-mon[79930]: pgmap v10903: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:44.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:31:44 smithi083 ceph-mon[77242]: pgmap v10903: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:45.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:31:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:31:44.877Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:31:45.320 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:31:45.610 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:31:45.610 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 9m ago 6h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:31:45.611 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 7m ago 6h 63.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:31:45.611 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 9m ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:31:45.611 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 7m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:31:45.611 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 7m ago 6h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:31:45.611 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 9m ago 6h 815M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:31:45.612 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 9m ago 6h 1344M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:31:45.612 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 7m ago 6h 578M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:31:45.612 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 9m ago 6h 774M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:31:45.612 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 9m ago 6h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:31:45.613 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 7m ago 6h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:31:45.613 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 9m ago 6h 947M 2420M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:31:45.613 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 9m ago 6h 936M 2420M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:31:45.613 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 9m ago 6h 730M 2420M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:31:45.613 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5h) 9m ago 6h 914M 2420M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:31:45.614 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 7m ago 6h 805M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:31:45.614 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 7m ago 6h 1029M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:31:45.614 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 7m ago 6h 898M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:31:45.614 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 7m ago 6h 1089M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:31:45.615 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 7m ago 6h 152M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:31:45.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:31:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:31:45.601Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:31:45.969 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:31:45.969 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:31:45.969 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:31:45.969 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:31:45.970 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:31:45.970 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:31:45.970 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:31:45.970 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:31:45.971 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:31:45.971 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:31:45.971 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:31:45.971 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:31:45.972 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:31:45.972 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:31:45.972 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:31:45.972 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:31:45.972 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:31:45.973 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:31:46.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:31:46 smithi153 ceph-mon[70044]: pgmap v10904: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:46.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:31:46 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1943684943' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:31:46.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:31:46 smithi083 ceph-mon[79930]: pgmap v10904: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:46.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:31:46 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1943684943' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:31:46.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:31:46 smithi083 ceph-mon[77242]: pgmap v10904: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:46.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:31:46 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1943684943' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:31:47.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:31:47 smithi153 ceph-mon[70044]: from='client.56746 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:31:47.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:31:47 smithi153 ceph-mon[70044]: from='client.46821 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:31:47.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:31:47 smithi083 ceph-mon[79930]: from='client.56746 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:31:47.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:31:47 smithi083 ceph-mon[79930]: from='client.46821 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:31:47.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:31:47 smithi083 ceph-mon[77242]: from='client.56746 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:31:47.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:31:47 smithi083 ceph-mon[77242]: from='client.46821 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:31:48.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:31:48 smithi153 ceph-mon[70044]: pgmap v10905: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:48.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:31:48 smithi083 ceph-mon[79930]: pgmap v10905: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:48.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:31:48 smithi083 ceph-mon[77242]: pgmap v10905: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:49.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:31:49 smithi153 ceph-mon[70044]: pgmap v10906: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:49.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:31:49 smithi083 ceph-mon[79930]: pgmap v10906: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:49.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:31:49 smithi083 ceph-mon[77242]: pgmap v10906: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:51.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:31:51 smithi153 ceph-mon[70044]: pgmap v10907: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:31:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:31:51] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T11:31:51.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:31:51 smithi083 ceph-mon[79930]: pgmap v10907: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:51.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:31:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:31:51.505Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:31:51.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:31:51 smithi083 ceph-mon[77242]: pgmap v10907: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:31:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:31:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:31:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:31:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:31:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:31:53.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:31:53 smithi153 ceph-mon[70044]: pgmap v10908: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:53.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:31:53 smithi083 ceph-mon[77242]: pgmap v10908: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:53.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:31:53 smithi083 ceph-mon[79930]: pgmap v10908: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:55.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:31:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:31:54.876Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:31:55.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:31:55 smithi153 ceph-mon[70044]: pgmap v10909: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:55.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:31:55 smithi083 ceph-mon[79930]: pgmap v10909: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:55.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:31:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:31:55.601Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:31:55.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:31:55 smithi083 ceph-mon[77242]: pgmap v10909: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:57.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:31:57 smithi153 ceph-mon[70044]: pgmap v10910: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:57.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:31:57 smithi083 ceph-mon[79930]: pgmap v10910: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:57.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:31:57 smithi083 ceph-mon[77242]: pgmap v10910: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:59.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:31:59 smithi153 ceph-mon[70044]: pgmap v10911: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:59.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:31:59 smithi083 ceph-mon[77242]: pgmap v10911: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:31:59.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:31:59 smithi083 ceph-mon[79930]: pgmap v10911: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:32:01.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:32:01 smithi153 ceph-mon[70044]: pgmap v10912: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:32:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:32:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:32:01] "GET /metrics HTTP/1.1" 200 225229 "" "Prometheus/2.33.4" 2022-09-29T11:32:01.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:32:01 smithi083 ceph-mon[79930]: pgmap v10912: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:32:01.964 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:32:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:32:01.507Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:32:01.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:32:01 smithi083 ceph-mon[77242]: pgmap v10912: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:32:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:32:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:32:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:32:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:32:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:32:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:32:03.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:32:03 smithi153 ceph-mon[70044]: pgmap v10913: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:32:03.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:32:03 smithi083 ceph-mon[79930]: pgmap v10913: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:32:03.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:32:03 smithi083 ceph-mon[77242]: pgmap v10913: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:32:05.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:32:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:32:04.878Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:32:05.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:32:05 smithi153 ceph-mon[70044]: pgmap v10914: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:32:05.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:32:05 smithi083 ceph-mon[79930]: pgmap v10914: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:32:05.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:32:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:32:05.602Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:32:05.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:32:05 smithi083 ceph-mon[77242]: pgmap v10914: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:32:07.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:32:07 smithi153 ceph-mon[70044]: pgmap v10915: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:32:07.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:32:07 smithi083 ceph-mon[79930]: pgmap v10915: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:32:07.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:32:07 smithi083 ceph-mon[77242]: pgmap v10915: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:32:09.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:32:09 smithi153 ceph-mon[70044]: pgmap v10916: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:32:09.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:32:09 smithi083 ceph-mon[77242]: pgmap v10916: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:32:09.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:32:09 smithi083 ceph-mon[79930]: pgmap v10916: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:32:11.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:32:11 smithi153 ceph-mon[70044]: pgmap v10917: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:32:11.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:32:11 smithi083 ceph-mon[79930]: pgmap v10917: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:32:11.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:32:11 smithi083 ceph-mon[77242]: pgmap v10917: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:32:11.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:32:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:32:11] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T11:32:11.964 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:32:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:32:11.507Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:32:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:32:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:32:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:32:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:32:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:32:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:32:13.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:32:13 smithi153 ceph-mon[70044]: pgmap v10918: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:32:13.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:32:13 smithi083 ceph-mon[79930]: pgmap v10918: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:32:13.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:32:13 smithi083 ceph-mon[77242]: pgmap v10918: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:32:15.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:32:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:32:14.878Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:32:15.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:32:15 smithi153 ceph-mon[70044]: pgmap v10919: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:32:15.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:32:15 smithi083 ceph-mon[79930]: pgmap v10919: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:32:15.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:32:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:32:15.602Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:32:15.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:32:15 smithi083 ceph-mon[77242]: pgmap v10919: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:32:16.295 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:32:16.585 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:32:16.585 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 9m ago 6h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:32:16.585 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 7m ago 6h 63.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:32:16.586 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 9m ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:32:16.586 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 7m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:32:16.586 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 7m ago 6h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:32:16.586 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 9m ago 6h 815M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:32:16.587 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 9m ago 6h 1344M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:32:16.587 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 7m ago 6h 578M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:32:16.587 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 9m ago 6h 774M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:32:16.587 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 9m ago 6h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:32:16.588 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 7m ago 6h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:32:16.588 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 9m ago 6h 947M 2420M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:32:16.588 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 9m ago 6h 936M 2420M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:32:16.588 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 9m ago 6h 730M 2420M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:32:16.589 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 9m ago 6h 914M 2420M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:32:16.589 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (5h) 7m ago 6h 805M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:32:16.589 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 7m ago 6h 1029M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:32:16.589 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 7m ago 6h 898M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:32:16.589 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 7m ago 6h 1089M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:32:16.590 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 7m ago 6h 152M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:32:16.943 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:32:16.943 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:32:16.943 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:32:16.944 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:32:16.944 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:32:16.944 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:32:16.944 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:32:16.945 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:32:16.945 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:32:16.945 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:32:16.945 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:32:16.945 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:32:16.946 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:32:16.946 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:32:16.946 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:32:16.946 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:32:16.947 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:32:16.947 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:32:17.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:32:17 smithi153 ceph-mon[70044]: from='client.46833 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:32:17.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:32:17 smithi153 ceph-mon[70044]: pgmap v10920: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:32:17.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:32:17 smithi153 ceph-mon[70044]: from='client.46839 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:32:17.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:32:17 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/285326821' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:32:17.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:32:17 smithi083 ceph-mon[79930]: from='client.46833 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:32:17.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:32:17 smithi083 ceph-mon[79930]: pgmap v10920: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:32:17.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:32:17 smithi083 ceph-mon[79930]: from='client.46839 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:32:17.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:32:17 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/285326821' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:32:17.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:32:17 smithi083 ceph-mon[77242]: from='client.46833 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:32:17.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:32:17 smithi083 ceph-mon[77242]: pgmap v10920: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:32:17.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:32:17 smithi083 ceph-mon[77242]: from='client.46839 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:32:17.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:32:17 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/285326821' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:32:19.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:32:19 smithi153 ceph-mon[70044]: pgmap v10921: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:32:19.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:32:19 smithi083 ceph-mon[79930]: pgmap v10921: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:32:19.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:32:19 smithi083 ceph-mon[77242]: pgmap v10921: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:32:21.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:32:21 smithi153 ceph-mon[70044]: pgmap v10922: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:32:21.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:32:21 smithi083 ceph-mon[77242]: pgmap v10922: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:32:21.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:32:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:32:21] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T11:32:21.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:32:21 smithi083 ceph-mon[79930]: pgmap v10922: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:32:21.964 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:32:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:32:21.508Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:32:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:32:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:32:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:32:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:32:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:32:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:32:23.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:32:23 smithi153 ceph-mon[70044]: pgmap v10923: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:32:23.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:32:23 smithi083 ceph-mon[79930]: pgmap v10923: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:32:23.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:32:23 smithi083 ceph-mon[77242]: pgmap v10923: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:32:25.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:32:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:32:24.879Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:32:25.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:32:25 smithi153 ceph-mon[70044]: pgmap v10924: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:32:25.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:32:25 smithi083 ceph-mon[79930]: pgmap v10924: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:32:25.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:32:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:32:25.603Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:32:25.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:32:25 smithi083 ceph-mon[77242]: pgmap v10924: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:32:27.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:32:27 smithi153 ceph-mon[70044]: pgmap v10925: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:32:27.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:32:27 smithi083 ceph-mon[79930]: pgmap v10925: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:32:27.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:32:27 smithi083 ceph-mon[77242]: pgmap v10925: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:32:29.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:32:29 smithi153 ceph-mon[70044]: pgmap v10926: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:32:29.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:32:29 smithi083 ceph-mon[79930]: pgmap v10926: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:32:29.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:32:29 smithi083 ceph-mon[77242]: pgmap v10926: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:32:31.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:32:31 smithi153 ceph-mon[70044]: pgmap v10927: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:32:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:32:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:32:31] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T11:32:31.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:32:31 smithi083 ceph-mon[79930]: pgmap v10927: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:32:31.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:32:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:32:31.508Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:32:31.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:32:31 smithi083 ceph-mon[77242]: pgmap v10927: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:32:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:32:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:32:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:32:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:32:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:32:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:32:33.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:32:33 smithi153 ceph-mon[70044]: pgmap v10928: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:32:33.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:32:33 smithi083 ceph-mon[79930]: pgmap v10928: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:32:33.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:32:33 smithi083 ceph-mon[77242]: pgmap v10928: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:32:35.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:32:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:32:34.879Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:32:35.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:32:35 smithi153 ceph-mon[70044]: pgmap v10929: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:32:35.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:32:35 smithi083 ceph-mon[79930]: pgmap v10929: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:32:35.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:32:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:32:35.604Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:32:35.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:32:35 smithi083 ceph-mon[77242]: pgmap v10929: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:32:37.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:32:37 smithi153 ceph-mon[70044]: pgmap v10930: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:32:37.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:32:37 smithi083 ceph-mon[79930]: pgmap v10930: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:32:37.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:32:37 smithi083 ceph-mon[77242]: pgmap v10930: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:32:39.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:32:39 smithi153 ceph-mon[70044]: pgmap v10931: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:32:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:32:39 smithi083 ceph-mon[79930]: pgmap v10931: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:32:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:32:39 smithi083 ceph-mon[77242]: pgmap v10931: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:32:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:32:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:32:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:32:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:32:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:32:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:32:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:32:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:32:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:32:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:32:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:32:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:32:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:32:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:32:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:32:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:32:40.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:32:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:32:40.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:32:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:32:40.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:32:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:32:40.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:32:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:32:41.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:32:41 smithi153 ceph-mon[70044]: pgmap v10932: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:32:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:32:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:32:41] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T11:32:41.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:32:41 smithi083 ceph-mon[79930]: pgmap v10932: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:32:41.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:32:41 smithi083 ceph-mon[77242]: pgmap v10932: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:32:41.964 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:32:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:32:41.508Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:32:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:32:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:32:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:32:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:32:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:32:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:32:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:32:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:32:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:32:42.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:32:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:32:42.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:32:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:32:42.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:32:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:32:42.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:32:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:32:42.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:32:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:32:42.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:32:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:32:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:32:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:32:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:32:44.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:32:43 smithi153 ceph-mon[70044]: pgmap v10933: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:32:44.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:32:43 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:32:44.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:32:43 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:32:44.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:32:43 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:32:44.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:32:43 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:32:44.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:32:43 smithi083 ceph-mon[79930]: pgmap v10933: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:32:44.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:32:43 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:32:44.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:32:43 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:32:44.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:32:43 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:32:44.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:32:43 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:32:44.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:32:43 smithi083 ceph-mon[77242]: pgmap v10933: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:32:44.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:32:43 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:32:44.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:32:43 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:32:44.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:32:43 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:32:44.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:32:43 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:32:45.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:32:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:32:44.880Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:32:45.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:32:45 smithi083 ceph-mon[79930]: pgmap v10934: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:32:45.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:32:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:32:45.603Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:32:45.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:32:45 smithi083 ceph-mon[77242]: pgmap v10934: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:32:46.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:32:45 smithi153 ceph-mon[70044]: pgmap v10934: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:32:47.274 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:32:47.576 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:32:47.576 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 0s ago 6h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:32:47.576 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 8m ago 6h 63.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:32:47.576 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 0s ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:32:47.577 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 8m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:32:47.577 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 8m ago 6h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:32:47.577 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 0s ago 6h 823M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:32:47.577 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 0s ago 6h 1379M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:32:47.578 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 8m ago 6h 578M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:32:47.578 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 0s ago 6h 795M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:32:47.578 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 0s ago 6h 29.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:32:47.578 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 8m ago 6h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:32:47.579 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 0s ago 6h 972M 2420M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:32:47.579 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 0s ago 6h 961M 2420M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:32:47.579 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 0s ago 6h 749M 2420M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:32:47.579 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 0s ago 6h 939M 2420M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:32:47.580 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 8m ago 6h 805M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:32:47.580 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 8m ago 6h 1029M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:32:47.580 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 8m ago 6h 898M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:32:47.580 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 8m ago 6h 1089M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:32:47.581 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 8m ago 6h 152M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:32:47.936 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:32:47.936 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:32:47.937 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:32:47.937 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:32:47.937 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:32:47.937 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:32:47.938 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:32:47.938 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:32:47.938 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:32:47.938 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:32:47.938 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:32:47.939 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:32:47.939 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:32:47.939 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:32:47.939 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:32:47.940 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:32:47.940 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:32:47.940 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:32:48.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:32:47 smithi153 ceph-mon[70044]: pgmap v10935: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:32:48.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:32:47 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:32:48.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:32:47 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:32:48.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:32:47 smithi153 ceph-mon[70044]: from='client.46851 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:32:48.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:32:47 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:32:48.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:32:47 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3369768684' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:32:48.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:32:47 smithi083 ceph-mon[79930]: pgmap v10935: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:32:48.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:32:47 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:32:48.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:32:47 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:32:48.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:32:47 smithi083 ceph-mon[79930]: from='client.46851 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:32:48.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:32:47 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:32:48.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:32:47 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3369768684' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:32:48.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:32:47 smithi083 ceph-mon[77242]: pgmap v10935: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:32:48.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:32:47 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:32:48.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:32:47 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:32:48.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:32:47 smithi083 ceph-mon[77242]: from='client.46851 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:32:48.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:32:47 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:32:48.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:32:47 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3369768684' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:32:49.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:32:49 smithi153 ceph-mon[70044]: from='client.56788 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:32:49.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:32:49 smithi083 ceph-mon[79930]: from='client.56788 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:32:49.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:32:49 smithi083 ceph-mon[77242]: from='client.56788 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:32:50.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:32:50 smithi153 ceph-mon[70044]: pgmap v10936: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:32:50.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:32:50 smithi083 ceph-mon[79930]: pgmap v10936: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:32:50.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:32:50 smithi083 ceph-mon[77242]: pgmap v10936: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:32:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:32:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:32:51] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T11:32:51.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:32:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:32:51.509Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:32:52.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:32:52 smithi153 ceph-mon[70044]: pgmap v10937: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:32:52.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:32:52 smithi083 ceph-mon[79930]: pgmap v10937: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:32:52.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:32:52 smithi083 ceph-mon[77242]: pgmap v10937: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:32:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:32:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:32:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:32:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:32:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:32:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:32:54.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:32:54 smithi153 ceph-mon[70044]: pgmap v10938: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:32:54.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:32:54 smithi083 ceph-mon[77242]: pgmap v10938: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:32:54.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:32:54 smithi083 ceph-mon[79930]: pgmap v10938: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:32:55.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:32:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:32:54.881Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:32:55.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:32:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:32:55.605Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:32:56.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:32:56 smithi153 ceph-mon[70044]: pgmap v10939: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:32:56.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:32:56 smithi083 ceph-mon[77242]: pgmap v10939: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:32:56.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:32:56 smithi083 ceph-mon[79930]: pgmap v10939: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:32:58.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:32:58 smithi153 ceph-mon[70044]: pgmap v10940: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:32:58.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:32:58 smithi083 ceph-mon[79930]: pgmap v10940: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:32:58.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:32:58 smithi083 ceph-mon[77242]: pgmap v10940: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:33:00 smithi153 ceph-mon[70044]: pgmap v10941: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:00.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:33:00 smithi083 ceph-mon[79930]: pgmap v10941: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:00.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:33:00 smithi083 ceph-mon[77242]: pgmap v10941: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:33:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:33:01] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T11:33:01.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:33:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:33:01.510Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:33:02.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:33:02 smithi153 ceph-mon[70044]: pgmap v10942: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:02.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:33:02 smithi083 ceph-mon[77242]: pgmap v10942: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:02.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:33:02 smithi083 ceph-mon[79930]: pgmap v10942: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:33:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:33:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:33:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:33:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:33:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:33:04.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:33:04 smithi153 ceph-mon[70044]: pgmap v10943: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:04.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:33:04 smithi083 ceph-mon[79930]: pgmap v10943: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:04.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:33:04 smithi083 ceph-mon[77242]: pgmap v10943: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:05.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:33:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:33:04.881Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:33:05.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:33:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:33:05.606Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:33:06.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:33:06 smithi153 ceph-mon[70044]: pgmap v10944: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:06.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:33:06 smithi083 ceph-mon[79930]: pgmap v10944: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:06.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:33:06 smithi083 ceph-mon[77242]: pgmap v10944: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:08.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:33:08 smithi153 ceph-mon[70044]: pgmap v10945: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:08.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:33:08 smithi083 ceph-mon[79930]: pgmap v10945: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:08.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:33:08 smithi083 ceph-mon[77242]: pgmap v10945: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:10.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:33:10 smithi153 ceph-mon[70044]: pgmap v10946: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:10.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:33:10 smithi083 ceph-mon[79930]: pgmap v10946: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:10.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:33:10 smithi083 ceph-mon[77242]: pgmap v10946: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:33:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:33:11] "GET /metrics HTTP/1.1" 200 225229 "" "Prometheus/2.33.4" 2022-09-29T11:33:11.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:33:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:33:11.510Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:33:12.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:33:12 smithi153 ceph-mon[70044]: pgmap v10947: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:12.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:33:12 smithi083 ceph-mon[77242]: pgmap v10947: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:12.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:33:12 smithi083 ceph-mon[79930]: pgmap v10947: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:33:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:33:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:33:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:33:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:33:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:33:14.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:33:14 smithi153 ceph-mon[70044]: pgmap v10948: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:14.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:33:14 smithi083 ceph-mon[77242]: pgmap v10948: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:14.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:33:14 smithi083 ceph-mon[79930]: pgmap v10948: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:15.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:33:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:33:14.882Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:33:15.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:33:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:33:15.607Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:33:16.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:33:16 smithi153 ceph-mon[70044]: pgmap v10949: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:16.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:33:16 smithi083 ceph-mon[77242]: pgmap v10949: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:16.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:33:16 smithi083 ceph-mon[79930]: pgmap v10949: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:18.259 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:33:18.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:33:18 smithi153 ceph-mon[70044]: pgmap v10950: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:18.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:33:18 smithi083 ceph-mon[79930]: pgmap v10950: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:18.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:33:18 smithi083 ceph-mon[77242]: pgmap v10950: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:18.547 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:33:18.548 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 31s ago 6h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:33:18.548 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 8m ago 6h 63.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:33:18.548 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 31s ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:33:18.548 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 8m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:33:18.549 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 8m ago 6h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:33:18.549 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 31s ago 6h 823M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:33:18.549 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 31s ago 6h 1379M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:33:18.550 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 8m ago 6h 578M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:33:18.550 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 31s ago 6h 795M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:33:18.550 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 31s ago 6h 29.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:33:18.550 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 8m ago 6h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:33:18.551 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 31s ago 6h 972M 2420M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:33:18.551 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 31s ago 6h 961M 2420M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:33:18.551 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 31s ago 6h 749M 2420M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:33:18.551 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 31s ago 6h 939M 2420M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:33:18.552 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 8m ago 6h 805M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:33:18.552 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (5h) 8m ago 6h 1029M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:33:18.552 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 8m ago 6h 898M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:33:18.552 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 8m ago 6h 1089M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:33:18.553 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 8m ago 6h 152M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:33:18.906 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:33:18.907 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:33:18.907 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:33:18.907 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:33:18.907 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:33:18.908 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:33:18.908 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:33:18.908 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:33:18.908 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:33:18.908 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:33:18.909 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:33:18.909 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:33:18.909 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:33:18.909 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:33:18.910 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:33:18.910 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:33:18.910 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:33:18.911 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:33:19.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:33:19 smithi153 ceph-mon[70044]: from='client.46869 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:33:19.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:33:19 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2724678733' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:33:19.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:33:19 smithi083 ceph-mon[79930]: from='client.46869 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:33:19.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:33:19 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2724678733' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:33:19.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:33:19 smithi083 ceph-mon[77242]: from='client.46869 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:33:19.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:33:19 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2724678733' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:33:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:33:20 smithi153 ceph-mon[70044]: pgmap v10951: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:33:20 smithi153 ceph-mon[70044]: from='client.56806 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:33:20.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:33:20 smithi083 ceph-mon[79930]: pgmap v10951: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:20.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:33:20 smithi083 ceph-mon[79930]: from='client.56806 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:33:20.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:33:20 smithi083 ceph-mon[77242]: pgmap v10951: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:20.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:33:20 smithi083 ceph-mon[77242]: from='client.56806 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:33:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:33:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:33:21] "GET /metrics HTTP/1.1" 200 225229 "" "Prometheus/2.33.4" 2022-09-29T11:33:21.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:33:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:33:21.511Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:33:22.316 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:33:22 smithi153 ceph-mon[70044]: pgmap v10952: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:22.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:33:22 smithi083 ceph-mon[79930]: pgmap v10952: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:22.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:33:22 smithi083 ceph-mon[77242]: pgmap v10952: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:33:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:33:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:33:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:33:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:33:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:33:24.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:33:24 smithi083 ceph-mon[79930]: pgmap v10953: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:24.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:33:24 smithi083 ceph-mon[77242]: pgmap v10953: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:24.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:33:24 smithi153 ceph-mon[70044]: pgmap v10953: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:25.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:33:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:33:24.883Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:33:25.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:33:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:33:25.608Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:33:26.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:33:26 smithi083 ceph-mon[79930]: pgmap v10954: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:26.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:33:26 smithi083 ceph-mon[77242]: pgmap v10954: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:33:26 smithi153 ceph-mon[70044]: pgmap v10954: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:28.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:33:28 smithi083 ceph-mon[77242]: pgmap v10955: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:28.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:33:28 smithi083 ceph-mon[79930]: pgmap v10955: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:33:28 smithi153 ceph-mon[70044]: pgmap v10955: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:30.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:33:30 smithi083 ceph-mon[79930]: pgmap v10956: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:30.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:33:30 smithi083 ceph-mon[77242]: pgmap v10956: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:33:30 smithi153 ceph-mon[70044]: pgmap v10956: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:33:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:33:31] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T11:33:31.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:33:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:33:31.511Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:33:32.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:33:32 smithi083 ceph-mon[79930]: pgmap v10957: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:32.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:33:32 smithi083 ceph-mon[77242]: pgmap v10957: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:33:32 smithi153 ceph-mon[70044]: pgmap v10957: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:32.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:33:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:33:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:33:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:33:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:33:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:33:34.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:33:34 smithi083 ceph-mon[77242]: pgmap v10958: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:34.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:33:34 smithi083 ceph-mon[79930]: pgmap v10958: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:33:34 smithi153 ceph-mon[70044]: pgmap v10958: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:35.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:33:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:33:34.883Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:33:35.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:33:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:33:35.608Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:33:36.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:33:36 smithi083 ceph-mon[79930]: pgmap v10959: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:36.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:33:36 smithi083 ceph-mon[77242]: pgmap v10959: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:33:36 smithi153 ceph-mon[70044]: pgmap v10959: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:38.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:33:38 smithi083 ceph-mon[79930]: pgmap v10960: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:38.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:33:38 smithi083 ceph-mon[77242]: pgmap v10960: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:33:38 smithi153 ceph-mon[70044]: pgmap v10960: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:40.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:33:40 smithi083 ceph-mon[77242]: pgmap v10961: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:33:40 smithi083 ceph-mon[79930]: pgmap v10961: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:33:40 smithi153 ceph-mon[70044]: pgmap v10961: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:41.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:33:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:33:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:33:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:33:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:33:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:33:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:33:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:33:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:33:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:33:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:33:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:33:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:33:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:33:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:33:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:33:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:33:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:33:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:33:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:33:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:33:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:33:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:33:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:33:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:33:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:33:41] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T11:33:41.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:33:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:33:41.512Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:33:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:33:42 smithi083 ceph-mon[79930]: pgmap v10962: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:42.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:33:42 smithi083 ceph-mon[77242]: pgmap v10962: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:33:42 smithi153 ceph-mon[70044]: pgmap v10962: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:42.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:33:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:33:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:33:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:33:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:33:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:33:44.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:33:44 smithi083 ceph-mon[79930]: pgmap v10963: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:44.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:33:44 smithi083 ceph-mon[77242]: pgmap v10963: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:44.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:33:44 smithi153 ceph-mon[70044]: pgmap v10963: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:45.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:33:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:33:44.884Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:33:45.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:33:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:33:45.608Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:33:46.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:33:46 smithi083 ceph-mon[77242]: pgmap v10964: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:46.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:33:46 smithi083 ceph-mon[79930]: pgmap v10964: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:46.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:33:46 smithi153 ceph-mon[70044]: pgmap v10964: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:48.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:33:48 smithi083 ceph-mon[79930]: pgmap v10965: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:48.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:33:48 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:33:48.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:33:48 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:33:48.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:33:48 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:33:48.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:33:48 smithi083 ceph-mon[77242]: pgmap v10965: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:48.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:33:48 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:33:48.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:33:48 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:33:48.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:33:48 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:33:48.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:33:48 smithi153 ceph-mon[70044]: pgmap v10965: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:48.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:33:48 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:33:48.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:33:48 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:33:48.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:33:48 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:33:49.224 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:33:49.512 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:33:49.513 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 62s ago 6h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:33:49.513 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 9m ago 6h 63.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:33:49.513 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 62s ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:33:49.514 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 9m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:33:49.514 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 9m ago 6h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:33:49.514 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 62s ago 6h 823M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:33:49.514 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 62s ago 6h 1379M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:33:49.515 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 9m ago 6h 578M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:33:49.515 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 62s ago 6h 795M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:33:49.515 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 62s ago 6h 29.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:33:49.515 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 9m ago 6h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:33:49.515 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 62s ago 6h 972M 2420M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:33:49.516 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 62s ago 6h 961M 2420M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:33:49.516 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 62s ago 6h 749M 2420M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:33:49.516 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 62s ago 6h 939M 2420M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:33:49.516 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 9m ago 6h 805M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:33:49.517 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 9m ago 6h 1029M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:33:49.517 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (5h) 9m ago 6h 898M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:33:49.517 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 9m ago 6h 1089M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:33:49.517 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 9m ago 6h 152M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:33:49.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:33:49 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:33:49.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:33:49 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:33:49.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:33:49 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:33:49.871 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:33:49.872 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:33:49.872 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:33:49.872 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:33:49.873 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:33:49.873 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:33:49.873 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:33:49.873 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:33:49.873 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:33:49.874 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:33:49.874 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:33:49.874 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:33:49.875 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:33:49.875 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:33:49.875 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:33:49.875 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:33:49.875 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:33:49.876 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:33:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:33:50 smithi153 ceph-mon[70044]: pgmap v10966: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:33:50 smithi153 ceph-mon[70044]: from='client.56818 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:33:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:33:50 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2024172682' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:33:50.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:33:50 smithi083 ceph-mon[77242]: pgmap v10966: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:50.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:33:50 smithi083 ceph-mon[77242]: from='client.56818 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:33:50.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:33:50 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2024172682' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:33:50.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:33:50 smithi083 ceph-mon[79930]: pgmap v10966: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:50.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:33:50 smithi083 ceph-mon[79930]: from='client.56818 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:33:50.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:33:50 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2024172682' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:33:51.507 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:33:51 smithi083 ceph-mon[79930]: from='client.46893 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:33:51.508 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:33:51 smithi083 ceph-mon[77242]: from='client.46893 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:33:51.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:33:51 smithi153 ceph-mon[70044]: from='client.46893 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:33:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:33:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:33:51] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T11:33:51.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:33:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:33:51.512Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:33:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:33:52 smithi153 ceph-mon[70044]: pgmap v10967: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:52.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:33:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:33:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:33:52.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:33:52 smithi083 ceph-mon[77242]: pgmap v10967: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:52.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:33:52 smithi083 ceph-mon[79930]: pgmap v10967: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:33:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:33:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:33:54.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:33:54 smithi153 ceph-mon[70044]: pgmap v10968: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:54.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:33:54 smithi083 ceph-mon[79930]: pgmap v10968: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:54.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:33:54 smithi083 ceph-mon[77242]: pgmap v10968: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:55.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:33:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:33:54.885Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:33:55.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:33:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:33:55.608Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:33:56.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:33:56 smithi153 ceph-mon[70044]: pgmap v10969: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:56.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:33:56 smithi083 ceph-mon[79930]: pgmap v10969: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:56.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:33:56 smithi083 ceph-mon[77242]: pgmap v10969: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:33:58 smithi153 ceph-mon[70044]: pgmap v10970: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:58.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:33:58 smithi083 ceph-mon[79930]: pgmap v10970: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:33:58.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:33:58 smithi083 ceph-mon[77242]: pgmap v10970: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:34:00 smithi153 ceph-mon[70044]: pgmap v10971: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:00.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:34:00 smithi083 ceph-mon[77242]: pgmap v10971: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:00.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:34:00 smithi083 ceph-mon[79930]: pgmap v10971: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:34:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:34:01] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T11:34:01.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:34:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:34:01.513Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:34:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:34:02 smithi153 ceph-mon[70044]: pgmap v10972: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:02.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:34:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:34:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:34:02.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:34:02 smithi083 ceph-mon[79930]: pgmap v10972: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:02.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:34:02 smithi083 ceph-mon[77242]: pgmap v10972: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:34:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:34:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:34:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:34:04 smithi153 ceph-mon[70044]: pgmap v10973: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:04.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:34:04 smithi083 ceph-mon[79930]: pgmap v10973: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:04.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:34:04 smithi083 ceph-mon[77242]: pgmap v10973: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:05.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:34:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:34:04.885Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:34:05.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:34:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:34:05.609Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:34:06.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:34:06 smithi153 ceph-mon[70044]: pgmap v10974: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:06.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:34:06 smithi083 ceph-mon[77242]: pgmap v10974: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:06.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:34:06 smithi083 ceph-mon[79930]: pgmap v10974: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:34:08 smithi153 ceph-mon[70044]: pgmap v10975: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:08.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:34:08 smithi083 ceph-mon[79930]: pgmap v10975: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:08.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:34:08 smithi083 ceph-mon[77242]: pgmap v10975: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:34:10 smithi153 ceph-mon[70044]: pgmap v10976: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:10.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:34:10 smithi083 ceph-mon[79930]: pgmap v10976: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:10.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:34:10 smithi083 ceph-mon[77242]: pgmap v10976: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:34:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:34:11] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T11:34:11.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:34:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:34:11.513Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:34:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:34:12 smithi153 ceph-mon[70044]: pgmap v10977: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:12.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:34:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:34:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:34:12.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:34:12 smithi083 ceph-mon[79930]: pgmap v10977: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:12.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:34:12 smithi083 ceph-mon[77242]: pgmap v10977: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:34:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:34:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:34:14.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:34:14 smithi153 ceph-mon[70044]: pgmap v10978: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:14.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:34:14 smithi083 ceph-mon[79930]: pgmap v10978: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:14.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:34:14 smithi083 ceph-mon[77242]: pgmap v10978: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:15.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:34:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:34:14.886Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:34:15.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:34:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:34:15.610Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:34:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:34:16 smithi153 ceph-mon[70044]: pgmap v10979: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:16.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:34:16 smithi083 ceph-mon[79930]: pgmap v10979: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:16.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:34:16 smithi083 ceph-mon[77242]: pgmap v10979: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:34:18 smithi153 ceph-mon[70044]: pgmap v10980: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:18.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:34:18 smithi083 ceph-mon[79930]: pgmap v10980: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:18.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:34:18 smithi083 ceph-mon[77242]: pgmap v10980: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:20.199 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:34:20.487 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:34:20.488 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 93s ago 6h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:34:20.488 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 9m ago 6h 63.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:34:20.488 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 93s ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:34:20.489 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 9m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:34:20.489 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 9m ago 6h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:34:20.489 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 93s ago 6h 823M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:34:20.489 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 93s ago 6h 1379M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:34:20.489 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 9m ago 6h 578M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:34:20.490 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 93s ago 6h 795M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:34:20.490 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 93s ago 6h 29.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:34:20.490 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 9m ago 6h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:34:20.490 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 93s ago 6h 972M 2420M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:34:20.490 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 93s ago 6h 961M 2420M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:34:20.491 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 93s ago 6h 749M 2420M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:34:20.491 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 93s ago 6h 939M 2420M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:34:20.491 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 9m ago 6h 805M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:34:20.491 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 9m ago 6h 1029M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:34:20.491 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 9m ago 6h 898M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:34:20.492 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (5h) 9m ago 6h 1089M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:34:20.492 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 9m ago 6h 152M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:34:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:34:20 smithi153 ceph-mon[70044]: pgmap v10981: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:20.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:34:20 smithi083 ceph-mon[79930]: pgmap v10981: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:20.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:34:20 smithi083 ceph-mon[77242]: pgmap v10981: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:20.846 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:34:20.846 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:34:20.846 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:34:20.847 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:34:20.847 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:34:20.847 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:34:20.847 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:34:20.848 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:34:20.848 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:34:20.848 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:34:20.848 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:34:20.849 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:34:20.849 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:34:20.849 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:34:20.849 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:34:20.850 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:34:20.850 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:34:20.850 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:34:21.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:34:21 smithi153 ceph-mon[70044]: from='client.46905 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:34:21.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:34:21 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2056855656' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:34:21.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:34:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:34:21] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T11:34:21.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:34:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:34:21.514Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:34:21.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:34:21 smithi083 ceph-mon[77242]: from='client.46905 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:34:21.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:34:21 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2056855656' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:34:21.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:34:21 smithi083 ceph-mon[79930]: from='client.46905 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:34:21.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:34:21 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2056855656' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:34:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:34:22 smithi153 ceph-mon[70044]: pgmap v10982: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:34:22 smithi153 ceph-mon[70044]: from='client.46911 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:34:22.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:34:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:34:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:34:22.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:34:22 smithi083 ceph-mon[79930]: pgmap v10982: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:22.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:34:22 smithi083 ceph-mon[79930]: from='client.46911 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:34:22.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:34:22 smithi083 ceph-mon[77242]: pgmap v10982: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:22.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:34:22 smithi083 ceph-mon[77242]: from='client.46911 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:34:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:34:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:34:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:34:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:34:24 smithi153 ceph-mon[70044]: pgmap v10983: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:24.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:34:24 smithi083 ceph-mon[79930]: pgmap v10983: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:24.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:34:24 smithi083 ceph-mon[77242]: pgmap v10983: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:25.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:34:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:34:24.887Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:34:25.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:34:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:34:25.610Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:34:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:34:26 smithi153 ceph-mon[70044]: pgmap v10984: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:26.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:34:26 smithi083 ceph-mon[79930]: pgmap v10984: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:26.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:34:26 smithi083 ceph-mon[77242]: pgmap v10984: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:34:28 smithi153 ceph-mon[70044]: pgmap v10985: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:28.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:34:28 smithi083 ceph-mon[79930]: pgmap v10985: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:28.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:34:28 smithi083 ceph-mon[77242]: pgmap v10985: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:34:30 smithi153 ceph-mon[70044]: pgmap v10986: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:30.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:34:30 smithi083 ceph-mon[77242]: pgmap v10986: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:30.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:34:30 smithi083 ceph-mon[79930]: pgmap v10986: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:34:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:34:31] "GET /metrics HTTP/1.1" 200 225229 "" "Prometheus/2.33.4" 2022-09-29T11:34:31.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:34:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:34:31.515Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:34:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:34:32 smithi153 ceph-mon[70044]: pgmap v10987: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:32.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:34:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:34:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:34:32.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:34:32 smithi083 ceph-mon[79930]: pgmap v10987: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:32.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:34:32 smithi083 ceph-mon[77242]: pgmap v10987: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:34:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:34:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:34:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:34:34 smithi153 ceph-mon[70044]: pgmap v10988: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:34.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:34:34 smithi083 ceph-mon[79930]: pgmap v10988: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:34.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:34:34 smithi083 ceph-mon[77242]: pgmap v10988: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:35.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:34:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:34:34.888Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:34:35.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:34:35 smithi153 ceph-mon[70044]: pgmap v10989: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:35.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:34:35 smithi083 ceph-mon[77242]: pgmap v10989: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:35.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:34:35 smithi083 ceph-mon[79930]: pgmap v10989: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:35.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:34:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:34:35.611Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:34:37.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:34:37 smithi153 ceph-mon[70044]: pgmap v10990: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:37.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:34:37 smithi083 ceph-mon[77242]: pgmap v10990: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:37.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:34:37 smithi083 ceph-mon[79930]: pgmap v10990: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:39.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:34:39 smithi153 ceph-mon[70044]: pgmap v10991: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:34:39 smithi083 ceph-mon[79930]: pgmap v10991: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:34:39 smithi083 ceph-mon[77242]: pgmap v10991: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:34:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:34:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:34:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:34:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:34:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:34:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:34:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:34:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:34:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:34:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:34:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:34:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:34:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:34:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:34:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:34:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:34:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:34:40.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:34:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:34:40.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:34:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:34:40.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:34:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:34:41.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:34:41 smithi153 ceph-mon[70044]: pgmap v10992: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:34:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:34:41] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T11:34:41.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:34:41 smithi083 ceph-mon[79930]: pgmap v10992: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:41.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:34:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:34:41.516Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:34:41.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:34:41 smithi083 ceph-mon[77242]: pgmap v10992: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:34:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:34:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:34:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:34:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:34:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:34:43.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:34:43 smithi153 ceph-mon[70044]: pgmap v10993: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:43.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:34:43 smithi083 ceph-mon[77242]: pgmap v10993: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:43.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:34:43 smithi083 ceph-mon[79930]: pgmap v10993: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:45.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:34:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:34:44.889Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:34:45.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:34:45 smithi153 ceph-mon[70044]: pgmap v10994: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:45.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:34:45 smithi083 ceph-mon[79930]: pgmap v10994: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:45.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:34:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:34:45.612Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:34:45.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:34:45 smithi083 ceph-mon[77242]: pgmap v10994: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:47.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:34:47 smithi153 ceph-mon[70044]: pgmap v10995: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:47.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:34:47 smithi083 ceph-mon[79930]: pgmap v10995: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:47.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:34:47 smithi083 ceph-mon[77242]: pgmap v10995: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:48.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:34:48 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:34:48.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:34:48 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:34:48.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:34:48 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:34:48.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:34:48 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:34:48.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:34:48 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:34:48.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:34:48 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:34:48.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:34:48 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:34:48.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:34:48 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:34:48.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:34:48 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:34:49.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:34:49 smithi153 ceph-mon[70044]: pgmap v10996: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:49.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:34:49 smithi083 ceph-mon[79930]: pgmap v10996: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:49.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:34:49 smithi083 ceph-mon[77242]: pgmap v10996: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:51.173 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:34:51.462 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:34:51.462 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 2m ago 6h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:34:51.463 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 10m ago 6h 63.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:34:51.463 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 2m ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:34:51.463 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 10m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:34:51.463 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 10m ago 6h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:34:51.464 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 2m ago 6h 823M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:34:51.464 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 2m ago 6h 1379M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:34:51.464 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 10m ago 6h 578M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:34:51.464 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 2m ago 6h 795M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:34:51.464 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 2m ago 6h 29.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:34:51.465 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 10m ago 6h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:34:51.465 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 2m ago 6h 972M 2420M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:34:51.465 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 2m ago 6h 961M 2420M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:34:51.465 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 2m ago 6h 749M 2420M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:34:51.466 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 2m ago 6h 939M 2420M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:34:51.466 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 10m ago 6h 805M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:34:51.466 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 10m ago 6h 1029M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:34:51.466 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 10m ago 6h 898M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:34:51.467 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 10m ago 6h 1089M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:34:51.467 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 10m ago 6h 152M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:34:51.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:34:51 smithi153 ceph-mon[70044]: pgmap v10997: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:51.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:34:51 smithi153 ceph-mon[70044]: from='client.56851 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:34:51.821 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:34:51.821 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:34:51.822 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:34:51.822 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:34:51.822 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:34:51.822 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:34:51.823 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:34:51.823 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:34:51.823 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:34:51.823 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:34:51.823 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:34:51.824 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:34:51.824 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:34:51.824 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:34:51.824 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:34:51.825 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:34:51.825 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:34:51.825 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:34:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:34:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:34:51] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T11:34:51.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:34:51 smithi083 ceph-mon[79930]: pgmap v10997: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:51.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:34:51 smithi083 ceph-mon[79930]: from='client.56851 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:34:51.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:34:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:34:51.515Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:34:51.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:34:51 smithi083 ceph-mon[77242]: pgmap v10997: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:51.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:34:51 smithi083 ceph-mon[77242]: from='client.56851 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:34:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:34:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:34:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:34:52.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:34:52 smithi153 ceph-mon[70044]: from='client.46929 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:34:52.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:34:52 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3495015281' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:34:52.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:34:52 smithi083 ceph-mon[77242]: from='client.46929 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:34:52.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:34:52 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3495015281' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:34:52.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:34:52 smithi083 ceph-mon[79930]: from='client.46929 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:34:52.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:34:52 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3495015281' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:34:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:34:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:34:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:34:54.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:34:54 smithi153 ceph-mon[70044]: pgmap v10998: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:54.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:34:54 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:34:54.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:34:54 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:34:54.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:34:54 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:34:54.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:34:54 smithi083 ceph-mon[77242]: pgmap v10998: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:54.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:34:54 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:34:54.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:34:54 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:34:54.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:34:54 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:34:54.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:34:54 smithi083 ceph-mon[79930]: pgmap v10998: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:54.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:34:54 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:34:54.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:34:54 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:34:54.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:34:54 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:34:55.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:34:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:34:54.890Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:34:55.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:34:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:34:55.612Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:34:56.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:34:56 smithi153 ceph-mon[70044]: pgmap v10999: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:56.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:34:56 smithi083 ceph-mon[79930]: pgmap v10999: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:56.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:34:56 smithi083 ceph-mon[77242]: pgmap v10999: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:58.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:34:58 smithi083 ceph-mon[79930]: pgmap v11000: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:58.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:34:58 smithi083 ceph-mon[77242]: pgmap v11000: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:34:58.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:34:58 smithi153 ceph-mon[70044]: pgmap v11000: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:00.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:35:00 smithi083 ceph-mon[79930]: pgmap v11001: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:00.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:35:00 smithi083 ceph-mon[77242]: pgmap v11001: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:35:00 smithi153 ceph-mon[70044]: pgmap v11001: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:35:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:35:01] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T11:35:01.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:35:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:35:01.516Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:35:02.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:35:02 smithi083 ceph-mon[79930]: pgmap v11002: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:02.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:35:02 smithi083 ceph-mon[77242]: pgmap v11002: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:35:02 smithi153 ceph-mon[70044]: pgmap v11002: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:02.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:35:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:35:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:35:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:35:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:35:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:35:04.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:35:04 smithi083 ceph-mon[77242]: pgmap v11003: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:04.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:35:04 smithi083 ceph-mon[79930]: pgmap v11003: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:35:04 smithi153 ceph-mon[70044]: pgmap v11003: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:05.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:35:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:35:04.890Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:35:05.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:35:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:35:05.614Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:35:06.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:35:06 smithi083 ceph-mon[79930]: pgmap v11004: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:06.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:35:06 smithi083 ceph-mon[77242]: pgmap v11004: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:06.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:35:06 smithi153 ceph-mon[70044]: pgmap v11004: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:08.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:35:08 smithi083 ceph-mon[79930]: pgmap v11005: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:08.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:35:08 smithi083 ceph-mon[77242]: pgmap v11005: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:35:08 smithi153 ceph-mon[70044]: pgmap v11005: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:10.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:35:10 smithi083 ceph-mon[79930]: pgmap v11006: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:10.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:35:10 smithi083 ceph-mon[77242]: pgmap v11006: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:35:10 smithi153 ceph-mon[70044]: pgmap v11006: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:11.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:35:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:35:11.517Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:35:11.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:35:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:35:11] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T11:35:12.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:35:12 smithi083 ceph-mon[77242]: pgmap v11007: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:12.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:35:12 smithi083 ceph-mon[79930]: pgmap v11007: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:35:12 smithi153 ceph-mon[70044]: pgmap v11007: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:12.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:35:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:35:12.320Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:35:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:35:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:35:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:35:14.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:35:14 smithi083 ceph-mon[79930]: pgmap v11008: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:14.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:35:14 smithi083 ceph-mon[77242]: pgmap v11008: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:14.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:35:14 smithi153 ceph-mon[70044]: pgmap v11008: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:15.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:35:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:35:14.890Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:35:15.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:35:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:35:15.613Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:35:16.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:35:16 smithi083 ceph-mon[79930]: pgmap v11009: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:16.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:35:16 smithi083 ceph-mon[77242]: pgmap v11009: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:35:16 smithi153 ceph-mon[70044]: pgmap v11009: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:18.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:35:18 smithi083 ceph-mon[79930]: pgmap v11010: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:18.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:35:18 smithi083 ceph-mon[77242]: pgmap v11010: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:35:18 smithi153 ceph-mon[70044]: pgmap v11010: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:20.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:35:20 smithi083 ceph-mon[79930]: pgmap v11011: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:20.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:35:20 smithi083 ceph-mon[77242]: pgmap v11011: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:35:20 smithi153 ceph-mon[70044]: pgmap v11011: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:35:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:35:21] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T11:35:21.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:35:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:35:21.518Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:35:22.149 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:35:22.440 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:35:22.441 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 2m ago 6h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:35:22.441 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 29s ago 6h 63.7M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:35:22.441 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 2m ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:35:22.442 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 29s ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:35:22.442 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 29s ago 6h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:35:22.442 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 2m ago 6h 823M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:35:22.442 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 2m ago 6h 1379M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:35:22.443 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 29s ago 6h 595M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:35:22.443 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 2m ago 6h 795M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:35:22.443 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 2m ago 6h 29.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:35:22.443 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 29s ago 6h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:35:22.443 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 2m ago 6h 972M 2420M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:35:22.444 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 2m ago 6h 961M 2420M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:35:22.444 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 2m ago 6h 749M 2420M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:35:22.444 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 2m ago 6h 939M 2420M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:35:22.444 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 29s ago 6h 827M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:35:22.445 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 29s ago 6h 1057M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:35:22.445 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 29s ago 6h 922M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:35:22.445 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 29s ago 6h 1119M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:35:22.445 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 29s ago 6h 154M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:35:22.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:35:22 smithi083 ceph-mon[79930]: pgmap v11012: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:22.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:35:22 smithi083 ceph-mon[77242]: pgmap v11012: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:35:22 smithi153 ceph-mon[70044]: pgmap v11012: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:22.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:35:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:35:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:35:22.796 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:35:22.796 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:35:22.797 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:35:22.797 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:35:22.797 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:35:22.797 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:35:22.798 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:35:22.798 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:35:22.798 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:35:22.799 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:35:22.799 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:35:22.799 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:35:22.799 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:35:22.800 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:35:22.800 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:35:22.800 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:35:22.800 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:35:22.800 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:35:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:35:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:35:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:35:23.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:35:23 smithi153 ceph-mon[70044]: from='client.46941 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:35:23.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:35:23 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/367168914' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:35:23.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:35:23 smithi083 ceph-mon[79930]: from='client.46941 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:35:23.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:35:23 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/367168914' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:35:23.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:35:23 smithi083 ceph-mon[77242]: from='client.46941 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:35:23.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:35:23 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/367168914' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:35:24.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:35:24 smithi083 ceph-mon[79930]: pgmap v11013: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:24.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:35:24 smithi083 ceph-mon[79930]: from='client.46947 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:35:24.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:35:24 smithi083 ceph-mon[77242]: pgmap v11013: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:24.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:35:24 smithi083 ceph-mon[77242]: from='client.46947 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:35:24.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:35:24 smithi153 ceph-mon[70044]: pgmap v11013: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:35:24 smithi153 ceph-mon[70044]: from='client.46947 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:35:25.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:35:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:35:24.891Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:35:25.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:35:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:35:25.617Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:35:26.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:35:26 smithi083 ceph-mon[79930]: pgmap v11014: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:26.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:35:26 smithi083 ceph-mon[77242]: pgmap v11014: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:35:26 smithi153 ceph-mon[70044]: pgmap v11014: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:28.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:35:28 smithi083 ceph-mon[77242]: pgmap v11015: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:28.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:35:28 smithi083 ceph-mon[79930]: pgmap v11015: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:35:28 smithi153 ceph-mon[70044]: pgmap v11015: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:30.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:35:30 smithi083 ceph-mon[77242]: pgmap v11016: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:30.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:35:30 smithi083 ceph-mon[79930]: pgmap v11016: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:35:30 smithi153 ceph-mon[70044]: pgmap v11016: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:35:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:35:31] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T11:35:31.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:35:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:35:31.519Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:35:32.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:35:32 smithi083 ceph-mon[79930]: pgmap v11017: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:32.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:35:32 smithi083 ceph-mon[77242]: pgmap v11017: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:35:32 smithi153 ceph-mon[70044]: pgmap v11017: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:32.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:35:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:35:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:35:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:35:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:35:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:35:34.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:35:34 smithi083 ceph-mon[79930]: pgmap v11018: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:34.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:35:34 smithi083 ceph-mon[77242]: pgmap v11018: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:34.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:35:34 smithi153 ceph-mon[70044]: pgmap v11018: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:35.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:35:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:35:34.891Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:35:35.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:35:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:35:35.615Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:35:36.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:35:36 smithi083 ceph-mon[77242]: pgmap v11019: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:36.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:35:36 smithi083 ceph-mon[79930]: pgmap v11019: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:35:36 smithi153 ceph-mon[70044]: pgmap v11019: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:38.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:35:38 smithi083 ceph-mon[79930]: pgmap v11020: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:38.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:35:38 smithi083 ceph-mon[77242]: pgmap v11020: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:35:38 smithi153 ceph-mon[70044]: pgmap v11020: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:40.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:35:40 smithi083 ceph-mon[77242]: pgmap v11021: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:35:40 smithi083 ceph-mon[79930]: pgmap v11021: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:35:40 smithi153 ceph-mon[70044]: pgmap v11021: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:41.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:35:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:35:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:35:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:35:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:35:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:35:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:35:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:35:41.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:35:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:35:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:35:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:35:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:35:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:35:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:35:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:35:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:35:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:35:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:35:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:35:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:35:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:35:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:35:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:35:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:35:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:35:41] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T11:35:41.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:35:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:35:41.520Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:35:42.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:35:42 smithi083 ceph-mon[77242]: pgmap v11022: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:42.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:35:42 smithi083 ceph-mon[79930]: pgmap v11022: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:35:42 smithi153 ceph-mon[70044]: pgmap v11022: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:42.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:35:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:35:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:35:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:35:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:35:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:35:44.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:35:44 smithi083 ceph-mon[79930]: pgmap v11023: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:44.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:35:44 smithi083 ceph-mon[77242]: pgmap v11023: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:44.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:35:44 smithi153 ceph-mon[70044]: pgmap v11023: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:45.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:35:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:35:44.892Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:35:45.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:35:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:35:45.616Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:35:46.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:35:46 smithi083 ceph-mon[79930]: pgmap v11024: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:46.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:35:46 smithi083 ceph-mon[77242]: pgmap v11024: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:46.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:35:46 smithi153 ceph-mon[70044]: pgmap v11024: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:48.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:35:48 smithi083 ceph-mon[77242]: pgmap v11025: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:48.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:35:48 smithi083 ceph-mon[79930]: pgmap v11025: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:48.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:35:48 smithi153 ceph-mon[70044]: pgmap v11025: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:50.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:35:50 smithi083 ceph-mon[79930]: pgmap v11026: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:50.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:35:50 smithi083 ceph-mon[77242]: pgmap v11026: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:35:50 smithi153 ceph-mon[70044]: pgmap v11026: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:35:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:35:51] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T11:35:51.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:35:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:35:51.521Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:35:52.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:35:52 smithi083 ceph-mon[77242]: pgmap v11027: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:52.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:35:52 smithi083 ceph-mon[79930]: pgmap v11027: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:35:52 smithi153 ceph-mon[70044]: pgmap v11027: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:52.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:35:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:35:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:35:53.122 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:35:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:35:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:35:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:35:53.414 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:35:53.414 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 3m ago 6h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:35:53.415 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 60s ago 6h 63.7M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:35:53.415 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 3m ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:35:53.415 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 60s ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:35:53.415 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 60s ago 6h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:35:53.416 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 3m ago 6h 823M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:35:53.416 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 3m ago 6h 1379M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:35:53.416 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 60s ago 6h 595M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:35:53.416 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 3m ago 6h 795M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:35:53.417 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 3m ago 6h 29.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:35:53.417 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 60s ago 6h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:35:53.417 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 3m ago 6h 972M 2420M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:35:53.417 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 3m ago 6h 961M 2420M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:35:53.418 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 3m ago 6h 749M 2420M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:35:53.418 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 3m ago 6h 939M 2420M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:35:53.418 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 60s ago 6h 827M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:35:53.418 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 60s ago 6h 1057M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:35:53.419 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 60s ago 6h 922M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:35:53.419 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 60s ago 6h 1119M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:35:53.419 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 60s ago 6h 154M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:35:53.783 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:35:53.784 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:35:53.784 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:35:53.784 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:35:53.784 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:35:53.785 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:35:53.785 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:35:53.785 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:35:53.785 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:35:53.786 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:35:53.786 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:35:53.786 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:35:53.787 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:35:53.787 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:35:53.787 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:35:53.787 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:35:53.787 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:35:53.788 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:35:54.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:35:54 smithi083 ceph-mon[79930]: pgmap v11028: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:54.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:35:54 smithi083 ceph-mon[79930]: from='client.56887 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:35:54.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:35:54 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:35:54.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:35:54 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:35:54.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:35:54 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:35:54.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:35:54 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3690355428' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:35:54.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:35:54 smithi083 ceph-mon[77242]: pgmap v11028: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:54.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:35:54 smithi083 ceph-mon[77242]: from='client.56887 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:35:54.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:35:54 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:35:54.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:35:54 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:35:54.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:35:54 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:35:54.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:35:54 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3690355428' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:35:54.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:35:54 smithi153 ceph-mon[70044]: pgmap v11028: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:54.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:35:54 smithi153 ceph-mon[70044]: from='client.56887 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:35:54.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:35:54 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:35:54.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:35:54 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:35:54.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:35:54 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:35:54.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:35:54 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3690355428' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:35:55.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:35:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:35:54.892Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:35:55.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:35:55 smithi153 ceph-mon[70044]: from='client.46965 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:35:55.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:35:55 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:35:55.618 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:35:55 smithi083 ceph-mon[79930]: from='client.46965 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:35:55.618 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:35:55 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:35:55.618 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:35:55 smithi083 ceph-mon[77242]: from='client.46965 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:35:55.619 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:35:55 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:35:55.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:35:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:35:55.616Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:35:56.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:35:56 smithi153 ceph-mon[70044]: pgmap v11029: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:56.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:35:56 smithi083 ceph-mon[79930]: pgmap v11029: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:56.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:35:56 smithi083 ceph-mon[77242]: pgmap v11029: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:35:58 smithi153 ceph-mon[70044]: pgmap v11030: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:58.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:35:58 smithi083 ceph-mon[79930]: pgmap v11030: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:35:58.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:35:58 smithi083 ceph-mon[77242]: pgmap v11030: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:36:00 smithi153 ceph-mon[70044]: pgmap v11031: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:00.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:36:00 smithi083 ceph-mon[79930]: pgmap v11031: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:00.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:36:00 smithi083 ceph-mon[77242]: pgmap v11031: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:36:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:36:01] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T11:36:01.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:36:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:36:01.522Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:36:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:36:02 smithi153 ceph-mon[70044]: pgmap v11032: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:02.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:36:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:36:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:36:02.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:36:02 smithi083 ceph-mon[79930]: pgmap v11032: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:02.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:36:02 smithi083 ceph-mon[77242]: pgmap v11032: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:36:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:36:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:36:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:36:04 smithi153 ceph-mon[70044]: pgmap v11033: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:04.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:36:04 smithi083 ceph-mon[79930]: pgmap v11033: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:04.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:36:04 smithi083 ceph-mon[77242]: pgmap v11033: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:05.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:36:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:36:04.893Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:36:05.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:36:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:36:05.617Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:36:06.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:36:06 smithi153 ceph-mon[70044]: pgmap v11034: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:06.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:36:06 smithi083 ceph-mon[77242]: pgmap v11034: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:06.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:36:06 smithi083 ceph-mon[79930]: pgmap v11034: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:36:08 smithi153 ceph-mon[70044]: pgmap v11035: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:08.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:36:08 smithi083 ceph-mon[77242]: pgmap v11035: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:08.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:36:08 smithi083 ceph-mon[79930]: pgmap v11035: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:36:10 smithi153 ceph-mon[70044]: pgmap v11036: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:10.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:36:10 smithi083 ceph-mon[77242]: pgmap v11036: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:10.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:36:10 smithi083 ceph-mon[79930]: pgmap v11036: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:36:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:36:11] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T11:36:11.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:36:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:36:11.522Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:36:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:36:12 smithi153 ceph-mon[70044]: pgmap v11037: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:12.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:36:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:36:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:36:12.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:36:12 smithi083 ceph-mon[79930]: pgmap v11037: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:12.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:36:12 smithi083 ceph-mon[77242]: pgmap v11037: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:36:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:36:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:36:14.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:36:14 smithi153 ceph-mon[70044]: pgmap v11038: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:14.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:36:14 smithi083 ceph-mon[79930]: pgmap v11038: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:14.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:36:14 smithi083 ceph-mon[77242]: pgmap v11038: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:15.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:36:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:36:14.894Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:36:15.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:36:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:36:15.616Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:36:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:36:16 smithi153 ceph-mon[70044]: pgmap v11039: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:16.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:36:16 smithi083 ceph-mon[79930]: pgmap v11039: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:16.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:36:16 smithi083 ceph-mon[77242]: pgmap v11039: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:36:18 smithi153 ceph-mon[70044]: pgmap v11040: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:18.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:36:18 smithi083 ceph-mon[79930]: pgmap v11040: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:18.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:36:18 smithi083 ceph-mon[77242]: pgmap v11040: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:36:20 smithi153 ceph-mon[70044]: pgmap v11041: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:20.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:36:20 smithi083 ceph-mon[77242]: pgmap v11041: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:20.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:36:20 smithi083 ceph-mon[79930]: pgmap v11041: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:36:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:36:21] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T11:36:21.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:36:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:36:21.523Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:36:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:36:22 smithi153 ceph-mon[70044]: pgmap v11042: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:22.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:36:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:36:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:36:22.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:36:22 smithi083 ceph-mon[79930]: pgmap v11042: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:22.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:36:22 smithi083 ceph-mon[77242]: pgmap v11042: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:36:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:36:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:36:24.118 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:36:24.411 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:36:24.411 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 3m ago 6h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:36:24.412 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 91s ago 6h 63.7M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:36:24.412 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 3m ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:36:24.412 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 91s ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:36:24.412 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 91s ago 6h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:36:24.412 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 3m ago 6h 823M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:36:24.413 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 3m ago 6h 1379M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:36:24.413 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 91s ago 6h 595M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:36:24.413 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 3m ago 6h 795M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:36:24.413 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 3m ago 6h 29.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:36:24.414 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 91s ago 6h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:36:24.414 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 3m ago 6h 972M 2420M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:36:24.414 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 3m ago 6h 961M 2420M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:36:24.414 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 3m ago 6h 749M 2420M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:36:24.414 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 3m ago 6h 939M 2420M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:36:24.415 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 91s ago 6h 827M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:36:24.415 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 91s ago 6h 1057M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:36:24.415 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 91s ago 6h 922M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:36:24.415 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 91s ago 6h 1119M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:36:24.415 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 91s ago 6h 154M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:36:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:36:24 smithi153 ceph-mon[70044]: pgmap v11043: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:24.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:36:24 smithi083 ceph-mon[79930]: pgmap v11043: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:24.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:36:24 smithi083 ceph-mon[77242]: pgmap v11043: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:24.767 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:36:24.768 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:36:24.768 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:36:24.768 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:36:24.768 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:36:24.769 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:36:24.769 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:36:24.769 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:36:24.769 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:36:24.770 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:36:24.770 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:36:24.770 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:36:24.770 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:36:24.770 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:36:24.771 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:36:24.771 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:36:24.771 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:36:24.771 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:36:25.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:36:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:36:24.894Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:36:25.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:36:25 smithi153 ceph-mon[70044]: from='client.46977 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:36:25.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:36:25 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3444315470' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:36:25.619 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:36:25 smithi083 ceph-mon[77242]: from='client.46977 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:36:25.619 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:36:25 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3444315470' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:36:25.620 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:36:25 smithi083 ceph-mon[79930]: from='client.46977 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:36:25.620 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:36:25 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3444315470' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:36:25.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:36:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:36:25.617Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:36:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:36:26 smithi153 ceph-mon[70044]: pgmap v11044: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:36:26 smithi153 ceph-mon[70044]: from='client.56911 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:36:26.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:36:26 smithi083 ceph-mon[77242]: pgmap v11044: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:26.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:36:26 smithi083 ceph-mon[77242]: from='client.56911 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:36:26.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:36:26 smithi083 ceph-mon[79930]: pgmap v11044: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:26.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:36:26 smithi083 ceph-mon[79930]: from='client.56911 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:36:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:36:28 smithi153 ceph-mon[70044]: pgmap v11045: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:28.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:36:28 smithi083 ceph-mon[79930]: pgmap v11045: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:28.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:36:28 smithi083 ceph-mon[77242]: pgmap v11045: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:36:30 smithi153 ceph-mon[70044]: pgmap v11046: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:30.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:36:30 smithi083 ceph-mon[79930]: pgmap v11046: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:30.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:36:30 smithi083 ceph-mon[77242]: pgmap v11046: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:36:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:36:31] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T11:36:31.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:36:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:36:31.523Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:36:32.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:36:32 smithi083 ceph-mon[79930]: pgmap v11047: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:32.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:36:32 smithi083 ceph-mon[77242]: pgmap v11047: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:32.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:36:32 smithi153 ceph-mon[70044]: pgmap v11047: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:36:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:36:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:36:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:36:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:36:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:36:34.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:36:34 smithi083 ceph-mon[77242]: pgmap v11048: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:34.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:36:34 smithi083 ceph-mon[79930]: pgmap v11048: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:34.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:36:34 smithi153 ceph-mon[70044]: pgmap v11048: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:35.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:36:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:36:34.895Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:36:35.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:36:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:36:35.618Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:36:36.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:36:36 smithi083 ceph-mon[77242]: pgmap v11049: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:36.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:36:36 smithi083 ceph-mon[79930]: pgmap v11049: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:36.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:36:36 smithi153 ceph-mon[70044]: pgmap v11049: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:38.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:36:38 smithi083 ceph-mon[79930]: pgmap v11050: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:38.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:36:38 smithi083 ceph-mon[77242]: pgmap v11050: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:38.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:36:38 smithi153 ceph-mon[70044]: pgmap v11050: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:36:40 smithi083 ceph-mon[79930]: pgmap v11051: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:36:40 smithi083 ceph-mon[77242]: pgmap v11051: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:36:40 smithi153 ceph-mon[70044]: pgmap v11051: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:41.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:36:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:36:41] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T11:36:41.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:36:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:36:41.524Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:36:41.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:36:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:36:41.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:36:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:36:41.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:36:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:36:41.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:36:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:36:41.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:36:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:36:41.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:36:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:36:41.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:36:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:36:41.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:36:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:36:41.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:36:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:36:41.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:36:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:36:41.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:36:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:36:41.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:36:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:36:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:36:42 smithi083 ceph-mon[79930]: pgmap v11052: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:36:42 smithi083 ceph-mon[77242]: pgmap v11052: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:36:42 smithi153 ceph-mon[70044]: pgmap v11052: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:36:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:36:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:36:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:36:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:36:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:36:43.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:36:43 smithi083 ceph-mon[79930]: pgmap v11053: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:43.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:36:43 smithi083 ceph-mon[77242]: pgmap v11053: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:43.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:36:43 smithi153 ceph-mon[70044]: pgmap v11053: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:45.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:36:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:36:44.896Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:36:45.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:36:45 smithi153 ceph-mon[70044]: pgmap v11054: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:45.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:36:45 smithi083 ceph-mon[79930]: pgmap v11054: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:45.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:36:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:36:45.619Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:36:45.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:36:45 smithi083 ceph-mon[77242]: pgmap v11054: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:47.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:36:47 smithi153 ceph-mon[70044]: pgmap v11055: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:47.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:36:47 smithi083 ceph-mon[79930]: pgmap v11055: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:47.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:36:47 smithi083 ceph-mon[77242]: pgmap v11055: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:49.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:36:49 smithi153 ceph-mon[70044]: pgmap v11056: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:49.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:36:49 smithi083 ceph-mon[79930]: pgmap v11056: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:49.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:36:49 smithi083 ceph-mon[77242]: pgmap v11056: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:51.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:36:51 smithi153 ceph-mon[70044]: pgmap v11057: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:36:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:36:51] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T11:36:51.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:36:51 smithi083 ceph-mon[79930]: pgmap v11057: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:51.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:36:51 smithi083 ceph-mon[77242]: pgmap v11057: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:51.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:36:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:36:51.524Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:36:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:36:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:36:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:36:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:36:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:36:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:36:53.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:36:53 smithi153 ceph-mon[70044]: pgmap v11058: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:53.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:36:53 smithi083 ceph-mon[79930]: pgmap v11058: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:53.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:36:53 smithi083 ceph-mon[77242]: pgmap v11058: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:54.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:36:54 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:36:54.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:36:54 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:36:54.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:36:54 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:36:54.904 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:36:54 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:36:54.905 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:36:54 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:36:54.905 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:36:54 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:36:54.906 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:36:54 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:36:54.906 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:36:54 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:36:54.906 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:36:54 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:36:55.103 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:36:55.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:36:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:36:54.902Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:36:55.391 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:36:55.391 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 4m ago 6h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:36:55.392 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 2m ago 6h 63.7M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:36:55.392 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 4m ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:36:55.392 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 2m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:36:55.393 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 2m ago 6h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:36:55.393 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 4m ago 6h 823M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:36:55.393 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 4m ago 6h 1379M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:36:55.393 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 2m ago 6h 595M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:36:55.394 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 4m ago 6h 795M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:36:55.394 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 4m ago 6h 29.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:36:55.394 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 2m ago 6h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:36:55.394 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 4m ago 6h 972M 2420M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:36:55.394 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 4m ago 6h 961M 2420M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:36:55.395 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 4m ago 6h 749M 2420M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:36:55.395 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 4m ago 6h 939M 2420M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:36:55.395 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 2m ago 6h 827M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:36:55.395 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 2m ago 6h 1057M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:36:55.396 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 2m ago 6h 922M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:36:55.396 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 2m ago 6h 1119M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:36:55.396 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 2m ago 6h 154M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:36:55.747 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:36:55.747 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:36:55.748 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:36:55.748 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:36:55.748 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:36:55.748 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:36:55.749 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:36:55.749 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:36:55.749 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:36:55.749 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:36:55.750 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:36:55.750 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:36:55.750 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:36:55.750 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:36:55.751 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:36:55.751 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:36:55.751 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:36:55.751 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:36:55.959 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:36:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:36:55.619Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:36:56.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:36:55 smithi083 ceph-mon[79930]: pgmap v11059: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:56.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:36:55 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:36:56.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:36:55 smithi083 ceph-mon[79930]: from='client.56923 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:36:56.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:36:55 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2003024233' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:36:56.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:36:55 smithi083 ceph-mon[77242]: pgmap v11059: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:56.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:36:55 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:36:56.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:36:55 smithi083 ceph-mon[77242]: from='client.56923 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:36:56.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:36:55 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2003024233' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:36:56.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:36:55 smithi153 ceph-mon[70044]: pgmap v11059: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:56.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:36:55 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:36:56.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:36:55 smithi153 ceph-mon[70044]: from='client.56923 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:36:56.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:36:55 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2003024233' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:36:57.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:36:56 smithi153 ceph-mon[70044]: from='client.56929 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:36:57.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:36:56 smithi083 ceph-mon[79930]: from='client.56929 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:36:57.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:36:56 smithi083 ceph-mon[77242]: from='client.56929 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:36:58.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:36:57 smithi153 ceph-mon[70044]: pgmap v11060: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:58.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:36:57 smithi083 ceph-mon[79930]: pgmap v11060: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:36:58.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:36:57 smithi083 ceph-mon[77242]: pgmap v11060: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:36:59 smithi153 ceph-mon[70044]: pgmap v11061: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:00.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:36:59 smithi083 ceph-mon[79930]: pgmap v11061: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:00.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:36:59 smithi083 ceph-mon[77242]: pgmap v11061: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:37:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:37:01] "GET /metrics HTTP/1.1" 200 225228 "" "Prometheus/2.33.4" 2022-09-29T11:37:01.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:37:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:37:01.525Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:37:02.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:37:01 smithi153 ceph-mon[70044]: pgmap v11062: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:02.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:37:01 smithi083 ceph-mon[77242]: pgmap v11062: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:02.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:37:01 smithi083 ceph-mon[79930]: pgmap v11062: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:37:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:37:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:37:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:37:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:37:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:37:04.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:37:03 smithi153 ceph-mon[70044]: pgmap v11063: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:04.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:37:03 smithi083 ceph-mon[79930]: pgmap v11063: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:04.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:37:03 smithi083 ceph-mon[77242]: pgmap v11063: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:05.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:37:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:37:04.897Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:37:05.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:37:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:37:05.620Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:37:06.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:37:05 smithi153 ceph-mon[70044]: pgmap v11064: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:06.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:37:05 smithi083 ceph-mon[79930]: pgmap v11064: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:06.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:37:05 smithi083 ceph-mon[77242]: pgmap v11064: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:08.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:37:07 smithi153 ceph-mon[70044]: pgmap v11065: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:08.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:37:07 smithi083 ceph-mon[79930]: pgmap v11065: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:08.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:37:07 smithi083 ceph-mon[77242]: pgmap v11065: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:10.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:37:09 smithi153 ceph-mon[70044]: pgmap v11066: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:10.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:37:09 smithi083 ceph-mon[79930]: pgmap v11066: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:10.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:37:09 smithi083 ceph-mon[77242]: pgmap v11066: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:37:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:37:11] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T11:37:11.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:37:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:37:11.526Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:37:12.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:37:11 smithi153 ceph-mon[70044]: pgmap v11067: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:12.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:37:11 smithi083 ceph-mon[79930]: pgmap v11067: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:12.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:37:11 smithi083 ceph-mon[77242]: pgmap v11067: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:37:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:37:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:37:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:37:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:37:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:37:14.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:37:13 smithi153 ceph-mon[70044]: pgmap v11068: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:14.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:37:13 smithi083 ceph-mon[79930]: pgmap v11068: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:14.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:37:13 smithi083 ceph-mon[77242]: pgmap v11068: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:15.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:37:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:37:14.897Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:37:15.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:37:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:37:15.620Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:37:16.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:37:16 smithi153 ceph-mon[70044]: pgmap v11069: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:16.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:37:15 smithi083 ceph-mon[79930]: pgmap v11069: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:16.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:37:15 smithi083 ceph-mon[77242]: pgmap v11069: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:18.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:37:18 smithi153 ceph-mon[70044]: pgmap v11070: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:18.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:37:18 smithi083 ceph-mon[79930]: pgmap v11070: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:18.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:37:18 smithi083 ceph-mon[77242]: pgmap v11070: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:37:20 smithi153 ceph-mon[70044]: pgmap v11071: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:20.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:37:20 smithi083 ceph-mon[79930]: pgmap v11071: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:20.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:37:20 smithi083 ceph-mon[77242]: pgmap v11071: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:37:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:37:21] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T11:37:21.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:37:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:37:21.526Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:37:22.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:37:22 smithi153 ceph-mon[70044]: pgmap v11072: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:22.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:37:22 smithi083 ceph-mon[79930]: pgmap v11072: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:22.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:37:22 smithi083 ceph-mon[77242]: pgmap v11072: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:37:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:37:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:37:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:37:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:37:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:37:24.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:37:24 smithi153 ceph-mon[70044]: pgmap v11073: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:24.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:37:24 smithi083 ceph-mon[79930]: pgmap v11073: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:24.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:37:24 smithi083 ceph-mon[77242]: pgmap v11073: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:25.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:37:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:37:24.897Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:37:25.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:37:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:37:25.622Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:37:26.074 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:37:26.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:37:26 smithi153 ceph-mon[70044]: pgmap v11074: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:26.361 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:37:26.361 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 4m ago 6h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:37:26.361 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 2m ago 6h 63.7M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:37:26.362 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 4m ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:37:26.362 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 2m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:37:26.362 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 2m ago 6h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:37:26.362 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 4m ago 6h 823M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:37:26.362 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 4m ago 6h 1379M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:37:26.363 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 2m ago 6h 595M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:37:26.363 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 4m ago 6h 795M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:37:26.363 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 4m ago 6h 29.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:37:26.363 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 2m ago 6h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:37:26.364 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 4m ago 6h 972M 2420M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:37:26.364 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 4m ago 6h 961M 2420M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:37:26.364 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 4m ago 6h 749M 2420M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:37:26.365 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 4m ago 6h 939M 2420M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:37:26.365 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 2m ago 6h 827M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:37:26.366 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 2m ago 6h 1057M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:37:26.366 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 2m ago 6h 922M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:37:26.367 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 2m ago 6h 1119M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:37:26.367 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 2m ago 6h 154M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:37:26.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:37:26 smithi083 ceph-mon[79930]: pgmap v11074: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:26.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:37:26 smithi083 ceph-mon[77242]: pgmap v11074: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:26.718 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:37:26.718 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:37:26.719 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:37:26.719 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:37:26.719 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:37:26.719 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:37:26.719 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:37:26.720 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:37:26.720 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:37:26.720 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:37:26.720 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:37:26.720 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:37:26.721 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:37:26.721 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:37:26.721 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:37:26.721 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:37:26.721 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:37:26.722 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:37:27.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:37:27 smithi153 ceph-mon[70044]: from='client.56941 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:37:27.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:37:27 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1925229608' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:37:27.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:37:27 smithi083 ceph-mon[79930]: from='client.56941 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:37:27.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:37:27 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1925229608' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:37:27.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:37:27 smithi083 ceph-mon[77242]: from='client.56941 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:37:27.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:37:27 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1925229608' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:37:28.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:37:28 smithi153 ceph-mon[70044]: from='client.47016 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:37:28.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:37:28 smithi153 ceph-mon[70044]: pgmap v11075: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:28.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:37:28 smithi083 ceph-mon[79930]: from='client.47016 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:37:28.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:37:28 smithi083 ceph-mon[79930]: pgmap v11075: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:28.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:37:28 smithi083 ceph-mon[77242]: from='client.47016 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:37:28.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:37:28 smithi083 ceph-mon[77242]: pgmap v11075: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:30.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:37:30 smithi153 ceph-mon[70044]: pgmap v11076: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:30.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:37:30 smithi083 ceph-mon[77242]: pgmap v11076: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:30.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:37:30 smithi083 ceph-mon[79930]: pgmap v11076: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:37:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:37:31] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T11:37:31.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:37:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:37:31.527Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:37:32.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:37:32 smithi153 ceph-mon[70044]: pgmap v11077: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:32.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:37:32 smithi083 ceph-mon[79930]: pgmap v11077: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:32.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:37:32 smithi083 ceph-mon[77242]: pgmap v11077: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:37:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:37:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:37:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:37:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:37:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:37:34.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:37:34 smithi153 ceph-mon[70044]: pgmap v11078: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:34.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:37:34 smithi083 ceph-mon[79930]: pgmap v11078: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:34.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:37:34 smithi083 ceph-mon[77242]: pgmap v11078: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:35.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:37:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:37:34.898Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:37:35.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:37:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:37:35.622Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:37:36.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:37:36 smithi153 ceph-mon[70044]: pgmap v11079: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:36.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:37:36 smithi083 ceph-mon[79930]: pgmap v11079: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:36.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:37:36 smithi083 ceph-mon[77242]: pgmap v11079: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:38.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:37:38 smithi153 ceph-mon[70044]: pgmap v11080: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:38.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:37:38 smithi083 ceph-mon[79930]: pgmap v11080: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:38.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:37:38 smithi083 ceph-mon[77242]: pgmap v11080: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:37:40 smithi153 ceph-mon[70044]: pgmap v11081: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:37:40 smithi083 ceph-mon[79930]: pgmap v11081: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:37:40 smithi083 ceph-mon[77242]: pgmap v11081: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:37:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:37:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:37:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:37:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:37:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:37:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:37:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:37:41.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:37:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:37:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:37:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:37:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:37:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:37:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:37:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:37:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:37:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:37:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:37:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:37:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:37:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:37:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:37:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:37:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:37:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:37:41] "GET /metrics HTTP/1.1" 200 225225 "" "Prometheus/2.33.4" 2022-09-29T11:37:41.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:37:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:37:41.528Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:37:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:37:42 smithi153 ceph-mon[70044]: pgmap v11082: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:42.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:37:42 smithi083 ceph-mon[77242]: pgmap v11082: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:42.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:37:42 smithi083 ceph-mon[79930]: pgmap v11082: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:37:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:37:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:37:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:37:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:37:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:37:44.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:37:44 smithi153 ceph-mon[70044]: pgmap v11083: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:44.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:37:44 smithi083 ceph-mon[79930]: pgmap v11083: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:44.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:37:44 smithi083 ceph-mon[77242]: pgmap v11083: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:45.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:37:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:37:44.899Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:37:45.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:37:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:37:45.622Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:37:46.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:37:46 smithi153 ceph-mon[70044]: pgmap v11084: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:46.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:37:46 smithi083 ceph-mon[79930]: pgmap v11084: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:46.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:37:46 smithi083 ceph-mon[77242]: pgmap v11084: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:48.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:37:48 smithi083 ceph-mon[79930]: pgmap v11085: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:48.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:37:48 smithi083 ceph-mon[77242]: pgmap v11085: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:48.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:37:48 smithi153 ceph-mon[70044]: pgmap v11085: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:50.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:37:50 smithi083 ceph-mon[79930]: pgmap v11086: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:50.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:37:50 smithi083 ceph-mon[77242]: pgmap v11086: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:37:50 smithi153 ceph-mon[70044]: pgmap v11086: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:37:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:37:51] "GET /metrics HTTP/1.1" 200 225225 "" "Prometheus/2.33.4" 2022-09-29T11:37:51.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:37:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:37:51.529Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:37:52.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:37:52 smithi083 ceph-mon[79930]: pgmap v11087: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:52.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:37:52 smithi083 ceph-mon[77242]: pgmap v11087: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:37:52 smithi153 ceph-mon[70044]: pgmap v11087: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:52.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:37:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:37:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:37:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:37:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:37:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:37:54.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:37:54 smithi083 ceph-mon[77242]: pgmap v11088: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:54.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:37:54 smithi083 ceph-mon[79930]: pgmap v11088: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:54.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:37:54 smithi153 ceph-mon[70044]: pgmap v11088: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:55.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:37:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:37:54.899Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:37:55.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:37:55 smithi153 ceph-mon[70044]: pgmap v11089: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:55.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:37:55 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:37:55.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:37:55 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:37:55.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:37:55 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:37:55.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:37:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:37:55.627Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:37:55.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:37:55 smithi083 ceph-mon[79930]: pgmap v11089: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:55.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:37:55 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:37:55.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:37:55 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:37:55.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:37:55 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:37:55.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:37:55 smithi083 ceph-mon[77242]: pgmap v11089: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:55.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:37:55 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:37:55.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:37:55 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:37:55.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:37:55 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:37:56.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:37:56 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:37:56.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:37:56 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:37:57.042 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:37:57.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:37:56 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:37:57.334 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:37:57.334 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 5m ago 6h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:37:57.334 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 3m ago 6h 63.7M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:37:57.335 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 5m ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:37:57.335 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 3m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:37:57.335 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 3m ago 6h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:37:57.335 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 5m ago 6h 823M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:37:57.335 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 5m ago 6h 1379M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:37:57.336 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 3m ago 6h 595M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:37:57.336 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 5m ago 6h 795M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:37:57.336 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 5m ago 6h 29.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:37:57.337 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 3m ago 6h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:37:57.337 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 5m ago 6h 972M 2420M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:37:57.337 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 5m ago 6h 961M 2420M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:37:57.337 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 5m ago 6h 749M 2420M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:37:57.338 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 5m ago 6h 939M 2420M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:37:57.338 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 3m ago 6h 827M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:37:57.338 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 3m ago 6h 1057M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:37:57.338 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 3m ago 6h 922M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:37:57.339 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 3m ago 6h 1119M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:37:57.339 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 3m ago 6h 154M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:37:57.692 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:37:57.693 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:37:57.693 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:37:57.693 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:37:57.694 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:37:57.694 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:37:57.694 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:37:57.695 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:37:57.695 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:37:57.695 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:37:57.695 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:37:57.696 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:37:57.696 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:37:57.696 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:37:57.696 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:37:57.696 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:37:57.697 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:37:57.697 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:37:57.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:37:57 smithi083 ceph-mon[79930]: pgmap v11090: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:57.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:37:57 smithi083 ceph-mon[79930]: from='client.56959 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:37:57.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:37:57 smithi083 ceph-mon[79930]: from='client.56965 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:37:57.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:37:57 smithi083 ceph-mon[77242]: pgmap v11090: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:57.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:37:57 smithi083 ceph-mon[77242]: from='client.56959 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:37:57.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:37:57 smithi083 ceph-mon[77242]: from='client.56965 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:37:58.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:37:57 smithi153 ceph-mon[70044]: pgmap v11090: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:58.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:37:57 smithi153 ceph-mon[70044]: from='client.56959 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:37:58.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:37:57 smithi153 ceph-mon[70044]: from='client.56965 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:37:58.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:37:58 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2081048591' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:37:58.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:37:58 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2081048591' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:37:59.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:37:58 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2081048591' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:37:59.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:37:59 smithi083 ceph-mon[79930]: pgmap v11091: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:37:59.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:37:59 smithi083 ceph-mon[77242]: pgmap v11091: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:38:00.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:37:59 smithi153 ceph-mon[70044]: pgmap v11091: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:38:01.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:38:01 smithi083 ceph-mon[79930]: pgmap v11092: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:38:01.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:38:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:38:01] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T11:38:01.964 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:38:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:38:01.530Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:38:01.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:38:01 smithi083 ceph-mon[77242]: pgmap v11092: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:38:02.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:38:01 smithi153 ceph-mon[70044]: pgmap v11092: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:38:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:38:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:38:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:38:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:38:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:38:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:38:03.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:38:03 smithi083 ceph-mon[79930]: pgmap v11093: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:38:03.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:38:03 smithi083 ceph-mon[77242]: pgmap v11093: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:38:04.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:38:03 smithi153 ceph-mon[70044]: pgmap v11093: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:38:05.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:38:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:38:04.900Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:38:05.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:38:05 smithi083 ceph-mon[79930]: pgmap v11094: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:38:05.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:38:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:38:05.624Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:38:05.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:38:05 smithi083 ceph-mon[77242]: pgmap v11094: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:38:06.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:38:05 smithi153 ceph-mon[70044]: pgmap v11094: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:38:07.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:38:07 smithi083 ceph-mon[79930]: pgmap v11095: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:38:07.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:38:07 smithi083 ceph-mon[77242]: pgmap v11095: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:38:08.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:38:07 smithi153 ceph-mon[70044]: pgmap v11095: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:38:09.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:38:09 smithi083 ceph-mon[79930]: pgmap v11096: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:38:09.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:38:09 smithi083 ceph-mon[77242]: pgmap v11096: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:38:10.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:38:09 smithi153 ceph-mon[70044]: pgmap v11096: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:38:11.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:38:11 smithi083 ceph-mon[79930]: pgmap v11097: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:38:11.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:38:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:38:11] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T11:38:11.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:38:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:38:11.530Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:38:11.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:38:11 smithi083 ceph-mon[77242]: pgmap v11097: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:38:12.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:38:11 smithi153 ceph-mon[70044]: pgmap v11097: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:38:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:38:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:38:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:38:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:38:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:38:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:38:13.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:38:13 smithi083 ceph-mon[79930]: pgmap v11098: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:38:13.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:38:13 smithi083 ceph-mon[77242]: pgmap v11098: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:38:14.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:38:13 smithi153 ceph-mon[70044]: pgmap v11098: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:38:15.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:38:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:38:14.900Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:38:15.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:38:15 smithi083 ceph-mon[77242]: pgmap v11099: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:38:15.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:38:15 smithi083 ceph-mon[79930]: pgmap v11099: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:38:15.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:38:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:38:15.624Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:38:16.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:38:15 smithi153 ceph-mon[70044]: pgmap v11099: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:38:17.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:38:17 smithi083 ceph-mon[79930]: pgmap v11100: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:38:17.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:38:17 smithi083 ceph-mon[77242]: pgmap v11100: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:38:18.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:38:17 smithi153 ceph-mon[70044]: pgmap v11100: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:38:19.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:38:19 smithi083 ceph-mon[79930]: pgmap v11101: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:38:19.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:38:19 smithi083 ceph-mon[77242]: pgmap v11101: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:38:20.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:38:19 smithi153 ceph-mon[70044]: pgmap v11101: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:38:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:38:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:38:21] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T11:38:21.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:38:21 smithi083 ceph-mon[79930]: pgmap v11102: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:38:21.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:38:21 smithi083 ceph-mon[77242]: pgmap v11102: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:38:21.964 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:38:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:38:21.531Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:38:22.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:38:21 smithi153 ceph-mon[70044]: pgmap v11102: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:38:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:38:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:38:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:38:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:38:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:38:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:38:23.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:38:23 smithi083 ceph-mon[77242]: pgmap v11103: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:38:23.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:38:23 smithi083 ceph-mon[79930]: pgmap v11103: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:38:24.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:38:23 smithi153 ceph-mon[70044]: pgmap v11103: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:38:25.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:38:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:38:24.901Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:38:25.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:38:25 smithi083 ceph-mon[79930]: pgmap v11104: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:38:25.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:38:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:38:25.624Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:38:25.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:38:25 smithi083 ceph-mon[77242]: pgmap v11104: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:38:26.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:38:25 smithi153 ceph-mon[70044]: pgmap v11104: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:38:27.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:38:27 smithi083 ceph-mon[79930]: pgmap v11105: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:38:27.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:38:27 smithi083 ceph-mon[77242]: pgmap v11105: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:38:28.021 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:38:28.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:38:27 smithi153 ceph-mon[70044]: pgmap v11105: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:38:28.307 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:38:28.308 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 5m ago 6h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:38:28.308 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 3m ago 6h 63.7M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:38:28.308 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 5m ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:38:28.308 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 3m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:38:28.309 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 3m ago 6h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:38:28.309 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 5m ago 6h 823M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:38:28.309 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 5m ago 6h 1379M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:38:28.309 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 3m ago 6h 595M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:38:28.309 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 5m ago 6h 795M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:38:28.310 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 5m ago 6h 29.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:38:28.310 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 3m ago 6h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:38:28.310 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 5m ago 6h 972M 2420M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:38:28.310 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 5m ago 6h 961M 2420M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:38:28.310 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 5m ago 6h 749M 2420M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:38:28.311 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 5m ago 6h 939M 2420M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:38:28.311 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 3m ago 6h 827M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:38:28.311 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 3m ago 6h 1057M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:38:28.311 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 3m ago 6h 922M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:38:28.311 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 3m ago 6h 1119M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:38:28.312 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 3m ago 6h 154M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:38:28.666 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:38:28.666 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:38:28.666 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:38:28.667 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:38:28.667 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:38:28.667 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:38:28.667 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:38:28.668 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:38:28.668 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:38:28.668 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:38:28.668 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:38:28.668 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:38:28.669 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:38:28.669 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:38:28.669 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:38:28.669 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:38:28.669 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:38:28.670 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:38:28.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:38:28 smithi083 ceph-mon[77242]: from='client.56977 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:38:28.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:38:28 smithi083 ceph-mon[77242]: from='client.56983 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:38:28.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:38:28 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1222029687' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:38:28.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:38:28 smithi083 ceph-mon[79930]: from='client.56977 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:38:28.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:38:28 smithi083 ceph-mon[79930]: from='client.56983 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:38:28.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:38:28 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1222029687' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:38:29.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:38:28 smithi153 ceph-mon[70044]: from='client.56977 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:38:29.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:38:28 smithi153 ceph-mon[70044]: from='client.56983 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:38:29.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:38:28 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1222029687' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:38:29.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:38:29 smithi083 ceph-mon[79930]: pgmap v11106: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:38:29.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:38:29 smithi083 ceph-mon[77242]: pgmap v11106: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:38:30.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:38:29 smithi153 ceph-mon[70044]: pgmap v11106: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:38:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:38:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:38:31] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T11:38:31.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:38:31 smithi083 ceph-mon[79930]: pgmap v11107: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:38:31.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:38:31 smithi083 ceph-mon[77242]: pgmap v11107: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:38:31.964 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:38:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:38:31.532Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:38:32.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:38:31 smithi153 ceph-mon[70044]: pgmap v11107: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:38:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:38:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:38:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:38:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:38:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:38:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:38:34.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:38:33 smithi153 ceph-mon[70044]: pgmap v11108: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:38:34.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:38:33 smithi083 ceph-mon[79930]: pgmap v11108: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:38:34.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:38:33 smithi083 ceph-mon[77242]: pgmap v11108: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:38:35.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:38:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:38:34.903Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:38:35.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:38:35 smithi083 ceph-mon[79930]: pgmap v11109: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:38:35.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:38:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:38:35.625Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:38:35.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:38:35 smithi083 ceph-mon[77242]: pgmap v11109: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:38:36.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:38:35 smithi153 ceph-mon[70044]: pgmap v11109: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:38:38.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:38:37 smithi153 ceph-mon[70044]: pgmap v11110: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:38:38.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:38:37 smithi083 ceph-mon[77242]: pgmap v11110: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:38:38.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:38:37 smithi083 ceph-mon[79930]: pgmap v11110: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:38:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:38:39 smithi153 ceph-mon[70044]: pgmap v11111: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:38:40.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:38:39 smithi083 ceph-mon[77242]: pgmap v11111: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:38:40.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:38:39 smithi083 ceph-mon[79930]: pgmap v11111: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:38:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:38:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:38:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:38:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:38:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:38:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:38:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:38:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:38:41.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:38:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:38:41.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:38:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:38:41.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:38:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:38:41.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:38:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:38:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:38:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:38:41.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:38:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:38:41.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:38:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:38:41.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:38:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:38:41.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:38:41 smithi083 ceph-mon[79930]: pgmap v11112: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:38:41.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:38:41 smithi083 ceph-mon[77242]: pgmap v11112: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:38:41.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:38:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:38:41] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T11:38:41.964 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:38:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:38:41.533Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:38:42.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:38:41 smithi153 ceph-mon[70044]: pgmap v11112: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:38:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:38:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:38:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:38:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:38:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:38:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:38:44.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:38:43 smithi153 ceph-mon[70044]: pgmap v11113: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:38:44.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:38:43 smithi083 ceph-mon[79930]: pgmap v11113: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:38:44.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:38:43 smithi083 ceph-mon[77242]: pgmap v11113: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:38:45.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:38:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:38:44.903Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:38:45.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:38:45 smithi083 ceph-mon[77242]: pgmap v11114: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:38:45.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:38:45 smithi083 ceph-mon[79930]: pgmap v11114: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:38:45.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:38:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:38:45.626Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:38:46.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:38:45 smithi153 ceph-mon[70044]: pgmap v11114: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:38:48.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:38:47 smithi153 ceph-mon[70044]: pgmap v11115: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:38:48.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:38:47 smithi083 ceph-mon[79930]: pgmap v11115: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:38:48.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:38:47 smithi083 ceph-mon[77242]: pgmap v11115: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:38:50.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:38:49 smithi153 ceph-mon[70044]: pgmap v11116: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:38:50.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:38:49 smithi083 ceph-mon[79930]: pgmap v11116: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:38:50.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:38:49 smithi083 ceph-mon[77242]: pgmap v11116: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:38:51.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:38:51 smithi083 ceph-mon[77242]: pgmap v11117: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:38:51.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:38:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:38:51] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T11:38:51.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:38:51 smithi083 ceph-mon[79930]: pgmap v11117: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:38:51.964 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:38:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:38:51.534Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:38:52.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:38:51 smithi153 ceph-mon[70044]: pgmap v11117: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:38:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:38:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:38:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:38:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:38:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:38:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:38:54.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:38:53 smithi153 ceph-mon[70044]: pgmap v11118: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:38:54.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:38:53 smithi083 ceph-mon[79930]: pgmap v11118: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:38:54.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:38:53 smithi083 ceph-mon[77242]: pgmap v11118: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:38:55.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:38:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:38:54.904Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:38:55.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:38:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:38:55.627Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:38:55.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:38:55 smithi083 ceph-mon[77242]: pgmap v11119: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:38:55.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:38:55 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:38:55.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:38:55 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:38:55.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:38:55 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:38:55.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:38:55 smithi083 ceph-mon[79930]: pgmap v11119: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:38:55.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:38:55 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:38:55.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:38:55 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:38:55.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:38:55 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:38:56.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:38:55 smithi153 ceph-mon[70044]: pgmap v11119: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:38:56.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:38:55 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:38:56.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:38:55 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:38:56.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:38:55 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:38:57.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:38:57 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:38:57.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:38:57 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:38:57.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:38:57 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:38:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:38:58 smithi153 ceph-mon[70044]: pgmap v11120: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:38:58.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:38:58 smithi083 ceph-mon[79930]: pgmap v11120: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:38:58.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:38:58 smithi083 ceph-mon[77242]: pgmap v11120: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:38:58.992 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:38:59.282 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:38:59.282 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 6m ago 6h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:38:59.283 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 4m ago 6h 63.7M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:38:59.283 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 6m ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:38:59.283 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 4m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:38:59.283 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 4m ago 6h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:38:59.283 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 6m ago 6h 823M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:38:59.284 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 6m ago 6h 1379M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:38:59.284 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 4m ago 6h 595M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:38:59.284 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 6m ago 6h 795M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:38:59.284 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 6m ago 6h 29.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:38:59.284 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 4m ago 6h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:38:59.285 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 6m ago 6h 972M 2420M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:38:59.285 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 6m ago 6h 961M 2420M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:38:59.285 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 6m ago 6h 749M 2420M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:38:59.285 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 6m ago 6h 939M 2420M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:38:59.286 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 4m ago 6h 827M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:38:59.286 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 4m ago 6h 1057M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:38:59.286 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 4m ago 6h 922M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:38:59.286 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 4m ago 6h 1119M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:38:59.287 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 4m ago 6h 154M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:38:59.642 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:38:59.642 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:38:59.642 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:38:59.643 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:38:59.643 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:38:59.643 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:38:59.643 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:38:59.644 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:38:59.644 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:38:59.644 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:38:59.644 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:38:59.644 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:38:59.645 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:38:59.645 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:38:59.645 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:38:59.645 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:38:59.646 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:38:59.646 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:39:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:39:00 smithi153 ceph-mon[70044]: pgmap v11121: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:39:00 smithi153 ceph-mon[70044]: from='client.56995 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:39:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:39:00 smithi153 ceph-mon[70044]: from='client.57001 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:39:00.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:39:00 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3680124203' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:39:00.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:39:00 smithi083 ceph-mon[79930]: pgmap v11121: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:00.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:39:00 smithi083 ceph-mon[79930]: from='client.56995 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:39:00.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:39:00 smithi083 ceph-mon[79930]: from='client.57001 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:39:00.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:39:00 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3680124203' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:39:00.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:39:00 smithi083 ceph-mon[77242]: pgmap v11121: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:00.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:39:00 smithi083 ceph-mon[77242]: from='client.56995 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:39:00.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:39:00 smithi083 ceph-mon[77242]: from='client.57001 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:39:00.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:39:00 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3680124203' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:39:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:39:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:39:01] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T11:39:01.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:39:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:39:01.534Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:39:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:39:02 smithi153 ceph-mon[70044]: pgmap v11122: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:02.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:39:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:39:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:39:02.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:39:02 smithi083 ceph-mon[79930]: pgmap v11122: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:02.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:39:02 smithi083 ceph-mon[77242]: pgmap v11122: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:39:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:39:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:39:04.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:39:04 smithi083 ceph-mon[79930]: pgmap v11123: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:04.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:39:04 smithi083 ceph-mon[77242]: pgmap v11123: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:04.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:39:04 smithi153 ceph-mon[70044]: pgmap v11123: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:05.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:39:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:39:04.905Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:39:05.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:39:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:39:05.628Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:39:06.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:39:06 smithi083 ceph-mon[77242]: pgmap v11124: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:06.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:39:06 smithi083 ceph-mon[79930]: pgmap v11124: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:06.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:39:06 smithi153 ceph-mon[70044]: pgmap v11124: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:08.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:39:08 smithi083 ceph-mon[79930]: pgmap v11125: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:08.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:39:08 smithi083 ceph-mon[77242]: pgmap v11125: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:08.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:39:08 smithi153 ceph-mon[70044]: pgmap v11125: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:10.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:39:10 smithi083 ceph-mon[79930]: pgmap v11126: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:10.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:39:10 smithi083 ceph-mon[77242]: pgmap v11126: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:10.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:39:10 smithi153 ceph-mon[70044]: pgmap v11126: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:39:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:39:11] "GET /metrics HTTP/1.1" 200 225229 "" "Prometheus/2.33.4" 2022-09-29T11:39:11.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:39:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:39:11.535Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:39:12.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:39:12 smithi083 ceph-mon[79930]: pgmap v11127: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:12.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:39:12 smithi083 ceph-mon[77242]: pgmap v11127: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:12.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:39:12 smithi153 ceph-mon[70044]: pgmap v11127: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:39:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:39:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:39:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:39:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:39:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:39:14.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:39:14 smithi083 ceph-mon[79930]: pgmap v11128: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:14.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:39:14 smithi083 ceph-mon[77242]: pgmap v11128: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:14.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:39:14 smithi153 ceph-mon[70044]: pgmap v11128: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:15.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:39:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:39:14.906Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:39:15.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:39:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:39:15.628Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:39:16.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:39:16 smithi083 ceph-mon[79930]: pgmap v11129: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:16.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:39:16 smithi083 ceph-mon[77242]: pgmap v11129: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:16.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:39:16 smithi153 ceph-mon[70044]: pgmap v11129: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:18.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:39:18 smithi083 ceph-mon[79930]: pgmap v11130: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:18.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:39:18 smithi083 ceph-mon[77242]: pgmap v11130: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:18.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:39:18 smithi153 ceph-mon[70044]: pgmap v11130: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:20.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:39:20 smithi083 ceph-mon[79930]: pgmap v11131: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:20.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:39:20 smithi083 ceph-mon[77242]: pgmap v11131: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:20.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:39:20 smithi153 ceph-mon[70044]: pgmap v11131: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:39:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:39:21] "GET /metrics HTTP/1.1" 200 225229 "" "Prometheus/2.33.4" 2022-09-29T11:39:21.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:39:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:39:21.536Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:39:22.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:39:22 smithi083 ceph-mon[77242]: pgmap v11132: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:22.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:39:22 smithi083 ceph-mon[79930]: pgmap v11132: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:22.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:39:22 smithi153 ceph-mon[70044]: pgmap v11132: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:39:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:39:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:39:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:39:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:39:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:39:24.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:39:24 smithi083 ceph-mon[79930]: pgmap v11133: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:24.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:39:24 smithi083 ceph-mon[77242]: pgmap v11133: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:24.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:39:24 smithi153 ceph-mon[70044]: pgmap v11133: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:25.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:39:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:39:24.907Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:39:25.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:39:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:39:25.629Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:39:26.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:39:26 smithi083 ceph-mon[79930]: pgmap v11134: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:26.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:39:26 smithi083 ceph-mon[77242]: pgmap v11134: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:26.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:39:26 smithi153 ceph-mon[70044]: pgmap v11134: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:28.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:39:28 smithi083 ceph-mon[77242]: pgmap v11135: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:28.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:39:28 smithi083 ceph-mon[79930]: pgmap v11135: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:28.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:39:28 smithi153 ceph-mon[70044]: pgmap v11135: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:29.967 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:39:30.253 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:39:30.253 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 6m ago 6h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:39:30.254 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 4m ago 6h 63.7M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:39:30.254 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 6m ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:39:30.254 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 4m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:39:30.254 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 4m ago 6h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:39:30.255 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 6m ago 6h 823M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:39:30.255 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 6m ago 6h 1379M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:39:30.255 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 4m ago 6h 595M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:39:30.255 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 6m ago 6h 795M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:39:30.256 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 6m ago 6h 29.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:39:30.256 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 4m ago 6h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:39:30.256 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 6m ago 6h 972M 2420M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:39:30.256 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 6m ago 6h 961M 2420M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:39:30.256 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 6m ago 6h 749M 2420M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:39:30.257 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 6m ago 6h 939M 2420M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:39:30.257 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 4m ago 6h 827M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:39:30.257 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 4m ago 6h 1057M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:39:30.257 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 4m ago 6h 922M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:39:30.258 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 4m ago 6h 1119M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:39:30.258 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 4m ago 6h 154M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:39:30.611 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:39:30.611 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:39:30.611 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:39:30.611 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:39:30.612 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:39:30.612 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:39:30.612 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:39:30.612 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:39:30.612 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:39:30.613 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:39:30.613 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:39:30.613 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:39:30.613 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:39:30.613 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:39:30.614 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:39:30.614 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:39:30.614 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:39:30.614 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:39:30.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:39:30 smithi083 ceph-mon[79930]: pgmap v11136: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:30.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:39:30 smithi083 ceph-mon[77242]: pgmap v11136: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:30.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:39:30 smithi153 ceph-mon[70044]: pgmap v11136: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:31.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:39:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:39:31] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T11:39:31.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:39:31 smithi083 ceph-mon[77242]: from='client.47082 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:39:31.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:39:31 smithi083 ceph-mon[77242]: from='client.47088 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:39:31.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:39:31 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1224444156' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:39:31.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:39:31 smithi083 ceph-mon[79930]: from='client.47082 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:39:31.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:39:31 smithi083 ceph-mon[79930]: from='client.47088 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:39:31.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:39:31 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1224444156' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:39:31.714 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:39:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:39:31.536Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:39:31.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:39:31 smithi153 ceph-mon[70044]: from='client.47082 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:39:31.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:39:31 smithi153 ceph-mon[70044]: from='client.47088 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:39:31.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:39:31 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1224444156' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:39:32.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:39:32 smithi083 ceph-mon[79930]: pgmap v11137: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:32.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:39:32 smithi083 ceph-mon[77242]: pgmap v11137: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:32.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:39:32 smithi153 ceph-mon[70044]: pgmap v11137: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:39:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:39:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:39:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:39:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:39:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:39:33.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:39:33 smithi083 ceph-mon[79930]: pgmap v11138: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:33.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:39:33 smithi083 ceph-mon[77242]: pgmap v11138: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:33.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:39:33 smithi153 ceph-mon[70044]: pgmap v11138: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:35.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:39:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:39:34.907Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:39:35.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:39:35 smithi083 ceph-mon[77242]: pgmap v11139: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:35.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:39:35 smithi083 ceph-mon[79930]: pgmap v11139: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:35.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:39:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:39:35.631Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:39:36.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:39:35 smithi153 ceph-mon[70044]: pgmap v11139: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:37.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:39:37 smithi083 ceph-mon[79930]: pgmap v11140: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:37.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:39:37 smithi083 ceph-mon[77242]: pgmap v11140: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:38.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:39:37 smithi153 ceph-mon[70044]: pgmap v11140: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:39:39 smithi083 ceph-mon[79930]: pgmap v11141: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:39:39 smithi083 ceph-mon[77242]: pgmap v11141: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:40.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:39:39 smithi153 ceph-mon[70044]: pgmap v11141: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:39:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:39:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:39:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:39:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:39:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:39:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:39:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:39:40.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:39:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:39:40.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:39:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:39:40.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:39:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:39:40.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:39:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:39:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:39:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:39:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:39:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:39:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:39:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:39:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:39:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:39:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:39:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:39:41] "GET /metrics HTTP/1.1" 200 225229 "" "Prometheus/2.33.4" 2022-09-29T11:39:41.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:39:41 smithi083 ceph-mon[79930]: pgmap v11142: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:41.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:39:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:39:41.537Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:39:41.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:39:41 smithi083 ceph-mon[77242]: pgmap v11142: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:42.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:39:41 smithi153 ceph-mon[70044]: pgmap v11142: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:39:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:39:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:39:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:39:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:39:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:39:43.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:39:43 smithi083 ceph-mon[77242]: pgmap v11143: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:43.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:39:43 smithi083 ceph-mon[79930]: pgmap v11143: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:44.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:39:43 smithi153 ceph-mon[70044]: pgmap v11143: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:45.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:39:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:39:44.907Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:39:45.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:39:45 smithi083 ceph-mon[79930]: pgmap v11144: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:45.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:39:45 smithi083 ceph-mon[77242]: pgmap v11144: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:45.964 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:39:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:39:45.631Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:39:46.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:39:45 smithi153 ceph-mon[70044]: pgmap v11144: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:47.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:39:47 smithi083 ceph-mon[79930]: pgmap v11145: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:47.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:39:47 smithi083 ceph-mon[77242]: pgmap v11145: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:48.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:39:47 smithi153 ceph-mon[70044]: pgmap v11145: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:49.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:39:49 smithi083 ceph-mon[79930]: pgmap v11146: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:49.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:39:49 smithi083 ceph-mon[77242]: pgmap v11146: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:50.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:39:49 smithi153 ceph-mon[70044]: pgmap v11146: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:39:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:39:51] "GET /metrics HTTP/1.1" 200 225229 "" "Prometheus/2.33.4" 2022-09-29T11:39:51.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:39:51 smithi083 ceph-mon[79930]: pgmap v11147: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:51.967 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:39:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:39:51.537Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:39:51.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:39:51 smithi083 ceph-mon[77242]: pgmap v11147: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:52.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:39:51 smithi153 ceph-mon[70044]: pgmap v11147: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:39:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:39:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:39:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:39:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:39:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:39:53.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:39:53 smithi083 ceph-mon[77242]: pgmap v11148: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:53.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:39:53 smithi083 ceph-mon[79930]: pgmap v11148: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:54.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:39:53 smithi153 ceph-mon[70044]: pgmap v11148: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:55.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:39:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:39:54.908Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:39:55.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:39:55 smithi083 ceph-mon[79930]: pgmap v11149: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:55.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:39:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:39:55.632Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:39:55.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:39:55 smithi083 ceph-mon[77242]: pgmap v11149: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:56.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:39:55 smithi153 ceph-mon[70044]: pgmap v11149: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:56.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:39:56 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:39:56.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:39:56 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:39:56.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:39:56 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:39:56.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:39:56 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:39:56.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:39:56 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:39:56.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:39:56 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:39:57.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:39:56 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:39:57.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:39:56 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:39:57.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:39:56 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:39:58.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:39:57 smithi083 ceph-mon[79930]: pgmap v11150: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:58.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:39:57 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:39:58.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:39:57 smithi083 ceph-mon[77242]: pgmap v11150: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:58.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:39:57 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:39:58.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:39:57 smithi153 ceph-mon[70044]: pgmap v11150: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:39:58.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:39:57 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:40:00.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:39:59 smithi083 ceph-mon[79930]: pgmap v11151: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:00.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:39:59 smithi083 ceph-mon[77242]: pgmap v11151: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:00.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:40:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[77219]: 2022-09-29T11:39:59.999+0000 7f160f665700 -1 log_channel(cluster) log [ERR] : overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T11:40:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:39:59 smithi153 ceph-mon[70044]: pgmap v11151: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:00.937 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:40:01.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:40:00 smithi083 ceph-mon[79930]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T11:40:01.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:40:00 smithi083 ceph-mon[77242]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T11:40:01.228 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:40:01.228 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 7m ago 6h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:40:01.229 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 5m ago 6h 63.7M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:40:01.229 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 7m ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:40:01.229 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 5m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:40:01.229 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 5m ago 6h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:40:01.230 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 7m ago 6h 823M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:40:01.230 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 7m ago 6h 1379M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:40:01.230 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 5m ago 6h 595M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:40:01.230 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 7m ago 6h 795M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:40:01.230 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 7m ago 6h 29.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:40:01.231 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 5m ago 6h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:40:01.231 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 7m ago 6h 972M 2420M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:40:01.231 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 7m ago 6h 961M 2420M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:40:01.231 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 7m ago 6h 749M 2420M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:40:01.232 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 7m ago 6h 939M 2420M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:40:01.232 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 5m ago 6h 827M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:40:01.232 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 5m ago 6h 1057M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:40:01.232 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 5m ago 6h 922M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:40:01.233 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 5m ago 6h 1119M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:40:01.233 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 5m ago 6h 154M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:40:01.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:40:00 smithi153 ceph-mon[70044]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T11:40:01.585 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:40:01.585 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:40:01.586 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:40:01.586 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:40:01.586 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:40:01.586 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:40:01.587 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:40:01.587 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:40:01.587 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:40:01.587 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:40:01.587 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:40:01.588 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:40:01.588 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:40:01.588 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:40:01.588 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:40:01.589 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:40:01.589 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:40:01.589 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:40:01.958 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:40:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:40:01] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T11:40:01.958 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:40:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:40:01.539Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:40:02.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:40:01 smithi083 ceph-mon[79930]: pgmap v11152: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:02.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:40:01 smithi083 ceph-mon[79930]: from='client.57031 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:40:02.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:40:01 smithi083 ceph-mon[79930]: from='client.47106 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:40:02.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:40:01 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1154747319' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:40:02.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:40:01 smithi083 ceph-mon[77242]: pgmap v11152: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:02.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:40:01 smithi083 ceph-mon[77242]: from='client.57031 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:40:02.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:40:01 smithi083 ceph-mon[77242]: from='client.47106 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:40:02.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:40:01 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1154747319' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:40:02.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:40:01 smithi153 ceph-mon[70044]: pgmap v11152: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:02.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:40:01 smithi153 ceph-mon[70044]: from='client.57031 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:40:02.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:40:01 smithi153 ceph-mon[70044]: from='client.47106 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:40:02.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:40:01 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1154747319' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:40:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:40:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:40:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:40:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:40:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:40:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:40:04.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:40:03 smithi083 ceph-mon[79930]: pgmap v11153: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:04.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:40:03 smithi083 ceph-mon[77242]: pgmap v11153: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:04.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:40:03 smithi153 ceph-mon[70044]: pgmap v11153: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:05.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:40:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:40:04.908Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:40:05.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:40:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:40:05.632Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:40:06.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:40:05 smithi153 ceph-mon[70044]: pgmap v11154: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:06.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:40:05 smithi083 ceph-mon[79930]: pgmap v11154: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:06.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:40:05 smithi083 ceph-mon[77242]: pgmap v11154: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:08.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:40:07 smithi153 ceph-mon[70044]: pgmap v11155: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:08.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:40:07 smithi083 ceph-mon[79930]: pgmap v11155: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:08.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:40:07 smithi083 ceph-mon[77242]: pgmap v11155: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:10.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:40:09 smithi153 ceph-mon[70044]: pgmap v11156: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:10.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:40:09 smithi083 ceph-mon[79930]: pgmap v11156: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:10.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:40:09 smithi083 ceph-mon[77242]: pgmap v11156: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:40:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:40:11] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T11:40:11.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:40:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:40:11.538Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:40:12.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:40:11 smithi153 ceph-mon[70044]: pgmap v11157: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:12.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:40:11 smithi083 ceph-mon[79930]: pgmap v11157: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:12.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:40:11 smithi083 ceph-mon[77242]: pgmap v11157: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:40:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:40:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:40:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:40:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:40:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:40:14.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:40:13 smithi153 ceph-mon[70044]: pgmap v11158: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:14.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:40:13 smithi083 ceph-mon[79930]: pgmap v11158: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:14.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:40:13 smithi083 ceph-mon[77242]: pgmap v11158: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:15.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:40:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:40:14.908Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:40:15.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:40:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:40:15.634Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:40:16.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:40:15 smithi153 ceph-mon[70044]: pgmap v11159: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:16.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:40:15 smithi083 ceph-mon[79930]: pgmap v11159: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:16.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:40:15 smithi083 ceph-mon[77242]: pgmap v11159: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:18.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:40:17 smithi153 ceph-mon[70044]: pgmap v11160: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:18.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:40:17 smithi083 ceph-mon[79930]: pgmap v11160: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:18.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:40:17 smithi083 ceph-mon[77242]: pgmap v11160: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:40:19 smithi153 ceph-mon[70044]: pgmap v11161: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:20.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:40:19 smithi083 ceph-mon[79930]: pgmap v11161: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:20.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:40:19 smithi083 ceph-mon[77242]: pgmap v11161: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:40:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:40:21] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T11:40:21.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:40:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:40:21.539Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:40:22.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:40:21 smithi153 ceph-mon[70044]: pgmap v11162: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:22.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:40:21 smithi083 ceph-mon[77242]: pgmap v11162: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:22.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:40:21 smithi083 ceph-mon[79930]: pgmap v11162: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:40:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:40:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:40:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:40:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:40:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:40:24.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:40:23 smithi153 ceph-mon[70044]: pgmap v11163: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:24.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:40:23 smithi083 ceph-mon[77242]: pgmap v11163: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:24.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:40:23 smithi083 ceph-mon[79930]: pgmap v11163: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:25.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:40:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:40:24.909Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:40:25.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:40:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:40:25.634Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:40:26.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:40:26 smithi153 ceph-mon[70044]: pgmap v11164: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:26.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:40:25 smithi083 ceph-mon[79930]: pgmap v11164: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:26.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:40:25 smithi083 ceph-mon[77242]: pgmap v11164: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:28.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:40:28 smithi153 ceph-mon[70044]: pgmap v11165: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:28.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:40:28 smithi083 ceph-mon[79930]: pgmap v11165: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:28.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:40:28 smithi083 ceph-mon[77242]: pgmap v11165: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:30.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:40:30 smithi153 ceph-mon[70044]: pgmap v11166: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:30.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:40:30 smithi083 ceph-mon[79930]: pgmap v11166: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:30.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:40:30 smithi083 ceph-mon[77242]: pgmap v11166: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:31.908 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:40:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:40:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:40:31] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T11:40:31.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:40:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:40:31.540Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:40:32.199 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:40:32.199 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 7m ago 6h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:40:32.200 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 5m ago 6h 63.7M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:40:32.200 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 7m ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:40:32.200 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 5m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:40:32.200 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 5m ago 6h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:40:32.201 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 7m ago 6h 823M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:40:32.201 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 7m ago 6h 1379M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:40:32.201 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 5m ago 6h 595M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:40:32.201 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 7m ago 6h 795M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:40:32.201 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 7m ago 6h 29.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:40:32.202 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 5m ago 6h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:40:32.202 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 7m ago 6h 972M 2420M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:40:32.202 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 7m ago 6h 961M 2420M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:40:32.202 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 7m ago 6h 749M 2420M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:40:32.203 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 7m ago 6h 939M 2420M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:40:32.203 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 5m ago 6h 827M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:40:32.203 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 5m ago 6h 1057M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:40:32.203 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 5m ago 6h 922M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:40:32.203 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 5m ago 6h 1119M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:40:32.204 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 5m ago 6h 154M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:40:32.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:40:32 smithi153 ceph-mon[70044]: pgmap v11167: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:32.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:40:32 smithi083 ceph-mon[79930]: pgmap v11167: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:32.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:40:32 smithi083 ceph-mon[77242]: pgmap v11167: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:32.558 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:40:32.558 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:40:32.559 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:40:32.559 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:40:32.559 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:40:32.559 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:40:32.560 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:40:32.560 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:40:32.560 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:40:32.560 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:40:32.561 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:40:32.561 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:40:32.561 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:40:32.561 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:40:32.562 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:40:32.562 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:40:32.562 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:40:32.562 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:40:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:40:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:40:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:40:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:40:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:40:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:40:33.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:40:33 smithi153 ceph-mon[70044]: from='client.47118 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:40:33.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:40:33 smithi153 ceph-mon[70044]: from='client.57055 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:40:33.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:40:33 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2456034174' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:40:33.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:40:33 smithi083 ceph-mon[77242]: from='client.47118 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:40:33.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:40:33 smithi083 ceph-mon[77242]: from='client.57055 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:40:33.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:40:33 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2456034174' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:40:33.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:40:33 smithi083 ceph-mon[79930]: from='client.47118 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:40:33.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:40:33 smithi083 ceph-mon[79930]: from='client.57055 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:40:33.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:40:33 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2456034174' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:40:34.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:40:34 smithi153 ceph-mon[70044]: pgmap v11168: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:34.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:40:34 smithi083 ceph-mon[79930]: pgmap v11168: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:34.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:40:34 smithi083 ceph-mon[77242]: pgmap v11168: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:35.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:40:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:40:34.909Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:40:35.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:40:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:40:35.635Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:40:36.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:40:36 smithi153 ceph-mon[70044]: pgmap v11169: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:36.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:40:36 smithi083 ceph-mon[79930]: pgmap v11169: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:36.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:40:36 smithi083 ceph-mon[77242]: pgmap v11169: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:38.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:40:38 smithi153 ceph-mon[70044]: pgmap v11170: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:38.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:40:38 smithi083 ceph-mon[79930]: pgmap v11170: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:38.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:40:38 smithi083 ceph-mon[77242]: pgmap v11170: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:40:40 smithi153 ceph-mon[70044]: pgmap v11171: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:40:40 smithi083 ceph-mon[79930]: pgmap v11171: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:40:40 smithi083 ceph-mon[77242]: pgmap v11171: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:40:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:40:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:40:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:40:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:40:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:40:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:40:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:40:41.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:40:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:40:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:40:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:40:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:40:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:40:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:40:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:40:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:40:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:40:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:40:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:40:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:40:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:40:41.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:40:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:40:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:40:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:40:41] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T11:40:41.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:40:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:40:41.540Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:40:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:40:42 smithi153 ceph-mon[70044]: pgmap v11172: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:42.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:40:42 smithi083 ceph-mon[79930]: pgmap v11172: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:42.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:40:42 smithi083 ceph-mon[77242]: pgmap v11172: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:40:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:40:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:40:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:40:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:40:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:40:44.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:40:44 smithi153 ceph-mon[70044]: pgmap v11173: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:44.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:40:44 smithi083 ceph-mon[79930]: pgmap v11173: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:44.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:40:44 smithi083 ceph-mon[77242]: pgmap v11173: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:45.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:40:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:40:44.910Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:40:45.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:40:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:40:45.635Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:40:46.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:40:46 smithi153 ceph-mon[70044]: pgmap v11174: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:46.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:40:46 smithi083 ceph-mon[79930]: pgmap v11174: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:46.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:40:46 smithi083 ceph-mon[77242]: pgmap v11174: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:48.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:40:48 smithi153 ceph-mon[70044]: pgmap v11175: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:48.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:40:48 smithi083 ceph-mon[79930]: pgmap v11175: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:48.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:40:48 smithi083 ceph-mon[77242]: pgmap v11175: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:50.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:40:50 smithi153 ceph-mon[70044]: pgmap v11176: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:50.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:40:50 smithi083 ceph-mon[79930]: pgmap v11176: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:50.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:40:50 smithi083 ceph-mon[77242]: pgmap v11176: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:40:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:40:51] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T11:40:51.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:40:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:40:51.541Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:40:52.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:40:52 smithi153 ceph-mon[70044]: pgmap v11177: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:52.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:40:52 smithi083 ceph-mon[79930]: pgmap v11177: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:52.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:40:52 smithi083 ceph-mon[77242]: pgmap v11177: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:40:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:40:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:40:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:40:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:40:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:40:54.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:40:54 smithi153 ceph-mon[70044]: pgmap v11178: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:54.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:40:54 smithi083 ceph-mon[79930]: pgmap v11178: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:54.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:40:54 smithi083 ceph-mon[77242]: pgmap v11178: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:55.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:40:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:40:54.911Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:40:55.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:40:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:40:55.636Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:40:56.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:40:56 smithi083 ceph-mon[79930]: pgmap v11179: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:56.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:40:56 smithi083 ceph-mon[77242]: pgmap v11179: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:56.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:40:56 smithi153 ceph-mon[70044]: pgmap v11179: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:57.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:40:57 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:40:57.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:40:57 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:40:57.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:40:57 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:40:57.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:40:57 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:40:57.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:40:57 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:40:57.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:40:57 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:40:57.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:40:57 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:40:57.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:40:57 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:40:57.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:40:57 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:40:58.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:40:58 smithi083 ceph-mon[79930]: pgmap v11180: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:58.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:40:58 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:40:58.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:40:58 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:40:58.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:40:58 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:40:58.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:40:58 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:40:58.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:40:58 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:40:58.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:40:58 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:40:58.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:40:58 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:40:58.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:40:58 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:40:58.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:40:58 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:40:58.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:40:58 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:40:58.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:40:58 smithi083 ceph-mon[77242]: pgmap v11180: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:58.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:40:58 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:40:58.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:40:58 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:40:58.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:40:58 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:40:58.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:40:58 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:40:58.467 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:40:58 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:40:58.467 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:40:58 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:40:58.467 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:40:58 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:40:58.467 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:40:58 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:40:58.468 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:40:58 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:40:58.468 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:40:58 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:40:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:40:58 smithi153 ceph-mon[70044]: pgmap v11180: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:40:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:40:58 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:40:58.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:40:58 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:40:58.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:40:58 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:40:58.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:40:58 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:40:58.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:40:58 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:40:58.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:40:58 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:40:58.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:40:58 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:40:58.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:40:58 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:40:58.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:40:58 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:40:58.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:40:58 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:40:59.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:40:59 smithi083 ceph-mon[79930]: Adjusting osd_memory_target on smithi083 to 2411M 2022-09-29T11:40:59.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:40:59 smithi083 ceph-mon[77242]: Adjusting osd_memory_target on smithi083 to 2411M 2022-09-29T11:40:59.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:40:59 smithi153 ceph-mon[70044]: Adjusting osd_memory_target on smithi083 to 2411M 2022-09-29T11:41:00.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:41:00 smithi083 ceph-mon[79930]: pgmap v11181: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:00.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:41:00 smithi083 ceph-mon[77242]: pgmap v11181: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:41:00 smithi153 ceph-mon[70044]: pgmap v11181: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:41:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:41:01] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T11:41:01.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:41:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:41:01.541Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:41:02.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:41:02 smithi083 ceph-mon[77242]: pgmap v11182: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:02.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:41:02 smithi083 ceph-mon[79930]: pgmap v11182: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:41:02 smithi153 ceph-mon[70044]: pgmap v11182: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:02.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:41:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:41:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:41:02.886 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:41:03.174 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:41:03.174 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 8m ago 6h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:41:03.175 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 6m ago 6h 63.7M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:41:03.175 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 8m ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:41:03.175 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 6m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:41:03.175 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 6m ago 6h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:41:03.175 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 8m ago 6h 823M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:41:03.176 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 8m ago 6h 1379M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:41:03.176 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 6m ago 6h 595M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:41:03.176 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 8m ago 6h 795M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:41:03.176 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 8m ago 6h 29.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:41:03.177 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 6m ago 6h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:41:03.177 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 8m ago 6h 972M 2411M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:41:03.177 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 8m ago 6h 961M 2411M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:41:03.177 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 8m ago 6h 749M 2411M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:41:03.178 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 8m ago 6h 939M 2411M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:41:03.178 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 6m ago 6h 827M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:41:03.178 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 6m ago 6h 1057M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:41:03.178 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 6m ago 6h 922M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:41:03.178 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 6m ago 6h 1119M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:41:03.179 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 6m ago 6h 154M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:41:03.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:41:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:41:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:41:03.539 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:41:03.539 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:41:03.540 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:41:03.540 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:41:03.540 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:41:03.540 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:41:03.540 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:41:03.541 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:41:03.541 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:41:03.541 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:41:03.541 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:41:03.542 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:41:03.542 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:41:03.542 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:41:03.542 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:41:03.542 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:41:03.543 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:41:03.543 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:41:04.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:41:04 smithi083 ceph-mon[79930]: pgmap v11183: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:04.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:41:04 smithi083 ceph-mon[79930]: from='client.47136 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:41:04.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:41:04 smithi083 ceph-mon[79930]: from='client.47142 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:41:04.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:41:04 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1963243659' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:41:04.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:41:04 smithi083 ceph-mon[77242]: pgmap v11183: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:04.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:41:04 smithi083 ceph-mon[77242]: from='client.47136 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:41:04.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:41:04 smithi083 ceph-mon[77242]: from='client.47142 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:41:04.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:41:04 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1963243659' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:41:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:41:04 smithi153 ceph-mon[70044]: pgmap v11183: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:41:04 smithi153 ceph-mon[70044]: from='client.47136 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:41:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:41:04 smithi153 ceph-mon[70044]: from='client.47142 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:41:04.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:41:04 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1963243659' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:41:05.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:41:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:41:04.911Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:41:05.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:41:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:41:05.636Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:41:06.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:41:06 smithi083 ceph-mon[77242]: pgmap v11184: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:06.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:41:06 smithi083 ceph-mon[79930]: pgmap v11184: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:06.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:41:06 smithi153 ceph-mon[70044]: pgmap v11184: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:08.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:41:08 smithi083 ceph-mon[79930]: pgmap v11185: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:08.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:41:08 smithi083 ceph-mon[77242]: pgmap v11185: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:41:08 smithi153 ceph-mon[70044]: pgmap v11185: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:10.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:41:10 smithi083 ceph-mon[79930]: pgmap v11186: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:10.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:41:10 smithi083 ceph-mon[77242]: pgmap v11186: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:41:10 smithi153 ceph-mon[70044]: pgmap v11186: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:41:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:41:11] "GET /metrics HTTP/1.1" 200 225229 "" "Prometheus/2.33.4" 2022-09-29T11:41:11.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:41:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:41:11.541Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:41:12.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:41:12 smithi083 ceph-mon[79930]: pgmap v11187: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:12.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:41:12 smithi083 ceph-mon[77242]: pgmap v11187: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:41:12 smithi153 ceph-mon[70044]: pgmap v11187: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:12.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:41:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:41:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:41:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:41:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:41:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:41:14.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:41:14 smithi083 ceph-mon[77242]: pgmap v11188: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:14.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:41:14 smithi083 ceph-mon[79930]: pgmap v11188: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:14.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:41:14 smithi153 ceph-mon[70044]: pgmap v11188: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:15.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:41:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:41:14.912Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:41:15.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:41:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:41:15.636Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:41:16.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:41:16 smithi083 ceph-mon[79930]: pgmap v11189: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:16.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:41:16 smithi083 ceph-mon[77242]: pgmap v11189: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:41:16 smithi153 ceph-mon[70044]: pgmap v11189: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:18.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:41:18 smithi083 ceph-mon[79930]: pgmap v11190: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:18.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:41:18 smithi083 ceph-mon[77242]: pgmap v11190: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:18.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:41:18 smithi153 ceph-mon[70044]: pgmap v11190: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:20.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:41:20 smithi083 ceph-mon[77242]: pgmap v11191: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:20.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:41:20 smithi083 ceph-mon[79930]: pgmap v11191: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:20.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:41:20 smithi153 ceph-mon[70044]: pgmap v11191: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:41:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:41:21] "GET /metrics HTTP/1.1" 200 225229 "" "Prometheus/2.33.4" 2022-09-29T11:41:21.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:41:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:41:21.541Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:41:22.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:41:22 smithi083 ceph-mon[79930]: pgmap v11192: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:22.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:41:22 smithi083 ceph-mon[77242]: pgmap v11192: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:41:22 smithi153 ceph-mon[70044]: pgmap v11192: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:22.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:41:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:41:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:41:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:41:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:41:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:41:24.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:41:24 smithi083 ceph-mon[79930]: pgmap v11193: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:24.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:41:24 smithi083 ceph-mon[77242]: pgmap v11193: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:41:24 smithi153 ceph-mon[70044]: pgmap v11193: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:25.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:41:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:41:24.913Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:41:25.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:41:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:41:25.637Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:41:26.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:41:26 smithi083 ceph-mon[79930]: pgmap v11194: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:26.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:41:26 smithi083 ceph-mon[77242]: pgmap v11194: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:41:26 smithi153 ceph-mon[70044]: pgmap v11194: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:28.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:41:28 smithi083 ceph-mon[79930]: pgmap v11195: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:28.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:41:28 smithi083 ceph-mon[77242]: pgmap v11195: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:41:28 smithi153 ceph-mon[70044]: pgmap v11195: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:30.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:41:30 smithi083 ceph-mon[79930]: pgmap v11196: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:30.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:41:30 smithi083 ceph-mon[77242]: pgmap v11196: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:30.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:41:30 smithi153 ceph-mon[70044]: pgmap v11196: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:41:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:41:31] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T11:41:31.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:41:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:41:31.542Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:41:32.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:41:32 smithi083 ceph-mon[79930]: pgmap v11197: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:32.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:41:32 smithi083 ceph-mon[77242]: pgmap v11197: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:41:32 smithi153 ceph-mon[70044]: pgmap v11197: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:32.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:41:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:41:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:41:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:41:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:41:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:41:33.868 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:41:34.155 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:41:34.156 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 8m ago 6h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:41:34.156 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 6m ago 6h 63.7M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:41:34.156 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 8m ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:41:34.156 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 6m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:41:34.157 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 6m ago 6h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:41:34.157 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 8m ago 6h 823M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:41:34.157 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 8m ago 6h 1379M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:41:34.157 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 6m ago 6h 595M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:41:34.158 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 8m ago 6h 795M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:41:34.158 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 8m ago 6h 29.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:41:34.158 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 6m ago 6h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:41:34.159 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 8m ago 6h 972M 2411M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:41:34.159 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 8m ago 6h 961M 2411M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:41:34.159 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 8m ago 6h 749M 2411M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:41:34.159 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 8m ago 6h 939M 2411M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:41:34.159 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 6m ago 6h 827M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:41:34.160 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 6m ago 6h 1057M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:41:34.160 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 6m ago 6h 922M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:41:34.160 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 6m ago 6h 1119M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:41:34.160 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 6m ago 6h 154M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:41:34.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:41:34 smithi083 ceph-mon[79930]: pgmap v11198: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:34.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:41:34 smithi083 ceph-mon[77242]: pgmap v11198: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:34.515 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:41:34.515 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:41:34.515 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:41:34.515 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:41:34.516 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:41:34.516 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:41:34.516 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:41:34.516 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:41:34.517 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:41:34.517 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:41:34.517 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:41:34.517 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:41:34.518 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:41:34.518 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:41:34.518 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:41:34.518 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:41:34.518 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:41:34.519 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:41:34.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:41:34 smithi153 ceph-mon[70044]: pgmap v11198: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:35.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:41:35 smithi083 ceph-mon[77242]: from='client.57085 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:41:35.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:41:35 smithi083 ceph-mon[77242]: from='client.57091 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:41:35.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:41:35 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3276334926' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:41:35.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:41:35 smithi083 ceph-mon[79930]: from='client.57085 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:41:35.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:41:35 smithi083 ceph-mon[79930]: from='client.57091 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:41:35.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:41:35 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3276334926' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:41:35.214 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:41:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:41:34.913Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:41:35.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:41:35 smithi153 ceph-mon[70044]: from='client.57085 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:41:35.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:41:35 smithi153 ceph-mon[70044]: from='client.57091 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:41:35.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:41:35 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3276334926' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:41:35.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:41:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:41:35.638Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:41:36.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:41:36 smithi083 ceph-mon[77242]: pgmap v11199: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:36.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:41:36 smithi083 ceph-mon[79930]: pgmap v11199: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:41:36 smithi153 ceph-mon[70044]: pgmap v11199: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:38.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:41:38 smithi083 ceph-mon[77242]: pgmap v11200: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:38.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:41:38 smithi083 ceph-mon[79930]: pgmap v11200: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:41:38 smithi153 ceph-mon[70044]: pgmap v11200: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:41:40 smithi083 ceph-mon[79930]: pgmap v11201: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:41:40 smithi083 ceph-mon[77242]: pgmap v11201: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:41:40 smithi153 ceph-mon[70044]: pgmap v11201: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:41.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:41:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:41:41.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:41:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:41:41.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:41:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:41:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:41:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:41:41.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:41:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:41:41.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:41:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:41:41.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:41:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:41:41.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:41:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:41:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:41:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:41:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:41:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:41:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:41:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:41:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:41:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:41:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:41:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:41:41] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T11:41:41.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:41:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:41:41.542Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:41:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:41:42 smithi083 ceph-mon[79930]: pgmap v11202: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:42.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:41:42 smithi083 ceph-mon[77242]: pgmap v11202: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:41:42 smithi153 ceph-mon[70044]: pgmap v11202: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:42.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:41:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:41:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:41:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:41:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:41:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:41:44.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:41:44 smithi083 ceph-mon[79930]: pgmap v11203: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:44.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:41:44 smithi083 ceph-mon[77242]: pgmap v11203: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:44.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:41:44 smithi153 ceph-mon[70044]: pgmap v11203: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:45.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:41:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:41:44.914Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:41:45.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:41:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:41:45.638Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:41:46.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:41:46 smithi083 ceph-mon[79930]: pgmap v11204: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:46.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:41:46 smithi083 ceph-mon[77242]: pgmap v11204: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:46.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:41:46 smithi153 ceph-mon[70044]: pgmap v11204: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:48.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:41:48 smithi083 ceph-mon[79930]: pgmap v11205: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:48.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:41:48 smithi083 ceph-mon[77242]: pgmap v11205: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:48.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:41:48 smithi153 ceph-mon[70044]: pgmap v11205: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:50.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:41:50 smithi083 ceph-mon[79930]: pgmap v11206: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:50.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:41:50 smithi083 ceph-mon[77242]: pgmap v11206: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:41:50 smithi153 ceph-mon[70044]: pgmap v11206: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:41:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:41:51] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T11:41:51.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:41:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:41:51.543Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:41:52.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:41:52 smithi083 ceph-mon[77242]: pgmap v11207: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:52.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:41:52 smithi083 ceph-mon[79930]: pgmap v11207: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:41:52 smithi153 ceph-mon[70044]: pgmap v11207: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:52.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:41:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:41:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:41:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:41:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:41:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:41:54.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:41:54 smithi083 ceph-mon[77242]: pgmap v11208: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:54.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:41:54 smithi083 ceph-mon[79930]: pgmap v11208: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:54.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:41:54 smithi153 ceph-mon[70044]: pgmap v11208: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:55.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:41:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:41:54.915Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:41:55.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:41:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:41:55.640Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:41:56.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:41:56 smithi083 ceph-mon[77242]: pgmap v11209: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:56.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:41:56 smithi083 ceph-mon[79930]: pgmap v11209: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:56.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:41:56 smithi153 ceph-mon[70044]: pgmap v11209: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:58.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:41:58 smithi083 ceph-mon[77242]: pgmap v11210: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:58.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:41:58 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:41:58.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:41:58 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:41:58.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:41:58 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:41:58.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:41:58 smithi083 ceph-mon[79930]: pgmap v11210: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:58.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:41:58 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:41:58.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:41:58 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:41:58.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:41:58 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:41:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:41:58 smithi153 ceph-mon[70044]: pgmap v11210: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:41:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:41:58 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:41:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:41:58 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:41:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:41:58 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:41:59.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:41:59 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:41:59.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:41:59 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:41:59.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:41:59 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:42:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:42:00 smithi153 ceph-mon[70044]: pgmap v11211: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:00.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:42:00 smithi083 ceph-mon[77242]: pgmap v11211: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:00.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:42:00 smithi083 ceph-mon[79930]: pgmap v11211: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:01.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:42:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:42:01] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T11:42:01.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:42:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:42:01.543Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:42:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:42:02 smithi153 ceph-mon[70044]: pgmap v11212: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:02.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:42:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:42:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:42:02.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:42:02 smithi083 ceph-mon[77242]: pgmap v11212: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:02.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:42:02 smithi083 ceph-mon[79930]: pgmap v11212: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:42:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:42:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:42:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:42:04 smithi153 ceph-mon[70044]: pgmap v11213: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:04.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:42:04 smithi083 ceph-mon[79930]: pgmap v11213: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:04.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:42:04 smithi083 ceph-mon[77242]: pgmap v11213: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:04.841 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:42:05.129 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:42:05.129 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 9m ago 6h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:42:05.129 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 7m ago 6h 63.7M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:42:05.130 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 9m ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:42:05.130 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 7m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:42:05.130 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 7m ago 6h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:42:05.131 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 9m ago 6h 823M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:42:05.131 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 9m ago 6h 1379M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:42:05.131 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 7m ago 6h 595M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:42:05.131 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 9m ago 6h 795M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:42:05.132 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 9m ago 6h 29.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:42:05.132 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 7m ago 6h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:42:05.132 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 9m ago 6h 972M 2411M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:42:05.132 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 9m ago 6h 961M 2411M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:42:05.133 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 9m ago 6h 749M 2411M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:42:05.133 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 9m ago 6h 939M 2411M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:42:05.133 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 7m ago 6h 827M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:42:05.133 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 7m ago 6h 1057M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:42:05.133 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 7m ago 6h 922M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:42:05.134 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 7m ago 6h 1119M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:42:05.134 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 7m ago 6h 154M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:42:05.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:42:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:42:04.914Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:42:05.487 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:42:05.487 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:42:05.488 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:42:05.488 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:42:05.488 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:42:05.488 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:42:05.489 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:42:05.489 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:42:05.489 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:42:05.489 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:42:05.490 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:42:05.490 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:42:05.490 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:42:05.490 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:42:05.491 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:42:05.491 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:42:05.491 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:42:05.491 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:42:05.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:42:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:42:05.639Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:42:06.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:42:06 smithi153 ceph-mon[70044]: pgmap v11214: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:06.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:42:06 smithi153 ceph-mon[70044]: from='client.47172 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:42:06.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:42:06 smithi153 ceph-mon[70044]: from='client.47178 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:42:06.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:42:06 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/435678041' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:42:06.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:42:06 smithi083 ceph-mon[79930]: pgmap v11214: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:06.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:42:06 smithi083 ceph-mon[79930]: from='client.47172 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:42:06.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:42:06 smithi083 ceph-mon[79930]: from='client.47178 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:42:06.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:42:06 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/435678041' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:42:06.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:42:06 smithi083 ceph-mon[77242]: pgmap v11214: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:06.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:42:06 smithi083 ceph-mon[77242]: from='client.47172 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:42:06.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:42:06 smithi083 ceph-mon[77242]: from='client.47178 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:42:06.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:42:06 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/435678041' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:42:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:42:08 smithi153 ceph-mon[70044]: pgmap v11215: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:08.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:42:08 smithi083 ceph-mon[79930]: pgmap v11215: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:08.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:42:08 smithi083 ceph-mon[77242]: pgmap v11215: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:42:10 smithi153 ceph-mon[70044]: pgmap v11216: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:10.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:42:10 smithi083 ceph-mon[77242]: pgmap v11216: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:10.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:42:10 smithi083 ceph-mon[79930]: pgmap v11216: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:42:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:42:11] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T11:42:11.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:42:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:42:11.544Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:42:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:42:12 smithi153 ceph-mon[70044]: pgmap v11217: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:12.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:42:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:42:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:42:12.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:42:12 smithi083 ceph-mon[79930]: pgmap v11217: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:12.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:42:12 smithi083 ceph-mon[77242]: pgmap v11217: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:42:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:42:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:42:14.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:42:14 smithi153 ceph-mon[70044]: pgmap v11218: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:14.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:42:14 smithi083 ceph-mon[77242]: pgmap v11218: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:14.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:42:14 smithi083 ceph-mon[79930]: pgmap v11218: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:15.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:42:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:42:14.916Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:42:15.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:42:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:42:15.641Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:42:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:42:16 smithi153 ceph-mon[70044]: pgmap v11219: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:16.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:42:16 smithi083 ceph-mon[77242]: pgmap v11219: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:16.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:42:16 smithi083 ceph-mon[79930]: pgmap v11219: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:42:18 smithi153 ceph-mon[70044]: pgmap v11220: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:18.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:42:18 smithi083 ceph-mon[77242]: pgmap v11220: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:18.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:42:18 smithi083 ceph-mon[79930]: pgmap v11220: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:42:20 smithi153 ceph-mon[70044]: pgmap v11221: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:20.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:42:20 smithi083 ceph-mon[77242]: pgmap v11221: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:20.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:42:20 smithi083 ceph-mon[79930]: pgmap v11221: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:42:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:42:21] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T11:42:21.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:42:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:42:21.544Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:42:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:42:22 smithi153 ceph-mon[70044]: pgmap v11222: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:22.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:42:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:42:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:42:22.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:42:22 smithi083 ceph-mon[79930]: pgmap v11222: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:22.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:42:22 smithi083 ceph-mon[77242]: pgmap v11222: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:42:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:42:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:42:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:42:24 smithi153 ceph-mon[70044]: pgmap v11223: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:24.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:42:24 smithi083 ceph-mon[79930]: pgmap v11223: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:24.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:42:24 smithi083 ceph-mon[77242]: pgmap v11223: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:25.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:42:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:42:24.916Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:42:25.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:42:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:42:25.642Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:42:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:42:26 smithi153 ceph-mon[70044]: pgmap v11224: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:26.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:42:26 smithi083 ceph-mon[79930]: pgmap v11224: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:26.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:42:26 smithi083 ceph-mon[77242]: pgmap v11224: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:28.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:42:28 smithi083 ceph-mon[79930]: pgmap v11225: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:28.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:42:28 smithi083 ceph-mon[77242]: pgmap v11225: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:28.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:42:28 smithi153 ceph-mon[70044]: pgmap v11225: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:30.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:42:30 smithi083 ceph-mon[79930]: pgmap v11226: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:30.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:42:30 smithi083 ceph-mon[77242]: pgmap v11226: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:30.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:42:30 smithi153 ceph-mon[70044]: pgmap v11226: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:42:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:42:31] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T11:42:31.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:42:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:42:31.544Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:42:32.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:42:32 smithi083 ceph-mon[77242]: pgmap v11227: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:32.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:42:32 smithi083 ceph-mon[79930]: pgmap v11227: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:32.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:42:32 smithi153 ceph-mon[70044]: pgmap v11227: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:42:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:42:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:42:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:42:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:42:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:42:34.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:42:34 smithi083 ceph-mon[79930]: pgmap v11228: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:34.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:42:34 smithi083 ceph-mon[77242]: pgmap v11228: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:34.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:42:34 smithi153 ceph-mon[70044]: pgmap v11228: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:35.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:42:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:42:34.917Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:42:35.818 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:42:35.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:42:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:42:35.644Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:42:36.106 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:42:36.107 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 9m ago 6h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:42:36.107 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 7m ago 6h 63.7M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:42:36.107 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 9m ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:42:36.107 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 7m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:42:36.108 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 7m ago 6h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:42:36.108 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 9m ago 6h 823M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:42:36.108 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 9m ago 6h 1379M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:42:36.108 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 7m ago 6h 595M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:42:36.109 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 9m ago 6h 795M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:42:36.109 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 9m ago 6h 29.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:42:36.109 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 7m ago 6h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:42:36.109 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 9m ago 6h 972M 2411M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:42:36.109 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 9m ago 6h 961M 2411M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:42:36.110 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 9m ago 6h 749M 2411M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:42:36.110 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 9m ago 6h 939M 2411M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:42:36.110 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 7m ago 6h 827M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:42:36.110 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 7m ago 6h 1057M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:42:36.111 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 7m ago 6h 922M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:42:36.111 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 7m ago 6h 1119M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:42:36.111 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 7m ago 6h 154M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:42:36.461 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:42:36.462 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:42:36.462 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:42:36.462 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:42:36.463 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:42:36.463 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:42:36.463 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:42:36.463 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:42:36.464 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:42:36.464 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:42:36.464 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:42:36.464 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:42:36.465 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:42:36.465 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:42:36.465 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:42:36.465 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:42:36.465 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:42:36.466 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:42:36.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:42:36 smithi083 ceph-mon[79930]: pgmap v11229: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:36.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:42:36 smithi083 ceph-mon[77242]: pgmap v11229: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:36.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:42:36 smithi153 ceph-mon[70044]: pgmap v11229: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:37.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:42:37 smithi083 ceph-mon[79930]: from='client.47190 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:42:37.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:42:37 smithi083 ceph-mon[79930]: from='client.47196 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:42:37.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:42:37 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1971087407' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:42:37.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:42:37 smithi083 ceph-mon[77242]: from='client.47190 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:42:37.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:42:37 smithi083 ceph-mon[77242]: from='client.47196 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:42:37.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:42:37 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1971087407' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:42:37.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:42:37 smithi153 ceph-mon[70044]: from='client.47190 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:42:37.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:42:37 smithi153 ceph-mon[70044]: from='client.47196 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:42:37.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:42:37 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1971087407' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:42:38.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:42:38 smithi083 ceph-mon[79930]: pgmap v11230: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:38.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:42:38 smithi083 ceph-mon[77242]: pgmap v11230: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:38.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:42:38 smithi153 ceph-mon[70044]: pgmap v11230: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:42:40 smithi083 ceph-mon[79930]: pgmap v11231: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:42:40 smithi083 ceph-mon[77242]: pgmap v11231: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:42:40 smithi153 ceph-mon[70044]: pgmap v11231: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:41.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:42:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:42:41] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T11:42:41.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:42:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:42:41.544Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:42:41.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:42:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:42:41.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:42:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:42:41.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:42:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:42:41.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:42:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:42:41.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:42:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:42:41.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:42:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:42:41.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:42:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:42:41.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:42:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:42:41.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:42:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:42:41.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:42:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:42:41.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:42:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:42:41.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:42:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:42:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:42:42 smithi083 ceph-mon[79930]: pgmap v11232: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:42:42 smithi083 ceph-mon[77242]: pgmap v11232: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:42:42 smithi153 ceph-mon[70044]: pgmap v11232: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:42:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:42:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:42:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:42:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:42:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:42:44.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:42:44 smithi083 ceph-mon[79930]: pgmap v11233: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:44.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:42:44 smithi083 ceph-mon[77242]: pgmap v11233: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:44.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:42:44 smithi153 ceph-mon[70044]: pgmap v11233: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:45.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:42:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:42:44.917Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:42:45.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:42:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:42:45.643Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:42:46.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:42:46 smithi083 ceph-mon[77242]: pgmap v11234: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:46.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:42:46 smithi083 ceph-mon[79930]: pgmap v11234: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:46.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:42:46 smithi153 ceph-mon[70044]: pgmap v11234: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:48.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:42:48 smithi083 ceph-mon[79930]: pgmap v11235: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:48.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:42:48 smithi083 ceph-mon[77242]: pgmap v11235: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:48.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:42:48 smithi153 ceph-mon[70044]: pgmap v11235: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:50.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:42:50 smithi083 ceph-mon[79930]: pgmap v11236: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:50.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:42:50 smithi083 ceph-mon[77242]: pgmap v11236: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:50.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:42:50 smithi153 ceph-mon[70044]: pgmap v11236: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:42:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:42:51] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T11:42:51.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:42:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:42:51.545Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:42:52.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:42:52 smithi083 ceph-mon[79930]: pgmap v11237: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:52.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:42:52 smithi083 ceph-mon[77242]: pgmap v11237: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:52.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:42:52 smithi153 ceph-mon[70044]: pgmap v11237: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:42:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:42:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:42:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:42:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:42:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:42:54.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:42:54 smithi083 ceph-mon[79930]: pgmap v11238: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:54.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:42:54 smithi083 ceph-mon[77242]: pgmap v11238: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:54.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:42:54 smithi153 ceph-mon[70044]: pgmap v11238: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:55.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:42:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:42:54.918Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:42:55.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:42:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:42:55.643Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:42:56.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:42:56 smithi083 ceph-mon[79930]: pgmap v11239: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:56.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:42:56 smithi083 ceph-mon[77242]: pgmap v11239: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:56.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:42:56 smithi153 ceph-mon[70044]: pgmap v11239: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:58.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:42:58 smithi083 ceph-mon[77242]: pgmap v11240: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:58.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:42:58 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:42:58.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:42:58 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:42:58.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:42:58 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:42:58.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:42:58 smithi083 ceph-mon[79930]: pgmap v11240: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:58.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:42:58 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:42:58.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:42:58 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:42:58.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:42:58 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:42:58.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:42:58 smithi153 ceph-mon[70044]: pgmap v11240: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:42:58.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:42:58 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:42:58.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:42:58 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:42:58.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:42:58 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:43:00.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:42:59 smithi153 ceph-mon[70044]: pgmap v11241: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:00.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:42:59 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:43:00.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:42:59 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:43:00.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:42:59 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:43:00.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:42:59 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:43:00.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:42:59 smithi083 ceph-mon[77242]: pgmap v11241: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:00.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:42:59 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:43:00.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:42:59 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:43:00.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:42:59 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:43:00.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:42:59 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:43:00.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:42:59 smithi083 ceph-mon[79930]: pgmap v11241: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:00.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:42:59 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:43:00.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:42:59 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:43:00.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:42:59 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:43:00.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:42:59 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:43:01.789 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:43:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:43:01] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T11:43:01.790 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:43:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:43:01.545Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:43:02.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:43:01 smithi153 ceph-mon[70044]: pgmap v11242: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:02.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:43:01 smithi083 ceph-mon[79930]: pgmap v11242: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:02.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:43:01 smithi083 ceph-mon[77242]: pgmap v11242: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:43:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:43:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:43:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:43:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:43:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:43:04.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:43:04 smithi083 ceph-mon[79930]: pgmap v11243: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:04.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:43:04 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:43:04.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:43:04 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:43:04.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:43:04 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:43:04.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:43:04 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:43:04.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:43:04 smithi083 ceph-mon[77242]: pgmap v11243: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:04.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:43:04 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:43:04.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:43:04 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:43:04.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:43:04 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:43:04.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:43:04 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:43:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:43:04 smithi153 ceph-mon[70044]: pgmap v11243: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:43:04 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:43:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:43:04 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:43:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:43:04 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:43:04.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:43:04 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:43:05.199 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:43:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:43:04.918Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:43:05.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:43:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:43:05.644Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:43:06.358 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:43:06 smithi083 ceph-mon[77242]: pgmap v11244: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:06.358 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:43:06 smithi083 ceph-mon[79930]: pgmap v11244: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:06.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:43:06 smithi153 ceph-mon[70044]: pgmap v11244: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:06.791 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:43:07.081 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:43:07.082 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 4s ago 6h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:43:07.082 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 8m ago 6h 63.7M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:43:07.082 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 4s ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:43:07.082 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 8m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:43:07.083 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 8m ago 6h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:43:07.083 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 4s ago 6h 831M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:43:07.083 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 4s ago 6h 1415M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:43:07.083 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 8m ago 6h 595M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:43:07.083 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 4s ago 6h 816M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:43:07.084 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 4s ago 6h 26.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:43:07.084 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 8m ago 6h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:43:07.084 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 4s ago 6h 997M 2411M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:43:07.084 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 4s ago 6h 986M 2411M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:43:07.085 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 4s ago 6h 769M 2411M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:43:07.085 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 4s ago 6h 964M 2411M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:43:07.085 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 8m ago 6h 827M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:43:07.085 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 8m ago 6h 1057M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:43:07.085 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 8m ago 6h 922M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:43:07.086 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 8m ago 6h 1119M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:43:07.086 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 8m ago 6h 154M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:43:07.439 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:43:07.440 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:43:07.440 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:43:07.440 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:43:07.440 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:43:07.440 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:43:07.441 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:43:07.441 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:43:07.441 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:43:07.441 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:43:07.442 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:43:07.442 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:43:07.442 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:43:07.442 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:43:07.442 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:43:07.443 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:43:07.443 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:43:07.443 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:43:07.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:43:07 smithi083 ceph-mon[77242]: pgmap v11245: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:07.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:43:07 smithi083 ceph-mon[77242]: Detected new or changed devices on smithi083 2022-09-29T11:43:07.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:43:07 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:43:07.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:43:07 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:43:07.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:43:07 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:43:07.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:43:07 smithi083 ceph-mon[77242]: from='client.57139 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:43:07.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:43:07 smithi083 ceph-mon[77242]: from='client.57145 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:43:07.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:43:07 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1239828975' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:43:07.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:43:07 smithi083 ceph-mon[79930]: pgmap v11245: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:07.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:43:07 smithi083 ceph-mon[79930]: Detected new or changed devices on smithi083 2022-09-29T11:43:07.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:43:07 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:43:07.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:43:07 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:43:07.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:43:07 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:43:07.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:43:07 smithi083 ceph-mon[79930]: from='client.57139 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:43:07.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:43:07 smithi083 ceph-mon[79930]: from='client.57145 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:43:07.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:43:07 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1239828975' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:43:08.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:43:07 smithi153 ceph-mon[70044]: pgmap v11245: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:08.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:43:07 smithi153 ceph-mon[70044]: Detected new or changed devices on smithi083 2022-09-29T11:43:08.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:43:07 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:43:08.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:43:07 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:43:08.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:43:07 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:43:08.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:43:07 smithi153 ceph-mon[70044]: from='client.57139 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:43:08.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:43:07 smithi153 ceph-mon[70044]: from='client.57145 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:43:08.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:43:07 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1239828975' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:43:09.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:43:09 smithi083 ceph-mon[79930]: pgmap v11246: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:09.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:43:09 smithi083 ceph-mon[77242]: pgmap v11246: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:10.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:43:09 smithi153 ceph-mon[70044]: pgmap v11246: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:11.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:43:11 smithi083 ceph-mon[77242]: pgmap v11247: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:11.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:43:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:43:11] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T11:43:11.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:43:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:43:11.546Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:43:11.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:43:11 smithi083 ceph-mon[79930]: pgmap v11247: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:12.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:43:11 smithi153 ceph-mon[70044]: pgmap v11247: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:43:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:43:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:43:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:43:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:43:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:43:13.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:43:13 smithi083 ceph-mon[79930]: pgmap v11248: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:13.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:43:13 smithi083 ceph-mon[77242]: pgmap v11248: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:14.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:43:13 smithi153 ceph-mon[70044]: pgmap v11248: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:15.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:43:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:43:14.919Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:43:15.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:43:15 smithi083 ceph-mon[77242]: pgmap v11249: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:15.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:43:15 smithi083 ceph-mon[79930]: pgmap v11249: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:15.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:43:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:43:15.645Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:43:16.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:43:15 smithi153 ceph-mon[70044]: pgmap v11249: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:17.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:43:17 smithi083 ceph-mon[77242]: pgmap v11250: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:17.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:43:17 smithi083 ceph-mon[79930]: pgmap v11250: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:18.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:43:17 smithi153 ceph-mon[70044]: pgmap v11250: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:19.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:43:19 smithi083 ceph-mon[79930]: pgmap v11251: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:19.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:43:19 smithi083 ceph-mon[77242]: pgmap v11251: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:20.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:43:19 smithi153 ceph-mon[70044]: pgmap v11251: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:43:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:43:21] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T11:43:21.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:43:21 smithi083 ceph-mon[79930]: pgmap v11252: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:21.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:43:21 smithi083 ceph-mon[77242]: pgmap v11252: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:21.964 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:43:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:43:21.547Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:43:22.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:43:21 smithi153 ceph-mon[70044]: pgmap v11252: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:43:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:43:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:43:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:43:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:43:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:43:23.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:43:23 smithi083 ceph-mon[79930]: pgmap v11253: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:23.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:43:23 smithi083 ceph-mon[77242]: pgmap v11253: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:24.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:43:23 smithi153 ceph-mon[70044]: pgmap v11253: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:25.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:43:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:43:24.919Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:43:25.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:43:25 smithi083 ceph-mon[79930]: pgmap v11254: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:25.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:43:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:43:25.646Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:43:25.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:43:25 smithi083 ceph-mon[77242]: pgmap v11254: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:26.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:43:25 smithi153 ceph-mon[70044]: pgmap v11254: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:27.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:43:27 smithi083 ceph-mon[79930]: pgmap v11255: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:27.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:43:27 smithi083 ceph-mon[77242]: pgmap v11255: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:28.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:43:27 smithi153 ceph-mon[70044]: pgmap v11255: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:29.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:43:29 smithi083 ceph-mon[79930]: pgmap v11256: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:29.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:43:29 smithi083 ceph-mon[77242]: pgmap v11256: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:30.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:43:29 smithi153 ceph-mon[70044]: pgmap v11256: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:31.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:43:31 smithi083 ceph-mon[77242]: pgmap v11257: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:31.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:43:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:43:31] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T11:43:31.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:43:31 smithi083 ceph-mon[79930]: pgmap v11257: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:31.964 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:43:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:43:31.547Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:43:32.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:43:31 smithi153 ceph-mon[70044]: pgmap v11257: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:43:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:43:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:43:33.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:43:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:43:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:43:33.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:43:33 smithi083 ceph-mon[77242]: pgmap v11258: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:33.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:43:33 smithi083 ceph-mon[79930]: pgmap v11258: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:34.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:43:33 smithi153 ceph-mon[70044]: pgmap v11258: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:35.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:43:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:43:34.921Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:43:35.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:43:35 smithi083 ceph-mon[77242]: pgmap v11259: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:35.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:43:35 smithi083 ceph-mon[79930]: pgmap v11259: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:35.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:43:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:43:35.647Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:43:36.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:43:35 smithi153 ceph-mon[70044]: pgmap v11259: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:37.767 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:43:37.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:43:37 smithi083 ceph-mon[79930]: pgmap v11260: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:37.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:43:37 smithi083 ceph-mon[77242]: pgmap v11260: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:38.056 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:43:38.056 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 34s ago 6h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:43:38.057 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 8m ago 6h 63.7M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:43:38.057 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 34s ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:43:38.057 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 8m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:43:38.057 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 8m ago 6h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:43:38.058 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 34s ago 6h 831M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:43:38.058 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 34s ago 6h 1415M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:43:38.058 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 8m ago 6h 595M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:43:38.058 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 34s ago 6h 816M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:43:38.059 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 34s ago 6h 26.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:43:38.059 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 8m ago 6h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:43:38.059 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 34s ago 6h 997M 2411M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:43:38.059 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 34s ago 6h 986M 2411M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:43:38.060 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 34s ago 6h 769M 2411M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:43:38.060 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 34s ago 6h 964M 2411M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:43:38.060 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 8m ago 6h 827M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:43:38.060 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 8m ago 6h 1057M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:43:38.061 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 8m ago 6h 922M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:43:38.061 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 8m ago 6h 1119M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:43:38.061 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 8m ago 6h 154M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:43:38.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:43:37 smithi153 ceph-mon[70044]: pgmap v11260: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:38.410 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:43:38.411 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:43:38.411 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:43:38.411 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:43:38.412 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:43:38.412 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:43:38.412 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:43:38.412 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:43:38.412 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:43:38.413 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:43:38.413 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:43:38.413 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:43:38.413 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:43:38.414 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:43:38.414 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:43:38.414 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:43:38.414 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:43:38.415 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:43:38.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:43:38 smithi083 ceph-mon[77242]: from='client.57157 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:43:38.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:43:38 smithi083 ceph-mon[77242]: from='client.47232 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:43:38.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:43:38 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1869655753' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:43:38.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:43:38 smithi083 ceph-mon[79930]: from='client.57157 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:43:38.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:43:38 smithi083 ceph-mon[79930]: from='client.47232 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:43:38.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:43:38 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1869655753' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:43:39.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:43:38 smithi153 ceph-mon[70044]: from='client.57157 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:43:39.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:43:38 smithi153 ceph-mon[70044]: from='client.47232 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:43:39.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:43:38 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1869655753' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:43:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:43:39 smithi083 ceph-mon[79930]: pgmap v11261: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:43:39 smithi083 ceph-mon[77242]: pgmap v11261: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:43:39 smithi153 ceph-mon[70044]: pgmap v11261: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:43:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:43:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:43:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:43:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:43:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:43:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:43:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:43:40.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:43:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:43:40.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:43:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:43:40.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:43:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:43:40.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:43:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:43:41.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:43:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:43:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:43:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:43:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:43:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:43:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:43:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:43:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:43:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:43:41] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T11:43:41.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:43:41 smithi083 ceph-mon[79930]: pgmap v11262: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:41.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:43:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:43:41.548Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:43:41.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:43:41 smithi083 ceph-mon[77242]: pgmap v11262: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:42.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:43:41 smithi153 ceph-mon[70044]: pgmap v11262: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:43:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:43:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:43:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:43:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:43:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:43:43.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:43:43 smithi083 ceph-mon[79930]: pgmap v11263: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:43.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:43:43 smithi083 ceph-mon[77242]: pgmap v11263: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:44.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:43:43 smithi153 ceph-mon[70044]: pgmap v11263: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:45.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:43:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:43:44.921Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:43:45.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:43:45 smithi083 ceph-mon[79930]: pgmap v11264: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:45.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:43:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:43:45.647Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:43:45.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:43:45 smithi083 ceph-mon[77242]: pgmap v11264: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:46.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:43:45 smithi153 ceph-mon[70044]: pgmap v11264: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:47.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:43:47 smithi083 ceph-mon[77242]: pgmap v11265: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:47.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:43:47 smithi083 ceph-mon[79930]: pgmap v11265: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:48.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:43:47 smithi153 ceph-mon[70044]: pgmap v11265: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:49.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:43:49 smithi083 ceph-mon[77242]: pgmap v11266: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:49.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:43:49 smithi083 ceph-mon[79930]: pgmap v11266: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:50.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:43:49 smithi153 ceph-mon[70044]: pgmap v11266: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:43:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:43:51] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T11:43:51.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:43:51 smithi083 ceph-mon[79930]: pgmap v11267: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:51.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:43:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:43:51.548Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:43:51.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:43:51 smithi083 ceph-mon[77242]: pgmap v11267: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:52.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:43:51 smithi153 ceph-mon[70044]: pgmap v11267: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:43:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:43:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:43:53.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:43:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:43:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:43:53.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:43:53 smithi083 ceph-mon[79930]: pgmap v11268: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:53.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:43:53 smithi083 ceph-mon[77242]: pgmap v11268: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:54.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:43:53 smithi153 ceph-mon[70044]: pgmap v11268: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:55.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:43:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:43:54.921Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:43:55.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:43:55 smithi083 ceph-mon[79930]: pgmap v11269: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:55.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:43:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:43:55.648Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:43:55.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:43:55 smithi083 ceph-mon[77242]: pgmap v11269: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:56.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:43:55 smithi153 ceph-mon[70044]: pgmap v11269: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:57.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:43:57 smithi083 ceph-mon[79930]: pgmap v11270: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:57.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:43:57 smithi083 ceph-mon[77242]: pgmap v11270: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:58.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:43:57 smithi153 ceph-mon[70044]: pgmap v11270: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:59.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:43:59 smithi083 ceph-mon[79930]: pgmap v11271: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:43:59.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:43:59 smithi083 ceph-mon[77242]: pgmap v11271: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:44:00.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:43:59 smithi153 ceph-mon[70044]: pgmap v11271: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:44:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:44:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:44:01] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T11:44:01.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:44:01 smithi083 ceph-mon[79930]: pgmap v11272: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:44:01.964 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:44:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:44:01.549Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:44:01.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:44:01 smithi083 ceph-mon[77242]: pgmap v11272: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:44:02.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:44:01 smithi153 ceph-mon[70044]: pgmap v11272: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:44:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:44:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:44:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:44:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:44:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:44:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:44:03.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:44:03 smithi083 ceph-mon[79930]: pgmap v11273: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:44:03.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:44:03 smithi083 ceph-mon[77242]: pgmap v11273: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:44:04.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:44:03 smithi153 ceph-mon[70044]: pgmap v11273: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:44:05.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:44:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:44:04.921Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:44:05.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:44:05 smithi083 ceph-mon[79930]: pgmap v11274: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:44:05.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:44:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:44:05.649Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:44:05.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:44:05 smithi083 ceph-mon[77242]: pgmap v11274: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:44:06.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:44:05 smithi153 ceph-mon[70044]: pgmap v11274: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:44:07.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:44:07 smithi083 ceph-mon[79930]: pgmap v11275: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:44:07.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:44:07 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:44:07.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:44:07 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:44:07.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:44:07 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:44:07.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:44:07 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:44:07.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:44:07 smithi083 ceph-mon[77242]: pgmap v11275: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:44:07.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:44:07 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:44:07.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:44:07 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:44:07.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:44:07 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:44:07.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:44:07 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:44:08.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:44:07 smithi153 ceph-mon[70044]: pgmap v11275: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:44:08.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:44:07 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:44:08.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:44:07 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:44:08.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:44:07 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:44:08.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:44:07 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:44:08.739 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:44:09.027 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:44:09.028 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 65s ago 6h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:44:09.028 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 9m ago 6h 63.7M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:44:09.028 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 65s ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:44:09.028 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 9m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:44:09.029 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 9m ago 6h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:44:09.029 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 65s ago 6h 831M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:44:09.029 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 65s ago 6h 1415M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:44:09.029 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 9m ago 6h 595M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:44:09.030 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 65s ago 6h 816M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:44:09.030 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 65s ago 6h 26.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:44:09.030 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 9m ago 6h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:44:09.030 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 65s ago 6h 997M 2411M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:44:09.030 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 65s ago 6h 986M 2411M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:44:09.031 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 65s ago 6h 769M 2411M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:44:09.031 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 65s ago 6h 964M 2411M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:44:09.031 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 9m ago 6h 827M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:44:09.031 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 9m ago 6h 1057M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:44:09.031 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 9m ago 6h 922M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:44:09.032 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 9m ago 6h 1119M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:44:09.032 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 9m ago 6h 154M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:44:09.383 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:44:09.383 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:44:09.384 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:44:09.384 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:44:09.384 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:44:09.384 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:44:09.385 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:44:09.385 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:44:09.385 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:44:09.385 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:44:09.385 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:44:09.386 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:44:09.386 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:44:09.386 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:44:09.386 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:44:09.386 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:44:09.387 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:44:09.387 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:44:10.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:44:09 smithi153 ceph-mon[70044]: pgmap v11276: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:44:10.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:44:09 smithi153 ceph-mon[70044]: from='client.47244 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:44:10.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:44:09 smithi153 ceph-mon[70044]: from='client.47250 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:44:10.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:44:09 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2389741877' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:44:10.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:44:09 smithi083 ceph-mon[79930]: pgmap v11276: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:44:10.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:44:09 smithi083 ceph-mon[79930]: from='client.47244 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:44:10.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:44:09 smithi083 ceph-mon[79930]: from='client.47250 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:44:10.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:44:09 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2389741877' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:44:10.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:44:09 smithi083 ceph-mon[77242]: pgmap v11276: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:44:10.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:44:09 smithi083 ceph-mon[77242]: from='client.47244 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:44:10.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:44:09 smithi083 ceph-mon[77242]: from='client.47250 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:44:10.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:44:09 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2389741877' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:44:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:44:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:44:11] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T11:44:11.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:44:11 smithi083 ceph-mon[79930]: pgmap v11277: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:44:11.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:44:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:44:11.550Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:44:11.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:44:11 smithi083 ceph-mon[77242]: pgmap v11277: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:44:12.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:44:11 smithi153 ceph-mon[70044]: pgmap v11277: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:44:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:44:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:44:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:44:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:44:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:44:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:44:14.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:44:13 smithi153 ceph-mon[70044]: pgmap v11278: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:44:14.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:44:13 smithi083 ceph-mon[79930]: pgmap v11278: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:44:14.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:44:13 smithi083 ceph-mon[77242]: pgmap v11278: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:44:15.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:44:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:44:14.922Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:44:15.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:44:15 smithi083 ceph-mon[79930]: pgmap v11279: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:44:15.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:44:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:44:15.650Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:44:15.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:44:15 smithi083 ceph-mon[77242]: pgmap v11279: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:44:16.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:44:15 smithi153 ceph-mon[70044]: pgmap v11279: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:44:18.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:44:17 smithi153 ceph-mon[70044]: pgmap v11280: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:44:18.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:44:17 smithi083 ceph-mon[77242]: pgmap v11280: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:44:18.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:44:17 smithi083 ceph-mon[79930]: pgmap v11280: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:44:20.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:44:19 smithi153 ceph-mon[70044]: pgmap v11281: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:44:20.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:44:19 smithi083 ceph-mon[79930]: pgmap v11281: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:44:20.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:44:19 smithi083 ceph-mon[77242]: pgmap v11281: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:44:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:44:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:44:21] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T11:44:21.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:44:21 smithi083 ceph-mon[79930]: pgmap v11282: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:44:21.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:44:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:44:21.550Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:44:21.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:44:21 smithi083 ceph-mon[77242]: pgmap v11282: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:44:22.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:44:21 smithi153 ceph-mon[70044]: pgmap v11282: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:44:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:44:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:44:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:44:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:44:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:44:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:44:24.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:44:23 smithi153 ceph-mon[70044]: pgmap v11283: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:44:24.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:44:23 smithi083 ceph-mon[79930]: pgmap v11283: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:44:24.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:44:23 smithi083 ceph-mon[77242]: pgmap v11283: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:44:25.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:44:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:44:24.923Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:44:25.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:44:25 smithi083 ceph-mon[79930]: pgmap v11284: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:44:25.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:44:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:44:25.650Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:44:25.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:44:25 smithi083 ceph-mon[77242]: pgmap v11284: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:44:26.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:44:25 smithi153 ceph-mon[70044]: pgmap v11284: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:44:28.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:44:27 smithi153 ceph-mon[70044]: pgmap v11285: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:44:28.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:44:27 smithi083 ceph-mon[79930]: pgmap v11285: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:44:28.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:44:27 smithi083 ceph-mon[77242]: pgmap v11285: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:44:30.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:44:29 smithi153 ceph-mon[70044]: pgmap v11286: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:44:30.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:44:29 smithi083 ceph-mon[79930]: pgmap v11286: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:44:30.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:44:29 smithi083 ceph-mon[77242]: pgmap v11286: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:44:31.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:44:31 smithi083 ceph-mon[77242]: pgmap v11287: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:44:31.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:44:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:44:31] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T11:44:31.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:44:31 smithi083 ceph-mon[79930]: pgmap v11287: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:44:31.964 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:44:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:44:31.551Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:44:32.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:44:31 smithi153 ceph-mon[70044]: pgmap v11287: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:44:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:44:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:44:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:44:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:44:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:44:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:44:34.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:44:33 smithi153 ceph-mon[70044]: pgmap v11288: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:44:34.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:44:33 smithi083 ceph-mon[77242]: pgmap v11288: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:44:34.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:44:33 smithi083 ceph-mon[79930]: pgmap v11288: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:44:35.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:44:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:44:34.924Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:44:35.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:44:35 smithi083 ceph-mon[79930]: pgmap v11289: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:44:35.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:44:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:44:35.651Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:44:35.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:44:35 smithi083 ceph-mon[77242]: pgmap v11289: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:44:36.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:44:35 smithi153 ceph-mon[70044]: pgmap v11289: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:44:38.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:44:37 smithi153 ceph-mon[70044]: pgmap v11290: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:44:38.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:44:37 smithi083 ceph-mon[77242]: pgmap v11290: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:44:38.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:44:37 smithi083 ceph-mon[79930]: pgmap v11290: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:44:39.708 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:44:40.000 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:44:40.000 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 96s ago 6h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:44:40.000 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 9m ago 6h 63.7M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:44:40.001 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 96s ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:44:40.001 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 9m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:44:40.001 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 9m ago 6h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:44:40.001 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 96s ago 6h 831M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:44:40.001 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 96s ago 6h 1415M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:44:40.002 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 9m ago 6h 595M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:44:40.002 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 96s ago 6h 816M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:44:40.002 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 96s ago 6h 26.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:44:40.002 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 9m ago 6h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:44:40.003 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 96s ago 6h 997M 2411M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:44:40.003 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 96s ago 6h 986M 2411M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:44:40.003 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 96s ago 6h 769M 2411M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:44:40.003 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 96s ago 6h 964M 2411M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:44:40.003 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 9m ago 6h 827M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:44:40.004 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 9m ago 6h 1057M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:44:40.004 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 9m ago 6h 922M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:44:40.004 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 9m ago 6h 1119M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:44:40.004 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 9m ago 6h 154M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:44:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:44:39 smithi153 ceph-mon[70044]: pgmap v11291: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:44:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:44:39 smithi083 ceph-mon[79930]: pgmap v11291: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:44:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:44:39 smithi083 ceph-mon[77242]: pgmap v11291: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:44:40.355 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:44:40.355 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:44:40.355 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:44:40.355 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:44:40.356 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:44:40.356 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:44:40.356 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:44:40.356 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:44:40.357 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:44:40.357 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:44:40.357 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:44:40.357 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:44:40.358 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:44:40.358 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:44:40.358 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:44:40.359 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:44:40.359 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:44:40.359 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:44:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:44:40 smithi153 ceph-mon[70044]: from='client.57193 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:44:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:44:40 smithi153 ceph-mon[70044]: from='client.47268 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:44:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:44:40 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3561769215' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:44:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:44:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:44:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:44:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:44:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:44:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:44:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:44:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:44:41.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:44:40 smithi083 ceph-mon[79930]: from='client.57193 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:44:41.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:44:40 smithi083 ceph-mon[79930]: from='client.47268 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:44:41.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:44:40 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3561769215' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:44:41.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:44:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:44:41.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:44:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:44:41.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:44:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:44:41.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:44:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:44:41.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:44:40 smithi083 ceph-mon[77242]: from='client.57193 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:44:41.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:44:40 smithi083 ceph-mon[77242]: from='client.47268 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:44:41.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:44:40 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3561769215' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:44:41.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:44:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:44:41.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:44:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:44:41.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:44:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:44:41.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:44:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:44:41.779 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:44:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:44:41] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T11:44:41.780 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:44:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:44:41.552Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:44:42.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:44:41 smithi153 ceph-mon[70044]: pgmap v11292: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:44:42.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:44:41 smithi083 ceph-mon[79930]: pgmap v11292: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:44:42.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:44:41 smithi083 ceph-mon[77242]: pgmap v11292: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:44:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:44:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:44:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:44:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:44:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:44:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:44:44.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:44:43 smithi153 ceph-mon[70044]: pgmap v11293: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:44:44.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:44:43 smithi083 ceph-mon[79930]: pgmap v11293: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:44:44.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:44:43 smithi083 ceph-mon[77242]: pgmap v11293: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:44:45.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:44:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:44:44.924Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:44:45.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:44:45 smithi083 ceph-mon[79930]: pgmap v11294: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:44:45.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:44:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:44:45.651Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:44:45.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:44:45 smithi083 ceph-mon[77242]: pgmap v11294: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:44:46.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:44:45 smithi153 ceph-mon[70044]: pgmap v11294: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:44:48.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:44:47 smithi153 ceph-mon[70044]: pgmap v11295: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:44:48.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:44:47 smithi083 ceph-mon[77242]: pgmap v11295: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:44:48.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:44:47 smithi083 ceph-mon[79930]: pgmap v11295: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:44:50.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:44:49 smithi153 ceph-mon[70044]: pgmap v11296: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:44:50.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:44:49 smithi083 ceph-mon[79930]: pgmap v11296: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:44:50.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:44:49 smithi083 ceph-mon[77242]: pgmap v11296: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:44:51.794 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:44:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:44:51] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T11:44:51.794 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:44:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:44:51.553Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:44:52.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:44:51 smithi153 ceph-mon[70044]: pgmap v11297: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:44:52.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:44:51 smithi083 ceph-mon[79930]: pgmap v11297: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:44:52.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:44:51 smithi083 ceph-mon[77242]: pgmap v11297: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:44:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:44:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:44:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:44:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:44:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:44:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:44:54.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:44:53 smithi153 ceph-mon[70044]: pgmap v11298: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:44:54.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:44:53 smithi083 ceph-mon[79930]: pgmap v11298: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:44:54.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:44:53 smithi083 ceph-mon[77242]: pgmap v11298: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:44:55.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:44:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:44:54.925Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:44:55.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:44:55 smithi083 ceph-mon[79930]: pgmap v11299: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:44:55.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:44:55 smithi083 ceph-mon[77242]: pgmap v11299: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:44:55.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:44:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:44:55.652Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:44:56.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:44:55 smithi153 ceph-mon[70044]: pgmap v11299: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:44:58.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:44:57 smithi153 ceph-mon[70044]: pgmap v11300: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:44:58.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:44:57 smithi083 ceph-mon[79930]: pgmap v11300: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:44:58.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:44:57 smithi083 ceph-mon[77242]: pgmap v11300: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:00.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:44:59 smithi153 ceph-mon[70044]: pgmap v11301: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:00.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:44:59 smithi083 ceph-mon[79930]: pgmap v11301: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:00.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:44:59 smithi083 ceph-mon[77242]: pgmap v11301: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:01.807 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:45:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:45:01] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T11:45:01.808 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:45:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:45:01.554Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:45:02.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:45:01 smithi083 ceph-mon[79930]: pgmap v11302: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:02.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:45:01 smithi083 ceph-mon[77242]: pgmap v11302: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:02.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:45:01 smithi153 ceph-mon[70044]: pgmap v11302: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:45:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:45:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:45:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:45:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:45:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:45:04.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:45:03 smithi083 ceph-mon[79930]: pgmap v11303: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:04.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:45:03 smithi083 ceph-mon[77242]: pgmap v11303: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:04.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:45:03 smithi153 ceph-mon[70044]: pgmap v11303: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:05.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:45:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:45:04.925Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:45:05.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:45:05 smithi083 ceph-mon[79930]: pgmap v11304: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:05.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:45:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:45:05.652Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:45:05.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:45:05 smithi083 ceph-mon[77242]: pgmap v11304: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:06.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:45:05 smithi153 ceph-mon[70044]: pgmap v11304: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:08.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:45:07 smithi083 ceph-mon[79930]: pgmap v11305: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:08.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:45:07 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:45:08.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:45:07 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:45:08.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:45:07 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:45:08.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:45:07 smithi083 ceph-mon[77242]: pgmap v11305: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:08.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:45:07 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:45:08.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:45:07 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:45:08.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:45:07 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:45:08.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:45:07 smithi153 ceph-mon[70044]: pgmap v11305: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:08.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:45:07 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:45:08.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:45:07 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:45:08.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:45:07 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:45:10.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:45:09 smithi083 ceph-mon[79930]: pgmap v11306: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:10.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:45:09 smithi083 ceph-mon[77242]: pgmap v11306: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:10.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:45:09 smithi153 ceph-mon[70044]: pgmap v11306: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:10.683 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:45:10.971 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:45:10.972 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 2m ago 6h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:45:10.972 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 10m ago 6h 63.7M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:45:10.972 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 2m ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:45:10.973 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 10m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:45:10.973 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 10m ago 6h 445M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:45:10.973 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 2m ago 6h 831M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:45:10.973 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 2m ago 6h 1415M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:45:10.974 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 10m ago 6h 595M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:45:10.974 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 2m ago 6h 816M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:45:10.974 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 2m ago 6h 26.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:45:10.974 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 10m ago 6h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:45:10.974 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 2m ago 6h 997M 2411M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:45:10.975 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 2m ago 6h 986M 2411M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:45:10.975 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 2m ago 6h 769M 2411M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:45:10.975 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 2m ago 6h 964M 2411M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:45:10.975 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 10m ago 6h 827M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:45:10.976 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 10m ago 6h 1057M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:45:10.976 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 10m ago 6h 922M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:45:10.976 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 10m ago 6h 1119M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:45:10.976 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 10m ago 6h 154M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:45:11.326 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:45:11.327 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:45:11.327 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:45:11.327 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:45:11.327 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:45:11.328 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:45:11.328 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:45:11.328 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:45:11.328 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:45:11.329 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:45:11.329 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:45:11.329 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:45:11.329 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:45:11.329 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:45:11.330 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:45:11.330 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:45:11.330 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:45:11.330 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:45:11.823 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:45:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:45:11] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T11:45:11.823 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:45:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:45:11.554Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:45:12.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:45:11 smithi083 ceph-mon[77242]: pgmap v11307: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:12.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:45:11 smithi083 ceph-mon[77242]: from='client.47280 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:45:12.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:45:11 smithi083 ceph-mon[77242]: from='client.47286 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:45:12.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:45:11 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1289531017' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:45:12.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:45:11 smithi083 ceph-mon[79930]: pgmap v11307: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:12.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:45:11 smithi083 ceph-mon[79930]: from='client.47280 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:45:12.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:45:11 smithi083 ceph-mon[79930]: from='client.47286 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:45:12.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:45:11 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1289531017' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:45:12.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:45:11 smithi153 ceph-mon[70044]: pgmap v11307: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:12.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:45:11 smithi153 ceph-mon[70044]: from='client.47280 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:45:12.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:45:11 smithi153 ceph-mon[70044]: from='client.47286 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:45:12.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:45:11 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1289531017' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:45:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:45:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:45:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:45:13.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:45:12 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:45:13.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:45:12 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:45:13.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:45:12 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:45:13.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:45:12 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:45:13.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:45:12 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:45:13.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:45:12 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:45:13.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:45:12 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:45:13.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:45:12 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:45:13.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:45:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:45:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:45:13.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:45:12 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:45:13.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:45:12 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:45:13.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:45:12 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:45:13.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:45:12 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:45:13.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:45:13 smithi153 ceph-mon[70044]: pgmap v11308: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:14.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:45:13 smithi083 ceph-mon[79930]: pgmap v11308: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:14.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:45:13 smithi083 ceph-mon[77242]: pgmap v11308: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:15.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:45:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:45:14.926Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:45:15.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:45:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:45:15.652Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:45:16.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:45:16 smithi083 ceph-mon[79930]: pgmap v11309: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:16.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:45:16 smithi083 ceph-mon[79930]: Detected new or changed devices on smithi153 2022-09-29T11:45:16.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:45:16 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:45:16.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:45:16 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:45:16.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:45:16 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:45:16.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:45:16 smithi083 ceph-mon[77242]: pgmap v11309: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:16.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:45:16 smithi083 ceph-mon[77242]: Detected new or changed devices on smithi153 2022-09-29T11:45:16.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:45:16 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:45:16.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:45:16 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:45:16.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:45:16 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:45:16.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:45:16 smithi153 ceph-mon[70044]: pgmap v11309: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:16.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:45:16 smithi153 ceph-mon[70044]: Detected new or changed devices on smithi153 2022-09-29T11:45:16.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:45:16 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:45:16.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:45:16 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:45:16.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:45:16 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:45:18.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:45:18 smithi083 ceph-mon[79930]: pgmap v11310: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:18.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:45:18 smithi083 ceph-mon[77242]: pgmap v11310: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:18.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:45:18 smithi153 ceph-mon[70044]: pgmap v11310: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:20.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:45:20 smithi083 ceph-mon[79930]: pgmap v11311: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:20.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:45:20 smithi083 ceph-mon[77242]: pgmap v11311: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:20.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:45:20 smithi153 ceph-mon[70044]: pgmap v11311: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:45:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:45:21] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T11:45:21.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:45:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:45:21.556Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:45:22.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:45:22 smithi083 ceph-mon[79930]: pgmap v11312: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:22.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:45:22 smithi083 ceph-mon[77242]: pgmap v11312: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:22.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:45:22 smithi153 ceph-mon[70044]: pgmap v11312: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:45:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:45:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:45:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:45:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:45:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:45:24.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:45:24 smithi083 ceph-mon[79930]: pgmap v11313: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:24.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:45:24 smithi083 ceph-mon[77242]: pgmap v11313: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:24.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:45:24 smithi153 ceph-mon[70044]: pgmap v11313: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:25.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:45:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:45:24.926Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:45:25.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:45:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:45:25.653Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:45:26.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:45:26 smithi083 ceph-mon[77242]: pgmap v11314: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:26.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:45:26 smithi083 ceph-mon[79930]: pgmap v11314: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:26.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:45:26 smithi153 ceph-mon[70044]: pgmap v11314: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:28.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:45:28 smithi083 ceph-mon[79930]: pgmap v11315: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:28.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:45:28 smithi083 ceph-mon[77242]: pgmap v11315: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:28.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:45:28 smithi153 ceph-mon[70044]: pgmap v11315: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:30.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:45:30 smithi083 ceph-mon[79930]: pgmap v11316: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:30.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:45:30 smithi083 ceph-mon[77242]: pgmap v11316: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:30.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:45:30 smithi153 ceph-mon[70044]: pgmap v11316: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:45:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:45:31] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T11:45:31.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:45:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:45:31.556Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:45:32.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:45:32 smithi083 ceph-mon[79930]: pgmap v11317: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:32.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:45:32 smithi083 ceph-mon[77242]: pgmap v11317: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:32.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:45:32 smithi153 ceph-mon[70044]: pgmap v11317: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:45:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:45:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:45:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:45:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:45:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:45:33.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:45:33 smithi083 ceph-mon[79930]: pgmap v11318: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:33.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:45:33 smithi083 ceph-mon[77242]: pgmap v11318: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:33.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:45:33 smithi153 ceph-mon[70044]: pgmap v11318: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:35.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:45:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:45:34.926Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:45:35.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:45:35 smithi083 ceph-mon[77242]: pgmap v11319: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:35.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:45:35 smithi083 ceph-mon[79930]: pgmap v11319: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:35.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:45:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:45:35.654Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:45:36.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:45:35 smithi153 ceph-mon[70044]: pgmap v11319: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:37.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:45:37 smithi083 ceph-mon[79930]: pgmap v11320: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:37.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:45:37 smithi083 ceph-mon[77242]: pgmap v11320: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:38.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:45:37 smithi153 ceph-mon[70044]: pgmap v11320: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:45:39 smithi083 ceph-mon[79930]: pgmap v11321: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:45:39 smithi083 ceph-mon[77242]: pgmap v11321: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:45:39 smithi153 ceph-mon[70044]: pgmap v11321: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:45:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:45:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:45:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:45:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:45:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:45:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:45:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:45:40.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:45:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:45:40.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:45:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:45:40.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:45:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:45:40.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:45:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:45:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:45:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:45:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:45:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:45:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:45:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:45:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:45:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:45:41.654 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:45:41.942 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:45:41.943 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 2m ago 6h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:45:41.943 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 30s ago 6h 64.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:45:41.944 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 2m ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:45:41.944 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 30s ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:45:41.944 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 30s ago 6h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:45:41.945 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 2m ago 6h 831M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:45:41.945 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 2m ago 6h 1415M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:45:41.945 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 30s ago 6h 610M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:45:41.945 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 2m ago 6h 816M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:45:41.946 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 2m ago 6h 26.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:45:41.946 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 30s ago 6h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:45:41.946 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 2m ago 6h 997M 2411M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:45:41.946 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 2m ago 6h 986M 2411M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:45:41.947 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 2m ago 6h 769M 2411M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:45:41.947 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 2m ago 6h 964M 2411M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:45:41.947 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 30s ago 6h 848M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:45:41.947 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 30s ago 6h 1085M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:45:41.948 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 30s ago 6h 947M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:45:41.948 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 30s ago 6h 1148M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:45:41.948 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 30s ago 6h 156M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:45:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:45:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:45:41] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T11:45:41.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:45:41 smithi083 ceph-mon[79930]: pgmap v11322: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:41.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:45:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:45:41.556Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:45:41.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:45:41 smithi083 ceph-mon[77242]: pgmap v11322: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:42.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:45:41 smithi153 ceph-mon[70044]: pgmap v11322: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:42.298 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:45:42.298 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:45:42.299 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:45:42.299 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:45:42.299 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:45:42.299 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:45:42.300 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:45:42.300 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:45:42.300 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:45:42.301 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:45:42.301 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:45:42.301 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:45:42.301 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:45:42.302 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:45:42.302 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:45:42.302 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:45:42.302 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:45:42.303 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:45:42.666 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:45:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:45:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:45:42.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:45:42 smithi083 ceph-mon[77242]: from='client.57229 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:45:42.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:45:42 smithi083 ceph-mon[77242]: from='client.57235 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:45:42.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:45:42 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3855209207' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:45:42.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:45:42 smithi083 ceph-mon[79930]: from='client.57229 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:45:42.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:45:42 smithi083 ceph-mon[79930]: from='client.57235 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:45:42.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:45:42 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3855209207' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:45:43.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:45:42 smithi153 ceph-mon[70044]: from='client.57229 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:45:43.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:45:42 smithi153 ceph-mon[70044]: from='client.57235 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:45:43.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:45:42 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3855209207' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:45:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:45:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:45:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:45:43.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:45:43 smithi083 ceph-mon[79930]: pgmap v11323: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:43.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:45:43 smithi083 ceph-mon[77242]: pgmap v11323: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:44.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:45:43 smithi153 ceph-mon[70044]: pgmap v11323: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:45.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:45:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:45:44.927Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:45:45.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:45:45 smithi083 ceph-mon[79930]: pgmap v11324: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:45.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:45:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:45:45.654Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:45:45.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:45:45 smithi083 ceph-mon[77242]: pgmap v11324: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:46.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:45:45 smithi153 ceph-mon[70044]: pgmap v11324: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:47.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:45:47 smithi083 ceph-mon[79930]: pgmap v11325: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:47.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:45:47 smithi083 ceph-mon[77242]: pgmap v11325: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:48.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:45:47 smithi153 ceph-mon[70044]: pgmap v11325: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:49.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:45:49 smithi083 ceph-mon[79930]: pgmap v11326: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:49.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:45:49 smithi083 ceph-mon[77242]: pgmap v11326: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:50.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:45:49 smithi153 ceph-mon[70044]: pgmap v11326: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:51.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:45:51 smithi083 ceph-mon[77242]: pgmap v11327: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:51.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:45:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:45:51] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T11:45:51.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:45:51 smithi083 ceph-mon[79930]: pgmap v11327: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:51.964 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:45:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:45:51.557Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:45:52.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:45:51 smithi153 ceph-mon[70044]: pgmap v11327: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:45:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:45:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:45:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:45:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:45:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:45:53.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:45:53 smithi083 ceph-mon[79930]: pgmap v11328: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:53.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:45:53 smithi083 ceph-mon[77242]: pgmap v11328: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:54.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:45:53 smithi153 ceph-mon[70044]: pgmap v11328: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:55.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:45:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:45:54.928Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:45:55.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:45:55 smithi083 ceph-mon[79930]: pgmap v11329: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:55.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:45:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:45:55.655Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:45:55.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:45:55 smithi083 ceph-mon[77242]: pgmap v11329: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:56.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:45:55 smithi153 ceph-mon[70044]: pgmap v11329: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:57.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:45:57 smithi083 ceph-mon[79930]: pgmap v11330: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:57.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:45:57 smithi083 ceph-mon[77242]: pgmap v11330: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:58.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:45:57 smithi153 ceph-mon[70044]: pgmap v11330: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:59.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:45:59 smithi083 ceph-mon[79930]: pgmap v11331: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:45:59.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:45:59 smithi083 ceph-mon[77242]: pgmap v11331: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:46:00.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:45:59 smithi153 ceph-mon[70044]: pgmap v11331: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:46:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:46:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:46:01] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T11:46:01.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:46:01 smithi083 ceph-mon[79930]: pgmap v11332: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:46:01.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:46:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:46:01.558Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:46:01.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:46:01 smithi083 ceph-mon[77242]: pgmap v11332: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:46:02.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:46:01 smithi153 ceph-mon[70044]: pgmap v11332: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:46:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:46:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:46:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:46:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:46:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:46:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:46:03.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:46:03 smithi083 ceph-mon[79930]: pgmap v11333: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:46:03.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:46:03 smithi083 ceph-mon[77242]: pgmap v11333: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:46:04.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:46:03 smithi153 ceph-mon[70044]: pgmap v11333: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:46:05.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:46:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:46:04.928Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:46:05.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:46:05 smithi083 ceph-mon[79930]: pgmap v11334: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:46:05.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:46:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:46:05.655Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:46:05.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:46:05 smithi083 ceph-mon[77242]: pgmap v11334: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:46:06.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:46:05 smithi153 ceph-mon[70044]: pgmap v11334: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:46:07.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:46:07 smithi083 ceph-mon[79930]: pgmap v11335: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:46:07.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:46:07 smithi083 ceph-mon[77242]: pgmap v11335: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:46:08.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:46:07 smithi153 ceph-mon[70044]: pgmap v11335: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:46:09.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:46:09 smithi083 ceph-mon[77242]: pgmap v11336: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:46:09.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:46:09 smithi083 ceph-mon[79930]: pgmap v11336: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:46:10.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:46:09 smithi153 ceph-mon[70044]: pgmap v11336: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:46:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:46:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:46:11] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T11:46:11.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:46:11 smithi083 ceph-mon[79930]: pgmap v11337: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:46:11.964 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:46:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:46:11.559Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:46:11.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:46:11 smithi083 ceph-mon[77242]: pgmap v11337: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:46:12.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:46:11 smithi153 ceph-mon[70044]: pgmap v11337: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:46:12.626 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:46:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:46:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:46:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:46:12.917 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:46:12.917 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 3m ago 6h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:46:12.917 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 61s ago 6h 64.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:46:12.917 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 3m ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:46:12.918 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 61s ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:46:12.918 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 61s ago 6h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:46:12.918 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 3m ago 6h 831M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:46:12.918 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 3m ago 6h 1415M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:46:12.919 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 61s ago 6h 610M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:46:12.919 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 3m ago 6h 816M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:46:12.919 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 3m ago 6h 26.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:46:12.919 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 61s ago 6h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:46:12.919 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 3m ago 6h 997M 2411M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:46:12.919 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 3m ago 6h 986M 2411M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:46:12.920 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 3m ago 6h 769M 2411M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:46:12.920 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 3m ago 6h 964M 2411M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:46:12.920 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 61s ago 6h 848M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:46:12.920 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 61s ago 6h 1085M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:46:12.920 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 61s ago 6h 947M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:46:12.921 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 61s ago 6h 1148M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:46:12.921 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 61s ago 6h 156M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:46:13.269 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:46:13.269 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:46:13.270 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:46:13.270 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:46:13.270 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:46:13.270 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:46:13.271 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:46:13.271 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:46:13.271 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:46:13.271 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:46:13.271 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:46:13.272 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:46:13.272 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:46:13.272 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:46:13.272 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:46:13.272 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:46:13.273 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:46:13.273 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:46:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:46:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:46:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:46:13.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:46:13 smithi083 ceph-mon[79930]: pgmap v11338: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:46:13.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:46:13 smithi083 ceph-mon[79930]: from='client.57247 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:46:13.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:46:13 smithi083 ceph-mon[79930]: from='client.47322 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:46:13.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:46:13 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3272543381' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:46:13.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:46:13 smithi083 ceph-mon[77242]: pgmap v11338: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:46:13.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:46:13 smithi083 ceph-mon[77242]: from='client.57247 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:46:13.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:46:13 smithi083 ceph-mon[77242]: from='client.47322 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:46:13.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:46:13 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3272543381' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:46:14.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:46:13 smithi153 ceph-mon[70044]: pgmap v11338: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:46:14.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:46:13 smithi153 ceph-mon[70044]: from='client.57247 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:46:14.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:46:13 smithi153 ceph-mon[70044]: from='client.47322 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:46:14.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:46:13 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3272543381' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:46:15.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:46:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:46:14.929Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:46:15.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:46:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:46:15.656Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:46:15.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:46:15 smithi083 ceph-mon[77242]: pgmap v11339: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:46:15.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:46:15 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:46:15.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:46:15 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:46:15.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:46:15 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:46:15.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:46:15 smithi083 ceph-mon[79930]: pgmap v11339: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:46:15.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:46:15 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:46:15.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:46:15 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:46:15.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:46:15 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:46:16.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:46:15 smithi153 ceph-mon[70044]: pgmap v11339: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:46:16.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:46:15 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:46:16.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:46:15 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:46:16.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:46:15 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:46:17.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:46:17 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:46:17.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:46:17 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:46:17.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:46:17 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:46:18.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:46:18 smithi083 ceph-mon[79930]: pgmap v11340: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:46:18.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:46:18 smithi083 ceph-mon[77242]: pgmap v11340: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:46:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:46:18 smithi153 ceph-mon[70044]: pgmap v11340: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:46:20.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:46:20 smithi083 ceph-mon[79930]: pgmap v11341: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:46:20.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:46:20 smithi083 ceph-mon[77242]: pgmap v11341: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:46:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:46:20 smithi153 ceph-mon[70044]: pgmap v11341: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:46:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:46:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:46:21] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T11:46:21.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:46:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:46:21.559Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:46:22.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:46:22 smithi083 ceph-mon[79930]: pgmap v11342: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:46:22.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:46:22 smithi083 ceph-mon[77242]: pgmap v11342: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:46:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:46:22 smithi153 ceph-mon[70044]: pgmap v11342: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:46:22.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:46:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:46:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:46:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:46:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:46:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:46:24.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:46:24 smithi083 ceph-mon[79930]: pgmap v11343: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:46:24.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:46:24 smithi083 ceph-mon[77242]: pgmap v11343: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:46:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:46:24 smithi153 ceph-mon[70044]: pgmap v11343: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:46:25.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:46:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:46:24.931Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:46:25.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:46:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:46:25.657Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:46:26.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:46:26 smithi083 ceph-mon[79930]: pgmap v11344: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:46:26.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:46:26 smithi083 ceph-mon[77242]: pgmap v11344: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:46:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:46:26 smithi153 ceph-mon[70044]: pgmap v11344: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:46:28.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:46:28 smithi083 ceph-mon[79930]: pgmap v11345: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:46:28.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:46:28 smithi083 ceph-mon[77242]: pgmap v11345: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:46:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:46:28 smithi153 ceph-mon[70044]: pgmap v11345: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:46:30.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:46:30 smithi083 ceph-mon[79930]: pgmap v11346: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:46:30.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:46:30 smithi083 ceph-mon[77242]: pgmap v11346: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:46:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:46:30 smithi153 ceph-mon[70044]: pgmap v11346: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:46:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:46:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:46:31] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T11:46:31.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:46:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:46:31.560Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:46:32.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:46:32 smithi083 ceph-mon[79930]: pgmap v11347: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:46:32.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:46:32 smithi083 ceph-mon[77242]: pgmap v11347: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:46:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:46:32 smithi153 ceph-mon[70044]: pgmap v11347: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:46:32.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:46:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:46:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:46:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:46:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:46:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:46:34.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:46:34 smithi083 ceph-mon[79930]: pgmap v11348: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:46:34.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:46:34 smithi083 ceph-mon[77242]: pgmap v11348: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:46:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:46:34 smithi153 ceph-mon[70044]: pgmap v11348: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:46:35.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:46:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:46:34.931Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:46:35.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:46:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:46:35.657Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:46:36.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:46:36 smithi083 ceph-mon[77242]: pgmap v11349: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:46:36.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:46:36 smithi083 ceph-mon[79930]: pgmap v11349: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:46:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:46:36 smithi153 ceph-mon[70044]: pgmap v11349: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:46:38.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:46:38 smithi083 ceph-mon[77242]: pgmap v11350: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:46:38.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:46:38 smithi083 ceph-mon[79930]: pgmap v11350: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:46:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:46:38 smithi153 ceph-mon[70044]: pgmap v11350: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:46:40.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:46:40 smithi083 ceph-mon[77242]: pgmap v11351: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:46:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:46:40 smithi083 ceph-mon[79930]: pgmap v11351: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:46:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:46:40 smithi153 ceph-mon[70044]: pgmap v11351: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:46:41.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:46:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:46:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:46:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:46:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:46:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:46:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:46:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:46:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:46:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:46:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:46:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:46:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:46:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:46:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:46:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:46:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:46:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:46:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:46:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:46:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:46:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:46:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:46:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:46:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:46:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:46:41] "GET /metrics HTTP/1.1" 200 225228 "" "Prometheus/2.33.4" 2022-09-29T11:46:41.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:46:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:46:41.561Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:46:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:46:42 smithi083 ceph-mon[79930]: pgmap v11352: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:46:42.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:46:42 smithi083 ceph-mon[77242]: pgmap v11352: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:46:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:46:42 smithi153 ceph-mon[70044]: pgmap v11352: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:46:42.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:46:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:46:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:46:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:46:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:46:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:46:43.605 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:46:43.901 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:46:43.901 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 3m ago 6h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:46:43.901 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 92s ago 6h 64.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:46:43.902 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 3m ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:46:43.902 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 92s ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:46:43.902 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 92s ago 6h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:46:43.903 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 3m ago 6h 831M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:46:43.903 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 3m ago 6h 1415M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:46:43.903 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 92s ago 6h 610M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:46:43.903 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 3m ago 6h 816M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:46:43.903 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 3m ago 6h 26.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:46:43.904 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 92s ago 6h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:46:43.904 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 3m ago 6h 997M 2411M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:46:43.904 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 3m ago 6h 986M 2411M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:46:43.904 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 3m ago 6h 769M 2411M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:46:43.905 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 3m ago 6h 964M 2411M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:46:43.905 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 92s ago 6h 848M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:46:43.905 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 92s ago 6h 1085M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:46:43.905 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 92s ago 6h 947M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:46:43.905 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 92s ago 6h 1148M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:46:43.906 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 92s ago 6h 156M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:46:44.258 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:46:44.258 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:46:44.259 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:46:44.259 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:46:44.259 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:46:44.259 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:46:44.260 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:46:44.260 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:46:44.260 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:46:44.260 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:46:44.260 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:46:44.261 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:46:44.261 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:46:44.261 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:46:44.261 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:46:44.262 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:46:44.262 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:46:44.262 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:46:44.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:46:44 smithi083 ceph-mon[79930]: pgmap v11353: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:46:44.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:46:44 smithi083 ceph-mon[77242]: pgmap v11353: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:46:44.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:46:44 smithi153 ceph-mon[70044]: pgmap v11353: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:46:45.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:46:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:46:44.931Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:46:45.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:46:45 smithi083 ceph-mon[77242]: from='client.57265 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:46:45.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:46:45 smithi083 ceph-mon[77242]: from='client.47340 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:46:45.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:46:45 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1537556424' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:46:45.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:46:45 smithi083 ceph-mon[79930]: from='client.57265 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:46:45.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:46:45 smithi083 ceph-mon[79930]: from='client.47340 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:46:45.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:46:45 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1537556424' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:46:45.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:46:45 smithi153 ceph-mon[70044]: from='client.57265 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:46:45.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:46:45 smithi153 ceph-mon[70044]: from='client.47340 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:46:45.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:46:45 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1537556424' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:46:45.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:46:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:46:45.658Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:46:46.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:46:46 smithi083 ceph-mon[79930]: pgmap v11354: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:46:46.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:46:46 smithi083 ceph-mon[77242]: pgmap v11354: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:46:46.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:46:46 smithi153 ceph-mon[70044]: pgmap v11354: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:46:48.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:46:48 smithi083 ceph-mon[79930]: pgmap v11355: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:46:48.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:46:48 smithi083 ceph-mon[77242]: pgmap v11355: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:46:48.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:46:48 smithi153 ceph-mon[70044]: pgmap v11355: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:46:50.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:46:50 smithi083 ceph-mon[77242]: pgmap v11356: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:46:50.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:46:50 smithi083 ceph-mon[79930]: pgmap v11356: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:46:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:46:50 smithi153 ceph-mon[70044]: pgmap v11356: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:46:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:46:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:46:51] "GET /metrics HTTP/1.1" 200 225228 "" "Prometheus/2.33.4" 2022-09-29T11:46:51.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:46:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:46:51.561Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:46:52.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:46:52 smithi083 ceph-mon[79930]: pgmap v11357: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:46:52.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:46:52 smithi083 ceph-mon[77242]: pgmap v11357: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:46:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:46:52 smithi153 ceph-mon[70044]: pgmap v11357: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:46:52.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:46:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:46:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:46:53.311 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:46:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:46:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:46:54.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:46:54 smithi083 ceph-mon[79930]: pgmap v11358: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:46:54.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:46:54 smithi083 ceph-mon[77242]: pgmap v11358: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:46:54.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:46:54 smithi153 ceph-mon[70044]: pgmap v11358: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:46:55.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:46:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:46:54.933Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:46:55.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:46:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:46:55.659Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:46:56.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:46:56 smithi083 ceph-mon[79930]: pgmap v11359: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:46:56.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:46:56 smithi083 ceph-mon[77242]: pgmap v11359: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:46:56.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:46:56 smithi153 ceph-mon[70044]: pgmap v11359: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:46:58.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:46:58 smithi083 ceph-mon[79930]: pgmap v11360: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:46:58.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:46:58 smithi083 ceph-mon[77242]: pgmap v11360: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:46:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:46:58 smithi153 ceph-mon[70044]: pgmap v11360: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:00.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:47:00 smithi083 ceph-mon[79930]: pgmap v11361: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:00.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:47:00 smithi083 ceph-mon[77242]: pgmap v11361: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:47:00 smithi153 ceph-mon[70044]: pgmap v11361: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:47:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:47:01] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T11:47:01.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:47:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:47:01.561Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:47:02.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:47:02 smithi083 ceph-mon[79930]: pgmap v11362: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:02.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:47:02 smithi083 ceph-mon[77242]: pgmap v11362: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:47:02 smithi153 ceph-mon[70044]: pgmap v11362: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:02.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:47:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:47:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:47:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:47:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:47:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:47:04.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:47:04 smithi083 ceph-mon[79930]: pgmap v11363: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:04.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:47:04 smithi083 ceph-mon[77242]: pgmap v11363: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:47:04 smithi153 ceph-mon[70044]: pgmap v11363: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:05.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:47:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:47:04.934Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:47:05.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:47:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:47:05.659Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:47:06.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:47:06 smithi083 ceph-mon[77242]: pgmap v11364: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:06.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:47:06 smithi083 ceph-mon[79930]: pgmap v11364: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:06.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:47:06 smithi153 ceph-mon[70044]: pgmap v11364: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:08.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:47:08 smithi083 ceph-mon[79930]: pgmap v11365: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:08.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:47:08 smithi083 ceph-mon[77242]: pgmap v11365: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:47:08 smithi153 ceph-mon[70044]: pgmap v11365: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:10.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:47:10 smithi083 ceph-mon[79930]: pgmap v11366: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:10.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:47:10 smithi083 ceph-mon[77242]: pgmap v11366: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:47:10 smithi153 ceph-mon[70044]: pgmap v11366: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:47:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:47:11] "GET /metrics HTTP/1.1" 200 225228 "" "Prometheus/2.33.4" 2022-09-29T11:47:11.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:47:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:47:11.562Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:47:12.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:47:12 smithi083 ceph-mon[79930]: pgmap v11367: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:12.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:47:12 smithi083 ceph-mon[77242]: pgmap v11367: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:47:12 smithi153 ceph-mon[70044]: pgmap v11367: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:12.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:47:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:47:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:47:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:47:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:47:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:47:14.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:47:14 smithi083 ceph-mon[77242]: pgmap v11368: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:14.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:47:14 smithi083 ceph-mon[79930]: pgmap v11368: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:14.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:47:14 smithi153 ceph-mon[70044]: pgmap v11368: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:14.584 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:47:14.875 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:47:14.875 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 4m ago 6h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:47:14.876 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 2m ago 6h 64.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:47:14.876 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 4m ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:47:14.876 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 2m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:47:14.876 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 2m ago 6h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:47:14.877 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 4m ago 6h 831M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:47:14.877 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 4m ago 6h 1415M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:47:14.877 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 2m ago 6h 610M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:47:14.877 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 4m ago 6h 816M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:47:14.877 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 4m ago 6h 26.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:47:14.878 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 2m ago 6h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:47:14.878 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 4m ago 6h 997M 2411M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:47:14.878 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 4m ago 6h 986M 2411M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:47:14.878 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 4m ago 6h 769M 2411M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:47:14.879 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 4m ago 6h 964M 2411M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:47:14.879 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 2m ago 6h 848M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:47:14.879 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 2m ago 6h 1085M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:47:14.879 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 2m ago 6h 947M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:47:14.879 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 2m ago 6h 1148M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:47:14.880 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 2m ago 6h 156M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:47:15.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:47:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:47:14.934Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:47:15.232 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:47:15.232 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:47:15.232 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:47:15.233 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:47:15.233 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:47:15.233 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:47:15.233 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:47:15.233 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:47:15.234 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:47:15.234 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:47:15.234 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:47:15.234 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:47:15.235 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:47:15.235 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:47:15.235 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:47:15.235 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:47:15.236 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:47:15.236 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:47:15.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:47:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:47:15.659Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:47:16.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:47:16 smithi083 ceph-mon[79930]: from='client.47352 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:47:16.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:47:16 smithi083 ceph-mon[79930]: pgmap v11369: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:16.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:47:16 smithi083 ceph-mon[79930]: from='client.57289 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:47:16.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:47:16 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2146711608' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:47:16.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:47:16 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:47:16.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:47:16 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:47:16.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:47:16 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:47:16.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:47:16 smithi083 ceph-mon[77242]: from='client.47352 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:47:16.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:47:16 smithi083 ceph-mon[77242]: pgmap v11369: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:16.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:47:16 smithi083 ceph-mon[77242]: from='client.57289 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:47:16.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:47:16 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2146711608' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:47:16.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:47:16 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:47:16.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:47:16 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:47:16.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:47:16 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:47:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:47:16 smithi153 ceph-mon[70044]: from='client.47352 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:47:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:47:16 smithi153 ceph-mon[70044]: pgmap v11369: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:47:16 smithi153 ceph-mon[70044]: from='client.57289 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:47:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:47:16 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2146711608' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:47:16.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:47:16 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:47:16.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:47:16 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:47:16.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:47:16 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:47:17.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:47:17 smithi083 ceph-mon[79930]: pgmap v11370: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:17.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:47:17 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:47:17.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:47:17 smithi083 ceph-mon[77242]: pgmap v11370: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:17.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:47:17 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:47:18.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:47:17 smithi153 ceph-mon[70044]: pgmap v11370: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:18.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:47:17 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:47:19.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:47:19 smithi083 ceph-mon[79930]: pgmap v11371: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:19.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:47:19 smithi083 ceph-mon[77242]: pgmap v11371: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:20.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:47:19 smithi153 ceph-mon[70044]: pgmap v11371: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:47:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:47:21] "GET /metrics HTTP/1.1" 200 225228 "" "Prometheus/2.33.4" 2022-09-29T11:47:21.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:47:21 smithi083 ceph-mon[79930]: pgmap v11372: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:21.964 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:47:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:47:21.563Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:47:21.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:47:21 smithi083 ceph-mon[77242]: pgmap v11372: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:22.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:47:21 smithi153 ceph-mon[70044]: pgmap v11372: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:47:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:47:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:47:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:47:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:47:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:47:24.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:47:23 smithi153 ceph-mon[70044]: pgmap v11373: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:24.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:47:23 smithi083 ceph-mon[79930]: pgmap v11373: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:24.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:47:23 smithi083 ceph-mon[77242]: pgmap v11373: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:25.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:47:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:47:24.935Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:47:25.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:47:25 smithi083 ceph-mon[79930]: pgmap v11374: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:25.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:47:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:47:25.661Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:47:25.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:47:25 smithi083 ceph-mon[77242]: pgmap v11374: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:26.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:47:25 smithi153 ceph-mon[70044]: pgmap v11374: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:28.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:47:27 smithi153 ceph-mon[70044]: pgmap v11375: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:28.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:47:27 smithi083 ceph-mon[79930]: pgmap v11375: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:28.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:47:27 smithi083 ceph-mon[77242]: pgmap v11375: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:30.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:47:29 smithi153 ceph-mon[70044]: pgmap v11376: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:30.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:47:29 smithi083 ceph-mon[79930]: pgmap v11376: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:30.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:47:29 smithi083 ceph-mon[77242]: pgmap v11376: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:47:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:47:31] "GET /metrics HTTP/1.1" 200 225229 "" "Prometheus/2.33.4" 2022-09-29T11:47:31.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:47:31 smithi083 ceph-mon[79930]: pgmap v11377: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:31.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:47:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:47:31.563Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:47:31.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:47:31 smithi083 ceph-mon[77242]: pgmap v11377: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:32.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:47:31 smithi153 ceph-mon[70044]: pgmap v11377: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:47:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:47:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:47:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:47:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:47:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:47:34.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:47:33 smithi153 ceph-mon[70044]: pgmap v11378: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:34.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:47:33 smithi083 ceph-mon[79930]: pgmap v11378: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:34.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:47:33 smithi083 ceph-mon[77242]: pgmap v11378: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:35.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:47:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:47:34.936Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:47:35.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:47:35 smithi083 ceph-mon[79930]: pgmap v11379: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:35.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:47:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:47:35.662Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:47:35.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:47:35 smithi083 ceph-mon[77242]: pgmap v11379: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:36.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:47:35 smithi153 ceph-mon[70044]: pgmap v11379: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:38.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:47:37 smithi153 ceph-mon[70044]: pgmap v11380: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:38.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:47:37 smithi083 ceph-mon[79930]: pgmap v11380: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:38.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:47:37 smithi083 ceph-mon[77242]: pgmap v11380: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:47:39 smithi153 ceph-mon[70044]: pgmap v11381: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:47:39 smithi083 ceph-mon[79930]: pgmap v11381: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:47:39 smithi083 ceph-mon[77242]: pgmap v11381: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:47:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:47:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:47:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:47:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:47:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:47:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:47:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:47:41.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:47:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:47:41.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:47:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:47:41.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:47:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:47:41.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:47:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:47:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:47:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:47:41.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:47:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:47:41.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:47:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:47:41.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:47:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:47:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:47:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:47:41] "GET /metrics HTTP/1.1" 200 225228 "" "Prometheus/2.33.4" 2022-09-29T11:47:41.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:47:41 smithi083 ceph-mon[79930]: pgmap v11382: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:41.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:47:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:47:41.564Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:47:41.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:47:41 smithi083 ceph-mon[77242]: pgmap v11382: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:42.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:47:41 smithi153 ceph-mon[70044]: pgmap v11382: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:47:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:47:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:47:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:47:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:47:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:47:44.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:47:43 smithi153 ceph-mon[70044]: pgmap v11383: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:44.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:47:43 smithi083 ceph-mon[79930]: pgmap v11383: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:44.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:47:43 smithi083 ceph-mon[77242]: pgmap v11383: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:45.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:47:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:47:44.936Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:47:45.555 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:47:45.845 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:47:45.846 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 4m ago 6h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:47:45.846 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 2m ago 6h 64.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:47:45.846 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 4m ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:47:45.846 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 2m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:47:45.847 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 2m ago 6h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:47:45.847 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 4m ago 6h 831M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:47:45.847 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 4m ago 6h 1415M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:47:45.847 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 2m ago 6h 610M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:47:45.847 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 4m ago 6h 816M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:47:45.848 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 4m ago 6h 26.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:47:45.848 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 2m ago 6h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:47:45.848 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 4m ago 6h 997M 2411M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:47:45.848 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 4m ago 6h 986M 2411M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:47:45.849 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 4m ago 6h 769M 2411M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:47:45.849 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 4m ago 6h 964M 2411M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:47:45.849 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 2m ago 6h 848M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:47:45.849 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 2m ago 6h 1085M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:47:45.849 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 2m ago 6h 947M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:47:45.850 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 2m ago 6h 1148M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:47:45.850 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 2m ago 6h 156M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:47:45.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:47:45 smithi083 ceph-mon[79930]: pgmap v11384: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:45.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:47:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:47:45.662Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:47:45.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:47:45 smithi083 ceph-mon[77242]: pgmap v11384: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:46.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:47:45 smithi153 ceph-mon[70044]: pgmap v11384: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:46.200 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:47:46.201 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:47:46.201 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:47:46.201 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:47:46.201 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:47:46.202 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:47:46.202 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:47:46.202 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:47:46.202 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:47:46.203 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:47:46.203 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:47:46.203 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:47:46.203 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:47:46.203 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:47:46.204 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:47:46.204 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:47:46.204 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:47:46.204 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:47:47.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:47:46 smithi153 ceph-mon[70044]: from='client.47370 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:47:47.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:47:46 smithi153 ceph-mon[70044]: from='client.57307 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:47:47.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:47:46 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1049783072' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:47:47.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:47:46 smithi083 ceph-mon[79930]: from='client.47370 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:47:47.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:47:46 smithi083 ceph-mon[79930]: from='client.57307 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:47:47.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:47:46 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1049783072' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:47:47.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:47:46 smithi083 ceph-mon[77242]: from='client.47370 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:47:47.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:47:46 smithi083 ceph-mon[77242]: from='client.57307 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:47:47.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:47:46 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1049783072' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:47:48.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:47:47 smithi153 ceph-mon[70044]: pgmap v11385: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:48.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:47:47 smithi083 ceph-mon[79930]: pgmap v11385: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:48.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:47:47 smithi083 ceph-mon[77242]: pgmap v11385: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:50.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:47:49 smithi153 ceph-mon[70044]: pgmap v11386: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:50.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:47:49 smithi083 ceph-mon[79930]: pgmap v11386: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:50.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:47:49 smithi083 ceph-mon[77242]: pgmap v11386: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:51.764 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:47:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:47:51] "GET /metrics HTTP/1.1" 200 225228 "" "Prometheus/2.33.4" 2022-09-29T11:47:51.764 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:47:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:47:51.565Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:47:52.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:47:51 smithi153 ceph-mon[70044]: pgmap v11387: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:52.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:47:51 smithi083 ceph-mon[79930]: pgmap v11387: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:52.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:47:51 smithi083 ceph-mon[77242]: pgmap v11387: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:47:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:47:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:47:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:47:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:47:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:47:54.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:47:53 smithi153 ceph-mon[70044]: pgmap v11388: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:54.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:47:53 smithi083 ceph-mon[79930]: pgmap v11388: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:54.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:47:53 smithi083 ceph-mon[77242]: pgmap v11388: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:55.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:47:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:47:54.937Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:47:55.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:47:55 smithi083 ceph-mon[79930]: pgmap v11389: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:55.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:47:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:47:55.663Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:47:55.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:47:55 smithi083 ceph-mon[77242]: pgmap v11389: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:56.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:47:55 smithi153 ceph-mon[70044]: pgmap v11389: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:58.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:47:57 smithi153 ceph-mon[70044]: pgmap v11390: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:58.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:47:57 smithi083 ceph-mon[77242]: pgmap v11390: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:47:58.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:47:57 smithi083 ceph-mon[79930]: pgmap v11390: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:00.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:47:59 smithi153 ceph-mon[70044]: pgmap v11391: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:00.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:47:59 smithi083 ceph-mon[77242]: pgmap v11391: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:00.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:47:59 smithi083 ceph-mon[79930]: pgmap v11391: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:01.784 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:48:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:48:01] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T11:48:01.784 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:48:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:48:01.565Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:48:02.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:48:01 smithi153 ceph-mon[70044]: pgmap v11392: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:02.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:48:01 smithi083 ceph-mon[79930]: pgmap v11392: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:02.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:48:01 smithi083 ceph-mon[77242]: pgmap v11392: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:48:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:48:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:48:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:48:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:48:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:48:04.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:48:03 smithi153 ceph-mon[70044]: pgmap v11393: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:04.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:48:03 smithi083 ceph-mon[79930]: pgmap v11393: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:04.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:48:03 smithi083 ceph-mon[77242]: pgmap v11393: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:05.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:48:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:48:04.937Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:48:05.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:48:05 smithi083 ceph-mon[79930]: pgmap v11394: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:05.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:48:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:48:05.664Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:48:05.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:48:05 smithi083 ceph-mon[77242]: pgmap v11394: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:06.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:48:05 smithi153 ceph-mon[70044]: pgmap v11394: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:08.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:48:07 smithi153 ceph-mon[70044]: pgmap v11395: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:08.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:48:07 smithi083 ceph-mon[79930]: pgmap v11395: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:08.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:48:07 smithi083 ceph-mon[77242]: pgmap v11395: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:10.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:48:09 smithi153 ceph-mon[70044]: pgmap v11396: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:10.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:48:09 smithi083 ceph-mon[79930]: pgmap v11396: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:10.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:48:09 smithi083 ceph-mon[77242]: pgmap v11396: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:11.802 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:48:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:48:11] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T11:48:11.803 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:48:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:48:11.566Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:48:12.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:48:11 smithi153 ceph-mon[70044]: pgmap v11397: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:12.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:48:11 smithi083 ceph-mon[79930]: pgmap v11397: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:12.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:48:11 smithi083 ceph-mon[77242]: pgmap v11397: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:48:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:48:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:48:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:48:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:48:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:48:14.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:48:13 smithi153 ceph-mon[70044]: pgmap v11398: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:14.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:48:13 smithi083 ceph-mon[77242]: pgmap v11398: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:14.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:48:13 smithi083 ceph-mon[79930]: pgmap v11398: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:15.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:48:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:48:14.938Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:48:15.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:48:15 smithi083 ceph-mon[79930]: pgmap v11399: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:15.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:48:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:48:15.664Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:48:15.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:48:15 smithi083 ceph-mon[77242]: pgmap v11399: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:16.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:48:15 smithi153 ceph-mon[70044]: pgmap v11399: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:16.519 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:48:16.814 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:48:16.814 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 5m ago 6h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:48:16.814 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 3m ago 6h 64.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:48:16.815 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 5m ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:48:16.815 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 3m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:48:16.815 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 3m ago 6h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:48:16.815 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 5m ago 6h 831M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:48:16.816 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 5m ago 6h 1415M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:48:16.816 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 3m ago 6h 610M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:48:16.816 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 5m ago 6h 816M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:48:16.816 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 5m ago 6h 26.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:48:16.816 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 3m ago 6h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:48:16.817 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 5m ago 6h 997M 2411M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:48:16.817 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 5m ago 6h 986M 2411M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:48:16.817 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 5m ago 6h 769M 2411M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:48:16.817 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 5m ago 6h 964M 2411M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:48:16.818 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 3m ago 6h 848M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:48:16.818 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 3m ago 6h 1085M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:48:16.818 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 3m ago 6h 947M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:48:16.818 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 3m ago 6h 1148M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:48:16.819 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 3m ago 6h 156M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:48:17.187 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:48:17.187 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:48:17.187 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:48:17.188 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:48:17.188 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:48:17.188 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:48:17.188 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:48:17.188 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:48:17.189 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:48:17.189 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:48:17.189 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:48:17.189 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:48:17.190 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:48:17.190 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:48:17.190 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:48:17.190 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:48:17.191 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:48:17.191 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:48:17.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:48:16 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:48:17.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:48:16 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:48:17.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:48:16 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:48:17.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:48:16 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:48:17.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:48:16 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:48:17.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:48:16 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:48:17.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:48:16 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:48:17.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:48:16 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:48:17.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:48:16 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:48:18.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:48:17 smithi083 ceph-mon[79930]: from='client.47388 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:48:18.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:48:17 smithi083 ceph-mon[79930]: pgmap v11400: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:18.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:48:17 smithi083 ceph-mon[79930]: from='client.57325 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:48:18.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:48:17 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1597905481' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:48:18.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:48:17 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:48:18.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:48:17 smithi083 ceph-mon[77242]: from='client.47388 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:48:18.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:48:17 smithi083 ceph-mon[77242]: pgmap v11400: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:18.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:48:17 smithi083 ceph-mon[77242]: from='client.57325 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:48:18.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:48:17 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1597905481' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:48:18.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:48:17 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:48:18.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:48:17 smithi153 ceph-mon[70044]: from='client.47388 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:48:18.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:48:17 smithi153 ceph-mon[70044]: pgmap v11400: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:18.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:48:17 smithi153 ceph-mon[70044]: from='client.57325 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:48:18.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:48:17 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1597905481' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:48:18.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:48:17 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:48:20.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:48:19 smithi083 ceph-mon[79930]: pgmap v11401: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:20.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:48:19 smithi083 ceph-mon[77242]: pgmap v11401: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:48:19 smithi153 ceph-mon[70044]: pgmap v11401: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:21.822 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:48:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:48:21] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T11:48:21.823 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:48:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:48:21.567Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:48:22.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:48:21 smithi083 ceph-mon[77242]: pgmap v11402: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:22.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:48:21 smithi083 ceph-mon[79930]: pgmap v11402: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:22.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:48:21 smithi153 ceph-mon[70044]: pgmap v11402: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:48:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:48:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:48:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:48:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:48:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:48:24.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:48:23 smithi083 ceph-mon[79930]: pgmap v11403: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:24.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:48:23 smithi083 ceph-mon[77242]: pgmap v11403: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:24.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:48:23 smithi153 ceph-mon[70044]: pgmap v11403: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:25.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:48:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:48:24.938Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:48:25.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:48:25 smithi083 ceph-mon[79930]: pgmap v11404: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:25.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:48:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:48:25.665Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:48:25.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:48:25 smithi083 ceph-mon[77242]: pgmap v11404: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:26.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:48:25 smithi153 ceph-mon[70044]: pgmap v11404: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:28.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:48:27 smithi083 ceph-mon[79930]: pgmap v11405: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:28.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:48:27 smithi083 ceph-mon[77242]: pgmap v11405: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:28.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:48:27 smithi153 ceph-mon[70044]: pgmap v11405: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:30.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:48:29 smithi083 ceph-mon[79930]: pgmap v11406: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:30.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:48:29 smithi083 ceph-mon[77242]: pgmap v11406: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:30.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:48:29 smithi153 ceph-mon[70044]: pgmap v11406: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:31.850 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:48:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:48:31] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T11:48:31.850 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:48:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:48:31.567Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:48:32.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:48:31 smithi083 ceph-mon[79930]: pgmap v11407: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:32.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:48:31 smithi083 ceph-mon[77242]: pgmap v11407: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:32.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:48:31 smithi153 ceph-mon[70044]: pgmap v11407: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:48:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:48:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:48:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:48:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:48:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:48:34.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:48:33 smithi083 ceph-mon[79930]: pgmap v11408: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:34.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:48:33 smithi083 ceph-mon[77242]: pgmap v11408: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:34.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:48:33 smithi153 ceph-mon[70044]: pgmap v11408: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:35.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:48:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:48:34.939Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:48:35.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:48:35 smithi083 ceph-mon[79930]: pgmap v11409: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:35.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:48:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:48:35.666Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:48:35.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:48:35 smithi083 ceph-mon[77242]: pgmap v11409: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:36.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:48:35 smithi153 ceph-mon[70044]: pgmap v11409: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:38.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:48:37 smithi083 ceph-mon[79930]: pgmap v11410: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:38.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:48:37 smithi083 ceph-mon[77242]: pgmap v11410: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:38.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:48:37 smithi153 ceph-mon[70044]: pgmap v11410: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:48:39 smithi083 ceph-mon[79930]: pgmap v11411: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:48:39 smithi083 ceph-mon[77242]: pgmap v11411: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:48:39 smithi153 ceph-mon[70044]: pgmap v11411: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:41.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:48:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:48:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:48:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:48:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:48:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:48:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:48:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:48:41.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:48:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:48:41.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:48:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:48:41.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:48:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:48:41.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:48:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:48:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:48:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:48:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:48:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:48:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:48:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:48:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:48:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:48:41.874 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:48:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:48:41] "GET /metrics HTTP/1.1" 200 225229 "" "Prometheus/2.33.4" 2022-09-29T11:48:41.875 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:48:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:48:41.568Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:48:42.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:48:41 smithi083 ceph-mon[79930]: pgmap v11412: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:42.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:48:41 smithi083 ceph-mon[77242]: pgmap v11412: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:48:41 smithi153 ceph-mon[70044]: pgmap v11412: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:48:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:48:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:48:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:48:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:48:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:48:44.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:48:43 smithi083 ceph-mon[79930]: pgmap v11413: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:44.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:48:43 smithi083 ceph-mon[77242]: pgmap v11413: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:44.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:48:43 smithi153 ceph-mon[70044]: pgmap v11413: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:45.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:48:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:48:44.939Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:48:45.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:48:45 smithi083 ceph-mon[79930]: pgmap v11414: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:45.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:48:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:48:45.666Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:48:45.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:48:45 smithi083 ceph-mon[77242]: pgmap v11414: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:46.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:48:45 smithi153 ceph-mon[70044]: pgmap v11414: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:47.513 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:48:47.803 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:48:47.803 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 5m ago 6h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:48:47.803 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 3m ago 6h 64.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:48:47.804 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 5m ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:48:47.804 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 3m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:48:47.804 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 3m ago 6h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:48:47.804 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 5m ago 6h 831M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:48:47.804 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 5m ago 6h 1415M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:48:47.805 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 3m ago 6h 610M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:48:47.805 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 5m ago 6h 816M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:48:47.805 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 5m ago 6h 26.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:48:47.805 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 3m ago 6h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:48:47.805 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 5m ago 6h 997M 2411M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:48:47.806 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 5m ago 6h 986M 2411M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:48:47.806 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 5m ago 6h 769M 2411M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:48:47.806 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 5m ago 6h 964M 2411M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:48:47.806 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 3m ago 6h 848M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:48:47.807 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 3m ago 6h 1085M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:48:47.807 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 3m ago 6h 947M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:48:47.807 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 3m ago 6h 1148M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:48:47.807 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 3m ago 6h 156M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:48:48.159 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:48:48.159 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:48:48.159 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:48:48.160 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:48:48.160 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:48:48.160 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:48:48.160 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:48:48.160 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:48:48.161 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:48:48.161 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:48:48.161 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:48:48.161 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:48:48.162 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:48:48.162 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:48:48.162 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:48:48.162 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:48:48.163 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:48:48.163 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:48:48.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:48:47 smithi083 ceph-mon[77242]: pgmap v11415: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:48.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:48:47 smithi083 ceph-mon[79930]: pgmap v11415: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:48.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:48:47 smithi153 ceph-mon[70044]: pgmap v11415: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:49.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:48:48 smithi083 ceph-mon[79930]: from='client.47406 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:48:49.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:48:48 smithi083 ceph-mon[79930]: from='client.47412 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:48:49.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:48:48 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2425075845' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:48:49.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:48:48 smithi083 ceph-mon[77242]: from='client.47406 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:48:49.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:48:48 smithi083 ceph-mon[77242]: from='client.47412 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:48:49.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:48:48 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2425075845' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:48:49.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:48:48 smithi153 ceph-mon[70044]: from='client.47406 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:48:49.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:48:48 smithi153 ceph-mon[70044]: from='client.47412 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:48:49.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:48:48 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2425075845' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:48:50.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:48:49 smithi083 ceph-mon[79930]: pgmap v11416: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:50.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:48:49 smithi083 ceph-mon[77242]: pgmap v11416: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:50.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:48:49 smithi153 ceph-mon[70044]: pgmap v11416: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:51.890 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:48:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:48:51] "GET /metrics HTTP/1.1" 200 225229 "" "Prometheus/2.33.4" 2022-09-29T11:48:51.891 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:48:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:48:51.569Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:48:52.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:48:51 smithi083 ceph-mon[79930]: pgmap v11417: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:52.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:48:51 smithi083 ceph-mon[77242]: pgmap v11417: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:52.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:48:51 smithi153 ceph-mon[70044]: pgmap v11417: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:48:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:48:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:48:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:48:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:48:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:48:54.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:48:53 smithi083 ceph-mon[79930]: pgmap v11418: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:54.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:48:53 smithi083 ceph-mon[77242]: pgmap v11418: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:54.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:48:53 smithi153 ceph-mon[70044]: pgmap v11418: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:55.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:48:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:48:54.940Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:48:55.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:48:55 smithi083 ceph-mon[79930]: pgmap v11419: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:55.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:48:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:48:55.666Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:48:55.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:48:55 smithi083 ceph-mon[77242]: pgmap v11419: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:56.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:48:55 smithi153 ceph-mon[70044]: pgmap v11419: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:58.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:48:57 smithi083 ceph-mon[79930]: pgmap v11420: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:58.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:48:57 smithi083 ceph-mon[77242]: pgmap v11420: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:48:58.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:48:57 smithi153 ceph-mon[70044]: pgmap v11420: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:00.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:48:59 smithi083 ceph-mon[79930]: pgmap v11421: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:00.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:48:59 smithi083 ceph-mon[77242]: pgmap v11421: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:48:59 smithi153 ceph-mon[70044]: pgmap v11421: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:01.903 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:49:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:49:01] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T11:49:01.904 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:49:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:49:01.569Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:49:02.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:49:01 smithi083 ceph-mon[77242]: pgmap v11422: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:02.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:49:01 smithi083 ceph-mon[79930]: pgmap v11422: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:02.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:49:01 smithi153 ceph-mon[70044]: pgmap v11422: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:49:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:49:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:49:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:49:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:49:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:49:04.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:49:03 smithi083 ceph-mon[77242]: pgmap v11423: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:04.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:49:03 smithi083 ceph-mon[79930]: pgmap v11423: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:04.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:49:03 smithi153 ceph-mon[70044]: pgmap v11423: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:05.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:49:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:49:04.941Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:49:05.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:49:05 smithi083 ceph-mon[77242]: pgmap v11424: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:05.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:49:05 smithi083 ceph-mon[79930]: pgmap v11424: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:05.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:49:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:49:05.667Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:49:06.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:49:05 smithi153 ceph-mon[70044]: pgmap v11424: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:08.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:49:07 smithi083 ceph-mon[77242]: pgmap v11425: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:08.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:49:07 smithi083 ceph-mon[79930]: pgmap v11425: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:08.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:49:07 smithi153 ceph-mon[70044]: pgmap v11425: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:10.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:49:09 smithi083 ceph-mon[77242]: pgmap v11426: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:10.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:49:09 smithi083 ceph-mon[79930]: pgmap v11426: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:10.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:49:09 smithi153 ceph-mon[70044]: pgmap v11426: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:11.917 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:49:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:49:11] "GET /metrics HTTP/1.1" 200 225229 "" "Prometheus/2.33.4" 2022-09-29T11:49:11.918 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:49:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:49:11.569Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:49:12.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:49:11 smithi083 ceph-mon[79930]: pgmap v11427: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:12.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:49:11 smithi083 ceph-mon[77242]: pgmap v11427: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:12.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:49:11 smithi153 ceph-mon[70044]: pgmap v11427: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:49:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:49:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:49:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:49:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:49:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:49:14.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:49:13 smithi083 ceph-mon[79930]: pgmap v11428: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:14.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:49:13 smithi083 ceph-mon[77242]: pgmap v11428: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:14.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:49:13 smithi153 ceph-mon[70044]: pgmap v11428: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:15.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:49:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:49:14.942Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:49:15.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:49:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:49:15.668Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:49:16.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:49:15 smithi083 ceph-mon[79930]: pgmap v11429: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:16.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:49:15 smithi083 ceph-mon[77242]: pgmap v11429: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:16.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:49:15 smithi153 ceph-mon[70044]: pgmap v11429: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:18.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:49:17 smithi083 ceph-mon[79930]: pgmap v11430: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:18.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:49:17 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:49:18.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:49:17 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:49:18.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:49:17 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:49:18.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:49:17 smithi083 ceph-mon[77242]: pgmap v11430: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:18.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:49:17 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:49:18.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:49:17 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:49:18.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:49:17 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:49:18.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:49:17 smithi153 ceph-mon[70044]: pgmap v11430: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:18.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:49:17 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:49:18.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:49:17 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:49:18.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:49:17 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:49:18.474 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:49:18.767 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:49:18.767 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 6m ago 6h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:49:18.768 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 4m ago 6h 64.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:49:18.768 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 6m ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:49:18.768 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 4m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:49:18.768 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 4m ago 6h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:49:18.769 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 6m ago 6h 831M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:49:18.769 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 6m ago 6h 1415M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:49:18.769 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 4m ago 6h 610M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:49:18.769 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 6m ago 6h 816M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:49:18.769 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 6m ago 6h 26.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:49:18.770 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 4m ago 6h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:49:18.770 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 6m ago 6h 997M 2411M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:49:18.770 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 6m ago 6h 986M 2411M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:49:18.770 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 6m ago 6h 769M 2411M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:49:18.771 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 6m ago 6h 964M 2411M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:49:18.771 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 4m ago 6h 848M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:49:18.771 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 4m ago 6h 1085M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:49:18.771 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 4m ago 6h 947M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:49:18.772 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 4m ago 6h 1148M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:49:18.772 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 4m ago 6h 156M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:49:19.128 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:49:19.128 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:49:19.128 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:49:19.129 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:49:19.129 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:49:19.129 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:49:19.129 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:49:19.129 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:49:19.130 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:49:19.130 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:49:19.130 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:49:19.130 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:49:19.131 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:49:19.131 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:49:19.131 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:49:19.131 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:49:19.131 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:49:19.132 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:49:19.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:49:19 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:49:19.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:49:19 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:49:19.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:49:19 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:49:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:49:20 smithi153 ceph-mon[70044]: from='client.57355 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:49:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:49:20 smithi153 ceph-mon[70044]: pgmap v11431: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:20.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:49:20 smithi153 ceph-mon[70044]: from='client.47430 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:49:20.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:49:20 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1111574425' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:49:20.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:49:20 smithi083 ceph-mon[79930]: from='client.57355 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:49:20.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:49:20 smithi083 ceph-mon[79930]: pgmap v11431: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:20.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:49:20 smithi083 ceph-mon[79930]: from='client.47430 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:49:20.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:49:20 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1111574425' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:49:20.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:49:20 smithi083 ceph-mon[77242]: from='client.57355 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:49:20.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:49:20 smithi083 ceph-mon[77242]: pgmap v11431: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:20.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:49:20 smithi083 ceph-mon[77242]: from='client.47430 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:49:20.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:49:20 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1111574425' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:49:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:49:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:49:21] "GET /metrics HTTP/1.1" 200 225229 "" "Prometheus/2.33.4" 2022-09-29T11:49:21.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:49:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:49:21.570Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:49:22.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:49:22 smithi153 ceph-mon[70044]: pgmap v11432: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:22.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:49:22 smithi083 ceph-mon[79930]: pgmap v11432: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:22.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:49:22 smithi083 ceph-mon[77242]: pgmap v11432: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:49:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:49:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:49:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:49:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:49:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:49:24.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:49:24 smithi153 ceph-mon[70044]: pgmap v11433: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:24.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:49:24 smithi083 ceph-mon[79930]: pgmap v11433: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:24.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:49:24 smithi083 ceph-mon[77242]: pgmap v11433: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:25.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:49:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:49:24.942Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:49:25.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:49:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:49:25.668Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:49:26.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:49:26 smithi153 ceph-mon[70044]: pgmap v11434: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:26.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:49:26 smithi083 ceph-mon[79930]: pgmap v11434: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:26.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:49:26 smithi083 ceph-mon[77242]: pgmap v11434: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:28.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:49:28 smithi153 ceph-mon[70044]: pgmap v11435: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:28.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:49:28 smithi083 ceph-mon[79930]: pgmap v11435: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:28.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:49:28 smithi083 ceph-mon[77242]: pgmap v11435: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:30.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:49:30 smithi153 ceph-mon[70044]: pgmap v11436: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:30.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:49:30 smithi083 ceph-mon[79930]: pgmap v11436: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:30.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:49:30 smithi083 ceph-mon[77242]: pgmap v11436: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:49:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:49:31] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T11:49:31.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:49:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:49:31.570Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:49:32.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:49:32 smithi153 ceph-mon[70044]: pgmap v11437: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:32.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:49:32 smithi083 ceph-mon[79930]: pgmap v11437: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:32.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:49:32 smithi083 ceph-mon[77242]: pgmap v11437: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:49:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:49:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:49:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:49:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:49:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:49:34.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:49:34 smithi153 ceph-mon[70044]: pgmap v11438: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:34.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:49:34 smithi083 ceph-mon[79930]: pgmap v11438: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:34.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:49:34 smithi083 ceph-mon[77242]: pgmap v11438: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:35.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:49:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:49:34.943Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:49:35.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:49:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:49:35.669Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:49:36.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:49:36 smithi153 ceph-mon[70044]: pgmap v11439: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:36.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:49:36 smithi083 ceph-mon[79930]: pgmap v11439: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:36.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:49:36 smithi083 ceph-mon[77242]: pgmap v11439: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:38.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:49:38 smithi083 ceph-mon[79930]: pgmap v11440: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:38.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:49:38 smithi083 ceph-mon[77242]: pgmap v11440: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:49:38 smithi153 ceph-mon[70044]: pgmap v11440: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:49:40 smithi083 ceph-mon[79930]: pgmap v11441: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:49:40 smithi083 ceph-mon[77242]: pgmap v11441: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:49:40 smithi153 ceph-mon[70044]: pgmap v11441: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:41.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:49:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:49:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:49:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:49:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:49:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:49:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:49:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:49:41.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:49:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:49:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:49:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:49:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:49:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:49:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:49:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:49:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:49:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:49:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:49:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:49:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:49:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:49:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:49:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:49:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:49:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:49:41] "GET /metrics HTTP/1.1" 200 225229 "" "Prometheus/2.33.4" 2022-09-29T11:49:41.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:49:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:49:41.571Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:49:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:49:42 smithi083 ceph-mon[79930]: pgmap v11442: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:42.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:49:42 smithi083 ceph-mon[77242]: pgmap v11442: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:49:42 smithi153 ceph-mon[70044]: pgmap v11442: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:42.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:49:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:49:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:49:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:49:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:49:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:49:44.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:49:44 smithi083 ceph-mon[79930]: pgmap v11443: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:44.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:49:44 smithi083 ceph-mon[77242]: pgmap v11443: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:44.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:49:44 smithi153 ceph-mon[70044]: pgmap v11443: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:45.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:49:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:49:44.944Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:49:45.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:49:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:49:45.669Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:49:46.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:49:46 smithi083 ceph-mon[79930]: pgmap v11444: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:46.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:49:46 smithi083 ceph-mon[77242]: pgmap v11444: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:46.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:49:46 smithi153 ceph-mon[70044]: pgmap v11444: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:48.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:49:48 smithi083 ceph-mon[77242]: pgmap v11445: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:48.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:49:48 smithi083 ceph-mon[79930]: pgmap v11445: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:48.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:49:48 smithi153 ceph-mon[70044]: pgmap v11445: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:49.456 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:49:49.748 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:49:49.748 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 6m ago 6h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:49:49.748 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 4m ago 6h 64.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:49:49.749 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 6m ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:49:49.749 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 4m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:49:49.749 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 4m ago 6h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:49:49.749 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 6m ago 6h 831M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:49:49.750 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 6m ago 6h 1415M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:49:49.750 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 4m ago 6h 610M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:49:49.750 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 6m ago 6h 816M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:49:49.750 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 6m ago 6h 26.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:49:49.751 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 4m ago 6h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:49:49.751 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 6m ago 6h 997M 2411M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:49:49.751 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 6m ago 6h 986M 2411M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:49:49.751 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 6m ago 6h 769M 2411M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:49:49.752 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 6m ago 6h 964M 2411M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:49:49.752 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 4m ago 6h 848M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:49:49.752 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 4m ago 6h 1085M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:49:49.752 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 4m ago 6h 947M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:49:49.753 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 4m ago 6h 1148M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:49:49.753 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 4m ago 6h 156M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:49:50.104 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:49:50.104 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:49:50.104 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:49:50.105 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:49:50.105 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:49:50.105 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:49:50.105 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:49:50.105 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:49:50.106 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:49:50.106 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:49:50.106 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:49:50.106 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:49:50.107 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:49:50.107 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:49:50.107 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:49:50.107 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:49:50.108 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:49:50.108 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:49:50.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:49:50 smithi083 ceph-mon[77242]: pgmap v11446: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:50.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:49:50 smithi083 ceph-mon[77242]: from='client.47442 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:49:50.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:49:50 smithi083 ceph-mon[79930]: pgmap v11446: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:50.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:49:50 smithi083 ceph-mon[79930]: from='client.47442 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:49:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:49:50 smithi153 ceph-mon[70044]: pgmap v11446: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:49:50 smithi153 ceph-mon[70044]: from='client.47442 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:49:51.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:49:51 smithi083 ceph-mon[79930]: from='client.47448 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:49:51.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:49:51 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2561776743' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:49:51.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:49:51 smithi083 ceph-mon[77242]: from='client.47448 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:49:51.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:49:51 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2561776743' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:49:51.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:49:51 smithi153 ceph-mon[70044]: from='client.47448 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:49:51.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:49:51 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2561776743' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:49:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:49:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:49:51] "GET /metrics HTTP/1.1" 200 225229 "" "Prometheus/2.33.4" 2022-09-29T11:49:51.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:49:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:49:51.571Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:49:52.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:49:52 smithi083 ceph-mon[77242]: pgmap v11447: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:52.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:49:52 smithi083 ceph-mon[79930]: pgmap v11447: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:49:52 smithi153 ceph-mon[70044]: pgmap v11447: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:52.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:49:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:49:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:49:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:49:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:49:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:49:54.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:49:54 smithi083 ceph-mon[77242]: pgmap v11448: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:54.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:49:54 smithi083 ceph-mon[79930]: pgmap v11448: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:54.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:49:54 smithi153 ceph-mon[70044]: pgmap v11448: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:55.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:49:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:49:54.945Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:49:55.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:49:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:49:55.670Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:49:56.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:49:56 smithi083 ceph-mon[79930]: pgmap v11449: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:56.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:49:56 smithi083 ceph-mon[77242]: pgmap v11449: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:56.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:49:56 smithi153 ceph-mon[70044]: pgmap v11449: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:58.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:49:58 smithi083 ceph-mon[79930]: pgmap v11450: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:58.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:49:58 smithi083 ceph-mon[77242]: pgmap v11450: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:49:58.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:49:58 smithi153 ceph-mon[70044]: pgmap v11450: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:00.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:50:00 smithi083 ceph-mon[79930]: pgmap v11451: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:00.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:50:00 smithi083 ceph-mon[79930]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T11:50:00.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:50:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[77219]: 2022-09-29T11:49:59.999+0000 7f160f665700 -1 log_channel(cluster) log [ERR] : overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T11:50:00.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:50:00 smithi083 ceph-mon[77242]: pgmap v11451: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:00.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:50:00 smithi083 ceph-mon[77242]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T11:50:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:50:00 smithi153 ceph-mon[70044]: pgmap v11451: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:50:00 smithi153 ceph-mon[70044]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T11:50:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:50:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:50:01] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T11:50:01.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:50:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:50:01.572Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:50:02.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:50:02 smithi083 ceph-mon[79930]: pgmap v11452: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:02.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:50:02 smithi083 ceph-mon[77242]: pgmap v11452: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:50:02 smithi153 ceph-mon[70044]: pgmap v11452: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:02.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:50:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:50:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:50:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:50:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:50:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:50:04.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:50:04 smithi083 ceph-mon[79930]: pgmap v11453: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:04.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:50:04 smithi083 ceph-mon[77242]: pgmap v11453: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:50:04 smithi153 ceph-mon[70044]: pgmap v11453: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:05.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:50:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:50:04.945Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:50:05.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:50:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:50:05.670Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:50:06.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:50:06 smithi083 ceph-mon[79930]: pgmap v11454: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:06.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:50:06 smithi083 ceph-mon[77242]: pgmap v11454: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:06.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:50:06 smithi153 ceph-mon[70044]: pgmap v11454: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:08.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:50:08 smithi083 ceph-mon[79930]: pgmap v11455: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:08.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:50:08 smithi083 ceph-mon[77242]: pgmap v11455: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:50:08 smithi153 ceph-mon[70044]: pgmap v11455: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:10.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:50:10 smithi083 ceph-mon[79930]: pgmap v11456: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:10.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:50:10 smithi083 ceph-mon[77242]: pgmap v11456: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:50:10 smithi153 ceph-mon[70044]: pgmap v11456: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:50:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:50:11] "GET /metrics HTTP/1.1" 200 225229 "" "Prometheus/2.33.4" 2022-09-29T11:50:11.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:50:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:50:11.573Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:50:12.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:50:12 smithi083 ceph-mon[79930]: pgmap v11457: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:12.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:50:12 smithi083 ceph-mon[77242]: pgmap v11457: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:50:12 smithi153 ceph-mon[70044]: pgmap v11457: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:12.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:50:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:50:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:50:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:50:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:50:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:50:14.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:50:14 smithi083 ceph-mon[79930]: pgmap v11458: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:14.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:50:14 smithi083 ceph-mon[77242]: pgmap v11458: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:14.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:50:14 smithi153 ceph-mon[70044]: pgmap v11458: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:15.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:50:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:50:14.945Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:50:15.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:50:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:50:15.671Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:50:16.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:50:16 smithi083 ceph-mon[79930]: pgmap v11459: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:16.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:50:16 smithi083 ceph-mon[77242]: pgmap v11459: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:50:16 smithi153 ceph-mon[70044]: pgmap v11459: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:18.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:50:18 smithi083 ceph-mon[79930]: pgmap v11460: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:18.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:50:18 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:50:18.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:50:18 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:50:18.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:50:18 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:50:18.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:50:18 smithi083 ceph-mon[77242]: pgmap v11460: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:18.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:50:18 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:50:18.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:50:18 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:50:18.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:50:18 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:50:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:50:18 smithi153 ceph-mon[70044]: pgmap v11460: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:50:18 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:50:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:50:18 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:50:18.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:50:18 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:50:19.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:50:19 smithi083 ceph-mon[79930]: pgmap v11461: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:19.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:50:19 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:50:19.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:50:19 smithi083 ceph-mon[77242]: pgmap v11461: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:19.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:50:19 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:50:20.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:50:19 smithi153 ceph-mon[70044]: pgmap v11461: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:20.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:50:19 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:50:20.431 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:50:20.724 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:50:20.725 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 7m ago 6h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:50:20.725 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 5m ago 6h 64.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:50:20.725 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 7m ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:50:20.725 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 5m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:50:20.726 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 5m ago 6h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:50:20.726 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 7m ago 6h 831M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:50:20.726 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 7m ago 6h 1415M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:50:20.726 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 5m ago 6h 610M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:50:20.726 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 7m ago 6h 816M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:50:20.727 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 7m ago 6h 26.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:50:20.727 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 5m ago 6h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:50:20.727 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 7m ago 6h 997M 2411M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:50:20.727 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 7m ago 6h 986M 2411M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:50:20.728 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 7m ago 6h 769M 2411M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:50:20.728 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 7m ago 6h 964M 2411M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:50:20.728 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 5m ago 6h 848M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:50:20.728 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 5m ago 6h 1085M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:50:20.729 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 5m ago 6h 947M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:50:20.729 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 5m ago 6h 1148M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:50:20.729 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 5m ago 6h 156M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:50:20.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:50:20 smithi083 ceph-mon[77242]: from='client.57391 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:50:20.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:50:20 smithi083 ceph-mon[79930]: from='client.57391 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:50:21.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:50:20 smithi153 ceph-mon[70044]: from='client.57391 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:50:21.078 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:50:21.078 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:50:21.079 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:50:21.079 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:50:21.079 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:50:21.079 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:50:21.080 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:50:21.080 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:50:21.080 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:50:21.080 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:50:21.081 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:50:21.081 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:50:21.081 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:50:21.081 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:50:21.082 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:50:21.082 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:50:21.082 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:50:21.082 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:50:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:50:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:50:21] "GET /metrics HTTP/1.1" 200 225229 "" "Prometheus/2.33.4" 2022-09-29T11:50:21.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:50:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:50:21.573Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:50:21.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:50:21 smithi083 ceph-mon[77242]: pgmap v11462: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:21.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:50:21 smithi083 ceph-mon[77242]: from='client.47466 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:50:21.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:50:21 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1442627625' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:50:21.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:50:21 smithi083 ceph-mon[79930]: pgmap v11462: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:21.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:50:21 smithi083 ceph-mon[79930]: from='client.47466 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:50:21.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:50:21 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1442627625' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:50:22.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:50:21 smithi153 ceph-mon[70044]: pgmap v11462: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:22.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:50:21 smithi153 ceph-mon[70044]: from='client.47466 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:50:22.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:50:21 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1442627625' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:50:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:50:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:50:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:50:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:50:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:50:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:50:24.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:50:23 smithi153 ceph-mon[70044]: pgmap v11463: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:24.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:50:23 smithi083 ceph-mon[79930]: pgmap v11463: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:24.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:50:23 smithi083 ceph-mon[77242]: pgmap v11463: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:25.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:50:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:50:24.947Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:50:25.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:50:25 smithi083 ceph-mon[77242]: pgmap v11464: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:25.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:50:25 smithi083 ceph-mon[79930]: pgmap v11464: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:25.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:50:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:50:25.671Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:50:26.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:50:25 smithi153 ceph-mon[70044]: pgmap v11464: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:28.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:50:27 smithi153 ceph-mon[70044]: pgmap v11465: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:28.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:50:27 smithi083 ceph-mon[79930]: pgmap v11465: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:28.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:50:27 smithi083 ceph-mon[77242]: pgmap v11465: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:30.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:50:29 smithi153 ceph-mon[70044]: pgmap v11466: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:30.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:50:29 smithi083 ceph-mon[79930]: pgmap v11466: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:30.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:50:29 smithi083 ceph-mon[77242]: pgmap v11466: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:50:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:50:31] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T11:50:31.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:50:31 smithi083 ceph-mon[79930]: pgmap v11467: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:31.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:50:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:50:31.574Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:50:31.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:50:31 smithi083 ceph-mon[77242]: pgmap v11467: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:32.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:50:31 smithi153 ceph-mon[70044]: pgmap v11467: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:50:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:50:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:50:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:50:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:50:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:50:34.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:50:33 smithi153 ceph-mon[70044]: pgmap v11468: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:34.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:50:33 smithi083 ceph-mon[77242]: pgmap v11468: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:34.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:50:33 smithi083 ceph-mon[79930]: pgmap v11468: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:35.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:50:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:50:34.947Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:50:35.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:50:35 smithi083 ceph-mon[79930]: pgmap v11469: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:35.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:50:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:50:35.672Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:50:35.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:50:35 smithi083 ceph-mon[77242]: pgmap v11469: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:36.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:50:35 smithi153 ceph-mon[70044]: pgmap v11469: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:38.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:50:37 smithi153 ceph-mon[70044]: pgmap v11470: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:38.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:50:37 smithi083 ceph-mon[77242]: pgmap v11470: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:38.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:50:37 smithi083 ceph-mon[79930]: pgmap v11470: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:50:39 smithi153 ceph-mon[70044]: pgmap v11471: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:50:39 smithi083 ceph-mon[79930]: pgmap v11471: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:50:39 smithi083 ceph-mon[77242]: pgmap v11471: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:50:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:50:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:50:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:50:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:50:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:50:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:50:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:50:41.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:50:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:50:41.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:50:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:50:41.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:50:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:50:41.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:50:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:50:41.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:50:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:50:41.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:50:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:50:41.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:50:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:50:41.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:50:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:50:41.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:50:41 smithi083 ceph-mon[77242]: pgmap v11472: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:41.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:50:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:50:41] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T11:50:41.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:50:41 smithi083 ceph-mon[79930]: pgmap v11472: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:41.964 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:50:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:50:41.575Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:50:42.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:50:41 smithi153 ceph-mon[70044]: pgmap v11472: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:50:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:50:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:50:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:50:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:50:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:50:44.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:50:43 smithi153 ceph-mon[70044]: pgmap v11473: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:44.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:50:43 smithi083 ceph-mon[79930]: pgmap v11473: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:44.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:50:43 smithi083 ceph-mon[77242]: pgmap v11473: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:45.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:50:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:50:44.948Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:50:45.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:50:45 smithi083 ceph-mon[79930]: pgmap v11474: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:45.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:50:45 smithi083 ceph-mon[77242]: pgmap v11474: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:45.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:50:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:50:45.672Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:50:46.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:50:45 smithi153 ceph-mon[70044]: pgmap v11474: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:48.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:50:47 smithi153 ceph-mon[70044]: pgmap v11475: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:48.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:50:47 smithi083 ceph-mon[79930]: pgmap v11475: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:48.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:50:47 smithi083 ceph-mon[77242]: pgmap v11475: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:50.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:50:49 smithi153 ceph-mon[70044]: pgmap v11476: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:50.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:50:49 smithi083 ceph-mon[79930]: pgmap v11476: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:50.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:50:49 smithi083 ceph-mon[77242]: pgmap v11476: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:51.403 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:50:51.690 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:50:51.691 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 7m ago 6h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:50:51.691 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 5m ago 6h 64.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:50:51.691 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 7m ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:50:51.692 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 5m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:50:51.692 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 5m ago 6h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:50:51.692 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 7m ago 6h 831M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:50:51.692 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 7m ago 6h 1415M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:50:51.692 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 5m ago 6h 610M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:50:51.693 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 7m ago 6h 816M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:50:51.693 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 7m ago 6h 26.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:50:51.693 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 5m ago 6h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:50:51.693 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 7m ago 6h 997M 2411M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:50:51.694 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 7m ago 6h 986M 2411M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:50:51.694 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 7m ago 6h 769M 2411M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:50:51.694 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 7m ago 6h 964M 2411M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:50:51.694 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 5m ago 6h 848M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:50:51.695 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 5m ago 6h 1085M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:50:51.695 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 5m ago 6h 947M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:50:51.695 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 5m ago 6h 1148M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:50:51.695 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 5m ago 6h 156M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:50:51.760 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:50:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:50:51] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T11:50:51.760 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:50:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:50:51.575Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:50:52.044 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:50:52.045 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:50:52.045 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:50:52.045 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:50:52.045 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:50:52.046 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:50:52.046 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:50:52.046 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:50:52.046 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:50:52.047 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:50:52.047 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:50:52.047 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:50:52.047 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:50:52.048 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:50:52.048 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:50:52.048 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:50:52.048 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:50:52.048 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:50:52.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:50:51 smithi153 ceph-mon[70044]: pgmap v11477: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:52.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:50:51 smithi153 ceph-mon[70044]: from='client.47478 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:50:52.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:50:51 smithi083 ceph-mon[77242]: pgmap v11477: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:52.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:50:51 smithi083 ceph-mon[77242]: from='client.47478 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:50:52.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:50:51 smithi083 ceph-mon[79930]: pgmap v11477: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:52.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:50:51 smithi083 ceph-mon[79930]: from='client.47478 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:50:52.768 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:50:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:50:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:50:53.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:50:52 smithi153 ceph-mon[70044]: from='client.57415 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:50:53.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:50:52 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2877842298' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:50:53.219 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:50:52 smithi083 ceph-mon[79930]: from='client.57415 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:50:53.219 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:50:52 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2877842298' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:50:53.220 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:50:52 smithi083 ceph-mon[77242]: from='client.57415 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:50:53.220 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:50:52 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2877842298' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:50:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:50:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:50:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:50:54.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:50:53 smithi153 ceph-mon[70044]: pgmap v11478: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:54.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:50:53 smithi083 ceph-mon[79930]: pgmap v11478: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:54.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:50:53 smithi083 ceph-mon[77242]: pgmap v11478: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:55.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:50:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:50:54.948Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:50:55.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:50:55 smithi083 ceph-mon[79930]: pgmap v11479: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:55.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:50:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:50:55.672Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:50:55.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:50:55 smithi083 ceph-mon[77242]: pgmap v11479: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:56.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:50:55 smithi153 ceph-mon[70044]: pgmap v11479: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:58.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:50:57 smithi153 ceph-mon[70044]: pgmap v11480: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:58.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:50:57 smithi083 ceph-mon[79930]: pgmap v11480: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:50:58.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:50:57 smithi083 ceph-mon[77242]: pgmap v11480: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:00.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:50:59 smithi153 ceph-mon[70044]: pgmap v11481: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:00.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:50:59 smithi083 ceph-mon[79930]: pgmap v11481: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:00.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:50:59 smithi083 ceph-mon[77242]: pgmap v11481: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:01.780 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:51:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:51:01] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T11:51:01.781 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:51:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:51:01.576Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:51:02.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:51:01 smithi153 ceph-mon[70044]: pgmap v11482: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:02.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:51:01 smithi083 ceph-mon[77242]: pgmap v11482: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:02.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:51:01 smithi083 ceph-mon[79930]: pgmap v11482: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:51:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:51:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:51:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:51:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:51:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:51:04.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:51:03 smithi153 ceph-mon[70044]: pgmap v11483: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:04.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:51:03 smithi083 ceph-mon[77242]: pgmap v11483: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:04.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:51:03 smithi083 ceph-mon[79930]: pgmap v11483: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:05.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:51:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:51:04.949Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:51:05.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:51:05 smithi083 ceph-mon[79930]: pgmap v11484: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:05.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:51:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:51:05.673Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:51:05.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:51:05 smithi083 ceph-mon[77242]: pgmap v11484: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:06.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:51:05 smithi153 ceph-mon[70044]: pgmap v11484: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:08.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:51:07 smithi153 ceph-mon[70044]: pgmap v11485: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:08.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:51:07 smithi083 ceph-mon[77242]: pgmap v11485: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:08.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:51:07 smithi083 ceph-mon[79930]: pgmap v11485: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:10.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:51:09 smithi153 ceph-mon[70044]: pgmap v11486: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:10.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:51:09 smithi083 ceph-mon[77242]: pgmap v11486: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:10.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:51:09 smithi083 ceph-mon[79930]: pgmap v11486: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:11.794 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:51:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:51:11.577Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:51:11.795 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:51:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:51:11] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T11:51:12.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:51:11 smithi153 ceph-mon[70044]: pgmap v11487: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:12.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:51:11 smithi083 ceph-mon[79930]: pgmap v11487: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:12.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:51:11 smithi083 ceph-mon[77242]: pgmap v11487: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:51:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:51:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:51:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:51:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:51:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:51:14.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:51:13 smithi153 ceph-mon[70044]: pgmap v11488: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:14.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:51:13 smithi083 ceph-mon[79930]: pgmap v11488: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:14.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:51:13 smithi083 ceph-mon[77242]: pgmap v11488: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:15.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:51:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:51:14.949Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:51:15.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:51:15 smithi083 ceph-mon[79930]: pgmap v11489: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:15.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:51:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:51:15.674Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:51:15.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:51:15 smithi083 ceph-mon[77242]: pgmap v11489: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:16.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:51:15 smithi153 ceph-mon[70044]: pgmap v11489: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:18.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:51:17 smithi153 ceph-mon[70044]: pgmap v11490: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:18.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:51:17 smithi083 ceph-mon[79930]: pgmap v11490: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:18.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:51:17 smithi083 ceph-mon[77242]: pgmap v11490: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:19.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:51:18 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:51:19.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:51:18 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:51:19.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:51:18 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:51:19.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:51:18 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:51:19.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:51:18 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:51:19.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:51:18 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:51:19.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:51:18 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:51:19.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:51:18 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:51:19.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:51:18 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:51:20.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:51:19 smithi083 ceph-mon[79930]: pgmap v11491: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:20.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:51:19 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:51:20.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:51:19 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:51:20.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:51:19 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:51:20.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:51:19 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:51:20.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:51:19 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:51:20.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:51:19 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:51:20.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:51:19 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:51:20.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:51:19 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:51:20.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:51:19 smithi083 ceph-mon[79930]: Adjusting osd_memory_target on smithi083 to 2402M 2022-09-29T11:51:20.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:51:19 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:51:20.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:51:19 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:51:20.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:51:19 smithi083 ceph-mon[77242]: pgmap v11491: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:20.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:51:19 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:51:20.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:51:19 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:51:20.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:51:19 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:51:20.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:51:19 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:51:20.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:51:19 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:51:20.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:51:19 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:51:20.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:51:19 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:51:20.218 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:51:19 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:51:20.218 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:51:19 smithi083 ceph-mon[77242]: Adjusting osd_memory_target on smithi083 to 2402M 2022-09-29T11:51:20.218 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:51:19 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:51:20.218 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:51:19 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:51:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:51:19 smithi153 ceph-mon[70044]: pgmap v11491: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:51:19 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:51:20.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:51:19 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:51:20.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:51:19 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:51:20.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:51:19 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:51:20.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:51:19 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:51:20.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:51:19 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:51:20.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:51:19 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:51:20.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:51:19 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T11:51:20.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:51:19 smithi153 ceph-mon[70044]: Adjusting osd_memory_target on smithi083 to 2402M 2022-09-29T11:51:20.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:51:19 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:51:20.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:51:19 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:51:21.813 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:51:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:51:21] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T11:51:21.814 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:51:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:51:21.578Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:51:22.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:51:21 smithi083 ceph-mon[79930]: pgmap v11492: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:22.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:51:21 smithi083 ceph-mon[77242]: pgmap v11492: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:22.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:51:21 smithi153 ceph-mon[70044]: pgmap v11492: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:22.370 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:51:22.658 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:51:22.659 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 8m ago 6h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:51:22.659 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 6m ago 6h 64.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:51:22.659 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 8m ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:51:22.659 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 6m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:51:22.660 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 6m ago 6h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:51:22.660 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 8m ago 6h 831M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:51:22.660 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 8m ago 6h 1415M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:51:22.660 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 6m ago 6h 610M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:51:22.661 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 8m ago 6h 816M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:51:22.661 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 8m ago 6h 26.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:51:22.661 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 6m ago 6h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:51:22.661 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 8m ago 6h 997M 2402M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:51:22.661 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 8m ago 6h 986M 2402M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:51:22.662 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 8m ago 6h 769M 2402M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:51:22.662 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 8m ago 6h 964M 2402M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:51:22.662 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 6m ago 6h 848M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:51:22.662 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 6m ago 6h 1085M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:51:22.663 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 6m ago 6h 947M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:51:22.663 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 6m ago 6h 1148M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:51:22.663 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 6m ago 6h 156M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:51:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:51:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:51:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:51:23.017 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:51:23.017 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:51:23.017 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:51:23.018 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:51:23.018 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:51:23.018 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:51:23.018 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:51:23.019 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:51:23.019 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:51:23.019 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:51:23.020 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:51:23.020 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:51:23.020 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:51:23.020 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:51:23.021 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:51:23.021 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:51:23.021 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:51:23.021 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:51:23.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:51:22 smithi083 ceph-mon[79930]: from='client.57427 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:51:23.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:51:22 smithi083 ceph-mon[77242]: from='client.57427 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:51:23.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:51:22 smithi153 ceph-mon[70044]: from='client.57427 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:51:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:51:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:51:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:51:24.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:51:23 smithi083 ceph-mon[79930]: from='client.47502 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:51:24.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:51:23 smithi083 ceph-mon[79930]: pgmap v11493: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:24.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:51:23 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1633360332' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:51:24.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:51:23 smithi083 ceph-mon[77242]: from='client.47502 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:51:24.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:51:23 smithi083 ceph-mon[77242]: pgmap v11493: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:24.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:51:23 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1633360332' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:51:24.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:51:23 smithi153 ceph-mon[70044]: from='client.47502 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:51:24.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:51:23 smithi153 ceph-mon[70044]: pgmap v11493: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:24.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:51:23 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1633360332' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:51:25.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:51:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:51:24.950Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:51:25.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:51:25 smithi083 ceph-mon[77242]: pgmap v11494: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:25.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:51:25 smithi083 ceph-mon[79930]: pgmap v11494: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:25.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:51:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:51:25.675Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:51:26.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:51:25 smithi153 ceph-mon[70044]: pgmap v11494: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:28.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:51:27 smithi083 ceph-mon[79930]: pgmap v11495: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:28.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:51:27 smithi083 ceph-mon[77242]: pgmap v11495: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:28.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:51:27 smithi153 ceph-mon[70044]: pgmap v11495: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:30.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:51:29 smithi083 ceph-mon[77242]: pgmap v11496: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:30.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:51:29 smithi083 ceph-mon[79930]: pgmap v11496: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:30.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:51:29 smithi153 ceph-mon[70044]: pgmap v11496: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:31.830 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:51:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:51:31] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T11:51:31.831 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:51:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:51:31.579Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:51:32.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:51:31 smithi083 ceph-mon[79930]: pgmap v11497: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:32.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:51:31 smithi083 ceph-mon[77242]: pgmap v11497: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:32.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:51:31 smithi153 ceph-mon[70044]: pgmap v11497: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:51:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:51:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:51:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:51:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:51:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:51:34.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:51:33 smithi083 ceph-mon[79930]: pgmap v11498: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:34.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:51:33 smithi083 ceph-mon[77242]: pgmap v11498: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:34.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:51:33 smithi153 ceph-mon[70044]: pgmap v11498: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:35.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:51:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:51:34.951Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:51:35.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:51:35 smithi083 ceph-mon[79930]: pgmap v11499: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:35.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:51:35 smithi083 ceph-mon[77242]: pgmap v11499: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:35.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:51:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:51:35.675Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:51:36.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:51:35 smithi153 ceph-mon[70044]: pgmap v11499: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:38.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:51:37 smithi083 ceph-mon[79930]: pgmap v11500: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:38.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:51:37 smithi083 ceph-mon[77242]: pgmap v11500: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:38.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:51:37 smithi153 ceph-mon[70044]: pgmap v11500: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:51:39 smithi083 ceph-mon[79930]: pgmap v11501: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:51:39 smithi083 ceph-mon[77242]: pgmap v11501: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:51:39 smithi153 ceph-mon[70044]: pgmap v11501: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:41.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:51:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:51:41.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:51:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:51:41.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:51:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:51:41.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:51:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:51:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:51:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:51:41.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:51:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:51:41.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:51:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:51:41.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:51:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:51:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:51:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:51:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:51:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:51:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:51:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:51:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:51:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:51:41.857 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:51:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:51:41] "GET /metrics HTTP/1.1" 200 225228 "" "Prometheus/2.33.4" 2022-09-29T11:51:41.858 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:51:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:51:41.579Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:51:42.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:51:41 smithi083 ceph-mon[77242]: pgmap v11502: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:42.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:51:41 smithi083 ceph-mon[79930]: pgmap v11502: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:51:41 smithi153 ceph-mon[70044]: pgmap v11502: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:51:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:51:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:51:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:51:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:51:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:51:44.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:51:43 smithi083 ceph-mon[79930]: pgmap v11503: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:44.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:51:43 smithi083 ceph-mon[77242]: pgmap v11503: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:44.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:51:43 smithi153 ceph-mon[70044]: pgmap v11503: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:45.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:51:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:51:44.952Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:51:45.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:51:45 smithi083 ceph-mon[79930]: pgmap v11504: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:45.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:51:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:51:45.675Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:51:45.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:51:45 smithi083 ceph-mon[77242]: pgmap v11504: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:46.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:51:45 smithi153 ceph-mon[70044]: pgmap v11504: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:48.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:51:47 smithi083 ceph-mon[79930]: pgmap v11505: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:48.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:51:47 smithi083 ceph-mon[77242]: pgmap v11505: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:48.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:51:47 smithi153 ceph-mon[70044]: pgmap v11505: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:50.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:51:49 smithi083 ceph-mon[79930]: pgmap v11506: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:50.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:51:49 smithi083 ceph-mon[77242]: pgmap v11506: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:50.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:51:49 smithi153 ceph-mon[70044]: pgmap v11506: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:51.872 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:51:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:51:51] "GET /metrics HTTP/1.1" 200 225228 "" "Prometheus/2.33.4" 2022-09-29T11:51:51.873 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:51:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:51:51.580Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:51:52.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:51:51 smithi083 ceph-mon[79930]: pgmap v11507: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:52.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:51:51 smithi083 ceph-mon[77242]: pgmap v11507: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:52.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:51:51 smithi153 ceph-mon[70044]: pgmap v11507: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:51:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:51:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:51:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:51:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:51:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:51:53.350 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:51:53.648 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:51:53.648 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 8m ago 6h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:51:53.649 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 6m ago 6h 64.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:51:53.649 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 8m ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:51:53.649 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 6m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:51:53.649 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 6m ago 6h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:51:53.649 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 8m ago 6h 831M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:51:53.650 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 8m ago 6h 1415M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:51:53.650 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 6m ago 6h 610M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:51:53.650 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 8m ago 6h 816M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:51:53.651 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 8m ago 6h 26.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:51:53.651 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 6m ago 6h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:51:53.651 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 8m ago 6h 997M 2402M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:51:53.651 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 8m ago 6h 986M 2402M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:51:53.651 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 8m ago 6h 769M 2402M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:51:53.652 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 8m ago 6h 964M 2402M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:51:53.652 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 6m ago 6h 848M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:51:53.652 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 6m ago 6h 1085M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:51:53.652 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 6m ago 6h 947M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:51:53.653 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 6m ago 6h 1148M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:51:53.653 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 6m ago 6h 156M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:51:54.011 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:51:54.011 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:51:54.012 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:51:54.012 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:51:54.012 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:51:54.012 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:51:54.013 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:51:54.013 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:51:54.013 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:51:54.013 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:51:54.013 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:51:54.014 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:51:54.014 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:51:54.014 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:51:54.014 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:51:54.015 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:51:54.015 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:51:54.015 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:51:54.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:51:53 smithi083 ceph-mon[79930]: pgmap v11508: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:54.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:51:53 smithi083 ceph-mon[79930]: from='client.47514 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:51:54.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:51:53 smithi083 ceph-mon[77242]: pgmap v11508: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:54.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:51:53 smithi083 ceph-mon[77242]: from='client.47514 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:51:54.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:51:53 smithi153 ceph-mon[70044]: pgmap v11508: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:54.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:51:53 smithi153 ceph-mon[70044]: from='client.47514 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:51:55.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:51:54 smithi083 ceph-mon[79930]: from='client.47520 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:51:55.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:51:54 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1293147157' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:51:55.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:51:54 smithi083 ceph-mon[77242]: from='client.47520 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:51:55.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:51:54 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1293147157' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:51:55.214 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:51:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:51:54.952Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:51:55.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:51:54 smithi153 ceph-mon[70044]: from='client.47520 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:51:55.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:51:54 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1293147157' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:51:55.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:51:55 smithi083 ceph-mon[79930]: pgmap v11509: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:55.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:51:55 smithi083 ceph-mon[77242]: pgmap v11509: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:55.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:51:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:51:55.676Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:51:56.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:51:55 smithi153 ceph-mon[70044]: pgmap v11509: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:58.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:51:57 smithi083 ceph-mon[79930]: pgmap v11510: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:58.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:51:57 smithi083 ceph-mon[77242]: pgmap v11510: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:51:58.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:51:57 smithi153 ceph-mon[70044]: pgmap v11510: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:00.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:51:59 smithi083 ceph-mon[79930]: pgmap v11511: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:00.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:51:59 smithi083 ceph-mon[77242]: pgmap v11511: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:51:59 smithi153 ceph-mon[70044]: pgmap v11511: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:01.893 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:52:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:52:01] "GET /metrics HTTP/1.1" 200 225229 "" "Prometheus/2.33.4" 2022-09-29T11:52:01.894 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:52:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:52:01.581Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:52:02.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:52:01 smithi083 ceph-mon[79930]: pgmap v11512: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:02.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:52:01 smithi083 ceph-mon[77242]: pgmap v11512: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:02.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:52:01 smithi153 ceph-mon[70044]: pgmap v11512: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:52:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:52:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:52:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:52:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:52:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:52:04.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:52:03 smithi083 ceph-mon[79930]: pgmap v11513: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:04.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:52:03 smithi083 ceph-mon[77242]: pgmap v11513: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:04.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:52:03 smithi153 ceph-mon[70044]: pgmap v11513: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:05.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:52:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:52:04.953Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:52:05.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:52:05 smithi083 ceph-mon[79930]: pgmap v11514: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:05.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:52:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:52:05.676Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:52:05.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:52:05 smithi083 ceph-mon[77242]: pgmap v11514: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:06.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:52:05 smithi153 ceph-mon[70044]: pgmap v11514: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:08.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:52:07 smithi083 ceph-mon[79930]: pgmap v11515: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:08.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:52:07 smithi083 ceph-mon[77242]: pgmap v11515: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:08.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:52:07 smithi153 ceph-mon[70044]: pgmap v11515: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:10.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:52:09 smithi083 ceph-mon[79930]: pgmap v11516: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:10.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:52:09 smithi083 ceph-mon[77242]: pgmap v11516: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:10.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:52:09 smithi153 ceph-mon[70044]: pgmap v11516: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:11.914 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:52:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:52:11] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T11:52:11.914 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:52:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:52:11.582Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:52:12.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:52:11 smithi083 ceph-mon[79930]: pgmap v11517: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:12.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:52:11 smithi083 ceph-mon[77242]: pgmap v11517: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:12.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:52:11 smithi153 ceph-mon[70044]: pgmap v11517: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:52:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:52:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:52:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:52:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:52:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:52:14.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:52:13 smithi083 ceph-mon[79930]: pgmap v11518: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:14.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:52:13 smithi083 ceph-mon[77242]: pgmap v11518: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:14.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:52:13 smithi153 ceph-mon[70044]: pgmap v11518: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:15.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:52:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:52:14.954Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:52:15.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:52:15 smithi083 ceph-mon[79930]: pgmap v11519: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:15.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:52:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:52:15.677Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:52:15.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:52:15 smithi083 ceph-mon[77242]: pgmap v11519: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:16.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:52:15 smithi153 ceph-mon[70044]: pgmap v11519: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:18.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:52:17 smithi083 ceph-mon[77242]: pgmap v11520: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:18.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:52:17 smithi083 ceph-mon[79930]: pgmap v11520: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:18.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:52:17 smithi153 ceph-mon[70044]: pgmap v11520: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:20.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:52:19 smithi083 ceph-mon[79930]: pgmap v11521: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:20.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:52:19 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:52:20.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:52:19 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:52:20.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:52:19 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:52:20.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:52:19 smithi083 ceph-mon[77242]: pgmap v11521: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:20.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:52:19 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:52:20.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:52:19 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:52:20.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:52:19 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:52:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:52:19 smithi153 ceph-mon[70044]: pgmap v11521: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:52:19 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:52:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:52:19 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:52:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:52:19 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:52:21.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:52:21 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:52:21.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:52:21 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:52:21.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:52:21 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:52:21.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:52:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:52:21.583Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:52:21.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:52:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:52:21] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T11:52:22.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:52:22 smithi153 ceph-mon[70044]: pgmap v11522: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:22.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:52:22 smithi083 ceph-mon[79930]: pgmap v11522: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:22.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:52:22 smithi083 ceph-mon[77242]: pgmap v11522: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:52:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:52:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:52:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:52:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:52:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:52:24.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:52:24 smithi153 ceph-mon[70044]: pgmap v11523: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:24.342 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:52:24.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:52:24 smithi083 ceph-mon[79930]: pgmap v11523: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:24.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:52:24 smithi083 ceph-mon[77242]: pgmap v11523: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:24.634 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:52:24.635 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 9m ago 6h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:52:24.635 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 7m ago 6h 64.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:52:24.635 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 9m ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:52:24.636 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 7m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:52:24.636 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 7m ago 6h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:52:24.636 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 9m ago 6h 831M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:52:24.636 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 9m ago 6h 1415M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:52:24.637 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 7m ago 6h 610M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:52:24.637 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 9m ago 6h 816M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:52:24.637 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 9m ago 6h 26.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:52:24.637 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 7m ago 6h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:52:24.638 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 9m ago 6h 997M 2402M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:52:24.638 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 9m ago 6h 986M 2402M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:52:24.638 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 9m ago 6h 769M 2402M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:52:24.638 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 9m ago 6h 964M 2402M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:52:24.639 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 7m ago 6h 848M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:52:24.639 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 7m ago 6h 1085M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:52:24.639 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 7m ago 6h 947M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:52:24.639 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 7m ago 6h 1148M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:52:24.639 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 7m ago 6h 156M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:52:24.990 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:52:24.991 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:52:24.991 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:52:24.991 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:52:24.991 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:52:24.992 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:52:24.992 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:52:24.992 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:52:24.992 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:52:24.993 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:52:24.993 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:52:24.993 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:52:24.993 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:52:24.993 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:52:24.994 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:52:24.994 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:52:24.994 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:52:24.994 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:52:25.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:52:25 smithi083 ceph-mon[79930]: from='client.47532 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:52:25.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:52:25 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/490117755' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:52:25.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:52:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:52:24.953Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:52:25.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:52:25 smithi083 ceph-mon[77242]: from='client.47532 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:52:25.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:52:25 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/490117755' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:52:25.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:52:25 smithi153 ceph-mon[70044]: from='client.47532 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:52:25.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:52:25 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/490117755' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:52:25.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:52:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:52:25.677Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:52:26.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:52:26 smithi153 ceph-mon[70044]: from='client.47538 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:52:26.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:52:26 smithi153 ceph-mon[70044]: pgmap v11524: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:26.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:52:26 smithi083 ceph-mon[77242]: from='client.47538 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:52:26.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:52:26 smithi083 ceph-mon[77242]: pgmap v11524: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:26.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:52:26 smithi083 ceph-mon[79930]: from='client.47538 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:52:26.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:52:26 smithi083 ceph-mon[79930]: pgmap v11524: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:28.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:52:28 smithi153 ceph-mon[70044]: pgmap v11525: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:28.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:52:28 smithi083 ceph-mon[79930]: pgmap v11525: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:28.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:52:28 smithi083 ceph-mon[77242]: pgmap v11525: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:30.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:52:30 smithi153 ceph-mon[70044]: pgmap v11526: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:30.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:52:30 smithi083 ceph-mon[79930]: pgmap v11526: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:30.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:52:30 smithi083 ceph-mon[77242]: pgmap v11526: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:52:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:52:31] "GET /metrics HTTP/1.1" 200 225228 "" "Prometheus/2.33.4" 2022-09-29T11:52:31.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:52:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:52:31.583Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:52:32.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:52:32 smithi153 ceph-mon[70044]: pgmap v11527: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:32.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:52:32 smithi083 ceph-mon[77242]: pgmap v11527: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:32.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:52:32 smithi083 ceph-mon[79930]: pgmap v11527: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:52:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:52:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:52:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:52:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:52:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:52:34.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:52:34 smithi153 ceph-mon[70044]: pgmap v11528: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:34.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:52:34 smithi083 ceph-mon[79930]: pgmap v11528: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:34.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:52:34 smithi083 ceph-mon[77242]: pgmap v11528: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:35.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:52:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:52:34.956Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:52:35.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:52:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:52:35.679Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:52:36.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:52:36 smithi083 ceph-mon[77242]: pgmap v11529: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:36.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:52:36 smithi083 ceph-mon[79930]: pgmap v11529: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:52:36 smithi153 ceph-mon[70044]: pgmap v11529: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:38.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:52:38 smithi083 ceph-mon[79930]: pgmap v11530: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:38.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:52:38 smithi083 ceph-mon[77242]: pgmap v11530: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:52:38 smithi153 ceph-mon[70044]: pgmap v11530: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:52:40 smithi083 ceph-mon[79930]: pgmap v11531: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:52:40 smithi083 ceph-mon[77242]: pgmap v11531: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:52:40 smithi153 ceph-mon[70044]: pgmap v11531: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:41.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:52:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:52:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:52:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:52:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:52:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:52:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:52:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:52:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:52:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:52:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:52:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:52:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:52:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:52:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:52:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:52:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:52:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:52:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:52:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:52:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:52:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:52:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:52:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:52:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:52:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:52:41] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T11:52:41.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:52:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:52:41.584Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:52:42.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:52:42 smithi083 ceph-mon[77242]: pgmap v11532: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:42.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:52:42 smithi083 ceph-mon[79930]: pgmap v11532: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:52:42 smithi153 ceph-mon[70044]: pgmap v11532: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:42.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:52:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:52:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:52:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:52:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:52:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:52:44.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:52:44 smithi083 ceph-mon[79930]: pgmap v11533: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:44.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:52:44 smithi083 ceph-mon[77242]: pgmap v11533: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:44.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:52:44 smithi153 ceph-mon[70044]: pgmap v11533: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:45.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:52:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:52:44.956Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:52:45.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:52:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:52:45.680Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:52:46.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:52:46 smithi083 ceph-mon[79930]: pgmap v11534: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:46.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:52:46 smithi083 ceph-mon[77242]: pgmap v11534: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:46.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:52:46 smithi153 ceph-mon[70044]: pgmap v11534: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:48.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:52:48 smithi083 ceph-mon[79930]: pgmap v11535: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:48.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:52:48 smithi083 ceph-mon[77242]: pgmap v11535: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:48.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:52:48 smithi153 ceph-mon[70044]: pgmap v11535: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:50.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:52:50 smithi083 ceph-mon[79930]: pgmap v11536: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:50.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:52:50 smithi083 ceph-mon[77242]: pgmap v11536: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:52:50 smithi153 ceph-mon[70044]: pgmap v11536: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:52:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:52:51] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T11:52:51.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:52:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:52:51.584Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:52:52.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:52:52 smithi083 ceph-mon[77242]: pgmap v11537: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:52.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:52:52 smithi083 ceph-mon[79930]: pgmap v11537: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:52:52 smithi153 ceph-mon[70044]: pgmap v11537: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:52.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:52:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:52:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:52:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:52:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:52:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:52:54.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:52:54 smithi083 ceph-mon[79930]: pgmap v11538: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:54.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:52:54 smithi083 ceph-mon[77242]: pgmap v11538: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:54.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:52:54 smithi153 ceph-mon[70044]: pgmap v11538: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:55.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:52:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:52:54.956Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:52:55.316 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:52:55.604 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:52:55.604 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 9m ago 6h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:52:55.605 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 7m ago 6h 64.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:52:55.605 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 9m ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:52:55.605 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 7m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:52:55.605 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 7m ago 6h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:52:55.606 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 9m ago 6h 831M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:52:55.606 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 9m ago 6h 1415M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:52:55.606 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 7m ago 6h 610M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:52:55.606 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 9m ago 6h 816M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:52:55.607 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 9m ago 6h 26.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:52:55.607 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 7m ago 6h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:52:55.607 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 9m ago 6h 997M 2402M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:52:55.607 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 9m ago 6h 986M 2402M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:52:55.608 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 9m ago 6h 769M 2402M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:52:55.608 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 9m ago 6h 964M 2402M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:52:55.608 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 7m ago 6h 848M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:52:55.608 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 7m ago 6h 1085M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:52:55.608 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 7m ago 6h 947M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:52:55.609 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 7m ago 6h 1148M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:52:55.609 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 7m ago 6h 156M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:52:55.962 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:52:55.962 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:52:55.963 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:52:55.963 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:52:55.963 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:52:55.963 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:52:55.963 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:52:55.964 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:52:55.964 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:52:55.964 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:52:55.964 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:52:55.965 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:52:55.965 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:52:55.965 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:52:55.965 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:52:55.965 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:52:55.966 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:52:55.966 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:52:55.966 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:52:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:52:55.680Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:52:56.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:52:56 smithi083 ceph-mon[79930]: pgmap v11539: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:56.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:52:56 smithi083 ceph-mon[79930]: from='client.47550 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:52:56.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:52:56 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2896776048' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:52:56.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:52:56 smithi083 ceph-mon[77242]: pgmap v11539: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:56.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:52:56 smithi083 ceph-mon[77242]: from='client.47550 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:52:56.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:52:56 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2896776048' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:52:56.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:52:56 smithi153 ceph-mon[70044]: pgmap v11539: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:56.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:52:56 smithi153 ceph-mon[70044]: from='client.47550 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:52:56.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:52:56 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2896776048' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:52:57.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:52:57 smithi083 ceph-mon[79930]: from='client.47556 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:52:57.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:52:57 smithi083 ceph-mon[77242]: from='client.47556 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:52:57.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:52:57 smithi153 ceph-mon[70044]: from='client.47556 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:52:58.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:52:58 smithi083 ceph-mon[77242]: pgmap v11540: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:58.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:52:58 smithi083 ceph-mon[79930]: pgmap v11540: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:52:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:52:58 smithi153 ceph-mon[70044]: pgmap v11540: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:00.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:53:00 smithi083 ceph-mon[79930]: pgmap v11541: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:00.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:53:00 smithi083 ceph-mon[77242]: pgmap v11541: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:53:00 smithi153 ceph-mon[70044]: pgmap v11541: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:53:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:53:01] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T11:53:01.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:53:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:53:01.584Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:53:02.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:53:02 smithi083 ceph-mon[79930]: pgmap v11542: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:02.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:53:02 smithi083 ceph-mon[77242]: pgmap v11542: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:53:02 smithi153 ceph-mon[70044]: pgmap v11542: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:02.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:53:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:53:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:53:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:53:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:53:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:53:04.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:53:04 smithi083 ceph-mon[79930]: pgmap v11543: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:04.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:53:04 smithi083 ceph-mon[77242]: pgmap v11543: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:53:04 smithi153 ceph-mon[70044]: pgmap v11543: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:05.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:53:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:53:04.956Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:53:05.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:53:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:53:05.682Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:53:06.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:53:06 smithi083 ceph-mon[79930]: pgmap v11544: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:06.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:53:06 smithi083 ceph-mon[77242]: pgmap v11544: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:06.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:53:06 smithi153 ceph-mon[70044]: pgmap v11544: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:08.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:53:08 smithi083 ceph-mon[79930]: pgmap v11545: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:08.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:53:08 smithi083 ceph-mon[77242]: pgmap v11545: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:53:08 smithi153 ceph-mon[70044]: pgmap v11545: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:10.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:53:10 smithi083 ceph-mon[77242]: pgmap v11546: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:10.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:53:10 smithi083 ceph-mon[79930]: pgmap v11546: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:53:10 smithi153 ceph-mon[70044]: pgmap v11546: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:53:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:53:11] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T11:53:11.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:53:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:53:11.584Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:53:12.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:53:12 smithi083 ceph-mon[77242]: pgmap v11547: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:12.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:53:12 smithi083 ceph-mon[79930]: pgmap v11547: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:53:12 smithi153 ceph-mon[70044]: pgmap v11547: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:12.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:53:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:53:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:53:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:53:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:53:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:53:14.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:53:14 smithi083 ceph-mon[79930]: pgmap v11548: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:14.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:53:14 smithi083 ceph-mon[77242]: pgmap v11548: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:14.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:53:14 smithi153 ceph-mon[70044]: pgmap v11548: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:15.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:53:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:53:14.957Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:53:15.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:53:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:53:15.682Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:53:16.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:53:16 smithi083 ceph-mon[79930]: pgmap v11549: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:16.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:53:16 smithi083 ceph-mon[77242]: pgmap v11549: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:53:16 smithi153 ceph-mon[70044]: pgmap v11549: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:18.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:53:18 smithi083 ceph-mon[77242]: pgmap v11550: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:18.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:53:18 smithi083 ceph-mon[79930]: pgmap v11550: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:53:18 smithi153 ceph-mon[70044]: pgmap v11550: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:20.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:53:20 smithi083 ceph-mon[79930]: pgmap v11551: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:20.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:53:20 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:53:20.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:53:20 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:53:20.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:53:20 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:53:20.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:53:20 smithi083 ceph-mon[77242]: pgmap v11551: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:20.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:53:20 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:53:20.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:53:20 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:53:20.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:53:20 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:53:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:53:20 smithi153 ceph-mon[70044]: pgmap v11551: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:53:20 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:53:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:53:20 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:53:20.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:53:20 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:53:21.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:53:21 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:53:21.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:53:21 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:53:21.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:53:21 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:53:21.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:53:21 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:53:21.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:53:21 smithi153 ceph-mon[70044]: pgmap v11552: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:53:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:53:21] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T11:53:21.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:53:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:53:21.585Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:53:21.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:53:21 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:53:21.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:53:21 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:53:21.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:53:21 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:53:21.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:53:21 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:53:21.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:53:21 smithi083 ceph-mon[77242]: pgmap v11552: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:21.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:53:21 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:53:21.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:53:21 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:53:21.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:53:21 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:53:21.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:53:21 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:53:21.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:53:21 smithi083 ceph-mon[79930]: pgmap v11552: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:53:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:53:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:53:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:53:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:53:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:53:24.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:53:23 smithi153 ceph-mon[70044]: pgmap v11553: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:24.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:53:23 smithi083 ceph-mon[79930]: pgmap v11553: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:24.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:53:23 smithi083 ceph-mon[77242]: pgmap v11553: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:25.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:53:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:53:24.957Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:53:25.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:53:25 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:53:25.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:53:25 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:53:25.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:53:25 smithi153 ceph-mon[70044]: pgmap v11554: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:25.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:53:25 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:53:25.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:53:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:53:25.682Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:53:25.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:53:25 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:53:25.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:53:25 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:53:25.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:53:25 smithi083 ceph-mon[77242]: pgmap v11554: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:25.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:53:25 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:53:25.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:53:25 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:53:25.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:53:25 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:53:25.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:53:25 smithi083 ceph-mon[79930]: pgmap v11554: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:25.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:53:25 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:53:26.281 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:53:26.571 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:53:26.571 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 2s ago 6h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:53:26.572 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 8m ago 6h 64.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:53:26.572 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 2s ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:53:26.572 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 8m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:53:26.572 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 8m ago 6h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:53:26.573 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 2s ago 6h 839M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:53:26.573 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 2s ago 6h 1452M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:53:26.573 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 8m ago 6h 610M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:53:26.573 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 2s ago 6h 838M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:53:26.573 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 2s ago 6h 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:53:26.574 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 8m ago 6h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:53:26.574 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 2s ago 6h 1047M 2402M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:53:26.574 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 2s ago 6h 1036M 2402M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:53:26.574 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 2s ago 6h 788M 2402M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:53:26.575 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 2s ago 6h 989M 2402M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:53:26.575 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 8m ago 6h 848M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:53:26.575 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 8m ago 6h 1085M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:53:26.575 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 8m ago 6h 947M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:53:26.576 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 8m ago 6h 1148M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:53:26.576 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 8m ago 6h 156M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:53:26.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:53:26 smithi153 ceph-mon[70044]: from='client.57499 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:53:26.926 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:53:26.926 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:53:26.927 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:53:26.927 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:53:26.927 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:53:26.927 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:53:26.928 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:53:26.928 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:53:26.928 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:53:26.928 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:53:26.929 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:53:26.929 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:53:26.929 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:53:26.929 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:53:26.930 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:53:26.930 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:53:26.930 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:53:26.930 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:53:26.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:53:26 smithi083 ceph-mon[79930]: from='client.57499 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:53:26.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:53:26 smithi083 ceph-mon[77242]: from='client.57499 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:53:27.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:53:27 smithi153 ceph-mon[70044]: from='client.47574 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:53:27.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:53:27 smithi153 ceph-mon[70044]: pgmap v11555: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:27.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:53:27 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/12025424' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:53:27.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:53:27 smithi083 ceph-mon[79930]: from='client.47574 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:53:27.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:53:27 smithi083 ceph-mon[79930]: pgmap v11555: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:27.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:53:27 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/12025424' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:53:27.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:53:27 smithi083 ceph-mon[77242]: from='client.47574 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:53:27.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:53:27 smithi083 ceph-mon[77242]: pgmap v11555: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:27.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:53:27 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/12025424' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:53:30.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:53:29 smithi153 ceph-mon[70044]: pgmap v11556: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:30.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:53:29 smithi083 ceph-mon[79930]: pgmap v11556: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:30.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:53:29 smithi083 ceph-mon[77242]: pgmap v11556: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:31.762 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:53:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:53:31] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T11:53:31.763 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:53:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:53:31.585Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:53:32.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:53:31 smithi153 ceph-mon[70044]: pgmap v11557: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:32.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:53:31 smithi083 ceph-mon[79930]: pgmap v11557: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:32.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:53:31 smithi083 ceph-mon[77242]: pgmap v11557: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:53:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:53:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:53:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:53:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:53:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:53:34.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:53:33 smithi153 ceph-mon[70044]: pgmap v11558: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:34.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:53:33 smithi083 ceph-mon[79930]: pgmap v11558: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:34.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:53:33 smithi083 ceph-mon[77242]: pgmap v11558: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:35.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:53:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:53:34.959Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:53:35.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:53:35 smithi083 ceph-mon[79930]: pgmap v11559: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:35.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:53:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:53:35.683Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:53:35.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:53:35 smithi083 ceph-mon[77242]: pgmap v11559: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:36.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:53:35 smithi153 ceph-mon[70044]: pgmap v11559: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:38.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:53:37 smithi153 ceph-mon[70044]: pgmap v11560: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:38.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:53:37 smithi083 ceph-mon[77242]: pgmap v11560: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:38.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:53:37 smithi083 ceph-mon[79930]: pgmap v11560: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:53:39 smithi153 ceph-mon[70044]: pgmap v11561: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:53:39 smithi083 ceph-mon[79930]: pgmap v11561: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:53:39 smithi083 ceph-mon[77242]: pgmap v11561: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:53:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:53:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:53:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:53:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:53:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:53:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:53:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:53:41.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:53:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:53:41.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:53:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:53:41.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:53:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:53:41.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:53:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:53:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:53:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:53:41.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:53:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:53:41.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:53:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:53:41.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:53:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:53:41.781 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:53:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:53:41] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T11:53:41.781 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:53:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:53:41.586Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:53:42.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:53:41 smithi153 ceph-mon[70044]: pgmap v11562: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:42.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:53:41 smithi083 ceph-mon[79930]: pgmap v11562: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:42.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:53:41 smithi083 ceph-mon[77242]: pgmap v11562: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:53:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:53:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:53:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:53:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:53:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:53:44.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:53:43 smithi153 ceph-mon[70044]: pgmap v11563: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:44.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:53:43 smithi083 ceph-mon[79930]: pgmap v11563: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:44.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:53:43 smithi083 ceph-mon[77242]: pgmap v11563: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:45.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:53:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:53:44.960Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:53:45.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:53:45 smithi083 ceph-mon[79930]: pgmap v11564: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:45.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:53:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:53:45.684Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:53:45.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:53:45 smithi083 ceph-mon[77242]: pgmap v11564: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:46.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:53:45 smithi153 ceph-mon[70044]: pgmap v11564: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:48.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:53:47 smithi153 ceph-mon[70044]: pgmap v11565: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:48.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:53:47 smithi083 ceph-mon[79930]: pgmap v11565: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:48.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:53:47 smithi083 ceph-mon[77242]: pgmap v11565: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:50.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:53:49 smithi153 ceph-mon[70044]: pgmap v11566: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:50.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:53:49 smithi083 ceph-mon[79930]: pgmap v11566: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:50.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:53:49 smithi083 ceph-mon[77242]: pgmap v11566: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:51.795 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:53:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:53:51] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T11:53:51.796 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:53:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:53:51.586Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:53:52.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:53:51 smithi153 ceph-mon[70044]: pgmap v11567: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:52.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:53:51 smithi083 ceph-mon[77242]: pgmap v11567: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:52.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:53:51 smithi083 ceph-mon[79930]: pgmap v11567: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:53:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:53:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:53:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:53:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:53:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:53:54.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:53:53 smithi153 ceph-mon[70044]: pgmap v11568: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:54.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:53:53 smithi083 ceph-mon[79930]: pgmap v11568: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:54.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:53:53 smithi083 ceph-mon[77242]: pgmap v11568: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:55.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:53:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:53:54.960Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:53:55.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:53:55 smithi083 ceph-mon[79930]: pgmap v11569: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:55.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:53:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:53:55.685Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:53:55.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:53:55 smithi083 ceph-mon[77242]: pgmap v11569: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:56.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:53:55 smithi153 ceph-mon[70044]: pgmap v11569: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:57.253 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:53:57.540 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:53:57.540 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 33s ago 6h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:53:57.541 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 8m ago 6h 64.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:53:57.541 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 33s ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:53:57.541 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 8m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:53:57.541 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 8m ago 6h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:53:57.541 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 33s ago 6h 839M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:53:57.542 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 33s ago 6h 1452M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:53:57.542 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 8m ago 6h 610M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:53:57.542 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 33s ago 6h 838M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:53:57.542 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 33s ago 6h 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:53:57.542 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 8m ago 6h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:53:57.543 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 33s ago 6h 1047M 2402M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:53:57.543 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 33s ago 6h 1036M 2402M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:53:57.543 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 33s ago 6h 788M 2402M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:53:57.543 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 33s ago 6h 989M 2402M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:53:57.543 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 8m ago 6h 848M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:53:57.544 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 8m ago 6h 1085M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:53:57.544 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 8m ago 6h 947M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:53:57.544 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 8m ago 6h 1148M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:53:57.544 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 8m ago 6h 156M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:53:57.898 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:53:57.899 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:53:57.899 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:53:57.899 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:53:57.899 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:53:57.900 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:53:57.900 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:53:57.900 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:53:57.900 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:53:57.901 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:53:57.901 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:53:57.901 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:53:57.901 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:53:57.902 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:53:57.902 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:53:57.902 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:53:57.902 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:53:57.903 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:53:58.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:53:57 smithi153 ceph-mon[70044]: pgmap v11570: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:58.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:53:57 smithi153 ceph-mon[70044]: from='client.57517 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:53:58.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:53:57 smithi083 ceph-mon[77242]: pgmap v11570: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:58.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:53:57 smithi083 ceph-mon[77242]: from='client.57517 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:53:58.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:53:57 smithi083 ceph-mon[79930]: pgmap v11570: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:53:58.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:53:57 smithi083 ceph-mon[79930]: from='client.57517 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:53:59.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:53:58 smithi083 ceph-mon[79930]: from='client.57523 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:53:59.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:53:58 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/396935302' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:53:59.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:53:58 smithi083 ceph-mon[77242]: from='client.57523 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:53:59.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:53:58 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/396935302' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:53:59.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:53:58 smithi153 ceph-mon[70044]: from='client.57523 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:53:59.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:53:58 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/396935302' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:54:00.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:53:59 smithi083 ceph-mon[79930]: pgmap v11571: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:00.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:53:59 smithi083 ceph-mon[77242]: pgmap v11571: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:53:59 smithi153 ceph-mon[70044]: pgmap v11571: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:01.819 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:54:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:54:01] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T11:54:01.819 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:54:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:54:01.587Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:54:02.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:54:01 smithi083 ceph-mon[79930]: pgmap v11572: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:02.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:54:01 smithi083 ceph-mon[77242]: pgmap v11572: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:02.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:54:01 smithi153 ceph-mon[70044]: pgmap v11572: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:54:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:54:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:54:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:54:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:54:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:54:04.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:54:03 smithi083 ceph-mon[77242]: pgmap v11573: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:04.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:54:03 smithi083 ceph-mon[79930]: pgmap v11573: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:04.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:54:03 smithi153 ceph-mon[70044]: pgmap v11573: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:05.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:54:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:54:04.961Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:54:05.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:54:05 smithi083 ceph-mon[79930]: pgmap v11574: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:05.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:54:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:54:05.686Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:54:05.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:54:05 smithi083 ceph-mon[77242]: pgmap v11574: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:06.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:54:05 smithi153 ceph-mon[70044]: pgmap v11574: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:08.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:54:07 smithi083 ceph-mon[79930]: pgmap v11575: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:08.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:54:07 smithi083 ceph-mon[77242]: pgmap v11575: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:08.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:54:07 smithi153 ceph-mon[70044]: pgmap v11575: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:10.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:54:09 smithi083 ceph-mon[79930]: pgmap v11576: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:10.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:54:09 smithi083 ceph-mon[77242]: pgmap v11576: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:10.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:54:09 smithi153 ceph-mon[70044]: pgmap v11576: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:11.835 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:54:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:54:11] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T11:54:11.835 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:54:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:54:11.587Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:54:12.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:54:11 smithi083 ceph-mon[79930]: pgmap v11577: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:12.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:54:11 smithi083 ceph-mon[77242]: pgmap v11577: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:12.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:54:11 smithi153 ceph-mon[70044]: pgmap v11577: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:54:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:54:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:54:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:54:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:54:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:54:14.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:54:13 smithi083 ceph-mon[79930]: pgmap v11578: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:14.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:54:13 smithi083 ceph-mon[77242]: pgmap v11578: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:14.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:54:13 smithi153 ceph-mon[70044]: pgmap v11578: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:15.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:54:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:54:14.961Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:54:15.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:54:15 smithi083 ceph-mon[79930]: pgmap v11579: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:15.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:54:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:54:15.687Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:54:15.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:54:15 smithi083 ceph-mon[77242]: pgmap v11579: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:16.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:54:15 smithi153 ceph-mon[70044]: pgmap v11579: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:18.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:54:17 smithi083 ceph-mon[77242]: pgmap v11580: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:18.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:54:17 smithi083 ceph-mon[79930]: pgmap v11580: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:18.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:54:17 smithi153 ceph-mon[70044]: pgmap v11580: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:20.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:54:19 smithi083 ceph-mon[79930]: pgmap v11581: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:20.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:54:19 smithi083 ceph-mon[77242]: pgmap v11581: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:54:19 smithi153 ceph-mon[70044]: pgmap v11581: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:21.849 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:54:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:54:21] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T11:54:21.850 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:54:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:54:21.588Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:54:22.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:54:21 smithi083 ceph-mon[79930]: pgmap v11582: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:22.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:54:21 smithi083 ceph-mon[77242]: pgmap v11582: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:22.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:54:21 smithi153 ceph-mon[70044]: pgmap v11582: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:54:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:54:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:54:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:54:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:54:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:54:24.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:54:23 smithi083 ceph-mon[79930]: pgmap v11583: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:24.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:54:23 smithi083 ceph-mon[77242]: pgmap v11583: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:24.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:54:23 smithi153 ceph-mon[70044]: pgmap v11583: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:25.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:54:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:54:24.962Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:54:25.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:54:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:54:25.695Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:54:25.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:54:25 smithi083 ceph-mon[77242]: pgmap v11584: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:25.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:54:25 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:54:25.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:54:25 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:54:25.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:54:25 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:54:25.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:54:25 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:54:25.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:54:25 smithi083 ceph-mon[79930]: pgmap v11584: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:25.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:54:25 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:54:25.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:54:25 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:54:25.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:54:25 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:54:25.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:54:25 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:54:26.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:54:25 smithi153 ceph-mon[70044]: pgmap v11584: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:26.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:54:25 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:54:26.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:54:25 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:54:26.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:54:25 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:54:26.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:54:25 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:54:28.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:54:27 smithi083 ceph-mon[79930]: pgmap v11585: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:28.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:54:27 smithi083 ceph-mon[77242]: pgmap v11585: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:28.225 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:54:28.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:54:27 smithi153 ceph-mon[70044]: pgmap v11585: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:28.513 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:54:28.514 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 64s ago 6h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:54:28.514 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 9m ago 6h 64.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:54:28.514 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 64s ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:54:28.515 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 9m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:54:28.515 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 9m ago 6h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:54:28.515 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 64s ago 6h 839M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:54:28.515 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 64s ago 6h 1452M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:54:28.516 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 9m ago 6h 610M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:54:28.516 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 64s ago 6h 838M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:54:28.516 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 64s ago 6h 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:54:28.516 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 9m ago 6h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:54:28.516 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 64s ago 6h 1047M 2402M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:54:28.517 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 64s ago 6h 1036M 2402M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:54:28.517 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 64s ago 6h 788M 2402M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:54:28.517 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 64s ago 6h 989M 2402M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:54:28.517 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 9m ago 6h 848M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:54:28.518 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 9m ago 6h 1085M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:54:28.518 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 9m ago 6h 947M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:54:28.518 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 9m ago 6h 1148M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:54:28.518 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 9m ago 6h 156M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:54:28.874 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:54:28.874 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:54:28.875 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:54:28.875 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:54:28.875 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:54:28.875 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:54:28.876 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:54:28.876 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:54:28.876 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:54:28.876 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:54:28.876 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:54:28.877 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:54:28.877 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:54:28.877 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:54:28.877 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:54:28.878 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:54:28.878 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:54:28.878 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:54:29.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:54:28 smithi083 ceph-mon[79930]: from='client.47604 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:54:29.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:54:28 smithi083 ceph-mon[77242]: from='client.47604 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:54:29.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:54:28 smithi153 ceph-mon[70044]: from='client.47604 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:54:30.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:54:29 smithi083 ceph-mon[79930]: from='client.47610 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:54:30.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:54:29 smithi083 ceph-mon[79930]: pgmap v11586: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:30.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:54:29 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3846661797' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:54:30.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:54:29 smithi083 ceph-mon[77242]: from='client.47610 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:54:30.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:54:29 smithi083 ceph-mon[77242]: pgmap v11586: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:30.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:54:29 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3846661797' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:54:30.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:54:29 smithi153 ceph-mon[70044]: from='client.47610 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:54:30.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:54:29 smithi153 ceph-mon[70044]: pgmap v11586: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:30.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:54:29 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3846661797' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:54:31.865 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:54:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:54:31] "GET /metrics HTTP/1.1" 200 225227 "" "Prometheus/2.33.4" 2022-09-29T11:54:31.866 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:54:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:54:31.588Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:54:32.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:54:31 smithi083 ceph-mon[77242]: pgmap v11587: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:32.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:54:31 smithi083 ceph-mon[79930]: pgmap v11587: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:32.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:54:31 smithi153 ceph-mon[70044]: pgmap v11587: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:54:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:54:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:54:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:54:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:54:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:54:34.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:54:33 smithi083 ceph-mon[79930]: pgmap v11588: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:34.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:54:33 smithi083 ceph-mon[77242]: pgmap v11588: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:34.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:54:33 smithi153 ceph-mon[70044]: pgmap v11588: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:35.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:54:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:54:34.963Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:54:35.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:54:35 smithi083 ceph-mon[79930]: pgmap v11589: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:35.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:54:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:54:35.688Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:54:35.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:54:35 smithi083 ceph-mon[77242]: pgmap v11589: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:36.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:54:35 smithi153 ceph-mon[70044]: pgmap v11589: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:38.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:54:37 smithi083 ceph-mon[79930]: pgmap v11590: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:38.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:54:37 smithi083 ceph-mon[77242]: pgmap v11590: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:38.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:54:37 smithi153 ceph-mon[70044]: pgmap v11590: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:54:39 smithi083 ceph-mon[79930]: pgmap v11591: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:54:39 smithi083 ceph-mon[77242]: pgmap v11591: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:54:39 smithi153 ceph-mon[70044]: pgmap v11591: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:41.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:54:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:54:41.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:54:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:54:41.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:54:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:54:41.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:54:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:54:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:54:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:54:41.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:54:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:54:41.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:54:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:54:41.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:54:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:54:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:54:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:54:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:54:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:54:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:54:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:54:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:54:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:54:41.882 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:54:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:54:41] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T11:54:41.882 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:54:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:54:41.589Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:54:42.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:54:41 smithi083 ceph-mon[79930]: pgmap v11592: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:42.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:54:41 smithi083 ceph-mon[77242]: pgmap v11592: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:54:41 smithi153 ceph-mon[70044]: pgmap v11592: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:54:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:54:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:54:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:54:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:54:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:54:44.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:54:43 smithi083 ceph-mon[79930]: pgmap v11593: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:44.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:54:43 smithi083 ceph-mon[77242]: pgmap v11593: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:44.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:54:43 smithi153 ceph-mon[70044]: pgmap v11593: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:45.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:54:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:54:44.964Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:54:45.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:54:45 smithi083 ceph-mon[77242]: pgmap v11594: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:45.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:54:45 smithi083 ceph-mon[79930]: pgmap v11594: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:45.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:54:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:54:45.689Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:54:46.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:54:45 smithi153 ceph-mon[70044]: pgmap v11594: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:48.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:54:47 smithi083 ceph-mon[79930]: pgmap v11595: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:48.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:54:47 smithi083 ceph-mon[77242]: pgmap v11595: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:48.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:54:47 smithi153 ceph-mon[70044]: pgmap v11595: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:50.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:54:49 smithi083 ceph-mon[79930]: pgmap v11596: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:50.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:54:49 smithi083 ceph-mon[77242]: pgmap v11596: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:50.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:54:49 smithi153 ceph-mon[70044]: pgmap v11596: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:51.906 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:54:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:54:51] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T11:54:51.906 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:54:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:54:51.589Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:54:52.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:54:51 smithi083 ceph-mon[77242]: pgmap v11597: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:52.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:54:51 smithi083 ceph-mon[79930]: pgmap v11597: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:52.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:54:51 smithi153 ceph-mon[70044]: pgmap v11597: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:54:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:54:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:54:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:54:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:54:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:54:54.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:54:53 smithi083 ceph-mon[79930]: pgmap v11598: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:54.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:54:53 smithi083 ceph-mon[77242]: pgmap v11598: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:54.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:54:53 smithi153 ceph-mon[70044]: pgmap v11598: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:54:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:54:54.964Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:54:55.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:54:55 smithi083 ceph-mon[79930]: pgmap v11599: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:55.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:54:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:54:55.689Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:54:55.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:54:55 smithi083 ceph-mon[77242]: pgmap v11599: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:56.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:54:55 smithi153 ceph-mon[70044]: pgmap v11599: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:58.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:54:57 smithi083 ceph-mon[79930]: pgmap v11600: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:58.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:54:57 smithi083 ceph-mon[77242]: pgmap v11600: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:58.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:54:57 smithi153 ceph-mon[70044]: pgmap v11600: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:54:59.202 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:54:59.490 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:54:59.491 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 94s ago 6h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:54:59.491 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 9m ago 6h 64.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:54:59.491 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 94s ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:54:59.492 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 9m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:54:59.492 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 9m ago 6h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:54:59.492 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 94s ago 6h 839M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:54:59.492 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 94s ago 6h 1452M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:54:59.492 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 9m ago 6h 610M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:54:59.493 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 94s ago 6h 838M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:54:59.493 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 94s ago 6h 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:54:59.493 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 9m ago 6h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:54:59.493 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 94s ago 6h 1047M 2402M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:54:59.494 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 94s ago 6h 1036M 2402M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:54:59.494 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 94s ago 6h 788M 2402M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:54:59.494 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 94s ago 6h 989M 2402M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:54:59.494 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 9m ago 6h 848M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:54:59.495 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 9m ago 6h 1085M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:54:59.495 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 9m ago 6h 947M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:54:59.495 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 9m ago 6h 1148M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:54:59.495 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 9m ago 6h 156M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:54:59.842 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:54:59.842 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:54:59.843 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:54:59.843 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:54:59.843 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:54:59.843 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:54:59.844 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:54:59.844 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:54:59.844 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:54:59.844 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:54:59.845 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:54:59.845 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:54:59.845 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:54:59.845 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:54:59.845 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:54:59.846 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:54:59.846 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:54:59.846 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:55:00.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:54:59 smithi083 ceph-mon[77242]: pgmap v11601: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:55:00.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:54:59 smithi083 ceph-mon[77242]: from='client.57553 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:55:00.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:54:59 smithi083 ceph-mon[77242]: from='client.47628 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:55:00.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:54:59 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3500004050' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:55:00.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:54:59 smithi083 ceph-mon[79930]: pgmap v11601: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:55:00.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:54:59 smithi083 ceph-mon[79930]: from='client.57553 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:55:00.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:54:59 smithi083 ceph-mon[79930]: from='client.47628 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:55:00.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:54:59 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3500004050' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:55:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:54:59 smithi153 ceph-mon[70044]: pgmap v11601: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:55:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:54:59 smithi153 ceph-mon[70044]: from='client.57553 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:55:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:54:59 smithi153 ceph-mon[70044]: from='client.47628 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:55:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:54:59 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3500004050' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:55:01.919 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:55:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:55:01] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T11:55:01.920 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:55:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:55:01.590Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:55:02.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:55:01 smithi083 ceph-mon[77242]: pgmap v11602: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:55:02.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:55:01 smithi083 ceph-mon[79930]: pgmap v11602: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:55:02.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:55:01 smithi153 ceph-mon[70044]: pgmap v11602: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:55:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:55:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:55:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:55:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:55:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:55:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:55:04.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:55:03 smithi083 ceph-mon[77242]: pgmap v11603: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:55:04.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:55:03 smithi083 ceph-mon[79930]: pgmap v11603: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:55:04.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:55:03 smithi153 ceph-mon[70044]: pgmap v11603: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:55:05.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:55:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:55:04.965Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:55:05.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:55:05 smithi083 ceph-mon[79930]: pgmap v11604: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:55:05.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:55:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:55:05.690Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:55:05.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:55:05 smithi083 ceph-mon[77242]: pgmap v11604: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:55:06.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:55:05 smithi153 ceph-mon[70044]: pgmap v11604: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:55:08.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:55:07 smithi083 ceph-mon[79930]: pgmap v11605: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:55:08.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:55:07 smithi083 ceph-mon[77242]: pgmap v11605: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:55:08.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:55:07 smithi153 ceph-mon[70044]: pgmap v11605: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:55:10.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:55:09 smithi083 ceph-mon[79930]: pgmap v11606: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:55:10.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:55:09 smithi083 ceph-mon[77242]: pgmap v11606: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:55:10.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:55:09 smithi153 ceph-mon[70044]: pgmap v11606: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:55:11.934 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:55:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:55:11] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T11:55:11.935 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:55:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:55:11.592Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:55:12.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:55:11 smithi083 ceph-mon[79930]: pgmap v11607: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:55:12.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:55:11 smithi083 ceph-mon[77242]: pgmap v11607: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:55:12.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:55:11 smithi153 ceph-mon[70044]: pgmap v11607: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:55:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:55:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:55:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:55:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:55:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:55:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:55:14.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:55:13 smithi083 ceph-mon[77242]: pgmap v11608: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:55:14.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:55:13 smithi083 ceph-mon[79930]: pgmap v11608: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:55:14.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:55:13 smithi153 ceph-mon[70044]: pgmap v11608: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:55:15.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:55:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:55:14.966Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:55:15.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:55:15 smithi083 ceph-mon[79930]: pgmap v11609: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:55:15.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:55:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:55:15.690Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:55:15.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:55:15 smithi083 ceph-mon[77242]: pgmap v11609: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:55:16.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:55:15 smithi153 ceph-mon[70044]: pgmap v11609: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:55:18.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:55:17 smithi083 ceph-mon[79930]: pgmap v11610: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:55:18.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:55:17 smithi083 ceph-mon[77242]: pgmap v11610: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:55:18.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:55:17 smithi153 ceph-mon[70044]: pgmap v11610: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:55:20.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:55:19 smithi083 ceph-mon[79930]: pgmap v11611: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:55:20.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:55:19 smithi083 ceph-mon[77242]: pgmap v11611: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:55:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:55:19 smithi153 ceph-mon[70044]: pgmap v11611: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:55:21.954 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:55:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:55:21] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T11:55:21.955 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:55:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:55:21.592Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:55:22.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:55:21 smithi083 ceph-mon[77242]: pgmap v11612: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:55:22.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:55:21 smithi083 ceph-mon[79930]: pgmap v11612: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:55:22.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:55:21 smithi153 ceph-mon[70044]: pgmap v11612: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:55:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:55:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:55:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:55:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:55:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:55:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:55:24.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:55:23 smithi083 ceph-mon[79930]: pgmap v11613: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:55:24.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:55:23 smithi083 ceph-mon[77242]: pgmap v11613: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:55:24.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:55:23 smithi153 ceph-mon[70044]: pgmap v11613: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:55:25.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:55:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:55:24.966Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:55:25.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:55:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:55:25.691Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:55:26.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:55:25 smithi083 ceph-mon[79930]: pgmap v11614: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:55:26.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:55:25 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:55:26.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:55:25 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:55:26.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:55:25 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:55:26.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:55:25 smithi083 ceph-mon[77242]: pgmap v11614: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:55:26.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:55:25 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:55:26.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:55:25 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:55:26.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:55:25 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:55:26.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:55:25 smithi153 ceph-mon[70044]: pgmap v11614: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:55:26.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:55:25 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:55:26.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:55:25 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:55:26.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:55:25 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:55:28.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:55:27 smithi153 ceph-mon[70044]: pgmap v11615: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:55:28.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:55:27 smithi083 ceph-mon[79930]: pgmap v11615: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:55:28.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:55:27 smithi083 ceph-mon[77242]: pgmap v11615: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:55:30.171 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:55:30.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:55:29 smithi153 ceph-mon[70044]: pgmap v11616: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:55:30.458 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:55:30.458 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 2m ago 6h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:55:30.459 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 10m ago 6h 64.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:55:30.459 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 2m ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:55:30.459 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 10m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:55:30.459 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 10m ago 6h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:55:30.460 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 2m ago 6h 839M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:55:30.460 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 2m ago 6h 1452M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:55:30.460 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 10m ago 6h 610M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:55:30.460 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 2m ago 6h 838M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:55:30.460 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 2m ago 6h 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:55:30.461 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 10m ago 6h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:55:30.461 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 2m ago 6h 1047M 2402M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:55:30.461 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 2m ago 6h 1036M 2402M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:55:30.461 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 2m ago 6h 788M 2402M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:55:30.462 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 2m ago 6h 989M 2402M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:55:30.462 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 10m ago 6h 848M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:55:30.462 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 10m ago 6h 1085M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:55:30.463 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 10m ago 6h 947M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:55:30.463 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 10m ago 6h 1148M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:55:30.463 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 10m ago 6h 156M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:55:30.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:55:29 smithi083 ceph-mon[79930]: pgmap v11616: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:55:30.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:55:29 smithi083 ceph-mon[77242]: pgmap v11616: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:55:30.815 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:55:30.816 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:55:30.816 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:55:30.816 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:55:30.816 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:55:30.817 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:55:30.817 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:55:30.817 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:55:30.817 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:55:30.817 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:55:30.818 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:55:30.818 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:55:30.818 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:55:30.819 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:55:30.819 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:55:30.819 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:55:30.819 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:55:30.819 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:55:31.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:55:30 smithi153 ceph-mon[70044]: from='client.47640 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:55:31.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:55:30 smithi153 ceph-mon[70044]: from='client.47646 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:55:31.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:55:30 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:55:31.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:55:30 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:55:31.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:55:30 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2885747272' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:55:31.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:55:30 smithi083 ceph-mon[79930]: from='client.47640 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:55:31.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:55:30 smithi083 ceph-mon[79930]: from='client.47646 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:55:31.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:55:30 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:55:31.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:55:30 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:55:31.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:55:30 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2885747272' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:55:31.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:55:30 smithi083 ceph-mon[77242]: from='client.47640 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:55:31.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:55:30 smithi083 ceph-mon[77242]: from='client.47646 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:55:31.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:55:30 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:55:31.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:55:30 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:55:31.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:55:30 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2885747272' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:55:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:55:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:55:31] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T11:55:31.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:55:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:55:31.592Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:55:32.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:55:32 smithi083 ceph-mon[79930]: pgmap v11617: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:55:32.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:55:32 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:55:32.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:55:32 smithi083 ceph-mon[77242]: pgmap v11617: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:55:32.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:55:32 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:55:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:55:32 smithi153 ceph-mon[70044]: pgmap v11617: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:55:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:55:32 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:55:32.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:55:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:55:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:55:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:55:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:55:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:55:34.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:55:34 smithi083 ceph-mon[79930]: pgmap v11618: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:55:34.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:55:34 smithi083 ceph-mon[77242]: pgmap v11618: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:55:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:55:34 smithi153 ceph-mon[70044]: pgmap v11618: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:55:35.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:55:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:55:34.967Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:55:35.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:55:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:55:35.692Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:55:36.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:55:36 smithi083 ceph-mon[79930]: pgmap v11619: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:55:36.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:55:36 smithi083 ceph-mon[77242]: pgmap v11619: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:55:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:55:36 smithi153 ceph-mon[70044]: pgmap v11619: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:55:38.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:55:38 smithi083 ceph-mon[79930]: pgmap v11620: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:55:38.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:55:38 smithi083 ceph-mon[77242]: pgmap v11620: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:55:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:55:38 smithi153 ceph-mon[70044]: pgmap v11620: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:55:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:55:40 smithi083 ceph-mon[79930]: pgmap v11621: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:55:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:55:40 smithi083 ceph-mon[77242]: pgmap v11621: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:55:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:55:40 smithi153 ceph-mon[70044]: pgmap v11621: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:55:41.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:55:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:55:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:55:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:55:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:55:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:55:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:55:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:55:41.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:55:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:55:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:55:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:55:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:55:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:55:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:55:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:55:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:55:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:55:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:55:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:55:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:55:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:55:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:55:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:55:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:55:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:55:41] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T11:55:41.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:55:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:55:41.593Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:55:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:55:42 smithi083 ceph-mon[79930]: pgmap v11622: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:55:42.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:55:42 smithi083 ceph-mon[77242]: pgmap v11622: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:55:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:55:42 smithi153 ceph-mon[70044]: pgmap v11622: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:55:42.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:55:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:55:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:55:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:55:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:55:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:55:44.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:55:44 smithi083 ceph-mon[79930]: pgmap v11623: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:55:44.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:55:44 smithi083 ceph-mon[77242]: pgmap v11623: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:55:44.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:55:44 smithi153 ceph-mon[70044]: pgmap v11623: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:55:45.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:55:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:55:44.968Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:55:45.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:55:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:55:45.693Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:55:46.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:55:46 smithi083 ceph-mon[77242]: pgmap v11624: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:55:46.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:55:46 smithi083 ceph-mon[79930]: pgmap v11624: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:55:46.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:55:46 smithi153 ceph-mon[70044]: pgmap v11624: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:55:48.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:55:48 smithi083 ceph-mon[77242]: pgmap v11625: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:55:48.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:55:48 smithi083 ceph-mon[79930]: pgmap v11625: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:55:48.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:55:48 smithi153 ceph-mon[70044]: pgmap v11625: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:55:50.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:55:50 smithi083 ceph-mon[77242]: pgmap v11626: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:55:50.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:55:50 smithi083 ceph-mon[79930]: pgmap v11626: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:55:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:55:50 smithi153 ceph-mon[70044]: pgmap v11626: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:55:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:55:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:55:51] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T11:55:51.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:55:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:55:51.594Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:55:52.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:55:52 smithi083 ceph-mon[77242]: pgmap v11627: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:55:52.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:55:52 smithi083 ceph-mon[79930]: pgmap v11627: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:55:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:55:52 smithi153 ceph-mon[70044]: pgmap v11627: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:55:52.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:55:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:55:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:55:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:55:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:55:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:55:54.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:55:54 smithi083 ceph-mon[79930]: pgmap v11628: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:55:54.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:55:54 smithi083 ceph-mon[77242]: pgmap v11628: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:55:54.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:55:54 smithi153 ceph-mon[70044]: pgmap v11628: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:55:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:55:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:55:54.968Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:55:55.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:55:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:55:55.693Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:55:56.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:55:56 smithi083 ceph-mon[79930]: pgmap v11629: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:55:56.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:55:56 smithi083 ceph-mon[77242]: pgmap v11629: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:55:56.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:55:56 smithi153 ceph-mon[70044]: pgmap v11629: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:55:58.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:55:58 smithi083 ceph-mon[79930]: pgmap v11630: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:55:58.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:55:58 smithi083 ceph-mon[77242]: pgmap v11630: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:55:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:55:58 smithi153 ceph-mon[70044]: pgmap v11630: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:00.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:56:00 smithi083 ceph-mon[79930]: pgmap v11631: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:00.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:56:00 smithi083 ceph-mon[77242]: pgmap v11631: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:56:00 smithi153 ceph-mon[70044]: pgmap v11631: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:01.141 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:56:01.429 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:56:01.430 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 2m ago 6h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:56:01.430 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 30s ago 6h 62.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:56:01.430 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 2m ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:56:01.430 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 30s ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:56:01.431 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 30s ago 6h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:56:01.431 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 2m ago 6h 839M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:56:01.431 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 2m ago 6h 1452M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:56:01.431 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 30s ago 6h 625M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:56:01.432 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 2m ago 6h 838M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:56:01.432 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 2m ago 6h 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:56:01.432 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 30s ago 6h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:56:01.432 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 2m ago 6h 1047M 2402M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:56:01.433 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 2m ago 6h 1036M 2402M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:56:01.433 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 2m ago 6h 788M 2402M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:56:01.433 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 2m ago 6h 989M 2402M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:56:01.433 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 30s ago 6h 870M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:56:01.433 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 30s ago 6h 1113M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:56:01.434 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 30s ago 6h 971M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:56:01.434 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 30s ago 6h 1179M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:56:01.434 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 30s ago 6h 159M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:56:01.785 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:56:01.785 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:56:01.786 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:56:01.786 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:56:01.786 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:56:01.787 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:56:01.787 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:56:01.787 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:56:01.787 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:56:01.788 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:56:01.788 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:56:01.788 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:56:01.788 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:56:01.788 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:56:01.789 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:56:01.789 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:56:01.789 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:56:01.789 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:56:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:56:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:56:01] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T11:56:01.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:56:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:56:01.593Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:56:02.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:56:02 smithi083 ceph-mon[77242]: pgmap v11632: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:02.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:56:02 smithi083 ceph-mon[77242]: from='client.57589 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:56:02.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:56:02 smithi083 ceph-mon[77242]: from='client.47664 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:56:02.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:56:02 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1098197758' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:56:02.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:56:02 smithi083 ceph-mon[79930]: pgmap v11632: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:02.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:56:02 smithi083 ceph-mon[79930]: from='client.57589 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:56:02.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:56:02 smithi083 ceph-mon[79930]: from='client.47664 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:56:02.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:56:02 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1098197758' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:56:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:56:02 smithi153 ceph-mon[70044]: pgmap v11632: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:56:02 smithi153 ceph-mon[70044]: from='client.57589 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:56:02.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:56:02 smithi153 ceph-mon[70044]: from='client.47664 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:56:02.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:56:02 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1098197758' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:56:02.559 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:56:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:56:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:56:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:56:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:56:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:56:04.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:56:04 smithi083 ceph-mon[79930]: pgmap v11633: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:04.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:56:04 smithi083 ceph-mon[77242]: pgmap v11633: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:56:04 smithi153 ceph-mon[70044]: pgmap v11633: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:05.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:56:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:56:04.968Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:56:05.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:56:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:56:05.694Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:56:06.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:56:06 smithi083 ceph-mon[77242]: pgmap v11634: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:06.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:56:06 smithi083 ceph-mon[79930]: pgmap v11634: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:06.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:56:06 smithi153 ceph-mon[70044]: pgmap v11634: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:08.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:56:08 smithi083 ceph-mon[79930]: pgmap v11635: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:08.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:56:08 smithi083 ceph-mon[77242]: pgmap v11635: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:56:08 smithi153 ceph-mon[70044]: pgmap v11635: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:10.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:56:10 smithi083 ceph-mon[79930]: pgmap v11636: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:10.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:56:10 smithi083 ceph-mon[77242]: pgmap v11636: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:56:10 smithi153 ceph-mon[70044]: pgmap v11636: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:56:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:56:11] "GET /metrics HTTP/1.1" 200 225228 "" "Prometheus/2.33.4" 2022-09-29T11:56:11.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:56:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:56:11.594Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:56:12.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:56:12 smithi083 ceph-mon[79930]: pgmap v11637: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:12.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:56:12 smithi083 ceph-mon[77242]: pgmap v11637: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:56:12 smithi153 ceph-mon[70044]: pgmap v11637: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:12.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:56:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:56:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:56:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:56:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:56:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:56:14.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:56:14 smithi083 ceph-mon[79930]: pgmap v11638: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:14.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:56:14 smithi083 ceph-mon[77242]: pgmap v11638: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:14.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:56:14 smithi153 ceph-mon[70044]: pgmap v11638: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:15.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:56:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:56:14.969Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:56:15.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:56:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:56:15.694Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:56:16.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:56:16 smithi083 ceph-mon[79930]: pgmap v11639: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:16.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:56:16 smithi083 ceph-mon[77242]: pgmap v11639: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:56:16 smithi153 ceph-mon[70044]: pgmap v11639: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:18.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:56:18 smithi083 ceph-mon[77242]: pgmap v11640: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:18.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:56:18 smithi083 ceph-mon[79930]: pgmap v11640: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:56:18 smithi153 ceph-mon[70044]: pgmap v11640: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:20.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:56:20 smithi083 ceph-mon[79930]: pgmap v11641: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:20.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:56:20 smithi083 ceph-mon[77242]: pgmap v11641: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:56:20 smithi153 ceph-mon[70044]: pgmap v11641: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:56:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:56:21] "GET /metrics HTTP/1.1" 200 225228 "" "Prometheus/2.33.4" 2022-09-29T11:56:21.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:56:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:56:21.595Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:56:22.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:56:22 smithi083 ceph-mon[77242]: pgmap v11642: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:22.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:56:22 smithi083 ceph-mon[79930]: pgmap v11642: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:56:22 smithi153 ceph-mon[70044]: pgmap v11642: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:22.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:56:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:56:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:56:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:56:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:56:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:56:24.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:56:24 smithi083 ceph-mon[77242]: pgmap v11643: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:24.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:56:24 smithi083 ceph-mon[79930]: pgmap v11643: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:56:24 smithi153 ceph-mon[70044]: pgmap v11643: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:25.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:56:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:56:24.970Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:56:25.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:56:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:56:25.695Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:56:26.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:56:26 smithi083 ceph-mon[79930]: pgmap v11644: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:26.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:56:26 smithi083 ceph-mon[77242]: pgmap v11644: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:56:26 smithi153 ceph-mon[70044]: pgmap v11644: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:28.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:56:28 smithi083 ceph-mon[79930]: pgmap v11645: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:28.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:56:28 smithi083 ceph-mon[77242]: pgmap v11645: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:56:28 smithi153 ceph-mon[70044]: pgmap v11645: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:30.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:56:30 smithi083 ceph-mon[79930]: pgmap v11646: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:30.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:56:30 smithi083 ceph-mon[77242]: pgmap v11646: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:56:30 smithi153 ceph-mon[70044]: pgmap v11646: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:31.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:56:31 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:56:31.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:56:31 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:56:31.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:56:31 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:56:31.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:56:31 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:56:31.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:56:31 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:56:31.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:56:31 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:56:31.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:56:31 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:56:31.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:56:31 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:56:31.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:56:31 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:56:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:56:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:56:31] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T11:56:31.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:56:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:56:31.595Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:56:32.098 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:56:32.388 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:56:32.388 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 3m ago 6h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:56:32.388 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 61s ago 6h 62.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:56:32.388 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 3m ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:56:32.389 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 61s ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:56:32.389 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 61s ago 6h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:56:32.389 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 3m ago 6h 839M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:56:32.389 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 3m ago 6h 1452M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:56:32.390 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 61s ago 6h 625M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:56:32.390 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 3m ago 6h 838M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:56:32.390 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 3m ago 6h 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:56:32.390 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 61s ago 6h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:56:32.391 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 3m ago 6h 1047M 2402M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:56:32.391 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 3m ago 6h 1036M 2402M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:56:32.391 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 3m ago 6h 788M 2402M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:56:32.391 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 3m ago 6h 989M 2402M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:56:32.391 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 61s ago 6h 870M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:56:32.392 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 61s ago 6h 1113M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:56:32.392 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 61s ago 6h 971M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:56:32.392 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 61s ago 6h 1179M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:56:32.392 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 61s ago 6h 159M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:56:32.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:56:32 smithi083 ceph-mon[79930]: pgmap v11647: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:32.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:56:32 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:56:32.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:56:32 smithi083 ceph-mon[77242]: pgmap v11647: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:32.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:56:32 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:56:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:56:32 smithi153 ceph-mon[70044]: pgmap v11647: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:56:32 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:56:32.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:56:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:56:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:56:32.741 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:56:32.741 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:56:32.741 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:56:32.742 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:56:32.742 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:56:32.742 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:56:32.742 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:56:32.743 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:56:32.743 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:56:32.743 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:56:32.743 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:56:32.744 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:56:32.744 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:56:32.744 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:56:32.744 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:56:32.745 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:56:32.745 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:56:32.745 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:56:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:56:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:56:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:56:33.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:56:33 smithi153 ceph-mon[70044]: from='client.47676 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:56:33.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:56:33 smithi153 ceph-mon[70044]: from='client.57613 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:56:33.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:56:33 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2900179742' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:56:33.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:56:33 smithi083 ceph-mon[79930]: from='client.47676 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:56:33.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:56:33 smithi083 ceph-mon[79930]: from='client.57613 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:56:33.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:56:33 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2900179742' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:56:33.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:56:33 smithi083 ceph-mon[77242]: from='client.47676 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:56:33.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:56:33 smithi083 ceph-mon[77242]: from='client.57613 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:56:33.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:56:33 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2900179742' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:56:34.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:56:34 smithi083 ceph-mon[79930]: pgmap v11648: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:34.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:56:34 smithi083 ceph-mon[77242]: pgmap v11648: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:56:34 smithi153 ceph-mon[70044]: pgmap v11648: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:35.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:56:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:56:34.970Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:56:35.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:56:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:56:35.695Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:56:36.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:56:36 smithi083 ceph-mon[79930]: pgmap v11649: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:36.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:56:36 smithi083 ceph-mon[77242]: pgmap v11649: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:56:36 smithi153 ceph-mon[70044]: pgmap v11649: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:56:38 smithi153 ceph-mon[70044]: pgmap v11650: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:38.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:56:38 smithi083 ceph-mon[79930]: pgmap v11650: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:38.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:56:38 smithi083 ceph-mon[77242]: pgmap v11650: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:56:40 smithi153 ceph-mon[70044]: pgmap v11651: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:56:40 smithi083 ceph-mon[79930]: pgmap v11651: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:56:40 smithi083 ceph-mon[77242]: pgmap v11651: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:41.507 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:56:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:56:41.507 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:56:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:56:41.507 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:56:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:56:41.508 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:56:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:56:41.508 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:56:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:56:41.508 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:56:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:56:41.509 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:56:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:56:41.509 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:56:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:56:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:56:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:56:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:56:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:56:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:56:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:56:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:56:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:56:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:56:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:56:41] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T11:56:41.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:56:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:56:41.596Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:56:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:56:42 smithi153 ceph-mon[70044]: pgmap v11652: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:42.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:56:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:56:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:56:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:56:42 smithi083 ceph-mon[79930]: pgmap v11652: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:56:42 smithi083 ceph-mon[77242]: pgmap v11652: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:56:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:56:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:56:44.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:56:44 smithi153 ceph-mon[70044]: pgmap v11653: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:44.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:56:44 smithi083 ceph-mon[79930]: pgmap v11653: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:44.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:56:44 smithi083 ceph-mon[77242]: pgmap v11653: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:45.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:56:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:56:44.971Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:56:45.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:56:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:56:45.696Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:56:46.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:56:46 smithi153 ceph-mon[70044]: pgmap v11654: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:46.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:56:46 smithi083 ceph-mon[77242]: pgmap v11654: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:46.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:56:46 smithi083 ceph-mon[79930]: pgmap v11654: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:48.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:56:48 smithi153 ceph-mon[70044]: pgmap v11655: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:48.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:56:48 smithi083 ceph-mon[77242]: pgmap v11655: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:48.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:56:48 smithi083 ceph-mon[79930]: pgmap v11655: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:56:50 smithi153 ceph-mon[70044]: pgmap v11656: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:50.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:56:50 smithi083 ceph-mon[79930]: pgmap v11656: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:50.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:56:50 smithi083 ceph-mon[77242]: pgmap v11656: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:56:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:56:51] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T11:56:51.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:56:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:56:51.597Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:56:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:56:52 smithi153 ceph-mon[70044]: pgmap v11657: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:52.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:56:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:56:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:56:52.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:56:52 smithi083 ceph-mon[79930]: pgmap v11657: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:52.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:56:52 smithi083 ceph-mon[77242]: pgmap v11657: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:56:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:56:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:56:54.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:56:54 smithi153 ceph-mon[70044]: pgmap v11658: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:54.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:56:54 smithi083 ceph-mon[79930]: pgmap v11658: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:54.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:56:54 smithi083 ceph-mon[77242]: pgmap v11658: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:56:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:56:54.972Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:56:55.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:56:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:56:55.696Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:56:56.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:56:56 smithi153 ceph-mon[70044]: pgmap v11659: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:56.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:56:56 smithi083 ceph-mon[79930]: pgmap v11659: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:56.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:56:56 smithi083 ceph-mon[77242]: pgmap v11659: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:56:58 smithi153 ceph-mon[70044]: pgmap v11660: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:58.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:56:58 smithi083 ceph-mon[77242]: pgmap v11660: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:56:58.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:56:58 smithi083 ceph-mon[79930]: pgmap v11660: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:57:00 smithi153 ceph-mon[70044]: pgmap v11661: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:00.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:57:00 smithi083 ceph-mon[79930]: pgmap v11661: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:00.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:57:00 smithi083 ceph-mon[77242]: pgmap v11661: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:57:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:57:01] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T11:57:01.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:57:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:57:01.597Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:57:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:57:02 smithi153 ceph-mon[70044]: pgmap v11662: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:02.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:57:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:57:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:57:02.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:57:02 smithi083 ceph-mon[79930]: pgmap v11662: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:02.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:57:02 smithi083 ceph-mon[77242]: pgmap v11662: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:03.064 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:57:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:57:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:57:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:57:03.354 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:57:03.354 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 3m ago 6h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:57:03.354 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 92s ago 6h 62.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:57:03.354 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 3m ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:57:03.355 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 92s ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:57:03.355 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 92s ago 6h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:57:03.355 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 3m ago 6h 839M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:57:03.355 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 3m ago 6h 1452M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:57:03.356 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 92s ago 6h 625M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:57:03.356 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 3m ago 6h 838M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:57:03.356 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 3m ago 6h 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:57:03.356 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 92s ago 6h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:57:03.356 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 3m ago 6h 1047M 2402M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:57:03.357 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 3m ago 6h 1036M 2402M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:57:03.357 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 3m ago 6h 788M 2402M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:57:03.357 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 3m ago 6h 989M 2402M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:57:03.357 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 92s ago 6h 870M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:57:03.358 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 92s ago 6h 1113M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:57:03.358 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 92s ago 6h 971M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:57:03.358 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 92s ago 6h 1179M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:57:03.358 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 92s ago 6h 159M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:57:03.723 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:57:03.723 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:57:03.723 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:57:03.724 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:57:03.724 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:57:03.724 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:57:03.724 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:57:03.724 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:57:03.725 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:57:03.725 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:57:03.725 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:57:03.725 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:57:03.726 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:57:03.726 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:57:03.726 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:57:03.726 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:57:03.726 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:57:03.727 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:57:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:57:04 smithi153 ceph-mon[70044]: pgmap v11663: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:57:04 smithi153 ceph-mon[70044]: from='client.47694 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:57:04.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:57:04 smithi153 ceph-mon[70044]: from='client.47700 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:57:04.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:57:04 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4086604417' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:57:04.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:57:04 smithi083 ceph-mon[79930]: pgmap v11663: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:04.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:57:04 smithi083 ceph-mon[79930]: from='client.47694 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:57:04.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:57:04 smithi083 ceph-mon[79930]: from='client.47700 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:57:04.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:57:04 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4086604417' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:57:04.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:57:04 smithi083 ceph-mon[77242]: pgmap v11663: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:04.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:57:04 smithi083 ceph-mon[77242]: from='client.47694 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:57:04.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:57:04 smithi083 ceph-mon[77242]: from='client.47700 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:57:04.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:57:04 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4086604417' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:57:05.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:57:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:57:04.973Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:57:05.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:57:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:57:05.697Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:57:06.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:57:06 smithi153 ceph-mon[70044]: pgmap v11664: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:06.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:57:06 smithi083 ceph-mon[79930]: pgmap v11664: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:06.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:57:06 smithi083 ceph-mon[77242]: pgmap v11664: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:57:08 smithi153 ceph-mon[70044]: pgmap v11665: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:08.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:57:08 smithi083 ceph-mon[79930]: pgmap v11665: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:08.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:57:08 smithi083 ceph-mon[77242]: pgmap v11665: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:57:10 smithi153 ceph-mon[70044]: pgmap v11666: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:10.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:57:10 smithi083 ceph-mon[79930]: pgmap v11666: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:10.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:57:10 smithi083 ceph-mon[77242]: pgmap v11666: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:57:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:57:11] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T11:57:11.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:57:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:57:11.598Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:57:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:57:12 smithi153 ceph-mon[70044]: pgmap v11667: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:12.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:57:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:57:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:57:12.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:57:12 smithi083 ceph-mon[77242]: pgmap v11667: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:12.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:57:12 smithi083 ceph-mon[79930]: pgmap v11667: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:57:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:57:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:57:14.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:57:14 smithi153 ceph-mon[70044]: pgmap v11668: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:14.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:57:14 smithi083 ceph-mon[79930]: pgmap v11668: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:14.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:57:14 smithi083 ceph-mon[77242]: pgmap v11668: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:15.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:57:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:57:14.973Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:57:15.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:57:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:57:15.697Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:57:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:57:16 smithi153 ceph-mon[70044]: pgmap v11669: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:16.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:57:16 smithi083 ceph-mon[79930]: pgmap v11669: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:16.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:57:16 smithi083 ceph-mon[77242]: pgmap v11669: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:57:18 smithi153 ceph-mon[70044]: pgmap v11670: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:18.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:57:18 smithi083 ceph-mon[77242]: pgmap v11670: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:18.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:57:18 smithi083 ceph-mon[79930]: pgmap v11670: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:57:20 smithi153 ceph-mon[70044]: pgmap v11671: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:20.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:57:20 smithi083 ceph-mon[79930]: pgmap v11671: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:20.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:57:20 smithi083 ceph-mon[77242]: pgmap v11671: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:57:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:57:21] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T11:57:21.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:57:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:57:21.599Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:57:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:57:22 smithi153 ceph-mon[70044]: pgmap v11672: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:22.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:57:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:57:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:57:22.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:57:22 smithi083 ceph-mon[77242]: pgmap v11672: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:22.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:57:22 smithi083 ceph-mon[79930]: pgmap v11672: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:57:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:57:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:57:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:57:24 smithi153 ceph-mon[70044]: pgmap v11673: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:24.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:57:24 smithi083 ceph-mon[79930]: pgmap v11673: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:24.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:57:24 smithi083 ceph-mon[77242]: pgmap v11673: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:25.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:57:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:57:24.974Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:57:25.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:57:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:57:25.698Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:57:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:57:26 smithi153 ceph-mon[70044]: pgmap v11674: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:26.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:57:26 smithi083 ceph-mon[79930]: pgmap v11674: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:26.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:57:26 smithi083 ceph-mon[77242]: pgmap v11674: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:57:28 smithi153 ceph-mon[70044]: pgmap v11675: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:28.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:57:28 smithi083 ceph-mon[79930]: pgmap v11675: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:28.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:57:28 smithi083 ceph-mon[77242]: pgmap v11675: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:57:30 smithi153 ceph-mon[70044]: pgmap v11676: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:30.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:57:30 smithi083 ceph-mon[79930]: pgmap v11676: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:30.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:57:30 smithi083 ceph-mon[77242]: pgmap v11676: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:57:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:57:31] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T11:57:31.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:57:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:57:31.599Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:57:32.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:57:32 smithi083 ceph-mon[79930]: pgmap v11677: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:32.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:57:32 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:57:32.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:57:32 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:57:32.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:57:32 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:57:32.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:57:32 smithi083 ceph-mon[77242]: pgmap v11677: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:32.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:57:32 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:57:32.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:57:32 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:57:32.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:57:32 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:57:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:57:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:57:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:57:32.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:57:32 smithi153 ceph-mon[70044]: pgmap v11677: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:32.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:57:32 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:57:32.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:57:32 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:57:32.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:57:32 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:57:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:57:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:57:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:57:33.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:57:33 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:57:33.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:57:33 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:57:33.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:57:33 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:57:34.052 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:57:34.339 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:57:34.340 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 4m ago 6h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:57:34.340 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 2m ago 6h 62.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:57:34.340 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 4m ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:57:34.341 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 2m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:57:34.341 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 2m ago 6h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:57:34.341 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 4m ago 6h 839M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:57:34.341 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 4m ago 6h 1452M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:57:34.342 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 2m ago 6h 625M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:57:34.342 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 4m ago 6h 838M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:57:34.342 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 4m ago 6h 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:57:34.342 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 2m ago 6h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:57:34.342 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 4m ago 6h 1047M 2402M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:57:34.343 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 4m ago 6h 1036M 2402M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:57:34.343 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 4m ago 6h 788M 2402M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:57:34.343 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 4m ago 6h 989M 2402M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:57:34.343 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 2m ago 6h 870M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:57:34.344 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 2m ago 6h 1113M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:57:34.344 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 2m ago 6h 971M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:57:34.344 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 2m ago 6h 1179M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:57:34.344 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 2m ago 6h 159M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:57:34.695 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:57:34.696 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:57:34.696 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:57:34.696 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:57:34.696 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:57:34.697 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:57:34.697 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:57:34.697 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:57:34.697 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:57:34.698 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:57:34.698 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:57:34.698 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:57:34.698 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:57:34.699 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:57:34.699 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:57:34.699 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:57:34.699 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:57:34.700 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:57:34.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:57:34 smithi083 ceph-mon[79930]: pgmap v11678: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:34.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:57:34 smithi083 ceph-mon[77242]: pgmap v11678: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:34.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:57:34 smithi153 ceph-mon[70044]: pgmap v11678: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:35.402 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:57:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:57:34.974Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:57:35.700 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:57:35 smithi083 ceph-mon[77242]: from='client.47712 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:57:35.701 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:57:35 smithi083 ceph-mon[77242]: from='client.47718 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:57:35.701 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:57:35 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3762680661' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:57:35.701 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:57:35 smithi083 ceph-mon[79930]: from='client.47712 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:57:35.702 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:57:35 smithi083 ceph-mon[79930]: from='client.47718 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:57:35.702 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:57:35 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3762680661' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:57:35.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:57:35 smithi153 ceph-mon[70044]: from='client.47712 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:57:35.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:57:35 smithi153 ceph-mon[70044]: from='client.47718 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:57:35.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:57:35 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3762680661' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:57:35.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:57:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:57:35.698Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:57:36.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:57:36 smithi083 ceph-mon[79930]: pgmap v11679: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:36.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:57:36 smithi083 ceph-mon[77242]: pgmap v11679: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:36.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:57:36 smithi153 ceph-mon[70044]: pgmap v11679: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:38.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:57:38 smithi083 ceph-mon[79930]: pgmap v11680: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:38.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:57:38 smithi083 ceph-mon[77242]: pgmap v11680: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:38.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:57:38 smithi153 ceph-mon[70044]: pgmap v11680: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:57:40 smithi083 ceph-mon[79930]: pgmap v11681: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:57:40 smithi083 ceph-mon[77242]: pgmap v11681: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:40.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:57:40 smithi153 ceph-mon[70044]: pgmap v11681: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:41.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:57:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:57:41] "GET /metrics HTTP/1.1" 200 225229 "" "Prometheus/2.33.4" 2022-09-29T11:57:41.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:57:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:57:41.600Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:57:41.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:57:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:57:41.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:57:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:57:41.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:57:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:57:41.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:57:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:57:41.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:57:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:57:41.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:57:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:57:41.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:57:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:57:41.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:57:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:57:41.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:57:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:57:41.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:57:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:57:41.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:57:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:57:41.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:57:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:57:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:57:42 smithi083 ceph-mon[79930]: pgmap v11682: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:57:42 smithi083 ceph-mon[77242]: pgmap v11682: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:57:42 smithi153 ceph-mon[70044]: pgmap v11682: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:57:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:57:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:57:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:57:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:57:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:57:44.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:57:44 smithi083 ceph-mon[79930]: pgmap v11683: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:44.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:57:44 smithi083 ceph-mon[77242]: pgmap v11683: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:44.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:57:44 smithi153 ceph-mon[70044]: pgmap v11683: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:45.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:57:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:57:44.975Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:57:45.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:57:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:57:45.699Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:57:46.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:57:46 smithi083 ceph-mon[79930]: pgmap v11684: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:46.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:57:46 smithi083 ceph-mon[77242]: pgmap v11684: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:46.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:57:46 smithi153 ceph-mon[70044]: pgmap v11684: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:48.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:57:48 smithi083 ceph-mon[77242]: pgmap v11685: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:48.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:57:48 smithi083 ceph-mon[79930]: pgmap v11685: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:48.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:57:48 smithi153 ceph-mon[70044]: pgmap v11685: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:50.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:57:50 smithi083 ceph-mon[77242]: pgmap v11686: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:50.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:57:50 smithi083 ceph-mon[79930]: pgmap v11686: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:50.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:57:50 smithi153 ceph-mon[70044]: pgmap v11686: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:57:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:57:51] "GET /metrics HTTP/1.1" 200 225229 "" "Prometheus/2.33.4" 2022-09-29T11:57:51.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:57:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:57:51.601Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:57:52.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:57:52 smithi083 ceph-mon[79930]: pgmap v11687: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:52.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:57:52 smithi083 ceph-mon[77242]: pgmap v11687: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:52.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:57:52 smithi153 ceph-mon[70044]: pgmap v11687: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:57:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:57:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:57:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:57:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:57:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:57:54.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:57:54 smithi083 ceph-mon[79930]: pgmap v11688: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:54.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:57:54 smithi083 ceph-mon[77242]: pgmap v11688: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:54.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:57:54 smithi153 ceph-mon[70044]: pgmap v11688: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:57:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:57:54.976Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:57:55.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:57:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:57:55.699Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:57:56.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:57:56 smithi083 ceph-mon[79930]: pgmap v11689: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:56.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:57:56 smithi083 ceph-mon[77242]: pgmap v11689: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:56.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:57:56 smithi153 ceph-mon[70044]: pgmap v11689: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:58.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:57:58 smithi083 ceph-mon[79930]: pgmap v11690: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:58.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:57:58 smithi083 ceph-mon[77242]: pgmap v11690: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:57:58.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:57:58 smithi153 ceph-mon[70044]: pgmap v11690: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:00.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:58:00 smithi083 ceph-mon[79930]: pgmap v11691: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:00.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:58:00 smithi083 ceph-mon[77242]: pgmap v11691: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:00.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:58:00 smithi153 ceph-mon[70044]: pgmap v11691: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:58:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:58:01] "GET /metrics HTTP/1.1" 200 225229 "" "Prometheus/2.33.4" 2022-09-29T11:58:01.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:58:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:58:01.602Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:58:02.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:58:02 smithi083 ceph-mon[79930]: pgmap v11692: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:02.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:58:02 smithi083 ceph-mon[77242]: pgmap v11692: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:02.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:58:02 smithi153 ceph-mon[70044]: pgmap v11692: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:58:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:58:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:58:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:58:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:58:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:58:04.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:58:04 smithi083 ceph-mon[77242]: pgmap v11693: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:04.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:58:04 smithi083 ceph-mon[79930]: pgmap v11693: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:04.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:58:04 smithi153 ceph-mon[70044]: pgmap v11693: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:05.027 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:58:05.315 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:58:05.315 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 4m ago 6h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:58:05.315 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 2m ago 6h 62.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:58:05.315 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 4m ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:58:05.316 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 2m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:58:05.316 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 2m ago 6h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:58:05.316 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 4m ago 6h 839M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:58:05.316 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 4m ago 6h 1452M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:58:05.316 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 2m ago 6h 625M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:58:05.317 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 4m ago 6h 838M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:58:05.317 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 4m ago 6h 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:58:05.317 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 2m ago 6h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:58:05.317 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 4m ago 6h 1047M 2402M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:58:05.318 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 4m ago 6h 1036M 2402M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:58:05.318 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 4m ago 6h 788M 2402M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:58:05.318 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 4m ago 6h 989M 2402M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:58:05.318 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 2m ago 6h 870M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:58:05.319 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 2m ago 6h 1113M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:58:05.319 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 2m ago 6h 971M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:58:05.319 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 2m ago 6h 1179M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:58:05.319 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 2m ago 6h 159M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:58:05.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:58:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:58:04.975Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:58:05.678 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:58:05.678 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:58:05.678 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:58:05.678 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:58:05.679 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:58:05.679 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:58:05.679 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:58:05.679 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:58:05.680 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:58:05.680 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:58:05.680 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:58:05.680 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:58:05.681 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:58:05.681 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:58:05.681 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:58:05.681 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:58:05.681 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:58:05.682 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:58:05.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:58:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:58:05.699Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:58:06.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:58:06 smithi083 ceph-mon[77242]: pgmap v11694: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:06.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:58:06 smithi083 ceph-mon[77242]: from='client.57661 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:58:06.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:58:06 smithi083 ceph-mon[77242]: from='client.57667 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:58:06.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:58:06 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3727830677' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:58:06.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:58:06 smithi083 ceph-mon[79930]: pgmap v11694: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:06.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:58:06 smithi083 ceph-mon[79930]: from='client.57661 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:58:06.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:58:06 smithi083 ceph-mon[79930]: from='client.57667 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:58:06.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:58:06 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3727830677' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:58:06.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:58:06 smithi153 ceph-mon[70044]: pgmap v11694: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:06.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:58:06 smithi153 ceph-mon[70044]: from='client.57661 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:58:06.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:58:06 smithi153 ceph-mon[70044]: from='client.57667 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:58:06.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:58:06 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3727830677' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:58:08.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:58:08 smithi083 ceph-mon[77242]: pgmap v11695: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:08.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:58:08 smithi083 ceph-mon[79930]: pgmap v11695: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:08.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:58:08 smithi153 ceph-mon[70044]: pgmap v11695: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:10.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:58:10 smithi083 ceph-mon[79930]: pgmap v11696: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:10.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:58:10 smithi083 ceph-mon[77242]: pgmap v11696: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:10.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:58:10 smithi153 ceph-mon[70044]: pgmap v11696: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:58:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:58:11] "GET /metrics HTTP/1.1" 200 225229 "" "Prometheus/2.33.4" 2022-09-29T11:58:11.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:58:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:58:11.603Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:58:12.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:58:12 smithi153 ceph-mon[70044]: pgmap v11697: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:58:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:58:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:58:12.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:58:12 smithi083 ceph-mon[79930]: pgmap v11697: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:12.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:58:12 smithi083 ceph-mon[77242]: pgmap v11697: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:58:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:58:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:58:14.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:58:14 smithi153 ceph-mon[70044]: pgmap v11698: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:14.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:58:14 smithi083 ceph-mon[79930]: pgmap v11698: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:14.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:58:14 smithi083 ceph-mon[77242]: pgmap v11698: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:15.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:58:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:58:14.978Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:58:15.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:58:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:58:15.700Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:58:16.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:58:16 smithi153 ceph-mon[70044]: pgmap v11699: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:16.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:58:16 smithi083 ceph-mon[79930]: pgmap v11699: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:16.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:58:16 smithi083 ceph-mon[77242]: pgmap v11699: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:18.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:58:18 smithi153 ceph-mon[70044]: pgmap v11700: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:18.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:58:18 smithi083 ceph-mon[79930]: pgmap v11700: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:18.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:58:18 smithi083 ceph-mon[77242]: pgmap v11700: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:20.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:58:20 smithi153 ceph-mon[70044]: pgmap v11701: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:20.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:58:20 smithi083 ceph-mon[79930]: pgmap v11701: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:20.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:58:20 smithi083 ceph-mon[77242]: pgmap v11701: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:58:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:58:21] "GET /metrics HTTP/1.1" 200 225229 "" "Prometheus/2.33.4" 2022-09-29T11:58:21.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:58:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:58:21.604Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:58:22.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:58:22 smithi153 ceph-mon[70044]: pgmap v11702: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:58:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:58:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:58:22.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:58:22 smithi083 ceph-mon[79930]: pgmap v11702: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:22.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:58:22 smithi083 ceph-mon[77242]: pgmap v11702: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:58:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:58:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:58:24.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:58:24 smithi153 ceph-mon[70044]: pgmap v11703: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:24.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:58:24 smithi083 ceph-mon[79930]: pgmap v11703: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:24.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:58:24 smithi083 ceph-mon[77242]: pgmap v11703: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:25.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:58:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:58:24.978Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:58:25.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:58:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:58:25.700Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:58:26.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:58:26 smithi153 ceph-mon[70044]: pgmap v11704: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:26.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:58:26 smithi083 ceph-mon[79930]: pgmap v11704: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:26.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:58:26 smithi083 ceph-mon[77242]: pgmap v11704: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:28.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:58:28 smithi153 ceph-mon[70044]: pgmap v11705: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:28.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:58:28 smithi083 ceph-mon[79930]: pgmap v11705: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:28.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:58:28 smithi083 ceph-mon[77242]: pgmap v11705: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:30.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:58:30 smithi153 ceph-mon[70044]: pgmap v11706: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:30.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:58:30 smithi083 ceph-mon[79930]: pgmap v11706: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:30.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:58:30 smithi083 ceph-mon[77242]: pgmap v11706: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:31.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:58:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:58:31.604Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:58:31.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:58:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:58:31] "GET /metrics HTTP/1.1" 200 225228 "" "Prometheus/2.33.4" 2022-09-29T11:58:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:58:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:58:32.316Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:58:32.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:58:32 smithi153 ceph-mon[70044]: pgmap v11707: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:32.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:58:32 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:58:32.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:58:32 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:58:32.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:58:32 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:58:32.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:58:32 smithi083 ceph-mon[79930]: pgmap v11707: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:32.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:58:32 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:58:32.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:58:32 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:58:32.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:58:32 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:58:32.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:58:32 smithi083 ceph-mon[77242]: pgmap v11707: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:32.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:58:32 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:58:32.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:58:32 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:58:32.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:58:32 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:58:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:58:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:58:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:58:34.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:58:34 smithi083 ceph-mon[79930]: pgmap v11708: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:34.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:58:34 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:58:34.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:58:34 smithi083 ceph-mon[77242]: pgmap v11708: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:34.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:58:34 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:58:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:58:34 smithi153 ceph-mon[70044]: pgmap v11708: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:58:34 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:58:35.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:58:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:58:34.979Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:58:35.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:58:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:58:35.701Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:58:36.007 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:58:36.297 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:58:36.297 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 5m ago 6h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:58:36.298 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 3m ago 6h 62.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:58:36.298 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 5m ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:58:36.298 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 3m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:58:36.299 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 3m ago 6h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:58:36.299 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 5m ago 6h 839M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:58:36.299 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 5m ago 6h 1452M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:58:36.299 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 3m ago 6h 625M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:58:36.299 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 5m ago 6h 838M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:58:36.300 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 5m ago 6h 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:58:36.300 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 3m ago 6h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:58:36.300 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 5m ago 6h 1047M 2402M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:58:36.300 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 5m ago 6h 1036M 2402M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:58:36.301 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 5m ago 6h 788M 2402M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:58:36.301 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 5m ago 6h 989M 2402M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:58:36.301 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 3m ago 6h 870M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:58:36.301 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 3m ago 6h 1113M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:58:36.302 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 3m ago 6h 971M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:58:36.302 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 3m ago 6h 1179M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:58:36.302 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 3m ago 6h 159M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:58:36.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:58:36 smithi083 ceph-mon[79930]: pgmap v11709: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:36.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:58:36 smithi083 ceph-mon[77242]: pgmap v11709: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:58:36 smithi153 ceph-mon[70044]: pgmap v11709: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:36.652 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:58:36.652 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:58:36.652 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:58:36.652 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:58:36.653 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:58:36.653 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:58:36.653 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:58:36.653 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:58:36.653 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:58:36.654 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:58:36.654 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:58:36.654 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:58:36.655 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:58:36.655 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:58:36.655 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:58:36.655 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:58:36.655 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:58:36.656 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:58:37.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:58:37 smithi083 ceph-mon[79930]: from='client.47748 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:58:37.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:58:37 smithi083 ceph-mon[79930]: from='client.57685 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:58:37.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:58:37 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2703252699' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:58:37.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:58:37 smithi083 ceph-mon[77242]: from='client.47748 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:58:37.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:58:37 smithi083 ceph-mon[77242]: from='client.57685 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:58:37.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:58:37 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2703252699' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:58:37.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:58:37 smithi153 ceph-mon[70044]: from='client.47748 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:58:37.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:58:37 smithi153 ceph-mon[70044]: from='client.57685 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:58:37.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:58:37 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2703252699' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:58:38.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:58:38 smithi083 ceph-mon[79930]: pgmap v11710: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:38.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:58:38 smithi083 ceph-mon[77242]: pgmap v11710: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:58:38 smithi153 ceph-mon[70044]: pgmap v11710: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:40.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:58:40 smithi083 ceph-mon[77242]: pgmap v11711: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:58:40 smithi083 ceph-mon[79930]: pgmap v11711: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:58:40 smithi153 ceph-mon[70044]: pgmap v11711: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:41.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:58:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:58:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:58:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:58:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:58:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:58:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:58:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:58:41.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:58:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:58:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:58:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:58:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:58:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:58:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:58:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:58:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:58:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:58:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:58:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:58:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:58:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:58:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:58:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:58:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:58:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:58:41] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T11:58:41.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:58:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:58:41.605Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:58:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:58:42 smithi083 ceph-mon[79930]: pgmap v11712: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:42.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:58:42 smithi083 ceph-mon[77242]: pgmap v11712: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:58:42 smithi153 ceph-mon[70044]: pgmap v11712: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:42.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:58:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:58:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:58:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:58:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:58:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:58:44.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:58:44 smithi083 ceph-mon[79930]: pgmap v11713: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:44.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:58:44 smithi083 ceph-mon[77242]: pgmap v11713: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:44.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:58:44 smithi153 ceph-mon[70044]: pgmap v11713: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:45.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:58:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:58:44.980Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:58:45.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:58:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:58:45.701Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:58:46.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:58:46 smithi083 ceph-mon[79930]: pgmap v11714: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:46.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:58:46 smithi083 ceph-mon[77242]: pgmap v11714: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:46.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:58:46 smithi153 ceph-mon[70044]: pgmap v11714: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:48.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:58:48 smithi083 ceph-mon[79930]: pgmap v11715: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:48.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:58:48 smithi083 ceph-mon[77242]: pgmap v11715: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:48.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:58:48 smithi153 ceph-mon[70044]: pgmap v11715: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:50.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:58:50 smithi083 ceph-mon[79930]: pgmap v11716: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:50.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:58:50 smithi083 ceph-mon[77242]: pgmap v11716: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:58:50 smithi153 ceph-mon[70044]: pgmap v11716: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:58:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:58:51] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T11:58:51.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:58:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:58:51.606Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:58:52.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:58:52 smithi083 ceph-mon[77242]: pgmap v11717: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:52.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:58:52 smithi083 ceph-mon[79930]: pgmap v11717: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:58:52 smithi153 ceph-mon[70044]: pgmap v11717: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:52.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:58:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:58:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:58:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:58:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:58:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:58:54.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:58:54 smithi083 ceph-mon[79930]: pgmap v11718: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:54.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:58:54 smithi083 ceph-mon[77242]: pgmap v11718: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:54.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:58:54 smithi153 ceph-mon[70044]: pgmap v11718: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:58:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:58:54.981Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:58:55.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:58:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:58:55.702Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:58:56.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:58:56 smithi083 ceph-mon[79930]: pgmap v11719: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:56.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:58:56 smithi083 ceph-mon[77242]: pgmap v11719: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:56.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:58:56 smithi153 ceph-mon[70044]: pgmap v11719: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:58.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:58:58 smithi083 ceph-mon[77242]: pgmap v11720: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:58.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:58:58 smithi083 ceph-mon[79930]: pgmap v11720: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:58:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:58:58 smithi153 ceph-mon[70044]: pgmap v11720: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:00.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:59:00 smithi083 ceph-mon[77242]: pgmap v11721: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:00.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:59:00 smithi083 ceph-mon[79930]: pgmap v11721: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:59:00 smithi153 ceph-mon[70044]: pgmap v11721: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:01.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:59:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:59:01.607Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:59:01.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:59:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:59:01] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T11:59:02.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:59:02 smithi083 ceph-mon[77242]: pgmap v11722: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:02.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:59:02 smithi083 ceph-mon[79930]: pgmap v11722: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:59:02 smithi153 ceph-mon[70044]: pgmap v11722: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:02.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:59:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:59:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:59:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:59:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:59:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:59:04.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:59:04 smithi083 ceph-mon[79930]: pgmap v11723: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:04.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:59:04 smithi083 ceph-mon[77242]: pgmap v11723: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:59:04 smithi153 ceph-mon[70044]: pgmap v11723: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:05.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:59:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:59:04.982Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:59:05.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:59:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:59:05.703Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:59:06.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:59:06 smithi083 ceph-mon[77242]: pgmap v11724: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:06.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:59:06 smithi083 ceph-mon[79930]: pgmap v11724: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:06.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:59:06 smithi153 ceph-mon[70044]: pgmap v11724: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:06.976 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:59:07.262 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:59:07.263 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 5m ago 6h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:59:07.263 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 3m ago 6h 62.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:59:07.263 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 5m ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:59:07.263 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 3m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:59:07.264 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 3m ago 6h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:59:07.264 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 5m ago 6h 839M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:59:07.264 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 5m ago 6h 1452M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:59:07.264 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 3m ago 6h 625M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:59:07.264 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 5m ago 6h 838M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:59:07.265 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 5m ago 6h 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:59:07.265 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 3m ago 6h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:59:07.265 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 5m ago 6h 1047M 2402M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:59:07.265 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 5m ago 6h 1036M 2402M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:59:07.266 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 5m ago 6h 788M 2402M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:59:07.266 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 5m ago 6h 989M 2402M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:59:07.266 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 3m ago 6h 870M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:59:07.266 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 3m ago 6h 1113M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:59:07.266 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 3m ago 6h 971M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:59:07.267 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 3m ago 6h 1179M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:59:07.267 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 3m ago 6h 159M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:59:07.620 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:59:07.620 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:59:07.620 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:59:07.621 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:59:07.621 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:59:07.621 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:59:07.621 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:59:07.621 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:59:07.622 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:59:07.622 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:59:07.622 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:59:07.622 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:59:07.623 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:59:07.623 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:59:07.623 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:59:07.623 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:59:07.623 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:59:07.624 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:59:08.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:59:08 smithi083 ceph-mon[79930]: pgmap v11725: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:08.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:59:08 smithi083 ceph-mon[79930]: from='client.57697 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:59:08.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:59:08 smithi083 ceph-mon[79930]: from='client.57703 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:59:08.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:59:08 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3658087274' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:59:08.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:59:08 smithi083 ceph-mon[77242]: pgmap v11725: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:08.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:59:08 smithi083 ceph-mon[77242]: from='client.57697 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:59:08.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:59:08 smithi083 ceph-mon[77242]: from='client.57703 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:59:08.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:59:08 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3658087274' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:59:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:59:08 smithi153 ceph-mon[70044]: pgmap v11725: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:59:08 smithi153 ceph-mon[70044]: from='client.57697 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:59:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:59:08 smithi153 ceph-mon[70044]: from='client.57703 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:59:08.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:59:08 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3658087274' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:59:10.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:59:10 smithi083 ceph-mon[79930]: pgmap v11726: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:10.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:59:10 smithi083 ceph-mon[77242]: pgmap v11726: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:59:10 smithi153 ceph-mon[70044]: pgmap v11726: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:59:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:59:11] "GET /metrics HTTP/1.1" 200 225227 "" "Prometheus/2.33.4" 2022-09-29T11:59:11.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:59:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:59:11.607Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:59:12.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:59:12 smithi083 ceph-mon[79930]: pgmap v11727: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:12.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:59:12 smithi083 ceph-mon[77242]: pgmap v11727: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:59:12 smithi153 ceph-mon[70044]: pgmap v11727: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:12.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:59:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:59:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:59:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:59:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:59:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:59:14.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:59:14 smithi083 ceph-mon[79930]: pgmap v11728: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:14.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:59:14 smithi083 ceph-mon[77242]: pgmap v11728: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:14.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:59:14 smithi153 ceph-mon[70044]: pgmap v11728: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:15.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:59:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:59:14.982Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:59:15.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:59:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:59:15.703Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:59:16.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:59:16 smithi083 ceph-mon[79930]: pgmap v11729: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:16.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:59:16 smithi083 ceph-mon[77242]: pgmap v11729: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:59:16 smithi153 ceph-mon[70044]: pgmap v11729: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:18.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:59:18 smithi083 ceph-mon[79930]: pgmap v11730: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:18.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:59:18 smithi083 ceph-mon[77242]: pgmap v11730: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:59:18 smithi153 ceph-mon[70044]: pgmap v11730: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:20.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:59:20 smithi083 ceph-mon[79930]: pgmap v11731: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:20.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:59:20 smithi083 ceph-mon[77242]: pgmap v11731: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:59:20 smithi153 ceph-mon[70044]: pgmap v11731: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:59:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:59:21] "GET /metrics HTTP/1.1" 200 225227 "" "Prometheus/2.33.4" 2022-09-29T11:59:21.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:59:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:59:21.608Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:59:22.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:59:22 smithi083 ceph-mon[77242]: pgmap v11732: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:22.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:59:22 smithi083 ceph-mon[79930]: pgmap v11732: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:59:22 smithi153 ceph-mon[70044]: pgmap v11732: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:22.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:59:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:59:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:59:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:59:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:59:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:59:24.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:59:24 smithi083 ceph-mon[77242]: pgmap v11733: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:24.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:59:24 smithi083 ceph-mon[79930]: pgmap v11733: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:59:24 smithi153 ceph-mon[70044]: pgmap v11733: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:25.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:59:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:59:24.983Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:59:25.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:59:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:59:25.703Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:59:26.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:59:26 smithi083 ceph-mon[79930]: pgmap v11734: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:26.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:59:26 smithi083 ceph-mon[77242]: pgmap v11734: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:59:26 smithi153 ceph-mon[70044]: pgmap v11734: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:28.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:59:28 smithi083 ceph-mon[79930]: pgmap v11735: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:28.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:59:28 smithi083 ceph-mon[77242]: pgmap v11735: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:59:28 smithi153 ceph-mon[70044]: pgmap v11735: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:30.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:59:30 smithi083 ceph-mon[79930]: pgmap v11736: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:30.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:59:30 smithi083 ceph-mon[77242]: pgmap v11736: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:59:30 smithi153 ceph-mon[70044]: pgmap v11736: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:59:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:59:31] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T11:59:31.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:59:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:59:31.609Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:59:32.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:59:32 smithi083 ceph-mon[79930]: pgmap v11737: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:32.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:59:32 smithi083 ceph-mon[77242]: pgmap v11737: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:59:32 smithi153 ceph-mon[70044]: pgmap v11737: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:32.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:59:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:59:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:59:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:59:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:59:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:59:33.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:59:33 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:59:33.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:59:33 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:59:33.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:59:33 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:59:33.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:59:33 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:59:33.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:59:33 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:59:33.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:59:33 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:59:33.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:59:33 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T11:59:33.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:59:33 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T11:59:33.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:59:33 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T11:59:34.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:59:34 smithi083 ceph-mon[79930]: pgmap v11738: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:34.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:59:34 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:59:34.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:59:34 smithi083 ceph-mon[77242]: pgmap v11738: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:34.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:59:34 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:59:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:59:34 smithi153 ceph-mon[70044]: pgmap v11738: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:59:34 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T11:59:35.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:59:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:59:34.983Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:59:35.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:59:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:59:35.704Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:59:36.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:59:36 smithi083 ceph-mon[77242]: pgmap v11739: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:36.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:59:36 smithi083 ceph-mon[79930]: pgmap v11739: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:59:36 smithi153 ceph-mon[70044]: pgmap v11739: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:37.946 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T11:59:38.233 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T11:59:38.233 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 6m ago 6h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T11:59:38.234 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 4m ago 6h 62.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T11:59:38.234 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 6m ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T11:59:38.234 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 4m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T11:59:38.234 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 4m ago 6h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T11:59:38.235 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 6m ago 6h 839M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T11:59:38.235 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 6m ago 6h 1452M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T11:59:38.235 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 4m ago 6h 625M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T11:59:38.235 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 6m ago 6h 838M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T11:59:38.236 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 6m ago 6h 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T11:59:38.236 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 4m ago 6h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T11:59:38.236 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 6m ago 6h 1047M 2402M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T11:59:38.236 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 6m ago 6h 1036M 2402M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T11:59:38.236 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 6m ago 6h 788M 2402M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T11:59:38.237 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 6m ago 6h 989M 2402M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T11:59:38.237 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 4m ago 6h 870M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T11:59:38.237 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 4m ago 6h 1113M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T11:59:38.237 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 4m ago 6h 971M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T11:59:38.237 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 4m ago 6h 1179M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T11:59:38.238 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 4m ago 6h 159M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T11:59:38.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:59:38 smithi083 ceph-mon[79930]: pgmap v11740: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:38.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:59:38 smithi083 ceph-mon[77242]: pgmap v11740: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:59:38 smithi153 ceph-mon[70044]: pgmap v11740: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:38.592 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T11:59:38.592 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T11:59:38.592 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T11:59:38.592 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:59:38.593 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T11:59:38.593 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T11:59:38.593 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:59:38.593 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T11:59:38.594 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T11:59:38.594 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:59:38.594 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T11:59:38.594 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T11:59:38.595 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T11:59:38.595 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T11:59:38.595 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T11:59:38.595 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T11:59:38.596 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T11:59:38.596 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T11:59:39.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:59:39 smithi083 ceph-mon[77242]: from='client.57715 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:59:39.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:59:39 smithi083 ceph-mon[77242]: from='client.47790 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:59:39.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:59:39 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3053757010' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:59:39.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:59:39 smithi083 ceph-mon[79930]: from='client.57715 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:59:39.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:59:39 smithi083 ceph-mon[79930]: from='client.47790 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:59:39.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:59:39 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3053757010' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:59:39.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:59:39 smithi153 ceph-mon[70044]: from='client.57715 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:59:39.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:59:39 smithi153 ceph-mon[70044]: from='client.47790 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T11:59:39.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:59:39 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3053757010' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T11:59:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:59:40 smithi083 ceph-mon[79930]: pgmap v11741: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:59:40 smithi083 ceph-mon[77242]: pgmap v11741: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:59:40 smithi153 ceph-mon[70044]: pgmap v11741: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:41.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:59:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:59:41.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:59:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:59:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:59:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:59:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:59:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:59:41.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:59:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:59:41.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:59:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:59:41.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:59:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:59:41.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:59:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:59:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:59:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:59:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:59:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:59:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:59:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T11:59:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:59:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T11:59:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:59:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:59:41] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T11:59:41.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:59:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:59:41.610Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:59:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:59:42 smithi083 ceph-mon[79930]: pgmap v11742: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:42.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:59:42 smithi083 ceph-mon[77242]: pgmap v11742: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:59:42 smithi153 ceph-mon[70044]: pgmap v11742: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:42.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:59:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:59:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:59:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:59:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:59:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:59:44.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:59:44 smithi083 ceph-mon[79930]: pgmap v11743: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:44.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:59:44 smithi083 ceph-mon[77242]: pgmap v11743: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:44.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:59:44 smithi153 ceph-mon[70044]: pgmap v11743: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:45.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:59:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:59:44.984Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:59:45.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:59:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:59:45.704Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:59:46.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:59:46 smithi083 ceph-mon[79930]: pgmap v11744: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:46.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:59:46 smithi083 ceph-mon[77242]: pgmap v11744: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:46.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:59:46 smithi153 ceph-mon[70044]: pgmap v11744: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:48.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:59:48 smithi083 ceph-mon[79930]: pgmap v11745: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:48.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:59:48 smithi083 ceph-mon[77242]: pgmap v11745: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:48.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:59:48 smithi153 ceph-mon[70044]: pgmap v11745: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:50.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:59:50 smithi083 ceph-mon[79930]: pgmap v11746: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:50.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:59:50 smithi083 ceph-mon[77242]: pgmap v11746: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:59:50 smithi153 ceph-mon[70044]: pgmap v11746: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 11:59:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:59:51] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T11:59:51.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:59:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:59:51.610Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:59:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:59:52 smithi153 ceph-mon[70044]: pgmap v11747: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:52.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 11:59:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T11:59:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T11:59:52.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:59:52 smithi083 ceph-mon[77242]: pgmap v11747: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:52.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:59:52 smithi083 ceph-mon[79930]: pgmap v11747: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 11:59:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:11:59:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T11:59:54.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:59:54 smithi153 ceph-mon[70044]: pgmap v11748: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:54.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:59:54 smithi083 ceph-mon[79930]: pgmap v11748: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:54.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:59:54 smithi083 ceph-mon[77242]: pgmap v11748: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:59:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:59:54.985Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:59:55.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 11:59:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T11:59:55.705Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T11:59:56.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:59:56 smithi153 ceph-mon[70044]: pgmap v11749: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:56.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:59:56 smithi083 ceph-mon[79930]: pgmap v11749: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:56.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:59:56 smithi083 ceph-mon[77242]: pgmap v11749: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 11:59:58 smithi153 ceph-mon[70044]: pgmap v11750: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:58.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 11:59:58 smithi083 ceph-mon[79930]: pgmap v11750: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T11:59:58.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 11:59:58 smithi083 ceph-mon[77242]: pgmap v11750: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:00.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:00:00 smithi083 ceph-mon[79930]: pgmap v11751: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:00.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:00:00 smithi083 ceph-mon[79930]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T12:00:00.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:00:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[77219]: 2022-09-29T11:59:59.999+0000 7f160f665700 -1 log_channel(cluster) log [ERR] : overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T12:00:00.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:00:00 smithi083 ceph-mon[77242]: pgmap v11751: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:00.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:00:00 smithi083 ceph-mon[77242]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T12:00:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:00:00 smithi153 ceph-mon[70044]: pgmap v11751: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:00:00 smithi153 ceph-mon[70044]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T12:00:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:00:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:00:01] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T12:00:01.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:00:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:00:01.611Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:00:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:00:02 smithi153 ceph-mon[70044]: pgmap v11752: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:02.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:00:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:00:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:00:02.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:00:02 smithi083 ceph-mon[77242]: pgmap v11752: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:02.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:00:02 smithi083 ceph-mon[79930]: pgmap v11752: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:00:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:00:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:00:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:00:04 smithi153 ceph-mon[70044]: pgmap v11753: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:04.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:00:04 smithi083 ceph-mon[77242]: pgmap v11753: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:04.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:00:04 smithi083 ceph-mon[79930]: pgmap v11753: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:05.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:00:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:00:04.985Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:00:05.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:00:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:00:05.704Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:00:06.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:00:06 smithi153 ceph-mon[70044]: pgmap v11754: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:06.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:00:06 smithi083 ceph-mon[77242]: pgmap v11754: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:06.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:00:06 smithi083 ceph-mon[79930]: pgmap v11754: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:00:08 smithi153 ceph-mon[70044]: pgmap v11755: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:08.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:00:08 smithi083 ceph-mon[79930]: pgmap v11755: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:08.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:00:08 smithi083 ceph-mon[77242]: pgmap v11755: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:08.920 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:00:09.209 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:00:09.209 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 6m ago 6h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:00:09.209 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 4m ago 6h 62.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:00:09.210 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 6m ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:00:09.210 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 4m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:00:09.210 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 4m ago 6h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:00:09.210 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 6m ago 6h 839M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:00:09.211 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 6m ago 6h 1452M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:00:09.211 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 4m ago 6h 625M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:00:09.211 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 6m ago 6h 838M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:00:09.211 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 6m ago 6h 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:00:09.212 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 4m ago 6h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:00:09.212 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 6m ago 6h 1047M 2402M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:00:09.212 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 6m ago 6h 1036M 2402M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:00:09.212 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 6m ago 6h 788M 2402M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:00:09.212 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 6m ago 6h 989M 2402M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:00:09.213 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 4m ago 6h 870M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:00:09.213 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 4m ago 6h 1113M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:00:09.213 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 4m ago 6h 971M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:00:09.213 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 4m ago 6h 1179M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:00:09.214 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 4m ago 6h 159M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:00:09.563 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:00:09.564 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:00:09.564 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:00:09.564 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:00:09.564 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:00:09.565 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:00:09.565 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:00:09.565 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:00:09.566 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:00:09.566 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:00:09.566 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:00:09.566 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:00:09.567 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:00:09.567 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:00:09.567 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:00:09.567 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:00:09.568 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:00:09.568 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:00:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:00:10 smithi153 ceph-mon[70044]: pgmap v11756: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:00:10 smithi153 ceph-mon[70044]: from='client.57733 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:00:10.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:00:10 smithi153 ceph-mon[70044]: from='client.47808 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:00:10.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:00:10 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2065194576' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:00:10.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:00:10 smithi083 ceph-mon[77242]: pgmap v11756: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:10.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:00:10 smithi083 ceph-mon[77242]: from='client.57733 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:00:10.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:00:10 smithi083 ceph-mon[77242]: from='client.47808 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:00:10.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:00:10 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2065194576' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:00:10.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:00:10 smithi083 ceph-mon[79930]: pgmap v11756: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:10.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:00:10 smithi083 ceph-mon[79930]: from='client.57733 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:00:10.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:00:10 smithi083 ceph-mon[79930]: from='client.47808 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:00:10.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:00:10 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2065194576' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:00:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:00:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:00:11] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T12:00:11.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:00:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:00:11.611Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:00:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:00:12 smithi153 ceph-mon[70044]: pgmap v11757: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:12.564 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:00:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:00:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:00:12.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:00:12 smithi083 ceph-mon[77242]: pgmap v11757: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:12.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:00:12 smithi083 ceph-mon[79930]: pgmap v11757: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:00:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:00:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:00:14.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:00:14 smithi153 ceph-mon[70044]: pgmap v11758: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:14.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:00:14 smithi083 ceph-mon[77242]: pgmap v11758: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:14.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:00:14 smithi083 ceph-mon[79930]: pgmap v11758: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:15.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:00:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:00:14.986Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:00:15.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:00:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:00:15.705Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:00:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:00:16 smithi153 ceph-mon[70044]: pgmap v11759: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:16.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:00:16 smithi083 ceph-mon[77242]: pgmap v11759: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:16.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:00:16 smithi083 ceph-mon[79930]: pgmap v11759: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:00:18 smithi153 ceph-mon[70044]: pgmap v11760: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:18.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:00:18 smithi083 ceph-mon[79930]: pgmap v11760: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:18.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:00:18 smithi083 ceph-mon[77242]: pgmap v11760: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:00:20 smithi153 ceph-mon[70044]: pgmap v11761: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:20.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:00:20 smithi083 ceph-mon[77242]: pgmap v11761: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:20.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:00:20 smithi083 ceph-mon[79930]: pgmap v11761: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:21.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:00:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:00:21.612Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:00:21.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:00:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:00:21] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T12:00:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:00:22 smithi153 ceph-mon[70044]: pgmap v11762: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:22.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:00:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:00:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:00:22.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:00:22 smithi083 ceph-mon[79930]: pgmap v11762: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:22.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:00:22 smithi083 ceph-mon[77242]: pgmap v11762: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:00:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:00:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:00:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:00:24 smithi153 ceph-mon[70044]: pgmap v11763: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:24.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:00:24 smithi083 ceph-mon[79930]: pgmap v11763: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:24.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:00:24 smithi083 ceph-mon[77242]: pgmap v11763: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:25.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:00:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:00:24.987Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:00:25.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:00:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:00:25.706Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:00:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:00:26 smithi153 ceph-mon[70044]: pgmap v11764: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:26.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:00:26 smithi083 ceph-mon[79930]: pgmap v11764: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:26.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:00:26 smithi083 ceph-mon[77242]: pgmap v11764: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:00:28 smithi153 ceph-mon[70044]: pgmap v11765: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:28.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:00:28 smithi083 ceph-mon[79930]: pgmap v11765: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:28.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:00:28 smithi083 ceph-mon[77242]: pgmap v11765: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:00:30 smithi153 ceph-mon[70044]: pgmap v11766: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:30.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:00:30 smithi083 ceph-mon[79930]: pgmap v11766: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:30.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:00:30 smithi083 ceph-mon[77242]: pgmap v11766: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:00:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:00:31] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T12:00:31.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:00:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:00:31.613Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:00:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:00:32 smithi153 ceph-mon[70044]: pgmap v11767: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:32.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:00:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:00:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:00:32.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:00:32 smithi083 ceph-mon[79930]: pgmap v11767: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:32.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:00:32 smithi083 ceph-mon[77242]: pgmap v11767: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:00:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:00:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:00:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:00:34 smithi153 ceph-mon[70044]: pgmap v11768: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:00:34 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:00:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:00:34 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:00:34.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:00:34 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:00:34.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:00:34 smithi083 ceph-mon[79930]: pgmap v11768: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:34.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:00:34 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:00:34.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:00:34 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:00:34.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:00:34 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:00:34.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:00:34 smithi083 ceph-mon[77242]: pgmap v11768: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:34.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:00:34 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:00:34.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:00:34 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:00:34.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:00:34 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:00:35.368 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:00:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:00:34.987Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:00:35.369 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:00:35 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:00:35.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:00:35 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:00:35.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:00:35 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:00:35.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:00:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:00:35.706Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:00:36.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:00:36 smithi083 ceph-mon[77242]: pgmap v11769: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:36.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:00:36 smithi083 ceph-mon[79930]: pgmap v11769: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:36.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:00:36 smithi153 ceph-mon[70044]: pgmap v11769: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:38.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:00:38 smithi083 ceph-mon[79930]: pgmap v11770: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:38.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:00:38 smithi083 ceph-mon[77242]: pgmap v11770: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:38.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:00:38 smithi153 ceph-mon[70044]: pgmap v11770: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:39.887 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:00:40.175 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:00:40.175 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 7m ago 6h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:00:40.175 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 5m ago 6h 62.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:00:40.175 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 7m ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:00:40.176 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 5m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:00:40.176 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 5m ago 6h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:00:40.176 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 7m ago 6h 839M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:00:40.176 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 7m ago 6h 1452M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:00:40.177 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 5m ago 6h 625M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:00:40.177 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 7m ago 6h 838M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:00:40.177 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 7m ago 6h 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:00:40.177 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 5m ago 6h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:00:40.177 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 7m ago 6h 1047M 2402M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:00:40.178 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 7m ago 6h 1036M 2402M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:00:40.178 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 7m ago 6h 788M 2402M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:00:40.178 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 7m ago 6h 989M 2402M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:00:40.178 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 5m ago 6h 870M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:00:40.179 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 5m ago 6h 1113M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:00:40.179 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 5m ago 6h 971M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:00:40.179 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 5m ago 6h 1179M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:00:40.179 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 5m ago 6h 159M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:00:40.534 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:00:40.534 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:00:40.534 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:00:40.534 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:00:40.535 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:00:40.535 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:00:40.535 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:00:40.535 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:00:40.536 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:00:40.536 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:00:40.536 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:00:40.536 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:00:40.536 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:00:40.537 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:00:40.537 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:00:40.537 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:00:40.537 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:00:40.538 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:00:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:00:40 smithi083 ceph-mon[79930]: pgmap v11771: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:00:40 smithi083 ceph-mon[77242]: pgmap v11771: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:00:40 smithi153 ceph-mon[70044]: pgmap v11771: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:41.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:00:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:00:41.614Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:00:41.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:00:41 smithi083 ceph-mon[77242]: from='client.57751 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:00:41.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:00:41 smithi083 ceph-mon[77242]: from='client.57757 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:00:41.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:00:41 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2753648118' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:00:41.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:00:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:00:41.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:00:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:00:41.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:00:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:00:41.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:00:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:00:41.715 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:00:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:00:41] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T12:00:41.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:00:41 smithi083 ceph-mon[79930]: from='client.57751 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:00:41.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:00:41 smithi083 ceph-mon[79930]: from='client.57757 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:00:41.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:00:41 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2753648118' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:00:41.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:00:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:00:41.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:00:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:00:41.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:00:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:00:41.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:00:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:00:41.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:00:41 smithi153 ceph-mon[70044]: from='client.57751 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:00:41.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:00:41 smithi153 ceph-mon[70044]: from='client.57757 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:00:41.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:00:41 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2753648118' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:00:41.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:00:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:00:41.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:00:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:00:41.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:00:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:00:41.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:00:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:00:42.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:00:42 smithi083 ceph-mon[77242]: pgmap v11772: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:42.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:00:42 smithi083 ceph-mon[79930]: pgmap v11772: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:00:42 smithi153 ceph-mon[70044]: pgmap v11772: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:00:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:00:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:00:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:00:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:00:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:00:44.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:00:44 smithi083 ceph-mon[79930]: pgmap v11773: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:44.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:00:44 smithi083 ceph-mon[77242]: pgmap v11773: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:44.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:00:44 smithi153 ceph-mon[70044]: pgmap v11773: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:45.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:00:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:00:44.989Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:00:45.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:00:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:00:45.707Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:00:46.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:00:46 smithi083 ceph-mon[79930]: pgmap v11774: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:46.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:00:46 smithi083 ceph-mon[77242]: pgmap v11774: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:46.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:00:46 smithi153 ceph-mon[70044]: pgmap v11774: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:48.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:00:48 smithi083 ceph-mon[79930]: pgmap v11775: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:48.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:00:48 smithi083 ceph-mon[77242]: pgmap v11775: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:48.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:00:48 smithi153 ceph-mon[70044]: pgmap v11775: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:50.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:00:50 smithi083 ceph-mon[79930]: pgmap v11776: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:50.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:00:50 smithi083 ceph-mon[77242]: pgmap v11776: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:50.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:00:50 smithi153 ceph-mon[70044]: pgmap v11776: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:51.967 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:00:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:00:51] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T12:00:51.968 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:00:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:00:51.615Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:00:52.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:00:52 smithi083 ceph-mon[77242]: pgmap v11777: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:52.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:00:52 smithi083 ceph-mon[79930]: pgmap v11777: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:52.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:00:52 smithi153 ceph-mon[70044]: pgmap v11777: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:00:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:00:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:00:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:00:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:00:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:00:54.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:00:54 smithi083 ceph-mon[77242]: pgmap v11778: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:54.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:00:54 smithi083 ceph-mon[79930]: pgmap v11778: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:54.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:00:54 smithi153 ceph-mon[70044]: pgmap v11778: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:00:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:00:54.989Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:00:55.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:00:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:00:55.707Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:00:56.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:00:56 smithi083 ceph-mon[79930]: pgmap v11779: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:56.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:00:56 smithi083 ceph-mon[77242]: pgmap v11779: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:56.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:00:56 smithi153 ceph-mon[70044]: pgmap v11779: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:58.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:00:58 smithi083 ceph-mon[79930]: pgmap v11780: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:58.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:00:58 smithi083 ceph-mon[77242]: pgmap v11780: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:00:58.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:00:58 smithi153 ceph-mon[70044]: pgmap v11780: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:00.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:01:00 smithi083 ceph-mon[77242]: pgmap v11781: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:00.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:01:00 smithi083 ceph-mon[79930]: pgmap v11781: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:00.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:01:00 smithi153 ceph-mon[70044]: pgmap v11781: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:01:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:01:01] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T12:01:01.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:01:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:01:01.616Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:01:02.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:01:02 smithi083 ceph-mon[77242]: pgmap v11782: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:02.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:01:02 smithi083 ceph-mon[79930]: pgmap v11782: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:02.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:01:02 smithi153 ceph-mon[70044]: pgmap v11782: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:01:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:01:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:01:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:01:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:01:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:01:04.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:01:04 smithi083 ceph-mon[79930]: pgmap v11783: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:04.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:01:04 smithi083 ceph-mon[77242]: pgmap v11783: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:04.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:01:04 smithi153 ceph-mon[70044]: pgmap v11783: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:05.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:01:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:01:04.990Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:01:05.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:01:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:01:05.708Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:01:06.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:01:06 smithi083 ceph-mon[79930]: pgmap v11784: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:06.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:01:06 smithi083 ceph-mon[77242]: pgmap v11784: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:06.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:01:06 smithi153 ceph-mon[70044]: pgmap v11784: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:08.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:01:08 smithi083 ceph-mon[79930]: pgmap v11785: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:08.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:01:08 smithi083 ceph-mon[77242]: pgmap v11785: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:08.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:01:08 smithi153 ceph-mon[70044]: pgmap v11785: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:10.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:01:10 smithi083 ceph-mon[79930]: pgmap v11786: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:10.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:01:10 smithi083 ceph-mon[77242]: pgmap v11786: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:10.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:01:10 smithi153 ceph-mon[70044]: pgmap v11786: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:10.864 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:01:11.156 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:01:11.157 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 7m ago 6h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:01:11.157 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 5m ago 6h 62.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:01:11.157 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 7m ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:01:11.157 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 5m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:01:11.158 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 5m ago 6h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:01:11.158 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 7m ago 6h 839M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:01:11.158 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 7m ago 6h 1452M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:01:11.158 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 5m ago 6h 625M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:01:11.159 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 7m ago 6h 838M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:01:11.159 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 7m ago 6h 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:01:11.159 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 5m ago 6h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:01:11.159 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 7m ago 6h 1047M 2402M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:01:11.160 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 7m ago 6h 1036M 2402M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:01:11.160 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 7m ago 6h 788M 2402M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:01:11.160 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 7m ago 6h 989M 2402M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:01:11.160 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 5m ago 6h 870M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:01:11.161 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 5m ago 6h 1113M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:01:11.161 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 5m ago 6h 971M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:01:11.161 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 5m ago 6h 1179M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:01:11.161 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 5m ago 6h 159M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:01:11.515 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:01:11.515 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:01:11.516 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:01:11.516 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:01:11.516 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:01:11.516 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:01:11.517 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:01:11.517 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:01:11.517 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:01:11.517 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:01:11.518 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:01:11.518 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:01:11.518 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:01:11.518 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:01:11.519 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:01:11.519 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:01:11.519 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:01:11.519 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:01:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:01:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:01:11] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T12:01:11.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:01:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:01:11.616Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:01:12.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:01:12 smithi083 ceph-mon[79930]: pgmap v11787: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:12.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:01:12 smithi083 ceph-mon[79930]: from='client.47838 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:01:12.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:01:12 smithi083 ceph-mon[79930]: from='client.47844 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:01:12.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:01:12 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2868076374' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:01:12.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:01:12 smithi083 ceph-mon[77242]: pgmap v11787: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:12.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:01:12 smithi083 ceph-mon[77242]: from='client.47838 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:01:12.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:01:12 smithi083 ceph-mon[77242]: from='client.47844 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:01:12.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:01:12 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2868076374' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:01:12.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:01:12 smithi153 ceph-mon[70044]: pgmap v11787: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:12.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:01:12 smithi153 ceph-mon[70044]: from='client.47838 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:01:12.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:01:12 smithi153 ceph-mon[70044]: from='client.47844 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:01:12.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:01:12 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2868076374' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:01:12.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:01:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:01:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:01:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:01:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:01:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:01:14.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:01:14 smithi083 ceph-mon[79930]: pgmap v11788: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:14.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:01:14 smithi083 ceph-mon[77242]: pgmap v11788: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:14.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:01:14 smithi153 ceph-mon[70044]: pgmap v11788: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:15.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:01:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:01:14.990Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:01:15.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:01:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:01:15.709Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:01:16.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:01:16 smithi083 ceph-mon[79930]: pgmap v11789: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:16.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:01:16 smithi083 ceph-mon[77242]: pgmap v11789: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:16.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:01:16 smithi153 ceph-mon[70044]: pgmap v11789: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:18.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:01:18 smithi083 ceph-mon[79930]: pgmap v11790: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:18.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:01:18 smithi083 ceph-mon[77242]: pgmap v11790: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:18.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:01:18 smithi153 ceph-mon[70044]: pgmap v11790: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:20.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:01:20 smithi083 ceph-mon[79930]: pgmap v11791: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:20.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:01:20 smithi083 ceph-mon[77242]: pgmap v11791: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:20.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:01:20 smithi153 ceph-mon[70044]: pgmap v11791: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:01:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:01:21] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T12:01:21.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:01:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:01:21.617Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:01:22.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:01:22 smithi083 ceph-mon[79930]: pgmap v11792: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:22.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:01:22 smithi083 ceph-mon[77242]: pgmap v11792: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:22.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:01:22 smithi153 ceph-mon[70044]: pgmap v11792: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:01:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:01:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:01:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:01:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:01:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:01:24.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:01:24 smithi083 ceph-mon[79930]: pgmap v11793: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:24.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:01:24 smithi083 ceph-mon[77242]: pgmap v11793: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:24.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:01:24 smithi153 ceph-mon[70044]: pgmap v11793: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:25.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:01:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:01:24.991Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:01:25.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:01:25 smithi083 ceph-mon[79930]: pgmap v11794: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:25.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:01:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:01:25.709Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:01:25.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:01:25 smithi083 ceph-mon[77242]: pgmap v11794: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:26.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:01:25 smithi153 ceph-mon[70044]: pgmap v11794: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:28.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:01:27 smithi083 ceph-mon[79930]: pgmap v11795: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:28.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:01:27 smithi083 ceph-mon[77242]: pgmap v11795: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:28.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:01:27 smithi153 ceph-mon[70044]: pgmap v11795: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:30.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:01:29 smithi083 ceph-mon[77242]: pgmap v11796: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:30.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:01:29 smithi083 ceph-mon[79930]: pgmap v11796: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:30.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:01:29 smithi153 ceph-mon[70044]: pgmap v11796: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:31.871 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:01:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:01:31] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T12:01:31.872 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:01:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:01:31.618Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:01:31.872 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:01:31 smithi083 ceph-mon[77242]: pgmap v11797: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:32.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:01:31 smithi083 ceph-mon[79930]: pgmap v11797: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:32.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:01:31 smithi153 ceph-mon[70044]: pgmap v11797: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:01:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:01:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:01:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:01:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:01:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:01:34.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:01:33 smithi083 ceph-mon[77242]: pgmap v11798: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:34.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:01:33 smithi083 ceph-mon[79930]: pgmap v11798: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:34.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:01:33 smithi153 ceph-mon[70044]: pgmap v11798: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:35.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:01:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:01:35.002Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:01:35.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:01:34 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:01:35.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:01:34 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:01:35.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:01:34 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:01:35.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:01:34 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:01:35.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:01:34 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:01:35.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:01:34 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:01:35.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:01:34 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:01:35.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:01:34 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:01:35.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:01:34 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:01:35.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:01:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:01:35.709Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:01:35.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:01:35 smithi083 ceph-mon[79930]: pgmap v11799: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:35.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:01:35 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:01:35.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:01:35 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:01:35.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:01:35 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:01:35.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:01:35 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:01:35.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:01:35 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:01:35.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:01:35 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:01:35.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:01:35 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:01:35.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:01:35 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:01:35.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:01:35 smithi083 ceph-mon[79930]: Adjusting osd_memory_target on smithi083 to 2393M 2022-09-29T12:01:35.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:01:35 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:01:35.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:01:35 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:01:35.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:01:35 smithi083 ceph-mon[77242]: pgmap v11799: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:35.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:01:35 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:01:35.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:01:35 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:01:35.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:01:35 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:01:35.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:01:35 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:01:35.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:01:35 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:01:35.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:01:35 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:01:35.968 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:01:35 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:01:35.968 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:01:35 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:01:35.968 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:01:35 smithi083 ceph-mon[77242]: Adjusting osd_memory_target on smithi083 to 2393M 2022-09-29T12:01:35.968 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:01:35 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:01:35.968 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:01:35 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:01:36.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:01:35 smithi153 ceph-mon[70044]: pgmap v11799: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:36.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:01:35 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:01:36.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:01:35 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:01:36.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:01:35 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:01:36.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:01:35 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:01:36.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:01:35 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:01:36.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:01:35 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:01:36.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:01:35 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:01:36.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:01:35 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:01:36.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:01:35 smithi153 ceph-mon[70044]: Adjusting osd_memory_target on smithi083 to 2393M 2022-09-29T12:01:36.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:01:35 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:01:36.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:01:35 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:01:38.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:01:37 smithi083 ceph-mon[79930]: pgmap v11800: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:38.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:01:37 smithi083 ceph-mon[77242]: pgmap v11800: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:38.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:01:37 smithi153 ceph-mon[70044]: pgmap v11800: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:01:39 smithi083 ceph-mon[79930]: pgmap v11801: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:01:39 smithi083 ceph-mon[77242]: pgmap v11801: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:01:39 smithi153 ceph-mon[70044]: pgmap v11801: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:41.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:01:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:01:41.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:01:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:01:41.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:01:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:01:41.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:01:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:01:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:01:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:01:41.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:01:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:01:41.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:01:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:01:41.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:01:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:01:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:01:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:01:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:01:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:01:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:01:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:01:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:01:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:01:41.843 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:01:41.889 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:01:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:01:41] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T12:01:41.889 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:01:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:01:41.618Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:01:42.134 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:01:42.134 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 8m ago 6h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:01:42.134 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 6m ago 6h 62.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:01:42.135 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 8m ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:01:42.135 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 6m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:01:42.135 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 6m ago 6h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:01:42.135 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 8m ago 6h 839M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:01:42.135 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 8m ago 6h 1452M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:01:42.136 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 6m ago 6h 625M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:01:42.136 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 8m ago 6h 838M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:01:42.136 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 8m ago 6h 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:01:42.136 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 6m ago 6h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:01:42.136 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 8m ago 6h 1047M 2393M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:01:42.137 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 8m ago 6h 1036M 2393M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:01:42.137 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 8m ago 6h 788M 2393M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:01:42.137 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 8m ago 6h 989M 2393M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:01:42.137 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 6m ago 6h 870M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:01:42.138 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 6m ago 6h 1113M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:01:42.138 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 6m ago 6h 971M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:01:42.138 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 6m ago 6h 1179M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:01:42.138 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 6m ago 6h 159M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:01:42.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:01:41 smithi083 ceph-mon[79930]: pgmap v11802: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:42.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:01:41 smithi083 ceph-mon[77242]: pgmap v11802: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:01:41 smithi153 ceph-mon[70044]: pgmap v11802: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:42.488 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:01:42.488 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:01:42.488 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:01:42.489 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:01:42.489 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:01:42.489 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:01:42.489 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:01:42.490 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:01:42.490 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:01:42.490 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:01:42.490 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:01:42.491 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:01:42.491 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:01:42.491 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:01:42.491 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:01:42.491 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:01:42.492 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:01:42.492 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:01:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:01:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:01:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:01:43.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:01:42 smithi083 ceph-mon[79930]: from='client.47856 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:01:43.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:01:42 smithi083 ceph-mon[79930]: from='client.57793 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:01:43.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:01:42 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2453338393' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:01:43.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:01:42 smithi083 ceph-mon[77242]: from='client.47856 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:01:43.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:01:42 smithi083 ceph-mon[77242]: from='client.57793 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:01:43.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:01:42 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2453338393' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:01:43.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:01:42 smithi153 ceph-mon[70044]: from='client.47856 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:01:43.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:01:42 smithi153 ceph-mon[70044]: from='client.57793 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:01:43.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:01:42 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2453338393' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:01:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:01:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:01:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:01:44.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:01:43 smithi083 ceph-mon[79930]: pgmap v11803: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:44.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:01:43 smithi083 ceph-mon[77242]: pgmap v11803: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:44.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:01:43 smithi153 ceph-mon[70044]: pgmap v11803: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:45.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:01:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:01:44.992Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:01:45.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:01:45 smithi083 ceph-mon[79930]: pgmap v11804: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:45.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:01:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:01:45.710Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:01:45.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:01:45 smithi083 ceph-mon[77242]: pgmap v11804: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:46.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:01:45 smithi153 ceph-mon[70044]: pgmap v11804: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:48.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:01:47 smithi083 ceph-mon[77242]: pgmap v11805: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:48.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:01:47 smithi083 ceph-mon[79930]: pgmap v11805: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:48.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:01:47 smithi153 ceph-mon[70044]: pgmap v11805: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:50.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:01:49 smithi083 ceph-mon[77242]: pgmap v11806: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:50.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:01:49 smithi083 ceph-mon[79930]: pgmap v11806: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:50.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:01:49 smithi153 ceph-mon[70044]: pgmap v11806: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:51.871 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:01:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:01:51] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T12:01:51.872 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:01:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:01:51.620Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:01:51.872 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:01:51 smithi083 ceph-mon[77242]: pgmap v11807: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:52.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:01:51 smithi083 ceph-mon[79930]: pgmap v11807: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:52.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:01:51 smithi153 ceph-mon[70044]: pgmap v11807: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:01:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:01:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:01:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:01:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:01:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:01:54.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:01:53 smithi083 ceph-mon[79930]: pgmap v11808: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:54.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:01:53 smithi083 ceph-mon[77242]: pgmap v11808: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:54.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:01:53 smithi153 ceph-mon[70044]: pgmap v11808: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:01:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:01:54.993Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:01:56.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:01:55 smithi083 ceph-mon[79930]: pgmap v11809: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:56.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:01:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:01:55.710Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:01:56.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:01:55 smithi083 ceph-mon[77242]: pgmap v11809: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:56.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:01:55 smithi153 ceph-mon[70044]: pgmap v11809: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:58.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:01:57 smithi083 ceph-mon[79930]: pgmap v11810: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:58.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:01:57 smithi083 ceph-mon[77242]: pgmap v11810: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:01:58.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:01:57 smithi153 ceph-mon[70044]: pgmap v11810: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:00.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:01:59 smithi083 ceph-mon[79930]: pgmap v11811: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:00.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:01:59 smithi083 ceph-mon[77242]: pgmap v11811: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:01:59 smithi153 ceph-mon[70044]: pgmap v11811: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:01.874 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:02:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:02:01] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T12:02:01.875 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:02:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:02:01.620Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:02:02.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:02:01 smithi083 ceph-mon[79930]: pgmap v11812: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:02.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:02:01 smithi083 ceph-mon[77242]: pgmap v11812: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:02.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:02:01 smithi153 ceph-mon[70044]: pgmap v11812: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:02:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:02:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:02:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:02:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:02:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:02:04.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:02:03 smithi083 ceph-mon[79930]: pgmap v11813: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:04.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:02:03 smithi083 ceph-mon[77242]: pgmap v11813: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:04.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:02:03 smithi153 ceph-mon[70044]: pgmap v11813: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:05.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:02:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:02:04.994Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:02:06.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:02:05 smithi083 ceph-mon[79930]: pgmap v11814: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:06.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:02:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:02:05.711Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:02:06.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:02:05 smithi083 ceph-mon[77242]: pgmap v11814: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:06.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:02:05 smithi153 ceph-mon[70044]: pgmap v11814: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:08.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:02:07 smithi083 ceph-mon[79930]: pgmap v11815: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:08.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:02:07 smithi083 ceph-mon[77242]: pgmap v11815: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:08.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:02:07 smithi153 ceph-mon[70044]: pgmap v11815: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:10.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:02:09 smithi083 ceph-mon[79930]: pgmap v11816: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:10.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:02:09 smithi083 ceph-mon[77242]: pgmap v11816: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:10.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:02:09 smithi153 ceph-mon[70044]: pgmap v11816: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:11.926 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:02:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:02:11] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T12:02:11.927 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:02:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:02:11.620Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:02:12.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:02:11 smithi083 ceph-mon[77242]: pgmap v11817: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:12.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:02:11 smithi083 ceph-mon[79930]: pgmap v11817: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:12.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:02:11 smithi153 ceph-mon[70044]: pgmap v11817: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:02:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:02:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:02:12.813 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:02:13.102 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:02:13.102 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 8m ago 6h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:02:13.102 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 6m ago 6h 62.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:02:13.103 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 8m ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:02:13.103 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 6m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:02:13.103 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 6m ago 6h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:02:13.103 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 8m ago 6h 839M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:02:13.104 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 8m ago 6h 1452M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:02:13.104 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 6m ago 6h 625M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:02:13.104 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 8m ago 6h 838M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:02:13.104 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 8m ago 6h 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:02:13.105 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 6m ago 6h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:02:13.105 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 8m ago 6h 1047M 2393M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:02:13.105 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 8m ago 6h 1036M 2393M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:02:13.105 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 8m ago 6h 788M 2393M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:02:13.105 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 8m ago 6h 989M 2393M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:02:13.106 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 6m ago 6h 870M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:02:13.106 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 6m ago 6h 1113M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:02:13.106 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 6m ago 6h 971M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:02:13.106 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 6m ago 6h 1179M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:02:13.107 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 6m ago 6h 159M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:02:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:02:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:02:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:02:13.466 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:02:13.466 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:02:13.466 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:02:13.466 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:02:13.467 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:02:13.467 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:02:13.467 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:02:13.467 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:02:13.468 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:02:13.468 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:02:13.468 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:02:13.468 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:02:13.468 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:02:13.469 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:02:13.469 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:02:13.469 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:02:13.469 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:02:13.469 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:02:14.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:02:13 smithi083 ceph-mon[79930]: from='client.47874 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:02:14.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:02:13 smithi083 ceph-mon[79930]: pgmap v11818: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:14.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:02:13 smithi083 ceph-mon[79930]: from='client.57811 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:02:14.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:02:13 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/193489708' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:02:14.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:02:13 smithi083 ceph-mon[77242]: from='client.47874 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:02:14.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:02:13 smithi083 ceph-mon[77242]: pgmap v11818: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:14.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:02:13 smithi083 ceph-mon[77242]: from='client.57811 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:02:14.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:02:13 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/193489708' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:02:14.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:02:13 smithi153 ceph-mon[70044]: from='client.47874 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:02:14.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:02:13 smithi153 ceph-mon[70044]: pgmap v11818: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:14.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:02:13 smithi153 ceph-mon[70044]: from='client.57811 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:02:14.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:02:13 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/193489708' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:02:15.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:02:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:02:14.994Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:02:16.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:02:15 smithi083 ceph-mon[79930]: pgmap v11819: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:16.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:02:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:02:15.711Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:02:16.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:02:15 smithi083 ceph-mon[77242]: pgmap v11819: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:16.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:02:15 smithi153 ceph-mon[70044]: pgmap v11819: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:18.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:02:17 smithi083 ceph-mon[79930]: pgmap v11820: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:18.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:02:17 smithi083 ceph-mon[77242]: pgmap v11820: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:18.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:02:17 smithi153 ceph-mon[70044]: pgmap v11820: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:20.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:02:19 smithi083 ceph-mon[79930]: pgmap v11821: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:20.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:02:19 smithi083 ceph-mon[77242]: pgmap v11821: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:02:19 smithi153 ceph-mon[70044]: pgmap v11821: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:21.883 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:02:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:02:21] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T12:02:21.883 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:02:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:02:21.620Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:02:22.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:02:21 smithi083 ceph-mon[79930]: pgmap v11822: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:22.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:02:21 smithi083 ceph-mon[77242]: pgmap v11822: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:22.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:02:21 smithi153 ceph-mon[70044]: pgmap v11822: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:02:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:02:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:02:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:02:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:02:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:02:24.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:02:23 smithi083 ceph-mon[79930]: pgmap v11823: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:24.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:02:23 smithi083 ceph-mon[77242]: pgmap v11823: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:24.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:02:23 smithi153 ceph-mon[70044]: pgmap v11823: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:25.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:02:24 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:02:24.995Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:02:26.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:02:25 smithi083 ceph-mon[79930]: pgmap v11824: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:26.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:02:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:02:25.712Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:02:26.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:02:25 smithi083 ceph-mon[77242]: pgmap v11824: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:26.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:02:25 smithi153 ceph-mon[70044]: pgmap v11824: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:28.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:02:27 smithi083 ceph-mon[79930]: pgmap v11825: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:28.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:02:27 smithi083 ceph-mon[77242]: pgmap v11825: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:28.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:02:27 smithi153 ceph-mon[70044]: pgmap v11825: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:30.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:02:29 smithi083 ceph-mon[79930]: pgmap v11826: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:30.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:02:29 smithi083 ceph-mon[77242]: pgmap v11826: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:30.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:02:29 smithi153 ceph-mon[70044]: pgmap v11826: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:31.933 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:02:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:02:31] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T12:02:31.933 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:02:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:02:31.621Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:02:32.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:02:31 smithi083 ceph-mon[77242]: pgmap v11827: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:32.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:02:31 smithi083 ceph-mon[79930]: pgmap v11827: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:32.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:02:31 smithi153 ceph-mon[70044]: pgmap v11827: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:02:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:02:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:02:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:02:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:02:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:02:34.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:02:33 smithi083 ceph-mon[77242]: pgmap v11828: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:34.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:02:33 smithi083 ceph-mon[79930]: pgmap v11828: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:34.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:02:33 smithi153 ceph-mon[70044]: pgmap v11828: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:35.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:02:34 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:02:34.996Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:02:36.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:02:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:02:35.712Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:02:36.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:02:35 smithi083 ceph-mon[79930]: pgmap v11829: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:36.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:02:35 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:02:36.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:02:35 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:02:36.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:02:35 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:02:36.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:02:35 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:02:36.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:02:35 smithi083 ceph-mon[77242]: pgmap v11829: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:36.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:02:35 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:02:36.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:02:35 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:02:36.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:02:35 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:02:36.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:02:35 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:02:36.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:02:35 smithi153 ceph-mon[70044]: pgmap v11829: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:36.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:02:35 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:02:36.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:02:35 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:02:36.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:02:35 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:02:36.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:02:35 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:02:38.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:02:37 smithi083 ceph-mon[79930]: pgmap v11830: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:38.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:02:37 smithi083 ceph-mon[77242]: pgmap v11830: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:38.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:02:37 smithi153 ceph-mon[70044]: pgmap v11830: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:02:39 smithi083 ceph-mon[79930]: pgmap v11831: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:02:39 smithi083 ceph-mon[77242]: pgmap v11831: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:02:39 smithi153 ceph-mon[70044]: pgmap v11831: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:41.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:02:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:02:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:02:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:02:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:02:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:02:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:02:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:02:41.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:02:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:02:41.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:02:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:02:41.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:02:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:02:41.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:02:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:02:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:02:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:02:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:02:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:02:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:02:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:02:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:02:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:02:41.939 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:02:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:02:41] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T12:02:41.940 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:02:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:02:41.622Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:02:42.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:02:41 smithi083 ceph-mon[79930]: pgmap v11832: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:42.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:02:41 smithi083 ceph-mon[77242]: pgmap v11832: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:02:41 smithi153 ceph-mon[70044]: pgmap v11832: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:02:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:02:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:02:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:02:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:02:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:02:43.798 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:02:44.091 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:02:44.091 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 9m ago 6h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:02:44.091 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 7m ago 6h 62.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:02:44.092 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 9m ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:02:44.092 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 7m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:02:44.092 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 7m ago 6h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:02:44.092 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 9m ago 6h 839M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:02:44.093 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 9m ago 6h 1452M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:02:44.093 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 7m ago 6h 625M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:02:44.093 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 9m ago 6h 838M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:02:44.093 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 9m ago 6h 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:02:44.094 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 7m ago 6h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:02:44.094 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 9m ago 6h 1047M 2393M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:02:44.094 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 9m ago 6h 1036M 2393M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:02:44.094 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 9m ago 6h 788M 2393M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:02:44.095 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 9m ago 6h 989M 2393M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:02:44.095 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 7m ago 6h 870M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:02:44.095 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 7m ago 6h 1113M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:02:44.095 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 7m ago 6h 971M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:02:44.096 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 7m ago 6h 1179M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:02:44.096 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 7m ago 6h 159M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:02:44.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:02:43 smithi083 ceph-mon[79930]: pgmap v11833: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:44.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:02:43 smithi083 ceph-mon[77242]: pgmap v11833: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:44.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:02:43 smithi153 ceph-mon[70044]: pgmap v11833: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:44.447 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:02:44.448 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:02:44.448 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:02:44.448 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:02:44.449 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:02:44.449 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:02:44.449 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:02:44.449 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:02:44.450 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:02:44.450 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:02:44.450 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:02:44.450 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:02:44.450 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:02:44.451 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:02:44.451 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:02:44.451 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:02:44.451 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:02:44.452 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:02:45.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:02:44 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:02:44.996Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:02:45.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:02:44 smithi083 ceph-mon[77242]: from='client.47892 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:02:45.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:02:44 smithi083 ceph-mon[77242]: from='client.57829 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:02:45.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:02:44 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/833959529' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:02:45.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:02:44 smithi083 ceph-mon[79930]: from='client.47892 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:02:45.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:02:44 smithi083 ceph-mon[79930]: from='client.57829 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:02:45.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:02:44 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/833959529' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:02:45.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:02:44 smithi153 ceph-mon[70044]: from='client.47892 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:02:45.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:02:44 smithi153 ceph-mon[70044]: from='client.57829 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:02:45.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:02:44 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/833959529' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:02:46.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:02:45 smithi083 ceph-mon[79930]: pgmap v11834: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:46.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:02:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:02:45.713Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:02:46.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:02:45 smithi083 ceph-mon[77242]: pgmap v11834: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:46.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:02:45 smithi153 ceph-mon[70044]: pgmap v11834: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:48.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:02:47 smithi083 ceph-mon[79930]: pgmap v11835: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:48.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:02:47 smithi083 ceph-mon[77242]: pgmap v11835: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:48.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:02:47 smithi153 ceph-mon[70044]: pgmap v11835: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:50.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:02:49 smithi083 ceph-mon[77242]: pgmap v11836: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:50.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:02:49 smithi083 ceph-mon[79930]: pgmap v11836: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:50.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:02:49 smithi153 ceph-mon[70044]: pgmap v11836: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:51.959 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:02:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:02:51] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T12:02:51.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:02:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:02:51.622Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:02:52.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:02:51 smithi083 ceph-mon[77242]: pgmap v11837: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:52.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:02:51 smithi083 ceph-mon[79930]: pgmap v11837: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:52.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:02:51 smithi153 ceph-mon[70044]: pgmap v11837: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:02:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:02:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:02:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:02:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:02:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:02:54.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:02:53 smithi083 ceph-mon[77242]: pgmap v11838: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:54.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:02:53 smithi083 ceph-mon[79930]: pgmap v11838: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:54.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:02:53 smithi153 ceph-mon[70044]: pgmap v11838: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:02:54 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:02:54.998Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:02:56.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:02:55 smithi083 ceph-mon[77242]: pgmap v11839: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:56.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:02:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:02:55.714Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:02:56.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:02:55 smithi083 ceph-mon[79930]: pgmap v11839: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:56.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:02:55 smithi153 ceph-mon[70044]: pgmap v11839: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:58.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:02:57 smithi083 ceph-mon[77242]: pgmap v11840: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:58.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:02:57 smithi083 ceph-mon[79930]: pgmap v11840: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:02:58.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:02:57 smithi153 ceph-mon[70044]: pgmap v11840: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:00.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:02:59 smithi083 ceph-mon[79930]: pgmap v11841: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:00.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:02:59 smithi083 ceph-mon[77242]: pgmap v11841: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:02:59 smithi153 ceph-mon[70044]: pgmap v11841: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:01.943 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:03:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:03:01] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T12:03:01.943 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:03:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:03:01.623Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:03:02.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:03:01 smithi083 ceph-mon[77242]: pgmap v11842: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:02.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:03:01 smithi083 ceph-mon[79930]: pgmap v11842: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:02.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:03:01 smithi153 ceph-mon[70044]: pgmap v11842: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:03:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:03:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:03:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:03:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:03:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:03:04.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:03:03 smithi083 ceph-mon[79930]: pgmap v11843: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:04.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:03:03 smithi083 ceph-mon[77242]: pgmap v11843: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:04.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:03:03 smithi153 ceph-mon[70044]: pgmap v11843: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:05.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:03:04 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:03:04.998Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:03:06.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:03:05 smithi083 ceph-mon[77242]: pgmap v11844: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:06.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:03:05 smithi083 ceph-mon[79930]: pgmap v11844: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:06.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:03:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:03:05.715Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:03:06.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:03:05 smithi153 ceph-mon[70044]: pgmap v11844: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:08.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:03:07 smithi083 ceph-mon[77242]: pgmap v11845: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:08.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:03:07 smithi083 ceph-mon[79930]: pgmap v11845: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:08.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:03:07 smithi153 ceph-mon[70044]: pgmap v11845: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:10.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:03:09 smithi083 ceph-mon[77242]: pgmap v11846: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:10.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:03:09 smithi083 ceph-mon[79930]: pgmap v11846: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:10.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:03:09 smithi153 ceph-mon[70044]: pgmap v11846: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:11.947 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:03:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:03:11] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T12:03:11.948 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:03:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:03:11.625Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:03:12.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:03:11 smithi083 ceph-mon[77242]: pgmap v11847: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:12.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:03:11 smithi083 ceph-mon[79930]: pgmap v11847: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:12.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:03:11 smithi153 ceph-mon[70044]: pgmap v11847: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:03:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:03:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:03:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:03:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:03:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:03:14.203 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:03:13 smithi083 ceph-mon[77242]: pgmap v11848: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:14.204 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:03:13 smithi083 ceph-mon[79930]: pgmap v11848: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:14.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:03:13 smithi153 ceph-mon[70044]: pgmap v11848: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:14.776 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:03:15.011 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:03:14 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:03:14.998Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:03:15.095 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:03:15.095 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 9m ago 6h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:03:15.095 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 7m ago 6h 62.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:03:15.095 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 9m ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:03:15.096 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 7m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:03:15.096 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 7m ago 6h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:03:15.096 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 9m ago 6h 839M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:03:15.096 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 9m ago 6h 1452M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:03:15.096 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 7m ago 6h 625M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:03:15.097 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 9m ago 6h 838M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:03:15.097 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 9m ago 6h 25.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:03:15.097 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 7m ago 6h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:03:15.098 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 9m ago 6h 1047M 2393M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:03:15.103 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 9m ago 6h 1036M 2393M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:03:15.103 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 9m ago 6h 788M 2393M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:03:15.103 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 9m ago 6h 989M 2393M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:03:15.103 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 7m ago 6h 870M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:03:15.104 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 7m ago 6h 1113M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:03:15.104 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 7m ago 6h 971M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:03:15.104 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 7m ago 6h 1179M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:03:15.105 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 7m ago 6h 159M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:03:15.476 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:03:15.476 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:03:15.476 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:03:15.477 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:03:15.477 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:03:15.477 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:03:15.477 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:03:15.478 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:03:15.478 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:03:15.478 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:03:15.478 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:03:15.478 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:03:15.479 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:03:15.479 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:03:15.479 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:03:15.479 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:03:15.480 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:03:15.480 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:03:16.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:03:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:03:15.715Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:03:16.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:03:15 smithi083 ceph-mon[77242]: from='client.57841 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:03:16.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:03:15 smithi083 ceph-mon[77242]: pgmap v11849: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:16.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:03:15 smithi083 ceph-mon[77242]: from='client.47916 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:03:16.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:03:15 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1842007630' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:03:16.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:03:15 smithi083 ceph-mon[79930]: from='client.57841 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:03:16.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:03:15 smithi083 ceph-mon[79930]: pgmap v11849: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:16.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:03:15 smithi083 ceph-mon[79930]: from='client.47916 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:03:16.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:03:15 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1842007630' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:03:16.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:03:15 smithi153 ceph-mon[70044]: from='client.57841 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:03:16.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:03:15 smithi153 ceph-mon[70044]: pgmap v11849: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:16.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:03:15 smithi153 ceph-mon[70044]: from='client.47916 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:03:16.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:03:15 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1842007630' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:03:18.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:03:17 smithi083 ceph-mon[79930]: pgmap v11850: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:18.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:03:17 smithi083 ceph-mon[77242]: pgmap v11850: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:18.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:03:17 smithi153 ceph-mon[70044]: pgmap v11850: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:20.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:03:19 smithi083 ceph-mon[77242]: pgmap v11851: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:20.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:03:19 smithi083 ceph-mon[79930]: pgmap v11851: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:03:19 smithi153 ceph-mon[70044]: pgmap v11851: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:21.951 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:03:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:03:21] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T12:03:21.951 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:03:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:03:21.625Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:03:22.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:03:21 smithi083 ceph-mon[77242]: pgmap v11852: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:22.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:03:21 smithi083 ceph-mon[79930]: pgmap v11852: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:22.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:03:21 smithi153 ceph-mon[70044]: pgmap v11852: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:03:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:03:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:03:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:03:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:03:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:03:24.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:03:23 smithi083 ceph-mon[77242]: pgmap v11853: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:24.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:03:23 smithi083 ceph-mon[79930]: pgmap v11853: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:24.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:03:23 smithi153 ceph-mon[70044]: pgmap v11853: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:25.391 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:03:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:03:24.999Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:03:25.887 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:03:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:03:25.716Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:03:26.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:03:25 smithi083 ceph-mon[79930]: pgmap v11854: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:26.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:03:25 smithi083 ceph-mon[77242]: pgmap v11854: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:26.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:03:25 smithi153 ceph-mon[70044]: pgmap v11854: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:28.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:03:27 smithi083 ceph-mon[79930]: pgmap v11855: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:28.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:03:27 smithi083 ceph-mon[77242]: pgmap v11855: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:28.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:03:27 smithi153 ceph-mon[70044]: pgmap v11855: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:30.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:03:29 smithi083 ceph-mon[79930]: pgmap v11856: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:30.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:03:29 smithi083 ceph-mon[77242]: pgmap v11856: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:30.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:03:29 smithi153 ceph-mon[70044]: pgmap v11856: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:31.953 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:03:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:03:31] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T12:03:31.954 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:03:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:03:31.626Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:03:32.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:03:31 smithi083 ceph-mon[77242]: pgmap v11857: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:32.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:03:31 smithi083 ceph-mon[79930]: pgmap v11857: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:32.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:03:31 smithi153 ceph-mon[70044]: pgmap v11857: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:03:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:03:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:03:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:03:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:03:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:03:34.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:03:33 smithi083 ceph-mon[79930]: pgmap v11858: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:34.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:03:33 smithi083 ceph-mon[77242]: pgmap v11858: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:34.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:03:33 smithi153 ceph-mon[70044]: pgmap v11858: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:35.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:03:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:03:35.000Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:03:36.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:03:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:03:35.718Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:03:36.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:03:35 smithi083 ceph-mon[79930]: pgmap v11859: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:36.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:03:35 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:03:36.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:03:35 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:03:36.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:03:35 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:03:36.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:03:35 smithi083 ceph-mon[77242]: pgmap v11859: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:36.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:03:35 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:03:36.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:03:35 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:03:36.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:03:35 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:03:36.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:03:35 smithi153 ceph-mon[70044]: pgmap v11859: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:36.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:03:35 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:03:36.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:03:35 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:03:36.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:03:35 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:03:37.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:03:37 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:03:37.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:03:37 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:03:37.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:03:37 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:03:37.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:03:37 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:03:37.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:03:37 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:03:37.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:03:37 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:03:37.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:03:37 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:03:37.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:03:37 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:03:37.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:03:37 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:03:37.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:03:37 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:03:37.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:03:37 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:03:37.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:03:37 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:03:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:03:38 smithi153 ceph-mon[70044]: pgmap v11860: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:38.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:03:38 smithi083 ceph-mon[79930]: pgmap v11860: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:38.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:03:38 smithi083 ceph-mon[77242]: pgmap v11860: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:03:40 smithi153 ceph-mon[70044]: pgmap v11861: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:03:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:03:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:03:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:03:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:03:40 smithi083 ceph-mon[79930]: pgmap v11861: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:03:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:03:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:03:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:03:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:03:40 smithi083 ceph-mon[77242]: pgmap v11861: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:03:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:03:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:03:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:03:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:03:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:03:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:03:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:03:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:03:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:03:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:03:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:03:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:03:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:03:41.627 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:03:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:03:41] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T12:03:41.628 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:03:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:03:41.628 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:03:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:03:41.629 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:03:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:03:41.629 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:03:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:03:41.629 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:03:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:03:41.629 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:03:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:03:41.630 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:03:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:03:41.630 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:03:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:03:41.630 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:03:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:03:41.630 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:03:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:03:41.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:03:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:03:41.625Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:03:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:03:42 smithi153 ceph-mon[70044]: pgmap v11862: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:42.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:03:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:03:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:03:42.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:03:42 smithi083 ceph-mon[77242]: pgmap v11862: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:42.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:03:42 smithi083 ceph-mon[79930]: pgmap v11862: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:03:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:03:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:03:44.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:03:44 smithi153 ceph-mon[70044]: pgmap v11863: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:44.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:03:44 smithi083 ceph-mon[77242]: pgmap v11863: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:44.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:03:44 smithi083 ceph-mon[79930]: pgmap v11863: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:45.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:03:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:03:45.001Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:03:45.807 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:03:46.097 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:03:46.098 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 5s ago 6h 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:03:46.098 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 8m ago 6h 62.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:03:46.098 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 5s ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:03:46.098 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 8m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:03:46.099 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 8m ago 6h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:03:46.099 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 5s ago 6h 847M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:03:46.099 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 5s ago 6h 1486M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:03:46.099 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 8m ago 6h 625M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:03:46.100 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 5s ago 6h 857M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:03:46.100 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 5s ago 6h 26.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:03:46.100 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 8m ago 6h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:03:46.100 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 5s ago 6h 1074M 2393M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:03:46.101 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 5s ago 6h 1061M 2393M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:03:46.101 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 5s ago 6h 808M 2393M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:03:46.101 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 5s ago 6h 1039M 2393M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:03:46.101 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 8m ago 6h 870M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:03:46.102 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 8m ago 6h 1113M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:03:46.102 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 8m ago 6h 971M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:03:46.102 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 8m ago 6h 1179M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:03:46.102 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 8m ago 6h 159M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:03:46.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:03:45 smithi083 ceph-mon[79930]: pgmap v11864: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:46.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:03:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:03:45.718Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:03:46.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:03:45 smithi083 ceph-mon[77242]: pgmap v11864: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:46.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:03:45 smithi153 ceph-mon[70044]: pgmap v11864: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:46.453 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:03:46.453 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:03:46.453 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:03:46.454 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:03:46.454 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:03:46.454 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:03:46.454 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:03:46.455 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:03:46.455 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:03:46.455 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:03:46.455 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:03:46.456 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:03:46.456 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:03:46.456 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:03:46.456 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:03:46.457 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:03:46.457 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:03:46.457 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:03:47.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:03:46 smithi083 ceph-mon[77242]: from='client.47928 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:03:47.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:03:46 smithi083 ceph-mon[77242]: from='client.57865 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:03:47.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:03:46 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2697858202' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:03:47.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:03:46 smithi083 ceph-mon[79930]: from='client.47928 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:03:47.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:03:46 smithi083 ceph-mon[79930]: from='client.57865 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:03:47.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:03:46 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2697858202' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:03:47.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:03:46 smithi153 ceph-mon[70044]: from='client.47928 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:03:47.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:03:46 smithi153 ceph-mon[70044]: from='client.57865 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:03:47.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:03:46 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2697858202' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:03:48.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:03:47 smithi083 ceph-mon[79930]: pgmap v11865: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:48.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:03:47 smithi083 ceph-mon[77242]: pgmap v11865: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:48.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:03:47 smithi153 ceph-mon[70044]: pgmap v11865: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:50.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:03:49 smithi083 ceph-mon[79930]: pgmap v11866: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:50.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:03:49 smithi083 ceph-mon[77242]: pgmap v11866: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:50.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:03:49 smithi153 ceph-mon[70044]: pgmap v11866: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:51.956 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:03:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:03:51] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T12:03:51.956 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:03:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:03:51.626Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:03:52.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:03:51 smithi083 ceph-mon[77242]: pgmap v11867: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:52.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:03:51 smithi083 ceph-mon[79930]: pgmap v11867: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:52.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:03:51 smithi153 ceph-mon[70044]: pgmap v11867: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:03:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:03:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:03:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:03:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:03:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:03:54.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:03:53 smithi083 ceph-mon[77242]: pgmap v11868: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:54.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:03:53 smithi083 ceph-mon[79930]: pgmap v11868: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:54.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:03:53 smithi153 ceph-mon[70044]: pgmap v11868: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:03:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:03:55.002Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:03:56.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:03:55 smithi083 ceph-mon[77242]: pgmap v11869: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:56.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:03:55 smithi083 ceph-mon[79930]: pgmap v11869: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:56.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:03:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:03:55.720Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:03:56.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:03:55 smithi153 ceph-mon[70044]: pgmap v11869: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:58.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:03:57 smithi083 ceph-mon[79930]: pgmap v11870: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:58.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:03:57 smithi083 ceph-mon[77242]: pgmap v11870: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:03:58.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:03:57 smithi153 ceph-mon[70044]: pgmap v11870: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:00.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:03:59 smithi083 ceph-mon[77242]: pgmap v11871: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:00.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:03:59 smithi083 ceph-mon[79930]: pgmap v11871: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:03:59 smithi153 ceph-mon[70044]: pgmap v11871: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:04:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:04:01] "GET /metrics HTTP/1.1" 200 225227 "" "Prometheus/2.33.4" 2022-09-29T12:04:01.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:04:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:04:01.627Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:04:02.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:04:01 smithi083 ceph-mon[79930]: pgmap v11872: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:02.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:04:01 smithi083 ceph-mon[77242]: pgmap v11872: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:02.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:04:01 smithi153 ceph-mon[70044]: pgmap v11872: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:04:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:04:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:04:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:04:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:04:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:04:04.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:04:03 smithi153 ceph-mon[70044]: pgmap v11873: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:04.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:04:03 smithi083 ceph-mon[79930]: pgmap v11873: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:04.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:04:03 smithi083 ceph-mon[77242]: pgmap v11873: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:05.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:04:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:04:05.002Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:04:06.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:04:05 smithi083 ceph-mon[79930]: pgmap v11874: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:06.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:04:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:04:05.721Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:04:06.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:04:05 smithi083 ceph-mon[77242]: pgmap v11874: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:06.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:04:05 smithi153 ceph-mon[70044]: pgmap v11874: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:08.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:04:07 smithi083 ceph-mon[79930]: pgmap v11875: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:08.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:04:07 smithi083 ceph-mon[77242]: pgmap v11875: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:08.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:04:07 smithi153 ceph-mon[70044]: pgmap v11875: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:10.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:04:09 smithi153 ceph-mon[70044]: pgmap v11876: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:10.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:04:09 smithi083 ceph-mon[77242]: pgmap v11876: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:10.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:04:09 smithi083 ceph-mon[79930]: pgmap v11876: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:04:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:04:11] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T12:04:11.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:04:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:04:11.628Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:04:12.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:04:11 smithi153 ceph-mon[70044]: pgmap v11877: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:12.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:04:11 smithi083 ceph-mon[79930]: pgmap v11877: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:12.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:04:11 smithi083 ceph-mon[77242]: pgmap v11877: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:04:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:04:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:04:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:04:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:04:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:04:14.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:04:13 smithi153 ceph-mon[70044]: pgmap v11878: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:14.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:04:13 smithi083 ceph-mon[79930]: pgmap v11878: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:14.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:04:13 smithi083 ceph-mon[77242]: pgmap v11878: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:15.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:04:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:04:15.003Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:04:16.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:04:15 smithi083 ceph-mon[77242]: pgmap v11879: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:16.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:04:15 smithi083 ceph-mon[79930]: pgmap v11879: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:16.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:04:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:04:15.721Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:04:16.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:04:15 smithi153 ceph-mon[70044]: pgmap v11879: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:16.780 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:04:17.069 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:04:17.070 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 36s ago 6h 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:04:17.070 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 8m ago 6h 62.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:04:17.070 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 36s ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:04:17.070 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 8m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:04:17.071 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 8m ago 6h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:04:17.071 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 36s ago 6h 847M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:04:17.071 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 36s ago 6h 1486M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:04:17.071 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 8m ago 6h 625M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:04:17.071 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 36s ago 6h 857M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:04:17.072 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 36s ago 6h 26.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:04:17.072 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 8m ago 6h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:04:17.072 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 36s ago 6h 1074M 2393M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:04:17.072 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 36s ago 6h 1061M 2393M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:04:17.073 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 36s ago 6h 808M 2393M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:04:17.073 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 36s ago 6h 1039M 2393M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:04:17.073 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 8m ago 6h 870M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:04:17.073 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 8m ago 6h 1113M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:04:17.074 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 8m ago 6h 971M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:04:17.074 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 8m ago 6h 1179M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:04:17.074 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 8m ago 6h 159M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:04:17.425 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:04:17.425 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:04:17.426 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:04:17.426 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:04:17.426 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:04:17.426 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:04:17.427 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:04:17.427 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:04:17.427 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:04:17.427 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:04:17.428 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:04:17.428 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:04:17.428 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:04:17.428 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:04:17.429 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:04:17.429 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:04:17.429 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:04:17.429 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:04:18.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:04:17 smithi083 ceph-mon[79930]: from='client.57877 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:04:18.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:04:17 smithi083 ceph-mon[79930]: pgmap v11880: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:18.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:04:17 smithi083 ceph-mon[79930]: from='client.57883 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:04:18.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:04:17 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3604418461' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:04:18.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:04:17 smithi083 ceph-mon[77242]: from='client.57877 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:04:18.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:04:17 smithi083 ceph-mon[77242]: pgmap v11880: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:18.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:04:17 smithi083 ceph-mon[77242]: from='client.57883 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:04:18.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:04:17 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3604418461' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:04:18.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:04:17 smithi153 ceph-mon[70044]: from='client.57877 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:04:18.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:04:17 smithi153 ceph-mon[70044]: pgmap v11880: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:18.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:04:17 smithi153 ceph-mon[70044]: from='client.57883 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:04:18.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:04:17 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3604418461' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:04:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:04:19 smithi153 ceph-mon[70044]: pgmap v11881: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:20.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:04:19 smithi083 ceph-mon[79930]: pgmap v11881: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:20.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:04:19 smithi083 ceph-mon[77242]: pgmap v11881: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:04:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:04:21] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T12:04:21.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:04:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:04:21.629Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:04:22.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:04:21 smithi153 ceph-mon[70044]: pgmap v11882: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:22.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:04:21 smithi083 ceph-mon[77242]: pgmap v11882: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:22.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:04:21 smithi083 ceph-mon[79930]: pgmap v11882: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:04:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:04:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:04:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:04:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:04:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:04:24.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:04:23 smithi153 ceph-mon[70044]: pgmap v11883: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:24.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:04:23 smithi083 ceph-mon[77242]: pgmap v11883: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:24.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:04:23 smithi083 ceph-mon[79930]: pgmap v11883: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:25.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:04:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:04:25.003Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:04:26.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:04:25 smithi083 ceph-mon[79930]: pgmap v11884: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:26.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:04:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:04:25.721Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:04:26.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:04:25 smithi083 ceph-mon[77242]: pgmap v11884: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:26.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:04:25 smithi153 ceph-mon[70044]: pgmap v11884: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:28.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:04:27 smithi153 ceph-mon[70044]: pgmap v11885: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:28.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:04:27 smithi083 ceph-mon[77242]: pgmap v11885: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:28.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:04:27 smithi083 ceph-mon[79930]: pgmap v11885: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:30.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:04:29 smithi153 ceph-mon[70044]: pgmap v11886: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:30.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:04:29 smithi083 ceph-mon[79930]: pgmap v11886: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:30.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:04:29 smithi083 ceph-mon[77242]: pgmap v11886: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:04:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:04:31] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T12:04:31.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:04:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:04:31.629Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:04:32.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:04:31 smithi153 ceph-mon[70044]: pgmap v11887: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:32.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:04:31 smithi083 ceph-mon[79930]: pgmap v11887: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:32.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:04:31 smithi083 ceph-mon[77242]: pgmap v11887: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:04:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:04:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:04:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:04:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:04:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:04:34.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:04:33 smithi153 ceph-mon[70044]: pgmap v11888: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:34.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:04:33 smithi083 ceph-mon[79930]: pgmap v11888: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:34.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:04:33 smithi083 ceph-mon[77242]: pgmap v11888: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:35.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:04:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:04:35.004Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:04:36.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:04:35 smithi083 ceph-mon[79930]: pgmap v11889: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:36.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:04:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:04:35.722Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:04:36.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:04:35 smithi083 ceph-mon[77242]: pgmap v11889: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:36.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:04:35 smithi153 ceph-mon[70044]: pgmap v11889: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:38.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:04:37 smithi153 ceph-mon[70044]: pgmap v11890: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:38.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:04:37 smithi083 ceph-mon[79930]: pgmap v11890: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:38.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:04:37 smithi083 ceph-mon[77242]: pgmap v11890: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:04:39 smithi153 ceph-mon[70044]: pgmap v11891: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:04:39 smithi083 ceph-mon[79930]: pgmap v11891: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:04:39 smithi083 ceph-mon[77242]: pgmap v11891: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:04:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:04:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:04:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:04:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:04:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:04:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:04:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:04:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:04:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:04:41.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:04:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:04:41.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:04:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:04:41.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:04:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:04:41.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:04:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:04:41.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:04:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:04:41.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:04:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:04:41.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:04:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:04:41.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:04:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:04:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:04:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:04:41.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:04:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:04:41.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:04:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:04:41.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:04:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:04:41.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:04:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:04:41.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:04:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:04:41.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:04:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:04:41.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:04:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:04:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:04:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:04:41] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T12:04:41.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:04:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:04:41.630Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:04:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:04:42 smithi083 ceph-mon[79930]: pgmap v11892: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:42.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:04:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:04:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:04:42 smithi083 ceph-mon[77242]: pgmap v11892: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:04:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:04:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:04:42 smithi153 ceph-mon[70044]: pgmap v11892: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:04:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:04:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:04:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:04:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:04:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:04:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:04:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:04:44.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:04:44 smithi083 ceph-mon[77242]: pgmap v11893: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:44.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:04:44 smithi083 ceph-mon[79930]: pgmap v11893: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:44.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:04:44 smithi153 ceph-mon[70044]: pgmap v11893: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:45.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:04:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:04:45.005Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:04:46.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:04:45 smithi083 ceph-mon[77242]: pgmap v11894: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:46.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:04:45 smithi083 ceph-mon[79930]: pgmap v11894: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:46.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:04:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:04:45.722Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:04:46.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:04:45 smithi153 ceph-mon[70044]: pgmap v11894: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:47.752 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:04:48.042 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:04:48.042 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 67s ago 6h 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:04:48.042 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 9m ago 6h 62.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:04:48.043 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 67s ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:04:48.043 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 9m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:04:48.043 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 9m ago 6h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:04:48.043 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 67s ago 6h 847M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:04:48.044 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 67s ago 6h 1486M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:04:48.044 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 9m ago 6h 625M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:04:48.044 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 67s ago 6h 857M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:04:48.044 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 67s ago 6h 26.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:04:48.044 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 9m ago 6h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:04:48.045 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 67s ago 6h 1074M 2393M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:04:48.045 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 67s ago 6h 1061M 2393M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:04:48.045 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 67s ago 6h 808M 2393M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:04:48.045 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 67s ago 6h 1039M 2393M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:04:48.046 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 9m ago 6h 870M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:04:48.046 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 9m ago 6h 1113M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:04:48.046 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 9m ago 6h 971M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:04:48.046 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 9m ago 6h 1179M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:04:48.046 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 9m ago 6h 159M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:04:48.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:04:47 smithi153 ceph-mon[70044]: pgmap v11895: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:48.396 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:04:48.397 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:04:48.397 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:04:48.397 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:04:48.397 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:04:48.398 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:04:48.398 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:04:48.398 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:04:48.399 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:04:48.399 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:04:48.399 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:04:48.399 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:04:48.399 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:04:48.400 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:04:48.400 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:04:48.400 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:04:48.400 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:04:48.401 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:04:48.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:04:47 smithi083 ceph-mon[79930]: pgmap v11895: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:48.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:04:47 smithi083 ceph-mon[77242]: pgmap v11895: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:49.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:04:48 smithi153 ceph-mon[70044]: from='client.47964 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:04:49.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:04:48 smithi153 ceph-mon[70044]: from='client.57901 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:04:49.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:04:48 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1663617729' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:04:49.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:04:48 smithi083 ceph-mon[79930]: from='client.47964 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:04:49.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:04:48 smithi083 ceph-mon[79930]: from='client.57901 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:04:49.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:04:48 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1663617729' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:04:49.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:04:48 smithi083 ceph-mon[77242]: from='client.47964 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:04:49.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:04:48 smithi083 ceph-mon[77242]: from='client.57901 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:04:49.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:04:48 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1663617729' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:04:50.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:04:49 smithi153 ceph-mon[70044]: pgmap v11896: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:50.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:04:49 smithi083 ceph-mon[79930]: pgmap v11896: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:50.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:04:49 smithi083 ceph-mon[77242]: pgmap v11896: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:04:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:04:51] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T12:04:51.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:04:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:04:51.630Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:04:52.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:04:51 smithi153 ceph-mon[70044]: pgmap v11897: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:52.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:04:51 smithi083 ceph-mon[79930]: pgmap v11897: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:52.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:04:51 smithi083 ceph-mon[77242]: pgmap v11897: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:04:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:04:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:04:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:04:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:04:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:04:54.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:04:53 smithi153 ceph-mon[70044]: pgmap v11898: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:54.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:04:53 smithi083 ceph-mon[79930]: pgmap v11898: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:54.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:04:53 smithi083 ceph-mon[77242]: pgmap v11898: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:04:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:04:55.005Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:04:56.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:04:55 smithi083 ceph-mon[79930]: pgmap v11899: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:56.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:04:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:04:55.723Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:04:56.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:04:55 smithi083 ceph-mon[77242]: pgmap v11899: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:56.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:04:55 smithi153 ceph-mon[70044]: pgmap v11899: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:58.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:04:57 smithi153 ceph-mon[70044]: pgmap v11900: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:58.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:04:57 smithi083 ceph-mon[79930]: pgmap v11900: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:04:58.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:04:57 smithi083 ceph-mon[77242]: pgmap v11900: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:04:59 smithi153 ceph-mon[70044]: pgmap v11901: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:00.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:04:59 smithi083 ceph-mon[79930]: pgmap v11901: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:00.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:04:59 smithi083 ceph-mon[77242]: pgmap v11901: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:05:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:05:01] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T12:05:01.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:05:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:05:01.631Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:05:02.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:05:01 smithi153 ceph-mon[70044]: pgmap v11902: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:02.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:05:01 smithi083 ceph-mon[79930]: pgmap v11902: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:02.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:05:01 smithi083 ceph-mon[77242]: pgmap v11902: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:05:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:05:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:05:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:05:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:05:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:05:04.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:05:03 smithi153 ceph-mon[70044]: pgmap v11903: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:04.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:05:03 smithi083 ceph-mon[79930]: pgmap v11903: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:04.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:05:03 smithi083 ceph-mon[77242]: pgmap v11903: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:05.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:05:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:05:05.006Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:05:06.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:05:05 smithi083 ceph-mon[79930]: pgmap v11904: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:06.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:05:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:05:05.724Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:05:06.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:05:05 smithi083 ceph-mon[77242]: pgmap v11904: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:06.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:05:05 smithi153 ceph-mon[70044]: pgmap v11904: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:08.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:05:07 smithi153 ceph-mon[70044]: pgmap v11905: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:08.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:05:07 smithi083 ceph-mon[79930]: pgmap v11905: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:08.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:05:07 smithi083 ceph-mon[77242]: pgmap v11905: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:10.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:05:09 smithi153 ceph-mon[70044]: pgmap v11906: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:10.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:05:09 smithi083 ceph-mon[79930]: pgmap v11906: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:10.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:05:09 smithi083 ceph-mon[77242]: pgmap v11906: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:05:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:05:11] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T12:05:11.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:05:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:05:11.632Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:05:12.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:05:11 smithi153 ceph-mon[70044]: pgmap v11907: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:12.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:05:11 smithi083 ceph-mon[79930]: pgmap v11907: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:12.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:05:11 smithi083 ceph-mon[77242]: pgmap v11907: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:05:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:05:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:05:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:05:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:05:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:05:14.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:05:13 smithi153 ceph-mon[70044]: pgmap v11908: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:14.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:05:13 smithi083 ceph-mon[79930]: pgmap v11908: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:14.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:05:13 smithi083 ceph-mon[77242]: pgmap v11908: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:15.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:05:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:05:15.008Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:05:16.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:05:15 smithi083 ceph-mon[79930]: pgmap v11909: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:16.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:05:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:05:15.724Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:05:16.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:05:15 smithi083 ceph-mon[77242]: pgmap v11909: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:16.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:05:15 smithi153 ceph-mon[70044]: pgmap v11909: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:18.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:05:17 smithi153 ceph-mon[70044]: pgmap v11910: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:18.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:05:17 smithi083 ceph-mon[79930]: pgmap v11910: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:18.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:05:17 smithi083 ceph-mon[77242]: pgmap v11910: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:18.726 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:05:19.016 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:05:19.016 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 98s ago 6h 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:05:19.017 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 9m ago 6h 62.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:05:19.017 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 98s ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:05:19.017 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 9m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:05:19.017 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 9m ago 6h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:05:19.018 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 98s ago 6h 847M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:05:19.018 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 98s ago 6h 1486M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:05:19.018 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 9m ago 6h 625M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:05:19.018 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 98s ago 6h 857M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:05:19.019 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 98s ago 6h 26.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:05:19.019 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 9m ago 6h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:05:19.019 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 98s ago 6h 1074M 2393M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:05:19.019 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 98s ago 6h 1061M 2393M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:05:19.020 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 98s ago 6h 808M 2393M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:05:19.020 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 98s ago 6h 1039M 2393M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:05:19.020 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 9m ago 6h 870M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:05:19.020 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 9m ago 6h 1113M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:05:19.020 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 9m ago 6h 971M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:05:19.021 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 9m ago 6h 1179M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:05:19.021 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 9m ago 6h 159M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:05:19.370 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:05:19.371 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:05:19.371 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:05:19.371 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:05:19.371 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:05:19.372 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:05:19.372 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:05:19.372 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:05:19.372 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:05:19.373 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:05:19.373 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:05:19.373 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:05:19.373 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:05:19.374 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:05:19.374 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:05:19.374 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:05:19.374 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:05:19.374 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:05:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:05:19 smithi153 ceph-mon[70044]: from='client.57913 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:05:20.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:05:19 smithi153 ceph-mon[70044]: pgmap v11911: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:20.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:05:19 smithi153 ceph-mon[70044]: from='client.47988 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:05:20.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:05:19 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/699251084' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:05:20.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:05:19 smithi083 ceph-mon[79930]: from='client.57913 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:05:20.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:05:19 smithi083 ceph-mon[79930]: pgmap v11911: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:20.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:05:19 smithi083 ceph-mon[79930]: from='client.47988 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:05:20.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:05:19 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/699251084' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:05:20.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:05:19 smithi083 ceph-mon[77242]: from='client.57913 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:05:20.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:05:19 smithi083 ceph-mon[77242]: pgmap v11911: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:20.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:05:19 smithi083 ceph-mon[77242]: from='client.47988 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:05:20.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:05:19 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/699251084' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:05:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:05:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:05:21] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T12:05:21.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:05:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:05:21.633Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:05:22.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:05:21 smithi153 ceph-mon[70044]: pgmap v11912: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:22.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:05:21 smithi083 ceph-mon[77242]: pgmap v11912: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:22.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:05:21 smithi083 ceph-mon[79930]: pgmap v11912: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:05:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:05:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:05:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:05:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:05:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:05:24.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:05:23 smithi153 ceph-mon[70044]: pgmap v11913: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:24.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:05:23 smithi083 ceph-mon[79930]: pgmap v11913: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:24.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:05:23 smithi083 ceph-mon[77242]: pgmap v11913: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:25.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:05:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:05:25.008Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:05:26.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:05:25 smithi083 ceph-mon[79930]: pgmap v11914: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:26.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:05:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:05:25.724Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:05:26.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:05:25 smithi083 ceph-mon[77242]: pgmap v11914: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:26.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:05:25 smithi153 ceph-mon[70044]: pgmap v11914: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:28.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:05:27 smithi153 ceph-mon[70044]: pgmap v11915: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:28.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:05:27 smithi083 ceph-mon[79930]: pgmap v11915: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:28.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:05:27 smithi083 ceph-mon[77242]: pgmap v11915: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:30.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:05:29 smithi153 ceph-mon[70044]: pgmap v11916: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:30.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:05:29 smithi083 ceph-mon[79930]: pgmap v11916: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:30.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:05:29 smithi083 ceph-mon[77242]: pgmap v11916: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:05:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:05:31] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T12:05:31.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:05:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:05:31.634Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:05:32.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:05:31 smithi153 ceph-mon[70044]: pgmap v11917: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:32.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:05:31 smithi083 ceph-mon[79930]: pgmap v11917: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:32.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:05:31 smithi083 ceph-mon[77242]: pgmap v11917: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:05:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:05:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:05:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:05:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:05:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:05:34.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:05:33 smithi153 ceph-mon[70044]: pgmap v11918: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:34.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:05:33 smithi083 ceph-mon[79930]: pgmap v11918: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:34.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:05:33 smithi083 ceph-mon[77242]: pgmap v11918: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:35.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:05:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:05:35.009Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:05:36.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:05:35 smithi083 ceph-mon[79930]: pgmap v11919: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:36.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:05:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:05:35.725Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:05:36.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:05:35 smithi083 ceph-mon[77242]: pgmap v11919: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:36.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:05:35 smithi153 ceph-mon[70044]: pgmap v11919: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:38.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:05:37 smithi153 ceph-mon[70044]: pgmap v11920: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:38.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:05:37 smithi083 ceph-mon[79930]: pgmap v11920: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:38.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:05:37 smithi083 ceph-mon[77242]: pgmap v11920: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:05:39 smithi153 ceph-mon[70044]: pgmap v11921: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:05:39 smithi083 ceph-mon[79930]: pgmap v11921: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:05:39 smithi083 ceph-mon[77242]: pgmap v11921: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:05:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:05:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:05:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:05:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:05:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:05:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:05:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:05:41.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:05:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:05:41.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:05:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:05:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:05:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:05:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:05:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:05:41.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:05:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:05:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:05:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:05:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:05:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:05:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:05:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:05:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:05:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:05:41] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T12:05:41.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:05:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:05:41.634Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:05:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:05:41 smithi153 ceph-mon[70044]: pgmap v11922: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:05:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:05:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:05:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:05:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:05:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:05:42.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:05:41 smithi083 ceph-mon[77242]: pgmap v11922: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:42.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:05:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:05:42.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:05:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:05:42.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:05:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:05:42.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:05:41 smithi083 ceph-mon[79930]: pgmap v11922: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:42.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:05:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:05:42.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:05:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:05:42.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:05:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:05:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:05:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:05:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:05:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:05:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:05:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:05:44.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:05:43 smithi153 ceph-mon[70044]: pgmap v11923: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:44.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:05:43 smithi083 ceph-mon[77242]: pgmap v11923: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:44.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:05:43 smithi083 ceph-mon[79930]: pgmap v11923: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:45.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:05:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:05:45.009Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:05:46.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:05:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:05:45.725Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:05:46.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:05:45 smithi083 ceph-mon[77242]: pgmap v11924: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:46.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:05:45 smithi083 ceph-mon[79930]: pgmap v11924: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:46.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:05:45 smithi153 ceph-mon[70044]: pgmap v11924: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:47.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:05:47 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:05:47.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:05:47 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:05:47.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:05:47 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:05:47.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:05:47 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:05:47.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:05:47 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:05:47.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:05:47 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:05:47.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:05:47 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:05:47.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:05:47 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:05:47.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:05:47 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:05:48.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:05:48 smithi083 ceph-mon[79930]: pgmap v11925: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:48.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:05:48 smithi083 ceph-mon[77242]: pgmap v11925: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:48.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:05:48 smithi153 ceph-mon[70044]: pgmap v11925: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:49.700 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:05:49.995 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:05:49.996 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 2m ago 6h 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:05:49.996 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 3s ago 6h 63.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:05:49.996 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 2m ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:05:49.997 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 3s ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:05:49.997 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 3s ago 6h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:05:49.997 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 2m ago 6h 847M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:05:49.997 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 2m ago 6h 1486M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:05:49.997 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 3s ago 6h 641M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:05:49.998 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 2m ago 6h 857M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:05:49.998 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 2m ago 6h 26.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:05:49.998 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 3s ago 6h 26.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:05:49.998 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 2m ago 6h 1074M 2393M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:05:49.999 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 2m ago 6h 1061M 2393M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:05:49.999 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 2m ago 6h 808M 2393M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:05:49.999 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 2m ago 6h 1039M 2393M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:05:49.999 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 3s ago 6h 892M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:05:49.999 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 3s ago 6h 1141M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:05:50.000 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 3s ago 6h 996M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:05:50.000 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 3s ago 6h 1209M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:05:50.000 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 3s ago 6h 163M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:05:50.360 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:05:50.360 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:05:50.361 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:05:50.361 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:05:50.361 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:05:50.362 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:05:50.362 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:05:50.362 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:05:50.362 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:05:50.363 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:05:50.363 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:05:50.363 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:05:50.363 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:05:50.363 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:05:50.364 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:05:50.364 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:05:50.364 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:05:50.364 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:05:50.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:05:50 smithi083 ceph-mon[79930]: pgmap v11926: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:50.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:05:50 smithi083 ceph-mon[77242]: pgmap v11926: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:05:50 smithi153 ceph-mon[70044]: pgmap v11926: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:51.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:05:51 smithi083 ceph-mon[79930]: from='client.48000 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:05:51.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:05:51 smithi083 ceph-mon[79930]: from='client.48006 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:05:51.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:05:51 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1635795936' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:05:51.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:05:51 smithi083 ceph-mon[77242]: from='client.48000 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:05:51.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:05:51 smithi083 ceph-mon[77242]: from='client.48006 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:05:51.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:05:51 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1635795936' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:05:51.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:05:51 smithi153 ceph-mon[70044]: from='client.48000 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:05:51.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:05:51 smithi153 ceph-mon[70044]: from='client.48006 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:05:51.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:05:51 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1635795936' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:05:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:05:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:05:51] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T12:05:51.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:05:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:05:51.636Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:05:52.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:05:52 smithi083 ceph-mon[79930]: pgmap v11927: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:52.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:05:52 smithi083 ceph-mon[77242]: pgmap v11927: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:05:52 smithi153 ceph-mon[70044]: pgmap v11927: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:52.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:05:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:05:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:05:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:05:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:05:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:05:54.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:05:54 smithi083 ceph-mon[79930]: pgmap v11928: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:54.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:05:54 smithi083 ceph-mon[77242]: pgmap v11928: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:54.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:05:54 smithi153 ceph-mon[70044]: pgmap v11928: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:05:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:05:55.010Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:05:56.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:05:55 smithi083 ceph-mon[79930]: pgmap v11929: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:56.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:05:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:05:55.726Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:05:56.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:05:55 smithi083 ceph-mon[77242]: pgmap v11929: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:56.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:05:55 smithi153 ceph-mon[70044]: pgmap v11929: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:58.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:05:57 smithi153 ceph-mon[70044]: pgmap v11930: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:58.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:05:57 smithi083 ceph-mon[79930]: pgmap v11930: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:05:58.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:05:57 smithi083 ceph-mon[77242]: pgmap v11930: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:05:59 smithi153 ceph-mon[70044]: pgmap v11931: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:00.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:05:59 smithi083 ceph-mon[79930]: pgmap v11931: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:00.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:05:59 smithi083 ceph-mon[77242]: pgmap v11931: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:01.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:06:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:06:01.636Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:06:01.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:06:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:06:01] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T12:06:02.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:06:01 smithi153 ceph-mon[70044]: pgmap v11932: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:02.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:06:01 smithi083 ceph-mon[79930]: pgmap v11932: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:02.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:06:01 smithi083 ceph-mon[77242]: pgmap v11932: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:06:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:06:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:06:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:06:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:06:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:06:04.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:06:04 smithi153 ceph-mon[70044]: pgmap v11933: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:04.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:06:04 smithi083 ceph-mon[77242]: pgmap v11933: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:04.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:06:04 smithi083 ceph-mon[79930]: pgmap v11933: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:05.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:06:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:06:05.010Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:06:06.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:06:05 smithi083 ceph-mon[79930]: pgmap v11934: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:06.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:06:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:06:05.727Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:06:06.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:06:05 smithi083 ceph-mon[77242]: pgmap v11934: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:06.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:06:05 smithi153 ceph-mon[70044]: pgmap v11934: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:08.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:06:07 smithi153 ceph-mon[70044]: pgmap v11935: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:08.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:06:07 smithi083 ceph-mon[79930]: pgmap v11935: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:08.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:06:07 smithi083 ceph-mon[77242]: pgmap v11935: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:10.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:06:10 smithi153 ceph-mon[70044]: pgmap v11936: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:10.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:06:09 smithi083 ceph-mon[79930]: pgmap v11936: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:10.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:06:09 smithi083 ceph-mon[77242]: pgmap v11936: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:11.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:06:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:06:11.636Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:06:11.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:06:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:06:11] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T12:06:12.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:06:12 smithi153 ceph-mon[70044]: pgmap v11937: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:12.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:06:12 smithi083 ceph-mon[79930]: pgmap v11937: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:12.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:06:12 smithi083 ceph-mon[77242]: pgmap v11937: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:06:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:06:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:06:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:06:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:06:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:06:14.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:06:14 smithi153 ceph-mon[70044]: pgmap v11938: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:14.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:06:14 smithi083 ceph-mon[79930]: pgmap v11938: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:14.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:06:14 smithi083 ceph-mon[77242]: pgmap v11938: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:15.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:06:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:06:15.011Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:06:16.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:06:15 smithi083 ceph-mon[79930]: pgmap v11939: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:16.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:06:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:06:15.728Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:06:16.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:06:15 smithi083 ceph-mon[77242]: pgmap v11939: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:16.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:06:15 smithi153 ceph-mon[70044]: pgmap v11939: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:18.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:06:17 smithi153 ceph-mon[70044]: pgmap v11940: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:18.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:06:17 smithi083 ceph-mon[79930]: pgmap v11940: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:18.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:06:17 smithi083 ceph-mon[77242]: pgmap v11940: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:06:20 smithi153 ceph-mon[70044]: pgmap v11941: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:20.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:06:19 smithi083 ceph-mon[77242]: pgmap v11941: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:20.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:06:19 smithi083 ceph-mon[79930]: pgmap v11941: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:20.687 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:06:20.973 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:06:20.974 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 2m ago 6h 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:06:20.974 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 34s ago 6h 63.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:06:20.974 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 2m ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:06:20.974 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 34s ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:06:20.975 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 34s ago 6h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:06:20.975 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 2m ago 6h 847M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:06:20.975 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 2m ago 6h 1486M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:06:20.975 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 34s ago 6h 641M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:06:20.976 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 2m ago 6h 857M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:06:20.976 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 2m ago 6h 26.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:06:20.976 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 34s ago 6h 26.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:06:20.976 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 2m ago 6h 1074M 2393M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:06:20.976 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 2m ago 6h 1061M 2393M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:06:20.977 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 2m ago 6h 808M 2393M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:06:20.977 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 2m ago 6h 1039M 2393M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:06:20.977 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 34s ago 6h 892M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:06:20.978 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 34s ago 6h 1141M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:06:20.978 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 34s ago 6h 996M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:06:20.978 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 34s ago 6h 1209M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:06:20.978 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 34s ago 6h 163M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:06:21.333 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:06:21.334 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:06:21.334 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:06:21.334 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:06:21.335 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:06:21.335 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:06:21.335 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:06:21.335 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:06:21.336 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:06:21.336 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:06:21.336 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:06:21.336 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:06:21.336 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:06:21.337 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:06:21.337 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:06:21.337 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:06:21.337 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:06:21.337 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:06:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:06:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:06:21] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T12:06:21.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:06:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:06:21.636Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:06:22.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:06:22 smithi153 ceph-mon[70044]: from='client.48018 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:06:22.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:06:22 smithi153 ceph-mon[70044]: pgmap v11942: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:22.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:06:22 smithi153 ceph-mon[70044]: from='client.48024 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:06:22.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:06:22 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2190920095' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:06:22.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:06:22 smithi083 ceph-mon[77242]: from='client.48018 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:06:22.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:06:22 smithi083 ceph-mon[77242]: pgmap v11942: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:22.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:06:22 smithi083 ceph-mon[77242]: from='client.48024 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:06:22.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:06:22 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2190920095' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:06:22.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:06:22 smithi083 ceph-mon[79930]: from='client.48018 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:06:22.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:06:22 smithi083 ceph-mon[79930]: pgmap v11942: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:22.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:06:22 smithi083 ceph-mon[79930]: from='client.48024 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:06:22.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:06:22 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2190920095' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:06:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:06:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:06:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:06:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:06:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:06:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:06:24.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:06:24 smithi153 ceph-mon[70044]: pgmap v11943: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:24.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:06:24 smithi083 ceph-mon[79930]: pgmap v11943: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:24.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:06:24 smithi083 ceph-mon[77242]: pgmap v11943: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:25.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:06:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:06:25.011Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:06:26.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:06:25 smithi083 ceph-mon[79930]: pgmap v11944: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:26.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:06:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:06:25.729Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:06:26.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:06:25 smithi083 ceph-mon[77242]: pgmap v11944: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:26.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:06:25 smithi153 ceph-mon[70044]: pgmap v11944: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:28.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:06:28 smithi153 ceph-mon[70044]: pgmap v11945: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:28.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:06:28 smithi083 ceph-mon[79930]: pgmap v11945: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:28.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:06:27 smithi083 ceph-mon[77242]: pgmap v11945: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:30.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:06:30 smithi153 ceph-mon[70044]: pgmap v11946: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:30.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:06:30 smithi083 ceph-mon[79930]: pgmap v11946: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:30.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:06:30 smithi083 ceph-mon[77242]: pgmap v11946: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:31.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:06:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:06:31.637Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:06:31.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:06:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:06:31] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T12:06:32.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:06:32 smithi153 ceph-mon[70044]: pgmap v11947: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:32.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:06:32 smithi083 ceph-mon[79930]: pgmap v11947: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:32.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:06:32 smithi083 ceph-mon[77242]: pgmap v11947: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:06:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:06:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:06:34.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:06:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:06:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:06:34.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:06:34 smithi153 ceph-mon[70044]: pgmap v11948: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:34.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:06:34 smithi083 ceph-mon[79930]: pgmap v11948: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:34.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:06:34 smithi083 ceph-mon[77242]: pgmap v11948: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:35.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:06:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:06:35.012Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:06:36.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:06:35 smithi083 ceph-mon[79930]: pgmap v11949: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:36.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:06:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:06:35.730Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:06:36.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:06:35 smithi083 ceph-mon[77242]: pgmap v11949: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:36.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:06:35 smithi153 ceph-mon[70044]: pgmap v11949: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:38.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:06:38 smithi153 ceph-mon[70044]: pgmap v11950: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:38.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:06:38 smithi083 ceph-mon[79930]: pgmap v11950: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:38.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:06:38 smithi083 ceph-mon[77242]: pgmap v11950: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:06:40 smithi153 ceph-mon[70044]: pgmap v11951: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:06:40 smithi083 ceph-mon[79930]: pgmap v11951: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:06:40 smithi083 ceph-mon[77242]: pgmap v11951: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:06:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:06:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:06:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:06:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:06:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:06:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:06:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:06:41.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:06:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:06:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:06:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:06:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:06:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:06:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:06:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:06:41.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:06:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:06:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:06:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:06:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:06:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:06:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:06:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:06:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:06:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:06:41] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T12:06:41.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:06:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:06:41.637Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:06:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:06:42 smithi153 ceph-mon[70044]: pgmap v11952: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:06:42 smithi083 ceph-mon[79930]: pgmap v11952: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:42.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:06:42 smithi083 ceph-mon[77242]: pgmap v11952: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:06:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:06:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:06:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:06:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:06:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:06:44.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:06:44 smithi153 ceph-mon[70044]: pgmap v11953: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:44.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:06:44 smithi083 ceph-mon[79930]: pgmap v11953: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:44.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:06:44 smithi083 ceph-mon[77242]: pgmap v11953: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:45.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:06:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:06:45.013Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:06:46.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:06:45 smithi083 ceph-mon[79930]: pgmap v11954: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:46.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:06:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:06:45.731Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:06:46.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:06:45 smithi083 ceph-mon[77242]: pgmap v11954: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:46.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:06:45 smithi153 ceph-mon[70044]: pgmap v11954: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:47.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:06:46 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:06:47.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:06:46 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:06:47.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:06:46 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:06:47.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:06:46 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:06:47.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:06:46 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:06:47.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:06:46 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:06:47.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:06:46 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:06:47.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:06:46 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:06:47.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:06:46 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:06:48.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:06:48 smithi083 ceph-mon[79930]: pgmap v11955: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:48.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:06:48 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:06:48.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:06:48 smithi083 ceph-mon[77242]: pgmap v11955: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:48.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:06:48 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:06:48.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:06:48 smithi153 ceph-mon[70044]: pgmap v11955: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:48.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:06:48 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:06:50.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:06:50 smithi083 ceph-mon[79930]: pgmap v11956: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:50.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:06:50 smithi083 ceph-mon[77242]: pgmap v11956: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:50.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:06:50 smithi153 ceph-mon[70044]: pgmap v11956: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:51.662 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:06:51.949 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:06:51.949 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 3m ago 6h 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:06:51.950 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 65s ago 6h 63.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:06:51.950 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 3m ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:06:51.950 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 65s ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:06:51.950 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 65s ago 6h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:06:51.950 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 3m ago 6h 847M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:06:51.951 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 3m ago 6h 1486M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:06:51.951 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 65s ago 6h 641M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:06:51.951 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 3m ago 6h 857M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:06:51.951 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 3m ago 6h 26.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:06:51.952 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 65s ago 6h 26.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:06:51.952 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 3m ago 6h 1074M 2393M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:06:51.952 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 3m ago 6h 1061M 2393M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:06:51.952 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 3m ago 6h 808M 2393M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:06:51.952 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 3m ago 6h 1039M 2393M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:06:51.953 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 65s ago 6h 892M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:06:51.953 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 65s ago 6h 1141M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:06:51.953 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 65s ago 6h 996M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:06:51.953 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 65s ago 6h 1209M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:06:51.953 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 65s ago 6h 163M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:06:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:06:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:06:51] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T12:06:51.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:06:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:06:51.638Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:06:52.305 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:06:52.305 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:06:52.306 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:06:52.306 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:06:52.306 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:06:52.306 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:06:52.307 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:06:52.307 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:06:52.307 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:06:52.307 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:06:52.307 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:06:52.308 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:06:52.308 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:06:52.308 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:06:52.308 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:06:52.308 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:06:52.309 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:06:52.309 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:06:52.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:06:52 smithi083 ceph-mon[79930]: pgmap v11957: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:52.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:06:52 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4265795235' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:06:52.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:06:52 smithi083 ceph-mon[77242]: pgmap v11957: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:52.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:06:52 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4265795235' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:06:52.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:06:52 smithi153 ceph-mon[70044]: pgmap v11957: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:52.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:06:52 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4265795235' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:06:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:06:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:06:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:06:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:06:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:06:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:06:53.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:06:53 smithi083 ceph-mon[79930]: from='client.48036 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:06:53.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:06:53 smithi083 ceph-mon[79930]: from='client.57973 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:06:53.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:06:53 smithi083 ceph-mon[77242]: from='client.48036 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:06:53.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:06:53 smithi083 ceph-mon[77242]: from='client.57973 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:06:53.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:06:53 smithi153 ceph-mon[70044]: from='client.48036 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:06:53.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:06:53 smithi153 ceph-mon[70044]: from='client.57973 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:06:54.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:06:54 smithi083 ceph-mon[79930]: pgmap v11958: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:54.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:06:54 smithi083 ceph-mon[77242]: pgmap v11958: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:54.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:06:54 smithi153 ceph-mon[70044]: pgmap v11958: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:06:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:06:55.014Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:06:56.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:06:55 smithi083 ceph-mon[79930]: pgmap v11959: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:56.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:06:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:06:55.732Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:06:56.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:06:55 smithi083 ceph-mon[77242]: pgmap v11959: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:56.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:06:55 smithi153 ceph-mon[70044]: pgmap v11959: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:58.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:06:58 smithi153 ceph-mon[70044]: pgmap v11960: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:58.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:06:58 smithi083 ceph-mon[79930]: pgmap v11960: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:06:58.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:06:58 smithi083 ceph-mon[77242]: pgmap v11960: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:07:00 smithi153 ceph-mon[70044]: pgmap v11961: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:00.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:07:00 smithi083 ceph-mon[79930]: pgmap v11961: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:00.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:07:00 smithi083 ceph-mon[77242]: pgmap v11961: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:07:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:07:01] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T12:07:01.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:07:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:07:01.639Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:07:02.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:07:02 smithi153 ceph-mon[70044]: pgmap v11962: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:02.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:07:02 smithi083 ceph-mon[77242]: pgmap v11962: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:02.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:07:02 smithi083 ceph-mon[79930]: pgmap v11962: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:07:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:07:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:07:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:07:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:07:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:07:04.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:07:04 smithi153 ceph-mon[70044]: pgmap v11963: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:04.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:07:04 smithi083 ceph-mon[79930]: pgmap v11963: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:04.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:07:04 smithi083 ceph-mon[77242]: pgmap v11963: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:05.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:07:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:07:05.014Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:07:06.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:07:05 smithi083 ceph-mon[79930]: pgmap v11964: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:06.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:07:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:07:05.732Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:07:06.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:07:05 smithi083 ceph-mon[77242]: pgmap v11964: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:06.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:07:05 smithi153 ceph-mon[70044]: pgmap v11964: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:08.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:07:08 smithi153 ceph-mon[70044]: pgmap v11965: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:08.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:07:08 smithi083 ceph-mon[79930]: pgmap v11965: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:08.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:07:08 smithi083 ceph-mon[77242]: pgmap v11965: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:10.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:07:10 smithi153 ceph-mon[70044]: pgmap v11966: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:10.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:07:10 smithi083 ceph-mon[79930]: pgmap v11966: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:10.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:07:10 smithi083 ceph-mon[77242]: pgmap v11966: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:07:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:07:11] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T12:07:11.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:07:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:07:11.639Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:07:12.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:07:12 smithi153 ceph-mon[70044]: pgmap v11967: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:12.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:07:12 smithi083 ceph-mon[79930]: pgmap v11967: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:12.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:07:12 smithi083 ceph-mon[77242]: pgmap v11967: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:07:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:07:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:07:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:07:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:07:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:07:14.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:07:14 smithi153 ceph-mon[70044]: pgmap v11968: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:14.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:07:14 smithi083 ceph-mon[79930]: pgmap v11968: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:14.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:07:14 smithi083 ceph-mon[77242]: pgmap v11968: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:15.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:07:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:07:15.014Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:07:16.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:07:15 smithi083 ceph-mon[79930]: pgmap v11969: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:16.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:07:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:07:15.732Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:07:16.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:07:15 smithi083 ceph-mon[77242]: pgmap v11969: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:16.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:07:15 smithi153 ceph-mon[70044]: pgmap v11969: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:18.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:07:18 smithi153 ceph-mon[70044]: pgmap v11970: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:18.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:07:18 smithi083 ceph-mon[79930]: pgmap v11970: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:18.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:07:18 smithi083 ceph-mon[77242]: pgmap v11970: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:07:20 smithi153 ceph-mon[70044]: pgmap v11971: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:20.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:07:20 smithi083 ceph-mon[79930]: pgmap v11971: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:20.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:07:20 smithi083 ceph-mon[77242]: pgmap v11971: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:07:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:07:21] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T12:07:21.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:07:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:07:21.640Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:07:22.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:07:22 smithi153 ceph-mon[70044]: pgmap v11972: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:22.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:07:22 smithi083 ceph-mon[79930]: pgmap v11972: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:22.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:07:22 smithi083 ceph-mon[77242]: pgmap v11972: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:22.632 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:07:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:07:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:07:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:07:22.918 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:07:22.918 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 3m ago 6h 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:07:22.918 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 96s ago 6h 63.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:07:22.919 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 3m ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:07:22.919 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 96s ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:07:22.919 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 96s ago 6h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:07:22.919 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 3m ago 6h 847M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:07:22.920 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 3m ago 6h 1486M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:07:22.920 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 96s ago 6h 641M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:07:22.920 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 3m ago 6h 857M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:07:22.920 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 3m ago 6h 26.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:07:22.920 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 96s ago 6h 26.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:07:22.921 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 3m ago 6h 1074M 2393M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:07:22.921 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 3m ago 6h 1061M 2393M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:07:22.921 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 3m ago 6h 808M 2393M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:07:22.921 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 3m ago 6h 1039M 2393M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:07:22.922 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 96s ago 6h 892M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:07:22.922 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 96s ago 6h 1141M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:07:22.922 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 96s ago 6h 996M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:07:22.922 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 96s ago 6h 1209M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:07:22.922 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 96s ago 6h 163M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:07:23.276 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:07:23.276 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:07:23.277 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:07:23.277 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:07:23.277 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:07:23.277 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:07:23.278 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:07:23.278 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:07:23.278 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:07:23.278 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:07:23.279 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:07:23.279 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:07:23.279 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:07:23.279 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:07:23.280 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:07:23.280 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:07:23.280 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:07:23.280 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:07:23.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:07:23 smithi153 ceph-mon[70044]: from='client.57985 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:07:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:07:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:07:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:07:23.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:07:23 smithi083 ceph-mon[79930]: from='client.57985 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:07:23.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:07:23 smithi083 ceph-mon[77242]: from='client.57985 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:07:24.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:07:24 smithi153 ceph-mon[70044]: from='client.48060 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:07:24.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:07:24 smithi153 ceph-mon[70044]: pgmap v11973: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:24.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:07:24 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1579239630' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:07:24.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:07:24 smithi083 ceph-mon[79930]: from='client.48060 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:07:24.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:07:24 smithi083 ceph-mon[79930]: pgmap v11973: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:24.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:07:24 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1579239630' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:07:24.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:07:24 smithi083 ceph-mon[77242]: from='client.48060 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:07:24.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:07:24 smithi083 ceph-mon[77242]: pgmap v11973: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:24.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:07:24 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1579239630' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:07:25.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:07:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:07:25.015Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:07:26.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:07:25 smithi083 ceph-mon[77242]: pgmap v11974: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:26.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:07:25 smithi083 ceph-mon[79930]: pgmap v11974: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:26.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:07:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:07:25.734Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:07:26.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:07:25 smithi153 ceph-mon[70044]: pgmap v11974: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:28.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:07:28 smithi153 ceph-mon[70044]: pgmap v11975: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:28.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:07:28 smithi083 ceph-mon[79930]: pgmap v11975: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:28.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:07:28 smithi083 ceph-mon[77242]: pgmap v11975: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:30.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:07:30 smithi153 ceph-mon[70044]: pgmap v11976: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:30.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:07:30 smithi083 ceph-mon[79930]: pgmap v11976: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:30.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:07:30 smithi083 ceph-mon[77242]: pgmap v11976: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:07:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:07:31] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T12:07:31.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:07:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:07:31.641Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:07:32.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:07:32 smithi153 ceph-mon[70044]: pgmap v11977: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:32.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:07:32 smithi083 ceph-mon[79930]: pgmap v11977: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:32.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:07:32 smithi083 ceph-mon[77242]: pgmap v11977: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:07:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:07:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:07:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:07:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:07:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:07:34.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:07:34 smithi153 ceph-mon[70044]: pgmap v11978: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:34.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:07:34 smithi083 ceph-mon[79930]: pgmap v11978: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:34.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:07:34 smithi083 ceph-mon[77242]: pgmap v11978: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:35.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:07:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:07:35.016Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:07:36.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:07:35 smithi083 ceph-mon[79930]: pgmap v11979: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:36.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:07:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:07:35.734Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:07:36.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:07:35 smithi083 ceph-mon[77242]: pgmap v11979: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:36.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:07:35 smithi153 ceph-mon[70044]: pgmap v11979: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:38.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:07:38 smithi153 ceph-mon[70044]: pgmap v11980: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:38.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:07:38 smithi083 ceph-mon[77242]: pgmap v11980: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:38.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:07:38 smithi083 ceph-mon[79930]: pgmap v11980: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:07:40 smithi153 ceph-mon[70044]: pgmap v11981: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:07:40 smithi083 ceph-mon[79930]: pgmap v11981: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:07:40 smithi083 ceph-mon[77242]: pgmap v11981: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:07:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:07:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:07:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:07:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:07:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:07:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:07:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:07:41.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:07:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:07:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:07:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:07:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:07:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:07:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:07:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:07:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:07:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:07:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:07:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:07:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:07:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:07:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:07:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:07:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:07:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:07:41] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T12:07:41.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:07:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:07:41.641Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:07:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:07:42 smithi153 ceph-mon[70044]: pgmap v11982: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:07:42 smithi083 ceph-mon[79930]: pgmap v11982: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:42.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:07:42 smithi083 ceph-mon[77242]: pgmap v11982: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:07:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:07:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:07:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:07:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:07:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:07:44.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:07:44 smithi153 ceph-mon[70044]: pgmap v11983: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:44.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:07:44 smithi083 ceph-mon[79930]: pgmap v11983: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:44.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:07:44 smithi083 ceph-mon[77242]: pgmap v11983: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:45.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:07:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:07:45.017Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:07:46.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:07:45 smithi083 ceph-mon[79930]: pgmap v11984: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:46.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:07:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:07:45.734Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:07:46.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:07:45 smithi083 ceph-mon[77242]: pgmap v11984: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:46.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:07:45 smithi153 ceph-mon[70044]: pgmap v11984: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:48.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:07:48 smithi153 ceph-mon[70044]: pgmap v11985: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:48.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:07:48 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:07:48.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:07:48 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:07:48.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:07:48 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:07:48.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:07:48 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:07:48.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:07:48 smithi083 ceph-mon[79930]: pgmap v11985: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:48.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:07:48 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:07:48.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:07:48 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:07:48.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:07:48 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:07:48.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:07:48 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:07:48.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:07:48 smithi083 ceph-mon[77242]: pgmap v11985: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:48.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:07:48 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:07:48.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:07:48 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:07:48.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:07:48 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:07:48.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:07:48 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:07:50.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:07:50 smithi153 ceph-mon[70044]: pgmap v11986: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:50.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:07:50 smithi083 ceph-mon[79930]: pgmap v11986: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:50.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:07:50 smithi083 ceph-mon[77242]: pgmap v11986: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:07:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:07:51] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T12:07:51.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:07:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:07:51.642Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:07:52.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:07:52 smithi153 ceph-mon[70044]: pgmap v11987: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:52.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:07:52 smithi083 ceph-mon[79930]: pgmap v11987: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:52.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:07:52 smithi083 ceph-mon[77242]: pgmap v11987: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:07:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:07:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:07:53.311 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:07:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:07:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:07:53.612 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:07:53.906 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:07:53.906 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 4m ago 6h 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:07:53.906 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 2m ago 6h 63.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:07:53.907 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 4m ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:07:53.907 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 2m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:07:53.907 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 2m ago 6h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:07:53.907 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 4m ago 6h 847M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:07:53.907 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 4m ago 6h 1486M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:07:53.908 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 2m ago 6h 641M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:07:53.908 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 4m ago 6h 857M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:07:53.908 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 4m ago 6h 26.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:07:53.909 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 2m ago 6h 26.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:07:53.909 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 4m ago 6h 1074M 2393M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:07:53.909 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 4m ago 6h 1061M 2393M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:07:53.909 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 4m ago 6h 808M 2393M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:07:53.909 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 4m ago 6h 1039M 2393M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:07:53.910 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 2m ago 6h 892M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:07:53.910 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 2m ago 6h 1141M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:07:53.910 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 2m ago 6h 996M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:07:53.910 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 2m ago 6h 1209M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:07:53.911 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 2m ago 6h 163M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:07:54.263 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:07:54.264 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:07:54.264 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:07:54.264 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:07:54.264 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:07:54.265 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:07:54.265 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:07:54.265 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:07:54.265 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:07:54.266 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:07:54.266 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:07:54.266 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:07:54.266 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:07:54.267 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:07:54.267 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:07:54.267 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:07:54.267 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:07:54.268 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:07:54.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:07:54 smithi153 ceph-mon[70044]: pgmap v11988: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:54.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:07:54 smithi153 ceph-mon[70044]: from='client.48072 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:07:54.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:07:54 smithi083 ceph-mon[79930]: pgmap v11988: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:54.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:07:54 smithi083 ceph-mon[79930]: from='client.48072 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:07:54.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:07:54 smithi083 ceph-mon[77242]: pgmap v11988: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:54.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:07:54 smithi083 ceph-mon[77242]: from='client.48072 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:07:55.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:07:55 smithi153 ceph-mon[70044]: from='client.48078 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:07:55.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:07:55 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4080074254' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:07:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:07:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:07:55.017Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:07:55.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:07:55 smithi083 ceph-mon[77242]: from='client.48078 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:07:55.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:07:55 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4080074254' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:07:55.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:07:55 smithi083 ceph-mon[79930]: from='client.48078 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:07:55.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:07:55 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4080074254' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:07:56.046 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:07:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:07:55.735Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:07:56.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:07:56 smithi153 ceph-mon[70044]: pgmap v11989: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:56.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:07:56 smithi083 ceph-mon[79930]: pgmap v11989: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:56.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:07:56 smithi083 ceph-mon[77242]: pgmap v11989: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:58.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:07:58 smithi153 ceph-mon[70044]: pgmap v11990: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:58.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:07:58 smithi083 ceph-mon[79930]: pgmap v11990: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:07:58.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:07:58 smithi083 ceph-mon[77242]: pgmap v11990: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:08:00 smithi153 ceph-mon[70044]: pgmap v11991: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:00.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:08:00 smithi083 ceph-mon[79930]: pgmap v11991: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:00.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:08:00 smithi083 ceph-mon[77242]: pgmap v11991: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:08:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:08:01] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T12:08:01.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:08:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:08:01.642Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:08:02.316 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:08:02 smithi153 ceph-mon[70044]: pgmap v11992: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:02.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:08:02 smithi083 ceph-mon[79930]: pgmap v11992: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:02.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:08:02 smithi083 ceph-mon[77242]: pgmap v11992: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:08:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:08:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:08:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:08:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:08:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:08:04.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:08:04 smithi083 ceph-mon[79930]: pgmap v11993: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:04.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:08:04 smithi083 ceph-mon[77242]: pgmap v11993: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:08:04 smithi153 ceph-mon[70044]: pgmap v11993: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:05.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:08:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:08:05.018Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:08:06.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:08:05 smithi083 ceph-mon[79930]: pgmap v11994: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:06.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:08:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:08:05.736Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:08:06.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:08:05 smithi083 ceph-mon[77242]: pgmap v11994: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:06.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:08:05 smithi153 ceph-mon[70044]: pgmap v11994: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:08.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:08:08 smithi153 ceph-mon[70044]: pgmap v11995: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:08.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:08:08 smithi083 ceph-mon[79930]: pgmap v11995: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:08.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:08:08 smithi083 ceph-mon[77242]: pgmap v11995: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:10.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:08:10 smithi153 ceph-mon[70044]: pgmap v11996: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:10.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:08:10 smithi083 ceph-mon[79930]: pgmap v11996: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:10.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:08:10 smithi083 ceph-mon[77242]: pgmap v11996: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:08:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:08:11] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T12:08:11.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:08:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:08:11.643Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:08:12.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:08:12 smithi153 ceph-mon[70044]: pgmap v11997: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:12.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:08:12 smithi083 ceph-mon[79930]: pgmap v11997: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:12.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:08:12 smithi083 ceph-mon[77242]: pgmap v11997: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:08:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:08:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:08:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:08:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:08:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:08:14.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:08:14 smithi153 ceph-mon[70044]: pgmap v11998: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:14.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:08:14 smithi083 ceph-mon[77242]: pgmap v11998: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:14.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:08:14 smithi083 ceph-mon[79930]: pgmap v11998: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:15.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:08:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:08:15.019Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:08:16.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:08:15 smithi083 ceph-mon[79930]: pgmap v11999: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:16.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:08:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:08:15.736Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:08:16.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:08:15 smithi083 ceph-mon[77242]: pgmap v11999: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:16.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:08:15 smithi153 ceph-mon[70044]: pgmap v11999: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:18.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:08:18 smithi153 ceph-mon[70044]: pgmap v12000: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:18.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:08:18 smithi083 ceph-mon[79930]: pgmap v12000: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:18.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:08:18 smithi083 ceph-mon[77242]: pgmap v12000: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:08:20 smithi153 ceph-mon[70044]: pgmap v12001: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:20.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:08:20 smithi083 ceph-mon[79930]: pgmap v12001: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:20.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:08:20 smithi083 ceph-mon[77242]: pgmap v12001: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:08:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:08:21] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T12:08:21.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:08:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:08:21.644Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:08:22.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:08:22 smithi153 ceph-mon[70044]: pgmap v12002: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:22.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:08:22 smithi083 ceph-mon[79930]: pgmap v12002: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:22.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:08:22 smithi083 ceph-mon[77242]: pgmap v12002: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:08:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:08:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:08:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:08:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:08:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:08:24.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:08:24 smithi153 ceph-mon[70044]: pgmap v12003: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:24.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:08:24 smithi083 ceph-mon[79930]: pgmap v12003: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:24.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:08:24 smithi083 ceph-mon[77242]: pgmap v12003: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:24.593 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:08:24.882 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:08:24.883 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 4m ago 6h 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:08:24.883 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 2m ago 6h 63.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:08:24.883 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 4m ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:08:24.883 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 2m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:08:24.884 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 2m ago 6h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:08:24.884 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 4m ago 6h 847M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:08:24.884 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 4m ago 6h 1486M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:08:24.884 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 2m ago 6h 641M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:08:24.885 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 4m ago 6h 857M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:08:24.885 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 4m ago 6h 26.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:08:24.885 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 2m ago 6h 26.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:08:24.885 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 4m ago 6h 1074M 2393M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:08:24.885 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 4m ago 6h 1061M 2393M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:08:24.886 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 4m ago 6h 808M 2393M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:08:24.886 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 4m ago 6h 1039M 2393M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:08:24.886 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 2m ago 6h 892M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:08:24.886 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 2m ago 6h 1141M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:08:24.886 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 2m ago 6h 996M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:08:24.887 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 2m ago 6h 1209M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:08:24.887 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 2m ago 6h 163M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:08:25.240 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:08:25.240 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:08:25.241 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:08:25.241 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:08:25.241 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:08:25.241 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:08:25.241 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:08:25.242 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:08:25.242 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:08:25.242 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:08:25.242 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:08:25.243 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:08:25.243 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:08:25.243 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:08:25.243 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:08:25.244 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:08:25.244 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:08:25.244 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:08:25.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:08:25 smithi153 ceph-mon[70044]: from='client.58021 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:08:25.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:08:25 smithi083 ceph-mon[79930]: from='client.58021 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:08:25.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:08:25 smithi083 ceph-mon[77242]: from='client.58021 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:08:25.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:08:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:08:25.019Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:08:26.048 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:08:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:08:25.736Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:08:26.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:08:26 smithi153 ceph-mon[70044]: from='client.58027 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:08:26.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:08:26 smithi153 ceph-mon[70044]: pgmap v12004: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:26.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:08:26 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2214954701' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:08:26.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:08:26 smithi083 ceph-mon[79930]: from='client.58027 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:08:26.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:08:26 smithi083 ceph-mon[79930]: pgmap v12004: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:26.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:08:26 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2214954701' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:08:26.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:08:26 smithi083 ceph-mon[77242]: from='client.58027 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:08:26.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:08:26 smithi083 ceph-mon[77242]: pgmap v12004: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:26.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:08:26 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2214954701' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:08:28.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:08:28 smithi153 ceph-mon[70044]: pgmap v12005: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:28.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:08:28 smithi083 ceph-mon[77242]: pgmap v12005: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:28.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:08:28 smithi083 ceph-mon[79930]: pgmap v12005: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:30.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:08:30 smithi153 ceph-mon[70044]: pgmap v12006: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:30.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:08:30 smithi083 ceph-mon[79930]: pgmap v12006: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:30.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:08:30 smithi083 ceph-mon[77242]: pgmap v12006: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:08:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:08:31] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T12:08:31.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:08:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:08:31.644Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:08:32.315 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:08:32 smithi153 ceph-mon[70044]: pgmap v12007: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:32.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:08:32 smithi083 ceph-mon[79930]: pgmap v12007: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:32.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:08:32 smithi083 ceph-mon[77242]: pgmap v12007: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:08:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:08:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:08:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:08:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:08:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:08:34.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:08:34 smithi083 ceph-mon[79930]: pgmap v12008: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:34.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:08:34 smithi083 ceph-mon[77242]: pgmap v12008: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:08:34 smithi153 ceph-mon[70044]: pgmap v12008: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:35.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:08:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:08:35.021Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:08:36.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:08:35 smithi083 ceph-mon[79930]: pgmap v12009: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:36.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:08:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:08:35.739Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:08:36.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:08:35 smithi083 ceph-mon[77242]: pgmap v12009: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:36.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:08:35 smithi153 ceph-mon[70044]: pgmap v12009: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:38.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:08:38 smithi153 ceph-mon[70044]: pgmap v12010: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:38.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:08:38 smithi083 ceph-mon[79930]: pgmap v12010: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:38.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:08:38 smithi083 ceph-mon[77242]: pgmap v12010: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:08:40 smithi153 ceph-mon[70044]: pgmap v12011: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:08:40 smithi083 ceph-mon[79930]: pgmap v12011: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:08:40 smithi083 ceph-mon[77242]: pgmap v12011: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:08:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:08:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:08:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:08:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:08:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:08:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:08:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:08:41.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:08:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:08:41.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:08:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:08:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:08:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:08:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:08:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:08:41.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:08:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:08:41.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:08:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:08:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:08:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:08:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:08:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:08:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:08:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:08:41] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T12:08:41.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:08:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:08:41.645Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:08:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:08:42 smithi153 ceph-mon[70044]: pgmap v12012: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:08:42 smithi083 ceph-mon[79930]: pgmap v12012: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:42.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:08:42 smithi083 ceph-mon[77242]: pgmap v12012: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:08:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:08:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:08:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:08:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:08:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:08:44.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:08:44 smithi153 ceph-mon[70044]: pgmap v12013: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:44.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:08:44 smithi083 ceph-mon[79930]: pgmap v12013: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:44.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:08:44 smithi083 ceph-mon[77242]: pgmap v12013: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:45.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:08:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:08:45.021Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:08:46.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:08:45 smithi083 ceph-mon[79930]: pgmap v12014: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:46.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:08:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:08:45.739Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:08:46.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:08:45 smithi083 ceph-mon[77242]: pgmap v12014: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:46.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:08:45 smithi153 ceph-mon[70044]: pgmap v12014: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:48.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:08:48 smithi153 ceph-mon[70044]: pgmap v12015: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:48.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:08:48 smithi083 ceph-mon[77242]: pgmap v12015: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:48.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:08:48 smithi083 ceph-mon[79930]: pgmap v12015: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:49.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:08:49 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:08:49.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:08:49 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:08:49.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:08:49 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:08:49.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:08:49 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:08:49.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:08:49 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:08:49.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:08:49 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:08:49.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:08:49 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:08:49.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:08:49 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:08:49.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:08:49 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:08:49.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:08:49 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:08:49.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:08:49 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:08:49.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:08:49 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:08:50.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:08:50 smithi153 ceph-mon[70044]: pgmap v12016: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:50.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:08:50 smithi083 ceph-mon[79930]: pgmap v12016: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:50.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:08:50 smithi083 ceph-mon[77242]: pgmap v12016: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:08:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:08:51] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T12:08:51.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:08:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:08:51.645Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:08:52.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:08:52 smithi153 ceph-mon[70044]: pgmap v12017: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:52.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:08:52 smithi083 ceph-mon[79930]: pgmap v12017: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:52.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:08:52 smithi083 ceph-mon[77242]: pgmap v12017: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:08:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:08:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:08:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:08:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:08:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:08:54.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:08:54 smithi153 ceph-mon[70044]: pgmap v12018: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:54.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:08:54 smithi083 ceph-mon[77242]: pgmap v12018: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:54.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:08:54 smithi083 ceph-mon[79930]: pgmap v12018: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:08:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:08:55.022Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:08:55.568 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:08:55.860 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:08:55.861 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 5m ago 6h 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:08:55.861 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 3m ago 6h 63.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:08:55.861 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 5m ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:08:55.862 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 3m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:08:55.862 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 3m ago 6h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:08:55.862 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 5m ago 6h 847M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:08:55.862 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 5m ago 6h 1486M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:08:55.862 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 3m ago 6h 641M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:08:55.863 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 5m ago 6h 857M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:08:55.863 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 5m ago 6h 26.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:08:55.863 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 3m ago 6h 26.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:08:55.863 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 5m ago 6h 1074M 2393M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:08:55.864 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 5m ago 6h 1061M 2393M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:08:55.864 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 5m ago 6h 808M 2393M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:08:55.864 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 5m ago 6h 1039M 2393M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:08:55.864 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 3m ago 6h 892M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:08:55.864 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 3m ago 6h 1141M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:08:55.865 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 3m ago 6h 996M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:08:55.865 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 3m ago 6h 1209M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:08:55.865 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 3m ago 6h 163M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:08:56.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:08:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:08:55.739Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:08:56.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:08:55 smithi083 ceph-mon[77242]: pgmap v12019: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:56.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:08:55 smithi083 ceph-mon[77242]: from='client.48108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:08:56.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:08:55 smithi083 ceph-mon[79930]: pgmap v12019: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:56.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:08:55 smithi083 ceph-mon[79930]: from='client.48108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:08:56.224 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:08:56.224 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:08:56.225 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:08:56.225 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:08:56.226 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:08:56.227 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:08:56.227 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:08:56.228 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:08:56.228 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:08:56.229 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:08:56.229 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:08:56.230 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:08:56.230 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:08:56.231 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:08:56.231 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:08:56.232 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:08:56.232 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:08:56.233 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:08:56.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:08:55 smithi153 ceph-mon[70044]: pgmap v12019: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:56.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:08:55 smithi153 ceph-mon[70044]: from='client.48108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:08:57.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:08:56 smithi153 ceph-mon[70044]: from='client.58045 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:08:57.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:08:56 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/652822528' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:08:57.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:08:56 smithi083 ceph-mon[77242]: from='client.58045 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:08:57.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:08:56 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/652822528' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:08:57.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:08:56 smithi083 ceph-mon[79930]: from='client.58045 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:08:57.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:08:56 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/652822528' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:08:58.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:08:57 smithi153 ceph-mon[70044]: pgmap v12020: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:58.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:08:57 smithi083 ceph-mon[79930]: pgmap v12020: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:08:58.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:08:57 smithi083 ceph-mon[77242]: pgmap v12020: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:09:00 smithi153 ceph-mon[70044]: pgmap v12021: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:00.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:09:00 smithi083 ceph-mon[79930]: pgmap v12021: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:00.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:09:00 smithi083 ceph-mon[77242]: pgmap v12021: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:09:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:09:01] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T12:09:01.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:09:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:09:01.645Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:09:02.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:09:02 smithi153 ceph-mon[70044]: pgmap v12022: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:02.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:09:02 smithi083 ceph-mon[79930]: pgmap v12022: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:02.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:09:02 smithi083 ceph-mon[77242]: pgmap v12022: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:09:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:09:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:09:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:09:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:09:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:09:04.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:09:04 smithi153 ceph-mon[70044]: pgmap v12023: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:04.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:09:04 smithi083 ceph-mon[77242]: pgmap v12023: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:04.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:09:04 smithi083 ceph-mon[79930]: pgmap v12023: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:05.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:09:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:09:05.022Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:09:06.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:09:05 smithi083 ceph-mon[79930]: pgmap v12024: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:06.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:09:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:09:05.740Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:09:06.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:09:05 smithi083 ceph-mon[77242]: pgmap v12024: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:06.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:09:05 smithi153 ceph-mon[70044]: pgmap v12024: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:08.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:09:08 smithi153 ceph-mon[70044]: pgmap v12025: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:08.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:09:08 smithi083 ceph-mon[77242]: pgmap v12025: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:08.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:09:08 smithi083 ceph-mon[79930]: pgmap v12025: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:10.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:09:10 smithi153 ceph-mon[70044]: pgmap v12026: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:10.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:09:10 smithi083 ceph-mon[79930]: pgmap v12026: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:10.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:09:10 smithi083 ceph-mon[77242]: pgmap v12026: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:09:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:09:11] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T12:09:11.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:09:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:09:11.647Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:09:12.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:09:12 smithi153 ceph-mon[70044]: pgmap v12027: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:12.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:09:12 smithi083 ceph-mon[77242]: pgmap v12027: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:12.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:09:12 smithi083 ceph-mon[79930]: pgmap v12027: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:09:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:09:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:09:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:09:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:09:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:09:14.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:09:14 smithi083 ceph-mon[79930]: pgmap v12028: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:14.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:09:14 smithi083 ceph-mon[77242]: pgmap v12028: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:14.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:09:14 smithi153 ceph-mon[70044]: pgmap v12028: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:15.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:09:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:09:15.022Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:09:16.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:09:15 smithi083 ceph-mon[79930]: pgmap v12029: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:16.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:09:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:09:15.741Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:09:16.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:09:15 smithi083 ceph-mon[77242]: pgmap v12029: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:16.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:09:15 smithi153 ceph-mon[70044]: pgmap v12029: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:18.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:09:18 smithi153 ceph-mon[70044]: pgmap v12030: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:18.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:09:18 smithi083 ceph-mon[79930]: pgmap v12030: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:18.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:09:18 smithi083 ceph-mon[77242]: pgmap v12030: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:20.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:09:20 smithi083 ceph-mon[79930]: pgmap v12031: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:20.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:09:20 smithi083 ceph-mon[77242]: pgmap v12031: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:09:20 smithi153 ceph-mon[70044]: pgmap v12031: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:09:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:09:21] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T12:09:21.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:09:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:09:21.647Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:09:22.316 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:09:22 smithi153 ceph-mon[70044]: pgmap v12032: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:22.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:09:22 smithi083 ceph-mon[77242]: pgmap v12032: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:22.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:09:22 smithi083 ceph-mon[79930]: pgmap v12032: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:09:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:09:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:09:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:09:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:09:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:09:24.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:09:24 smithi083 ceph-mon[79930]: pgmap v12033: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:24.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:09:24 smithi083 ceph-mon[77242]: pgmap v12033: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:09:24 smithi153 ceph-mon[70044]: pgmap v12033: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:25.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:09:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:09:25.023Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:09:26.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:09:25 smithi083 ceph-mon[79930]: pgmap v12034: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:26.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:09:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:09:25.742Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:09:26.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:09:25 smithi083 ceph-mon[77242]: pgmap v12034: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:26.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:09:25 smithi153 ceph-mon[70044]: pgmap v12034: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:26.549 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:09:26.840 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:09:26.840 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 5m ago 6h 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:09:26.840 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 3m ago 6h 63.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:09:26.840 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 5m ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:09:26.841 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 3m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:09:26.841 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 3m ago 6h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:09:26.841 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 5m ago 6h 847M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:09:26.841 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 5m ago 6h 1486M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:09:26.842 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 3m ago 6h 641M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:09:26.842 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 5m ago 6h 857M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:09:26.842 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 5m ago 6h 26.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:09:26.842 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 3m ago 6h 26.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:09:26.842 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 5m ago 6h 1074M 2393M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:09:26.843 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 5m ago 6h 1061M 2393M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:09:26.843 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 5m ago 6h 808M 2393M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:09:26.843 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 5m ago 6h 1039M 2393M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:09:26.843 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 3m ago 6h 892M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:09:26.844 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 3m ago 6h 1141M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:09:26.844 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 3m ago 6h 996M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:09:26.844 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 3m ago 6h 1209M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:09:26.844 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 3m ago 6h 163M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:09:27.197 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:09:27.197 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:09:27.198 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:09:27.198 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:09:27.198 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:09:27.198 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:09:27.199 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:09:27.199 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:09:27.199 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:09:27.199 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:09:27.200 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:09:27.200 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:09:27.200 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:09:27.200 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:09:27.201 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:09:27.201 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:09:27.201 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:09:27.201 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:09:27.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:09:26 smithi153 ceph-mon[70044]: from='client.48126 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:09:27.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:09:26 smithi083 ceph-mon[77242]: from='client.48126 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:09:27.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:09:26 smithi083 ceph-mon[79930]: from='client.48126 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:09:28.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:09:27 smithi153 ceph-mon[70044]: from='client.58063 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:09:28.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:09:27 smithi153 ceph-mon[70044]: pgmap v12035: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:28.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:09:27 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1932432716' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:09:28.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:09:27 smithi083 ceph-mon[79930]: from='client.58063 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:09:28.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:09:27 smithi083 ceph-mon[79930]: pgmap v12035: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:28.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:09:27 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1932432716' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:09:28.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:09:27 smithi083 ceph-mon[77242]: from='client.58063 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:09:28.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:09:27 smithi083 ceph-mon[77242]: pgmap v12035: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:28.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:09:27 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1932432716' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:09:30.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:09:30 smithi083 ceph-mon[79930]: pgmap v12036: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:30.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:09:30 smithi083 ceph-mon[77242]: pgmap v12036: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:09:30 smithi153 ceph-mon[70044]: pgmap v12036: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:09:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:09:31] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T12:09:31.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:09:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:09:31.648Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:09:32.315 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:09:32 smithi153 ceph-mon[70044]: pgmap v12037: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:32.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:09:32 smithi083 ceph-mon[79930]: pgmap v12037: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:32.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:09:32 smithi083 ceph-mon[77242]: pgmap v12037: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:09:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:09:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:09:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:09:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:09:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:09:34.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:09:34 smithi083 ceph-mon[79930]: pgmap v12038: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:34.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:09:34 smithi083 ceph-mon[77242]: pgmap v12038: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:09:34 smithi153 ceph-mon[70044]: pgmap v12038: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:35.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:09:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:09:35.023Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:09:36.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:09:35 smithi083 ceph-mon[79930]: pgmap v12039: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:36.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:09:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:09:35.742Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:09:36.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:09:35 smithi083 ceph-mon[77242]: pgmap v12039: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:36.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:09:35 smithi153 ceph-mon[70044]: pgmap v12039: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:38.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:09:38 smithi083 ceph-mon[79930]: pgmap v12040: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:38.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:09:38 smithi083 ceph-mon[77242]: pgmap v12040: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:09:38 smithi153 ceph-mon[70044]: pgmap v12040: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:09:40 smithi083 ceph-mon[79930]: pgmap v12041: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:09:40 smithi083 ceph-mon[77242]: pgmap v12041: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:09:40 smithi153 ceph-mon[70044]: pgmap v12041: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:41.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:09:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:09:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:09:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:09:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:09:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:09:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:09:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:09:41.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:09:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:09:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:09:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:09:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:09:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:09:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:09:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:09:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:09:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:09:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:09:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:09:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:09:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:09:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:09:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:09:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:09:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:09:41] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T12:09:41.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:09:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:09:41.648Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:09:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:09:42 smithi083 ceph-mon[79930]: pgmap v12042: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:42.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:09:42 smithi083 ceph-mon[77242]: pgmap v12042: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:09:42 smithi153 ceph-mon[70044]: pgmap v12042: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:42.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:09:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:09:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:09:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:09:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:09:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:09:44.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:09:44 smithi083 ceph-mon[79930]: pgmap v12043: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:44.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:09:44 smithi083 ceph-mon[77242]: pgmap v12043: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:44.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:09:44 smithi153 ceph-mon[70044]: pgmap v12043: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:45.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:09:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:09:45.023Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:09:46.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:09:45 smithi083 ceph-mon[79930]: pgmap v12044: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:46.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:09:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:09:45.743Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:09:46.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:09:45 smithi083 ceph-mon[77242]: pgmap v12044: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:46.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:09:45 smithi153 ceph-mon[70044]: pgmap v12044: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:48.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:09:48 smithi083 ceph-mon[79930]: pgmap v12045: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:48.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:09:48 smithi083 ceph-mon[77242]: pgmap v12045: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:48.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:09:48 smithi153 ceph-mon[70044]: pgmap v12045: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:49.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:09:49 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:09:49.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:09:49 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:09:49.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:09:49 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:09:49.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:09:49 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:09:49.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:09:49 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:09:49.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:09:49 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:09:49.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:09:49 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:09:49.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:09:49 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:09:49.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:09:49 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:09:50.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:09:50 smithi083 ceph-mon[79930]: pgmap v12046: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:50.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:09:50 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:09:50.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:09:50 smithi083 ceph-mon[77242]: pgmap v12046: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:50.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:09:50 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:09:50.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:09:50 smithi153 ceph-mon[70044]: pgmap v12046: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:50.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:09:50 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:09:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:09:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:09:51] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T12:09:51.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:09:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:09:51.648Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:09:52.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:09:52 smithi083 ceph-mon[77242]: pgmap v12047: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:52.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:09:52 smithi083 ceph-mon[79930]: pgmap v12047: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:52.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:09:52 smithi153 ceph-mon[70044]: pgmap v12047: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:09:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:09:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:09:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:09:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:09:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:09:54.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:09:54 smithi083 ceph-mon[77242]: pgmap v12048: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:54.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:09:54 smithi083 ceph-mon[79930]: pgmap v12048: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:54.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:09:54 smithi153 ceph-mon[70044]: pgmap v12048: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:09:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:09:55.024Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:09:56.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:09:55 smithi083 ceph-mon[77242]: pgmap v12049: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:56.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:09:55 smithi083 ceph-mon[79930]: pgmap v12049: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:56.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:09:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:09:55.744Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:09:56.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:09:55 smithi153 ceph-mon[70044]: pgmap v12049: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:57.522 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:09:57.811 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:09:57.811 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 6m ago 6h 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:09:57.812 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 4m ago 6h 63.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:09:57.812 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 6m ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:09:57.812 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 4m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:09:57.812 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 4m ago 6h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:09:57.813 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 6m ago 6h 847M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:09:57.813 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 6m ago 6h 1486M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:09:57.813 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 4m ago 6h 641M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:09:57.813 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 6m ago 6h 857M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:09:57.813 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 6m ago 6h 26.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:09:57.814 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 4m ago 6h 26.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:09:57.814 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 6m ago 6h 1074M 2393M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:09:57.814 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 6m ago 6h 1061M 2393M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:09:57.814 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 6m ago 6h 808M 2393M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:09:57.815 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 6m ago 6h 1039M 2393M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:09:57.815 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 4m ago 6h 892M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:09:57.815 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 4m ago 6h 1141M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:09:57.816 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 4m ago 6h 996M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:09:57.816 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 4m ago 6h 1209M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:09:57.816 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 4m ago 6h 163M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:09:58.167 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:09:58.167 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:09:58.168 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:09:58.168 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:09:58.168 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:09:58.169 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:09:58.169 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:09:58.169 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:09:58.169 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:09:58.170 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:09:58.170 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:09:58.170 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:09:58.170 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:09:58.171 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:09:58.171 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:09:58.171 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:09:58.171 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:09:58.172 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:09:58.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:09:58 smithi083 ceph-mon[79930]: pgmap v12050: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:58.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:09:58 smithi083 ceph-mon[79930]: from='client.58075 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:09:58.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:09:58 smithi083 ceph-mon[77242]: pgmap v12050: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:58.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:09:58 smithi083 ceph-mon[77242]: from='client.58075 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:09:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:09:58 smithi153 ceph-mon[70044]: pgmap v12050: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:09:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:09:58 smithi153 ceph-mon[70044]: from='client.58075 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:09:59.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:09:59 smithi083 ceph-mon[79930]: from='client.48150 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:09:59.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:09:59 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3101100455' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:09:59.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:09:59 smithi083 ceph-mon[77242]: from='client.48150 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:09:59.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:09:59 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3101100455' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:09:59.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:09:59 smithi153 ceph-mon[70044]: from='client.48150 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:09:59.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:09:59 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3101100455' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:10:00.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:10:00 smithi083 ceph-mon[79930]: pgmap v12051: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:00.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:10:00 smithi083 ceph-mon[79930]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T12:10:00.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:10:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[77219]: 2022-09-29T12:09:59.999+0000 7f160f665700 -1 log_channel(cluster) log [ERR] : overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T12:10:00.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:10:00 smithi083 ceph-mon[77242]: pgmap v12051: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:00.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:10:00 smithi083 ceph-mon[77242]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T12:10:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:10:00 smithi153 ceph-mon[70044]: pgmap v12051: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:10:00 smithi153 ceph-mon[70044]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T12:10:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:10:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:10:01] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T12:10:01.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:10:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:10:01.649Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:10:02.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:10:02 smithi083 ceph-mon[79930]: pgmap v12052: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:02.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:10:02 smithi083 ceph-mon[77242]: pgmap v12052: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:10:02 smithi153 ceph-mon[70044]: pgmap v12052: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:02.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:10:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:10:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:10:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:10:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:10:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:10:04.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:10:04 smithi083 ceph-mon[79930]: pgmap v12053: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:04.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:10:04 smithi083 ceph-mon[77242]: pgmap v12053: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:10:04 smithi153 ceph-mon[70044]: pgmap v12053: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:05.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:10:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:10:05.024Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:10:06.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:10:05 smithi083 ceph-mon[79930]: pgmap v12054: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:06.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:10:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:10:05.745Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:10:06.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:10:05 smithi083 ceph-mon[77242]: pgmap v12054: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:06.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:10:05 smithi153 ceph-mon[70044]: pgmap v12054: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:08.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:10:08 smithi083 ceph-mon[77242]: pgmap v12055: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:08.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:10:08 smithi083 ceph-mon[79930]: pgmap v12055: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:10:08 smithi153 ceph-mon[70044]: pgmap v12055: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:10.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:10:10 smithi083 ceph-mon[79930]: pgmap v12056: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:10.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:10:10 smithi083 ceph-mon[77242]: pgmap v12056: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:10:10 smithi153 ceph-mon[70044]: pgmap v12056: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:10:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:10:11] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T12:10:11.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:10:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:10:11.651Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:10:12.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:10:12 smithi083 ceph-mon[77242]: pgmap v12057: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:12.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:10:12 smithi083 ceph-mon[79930]: pgmap v12057: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:10:12 smithi153 ceph-mon[70044]: pgmap v12057: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:12.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:10:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:10:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:10:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:10:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:10:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:10:14.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:10:14 smithi083 ceph-mon[79930]: pgmap v12058: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:14.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:10:14 smithi083 ceph-mon[77242]: pgmap v12058: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:14.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:10:14 smithi153 ceph-mon[70044]: pgmap v12058: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:15.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:10:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:10:15.025Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:10:16.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:10:15 smithi083 ceph-mon[77242]: pgmap v12059: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:16.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:10:15 smithi083 ceph-mon[79930]: pgmap v12059: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:16.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:10:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:10:15.745Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:10:16.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:10:15 smithi153 ceph-mon[70044]: pgmap v12059: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:18.467 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:10:18 smithi083 ceph-mon[79930]: pgmap v12060: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:18.468 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:10:18 smithi083 ceph-mon[77242]: pgmap v12060: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:10:18 smithi153 ceph-mon[70044]: pgmap v12060: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:20.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:10:20 smithi083 ceph-mon[79930]: pgmap v12061: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:20.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:10:20 smithi083 ceph-mon[77242]: pgmap v12061: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:10:20 smithi153 ceph-mon[70044]: pgmap v12061: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:10:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:10:21] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T12:10:21.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:10:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:10:21.649Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:10:22.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:10:22 smithi083 ceph-mon[79930]: pgmap v12062: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:22.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:10:22 smithi083 ceph-mon[77242]: pgmap v12062: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:10:22 smithi153 ceph-mon[70044]: pgmap v12062: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:22.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:10:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:10:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:10:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:10:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:10:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:10:24.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:10:24 smithi083 ceph-mon[79930]: pgmap v12063: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:24.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:10:24 smithi083 ceph-mon[77242]: pgmap v12063: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:10:24 smithi153 ceph-mon[70044]: pgmap v12063: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:25.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:10:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:10:25.025Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:10:26.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:10:25 smithi083 ceph-mon[79930]: pgmap v12064: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:26.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:10:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:10:25.745Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:10:26.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:10:25 smithi083 ceph-mon[77242]: pgmap v12064: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:26.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:10:25 smithi153 ceph-mon[70044]: pgmap v12064: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:28.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:10:28 smithi083 ceph-mon[79930]: pgmap v12065: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:28.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:10:28 smithi083 ceph-mon[77242]: pgmap v12065: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:28.493 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:10:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:10:28 smithi153 ceph-mon[70044]: pgmap v12065: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:28.782 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:10:28.782 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 6m ago 6h 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:10:28.783 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 4m ago 6h 63.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:10:28.783 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 6m ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:10:28.783 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 4m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:10:28.784 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 4m ago 6h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:10:28.784 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 6m ago 6h 847M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:10:28.784 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 6m ago 6h 1486M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:10:28.785 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 4m ago 6h 641M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:10:28.785 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 6m ago 6h 857M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:10:28.785 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 6m ago 6h 26.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:10:28.786 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 4m ago 6h 26.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:10:28.786 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 6m ago 6h 1074M 2393M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:10:28.786 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 6m ago 6h 1061M 2393M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:10:28.787 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 6m ago 6h 808M 2393M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:10:28.787 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 6m ago 6h 1039M 2393M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:10:28.787 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 4m ago 6h 892M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:10:28.787 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 4m ago 6h 1141M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:10:28.788 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 4m ago 6h 996M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:10:28.788 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 4m ago 6h 1209M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:10:28.788 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 4m ago 6h 163M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:10:29.140 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:10:29.140 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:10:29.140 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:10:29.141 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:10:29.141 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:10:29.141 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:10:29.141 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:10:29.142 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:10:29.142 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:10:29.142 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:10:29.142 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:10:29.143 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:10:29.143 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:10:29.143 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:10:29.143 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:10:29.143 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:10:29.144 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:10:29.144 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:10:29.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:10:29 smithi083 ceph-mon[79930]: from='client.58093 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:10:29.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:10:29 smithi083 ceph-mon[77242]: from='client.58093 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:10:29.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:10:29 smithi153 ceph-mon[70044]: from='client.58093 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:10:30.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:10:30 smithi083 ceph-mon[79930]: from='client.58099 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:10:30.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:10:30 smithi083 ceph-mon[79930]: pgmap v12066: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:30.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:10:30 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3121586764' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:10:30.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:10:30 smithi083 ceph-mon[77242]: from='client.58099 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:10:30.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:10:30 smithi083 ceph-mon[77242]: pgmap v12066: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:30.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:10:30 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3121586764' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:10:30.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:10:30 smithi153 ceph-mon[70044]: from='client.58099 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:10:30.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:10:30 smithi153 ceph-mon[70044]: pgmap v12066: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:30.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:10:30 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3121586764' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:10:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:10:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:10:31] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T12:10:31.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:10:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:10:31.650Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:10:32.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:10:32 smithi083 ceph-mon[79930]: pgmap v12067: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:32.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:10:32 smithi083 ceph-mon[77242]: pgmap v12067: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:10:32 smithi153 ceph-mon[70044]: pgmap v12067: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:32.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:10:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:10:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:10:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:10:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:10:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:10:34.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:10:34 smithi083 ceph-mon[79930]: pgmap v12068: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:34.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:10:34 smithi083 ceph-mon[77242]: pgmap v12068: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:10:34 smithi153 ceph-mon[70044]: pgmap v12068: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:35.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:10:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:10:35.026Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:10:36.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:10:35 smithi083 ceph-mon[79930]: pgmap v12069: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:36.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:10:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:10:35.746Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:10:36.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:10:35 smithi083 ceph-mon[77242]: pgmap v12069: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:36.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:10:35 smithi153 ceph-mon[70044]: pgmap v12069: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:38.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:10:38 smithi083 ceph-mon[79930]: pgmap v12070: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:38.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:10:38 smithi083 ceph-mon[77242]: pgmap v12070: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:10:38 smithi153 ceph-mon[70044]: pgmap v12070: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:10:40 smithi083 ceph-mon[79930]: pgmap v12071: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:10:40 smithi083 ceph-mon[77242]: pgmap v12071: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:10:40 smithi153 ceph-mon[70044]: pgmap v12071: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:41.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:10:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:10:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:10:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:10:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:10:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:10:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:10:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:10:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:10:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:10:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:10:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:10:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:10:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:10:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:10:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:10:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:10:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:10:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:10:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:10:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:10:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:10:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:10:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:10:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:10:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:10:41] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T12:10:41.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:10:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:10:41.650Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:10:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:10:42 smithi083 ceph-mon[79930]: pgmap v12072: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:42.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:10:42 smithi083 ceph-mon[77242]: pgmap v12072: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:10:42 smithi153 ceph-mon[70044]: pgmap v12072: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:42.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:10:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:10:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:10:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:10:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:10:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:10:44.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:10:44 smithi083 ceph-mon[79930]: pgmap v12073: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:44.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:10:44 smithi083 ceph-mon[77242]: pgmap v12073: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:44.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:10:44 smithi153 ceph-mon[70044]: pgmap v12073: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:45.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:10:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:10:45.026Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:10:46.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:10:45 smithi083 ceph-mon[79930]: pgmap v12074: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:46.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:10:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:10:45.747Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:10:46.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:10:45 smithi083 ceph-mon[77242]: pgmap v12074: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:46.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:10:45 smithi153 ceph-mon[70044]: pgmap v12074: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:48.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:10:48 smithi083 ceph-mon[79930]: pgmap v12075: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:48.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:10:48 smithi083 ceph-mon[77242]: pgmap v12075: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:48.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:10:48 smithi153 ceph-mon[70044]: pgmap v12075: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:50.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:10:50 smithi083 ceph-mon[79930]: pgmap v12076: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:50.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:10:50 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:10:50.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:10:50 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:10:50.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:10:50 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:10:50.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:10:50 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:10:50.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:10:50 smithi083 ceph-mon[77242]: pgmap v12076: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:50.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:10:50 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:10:50.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:10:50 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:10:50.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:10:50 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:10:50.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:10:50 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:10:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:10:50 smithi153 ceph-mon[70044]: pgmap v12076: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:10:50 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:10:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:10:50 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:10:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:10:50 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:10:50.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:10:50 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:10:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:10:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:10:51] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T12:10:51.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:10:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:10:51.651Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:10:52.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:10:52 smithi083 ceph-mon[77242]: pgmap v12077: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:52.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:10:52 smithi083 ceph-mon[79930]: pgmap v12077: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:10:52 smithi153 ceph-mon[70044]: pgmap v12077: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:52.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:10:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:10:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:10:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:10:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:10:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:10:54.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:10:54 smithi083 ceph-mon[79930]: pgmap v12078: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:54.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:10:54 smithi083 ceph-mon[77242]: pgmap v12078: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:54.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:10:54 smithi153 ceph-mon[70044]: pgmap v12078: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:10:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:10:55.027Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:10:56.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:10:55 smithi083 ceph-mon[77242]: pgmap v12079: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:56.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:10:55 smithi083 ceph-mon[79930]: pgmap v12079: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:56.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:10:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:10:55.747Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:10:56.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:10:55 smithi153 ceph-mon[70044]: pgmap v12079: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:58.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:10:58 smithi083 ceph-mon[79930]: pgmap v12080: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:58.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:10:58 smithi083 ceph-mon[77242]: pgmap v12080: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:10:58 smithi153 ceph-mon[70044]: pgmap v12080: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:10:59.465 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:10:59.752 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:10:59.752 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 7m ago 6h 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:10:59.752 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 5m ago 6h 63.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:10:59.753 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 7m ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:10:59.753 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 5m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:10:59.753 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 5m ago 6h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:10:59.753 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 7m ago 6h 847M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:10:59.754 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 7m ago 6h 1486M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:10:59.754 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 5m ago 6h 641M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:10:59.754 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 7m ago 6h 857M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:10:59.754 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 7m ago 6h 26.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:10:59.755 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 5m ago 6h 26.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:10:59.755 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 7m ago 6h 1074M 2393M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:10:59.755 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 7m ago 6h 1061M 2393M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:10:59.755 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 7m ago 6h 808M 2393M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:10:59.756 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 7m ago 6h 1039M 2393M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:10:59.756 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 5m ago 6h 892M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:10:59.756 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 5m ago 6h 1141M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:10:59.756 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 5m ago 6h 996M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:10:59.756 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 5m ago 6h 1209M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:10:59.757 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 5m ago 6h 163M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:11:00.111 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:11:00.112 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:11:00.112 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:11:00.112 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:11:00.113 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:11:00.113 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:11:00.113 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:11:00.113 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:11:00.114 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:11:00.114 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:11:00.114 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:11:00.114 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:11:00.115 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:11:00.115 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:11:00.115 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:11:00.115 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:11:00.116 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:11:00.116 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:11:00.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:11:00 smithi083 ceph-mon[77242]: pgmap v12081: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:11:00.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:11:00 smithi083 ceph-mon[77242]: from='client.48180 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:11:00.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:11:00 smithi083 ceph-mon[79930]: pgmap v12081: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:11:00.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:11:00 smithi083 ceph-mon[79930]: from='client.48180 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:11:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:11:00 smithi153 ceph-mon[70044]: pgmap v12081: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:11:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:11:00 smithi153 ceph-mon[70044]: from='client.48180 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:11:01.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:11:01 smithi083 ceph-mon[79930]: from='client.48186 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:11:01.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:11:01 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2834190949' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:11:01.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:11:01 smithi083 ceph-mon[77242]: from='client.48186 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:11:01.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:11:01 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2834190949' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:11:01.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:11:01 smithi153 ceph-mon[70044]: from='client.48186 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:11:01.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:11:01 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2834190949' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:11:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:11:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:11:01] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T12:11:01.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:11:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:11:01.651Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:11:02.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:11:02 smithi083 ceph-mon[79930]: pgmap v12082: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:11:02.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:11:02 smithi083 ceph-mon[77242]: pgmap v12082: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:11:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:11:02 smithi153 ceph-mon[70044]: pgmap v12082: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:11:02.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:11:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:11:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:11:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:11:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:11:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:11:04.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:11:04 smithi083 ceph-mon[79930]: pgmap v12083: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:11:04.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:11:04 smithi083 ceph-mon[77242]: pgmap v12083: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:11:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:11:04 smithi153 ceph-mon[70044]: pgmap v12083: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:11:05.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:11:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:11:05.027Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:11:06.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:11:05 smithi083 ceph-mon[79930]: pgmap v12084: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:11:06.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:11:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:11:05.748Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:11:06.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:11:05 smithi083 ceph-mon[77242]: pgmap v12084: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:11:06.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:11:05 smithi153 ceph-mon[70044]: pgmap v12084: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:11:08.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:11:08 smithi083 ceph-mon[77242]: pgmap v12085: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:11:08.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:11:08 smithi083 ceph-mon[79930]: pgmap v12085: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:11:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:11:08 smithi153 ceph-mon[70044]: pgmap v12085: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:11:10.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:11:10 smithi083 ceph-mon[77242]: pgmap v12086: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:11:10.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:11:10 smithi083 ceph-mon[79930]: pgmap v12086: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:11:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:11:10 smithi153 ceph-mon[70044]: pgmap v12086: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:11:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:11:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:11:11] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T12:11:11.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:11:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:11:11.654Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:11:12.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:11:12 smithi083 ceph-mon[77242]: pgmap v12087: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:11:12.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:11:12 smithi083 ceph-mon[79930]: pgmap v12087: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:11:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:11:12 smithi153 ceph-mon[70044]: pgmap v12087: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:11:12.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:11:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:11:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:11:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:11:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:11:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:11:14.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:11:14 smithi083 ceph-mon[79930]: pgmap v12088: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:11:14.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:11:14 smithi083 ceph-mon[77242]: pgmap v12088: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:11:14.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:11:14 smithi153 ceph-mon[70044]: pgmap v12088: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:11:15.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:11:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:11:15.028Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:11:16.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:11:15 smithi083 ceph-mon[79930]: pgmap v12089: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:11:16.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:11:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:11:15.748Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:11:16.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:11:15 smithi083 ceph-mon[77242]: pgmap v12089: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:11:16.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:11:15 smithi153 ceph-mon[70044]: pgmap v12089: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:11:18.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:11:18 smithi083 ceph-mon[79930]: pgmap v12090: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:11:18.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:11:18 smithi083 ceph-mon[77242]: pgmap v12090: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:11:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:11:18 smithi153 ceph-mon[70044]: pgmap v12090: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:11:20.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:11:20 smithi083 ceph-mon[79930]: pgmap v12091: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:11:20.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:11:20 smithi083 ceph-mon[77242]: pgmap v12091: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:11:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:11:20 smithi153 ceph-mon[70044]: pgmap v12091: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:11:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:11:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:11:21] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T12:11:21.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:11:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:11:21.653Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:11:22.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:11:22 smithi083 ceph-mon[79930]: pgmap v12092: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:11:22.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:11:22 smithi083 ceph-mon[77242]: pgmap v12092: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:11:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:11:22 smithi153 ceph-mon[70044]: pgmap v12092: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:11:22.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:11:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:11:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:11:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:11:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:11:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:11:24.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:11:24 smithi083 ceph-mon[77242]: pgmap v12093: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:11:24.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:11:24 smithi083 ceph-mon[79930]: pgmap v12093: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:11:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:11:24 smithi153 ceph-mon[70044]: pgmap v12093: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:11:25.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:11:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:11:25.028Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:11:26.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:11:25 smithi083 ceph-mon[79930]: pgmap v12094: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:11:26.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:11:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:11:25.749Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:11:26.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:11:25 smithi083 ceph-mon[77242]: pgmap v12094: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:11:26.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:11:25 smithi153 ceph-mon[70044]: pgmap v12094: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:11:28.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:11:28 smithi083 ceph-mon[79930]: pgmap v12095: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:11:28.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:11:28 smithi083 ceph-mon[77242]: pgmap v12095: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:11:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:11:28 smithi153 ceph-mon[70044]: pgmap v12095: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:11:30.439 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:11:30.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:11:30 smithi083 ceph-mon[79930]: pgmap v12096: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:11:30.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:11:30 smithi083 ceph-mon[77242]: pgmap v12096: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:11:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:11:30 smithi153 ceph-mon[70044]: pgmap v12096: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:11:30.729 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:11:30.729 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 7m ago 6h 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:11:30.730 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 5m ago 6h 63.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:11:30.730 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 7m ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:11:30.730 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 5m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:11:30.731 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 5m ago 6h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:11:30.731 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 7m ago 6h 847M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:11:30.731 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 7m ago 6h 1486M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:11:30.731 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 5m ago 6h 641M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:11:30.731 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 7m ago 6h 857M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:11:30.732 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 7m ago 6h 26.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:11:30.732 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 5m ago 6h 26.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:11:30.732 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 7m ago 6h 1074M 2393M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:11:30.732 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 7m ago 6h 1061M 2393M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:11:30.733 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 7m ago 6h 808M 2393M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:11:30.733 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 7m ago 6h 1039M 2393M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:11:30.733 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 5m ago 6h 892M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:11:30.733 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 5m ago 6h 1141M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:11:30.734 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 5m ago 6h 996M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:11:30.734 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 5m ago 6h 1209M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:11:30.734 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 5m ago 6h 163M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:11:31.087 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:11:31.087 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:11:31.087 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:11:31.088 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:11:31.088 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:11:31.088 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:11:31.088 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:11:31.089 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:11:31.089 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:11:31.089 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:11:31.089 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:11:31.089 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:11:31.090 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:11:31.090 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:11:31.090 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:11:31.090 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:11:31.090 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:11:31.091 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:11:31.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:11:31 smithi083 ceph-mon[79930]: from='client.58129 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:11:31.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:11:31 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2508910' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:11:31.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:11:31 smithi083 ceph-mon[77242]: from='client.58129 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:11:31.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:11:31 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2508910' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:11:31.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:11:31 smithi153 ceph-mon[70044]: from='client.58129 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:11:31.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:11:31 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2508910' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:11:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:11:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:11:31] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T12:11:31.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:11:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:11:31.653Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:11:32.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:11:32 smithi083 ceph-mon[79930]: from='client.58135 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:11:32.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:11:32 smithi083 ceph-mon[79930]: pgmap v12097: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:11:32.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:11:32 smithi083 ceph-mon[77242]: from='client.58135 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:11:32.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:11:32 smithi083 ceph-mon[77242]: pgmap v12097: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:11:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:11:32 smithi153 ceph-mon[70044]: from='client.58135 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:11:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:11:32 smithi153 ceph-mon[70044]: pgmap v12097: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:11:32.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:11:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:11:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:11:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:11:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:11:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:11:34.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:11:34 smithi083 ceph-mon[79930]: pgmap v12098: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:11:34.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:11:34 smithi083 ceph-mon[77242]: pgmap v12098: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:11:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:11:34 smithi153 ceph-mon[70044]: pgmap v12098: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:11:35.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:11:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:11:35.029Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:11:36.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:11:36 smithi083 ceph-mon[79930]: pgmap v12099: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:11:36.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:11:35 smithi083 ceph-mon[77242]: pgmap v12099: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:11:36.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:11:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:11:35.751Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:11:36.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:11:35 smithi153 ceph-mon[70044]: pgmap v12099: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:11:38.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:11:38 smithi083 ceph-mon[79930]: pgmap v12100: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:11:38.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:11:38 smithi083 ceph-mon[77242]: pgmap v12100: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:11:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:11:38 smithi153 ceph-mon[70044]: pgmap v12100: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:11:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:11:40 smithi083 ceph-mon[79930]: pgmap v12101: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:11:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:11:40 smithi083 ceph-mon[77242]: pgmap v12101: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:11:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:11:40 smithi153 ceph-mon[70044]: pgmap v12101: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:11:41.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:11:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:11:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:11:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:11:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:11:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:11:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:11:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:11:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:11:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:11:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:11:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:11:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:11:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:11:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:11:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:11:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:11:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:11:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:11:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:11:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:11:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:11:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:11:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:11:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:11:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:11:41] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T12:11:41.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:11:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:11:41.654Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:11:42.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:11:42 smithi083 ceph-mon[77242]: pgmap v12102: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:11:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:11:42 smithi083 ceph-mon[79930]: pgmap v12102: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:11:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:11:42 smithi153 ceph-mon[70044]: pgmap v12102: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:11:42.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:11:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:11:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:11:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:11:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:11:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:11:44.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:11:44 smithi083 ceph-mon[79930]: pgmap v12103: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:11:44.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:11:44 smithi083 ceph-mon[77242]: pgmap v12103: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:11:44.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:11:44 smithi153 ceph-mon[70044]: pgmap v12103: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:11:45.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:11:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:11:45.030Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:11:46.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:11:45 smithi083 ceph-mon[79930]: pgmap v12104: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:11:46.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:11:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:11:45.751Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:11:46.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:11:45 smithi083 ceph-mon[77242]: pgmap v12104: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:11:46.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:11:45 smithi153 ceph-mon[70044]: pgmap v12104: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:11:48.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:11:48 smithi083 ceph-mon[79930]: pgmap v12105: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:11:48.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:11:48 smithi083 ceph-mon[77242]: pgmap v12105: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:11:48.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:11:48 smithi153 ceph-mon[70044]: pgmap v12105: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:11:50.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:11:50 smithi083 ceph-mon[79930]: pgmap v12106: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:11:50.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:11:50 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:11:50.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:11:50 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:11:50.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:11:50 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:11:50.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:11:50 smithi083 ceph-mon[77242]: pgmap v12106: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:11:50.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:11:50 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:11:50.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:11:50 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:11:50.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:11:50 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:11:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:11:50 smithi153 ceph-mon[70044]: pgmap v12106: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:11:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:11:50 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:11:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:11:50 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:11:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:11:50 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:11:51.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:11:51 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:11:51.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:11:51 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:11:51.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:11:51 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:11:51.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:11:51 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:11:51.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:11:51 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:11:51.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:11:51 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:11:51.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:11:51 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:11:51.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:11:51 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:11:51.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:11:51 smithi083 ceph-mon[79930]: Adjusting osd_memory_target on smithi083 to 2384M 2022-09-29T12:11:51.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:11:51 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:11:51.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:11:51 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:11:51.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:11:51 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:11:51.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:11:51 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:11:51.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:11:51 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:11:51.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:11:51 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:11:51.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:11:51 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:11:51.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:11:51 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:11:51.467 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:11:51 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:11:51.467 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:11:51 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:11:51.467 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:11:51 smithi083 ceph-mon[77242]: Adjusting osd_memory_target on smithi083 to 2384M 2022-09-29T12:11:51.467 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:11:51 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:11:51.468 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:11:51 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:11:51.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:11:51 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:11:51.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:11:51 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:11:51.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:11:51 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:11:51.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:11:51 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:11:51.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:11:51 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:11:51.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:11:51 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:11:51.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:11:51 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:11:51.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:11:51 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:11:51.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:11:51 smithi153 ceph-mon[70044]: Adjusting osd_memory_target on smithi083 to 2384M 2022-09-29T12:11:51.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:11:51 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:11:51.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:11:51 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:11:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:11:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:11:51] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T12:11:51.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:11:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:11:51.655Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:11:52.649 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:11:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:11:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:11:52.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:11:52 smithi083 ceph-mon[77242]: pgmap v12107: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:11:52.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:11:52 smithi083 ceph-mon[79930]: pgmap v12107: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:11:53.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:11:52 smithi153 ceph-mon[70044]: pgmap v12107: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:11:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:11:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:11:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:11:54.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:11:54 smithi083 ceph-mon[79930]: pgmap v12108: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:11:54.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:11:54 smithi083 ceph-mon[77242]: pgmap v12108: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:11:55.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:11:54 smithi153 ceph-mon[70044]: pgmap v12108: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:11:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:11:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:11:55.031Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:11:56.007 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:11:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:11:55.752Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:11:56.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:11:56 smithi153 ceph-mon[70044]: pgmap v12109: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:11:56.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:11:56 smithi083 ceph-mon[79930]: pgmap v12109: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:11:56.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:11:56 smithi083 ceph-mon[77242]: pgmap v12109: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:11:58.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:11:58 smithi083 ceph-mon[79930]: pgmap v12110: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:11:58.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:11:58 smithi083 ceph-mon[77242]: pgmap v12110: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:11:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:11:58 smithi153 ceph-mon[70044]: pgmap v12110: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:00.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:12:00 smithi083 ceph-mon[77242]: pgmap v12111: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:00.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:12:00 smithi083 ceph-mon[79930]: pgmap v12111: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:12:00 smithi153 ceph-mon[70044]: pgmap v12111: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:01.414 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:12:01.704 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:12:01.704 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 8m ago 6h 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:12:01.704 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 6m ago 6h 63.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:12:01.704 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 8m ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:12:01.705 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 6m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:12:01.705 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 6m ago 6h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:12:01.705 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 8m ago 6h 847M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:12:01.705 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 8m ago 6h 1486M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:12:01.706 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 6m ago 6h 641M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:12:01.706 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 8m ago 6h 857M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:12:01.706 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 8m ago 6h 26.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:12:01.706 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 6m ago 6h 26.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:12:01.707 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 8m ago 6h 1074M 2384M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:12:01.707 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 8m ago 6h 1061M 2384M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:12:01.707 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 8m ago 6h 808M 2384M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:12:01.707 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 8m ago 6h 1039M 2384M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:12:01.707 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 6m ago 6h 892M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:12:01.708 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 6m ago 6h 1141M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:12:01.708 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 6m ago 6h 996M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:12:01.708 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 6m ago 6h 1209M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:12:01.708 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 6m ago 6h 163M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:12:01.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:12:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:12:01] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T12:12:01.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:12:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:12:01.654Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:12:02.062 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:12:02.063 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:12:02.063 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:12:02.063 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:12:02.063 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:12:02.063 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:12:02.064 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:12:02.064 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:12:02.064 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:12:02.064 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:12:02.065 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:12:02.065 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:12:02.065 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:12:02.065 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:12:02.065 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:12:02.066 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:12:02.066 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:12:02.066 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:12:02.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:12:02 smithi083 ceph-mon[79930]: pgmap v12112: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:02.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:12:02 smithi083 ceph-mon[79930]: from='client.48216 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:12:02.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:12:02 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1742501097' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:12:02.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:12:02 smithi083 ceph-mon[77242]: pgmap v12112: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:02.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:12:02 smithi083 ceph-mon[77242]: from='client.48216 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:12:02.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:12:02 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1742501097' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:12:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:12:02 smithi153 ceph-mon[70044]: pgmap v12112: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:12:02 smithi153 ceph-mon[70044]: from='client.48216 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:12:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:12:02 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1742501097' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:12:02.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:12:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:12:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:12:03.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:12:03 smithi153 ceph-mon[70044]: from='client.58153 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:12:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:12:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:12:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:12:03.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:12:03 smithi083 ceph-mon[79930]: from='client.58153 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:12:03.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:12:03 smithi083 ceph-mon[77242]: from='client.58153 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:12:04.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:12:04 smithi083 ceph-mon[79930]: pgmap v12113: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:04.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:12:04 smithi083 ceph-mon[77242]: pgmap v12113: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:12:04 smithi153 ceph-mon[70044]: pgmap v12113: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:05.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:12:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:12:05.032Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:12:06.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:12:06 smithi083 ceph-mon[79930]: pgmap v12114: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:06.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:12:06 smithi083 ceph-mon[77242]: pgmap v12114: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:06.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:12:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:12:05.753Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:12:06.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:12:06 smithi153 ceph-mon[70044]: pgmap v12114: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:08.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:12:08 smithi083 ceph-mon[79930]: pgmap v12115: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:08.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:12:08 smithi083 ceph-mon[77242]: pgmap v12115: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:12:08 smithi153 ceph-mon[70044]: pgmap v12115: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:10.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:12:10 smithi083 ceph-mon[79930]: pgmap v12116: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:10.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:12:10 smithi083 ceph-mon[77242]: pgmap v12116: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:12:10 smithi153 ceph-mon[70044]: pgmap v12116: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:12:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:12:11] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T12:12:11.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:12:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:12:11.656Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:12:12.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:12:12 smithi083 ceph-mon[77242]: pgmap v12117: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:12.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:12:12 smithi083 ceph-mon[79930]: pgmap v12117: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:12:12 smithi153 ceph-mon[70044]: pgmap v12117: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:12.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:12:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:12:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:12:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:12:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:12:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:12:14.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:12:14 smithi083 ceph-mon[79930]: pgmap v12118: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:14.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:12:14 smithi083 ceph-mon[77242]: pgmap v12118: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:14.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:12:14 smithi153 ceph-mon[70044]: pgmap v12118: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:15.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:12:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:12:15.032Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:12:16.006 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:12:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:12:15.753Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:12:16.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:12:16 smithi153 ceph-mon[70044]: pgmap v12119: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:16.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:12:16 smithi083 ceph-mon[79930]: pgmap v12119: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:16.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:12:16 smithi083 ceph-mon[77242]: pgmap v12119: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:18.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:12:18 smithi083 ceph-mon[79930]: pgmap v12120: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:18.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:12:18 smithi083 ceph-mon[77242]: pgmap v12120: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:12:18 smithi153 ceph-mon[70044]: pgmap v12120: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:20.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:12:20 smithi083 ceph-mon[79930]: pgmap v12121: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:20.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:12:20 smithi083 ceph-mon[77242]: pgmap v12121: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:12:20 smithi153 ceph-mon[70044]: pgmap v12121: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:12:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:12:21] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T12:12:21.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:12:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:12:21.656Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:12:22.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:12:22 smithi083 ceph-mon[79930]: pgmap v12122: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:22.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:12:22 smithi083 ceph-mon[77242]: pgmap v12122: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:12:22 smithi153 ceph-mon[70044]: pgmap v12122: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:22.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:12:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:12:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:12:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:12:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:12:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:12:24.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:12:24 smithi083 ceph-mon[77242]: pgmap v12123: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:24.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:12:24 smithi083 ceph-mon[79930]: pgmap v12123: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:12:24 smithi153 ceph-mon[70044]: pgmap v12123: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:25.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:12:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:12:25.032Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:12:26.008 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:12:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:12:25.754Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:12:26.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:12:26 smithi153 ceph-mon[70044]: pgmap v12124: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:26.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:12:26 smithi083 ceph-mon[77242]: pgmap v12124: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:26.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:12:26 smithi083 ceph-mon[79930]: pgmap v12124: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:28.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:12:28 smithi083 ceph-mon[79930]: pgmap v12125: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:28.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:12:28 smithi083 ceph-mon[77242]: pgmap v12125: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:12:28 smithi153 ceph-mon[70044]: pgmap v12125: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:30.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:12:30 smithi083 ceph-mon[79930]: pgmap v12126: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:30.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:12:30 smithi083 ceph-mon[77242]: pgmap v12126: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:12:30 smithi153 ceph-mon[70044]: pgmap v12126: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:12:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:12:31] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T12:12:31.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:12:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:12:31.657Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:12:32.392 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:12:32.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:12:32 smithi083 ceph-mon[79930]: pgmap v12127: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:32.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:12:32 smithi083 ceph-mon[77242]: pgmap v12127: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:12:32 smithi153 ceph-mon[70044]: pgmap v12127: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:32.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:12:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:12:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:12:32.679 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:12:32.679 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 8m ago 6h 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:12:32.679 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 6m ago 6h 63.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:12:32.679 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 8m ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:12:32.680 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 6m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:12:32.680 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 6m ago 6h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:12:32.680 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 8m ago 6h 847M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:12:32.680 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 8m ago 6h 1486M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:12:32.681 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 6m ago 6h 641M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:12:32.681 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 8m ago 6h 857M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:12:32.681 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 8m ago 6h 26.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:12:32.681 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 6m ago 6h 26.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:12:32.681 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 8m ago 6h 1074M 2384M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:12:32.682 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 8m ago 6h 1061M 2384M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:12:32.682 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 8m ago 6h 808M 2384M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:12:32.682 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 8m ago 6h 1039M 2384M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:12:32.682 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 6m ago 6h 892M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:12:32.683 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 6m ago 6h 1141M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:12:32.683 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 6m ago 6h 996M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:12:32.683 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 6m ago 6h 1209M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:12:32.683 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 6m ago 6h 163M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:12:33.034 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:12:33.035 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:12:33.035 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:12:33.035 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:12:33.035 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:12:33.036 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:12:33.036 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:12:33.036 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:12:33.036 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:12:33.036 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:12:33.037 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:12:33.037 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:12:33.037 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:12:33.037 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:12:33.038 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:12:33.038 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:12:33.038 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:12:33.038 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:12:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:12:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:12:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:12:33.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:12:33 smithi153 ceph-mon[70044]: from='client.48234 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:12:33.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:12:33 smithi153 ceph-mon[70044]: from='client.58171 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:12:33.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:12:33 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4202595136' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:12:33.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:12:33 smithi083 ceph-mon[79930]: from='client.48234 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:12:33.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:12:33 smithi083 ceph-mon[79930]: from='client.58171 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:12:33.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:12:33 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4202595136' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:12:33.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:12:33 smithi083 ceph-mon[77242]: from='client.48234 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:12:33.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:12:33 smithi083 ceph-mon[77242]: from='client.58171 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:12:33.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:12:33 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4202595136' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:12:34.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:12:34 smithi083 ceph-mon[79930]: pgmap v12128: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:34.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:12:34 smithi083 ceph-mon[77242]: pgmap v12128: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:12:34 smithi153 ceph-mon[70044]: pgmap v12128: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:35.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:12:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:12:35.034Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:12:36.011 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:12:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:12:35.755Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:12:36.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:12:36 smithi153 ceph-mon[70044]: pgmap v12129: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:36.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:12:36 smithi083 ceph-mon[77242]: pgmap v12129: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:36.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:12:36 smithi083 ceph-mon[79930]: pgmap v12129: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:38.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:12:38 smithi083 ceph-mon[77242]: pgmap v12130: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:38.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:12:38 smithi083 ceph-mon[79930]: pgmap v12130: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:12:38 smithi153 ceph-mon[70044]: pgmap v12130: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:12:40 smithi083 ceph-mon[79930]: pgmap v12131: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:12:40 smithi083 ceph-mon[77242]: pgmap v12131: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:12:40 smithi153 ceph-mon[70044]: pgmap v12131: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:41.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:12:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:12:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:12:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:12:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:12:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:12:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:12:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:12:41.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:12:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:12:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:12:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:12:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:12:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:12:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:12:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:12:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:12:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:12:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:12:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:12:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:12:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:12:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:12:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:12:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:12:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:12:41] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T12:12:41.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:12:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:12:41.657Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:12:42.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:12:42 smithi083 ceph-mon[77242]: pgmap v12132: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:42.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:12:42 smithi083 ceph-mon[79930]: pgmap v12132: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:12:42 smithi153 ceph-mon[70044]: pgmap v12132: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:42.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:12:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:12:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:12:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:12:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:12:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:12:44.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:12:44 smithi083 ceph-mon[77242]: pgmap v12133: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:44.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:12:44 smithi083 ceph-mon[79930]: pgmap v12133: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:44.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:12:44 smithi153 ceph-mon[70044]: pgmap v12133: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:45.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:12:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:12:45.034Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:12:46.012 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:12:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:12:45.756Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:12:46.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:12:46 smithi153 ceph-mon[70044]: pgmap v12134: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:46.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:12:46 smithi083 ceph-mon[79930]: pgmap v12134: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:46.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:12:46 smithi083 ceph-mon[77242]: pgmap v12134: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:48.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:12:48 smithi083 ceph-mon[79930]: pgmap v12135: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:48.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:12:48 smithi083 ceph-mon[77242]: pgmap v12135: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:48.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:12:48 smithi153 ceph-mon[70044]: pgmap v12135: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:50.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:12:50 smithi083 ceph-mon[79930]: pgmap v12136: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:50.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:12:50 smithi083 ceph-mon[77242]: pgmap v12136: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:12:50 smithi153 ceph-mon[70044]: pgmap v12136: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:51.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:12:51 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:12:51.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:12:51 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:12:51.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:12:51 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:12:51.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:12:51 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:12:51.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:12:51 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:12:51.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:12:51 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:12:51.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:12:51 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:12:51.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:12:51 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:12:51.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:12:51 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:12:51.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:12:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:12:51.657Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:12:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:12:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:12:51] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T12:12:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:12:52 smithi153 ceph-mon[70044]: pgmap v12137: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:12:52 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:12:52.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:12:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:12:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:12:52.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:12:52 smithi083 ceph-mon[79930]: pgmap v12137: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:52.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:12:52 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:12:52.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:12:52 smithi083 ceph-mon[77242]: pgmap v12137: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:52.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:12:52 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:12:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:12:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:12:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:12:54.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:12:54 smithi153 ceph-mon[70044]: pgmap v12138: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:54.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:12:54 smithi083 ceph-mon[79930]: pgmap v12138: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:54.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:12:54 smithi083 ceph-mon[77242]: pgmap v12138: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:12:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:12:55.035Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:12:56.017 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:12:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:12:55.756Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:12:56.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:12:56 smithi153 ceph-mon[70044]: pgmap v12139: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:56.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:12:56 smithi083 ceph-mon[79930]: pgmap v12139: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:56.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:12:56 smithi083 ceph-mon[77242]: pgmap v12139: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:58.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:12:58 smithi083 ceph-mon[79930]: pgmap v12140: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:58.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:12:58 smithi083 ceph-mon[77242]: pgmap v12140: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:12:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:12:58 smithi153 ceph-mon[70044]: pgmap v12140: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:00.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:13:00 smithi083 ceph-mon[79930]: pgmap v12141: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:00.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:13:00 smithi083 ceph-mon[77242]: pgmap v12141: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:13:00 smithi153 ceph-mon[70044]: pgmap v12141: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:13:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:13:01] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T12:13:01.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:13:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:13:01.659Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:13:02.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:13:02 smithi083 ceph-mon[77242]: pgmap v12142: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:02.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:13:02 smithi083 ceph-mon[79930]: pgmap v12142: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:13:02 smithi153 ceph-mon[70044]: pgmap v12142: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:02.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:13:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:13:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:13:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:13:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:13:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:13:03.364 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:13:03.658 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:13:03.658 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 9m ago 6h 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:13:03.658 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 7m ago 6h 63.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:13:03.659 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 9m ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:13:03.659 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 7m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:13:03.659 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 7m ago 6h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:13:03.659 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 9m ago 6h 847M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:13:03.660 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 9m ago 6h 1486M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:13:03.660 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 7m ago 6h 641M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:13:03.660 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 9m ago 6h 857M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:13:03.660 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 9m ago 6h 26.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:13:03.661 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 7m ago 6h 26.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:13:03.661 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 9m ago 6h 1074M 2384M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:13:03.661 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 9m ago 6h 1061M 2384M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:13:03.661 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 9m ago 6h 808M 2384M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:13:03.661 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 9m ago 6h 1039M 2384M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:13:03.662 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 7m ago 6h 892M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:13:03.662 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 7m ago 6h 1141M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:13:03.662 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 7m ago 6h 996M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:13:03.662 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 7m ago 6h 1209M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:13:03.663 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 7m ago 6h 163M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:13:04.017 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:13:04.017 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:13:04.017 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:13:04.018 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:13:04.018 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:13:04.018 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:13:04.018 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:13:04.018 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:13:04.019 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:13:04.019 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:13:04.019 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:13:04.019 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:13:04.020 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:13:04.020 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:13:04.020 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:13:04.020 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:13:04.020 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:13:04.021 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:13:04.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:13:04 smithi083 ceph-mon[79930]: pgmap v12143: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:04.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:13:04 smithi083 ceph-mon[79930]: from='client.58183 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:13:04.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:13:04 smithi083 ceph-mon[79930]: from='client.58189 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:13:04.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:13:04 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3746710523' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:13:04.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:13:04 smithi083 ceph-mon[77242]: pgmap v12143: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:04.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:13:04 smithi083 ceph-mon[77242]: from='client.58183 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:13:04.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:13:04 smithi083 ceph-mon[77242]: from='client.58189 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:13:04.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:13:04 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3746710523' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:13:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:13:04 smithi153 ceph-mon[70044]: pgmap v12143: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:13:04 smithi153 ceph-mon[70044]: from='client.58183 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:13:04.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:13:04 smithi153 ceph-mon[70044]: from='client.58189 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:13:04.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:13:04 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3746710523' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:13:05.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:13:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:13:05.035Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:13:06.019 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:13:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:13:05.757Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:13:06.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:13:06 smithi153 ceph-mon[70044]: pgmap v12144: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:06.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:13:06 smithi083 ceph-mon[79930]: pgmap v12144: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:06.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:13:06 smithi083 ceph-mon[77242]: pgmap v12144: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:08.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:13:08 smithi083 ceph-mon[79930]: pgmap v12145: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:08.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:13:08 smithi083 ceph-mon[77242]: pgmap v12145: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:13:08 smithi153 ceph-mon[70044]: pgmap v12145: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:10.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:13:10 smithi083 ceph-mon[79930]: pgmap v12146: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:10.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:13:10 smithi083 ceph-mon[77242]: pgmap v12146: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:13:10 smithi153 ceph-mon[70044]: pgmap v12146: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:13:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:13:11] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T12:13:11.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:13:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:13:11.660Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:13:12.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:13:12 smithi083 ceph-mon[79930]: pgmap v12147: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:12.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:13:12 smithi083 ceph-mon[77242]: pgmap v12147: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:13:12 smithi153 ceph-mon[70044]: pgmap v12147: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:12.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:13:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:13:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:13:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:13:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:13:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:13:14.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:13:14 smithi083 ceph-mon[79930]: pgmap v12148: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:14.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:13:14 smithi083 ceph-mon[77242]: pgmap v12148: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:14.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:13:14 smithi153 ceph-mon[70044]: pgmap v12148: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:15.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:13:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:13:15.037Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:13:16.016 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:13:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:13:15.757Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:13:16.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:13:16 smithi153 ceph-mon[70044]: pgmap v12149: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:16.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:13:16 smithi083 ceph-mon[79930]: pgmap v12149: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:16.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:13:16 smithi083 ceph-mon[77242]: pgmap v12149: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:18.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:13:18 smithi083 ceph-mon[77242]: pgmap v12150: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:18.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:13:18 smithi083 ceph-mon[79930]: pgmap v12150: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:13:18 smithi153 ceph-mon[70044]: pgmap v12150: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:20.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:13:20 smithi083 ceph-mon[77242]: pgmap v12151: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:20.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:13:20 smithi083 ceph-mon[79930]: pgmap v12151: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:13:20 smithi153 ceph-mon[70044]: pgmap v12151: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:13:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:13:21] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T12:13:21.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:13:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:13:21.659Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:13:22.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:13:22 smithi083 ceph-mon[77242]: pgmap v12152: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:22.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:13:22 smithi083 ceph-mon[79930]: pgmap v12152: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:13:22 smithi153 ceph-mon[70044]: pgmap v12152: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:22.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:13:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:13:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:13:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:13:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:13:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:13:24.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:13:24 smithi083 ceph-mon[79930]: pgmap v12153: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:24.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:13:24 smithi083 ceph-mon[77242]: pgmap v12153: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:13:24 smithi153 ceph-mon[70044]: pgmap v12153: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:25.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:13:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:13:25.038Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:13:26.022 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:13:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:13:25.757Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:13:26.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:13:26 smithi153 ceph-mon[70044]: pgmap v12154: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:26.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:13:26 smithi083 ceph-mon[79930]: pgmap v12154: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:26.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:13:26 smithi083 ceph-mon[77242]: pgmap v12154: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:28.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:13:28 smithi083 ceph-mon[79930]: pgmap v12155: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:28.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:13:28 smithi083 ceph-mon[77242]: pgmap v12155: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:13:28 smithi153 ceph-mon[70044]: pgmap v12155: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:30.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:13:30 smithi083 ceph-mon[77242]: pgmap v12156: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:30.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:13:30 smithi083 ceph-mon[79930]: pgmap v12156: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:13:30 smithi153 ceph-mon[70044]: pgmap v12156: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:13:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:13:31] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T12:13:31.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:13:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:13:31.660Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:13:32.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:13:32 smithi083 ceph-mon[77242]: pgmap v12157: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:32.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:13:32 smithi083 ceph-mon[79930]: pgmap v12157: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:13:32 smithi153 ceph-mon[70044]: pgmap v12157: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:32.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:13:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:13:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:13:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:13:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:13:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:13:34.343 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:13:34.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:13:34 smithi083 ceph-mon[77242]: pgmap v12158: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:34.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:13:34 smithi083 ceph-mon[79930]: pgmap v12158: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:13:34 smithi153 ceph-mon[70044]: pgmap v12158: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:34.633 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:13:34.633 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 9m ago 6h 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:13:34.633 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 7m ago 6h 63.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:13:34.633 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 9m ago 6h 25.8M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:13:34.634 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 7m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:13:34.634 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 7m ago 6h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:13:34.634 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 9m ago 6h 847M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:13:34.634 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 9m ago 6h 1486M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:13:34.635 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 7m ago 6h 641M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:13:34.635 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 9m ago 6h 857M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:13:34.635 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 9m ago 6h 26.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:13:34.635 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 7m ago 6h 26.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:13:34.636 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 9m ago 6h 1074M 2384M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:13:34.636 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 9m ago 6h 1061M 2384M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:13:34.636 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 9m ago 6h 808M 2384M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:13:34.636 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 9m ago 6h 1039M 2384M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:13:34.637 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 7m ago 6h 892M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:13:34.637 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 7m ago 6h 1141M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:13:34.637 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 7m ago 6h 996M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:13:34.637 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 7m ago 6h 1209M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:13:34.637 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 7m ago 6h 163M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:13:34.992 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:13:34.993 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:13:34.993 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:13:34.993 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:13:34.993 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:13:34.994 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:13:34.994 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:13:34.994 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:13:34.994 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:13:34.995 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:13:34.995 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:13:34.995 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:13:34.995 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:13:34.996 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:13:34.996 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:13:34.996 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:13:34.996 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:13:34.996 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:13:35.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:13:35 smithi083 ceph-mon[77242]: from='client.48270 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:13:35.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:13:35 smithi083 ceph-mon[77242]: from='client.58207 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:13:35.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:13:35 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3195141302' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:13:35.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:13:35 smithi083 ceph-mon[79930]: from='client.48270 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:13:35.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:13:35 smithi083 ceph-mon[79930]: from='client.58207 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:13:35.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:13:35 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3195141302' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:13:35.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:13:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:13:35.037Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:13:35.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:13:35 smithi153 ceph-mon[70044]: from='client.48270 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:13:35.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:13:35 smithi153 ceph-mon[70044]: from='client.58207 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:13:35.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:13:35 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3195141302' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:13:36.145 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:13:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:13:35.757Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:13:36.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:13:36 smithi083 ceph-mon[79930]: pgmap v12159: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:36.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:13:36 smithi083 ceph-mon[77242]: pgmap v12159: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:13:36 smithi153 ceph-mon[70044]: pgmap v12159: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:38.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:13:38 smithi083 ceph-mon[77242]: pgmap v12160: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:38.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:13:38 smithi083 ceph-mon[79930]: pgmap v12160: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:13:38 smithi153 ceph-mon[70044]: pgmap v12160: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:13:40 smithi083 ceph-mon[79930]: pgmap v12161: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:13:40 smithi083 ceph-mon[77242]: pgmap v12161: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:13:40 smithi153 ceph-mon[70044]: pgmap v12161: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:41.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:13:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:13:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:13:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:13:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:13:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:13:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:13:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:13:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:13:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:13:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:13:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:13:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:13:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:13:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:13:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:13:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:13:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:13:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:13:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:13:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:13:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:13:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:13:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:13:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:13:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:13:41] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T12:13:41.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:13:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:13:41.660Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:13:42.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:13:42 smithi083 ceph-mon[77242]: pgmap v12162: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:42.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:13:42 smithi083 ceph-mon[79930]: pgmap v12162: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:13:42 smithi153 ceph-mon[70044]: pgmap v12162: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:42.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:13:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:13:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:13:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:13:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:13:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:13:44.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:13:44 smithi083 ceph-mon[79930]: pgmap v12163: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:44.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:13:44 smithi083 ceph-mon[77242]: pgmap v12163: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:44.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:13:44 smithi153 ceph-mon[70044]: pgmap v12163: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:45.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:13:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:13:45.038Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:13:46.028 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:13:46 smithi083 ceph-mon[77242]: pgmap v12164: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:46.029 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:13:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:13:45.757Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:13:46.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:13:46 smithi153 ceph-mon[70044]: pgmap v12164: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:46.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:13:46 smithi083 ceph-mon[79930]: pgmap v12164: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:48.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:13:48 smithi083 ceph-mon[79930]: pgmap v12165: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:48.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:13:48 smithi083 ceph-mon[77242]: pgmap v12165: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:48.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:13:48 smithi153 ceph-mon[70044]: pgmap v12165: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:50.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:13:50 smithi083 ceph-mon[79930]: pgmap v12166: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:50.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:13:50 smithi083 ceph-mon[77242]: pgmap v12166: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:13:50 smithi153 ceph-mon[70044]: pgmap v12166: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:13:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:13:51] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T12:13:51.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:13:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:13:51.660Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:13:52.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:13:52 smithi083 ceph-mon[77242]: pgmap v12167: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:52.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:13:52 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:13:52.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:13:52 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:13:52.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:13:52 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:13:52.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:13:52 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:13:52.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:13:52 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:13:52.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:13:52 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:13:52.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:13:52 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:13:52.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:13:52 smithi083 ceph-mon[79930]: pgmap v12167: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:52.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:13:52 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:13:52.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:13:52 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:13:52.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:13:52 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:13:52.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:13:52 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:13:52.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:13:52 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:13:52.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:13:52 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:13:52.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:13:52 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:13:52.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:13:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:13:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:13:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:13:52 smithi153 ceph-mon[70044]: pgmap v12167: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:13:52 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:13:52.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:13:52 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:13:52.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:13:52 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:13:52.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:13:52 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:13:52.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:13:52 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:13:52.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:13:52 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:13:52.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:13:52 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:13:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:13:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:13:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:13:54.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:13:54 smithi083 ceph-mon[79930]: pgmap v12168: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:54.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:13:54 smithi083 ceph-mon[77242]: pgmap v12168: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:54.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:13:54 smithi153 ceph-mon[70044]: pgmap v12168: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:13:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:13:55.038Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:13:56.030 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:13:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:13:55.758Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:13:56.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:13:56 smithi153 ceph-mon[70044]: pgmap v12169: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:56.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:13:56 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:13:56.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:13:56 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:13:56.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:13:56 smithi083 ceph-mon[79930]: pgmap v12169: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:56.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:13:56 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:13:56.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:13:56 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:13:56.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:13:56 smithi083 ceph-mon[77242]: pgmap v12169: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:56.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:13:56 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:13:56.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:13:56 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:13:57.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:13:57 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:13:57.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:13:57 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:13:57.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:13:57 smithi083 ceph-mon[79930]: pgmap v12170: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:57.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:13:57 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:13:57.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:13:57 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:13:57.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:13:57 smithi083 ceph-mon[77242]: pgmap v12170: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:13:58.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:13:57 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:13:58.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:13:57 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:13:58.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:13:57 smithi153 ceph-mon[70044]: pgmap v12170: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:00.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:14:00 smithi083 ceph-mon[79930]: pgmap v12171: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:00.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:14:00 smithi083 ceph-mon[79930]: Detected new or changed devices on smithi083 2022-09-29T12:14:00.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:14:00 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:14:00.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:14:00 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:14:00.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:14:00 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:14:00.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:14:00 smithi083 ceph-mon[77242]: pgmap v12171: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:00.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:14:00 smithi083 ceph-mon[77242]: Detected new or changed devices on smithi083 2022-09-29T12:14:00.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:14:00 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:14:00.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:14:00 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:14:00.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:14:00 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:14:00.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:14:00 smithi153 ceph-mon[70044]: pgmap v12171: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:00.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:14:00 smithi153 ceph-mon[70044]: Detected new or changed devices on smithi083 2022-09-29T12:14:00.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:14:00 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:14:00.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:14:00 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:14:00.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:14:00 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:14:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:14:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:14:01] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T12:14:01.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:14:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:14:01.661Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:14:02.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:14:02 smithi153 ceph-mon[70044]: pgmap v12172: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:14:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:14:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:14:02.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:14:02 smithi083 ceph-mon[79930]: pgmap v12172: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:02.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:14:02 smithi083 ceph-mon[77242]: pgmap v12172: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:14:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:14:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:14:04.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:14:04 smithi153 ceph-mon[70044]: pgmap v12173: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:04.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:14:04 smithi083 ceph-mon[77242]: pgmap v12173: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:04.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:14:04 smithi083 ceph-mon[79930]: pgmap v12173: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:05.322 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:14:05.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:14:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:14:05.040Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:14:05.612 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:14:05.612 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 9s ago 6h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:14:05.613 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 8m ago 6h 63.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:14:05.613 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 9s ago 6h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:14:05.613 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 8m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:14:05.613 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 8m ago 6h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:14:05.614 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 9s ago 6h 856M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:14:05.614 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 9s ago 6h 1523M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:14:05.614 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 8m ago 6h 641M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:14:05.614 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 9s ago 6h 879M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:14:05.615 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 9s ago 6h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:14:05.615 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 8m ago 6h 26.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:14:05.615 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 9s ago 6h 1099M 2384M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:14:05.615 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 9s ago 6h 1087M 2384M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:14:05.616 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 9s ago 6h 827M 2384M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:14:05.616 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 9s ago 6h 1063M 2384M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:14:05.616 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 8m ago 6h 892M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:14:05.616 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 8m ago 6h 1141M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:14:05.616 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 8m ago 6h 996M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:14:05.617 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 8m ago 6h 1209M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:14:05.617 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 8m ago 6h 163M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:14:05.972 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:14:05.973 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:14:05.973 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:14:05.973 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:14:05.974 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:14:05.974 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:14:05.974 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:14:05.974 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:14:05.975 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:14:05.975 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:14:05.975 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:14:05.975 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:14:05.976 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:14:05.976 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:14:05.976 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:14:05.976 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:14:05.977 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:14:05.977 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:14:06.033 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:14:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:14:05.758Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:14:06.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:14:06 smithi153 ceph-mon[70044]: pgmap v12174: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:06.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:14:06 smithi153 ceph-mon[70044]: from='client.58219 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:14:06.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:14:06 smithi153 ceph-mon[70044]: from='client.48294 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:14:06.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:14:06 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2613552048' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:14:06.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:14:06 smithi083 ceph-mon[77242]: pgmap v12174: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:06.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:14:06 smithi083 ceph-mon[77242]: from='client.58219 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:14:06.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:14:06 smithi083 ceph-mon[77242]: from='client.48294 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:14:06.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:14:06 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2613552048' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:14:06.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:14:06 smithi083 ceph-mon[79930]: pgmap v12174: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:06.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:14:06 smithi083 ceph-mon[79930]: from='client.58219 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:14:06.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:14:06 smithi083 ceph-mon[79930]: from='client.48294 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:14:06.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:14:06 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2613552048' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:14:08.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:14:08 smithi083 ceph-mon[77242]: pgmap v12175: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:08.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:14:08 smithi083 ceph-mon[79930]: pgmap v12175: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:14:08 smithi153 ceph-mon[70044]: pgmap v12175: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:10.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:14:10 smithi083 ceph-mon[77242]: pgmap v12176: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:10.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:14:10 smithi083 ceph-mon[79930]: pgmap v12176: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:14:10 smithi153 ceph-mon[70044]: pgmap v12176: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:14:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:14:11] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T12:14:11.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:14:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:14:11.662Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:14:12.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:14:12 smithi083 ceph-mon[77242]: pgmap v12177: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:12.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:14:12 smithi083 ceph-mon[79930]: pgmap v12177: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:14:12 smithi153 ceph-mon[70044]: pgmap v12177: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:12.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:14:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:14:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:14:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:14:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:14:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:14:14.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:14:14 smithi083 ceph-mon[77242]: pgmap v12178: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:14.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:14:14 smithi083 ceph-mon[79930]: pgmap v12178: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:14.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:14:14 smithi153 ceph-mon[70044]: pgmap v12178: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:15.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:14:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:14:15.039Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:14:16.034 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:14:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:14:15.759Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:14:16.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:14:16 smithi153 ceph-mon[70044]: pgmap v12179: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:16.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:14:16 smithi083 ceph-mon[77242]: pgmap v12179: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:16.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:14:16 smithi083 ceph-mon[79930]: pgmap v12179: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:18.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:14:18 smithi083 ceph-mon[79930]: pgmap v12180: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:18.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:14:18 smithi083 ceph-mon[77242]: pgmap v12180: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:14:18 smithi153 ceph-mon[70044]: pgmap v12180: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:20.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:14:20 smithi083 ceph-mon[79930]: pgmap v12181: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:20.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:14:20 smithi083 ceph-mon[77242]: pgmap v12181: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:14:20 smithi153 ceph-mon[70044]: pgmap v12181: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:14:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:14:21] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T12:14:21.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:14:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:14:21.663Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:14:22.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:14:22 smithi083 ceph-mon[79930]: pgmap v12182: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:22.492 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:14:22 smithi083 ceph-mon[77242]: pgmap v12182: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:14:22 smithi153 ceph-mon[70044]: pgmap v12182: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:22.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:14:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:14:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:14:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:14:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:14:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:14:24.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:14:24 smithi083 ceph-mon[79930]: pgmap v12183: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:24.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:14:24 smithi083 ceph-mon[77242]: pgmap v12183: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:14:24 smithi153 ceph-mon[70044]: pgmap v12183: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:25.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:14:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:14:25.040Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:14:26.037 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:14:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:14:25.759Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:14:26.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:14:26 smithi153 ceph-mon[70044]: pgmap v12184: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:26.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:14:26 smithi083 ceph-mon[79930]: pgmap v12184: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:26.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:14:26 smithi083 ceph-mon[77242]: pgmap v12184: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:28.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:14:28 smithi083 ceph-mon[77242]: pgmap v12185: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:28.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:14:28 smithi083 ceph-mon[79930]: pgmap v12185: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:14:28 smithi153 ceph-mon[70044]: pgmap v12185: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:30.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:14:30 smithi083 ceph-mon[79930]: pgmap v12186: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:30.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:14:30 smithi083 ceph-mon[77242]: pgmap v12186: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:14:30 smithi153 ceph-mon[70044]: pgmap v12186: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:14:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:14:31] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T12:14:31.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:14:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:14:31.663Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:14:32.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:14:32 smithi083 ceph-mon[79930]: pgmap v12187: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:32.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:14:32 smithi083 ceph-mon[77242]: pgmap v12187: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:14:32 smithi153 ceph-mon[70044]: pgmap v12187: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:32.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:14:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:14:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:14:33.312 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:14:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:14:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:14:34.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:14:34 smithi083 ceph-mon[79930]: pgmap v12188: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:34.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:14:34 smithi083 ceph-mon[77242]: pgmap v12188: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:14:34 smithi153 ceph-mon[70044]: pgmap v12188: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:35.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:14:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:14:35.040Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:14:36.039 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:14:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:14:35.760Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:14:36.301 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:14:36.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:14:36 smithi153 ceph-mon[70044]: pgmap v12189: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:36.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:14:36 smithi083 ceph-mon[79930]: pgmap v12189: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:36.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:14:36 smithi083 ceph-mon[77242]: pgmap v12189: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:36.590 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:14:36.590 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 40s ago 6h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:14:36.591 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 8m ago 6h 63.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:14:36.591 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 40s ago 6h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:14:36.591 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 8m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:14:36.592 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 8m ago 6h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:14:36.592 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 40s ago 6h 856M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:14:36.592 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 40s ago 6h 1523M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:14:36.592 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 8m ago 6h 641M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:14:36.593 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 40s ago 6h 879M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:14:36.593 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 40s ago 6h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:14:36.593 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 8m ago 6h 26.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:14:36.594 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 40s ago 6h 1099M 2384M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:14:36.594 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 40s ago 6h 1087M 2384M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:14:36.594 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 40s ago 6h 827M 2384M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:14:36.595 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 40s ago 6h 1063M 2384M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:14:36.595 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 8m ago 6h 892M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:14:36.595 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 8m ago 6h 1141M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:14:36.595 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 8m ago 6h 996M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:14:36.596 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 8m ago 6h 1209M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:14:36.596 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 8m ago 6h 163M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:14:36.947 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:14:36.947 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:14:36.947 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:14:36.948 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:14:36.948 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:14:36.948 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:14:36.948 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:14:36.948 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:14:36.949 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:14:36.949 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:14:36.949 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:14:36.949 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:14:36.950 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:14:36.950 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:14:36.950 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:14:36.950 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:14:36.951 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:14:36.951 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:14:37.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:14:37 smithi153 ceph-mon[70044]: from='client.58237 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:14:37.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:14:37 smithi153 ceph-mon[70044]: from='client.48312 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:14:37.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:14:37 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/721734887' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:14:37.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:14:37 smithi083 ceph-mon[77242]: from='client.58237 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:14:37.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:14:37 smithi083 ceph-mon[77242]: from='client.48312 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:14:37.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:14:37 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/721734887' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:14:37.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:14:37 smithi083 ceph-mon[79930]: from='client.58237 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:14:37.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:14:37 smithi083 ceph-mon[79930]: from='client.48312 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:14:37.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:14:37 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/721734887' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:14:38.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:14:38 smithi153 ceph-mon[70044]: pgmap v12190: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:38.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:14:38 smithi083 ceph-mon[77242]: pgmap v12190: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:38.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:14:38 smithi083 ceph-mon[79930]: pgmap v12190: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:14:40 smithi083 ceph-mon[79930]: pgmap v12191: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:40.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:14:40 smithi083 ceph-mon[77242]: pgmap v12191: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:14:40 smithi153 ceph-mon[70044]: pgmap v12191: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:41.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:14:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:14:41.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:14:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:14:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:14:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:14:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:14:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:14:41.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:14:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:14:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:14:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:14:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:14:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:14:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:14:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:14:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:14:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:14:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:14:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:14:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:14:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:14:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:14:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:14:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:14:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:14:41] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T12:14:41.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:14:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:14:41.664Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:14:42.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:14:42 smithi083 ceph-mon[77242]: pgmap v12192: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:42.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:14:42 smithi083 ceph-mon[79930]: pgmap v12192: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:42.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:14:42 smithi153 ceph-mon[70044]: pgmap v12192: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:42.560 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:14:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:14:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:14:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:14:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:14:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:14:44.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:14:44 smithi083 ceph-mon[79930]: pgmap v12193: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:44.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:14:44 smithi083 ceph-mon[77242]: pgmap v12193: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:44.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:14:44 smithi153 ceph-mon[70044]: pgmap v12193: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:45.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:14:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:14:45.040Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:14:46.040 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:14:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:14:45.761Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:14:46.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:14:46 smithi153 ceph-mon[70044]: pgmap v12194: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:46.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:14:46 smithi083 ceph-mon[79930]: pgmap v12194: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:46.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:14:46 smithi083 ceph-mon[77242]: pgmap v12194: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:48.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:14:48 smithi083 ceph-mon[79930]: pgmap v12195: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:48.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:14:48 smithi083 ceph-mon[77242]: pgmap v12195: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:48.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:14:48 smithi153 ceph-mon[70044]: pgmap v12195: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:50.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:14:50 smithi083 ceph-mon[79930]: pgmap v12196: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:50.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:14:50 smithi083 ceph-mon[77242]: pgmap v12196: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:14:50 smithi153 ceph-mon[70044]: pgmap v12196: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:14:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:14:51] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T12:14:51.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:14:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:14:51.665Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:14:52.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:14:52 smithi083 ceph-mon[77242]: pgmap v12197: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:52.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:14:52 smithi083 ceph-mon[79930]: pgmap v12197: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:14:52 smithi153 ceph-mon[70044]: pgmap v12197: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:52.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:14:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:14:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:14:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:14:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:14:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:14:54.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:14:54 smithi083 ceph-mon[79930]: pgmap v12198: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:54.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:14:54 smithi083 ceph-mon[77242]: pgmap v12198: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:54.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:14:54 smithi153 ceph-mon[70044]: pgmap v12198: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:14:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:14:55.041Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:14:56.044 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:14:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:14:55.761Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:14:56.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:14:56 smithi153 ceph-mon[70044]: pgmap v12199: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:56.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:14:56 smithi083 ceph-mon[79930]: pgmap v12199: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:56.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:14:56 smithi083 ceph-mon[77242]: pgmap v12199: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:58.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:14:58 smithi083 ceph-mon[79930]: pgmap v12200: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:58.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:14:58 smithi083 ceph-mon[77242]: pgmap v12200: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:14:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:14:58 smithi153 ceph-mon[70044]: pgmap v12200: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:00.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:15:00 smithi083 ceph-mon[79930]: pgmap v12201: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:00.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:15:00 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:15:00.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:15:00 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:15:00.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:15:00 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:15:00.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:15:00 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:15:00.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:15:00 smithi083 ceph-mon[77242]: pgmap v12201: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:00.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:15:00 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:15:00.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:15:00 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:15:00.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:15:00 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:15:00.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:15:00 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:15:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:15:00 smithi153 ceph-mon[70044]: pgmap v12201: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:15:00 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:15:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:15:00 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:15:00.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:15:00 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:15:00.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:15:00 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:15:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:15:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:15:01] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T12:15:01.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:15:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:15:01.665Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:15:02.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:15:02 smithi083 ceph-mon[79930]: pgmap v12202: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:02.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:15:02 smithi083 ceph-mon[77242]: pgmap v12202: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:15:02 smithi153 ceph-mon[70044]: pgmap v12202: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:02.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:15:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:15:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:15:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:15:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:15:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:15:04.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:15:04 smithi083 ceph-mon[79930]: pgmap v12203: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:04.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:15:04 smithi083 ceph-mon[77242]: pgmap v12203: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:15:04 smithi153 ceph-mon[70044]: pgmap v12203: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:05.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:15:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:15:05.042Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:15:06.044 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:15:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:15:05.762Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:15:06.044 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:15:06 smithi083 ceph-mon[77242]: pgmap v12204: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:06.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:15:06 smithi153 ceph-mon[70044]: pgmap v12204: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:06.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:15:06 smithi083 ceph-mon[79930]: pgmap v12204: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:07.280 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:15:07.566 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:15:07.567 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 71s ago 6h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:15:07.567 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 9m ago 6h 63.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:15:07.567 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 71s ago 6h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:15:07.568 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 9m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:15:07.568 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 9m ago 6h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:15:07.568 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 71s ago 6h 856M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:15:07.568 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 71s ago 6h 1523M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:15:07.568 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 9m ago 6h 641M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:15:07.569 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 71s ago 6h 879M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:15:07.569 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 71s ago 6h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:15:07.569 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 9m ago 6h 26.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:15:07.569 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 71s ago 6h 1099M 2384M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:15:07.570 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 71s ago 6h 1087M 2384M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:15:07.570 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 71s ago 6h 827M 2384M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:15:07.570 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 71s ago 6h 1063M 2384M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:15:07.570 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 9m ago 6h 892M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:15:07.571 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 9m ago 6h 1141M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:15:07.571 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 9m ago 6h 996M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:15:07.571 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 9m ago 6h 1209M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:15:07.571 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 9m ago 6h 163M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:15:07.922 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:15:07.922 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:15:07.923 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:15:07.923 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:15:07.923 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:15:07.923 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:15:07.924 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:15:07.924 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:15:07.924 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:15:07.924 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:15:07.924 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:15:07.925 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:15:07.925 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:15:07.925 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:15:07.925 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:15:07.926 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:15:07.926 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:15:07.926 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:15:08.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:15:08 smithi083 ceph-mon[79930]: pgmap v12205: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:08.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:15:08 smithi083 ceph-mon[79930]: from='client.58255 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:15:08.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:15:08 smithi083 ceph-mon[79930]: from='client.58261 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:15:08.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:15:08 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2590590845' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:15:08.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:15:08 smithi083 ceph-mon[77242]: pgmap v12205: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:08.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:15:08 smithi083 ceph-mon[77242]: from='client.58255 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:15:08.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:15:08 smithi083 ceph-mon[77242]: from='client.58261 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:15:08.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:15:08 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2590590845' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:15:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:15:08 smithi153 ceph-mon[70044]: pgmap v12205: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:15:08 smithi153 ceph-mon[70044]: from='client.58255 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:15:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:15:08 smithi153 ceph-mon[70044]: from='client.58261 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:15:08.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:15:08 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2590590845' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:15:10.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:15:10 smithi083 ceph-mon[79930]: pgmap v12206: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:10.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:15:10 smithi083 ceph-mon[77242]: pgmap v12206: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:15:10 smithi153 ceph-mon[70044]: pgmap v12206: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:11.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:15:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:15:11.666Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:15:11.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:15:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:15:11] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T12:15:12.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:15:12 smithi083 ceph-mon[79930]: pgmap v12207: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:12.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:15:12 smithi083 ceph-mon[77242]: pgmap v12207: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:15:12 smithi153 ceph-mon[70044]: pgmap v12207: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:12.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:15:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:15:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:15:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:15:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:15:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:15:14.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:15:14 smithi083 ceph-mon[79930]: pgmap v12208: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:14.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:15:14 smithi083 ceph-mon[77242]: pgmap v12208: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:14.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:15:14 smithi153 ceph-mon[70044]: pgmap v12208: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:15.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:15:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:15:15.043Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:15:16.046 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:15:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:15:15.762Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:15:16.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:15:16 smithi153 ceph-mon[70044]: pgmap v12209: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:16.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:15:16 smithi083 ceph-mon[79930]: pgmap v12209: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:16.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:15:16 smithi083 ceph-mon[77242]: pgmap v12209: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:18.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:15:18 smithi083 ceph-mon[79930]: pgmap v12210: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:18.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:15:18 smithi083 ceph-mon[77242]: pgmap v12210: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:15:18 smithi153 ceph-mon[70044]: pgmap v12210: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:20.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:15:20 smithi083 ceph-mon[79930]: pgmap v12211: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:20.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:15:20 smithi083 ceph-mon[77242]: pgmap v12211: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:15:20 smithi153 ceph-mon[70044]: pgmap v12211: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:15:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:15:21] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T12:15:21.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:15:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:15:21.666Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:15:22.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:15:22 smithi083 ceph-mon[79930]: pgmap v12212: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:22.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:15:22 smithi083 ceph-mon[77242]: pgmap v12212: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:15:22 smithi153 ceph-mon[70044]: pgmap v12212: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:22.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:15:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:15:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:15:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:15:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:15:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:15:24.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:15:24 smithi083 ceph-mon[79930]: pgmap v12213: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:24.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:15:24 smithi083 ceph-mon[77242]: pgmap v12213: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:15:24 smithi153 ceph-mon[70044]: pgmap v12213: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:25.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:15:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:15:25.044Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:15:26.049 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:15:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:15:25.763Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:15:26.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:15:26 smithi153 ceph-mon[70044]: pgmap v12214: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:26.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:15:26 smithi083 ceph-mon[77242]: pgmap v12214: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:26.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:15:26 smithi083 ceph-mon[79930]: pgmap v12214: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:28.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:15:28 smithi083 ceph-mon[77242]: pgmap v12215: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:28.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:15:28 smithi083 ceph-mon[79930]: pgmap v12215: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:15:28 smithi153 ceph-mon[70044]: pgmap v12215: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:30.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:15:30 smithi083 ceph-mon[79930]: pgmap v12216: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:30.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:15:30 smithi083 ceph-mon[77242]: pgmap v12216: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:15:30 smithi153 ceph-mon[70044]: pgmap v12216: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:15:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:15:31] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T12:15:31.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:15:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:15:31.667Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:15:32.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:15:32 smithi083 ceph-mon[79930]: pgmap v12217: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:32.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:15:32 smithi083 ceph-mon[77242]: pgmap v12217: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:15:32 smithi153 ceph-mon[70044]: pgmap v12217: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:32.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:15:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:15:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:15:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:15:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:15:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:15:34.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:15:34 smithi083 ceph-mon[79930]: pgmap v12218: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:34.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:15:34 smithi083 ceph-mon[77242]: pgmap v12218: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:15:34 smithi153 ceph-mon[70044]: pgmap v12218: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:35.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:15:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:15:35.045Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:15:36.052 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:15:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:15:35.764Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:15:36.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:15:36 smithi153 ceph-mon[70044]: pgmap v12219: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:36.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:15:36 smithi083 ceph-mon[79930]: pgmap v12219: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:36.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:15:36 smithi083 ceph-mon[77242]: pgmap v12219: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:38.251 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:15:38.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:15:38 smithi083 ceph-mon[79930]: pgmap v12220: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:38.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:15:38 smithi083 ceph-mon[77242]: pgmap v12220: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:38.540 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:15:38.540 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 102s ago 6h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:15:38.540 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 9m ago 6h 63.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:15:38.541 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 102s ago 6h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:15:38.541 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 9m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:15:38.541 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 9m ago 6h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:15:38.541 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 102s ago 6h 856M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:15:38.542 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 102s ago 6h 1523M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:15:38.542 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 9m ago 6h 641M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:15:38.542 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 102s ago 6h 879M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:15:38.542 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 102s ago 6h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:15:38.543 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 9m ago 6h 26.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:15:38.543 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 102s ago 6h 1099M 2384M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:15:38.543 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 102s ago 6h 1087M 2384M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:15:38.543 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 102s ago 6h 827M 2384M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:15:38.544 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 102s ago 6h 1063M 2384M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:15:38.544 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 9m ago 6h 892M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:15:38.544 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 9m ago 6h 1141M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:15:38.544 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 9m ago 6h 996M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:15:38.545 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 9m ago 6h 1209M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:15:38.545 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 9m ago 6h 163M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:15:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:15:38 smithi153 ceph-mon[70044]: pgmap v12220: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:38.897 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:15:38.897 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:15:38.897 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:15:38.897 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:15:38.898 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:15:38.898 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:15:38.898 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:15:38.899 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:15:38.899 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:15:38.899 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:15:38.899 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:15:38.899 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:15:38.900 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:15:38.900 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:15:38.900 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:15:38.900 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:15:38.900 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:15:38.901 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:15:39.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:15:39 smithi083 ceph-mon[77242]: from='client.58273 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:15:39.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:15:39 smithi083 ceph-mon[77242]: from='client.48348 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:15:39.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:15:39 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3215785561' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:15:39.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:15:39 smithi083 ceph-mon[79930]: from='client.58273 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:15:39.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:15:39 smithi083 ceph-mon[79930]: from='client.48348 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:15:39.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:15:39 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3215785561' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:15:39.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:15:39 smithi153 ceph-mon[70044]: from='client.58273 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:15:39.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:15:39 smithi153 ceph-mon[70044]: from='client.48348 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:15:39.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:15:39 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3215785561' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:15:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:15:40 smithi083 ceph-mon[79930]: pgmap v12221: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:15:40 smithi083 ceph-mon[77242]: pgmap v12221: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:15:40 smithi153 ceph-mon[70044]: pgmap v12221: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:41.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:15:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:15:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:15:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:15:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:15:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:15:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:15:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:15:41.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:15:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:15:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:15:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:15:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:15:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:15:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:15:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:15:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:15:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:15:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:15:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:15:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:15:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:15:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:15:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:15:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:15:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:15:41] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T12:15:41.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:15:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:15:41.667Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:15:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:15:42 smithi083 ceph-mon[79930]: pgmap v12222: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:42.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:15:42 smithi083 ceph-mon[77242]: pgmap v12222: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:15:42 smithi153 ceph-mon[70044]: pgmap v12222: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:42.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:15:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:15:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:15:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:15:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:15:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:15:44.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:15:44 smithi083 ceph-mon[79930]: pgmap v12223: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:44.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:15:44 smithi083 ceph-mon[77242]: pgmap v12223: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:44.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:15:44 smithi153 ceph-mon[70044]: pgmap v12223: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:45.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:15:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:15:45.046Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:15:46.052 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:15:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:15:45.764Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:15:46.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:15:46 smithi153 ceph-mon[70044]: pgmap v12224: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:46.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:15:46 smithi083 ceph-mon[77242]: pgmap v12224: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:46.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:15:46 smithi083 ceph-mon[79930]: pgmap v12224: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:48.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:15:48 smithi083 ceph-mon[79930]: pgmap v12225: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:48.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:15:48 smithi083 ceph-mon[77242]: pgmap v12225: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:48.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:15:48 smithi153 ceph-mon[70044]: pgmap v12225: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:50.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:15:50 smithi083 ceph-mon[79930]: pgmap v12226: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:50.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:15:50 smithi083 ceph-mon[77242]: pgmap v12226: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:15:50 smithi153 ceph-mon[70044]: pgmap v12226: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:15:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:15:51] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T12:15:51.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:15:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:15:51.668Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:15:52.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:15:52 smithi083 ceph-mon[79930]: pgmap v12227: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:52.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:15:52 smithi083 ceph-mon[77242]: pgmap v12227: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:15:52 smithi153 ceph-mon[70044]: pgmap v12227: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:52.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:15:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:15:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:15:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:15:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:15:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:15:54.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:15:54 smithi083 ceph-mon[79930]: pgmap v12228: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:54.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:15:54 smithi083 ceph-mon[77242]: pgmap v12228: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:54.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:15:54 smithi153 ceph-mon[70044]: pgmap v12228: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:15:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:15:55.045Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:15:56.056 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:15:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:15:55.765Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:15:56.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:15:56 smithi153 ceph-mon[70044]: pgmap v12229: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:56.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:15:56 smithi083 ceph-mon[77242]: pgmap v12229: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:56.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:15:56 smithi083 ceph-mon[79930]: pgmap v12229: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:58.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:15:58 smithi083 ceph-mon[77242]: pgmap v12230: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:58.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:15:58 smithi083 ceph-mon[79930]: pgmap v12230: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:15:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:15:58 smithi153 ceph-mon[70044]: pgmap v12230: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:00.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:16:00 smithi083 ceph-mon[79930]: pgmap v12231: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:00.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:16:00 smithi083 ceph-mon[77242]: pgmap v12231: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:16:00 smithi153 ceph-mon[70044]: pgmap v12231: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:01.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:16:01 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:16:01.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:16:01 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:16:01.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:16:01 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:16:01.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:16:01 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:16:01.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:16:01 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:16:01.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:16:01 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:16:01.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:16:01 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:16:01.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:16:01 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:16:01.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:16:01 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:16:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:16:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:16:01] "GET /metrics HTTP/1.1" 200 225227 "" "Prometheus/2.33.4" 2022-09-29T12:16:01.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:16:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:16:01.669Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:16:02.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:16:02 smithi083 ceph-mon[79930]: pgmap v12232: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:02.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:16:02 smithi083 ceph-mon[77242]: pgmap v12232: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:16:02 smithi153 ceph-mon[70044]: pgmap v12232: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:02.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:16:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:16:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:16:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:16:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:16:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:16:04.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:16:04 smithi083 ceph-mon[79930]: pgmap v12233: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:04.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:16:04 smithi083 ceph-mon[77242]: pgmap v12233: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:16:04 smithi153 ceph-mon[70044]: pgmap v12233: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:05.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:16:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:16:05.045Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:16:06.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:16:05 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:16:06.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:16:05 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:16:06.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:16:05 smithi153 ceph-mon[70044]: pgmap v12234: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:06.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:16:05 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:16:06.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:16:05 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:16:06.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:16:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:16:05.766Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:16:06.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:16:05 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:16:06.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:16:05 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:16:06.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:16:05 smithi083 ceph-mon[79930]: pgmap v12234: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:06.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:16:05 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:16:06.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:16:05 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:16:06.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:16:05 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:16:06.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:16:05 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:16:06.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:16:05 smithi083 ceph-mon[77242]: pgmap v12234: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:06.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:16:05 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:16:06.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:16:05 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:16:08.252 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:16:08 smithi153 ceph-mon[70044]: pgmap v12235: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:08.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:16:08 smithi083 ceph-mon[77242]: pgmap v12235: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:08.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:16:08 smithi083 ceph-mon[79930]: pgmap v12235: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:09.222 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:16:09.513 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:16:09.513 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 2m ago 6h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:16:09.513 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 4s ago 6h 62.7M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:16:09.513 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 2m ago 6h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:16:09.514 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 4s ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:16:09.514 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 4s ago 6h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:16:09.514 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 2m ago 6h 856M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:16:09.515 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 2m ago 6h 1523M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:16:09.515 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 4s ago 6h 658M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:16:09.515 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 2m ago 6h 879M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:16:09.515 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 2m ago 6h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:16:09.515 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 4s ago 6h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:16:09.516 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 2m ago 6h 1099M 2384M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:16:09.516 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 2m ago 6h 1087M 2384M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:16:09.516 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 2m ago 6h 827M 2384M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:16:09.516 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 2m ago 6h 1063M 2384M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:16:09.517 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 4s ago 6h 914M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:16:09.517 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 4s ago 6h 1170M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:16:09.517 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 4s ago 6h 1045M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:16:09.517 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 4s ago 6h 1240M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:16:09.517 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 4s ago 6h 166M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:16:09.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:16:09 smithi083 ceph-mon[79930]: Detected new or changed devices on smithi153 2022-09-29T12:16:09.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:16:09 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:16:09.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:16:09 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:16:09.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:16:09 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:16:09.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:16:09 smithi083 ceph-mon[77242]: Detected new or changed devices on smithi153 2022-09-29T12:16:09.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:16:09 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:16:09.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:16:09 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:16:09.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:16:09 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:16:09.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:16:09 smithi153 ceph-mon[70044]: Detected new or changed devices on smithi153 2022-09-29T12:16:09.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:16:09 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:16:09.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:16:09 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:16:09.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:16:09 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:16:09.871 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:16:09.871 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:16:09.871 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:16:09.871 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:16:09.872 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:16:09.872 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:16:09.872 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:16:09.872 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:16:09.873 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:16:09.873 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:16:09.873 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:16:09.873 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:16:09.873 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:16:09.874 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:16:09.874 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:16:09.874 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:16:09.874 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:16:09.875 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:16:10.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:16:10 smithi083 ceph-mon[79930]: pgmap v12236: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:10.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:16:10 smithi083 ceph-mon[79930]: from='client.48360 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:16:10.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:16:10 smithi083 ceph-mon[79930]: from='client.58297 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:16:10.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:16:10 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1292038548' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:16:10.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:16:10 smithi083 ceph-mon[77242]: pgmap v12236: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:10.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:16:10 smithi083 ceph-mon[77242]: from='client.48360 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:16:10.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:16:10 smithi083 ceph-mon[77242]: from='client.58297 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:16:10.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:16:10 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1292038548' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:16:10.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:16:10 smithi153 ceph-mon[70044]: pgmap v12236: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:10.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:16:10 smithi153 ceph-mon[70044]: from='client.48360 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:16:10.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:16:10 smithi153 ceph-mon[70044]: from='client.58297 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:16:10.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:16:10 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1292038548' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:16:11.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:16:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:16:11.670Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:16:11.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:16:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:16:11] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T12:16:12.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:16:12 smithi083 ceph-mon[79930]: pgmap v12237: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:12.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:16:12 smithi083 ceph-mon[77242]: pgmap v12237: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:12.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:16:12 smithi153 ceph-mon[70044]: pgmap v12237: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:16:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:16:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:16:13.312 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:16:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:16:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:16:14.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:16:14 smithi083 ceph-mon[79930]: pgmap v12238: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:14.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:16:14 smithi083 ceph-mon[77242]: pgmap v12238: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:14.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:16:14 smithi153 ceph-mon[70044]: pgmap v12238: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:15.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:16:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:16:15.046Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:16:16.060 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:16:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:16:15.766Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:16:16.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:16:16 smithi083 ceph-mon[79930]: pgmap v12239: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:16.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:16:16 smithi083 ceph-mon[77242]: pgmap v12239: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:16:16 smithi153 ceph-mon[70044]: pgmap v12239: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:18.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:16:18 smithi083 ceph-mon[79930]: pgmap v12240: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:18.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:16:18 smithi083 ceph-mon[77242]: pgmap v12240: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:16:18 smithi153 ceph-mon[70044]: pgmap v12240: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:20.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:16:20 smithi083 ceph-mon[79930]: pgmap v12241: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:20.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:16:20 smithi083 ceph-mon[77242]: pgmap v12241: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:16:20 smithi153 ceph-mon[70044]: pgmap v12241: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:16:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:16:21] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T12:16:21.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:16:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:16:21.671Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:16:22.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:16:22 smithi083 ceph-mon[77242]: pgmap v12242: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:22.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:16:22 smithi083 ceph-mon[79930]: pgmap v12242: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:16:22 smithi153 ceph-mon[70044]: pgmap v12242: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:22.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:16:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:16:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:16:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:16:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:16:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:16:24.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:16:24 smithi083 ceph-mon[79930]: pgmap v12243: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:24.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:16:24 smithi083 ceph-mon[77242]: pgmap v12243: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:16:24 smithi153 ceph-mon[70044]: pgmap v12243: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:25.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:16:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:16:25.046Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:16:26.062 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:16:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:16:25.767Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:16:26.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:16:26 smithi083 ceph-mon[77242]: pgmap v12244: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:26.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:16:26 smithi083 ceph-mon[79930]: pgmap v12244: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:16:26 smithi153 ceph-mon[70044]: pgmap v12244: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:28.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:16:28 smithi083 ceph-mon[79930]: pgmap v12245: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:28.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:16:28 smithi083 ceph-mon[77242]: pgmap v12245: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:16:28 smithi153 ceph-mon[70044]: pgmap v12245: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:30.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:16:30 smithi083 ceph-mon[79930]: pgmap v12246: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:30.488 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:16:30 smithi083 ceph-mon[77242]: pgmap v12246: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:16:30 smithi153 ceph-mon[70044]: pgmap v12246: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:16:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:16:31] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T12:16:31.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:16:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:16:31.672Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:16:32.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:16:32 smithi083 ceph-mon[79930]: pgmap v12247: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:32.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:16:32 smithi083 ceph-mon[77242]: pgmap v12247: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:16:32 smithi153 ceph-mon[70044]: pgmap v12247: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:32.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:16:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:16:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:16:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:16:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:16:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:16:34.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:16:34 smithi083 ceph-mon[79930]: pgmap v12248: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:34.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:16:34 smithi083 ceph-mon[77242]: pgmap v12248: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:16:34 smithi153 ceph-mon[70044]: pgmap v12248: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:35.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:16:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:16:35.047Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:16:36.063 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:16:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:16:35.767Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:16:36.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:16:36 smithi083 ceph-mon[79930]: pgmap v12249: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:36.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:16:36 smithi083 ceph-mon[77242]: pgmap v12249: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:16:36 smithi153 ceph-mon[70044]: pgmap v12249: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:38.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:16:38 smithi083 ceph-mon[79930]: pgmap v12250: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:38.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:16:38 smithi083 ceph-mon[77242]: pgmap v12250: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:16:38 smithi153 ceph-mon[70044]: pgmap v12250: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:40.195 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:16:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:16:40 smithi083 ceph-mon[79930]: pgmap v12251: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:40.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:16:40 smithi083 ceph-mon[77242]: pgmap v12251: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:40.485 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:16:40.485 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 2m ago 6h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:16:40.486 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 35s ago 6h 62.7M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:16:40.486 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 2m ago 6h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:16:40.486 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 35s ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:16:40.486 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 35s ago 6h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:16:40.487 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 2m ago 6h 856M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:16:40.487 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 2m ago 6h 1523M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:16:40.487 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 35s ago 6h 658M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:16:40.487 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 2m ago 6h 879M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:16:40.487 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 2m ago 6h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:16:40.488 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 35s ago 6h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:16:40.488 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 2m ago 6h 1099M 2384M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:16:40.488 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 2m ago 6h 1087M 2384M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:16:40.488 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 2m ago 6h 827M 2384M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:16:40.488 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 2m ago 6h 1063M 2384M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:16:40.489 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 35s ago 6h 914M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:16:40.489 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 35s ago 6h 1170M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:16:40.489 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 35s ago 6h 1045M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:16:40.489 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 35s ago 6h 1240M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:16:40.489 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 35s ago 6h 166M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:16:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:16:40 smithi153 ceph-mon[70044]: pgmap v12251: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:40.840 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:16:40.840 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:16:40.840 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:16:40.841 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:16:40.841 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:16:40.841 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:16:40.841 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:16:40.842 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:16:40.842 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:16:40.842 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:16:40.842 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:16:40.843 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:16:40.843 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:16:40.843 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:16:40.843 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:16:40.843 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:16:40.844 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:16:40.844 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:16:41.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:16:41 smithi083 ceph-mon[79930]: from='client.48378 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:16:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:16:41 smithi083 ceph-mon[79930]: from='client.58315 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:16:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:16:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:16:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:16:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:16:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:16:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:16:41.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:16:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:16:41.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:16:41 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3737749302' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:16:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:16:41 smithi083 ceph-mon[77242]: from='client.48378 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:16:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:16:41 smithi083 ceph-mon[77242]: from='client.58315 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:16:41.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:16:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:16:41.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:16:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:16:41.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:16:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:16:41.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:16:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:16:41.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:16:41 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3737749302' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:16:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:16:41 smithi153 ceph-mon[70044]: from='client.48378 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:16:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:16:41 smithi153 ceph-mon[70044]: from='client.58315 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:16:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:16:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:16:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:16:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:16:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:16:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:16:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:16:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:16:41.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:16:41 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3737749302' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:16:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:16:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:16:41] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T12:16:41.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:16:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:16:41.672Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:16:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:16:42 smithi083 ceph-mon[79930]: pgmap v12252: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:42.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:16:42 smithi083 ceph-mon[77242]: pgmap v12252: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:16:42 smithi153 ceph-mon[70044]: pgmap v12252: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:42.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:16:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:16:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:16:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:16:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:16:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:16:44.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:16:44 smithi083 ceph-mon[79930]: pgmap v12253: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:44.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:16:44 smithi083 ceph-mon[77242]: pgmap v12253: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:44.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:16:44 smithi153 ceph-mon[70044]: pgmap v12253: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:45.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:16:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:16:45.048Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:16:46.065 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:16:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:16:45.768Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:16:46.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:16:46 smithi083 ceph-mon[79930]: pgmap v12254: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:46.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:16:46 smithi083 ceph-mon[77242]: pgmap v12254: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:46.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:16:46 smithi153 ceph-mon[70044]: pgmap v12254: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:48.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:16:48 smithi083 ceph-mon[79930]: pgmap v12255: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:48.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:16:48 smithi083 ceph-mon[77242]: pgmap v12255: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:48.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:16:48 smithi153 ceph-mon[70044]: pgmap v12255: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:50.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:16:50 smithi083 ceph-mon[77242]: pgmap v12256: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:50.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:16:50 smithi083 ceph-mon[79930]: pgmap v12256: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:16:50 smithi153 ceph-mon[70044]: pgmap v12256: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:16:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:16:51] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T12:16:51.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:16:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:16:51.673Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:16:52.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:16:52 smithi083 ceph-mon[79930]: pgmap v12257: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:52.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:16:52 smithi083 ceph-mon[77242]: pgmap v12257: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:16:52 smithi153 ceph-mon[70044]: pgmap v12257: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:52.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:16:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:16:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:16:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:16:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:16:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:16:54.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:16:54 smithi083 ceph-mon[79930]: pgmap v12258: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:54.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:16:54 smithi083 ceph-mon[77242]: pgmap v12258: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:54.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:16:54 smithi153 ceph-mon[70044]: pgmap v12258: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:16:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:16:55.049Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:16:56.069 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:16:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:16:55.768Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:16:56.070 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:16:56 smithi083 ceph-mon[77242]: pgmap v12259: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:56.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:16:56 smithi083 ceph-mon[79930]: pgmap v12259: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:56.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:16:56 smithi153 ceph-mon[70044]: pgmap v12259: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:58.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:16:58 smithi083 ceph-mon[79930]: pgmap v12260: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:58.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:16:58 smithi083 ceph-mon[77242]: pgmap v12260: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:16:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:16:58 smithi153 ceph-mon[70044]: pgmap v12260: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:00.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:17:00 smithi083 ceph-mon[79930]: pgmap v12261: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:00.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:17:00 smithi083 ceph-mon[77242]: pgmap v12261: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:17:00 smithi153 ceph-mon[70044]: pgmap v12261: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:17:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:17:01] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T12:17:01.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:17:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:17:01.674Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:17:02.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:17:02 smithi083 ceph-mon[79930]: pgmap v12262: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:02.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:17:02 smithi083 ceph-mon[77242]: pgmap v12262: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:17:02 smithi153 ceph-mon[70044]: pgmap v12262: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:02.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:17:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:17:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:17:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:17:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:17:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:17:04.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:17:04 smithi083 ceph-mon[79930]: pgmap v12263: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:04.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:17:04 smithi083 ceph-mon[77242]: pgmap v12263: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:17:04 smithi153 ceph-mon[70044]: pgmap v12263: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:05.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:17:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:17:05.050Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:17:06.068 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:17:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:17:05.769Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:17:06.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:17:06 smithi083 ceph-mon[79930]: pgmap v12264: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:06.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:17:06 smithi083 ceph-mon[77242]: pgmap v12264: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:06.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:17:06 smithi153 ceph-mon[70044]: pgmap v12264: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:08.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:17:08 smithi083 ceph-mon[79930]: pgmap v12265: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:08.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:17:08 smithi083 ceph-mon[77242]: pgmap v12265: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:17:08 smithi153 ceph-mon[70044]: pgmap v12265: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:09.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:17:09 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:17:09.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:17:09 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:17:09.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:17:09 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:17:09.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:17:09 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:17:09.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:17:09 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:17:09.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:17:09 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:17:09.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:17:09 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:17:09.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:17:09 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:17:09.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:17:09 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:17:09.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:17:09 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:17:09.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:17:09 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:17:09.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:17:09 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:17:10.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:17:10 smithi083 ceph-mon[79930]: pgmap v12266: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:10.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:17:10 smithi083 ceph-mon[77242]: pgmap v12266: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:17:10 smithi153 ceph-mon[70044]: pgmap v12266: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:11.168 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:17:11.456 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:17:11.456 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 3m ago 6h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:17:11.456 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 66s ago 6h 62.7M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:17:11.457 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 3m ago 6h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:17:11.457 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 66s ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:17:11.457 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 66s ago 6h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:17:11.457 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 3m ago 6h 856M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:17:11.458 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 3m ago 6h 1523M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:17:11.458 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 66s ago 6h 658M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:17:11.458 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 3m ago 6h 879M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:17:11.458 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 3m ago 6h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:17:11.459 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 66s ago 6h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:17:11.459 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 3m ago 6h 1099M 2384M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:17:11.459 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 3m ago 6h 1087M 2384M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:17:11.459 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 3m ago 6h 827M 2384M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:17:11.459 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 3m ago 6h 1063M 2384M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:17:11.460 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 66s ago 6h 914M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:17:11.460 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 66s ago 6h 1170M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:17:11.460 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 66s ago 6h 1045M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:17:11.460 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 66s ago 6h 1240M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:17:11.460 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 66s ago 6h 166M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:17:11.816 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:17:11.817 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:17:11.817 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:17:11.817 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:17:11.817 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:17:11.818 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:17:11.818 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:17:11.818 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:17:11.818 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:17:11.819 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:17:11.819 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:17:11.819 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:17:11.819 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:17:11.820 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:17:11.820 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:17:11.820 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:17:11.820 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:17:11.820 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:17:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:17:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:17:11] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T12:17:11.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:17:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:17:11.674Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:17:12.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:17:12 smithi083 ceph-mon[79930]: pgmap v12267: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:12.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:17:12 smithi083 ceph-mon[79930]: from='client.48396 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:17:12.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:17:12 smithi083 ceph-mon[79930]: from='client.58333 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:17:12.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:17:12 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1358366619' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:17:12.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:17:12 smithi083 ceph-mon[77242]: pgmap v12267: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:12.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:17:12 smithi083 ceph-mon[77242]: from='client.48396 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:17:12.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:17:12 smithi083 ceph-mon[77242]: from='client.58333 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:17:12.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:17:12 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1358366619' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:17:12.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:17:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:17:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:17:12.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:17:12 smithi153 ceph-mon[70044]: pgmap v12267: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:12.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:17:12 smithi153 ceph-mon[70044]: from='client.48396 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:17:12.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:17:12 smithi153 ceph-mon[70044]: from='client.58333 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:17:12.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:17:12 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1358366619' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:17:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:17:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:17:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:17:14.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:17:14 smithi083 ceph-mon[79930]: pgmap v12268: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:14.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:17:14 smithi083 ceph-mon[77242]: pgmap v12268: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:14.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:17:14 smithi153 ceph-mon[70044]: pgmap v12268: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:15.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:17:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:17:15.050Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:17:16.073 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:17:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:17:15.770Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:17:16.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:17:16 smithi083 ceph-mon[79930]: pgmap v12269: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:16.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:17:16 smithi083 ceph-mon[77242]: pgmap v12269: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:17:16 smithi153 ceph-mon[70044]: pgmap v12269: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:18.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:17:18 smithi083 ceph-mon[79930]: pgmap v12270: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:18.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:17:18 smithi083 ceph-mon[77242]: pgmap v12270: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:17:18 smithi153 ceph-mon[70044]: pgmap v12270: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:20.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:17:20 smithi083 ceph-mon[79930]: pgmap v12271: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:20.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:17:20 smithi083 ceph-mon[77242]: pgmap v12271: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:17:20 smithi153 ceph-mon[70044]: pgmap v12271: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:17:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:17:21] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T12:17:21.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:17:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:17:21.675Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:17:22.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:17:22 smithi083 ceph-mon[79930]: pgmap v12272: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:22.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:17:22 smithi083 ceph-mon[77242]: pgmap v12272: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:17:22 smithi153 ceph-mon[70044]: pgmap v12272: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:22.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:17:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:17:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:17:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:17:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:17:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:17:24.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:17:24 smithi083 ceph-mon[79930]: pgmap v12273: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:24.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:17:24 smithi083 ceph-mon[77242]: pgmap v12273: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:17:24 smithi153 ceph-mon[70044]: pgmap v12273: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:25.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:17:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:17:25.051Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:17:26.077 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:17:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:17:25.771Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:17:26.077 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:17:26 smithi083 ceph-mon[77242]: pgmap v12274: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:26.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:17:26 smithi083 ceph-mon[79930]: pgmap v12274: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:17:26 smithi153 ceph-mon[70044]: pgmap v12274: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:28.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:17:28 smithi083 ceph-mon[77242]: pgmap v12275: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:28.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:17:28 smithi083 ceph-mon[79930]: pgmap v12275: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:17:28 smithi153 ceph-mon[70044]: pgmap v12275: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:30.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:17:30 smithi083 ceph-mon[79930]: pgmap v12276: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:30.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:17:30 smithi083 ceph-mon[77242]: pgmap v12276: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:17:30 smithi153 ceph-mon[70044]: pgmap v12276: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:17:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:17:31] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T12:17:31.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:17:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:17:31.676Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:17:32.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:17:32 smithi083 ceph-mon[77242]: pgmap v12277: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:32.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:17:32 smithi083 ceph-mon[79930]: pgmap v12277: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:17:32 smithi153 ceph-mon[70044]: pgmap v12277: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:32.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:17:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:17:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:17:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:17:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:17:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:17:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:17:34 smithi153 ceph-mon[70044]: pgmap v12278: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:34.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:17:34 smithi083 ceph-mon[79930]: pgmap v12278: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:34.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:17:34 smithi083 ceph-mon[77242]: pgmap v12278: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:35.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:17:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:17:35.052Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:17:36.072 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:17:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:17:35.771Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:17:36.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:17:36 smithi083 ceph-mon[77242]: pgmap v12279: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:36.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:17:36 smithi083 ceph-mon[79930]: pgmap v12279: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:17:36 smithi153 ceph-mon[70044]: pgmap v12279: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:38.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:17:38 smithi083 ceph-mon[79930]: pgmap v12280: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:38.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:17:38 smithi083 ceph-mon[77242]: pgmap v12280: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:17:38 smithi153 ceph-mon[70044]: pgmap v12280: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:17:40 smithi153 ceph-mon[70044]: pgmap v12281: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:17:40 smithi083 ceph-mon[79930]: pgmap v12281: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:17:40 smithi083 ceph-mon[77242]: pgmap v12281: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:41.507 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:17:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:17:41.508 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:17:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:17:41.508 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:17:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:17:41.508 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:17:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:17:41.509 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:17:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:17:41.509 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:17:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:17:41.509 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:17:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:17:41.509 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:17:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:17:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:17:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:17:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:17:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:17:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:17:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:17:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:17:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:17:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:17:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:17:41] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T12:17:41.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:17:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:17:41.676Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:17:42.143 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:17:42.433 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:17:42.433 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 3m ago 6h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:17:42.433 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 97s ago 6h 62.7M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:17:42.434 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 3m ago 6h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:17:42.434 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 97s ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:17:42.434 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 97s ago 6h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:17:42.434 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 3m ago 6h 856M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:17:42.435 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 3m ago 6h 1523M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:17:42.435 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 97s ago 6h 658M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:17:42.435 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 3m ago 6h 879M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:17:42.435 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 3m ago 6h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:17:42.436 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 97s ago 6h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:17:42.436 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 3m ago 6h 1099M 2384M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:17:42.436 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 3m ago 6h 1087M 2384M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:17:42.436 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 3m ago 6h 827M 2384M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:17:42.437 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 3m ago 6h 1063M 2384M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:17:42.437 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 97s ago 6h 914M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:17:42.437 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 97s ago 6h 1170M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:17:42.437 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 97s ago 6h 1045M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:17:42.437 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 97s ago 6h 1240M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:17:42.438 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 97s ago 6h 166M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:17:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:17:42 smithi153 ceph-mon[70044]: pgmap v12282: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:42.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:17:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:17:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:17:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:17:42 smithi083 ceph-mon[79930]: pgmap v12282: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:42.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:17:42 smithi083 ceph-mon[77242]: pgmap v12282: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:42.792 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:17:42.792 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:17:42.793 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:17:42.793 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:17:42.793 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:17:42.793 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:17:42.794 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:17:42.794 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:17:42.794 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:17:42.794 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:17:42.795 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:17:42.795 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:17:42.795 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:17:42.795 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:17:42.795 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:17:42.796 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:17:42.796 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:17:42.796 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:17:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:17:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:17:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:17:43.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:17:43 smithi153 ceph-mon[70044]: from='client.58345 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:17:43.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:17:43 smithi153 ceph-mon[70044]: from='client.48420 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:17:43.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:17:43 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3776588786' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:17:43.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:17:43 smithi083 ceph-mon[79930]: from='client.58345 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:17:43.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:17:43 smithi083 ceph-mon[79930]: from='client.48420 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:17:43.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:17:43 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3776588786' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:17:43.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:17:43 smithi083 ceph-mon[77242]: from='client.58345 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:17:43.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:17:43 smithi083 ceph-mon[77242]: from='client.48420 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:17:43.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:17:43 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3776588786' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:17:44.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:17:44 smithi153 ceph-mon[70044]: pgmap v12283: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:44.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:17:44 smithi083 ceph-mon[79930]: pgmap v12283: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:44.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:17:44 smithi083 ceph-mon[77242]: pgmap v12283: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:45.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:17:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:17:45.053Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:17:46.079 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:17:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:17:45.772Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:17:46.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:17:46 smithi083 ceph-mon[79930]: pgmap v12284: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:46.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:17:46 smithi083 ceph-mon[77242]: pgmap v12284: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:46.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:17:46 smithi153 ceph-mon[70044]: pgmap v12284: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:48.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:17:48 smithi083 ceph-mon[79930]: pgmap v12285: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:48.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:17:48 smithi083 ceph-mon[77242]: pgmap v12285: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:48.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:17:48 smithi153 ceph-mon[70044]: pgmap v12285: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:17:50 smithi153 ceph-mon[70044]: pgmap v12286: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:50.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:17:50 smithi083 ceph-mon[79930]: pgmap v12286: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:50.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:17:50 smithi083 ceph-mon[77242]: pgmap v12286: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:17:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:17:51] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T12:17:51.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:17:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:17:51.677Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:17:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:17:52 smithi153 ceph-mon[70044]: pgmap v12287: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:52.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:17:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:17:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:17:52.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:17:52 smithi083 ceph-mon[79930]: pgmap v12287: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:52.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:17:52 smithi083 ceph-mon[77242]: pgmap v12287: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:17:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:17:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:17:54.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:17:54 smithi153 ceph-mon[70044]: pgmap v12288: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:54.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:17:54 smithi083 ceph-mon[77242]: pgmap v12288: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:54.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:17:54 smithi083 ceph-mon[79930]: pgmap v12288: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:17:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:17:55.054Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:17:56.081 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:17:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:17:55.773Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:17:56.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:17:56 smithi083 ceph-mon[77242]: pgmap v12289: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:56.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:17:56 smithi083 ceph-mon[79930]: pgmap v12289: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:56.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:17:56 smithi153 ceph-mon[70044]: pgmap v12289: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:17:58 smithi153 ceph-mon[70044]: pgmap v12290: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:58.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:17:58 smithi083 ceph-mon[77242]: pgmap v12290: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:17:58.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:17:58 smithi083 ceph-mon[79930]: pgmap v12290: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:18:00 smithi153 ceph-mon[70044]: pgmap v12291: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:00.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:18:00 smithi083 ceph-mon[77242]: pgmap v12291: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:00.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:18:00 smithi083 ceph-mon[79930]: pgmap v12291: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:18:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:18:01] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T12:18:01.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:18:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:18:01.678Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:18:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:18:02 smithi153 ceph-mon[70044]: pgmap v12292: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:02.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:18:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:18:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:18:02.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:18:02 smithi083 ceph-mon[77242]: pgmap v12292: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:02.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:18:02 smithi083 ceph-mon[79930]: pgmap v12292: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:18:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:18:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:18:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:18:04 smithi153 ceph-mon[70044]: pgmap v12293: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:04.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:18:04 smithi083 ceph-mon[79930]: pgmap v12293: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:04.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:18:04 smithi083 ceph-mon[77242]: pgmap v12293: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:05.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:18:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:18:05.054Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:18:06.084 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:18:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:18:05.774Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:18:06.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:18:06 smithi083 ceph-mon[79930]: pgmap v12294: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:06.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:18:06 smithi083 ceph-mon[77242]: pgmap v12294: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:06.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:18:06 smithi153 ceph-mon[70044]: pgmap v12294: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:18:08 smithi153 ceph-mon[70044]: pgmap v12295: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:08.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:18:08 smithi083 ceph-mon[79930]: pgmap v12295: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:08.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:18:08 smithi083 ceph-mon[77242]: pgmap v12295: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:09.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:18:09 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:18:09.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:18:09 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:18:09.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:18:09 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:18:09.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:18:09 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:18:09.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:18:09 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:18:09.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:18:09 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:18:09.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:18:09 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:18:09.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:18:09 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:18:09.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:18:09 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:18:10.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:18:10 smithi083 ceph-mon[79930]: pgmap v12296: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:10.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:18:10 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:18:10.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:18:10 smithi083 ceph-mon[77242]: pgmap v12296: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:10.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:18:10 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:18:11.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:18:10 smithi153 ceph-mon[70044]: pgmap v12296: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:11.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:18:10 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:18:11.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:18:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:18:11.679Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:18:11.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:18:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:18:11] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T12:18:12.709 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:18:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:18:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:18:12.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:18:12 smithi083 ceph-mon[77242]: pgmap v12297: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:12.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:18:12 smithi083 ceph-mon[79930]: pgmap v12297: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:13.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:18:12 smithi153 ceph-mon[70044]: pgmap v12297: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:13.117 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:18:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:18:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:18:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:18:13.410 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:18:13.410 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 4m ago 6h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:18:13.410 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 2m ago 6h 62.7M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:18:13.411 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 4m ago 6h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:18:13.411 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 2m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:18:13.411 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 2m ago 6h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:18:13.411 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 4m ago 6h 856M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:18:13.411 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 4m ago 6h 1523M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:18:13.412 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 2m ago 6h 658M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:18:13.412 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 4m ago 6h 879M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:18:13.412 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 4m ago 6h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:18:13.412 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 2m ago 6h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:18:13.413 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 4m ago 6h 1099M 2384M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:18:13.413 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 4m ago 6h 1087M 2384M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:18:13.413 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 4m ago 6h 827M 2384M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:18:13.413 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 4m ago 6h 1063M 2384M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:18:13.414 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 2m ago 6h 914M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:18:13.414 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 2m ago 6h 1170M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:18:13.414 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 2m ago 6h 1045M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:18:13.414 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 2m ago 6h 1240M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:18:13.415 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 2m ago 6h 166M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:18:13.770 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:18:13.771 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:18:13.771 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:18:13.771 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:18:13.771 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:18:13.772 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:18:13.772 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:18:13.772 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:18:13.772 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:18:13.773 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:18:13.773 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:18:13.773 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:18:13.773 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:18:13.773 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:18:13.774 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:18:13.774 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:18:13.774 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:18:13.774 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:18:14.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:18:14 smithi083 ceph-mon[79930]: from='client.48432 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:18:14.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:18:14 smithi083 ceph-mon[79930]: pgmap v12298: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:14.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:18:14 smithi083 ceph-mon[79930]: from='client.48438 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:18:14.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:18:14 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1291561996' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:18:14.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:18:14 smithi083 ceph-mon[77242]: from='client.48432 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:18:14.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:18:14 smithi083 ceph-mon[77242]: pgmap v12298: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:14.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:18:14 smithi083 ceph-mon[77242]: from='client.48438 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:18:14.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:18:14 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1291561996' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:18:15.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:18:14 smithi153 ceph-mon[70044]: from='client.48432 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:18:15.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:18:14 smithi153 ceph-mon[70044]: pgmap v12298: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:15.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:18:14 smithi153 ceph-mon[70044]: from='client.48438 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:18:15.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:18:14 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1291561996' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:18:15.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:18:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:18:15.056Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:18:16.083 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:18:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:18:15.775Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:18:16.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:18:16 smithi083 ceph-mon[79930]: pgmap v12299: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:16.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:18:16 smithi083 ceph-mon[77242]: pgmap v12299: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:18:16 smithi153 ceph-mon[70044]: pgmap v12299: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:18:18 smithi153 ceph-mon[70044]: pgmap v12300: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:18.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:18:18 smithi083 ceph-mon[79930]: pgmap v12300: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:18.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:18:18 smithi083 ceph-mon[77242]: pgmap v12300: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:18:20 smithi153 ceph-mon[70044]: pgmap v12301: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:20.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:18:20 smithi083 ceph-mon[79930]: pgmap v12301: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:20.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:18:20 smithi083 ceph-mon[77242]: pgmap v12301: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:18:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:18:21] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T12:18:21.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:18:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:18:21.680Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:18:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:18:22 smithi153 ceph-mon[70044]: pgmap v12302: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:22.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:18:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:18:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:18:22.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:18:22 smithi083 ceph-mon[79930]: pgmap v12302: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:22.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:18:22 smithi083 ceph-mon[77242]: pgmap v12302: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:18:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:18:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:18:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:18:24 smithi153 ceph-mon[70044]: pgmap v12303: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:24.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:18:24 smithi083 ceph-mon[79930]: pgmap v12303: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:24.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:18:24 smithi083 ceph-mon[77242]: pgmap v12303: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:25.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:18:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:18:25.056Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:18:26.086 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:18:26 smithi083 ceph-mon[77242]: pgmap v12304: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:26.086 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:18:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:18:25.775Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:18:26.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:18:26 smithi083 ceph-mon[79930]: pgmap v12304: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:18:26 smithi153 ceph-mon[70044]: pgmap v12304: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:18:28 smithi153 ceph-mon[70044]: pgmap v12305: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:28.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:18:28 smithi083 ceph-mon[79930]: pgmap v12305: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:28.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:18:28 smithi083 ceph-mon[77242]: pgmap v12305: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:18:30 smithi153 ceph-mon[70044]: pgmap v12306: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:30.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:18:30 smithi083 ceph-mon[79930]: pgmap v12306: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:30.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:18:30 smithi083 ceph-mon[77242]: pgmap v12306: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:18:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:18:31] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T12:18:31.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:18:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:18:31.680Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:18:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:18:32 smithi153 ceph-mon[70044]: pgmap v12307: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:32.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:18:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:18:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:18:32.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:18:32 smithi083 ceph-mon[79930]: pgmap v12307: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:32.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:18:32 smithi083 ceph-mon[77242]: pgmap v12307: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:18:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:18:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:18:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:18:34 smithi153 ceph-mon[70044]: pgmap v12308: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:34.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:18:34 smithi083 ceph-mon[77242]: pgmap v12308: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:34.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:18:34 smithi083 ceph-mon[79930]: pgmap v12308: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:35.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:18:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:18:35.056Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:18:36.088 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:18:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:18:35.776Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:18:36.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:18:36 smithi083 ceph-mon[79930]: pgmap v12309: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:36.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:18:36 smithi083 ceph-mon[77242]: pgmap v12309: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:18:36 smithi153 ceph-mon[70044]: pgmap v12309: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:18:38 smithi153 ceph-mon[70044]: pgmap v12310: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:38.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:18:38 smithi083 ceph-mon[79930]: pgmap v12310: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:38.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:18:38 smithi083 ceph-mon[77242]: pgmap v12310: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:18:40 smithi153 ceph-mon[70044]: pgmap v12311: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:18:40 smithi083 ceph-mon[79930]: pgmap v12311: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:18:40 smithi083 ceph-mon[77242]: pgmap v12311: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:41.507 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:18:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:18:41.508 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:18:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:18:41.508 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:18:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:18:41.508 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:18:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:18:41.509 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:18:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:18:41.509 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:18:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:18:41.509 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:18:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:18:41.509 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:18:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:18:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:18:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:18:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:18:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:18:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:18:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:18:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:18:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:18:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:18:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:18:41] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T12:18:41.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:18:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:18:41.680Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:18:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:18:42 smithi153 ceph-mon[70044]: pgmap v12312: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:42.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:18:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:18:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:18:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:18:42 smithi083 ceph-mon[79930]: pgmap v12312: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:42.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:18:42 smithi083 ceph-mon[77242]: pgmap v12312: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:18:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:18:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:18:44.099 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:18:44.390 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:18:44.390 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 4m ago 6h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:18:44.390 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 2m ago 6h 62.7M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:18:44.391 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 4m ago 6h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:18:44.391 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 2m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:18:44.391 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 2m ago 6h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:18:44.391 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 4m ago 6h 856M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:18:44.392 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 4m ago 6h 1523M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:18:44.392 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 2m ago 6h 658M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:18:44.392 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 4m ago 6h 879M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:18:44.392 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 4m ago 6h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:18:44.393 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 2m ago 6h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:18:44.393 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 4m ago 6h 1099M 2384M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:18:44.393 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 4m ago 6h 1087M 2384M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:18:44.393 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 4m ago 6h 827M 2384M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:18:44.393 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 4m ago 6h 1063M 2384M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:18:44.394 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 2m ago 6h 914M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:18:44.394 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 2m ago 6h 1170M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:18:44.394 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 2m ago 6h 1045M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:18:44.394 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 2m ago 6h 1240M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:18:44.395 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 2m ago 6h 166M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:18:44.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:18:44 smithi153 ceph-mon[70044]: pgmap v12313: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:44.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:18:44 smithi083 ceph-mon[79930]: pgmap v12313: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:44.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:18:44 smithi083 ceph-mon[77242]: pgmap v12313: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:44.745 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:18:44.745 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:18:44.746 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:18:44.746 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:18:44.746 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:18:44.746 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:18:44.746 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:18:44.747 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:18:44.747 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:18:44.747 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:18:44.747 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:18:44.748 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:18:44.748 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:18:44.748 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:18:44.748 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:18:44.748 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:18:44.749 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:18:44.749 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:18:45.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:18:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:18:45.056Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:18:45.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:18:45 smithi083 ceph-mon[77242]: from='client.58381 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:18:45.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:18:45 smithi083 ceph-mon[77242]: from='client.58387 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:18:45.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:18:45 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/984104629' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:18:45.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:18:45 smithi083 ceph-mon[79930]: from='client.58381 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:18:45.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:18:45 smithi083 ceph-mon[79930]: from='client.58387 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:18:45.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:18:45 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/984104629' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:18:45.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:18:45 smithi153 ceph-mon[70044]: from='client.58381 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:18:45.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:18:45 smithi153 ceph-mon[70044]: from='client.58387 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:18:45.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:18:45 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/984104629' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:18:46.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:18:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:18:45.777Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:18:46.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:18:46 smithi153 ceph-mon[70044]: pgmap v12314: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:46.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:18:46 smithi083 ceph-mon[77242]: pgmap v12314: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:46.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:18:46 smithi083 ceph-mon[79930]: pgmap v12314: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:48.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:18:48 smithi153 ceph-mon[70044]: pgmap v12315: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:48.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:18:48 smithi083 ceph-mon[79930]: pgmap v12315: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:48.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:18:48 smithi083 ceph-mon[77242]: pgmap v12315: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:50.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:18:50 smithi153 ceph-mon[70044]: pgmap v12316: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:50.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:18:50 smithi083 ceph-mon[79930]: pgmap v12316: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:50.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:18:50 smithi083 ceph-mon[77242]: pgmap v12316: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:18:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:18:51] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T12:18:51.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:18:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:18:51.681Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:18:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:18:52 smithi153 ceph-mon[70044]: pgmap v12317: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:52.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:18:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:18:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:18:52.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:18:52 smithi083 ceph-mon[77242]: pgmap v12317: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:52.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:18:52 smithi083 ceph-mon[79930]: pgmap v12317: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:18:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:18:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:18:54.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:18:54 smithi153 ceph-mon[70044]: pgmap v12318: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:54.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:18:54 smithi083 ceph-mon[77242]: pgmap v12318: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:54.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:18:54 smithi083 ceph-mon[79930]: pgmap v12318: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:18:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:18:55.057Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:18:56.100 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:18:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:18:55.777Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:18:56.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:18:56 smithi083 ceph-mon[79930]: pgmap v12319: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:56.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:18:56 smithi083 ceph-mon[77242]: pgmap v12319: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:56.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:18:56 smithi153 ceph-mon[70044]: pgmap v12319: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:18:58 smithi153 ceph-mon[70044]: pgmap v12320: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:58.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:18:58 smithi083 ceph-mon[79930]: pgmap v12320: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:18:58.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:18:58 smithi083 ceph-mon[77242]: pgmap v12320: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:19:00 smithi153 ceph-mon[70044]: pgmap v12321: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:00.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:19:00 smithi083 ceph-mon[79930]: pgmap v12321: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:00.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:19:00 smithi083 ceph-mon[77242]: pgmap v12321: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:19:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:19:01] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T12:19:01.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:19:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:19:01.682Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:19:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:19:02 smithi153 ceph-mon[70044]: pgmap v12322: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:02.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:19:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:19:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:19:02.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:19:02 smithi083 ceph-mon[79930]: pgmap v12322: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:02.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:19:02 smithi083 ceph-mon[77242]: pgmap v12322: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:19:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:19:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:19:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:19:04 smithi153 ceph-mon[70044]: pgmap v12323: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:04.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:19:04 smithi083 ceph-mon[79930]: pgmap v12323: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:04.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:19:04 smithi083 ceph-mon[77242]: pgmap v12323: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:05.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:19:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:19:05.057Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:19:06.094 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:19:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:19:05.778Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:19:06.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:19:06 smithi083 ceph-mon[79930]: pgmap v12324: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:06.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:19:06 smithi083 ceph-mon[77242]: pgmap v12324: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:06.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:19:06 smithi153 ceph-mon[70044]: pgmap v12324: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:19:08 smithi153 ceph-mon[70044]: pgmap v12325: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:08.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:19:08 smithi083 ceph-mon[79930]: pgmap v12325: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:08.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:19:08 smithi083 ceph-mon[77242]: pgmap v12325: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:19:10 smithi153 ceph-mon[70044]: pgmap v12326: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:19:10 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:19:10.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:19:10 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:19:10.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:19:10 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:19:10.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:19:10 smithi083 ceph-mon[79930]: pgmap v12326: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:10.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:19:10 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:19:10.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:19:10 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:19:10.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:19:10 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:19:10.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:19:10 smithi083 ceph-mon[77242]: pgmap v12326: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:10.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:19:10 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:19:10.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:19:10 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:19:10.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:19:10 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:19:11.685 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:19:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:19:11] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T12:19:11.685 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:19:11 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:19:11.686 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:19:11 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:19:11.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:19:11 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:19:11.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:19:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:19:11.683Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:19:12.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:19:12 smithi083 ceph-mon[79930]: pgmap v12327: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:12.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:19:12 smithi083 ceph-mon[77242]: pgmap v12327: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:12.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:19:12 smithi153 ceph-mon[70044]: pgmap v12327: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:19:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:19:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:19:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:19:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:19:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:19:14.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:19:14 smithi083 ceph-mon[79930]: pgmap v12328: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:14.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:19:14 smithi083 ceph-mon[77242]: pgmap v12328: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:14.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:19:14 smithi153 ceph-mon[70044]: pgmap v12328: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:15.072 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:19:15.360 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:19:15.360 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 5m ago 6h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:19:15.361 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 3m ago 6h 62.7M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:19:15.361 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 5m ago 6h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:19:15.361 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 3m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:19:15.361 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 3m ago 6h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:19:15.362 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 5m ago 6h 856M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:19:15.362 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 5m ago 6h 1523M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:19:15.362 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 3m ago 6h 658M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:19:15.362 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 5m ago 6h 879M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:19:15.363 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 5m ago 6h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:19:15.363 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 3m ago 6h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:19:15.363 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 5m ago 6h 1099M 2384M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:19:15.363 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 5m ago 6h 1087M 2384M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:19:15.364 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 5m ago 6h 827M 2384M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:19:15.364 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 5m ago 6h 1063M 2384M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:19:15.364 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 3m ago 6h 914M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:19:15.364 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 3m ago 6h 1170M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:19:15.365 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 3m ago 6h 1045M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:19:15.365 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 3m ago 6h 1240M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:19:15.365 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 3m ago 6h 166M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:19:15.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:19:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:19:15.058Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:19:15.717 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:19:15.718 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:19:15.718 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:19:15.718 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:19:15.719 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:19:15.719 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:19:15.719 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:19:15.719 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:19:15.720 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:19:15.720 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:19:15.720 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:19:15.720 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:19:15.720 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:19:15.721 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:19:15.721 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:19:15.721 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:19:15.721 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:19:15.722 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:19:16.096 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:19:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:19:15.778Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:19:16.097 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:19:16 smithi083 ceph-mon[77242]: from='client.48468 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:19:16.097 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:19:16 smithi083 ceph-mon[77242]: pgmap v12329: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:16.097 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:19:16 smithi083 ceph-mon[77242]: from='client.48474 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:19:16.097 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:19:16 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/25339031' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:19:16.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:19:16 smithi083 ceph-mon[79930]: from='client.48468 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:19:16.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:19:16 smithi083 ceph-mon[79930]: pgmap v12329: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:16.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:19:16 smithi083 ceph-mon[79930]: from='client.48474 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:19:16.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:19:16 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/25339031' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:19:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:19:16 smithi153 ceph-mon[70044]: from='client.48468 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:19:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:19:16 smithi153 ceph-mon[70044]: pgmap v12329: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:19:16 smithi153 ceph-mon[70044]: from='client.48474 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:19:16.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:19:16 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/25339031' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:19:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:19:18 smithi153 ceph-mon[70044]: pgmap v12330: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:18.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:19:18 smithi083 ceph-mon[79930]: pgmap v12330: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:18.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:19:18 smithi083 ceph-mon[77242]: pgmap v12330: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:19:20 smithi153 ceph-mon[70044]: pgmap v12331: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:20.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:19:20 smithi083 ceph-mon[79930]: pgmap v12331: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:20.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:19:20 smithi083 ceph-mon[77242]: pgmap v12331: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:19:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:19:21] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T12:19:21.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:19:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:19:21.684Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:19:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:19:22 smithi153 ceph-mon[70044]: pgmap v12332: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:22.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:19:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:19:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:19:22.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:19:22 smithi083 ceph-mon[79930]: pgmap v12332: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:22.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:19:22 smithi083 ceph-mon[77242]: pgmap v12332: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:19:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:19:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:19:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:19:24 smithi153 ceph-mon[70044]: pgmap v12333: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:24.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:19:24 smithi083 ceph-mon[79930]: pgmap v12333: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:24.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:19:24 smithi083 ceph-mon[77242]: pgmap v12333: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:25.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:19:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:19:25.059Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:19:26.100 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:19:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:19:25.779Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:19:26.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:19:26 smithi083 ceph-mon[79930]: pgmap v12334: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:26.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:19:26 smithi083 ceph-mon[77242]: pgmap v12334: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:19:26 smithi153 ceph-mon[70044]: pgmap v12334: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:19:28 smithi153 ceph-mon[70044]: pgmap v12335: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:28.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:19:28 smithi083 ceph-mon[79930]: pgmap v12335: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:28.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:19:28 smithi083 ceph-mon[77242]: pgmap v12335: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:19:30 smithi153 ceph-mon[70044]: pgmap v12336: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:30.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:19:30 smithi083 ceph-mon[79930]: pgmap v12336: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:30.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:19:30 smithi083 ceph-mon[77242]: pgmap v12336: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:19:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:19:31] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T12:19:31.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:19:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:19:31.684Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:19:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:19:32 smithi153 ceph-mon[70044]: pgmap v12337: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:32.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:19:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:19:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:19:32.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:19:32 smithi083 ceph-mon[79930]: pgmap v12337: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:32.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:19:32 smithi083 ceph-mon[77242]: pgmap v12337: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:19:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:19:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:19:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:19:34 smithi153 ceph-mon[70044]: pgmap v12338: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:34.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:19:34 smithi083 ceph-mon[79930]: pgmap v12338: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:34.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:19:34 smithi083 ceph-mon[77242]: pgmap v12338: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:35.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:19:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:19:35.059Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:19:36.102 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:19:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:19:35.780Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:19:36.103 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:19:36 smithi083 ceph-mon[77242]: pgmap v12339: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:36.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:19:36 smithi083 ceph-mon[79930]: pgmap v12339: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:19:36 smithi153 ceph-mon[70044]: pgmap v12339: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:19:38 smithi153 ceph-mon[70044]: pgmap v12340: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:38.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:19:38 smithi083 ceph-mon[77242]: pgmap v12340: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:38.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:19:38 smithi083 ceph-mon[79930]: pgmap v12340: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:19:40 smithi153 ceph-mon[70044]: pgmap v12341: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:19:40 smithi083 ceph-mon[79930]: pgmap v12341: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:19:40 smithi083 ceph-mon[77242]: pgmap v12341: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:41.507 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:19:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:19:41.508 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:19:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:19:41.508 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:19:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:19:41.508 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:19:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:19:41.509 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:19:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:19:41.509 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:19:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:19:41.509 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:19:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:19:41.509 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:19:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:19:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:19:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:19:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:19:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:19:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:19:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:19:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:19:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:19:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:19:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:19:41] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T12:19:41.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:19:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:19:41.685Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:19:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:19:42 smithi153 ceph-mon[70044]: pgmap v12342: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:42.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:19:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:19:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:19:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:19:42 smithi083 ceph-mon[79930]: pgmap v12342: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:19:42 smithi083 ceph-mon[77242]: pgmap v12342: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:19:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:19:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:19:44.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:19:44 smithi153 ceph-mon[70044]: pgmap v12343: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:44.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:19:44 smithi083 ceph-mon[77242]: pgmap v12343: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:44.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:19:44 smithi083 ceph-mon[79930]: pgmap v12343: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:45.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:19:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:19:45.060Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:19:46.044 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:19:46.104 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:19:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:19:45.780Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:19:46.105 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:19:46 smithi083 ceph-mon[77242]: pgmap v12344: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:46.337 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:19:46.337 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 5m ago 6h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:19:46.338 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 3m ago 6h 62.7M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:19:46.338 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 5m ago 6h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:19:46.338 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 3m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:19:46.338 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 3m ago 6h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:19:46.339 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 5m ago 6h 856M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:19:46.339 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 5m ago 6h 1523M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:19:46.339 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 3m ago 6h 658M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:19:46.339 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 5m ago 6h 879M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:19:46.340 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 5m ago 6h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:19:46.340 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 3m ago 6h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:19:46.340 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 5m ago 6h 1099M 2384M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:19:46.340 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 5m ago 6h 1087M 2384M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:19:46.341 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 5m ago 6h 827M 2384M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:19:46.341 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 5m ago 6h 1063M 2384M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:19:46.341 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 3m ago 6h 914M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:19:46.341 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 3m ago 6h 1170M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:19:46.341 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 3m ago 6h 1045M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:19:46.342 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 3m ago 6h 1240M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:19:46.342 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 3m ago 6h 166M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:19:46.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:19:46 smithi083 ceph-mon[79930]: pgmap v12344: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:46.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:19:46 smithi153 ceph-mon[70044]: pgmap v12344: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:46.694 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:19:46.694 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:19:46.694 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:19:46.694 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:19:46.695 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:19:46.695 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:19:46.695 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:19:46.695 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:19:46.696 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:19:46.696 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:19:46.696 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:19:46.696 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:19:46.697 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:19:46.697 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:19:46.697 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:19:46.697 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:19:46.697 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:19:46.698 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:19:47.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:19:47 smithi083 ceph-mon[79930]: from='client.48486 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:19:47.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:19:47 smithi083 ceph-mon[79930]: from='client.58423 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:19:47.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:19:47 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1907755546' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:19:47.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:19:47 smithi083 ceph-mon[77242]: from='client.48486 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:19:47.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:19:47 smithi083 ceph-mon[77242]: from='client.58423 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:19:47.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:19:47 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1907755546' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:19:47.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:19:47 smithi153 ceph-mon[70044]: from='client.48486 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:19:47.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:19:47 smithi153 ceph-mon[70044]: from='client.58423 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:19:47.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:19:47 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1907755546' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:19:48.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:19:48 smithi083 ceph-mon[79930]: pgmap v12345: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:48.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:19:48 smithi083 ceph-mon[77242]: pgmap v12345: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:48.471 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:19:48 smithi153 ceph-mon[70044]: pgmap v12345: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:19:50 smithi153 ceph-mon[70044]: pgmap v12346: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:50.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:19:50 smithi083 ceph-mon[79930]: pgmap v12346: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:50.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:19:50 smithi083 ceph-mon[77242]: pgmap v12346: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:19:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:19:51] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T12:19:51.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:19:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:19:51.685Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:19:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:19:52 smithi153 ceph-mon[70044]: pgmap v12347: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:52.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:19:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:19:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:19:52.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:19:52 smithi083 ceph-mon[79930]: pgmap v12347: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:52.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:19:52 smithi083 ceph-mon[77242]: pgmap v12347: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:19:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:19:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:19:54.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:19:54 smithi153 ceph-mon[70044]: pgmap v12348: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:54.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:19:54 smithi083 ceph-mon[79930]: pgmap v12348: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:54.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:19:54 smithi083 ceph-mon[77242]: pgmap v12348: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:19:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:19:55.061Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:19:56.108 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:19:56 smithi083 ceph-mon[77242]: pgmap v12349: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:56.108 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:19:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:19:55.781Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:19:56.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:19:56 smithi083 ceph-mon[79930]: pgmap v12349: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:56.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:19:56 smithi153 ceph-mon[70044]: pgmap v12349: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:19:58 smithi153 ceph-mon[70044]: pgmap v12350: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:58.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:19:58 smithi083 ceph-mon[79930]: pgmap v12350: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:19:58.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:19:58 smithi083 ceph-mon[77242]: pgmap v12350: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:00.253 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:20:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[77219]: 2022-09-29T12:20:00.000+0000 7f160f665700 -1 log_channel(cluster) log [ERR] : overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T12:20:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:20:00 smithi153 ceph-mon[70044]: pgmap v12351: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:20:00 smithi153 ceph-mon[70044]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T12:20:00.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:20:00 smithi083 ceph-mon[77242]: pgmap v12351: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:00.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:20:00 smithi083 ceph-mon[77242]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T12:20:00.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:20:00 smithi083 ceph-mon[79930]: pgmap v12351: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:00.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:20:00 smithi083 ceph-mon[79930]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T12:20:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:20:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:20:01] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T12:20:01.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:20:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:20:01.686Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:20:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:20:02 smithi153 ceph-mon[70044]: pgmap v12352: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:02.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:20:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:20:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:20:02.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:20:02 smithi083 ceph-mon[79930]: pgmap v12352: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:02.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:20:02 smithi083 ceph-mon[77242]: pgmap v12352: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:20:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:20:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:20:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:20:04 smithi153 ceph-mon[70044]: pgmap v12353: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:04.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:20:04 smithi083 ceph-mon[79930]: pgmap v12353: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:04.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:20:04 smithi083 ceph-mon[77242]: pgmap v12353: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:05.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:20:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:20:05.062Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:20:06.109 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:20:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:20:05.782Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:20:06.110 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:20:06 smithi083 ceph-mon[77242]: pgmap v12354: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:06.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:20:06 smithi083 ceph-mon[79930]: pgmap v12354: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:06.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:20:06 smithi153 ceph-mon[70044]: pgmap v12354: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:20:08 smithi153 ceph-mon[70044]: pgmap v12355: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:08.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:20:08 smithi083 ceph-mon[79930]: pgmap v12355: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:08.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:20:08 smithi083 ceph-mon[77242]: pgmap v12355: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:20:10 smithi153 ceph-mon[70044]: pgmap v12356: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:10.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:20:10 smithi083 ceph-mon[79930]: pgmap v12356: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:10.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:20:10 smithi083 ceph-mon[77242]: pgmap v12356: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:11.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:20:11 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:20:11.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:20:11 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:20:11.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:20:11 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:20:11.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:20:11 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:20:11.689 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:20:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:20:11] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T12:20:11.689 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:20:11 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:20:11.689 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:20:11 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:20:11.690 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:20:11 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:20:11.690 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:20:11 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:20:11.690 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:20:11 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:20:11.690 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:20:11 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:20:11.691 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:20:11 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:20:11.691 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:20:11 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:20:11.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:20:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:20:11.687Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:20:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:20:12 smithi153 ceph-mon[70044]: pgmap v12357: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:12.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:20:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:20:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:20:12.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:20:12 smithi083 ceph-mon[77242]: pgmap v12357: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:12.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:20:12 smithi083 ceph-mon[79930]: pgmap v12357: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:20:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:20:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:20:14.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:20:14 smithi153 ceph-mon[70044]: pgmap v12358: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:14.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:20:14 smithi083 ceph-mon[77242]: pgmap v12358: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:14.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:20:14 smithi083 ceph-mon[79930]: pgmap v12358: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:15.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:20:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:20:15.062Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:20:16.113 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:20:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:20:15.783Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:20:16.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:20:16 smithi083 ceph-mon[79930]: pgmap v12359: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:16.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:20:16 smithi083 ceph-mon[77242]: pgmap v12359: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:20:16 smithi153 ceph-mon[70044]: pgmap v12359: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:17.022 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:20:17.314 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:20:17.314 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 6m ago 6h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:20:17.314 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 4m ago 6h 62.7M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:20:17.315 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 6m ago 6h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:20:17.315 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 4m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:20:17.315 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 4m ago 6h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:20:17.315 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 6m ago 6h 856M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:20:17.316 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 6m ago 6h 1523M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:20:17.316 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 4m ago 6h 658M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:20:17.316 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 6m ago 6h 879M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:20:17.316 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 6m ago 6h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:20:17.316 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 4m ago 6h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:20:17.317 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 6m ago 6h 1099M 2384M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:20:17.317 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 6m ago 6h 1087M 2384M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:20:17.317 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 6m ago 6h 827M 2384M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:20:17.317 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 6m ago 6h 1063M 2384M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:20:17.318 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 4m ago 6h 914M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:20:17.318 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 4m ago 6h 1170M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:20:17.318 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 4m ago 6h 1045M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:20:17.318 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 4m ago 6h 1240M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:20:17.319 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 4m ago 6h 166M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:20:17.664 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:20:17.665 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:20:17.665 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:20:17.665 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:20:17.665 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:20:17.666 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:20:17.666 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:20:17.666 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:20:17.666 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:20:17.667 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:20:17.667 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:20:17.667 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:20:17.667 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:20:17.667 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:20:17.668 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:20:17.668 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:20:17.668 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:20:17.668 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:20:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:20:18 smithi153 ceph-mon[70044]: from='client.58435 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:20:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:20:18 smithi153 ceph-mon[70044]: pgmap v12360: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:18.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:20:18 smithi153 ceph-mon[70044]: from='client.58441 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:20:18.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:20:18 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3931824035' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:20:18.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:20:18 smithi083 ceph-mon[79930]: from='client.58435 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:20:18.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:20:18 smithi083 ceph-mon[79930]: pgmap v12360: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:18.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:20:18 smithi083 ceph-mon[79930]: from='client.58441 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:20:18.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:20:18 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3931824035' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:20:18.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:20:18 smithi083 ceph-mon[77242]: from='client.58435 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:20:18.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:20:18 smithi083 ceph-mon[77242]: pgmap v12360: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:18.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:20:18 smithi083 ceph-mon[77242]: from='client.58441 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:20:18.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:20:18 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3931824035' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:20:20.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:20:20 smithi153 ceph-mon[70044]: pgmap v12361: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:20.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:20:20 smithi083 ceph-mon[79930]: pgmap v12361: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:20.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:20:20 smithi083 ceph-mon[77242]: pgmap v12361: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:20:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:20:21] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T12:20:21.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:20:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:20:21.688Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:20:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:20:22 smithi153 ceph-mon[70044]: pgmap v12362: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:22.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:20:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:20:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:20:22.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:20:22 smithi083 ceph-mon[77242]: pgmap v12362: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:22.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:20:22 smithi083 ceph-mon[79930]: pgmap v12362: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:20:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:20:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:20:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:20:24 smithi153 ceph-mon[70044]: pgmap v12363: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:24.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:20:24 smithi083 ceph-mon[79930]: pgmap v12363: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:24.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:20:24 smithi083 ceph-mon[77242]: pgmap v12363: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:25.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:20:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:20:25.062Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:20:26.114 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:20:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:20:25.784Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:20:26.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:20:26 smithi083 ceph-mon[79930]: pgmap v12364: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:26.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:20:26 smithi083 ceph-mon[77242]: pgmap v12364: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:20:26 smithi153 ceph-mon[70044]: pgmap v12364: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:20:28 smithi153 ceph-mon[70044]: pgmap v12365: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:28.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:20:28 smithi083 ceph-mon[79930]: pgmap v12365: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:28.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:20:28 smithi083 ceph-mon[77242]: pgmap v12365: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:20:30 smithi153 ceph-mon[70044]: pgmap v12366: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:30.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:20:30 smithi083 ceph-mon[79930]: pgmap v12366: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:30.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:20:30 smithi083 ceph-mon[77242]: pgmap v12366: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:20:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:20:31] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T12:20:31.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:20:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:20:31.689Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:20:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:20:32 smithi153 ceph-mon[70044]: pgmap v12367: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:32.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:20:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:20:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:20:32.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:20:32 smithi083 ceph-mon[77242]: pgmap v12367: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:32.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:20:32 smithi083 ceph-mon[79930]: pgmap v12367: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:20:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:20:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:20:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:20:34 smithi153 ceph-mon[70044]: pgmap v12368: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:34.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:20:34 smithi083 ceph-mon[79930]: pgmap v12368: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:34.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:20:34 smithi083 ceph-mon[77242]: pgmap v12368: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:35.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:20:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:20:35.062Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:20:36.116 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:20:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:20:35.785Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:20:36.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:20:36 smithi083 ceph-mon[79930]: pgmap v12369: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:36.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:20:36 smithi083 ceph-mon[77242]: pgmap v12369: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:20:36 smithi153 ceph-mon[70044]: pgmap v12369: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:20:38 smithi153 ceph-mon[70044]: pgmap v12370: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:38.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:20:38 smithi083 ceph-mon[79930]: pgmap v12370: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:38.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:20:38 smithi083 ceph-mon[77242]: pgmap v12370: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:20:40 smithi153 ceph-mon[70044]: pgmap v12371: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:20:40 smithi083 ceph-mon[79930]: pgmap v12371: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:20:40 smithi083 ceph-mon[77242]: pgmap v12371: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:20:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:20:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:20:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:20:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:20:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:20:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:20:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:20:41.692 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:20:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:20:41] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T12:20:41.693 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:20:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:20:41.693 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:20:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:20:41.693 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:20:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:20:41.693 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:20:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:20:41.694 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:20:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:20:41.694 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:20:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:20:41.694 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:20:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:20:41.695 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:20:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:20:41.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:20:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:20:41.689Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:20:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:20:42 smithi153 ceph-mon[70044]: pgmap v12372: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:42.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:20:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:20:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:20:42.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:20:42 smithi083 ceph-mon[77242]: pgmap v12372: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:42.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:20:42 smithi083 ceph-mon[79930]: pgmap v12372: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:20:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:20:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:20:44.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:20:44 smithi153 ceph-mon[70044]: pgmap v12373: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:44.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:20:44 smithi083 ceph-mon[79930]: pgmap v12373: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:44.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:20:44 smithi083 ceph-mon[77242]: pgmap v12373: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:45.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:20:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:20:45.063Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:20:46.120 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:20:46 smithi083 ceph-mon[77242]: pgmap v12374: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:46.121 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:20:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:20:45.785Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:20:46.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:20:46 smithi083 ceph-mon[79930]: pgmap v12374: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:46.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:20:46 smithi153 ceph-mon[70044]: pgmap v12374: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:47.985 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:20:48.278 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:20:48.278 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 6m ago 6h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:20:48.278 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 4m ago 6h 62.7M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:20:48.279 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 6m ago 6h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:20:48.279 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 4m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:20:48.279 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 4m ago 6h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:20:48.279 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 6m ago 6h 856M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:20:48.279 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 6m ago 7h 1523M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:20:48.280 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 4m ago 6h 658M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:20:48.280 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 6m ago 6h 879M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:20:48.280 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 6m ago 6h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:20:48.280 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 4m ago 6h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:20:48.280 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 6m ago 6h 1099M 2384M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:20:48.281 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 6m ago 6h 1087M 2384M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:20:48.281 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 6m ago 6h 827M 2384M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:20:48.281 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 6m ago 6h 1063M 2384M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:20:48.281 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 4m ago 6h 914M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:20:48.282 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 4m ago 6h 1170M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:20:48.282 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 4m ago 6h 1045M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:20:48.282 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 4m ago 6h 1240M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:20:48.282 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 4m ago 6h 166M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:20:48.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:20:48 smithi153 ceph-mon[70044]: pgmap v12375: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:48.631 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:20:48.632 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:20:48.632 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:20:48.632 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:20:48.632 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:20:48.633 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:20:48.633 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:20:48.633 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:20:48.633 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:20:48.634 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:20:48.639 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:20:48.639 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:20:48.639 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:20:48.639 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:20:48.640 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:20:48.640 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:20:48.640 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:20:48.640 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:20:48.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:20:48 smithi083 ceph-mon[79930]: pgmap v12375: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:48.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:20:48 smithi083 ceph-mon[77242]: pgmap v12375: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:49.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:20:49 smithi153 ceph-mon[70044]: from='client.48522 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:20:49.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:20:49 smithi153 ceph-mon[70044]: from='client.58459 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:20:49.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:20:49 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2865064911' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:20:49.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:20:49 smithi083 ceph-mon[79930]: from='client.48522 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:20:49.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:20:49 smithi083 ceph-mon[79930]: from='client.58459 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:20:49.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:20:49 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2865064911' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:20:49.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:20:49 smithi083 ceph-mon[77242]: from='client.48522 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:20:49.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:20:49 smithi083 ceph-mon[77242]: from='client.58459 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:20:49.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:20:49 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2865064911' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:20:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:20:50 smithi153 ceph-mon[70044]: pgmap v12376: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:50.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:20:50 smithi083 ceph-mon[79930]: pgmap v12376: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:50.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:20:50 smithi083 ceph-mon[77242]: pgmap v12376: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:20:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:20:51] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T12:20:51.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:20:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:20:51.689Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:20:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:20:52 smithi153 ceph-mon[70044]: pgmap v12377: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:52.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:20:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:20:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:20:52.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:20:52 smithi083 ceph-mon[79930]: pgmap v12377: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:52.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:20:52 smithi083 ceph-mon[77242]: pgmap v12377: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:20:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:20:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:20:54.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:20:54 smithi153 ceph-mon[70044]: pgmap v12378: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:54.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:20:54 smithi083 ceph-mon[79930]: pgmap v12378: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:54.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:20:54 smithi083 ceph-mon[77242]: pgmap v12378: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:20:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:20:55.063Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:20:56.120 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:20:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:20:55.786Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:20:56.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:20:56 smithi083 ceph-mon[79930]: pgmap v12379: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:56.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:20:56 smithi083 ceph-mon[77242]: pgmap v12379: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:56.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:20:56 smithi153 ceph-mon[70044]: pgmap v12379: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:20:58 smithi153 ceph-mon[70044]: pgmap v12380: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:58.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:20:58 smithi083 ceph-mon[79930]: pgmap v12380: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:20:58.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:20:58 smithi083 ceph-mon[77242]: pgmap v12380: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:21:00 smithi153 ceph-mon[70044]: pgmap v12381: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:00.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:21:00 smithi083 ceph-mon[79930]: pgmap v12381: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:00.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:21:00 smithi083 ceph-mon[77242]: pgmap v12381: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:21:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:21:01] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T12:21:01.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:21:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:21:01.690Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:21:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:21:02 smithi153 ceph-mon[70044]: pgmap v12382: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:02.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:21:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:21:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:21:02.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:21:02 smithi083 ceph-mon[79930]: pgmap v12382: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:02.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:21:02 smithi083 ceph-mon[77242]: pgmap v12382: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:21:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:21:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:21:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:21:04 smithi153 ceph-mon[70044]: pgmap v12383: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:04.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:21:04 smithi083 ceph-mon[79930]: pgmap v12383: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:04.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:21:04 smithi083 ceph-mon[77242]: pgmap v12383: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:05.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:21:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:21:05.063Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:21:06.125 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:21:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:21:05.787Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:21:06.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:21:06 smithi083 ceph-mon[79930]: pgmap v12384: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:06.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:21:06 smithi083 ceph-mon[77242]: pgmap v12384: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:06.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:21:06 smithi153 ceph-mon[70044]: pgmap v12384: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:21:08 smithi153 ceph-mon[70044]: pgmap v12385: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:08.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:21:08 smithi083 ceph-mon[79930]: pgmap v12385: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:08.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:21:08 smithi083 ceph-mon[77242]: pgmap v12385: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:21:10 smithi153 ceph-mon[70044]: pgmap v12386: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:10.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:21:10 smithi083 ceph-mon[79930]: pgmap v12386: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:10.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:21:10 smithi083 ceph-mon[77242]: pgmap v12386: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:11.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:21:11 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:21:11.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:21:11 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:21:11.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:21:11 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:21:11.693 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:21:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:21:11] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T12:21:11.694 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:21:11 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:21:11.694 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:21:11 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:21:11.694 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:21:11 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:21:11.694 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:21:11 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:21:11.695 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:21:11 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:21:11.695 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:21:11 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:21:11.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:21:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:21:11.690Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:21:12.665 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:21:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:21:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:21:12.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:21:12 smithi083 ceph-mon[77242]: pgmap v12387: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:12.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:21:12 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:21:12.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:21:12 smithi083 ceph-mon[79930]: pgmap v12387: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:12.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:21:12 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:21:13.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:21:12 smithi153 ceph-mon[70044]: pgmap v12387: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:13.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:21:12 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:21:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:21:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:21:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:21:14.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:21:14 smithi083 ceph-mon[79930]: pgmap v12388: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:14.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:21:14 smithi083 ceph-mon[77242]: pgmap v12388: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:15.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:21:14 smithi153 ceph-mon[70044]: pgmap v12388: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:15.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:21:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:21:15.064Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:21:16.126 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:21:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:21:15.787Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:21:16.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:21:16 smithi083 ceph-mon[79930]: pgmap v12389: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:16.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:21:16 smithi083 ceph-mon[77242]: pgmap v12389: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:21:16 smithi153 ceph-mon[70044]: pgmap v12389: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:21:18 smithi153 ceph-mon[70044]: pgmap v12390: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:18.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:21:18 smithi083 ceph-mon[79930]: pgmap v12390: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:18.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:21:18 smithi083 ceph-mon[77242]: pgmap v12390: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:18.962 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:21:19.251 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:21:19.252 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 7m ago 6h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:21:19.252 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 5m ago 6h 62.7M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:21:19.252 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 7m ago 6h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:21:19.253 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 5m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:21:19.253 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 5m ago 6h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:21:19.253 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 7m ago 7h 856M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:21:19.253 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 7m ago 7h 1523M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:21:19.254 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 5m ago 6h 658M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:21:19.254 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 7m ago 6h 879M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:21:19.254 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 7m ago 6h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:21:19.254 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 5m ago 6h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:21:19.255 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 7m ago 6h 1099M 2384M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:21:19.255 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 7m ago 6h 1087M 2384M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:21:19.255 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 7m ago 6h 827M 2384M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:21:19.255 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 7m ago 6h 1063M 2384M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:21:19.256 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 5m ago 6h 914M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:21:19.256 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 5m ago 6h 1170M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:21:19.256 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 5m ago 6h 1045M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:21:19.256 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 5m ago 6h 1240M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:21:19.257 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 5m ago 6h 166M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:21:19.607 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:21:19.607 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:21:19.608 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:21:19.608 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:21:19.608 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:21:19.608 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:21:19.609 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:21:19.609 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:21:19.610 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:21:19.610 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:21:19.611 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:21:19.611 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:21:19.611 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:21:19.612 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:21:19.612 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:21:19.613 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:21:19.613 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:21:19.613 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:21:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:21:20 smithi153 ceph-mon[70044]: from='client.58471 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:21:20.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:21:20 smithi153 ceph-mon[70044]: pgmap v12391: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:20.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:21:20 smithi153 ceph-mon[70044]: from='client.48546 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:21:20.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:21:20 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3882538470' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:21:20.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:21:20 smithi083 ceph-mon[79930]: from='client.58471 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:21:20.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:21:20 smithi083 ceph-mon[79930]: pgmap v12391: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:20.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:21:20 smithi083 ceph-mon[79930]: from='client.48546 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:21:20.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:21:20 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3882538470' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:21:20.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:21:20 smithi083 ceph-mon[77242]: from='client.58471 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:21:20.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:21:20 smithi083 ceph-mon[77242]: pgmap v12391: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:20.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:21:20 smithi083 ceph-mon[77242]: from='client.48546 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:21:20.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:21:20 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3882538470' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:21:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:21:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:21:21] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T12:21:21.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:21:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:21:21.691Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:21:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:21:22 smithi153 ceph-mon[70044]: pgmap v12392: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:22.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:21:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:21:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:21:22.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:21:22 smithi083 ceph-mon[77242]: pgmap v12392: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:22.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:21:22 smithi083 ceph-mon[79930]: pgmap v12392: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:21:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:21:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:21:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:21:24 smithi153 ceph-mon[70044]: pgmap v12393: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:24.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:21:24 smithi083 ceph-mon[79930]: pgmap v12393: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:24.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:21:24 smithi083 ceph-mon[77242]: pgmap v12393: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:25.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:21:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:21:25.065Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:21:26.130 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:21:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:21:25.788Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:21:26.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:21:26 smithi083 ceph-mon[79930]: pgmap v12394: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:26.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:21:26 smithi083 ceph-mon[77242]: pgmap v12394: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:21:26 smithi153 ceph-mon[70044]: pgmap v12394: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:21:28 smithi153 ceph-mon[70044]: pgmap v12395: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:28.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:21:28 smithi083 ceph-mon[79930]: pgmap v12395: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:28.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:21:28 smithi083 ceph-mon[77242]: pgmap v12395: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:21:30 smithi153 ceph-mon[70044]: pgmap v12396: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:30.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:21:30 smithi083 ceph-mon[79930]: pgmap v12396: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:30.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:21:30 smithi083 ceph-mon[77242]: pgmap v12396: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:21:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:21:31] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T12:21:31.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:21:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:21:31.692Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:21:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:21:32 smithi153 ceph-mon[70044]: pgmap v12397: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:32.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:21:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:21:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:21:32.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:21:32 smithi083 ceph-mon[79930]: pgmap v12397: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:32.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:21:32 smithi083 ceph-mon[77242]: pgmap v12397: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:21:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:21:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:21:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:21:34 smithi153 ceph-mon[70044]: pgmap v12398: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:34.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:21:34 smithi083 ceph-mon[79930]: pgmap v12398: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:34.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:21:34 smithi083 ceph-mon[77242]: pgmap v12398: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:35.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:21:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:21:35.065Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:21:36.133 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:21:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:21:35.789Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:21:36.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:21:36 smithi083 ceph-mon[79930]: pgmap v12399: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:36.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:21:36 smithi083 ceph-mon[77242]: pgmap v12399: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:21:36 smithi153 ceph-mon[70044]: pgmap v12399: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:21:38 smithi153 ceph-mon[70044]: pgmap v12400: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:38.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:21:38 smithi083 ceph-mon[79930]: pgmap v12400: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:38.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:21:38 smithi083 ceph-mon[77242]: pgmap v12400: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:21:40 smithi153 ceph-mon[70044]: pgmap v12401: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:21:40 smithi083 ceph-mon[79930]: pgmap v12401: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:21:40 smithi083 ceph-mon[77242]: pgmap v12401: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:21:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:21:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:21:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:21:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:21:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:21:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:21:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:21:41.695 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:21:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:21:41] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T12:21:41.696 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:21:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:21:41.696 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:21:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:21:41.696 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:21:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:21:41.697 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:21:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:21:41.697 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:21:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:21:41.697 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:21:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:21:41.697 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:21:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:21:41.698 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:21:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:21:41.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:21:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:21:41.693Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:21:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:21:42 smithi153 ceph-mon[70044]: pgmap v12402: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:42.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:21:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:21:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:21:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:21:42 smithi083 ceph-mon[79930]: pgmap v12402: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:21:42 smithi083 ceph-mon[77242]: pgmap v12402: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:21:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:21:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:21:44.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:21:44 smithi153 ceph-mon[70044]: pgmap v12403: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:44.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:21:44 smithi083 ceph-mon[77242]: pgmap v12403: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:44.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:21:44 smithi083 ceph-mon[79930]: pgmap v12403: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:45.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:21:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:21:45.066Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:21:46.133 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:21:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:21:45.789Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:21:46.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:21:46 smithi083 ceph-mon[77242]: pgmap v12404: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:46.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:21:46 smithi083 ceph-mon[79930]: pgmap v12404: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:46.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:21:46 smithi153 ceph-mon[70044]: pgmap v12404: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:48.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:21:48 smithi153 ceph-mon[70044]: pgmap v12405: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:48.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:21:48 smithi083 ceph-mon[79930]: pgmap v12405: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:48.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:21:48 smithi083 ceph-mon[77242]: pgmap v12405: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:49.933 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:21:50.224 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:21:50.225 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 7m ago 6h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:21:50.225 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 5m ago 6h 62.7M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:21:50.225 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 7m ago 6h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:21:50.225 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 5m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:21:50.226 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 5m ago 6h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:21:50.226 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 7m ago 7h 856M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:21:50.226 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 7m ago 7h 1523M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:21:50.226 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 5m ago 6h 658M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:21:50.227 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 7m ago 6h 879M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:21:50.227 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 7m ago 6h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:21:50.227 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 5m ago 6h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:21:50.227 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 7m ago 6h 1099M 2384M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:21:50.227 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 7m ago 6h 1087M 2384M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:21:50.228 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 7m ago 6h 827M 2384M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:21:50.228 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 7m ago 6h 1063M 2384M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:21:50.228 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 5m ago 6h 914M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:21:50.228 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 5m ago 6h 1170M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:21:50.228 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 5m ago 6h 1045M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:21:50.229 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 5m ago 6h 1240M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:21:50.229 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 5m ago 6h 166M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:21:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:21:50 smithi153 ceph-mon[70044]: pgmap v12406: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:50.578 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:21:50.579 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:21:50.579 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:21:50.579 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:21:50.579 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:21:50.580 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:21:50.580 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:21:50.580 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:21:50.580 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:21:50.580 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:21:50.581 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:21:50.581 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:21:50.581 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:21:50.581 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:21:50.581 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:21:50.582 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:21:50.582 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:21:50.582 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:21:50.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:21:50 smithi083 ceph-mon[77242]: pgmap v12406: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:50.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:21:50 smithi083 ceph-mon[79930]: pgmap v12406: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:51.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:21:51 smithi153 ceph-mon[70044]: from='client.58489 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:21:51.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:21:51 smithi153 ceph-mon[70044]: from='client.58495 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:21:51.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:21:51 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4194665219' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:21:51.696 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:21:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:21:51] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T12:21:51.697 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:21:51 smithi083 ceph-mon[77242]: from='client.58489 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:21:51.697 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:21:51 smithi083 ceph-mon[77242]: from='client.58495 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:21:51.697 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:21:51 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4194665219' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:21:51.698 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:21:51 smithi083 ceph-mon[79930]: from='client.58489 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:21:51.698 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:21:51 smithi083 ceph-mon[79930]: from='client.58495 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:21:51.698 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:21:51 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4194665219' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:21:51.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:21:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:21:51.694Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:21:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:21:52 smithi153 ceph-mon[70044]: pgmap v12407: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:52.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:21:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:21:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:21:52.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:21:52 smithi083 ceph-mon[77242]: pgmap v12407: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:52.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:21:52 smithi083 ceph-mon[79930]: pgmap v12407: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:21:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:21:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:21:54.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:21:54 smithi153 ceph-mon[70044]: pgmap v12408: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:54.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:21:54 smithi083 ceph-mon[79930]: pgmap v12408: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:54.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:21:54 smithi083 ceph-mon[77242]: pgmap v12408: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:21:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:21:55.067Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:21:56.136 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:21:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:21:55.789Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:21:56.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:21:56 smithi083 ceph-mon[79930]: pgmap v12409: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:56.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:21:56 smithi083 ceph-mon[77242]: pgmap v12409: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:56.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:21:56 smithi153 ceph-mon[70044]: pgmap v12409: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:21:58 smithi153 ceph-mon[70044]: pgmap v12410: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:58.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:21:58 smithi083 ceph-mon[79930]: pgmap v12410: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:21:58.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:21:58 smithi083 ceph-mon[77242]: pgmap v12410: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:22:00 smithi153 ceph-mon[70044]: pgmap v12411: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:00.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:22:00 smithi083 ceph-mon[79930]: pgmap v12411: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:00.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:22:00 smithi083 ceph-mon[77242]: pgmap v12411: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:22:01] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T12:22:01.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:22:01.694Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:22:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:22:02 smithi153 ceph-mon[70044]: pgmap v12412: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:02.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:22:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:22:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:22:02.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:22:02 smithi083 ceph-mon[79930]: pgmap v12412: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:02.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:22:02 smithi083 ceph-mon[77242]: pgmap v12412: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:22:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:22:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:22:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:22:04 smithi153 ceph-mon[70044]: pgmap v12413: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:04.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:22:04 smithi083 ceph-mon[79930]: pgmap v12413: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:04.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:22:04 smithi083 ceph-mon[77242]: pgmap v12413: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:05.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:22:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:22:05.067Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:22:06.137 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:22:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:22:05.790Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:22:06.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:22:06 smithi083 ceph-mon[79930]: pgmap v12414: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:06.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:22:06 smithi083 ceph-mon[77242]: pgmap v12414: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:06.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:22:06 smithi153 ceph-mon[70044]: pgmap v12414: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:22:08 smithi153 ceph-mon[70044]: pgmap v12415: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:08.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:22:08 smithi083 ceph-mon[79930]: pgmap v12415: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:08.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:22:08 smithi083 ceph-mon[77242]: pgmap v12415: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:22:10 smithi153 ceph-mon[70044]: pgmap v12416: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:10.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:22:10 smithi083 ceph-mon[79930]: pgmap v12416: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:10.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:22:10 smithi083 ceph-mon[77242]: pgmap v12416: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:22:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:22:11] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T12:22:11.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:22:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:22:11.695Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:22:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:22:12 smithi153 ceph-mon[70044]: pgmap v12417: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:12.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:22:12 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:22:12.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:22:12 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:22:12.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:22:12 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:22:12.559 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:22:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:22:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:22:12.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:22:12 smithi083 ceph-mon[77242]: pgmap v12417: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:12.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:22:12 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:22:12.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:22:12 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:22:12.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:22:12 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:22:12.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:22:12 smithi083 ceph-mon[79930]: pgmap v12417: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:12.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:22:12 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:22:12.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:22:12 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:22:12.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:22:12 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:22:13.297 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:22:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:22:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:22:13.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:22:13 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:22:13.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:22:13 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:22:13.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:22:13 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:22:13.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:22:13 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:22:13.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:22:13 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:22:13.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:22:13 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:22:13.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:22:13 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:22:13.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:22:13 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:22:13.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:22:13 smithi153 ceph-mon[70044]: Adjusting osd_memory_target on smithi083 to 2375M 2022-09-29T12:22:13.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:22:13 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:22:13.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:22:13 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:22:13.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:22:13 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:22:13.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:22:13 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:22:13.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:22:13 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:22:13.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:22:13 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:22:13.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:22:13 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:22:13.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:22:13 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:22:13.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:22:13 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:22:13.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:22:13 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:22:13.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:22:13 smithi083 ceph-mon[79930]: Adjusting osd_memory_target on smithi083 to 2375M 2022-09-29T12:22:13.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:22:13 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:22:13.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:22:13 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:22:13.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:22:13 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:22:13.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:22:13 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:22:13.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:22:13 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:22:13.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:22:13 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:22:13.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:22:13 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:22:13.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:22:13 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:22:13.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:22:13 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:22:13.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:22:13 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:22:13.718 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:22:13 smithi083 ceph-mon[77242]: Adjusting osd_memory_target on smithi083 to 2375M 2022-09-29T12:22:13.718 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:22:13 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:22:13.718 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:22:13 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:22:14.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:22:14 smithi083 ceph-mon[79930]: pgmap v12418: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:14.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:22:14 smithi083 ceph-mon[77242]: pgmap v12418: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:14.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:22:14 smithi153 ceph-mon[70044]: pgmap v12418: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:15.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:22:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:22:15.069Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:22:16.141 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:22:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:22:15.791Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:22:16.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:22:16 smithi083 ceph-mon[79930]: pgmap v12419: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:16.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:22:16 smithi083 ceph-mon[77242]: pgmap v12419: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:22:16 smithi153 ceph-mon[70044]: pgmap v12419: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:22:18 smithi153 ceph-mon[70044]: pgmap v12420: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:18.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:22:18 smithi083 ceph-mon[79930]: pgmap v12420: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:18.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:22:18 smithi083 ceph-mon[77242]: pgmap v12420: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:22:20 smithi153 ceph-mon[70044]: pgmap v12421: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:20.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:22:20 smithi083 ceph-mon[79930]: pgmap v12421: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:20.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:22:20 smithi083 ceph-mon[77242]: pgmap v12421: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:20.903 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:22:21.190 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:22:21.190 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 8m ago 6h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:22:21.191 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 6m ago 6h 62.7M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:22:21.191 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 8m ago 6h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:22:21.191 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 6m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:22:21.191 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 6m ago 6h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:22:21.192 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 8m ago 7h 856M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:22:21.192 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 8m ago 7h 1523M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:22:21.192 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 6m ago 7h 658M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:22:21.192 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 8m ago 7h 879M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:22:21.193 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 8m ago 6h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:22:21.193 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 6m ago 6h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:22:21.193 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 8m ago 6h 1099M 2375M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:22:21.193 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 8m ago 6h 1087M 2375M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:22:21.193 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 8m ago 6h 827M 2375M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:22:21.194 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 8m ago 6h 1063M 2375M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:22:21.194 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 6m ago 6h 914M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:22:21.194 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 6m ago 6h 1170M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:22:21.194 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 6m ago 6h 1045M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:22:21.195 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 6m ago 6h 1240M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:22:21.195 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 6m ago 6h 166M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:22:21.548 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:22:21.548 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:22:21.549 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:22:21.549 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:22:21.549 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:22:21.549 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:22:21.550 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:22:21.550 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:22:21.550 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:22:21.550 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:22:21.551 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:22:21.551 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:22:21.551 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:22:21.551 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:22:21.551 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:22:21.552 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:22:21.552 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:22:21.552 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:22:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:22:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:22:21] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T12:22:21.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:22:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:22:21.695Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:22:22.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:22:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:22:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:22:22.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:22:22 smithi153 ceph-mon[70044]: from='client.58507 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:22:22.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:22:22 smithi153 ceph-mon[70044]: from='client.58513 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:22:22.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:22:22 smithi153 ceph-mon[70044]: pgmap v12422: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:22.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:22:22 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1421033449' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:22:22.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:22:22 smithi083 ceph-mon[77242]: from='client.58507 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:22:22.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:22:22 smithi083 ceph-mon[77242]: from='client.58513 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:22:22.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:22:22 smithi083 ceph-mon[77242]: pgmap v12422: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:22.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:22:22 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1421033449' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:22:22.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:22:22 smithi083 ceph-mon[79930]: from='client.58507 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:22:22.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:22:22 smithi083 ceph-mon[79930]: from='client.58513 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:22:22.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:22:22 smithi083 ceph-mon[79930]: pgmap v12422: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:22.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:22:22 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1421033449' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:22:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:22:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:22:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:22:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:22:24 smithi153 ceph-mon[70044]: pgmap v12423: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:24.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:22:24 smithi083 ceph-mon[77242]: pgmap v12423: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:24.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:22:24 smithi083 ceph-mon[79930]: pgmap v12423: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:25.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:22:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:22:25.069Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:22:26.143 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:22:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:22:25.792Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:22:26.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:22:26 smithi083 ceph-mon[79930]: pgmap v12424: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:26.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:22:26 smithi083 ceph-mon[77242]: pgmap v12424: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:22:26 smithi153 ceph-mon[70044]: pgmap v12424: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:22:28 smithi153 ceph-mon[70044]: pgmap v12425: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:28.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:22:28 smithi083 ceph-mon[79930]: pgmap v12425: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:28.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:22:28 smithi083 ceph-mon[77242]: pgmap v12425: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:22:30 smithi153 ceph-mon[70044]: pgmap v12426: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:30.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:22:30 smithi083 ceph-mon[79930]: pgmap v12426: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:30.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:22:30 smithi083 ceph-mon[77242]: pgmap v12426: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:22:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:22:31] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T12:22:31.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:22:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:22:31.697Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:22:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:22:32 smithi153 ceph-mon[70044]: pgmap v12427: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:32.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:22:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:22:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:22:32.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:22:32 smithi083 ceph-mon[79930]: pgmap v12427: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:32.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:22:32 smithi083 ceph-mon[77242]: pgmap v12427: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:22:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:22:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:22:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:22:34 smithi153 ceph-mon[70044]: pgmap v12428: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:34.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:22:34 smithi083 ceph-mon[79930]: pgmap v12428: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:34.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:22:34 smithi083 ceph-mon[77242]: pgmap v12428: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:35.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:22:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:22:35.070Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:22:36.143 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:22:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:22:35.792Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:22:36.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:22:36 smithi083 ceph-mon[79930]: pgmap v12429: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:36.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:22:36 smithi083 ceph-mon[77242]: pgmap v12429: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:22:36 smithi153 ceph-mon[70044]: pgmap v12429: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:22:38 smithi153 ceph-mon[70044]: pgmap v12430: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:38.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:22:38 smithi083 ceph-mon[79930]: pgmap v12430: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:38.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:22:38 smithi083 ceph-mon[77242]: pgmap v12430: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:22:40 smithi153 ceph-mon[70044]: pgmap v12431: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:22:40 smithi083 ceph-mon[77242]: pgmap v12431: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:22:40 smithi083 ceph-mon[79930]: pgmap v12431: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:41.700 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:22:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:22:41] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T12:22:41.701 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:22:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:22:41.702 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:22:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:22:41.702 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:22:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:22:41.703 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:22:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:22:41.703 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:22:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:22:41.704 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:22:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:22:41.704 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:22:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:22:41.705 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:22:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:22:41.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:22:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:22:41.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:22:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:22:41.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:22:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:22:41.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:22:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:22:41.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:22:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:22:41.697Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:22:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:22:42 smithi083 ceph-mon[79930]: pgmap v12432: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:22:42 smithi083 ceph-mon[77242]: pgmap v12432: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:22:42 smithi153 ceph-mon[70044]: pgmap v12432: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:22:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:22:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:22:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:22:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:22:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:22:44.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:22:44 smithi083 ceph-mon[79930]: pgmap v12433: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:44.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:22:44 smithi083 ceph-mon[77242]: pgmap v12433: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:44.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:22:44 smithi153 ceph-mon[70044]: pgmap v12433: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:45.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:22:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:22:45.071Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:22:46.147 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:22:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:22:45.792Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:22:46.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:22:46 smithi083 ceph-mon[79930]: pgmap v12434: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:46.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:22:46 smithi083 ceph-mon[77242]: pgmap v12434: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:46.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:22:46 smithi153 ceph-mon[70044]: pgmap v12434: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:48.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:22:48 smithi153 ceph-mon[70044]: pgmap v12435: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:48.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:22:48 smithi083 ceph-mon[79930]: pgmap v12435: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:48.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:22:48 smithi083 ceph-mon[77242]: pgmap v12435: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:22:50 smithi153 ceph-mon[70044]: pgmap v12436: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:50.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:22:50 smithi083 ceph-mon[79930]: pgmap v12436: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:50.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:22:50 smithi083 ceph-mon[77242]: pgmap v12436: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:51.877 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:22:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:22:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:22:51] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T12:22:51.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:22:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:22:51.698Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:22:52.162 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:22:52.163 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 8m ago 6h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:22:52.163 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 6m ago 6h 62.7M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:22:52.163 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 8m ago 6h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:22:52.163 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 6m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:22:52.164 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 6m ago 7h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:22:52.164 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 8m ago 7h 856M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:22:52.164 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 8m ago 7h 1523M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:22:52.164 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 6m ago 7h 658M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:22:52.164 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 8m ago 7h 879M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:22:52.165 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 8m ago 6h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:22:52.165 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 6m ago 6h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:22:52.165 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 8m ago 7h 1099M 2375M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:22:52.165 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 8m ago 6h 1087M 2375M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:22:52.166 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 8m ago 6h 827M 2375M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:22:52.166 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 8m ago 6h 1063M 2375M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:22:52.166 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 6m ago 6h 914M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:22:52.166 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 6m ago 6h 1170M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:22:52.166 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 6m ago 6h 1045M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:22:52.167 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 6m ago 6h 1240M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:22:52.167 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 6m ago 6h 166M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:22:52.520 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:22:52.521 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:22:52.521 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:22:52.521 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:22:52.522 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:22:52.522 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:22:52.522 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:22:52.522 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:22:52.523 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:22:52.523 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:22:52.523 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:22:52.523 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:22:52.523 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:22:52.524 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:22:52.524 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:22:52.524 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:22:52.524 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:22:52.525 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:22:52.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:22:52 smithi083 ceph-mon[79930]: pgmap v12437: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:52.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:22:52 smithi083 ceph-mon[77242]: pgmap v12437: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:52.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:22:52 smithi153 ceph-mon[70044]: pgmap v12437: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:22:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:22:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:22:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:22:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:22:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:22:53.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:22:53 smithi083 ceph-mon[79930]: from='client.58525 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:22:53.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:22:53 smithi083 ceph-mon[79930]: from='client.48600 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:22:53.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:22:53 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1656752604' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:22:53.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:22:53 smithi083 ceph-mon[77242]: from='client.58525 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:22:53.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:22:53 smithi083 ceph-mon[77242]: from='client.48600 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:22:53.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:22:53 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1656752604' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:22:53.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:22:53 smithi153 ceph-mon[70044]: from='client.58525 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:22:53.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:22:53 smithi153 ceph-mon[70044]: from='client.48600 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:22:53.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:22:53 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1656752604' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:22:54.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:22:54 smithi083 ceph-mon[77242]: pgmap v12438: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:54.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:22:54 smithi083 ceph-mon[79930]: pgmap v12438: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:54.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:22:54 smithi153 ceph-mon[70044]: pgmap v12438: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:22:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:22:55.072Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:22:56.147 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:22:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:22:55.793Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:22:56.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:22:56 smithi083 ceph-mon[79930]: pgmap v12439: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:56.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:22:56 smithi083 ceph-mon[77242]: pgmap v12439: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:56.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:22:56 smithi153 ceph-mon[70044]: pgmap v12439: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:22:58 smithi153 ceph-mon[70044]: pgmap v12440: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:58.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:22:58 smithi083 ceph-mon[77242]: pgmap v12440: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:22:58.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:22:58 smithi083 ceph-mon[79930]: pgmap v12440: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:00.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:23:00 smithi083 ceph-mon[79930]: pgmap v12441: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:00.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:23:00 smithi083 ceph-mon[77242]: pgmap v12441: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:00.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:23:00 smithi153 ceph-mon[70044]: pgmap v12441: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:23:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:23:01] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T12:23:01.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:23:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:23:01.698Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:23:02.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:23:02 smithi083 ceph-mon[79930]: pgmap v12442: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:02.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:23:02 smithi083 ceph-mon[77242]: pgmap v12442: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:02.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:23:02 smithi153 ceph-mon[70044]: pgmap v12442: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:23:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:23:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:23:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:23:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:23:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:23:04.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:23:04 smithi083 ceph-mon[79930]: pgmap v12443: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:04.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:23:04 smithi083 ceph-mon[77242]: pgmap v12443: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:04.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:23:04 smithi153 ceph-mon[70044]: pgmap v12443: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:05.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:23:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:23:05.073Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:23:06.150 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:23:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:23:05.794Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:23:06.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:23:06 smithi083 ceph-mon[77242]: pgmap v12444: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:06.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:23:06 smithi083 ceph-mon[79930]: pgmap v12444: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:06.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:23:06 smithi153 ceph-mon[70044]: pgmap v12444: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:08.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:23:08 smithi083 ceph-mon[79930]: pgmap v12445: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:08.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:23:08 smithi083 ceph-mon[77242]: pgmap v12445: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:08.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:23:08 smithi153 ceph-mon[70044]: pgmap v12445: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:10.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:23:10 smithi083 ceph-mon[77242]: pgmap v12446: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:10.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:23:10 smithi083 ceph-mon[79930]: pgmap v12446: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:10.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:23:10 smithi153 ceph-mon[70044]: pgmap v12446: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:23:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:23:11] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T12:23:11.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:23:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:23:11.699Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:23:12.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:23:12 smithi083 ceph-mon[79930]: pgmap v12447: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:12.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:23:12 smithi083 ceph-mon[77242]: pgmap v12447: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:12.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:23:12 smithi153 ceph-mon[70044]: pgmap v12447: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:23:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:23:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:23:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:23:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:23:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:23:13.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:23:13 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:23:13.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:23:13 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:23:13.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:23:13 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:23:13.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:23:13 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:23:13.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:23:13 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:23:13.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:23:13 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:23:13.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:23:13 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:23:13.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:23:13 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:23:13.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:23:13 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:23:13.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:23:13 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:23:13.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:23:13 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:23:13.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:23:13 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:23:14.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:23:14 smithi083 ceph-mon[79930]: pgmap v12448: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:14.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:23:14 smithi083 ceph-mon[77242]: pgmap v12448: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:14.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:23:14 smithi153 ceph-mon[70044]: pgmap v12448: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:15.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:23:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:23:15.073Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:23:16.152 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:23:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:23:15.794Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:23:16.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:23:16 smithi083 ceph-mon[79930]: pgmap v12449: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:16.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:23:16 smithi083 ceph-mon[77242]: pgmap v12449: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:23:16 smithi153 ceph-mon[70044]: pgmap v12449: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:18.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:23:18 smithi083 ceph-mon[77242]: pgmap v12450: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:18.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:23:18 smithi083 ceph-mon[79930]: pgmap v12450: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:18.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:23:18 smithi153 ceph-mon[70044]: pgmap v12450: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:20.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:23:20 smithi083 ceph-mon[79930]: pgmap v12451: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:20.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:23:20 smithi083 ceph-mon[77242]: pgmap v12451: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:20.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:23:20 smithi153 ceph-mon[70044]: pgmap v12451: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:23:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:23:21] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T12:23:21.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:23:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:23:21.700Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:23:22.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:23:22 smithi083 ceph-mon[79930]: pgmap v12452: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:22.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:23:22 smithi083 ceph-mon[77242]: pgmap v12452: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:22.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:23:22 smithi153 ceph-mon[70044]: pgmap v12452: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:23:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:23:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:23:22.847 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:23:23.136 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:23:23.137 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 9m ago 6h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:23:23.137 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 7m ago 6h 62.7M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:23:23.137 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 9m ago 6h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:23:23.137 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 7m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:23:23.137 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 7m ago 7h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:23:23.138 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 9m ago 7h 856M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:23:23.138 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 9m ago 7h 1523M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:23:23.138 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 7m ago 7h 658M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:23:23.138 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 9m ago 7h 879M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:23:23.139 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 9m ago 6h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:23:23.139 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 7m ago 6h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:23:23.139 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 9m ago 7h 1099M 2375M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:23:23.139 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 9m ago 7h 1087M 2375M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:23:23.139 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 9m ago 7h 827M 2375M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:23:23.140 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 9m ago 6h 1063M 2375M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:23:23.140 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 7m ago 6h 914M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:23:23.140 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 7m ago 6h 1170M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:23:23.140 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 7m ago 6h 1045M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:23:23.140 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 7m ago 6h 1240M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:23:23.141 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 7m ago 6h 166M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:23:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:23:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:23:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:23:23.497 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:23:23.498 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:23:23.498 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:23:23.498 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:23:23.499 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:23:23.499 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:23:23.499 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:23:23.499 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:23:23.499 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:23:23.500 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:23:23.500 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:23:23.500 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:23:23.500 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:23:23.501 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:23:23.501 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:23:23.501 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:23:23.501 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:23:23.502 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:23:24.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:23:24 smithi083 ceph-mon[77242]: from='client.48612 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:23:24.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:23:24 smithi083 ceph-mon[77242]: from='client.48618 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:23:24.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:23:24 smithi083 ceph-mon[77242]: pgmap v12453: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:24.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:23:24 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4199325523' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:23:24.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:23:24 smithi083 ceph-mon[79930]: from='client.48612 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:23:24.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:23:24 smithi083 ceph-mon[79930]: from='client.48618 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:23:24.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:23:24 smithi083 ceph-mon[79930]: pgmap v12453: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:24.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:23:24 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4199325523' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:23:24.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:23:24 smithi153 ceph-mon[70044]: from='client.48612 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:23:24.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:23:24 smithi153 ceph-mon[70044]: from='client.48618 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:23:24.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:23:24 smithi153 ceph-mon[70044]: pgmap v12453: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:24.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:23:24 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4199325523' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:23:25.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:23:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:23:25.074Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:23:26.154 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:23:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:23:25.795Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:23:26.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:23:26 smithi083 ceph-mon[79930]: pgmap v12454: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:26.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:23:26 smithi083 ceph-mon[77242]: pgmap v12454: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:23:26 smithi153 ceph-mon[70044]: pgmap v12454: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:28.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:23:28 smithi083 ceph-mon[79930]: pgmap v12455: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:28.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:23:28 smithi083 ceph-mon[77242]: pgmap v12455: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:28.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:23:28 smithi153 ceph-mon[70044]: pgmap v12455: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:30.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:23:30 smithi083 ceph-mon[79930]: pgmap v12456: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:30.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:23:30 smithi083 ceph-mon[77242]: pgmap v12456: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:30.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:23:30 smithi153 ceph-mon[70044]: pgmap v12456: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:23:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:23:31] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T12:23:31.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:23:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:23:31.700Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:23:32.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:23:32 smithi083 ceph-mon[79930]: pgmap v12457: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:32.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:23:32 smithi083 ceph-mon[77242]: pgmap v12457: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:32.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:23:32 smithi153 ceph-mon[70044]: pgmap v12457: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:23:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:23:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:23:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:23:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:23:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:23:34.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:23:34 smithi083 ceph-mon[79930]: pgmap v12458: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:34.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:23:34 smithi083 ceph-mon[77242]: pgmap v12458: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:34.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:23:34 smithi153 ceph-mon[70044]: pgmap v12458: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:35.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:23:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:23:35.074Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:23:36.157 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:23:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:23:35.796Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:23:36.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:23:36 smithi083 ceph-mon[77242]: pgmap v12459: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:36.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:23:36 smithi083 ceph-mon[79930]: pgmap v12459: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:23:36 smithi153 ceph-mon[70044]: pgmap v12459: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:38.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:23:38 smithi083 ceph-mon[77242]: pgmap v12460: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:38.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:23:38 smithi083 ceph-mon[79930]: pgmap v12460: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:38.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:23:38 smithi153 ceph-mon[70044]: pgmap v12460: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:23:40 smithi083 ceph-mon[79930]: pgmap v12461: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:23:40 smithi083 ceph-mon[77242]: pgmap v12461: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:23:40 smithi153 ceph-mon[70044]: pgmap v12461: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:41.703 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:23:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:23:41] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T12:23:41.704 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:23:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:23:41.704 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:23:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:23:41.704 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:23:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:23:41.705 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:23:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:23:41.705 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:23:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:23:41.705 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:23:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:23:41.705 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:23:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:23:41.706 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:23:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:23:41.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:23:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:23:41.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:23:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:23:41.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:23:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:23:41.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:23:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:23:41.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:23:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:23:41.701Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:23:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:23:42 smithi083 ceph-mon[79930]: pgmap v12462: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:23:42 smithi083 ceph-mon[77242]: pgmap v12462: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:23:42 smithi153 ceph-mon[70044]: pgmap v12462: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:23:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:23:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:23:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:23:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:23:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:23:44.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:23:44 smithi083 ceph-mon[79930]: pgmap v12463: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:44.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:23:44 smithi083 ceph-mon[77242]: pgmap v12463: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:44.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:23:44 smithi153 ceph-mon[70044]: pgmap v12463: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:45.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:23:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:23:45.075Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:23:46.158 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:23:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:23:45.796Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:23:46.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:23:46 smithi083 ceph-mon[77242]: pgmap v12464: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:46.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:23:46 smithi083 ceph-mon[79930]: pgmap v12464: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:46.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:23:46 smithi153 ceph-mon[70044]: pgmap v12464: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:48.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:23:48 smithi083 ceph-mon[79930]: pgmap v12465: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:48.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:23:48 smithi083 ceph-mon[77242]: pgmap v12465: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:48.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:23:48 smithi153 ceph-mon[70044]: pgmap v12465: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:50.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:23:50 smithi083 ceph-mon[79930]: pgmap v12466: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:50.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:23:50 smithi083 ceph-mon[77242]: pgmap v12466: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:50.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:23:50 smithi153 ceph-mon[70044]: pgmap v12466: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:23:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:23:51] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T12:23:51.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:23:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:23:51.702Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:23:52.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:23:52 smithi083 ceph-mon[79930]: pgmap v12467: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:52.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:23:52 smithi083 ceph-mon[77242]: pgmap v12467: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:52.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:23:52 smithi153 ceph-mon[70044]: pgmap v12467: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:23:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:23:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:23:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:23:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:23:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:23:53.824 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:23:54.113 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:23:54.113 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 9m ago 6h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:23:54.113 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 7m ago 6h 62.7M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:23:54.114 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 9m ago 6h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:23:54.114 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 7m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:23:54.114 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 7m ago 7h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:23:54.114 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 9m ago 7h 856M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:23:54.115 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 9m ago 7h 1523M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:23:54.115 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 7m ago 7h 658M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:23:54.115 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 9m ago 7h 879M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:23:54.115 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 9m ago 6h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:23:54.115 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 7m ago 6h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:23:54.116 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 9m ago 7h 1099M 2375M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:23:54.116 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 9m ago 7h 1087M 2375M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:23:54.116 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 9m ago 7h 827M 2375M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:23:54.116 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 9m ago 7h 1063M 2375M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:23:54.116 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 7m ago 7h 914M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:23:54.117 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 7m ago 6h 1170M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:23:54.117 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 7m ago 6h 1045M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:23:54.117 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 7m ago 6h 1240M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:23:54.117 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 7m ago 6h 166M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:23:54.489 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:23:54.490 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:23:54.490 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:23:54.490 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:23:54.490 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:23:54.491 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:23:54.491 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:23:54.491 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:23:54.491 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:23:54.492 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:23:54.492 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:23:54.492 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:23:54.492 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:23:54.492 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:23:54.493 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:23:54.493 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:23:54.493 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:23:54.493 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:23:54.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:23:54 smithi083 ceph-mon[79930]: pgmap v12468: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:54.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:23:54 smithi083 ceph-mon[77242]: pgmap v12468: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:54.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:23:54 smithi153 ceph-mon[70044]: pgmap v12468: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:55.332 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:23:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:23:55.075Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:23:55.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:23:55 smithi083 ceph-mon[79930]: from='client.58561 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:23:55.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:23:55 smithi083 ceph-mon[79930]: from='client.48636 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:23:55.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:23:55 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/431338815' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:23:55.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:23:55 smithi083 ceph-mon[77242]: from='client.58561 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:23:55.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:23:55 smithi083 ceph-mon[77242]: from='client.48636 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:23:55.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:23:55 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/431338815' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:23:55.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:23:55 smithi153 ceph-mon[70044]: from='client.58561 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:23:55.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:23:55 smithi153 ceph-mon[70044]: from='client.48636 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:23:55.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:23:55 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/431338815' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:23:56.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:23:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:23:55.797Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:23:56.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:23:56 smithi083 ceph-mon[77242]: pgmap v12469: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:56.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:23:56 smithi083 ceph-mon[79930]: pgmap v12469: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:56.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:23:56 smithi153 ceph-mon[70044]: pgmap v12469: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:58.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:23:58 smithi083 ceph-mon[79930]: pgmap v12470: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:58.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:23:58 smithi083 ceph-mon[77242]: pgmap v12470: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:23:58.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:23:58 smithi153 ceph-mon[70044]: pgmap v12470: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:00.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:24:00 smithi083 ceph-mon[77242]: pgmap v12471: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:00.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:24:00 smithi083 ceph-mon[79930]: pgmap v12471: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:00.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:24:00 smithi153 ceph-mon[70044]: pgmap v12471: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:24:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:24:01] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T12:24:01.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:24:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:24:01.703Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:24:02.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:24:02 smithi083 ceph-mon[79930]: pgmap v12472: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:02.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:24:02 smithi083 ceph-mon[77242]: pgmap v12472: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:02.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:24:02 smithi153 ceph-mon[70044]: pgmap v12472: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:24:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:24:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:24:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:24:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:24:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:24:04.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:24:04 smithi083 ceph-mon[79930]: pgmap v12473: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:04.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:24:04 smithi083 ceph-mon[77242]: pgmap v12473: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:04.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:24:04 smithi153 ceph-mon[70044]: pgmap v12473: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:05.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:24:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:24:05.077Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:24:06.164 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:24:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:24:05.797Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:24:06.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:24:06 smithi083 ceph-mon[79930]: pgmap v12474: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:06.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:24:06 smithi083 ceph-mon[77242]: pgmap v12474: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:06.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:24:06 smithi153 ceph-mon[70044]: pgmap v12474: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:08.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:24:08 smithi083 ceph-mon[79930]: pgmap v12475: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:08.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:24:08 smithi083 ceph-mon[77242]: pgmap v12475: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:08.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:24:08 smithi153 ceph-mon[70044]: pgmap v12475: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:10.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:24:10 smithi083 ceph-mon[77242]: pgmap v12476: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:10.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:24:10 smithi083 ceph-mon[79930]: pgmap v12476: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:10.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:24:10 smithi153 ceph-mon[70044]: pgmap v12476: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:24:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:24:11] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T12:24:11.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:24:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:24:11.703Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:24:12.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:24:12 smithi083 ceph-mon[79930]: pgmap v12477: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:12.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:24:12 smithi083 ceph-mon[77242]: pgmap v12477: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:12.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:24:12 smithi153 ceph-mon[70044]: pgmap v12477: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:24:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:24:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:24:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:24:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:24:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:24:13.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:24:13 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:24:13.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:24:13 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:24:13.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:24:13 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:24:13.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:24:13 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:24:13.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:24:13 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:24:13.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:24:13 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:24:13.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:24:13 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:24:13.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:24:13 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:24:13.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:24:13 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:24:14.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:24:14 smithi153 ceph-mon[70044]: pgmap v12478: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:14.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:24:14 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:24:14.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:24:14 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:24:14.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:24:14 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:24:14.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:24:14 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:24:14.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:24:14 smithi083 ceph-mon[79930]: pgmap v12478: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:14.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:24:14 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:24:14.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:24:14 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:24:14.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:24:14 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:24:14.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:24:14 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:24:14.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:24:14 smithi083 ceph-mon[77242]: pgmap v12478: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:14.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:24:14 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:24:14.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:24:14 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:24:14.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:24:14 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:24:14.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:24:14 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:24:15.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:24:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:24:15.077Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:24:16.165 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:24:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:24:15.797Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:24:16.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:24:16 smithi083 ceph-mon[79930]: pgmap v12479: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:16.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:24:16 smithi083 ceph-mon[77242]: pgmap v12479: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:16.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:24:16 smithi153 ceph-mon[70044]: pgmap v12479: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:18.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:24:18 smithi083 ceph-mon[79930]: pgmap v12480: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:18.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:24:18 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:24:18.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:24:18 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:24:18.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:24:18 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:24:18.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:24:18 smithi083 ceph-mon[77242]: pgmap v12480: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:18.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:24:18 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:24:18.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:24:18 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:24:18.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:24:18 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:24:19.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:24:18 smithi153 ceph-mon[70044]: pgmap v12480: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:19.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:24:18 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:24:19.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:24:18 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:24:19.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:24:18 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:24:20.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:24:20 smithi083 ceph-mon[79930]: pgmap v12481: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:20.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:24:20 smithi083 ceph-mon[77242]: pgmap v12481: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:21.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:24:20 smithi153 ceph-mon[70044]: pgmap v12481: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:24:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:24:21] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T12:24:21.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:24:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:24:21.704Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:24:22.711 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:24:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:24:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:24:22.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:24:22 smithi083 ceph-mon[79930]: pgmap v12482: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:22.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:24:22 smithi083 ceph-mon[77242]: pgmap v12482: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:23.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:24:22 smithi153 ceph-mon[70044]: pgmap v12482: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:24:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:24:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:24:24.814 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:24:25.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:24:24 smithi153 ceph-mon[70044]: pgmap v12483: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:25.080 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:24:24 smithi083 ceph-mon[79930]: pgmap v12483: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:25.080 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:24:24 smithi083 ceph-mon[77242]: pgmap v12483: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:25.103 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:24:25.104 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 7s ago 6h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:24:25.104 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 8m ago 6h 62.7M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:24:25.104 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 7s ago 6h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:24:25.104 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 8m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:24:25.105 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 8m ago 7h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:24:25.105 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 7s ago 7h 864M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:24:25.105 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 7s ago 7h 1559M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:24:25.105 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 8m ago 7h 658M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:24:25.105 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 7s ago 7h 902M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:24:25.106 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 7s ago 6h 28.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:24:25.106 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 8m ago 6h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:24:25.106 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 7s ago 7h 1126M 2375M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:24:25.106 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 7s ago 7h 1114M 2375M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:24:25.107 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 7s ago 7h 847M 2375M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:24:25.107 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 7s ago 7h 1090M 2375M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:24:25.107 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 8m ago 7h 914M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:24:25.107 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 8m ago 7h 1170M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:24:25.107 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 8m ago 7h 1045M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:24:25.108 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 8m ago 6h 1240M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:24:25.108 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 8m ago 6h 166M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:24:25.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:24:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:24:25.077Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:24:25.463 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:24:25.464 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:24:25.464 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:24:25.464 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:24:25.464 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:24:25.465 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:24:25.465 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:24:25.465 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:24:25.465 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:24:25.465 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:24:25.466 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:24:25.466 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:24:25.466 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:24:25.466 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:24:25.467 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:24:25.467 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:24:25.467 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:24:25.467 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:24:26.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:24:25 smithi153 ceph-mon[70044]: from='client.48648 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:24:26.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:24:25 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1715826499' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:24:26.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:24:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:24:25.798Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:24:26.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:24:25 smithi083 ceph-mon[77242]: from='client.48648 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:24:26.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:24:25 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1715826499' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:24:26.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:24:25 smithi083 ceph-mon[79930]: from='client.48648 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:24:26.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:24:25 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1715826499' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:24:27.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:24:26 smithi153 ceph-mon[70044]: from='client.48654 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:24:27.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:24:26 smithi153 ceph-mon[70044]: pgmap v12484: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:27.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:24:26 smithi083 ceph-mon[79930]: from='client.48654 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:24:27.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:24:26 smithi083 ceph-mon[79930]: pgmap v12484: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:27.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:24:26 smithi083 ceph-mon[77242]: from='client.48654 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:24:27.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:24:26 smithi083 ceph-mon[77242]: pgmap v12484: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:29.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:24:28 smithi153 ceph-mon[70044]: pgmap v12485: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:29.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:24:28 smithi083 ceph-mon[79930]: pgmap v12485: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:29.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:24:28 smithi083 ceph-mon[77242]: pgmap v12485: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:31.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:24:30 smithi153 ceph-mon[70044]: pgmap v12486: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:31.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:24:30 smithi083 ceph-mon[79930]: pgmap v12486: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:31.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:24:30 smithi083 ceph-mon[77242]: pgmap v12486: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:24:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:24:31] "GET /metrics HTTP/1.1" 200 225225 "" "Prometheus/2.33.4" 2022-09-29T12:24:31.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:24:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:24:31.704Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:24:32.726 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:24:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:24:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:24:33.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:24:32 smithi153 ceph-mon[70044]: pgmap v12487: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:33.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:24:32 smithi083 ceph-mon[79930]: pgmap v12487: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:33.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:24:32 smithi083 ceph-mon[77242]: pgmap v12487: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:24:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:24:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:24:35.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:24:34 smithi153 ceph-mon[70044]: pgmap v12488: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:35.081 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:24:34 smithi083 ceph-mon[79930]: pgmap v12488: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:35.082 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:24:34 smithi083 ceph-mon[77242]: pgmap v12488: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:35.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:24:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:24:35.079Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:24:36.170 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:24:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:24:35.800Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:24:36.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:24:36 smithi083 ceph-mon[79930]: pgmap v12489: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:36.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:24:36 smithi083 ceph-mon[77242]: pgmap v12489: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:24:36 smithi153 ceph-mon[70044]: pgmap v12489: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:38.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:24:38 smithi083 ceph-mon[79930]: pgmap v12490: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:38.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:24:38 smithi083 ceph-mon[77242]: pgmap v12490: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:38.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:24:38 smithi153 ceph-mon[70044]: pgmap v12490: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:24:40 smithi083 ceph-mon[79930]: pgmap v12491: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:24:40 smithi083 ceph-mon[77242]: pgmap v12491: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:24:40 smithi153 ceph-mon[70044]: pgmap v12491: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:41.708 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:24:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:24:41] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T12:24:41.708 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:24:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:24:41.709 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:24:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:24:41.709 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:24:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:24:41.709 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:24:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:24:41.710 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:24:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:24:41.710 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:24:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:24:41.710 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:24:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:24:41.710 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:24:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:24:41.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:24:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:24:41.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:24:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:24:41.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:24:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:24:41.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:24:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:24:41.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:24:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:24:41.705Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:24:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:24:42 smithi083 ceph-mon[79930]: pgmap v12492: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:24:42 smithi083 ceph-mon[77242]: pgmap v12492: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:24:42 smithi153 ceph-mon[70044]: pgmap v12492: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:24:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:24:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:24:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:24:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:24:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:24:44.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:24:44 smithi083 ceph-mon[79930]: pgmap v12493: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:44.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:24:44 smithi083 ceph-mon[77242]: pgmap v12493: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:44.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:24:44 smithi153 ceph-mon[70044]: pgmap v12493: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:45.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:24:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:24:45.080Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:24:46.172 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:24:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:24:45.800Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:24:46.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:24:46 smithi083 ceph-mon[79930]: pgmap v12494: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:46.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:24:46 smithi083 ceph-mon[77242]: pgmap v12494: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:46.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:24:46 smithi153 ceph-mon[70044]: pgmap v12494: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:48.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:24:48 smithi083 ceph-mon[79930]: pgmap v12495: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:48.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:24:48 smithi083 ceph-mon[77242]: pgmap v12495: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:48.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:24:48 smithi153 ceph-mon[70044]: pgmap v12495: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:50.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:24:50 smithi083 ceph-mon[79930]: pgmap v12496: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:50.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:24:50 smithi083 ceph-mon[77242]: pgmap v12496: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:50.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:24:50 smithi153 ceph-mon[70044]: pgmap v12496: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:24:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:24:51] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T12:24:51.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:24:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:24:51.705Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:24:52.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:24:52 smithi083 ceph-mon[79930]: pgmap v12497: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:52.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:24:52 smithi083 ceph-mon[77242]: pgmap v12497: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:52.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:24:52 smithi153 ceph-mon[70044]: pgmap v12497: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:24:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:24:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:24:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:24:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:24:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:24:54.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:24:54 smithi083 ceph-mon[79930]: pgmap v12498: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:54.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:24:54 smithi083 ceph-mon[77242]: pgmap v12498: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:54.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:24:54 smithi153 ceph-mon[70044]: pgmap v12498: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:24:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:24:55.081Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:24:55.789 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:24:56.079 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:24:56.079 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 38s ago 6h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:24:56.079 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 8m ago 6h 62.7M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:24:56.080 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (6h) 38s ago 6h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:24:56.080 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (6h) 8m ago 6h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:24:56.080 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 8m ago 7h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:24:56.080 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 38s ago 7h 864M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:24:56.081 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 38s ago 7h 1559M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:24:56.081 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 8m ago 7h 658M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:24:56.081 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 38s ago 7h 902M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:24:56.081 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 38s ago 6h 28.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:24:56.081 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 8m ago 6h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:24:56.082 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 38s ago 7h 1126M 2375M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:24:56.082 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 38s ago 7h 1114M 2375M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:24:56.082 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 38s ago 7h 847M 2375M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:24:56.082 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 38s ago 7h 1090M 2375M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:24:56.083 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 8m ago 7h 914M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:24:56.083 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 8m ago 7h 1170M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:24:56.083 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 8m ago 7h 1045M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:24:56.083 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 8m ago 7h 1240M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:24:56.083 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 8m ago 6h 166M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:24:56.175 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:24:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:24:55.800Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:24:56.440 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:24:56.440 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:24:56.440 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:24:56.440 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:24:56.441 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:24:56.441 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:24:56.441 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:24:56.441 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:24:56.442 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:24:56.442 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:24:56.442 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:24:56.442 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:24:56.443 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:24:56.443 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:24:56.443 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:24:56.443 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:24:56.443 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:24:56.444 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:24:56.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:24:56 smithi083 ceph-mon[79930]: pgmap v12499: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:56.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:24:56 smithi083 ceph-mon[79930]: from='client.48666 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:24:56.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:24:56 smithi083 ceph-mon[77242]: pgmap v12499: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:56.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:24:56 smithi083 ceph-mon[77242]: from='client.48666 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:24:56.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:24:56 smithi153 ceph-mon[70044]: pgmap v12499: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:56.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:24:56 smithi153 ceph-mon[70044]: from='client.48666 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:24:57.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:24:57 smithi083 ceph-mon[79930]: from='client.58603 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:24:57.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:24:57 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2876341888' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:24:57.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:24:57 smithi083 ceph-mon[77242]: from='client.58603 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:24:57.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:24:57 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2876341888' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:24:57.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:24:57 smithi153 ceph-mon[70044]: from='client.58603 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:24:57.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:24:57 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2876341888' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:24:58.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:24:58 smithi083 ceph-mon[79930]: pgmap v12500: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:58.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:24:58 smithi083 ceph-mon[77242]: pgmap v12500: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:24:58.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:24:58 smithi153 ceph-mon[70044]: pgmap v12500: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:00.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:25:00 smithi083 ceph-mon[79930]: pgmap v12501: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:00.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:25:00 smithi083 ceph-mon[77242]: pgmap v12501: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:00.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:25:00 smithi153 ceph-mon[70044]: pgmap v12501: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:01.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:25:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:25:01] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T12:25:01.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:25:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:25:01.706Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:25:02.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:25:02 smithi083 ceph-mon[77242]: pgmap v12502: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:02.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:25:02 smithi083 ceph-mon[79930]: pgmap v12502: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:02.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:25:02 smithi153 ceph-mon[70044]: pgmap v12502: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:25:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:25:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:25:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:25:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:25:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:25:04.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:25:04 smithi083 ceph-mon[77242]: pgmap v12503: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:04.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:25:04 smithi083 ceph-mon[79930]: pgmap v12503: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:04.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:25:04 smithi153 ceph-mon[70044]: pgmap v12503: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:05.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:25:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:25:05.081Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:25:06.178 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:25:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:25:05.802Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:25:06.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:25:06 smithi083 ceph-mon[79930]: pgmap v12504: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:06.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:25:06 smithi083 ceph-mon[77242]: pgmap v12504: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:06.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:25:06 smithi153 ceph-mon[70044]: pgmap v12504: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:08.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:25:08 smithi083 ceph-mon[79930]: pgmap v12505: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:08.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:25:08 smithi083 ceph-mon[77242]: pgmap v12505: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:08.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:25:08 smithi153 ceph-mon[70044]: pgmap v12505: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:10.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:25:10 smithi083 ceph-mon[77242]: pgmap v12506: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:10.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:25:10 smithi083 ceph-mon[79930]: pgmap v12506: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:10.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:25:10 smithi153 ceph-mon[70044]: pgmap v12506: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:25:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:25:11] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T12:25:11.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:25:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:25:11.707Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:25:12.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:25:12 smithi083 ceph-mon[77242]: pgmap v12507: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:12.727 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:25:12 smithi083 ceph-mon[79930]: pgmap v12507: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:12.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:25:12 smithi153 ceph-mon[70044]: pgmap v12507: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:25:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:25:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:25:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:25:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:25:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:25:14.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:25:14 smithi083 ceph-mon[77242]: pgmap v12508: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:14.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:25:14 smithi083 ceph-mon[79930]: pgmap v12508: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:14.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:25:14 smithi153 ceph-mon[70044]: pgmap v12508: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:15.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:25:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:25:15.082Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:25:16.175 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:25:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:25:15.802Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:25:16.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:25:16 smithi083 ceph-mon[79930]: pgmap v12509: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:16.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:25:16 smithi083 ceph-mon[77242]: pgmap v12509: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:16.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:25:16 smithi153 ceph-mon[70044]: pgmap v12509: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:18.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:25:18 smithi083 ceph-mon[79930]: pgmap v12510: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:18.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:25:18 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:25:18.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:25:18 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:25:18.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:25:18 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:25:18.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:25:18 smithi083 ceph-mon[77242]: pgmap v12510: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:18.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:25:18 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:25:18.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:25:18 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:25:18.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:25:18 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:25:18.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:25:18 smithi153 ceph-mon[70044]: pgmap v12510: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:18.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:25:18 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:25:18.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:25:18 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:25:18.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:25:18 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:25:20.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:25:19 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:25:20.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:25:19 smithi083 ceph-mon[79930]: pgmap v12511: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:20.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:25:19 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:25:20.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:25:19 smithi083 ceph-mon[77242]: pgmap v12511: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:20.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:25:19 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:25:20.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:25:19 smithi153 ceph-mon[70044]: pgmap v12511: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:25:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:25:21] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T12:25:21.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:25:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:25:21.708Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:25:22.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:25:22 smithi083 ceph-mon[77242]: pgmap v12512: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:22.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:25:22 smithi083 ceph-mon[79930]: pgmap v12512: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:22.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:25:22 smithi153 ceph-mon[70044]: pgmap v12512: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:25:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:25:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:25:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:25:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:25:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:25:24.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:25:24 smithi083 ceph-mon[79930]: pgmap v12513: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:24.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:25:24 smithi083 ceph-mon[77242]: pgmap v12513: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:24.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:25:24 smithi153 ceph-mon[70044]: pgmap v12513: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:25.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:25:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:25:25.083Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:25:26.184 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:25:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:25:25.803Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:25:26.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:25:26 smithi083 ceph-mon[79930]: pgmap v12514: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:26.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:25:26 smithi083 ceph-mon[77242]: pgmap v12514: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:25:26 smithi153 ceph-mon[70044]: pgmap v12514: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:26.771 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:25:27.058 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:25:27.059 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 69s ago 6h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:25:27.059 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 9m ago 6h 62.7M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:25:27.059 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 69s ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:25:27.059 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 9m ago 7h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:25:27.060 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 9m ago 7h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:25:27.060 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 69s ago 7h 864M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:25:27.060 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 69s ago 7h 1559M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:25:27.060 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 9m ago 7h 658M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:25:27.060 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 69s ago 7h 902M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:25:27.061 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (6h) 69s ago 6h 28.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:25:27.061 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (6h) 9m ago 6h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:25:27.061 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 69s ago 7h 1126M 2375M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:25:27.061 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 69s ago 7h 1114M 2375M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:25:27.062 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 69s ago 7h 847M 2375M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:25:27.062 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 69s ago 7h 1090M 2375M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:25:27.062 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 9m ago 7h 914M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:25:27.062 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 9m ago 7h 1170M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:25:27.062 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 9m ago 7h 1045M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:25:27.063 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 9m ago 7h 1240M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:25:27.063 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 9m ago 6h 166M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:25:27.417 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:25:27.417 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:25:27.417 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:25:27.418 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:25:27.418 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:25:27.418 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:25:27.418 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:25:27.419 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:25:27.419 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:25:27.419 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:25:27.419 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:25:27.419 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:25:27.420 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:25:27.420 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:25:27.420 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:25:27.420 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:25:27.420 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:25:27.421 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:25:27.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:25:27 smithi083 ceph-mon[79930]: from='client.48684 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:25:27.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:25:27 smithi083 ceph-mon[77242]: from='client.48684 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:25:27.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:25:27 smithi153 ceph-mon[70044]: from='client.48684 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:25:28.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:25:28 smithi083 ceph-mon[79930]: from='client.58621 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:25:28.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:25:28 smithi083 ceph-mon[79930]: pgmap v12515: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:28.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:25:28 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3787134386' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:25:28.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:25:28 smithi083 ceph-mon[77242]: from='client.58621 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:25:28.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:25:28 smithi083 ceph-mon[77242]: pgmap v12515: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:28.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:25:28 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3787134386' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:25:28.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:25:28 smithi153 ceph-mon[70044]: from='client.58621 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:25:28.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:25:28 smithi153 ceph-mon[70044]: pgmap v12515: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:28.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:25:28 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3787134386' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:25:30.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:25:30 smithi083 ceph-mon[79930]: pgmap v12516: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:30.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:25:30 smithi083 ceph-mon[77242]: pgmap v12516: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:30.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:25:30 smithi153 ceph-mon[70044]: pgmap v12516: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:25:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:25:31] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T12:25:31.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:25:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:25:31.709Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:25:32.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:25:32 smithi083 ceph-mon[77242]: pgmap v12517: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:32.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:25:32 smithi083 ceph-mon[79930]: pgmap v12517: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:32.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:25:32 smithi153 ceph-mon[70044]: pgmap v12517: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:25:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:25:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:25:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:25:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:25:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:25:34.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:25:34 smithi083 ceph-mon[79930]: pgmap v12518: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:34.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:25:34 smithi083 ceph-mon[77242]: pgmap v12518: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:34.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:25:34 smithi153 ceph-mon[70044]: pgmap v12518: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:35.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:25:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:25:35.083Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:25:36.184 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:25:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:25:35.804Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:25:36.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:25:36 smithi083 ceph-mon[77242]: pgmap v12519: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:36.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:25:36 smithi083 ceph-mon[79930]: pgmap v12519: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:25:36 smithi153 ceph-mon[70044]: pgmap v12519: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:38.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:25:38 smithi083 ceph-mon[77242]: pgmap v12520: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:38.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:25:38 smithi083 ceph-mon[79930]: pgmap v12520: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:38.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:25:38 smithi153 ceph-mon[70044]: pgmap v12520: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:25:40 smithi083 ceph-mon[79930]: pgmap v12521: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:25:40 smithi083 ceph-mon[77242]: pgmap v12521: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:25:40 smithi153 ceph-mon[70044]: pgmap v12521: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:41.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:25:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:25:41] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T12:25:41.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:25:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:25:41.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:25:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:25:41.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:25:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:25:41.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:25:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:25:41.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:25:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:25:41.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:25:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:25:41.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:25:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:25:41.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:25:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:25:41.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:25:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:25:41.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:25:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:25:41.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:25:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:25:41.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:25:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:25:41.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:25:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:25:41.710Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:25:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:25:42 smithi083 ceph-mon[79930]: pgmap v12522: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:25:42 smithi083 ceph-mon[77242]: pgmap v12522: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:25:42 smithi153 ceph-mon[70044]: pgmap v12522: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:42.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:25:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:25:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:25:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:25:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:25:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:25:44.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:25:44 smithi083 ceph-mon[79930]: pgmap v12523: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:44.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:25:44 smithi083 ceph-mon[77242]: pgmap v12523: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:44.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:25:44 smithi153 ceph-mon[70044]: pgmap v12523: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:45.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:25:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:25:45.084Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:25:46.186 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:25:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:25:45.804Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:25:46.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:25:46 smithi083 ceph-mon[79930]: pgmap v12524: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:46.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:25:46 smithi083 ceph-mon[77242]: pgmap v12524: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:46.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:25:46 smithi153 ceph-mon[70044]: pgmap v12524: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:48.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:25:48 smithi083 ceph-mon[79930]: pgmap v12525: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:48.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:25:48 smithi083 ceph-mon[77242]: pgmap v12525: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:48.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:25:48 smithi153 ceph-mon[70044]: pgmap v12525: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:50.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:25:50 smithi083 ceph-mon[77242]: pgmap v12526: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:50.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:25:50 smithi083 ceph-mon[79930]: pgmap v12526: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:50.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:25:50 smithi153 ceph-mon[70044]: pgmap v12526: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:25:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:25:51] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T12:25:51.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:25:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:25:51.711Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:25:52.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:25:52 smithi083 ceph-mon[79930]: pgmap v12527: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:52.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:25:52 smithi083 ceph-mon[77242]: pgmap v12527: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:52.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:25:52 smithi153 ceph-mon[70044]: pgmap v12527: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:25:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:25:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:25:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:25:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:25:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:25:54.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:25:54 smithi083 ceph-mon[79930]: pgmap v12528: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:54.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:25:54 smithi083 ceph-mon[77242]: pgmap v12528: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:54.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:25:54 smithi153 ceph-mon[70044]: pgmap v12528: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:25:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:25:55.084Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:25:56.188 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:25:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:25:55.804Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:25:56.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:25:56 smithi083 ceph-mon[77242]: pgmap v12529: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:56.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:25:56 smithi083 ceph-mon[79930]: pgmap v12529: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:56.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:25:56 smithi153 ceph-mon[70044]: pgmap v12529: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:57.742 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:25:58.031 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:25:58.032 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (6h) 100s ago 7h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:25:58.032 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (6h) 9m ago 6h 62.7M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:25:58.032 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 100s ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:25:58.033 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 9m ago 7h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:25:58.033 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 9m ago 7h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:25:58.033 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 100s ago 7h 864M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:25:58.033 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 100s ago 7h 1559M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:25:58.034 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 9m ago 7h 658M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:25:58.034 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 100s ago 7h 902M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:25:58.034 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 100s ago 7h 28.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:25:58.034 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 9m ago 7h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:25:58.035 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 100s ago 7h 1126M 2375M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:25:58.035 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 100s ago 7h 1114M 2375M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:25:58.035 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 100s ago 7h 847M 2375M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:25:58.035 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 100s ago 7h 1090M 2375M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:25:58.035 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 9m ago 7h 914M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:25:58.036 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 9m ago 7h 1170M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:25:58.036 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 9m ago 7h 1045M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:25:58.036 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 9m ago 7h 1240M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:25:58.036 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 9m ago 7h 166M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:25:58.390 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:25:58.390 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:25:58.390 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:25:58.391 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:25:58.391 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:25:58.391 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:25:58.391 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:25:58.392 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:25:58.392 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:25:58.392 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:25:58.392 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:25:58.393 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:25:58.393 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:25:58.393 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:25:58.393 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:25:58.393 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:25:58.394 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:25:58.394 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:25:58.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:25:58 smithi083 ceph-mon[79930]: pgmap v12530: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:58.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:25:58 smithi083 ceph-mon[79930]: from='client.48702 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:25:58.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:25:58 smithi083 ceph-mon[77242]: pgmap v12530: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:58.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:25:58 smithi083 ceph-mon[77242]: from='client.48702 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:25:58.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:25:58 smithi153 ceph-mon[70044]: pgmap v12530: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:25:58.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:25:58 smithi153 ceph-mon[70044]: from='client.48702 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:25:59.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:25:59 smithi083 ceph-mon[79930]: from='client.58639 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:25:59.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:25:59 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3839062796' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:25:59.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:25:59 smithi083 ceph-mon[77242]: from='client.58639 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:25:59.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:25:59 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3839062796' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:25:59.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:25:59 smithi153 ceph-mon[70044]: from='client.58639 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:25:59.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:25:59 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3839062796' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:26:00.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:26:00 smithi083 ceph-mon[79930]: pgmap v12531: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:00.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:26:00 smithi083 ceph-mon[77242]: pgmap v12531: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:00.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:26:00 smithi153 ceph-mon[70044]: pgmap v12531: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:26:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:26:01] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T12:26:01.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:26:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:26:01.712Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:26:02.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:26:02 smithi083 ceph-mon[79930]: pgmap v12532: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:02.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:26:02 smithi083 ceph-mon[77242]: pgmap v12532: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:02.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:26:02 smithi153 ceph-mon[70044]: pgmap v12532: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:26:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:26:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:26:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:26:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:26:04.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:26:04 smithi083 ceph-mon[79930]: pgmap v12533: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:04.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:26:04 smithi083 ceph-mon[77242]: pgmap v12533: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:04.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:26:04 smithi153 ceph-mon[70044]: pgmap v12533: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:05.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:26:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:26:05.084Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:26:06.191 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:26:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:26:05.805Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:26:06.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:26:06 smithi083 ceph-mon[79930]: pgmap v12534: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:06.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:26:06 smithi083 ceph-mon[77242]: pgmap v12534: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:06.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:26:06 smithi153 ceph-mon[70044]: pgmap v12534: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:08.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:26:08 smithi083 ceph-mon[79930]: pgmap v12535: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:08.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:26:08 smithi083 ceph-mon[77242]: pgmap v12535: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:08.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:26:08 smithi153 ceph-mon[70044]: pgmap v12535: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:10.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:26:10 smithi083 ceph-mon[79930]: pgmap v12536: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:10.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:26:10 smithi083 ceph-mon[77242]: pgmap v12536: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:10.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:26:10 smithi153 ceph-mon[70044]: pgmap v12536: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:26:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:26:11] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T12:26:11.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:26:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:26:11.712Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:26:12.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:26:12 smithi083 ceph-mon[79930]: pgmap v12537: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:12.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:26:12 smithi083 ceph-mon[77242]: pgmap v12537: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:12.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:26:12 smithi153 ceph-mon[70044]: pgmap v12537: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:26:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:26:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:26:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:26:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:26:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:26:14.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:26:14 smithi083 ceph-mon[79930]: pgmap v12538: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:14.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:26:14 smithi083 ceph-mon[77242]: pgmap v12538: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:14.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:26:14 smithi153 ceph-mon[70044]: pgmap v12538: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:15.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:26:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:26:15.085Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:26:16.192 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:26:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:26:15.805Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:26:16.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:26:16 smithi083 ceph-mon[77242]: pgmap v12539: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:16.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:26:16 smithi083 ceph-mon[79930]: pgmap v12539: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:16.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:26:16 smithi153 ceph-mon[70044]: pgmap v12539: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:18.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:26:18 smithi083 ceph-mon[79930]: pgmap v12540: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:18.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:26:18 smithi083 ceph-mon[77242]: pgmap v12540: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:18.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:26:18 smithi153 ceph-mon[70044]: pgmap v12540: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:19.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:26:19 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:26:19.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:26:19 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:26:19.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:26:19 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:26:19.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:26:19 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:26:19.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:26:19 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:26:19.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:26:19 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:26:19.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:26:19 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:26:19.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:26:19 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:26:19.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:26:19 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:26:20.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:26:20 smithi083 ceph-mon[77242]: pgmap v12541: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:20.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:26:20 smithi083 ceph-mon[79930]: pgmap v12541: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:20.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:26:20 smithi153 ceph-mon[70044]: pgmap v12541: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:26:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:26:21] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T12:26:21.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:26:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:26:21.714Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:26:22.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:26:22 smithi083 ceph-mon[79930]: pgmap v12542: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:22.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:26:22 smithi083 ceph-mon[77242]: pgmap v12542: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:22.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:26:22 smithi153 ceph-mon[70044]: pgmap v12542: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:26:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:26:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:26:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:26:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:26:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:26:24.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:26:24 smithi153 ceph-mon[70044]: pgmap v12543: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:24.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:26:24 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:26:24.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:26:24 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:26:24.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:26:24 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:26:24.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:26:24 smithi083 ceph-mon[77242]: pgmap v12543: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:24.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:26:24 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:26:24.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:26:24 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:26:24.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:26:24 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:26:24.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:26:24 smithi083 ceph-mon[79930]: pgmap v12543: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:24.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:26:24 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:26:24.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:26:24 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:26:24.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:26:24 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:26:25.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:26:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:26:25.086Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:26:26.195 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:26:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:26:25.806Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:26:26.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:26:26 smithi083 ceph-mon[79930]: pgmap v12544: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:26.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:26:26 smithi083 ceph-mon[77242]: pgmap v12544: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:26:26 smithi153 ceph-mon[70044]: pgmap v12544: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:28.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:26:28 smithi083 ceph-mon[79930]: pgmap v12545: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:28.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:26:28 smithi083 ceph-mon[77242]: pgmap v12545: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:28.719 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:26:28.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:26:28 smithi153 ceph-mon[70044]: pgmap v12545: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:29.009 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:26:29.009 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 2m ago 7h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:26:29.009 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 5s ago 7h 64.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:26:29.010 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 2m ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:26:29.010 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 5s ago 7h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:26:29.010 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 5s ago 7h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:26:29.010 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 2m ago 7h 864M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:26:29.011 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 2m ago 7h 1559M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:26:29.011 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 5s ago 7h 674M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:26:29.011 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 2m ago 7h 902M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:26:29.011 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 2m ago 7h 28.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:26:29.012 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 5s ago 7h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:26:29.012 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 2m ago 7h 1126M 2375M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:26:29.012 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 2m ago 7h 1114M 2375M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:26:29.012 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 2m ago 7h 847M 2375M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:26:29.013 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 2m ago 7h 1090M 2375M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:26:29.013 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 5s ago 7h 936M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:26:29.013 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 5s ago 7h 1198M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:26:29.014 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 5s ago 7h 1071M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:26:29.014 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 5s ago 7h 1270M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:26:29.014 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 5s ago 7h 169M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:26:29.363 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:26:29.364 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:26:29.364 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:26:29.364 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:26:29.365 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:26:29.365 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:26:29.365 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:26:29.365 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:26:29.365 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:26:29.366 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:26:29.366 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:26:29.366 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:26:29.366 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:26:29.367 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:26:29.367 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:26:29.367 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:26:29.367 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:26:29.368 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:26:29.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:26:29 smithi083 ceph-mon[79930]: from='client.48720 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:26:29.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:26:29 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1557426255' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:26:29.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:26:29 smithi083 ceph-mon[77242]: from='client.48720 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:26:29.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:26:29 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1557426255' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:26:29.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:26:29 smithi153 ceph-mon[70044]: from='client.48720 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:26:29.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:26:29 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1557426255' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:26:30.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:26:30 smithi083 ceph-mon[79930]: from='client.48726 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:26:30.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:26:30 smithi083 ceph-mon[79930]: pgmap v12546: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:30.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:26:30 smithi083 ceph-mon[77242]: from='client.48726 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:26:30.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:26:30 smithi083 ceph-mon[77242]: pgmap v12546: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:30.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:26:30 smithi153 ceph-mon[70044]: from='client.48726 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:26:30.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:26:30 smithi153 ceph-mon[70044]: pgmap v12546: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:26:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:26:31] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T12:26:31.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:26:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:26:31.714Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:26:32.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:26:32 smithi083 ceph-mon[79930]: pgmap v12547: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:32.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:26:32 smithi083 ceph-mon[77242]: pgmap v12547: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:32.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:26:32 smithi153 ceph-mon[70044]: pgmap v12547: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:26:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:26:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:26:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:26:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:26:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:26:34.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:26:34 smithi083 ceph-mon[79930]: pgmap v12548: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:34.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:26:34 smithi083 ceph-mon[77242]: pgmap v12548: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:34.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:26:34 smithi153 ceph-mon[70044]: pgmap v12548: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:35.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:26:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:26:35.086Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:26:36.197 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:26:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:26:35.807Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:26:36.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:26:36 smithi083 ceph-mon[79930]: pgmap v12549: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:36.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:26:36 smithi083 ceph-mon[77242]: pgmap v12549: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:36.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:26:36 smithi153 ceph-mon[70044]: pgmap v12549: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:38.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:26:38 smithi083 ceph-mon[77242]: pgmap v12550: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:38.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:26:38 smithi083 ceph-mon[79930]: pgmap v12550: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:38.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:26:38 smithi153 ceph-mon[70044]: pgmap v12550: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:26:40 smithi083 ceph-mon[79930]: pgmap v12551: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:26:40 smithi083 ceph-mon[77242]: pgmap v12551: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:26:40 smithi153 ceph-mon[70044]: pgmap v12551: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:41.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:26:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:26:41] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T12:26:41.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:26:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:26:41.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:26:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:26:41.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:26:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:26:41.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:26:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:26:41.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:26:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:26:41.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:26:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:26:41.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:26:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:26:41.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:26:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:26:41.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:26:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:26:41.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:26:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:26:41.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:26:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:26:41.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:26:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:26:42.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:26:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:26:41.714Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:26:42.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:26:42 smithi083 ceph-mon[77242]: pgmap v12552: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:42.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:26:42 smithi083 ceph-mon[79930]: pgmap v12552: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:26:42 smithi153 ceph-mon[70044]: pgmap v12552: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:26:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:26:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:26:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:26:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:26:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:26:44.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:26:44 smithi083 ceph-mon[77242]: pgmap v12553: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:44.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:26:44 smithi083 ceph-mon[79930]: pgmap v12553: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:44.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:26:44 smithi153 ceph-mon[70044]: pgmap v12553: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:45.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:26:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:26:45.087Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:26:46.199 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:26:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:26:45.807Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:26:46.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:26:46 smithi083 ceph-mon[79930]: pgmap v12554: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:46.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:26:46 smithi083 ceph-mon[77242]: pgmap v12554: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:46.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:26:46 smithi153 ceph-mon[70044]: pgmap v12554: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:48.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:26:48 smithi083 ceph-mon[79930]: pgmap v12555: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:48.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:26:48 smithi083 ceph-mon[77242]: pgmap v12555: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:48.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:26:48 smithi153 ceph-mon[70044]: pgmap v12555: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:50.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:26:50 smithi083 ceph-mon[79930]: pgmap v12556: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:50.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:26:50 smithi083 ceph-mon[77242]: pgmap v12556: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:50.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:26:50 smithi153 ceph-mon[70044]: pgmap v12556: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:26:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:26:51] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T12:26:51.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:26:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:26:51.715Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:26:52.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:26:52 smithi083 ceph-mon[77242]: pgmap v12557: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:52.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:26:52 smithi083 ceph-mon[79930]: pgmap v12557: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:52.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:26:52 smithi153 ceph-mon[70044]: pgmap v12557: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:52.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:26:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:26:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:26:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:26:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:26:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:26:54.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:26:54 smithi083 ceph-mon[79930]: pgmap v12558: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:54.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:26:54 smithi083 ceph-mon[77242]: pgmap v12558: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:54.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:26:54 smithi153 ceph-mon[70044]: pgmap v12558: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:26:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:26:55.088Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:26:56.201 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:26:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:26:55.807Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:26:56.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:26:56 smithi083 ceph-mon[77242]: pgmap v12559: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:56.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:26:56 smithi083 ceph-mon[79930]: pgmap v12559: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:56.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:26:56 smithi153 ceph-mon[70044]: pgmap v12559: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:58.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:26:58 smithi083 ceph-mon[77242]: pgmap v12560: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:58.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:26:58 smithi083 ceph-mon[79930]: pgmap v12560: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:58.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:26:58 smithi153 ceph-mon[70044]: pgmap v12560: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:26:59.685 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:26:59.973 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:26:59.974 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 2m ago 7h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:26:59.974 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 36s ago 7h 64.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:26:59.974 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 2m ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:26:59.975 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 36s ago 7h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:26:59.975 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 36s ago 7h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:26:59.975 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 2m ago 7h 864M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:26:59.975 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 2m ago 7h 1559M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:26:59.976 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 36s ago 7h 674M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:26:59.976 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 2m ago 7h 902M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:26:59.976 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 2m ago 7h 28.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:26:59.976 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 36s ago 7h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:26:59.977 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 2m ago 7h 1126M 2375M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:26:59.977 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 2m ago 7h 1114M 2375M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:26:59.977 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 2m ago 7h 847M 2375M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:26:59.977 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 2m ago 7h 1090M 2375M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:26:59.977 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 36s ago 7h 936M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:26:59.978 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 36s ago 7h 1198M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:26:59.978 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 36s ago 7h 1071M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:26:59.978 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 36s ago 7h 1270M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:26:59.979 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 36s ago 7h 169M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:27:00.330 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:27:00.330 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:27:00.331 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:27:00.331 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:27:00.331 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:27:00.331 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:27:00.331 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:27:00.332 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:27:00.332 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:27:00.332 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:27:00.332 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:27:00.332 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:27:00.333 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:27:00.333 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:27:00.333 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:27:00.333 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:27:00.333 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:27:00.334 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:27:00.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:27:00 smithi083 ceph-mon[77242]: pgmap v12561: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:27:00.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:27:00 smithi083 ceph-mon[77242]: from='client.48738 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:27:00.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:27:00 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1000226330' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:27:00.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:27:00 smithi083 ceph-mon[79930]: pgmap v12561: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:27:00.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:27:00 smithi083 ceph-mon[79930]: from='client.48738 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:27:00.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:27:00 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1000226330' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:27:00.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:27:00 smithi153 ceph-mon[70044]: pgmap v12561: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:27:00.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:27:00 smithi153 ceph-mon[70044]: from='client.48738 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:27:00.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:27:00 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1000226330' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:27:01.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:27:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:27:01] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T12:27:01.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:27:01 smithi083 ceph-mon[79930]: from='client.58675 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:27:01.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:27:01 smithi083 ceph-mon[77242]: from='client.58675 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:27:01.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:27:01 smithi153 ceph-mon[70044]: from='client.58675 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:27:02.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:27:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:27:01.716Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:27:02.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:27:02 smithi083 ceph-mon[79930]: pgmap v12562: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:27:02.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:27:02 smithi083 ceph-mon[77242]: pgmap v12562: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:27:02.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:27:02 smithi153 ceph-mon[70044]: pgmap v12562: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:27:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:27:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:27:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:27:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:27:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:27:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:27:04.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:27:04 smithi083 ceph-mon[79930]: pgmap v12563: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:27:04.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:27:04 smithi083 ceph-mon[77242]: pgmap v12563: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:27:04.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:27:04 smithi153 ceph-mon[70044]: pgmap v12563: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:27:05.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:27:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:27:05.088Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:27:06.205 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:27:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:27:05.808Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:27:06.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:27:06 smithi083 ceph-mon[77242]: pgmap v12564: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:27:06.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:27:06 smithi083 ceph-mon[79930]: pgmap v12564: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:27:06.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:27:06 smithi153 ceph-mon[70044]: pgmap v12564: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:27:08.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:27:08 smithi083 ceph-mon[79930]: pgmap v12565: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:27:08.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:27:08 smithi083 ceph-mon[77242]: pgmap v12565: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:27:08.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:27:08 smithi153 ceph-mon[70044]: pgmap v12565: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:27:10.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:27:10 smithi083 ceph-mon[77242]: pgmap v12566: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:27:10.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:27:10 smithi083 ceph-mon[79930]: pgmap v12566: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:27:10.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:27:10 smithi153 ceph-mon[70044]: pgmap v12566: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:27:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:27:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:27:11] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T12:27:11.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:27:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:27:11.716Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:27:12.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:27:12 smithi083 ceph-mon[77242]: pgmap v12567: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:27:12.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:27:12 smithi083 ceph-mon[79930]: pgmap v12567: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:27:12.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:27:12 smithi153 ceph-mon[70044]: pgmap v12567: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:27:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:27:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:27:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:27:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:27:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:27:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:27:14.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:27:14 smithi083 ceph-mon[79930]: pgmap v12568: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:27:14.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:27:14 smithi083 ceph-mon[77242]: pgmap v12568: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:27:14.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:27:14 smithi153 ceph-mon[70044]: pgmap v12568: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:27:15.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:27:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:27:15.089Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:27:16.206 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:27:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:27:15.807Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:27:16.207 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:27:16 smithi083 ceph-mon[77242]: pgmap v12569: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:27:16.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:27:16 smithi083 ceph-mon[79930]: pgmap v12569: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:27:16.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:27:16 smithi153 ceph-mon[70044]: pgmap v12569: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:27:18.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:27:18 smithi083 ceph-mon[79930]: pgmap v12570: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:27:18.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:27:18 smithi083 ceph-mon[77242]: pgmap v12570: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:27:18.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:27:18 smithi153 ceph-mon[70044]: pgmap v12570: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:27:20.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:27:20 smithi083 ceph-mon[77242]: pgmap v12571: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:27:20.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:27:20 smithi083 ceph-mon[79930]: pgmap v12571: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:27:20.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:27:20 smithi153 ceph-mon[70044]: pgmap v12571: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:27:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:27:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:27:21] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T12:27:21.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:27:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:27:21.717Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:27:22.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:27:22 smithi083 ceph-mon[79930]: pgmap v12572: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:27:22.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:27:22 smithi083 ceph-mon[77242]: pgmap v12572: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:27:22.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:27:22 smithi153 ceph-mon[70044]: pgmap v12572: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:27:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:27:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:27:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:27:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:27:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:27:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:27:24.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:27:24 smithi083 ceph-mon[79930]: pgmap v12573: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:27:24.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:27:24 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:27:24.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:27:24 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:27:24.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:27:24 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:27:24.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:27:24 smithi083 ceph-mon[77242]: pgmap v12573: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:27:24.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:27:24 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:27:24.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:27:24 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:27:24.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:27:24 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:27:24.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:27:24 smithi153 ceph-mon[70044]: pgmap v12573: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:27:24.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:27:24 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:27:24.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:27:24 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:27:24.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:27:24 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:27:25.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:27:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:27:25.089Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:27:26.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:27:25 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:27:26.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:27:25 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:27:26.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:27:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:27:25.808Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:27:26.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:27:25 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:27:27.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:27:26 smithi153 ceph-mon[70044]: pgmap v12574: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:27:27.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:27:26 smithi083 ceph-mon[79930]: pgmap v12574: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:27:27.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:27:26 smithi083 ceph-mon[77242]: pgmap v12574: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:27:29.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:27:28 smithi153 ceph-mon[70044]: pgmap v12575: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:27:29.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:27:28 smithi083 ceph-mon[79930]: pgmap v12575: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:27:29.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:27:28 smithi083 ceph-mon[77242]: pgmap v12575: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:27:30.660 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:27:30.948 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:27:30.948 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 3m ago 7h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:27:30.948 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 67s ago 7h 64.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:27:30.949 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 3m ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:27:30.949 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 67s ago 7h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:27:30.949 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 67s ago 7h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:27:30.949 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 3m ago 7h 864M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:27:30.949 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 3m ago 7h 1559M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:27:30.950 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 67s ago 7h 674M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:27:30.950 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 3m ago 7h 902M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:27:30.950 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 3m ago 7h 28.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:27:30.950 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 67s ago 7h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:27:30.951 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 3m ago 7h 1126M 2375M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:27:30.951 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 3m ago 7h 1114M 2375M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:27:30.951 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 3m ago 7h 847M 2375M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:27:30.951 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 3m ago 7h 1090M 2375M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:27:30.951 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 67s ago 7h 936M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:27:30.952 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 67s ago 7h 1198M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:27:30.952 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 67s ago 7h 1071M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:27:30.952 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 67s ago 7h 1270M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:27:30.952 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 67s ago 7h 169M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:27:31.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:27:30 smithi153 ceph-mon[70044]: pgmap v12576: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:27:31.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:27:30 smithi083 ceph-mon[79930]: pgmap v12576: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:27:31.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:27:30 smithi083 ceph-mon[77242]: pgmap v12576: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:27:31.302 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:27:31.303 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:27:31.303 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:27:31.303 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:27:31.303 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:27:31.304 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:27:31.304 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:27:31.304 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:27:31.304 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:27:31.304 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:27:31.305 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:27:31.305 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:27:31.305 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:27:31.305 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:27:31.305 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:27:31.306 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:27:31.306 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:27:31.306 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:27:31.762 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:27:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:27:31] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T12:27:31.763 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:27:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:27:31.717Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:27:32.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:27:31 smithi153 ceph-mon[70044]: from='client.58687 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:27:32.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:27:31 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2709155912' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:27:32.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:27:31 smithi083 ceph-mon[79930]: from='client.58687 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:27:32.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:27:31 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2709155912' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:27:32.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:27:31 smithi083 ceph-mon[77242]: from='client.58687 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:27:32.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:27:31 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2709155912' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:27:32.767 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:27:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:27:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:27:33.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:27:32 smithi153 ceph-mon[70044]: from='client.58693 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:27:33.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:27:32 smithi153 ceph-mon[70044]: pgmap v12577: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:27:33.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:27:32 smithi083 ceph-mon[79930]: from='client.58693 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:27:33.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:27:32 smithi083 ceph-mon[79930]: pgmap v12577: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:27:33.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:27:32 smithi083 ceph-mon[77242]: from='client.58693 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:27:33.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:27:32 smithi083 ceph-mon[77242]: pgmap v12577: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:27:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:27:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:27:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:27:35.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:27:34 smithi153 ceph-mon[70044]: pgmap v12578: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:27:35.093 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:27:34 smithi083 ceph-mon[79930]: pgmap v12578: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:27:35.093 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:27:34 smithi083 ceph-mon[77242]: pgmap v12578: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:27:35.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:27:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:27:35.091Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:27:36.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:27:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:27:35.809Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:27:36.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:27:36 smithi153 ceph-mon[70044]: pgmap v12579: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:27:36.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:27:36 smithi083 ceph-mon[79930]: pgmap v12579: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:27:36.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:27:36 smithi083 ceph-mon[77242]: pgmap v12579: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:27:38.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:27:38 smithi083 ceph-mon[79930]: pgmap v12580: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:27:38.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:27:38 smithi083 ceph-mon[77242]: pgmap v12580: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:27:38.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:27:38 smithi153 ceph-mon[70044]: pgmap v12580: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:27:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:27:40 smithi083 ceph-mon[79930]: pgmap v12581: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:27:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:27:40 smithi083 ceph-mon[77242]: pgmap v12581: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:27:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:27:40 smithi153 ceph-mon[70044]: pgmap v12581: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:27:41.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:27:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:27:41] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T12:27:41.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:27:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:27:41.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:27:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:27:41.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:27:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:27:41.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:27:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:27:41.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:27:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:27:41.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:27:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:27:41.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:27:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:27:41.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:27:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:27:41.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:27:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:27:41.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:27:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:27:41.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:27:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:27:41.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:27:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:27:42.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:27:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:27:41.718Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:27:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:27:42 smithi083 ceph-mon[79930]: pgmap v12582: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:27:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:27:42 smithi083 ceph-mon[77242]: pgmap v12582: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:27:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:27:42 smithi153 ceph-mon[70044]: pgmap v12582: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:27:42.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:27:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:27:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:27:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:27:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:27:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:27:44.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:27:44 smithi083 ceph-mon[77242]: pgmap v12583: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:27:44.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:27:44 smithi083 ceph-mon[79930]: pgmap v12583: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:27:44.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:27:44 smithi153 ceph-mon[70044]: pgmap v12583: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:27:45.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:27:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:27:45.091Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:27:46.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:27:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:27:45.810Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:27:46.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:27:46 smithi153 ceph-mon[70044]: pgmap v12584: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:27:46.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:27:46 smithi083 ceph-mon[77242]: pgmap v12584: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:27:46.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:27:46 smithi083 ceph-mon[79930]: pgmap v12584: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:27:48.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:27:48 smithi083 ceph-mon[77242]: pgmap v12585: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:27:48.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:27:48 smithi083 ceph-mon[79930]: pgmap v12585: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:27:48.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:27:48 smithi153 ceph-mon[70044]: pgmap v12585: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:27:50.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:27:50 smithi083 ceph-mon[79930]: pgmap v12586: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:27:50.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:27:50 smithi083 ceph-mon[77242]: pgmap v12586: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:27:50.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:27:50 smithi153 ceph-mon[70044]: pgmap v12586: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:27:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:27:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:27:51] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T12:27:51.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:27:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:27:51.719Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:27:52.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:27:52 smithi083 ceph-mon[79930]: pgmap v12587: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:27:52.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:27:52 smithi083 ceph-mon[77242]: pgmap v12587: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:27:52.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:27:52 smithi153 ceph-mon[70044]: pgmap v12587: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:27:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:27:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:27:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:27:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:27:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:27:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:27:54.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:27:54 smithi083 ceph-mon[79930]: pgmap v12588: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:27:54.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:27:54 smithi083 ceph-mon[77242]: pgmap v12588: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:27:54.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:27:54 smithi153 ceph-mon[70044]: pgmap v12588: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:27:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:27:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:27:55.091Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:27:56.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:27:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:27:55.810Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:27:56.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:27:56 smithi153 ceph-mon[70044]: pgmap v12589: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:27:56.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:27:56 smithi083 ceph-mon[79930]: pgmap v12589: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:27:56.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:27:56 smithi083 ceph-mon[77242]: pgmap v12589: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:27:58.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:27:58 smithi083 ceph-mon[79930]: pgmap v12590: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:27:58.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:27:58 smithi083 ceph-mon[77242]: pgmap v12590: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:27:58.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:27:58 smithi153 ceph-mon[70044]: pgmap v12590: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:00.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:28:00 smithi083 ceph-mon[79930]: pgmap v12591: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:00.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:28:00 smithi083 ceph-mon[77242]: pgmap v12591: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:00.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:28:00 smithi153 ceph-mon[70044]: pgmap v12591: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:01.632 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:28:01.919 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:28:01.920 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 3m ago 7h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:28:01.920 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 98s ago 7h 64.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:28:01.920 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 3m ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:28:01.920 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 98s ago 7h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:28:01.920 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 98s ago 7h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:28:01.921 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 3m ago 7h 864M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:28:01.921 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 3m ago 7h 1559M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:28:01.921 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 98s ago 7h 674M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:28:01.921 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 3m ago 7h 902M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:28:01.921 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 3m ago 7h 28.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:28:01.922 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 98s ago 7h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:28:01.922 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 3m ago 7h 1126M 2375M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:28:01.922 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 3m ago 7h 1114M 2375M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:28:01.922 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 3m ago 7h 847M 2375M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:28:01.923 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 3m ago 7h 1090M 2375M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:28:01.923 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 98s ago 7h 936M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:28:01.923 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 98s ago 7h 1198M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:28:01.923 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 98s ago 7h 1071M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:28:01.923 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 98s ago 7h 1270M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:28:01.924 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 98s ago 7h 169M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:28:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:28:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:28:01] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T12:28:01.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:28:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:28:01.720Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:28:02.275 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:28:02.275 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:28:02.275 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:28:02.276 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:28:02.276 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:28:02.276 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:28:02.276 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:28:02.277 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:28:02.277 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:28:02.277 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:28:02.277 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:28:02.278 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:28:02.278 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:28:02.278 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:28:02.278 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:28:02.279 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:28:02.279 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:28:02.279 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:28:02.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:28:02 smithi083 ceph-mon[79930]: pgmap v12592: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:02.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:28:02 smithi083 ceph-mon[79930]: from='client.58705 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:28:02.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:28:02 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/932889339' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:28:02.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:28:02 smithi083 ceph-mon[77242]: pgmap v12592: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:02.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:28:02 smithi083 ceph-mon[77242]: from='client.58705 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:28:02.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:28:02 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/932889339' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:28:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:28:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:28:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:28:02.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:28:02 smithi153 ceph-mon[70044]: pgmap v12592: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:02.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:28:02 smithi153 ceph-mon[70044]: from='client.58705 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:28:02.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:28:02 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/932889339' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:28:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:28:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:28:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:28:03.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:28:03 smithi083 ceph-mon[79930]: from='client.48780 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:28:03.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:28:03 smithi083 ceph-mon[77242]: from='client.48780 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:28:03.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:28:03 smithi153 ceph-mon[70044]: from='client.48780 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:28:04.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:28:04 smithi083 ceph-mon[79930]: pgmap v12593: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:04.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:28:04 smithi083 ceph-mon[77242]: pgmap v12593: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:04.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:28:04 smithi153 ceph-mon[70044]: pgmap v12593: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:05.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:28:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:28:05.092Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:28:06.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:28:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:28:05.811Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:28:06.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:28:06 smithi153 ceph-mon[70044]: pgmap v12594: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:06.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:28:06 smithi083 ceph-mon[77242]: pgmap v12594: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:06.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:28:06 smithi083 ceph-mon[79930]: pgmap v12594: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:08.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:28:08 smithi083 ceph-mon[79930]: pgmap v12595: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:08.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:28:08 smithi083 ceph-mon[77242]: pgmap v12595: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:08.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:28:08 smithi153 ceph-mon[70044]: pgmap v12595: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:10.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:28:10 smithi083 ceph-mon[79930]: pgmap v12596: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:10.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:28:10 smithi083 ceph-mon[77242]: pgmap v12596: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:10.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:28:10 smithi153 ceph-mon[70044]: pgmap v12596: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:28:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:28:11] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T12:28:11.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:28:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:28:11.720Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:28:12.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:28:12 smithi083 ceph-mon[77242]: pgmap v12597: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:12.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:28:12 smithi083 ceph-mon[79930]: pgmap v12597: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:12.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:28:12 smithi153 ceph-mon[70044]: pgmap v12597: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:28:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:28:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:28:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:28:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:28:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:28:14.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:28:14 smithi083 ceph-mon[79930]: pgmap v12598: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:14.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:28:14 smithi083 ceph-mon[77242]: pgmap v12598: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:14.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:28:14 smithi153 ceph-mon[70044]: pgmap v12598: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:15.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:28:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:28:15.092Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:28:16.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:28:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:28:15.811Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:28:16.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:28:16 smithi153 ceph-mon[70044]: pgmap v12599: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:16.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:28:16 smithi083 ceph-mon[79930]: pgmap v12599: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:16.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:28:16 smithi083 ceph-mon[77242]: pgmap v12599: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:18.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:28:18 smithi083 ceph-mon[79930]: pgmap v12600: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:18.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:28:18 smithi083 ceph-mon[77242]: pgmap v12600: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:18.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:28:18 smithi153 ceph-mon[70044]: pgmap v12600: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:20.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:28:20 smithi083 ceph-mon[79930]: pgmap v12601: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:20.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:28:20 smithi083 ceph-mon[77242]: pgmap v12601: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:20.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:28:20 smithi153 ceph-mon[70044]: pgmap v12601: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:28:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:28:21] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T12:28:21.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:28:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:28:21.721Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:28:22.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:28:22 smithi083 ceph-mon[79930]: pgmap v12602: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:22.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:28:22 smithi083 ceph-mon[77242]: pgmap v12602: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:22.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:28:22 smithi153 ceph-mon[70044]: pgmap v12602: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:28:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:28:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:28:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:28:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:28:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:28:24.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:28:24 smithi083 ceph-mon[77242]: pgmap v12603: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:24.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:28:24 smithi083 ceph-mon[79930]: pgmap v12603: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:24.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:28:24 smithi153 ceph-mon[70044]: pgmap v12603: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:25.416 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:28:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:28:25.091Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:28:25.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:28:25 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:28:25.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:28:25 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:28:25.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:28:25 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:28:25.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:28:25 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:28:25.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:28:25 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:28:25.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:28:25 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:28:25.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:28:25 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:28:25.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:28:25 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:28:25.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:28:25 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:28:25.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:28:25 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:28:25.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:28:25 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:28:25.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:28:25 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:28:26.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:28:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:28:25.811Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:28:26.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:28:26 smithi083 ceph-mon[79930]: pgmap v12604: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:26.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:28:26 smithi083 ceph-mon[77242]: pgmap v12604: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:26.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:28:26 smithi153 ceph-mon[70044]: pgmap v12604: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:28.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:28:28 smithi083 ceph-mon[79930]: pgmap v12605: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:28.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:28:28 smithi083 ceph-mon[77242]: pgmap v12605: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:28.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:28:28 smithi153 ceph-mon[70044]: pgmap v12605: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:30.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:28:30 smithi083 ceph-mon[79930]: pgmap v12606: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:30.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:28:30 smithi083 ceph-mon[77242]: pgmap v12606: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:30.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:28:30 smithi153 ceph-mon[70044]: pgmap v12606: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:28:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:28:31] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T12:28:31.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:28:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:28:31.721Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:28:32.604 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:28:32.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:28:32 smithi083 ceph-mon[79930]: pgmap v12607: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:32.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:28:32 smithi083 ceph-mon[77242]: pgmap v12607: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:32.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:28:32 smithi153 ceph-mon[70044]: pgmap v12607: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:32.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:28:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:28:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:28:32.890 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:28:32.890 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 4m ago 7h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:28:32.891 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 2m ago 7h 64.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:28:32.891 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 4m ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:28:32.891 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 2m ago 7h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:28:32.891 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 2m ago 7h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:28:32.892 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (6h) 4m ago 7h 864M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:28:32.892 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 4m ago 7h 1559M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:28:32.892 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 2m ago 7h 674M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:28:32.892 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 4m ago 7h 902M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:28:32.893 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 4m ago 7h 28.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:28:32.893 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 2m ago 7h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:28:32.893 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 4m ago 7h 1126M 2375M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:28:32.893 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 4m ago 7h 1114M 2375M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:28:32.893 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 4m ago 7h 847M 2375M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:28:32.894 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 4m ago 7h 1090M 2375M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:28:32.894 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 2m ago 7h 936M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:28:32.894 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 2m ago 7h 1198M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:28:32.894 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 2m ago 7h 1071M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:28:32.895 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 2m ago 7h 1270M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:28:32.895 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 2m ago 7h 169M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:28:33.246 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:28:33.247 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:28:33.247 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:28:33.247 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:28:33.247 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:28:33.248 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:28:33.248 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:28:33.248 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:28:33.248 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:28:33.249 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:28:33.249 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:28:33.249 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:28:33.249 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:28:33.249 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:28:33.250 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:28:33.250 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:28:33.250 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:28:33.251 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:28:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:28:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:28:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:28:33.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:28:33 smithi083 ceph-mon[79930]: from='client.48792 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:28:33.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:28:33 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3883144669' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:28:33.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:28:33 smithi083 ceph-mon[77242]: from='client.48792 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:28:33.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:28:33 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3883144669' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:28:33.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:28:33 smithi153 ceph-mon[70044]: from='client.48792 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:28:33.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:28:33 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3883144669' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:28:34.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:28:34 smithi083 ceph-mon[79930]: from='client.48798 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:28:34.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:28:34 smithi083 ceph-mon[79930]: pgmap v12608: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:34.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:28:34 smithi083 ceph-mon[77242]: from='client.48798 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:28:34.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:28:34 smithi083 ceph-mon[77242]: pgmap v12608: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:34.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:28:34 smithi153 ceph-mon[70044]: from='client.48798 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:28:34.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:28:34 smithi153 ceph-mon[70044]: pgmap v12608: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:35.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:28:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:28:35.093Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:28:36.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:28:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:28:35.812Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:28:36.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:28:36 smithi153 ceph-mon[70044]: pgmap v12609: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:36.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:28:36 smithi083 ceph-mon[79930]: pgmap v12609: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:36.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:28:36 smithi083 ceph-mon[77242]: pgmap v12609: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:38.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:28:38 smithi083 ceph-mon[79930]: pgmap v12610: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:38.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:28:38 smithi083 ceph-mon[77242]: pgmap v12610: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:38.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:28:38 smithi153 ceph-mon[70044]: pgmap v12610: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:28:40 smithi083 ceph-mon[79930]: pgmap v12611: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:28:40 smithi083 ceph-mon[77242]: pgmap v12611: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:28:40 smithi153 ceph-mon[70044]: pgmap v12611: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:41.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:28:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:28:41] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T12:28:41.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:28:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:28:41.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:28:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:28:41.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:28:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:28:41.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:28:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:28:41.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:28:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:28:41.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:28:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:28:41.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:28:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:28:41.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:28:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:28:41.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:28:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:28:41.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:28:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:28:41.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:28:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:28:41.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:28:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:28:42.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:28:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:28:41.722Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:28:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:28:42 smithi083 ceph-mon[79930]: pgmap v12612: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:28:42 smithi083 ceph-mon[77242]: pgmap v12612: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:28:42 smithi153 ceph-mon[70044]: pgmap v12612: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:28:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:28:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:28:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:28:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:28:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:28:44.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:28:44 smithi083 ceph-mon[79930]: pgmap v12613: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:44.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:28:44 smithi083 ceph-mon[77242]: pgmap v12613: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:44.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:28:44 smithi153 ceph-mon[70044]: pgmap v12613: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:45.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:28:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:28:45.094Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:28:46.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:28:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:28:45.813Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:28:46.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:28:46 smithi153 ceph-mon[70044]: pgmap v12614: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:46.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:28:46 smithi083 ceph-mon[77242]: pgmap v12614: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:46.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:28:46 smithi083 ceph-mon[79930]: pgmap v12614: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:48.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:28:48 smithi083 ceph-mon[79930]: pgmap v12615: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:48.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:28:48 smithi083 ceph-mon[77242]: pgmap v12615: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:48.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:28:48 smithi153 ceph-mon[70044]: pgmap v12615: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:50.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:28:50 smithi083 ceph-mon[79930]: pgmap v12616: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:50.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:28:50 smithi083 ceph-mon[77242]: pgmap v12616: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:50.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:28:50 smithi153 ceph-mon[70044]: pgmap v12616: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:28:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:28:51] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T12:28:51.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:28:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:28:51.722Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:28:52.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:28:52 smithi083 ceph-mon[79930]: pgmap v12617: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:52.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:28:52 smithi083 ceph-mon[77242]: pgmap v12617: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:52.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:28:52 smithi153 ceph-mon[70044]: pgmap v12617: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:28:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:28:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:28:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:28:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:28:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:28:54.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:28:54 smithi083 ceph-mon[77242]: pgmap v12618: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:54.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:28:54 smithi083 ceph-mon[79930]: pgmap v12618: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:54.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:28:54 smithi153 ceph-mon[70044]: pgmap v12618: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:28:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:28:55.094Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:28:56.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:28:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:28:55.814Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:28:56.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:28:56 smithi153 ceph-mon[70044]: pgmap v12619: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:56.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:28:56 smithi083 ceph-mon[77242]: pgmap v12619: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:56.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:28:56 smithi083 ceph-mon[79930]: pgmap v12619: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:58.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:28:58 smithi083 ceph-mon[77242]: pgmap v12620: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:58.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:28:58 smithi083 ceph-mon[79930]: pgmap v12620: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:28:58.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:28:58 smithi153 ceph-mon[70044]: pgmap v12620: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:00.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:29:00 smithi083 ceph-mon[79930]: pgmap v12621: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:00.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:29:00 smithi083 ceph-mon[77242]: pgmap v12621: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:00.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:29:00 smithi153 ceph-mon[70044]: pgmap v12621: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:01.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:29:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:29:01] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T12:29:01.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:29:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:29:01.723Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:29:02.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:29:02 smithi083 ceph-mon[79930]: pgmap v12622: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:02.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:29:02 smithi083 ceph-mon[77242]: pgmap v12622: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:02.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:29:02 smithi153 ceph-mon[70044]: pgmap v12622: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:29:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:29:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:29:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:29:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:29:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:29:03.580 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:29:03.875 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:29:03.876 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 4m ago 7h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:29:03.876 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 2m ago 7h 64.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:29:03.876 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 4m ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:29:03.877 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 2m ago 7h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:29:03.877 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (6h) 2m ago 7h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:29:03.877 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 4m ago 7h 864M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:29:03.877 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (6h) 4m ago 7h 1559M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:29:03.878 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 2m ago 7h 674M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:29:03.878 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 4m ago 7h 902M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:29:03.878 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 4m ago 7h 28.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:29:03.878 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 2m ago 7h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:29:03.879 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 4m ago 7h 1126M 2375M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:29:03.879 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 4m ago 7h 1114M 2375M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:29:03.879 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 4m ago 7h 847M 2375M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:29:03.879 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 4m ago 7h 1090M 2375M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:29:03.880 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 2m ago 7h 936M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:29:03.880 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 2m ago 7h 1198M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:29:03.880 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 2m ago 7h 1071M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:29:03.880 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 2m ago 7h 1270M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:29:03.881 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (6h) 2m ago 7h 169M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:29:04.228 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:29:04.229 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:29:04.229 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:29:04.229 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:29:04.229 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:29:04.230 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:29:04.230 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:29:04.230 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:29:04.230 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:29:04.231 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:29:04.231 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:29:04.231 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:29:04.231 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:29:04.232 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:29:04.232 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:29:04.232 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:29:04.232 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:29:04.232 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:29:04.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:29:04 smithi083 ceph-mon[79930]: pgmap v12623: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:04.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:29:04 smithi083 ceph-mon[79930]: from='client.48810 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:29:04.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:29:04 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2743506936' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:29:04.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:29:04 smithi083 ceph-mon[77242]: pgmap v12623: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:04.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:29:04 smithi083 ceph-mon[77242]: from='client.48810 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:29:04.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:29:04 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2743506936' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:29:04.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:29:04 smithi153 ceph-mon[70044]: pgmap v12623: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:04.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:29:04 smithi153 ceph-mon[70044]: from='client.48810 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:29:04.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:29:04 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2743506936' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:29:05.428 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:29:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:29:05.094Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:29:05.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:29:05 smithi083 ceph-mon[79930]: from='client.48816 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:29:05.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:29:05 smithi083 ceph-mon[77242]: from='client.48816 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:29:05.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:29:05 smithi153 ceph-mon[70044]: from='client.48816 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:29:06.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:29:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:29:05.815Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:29:06.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:29:06 smithi083 ceph-mon[79930]: pgmap v12624: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:06.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:29:06 smithi083 ceph-mon[77242]: pgmap v12624: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:06.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:29:06 smithi153 ceph-mon[70044]: pgmap v12624: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:08.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:29:08 smithi083 ceph-mon[79930]: pgmap v12625: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:08.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:29:08 smithi083 ceph-mon[77242]: pgmap v12625: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:08.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:29:08 smithi153 ceph-mon[70044]: pgmap v12625: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:10.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:29:10 smithi083 ceph-mon[79930]: pgmap v12626: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:10.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:29:10 smithi083 ceph-mon[77242]: pgmap v12626: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:10.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:29:10 smithi153 ceph-mon[70044]: pgmap v12626: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:29:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:29:11] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T12:29:11.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:29:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:29:11.724Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:29:12.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:29:12 smithi083 ceph-mon[79930]: pgmap v12627: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:12.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:29:12 smithi083 ceph-mon[77242]: pgmap v12627: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:12.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:29:12 smithi153 ceph-mon[70044]: pgmap v12627: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:29:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:29:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:29:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:29:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:29:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:29:14.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:29:14 smithi083 ceph-mon[79930]: pgmap v12628: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:14.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:29:14 smithi083 ceph-mon[77242]: pgmap v12628: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:14.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:29:14 smithi153 ceph-mon[70044]: pgmap v12628: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:15.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:29:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:29:15.095Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:29:16.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:29:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:29:15.815Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:29:16.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:29:16 smithi153 ceph-mon[70044]: pgmap v12629: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:16.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:29:16 smithi083 ceph-mon[79930]: pgmap v12629: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:16.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:29:16 smithi083 ceph-mon[77242]: pgmap v12629: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:18.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:29:18 smithi083 ceph-mon[79930]: pgmap v12630: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:18.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:29:18 smithi083 ceph-mon[77242]: pgmap v12630: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:18.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:29:18 smithi153 ceph-mon[70044]: pgmap v12630: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:20.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:29:20 smithi083 ceph-mon[79930]: pgmap v12631: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:20.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:29:20 smithi083 ceph-mon[77242]: pgmap v12631: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:20.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:29:20 smithi153 ceph-mon[70044]: pgmap v12631: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:21.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:29:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:29:21] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T12:29:21.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:29:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:29:21.724Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:29:22.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:29:22 smithi083 ceph-mon[79930]: pgmap v12632: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:22.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:29:22 smithi083 ceph-mon[77242]: pgmap v12632: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:22.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:29:22 smithi153 ceph-mon[70044]: pgmap v12632: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:29:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:29:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:29:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:29:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:29:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:29:24.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:29:24 smithi083 ceph-mon[79930]: pgmap v12633: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:24.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:29:24 smithi083 ceph-mon[77242]: pgmap v12633: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:24.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:29:24 smithi153 ceph-mon[70044]: pgmap v12633: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:25.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:29:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:29:25.097Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:29:26.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:29:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:29:25.815Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:29:26.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:29:26 smithi153 ceph-mon[70044]: pgmap v12634: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:26.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:29:26 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:29:26.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:29:26 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:29:26.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:29:26 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:29:26.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:29:26 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:29:26.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:29:26 smithi083 ceph-mon[79930]: pgmap v12634: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:26.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:29:26 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:29:26.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:29:26 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:29:26.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:29:26 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:29:26.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:29:26 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:29:26.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:29:26 smithi083 ceph-mon[77242]: pgmap v12634: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:26.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:29:26 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:29:26.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:29:26 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:29:26.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:29:26 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:29:26.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:29:26 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:29:28.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:29:28 smithi083 ceph-mon[79930]: pgmap v12635: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:28.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:29:28 smithi083 ceph-mon[77242]: pgmap v12635: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:28.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:29:28 smithi153 ceph-mon[70044]: pgmap v12635: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:30.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:29:30 smithi083 ceph-mon[79930]: pgmap v12636: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:30.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:29:30 smithi083 ceph-mon[77242]: pgmap v12636: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:30.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:29:30 smithi153 ceph-mon[70044]: pgmap v12636: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:31.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:29:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:29:31] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T12:29:31.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:29:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:29:31.725Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:29:32.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:29:32 smithi083 ceph-mon[79930]: pgmap v12637: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:32.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:29:32 smithi083 ceph-mon[77242]: pgmap v12637: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:32.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:29:32 smithi153 ceph-mon[70044]: pgmap v12637: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:32.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:29:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:29:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:29:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:29:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:29:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:29:34.558 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:29:34.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:29:34 smithi083 ceph-mon[79930]: pgmap v12638: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:34.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:29:34 smithi083 ceph-mon[77242]: pgmap v12638: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:34.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:29:34 smithi153 ceph-mon[70044]: pgmap v12638: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:34.848 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:29:34.848 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 5m ago 7h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:29:34.848 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 3m ago 7h 64.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:29:34.848 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 5m ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:29:34.849 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 3m ago 7h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:29:34.849 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 3m ago 7h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:29:34.849 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 5m ago 7h 864M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:29:34.849 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 5m ago 7h 1559M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:29:34.850 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 3m ago 7h 674M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:29:34.850 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (6h) 5m ago 7h 902M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:29:34.850 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 5m ago 7h 28.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:29:34.850 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 3m ago 7h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:29:34.851 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 5m ago 7h 1126M 2375M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:29:34.851 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 5m ago 7h 1114M 2375M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:29:34.851 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 5m ago 7h 847M 2375M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:29:34.851 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 5m ago 7h 1090M 2375M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:29:34.851 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 3m ago 7h 936M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:29:34.852 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 3m ago 7h 1198M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:29:34.852 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 3m ago 7h 1071M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:29:34.852 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 3m ago 7h 1270M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:29:34.852 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 3m ago 7h 169M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:29:35.203 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:29:35.203 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:29:35.203 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:29:35.203 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:29:35.204 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:29:35.204 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:29:35.204 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:29:35.204 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:29:35.205 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:29:35.205 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:29:35.205 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:29:35.205 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:29:35.205 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:29:35.206 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:29:35.206 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:29:35.206 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:29:35.206 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:29:35.207 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:29:35.438 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:29:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:29:35.096Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:29:35.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:29:35 smithi083 ceph-mon[77242]: from='client.58759 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:29:35.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:29:35 smithi083 ceph-mon[77242]: from='client.48834 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:29:35.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:29:35 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3426414293' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:29:35.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:29:35 smithi083 ceph-mon[79930]: from='client.58759 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:29:35.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:29:35 smithi083 ceph-mon[79930]: from='client.48834 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:29:35.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:29:35 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3426414293' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:29:35.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:29:35 smithi153 ceph-mon[70044]: from='client.58759 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:29:35.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:29:35 smithi153 ceph-mon[70044]: from='client.48834 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:29:35.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:29:35 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3426414293' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:29:36.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:29:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:29:35.816Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:29:36.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:29:36 smithi083 ceph-mon[79930]: pgmap v12639: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:36.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:29:36 smithi083 ceph-mon[77242]: pgmap v12639: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:36.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:29:36 smithi153 ceph-mon[70044]: pgmap v12639: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:38.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:29:38 smithi083 ceph-mon[79930]: pgmap v12640: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:38.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:29:38 smithi083 ceph-mon[77242]: pgmap v12640: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:38.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:29:38 smithi153 ceph-mon[70044]: pgmap v12640: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:29:40 smithi083 ceph-mon[79930]: pgmap v12641: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:29:40 smithi083 ceph-mon[77242]: pgmap v12641: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:29:40 smithi153 ceph-mon[70044]: pgmap v12641: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:41.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:29:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:29:41] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T12:29:41.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:29:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:29:41.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:29:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:29:41.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:29:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:29:41.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:29:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:29:41.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:29:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:29:41.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:29:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:29:41.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:29:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:29:41.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:29:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:29:41.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:29:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:29:41.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:29:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:29:41.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:29:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:29:41.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:29:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:29:42.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:29:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:29:41.726Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:29:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:29:42 smithi083 ceph-mon[79930]: pgmap v12642: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:29:42 smithi083 ceph-mon[77242]: pgmap v12642: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:29:42 smithi153 ceph-mon[70044]: pgmap v12642: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:42.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:29:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:29:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:29:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:29:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:29:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:29:44.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:29:44 smithi083 ceph-mon[79930]: pgmap v12643: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:44.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:29:44 smithi083 ceph-mon[77242]: pgmap v12643: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:44.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:29:44 smithi153 ceph-mon[70044]: pgmap v12643: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:45.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:29:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:29:45.098Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:29:46.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:29:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:29:45.817Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:29:46.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:29:46 smithi153 ceph-mon[70044]: pgmap v12644: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:46.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:29:46 smithi083 ceph-mon[77242]: pgmap v12644: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:46.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:29:46 smithi083 ceph-mon[79930]: pgmap v12644: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:48.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:29:48 smithi083 ceph-mon[79930]: pgmap v12645: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:48.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:29:48 smithi083 ceph-mon[77242]: pgmap v12645: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:48.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:29:48 smithi153 ceph-mon[70044]: pgmap v12645: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:50.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:29:50 smithi083 ceph-mon[79930]: pgmap v12646: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:50.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:29:50 smithi083 ceph-mon[77242]: pgmap v12646: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:50.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:29:50 smithi153 ceph-mon[70044]: pgmap v12646: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:51.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:29:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:29:51] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T12:29:51.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:29:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:29:51.727Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:29:52.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:29:52 smithi083 ceph-mon[77242]: pgmap v12647: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:52.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:29:52 smithi083 ceph-mon[79930]: pgmap v12647: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:52.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:29:52 smithi153 ceph-mon[70044]: pgmap v12647: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:29:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:29:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:29:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:29:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:29:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:29:54.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:29:54 smithi083 ceph-mon[79930]: pgmap v12648: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:54.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:29:54 smithi083 ceph-mon[77242]: pgmap v12648: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:54.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:29:54 smithi153 ceph-mon[70044]: pgmap v12648: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:29:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:29:55.098Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:29:56.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:29:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:29:55.818Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:29:56.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:29:56 smithi153 ceph-mon[70044]: pgmap v12649: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:56.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:29:56 smithi083 ceph-mon[79930]: pgmap v12649: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:56.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:29:56 smithi083 ceph-mon[77242]: pgmap v12649: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:58.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:29:58 smithi083 ceph-mon[77242]: pgmap v12650: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:58.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:29:58 smithi083 ceph-mon[79930]: pgmap v12650: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:29:58.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:29:58 smithi153 ceph-mon[70044]: pgmap v12650: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:00.438 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:30:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[77219]: 2022-09-29T12:29:59.999+0000 7f160f665700 -1 log_channel(cluster) log [ERR] : overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T12:30:00.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:30:00 smithi083 ceph-mon[77242]: pgmap v12651: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:00.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:30:00 smithi083 ceph-mon[77242]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T12:30:00.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:30:00 smithi083 ceph-mon[79930]: pgmap v12651: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:00.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:30:00 smithi083 ceph-mon[79930]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T12:30:00.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:30:00 smithi153 ceph-mon[70044]: pgmap v12651: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:00.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:30:00 smithi153 ceph-mon[70044]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T12:30:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:30:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:30:01] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T12:30:01.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:30:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:30:01.727Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:30:02.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:30:02 smithi083 ceph-mon[77242]: pgmap v12652: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:02.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:30:02 smithi083 ceph-mon[79930]: pgmap v12652: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:02.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:30:02 smithi153 ceph-mon[70044]: pgmap v12652: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:02.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:30:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:30:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:30:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:30:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:30:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:30:04.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:30:04 smithi083 ceph-mon[77242]: pgmap v12653: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:04.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:30:04 smithi083 ceph-mon[79930]: pgmap v12653: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:04.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:30:04 smithi153 ceph-mon[70044]: pgmap v12653: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:05.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:30:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:30:05.099Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:30:05.534 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:30:05.821 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:30:05.822 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 5m ago 7h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:30:05.822 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 3m ago 7h 64.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:30:05.822 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 5m ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:30:05.822 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 3m ago 7h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:30:05.822 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 3m ago 7h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:30:05.823 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 5m ago 7h 864M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:30:05.823 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 5m ago 7h 1559M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:30:05.823 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (6h) 3m ago 7h 674M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:30:05.823 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 5m ago 7h 902M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:30:05.824 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 5m ago 7h 28.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:30:05.824 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 3m ago 7h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:30:05.824 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 5m ago 7h 1126M 2375M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:30:05.824 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 5m ago 7h 1114M 2375M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:30:05.824 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 5m ago 7h 847M 2375M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:30:05.825 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 5m ago 7h 1090M 2375M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:30:05.825 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 3m ago 7h 936M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:30:05.825 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 3m ago 7h 1198M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:30:05.825 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 3m ago 7h 1071M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:30:05.825 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 3m ago 7h 1270M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:30:05.826 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 3m ago 7h 169M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:30:06.177 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:30:06.178 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:30:06.178 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:30:06.178 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:30:06.178 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:30:06.179 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:30:06.179 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:30:06.179 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:30:06.179 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:30:06.180 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:30:06.180 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:30:06.180 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:30:06.180 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:30:06.180 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:30:06.181 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:30:06.181 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:30:06.181 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:30:06.181 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:30:06.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:30:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:30:05.822Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:30:06.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:30:06 smithi153 ceph-mon[70044]: pgmap v12654: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:06.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:30:06 smithi153 ceph-mon[70044]: from='client.58777 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:30:06.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:30:06 smithi153 ceph-mon[70044]: from='client.58783 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:30:06.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:30:06 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/842829253' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:30:06.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:30:06 smithi083 ceph-mon[77242]: pgmap v12654: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:06.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:30:06 smithi083 ceph-mon[77242]: from='client.58777 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:30:06.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:30:06 smithi083 ceph-mon[77242]: from='client.58783 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:30:06.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:30:06 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/842829253' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:30:06.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:30:06 smithi083 ceph-mon[79930]: pgmap v12654: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:06.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:30:06 smithi083 ceph-mon[79930]: from='client.58777 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:30:06.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:30:06 smithi083 ceph-mon[79930]: from='client.58783 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:30:06.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:30:06 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/842829253' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:30:08.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:30:08 smithi083 ceph-mon[79930]: pgmap v12655: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:08.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:30:08 smithi083 ceph-mon[77242]: pgmap v12655: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:08.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:30:08 smithi153 ceph-mon[70044]: pgmap v12655: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:10.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:30:10 smithi083 ceph-mon[79930]: pgmap v12656: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:10.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:30:10 smithi083 ceph-mon[77242]: pgmap v12656: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:10.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:30:10 smithi153 ceph-mon[70044]: pgmap v12656: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:11.967 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:30:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:30:11] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T12:30:11.967 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:30:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:30:11.728Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:30:12.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:30:12 smithi083 ceph-mon[79930]: pgmap v12657: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:12.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:30:12 smithi083 ceph-mon[77242]: pgmap v12657: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:12.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:30:12 smithi153 ceph-mon[70044]: pgmap v12657: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:30:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:30:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:30:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:30:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:30:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:30:14.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:30:14 smithi083 ceph-mon[79930]: pgmap v12658: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:14.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:30:14 smithi083 ceph-mon[77242]: pgmap v12658: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:14.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:30:14 smithi153 ceph-mon[70044]: pgmap v12658: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:15.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:30:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:30:15.099Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:30:16.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:30:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:30:15.819Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:30:16.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:30:16 smithi153 ceph-mon[70044]: pgmap v12659: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:16.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:30:16 smithi083 ceph-mon[79930]: pgmap v12659: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:16.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:30:16 smithi083 ceph-mon[77242]: pgmap v12659: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:18.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:30:18 smithi083 ceph-mon[79930]: pgmap v12660: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:18.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:30:18 smithi083 ceph-mon[77242]: pgmap v12660: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:18.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:30:18 smithi153 ceph-mon[70044]: pgmap v12660: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:20.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:30:20 smithi083 ceph-mon[79930]: pgmap v12661: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:20.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:30:20 smithi083 ceph-mon[77242]: pgmap v12661: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:20.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:30:20 smithi153 ceph-mon[70044]: pgmap v12661: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:30:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:30:21] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T12:30:21.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:30:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:30:21.729Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:30:22.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:30:22 smithi083 ceph-mon[79930]: pgmap v12662: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:22.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:30:22 smithi083 ceph-mon[77242]: pgmap v12662: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:22.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:30:22 smithi153 ceph-mon[70044]: pgmap v12662: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:30:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:30:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:30:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:30:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:30:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:30:24.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:30:24 smithi083 ceph-mon[79930]: pgmap v12663: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:24.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:30:24 smithi083 ceph-mon[77242]: pgmap v12663: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:24.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:30:24 smithi153 ceph-mon[70044]: pgmap v12663: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:25.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:30:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:30:25.100Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:30:26.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:30:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:30:25.820Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:30:26.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:30:26 smithi153 ceph-mon[70044]: pgmap v12664: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:26.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:30:26 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:30:26.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:30:26 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:30:26.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:30:26 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:30:26.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:30:26 smithi083 ceph-mon[79930]: pgmap v12664: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:26.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:30:26 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:30:26.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:30:26 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:30:26.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:30:26 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:30:26.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:30:26 smithi083 ceph-mon[77242]: pgmap v12664: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:26.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:30:26 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:30:26.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:30:26 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:30:26.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:30:26 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:30:28.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:30:27 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:30:28.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:30:27 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:30:28.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:30:27 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:30:29.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:30:28 smithi153 ceph-mon[70044]: pgmap v12665: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:29.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:30:28 smithi083 ceph-mon[79930]: pgmap v12665: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:29.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:30:28 smithi083 ceph-mon[77242]: pgmap v12665: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:31.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:30:30 smithi153 ceph-mon[70044]: pgmap v12666: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:31.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:30:30 smithi083 ceph-mon[77242]: pgmap v12666: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:31.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:30:30 smithi083 ceph-mon[79930]: pgmap v12666: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:30:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:30:31] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T12:30:31.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:30:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:30:31.731Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:30:32.731 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:30:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:30:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:30:33.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:30:32 smithi153 ceph-mon[70044]: pgmap v12667: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:33.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:30:32 smithi083 ceph-mon[79930]: pgmap v12667: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:33.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:30:32 smithi083 ceph-mon[77242]: pgmap v12667: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:30:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:30:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:30:35.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:30:34 smithi153 ceph-mon[70044]: pgmap v12668: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:35.103 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:30:34 smithi083 ceph-mon[79930]: pgmap v12668: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:35.104 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:30:34 smithi083 ceph-mon[77242]: pgmap v12668: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:35.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:30:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:30:35.101Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:30:36.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:30:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:30:35.821Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:30:36.506 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:30:36.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:30:36 smithi153 ceph-mon[70044]: pgmap v12669: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:36.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:30:36 smithi083 ceph-mon[79930]: pgmap v12669: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:36.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:30:36 smithi083 ceph-mon[77242]: pgmap v12669: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:36.797 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:30:36.798 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 6m ago 7h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:30:36.798 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 4m ago 7h 64.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:30:36.798 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 6m ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:30:36.798 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 4m ago 7h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:30:36.799 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 4m ago 7h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:30:36.799 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 6m ago 7h 864M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:30:36.799 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 6m ago 7h 1559M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:30:36.799 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 4m ago 7h 674M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:30:36.800 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 6m ago 7h 902M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:30:36.800 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 6m ago 7h 28.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:30:36.800 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 4m ago 7h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:30:36.800 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6h) 6m ago 7h 1126M 2375M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:30:36.801 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 6m ago 7h 1114M 2375M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:30:36.801 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 6m ago 7h 847M 2375M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:30:36.801 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 6m ago 7h 1090M 2375M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:30:36.801 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 4m ago 7h 936M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:30:36.801 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 4m ago 7h 1198M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:30:36.802 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 4m ago 7h 1071M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:30:36.802 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 4m ago 7h 1270M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:30:36.802 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 4m ago 7h 169M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:30:37.154 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:30:37.155 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:30:37.155 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:30:37.155 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:30:37.155 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:30:37.156 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:30:37.156 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:30:37.156 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:30:37.156 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:30:37.156 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:30:37.157 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:30:37.157 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:30:37.157 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:30:37.157 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:30:37.157 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:30:37.158 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:30:37.158 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:30:37.158 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:30:37.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:30:37 smithi153 ceph-mon[70044]: from='client.48864 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:30:37.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:30:37 smithi153 ceph-mon[70044]: from='client.48870 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:30:37.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:30:37 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1451642961' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:30:37.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:30:37 smithi083 ceph-mon[79930]: from='client.48864 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:30:37.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:30:37 smithi083 ceph-mon[79930]: from='client.48870 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:30:37.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:30:37 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1451642961' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:30:37.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:30:37 smithi083 ceph-mon[77242]: from='client.48864 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:30:37.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:30:37 smithi083 ceph-mon[77242]: from='client.48870 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:30:37.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:30:37 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1451642961' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:30:38.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:30:38 smithi153 ceph-mon[70044]: pgmap v12670: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:38.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:30:38 smithi083 ceph-mon[79930]: pgmap v12670: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:38.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:30:38 smithi083 ceph-mon[77242]: pgmap v12670: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:30:40 smithi083 ceph-mon[79930]: pgmap v12671: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:30:40 smithi083 ceph-mon[77242]: pgmap v12671: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:30:40 smithi153 ceph-mon[70044]: pgmap v12671: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:41.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:30:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:30:41] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T12:30:41.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:30:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:30:41.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:30:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:30:41.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:30:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:30:41.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:30:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:30:41.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:30:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:30:41.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:30:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:30:41.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:30:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:30:41.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:30:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:30:41.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:30:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:30:41.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:30:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:30:41.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:30:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:30:41.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:30:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:30:42.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:30:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:30:41.730Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:30:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:30:42 smithi083 ceph-mon[79930]: pgmap v12672: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:30:42 smithi083 ceph-mon[77242]: pgmap v12672: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:30:42 smithi153 ceph-mon[70044]: pgmap v12672: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:42.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:30:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:30:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:30:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:30:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:30:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:30:44.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:30:44 smithi083 ceph-mon[79930]: pgmap v12673: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:44.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:30:44 smithi083 ceph-mon[77242]: pgmap v12673: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:44.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:30:44 smithi153 ceph-mon[70044]: pgmap v12673: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:45.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:30:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:30:45.101Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:30:46.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:30:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:30:45.821Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:30:46.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:30:46 smithi153 ceph-mon[70044]: pgmap v12674: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:46.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:30:46 smithi083 ceph-mon[79930]: pgmap v12674: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:46.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:30:46 smithi083 ceph-mon[77242]: pgmap v12674: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:48.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:30:48 smithi083 ceph-mon[79930]: pgmap v12675: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:48.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:30:48 smithi083 ceph-mon[77242]: pgmap v12675: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:48.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:30:48 smithi153 ceph-mon[70044]: pgmap v12675: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:50.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:30:50 smithi083 ceph-mon[79930]: pgmap v12676: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:50.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:30:50 smithi083 ceph-mon[77242]: pgmap v12676: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:50.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:30:50 smithi153 ceph-mon[70044]: pgmap v12676: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:30:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:30:51] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T12:30:51.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:30:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:30:51.731Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:30:52.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:30:52 smithi083 ceph-mon[79930]: pgmap v12677: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:52.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:30:52 smithi083 ceph-mon[77242]: pgmap v12677: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:52.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:30:52 smithi153 ceph-mon[70044]: pgmap v12677: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:30:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:30:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:30:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:30:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:30:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:30:54.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:30:54 smithi083 ceph-mon[79930]: pgmap v12678: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:54.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:30:54 smithi083 ceph-mon[77242]: pgmap v12678: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:54.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:30:54 smithi153 ceph-mon[70044]: pgmap v12678: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:30:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:30:55.102Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:30:56.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:30:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:30:55.822Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:30:56.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:30:56 smithi153 ceph-mon[70044]: pgmap v12679: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:56.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:30:56 smithi083 ceph-mon[79930]: pgmap v12679: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:56.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:30:56 smithi083 ceph-mon[77242]: pgmap v12679: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:58.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:30:58 smithi083 ceph-mon[79930]: pgmap v12680: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:58.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:30:58 smithi083 ceph-mon[77242]: pgmap v12680: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:30:58.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:30:58 smithi153 ceph-mon[70044]: pgmap v12680: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:00.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:31:00 smithi083 ceph-mon[79930]: pgmap v12681: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:00.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:31:00 smithi083 ceph-mon[77242]: pgmap v12681: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:00.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:31:00 smithi153 ceph-mon[70044]: pgmap v12681: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:31:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:31:01] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T12:31:01.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:31:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:31:01.731Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:31:02.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:31:02 smithi083 ceph-mon[79930]: pgmap v12682: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:02.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:31:02 smithi083 ceph-mon[77242]: pgmap v12682: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:02.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:31:02 smithi153 ceph-mon[70044]: pgmap v12682: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:31:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:31:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:31:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:31:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:31:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:31:04.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:31:04 smithi153 ceph-mon[70044]: pgmap v12683: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:04.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:31:04 smithi083 ceph-mon[79930]: pgmap v12683: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:04.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:31:04 smithi083 ceph-mon[77242]: pgmap v12683: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:05.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:31:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:31:05.102Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:31:06.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:31:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:31:05.823Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:31:06.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:31:06 smithi153 ceph-mon[70044]: pgmap v12684: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:06.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:31:06 smithi083 ceph-mon[77242]: pgmap v12684: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:06.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:31:06 smithi083 ceph-mon[79930]: pgmap v12684: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:07.482 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:31:07.768 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:31:07.769 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 6m ago 7h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:31:07.769 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 4m ago 7h 64.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:31:07.769 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 6m ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:31:07.769 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 4m ago 7h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:31:07.770 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 4m ago 7h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:31:07.770 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 6m ago 7h 864M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:31:07.770 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 6m ago 7h 1559M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:31:07.770 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 4m ago 7h 674M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:31:07.771 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 6m ago 7h 902M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:31:07.771 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 6m ago 7h 28.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:31:07.771 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 4m ago 7h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:31:07.771 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 6m ago 7h 1126M 2375M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:31:07.771 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6h) 6m ago 7h 1114M 2375M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:31:07.772 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 6m ago 7h 847M 2375M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:31:07.772 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 6m ago 7h 1090M 2375M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:31:07.772 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 4m ago 7h 936M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:31:07.772 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 4m ago 7h 1198M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:31:07.773 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 4m ago 7h 1071M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:31:07.773 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 4m ago 7h 1270M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:31:07.773 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 4m ago 7h 169M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:31:08.123 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:31:08.123 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:31:08.123 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:31:08.123 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:31:08.124 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:31:08.124 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:31:08.124 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:31:08.124 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:31:08.125 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:31:08.125 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:31:08.125 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:31:08.125 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:31:08.125 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:31:08.126 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:31:08.126 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:31:08.126 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:31:08.126 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:31:08.127 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:31:08.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:31:08 smithi083 ceph-mon[77242]: pgmap v12685: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:08.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:31:08 smithi083 ceph-mon[77242]: from='client.58813 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:31:08.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:31:08 smithi083 ceph-mon[77242]: from='client.58819 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:31:08.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:31:08 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3609489593' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:31:08.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:31:08 smithi083 ceph-mon[79930]: pgmap v12685: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:08.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:31:08 smithi083 ceph-mon[79930]: from='client.58813 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:31:08.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:31:08 smithi083 ceph-mon[79930]: from='client.58819 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:31:08.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:31:08 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3609489593' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:31:08.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:31:08 smithi153 ceph-mon[70044]: pgmap v12685: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:08.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:31:08 smithi153 ceph-mon[70044]: from='client.58813 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:31:08.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:31:08 smithi153 ceph-mon[70044]: from='client.58819 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:31:08.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:31:08 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3609489593' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:31:10.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:31:10 smithi083 ceph-mon[79930]: pgmap v12686: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:10.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:31:10 smithi083 ceph-mon[77242]: pgmap v12686: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:10.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:31:10 smithi153 ceph-mon[70044]: pgmap v12686: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:11.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:31:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:31:11.731Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:31:11.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:31:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:31:11] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T12:31:12.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:31:12 smithi153 ceph-mon[70044]: pgmap v12687: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:12.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:31:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:31:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:31:12.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:31:12 smithi083 ceph-mon[77242]: pgmap v12687: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:12.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:31:12 smithi083 ceph-mon[79930]: pgmap v12687: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:31:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:31:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:31:14.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:31:14 smithi153 ceph-mon[70044]: pgmap v12688: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:14.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:31:14 smithi083 ceph-mon[77242]: pgmap v12688: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:14.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:31:14 smithi083 ceph-mon[79930]: pgmap v12688: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:15.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:31:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:31:15.103Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:31:16.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:31:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:31:15.824Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:31:16.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:31:16 smithi153 ceph-mon[70044]: pgmap v12689: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:16.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:31:16 smithi083 ceph-mon[77242]: pgmap v12689: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:16.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:31:16 smithi083 ceph-mon[79930]: pgmap v12689: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:18.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:31:18 smithi083 ceph-mon[79930]: pgmap v12690: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:18.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:31:18 smithi083 ceph-mon[77242]: pgmap v12690: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:18.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:31:18 smithi153 ceph-mon[70044]: pgmap v12690: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:20.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:31:20 smithi153 ceph-mon[70044]: pgmap v12691: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:20.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:31:20 smithi083 ceph-mon[77242]: pgmap v12691: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:20.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:31:20 smithi083 ceph-mon[79930]: pgmap v12691: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:21.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:31:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:31:21] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T12:31:21.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:31:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:31:21.732Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:31:22.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:31:22 smithi153 ceph-mon[70044]: pgmap v12692: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:31:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:31:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:31:22.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:31:22 smithi083 ceph-mon[77242]: pgmap v12692: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:22.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:31:22 smithi083 ceph-mon[79930]: pgmap v12692: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:31:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:31:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:31:24.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:31:24 smithi153 ceph-mon[70044]: pgmap v12693: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:24.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:31:24 smithi083 ceph-mon[77242]: pgmap v12693: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:24.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:31:24 smithi083 ceph-mon[79930]: pgmap v12693: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:25.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:31:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:31:25.103Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:31:26.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:31:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:31:25.825Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:31:26.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:31:26 smithi153 ceph-mon[70044]: pgmap v12694: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:26.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:31:26 smithi083 ceph-mon[77242]: pgmap v12694: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:26.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:31:26 smithi083 ceph-mon[79930]: pgmap v12694: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:27.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:31:27 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:31:27.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:31:27 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:31:27.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:31:27 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:31:27.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:31:27 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:31:27.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:31:27 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:31:27.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:31:27 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:31:27.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:31:27 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:31:27.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:31:27 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:31:27.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:31:27 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:31:28.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:31:28 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:31:28.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:31:28 smithi083 ceph-mon[79930]: pgmap v12695: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:28.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:31:28 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:31:28.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:31:28 smithi083 ceph-mon[77242]: pgmap v12695: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:28.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:31:28 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:31:28.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:31:28 smithi153 ceph-mon[70044]: pgmap v12695: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:30.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:31:30 smithi153 ceph-mon[70044]: pgmap v12696: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:30.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:31:30 smithi083 ceph-mon[79930]: pgmap v12696: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:30.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:31:30 smithi083 ceph-mon[77242]: pgmap v12696: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:31:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:31:31] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T12:31:31.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:31:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:31:31.733Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:31:32.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:31:32 smithi153 ceph-mon[70044]: pgmap v12697: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:32.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:31:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:31:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:31:32.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:31:32 smithi083 ceph-mon[77242]: pgmap v12697: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:32.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:31:32 smithi083 ceph-mon[79930]: pgmap v12697: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:31:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:31:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:31:34.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:31:34 smithi153 ceph-mon[70044]: pgmap v12698: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:34.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:31:34 smithi083 ceph-mon[79930]: pgmap v12698: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:34.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:31:34 smithi083 ceph-mon[77242]: pgmap v12698: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:35.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:31:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:31:35.104Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:31:36.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:31:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:31:35.826Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:31:36.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:31:36 smithi153 ceph-mon[70044]: pgmap v12699: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:36.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:31:36 smithi083 ceph-mon[77242]: pgmap v12699: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:36.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:31:36 smithi083 ceph-mon[79930]: pgmap v12699: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:38.450 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:31:38.738 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:31:38.738 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 7m ago 7h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:31:38.739 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 5m ago 7h 64.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:31:38.739 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 7m ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:31:38.739 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 5m ago 7h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:31:38.739 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 5m ago 7h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:31:38.740 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 7m ago 7h 864M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:31:38.740 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 7m ago 7h 1559M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:31:38.740 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 5m ago 7h 674M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:31:38.740 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 7m ago 7h 902M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:31:38.741 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 7m ago 7h 28.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:31:38.741 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 5m ago 7h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:31:38.741 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 7m ago 7h 1126M 2375M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:31:38.741 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 7m ago 7h 1114M 2375M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:31:38.741 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6h) 7m ago 7h 847M 2375M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:31:38.742 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 7m ago 7h 1090M 2375M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:31:38.742 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 5m ago 7h 936M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:31:38.742 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 5m ago 7h 1198M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:31:38.743 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 5m ago 7h 1071M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:31:38.743 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 5m ago 7h 1270M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:31:38.743 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 5m ago 7h 169M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:31:38.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:31:38 smithi153 ceph-mon[70044]: pgmap v12700: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:38.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:31:38 smithi083 ceph-mon[77242]: pgmap v12700: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:38.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:31:38 smithi083 ceph-mon[79930]: pgmap v12700: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:39.092 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:31:39.093 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:31:39.093 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:31:39.093 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:31:39.093 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:31:39.094 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:31:39.094 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:31:39.094 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:31:39.094 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:31:39.094 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:31:39.095 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:31:39.095 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:31:39.095 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:31:39.095 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:31:39.096 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:31:39.096 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:31:39.096 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:31:39.096 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:31:39.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:31:39 smithi153 ceph-mon[70044]: from='client.58831 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:31:39.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:31:39 smithi153 ceph-mon[70044]: from='client.48906 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:31:39.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:31:39 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2027535080' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:31:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:31:39 smithi083 ceph-mon[79930]: from='client.58831 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:31:39.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:31:39 smithi083 ceph-mon[79930]: from='client.48906 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:31:39.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:31:39 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2027535080' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:31:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:31:39 smithi083 ceph-mon[77242]: from='client.58831 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:31:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:31:39 smithi083 ceph-mon[77242]: from='client.48906 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:31:39.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:31:39 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2027535080' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:31:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:31:40 smithi153 ceph-mon[70044]: pgmap v12701: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:31:40 smithi083 ceph-mon[79930]: pgmap v12701: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:31:40 smithi083 ceph-mon[77242]: pgmap v12701: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:41.736 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:31:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:31:41] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T12:31:41.736 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:31:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:31:41.737 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:31:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:31:41.737 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:31:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:31:41.737 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:31:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:31:41.737 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:31:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:31:41.738 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:31:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:31:41.738 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:31:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:31:41.738 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:31:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:31:41.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:31:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:31:41.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:31:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:31:41.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:31:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:31:41.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:31:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:31:42.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:31:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:31:41.733Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:31:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:31:42 smithi153 ceph-mon[70044]: pgmap v12702: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:42.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:31:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:31:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:31:42.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:31:42 smithi083 ceph-mon[79930]: pgmap v12702: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:42.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:31:42 smithi083 ceph-mon[77242]: pgmap v12702: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:31:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:31:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:31:44.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:31:44 smithi153 ceph-mon[70044]: pgmap v12703: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:44.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:31:44 smithi083 ceph-mon[79930]: pgmap v12703: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:44.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:31:44 smithi083 ceph-mon[77242]: pgmap v12703: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:45.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:31:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:31:45.105Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:31:46.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:31:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:31:45.826Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:31:46.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:31:46 smithi153 ceph-mon[70044]: pgmap v12704: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:46.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:31:46 smithi083 ceph-mon[77242]: pgmap v12704: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:46.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:31:46 smithi083 ceph-mon[79930]: pgmap v12704: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:48.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:31:48 smithi153 ceph-mon[70044]: pgmap v12705: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:48.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:31:48 smithi083 ceph-mon[77242]: pgmap v12705: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:48.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:31:48 smithi083 ceph-mon[79930]: pgmap v12705: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:50.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:31:50 smithi153 ceph-mon[70044]: pgmap v12706: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:50.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:31:50 smithi083 ceph-mon[77242]: pgmap v12706: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:50.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:31:50 smithi083 ceph-mon[79930]: pgmap v12706: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:31:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:31:51] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T12:31:51.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:31:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:31:51.734Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:31:52.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:31:52 smithi153 ceph-mon[70044]: pgmap v12707: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:52.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:31:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:31:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:31:52.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:31:52 smithi083 ceph-mon[79930]: pgmap v12707: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:52.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:31:52 smithi083 ceph-mon[77242]: pgmap v12707: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:31:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:31:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:31:54.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:31:54 smithi153 ceph-mon[70044]: pgmap v12708: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:54.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:31:54 smithi083 ceph-mon[77242]: pgmap v12708: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:54.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:31:54 smithi083 ceph-mon[79930]: pgmap v12708: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:31:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:31:55.106Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:31:56.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:31:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:31:55.827Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:31:56.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:31:56 smithi153 ceph-mon[70044]: pgmap v12709: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:56.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:31:56 smithi083 ceph-mon[77242]: pgmap v12709: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:56.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:31:56 smithi083 ceph-mon[79930]: pgmap v12709: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:58.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:31:58 smithi153 ceph-mon[70044]: pgmap v12710: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:58.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:31:58 smithi083 ceph-mon[79930]: pgmap v12710: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:31:58.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:31:58 smithi083 ceph-mon[77242]: pgmap v12710: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:00.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:32:00 smithi153 ceph-mon[70044]: pgmap v12711: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:00.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:32:00 smithi083 ceph-mon[79930]: pgmap v12711: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:00.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:32:00 smithi083 ceph-mon[77242]: pgmap v12711: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:01.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:32:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:32:01] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T12:32:01.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:32:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:32:01.735Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:32:02.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:32:02 smithi153 ceph-mon[70044]: pgmap v12712: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:02.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:32:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:32:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:32:02.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:32:02 smithi083 ceph-mon[77242]: pgmap v12712: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:02.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:32:02 smithi083 ceph-mon[79930]: pgmap v12712: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:32:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:32:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:32:04.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:32:04 smithi153 ceph-mon[70044]: pgmap v12713: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:04.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:32:04 smithi083 ceph-mon[79930]: pgmap v12713: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:04.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:32:04 smithi083 ceph-mon[77242]: pgmap v12713: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:05.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:32:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:32:05.106Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:32:06.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:32:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:32:05.828Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:32:06.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:32:06 smithi153 ceph-mon[70044]: pgmap v12714: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:06.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:32:06 smithi083 ceph-mon[79930]: pgmap v12714: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:06.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:32:06 smithi083 ceph-mon[77242]: pgmap v12714: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:08.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:32:08 smithi153 ceph-mon[70044]: pgmap v12715: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:08.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:32:08 smithi083 ceph-mon[79930]: pgmap v12715: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:08.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:32:08 smithi083 ceph-mon[77242]: pgmap v12715: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:09.420 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:32:09.716 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:32:09.717 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 7m ago 7h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:32:09.717 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 5m ago 7h 64.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:32:09.717 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 7m ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:32:09.718 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 5m ago 7h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:32:09.718 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 5m ago 7h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:32:09.718 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 7m ago 7h 864M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:32:09.718 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 7m ago 7h 1559M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:32:09.719 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 5m ago 7h 674M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:32:09.719 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 7m ago 7h 902M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:32:09.719 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 7m ago 7h 28.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:32:09.719 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 5m ago 7h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:32:09.719 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 7m ago 7h 1126M 2375M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:32:09.720 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 7m ago 7h 1114M 2375M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:32:09.720 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 7m ago 7h 847M 2375M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:32:09.720 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6h) 7m ago 7h 1090M 2375M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:32:09.720 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 5m ago 7h 936M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:32:09.720 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 5m ago 7h 1198M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:32:09.721 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 5m ago 7h 1071M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:32:09.721 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 5m ago 7h 1270M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:32:09.721 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 5m ago 7h 169M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:32:10.073 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:32:10.073 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:32:10.074 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:32:10.074 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:32:10.074 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:32:10.074 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:32:10.075 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:32:10.075 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:32:10.075 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:32:10.075 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:32:10.075 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:32:10.076 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:32:10.076 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:32:10.076 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:32:10.076 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:32:10.077 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:32:10.077 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:32:10.077 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:32:10.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:32:10 smithi153 ceph-mon[70044]: from='client.48918 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:32:10.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:32:10 smithi153 ceph-mon[70044]: pgmap v12716: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:10.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:32:10 smithi153 ceph-mon[70044]: from='client.48924 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:32:10.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:32:10 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/343784700' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:32:10.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:32:10 smithi083 ceph-mon[79930]: from='client.48918 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:32:10.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:32:10 smithi083 ceph-mon[79930]: pgmap v12716: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:10.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:32:10 smithi083 ceph-mon[79930]: from='client.48924 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:32:10.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:32:10 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/343784700' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:32:10.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:32:10 smithi083 ceph-mon[77242]: from='client.48918 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:32:10.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:32:10 smithi083 ceph-mon[77242]: pgmap v12716: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:10.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:32:10 smithi083 ceph-mon[77242]: from='client.48924 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:32:10.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:32:10 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/343784700' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:32:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:32:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:32:11] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T12:32:11.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:32:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:32:11.735Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:32:12.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:32:12 smithi153 ceph-mon[70044]: pgmap v12717: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:12.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:32:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:32:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:32:12.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:32:12 smithi083 ceph-mon[79930]: pgmap v12717: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:12.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:32:12 smithi083 ceph-mon[77242]: pgmap v12717: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:32:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:32:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:32:14.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:32:14 smithi153 ceph-mon[70044]: pgmap v12718: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:14.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:32:14 smithi083 ceph-mon[77242]: pgmap v12718: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:14.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:32:14 smithi083 ceph-mon[79930]: pgmap v12718: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:15.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:32:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:32:15.107Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:32:16.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:32:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:32:15.828Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:32:16.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:32:16 smithi153 ceph-mon[70044]: pgmap v12719: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:16.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:32:16 smithi083 ceph-mon[77242]: pgmap v12719: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:16.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:32:16 smithi083 ceph-mon[79930]: pgmap v12719: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:18.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:32:18 smithi153 ceph-mon[70044]: pgmap v12720: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:18.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:32:18 smithi083 ceph-mon[77242]: pgmap v12720: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:18.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:32:18 smithi083 ceph-mon[79930]: pgmap v12720: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:20.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:32:20 smithi153 ceph-mon[70044]: pgmap v12721: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:20.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:32:20 smithi083 ceph-mon[79930]: pgmap v12721: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:20.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:32:20 smithi083 ceph-mon[77242]: pgmap v12721: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:32:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:32:21] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T12:32:21.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:32:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:32:21.736Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:32:22.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:32:22 smithi153 ceph-mon[70044]: pgmap v12722: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:22.830 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:32:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:32:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:32:22.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:32:22 smithi083 ceph-mon[77242]: pgmap v12722: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:22.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:32:22 smithi083 ceph-mon[79930]: pgmap v12722: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:32:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:32:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:32:24.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:32:24 smithi153 ceph-mon[70044]: pgmap v12723: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:24.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:32:24 smithi083 ceph-mon[79930]: pgmap v12723: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:24.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:32:24 smithi083 ceph-mon[77242]: pgmap v12723: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:25.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:32:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:32:25.107Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:32:26.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:32:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:32:25.828Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:32:26.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:32:26 smithi153 ceph-mon[70044]: pgmap v12724: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:26.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:32:26 smithi083 ceph-mon[77242]: pgmap v12724: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:26.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:32:26 smithi083 ceph-mon[79930]: pgmap v12724: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:27.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:32:27 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:32:27.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:32:27 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:32:27.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:32:27 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:32:27.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:32:27 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:32:27.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:32:27 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:32:27.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:32:27 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:32:27.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:32:27 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:32:27.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:32:27 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:32:27.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:32:27 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:32:28.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:32:28 smithi153 ceph-mon[70044]: pgmap v12725: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:28.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:32:28 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:32:28.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:32:28 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:32:28.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:32:28 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:32:28.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:32:28 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:32:28.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:32:28 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:32:28.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:32:28 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:32:28.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:32:28 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:32:28.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:32:28 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:32:28.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:32:28 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:32:28.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:32:28 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:32:28.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:32:28 smithi083 ceph-mon[79930]: pgmap v12725: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:28.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:32:28 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:32:28.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:32:28 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:32:28.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:32:28 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:32:28.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:32:28 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:32:28.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:32:28 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:32:28.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:32:28 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:32:28.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:32:28 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:32:28.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:32:28 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:32:28.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:32:28 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:32:28.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:32:28 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:32:28.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:32:28 smithi083 ceph-mon[77242]: pgmap v12725: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:28.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:32:28 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:32:28.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:32:28 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:32:28.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:32:28 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:32:28.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:32:28 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:32:28.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:32:28 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:32:28.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:32:28 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:32:28.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:32:28 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:32:28.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:32:28 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:32:28.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:32:28 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:32:28.968 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:32:28 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:32:29.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:32:29 smithi153 ceph-mon[70044]: Adjusting osd_memory_target on smithi083 to 2366M 2022-09-29T12:32:29.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:32:29 smithi083 ceph-mon[79930]: Adjusting osd_memory_target on smithi083 to 2366M 2022-09-29T12:32:29.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:32:29 smithi083 ceph-mon[77242]: Adjusting osd_memory_target on smithi083 to 2366M 2022-09-29T12:32:30.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:32:30 smithi153 ceph-mon[70044]: pgmap v12726: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:30.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:32:30 smithi083 ceph-mon[77242]: pgmap v12726: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:30.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:32:30 smithi083 ceph-mon[79930]: pgmap v12726: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:31.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:32:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:32:31.737Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:32:31.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:32:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:32:31] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T12:32:32.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:32:32 smithi153 ceph-mon[70044]: pgmap v12727: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:32:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:32:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:32:32.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:32:32 smithi083 ceph-mon[79930]: pgmap v12727: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:32.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:32:32 smithi083 ceph-mon[77242]: pgmap v12727: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:32:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:32:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:32:34.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:32:34 smithi153 ceph-mon[70044]: pgmap v12728: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:34.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:32:34 smithi083 ceph-mon[79930]: pgmap v12728: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:34.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:32:34 smithi083 ceph-mon[77242]: pgmap v12728: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:35.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:32:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:32:35.108Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:32:36.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:32:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:32:35.828Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:32:36.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:32:36 smithi153 ceph-mon[70044]: pgmap v12729: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:36.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:32:36 smithi083 ceph-mon[79930]: pgmap v12729: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:36.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:32:36 smithi083 ceph-mon[77242]: pgmap v12729: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:38.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:32:38 smithi153 ceph-mon[70044]: pgmap v12730: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:38.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:32:38 smithi083 ceph-mon[79930]: pgmap v12730: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:38.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:32:38 smithi083 ceph-mon[77242]: pgmap v12730: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:40.405 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:32:40.693 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:32:40.694 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 8m ago 7h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:32:40.694 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 6m ago 7h 64.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:32:40.694 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 8m ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:32:40.695 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 6m ago 7h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:32:40.695 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 6m ago 7h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:32:40.695 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 8m ago 7h 864M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:32:40.695 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 8m ago 7h 1559M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:32:40.696 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 6m ago 7h 674M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:32:40.696 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 8m ago 7h 902M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:32:40.696 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 8m ago 7h 28.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:32:40.696 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 6m ago 7h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:32:40.696 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 8m ago 7h 1126M 2366M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:32:40.697 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 8m ago 7h 1114M 2366M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:32:40.697 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 8m ago 7h 847M 2366M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:32:40.697 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 8m ago 7h 1090M 2366M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:32:40.697 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (6h) 6m ago 7h 936M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:32:40.697 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 6m ago 7h 1198M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:32:40.698 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 6m ago 7h 1071M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:32:40.698 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 6m ago 7h 1270M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:32:40.698 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 6m ago 7h 169M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:32:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:32:40 smithi153 ceph-mon[70044]: pgmap v12731: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:32:40 smithi083 ceph-mon[79930]: pgmap v12731: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:32:40 smithi083 ceph-mon[77242]: pgmap v12731: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:41.053 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:32:41.053 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:32:41.053 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:32:41.053 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:32:41.054 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:32:41.054 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:32:41.054 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:32:41.054 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:32:41.055 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:32:41.055 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:32:41.055 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:32:41.055 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:32:41.055 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:32:41.056 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:32:41.056 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:32:41.056 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:32:41.056 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:32:41.056 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:32:41.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:32:41 smithi153 ceph-mon[70044]: from='client.48936 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:32:41.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:32:41 smithi153 ceph-mon[70044]: from='client.48942 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:32:41.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:32:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:32:41.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:32:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:32:41.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:32:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:32:41.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:32:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:32:41.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:32:41 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/152641968' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:32:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:32:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:32:41] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T12:32:41.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:32:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:32:41.737Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:32:41.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:32:41 smithi083 ceph-mon[79930]: from='client.48936 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:32:41.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:32:41 smithi083 ceph-mon[79930]: from='client.48942 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:32:41.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:32:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:32:41.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:32:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:32:41.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:32:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:32:41.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:32:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:32:41.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:32:41 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/152641968' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:32:41.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:32:41 smithi083 ceph-mon[77242]: from='client.48936 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:32:41.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:32:41 smithi083 ceph-mon[77242]: from='client.48942 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:32:41.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:32:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:32:41.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:32:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:32:41.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:32:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:32:41.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:32:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:32:41.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:32:41 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/152641968' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:32:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:32:42 smithi153 ceph-mon[70044]: pgmap v12732: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:32:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:32:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:32:42.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:32:42 smithi083 ceph-mon[77242]: pgmap v12732: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:42.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:32:42 smithi083 ceph-mon[79930]: pgmap v12732: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:32:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:32:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:32:44.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:32:44 smithi153 ceph-mon[70044]: pgmap v12733: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:44.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:32:44 smithi083 ceph-mon[79930]: pgmap v12733: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:44.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:32:44 smithi083 ceph-mon[77242]: pgmap v12733: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:45.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:32:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:32:45.108Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:32:46.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:32:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:32:45.830Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:32:46.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:32:46 smithi153 ceph-mon[70044]: pgmap v12734: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:46.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:32:46 smithi083 ceph-mon[79930]: pgmap v12734: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:46.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:32:46 smithi083 ceph-mon[77242]: pgmap v12734: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:48.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:32:48 smithi153 ceph-mon[70044]: pgmap v12735: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:48.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:32:48 smithi083 ceph-mon[79930]: pgmap v12735: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:48.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:32:48 smithi083 ceph-mon[77242]: pgmap v12735: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:50.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:32:50 smithi153 ceph-mon[70044]: pgmap v12736: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:50.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:32:50 smithi083 ceph-mon[79930]: pgmap v12736: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:50.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:32:50 smithi083 ceph-mon[77242]: pgmap v12736: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:51.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:32:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:32:51] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T12:32:51.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:32:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:32:51.739Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:32:52.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:32:52 smithi153 ceph-mon[70044]: pgmap v12737: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:52.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:32:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:32:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:32:52.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:32:52 smithi083 ceph-mon[79930]: pgmap v12737: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:52.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:32:52 smithi083 ceph-mon[77242]: pgmap v12737: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:32:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:32:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:32:54.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:32:54 smithi153 ceph-mon[70044]: pgmap v12738: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:54.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:32:54 smithi083 ceph-mon[79930]: pgmap v12738: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:54.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:32:54 smithi083 ceph-mon[77242]: pgmap v12738: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:32:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:32:55.109Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:32:56.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:32:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:32:55.831Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:32:56.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:32:56 smithi153 ceph-mon[70044]: pgmap v12739: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:56.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:32:56 smithi083 ceph-mon[79930]: pgmap v12739: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:56.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:32:56 smithi083 ceph-mon[77242]: pgmap v12739: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:58.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:32:58 smithi153 ceph-mon[70044]: pgmap v12740: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:58.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:32:58 smithi083 ceph-mon[79930]: pgmap v12740: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:32:58.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:32:58 smithi083 ceph-mon[77242]: pgmap v12740: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:00.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:33:00 smithi153 ceph-mon[70044]: pgmap v12741: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:00.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:33:00 smithi083 ceph-mon[79930]: pgmap v12741: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:00.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:33:00 smithi083 ceph-mon[77242]: pgmap v12741: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:01.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:33:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:33:01] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T12:33:01.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:33:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:33:01.739Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:33:02.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:33:02 smithi153 ceph-mon[70044]: pgmap v12742: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:33:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:33:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:33:02.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:33:02 smithi083 ceph-mon[79930]: pgmap v12742: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:02.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:33:02 smithi083 ceph-mon[77242]: pgmap v12742: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:33:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:33:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:33:04.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:33:04 smithi153 ceph-mon[70044]: pgmap v12743: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:04.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:33:04 smithi083 ceph-mon[79930]: pgmap v12743: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:04.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:33:04 smithi083 ceph-mon[77242]: pgmap v12743: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:05.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:33:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:33:05.109Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:33:06.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:33:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:33:05.831Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:33:06.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:33:06 smithi153 ceph-mon[70044]: pgmap v12744: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:06.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:33:06 smithi083 ceph-mon[77242]: pgmap v12744: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:06.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:33:06 smithi083 ceph-mon[79930]: pgmap v12744: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:08.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:33:08 smithi153 ceph-mon[70044]: pgmap v12745: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:08.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:33:08 smithi083 ceph-mon[79930]: pgmap v12745: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:08.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:33:08 smithi083 ceph-mon[77242]: pgmap v12745: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:10.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:33:10 smithi153 ceph-mon[70044]: pgmap v12746: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:10.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:33:10 smithi083 ceph-mon[79930]: pgmap v12746: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:10.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:33:10 smithi083 ceph-mon[77242]: pgmap v12746: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:11.379 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:33:11.669 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:33:11.670 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 8m ago 7h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:33:11.670 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 6m ago 7h 64.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:33:11.670 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 8m ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:33:11.670 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 6m ago 7h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:33:11.671 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 6m ago 7h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:33:11.671 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 8m ago 7h 864M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:33:11.671 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 8m ago 7h 1559M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:33:11.671 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 6m ago 7h 674M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:33:11.671 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 8m ago 7h 902M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:33:11.672 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 8m ago 7h 28.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:33:11.672 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 6m ago 7h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:33:11.672 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 8m ago 7h 1126M 2366M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:33:11.672 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 8m ago 7h 1114M 2366M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:33:11.672 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 8m ago 7h 847M 2366M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:33:11.673 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 8m ago 7h 1090M 2366M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:33:11.673 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 6m ago 7h 936M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:33:11.673 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (6h) 6m ago 7h 1198M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:33:11.673 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 6m ago 7h 1071M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:33:11.673 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 6m ago 7h 1270M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:33:11.674 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 6m ago 7h 169M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:33:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:33:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:33:11] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T12:33:11.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:33:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:33:11.740Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:33:12.027 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:33:12.027 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:33:12.027 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:33:12.027 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:33:12.028 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:33:12.028 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:33:12.028 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:33:12.028 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:33:12.029 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:33:12.029 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:33:12.029 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:33:12.029 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:33:12.029 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:33:12.030 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:33:12.030 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:33:12.030 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:33:12.030 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:33:12.031 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:33:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:33:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:33:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:33:12.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:33:12 smithi153 ceph-mon[70044]: from='client.58885 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:33:12.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:33:12 smithi153 ceph-mon[70044]: pgmap v12747: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:12.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:33:12 smithi153 ceph-mon[70044]: from='client.58891 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:33:12.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:33:12 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3233502408' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:33:12.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:33:12 smithi083 ceph-mon[79930]: from='client.58885 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:33:12.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:33:12 smithi083 ceph-mon[79930]: pgmap v12747: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:12.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:33:12 smithi083 ceph-mon[79930]: from='client.58891 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:33:12.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:33:12 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3233502408' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:33:12.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:33:12 smithi083 ceph-mon[77242]: from='client.58885 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:33:12.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:33:12 smithi083 ceph-mon[77242]: pgmap v12747: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:12.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:33:12 smithi083 ceph-mon[77242]: from='client.58891 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:33:12.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:33:12 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3233502408' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:33:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:33:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:33:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:33:14.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:33:14 smithi153 ceph-mon[70044]: pgmap v12748: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:14.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:33:14 smithi083 ceph-mon[79930]: pgmap v12748: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:14.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:33:14 smithi083 ceph-mon[77242]: pgmap v12748: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:15.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:33:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:33:15.110Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:33:16.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:33:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:33:15.831Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:33:16.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:33:16 smithi153 ceph-mon[70044]: pgmap v12749: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:16.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:33:16 smithi083 ceph-mon[79930]: pgmap v12749: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:16.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:33:16 smithi083 ceph-mon[77242]: pgmap v12749: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:18.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:33:18 smithi153 ceph-mon[70044]: pgmap v12750: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:18.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:33:18 smithi083 ceph-mon[79930]: pgmap v12750: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:18.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:33:18 smithi083 ceph-mon[77242]: pgmap v12750: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:20.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:33:20 smithi153 ceph-mon[70044]: pgmap v12751: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:20.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:33:20 smithi083 ceph-mon[79930]: pgmap v12751: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:20.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:33:20 smithi083 ceph-mon[77242]: pgmap v12751: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:33:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:33:21] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T12:33:21.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:33:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:33:21.741Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:33:22.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:33:22 smithi153 ceph-mon[70044]: pgmap v12752: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:33:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:33:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:33:22.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:33:22 smithi083 ceph-mon[79930]: pgmap v12752: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:22.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:33:22 smithi083 ceph-mon[77242]: pgmap v12752: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:33:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:33:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:33:24.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:33:24 smithi153 ceph-mon[70044]: pgmap v12753: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:24.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:33:24 smithi083 ceph-mon[79930]: pgmap v12753: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:24.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:33:24 smithi083 ceph-mon[77242]: pgmap v12753: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:25.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:33:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:33:25.110Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:33:26.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:33:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:33:25.832Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:33:26.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:33:26 smithi153 ceph-mon[70044]: pgmap v12754: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:26.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:33:26 smithi083 ceph-mon[79930]: pgmap v12754: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:26.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:33:26 smithi083 ceph-mon[77242]: pgmap v12754: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:28.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:33:28 smithi153 ceph-mon[70044]: pgmap v12755: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:28.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:33:28 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:33:28.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:33:28 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:33:28.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:33:28 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:33:28.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:33:28 smithi083 ceph-mon[79930]: pgmap v12755: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:28.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:33:28 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:33:28.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:33:28 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:33:28.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:33:28 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:33:28.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:33:28 smithi083 ceph-mon[77242]: pgmap v12755: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:28.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:33:28 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:33:28.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:33:28 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:33:28.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:33:28 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:33:30.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:33:29 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:33:30.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:33:29 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:33:30.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:33:29 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:33:31.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:33:30 smithi153 ceph-mon[70044]: pgmap v12756: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:31.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:33:30 smithi083 ceph-mon[79930]: pgmap v12756: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:31.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:33:30 smithi083 ceph-mon[77242]: pgmap v12756: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:33:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:33:31] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T12:33:31.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:33:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:33:31.741Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:33:32.733 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:33:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:33:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:33:33.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:33:32 smithi153 ceph-mon[70044]: pgmap v12757: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:33.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:33:32 smithi083 ceph-mon[77242]: pgmap v12757: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:33.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:33:32 smithi083 ceph-mon[79930]: pgmap v12757: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:33:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:33:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:33:35.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:33:34 smithi153 ceph-mon[70044]: pgmap v12758: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:35.113 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:33:34 smithi083 ceph-mon[79930]: pgmap v12758: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:35.114 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:33:34 smithi083 ceph-mon[77242]: pgmap v12758: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:35.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:33:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:33:35.111Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:33:36.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:33:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:33:35.833Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:33:36.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:33:36 smithi153 ceph-mon[70044]: pgmap v12759: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:36.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:33:36 smithi083 ceph-mon[77242]: pgmap v12759: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:36.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:33:36 smithi083 ceph-mon[79930]: pgmap v12759: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:38.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:33:38 smithi153 ceph-mon[70044]: pgmap v12760: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:38.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:33:38 smithi083 ceph-mon[79930]: pgmap v12760: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:38.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:33:38 smithi083 ceph-mon[77242]: pgmap v12760: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:33:40 smithi153 ceph-mon[70044]: pgmap v12761: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:33:40 smithi083 ceph-mon[79930]: pgmap v12761: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:33:40 smithi083 ceph-mon[77242]: pgmap v12761: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:41.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:33:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:33:41.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:33:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:33:41.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:33:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:33:41.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:33:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:33:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:33:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:33:41] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T12:33:41.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:33:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:33:41.742Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:33:41.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:33:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:33:41.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:33:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:33:41.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:33:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:33:41.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:33:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:33:41.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:33:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:33:41.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:33:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:33:41.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:33:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:33:41.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:33:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:33:42.349 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:33:42.637 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:33:42.638 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 9m ago 7h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:33:42.638 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 7m ago 7h 64.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:33:42.638 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 9m ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:33:42.638 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 7m ago 7h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:33:42.639 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 7m ago 7h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:33:42.639 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 9m ago 7h 864M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:33:42.639 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 9m ago 7h 1559M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:33:42.639 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 7m ago 7h 674M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:33:42.640 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 9m ago 7h 902M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:33:42.640 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 9m ago 7h 28.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:33:42.640 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 7m ago 7h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:33:42.640 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 9m ago 7h 1126M 2366M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:33:42.641 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 9m ago 7h 1114M 2366M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:33:42.641 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 9m ago 7h 847M 2366M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:33:42.641 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 9m ago 7h 1090M 2366M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:33:42.641 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 7m ago 7h 936M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:33:42.642 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 7m ago 7h 1198M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:33:42.642 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (6h) 7m ago 7h 1071M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:33:42.642 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 7m ago 7h 1270M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:33:42.643 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 7m ago 7h 169M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:33:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:33:42 smithi153 ceph-mon[70044]: pgmap v12762: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:42.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:33:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:33:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:33:42.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:33:42 smithi083 ceph-mon[79930]: pgmap v12762: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:42.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:33:42 smithi083 ceph-mon[77242]: pgmap v12762: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:42.995 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:33:42.995 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:33:42.995 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:33:42.996 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:33:42.996 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:33:42.996 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:33:42.996 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:33:42.996 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:33:42.997 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:33:42.997 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:33:42.997 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:33:42.997 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:33:42.998 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:33:42.998 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:33:42.998 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:33:42.998 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:33:42.998 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:33:42.999 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:33:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:33:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:33:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:33:43.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:33:43 smithi153 ceph-mon[70044]: from='client.48972 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:33:43.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:33:43 smithi153 ceph-mon[70044]: from='client.48978 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:33:43.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:33:43 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4170513748' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:33:43.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:33:43 smithi083 ceph-mon[79930]: from='client.48972 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:33:43.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:33:43 smithi083 ceph-mon[79930]: from='client.48978 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:33:43.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:33:43 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4170513748' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:33:43.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:33:43 smithi083 ceph-mon[77242]: from='client.48972 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:33:43.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:33:43 smithi083 ceph-mon[77242]: from='client.48978 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:33:43.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:33:43 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4170513748' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:33:44.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:33:44 smithi153 ceph-mon[70044]: pgmap v12763: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:44.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:33:44 smithi083 ceph-mon[79930]: pgmap v12763: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:44.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:33:44 smithi083 ceph-mon[77242]: pgmap v12763: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:45.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:33:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:33:45.112Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:33:46.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:33:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:33:45.833Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:33:46.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:33:46 smithi153 ceph-mon[70044]: pgmap v12764: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:46.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:33:46 smithi083 ceph-mon[77242]: pgmap v12764: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:46.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:33:46 smithi083 ceph-mon[79930]: pgmap v12764: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:48.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:33:48 smithi153 ceph-mon[70044]: pgmap v12765: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:48.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:33:48 smithi083 ceph-mon[79930]: pgmap v12765: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:48.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:33:48 smithi083 ceph-mon[77242]: pgmap v12765: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:50.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:33:50 smithi153 ceph-mon[70044]: pgmap v12766: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:50.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:33:50 smithi083 ceph-mon[79930]: pgmap v12766: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:50.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:33:50 smithi083 ceph-mon[77242]: pgmap v12766: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:51.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:33:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:33:51] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T12:33:51.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:33:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:33:51.743Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:33:52.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:33:52 smithi153 ceph-mon[70044]: pgmap v12767: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:52.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:33:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:33:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:33:52.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:33:52 smithi083 ceph-mon[79930]: pgmap v12767: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:52.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:33:52 smithi083 ceph-mon[77242]: pgmap v12767: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:33:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:33:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:33:54.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:33:54 smithi153 ceph-mon[70044]: pgmap v12768: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:54.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:33:54 smithi083 ceph-mon[79930]: pgmap v12768: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:54.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:33:54 smithi083 ceph-mon[77242]: pgmap v12768: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:33:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:33:55.112Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:33:56.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:33:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:33:55.834Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:33:56.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:33:56 smithi153 ceph-mon[70044]: pgmap v12769: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:56.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:33:56 smithi083 ceph-mon[79930]: pgmap v12769: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:56.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:33:56 smithi083 ceph-mon[77242]: pgmap v12769: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:58.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:33:58 smithi153 ceph-mon[70044]: pgmap v12770: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:58.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:33:58 smithi083 ceph-mon[79930]: pgmap v12770: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:33:58.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:33:58 smithi083 ceph-mon[77242]: pgmap v12770: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:00.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:34:00 smithi153 ceph-mon[70044]: pgmap v12771: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:00.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:34:00 smithi083 ceph-mon[77242]: pgmap v12771: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:00.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:34:00 smithi083 ceph-mon[79930]: pgmap v12771: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:34:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:34:01] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T12:34:01.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:34:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:34:01.743Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:34:02.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:34:02 smithi153 ceph-mon[70044]: pgmap v12772: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:02.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:34:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:34:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:34:02.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:34:02 smithi083 ceph-mon[79930]: pgmap v12772: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:02.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:34:02 smithi083 ceph-mon[77242]: pgmap v12772: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:34:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:34:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:34:04.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:34:04 smithi153 ceph-mon[70044]: pgmap v12773: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:04.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:34:04 smithi083 ceph-mon[79930]: pgmap v12773: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:04.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:34:04 smithi083 ceph-mon[77242]: pgmap v12773: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:05.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:34:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:34:05.112Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:34:06.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:34:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:34:05.834Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:34:06.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:34:06 smithi153 ceph-mon[70044]: pgmap v12774: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:06.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:34:06 smithi083 ceph-mon[79930]: pgmap v12774: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:06.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:34:06 smithi083 ceph-mon[77242]: pgmap v12774: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:08.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:34:08 smithi153 ceph-mon[70044]: pgmap v12775: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:08.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:34:08 smithi083 ceph-mon[77242]: pgmap v12775: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:08.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:34:08 smithi083 ceph-mon[79930]: pgmap v12775: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:10.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:34:10 smithi153 ceph-mon[70044]: pgmap v12776: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:10.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:34:10 smithi083 ceph-mon[79930]: pgmap v12776: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:10.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:34:10 smithi083 ceph-mon[77242]: pgmap v12776: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:34:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:34:11] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T12:34:11.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:34:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:34:11.744Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:34:12.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:34:12 smithi153 ceph-mon[70044]: pgmap v12777: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:12.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:34:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:34:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:34:12.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:34:12 smithi083 ceph-mon[79930]: pgmap v12777: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:12.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:34:12 smithi083 ceph-mon[77242]: pgmap v12777: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:34:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:34:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:34:13.319 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:34:13.621 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:34:13.621 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 9m ago 7h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:34:13.622 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 7m ago 7h 64.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:34:13.622 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 9m ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:34:13.622 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 7m ago 7h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:34:13.623 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 7m ago 7h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:34:13.623 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 9m ago 7h 864M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:34:13.623 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 9m ago 7h 1559M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:34:13.623 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 7m ago 7h 674M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:34:13.624 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 9m ago 7h 902M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:34:13.624 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 9m ago 7h 28.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:34:13.624 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 7m ago 7h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:34:13.624 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 9m ago 7h 1126M 2366M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:34:13.625 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 9m ago 7h 1114M 2366M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:34:13.625 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 9m ago 7h 847M 2366M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:34:13.625 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 9m ago 7h 1090M 2366M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:34:13.625 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 7m ago 7h 936M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:34:13.625 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 7m ago 7h 1198M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:34:13.626 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 7m ago 7h 1071M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:34:13.626 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (6h) 7m ago 7h 1270M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:34:13.626 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 7m ago 7h 169M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:34:13.988 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:34:13.988 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:34:13.988 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:34:13.989 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:34:13.989 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:34:13.989 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:34:13.989 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:34:13.989 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:34:13.990 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:34:13.990 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:34:13.990 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:34:13.990 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:34:13.991 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:34:13.991 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:34:13.991 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:34:13.991 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:34:13.991 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:34:13.992 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:34:14.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:34:14 smithi153 ceph-mon[70044]: from='client.48990 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:34:14.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:34:14 smithi153 ceph-mon[70044]: pgmap v12778: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:14.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:34:14 smithi153 ceph-mon[70044]: from='client.48996 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:34:14.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:34:14 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/206123449' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:34:14.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:34:14 smithi083 ceph-mon[79930]: from='client.48990 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:34:14.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:34:14 smithi083 ceph-mon[79930]: pgmap v12778: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:14.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:34:14 smithi083 ceph-mon[79930]: from='client.48996 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:34:14.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:34:14 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/206123449' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:34:14.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:34:14 smithi083 ceph-mon[77242]: from='client.48990 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:34:14.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:34:14 smithi083 ceph-mon[77242]: pgmap v12778: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:14.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:34:14 smithi083 ceph-mon[77242]: from='client.48996 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:34:14.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:34:14 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/206123449' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:34:15.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:34:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:34:15.113Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:34:16.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:34:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:34:15.835Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:34:16.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:34:16 smithi153 ceph-mon[70044]: pgmap v12779: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:16.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:34:16 smithi083 ceph-mon[79930]: pgmap v12779: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:16.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:34:16 smithi083 ceph-mon[77242]: pgmap v12779: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:18.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:34:18 smithi153 ceph-mon[70044]: pgmap v12780: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:18.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:34:18 smithi083 ceph-mon[79930]: pgmap v12780: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:18.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:34:18 smithi083 ceph-mon[77242]: pgmap v12780: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:20.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:34:20 smithi153 ceph-mon[70044]: pgmap v12781: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:20.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:34:20 smithi083 ceph-mon[79930]: pgmap v12781: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:20.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:34:20 smithi083 ceph-mon[77242]: pgmap v12781: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:21.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:34:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:34:21] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T12:34:21.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:34:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:34:21.745Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:34:22.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:34:22 smithi153 ceph-mon[70044]: pgmap v12782: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:22.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:34:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:34:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:34:22.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:34:22 smithi083 ceph-mon[79930]: pgmap v12782: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:22.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:34:22 smithi083 ceph-mon[77242]: pgmap v12782: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:34:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:34:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:34:24.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:34:24 smithi153 ceph-mon[70044]: pgmap v12783: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:24.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:34:24 smithi083 ceph-mon[79930]: pgmap v12783: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:24.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:34:24 smithi083 ceph-mon[77242]: pgmap v12783: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:25.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:34:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:34:25.113Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:34:26.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:34:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:34:25.836Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:34:26.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:34:26 smithi153 ceph-mon[70044]: pgmap v12784: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:26.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:34:26 smithi083 ceph-mon[77242]: pgmap v12784: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:26.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:34:26 smithi083 ceph-mon[79930]: pgmap v12784: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:28.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:34:28 smithi153 ceph-mon[70044]: pgmap v12785: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:28.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:34:28 smithi083 ceph-mon[79930]: pgmap v12785: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:28.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:34:28 smithi083 ceph-mon[77242]: pgmap v12785: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:29.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:34:29 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:34:29.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:34:29 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:34:29.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:34:29 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:34:29.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:34:29 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:34:29.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:34:29 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:34:29.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:34:29 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:34:29.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:34:29 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:34:29.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:34:29 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:34:29.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:34:29 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:34:29.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:34:29 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:34:29.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:34:29 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:34:29.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:34:29 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:34:29.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:34:29 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:34:29.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:34:29 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:34:29.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:34:29 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:34:29.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:34:29 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:34:29.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:34:29 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:34:29.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:34:29 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:34:29.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:34:29 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:34:29.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:34:29 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:34:29.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:34:29 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:34:30.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:34:30 smithi153 ceph-mon[70044]: pgmap v12786: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:30.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:34:30 smithi083 ceph-mon[79930]: pgmap v12786: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:30.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:34:30 smithi083 ceph-mon[77242]: pgmap v12786: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:34:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:34:31] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T12:34:31.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:34:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:34:31.744Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:34:32.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:34:32 smithi153 ceph-mon[70044]: pgmap v12787: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:32.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:34:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:34:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:34:32.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:34:32 smithi083 ceph-mon[79930]: pgmap v12787: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:32.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:34:32 smithi083 ceph-mon[77242]: pgmap v12787: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:34:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:34:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:34:34.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:34:34 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:34:34.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:34:34 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:34:34.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:34:34 smithi083 ceph-mon[79930]: pgmap v12788: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:34.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:34:34 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:34:34.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:34:34 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:34:34.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:34:34 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:34:34.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:34:34 smithi083 ceph-mon[77242]: pgmap v12788: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:34.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:34:34 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:34:34.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:34:34 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:34:34.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:34:34 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:34:34.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:34:34 smithi153 ceph-mon[70044]: pgmap v12788: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:34.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:34:34 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:34:35.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:34:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:34:35.114Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:34:36.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:34:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:34:35.836Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:34:36.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:34:36 smithi153 ceph-mon[70044]: pgmap v12789: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:36.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:34:36 smithi083 ceph-mon[77242]: pgmap v12789: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:36.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:34:36 smithi083 ceph-mon[79930]: pgmap v12789: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:38.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:34:38 smithi153 ceph-mon[70044]: pgmap v12790: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:38.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:34:38 smithi083 ceph-mon[79930]: pgmap v12790: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:38.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:34:38 smithi083 ceph-mon[77242]: pgmap v12790: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:34:40 smithi153 ceph-mon[70044]: pgmap v12791: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:34:40 smithi083 ceph-mon[79930]: pgmap v12791: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:34:40 smithi083 ceph-mon[77242]: pgmap v12791: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:41.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:34:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:34:41.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:34:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:34:41.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:34:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:34:41.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:34:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:34:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:34:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:34:41] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T12:34:41.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:34:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:34:41.746Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:34:41.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:34:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:34:41.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:34:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:34:41.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:34:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:34:41.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:34:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:34:41.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:34:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:34:41.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:34:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:34:41.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:34:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:34:41.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:34:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:34:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:34:42 smithi153 ceph-mon[70044]: pgmap v12792: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:42.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:34:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:34:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:34:42.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:34:42 smithi083 ceph-mon[79930]: pgmap v12792: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:42.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:34:42 smithi083 ceph-mon[77242]: pgmap v12792: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:34:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:34:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:34:44.310 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:34:44.599 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:34:44.599 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 11s ago 7h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:34:44.599 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 8m ago 7h 64.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:34:44.599 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 11s ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:34:44.600 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 8m ago 7h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:34:44.600 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 8m ago 7h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:34:44.600 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 11s ago 7h 872M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:34:44.600 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 11s ago 7h 1595M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:34:44.600 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 8m ago 7h 674M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:34:44.601 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 11s ago 7h 924M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:34:44.601 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 11s ago 7h 26.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:34:44.601 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 8m ago 7h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:34:44.601 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 11s ago 7h 1152M 2366M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:34:44.602 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 11s ago 7h 1139M 2366M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:34:44.602 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 11s ago 7h 867M 2366M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:34:44.602 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 11s ago 7h 1115M 2366M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:34:44.602 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 8m ago 7h 936M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:34:44.602 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 8m ago 7h 1198M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:34:44.603 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 8m ago 7h 1071M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:34:44.603 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 8m ago 7h 1270M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:34:44.603 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 8m ago 7h 169M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:34:44.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:34:44 smithi153 ceph-mon[70044]: pgmap v12793: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:44.953 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:34:44.953 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:34:44.953 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:34:44.953 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:34:44.954 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:34:44.954 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:34:44.954 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:34:44.955 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:34:44.955 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:34:44.955 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:34:44.955 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:34:44.955 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:34:44.956 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:34:44.956 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:34:44.956 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:34:44.956 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:34:44.956 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:34:44.957 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:34:44.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:34:44 smithi083 ceph-mon[79930]: pgmap v12793: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:44.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:34:44 smithi083 ceph-mon[77242]: pgmap v12793: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:45.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:34:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:34:45.114Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:34:45.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:34:45 smithi153 ceph-mon[70044]: from='client.58939 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:34:45.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:34:45 smithi153 ceph-mon[70044]: from='client.58945 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:34:45.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:34:45 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2327543251' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:34:45.839 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:34:45 smithi083 ceph-mon[79930]: from='client.58939 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:34:45.839 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:34:45 smithi083 ceph-mon[79930]: from='client.58945 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:34:45.839 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:34:45 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2327543251' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:34:45.840 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:34:45 smithi083 ceph-mon[77242]: from='client.58939 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:34:45.840 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:34:45 smithi083 ceph-mon[77242]: from='client.58945 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:34:45.840 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:34:45 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2327543251' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:34:46.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:34:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:34:45.836Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:34:46.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:34:46 smithi153 ceph-mon[70044]: pgmap v12794: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:46.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:34:46 smithi083 ceph-mon[79930]: pgmap v12794: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:46.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:34:46 smithi083 ceph-mon[77242]: pgmap v12794: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:48.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:34:48 smithi153 ceph-mon[70044]: pgmap v12795: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:48.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:34:48 smithi083 ceph-mon[79930]: pgmap v12795: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:48.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:34:48 smithi083 ceph-mon[77242]: pgmap v12795: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:50.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:34:50 smithi153 ceph-mon[70044]: pgmap v12796: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:50.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:34:50 smithi083 ceph-mon[79930]: pgmap v12796: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:50.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:34:50 smithi083 ceph-mon[77242]: pgmap v12796: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:34:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:34:51] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T12:34:51.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:34:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:34:51.747Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:34:52.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:34:52 smithi153 ceph-mon[70044]: pgmap v12797: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:52.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:34:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:34:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:34:52.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:34:52 smithi083 ceph-mon[79930]: pgmap v12797: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:52.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:34:52 smithi083 ceph-mon[77242]: pgmap v12797: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:34:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:34:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:34:54.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:34:54 smithi083 ceph-mon[79930]: pgmap v12798: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:54.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:34:54 smithi083 ceph-mon[77242]: pgmap v12798: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:55.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:34:54 smithi153 ceph-mon[70044]: pgmap v12798: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:34:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:34:55.116Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:34:56.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:34:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:34:55.838Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:34:56.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:34:56 smithi083 ceph-mon[79930]: pgmap v12799: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:56.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:34:56 smithi083 ceph-mon[77242]: pgmap v12799: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:56.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:34:56 smithi153 ceph-mon[70044]: pgmap v12799: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:58.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:34:58 smithi153 ceph-mon[70044]: pgmap v12800: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:58.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:34:58 smithi083 ceph-mon[77242]: pgmap v12800: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:34:58.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:34:58 smithi083 ceph-mon[79930]: pgmap v12800: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:00.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:35:00 smithi153 ceph-mon[70044]: pgmap v12801: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:00.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:35:00 smithi083 ceph-mon[79930]: pgmap v12801: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:00.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:35:00 smithi083 ceph-mon[77242]: pgmap v12801: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:01.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:35:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:35:01] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T12:35:01.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:35:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:35:01.748Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:35:02.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:35:02 smithi153 ceph-mon[70044]: pgmap v12802: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:02.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:35:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:35:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:35:02.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:35:02 smithi083 ceph-mon[79930]: pgmap v12802: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:02.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:35:02 smithi083 ceph-mon[77242]: pgmap v12802: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:35:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:35:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:35:04.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:35:04 smithi153 ceph-mon[70044]: pgmap v12803: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:04.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:35:04 smithi083 ceph-mon[77242]: pgmap v12803: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:04.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:35:04 smithi083 ceph-mon[79930]: pgmap v12803: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:05.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:35:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:35:05.116Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:35:06.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:35:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:35:05.838Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:35:06.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:35:06 smithi083 ceph-mon[79930]: pgmap v12804: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:06.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:35:06 smithi083 ceph-mon[77242]: pgmap v12804: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:06.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:35:06 smithi153 ceph-mon[70044]: pgmap v12804: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:08.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:35:08 smithi153 ceph-mon[70044]: pgmap v12805: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:08.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:35:08 smithi083 ceph-mon[79930]: pgmap v12805: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:08.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:35:08 smithi083 ceph-mon[77242]: pgmap v12805: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:10.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:35:10 smithi153 ceph-mon[70044]: pgmap v12806: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:10.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:35:10 smithi083 ceph-mon[79930]: pgmap v12806: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:10.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:35:10 smithi083 ceph-mon[77242]: pgmap v12806: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:35:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:35:11] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T12:35:11.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:35:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:35:11.748Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:35:12.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:35:12 smithi153 ceph-mon[70044]: pgmap v12807: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:35:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:35:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:35:12.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:35:12 smithi083 ceph-mon[77242]: pgmap v12807: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:12.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:35:12 smithi083 ceph-mon[79930]: pgmap v12807: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:35:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:35:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:35:14.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:35:14 smithi153 ceph-mon[70044]: pgmap v12808: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:14.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:35:14 smithi083 ceph-mon[79930]: pgmap v12808: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:14.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:35:14 smithi083 ceph-mon[77242]: pgmap v12808: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:15.279 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:35:15.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:35:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:35:15.116Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:35:15.569 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:35:15.569 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 42s ago 7h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:35:15.569 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 8m ago 7h 64.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:35:15.570 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 42s ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:35:15.570 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 8m ago 7h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:35:15.570 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 8m ago 7h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:35:15.570 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 42s ago 7h 872M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:35:15.571 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 42s ago 7h 1595M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:35:15.571 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 8m ago 7h 674M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:35:15.571 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 42s ago 7h 924M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:35:15.571 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 42s ago 7h 26.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:35:15.571 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 8m ago 7h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:35:15.572 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 42s ago 7h 1152M 2366M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:35:15.572 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 42s ago 7h 1139M 2366M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:35:15.572 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 42s ago 7h 867M 2366M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:35:15.572 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 42s ago 7h 1115M 2366M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:35:15.572 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 8m ago 7h 936M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:35:15.573 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 8m ago 7h 1198M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:35:15.573 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 8m ago 7h 1071M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:35:15.573 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 8m ago 7h 1270M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:35:15.573 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 8m ago 7h 169M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:35:15.926 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:35:15.926 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:35:15.927 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:35:15.927 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:35:15.927 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:35:15.927 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:35:15.927 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:35:15.928 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:35:15.928 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:35:15.928 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:35:15.928 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:35:15.928 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:35:15.929 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:35:15.929 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:35:15.929 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:35:15.929 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:35:15.929 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:35:15.930 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:35:16.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:35:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:35:15.838Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:35:16.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:35:16 smithi083 ceph-mon[77242]: from='client.49026 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:35:16.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:35:16 smithi083 ceph-mon[77242]: pgmap v12809: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:16.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:35:16 smithi083 ceph-mon[77242]: from='client.58963 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:35:16.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:35:16 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/760001364' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:35:16.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:35:16 smithi083 ceph-mon[79930]: from='client.49026 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:35:16.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:35:16 smithi083 ceph-mon[79930]: pgmap v12809: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:16.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:35:16 smithi083 ceph-mon[79930]: from='client.58963 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:35:16.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:35:16 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/760001364' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:35:16.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:35:16 smithi153 ceph-mon[70044]: from='client.49026 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:35:16.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:35:16 smithi153 ceph-mon[70044]: pgmap v12809: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:16.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:35:16 smithi153 ceph-mon[70044]: from='client.58963 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:35:16.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:35:16 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/760001364' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:35:18.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:35:18 smithi153 ceph-mon[70044]: pgmap v12810: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:18.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:35:18 smithi083 ceph-mon[79930]: pgmap v12810: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:18.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:35:18 smithi083 ceph-mon[77242]: pgmap v12810: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:20.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:35:20 smithi153 ceph-mon[70044]: pgmap v12811: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:20.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:35:20 smithi083 ceph-mon[79930]: pgmap v12811: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:20.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:35:20 smithi083 ceph-mon[77242]: pgmap v12811: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:35:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:35:21] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T12:35:21.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:35:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:35:21.749Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:35:22.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:35:22 smithi153 ceph-mon[70044]: pgmap v12812: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:22.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:35:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:35:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:35:22.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:35:22 smithi083 ceph-mon[79930]: pgmap v12812: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:22.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:35:22 smithi083 ceph-mon[77242]: pgmap v12812: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:35:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:35:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:35:24.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:35:24 smithi153 ceph-mon[70044]: pgmap v12813: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:24.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:35:24 smithi083 ceph-mon[79930]: pgmap v12813: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:24.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:35:24 smithi083 ceph-mon[77242]: pgmap v12813: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:25.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:35:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:35:25.117Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:35:26.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:35:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:35:25.841Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:35:26.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:35:26 smithi083 ceph-mon[77242]: pgmap v12814: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:26.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:35:26 smithi083 ceph-mon[79930]: pgmap v12814: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:26.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:35:26 smithi153 ceph-mon[70044]: pgmap v12814: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:28.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:35:28 smithi153 ceph-mon[70044]: pgmap v12815: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:28.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:35:28 smithi083 ceph-mon[77242]: pgmap v12815: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:28.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:35:28 smithi083 ceph-mon[79930]: pgmap v12815: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:30.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:35:30 smithi153 ceph-mon[70044]: pgmap v12816: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:30.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:35:30 smithi083 ceph-mon[79930]: pgmap v12816: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:30.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:35:30 smithi083 ceph-mon[77242]: pgmap v12816: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:35:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:35:31] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T12:35:31.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:35:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:35:31.750Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:35:32.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:35:32 smithi153 ceph-mon[70044]: pgmap v12817: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:32.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:35:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:35:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:35:32.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:35:32 smithi083 ceph-mon[79930]: pgmap v12817: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:32.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:35:32 smithi083 ceph-mon[77242]: pgmap v12817: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:35:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:35:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:35:34.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:35:34 smithi153 ceph-mon[70044]: pgmap v12818: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:34.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:35:34 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:35:34.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:35:34 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:35:34.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:35:34 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:35:34.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:35:34 smithi083 ceph-mon[77242]: pgmap v12818: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:34.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:35:34 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:35:34.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:35:34 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:35:34.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:35:34 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:35:34.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:35:34 smithi083 ceph-mon[79930]: pgmap v12818: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:34.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:35:34 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:35:34.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:35:34 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:35:34.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:35:34 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:35:35.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:35:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:35:35.117Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:35:35.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:35:35 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:35:35.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:35:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:35:35.840Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:35:35.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:35:35 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:35:36.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:35:35 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:35:36.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:35:36 smithi083 ceph-mon[79930]: pgmap v12819: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:36.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:35:36 smithi083 ceph-mon[77242]: pgmap v12819: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:37.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:35:36 smithi153 ceph-mon[70044]: pgmap v12819: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:38.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:35:38 smithi083 ceph-mon[77242]: pgmap v12820: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:38.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:35:38 smithi083 ceph-mon[79930]: pgmap v12820: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:39.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:35:38 smithi153 ceph-mon[70044]: pgmap v12820: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:40.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:35:40 smithi083 ceph-mon[77242]: pgmap v12821: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:35:40 smithi083 ceph-mon[79930]: pgmap v12821: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:35:40 smithi153 ceph-mon[70044]: pgmap v12821: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:41.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:35:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:35:41] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T12:35:41.964 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:35:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:35:41.750Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:35:41.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:35:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:35:41.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:35:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:35:41.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:35:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:35:41.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:35:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:35:41.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:35:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:35:41.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:35:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:35:41.968 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:35:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:35:41.968 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:35:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:35:42.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:35:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:35:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:35:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:35:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:35:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:35:42.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:35:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:35:42.657 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:35:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:35:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:35:42.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:35:42 smithi083 ceph-mon[77242]: pgmap v12822: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:42.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:35:42 smithi083 ceph-mon[79930]: pgmap v12822: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:43.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:35:42 smithi153 ceph-mon[70044]: pgmap v12822: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:35:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:35:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:35:44.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:35:44 smithi083 ceph-mon[79930]: pgmap v12823: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:44.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:35:44 smithi083 ceph-mon[77242]: pgmap v12823: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:45.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:35:44 smithi153 ceph-mon[70044]: pgmap v12823: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:45.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:35:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:35:45.118Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:35:46.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:35:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:35:45.841Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:35:46.253 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:35:46.542 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:35:46.542 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 73s ago 7h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:35:46.542 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 9m ago 7h 64.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:35:46.543 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 73s ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:35:46.543 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 9m ago 7h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:35:46.543 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 9m ago 7h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:35:46.543 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 73s ago 7h 872M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:35:46.544 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 73s ago 7h 1595M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:35:46.544 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 9m ago 7h 674M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:35:46.544 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 73s ago 7h 924M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:35:46.544 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 73s ago 7h 26.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:35:46.544 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 9m ago 7h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:35:46.545 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 73s ago 7h 1152M 2366M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:35:46.545 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 73s ago 7h 1139M 2366M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:35:46.545 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 73s ago 7h 867M 2366M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:35:46.545 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 73s ago 7h 1115M 2366M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:35:46.546 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 9m ago 7h 936M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:35:46.546 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 9m ago 7h 1198M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:35:46.546 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 9m ago 7h 1071M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:35:46.546 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 9m ago 7h 1270M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:35:46.547 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 9m ago 7h 169M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:35:46.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:35:46 smithi083 ceph-mon[77242]: pgmap v12824: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:46.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:35:46 smithi083 ceph-mon[79930]: pgmap v12824: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:46.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:35:46 smithi153 ceph-mon[70044]: pgmap v12824: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:46.897 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:35:46.898 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:35:46.898 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:35:46.898 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:35:46.899 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:35:46.899 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:35:46.899 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:35:46.899 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:35:46.900 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:35:46.900 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:35:46.900 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:35:46.900 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:35:46.900 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:35:46.901 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:35:46.901 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:35:46.901 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:35:46.901 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:35:46.902 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:35:47.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:35:47 smithi083 ceph-mon[77242]: from='client.58975 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:35:47.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:35:47 smithi083 ceph-mon[77242]: from='client.49050 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:35:47.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:35:47 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2717977498' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:35:47.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:35:47 smithi083 ceph-mon[79930]: from='client.58975 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:35:47.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:35:47 smithi083 ceph-mon[79930]: from='client.49050 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:35:47.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:35:47 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2717977498' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:35:47.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:35:47 smithi153 ceph-mon[70044]: from='client.58975 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:35:47.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:35:47 smithi153 ceph-mon[70044]: from='client.49050 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:35:47.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:35:47 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2717977498' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:35:48.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:35:48 smithi083 ceph-mon[79930]: pgmap v12825: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:48.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:35:48 smithi083 ceph-mon[77242]: pgmap v12825: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:48.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:35:48 smithi153 ceph-mon[70044]: pgmap v12825: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:50.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:35:50 smithi153 ceph-mon[70044]: pgmap v12826: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:50.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:35:50 smithi083 ceph-mon[77242]: pgmap v12826: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:50.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:35:50 smithi083 ceph-mon[79930]: pgmap v12826: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:35:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:35:51] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T12:35:51.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:35:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:35:51.751Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:35:52.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:35:52 smithi153 ceph-mon[70044]: pgmap v12827: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:52.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:35:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:35:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:35:52.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:35:52 smithi083 ceph-mon[77242]: pgmap v12827: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:52.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:35:52 smithi083 ceph-mon[79930]: pgmap v12827: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:35:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:35:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:35:54.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:35:54 smithi153 ceph-mon[70044]: pgmap v12828: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:54.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:35:54 smithi083 ceph-mon[79930]: pgmap v12828: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:54.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:35:54 smithi083 ceph-mon[77242]: pgmap v12828: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:35:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:35:55.118Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:35:56.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:35:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:35:55.842Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:35:56.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:35:56 smithi083 ceph-mon[79930]: pgmap v12829: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:56.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:35:56 smithi083 ceph-mon[77242]: pgmap v12829: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:56.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:35:56 smithi153 ceph-mon[70044]: pgmap v12829: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:58.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:35:58 smithi153 ceph-mon[70044]: pgmap v12830: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:58.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:35:58 smithi083 ceph-mon[79930]: pgmap v12830: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:35:58.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:35:58 smithi083 ceph-mon[77242]: pgmap v12830: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:00.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:36:00 smithi153 ceph-mon[70044]: pgmap v12831: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:00.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:36:00 smithi083 ceph-mon[79930]: pgmap v12831: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:00.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:36:00 smithi083 ceph-mon[77242]: pgmap v12831: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:01.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:36:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:36:01] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T12:36:01.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:36:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:36:01.752Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:36:02.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:36:02 smithi153 ceph-mon[70044]: pgmap v12832: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:36:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:36:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:36:02.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:36:02 smithi083 ceph-mon[77242]: pgmap v12832: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:02.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:36:02 smithi083 ceph-mon[79930]: pgmap v12832: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:36:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:36:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:36:04.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:36:04 smithi153 ceph-mon[70044]: pgmap v12833: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:04.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:36:04 smithi083 ceph-mon[77242]: pgmap v12833: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:04.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:36:04 smithi083 ceph-mon[79930]: pgmap v12833: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:05.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:36:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:36:05.119Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:36:06.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:36:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:36:05.842Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:36:06.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:36:06 smithi083 ceph-mon[77242]: pgmap v12834: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:06.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:36:06 smithi083 ceph-mon[79930]: pgmap v12834: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:06.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:36:06 smithi153 ceph-mon[70044]: pgmap v12834: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:08.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:36:08 smithi153 ceph-mon[70044]: pgmap v12835: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:08.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:36:08 smithi083 ceph-mon[79930]: pgmap v12835: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:08.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:36:08 smithi083 ceph-mon[77242]: pgmap v12835: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:10.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:36:10 smithi153 ceph-mon[70044]: pgmap v12836: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:10.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:36:10 smithi083 ceph-mon[79930]: pgmap v12836: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:10.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:36:10 smithi083 ceph-mon[77242]: pgmap v12836: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:11.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:36:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:36:11.752Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:36:11.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:36:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:36:11] "GET /metrics HTTP/1.1" 200 225229 "" "Prometheus/2.33.4" 2022-09-29T12:36:12.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:36:12 smithi153 ceph-mon[70044]: pgmap v12837: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:12.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:36:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:36:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:36:12.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:36:12 smithi083 ceph-mon[79930]: pgmap v12837: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:12.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:36:12 smithi083 ceph-mon[77242]: pgmap v12837: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:36:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:36:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:36:14.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:36:14 smithi083 ceph-mon[79930]: pgmap v12838: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:14.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:36:14 smithi083 ceph-mon[77242]: pgmap v12838: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:15.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:36:14 smithi153 ceph-mon[70044]: pgmap v12838: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:15.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:36:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:36:15.120Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:36:16.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:36:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:36:15.843Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:36:16.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:36:16 smithi083 ceph-mon[79930]: pgmap v12839: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:16.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:36:16 smithi083 ceph-mon[77242]: pgmap v12839: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:16.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:36:16 smithi153 ceph-mon[70044]: pgmap v12839: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:17.223 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:36:17.513 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:36:17.513 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 104s ago 7h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:36:17.513 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 9m ago 7h 64.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:36:17.513 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 104s ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:36:17.514 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 9m ago 7h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:36:17.514 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 9m ago 7h 446M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:36:17.514 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 104s ago 7h 872M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:36:17.515 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 104s ago 7h 1595M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:36:17.515 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 9m ago 7h 674M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:36:17.515 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 104s ago 7h 924M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:36:17.515 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 104s ago 7h 26.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:36:17.515 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 9m ago 7h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:36:17.516 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 104s ago 7h 1152M 2366M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:36:17.516 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 104s ago 7h 1139M 2366M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:36:17.516 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 104s ago 7h 867M 2366M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:36:17.516 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 104s ago 7h 1115M 2366M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:36:17.517 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 9m ago 7h 936M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:36:17.517 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 9m ago 7h 1198M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:36:17.517 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 9m ago 7h 1071M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:36:17.517 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 9m ago 7h 1270M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:36:17.518 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 9m ago 7h 169M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:36:17.868 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:36:17.869 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:36:17.869 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:36:17.869 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:36:17.870 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:36:17.870 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:36:17.870 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:36:17.871 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:36:17.871 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:36:17.871 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:36:17.871 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:36:17.872 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:36:17.872 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:36:17.872 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:36:17.872 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:36:17.873 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:36:17.873 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:36:17.873 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:36:18.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:36:18 smithi153 ceph-mon[70044]: from='client.58993 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:36:18.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:36:18 smithi153 ceph-mon[70044]: pgmap v12840: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:18.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:36:18 smithi153 ceph-mon[70044]: from='client.49068 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:36:18.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:36:18 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/134448693' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:36:18.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:36:18 smithi083 ceph-mon[77242]: from='client.58993 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:36:18.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:36:18 smithi083 ceph-mon[77242]: pgmap v12840: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:18.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:36:18 smithi083 ceph-mon[77242]: from='client.49068 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:36:18.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:36:18 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/134448693' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:36:18.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:36:18 smithi083 ceph-mon[79930]: from='client.58993 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:36:18.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:36:18 smithi083 ceph-mon[79930]: pgmap v12840: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:18.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:36:18 smithi083 ceph-mon[79930]: from='client.49068 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:36:18.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:36:18 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/134448693' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:36:20.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:36:20 smithi083 ceph-mon[77242]: pgmap v12841: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:20.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:36:20 smithi083 ceph-mon[79930]: pgmap v12841: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:21.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:36:20 smithi153 ceph-mon[70044]: pgmap v12841: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:36:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:36:21] "GET /metrics HTTP/1.1" 200 225229 "" "Prometheus/2.33.4" 2022-09-29T12:36:21.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:36:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:36:21.752Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:36:22.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:36:22 smithi153 ceph-mon[70044]: pgmap v12842: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:22.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:36:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:36:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:36:22.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:36:22 smithi083 ceph-mon[77242]: pgmap v12842: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:22.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:36:22 smithi083 ceph-mon[79930]: pgmap v12842: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:36:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:36:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:36:24.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:36:24 smithi083 ceph-mon[79930]: pgmap v12843: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:24.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:36:24 smithi083 ceph-mon[77242]: pgmap v12843: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:25.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:36:24 smithi153 ceph-mon[70044]: pgmap v12843: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:25.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:36:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:36:25.121Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:36:26.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:36:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:36:25.844Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:36:26.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:36:26 smithi083 ceph-mon[77242]: pgmap v12844: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:26.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:36:26 smithi083 ceph-mon[79930]: pgmap v12844: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:26.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:36:26 smithi153 ceph-mon[70044]: pgmap v12844: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:28.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:36:28 smithi153 ceph-mon[70044]: pgmap v12845: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:28.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:36:28 smithi083 ceph-mon[79930]: pgmap v12845: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:28.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:36:28 smithi083 ceph-mon[77242]: pgmap v12845: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:30.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:36:30 smithi083 ceph-mon[79930]: pgmap v12846: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:30.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:36:30 smithi083 ceph-mon[77242]: pgmap v12846: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:31.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:36:30 smithi153 ceph-mon[70044]: pgmap v12846: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:31.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:36:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:36:31.753Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:36:31.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:36:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:36:31] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T12:36:32.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:36:32 smithi153 ceph-mon[70044]: pgmap v12847: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:32.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:36:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:36:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:36:32.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:36:32 smithi083 ceph-mon[77242]: pgmap v12847: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:32.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:36:32 smithi083 ceph-mon[79930]: pgmap v12847: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:36:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:36:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:36:34.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:36:34 smithi083 ceph-mon[79930]: pgmap v12848: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:34.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:36:34 smithi083 ceph-mon[77242]: pgmap v12848: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:35.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:36:34 smithi153 ceph-mon[70044]: pgmap v12848: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:35.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:36:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:36:35.121Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:36:35.846 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:36:35 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:36:35.846 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:36:35 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:36:35.847 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:36:35 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:36:35.847 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:36:35 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:36:35.847 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:36:35 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:36:35.848 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:36:35 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:36:36.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:36:35 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:36:36.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:36:35 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:36:36.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:36:35 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:36:36.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:36:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:36:35.843Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:36:36.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:36:36 smithi083 ceph-mon[77242]: pgmap v12849: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:36.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:36:36 smithi083 ceph-mon[79930]: pgmap v12849: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:37.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:36:36 smithi153 ceph-mon[70044]: pgmap v12849: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:38.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:36:38 smithi083 ceph-mon[79930]: pgmap v12850: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:38.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:36:38 smithi083 ceph-mon[77242]: pgmap v12850: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:39.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:36:38 smithi153 ceph-mon[70044]: pgmap v12850: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:36:40 smithi083 ceph-mon[79930]: pgmap v12851: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:36:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:36:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:36:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:36:40.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:36:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:36:40.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:36:40 smithi083 ceph-mon[77242]: pgmap v12851: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:40.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:36:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:36:40.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:36:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:36:40.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:36:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:36:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:36:40 smithi153 ceph-mon[70044]: pgmap v12851: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:41.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:36:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:36:41.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:36:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:36:41.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:36:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:36:41.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:36:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:36:41] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T12:36:41.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:36:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:36:41.753Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:36:41.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:36:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:36:41.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:36:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:36:41.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:36:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:36:41.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:36:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:36:41.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:36:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:36:41.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:36:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:36:41.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:36:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:36:41.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:36:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:36:42.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:36:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:36:42.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:36:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:36:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:36:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:36:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:36:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:36:42.572 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:36:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:36:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:36:42.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:36:42 smithi083 ceph-mon[79930]: pgmap v12852: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:42.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:36:42 smithi083 ceph-mon[77242]: pgmap v12852: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:43.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:36:42 smithi153 ceph-mon[70044]: pgmap v12852: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:36:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:36:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:36:44.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:36:44 smithi083 ceph-mon[79930]: pgmap v12853: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:44.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:36:44 smithi083 ceph-mon[77242]: pgmap v12853: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:45.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:36:44 smithi153 ceph-mon[70044]: pgmap v12853: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:45.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:36:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:36:45.122Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:36:46.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:36:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:36:45.844Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:36:46.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:36:46 smithi083 ceph-mon[77242]: pgmap v12854: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:46.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:36:46 smithi083 ceph-mon[79930]: pgmap v12854: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:46.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:36:46 smithi153 ceph-mon[70044]: pgmap v12854: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:48.196 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:36:48.487 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:36:48.487 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 2m ago 7h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:36:48.488 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 8s ago 7h 64.1M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:36:48.488 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 2m ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:36:48.488 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 8s ago 7h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:36:48.488 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 8s ago 7h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:36:48.489 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 2m ago 7h 872M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:36:48.489 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 2m ago 7h 1595M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:36:48.489 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 8s ago 7h 688M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:36:48.490 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 2m ago 7h 924M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:36:48.490 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 2m ago 7h 26.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:36:48.490 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 8s ago 7h 25.8M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:36:48.490 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 2m ago 7h 1152M 2366M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:36:48.491 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 2m ago 7h 1139M 2366M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:36:48.491 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 2m ago 7h 867M 2366M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:36:48.491 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 2m ago 7h 1115M 2366M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:36:48.491 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 8s ago 7h 957M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:36:48.491 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 8s ago 7h 1226M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:36:48.492 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 8s ago 7h 1095M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:36:48.492 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 8s ago 7h 1300M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:36:48.492 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 8s ago 7h 171M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:36:48.846 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:36:48.846 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:36:48.846 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:36:48.847 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:36:48.847 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:36:48.847 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:36:48.847 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:36:48.848 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:36:48.848 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:36:48.848 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:36:48.848 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:36:48.849 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:36:48.849 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:36:48.849 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:36:48.849 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:36:48.850 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:36:48.850 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:36:48.850 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:36:48.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:36:48 smithi083 ceph-mon[79930]: pgmap v12855: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:48.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:36:48 smithi083 ceph-mon[77242]: pgmap v12855: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:49.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:36:48 smithi153 ceph-mon[70044]: pgmap v12855: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:49.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:36:49 smithi083 ceph-mon[79930]: from='client.49080 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:36:49.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:36:49 smithi083 ceph-mon[79930]: from='client.59017 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:36:49.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:36:49 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1119168244' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:36:49.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:36:49 smithi083 ceph-mon[77242]: from='client.49080 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:36:49.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:36:49 smithi083 ceph-mon[77242]: from='client.59017 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:36:49.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:36:49 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1119168244' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:36:50.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:36:49 smithi153 ceph-mon[70044]: from='client.49080 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:36:50.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:36:49 smithi153 ceph-mon[70044]: from='client.59017 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:36:50.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:36:49 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1119168244' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:36:50.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:36:50 smithi083 ceph-mon[79930]: pgmap v12856: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:50.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:36:50 smithi083 ceph-mon[77242]: pgmap v12856: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:51.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:36:50 smithi153 ceph-mon[70044]: pgmap v12856: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:51.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:36:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:36:51.754Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:36:51.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:36:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:36:51] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T12:36:52.574 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:36:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:36:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:36:52.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:36:52 smithi083 ceph-mon[77242]: pgmap v12857: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:52.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:36:52 smithi083 ceph-mon[79930]: pgmap v12857: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:53.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:36:52 smithi153 ceph-mon[70044]: pgmap v12857: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:36:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:36:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:36:54.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:36:54 smithi083 ceph-mon[79930]: pgmap v12858: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:54.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:36:54 smithi083 ceph-mon[77242]: pgmap v12858: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:55.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:36:54 smithi153 ceph-mon[70044]: pgmap v12858: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:36:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:36:55.124Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:36:56.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:36:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:36:55.846Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:36:56.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:36:56 smithi083 ceph-mon[79930]: pgmap v12859: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:56.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:36:56 smithi083 ceph-mon[77242]: pgmap v12859: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:56.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:36:56 smithi153 ceph-mon[70044]: pgmap v12859: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:58.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:36:58 smithi083 ceph-mon[79930]: pgmap v12860: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:58.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:36:58 smithi083 ceph-mon[77242]: pgmap v12860: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:36:59.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:36:58 smithi153 ceph-mon[70044]: pgmap v12860: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:00.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:37:00 smithi083 ceph-mon[79930]: pgmap v12861: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:00.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:37:00 smithi083 ceph-mon[77242]: pgmap v12861: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:01.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:37:00 smithi153 ceph-mon[70044]: pgmap v12861: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:01.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:37:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:37:01] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T12:37:01.964 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:37:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:37:01.754Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:37:02.576 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:37:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:37:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:37:02.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:37:02 smithi083 ceph-mon[79930]: pgmap v12862: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:02.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:37:02 smithi083 ceph-mon[77242]: pgmap v12862: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:03.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:37:02 smithi153 ceph-mon[70044]: pgmap v12862: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:37:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:37:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:37:04.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:37:04 smithi083 ceph-mon[79930]: pgmap v12863: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:04.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:37:04 smithi083 ceph-mon[77242]: pgmap v12863: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:05.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:37:04 smithi153 ceph-mon[70044]: pgmap v12863: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:05.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:37:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:37:05.124Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:37:06.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:37:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:37:05.846Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:37:06.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:37:06 smithi083 ceph-mon[79930]: pgmap v12864: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:06.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:37:06 smithi083 ceph-mon[77242]: pgmap v12864: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:06.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:37:06 smithi153 ceph-mon[70044]: pgmap v12864: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:08.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:37:08 smithi083 ceph-mon[79930]: pgmap v12865: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:08.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:37:08 smithi083 ceph-mon[77242]: pgmap v12865: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:09.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:37:08 smithi153 ceph-mon[70044]: pgmap v12865: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:10.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:37:10 smithi083 ceph-mon[79930]: pgmap v12866: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:10.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:37:10 smithi083 ceph-mon[77242]: pgmap v12866: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:11.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:37:10 smithi153 ceph-mon[70044]: pgmap v12866: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:11.758 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:37:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:37:11] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T12:37:12.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:37:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:37:11.755Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:37:12.575 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:37:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:37:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:37:12.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:37:12 smithi083 ceph-mon[79930]: pgmap v12867: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:12.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:37:12 smithi083 ceph-mon[77242]: pgmap v12867: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:13.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:37:12 smithi153 ceph-mon[70044]: pgmap v12867: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:37:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:37:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:37:14.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:37:14 smithi083 ceph-mon[79930]: pgmap v12868: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:14.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:37:14 smithi083 ceph-mon[77242]: pgmap v12868: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:15.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:37:14 smithi153 ceph-mon[70044]: pgmap v12868: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:15.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:37:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:37:15.125Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:37:16.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:37:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:37:15.847Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:37:16.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:37:16 smithi083 ceph-mon[79930]: pgmap v12869: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:16.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:37:16 smithi083 ceph-mon[77242]: pgmap v12869: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:16.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:37:16 smithi153 ceph-mon[70044]: pgmap v12869: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:18.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:37:18 smithi083 ceph-mon[79930]: pgmap v12870: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:18.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:37:18 smithi083 ceph-mon[77242]: pgmap v12870: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:19.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:37:18 smithi153 ceph-mon[70044]: pgmap v12870: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:19.170 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:37:19.457 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:37:19.458 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 2m ago 7h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:37:19.458 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 39s ago 7h 64.1M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:37:19.458 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 2m ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:37:19.458 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 39s ago 7h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:37:19.459 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 39s ago 7h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:37:19.459 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 2m ago 7h 872M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:37:19.459 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 2m ago 7h 1595M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:37:19.459 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 39s ago 7h 688M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:37:19.459 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 2m ago 7h 924M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:37:19.460 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 2m ago 7h 26.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:37:19.460 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 39s ago 7h 25.8M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:37:19.460 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 2m ago 7h 1152M 2366M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:37:19.460 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 2m ago 7h 1139M 2366M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:37:19.460 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 2m ago 7h 867M 2366M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:37:19.461 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 2m ago 7h 1115M 2366M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:37:19.461 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 39s ago 7h 957M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:37:19.461 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 39s ago 7h 1226M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:37:19.461 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 39s ago 7h 1095M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:37:19.462 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 39s ago 7h 1300M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:37:19.462 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 39s ago 7h 171M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:37:19.815 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:37:19.815 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:37:19.815 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:37:19.816 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:37:19.816 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:37:19.816 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:37:19.816 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:37:19.817 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:37:19.817 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:37:19.817 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:37:19.817 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:37:19.818 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:37:19.818 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:37:19.818 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:37:19.818 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:37:19.819 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:37:19.819 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:37:19.819 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:37:20.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:37:20 smithi083 ceph-mon[79930]: from='client.59029 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:37:20.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:37:20 smithi083 ceph-mon[79930]: from='client.59035 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:37:20.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:37:20 smithi083 ceph-mon[79930]: pgmap v12871: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:20.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:37:20 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1421796285' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:37:20.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:37:20 smithi083 ceph-mon[77242]: from='client.59029 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:37:20.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:37:20 smithi083 ceph-mon[77242]: from='client.59035 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:37:20.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:37:20 smithi083 ceph-mon[77242]: pgmap v12871: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:20.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:37:20 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1421796285' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:37:21.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:37:20 smithi153 ceph-mon[70044]: from='client.59029 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:37:21.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:37:20 smithi153 ceph-mon[70044]: from='client.59035 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:37:21.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:37:20 smithi153 ceph-mon[70044]: pgmap v12871: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:21.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:37:20 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1421796285' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:37:21.758 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:37:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:37:21] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T12:37:22.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:37:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:37:21.756Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:37:22.579 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:37:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:37:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:37:22.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:37:22 smithi083 ceph-mon[77242]: pgmap v12872: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:22.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:37:22 smithi083 ceph-mon[79930]: pgmap v12872: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:23.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:37:22 smithi153 ceph-mon[70044]: pgmap v12872: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:37:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:37:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:37:24.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:37:24 smithi083 ceph-mon[77242]: pgmap v12873: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:24.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:37:24 smithi083 ceph-mon[79930]: pgmap v12873: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:25.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:37:24 smithi153 ceph-mon[70044]: pgmap v12873: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:25.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:37:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:37:25.125Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:37:26.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:37:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:37:25.847Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:37:26.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:37:26 smithi083 ceph-mon[79930]: pgmap v12874: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:26.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:37:26 smithi083 ceph-mon[77242]: pgmap v12874: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:26.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:37:26 smithi153 ceph-mon[70044]: pgmap v12874: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:28.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:37:28 smithi083 ceph-mon[79930]: pgmap v12875: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:28.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:37:28 smithi083 ceph-mon[77242]: pgmap v12875: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:29.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:37:28 smithi153 ceph-mon[70044]: pgmap v12875: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:30.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:37:30 smithi083 ceph-mon[77242]: pgmap v12876: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:30.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:37:30 smithi083 ceph-mon[79930]: pgmap v12876: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:31.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:37:30 smithi153 ceph-mon[70044]: pgmap v12876: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:31.759 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:37:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:37:31] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T12:37:32.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:37:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:37:31.756Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:37:32.584 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:37:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:37:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:37:32.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:37:32 smithi083 ceph-mon[77242]: pgmap v12877: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:32.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:37:32 smithi083 ceph-mon[79930]: pgmap v12877: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:33.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:37:32 smithi153 ceph-mon[70044]: pgmap v12877: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:37:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:37:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:37:34.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:37:34 smithi083 ceph-mon[79930]: pgmap v12878: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:34.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:37:34 smithi083 ceph-mon[77242]: pgmap v12878: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:35.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:37:34 smithi153 ceph-mon[70044]: pgmap v12878: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:35.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:37:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:37:35.126Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:37:36.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:37:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:37:35.848Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:37:36.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:37:36 smithi083 ceph-mon[77242]: pgmap v12879: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:36.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:37:36 smithi083 ceph-mon[79930]: pgmap v12879: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:36.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:37:36 smithi153 ceph-mon[70044]: pgmap v12879: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:38.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:37:38 smithi083 ceph-mon[79930]: pgmap v12880: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:38.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:37:38 smithi083 ceph-mon[77242]: pgmap v12880: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:39.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:37:38 smithi153 ceph-mon[70044]: pgmap v12880: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:37:40 smithi083 ceph-mon[79930]: pgmap v12881: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:37:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:37:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:37:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:37:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:37:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:37:40.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:37:40 smithi083 ceph-mon[77242]: pgmap v12881: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:40.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:37:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:37:40.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:37:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:37:40.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:37:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:37:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:37:40 smithi153 ceph-mon[70044]: pgmap v12881: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:37:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:37:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:37:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:37:41.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:37:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:37:41.759 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:37:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:37:41] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T12:37:42.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:37:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:37:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:37:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:37:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:37:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:37:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:37:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:37:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:37:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:37:42.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:37:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:37:41.756Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:37:42.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:37:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:37:42.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:37:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:37:42.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:37:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:37:42.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:37:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:37:42.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:37:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:37:42.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:37:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:37:42.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:37:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:37:42.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:37:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:37:42.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:37:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:37:42.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:37:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:37:42.770 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:37:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:37:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:37:43.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:37:42 smithi153 ceph-mon[70044]: pgmap v12882: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:43.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:37:42 smithi083 ceph-mon[79930]: pgmap v12882: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:43.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:37:42 smithi083 ceph-mon[77242]: pgmap v12882: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:37:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:37:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:37:45.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:37:44 smithi153 ceph-mon[70044]: pgmap v12883: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:45.129 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:37:44 smithi083 ceph-mon[79930]: pgmap v12883: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:45.129 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:37:44 smithi083 ceph-mon[77242]: pgmap v12883: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:45.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:37:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:37:45.126Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:37:46.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:37:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:37:45.848Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:37:46.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:37:46 smithi083 ceph-mon[79930]: pgmap v12884: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:46.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:37:46 smithi083 ceph-mon[77242]: pgmap v12884: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:46.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:37:46 smithi153 ceph-mon[70044]: pgmap v12884: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:48.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:37:48 smithi083 ceph-mon[79930]: pgmap v12885: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:48.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:37:48 smithi083 ceph-mon[77242]: pgmap v12885: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:49.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:37:48 smithi153 ceph-mon[70044]: pgmap v12885: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:50.140 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:37:50.430 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:37:50.430 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 3m ago 7h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:37:50.430 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 70s ago 7h 64.1M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:37:50.430 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 3m ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:37:50.431 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 70s ago 7h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:37:50.431 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 70s ago 7h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:37:50.431 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 3m ago 7h 872M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:37:50.431 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 3m ago 7h 1595M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:37:50.431 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 70s ago 7h 688M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:37:50.432 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 3m ago 7h 924M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:37:50.432 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 3m ago 7h 26.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:37:50.432 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 70s ago 7h 25.8M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:37:50.432 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 3m ago 7h 1152M 2366M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:37:50.433 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 3m ago 7h 1139M 2366M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:37:50.433 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 3m ago 7h 867M 2366M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:37:50.433 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 3m ago 7h 1115M 2366M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:37:50.433 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 70s ago 7h 957M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:37:50.433 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 70s ago 7h 1226M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:37:50.434 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 70s ago 7h 1095M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:37:50.434 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 70s ago 7h 1300M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:37:50.434 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 70s ago 7h 171M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:37:50.790 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:37:50.790 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:37:50.790 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:37:50.791 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:37:50.791 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:37:50.791 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:37:50.791 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:37:50.792 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:37:50.792 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:37:50.792 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:37:50.792 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:37:50.793 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:37:50.793 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:37:50.793 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:37:50.793 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:37:50.794 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:37:50.794 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:37:50.794 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:37:50.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:37:50 smithi083 ceph-mon[77242]: pgmap v12886: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:50.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:37:50 smithi083 ceph-mon[79930]: pgmap v12886: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:51.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:37:50 smithi153 ceph-mon[70044]: pgmap v12886: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:51.760 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:37:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:37:51] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T12:37:51.760 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:37:51 smithi083 ceph-mon[79930]: from='client.59047 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:37:51.761 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:37:51 smithi083 ceph-mon[79930]: from='client.49122 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:37:51.761 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:37:51 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1650878716' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:37:51.762 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:37:51 smithi083 ceph-mon[77242]: from='client.59047 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:37:51.762 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:37:51 smithi083 ceph-mon[77242]: from='client.49122 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:37:51.762 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:37:51 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1650878716' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:37:52.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:37:51 smithi153 ceph-mon[70044]: from='client.59047 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:37:52.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:37:51 smithi153 ceph-mon[70044]: from='client.49122 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:37:52.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:37:51 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1650878716' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:37:52.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:37:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:37:51.757Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:37:52.598 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:37:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:37:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:37:52.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:37:52 smithi083 ceph-mon[79930]: pgmap v12887: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:52.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:37:52 smithi083 ceph-mon[77242]: pgmap v12887: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:53.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:37:52 smithi153 ceph-mon[70044]: pgmap v12887: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:37:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:37:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:37:54.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:37:54 smithi083 ceph-mon[77242]: pgmap v12888: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:54.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:37:54 smithi083 ceph-mon[79930]: pgmap v12888: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:55.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:37:54 smithi153 ceph-mon[70044]: pgmap v12888: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:37:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:37:55.127Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:37:56.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:37:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:37:55.849Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:37:56.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:37:56 smithi083 ceph-mon[77242]: pgmap v12889: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:56.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:37:56 smithi083 ceph-mon[79930]: pgmap v12889: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:56.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:37:56 smithi153 ceph-mon[70044]: pgmap v12889: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:58.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:37:58 smithi083 ceph-mon[79930]: pgmap v12890: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:58.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:37:58 smithi083 ceph-mon[77242]: pgmap v12890: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:37:59.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:37:58 smithi153 ceph-mon[70044]: pgmap v12890: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:00.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:38:00 smithi083 ceph-mon[79930]: pgmap v12891: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:00.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:38:00 smithi083 ceph-mon[77242]: pgmap v12891: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:01.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:38:00 smithi153 ceph-mon[70044]: pgmap v12891: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:01.761 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:38:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:38:01] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T12:38:02.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:38:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:38:01.758Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:38:02.594 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:38:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:38:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:38:02.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:38:02 smithi083 ceph-mon[79930]: pgmap v12892: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:02.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:38:02 smithi083 ceph-mon[77242]: pgmap v12892: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:03.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:38:02 smithi153 ceph-mon[70044]: pgmap v12892: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:38:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:38:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:38:04.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:38:04 smithi083 ceph-mon[79930]: pgmap v12893: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:04.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:38:04 smithi083 ceph-mon[77242]: pgmap v12893: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:05.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:38:04 smithi153 ceph-mon[70044]: pgmap v12893: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:05.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:38:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:38:05.127Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:38:06.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:38:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:38:05.851Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:38:06.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:38:06 smithi083 ceph-mon[77242]: pgmap v12894: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:06.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:38:06 smithi083 ceph-mon[79930]: pgmap v12894: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:06.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:38:06 smithi153 ceph-mon[70044]: pgmap v12894: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:08.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:38:08 smithi083 ceph-mon[79930]: pgmap v12895: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:08.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:38:08 smithi083 ceph-mon[77242]: pgmap v12895: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:09.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:38:08 smithi153 ceph-mon[70044]: pgmap v12895: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:10.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:38:10 smithi083 ceph-mon[79930]: pgmap v12896: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:10.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:38:10 smithi083 ceph-mon[77242]: pgmap v12896: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:11.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:38:10 smithi153 ceph-mon[70044]: pgmap v12896: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:11.761 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:38:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:38:11] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T12:38:12.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:38:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:38:11.759Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:38:12.595 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:38:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:38:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:38:12.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:38:12 smithi083 ceph-mon[79930]: pgmap v12897: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:12.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:38:12 smithi083 ceph-mon[77242]: pgmap v12897: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:13.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:38:12 smithi153 ceph-mon[70044]: pgmap v12897: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:38:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:38:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:38:14.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:38:14 smithi083 ceph-mon[77242]: pgmap v12898: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:14.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:38:14 smithi083 ceph-mon[79930]: pgmap v12898: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:15.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:38:14 smithi153 ceph-mon[70044]: pgmap v12898: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:15.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:38:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:38:15.128Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:38:16.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:38:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:38:15.851Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:38:16.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:38:16 smithi083 ceph-mon[79930]: pgmap v12899: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:16.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:38:16 smithi083 ceph-mon[77242]: pgmap v12899: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:16.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:38:16 smithi153 ceph-mon[70044]: pgmap v12899: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:18.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:38:18 smithi083 ceph-mon[77242]: pgmap v12900: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:18.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:38:18 smithi083 ceph-mon[79930]: pgmap v12900: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:19.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:38:18 smithi153 ceph-mon[70044]: pgmap v12900: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:20.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:38:20 smithi083 ceph-mon[79930]: pgmap v12901: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:20.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:38:20 smithi083 ceph-mon[77242]: pgmap v12901: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:21.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:38:20 smithi153 ceph-mon[70044]: pgmap v12901: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:21.117 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:38:21.404 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:38:21.404 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 3m ago 7h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:38:21.405 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 101s ago 7h 64.1M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:38:21.405 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 3m ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:38:21.405 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 101s ago 7h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:38:21.405 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 101s ago 7h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:38:21.406 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 3m ago 7h 872M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:38:21.406 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 3m ago 7h 1595M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:38:21.406 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 101s ago 7h 688M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:38:21.406 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 3m ago 7h 924M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:38:21.406 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 3m ago 7h 26.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:38:21.407 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 101s ago 7h 25.8M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:38:21.407 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 3m ago 7h 1152M 2366M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:38:21.407 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 3m ago 7h 1139M 2366M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:38:21.407 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 3m ago 7h 867M 2366M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:38:21.408 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 3m ago 7h 1115M 2366M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:38:21.408 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 101s ago 7h 957M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:38:21.408 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 101s ago 7h 1226M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:38:21.408 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 101s ago 7h 1095M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:38:21.409 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 101s ago 7h 1300M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:38:21.409 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 101s ago 7h 171M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:38:21.759 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:38:21.759 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:38:21.760 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:38:21.760 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:38:21.760 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:38:21.760 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:38:21.761 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:38:21.761 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:38:21.761 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:38:21.761 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:38:21.762 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:38:21.762 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:38:21.762 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:38:21.762 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:38:21.763 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:38:21.763 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:38:21.763 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:38:21.763 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:38:21.764 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:38:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:38:21] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T12:38:22.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:38:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:38:21.759Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:38:22.598 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:38:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:38:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:38:22.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:38:22 smithi083 ceph-mon[77242]: from='client.59065 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:38:22.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:38:22 smithi083 ceph-mon[77242]: from='client.59071 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:38:22.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:38:22 smithi083 ceph-mon[77242]: pgmap v12902: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:22.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:38:22 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1292110734' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:38:22.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:38:22 smithi083 ceph-mon[79930]: from='client.59065 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:38:22.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:38:22 smithi083 ceph-mon[79930]: from='client.59071 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:38:22.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:38:22 smithi083 ceph-mon[79930]: pgmap v12902: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:22.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:38:22 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1292110734' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:38:23.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:38:22 smithi153 ceph-mon[70044]: from='client.59065 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:38:23.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:38:22 smithi153 ceph-mon[70044]: from='client.59071 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:38:23.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:38:22 smithi153 ceph-mon[70044]: pgmap v12902: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:23.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:38:22 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1292110734' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:38:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:38:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:38:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:38:24.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:38:24 smithi083 ceph-mon[79930]: pgmap v12903: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:24.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:38:24 smithi083 ceph-mon[77242]: pgmap v12903: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:25.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:38:24 smithi153 ceph-mon[70044]: pgmap v12903: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:25.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:38:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:38:25.129Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:38:26.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:38:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:38:25.852Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:38:26.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:38:26 smithi083 ceph-mon[77242]: pgmap v12904: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:26.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:38:26 smithi083 ceph-mon[79930]: pgmap v12904: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:26.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:38:26 smithi153 ceph-mon[70044]: pgmap v12904: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:28.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:38:28 smithi083 ceph-mon[79930]: pgmap v12905: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:28.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:38:28 smithi083 ceph-mon[77242]: pgmap v12905: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:29.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:38:28 smithi153 ceph-mon[70044]: pgmap v12905: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:30.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:38:30 smithi083 ceph-mon[77242]: pgmap v12906: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:30.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:38:30 smithi083 ceph-mon[79930]: pgmap v12906: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:31.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:38:30 smithi153 ceph-mon[70044]: pgmap v12906: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:31.763 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:38:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:38:31] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T12:38:32.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:38:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:38:31.760Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:38:32.602 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:38:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:38:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:38:32.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:38:32 smithi083 ceph-mon[79930]: pgmap v12907: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:32.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:38:32 smithi083 ceph-mon[77242]: pgmap v12907: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:33.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:38:32 smithi153 ceph-mon[70044]: pgmap v12907: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:38:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:38:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:38:34.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:38:34 smithi083 ceph-mon[77242]: pgmap v12908: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:34.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:38:34 smithi083 ceph-mon[79930]: pgmap v12908: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:35.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:38:34 smithi153 ceph-mon[70044]: pgmap v12908: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:35.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:38:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:38:35.129Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:38:36.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:38:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:38:35.853Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:38:36.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:38:36 smithi083 ceph-mon[77242]: pgmap v12909: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:36.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:38:36 smithi083 ceph-mon[79930]: pgmap v12909: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:36.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:38:36 smithi153 ceph-mon[70044]: pgmap v12909: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:38.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:38:38 smithi083 ceph-mon[77242]: pgmap v12910: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:38.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:38:38 smithi083 ceph-mon[79930]: pgmap v12910: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:39.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:38:38 smithi153 ceph-mon[70044]: pgmap v12910: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:38:40 smithi083 ceph-mon[79930]: pgmap v12911: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:38:40 smithi083 ceph-mon[77242]: pgmap v12911: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:38:40 smithi153 ceph-mon[70044]: pgmap v12911: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:41.763 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:38:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:38:41] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T12:38:41.764 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:38:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:38:41.764 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:38:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:38:41.764 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:38:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:38:41.764 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:38:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:38:41.765 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:38:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:38:41.765 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:38:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:38:41.765 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:38:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:38:41.765 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:38:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:38:41.766 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:38:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:38:41.766 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:38:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:38:41.766 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:38:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:38:41.766 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:38:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:38:41.767 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:38:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:38:41.767 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:38:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:38:41.767 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:38:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:38:41.767 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:38:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:38:42.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:38:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:38:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:38:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:38:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:38:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:38:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:38:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:38:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:38:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:38:42.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:38:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:38:42.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:38:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:38:42.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:38:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:38:42.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:38:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:38:41.760Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:38:42.606 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:38:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:38:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:38:42.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:38:42 smithi083 ceph-mon[79930]: pgmap v12912: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:42.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:38:42 smithi083 ceph-mon[77242]: pgmap v12912: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:43.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:38:42 smithi153 ceph-mon[70044]: pgmap v12912: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:38:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:38:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:38:44.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:38:44 smithi083 ceph-mon[79930]: pgmap v12913: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:44.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:38:44 smithi083 ceph-mon[77242]: pgmap v12913: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:45.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:38:44 smithi153 ceph-mon[70044]: pgmap v12913: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:45.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:38:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:38:45.130Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:38:46.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:38:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:38:45.854Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:38:46.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:38:46 smithi083 ceph-mon[79930]: pgmap v12914: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:46.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:38:46 smithi083 ceph-mon[77242]: pgmap v12914: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:46.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:38:46 smithi153 ceph-mon[70044]: pgmap v12914: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:48.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:38:48 smithi083 ceph-mon[77242]: pgmap v12915: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:48.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:38:48 smithi083 ceph-mon[79930]: pgmap v12915: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:49.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:38:48 smithi153 ceph-mon[70044]: pgmap v12915: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:50.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:38:50 smithi083 ceph-mon[79930]: pgmap v12916: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:50.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:38:50 smithi083 ceph-mon[77242]: pgmap v12916: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:51.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:38:50 smithi153 ceph-mon[70044]: pgmap v12916: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:51.765 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:38:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:38:51] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T12:38:52.084 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:38:52.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:38:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:38:51.762Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:38:52.374 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:38:52.374 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 4m ago 7h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:38:52.374 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 2m ago 7h 64.1M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:38:52.375 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 4m ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:38:52.375 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 2m ago 7h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:38:52.375 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 2m ago 7h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:38:52.375 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 4m ago 7h 872M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:38:52.376 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 4m ago 7h 1595M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:38:52.376 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 2m ago 7h 688M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:38:52.376 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 4m ago 7h 924M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:38:52.376 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 4m ago 7h 26.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:38:52.377 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 2m ago 7h 25.8M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:38:52.377 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 4m ago 7h 1152M 2366M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:38:52.377 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 4m ago 7h 1139M 2366M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:38:52.377 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 4m ago 7h 867M 2366M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:38:52.378 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 4m ago 7h 1115M 2366M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:38:52.378 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 2m ago 7h 957M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:38:52.378 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 2m ago 7h 1226M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:38:52.378 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 2m ago 7h 1095M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:38:52.379 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 2m ago 7h 1300M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:38:52.379 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 2m ago 7h 171M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:38:52.606 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:38:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:38:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:38:52.731 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:38:52.731 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:38:52.732 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:38:52.732 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:38:52.732 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:38:52.732 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:38:52.733 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:38:52.733 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:38:52.733 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:38:52.734 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:38:52.734 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:38:52.734 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:38:52.734 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:38:52.735 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:38:52.735 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:38:52.735 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:38:52.735 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:38:52.736 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:38:52.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:38:52 smithi083 ceph-mon[77242]: pgmap v12917: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:52.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:38:52 smithi083 ceph-mon[79930]: pgmap v12917: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:53.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:38:52 smithi153 ceph-mon[70044]: pgmap v12917: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:38:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:38:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:38:53.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:38:53 smithi083 ceph-mon[77242]: from='client.49152 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:38:53.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:38:53 smithi083 ceph-mon[77242]: from='client.59089 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:38:53.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:38:53 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3665737659' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:38:53.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:38:53 smithi083 ceph-mon[79930]: from='client.49152 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:38:53.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:38:53 smithi083 ceph-mon[79930]: from='client.59089 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:38:53.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:38:53 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3665737659' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:38:54.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:38:53 smithi153 ceph-mon[70044]: from='client.49152 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:38:54.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:38:53 smithi153 ceph-mon[70044]: from='client.59089 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:38:54.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:38:53 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3665737659' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:38:54.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:38:54 smithi083 ceph-mon[79930]: pgmap v12918: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:54.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:38:54 smithi083 ceph-mon[77242]: pgmap v12918: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:55.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:38:54 smithi153 ceph-mon[70044]: pgmap v12918: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:38:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:38:55.131Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:38:56.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:38:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:38:55.854Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:38:56.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:38:56 smithi083 ceph-mon[79930]: pgmap v12919: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:56.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:38:56 smithi083 ceph-mon[77242]: pgmap v12919: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:56.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:38:56 smithi153 ceph-mon[70044]: pgmap v12919: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:58.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:38:58 smithi083 ceph-mon[79930]: pgmap v12920: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:58.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:38:58 smithi083 ceph-mon[77242]: pgmap v12920: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:38:59.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:38:58 smithi153 ceph-mon[70044]: pgmap v12920: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:00.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:39:00 smithi083 ceph-mon[79930]: pgmap v12921: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:00.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:39:00 smithi083 ceph-mon[77242]: pgmap v12921: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:01.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:39:00 smithi153 ceph-mon[70044]: pgmap v12921: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:01.766 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:39:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:39:01] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T12:39:02.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:39:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:39:01.763Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:39:02.610 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:39:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:39:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:39:02.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:39:02 smithi083 ceph-mon[79930]: pgmap v12922: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:02.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:39:02 smithi083 ceph-mon[77242]: pgmap v12922: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:03.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:39:02 smithi153 ceph-mon[70044]: pgmap v12922: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:39:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:39:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:39:04.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:39:04 smithi083 ceph-mon[79930]: pgmap v12923: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:04.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:39:04 smithi083 ceph-mon[77242]: pgmap v12923: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:05.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:39:04 smithi153 ceph-mon[70044]: pgmap v12923: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:05.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:39:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:39:05.131Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:39:06.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:39:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:39:05.855Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:39:06.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:39:06 smithi083 ceph-mon[79930]: pgmap v12924: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:06.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:39:06 smithi083 ceph-mon[77242]: pgmap v12924: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:06.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:39:06 smithi153 ceph-mon[70044]: pgmap v12924: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:08.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:39:08 smithi083 ceph-mon[79930]: pgmap v12925: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:08.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:39:08 smithi083 ceph-mon[77242]: pgmap v12925: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:09.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:39:08 smithi153 ceph-mon[70044]: pgmap v12925: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:10.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:39:10 smithi083 ceph-mon[79930]: pgmap v12926: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:10.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:39:10 smithi083 ceph-mon[77242]: pgmap v12926: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:11.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:39:10 smithi153 ceph-mon[70044]: pgmap v12926: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:11.767 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:39:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:39:11] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T12:39:12.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:39:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:39:11.764Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:39:12.614 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:39:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:39:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:39:12.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:39:12 smithi083 ceph-mon[79930]: pgmap v12927: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:12.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:39:12 smithi083 ceph-mon[77242]: pgmap v12927: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:13.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:39:12 smithi153 ceph-mon[70044]: pgmap v12927: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:39:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:39:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:39:14.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:39:14 smithi083 ceph-mon[79930]: pgmap v12928: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:14.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:39:14 smithi083 ceph-mon[77242]: pgmap v12928: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:15.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:39:14 smithi153 ceph-mon[70044]: pgmap v12928: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:15.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:39:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:39:15.132Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:39:16.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:39:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:39:15.856Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:39:16.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:39:16 smithi083 ceph-mon[77242]: pgmap v12929: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:16.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:39:16 smithi083 ceph-mon[79930]: pgmap v12929: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:16.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:39:16 smithi153 ceph-mon[70044]: pgmap v12929: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:18.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:39:18 smithi083 ceph-mon[79930]: pgmap v12930: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:18.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:39:18 smithi083 ceph-mon[77242]: pgmap v12930: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:19.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:39:18 smithi153 ceph-mon[70044]: pgmap v12930: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:20.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:39:20 smithi083 ceph-mon[79930]: pgmap v12931: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:20.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:39:20 smithi083 ceph-mon[77242]: pgmap v12931: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:21.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:39:20 smithi153 ceph-mon[70044]: pgmap v12931: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:21.767 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:39:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:39:21] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T12:39:22.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:39:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:39:21.765Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:39:22.616 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:39:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:39:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:39:22.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:39:22 smithi083 ceph-mon[79930]: pgmap v12932: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:22.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:39:22 smithi083 ceph-mon[77242]: pgmap v12932: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:23.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:39:22 smithi153 ceph-mon[70044]: pgmap v12932: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:23.057 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:39:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:39:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:39:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:39:23.347 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:39:23.348 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 4m ago 7h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:39:23.348 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 2m ago 7h 64.1M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:39:23.348 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 4m ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:39:23.348 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 2m ago 7h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:39:23.349 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 2m ago 7h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:39:23.349 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 4m ago 7h 872M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:39:23.349 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 4m ago 7h 1595M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:39:23.349 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 2m ago 7h 688M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:39:23.349 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 4m ago 7h 924M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:39:23.350 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 4m ago 7h 26.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:39:23.350 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 2m ago 7h 25.8M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:39:23.350 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 4m ago 7h 1152M 2366M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:39:23.350 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 4m ago 7h 1139M 2366M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:39:23.351 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 4m ago 7h 867M 2366M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:39:23.351 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 4m ago 7h 1115M 2366M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:39:23.351 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 2m ago 7h 957M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:39:23.351 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 2m ago 7h 1226M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:39:23.352 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 2m ago 7h 1095M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:39:23.352 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 2m ago 7h 1300M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:39:23.352 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 2m ago 7h 171M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:39:23.710 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:39:23.710 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:39:23.710 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:39:23.711 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:39:23.711 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:39:23.711 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:39:23.711 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:39:23.711 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:39:23.712 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:39:23.712 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:39:23.712 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:39:23.712 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:39:23.713 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:39:23.713 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:39:23.713 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:39:23.713 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:39:23.713 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:39:23.714 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:39:24.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:39:24 smithi083 ceph-mon[79930]: from='client.59101 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:39:24.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:39:24 smithi083 ceph-mon[79930]: from='client.49176 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:39:24.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:39:24 smithi083 ceph-mon[79930]: pgmap v12933: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:24.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:39:24 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1137027804' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:39:24.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:39:24 smithi083 ceph-mon[77242]: from='client.59101 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:39:24.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:39:24 smithi083 ceph-mon[77242]: from='client.49176 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:39:24.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:39:24 smithi083 ceph-mon[77242]: pgmap v12933: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:24.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:39:24 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1137027804' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:39:25.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:39:24 smithi153 ceph-mon[70044]: from='client.59101 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:39:25.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:39:24 smithi153 ceph-mon[70044]: from='client.49176 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:39:25.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:39:24 smithi153 ceph-mon[70044]: pgmap v12933: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:25.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:39:24 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1137027804' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:39:25.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:39:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:39:25.133Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:39:26.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:39:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:39:25.856Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:39:26.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:39:26 smithi083 ceph-mon[79930]: pgmap v12934: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:26.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:39:26 smithi083 ceph-mon[77242]: pgmap v12934: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:26.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:39:26 smithi153 ceph-mon[70044]: pgmap v12934: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:28.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:39:28 smithi083 ceph-mon[79930]: pgmap v12935: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:28.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:39:28 smithi083 ceph-mon[77242]: pgmap v12935: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:29.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:39:28 smithi153 ceph-mon[70044]: pgmap v12935: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:30.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:39:30 smithi083 ceph-mon[79930]: pgmap v12936: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:30.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:39:30 smithi083 ceph-mon[77242]: pgmap v12936: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:31.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:39:30 smithi153 ceph-mon[70044]: pgmap v12936: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:31.768 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:39:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:39:31] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T12:39:32.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:39:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:39:31.765Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:39:32.621 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:39:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:39:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:39:32.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:39:32 smithi083 ceph-mon[79930]: pgmap v12937: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:32.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:39:32 smithi083 ceph-mon[77242]: pgmap v12937: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:33.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:39:32 smithi153 ceph-mon[70044]: pgmap v12937: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:39:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:39:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:39:34.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:39:34 smithi083 ceph-mon[77242]: pgmap v12938: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:34.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:39:34 smithi083 ceph-mon[79930]: pgmap v12938: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:35.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:39:34 smithi153 ceph-mon[70044]: pgmap v12938: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:35.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:39:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:39:35.133Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:39:36.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:39:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:39:35.857Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:39:36.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:39:36 smithi083 ceph-mon[79930]: pgmap v12939: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:36.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:39:36 smithi083 ceph-mon[77242]: pgmap v12939: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:36.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:39:36 smithi153 ceph-mon[70044]: pgmap v12939: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:38.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:39:38 smithi083 ceph-mon[77242]: pgmap v12940: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:38.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:39:38 smithi083 ceph-mon[79930]: pgmap v12940: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:39.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:39:38 smithi153 ceph-mon[70044]: pgmap v12940: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:39:40 smithi083 ceph-mon[79930]: pgmap v12941: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:39:40 smithi083 ceph-mon[77242]: pgmap v12941: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:39:40 smithi153 ceph-mon[70044]: pgmap v12941: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:41.767 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:39:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:39:41] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T12:39:41.768 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:39:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:39:41.768 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:39:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:39:41.769 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:39:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:39:41.769 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:39:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:39:41.769 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:39:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:39:41.769 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:39:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:39:41.770 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:39:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:39:41.770 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:39:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:39:41.770 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:39:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:39:41.771 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:39:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:39:41.771 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:39:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:39:41.771 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:39:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:39:41.771 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:39:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:39:41.771 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:39:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:39:42.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:39:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:39:42.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:39:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:39:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:39:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:39:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:39:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:39:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:39:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:39:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:39:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:39:42.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:39:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:39:42.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:39:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:39:41.765Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:39:42.625 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:39:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:39:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:39:42.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:39:42 smithi083 ceph-mon[79930]: pgmap v12942: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:42.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:39:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:39:42.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:39:42 smithi083 ceph-mon[77242]: pgmap v12942: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:42.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:39:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:39:43.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:39:42 smithi153 ceph-mon[70044]: pgmap v12942: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:43.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:39:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:39:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:39:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:39:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:39:44.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:39:44 smithi083 ceph-mon[79930]: pgmap v12943: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:44.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:39:44 smithi083 ceph-mon[77242]: pgmap v12943: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:45.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:39:44 smithi153 ceph-mon[70044]: pgmap v12943: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:45.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:39:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:39:45.134Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:39:46.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:39:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:39:45.858Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:39:46.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:39:46 smithi083 ceph-mon[79930]: pgmap v12944: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:46.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:39:46 smithi083 ceph-mon[77242]: pgmap v12944: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:46.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:39:46 smithi153 ceph-mon[70044]: pgmap v12944: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:48.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:39:48 smithi083 ceph-mon[79930]: pgmap v12945: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:48.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:39:48 smithi083 ceph-mon[77242]: pgmap v12945: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:49.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:39:48 smithi153 ceph-mon[70044]: pgmap v12945: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:50.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:39:50 smithi083 ceph-mon[77242]: pgmap v12946: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:50.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:39:50 smithi083 ceph-mon[79930]: pgmap v12946: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:51.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:39:50 smithi153 ceph-mon[70044]: pgmap v12946: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:51.769 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:39:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:39:51] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T12:39:52.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:39:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:39:51.766Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:39:52.624 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:39:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:39:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:39:52.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:39:52 smithi083 ceph-mon[79930]: pgmap v12947: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:52.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:39:52 smithi083 ceph-mon[77242]: pgmap v12947: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:53.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:39:52 smithi153 ceph-mon[70044]: pgmap v12947: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:39:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:39:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:39:54.039 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:39:54.327 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:39:54.328 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 5m ago 7h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:39:54.328 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 3m ago 7h 64.1M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:39:54.328 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 5m ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:39:54.328 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 3m ago 7h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:39:54.329 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 3m ago 7h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:39:54.329 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 5m ago 7h 872M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:39:54.329 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 5m ago 7h 1595M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:39:54.329 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 3m ago 7h 688M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:39:54.329 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 5m ago 7h 924M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:39:54.330 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 5m ago 7h 26.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:39:54.330 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 3m ago 7h 25.8M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:39:54.330 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 5m ago 7h 1152M 2366M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:39:54.330 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 5m ago 7h 1139M 2366M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:39:54.331 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 5m ago 7h 867M 2366M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:39:54.331 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 5m ago 7h 1115M 2366M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:39:54.331 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 3m ago 7h 957M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:39:54.331 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 3m ago 7h 1226M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:39:54.331 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 3m ago 7h 1095M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:39:54.332 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 3m ago 7h 1300M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:39:54.332 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 3m ago 7h 171M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:39:54.688 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:39:54.689 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:39:54.689 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:39:54.690 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:39:54.690 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:39:54.690 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:39:54.690 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:39:54.690 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:39:54.691 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:39:54.691 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:39:54.691 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:39:54.691 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:39:54.692 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:39:54.692 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:39:54.692 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:39:54.692 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:39:54.693 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:39:54.693 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:39:54.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:39:54 smithi083 ceph-mon[79930]: pgmap v12948: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:54.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:39:54 smithi083 ceph-mon[77242]: pgmap v12948: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:55.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:39:54 smithi153 ceph-mon[70044]: pgmap v12948: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:39:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:39:55.133Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:39:55.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:39:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:39:55.858Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:39:55.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:39:55 smithi083 ceph-mon[77242]: from='client.59119 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:39:55.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:39:55 smithi083 ceph-mon[77242]: from='client.49194 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:39:55.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:39:55 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/474602424' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:39:55.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:39:55 smithi083 ceph-mon[79930]: from='client.59119 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:39:55.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:39:55 smithi083 ceph-mon[79930]: from='client.49194 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:39:55.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:39:55 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/474602424' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:39:56.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:39:55 smithi153 ceph-mon[70044]: from='client.59119 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:39:56.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:39:55 smithi153 ceph-mon[70044]: from='client.49194 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:39:56.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:39:55 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/474602424' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:39:56.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:39:56 smithi083 ceph-mon[79930]: pgmap v12949: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:56.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:39:56 smithi083 ceph-mon[77242]: pgmap v12949: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:57.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:39:56 smithi153 ceph-mon[70044]: pgmap v12949: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:58.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:39:58 smithi083 ceph-mon[79930]: pgmap v12950: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:58.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:39:58 smithi083 ceph-mon[77242]: pgmap v12950: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:39:59.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:39:58 smithi153 ceph-mon[70044]: pgmap v12950: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:00.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:40:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[77219]: 2022-09-29T12:39:59.999+0000 7f160f665700 -1 log_channel(cluster) log [ERR] : overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T12:40:00.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:40:00 smithi083 ceph-mon[77242]: pgmap v12951: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:00.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:40:00 smithi083 ceph-mon[77242]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T12:40:00.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:40:00 smithi083 ceph-mon[79930]: pgmap v12951: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:00.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:40:00 smithi083 ceph-mon[79930]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T12:40:01.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:40:00 smithi153 ceph-mon[70044]: pgmap v12951: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:01.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:40:00 smithi153 ceph-mon[70044]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T12:40:01.771 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:40:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:40:01] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T12:40:02.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:40:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:40:01.769Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:40:02.641 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:40:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:40:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:40:02.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:40:02 smithi083 ceph-mon[77242]: pgmap v12952: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:02.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:40:02 smithi083 ceph-mon[79930]: pgmap v12952: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:03.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:40:02 smithi153 ceph-mon[70044]: pgmap v12952: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:40:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:40:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:40:04.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:40:04 smithi083 ceph-mon[77242]: pgmap v12953: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:04.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:40:04 smithi083 ceph-mon[79930]: pgmap v12953: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:05.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:40:04 smithi153 ceph-mon[70044]: pgmap v12953: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:05.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:40:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:40:05.135Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:40:06.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:40:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:40:05.859Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:40:06.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:40:06 smithi083 ceph-mon[79930]: pgmap v12954: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:06.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:40:06 smithi083 ceph-mon[77242]: pgmap v12954: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:06.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:40:06 smithi153 ceph-mon[70044]: pgmap v12954: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:08.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:40:08 smithi083 ceph-mon[79930]: pgmap v12955: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:08.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:40:08 smithi083 ceph-mon[77242]: pgmap v12955: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:09.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:40:08 smithi153 ceph-mon[70044]: pgmap v12955: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:10.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:40:10 smithi083 ceph-mon[79930]: pgmap v12956: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:10.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:40:10 smithi083 ceph-mon[77242]: pgmap v12956: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:11.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:40:10 smithi153 ceph-mon[70044]: pgmap v12956: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:11.769 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:40:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:40:11] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T12:40:12.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:40:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:40:11.767Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:40:12.633 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:40:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:40:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:40:12.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:40:12 smithi083 ceph-mon[79930]: pgmap v12957: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:12.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:40:12 smithi083 ceph-mon[77242]: pgmap v12957: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:13.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:40:12 smithi153 ceph-mon[70044]: pgmap v12957: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:40:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:40:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:40:14.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:40:14 smithi083 ceph-mon[79930]: pgmap v12958: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:14.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:40:14 smithi083 ceph-mon[77242]: pgmap v12958: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:15.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:40:14 smithi153 ceph-mon[70044]: pgmap v12958: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:15.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:40:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:40:15.136Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:40:16.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:40:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:40:15.860Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:40:16.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:40:16 smithi083 ceph-mon[77242]: pgmap v12959: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:16.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:40:16 smithi083 ceph-mon[79930]: pgmap v12959: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:16.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:40:16 smithi153 ceph-mon[70044]: pgmap v12959: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:18.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:40:18 smithi083 ceph-mon[79930]: pgmap v12960: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:18.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:40:18 smithi083 ceph-mon[77242]: pgmap v12960: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:19.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:40:18 smithi153 ceph-mon[70044]: pgmap v12960: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:20.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:40:20 smithi083 ceph-mon[79930]: pgmap v12961: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:20.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:40:20 smithi083 ceph-mon[77242]: pgmap v12961: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:21.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:40:20 smithi153 ceph-mon[70044]: pgmap v12961: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:21.771 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:40:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:40:21] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T12:40:22.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:40:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:40:21.769Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:40:22.635 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:40:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:40:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:40:22.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:40:22 smithi083 ceph-mon[79930]: pgmap v12962: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:22.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:40:22 smithi083 ceph-mon[77242]: pgmap v12962: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:23.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:40:22 smithi153 ceph-mon[70044]: pgmap v12962: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:40:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:40:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:40:24.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:40:24 smithi083 ceph-mon[79930]: pgmap v12963: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:24.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:40:24 smithi083 ceph-mon[77242]: pgmap v12963: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:25.018 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:40:25.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:40:24 smithi153 ceph-mon[70044]: pgmap v12963: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:25.307 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:40:25.307 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 5m ago 7h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:40:25.308 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 3m ago 7h 64.1M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:40:25.308 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 5m ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:40:25.308 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 3m ago 7h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:40:25.308 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 3m ago 7h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:40:25.309 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 5m ago 7h 872M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:40:25.309 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 5m ago 7h 1595M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:40:25.309 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 3m ago 7h 688M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:40:25.309 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 5m ago 7h 924M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:40:25.310 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 5m ago 7h 26.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:40:25.310 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 3m ago 7h 25.8M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:40:25.310 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 5m ago 7h 1152M 2366M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:40:25.311 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 5m ago 7h 1139M 2366M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:40:25.311 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 5m ago 7h 867M 2366M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:40:25.311 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 5m ago 7h 1115M 2366M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:40:25.311 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 3m ago 7h 957M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:40:25.312 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 3m ago 7h 1226M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:40:25.312 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 3m ago 7h 1095M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:40:25.312 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 3m ago 7h 1300M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:40:25.312 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 3m ago 7h 171M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:40:25.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:40:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:40:25.135Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:40:25.665 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:40:25.666 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:40:25.666 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:40:25.666 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:40:25.666 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:40:25.667 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:40:25.667 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:40:25.667 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:40:25.667 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:40:25.668 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:40:25.668 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:40:25.668 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:40:25.668 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:40:25.668 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:40:25.669 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:40:25.669 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:40:25.669 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:40:25.669 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:40:26.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:40:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:40:25.860Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:40:26.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:40:26 smithi083 ceph-mon[79930]: from='client.59137 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:40:26.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:40:26 smithi083 ceph-mon[79930]: from='client.49212 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:40:26.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:40:26 smithi083 ceph-mon[79930]: pgmap v12964: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:26.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:40:26 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2411862135' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:40:26.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:40:26 smithi083 ceph-mon[77242]: from='client.59137 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:40:26.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:40:26 smithi083 ceph-mon[77242]: from='client.49212 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:40:26.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:40:26 smithi083 ceph-mon[77242]: pgmap v12964: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:26.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:40:26 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2411862135' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:40:26.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:40:26 smithi153 ceph-mon[70044]: from='client.59137 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:40:26.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:40:26 smithi153 ceph-mon[70044]: from='client.49212 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:40:26.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:40:26 smithi153 ceph-mon[70044]: pgmap v12964: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:26.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:40:26 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2411862135' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:40:28.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:40:28 smithi083 ceph-mon[79930]: pgmap v12965: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:28.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:40:28 smithi083 ceph-mon[77242]: pgmap v12965: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:29.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:40:28 smithi153 ceph-mon[70044]: pgmap v12965: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:30.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:40:30 smithi083 ceph-mon[79930]: pgmap v12966: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:30.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:40:30 smithi083 ceph-mon[77242]: pgmap v12966: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:31.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:40:30 smithi153 ceph-mon[70044]: pgmap v12966: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:31.771 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:40:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:40:31] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T12:40:32.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:40:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:40:31.769Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:40:32.635 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:40:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:40:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:40:32.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:40:32 smithi083 ceph-mon[79930]: pgmap v12967: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:32.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:40:32 smithi083 ceph-mon[77242]: pgmap v12967: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:33.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:40:32 smithi153 ceph-mon[70044]: pgmap v12967: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:40:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:40:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:40:34.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:40:34 smithi083 ceph-mon[77242]: pgmap v12968: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:34.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:40:34 smithi083 ceph-mon[79930]: pgmap v12968: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:35.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:40:34 smithi153 ceph-mon[70044]: pgmap v12968: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:35.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:40:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:40:35.137Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:40:36.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:40:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:40:35.861Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:40:36.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:40:36 smithi083 ceph-mon[77242]: pgmap v12969: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:36.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:40:36 smithi083 ceph-mon[79930]: pgmap v12969: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:36.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:40:36 smithi153 ceph-mon[70044]: pgmap v12969: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:38.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:40:38 smithi083 ceph-mon[79930]: pgmap v12970: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:38.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:40:38 smithi083 ceph-mon[77242]: pgmap v12970: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:39.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:40:38 smithi153 ceph-mon[70044]: pgmap v12970: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:40:40 smithi083 ceph-mon[79930]: pgmap v12971: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:40:40 smithi083 ceph-mon[77242]: pgmap v12971: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:40:40 smithi153 ceph-mon[70044]: pgmap v12971: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:41.772 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:40:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:40:41] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T12:40:41.773 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:40:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:40:41.773 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:40:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:40:41.774 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:40:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:40:41.774 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:40:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:40:41.774 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:40:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:40:41.775 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:40:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:40:41.775 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:40:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:40:41.775 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:40:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:40:42.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:40:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:40:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:40:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:40:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:40:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:40:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:40:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:40:42.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:40:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:40:41.770Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:40:42.642 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:40:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:40:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:40:42.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:40:42 smithi083 ceph-mon[79930]: pgmap v12972: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:42.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:40:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:40:42.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:40:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:40:42.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:40:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:40:42.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:40:42 smithi083 ceph-mon[77242]: pgmap v12972: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:42.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:40:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:40:42.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:40:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:40:42.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:40:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:40:43.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:40:42 smithi153 ceph-mon[70044]: pgmap v12972: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:43.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:40:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:40:43.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:40:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:40:43.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:40:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:40:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:40:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:40:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:40:44.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:40:43 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:40:44.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:40:43 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:40:44.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:40:43 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:40:45.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:40:44 smithi153 ceph-mon[70044]: pgmap v12973: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:45.140 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:40:44 smithi083 ceph-mon[79930]: pgmap v12973: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:45.141 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:40:44 smithi083 ceph-mon[77242]: pgmap v12973: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:45.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:40:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:40:45.137Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:40:46.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:40:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:40:45.861Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:40:46.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:40:46 smithi083 ceph-mon[79930]: pgmap v12974: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:46.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:40:46 smithi083 ceph-mon[77242]: pgmap v12974: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:46.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:40:46 smithi153 ceph-mon[70044]: pgmap v12974: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:48.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:40:48 smithi083 ceph-mon[79930]: pgmap v12975: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:48.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:40:48 smithi083 ceph-mon[77242]: pgmap v12975: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:49.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:40:48 smithi153 ceph-mon[70044]: pgmap v12975: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:50.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:40:50 smithi083 ceph-mon[79930]: pgmap v12976: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:50.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:40:50 smithi083 ceph-mon[77242]: pgmap v12976: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:51.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:40:50 smithi153 ceph-mon[70044]: pgmap v12976: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:51.773 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:40:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:40:51] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T12:40:52.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:40:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:40:51.771Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:40:52.643 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:40:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:40:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:40:52.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:40:52 smithi083 ceph-mon[77242]: pgmap v12977: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:52.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:40:52 smithi083 ceph-mon[79930]: pgmap v12977: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:53.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:40:52 smithi153 ceph-mon[70044]: pgmap v12977: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:40:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:40:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:40:54.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:40:54 smithi083 ceph-mon[79930]: pgmap v12978: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:54.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:40:54 smithi083 ceph-mon[77242]: pgmap v12978: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:55.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:40:54 smithi153 ceph-mon[70044]: pgmap v12978: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:40:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:40:55.138Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:40:55.997 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:40:56.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:40:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:40:55.862Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:40:56.289 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:40:56.290 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 6m ago 7h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:40:56.290 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 4m ago 7h 64.1M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:40:56.290 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 6m ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:40:56.290 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 4m ago 7h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:40:56.291 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 4m ago 7h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:40:56.291 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 6m ago 7h 872M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:40:56.291 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 6m ago 7h 1595M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:40:56.291 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 4m ago 7h 688M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:40:56.292 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 6m ago 7h 924M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:40:56.292 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 6m ago 7h 26.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:40:56.292 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 4m ago 7h 25.8M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:40:56.292 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 6m ago 7h 1152M 2366M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:40:56.293 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 6m ago 7h 1139M 2366M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:40:56.293 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 6m ago 7h 867M 2366M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:40:56.293 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 6m ago 7h 1115M 2366M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:40:56.293 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 4m ago 7h 957M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:40:56.293 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 4m ago 7h 1226M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:40:56.294 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 4m ago 7h 1095M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:40:56.294 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 4m ago 7h 1300M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:40:56.294 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 4m ago 7h 171M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:40:56.648 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:40:56.649 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:40:56.649 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:40:56.649 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:40:56.649 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:40:56.650 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:40:56.650 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:40:56.650 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:40:56.650 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:40:56.651 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:40:56.651 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:40:56.651 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:40:56.651 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:40:56.652 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:40:56.652 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:40:56.652 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:40:56.652 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:40:56.652 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:40:56.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:40:56 smithi083 ceph-mon[79930]: pgmap v12979: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:56.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:40:56 smithi083 ceph-mon[77242]: pgmap v12979: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:56.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:40:56 smithi153 ceph-mon[70044]: pgmap v12979: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:57.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:40:57 smithi083 ceph-mon[77242]: from='client.59155 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:40:57.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:40:57 smithi083 ceph-mon[77242]: from='client.59161 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:40:57.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:40:57 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3123314854' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:40:57.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:40:57 smithi083 ceph-mon[79930]: from='client.59155 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:40:57.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:40:57 smithi083 ceph-mon[79930]: from='client.59161 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:40:57.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:40:57 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3123314854' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:40:57.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:40:57 smithi153 ceph-mon[70044]: from='client.59155 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:40:57.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:40:57 smithi153 ceph-mon[70044]: from='client.59161 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:40:57.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:40:57 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3123314854' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:40:58.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:40:58 smithi083 ceph-mon[77242]: pgmap v12980: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:58.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:40:58 smithi083 ceph-mon[79930]: pgmap v12980: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:40:58.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:40:58 smithi153 ceph-mon[70044]: pgmap v12980: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:00.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:41:00 smithi083 ceph-mon[79930]: pgmap v12981: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:00.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:41:00 smithi083 ceph-mon[77242]: pgmap v12981: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:01.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:41:00 smithi153 ceph-mon[70044]: pgmap v12981: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:01.774 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:41:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:41:01] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T12:41:02.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:41:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:41:01.771Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:41:02.644 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:41:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:41:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:41:02.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:41:02 smithi083 ceph-mon[77242]: pgmap v12982: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:02.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:41:02 smithi083 ceph-mon[79930]: pgmap v12982: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:03.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:41:02 smithi153 ceph-mon[70044]: pgmap v12982: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:41:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:41:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:41:04.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:41:04 smithi083 ceph-mon[79930]: pgmap v12983: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:04.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:41:04 smithi083 ceph-mon[77242]: pgmap v12983: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:05.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:41:04 smithi153 ceph-mon[70044]: pgmap v12983: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:05.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:41:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:41:05.138Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:41:06.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:41:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:41:05.863Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:41:06.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:41:06 smithi083 ceph-mon[79930]: pgmap v12984: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:06.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:41:06 smithi083 ceph-mon[77242]: pgmap v12984: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:06.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:41:06 smithi153 ceph-mon[70044]: pgmap v12984: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:08.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:41:08 smithi083 ceph-mon[77242]: pgmap v12985: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:08.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:41:08 smithi083 ceph-mon[79930]: pgmap v12985: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:09.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:41:08 smithi153 ceph-mon[70044]: pgmap v12985: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:10.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:41:10 smithi083 ceph-mon[79930]: pgmap v12986: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:10.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:41:10 smithi083 ceph-mon[77242]: pgmap v12986: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:11.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:41:10 smithi153 ceph-mon[70044]: pgmap v12986: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:11.775 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:41:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:41:11] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T12:41:12.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:41:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:41:11.772Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:41:12.648 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:41:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:41:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:41:12.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:41:12 smithi083 ceph-mon[79930]: pgmap v12987: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:12.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:41:12 smithi083 ceph-mon[77242]: pgmap v12987: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:13.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:41:12 smithi153 ceph-mon[70044]: pgmap v12987: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:41:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:41:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:41:14.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:41:14 smithi083 ceph-mon[79930]: pgmap v12988: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:14.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:41:14 smithi083 ceph-mon[77242]: pgmap v12988: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:15.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:41:14 smithi153 ceph-mon[70044]: pgmap v12988: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:15.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:41:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:41:15.139Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:41:16.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:41:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:41:15.863Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:41:16.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:41:16 smithi083 ceph-mon[79930]: pgmap v12989: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:16.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:41:16 smithi083 ceph-mon[77242]: pgmap v12989: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:16.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:41:16 smithi153 ceph-mon[70044]: pgmap v12989: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:18.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:41:18 smithi083 ceph-mon[77242]: pgmap v12990: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:18.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:41:18 smithi083 ceph-mon[79930]: pgmap v12990: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:19.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:41:18 smithi153 ceph-mon[70044]: pgmap v12990: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:20.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:41:20 smithi083 ceph-mon[79930]: pgmap v12991: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:20.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:41:20 smithi083 ceph-mon[77242]: pgmap v12991: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:21.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:41:20 smithi153 ceph-mon[70044]: pgmap v12991: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:21.775 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:41:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:41:21] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T12:41:22.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:41:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:41:21.773Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:41:22.654 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:41:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:41:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:41:22.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:41:22 smithi083 ceph-mon[77242]: pgmap v12992: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:22.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:41:22 smithi083 ceph-mon[79930]: pgmap v12992: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:23.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:41:22 smithi153 ceph-mon[70044]: pgmap v12992: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:41:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:41:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:41:24.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:41:24 smithi083 ceph-mon[79930]: pgmap v12993: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:24.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:41:24 smithi083 ceph-mon[77242]: pgmap v12993: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:25.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:41:24 smithi153 ceph-mon[70044]: pgmap v12993: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:25.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:41:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:41:25.139Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:41:26.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:41:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:41:25.864Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:41:26.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:41:26 smithi083 ceph-mon[79930]: pgmap v12994: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:26.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:41:26 smithi083 ceph-mon[77242]: pgmap v12994: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:26.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:41:26 smithi153 ceph-mon[70044]: pgmap v12994: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:26.974 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:41:27.265 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:41:27.265 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 6m ago 7h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:41:27.265 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 4m ago 7h 64.1M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:41:27.265 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 6m ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:41:27.266 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 4m ago 7h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:41:27.266 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 4m ago 7h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:41:27.266 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 6m ago 7h 872M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:41:27.266 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 6m ago 7h 1595M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:41:27.267 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 4m ago 7h 688M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:41:27.267 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 6m ago 7h 924M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:41:27.267 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 6m ago 7h 26.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:41:27.267 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 4m ago 7h 25.8M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:41:27.268 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 6m ago 7h 1152M 2366M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:41:27.268 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 6m ago 7h 1139M 2366M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:41:27.268 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 6m ago 7h 867M 2366M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:41:27.268 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 6m ago 7h 1115M 2366M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:41:27.268 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 4m ago 7h 957M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:41:27.269 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 4m ago 7h 1226M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:41:27.269 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 4m ago 7h 1095M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:41:27.269 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 4m ago 7h 1300M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:41:27.269 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 4m ago 7h 171M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:41:27.617 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:41:27.618 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:41:27.618 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:41:27.618 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:41:27.618 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:41:27.619 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:41:27.619 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:41:27.619 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:41:27.619 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:41:27.619 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:41:27.620 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:41:27.620 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:41:27.620 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:41:27.620 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:41:27.621 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:41:27.621 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:41:27.621 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:41:27.621 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:41:27.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:41:27 smithi083 ceph-mon[79930]: from='client.59173 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:41:27.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:41:27 smithi083 ceph-mon[77242]: from='client.59173 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:41:27.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:41:27 smithi153 ceph-mon[70044]: from='client.59173 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:41:28.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:41:28 smithi083 ceph-mon[79930]: from='client.49248 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:41:28.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:41:28 smithi083 ceph-mon[79930]: pgmap v12995: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:28.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:41:28 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2486378246' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:41:28.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:41:28 smithi083 ceph-mon[77242]: from='client.49248 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:41:28.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:41:28 smithi083 ceph-mon[77242]: pgmap v12995: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:28.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:41:28 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2486378246' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:41:28.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:41:28 smithi153 ceph-mon[70044]: from='client.49248 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:41:28.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:41:28 smithi153 ceph-mon[70044]: pgmap v12995: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:28.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:41:28 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2486378246' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:41:30.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:41:30 smithi083 ceph-mon[79930]: pgmap v12996: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:30.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:41:30 smithi083 ceph-mon[77242]: pgmap v12996: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:31.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:41:30 smithi153 ceph-mon[70044]: pgmap v12996: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:31.777 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:41:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:41:31] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T12:41:32.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:41:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:41:31.774Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:41:32.652 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:41:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:41:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:41:32.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:41:32 smithi083 ceph-mon[79930]: pgmap v12997: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:32.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:41:32 smithi083 ceph-mon[77242]: pgmap v12997: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:33.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:41:32 smithi153 ceph-mon[70044]: pgmap v12997: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:41:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:41:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:41:34.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:41:34 smithi083 ceph-mon[79930]: pgmap v12998: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:34.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:41:34 smithi083 ceph-mon[77242]: pgmap v12998: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:35.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:41:34 smithi153 ceph-mon[70044]: pgmap v12998: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:35.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:41:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:41:35.141Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:41:36.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:41:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:41:35.865Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:41:36.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:41:36 smithi083 ceph-mon[79930]: pgmap v12999: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:36.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:41:36 smithi083 ceph-mon[77242]: pgmap v12999: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:36.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:41:36 smithi153 ceph-mon[70044]: pgmap v12999: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:38.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:41:38 smithi083 ceph-mon[79930]: pgmap v13000: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:38.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:41:38 smithi083 ceph-mon[77242]: pgmap v13000: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:39.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:41:38 smithi153 ceph-mon[70044]: pgmap v13000: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:41:40 smithi083 ceph-mon[79930]: pgmap v13001: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:41:40 smithi083 ceph-mon[77242]: pgmap v13001: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:41.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:41:40 smithi153 ceph-mon[70044]: pgmap v13001: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:41.777 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:41:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:41:41] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T12:41:41.778 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:41:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:41:41.778 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:41:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:41:41.779 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:41:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:41:41.779 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:41:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:41:41.779 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:41:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:41:41.779 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:41:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:41:41.780 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:41:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:41:41.780 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:41:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:41:42.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:41:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:41:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:41:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:41:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:41:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:41:42.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:41:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:41:42.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:41:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:41:41.775Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:41:42.666 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:41:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:41:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:41:42.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:41:42 smithi083 ceph-mon[79930]: pgmap v13002: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:42.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:41:42 smithi083 ceph-mon[77242]: pgmap v13002: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:43.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:41:42 smithi153 ceph-mon[70044]: pgmap v13002: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:41:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:41:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:41:43.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:41:43 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:41:43.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:41:43 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:41:43.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:41:43 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:41:43.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:41:43 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:41:43.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:41:43 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:41:43.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:41:43 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:41:43.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:41:43 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:41:43.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:41:43 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:41:44.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:41:43 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:41:44.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:41:43 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:41:44.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:41:43 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:41:44.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:41:43 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:41:44.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:41:44 smithi083 ceph-mon[79930]: pgmap v13003: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:44.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:41:44 smithi083 ceph-mon[77242]: pgmap v13003: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:45.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:41:44 smithi153 ceph-mon[70044]: pgmap v13003: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:45.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:41:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:41:45.141Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:41:46.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:41:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:41:45.866Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:41:46.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:41:46 smithi083 ceph-mon[79930]: pgmap v13004: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:46.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:41:46 smithi083 ceph-mon[77242]: pgmap v13004: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:46.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:41:46 smithi153 ceph-mon[70044]: pgmap v13004: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:48.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:41:48 smithi083 ceph-mon[79930]: pgmap v13005: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:48.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:41:48 smithi083 ceph-mon[77242]: pgmap v13005: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:49.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:41:48 smithi153 ceph-mon[70044]: pgmap v13005: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:50.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:41:50 smithi083 ceph-mon[77242]: pgmap v13006: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:50.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:41:50 smithi083 ceph-mon[79930]: pgmap v13006: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:51.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:41:50 smithi153 ceph-mon[70044]: pgmap v13006: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:51.777 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:41:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:41:51] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T12:41:52.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:41:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:41:51.775Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:41:52.663 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:41:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:41:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:41:52.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:41:52 smithi083 ceph-mon[79930]: pgmap v13007: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:52.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:41:52 smithi083 ceph-mon[77242]: pgmap v13007: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:53.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:41:52 smithi153 ceph-mon[70044]: pgmap v13007: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:41:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:41:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:41:54.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:41:54 smithi083 ceph-mon[79930]: pgmap v13008: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:54.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:41:54 smithi083 ceph-mon[77242]: pgmap v13008: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:55.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:41:54 smithi153 ceph-mon[70044]: pgmap v13008: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:55.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:41:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:41:55.141Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:41:56.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:41:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:41:55.867Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:41:56.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:41:56 smithi083 ceph-mon[77242]: pgmap v13009: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:56.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:41:56 smithi083 ceph-mon[79930]: pgmap v13009: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:56.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:41:56 smithi153 ceph-mon[70044]: pgmap v13009: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:57.945 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:41:58.235 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:41:58.235 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 7m ago 7h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:41:58.235 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 5m ago 7h 64.1M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:41:58.235 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 7m ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:41:58.236 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 5m ago 7h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:41:58.236 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 5m ago 7h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:41:58.236 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 7m ago 7h 872M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:41:58.236 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 7m ago 7h 1595M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:41:58.237 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 5m ago 7h 688M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:41:58.237 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 7m ago 7h 924M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:41:58.237 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 7m ago 7h 26.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:41:58.238 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 5m ago 7h 25.8M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:41:58.238 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 7m ago 7h 1152M 2366M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:41:58.238 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 7m ago 7h 1139M 2366M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:41:58.238 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 7m ago 7h 867M 2366M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:41:58.239 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 7m ago 7h 1115M 2366M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:41:58.239 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 5m ago 7h 957M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:41:58.239 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 5m ago 7h 1226M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:41:58.239 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 5m ago 7h 1095M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:41:58.239 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 5m ago 7h 1300M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:41:58.240 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 5m ago 7h 171M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:41:58.590 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:41:58.590 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:41:58.591 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:41:58.591 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:41:58.591 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:41:58.591 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:41:58.592 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:41:58.592 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:41:58.592 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:41:58.592 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:41:58.593 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:41:58.593 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:41:58.593 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:41:58.593 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:41:58.593 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:41:58.594 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:41:58.594 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:41:58.594 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:41:58.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:41:58 smithi083 ceph-mon[79930]: pgmap v13010: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:58.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:41:58 smithi083 ceph-mon[79930]: from='client.49260 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:41:58.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:41:58 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/139704580' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:41:58.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:41:58 smithi083 ceph-mon[77242]: pgmap v13010: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:58.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:41:58 smithi083 ceph-mon[77242]: from='client.49260 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:41:58.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:41:58 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/139704580' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:41:59.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:41:58 smithi153 ceph-mon[70044]: pgmap v13010: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:41:59.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:41:58 smithi153 ceph-mon[70044]: from='client.49260 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:41:59.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:41:58 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/139704580' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:41:59.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:41:59 smithi083 ceph-mon[79930]: from='client.49266 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:41:59.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:41:59 smithi083 ceph-mon[77242]: from='client.49266 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:42:00.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:41:59 smithi153 ceph-mon[70044]: from='client.49266 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:42:00.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:42:00 smithi083 ceph-mon[79930]: pgmap v13011: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:00.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:42:00 smithi083 ceph-mon[77242]: pgmap v13011: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:01.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:42:00 smithi153 ceph-mon[70044]: pgmap v13011: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:01.778 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:42:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:42:01] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T12:42:02.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:42:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:42:01.776Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:42:02.670 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:42:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:42:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:42:02.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:42:02 smithi083 ceph-mon[79930]: pgmap v13012: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:02.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:42:02 smithi083 ceph-mon[77242]: pgmap v13012: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:03.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:42:02 smithi153 ceph-mon[70044]: pgmap v13012: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:42:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:42:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:42:04.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:42:04 smithi083 ceph-mon[79930]: pgmap v13013: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:04.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:42:04 smithi083 ceph-mon[77242]: pgmap v13013: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:05.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:42:04 smithi153 ceph-mon[70044]: pgmap v13013: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:05.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:42:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:42:05.142Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:42:06.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:42:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:42:05.867Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:42:06.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:42:06 smithi083 ceph-mon[77242]: pgmap v13014: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:06.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:42:06 smithi083 ceph-mon[79930]: pgmap v13014: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:06.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:42:06 smithi153 ceph-mon[70044]: pgmap v13014: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:08.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:42:08 smithi083 ceph-mon[77242]: pgmap v13015: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:08.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:42:08 smithi083 ceph-mon[79930]: pgmap v13015: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:09.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:42:08 smithi153 ceph-mon[70044]: pgmap v13015: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:10.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:42:10 smithi083 ceph-mon[79930]: pgmap v13016: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:10.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:42:10 smithi083 ceph-mon[77242]: pgmap v13016: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:11.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:42:10 smithi153 ceph-mon[70044]: pgmap v13016: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:11.779 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:42:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:42:11] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T12:42:12.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:42:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:42:11.776Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:42:12.669 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:42:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:42:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:42:12.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:42:12 smithi083 ceph-mon[79930]: pgmap v13017: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:12.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:42:12 smithi083 ceph-mon[77242]: pgmap v13017: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:13.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:42:12 smithi153 ceph-mon[70044]: pgmap v13017: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:13.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:42:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:42:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:42:14.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:42:14 smithi083 ceph-mon[77242]: pgmap v13018: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:14.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:42:14 smithi083 ceph-mon[79930]: pgmap v13018: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:15.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:42:14 smithi153 ceph-mon[70044]: pgmap v13018: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:15.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:42:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:42:15.142Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:42:16.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:42:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:42:15.867Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:42:16.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:42:16 smithi083 ceph-mon[79930]: pgmap v13019: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:16.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:42:16 smithi083 ceph-mon[77242]: pgmap v13019: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:16.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:42:16 smithi153 ceph-mon[70044]: pgmap v13019: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:18.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:42:18 smithi083 ceph-mon[79930]: pgmap v13020: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:18.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:42:18 smithi083 ceph-mon[77242]: pgmap v13020: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:19.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:42:18 smithi153 ceph-mon[70044]: pgmap v13020: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:20.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:42:20 smithi083 ceph-mon[79930]: pgmap v13021: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:20.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:42:20 smithi083 ceph-mon[77242]: pgmap v13021: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:21.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:42:20 smithi153 ceph-mon[70044]: pgmap v13021: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:21.782 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:42:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:42:21] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T12:42:22.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:42:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:42:21.778Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:42:22.673 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:42:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:42:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:42:22.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:42:22 smithi083 ceph-mon[79930]: pgmap v13022: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:22.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:42:22 smithi083 ceph-mon[77242]: pgmap v13022: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:23.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:42:22 smithi153 ceph-mon[70044]: pgmap v13022: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:23.310 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:42:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:42:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:42:24.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:42:24 smithi083 ceph-mon[79930]: pgmap v13023: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:24.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:42:24 smithi083 ceph-mon[77242]: pgmap v13023: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:25.067 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:42:24 smithi153 ceph-mon[70044]: pgmap v13023: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:25.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:42:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:42:25.143Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:42:26.214 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:42:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:42:25.868Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:42:26.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:42:26 smithi083 ceph-mon[79930]: pgmap v13024: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:26.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:42:26 smithi083 ceph-mon[77242]: pgmap v13024: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:26.816 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:42:26 smithi153 ceph-mon[70044]: pgmap v13024: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:28.926 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:42:28.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:42:28 smithi083 ceph-mon[79930]: pgmap v13025: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:28.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:42:28 smithi083 ceph-mon[77242]: pgmap v13025: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:29.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:42:28 smithi153 ceph-mon[70044]: pgmap v13025: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:29.204 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:42:29.205 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 7m ago 7h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:42:29.205 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 5m ago 7h 64.1M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:42:29.206 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 7m ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:42:29.208 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 5m ago 7h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:42:29.209 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 5m ago 7h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:42:29.209 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 7m ago 7h 872M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:42:29.210 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 7m ago 7h 1595M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:42:29.210 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 5m ago 7h 688M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:42:29.210 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 7m ago 7h 924M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:42:29.211 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 7m ago 7h 26.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:42:29.211 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 5m ago 7h 25.8M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:42:29.212 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 7m ago 7h 1152M 2366M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:42:29.213 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 7m ago 7h 1139M 2366M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:42:29.215 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 7m ago 7h 867M 2366M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:42:29.215 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 7m ago 7h 1115M 2366M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:42:29.228 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 5m ago 7h 957M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:42:29.229 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 5m ago 7h 1226M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:42:29.230 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 5m ago 7h 1095M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:42:29.238 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 5m ago 7h 1300M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:42:29.239 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 5m ago 7h 171M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:42:29.562 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:42:29.563 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:42:29.563 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:42:29.564 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:42:29.564 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:42:29.565 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:42:29.565 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:42:29.565 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:42:29.605 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:42:29.605 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:42:29.615 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:42:29.615 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:42:29.616 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:42:29.616 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:42:29.642 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:42:29.642 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:42:29.642 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:42:29.642 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:42:29.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:42:29 smithi083 ceph-mon[79930]: from='client.49278 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:42:29.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:42:29 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4072976202' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:42:29.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:42:29 smithi083 ceph-mon[77242]: from='client.49278 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:42:29.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:42:29 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4072976202' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:42:30.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:42:29 smithi153 ceph-mon[70044]: from='client.49278 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:42:30.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:42:29 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4072976202' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:42:30.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:42:30 smithi083 ceph-mon[79930]: from='client.59215 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:42:30.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:42:30 smithi083 ceph-mon[79930]: pgmap v13026: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:30.968 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:42:30 smithi083 ceph-mon[77242]: from='client.59215 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:42:30.969 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:42:30 smithi083 ceph-mon[77242]: pgmap v13026: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:31.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:42:30 smithi153 ceph-mon[70044]: from='client.59215 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:42:31.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:42:30 smithi153 ceph-mon[70044]: pgmap v13026: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:31.784 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:42:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:42:31] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T12:42:32.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:42:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:42:31.778Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:42:32.690 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:42:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:42:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:42:32.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:42:32 smithi083 ceph-mon[79930]: pgmap v13027: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:32.968 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:42:32 smithi083 ceph-mon[77242]: pgmap v13027: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:33.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:42:32 smithi153 ceph-mon[70044]: pgmap v13027: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:42:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:42:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:42:34.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:42:34 smithi083 ceph-mon[77242]: pgmap v13028: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:34.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:42:34 smithi083 ceph-mon[79930]: pgmap v13028: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:35.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:42:34 smithi153 ceph-mon[70044]: pgmap v13028: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:35.470 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:42:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:42:35.144Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:42:36.227 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:42:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:42:35.869Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:42:36.719 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:42:36 smithi083 ceph-mon[79930]: pgmap v13029: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:36.720 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:42:36 smithi083 ceph-mon[77242]: pgmap v13029: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:36.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:42:36 smithi153 ceph-mon[70044]: pgmap v13029: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:38.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:42:38 smithi083 ceph-mon[77242]: pgmap v13030: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:38.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:42:38 smithi083 ceph-mon[79930]: pgmap v13030: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:39.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:42:38 smithi153 ceph-mon[70044]: pgmap v13030: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:42:40 smithi083 ceph-mon[79930]: pgmap v13031: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:40.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:42:40 smithi083 ceph-mon[77242]: pgmap v13031: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:41.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:42:40 smithi153 ceph-mon[70044]: pgmap v13031: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:41.782 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:42:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:42:41] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T12:42:41.784 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:42:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:42:41.784 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:42:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:42:41.785 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:42:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:42:41.785 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:42:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:42:41.787 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:42:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:42:41.788 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:42:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:42:41.788 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:42:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:42:41.788 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:42:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:42:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:42:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:42:42.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:42:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:42:42.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:42:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:42:42.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:42:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:42:42.218 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:42:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:42:41.779Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:42:42.682 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:42:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:42:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:42:42.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:42:42 smithi083 ceph-mon[77242]: pgmap v13032: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:42.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:42:42 smithi083 ceph-mon[79930]: pgmap v13032: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:43.030 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:42:42 smithi153 ceph-mon[70044]: pgmap v13032: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:43.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:42:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:42:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:42:43.970 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:42:43 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:42:43.971 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:42:43 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:42:43.973 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:42:43 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:42:43.973 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:42:43 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:42:43.974 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:42:43 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:42:43.974 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:42:43 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:42:44.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:42:43 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:42:44.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:42:43 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:42:44.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:42:43 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:42:44.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:42:44 smithi083 ceph-mon[79930]: pgmap v13033: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:44.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:42:44 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:42:44.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:42:44 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:42:44.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:42:44 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:42:44.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:42:44 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:42:44.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:42:44 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:42:44.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:42:44 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:42:44.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:42:44 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:42:44.968 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:42:44 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:42:44.968 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:42:44 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:42:44.968 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:42:44 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:42:44.969 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:42:44 smithi083 ceph-mon[77242]: pgmap v13033: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:44.970 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:42:44 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:42:44.970 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:42:44 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:42:44.971 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:42:44 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:42:44.971 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:42:44 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:42:44.972 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:42:44 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:42:44.972 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:42:44 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:42:44.972 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:42:44 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:42:44.973 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:42:44 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:42:44.973 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:42:44 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:42:44.974 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:42:44 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:42:45.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:42:44 smithi153 ceph-mon[70044]: pgmap v13033: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:45.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:42:44 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:42:45.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:42:44 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:42:45.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:42:44 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:42:45.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:42:44 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:42:45.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:42:44 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:42:45.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:42:44 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:42:45.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:42:44 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:42:45.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:42:44 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:42:45.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:42:44 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:42:45.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:42:44 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:42:45.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:42:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:42:45.145Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:42:45.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:42:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:42:45.870Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:42:45.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:42:45 smithi083 ceph-mon[77242]: Adjusting osd_memory_target on smithi083 to 2357M 2022-09-29T12:42:45.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:42:45 smithi083 ceph-mon[79930]: Adjusting osd_memory_target on smithi083 to 2357M 2022-09-29T12:42:46.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:42:45 smithi153 ceph-mon[70044]: Adjusting osd_memory_target on smithi083 to 2357M 2022-09-29T12:42:46.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:42:46 smithi083 ceph-mon[77242]: pgmap v13034: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:46.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:42:46 smithi083 ceph-mon[79930]: pgmap v13034: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:47.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:42:46 smithi153 ceph-mon[70044]: pgmap v13034: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:48.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:42:48 smithi083 ceph-mon[77242]: pgmap v13035: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:48.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:42:48 smithi083 ceph-mon[79930]: pgmap v13035: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:49.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:42:48 smithi153 ceph-mon[70044]: pgmap v13035: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:50.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:42:50 smithi083 ceph-mon[77242]: pgmap v13036: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:50.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:42:50 smithi083 ceph-mon[79930]: pgmap v13036: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:51.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:42:50 smithi153 ceph-mon[70044]: pgmap v13036: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:51.783 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:42:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:42:51] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T12:42:52.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:42:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:42:51.779Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:42:52.700 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:42:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:42:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:42:52.971 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:42:52 smithi083 ceph-mon[79930]: pgmap v13037: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:52.972 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:42:52 smithi083 ceph-mon[77242]: pgmap v13037: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:53.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:42:52 smithi153 ceph-mon[70044]: pgmap v13037: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:42:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:42:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:42:54.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:42:54 smithi083 ceph-mon[79930]: pgmap v13038: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:54.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:42:54 smithi083 ceph-mon[77242]: pgmap v13038: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:55.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:42:54 smithi153 ceph-mon[70044]: pgmap v13038: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:55.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:42:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:42:55.145Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:42:56.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:42:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:42:55.870Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:42:56.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:42:56 smithi083 ceph-mon[77242]: pgmap v13039: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:56.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:42:56 smithi083 ceph-mon[79930]: pgmap v13039: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:56.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:42:56 smithi153 ceph-mon[70044]: pgmap v13039: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:58.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:42:58 smithi083 ceph-mon[79930]: pgmap v13040: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:58.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:42:58 smithi083 ceph-mon[77242]: pgmap v13040: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:59.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:42:58 smithi153 ceph-mon[70044]: pgmap v13040: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:42:59.888 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:43:00.175 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:43:00.175 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 8m ago 7h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:43:00.175 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 6m ago 7h 64.1M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:43:00.176 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 8m ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:43:00.176 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 6m ago 7h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:43:00.177 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 6m ago 7h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:43:00.177 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 8m ago 7h 872M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:43:00.178 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 8m ago 7h 1595M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:43:00.178 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 6m ago 7h 688M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:43:00.178 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 8m ago 7h 924M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:43:00.179 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 8m ago 7h 26.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:43:00.179 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 6m ago 7h 25.8M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:43:00.180 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 8m ago 7h 1152M 2357M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:43:00.180 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 8m ago 7h 1139M 2357M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:43:00.181 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 8m ago 7h 867M 2357M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:43:00.181 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 8m ago 7h 1115M 2357M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:43:00.181 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 6m ago 7h 957M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:43:00.182 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 6m ago 7h 1226M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:43:00.182 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 6m ago 7h 1095M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:43:00.183 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 6m ago 7h 1300M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:43:00.184 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 6m ago 7h 171M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:43:00.533 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:43:00.534 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:43:00.534 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:43:00.534 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:43:00.534 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:43:00.535 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:43:00.535 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:43:00.535 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:43:00.535 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:43:00.536 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:43:00.536 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:43:00.536 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:43:00.537 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:43:00.537 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:43:00.537 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:43:00.538 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:43:00.538 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:43:00.539 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:43:00.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:43:00 smithi083 ceph-mon[77242]: pgmap v13041: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:43:00.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:43:00 smithi083 ceph-mon[77242]: from='client.49296 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:43:00.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:43:00 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2015397145' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:43:00.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:43:00 smithi083 ceph-mon[79930]: pgmap v13041: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:43:00.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:43:00 smithi083 ceph-mon[79930]: from='client.49296 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:43:00.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:43:00 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2015397145' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:43:01.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:43:00 smithi153 ceph-mon[70044]: pgmap v13041: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:43:01.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:43:00 smithi153 ceph-mon[70044]: from='client.49296 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:43:01.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:43:00 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2015397145' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:43:01.783 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:43:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:43:01] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T12:43:01.784 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:43:01 smithi083 ceph-mon[79930]: from='client.59233 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:43:01.786 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:43:01 smithi083 ceph-mon[77242]: from='client.59233 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:43:02.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:43:01 smithi153 ceph-mon[70044]: from='client.59233 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:43:02.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:43:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:43:01.780Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:43:02.686 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:43:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:43:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:43:02.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:43:02 smithi083 ceph-mon[79930]: pgmap v13042: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:43:02.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:43:02 smithi083 ceph-mon[77242]: pgmap v13042: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:43:03.028 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:43:02 smithi153 ceph-mon[70044]: pgmap v13042: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:43:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:43:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:43:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:43:04.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:43:04 smithi083 ceph-mon[79930]: pgmap v13043: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:43:04.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:43:04 smithi083 ceph-mon[77242]: pgmap v13043: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:43:05.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:43:04 smithi153 ceph-mon[70044]: pgmap v13043: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:43:05.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:43:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:43:05.146Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:43:06.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:43:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:43:05.871Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:43:06.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:43:06 smithi083 ceph-mon[77242]: pgmap v13044: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:43:06.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:43:06 smithi083 ceph-mon[79930]: pgmap v13044: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:43:06.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:43:06 smithi153 ceph-mon[70044]: pgmap v13044: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:43:08.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:43:08 smithi083 ceph-mon[79930]: pgmap v13045: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:43:08.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:43:08 smithi083 ceph-mon[77242]: pgmap v13045: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:43:09.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:43:08 smithi153 ceph-mon[70044]: pgmap v13045: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:43:10.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:43:10 smithi083 ceph-mon[79930]: pgmap v13046: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:43:10.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:43:10 smithi083 ceph-mon[77242]: pgmap v13046: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:43:11.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:43:10 smithi153 ceph-mon[70044]: pgmap v13046: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:43:11.784 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:43:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:43:11] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T12:43:12.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:43:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:43:11.781Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:43:12.707 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:43:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:43:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:43:12.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:43:12 smithi083 ceph-mon[79930]: pgmap v13047: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:43:12.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:43:12 smithi083 ceph-mon[77242]: pgmap v13047: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:43:13.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:43:12 smithi153 ceph-mon[70044]: pgmap v13047: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:43:13.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:43:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:43:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:43:14.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:43:14 smithi083 ceph-mon[79930]: pgmap v13048: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:43:14.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:43:14 smithi083 ceph-mon[77242]: pgmap v13048: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:43:15.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:43:14 smithi153 ceph-mon[70044]: pgmap v13048: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:43:15.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:43:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:43:15.147Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:43:16.215 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:43:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:43:15.871Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:43:16.740 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:43:16 smithi083 ceph-mon[79930]: pgmap v13049: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:43:16.741 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:43:16 smithi083 ceph-mon[77242]: pgmap v13049: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:43:16.813 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:43:16 smithi153 ceph-mon[70044]: pgmap v13049: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:43:18.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:43:18 smithi083 ceph-mon[79930]: pgmap v13050: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:43:18.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:43:18 smithi083 ceph-mon[77242]: pgmap v13050: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:43:19.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:43:18 smithi153 ceph-mon[70044]: pgmap v13050: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:43:20.975 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:43:20 smithi083 ceph-mon[79930]: pgmap v13051: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:43:20.975 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:43:20 smithi083 ceph-mon[77242]: pgmap v13051: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:43:21.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:43:20 smithi153 ceph-mon[70044]: pgmap v13051: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:43:21.787 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:43:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:43:21] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T12:43:22.224 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:43:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:43:21.781Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:43:22.706 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:43:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:43:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:43:23.023 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:43:22 smithi083 ceph-mon[79930]: pgmap v13052: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:43:23.024 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:43:22 smithi083 ceph-mon[77242]: pgmap v13052: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:43:23.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:43:22 smithi153 ceph-mon[70044]: pgmap v13052: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:43:23.311 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:43:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:43:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:43:24.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:43:24 smithi083 ceph-mon[79930]: pgmap v13053: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:43:24.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:43:24 smithi083 ceph-mon[77242]: pgmap v13053: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:43:25.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:43:24 smithi153 ceph-mon[70044]: pgmap v13053: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:43:25.467 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:43:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:43:25.147Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:43:26.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:43:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:43:25.871Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:43:26.722 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:43:26 smithi083 ceph-mon[79930]: pgmap v13054: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:43:26.722 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:43:26 smithi083 ceph-mon[77242]: pgmap v13054: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:43:26.813 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:43:26 smithi153 ceph-mon[70044]: pgmap v13054: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:43:28.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:43:28 smithi083 ceph-mon[79930]: pgmap v13055: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:43:28.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:43:28 smithi083 ceph-mon[77242]: pgmap v13055: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:43:29.063 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:43:28 smithi153 ceph-mon[70044]: pgmap v13055: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:43:30.864 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:43:30.970 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:43:30 smithi083 ceph-mon[79930]: pgmap v13056: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:43:30.971 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:43:30 smithi083 ceph-mon[77242]: pgmap v13056: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:43:31.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:43:30 smithi153 ceph-mon[70044]: pgmap v13056: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:43:31.148 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:43:31.149 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 8m ago 7h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:43:31.149 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 6m ago 7h 64.1M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:43:31.155 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 8m ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:43:31.155 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 6m ago 7h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:43:31.155 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 6m ago 7h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:43:31.156 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 8m ago 7h 872M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:43:31.156 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 8m ago 7h 1595M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:43:31.156 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 6m ago 7h 688M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:43:31.157 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 8m ago 7h 924M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:43:31.163 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 8m ago 7h 26.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:43:31.164 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 6m ago 7h 25.8M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:43:31.164 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 8m ago 7h 1152M 2357M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:43:31.165 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 8m ago 7h 1139M 2357M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:43:31.165 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 8m ago 7h 867M 2357M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:43:31.165 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 8m ago 7h 1115M 2357M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:43:31.166 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 6m ago 7h 957M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:43:31.166 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 6m ago 7h 1226M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:43:31.166 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 6m ago 7h 1095M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:43:31.167 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 6m ago 7h 1300M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:43:31.167 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 6m ago 7h 171M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:43:31.501 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:43:31.502 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:43:31.502 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:43:31.502 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:43:31.502 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:43:31.503 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:43:31.503 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:43:31.503 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:43:31.504 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:43:31.504 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:43:31.504 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:43:31.504 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:43:31.504 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:43:31.505 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:43:31.505 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:43:31.505 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:43:31.505 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:43:31.506 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:43:31.794 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:43:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:43:31] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T12:43:31.795 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:43:31 smithi083 ceph-mon[77242]: from='client.59245 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:43:31.795 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:43:31 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/641259035' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:43:31.796 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:43:31 smithi083 ceph-mon[79930]: from='client.59245 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:43:31.796 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:43:31 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/641259035' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:43:32.066 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:43:31 smithi153 ceph-mon[70044]: from='client.59245 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:43:32.067 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:43:31 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/641259035' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:43:32.214 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:43:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:43:31.781Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:43:32.694 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:43:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:43:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:43:32.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:43:32 smithi083 ceph-mon[77242]: from='client.49320 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:43:32.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:43:32 smithi083 ceph-mon[77242]: pgmap v13057: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:43:32.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:43:32 smithi083 ceph-mon[79930]: from='client.49320 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:43:32.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:43:32 smithi083 ceph-mon[79930]: pgmap v13057: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:43:33.042 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:43:32 smithi153 ceph-mon[70044]: from='client.49320 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:43:33.042 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:43:32 smithi153 ceph-mon[70044]: pgmap v13057: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:43:33.310 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:43:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:43:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:43:34.973 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:43:34 smithi083 ceph-mon[79930]: pgmap v13058: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:43:34.974 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:43:34 smithi083 ceph-mon[77242]: pgmap v13058: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:43:35.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:43:34 smithi153 ceph-mon[70044]: pgmap v13058: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:43:35.471 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:43:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:43:35.148Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:43:36.219 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:43:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:43:35.873Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:43:36.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:43:36 smithi083 ceph-mon[79930]: pgmap v13059: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:43:36.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:43:36 smithi083 ceph-mon[77242]: pgmap v13059: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:43:36.827 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:43:36 smithi153 ceph-mon[70044]: pgmap v13059: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:43:38.970 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:43:38 smithi083 ceph-mon[79930]: pgmap v13060: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:43:38.973 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:43:38 smithi083 ceph-mon[77242]: pgmap v13060: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:43:39.066 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:43:38 smithi153 ceph-mon[70044]: pgmap v13060: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:43:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:43:40 smithi083 ceph-mon[79930]: pgmap v13061: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:43:40.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:43:40 smithi083 ceph-mon[77242]: pgmap v13061: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:43:41.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:43:40 smithi153 ceph-mon[70044]: pgmap v13061: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:43:41.788 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:43:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:43:41] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T12:43:41.789 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:43:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:43:41.789 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:43:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:43:41.790 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:43:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:43:41.790 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:43:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:43:41.790 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:43:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:43:41.791 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:43:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:43:41.791 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:43:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:43:41.791 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:43:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:43:42.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:43:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:43:42.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:43:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:43:42.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:43:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:43:42.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:43:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:43:42.229 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:43:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:43:41.782Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:43:42.700 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:43:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:43:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:43:42.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:43:42 smithi083 ceph-mon[79930]: pgmap v13062: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:43:42.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:43:42 smithi083 ceph-mon[77242]: pgmap v13062: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:43:43.031 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:43:42 smithi153 ceph-mon[70044]: pgmap v13062: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:43:43.310 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:43:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:43:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:43:44.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:43:44 smithi083 ceph-mon[77242]: pgmap v13063: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:43:44.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:43:44 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:43:44.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:43:44 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:43:44.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:43:44 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:43:44.973 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:43:44 smithi083 ceph-mon[79930]: pgmap v13063: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:43:44.977 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:43:44 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:43:44.979 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:43:44 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:43:44.979 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:43:44 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:43:45.064 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:43:44 smithi153 ceph-mon[70044]: pgmap v13063: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:43:45.065 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:43:44 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:43:45.070 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:43:44 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:43:45.071 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:43:44 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:43:45.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:43:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:43:45.148Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:43:46.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:43:45 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:43:46.219 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:43:45 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:43:46.220 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:43:45 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:43:46.221 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:43:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:43:45.873Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:43:47.064 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:43:46 smithi153 ceph-mon[70044]: pgmap v13064: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:43:47.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:43:46 smithi083 ceph-mon[79930]: pgmap v13064: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:43:47.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:43:46 smithi083 ceph-mon[77242]: pgmap v13064: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:43:49.077 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:43:48 smithi153 ceph-mon[70044]: pgmap v13065: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:43:49.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:43:48 smithi083 ceph-mon[79930]: pgmap v13065: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:43:49.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:43:48 smithi083 ceph-mon[77242]: pgmap v13065: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:43:51.072 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:43:50 smithi153 ceph-mon[70044]: pgmap v13066: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:43:51.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:43:50 smithi083 ceph-mon[77242]: pgmap v13066: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:43:51.220 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:43:50 smithi083 ceph-mon[79930]: pgmap v13066: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:43:51.795 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:43:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:43:51] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T12:43:52.227 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:43:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:43:51.782Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:43:52.791 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:43:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:43:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:43:53.064 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:43:52 smithi153 ceph-mon[70044]: pgmap v13067: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:43:53.065 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:43:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:43:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:43:53.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:43:52 smithi083 ceph-mon[79930]: pgmap v13067: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:43:53.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:43:52 smithi083 ceph-mon[77242]: pgmap v13067: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:43:55.066 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:43:54 smithi153 ceph-mon[70044]: pgmap v13068: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:43:55.154 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:43:54 smithi083 ceph-mon[79930]: pgmap v13068: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:43:55.155 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:43:54 smithi083 ceph-mon[77242]: pgmap v13068: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:43:55.466 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:43:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:43:55.149Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:43:56.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:43:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:43:55.874Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:43:56.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:43:56 smithi083 ceph-mon[79930]: pgmap v13069: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:43:56.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:43:56 smithi083 ceph-mon[77242]: pgmap v13069: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:43:56.817 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:43:56 smithi153 ceph-mon[70044]: pgmap v13069: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:43:58.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:43:58 smithi083 ceph-mon[79930]: pgmap v13070: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:43:58.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:43:58 smithi083 ceph-mon[77242]: pgmap v13070: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:43:59.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:43:58 smithi153 ceph-mon[70044]: pgmap v13070: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:00.968 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:44:00 smithi083 ceph-mon[77242]: pgmap v13071: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:00.969 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:44:00 smithi083 ceph-mon[79930]: pgmap v13071: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:01.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:44:00 smithi153 ceph-mon[70044]: pgmap v13071: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:01.786 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:44:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:44:01] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T12:44:01.831 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:44:02.122 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:44:02.123 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 9m ago 7h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:44:02.123 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 7m ago 7h 64.1M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:44:02.123 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 9m ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:44:02.123 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 7m ago 7h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:44:02.123 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 7m ago 7h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:44:02.124 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 9m ago 7h 872M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:44:02.124 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 9m ago 7h 1595M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:44:02.124 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 7m ago 7h 688M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:44:02.124 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 9m ago 7h 924M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:44:02.125 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 9m ago 7h 26.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:44:02.125 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 7m ago 7h 25.8M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:44:02.125 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 9m ago 7h 1152M 2357M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:44:02.125 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 9m ago 7h 1139M 2357M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:44:02.126 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 9m ago 7h 867M 2357M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:44:02.147 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 9m ago 7h 1115M 2357M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:44:02.148 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 7m ago 7h 957M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:44:02.148 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 7m ago 7h 1226M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:44:02.149 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 7m ago 7h 1095M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:44:02.149 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 7m ago 7h 1300M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:44:02.149 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 7m ago 7h 171M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:44:02.215 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:44:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:44:01.782Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:44:02.477 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:44:02.478 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:44:02.478 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:44:02.479 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:44:02.479 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:44:02.479 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:44:02.480 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:44:02.480 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:44:02.481 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:44:02.481 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:44:02.483 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:44:02.486 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:44:02.487 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:44:02.489 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:44:02.489 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:44:02.490 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:44:02.490 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:44:02.491 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:44:02.704 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:44:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:44:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:44:02.974 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:44:02 smithi083 ceph-mon[77242]: pgmap v13072: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:02.975 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:44:02 smithi083 ceph-mon[77242]: from='client.49332 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:44:02.976 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:44:02 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3651365951' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:44:02.976 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:44:02 smithi083 ceph-mon[79930]: pgmap v13072: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:02.977 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:44:02 smithi083 ceph-mon[79930]: from='client.49332 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:44:02.977 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:44:02 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3651365951' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:44:03.027 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:44:02 smithi153 ceph-mon[70044]: pgmap v13072: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:03.027 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:44:02 smithi153 ceph-mon[70044]: from='client.49332 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:44:03.028 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:44:02 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3651365951' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:44:03.312 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:44:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:44:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:44:03.980 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:44:03 smithi083 ceph-mon[79930]: from='client.49338 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:44:03.980 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:44:03 smithi083 ceph-mon[77242]: from='client.49338 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:44:04.104 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:44:03 smithi153 ceph-mon[70044]: from='client.49338 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:44:04.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:44:04 smithi083 ceph-mon[79930]: pgmap v13073: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:04.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:44:04 smithi083 ceph-mon[77242]: pgmap v13073: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:05.071 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:44:04 smithi153 ceph-mon[70044]: pgmap v13073: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:05.466 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:44:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:44:05.151Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:44:06.216 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:44:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:44:05.875Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:44:06.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:44:06 smithi083 ceph-mon[79930]: pgmap v13074: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:06.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:44:06 smithi083 ceph-mon[77242]: pgmap v13074: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:06.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:44:06 smithi153 ceph-mon[70044]: pgmap v13074: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:08.969 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:44:08 smithi083 ceph-mon[79930]: pgmap v13075: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:08.970 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:44:08 smithi083 ceph-mon[77242]: pgmap v13075: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:09.106 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:44:08 smithi153 ceph-mon[70044]: pgmap v13075: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:10.970 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:44:10 smithi083 ceph-mon[77242]: pgmap v13076: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:10.971 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:44:10 smithi083 ceph-mon[79930]: pgmap v13076: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:11.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:44:10 smithi153 ceph-mon[70044]: pgmap v13076: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:11.788 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:44:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:44:11] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T12:44:12.215 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:44:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:44:11.782Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:44:12.711 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:44:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:44:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:44:12.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:44:12 smithi083 ceph-mon[79930]: pgmap v13077: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:12.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:44:12 smithi083 ceph-mon[77242]: pgmap v13077: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:13.030 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:44:12 smithi153 ceph-mon[70044]: pgmap v13077: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:13.310 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:44:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:44:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:44:14.969 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:44:14 smithi083 ceph-mon[77242]: pgmap v13078: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:14.971 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:44:14 smithi083 ceph-mon[79930]: pgmap v13078: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:15.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:44:14 smithi153 ceph-mon[70044]: pgmap v13078: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:15.466 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:44:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:44:15.151Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:44:16.216 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:44:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:44:15.876Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:44:16.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:44:16 smithi083 ceph-mon[77242]: pgmap v13079: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:16.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:44:16 smithi083 ceph-mon[79930]: pgmap v13079: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:16.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:44:16 smithi153 ceph-mon[70044]: pgmap v13079: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:18.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:44:18 smithi083 ceph-mon[79930]: pgmap v13080: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:18.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:44:18 smithi083 ceph-mon[77242]: pgmap v13080: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:19.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:44:18 smithi153 ceph-mon[70044]: pgmap v13080: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:20.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:44:20 smithi083 ceph-mon[77242]: pgmap v13081: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:20.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:44:20 smithi083 ceph-mon[79930]: pgmap v13081: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:21.068 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:44:20 smithi153 ceph-mon[70044]: pgmap v13081: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:21.792 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:44:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:44:21] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T12:44:22.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:44:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:44:21.784Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:44:22.707 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:44:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:44:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:44:22.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:44:22 smithi083 ceph-mon[77242]: pgmap v13082: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:22.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:44:22 smithi083 ceph-mon[79930]: pgmap v13082: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:23.029 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:44:22 smithi153 ceph-mon[70044]: pgmap v13082: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:23.314 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:44:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:44:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:44:24.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:44:24 smithi083 ceph-mon[77242]: pgmap v13083: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:24.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:44:24 smithi083 ceph-mon[79930]: pgmap v13083: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:25.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:44:24 smithi153 ceph-mon[70044]: pgmap v13083: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:25.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:44:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:44:25.152Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:44:26.214 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:44:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:44:25.876Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:44:26.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:44:26 smithi083 ceph-mon[77242]: pgmap v13084: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:26.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:44:26 smithi083 ceph-mon[79930]: pgmap v13084: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:26.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:44:26 smithi153 ceph-mon[70044]: pgmap v13084: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:28.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:44:28 smithi083 ceph-mon[77242]: pgmap v13085: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:28.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:44:28 smithi083 ceph-mon[79930]: pgmap v13085: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:29.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:44:28 smithi153 ceph-mon[70044]: pgmap v13085: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:30.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:44:30 smithi083 ceph-mon[79930]: pgmap v13086: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:30.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:44:30 smithi083 ceph-mon[77242]: pgmap v13086: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:31.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:44:30 smithi153 ceph-mon[70044]: pgmap v13086: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:31.788 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:44:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:44:31] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T12:44:32.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:44:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:44:31.784Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:44:32.710 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:44:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:44:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:44:32.803 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:44:32.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:44:32 smithi083 ceph-mon[79930]: pgmap v13087: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:32.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:44:32 smithi083 ceph-mon[77242]: pgmap v13087: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:33.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:44:32 smithi153 ceph-mon[70044]: pgmap v13087: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:33.092 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:44:33.093 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 9m ago 7h 26.8M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:44:33.093 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 7m ago 7h 64.1M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:44:33.094 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 9m ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:44:33.094 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 7m ago 7h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:44:33.095 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 7m ago 7h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:44:33.095 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 9m ago 7h 872M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:44:33.096 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 9m ago 7h 1595M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:44:33.096 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 7m ago 7h 688M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:44:33.096 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 9m ago 7h 924M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:44:33.097 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 9m ago 7h 26.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:44:33.097 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 7m ago 7h 25.8M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:44:33.104 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 9m ago 7h 1152M 2357M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:44:33.106 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 9m ago 7h 1139M 2357M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:44:33.106 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 9m ago 7h 867M 2357M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:44:33.106 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 9m ago 7h 1115M 2357M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:44:33.106 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 7m ago 7h 957M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:44:33.107 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 7m ago 7h 1226M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:44:33.107 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 7m ago 7h 1095M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:44:33.107 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 7m ago 7h 1300M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:44:33.107 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 7m ago 7h 171M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:44:33.310 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:44:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:44:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:44:33.465 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:44:33.466 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:44:33.466 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:44:33.466 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:44:33.466 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:44:33.467 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:44:33.467 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:44:33.467 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:44:33.467 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:44:33.468 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:44:33.468 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:44:33.468 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:44:33.469 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:44:33.469 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:44:33.469 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:44:33.469 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:44:33.479 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:44:33.479 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:44:34.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:44:33 smithi153 ceph-mon[70044]: from='client.59281 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:44:34.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:44:33 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3760511940' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:44:34.218 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:44:33 smithi083 ceph-mon[79930]: from='client.59281 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:44:34.219 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:44:33 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3760511940' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:44:34.219 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:44:33 smithi083 ceph-mon[77242]: from='client.59281 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:44:34.219 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:44:33 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3760511940' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:44:35.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:44:34 smithi153 ceph-mon[70044]: from='client.59287 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:44:35.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:44:34 smithi153 ceph-mon[70044]: pgmap v13088: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:35.158 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:44:34 smithi083 ceph-mon[79930]: from='client.59287 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:44:35.158 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:44:34 smithi083 ceph-mon[79930]: pgmap v13088: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:35.159 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:44:34 smithi083 ceph-mon[77242]: from='client.59287 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:44:35.159 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:44:34 smithi083 ceph-mon[77242]: pgmap v13088: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:35.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:44:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:44:35.153Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:44:36.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:44:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:44:35.877Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:44:36.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:44:36 smithi083 ceph-mon[77242]: pgmap v13089: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:36.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:44:36 smithi083 ceph-mon[79930]: pgmap v13089: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:36.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:44:36 smithi153 ceph-mon[70044]: pgmap v13089: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:38.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:44:38 smithi083 ceph-mon[77242]: pgmap v13090: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:38.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:44:38 smithi083 ceph-mon[79930]: pgmap v13090: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:39.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:44:38 smithi153 ceph-mon[70044]: pgmap v13090: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:44:40 smithi083 ceph-mon[79930]: pgmap v13091: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:44:40 smithi083 ceph-mon[77242]: pgmap v13091: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:44:40 smithi153 ceph-mon[70044]: pgmap v13091: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:41.794 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:44:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:44:41] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T12:44:41.795 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:44:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:44:41.796 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:44:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:44:41.796 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:44:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:44:41.797 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:44:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:44:41.797 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:44:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:44:41.797 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:44:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:44:41.798 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:44:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:44:41.798 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:44:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:44:42.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:44:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:44:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:44:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:44:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:44:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:44:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:44:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:44:42.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:44:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:44:41.784Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:44:42.721 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:44:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:44:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:44:43.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:44:42 smithi153 ceph-mon[70044]: pgmap v13092: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:43.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:44:42 smithi083 ceph-mon[79930]: pgmap v13092: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:43.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:44:42 smithi083 ceph-mon[77242]: pgmap v13092: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:44:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:44:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:44:45.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:44:44 smithi153 ceph-mon[70044]: pgmap v13093: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:45.157 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:44:44 smithi083 ceph-mon[79930]: pgmap v13093: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:45.158 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:44:44 smithi083 ceph-mon[77242]: pgmap v13093: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:45.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:44:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:44:45.153Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:44:46.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:44:45 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:44:46.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:44:45 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:44:46.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:44:45 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:44:46.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:44:45 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:44:46.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:44:45 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:44:46.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:44:45 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:44:46.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:44:45 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:44:46.215 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:44:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:44:45.877Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:44:46.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:44:45 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:44:46.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:44:45 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:44:46.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:44:45 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:44:46.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:44:45 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:44:46.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:44:45 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:44:46.218 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:44:45 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:44:46.219 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:44:45 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:44:46.219 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:44:45 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:44:46.220 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:44:45 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:44:46.220 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:44:45 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:44:46.221 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:44:45 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:44:46.221 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:44:45 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:44:46.224 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:44:45 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:44:46.224 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:44:45 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:44:47.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:44:46 smithi153 ceph-mon[70044]: pgmap v13094: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:47.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:44:46 smithi083 ceph-mon[79930]: pgmap v13094: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:47.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:44:46 smithi083 ceph-mon[77242]: pgmap v13094: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:49.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:44:48 smithi153 ceph-mon[70044]: pgmap v13095: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:49.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:44:48 smithi083 ceph-mon[79930]: pgmap v13095: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:49.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:44:48 smithi083 ceph-mon[77242]: pgmap v13095: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:50.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:44:50 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:44:50.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:44:50 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:44:50.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:44:50 smithi153 ceph-mon[70044]: pgmap v13096: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:50.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:44:50 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:44:50.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:44:50 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:44:50.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:44:50 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:44:50.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:44:50 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:44:50.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:44:50 smithi083 ceph-mon[79930]: pgmap v13096: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:50.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:44:50 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:44:50.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:44:50 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:44:50.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:44:50 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:44:50.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:44:50 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:44:50.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:44:50 smithi083 ceph-mon[77242]: pgmap v13096: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:50.968 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:44:50 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:44:50.968 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:44:50 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:44:51.640 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:44:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:44:51] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T12:44:51.964 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:44:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:44:51.784Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:44:52.711 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:44:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:44:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:44:52.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:44:52 smithi083 ceph-mon[79930]: pgmap v13097: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:52.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:44:52 smithi083 ceph-mon[77242]: pgmap v13097: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:53.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:44:52 smithi153 ceph-mon[70044]: pgmap v13097: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:53.311 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:44:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:44:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:44:54.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:44:54 smithi153 ceph-mon[70044]: Detected new or changed devices on smithi083 2022-09-29T12:44:54.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:44:54 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:44:54.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:44:54 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:44:54.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:44:54 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:44:54.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:44:54 smithi153 ceph-mon[70044]: pgmap v13098: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:54.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:44:54 smithi083 ceph-mon[77242]: Detected new or changed devices on smithi083 2022-09-29T12:44:54.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:44:54 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:44:54.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:44:54 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:44:54.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:44:54 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:44:54.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:44:54 smithi083 ceph-mon[77242]: pgmap v13098: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:54.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:44:54 smithi083 ceph-mon[79930]: Detected new or changed devices on smithi083 2022-09-29T12:44:54.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:44:54 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:44:54.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:44:54 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:44:54.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:44:54 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:44:54.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:44:54 smithi083 ceph-mon[79930]: pgmap v13098: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:55.465 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:44:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:44:55.154Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:44:56.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:44:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:44:55.878Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:44:56.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:44:56 smithi083 ceph-mon[77242]: pgmap v13099: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:56.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:44:56 smithi083 ceph-mon[79930]: pgmap v13099: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:56.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:44:56 smithi153 ceph-mon[70044]: pgmap v13099: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:59.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:44:58 smithi153 ceph-mon[70044]: pgmap v13100: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:59.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:44:58 smithi083 ceph-mon[79930]: pgmap v13100: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:44:59.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:44:58 smithi083 ceph-mon[77242]: pgmap v13100: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:01.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:45:00 smithi153 ceph-mon[70044]: pgmap v13101: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:01.223 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:45:00 smithi083 ceph-mon[77242]: pgmap v13101: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:01.224 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:45:00 smithi083 ceph-mon[79930]: pgmap v13101: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:01.789 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:45:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:45:01] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T12:45:02.214 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:45:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:45:01.785Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:45:02.721 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:45:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:45:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:45:03.027 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:45:02 smithi153 ceph-mon[70044]: pgmap v13102: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:03.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:45:02 smithi083 ceph-mon[79930]: pgmap v13102: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:03.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:45:02 smithi083 ceph-mon[77242]: pgmap v13102: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:03.312 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:45:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:45:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:45:03.792 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:45:04.083 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:45:04.084 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 14s ago 7h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:45:04.084 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 8m ago 7h 64.1M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:45:04.085 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 14s ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:45:04.085 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 8m ago 7h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:45:04.086 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 8m ago 7h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:45:04.086 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 14s ago 7h 880M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:45:04.087 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 14s ago 7h 1630M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:45:04.087 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 8m ago 7h 688M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:45:04.087 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 14s ago 7h 943M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:45:04.088 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 14s ago 7h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:45:04.089 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 8m ago 7h 25.8M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:45:04.096 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 14s ago 7h 1177M 2357M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:45:04.097 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 14s ago 7h 1165M 2357M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:45:04.097 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 14s ago 7h 886M 2357M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:45:04.105 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 14s ago 7h 1140M 2357M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:45:04.105 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 8m ago 7h 957M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:45:04.106 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 8m ago 7h 1226M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:45:04.106 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 8m ago 7h 1095M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:45:04.107 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 8m ago 7h 1300M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:45:04.107 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 8m ago 7h 171M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:45:04.439 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:45:04.439 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:45:04.439 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:45:04.440 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:45:04.440 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:45:04.440 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:45:04.440 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:45:04.441 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:45:04.441 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:45:04.441 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:45:04.441 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:45:04.442 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:45:04.442 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:45:04.442 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:45:04.442 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:45:04.443 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:45:04.443 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:45:04.443 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:45:05.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:45:04 smithi153 ceph-mon[70044]: pgmap v13103: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:05.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:45:04 smithi153 ceph-mon[70044]: from='client.49368 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:45:05.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:45:04 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/55019527' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:45:05.164 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:45:04 smithi083 ceph-mon[79930]: pgmap v13103: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:05.165 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:45:04 smithi083 ceph-mon[79930]: from='client.49368 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:45:05.165 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:45:04 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/55019527' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:45:05.167 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:45:04 smithi083 ceph-mon[77242]: pgmap v13103: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:05.167 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:45:04 smithi083 ceph-mon[77242]: from='client.49368 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:45:05.168 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:45:04 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/55019527' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:45:05.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:45:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:45:05.155Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:45:06.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:45:05 smithi153 ceph-mon[70044]: from='client.59305 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:45:06.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:45:05 smithi083 ceph-mon[79930]: from='client.59305 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:45:06.219 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:45:05 smithi083 ceph-mon[77242]: from='client.59305 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:45:06.224 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:45:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:45:05.879Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:45:07.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:45:06 smithi153 ceph-mon[70044]: pgmap v13104: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:07.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:45:06 smithi083 ceph-mon[79930]: pgmap v13104: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:07.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:45:06 smithi083 ceph-mon[77242]: pgmap v13104: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:09.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:45:08 smithi153 ceph-mon[70044]: pgmap v13105: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:09.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:45:08 smithi083 ceph-mon[79930]: pgmap v13105: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:09.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:45:08 smithi083 ceph-mon[77242]: pgmap v13105: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:11.066 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:45:10 smithi153 ceph-mon[70044]: pgmap v13106: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:11.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:45:10 smithi083 ceph-mon[79930]: pgmap v13106: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:11.229 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:45:10 smithi083 ceph-mon[77242]: pgmap v13106: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:11.788 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:45:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:45:11] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T12:45:12.214 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:45:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:45:11.786Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:45:12.738 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:45:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:45:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:45:13.032 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:45:12 smithi153 ceph-mon[70044]: pgmap v13107: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:13.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:45:12 smithi083 ceph-mon[77242]: pgmap v13107: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:13.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:45:12 smithi083 ceph-mon[79930]: pgmap v13107: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:13.314 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:45:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:45:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:45:15.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:45:14 smithi153 ceph-mon[70044]: pgmap v13108: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:15.159 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:45:14 smithi083 ceph-mon[79930]: pgmap v13108: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:15.160 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:45:14 smithi083 ceph-mon[77242]: pgmap v13108: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:15.465 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:45:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:45:15.156Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:45:16.214 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:45:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:45:15.879Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:45:16.722 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:45:16 smithi083 ceph-mon[77242]: pgmap v13109: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:16.723 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:45:16 smithi083 ceph-mon[79930]: pgmap v13109: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:16.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:45:16 smithi153 ceph-mon[70044]: pgmap v13109: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:19.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:45:18 smithi153 ceph-mon[70044]: pgmap v13110: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:19.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:45:18 smithi083 ceph-mon[77242]: pgmap v13110: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:19.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:45:18 smithi083 ceph-mon[79930]: pgmap v13110: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:21.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:45:20 smithi153 ceph-mon[70044]: pgmap v13111: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:21.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:45:20 smithi083 ceph-mon[79930]: pgmap v13111: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:21.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:45:20 smithi083 ceph-mon[77242]: pgmap v13111: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:21.805 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:45:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:45:21] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T12:45:22.214 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:45:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:45:21.787Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:45:22.727 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:45:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:45:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:45:23.027 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:45:22 smithi153 ceph-mon[70044]: pgmap v13112: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:23.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:45:22 smithi083 ceph-mon[77242]: pgmap v13112: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:23.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:45:22 smithi083 ceph-mon[79930]: pgmap v13112: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:23.311 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:45:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:45:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:45:25.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:45:24 smithi153 ceph-mon[70044]: pgmap v13113: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:25.163 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:45:24 smithi083 ceph-mon[79930]: pgmap v13113: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:25.163 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:45:24 smithi083 ceph-mon[77242]: pgmap v13113: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:25.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:45:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:45:25.156Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:45:26.216 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:45:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:45:25.879Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:45:26.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:45:26 smithi083 ceph-mon[79930]: pgmap v13114: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:26.718 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:45:26 smithi083 ceph-mon[77242]: pgmap v13114: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:26.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:45:26 smithi153 ceph-mon[70044]: pgmap v13114: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:29.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:45:28 smithi153 ceph-mon[70044]: pgmap v13115: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:29.222 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:45:28 smithi083 ceph-mon[79930]: pgmap v13115: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:29.223 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:45:28 smithi083 ceph-mon[77242]: pgmap v13115: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:31.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:45:30 smithi153 ceph-mon[70044]: pgmap v13116: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:31.218 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:45:30 smithi083 ceph-mon[79930]: pgmap v13116: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:31.219 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:45:30 smithi083 ceph-mon[77242]: pgmap v13116: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:31.793 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:45:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:45:31] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T12:45:32.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:45:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:45:31.787Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:45:32.734 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:45:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:45:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:45:33.028 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:45:32 smithi153 ceph-mon[70044]: pgmap v13117: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:33.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:45:32 smithi083 ceph-mon[77242]: pgmap v13117: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:33.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:45:32 smithi083 ceph-mon[79930]: pgmap v13117: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:33.318 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:45:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:45:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:45:34.766 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:45:35.057 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:45:35.058 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 45s ago 7h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:45:35.058 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 8m ago 7h 64.1M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:45:35.058 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 45s ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:45:35.058 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 8m ago 7h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:45:35.059 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 8m ago 7h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:45:35.059 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 45s ago 7h 880M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:45:35.059 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 45s ago 7h 1630M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:45:35.059 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 8m ago 7h 688M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:45:35.060 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 45s ago 7h 943M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:45:35.060 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 45s ago 7h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:45:35.060 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 8m ago 7h 25.8M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:45:35.060 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 45s ago 7h 1177M 2357M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:45:35.068 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 45s ago 7h 1165M 2357M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:45:35.087 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 45s ago 7h 886M 2357M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:45:35.087 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 45s ago 7h 1140M 2357M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:45:35.088 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 8m ago 7h 957M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:45:35.088 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 8m ago 7h 1226M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:45:35.089 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 8m ago 7h 1095M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:45:35.089 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 8m ago 7h 1300M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:45:35.095 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 8m ago 7h 171M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:45:35.097 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:45:34 smithi153 ceph-mon[70044]: pgmap v13118: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:35.166 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:45:34 smithi083 ceph-mon[79930]: pgmap v13118: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:35.166 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:45:34 smithi083 ceph-mon[77242]: pgmap v13118: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:35.415 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:45:35.415 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:45:35.416 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:45:35.416 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:45:35.417 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:45:35.417 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:45:35.417 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:45:35.437 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:45:35.442 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:45:35.442 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:45:35.443 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:45:35.443 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:45:35.444 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:45:35.444 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:45:35.445 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:45:35.445 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:45:35.445 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:45:35.453 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:45:35.470 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:45:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:45:35.157Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:45:36.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:45:35 smithi153 ceph-mon[70044]: from='client.49386 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:45:36.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:45:35 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2066676775' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:45:36.217 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:45:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:45:35.880Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:45:36.218 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:45:35 smithi083 ceph-mon[77242]: from='client.49386 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:45:36.230 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:45:35 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2066676775' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:45:36.231 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:45:35 smithi083 ceph-mon[79930]: from='client.49386 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:45:36.231 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:45:35 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2066676775' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:45:37.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:45:36 smithi153 ceph-mon[70044]: from='client.59323 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:45:37.063 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:45:36 smithi153 ceph-mon[70044]: pgmap v13119: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:37.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:45:36 smithi083 ceph-mon[79930]: from='client.59323 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:45:37.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:45:36 smithi083 ceph-mon[79930]: pgmap v13119: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:37.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:45:36 smithi083 ceph-mon[77242]: from='client.59323 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:45:37.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:45:36 smithi083 ceph-mon[77242]: pgmap v13119: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:39.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:45:38 smithi153 ceph-mon[70044]: pgmap v13120: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:39.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:45:38 smithi083 ceph-mon[79930]: pgmap v13120: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:39.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:45:38 smithi083 ceph-mon[77242]: pgmap v13120: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:41.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:45:40 smithi153 ceph-mon[70044]: pgmap v13121: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:41.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:45:40 smithi083 ceph-mon[79930]: pgmap v13121: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:41.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:45:40 smithi083 ceph-mon[77242]: pgmap v13121: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:41.791 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:45:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:45:41] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T12:45:41.792 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:45:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:45:41.792 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:45:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:45:41.792 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:45:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:45:41.792 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:45:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:45:41.793 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:45:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:45:41.793 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:45:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:45:41.793 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:45:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:45:41.793 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:45:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:45:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:45:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:45:42.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:45:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:45:42.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:45:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:45:42.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:45:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:45:42.226 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:45:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:45:41.788Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:45:42.759 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:45:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:45:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:45:43.028 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:45:42 smithi153 ceph-mon[70044]: pgmap v13122: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:43.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:45:42 smithi083 ceph-mon[79930]: pgmap v13122: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:43.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:45:42 smithi083 ceph-mon[77242]: pgmap v13122: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:43.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:45:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:45:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:45:45.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:45:44 smithi153 ceph-mon[70044]: pgmap v13123: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:45.166 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:45:44 smithi083 ceph-mon[79930]: pgmap v13123: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:45.167 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:45:44 smithi083 ceph-mon[77242]: pgmap v13123: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:45.468 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:45:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:45:45.158Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:45:46.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:45:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:45:45.881Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:45:46.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:45:46 smithi083 ceph-mon[79930]: pgmap v13124: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:46.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:45:46 smithi083 ceph-mon[77242]: pgmap v13124: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:46.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:45:46 smithi153 ceph-mon[70044]: pgmap v13124: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:49.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:45:48 smithi153 ceph-mon[70044]: pgmap v13125: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:49.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:45:48 smithi083 ceph-mon[79930]: pgmap v13125: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:49.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:45:48 smithi083 ceph-mon[77242]: pgmap v13125: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:51.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:45:50 smithi153 ceph-mon[70044]: pgmap v13126: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:51.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:45:50 smithi083 ceph-mon[79930]: pgmap v13126: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:51.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:45:50 smithi083 ceph-mon[77242]: pgmap v13126: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:51.791 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:45:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:45:51] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T12:45:52.215 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:45:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:45:51.788Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:45:52.742 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:45:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:45:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:45:53.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:45:52 smithi153 ceph-mon[70044]: pgmap v13127: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:53.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:45:52 smithi083 ceph-mon[79930]: pgmap v13127: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:53.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:45:52 smithi083 ceph-mon[77242]: pgmap v13127: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:53.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:45:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:45:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:45:54.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:45:53 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:45:54.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:45:53 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:45:54.063 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:45:53 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:45:54.063 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:45:53 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:45:54.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:45:53 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:45:54.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:45:53 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:45:54.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:45:53 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:45:54.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:45:53 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:45:54.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:45:53 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:45:54.230 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:45:53 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:45:54.231 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:45:53 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:45:54.231 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:45:53 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:45:55.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:45:54 smithi153 ceph-mon[70044]: pgmap v13128: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:55.164 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:45:54 smithi083 ceph-mon[79930]: pgmap v13128: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:55.165 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:45:54 smithi083 ceph-mon[77242]: pgmap v13128: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:55.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:45:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:45:55.158Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:45:56.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:45:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:45:55.881Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:45:56.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:45:56 smithi083 ceph-mon[79930]: pgmap v13129: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:56.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:45:56 smithi083 ceph-mon[77242]: pgmap v13129: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:56.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:45:56 smithi153 ceph-mon[70044]: pgmap v13129: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:59.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:45:58 smithi153 ceph-mon[70044]: pgmap v13130: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:59.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:45:58 smithi083 ceph-mon[79930]: pgmap v13130: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:45:59.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:45:58 smithi083 ceph-mon[77242]: pgmap v13130: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:01.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:46:00 smithi153 ceph-mon[70044]: pgmap v13131: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:01.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:46:00 smithi083 ceph-mon[77242]: pgmap v13131: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:01.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:46:00 smithi083 ceph-mon[79930]: pgmap v13131: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:01.792 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:46:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:46:01] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T12:46:02.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:46:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:46:01.789Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:46:02.738 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:46:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:46:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:46:03.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:46:02 smithi153 ceph-mon[70044]: pgmap v13132: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:03.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:46:02 smithi083 ceph-mon[77242]: pgmap v13132: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:03.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:46:02 smithi083 ceph-mon[79930]: pgmap v13132: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:46:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:46:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:46:05.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:46:04 smithi153 ceph-mon[70044]: pgmap v13133: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:05.162 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:46:04 smithi083 ceph-mon[79930]: pgmap v13133: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:05.163 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:46:04 smithi083 ceph-mon[77242]: pgmap v13133: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:05.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:46:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:46:05.159Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:46:05.741 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:46:06.033 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:46:06.033 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 76s ago 7h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:46:06.033 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 9m ago 7h 64.1M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:46:06.034 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 76s ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:46:06.034 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 9m ago 7h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:46:06.034 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 9m ago 7h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:46:06.034 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 76s ago 7h 880M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:46:06.035 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 76s ago 7h 1630M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:46:06.035 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 9m ago 7h 688M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:46:06.035 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 76s ago 7h 943M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:46:06.035 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 76s ago 7h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:46:06.035 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 9m ago 7h 25.8M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:46:06.036 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 76s ago 7h 1177M 2357M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:46:06.036 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 76s ago 7h 1165M 2357M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:46:06.036 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 76s ago 7h 886M 2357M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:46:06.036 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 76s ago 7h 1140M 2357M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:46:06.037 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 9m ago 7h 957M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:46:06.037 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 9m ago 7h 1226M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:46:06.037 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 9m ago 7h 1095M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:46:06.038 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 9m ago 7h 1300M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:46:06.038 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 9m ago 7h 171M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:46:06.214 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:46:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:46:05.881Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:46:06.388 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:46:06.388 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:46:06.389 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:46:06.389 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:46:06.390 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:46:06.390 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:46:06.390 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:46:06.391 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:46:06.391 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:46:06.392 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:46:06.392 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:46:06.393 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:46:06.393 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:46:06.394 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:46:06.403 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:46:06.403 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:46:06.404 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:46:06.404 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:46:06.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:46:06 smithi083 ceph-mon[77242]: pgmap v13134: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:06.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:46:06 smithi083 ceph-mon[77242]: from='client.49404 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:46:06.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:46:06 smithi083 ceph-mon[77242]: from='client.59341 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:46:06.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:46:06 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2382248003' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:46:06.718 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:46:06 smithi083 ceph-mon[79930]: pgmap v13134: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:06.719 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:46:06 smithi083 ceph-mon[79930]: from='client.49404 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:46:06.719 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:46:06 smithi083 ceph-mon[79930]: from='client.59341 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:46:06.719 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:46:06 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2382248003' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:46:06.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:46:06 smithi153 ceph-mon[70044]: pgmap v13134: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:06.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:46:06 smithi153 ceph-mon[70044]: from='client.49404 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:46:06.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:46:06 smithi153 ceph-mon[70044]: from='client.59341 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:46:06.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:46:06 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2382248003' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:46:09.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:46:08 smithi153 ceph-mon[70044]: pgmap v13135: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:09.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:46:08 smithi083 ceph-mon[77242]: pgmap v13135: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:09.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:46:08 smithi083 ceph-mon[79930]: pgmap v13135: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:11.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:46:10 smithi153 ceph-mon[70044]: pgmap v13136: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:11.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:46:10 smithi083 ceph-mon[77242]: pgmap v13136: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:11.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:46:10 smithi083 ceph-mon[79930]: pgmap v13136: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:11.792 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:46:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:46:11] "GET /metrics HTTP/1.1" 200 225229 "" "Prometheus/2.33.4" 2022-09-29T12:46:12.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:46:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:46:11.790Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:46:12.742 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:46:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:46:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:46:13.027 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:46:12 smithi153 ceph-mon[70044]: pgmap v13137: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:13.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:46:12 smithi083 ceph-mon[79930]: pgmap v13137: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:13.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:46:12 smithi083 ceph-mon[77242]: pgmap v13137: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:46:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:46:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:46:15.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:46:14 smithi153 ceph-mon[70044]: pgmap v13138: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:15.163 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:46:14 smithi083 ceph-mon[79930]: pgmap v13138: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:15.163 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:46:14 smithi083 ceph-mon[77242]: pgmap v13138: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:15.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:46:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:46:15.160Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:46:16.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:46:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:46:15.882Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:46:16.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:46:16 smithi153 ceph-mon[70044]: pgmap v13139: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:16.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:46:16 smithi083 ceph-mon[79930]: pgmap v13139: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:16.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:46:16 smithi083 ceph-mon[77242]: pgmap v13139: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:19.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:46:18 smithi153 ceph-mon[70044]: pgmap v13140: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:19.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:46:18 smithi083 ceph-mon[77242]: pgmap v13140: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:19.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:46:18 smithi083 ceph-mon[79930]: pgmap v13140: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:21.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:46:20 smithi153 ceph-mon[70044]: pgmap v13141: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:21.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:46:20 smithi083 ceph-mon[79930]: pgmap v13141: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:21.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:46:20 smithi083 ceph-mon[77242]: pgmap v13141: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:21.793 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:46:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:46:21] "GET /metrics HTTP/1.1" 200 225229 "" "Prometheus/2.33.4" 2022-09-29T12:46:22.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:46:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:46:21.791Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:46:22.745 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:46:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:46:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:46:23.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:46:22 smithi153 ceph-mon[70044]: pgmap v13142: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:23.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:46:22 smithi083 ceph-mon[77242]: pgmap v13142: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:23.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:46:22 smithi083 ceph-mon[79930]: pgmap v13142: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:46:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:46:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:46:25.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:46:24 smithi153 ceph-mon[70044]: pgmap v13143: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:25.163 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:46:24 smithi083 ceph-mon[79930]: pgmap v13143: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:25.164 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:46:24 smithi083 ceph-mon[77242]: pgmap v13143: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:25.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:46:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:46:25.160Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:46:26.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:46:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:46:25.882Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:46:26.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:46:26 smithi153 ceph-mon[70044]: pgmap v13144: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:26.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:46:26 smithi083 ceph-mon[79930]: pgmap v13144: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:26.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:46:26 smithi083 ceph-mon[77242]: pgmap v13144: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:29.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:46:28 smithi153 ceph-mon[70044]: pgmap v13145: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:29.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:46:28 smithi083 ceph-mon[79930]: pgmap v13145: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:29.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:46:28 smithi083 ceph-mon[77242]: pgmap v13145: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:31.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:46:30 smithi153 ceph-mon[70044]: pgmap v13146: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:31.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:46:30 smithi083 ceph-mon[79930]: pgmap v13146: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:31.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:46:30 smithi083 ceph-mon[77242]: pgmap v13146: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:31.793 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:46:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:46:31] "GET /metrics HTTP/1.1" 200 225228 "" "Prometheus/2.33.4" 2022-09-29T12:46:32.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:46:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:46:31.790Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:46:32.750 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:46:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:46:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:46:33.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:46:32 smithi153 ceph-mon[70044]: pgmap v13147: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:33.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:46:32 smithi083 ceph-mon[79930]: pgmap v13147: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:33.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:46:32 smithi083 ceph-mon[77242]: pgmap v13147: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:33.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:46:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:46:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:46:35.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:46:34 smithi153 ceph-mon[70044]: pgmap v13148: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:35.164 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:46:34 smithi083 ceph-mon[79930]: pgmap v13148: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:35.165 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:46:34 smithi083 ceph-mon[77242]: pgmap v13148: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:35.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:46:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:46:35.161Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:46:36.214 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:46:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:46:35.883Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:46:36.713 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:46:36.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:46:36 smithi153 ceph-mon[70044]: pgmap v13149: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:36.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:46:36 smithi083 ceph-mon[79930]: pgmap v13149: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:36.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:46:36 smithi083 ceph-mon[77242]: pgmap v13149: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:37.003 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:46:37.004 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 107s ago 7h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:46:37.004 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 9m ago 7h 64.1M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:46:37.005 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 107s ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:46:37.005 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 9m ago 7h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:46:37.006 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 9m ago 7h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:46:37.006 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 107s ago 7h 880M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:46:37.006 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 107s ago 7h 1630M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:46:37.007 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 9m ago 7h 688M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:46:37.007 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 107s ago 7h 943M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:46:37.008 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 107s ago 7h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:46:37.008 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 9m ago 7h 25.8M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:46:37.008 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 107s ago 7h 1177M 2357M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:46:37.009 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 107s ago 7h 1165M 2357M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:46:37.009 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 107s ago 7h 886M 2357M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:46:37.009 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 107s ago 7h 1140M 2357M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:46:37.010 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 9m ago 7h 957M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:46:37.010 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 9m ago 7h 1226M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:46:37.011 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 9m ago 7h 1095M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:46:37.011 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 9m ago 7h 1300M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:46:37.012 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 9m ago 7h 171M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:46:37.357 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:46:37.358 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:46:37.359 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:46:37.360 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:46:37.360 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:46:37.362 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:46:37.362 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:46:37.363 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:46:37.363 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:46:37.363 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:46:37.364 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:46:37.364 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:46:37.365 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:46:37.365 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:46:37.366 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:46:37.366 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:46:37.367 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:46:37.367 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:46:37.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:46:37 smithi153 ceph-mon[70044]: from='client.49422 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:46:37.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:46:37 smithi153 ceph-mon[70044]: from='client.49428 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:46:37.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:46:37 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/843592239' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:46:37.974 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:46:37 smithi083 ceph-mon[77242]: from='client.49422 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:46:37.975 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:46:37 smithi083 ceph-mon[77242]: from='client.49428 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:46:37.975 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:46:37 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/843592239' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:46:37.976 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:46:37 smithi083 ceph-mon[79930]: from='client.49422 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:46:37.977 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:46:37 smithi083 ceph-mon[79930]: from='client.49428 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:46:37.977 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:46:37 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/843592239' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:46:38.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:46:38 smithi153 ceph-mon[70044]: pgmap v13150: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:38.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:46:38 smithi083 ceph-mon[79930]: pgmap v13150: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:38.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:46:38 smithi083 ceph-mon[77242]: pgmap v13150: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:46:40 smithi153 ceph-mon[70044]: pgmap v13151: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:41.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:46:40 smithi083 ceph-mon[79930]: pgmap v13151: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:41.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:46:40 smithi083 ceph-mon[77242]: pgmap v13151: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:41.794 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:46:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:46:41] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T12:46:41.795 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:46:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:46:41.795 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:46:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:46:41.796 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:46:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:46:41.796 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:46:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:46:41.797 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:46:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:46:41.797 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:46:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:46:41.798 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:46:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:46:41.798 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:46:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:46:42.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:46:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:46:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:46:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:46:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:46:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:46:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:46:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:46:42.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:46:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:46:41.791Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:46:42.753 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:46:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:46:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:46:43.026 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:46:42 smithi153 ceph-mon[70044]: pgmap v13152: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:43.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:46:42 smithi083 ceph-mon[79930]: pgmap v13152: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:43.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:46:42 smithi083 ceph-mon[77242]: pgmap v13152: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:43.310 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:46:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:46:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:46:45.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:46:44 smithi153 ceph-mon[70044]: pgmap v13153: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:45.164 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:46:44 smithi083 ceph-mon[79930]: pgmap v13153: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:45.164 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:46:44 smithi083 ceph-mon[77242]: pgmap v13153: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:45.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:46:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:46:45.161Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:46:46.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:46:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:46:45.884Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:46:46.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:46:46 smithi153 ceph-mon[70044]: pgmap v13154: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:46.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:46:46 smithi083 ceph-mon[77242]: pgmap v13154: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:46.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:46:46 smithi083 ceph-mon[79930]: pgmap v13154: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:49.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:46:48 smithi153 ceph-mon[70044]: pgmap v13155: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:49.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:46:48 smithi083 ceph-mon[77242]: pgmap v13155: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:49.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:46:48 smithi083 ceph-mon[79930]: pgmap v13155: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:51.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:46:50 smithi153 ceph-mon[70044]: pgmap v13156: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:51.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:46:50 smithi083 ceph-mon[79930]: pgmap v13156: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:51.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:46:50 smithi083 ceph-mon[77242]: pgmap v13156: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:51.794 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:46:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:46:51] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T12:46:52.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:46:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:46:51.792Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:46:52.753 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:46:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:46:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:46:53.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:46:52 smithi153 ceph-mon[70044]: pgmap v13157: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:53.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:46:52 smithi083 ceph-mon[79930]: pgmap v13157: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:53.221 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:46:52 smithi083 ceph-mon[77242]: pgmap v13157: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:53.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:46:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:46:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:46:55.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:46:54 smithi153 ceph-mon[70044]: pgmap v13158: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:55.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:46:54 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:46:55.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:46:54 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:46:55.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:46:54 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:46:55.164 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:46:54 smithi083 ceph-mon[79930]: pgmap v13158: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:55.164 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:46:54 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:46:55.165 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:46:54 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:46:55.165 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:46:54 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:46:55.165 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:46:54 smithi083 ceph-mon[77242]: pgmap v13158: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:55.166 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:46:54 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:46:55.166 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:46:54 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:46:55.166 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:46:54 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:46:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:46:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:46:55.161Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:46:56.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:46:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:46:55.885Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:46:56.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:46:56 smithi153 ceph-mon[70044]: pgmap v13159: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:56.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:46:56 smithi083 ceph-mon[79930]: pgmap v13159: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:56.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:46:56 smithi083 ceph-mon[77242]: pgmap v13159: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:59.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:46:58 smithi153 ceph-mon[70044]: pgmap v13160: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:59.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:46:58 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:46:59.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:46:58 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:46:59.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:46:58 smithi083 ceph-mon[79930]: pgmap v13160: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:59.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:46:58 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:46:59.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:46:58 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:46:59.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:46:58 smithi083 ceph-mon[77242]: pgmap v13160: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:46:59.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:46:58 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:46:59.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:46:58 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:47:00.669 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:47:00 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:47:00.669 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:47:00 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:47:00.670 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:47:00 smithi153 ceph-mon[70044]: pgmap v13161: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:47:00.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:47:00 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:47:00.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:47:00 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:47:00.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:47:00 smithi083 ceph-mon[79930]: pgmap v13161: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:47:00.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:47:00 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:47:00.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:47:00 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:47:00.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:47:00 smithi083 ceph-mon[77242]: pgmap v13161: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:47:01.795 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:47:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:47:01] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T12:47:02.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:47:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:47:01.792Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:47:02.752 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:47:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:47:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:47:03.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:47:02 smithi153 ceph-mon[70044]: pgmap v13162: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:47:03.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:47:02 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:47:03.026 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:47:02 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:47:03.026 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:47:02 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:47:03.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:47:02 smithi083 ceph-mon[79930]: pgmap v13162: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:47:03.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:47:02 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:47:03.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:47:02 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:47:03.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:47:02 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:47:03.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:47:02 smithi083 ceph-mon[77242]: pgmap v13162: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:47:03.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:47:02 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:47:03.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:47:02 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:47:03.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:47:02 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:47:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:47:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:47:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:47:04.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:47:03 smithi153 ceph-mon[70044]: Detected new or changed devices on smithi153 2022-09-29T12:47:04.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:47:03 smithi083 ceph-mon[79930]: Detected new or changed devices on smithi153 2022-09-29T12:47:04.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:47:03 smithi083 ceph-mon[77242]: Detected new or changed devices on smithi153 2022-09-29T12:47:05.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:47:04 smithi153 ceph-mon[70044]: pgmap v13163: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:47:05.165 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:47:04 smithi083 ceph-mon[79930]: pgmap v13163: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:47:05.166 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:47:04 smithi083 ceph-mon[77242]: pgmap v13163: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:47:05.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:47:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:47:05.162Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:47:06.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:47:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:47:05.886Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:47:06.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:47:06 smithi153 ceph-mon[70044]: pgmap v13164: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:47:06.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:47:06 smithi083 ceph-mon[79930]: pgmap v13164: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:47:06.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:47:06 smithi083 ceph-mon[77242]: pgmap v13164: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:47:07.684 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:47:07.974 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:47:07.974 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 2m ago 7h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:47:07.975 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 9s ago 7h 64.1M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:47:07.975 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 2m ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:47:07.975 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 9s ago 7h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:47:07.976 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 9s ago 7h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:47:07.976 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 2m ago 7h 880M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:47:07.976 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 2m ago 7h 1630M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:47:07.976 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 9s ago 7h 702M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:47:07.977 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 2m ago 7h 943M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:47:07.977 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 2m ago 7h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:47:07.977 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 9s ago 7h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:47:07.977 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 2m ago 7h 1177M 2357M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:47:07.977 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 2m ago 7h 1165M 2357M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:47:07.978 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 2m ago 7h 886M 2357M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:47:07.978 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 2m ago 7h 1140M 2357M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:47:07.978 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 9s ago 7h 979M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:47:07.978 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 9s ago 7h 1254M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:47:07.979 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 9s ago 7h 1121M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:47:07.979 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 9s ago 7h 1331M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:47:07.979 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 9s ago 7h 176M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:47:08.328 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:47:08.329 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:47:08.329 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:47:08.329 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:47:08.330 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:47:08.330 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:47:08.331 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:47:08.331 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:47:08.332 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:47:08.332 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:47:08.333 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:47:08.333 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:47:08.334 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:47:08.334 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:47:08.335 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:47:08.335 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:47:08.336 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:47:08.336 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:47:09.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:47:08 smithi153 ceph-mon[70044]: from='client.49440 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:47:09.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:47:08 smithi153 ceph-mon[70044]: pgmap v13165: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:47:09.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:47:08 smithi153 ceph-mon[70044]: from='client.59377 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:47:09.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:47:08 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4287261864' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:47:09.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:47:08 smithi083 ceph-mon[77242]: from='client.49440 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:47:09.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:47:08 smithi083 ceph-mon[77242]: pgmap v13165: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:47:09.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:47:08 smithi083 ceph-mon[77242]: from='client.59377 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:47:09.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:47:08 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4287261864' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:47:09.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:47:08 smithi083 ceph-mon[79930]: from='client.49440 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:47:09.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:47:08 smithi083 ceph-mon[79930]: pgmap v13165: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:47:09.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:47:08 smithi083 ceph-mon[79930]: from='client.59377 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:47:09.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:47:08 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4287261864' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:47:11.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:47:10 smithi153 ceph-mon[70044]: pgmap v13166: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:47:11.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:47:10 smithi083 ceph-mon[77242]: pgmap v13166: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:47:11.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:47:10 smithi083 ceph-mon[79930]: pgmap v13166: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:47:11.795 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:47:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:47:11] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T12:47:12.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:47:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:47:11.792Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:47:12.759 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:47:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:47:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:47:13.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:47:12 smithi153 ceph-mon[70044]: pgmap v13167: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:47:13.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:47:12 smithi083 ceph-mon[77242]: pgmap v13167: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:47:13.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:47:12 smithi083 ceph-mon[79930]: pgmap v13167: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:47:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:47:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:47:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:47:15.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:47:14 smithi153 ceph-mon[70044]: pgmap v13168: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:47:15.165 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:47:14 smithi083 ceph-mon[79930]: pgmap v13168: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:47:15.166 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:47:14 smithi083 ceph-mon[77242]: pgmap v13168: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:47:15.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:47:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:47:15.163Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:47:16.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:47:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:47:15.887Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:47:16.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:47:16 smithi153 ceph-mon[70044]: pgmap v13169: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:47:16.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:47:16 smithi083 ceph-mon[79930]: pgmap v13169: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:47:16.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:47:16 smithi083 ceph-mon[77242]: pgmap v13169: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:47:19.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:47:18 smithi153 ceph-mon[70044]: pgmap v13170: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:47:19.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:47:18 smithi083 ceph-mon[79930]: pgmap v13170: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:47:19.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:47:18 smithi083 ceph-mon[77242]: pgmap v13170: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:47:21.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:47:20 smithi153 ceph-mon[70044]: pgmap v13171: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:47:21.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:47:20 smithi083 ceph-mon[77242]: pgmap v13171: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:47:21.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:47:20 smithi083 ceph-mon[79930]: pgmap v13171: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:47:21.796 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:47:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:47:21] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T12:47:22.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:47:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:47:21.793Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:47:22.762 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:47:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:47:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:47:23.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:47:22 smithi153 ceph-mon[70044]: pgmap v13172: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:47:23.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:47:22 smithi083 ceph-mon[79930]: pgmap v13172: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:47:23.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:47:22 smithi083 ceph-mon[77242]: pgmap v13172: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:47:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:47:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:47:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:47:25.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:47:24 smithi153 ceph-mon[70044]: pgmap v13173: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:47:25.167 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:47:24 smithi083 ceph-mon[79930]: pgmap v13173: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:47:25.167 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:47:24 smithi083 ceph-mon[77242]: pgmap v13173: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:47:25.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:47:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:47:25.164Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:47:26.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:47:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:47:25.887Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:47:26.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:47:26 smithi153 ceph-mon[70044]: pgmap v13174: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:47:26.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:47:26 smithi083 ceph-mon[77242]: pgmap v13174: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:47:26.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:47:26 smithi083 ceph-mon[79930]: pgmap v13174: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:47:29.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:47:28 smithi153 ceph-mon[70044]: pgmap v13175: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:47:29.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:47:28 smithi083 ceph-mon[79930]: pgmap v13175: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:47:29.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:47:28 smithi083 ceph-mon[77242]: pgmap v13175: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:47:31.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:47:30 smithi153 ceph-mon[70044]: pgmap v13176: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:47:31.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:47:30 smithi083 ceph-mon[79930]: pgmap v13176: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:47:31.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:47:30 smithi083 ceph-mon[77242]: pgmap v13176: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:47:31.797 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:47:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:47:31] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T12:47:32.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:47:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:47:31.795Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:47:32.768 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:47:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:47:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:47:33.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:47:32 smithi153 ceph-mon[70044]: pgmap v13177: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:47:33.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:47:32 smithi083 ceph-mon[79930]: pgmap v13177: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:47:33.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:47:32 smithi083 ceph-mon[77242]: pgmap v13177: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:47:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:47:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:47:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:47:35.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:47:34 smithi153 ceph-mon[70044]: pgmap v13178: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:47:35.167 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:47:34 smithi083 ceph-mon[77242]: pgmap v13178: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:47:35.168 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:47:34 smithi083 ceph-mon[79930]: pgmap v13178: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:47:35.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:47:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:47:35.165Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:47:36.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:47:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:47:35.888Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:47:36.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:47:36 smithi153 ceph-mon[70044]: pgmap v13179: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:47:36.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:47:36 smithi083 ceph-mon[77242]: pgmap v13179: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:47:36.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:47:36 smithi083 ceph-mon[79930]: pgmap v13179: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:47:38.653 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:47:38.941 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:47:38.941 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 2m ago 7h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:47:38.941 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 40s ago 7h 64.1M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:47:38.942 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 2m ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:47:38.942 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 40s ago 7h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:47:38.942 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 40s ago 7h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:47:38.942 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 2m ago 7h 880M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:47:38.942 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 2m ago 7h 1630M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:47:38.943 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 40s ago 7h 702M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:47:38.943 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 2m ago 7h 943M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:47:38.943 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 2m ago 7h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:47:38.943 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 40s ago 7h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:47:38.944 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 2m ago 7h 1177M 2357M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:47:38.944 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 2m ago 7h 1165M 2357M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:47:38.944 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 2m ago 7h 886M 2357M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:47:38.944 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 2m ago 7h 1140M 2357M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:47:38.945 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 40s ago 7h 979M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:47:38.945 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 40s ago 7h 1254M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:47:38.945 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 40s ago 7h 1121M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:47:38.945 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 40s ago 7h 1331M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:47:38.945 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 40s ago 7h 176M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:47:39.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:47:38 smithi153 ceph-mon[70044]: pgmap v13180: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:47:39.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:47:38 smithi083 ceph-mon[79930]: pgmap v13180: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:47:39.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:47:38 smithi083 ceph-mon[77242]: pgmap v13180: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:47:39.300 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:47:39.301 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:47:39.301 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:47:39.301 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:47:39.302 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:47:39.302 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:47:39.302 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:47:39.302 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:47:39.303 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:47:39.303 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:47:39.303 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:47:39.303 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:47:39.304 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:47:39.304 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:47:39.304 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:47:39.304 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:47:39.305 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:47:39.305 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:47:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:47:39 smithi153 ceph-mon[70044]: from='client.49458 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:47:40.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:47:39 smithi153 ceph-mon[70044]: from='client.59395 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:47:40.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:47:39 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/962283411' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:47:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:47:39 smithi083 ceph-mon[79930]: from='client.49458 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:47:40.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:47:39 smithi083 ceph-mon[79930]: from='client.59395 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:47:40.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:47:39 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/962283411' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:47:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:47:39 smithi083 ceph-mon[77242]: from='client.49458 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:47:40.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:47:39 smithi083 ceph-mon[77242]: from='client.59395 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:47:40.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:47:39 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/962283411' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:47:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:47:40 smithi153 ceph-mon[70044]: pgmap v13181: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:47:41.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:47:40 smithi083 ceph-mon[79930]: pgmap v13181: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:47:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:47:40 smithi083 ceph-mon[77242]: pgmap v13181: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:47:41.774 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:47:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:47:41] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T12:47:41.775 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:47:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:47:41.775 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:47:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:47:41.775 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:47:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:47:41.776 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:47:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:47:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:47:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:47:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:47:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:47:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:47:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:47:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:47:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:47:42.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:47:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:47:41.795Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:47:42.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:47:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:47:42.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:47:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:47:42.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:47:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:47:42.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:47:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:47:42.776 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:47:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:47:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:47:43.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:47:42 smithi153 ceph-mon[70044]: pgmap v13182: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:47:43.059 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:47:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:47:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:47:43.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:47:42 smithi083 ceph-mon[77242]: pgmap v13182: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:47:43.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:47:42 smithi083 ceph-mon[79930]: pgmap v13182: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:47:45.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:47:44 smithi153 ceph-mon[70044]: pgmap v13183: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:47:45.175 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:47:44 smithi083 ceph-mon[79930]: pgmap v13183: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:47:45.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:47:44 smithi083 ceph-mon[77242]: pgmap v13183: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:47:45.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:47:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:47:45.166Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:47:46.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:47:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:47:45.889Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:47:46.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:47:46 smithi153 ceph-mon[70044]: pgmap v13184: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:47:46.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:47:46 smithi083 ceph-mon[77242]: pgmap v13184: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:47:46.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:47:46 smithi083 ceph-mon[79930]: pgmap v13184: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:47:49.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:47:48 smithi153 ceph-mon[70044]: pgmap v13185: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:47:49.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:47:48 smithi083 ceph-mon[77242]: pgmap v13185: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:47:49.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:47:48 smithi083 ceph-mon[79930]: pgmap v13185: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:47:51.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:47:50 smithi153 ceph-mon[70044]: pgmap v13186: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:47:51.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:47:50 smithi083 ceph-mon[79930]: pgmap v13186: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:47:51.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:47:50 smithi083 ceph-mon[77242]: pgmap v13186: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:47:51.798 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:47:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:47:51] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T12:47:52.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:47:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:47:51.795Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:47:52.773 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:47:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:47:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:47:53.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:47:52 smithi153 ceph-mon[70044]: pgmap v13187: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:47:53.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:47:52 smithi083 ceph-mon[79930]: pgmap v13187: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:47:53.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:47:52 smithi083 ceph-mon[77242]: pgmap v13187: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:47:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:47:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:47:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:47:55.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:47:54 smithi153 ceph-mon[70044]: pgmap v13188: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:47:55.168 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:47:54 smithi083 ceph-mon[79930]: pgmap v13188: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:47:55.169 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:47:54 smithi083 ceph-mon[77242]: pgmap v13188: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:47:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:47:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:47:55.166Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:47:56.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:47:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:47:55.889Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:47:56.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:47:56 smithi153 ceph-mon[70044]: pgmap v13189: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:47:56.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:47:56 smithi083 ceph-mon[79930]: pgmap v13189: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:47:56.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:47:56 smithi083 ceph-mon[77242]: pgmap v13189: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:47:59.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:47:58 smithi153 ceph-mon[70044]: pgmap v13190: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:47:59.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:47:58 smithi083 ceph-mon[79930]: pgmap v13190: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:47:59.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:47:58 smithi083 ceph-mon[77242]: pgmap v13190: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:01.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:48:00 smithi153 ceph-mon[70044]: pgmap v13191: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:01.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:48:00 smithi083 ceph-mon[79930]: pgmap v13191: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:01.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:48:00 smithi083 ceph-mon[77242]: pgmap v13191: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:01.799 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:48:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:48:01] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T12:48:02.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:48:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:48:01.796Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:48:02.777 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:48:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:48:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:48:03.059 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:48:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:48:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:48:03.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:48:02 smithi153 ceph-mon[70044]: pgmap v13192: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:03.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:48:02 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:48:03.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:48:02 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:48:03.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:48:02 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:48:03.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:48:02 smithi083 ceph-mon[79930]: pgmap v13192: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:03.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:48:02 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:48:03.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:48:02 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:48:03.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:48:02 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:48:03.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:48:02 smithi083 ceph-mon[77242]: pgmap v13192: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:03.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:48:02 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:48:03.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:48:02 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:48:03.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:48:02 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:48:04.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:48:03 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:48:04.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:48:03 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:48:04.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:48:03 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:48:05.169 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:48:04 smithi083 ceph-mon[79930]: pgmap v13193: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:05.170 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:48:04 smithi083 ceph-mon[77242]: pgmap v13193: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:05.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:48:04 smithi153 ceph-mon[70044]: pgmap v13193: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:05.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:48:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:48:05.167Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:48:06.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:48:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:48:05.890Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:48:06.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:48:06 smithi153 ceph-mon[70044]: pgmap v13194: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:06.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:48:06 smithi083 ceph-mon[79930]: pgmap v13194: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:06.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:48:06 smithi083 ceph-mon[77242]: pgmap v13194: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:09.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:48:08 smithi153 ceph-mon[70044]: pgmap v13195: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:09.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:48:08 smithi083 ceph-mon[77242]: pgmap v13195: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:09.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:48:08 smithi083 ceph-mon[79930]: pgmap v13195: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:09.625 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:48:09.918 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:48:09.918 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 3m ago 7h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:48:09.919 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 71s ago 7h 64.1M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:48:09.919 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 3m ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:48:09.919 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 71s ago 7h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:48:09.919 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 71s ago 7h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:48:09.919 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 3m ago 7h 880M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:48:09.920 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 3m ago 7h 1630M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:48:09.920 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 71s ago 7h 702M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:48:09.920 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 3m ago 7h 943M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:48:09.920 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 3m ago 7h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:48:09.921 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 71s ago 7h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:48:09.921 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 3m ago 7h 1177M 2357M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:48:09.921 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 3m ago 7h 1165M 2357M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:48:09.921 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 3m ago 7h 886M 2357M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:48:09.921 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 3m ago 7h 1140M 2357M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:48:09.922 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 71s ago 7h 979M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:48:09.922 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 71s ago 7h 1254M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:48:09.922 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 71s ago 7h 1121M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:48:09.922 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 71s ago 7h 1331M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:48:09.923 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 71s ago 7h 176M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:48:10.274 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:48:10.274 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:48:10.275 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:48:10.275 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:48:10.275 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:48:10.275 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:48:10.276 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:48:10.276 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:48:10.276 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:48:10.276 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:48:10.276 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:48:10.277 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:48:10.277 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:48:10.277 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:48:10.277 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:48:10.277 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:48:10.278 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:48:10.278 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:48:11.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:48:10 smithi153 ceph-mon[70044]: from='client.59407 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:48:11.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:48:10 smithi153 ceph-mon[70044]: pgmap v13196: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:11.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:48:10 smithi153 ceph-mon[70044]: from='client.49482 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:48:11.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:48:10 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4211734612' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:48:11.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:48:10 smithi083 ceph-mon[77242]: from='client.59407 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:48:11.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:48:10 smithi083 ceph-mon[77242]: pgmap v13196: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:11.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:48:10 smithi083 ceph-mon[77242]: from='client.49482 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:48:11.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:48:10 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4211734612' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:48:11.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:48:10 smithi083 ceph-mon[79930]: from='client.59407 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:48:11.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:48:10 smithi083 ceph-mon[79930]: pgmap v13196: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:11.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:48:10 smithi083 ceph-mon[79930]: from='client.49482 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:48:11.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:48:10 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4211734612' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:48:11.800 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:48:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:48:11] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T12:48:12.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:48:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:48:11.797Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:48:12.778 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:48:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:48:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:48:13.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:48:12 smithi153 ceph-mon[70044]: pgmap v13197: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:13.059 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:48:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:48:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:48:13.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:48:12 smithi083 ceph-mon[79930]: pgmap v13197: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:13.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:48:12 smithi083 ceph-mon[77242]: pgmap v13197: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:15.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:48:14 smithi153 ceph-mon[70044]: pgmap v13198: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:15.171 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:48:14 smithi083 ceph-mon[79930]: pgmap v13198: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:15.171 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:48:14 smithi083 ceph-mon[77242]: pgmap v13198: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:15.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:48:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:48:15.168Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:48:16.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:48:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:48:15.890Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:48:16.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:48:16 smithi153 ceph-mon[70044]: pgmap v13199: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:16.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:48:16 smithi083 ceph-mon[79930]: pgmap v13199: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:16.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:48:16 smithi083 ceph-mon[77242]: pgmap v13199: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:19.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:48:18 smithi153 ceph-mon[70044]: pgmap v13200: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:19.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:48:18 smithi083 ceph-mon[77242]: pgmap v13200: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:19.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:48:18 smithi083 ceph-mon[79930]: pgmap v13200: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:21.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:48:20 smithi153 ceph-mon[70044]: pgmap v13201: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:21.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:48:20 smithi083 ceph-mon[79930]: pgmap v13201: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:21.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:48:20 smithi083 ceph-mon[77242]: pgmap v13201: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:21.802 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:48:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:48:21] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T12:48:22.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:48:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:48:21.799Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:48:22.784 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:48:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:48:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:48:23.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:48:22 smithi153 ceph-mon[70044]: pgmap v13202: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:23.059 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:48:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:48:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:48:23.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:48:22 smithi083 ceph-mon[79930]: pgmap v13202: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:23.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:48:22 smithi083 ceph-mon[77242]: pgmap v13202: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:25.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:48:24 smithi153 ceph-mon[70044]: pgmap v13203: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:25.171 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:48:24 smithi083 ceph-mon[79930]: pgmap v13203: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:25.171 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:48:24 smithi083 ceph-mon[77242]: pgmap v13203: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:25.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:48:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:48:25.168Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:48:26.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:48:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:48:25.890Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:48:26.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:48:26 smithi153 ceph-mon[70044]: pgmap v13204: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:26.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:48:26 smithi083 ceph-mon[79930]: pgmap v13204: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:26.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:48:26 smithi083 ceph-mon[77242]: pgmap v13204: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:29.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:48:28 smithi153 ceph-mon[70044]: pgmap v13205: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:29.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:48:28 smithi083 ceph-mon[79930]: pgmap v13205: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:29.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:48:28 smithi083 ceph-mon[77242]: pgmap v13205: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:31.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:48:30 smithi153 ceph-mon[70044]: pgmap v13206: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:31.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:48:30 smithi083 ceph-mon[79930]: pgmap v13206: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:31.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:48:30 smithi083 ceph-mon[77242]: pgmap v13206: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:31.801 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:48:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:48:31] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T12:48:32.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:48:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:48:31.798Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:48:32.787 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:48:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:48:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:48:33.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:48:32 smithi153 ceph-mon[70044]: pgmap v13207: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:33.059 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:48:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:48:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:48:33.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:48:32 smithi083 ceph-mon[79930]: pgmap v13207: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:33.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:48:32 smithi083 ceph-mon[77242]: pgmap v13207: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:35.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:48:34 smithi153 ceph-mon[70044]: pgmap v13208: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:35.171 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:48:34 smithi083 ceph-mon[79930]: pgmap v13208: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:35.172 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:48:34 smithi083 ceph-mon[77242]: pgmap v13208: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:35.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:48:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:48:35.169Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:48:36.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:48:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:48:35.891Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:48:36.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:48:36 smithi153 ceph-mon[70044]: pgmap v13209: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:36.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:48:36 smithi083 ceph-mon[77242]: pgmap v13209: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:36.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:48:36 smithi083 ceph-mon[79930]: pgmap v13209: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:39.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:48:38 smithi153 ceph-mon[70044]: pgmap v13210: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:39.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:48:38 smithi083 ceph-mon[77242]: pgmap v13210: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:39.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:48:38 smithi083 ceph-mon[79930]: pgmap v13210: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:40.603 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:48:40.894 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:48:40.894 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 3m ago 7h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:48:40.894 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 102s ago 7h 64.1M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:48:40.895 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 3m ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:48:40.895 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 102s ago 7h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:48:40.895 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 102s ago 7h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:48:40.895 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 3m ago 7h 880M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:48:40.896 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 3m ago 7h 1630M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:48:40.896 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 102s ago 7h 702M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:48:40.896 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 3m ago 7h 943M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:48:40.896 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 3m ago 7h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:48:40.896 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 102s ago 7h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:48:40.897 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 3m ago 7h 1177M 2357M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:48:40.897 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 3m ago 7h 1165M 2357M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:48:40.897 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 3m ago 7h 886M 2357M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:48:40.897 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 3m ago 7h 1140M 2357M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:48:40.898 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 102s ago 7h 979M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:48:40.898 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 102s ago 7h 1254M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:48:40.898 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 102s ago 7h 1121M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:48:40.898 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 102s ago 7h 1331M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:48:40.898 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 102s ago 7h 176M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:48:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:48:40 smithi153 ceph-mon[70044]: pgmap v13211: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:41.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:48:40 smithi083 ceph-mon[79930]: pgmap v13211: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:48:40 smithi083 ceph-mon[77242]: pgmap v13211: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:41.251 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:48:41.251 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:48:41.252 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:48:41.252 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:48:41.252 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:48:41.252 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:48:41.253 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:48:41.253 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:48:41.253 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:48:41.253 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:48:41.254 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:48:41.254 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:48:41.254 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:48:41.254 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:48:41.254 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:48:41.255 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:48:41.255 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:48:41.255 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:48:41.790 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:48:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:48:41] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T12:48:42.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:48:41 smithi153 ceph-mon[70044]: from='client.59425 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:48:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:48:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:48:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:48:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:48:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:48:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:48:42.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:48:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:48:42.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:48:41 smithi153 ceph-mon[70044]: from='client.49500 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:48:42.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:48:41 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3990179371' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:48:42.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:48:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:48:41.799Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:48:42.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:48:41 smithi083 ceph-mon[79930]: from='client.59425 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:48:42.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:48:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:48:42.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:48:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:48:42.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:48:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:48:42.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:48:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:48:42.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:48:41 smithi083 ceph-mon[79930]: from='client.49500 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:48:42.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:48:41 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3990179371' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:48:42.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:48:41 smithi083 ceph-mon[77242]: from='client.59425 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:48:42.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:48:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:48:42.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:48:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:48:42.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:48:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:48:42.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:48:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:48:42.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:48:41 smithi083 ceph-mon[77242]: from='client.49500 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:48:42.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:48:41 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3990179371' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:48:42.792 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:48:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:48:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:48:43.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:48:42 smithi153 ceph-mon[70044]: pgmap v13212: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:43.059 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:48:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:48:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:48:43.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:48:42 smithi083 ceph-mon[79930]: pgmap v13212: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:43.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:48:42 smithi083 ceph-mon[77242]: pgmap v13212: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:45.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:48:44 smithi153 ceph-mon[70044]: pgmap v13213: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:45.172 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:48:44 smithi083 ceph-mon[79930]: pgmap v13213: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:45.173 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:48:44 smithi083 ceph-mon[77242]: pgmap v13213: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:45.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:48:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:48:45.170Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:48:46.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:48:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:48:45.891Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:48:46.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:48:46 smithi153 ceph-mon[70044]: pgmap v13214: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:46.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:48:46 smithi083 ceph-mon[77242]: pgmap v13214: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:46.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:48:46 smithi083 ceph-mon[79930]: pgmap v13214: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:49.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:48:48 smithi153 ceph-mon[70044]: pgmap v13215: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:49.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:48:48 smithi083 ceph-mon[79930]: pgmap v13215: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:49.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:48:48 smithi083 ceph-mon[77242]: pgmap v13215: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:51.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:48:50 smithi153 ceph-mon[70044]: pgmap v13216: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:51.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:48:50 smithi083 ceph-mon[77242]: pgmap v13216: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:51.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:48:50 smithi083 ceph-mon[79930]: pgmap v13216: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:51.802 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:48:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:48:51] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T12:48:52.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:48:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:48:51.800Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:48:52.791 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:48:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:48:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:48:53.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:48:52 smithi153 ceph-mon[70044]: pgmap v13217: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:53.059 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:48:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:48:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:48:53.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:48:52 smithi083 ceph-mon[77242]: pgmap v13217: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:53.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:48:52 smithi083 ceph-mon[79930]: pgmap v13217: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:55.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:48:54 smithi153 ceph-mon[70044]: pgmap v13218: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:55.173 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:48:54 smithi083 ceph-mon[79930]: pgmap v13218: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:55.174 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:48:54 smithi083 ceph-mon[77242]: pgmap v13218: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:48:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:48:55.171Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:48:56.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:48:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:48:55.892Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:48:56.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:48:56 smithi153 ceph-mon[70044]: pgmap v13219: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:56.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:48:56 smithi083 ceph-mon[79930]: pgmap v13219: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:56.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:48:56 smithi083 ceph-mon[77242]: pgmap v13219: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:59.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:48:58 smithi153 ceph-mon[70044]: pgmap v13220: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:59.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:48:58 smithi083 ceph-mon[79930]: pgmap v13220: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:48:59.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:48:58 smithi083 ceph-mon[77242]: pgmap v13220: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:01.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:49:00 smithi153 ceph-mon[70044]: pgmap v13221: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:01.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:49:00 smithi083 ceph-mon[79930]: pgmap v13221: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:01.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:49:00 smithi083 ceph-mon[77242]: pgmap v13221: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:01.804 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:49:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:49:01] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T12:49:02.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:49:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:49:01.801Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:49:02.796 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:49:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:49:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:49:03.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:49:02 smithi153 ceph-mon[70044]: pgmap v13222: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:03.059 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:49:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:49:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:49:03.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:49:02 smithi083 ceph-mon[79930]: pgmap v13222: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:03.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:49:02 smithi083 ceph-mon[77242]: pgmap v13222: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:04.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:49:03 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:49:04.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:49:03 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:49:04.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:49:03 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:49:04.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:49:03 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:49:04.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:49:03 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:49:04.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:49:03 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:49:04.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:49:03 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:49:04.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:49:03 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:49:04.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:49:03 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:49:04.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:49:03 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:49:04.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:49:03 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:49:04.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:49:03 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:49:05.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:49:04 smithi153 ceph-mon[70044]: pgmap v13223: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:05.174 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:49:04 smithi083 ceph-mon[79930]: pgmap v13223: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:05.174 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:49:04 smithi083 ceph-mon[77242]: pgmap v13223: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:05.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:49:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:49:05.171Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:49:06.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:49:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:49:05.893Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:49:06.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:49:06 smithi153 ceph-mon[70044]: pgmap v13224: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:06.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:49:06 smithi083 ceph-mon[77242]: pgmap v13224: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:06.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:49:06 smithi083 ceph-mon[79930]: pgmap v13224: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:09.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:49:08 smithi153 ceph-mon[70044]: pgmap v13225: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:09.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:49:08 smithi083 ceph-mon[77242]: pgmap v13225: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:09.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:49:08 smithi083 ceph-mon[79930]: pgmap v13225: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:11.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:49:10 smithi153 ceph-mon[70044]: pgmap v13226: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:11.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:49:10 smithi083 ceph-mon[79930]: pgmap v13226: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:11.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:49:10 smithi083 ceph-mon[77242]: pgmap v13226: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:11.579 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:49:11.802 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:49:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:49:11] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T12:49:11.869 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:49:11.869 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 4m ago 7h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:49:11.869 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 2m ago 7h 64.1M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:49:11.869 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 4m ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:49:11.870 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 2m ago 7h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:49:11.870 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 2m ago 7h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:49:11.870 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 4m ago 7h 880M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:49:11.871 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 4m ago 7h 1630M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:49:11.871 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 2m ago 7h 702M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:49:11.871 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 4m ago 7h 943M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:49:11.871 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 4m ago 7h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:49:11.872 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 2m ago 7h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:49:11.872 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 4m ago 7h 1177M 2357M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:49:11.872 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 4m ago 7h 1165M 2357M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:49:11.872 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 4m ago 7h 886M 2357M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:49:11.873 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 4m ago 7h 1140M 2357M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:49:11.873 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 2m ago 7h 979M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:49:11.873 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 2m ago 7h 1254M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:49:11.873 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 2m ago 7h 1121M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:49:11.874 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 2m ago 7h 1331M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:49:11.874 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 2m ago 7h 176M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:49:12.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:49:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:49:11.800Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:49:12.222 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:49:12.222 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:49:12.222 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:49:12.223 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:49:12.223 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:49:12.223 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:49:12.223 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:49:12.224 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:49:12.224 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:49:12.224 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:49:12.224 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:49:12.225 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:49:12.225 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:49:12.225 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:49:12.225 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:49:12.226 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:49:12.226 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:49:12.226 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:49:12.797 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:49:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:49:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:49:13.058 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:49:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:49:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:49:13.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:49:12 smithi153 ceph-mon[70044]: from='client.49512 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:49:13.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:49:12 smithi153 ceph-mon[70044]: pgmap v13227: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:13.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:49:12 smithi153 ceph-mon[70044]: from='client.59449 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:49:13.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:49:12 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3343639990' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:49:13.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:49:12 smithi083 ceph-mon[79930]: from='client.49512 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:49:13.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:49:12 smithi083 ceph-mon[79930]: pgmap v13227: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:13.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:49:12 smithi083 ceph-mon[79930]: from='client.59449 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:49:13.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:49:12 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3343639990' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:49:13.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:49:12 smithi083 ceph-mon[77242]: from='client.49512 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:49:13.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:49:12 smithi083 ceph-mon[77242]: pgmap v13227: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:13.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:49:12 smithi083 ceph-mon[77242]: from='client.59449 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:49:13.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:49:12 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3343639990' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:49:15.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:49:14 smithi153 ceph-mon[70044]: pgmap v13228: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:15.174 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:49:14 smithi083 ceph-mon[79930]: pgmap v13228: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:15.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:49:14 smithi083 ceph-mon[77242]: pgmap v13228: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:15.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:49:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:49:15.172Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:49:16.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:49:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:49:15.893Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:49:16.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:49:16 smithi153 ceph-mon[70044]: pgmap v13229: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:16.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:49:16 smithi083 ceph-mon[79930]: pgmap v13229: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:16.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:49:16 smithi083 ceph-mon[77242]: pgmap v13229: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:19.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:49:18 smithi153 ceph-mon[70044]: pgmap v13230: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:19.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:49:18 smithi083 ceph-mon[79930]: pgmap v13230: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:19.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:49:18 smithi083 ceph-mon[77242]: pgmap v13230: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:21.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:49:20 smithi153 ceph-mon[70044]: pgmap v13231: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:21.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:49:20 smithi083 ceph-mon[79930]: pgmap v13231: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:21.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:49:20 smithi083 ceph-mon[77242]: pgmap v13231: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:21.804 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:49:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:49:21] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T12:49:22.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:49:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:49:21.801Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:49:22.801 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:49:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:49:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:49:23.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:49:22 smithi153 ceph-mon[70044]: pgmap v13232: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:23.059 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:49:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:49:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:49:23.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:49:22 smithi083 ceph-mon[79930]: pgmap v13232: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:23.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:49:22 smithi083 ceph-mon[77242]: pgmap v13232: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:25.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:49:24 smithi153 ceph-mon[70044]: pgmap v13233: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:25.175 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:49:24 smithi083 ceph-mon[79930]: pgmap v13233: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:25.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:49:24 smithi083 ceph-mon[77242]: pgmap v13233: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:25.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:49:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:49:25.173Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:49:26.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:49:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:49:25.894Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:49:26.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:49:26 smithi153 ceph-mon[70044]: pgmap v13234: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:26.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:49:26 smithi083 ceph-mon[79930]: pgmap v13234: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:26.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:49:26 smithi083 ceph-mon[77242]: pgmap v13234: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:29.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:49:28 smithi153 ceph-mon[70044]: pgmap v13235: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:29.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:49:28 smithi083 ceph-mon[79930]: pgmap v13235: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:29.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:49:28 smithi083 ceph-mon[77242]: pgmap v13235: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:31.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:49:30 smithi153 ceph-mon[70044]: pgmap v13236: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:31.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:49:30 smithi083 ceph-mon[79930]: pgmap v13236: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:31.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:49:30 smithi083 ceph-mon[77242]: pgmap v13236: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:31.804 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:49:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:49:31] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T12:49:32.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:49:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:49:31.802Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:49:32.805 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:49:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:49:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:49:33.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:49:32 smithi153 ceph-mon[70044]: pgmap v13237: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:33.059 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:49:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:49:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:49:33.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:49:32 smithi083 ceph-mon[77242]: pgmap v13237: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:33.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:49:32 smithi083 ceph-mon[79930]: pgmap v13237: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:35.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:49:34 smithi153 ceph-mon[70044]: pgmap v13238: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:35.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:49:34 smithi083 ceph-mon[79930]: pgmap v13238: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:35.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:49:34 smithi083 ceph-mon[77242]: pgmap v13238: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:35.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:49:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:49:35.173Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:49:36.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:49:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:49:35.894Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:49:36.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:49:36 smithi153 ceph-mon[70044]: pgmap v13239: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:36.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:49:36 smithi083 ceph-mon[79930]: pgmap v13239: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:36.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:49:36 smithi083 ceph-mon[77242]: pgmap v13239: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:39.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:49:38 smithi153 ceph-mon[70044]: pgmap v13240: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:39.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:49:38 smithi083 ceph-mon[79930]: pgmap v13240: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:39.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:49:38 smithi083 ceph-mon[77242]: pgmap v13240: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:49:40 smithi153 ceph-mon[70044]: pgmap v13241: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:49:40 smithi083 ceph-mon[77242]: pgmap v13241: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:41.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:49:40 smithi083 ceph-mon[79930]: pgmap v13241: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:41.806 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:49:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:49:41] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T12:49:42.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:49:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:49:41.803Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:49:42.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:49:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:49:42.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:49:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:49:42.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:49:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:49:42.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:49:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:49:42.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:49:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:49:42.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:49:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:49:42.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:49:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:49:42.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:49:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:49:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:49:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:49:42.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:49:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:49:42.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:49:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:49:42.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:49:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:49:42.545 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:49:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:49:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:49:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:49:42.834 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:49:42.834 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 4m ago 7h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:49:42.834 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 2m ago 7h 64.1M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:49:42.835 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 4m ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:49:42.835 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 2m ago 7h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:49:42.835 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 2m ago 7h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:49:42.835 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 4m ago 7h 880M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:49:42.836 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 4m ago 7h 1630M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:49:42.836 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 2m ago 7h 702M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:49:42.836 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 4m ago 7h 943M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:49:42.836 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 4m ago 7h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:49:42.836 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 2m ago 7h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:49:42.837 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 4m ago 7h 1177M 2357M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:49:42.837 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 4m ago 7h 1165M 2357M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:49:42.837 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 4m ago 7h 886M 2357M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:49:42.837 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 4m ago 7h 1140M 2357M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:49:42.838 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 2m ago 7h 979M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:49:42.838 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 2m ago 7h 1254M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:49:42.838 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 2m ago 7h 1121M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:49:42.838 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 2m ago 7h 1331M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:49:42.838 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 2m ago 7h 176M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:49:43.186 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:49:43.187 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:49:43.187 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:49:43.187 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:49:43.187 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:49:43.187 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:49:43.188 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:49:43.188 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:49:43.188 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:49:43.188 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:49:43.188 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:49:43.189 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:49:43.189 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:49:43.189 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:49:43.189 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:49:43.189 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:49:43.190 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:49:43.190 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:49:43.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:49:42 smithi083 ceph-mon[79930]: pgmap v13242: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:43.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:49:42 smithi083 ceph-mon[77242]: pgmap v13242: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:43.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:49:42 smithi153 ceph-mon[70044]: pgmap v13242: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:43.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:49:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:49:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:49:44.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:49:43 smithi083 ceph-mon[77242]: from='client.49530 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:49:44.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:49:43 smithi083 ceph-mon[77242]: from='client.49536 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:49:44.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:49:43 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/437414573' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:49:44.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:49:43 smithi083 ceph-mon[79930]: from='client.49530 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:49:44.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:49:43 smithi083 ceph-mon[79930]: from='client.49536 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:49:44.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:49:43 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/437414573' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:49:44.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:49:43 smithi153 ceph-mon[70044]: from='client.49530 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:49:44.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:49:43 smithi153 ceph-mon[70044]: from='client.49536 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:49:44.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:49:43 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/437414573' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:49:45.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:49:44 smithi083 ceph-mon[79930]: pgmap v13243: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:45.178 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:49:44 smithi083 ceph-mon[77242]: pgmap v13243: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:45.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:49:44 smithi153 ceph-mon[70044]: pgmap v13243: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:45.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:49:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:49:45.174Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:49:46.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:49:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:49:45.895Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:49:46.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:49:46 smithi153 ceph-mon[70044]: pgmap v13244: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:46.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:49:46 smithi083 ceph-mon[79930]: pgmap v13244: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:46.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:49:46 smithi083 ceph-mon[77242]: pgmap v13244: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:49.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:49:48 smithi153 ceph-mon[70044]: pgmap v13245: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:49.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:49:48 smithi083 ceph-mon[77242]: pgmap v13245: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:49.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:49:48 smithi083 ceph-mon[79930]: pgmap v13245: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:51.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:49:50 smithi083 ceph-mon[79930]: pgmap v13246: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:51.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:49:50 smithi083 ceph-mon[77242]: pgmap v13246: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:51.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:49:50 smithi153 ceph-mon[70044]: pgmap v13246: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:51.806 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:49:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:49:51] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T12:49:52.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:49:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:49:51.804Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:49:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:49:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:49:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:49:53.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:49:52 smithi083 ceph-mon[79930]: pgmap v13247: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:53.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:49:52 smithi083 ceph-mon[77242]: pgmap v13247: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:53.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:49:52 smithi153 ceph-mon[70044]: pgmap v13247: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:53.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:49:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:49:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:49:55.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:49:54 smithi083 ceph-mon[79930]: pgmap v13248: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:55.177 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:49:54 smithi083 ceph-mon[77242]: pgmap v13248: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:55.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:49:54 smithi153 ceph-mon[70044]: pgmap v13248: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:49:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:49:55.174Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:49:56.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:49:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:49:55.896Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:49:56.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:49:56 smithi153 ceph-mon[70044]: pgmap v13249: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:56.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:49:56 smithi083 ceph-mon[79930]: pgmap v13249: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:56.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:49:56 smithi083 ceph-mon[77242]: pgmap v13249: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:59.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:49:58 smithi153 ceph-mon[70044]: pgmap v13250: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:59.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:49:58 smithi083 ceph-mon[79930]: pgmap v13250: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:49:59.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:49:58 smithi083 ceph-mon[77242]: pgmap v13250: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:00.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:50:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[77219]: 2022-09-29T12:49:59.999+0000 7f160f665700 -1 log_channel(cluster) log [ERR] : overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T12:50:01.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:50:00 smithi083 ceph-mon[79930]: pgmap v13251: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:01.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:50:00 smithi083 ceph-mon[79930]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T12:50:01.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:50:00 smithi083 ceph-mon[77242]: pgmap v13251: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:01.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:50:00 smithi083 ceph-mon[77242]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T12:50:01.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:50:00 smithi153 ceph-mon[70044]: pgmap v13251: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:01.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:50:00 smithi153 ceph-mon[70044]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T12:50:01.806 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:50:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:50:01] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T12:50:02.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:50:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:50:01.804Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:50:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:50:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:50:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:50:03.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:50:02 smithi083 ceph-mon[79930]: pgmap v13252: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:03.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:50:02 smithi083 ceph-mon[77242]: pgmap v13252: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:03.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:50:02 smithi153 ceph-mon[70044]: pgmap v13252: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:50:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:50:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:50:04.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:50:03 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:50:04.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:50:03 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:50:04.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:50:03 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:50:04.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:50:03 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:50:04.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:50:03 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:50:04.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:50:03 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:50:04.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:50:03 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:50:04.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:50:03 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:50:04.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:50:03 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:50:05.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:50:04 smithi083 ceph-mon[79930]: pgmap v13253: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:05.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:50:04 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:50:05.178 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:50:04 smithi083 ceph-mon[77242]: pgmap v13253: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:05.178 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:50:04 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:50:05.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:50:04 smithi153 ceph-mon[70044]: pgmap v13253: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:05.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:50:04 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:50:05.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:50:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:50:05.174Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:50:06.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:50:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:50:05.896Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:50:06.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:50:06 smithi153 ceph-mon[70044]: pgmap v13254: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:06.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:50:06 smithi083 ceph-mon[79930]: pgmap v13254: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:06.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:50:06 smithi083 ceph-mon[77242]: pgmap v13254: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:09.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:50:08 smithi083 ceph-mon[79930]: pgmap v13255: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:09.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:50:08 smithi083 ceph-mon[77242]: pgmap v13255: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:09.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:50:08 smithi153 ceph-mon[70044]: pgmap v13255: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:11.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:50:10 smithi083 ceph-mon[79930]: pgmap v13256: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:11.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:50:10 smithi083 ceph-mon[77242]: pgmap v13256: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:11.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:50:10 smithi153 ceph-mon[70044]: pgmap v13256: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:11.807 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:50:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:50:11] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T12:50:12.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:50:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:50:11.805Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:50:12.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:50:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:50:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:50:13.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:50:12 smithi083 ceph-mon[79930]: pgmap v13257: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:13.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:50:12 smithi083 ceph-mon[77242]: pgmap v13257: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:13.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:50:12 smithi153 ceph-mon[70044]: pgmap v13257: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:13.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:50:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:50:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:50:13.518 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:50:13.815 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:50:13.815 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 5m ago 7h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:50:13.815 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 3m ago 7h 64.1M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:50:13.816 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 5m ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:50:13.816 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 3m ago 7h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:50:13.816 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 3m ago 7h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:50:13.816 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 5m ago 7h 880M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:50:13.817 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 5m ago 7h 1630M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:50:13.817 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 3m ago 7h 702M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:50:13.817 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 5m ago 7h 943M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:50:13.817 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 5m ago 7h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:50:13.818 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 3m ago 7h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:50:13.818 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 5m ago 7h 1177M 2357M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:50:13.818 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 5m ago 7h 1165M 2357M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:50:13.818 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 5m ago 7h 886M 2357M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:50:13.819 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 5m ago 7h 1140M 2357M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:50:13.819 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 3m ago 7h 979M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:50:13.819 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 3m ago 7h 1254M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:50:13.819 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 3m ago 7h 1121M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:50:13.819 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 3m ago 7h 1331M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:50:13.820 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 3m ago 7h 176M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:50:14.170 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:50:14.170 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:50:14.171 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:50:14.171 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:50:14.171 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:50:14.171 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:50:14.172 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:50:14.172 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:50:14.172 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:50:14.172 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:50:14.173 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:50:14.173 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:50:14.173 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:50:14.173 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:50:14.173 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:50:14.174 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:50:14.174 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:50:14.174 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:50:15.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:50:14 smithi083 ceph-mon[77242]: from='client.49548 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:50:15.177 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:50:14 smithi083 ceph-mon[77242]: pgmap v13258: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:15.177 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:50:14 smithi083 ceph-mon[77242]: from='client.59485 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:50:15.177 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:50:14 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/292438515' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:50:15.178 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:50:14 smithi083 ceph-mon[79930]: from='client.49548 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:50:15.178 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:50:14 smithi083 ceph-mon[79930]: pgmap v13258: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:15.178 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:50:14 smithi083 ceph-mon[79930]: from='client.59485 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:50:15.178 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:50:14 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/292438515' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:50:15.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:50:14 smithi153 ceph-mon[70044]: from='client.49548 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:50:15.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:50:14 smithi153 ceph-mon[70044]: pgmap v13258: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:15.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:50:14 smithi153 ceph-mon[70044]: from='client.59485 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:50:15.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:50:14 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/292438515' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:50:15.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:50:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:50:15.174Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:50:16.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:50:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:50:15.896Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:50:16.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:50:16 smithi153 ceph-mon[70044]: pgmap v13259: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:16.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:50:16 smithi083 ceph-mon[79930]: pgmap v13259: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:16.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:50:16 smithi083 ceph-mon[77242]: pgmap v13259: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:19.067 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:50:18 smithi153 ceph-mon[70044]: pgmap v13260: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:19.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:50:18 smithi083 ceph-mon[79930]: pgmap v13260: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:19.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:50:18 smithi083 ceph-mon[77242]: pgmap v13260: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:21.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:50:20 smithi083 ceph-mon[79930]: pgmap v13261: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:21.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:50:20 smithi083 ceph-mon[77242]: pgmap v13261: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:21.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:50:20 smithi153 ceph-mon[70044]: pgmap v13261: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:21.809 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:50:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:50:21] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T12:50:22.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:50:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:50:21.806Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:50:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:50:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:50:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:50:23.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:50:22 smithi083 ceph-mon[77242]: pgmap v13262: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:23.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:50:22 smithi083 ceph-mon[79930]: pgmap v13262: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:23.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:50:22 smithi153 ceph-mon[70044]: pgmap v13262: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:23.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:50:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:50:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:50:25.178 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:50:24 smithi083 ceph-mon[79930]: pgmap v13263: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:25.179 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:50:24 smithi083 ceph-mon[77242]: pgmap v13263: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:25.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:50:24 smithi153 ceph-mon[70044]: pgmap v13263: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:25.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:50:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:50:25.175Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:50:26.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:50:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:50:25.897Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:50:26.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:50:26 smithi153 ceph-mon[70044]: pgmap v13264: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:26.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:50:26 smithi083 ceph-mon[79930]: pgmap v13264: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:26.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:50:26 smithi083 ceph-mon[77242]: pgmap v13264: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:29.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:50:28 smithi083 ceph-mon[79930]: pgmap v13265: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:29.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:50:28 smithi083 ceph-mon[77242]: pgmap v13265: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:29.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:50:28 smithi153 ceph-mon[70044]: pgmap v13265: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:31.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:50:30 smithi083 ceph-mon[77242]: pgmap v13266: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:31.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:50:30 smithi083 ceph-mon[79930]: pgmap v13266: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:31.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:50:30 smithi153 ceph-mon[70044]: pgmap v13266: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:31.809 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:50:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:50:31] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T12:50:32.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:50:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:50:31.806Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:50:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:50:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:50:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:50:33.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:50:32 smithi083 ceph-mon[79930]: pgmap v13267: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:33.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:50:32 smithi083 ceph-mon[77242]: pgmap v13267: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:33.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:50:32 smithi153 ceph-mon[70044]: pgmap v13267: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:33.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:50:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:50:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:50:35.179 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:50:34 smithi083 ceph-mon[79930]: pgmap v13268: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:35.180 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:50:34 smithi083 ceph-mon[77242]: pgmap v13268: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:35.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:50:34 smithi153 ceph-mon[70044]: pgmap v13268: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:35.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:50:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:50:35.177Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:50:36.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:50:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:50:35.898Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:50:36.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:50:36 smithi153 ceph-mon[70044]: pgmap v13269: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:36.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:50:36 smithi083 ceph-mon[77242]: pgmap v13269: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:36.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:50:36 smithi083 ceph-mon[79930]: pgmap v13269: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:39.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:50:38 smithi083 ceph-mon[77242]: pgmap v13270: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:39.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:50:38 smithi083 ceph-mon[79930]: pgmap v13270: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:39.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:50:38 smithi153 ceph-mon[70044]: pgmap v13270: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:41.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:50:40 smithi083 ceph-mon[77242]: pgmap v13271: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:41.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:50:40 smithi083 ceph-mon[79930]: pgmap v13271: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:50:40 smithi153 ceph-mon[70044]: pgmap v13271: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:41.809 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:50:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:50:41] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T12:50:42.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:50:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:50:41.807Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:50:42.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:50:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:50:42.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:50:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:50:42.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:50:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:50:42.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:50:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:50:42.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:50:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:50:42.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:50:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:50:42.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:50:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:50:42.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:50:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:50:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:50:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:50:42.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:50:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:50:42.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:50:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:50:42.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:50:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:50:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:50:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:50:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:50:43.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:50:42 smithi083 ceph-mon[77242]: pgmap v13272: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:43.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:50:42 smithi083 ceph-mon[79930]: pgmap v13272: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:43.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:50:42 smithi153 ceph-mon[70044]: pgmap v13272: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:43.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:50:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:50:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:50:44.497 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:50:44.785 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:50:44.786 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 5m ago 7h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:50:44.786 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 3m ago 7h 64.1M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:50:44.786 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 5m ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:50:44.786 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 3m ago 7h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:50:44.786 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 3m ago 7h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:50:44.787 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 5m ago 7h 880M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:50:44.787 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 5m ago 7h 1630M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:50:44.787 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 3m ago 7h 702M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:50:44.787 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 5m ago 7h 943M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:50:44.788 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 5m ago 7h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:50:44.788 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 3m ago 7h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:50:44.788 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 5m ago 7h 1177M 2357M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:50:44.788 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 5m ago 7h 1165M 2357M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:50:44.788 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 5m ago 7h 886M 2357M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:50:44.789 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 5m ago 7h 1140M 2357M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:50:44.789 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 3m ago 7h 979M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:50:44.789 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 3m ago 7h 1254M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:50:44.789 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 3m ago 7h 1121M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:50:44.790 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 3m ago 7h 1331M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:50:44.790 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 3m ago 7h 176M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:50:45.146 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:50:45.147 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:50:45.147 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:50:45.147 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:50:45.147 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:50:45.147 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:50:45.148 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:50:45.148 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:50:45.148 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:50:45.148 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:50:45.149 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:50:45.149 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:50:45.149 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:50:45.149 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:50:45.150 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:50:45.150 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:50:45.150 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:50:45.150 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:50:45.178 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:50:44 smithi083 ceph-mon[79930]: pgmap v13273: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:45.178 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:50:44 smithi083 ceph-mon[77242]: pgmap v13273: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:45.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:50:44 smithi153 ceph-mon[70044]: pgmap v13273: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:45.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:50:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:50:45.176Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:50:46.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:50:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:50:45.897Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:50:46.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:50:45 smithi083 ceph-mon[79930]: from='client.49566 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:50:46.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:50:45 smithi083 ceph-mon[79930]: from='client.59503 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:50:46.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:50:45 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3193469740' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:50:46.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:50:45 smithi083 ceph-mon[77242]: from='client.49566 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:50:46.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:50:45 smithi083 ceph-mon[77242]: from='client.59503 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:50:46.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:50:45 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3193469740' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:50:46.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:50:45 smithi153 ceph-mon[70044]: from='client.49566 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:50:46.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:50:45 smithi153 ceph-mon[70044]: from='client.59503 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:50:46.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:50:45 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3193469740' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:50:47.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:50:46 smithi083 ceph-mon[79930]: pgmap v13274: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:47.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:50:46 smithi083 ceph-mon[77242]: pgmap v13274: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:47.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:50:46 smithi153 ceph-mon[70044]: pgmap v13274: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:49.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:50:48 smithi083 ceph-mon[79930]: pgmap v13275: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:49.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:50:48 smithi083 ceph-mon[77242]: pgmap v13275: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:49.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:50:48 smithi153 ceph-mon[70044]: pgmap v13275: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:51.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:50:50 smithi083 ceph-mon[79930]: pgmap v13276: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:51.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:50:50 smithi083 ceph-mon[77242]: pgmap v13276: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:51.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:50:50 smithi153 ceph-mon[70044]: pgmap v13276: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:51.810 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:50:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:50:51] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T12:50:52.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:50:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:50:51.808Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:50:52.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:50:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:50:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:50:53.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:50:52 smithi083 ceph-mon[79930]: pgmap v13277: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:53.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:50:52 smithi083 ceph-mon[77242]: pgmap v13277: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:53.312 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:50:52 smithi153 ceph-mon[70044]: pgmap v13277: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:53.313 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:50:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:50:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:50:55.180 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:50:54 smithi083 ceph-mon[79930]: pgmap v13278: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:55.180 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:50:54 smithi083 ceph-mon[77242]: pgmap v13278: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:55.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:50:54 smithi153 ceph-mon[70044]: pgmap v13278: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:50:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:50:55.177Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:50:56.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:50:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:50:55.898Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:50:56.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:50:56 smithi153 ceph-mon[70044]: pgmap v13279: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:56.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:50:56 smithi083 ceph-mon[79930]: pgmap v13279: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:56.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:50:56 smithi083 ceph-mon[77242]: pgmap v13279: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:59.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:50:58 smithi083 ceph-mon[79930]: pgmap v13280: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:59.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:50:58 smithi083 ceph-mon[77242]: pgmap v13280: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:50:59.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:50:58 smithi153 ceph-mon[70044]: pgmap v13280: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:01.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:51:00 smithi083 ceph-mon[79930]: pgmap v13281: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:01.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:51:00 smithi083 ceph-mon[77242]: pgmap v13281: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:01.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:51:00 smithi153 ceph-mon[70044]: pgmap v13281: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:01.811 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:51:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:51:01] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T12:51:02.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:51:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:51:01.808Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:51:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:51:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:51:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:51:03.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:51:02 smithi083 ceph-mon[79930]: pgmap v13282: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:03.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:51:02 smithi083 ceph-mon[77242]: pgmap v13282: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:03.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:51:02 smithi153 ceph-mon[70044]: pgmap v13282: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:51:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:51:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:51:05.180 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:51:04 smithi083 ceph-mon[79930]: pgmap v13283: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:05.180 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:51:04 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:51:05.181 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:51:04 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:51:05.181 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:51:04 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:51:05.181 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:51:04 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:51:05.181 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:51:04 smithi083 ceph-mon[77242]: pgmap v13283: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:05.182 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:51:04 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:51:05.182 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:51:04 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:51:05.182 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:51:04 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:51:05.182 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:51:04 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:51:05.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:51:04 smithi153 ceph-mon[70044]: pgmap v13283: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:05.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:51:04 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:51:05.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:51:04 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:51:05.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:51:04 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:51:05.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:51:04 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:51:05.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:51:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:51:05.177Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:51:06.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:51:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:51:05.898Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:51:06.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:51:06 smithi153 ceph-mon[70044]: pgmap v13284: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:06.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:51:06 smithi083 ceph-mon[77242]: pgmap v13284: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:06.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:51:06 smithi083 ceph-mon[79930]: pgmap v13284: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:09.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:51:08 smithi083 ceph-mon[77242]: pgmap v13285: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:09.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:51:08 smithi083 ceph-mon[79930]: pgmap v13285: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:09.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:51:08 smithi153 ceph-mon[70044]: pgmap v13285: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:11.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:51:10 smithi083 ceph-mon[77242]: pgmap v13286: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:11.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:51:10 smithi083 ceph-mon[79930]: pgmap v13286: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:11.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:51:10 smithi153 ceph-mon[70044]: pgmap v13286: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:11.811 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:51:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:51:11] "GET /metrics HTTP/1.1" 200 225229 "" "Prometheus/2.33.4" 2022-09-29T12:51:12.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:51:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:51:11.809Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:51:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:51:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:51:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:51:13.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:51:12 smithi083 ceph-mon[79930]: pgmap v13287: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:13.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:51:12 smithi083 ceph-mon[77242]: pgmap v13287: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:13.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:51:12 smithi153 ceph-mon[70044]: pgmap v13287: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:13.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:51:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:51:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:51:15.181 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:51:14 smithi083 ceph-mon[79930]: pgmap v13288: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:15.182 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:51:14 smithi083 ceph-mon[77242]: pgmap v13288: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:15.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:51:14 smithi153 ceph-mon[70044]: pgmap v13288: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:15.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:51:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:51:15.179Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:51:15.476 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:51:15.763 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:51:15.764 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 6m ago 7h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:51:15.764 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 4m ago 7h 64.1M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:51:15.764 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 6m ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:51:15.764 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 4m ago 7h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:51:15.765 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 4m ago 7h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:51:15.765 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 6m ago 7h 880M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:51:15.765 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 6m ago 7h 1630M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:51:15.765 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 4m ago 7h 702M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:51:15.766 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 6m ago 7h 943M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:51:15.766 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 6m ago 7h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:51:15.766 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 4m ago 7h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:51:15.766 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 6m ago 7h 1177M 2357M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:51:15.767 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 6m ago 7h 1165M 2357M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:51:15.767 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 6m ago 7h 886M 2357M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:51:15.767 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 6m ago 7h 1140M 2357M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:51:15.767 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 4m ago 7h 979M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:51:15.767 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 4m ago 7h 1254M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:51:15.768 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 4m ago 7h 1121M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:51:15.768 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 4m ago 7h 1331M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:51:15.768 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 4m ago 7h 176M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:51:16.116 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:51:16.117 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:51:16.117 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:51:16.117 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:51:16.118 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:51:16.118 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:51:16.118 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:51:16.118 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:51:16.119 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:51:16.119 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:51:16.119 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:51:16.119 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:51:16.119 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:51:16.120 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:51:16.120 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:51:16.120 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:51:16.120 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:51:16.121 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:51:16.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:51:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:51:15.898Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:51:16.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:51:16 smithi153 ceph-mon[70044]: from='client.49584 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:51:16.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:51:16 smithi153 ceph-mon[70044]: from='client.49590 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:51:16.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:51:16 smithi153 ceph-mon[70044]: pgmap v13289: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:16.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:51:16 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/111723759' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:51:16.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:51:16 smithi083 ceph-mon[79930]: from='client.49584 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:51:16.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:51:16 smithi083 ceph-mon[79930]: from='client.49590 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:51:16.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:51:16 smithi083 ceph-mon[79930]: pgmap v13289: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:16.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:51:16 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/111723759' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:51:16.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:51:16 smithi083 ceph-mon[77242]: from='client.49584 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:51:16.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:51:16 smithi083 ceph-mon[77242]: from='client.49590 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:51:16.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:51:16 smithi083 ceph-mon[77242]: pgmap v13289: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:16.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:51:16 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/111723759' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:51:19.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:51:18 smithi083 ceph-mon[79930]: pgmap v13290: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:19.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:51:18 smithi083 ceph-mon[77242]: pgmap v13290: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:19.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:51:18 smithi153 ceph-mon[70044]: pgmap v13290: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:21.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:51:20 smithi083 ceph-mon[79930]: pgmap v13291: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:21.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:51:20 smithi083 ceph-mon[77242]: pgmap v13291: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:21.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:51:20 smithi153 ceph-mon[70044]: pgmap v13291: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:21.812 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:51:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:51:21] "GET /metrics HTTP/1.1" 200 225229 "" "Prometheus/2.33.4" 2022-09-29T12:51:22.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:51:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:51:21.809Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:51:22.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:51:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:51:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:51:23.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:51:22 smithi083 ceph-mon[79930]: pgmap v13292: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:23.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:51:22 smithi083 ceph-mon[77242]: pgmap v13292: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:23.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:51:22 smithi153 ceph-mon[70044]: pgmap v13292: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:23.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:51:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:51:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:51:25.182 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:51:24 smithi083 ceph-mon[79930]: pgmap v13293: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:25.183 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:51:24 smithi083 ceph-mon[77242]: pgmap v13293: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:25.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:51:24 smithi153 ceph-mon[70044]: pgmap v13293: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:25.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:51:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:51:25.180Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:51:26.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:51:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:51:25.899Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:51:26.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:51:26 smithi153 ceph-mon[70044]: pgmap v13294: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:26.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:51:26 smithi083 ceph-mon[79930]: pgmap v13294: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:26.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:51:26 smithi083 ceph-mon[77242]: pgmap v13294: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:29.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:51:28 smithi083 ceph-mon[79930]: pgmap v13295: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:29.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:51:28 smithi083 ceph-mon[77242]: pgmap v13295: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:29.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:51:28 smithi153 ceph-mon[70044]: pgmap v13295: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:31.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:51:30 smithi083 ceph-mon[77242]: pgmap v13296: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:31.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:51:30 smithi083 ceph-mon[79930]: pgmap v13296: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:31.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:51:30 smithi153 ceph-mon[70044]: pgmap v13296: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:31.813 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:51:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:51:31] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T12:51:32.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:51:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:51:31.811Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:51:32.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:51:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:51:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:51:33.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:51:32 smithi083 ceph-mon[79930]: pgmap v13297: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:33.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:51:32 smithi083 ceph-mon[77242]: pgmap v13297: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:33.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:51:32 smithi153 ceph-mon[70044]: pgmap v13297: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:33.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:51:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:51:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:51:35.183 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:51:34 smithi083 ceph-mon[79930]: pgmap v13298: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:35.184 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:51:34 smithi083 ceph-mon[77242]: pgmap v13298: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:35.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:51:34 smithi153 ceph-mon[70044]: pgmap v13298: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:35.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:51:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:51:35.181Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:51:36.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:51:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:51:35.900Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:51:36.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:51:36 smithi153 ceph-mon[70044]: pgmap v13299: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:36.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:51:36 smithi083 ceph-mon[79930]: pgmap v13299: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:36.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:51:36 smithi083 ceph-mon[77242]: pgmap v13299: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:39.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:51:38 smithi083 ceph-mon[79930]: pgmap v13300: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:39.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:51:38 smithi083 ceph-mon[77242]: pgmap v13300: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:39.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:51:38 smithi153 ceph-mon[70044]: pgmap v13300: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:41.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:51:40 smithi083 ceph-mon[79930]: pgmap v13301: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:51:40 smithi083 ceph-mon[77242]: pgmap v13301: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:51:40 smithi153 ceph-mon[70044]: pgmap v13301: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:41.814 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:51:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:51:41] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T12:51:42.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:51:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:51:41.811Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:51:42.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:51:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:51:42.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:51:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:51:42.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:51:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:51:42.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:51:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:51:42.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:51:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:51:42.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:51:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:51:42.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:51:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:51:42.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:51:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:51:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:51:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:51:42.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:51:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:51:42.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:51:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:51:42.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:51:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:51:42.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:51:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:51:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:51:43.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:51:42 smithi083 ceph-mon[79930]: pgmap v13302: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:43.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:51:42 smithi083 ceph-mon[77242]: pgmap v13302: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:43.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:51:42 smithi153 ceph-mon[70044]: pgmap v13302: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:43.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:51:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:51:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:51:45.184 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:51:44 smithi083 ceph-mon[79930]: pgmap v13303: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:45.185 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:51:44 smithi083 ceph-mon[77242]: pgmap v13303: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:45.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:51:44 smithi153 ceph-mon[70044]: pgmap v13303: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:45.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:51:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:51:45.181Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:51:46.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:51:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:51:45.901Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:51:46.443 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:51:46.736 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:51:46.737 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 6m ago 7h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:51:46.737 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 4m ago 7h 64.1M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:51:46.737 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 6m ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:51:46.738 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 4m ago 7h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:51:46.738 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 4m ago 7h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:51:46.738 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 6m ago 7h 880M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:51:46.738 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 6m ago 7h 1630M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:51:46.739 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 4m ago 7h 702M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:51:46.739 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 6m ago 7h 943M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:51:46.739 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 6m ago 7h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:51:46.739 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 4m ago 7h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:51:46.740 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 6m ago 7h 1177M 2357M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:51:46.740 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 6m ago 7h 1165M 2357M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:51:46.740 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 6m ago 7h 886M 2357M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:51:46.740 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 6m ago 7h 1140M 2357M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:51:46.741 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 4m ago 7h 979M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:51:46.741 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 4m ago 7h 1254M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:51:46.741 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 4m ago 7h 1121M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:51:46.741 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 4m ago 7h 1331M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:51:46.742 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 4m ago 7h 176M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:51:46.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:51:46 smithi153 ceph-mon[70044]: pgmap v13304: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:46.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:51:46 smithi083 ceph-mon[79930]: pgmap v13304: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:46.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:51:46 smithi083 ceph-mon[77242]: pgmap v13304: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:47.091 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:51:47.091 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:51:47.091 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:51:47.092 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:51:47.092 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:51:47.092 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:51:47.092 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:51:47.093 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:51:47.093 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:51:47.093 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:51:47.093 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:51:47.093 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:51:47.094 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:51:47.094 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:51:47.094 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:51:47.094 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:51:47.095 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:51:47.095 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:51:47.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:51:47 smithi153 ceph-mon[70044]: from='client.49602 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:51:47.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:51:47 smithi153 ceph-mon[70044]: from='client.49608 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:51:47.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:51:47 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3302077373' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:51:47.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:51:47 smithi083 ceph-mon[79930]: from='client.49602 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:51:47.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:51:47 smithi083 ceph-mon[79930]: from='client.49608 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:51:47.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:51:47 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3302077373' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:51:47.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:51:47 smithi083 ceph-mon[77242]: from='client.49602 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:51:47.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:51:47 smithi083 ceph-mon[77242]: from='client.49608 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:51:47.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:51:47 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3302077373' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:51:48.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:51:48 smithi153 ceph-mon[70044]: pgmap v13305: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:48.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:51:48 smithi083 ceph-mon[79930]: pgmap v13305: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:48.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:51:48 smithi083 ceph-mon[77242]: pgmap v13305: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:51.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:51:50 smithi083 ceph-mon[79930]: pgmap v13306: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:51.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:51:50 smithi083 ceph-mon[77242]: pgmap v13306: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:51.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:51:50 smithi153 ceph-mon[70044]: pgmap v13306: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:51.815 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:51:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:51:51] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T12:51:52.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:51:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:51:51.812Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:51:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:51:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:51:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:51:53.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:51:52 smithi083 ceph-mon[79930]: pgmap v13307: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:53.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:51:52 smithi083 ceph-mon[77242]: pgmap v13307: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:53.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:51:52 smithi153 ceph-mon[70044]: pgmap v13307: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:53.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:51:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:51:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:51:55.185 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:51:54 smithi083 ceph-mon[79930]: pgmap v13308: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:55.185 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:51:54 smithi083 ceph-mon[77242]: pgmap v13308: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:55.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:51:54 smithi153 ceph-mon[70044]: pgmap v13308: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:51:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:51:55.182Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:51:56.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:51:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:51:55.901Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:51:56.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:51:56 smithi153 ceph-mon[70044]: pgmap v13309: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:56.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:51:56 smithi083 ceph-mon[77242]: pgmap v13309: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:56.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:51:56 smithi083 ceph-mon[79930]: pgmap v13309: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:59.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:51:58 smithi083 ceph-mon[79930]: pgmap v13310: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:59.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:51:58 smithi083 ceph-mon[77242]: pgmap v13310: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:51:59.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:51:58 smithi153 ceph-mon[70044]: pgmap v13310: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:01.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:52:00 smithi083 ceph-mon[79930]: pgmap v13311: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:01.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:52:00 smithi083 ceph-mon[77242]: pgmap v13311: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:01.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:52:00 smithi153 ceph-mon[70044]: pgmap v13311: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:01.815 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:52:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:52:01] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T12:52:02.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:52:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:52:01.812Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:52:02.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:52:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:52:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:52:03.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:52:02 smithi083 ceph-mon[79930]: pgmap v13312: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:03.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:52:02 smithi083 ceph-mon[77242]: pgmap v13312: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:03.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:52:02 smithi153 ceph-mon[70044]: pgmap v13312: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:52:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:52:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:52:05.184 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:52:04 smithi083 ceph-mon[79930]: pgmap v13313: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:05.184 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:52:04 smithi083 ceph-mon[77242]: pgmap v13313: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:05.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:52:04 smithi153 ceph-mon[70044]: pgmap v13313: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:05.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:52:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:52:05.181Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:52:06.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:52:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:52:05.901Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:52:06.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:52:05 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:52:06.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:52:05 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:52:06.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:52:05 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:52:06.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:52:05 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:52:06.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:52:05 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:52:06.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:52:05 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:52:06.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:52:05 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:52:06.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:52:05 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:52:06.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:52:05 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:52:06.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:52:05 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:52:06.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:52:05 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:52:06.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:52:05 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:52:07.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:52:06 smithi083 ceph-mon[77242]: pgmap v13314: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:07.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:52:06 smithi083 ceph-mon[79930]: pgmap v13314: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:07.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:52:06 smithi153 ceph-mon[70044]: pgmap v13314: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:09.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:52:08 smithi083 ceph-mon[79930]: pgmap v13315: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:09.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:52:08 smithi083 ceph-mon[77242]: pgmap v13315: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:09.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:52:08 smithi153 ceph-mon[70044]: pgmap v13315: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:11.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:52:10 smithi083 ceph-mon[79930]: pgmap v13316: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:11.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:52:10 smithi083 ceph-mon[77242]: pgmap v13316: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:11.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:52:10 smithi153 ceph-mon[70044]: pgmap v13316: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:11.816 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:52:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:52:11] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T12:52:12.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:52:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:52:11.813Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:52:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:52:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:52:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:52:13.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:52:12 smithi083 ceph-mon[79930]: pgmap v13317: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:13.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:52:12 smithi083 ceph-mon[77242]: pgmap v13317: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:13.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:52:12 smithi153 ceph-mon[70044]: pgmap v13317: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:13.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:52:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:52:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:52:15.185 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:52:14 smithi083 ceph-mon[79930]: pgmap v13318: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:15.186 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:52:14 smithi083 ceph-mon[77242]: pgmap v13318: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:15.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:52:14 smithi153 ceph-mon[70044]: pgmap v13318: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:15.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:52:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:52:15.183Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:52:16.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:52:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:52:15.902Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:52:16.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:52:16 smithi153 ceph-mon[70044]: pgmap v13319: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:16.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:52:16 smithi083 ceph-mon[79930]: pgmap v13319: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:16.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:52:16 smithi083 ceph-mon[77242]: pgmap v13319: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:17.418 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:52:17.707 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:52:17.708 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 7m ago 7h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:52:17.708 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 5m ago 7h 64.1M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:52:17.708 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 7m ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:52:17.709 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 5m ago 7h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:52:17.709 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 5m ago 7h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:52:17.709 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 7m ago 7h 880M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:52:17.709 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 7m ago 7h 1630M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:52:17.710 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 5m ago 7h 702M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:52:17.710 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 7m ago 7h 943M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:52:17.710 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 7m ago 7h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:52:17.710 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 5m ago 7h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:52:17.710 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 7m ago 7h 1177M 2357M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:52:17.711 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 7m ago 7h 1165M 2357M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:52:17.711 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 7m ago 7h 886M 2357M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:52:17.711 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 7m ago 7h 1140M 2357M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:52:17.711 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 5m ago 7h 979M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:52:17.712 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 5m ago 7h 1254M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:52:17.712 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 5m ago 7h 1121M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:52:17.712 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 5m ago 7h 1331M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:52:17.712 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 5m ago 7h 176M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:52:18.065 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:52:18.065 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:52:18.065 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:52:18.066 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:52:18.066 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:52:18.066 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:52:18.066 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:52:18.067 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:52:18.067 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:52:18.067 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:52:18.067 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:52:18.068 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:52:18.068 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:52:18.068 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:52:18.068 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:52:18.069 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:52:18.069 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:52:18.069 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:52:19.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:52:18 smithi083 ceph-mon[79930]: from='client.59551 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:52:19.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:52:18 smithi083 ceph-mon[79930]: from='client.59557 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:52:19.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:52:18 smithi083 ceph-mon[79930]: pgmap v13320: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:19.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:52:18 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3976864797' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:52:19.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:52:18 smithi083 ceph-mon[77242]: from='client.59551 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:52:19.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:52:18 smithi083 ceph-mon[77242]: from='client.59557 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:52:19.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:52:18 smithi083 ceph-mon[77242]: pgmap v13320: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:19.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:52:18 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3976864797' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:52:19.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:52:18 smithi153 ceph-mon[70044]: from='client.59551 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:52:19.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:52:18 smithi153 ceph-mon[70044]: from='client.59557 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:52:19.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:52:18 smithi153 ceph-mon[70044]: pgmap v13320: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:19.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:52:18 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3976864797' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:52:21.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:52:20 smithi083 ceph-mon[79930]: pgmap v13321: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:21.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:52:20 smithi083 ceph-mon[77242]: pgmap v13321: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:21.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:52:20 smithi153 ceph-mon[70044]: pgmap v13321: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:21.816 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:52:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:52:21] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T12:52:22.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:52:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:52:21.814Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:52:22.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:52:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:52:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:52:23.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:52:22 smithi083 ceph-mon[79930]: pgmap v13322: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:23.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:52:22 smithi083 ceph-mon[77242]: pgmap v13322: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:23.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:52:22 smithi153 ceph-mon[70044]: pgmap v13322: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:23.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:52:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:52:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:52:25.186 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:52:24 smithi083 ceph-mon[79930]: pgmap v13323: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:25.186 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:52:24 smithi083 ceph-mon[77242]: pgmap v13323: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:25.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:52:24 smithi153 ceph-mon[70044]: pgmap v13323: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:25.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:52:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:52:25.183Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:52:26.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:52:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:52:25.902Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:52:26.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:52:26 smithi153 ceph-mon[70044]: pgmap v13324: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:26.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:52:26 smithi083 ceph-mon[77242]: pgmap v13324: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:26.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:52:26 smithi083 ceph-mon[79930]: pgmap v13324: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:29.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:52:28 smithi083 ceph-mon[77242]: pgmap v13325: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:29.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:52:28 smithi083 ceph-mon[79930]: pgmap v13325: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:29.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:52:28 smithi153 ceph-mon[70044]: pgmap v13325: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:31.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:52:30 smithi083 ceph-mon[77242]: pgmap v13326: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:31.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:52:30 smithi083 ceph-mon[79930]: pgmap v13326: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:31.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:52:30 smithi153 ceph-mon[70044]: pgmap v13326: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:31.817 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:52:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:52:31] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T12:52:32.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:52:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:52:31.814Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:52:32.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:52:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:52:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:52:33.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:52:32 smithi083 ceph-mon[77242]: pgmap v13327: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:33.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:52:32 smithi083 ceph-mon[79930]: pgmap v13327: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:33.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:52:32 smithi153 ceph-mon[70044]: pgmap v13327: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:33.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:52:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:52:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:52:35.187 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:52:34 smithi083 ceph-mon[79930]: pgmap v13328: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:35.187 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:52:34 smithi083 ceph-mon[77242]: pgmap v13328: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:35.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:52:34 smithi153 ceph-mon[70044]: pgmap v13328: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:35.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:52:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:52:35.184Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:52:36.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:52:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:52:35.902Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:52:36.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:52:36 smithi153 ceph-mon[70044]: pgmap v13329: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:36.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:52:36 smithi083 ceph-mon[77242]: pgmap v13329: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:36.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:52:36 smithi083 ceph-mon[79930]: pgmap v13329: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:39.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:52:38 smithi083 ceph-mon[79930]: pgmap v13330: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:39.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:52:38 smithi083 ceph-mon[77242]: pgmap v13330: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:39.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:52:38 smithi153 ceph-mon[70044]: pgmap v13330: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:41.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:52:40 smithi083 ceph-mon[79930]: pgmap v13331: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:52:40 smithi083 ceph-mon[77242]: pgmap v13331: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:52:40 smithi153 ceph-mon[70044]: pgmap v13331: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:41.818 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:52:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:52:41] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T12:52:42.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:52:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:52:41.815Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:52:42.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:52:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:52:42.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:52:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:52:42.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:52:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:52:42.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:52:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:52:42.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:52:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:52:42.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:52:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:52:42.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:52:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:52:42.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:52:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:52:42.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:52:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:52:42.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:52:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:52:42.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:52:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:52:42.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:52:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:52:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:52:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:52:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:52:43.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:52:42 smithi083 ceph-mon[79930]: pgmap v13332: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:43.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:52:42 smithi083 ceph-mon[77242]: pgmap v13332: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:43.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:52:42 smithi153 ceph-mon[70044]: pgmap v13332: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:43.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:52:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:52:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:52:45.187 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:52:44 smithi083 ceph-mon[77242]: pgmap v13333: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:45.188 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:52:44 smithi083 ceph-mon[79930]: pgmap v13333: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:45.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:52:44 smithi153 ceph-mon[70044]: pgmap v13333: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:45.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:52:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:52:45.185Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:52:46.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:52:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:52:45.903Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:52:46.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:52:46 smithi153 ceph-mon[70044]: pgmap v13334: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:46.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:52:46 smithi083 ceph-mon[79930]: pgmap v13334: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:46.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:52:46 smithi083 ceph-mon[77242]: pgmap v13334: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:48.394 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:52:48.681 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:52:48.682 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 7m ago 7h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:52:48.682 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 5m ago 7h 64.1M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:52:48.682 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 7m ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:52:48.683 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 5m ago 7h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:52:48.683 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 5m ago 7h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:52:48.683 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 7m ago 7h 880M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:52:48.683 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 7m ago 7h 1630M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:52:48.684 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 5m ago 7h 702M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:52:48.684 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 7m ago 7h 943M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:52:48.684 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 7m ago 7h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:52:48.684 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 5m ago 7h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:52:48.684 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 7m ago 7h 1177M 2357M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:52:48.685 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 7m ago 7h 1165M 2357M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:52:48.685 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 7m ago 7h 886M 2357M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:52:48.685 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 7m ago 7h 1140M 2357M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:52:48.685 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 5m ago 7h 979M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:52:48.686 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 5m ago 7h 1254M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:52:48.686 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 5m ago 7h 1121M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:52:48.686 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 5m ago 7h 1331M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:52:48.687 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 5m ago 7h 176M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:52:49.039 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:52:49.040 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:52:49.040 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:52:49.040 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:52:49.041 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:52:49.041 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:52:49.041 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:52:49.041 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:52:49.042 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:52:49.042 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:52:49.042 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:52:49.042 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:52:49.043 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:52:49.043 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:52:49.043 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:52:49.043 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:52:49.044 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:52:49.044 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:52:49.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:52:48 smithi083 ceph-mon[79930]: pgmap v13335: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:49.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:52:48 smithi083 ceph-mon[77242]: pgmap v13335: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:49.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:52:48 smithi153 ceph-mon[70044]: pgmap v13335: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:50.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:52:49 smithi083 ceph-mon[77242]: from='client.59569 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:52:50.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:52:49 smithi083 ceph-mon[77242]: from='client.59575 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:52:50.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:52:49 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3741488527' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:52:50.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:52:49 smithi083 ceph-mon[79930]: from='client.59569 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:52:50.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:52:49 smithi083 ceph-mon[79930]: from='client.59575 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:52:50.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:52:49 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3741488527' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:52:50.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:52:49 smithi153 ceph-mon[70044]: from='client.59569 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:52:50.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:52:49 smithi153 ceph-mon[70044]: from='client.59575 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:52:50.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:52:49 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3741488527' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:52:51.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:52:50 smithi083 ceph-mon[77242]: pgmap v13336: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:51.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:52:50 smithi083 ceph-mon[79930]: pgmap v13336: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:51.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:52:50 smithi153 ceph-mon[70044]: pgmap v13336: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:51.819 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:52:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:52:51] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T12:52:52.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:52:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:52:51.816Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:52:52.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:52:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:52:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:52:53.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:52:52 smithi083 ceph-mon[79930]: pgmap v13337: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:53.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:52:52 smithi083 ceph-mon[77242]: pgmap v13337: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:53.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:52:52 smithi153 ceph-mon[70044]: pgmap v13337: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:53.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:52:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:52:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:52:55.189 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:52:54 smithi083 ceph-mon[79930]: pgmap v13338: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:55.190 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:52:54 smithi083 ceph-mon[77242]: pgmap v13338: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:55.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:52:54 smithi153 ceph-mon[70044]: pgmap v13338: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:52:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:52:55.186Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:52:56.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:52:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:52:55.903Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:52:56.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:52:56 smithi153 ceph-mon[70044]: pgmap v13339: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:56.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:52:56 smithi083 ceph-mon[77242]: pgmap v13339: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:56.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:52:56 smithi083 ceph-mon[79930]: pgmap v13339: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:59.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:52:58 smithi083 ceph-mon[79930]: pgmap v13340: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:59.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:52:58 smithi083 ceph-mon[77242]: pgmap v13340: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:52:59.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:52:58 smithi153 ceph-mon[70044]: pgmap v13340: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:01.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:53:00 smithi083 ceph-mon[79930]: pgmap v13341: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:01.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:53:00 smithi083 ceph-mon[77242]: pgmap v13341: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:01.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:53:00 smithi153 ceph-mon[70044]: pgmap v13341: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:01.820 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:53:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:53:01] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T12:53:02.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:53:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:53:01.817Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:53:02.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:53:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:53:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:53:03.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:53:02 smithi083 ceph-mon[77242]: pgmap v13342: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:03.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:53:02 smithi083 ceph-mon[79930]: pgmap v13342: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:03.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:53:02 smithi153 ceph-mon[70044]: pgmap v13342: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:53:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:53:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:53:05.189 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:53:04 smithi083 ceph-mon[79930]: pgmap v13343: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:05.189 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:53:04 smithi083 ceph-mon[77242]: pgmap v13343: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:05.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:53:04 smithi153 ceph-mon[70044]: pgmap v13343: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:05.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:53:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:53:05.186Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:53:06.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:53:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:53:05.903Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:53:06.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:53:05 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:53:06.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:53:05 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:53:06.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:53:05 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:53:06.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:53:05 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:53:06.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:53:05 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:53:06.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:53:05 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:53:06.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:53:05 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:53:06.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:53:05 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:53:06.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:53:05 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:53:07.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:53:06 smithi083 ceph-mon[77242]: pgmap v13344: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:07.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:53:06 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:53:07.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:53:06 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:53:07.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:53:06 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:53:07.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:53:06 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:53:07.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:53:06 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:53:07.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:53:06 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:53:07.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:53:06 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:53:07.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:53:06 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:53:07.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:53:06 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:53:07.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:53:06 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:53:07.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:53:06 smithi083 ceph-mon[79930]: pgmap v13344: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:07.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:53:06 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:53:07.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:53:06 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:53:07.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:53:06 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:53:07.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:53:06 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:53:07.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:53:06 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:53:07.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:53:06 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:53:07.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:53:06 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:53:07.218 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:53:06 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:53:07.218 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:53:06 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:53:07.218 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:53:06 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:53:07.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:53:06 smithi153 ceph-mon[70044]: pgmap v13344: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:07.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:53:06 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:53:07.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:53:06 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:53:07.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:53:06 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:53:07.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:53:06 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:53:07.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:53:06 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:53:07.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:53:06 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:53:07.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:53:06 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:53:07.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:53:06 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T12:53:07.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:53:06 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:53:07.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:53:06 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:53:08.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:53:07 smithi083 ceph-mon[79930]: Adjusting osd_memory_target on smithi083 to 2348M 2022-09-29T12:53:08.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:53:07 smithi083 ceph-mon[77242]: Adjusting osd_memory_target on smithi083 to 2348M 2022-09-29T12:53:08.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:53:07 smithi153 ceph-mon[70044]: Adjusting osd_memory_target on smithi083 to 2348M 2022-09-29T12:53:09.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:53:08 smithi083 ceph-mon[79930]: pgmap v13345: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:09.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:53:08 smithi083 ceph-mon[77242]: pgmap v13345: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:09.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:53:08 smithi153 ceph-mon[70044]: pgmap v13345: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:11.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:53:10 smithi083 ceph-mon[79930]: pgmap v13346: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:11.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:53:10 smithi083 ceph-mon[77242]: pgmap v13346: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:11.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:53:10 smithi153 ceph-mon[70044]: pgmap v13346: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:11.820 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:53:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:53:11] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T12:53:12.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:53:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:53:11.817Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:53:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:53:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:53:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:53:13.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:53:12 smithi083 ceph-mon[77242]: pgmap v13347: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:13.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:53:12 smithi083 ceph-mon[79930]: pgmap v13347: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:13.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:53:12 smithi153 ceph-mon[70044]: pgmap v13347: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:13.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:53:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:53:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:53:15.189 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:53:14 smithi083 ceph-mon[79930]: pgmap v13348: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:15.190 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:53:14 smithi083 ceph-mon[77242]: pgmap v13348: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:15.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:53:14 smithi153 ceph-mon[70044]: pgmap v13348: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:15.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:53:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:53:15.186Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:53:16.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:53:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:53:15.905Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:53:16.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:53:16 smithi083 ceph-mon[79930]: pgmap v13349: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:16.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:53:16 smithi083 ceph-mon[77242]: pgmap v13349: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:17.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:53:16 smithi153 ceph-mon[70044]: pgmap v13349: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:19.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:53:18 smithi083 ceph-mon[77242]: pgmap v13350: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:19.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:53:18 smithi083 ceph-mon[79930]: pgmap v13350: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:19.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:53:18 smithi153 ceph-mon[70044]: pgmap v13350: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:19.365 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:53:19.655 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:53:19.656 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 8m ago 7h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:53:19.656 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 6m ago 7h 64.1M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:53:19.656 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 8m ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:53:19.656 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 6m ago 7h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:53:19.657 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 6m ago 7h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:53:19.657 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 8m ago 7h 880M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:53:19.657 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 8m ago 7h 1630M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:53:19.657 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 6m ago 7h 702M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:53:19.657 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 8m ago 7h 943M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:53:19.658 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 8m ago 7h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:53:19.658 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 6m ago 7h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:53:19.658 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 8m ago 7h 1177M 2348M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:53:19.658 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 8m ago 7h 1165M 2348M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:53:19.659 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 8m ago 7h 886M 2348M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:53:19.659 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 8m ago 7h 1140M 2348M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:53:19.659 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 6m ago 7h 979M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:53:19.659 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 6m ago 7h 1254M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:53:19.660 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 6m ago 7h 1121M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:53:19.660 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 6m ago 7h 1331M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:53:19.660 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 6m ago 7h 176M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:53:20.012 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:53:20.012 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:53:20.012 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:53:20.013 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:53:20.013 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:53:20.013 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:53:20.014 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:53:20.014 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:53:20.014 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:53:20.014 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:53:20.015 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:53:20.015 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:53:20.015 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:53:20.015 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:53:20.016 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:53:20.016 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:53:20.016 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:53:20.016 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:53:21.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:53:20 smithi083 ceph-mon[79930]: from='client.59587 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:53:21.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:53:20 smithi083 ceph-mon[79930]: from='client.59593 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:53:21.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:53:20 smithi083 ceph-mon[79930]: pgmap v13351: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:21.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:53:20 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1995073142' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:53:21.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:53:20 smithi083 ceph-mon[77242]: from='client.59587 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:53:21.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:53:20 smithi083 ceph-mon[77242]: from='client.59593 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:53:21.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:53:20 smithi083 ceph-mon[77242]: pgmap v13351: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:21.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:53:20 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1995073142' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:53:21.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:53:20 smithi153 ceph-mon[70044]: from='client.59587 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:53:21.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:53:20 smithi153 ceph-mon[70044]: from='client.59593 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:53:21.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:53:20 smithi153 ceph-mon[70044]: pgmap v13351: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:21.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:53:20 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1995073142' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:53:21.821 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:53:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:53:21] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T12:53:22.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:53:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:53:21.818Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:53:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:53:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:53:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:53:23.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:53:22 smithi083 ceph-mon[77242]: pgmap v13352: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:23.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:53:22 smithi083 ceph-mon[79930]: pgmap v13352: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:53:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:53:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:53:23.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:53:22 smithi153 ceph-mon[70044]: pgmap v13352: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:25.190 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:53:24 smithi083 ceph-mon[79930]: pgmap v13353: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:25.190 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:53:24 smithi083 ceph-mon[77242]: pgmap v13353: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:25.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:53:24 smithi153 ceph-mon[70044]: pgmap v13353: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:25.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:53:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:53:25.187Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:53:26.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:53:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:53:25.905Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:53:26.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:53:26 smithi083 ceph-mon[77242]: pgmap v13354: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:26.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:53:26 smithi083 ceph-mon[79930]: pgmap v13354: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:27.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:53:26 smithi153 ceph-mon[70044]: pgmap v13354: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:29.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:53:28 smithi083 ceph-mon[79930]: pgmap v13355: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:29.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:53:28 smithi083 ceph-mon[77242]: pgmap v13355: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:29.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:53:28 smithi153 ceph-mon[70044]: pgmap v13355: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:31.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:53:30 smithi083 ceph-mon[79930]: pgmap v13356: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:31.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:53:30 smithi083 ceph-mon[77242]: pgmap v13356: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:31.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:53:30 smithi153 ceph-mon[70044]: pgmap v13356: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:31.822 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:53:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:53:31] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T12:53:32.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:53:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:53:31.819Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:53:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:53:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:53:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:53:33.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:53:32 smithi083 ceph-mon[79930]: pgmap v13357: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:33.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:53:32 smithi083 ceph-mon[77242]: pgmap v13357: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:33.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:53:32 smithi153 ceph-mon[70044]: pgmap v13357: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:33.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:53:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:53:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:53:35.191 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:53:34 smithi083 ceph-mon[79930]: pgmap v13358: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:35.192 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:53:34 smithi083 ceph-mon[77242]: pgmap v13358: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:35.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:53:34 smithi153 ceph-mon[70044]: pgmap v13358: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:35.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:53:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:53:35.189Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:53:36.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:53:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:53:35.906Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:53:36.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:53:36 smithi083 ceph-mon[79930]: pgmap v13359: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:36.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:53:36 smithi083 ceph-mon[77242]: pgmap v13359: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:37.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:53:36 smithi153 ceph-mon[70044]: pgmap v13359: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:39.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:53:38 smithi083 ceph-mon[79930]: pgmap v13360: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:39.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:53:38 smithi083 ceph-mon[77242]: pgmap v13360: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:39.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:53:38 smithi153 ceph-mon[70044]: pgmap v13360: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:41.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:53:40 smithi083 ceph-mon[79930]: pgmap v13361: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:41.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:53:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:53:41.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:53:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:53:41.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:53:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:53:41.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:53:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:53:41.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:53:40 smithi083 ceph-mon[77242]: pgmap v13361: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:41.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:53:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:53:41.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:53:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:53:41.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:53:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:53:41.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:53:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:53:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:53:40 smithi153 ceph-mon[70044]: pgmap v13361: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:41.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:53:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:53:41.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:53:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:53:41.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:53:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:53:41.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:53:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:53:41.822 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:53:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:53:41] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T12:53:42.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:53:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:53:41.820Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:53:42.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:53:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:53:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:53:43.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:53:42 smithi083 ceph-mon[79930]: pgmap v13362: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:43.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:53:42 smithi083 ceph-mon[77242]: pgmap v13362: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:43.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:53:42 smithi153 ceph-mon[70044]: pgmap v13362: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:43.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:53:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:53:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:53:45.191 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:53:44 smithi083 ceph-mon[79930]: pgmap v13363: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:45.192 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:53:44 smithi083 ceph-mon[77242]: pgmap v13363: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:45.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:53:44 smithi153 ceph-mon[70044]: pgmap v13363: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:45.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:53:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:53:45.189Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:53:46.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:53:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:53:45.907Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:53:46.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:53:46 smithi083 ceph-mon[77242]: pgmap v13364: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:46.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:53:46 smithi083 ceph-mon[79930]: pgmap v13364: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:47.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:53:46 smithi153 ceph-mon[70044]: pgmap v13364: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:49.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:53:48 smithi083 ceph-mon[77242]: pgmap v13365: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:49.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:53:48 smithi083 ceph-mon[79930]: pgmap v13365: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:49.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:53:48 smithi153 ceph-mon[70044]: pgmap v13365: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:50.336 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:53:50.624 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:53:50.624 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 9m ago 7h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:53:50.625 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 6m ago 7h 64.1M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:53:50.625 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 9m ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:53:50.625 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 6m ago 7h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:53:50.625 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 6m ago 7h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:53:50.625 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 9m ago 7h 880M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:53:50.626 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 9m ago 7h 1630M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:53:50.626 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 6m ago 7h 702M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:53:50.626 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 9m ago 7h 943M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:53:50.626 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 9m ago 7h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:53:50.627 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 6m ago 7h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:53:50.627 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 9m ago 7h 1177M 2348M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:53:50.627 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 9m ago 7h 1165M 2348M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:53:50.627 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 9m ago 7h 886M 2348M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:53:50.628 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 9m ago 7h 1140M 2348M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:53:50.628 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 6m ago 7h 979M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:53:50.628 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 6m ago 7h 1254M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:53:50.628 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 6m ago 7h 1121M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:53:50.628 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 6m ago 7h 1331M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:53:50.629 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 6m ago 7h 176M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:53:50.985 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:53:50.985 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:53:50.985 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:53:50.986 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:53:50.986 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:53:50.986 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:53:50.986 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:53:50.987 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:53:50.987 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:53:50.987 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:53:50.987 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:53:50.987 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:53:50.988 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:53:50.988 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:53:50.988 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:53:50.988 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:53:50.989 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:53:50.989 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:53:51.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:53:50 smithi083 ceph-mon[77242]: pgmap v13366: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:51.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:53:50 smithi083 ceph-mon[79930]: pgmap v13366: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:51.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:53:50 smithi153 ceph-mon[70044]: pgmap v13366: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:51.823 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:53:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:53:51] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T12:53:52.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:53:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:53:51.820Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:53:52.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:53:51 smithi083 ceph-mon[77242]: from='client.49674 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:53:52.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:53:51 smithi083 ceph-mon[77242]: from='client.49680 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:53:52.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:53:51 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3611938459' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:53:52.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:53:51 smithi083 ceph-mon[79930]: from='client.49674 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:53:52.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:53:51 smithi083 ceph-mon[79930]: from='client.49680 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:53:52.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:53:51 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3611938459' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:53:52.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:53:51 smithi153 ceph-mon[70044]: from='client.49674 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:53:52.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:53:51 smithi153 ceph-mon[70044]: from='client.49680 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:53:52.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:53:51 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3611938459' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:53:52.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:53:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:53:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:53:53.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:53:52 smithi083 ceph-mon[79930]: pgmap v13367: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:53.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:53:52 smithi083 ceph-mon[77242]: pgmap v13367: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:53.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:53:52 smithi153 ceph-mon[70044]: pgmap v13367: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:53.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:53:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:53:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:53:55.192 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:53:54 smithi083 ceph-mon[79930]: pgmap v13368: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:55.192 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:53:54 smithi083 ceph-mon[77242]: pgmap v13368: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:55.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:53:54 smithi153 ceph-mon[70044]: pgmap v13368: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:53:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:53:55.189Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:53:56.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:53:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:53:55.907Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:53:56.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:53:56 smithi083 ceph-mon[79930]: pgmap v13369: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:56.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:53:56 smithi083 ceph-mon[77242]: pgmap v13369: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:57.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:53:56 smithi153 ceph-mon[70044]: pgmap v13369: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:59.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:53:58 smithi083 ceph-mon[79930]: pgmap v13370: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:59.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:53:58 smithi083 ceph-mon[77242]: pgmap v13370: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:53:59.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:53:58 smithi153 ceph-mon[70044]: pgmap v13370: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:01.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:54:00 smithi083 ceph-mon[79930]: pgmap v13371: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:01.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:54:00 smithi083 ceph-mon[77242]: pgmap v13371: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:01.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:54:00 smithi153 ceph-mon[70044]: pgmap v13371: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:01.823 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:54:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:54:01] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T12:54:02.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:54:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:54:01.821Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:54:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:54:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:54:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:54:03.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:54:02 smithi083 ceph-mon[77242]: pgmap v13372: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:03.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:54:02 smithi083 ceph-mon[79930]: pgmap v13372: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:03.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:54:02 smithi153 ceph-mon[70044]: pgmap v13372: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:54:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:54:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:54:05.192 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:54:04 smithi083 ceph-mon[79930]: pgmap v13373: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:05.193 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:54:04 smithi083 ceph-mon[77242]: pgmap v13373: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:05.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:54:04 smithi153 ceph-mon[70044]: pgmap v13373: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:05.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:54:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:54:05.190Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:54:06.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:54:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:54:05.908Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:54:06.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:54:06 smithi083 ceph-mon[79930]: pgmap v13374: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:06.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:54:06 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:54:06.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:54:06 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:54:06.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:54:06 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:54:06.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:54:06 smithi083 ceph-mon[77242]: pgmap v13374: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:06.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:54:06 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:54:06.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:54:06 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:54:06.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:54:06 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:54:07.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:54:06 smithi153 ceph-mon[70044]: pgmap v13374: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:07.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:54:06 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:54:07.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:54:06 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:54:07.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:54:06 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:54:08.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:54:07 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:54:08.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:54:07 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:54:08.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:54:07 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:54:09.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:54:08 smithi153 ceph-mon[70044]: pgmap v13375: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:09.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:54:08 smithi083 ceph-mon[79930]: pgmap v13375: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:09.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:54:08 smithi083 ceph-mon[77242]: pgmap v13375: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:11.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:54:10 smithi083 ceph-mon[79930]: pgmap v13376: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:11.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:54:10 smithi083 ceph-mon[77242]: pgmap v13376: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:11.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:54:10 smithi153 ceph-mon[70044]: pgmap v13376: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:11.824 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:54:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:54:11] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T12:54:12.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:54:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:54:11.821Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:54:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:54:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:54:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:54:13.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:54:12 smithi083 ceph-mon[79930]: pgmap v13377: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:13.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:54:12 smithi083 ceph-mon[77242]: pgmap v13377: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:13.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:54:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:54:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:54:13.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:54:12 smithi153 ceph-mon[70044]: pgmap v13377: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:15.193 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:54:14 smithi083 ceph-mon[79930]: pgmap v13378: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:15.194 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:54:14 smithi083 ceph-mon[77242]: pgmap v13378: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:15.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:54:14 smithi153 ceph-mon[70044]: pgmap v13378: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:15.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:54:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:54:15.190Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:54:16.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:54:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:54:15.908Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:54:16.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:54:16 smithi083 ceph-mon[79930]: pgmap v13379: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:16.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:54:16 smithi083 ceph-mon[77242]: pgmap v13379: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:17.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:54:16 smithi153 ceph-mon[70044]: pgmap v13379: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:19.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:54:18 smithi083 ceph-mon[79930]: pgmap v13380: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:19.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:54:18 smithi083 ceph-mon[77242]: pgmap v13380: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:19.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:54:18 smithi153 ceph-mon[70044]: pgmap v13380: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:21.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:54:20 smithi083 ceph-mon[79930]: pgmap v13381: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:21.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:54:20 smithi083 ceph-mon[77242]: pgmap v13381: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:21.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:54:20 smithi153 ceph-mon[70044]: pgmap v13381: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:21.311 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:54:21.604 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:54:21.604 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 9m ago 7h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:54:21.604 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 7m ago 7h 64.1M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:54:21.605 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 9m ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:54:21.605 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 7m ago 7h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:54:21.605 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 7m ago 7h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:54:21.605 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 9m ago 7h 880M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:54:21.606 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 9m ago 7h 1630M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:54:21.606 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 7m ago 7h 702M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:54:21.606 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 9m ago 7h 943M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:54:21.606 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 9m ago 7h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:54:21.607 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 7m ago 7h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:54:21.607 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 9m ago 7h 1177M 2348M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:54:21.607 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 9m ago 7h 1165M 2348M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:54:21.607 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 9m ago 7h 886M 2348M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:54:21.607 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 9m ago 7h 1140M 2348M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:54:21.608 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 7m ago 7h 979M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:54:21.608 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 7m ago 7h 1254M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:54:21.608 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 7m ago 7h 1121M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:54:21.608 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 7m ago 7h 1331M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:54:21.609 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 7m ago 7h 176M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:54:21.823 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:54:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:54:21] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T12:54:21.962 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:54:21.963 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:54:21.963 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:54:21.963 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:54:21.964 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:54:21.964 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:54:21.964 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:54:21.964 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:54:21.965 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:54:21.965 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:54:21.965 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:54:21.965 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:54:21.965 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:54:21.966 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:54:21.966 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:54:21.966 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:54:21.966 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:54:21.967 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:54:22.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:54:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:54:21.821Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:54:22.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:54:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:54:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:54:23.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:54:22 smithi083 ceph-mon[79930]: from='client.49692 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:54:23.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:54:22 smithi083 ceph-mon[79930]: from='client.59629 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:54:23.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:54:22 smithi083 ceph-mon[79930]: pgmap v13382: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:23.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:54:22 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4288331629' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:54:23.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:54:22 smithi083 ceph-mon[77242]: from='client.49692 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:54:23.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:54:22 smithi083 ceph-mon[77242]: from='client.59629 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:54:23.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:54:22 smithi083 ceph-mon[77242]: pgmap v13382: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:23.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:54:22 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4288331629' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:54:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:54:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:54:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:54:23.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:54:22 smithi153 ceph-mon[70044]: from='client.49692 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:54:23.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:54:22 smithi153 ceph-mon[70044]: from='client.59629 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:54:23.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:54:22 smithi153 ceph-mon[70044]: pgmap v13382: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:23.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:54:22 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4288331629' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:54:25.193 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:54:24 smithi083 ceph-mon[79930]: pgmap v13383: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:25.194 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:54:24 smithi083 ceph-mon[77242]: pgmap v13383: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:25.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:54:24 smithi153 ceph-mon[70044]: pgmap v13383: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:25.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:54:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:54:25.191Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:54:26.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:54:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:54:25.909Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:54:26.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:54:26 smithi083 ceph-mon[79930]: pgmap v13384: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:26.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:54:26 smithi083 ceph-mon[77242]: pgmap v13384: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:27.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:54:26 smithi153 ceph-mon[70044]: pgmap v13384: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:29.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:54:28 smithi083 ceph-mon[79930]: pgmap v13385: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:29.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:54:28 smithi083 ceph-mon[77242]: pgmap v13385: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:29.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:54:28 smithi153 ceph-mon[70044]: pgmap v13385: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:31.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:54:30 smithi083 ceph-mon[79930]: pgmap v13386: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:31.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:54:30 smithi083 ceph-mon[77242]: pgmap v13386: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:31.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:54:30 smithi153 ceph-mon[70044]: pgmap v13386: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:31.825 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:54:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:54:31] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T12:54:32.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:54:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:54:31.822Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:54:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:54:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:54:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:54:33.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:54:32 smithi083 ceph-mon[79930]: pgmap v13387: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:33.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:54:32 smithi083 ceph-mon[77242]: pgmap v13387: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:33.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:54:32 smithi153 ceph-mon[70044]: pgmap v13387: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:33.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:54:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:54:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:54:35.194 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:54:34 smithi083 ceph-mon[79930]: pgmap v13388: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:35.195 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:54:34 smithi083 ceph-mon[77242]: pgmap v13388: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:35.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:54:34 smithi153 ceph-mon[70044]: pgmap v13388: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:35.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:54:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:54:35.191Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:54:36.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:54:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:54:35.910Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:54:36.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:54:36 smithi083 ceph-mon[77242]: pgmap v13389: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:36.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:54:36 smithi083 ceph-mon[79930]: pgmap v13389: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:37.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:54:36 smithi153 ceph-mon[70044]: pgmap v13389: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:39.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:54:38 smithi083 ceph-mon[79930]: pgmap v13390: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:39.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:54:38 smithi083 ceph-mon[77242]: pgmap v13390: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:39.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:54:38 smithi153 ceph-mon[70044]: pgmap v13390: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:41.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:54:40 smithi083 ceph-mon[79930]: pgmap v13391: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:41.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:54:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:54:41.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:54:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:54:41.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:54:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:54:41.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:54:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:54:41.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:54:40 smithi083 ceph-mon[77242]: pgmap v13391: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:41.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:54:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:54:41.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:54:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:54:41.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:54:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:54:41.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:54:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:54:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:54:40 smithi153 ceph-mon[70044]: pgmap v13391: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:41.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:54:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:54:41.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:54:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:54:41.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:54:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:54:41.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:54:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:54:41.825 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:54:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:54:41] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T12:54:42.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:54:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:54:41.823Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:54:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:54:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:54:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:54:43.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:54:42 smithi083 ceph-mon[79930]: pgmap v13392: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:43.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:54:42 smithi083 ceph-mon[77242]: pgmap v13392: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:43.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:54:42 smithi153 ceph-mon[70044]: pgmap v13392: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:43.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:54:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:54:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:54:45.195 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:54:44 smithi083 ceph-mon[79930]: pgmap v13393: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:45.195 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:54:44 smithi083 ceph-mon[77242]: pgmap v13393: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:45.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:54:44 smithi153 ceph-mon[70044]: pgmap v13393: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:45.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:54:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:54:45.192Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:54:46.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:54:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:54:45.910Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:54:46.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:54:46 smithi083 ceph-mon[79930]: pgmap v13394: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:46.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:54:46 smithi083 ceph-mon[77242]: pgmap v13394: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:47.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:54:46 smithi153 ceph-mon[70044]: pgmap v13394: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:49.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:54:48 smithi083 ceph-mon[79930]: pgmap v13395: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:49.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:54:48 smithi083 ceph-mon[77242]: pgmap v13395: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:49.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:54:48 smithi153 ceph-mon[70044]: pgmap v13395: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:51.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:54:50 smithi083 ceph-mon[79930]: pgmap v13396: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:51.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:54:50 smithi083 ceph-mon[77242]: pgmap v13396: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:51.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:54:50 smithi153 ceph-mon[70044]: pgmap v13396: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:51.826 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:54:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:54:51] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T12:54:52.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:54:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:54:51.823Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:54:52.287 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:54:52.574 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:54:52.575 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 10m ago 7h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:54:52.575 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 7m ago 7h 64.1M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:54:52.575 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 10m ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:54:52.576 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 7m ago 7h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:54:52.576 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 7m ago 7h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:54:52.576 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 10m ago 7h 880M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:54:52.576 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 10m ago 7h 1630M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:54:52.576 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 7m ago 7h 702M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:54:52.577 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 10m ago 7h 943M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:54:52.577 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 10m ago 7h 26.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:54:52.577 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 7m ago 7h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:54:52.577 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 10m ago 7h 1177M 2348M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:54:52.578 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 10m ago 7h 1165M 2348M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:54:52.578 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 10m ago 7h 886M 2348M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:54:52.578 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 10m ago 7h 1140M 2348M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:54:52.578 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 7m ago 7h 979M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:54:52.579 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 7m ago 7h 1254M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:54:52.579 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 7m ago 7h 1121M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:54:52.579 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 7m ago 7h 1331M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:54:52.579 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 7m ago 7h 176M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:54:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:54:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:54:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:54:52.934 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:54:52.935 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:54:52.935 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:54:52.935 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:54:52.935 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:54:52.936 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:54:52.936 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:54:52.936 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:54:52.936 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:54:52.937 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:54:52.937 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:54:52.937 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:54:52.937 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:54:52.938 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:54:52.938 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:54:52.938 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:54:52.938 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:54:52.939 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:54:53.219 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:54:52 smithi083 ceph-mon[77242]: pgmap v13397: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:53.219 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:54:52 smithi083 ceph-mon[79930]: pgmap v13397: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:53.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:54:52 smithi153 ceph-mon[70044]: pgmap v13397: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:53.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:54:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:54:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:54:54.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:54:53 smithi083 ceph-mon[79930]: from='client.59641 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:54:54.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:54:53 smithi083 ceph-mon[79930]: from='client.49716 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:54:54.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:54:53 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4074199792' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:54:54.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:54:53 smithi083 ceph-mon[77242]: from='client.59641 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:54:54.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:54:53 smithi083 ceph-mon[77242]: from='client.49716 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:54:54.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:54:53 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4074199792' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:54:54.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:54:53 smithi153 ceph-mon[70044]: from='client.59641 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:54:54.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:54:53 smithi153 ceph-mon[70044]: from='client.49716 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:54:54.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:54:53 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4074199792' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:54:55.195 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:54:54 smithi083 ceph-mon[79930]: pgmap v13398: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:55.196 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:54:54 smithi083 ceph-mon[77242]: pgmap v13398: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:55.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:54:54 smithi153 ceph-mon[70044]: pgmap v13398: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:54:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:54:55.193Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:54:56.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:54:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:54:55.911Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:54:56.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:54:56 smithi083 ceph-mon[77242]: pgmap v13399: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:56.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:54:56 smithi083 ceph-mon[79930]: pgmap v13399: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:57.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:54:56 smithi153 ceph-mon[70044]: pgmap v13399: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:59.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:54:58 smithi083 ceph-mon[79930]: pgmap v13400: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:59.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:54:58 smithi083 ceph-mon[77242]: pgmap v13400: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:54:59.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:54:58 smithi153 ceph-mon[70044]: pgmap v13400: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:01.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:55:00 smithi083 ceph-mon[79930]: pgmap v13401: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:01.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:55:00 smithi083 ceph-mon[77242]: pgmap v13401: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:01.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:55:00 smithi153 ceph-mon[70044]: pgmap v13401: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:01.826 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:55:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:55:01] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T12:55:02.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:55:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:55:01.824Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:55:02.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:55:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:55:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:55:03.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:55:02 smithi083 ceph-mon[79930]: pgmap v13402: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:03.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:55:02 smithi083 ceph-mon[77242]: pgmap v13402: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:03.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:55:02 smithi153 ceph-mon[70044]: pgmap v13402: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:55:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:55:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:55:05.195 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:55:04 smithi083 ceph-mon[79930]: pgmap v13403: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:05.196 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:55:04 smithi083 ceph-mon[77242]: pgmap v13403: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:05.312 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:55:04 smithi153 ceph-mon[70044]: pgmap v13403: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:05.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:55:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:55:05.193Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:55:06.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:55:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:55:05.911Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:55:06.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:55:06 smithi083 ceph-mon[79930]: pgmap v13404: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:06.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:55:06 smithi083 ceph-mon[77242]: pgmap v13404: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:07.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:55:06 smithi153 ceph-mon[70044]: pgmap v13404: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:07.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:55:07 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:55:07.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:55:07 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:55:07.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:55:07 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:55:07.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:55:07 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:55:07.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:55:07 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:55:07.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:55:07 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:55:07.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:55:07 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:55:07.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:55:07 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:55:07.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:55:07 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:55:07.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:55:07 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:55:07.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:55:07 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:55:07.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:55:07 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:55:07.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:55:07 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:55:07.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:55:07 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:55:08.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:55:07 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:55:08.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:55:07 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:55:08.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:55:07 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:55:08.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:55:07 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:55:08.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:55:07 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:55:08.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:55:07 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:55:08.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:55:07 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:55:08.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:55:08 smithi083 ceph-mon[79930]: pgmap v13405: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:08.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:55:08 smithi083 ceph-mon[77242]: pgmap v13405: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:09.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:55:08 smithi153 ceph-mon[70044]: pgmap v13405: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:11.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:55:10 smithi083 ceph-mon[79930]: pgmap v13406: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:11.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:55:10 smithi083 ceph-mon[77242]: pgmap v13406: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:11.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:55:10 smithi153 ceph-mon[70044]: pgmap v13406: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:11.826 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:55:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:55:11] "GET /metrics HTTP/1.1" 200 225229 "" "Prometheus/2.33.4" 2022-09-29T12:55:12.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:55:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:55:11.823Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:55:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:55:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:55:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:55:12.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:55:12 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:55:12.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:55:12 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:55:12.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:55:12 smithi153 ceph-mon[70044]: pgmap v13407: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:12.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:55:12 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:55:12.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:55:12 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:55:12.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:55:12 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:55:12.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:55:12 smithi083 ceph-mon[79930]: pgmap v13407: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:12.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:55:12 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:55:12.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:55:12 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:55:12.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:55:12 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:55:12.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:55:12 smithi083 ceph-mon[77242]: pgmap v13407: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:12.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:55:12 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:55:13.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:55:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:55:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:55:15.196 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:55:14 smithi083 ceph-mon[79930]: pgmap v13408: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:15.197 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:55:14 smithi083 ceph-mon[77242]: pgmap v13408: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:15.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:55:14 smithi153 ceph-mon[70044]: pgmap v13408: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:15.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:55:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:55:15.193Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:55:16.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:55:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:55:15.912Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:55:16.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:55:16 smithi083 ceph-mon[79930]: pgmap v13409: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:16.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:55:16 smithi083 ceph-mon[77242]: pgmap v13409: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:17.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:55:16 smithi153 ceph-mon[70044]: pgmap v13409: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:19.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:55:18 smithi083 ceph-mon[79930]: pgmap v13410: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:19.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:55:18 smithi083 ceph-mon[77242]: pgmap v13410: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:19.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:55:18 smithi153 ceph-mon[70044]: pgmap v13410: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:21.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:55:20 smithi083 ceph-mon[77242]: pgmap v13411: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:21.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:55:20 smithi083 ceph-mon[79930]: pgmap v13411: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:21.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:55:20 smithi153 ceph-mon[70044]: pgmap v13411: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:21.827 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:55:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:55:21] "GET /metrics HTTP/1.1" 200 225229 "" "Prometheus/2.33.4" 2022-09-29T12:55:22.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:55:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:55:21.824Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:55:22.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:55:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:55:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:55:23.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:55:22 smithi083 ceph-mon[79930]: pgmap v13412: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:23.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:55:22 smithi083 ceph-mon[77242]: pgmap v13412: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:23.258 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:55:23.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:55:22 smithi153 ceph-mon[70044]: pgmap v13412: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:23.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:55:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:55:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:55:23.547 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:55:23.548 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 12s ago 7h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:55:23.548 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 8m ago 7h 64.1M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:55:23.548 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 12s ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:55:23.548 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 8m ago 7h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:55:23.549 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 8m ago 7h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:55:23.549 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 12s ago 7h 888M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:55:23.549 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 12s ago 7h 1666M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:55:23.549 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 8m ago 7h 702M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:55:23.549 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 12s ago 7h 965M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:55:23.550 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 12s ago 7h 24.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:55:23.550 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 8m ago 7h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:55:23.550 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 12s ago 7h 1204M 2348M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:55:23.550 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 12s ago 7h 1191M 2348M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:55:23.551 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 12s ago 7h 906M 2348M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:55:23.551 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 12s ago 7h 1166M 2348M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:55:23.551 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 8m ago 7h 979M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:55:23.551 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 8m ago 7h 1254M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:55:23.551 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 8m ago 7h 1121M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:55:23.552 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 8m ago 7h 1331M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:55:23.552 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 8m ago 7h 176M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:55:23.905 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:55:23.905 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:55:23.906 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:55:23.906 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:55:23.906 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:55:23.906 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:55:23.907 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:55:23.907 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:55:23.907 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:55:23.907 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:55:23.908 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:55:23.908 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:55:23.908 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:55:23.908 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:55:23.908 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:55:23.909 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:55:23.909 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:55:23.909 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:55:24.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:55:23 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1784124090' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:55:24.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:55:23 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1784124090' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:55:24.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:55:23 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1784124090' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:55:25.196 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:55:24 smithi083 ceph-mon[79930]: from='client.49728 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:55:25.197 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:55:24 smithi083 ceph-mon[79930]: from='client.59665 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:55:25.197 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:55:24 smithi083 ceph-mon[79930]: pgmap v13413: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:25.197 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:55:24 smithi083 ceph-mon[77242]: from='client.49728 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:55:25.198 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:55:24 smithi083 ceph-mon[77242]: from='client.59665 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:55:25.198 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:55:24 smithi083 ceph-mon[77242]: pgmap v13413: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:25.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:55:24 smithi153 ceph-mon[70044]: from='client.49728 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:55:25.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:55:24 smithi153 ceph-mon[70044]: from='client.59665 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:55:25.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:55:24 smithi153 ceph-mon[70044]: pgmap v13413: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:25.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:55:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:55:25.193Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:55:26.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:55:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:55:25.912Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:55:26.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:55:26 smithi083 ceph-mon[79930]: pgmap v13414: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:26.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:55:26 smithi083 ceph-mon[77242]: pgmap v13414: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:27.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:55:26 smithi153 ceph-mon[70044]: pgmap v13414: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:29.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:55:28 smithi083 ceph-mon[77242]: pgmap v13415: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:29.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:55:28 smithi083 ceph-mon[79930]: pgmap v13415: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:29.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:55:28 smithi153 ceph-mon[70044]: pgmap v13415: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:31.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:55:30 smithi083 ceph-mon[79930]: pgmap v13416: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:31.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:55:30 smithi083 ceph-mon[77242]: pgmap v13416: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:31.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:55:30 smithi153 ceph-mon[70044]: pgmap v13416: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:31.828 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:55:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:55:31] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T12:55:32.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:55:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:55:31.825Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:55:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:55:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:55:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:55:33.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:55:32 smithi083 ceph-mon[77242]: pgmap v13417: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:33.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:55:32 smithi083 ceph-mon[79930]: pgmap v13417: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:33.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:55:32 smithi153 ceph-mon[70044]: pgmap v13417: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:33.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:55:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:55:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:55:35.197 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:55:34 smithi083 ceph-mon[79930]: pgmap v13418: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:35.198 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:55:34 smithi083 ceph-mon[77242]: pgmap v13418: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:35.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:55:34 smithi153 ceph-mon[70044]: pgmap v13418: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:35.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:55:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:55:35.195Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:55:36.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:55:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:55:35.912Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:55:36.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:55:36 smithi083 ceph-mon[79930]: pgmap v13419: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:36.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:55:36 smithi083 ceph-mon[77242]: pgmap v13419: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:37.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:55:36 smithi153 ceph-mon[70044]: pgmap v13419: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:39.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:55:38 smithi083 ceph-mon[79930]: pgmap v13420: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:39.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:55:38 smithi083 ceph-mon[77242]: pgmap v13420: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:39.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:55:38 smithi153 ceph-mon[70044]: pgmap v13420: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:41.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:55:40 smithi083 ceph-mon[79930]: pgmap v13421: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:41.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:55:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:55:41.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:55:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:55:41.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:55:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:55:41.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:55:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:55:41.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:55:40 smithi083 ceph-mon[77242]: pgmap v13421: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:41.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:55:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:55:41.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:55:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:55:41.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:55:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:55:41.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:55:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:55:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:55:40 smithi153 ceph-mon[70044]: pgmap v13421: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:41.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:55:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:55:41.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:55:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:55:41.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:55:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:55:41.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:55:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:55:41.828 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:55:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:55:41] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T12:55:42.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:55:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:55:41.826Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:55:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:55:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:55:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:55:43.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:55:42 smithi083 ceph-mon[77242]: pgmap v13422: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:43.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:55:42 smithi083 ceph-mon[79930]: pgmap v13422: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:43.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:55:42 smithi153 ceph-mon[70044]: pgmap v13422: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:43.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:55:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:55:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:55:45.198 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:55:44 smithi083 ceph-mon[79930]: pgmap v13423: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:45.199 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:55:44 smithi083 ceph-mon[77242]: pgmap v13423: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:45.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:55:44 smithi153 ceph-mon[70044]: pgmap v13423: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:45.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:55:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:55:45.195Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:55:46.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:55:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:55:45.913Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:55:46.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:55:46 smithi083 ceph-mon[79930]: pgmap v13424: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:46.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:55:46 smithi083 ceph-mon[77242]: pgmap v13424: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:47.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:55:46 smithi153 ceph-mon[70044]: pgmap v13424: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:49.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:55:48 smithi083 ceph-mon[79930]: pgmap v13425: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:49.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:55:48 smithi083 ceph-mon[77242]: pgmap v13425: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:49.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:55:48 smithi153 ceph-mon[70044]: pgmap v13425: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:51.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:55:50 smithi083 ceph-mon[79930]: pgmap v13426: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:51.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:55:50 smithi083 ceph-mon[77242]: pgmap v13426: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:51.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:55:50 smithi153 ceph-mon[70044]: pgmap v13426: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:51.828 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:55:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:55:51] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T12:55:52.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:55:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:55:51.826Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:55:52.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:55:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:55:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:55:53.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:55:52 smithi083 ceph-mon[77242]: pgmap v13427: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:53.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:55:52 smithi083 ceph-mon[79930]: pgmap v13427: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:53.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:55:52 smithi153 ceph-mon[70044]: pgmap v13427: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:53.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:55:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:55:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:55:54.232 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:55:54.524 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:55:54.524 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 43s ago 7h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:55:54.524 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 8m ago 7h 64.1M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:55:54.525 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 43s ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:55:54.525 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 8m ago 7h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:55:54.525 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 8m ago 7h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:55:54.525 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 43s ago 7h 888M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:55:54.526 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 43s ago 7h 1666M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:55:54.526 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 8m ago 7h 702M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:55:54.526 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 43s ago 7h 965M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:55:54.526 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 43s ago 7h 24.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:55:54.527 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 8m ago 7h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:55:54.527 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 43s ago 7h 1204M 2348M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:55:54.527 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 43s ago 7h 1191M 2348M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:55:54.527 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 43s ago 7h 906M 2348M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:55:54.528 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 43s ago 7h 1166M 2348M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:55:54.528 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 8m ago 7h 979M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:55:54.528 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 8m ago 7h 1254M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:55:54.528 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 8m ago 7h 1121M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:55:54.528 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 8m ago 7h 1331M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:55:54.529 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 8m ago 7h 176M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:55:54.879 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:55:54.879 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:55:54.880 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:55:54.880 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:55:54.880 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:55:54.880 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:55:54.881 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:55:54.881 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:55:54.881 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:55:54.881 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:55:54.882 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:55:54.882 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:55:54.882 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:55:54.882 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:55:54.883 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:55:54.883 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:55:54.883 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:55:54.883 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:55:55.198 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:55:54 smithi083 ceph-mon[79930]: pgmap v13428: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:55.198 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:55:54 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2041567642' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:55:55.199 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:55:54 smithi083 ceph-mon[77242]: pgmap v13428: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:55.199 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:55:54 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2041567642' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:55:55.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:55:54 smithi153 ceph-mon[70044]: pgmap v13428: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:55.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:55:54 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2041567642' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:55:55.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:55:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:55:55.195Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:55:56.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:55:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:55:55.913Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:55:56.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:55:55 smithi083 ceph-mon[77242]: from='client.59677 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:55:56.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:55:55 smithi083 ceph-mon[77242]: from='client.49752 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:55:56.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:55:55 smithi083 ceph-mon[79930]: from='client.59677 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:55:56.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:55:55 smithi083 ceph-mon[79930]: from='client.49752 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:55:56.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:55:55 smithi153 ceph-mon[70044]: from='client.59677 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:55:56.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:55:55 smithi153 ceph-mon[70044]: from='client.49752 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:55:57.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:55:56 smithi083 ceph-mon[77242]: pgmap v13429: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:57.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:55:56 smithi083 ceph-mon[79930]: pgmap v13429: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:57.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:55:56 smithi153 ceph-mon[70044]: pgmap v13429: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:59.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:55:58 smithi083 ceph-mon[77242]: pgmap v13430: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:59.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:55:58 smithi083 ceph-mon[79930]: pgmap v13430: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:55:59.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:55:58 smithi153 ceph-mon[70044]: pgmap v13430: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:01.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:56:00 smithi083 ceph-mon[79930]: pgmap v13431: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:01.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:56:00 smithi083 ceph-mon[77242]: pgmap v13431: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:01.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:56:00 smithi153 ceph-mon[70044]: pgmap v13431: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:01.830 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:56:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:56:01] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T12:56:02.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:56:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:56:01.827Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:56:02.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:56:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:56:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:56:03.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:56:02 smithi083 ceph-mon[79930]: pgmap v13432: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:03.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:56:02 smithi083 ceph-mon[77242]: pgmap v13432: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:03.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:56:02 smithi153 ceph-mon[70044]: pgmap v13432: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:56:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:56:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:56:05.199 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:56:04 smithi083 ceph-mon[79930]: pgmap v13433: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:05.199 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:56:04 smithi083 ceph-mon[77242]: pgmap v13433: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:05.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:56:04 smithi153 ceph-mon[70044]: pgmap v13433: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:05.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:56:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:56:05.196Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:56:06.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:56:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:56:05.914Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:56:06.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:56:06 smithi083 ceph-mon[79930]: pgmap v13434: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:06.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:56:06 smithi083 ceph-mon[77242]: pgmap v13434: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:07.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:56:06 smithi153 ceph-mon[70044]: pgmap v13434: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:09.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:56:08 smithi083 ceph-mon[77242]: pgmap v13435: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:09.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:56:08 smithi083 ceph-mon[79930]: pgmap v13435: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:09.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:56:08 smithi153 ceph-mon[70044]: pgmap v13435: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:11.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:56:10 smithi083 ceph-mon[77242]: pgmap v13436: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:11.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:56:10 smithi083 ceph-mon[79930]: pgmap v13436: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:11.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:56:10 smithi153 ceph-mon[70044]: pgmap v13436: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:11.830 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:56:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:56:11] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T12:56:12.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:56:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:56:11.827Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:56:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:56:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:56:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:56:13.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:56:12 smithi083 ceph-mon[79930]: pgmap v13437: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:13.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:56:12 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:56:13.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:56:12 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:56:13.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:56:12 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:56:13.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:56:12 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:56:13.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:56:12 smithi083 ceph-mon[77242]: pgmap v13437: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:13.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:56:12 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:56:13.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:56:12 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:56:13.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:56:12 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:56:13.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:56:12 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:56:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:56:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:56:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:56:13.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:56:12 smithi153 ceph-mon[70044]: pgmap v13437: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:13.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:56:12 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:56:13.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:56:12 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:56:13.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:56:12 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:56:13.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:56:12 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:56:15.200 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:56:14 smithi083 ceph-mon[79930]: pgmap v13438: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:15.200 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:56:14 smithi083 ceph-mon[77242]: pgmap v13438: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:15.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:56:14 smithi153 ceph-mon[70044]: pgmap v13438: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:15.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:56:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:56:15.197Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:56:16.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:56:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:56:15.915Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:56:16.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:56:16 smithi083 ceph-mon[79930]: pgmap v13439: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:16.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:56:16 smithi083 ceph-mon[77242]: pgmap v13439: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:17.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:56:16 smithi153 ceph-mon[70044]: pgmap v13439: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:19.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:56:18 smithi083 ceph-mon[79930]: pgmap v13440: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:19.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:56:18 smithi083 ceph-mon[77242]: pgmap v13440: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:19.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:56:18 smithi153 ceph-mon[70044]: pgmap v13440: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:21.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:56:20 smithi083 ceph-mon[79930]: pgmap v13441: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:21.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:56:20 smithi083 ceph-mon[77242]: pgmap v13441: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:21.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:56:20 smithi153 ceph-mon[70044]: pgmap v13441: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:21.831 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:56:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:56:21] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T12:56:22.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:56:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:56:21.828Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:56:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:56:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:56:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:56:23.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:56:22 smithi083 ceph-mon[77242]: pgmap v13442: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:23.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:56:22 smithi083 ceph-mon[79930]: pgmap v13442: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:23.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:56:22 smithi153 ceph-mon[70044]: pgmap v13442: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:23.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:56:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:56:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:56:25.199 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:56:24 smithi083 ceph-mon[79930]: pgmap v13443: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:25.200 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:56:24 smithi083 ceph-mon[77242]: pgmap v13443: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:25.207 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:56:25.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:56:24 smithi153 ceph-mon[70044]: pgmap v13443: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:25.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:56:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:56:25.196Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:56:25.494 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:56:25.495 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 73s ago 7h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:56:25.495 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 9m ago 7h 64.1M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:56:25.495 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 73s ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:56:25.495 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 9m ago 7h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:56:25.495 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 9m ago 7h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:56:25.496 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 73s ago 7h 888M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:56:25.496 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 73s ago 7h 1666M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:56:25.496 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 9m ago 7h 702M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:56:25.496 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 73s ago 7h 965M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:56:25.497 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 73s ago 7h 24.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:56:25.497 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 9m ago 7h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:56:25.497 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 73s ago 7h 1204M 2348M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:56:25.497 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 73s ago 7h 1191M 2348M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:56:25.497 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 73s ago 7h 906M 2348M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:56:25.498 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 73s ago 7h 1166M 2348M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:56:25.498 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 9m ago 7h 979M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:56:25.498 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 9m ago 7h 1254M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:56:25.498 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 9m ago 7h 1121M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:56:25.499 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 9m ago 7h 1331M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:56:25.499 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 9m ago 7h 176M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:56:25.852 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:56:25.852 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:56:25.852 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:56:25.853 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:56:25.853 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:56:25.853 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:56:25.853 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:56:25.854 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:56:25.854 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:56:25.854 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:56:25.854 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:56:25.855 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:56:25.855 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:56:25.855 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:56:25.855 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:56:25.856 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:56:25.856 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:56:25.856 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:56:26.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:56:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:56:25.915Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:56:26.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:56:26 smithi083 ceph-mon[79930]: from='client.59695 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:56:26.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:56:26 smithi083 ceph-mon[79930]: from='client.49770 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:56:26.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:56:26 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1092553835' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:56:26.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:56:26 smithi083 ceph-mon[79930]: pgmap v13444: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:26.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:56:26 smithi083 ceph-mon[77242]: from='client.59695 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:56:26.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:56:26 smithi083 ceph-mon[77242]: from='client.49770 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:56:26.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:56:26 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1092553835' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:56:26.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:56:26 smithi083 ceph-mon[77242]: pgmap v13444: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:27.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:56:26 smithi153 ceph-mon[70044]: from='client.59695 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:56:27.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:56:26 smithi153 ceph-mon[70044]: from='client.49770 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:56:27.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:56:26 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1092553835' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:56:27.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:56:26 smithi153 ceph-mon[70044]: pgmap v13444: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:29.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:56:28 smithi083 ceph-mon[79930]: pgmap v13445: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:29.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:56:28 smithi083 ceph-mon[77242]: pgmap v13445: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:29.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:56:28 smithi153 ceph-mon[70044]: pgmap v13445: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:31.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:56:30 smithi083 ceph-mon[79930]: pgmap v13446: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:31.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:56:30 smithi083 ceph-mon[77242]: pgmap v13446: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:31.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:56:30 smithi153 ceph-mon[70044]: pgmap v13446: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:31.832 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:56:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:56:31] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T12:56:32.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:56:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:56:31.829Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:56:32.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:56:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:56:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:56:33.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:56:32 smithi083 ceph-mon[77242]: pgmap v13447: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:33.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:56:32 smithi083 ceph-mon[79930]: pgmap v13447: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:33.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:56:32 smithi153 ceph-mon[70044]: pgmap v13447: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:33.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:56:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:56:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:56:35.200 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:56:34 smithi083 ceph-mon[79930]: pgmap v13448: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:35.201 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:56:34 smithi083 ceph-mon[77242]: pgmap v13448: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:35.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:56:34 smithi153 ceph-mon[70044]: pgmap v13448: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:35.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:56:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:56:35.197Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:56:36.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:56:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:56:35.916Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:56:36.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:56:36 smithi083 ceph-mon[79930]: pgmap v13449: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:36.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:56:36 smithi083 ceph-mon[77242]: pgmap v13449: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:37.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:56:36 smithi153 ceph-mon[70044]: pgmap v13449: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:39.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:56:38 smithi083 ceph-mon[79930]: pgmap v13450: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:39.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:56:38 smithi083 ceph-mon[77242]: pgmap v13450: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:39.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:56:38 smithi153 ceph-mon[70044]: pgmap v13450: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:41.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:56:40 smithi083 ceph-mon[79930]: pgmap v13451: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:41.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:56:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:56:41.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:56:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:56:41.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:56:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:56:41.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:56:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:56:41.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:56:40 smithi083 ceph-mon[77242]: pgmap v13451: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:41.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:56:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:56:41.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:56:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:56:41.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:56:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:56:41.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:56:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:56:41.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:56:40 smithi153 ceph-mon[70044]: pgmap v13451: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:41.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:56:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:56:41.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:56:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:56:41.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:56:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:56:41.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:56:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:56:41.832 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:56:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:56:41] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T12:56:42.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:56:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:56:41.830Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:56:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:56:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:56:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:56:43.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:56:42 smithi083 ceph-mon[79930]: pgmap v13452: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:43.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:56:42 smithi083 ceph-mon[77242]: pgmap v13452: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:43.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:56:42 smithi153 ceph-mon[70044]: pgmap v13452: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:43.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:56:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:56:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:56:45.200 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:56:44 smithi083 ceph-mon[79930]: pgmap v13453: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:45.201 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:56:44 smithi083 ceph-mon[77242]: pgmap v13453: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:45.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:56:44 smithi153 ceph-mon[70044]: pgmap v13453: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:45.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:56:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:56:45.198Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:56:46.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:56:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:56:45.917Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:56:46.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:56:46 smithi083 ceph-mon[79930]: pgmap v13454: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:46.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:56:46 smithi083 ceph-mon[77242]: pgmap v13454: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:47.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:56:46 smithi153 ceph-mon[70044]: pgmap v13454: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:49.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:56:48 smithi083 ceph-mon[79930]: pgmap v13455: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:49.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:56:48 smithi083 ceph-mon[77242]: pgmap v13455: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:49.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:56:48 smithi153 ceph-mon[70044]: pgmap v13455: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:51.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:56:50 smithi083 ceph-mon[79930]: pgmap v13456: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:51.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:56:50 smithi083 ceph-mon[77242]: pgmap v13456: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:51.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:56:50 smithi153 ceph-mon[70044]: pgmap v13456: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:51.834 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:56:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:56:51] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T12:56:52.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:56:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:56:51.831Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:56:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:56:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:56:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:56:53.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:56:52 smithi083 ceph-mon[79930]: pgmap v13457: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:53.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:56:52 smithi083 ceph-mon[77242]: pgmap v13457: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:53.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:56:52 smithi153 ceph-mon[70044]: pgmap v13457: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:53.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:56:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:56:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:56:55.201 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:56:54 smithi083 ceph-mon[79930]: pgmap v13458: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:55.201 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:56:54 smithi083 ceph-mon[77242]: pgmap v13458: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:55.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:56:54 smithi153 ceph-mon[70044]: pgmap v13458: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:55.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:56:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:56:55.198Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:56:56.180 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:56:56.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:56:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:56:55.919Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:56:56.470 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:56:56.471 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 104s ago 7h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:56:56.471 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 9m ago 7h 64.1M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:56:56.471 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 104s ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:56:56.471 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 9m ago 7h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:56:56.472 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 9m ago 7h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:56:56.472 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 104s ago 7h 888M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:56:56.472 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 104s ago 7h 1666M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:56:56.472 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 9m ago 7h 702M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:56:56.473 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 104s ago 7h 965M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:56:56.473 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 104s ago 7h 24.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:56:56.473 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 9m ago 7h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:56:56.473 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 104s ago 7h 1204M 2348M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:56:56.473 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 104s ago 7h 1191M 2348M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:56:56.474 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 104s ago 7h 906M 2348M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:56:56.474 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 104s ago 7h 1166M 2348M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:56:56.474 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 9m ago 7h 979M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:56:56.474 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 9m ago 7h 1254M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:56:56.475 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 9m ago 7h 1121M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:56:56.475 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 9m ago 7h 1331M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:56:56.475 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 9m ago 7h 176M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:56:56.828 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:56:56.828 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:56:56.828 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:56:56.829 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:56:56.829 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:56:56.829 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:56:56.829 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:56:56.830 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:56:56.830 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:56:56.830 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:56:56.830 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:56:56.831 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:56:56.831 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:56:56.831 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:56:56.831 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:56:56.831 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:56:56.832 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:56:56.832 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:56:56.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:56:56 smithi083 ceph-mon[77242]: pgmap v13459: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:56.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:56:56 smithi083 ceph-mon[79930]: pgmap v13459: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:57.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:56:56 smithi153 ceph-mon[70044]: pgmap v13459: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:57.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:56:57 smithi083 ceph-mon[79930]: from='client.49782 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:56:57.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:56:57 smithi083 ceph-mon[79930]: from='client.59719 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:56:57.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:56:57 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/671352202' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:56:57.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:56:57 smithi083 ceph-mon[77242]: from='client.49782 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:56:57.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:56:57 smithi083 ceph-mon[77242]: from='client.59719 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:56:57.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:56:57 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/671352202' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:56:58.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:56:57 smithi153 ceph-mon[70044]: from='client.49782 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:56:58.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:56:57 smithi153 ceph-mon[70044]: from='client.59719 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:56:58.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:56:57 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/671352202' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:56:58.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:56:58 smithi083 ceph-mon[79930]: pgmap v13460: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:58.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:56:58 smithi083 ceph-mon[77242]: pgmap v13460: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:56:59.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:56:58 smithi153 ceph-mon[70044]: pgmap v13460: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:01.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:57:00 smithi083 ceph-mon[79930]: pgmap v13461: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:01.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:57:00 smithi083 ceph-mon[77242]: pgmap v13461: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:01.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:57:00 smithi153 ceph-mon[70044]: pgmap v13461: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:01.834 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:57:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:57:01] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T12:57:02.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:57:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:57:01.832Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:57:02.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:57:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:57:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:57:03.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:57:02 smithi083 ceph-mon[79930]: pgmap v13462: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:03.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:57:02 smithi083 ceph-mon[77242]: pgmap v13462: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:03.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:57:02 smithi153 ceph-mon[70044]: pgmap v13462: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:57:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:57:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:57:05.202 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:57:04 smithi083 ceph-mon[79930]: pgmap v13463: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:05.203 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:57:04 smithi083 ceph-mon[77242]: pgmap v13463: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:05.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:57:04 smithi153 ceph-mon[70044]: pgmap v13463: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:05.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:57:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:57:05.199Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:57:06.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:57:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:57:05.919Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:57:06.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:57:06 smithi083 ceph-mon[77242]: pgmap v13464: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:06.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:57:06 smithi083 ceph-mon[79930]: pgmap v13464: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:07.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:57:06 smithi153 ceph-mon[70044]: pgmap v13464: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:09.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:57:08 smithi083 ceph-mon[77242]: pgmap v13465: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:09.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:57:08 smithi083 ceph-mon[79930]: pgmap v13465: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:09.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:57:08 smithi153 ceph-mon[70044]: pgmap v13465: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:11.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:57:10 smithi083 ceph-mon[79930]: pgmap v13466: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:11.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:57:10 smithi083 ceph-mon[77242]: pgmap v13466: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:11.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:57:10 smithi153 ceph-mon[70044]: pgmap v13466: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:11.834 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:57:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:57:11] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T12:57:12.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:57:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:57:11.832Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:57:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:57:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:57:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:57:13.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:57:12 smithi083 ceph-mon[79930]: pgmap v13467: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:13.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:57:12 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:57:13.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:57:12 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:57:13.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:57:12 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:57:13.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:57:12 smithi083 ceph-mon[77242]: pgmap v13467: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:13.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:57:12 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:57:13.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:57:12 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:57:13.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:57:12 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:57:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:57:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:57:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:57:13.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:57:12 smithi153 ceph-mon[70044]: pgmap v13467: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:13.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:57:12 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:57:13.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:57:12 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:57:13.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:57:12 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:57:15.203 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:57:14 smithi083 ceph-mon[79930]: pgmap v13468: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:15.204 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:57:14 smithi083 ceph-mon[77242]: pgmap v13468: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:15.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:57:14 smithi153 ceph-mon[70044]: pgmap v13468: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:15.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:57:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:57:15.200Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:57:16.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:57:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:57:15.919Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:57:16.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:57:16 smithi083 ceph-mon[79930]: pgmap v13469: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:16.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:57:16 smithi083 ceph-mon[77242]: pgmap v13469: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:17.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:57:16 smithi153 ceph-mon[70044]: pgmap v13469: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:18.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:57:18 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:57:18.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:57:18 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:57:18.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:57:18 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:57:18.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:57:18 smithi153 ceph-mon[70044]: pgmap v13470: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:18.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:57:18 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:57:18.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:57:18 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:57:18.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:57:18 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:57:18.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:57:18 smithi083 ceph-mon[77242]: pgmap v13470: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:18.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:57:18 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:57:18.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:57:18 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:57:18.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:57:18 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:57:18.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:57:18 smithi083 ceph-mon[79930]: pgmap v13470: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:21.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:57:20 smithi083 ceph-mon[79930]: pgmap v13471: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:21.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:57:20 smithi083 ceph-mon[77242]: pgmap v13471: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:21.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:57:20 smithi153 ceph-mon[70044]: pgmap v13471: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:21.836 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:57:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:57:21] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T12:57:22.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:57:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:57:21.833Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:57:22.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:57:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:57:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:57:23.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:57:22 smithi083 ceph-mon[79930]: pgmap v13472: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:23.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:57:22 smithi083 ceph-mon[77242]: pgmap v13472: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:23.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:57:22 smithi153 ceph-mon[70044]: pgmap v13472: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:23.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:57:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:57:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:57:25.203 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:57:24 smithi083 ceph-mon[79930]: pgmap v13473: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:25.204 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:57:24 smithi083 ceph-mon[77242]: pgmap v13473: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:25.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:57:24 smithi153 ceph-mon[70044]: pgmap v13473: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:25.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:57:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:57:25.201Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:57:26.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:57:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:57:25.920Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:57:26.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:57:26 smithi083 ceph-mon[79930]: pgmap v13474: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:26.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:57:26 smithi083 ceph-mon[77242]: pgmap v13474: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:27.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:57:26 smithi153 ceph-mon[70044]: pgmap v13474: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:27.155 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:57:27.442 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:57:27.442 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 2m ago 7h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:57:27.443 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 10s ago 7h 63.1M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:57:27.443 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 2m ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:57:27.443 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 10s ago 7h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:57:27.443 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 10s ago 7h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:57:27.444 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 2m ago 7h 888M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:57:27.444 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 2m ago 7h 1666M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:57:27.444 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 10s ago 7h 721M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:57:27.444 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 2m ago 7h 965M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:57:27.445 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 2m ago 7h 24.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:57:27.445 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 10s ago 7h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:57:27.445 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 2m ago 7h 1204M 2348M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:57:27.445 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 2m ago 7h 1191M 2348M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:57:27.445 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 2m ago 7h 906M 2348M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:57:27.446 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 2m ago 7h 1166M 2348M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:57:27.446 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 10s ago 7h 1025M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:57:27.446 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 10s ago 7h 1283M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:57:27.446 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 10s ago 7h 1146M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:57:27.447 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 10s ago 7h 1360M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:57:27.447 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 10s ago 7h 177M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:57:27.795 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:57:27.796 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:57:27.796 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:57:27.796 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:57:27.797 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:57:27.797 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:57:27.797 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:57:27.797 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:57:27.798 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:57:27.798 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:57:27.798 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:57:27.798 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:57:27.799 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:57:27.799 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:57:27.799 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:57:27.799 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:57:27.800 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:57:27.800 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:57:28.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:57:27 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/703212383' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:57:28.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:57:27 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/703212383' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:57:28.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:57:27 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/703212383' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:57:29.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:57:28 smithi083 ceph-mon[79930]: from='client.49800 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:57:29.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:57:28 smithi083 ceph-mon[79930]: from='client.49806 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:57:29.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:57:28 smithi083 ceph-mon[79930]: pgmap v13475: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:29.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:57:28 smithi083 ceph-mon[77242]: from='client.49800 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:57:29.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:57:28 smithi083 ceph-mon[77242]: from='client.49806 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:57:29.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:57:28 smithi083 ceph-mon[77242]: pgmap v13475: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:29.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:57:28 smithi153 ceph-mon[70044]: from='client.49800 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:57:29.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:57:28 smithi153 ceph-mon[70044]: from='client.49806 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:57:29.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:57:28 smithi153 ceph-mon[70044]: pgmap v13475: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:31.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:57:30 smithi083 ceph-mon[79930]: pgmap v13476: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:31.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:57:30 smithi083 ceph-mon[77242]: pgmap v13476: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:31.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:57:30 smithi153 ceph-mon[70044]: pgmap v13476: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:31.836 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:57:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:57:31] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T12:57:32.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:57:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:57:31.834Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:57:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:57:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:57:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:57:33.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:57:32 smithi083 ceph-mon[79930]: pgmap v13477: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:33.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:57:32 smithi083 ceph-mon[77242]: pgmap v13477: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:33.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:57:32 smithi153 ceph-mon[70044]: pgmap v13477: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:33.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:57:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:57:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:57:35.203 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:57:34 smithi083 ceph-mon[79930]: pgmap v13478: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:35.204 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:57:34 smithi083 ceph-mon[77242]: pgmap v13478: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:35.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:57:34 smithi153 ceph-mon[70044]: pgmap v13478: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:35.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:57:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:57:35.201Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:57:36.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:57:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:57:35.921Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:57:36.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:57:36 smithi083 ceph-mon[77242]: pgmap v13479: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:36.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:57:36 smithi083 ceph-mon[79930]: pgmap v13479: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:37.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:57:36 smithi153 ceph-mon[70044]: pgmap v13479: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:39.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:57:38 smithi083 ceph-mon[77242]: pgmap v13480: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:39.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:57:38 smithi083 ceph-mon[79930]: pgmap v13480: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:39.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:57:38 smithi153 ceph-mon[70044]: pgmap v13480: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:41.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:57:40 smithi083 ceph-mon[79930]: pgmap v13481: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:41.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:57:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:57:41.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:57:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:57:41.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:57:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:57:41.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:57:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:57:41.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:57:40 smithi083 ceph-mon[77242]: pgmap v13481: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:41.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:57:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:57:41.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:57:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:57:41.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:57:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:57:41.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:57:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:57:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:57:40 smithi153 ceph-mon[70044]: pgmap v13481: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:41.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:57:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:57:41.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:57:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:57:41.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:57:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:57:41.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:57:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:57:41.837 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:57:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:57:41] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T12:57:42.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:57:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:57:41.834Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:57:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:57:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:57:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:57:43.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:57:42 smithi083 ceph-mon[79930]: pgmap v13482: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:43.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:57:42 smithi083 ceph-mon[77242]: pgmap v13482: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:43.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:57:42 smithi153 ceph-mon[70044]: pgmap v13482: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:43.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:57:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:57:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:57:45.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:57:44 smithi083 ceph-mon[79930]: pgmap v13483: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:45.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:57:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:57:45.202Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:57:45.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:57:44 smithi083 ceph-mon[77242]: pgmap v13483: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:45.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:57:44 smithi153 ceph-mon[70044]: pgmap v13483: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:46.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:57:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:57:45.922Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:57:46.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:57:46 smithi083 ceph-mon[79930]: pgmap v13484: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:46.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:57:46 smithi083 ceph-mon[77242]: pgmap v13484: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:47.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:57:46 smithi153 ceph-mon[70044]: pgmap v13484: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:49.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:57:48 smithi083 ceph-mon[79930]: pgmap v13485: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:49.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:57:48 smithi083 ceph-mon[77242]: pgmap v13485: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:49.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:57:48 smithi153 ceph-mon[70044]: pgmap v13485: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:51.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:57:50 smithi083 ceph-mon[79930]: pgmap v13486: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:51.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:57:50 smithi083 ceph-mon[77242]: pgmap v13486: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:51.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:57:50 smithi153 ceph-mon[70044]: pgmap v13486: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:51.838 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:57:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:57:51] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T12:57:52.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:57:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:57:51.835Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:57:52.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:57:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:57:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:57:53.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:57:52 smithi083 ceph-mon[79930]: pgmap v13487: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:53.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:57:52 smithi083 ceph-mon[77242]: pgmap v13487: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:53.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:57:52 smithi153 ceph-mon[70044]: pgmap v13487: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:53.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:57:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:57:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:57:55.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:57:54 smithi153 ceph-mon[70044]: pgmap v13488: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:55.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:57:54 smithi083 ceph-mon[79930]: pgmap v13488: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:55.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:57:54 smithi083 ceph-mon[77242]: pgmap v13488: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:55.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:57:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:57:55.202Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:57:56.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:57:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:57:55.922Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:57:56.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:57:56 smithi083 ceph-mon[79930]: pgmap v13489: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:56.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:57:56 smithi083 ceph-mon[77242]: pgmap v13489: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:57.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:57:56 smithi153 ceph-mon[70044]: pgmap v13489: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:58.125 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:57:58.414 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:57:58.415 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 2m ago 7h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:57:58.415 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 41s ago 7h 63.1M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:57:58.415 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 2m ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:57:58.416 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 41s ago 7h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:57:58.416 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 41s ago 7h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:57:58.416 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 2m ago 7h 888M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:57:58.416 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 2m ago 7h 1666M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:57:58.416 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 41s ago 7h 721M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:57:58.417 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 2m ago 7h 965M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:57:58.417 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 2m ago 7h 24.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:57:58.417 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 41s ago 7h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:57:58.417 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 2m ago 7h 1204M 2348M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:57:58.418 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 2m ago 7h 1191M 2348M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:57:58.418 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 2m ago 7h 906M 2348M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:57:58.418 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 2m ago 7h 1166M 2348M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:57:58.418 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 41s ago 7h 1025M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:57:58.418 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 41s ago 7h 1283M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:57:58.419 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 41s ago 7h 1146M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:57:58.419 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 41s ago 7h 1360M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:57:58.419 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 41s ago 7h 177M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:57:58.768 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:57:58.768 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:57:58.768 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:57:58.769 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:57:58.769 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:57:58.769 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:57:58.769 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:57:58.770 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:57:58.770 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:57:58.770 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:57:58.770 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:57:58.771 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:57:58.771 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:57:58.771 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:57:58.771 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:57:58.772 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:57:58.772 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:57:58.772 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:57:59.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:57:58 smithi083 ceph-mon[77242]: pgmap v13490: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:59.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:57:58 smithi083 ceph-mon[77242]: from='client.49818 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:57:59.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:57:58 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/142467336' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:57:59.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:57:58 smithi083 ceph-mon[79930]: pgmap v13490: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:59.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:57:58 smithi083 ceph-mon[79930]: from='client.49818 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:57:59.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:57:58 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/142467336' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:57:59.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:57:58 smithi153 ceph-mon[70044]: pgmap v13490: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:57:59.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:57:58 smithi153 ceph-mon[70044]: from='client.49818 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:57:59.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:57:58 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/142467336' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:58:00.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:57:59 smithi083 ceph-mon[79930]: from='client.59755 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:58:00.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:57:59 smithi083 ceph-mon[77242]: from='client.59755 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:58:00.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:57:59 smithi153 ceph-mon[70044]: from='client.59755 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:58:01.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:58:00 smithi083 ceph-mon[79930]: pgmap v13491: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:58:01.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:58:00 smithi083 ceph-mon[77242]: pgmap v13491: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:58:01.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:58:00 smithi153 ceph-mon[70044]: pgmap v13491: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:58:01.838 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:58:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:58:01] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T12:58:02.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:58:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:58:01.836Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:58:02.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:58:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:58:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:58:03.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:58:02 smithi153 ceph-mon[70044]: pgmap v13492: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:58:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:58:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:58:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:58:03.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:58:02 smithi083 ceph-mon[79930]: pgmap v13492: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:58:03.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:58:02 smithi083 ceph-mon[77242]: pgmap v13492: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:58:05.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:58:04 smithi153 ceph-mon[70044]: pgmap v13493: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:58:05.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:58:04 smithi083 ceph-mon[79930]: pgmap v13493: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:58:05.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:58:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:58:05.203Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:58:05.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:58:04 smithi083 ceph-mon[77242]: pgmap v13493: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:58:06.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:58:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:58:05.923Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:58:06.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:58:06 smithi083 ceph-mon[79930]: pgmap v13494: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:58:06.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:58:06 smithi083 ceph-mon[77242]: pgmap v13494: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:58:07.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:58:06 smithi153 ceph-mon[70044]: pgmap v13494: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:58:09.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:58:08 smithi083 ceph-mon[79930]: pgmap v13495: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:58:09.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:58:08 smithi083 ceph-mon[77242]: pgmap v13495: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:58:09.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:58:08 smithi153 ceph-mon[70044]: pgmap v13495: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:58:11.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:58:10 smithi083 ceph-mon[79930]: pgmap v13496: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:58:11.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:58:10 smithi083 ceph-mon[77242]: pgmap v13496: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:58:11.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:58:10 smithi153 ceph-mon[70044]: pgmap v13496: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:58:11.839 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:58:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:58:11] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T12:58:12.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:58:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:58:11.836Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:58:12.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:58:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:58:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:58:13.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:58:12 smithi153 ceph-mon[70044]: pgmap v13497: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:58:13.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:58:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:58:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:58:13.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:58:12 smithi083 ceph-mon[77242]: pgmap v13497: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:58:13.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:58:12 smithi083 ceph-mon[79930]: pgmap v13497: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:58:15.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:58:14 smithi153 ceph-mon[70044]: pgmap v13498: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:58:15.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:58:14 smithi083 ceph-mon[79930]: pgmap v13498: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:58:15.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:58:14 smithi083 ceph-mon[77242]: pgmap v13498: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:58:15.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:58:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:58:15.204Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:58:16.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:58:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:58:15.924Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:58:16.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:58:16 smithi083 ceph-mon[79930]: pgmap v13499: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:58:16.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:58:16 smithi083 ceph-mon[77242]: pgmap v13499: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:58:17.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:58:16 smithi153 ceph-mon[70044]: pgmap v13499: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:58:18.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:58:17 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:58:18.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:58:17 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:58:18.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:58:17 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:58:18.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:58:17 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:58:18.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:58:17 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:58:18.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:58:17 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:58:18.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:58:17 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:58:18.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:58:17 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:58:18.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:58:17 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:58:19.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:58:18 smithi153 ceph-mon[70044]: pgmap v13500: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:58:19.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:58:18 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:58:19.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:58:18 smithi083 ceph-mon[79930]: pgmap v13500: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:58:19.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:58:18 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:58:19.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:58:18 smithi083 ceph-mon[77242]: pgmap v13500: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:58:19.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:58:18 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:58:21.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:58:20 smithi153 ceph-mon[70044]: pgmap v13501: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:58:21.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:58:20 smithi083 ceph-mon[79930]: pgmap v13501: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:58:21.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:58:20 smithi083 ceph-mon[77242]: pgmap v13501: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:58:21.839 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:58:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:58:21] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T12:58:22.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:58:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:58:21.837Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:58:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:58:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:58:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:58:23.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:58:22 smithi153 ceph-mon[70044]: pgmap v13502: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:58:23.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:58:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:58:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:58:23.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:58:22 smithi083 ceph-mon[79930]: pgmap v13502: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:58:23.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:58:22 smithi083 ceph-mon[77242]: pgmap v13502: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:58:25.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:58:24 smithi153 ceph-mon[70044]: pgmap v13503: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:58:25.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:58:24 smithi083 ceph-mon[79930]: pgmap v13503: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:58:25.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:58:24 smithi083 ceph-mon[77242]: pgmap v13503: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:58:25.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:58:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:58:25.205Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:58:26.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:58:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:58:25.924Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:58:26.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:58:26 smithi083 ceph-mon[79930]: pgmap v13504: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:58:26.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:58:26 smithi083 ceph-mon[77242]: pgmap v13504: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:58:27.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:58:26 smithi153 ceph-mon[70044]: pgmap v13504: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:58:29.097 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:58:29.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:58:28 smithi153 ceph-mon[70044]: pgmap v13505: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:58:29.385 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:58:29.386 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 3m ago 7h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:58:29.387 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 72s ago 7h 63.1M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:58:29.387 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 3m ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:58:29.387 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 72s ago 7h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:58:29.388 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 72s ago 7h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:58:29.388 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 3m ago 7h 888M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:58:29.389 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 3m ago 7h 1666M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:58:29.389 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 72s ago 7h 721M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:58:29.389 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 3m ago 7h 965M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:58:29.390 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 3m ago 7h 24.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:58:29.390 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 72s ago 7h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:58:29.391 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 3m ago 7h 1204M 2348M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:58:29.391 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 3m ago 7h 1191M 2348M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:58:29.391 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 3m ago 7h 906M 2348M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:58:29.391 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 3m ago 7h 1166M 2348M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:58:29.392 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 72s ago 7h 1025M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:58:29.392 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 72s ago 7h 1283M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:58:29.392 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 72s ago 7h 1146M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:58:29.392 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 72s ago 7h 1360M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:58:29.393 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 72s ago 7h 177M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:58:29.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:58:28 smithi083 ceph-mon[79930]: pgmap v13505: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:58:29.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:58:28 smithi083 ceph-mon[77242]: pgmap v13505: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:58:29.738 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:58:29.738 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:58:29.739 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:58:29.739 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:58:29.739 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:58:29.739 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:58:29.740 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:58:29.740 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:58:29.740 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:58:29.740 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:58:29.740 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:58:29.741 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:58:29.741 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:58:29.741 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:58:29.741 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:58:29.741 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:58:29.742 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:58:29.742 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:58:30.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:58:29 smithi153 ceph-mon[70044]: from='client.49836 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:58:30.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:58:29 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2112683271' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:58:30.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:58:29 smithi083 ceph-mon[79930]: from='client.49836 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:58:30.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:58:29 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2112683271' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:58:30.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:58:29 smithi083 ceph-mon[77242]: from='client.49836 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:58:30.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:58:29 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2112683271' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:58:31.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:58:30 smithi153 ceph-mon[70044]: from='client.59773 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:58:31.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:58:30 smithi153 ceph-mon[70044]: pgmap v13506: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:58:31.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:58:30 smithi083 ceph-mon[79930]: from='client.59773 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:58:31.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:58:30 smithi083 ceph-mon[79930]: pgmap v13506: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:58:31.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:58:30 smithi083 ceph-mon[77242]: from='client.59773 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:58:31.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:58:30 smithi083 ceph-mon[77242]: pgmap v13506: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:58:31.840 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:58:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:58:31] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T12:58:32.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:58:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:58:31.837Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:58:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:58:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:58:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:58:33.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:58:32 smithi153 ceph-mon[70044]: pgmap v13507: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:58:33.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:58:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:58:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:58:33.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:58:32 smithi083 ceph-mon[77242]: pgmap v13507: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:58:33.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:58:32 smithi083 ceph-mon[79930]: pgmap v13507: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:58:35.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:58:34 smithi153 ceph-mon[70044]: pgmap v13508: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:58:35.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:58:34 smithi083 ceph-mon[79930]: pgmap v13508: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:58:35.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:58:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:58:35.206Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:58:35.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:58:34 smithi083 ceph-mon[77242]: pgmap v13508: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:58:36.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:58:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:58:35.925Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:58:36.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:58:36 smithi083 ceph-mon[79930]: pgmap v13509: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:58:36.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:58:36 smithi083 ceph-mon[77242]: pgmap v13509: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:58:37.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:58:36 smithi153 ceph-mon[70044]: pgmap v13509: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:58:39.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:58:38 smithi153 ceph-mon[70044]: pgmap v13510: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:58:39.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:58:38 smithi083 ceph-mon[79930]: pgmap v13510: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:58:39.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:58:38 smithi083 ceph-mon[77242]: pgmap v13510: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:58:41.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:58:40 smithi153 ceph-mon[70044]: pgmap v13511: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:58:41.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:58:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:58:41.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:58:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:58:41.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:58:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:58:41.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:58:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:58:41.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:58:40 smithi083 ceph-mon[77242]: pgmap v13511: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:58:41.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:58:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:58:41.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:58:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:58:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:58:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:58:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:58:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:58:41.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:58:40 smithi083 ceph-mon[79930]: pgmap v13511: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:58:41.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:58:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:58:41.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:58:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:58:41.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:58:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:58:41.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:58:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:58:41.840 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:58:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:58:41] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T12:58:42.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:58:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:58:41.838Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:58:42.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:58:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:58:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:58:43.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:58:42 smithi153 ceph-mon[70044]: pgmap v13512: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:58:43.310 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:58:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:58:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:58:43.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:58:42 smithi083 ceph-mon[79930]: pgmap v13512: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:58:43.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:58:42 smithi083 ceph-mon[77242]: pgmap v13512: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:58:45.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:58:44 smithi153 ceph-mon[70044]: pgmap v13513: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:58:45.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:58:44 smithi083 ceph-mon[79930]: pgmap v13513: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:58:45.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:58:44 smithi083 ceph-mon[77242]: pgmap v13513: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:58:45.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:58:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:58:45.206Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:58:46.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:58:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:58:45.925Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:58:46.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:58:46 smithi083 ceph-mon[79930]: pgmap v13514: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:58:46.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:58:46 smithi083 ceph-mon[77242]: pgmap v13514: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:58:47.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:58:46 smithi153 ceph-mon[70044]: pgmap v13514: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:58:49.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:58:48 smithi153 ceph-mon[70044]: pgmap v13515: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:58:49.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:58:48 smithi083 ceph-mon[77242]: pgmap v13515: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:58:49.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:58:48 smithi083 ceph-mon[79930]: pgmap v13515: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:58:51.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:58:50 smithi153 ceph-mon[70044]: pgmap v13516: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:58:51.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:58:50 smithi083 ceph-mon[79930]: pgmap v13516: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:58:51.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:58:50 smithi083 ceph-mon[77242]: pgmap v13516: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:58:51.841 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:58:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:58:51] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T12:58:52.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:58:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:58:51.839Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:58:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:58:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:58:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:58:53.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:58:52 smithi153 ceph-mon[70044]: pgmap v13517: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:58:53.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:58:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:58:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:58:53.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:58:52 smithi083 ceph-mon[79930]: pgmap v13517: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:58:53.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:58:52 smithi083 ceph-mon[77242]: pgmap v13517: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:58:55.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:58:54 smithi153 ceph-mon[70044]: pgmap v13518: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:58:55.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:58:54 smithi083 ceph-mon[77242]: pgmap v13518: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:58:55.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:58:54 smithi083 ceph-mon[79930]: pgmap v13518: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:58:55.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:58:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:58:55.207Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:58:56.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:58:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:58:55.927Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:58:56.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:58:56 smithi083 ceph-mon[79930]: pgmap v13519: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:58:56.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:58:56 smithi083 ceph-mon[77242]: pgmap v13519: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:58:57.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:58:56 smithi153 ceph-mon[70044]: pgmap v13519: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:58:59.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:58:58 smithi153 ceph-mon[70044]: pgmap v13520: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:58:59.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:58:58 smithi083 ceph-mon[77242]: pgmap v13520: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:58:59.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:58:58 smithi083 ceph-mon[79930]: pgmap v13520: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:00.067 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:59:00.358 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:59:00.358 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 3m ago 7h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:59:00.359 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 103s ago 7h 63.1M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:59:00.359 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 3m ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:59:00.359 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 103s ago 7h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:59:00.359 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 103s ago 7h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:59:00.360 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 3m ago 7h 888M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:59:00.360 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 3m ago 7h 1666M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:59:00.360 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 103s ago 7h 721M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:59:00.360 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 3m ago 7h 965M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:59:00.360 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 3m ago 7h 24.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:59:00.361 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 103s ago 7h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:59:00.361 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 3m ago 7h 1204M 2348M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:59:00.361 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 3m ago 7h 1191M 2348M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:59:00.361 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 3m ago 7h 906M 2348M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:59:00.362 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 3m ago 7h 1166M 2348M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:59:00.362 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 103s ago 7h 1025M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:59:00.362 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 103s ago 7h 1283M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:59:00.362 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 103s ago 7h 1146M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:59:00.362 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 103s ago 7h 1360M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:59:00.363 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 103s ago 7h 177M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:59:00.717 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:59:00.718 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:59:00.718 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:59:00.718 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:59:00.719 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:59:00.719 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:59:00.719 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:59:00.719 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:59:00.720 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:59:00.720 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:59:00.720 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:59:00.720 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:59:00.721 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:59:00.721 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:59:00.721 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:59:00.721 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:59:00.722 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:59:00.722 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:59:01.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:59:00 smithi153 ceph-mon[70044]: pgmap v13521: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:01.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:59:00 smithi153 ceph-mon[70044]: from='client.59785 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:59:01.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:59:00 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3453845315' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:59:01.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:59:00 smithi083 ceph-mon[79930]: pgmap v13521: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:01.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:59:00 smithi083 ceph-mon[79930]: from='client.59785 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:59:01.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:59:00 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3453845315' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:59:01.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:59:00 smithi083 ceph-mon[77242]: pgmap v13521: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:01.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:59:00 smithi083 ceph-mon[77242]: from='client.59785 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:59:01.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:59:00 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3453845315' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:59:01.842 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:59:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:59:01] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T12:59:02.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:59:01 smithi083 ceph-mon[79930]: from='client.59791 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:59:02.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:59:01 smithi083 ceph-mon[77242]: from='client.59791 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:59:02.214 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:59:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:59:01.840Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:59:02.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:59:01 smithi153 ceph-mon[70044]: from='client.59791 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:59:02.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:59:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:59:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:59:03.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:59:02 smithi153 ceph-mon[70044]: pgmap v13522: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:59:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:59:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:59:03.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:59:02 smithi083 ceph-mon[79930]: pgmap v13522: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:03.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:59:02 smithi083 ceph-mon[77242]: pgmap v13522: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:05.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:59:04 smithi153 ceph-mon[70044]: pgmap v13523: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:05.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:59:04 smithi083 ceph-mon[77242]: pgmap v13523: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:05.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:59:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:59:05.207Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:59:05.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:59:04 smithi083 ceph-mon[79930]: pgmap v13523: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:06.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:59:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:59:05.928Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:59:06.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:59:06 smithi083 ceph-mon[77242]: pgmap v13524: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:06.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:59:06 smithi083 ceph-mon[79930]: pgmap v13524: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:07.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:59:06 smithi153 ceph-mon[70044]: pgmap v13524: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:09.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:59:08 smithi153 ceph-mon[70044]: pgmap v13525: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:09.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:59:08 smithi083 ceph-mon[79930]: pgmap v13525: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:09.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:59:08 smithi083 ceph-mon[77242]: pgmap v13525: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:11.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:59:10 smithi153 ceph-mon[70044]: pgmap v13526: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:11.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:59:10 smithi083 ceph-mon[79930]: pgmap v13526: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:11.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:59:10 smithi083 ceph-mon[77242]: pgmap v13526: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:11.843 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:59:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:59:11] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T12:59:12.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:59:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:59:11.840Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:59:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:59:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:59:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:59:13.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:59:12 smithi153 ceph-mon[70044]: pgmap v13527: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:13.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:59:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:59:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:59:13.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:59:12 smithi083 ceph-mon[79930]: pgmap v13527: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:13.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:59:12 smithi083 ceph-mon[77242]: pgmap v13527: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:15.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:59:14 smithi153 ceph-mon[70044]: pgmap v13528: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:15.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:59:14 smithi083 ceph-mon[79930]: pgmap v13528: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:15.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:59:14 smithi083 ceph-mon[77242]: pgmap v13528: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:15.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:59:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:59:15.208Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:59:16.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:59:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:59:15.928Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:59:16.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:59:16 smithi083 ceph-mon[79930]: pgmap v13529: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:16.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:59:16 smithi083 ceph-mon[77242]: pgmap v13529: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:17.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:59:16 smithi153 ceph-mon[70044]: pgmap v13529: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:19.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:59:18 smithi153 ceph-mon[70044]: pgmap v13530: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:19.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:59:18 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:59:19.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:59:18 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:59:19.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:59:18 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:59:19.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:59:18 smithi083 ceph-mon[79930]: pgmap v13530: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:19.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:59:18 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:59:19.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:59:18 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:59:19.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:59:18 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:59:19.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:59:18 smithi083 ceph-mon[77242]: pgmap v13530: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:19.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:59:18 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T12:59:19.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:59:18 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T12:59:19.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:59:18 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T12:59:20.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:59:20 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:59:20.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:59:20 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:59:20.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:59:20 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T12:59:21.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:59:21 smithi083 ceph-mon[77242]: pgmap v13531: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:21.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:59:21 smithi083 ceph-mon[79930]: pgmap v13531: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:21.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:59:21 smithi153 ceph-mon[70044]: pgmap v13531: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:21.844 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:59:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:59:21] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T12:59:22.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:59:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:59:21.841Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:59:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:59:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:59:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:59:23.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:59:23 smithi153 ceph-mon[70044]: pgmap v13532: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:23.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:59:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:59:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:59:23.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:59:23 smithi083 ceph-mon[77242]: pgmap v13532: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:23.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:59:23 smithi083 ceph-mon[79930]: pgmap v13532: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:25.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:59:25 smithi083 ceph-mon[79930]: pgmap v13533: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:25.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:59:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:59:25.209Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:59:25.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:59:25 smithi083 ceph-mon[77242]: pgmap v13533: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:25.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:59:25 smithi153 ceph-mon[70044]: pgmap v13533: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:26.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:59:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:59:25.929Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:59:26.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:59:26 smithi083 ceph-mon[79930]: pgmap v13534: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:26.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:59:26 smithi083 ceph-mon[77242]: pgmap v13534: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:27.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:59:26 smithi153 ceph-mon[70044]: pgmap v13534: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:29.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:59:28 smithi153 ceph-mon[70044]: pgmap v13535: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:29.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:59:28 smithi083 ceph-mon[79930]: pgmap v13535: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:29.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:59:28 smithi083 ceph-mon[77242]: pgmap v13535: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:31.041 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T12:59:31.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:59:30 smithi153 ceph-mon[70044]: pgmap v13536: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:31.327 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T12:59:31.328 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 4m ago 7h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T12:59:31.328 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 2m ago 7h 63.1M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T12:59:31.328 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 4m ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T12:59:31.328 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 2m ago 7h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T12:59:31.328 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 2m ago 7h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T12:59:31.329 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 4m ago 7h 888M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T12:59:31.329 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 4m ago 7h 1666M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T12:59:31.329 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 2m ago 7h 721M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T12:59:31.329 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 4m ago 7h 965M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T12:59:31.330 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 4m ago 7h 24.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T12:59:31.330 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 2m ago 7h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T12:59:31.330 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 4m ago 7h 1204M 2348M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T12:59:31.330 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 4m ago 7h 1191M 2348M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T12:59:31.331 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 4m ago 7h 906M 2348M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T12:59:31.331 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 4m ago 7h 1166M 2348M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T12:59:31.331 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 2m ago 7h 1025M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T12:59:31.331 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 2m ago 7h 1283M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T12:59:31.331 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 2m ago 7h 1146M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T12:59:31.332 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 2m ago 7h 1360M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T12:59:31.332 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 2m ago 7h 177M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T12:59:31.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:59:30 smithi083 ceph-mon[79930]: pgmap v13536: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:31.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:59:30 smithi083 ceph-mon[77242]: pgmap v13536: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:31.683 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T12:59:31.683 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T12:59:31.684 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T12:59:31.684 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:59:31.684 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T12:59:31.684 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T12:59:31.685 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:59:31.685 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T12:59:31.685 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T12:59:31.685 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:59:31.685 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T12:59:31.686 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T12:59:31.686 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T12:59:31.686 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T12:59:31.686 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T12:59:31.687 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T12:59:31.687 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T12:59:31.687 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T12:59:31.844 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:59:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:59:31] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T12:59:32.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:59:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:59:31.841Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:59:32.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:59:31 smithi083 ceph-mon[77242]: from='client.59803 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:59:32.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:59:31 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2103487476' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:59:32.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:59:31 smithi083 ceph-mon[79930]: from='client.59803 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:59:32.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:59:31 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2103487476' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:59:32.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:59:31 smithi153 ceph-mon[70044]: from='client.59803 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:59:32.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:59:31 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2103487476' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T12:59:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:59:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:59:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:59:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:59:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:59:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:59:33.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:59:32 smithi153 ceph-mon[70044]: from='client.59809 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:59:33.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:59:32 smithi153 ceph-mon[70044]: pgmap v13537: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:33.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:59:32 smithi083 ceph-mon[79930]: from='client.59809 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:59:33.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:59:32 smithi083 ceph-mon[79930]: pgmap v13537: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:33.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:59:32 smithi083 ceph-mon[77242]: from='client.59809 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T12:59:33.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:59:32 smithi083 ceph-mon[77242]: pgmap v13537: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:35.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:59:34 smithi153 ceph-mon[70044]: pgmap v13538: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:35.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:59:34 smithi083 ceph-mon[79930]: pgmap v13538: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:35.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:59:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:59:35.210Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:59:35.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:59:34 smithi083 ceph-mon[77242]: pgmap v13538: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:36.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:59:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:59:35.929Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:59:36.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:59:36 smithi083 ceph-mon[79930]: pgmap v13539: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:36.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:59:36 smithi083 ceph-mon[77242]: pgmap v13539: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:37.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:59:36 smithi153 ceph-mon[70044]: pgmap v13539: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:39.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:59:38 smithi153 ceph-mon[70044]: pgmap v13540: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:39.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:59:38 smithi083 ceph-mon[79930]: pgmap v13540: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:39.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:59:38 smithi083 ceph-mon[77242]: pgmap v13540: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:59:40 smithi153 ceph-mon[70044]: pgmap v13541: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:41.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:59:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:59:41.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:59:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:59:41.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:59:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:59:41.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:59:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:59:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:59:40 smithi083 ceph-mon[79930]: pgmap v13541: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:59:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:59:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:59:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:59:41.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:59:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:59:41.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:59:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:59:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:59:40 smithi083 ceph-mon[77242]: pgmap v13541: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:59:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:59:41.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:59:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:59:41.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:59:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T12:59:41.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:59:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T12:59:41.845 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:59:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:59:41] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T12:59:42.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:59:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:59:41.842Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:59:42.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:59:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:59:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:59:43.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:59:42 smithi153 ceph-mon[70044]: pgmap v13542: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:43.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:59:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:59:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:59:43.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:59:42 smithi083 ceph-mon[79930]: pgmap v13542: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:43.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:59:42 smithi083 ceph-mon[77242]: pgmap v13542: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:45.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:59:44 smithi153 ceph-mon[70044]: pgmap v13543: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:45.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:59:44 smithi083 ceph-mon[79930]: pgmap v13543: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:45.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:59:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:59:45.211Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:59:45.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:59:44 smithi083 ceph-mon[77242]: pgmap v13543: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:46.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:59:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:59:45.930Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:59:46.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:59:46 smithi083 ceph-mon[79930]: pgmap v13544: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:46.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:59:46 smithi083 ceph-mon[77242]: pgmap v13544: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:47.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:59:46 smithi153 ceph-mon[70044]: pgmap v13544: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:49.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:59:48 smithi153 ceph-mon[70044]: pgmap v13545: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:49.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:59:48 smithi083 ceph-mon[79930]: pgmap v13545: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:49.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:59:48 smithi083 ceph-mon[77242]: pgmap v13545: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:51.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:59:50 smithi153 ceph-mon[70044]: pgmap v13546: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:51.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:59:50 smithi083 ceph-mon[79930]: pgmap v13546: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:51.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:59:50 smithi083 ceph-mon[77242]: pgmap v13546: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:51.845 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 12:59:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:59:51] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T12:59:52.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:59:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:59:51.843Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:59:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 12:59:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T12:59:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T12:59:53.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:59:52 smithi153 ceph-mon[70044]: pgmap v13547: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:53.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 12:59:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:12:59:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T12:59:53.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:59:52 smithi083 ceph-mon[79930]: pgmap v13547: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:53.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:59:52 smithi083 ceph-mon[77242]: pgmap v13547: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:55.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:59:54 smithi153 ceph-mon[70044]: pgmap v13548: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:55.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:59:54 smithi083 ceph-mon[79930]: pgmap v13548: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:55.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:59:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:59:55.210Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:59:55.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:59:54 smithi083 ceph-mon[77242]: pgmap v13548: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:56.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 12:59:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T12:59:55.931Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T12:59:56.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:59:56 smithi083 ceph-mon[77242]: pgmap v13549: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:56.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:59:56 smithi083 ceph-mon[79930]: pgmap v13549: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:57.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:59:56 smithi153 ceph-mon[70044]: pgmap v13549: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:59.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 12:59:58 smithi153 ceph-mon[70044]: pgmap v13550: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:59.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 12:59:58 smithi083 ceph-mon[79930]: pgmap v13550: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T12:59:59.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 12:59:58 smithi083 ceph-mon[77242]: pgmap v13550: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:00.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:00:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[77219]: 2022-09-29T12:59:59.999+0000 7f160f665700 -1 log_channel(cluster) log [ERR] : overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T13:00:01.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:00:00 smithi153 ceph-mon[70044]: pgmap v13551: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:01.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:00:00 smithi153 ceph-mon[70044]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T13:00:01.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:00:00 smithi083 ceph-mon[79930]: pgmap v13551: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:01.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:00:00 smithi083 ceph-mon[79930]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T13:00:01.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:00:00 smithi083 ceph-mon[77242]: pgmap v13551: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:01.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:00:00 smithi083 ceph-mon[77242]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T13:00:01.845 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:00:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:00:01] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T13:00:02.010 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:00:02.058 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:00:01 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:00:01.564Z caller=compact.go:519 level=info component=tsdb msg="write block" mint=1664445600138 maxt=1664452800000 ulid=01GE4NA8KA55YEQREYPDKD6DBD duration=49.842803ms 2022-09-29T13:00:02.058 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:00:01 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:00:01.567Z caller=head.go:827 level=info component=tsdb msg="Head GC completed" duration=3.004216ms 2022-09-29T13:00:02.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:00:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:00:01.843Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:00:02.296 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:00:02.297 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 4m ago 7h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:00:02.297 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 2m ago 7h 63.1M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:00:02.297 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 4m ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:00:02.297 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 2m ago 7h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:00:02.297 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 2m ago 7h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:00:02.298 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 4m ago 7h 888M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:00:02.298 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 4m ago 7h 1666M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:00:02.298 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 2m ago 7h 721M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:00:02.298 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 4m ago 7h 965M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:00:02.299 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 4m ago 7h 24.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:00:02.299 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 2m ago 7h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:00:02.299 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 4m ago 7h 1204M 2348M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:00:02.299 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 4m ago 7h 1191M 2348M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:00:02.299 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 4m ago 7h 906M 2348M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:00:02.300 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 4m ago 7h 1166M 2348M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:00:02.300 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 2m ago 7h 1025M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:00:02.300 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 2m ago 7h 1283M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:00:02.300 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 2m ago 7h 1146M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:00:02.300 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 2m ago 7h 1360M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:00:02.301 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 2m ago 7h 177M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:00:02.650 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:00:02.650 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:00:02.651 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:00:02.651 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:00:02.651 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:00:02.651 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:00:02.652 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:00:02.652 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:00:02.652 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:00:02.652 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:00:02.653 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:00:02.653 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:00:02.653 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:00:02.653 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:00:02.653 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:00:02.654 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:00:02.654 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:00:02.654 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:00:02.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:00:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:00:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:00:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:00:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:00:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:00:03.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:00:02 smithi153 ceph-mon[70044]: pgmap v13552: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:03.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:00:02 smithi153 ceph-mon[70044]: from='client.59821 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:00:03.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:00:02 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2262611505' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:00:03.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:00:02 smithi083 ceph-mon[77242]: pgmap v13552: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:03.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:00:02 smithi083 ceph-mon[77242]: from='client.59821 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:00:03.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:00:02 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2262611505' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:00:03.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:00:02 smithi083 ceph-mon[79930]: pgmap v13552: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:03.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:00:02 smithi083 ceph-mon[79930]: from='client.59821 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:00:03.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:00:02 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2262611505' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:00:04.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:00:04 smithi153 ceph-mon[70044]: from='client.59827 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:00:04.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:00:04 smithi083 ceph-mon[79930]: from='client.59827 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:00:04.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:00:04 smithi083 ceph-mon[77242]: from='client.59827 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:00:05.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:00:05 smithi153 ceph-mon[70044]: pgmap v13553: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:05.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:00:05 smithi083 ceph-mon[79930]: pgmap v13553: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:05.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:00:05 smithi083 ceph-mon[77242]: pgmap v13553: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:05.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:00:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:00:05.210Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:00:06.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:00:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:00:05.932Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:00:06.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:00:06 smithi083 ceph-mon[79930]: pgmap v13554: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:06.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:00:06 smithi083 ceph-mon[77242]: pgmap v13554: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:07.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:00:06 smithi153 ceph-mon[70044]: pgmap v13554: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:09.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:00:08 smithi153 ceph-mon[70044]: pgmap v13555: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:09.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:00:08 smithi083 ceph-mon[77242]: pgmap v13555: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:09.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:00:08 smithi083 ceph-mon[79930]: pgmap v13555: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:11.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:00:10 smithi153 ceph-mon[70044]: pgmap v13556: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:11.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:00:10 smithi083 ceph-mon[79930]: pgmap v13556: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:11.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:00:10 smithi083 ceph-mon[77242]: pgmap v13556: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:11.847 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:00:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:00:11] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T13:00:12.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:00:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:00:11.844Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:00:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:00:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:00:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:00:13.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:00:13 smithi153 ceph-mon[70044]: pgmap v13557: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:13.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:00:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:00:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:00:13.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:00:13 smithi083 ceph-mon[77242]: pgmap v13557: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:13.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:00:13 smithi083 ceph-mon[79930]: pgmap v13557: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:15.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:00:15 smithi153 ceph-mon[70044]: pgmap v13558: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:15.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:00:15 smithi083 ceph-mon[79930]: pgmap v13558: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:15.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:00:15 smithi083 ceph-mon[77242]: pgmap v13558: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:15.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:00:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:00:15.211Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:00:16.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:00:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:00:15.932Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:00:16.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:00:16 smithi083 ceph-mon[79930]: pgmap v13559: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:16.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:00:16 smithi083 ceph-mon[77242]: pgmap v13559: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:17.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:00:16 smithi153 ceph-mon[70044]: pgmap v13559: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:19.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:00:18 smithi153 ceph-mon[70044]: pgmap v13560: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:19.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:00:18 smithi083 ceph-mon[77242]: pgmap v13560: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:19.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:00:19 smithi083 ceph-mon[79930]: pgmap v13560: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:20.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:00:20 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:00:20.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:00:20 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:00:20.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:00:20 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:00:20.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:00:20 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:00:20.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:00:20 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:00:20.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:00:20 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:00:20.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:00:20 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:00:20.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:00:20 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:00:20.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:00:20 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:00:20.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:00:20 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:00:20.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:00:20 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:00:20.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:00:20 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:00:21.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:00:21 smithi153 ceph-mon[70044]: pgmap v13561: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:21.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:00:21 smithi083 ceph-mon[79930]: pgmap v13561: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:21.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:00:21 smithi083 ceph-mon[77242]: pgmap v13561: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:21.848 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:00:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:00:21] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T13:00:22.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:00:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:00:21.845Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:00:22.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:00:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:00:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:00:23.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:00:23 smithi153 ceph-mon[70044]: pgmap v13562: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:23.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:00:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:00:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:00:23.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:00:23 smithi083 ceph-mon[77242]: pgmap v13562: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:23.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:00:23 smithi083 ceph-mon[79930]: pgmap v13562: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:25.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:00:25 smithi153 ceph-mon[70044]: pgmap v13563: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:25.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:00:25 smithi083 ceph-mon[77242]: pgmap v13563: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:25.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:00:25 smithi083 ceph-mon[79930]: pgmap v13563: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:25.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:00:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:00:25.212Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:00:26.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:00:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:00:25.933Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:00:26.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:00:26 smithi083 ceph-mon[79930]: pgmap v13564: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:26.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:00:26 smithi083 ceph-mon[77242]: pgmap v13564: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:27.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:00:26 smithi153 ceph-mon[70044]: pgmap v13564: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:29.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:00:29 smithi153 ceph-mon[70044]: pgmap v13565: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:29.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:00:29 smithi083 ceph-mon[79930]: pgmap v13565: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:29.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:00:29 smithi083 ceph-mon[77242]: pgmap v13565: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:31.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:00:31 smithi153 ceph-mon[70044]: pgmap v13566: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:31.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:00:31 smithi083 ceph-mon[79930]: pgmap v13566: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:31.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:00:31 smithi083 ceph-mon[77242]: pgmap v13566: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:31.847 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:00:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:00:31] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T13:00:32.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:00:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:00:31.845Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:00:32.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:00:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:00:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:00:32.974 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:00:33.264 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:00:33.265 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 5m ago 7h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:00:33.265 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 3m ago 7h 63.1M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:00:33.265 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 5m ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:00:33.266 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 3m ago 7h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:00:33.266 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 3m ago 7h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:00:33.266 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 5m ago 7h 888M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:00:33.266 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 5m ago 7h 1666M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:00:33.266 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 3m ago 7h 721M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:00:33.267 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 5m ago 7h 965M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:00:33.267 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 5m ago 7h 24.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:00:33.267 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 3m ago 7h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:00:33.267 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 5m ago 7h 1204M 2348M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:00:33.268 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 5m ago 7h 1191M 2348M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:00:33.268 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 5m ago 7h 906M 2348M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:00:33.268 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 5m ago 7h 1166M 2348M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:00:33.268 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 3m ago 7h 1025M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:00:33.268 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 3m ago 7h 1283M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:00:33.269 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 3m ago 7h 1146M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:00:33.269 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 3m ago 7h 1360M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:00:33.269 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 3m ago 7h 177M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:00:33.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:00:33 smithi153 ceph-mon[70044]: pgmap v13567: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:33.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:00:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:00:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:00:33.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:00:33 smithi083 ceph-mon[79930]: pgmap v13567: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:33.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:00:33 smithi083 ceph-mon[77242]: pgmap v13567: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:33.623 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:00:33.623 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:00:33.624 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:00:33.624 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:00:33.624 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:00:33.624 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:00:33.624 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:00:33.625 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:00:33.625 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:00:33.625 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:00:33.625 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:00:33.626 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:00:33.626 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:00:33.626 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:00:33.626 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:00:33.626 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:00:33.627 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:00:33.627 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:00:34.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:00:34 smithi153 ceph-mon[70044]: from='client.59839 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:00:34.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:00:34 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3530328237' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:00:34.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:00:34 smithi083 ceph-mon[79930]: from='client.59839 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:00:34.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:00:34 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3530328237' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:00:34.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:00:34 smithi083 ceph-mon[77242]: from='client.59839 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:00:34.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:00:34 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3530328237' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:00:35.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:00:35 smithi153 ceph-mon[70044]: from='client.59845 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:00:35.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:00:35 smithi153 ceph-mon[70044]: pgmap v13568: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:35.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:00:35 smithi083 ceph-mon[77242]: from='client.59845 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:00:35.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:00:35 smithi083 ceph-mon[77242]: pgmap v13568: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:35.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:00:35 smithi083 ceph-mon[79930]: from='client.59845 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:00:35.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:00:35 smithi083 ceph-mon[79930]: pgmap v13568: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:35.465 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:00:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:00:35.213Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:00:36.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:00:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:00:35.934Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:00:36.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:00:36 smithi083 ceph-mon[79930]: pgmap v13569: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:36.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:00:36 smithi083 ceph-mon[77242]: pgmap v13569: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:37.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:00:36 smithi153 ceph-mon[70044]: pgmap v13569: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:39.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:00:39 smithi153 ceph-mon[70044]: pgmap v13570: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:39.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:00:39 smithi083 ceph-mon[79930]: pgmap v13570: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:39.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:00:39 smithi083 ceph-mon[77242]: pgmap v13570: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:00:41 smithi153 ceph-mon[70044]: pgmap v13571: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:41.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:00:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:00:41.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:00:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:00:41.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:00:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:00:41.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:00:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:00:41.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:00:41 smithi083 ceph-mon[77242]: pgmap v13571: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:41.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:00:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:00:41.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:00:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:00:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:00:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:00:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:00:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:00:41.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:00:41 smithi083 ceph-mon[79930]: pgmap v13571: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:41.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:00:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:00:41.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:00:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:00:41.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:00:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:00:41.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:00:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:00:41.848 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:00:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:00:41] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T13:00:42.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:00:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:00:41.846Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:00:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:00:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:00:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:00:43.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:00:43 smithi153 ceph-mon[70044]: pgmap v13572: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:43.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:00:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:00:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:00:43.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:00:43 smithi083 ceph-mon[79930]: pgmap v13572: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:43.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:00:43 smithi083 ceph-mon[77242]: pgmap v13572: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:45.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:00:45 smithi153 ceph-mon[70044]: pgmap v13573: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:45.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:00:45 smithi083 ceph-mon[79930]: pgmap v13573: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:45.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:00:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:00:45.214Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:00:45.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:00:45 smithi083 ceph-mon[77242]: pgmap v13573: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:46.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:00:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:00:45.934Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:00:46.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:00:46 smithi083 ceph-mon[79930]: pgmap v13574: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:46.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:00:46 smithi083 ceph-mon[77242]: pgmap v13574: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:47.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:00:46 smithi153 ceph-mon[70044]: pgmap v13574: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:49.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:00:49 smithi153 ceph-mon[70044]: pgmap v13575: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:49.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:00:49 smithi083 ceph-mon[79930]: pgmap v13575: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:49.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:00:49 smithi083 ceph-mon[77242]: pgmap v13575: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:51.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:00:51 smithi153 ceph-mon[70044]: pgmap v13576: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:51.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:00:51 smithi083 ceph-mon[79930]: pgmap v13576: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:51.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:00:51 smithi083 ceph-mon[77242]: pgmap v13576: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:51.848 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:00:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:00:51] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T13:00:52.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:00:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:00:51.846Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:00:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:00:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:00:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:00:53.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:00:53 smithi153 ceph-mon[70044]: pgmap v13577: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:53.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:00:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:00:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:00:53.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:00:53 smithi083 ceph-mon[79930]: pgmap v13577: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:53.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:00:53 smithi083 ceph-mon[77242]: pgmap v13577: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:55.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:00:55 smithi153 ceph-mon[70044]: pgmap v13578: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:55.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:00:55 smithi083 ceph-mon[79930]: pgmap v13578: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:55.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:00:55 smithi083 ceph-mon[77242]: pgmap v13578: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:55.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:00:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:00:55.214Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:00:56.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:00:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:00:55.935Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:00:56.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:00:56 smithi083 ceph-mon[77242]: pgmap v13579: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:56.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:00:56 smithi083 ceph-mon[79930]: pgmap v13579: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:57.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:00:56 smithi153 ceph-mon[70044]: pgmap v13579: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:59.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:00:59 smithi153 ceph-mon[70044]: pgmap v13580: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:59.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:00:59 smithi083 ceph-mon[79930]: pgmap v13580: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:00:59.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:00:59 smithi083 ceph-mon[77242]: pgmap v13580: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:01.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:01:01 smithi153 ceph-mon[70044]: pgmap v13581: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:01.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:01:01 smithi083 ceph-mon[79930]: pgmap v13581: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:01.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:01:01 smithi083 ceph-mon[77242]: pgmap v13581: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:01.849 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:01:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:01:01] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T13:01:02.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:01:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:01:01.846Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:01:02.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:01:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:01:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:01:03.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:01:03 smithi153 ceph-mon[70044]: pgmap v13582: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:01:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:01:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:01:03.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:01:03 smithi083 ceph-mon[77242]: pgmap v13582: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:03.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:01:03 smithi083 ceph-mon[79930]: pgmap v13582: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:03.952 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:01:04.240 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:01:04.240 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 5m ago 7h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:01:04.241 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 3m ago 7h 63.1M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:01:04.241 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 5m ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:01:04.241 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 3m ago 7h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:01:04.241 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 3m ago 7h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:01:04.242 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 5m ago 7h 888M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:01:04.242 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 5m ago 7h 1666M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:01:04.242 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 3m ago 7h 721M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:01:04.242 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 5m ago 7h 965M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:01:04.243 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 5m ago 7h 24.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:01:04.243 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 3m ago 7h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:01:04.243 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 5m ago 7h 1204M 2348M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:01:04.243 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 5m ago 7h 1191M 2348M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:01:04.244 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 5m ago 7h 906M 2348M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:01:04.244 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 5m ago 7h 1166M 2348M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:01:04.244 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 3m ago 7h 1025M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:01:04.244 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 3m ago 7h 1283M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:01:04.244 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 3m ago 7h 1146M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:01:04.245 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 3m ago 7h 1360M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:01:04.245 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 3m ago 7h 177M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:01:04.597 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:01:04.597 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:01:04.598 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:01:04.598 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:01:04.598 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:01:04.598 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:01:04.599 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:01:04.599 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:01:04.599 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:01:04.599 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:01:04.600 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:01:04.600 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:01:04.600 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:01:04.600 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:01:04.601 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:01:04.601 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:01:04.601 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:01:04.601 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:01:05.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:01:05 smithi153 ceph-mon[70044]: from='client.59857 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:01:05.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:01:05 smithi153 ceph-mon[70044]: pgmap v13583: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:05.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:01:05 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/92499898' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:01:05.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:01:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:01:05.215Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:01:05.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:01:05 smithi083 ceph-mon[77242]: from='client.59857 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:01:05.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:01:05 smithi083 ceph-mon[77242]: pgmap v13583: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:05.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:01:05 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/92499898' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:01:05.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:01:05 smithi083 ceph-mon[79930]: from='client.59857 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:01:05.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:01:05 smithi083 ceph-mon[79930]: pgmap v13583: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:05.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:01:05 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/92499898' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:01:06.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:01:06 smithi083 ceph-mon[79930]: from='client.59863 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:01:06.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:01:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:01:05.935Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:01:06.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:01:06 smithi083 ceph-mon[77242]: from='client.59863 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:01:06.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:01:06 smithi153 ceph-mon[70044]: from='client.59863 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:01:07.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:01:07 smithi153 ceph-mon[70044]: pgmap v13584: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:07.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:01:07 smithi083 ceph-mon[79930]: pgmap v13584: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:07.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:01:07 smithi083 ceph-mon[77242]: pgmap v13584: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:09.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:01:09 smithi153 ceph-mon[70044]: pgmap v13585: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:09.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:01:09 smithi083 ceph-mon[79930]: pgmap v13585: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:09.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:01:09 smithi083 ceph-mon[77242]: pgmap v13585: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:11.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:01:11 smithi153 ceph-mon[70044]: pgmap v13586: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:11.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:01:11 smithi083 ceph-mon[79930]: pgmap v13586: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:11.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:01:11 smithi083 ceph-mon[77242]: pgmap v13586: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:11.850 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:01:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:01:11] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T13:01:12.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:01:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:01:11.847Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:01:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:01:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:01:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:01:13.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:01:13 smithi153 ceph-mon[70044]: pgmap v13587: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:13.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:01:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:01:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:01:13.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:01:13 smithi083 ceph-mon[79930]: pgmap v13587: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:13.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:01:13 smithi083 ceph-mon[77242]: pgmap v13587: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:15.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:01:15 smithi153 ceph-mon[70044]: pgmap v13588: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:15.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:01:15 smithi083 ceph-mon[79930]: pgmap v13588: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:15.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:01:15 smithi083 ceph-mon[77242]: pgmap v13588: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:15.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:01:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:01:15.216Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:01:16.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:01:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:01:15.936Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:01:16.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:01:16 smithi083 ceph-mon[79930]: pgmap v13589: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:16.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:01:16 smithi083 ceph-mon[77242]: pgmap v13589: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:17.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:01:16 smithi153 ceph-mon[70044]: pgmap v13589: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:19.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:01:19 smithi153 ceph-mon[70044]: pgmap v13590: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:19.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:01:19 smithi083 ceph-mon[79930]: pgmap v13590: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:19.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:01:19 smithi083 ceph-mon[77242]: pgmap v13590: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:20.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:01:20 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:01:20.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:01:20 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:01:20.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:01:20 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:01:20.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:01:20 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:01:20.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:01:20 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:01:20.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:01:20 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:01:20.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:01:20 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:01:20.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:01:20 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:01:20.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:01:20 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:01:21.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:01:21 smithi153 ceph-mon[70044]: pgmap v13591: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:21.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:01:21 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:01:21.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:01:21 smithi083 ceph-mon[79930]: pgmap v13591: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:21.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:01:21 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:01:21.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:01:21 smithi083 ceph-mon[77242]: pgmap v13591: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:21.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:01:21 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:01:21.850 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:01:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:01:21] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T13:01:22.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:01:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:01:21.848Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:01:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:01:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:01:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:01:23.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:01:23 smithi153 ceph-mon[70044]: pgmap v13592: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:23.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:01:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:01:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:01:23.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:01:23 smithi083 ceph-mon[79930]: pgmap v13592: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:23.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:01:23 smithi083 ceph-mon[77242]: pgmap v13592: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:25.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:01:25 smithi153 ceph-mon[70044]: pgmap v13593: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:25.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:01:25 smithi083 ceph-mon[79930]: pgmap v13593: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:25.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:01:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:01:25.216Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:01:25.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:01:25 smithi083 ceph-mon[77242]: pgmap v13593: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:26.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:01:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:01:25.937Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:01:26.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:01:26 smithi083 ceph-mon[79930]: pgmap v13594: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:26.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:01:26 smithi083 ceph-mon[77242]: pgmap v13594: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:27.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:01:26 smithi153 ceph-mon[70044]: pgmap v13594: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:29.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:01:29 smithi153 ceph-mon[70044]: pgmap v13595: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:29.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:01:29 smithi083 ceph-mon[79930]: pgmap v13595: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:29.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:01:29 smithi083 ceph-mon[77242]: pgmap v13595: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:31.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:01:31 smithi153 ceph-mon[70044]: pgmap v13596: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:31.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:01:31 smithi083 ceph-mon[79930]: pgmap v13596: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:31.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:01:31 smithi083 ceph-mon[77242]: pgmap v13596: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:31.851 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:01:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:01:31] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T13:01:32.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:01:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:01:31.848Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:01:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:01:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:01:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:01:33.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:01:33 smithi153 ceph-mon[70044]: pgmap v13597: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:33.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:01:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:01:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:01:33.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:01:33 smithi083 ceph-mon[79930]: pgmap v13597: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:33.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:01:33 smithi083 ceph-mon[77242]: pgmap v13597: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:34.929 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:01:35.218 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:01:35.218 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 6m ago 7h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:01:35.219 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 4m ago 7h 63.1M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:01:35.219 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 6m ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:01:35.219 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 4m ago 7h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:01:35.219 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 4m ago 7h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:01:35.220 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 6m ago 7h 888M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:01:35.220 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 6m ago 7h 1666M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:01:35.220 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 4m ago 7h 721M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:01:35.220 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 6m ago 7h 965M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:01:35.221 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 6m ago 7h 24.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:01:35.221 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 4m ago 7h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:01:35.221 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 6m ago 7h 1204M 2348M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:01:35.221 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 6m ago 7h 1191M 2348M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:01:35.221 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 6m ago 7h 906M 2348M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:01:35.222 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 6m ago 7h 1166M 2348M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:01:35.222 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 4m ago 7h 1025M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:01:35.222 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 4m ago 7h 1283M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:01:35.222 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 4m ago 7h 1146M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:01:35.223 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 4m ago 7h 1360M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:01:35.223 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 4m ago 7h 177M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:01:35.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:01:35 smithi153 ceph-mon[70044]: pgmap v13598: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:35.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:01:35 smithi083 ceph-mon[79930]: pgmap v13598: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:35.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:01:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:01:35.218Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:01:35.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:01:35 smithi083 ceph-mon[77242]: pgmap v13598: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:35.572 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:01:35.572 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:01:35.573 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:01:35.573 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:01:35.573 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:01:35.573 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:01:35.574 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:01:35.574 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:01:35.574 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:01:35.574 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:01:35.575 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:01:35.575 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:01:35.575 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:01:35.575 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:01:35.576 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:01:35.576 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:01:35.576 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:01:35.576 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:01:36.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:01:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:01:35.937Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:01:36.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:01:36 smithi083 ceph-mon[77242]: from='client.59875 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:01:36.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:01:36 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1204935642' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:01:36.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:01:36 smithi083 ceph-mon[79930]: from='client.59875 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:01:36.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:01:36 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1204935642' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:01:36.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:01:36 smithi153 ceph-mon[70044]: from='client.59875 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:01:36.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:01:36 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1204935642' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:01:37.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:01:37 smithi153 ceph-mon[70044]: from='client.59881 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:01:37.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:01:37 smithi153 ceph-mon[70044]: pgmap v13599: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:37.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:01:37 smithi083 ceph-mon[77242]: from='client.59881 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:01:37.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:01:37 smithi083 ceph-mon[77242]: pgmap v13599: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:37.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:01:37 smithi083 ceph-mon[79930]: from='client.59881 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:01:37.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:01:37 smithi083 ceph-mon[79930]: pgmap v13599: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:39.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:01:39 smithi153 ceph-mon[70044]: pgmap v13600: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:39.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:01:39 smithi083 ceph-mon[79930]: pgmap v13600: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:39.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:01:39 smithi083 ceph-mon[77242]: pgmap v13600: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:01:41 smithi153 ceph-mon[70044]: pgmap v13601: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:41.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:01:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:01:41.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:01:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:01:41.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:01:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:01:41.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:01:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:01:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:01:41 smithi083 ceph-mon[79930]: pgmap v13601: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:01:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:01:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:01:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:01:41.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:01:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:01:41.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:01:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:01:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:01:41 smithi083 ceph-mon[77242]: pgmap v13601: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:41.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:01:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:01:41.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:01:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:01:41.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:01:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:01:41.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:01:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:01:41.851 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:01:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:01:41] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T13:01:42.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:01:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:01:41.849Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:01:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:01:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:01:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:01:43.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:01:43 smithi153 ceph-mon[70044]: pgmap v13602: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:43.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:01:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:01:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:01:43.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:01:43 smithi083 ceph-mon[79930]: pgmap v13602: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:43.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:01:43 smithi083 ceph-mon[77242]: pgmap v13602: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:45.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:01:45 smithi153 ceph-mon[70044]: pgmap v13603: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:45.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:01:45 smithi083 ceph-mon[79930]: pgmap v13603: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:45.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:01:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:01:45.218Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:01:45.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:01:45 smithi083 ceph-mon[77242]: pgmap v13603: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:46.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:01:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:01:45.939Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:01:46.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:01:46 smithi083 ceph-mon[79930]: pgmap v13604: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:46.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:01:46 smithi083 ceph-mon[77242]: pgmap v13604: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:47.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:01:46 smithi153 ceph-mon[70044]: pgmap v13604: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:49.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:01:49 smithi153 ceph-mon[70044]: pgmap v13605: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:49.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:01:49 smithi083 ceph-mon[77242]: pgmap v13605: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:49.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:01:49 smithi083 ceph-mon[79930]: pgmap v13605: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:51.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:01:51 smithi153 ceph-mon[70044]: pgmap v13606: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:51.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:01:51 smithi083 ceph-mon[77242]: pgmap v13606: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:51.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:01:51 smithi083 ceph-mon[79930]: pgmap v13606: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:51.852 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:01:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:01:51] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T13:01:52.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:01:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:01:51.849Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:01:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:01:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:01:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:01:53.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:01:53 smithi153 ceph-mon[70044]: pgmap v13607: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:53.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:01:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:01:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:01:53.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:01:53 smithi083 ceph-mon[79930]: pgmap v13607: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:53.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:01:53 smithi083 ceph-mon[77242]: pgmap v13607: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:55.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:01:55 smithi153 ceph-mon[70044]: pgmap v13608: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:55.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:01:55 smithi083 ceph-mon[77242]: pgmap v13608: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:55.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:01:55 smithi083 ceph-mon[79930]: pgmap v13608: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:55.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:01:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:01:55.218Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:01:56.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:01:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:01:55.940Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:01:56.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:01:56 smithi083 ceph-mon[77242]: pgmap v13609: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:56.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:01:56 smithi083 ceph-mon[79930]: pgmap v13609: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:57.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:01:56 smithi153 ceph-mon[70044]: pgmap v13609: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:59.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:01:59 smithi153 ceph-mon[70044]: pgmap v13610: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:59.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:01:59 smithi083 ceph-mon[79930]: pgmap v13610: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:01:59.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:01:59 smithi083 ceph-mon[77242]: pgmap v13610: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:01.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:02:01 smithi153 ceph-mon[70044]: pgmap v13611: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:01.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:02:01 smithi083 ceph-mon[79930]: pgmap v13611: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:01.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:02:01 smithi083 ceph-mon[77242]: pgmap v13611: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:01.852 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:02:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:02:01] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T13:02:02.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:02:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:02:01.850Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:02:02.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:02:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:02:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:02:03.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:02:03 smithi153 ceph-mon[70044]: pgmap v13612: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:02:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:02:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:02:03.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:02:03 smithi083 ceph-mon[79930]: pgmap v13612: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:03.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:02:03 smithi083 ceph-mon[77242]: pgmap v13612: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:05.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:02:05 smithi153 ceph-mon[70044]: pgmap v13613: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:05.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:02:05 smithi083 ceph-mon[79930]: pgmap v13613: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:05.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:02:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:02:05.219Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:02:05.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:02:05 smithi083 ceph-mon[77242]: pgmap v13613: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:05.900 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:02:06.194 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:02:06.195 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 6m ago 7h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:02:06.195 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 4m ago 7h 63.1M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:02:06.195 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 6m ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:02:06.195 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 4m ago 7h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:02:06.196 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 4m ago 7h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:02:06.196 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 6m ago 7h 888M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:02:06.196 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 6m ago 7h 1666M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:02:06.196 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 4m ago 7h 721M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:02:06.197 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 6m ago 7h 965M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:02:06.197 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 6m ago 7h 24.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:02:06.197 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 4m ago 7h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:02:06.197 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 6m ago 7h 1204M 2348M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:02:06.198 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 6m ago 7h 1191M 2348M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:02:06.198 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 6m ago 7h 906M 2348M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:02:06.198 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 6m ago 7h 1166M 2348M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:02:06.198 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 4m ago 7h 1025M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:02:06.198 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 4m ago 7h 1283M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:02:06.199 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 4m ago 7h 1146M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:02:06.199 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 4m ago 7h 1360M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:02:06.199 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 4m ago 7h 177M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:02:06.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:02:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:02:05.940Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:02:06.552 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:02:06.552 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:02:06.552 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:02:06.552 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:02:06.553 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:02:06.553 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:02:06.553 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:02:06.553 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:02:06.554 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:02:06.554 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:02:06.554 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:02:06.554 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:02:06.554 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:02:06.555 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:02:06.555 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:02:06.555 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:02:06.555 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:02:06.555 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:02:06.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:02:06 smithi083 ceph-mon[79930]: from='client.59893 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:02:06.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:02:06 smithi083 ceph-mon[79930]: pgmap v13614: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:06.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:02:06 smithi083 ceph-mon[79930]: from='client.49968 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:02:06.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:02:06 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/144479025' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:02:06.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:02:06 smithi083 ceph-mon[77242]: from='client.59893 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:02:06.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:02:06 smithi083 ceph-mon[77242]: pgmap v13614: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:06.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:02:06 smithi083 ceph-mon[77242]: from='client.49968 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:02:06.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:02:06 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/144479025' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:02:07.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:02:06 smithi153 ceph-mon[70044]: from='client.59893 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:02:07.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:02:06 smithi153 ceph-mon[70044]: pgmap v13614: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:07.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:02:06 smithi153 ceph-mon[70044]: from='client.49968 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:02:07.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:02:06 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/144479025' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:02:09.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:02:09 smithi153 ceph-mon[70044]: pgmap v13615: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:09.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:02:09 smithi083 ceph-mon[79930]: pgmap v13615: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:09.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:02:09 smithi083 ceph-mon[77242]: pgmap v13615: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:11.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:02:11 smithi153 ceph-mon[70044]: pgmap v13616: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:11.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:02:11 smithi083 ceph-mon[79930]: pgmap v13616: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:11.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:02:11 smithi083 ceph-mon[77242]: pgmap v13616: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:11.853 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:02:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:02:11] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T13:02:12.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:02:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:02:11.851Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:02:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:02:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:02:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:02:13.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:02:13 smithi153 ceph-mon[70044]: pgmap v13617: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:13.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:02:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:02:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:02:13.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:02:13 smithi083 ceph-mon[79930]: pgmap v13617: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:13.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:02:13 smithi083 ceph-mon[77242]: pgmap v13617: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:15.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:02:15 smithi153 ceph-mon[70044]: pgmap v13618: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:15.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:02:15 smithi083 ceph-mon[79930]: pgmap v13618: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:15.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:02:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:02:15.220Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:02:15.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:02:15 smithi083 ceph-mon[77242]: pgmap v13618: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:16.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:02:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:02:15.941Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:02:16.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:02:16 smithi083 ceph-mon[79930]: pgmap v13619: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:16.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:02:16 smithi083 ceph-mon[77242]: pgmap v13619: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:17.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:02:16 smithi153 ceph-mon[70044]: pgmap v13619: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:19.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:02:19 smithi153 ceph-mon[70044]: pgmap v13620: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:19.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:02:19 smithi083 ceph-mon[77242]: pgmap v13620: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:19.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:02:19 smithi083 ceph-mon[79930]: pgmap v13620: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:21.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:02:21 smithi153 ceph-mon[70044]: pgmap v13621: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:21.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:02:21 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:02:21.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:02:21 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:02:21.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:02:21 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:02:21.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:02:21 smithi083 ceph-mon[79930]: pgmap v13621: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:21.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:02:21 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:02:21.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:02:21 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:02:21.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:02:21 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:02:21.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:02:21 smithi083 ceph-mon[77242]: pgmap v13621: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:21.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:02:21 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:02:21.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:02:21 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:02:21.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:02:21 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:02:21.854 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:02:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:02:21] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T13:02:22.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:02:22 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:02:22.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:02:22 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:02:22.214 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:02:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:02:21.851Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:02:22.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:02:22 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:02:22.559 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:02:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:02:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:02:23.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:02:23 smithi153 ceph-mon[70044]: pgmap v13622: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:23.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:02:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:02:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:02:23.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:02:23 smithi083 ceph-mon[77242]: pgmap v13622: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:23.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:02:23 smithi083 ceph-mon[79930]: pgmap v13622: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:25.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:02:25 smithi083 ceph-mon[79930]: pgmap v13623: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:25.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:02:25 smithi083 ceph-mon[77242]: pgmap v13623: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:25.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:02:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:02:25.221Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:02:25.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:02:25 smithi153 ceph-mon[70044]: pgmap v13623: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:26.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:02:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:02:25.942Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:02:26.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:02:26 smithi083 ceph-mon[79930]: pgmap v13624: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:26.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:02:26 smithi083 ceph-mon[77242]: pgmap v13624: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:27.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:02:26 smithi153 ceph-mon[70044]: pgmap v13624: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:29.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:02:29 smithi153 ceph-mon[70044]: pgmap v13625: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:29.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:02:29 smithi083 ceph-mon[79930]: pgmap v13625: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:29.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:02:29 smithi083 ceph-mon[77242]: pgmap v13625: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:31.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:02:31 smithi153 ceph-mon[70044]: pgmap v13626: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:31.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:02:31 smithi083 ceph-mon[77242]: pgmap v13626: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:31.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:02:31 smithi083 ceph-mon[79930]: pgmap v13626: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:31.855 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:02:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:02:31] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T13:02:32.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:02:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:02:31.852Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:02:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:02:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:02:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:02:33.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:02:33 smithi153 ceph-mon[70044]: pgmap v13627: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:33.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:02:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:02:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:02:33.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:02:33 smithi083 ceph-mon[79930]: pgmap v13627: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:33.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:02:33 smithi083 ceph-mon[77242]: pgmap v13627: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:35.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:02:35 smithi153 ceph-mon[70044]: pgmap v13628: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:35.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:02:35 smithi083 ceph-mon[79930]: pgmap v13628: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:35.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:02:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:02:35.221Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:02:35.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:02:35 smithi083 ceph-mon[77242]: pgmap v13628: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:36.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:02:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:02:35.943Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:02:36.879 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:02:36.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:02:36 smithi083 ceph-mon[79930]: pgmap v13629: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:36.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:02:36 smithi083 ceph-mon[77242]: pgmap v13629: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:37.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:02:36 smithi153 ceph-mon[70044]: pgmap v13629: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:37.166 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:02:37.167 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 7m ago 7h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:02:37.167 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 5m ago 7h 63.1M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:02:37.167 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 7m ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:02:37.167 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 5m ago 7h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:02:37.167 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 5m ago 7h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:02:37.168 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 7m ago 7h 888M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:02:37.168 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 7m ago 7h 1666M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:02:37.168 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 5m ago 7h 721M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:02:37.168 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 7m ago 7h 965M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:02:37.168 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 7m ago 7h 24.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:02:37.169 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 5m ago 7h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:02:37.169 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 7m ago 7h 1204M 2348M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:02:37.169 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 7m ago 7h 1191M 2348M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:02:37.169 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 7m ago 7h 906M 2348M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:02:37.170 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 7m ago 7h 1166M 2348M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:02:37.170 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 5m ago 7h 1025M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:02:37.170 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 5m ago 7h 1283M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:02:37.170 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 5m ago 7h 1146M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:02:37.171 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 5m ago 7h 1360M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:02:37.171 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 5m ago 7h 177M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:02:37.521 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:02:37.521 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:02:37.521 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:02:37.521 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:02:37.522 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:02:37.522 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:02:37.522 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:02:37.522 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:02:37.523 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:02:37.523 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:02:37.523 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:02:37.523 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:02:37.523 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:02:37.524 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:02:37.524 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:02:37.524 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:02:37.524 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:02:37.525 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:02:37.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:02:37 smithi083 ceph-mon[77242]: from='client.49980 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:02:37.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:02:37 smithi083 ceph-mon[77242]: from='client.49986 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:02:37.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:02:37 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3769527250' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:02:37.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:02:37 smithi083 ceph-mon[79930]: from='client.49980 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:02:37.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:02:37 smithi083 ceph-mon[79930]: from='client.49986 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:02:37.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:02:37 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3769527250' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:02:38.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:02:37 smithi153 ceph-mon[70044]: from='client.49980 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:02:38.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:02:37 smithi153 ceph-mon[70044]: from='client.49986 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:02:38.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:02:37 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3769527250' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:02:38.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:02:38 smithi083 ceph-mon[79930]: pgmap v13630: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:38.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:02:38 smithi083 ceph-mon[77242]: pgmap v13630: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:39.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:02:38 smithi153 ceph-mon[70044]: pgmap v13630: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:41.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:02:41 smithi153 ceph-mon[70044]: pgmap v13631: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:41.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:02:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:02:41.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:02:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:02:41.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:02:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:02:41.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:02:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:02:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:02:41 smithi083 ceph-mon[79930]: pgmap v13631: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:02:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:02:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:02:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:02:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:02:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:02:41.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:02:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:02:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:02:41 smithi083 ceph-mon[77242]: pgmap v13631: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:02:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:02:41.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:02:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:02:41.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:02:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:02:41.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:02:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:02:41.855 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:02:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:02:41] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T13:02:42.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:02:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:02:41.852Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:02:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:02:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:02:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:02:43.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:02:43 smithi153 ceph-mon[70044]: pgmap v13632: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:43.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:02:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:02:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:02:43.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:02:43 smithi083 ceph-mon[77242]: pgmap v13632: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:43.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:02:43 smithi083 ceph-mon[79930]: pgmap v13632: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:45.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:02:45 smithi153 ceph-mon[70044]: pgmap v13633: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:45.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:02:45 smithi083 ceph-mon[79930]: pgmap v13633: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:45.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:02:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:02:45.222Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:02:45.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:02:45 smithi083 ceph-mon[77242]: pgmap v13633: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:46.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:02:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:02:45.943Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:02:46.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:02:46 smithi083 ceph-mon[79930]: pgmap v13634: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:46.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:02:46 smithi083 ceph-mon[77242]: pgmap v13634: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:47.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:02:46 smithi153 ceph-mon[70044]: pgmap v13634: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:49.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:02:49 smithi153 ceph-mon[70044]: pgmap v13635: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:49.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:02:49 smithi083 ceph-mon[77242]: pgmap v13635: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:49.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:02:49 smithi083 ceph-mon[79930]: pgmap v13635: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:51.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:02:51 smithi153 ceph-mon[70044]: pgmap v13636: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:51.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:02:51 smithi083 ceph-mon[79930]: pgmap v13636: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:51.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:02:51 smithi083 ceph-mon[77242]: pgmap v13636: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:51.856 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:02:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:02:51] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T13:02:52.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:02:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:02:51.853Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:02:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:02:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:02:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:02:53.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:02:53 smithi153 ceph-mon[70044]: pgmap v13637: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:53.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:02:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:02:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:02:53.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:02:53 smithi083 ceph-mon[79930]: pgmap v13637: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:53.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:02:53 smithi083 ceph-mon[77242]: pgmap v13637: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:55.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:02:55 smithi153 ceph-mon[70044]: pgmap v13638: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:55.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:02:55 smithi083 ceph-mon[79930]: pgmap v13638: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:55.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:02:55 smithi083 ceph-mon[77242]: pgmap v13638: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:55.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:02:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:02:55.223Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:02:56.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:02:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:02:55.944Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:02:56.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:02:56 smithi083 ceph-mon[79930]: pgmap v13639: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:56.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:02:56 smithi083 ceph-mon[77242]: pgmap v13639: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:57.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:02:56 smithi153 ceph-mon[70044]: pgmap v13639: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:59.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:02:59 smithi153 ceph-mon[70044]: pgmap v13640: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:59.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:02:59 smithi083 ceph-mon[77242]: pgmap v13640: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:02:59.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:02:59 smithi083 ceph-mon[79930]: pgmap v13640: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:01.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:03:01 smithi153 ceph-mon[70044]: pgmap v13641: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:01.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:03:01 smithi083 ceph-mon[79930]: pgmap v13641: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:01.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:03:01 smithi083 ceph-mon[77242]: pgmap v13641: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:01.857 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:03:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:03:01] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T13:03:02.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:03:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:03:01.854Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:03:02.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:03:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:03:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:03:03.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:03:03 smithi153 ceph-mon[70044]: pgmap v13642: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:03:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:03:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:03:03.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:03:03 smithi083 ceph-mon[77242]: pgmap v13642: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:03.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:03:03 smithi083 ceph-mon[79930]: pgmap v13642: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:05.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:03:05 smithi083 ceph-mon[79930]: pgmap v13643: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:05.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:03:05 smithi083 ceph-mon[77242]: pgmap v13643: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:05.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:03:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:03:05.224Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:03:05.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:03:05 smithi153 ceph-mon[70044]: pgmap v13643: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:06.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:03:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:03:05.945Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:03:06.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:03:06 smithi083 ceph-mon[79930]: pgmap v13644: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:06.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:03:06 smithi083 ceph-mon[77242]: pgmap v13644: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:07.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:03:06 smithi153 ceph-mon[70044]: pgmap v13644: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:07.845 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:03:08.133 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:03:08.133 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 7m ago 7h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:03:08.134 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 5m ago 7h 63.1M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:03:08.134 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 7m ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:03:08.134 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 5m ago 7h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:03:08.134 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 5m ago 7h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:03:08.135 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 7m ago 7h 888M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:03:08.135 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 7m ago 7h 1666M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:03:08.135 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 5m ago 7h 721M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:03:08.135 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 7m ago 7h 965M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:03:08.136 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 7m ago 7h 24.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:03:08.136 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 5m ago 7h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:03:08.136 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 7m ago 7h 1204M 2348M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:03:08.136 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 7m ago 7h 1191M 2348M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:03:08.136 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 7m ago 7h 906M 2348M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:03:08.137 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 7m ago 7h 1166M 2348M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:03:08.137 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 5m ago 7h 1025M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:03:08.137 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 5m ago 7h 1283M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:03:08.137 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 5m ago 7h 1146M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:03:08.137 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 5m ago 7h 1360M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:03:08.138 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 5m ago 7h 177M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:03:08.487 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:03:08.488 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:03:08.488 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:03:08.488 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:03:08.488 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:03:08.489 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:03:08.489 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:03:08.489 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:03:08.489 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:03:08.489 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:03:08.490 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:03:08.490 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:03:08.490 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:03:08.490 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:03:08.491 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:03:08.491 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:03:08.491 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:03:08.491 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:03:09.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:03:09 smithi153 ceph-mon[70044]: from='client.59929 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:03:09.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:03:09 smithi153 ceph-mon[70044]: pgmap v13645: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:09.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:03:09 smithi153 ceph-mon[70044]: from='client.50004 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:03:09.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:03:09 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/600853707' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:03:09.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:03:09 smithi083 ceph-mon[77242]: from='client.59929 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:03:09.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:03:09 smithi083 ceph-mon[77242]: pgmap v13645: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:09.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:03:09 smithi083 ceph-mon[77242]: from='client.50004 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:03:09.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:03:09 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/600853707' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:03:09.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:03:09 smithi083 ceph-mon[79930]: from='client.59929 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:03:09.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:03:09 smithi083 ceph-mon[79930]: pgmap v13645: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:09.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:03:09 smithi083 ceph-mon[79930]: from='client.50004 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:03:09.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:03:09 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/600853707' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:03:11.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:03:11 smithi083 ceph-mon[79930]: pgmap v13646: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:11.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:03:11 smithi083 ceph-mon[77242]: pgmap v13646: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:11.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:03:11 smithi153 ceph-mon[70044]: pgmap v13646: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:11.857 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:03:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:03:11] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T13:03:12.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:03:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:03:11.855Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:03:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:03:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:03:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:03:13.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:03:13 smithi153 ceph-mon[70044]: pgmap v13647: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:13.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:03:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:03:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:03:13.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:03:13 smithi083 ceph-mon[77242]: pgmap v13647: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:13.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:03:13 smithi083 ceph-mon[79930]: pgmap v13647: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:15.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:03:15 smithi083 ceph-mon[77242]: pgmap v13648: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:15.468 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:03:15 smithi083 ceph-mon[79930]: pgmap v13648: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:15.469 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:03:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:03:15.224Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:03:15.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:03:15 smithi153 ceph-mon[70044]: pgmap v13648: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:16.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:03:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:03:15.946Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:03:16.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:03:16 smithi083 ceph-mon[79930]: pgmap v13649: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:16.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:03:16 smithi083 ceph-mon[77242]: pgmap v13649: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:17.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:03:16 smithi153 ceph-mon[70044]: pgmap v13649: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:19.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:03:19 smithi153 ceph-mon[70044]: pgmap v13650: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:19.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:03:19 smithi083 ceph-mon[79930]: pgmap v13650: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:19.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:03:19 smithi083 ceph-mon[77242]: pgmap v13650: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:21.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:03:21 smithi083 ceph-mon[79930]: pgmap v13651: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:21.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:03:21 smithi083 ceph-mon[77242]: pgmap v13651: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:21.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:03:21 smithi153 ceph-mon[70044]: pgmap v13651: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:21.858 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:03:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:03:21] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T13:03:22.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:03:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:03:21.855Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:03:22.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:03:22 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:03:22.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:03:22 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:03:22.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:03:22 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:03:22.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:03:22 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:03:22.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:03:22 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:03:22.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:03:22 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:03:22.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:03:22 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:03:22.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:03:22 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:03:22.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:03:22 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:03:22.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:03:22 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:03:22.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:03:22 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:03:22.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:03:22 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:03:22.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:03:22 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:03:22.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:03:22 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:03:22.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:03:22 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:03:22.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:03:22 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:03:22.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:03:22 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:03:22.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:03:22 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:03:22.218 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:03:22 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:03:22.218 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:03:22 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:03:22.218 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:03:22 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:03:22.218 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:03:22 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:03:22.218 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:03:22 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:03:22.219 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:03:22 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:03:22.219 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:03:22 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:03:22.219 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:03:22 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:03:22.317 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:03:22 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:03:22.317 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:03:22 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:03:22.318 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:03:22 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:03:22.318 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:03:22 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:03:22.318 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:03:22 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:03:22.318 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:03:22 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:03:22.319 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:03:22 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:03:22.319 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:03:22 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:03:22.319 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:03:22 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:03:22.319 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:03:22 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:03:22.320 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:03:22 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:03:22.320 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:03:22 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:03:22.320 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:03:22 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:03:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:03:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:03:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:03:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:03:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:03:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:03:23.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:03:23 smithi153 ceph-mon[70044]: Adjusting osd_memory_target on smithi083 to 2339M 2022-09-29T13:03:23.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:03:23 smithi153 ceph-mon[70044]: pgmap v13652: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:23.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:03:23 smithi083 ceph-mon[79930]: Adjusting osd_memory_target on smithi083 to 2339M 2022-09-29T13:03:23.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:03:23 smithi083 ceph-mon[79930]: pgmap v13652: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:23.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:03:23 smithi083 ceph-mon[77242]: Adjusting osd_memory_target on smithi083 to 2339M 2022-09-29T13:03:23.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:03:23 smithi083 ceph-mon[77242]: pgmap v13652: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:25.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:03:25 smithi083 ceph-mon[79930]: pgmap v13653: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:25.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:03:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:03:25.224Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:03:25.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:03:25 smithi083 ceph-mon[77242]: pgmap v13653: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:25.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:03:25 smithi153 ceph-mon[70044]: pgmap v13653: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:26.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:03:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:03:25.946Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:03:26.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:03:26 smithi083 ceph-mon[79930]: pgmap v13654: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:26.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:03:26 smithi083 ceph-mon[77242]: pgmap v13654: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:27.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:03:26 smithi153 ceph-mon[70044]: pgmap v13654: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:29.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:03:29 smithi083 ceph-mon[77242]: pgmap v13655: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:29.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:03:29 smithi083 ceph-mon[79930]: pgmap v13655: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:29.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:03:29 smithi153 ceph-mon[70044]: pgmap v13655: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:31.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:03:31 smithi083 ceph-mon[79930]: pgmap v13656: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:31.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:03:31 smithi083 ceph-mon[77242]: pgmap v13656: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:31.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:03:31 smithi153 ceph-mon[70044]: pgmap v13656: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:31.859 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:03:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:03:31] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T13:03:32.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:03:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:03:31.856Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:03:32.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:03:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:03:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:03:33.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:03:33 smithi153 ceph-mon[70044]: pgmap v13657: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:33.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:03:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:03:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:03:33.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:03:33 smithi083 ceph-mon[79930]: pgmap v13657: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:33.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:03:33 smithi083 ceph-mon[77242]: pgmap v13657: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:35.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:03:35 smithi083 ceph-mon[79930]: pgmap v13658: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:35.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:03:35 smithi083 ceph-mon[77242]: pgmap v13658: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:35.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:03:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:03:35.225Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:03:35.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:03:35 smithi153 ceph-mon[70044]: pgmap v13658: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:36.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:03:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:03:35.947Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:03:36.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:03:36 smithi083 ceph-mon[79930]: pgmap v13659: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:36.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:03:36 smithi083 ceph-mon[77242]: pgmap v13659: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:37.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:03:36 smithi153 ceph-mon[70044]: pgmap v13659: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:38.813 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:03:39.103 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:03:39.104 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 8m ago 7h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:03:39.104 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 6m ago 7h 63.1M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:03:39.104 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 8m ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:03:39.105 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 6m ago 7h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:03:39.105 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 6m ago 7h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:03:39.105 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 8m ago 7h 888M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:03:39.105 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 8m ago 7h 1666M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:03:39.106 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 6m ago 7h 721M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:03:39.106 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 8m ago 7h 965M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:03:39.106 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 8m ago 7h 24.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:03:39.106 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 6m ago 7h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:03:39.107 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 8m ago 7h 1204M 2339M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:03:39.107 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 8m ago 7h 1191M 2339M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:03:39.107 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 8m ago 7h 906M 2339M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:03:39.107 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 8m ago 7h 1166M 2339M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:03:39.108 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 6m ago 7h 1025M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:03:39.108 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 6m ago 7h 1283M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:03:39.108 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 6m ago 7h 1146M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:03:39.108 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 6m ago 7h 1360M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:03:39.109 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 6m ago 7h 177M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:03:39.456 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:03:39.456 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:03:39.456 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:03:39.457 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:03:39.457 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:03:39.457 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:03:39.457 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:03:39.457 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:03:39.458 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:03:39.458 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:03:39.458 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:03:39.459 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:03:39.459 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:03:39.459 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:03:39.459 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:03:39.459 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:03:39.460 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:03:39.460 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:03:39.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:03:39 smithi083 ceph-mon[79930]: pgmap v13660: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:39.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:03:39 smithi083 ceph-mon[77242]: pgmap v13660: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:39.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:03:39 smithi153 ceph-mon[70044]: pgmap v13660: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:03:40 smithi083 ceph-mon[79930]: from='client.50016 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:03:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:03:40 smithi083 ceph-mon[79930]: from='client.59953 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:03:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:03:40 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1650795366' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:03:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:03:40 smithi083 ceph-mon[77242]: from='client.50016 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:03:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:03:40 smithi083 ceph-mon[77242]: from='client.59953 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:03:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:03:40 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1650795366' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:03:40.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:03:40 smithi153 ceph-mon[70044]: from='client.50016 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:03:40.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:03:40 smithi153 ceph-mon[70044]: from='client.59953 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:03:40.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:03:40 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1650795366' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:03:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:03:41 smithi083 ceph-mon[79930]: pgmap v13661: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:03:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:03:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:03:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:03:41.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:03:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:03:41.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:03:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:03:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:03:41 smithi083 ceph-mon[77242]: pgmap v13661: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:41.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:03:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:03:41.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:03:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:03:41.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:03:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:03:41.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:03:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:03:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:03:41 smithi153 ceph-mon[70044]: pgmap v13661: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:41.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:03:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:03:41.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:03:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:03:41.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:03:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:03:41.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:03:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:03:41.860 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:03:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:03:41] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T13:03:42.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:03:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:03:41.857Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:03:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:03:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:03:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:03:43.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:03:43 smithi153 ceph-mon[70044]: pgmap v13662: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:43.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:03:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:03:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:03:43.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:03:43 smithi083 ceph-mon[79930]: pgmap v13662: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:43.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:03:43 smithi083 ceph-mon[77242]: pgmap v13662: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:45.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:03:45 smithi083 ceph-mon[79930]: pgmap v13663: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:45.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:03:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:03:45.226Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:03:45.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:03:45 smithi083 ceph-mon[77242]: pgmap v13663: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:45.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:03:45 smithi153 ceph-mon[70044]: pgmap v13663: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:46.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:03:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:03:45.948Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:03:46.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:03:46 smithi083 ceph-mon[79930]: pgmap v13664: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:46.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:03:46 smithi083 ceph-mon[77242]: pgmap v13664: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:47.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:03:46 smithi153 ceph-mon[70044]: pgmap v13664: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:49.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:03:49 smithi083 ceph-mon[79930]: pgmap v13665: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:49.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:03:49 smithi083 ceph-mon[77242]: pgmap v13665: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:49.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:03:49 smithi153 ceph-mon[70044]: pgmap v13665: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:51.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:03:51 smithi083 ceph-mon[79930]: pgmap v13666: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:51.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:03:51 smithi083 ceph-mon[77242]: pgmap v13666: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:51.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:03:51 smithi153 ceph-mon[70044]: pgmap v13666: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:51.861 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:03:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:03:51] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T13:03:52.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:03:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:03:51.858Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:03:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:03:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:03:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:03:53.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:03:53 smithi153 ceph-mon[70044]: pgmap v13667: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:53.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:03:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:03:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:03:53.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:03:53 smithi083 ceph-mon[77242]: pgmap v13667: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:53.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:03:53 smithi083 ceph-mon[79930]: pgmap v13667: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:55.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:03:55 smithi083 ceph-mon[77242]: pgmap v13668: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:55.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:03:55 smithi083 ceph-mon[79930]: pgmap v13668: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:55.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:03:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:03:55.225Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:03:55.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:03:55 smithi153 ceph-mon[70044]: pgmap v13668: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:56.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:03:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:03:55.949Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:03:56.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:03:56 smithi083 ceph-mon[79930]: pgmap v13669: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:56.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:03:56 smithi083 ceph-mon[77242]: pgmap v13669: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:57.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:03:56 smithi153 ceph-mon[70044]: pgmap v13669: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:59.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:03:59 smithi083 ceph-mon[77242]: pgmap v13670: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:59.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:03:59 smithi083 ceph-mon[79930]: pgmap v13670: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:03:59.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:03:59 smithi153 ceph-mon[70044]: pgmap v13670: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:01.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:04:01 smithi083 ceph-mon[79930]: pgmap v13671: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:01.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:04:01 smithi083 ceph-mon[77242]: pgmap v13671: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:01.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:04:01 smithi153 ceph-mon[70044]: pgmap v13671: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:01.861 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:04:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:04:01] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T13:04:02.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:04:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:04:01.858Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:04:02.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:04:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:04:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:04:03.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:04:03 smithi153 ceph-mon[70044]: pgmap v13672: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:04:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:04:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:04:03.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:04:03 smithi083 ceph-mon[77242]: pgmap v13672: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:03.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:04:03 smithi083 ceph-mon[79930]: pgmap v13672: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:05.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:04:05 smithi083 ceph-mon[79930]: pgmap v13673: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:05.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:04:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:04:05.226Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:04:05.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:04:05 smithi083 ceph-mon[77242]: pgmap v13673: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:05.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:04:05 smithi153 ceph-mon[70044]: pgmap v13673: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:06.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:04:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:04:05.950Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:04:06.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:04:06 smithi083 ceph-mon[79930]: pgmap v13674: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:06.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:04:06 smithi083 ceph-mon[77242]: pgmap v13674: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:07.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:04:06 smithi153 ceph-mon[70044]: pgmap v13674: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:09.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:04:09 smithi083 ceph-mon[77242]: pgmap v13675: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:09.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:04:09 smithi083 ceph-mon[79930]: pgmap v13675: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:09.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:04:09 smithi153 ceph-mon[70044]: pgmap v13675: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:09.782 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:04:10.071 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:04:10.072 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 8m ago 7h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:04:10.072 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 6m ago 7h 63.1M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:04:10.072 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 8m ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:04:10.073 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 6m ago 7h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:04:10.073 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 6m ago 7h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:04:10.073 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 8m ago 7h 888M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:04:10.073 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 8m ago 7h 1666M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:04:10.073 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 6m ago 7h 721M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:04:10.074 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 8m ago 7h 965M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:04:10.074 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 8m ago 7h 24.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:04:10.074 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 6m ago 7h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:04:10.074 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 8m ago 7h 1204M 2339M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:04:10.075 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 8m ago 7h 1191M 2339M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:04:10.075 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 8m ago 7h 906M 2339M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:04:10.075 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 8m ago 7h 1166M 2339M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:04:10.075 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 6m ago 7h 1025M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:04:10.076 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 6m ago 7h 1283M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:04:10.076 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 6m ago 7h 1146M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:04:10.076 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 6m ago 7h 1360M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:04:10.076 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 6m ago 7h 177M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:04:10.429 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:04:10.429 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:04:10.430 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:04:10.430 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:04:10.430 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:04:10.431 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:04:10.431 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:04:10.431 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:04:10.431 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:04:10.431 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:04:10.432 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:04:10.432 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:04:10.432 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:04:10.432 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:04:10.432 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:04:10.433 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:04:10.433 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:04:10.433 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:04:11.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:04:11 smithi083 ceph-mon[79930]: from='client.59965 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:04:11.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:04:11 smithi083 ceph-mon[79930]: pgmap v13676: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:11.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:04:11 smithi083 ceph-mon[79930]: from='client.50040 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:04:11.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:04:11 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3493454129' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:04:11.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:04:11 smithi083 ceph-mon[77242]: from='client.59965 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:04:11.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:04:11 smithi083 ceph-mon[77242]: pgmap v13676: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:11.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:04:11 smithi083 ceph-mon[77242]: from='client.50040 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:04:11.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:04:11 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3493454129' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:04:11.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:04:11 smithi153 ceph-mon[70044]: from='client.59965 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:04:11.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:04:11 smithi153 ceph-mon[70044]: pgmap v13676: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:11.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:04:11 smithi153 ceph-mon[70044]: from='client.50040 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:04:11.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:04:11 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3493454129' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:04:11.862 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:04:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:04:11] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T13:04:12.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:04:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:04:11.859Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:04:12.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:04:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:04:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:04:13.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:04:13 smithi153 ceph-mon[70044]: pgmap v13677: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:13.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:04:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:04:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:04:13.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:04:13 smithi083 ceph-mon[77242]: pgmap v13677: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:13.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:04:13 smithi083 ceph-mon[79930]: pgmap v13677: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:15.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:04:15 smithi083 ceph-mon[79930]: pgmap v13678: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:15.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:04:15 smithi083 ceph-mon[77242]: pgmap v13678: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:15.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:04:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:04:15.226Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:04:15.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:04:15 smithi153 ceph-mon[70044]: pgmap v13678: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:16.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:04:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:04:15.950Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:04:16.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:04:16 smithi083 ceph-mon[79930]: pgmap v13679: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:16.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:04:16 smithi083 ceph-mon[77242]: pgmap v13679: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:17.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:04:16 smithi153 ceph-mon[70044]: pgmap v13679: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:19.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:04:19 smithi083 ceph-mon[79930]: pgmap v13680: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:19.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:04:19 smithi083 ceph-mon[77242]: pgmap v13680: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:19.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:04:19 smithi153 ceph-mon[70044]: pgmap v13680: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:21.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:04:21 smithi083 ceph-mon[79930]: pgmap v13681: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:21.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:04:21 smithi083 ceph-mon[77242]: pgmap v13681: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:21.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:04:21 smithi153 ceph-mon[70044]: pgmap v13681: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:21.862 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:04:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:04:21] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T13:04:22.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:04:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:04:21.859Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:04:22.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:04:22 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:04:22.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:04:22 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:04:22.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:04:22 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:04:22.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:04:22 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:04:22.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:04:22 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:04:22.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:04:22 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:04:22.559 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:04:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:04:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:04:22.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:04:22 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:04:22.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:04:22 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:04:22.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:04:22 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:04:23.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:04:23 smithi153 ceph-mon[70044]: pgmap v13682: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:23.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:04:23 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:04:23.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:04:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:04:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:04:23.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:04:23 smithi083 ceph-mon[79930]: pgmap v13682: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:23.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:04:23 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:04:23.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:04:23 smithi083 ceph-mon[77242]: pgmap v13682: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:23.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:04:23 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:04:25.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:04:25 smithi083 ceph-mon[79930]: pgmap v13683: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:25.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:04:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:04:25.227Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:04:25.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:04:25 smithi083 ceph-mon[77242]: pgmap v13683: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:25.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:04:25 smithi153 ceph-mon[70044]: pgmap v13683: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:26.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:04:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:04:25.951Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:04:27.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:04:26 smithi153 ceph-mon[70044]: pgmap v13684: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:27.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:04:26 smithi083 ceph-mon[77242]: pgmap v13684: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:27.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:04:26 smithi083 ceph-mon[79930]: pgmap v13684: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:29.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:04:29 smithi083 ceph-mon[79930]: pgmap v13685: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:29.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:04:29 smithi083 ceph-mon[77242]: pgmap v13685: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:29.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:04:29 smithi153 ceph-mon[70044]: pgmap v13685: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:31.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:04:31 smithi083 ceph-mon[79930]: pgmap v13686: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:31.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:04:31 smithi083 ceph-mon[77242]: pgmap v13686: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:31.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:04:31 smithi153 ceph-mon[70044]: pgmap v13686: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:31.864 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:04:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:04:31] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T13:04:32.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:04:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:04:31.861Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:04:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:04:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:04:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:04:33.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:04:33 smithi153 ceph-mon[70044]: pgmap v13687: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:33.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:04:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:04:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:04:33.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:04:33 smithi083 ceph-mon[79930]: pgmap v13687: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:33.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:04:33 smithi083 ceph-mon[77242]: pgmap v13687: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:35.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:04:35 smithi083 ceph-mon[77242]: pgmap v13688: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:35.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:04:35 smithi083 ceph-mon[79930]: pgmap v13688: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:35.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:04:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:04:35.227Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:04:35.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:04:35 smithi153 ceph-mon[70044]: pgmap v13688: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:36.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:04:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:04:35.951Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:04:37.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:04:36 smithi153 ceph-mon[70044]: pgmap v13689: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:37.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:04:36 smithi083 ceph-mon[79930]: pgmap v13689: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:37.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:04:36 smithi083 ceph-mon[77242]: pgmap v13689: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:39.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:04:39 smithi083 ceph-mon[79930]: pgmap v13690: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:39.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:04:39 smithi083 ceph-mon[77242]: pgmap v13690: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:39.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:04:39 smithi153 ceph-mon[70044]: pgmap v13690: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:40.758 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:04:41.048 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:04:41.048 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 9m ago 7h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:04:41.049 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 7m ago 7h 63.1M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:04:41.049 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 9m ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:04:41.049 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 7m ago 7h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:04:41.049 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 7m ago 7h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:04:41.050 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 9m ago 7h 888M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:04:41.050 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 9m ago 7h 1666M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:04:41.050 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 7m ago 7h 721M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:04:41.050 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 9m ago 7h 965M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:04:41.051 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 9m ago 7h 24.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:04:41.051 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 7m ago 7h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:04:41.051 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 9m ago 7h 1204M 2339M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:04:41.051 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 9m ago 7h 1191M 2339M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:04:41.051 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 9m ago 7h 906M 2339M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:04:41.052 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 9m ago 7h 1166M 2339M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:04:41.052 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 7m ago 7h 1025M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:04:41.052 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 7m ago 7h 1283M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:04:41.052 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 7m ago 7h 1146M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:04:41.053 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 7m ago 7h 1360M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:04:41.053 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 7m ago 7h 177M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:04:41.403 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:04:41.404 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:04:41.404 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:04:41.404 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:04:41.405 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:04:41.405 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:04:41.405 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:04:41.405 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:04:41.406 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:04:41.406 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:04:41.406 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:04:41.406 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:04:41.407 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:04:41.407 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:04:41.407 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:04:41.407 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:04:41.407 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:04:41.408 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:04:41.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:04:41 smithi083 ceph-mon[79930]: pgmap v13691: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:04:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:04:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:04:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:04:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:04:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:04:41.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:04:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:04:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:04:41 smithi083 ceph-mon[77242]: pgmap v13691: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:04:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:04:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:04:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:04:41.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:04:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:04:41.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:04:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:04:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:04:41 smithi153 ceph-mon[70044]: pgmap v13691: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:41.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:04:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:04:41.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:04:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:04:41.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:04:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:04:41.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:04:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:04:41.863 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:04:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:04:41] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T13:04:42.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:04:42 smithi083 ceph-mon[79930]: from='client.50052 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:04:42.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:04:42 smithi083 ceph-mon[79930]: from='client.50058 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:04:42.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:04:42 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/906670840' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:04:42.214 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:04:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:04:41.861Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:04:42.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:04:42 smithi083 ceph-mon[77242]: from='client.50052 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:04:42.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:04:42 smithi083 ceph-mon[77242]: from='client.50058 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:04:42.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:04:42 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/906670840' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:04:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:04:42 smithi153 ceph-mon[70044]: from='client.50052 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:04:42.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:04:42 smithi153 ceph-mon[70044]: from='client.50058 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:04:42.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:04:42 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/906670840' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:04:42.560 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:04:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:04:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:04:43.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:04:43 smithi153 ceph-mon[70044]: pgmap v13692: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:43.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:04:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:04:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:04:43.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:04:43 smithi083 ceph-mon[77242]: pgmap v13692: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:43.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:04:43 smithi083 ceph-mon[79930]: pgmap v13692: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:45.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:04:45 smithi083 ceph-mon[79930]: pgmap v13693: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:45.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:04:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:04:45.228Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:04:45.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:04:45 smithi083 ceph-mon[77242]: pgmap v13693: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:45.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:04:45 smithi153 ceph-mon[70044]: pgmap v13693: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:46.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:04:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:04:45.952Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:04:47.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:04:46 smithi153 ceph-mon[70044]: pgmap v13694: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:47.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:04:46 smithi083 ceph-mon[79930]: pgmap v13694: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:47.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:04:46 smithi083 ceph-mon[77242]: pgmap v13694: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:49.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:04:49 smithi083 ceph-mon[79930]: pgmap v13695: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:49.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:04:49 smithi083 ceph-mon[77242]: pgmap v13695: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:49.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:04:49 smithi153 ceph-mon[70044]: pgmap v13695: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:51.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:04:51 smithi083 ceph-mon[79930]: pgmap v13696: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:51.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:04:51 smithi083 ceph-mon[77242]: pgmap v13696: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:51.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:04:51 smithi153 ceph-mon[70044]: pgmap v13696: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:51.865 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:04:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:04:51] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T13:04:52.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:04:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:04:51.862Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:04:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:04:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:04:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:04:53.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:04:53 smithi153 ceph-mon[70044]: pgmap v13697: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:53.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:04:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:04:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:04:53.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:04:53 smithi083 ceph-mon[79930]: pgmap v13697: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:53.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:04:53 smithi083 ceph-mon[77242]: pgmap v13697: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:55.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:04:55 smithi083 ceph-mon[79930]: pgmap v13698: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:55.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:04:55 smithi083 ceph-mon[77242]: pgmap v13698: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:55.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:04:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:04:55.228Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:04:55.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:04:55 smithi153 ceph-mon[70044]: pgmap v13698: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:56.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:04:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:04:55.953Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:04:57.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:04:56 smithi153 ceph-mon[70044]: pgmap v13699: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:57.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:04:56 smithi083 ceph-mon[79930]: pgmap v13699: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:57.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:04:56 smithi083 ceph-mon[77242]: pgmap v13699: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:59.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:04:59 smithi083 ceph-mon[79930]: pgmap v13700: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:59.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:04:59 smithi083 ceph-mon[77242]: pgmap v13700: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:04:59.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:04:59 smithi153 ceph-mon[70044]: pgmap v13700: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:01.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:05:01 smithi083 ceph-mon[79930]: pgmap v13701: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:01.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:05:01 smithi083 ceph-mon[77242]: pgmap v13701: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:01.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:05:01 smithi153 ceph-mon[70044]: pgmap v13701: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:01.865 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:05:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:05:01] "GET /metrics HTTP/1.1" 200 225230 "" "Prometheus/2.33.4" 2022-09-29T13:05:02.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:05:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:05:01.862Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:05:02.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:05:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:05:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:05:03.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:05:03 smithi153 ceph-mon[70044]: pgmap v13702: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:05:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:05:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:05:03.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:05:03 smithi083 ceph-mon[77242]: pgmap v13702: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:03.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:05:03 smithi083 ceph-mon[79930]: pgmap v13702: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:05.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:05:05 smithi083 ceph-mon[77242]: pgmap v13703: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:05.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:05:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:05:05.229Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:05:05.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:05:05 smithi083 ceph-mon[79930]: pgmap v13703: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:05.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:05:05 smithi153 ceph-mon[70044]: pgmap v13703: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:06.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:05:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:05:05.952Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:05:07.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:05:06 smithi153 ceph-mon[70044]: pgmap v13704: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:07.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:05:06 smithi083 ceph-mon[79930]: pgmap v13704: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:07.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:05:06 smithi083 ceph-mon[77242]: pgmap v13704: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:09.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:05:09 smithi083 ceph-mon[79930]: pgmap v13705: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:09.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:05:09 smithi083 ceph-mon[77242]: pgmap v13705: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:09.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:05:09 smithi153 ceph-mon[70044]: pgmap v13705: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:11.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:05:11 smithi083 ceph-mon[79930]: pgmap v13706: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:11.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:05:11 smithi083 ceph-mon[77242]: pgmap v13706: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:11.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:05:11 smithi153 ceph-mon[70044]: pgmap v13706: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:11.732 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:05:11.865 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:05:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:05:11] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T13:05:12.020 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:05:12.020 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 10m ago 7h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:05:12.021 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 7m ago 7h 63.1M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:05:12.021 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 10m ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:05:12.021 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 7m ago 7h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:05:12.021 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 7m ago 7h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:05:12.022 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 10m ago 7h 888M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:05:12.022 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 10m ago 7h 1666M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:05:12.022 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 7m ago 7h 721M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:05:12.022 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 10m ago 7h 965M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:05:12.023 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 10m ago 7h 24.2M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:05:12.023 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 7m ago 7h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:05:12.023 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 10m ago 7h 1204M 2339M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:05:12.023 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 10m ago 7h 1191M 2339M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:05:12.024 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 10m ago 7h 906M 2339M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:05:12.024 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 10m ago 7h 1166M 2339M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:05:12.024 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 7m ago 7h 1025M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:05:12.024 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 7m ago 7h 1283M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:05:12.025 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 7m ago 7h 1146M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:05:12.025 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 7m ago 7h 1360M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:05:12.025 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 7m ago 7h 177M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:05:12.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:05:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:05:11.862Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:05:12.381 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:05:12.381 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:05:12.381 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:05:12.382 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:05:12.382 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:05:12.382 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:05:12.382 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:05:12.383 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:05:12.383 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:05:12.383 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:05:12.383 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:05:12.384 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:05:12.384 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:05:12.384 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:05:12.384 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:05:12.385 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:05:12.385 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:05:12.385 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:05:12.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:05:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:05:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:05:13.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:05:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:05:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:05:13.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:05:13 smithi153 ceph-mon[70044]: from='client.50070 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:05:13.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:05:13 smithi153 ceph-mon[70044]: from='client.50076 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:05:13.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:05:13 smithi153 ceph-mon[70044]: pgmap v13707: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:13.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:05:13 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1329184363' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:05:13.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:05:13 smithi083 ceph-mon[79930]: from='client.50070 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:05:13.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:05:13 smithi083 ceph-mon[79930]: from='client.50076 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:05:13.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:05:13 smithi083 ceph-mon[79930]: pgmap v13707: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:13.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:05:13 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1329184363' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:05:13.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:05:13 smithi083 ceph-mon[77242]: from='client.50070 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:05:13.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:05:13 smithi083 ceph-mon[77242]: from='client.50076 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:05:13.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:05:13 smithi083 ceph-mon[77242]: pgmap v13707: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:13.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:05:13 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1329184363' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:05:15.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:05:15 smithi083 ceph-mon[79930]: pgmap v13708: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:15.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:05:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:05:15.229Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:05:15.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:05:15 smithi083 ceph-mon[77242]: pgmap v13708: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:15.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:05:15 smithi153 ceph-mon[70044]: pgmap v13708: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:16.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:05:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:05:15.953Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:05:17.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:05:16 smithi153 ceph-mon[70044]: pgmap v13709: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:17.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:05:16 smithi083 ceph-mon[77242]: pgmap v13709: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:17.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:05:16 smithi083 ceph-mon[79930]: pgmap v13709: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:19.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:05:19 smithi083 ceph-mon[77242]: pgmap v13710: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:19.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:05:19 smithi083 ceph-mon[79930]: pgmap v13710: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:19.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:05:19 smithi153 ceph-mon[70044]: pgmap v13710: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:21.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:05:21 smithi083 ceph-mon[79930]: pgmap v13711: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:21.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:05:21 smithi083 ceph-mon[77242]: pgmap v13711: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:21.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:05:21 smithi153 ceph-mon[70044]: pgmap v13711: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:21.866 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:05:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:05:21] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T13:05:22.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:05:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:05:21.864Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:05:22.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:05:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:05:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:05:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:05:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:05:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:05:23.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:05:23 smithi153 ceph-mon[70044]: pgmap v13712: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:23.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:05:23 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:05:23.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:05:23 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:05:23.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:05:23 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:05:23.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:05:23 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:05:23.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:05:23 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:05:23.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:05:23 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:05:23.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:05:23 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:05:23.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:05:23 smithi083 ceph-mon[79930]: pgmap v13712: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:23.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:05:23 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:05:23.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:05:23 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:05:23.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:05:23 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:05:23.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:05:23 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:05:23.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:05:23 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:05:23.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:05:23 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:05:23.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:05:23 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:05:23.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:05:23 smithi083 ceph-mon[77242]: pgmap v13712: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:23.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:05:23 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:05:23.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:05:23 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:05:23.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:05:23 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:05:23.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:05:23 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:05:23.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:05:23 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:05:23.467 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:05:23 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:05:23.467 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:05:23 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:05:25.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:05:25 smithi083 ceph-mon[79930]: pgmap v13713: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:25.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:05:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:05:25.229Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:05:25.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:05:25 smithi083 ceph-mon[77242]: pgmap v13713: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:25.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:05:25 smithi153 ceph-mon[70044]: pgmap v13713: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:26.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:05:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:05:25.954Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:05:27.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:05:26 smithi153 ceph-mon[70044]: pgmap v13714: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:27.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:05:26 smithi083 ceph-mon[79930]: pgmap v13714: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:27.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:05:26 smithi083 ceph-mon[77242]: pgmap v13714: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:28.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:05:28 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:05:28.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:05:28 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:05:28.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:05:28 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:05:28.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:05:28 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:05:28.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:05:28 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:05:28.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:05:28 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:05:28.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:05:28 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:05:28.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:05:28 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:05:28.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:05:28 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:05:29.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:05:29 smithi083 ceph-mon[79930]: pgmap v13715: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:29.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:05:29 smithi083 ceph-mon[77242]: pgmap v13715: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:29.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:05:29 smithi153 ceph-mon[70044]: pgmap v13715: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:31.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:05:31 smithi083 ceph-mon[79930]: pgmap v13716: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:31.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:05:31 smithi083 ceph-mon[77242]: pgmap v13716: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:31.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:05:31 smithi153 ceph-mon[70044]: pgmap v13716: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:31.867 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:05:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:05:31] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T13:05:32.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:05:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:05:31.865Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:05:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:05:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:05:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:05:33.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:05:33 smithi153 ceph-mon[70044]: pgmap v13717: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:33.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:05:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:05:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:05:33.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:05:33 smithi083 ceph-mon[79930]: pgmap v13717: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:33.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:05:33 smithi083 ceph-mon[77242]: pgmap v13717: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:35.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:05:35 smithi083 ceph-mon[79930]: pgmap v13718: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:35.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:05:35 smithi083 ceph-mon[77242]: pgmap v13718: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:35.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:05:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:05:35.230Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:05:35.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:05:35 smithi153 ceph-mon[70044]: pgmap v13718: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:36.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:05:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:05:35.955Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:05:37.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:05:36 smithi153 ceph-mon[70044]: pgmap v13719: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:37.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:05:36 smithi083 ceph-mon[77242]: pgmap v13719: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:37.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:05:36 smithi083 ceph-mon[79930]: pgmap v13719: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:39.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:05:39 smithi083 ceph-mon[77242]: pgmap v13720: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:39.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:05:39 smithi083 ceph-mon[79930]: pgmap v13720: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:39.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:05:39 smithi153 ceph-mon[70044]: pgmap v13720: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:05:41 smithi083 ceph-mon[79930]: pgmap v13721: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:05:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:05:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:05:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:05:41.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:05:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:05:41.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:05:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:05:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:05:41 smithi083 ceph-mon[77242]: pgmap v13721: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:41.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:05:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:05:41.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:05:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:05:41.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:05:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:05:41.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:05:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:05:41.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:05:41 smithi153 ceph-mon[70044]: pgmap v13721: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:41.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:05:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:05:41.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:05:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:05:41.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:05:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:05:41.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:05:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:05:41.868 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:05:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:05:41] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T13:05:42.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:05:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:05:41.865Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:05:42.705 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:05:42.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:05:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:05:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:05:42.997 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:05:42.998 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 15s ago 7h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:05:42.998 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 8m ago 7h 63.1M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:05:42.998 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 15s ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:05:42.998 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 8m ago 7h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:05:42.999 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 8m ago 7h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:05:42.999 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 15s ago 7h 896M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:05:42.999 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 15s ago 7h 1701M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:05:42.999 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 8m ago 7h 721M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:05:43.000 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 15s ago 7h 987M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:05:43.000 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 15s ago 7h 26.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:05:43.000 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 8m ago 7h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:05:43.000 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 15s ago 7h 1229M 2339M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:05:43.000 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 15s ago 7h 1217M 2339M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:05:43.001 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 15s ago 7h 926M 2339M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:05:43.001 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 15s ago 7h 1191M 2339M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:05:43.001 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 8m ago 7h 1025M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:05:43.001 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 8m ago 7h 1283M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:05:43.002 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 8m ago 7h 1146M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:05:43.002 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 8m ago 7h 1360M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:05:43.002 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 8m ago 7h 177M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:05:43.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:05:43 smithi153 ceph-mon[70044]: pgmap v13722: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:43.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:05:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:05:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:05:43.357 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:05:43.357 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:05:43.357 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:05:43.357 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:05:43.358 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:05:43.358 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:05:43.358 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:05:43.358 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:05:43.359 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:05:43.359 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:05:43.359 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:05:43.359 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:05:43.360 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:05:43.360 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:05:43.360 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:05:43.360 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:05:43.360 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:05:43.361 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:05:43.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:05:43 smithi083 ceph-mon[79930]: pgmap v13722: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:43.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:05:43 smithi083 ceph-mon[77242]: pgmap v13722: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:44.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:05:44 smithi083 ceph-mon[79930]: from='client.50088 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:05:44.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:05:44 smithi083 ceph-mon[79930]: from='client.60025 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:05:44.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:05:44 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/492298833' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:05:44.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:05:44 smithi083 ceph-mon[77242]: from='client.50088 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:05:44.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:05:44 smithi083 ceph-mon[77242]: from='client.60025 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:05:44.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:05:44 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/492298833' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:05:44.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:05:44 smithi153 ceph-mon[70044]: from='client.50088 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:05:44.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:05:44 smithi153 ceph-mon[70044]: from='client.60025 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:05:44.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:05:44 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/492298833' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:05:45.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:05:45 smithi083 ceph-mon[79930]: pgmap v13723: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:45.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:05:45 smithi083 ceph-mon[77242]: pgmap v13723: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:45.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:05:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:05:45.231Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:05:45.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:05:45 smithi153 ceph-mon[70044]: pgmap v13723: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:46.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:05:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:05:45.956Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:05:47.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:05:46 smithi153 ceph-mon[70044]: pgmap v13724: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:47.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:05:46 smithi083 ceph-mon[77242]: pgmap v13724: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:47.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:05:46 smithi083 ceph-mon[79930]: pgmap v13724: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:49.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:05:49 smithi083 ceph-mon[79930]: pgmap v13725: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:49.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:05:49 smithi083 ceph-mon[77242]: pgmap v13725: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:49.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:05:49 smithi153 ceph-mon[70044]: pgmap v13725: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:51.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:05:51 smithi083 ceph-mon[79930]: pgmap v13726: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:51.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:05:51 smithi083 ceph-mon[77242]: pgmap v13726: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:51.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:05:51 smithi153 ceph-mon[70044]: pgmap v13726: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:51.868 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:05:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:05:51] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T13:05:52.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:05:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:05:51.866Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:05:52.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:05:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:05:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:05:53.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:05:53 smithi153 ceph-mon[70044]: pgmap v13727: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:53.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:05:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:05:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:05:53.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:05:53 smithi083 ceph-mon[79930]: pgmap v13727: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:53.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:05:53 smithi083 ceph-mon[77242]: pgmap v13727: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:55.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:05:55 smithi083 ceph-mon[77242]: pgmap v13728: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:55.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:05:55 smithi083 ceph-mon[79930]: pgmap v13728: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:55.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:05:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:05:55.231Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:05:55.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:05:55 smithi153 ceph-mon[70044]: pgmap v13728: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:56.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:05:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:05:55.957Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:05:57.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:05:56 smithi153 ceph-mon[70044]: pgmap v13729: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:57.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:05:56 smithi083 ceph-mon[79930]: pgmap v13729: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:57.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:05:56 smithi083 ceph-mon[77242]: pgmap v13729: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:59.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:05:59 smithi083 ceph-mon[77242]: pgmap v13730: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:59.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:05:59 smithi083 ceph-mon[79930]: pgmap v13730: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:05:59.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:05:59 smithi153 ceph-mon[70044]: pgmap v13730: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:01.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:06:01 smithi083 ceph-mon[77242]: pgmap v13731: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:01.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:06:01 smithi083 ceph-mon[79930]: pgmap v13731: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:01.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:06:01 smithi153 ceph-mon[70044]: pgmap v13731: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:01.870 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:06:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:06:01] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T13:06:02.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:06:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:06:01.867Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:06:02.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:06:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:06:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:06:03.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:06:03 smithi153 ceph-mon[70044]: pgmap v13732: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:06:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:06:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:06:03.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:06:03 smithi083 ceph-mon[79930]: pgmap v13732: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:03.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:06:03 smithi083 ceph-mon[77242]: pgmap v13732: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:05.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:06:05 smithi083 ceph-mon[79930]: pgmap v13733: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:05.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:06:05 smithi083 ceph-mon[77242]: pgmap v13733: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:05.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:06:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:06:05.232Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:06:05.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:06:05 smithi153 ceph-mon[70044]: pgmap v13733: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:06.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:06:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:06:05.957Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:06:07.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:06:06 smithi153 ceph-mon[70044]: pgmap v13734: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:07.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:06:06 smithi083 ceph-mon[79930]: pgmap v13734: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:07.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:06:06 smithi083 ceph-mon[77242]: pgmap v13734: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:09.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:06:09 smithi083 ceph-mon[79930]: pgmap v13735: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:09.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:06:09 smithi083 ceph-mon[77242]: pgmap v13735: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:09.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:06:09 smithi153 ceph-mon[70044]: pgmap v13735: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:11.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:06:11 smithi083 ceph-mon[79930]: pgmap v13736: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:11.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:06:11 smithi083 ceph-mon[77242]: pgmap v13736: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:11.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:06:11 smithi153 ceph-mon[70044]: pgmap v13736: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:11.870 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:06:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:06:11] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T13:06:12.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:06:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:06:11.868Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:06:12.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:06:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:06:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:06:13.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:06:13 smithi153 ceph-mon[70044]: pgmap v13737: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:13.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:06:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:06:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:06:13.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:06:13 smithi083 ceph-mon[79930]: pgmap v13737: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:13.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:06:13 smithi083 ceph-mon[77242]: pgmap v13737: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:13.694 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:06:13.985 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:06:13.985 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 46s ago 7h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:06:13.985 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 8m ago 7h 63.1M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:06:13.986 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 46s ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:06:13.986 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 8m ago 7h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:06:13.986 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 8m ago 7h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:06:13.987 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 46s ago 7h 896M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:06:13.987 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 46s ago 7h 1701M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:06:13.987 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 8m ago 7h 721M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:06:13.987 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 46s ago 7h 987M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:06:13.987 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 46s ago 7h 26.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:06:13.988 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 8m ago 7h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:06:13.988 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 46s ago 7h 1229M 2339M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:06:13.988 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 46s ago 7h 1217M 2339M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:06:13.988 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 46s ago 7h 926M 2339M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:06:13.989 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 46s ago 7h 1191M 2339M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:06:13.989 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 8m ago 7h 1025M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:06:13.989 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 8m ago 7h 1283M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:06:13.989 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 8m ago 7h 1146M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:06:13.989 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 8m ago 7h 1360M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:06:13.990 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 8m ago 7h 177M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:06:14.339 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:06:14.340 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:06:14.340 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:06:14.340 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:06:14.340 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:06:14.341 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:06:14.341 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:06:14.341 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:06:14.341 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:06:14.342 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:06:14.342 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:06:14.342 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:06:14.342 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:06:14.343 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:06:14.343 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:06:14.343 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:06:14.343 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:06:14.343 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:06:15.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:06:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:06:15.233Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:06:15.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:06:15 smithi083 ceph-mon[79930]: from='client.60037 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:06:15.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:06:15 smithi083 ceph-mon[79930]: from='client.50112 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:06:15.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:06:15 smithi083 ceph-mon[79930]: pgmap v13738: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:15.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:06:15 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/595723146' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:06:15.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:06:15 smithi083 ceph-mon[77242]: from='client.60037 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:06:15.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:06:15 smithi083 ceph-mon[77242]: from='client.50112 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:06:15.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:06:15 smithi083 ceph-mon[77242]: pgmap v13738: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:15.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:06:15 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/595723146' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:06:15.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:06:15 smithi153 ceph-mon[70044]: from='client.60037 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:06:15.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:06:15 smithi153 ceph-mon[70044]: from='client.50112 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:06:15.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:06:15 smithi153 ceph-mon[70044]: pgmap v13738: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:15.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:06:15 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/595723146' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:06:16.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:06:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:06:15.958Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:06:17.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:06:16 smithi153 ceph-mon[70044]: pgmap v13739: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:17.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:06:16 smithi083 ceph-mon[79930]: pgmap v13739: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:17.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:06:16 smithi083 ceph-mon[77242]: pgmap v13739: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:19.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:06:19 smithi083 ceph-mon[79930]: pgmap v13740: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:19.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:06:19 smithi083 ceph-mon[77242]: pgmap v13740: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:19.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:06:19 smithi153 ceph-mon[70044]: pgmap v13740: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:21.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:06:21 smithi083 ceph-mon[79930]: pgmap v13741: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:21.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:06:21 smithi083 ceph-mon[77242]: pgmap v13741: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:21.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:06:21 smithi153 ceph-mon[70044]: pgmap v13741: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:21.871 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:06:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:06:21] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T13:06:22.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:06:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:06:21.868Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:06:22.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:06:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:06:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:06:23.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:06:23 smithi153 ceph-mon[70044]: pgmap v13742: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:23.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:06:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:06:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:06:23.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:06:23 smithi083 ceph-mon[79930]: pgmap v13742: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:23.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:06:23 smithi083 ceph-mon[77242]: pgmap v13742: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:25.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:06:25 smithi083 ceph-mon[79930]: pgmap v13743: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:25.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:06:25 smithi083 ceph-mon[77242]: pgmap v13743: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:25.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:06:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:06:25.233Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:06:25.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:06:25 smithi153 ceph-mon[70044]: pgmap v13743: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:26.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:06:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:06:25.958Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:06:27.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:06:26 smithi153 ceph-mon[70044]: pgmap v13744: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:27.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:06:26 smithi083 ceph-mon[77242]: pgmap v13744: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:27.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:06:26 smithi083 ceph-mon[79930]: pgmap v13744: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:28.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:06:27 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:06:28.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:06:27 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:06:28.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:06:27 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:06:28.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:06:27 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:06:28.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:06:27 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:06:28.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:06:27 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:06:28.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:06:27 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:06:28.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:06:27 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:06:28.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:06:27 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:06:29.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:06:28 smithi153 ceph-mon[70044]: pgmap v13745: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:29.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:06:28 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:06:29.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:06:28 smithi083 ceph-mon[79930]: pgmap v13745: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:29.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:06:28 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:06:29.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:06:28 smithi083 ceph-mon[77242]: pgmap v13745: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:29.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:06:28 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:06:31.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:06:31 smithi083 ceph-mon[79930]: pgmap v13746: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:31.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:06:31 smithi083 ceph-mon[77242]: pgmap v13746: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:31.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:06:31 smithi153 ceph-mon[70044]: pgmap v13746: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:31.872 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:06:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:06:31] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T13:06:32.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:06:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:06:31.869Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:06:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:06:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:06:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:06:33.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:06:33 smithi153 ceph-mon[70044]: pgmap v13747: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:33.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:06:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:06:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:06:33.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:06:33 smithi083 ceph-mon[77242]: pgmap v13747: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:33.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:06:33 smithi083 ceph-mon[79930]: pgmap v13747: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:35.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:06:35 smithi083 ceph-mon[79930]: pgmap v13748: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:35.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:06:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:06:35.233Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:06:35.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:06:35 smithi083 ceph-mon[77242]: pgmap v13748: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:35.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:06:35 smithi153 ceph-mon[70044]: pgmap v13748: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:36.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:06:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:06:35.959Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:06:37.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:06:36 smithi153 ceph-mon[70044]: pgmap v13749: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:37.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:06:36 smithi083 ceph-mon[79930]: pgmap v13749: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:37.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:06:36 smithi083 ceph-mon[77242]: pgmap v13749: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:39.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:06:39 smithi083 ceph-mon[79930]: pgmap v13750: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:39.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:06:39 smithi083 ceph-mon[77242]: pgmap v13750: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:39.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:06:39 smithi153 ceph-mon[70044]: pgmap v13750: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:06:41 smithi083 ceph-mon[79930]: pgmap v13751: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:06:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:06:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:06:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:06:41.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:06:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:06:41.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:06:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:06:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:06:41 smithi083 ceph-mon[77242]: pgmap v13751: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:41.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:06:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:06:41.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:06:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:06:41.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:06:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:06:41.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:06:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:06:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:06:41 smithi153 ceph-mon[70044]: pgmap v13751: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:41.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:06:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:06:41.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:06:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:06:41.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:06:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:06:41.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:06:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:06:41.873 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:06:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:06:41] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T13:06:42.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:06:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:06:41.870Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:06:42.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:06:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:06:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:06:43.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:06:43 smithi153 ceph-mon[70044]: pgmap v13752: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:43.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:06:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:06:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:06:43.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:06:43 smithi083 ceph-mon[79930]: pgmap v13752: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:43.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:06:43 smithi083 ceph-mon[77242]: pgmap v13752: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:44.665 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:06:44.952 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:06:44.952 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 77s ago 7h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:06:44.953 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 9m ago 7h 63.1M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:06:44.953 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 77s ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:06:44.953 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 9m ago 7h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:06:44.953 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 9m ago 7h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:06:44.953 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 77s ago 7h 896M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:06:44.954 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 77s ago 7h 1701M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:06:44.954 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 9m ago 7h 721M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:06:44.954 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 77s ago 7h 987M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:06:44.954 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 77s ago 7h 26.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:06:44.955 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 9m ago 7h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:06:44.955 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 77s ago 7h 1229M 2339M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:06:44.955 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 77s ago 7h 1217M 2339M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:06:44.955 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 77s ago 7h 926M 2339M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:06:44.955 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 77s ago 7h 1191M 2339M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:06:44.956 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 9m ago 7h 1025M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:06:44.956 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 9m ago 7h 1283M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:06:44.956 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 9m ago 7h 1146M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:06:44.956 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 9m ago 7h 1360M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:06:44.956 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 9m ago 7h 177M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:06:45.308 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:06:45.309 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:06:45.309 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:06:45.309 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:06:45.310 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:06:45.310 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:06:45.310 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:06:45.310 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:06:45.310 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:06:45.311 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:06:45.311 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:06:45.311 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:06:45.311 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:06:45.312 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:06:45.312 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:06:45.312 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:06:45.312 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:06:45.312 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:06:45.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:06:45 smithi083 ceph-mon[79930]: pgmap v13753: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:45.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:06:45 smithi083 ceph-mon[77242]: pgmap v13753: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:45.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:06:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:06:45.233Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:06:45.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:06:45 smithi153 ceph-mon[70044]: pgmap v13753: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:46.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:06:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:06:45.959Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:06:46.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:06:46 smithi083 ceph-mon[79930]: from='client.60055 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:06:46.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:06:46 smithi083 ceph-mon[79930]: from='client.60061 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:06:46.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:06:46 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4205301492' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:06:46.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:06:46 smithi083 ceph-mon[77242]: from='client.60055 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:06:46.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:06:46 smithi083 ceph-mon[77242]: from='client.60061 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:06:46.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:06:46 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4205301492' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:06:46.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:06:46 smithi153 ceph-mon[70044]: from='client.60055 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:06:46.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:06:46 smithi153 ceph-mon[70044]: from='client.60061 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:06:46.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:06:46 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4205301492' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:06:47.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:06:47 smithi083 ceph-mon[79930]: pgmap v13754: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:47.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:06:47 smithi083 ceph-mon[77242]: pgmap v13754: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:47.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:06:47 smithi153 ceph-mon[70044]: pgmap v13754: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:49.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:06:49 smithi083 ceph-mon[79930]: pgmap v13755: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:49.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:06:49 smithi083 ceph-mon[77242]: pgmap v13755: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:49.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:06:49 smithi153 ceph-mon[70044]: pgmap v13755: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:51.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:06:51 smithi083 ceph-mon[79930]: pgmap v13756: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:51.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:06:51 smithi083 ceph-mon[77242]: pgmap v13756: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:51.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:06:51 smithi153 ceph-mon[70044]: pgmap v13756: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:51.873 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:06:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:06:51] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T13:06:52.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:06:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:06:51.871Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:06:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:06:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:06:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:06:53.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:06:53 smithi153 ceph-mon[70044]: pgmap v13757: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:53.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:06:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:06:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:06:53.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:06:53 smithi083 ceph-mon[79930]: pgmap v13757: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:53.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:06:53 smithi083 ceph-mon[77242]: pgmap v13757: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:55.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:06:55 smithi083 ceph-mon[79930]: pgmap v13758: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:55.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:06:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:06:55.234Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:06:55.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:06:55 smithi083 ceph-mon[77242]: pgmap v13758: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:55.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:06:55 smithi153 ceph-mon[70044]: pgmap v13758: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:56.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:06:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:06:55.960Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:06:57.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:06:56 smithi153 ceph-mon[70044]: pgmap v13759: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:57.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:06:56 smithi083 ceph-mon[79930]: pgmap v13759: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:57.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:06:56 smithi083 ceph-mon[77242]: pgmap v13759: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:59.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:06:59 smithi083 ceph-mon[79930]: pgmap v13760: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:59.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:06:59 smithi083 ceph-mon[77242]: pgmap v13760: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:06:59.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:06:59 smithi153 ceph-mon[70044]: pgmap v13760: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:01.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:07:01 smithi083 ceph-mon[79930]: pgmap v13761: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:01.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:07:01 smithi083 ceph-mon[77242]: pgmap v13761: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:01.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:07:01 smithi153 ceph-mon[70044]: pgmap v13761: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:01.875 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:07:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:07:01] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T13:07:02.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:07:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:07:01.872Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:07:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:07:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:07:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:07:03.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:07:03 smithi153 ceph-mon[70044]: pgmap v13762: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:07:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:07:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:07:03.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:07:03 smithi083 ceph-mon[79930]: pgmap v13762: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:03.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:07:03 smithi083 ceph-mon[77242]: pgmap v13762: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:05.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:07:05 smithi083 ceph-mon[79930]: pgmap v13763: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:05.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:07:05 smithi083 ceph-mon[77242]: pgmap v13763: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:05.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:07:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:07:05.235Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:07:05.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:07:05 smithi153 ceph-mon[70044]: pgmap v13763: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:06.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:07:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:07:05.960Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:07:07.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:07:06 smithi153 ceph-mon[70044]: pgmap v13764: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:07.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:07:06 smithi083 ceph-mon[79930]: pgmap v13764: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:07.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:07:06 smithi083 ceph-mon[77242]: pgmap v13764: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:09.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:07:09 smithi083 ceph-mon[79930]: pgmap v13765: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:09.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:07:09 smithi083 ceph-mon[77242]: pgmap v13765: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:09.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:07:09 smithi153 ceph-mon[70044]: pgmap v13765: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:11.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:07:11 smithi083 ceph-mon[79930]: pgmap v13766: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:11.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:07:11 smithi083 ceph-mon[77242]: pgmap v13766: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:11.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:07:11 smithi153 ceph-mon[70044]: pgmap v13766: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:11.875 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:07:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:07:11] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T13:07:12.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:07:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:07:11.872Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:07:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:07:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:07:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:07:13.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:07:13 smithi153 ceph-mon[70044]: pgmap v13767: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:13.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:07:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:07:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:07:13.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:07:13 smithi083 ceph-mon[79930]: pgmap v13767: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:13.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:07:13 smithi083 ceph-mon[77242]: pgmap v13767: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:15.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:07:15 smithi083 ceph-mon[79930]: pgmap v13768: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:15.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:07:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:07:15.236Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:07:15.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:07:15 smithi083 ceph-mon[77242]: pgmap v13768: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:15.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:07:15 smithi153 ceph-mon[70044]: pgmap v13768: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:15.636 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:07:15.930 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:07:15.930 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 108s ago 7h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:07:15.930 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 9m ago 7h 63.1M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:07:15.931 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 108s ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:07:15.931 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 9m ago 7h 28.1M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:07:15.931 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 9m ago 7h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:07:15.931 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 108s ago 7h 896M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:07:15.932 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 108s ago 7h 1701M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:07:15.932 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 9m ago 7h 721M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:07:15.932 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 108s ago 7h 987M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:07:15.932 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 108s ago 7h 26.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:07:15.933 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 9m ago 7h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:07:15.933 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 108s ago 7h 1229M 2339M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:07:15.933 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 108s ago 7h 1217M 2339M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:07:15.933 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 108s ago 7h 926M 2339M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:07:15.934 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 108s ago 7h 1191M 2339M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:07:15.934 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 9m ago 7h 1025M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:07:15.934 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 9m ago 7h 1283M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:07:15.934 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 9m ago 7h 1146M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:07:15.935 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 9m ago 7h 1360M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:07:15.935 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 9m ago 7h 177M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:07:16.288 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:07:16.288 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:07:16.289 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:07:16.289 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:07:16.289 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:07:16.289 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:07:16.290 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:07:16.290 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:07:16.290 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:07:16.290 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:07:16.291 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:07:16.291 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:07:16.291 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:07:16.291 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:07:16.292 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:07:16.292 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:07:16.292 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:07:16.292 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:07:16.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:07:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:07:15.960Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:07:17.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:07:16 smithi153 ceph-mon[70044]: from='client.60073 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:07:17.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:07:16 smithi153 ceph-mon[70044]: from='client.60079 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:07:17.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:07:16 smithi153 ceph-mon[70044]: pgmap v13769: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:17.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:07:16 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1987401410' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:07:17.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:07:16 smithi083 ceph-mon[79930]: from='client.60073 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:07:17.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:07:16 smithi083 ceph-mon[79930]: from='client.60079 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:07:17.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:07:16 smithi083 ceph-mon[79930]: pgmap v13769: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:17.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:07:16 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1987401410' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:07:17.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:07:16 smithi083 ceph-mon[77242]: from='client.60073 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:07:17.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:07:16 smithi083 ceph-mon[77242]: from='client.60079 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:07:17.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:07:16 smithi083 ceph-mon[77242]: pgmap v13769: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:17.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:07:16 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1987401410' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:07:19.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:07:19 smithi083 ceph-mon[79930]: pgmap v13770: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:19.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:07:19 smithi083 ceph-mon[77242]: pgmap v13770: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:19.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:07:19 smithi153 ceph-mon[70044]: pgmap v13770: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:21.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:07:21 smithi083 ceph-mon[79930]: pgmap v13771: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:21.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:07:21 smithi083 ceph-mon[77242]: pgmap v13771: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:21.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:07:21 smithi153 ceph-mon[70044]: pgmap v13771: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:21.876 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:07:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:07:21] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T13:07:22.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:07:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:07:21.873Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:07:22.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:07:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:07:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:07:23.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:07:23 smithi153 ceph-mon[70044]: pgmap v13772: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:23.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:07:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:07:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:07:23.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:07:23 smithi083 ceph-mon[79930]: pgmap v13772: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:23.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:07:23 smithi083 ceph-mon[77242]: pgmap v13772: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:25.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:07:25 smithi083 ceph-mon[79930]: pgmap v13773: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:25.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:07:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:07:25.236Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:07:25.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:07:25 smithi083 ceph-mon[77242]: pgmap v13773: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:25.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:07:25 smithi153 ceph-mon[70044]: pgmap v13773: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:26.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:07:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:07:25.962Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:07:27.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:07:26 smithi153 ceph-mon[70044]: pgmap v13774: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:27.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:07:26 smithi083 ceph-mon[79930]: pgmap v13774: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:27.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:07:26 smithi083 ceph-mon[77242]: pgmap v13774: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:29.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:07:29 smithi083 ceph-mon[79930]: pgmap v13775: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:29.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:07:29 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:07:29.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:07:29 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:07:29.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:07:29 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:07:29.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:07:29 smithi083 ceph-mon[77242]: pgmap v13775: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:29.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:07:29 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:07:29.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:07:29 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:07:29.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:07:29 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:07:29.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:07:29 smithi153 ceph-mon[70044]: pgmap v13775: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:29.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:07:29 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:07:29.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:07:29 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:07:29.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:07:29 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:07:31.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:07:31 smithi083 ceph-mon[79930]: pgmap v13776: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:31.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:07:31 smithi083 ceph-mon[77242]: pgmap v13776: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:31.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:07:31 smithi153 ceph-mon[70044]: pgmap v13776: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:31.877 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:07:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:07:31] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T13:07:32.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:07:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:07:31.874Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:07:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:07:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:07:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:07:33.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:07:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:07:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:07:33.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:07:33 smithi153 ceph-mon[70044]: pgmap v13777: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:33.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:07:33 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:07:33.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:07:33 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:07:33.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:07:33 smithi083 ceph-mon[79930]: pgmap v13777: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:33.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:07:33 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:07:33.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:07:33 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:07:33.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:07:33 smithi083 ceph-mon[77242]: pgmap v13777: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:33.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:07:33 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:07:33.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:07:33 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:07:34.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:07:34 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:07:34.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:07:34 smithi083 ceph-mon[79930]: pgmap v13778: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:34.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:07:34 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:07:34.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:07:34 smithi083 ceph-mon[77242]: pgmap v13778: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:35.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:07:34 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:07:35.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:07:34 smithi153 ceph-mon[70044]: pgmap v13778: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:35.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:07:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:07:35.238Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:07:36.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:07:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:07:35.962Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:07:37.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:07:36 smithi153 ceph-mon[70044]: pgmap v13779: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:37.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:07:36 smithi083 ceph-mon[77242]: pgmap v13779: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:37.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:07:36 smithi083 ceph-mon[79930]: pgmap v13779: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:39.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:07:39 smithi083 ceph-mon[79930]: pgmap v13780: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:39.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:07:39 smithi083 ceph-mon[77242]: pgmap v13780: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:39.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:07:39 smithi153 ceph-mon[70044]: pgmap v13780: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:07:41 smithi083 ceph-mon[79930]: pgmap v13781: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:07:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:07:41.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:07:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:07:41.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:07:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:07:41.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:07:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:07:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:07:41 smithi083 ceph-mon[77242]: pgmap v13781: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:41.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:07:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:07:41.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:07:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:07:41.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:07:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:07:41.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:07:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:07:41.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:07:41 smithi153 ceph-mon[70044]: pgmap v13781: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:41.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:07:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:07:41.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:07:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:07:41.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:07:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:07:41.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:07:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:07:41.877 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:07:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:07:41] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T13:07:42.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:07:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:07:41.874Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:07:42.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:07:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:07:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:07:43.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:07:43 smithi153 ceph-mon[70044]: pgmap v13782: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:43.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:07:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:07:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:07:43.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:07:43 smithi083 ceph-mon[79930]: pgmap v13782: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:43.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:07:43 smithi083 ceph-mon[77242]: pgmap v13782: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:45.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:07:45 smithi083 ceph-mon[79930]: pgmap v13783: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:45.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:07:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:07:45.238Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:07:45.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:07:45 smithi083 ceph-mon[77242]: pgmap v13783: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:45.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:07:45 smithi153 ceph-mon[70044]: pgmap v13783: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:46.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:07:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:07:45.962Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:07:46.616 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:07:46.906 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:07:46.907 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 2m ago 7h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:07:46.907 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 13s ago 7h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:07:46.907 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 2m ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:07:46.907 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 13s ago 7h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:07:46.908 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 13s ago 7h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:07:46.908 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 2m ago 7h 896M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:07:46.908 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 2m ago 7h 1701M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:07:46.909 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 13s ago 7h 736M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:07:46.909 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 2m ago 7h 987M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:07:46.909 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 2m ago 7h 26.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:07:46.910 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 13s ago 7h 25.7M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:07:46.910 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 2m ago 7h 1229M 2339M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:07:46.910 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 2m ago 7h 1217M 2339M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:07:46.911 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 2m ago 7h 926M 2339M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:07:46.912 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 2m ago 7h 1191M 2339M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:07:46.912 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 13s ago 7h 1047M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:07:46.912 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 13s ago 7h 1311M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:07:46.913 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 13s ago 7h 1172M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:07:46.913 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 13s ago 7h 1391M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:07:46.914 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 13s ago 7h 174M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:07:47.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:07:46 smithi153 ceph-mon[70044]: pgmap v13784: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:47.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:07:46 smithi083 ceph-mon[79930]: pgmap v13784: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:47.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:07:46 smithi083 ceph-mon[77242]: pgmap v13784: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:47.264 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:07:47.264 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:07:47.265 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:07:47.265 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:07:47.265 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:07:47.265 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:07:47.266 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:07:47.266 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:07:47.266 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:07:47.266 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:07:47.266 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:07:47.267 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:07:47.267 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:07:47.267 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:07:47.267 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:07:47.267 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:07:47.268 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:07:47.268 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:07:48.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:07:47 smithi153 ceph-mon[70044]: from='client.60091 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:07:48.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:07:47 smithi153 ceph-mon[70044]: from='client.50166 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:07:48.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:07:47 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2989570859' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:07:48.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:07:47 smithi083 ceph-mon[79930]: from='client.60091 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:07:48.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:07:47 smithi083 ceph-mon[79930]: from='client.50166 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:07:48.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:07:47 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2989570859' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:07:48.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:07:47 smithi083 ceph-mon[77242]: from='client.60091 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:07:48.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:07:47 smithi083 ceph-mon[77242]: from='client.50166 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:07:48.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:07:47 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2989570859' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:07:49.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:07:48 smithi153 ceph-mon[70044]: pgmap v13785: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:49.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:07:48 smithi083 ceph-mon[79930]: pgmap v13785: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:49.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:07:48 smithi083 ceph-mon[77242]: pgmap v13785: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:51.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:07:51 smithi083 ceph-mon[79930]: pgmap v13786: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:51.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:07:51 smithi083 ceph-mon[77242]: pgmap v13786: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:51.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:07:51 smithi153 ceph-mon[70044]: pgmap v13786: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:51.877 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:07:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:07:51] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T13:07:52.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:07:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:07:51.875Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:07:52.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:07:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:07:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:07:53.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:07:53 smithi153 ceph-mon[70044]: pgmap v13787: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:53.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:07:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:07:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:07:53.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:07:53 smithi083 ceph-mon[79930]: pgmap v13787: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:53.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:07:53 smithi083 ceph-mon[77242]: pgmap v13787: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:55.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:07:55 smithi083 ceph-mon[79930]: pgmap v13788: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:55.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:07:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:07:55.239Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:07:55.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:07:55 smithi083 ceph-mon[77242]: pgmap v13788: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:55.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:07:55 smithi153 ceph-mon[70044]: pgmap v13788: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:56.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:07:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:07:55.963Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:07:57.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:07:56 smithi153 ceph-mon[70044]: pgmap v13789: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:57.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:07:56 smithi083 ceph-mon[77242]: pgmap v13789: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:57.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:07:56 smithi083 ceph-mon[79930]: pgmap v13789: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:59.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:07:59 smithi083 ceph-mon[79930]: pgmap v13790: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:59.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:07:59 smithi083 ceph-mon[77242]: pgmap v13790: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:07:59.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:07:59 smithi153 ceph-mon[70044]: pgmap v13790: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:01.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:08:01 smithi083 ceph-mon[77242]: pgmap v13791: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:01.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:08:01 smithi083 ceph-mon[79930]: pgmap v13791: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:01.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:08:01 smithi153 ceph-mon[70044]: pgmap v13791: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:01.878 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:08:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:08:01] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T13:08:02.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:08:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:08:01.875Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:08:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:08:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:08:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:08:03.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:08:03 smithi153 ceph-mon[70044]: pgmap v13792: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:08:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:08:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:08:03.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:08:03 smithi083 ceph-mon[79930]: pgmap v13792: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:03.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:08:03 smithi083 ceph-mon[77242]: pgmap v13792: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:05.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:08:05 smithi083 ceph-mon[79930]: pgmap v13793: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:05.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:08:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:08:05.239Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:08:05.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:08:05 smithi083 ceph-mon[77242]: pgmap v13793: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:05.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:08:05 smithi153 ceph-mon[70044]: pgmap v13793: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:06.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:08:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:08:05.964Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:08:07.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:08:06 smithi153 ceph-mon[70044]: pgmap v13794: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:07.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:08:06 smithi083 ceph-mon[77242]: pgmap v13794: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:07.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:08:06 smithi083 ceph-mon[79930]: pgmap v13794: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:09.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:08:09 smithi083 ceph-mon[79930]: pgmap v13795: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:09.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:08:09 smithi083 ceph-mon[77242]: pgmap v13795: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:09.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:08:09 smithi153 ceph-mon[70044]: pgmap v13795: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:11.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:08:11 smithi083 ceph-mon[77242]: pgmap v13796: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:11.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:08:11 smithi083 ceph-mon[79930]: pgmap v13796: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:11.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:08:11 smithi153 ceph-mon[70044]: pgmap v13796: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:11.879 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:08:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:08:11] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T13:08:12.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:08:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:08:11.877Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:08:12.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:08:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:08:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:08:13.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:08:13 smithi153 ceph-mon[70044]: pgmap v13797: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:13.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:08:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:08:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:08:13.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:08:13 smithi083 ceph-mon[79930]: pgmap v13797: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:13.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:08:13 smithi083 ceph-mon[77242]: pgmap v13797: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:15.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:08:15 smithi083 ceph-mon[79930]: pgmap v13798: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:15.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:08:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:08:15.241Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:08:15.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:08:15 smithi083 ceph-mon[77242]: pgmap v13798: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:15.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:08:15 smithi153 ceph-mon[70044]: pgmap v13798: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:16.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:08:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:08:15.965Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:08:17.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:08:16 smithi153 ceph-mon[70044]: pgmap v13799: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:17.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:08:16 smithi083 ceph-mon[77242]: pgmap v13799: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:17.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:08:16 smithi083 ceph-mon[79930]: pgmap v13799: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:17.589 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:08:17.878 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:08:17.878 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 2m ago 7h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:08:17.879 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 44s ago 7h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:08:17.879 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 2m ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:08:17.879 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 44s ago 7h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:08:17.879 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 44s ago 7h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:08:17.879 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 2m ago 7h 896M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:08:17.880 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 2m ago 7h 1701M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:08:17.880 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 44s ago 7h 736M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:08:17.880 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 2m ago 7h 987M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:08:17.880 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 2m ago 7h 26.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:08:17.881 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 44s ago 7h 25.7M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:08:17.881 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 2m ago 7h 1229M 2339M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:08:17.881 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 2m ago 7h 1217M 2339M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:08:17.881 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 2m ago 7h 926M 2339M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:08:17.881 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 2m ago 7h 1191M 2339M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:08:17.882 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 44s ago 7h 1047M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:08:17.882 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 44s ago 7h 1311M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:08:17.882 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 44s ago 7h 1172M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:08:17.883 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 44s ago 7h 1391M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:08:17.883 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 44s ago 7h 174M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:08:18.235 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:08:18.235 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:08:18.235 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:08:18.235 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:08:18.236 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:08:18.236 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:08:18.236 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:08:18.236 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:08:18.236 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:08:18.237 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:08:18.237 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:08:18.237 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:08:18.237 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:08:18.238 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:08:18.238 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:08:18.238 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:08:18.238 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:08:18.238 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:08:19.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:08:19 smithi083 ceph-mon[77242]: from='client.60109 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:08:19.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:08:19 smithi083 ceph-mon[77242]: from='client.60115 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:08:19.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:08:19 smithi083 ceph-mon[77242]: pgmap v13800: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:19.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:08:19 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2306267319' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:08:19.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:08:19 smithi083 ceph-mon[79930]: from='client.60109 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:08:19.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:08:19 smithi083 ceph-mon[79930]: from='client.60115 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:08:19.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:08:19 smithi083 ceph-mon[79930]: pgmap v13800: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:19.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:08:19 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2306267319' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:08:19.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:08:19 smithi153 ceph-mon[70044]: from='client.60109 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:08:19.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:08:19 smithi153 ceph-mon[70044]: from='client.60115 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:08:19.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:08:19 smithi153 ceph-mon[70044]: pgmap v13800: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:19.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:08:19 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2306267319' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:08:21.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:08:21 smithi083 ceph-mon[79930]: pgmap v13801: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:21.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:08:21 smithi083 ceph-mon[77242]: pgmap v13801: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:21.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:08:21 smithi153 ceph-mon[70044]: pgmap v13801: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:21.880 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:08:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:08:21] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T13:08:22.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:08:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:08:21.877Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:08:22.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:08:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:08:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:08:23.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:08:23 smithi153 ceph-mon[70044]: pgmap v13802: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:23.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:08:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:08:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:08:23.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:08:23 smithi083 ceph-mon[77242]: pgmap v13802: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:23.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:08:23 smithi083 ceph-mon[79930]: pgmap v13802: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:25.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:08:25 smithi083 ceph-mon[79930]: pgmap v13803: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:25.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:08:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:08:25.241Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:08:25.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:08:25 smithi083 ceph-mon[77242]: pgmap v13803: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:25.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:08:25 smithi153 ceph-mon[70044]: pgmap v13803: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:26.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:08:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:08:25.966Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:08:27.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:08:26 smithi153 ceph-mon[70044]: pgmap v13804: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:27.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:08:26 smithi083 ceph-mon[79930]: pgmap v13804: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:27.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:08:26 smithi083 ceph-mon[77242]: pgmap v13804: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:29.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:08:29 smithi083 ceph-mon[79930]: pgmap v13805: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:29.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:08:29 smithi083 ceph-mon[77242]: pgmap v13805: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:29.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:08:29 smithi153 ceph-mon[70044]: pgmap v13805: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:31.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:08:31 smithi083 ceph-mon[79930]: pgmap v13806: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:31.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:08:31 smithi083 ceph-mon[77242]: pgmap v13806: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:31.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:08:31 smithi153 ceph-mon[70044]: pgmap v13806: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:31.880 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:08:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:08:31] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T13:08:32.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:08:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:08:31.877Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:08:32.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:08:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:08:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:08:33.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:08:33 smithi153 ceph-mon[70044]: pgmap v13807: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:33.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:08:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:08:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:08:33.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:08:33 smithi083 ceph-mon[79930]: pgmap v13807: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:33.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:08:33 smithi083 ceph-mon[77242]: pgmap v13807: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:34.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:08:34 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:08:34.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:08:34 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:08:34.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:08:34 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:08:34.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:08:34 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:08:34.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:08:34 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:08:34.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:08:34 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:08:34.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:08:34 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:08:34.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:08:34 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:08:34.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:08:34 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:08:35.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:08:35 smithi083 ceph-mon[77242]: pgmap v13808: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:35.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:08:35 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:08:35.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:08:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:08:35.241Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:08:35.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:08:35 smithi083 ceph-mon[79930]: pgmap v13808: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:35.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:08:35 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:08:35.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:08:35 smithi153 ceph-mon[70044]: pgmap v13808: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:35.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:08:35 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:08:36.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:08:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:08:35.966Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:08:37.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:08:36 smithi153 ceph-mon[70044]: pgmap v13809: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:37.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:08:36 smithi083 ceph-mon[79930]: pgmap v13809: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:37.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:08:36 smithi083 ceph-mon[77242]: pgmap v13809: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:39.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:08:39 smithi083 ceph-mon[79930]: pgmap v13810: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:39.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:08:39 smithi083 ceph-mon[77242]: pgmap v13810: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:39.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:08:39 smithi153 ceph-mon[70044]: pgmap v13810: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:08:41 smithi083 ceph-mon[79930]: pgmap v13811: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:08:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:08:41.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:08:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:08:41.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:08:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:08:41.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:08:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:08:41.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:08:41 smithi083 ceph-mon[77242]: pgmap v13811: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:41.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:08:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:08:41.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:08:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:08:41.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:08:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:08:41.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:08:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:08:41.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:08:41 smithi153 ceph-mon[70044]: pgmap v13811: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:41.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:08:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:08:41.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:08:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:08:41.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:08:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:08:41.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:08:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:08:41.881 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:08:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:08:41] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T13:08:42.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:08:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:08:41.878Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:08:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:08:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:08:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:08:43.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:08:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:08:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:08:43.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:08:43 smithi153 ceph-mon[70044]: pgmap v13812: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:43.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:08:43 smithi083 ceph-mon[79930]: pgmap v13812: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:43.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:08:43 smithi083 ceph-mon[77242]: pgmap v13812: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:45.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:08:45 smithi083 ceph-mon[79930]: pgmap v13813: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:45.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:08:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:08:45.242Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:08:45.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:08:45 smithi083 ceph-mon[77242]: pgmap v13813: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:45.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:08:45 smithi153 ceph-mon[70044]: pgmap v13813: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:46.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:08:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:08:45.967Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:08:47.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:08:46 smithi153 ceph-mon[70044]: pgmap v13814: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:47.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:08:46 smithi083 ceph-mon[79930]: pgmap v13814: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:47.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:08:46 smithi083 ceph-mon[77242]: pgmap v13814: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:48.564 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:08:48.851 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:08:48.852 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 3m ago 7h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:08:48.852 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 75s ago 7h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:08:48.852 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 3m ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:08:48.852 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 75s ago 7h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:08:48.853 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 75s ago 7h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:08:48.853 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 3m ago 7h 896M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:08:48.853 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 3m ago 7h 1701M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:08:48.854 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 75s ago 7h 736M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:08:48.854 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 3m ago 7h 987M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:08:48.854 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 3m ago 7h 26.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:08:48.854 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 75s ago 7h 25.7M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:08:48.855 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 3m ago 7h 1229M 2339M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:08:48.855 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 3m ago 7h 1217M 2339M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:08:48.855 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 3m ago 7h 926M 2339M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:08:48.855 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 3m ago 7h 1191M 2339M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:08:48.856 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 75s ago 7h 1047M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:08:48.856 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 75s ago 7h 1311M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:08:48.856 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 75s ago 7h 1172M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:08:48.856 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 75s ago 7h 1391M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:08:48.857 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 75s ago 7h 174M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:08:49.205 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:08:49.205 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:08:49.205 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:08:49.206 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:08:49.206 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:08:49.206 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:08:49.206 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:08:49.207 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:08:49.207 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:08:49.207 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:08:49.207 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:08:49.208 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:08:49.208 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:08:49.208 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:08:49.208 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:08:49.208 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:08:49.209 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:08:49.209 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:08:49.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:08:49 smithi083 ceph-mon[77242]: pgmap v13815: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:49.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:08:49 smithi083 ceph-mon[79930]: pgmap v13815: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:49.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:08:49 smithi153 ceph-mon[70044]: pgmap v13815: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:50.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:08:50 smithi083 ceph-mon[79930]: from='client.60127 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:08:50.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:08:50 smithi083 ceph-mon[79930]: from='client.50202 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:08:50.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:08:50 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1697101978' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:08:50.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:08:50 smithi083 ceph-mon[77242]: from='client.60127 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:08:50.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:08:50 smithi083 ceph-mon[77242]: from='client.50202 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:08:50.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:08:50 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1697101978' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:08:50.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:08:50 smithi153 ceph-mon[70044]: from='client.60127 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:08:50.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:08:50 smithi153 ceph-mon[70044]: from='client.50202 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:08:50.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:08:50 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1697101978' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:08:51.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:08:51 smithi083 ceph-mon[79930]: pgmap v13816: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:51.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:08:51 smithi083 ceph-mon[77242]: pgmap v13816: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:51.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:08:51 smithi153 ceph-mon[70044]: pgmap v13816: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:51.882 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:08:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:08:51] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T13:08:52.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:08:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:08:51.879Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:08:52.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:08:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:08:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:08:53.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:08:53 smithi153 ceph-mon[70044]: pgmap v13817: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:53.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:08:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:08:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:08:53.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:08:53 smithi083 ceph-mon[77242]: pgmap v13817: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:53.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:08:53 smithi083 ceph-mon[79930]: pgmap v13817: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:55.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:08:55 smithi083 ceph-mon[79930]: pgmap v13818: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:55.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:08:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:08:55.243Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:08:55.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:08:55 smithi083 ceph-mon[77242]: pgmap v13818: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:55.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:08:55 smithi153 ceph-mon[70044]: pgmap v13818: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:56.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:08:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:08:55.966Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:08:57.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:08:56 smithi153 ceph-mon[70044]: pgmap v13819: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:57.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:08:56 smithi083 ceph-mon[79930]: pgmap v13819: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:57.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:08:56 smithi083 ceph-mon[77242]: pgmap v13819: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:59.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:08:59 smithi083 ceph-mon[79930]: pgmap v13820: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:59.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:08:59 smithi083 ceph-mon[77242]: pgmap v13820: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:08:59.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:08:59 smithi153 ceph-mon[70044]: pgmap v13820: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:01.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:09:01 smithi083 ceph-mon[79930]: pgmap v13821: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:01.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:09:01 smithi083 ceph-mon[77242]: pgmap v13821: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:01.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:09:01 smithi153 ceph-mon[70044]: pgmap v13821: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:01.882 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:09:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:09:01] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T13:09:02.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:09:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:09:01.879Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:09:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:09:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:09:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:09:03.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:09:03 smithi153 ceph-mon[70044]: pgmap v13822: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:09:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:09:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:09:03.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:09:03 smithi083 ceph-mon[77242]: pgmap v13822: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:03.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:09:03 smithi083 ceph-mon[79930]: pgmap v13822: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:05.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:09:05 smithi083 ceph-mon[79930]: pgmap v13823: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:05.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:09:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:09:05.244Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:09:05.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:09:05 smithi083 ceph-mon[77242]: pgmap v13823: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:05.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:09:05 smithi153 ceph-mon[70044]: pgmap v13823: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:06.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:09:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:09:05.967Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:09:07.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:09:06 smithi153 ceph-mon[70044]: pgmap v13824: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:07.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:09:06 smithi083 ceph-mon[77242]: pgmap v13824: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:07.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:09:06 smithi083 ceph-mon[79930]: pgmap v13824: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:09.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:09:09 smithi083 ceph-mon[79930]: pgmap v13825: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:09.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:09:09 smithi083 ceph-mon[77242]: pgmap v13825: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:09.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:09:09 smithi153 ceph-mon[70044]: pgmap v13825: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:11.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:09:11 smithi083 ceph-mon[79930]: pgmap v13826: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:11.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:09:11 smithi083 ceph-mon[77242]: pgmap v13826: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:11.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:09:11 smithi153 ceph-mon[70044]: pgmap v13826: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:11.883 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:09:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:09:11] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T13:09:12.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:09:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:09:11.880Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:09:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:09:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:09:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:09:13.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:09:13 smithi153 ceph-mon[70044]: pgmap v13827: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:13.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:09:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:09:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:09:13.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:09:13 smithi083 ceph-mon[79930]: pgmap v13827: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:13.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:09:13 smithi083 ceph-mon[77242]: pgmap v13827: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:15.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:09:15 smithi083 ceph-mon[79930]: pgmap v13828: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:15.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:09:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:09:15.244Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:09:15.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:09:15 smithi083 ceph-mon[77242]: pgmap v13828: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:15.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:09:15 smithi153 ceph-mon[70044]: pgmap v13828: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:16.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:09:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:09:15.968Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:09:17.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:09:16 smithi153 ceph-mon[70044]: pgmap v13829: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:17.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:09:16 smithi083 ceph-mon[79930]: pgmap v13829: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:17.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:09:16 smithi083 ceph-mon[77242]: pgmap v13829: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:19.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:09:19 smithi083 ceph-mon[79930]: pgmap v13830: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:19.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:09:19 smithi083 ceph-mon[77242]: pgmap v13830: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:19.534 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:09:19.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:09:19 smithi153 ceph-mon[70044]: pgmap v13830: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:19.823 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:09:19.824 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 3m ago 7h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:09:19.824 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 106s ago 7h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:09:19.824 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 3m ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:09:19.824 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 106s ago 7h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:09:19.825 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 106s ago 7h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:09:19.825 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 3m ago 7h 896M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:09:19.825 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 3m ago 7h 1701M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:09:19.825 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 106s ago 7h 736M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:09:19.826 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 3m ago 7h 987M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:09:19.826 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 3m ago 7h 26.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:09:19.826 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 106s ago 7h 25.7M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:09:19.827 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 3m ago 7h 1229M 2339M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:09:19.827 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 3m ago 7h 1217M 2339M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:09:19.828 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 3m ago 7h 926M 2339M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:09:19.828 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 3m ago 7h 1191M 2339M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:09:19.828 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 106s ago 7h 1047M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:09:19.829 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 106s ago 7h 1311M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:09:19.829 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 106s ago 7h 1172M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:09:19.829 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 106s ago 7h 1391M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:09:19.829 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 106s ago 7h 174M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:09:20.181 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:09:20.182 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:09:20.182 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:09:20.182 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:09:20.182 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:09:20.183 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:09:20.183 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:09:20.183 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:09:20.183 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:09:20.184 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:09:20.184 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:09:20.184 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:09:20.184 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:09:20.184 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:09:20.185 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:09:20.185 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:09:20.185 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:09:20.185 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:09:21.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:09:21 smithi083 ceph-mon[79930]: from='client.60145 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:09:21.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:09:21 smithi083 ceph-mon[79930]: from='client.60151 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:09:21.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:09:21 smithi083 ceph-mon[79930]: pgmap v13831: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:21.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:09:21 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3362053710' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:09:21.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:09:21 smithi083 ceph-mon[77242]: from='client.60145 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:09:21.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:09:21 smithi083 ceph-mon[77242]: from='client.60151 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:09:21.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:09:21 smithi083 ceph-mon[77242]: pgmap v13831: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:21.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:09:21 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3362053710' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:09:21.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:09:21 smithi153 ceph-mon[70044]: from='client.60145 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:09:21.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:09:21 smithi153 ceph-mon[70044]: from='client.60151 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:09:21.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:09:21 smithi153 ceph-mon[70044]: pgmap v13831: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:21.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:09:21 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3362053710' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:09:21.884 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:09:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:09:21] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T13:09:22.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:09:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:09:21.881Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:09:22.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:09:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:09:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:09:23.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:09:23 smithi153 ceph-mon[70044]: pgmap v13832: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:23.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:09:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:09:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:09:23.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:09:23 smithi083 ceph-mon[79930]: pgmap v13832: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:23.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:09:23 smithi083 ceph-mon[77242]: pgmap v13832: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:25.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:09:25 smithi083 ceph-mon[79930]: pgmap v13833: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:25.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:09:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:09:25.246Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:09:25.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:09:25 smithi083 ceph-mon[77242]: pgmap v13833: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:25.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:09:25 smithi153 ceph-mon[70044]: pgmap v13833: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:26.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:09:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:09:25.969Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:09:27.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:09:26 smithi153 ceph-mon[70044]: pgmap v13834: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:27.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:09:26 smithi083 ceph-mon[79930]: pgmap v13834: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:27.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:09:26 smithi083 ceph-mon[77242]: pgmap v13834: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:29.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:09:29 smithi083 ceph-mon[77242]: pgmap v13835: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:29.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:09:29 smithi083 ceph-mon[79930]: pgmap v13835: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:29.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:09:29 smithi153 ceph-mon[70044]: pgmap v13835: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:31.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:09:31 smithi083 ceph-mon[79930]: pgmap v13836: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:31.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:09:31 smithi083 ceph-mon[77242]: pgmap v13836: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:31.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:09:31 smithi153 ceph-mon[70044]: pgmap v13836: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:31.884 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:09:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:09:31] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T13:09:32.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:09:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:09:31.881Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:09:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:09:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:09:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:09:33.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:09:33 smithi153 ceph-mon[70044]: pgmap v13837: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:33.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:09:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:09:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:09:33.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:09:33 smithi083 ceph-mon[79930]: pgmap v13837: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:33.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:09:33 smithi083 ceph-mon[77242]: pgmap v13837: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:35.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:09:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:09:35.246Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:09:35.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:09:35 smithi083 ceph-mon[79930]: pgmap v13838: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:35.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:09:35 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:09:35.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:09:35 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:09:35.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:09:35 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:09:35.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:09:35 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:09:35.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:09:35 smithi083 ceph-mon[77242]: pgmap v13838: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:35.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:09:35 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:09:35.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:09:35 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:09:35.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:09:35 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:09:35.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:09:35 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:09:35.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:09:35 smithi153 ceph-mon[70044]: pgmap v13838: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:35.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:09:35 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:09:35.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:09:35 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:09:35.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:09:35 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:09:35.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:09:35 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:09:36.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:09:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:09:35.969Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:09:37.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:09:36 smithi153 ceph-mon[70044]: pgmap v13839: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:37.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:09:36 smithi083 ceph-mon[79930]: pgmap v13839: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:37.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:09:36 smithi083 ceph-mon[77242]: pgmap v13839: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:39.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:09:39 smithi083 ceph-mon[79930]: pgmap v13840: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:39.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:09:39 smithi083 ceph-mon[77242]: pgmap v13840: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:39.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:09:39 smithi153 ceph-mon[70044]: pgmap v13840: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:41.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:09:41 smithi083 ceph-mon[77242]: pgmap v13841: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:41.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:09:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:09:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:09:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:09:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:09:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:09:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:09:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:09:41.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:09:41 smithi083 ceph-mon[79930]: pgmap v13841: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:41.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:09:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:09:41.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:09:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:09:41.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:09:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:09:41.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:09:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:09:41.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:09:41 smithi153 ceph-mon[70044]: pgmap v13841: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:41.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:09:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:09:41.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:09:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:09:41.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:09:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:09:41.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:09:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:09:41.885 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:09:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:09:41] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T13:09:42.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:09:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:09:41.882Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:09:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:09:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:09:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:09:43.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:09:43 smithi153 ceph-mon[70044]: pgmap v13842: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:43.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:09:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:09:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:09:43.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:09:43 smithi083 ceph-mon[77242]: pgmap v13842: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:43.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:09:43 smithi083 ceph-mon[79930]: pgmap v13842: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:45.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:09:45 smithi083 ceph-mon[79930]: pgmap v13843: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:45.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:09:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:09:45.247Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:09:45.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:09:45 smithi083 ceph-mon[77242]: pgmap v13843: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:45.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:09:45 smithi153 ceph-mon[70044]: pgmap v13843: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:46.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:09:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:09:45.970Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:09:47.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:09:46 smithi153 ceph-mon[70044]: pgmap v13844: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:47.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:09:46 smithi083 ceph-mon[77242]: pgmap v13844: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:47.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:09:46 smithi083 ceph-mon[79930]: pgmap v13844: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:49.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:09:49 smithi083 ceph-mon[79930]: pgmap v13845: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:49.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:09:49 smithi083 ceph-mon[77242]: pgmap v13845: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:49.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:09:49 smithi153 ceph-mon[70044]: pgmap v13845: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:50.509 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:09:50.797 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:09:50.798 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 4m ago 7h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:09:50.798 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 2m ago 7h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:09:50.798 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 4m ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:09:50.799 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 2m ago 7h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:09:50.799 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 2m ago 7h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:09:50.799 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 4m ago 7h 896M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:09:50.799 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 4m ago 7h 1701M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:09:50.800 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 2m ago 7h 736M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:09:50.800 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 4m ago 7h 987M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:09:50.800 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 4m ago 7h 26.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:09:50.800 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 2m ago 7h 25.7M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:09:50.800 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 4m ago 7h 1229M 2339M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:09:50.801 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 4m ago 7h 1217M 2339M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:09:50.801 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 4m ago 7h 926M 2339M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:09:50.801 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 4m ago 7h 1191M 2339M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:09:50.801 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 2m ago 7h 1047M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:09:50.801 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 2m ago 7h 1311M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:09:50.802 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 2m ago 7h 1172M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:09:50.802 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 2m ago 7h 1391M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:09:50.802 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 2m ago 7h 174M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:09:51.152 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:09:51.152 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:09:51.153 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:09:51.153 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:09:51.153 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:09:51.153 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:09:51.153 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:09:51.154 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:09:51.154 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:09:51.154 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:09:51.154 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:09:51.155 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:09:51.155 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:09:51.155 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:09:51.155 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:09:51.155 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:09:51.156 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:09:51.156 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:09:51.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:09:51 smithi083 ceph-mon[79930]: pgmap v13846: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:51.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:09:51 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3452043895' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:09:51.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:09:51 smithi083 ceph-mon[77242]: pgmap v13846: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:51.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:09:51 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3452043895' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:09:51.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:09:51 smithi153 ceph-mon[70044]: pgmap v13846: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:51.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:09:51 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3452043895' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:09:51.884 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:09:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:09:51] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T13:09:52.190 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:09:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:09:51.882Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:09:52.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:09:52 smithi083 ceph-mon[77242]: from='client.50232 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:09:52.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:09:52 smithi083 ceph-mon[77242]: from='client.60169 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:09:52.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:09:52 smithi083 ceph-mon[79930]: from='client.50232 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:09:52.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:09:52 smithi083 ceph-mon[79930]: from='client.60169 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:09:52.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:09:52 smithi153 ceph-mon[70044]: from='client.50232 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:09:52.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:09:52 smithi153 ceph-mon[70044]: from='client.60169 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:09:52.559 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:09:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:09:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:09:53.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:09:53 smithi153 ceph-mon[70044]: pgmap v13847: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:53.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:09:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:09:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:09:53.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:09:53 smithi083 ceph-mon[77242]: pgmap v13847: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:53.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:09:53 smithi083 ceph-mon[79930]: pgmap v13847: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:55.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:09:55 smithi083 ceph-mon[79930]: pgmap v13848: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:55.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:09:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:09:55.248Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:09:55.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:09:55 smithi083 ceph-mon[77242]: pgmap v13848: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:55.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:09:55 smithi153 ceph-mon[70044]: pgmap v13848: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:56.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:09:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:09:55.970Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:09:57.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:09:56 smithi153 ceph-mon[70044]: pgmap v13849: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:57.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:09:56 smithi083 ceph-mon[79930]: pgmap v13849: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:57.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:09:56 smithi083 ceph-mon[77242]: pgmap v13849: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:59.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:09:59 smithi083 ceph-mon[79930]: pgmap v13850: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:59.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:09:59 smithi083 ceph-mon[77242]: pgmap v13850: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:09:59.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:09:59 smithi153 ceph-mon[70044]: pgmap v13850: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:00.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:10:00 smithi083 ceph-mon[79930]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T13:10:00.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:10:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[77219]: 2022-09-29T13:09:59.999+0000 7f160f665700 -1 log_channel(cluster) log [ERR] : overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T13:10:00.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:10:00 smithi083 ceph-mon[77242]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T13:10:00.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:10:00 smithi153 ceph-mon[70044]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T13:10:01.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:10:01 smithi083 ceph-mon[79930]: pgmap v13851: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:01.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:10:01 smithi083 ceph-mon[77242]: pgmap v13851: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:01.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:10:01 smithi153 ceph-mon[70044]: pgmap v13851: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:01.886 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:10:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:10:01] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T13:10:02.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:10:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:10:01.882Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:10:02.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:10:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:10:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:10:03.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:10:03 smithi153 ceph-mon[70044]: pgmap v13852: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:10:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:10:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:10:03.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:10:03 smithi083 ceph-mon[79930]: pgmap v13852: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:03.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:10:03 smithi083 ceph-mon[77242]: pgmap v13852: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:05.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:10:05 smithi083 ceph-mon[77242]: pgmap v13853: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:05.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:10:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:10:05.249Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:10:05.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:10:05 smithi083 ceph-mon[79930]: pgmap v13853: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:05.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:10:05 smithi153 ceph-mon[70044]: pgmap v13853: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:06.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:10:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:10:05.971Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:10:07.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:10:06 smithi153 ceph-mon[70044]: pgmap v13854: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:07.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:10:06 smithi083 ceph-mon[79930]: pgmap v13854: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:07.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:10:06 smithi083 ceph-mon[77242]: pgmap v13854: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:09.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:10:09 smithi083 ceph-mon[79930]: pgmap v13855: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:09.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:10:09 smithi083 ceph-mon[77242]: pgmap v13855: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:09.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:10:09 smithi153 ceph-mon[70044]: pgmap v13855: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:11.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:10:11 smithi083 ceph-mon[79930]: pgmap v13856: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:11.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:10:11 smithi083 ceph-mon[77242]: pgmap v13856: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:11.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:10:11 smithi153 ceph-mon[70044]: pgmap v13856: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:11.886 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:10:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:10:11] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T13:10:12.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:10:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:10:11.883Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:10:12.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:10:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:10:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:10:13.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:10:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:10:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:10:13.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:10:13 smithi153 ceph-mon[70044]: pgmap v13857: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:13.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:10:13 smithi083 ceph-mon[79930]: pgmap v13857: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:13.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:10:13 smithi083 ceph-mon[77242]: pgmap v13857: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:15.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:10:15 smithi083 ceph-mon[79930]: pgmap v13858: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:15.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:10:15 smithi083 ceph-mon[77242]: pgmap v13858: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:15.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:10:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:10:15.250Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:10:15.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:10:15 smithi153 ceph-mon[70044]: pgmap v13858: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:16.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:10:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:10:15.972Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:10:17.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:10:16 smithi153 ceph-mon[70044]: pgmap v13859: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:17.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:10:16 smithi083 ceph-mon[77242]: pgmap v13859: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:17.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:10:16 smithi083 ceph-mon[79930]: pgmap v13859: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:19.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:10:19 smithi083 ceph-mon[79930]: pgmap v13860: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:19.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:10:19 smithi083 ceph-mon[77242]: pgmap v13860: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:19.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:10:19 smithi153 ceph-mon[70044]: pgmap v13860: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:21.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:10:21 smithi083 ceph-mon[77242]: pgmap v13861: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:21.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:10:21 smithi083 ceph-mon[79930]: pgmap v13861: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:21.481 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:10:21.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:10:21 smithi153 ceph-mon[70044]: pgmap v13861: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:21.768 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:10:21.768 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 4m ago 7h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:10:21.769 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 2m ago 7h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:10:21.769 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 4m ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:10:21.769 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 2m ago 7h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:10:21.769 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 2m ago 7h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:10:21.769 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 4m ago 7h 896M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:10:21.770 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 4m ago 7h 1701M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:10:21.770 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 2m ago 7h 736M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:10:21.770 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 4m ago 7h 987M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:10:21.770 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 4m ago 7h 26.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:10:21.771 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 2m ago 7h 25.7M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:10:21.771 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 4m ago 7h 1229M 2339M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:10:21.771 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 4m ago 7h 1217M 2339M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:10:21.771 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 4m ago 7h 926M 2339M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:10:21.772 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 4m ago 7h 1191M 2339M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:10:21.772 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 2m ago 7h 1047M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:10:21.772 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 2m ago 7h 1311M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:10:21.772 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 2m ago 7h 1172M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:10:21.772 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 2m ago 7h 1391M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:10:21.773 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 2m ago 7h 174M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:10:21.885 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:10:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:10:21] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T13:10:22.125 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:10:22.126 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:10:22.126 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:10:22.126 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:10:22.126 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:10:22.127 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:10:22.127 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:10:22.127 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:10:22.127 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:10:22.127 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:10:22.128 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:10:22.128 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:10:22.128 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:10:22.128 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:10:22.129 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:10:22.129 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:10:22.129 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:10:22.129 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:10:22.197 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:10:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:10:21.883Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:10:22.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:10:22 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3348910874' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:10:22.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:10:22 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3348910874' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:10:22.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:10:22 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3348910874' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:10:22.559 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:10:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:10:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:10:23.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:10:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:10:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:10:23.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:10:23 smithi153 ceph-mon[70044]: from='client.50250 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:10:23.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:10:23 smithi153 ceph-mon[70044]: from='client.50256 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:10:23.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:10:23 smithi153 ceph-mon[70044]: pgmap v13862: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:23.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:10:23 smithi083 ceph-mon[79930]: from='client.50250 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:10:23.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:10:23 smithi083 ceph-mon[79930]: from='client.50256 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:10:23.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:10:23 smithi083 ceph-mon[79930]: pgmap v13862: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:23.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:10:23 smithi083 ceph-mon[77242]: from='client.50250 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:10:23.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:10:23 smithi083 ceph-mon[77242]: from='client.50256 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:10:23.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:10:23 smithi083 ceph-mon[77242]: pgmap v13862: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:25.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:10:25 smithi083 ceph-mon[79930]: pgmap v13863: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:25.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:10:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:10:25.250Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:10:25.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:10:25 smithi083 ceph-mon[77242]: pgmap v13863: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:25.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:10:25 smithi153 ceph-mon[70044]: pgmap v13863: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:26.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:10:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:10:25.973Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:10:27.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:10:26 smithi153 ceph-mon[70044]: pgmap v13864: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:27.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:10:26 smithi083 ceph-mon[77242]: pgmap v13864: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:27.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:10:26 smithi083 ceph-mon[79930]: pgmap v13864: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:29.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:10:29 smithi083 ceph-mon[79930]: pgmap v13865: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:29.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:10:29 smithi083 ceph-mon[77242]: pgmap v13865: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:29.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:10:29 smithi153 ceph-mon[70044]: pgmap v13865: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:31.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:10:31 smithi083 ceph-mon[79930]: pgmap v13866: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:31.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:10:31 smithi083 ceph-mon[77242]: pgmap v13866: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:31.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:10:31 smithi153 ceph-mon[70044]: pgmap v13866: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:31.887 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:10:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:10:31] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T13:10:32.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:10:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:10:31.884Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:10:32.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:10:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:10:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:10:33.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:10:33 smithi153 ceph-mon[70044]: pgmap v13867: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:33.310 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:10:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:10:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:10:33.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:10:33 smithi083 ceph-mon[79930]: pgmap v13867: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:33.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:10:33 smithi083 ceph-mon[77242]: pgmap v13867: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:35.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:10:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:10:35.250Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:10:35.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:10:35 smithi083 ceph-mon[79930]: pgmap v13868: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:35.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:10:35 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:10:35.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:10:35 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:10:35.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:10:35 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:10:35.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:10:35 smithi083 ceph-mon[77242]: pgmap v13868: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:35.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:10:35 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:10:35.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:10:35 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:10:35.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:10:35 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:10:35.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:10:35 smithi153 ceph-mon[70044]: pgmap v13868: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:35.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:10:35 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:10:35.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:10:35 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:10:35.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:10:35 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:10:36.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:10:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:10:35.972Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:10:36.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:10:36 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:10:36.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:10:36 smithi153 ceph-mon[70044]: pgmap v13869: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:36.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:10:36 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:10:36.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:10:36 smithi083 ceph-mon[77242]: pgmap v13869: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:36.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:10:36 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:10:36.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:10:36 smithi083 ceph-mon[79930]: pgmap v13869: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:39.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:10:39 smithi083 ceph-mon[79930]: pgmap v13870: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:39.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:10:39 smithi083 ceph-mon[77242]: pgmap v13870: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:39.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:10:39 smithi153 ceph-mon[70044]: pgmap v13870: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:10:41 smithi083 ceph-mon[79930]: pgmap v13871: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:10:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:10:41.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:10:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:10:41.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:10:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:10:41.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:10:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:10:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:10:41 smithi083 ceph-mon[77242]: pgmap v13871: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:41.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:10:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:10:41.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:10:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:10:41.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:10:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:10:41.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:10:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:10:41.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:10:41 smithi153 ceph-mon[70044]: pgmap v13871: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:41.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:10:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:10:41.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:10:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:10:41.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:10:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:10:41.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:10:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:10:41.888 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:10:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:10:41] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T13:10:42.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:10:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:10:41.885Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:10:42.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:10:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:10:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:10:43.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:10:43 smithi153 ceph-mon[70044]: pgmap v13872: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:43.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:10:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:10:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:10:43.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:10:43 smithi083 ceph-mon[79930]: pgmap v13872: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:43.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:10:43 smithi083 ceph-mon[77242]: pgmap v13872: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:45.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:10:45 smithi083 ceph-mon[79930]: pgmap v13873: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:45.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:10:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:10:45.251Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:10:45.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:10:45 smithi083 ceph-mon[77242]: pgmap v13873: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:45.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:10:45 smithi153 ceph-mon[70044]: pgmap v13873: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:46.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:10:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:10:45.973Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:10:47.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:10:46 smithi153 ceph-mon[70044]: pgmap v13874: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:47.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:10:46 smithi083 ceph-mon[79930]: pgmap v13874: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:47.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:10:46 smithi083 ceph-mon[77242]: pgmap v13874: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:49.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:10:49 smithi083 ceph-mon[77242]: pgmap v13875: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:49.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:10:49 smithi083 ceph-mon[79930]: pgmap v13875: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:49.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:10:49 smithi153 ceph-mon[70044]: pgmap v13875: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:51.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:10:51 smithi083 ceph-mon[77242]: pgmap v13876: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:51.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:10:51 smithi083 ceph-mon[79930]: pgmap v13876: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:51.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:10:51 smithi153 ceph-mon[70044]: pgmap v13876: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:51.888 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:10:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:10:51] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T13:10:52.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:10:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:10:51.885Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:10:52.452 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:10:52.741 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:10:52.741 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 5m ago 7h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:10:52.741 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 3m ago 7h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:10:52.742 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 5m ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:10:52.742 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 3m ago 7h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:10:52.742 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 3m ago 7h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:10:52.743 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 5m ago 7h 896M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:10:52.743 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 5m ago 7h 1701M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:10:52.743 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 3m ago 7h 736M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:10:52.743 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 5m ago 7h 987M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:10:52.744 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 5m ago 7h 26.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:10:52.744 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 3m ago 7h 25.7M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:10:52.744 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 5m ago 7h 1229M 2339M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:10:52.744 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 5m ago 7h 1217M 2339M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:10:52.745 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 5m ago 7h 926M 2339M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:10:52.745 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 5m ago 7h 1191M 2339M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:10:52.745 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 3m ago 7h 1047M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:10:52.745 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 3m ago 7h 1311M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:10:52.746 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 3m ago 7h 1172M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:10:52.746 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 3m ago 7h 1391M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:10:52.746 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 3m ago 7h 174M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:10:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:10:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:10:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:10:53.095 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:10:53.095 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:10:53.095 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:10:53.096 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:10:53.096 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:10:53.096 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:10:53.096 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:10:53.096 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:10:53.097 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:10:53.097 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:10:53.097 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:10:53.097 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:10:53.098 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:10:53.098 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:10:53.098 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:10:53.098 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:10:53.099 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:10:53.099 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:10:53.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:10:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:10:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:10:53.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:10:53 smithi153 ceph-mon[70044]: pgmap v13877: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:53.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:10:53 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/984780665' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:10:53.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:10:53 smithi083 ceph-mon[79930]: pgmap v13877: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:53.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:10:53 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/984780665' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:10:53.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:10:53 smithi083 ceph-mon[77242]: pgmap v13877: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:53.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:10:53 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/984780665' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:10:54.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:10:54 smithi083 ceph-mon[79930]: from='client.50268 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:10:54.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:10:54 smithi083 ceph-mon[79930]: from='client.50274 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:10:54.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:10:54 smithi083 ceph-mon[77242]: from='client.50268 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:10:54.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:10:54 smithi083 ceph-mon[77242]: from='client.50274 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:10:54.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:10:54 smithi153 ceph-mon[70044]: from='client.50268 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:10:54.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:10:54 smithi153 ceph-mon[70044]: from='client.50274 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:10:55.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:10:55 smithi083 ceph-mon[77242]: pgmap v13878: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:55.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:10:55 smithi083 ceph-mon[79930]: pgmap v13878: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:55.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:10:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:10:55.252Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:10:55.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:10:55 smithi153 ceph-mon[70044]: pgmap v13878: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:56.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:10:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:10:55.974Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:10:57.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:10:56 smithi153 ceph-mon[70044]: pgmap v13879: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:57.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:10:56 smithi083 ceph-mon[77242]: pgmap v13879: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:57.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:10:56 smithi083 ceph-mon[79930]: pgmap v13879: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:59.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:10:59 smithi083 ceph-mon[79930]: pgmap v13880: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:59.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:10:59 smithi083 ceph-mon[77242]: pgmap v13880: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:10:59.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:10:59 smithi153 ceph-mon[70044]: pgmap v13880: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:01.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:11:01 smithi083 ceph-mon[79930]: pgmap v13881: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:01.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:11:01 smithi083 ceph-mon[77242]: pgmap v13881: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:01.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:11:01 smithi153 ceph-mon[70044]: pgmap v13881: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:01.889 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:11:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:11:01] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T13:11:02.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:11:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:11:01.886Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:11:02.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:11:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:11:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:11:03.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:11:03 smithi153 ceph-mon[70044]: pgmap v13882: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:11:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:11:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:11:03.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:11:03 smithi083 ceph-mon[79930]: pgmap v13882: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:03.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:11:03 smithi083 ceph-mon[77242]: pgmap v13882: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:05.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:11:05 smithi153 ceph-mon[70044]: pgmap v13883: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:05.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:11:05 smithi083 ceph-mon[77242]: pgmap v13883: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:05.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:11:05 smithi083 ceph-mon[79930]: pgmap v13883: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:05.714 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:11:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:11:05.252Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:11:06.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:11:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:11:05.975Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:11:07.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:11:06 smithi153 ceph-mon[70044]: pgmap v13884: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:07.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:11:06 smithi083 ceph-mon[77242]: pgmap v13884: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:07.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:11:06 smithi083 ceph-mon[79930]: pgmap v13884: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:09.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:11:09 smithi083 ceph-mon[79930]: pgmap v13885: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:09.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:11:09 smithi083 ceph-mon[77242]: pgmap v13885: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:09.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:11:09 smithi153 ceph-mon[70044]: pgmap v13885: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:11.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:11:11 smithi083 ceph-mon[77242]: pgmap v13886: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:11.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:11:11 smithi083 ceph-mon[79930]: pgmap v13886: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:11.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:11:11 smithi153 ceph-mon[70044]: pgmap v13886: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:11.890 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:11:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:11:11] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T13:11:12.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:11:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:11:11.887Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:11:12.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:11:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:11:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:11:13.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:11:13 smithi153 ceph-mon[70044]: pgmap v13887: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:13.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:11:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:11:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:11:13.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:11:13 smithi083 ceph-mon[77242]: pgmap v13887: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:13.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:11:13 smithi083 ceph-mon[79930]: pgmap v13887: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:15.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:11:15 smithi153 ceph-mon[70044]: pgmap v13888: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:15.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:11:15 smithi083 ceph-mon[77242]: pgmap v13888: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:15.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:11:15 smithi083 ceph-mon[79930]: pgmap v13888: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:15.714 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:11:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:11:15.253Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:11:16.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:11:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:11:15.975Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:11:17.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:11:16 smithi153 ceph-mon[70044]: pgmap v13889: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:17.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:11:16 smithi083 ceph-mon[79930]: pgmap v13889: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:17.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:11:16 smithi083 ceph-mon[77242]: pgmap v13889: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:19.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:11:19 smithi083 ceph-mon[79930]: pgmap v13890: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:19.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:11:19 smithi083 ceph-mon[77242]: pgmap v13890: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:19.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:11:19 smithi153 ceph-mon[70044]: pgmap v13890: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:21.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:11:21 smithi083 ceph-mon[79930]: pgmap v13891: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:21.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:11:21 smithi083 ceph-mon[77242]: pgmap v13891: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:21.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:11:21 smithi153 ceph-mon[70044]: pgmap v13891: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:21.891 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:11:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:11:21] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T13:11:22.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:11:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:11:21.888Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:11:22.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:11:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:11:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:11:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:11:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:11:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:11:23.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:11:23 smithi153 ceph-mon[70044]: pgmap v13892: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:23.431 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:11:23.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:11:23 smithi083 ceph-mon[79930]: pgmap v13892: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:23.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:11:23 smithi083 ceph-mon[77242]: pgmap v13892: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:23.730 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:11:23.730 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 5m ago 7h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:11:23.730 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 3m ago 7h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:11:23.731 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 5m ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:11:23.731 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 3m ago 7h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:11:23.731 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 3m ago 7h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:11:23.731 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 5m ago 7h 896M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:11:23.732 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 5m ago 7h 1701M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:11:23.732 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 3m ago 7h 736M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:11:23.732 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 5m ago 7h 987M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:11:23.732 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 5m ago 7h 26.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:11:23.733 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 3m ago 7h 25.7M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:11:23.733 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 5m ago 7h 1229M 2339M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:11:23.733 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 5m ago 7h 1217M 2339M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:11:23.733 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 5m ago 7h 926M 2339M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:11:23.733 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 5m ago 7h 1191M 2339M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:11:23.734 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 3m ago 7h 1047M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:11:23.734 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 3m ago 7h 1311M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:11:23.734 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 3m ago 7h 1172M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:11:23.734 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 3m ago 7h 1391M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:11:23.735 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 3m ago 7h 174M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:11:24.087 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:11:24.087 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:11:24.087 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:11:24.087 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:11:24.088 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:11:24.088 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:11:24.088 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:11:24.088 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:11:24.089 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:11:24.089 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:11:24.089 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:11:24.089 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:11:24.090 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:11:24.090 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:11:24.090 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:11:24.090 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:11:24.090 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:11:24.091 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:11:24.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:11:24 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2152837423' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:11:24.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:11:24 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2152837423' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:11:24.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:11:24 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2152837423' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:11:25.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:11:25 smithi153 ceph-mon[70044]: from='client.60217 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:11:25.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:11:25 smithi153 ceph-mon[70044]: from='client.50292 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:11:25.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:11:25 smithi153 ceph-mon[70044]: pgmap v13893: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:25.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:11:25 smithi083 ceph-mon[79930]: from='client.60217 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:11:25.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:11:25 smithi083 ceph-mon[79930]: from='client.50292 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:11:25.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:11:25 smithi083 ceph-mon[79930]: pgmap v13893: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:25.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:11:25 smithi083 ceph-mon[77242]: from='client.60217 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:11:25.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:11:25 smithi083 ceph-mon[77242]: from='client.50292 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:11:25.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:11:25 smithi083 ceph-mon[77242]: pgmap v13893: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:25.715 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:11:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:11:25.254Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:11:26.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:11:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:11:25.975Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:11:27.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:11:26 smithi153 ceph-mon[70044]: pgmap v13894: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:27.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:11:26 smithi083 ceph-mon[79930]: pgmap v13894: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:27.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:11:26 smithi083 ceph-mon[77242]: pgmap v13894: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:29.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:11:29 smithi083 ceph-mon[77242]: pgmap v13895: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:29.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:11:29 smithi083 ceph-mon[79930]: pgmap v13895: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:29.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:11:29 smithi153 ceph-mon[70044]: pgmap v13895: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:31.509 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:11:31 smithi083 ceph-mon[79930]: pgmap v13896: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:31.509 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:11:31 smithi083 ceph-mon[77242]: pgmap v13896: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:31.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:11:31 smithi153 ceph-mon[70044]: pgmap v13896: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:31.891 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:11:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:11:31] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T13:11:32.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:11:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:11:31.889Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:11:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:11:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:11:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:11:33.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:11:33 smithi153 ceph-mon[70044]: pgmap v13897: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:33.310 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:11:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:11:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:11:33.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:11:33 smithi083 ceph-mon[79930]: pgmap v13897: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:33.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:11:33 smithi083 ceph-mon[77242]: pgmap v13897: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:35.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:11:35 smithi153 ceph-mon[70044]: pgmap v13898: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:35.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:11:35 smithi083 ceph-mon[79930]: pgmap v13898: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:35.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:11:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:11:35.254Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:11:35.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:11:35 smithi083 ceph-mon[77242]: pgmap v13898: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:36.224 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:11:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:11:35.975Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:11:36.225 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:11:36 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:11:36.225 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:11:36 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:11:36.225 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:11:36 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:11:36.225 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:11:36 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:11:36.226 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:11:36 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:11:36.226 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:11:36 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:11:36.226 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:11:36 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:11:36.226 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:11:36 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:11:36.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:11:36 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:11:36.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:11:36 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:11:36.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:11:36 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:11:36.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:11:36 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:11:37.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:11:37 smithi153 ceph-mon[70044]: pgmap v13899: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:37.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:11:37 smithi083 ceph-mon[79930]: pgmap v13899: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:37.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:11:37 smithi083 ceph-mon[77242]: pgmap v13899: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:39.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:11:39 smithi153 ceph-mon[70044]: pgmap v13900: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:39.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:11:39 smithi083 ceph-mon[79930]: pgmap v13900: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:39.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:11:39 smithi083 ceph-mon[77242]: pgmap v13900: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:41.509 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:11:41 smithi083 ceph-mon[79930]: pgmap v13901: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:41.509 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:11:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:11:41.509 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:11:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:11:41.510 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:11:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:11:41.510 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:11:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:11:41.510 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:11:41 smithi083 ceph-mon[77242]: pgmap v13901: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:41.511 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:11:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:11:41.511 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:11:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:11:41.511 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:11:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:11:41.511 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:11:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:11:41.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:11:41 smithi153 ceph-mon[70044]: pgmap v13901: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:41.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:11:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:11:41.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:11:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:11:41.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:11:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:11:41.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:11:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:11:41.892 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:11:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:11:41] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T13:11:42.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:11:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:11:41.889Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:11:42.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:11:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:11:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:11:43.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:11:43 smithi153 ceph-mon[70044]: pgmap v13902: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:43.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:11:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:11:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:11:43.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:11:43 smithi083 ceph-mon[79930]: pgmap v13902: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:43.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:11:43 smithi083 ceph-mon[77242]: pgmap v13902: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:45.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:11:45 smithi153 ceph-mon[70044]: pgmap v13903: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:45.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:11:45 smithi083 ceph-mon[79930]: pgmap v13903: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:45.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:11:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:11:45.255Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:11:45.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:11:45 smithi083 ceph-mon[77242]: pgmap v13903: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:46.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:11:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:11:45.976Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:11:47.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:11:46 smithi083 ceph-mon[79930]: pgmap v13904: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:47.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:11:46 smithi083 ceph-mon[77242]: pgmap v13904: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:47.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:11:46 smithi153 ceph-mon[70044]: pgmap v13904: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:49.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:11:49 smithi153 ceph-mon[70044]: pgmap v13905: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:49.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:11:49 smithi083 ceph-mon[79930]: pgmap v13905: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:49.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:11:49 smithi083 ceph-mon[77242]: pgmap v13905: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:51.508 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:11:51 smithi083 ceph-mon[79930]: pgmap v13906: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:51.508 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:11:51 smithi083 ceph-mon[77242]: pgmap v13906: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:51.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:11:51 smithi153 ceph-mon[70044]: pgmap v13906: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:51.893 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:11:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:11:51] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T13:11:52.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:11:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:11:51.890Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:11:52.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:11:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:11:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:11:53.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:11:53 smithi153 ceph-mon[70044]: pgmap v13907: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:53.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:11:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:11:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:11:53.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:11:53 smithi083 ceph-mon[79930]: pgmap v13907: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:53.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:11:53 smithi083 ceph-mon[77242]: pgmap v13907: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:54.417 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:11:54.705 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:11:54.706 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 6m ago 7h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:11:54.706 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 4m ago 7h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:11:54.706 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 6m ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:11:54.707 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 4m ago 7h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:11:54.707 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 4m ago 7h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:11:54.707 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 6m ago 7h 896M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:11:54.707 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 6m ago 7h 1701M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:11:54.707 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 4m ago 7h 736M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:11:54.708 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 6m ago 7h 987M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:11:54.708 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 6m ago 7h 26.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:11:54.708 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 4m ago 7h 25.7M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:11:54.708 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 6m ago 7h 1229M 2339M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:11:54.709 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 6m ago 7h 1217M 2339M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:11:54.709 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 6m ago 7h 926M 2339M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:11:54.709 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 6m ago 7h 1191M 2339M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:11:54.709 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 4m ago 7h 1047M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:11:54.710 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 4m ago 7h 1311M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:11:54.710 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 4m ago 7h 1172M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:11:54.710 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 4m ago 7h 1391M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:11:54.710 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 4m ago 7h 174M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:11:55.056 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:11:55.056 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:11:55.056 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:11:55.056 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:11:55.057 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:11:55.057 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:11:55.057 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:11:55.057 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:11:55.058 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:11:55.058 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:11:55.058 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:11:55.058 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:11:55.058 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:11:55.059 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:11:55.059 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:11:55.059 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:11:55.059 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:11:55.060 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:11:55.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:11:55 smithi153 ceph-mon[70044]: pgmap v13908: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:55.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:11:55 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1530639281' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:11:55.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:11:55 smithi083 ceph-mon[77242]: pgmap v13908: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:55.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:11:55 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1530639281' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:11:55.714 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:11:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:11:55.254Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:11:55.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:11:55 smithi083 ceph-mon[79930]: pgmap v13908: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:55.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:11:55 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1530639281' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:11:56.230 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:11:56 smithi083 ceph-mon[79930]: from='client.50304 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:11:56.231 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:11:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:11:55.977Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:11:56.231 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:11:56 smithi083 ceph-mon[77242]: from='client.50304 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:11:56.231 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:11:56 smithi083 ceph-mon[77242]: from='client.60241 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:11:56.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:11:56 smithi153 ceph-mon[70044]: from='client.50304 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:11:56.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:11:56 smithi153 ceph-mon[70044]: from='client.60241 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:11:57.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:11:57 smithi153 ceph-mon[70044]: pgmap v13909: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:57.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:11:57 smithi083 ceph-mon[77242]: pgmap v13909: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:57.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:11:56 smithi083 ceph-mon[79930]: from='client.60241 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:11:57.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:11:57 smithi083 ceph-mon[79930]: pgmap v13909: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:59.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:11:59 smithi153 ceph-mon[70044]: pgmap v13910: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:59.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:11:59 smithi083 ceph-mon[77242]: pgmap v13910: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:11:59.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:11:59 smithi083 ceph-mon[79930]: pgmap v13910: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:01.509 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:12:01 smithi083 ceph-mon[79930]: pgmap v13911: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:01.509 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:12:01 smithi083 ceph-mon[77242]: pgmap v13911: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:01.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:12:01 smithi153 ceph-mon[70044]: pgmap v13911: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:01.894 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:12:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:12:01] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T13:12:02.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:12:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:12:01.892Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:12:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:12:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:12:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:12:03.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:12:03 smithi153 ceph-mon[70044]: pgmap v13912: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:12:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:12:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:12:03.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:12:03 smithi083 ceph-mon[79930]: pgmap v13912: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:03.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:12:03 smithi083 ceph-mon[77242]: pgmap v13912: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:05.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:12:05 smithi153 ceph-mon[70044]: pgmap v13913: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:05.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:12:05 smithi083 ceph-mon[79930]: pgmap v13913: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:05.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:12:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:12:05.256Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:12:05.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:12:05 smithi083 ceph-mon[77242]: pgmap v13913: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:06.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:12:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:12:05.978Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:12:07.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:12:06 smithi083 ceph-mon[79930]: pgmap v13914: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:07.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:12:06 smithi083 ceph-mon[77242]: pgmap v13914: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:07.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:12:06 smithi153 ceph-mon[70044]: pgmap v13914: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:09.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:12:09 smithi153 ceph-mon[70044]: pgmap v13915: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:09.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:12:09 smithi083 ceph-mon[79930]: pgmap v13915: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:09.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:12:09 smithi083 ceph-mon[77242]: pgmap v13915: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:11.509 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:12:11 smithi083 ceph-mon[79930]: pgmap v13916: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:11.509 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:12:11 smithi083 ceph-mon[77242]: pgmap v13916: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:11.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:12:11 smithi153 ceph-mon[70044]: pgmap v13916: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:11.894 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:12:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:12:11] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T13:12:12.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:12:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:12:11.891Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:12:12.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:12:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:12:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:12:13.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:12:13 smithi153 ceph-mon[70044]: pgmap v13917: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:13.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:12:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:12:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:12:13.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:12:13 smithi083 ceph-mon[77242]: pgmap v13917: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:13.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:12:13 smithi083 ceph-mon[79930]: pgmap v13917: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:15.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:12:15 smithi153 ceph-mon[70044]: pgmap v13918: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:15.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:12:15 smithi083 ceph-mon[79930]: pgmap v13918: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:15.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:12:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:12:15.257Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:12:15.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:12:15 smithi083 ceph-mon[77242]: pgmap v13918: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:16.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:12:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:12:15.979Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:12:17.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:12:16 smithi083 ceph-mon[79930]: pgmap v13919: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:17.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:12:16 smithi083 ceph-mon[77242]: pgmap v13919: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:17.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:12:16 smithi153 ceph-mon[70044]: pgmap v13919: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:19.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:12:19 smithi153 ceph-mon[70044]: pgmap v13920: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:19.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:12:19 smithi083 ceph-mon[79930]: pgmap v13920: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:19.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:12:19 smithi083 ceph-mon[77242]: pgmap v13920: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:21.509 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:12:21 smithi083 ceph-mon[79930]: pgmap v13921: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:21.509 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:12:21 smithi083 ceph-mon[77242]: pgmap v13921: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:21.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:12:21 smithi153 ceph-mon[70044]: pgmap v13921: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:21.895 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:12:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:12:21] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T13:12:22.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:12:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:12:21.892Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:12:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:12:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:12:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:12:23.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:12:23 smithi153 ceph-mon[70044]: pgmap v13922: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:23.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:12:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:12:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:12:23.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:12:23 smithi083 ceph-mon[77242]: pgmap v13922: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:23.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:12:23 smithi083 ceph-mon[79930]: pgmap v13922: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:25.386 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:12:25.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:12:25 smithi153 ceph-mon[70044]: pgmap v13923: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:25.673 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:12:25.674 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 6m ago 7h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:12:25.674 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 4m ago 7h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:12:25.674 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 6m ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:12:25.674 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 4m ago 7h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:12:25.675 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 4m ago 7h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:12:25.675 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 6m ago 7h 896M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:12:25.675 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 6m ago 7h 1701M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:12:25.675 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 4m ago 7h 736M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:12:25.676 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 6m ago 7h 987M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:12:25.676 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 6m ago 7h 26.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:12:25.676 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 4m ago 7h 25.7M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:12:25.676 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 6m ago 7h 1229M 2339M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:12:25.676 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 6m ago 7h 1217M 2339M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:12:25.677 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 6m ago 7h 926M 2339M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:12:25.677 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 6m ago 7h 1191M 2339M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:12:25.677 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 4m ago 7h 1047M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:12:25.677 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 4m ago 7h 1311M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:12:25.678 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 4m ago 7h 1172M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:12:25.678 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 4m ago 7h 1391M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:12:25.678 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 4m ago 7h 174M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:12:25.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:12:25 smithi083 ceph-mon[79930]: pgmap v13923: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:25.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:12:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:12:25.258Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:12:25.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:12:25 smithi083 ceph-mon[77242]: pgmap v13923: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:26.031 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:12:26.032 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:12:26.032 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:12:26.032 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:12:26.032 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:12:26.033 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:12:26.033 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:12:26.033 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:12:26.033 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:12:26.034 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:12:26.034 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:12:26.034 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:12:26.034 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:12:26.035 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:12:26.035 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:12:26.035 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:12:26.035 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:12:26.036 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:12:26.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:12:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:12:25.979Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:12:27.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:12:26 smithi083 ceph-mon[79930]: from='client.60253 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:12:27.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:12:26 smithi083 ceph-mon[79930]: from='client.50328 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:12:27.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:12:26 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2431440882' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:12:27.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:12:26 smithi083 ceph-mon[79930]: pgmap v13924: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:27.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:12:26 smithi083 ceph-mon[77242]: from='client.60253 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:12:27.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:12:26 smithi083 ceph-mon[77242]: from='client.50328 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:12:27.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:12:26 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2431440882' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:12:27.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:12:26 smithi083 ceph-mon[77242]: pgmap v13924: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:27.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:12:26 smithi153 ceph-mon[70044]: from='client.60253 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:12:27.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:12:26 smithi153 ceph-mon[70044]: from='client.50328 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:12:27.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:12:26 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2431440882' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:12:27.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:12:26 smithi153 ceph-mon[70044]: pgmap v13924: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:29.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:12:29 smithi153 ceph-mon[70044]: pgmap v13925: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:29.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:12:29 smithi083 ceph-mon[79930]: pgmap v13925: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:29.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:12:29 smithi083 ceph-mon[77242]: pgmap v13925: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:31.508 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:12:31 smithi083 ceph-mon[79930]: pgmap v13926: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:31.509 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:12:31 smithi083 ceph-mon[77242]: pgmap v13926: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:31.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:12:31 smithi153 ceph-mon[70044]: pgmap v13926: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:31.896 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:12:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:12:31] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T13:12:32.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:12:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:12:31.893Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:12:32.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:12:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:12:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:12:33.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:12:33 smithi153 ceph-mon[70044]: pgmap v13927: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:33.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:12:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:12:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:12:33.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:12:33 smithi083 ceph-mon[79930]: pgmap v13927: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:33.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:12:33 smithi083 ceph-mon[77242]: pgmap v13927: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:35.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:12:35 smithi153 ceph-mon[70044]: pgmap v13928: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:35.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:12:35 smithi083 ceph-mon[79930]: pgmap v13928: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:35.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:12:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:12:35.258Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:12:35.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:12:35 smithi083 ceph-mon[77242]: pgmap v13928: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:36.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:12:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:12:35.981Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:12:37.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:12:36 smithi083 ceph-mon[79930]: pgmap v13929: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:37.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:12:36 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:12:37.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:12:36 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:12:37.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:12:36 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:12:37.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:12:36 smithi083 ceph-mon[77242]: pgmap v13929: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:37.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:12:36 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:12:37.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:12:36 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:12:37.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:12:36 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:12:37.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:12:36 smithi153 ceph-mon[70044]: pgmap v13929: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:37.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:12:36 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:12:37.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:12:36 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:12:37.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:12:36 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:12:38.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:12:37 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:12:38.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:12:37 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:12:38.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:12:37 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:12:39.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:12:38 smithi083 ceph-mon[79930]: pgmap v13930: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:39.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:12:38 smithi083 ceph-mon[77242]: pgmap v13930: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:39.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:12:38 smithi153 ceph-mon[70044]: pgmap v13930: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:41.508 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:12:41 smithi083 ceph-mon[77242]: pgmap v13931: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:41.508 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:12:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:12:41.509 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:12:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:12:41.509 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:12:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:12:41.509 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:12:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:12:41.510 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:12:41 smithi083 ceph-mon[79930]: pgmap v13931: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:41.510 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:12:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:12:41.510 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:12:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:12:41.510 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:12:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:12:41.511 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:12:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:12:41.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:12:41 smithi153 ceph-mon[70044]: pgmap v13931: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:41.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:12:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:12:41.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:12:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:12:41.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:12:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:12:41.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:12:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:12:41.897 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:12:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:12:41] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T13:12:42.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:12:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:12:41.894Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:12:42.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:12:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:12:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:12:43.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:12:43 smithi153 ceph-mon[70044]: pgmap v13932: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:43.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:12:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:12:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:12:43.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:12:43 smithi083 ceph-mon[77242]: pgmap v13932: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:43.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:12:43 smithi083 ceph-mon[79930]: pgmap v13932: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:45.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:12:45 smithi153 ceph-mon[70044]: pgmap v13933: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:45.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:12:45 smithi083 ceph-mon[79930]: pgmap v13933: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:45.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:12:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:12:45.259Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:12:45.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:12:45 smithi083 ceph-mon[77242]: pgmap v13933: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:46.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:12:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:12:45.981Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:12:47.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:12:46 smithi083 ceph-mon[77242]: pgmap v13934: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:47.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:12:46 smithi083 ceph-mon[79930]: pgmap v13934: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:47.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:12:46 smithi153 ceph-mon[70044]: pgmap v13934: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:49.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:12:49 smithi153 ceph-mon[70044]: pgmap v13935: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:49.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:12:49 smithi083 ceph-mon[79930]: pgmap v13935: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:49.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:12:49 smithi083 ceph-mon[77242]: pgmap v13935: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:51.508 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:12:51 smithi083 ceph-mon[79930]: pgmap v13936: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:51.509 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:12:51 smithi083 ceph-mon[77242]: pgmap v13936: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:51.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:12:51 smithi153 ceph-mon[70044]: pgmap v13936: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:51.898 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:12:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:12:51] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T13:12:52.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:12:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:12:51.896Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:12:52.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:12:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:12:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:12:53.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:12:53 smithi153 ceph-mon[70044]: pgmap v13937: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:53.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:12:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:12:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:12:53.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:12:53 smithi083 ceph-mon[79930]: pgmap v13937: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:53.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:12:53 smithi083 ceph-mon[77242]: pgmap v13937: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:55.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:12:55 smithi153 ceph-mon[70044]: pgmap v13938: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:55.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:12:55 smithi083 ceph-mon[79930]: pgmap v13938: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:55.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:12:55 smithi083 ceph-mon[77242]: pgmap v13938: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:55.714 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:12:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:12:55.259Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:12:56.360 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:12:56.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:12:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:12:55.982Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:12:56.652 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:12:56.652 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 7m ago 7h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:12:56.652 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 5m ago 7h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:12:56.653 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 7m ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:12:56.653 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 5m ago 7h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:12:56.653 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 5m ago 7h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:12:56.653 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 7m ago 7h 896M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:12:56.654 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 7m ago 7h 1701M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:12:56.654 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 5m ago 7h 736M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:12:56.654 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 7m ago 7h 987M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:12:56.655 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 7m ago 7h 26.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:12:56.655 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 5m ago 7h 25.7M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:12:56.655 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 7m ago 7h 1229M 2339M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:12:56.655 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 7m ago 7h 1217M 2339M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:12:56.656 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 7m ago 7h 926M 2339M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:12:56.656 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 7m ago 7h 1191M 2339M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:12:56.656 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 5m ago 7h 1047M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:12:56.656 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 5m ago 7h 1311M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:12:56.656 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 5m ago 7h 1172M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:12:56.657 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 5m ago 7h 1391M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:12:56.657 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 5m ago 7h 174M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:12:57.008 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:12:57.008 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:12:57.008 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:12:57.008 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:12:57.009 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:12:57.009 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:12:57.009 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:12:57.009 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:12:57.010 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:12:57.010 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:12:57.010 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:12:57.010 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:12:57.011 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:12:57.011 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:12:57.011 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:12:57.011 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:12:57.011 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:12:57.012 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:12:57.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:12:56 smithi083 ceph-mon[79930]: pgmap v13939: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:57.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:12:56 smithi083 ceph-mon[77242]: pgmap v13939: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:57.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:12:56 smithi153 ceph-mon[70044]: pgmap v13939: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:58.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:12:57 smithi083 ceph-mon[79930]: from='client.50340 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:12:58.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:12:57 smithi083 ceph-mon[79930]: from='client.50346 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:12:58.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:12:57 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/224283891' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:12:58.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:12:57 smithi083 ceph-mon[77242]: from='client.50340 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:12:58.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:12:57 smithi083 ceph-mon[77242]: from='client.50346 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:12:58.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:12:57 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/224283891' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:12:58.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:12:57 smithi153 ceph-mon[70044]: from='client.50340 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:12:58.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:12:57 smithi153 ceph-mon[70044]: from='client.50346 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:12:58.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:12:57 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/224283891' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:12:59.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:12:58 smithi083 ceph-mon[77242]: pgmap v13940: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:59.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:12:58 smithi083 ceph-mon[79930]: pgmap v13940: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:12:59.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:12:58 smithi153 ceph-mon[70044]: pgmap v13940: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:01.509 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:13:01 smithi083 ceph-mon[79930]: pgmap v13941: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:01.510 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:13:01 smithi083 ceph-mon[77242]: pgmap v13941: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:01.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:13:01 smithi153 ceph-mon[70044]: pgmap v13941: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:01.900 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:13:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:13:01] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T13:13:02.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:13:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:13:01.896Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:13:02.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:13:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:13:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:13:03.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:13:03 smithi153 ceph-mon[70044]: pgmap v13942: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:13:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:13:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:13:03.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:13:03 smithi083 ceph-mon[77242]: pgmap v13942: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:03.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:13:03 smithi083 ceph-mon[79930]: pgmap v13942: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:05.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:13:05 smithi153 ceph-mon[70044]: pgmap v13943: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:05.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:13:05 smithi083 ceph-mon[79930]: pgmap v13943: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:05.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:13:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:13:05.260Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:13:05.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:13:05 smithi083 ceph-mon[77242]: pgmap v13943: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:06.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:13:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:13:05.983Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:13:07.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:13:06 smithi083 ceph-mon[77242]: pgmap v13944: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:07.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:13:06 smithi083 ceph-mon[79930]: pgmap v13944: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:07.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:13:06 smithi153 ceph-mon[70044]: pgmap v13944: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:09.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:13:09 smithi153 ceph-mon[70044]: pgmap v13945: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:09.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:13:09 smithi083 ceph-mon[79930]: pgmap v13945: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:09.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:13:09 smithi083 ceph-mon[77242]: pgmap v13945: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:11.509 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:13:11 smithi083 ceph-mon[79930]: pgmap v13946: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:11.509 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:13:11 smithi083 ceph-mon[77242]: pgmap v13946: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:11.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:13:11 smithi153 ceph-mon[70044]: pgmap v13946: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:11.899 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:13:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:13:11] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T13:13:12.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:13:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:13:11.897Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:13:12.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:13:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:13:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:13:13.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:13:13 smithi153 ceph-mon[70044]: pgmap v13947: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:13.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:13:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:13:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:13:13.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:13:13 smithi083 ceph-mon[77242]: pgmap v13947: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:13.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:13:13 smithi083 ceph-mon[79930]: pgmap v13947: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:15.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:13:15 smithi153 ceph-mon[70044]: pgmap v13948: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:15.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:13:15 smithi083 ceph-mon[79930]: pgmap v13948: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:15.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:13:15 smithi083 ceph-mon[77242]: pgmap v13948: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:15.714 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:13:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:13:15.261Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:13:16.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:13:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:13:15.983Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:13:17.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:13:16 smithi083 ceph-mon[79930]: pgmap v13949: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:17.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:13:16 smithi083 ceph-mon[77242]: pgmap v13949: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:17.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:13:16 smithi153 ceph-mon[70044]: pgmap v13949: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:19.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:13:19 smithi153 ceph-mon[70044]: pgmap v13950: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:19.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:13:19 smithi083 ceph-mon[79930]: pgmap v13950: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:19.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:13:19 smithi083 ceph-mon[77242]: pgmap v13950: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:21.508 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:13:21 smithi083 ceph-mon[77242]: pgmap v13951: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:21.509 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:13:21 smithi083 ceph-mon[79930]: pgmap v13951: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:21.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:13:21 smithi153 ceph-mon[70044]: pgmap v13951: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:21.899 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:13:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:13:21] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T13:13:22.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:13:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:13:21.897Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:13:22.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:13:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:13:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:13:23.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:13:23 smithi153 ceph-mon[70044]: pgmap v13952: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:23.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:13:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:13:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:13:23.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:13:23 smithi083 ceph-mon[79930]: pgmap v13952: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:23.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:13:23 smithi083 ceph-mon[77242]: pgmap v13952: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:25.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:13:25 smithi153 ceph-mon[70044]: pgmap v13953: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:25.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:13:25 smithi083 ceph-mon[79930]: pgmap v13953: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:25.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:13:25 smithi083 ceph-mon[77242]: pgmap v13953: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:25.714 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:13:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:13:25.261Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:13:26.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:13:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:13:25.984Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:13:27.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:13:26 smithi083 ceph-mon[79930]: pgmap v13954: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:27.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:13:26 smithi083 ceph-mon[77242]: pgmap v13954: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:27.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:13:26 smithi153 ceph-mon[70044]: pgmap v13954: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:27.331 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:13:27.618 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:13:27.618 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 8m ago 7h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:13:27.618 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 5m ago 7h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:13:27.619 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 8m ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:13:27.619 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 5m ago 7h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:13:27.619 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 5m ago 7h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:13:27.619 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 8m ago 7h 896M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:13:27.620 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 8m ago 7h 1701M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:13:27.620 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 5m ago 7h 736M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:13:27.620 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 8m ago 7h 987M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:13:27.620 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 8m ago 7h 26.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:13:27.620 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 5m ago 7h 25.7M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:13:27.621 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 8m ago 7h 1229M 2339M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:13:27.621 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 8m ago 7h 1217M 2339M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:13:27.621 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 8m ago 7h 926M 2339M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:13:27.621 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 8m ago 7h 1191M 2339M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:13:27.622 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 5m ago 7h 1047M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:13:27.622 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 5m ago 7h 1311M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:13:27.622 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 5m ago 7h 1172M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:13:27.622 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 5m ago 7h 1391M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:13:27.622 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 5m ago 7h 174M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:13:27.972 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:13:27.972 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:13:27.972 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:13:27.972 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:13:27.973 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:13:27.973 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:13:27.973 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:13:27.973 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:13:27.974 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:13:27.974 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:13:27.974 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:13:27.974 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:13:27.974 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:13:27.975 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:13:27.975 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:13:27.975 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:13:27.975 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:13:27.976 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:13:28.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:13:28 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2207810110' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:13:28.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:13:28 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2207810110' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:13:28.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:13:28 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2207810110' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:13:29.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:13:29 smithi153 ceph-mon[70044]: from='client.60289 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:13:29.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:13:29 smithi153 ceph-mon[70044]: from='client.60295 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:13:29.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:13:29 smithi153 ceph-mon[70044]: pgmap v13955: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:29.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:13:29 smithi083 ceph-mon[79930]: from='client.60289 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:13:29.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:13:29 smithi083 ceph-mon[79930]: from='client.60295 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:13:29.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:13:29 smithi083 ceph-mon[79930]: pgmap v13955: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:29.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:13:29 smithi083 ceph-mon[77242]: from='client.60289 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:13:29.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:13:29 smithi083 ceph-mon[77242]: from='client.60295 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:13:29.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:13:29 smithi083 ceph-mon[77242]: pgmap v13955: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:31.508 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:13:31 smithi083 ceph-mon[79930]: pgmap v13956: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:31.509 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:13:31 smithi083 ceph-mon[77242]: pgmap v13956: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:31.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:13:31 smithi153 ceph-mon[70044]: pgmap v13956: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:31.901 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:13:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:13:31] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T13:13:32.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:13:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:13:31.898Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:13:32.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:13:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:13:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:13:33.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:13:33 smithi153 ceph-mon[70044]: pgmap v13957: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:33.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:13:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:13:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:13:33.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:13:33 smithi083 ceph-mon[79930]: pgmap v13957: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:33.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:13:33 smithi083 ceph-mon[77242]: pgmap v13957: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:35.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:13:35 smithi153 ceph-mon[70044]: pgmap v13958: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:35.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:13:35 smithi083 ceph-mon[79930]: pgmap v13958: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:35.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:13:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:13:35.263Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:13:35.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:13:35 smithi083 ceph-mon[77242]: pgmap v13958: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:36.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:13:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:13:35.986Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:13:37.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:13:36 smithi083 ceph-mon[79930]: pgmap v13959: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:37.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:13:36 smithi083 ceph-mon[77242]: pgmap v13959: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:37.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:13:36 smithi153 ceph-mon[70044]: pgmap v13959: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:38.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:13:37 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:13:38.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:13:37 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:13:38.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:13:37 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:13:38.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:13:37 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:13:38.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:13:37 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:13:38.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:13:37 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:13:38.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:13:37 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:13:38.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:13:37 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:13:38.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:13:37 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:13:38.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:13:37 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:13:38.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:13:37 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:13:38.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:13:37 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:13:38.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:13:37 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:13:38.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:13:37 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:13:38.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:13:37 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:13:38.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:13:37 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:13:38.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:13:37 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:13:38.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:13:37 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:13:38.218 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:13:37 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:13:38.218 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:13:37 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:13:38.218 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:13:37 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:13:38.218 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:13:37 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:13:38.219 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:13:37 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:13:38.219 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:13:37 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:13:38.219 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:13:37 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:13:38.219 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:13:37 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:13:38.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:13:37 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:13:38.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:13:37 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:13:38.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:13:37 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:13:38.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:13:37 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:13:38.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:13:37 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:13:38.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:13:37 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:13:38.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:13:37 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:13:38.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:13:37 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:13:38.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:13:37 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:13:38.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:13:37 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:13:38.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:13:37 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:13:38.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:13:37 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:13:38.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:13:37 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:13:39.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:13:38 smithi083 ceph-mon[79930]: Adjusting osd_memory_target on smithi083 to 2330M 2022-09-29T13:13:39.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:13:38 smithi083 ceph-mon[79930]: pgmap v13960: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:39.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:13:38 smithi083 ceph-mon[77242]: Adjusting osd_memory_target on smithi083 to 2330M 2022-09-29T13:13:39.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:13:38 smithi083 ceph-mon[77242]: pgmap v13960: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:39.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:13:38 smithi153 ceph-mon[70044]: Adjusting osd_memory_target on smithi083 to 2330M 2022-09-29T13:13:39.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:13:38 smithi153 ceph-mon[70044]: pgmap v13960: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:41.508 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:13:41 smithi083 ceph-mon[79930]: pgmap v13961: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:41.509 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:13:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:13:41.509 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:13:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:13:41.509 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:13:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:13:41.510 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:13:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:13:41.510 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:13:41 smithi083 ceph-mon[77242]: pgmap v13961: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:41.510 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:13:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:13:41.510 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:13:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:13:41.511 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:13:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:13:41.511 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:13:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:13:41.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:13:41 smithi153 ceph-mon[70044]: pgmap v13961: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:41.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:13:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:13:41.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:13:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:13:41.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:13:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:13:41.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:13:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:13:41.902 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:13:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:13:41] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T13:13:42.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:13:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:13:41.899Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:13:42.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:13:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:13:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:13:43.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:13:43 smithi153 ceph-mon[70044]: pgmap v13962: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:43.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:13:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:13:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:13:43.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:13:43 smithi083 ceph-mon[77242]: pgmap v13962: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:43.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:13:43 smithi083 ceph-mon[79930]: pgmap v13962: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:45.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:13:45 smithi153 ceph-mon[70044]: pgmap v13963: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:45.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:13:45 smithi083 ceph-mon[79930]: pgmap v13963: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:45.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:13:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:13:45.263Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:13:45.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:13:45 smithi083 ceph-mon[77242]: pgmap v13963: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:46.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:13:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:13:45.986Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:13:47.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:13:46 smithi083 ceph-mon[79930]: pgmap v13964: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:47.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:13:46 smithi083 ceph-mon[77242]: pgmap v13964: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:47.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:13:46 smithi153 ceph-mon[70044]: pgmap v13964: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:49.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:13:49 smithi153 ceph-mon[70044]: pgmap v13965: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:49.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:13:49 smithi083 ceph-mon[77242]: pgmap v13965: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:49.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:13:49 smithi083 ceph-mon[79930]: pgmap v13965: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:51.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:13:51 smithi153 ceph-mon[70044]: pgmap v13966: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:51.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:13:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:13:51] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T13:13:51.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:13:51 smithi083 ceph-mon[79930]: pgmap v13966: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:51.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:13:51 smithi083 ceph-mon[77242]: pgmap v13966: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:52.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:13:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:13:51.900Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:13:52.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:13:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:13:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:13:53.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:13:53 smithi153 ceph-mon[70044]: pgmap v13967: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:53.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:13:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:13:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:13:53.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:13:53 smithi083 ceph-mon[79930]: pgmap v13967: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:53.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:13:53 smithi083 ceph-mon[77242]: pgmap v13967: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:55.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:13:55 smithi153 ceph-mon[70044]: pgmap v13968: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:55.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:13:55 smithi083 ceph-mon[79930]: pgmap v13968: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:55.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:13:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:13:55.264Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:13:55.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:13:55 smithi083 ceph-mon[77242]: pgmap v13968: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:56.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:13:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:13:55.986Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:13:57.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:13:56 smithi083 ceph-mon[79930]: pgmap v13969: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:57.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:13:56 smithi083 ceph-mon[77242]: pgmap v13969: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:57.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:13:56 smithi153 ceph-mon[70044]: pgmap v13969: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:58.305 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:13:58.590 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:13:58.590 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 8m ago 7h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:13:58.590 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 6m ago 7h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:13:58.591 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 8m ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:13:58.591 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 6m ago 7h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:13:58.591 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 6m ago 7h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:13:58.591 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 8m ago 7h 896M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:13:58.592 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 8m ago 7h 1701M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:13:58.592 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 6m ago 7h 736M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:13:58.592 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 8m ago 7h 987M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:13:58.592 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 8m ago 7h 26.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:13:58.593 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 6m ago 7h 25.7M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:13:58.593 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 8m ago 7h 1229M 2330M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:13:58.593 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 8m ago 7h 1217M 2330M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:13:58.593 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 8m ago 7h 926M 2330M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:13:58.593 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 8m ago 7h 1191M 2330M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:13:58.594 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 6m ago 7h 1047M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:13:58.594 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 6m ago 7h 1311M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:13:58.594 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 6m ago 7h 1172M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:13:58.594 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 6m ago 7h 1391M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:13:58.595 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 6m ago 7h 174M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:13:58.945 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:13:58.945 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:13:58.946 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:13:58.946 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:13:58.946 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:13:58.946 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:13:58.947 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:13:58.947 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:13:58.947 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:13:58.947 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:13:58.947 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:13:58.948 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:13:58.948 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:13:58.948 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:13:58.948 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:13:58.949 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:13:58.949 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:13:58.949 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:13:59.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:13:59 smithi153 ceph-mon[70044]: pgmap v13970: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:59.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:13:59 smithi153 ceph-mon[70044]: from='client.50376 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:13:59.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:13:59 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2790955604' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:13:59.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:13:59 smithi083 ceph-mon[79930]: pgmap v13970: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:59.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:13:59 smithi083 ceph-mon[79930]: from='client.50376 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:13:59.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:13:59 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2790955604' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:13:59.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:13:59 smithi083 ceph-mon[77242]: pgmap v13970: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:13:59.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:13:59 smithi083 ceph-mon[77242]: from='client.50376 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:13:59.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:13:59 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2790955604' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:14:00.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:14:00 smithi153 ceph-mon[70044]: from='client.60313 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:14:00.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:14:00 smithi083 ceph-mon[79930]: from='client.60313 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:14:00.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:14:00 smithi083 ceph-mon[77242]: from='client.60313 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:14:01.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:14:01 smithi153 ceph-mon[70044]: pgmap v13971: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:14:01.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:14:01 smithi083 ceph-mon[79930]: pgmap v13971: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:14:01.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:14:01 smithi083 ceph-mon[77242]: pgmap v13971: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:14:01.714 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:14:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:14:01] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T13:14:02.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:14:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:14:01.900Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:14:02.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:14:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:14:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:14:03.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:14:03 smithi153 ceph-mon[70044]: pgmap v13972: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:14:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:14:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:14:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:14:03.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:14:03 smithi083 ceph-mon[79930]: pgmap v13972: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:14:03.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:14:03 smithi083 ceph-mon[77242]: pgmap v13972: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:14:05.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:14:05 smithi153 ceph-mon[70044]: pgmap v13973: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:14:05.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:14:05 smithi083 ceph-mon[79930]: pgmap v13973: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:14:05.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:14:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:14:05.263Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:14:05.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:14:05 smithi083 ceph-mon[77242]: pgmap v13973: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:14:06.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:14:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:14:05.987Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:14:07.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:14:06 smithi083 ceph-mon[79930]: pgmap v13974: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:14:07.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:14:06 smithi083 ceph-mon[77242]: pgmap v13974: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:14:07.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:14:06 smithi153 ceph-mon[70044]: pgmap v13974: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:14:09.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:14:09 smithi153 ceph-mon[70044]: pgmap v13975: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:14:09.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:14:09 smithi083 ceph-mon[79930]: pgmap v13975: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:14:09.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:14:09 smithi083 ceph-mon[77242]: pgmap v13975: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:14:11.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:14:11 smithi153 ceph-mon[70044]: pgmap v13976: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:14:11.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:14:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:14:11] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T13:14:11.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:14:11 smithi083 ceph-mon[79930]: pgmap v13976: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:14:11.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:14:11 smithi083 ceph-mon[77242]: pgmap v13976: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:14:12.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:14:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:14:11.901Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:14:12.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:14:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:14:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:14:13.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:14:13 smithi153 ceph-mon[70044]: pgmap v13977: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:14:13.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:14:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:14:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:14:13.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:14:13 smithi083 ceph-mon[79930]: pgmap v13977: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:14:13.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:14:13 smithi083 ceph-mon[77242]: pgmap v13977: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:14:15.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:14:15 smithi153 ceph-mon[70044]: pgmap v13978: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:14:15.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:14:15 smithi083 ceph-mon[79930]: pgmap v13978: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:14:15.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:14:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:14:15.264Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:14:15.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:14:15 smithi083 ceph-mon[77242]: pgmap v13978: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:14:16.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:14:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:14:15.988Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:14:17.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:14:16 smithi083 ceph-mon[79930]: pgmap v13979: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:14:17.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:14:16 smithi083 ceph-mon[77242]: pgmap v13979: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:14:17.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:14:16 smithi153 ceph-mon[70044]: pgmap v13979: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:14:19.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:14:19 smithi153 ceph-mon[70044]: pgmap v13980: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:14:19.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:14:19 smithi083 ceph-mon[79930]: pgmap v13980: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:14:19.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:14:19 smithi083 ceph-mon[77242]: pgmap v13980: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:14:21.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:14:21 smithi153 ceph-mon[70044]: pgmap v13981: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:14:21.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:14:21 smithi083 ceph-mon[77242]: pgmap v13981: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:14:21.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:14:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:14:21] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T13:14:21.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:14:21 smithi083 ceph-mon[79930]: pgmap v13981: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:14:22.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:14:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:14:21.901Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:14:22.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:14:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:14:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:14:23.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:14:23 smithi153 ceph-mon[70044]: pgmap v13982: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:14:23.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:14:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:14:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:14:23.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:14:23 smithi083 ceph-mon[77242]: pgmap v13982: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:14:23.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:14:23 smithi083 ceph-mon[79930]: pgmap v13982: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:14:25.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:14:25 smithi153 ceph-mon[70044]: pgmap v13983: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:14:25.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:14:25 smithi083 ceph-mon[77242]: pgmap v13983: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:14:25.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:14:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:14:25.265Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:14:25.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:14:25 smithi083 ceph-mon[79930]: pgmap v13983: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:14:26.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:14:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:14:25.988Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:14:27.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:14:26 smithi083 ceph-mon[77242]: pgmap v13984: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:14:27.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:14:26 smithi083 ceph-mon[79930]: pgmap v13984: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:14:27.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:14:26 smithi153 ceph-mon[70044]: pgmap v13984: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:14:29.269 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:14:29.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:14:29 smithi153 ceph-mon[70044]: pgmap v13985: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:14:29.559 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:14:29.560 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 9m ago 7h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:14:29.560 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 6m ago 7h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:14:29.560 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 9m ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:14:29.560 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 6m ago 7h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:14:29.561 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 6m ago 7h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:14:29.561 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 9m ago 7h 896M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:14:29.561 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 9m ago 7h 1701M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:14:29.561 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 6m ago 7h 736M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:14:29.561 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 9m ago 7h 987M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:14:29.562 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 9m ago 7h 26.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:14:29.562 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 6m ago 7h 25.7M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:14:29.562 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 9m ago 7h 1229M 2330M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:14:29.562 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 9m ago 7h 1217M 2330M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:14:29.563 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 9m ago 7h 926M 2330M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:14:29.563 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 9m ago 7h 1191M 2330M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:14:29.563 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 6m ago 7h 1047M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:14:29.563 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 6m ago 7h 1311M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:14:29.564 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 6m ago 7h 1172M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:14:29.564 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 6m ago 7h 1391M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:14:29.564 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 6m ago 7h 174M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:14:29.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:14:29 smithi083 ceph-mon[79930]: pgmap v13985: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:14:29.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:14:29 smithi083 ceph-mon[77242]: pgmap v13985: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:14:29.915 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:14:29.916 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:14:29.916 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:14:29.916 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:14:29.916 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:14:29.917 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:14:29.917 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:14:29.917 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:14:29.917 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:14:29.918 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:14:29.918 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:14:29.918 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:14:29.918 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:14:29.918 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:14:29.919 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:14:29.919 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:14:29.919 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:14:29.919 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:14:30.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:14:30 smithi153 ceph-mon[70044]: from='client.50394 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:14:30.561 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:14:30 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1256488391' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:14:30.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:14:30 smithi083 ceph-mon[79930]: from='client.50394 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:14:30.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:14:30 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1256488391' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:14:30.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:14:30 smithi083 ceph-mon[77242]: from='client.50394 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:14:30.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:14:30 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1256488391' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:14:31.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:14:31 smithi153 ceph-mon[70044]: from='client.50400 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:14:31.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:14:31 smithi153 ceph-mon[70044]: pgmap v13986: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:14:31.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:14:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:14:31] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T13:14:31.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:14:31 smithi083 ceph-mon[79930]: from='client.50400 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:14:31.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:14:31 smithi083 ceph-mon[79930]: pgmap v13986: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:14:31.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:14:31 smithi083 ceph-mon[77242]: from='client.50400 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:14:31.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:14:31 smithi083 ceph-mon[77242]: pgmap v13986: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:14:32.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:14:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:14:31.902Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:14:32.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:14:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:14:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:14:33.275 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:14:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:14:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:14:33.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:14:33 smithi153 ceph-mon[70044]: pgmap v13987: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:14:33.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:14:33 smithi083 ceph-mon[77242]: pgmap v13987: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:14:33.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:14:33 smithi083 ceph-mon[79930]: pgmap v13987: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:14:35.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:14:35 smithi153 ceph-mon[70044]: pgmap v13988: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:14:35.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:14:35 smithi083 ceph-mon[79930]: pgmap v13988: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:14:35.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:14:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:14:35.265Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:14:35.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:14:35 smithi083 ceph-mon[77242]: pgmap v13988: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:14:36.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:14:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:14:35.990Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:14:37.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:14:36 smithi083 ceph-mon[79930]: pgmap v13989: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:14:37.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:14:36 smithi083 ceph-mon[77242]: pgmap v13989: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:14:37.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:14:36 smithi153 ceph-mon[70044]: pgmap v13989: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:14:38.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:14:37 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:14:38.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:14:37 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:14:38.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:14:37 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:14:38.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:14:37 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:14:38.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:14:37 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:14:38.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:14:37 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:14:38.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:14:37 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:14:38.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:14:37 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:14:38.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:14:37 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:14:39.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:14:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:14:39.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:14:39 smithi083 ceph-mon[79930]: pgmap v13990: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:14:39.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:14:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:14:39.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:14:39 smithi083 ceph-mon[77242]: pgmap v13990: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:14:39.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:14:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:14:39.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:14:39 smithi153 ceph-mon[70044]: pgmap v13990: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:14:41.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:14:41 smithi153 ceph-mon[70044]: pgmap v13991: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:14:41.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:14:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:14:41.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:14:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:14:41.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:14:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:14:41.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:14:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:14:41.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:14:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:14:41] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T13:14:41.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:14:41 smithi083 ceph-mon[79930]: pgmap v13991: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:14:41.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:14:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:14:41.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:14:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:14:41.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:14:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:14:41.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:14:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:14:41.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:14:41 smithi083 ceph-mon[77242]: pgmap v13991: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:14:41.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:14:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:14:41.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:14:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:14:41.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:14:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:14:41.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:14:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:14:42.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:14:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:14:41.903Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:14:42.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:14:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:14:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:14:43.275 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:14:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:14:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:14:43.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:14:43 smithi153 ceph-mon[70044]: pgmap v13992: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:14:43.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:14:43 smithi083 ceph-mon[79930]: pgmap v13992: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:14:43.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:14:43 smithi083 ceph-mon[77242]: pgmap v13992: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:14:45.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:14:45 smithi153 ceph-mon[70044]: pgmap v13993: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:14:45.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:14:45 smithi083 ceph-mon[77242]: pgmap v13993: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:14:45.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:14:45 smithi083 ceph-mon[79930]: pgmap v13993: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:14:45.714 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:14:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:14:45.266Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:14:46.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:14:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:14:45.991Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:14:47.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:14:46 smithi083 ceph-mon[79930]: pgmap v13994: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:14:47.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:14:46 smithi083 ceph-mon[77242]: pgmap v13994: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:14:47.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:14:46 smithi153 ceph-mon[70044]: pgmap v13994: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:14:49.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:14:49 smithi153 ceph-mon[70044]: pgmap v13995: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:14:49.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:14:49 smithi083 ceph-mon[77242]: pgmap v13995: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:14:49.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:14:49 smithi083 ceph-mon[79930]: pgmap v13995: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:14:51.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:14:51 smithi153 ceph-mon[70044]: pgmap v13996: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:14:51.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:14:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:14:51] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T13:14:51.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:14:51 smithi083 ceph-mon[79930]: pgmap v13996: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:14:51.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:14:51 smithi083 ceph-mon[77242]: pgmap v13996: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:14:52.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:14:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:14:51.903Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:14:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:14:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:14:52.316Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:14:53.279 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:14:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:14:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:14:53.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:14:53 smithi153 ceph-mon[70044]: pgmap v13997: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:14:53.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:14:53 smithi083 ceph-mon[79930]: pgmap v13997: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:14:53.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:14:53 smithi083 ceph-mon[77242]: pgmap v13997: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:14:55.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:14:55 smithi153 ceph-mon[70044]: pgmap v13998: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:14:55.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:14:55 smithi083 ceph-mon[79930]: pgmap v13998: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:14:55.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:14:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:14:55.266Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:14:55.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:14:55 smithi083 ceph-mon[77242]: pgmap v13998: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:14:56.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:14:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:14:55.991Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:14:57.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:14:56 smithi083 ceph-mon[79930]: pgmap v13999: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:14:57.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:14:56 smithi083 ceph-mon[77242]: pgmap v13999: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:14:57.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:14:56 smithi153 ceph-mon[70044]: pgmap v13999: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:14:59.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:14:59 smithi153 ceph-mon[70044]: pgmap v14000: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:14:59.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:14:59 smithi083 ceph-mon[77242]: pgmap v14000: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:14:59.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:14:59 smithi083 ceph-mon[79930]: pgmap v14000: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:00.239 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:15:00.529 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:15:00.529 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 9m ago 7h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:15:00.530 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 7m ago 7h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:15:00.530 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 9m ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:15:00.530 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 7m ago 7h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:15:00.530 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 7m ago 7h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:15:00.531 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 9m ago 7h 896M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:15:00.531 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 9m ago 7h 1701M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:15:00.531 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 7m ago 7h 736M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:15:00.531 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 9m ago 7h 987M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:15:00.532 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 9m ago 7h 26.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:15:00.532 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 7m ago 7h 25.7M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:15:00.532 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 9m ago 7h 1229M 2330M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:15:00.532 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 9m ago 7h 1217M 2330M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:15:00.533 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 9m ago 7h 926M 2330M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:15:00.533 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 9m ago 7h 1191M 2330M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:15:00.533 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 7m ago 7h 1047M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:15:00.533 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 7m ago 7h 1311M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:15:00.534 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 7m ago 7h 1172M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:15:00.534 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 7m ago 7h 1391M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:15:00.534 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 7m ago 7h 174M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:15:00.885 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:15:00.885 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:15:00.886 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:15:00.886 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:15:00.886 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:15:00.887 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:15:00.887 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:15:00.887 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:15:00.887 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:15:00.888 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:15:00.888 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:15:00.888 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:15:00.888 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:15:00.889 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:15:00.889 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:15:00.889 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:15:00.889 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:15:00.890 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:15:01.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:15:01 smithi153 ceph-mon[70044]: pgmap v14001: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:01.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:15:01 smithi153 ceph-mon[70044]: from='client.50412 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:15:01.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:15:01 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/322345549' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:15:01.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:15:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:15:01] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T13:15:01.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:15:01 smithi083 ceph-mon[77242]: pgmap v14001: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:01.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:15:01 smithi083 ceph-mon[77242]: from='client.50412 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:15:01.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:15:01 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/322345549' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:15:01.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:15:01 smithi083 ceph-mon[79930]: pgmap v14001: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:01.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:15:01 smithi083 ceph-mon[79930]: from='client.50412 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:15:01.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:15:01 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/322345549' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:15:02.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:15:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:15:01.905Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:15:02.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:15:02 smithi153 ceph-mon[70044]: from='client.50418 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:15:02.559 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:15:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:15:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:15:02.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:15:02 smithi083 ceph-mon[79930]: from='client.50418 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:15:02.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:15:02 smithi083 ceph-mon[77242]: from='client.50418 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:15:03.287 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:15:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:15:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:15:03.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:15:03 smithi153 ceph-mon[70044]: pgmap v14002: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:03.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:15:03 smithi083 ceph-mon[79930]: pgmap v14002: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:03.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:15:03 smithi083 ceph-mon[77242]: pgmap v14002: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:05.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:15:05 smithi153 ceph-mon[70044]: pgmap v14003: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:05.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:15:05 smithi083 ceph-mon[79930]: pgmap v14003: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:05.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:15:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:15:05.267Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:15:05.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:15:05 smithi083 ceph-mon[77242]: pgmap v14003: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:06.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:15:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:15:05.992Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:15:07.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:15:06 smithi083 ceph-mon[79930]: pgmap v14004: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:07.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:15:06 smithi083 ceph-mon[77242]: pgmap v14004: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:07.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:15:06 smithi153 ceph-mon[70044]: pgmap v14004: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:09.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:15:09 smithi153 ceph-mon[70044]: pgmap v14005: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:09.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:15:09 smithi083 ceph-mon[79930]: pgmap v14005: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:09.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:15:09 smithi083 ceph-mon[77242]: pgmap v14005: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:11.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:15:11 smithi153 ceph-mon[70044]: pgmap v14006: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:11.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:15:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:15:11] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T13:15:11.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:15:11 smithi083 ceph-mon[79930]: pgmap v14006: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:11.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:15:11 smithi083 ceph-mon[77242]: pgmap v14006: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:12.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:15:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:15:11.905Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:15:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:15:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:15:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:15:13.288 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:15:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:15:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:15:13.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:15:13 smithi153 ceph-mon[70044]: pgmap v14007: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:13.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:15:13 smithi083 ceph-mon[77242]: pgmap v14007: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:13.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:15:13 smithi083 ceph-mon[79930]: pgmap v14007: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:15.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:15:15 smithi153 ceph-mon[70044]: pgmap v14008: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:15.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:15:15 smithi083 ceph-mon[79930]: pgmap v14008: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:15.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:15:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:15:15.267Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:15:15.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:15:15 smithi083 ceph-mon[77242]: pgmap v14008: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:16.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:15:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:15:15.992Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:15:17.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:15:16 smithi083 ceph-mon[77242]: pgmap v14009: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:17.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:15:16 smithi083 ceph-mon[79930]: pgmap v14009: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:17.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:15:16 smithi153 ceph-mon[70044]: pgmap v14009: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:19.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:15:19 smithi153 ceph-mon[70044]: pgmap v14010: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:19.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:15:19 smithi083 ceph-mon[79930]: pgmap v14010: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:19.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:15:19 smithi083 ceph-mon[77242]: pgmap v14010: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:21.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:15:21 smithi153 ceph-mon[70044]: pgmap v14011: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:21.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:15:21 smithi083 ceph-mon[79930]: pgmap v14011: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:21.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:15:21 smithi083 ceph-mon[77242]: pgmap v14011: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:21.714 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:15:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:15:21] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T13:15:22.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:15:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:15:21.906Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:15:22.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:15:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:15:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:15:23.290 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:15:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:15:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:15:23.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:15:23 smithi153 ceph-mon[70044]: pgmap v14012: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:23.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:15:23 smithi083 ceph-mon[79930]: pgmap v14012: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:23.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:15:23 smithi083 ceph-mon[77242]: pgmap v14012: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:25.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:15:25 smithi153 ceph-mon[70044]: pgmap v14013: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:25.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:15:25 smithi083 ceph-mon[77242]: pgmap v14013: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:25.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:15:25 smithi083 ceph-mon[79930]: pgmap v14013: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:25.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:15:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:15:25.267Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:15:26.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:15:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:15:25.993Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:15:27.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:15:26 smithi083 ceph-mon[79930]: pgmap v14014: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:27.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:15:26 smithi083 ceph-mon[77242]: pgmap v14014: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:27.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:15:26 smithi153 ceph-mon[70044]: pgmap v14014: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:29.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:15:29 smithi153 ceph-mon[70044]: pgmap v14015: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:29.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:15:29 smithi083 ceph-mon[79930]: pgmap v14015: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:29.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:15:29 smithi083 ceph-mon[77242]: pgmap v14015: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:31.211 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:15:31.500 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:15:31.501 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 10m ago 7h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:15:31.501 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 7m ago 7h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:15:31.501 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 10m ago 7h 25.9M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:15:31.501 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 7m ago 7h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:15:31.502 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 7m ago 7h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:15:31.502 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 10m ago 7h 896M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:15:31.502 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 10m ago 7h 1701M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:15:31.503 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 7m ago 7h 736M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:15:31.503 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 10m ago 7h 987M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:15:31.503 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 10m ago 7h 26.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:15:31.503 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 7m ago 7h 25.7M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:15:31.504 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 10m ago 7h 1229M 2330M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:15:31.504 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 10m ago 7h 1217M 2330M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:15:31.504 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 10m ago 7h 926M 2330M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:15:31.504 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 10m ago 7h 1191M 2330M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:15:31.504 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 7m ago 7h 1047M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:15:31.505 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 7m ago 7h 1311M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:15:31.505 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 7m ago 7h 1172M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:15:31.505 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 7m ago 7h 1391M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:15:31.505 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 7m ago 7h 174M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:15:31.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:15:31 smithi153 ceph-mon[70044]: pgmap v14016: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:31.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:15:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:15:31] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T13:15:31.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:15:31 smithi083 ceph-mon[79930]: pgmap v14016: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:31.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:15:31 smithi083 ceph-mon[77242]: pgmap v14016: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:31.860 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:15:31.861 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:15:31.861 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:15:31.861 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:15:31.861 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:15:31.862 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:15:31.862 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:15:31.862 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:15:31.862 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:15:31.862 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:15:31.863 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:15:31.863 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:15:31.863 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:15:31.863 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:15:31.864 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:15:31.864 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:15:31.864 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:15:31.864 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:15:32.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:15:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:15:31.905Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:15:32.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:15:32 smithi153 ceph-mon[70044]: from='client.50430 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:15:32.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:15:32 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1711744678' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:15:32.559 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:15:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:15:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:15:32.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:15:32 smithi083 ceph-mon[79930]: from='client.50430 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:15:32.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:15:32 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1711744678' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:15:32.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:15:32 smithi083 ceph-mon[77242]: from='client.50430 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:15:32.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:15:32 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1711744678' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:15:33.299 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:15:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:15:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:15:33.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:15:33 smithi153 ceph-mon[70044]: from='client.60367 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:15:33.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:15:33 smithi153 ceph-mon[70044]: pgmap v14017: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:33.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:15:33 smithi083 ceph-mon[79930]: from='client.60367 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:15:33.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:15:33 smithi083 ceph-mon[79930]: pgmap v14017: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:33.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:15:33 smithi083 ceph-mon[77242]: from='client.60367 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:15:33.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:15:33 smithi083 ceph-mon[77242]: pgmap v14017: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:35.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:15:35 smithi153 ceph-mon[70044]: pgmap v14018: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:35.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:15:35 smithi083 ceph-mon[79930]: pgmap v14018: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:35.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:15:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:15:35.268Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:15:35.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:15:35 smithi083 ceph-mon[77242]: pgmap v14018: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:36.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:15:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:15:35.994Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:15:37.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:15:36 smithi083 ceph-mon[79930]: pgmap v14019: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:37.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:15:36 smithi083 ceph-mon[77242]: pgmap v14019: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:37.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:15:36 smithi153 ceph-mon[70044]: pgmap v14019: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:38.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:15:38 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:15:38.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:15:38 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:15:38.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:15:38 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:15:38.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:15:38 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:15:38.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:15:38 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:15:38.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:15:38 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:15:38.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:15:38 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:15:38.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:15:38 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:15:38.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:15:38 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:15:39.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:15:39 smithi153 ceph-mon[70044]: pgmap v14020: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:39.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:15:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:15:39.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:15:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:15:39.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:15:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:15:39.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:15:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:15:39.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:15:39 smithi083 ceph-mon[77242]: pgmap v14020: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:39.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:15:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:15:39.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:15:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:15:39.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:15:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:15:39.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:15:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:15:39.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:15:39 smithi083 ceph-mon[79930]: pgmap v14020: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:39.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:15:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:15:39.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:15:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:15:39.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:15:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:15:39.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:15:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:15:41.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:15:41 smithi153 ceph-mon[70044]: pgmap v14021: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:41.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:15:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:15:41.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:15:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:15:41.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:15:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:15:41.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:15:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:15:41.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:15:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:15:41] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T13:15:41.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:15:41 smithi083 ceph-mon[79930]: pgmap v14021: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:41.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:15:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:15:41.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:15:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:15:41.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:15:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:15:41.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:15:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:15:41.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:15:41 smithi083 ceph-mon[77242]: pgmap v14021: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:41.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:15:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:15:41.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:15:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:15:41.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:15:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:15:41.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:15:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:15:42.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:15:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:15:41.907Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:15:42.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:15:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:15:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:15:43.302 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:15:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:15:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:15:43.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:15:43 smithi153 ceph-mon[70044]: pgmap v14022: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:43.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:15:43 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:15:43.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:15:43 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:15:43.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:15:43 smithi083 ceph-mon[79930]: pgmap v14022: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:43.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:15:43 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:15:43.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:15:43 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:15:43.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:15:43 smithi083 ceph-mon[77242]: pgmap v14022: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:43.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:15:43 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:15:43.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:15:43 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:15:44.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:15:44 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:15:44.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:15:44 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:15:44.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:15:44 smithi083 ceph-mon[79930]: pgmap v14023: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:44.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:15:44 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:15:44.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:15:44 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:15:44.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:15:44 smithi083 ceph-mon[77242]: pgmap v14023: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:45.298 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:15:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:15:45.268Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:15:45.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:15:44 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:15:45.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:15:44 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:15:45.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:15:44 smithi153 ceph-mon[70044]: pgmap v14023: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:46.427 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:15:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:15:45.996Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:15:47.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:15:46 smithi083 ceph-mon[79930]: pgmap v14024: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:47.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:15:46 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:15:47.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:15:46 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:15:47.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:15:46 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:15:47.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:15:46 smithi083 ceph-mon[77242]: pgmap v14024: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:47.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:15:46 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:15:47.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:15:46 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:15:47.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:15:46 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:15:47.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:15:46 smithi153 ceph-mon[70044]: pgmap v14024: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:47.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:15:46 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:15:47.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:15:46 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:15:47.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:15:46 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:15:48.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:15:47 smithi083 ceph-mon[77242]: Detected new or changed devices on smithi083 2022-09-29T13:15:48.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:15:47 smithi083 ceph-mon[79930]: Detected new or changed devices on smithi083 2022-09-29T13:15:48.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:15:47 smithi153 ceph-mon[70044]: Detected new or changed devices on smithi083 2022-09-29T13:15:49.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:15:48 smithi083 ceph-mon[77242]: pgmap v14025: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:49.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:15:48 smithi083 ceph-mon[79930]: pgmap v14025: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:49.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:15:48 smithi153 ceph-mon[70044]: pgmap v14025: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:51.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:15:51 smithi153 ceph-mon[70044]: pgmap v14026: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:51.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:15:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:15:51] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T13:15:51.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:15:51 smithi083 ceph-mon[77242]: pgmap v14026: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:51.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:15:51 smithi083 ceph-mon[79930]: pgmap v14026: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:52.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:15:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:15:51.908Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:15:52.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:15:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:15:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:15:53.294 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:15:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:15:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:15:53.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:15:53 smithi153 ceph-mon[70044]: pgmap v14027: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:53.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:15:53 smithi083 ceph-mon[79930]: pgmap v14027: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:53.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:15:53 smithi083 ceph-mon[77242]: pgmap v14027: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:55.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:15:55 smithi153 ceph-mon[70044]: pgmap v14028: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:55.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:15:55 smithi083 ceph-mon[79930]: pgmap v14028: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:55.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:15:55 smithi083 ceph-mon[77242]: pgmap v14028: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:55.714 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:15:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:15:55.269Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:15:56.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:15:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:15:55.996Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:15:57.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:15:56 smithi083 ceph-mon[77242]: pgmap v14029: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:57.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:15:56 smithi083 ceph-mon[79930]: pgmap v14029: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:57.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:15:56 smithi153 ceph-mon[70044]: pgmap v14029: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:59.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:15:59 smithi153 ceph-mon[70044]: pgmap v14030: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:59.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:15:59 smithi083 ceph-mon[79930]: pgmap v14030: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:15:59.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:15:59 smithi083 ceph-mon[77242]: pgmap v14030: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:01.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:16:01 smithi153 ceph-mon[70044]: pgmap v14031: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:01.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:16:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:16:01] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T13:16:01.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:16:01 smithi083 ceph-mon[79930]: pgmap v14031: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:01.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:16:01 smithi083 ceph-mon[77242]: pgmap v14031: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:02.190 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:16:02.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:16:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:16:01.910Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:16:02.480 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:16:02.480 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 19s ago 7h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:16:02.480 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 8m ago 7h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:16:02.481 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 19s ago 7h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:16:02.481 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 8m ago 7h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:16:02.481 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 8m ago 7h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:16:02.481 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 19s ago 7h 903M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:16:02.481 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 19s ago 7h 1736M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:16:02.482 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 8m ago 7h 736M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:16:02.482 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 19s ago 7h 1031M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:16:02.482 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 19s ago 7h 23.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:16:02.482 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 8m ago 7h 25.7M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:16:02.483 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 19s ago 7h 1256M 2330M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:16:02.483 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 19s ago 7h 1244M 2330M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:16:02.483 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 19s ago 7h 945M 2330M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:16:02.483 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 19s ago 7h 1217M 2330M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:16:02.484 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 8m ago 7h 1047M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:16:02.484 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 8m ago 7h 1311M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:16:02.484 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 8m ago 7h 1172M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:16:02.484 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 8m ago 7h 1391M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:16:02.484 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 8m ago 7h 174M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:16:02.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:16:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:16:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:16:02.836 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:16:02.836 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:16:02.836 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:16:02.836 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:16:02.837 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:16:02.837 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:16:02.837 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:16:02.837 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:16:02.838 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:16:02.838 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:16:02.838 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:16:02.838 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:16:02.839 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:16:02.839 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:16:02.839 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:16:02.839 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:16:02.839 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:16:02.840 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:16:03.300 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:16:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:16:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:16:03.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:16:03 smithi153 ceph-mon[70044]: from='client.60379 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:16:03.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:16:03 smithi153 ceph-mon[70044]: pgmap v14032: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:03.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:16:03 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3703634654' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:16:03.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:16:03 smithi083 ceph-mon[77242]: from='client.60379 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:16:03.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:16:03 smithi083 ceph-mon[77242]: pgmap v14032: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:03.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:16:03 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3703634654' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:16:03.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:16:03 smithi083 ceph-mon[79930]: from='client.60379 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:16:03.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:16:03 smithi083 ceph-mon[79930]: pgmap v14032: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:03.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:16:03 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3703634654' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:16:04.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:16:04 smithi153 ceph-mon[70044]: from='client.50454 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:16:04.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:16:04 smithi083 ceph-mon[79930]: from='client.50454 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:16:04.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:16:04 smithi083 ceph-mon[77242]: from='client.50454 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:16:05.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:16:05 smithi153 ceph-mon[70044]: pgmap v14033: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:05.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:16:05 smithi083 ceph-mon[79930]: pgmap v14033: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:05.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:16:05 smithi083 ceph-mon[77242]: pgmap v14033: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:05.714 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:16:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:16:05.270Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:16:06.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:16:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:16:05.997Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:16:07.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:16:06 smithi083 ceph-mon[79930]: pgmap v14034: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:07.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:16:06 smithi083 ceph-mon[77242]: pgmap v14034: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:07.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:16:06 smithi153 ceph-mon[70044]: pgmap v14034: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:09.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:16:09 smithi153 ceph-mon[70044]: pgmap v14035: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:09.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:16:09 smithi083 ceph-mon[77242]: pgmap v14035: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:09.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:16:09 smithi083 ceph-mon[79930]: pgmap v14035: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:11.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:16:11 smithi153 ceph-mon[70044]: pgmap v14036: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:11.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:16:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:16:11] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T13:16:11.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:16:11 smithi083 ceph-mon[79930]: pgmap v14036: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:11.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:16:11 smithi083 ceph-mon[77242]: pgmap v14036: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:12.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:16:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:16:11.908Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:16:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:16:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:16:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:16:13.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:16:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:16:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:16:13.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:16:13 smithi153 ceph-mon[70044]: pgmap v14037: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:13.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:16:13 smithi083 ceph-mon[79930]: pgmap v14037: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:13.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:16:13 smithi083 ceph-mon[77242]: pgmap v14037: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:15.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:16:15 smithi083 ceph-mon[79930]: pgmap v14038: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:15.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:16:15 smithi083 ceph-mon[77242]: pgmap v14038: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:15.714 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:16:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:16:15.270Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:16:15.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:16:15 smithi153 ceph-mon[70044]: pgmap v14038: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:16.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:16:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:16:15.997Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:16:17.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:16:16 smithi083 ceph-mon[79930]: pgmap v14039: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:17.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:16:16 smithi083 ceph-mon[77242]: pgmap v14039: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:17.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:16:16 smithi153 ceph-mon[70044]: pgmap v14039: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:19.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:16:19 smithi153 ceph-mon[70044]: pgmap v14040: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:19.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:16:19 smithi083 ceph-mon[79930]: pgmap v14040: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:19.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:16:19 smithi083 ceph-mon[77242]: pgmap v14040: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:21.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:16:21 smithi153 ceph-mon[70044]: pgmap v14041: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:21.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:16:21 smithi083 ceph-mon[77242]: pgmap v14041: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:21.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:16:21 smithi083 ceph-mon[79930]: pgmap v14041: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:21.714 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:16:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:16:21] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T13:16:22.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:16:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:16:21.910Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:16:22.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:16:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:16:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:16:23.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:16:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:16:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:16:23.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:16:23 smithi153 ceph-mon[70044]: pgmap v14042: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:23.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:16:23 smithi083 ceph-mon[79930]: pgmap v14042: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:23.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:16:23 smithi083 ceph-mon[77242]: pgmap v14042: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:25.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:16:25 smithi083 ceph-mon[79930]: pgmap v14043: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:25.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:16:25 smithi083 ceph-mon[77242]: pgmap v14043: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:25.714 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:16:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:16:25.270Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:16:25.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:16:25 smithi153 ceph-mon[70044]: pgmap v14043: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:26.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:16:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:16:25.998Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:16:27.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:16:26 smithi083 ceph-mon[79930]: pgmap v14044: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:27.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:16:26 smithi083 ceph-mon[77242]: pgmap v14044: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:27.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:16:26 smithi153 ceph-mon[70044]: pgmap v14044: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:29.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:16:29 smithi153 ceph-mon[70044]: pgmap v14045: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:29.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:16:29 smithi083 ceph-mon[79930]: pgmap v14045: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:29.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:16:29 smithi083 ceph-mon[77242]: pgmap v14045: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:31.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:16:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:16:31] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T13:16:31.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:16:31 smithi083 ceph-mon[79930]: pgmap v14046: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:31.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:16:31 smithi083 ceph-mon[77242]: pgmap v14046: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:31.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:16:31 smithi153 ceph-mon[70044]: pgmap v14046: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:32.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:16:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:16:31.910Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:16:32.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:16:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:16:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:16:33.161 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:16:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:16:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:16:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:16:33.458 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:16:33.459 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 50s ago 7h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:16:33.459 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 9m ago 7h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:16:33.459 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 50s ago 7h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:16:33.459 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 9m ago 7h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:16:33.460 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 9m ago 7h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:16:33.460 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 50s ago 7h 903M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:16:33.460 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 50s ago 7h 1736M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:16:33.460 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 9m ago 7h 736M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:16:33.461 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 50s ago 7h 1031M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:16:33.461 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 50s ago 7h 23.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:16:33.461 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 9m ago 7h 25.7M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:16:33.461 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 50s ago 7h 1256M 2330M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:16:33.461 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 50s ago 7h 1244M 2330M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:16:33.462 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 50s ago 7h 945M 2330M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:16:33.462 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 50s ago 7h 1217M 2330M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:16:33.462 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 9m ago 7h 1047M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:16:33.462 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 9m ago 7h 1311M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:16:33.463 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 9m ago 7h 1172M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:16:33.463 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 9m ago 7h 1391M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:16:33.463 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 9m ago 7h 174M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:16:33.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:16:33 smithi083 ceph-mon[79930]: pgmap v14047: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:33.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:16:33 smithi083 ceph-mon[77242]: pgmap v14047: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:33.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:16:33 smithi153 ceph-mon[70044]: pgmap v14047: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:33.817 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:16:33.817 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:16:33.818 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:16:33.818 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:16:33.818 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:16:33.818 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:16:33.818 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:16:33.819 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:16:33.819 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:16:33.819 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:16:33.819 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:16:33.820 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:16:33.820 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:16:33.820 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:16:33.820 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:16:33.821 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:16:33.821 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:16:33.821 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:16:34.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:16:34 smithi083 ceph-mon[79930]: from='client.60397 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:16:34.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:16:34 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4243444479' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:16:34.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:16:34 smithi083 ceph-mon[77242]: from='client.60397 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:16:34.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:16:34 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4243444479' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:16:34.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:16:34 smithi153 ceph-mon[70044]: from='client.60397 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:16:34.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:16:34 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4243444479' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:16:35.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:16:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:16:35.271Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:16:35.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:16:35 smithi083 ceph-mon[77242]: from='client.60403 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:16:35.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:16:35 smithi083 ceph-mon[77242]: pgmap v14048: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:35.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:16:35 smithi083 ceph-mon[79930]: from='client.60403 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:16:35.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:16:35 smithi083 ceph-mon[79930]: pgmap v14048: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:35.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:16:35 smithi153 ceph-mon[70044]: from='client.60403 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:16:35.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:16:35 smithi153 ceph-mon[70044]: pgmap v14048: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:36.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:16:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:16:35.999Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:16:37.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:16:36 smithi083 ceph-mon[79930]: pgmap v14049: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:37.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:16:36 smithi083 ceph-mon[77242]: pgmap v14049: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:37.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:16:36 smithi153 ceph-mon[70044]: pgmap v14049: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:39.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:16:39 smithi153 ceph-mon[70044]: pgmap v14050: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:39.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:16:39 smithi083 ceph-mon[79930]: pgmap v14050: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:39.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:16:39 smithi083 ceph-mon[77242]: pgmap v14050: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:41.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:16:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:16:41] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T13:16:41.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:16:41 smithi083 ceph-mon[79930]: pgmap v14051: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:41.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:16:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:16:41.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:16:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:16:41.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:16:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:16:41.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:16:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:16:41.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:16:41 smithi083 ceph-mon[77242]: pgmap v14051: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:41.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:16:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:16:41.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:16:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:16:41.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:16:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:16:41.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:16:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:16:41.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:16:41 smithi153 ceph-mon[70044]: pgmap v14051: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:41.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:16:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:16:41.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:16:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:16:41.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:16:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:16:41.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:16:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:16:42.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:16:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:16:41.910Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:16:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:16:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:16:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:16:43.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:16:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:16:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:16:43.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:16:43 smithi083 ceph-mon[79930]: pgmap v14052: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:43.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:16:43 smithi083 ceph-mon[77242]: pgmap v14052: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:43.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:16:43 smithi153 ceph-mon[70044]: pgmap v14052: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:45.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:16:45 smithi083 ceph-mon[79930]: pgmap v14053: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:45.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:16:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:16:45.272Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:16:45.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:16:45 smithi083 ceph-mon[77242]: pgmap v14053: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:45.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:16:45 smithi153 ceph-mon[70044]: pgmap v14053: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:46.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:16:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:16:45.999Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:16:47.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:16:46 smithi083 ceph-mon[79930]: pgmap v14054: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:47.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:16:46 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:16:47.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:16:46 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:16:47.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:16:46 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:16:47.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:16:46 smithi083 ceph-mon[77242]: pgmap v14054: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:47.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:16:46 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:16:47.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:16:46 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:16:47.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:16:46 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:16:47.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:16:46 smithi153 ceph-mon[70044]: pgmap v14054: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:47.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:16:46 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:16:47.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:16:46 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:16:47.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:16:46 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:16:48.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:16:48 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:16:48.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:16:48 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:16:48.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:16:48 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:16:49.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:16:49 smithi083 ceph-mon[79930]: pgmap v14055: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:49.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:16:49 smithi083 ceph-mon[77242]: pgmap v14055: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:49.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:16:49 smithi153 ceph-mon[70044]: pgmap v14055: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:51.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:16:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:16:51] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T13:16:51.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:16:51 smithi083 ceph-mon[79930]: pgmap v14056: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:51.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:16:51 smithi083 ceph-mon[77242]: pgmap v14056: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:51.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:16:51 smithi153 ceph-mon[70044]: pgmap v14056: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:52.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:16:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:16:51.911Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:16:52.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:16:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:16:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:16:53.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:16:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:16:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:16:53.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:16:53 smithi083 ceph-mon[79930]: pgmap v14057: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:53.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:16:53 smithi083 ceph-mon[77242]: pgmap v14057: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:53.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:16:53 smithi153 ceph-mon[70044]: pgmap v14057: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:55.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:16:55 smithi083 ceph-mon[79930]: pgmap v14058: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:55.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:16:55 smithi083 ceph-mon[77242]: pgmap v14058: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:55.714 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:16:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:16:55.273Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:16:55.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:16:55 smithi153 ceph-mon[70044]: pgmap v14058: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:56.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:16:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:16:56.000Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:16:57.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:16:56 smithi083 ceph-mon[79930]: pgmap v14059: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:57.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:16:56 smithi083 ceph-mon[77242]: pgmap v14059: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:57.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:16:56 smithi153 ceph-mon[70044]: pgmap v14059: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:59.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:16:59 smithi083 ceph-mon[77242]: pgmap v14060: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:59.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:16:59 smithi083 ceph-mon[79930]: pgmap v14060: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:16:59.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:16:59 smithi153 ceph-mon[70044]: pgmap v14060: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:01.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:17:01 smithi083 ceph-mon[77242]: pgmap v14061: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:01.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:17:01 smithi083 ceph-mon[79930]: pgmap v14061: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:01.714 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:17:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:17:01] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T13:17:01.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:17:01 smithi153 ceph-mon[70044]: pgmap v14061: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:02.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:17:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:17:01.911Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:17:02.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:17:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:17:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:17:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:17:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:17:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:17:03.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:17:03 smithi083 ceph-mon[79930]: pgmap v14062: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:03.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:17:03 smithi083 ceph-mon[77242]: pgmap v14062: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:03.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:17:03 smithi153 ceph-mon[70044]: pgmap v14062: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:04.148 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:17:04.440 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:17:04.440 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 81s ago 7h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:17:04.441 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 9m ago 7h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:17:04.441 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 81s ago 7h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:17:04.441 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 9m ago 7h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:17:04.441 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 9m ago 7h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:17:04.442 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 81s ago 7h 903M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:17:04.442 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 81s ago 7h 1736M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:17:04.442 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 9m ago 7h 736M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:17:04.443 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 81s ago 7h 1031M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:17:04.443 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 81s ago 7h 23.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:17:04.443 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 9m ago 7h 25.7M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:17:04.443 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 81s ago 7h 1256M 2330M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:17:04.443 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 81s ago 7h 1244M 2330M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:17:04.444 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 81s ago 7h 945M 2330M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:17:04.444 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 81s ago 7h 1217M 2330M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:17:04.444 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 9m ago 7h 1047M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:17:04.444 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 9m ago 7h 1311M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:17:04.445 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 9m ago 7h 1172M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:17:04.445 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 9m ago 7h 1391M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:17:04.445 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 9m ago 7h 174M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:17:04.793 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:17:04.794 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:17:04.794 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:17:04.794 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:17:04.794 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:17:04.795 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:17:04.795 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:17:04.795 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:17:04.795 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:17:04.795 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:17:04.796 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:17:04.796 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:17:04.796 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:17:04.796 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:17:04.797 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:17:04.797 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:17:04.797 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:17:04.797 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:17:05.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:17:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:17:05.273Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:17:05.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:17:05 smithi083 ceph-mon[77242]: from='client.60415 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:17:05.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:17:05 smithi083 ceph-mon[77242]: pgmap v14063: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:05.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:17:05 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1153553242' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:17:05.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:17:05 smithi083 ceph-mon[79930]: from='client.60415 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:17:05.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:17:05 smithi083 ceph-mon[79930]: pgmap v14063: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:05.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:17:05 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1153553242' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:17:05.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:17:05 smithi153 ceph-mon[70044]: from='client.60415 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:17:05.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:17:05 smithi153 ceph-mon[70044]: pgmap v14063: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:05.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:17:05 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1153553242' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:17:06.323 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:17:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:17:06.001Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:17:06.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:17:06 smithi083 ceph-mon[79930]: from='client.60421 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:17:06.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:17:06 smithi083 ceph-mon[77242]: from='client.60421 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:17:06.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:17:06 smithi153 ceph-mon[70044]: from='client.60421 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:17:07.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:17:07 smithi083 ceph-mon[79930]: pgmap v14064: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:07.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:17:07 smithi083 ceph-mon[77242]: pgmap v14064: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:07.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:17:07 smithi153 ceph-mon[70044]: pgmap v14064: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:09.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:17:09 smithi083 ceph-mon[79930]: pgmap v14065: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:09.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:17:09 smithi083 ceph-mon[77242]: pgmap v14065: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:09.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:17:09 smithi153 ceph-mon[70044]: pgmap v14065: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:11.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:17:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:17:11] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T13:17:11.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:17:11 smithi083 ceph-mon[79930]: pgmap v14066: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:11.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:17:11 smithi083 ceph-mon[77242]: pgmap v14066: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:11.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:17:11 smithi153 ceph-mon[70044]: pgmap v14066: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:12.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:17:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:17:11.912Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:17:12.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:17:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:17:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:17:13.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:17:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:17:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:17:13.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:17:13 smithi083 ceph-mon[79930]: pgmap v14067: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:13.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:17:13 smithi083 ceph-mon[77242]: pgmap v14067: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:13.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:17:13 smithi153 ceph-mon[70044]: pgmap v14067: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:14.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:17:14 smithi083 ceph-mon[79930]: pgmap v14068: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:14.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:17:14 smithi083 ceph-mon[77242]: pgmap v14068: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:14.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:17:14 smithi153 ceph-mon[70044]: pgmap v14068: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:15.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:17:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:17:15.274Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:17:16.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:17:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:17:16.001Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:17:17.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:17:16 smithi083 ceph-mon[77242]: pgmap v14069: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:17.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:17:16 smithi083 ceph-mon[79930]: pgmap v14069: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:17.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:17:16 smithi153 ceph-mon[70044]: pgmap v14069: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:19.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:17:19 smithi083 ceph-mon[79930]: pgmap v14070: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:19.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:17:19 smithi083 ceph-mon[77242]: pgmap v14070: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:19.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:17:19 smithi153 ceph-mon[70044]: pgmap v14070: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:21.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:17:21 smithi083 ceph-mon[79930]: pgmap v14071: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:21.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:17:21 smithi083 ceph-mon[77242]: pgmap v14071: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:21.714 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:17:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:17:21] "GET /metrics HTTP/1.1" 200 225231 "" "Prometheus/2.33.4" 2022-09-29T13:17:21.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:17:21 smithi153 ceph-mon[70044]: pgmap v14071: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:22.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:17:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:17:21.913Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:17:22.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:17:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:17:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:17:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:17:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:17:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:17:23.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:17:23 smithi083 ceph-mon[79930]: pgmap v14072: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:23.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:17:23 smithi083 ceph-mon[77242]: pgmap v14072: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:23.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:17:23 smithi153 ceph-mon[70044]: pgmap v14072: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:25.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:17:25 smithi083 ceph-mon[79930]: pgmap v14073: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:25.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:17:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:17:25.275Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:17:25.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:17:25 smithi083 ceph-mon[77242]: pgmap v14073: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:25.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:17:25 smithi153 ceph-mon[70044]: pgmap v14073: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:26.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:17:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:17:26.001Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:17:27.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:17:26 smithi083 ceph-mon[79930]: pgmap v14074: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:27.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:17:26 smithi083 ceph-mon[77242]: pgmap v14074: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:27.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:17:26 smithi153 ceph-mon[70044]: pgmap v14074: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:29.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:17:29 smithi083 ceph-mon[79930]: pgmap v14075: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:29.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:17:29 smithi083 ceph-mon[77242]: pgmap v14075: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:29.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:17:29 smithi153 ceph-mon[70044]: pgmap v14075: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:31.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:17:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:17:31] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T13:17:31.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:17:31 smithi083 ceph-mon[79930]: pgmap v14076: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:31.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:17:31 smithi083 ceph-mon[77242]: pgmap v14076: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:31.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:17:31 smithi153 ceph-mon[70044]: pgmap v14076: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:32.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:17:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:17:31.913Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:17:32.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:17:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:17:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:17:33.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:17:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:17:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:17:33.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:17:33 smithi083 ceph-mon[79930]: pgmap v14077: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:33.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:17:33 smithi083 ceph-mon[77242]: pgmap v14077: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:33.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:17:33 smithi153 ceph-mon[70044]: pgmap v14077: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:35.119 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:17:35.410 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:17:35.411 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 112s ago 7h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:17:35.411 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 10m ago 7h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:17:35.411 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 112s ago 7h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:17:35.412 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 10m ago 7h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:17:35.412 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 10m ago 7h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:17:35.412 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 112s ago 7h 903M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:17:35.412 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 112s ago 7h 1736M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:17:35.412 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 10m ago 7h 736M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:17:35.413 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 112s ago 7h 1031M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:17:35.413 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 112s ago 7h 23.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:17:35.413 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 10m ago 7h 25.7M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:17:35.413 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 112s ago 7h 1256M 2330M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:17:35.414 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 112s ago 7h 1244M 2330M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:17:35.414 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 112s ago 7h 945M 2330M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:17:35.414 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 112s ago 7h 1217M 2330M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:17:35.414 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 10m ago 7h 1047M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:17:35.414 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 10m ago 7h 1311M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:17:35.415 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 10m ago 7h 1172M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:17:35.415 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 10m ago 7h 1391M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:17:35.415 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 10m ago 7h 174M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:17:35.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:17:35 smithi083 ceph-mon[79930]: pgmap v14078: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:35.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:17:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:17:35.274Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:17:35.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:17:35 smithi083 ceph-mon[77242]: pgmap v14078: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:35.771 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:17:35.772 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:17:35.772 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:17:35.772 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:17:35.772 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:17:35.773 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:17:35.773 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:17:35.773 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:17:35.774 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:17:35.774 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:17:35.774 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:17:35.774 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:17:35.775 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:17:35.775 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:17:35.775 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:17:35.775 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:17:35.776 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:17:35.776 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:17:35.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:17:35 smithi153 ceph-mon[70044]: pgmap v14078: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:36.335 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:17:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:17:36.001Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:17:36.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:17:36 smithi083 ceph-mon[79930]: from='client.50502 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:17:36.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:17:36 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3634846966' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:17:36.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:17:36 smithi083 ceph-mon[77242]: from='client.50502 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:17:36.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:17:36 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3634846966' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:17:36.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:17:36 smithi153 ceph-mon[70044]: from='client.50502 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:17:36.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:17:36 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3634846966' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:17:37.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:17:37 smithi083 ceph-mon[79930]: from='client.60439 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:17:37.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:17:37 smithi083 ceph-mon[79930]: pgmap v14079: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:37.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:17:37 smithi083 ceph-mon[77242]: from='client.60439 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:17:37.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:17:37 smithi083 ceph-mon[77242]: pgmap v14079: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:37.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:17:37 smithi153 ceph-mon[70044]: from='client.60439 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:17:37.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:17:37 smithi153 ceph-mon[70044]: pgmap v14079: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:39.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:17:39 smithi083 ceph-mon[79930]: pgmap v14080: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:39.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:17:39 smithi083 ceph-mon[77242]: pgmap v14080: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:39.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:17:39 smithi153 ceph-mon[70044]: pgmap v14080: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:17:40 smithi083 ceph-mon[77242]: pgmap v14081: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:17:40 smithi083 ceph-mon[79930]: pgmap v14081: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:40.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:17:40 smithi153 ceph-mon[70044]: pgmap v14081: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:41.883 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:17:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:17:41] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T13:17:42.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:17:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:17:41.915Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:17:42.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:17:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:17:42.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:17:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:17:42.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:17:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:17:42.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:17:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:17:42.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:17:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:17:42.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:17:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:17:42.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:17:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:17:42.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:17:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:17:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:17:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:17:42.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:17:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:17:42.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:17:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:17:42.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:17:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:17:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:17:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:17:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:17:43.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:17:42 smithi083 ceph-mon[77242]: pgmap v14082: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:43.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:17:42 smithi083 ceph-mon[79930]: pgmap v14082: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:43.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:17:42 smithi153 ceph-mon[70044]: pgmap v14082: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:43.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:17:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:17:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:17:45.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:17:45 smithi083 ceph-mon[79930]: pgmap v14083: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:45.714 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:17:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:17:45.276Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:17:45.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:17:45 smithi083 ceph-mon[77242]: pgmap v14083: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:45.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:17:45 smithi153 ceph-mon[70044]: pgmap v14083: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:46.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:17:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:17:46.002Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:17:47.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:17:46 smithi083 ceph-mon[79930]: pgmap v14084: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:47.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:17:46 smithi083 ceph-mon[77242]: pgmap v14084: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:47.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:17:46 smithi153 ceph-mon[70044]: pgmap v14084: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:48.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:17:47 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:17:48.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:17:47 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:17:48.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:17:47 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:17:48.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:17:47 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:17:48.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:17:47 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:17:48.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:17:47 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:17:48.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:17:47 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:17:48.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:17:47 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:17:48.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:17:47 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:17:49.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:17:48 smithi083 ceph-mon[79930]: pgmap v14085: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:49.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:17:48 smithi083 ceph-mon[77242]: pgmap v14085: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:49.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:17:48 smithi153 ceph-mon[70044]: pgmap v14085: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:51.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:17:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:17:51] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T13:17:51.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:17:51 smithi083 ceph-mon[79930]: pgmap v14086: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:51.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:17:51 smithi083 ceph-mon[77242]: pgmap v14086: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:51.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:17:51 smithi153 ceph-mon[70044]: pgmap v14086: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:52.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:17:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:17:51.915Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:17:52.661 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:17:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:17:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:17:52.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:17:52 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:17:52.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:17:52 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:17:52.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:17:52 smithi083 ceph-mon[77242]: pgmap v14087: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:52.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:17:52 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:17:52.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:17:52 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:17:52.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:17:52 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:17:52.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:17:52 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:17:52.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:17:52 smithi083 ceph-mon[79930]: pgmap v14087: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:52.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:17:52 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:17:52.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:17:52 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:17:53.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:17:52 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:17:53.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:17:52 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:17:53.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:17:52 smithi153 ceph-mon[70044]: pgmap v14087: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:53.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:17:52 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:17:53.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:17:52 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:17:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:17:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:17:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:17:55.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:17:55 smithi153 ceph-mon[70044]: pgmap v14088: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:55.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:17:55 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:17:55.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:17:55 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:17:55.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:17:55 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:17:55.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:17:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:17:55.275Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:17:55.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:17:55 smithi083 ceph-mon[79930]: pgmap v14088: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:55.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:17:55 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:17:55.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:17:55 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:17:55.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:17:55 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:17:55.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:17:55 smithi083 ceph-mon[77242]: pgmap v14088: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:55.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:17:55 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:17:55.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:17:55 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:17:55.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:17:55 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:17:56.331 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:17:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:17:56.003Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:17:56.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:17:56 smithi083 ceph-mon[79930]: Detected new or changed devices on smithi153 2022-09-29T13:17:56.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:17:56 smithi083 ceph-mon[77242]: Detected new or changed devices on smithi153 2022-09-29T13:17:56.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:17:56 smithi153 ceph-mon[70044]: Detected new or changed devices on smithi153 2022-09-29T13:17:57.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:17:57 smithi083 ceph-mon[79930]: pgmap v14089: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:57.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:17:57 smithi083 ceph-mon[77242]: pgmap v14089: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:57.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:17:57 smithi153 ceph-mon[70044]: pgmap v14089: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:59.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:17:59 smithi083 ceph-mon[79930]: pgmap v14090: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:59.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:17:59 smithi083 ceph-mon[77242]: pgmap v14090: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:17:59.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:17:59 smithi153 ceph-mon[70044]: pgmap v14090: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:01.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:18:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:18:01] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T13:18:01.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:18:01 smithi083 ceph-mon[79930]: pgmap v14091: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:01.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:18:01 smithi083 ceph-mon[77242]: pgmap v14091: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:01.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:18:01 smithi153 ceph-mon[70044]: pgmap v14091: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:02.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:18:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:18:01.915Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:18:02.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:18:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:18:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:18:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:18:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:18:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:18:03.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:18:03 smithi083 ceph-mon[79930]: pgmap v14092: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:03.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:18:03 smithi083 ceph-mon[77242]: pgmap v14092: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:03.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:18:03 smithi153 ceph-mon[70044]: pgmap v14092: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:05.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:18:05 smithi083 ceph-mon[79930]: pgmap v14093: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:05.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:18:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:18:05.277Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:18:05.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:18:05 smithi083 ceph-mon[77242]: pgmap v14093: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:05.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:18:05 smithi153 ceph-mon[70044]: pgmap v14093: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:06.098 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:18:06.390 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:18:06.390 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 2m ago 7h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:18:06.391 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 14s ago 7h 63.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:18:06.391 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 2m ago 7h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:18:06.391 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 14s ago 7h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:18:06.391 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 14s ago 7h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:18:06.392 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 2m ago 7h 903M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:18:06.392 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 2m ago 7h 1736M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:18:06.392 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 14s ago 7h 752M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:18:06.392 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 2m ago 7h 1031M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:18:06.393 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 2m ago 7h 23.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:18:06.393 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 14s ago 7h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:18:06.393 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 2m ago 7h 1256M 2330M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:18:06.393 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 2m ago 7h 1244M 2330M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:18:06.393 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 2m ago 7h 945M 2330M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:18:06.394 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 2m ago 7h 1217M 2330M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:18:06.394 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 14s ago 7h 1069M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:18:06.394 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 14s ago 7h 1339M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:18:06.394 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 14s ago 7h 1197M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:18:06.395 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 14s ago 7h 1421M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:18:06.395 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 14s ago 7h 176M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:18:06.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:18:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:18:06.003Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:18:06.745 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:18:06.746 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:18:06.746 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:18:06.746 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:18:06.746 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:18:06.747 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:18:06.747 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:18:06.747 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:18:06.747 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:18:06.748 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:18:06.748 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:18:06.748 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:18:06.748 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:18:06.749 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:18:06.749 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:18:06.749 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:18:06.749 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:18:06.750 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:18:07.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:18:06 smithi083 ceph-mon[77242]: from='client.60451 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:18:07.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:18:06 smithi083 ceph-mon[77242]: pgmap v14094: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:07.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:18:06 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1350528226' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:18:07.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:18:06 smithi083 ceph-mon[79930]: from='client.60451 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:18:07.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:18:06 smithi083 ceph-mon[79930]: pgmap v14094: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:07.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:18:06 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1350528226' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:18:07.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:18:06 smithi153 ceph-mon[70044]: from='client.60451 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:18:07.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:18:06 smithi153 ceph-mon[70044]: pgmap v14094: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:07.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:18:06 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1350528226' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:18:08.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:18:07 smithi083 ceph-mon[79930]: from='client.50526 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:18:08.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:18:07 smithi083 ceph-mon[77242]: from='client.50526 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:18:08.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:18:07 smithi153 ceph-mon[70044]: from='client.50526 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:18:09.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:18:08 smithi083 ceph-mon[79930]: pgmap v14095: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:09.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:18:08 smithi083 ceph-mon[77242]: pgmap v14095: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:09.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:18:08 smithi153 ceph-mon[70044]: pgmap v14095: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:11.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:18:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:18:11] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T13:18:11.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:18:11 smithi083 ceph-mon[79930]: pgmap v14096: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:11.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:18:11 smithi083 ceph-mon[77242]: pgmap v14096: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:11.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:18:11 smithi153 ceph-mon[70044]: pgmap v14096: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:12.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:18:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:18:11.916Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:18:12.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:18:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:18:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:18:13.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:18:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:18:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:18:13.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:18:13 smithi083 ceph-mon[79930]: pgmap v14097: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:13.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:18:13 smithi083 ceph-mon[77242]: pgmap v14097: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:13.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:18:13 smithi153 ceph-mon[70044]: pgmap v14097: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:15.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:18:15 smithi083 ceph-mon[79930]: pgmap v14098: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:15.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:18:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:18:15.277Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:18:15.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:18:15 smithi083 ceph-mon[77242]: pgmap v14098: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:15.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:18:15 smithi153 ceph-mon[70044]: pgmap v14098: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:16.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:18:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:18:16.004Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:18:17.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:18:16 smithi083 ceph-mon[79930]: pgmap v14099: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:17.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:18:16 smithi083 ceph-mon[77242]: pgmap v14099: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:17.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:18:16 smithi153 ceph-mon[70044]: pgmap v14099: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:19.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:18:19 smithi083 ceph-mon[79930]: pgmap v14100: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:19.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:18:19 smithi083 ceph-mon[77242]: pgmap v14100: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:19.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:18:19 smithi153 ceph-mon[70044]: pgmap v14100: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:21.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:18:21 smithi083 ceph-mon[77242]: pgmap v14101: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:21.714 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:18:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:18:21] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T13:18:21.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:18:21 smithi083 ceph-mon[79930]: pgmap v14101: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:21.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:18:21 smithi153 ceph-mon[70044]: pgmap v14101: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:22.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:18:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:18:21.916Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:18:22.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:18:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:18:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:18:23.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:18:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:18:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:18:23.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:18:23 smithi083 ceph-mon[79930]: pgmap v14102: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:23.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:18:23 smithi083 ceph-mon[77242]: pgmap v14102: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:23.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:18:23 smithi153 ceph-mon[70044]: pgmap v14102: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:25.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:18:25 smithi083 ceph-mon[79930]: pgmap v14103: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:25.714 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:18:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:18:25.278Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:18:25.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:18:25 smithi083 ceph-mon[77242]: pgmap v14103: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:25.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:18:25 smithi153 ceph-mon[70044]: pgmap v14103: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:26.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:18:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:18:26.006Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:18:27.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:18:26 smithi083 ceph-mon[79930]: pgmap v14104: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:27.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:18:26 smithi083 ceph-mon[77242]: pgmap v14104: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:27.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:18:26 smithi153 ceph-mon[70044]: pgmap v14104: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:29.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:18:29 smithi083 ceph-mon[79930]: pgmap v14105: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:29.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:18:29 smithi083 ceph-mon[77242]: pgmap v14105: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:29.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:18:29 smithi153 ceph-mon[70044]: pgmap v14105: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:31.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:18:31 smithi083 ceph-mon[77242]: pgmap v14106: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:31.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:18:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:18:31] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T13:18:31.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:18:31 smithi083 ceph-mon[79930]: pgmap v14106: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:31.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:18:31 smithi153 ceph-mon[70044]: pgmap v14106: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:32.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:18:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:18:31.917Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:18:32.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:18:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:18:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:18:33.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:18:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:18:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:18:33.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:18:33 smithi083 ceph-mon[79930]: pgmap v14107: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:33.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:18:33 smithi083 ceph-mon[77242]: pgmap v14107: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:33.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:18:33 smithi153 ceph-mon[70044]: pgmap v14107: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:35.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:18:35 smithi083 ceph-mon[77242]: pgmap v14108: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:35.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:18:35 smithi083 ceph-mon[79930]: pgmap v14108: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:35.714 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:18:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:18:35.279Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:18:35.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:18:35 smithi153 ceph-mon[70044]: pgmap v14108: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:36.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:18:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:18:36.006Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:18:37.071 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:18:37.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:18:36 smithi083 ceph-mon[77242]: pgmap v14109: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:37.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:18:36 smithi083 ceph-mon[79930]: pgmap v14109: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:37.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:18:36 smithi153 ceph-mon[70044]: pgmap v14109: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:37.360 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:18:37.360 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 2m ago 7h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:18:37.361 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 45s ago 7h 63.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:18:37.361 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 2m ago 7h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:18:37.361 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 45s ago 7h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:18:37.361 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 45s ago 7h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:18:37.362 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 2m ago 7h 903M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:18:37.362 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 2m ago 7h 1736M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:18:37.362 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 45s ago 7h 752M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:18:37.362 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 2m ago 7h 1031M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:18:37.363 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 2m ago 7h 23.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:18:37.363 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 45s ago 7h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:18:37.363 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 2m ago 7h 1256M 2330M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:18:37.363 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 2m ago 7h 1244M 2330M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:18:37.364 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 2m ago 7h 945M 2330M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:18:37.364 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 2m ago 7h 1217M 2330M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:18:37.364 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 45s ago 7h 1069M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:18:37.364 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 45s ago 7h 1339M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:18:37.364 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 45s ago 7h 1197M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:18:37.365 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 45s ago 7h 1421M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:18:37.365 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 45s ago 7h 176M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:18:37.716 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:18:37.717 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:18:37.717 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:18:37.717 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:18:37.718 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:18:37.718 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:18:37.718 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:18:37.718 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:18:37.718 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:18:37.719 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:18:37.719 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:18:37.719 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:18:37.719 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:18:37.720 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:18:37.720 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:18:37.720 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:18:37.720 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:18:37.721 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:18:38.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:18:37 smithi083 ceph-mon[79930]: from='client.60469 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:18:38.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:18:37 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1851642979' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:18:38.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:18:37 smithi083 ceph-mon[77242]: from='client.60469 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:18:38.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:18:37 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1851642979' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:18:38.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:18:37 smithi153 ceph-mon[70044]: from='client.60469 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:18:38.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:18:37 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1851642979' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:18:39.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:18:38 smithi083 ceph-mon[79930]: from='client.50544 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:18:39.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:18:38 smithi083 ceph-mon[79930]: pgmap v14110: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:39.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:18:38 smithi083 ceph-mon[77242]: from='client.50544 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:18:39.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:18:38 smithi083 ceph-mon[77242]: pgmap v14110: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:39.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:18:38 smithi153 ceph-mon[70044]: from='client.50544 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:18:39.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:18:38 smithi153 ceph-mon[70044]: pgmap v14110: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:41.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:18:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:18:41] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T13:18:41.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:18:41 smithi083 ceph-mon[77242]: pgmap v14111: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:41.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:18:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:18:41.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:18:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:18:41.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:18:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:18:41.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:18:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:18:41.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:18:41 smithi083 ceph-mon[79930]: pgmap v14111: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:41.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:18:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:18:41.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:18:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:18:41.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:18:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:18:41.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:18:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:18:41.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:18:41 smithi153 ceph-mon[70044]: pgmap v14111: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:41.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:18:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:18:41.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:18:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:18:41.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:18:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:18:41.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:18:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:18:42.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:18:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:18:41.918Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:18:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:18:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:18:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:18:43.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:18:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:18:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:18:43.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:18:43 smithi083 ceph-mon[79930]: pgmap v14112: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:43.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:18:43 smithi083 ceph-mon[77242]: pgmap v14112: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:43.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:18:43 smithi153 ceph-mon[70044]: pgmap v14112: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:45.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:18:45 smithi083 ceph-mon[77242]: pgmap v14113: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:45.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:18:45 smithi083 ceph-mon[79930]: pgmap v14113: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:45.714 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:18:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:18:45.280Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:18:45.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:18:45 smithi153 ceph-mon[70044]: pgmap v14113: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:46.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:18:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:18:46.006Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:18:47.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:18:46 smithi083 ceph-mon[79930]: pgmap v14114: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:47.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:18:46 smithi083 ceph-mon[77242]: pgmap v14114: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:47.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:18:46 smithi153 ceph-mon[70044]: pgmap v14114: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:49.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:18:49 smithi083 ceph-mon[77242]: pgmap v14115: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:49.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:18:49 smithi083 ceph-mon[79930]: pgmap v14115: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:49.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:18:49 smithi153 ceph-mon[70044]: pgmap v14115: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:51.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:18:51 smithi083 ceph-mon[77242]: pgmap v14116: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:51.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:18:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:18:51] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T13:18:51.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:18:51 smithi083 ceph-mon[79930]: pgmap v14116: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:51.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:18:51 smithi153 ceph-mon[70044]: pgmap v14116: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:52.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:18:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:18:51.918Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:18:52.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:18:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:18:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:18:53.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:18:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:18:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:18:53.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:18:53 smithi083 ceph-mon[77242]: pgmap v14117: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:53.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:18:53 smithi083 ceph-mon[79930]: pgmap v14117: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:53.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:18:53 smithi153 ceph-mon[70044]: pgmap v14117: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:54.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:18:54 smithi083 ceph-mon[77242]: pgmap v14118: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:54.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:18:54 smithi083 ceph-mon[79930]: pgmap v14118: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:54.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:18:54 smithi153 ceph-mon[70044]: pgmap v14118: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:55.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:18:55 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:18:55.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:18:55 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:18:55.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:18:55 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:18:55.714 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:18:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:18:55.280Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:18:55.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:18:55 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:18:55.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:18:55 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:18:55.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:18:55 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:18:55.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:18:55 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:18:55.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:18:55 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:18:55.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:18:55 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:18:56.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:18:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:18:56.007Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:18:57.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:18:56 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:18:57.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:18:56 smithi083 ceph-mon[77242]: pgmap v14119: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:57.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:18:56 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:18:57.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:18:56 smithi083 ceph-mon[79930]: pgmap v14119: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:57.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:18:56 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:18:57.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:18:56 smithi153 ceph-mon[70044]: pgmap v14119: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:59.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:18:59 smithi083 ceph-mon[79930]: pgmap v14120: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:59.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:18:59 smithi083 ceph-mon[77242]: pgmap v14120: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:18:59.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:18:59 smithi153 ceph-mon[70044]: pgmap v14120: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:01.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:19:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:19:01] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T13:19:01.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:19:01 smithi083 ceph-mon[79930]: pgmap v14121: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:01.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:19:01 smithi083 ceph-mon[77242]: pgmap v14121: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:01.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:19:01 smithi153 ceph-mon[70044]: pgmap v14121: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:02.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:19:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:19:01.919Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:19:02.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:19:02 smithi083 ceph-mon[79930]: pgmap v14122: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:02.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:19:02 smithi083 ceph-mon[77242]: pgmap v14122: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:02.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:19:02 smithi153 ceph-mon[70044]: pgmap v14122: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:02.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:19:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:19:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:19:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:19:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:19:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:19:05.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:19:05 smithi083 ceph-mon[79930]: pgmap v14123: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:05.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:19:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:19:05.281Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:19:05.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:19:05 smithi083 ceph-mon[77242]: pgmap v14123: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:05.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:19:05 smithi153 ceph-mon[70044]: pgmap v14123: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:06.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:19:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:19:06.008Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:19:07.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:19:06 smithi083 ceph-mon[79930]: pgmap v14124: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:07.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:19:06 smithi083 ceph-mon[77242]: pgmap v14124: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:07.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:19:06 smithi153 ceph-mon[70044]: pgmap v14124: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:08.047 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:19:08.337 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:19:08.337 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 3m ago 7h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:19:08.337 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 76s ago 7h 63.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:19:08.337 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 3m ago 7h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:19:08.338 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 76s ago 7h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:19:08.338 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 76s ago 7h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:19:08.338 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 3m ago 7h 903M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:19:08.338 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 3m ago 7h 1736M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:19:08.339 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 76s ago 7h 752M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:19:08.339 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 3m ago 7h 1031M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:19:08.339 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 3m ago 7h 23.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:19:08.339 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 76s ago 7h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:19:08.340 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 3m ago 7h 1256M 2330M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:19:08.340 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 3m ago 7h 1244M 2330M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:19:08.340 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 3m ago 7h 945M 2330M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:19:08.340 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 3m ago 7h 1217M 2330M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:19:08.340 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 76s ago 7h 1069M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:19:08.341 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 76s ago 7h 1339M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:19:08.341 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 76s ago 7h 1197M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:19:08.341 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 76s ago 7h 1421M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:19:08.341 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 76s ago 7h 176M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:19:08.692 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:19:08.692 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:19:08.692 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:19:08.693 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:19:08.693 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:19:08.693 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:19:08.693 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:19:08.694 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:19:08.694 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:19:08.694 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:19:08.694 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:19:08.695 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:19:08.695 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:19:08.695 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:19:08.695 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:19:08.695 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:19:08.696 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:19:08.696 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:19:09.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:19:09 smithi083 ceph-mon[77242]: from='client.50556 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:19:09.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:19:09 smithi083 ceph-mon[77242]: pgmap v14125: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:09.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:19:09 smithi083 ceph-mon[77242]: from='client.50562 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:19:09.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:19:09 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/99081663' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:19:09.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:19:09 smithi083 ceph-mon[79930]: from='client.50556 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:19:09.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:19:09 smithi083 ceph-mon[79930]: pgmap v14125: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:09.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:19:09 smithi083 ceph-mon[79930]: from='client.50562 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:19:09.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:19:09 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/99081663' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:19:09.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:19:09 smithi153 ceph-mon[70044]: from='client.50556 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:19:09.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:19:09 smithi153 ceph-mon[70044]: pgmap v14125: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:09.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:19:09 smithi153 ceph-mon[70044]: from='client.50562 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:19:09.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:19:09 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/99081663' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:19:11.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:19:11 smithi083 ceph-mon[79930]: pgmap v14126: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:11.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:19:11 smithi083 ceph-mon[77242]: pgmap v14126: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:11.714 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:19:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:19:11] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T13:19:11.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:19:11 smithi153 ceph-mon[70044]: pgmap v14126: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:12.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:19:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:19:11.919Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:19:12.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:19:12 smithi083 ceph-mon[77242]: pgmap v14127: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:12.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:19:12 smithi083 ceph-mon[79930]: pgmap v14127: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:12.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:19:12 smithi153 ceph-mon[70044]: pgmap v14127: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:12.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:19:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:19:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:19:13.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:19:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:19:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:19:15.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:19:15 smithi083 ceph-mon[79930]: pgmap v14128: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:15.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:19:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:19:15.282Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:19:15.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:19:15 smithi083 ceph-mon[77242]: pgmap v14128: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:15.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:19:15 smithi153 ceph-mon[70044]: pgmap v14128: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:16.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:19:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:19:16.008Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:19:17.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:19:16 smithi083 ceph-mon[79930]: pgmap v14129: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:17.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:19:16 smithi083 ceph-mon[77242]: pgmap v14129: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:17.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:19:16 smithi153 ceph-mon[70044]: pgmap v14129: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:19.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:19:19 smithi083 ceph-mon[79930]: pgmap v14130: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:19.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:19:19 smithi083 ceph-mon[77242]: pgmap v14130: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:19.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:19:19 smithi153 ceph-mon[70044]: pgmap v14130: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:20.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:19:20 smithi083 ceph-mon[79930]: pgmap v14131: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:20.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:19:20 smithi083 ceph-mon[77242]: pgmap v14131: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:20.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:19:20 smithi153 ceph-mon[70044]: pgmap v14131: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:21.922 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:19:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:19:21] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T13:19:22.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:19:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:19:21.920Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:19:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:19:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:19:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:19:23.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:19:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:19:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:19:23.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:19:23 smithi083 ceph-mon[79930]: pgmap v14132: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:23.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:19:23 smithi083 ceph-mon[77242]: pgmap v14132: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:23.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:19:23 smithi153 ceph-mon[70044]: pgmap v14132: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:24.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:19:24 smithi083 ceph-mon[79930]: pgmap v14133: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:24.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:19:24 smithi083 ceph-mon[77242]: pgmap v14133: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:24.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:19:24 smithi153 ceph-mon[70044]: pgmap v14133: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:25.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:19:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:19:25.283Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:19:26.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:19:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:19:26.009Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:19:27.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:19:26 smithi083 ceph-mon[79930]: pgmap v14134: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:27.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:19:26 smithi083 ceph-mon[77242]: pgmap v14134: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:27.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:19:26 smithi153 ceph-mon[70044]: pgmap v14134: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:29.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:19:29 smithi083 ceph-mon[79930]: pgmap v14135: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:29.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:19:29 smithi083 ceph-mon[77242]: pgmap v14135: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:29.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:19:29 smithi153 ceph-mon[70044]: pgmap v14135: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:30.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:19:30 smithi083 ceph-mon[79930]: pgmap v14136: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:30.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:19:30 smithi083 ceph-mon[77242]: pgmap v14136: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:30.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:19:30 smithi153 ceph-mon[70044]: pgmap v14136: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:31.923 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:19:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:19:31] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T13:19:32.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:19:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:19:31.920Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:19:32.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:19:32 smithi083 ceph-mon[79930]: pgmap v14137: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:32.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:19:32 smithi083 ceph-mon[77242]: pgmap v14137: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:32.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:19:32 smithi153 ceph-mon[70044]: pgmap v14137: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:32.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:19:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:19:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:19:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:19:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:19:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:19:34.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:19:34 smithi083 ceph-mon[79930]: pgmap v14138: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:34.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:19:34 smithi083 ceph-mon[77242]: pgmap v14138: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:34.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:19:34 smithi153 ceph-mon[70044]: pgmap v14138: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:35.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:19:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:19:35.283Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:19:36.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:19:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:19:36.010Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:19:37.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:19:36 smithi083 ceph-mon[79930]: pgmap v14139: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:37.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:19:36 smithi083 ceph-mon[77242]: pgmap v14139: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:37.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:19:36 smithi153 ceph-mon[70044]: pgmap v14139: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:38.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:19:38 smithi083 ceph-mon[79930]: pgmap v14140: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:38.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:19:38 smithi083 ceph-mon[77242]: pgmap v14140: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:38.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:19:38 smithi153 ceph-mon[70044]: pgmap v14140: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:39.019 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:19:39.306 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:19:39.307 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 3m ago 7h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:19:39.307 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 107s ago 7h 63.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:19:39.307 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 3m ago 7h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:19:39.307 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 107s ago 7h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:19:39.308 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 107s ago 7h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:19:39.308 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 3m ago 7h 903M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:19:39.308 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 3m ago 7h 1736M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:19:39.309 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 107s ago 7h 752M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:19:39.309 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 3m ago 7h 1031M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:19:39.309 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 3m ago 7h 23.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:19:39.309 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 107s ago 7h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:19:39.310 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 3m ago 7h 1256M 2330M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:19:39.310 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 3m ago 7h 1244M 2330M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:19:39.310 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 3m ago 7h 945M 2330M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:19:39.310 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 3m ago 7h 1217M 2330M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:19:39.311 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 107s ago 7h 1069M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:19:39.311 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 107s ago 7h 1339M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:19:39.311 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 107s ago 7h 1197M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:19:39.311 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 107s ago 7h 1421M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:19:39.312 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 107s ago 7h 176M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:19:39.666 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:19:39.667 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:19:39.667 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:19:39.667 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:19:39.667 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:19:39.667 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:19:39.668 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:19:39.668 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:19:39.668 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:19:39.668 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:19:39.669 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:19:39.669 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:19:39.669 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:19:39.669 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:19:39.670 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:19:39.670 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:19:39.670 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:19:39.670 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:19:39.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:19:39 smithi083 ceph-mon[79930]: from='client.60505 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:19:39.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:19:39 smithi083 ceph-mon[79930]: from='client.60511 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:19:39.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:19:39 smithi083 ceph-mon[77242]: from='client.60505 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:19:39.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:19:39 smithi083 ceph-mon[77242]: from='client.60511 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:19:39.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:19:39 smithi153 ceph-mon[70044]: from='client.60505 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:19:39.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:19:39 smithi153 ceph-mon[70044]: from='client.60511 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:19:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:19:40 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1419796678' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:19:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:19:40 smithi083 ceph-mon[79930]: pgmap v14141: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:19:40 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1419796678' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:19:40.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:19:40 smithi083 ceph-mon[77242]: pgmap v14141: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:40.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:19:40 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1419796678' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:19:40.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:19:40 smithi153 ceph-mon[70044]: pgmap v14141: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:41.912 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:19:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:19:41] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T13:19:42.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:19:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:19:41.920Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:19:42.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:19:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:19:42.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:19:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:19:42.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:19:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:19:42.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:19:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:19:42.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:19:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:19:42.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:19:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:19:42.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:19:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:19:42.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:19:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:19:42.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:19:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:19:42.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:19:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:19:42.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:19:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:19:42.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:19:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:19:42.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:19:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:19:42.316Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:19:43.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:19:42 smithi083 ceph-mon[79930]: pgmap v14142: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:43.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:19:42 smithi083 ceph-mon[77242]: pgmap v14142: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:43.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:19:42 smithi153 ceph-mon[70044]: pgmap v14142: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:43.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:19:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:19:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:19:44.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:19:44 smithi083 ceph-mon[79930]: pgmap v14143: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:44.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:19:44 smithi083 ceph-mon[77242]: pgmap v14143: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:44.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:19:44 smithi153 ceph-mon[70044]: pgmap v14143: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:45.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:19:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:19:45.284Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:19:46.465 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:19:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:19:46.010Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:19:47.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:19:46 smithi083 ceph-mon[79930]: pgmap v14144: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:47.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:19:46 smithi083 ceph-mon[77242]: pgmap v14144: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:47.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:19:46 smithi153 ceph-mon[70044]: pgmap v14144: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:48.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:19:48 smithi083 ceph-mon[79930]: pgmap v14145: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:48.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:19:48 smithi083 ceph-mon[77242]: pgmap v14145: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:48.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:19:48 smithi153 ceph-mon[70044]: pgmap v14145: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:50.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:19:50 smithi083 ceph-mon[79930]: pgmap v14146: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:50.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:19:50 smithi083 ceph-mon[77242]: pgmap v14146: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:50.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:19:50 smithi153 ceph-mon[70044]: pgmap v14146: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:51.923 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:19:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:19:51] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T13:19:52.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:19:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:19:51.921Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:19:52.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:19:52 smithi083 ceph-mon[77242]: pgmap v14147: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:52.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:19:52 smithi083 ceph-mon[79930]: pgmap v14147: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:52.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:19:52 smithi153 ceph-mon[70044]: pgmap v14147: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:52.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:19:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:19:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:19:53.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:19:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:19:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:19:54.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:19:54 smithi083 ceph-mon[79930]: pgmap v14148: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:54.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:19:54 smithi083 ceph-mon[77242]: pgmap v14148: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:54.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:19:54 smithi153 ceph-mon[70044]: pgmap v14148: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:55.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:19:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:19:55.285Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:19:56.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:19:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:19:56.010Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:19:56.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:19:55 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:19:56.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:19:55 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:19:56.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:19:55 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:19:56.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:19:55 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:19:56.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:19:55 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:19:56.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:19:55 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:19:56.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:19:55 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:19:56.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:19:55 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:19:56.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:19:55 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:19:57.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:19:56 smithi083 ceph-mon[79930]: pgmap v14149: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:57.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:19:56 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:19:57.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:19:56 smithi083 ceph-mon[77242]: pgmap v14149: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:57.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:19:56 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:19:57.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:19:56 smithi153 ceph-mon[70044]: pgmap v14149: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:57.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:19:56 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:19:58.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:19:58 smithi083 ceph-mon[79930]: pgmap v14150: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:58.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:19:58 smithi083 ceph-mon[77242]: pgmap v14150: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:19:58.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:19:58 smithi153 ceph-mon[70044]: pgmap v14150: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:00.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:20:00 smithi153 ceph-mon[70044]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T13:20:00.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:20:00 smithi083 ceph-mon[79930]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T13:20:00.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:20:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[77219]: 2022-09-29T13:19:59.999+0000 7f160f665700 -1 log_channel(cluster) log [ERR] : overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T13:20:00.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:20:00 smithi083 ceph-mon[77242]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T13:20:01.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:20:01 smithi083 ceph-mon[79930]: pgmap v14151: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:01.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:20:01 smithi083 ceph-mon[77242]: pgmap v14151: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:01.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:20:01 smithi153 ceph-mon[70044]: pgmap v14151: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:01.925 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:20:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:20:01] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T13:20:02.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:20:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:20:01.922Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:20:02.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:20:02 smithi083 ceph-mon[79930]: pgmap v14152: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:02.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:20:02 smithi083 ceph-mon[77242]: pgmap v14152: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:02.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:20:02 smithi153 ceph-mon[70044]: pgmap v14152: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:02.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:20:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:20:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:20:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:20:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:20:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:20:04.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:20:04 smithi083 ceph-mon[79930]: pgmap v14153: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:04.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:20:04 smithi083 ceph-mon[77242]: pgmap v14153: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:04.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:20:04 smithi153 ceph-mon[70044]: pgmap v14153: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:05.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:20:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:20:05.285Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:20:06.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:20:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:20:06.011Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:20:07.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:20:06 smithi083 ceph-mon[79930]: pgmap v14154: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:07.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:20:06 smithi083 ceph-mon[77242]: pgmap v14154: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:07.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:20:06 smithi153 ceph-mon[70044]: pgmap v14154: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:08.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:20:08 smithi083 ceph-mon[79930]: pgmap v14155: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:08.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:20:08 smithi083 ceph-mon[77242]: pgmap v14155: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:08.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:20:08 smithi153 ceph-mon[70044]: pgmap v14155: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:09.992 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:20:10.281 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:20:10.282 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 4m ago 7h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:20:10.282 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 2m ago 7h 63.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:20:10.283 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 4m ago 7h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:20:10.283 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 2m ago 7h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:20:10.283 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 2m ago 7h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:20:10.283 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 4m ago 7h 903M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:20:10.283 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 4m ago 7h 1736M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:20:10.284 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 2m ago 7h 752M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:20:10.284 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 4m ago 7h 1031M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:20:10.284 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 4m ago 7h 23.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:20:10.284 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 2m ago 7h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:20:10.285 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 4m ago 7h 1256M 2330M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:20:10.285 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 4m ago 7h 1244M 2330M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:20:10.285 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 4m ago 7h 945M 2330M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:20:10.285 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 4m ago 7h 1217M 2330M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:20:10.286 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 2m ago 7h 1069M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:20:10.286 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 2m ago 7h 1339M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:20:10.287 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 2m ago 7h 1197M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:20:10.287 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 2m ago 7h 1421M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:20:10.287 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 2m ago 7h 176M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:20:10.642 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:20:10.642 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:20:10.642 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:20:10.643 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:20:10.643 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:20:10.643 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:20:10.644 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:20:10.644 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:20:10.644 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:20:10.645 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:20:10.645 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:20:10.646 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:20:10.646 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:20:10.647 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:20:10.647 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:20:10.648 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:20:10.648 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:20:10.649 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:20:10.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:20:10 smithi083 ceph-mon[77242]: from='client.50592 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:20:10.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:20:10 smithi083 ceph-mon[77242]: from='client.60529 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:20:10.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:20:10 smithi083 ceph-mon[77242]: pgmap v14156: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:10.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:20:10 smithi083 ceph-mon[79930]: from='client.50592 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:20:10.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:20:10 smithi083 ceph-mon[79930]: from='client.60529 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:20:10.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:20:10 smithi083 ceph-mon[79930]: pgmap v14156: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:10.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:20:10 smithi153 ceph-mon[70044]: from='client.50592 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:20:10.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:20:10 smithi153 ceph-mon[70044]: from='client.60529 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:20:10.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:20:10 smithi153 ceph-mon[70044]: pgmap v14156: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:11.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:20:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:20:11] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T13:20:11.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:20:11 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/933741411' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:20:11.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:20:11 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/933741411' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:20:11.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:20:11 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/933741411' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:20:12.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:20:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:20:11.922Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:20:12.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:20:12 smithi083 ceph-mon[79930]: pgmap v14157: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:12.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:20:12 smithi083 ceph-mon[77242]: pgmap v14157: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:12.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:20:12 smithi153 ceph-mon[70044]: pgmap v14157: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:12.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:20:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:20:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:20:13.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:20:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:20:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:20:14.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:20:14 smithi083 ceph-mon[79930]: pgmap v14158: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:14.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:20:14 smithi083 ceph-mon[77242]: pgmap v14158: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:14.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:20:14 smithi153 ceph-mon[70044]: pgmap v14158: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:15.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:20:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:20:15.284Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:20:16.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:20:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:20:16.012Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:20:17.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:20:16 smithi083 ceph-mon[79930]: pgmap v14159: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:17.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:20:16 smithi083 ceph-mon[77242]: pgmap v14159: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:17.312 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:20:16 smithi153 ceph-mon[70044]: pgmap v14159: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:18.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:20:18 smithi083 ceph-mon[79930]: pgmap v14160: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:18.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:20:18 smithi083 ceph-mon[77242]: pgmap v14160: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:18.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:20:18 smithi153 ceph-mon[70044]: pgmap v14160: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:20.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:20:20 smithi083 ceph-mon[79930]: pgmap v14161: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:20.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:20:20 smithi083 ceph-mon[77242]: pgmap v14161: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:20.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:20:20 smithi153 ceph-mon[70044]: pgmap v14161: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:21.925 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:20:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:20:21] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T13:20:22.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:20:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:20:21.923Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:20:22.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:20:22 smithi083 ceph-mon[79930]: pgmap v14162: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:22.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:20:22 smithi083 ceph-mon[77242]: pgmap v14162: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:22.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:20:22 smithi153 ceph-mon[70044]: pgmap v14162: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:22.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:20:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:20:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:20:23.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:20:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:20:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:20:24.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:20:24 smithi083 ceph-mon[79930]: pgmap v14163: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:24.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:20:24 smithi083 ceph-mon[77242]: pgmap v14163: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:24.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:20:24 smithi153 ceph-mon[70044]: pgmap v14163: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:25.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:20:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:20:25.286Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:20:26.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:20:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:20:26.013Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:20:27.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:20:26 smithi083 ceph-mon[79930]: pgmap v14164: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:27.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:20:26 smithi083 ceph-mon[77242]: pgmap v14164: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:27.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:20:26 smithi153 ceph-mon[70044]: pgmap v14164: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:28.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:20:28 smithi083 ceph-mon[79930]: pgmap v14165: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:28.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:20:28 smithi083 ceph-mon[77242]: pgmap v14165: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:28.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:20:28 smithi153 ceph-mon[70044]: pgmap v14165: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:30.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:20:30 smithi083 ceph-mon[79930]: pgmap v14166: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:30.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:20:30 smithi083 ceph-mon[77242]: pgmap v14166: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:30.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:20:30 smithi153 ceph-mon[70044]: pgmap v14166: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:31.926 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:20:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:20:31] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T13:20:32.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:20:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:20:31.923Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:20:32.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:20:32 smithi083 ceph-mon[79930]: pgmap v14167: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:32.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:20:32 smithi083 ceph-mon[77242]: pgmap v14167: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:32.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:20:32 smithi153 ceph-mon[70044]: pgmap v14167: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:32.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:20:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:20:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:20:33.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:20:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:20:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:20:34.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:20:34 smithi083 ceph-mon[79930]: pgmap v14168: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:34.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:20:34 smithi083 ceph-mon[77242]: pgmap v14168: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:34.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:20:34 smithi153 ceph-mon[70044]: pgmap v14168: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:35.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:20:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:20:35.287Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:20:36.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:20:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:20:36.014Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:20:37.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:20:36 smithi083 ceph-mon[79930]: pgmap v14169: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:37.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:20:36 smithi083 ceph-mon[77242]: pgmap v14169: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:37.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:20:36 smithi153 ceph-mon[70044]: pgmap v14169: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:38.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:20:38 smithi083 ceph-mon[79930]: pgmap v14170: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:38.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:20:38 smithi083 ceph-mon[77242]: pgmap v14170: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:38.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:20:38 smithi153 ceph-mon[70044]: pgmap v14170: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:20:40 smithi083 ceph-mon[79930]: pgmap v14171: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:40.740 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:20:40 smithi083 ceph-mon[77242]: pgmap v14171: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:40.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:20:40 smithi153 ceph-mon[70044]: pgmap v14171: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:40.967 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:20:41.257 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:20:41.257 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 4m ago 7h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:20:41.258 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 2m ago 7h 63.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:20:41.258 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 4m ago 7h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:20:41.258 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 2m ago 7h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:20:41.258 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 2m ago 7h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:20:41.259 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 4m ago 7h 903M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:20:41.259 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 4m ago 7h 1736M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:20:41.259 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 2m ago 7h 752M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:20:41.259 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 4m ago 7h 1031M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:20:41.260 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 4m ago 7h 23.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:20:41.260 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 2m ago 7h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:20:41.260 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 4m ago 7h 1256M 2330M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:20:41.260 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 4m ago 7h 1244M 2330M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:20:41.261 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 4m ago 7h 945M 2330M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:20:41.261 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 4m ago 7h 1217M 2330M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:20:41.261 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 2m ago 7h 1069M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:20:41.261 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 2m ago 7h 1339M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:20:41.262 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 2m ago 7h 1197M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:20:41.262 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 2m ago 7h 1421M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:20:41.262 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 2m ago 7h 176M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:20:41.611 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:20:41.612 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:20:41.612 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:20:41.612 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:20:41.612 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:20:41.613 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:20:41.613 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:20:41.613 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:20:41.613 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:20:41.613 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:20:41.614 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:20:41.614 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:20:41.614 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:20:41.614 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:20:41.615 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:20:41.615 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:20:41.615 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:20:41.615 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:20:41.924 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:20:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:20:41] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T13:20:42.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:20:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:20:41.924Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:20:42.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:20:41 smithi083 ceph-mon[79930]: from='client.60541 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:20:42.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:20:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:20:42.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:20:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:20:42.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:20:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:20:42.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:20:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:20:42.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:20:41 smithi083 ceph-mon[79930]: from='client.60547 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:20:42.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:20:41 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/286053923' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:20:42.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:20:41 smithi083 ceph-mon[77242]: from='client.60541 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:20:42.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:20:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:20:42.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:20:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:20:42.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:20:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:20:42.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:20:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:20:42.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:20:41 smithi083 ceph-mon[77242]: from='client.60547 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:20:42.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:20:41 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/286053923' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:20:42.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:20:41 smithi153 ceph-mon[70044]: from='client.60541 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:20:42.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:20:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:20:42.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:20:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:20:42.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:20:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:20:42.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:20:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:20:42.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:20:41 smithi153 ceph-mon[70044]: from='client.60547 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:20:42.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:20:41 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/286053923' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:20:42.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:20:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:20:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:20:43.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:20:42 smithi083 ceph-mon[79930]: pgmap v14172: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:43.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:20:42 smithi083 ceph-mon[77242]: pgmap v14172: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:43.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:20:42 smithi153 ceph-mon[70044]: pgmap v14172: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:43.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:20:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:20:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:20:44.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:20:44 smithi083 ceph-mon[79930]: pgmap v14173: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:44.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:20:44 smithi083 ceph-mon[77242]: pgmap v14173: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:44.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:20:44 smithi153 ceph-mon[70044]: pgmap v14173: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:45.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:20:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:20:45.288Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:20:46.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:20:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:20:46.014Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:20:47.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:20:46 smithi083 ceph-mon[77242]: pgmap v14174: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:47.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:20:46 smithi083 ceph-mon[79930]: pgmap v14174: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:47.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:20:46 smithi153 ceph-mon[70044]: pgmap v14174: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:48.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:20:48 smithi083 ceph-mon[79930]: pgmap v14175: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:48.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:20:48 smithi083 ceph-mon[77242]: pgmap v14175: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:48.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:20:48 smithi153 ceph-mon[70044]: pgmap v14175: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:50.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:20:50 smithi083 ceph-mon[79930]: pgmap v14176: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:50.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:20:50 smithi083 ceph-mon[77242]: pgmap v14176: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:50.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:20:50 smithi153 ceph-mon[70044]: pgmap v14176: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:51.927 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:20:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:20:51] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T13:20:52.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:20:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:20:51.924Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:20:52.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:20:52 smithi083 ceph-mon[79930]: pgmap v14177: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:52.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:20:52 smithi083 ceph-mon[77242]: pgmap v14177: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:52.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:20:52 smithi153 ceph-mon[70044]: pgmap v14177: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:52.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:20:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:20:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:20:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:20:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:20:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:20:54.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:20:54 smithi083 ceph-mon[79930]: pgmap v14178: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:54.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:20:54 smithi083 ceph-mon[77242]: pgmap v14178: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:54.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:20:54 smithi153 ceph-mon[70044]: pgmap v14178: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:55.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:20:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:20:55.288Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:20:56.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:20:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:20:56.015Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:20:57.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:20:56 smithi083 ceph-mon[79930]: pgmap v14179: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:57.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:20:56 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:20:57.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:20:56 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:20:57.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:20:56 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:20:57.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:20:56 smithi083 ceph-mon[77242]: pgmap v14179: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:57.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:20:56 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:20:57.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:20:56 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:20:57.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:20:56 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:20:57.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:20:56 smithi153 ceph-mon[70044]: pgmap v14179: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:57.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:20:56 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:20:57.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:20:56 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:20:57.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:20:56 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:20:58.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:20:58 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:20:58.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:20:58 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:20:58.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:20:58 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:20:59.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:20:59 smithi083 ceph-mon[79930]: pgmap v14180: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:59.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:20:59 smithi083 ceph-mon[77242]: pgmap v14180: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:20:59.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:20:59 smithi153 ceph-mon[70044]: pgmap v14180: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:00.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:21:00 smithi083 ceph-mon[77242]: pgmap v14181: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:00.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:21:00 smithi083 ceph-mon[79930]: pgmap v14181: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:00.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:21:00 smithi153 ceph-mon[70044]: pgmap v14181: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:01.928 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:21:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:21:01] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T13:21:02.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:21:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:21:01.925Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:21:02.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:21:02 smithi083 ceph-mon[79930]: pgmap v14182: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:02.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:21:02 smithi083 ceph-mon[77242]: pgmap v14182: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:02.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:21:02 smithi153 ceph-mon[70044]: pgmap v14182: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:02.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:21:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:21:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:21:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:21:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:21:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:21:04.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:21:04 smithi083 ceph-mon[79930]: pgmap v14183: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:04.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:21:04 smithi083 ceph-mon[77242]: pgmap v14183: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:04.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:21:04 smithi153 ceph-mon[70044]: pgmap v14183: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:05.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:21:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:21:05.288Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:21:06.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:21:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:21:06.016Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:21:07.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:21:06 smithi083 ceph-mon[79930]: pgmap v14184: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:07.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:21:06 smithi083 ceph-mon[77242]: pgmap v14184: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:07.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:21:06 smithi153 ceph-mon[70044]: pgmap v14184: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:08.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:21:08 smithi083 ceph-mon[79930]: pgmap v14185: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:08.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:21:08 smithi083 ceph-mon[77242]: pgmap v14185: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:08.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:21:08 smithi153 ceph-mon[70044]: pgmap v14185: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:10.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:21:10 smithi083 ceph-mon[79930]: pgmap v14186: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:10.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:21:10 smithi083 ceph-mon[77242]: pgmap v14186: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:10.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:21:10 smithi153 ceph-mon[70044]: pgmap v14186: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:11.928 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:21:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:21:11] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T13:21:11.945 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:21:12.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:21:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:21:11.925Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:21:12.234 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:21:12.234 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 5m ago 7h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:21:12.235 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 3m ago 7h 63.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:21:12.235 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 5m ago 7h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:21:12.235 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 3m ago 7h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:21:12.235 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 3m ago 7h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:21:12.236 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 5m ago 8h 903M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:21:12.236 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 5m ago 8h 1736M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:21:12.236 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 3m ago 7h 752M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:21:12.236 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 5m ago 7h 1031M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:21:12.236 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 5m ago 7h 23.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:21:12.237 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 3m ago 7h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:21:12.237 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 5m ago 7h 1256M 2330M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:21:12.237 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 5m ago 7h 1244M 2330M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:21:12.237 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 5m ago 7h 945M 2330M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:21:12.238 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 5m ago 7h 1217M 2330M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:21:12.238 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 3m ago 7h 1069M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:21:12.238 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 3m ago 7h 1339M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:21:12.238 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 3m ago 7h 1197M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:21:12.239 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 3m ago 7h 1421M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:21:12.239 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 3m ago 7h 176M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:21:12.597 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:21:12.598 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:21:12.598 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:21:12.598 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:21:12.598 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:21:12.599 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:21:12.599 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:21:12.599 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:21:12.599 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:21:12.600 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:21:12.600 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:21:12.600 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:21:12.600 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:21:12.600 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:21:12.601 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:21:12.601 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:21:12.601 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:21:12.601 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:21:12.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:21:12 smithi083 ceph-mon[79930]: from='client.50628 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:21:12.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:21:12 smithi083 ceph-mon[79930]: from='client.50634 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:21:12.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:21:12 smithi083 ceph-mon[79930]: pgmap v14187: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:12.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:21:12 smithi083 ceph-mon[77242]: from='client.50628 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:21:12.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:21:12 smithi083 ceph-mon[77242]: from='client.50634 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:21:12.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:21:12 smithi083 ceph-mon[77242]: pgmap v14187: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:12.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:21:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:21:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:21:12.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:21:12 smithi153 ceph-mon[70044]: from='client.50628 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:21:12.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:21:12 smithi153 ceph-mon[70044]: from='client.50634 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:21:12.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:21:12 smithi153 ceph-mon[70044]: pgmap v14187: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:13.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:21:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:21:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:21:13.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:21:13 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1231009437' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:21:13.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:21:13 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1231009437' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:21:13.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:21:13 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1231009437' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:21:14.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:21:14 smithi083 ceph-mon[79930]: pgmap v14188: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:14.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:21:14 smithi083 ceph-mon[77242]: pgmap v14188: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:14.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:21:14 smithi153 ceph-mon[70044]: pgmap v14188: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:15.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:21:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:21:15.289Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:21:16.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:21:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:21:16.017Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:21:17.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:21:16 smithi083 ceph-mon[79930]: pgmap v14189: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:17.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:21:16 smithi083 ceph-mon[77242]: pgmap v14189: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:17.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:21:16 smithi153 ceph-mon[70044]: pgmap v14189: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:18.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:21:18 smithi083 ceph-mon[79930]: pgmap v14190: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:18.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:21:18 smithi083 ceph-mon[77242]: pgmap v14190: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:18.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:21:18 smithi153 ceph-mon[70044]: pgmap v14190: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:20.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:21:20 smithi083 ceph-mon[79930]: pgmap v14191: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:20.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:21:20 smithi083 ceph-mon[77242]: pgmap v14191: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:20.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:21:20 smithi153 ceph-mon[70044]: pgmap v14191: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:21.929 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:21:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:21:21] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T13:21:22.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:21:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:21:21.926Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:21:22.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:21:22 smithi083 ceph-mon[79930]: pgmap v14192: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:22.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:21:22 smithi083 ceph-mon[77242]: pgmap v14192: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:22.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:21:22 smithi153 ceph-mon[70044]: pgmap v14192: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:22.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:21:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:21:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:21:23.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:21:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:21:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:21:24.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:21:24 smithi083 ceph-mon[77242]: pgmap v14193: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:24.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:21:24 smithi083 ceph-mon[79930]: pgmap v14193: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:24.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:21:24 smithi153 ceph-mon[70044]: pgmap v14193: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:25.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:21:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:21:25.290Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:21:26.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:21:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:21:26.017Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:21:27.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:21:26 smithi083 ceph-mon[79930]: pgmap v14194: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:27.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:21:26 smithi083 ceph-mon[77242]: pgmap v14194: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:27.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:21:26 smithi153 ceph-mon[70044]: pgmap v14194: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:28.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:21:28 smithi083 ceph-mon[79930]: pgmap v14195: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:28.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:21:28 smithi083 ceph-mon[77242]: pgmap v14195: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:28.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:21:28 smithi153 ceph-mon[70044]: pgmap v14195: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:30.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:21:30 smithi083 ceph-mon[79930]: pgmap v14196: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:30.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:21:30 smithi083 ceph-mon[77242]: pgmap v14196: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:30.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:21:30 smithi153 ceph-mon[70044]: pgmap v14196: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:31.929 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:21:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:21:31] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T13:21:32.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:21:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:21:31.927Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:21:32.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:21:32 smithi083 ceph-mon[77242]: pgmap v14197: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:32.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:21:32 smithi083 ceph-mon[79930]: pgmap v14197: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:32.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:21:32 smithi153 ceph-mon[70044]: pgmap v14197: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:32.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:21:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:21:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:21:33.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:21:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:21:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:21:34.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:21:34 smithi083 ceph-mon[77242]: pgmap v14198: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:34.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:21:34 smithi083 ceph-mon[79930]: pgmap v14198: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:34.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:21:34 smithi153 ceph-mon[70044]: pgmap v14198: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:35.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:21:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:21:35.291Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:21:36.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:21:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:21:36.017Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:21:37.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:21:36 smithi083 ceph-mon[79930]: pgmap v14199: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:37.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:21:36 smithi083 ceph-mon[77242]: pgmap v14199: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:37.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:21:36 smithi153 ceph-mon[70044]: pgmap v14199: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:38.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:21:38 smithi083 ceph-mon[79930]: pgmap v14200: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:38.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:21:38 smithi083 ceph-mon[77242]: pgmap v14200: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:38.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:21:38 smithi153 ceph-mon[70044]: pgmap v14200: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:21:40 smithi083 ceph-mon[79930]: pgmap v14201: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:21:40 smithi083 ceph-mon[77242]: pgmap v14201: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:40.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:21:40 smithi153 ceph-mon[70044]: pgmap v14201: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:41.930 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:21:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:21:41] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T13:21:42.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:21:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:21:41.927Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:21:42.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:21:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:21:42.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:21:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:21:42.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:21:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:21:42.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:21:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:21:42.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:21:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:21:42.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:21:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:21:42.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:21:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:21:42.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:21:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:21:42.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:21:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:21:42.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:21:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:21:42.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:21:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:21:42.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:21:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:21:42.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:21:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:21:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:21:42.928 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:21:43.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:21:42 smithi083 ceph-mon[79930]: pgmap v14202: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:43.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:21:42 smithi083 ceph-mon[77242]: pgmap v14202: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:43.217 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:21:43.217 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 6m ago 7h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:21:43.217 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 3m ago 7h 63.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:21:43.218 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 6m ago 7h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:21:43.218 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 3m ago 7h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:21:43.218 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 3m ago 7h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:21:43.218 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 6m ago 8h 903M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:21:43.219 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 6m ago 8h 1736M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:21:43.219 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 3m ago 7h 752M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:21:43.219 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 6m ago 7h 1031M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:21:43.219 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 6m ago 7h 23.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:21:43.220 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 3m ago 7h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:21:43.220 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 6m ago 7h 1256M 2330M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:21:43.220 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 6m ago 7h 1244M 2330M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:21:43.220 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 6m ago 7h 945M 2330M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:21:43.221 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 6m ago 7h 1217M 2330M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:21:43.221 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 3m ago 7h 1069M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:21:43.221 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 3m ago 7h 1339M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:21:43.221 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 3m ago 7h 1197M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:21:43.222 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 3m ago 7h 1421M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:21:43.222 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 3m ago 7h 176M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:21:43.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:21:42 smithi153 ceph-mon[70044]: pgmap v14202: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:43.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:21:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:21:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:21:43.582 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:21:43.583 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:21:43.583 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:21:43.583 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:21:43.584 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:21:43.584 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:21:43.584 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:21:43.585 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:21:43.585 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:21:43.585 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:21:43.585 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:21:43.586 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:21:43.586 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:21:43.586 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:21:43.586 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:21:43.587 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:21:43.587 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:21:43.588 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:21:44.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:21:43 smithi083 ceph-mon[79930]: from='client.50646 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:21:44.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:21:43 smithi083 ceph-mon[79930]: from='client.50652 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:21:44.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:21:43 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4103079375' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:21:44.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:21:43 smithi083 ceph-mon[77242]: from='client.50646 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:21:44.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:21:43 smithi083 ceph-mon[77242]: from='client.50652 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:21:44.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:21:43 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4103079375' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:21:44.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:21:43 smithi153 ceph-mon[70044]: from='client.50646 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:21:44.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:21:43 smithi153 ceph-mon[70044]: from='client.50652 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:21:44.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:21:43 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4103079375' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:21:45.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:21:44 smithi083 ceph-mon[79930]: pgmap v14203: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:45.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:21:44 smithi083 ceph-mon[77242]: pgmap v14203: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:45.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:21:44 smithi153 ceph-mon[70044]: pgmap v14203: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:45.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:21:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:21:45.292Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:21:46.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:21:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:21:46.017Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:21:47.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:21:46 smithi083 ceph-mon[79930]: pgmap v14204: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:47.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:21:46 smithi083 ceph-mon[77242]: pgmap v14204: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:47.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:21:46 smithi153 ceph-mon[70044]: pgmap v14204: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:48.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:21:48 smithi083 ceph-mon[77242]: pgmap v14205: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:48.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:21:48 smithi083 ceph-mon[79930]: pgmap v14205: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:48.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:21:48 smithi153 ceph-mon[70044]: pgmap v14205: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:50.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:21:50 smithi083 ceph-mon[77242]: pgmap v14206: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:50.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:21:50 smithi083 ceph-mon[79930]: pgmap v14206: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:50.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:21:50 smithi153 ceph-mon[70044]: pgmap v14206: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:51.931 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:21:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:21:51] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T13:21:52.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:21:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:21:51.928Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:21:52.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:21:52 smithi083 ceph-mon[77242]: pgmap v14207: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:52.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:21:52 smithi083 ceph-mon[79930]: pgmap v14207: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:52.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:21:52 smithi153 ceph-mon[70044]: pgmap v14207: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:52.810 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:21:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:21:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:21:53.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:21:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:21:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:21:54.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:21:54 smithi083 ceph-mon[77242]: pgmap v14208: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:54.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:21:54 smithi083 ceph-mon[79930]: pgmap v14208: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:54.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:21:54 smithi153 ceph-mon[70044]: pgmap v14208: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:55.714 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:21:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:21:55.292Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:21:56.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:21:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:21:56.018Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:21:57.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:21:56 smithi083 ceph-mon[79930]: pgmap v14209: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:57.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:21:56 smithi083 ceph-mon[77242]: pgmap v14209: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:57.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:21:56 smithi153 ceph-mon[70044]: pgmap v14209: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:58.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:21:57 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:21:58.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:21:57 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:21:58.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:21:57 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:21:58.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:21:57 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:21:58.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:21:57 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:21:58.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:21:57 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:21:58.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:21:57 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:21:58.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:21:57 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:21:58.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:21:57 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:21:58.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:21:57 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:21:58.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:21:57 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:21:58.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:21:57 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:21:59.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:21:58 smithi083 ceph-mon[79930]: pgmap v14210: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:59.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:21:58 smithi083 ceph-mon[77242]: pgmap v14210: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:21:59.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:21:58 smithi153 ceph-mon[70044]: pgmap v14210: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:00.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:22:00 smithi083 ceph-mon[77242]: pgmap v14211: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:00.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:22:00 smithi083 ceph-mon[79930]: pgmap v14211: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:00.813 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:22:00 smithi153 ceph-mon[70044]: pgmap v14211: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:01.934 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:22:01] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T13:22:02.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:22:01.929Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:22:02.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:22:02 smithi083 ceph-mon[79930]: pgmap v14212: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:02.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:22:02 smithi083 ceph-mon[77242]: pgmap v14212: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:02.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:22:02 smithi153 ceph-mon[70044]: pgmap v14212: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:02.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:22:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:22:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:22:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:22:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:22:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:22:04.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:22:04 smithi083 ceph-mon[77242]: pgmap v14213: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:04.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:22:04 smithi083 ceph-mon[79930]: pgmap v14213: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:04.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:22:04 smithi153 ceph-mon[70044]: pgmap v14213: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:05.715 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:22:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:22:05.293Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:22:06.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:22:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:22:06.018Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:22:07.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:22:06 smithi083 ceph-mon[79930]: pgmap v14214: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:07.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:22:06 smithi083 ceph-mon[77242]: pgmap v14214: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:07.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:22:06 smithi153 ceph-mon[70044]: pgmap v14214: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:08.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:22:08 smithi083 ceph-mon[79930]: pgmap v14215: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:08.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:22:08 smithi083 ceph-mon[77242]: pgmap v14215: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:08.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:22:08 smithi153 ceph-mon[70044]: pgmap v14215: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:10.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:22:10 smithi083 ceph-mon[79930]: pgmap v14216: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:10.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:22:10 smithi083 ceph-mon[77242]: pgmap v14216: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:10.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:22:10 smithi153 ceph-mon[70044]: pgmap v14216: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:11.932 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:22:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:22:11] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T13:22:12.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:22:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:22:11.929Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:22:12.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:22:12 smithi083 ceph-mon[79930]: pgmap v14217: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:12.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:22:12 smithi083 ceph-mon[77242]: pgmap v14217: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:12.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:22:12 smithi153 ceph-mon[70044]: pgmap v14217: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:12.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:22:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:22:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:22:13.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:22:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:22:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:22:13.907 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:22:14.191 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:22:14.192 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 6m ago 7h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:22:14.192 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 4m ago 7h 63.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:22:14.192 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 6m ago 7h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:22:14.193 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 4m ago 7h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:22:14.193 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 4m ago 7h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:22:14.193 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 6m ago 8h 903M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:22:14.193 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 6m ago 8h 1736M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:22:14.194 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 4m ago 8h 752M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:22:14.194 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 6m ago 8h 1031M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:22:14.194 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 6m ago 7h 23.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:22:14.194 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 4m ago 7h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:22:14.195 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 6m ago 7h 1256M 2330M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:22:14.195 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 6m ago 7h 1244M 2330M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:22:14.195 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 6m ago 7h 945M 2330M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:22:14.196 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 6m ago 7h 1217M 2330M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:22:14.196 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 4m ago 7h 1069M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:22:14.197 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 4m ago 7h 1339M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:22:14.197 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 4m ago 7h 1197M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:22:14.197 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 4m ago 7h 1421M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:22:14.198 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 4m ago 7h 176M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:22:14.552 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:22:14.553 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:22:14.553 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:22:14.553 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:22:14.554 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:22:14.554 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:22:14.554 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:22:14.554 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:22:14.555 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:22:14.555 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:22:14.555 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:22:14.555 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:22:14.556 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:22:14.556 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:22:14.556 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:22:14.557 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:22:14.557 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:22:14.557 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:22:14.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:22:14 smithi083 ceph-mon[77242]: from='client.50664 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:22:14.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:22:14 smithi083 ceph-mon[77242]: from='client.50670 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:22:14.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:22:14 smithi083 ceph-mon[77242]: pgmap v14218: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:14.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:22:14 smithi083 ceph-mon[79930]: from='client.50664 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:22:14.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:22:14 smithi083 ceph-mon[79930]: from='client.50670 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:22:14.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:22:14 smithi083 ceph-mon[79930]: pgmap v14218: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:14.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:22:14 smithi153 ceph-mon[70044]: from='client.50664 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:22:14.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:22:14 smithi153 ceph-mon[70044]: from='client.50670 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:22:14.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:22:14 smithi153 ceph-mon[70044]: pgmap v14218: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:15.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:22:15 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2370279676' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:22:15.715 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:22:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:22:15.293Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:22:15.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:22:15 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2370279676' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:22:15.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:22:15 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2370279676' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:22:16.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:22:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:22:16.019Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:22:17.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:22:16 smithi083 ceph-mon[79930]: pgmap v14219: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:17.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:22:16 smithi083 ceph-mon[77242]: pgmap v14219: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:17.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:22:16 smithi153 ceph-mon[70044]: pgmap v14219: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:18.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:22:18 smithi083 ceph-mon[79930]: pgmap v14220: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:18.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:22:18 smithi083 ceph-mon[77242]: pgmap v14220: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:18.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:22:18 smithi153 ceph-mon[70044]: pgmap v14220: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:20.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:22:20 smithi083 ceph-mon[77242]: pgmap v14221: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:20.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:22:20 smithi083 ceph-mon[79930]: pgmap v14221: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:20.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:22:20 smithi153 ceph-mon[70044]: pgmap v14221: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:21.933 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:22:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:22:21] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T13:22:22.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:22:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:22:21.930Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:22:22.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:22:22 smithi083 ceph-mon[77242]: pgmap v14222: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:22.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:22:22 smithi083 ceph-mon[79930]: pgmap v14222: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:22.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:22:22 smithi153 ceph-mon[70044]: pgmap v14222: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:22.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:22:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:22:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:22:23.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:22:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:22:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:22:24.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:22:24 smithi083 ceph-mon[79930]: pgmap v14223: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:24.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:22:24 smithi083 ceph-mon[77242]: pgmap v14223: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:24.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:22:24 smithi153 ceph-mon[70044]: pgmap v14223: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:25.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:22:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:22:25.294Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:22:26.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:22:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:22:26.020Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:22:27.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:22:26 smithi083 ceph-mon[79930]: pgmap v14224: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:27.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:22:26 smithi083 ceph-mon[77242]: pgmap v14224: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:27.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:22:26 smithi153 ceph-mon[70044]: pgmap v14224: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:28.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:22:28 smithi083 ceph-mon[79930]: pgmap v14225: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:28.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:22:28 smithi083 ceph-mon[77242]: pgmap v14225: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:28.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:22:28 smithi153 ceph-mon[70044]: pgmap v14225: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:30.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:22:30 smithi083 ceph-mon[79930]: pgmap v14226: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:30.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:22:30 smithi083 ceph-mon[77242]: pgmap v14226: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:30.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:22:30 smithi153 ceph-mon[70044]: pgmap v14226: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:31.934 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:22:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:22:31] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T13:22:32.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:22:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:22:31.931Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:22:32.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:22:32 smithi083 ceph-mon[79930]: pgmap v14227: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:32.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:22:32 smithi083 ceph-mon[77242]: pgmap v14227: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:32.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:22:32 smithi153 ceph-mon[70044]: pgmap v14227: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:32.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:22:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:22:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:22:33.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:22:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:22:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:22:34.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:22:34 smithi083 ceph-mon[79930]: pgmap v14228: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:34.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:22:34 smithi083 ceph-mon[77242]: pgmap v14228: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:34.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:22:34 smithi153 ceph-mon[70044]: pgmap v14228: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:35.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:22:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:22:35.295Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:22:36.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:22:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:22:36.020Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:22:37.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:22:36 smithi083 ceph-mon[79930]: pgmap v14229: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:37.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:22:36 smithi083 ceph-mon[77242]: pgmap v14229: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:37.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:22:36 smithi153 ceph-mon[70044]: pgmap v14229: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:38.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:22:38 smithi083 ceph-mon[79930]: pgmap v14230: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:38.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:22:38 smithi083 ceph-mon[77242]: pgmap v14230: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:38.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:22:38 smithi153 ceph-mon[70044]: pgmap v14230: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:40.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:22:40 smithi083 ceph-mon[79930]: pgmap v14231: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:40.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:22:40 smithi083 ceph-mon[77242]: pgmap v14231: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:40.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:22:40 smithi153 ceph-mon[70044]: pgmap v14231: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:41.934 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:22:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:22:41] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T13:22:42.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:22:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:22:42.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:22:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:22:42.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:22:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:22:42.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:22:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:22:42.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:22:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:22:42.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:22:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:22:42.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:22:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:22:42.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:22:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:22:42.216 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:22:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:22:41.932Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:22:42.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:22:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:22:42.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:22:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:22:42.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:22:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:22:42.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:22:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:22:42.810 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:22:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:22:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:22:43.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:22:42 smithi083 ceph-mon[79930]: pgmap v14232: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:43.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:22:42 smithi083 ceph-mon[77242]: pgmap v14232: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:43.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:22:42 smithi153 ceph-mon[70044]: pgmap v14232: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:43.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:22:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:22:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:22:44.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:22:44 smithi083 ceph-mon[79930]: pgmap v14233: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:44.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:22:44 smithi083 ceph-mon[77242]: pgmap v14233: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:44.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:22:44 smithi153 ceph-mon[70044]: pgmap v14233: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:44.881 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:22:45.175 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:22:45.176 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 7m ago 7h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:22:45.176 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 4m ago 7h 63.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:22:45.177 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 7m ago 7h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:22:45.177 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 4m ago 7h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:22:45.177 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 4m ago 8h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:22:45.177 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 7m ago 8h 903M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:22:45.178 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 7m ago 8h 1736M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:22:45.178 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 4m ago 8h 752M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:22:45.178 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 7m ago 8h 1031M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:22:45.178 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 7m ago 7h 23.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:22:45.179 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 4m ago 7h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:22:45.179 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 7m ago 8h 1256M 2330M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:22:45.179 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 7m ago 7h 1244M 2330M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:22:45.180 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 7m ago 7h 945M 2330M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:22:45.180 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 7m ago 7h 1217M 2330M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:22:45.180 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 4m ago 7h 1069M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:22:45.180 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 4m ago 7h 1339M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:22:45.181 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 4m ago 7h 1197M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:22:45.181 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 4m ago 7h 1421M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:22:45.182 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 4m ago 7h 176M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:22:45.534 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:22:45.534 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:22:45.534 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:22:45.535 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:22:45.535 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:22:45.535 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:22:45.535 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:22:45.535 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:22:45.536 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:22:45.536 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:22:45.536 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:22:45.536 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:22:45.537 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:22:45.537 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:22:45.537 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:22:45.537 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:22:45.537 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:22:45.538 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:22:45.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:22:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:22:45.295Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:22:45.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:22:45 smithi083 ceph-mon[77242]: from='client.50682 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:22:45.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:22:45 smithi083 ceph-mon[77242]: from='client.60619 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:22:45.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:22:45 smithi083 ceph-mon[79930]: from='client.50682 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:22:45.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:22:45 smithi083 ceph-mon[79930]: from='client.60619 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:22:45.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:22:45 smithi153 ceph-mon[70044]: from='client.50682 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:22:45.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:22:45 smithi153 ceph-mon[70044]: from='client.60619 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:22:46.445 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:22:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:22:46.021Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:22:46.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:22:46 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1726304190' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:22:46.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:22:46 smithi083 ceph-mon[79930]: pgmap v14234: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:46.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:22:46 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1726304190' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:22:46.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:22:46 smithi083 ceph-mon[77242]: pgmap v14234: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:46.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:22:46 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1726304190' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:22:46.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:22:46 smithi153 ceph-mon[70044]: pgmap v14234: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:48.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:22:48 smithi083 ceph-mon[79930]: pgmap v14235: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:48.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:22:48 smithi083 ceph-mon[77242]: pgmap v14235: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:48.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:22:48 smithi153 ceph-mon[70044]: pgmap v14235: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:50.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:22:50 smithi083 ceph-mon[79930]: pgmap v14236: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:50.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:22:50 smithi083 ceph-mon[77242]: pgmap v14236: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:50.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:22:50 smithi153 ceph-mon[70044]: pgmap v14236: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:51.935 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:22:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:22:51] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T13:22:52.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:22:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:22:51.932Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:22:52.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:22:52 smithi083 ceph-mon[79930]: pgmap v14237: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:52.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:22:52 smithi083 ceph-mon[77242]: pgmap v14237: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:52.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:22:52 smithi153 ceph-mon[70044]: pgmap v14237: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:52.810 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:22:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:22:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:22:53.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:22:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:22:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:22:54.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:22:54 smithi083 ceph-mon[79930]: pgmap v14238: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:54.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:22:54 smithi083 ceph-mon[77242]: pgmap v14238: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:54.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:22:54 smithi153 ceph-mon[70044]: pgmap v14238: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:55.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:22:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:22:55.297Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:22:56.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:22:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:22:56.022Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:22:57.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:22:56 smithi083 ceph-mon[79930]: pgmap v14239: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:57.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:22:56 smithi083 ceph-mon[77242]: pgmap v14239: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:57.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:22:56 smithi153 ceph-mon[70044]: pgmap v14239: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:58.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:22:57 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:22:58.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:22:57 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:22:58.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:22:57 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:22:58.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:22:57 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:22:58.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:22:57 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:22:58.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:22:57 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:22:58.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:22:57 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:22:58.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:22:57 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:22:58.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:22:57 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:22:59.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:22:58 smithi153 ceph-mon[70044]: pgmap v14240: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:59.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:22:58 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:22:59.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:22:58 smithi083 ceph-mon[77242]: pgmap v14240: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:59.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:22:58 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:22:59.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:22:58 smithi083 ceph-mon[79930]: pgmap v14240: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:22:59.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:22:58 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:23:00.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:23:00 smithi083 ceph-mon[77242]: pgmap v14241: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:00.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:23:00 smithi083 ceph-mon[79930]: pgmap v14241: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:00.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:23:00 smithi153 ceph-mon[70044]: pgmap v14241: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:01.936 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:23:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:23:01] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T13:23:02.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:23:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:23:01.933Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:23:02.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:23:02 smithi083 ceph-mon[79930]: pgmap v14242: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:02.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:23:02 smithi083 ceph-mon[77242]: pgmap v14242: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:02.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:23:02 smithi153 ceph-mon[70044]: pgmap v14242: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:02.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:23:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:23:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:23:03.310 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:23:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:23:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:23:04.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:23:04 smithi083 ceph-mon[77242]: pgmap v14243: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:04.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:23:04 smithi083 ceph-mon[79930]: pgmap v14243: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:04.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:23:04 smithi153 ceph-mon[70044]: pgmap v14243: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:05.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:23:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:23:05.297Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:23:06.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:23:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:23:06.023Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:23:07.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:23:06 smithi153 ceph-mon[70044]: pgmap v14244: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:07.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:23:06 smithi083 ceph-mon[79930]: pgmap v14244: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:07.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:23:06 smithi083 ceph-mon[77242]: pgmap v14244: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:08.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:23:08 smithi083 ceph-mon[79930]: pgmap v14245: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:08.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:23:08 smithi083 ceph-mon[77242]: pgmap v14245: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:08.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:23:08 smithi153 ceph-mon[70044]: pgmap v14245: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:10.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:23:10 smithi083 ceph-mon[79930]: pgmap v14246: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:10.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:23:10 smithi083 ceph-mon[77242]: pgmap v14246: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:10.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:23:10 smithi153 ceph-mon[70044]: pgmap v14246: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:11.938 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:23:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:23:11] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T13:23:12.214 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:23:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:23:11.933Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:23:12.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:23:12 smithi083 ceph-mon[79930]: pgmap v14247: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:12.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:23:12 smithi083 ceph-mon[77242]: pgmap v14247: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:12.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:23:12 smithi153 ceph-mon[70044]: pgmap v14247: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:12.810 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:23:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:23:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:23:13.311 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:23:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:23:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:23:14.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:23:14 smithi083 ceph-mon[79930]: pgmap v14248: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:14.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:23:14 smithi083 ceph-mon[77242]: pgmap v14248: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:14.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:23:14 smithi153 ceph-mon[70044]: pgmap v14248: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:15.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:23:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:23:15.297Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:23:15.858 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:23:16.147 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:23:16.148 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 7m ago 7h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:23:16.148 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 5m ago 7h 63.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:23:16.149 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 7m ago 7h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:23:16.149 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 5m ago 7h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:23:16.149 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 5m ago 8h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:23:16.150 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 7m ago 8h 903M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:23:16.150 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 7m ago 8h 1736M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:23:16.151 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 5m ago 8h 752M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:23:16.151 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 7m ago 8h 1031M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:23:16.152 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 7m ago 7h 23.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:23:16.152 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 5m ago 7h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:23:16.153 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 7m ago 8h 1256M 2330M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:23:16.153 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 7m ago 8h 1244M 2330M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:23:16.154 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 7m ago 8h 945M 2330M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:23:16.154 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 7m ago 7h 1217M 2330M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:23:16.155 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 5m ago 7h 1069M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:23:16.155 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 5m ago 7h 1339M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:23:16.156 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 5m ago 7h 1197M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:23:16.156 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 5m ago 7h 1421M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:23:16.156 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 5m ago 7h 176M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:23:16.467 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:23:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:23:16.023Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:23:16.508 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:23:16.508 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:23:16.508 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:23:16.509 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:23:16.509 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:23:16.509 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:23:16.509 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:23:16.510 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:23:16.510 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:23:16.511 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:23:16.511 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:23:16.511 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:23:16.511 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:23:16.512 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:23:16.512 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:23:16.512 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:23:16.512 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:23:16.513 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:23:17.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:23:16 smithi153 ceph-mon[70044]: from='client.50700 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:23:17.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:23:16 smithi153 ceph-mon[70044]: from='client.50706 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:23:17.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:23:16 smithi153 ceph-mon[70044]: pgmap v14249: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:17.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:23:16 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2244419030' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:23:17.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:23:16 smithi083 ceph-mon[79930]: from='client.50700 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:23:17.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:23:16 smithi083 ceph-mon[79930]: from='client.50706 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:23:17.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:23:16 smithi083 ceph-mon[79930]: pgmap v14249: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:17.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:23:16 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2244419030' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:23:17.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:23:16 smithi083 ceph-mon[77242]: from='client.50700 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:23:17.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:23:16 smithi083 ceph-mon[77242]: from='client.50706 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:23:17.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:23:16 smithi083 ceph-mon[77242]: pgmap v14249: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:17.467 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:23:16 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2244419030' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:23:18.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:23:18 smithi083 ceph-mon[79930]: pgmap v14250: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:18.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:23:18 smithi083 ceph-mon[77242]: pgmap v14250: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:18.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:23:18 smithi153 ceph-mon[70044]: pgmap v14250: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:20.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:23:20 smithi083 ceph-mon[79930]: pgmap v14251: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:20.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:23:20 smithi083 ceph-mon[77242]: pgmap v14251: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:20.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:23:20 smithi153 ceph-mon[70044]: pgmap v14251: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:21.937 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:23:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:23:21] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T13:23:22.215 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:23:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:23:21.934Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:23:22.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:23:22 smithi083 ceph-mon[79930]: pgmap v14252: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:22.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:23:22 smithi083 ceph-mon[77242]: pgmap v14252: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:22.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:23:22 smithi153 ceph-mon[70044]: pgmap v14252: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:22.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:23:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:23:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:23:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:23:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:23:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:23:24.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:23:24 smithi083 ceph-mon[77242]: pgmap v14253: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:24.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:23:24 smithi083 ceph-mon[79930]: pgmap v14253: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:24.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:23:24 smithi153 ceph-mon[70044]: pgmap v14253: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:25.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:23:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:23:25.297Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:23:26.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:23:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:23:26.024Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:23:27.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:23:26 smithi153 ceph-mon[70044]: pgmap v14254: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:27.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:23:26 smithi083 ceph-mon[79930]: pgmap v14254: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:27.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:23:26 smithi083 ceph-mon[77242]: pgmap v14254: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:28.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:23:28 smithi083 ceph-mon[77242]: pgmap v14255: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:28.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:23:28 smithi083 ceph-mon[79930]: pgmap v14255: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:28.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:23:28 smithi153 ceph-mon[70044]: pgmap v14255: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:30.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:23:30 smithi083 ceph-mon[77242]: pgmap v14256: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:30.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:23:30 smithi083 ceph-mon[79930]: pgmap v14256: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:30.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:23:30 smithi153 ceph-mon[70044]: pgmap v14256: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:31.937 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:23:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:23:31] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T13:23:32.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:23:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:23:31.935Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:23:32.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:23:32 smithi083 ceph-mon[77242]: pgmap v14257: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:32.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:23:32 smithi083 ceph-mon[79930]: pgmap v14257: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:32.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:23:32 smithi153 ceph-mon[70044]: pgmap v14257: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:32.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:23:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:23:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:23:33.311 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:23:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:23:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:23:34.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:23:34 smithi083 ceph-mon[77242]: pgmap v14258: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:34.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:23:34 smithi083 ceph-mon[79930]: pgmap v14258: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:34.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:23:34 smithi153 ceph-mon[70044]: pgmap v14258: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:35.715 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:23:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:23:35.298Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:23:36.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:23:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:23:36.025Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:23:37.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:23:36 smithi153 ceph-mon[70044]: pgmap v14259: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:37.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:23:36 smithi083 ceph-mon[77242]: pgmap v14259: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:37.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:23:36 smithi083 ceph-mon[79930]: pgmap v14259: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:38.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:23:38 smithi083 ceph-mon[77242]: pgmap v14260: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:38.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:23:38 smithi083 ceph-mon[79930]: pgmap v14260: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:38.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:23:38 smithi153 ceph-mon[70044]: pgmap v14260: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:23:40 smithi083 ceph-mon[79930]: pgmap v14261: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:40.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:23:40 smithi083 ceph-mon[77242]: pgmap v14261: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:23:40 smithi153 ceph-mon[70044]: pgmap v14261: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:41.938 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:23:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:23:41] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T13:23:42.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:23:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:23:41.935Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:23:42.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:23:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:23:42.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:23:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:23:42.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:23:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:23:42.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:23:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:23:42.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:23:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:23:42.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:23:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:23:42.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:23:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:23:42.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:23:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:23:42.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:23:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:23:42.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:23:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:23:42.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:23:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:23:42.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:23:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:23:42.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:23:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:23:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:23:43.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:23:42 smithi153 ceph-mon[70044]: pgmap v14262: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:43.310 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:23:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:23:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:23:43.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:23:42 smithi083 ceph-mon[79930]: pgmap v14262: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:43.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:23:42 smithi083 ceph-mon[77242]: pgmap v14262: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:44.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:23:44 smithi083 ceph-mon[79930]: pgmap v14263: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:44.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:23:44 smithi083 ceph-mon[77242]: pgmap v14263: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:44.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:23:44 smithi153 ceph-mon[70044]: pgmap v14263: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:45.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:23:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:23:45.299Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:23:46.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:23:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:23:46.026Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:23:46.833 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:23:47.127 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:23:47.128 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 8m ago 7h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:23:47.128 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 5m ago 7h 63.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:23:47.128 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 8m ago 7h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:23:47.129 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 5m ago 7h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:23:47.129 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 5m ago 8h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:23:47.129 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 8m ago 8h 903M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:23:47.129 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 8m ago 8h 1736M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:23:47.130 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 5m ago 8h 752M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:23:47.130 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 8m ago 8h 1031M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:23:47.130 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 8m ago 7h 23.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:23:47.130 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 5m ago 7h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:23:47.131 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 8m ago 8h 1256M 2330M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:23:47.131 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 8m ago 8h 1244M 2330M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:23:47.131 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 8m ago 8h 945M 2330M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:23:47.131 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 8m ago 8h 1217M 2330M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:23:47.132 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 5m ago 7h 1069M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:23:47.132 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 5m ago 7h 1339M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:23:47.132 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 5m ago 7h 1197M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:23:47.132 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 5m ago 7h 1421M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:23:47.132 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 5m ago 7h 176M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:23:47.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:23:46 smithi153 ceph-mon[70044]: pgmap v14264: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:47.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:23:46 smithi083 ceph-mon[79930]: pgmap v14264: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:47.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:23:46 smithi083 ceph-mon[77242]: pgmap v14264: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:47.483 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:23:47.484 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:23:47.484 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:23:47.484 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:23:47.485 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:23:47.485 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:23:47.485 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:23:47.485 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:23:47.486 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:23:47.486 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:23:47.486 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:23:47.486 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:23:47.487 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:23:47.487 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:23:47.487 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:23:47.488 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:23:47.488 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:23:47.488 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:23:48.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:23:47 smithi153 ceph-mon[70044]: from='client.60649 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:23:48.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:23:47 smithi153 ceph-mon[70044]: from='client.50724 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:23:48.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:23:47 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1256832473' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:23:48.470 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:23:47 smithi083 ceph-mon[79930]: from='client.60649 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:23:48.471 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:23:47 smithi083 ceph-mon[79930]: from='client.50724 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:23:48.471 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:23:47 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1256832473' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:23:48.472 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:23:47 smithi083 ceph-mon[77242]: from='client.60649 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:23:48.472 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:23:47 smithi083 ceph-mon[77242]: from='client.50724 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:23:48.472 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:23:47 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1256832473' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:23:49.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:23:48 smithi153 ceph-mon[70044]: pgmap v14265: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:49.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:23:48 smithi083 ceph-mon[79930]: pgmap v14265: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:49.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:23:48 smithi083 ceph-mon[77242]: pgmap v14265: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:50.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:23:50 smithi083 ceph-mon[79930]: pgmap v14266: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:50.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:23:50 smithi083 ceph-mon[77242]: pgmap v14266: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:50.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:23:50 smithi153 ceph-mon[70044]: pgmap v14266: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:51.938 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:23:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:23:51] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T13:23:52.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:23:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:23:51.935Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:23:52.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:23:52 smithi083 ceph-mon[77242]: pgmap v14267: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:52.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:23:52 smithi083 ceph-mon[79930]: pgmap v14267: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:52.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:23:52 smithi153 ceph-mon[70044]: pgmap v14267: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:52.810 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:23:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:23:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:23:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:23:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:23:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:23:54.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:23:54 smithi083 ceph-mon[79930]: pgmap v14268: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:54.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:23:54 smithi083 ceph-mon[77242]: pgmap v14268: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:54.814 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:23:54 smithi153 ceph-mon[70044]: pgmap v14268: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:55.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:23:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:23:55.299Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:23:56.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:23:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:23:56.027Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:23:57.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:23:56 smithi153 ceph-mon[70044]: pgmap v14269: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:57.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:23:56 smithi083 ceph-mon[79930]: pgmap v14269: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:57.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:23:56 smithi083 ceph-mon[77242]: pgmap v14269: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:58.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:23:58 smithi083 ceph-mon[79930]: pgmap v14270: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:58.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:23:58 smithi083 ceph-mon[77242]: pgmap v14270: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:58.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:23:58 smithi153 ceph-mon[70044]: pgmap v14270: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:23:59.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:23:59 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:23:59.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:23:59 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:23:59.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:23:59 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:23:59.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:23:59 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:23:59.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:23:59 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:23:59.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:23:59 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:23:59.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:23:59 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:23:59.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:23:59 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:23:59.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:23:59 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:23:59.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:23:59 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:23:59.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:23:59 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:23:59.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:23:59 smithi083 ceph-mon[79930]: Adjusting osd_memory_target on smithi083 to 2320M 2022-09-29T13:23:59.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:23:59 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:23:59.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:23:59 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:23:59.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:23:59 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:23:59.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:23:59 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:23:59.718 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:23:59 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:23:59.718 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:23:59 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:23:59.718 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:23:59 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:23:59.718 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:23:59 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:23:59.719 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:23:59 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:23:59.719 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:23:59 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:23:59.719 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:23:59 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:23:59.719 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:23:59 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:23:59.720 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:23:59 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:23:59.720 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:23:59 smithi083 ceph-mon[77242]: Adjusting osd_memory_target on smithi083 to 2320M 2022-09-29T13:23:59.720 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:23:59 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:23:59.720 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:23:59 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:23:59.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:23:59 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:23:59.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:23:59 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:23:59.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:23:59 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:23:59.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:23:59 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:23:59.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:23:59 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:23:59.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:23:59 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:23:59.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:23:59 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:23:59.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:23:59 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:23:59.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:23:59 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:23:59.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:23:59 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:23:59.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:23:59 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:23:59.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:23:59 smithi153 ceph-mon[70044]: Adjusting osd_memory_target on smithi083 to 2320M 2022-09-29T13:23:59.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:23:59 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:23:59.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:23:59 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:24:00.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:24:00 smithi153 ceph-mon[70044]: pgmap v14271: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:00.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:24:00 smithi083 ceph-mon[79930]: pgmap v14271: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:00.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:24:00 smithi083 ceph-mon[77242]: pgmap v14271: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:01.940 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:24:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:24:01] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T13:24:02.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:24:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:24:01.937Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:24:02.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:24:02 smithi083 ceph-mon[79930]: pgmap v14272: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:02.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:24:02 smithi083 ceph-mon[77242]: pgmap v14272: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:02.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:24:02 smithi153 ceph-mon[70044]: pgmap v14272: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:02.810 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:24:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:24:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:24:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:24:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:24:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:24:04.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:24:04 smithi083 ceph-mon[79930]: pgmap v14273: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:04.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:24:04 smithi083 ceph-mon[77242]: pgmap v14273: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:04.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:24:04 smithi153 ceph-mon[70044]: pgmap v14273: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:05.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:24:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:24:05.300Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:24:06.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:24:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:24:06.027Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:24:07.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:24:06 smithi153 ceph-mon[70044]: pgmap v14274: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:07.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:24:06 smithi083 ceph-mon[79930]: pgmap v14274: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:07.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:24:06 smithi083 ceph-mon[77242]: pgmap v14274: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:08.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:24:08 smithi083 ceph-mon[79930]: pgmap v14275: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:08.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:24:08 smithi083 ceph-mon[77242]: pgmap v14275: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:08.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:24:08 smithi153 ceph-mon[70044]: pgmap v14275: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:10.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:24:10 smithi083 ceph-mon[79930]: pgmap v14276: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:10.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:24:10 smithi083 ceph-mon[77242]: pgmap v14276: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:10.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:24:10 smithi153 ceph-mon[70044]: pgmap v14276: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:11.940 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:24:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:24:11] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T13:24:12.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:24:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:24:11.937Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:24:12.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:24:12 smithi083 ceph-mon[79930]: pgmap v14277: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:12.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:24:12 smithi083 ceph-mon[77242]: pgmap v14277: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:12.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:24:12 smithi153 ceph-mon[70044]: pgmap v14277: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:12.810 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:24:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:24:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:24:13.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:24:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:24:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:24:14.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:24:14 smithi083 ceph-mon[79930]: pgmap v14278: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:14.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:24:14 smithi083 ceph-mon[77242]: pgmap v14278: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:14.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:24:14 smithi153 ceph-mon[70044]: pgmap v14278: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:15.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:24:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:24:15.300Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:24:16.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:24:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:24:16.028Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:24:17.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:24:16 smithi153 ceph-mon[70044]: pgmap v14279: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:17.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:24:16 smithi083 ceph-mon[79930]: pgmap v14279: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:17.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:24:16 smithi083 ceph-mon[77242]: pgmap v14279: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:17.812 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:24:18.099 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:24:18.100 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 8m ago 7h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:24:18.100 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 6m ago 7h 63.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:24:18.100 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 8m ago 7h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:24:18.101 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 6m ago 7h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:24:18.101 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 6m ago 8h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:24:18.101 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 8m ago 8h 903M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:24:18.101 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 8m ago 8h 1736M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:24:18.101 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 6m ago 8h 752M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:24:18.102 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 8m ago 8h 1031M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:24:18.102 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 8m ago 7h 23.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:24:18.102 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 6m ago 7h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:24:18.102 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 8m ago 8h 1256M 2320M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:24:18.103 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 8m ago 8h 1244M 2320M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:24:18.103 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 8m ago 8h 945M 2320M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:24:18.103 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 8m ago 8h 1217M 2320M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:24:18.103 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 6m ago 8h 1069M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:24:18.104 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 6m ago 8h 1339M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:24:18.104 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 6m ago 7h 1197M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:24:18.104 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 6m ago 7h 1421M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:24:18.104 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 6m ago 7h 176M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:24:18.459 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:24:18.459 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:24:18.459 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:24:18.460 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:24:18.460 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:24:18.460 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:24:18.460 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:24:18.461 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:24:18.461 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:24:18.461 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:24:18.461 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:24:18.461 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:24:18.462 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:24:18.462 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:24:18.462 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:24:18.462 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:24:18.463 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:24:18.463 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:24:18.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:24:18 smithi083 ceph-mon[77242]: from='client.60667 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:24:18.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:24:18 smithi083 ceph-mon[77242]: from='client.50742 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:24:18.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:24:18 smithi083 ceph-mon[77242]: pgmap v14280: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:18.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:24:18 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2780299275' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:24:18.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:24:18 smithi083 ceph-mon[79930]: from='client.60667 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:24:18.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:24:18 smithi083 ceph-mon[79930]: from='client.50742 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:24:18.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:24:18 smithi083 ceph-mon[79930]: pgmap v14280: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:18.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:24:18 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2780299275' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:24:18.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:24:18 smithi153 ceph-mon[70044]: from='client.60667 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:24:18.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:24:18 smithi153 ceph-mon[70044]: from='client.50742 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:24:18.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:24:18 smithi153 ceph-mon[70044]: pgmap v14280: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:18.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:24:18 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2780299275' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:24:20.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:24:20 smithi083 ceph-mon[79930]: pgmap v14281: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:20.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:24:20 smithi083 ceph-mon[77242]: pgmap v14281: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:20.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:24:20 smithi153 ceph-mon[70044]: pgmap v14281: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:21.941 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:24:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:24:21] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T13:24:22.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:24:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:24:21.938Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:24:22.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:24:22 smithi083 ceph-mon[79930]: pgmap v14282: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:22.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:24:22 smithi083 ceph-mon[77242]: pgmap v14282: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:22.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:24:22 smithi153 ceph-mon[70044]: pgmap v14282: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:22.810 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:24:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:24:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:24:23.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:24:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:24:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:24:24.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:24:24 smithi083 ceph-mon[77242]: pgmap v14283: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:24.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:24:24 smithi083 ceph-mon[79930]: pgmap v14283: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:24.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:24:24 smithi153 ceph-mon[70044]: pgmap v14283: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:25.715 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:24:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:24:25.302Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:24:26.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:24:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:24:26.029Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:24:27.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:24:26 smithi153 ceph-mon[70044]: pgmap v14284: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:27.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:24:26 smithi083 ceph-mon[79930]: pgmap v14284: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:27.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:24:26 smithi083 ceph-mon[77242]: pgmap v14284: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:28.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:24:28 smithi083 ceph-mon[77242]: pgmap v14285: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:28.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:24:28 smithi083 ceph-mon[79930]: pgmap v14285: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:28.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:24:28 smithi153 ceph-mon[70044]: pgmap v14285: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:30.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:24:30 smithi083 ceph-mon[77242]: pgmap v14286: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:30.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:24:30 smithi083 ceph-mon[79930]: pgmap v14286: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:30.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:24:30 smithi153 ceph-mon[70044]: pgmap v14286: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:31.941 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:24:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:24:31] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T13:24:32.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:24:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:24:31.938Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:24:32.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:24:32 smithi083 ceph-mon[77242]: pgmap v14287: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:32.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:24:32 smithi083 ceph-mon[79930]: pgmap v14287: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:32.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:24:32 smithi153 ceph-mon[70044]: pgmap v14287: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:32.810 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:24:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:24:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:24:33.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:24:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:24:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:24:34.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:24:34 smithi083 ceph-mon[79930]: pgmap v14288: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:34.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:24:34 smithi083 ceph-mon[77242]: pgmap v14288: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:34.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:24:34 smithi153 ceph-mon[70044]: pgmap v14288: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:35.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:24:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:24:35.303Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:24:36.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:24:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:24:36.030Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:24:37.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:24:36 smithi153 ceph-mon[70044]: pgmap v14289: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:37.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:24:36 smithi083 ceph-mon[79930]: pgmap v14289: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:37.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:24:36 smithi083 ceph-mon[77242]: pgmap v14289: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:38.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:24:38 smithi083 ceph-mon[79930]: pgmap v14290: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:38.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:24:38 smithi083 ceph-mon[77242]: pgmap v14290: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:38.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:24:38 smithi153 ceph-mon[70044]: pgmap v14290: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:24:40 smithi083 ceph-mon[79930]: pgmap v14291: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:24:40 smithi083 ceph-mon[77242]: pgmap v14291: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:40.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:24:40 smithi153 ceph-mon[70044]: pgmap v14291: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:41.942 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:24:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:24:41] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T13:24:42.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:24:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:24:41.939Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:24:42.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:24:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:24:42.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:24:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:24:42.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:24:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:24:42.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:24:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:24:42.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:24:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:24:42.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:24:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:24:42.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:24:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:24:42.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:24:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:24:42.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:24:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:24:42.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:24:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:24:42.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:24:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:24:42.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:24:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:24:42.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:24:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:24:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:24:43.313 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:24:42 smithi153 ceph-mon[70044]: pgmap v14292: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:43.314 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:24:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:24:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:24:43.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:24:42 smithi083 ceph-mon[77242]: pgmap v14292: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:43.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:24:42 smithi083 ceph-mon[79930]: pgmap v14292: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:44.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:24:44 smithi153 ceph-mon[70044]: pgmap v14293: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:44.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:24:44 smithi083 ceph-mon[77242]: pgmap v14293: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:44.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:24:44 smithi083 ceph-mon[79930]: pgmap v14293: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:45.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:24:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:24:45.304Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:24:46.465 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:24:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:24:46.030Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:24:47.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:24:46 smithi153 ceph-mon[70044]: pgmap v14294: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:47.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:24:46 smithi083 ceph-mon[77242]: pgmap v14294: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:47.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:24:46 smithi083 ceph-mon[79930]: pgmap v14294: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:48.781 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:24:48.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:24:48 smithi153 ceph-mon[70044]: pgmap v14295: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:48.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:24:48 smithi083 ceph-mon[79930]: pgmap v14295: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:48.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:24:48 smithi083 ceph-mon[77242]: pgmap v14295: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:49.070 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:24:49.070 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 9m ago 7h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:24:49.071 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 6m ago 7h 63.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:24:49.071 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (7h) 9m ago 7h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:24:49.072 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (7h) 6m ago 7h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:24:49.072 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 6m ago 8h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:24:49.072 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 9m ago 8h 903M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:24:49.073 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 9m ago 8h 1736M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:24:49.073 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 6m ago 8h 752M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:24:49.073 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 9m ago 8h 1031M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:24:49.074 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 9m ago 7h 23.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:24:49.074 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 6m ago 7h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:24:49.075 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 9m ago 8h 1256M 2320M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:24:49.075 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 9m ago 8h 1244M 2320M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:24:49.076 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 9m ago 8h 945M 2320M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:24:49.076 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 9m ago 8h 1217M 2320M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:24:49.076 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 6m ago 8h 1069M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:24:49.077 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 6m ago 8h 1339M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:24:49.077 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 6m ago 8h 1197M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:24:49.079 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 6m ago 8h 1421M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:24:49.079 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 6m ago 7h 176M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:24:49.425 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:24:49.425 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:24:49.426 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:24:49.426 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:24:49.427 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:24:49.427 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:24:49.428 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:24:49.428 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:24:49.429 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:24:49.429 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:24:49.434 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:24:49.434 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:24:49.435 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:24:49.435 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:24:49.436 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:24:49.436 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:24:49.437 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:24:49.437 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:24:49.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:24:49 smithi153 ceph-mon[70044]: from='client.50754 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:24:49.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:24:49 smithi153 ceph-mon[70044]: from='client.60691 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:24:49.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:24:49 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2517151337' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:24:49.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:24:49 smithi083 ceph-mon[77242]: from='client.50754 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:24:49.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:24:49 smithi083 ceph-mon[77242]: from='client.60691 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:24:49.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:24:49 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2517151337' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:24:49.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:24:49 smithi083 ceph-mon[79930]: from='client.50754 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:24:49.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:24:49 smithi083 ceph-mon[79930]: from='client.60691 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:24:49.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:24:49 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2517151337' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:24:50.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:24:50 smithi153 ceph-mon[70044]: pgmap v14296: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:50.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:24:50 smithi083 ceph-mon[79930]: pgmap v14296: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:50.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:24:50 smithi083 ceph-mon[77242]: pgmap v14296: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:51.942 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:24:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:24:51] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T13:24:52.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:24:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:24:51.940Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:24:52.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:24:52 smithi153 ceph-mon[70044]: pgmap v14297: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:52.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:24:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:24:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:24:52.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:24:52 smithi083 ceph-mon[77242]: pgmap v14297: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:52.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:24:52 smithi083 ceph-mon[79930]: pgmap v14297: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:53.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:24:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:24:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:24:54.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:24:54 smithi153 ceph-mon[70044]: pgmap v14298: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:54.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:24:54 smithi083 ceph-mon[77242]: pgmap v14298: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:54.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:24:54 smithi083 ceph-mon[79930]: pgmap v14298: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:55.714 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:24:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:24:55.304Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:24:56.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:24:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:24:56.031Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:24:57.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:24:56 smithi153 ceph-mon[70044]: pgmap v14299: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:57.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:24:56 smithi083 ceph-mon[79930]: pgmap v14299: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:57.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:24:56 smithi083 ceph-mon[77242]: pgmap v14299: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:58.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:24:58 smithi153 ceph-mon[70044]: pgmap v14300: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:58.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:24:58 smithi083 ceph-mon[79930]: pgmap v14300: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:58.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:24:58 smithi083 ceph-mon[77242]: pgmap v14300: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:24:59.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:24:59 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:24:59.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:24:59 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:24:59.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:24:59 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:24:59.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:24:59 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:24:59.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:24:59 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:24:59.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:24:59 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:24:59.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:24:59 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:24:59.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:24:59 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:24:59.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:24:59 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:25:01.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:25:00 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:25:01.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:25:00 smithi153 ceph-mon[70044]: pgmap v14301: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:25:01.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:25:00 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:25:01.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:25:00 smithi083 ceph-mon[79930]: pgmap v14301: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:25:01.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:25:00 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:25:01.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:25:00 smithi083 ceph-mon[77242]: pgmap v14301: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:25:01.943 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:25:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:25:01] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T13:25:02.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:25:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:25:01.940Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:25:02.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:25:02 smithi153 ceph-mon[70044]: pgmap v14302: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:25:02.810 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:25:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:25:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:25:02.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:25:02 smithi083 ceph-mon[79930]: pgmap v14302: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:25:02.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:25:02 smithi083 ceph-mon[77242]: pgmap v14302: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:25:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:25:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:25:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:25:04.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:25:04 smithi153 ceph-mon[70044]: pgmap v14303: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:25:04.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:25:04 smithi083 ceph-mon[79930]: pgmap v14303: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:25:04.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:25:04 smithi083 ceph-mon[77242]: pgmap v14303: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:25:05.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:25:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:25:05.305Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:25:06.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:25:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:25:06.032Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:25:07.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:25:06 smithi153 ceph-mon[70044]: pgmap v14304: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:25:07.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:25:06 smithi083 ceph-mon[79930]: pgmap v14304: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:25:07.467 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:25:06 smithi083 ceph-mon[77242]: pgmap v14304: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:25:08.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:25:08 smithi153 ceph-mon[70044]: pgmap v14305: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:25:08.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:25:08 smithi083 ceph-mon[77242]: pgmap v14305: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:25:08.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:25:08 smithi083 ceph-mon[79930]: pgmap v14305: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:25:10.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:25:10 smithi153 ceph-mon[70044]: pgmap v14306: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:25:10.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:25:10 smithi083 ceph-mon[79930]: pgmap v14306: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:25:10.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:25:10 smithi083 ceph-mon[77242]: pgmap v14306: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:25:11.944 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:25:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:25:11] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T13:25:12.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:25:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:25:11.941Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:25:12.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:25:12 smithi153 ceph-mon[70044]: pgmap v14307: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:25:12.810 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:25:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:25:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:25:12.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:25:12 smithi083 ceph-mon[79930]: pgmap v14307: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:25:12.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:25:12 smithi083 ceph-mon[77242]: pgmap v14307: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:25:13.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:25:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:25:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:25:14.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:25:14 smithi153 ceph-mon[70044]: pgmap v14308: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:25:14.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:25:14 smithi083 ceph-mon[77242]: pgmap v14308: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:25:14.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:25:14 smithi083 ceph-mon[79930]: pgmap v14308: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:25:15.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:25:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:25:15.306Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:25:16.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:25:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:25:16.032Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:25:17.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:25:16 smithi153 ceph-mon[70044]: pgmap v14309: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:25:17.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:25:16 smithi083 ceph-mon[79930]: pgmap v14309: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:25:17.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:25:16 smithi083 ceph-mon[77242]: pgmap v14309: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:25:18.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:25:18 smithi153 ceph-mon[70044]: pgmap v14310: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:25:18.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:25:18 smithi083 ceph-mon[79930]: pgmap v14310: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:25:18.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:25:18 smithi083 ceph-mon[77242]: pgmap v14310: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:25:19.750 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:25:20.039 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:25:20.039 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 9m ago 7h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:25:20.039 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 7m ago 7h 63.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:25:20.039 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 9m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:25:20.040 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 7m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:25:20.040 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 7m ago 8h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:25:20.040 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 9m ago 8h 903M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:25:20.040 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 9m ago 8h 1736M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:25:20.041 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 7m ago 8h 752M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:25:20.041 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 9m ago 8h 1031M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:25:20.041 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (7h) 9m ago 7h 23.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:25:20.041 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (7h) 7m ago 7h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:25:20.042 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 9m ago 8h 1256M 2320M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:25:20.042 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 9m ago 8h 1244M 2320M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:25:20.042 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 9m ago 8h 945M 2320M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:25:20.042 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 9m ago 8h 1217M 2320M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:25:20.043 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 7m ago 8h 1069M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:25:20.043 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 7m ago 8h 1339M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:25:20.043 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 7m ago 8h 1197M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:25:20.043 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 7m ago 8h 1421M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:25:20.044 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 7m ago 7h 176M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:25:20.395 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:25:20.396 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:25:20.396 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:25:20.397 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:25:20.397 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:25:20.398 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:25:20.399 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:25:20.403 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:25:20.403 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:25:20.404 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:25:20.404 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:25:20.404 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:25:20.405 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:25:20.405 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:25:20.406 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:25:20.406 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:25:20.407 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:25:20.407 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:25:20.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:25:20 smithi083 ceph-mon[79930]: from='client.50772 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:25:20.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:25:20 smithi083 ceph-mon[79930]: from='client.50778 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:25:20.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:25:20 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2936528742' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:25:20.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:25:20 smithi083 ceph-mon[79930]: pgmap v14311: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:25:20.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:25:20 smithi083 ceph-mon[77242]: from='client.50772 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:25:20.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:25:20 smithi083 ceph-mon[77242]: from='client.50778 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:25:20.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:25:20 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2936528742' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:25:20.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:25:20 smithi083 ceph-mon[77242]: pgmap v14311: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:25:20.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:25:20 smithi153 ceph-mon[70044]: from='client.50772 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:25:20.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:25:20 smithi153 ceph-mon[70044]: from='client.50778 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:25:20.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:25:20 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2936528742' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:25:20.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:25:20 smithi153 ceph-mon[70044]: pgmap v14311: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:25:21.951 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:25:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:25:21] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T13:25:22.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:25:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:25:21.941Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:25:22.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:25:22 smithi153 ceph-mon[70044]: pgmap v14312: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:25:22.810 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:25:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:25:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:25:22.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:25:22 smithi083 ceph-mon[79930]: pgmap v14312: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:25:22.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:25:22 smithi083 ceph-mon[77242]: pgmap v14312: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:25:23.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:25:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:25:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:25:24.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:25:24 smithi153 ceph-mon[70044]: pgmap v14313: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:25:24.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:25:24 smithi083 ceph-mon[77242]: pgmap v14313: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:25:24.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:25:24 smithi083 ceph-mon[79930]: pgmap v14313: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:25:25.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:25:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:25:25.307Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:25:26.470 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:25:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:25:26.033Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:25:27.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:25:26 smithi153 ceph-mon[70044]: pgmap v14314: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:25:27.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:25:26 smithi083 ceph-mon[79930]: pgmap v14314: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:25:27.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:25:26 smithi083 ceph-mon[77242]: pgmap v14314: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:25:29.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:25:29 smithi153 ceph-mon[70044]: pgmap v14315: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:25:29.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:25:29 smithi083 ceph-mon[77242]: pgmap v14315: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:25:29.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:25:29 smithi083 ceph-mon[79930]: pgmap v14315: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:25:31.944 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:25:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:25:31] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T13:25:32.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:25:31 smithi083 ceph-mon[77242]: pgmap v14316: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:25:32.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:25:31 smithi083 ceph-mon[79930]: pgmap v14316: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:25:32.214 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:25:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:25:31.942Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:25:32.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:25:31 smithi153 ceph-mon[70044]: pgmap v14316: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:25:32.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:25:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:25:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:25:33.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:25:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:25:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:25:34.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:25:33 smithi153 ceph-mon[70044]: pgmap v14317: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:25:34.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:25:33 smithi083 ceph-mon[77242]: pgmap v14317: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:25:34.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:25:33 smithi083 ceph-mon[79930]: pgmap v14317: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:25:35.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:25:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:25:35.307Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:25:36.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:25:35 smithi153 ceph-mon[70044]: pgmap v14318: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:25:36.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:25:35 smithi083 ceph-mon[77242]: pgmap v14318: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:25:36.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:25:35 smithi083 ceph-mon[79930]: pgmap v14318: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:25:36.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:25:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:25:36.033Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:25:38.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:25:37 smithi153 ceph-mon[70044]: pgmap v14319: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:25:38.467 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:25:37 smithi083 ceph-mon[77242]: pgmap v14319: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:25:38.468 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:25:37 smithi083 ceph-mon[79930]: pgmap v14319: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:25:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:25:39 smithi153 ceph-mon[70044]: pgmap v14320: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:25:40.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:25:39 smithi083 ceph-mon[77242]: pgmap v14320: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:25:40.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:25:39 smithi083 ceph-mon[79930]: pgmap v14320: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:25:41.945 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:25:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:25:41] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T13:25:42.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:25:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:25:41.942Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:25:42.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:25:42 smithi083 ceph-mon[79930]: pgmap v14321: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:25:42.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:25:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:25:42.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:25:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:25:42.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:25:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:25:42.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:25:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:25:42.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:25:42 smithi083 ceph-mon[77242]: pgmap v14321: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:25:42.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:25:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:25:42.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:25:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:25:42.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:25:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:25:42.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:25:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:25:42.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:25:42 smithi153 ceph-mon[70044]: pgmap v14321: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:25:42.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:25:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:25:42.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:25:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:25:42.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:25:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:25:42.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:25:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:25:42.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:25:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:25:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:25:43.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:25:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:25:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:25:44.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:25:44 smithi153 ceph-mon[70044]: pgmap v14322: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:25:44.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:25:44 smithi083 ceph-mon[79930]: pgmap v14322: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:25:44.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:25:44 smithi083 ceph-mon[77242]: pgmap v14322: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:25:45.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:25:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:25:45.307Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:25:46.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:25:46 smithi153 ceph-mon[70044]: pgmap v14323: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:25:46.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:25:46 smithi083 ceph-mon[77242]: pgmap v14323: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:25:46.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:25:46 smithi083 ceph-mon[79930]: pgmap v14323: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:25:46.465 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:25:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:25:46.034Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:25:48.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:25:48 smithi153 ceph-mon[70044]: pgmap v14324: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:25:48.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:25:48 smithi083 ceph-mon[79930]: pgmap v14324: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:25:48.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:25:48 smithi083 ceph-mon[77242]: pgmap v14324: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:25:50.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:25:50 smithi153 ceph-mon[70044]: pgmap v14325: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:25:50.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:25:50 smithi083 ceph-mon[79930]: pgmap v14325: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:25:50.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:25:50 smithi083 ceph-mon[77242]: pgmap v14325: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:25:50.724 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:25:51.018 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:25:51.019 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7h) 10m ago 8h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:25:51.019 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (7h) 7m ago 7h 63.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:25:51.019 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 10m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:25:51.019 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 7m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:25:51.020 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 7m ago 8h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:25:51.020 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 10m ago 8h 903M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:25:51.020 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 10m ago 8h 1736M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:25:51.020 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 7m ago 8h 752M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:25:51.020 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 10m ago 8h 1031M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:25:51.021 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 10m ago 8h 23.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:25:51.022 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 7m ago 8h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:25:51.022 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 10m ago 8h 1256M 2320M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:25:51.022 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 10m ago 8h 1244M 2320M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:25:51.022 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 10m ago 8h 945M 2320M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:25:51.022 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 10m ago 8h 1217M 2320M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:25:51.023 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 7m ago 8h 1069M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:25:51.023 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 7m ago 8h 1339M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:25:51.023 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 7m ago 8h 1197M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:25:51.023 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 7m ago 8h 1421M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:25:51.024 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 7m ago 8h 176M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:25:51.371 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:25:51.371 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:25:51.372 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:25:51.372 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:25:51.372 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:25:51.372 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:25:51.372 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:25:51.373 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:25:51.373 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:25:51.373 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:25:51.373 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:25:51.374 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:25:51.374 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:25:51.374 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:25:51.375 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:25:51.375 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:25:51.375 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:25:51.375 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:25:51.945 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:25:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:25:51] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T13:25:52.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:25:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:25:51.942Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:25:52.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:25:52 smithi083 ceph-mon[79930]: pgmap v14326: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:25:52.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:25:52 smithi083 ceph-mon[79930]: from='client.50790 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:25:52.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:25:52 smithi083 ceph-mon[79930]: from='client.50796 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:25:52.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:25:52 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/552872057' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:25:52.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:25:52 smithi083 ceph-mon[77242]: pgmap v14326: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:25:52.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:25:52 smithi083 ceph-mon[77242]: from='client.50790 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:25:52.218 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:25:52 smithi083 ceph-mon[77242]: from='client.50796 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:25:52.218 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:25:52 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/552872057' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:25:52.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:25:52 smithi153 ceph-mon[70044]: pgmap v14326: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:25:52.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:25:52 smithi153 ceph-mon[70044]: from='client.50790 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:25:52.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:25:52 smithi153 ceph-mon[70044]: from='client.50796 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:25:52.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:25:52 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/552872057' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:25:52.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:25:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:25:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:25:53.313 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:25:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:25:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:25:54.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:25:54 smithi153 ceph-mon[70044]: pgmap v14327: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:25:54.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:25:54 smithi083 ceph-mon[79930]: pgmap v14327: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:25:54.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:25:54 smithi083 ceph-mon[77242]: pgmap v14327: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:25:55.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:25:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:25:55.307Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:25:56.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:25:56 smithi153 ceph-mon[70044]: pgmap v14328: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:25:56.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:25:56 smithi083 ceph-mon[79930]: pgmap v14328: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:25:56.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:25:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:25:56.034Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:25:56.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:25:56 smithi083 ceph-mon[77242]: pgmap v14328: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:25:58.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:25:58 smithi153 ceph-mon[70044]: pgmap v14329: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:25:58.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:25:58 smithi083 ceph-mon[79930]: pgmap v14329: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:25:58.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:25:58 smithi083 ceph-mon[77242]: pgmap v14329: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:00.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:26:00 smithi153 ceph-mon[70044]: pgmap v14330: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:00.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:26:00 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:26:00.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:26:00 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:26:00.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:26:00 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:26:00.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:26:00 smithi083 ceph-mon[79930]: pgmap v14330: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:00.467 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:26:00 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:26:00.467 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:26:00 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:26:00.468 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:26:00 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:26:00.469 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:26:00 smithi083 ceph-mon[77242]: pgmap v14330: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:00.469 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:26:00 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:26:00.470 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:26:00 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:26:00.470 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:26:00 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:26:01.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:26:01 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:26:01.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:26:01 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:26:01.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:26:01 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:26:01.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:26:01 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:26:01.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:26:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:26:01] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T13:26:01.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:26:01 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:26:01.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:26:01 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:26:01.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:26:01 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:26:01.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:26:01 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:26:01.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:26:01 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:26:01.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:26:01 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:26:01.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:26:01 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:26:01.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:26:01 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:26:02.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:26:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:26:01.943Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:26:02.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:26:02 smithi153 ceph-mon[70044]: pgmap v14331: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:02.559 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:26:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:26:02.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:26:02 smithi083 ceph-mon[79930]: pgmap v14331: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:02.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:26:02 smithi083 ceph-mon[77242]: pgmap v14331: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:26:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:26:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:26:04.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:26:04 smithi083 ceph-mon[79930]: pgmap v14332: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:04.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:26:04 smithi083 ceph-mon[77242]: pgmap v14332: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:04.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:26:04 smithi153 ceph-mon[70044]: pgmap v14332: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:05.572 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:26:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:26:05.307Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:26:05.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:26:05 smithi083 ceph-mon[77242]: pgmap v14333: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:05.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:26:05 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:26:05.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:26:05 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:26:05.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:26:05 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:26:05.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:26:05 smithi083 ceph-mon[79930]: pgmap v14333: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:05.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:26:05 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:26:05.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:26:05 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:26:05.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:26:05 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:26:06.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:26:05 smithi153 ceph-mon[70044]: pgmap v14333: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:06.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:26:05 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:26:06.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:26:05 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:26:06.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:26:05 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:26:06.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:26:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:26:06.034Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:26:08.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:26:07 smithi153 ceph-mon[70044]: pgmap v14334: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:08.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:26:07 smithi083 ceph-mon[79930]: pgmap v14334: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:08.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:26:07 smithi083 ceph-mon[77242]: pgmap v14334: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:10.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:26:09 smithi153 ceph-mon[70044]: pgmap v14335: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:10.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:26:09 smithi083 ceph-mon[79930]: pgmap v14335: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:10.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:26:09 smithi083 ceph-mon[77242]: pgmap v14335: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:11.946 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:26:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:26:11] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T13:26:12.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:26:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:26:11.943Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:26:12.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:26:12 smithi083 ceph-mon[77242]: pgmap v14336: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:12.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:26:12 smithi083 ceph-mon[79930]: pgmap v14336: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:12.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:26:12 smithi153 ceph-mon[70044]: pgmap v14336: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:12.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:26:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:26:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:26:13.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:26:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:26:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:26:14.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:26:14 smithi153 ceph-mon[70044]: pgmap v14337: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:14.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:26:14 smithi083 ceph-mon[79930]: pgmap v14337: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:14.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:26:14 smithi083 ceph-mon[77242]: pgmap v14337: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:15.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:26:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:26:15.309Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:26:16.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:26:16 smithi153 ceph-mon[70044]: pgmap v14338: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:16.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:26:16 smithi083 ceph-mon[79930]: pgmap v14338: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:16.466 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:26:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:26:16.036Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:26:16.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:26:16 smithi083 ceph-mon[77242]: pgmap v14338: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:18.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:26:18 smithi153 ceph-mon[70044]: pgmap v14339: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:18.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:26:18 smithi083 ceph-mon[79930]: pgmap v14339: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:18.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:26:18 smithi083 ceph-mon[77242]: pgmap v14339: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:20.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:26:20 smithi153 ceph-mon[70044]: pgmap v14340: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:20.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:26:20 smithi083 ceph-mon[79930]: pgmap v14340: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:20.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:26:20 smithi083 ceph-mon[77242]: pgmap v14340: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:21.699 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:26:21.946 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:26:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:26:21] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T13:26:21.989 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:26:21.989 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 17s ago 8h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:26:21.991 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 8m ago 8h 63.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:26:21.991 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 17s ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:26:21.991 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 8m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:26:21.991 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 8m ago 8h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:26:21.992 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 17s ago 8h 911M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:26:21.992 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 17s ago 8h 1772M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:26:21.992 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 8m ago 8h 752M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:26:21.992 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 17s ago 8h 1055M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:26:21.993 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 17s ago 8h 23.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:26:21.993 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 8m ago 8h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:26:21.993 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 17s ago 8h 1282M 2320M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:26:21.993 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 17s ago 8h 1269M 2320M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:26:21.993 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 17s ago 8h 965M 2320M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:26:21.994 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 17s ago 8h 1243M 2320M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:26:21.994 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 8m ago 8h 1069M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:26:21.994 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 8m ago 8h 1339M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:26:21.994 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 8m ago 8h 1197M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:26:21.995 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 8m ago 8h 1421M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:26:21.995 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 8m ago 8h 176M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:26:22.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:26:22 smithi083 ceph-mon[79930]: pgmap v14341: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:22.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:26:22 smithi083 ceph-mon[77242]: pgmap v14341: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:22.215 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:26:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:26:21.944Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:26:22.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:26:22 smithi153 ceph-mon[70044]: pgmap v14341: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:22.347 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:26:22.348 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:26:22.348 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:26:22.348 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:26:22.348 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:26:22.349 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:26:22.349 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:26:22.349 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:26:22.349 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:26:22.350 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:26:22.350 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:26:22.350 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:26:22.350 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:26:22.351 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:26:22.351 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:26:22.351 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:26:22.351 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:26:22.352 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:26:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:26:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:26:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:26:23.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:26:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:26:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:26:23.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:26:23 smithi153 ceph-mon[70044]: from='client.50808 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:26:23.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:26:23 smithi153 ceph-mon[70044]: from='client.50814 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:26:23.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:26:23 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2006522348' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:26:23.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:26:23 smithi083 ceph-mon[79930]: from='client.50808 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:26:23.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:26:23 smithi083 ceph-mon[79930]: from='client.50814 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:26:23.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:26:23 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2006522348' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:26:23.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:26:23 smithi083 ceph-mon[77242]: from='client.50808 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:26:23.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:26:23 smithi083 ceph-mon[77242]: from='client.50814 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:26:23.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:26:23 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2006522348' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:26:24.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:26:24 smithi153 ceph-mon[70044]: pgmap v14342: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:24.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:26:24 smithi083 ceph-mon[79930]: pgmap v14342: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:24.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:26:24 smithi083 ceph-mon[77242]: pgmap v14342: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:25.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:26:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:26:25.309Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:26:26.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:26:26 smithi153 ceph-mon[70044]: pgmap v14343: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:26.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:26:26 smithi083 ceph-mon[79930]: pgmap v14343: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:26.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:26:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:26:26.037Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:26:26.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:26:26 smithi083 ceph-mon[77242]: pgmap v14343: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:28.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:26:28 smithi153 ceph-mon[70044]: pgmap v14344: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:28.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:26:28 smithi083 ceph-mon[79930]: pgmap v14344: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:28.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:26:28 smithi083 ceph-mon[77242]: pgmap v14344: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:30.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:26:30 smithi153 ceph-mon[70044]: pgmap v14345: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:30.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:26:30 smithi083 ceph-mon[79930]: pgmap v14345: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:30.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:26:30 smithi083 ceph-mon[77242]: pgmap v14345: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:31.948 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:26:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:26:31] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T13:26:32.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:26:32 smithi083 ceph-mon[79930]: pgmap v14346: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:32.214 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:26:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:26:31.945Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:26:32.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:26:32 smithi083 ceph-mon[77242]: pgmap v14346: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:32.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:26:32 smithi153 ceph-mon[70044]: pgmap v14346: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:32.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:26:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:26:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:26:33.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:26:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:26:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:26:34.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:26:34 smithi153 ceph-mon[70044]: pgmap v14347: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:34.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:26:34 smithi083 ceph-mon[79930]: pgmap v14347: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:34.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:26:34 smithi083 ceph-mon[77242]: pgmap v14347: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:35.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:26:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:26:35.310Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:26:36.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:26:36 smithi153 ceph-mon[70044]: pgmap v14348: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:36.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:26:36 smithi083 ceph-mon[79930]: pgmap v14348: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:36.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:26:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:26:36.037Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:26:36.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:26:36 smithi083 ceph-mon[77242]: pgmap v14348: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:38.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:26:38 smithi153 ceph-mon[70044]: pgmap v14349: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:38.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:26:38 smithi083 ceph-mon[79930]: pgmap v14349: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:38.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:26:38 smithi083 ceph-mon[77242]: pgmap v14349: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:26:40 smithi153 ceph-mon[70044]: pgmap v14350: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:26:40 smithi083 ceph-mon[79930]: pgmap v14350: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:40.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:26:40 smithi083 ceph-mon[77242]: pgmap v14350: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:41.948 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:26:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:26:41] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T13:26:42.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:26:42 smithi083 ceph-mon[79930]: pgmap v14351: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:42.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:26:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:26:42.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:26:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:26:42.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:26:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:26:42.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:26:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:26:42.216 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:26:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:26:41.945Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:26:42.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:26:42 smithi083 ceph-mon[77242]: pgmap v14351: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:42.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:26:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:26:42.218 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:26:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:26:42.218 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:26:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:26:42.219 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:26:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:26:42.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:26:42 smithi153 ceph-mon[70044]: pgmap v14351: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:42.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:26:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:26:42.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:26:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:26:42.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:26:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:26:42.312 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:26:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:26:42.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:26:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:26:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:26:43.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:26:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:26:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:26:44.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:26:44 smithi153 ceph-mon[70044]: pgmap v14352: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:44.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:26:44 smithi083 ceph-mon[79930]: pgmap v14352: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:44.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:26:44 smithi083 ceph-mon[77242]: pgmap v14352: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:45.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:26:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:26:45.310Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:26:46.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:26:46 smithi153 ceph-mon[70044]: pgmap v14353: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:46.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:26:46 smithi083 ceph-mon[79930]: pgmap v14353: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:46.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:26:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:26:46.037Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:26:46.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:26:46 smithi083 ceph-mon[77242]: pgmap v14353: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:48.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:26:48 smithi153 ceph-mon[70044]: pgmap v14354: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:48.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:26:48 smithi083 ceph-mon[79930]: pgmap v14354: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:48.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:26:48 smithi083 ceph-mon[77242]: pgmap v14354: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:50.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:26:50 smithi153 ceph-mon[70044]: pgmap v14355: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:50.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:26:50 smithi083 ceph-mon[79930]: pgmap v14355: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:50.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:26:50 smithi083 ceph-mon[77242]: pgmap v14355: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:51.948 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:26:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:26:51] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T13:26:52.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:26:52 smithi083 ceph-mon[79930]: pgmap v14356: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:52.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:26:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:26:51.946Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:26:52.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:26:52 smithi083 ceph-mon[77242]: pgmap v14356: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:52.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:26:52 smithi153 ceph-mon[70044]: pgmap v14356: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:52.674 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:26:52.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:26:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:26:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:26:52.963 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:26:52.964 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 48s ago 8h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:26:52.964 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 9m ago 8h 63.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:26:52.965 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 48s ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:26:52.965 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 9m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:26:52.966 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 9m ago 8h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:26:52.966 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 48s ago 8h 911M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:26:52.967 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 48s ago 8h 1772M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:26:52.967 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 9m ago 8h 752M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:26:52.968 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 48s ago 8h 1055M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:26:52.968 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 48s ago 8h 23.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:26:52.968 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 9m ago 8h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:26:52.969 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 48s ago 8h 1282M 2320M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:26:52.969 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 48s ago 8h 1269M 2320M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:26:52.969 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 48s ago 8h 965M 2320M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:26:52.969 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 48s ago 8h 1243M 2320M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:26:52.970 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 9m ago 8h 1069M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:26:52.970 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 9m ago 8h 1339M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:26:52.970 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 9m ago 8h 1197M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:26:52.971 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 9m ago 8h 1421M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:26:52.971 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 9m ago 8h 176M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:26:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:26:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:26:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:26:53.322 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:26:53.323 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:26:53.323 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:26:53.323 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:26:53.323 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:26:53.324 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:26:53.324 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:26:53.324 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:26:53.324 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:26:53.325 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:26:53.325 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:26:53.325 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:26:53.325 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:26:53.325 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:26:53.326 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:26:53.326 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:26:53.326 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:26:53.326 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:26:54.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:26:54 smithi153 ceph-mon[70044]: pgmap v14357: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:54.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:26:54 smithi153 ceph-mon[70044]: from='client.50826 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:26:54.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:26:54 smithi153 ceph-mon[70044]: from='client.60763 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:26:54.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:26:54 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/474053153' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:26:54.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:26:54 smithi083 ceph-mon[79930]: pgmap v14357: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:54.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:26:54 smithi083 ceph-mon[79930]: from='client.50826 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:26:54.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:26:54 smithi083 ceph-mon[79930]: from='client.60763 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:26:54.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:26:54 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/474053153' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:26:54.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:26:54 smithi083 ceph-mon[77242]: pgmap v14357: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:54.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:26:54 smithi083 ceph-mon[77242]: from='client.50826 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:26:54.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:26:54 smithi083 ceph-mon[77242]: from='client.60763 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:26:54.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:26:54 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/474053153' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:26:55.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:26:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:26:55.311Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:26:56.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:26:56 smithi153 ceph-mon[70044]: pgmap v14358: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:56.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:26:56 smithi083 ceph-mon[79930]: pgmap v14358: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:56.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:26:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:26:56.038Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:26:56.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:26:56 smithi083 ceph-mon[77242]: pgmap v14358: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:58.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:26:58 smithi153 ceph-mon[70044]: pgmap v14359: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:58.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:26:58 smithi083 ceph-mon[77242]: pgmap v14359: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:26:58.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:26:58 smithi083 ceph-mon[79930]: pgmap v14359: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:00.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:27:00 smithi153 ceph-mon[70044]: pgmap v14360: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:00.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:27:00 smithi083 ceph-mon[79930]: pgmap v14360: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:00.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:27:00 smithi083 ceph-mon[77242]: pgmap v14360: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:01.949 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:27:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:27:01] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T13:27:02.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:27:02 smithi083 ceph-mon[79930]: pgmap v14361: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:02.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:27:02 smithi083 ceph-mon[77242]: pgmap v14361: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:02.214 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:27:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:27:01.946Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:27:02.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:27:02 smithi153 ceph-mon[70044]: pgmap v14361: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:27:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:27:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:27:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:27:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:27:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:27:04.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:27:04 smithi153 ceph-mon[70044]: pgmap v14362: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:04.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:27:04 smithi083 ceph-mon[77242]: pgmap v14362: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:04.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:27:04 smithi083 ceph-mon[79930]: pgmap v14362: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:05.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:27:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:27:05.312Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:27:06.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:27:06 smithi153 ceph-mon[70044]: pgmap v14363: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:06.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:27:06 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:27:06.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:27:06 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:27:06.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:27:06 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:27:06.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:27:06 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:27:06.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:27:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:27:06.039Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:27:06.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:27:06 smithi083 ceph-mon[77242]: pgmap v14363: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:06.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:27:06 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:27:06.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:27:06 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:27:06.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:27:06 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:27:06.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:27:06 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:27:06.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:27:06 smithi083 ceph-mon[79930]: pgmap v14363: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:06.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:27:06 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:27:06.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:27:06 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:27:06.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:27:06 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:27:06.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:27:06 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:27:08.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:27:08 smithi153 ceph-mon[70044]: pgmap v14364: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:08.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:27:08 smithi083 ceph-mon[79930]: pgmap v14364: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:08.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:27:08 smithi083 ceph-mon[77242]: pgmap v14364: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:10.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:27:10 smithi153 ceph-mon[70044]: pgmap v14365: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:10.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:27:10 smithi083 ceph-mon[79930]: pgmap v14365: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:10.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:27:10 smithi083 ceph-mon[77242]: pgmap v14365: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:11.949 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:27:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:27:11] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T13:27:12.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:27:12 smithi083 ceph-mon[79930]: pgmap v14366: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:12.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:27:12 smithi083 ceph-mon[77242]: pgmap v14366: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:12.215 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:27:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:27:11.947Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:27:12.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:27:12 smithi153 ceph-mon[70044]: pgmap v14366: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:12.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:27:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:27:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:27:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:27:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:27:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:27:14.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:27:14 smithi153 ceph-mon[70044]: pgmap v14367: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:14.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:27:14 smithi083 ceph-mon[79930]: pgmap v14367: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:14.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:27:14 smithi083 ceph-mon[77242]: pgmap v14367: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:15.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:27:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:27:15.313Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:27:16.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:27:16 smithi083 ceph-mon[79930]: pgmap v14368: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:16.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:27:16 smithi083 ceph-mon[77242]: pgmap v14368: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:16.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:27:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:27:16.040Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:27:16.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:27:16 smithi153 ceph-mon[70044]: pgmap v14368: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:18.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:27:18 smithi083 ceph-mon[79930]: pgmap v14369: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:18.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:27:18 smithi083 ceph-mon[77242]: pgmap v14369: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:18.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:27:18 smithi153 ceph-mon[70044]: pgmap v14369: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:20.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:27:20 smithi083 ceph-mon[79930]: pgmap v14370: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:20.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:27:20 smithi083 ceph-mon[77242]: pgmap v14370: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:20.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:27:20 smithi153 ceph-mon[70044]: pgmap v14370: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:21.949 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:27:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:27:21] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T13:27:22.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:27:22 smithi083 ceph-mon[77242]: pgmap v14371: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:22.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:27:22 smithi083 ceph-mon[79930]: pgmap v14371: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:22.214 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:27:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:27:21.947Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:27:22.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:27:22 smithi153 ceph-mon[70044]: pgmap v14371: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:22.559 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:27:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:27:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:27:23.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:27:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:27:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:27:23.661 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:27:23.951 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:27:23.952 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 79s ago 8h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:27:23.952 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 9m ago 8h 63.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:27:23.952 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 79s ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:27:23.953 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 9m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:27:23.953 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 9m ago 8h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:27:23.953 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 79s ago 8h 911M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:27:23.953 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 79s ago 8h 1772M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:27:23.953 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 9m ago 8h 752M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:27:23.954 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 79s ago 8h 1055M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:27:23.954 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 79s ago 8h 23.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:27:23.954 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 9m ago 8h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:27:23.954 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 79s ago 8h 1282M 2320M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:27:23.955 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 79s ago 8h 1269M 2320M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:27:23.955 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 79s ago 8h 965M 2320M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:27:23.955 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 79s ago 8h 1243M 2320M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:27:23.955 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 9m ago 8h 1069M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:27:23.956 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 9m ago 8h 1339M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:27:23.956 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 9m ago 8h 1197M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:27:23.956 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 9m ago 8h 1421M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:27:23.956 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 9m ago 8h 176M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:27:24.309 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:27:24.309 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:27:24.310 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:27:24.310 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:27:24.311 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:27:24.311 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:27:24.311 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:27:24.311 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:27:24.312 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:27:24.312 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:27:24.312 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:27:24.312 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:27:24.312 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:27:24.313 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:27:24.313 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:27:24.313 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:27:24.313 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:27:24.314 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:27:24.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:27:24 smithi083 ceph-mon[79930]: pgmap v14372: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:24.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:27:24 smithi083 ceph-mon[77242]: pgmap v14372: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:24.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:27:24 smithi153 ceph-mon[70044]: pgmap v14372: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:25.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:27:25 smithi083 ceph-mon[79930]: from='client.50844 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:27:25.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:27:25 smithi083 ceph-mon[79930]: from='client.60781 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:27:25.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:27:25 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1539050045' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:27:25.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:27:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:27:25.313Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:27:25.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:27:25 smithi083 ceph-mon[77242]: from='client.50844 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:27:25.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:27:25 smithi083 ceph-mon[77242]: from='client.60781 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:27:25.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:27:25 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1539050045' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:27:25.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:27:25 smithi153 ceph-mon[70044]: from='client.50844 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:27:25.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:27:25 smithi153 ceph-mon[70044]: from='client.60781 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:27:25.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:27:25 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1539050045' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:27:26.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:27:26 smithi083 ceph-mon[79930]: pgmap v14373: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:26.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:27:26 smithi083 ceph-mon[77242]: pgmap v14373: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:26.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:27:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:27:26.041Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:27:26.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:27:26 smithi153 ceph-mon[70044]: pgmap v14373: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:28.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:27:28 smithi083 ceph-mon[79930]: pgmap v14374: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:28.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:27:28 smithi083 ceph-mon[77242]: pgmap v14374: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:28.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:27:28 smithi153 ceph-mon[70044]: pgmap v14374: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:30.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:27:30 smithi083 ceph-mon[79930]: pgmap v14375: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:30.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:27:30 smithi083 ceph-mon[77242]: pgmap v14375: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:30.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:27:30 smithi153 ceph-mon[70044]: pgmap v14375: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:31.951 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:27:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:27:31] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T13:27:32.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:27:32 smithi083 ceph-mon[79930]: pgmap v14376: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:32.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:27:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:27:31.948Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:27:32.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:27:32 smithi083 ceph-mon[77242]: pgmap v14376: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:32.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:27:32 smithi153 ceph-mon[70044]: pgmap v14376: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:32.559 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:27:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:27:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:27:33.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:27:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:27:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:27:34.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:27:34 smithi083 ceph-mon[79930]: pgmap v14377: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:34.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:27:34 smithi083 ceph-mon[77242]: pgmap v14377: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:34.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:27:34 smithi153 ceph-mon[70044]: pgmap v14377: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:35.717 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:27:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:27:35.314Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:27:36.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:27:36 smithi083 ceph-mon[79930]: pgmap v14378: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:36.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:27:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:27:36.041Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:27:36.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:27:36 smithi083 ceph-mon[77242]: pgmap v14378: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:36.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:27:36 smithi153 ceph-mon[70044]: pgmap v14378: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:38.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:27:38 smithi083 ceph-mon[79930]: pgmap v14379: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:38.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:27:38 smithi083 ceph-mon[77242]: pgmap v14379: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:38.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:27:38 smithi153 ceph-mon[70044]: pgmap v14379: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:27:40 smithi083 ceph-mon[79930]: pgmap v14380: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:27:40 smithi083 ceph-mon[77242]: pgmap v14380: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:40.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:27:40 smithi153 ceph-mon[70044]: pgmap v14380: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:41.951 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:27:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:27:41] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T13:27:42.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:27:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:27:41.948Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:27:42.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:27:42 smithi083 ceph-mon[77242]: pgmap v14381: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:42.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:27:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:27:42.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:27:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:27:42.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:27:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:27:42.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:27:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:27:42.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:27:42 smithi083 ceph-mon[79930]: pgmap v14381: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:42.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:27:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:27:42.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:27:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:27:42.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:27:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:27:42.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:27:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:27:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:27:42 smithi153 ceph-mon[70044]: pgmap v14381: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:42.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:27:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:27:42.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:27:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:27:42.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:27:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:27:42.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:27:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:27:42.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:27:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:27:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:27:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:27:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:27:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:27:44.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:27:44 smithi153 ceph-mon[70044]: pgmap v14382: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:44.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:27:44 smithi083 ceph-mon[79930]: pgmap v14382: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:44.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:27:44 smithi083 ceph-mon[77242]: pgmap v14382: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:45.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:27:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:27:45.315Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:27:46.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:27:46 smithi153 ceph-mon[70044]: pgmap v14383: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:46.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:27:46 smithi083 ceph-mon[79930]: pgmap v14383: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:46.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:27:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:27:46.041Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:27:46.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:27:46 smithi083 ceph-mon[77242]: pgmap v14383: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:48.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:27:48 smithi153 ceph-mon[70044]: pgmap v14384: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:48.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:27:48 smithi083 ceph-mon[79930]: pgmap v14384: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:48.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:27:48 smithi083 ceph-mon[77242]: pgmap v14384: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:50.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:27:50 smithi153 ceph-mon[70044]: pgmap v14385: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:50.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:27:50 smithi083 ceph-mon[79930]: pgmap v14385: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:50.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:27:50 smithi083 ceph-mon[77242]: pgmap v14385: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:51.952 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:27:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:27:51] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T13:27:52.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:27:52 smithi083 ceph-mon[77242]: pgmap v14386: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:52.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:27:52 smithi083 ceph-mon[79930]: pgmap v14386: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:52.215 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:27:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:27:51.949Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:27:52.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:27:52 smithi153 ceph-mon[70044]: pgmap v14386: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:52.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:27:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:27:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:27:53.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:27:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:27:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:27:54.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:27:54 smithi153 ceph-mon[70044]: pgmap v14387: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:54.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:27:54 smithi083 ceph-mon[79930]: pgmap v14387: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:54.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:27:54 smithi083 ceph-mon[77242]: pgmap v14387: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:54.642 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:27:54.929 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:27:54.929 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 110s ago 8h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:27:54.929 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 10m ago 8h 63.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:27:54.930 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 110s ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:27:54.930 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 10m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:27:54.930 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 10m ago 8h 447M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:27:54.930 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 110s ago 8h 911M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:27:54.931 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 110s ago 8h 1772M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:27:54.931 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 10m ago 8h 752M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:27:54.931 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 110s ago 8h 1055M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:27:54.931 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 110s ago 8h 23.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:27:54.932 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 10m ago 8h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:27:54.932 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 110s ago 8h 1282M 2320M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:27:54.932 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 110s ago 8h 1269M 2320M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:27:54.932 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 110s ago 8h 965M 2320M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:27:54.933 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 110s ago 8h 1243M 2320M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:27:54.933 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 10m ago 8h 1069M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:27:54.933 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 10m ago 8h 1339M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:27:54.933 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 10m ago 8h 1197M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:27:54.934 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 10m ago 8h 1421M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:27:54.934 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 10m ago 8h 176M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:27:55.281 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:27:55.282 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:27:55.282 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:27:55.282 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:27:55.283 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:27:55.283 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:27:55.283 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:27:55.284 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:27:55.284 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:27:55.285 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:27:55.285 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:27:55.286 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:27:55.286 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:27:55.287 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:27:55.287 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:27:55.287 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:27:55.288 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:27:55.288 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:27:55.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:27:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:27:55.314Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:27:56.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:27:56 smithi153 ceph-mon[70044]: pgmap v14388: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:56.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:27:56 smithi153 ceph-mon[70044]: from='client.60793 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:27:56.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:27:56 smithi153 ceph-mon[70044]: from='client.50868 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:27:56.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:27:56 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2039075107' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:27:56.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:27:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:27:56.041Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:27:56.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:27:56 smithi083 ceph-mon[79930]: pgmap v14388: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:56.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:27:56 smithi083 ceph-mon[79930]: from='client.60793 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:27:56.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:27:56 smithi083 ceph-mon[79930]: from='client.50868 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:27:56.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:27:56 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2039075107' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:27:56.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:27:56 smithi083 ceph-mon[77242]: pgmap v14388: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:56.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:27:56 smithi083 ceph-mon[77242]: from='client.60793 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:27:56.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:27:56 smithi083 ceph-mon[77242]: from='client.50868 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:27:56.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:27:56 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2039075107' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:27:58.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:27:58 smithi153 ceph-mon[70044]: pgmap v14389: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:58.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:27:58 smithi083 ceph-mon[79930]: pgmap v14389: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:27:58.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:27:58 smithi083 ceph-mon[77242]: pgmap v14389: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:00.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:28:00 smithi153 ceph-mon[70044]: pgmap v14390: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:00.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:28:00 smithi083 ceph-mon[79930]: pgmap v14390: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:00.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:28:00 smithi083 ceph-mon[77242]: pgmap v14390: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:01.953 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:28:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:28:01] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T13:28:02.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:28:02 smithi083 ceph-mon[79930]: pgmap v14391: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:02.214 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:28:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:28:01.950Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:28:02.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:28:02 smithi083 ceph-mon[77242]: pgmap v14391: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:02.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:28:02 smithi153 ceph-mon[70044]: pgmap v14391: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:02.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:28:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:28:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:28:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:28:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:28:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:28:04.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:28:04 smithi153 ceph-mon[70044]: pgmap v14392: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:04.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:28:04 smithi083 ceph-mon[79930]: pgmap v14392: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:04.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:28:04 smithi083 ceph-mon[77242]: pgmap v14392: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:05.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:28:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:28:05.315Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:28:06.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:28:06 smithi153 ceph-mon[70044]: pgmap v14393: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:06.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:28:06 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:28:06.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:28:06 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:28:06.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:28:06 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:28:06.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:28:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:28:06.042Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:28:06.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:28:06 smithi083 ceph-mon[79930]: pgmap v14393: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:06.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:28:06 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:28:06.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:28:06 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:28:06.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:28:06 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:28:06.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:28:06 smithi083 ceph-mon[77242]: pgmap v14393: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:06.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:28:06 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:28:06.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:28:06 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:28:06.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:28:06 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:28:08.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:28:08 smithi153 ceph-mon[70044]: pgmap v14394: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:08.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:28:08 smithi083 ceph-mon[79930]: pgmap v14394: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:08.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:28:08 smithi083 ceph-mon[77242]: pgmap v14394: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:10.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:28:10 smithi153 ceph-mon[70044]: pgmap v14395: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:10.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:28:10 smithi083 ceph-mon[79930]: pgmap v14395: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:10.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:28:10 smithi083 ceph-mon[77242]: pgmap v14395: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:11.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:28:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:28:11] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T13:28:11.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:28:11 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:28:11.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:28:11 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:28:11.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:28:11 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:28:11.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:28:11 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:28:11.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:28:11 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:28:11.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:28:11 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:28:11.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:28:11 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:28:11.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:28:11 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:28:11.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:28:11 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:28:12.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:28:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:28:11.951Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:28:12.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:28:12 smithi083 ceph-mon[79930]: pgmap v14396: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:12.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:28:12 smithi083 ceph-mon[77242]: pgmap v14396: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:12.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:28:12 smithi153 ceph-mon[70044]: pgmap v14396: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:12.810 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:28:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:28:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:28:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:28:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:28:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:28:14.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:28:14 smithi083 ceph-mon[79930]: pgmap v14397: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:14.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:28:14 smithi083 ceph-mon[77242]: pgmap v14397: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:14.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:28:14 smithi153 ceph-mon[70044]: pgmap v14397: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:15.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:28:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:28:15.316Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:28:16.440 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:28:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:28:16.043Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:28:16.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:28:16 smithi083 ceph-mon[79930]: pgmap v14398: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:16.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:28:16 smithi083 ceph-mon[77242]: pgmap v14398: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:16.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:28:16 smithi153 ceph-mon[70044]: pgmap v14398: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:18.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:28:18 smithi083 ceph-mon[77242]: pgmap v14399: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:18.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:28:18 smithi083 ceph-mon[79930]: pgmap v14399: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:18.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:28:18 smithi153 ceph-mon[70044]: pgmap v14399: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:20.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:28:20 smithi083 ceph-mon[79930]: pgmap v14400: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:20.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:28:20 smithi083 ceph-mon[77242]: pgmap v14400: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:20.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:28:20 smithi153 ceph-mon[70044]: pgmap v14400: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:21.955 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:28:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:28:21] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T13:28:22.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:28:22 smithi083 ceph-mon[79930]: pgmap v14401: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:22.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:28:22 smithi083 ceph-mon[77242]: pgmap v14401: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:22.214 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:28:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:28:21.952Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:28:22.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:28:22 smithi153 ceph-mon[70044]: pgmap v14401: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:22.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:28:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:28:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:28:23.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:28:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:28:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:28:24.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:28:24 smithi153 ceph-mon[70044]: pgmap v14402: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:24.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:28:24 smithi083 ceph-mon[79930]: pgmap v14402: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:24.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:28:24 smithi083 ceph-mon[77242]: pgmap v14402: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:25.606 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:28:25.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:28:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:28:25.316Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:28:25.895 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:28:25.895 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 2m ago 8h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:28:25.895 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 15s ago 8h 63.7M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:28:25.896 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 2m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:28:25.896 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 15s ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:28:25.896 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 15s ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:28:25.896 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (7h) 2m ago 8h 911M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:28:25.897 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 2m ago 8h 1772M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:28:25.897 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 15s ago 8h 766M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:28:25.897 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 2m ago 8h 1055M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:28:25.897 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 2m ago 8h 23.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:28:25.898 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 15s ago 8h 25.7M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:28:25.898 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 2m ago 8h 1282M 2320M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:28:25.898 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 2m ago 8h 1269M 2320M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:28:25.898 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 2m ago 8h 965M 2320M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:28:25.899 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 2m ago 8h 1243M 2320M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:28:25.899 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 15s ago 8h 1091M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:28:25.899 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 15s ago 8h 1368M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:28:25.899 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 15s ago 8h 1222M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:28:25.900 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 15s ago 8h 1452M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:28:25.900 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 15s ago 8h 182M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:28:26.256 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:28:26.256 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:28:26.257 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:28:26.257 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:28:26.257 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:28:26.258 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:28:26.258 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:28:26.259 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:28:26.259 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:28:26.259 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:28:26.260 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:28:26.260 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:28:26.261 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:28:26.261 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:28:26.262 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:28:26.262 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:28:26.262 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:28:26.263 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:28:26.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:28:26 smithi153 ceph-mon[70044]: pgmap v14403: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:26.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:28:26 smithi083 ceph-mon[79930]: pgmap v14403: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:26.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:28:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:28:26.043Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:28:26.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:28:26 smithi083 ceph-mon[77242]: pgmap v14403: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:27.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:28:27 smithi153 ceph-mon[70044]: from='client.60811 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:28:27.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:28:27 smithi153 ceph-mon[70044]: from='client.60817 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:28:27.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:28:27 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3572777957' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:28:27.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:28:27 smithi083 ceph-mon[79930]: from='client.60811 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:28:27.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:28:27 smithi083 ceph-mon[79930]: from='client.60817 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:28:27.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:28:27 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3572777957' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:28:27.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:28:27 smithi083 ceph-mon[77242]: from='client.60811 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:28:27.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:28:27 smithi083 ceph-mon[77242]: from='client.60817 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:28:27.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:28:27 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3572777957' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:28:28.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:28:28 smithi153 ceph-mon[70044]: pgmap v14404: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:28.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:28:28 smithi083 ceph-mon[79930]: pgmap v14404: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:28.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:28:28 smithi083 ceph-mon[77242]: pgmap v14404: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:30.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:28:30 smithi153 ceph-mon[70044]: pgmap v14405: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:30.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:28:30 smithi083 ceph-mon[79930]: pgmap v14405: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:30.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:28:30 smithi083 ceph-mon[77242]: pgmap v14405: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:31.955 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:28:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:28:31] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T13:28:32.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:28:32 smithi083 ceph-mon[77242]: pgmap v14406: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:32.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:28:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:28:31.952Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:28:32.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:28:32 smithi083 ceph-mon[79930]: pgmap v14406: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:32.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:28:32 smithi153 ceph-mon[70044]: pgmap v14406: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:32.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:28:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:28:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:28:33.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:28:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:28:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:28:34.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:28:34 smithi153 ceph-mon[70044]: pgmap v14407: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:34.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:28:34 smithi083 ceph-mon[77242]: pgmap v14407: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:34.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:28:34 smithi083 ceph-mon[79930]: pgmap v14407: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:35.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:28:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:28:35.317Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:28:36.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:28:36 smithi153 ceph-mon[70044]: pgmap v14408: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:36.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:28:36 smithi083 ceph-mon[79930]: pgmap v14408: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:36.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:28:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:28:36.044Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:28:36.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:28:36 smithi083 ceph-mon[77242]: pgmap v14408: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:38.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:28:38 smithi153 ceph-mon[70044]: pgmap v14409: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:38.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:28:38 smithi083 ceph-mon[79930]: pgmap v14409: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:38.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:28:38 smithi083 ceph-mon[77242]: pgmap v14409: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:28:40 smithi083 ceph-mon[79930]: pgmap v14410: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:28:40 smithi083 ceph-mon[77242]: pgmap v14410: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:28:40 smithi153 ceph-mon[70044]: pgmap v14410: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:41.956 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:28:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:28:41] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T13:28:42.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:28:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:28:41.954Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:28:42.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:28:42 smithi083 ceph-mon[77242]: pgmap v14411: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:42.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:28:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:28:42.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:28:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:28:42.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:28:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:28:42.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:28:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:28:42.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:28:42 smithi083 ceph-mon[79930]: pgmap v14411: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:42.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:28:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:28:42.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:28:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:28:42.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:28:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:28:42.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:28:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:28:42.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:28:42 smithi153 ceph-mon[70044]: pgmap v14411: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:42.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:28:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:28:42.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:28:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:28:42.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:28:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:28:42.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:28:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:28:42.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:28:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:28:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:28:43.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:28:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:28:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:28:44.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:28:44 smithi153 ceph-mon[70044]: pgmap v14412: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:44.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:28:44 smithi083 ceph-mon[79930]: pgmap v14412: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:44.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:28:44 smithi083 ceph-mon[77242]: pgmap v14412: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:45.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:28:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:28:45.317Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:28:46.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:28:46 smithi153 ceph-mon[70044]: pgmap v14413: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:46.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:28:46 smithi083 ceph-mon[79930]: pgmap v14413: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:46.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:28:46 smithi083 ceph-mon[77242]: pgmap v14413: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:46.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:28:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:28:46.045Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:28:48.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:28:48 smithi153 ceph-mon[70044]: pgmap v14414: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:48.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:28:48 smithi083 ceph-mon[79930]: pgmap v14414: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:48.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:28:48 smithi083 ceph-mon[77242]: pgmap v14414: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:50.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:28:50 smithi153 ceph-mon[70044]: pgmap v14415: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:50.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:28:50 smithi083 ceph-mon[79930]: pgmap v14415: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:50.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:28:50 smithi083 ceph-mon[77242]: pgmap v14415: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:51.956 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:28:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:28:51] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T13:28:52.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:28:52 smithi083 ceph-mon[77242]: pgmap v14416: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:52.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:28:52 smithi083 ceph-mon[79930]: pgmap v14416: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:52.214 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:28:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:28:51.953Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:28:52.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:28:52 smithi153 ceph-mon[70044]: pgmap v14416: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:28:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:28:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:28:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:28:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:28:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:28:54.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:28:54 smithi153 ceph-mon[70044]: pgmap v14417: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:54.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:28:54 smithi083 ceph-mon[79930]: pgmap v14417: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:54.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:28:54 smithi083 ceph-mon[77242]: pgmap v14417: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:55.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:28:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:28:55.319Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:28:56.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:28:56 smithi153 ceph-mon[70044]: pgmap v14418: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:56.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:28:56 smithi083 ceph-mon[77242]: pgmap v14418: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:56.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:28:56 smithi083 ceph-mon[79930]: pgmap v14418: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:56.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:28:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:28:56.045Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:28:56.582 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:28:56.870 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:28:56.870 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 2m ago 8h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:28:56.871 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 46s ago 8h 63.7M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:28:56.871 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 2m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:28:56.872 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 46s ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:28:56.872 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (7h) 46s ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:28:56.872 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 2m ago 8h 911M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:28:56.872 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 2m ago 8h 1772M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:28:56.873 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 46s ago 8h 766M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:28:56.873 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 2m ago 8h 1055M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:28:56.873 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 2m ago 8h 23.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:28:56.874 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 46s ago 8h 25.7M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:28:56.874 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 2m ago 8h 1282M 2320M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:28:56.874 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 2m ago 8h 1269M 2320M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:28:56.875 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 2m ago 8h 965M 2320M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:28:56.875 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 2m ago 8h 1243M 2320M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:28:56.875 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 46s ago 8h 1091M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:28:56.876 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 46s ago 8h 1368M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:28:56.876 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 46s ago 8h 1222M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:28:56.876 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 46s ago 8h 1452M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:28:56.876 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (7h) 46s ago 8h 182M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:28:57.225 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:28:57.226 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:28:57.226 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:28:57.226 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:28:57.227 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:28:57.227 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:28:57.227 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:28:57.227 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:28:57.228 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:28:57.228 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:28:57.228 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:28:57.228 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:28:57.228 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:28:57.229 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:28:57.229 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:28:57.229 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:28:57.229 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:28:57.230 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:28:58.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:28:58 smithi083 ceph-mon[79930]: pgmap v14419: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:58.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:28:58 smithi083 ceph-mon[79930]: from='client.60829 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:28:58.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:28:58 smithi083 ceph-mon[79930]: from='client.50904 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:28:58.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:28:58 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3353399989' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:28:58.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:28:58 smithi083 ceph-mon[77242]: pgmap v14419: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:58.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:28:58 smithi083 ceph-mon[77242]: from='client.60829 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:28:58.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:28:58 smithi083 ceph-mon[77242]: from='client.50904 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:28:58.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:28:58 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3353399989' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:28:58.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:28:58 smithi153 ceph-mon[70044]: pgmap v14419: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:28:58.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:28:58 smithi153 ceph-mon[70044]: from='client.60829 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:28:58.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:28:58 smithi153 ceph-mon[70044]: from='client.50904 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:28:58.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:28:58 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3353399989' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:29:00.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:29:00 smithi083 ceph-mon[79930]: pgmap v14420: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:00.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:29:00 smithi083 ceph-mon[77242]: pgmap v14420: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:00.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:29:00 smithi153 ceph-mon[70044]: pgmap v14420: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:01.957 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:29:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:29:01] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T13:29:02.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:29:02 smithi083 ceph-mon[79930]: pgmap v14421: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:02.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:29:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:29:01.954Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:29:02.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:29:02 smithi083 ceph-mon[77242]: pgmap v14421: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:02.318 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:29:02 smithi153 ceph-mon[70044]: pgmap v14421: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:29:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:29:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:29:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:29:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:29:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:29:04.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:29:04 smithi083 ceph-mon[79930]: pgmap v14422: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:04.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:29:04 smithi083 ceph-mon[77242]: pgmap v14422: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:04.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:29:04 smithi153 ceph-mon[70044]: pgmap v14422: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:05.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:29:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:29:05.320Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:29:06.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:29:06 smithi083 ceph-mon[79930]: pgmap v14423: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:06.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:29:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:29:06.047Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:29:06.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:29:06 smithi083 ceph-mon[77242]: pgmap v14423: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:06.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:29:06 smithi153 ceph-mon[70044]: pgmap v14423: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:08.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:29:08 smithi083 ceph-mon[79930]: pgmap v14424: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:08.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:29:08 smithi083 ceph-mon[77242]: pgmap v14424: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:08.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:29:08 smithi153 ceph-mon[70044]: pgmap v14424: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:10.469 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:29:10 smithi083 ceph-mon[77242]: pgmap v14425: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:10.470 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:29:10 smithi083 ceph-mon[79930]: pgmap v14425: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:10.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:29:10 smithi153 ceph-mon[70044]: pgmap v14425: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:11.957 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:29:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:29:11] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T13:29:12.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:29:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:29:11.954Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:29:12.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:29:12 smithi083 ceph-mon[77242]: pgmap v14426: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:12.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:29:12 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:29:12.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:29:12 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:29:12.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:29:12 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:29:12.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:29:12 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:29:12.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:29:12 smithi083 ceph-mon[79930]: pgmap v14426: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:12.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:29:12 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:29:12.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:29:12 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:29:12.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:29:12 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:29:12.218 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:29:12 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:29:12.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:29:12 smithi153 ceph-mon[70044]: pgmap v14426: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:12.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:29:12 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:29:12.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:29:12 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:29:12.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:29:12 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:29:12.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:29:12 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:29:12.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:29:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:29:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:29:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:29:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:29:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:29:14.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:29:14 smithi153 ceph-mon[70044]: pgmap v14427: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:14.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:29:14 smithi083 ceph-mon[79930]: pgmap v14427: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:14.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:29:14 smithi083 ceph-mon[77242]: pgmap v14427: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:15.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:29:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:29:15.320Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:29:16.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:29:16 smithi153 ceph-mon[70044]: pgmap v14428: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:16.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:29:16 smithi083 ceph-mon[79930]: pgmap v14428: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:16.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:29:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:29:16.048Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:29:16.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:29:16 smithi083 ceph-mon[77242]: pgmap v14428: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:18.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:29:18 smithi153 ceph-mon[70044]: pgmap v14429: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:18.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:29:18 smithi083 ceph-mon[79930]: pgmap v14429: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:18.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:29:18 smithi083 ceph-mon[77242]: pgmap v14429: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:20.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:29:20 smithi153 ceph-mon[70044]: pgmap v14430: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:20.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:29:20 smithi083 ceph-mon[79930]: pgmap v14430: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:20.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:29:20 smithi083 ceph-mon[77242]: pgmap v14430: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:21.958 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:29:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:29:21] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T13:29:22.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:29:22 smithi083 ceph-mon[77242]: pgmap v14431: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:22.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:29:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:29:21.955Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:29:22.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:29:22 smithi083 ceph-mon[79930]: pgmap v14431: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:22.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:29:22 smithi153 ceph-mon[70044]: pgmap v14431: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:22.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:29:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:29:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:29:23.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:29:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:29:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:29:24.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:29:24 smithi083 ceph-mon[79930]: pgmap v14432: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:24.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:29:24 smithi083 ceph-mon[77242]: pgmap v14432: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:24.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:29:24 smithi153 ceph-mon[70044]: pgmap v14432: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:25.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:29:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:29:25.321Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:29:26.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:29:26 smithi083 ceph-mon[77242]: pgmap v14433: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:26.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:29:26 smithi083 ceph-mon[79930]: pgmap v14433: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:26.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:29:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:29:26.047Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:29:26.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:29:26 smithi153 ceph-mon[70044]: pgmap v14433: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:27.555 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:29:27.842 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:29:27.842 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 3m ago 8h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:29:27.843 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 77s ago 8h 63.7M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:29:27.843 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 3m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:29:27.843 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 77s ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:29:27.843 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 77s ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:29:27.844 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 3m ago 8h 911M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:29:27.844 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (7h) 3m ago 8h 1772M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:29:27.844 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 77s ago 8h 766M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:29:27.844 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (7h) 3m ago 8h 1055M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:29:27.845 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 3m ago 8h 23.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:29:27.845 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 77s ago 8h 25.7M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:29:27.845 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 3m ago 8h 1282M 2320M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:29:27.845 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 3m ago 8h 1269M 2320M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:29:27.845 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 3m ago 8h 965M 2320M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:29:27.846 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 3m ago 8h 1243M 2320M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:29:27.846 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 77s ago 8h 1091M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:29:27.846 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 77s ago 8h 1368M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:29:27.846 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 77s ago 8h 1222M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:29:27.847 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 77s ago 8h 1452M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:29:27.847 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 77s ago 8h 182M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:29:28.198 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:29:28.198 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:29:28.198 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:29:28.199 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:29:28.199 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:29:28.199 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:29:28.200 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:29:28.200 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:29:28.200 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:29:28.200 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:29:28.201 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:29:28.201 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:29:28.201 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:29:28.202 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:29:28.202 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:29:28.202 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:29:28.203 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:29:28.203 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:29:28.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:29:28 smithi083 ceph-mon[79930]: pgmap v14434: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:28.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:29:28 smithi083 ceph-mon[77242]: pgmap v14434: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:28.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:29:28 smithi153 ceph-mon[70044]: pgmap v14434: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:29.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:29:29 smithi083 ceph-mon[79930]: from='client.50916 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:29:29.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:29:29 smithi083 ceph-mon[79930]: from='client.50922 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:29:29.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:29:29 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/594272560' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:29:29.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:29:29 smithi083 ceph-mon[77242]: from='client.50916 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:29:29.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:29:29 smithi083 ceph-mon[77242]: from='client.50922 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:29:29.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:29:29 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/594272560' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:29:29.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:29:29 smithi153 ceph-mon[70044]: from='client.50916 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:29:29.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:29:29 smithi153 ceph-mon[70044]: from='client.50922 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:29:29.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:29:29 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/594272560' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:29:30.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:29:30 smithi083 ceph-mon[77242]: pgmap v14435: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:30.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:29:30 smithi083 ceph-mon[79930]: pgmap v14435: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:30.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:29:30 smithi153 ceph-mon[70044]: pgmap v14435: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:31.958 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:29:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:29:31] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T13:29:32.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:29:32 smithi083 ceph-mon[79930]: pgmap v14436: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:32.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:29:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:29:31.956Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:29:32.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:29:32 smithi083 ceph-mon[77242]: pgmap v14436: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:32.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:29:32 smithi153 ceph-mon[70044]: pgmap v14436: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:32.559 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:29:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:29:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:29:33.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:29:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:29:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:29:34.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:29:34 smithi083 ceph-mon[79930]: pgmap v14437: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:34.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:29:34 smithi083 ceph-mon[77242]: pgmap v14437: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:34.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:29:34 smithi153 ceph-mon[70044]: pgmap v14437: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:35.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:29:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:29:35.321Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:29:36.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:29:36 smithi083 ceph-mon[77242]: pgmap v14438: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:36.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:29:36 smithi083 ceph-mon[79930]: pgmap v14438: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:36.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:29:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:29:36.047Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:29:36.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:29:36 smithi153 ceph-mon[70044]: pgmap v14438: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:38.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:29:38 smithi083 ceph-mon[77242]: pgmap v14439: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:38.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:29:38 smithi083 ceph-mon[79930]: pgmap v14439: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:38.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:29:38 smithi153 ceph-mon[70044]: pgmap v14439: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:29:40 smithi083 ceph-mon[79930]: pgmap v14440: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:29:40 smithi083 ceph-mon[77242]: pgmap v14440: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:40.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:29:40 smithi153 ceph-mon[70044]: pgmap v14440: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:41.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:29:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:29:41.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:29:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:29:41.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:29:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:29:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:29:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:29:41.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:29:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:29:41.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:29:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:29:41.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:29:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:29:41.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:29:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:29:41.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:29:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:29:41.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:29:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:29:41.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:29:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:29:41.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:29:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:29:41.959 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:29:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:29:41] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T13:29:42.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:29:42 smithi083 ceph-mon[79930]: pgmap v14441: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:42.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:29:42 smithi083 ceph-mon[77242]: pgmap v14441: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:42.214 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:29:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:29:41.956Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:29:42.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:29:42 smithi153 ceph-mon[70044]: pgmap v14441: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:42.559 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:29:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:29:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:29:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:29:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:29:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:29:44.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:29:44 smithi083 ceph-mon[77242]: pgmap v14442: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:44.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:29:44 smithi083 ceph-mon[79930]: pgmap v14442: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:44.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:29:44 smithi153 ceph-mon[70044]: pgmap v14442: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:45.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:29:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:29:45.322Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:29:46.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:29:46 smithi083 ceph-mon[77242]: pgmap v14443: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:46.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:29:46 smithi083 ceph-mon[79930]: pgmap v14443: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:46.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:29:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:29:46.048Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:29:46.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:29:46 smithi153 ceph-mon[70044]: pgmap v14443: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:48.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:29:48 smithi083 ceph-mon[79930]: pgmap v14444: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:48.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:29:48 smithi083 ceph-mon[77242]: pgmap v14444: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:48.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:29:48 smithi153 ceph-mon[70044]: pgmap v14444: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:50.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:29:50 smithi083 ceph-mon[79930]: pgmap v14445: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:50.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:29:50 smithi083 ceph-mon[77242]: pgmap v14445: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:50.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:29:50 smithi153 ceph-mon[70044]: pgmap v14445: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:51.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:29:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:29:51] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T13:29:52.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:29:52 smithi083 ceph-mon[79930]: pgmap v14446: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:52.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:29:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:29:51.957Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:29:52.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:29:52 smithi083 ceph-mon[77242]: pgmap v14446: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:52.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:29:52 smithi153 ceph-mon[70044]: pgmap v14446: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:52.559 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:29:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:29:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:29:53.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:29:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:29:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:29:54.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:29:54 smithi083 ceph-mon[79930]: pgmap v14447: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:54.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:29:54 smithi083 ceph-mon[77242]: pgmap v14447: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:54.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:29:54 smithi153 ceph-mon[70044]: pgmap v14447: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:55.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:29:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:29:55.322Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:29:56.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:29:56 smithi083 ceph-mon[79930]: pgmap v14448: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:56.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:29:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:29:56.049Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:29:56.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:29:56 smithi083 ceph-mon[77242]: pgmap v14448: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:56.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:29:56 smithi153 ceph-mon[70044]: pgmap v14448: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:58.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:29:58 smithi083 ceph-mon[79930]: pgmap v14449: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:58.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:29:58 smithi083 ceph-mon[77242]: pgmap v14449: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:58.524 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:29:58.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:29:58 smithi153 ceph-mon[70044]: pgmap v14449: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:29:58.812 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:29:58.812 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 3m ago 8h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:29:58.812 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 108s ago 8h 63.7M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:29:58.813 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 3m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:29:58.813 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 108s ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:29:58.813 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 108s ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:29:58.813 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 3m ago 8h 911M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:29:58.814 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 3m ago 8h 1772M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:29:58.814 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (7h) 108s ago 8h 766M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:29:58.814 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 3m ago 8h 1055M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:29:58.814 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 3m ago 8h 23.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:29:58.815 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 108s ago 8h 25.7M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:29:58.815 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 3m ago 8h 1282M 2320M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:29:58.815 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 3m ago 8h 1269M 2320M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:29:58.815 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 3m ago 8h 965M 2320M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:29:58.816 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 3m ago 8h 1243M 2320M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:29:58.816 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 108s ago 8h 1091M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:29:58.816 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 108s ago 8h 1368M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:29:58.816 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 108s ago 8h 1222M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:29:58.816 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 108s ago 8h 1452M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:29:58.817 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 108s ago 8h 182M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:29:59.172 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:29:59.172 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:29:59.172 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:29:59.173 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:29:59.173 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:29:59.173 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:29:59.173 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:29:59.174 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:29:59.174 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:29:59.174 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:29:59.174 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:29:59.175 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:29:59.175 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:29:59.175 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:29:59.175 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:29:59.175 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:29:59.176 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:29:59.176 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:30:00.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:30:00 smithi083 ceph-mon[79930]: pgmap v14450: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:30:00.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:30:00 smithi083 ceph-mon[79930]: from='client.50934 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:30:00.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:30:00 smithi083 ceph-mon[79930]: from='client.50940 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:30:00.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:30:00 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3117458250' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:30:00.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:30:00 smithi083 ceph-mon[79930]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T13:30:00.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:30:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[77219]: 2022-09-29T13:29:59.999+0000 7f160f665700 -1 log_channel(cluster) log [ERR] : overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T13:30:00.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:30:00 smithi083 ceph-mon[77242]: pgmap v14450: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:30:00.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:30:00 smithi083 ceph-mon[77242]: from='client.50934 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:30:00.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:30:00 smithi083 ceph-mon[77242]: from='client.50940 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:30:00.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:30:00 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3117458250' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:30:00.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:30:00 smithi083 ceph-mon[77242]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T13:30:00.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:30:00 smithi153 ceph-mon[70044]: pgmap v14450: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:30:00.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:30:00 smithi153 ceph-mon[70044]: from='client.50934 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:30:00.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:30:00 smithi153 ceph-mon[70044]: from='client.50940 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:30:00.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:30:00 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3117458250' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:30:00.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:30:00 smithi153 ceph-mon[70044]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T13:30:01.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:30:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:30:01] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T13:30:02.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:30:02 smithi083 ceph-mon[79930]: pgmap v14451: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:30:02.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:30:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:30:01.957Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:30:02.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:30:02 smithi083 ceph-mon[77242]: pgmap v14451: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:30:02.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:30:02 smithi153 ceph-mon[70044]: pgmap v14451: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:30:02.559 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:30:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:30:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:30:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:30:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:30:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:30:04.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:30:04 smithi083 ceph-mon[79930]: pgmap v14452: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:30:04.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:30:04 smithi083 ceph-mon[77242]: pgmap v14452: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:30:04.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:30:04 smithi153 ceph-mon[70044]: pgmap v14452: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:30:05.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:30:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:30:05.323Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:30:06.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:30:06 smithi083 ceph-mon[79930]: pgmap v14453: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:30:06.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:30:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:30:06.050Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:30:06.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:30:06 smithi083 ceph-mon[77242]: pgmap v14453: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:30:06.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:30:06 smithi153 ceph-mon[70044]: pgmap v14453: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:30:08.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:30:08 smithi083 ceph-mon[77242]: pgmap v14454: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:30:08.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:30:08 smithi083 ceph-mon[79930]: pgmap v14454: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:30:08.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:30:08 smithi153 ceph-mon[70044]: pgmap v14454: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:30:10.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:30:10 smithi083 ceph-mon[77242]: pgmap v14455: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:30:10.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:30:10 smithi083 ceph-mon[79930]: pgmap v14455: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:30:10.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:30:10 smithi153 ceph-mon[70044]: pgmap v14455: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:30:11.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:30:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:30:11] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T13:30:12.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:30:12 smithi083 ceph-mon[79930]: pgmap v14456: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:30:12.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:30:12 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:30:12.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:30:12 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:30:12.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:30:12 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:30:12.214 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:30:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:30:11.957Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:30:12.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:30:12 smithi083 ceph-mon[77242]: pgmap v14456: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:30:12.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:30:12 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:30:12.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:30:12 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:30:12.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:30:12 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:30:12.559 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:30:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:30:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:30:12.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:30:12 smithi153 ceph-mon[70044]: pgmap v14456: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:30:12.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:30:12 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:30:12.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:30:12 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:30:12.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:30:12 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:30:13.284 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:30:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:30:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:30:13.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:30:13 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:30:13.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:30:13 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:30:13.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:30:13 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:30:14.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:30:14 smithi153 ceph-mon[70044]: pgmap v14457: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:30:14.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:30:14 smithi083 ceph-mon[79930]: pgmap v14457: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:30:14.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:30:14 smithi083 ceph-mon[77242]: pgmap v14457: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:30:15.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:30:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:30:15.323Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:30:16.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:30:16 smithi083 ceph-mon[79930]: pgmap v14458: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:30:16.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:30:16 smithi083 ceph-mon[77242]: pgmap v14458: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:30:16.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:30:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:30:16.050Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:30:16.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:30:16 smithi153 ceph-mon[70044]: pgmap v14458: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:30:18.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:30:18 smithi153 ceph-mon[70044]: pgmap v14459: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:30:18.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:30:18 smithi083 ceph-mon[79930]: pgmap v14459: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:30:18.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:30:18 smithi083 ceph-mon[77242]: pgmap v14459: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:30:20.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:30:20 smithi153 ceph-mon[70044]: pgmap v14460: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:30:20.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:30:20 smithi083 ceph-mon[79930]: pgmap v14460: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:30:20.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:30:20 smithi083 ceph-mon[77242]: pgmap v14460: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:30:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:30:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:30:21] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T13:30:22.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:30:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:30:21.958Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:30:22.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:30:22 smithi153 ceph-mon[70044]: pgmap v14461: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:30:22.559 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:30:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:30:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:30:22.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:30:22 smithi083 ceph-mon[79930]: pgmap v14461: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:30:22.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:30:22 smithi083 ceph-mon[77242]: pgmap v14461: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:30:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:30:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:30:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:30:24.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:30:24 smithi153 ceph-mon[70044]: pgmap v14462: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:30:24.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:30:24 smithi083 ceph-mon[77242]: pgmap v14462: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:30:24.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:30:24 smithi083 ceph-mon[79930]: pgmap v14462: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:30:25.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:30:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:30:25.323Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:30:26.304 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:30:26 smithi083 ceph-mon[77242]: pgmap v14463: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:30:26.304 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:30:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:30:26.051Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:30:26.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:30:26 smithi153 ceph-mon[70044]: pgmap v14463: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:30:26.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:30:26 smithi083 ceph-mon[79930]: pgmap v14463: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:30:28.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:30:28 smithi153 ceph-mon[70044]: pgmap v14464: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:30:28.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:30:28 smithi083 ceph-mon[79930]: pgmap v14464: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:30:28.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:30:28 smithi083 ceph-mon[77242]: pgmap v14464: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:30:29.498 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:30:29.787 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:30:29.787 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 4m ago 8h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:30:29.787 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 2m ago 8h 63.7M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:30:29.787 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 4m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:30:29.788 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 2m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:30:29.788 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 2m ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:30:29.788 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 4m ago 8h 911M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:30:29.788 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 4m ago 8h 1772M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:30:29.789 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 2m ago 8h 766M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:30:29.789 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 4m ago 8h 1055M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:30:29.789 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 4m ago 8h 23.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:30:29.789 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 2m ago 8h 25.7M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:30:29.790 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7h) 4m ago 8h 1282M 2320M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:30:29.790 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 4m ago 8h 1269M 2320M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:30:29.790 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 4m ago 8h 965M 2320M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:30:29.790 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 4m ago 8h 1243M 2320M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:30:29.791 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 2m ago 8h 1091M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:30:29.791 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 2m ago 8h 1368M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:30:29.791 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 2m ago 8h 1222M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:30:29.791 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 2m ago 8h 1452M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:30:29.792 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 2m ago 8h 182M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:30:30.145 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:30:30.145 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:30:30.146 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:30:30.146 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:30:30.146 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:30:30.146 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:30:30.147 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:30:30.147 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:30:30.147 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:30:30.147 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:30:30.148 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:30:30.148 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:30:30.148 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:30:30.148 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:30:30.148 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:30:30.149 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:30:30.149 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:30:30.149 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:30:30.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:30:30 smithi083 ceph-mon[79930]: pgmap v14465: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:30:30.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:30:30 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2922397739' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:30:30.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:30:30 smithi083 ceph-mon[77242]: pgmap v14465: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:30:30.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:30:30 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2922397739' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:30:30.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:30:30 smithi153 ceph-mon[70044]: pgmap v14465: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:30:30.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:30:30 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2922397739' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:30:31.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:30:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:30:31] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T13:30:31.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:30:31 smithi083 ceph-mon[79930]: from='client.50952 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:30:31.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:30:31 smithi083 ceph-mon[79930]: from='client.50958 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:30:31.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:30:31 smithi083 ceph-mon[77242]: from='client.50952 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:30:31.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:30:31 smithi083 ceph-mon[77242]: from='client.50958 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:30:31.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:30:31 smithi153 ceph-mon[70044]: from='client.50952 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:30:31.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:30:31 smithi153 ceph-mon[70044]: from='client.50958 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:30:32.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:30:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:30:31.959Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:30:32.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:30:32 smithi083 ceph-mon[77242]: pgmap v14466: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:30:32.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:30:32 smithi083 ceph-mon[79930]: pgmap v14466: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:30:32.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:30:32 smithi153 ceph-mon[70044]: pgmap v14466: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:30:32.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:30:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:30:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:30:33.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:30:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:30:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:30:34.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:30:34 smithi083 ceph-mon[79930]: pgmap v14467: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:30:34.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:30:34 smithi083 ceph-mon[77242]: pgmap v14467: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:30:34.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:30:34 smithi153 ceph-mon[70044]: pgmap v14467: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:30:35.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:30:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:30:35.324Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:30:36.327 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:30:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:30:36.052Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:30:36.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:30:36 smithi083 ceph-mon[79930]: pgmap v14468: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:30:36.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:30:36 smithi083 ceph-mon[77242]: pgmap v14468: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:30:36.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:30:36 smithi153 ceph-mon[70044]: pgmap v14468: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:30:38.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:30:38 smithi083 ceph-mon[79930]: pgmap v14469: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:30:38.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:30:38 smithi083 ceph-mon[77242]: pgmap v14469: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:30:38.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:30:38 smithi153 ceph-mon[70044]: pgmap v14469: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:30:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:30:40 smithi083 ceph-mon[79930]: pgmap v14470: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:30:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:30:40 smithi083 ceph-mon[77242]: pgmap v14470: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:30:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:30:40 smithi153 ceph-mon[70044]: pgmap v14470: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:30:41.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:30:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:30:41] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T13:30:41.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:30:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:30:41.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:30:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:30:41.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:30:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:30:41.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:30:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:30:41.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:30:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:30:41.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:30:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:30:41.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:30:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:30:41.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:30:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:30:41.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:30:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:30:41.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:30:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:30:41.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:30:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:30:41.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:30:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:30:42.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:30:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:30:41.959Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:30:42.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:30:42 smithi083 ceph-mon[79930]: pgmap v14471: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:30:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:30:42 smithi083 ceph-mon[77242]: pgmap v14471: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:30:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:30:42 smithi153 ceph-mon[70044]: pgmap v14471: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:30:42.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:30:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:30:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:30:43.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:30:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:30:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:30:44.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:30:44 smithi083 ceph-mon[79930]: pgmap v14472: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:30:44.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:30:44 smithi083 ceph-mon[77242]: pgmap v14472: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:30:44.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:30:44 smithi153 ceph-mon[70044]: pgmap v14472: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:30:45.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:30:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:30:45.325Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:30:46.350 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:30:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:30:46.052Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:30:46.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:30:46 smithi083 ceph-mon[79930]: pgmap v14473: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:30:46.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:30:46 smithi083 ceph-mon[77242]: pgmap v14473: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:30:46.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:30:46 smithi153 ceph-mon[70044]: pgmap v14473: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:30:48.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:30:48 smithi083 ceph-mon[79930]: pgmap v14474: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:30:48.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:30:48 smithi083 ceph-mon[77242]: pgmap v14474: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:30:48.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:30:48 smithi153 ceph-mon[70044]: pgmap v14474: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:30:50.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:30:50 smithi083 ceph-mon[79930]: pgmap v14475: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:30:50.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:30:50 smithi083 ceph-mon[77242]: pgmap v14475: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:30:50.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:30:50 smithi153 ceph-mon[70044]: pgmap v14475: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:30:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:30:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:30:51] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T13:30:52.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:30:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:30:51.960Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:30:52.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:30:52 smithi083 ceph-mon[79930]: pgmap v14476: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:30:52.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:30:52 smithi083 ceph-mon[77242]: pgmap v14476: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:30:52.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:30:52 smithi153 ceph-mon[70044]: pgmap v14476: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:30:52.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:30:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:30:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:30:53.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:30:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:30:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:30:54.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:30:54 smithi083 ceph-mon[79930]: pgmap v14477: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:30:54.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:30:54 smithi083 ceph-mon[77242]: pgmap v14477: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:30:54.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:30:54 smithi153 ceph-mon[70044]: pgmap v14477: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:30:55.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:30:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:30:55.326Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:30:56.365 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:30:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:30:56.053Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:30:56.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:30:56 smithi083 ceph-mon[77242]: pgmap v14478: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:30:56.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:30:56 smithi083 ceph-mon[79930]: pgmap v14478: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:30:56.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:30:56 smithi153 ceph-mon[70044]: pgmap v14478: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:30:58.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:30:58 smithi083 ceph-mon[77242]: pgmap v14479: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:30:58.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:30:58 smithi083 ceph-mon[79930]: pgmap v14479: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:30:58.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:30:58 smithi153 ceph-mon[70044]: pgmap v14479: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:00.472 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:31:00.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:31:00 smithi083 ceph-mon[79930]: pgmap v14480: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:00.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:31:00 smithi083 ceph-mon[77242]: pgmap v14480: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:00.760 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:31:00.760 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 4m ago 8h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:31:00.761 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 2m ago 8h 63.7M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:31:00.761 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 4m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:31:00.761 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 2m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:31:00.761 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 2m ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:31:00.762 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 4m ago 8h 911M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:31:00.762 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 4m ago 8h 1772M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:31:00.762 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 2m ago 8h 766M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:31:00.762 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 4m ago 8h 1055M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:31:00.763 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 4m ago 8h 23.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:31:00.763 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 2m ago 8h 25.7M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:31:00.763 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 4m ago 8h 1282M 2320M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:31:00.763 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7h) 4m ago 8h 1269M 2320M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:31:00.763 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 4m ago 8h 965M 2320M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:31:00.764 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 4m ago 8h 1243M 2320M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:31:00.764 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 2m ago 8h 1091M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:31:00.764 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 2m ago 8h 1368M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:31:00.764 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 2m ago 8h 1222M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:31:00.765 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 2m ago 8h 1452M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:31:00.765 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 2m ago 8h 182M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:31:00.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:31:00 smithi153 ceph-mon[70044]: pgmap v14480: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:01.116 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:31:01.117 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:31:01.117 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:31:01.117 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:31:01.117 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:31:01.118 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:31:01.118 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:31:01.118 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:31:01.118 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:31:01.119 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:31:01.119 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:31:01.119 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:31:01.119 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:31:01.120 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:31:01.120 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:31:01.120 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:31:01.120 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:31:01.121 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:31:01.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:31:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:31:01] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T13:31:01.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:31:01 smithi083 ceph-mon[77242]: from='client.50970 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:31:01.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:31:01 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2322826435' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:31:01.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:31:01 smithi083 ceph-mon[79930]: from='client.50970 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:31:01.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:31:01 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2322826435' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:31:01.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:31:01 smithi153 ceph-mon[70044]: from='client.50970 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:31:01.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:31:01 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2322826435' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:31:02.382 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:31:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:31:01.960Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:31:02.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:31:02 smithi083 ceph-mon[79930]: pgmap v14481: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:02.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:31:02 smithi083 ceph-mon[79930]: from='client.60907 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:31:02.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:31:02 smithi083 ceph-mon[77242]: pgmap v14481: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:02.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:31:02 smithi083 ceph-mon[77242]: from='client.60907 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:31:02.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:31:02 smithi153 ceph-mon[70044]: pgmap v14481: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:02.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:31:02 smithi153 ceph-mon[70044]: from='client.60907 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:31:02.810 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:31:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:31:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:31:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:31:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:31:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:31:04.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:31:04 smithi083 ceph-mon[77242]: pgmap v14482: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:04.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:31:04 smithi083 ceph-mon[79930]: pgmap v14482: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:04.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:31:04 smithi153 ceph-mon[70044]: pgmap v14482: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:05.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:31:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:31:05.327Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:31:06.390 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:31:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:31:06.054Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:31:06.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:31:06 smithi083 ceph-mon[79930]: pgmap v14483: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:06.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:31:06 smithi083 ceph-mon[77242]: pgmap v14483: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:06.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:31:06 smithi153 ceph-mon[70044]: pgmap v14483: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:08.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:31:08 smithi083 ceph-mon[77242]: pgmap v14484: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:08.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:31:08 smithi083 ceph-mon[79930]: pgmap v14484: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:08.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:31:08 smithi153 ceph-mon[70044]: pgmap v14484: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:10.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:31:10 smithi083 ceph-mon[79930]: pgmap v14485: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:10.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:31:10 smithi083 ceph-mon[77242]: pgmap v14485: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:10.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:31:10 smithi153 ceph-mon[70044]: pgmap v14485: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:11.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:31:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:31:11] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T13:31:12.402 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:31:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:31:11.961Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:31:12.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:31:12 smithi083 ceph-mon[79930]: pgmap v14486: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:12.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:31:12 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:31:12.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:31:12 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:31:12.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:31:12 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:31:12.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:31:12 smithi083 ceph-mon[77242]: pgmap v14486: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:12.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:31:12 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:31:12.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:31:12 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:31:12.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:31:12 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:31:12.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:31:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:31:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:31:12.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:31:12 smithi153 ceph-mon[70044]: pgmap v14486: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:12.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:31:12 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:31:12.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:31:12 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:31:12.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:31:12 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:31:13.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:31:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:31:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:31:14.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:31:13 smithi083 ceph-mon[77242]: pgmap v14487: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:14.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:31:13 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:31:14.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:31:13 smithi083 ceph-mon[79930]: pgmap v14487: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:14.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:31:13 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:31:14.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:31:13 smithi153 ceph-mon[70044]: pgmap v14487: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:14.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:31:13 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:31:15.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:31:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:31:15.328Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:31:16.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:31:15 smithi083 ceph-mon[79930]: pgmap v14488: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:16.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:31:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:31:16.054Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:31:16.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:31:15 smithi083 ceph-mon[77242]: pgmap v14488: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:16.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:31:15 smithi153 ceph-mon[70044]: pgmap v14488: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:18.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:31:18 smithi083 ceph-mon[79930]: pgmap v14489: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:18.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:31:18 smithi083 ceph-mon[77242]: pgmap v14489: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:18.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:31:18 smithi153 ceph-mon[70044]: pgmap v14489: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:20.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:31:20 smithi083 ceph-mon[79930]: pgmap v14490: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:20.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:31:20 smithi083 ceph-mon[77242]: pgmap v14490: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:20.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:31:20 smithi153 ceph-mon[70044]: pgmap v14490: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:31:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:31:21] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T13:31:22.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:31:22 smithi083 ceph-mon[77242]: pgmap v14491: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:22.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:31:22 smithi083 ceph-mon[79930]: pgmap v14491: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:22.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:31:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:31:21.962Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:31:22.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:31:22 smithi153 ceph-mon[70044]: pgmap v14491: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:22.559 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:31:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:31:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:31:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:31:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:31:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:31:24.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:31:24 smithi083 ceph-mon[79930]: pgmap v14492: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:24.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:31:24 smithi083 ceph-mon[77242]: pgmap v14492: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:24.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:31:24 smithi153 ceph-mon[70044]: pgmap v14492: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:25.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:31:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:31:25.328Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:31:26.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:31:26 smithi083 ceph-mon[79930]: pgmap v14493: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:26.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:31:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:31:26.054Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:31:26.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:31:26 smithi083 ceph-mon[77242]: pgmap v14493: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:26.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:31:26 smithi153 ceph-mon[70044]: pgmap v14493: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:28.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:31:28 smithi083 ceph-mon[77242]: pgmap v14494: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:28.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:31:28 smithi083 ceph-mon[79930]: pgmap v14494: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:28.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:31:28 smithi153 ceph-mon[70044]: pgmap v14494: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:30.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:31:30 smithi083 ceph-mon[79930]: pgmap v14495: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:30.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:31:30 smithi083 ceph-mon[77242]: pgmap v14495: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:30.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:31:30 smithi153 ceph-mon[70044]: pgmap v14495: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:31.444 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:31:31.732 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:31:31.732 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 5m ago 8h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:31:31.732 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 3m ago 8h 63.7M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:31:31.733 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 5m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:31:31.733 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 3m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:31:31.733 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 3m ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:31:31.733 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 5m ago 8h 911M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:31:31.734 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 5m ago 8h 1772M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:31:31.734 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 3m ago 8h 766M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:31:31.734 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 5m ago 8h 1055M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:31:31.734 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 5m ago 8h 23.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:31:31.735 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 3m ago 8h 25.7M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:31:31.735 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 5m ago 8h 1282M 2320M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:31:31.735 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 5m ago 8h 1269M 2320M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:31:31.735 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7h) 5m ago 8h 965M 2320M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:31:31.735 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 5m ago 8h 1243M 2320M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:31:31.736 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 3m ago 8h 1091M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:31:31.736 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 3m ago 8h 1368M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:31:31.736 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 3m ago 8h 1222M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:31:31.736 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 3m ago 8h 1452M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:31:31.737 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 3m ago 8h 182M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:31:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:31:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:31:31] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T13:31:32.089 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:31:32.089 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:31:32.089 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:31:32.090 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:31:32.090 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:31:32.090 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:31:32.090 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:31:32.091 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:31:32.091 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:31:32.091 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:31:32.091 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:31:32.091 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:31:32.092 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:31:32.092 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:31:32.092 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:31:32.092 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:31:32.093 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:31:32.093 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:31:32.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:31:32 smithi083 ceph-mon[79930]: pgmap v14496: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:32.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:31:32 smithi083 ceph-mon[79930]: from='client.60919 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:31:32.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:31:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:31:31.962Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:31:32.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:31:32 smithi083 ceph-mon[77242]: pgmap v14496: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:32.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:31:32 smithi083 ceph-mon[77242]: from='client.60919 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:31:32.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:31:32 smithi153 ceph-mon[70044]: pgmap v14496: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:32.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:31:32 smithi153 ceph-mon[70044]: from='client.60919 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:31:32.560 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:31:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:31:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:31:33.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:31:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:31:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:31:33.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:31:33 smithi153 ceph-mon[70044]: from='client.60925 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:31:33.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:31:33 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3568217148' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:31:33.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:31:33 smithi083 ceph-mon[79930]: from='client.60925 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:31:33.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:31:33 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3568217148' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:31:33.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:31:33 smithi083 ceph-mon[77242]: from='client.60925 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:31:33.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:31:33 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3568217148' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:31:34.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:31:34 smithi083 ceph-mon[79930]: pgmap v14497: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:34.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:31:34 smithi083 ceph-mon[77242]: pgmap v14497: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:34.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:31:34 smithi153 ceph-mon[70044]: pgmap v14497: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:35.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:31:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:31:35.329Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:31:36.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:31:36 smithi083 ceph-mon[77242]: pgmap v14498: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:36.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:31:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:31:36.055Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:31:36.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:31:36 smithi083 ceph-mon[79930]: pgmap v14498: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:36.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:31:36 smithi153 ceph-mon[70044]: pgmap v14498: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:38.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:31:38 smithi083 ceph-mon[79930]: pgmap v14499: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:38.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:31:38 smithi083 ceph-mon[77242]: pgmap v14499: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:38.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:31:38 smithi153 ceph-mon[70044]: pgmap v14499: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:40.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:31:40 smithi083 ceph-mon[77242]: pgmap v14500: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:31:40 smithi083 ceph-mon[79930]: pgmap v14500: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:31:40 smithi153 ceph-mon[70044]: pgmap v14500: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:31:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:31:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:31:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:31:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:31:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:31:41.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:31:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:31:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:31:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:31:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:31:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:31:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:31:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:31:41.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:31:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:31:41.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:31:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:31:41.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:31:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:31:41.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:31:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:31:41.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:31:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:31:41.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:31:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:31:41] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T13:31:42.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:31:42 smithi083 ceph-mon[77242]: pgmap v14501: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:42.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:31:42 smithi083 ceph-mon[79930]: pgmap v14501: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:42.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:31:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:31:41.964Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:31:42.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:31:42 smithi153 ceph-mon[70044]: pgmap v14501: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:42.559 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:31:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:31:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:31:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:31:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:31:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:31:44.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:31:44 smithi083 ceph-mon[79930]: pgmap v14502: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:44.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:31:44 smithi083 ceph-mon[77242]: pgmap v14502: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:44.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:31:44 smithi153 ceph-mon[70044]: pgmap v14502: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:45.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:31:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:31:45.330Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:31:46.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:31:46 smithi083 ceph-mon[79930]: pgmap v14503: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:46.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:31:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:31:46.056Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:31:46.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:31:46 smithi083 ceph-mon[77242]: pgmap v14503: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:46.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:31:46 smithi153 ceph-mon[70044]: pgmap v14503: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:48.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:31:48 smithi083 ceph-mon[79930]: pgmap v14504: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:48.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:31:48 smithi083 ceph-mon[77242]: pgmap v14504: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:48.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:31:48 smithi153 ceph-mon[70044]: pgmap v14504: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:50.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:31:50 smithi083 ceph-mon[79930]: pgmap v14505: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:50.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:31:50 smithi083 ceph-mon[77242]: pgmap v14505: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:50.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:31:50 smithi153 ceph-mon[70044]: pgmap v14505: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:31:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:31:51] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T13:31:52.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:31:52 smithi083 ceph-mon[79930]: pgmap v14506: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:52.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:31:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:31:51.964Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:31:52.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:31:52 smithi083 ceph-mon[77242]: pgmap v14506: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:52.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:31:52 smithi153 ceph-mon[70044]: pgmap v14506: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:52.559 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:31:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:31:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:31:53.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:31:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:31:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:31:54.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:31:54 smithi083 ceph-mon[77242]: pgmap v14507: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:54.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:31:54 smithi083 ceph-mon[79930]: pgmap v14507: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:54.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:31:54 smithi153 ceph-mon[70044]: pgmap v14507: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:55.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:31:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:31:55.330Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:31:56.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:31:56 smithi083 ceph-mon[79930]: pgmap v14508: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:56.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:31:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:31:56.056Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:31:56.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:31:56 smithi083 ceph-mon[77242]: pgmap v14508: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:56.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:31:56 smithi153 ceph-mon[70044]: pgmap v14508: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:58.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:31:58 smithi083 ceph-mon[79930]: pgmap v14509: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:58.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:31:58 smithi083 ceph-mon[77242]: pgmap v14509: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:31:58.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:31:58 smithi153 ceph-mon[70044]: pgmap v14509: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:00.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:32:00 smithi083 ceph-mon[79930]: pgmap v14510: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:00.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:32:00 smithi083 ceph-mon[77242]: pgmap v14510: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:00.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:32:00 smithi153 ceph-mon[70044]: pgmap v14510: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:01.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:32:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:32:01] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T13:32:02.416 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:32:02.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:32:02 smithi083 ceph-mon[79930]: pgmap v14511: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:02.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:32:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:32:01.965Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:32:02.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:32:02 smithi083 ceph-mon[77242]: pgmap v14511: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:02.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:32:02 smithi153 ceph-mon[70044]: pgmap v14511: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:02.559 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:32:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:32:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:32:02.704 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:32:02.704 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 5m ago 8h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:32:02.704 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 3m ago 8h 63.7M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:32:02.704 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 5m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:32:02.705 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 3m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:32:02.705 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 3m ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:32:02.705 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 5m ago 8h 911M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:32:02.705 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 5m ago 8h 1772M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:32:02.706 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 3m ago 8h 766M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:32:02.706 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 5m ago 8h 1055M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:32:02.706 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 5m ago 8h 23.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:32:02.706 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 3m ago 8h 25.7M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:32:02.706 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 5m ago 8h 1282M 2320M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:32:02.707 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 5m ago 8h 1269M 2320M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:32:02.707 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 5m ago 8h 965M 2320M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:32:02.707 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7h) 5m ago 8h 1243M 2320M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:32:02.707 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 3m ago 8h 1091M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:32:02.707 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 3m ago 8h 1368M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:32:02.708 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 3m ago 8h 1222M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:32:02.708 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 3m ago 8h 1452M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:32:02.708 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 3m ago 8h 182M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:32:03.059 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:32:03.059 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:32:03.059 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:32:03.060 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:32:03.060 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:32:03.060 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:32:03.060 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:32:03.061 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:32:03.061 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:32:03.061 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:32:03.062 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:32:03.062 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:32:03.062 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:32:03.062 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:32:03.062 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:32:03.063 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:32:03.063 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:32:03.063 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:32:03.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:32:03 smithi153 ceph-mon[70044]: from='client.60937 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:32:03.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:32:03 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1087231075' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:32:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:32:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:32:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:32:03.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:32:03 smithi083 ceph-mon[79930]: from='client.60937 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:32:03.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:32:03 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1087231075' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:32:03.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:32:03 smithi083 ceph-mon[77242]: from='client.60937 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:32:03.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:32:03 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1087231075' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:32:04.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:32:04 smithi083 ceph-mon[77242]: pgmap v14512: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:04.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:32:04 smithi083 ceph-mon[77242]: from='client.60943 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:32:04.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:32:04 smithi083 ceph-mon[79930]: pgmap v14512: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:04.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:32:04 smithi083 ceph-mon[79930]: from='client.60943 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:32:04.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:32:04 smithi153 ceph-mon[70044]: pgmap v14512: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:04.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:32:04 smithi153 ceph-mon[70044]: from='client.60943 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:32:05.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:32:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:32:05.331Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:32:06.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:32:06 smithi083 ceph-mon[79930]: pgmap v14513: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:06.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:32:06 smithi083 ceph-mon[77242]: pgmap v14513: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:06.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:32:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:32:06.057Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:32:06.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:32:06 smithi153 ceph-mon[70044]: pgmap v14513: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:08.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:32:08 smithi083 ceph-mon[77242]: pgmap v14514: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:08.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:32:08 smithi083 ceph-mon[79930]: pgmap v14514: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:08.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:32:08 smithi153 ceph-mon[70044]: pgmap v14514: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:10.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:32:10 smithi083 ceph-mon[79930]: pgmap v14515: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:10.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:32:10 smithi083 ceph-mon[77242]: pgmap v14515: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:10.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:32:10 smithi153 ceph-mon[70044]: pgmap v14515: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:32:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:32:11] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T13:32:12.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:32:12 smithi083 ceph-mon[79930]: pgmap v14516: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:12.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:32:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:32:11.966Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:32:12.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:32:12 smithi083 ceph-mon[77242]: pgmap v14516: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:12.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:32:12 smithi153 ceph-mon[70044]: pgmap v14516: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:12.559 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:32:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:32:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:32:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:32:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:32:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:32:13.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:32:13 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:32:13.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:32:13 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:32:13.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:32:13 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:32:13.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:32:13 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:32:13.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:32:13 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:32:13.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:32:13 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:32:13.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:32:13 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:32:13.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:32:13 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:32:13.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:32:13 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:32:14.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:32:14 smithi083 ceph-mon[79930]: pgmap v14517: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:14.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:32:14 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:32:14.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:32:14 smithi083 ceph-mon[77242]: pgmap v14517: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:14.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:32:14 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:32:14.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:32:14 smithi153 ceph-mon[70044]: pgmap v14517: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:14.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:32:14 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:32:15.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:32:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:32:15.331Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:32:16.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:32:16 smithi083 ceph-mon[79930]: pgmap v14518: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:16.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:32:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:32:16.057Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:32:16.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:32:16 smithi083 ceph-mon[77242]: pgmap v14518: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:16.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:32:16 smithi153 ceph-mon[70044]: pgmap v14518: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:18.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:32:18 smithi083 ceph-mon[79930]: pgmap v14519: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:18.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:32:18 smithi083 ceph-mon[77242]: pgmap v14519: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:18.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:32:18 smithi153 ceph-mon[70044]: pgmap v14519: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:20.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:32:20 smithi083 ceph-mon[79930]: pgmap v14520: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:20.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:32:20 smithi083 ceph-mon[77242]: pgmap v14520: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:20.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:32:20 smithi153 ceph-mon[70044]: pgmap v14520: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:21.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:32:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:32:21] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T13:32:22.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:32:22 smithi083 ceph-mon[79930]: pgmap v14521: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:22.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:32:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:32:21.967Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:32:22.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:32:22 smithi083 ceph-mon[77242]: pgmap v14521: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:22.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:32:22 smithi153 ceph-mon[70044]: pgmap v14521: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:22.559 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:32:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:32:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:32:23.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:32:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:32:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:32:24.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:32:24 smithi083 ceph-mon[79930]: pgmap v14522: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:24.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:32:24 smithi083 ceph-mon[77242]: pgmap v14522: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:24.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:32:24 smithi153 ceph-mon[70044]: pgmap v14522: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:25.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:32:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:32:25.332Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:32:26.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:32:26 smithi083 ceph-mon[79930]: pgmap v14523: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:26.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:32:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:32:26.057Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:32:26.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:32:26 smithi083 ceph-mon[77242]: pgmap v14523: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:26.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:32:26 smithi153 ceph-mon[70044]: pgmap v14523: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:28.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:32:28 smithi083 ceph-mon[79930]: pgmap v14524: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:28.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:32:28 smithi083 ceph-mon[77242]: pgmap v14524: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:28.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:32:28 smithi153 ceph-mon[70044]: pgmap v14524: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:30.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:32:30 smithi083 ceph-mon[77242]: pgmap v14525: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:30.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:32:30 smithi083 ceph-mon[79930]: pgmap v14525: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:30.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:32:30 smithi153 ceph-mon[70044]: pgmap v14525: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:31.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:32:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:32:31] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T13:32:32.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:32:32 smithi083 ceph-mon[77242]: pgmap v14526: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:32.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:32:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:32:31.967Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:32:32.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:32:32 smithi083 ceph-mon[79930]: pgmap v14526: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:32.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:32:32 smithi153 ceph-mon[70044]: pgmap v14526: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:32.560 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:32:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:32:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:32:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:32:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:32:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:32:33.391 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:32:33.686 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:32:33.686 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 6m ago 8h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:32:33.687 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 4m ago 8h 63.7M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:32:33.687 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 6m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:32:33.687 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 4m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:32:33.688 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 4m ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:32:33.688 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 6m ago 8h 911M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:32:33.689 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 6m ago 8h 1772M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:32:33.689 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 4m ago 8h 766M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:32:33.689 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 6m ago 8h 1055M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:32:33.690 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 6m ago 8h 23.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:32:33.691 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 4m ago 8h 25.7M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:32:33.691 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 6m ago 8h 1282M 2320M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:32:33.692 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 6m ago 8h 1269M 2320M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:32:33.692 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 6m ago 8h 965M 2320M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:32:33.693 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 6m ago 8h 1243M 2320M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:32:33.693 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (7h) 4m ago 8h 1091M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:32:33.693 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 4m ago 8h 1368M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:32:33.694 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 4m ago 8h 1222M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:32:33.694 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 4m ago 8h 1452M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:32:33.694 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 4m ago 8h 182M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:32:34.049 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:32:34.050 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:32:34.050 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:32:34.050 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:32:34.051 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:32:34.051 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:32:34.051 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:32:34.051 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:32:34.052 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:32:34.052 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:32:34.052 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:32:34.052 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:32:34.053 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:32:34.053 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:32:34.053 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:32:34.053 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:32:34.053 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:32:34.054 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:32:34.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:32:34 smithi083 ceph-mon[79930]: pgmap v14527: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:34.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:32:34 smithi083 ceph-mon[79930]: from='client.60955 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:32:34.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:32:34 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3957892774' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:32:34.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:32:34 smithi083 ceph-mon[77242]: pgmap v14527: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:34.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:32:34 smithi083 ceph-mon[77242]: from='client.60955 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:32:34.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:32:34 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3957892774' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:32:34.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:32:34 smithi153 ceph-mon[70044]: pgmap v14527: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:34.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:32:34 smithi153 ceph-mon[70044]: from='client.60955 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:32:34.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:32:34 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3957892774' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:32:35.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:32:35 smithi083 ceph-mon[77242]: from='client.51030 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:32:35.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:32:35 smithi083 ceph-mon[79930]: from='client.51030 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:32:35.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:32:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:32:35.333Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:32:35.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:32:35 smithi153 ceph-mon[70044]: from='client.51030 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:32:36.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:32:36 smithi083 ceph-mon[77242]: pgmap v14528: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:36.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:32:36 smithi083 ceph-mon[79930]: pgmap v14528: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:36.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:32:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:32:36.058Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:32:36.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:32:36 smithi153 ceph-mon[70044]: pgmap v14528: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:38.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:32:38 smithi083 ceph-mon[79930]: pgmap v14529: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:38.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:32:38 smithi083 ceph-mon[77242]: pgmap v14529: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:38.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:32:38 smithi153 ceph-mon[70044]: pgmap v14529: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:32:40 smithi083 ceph-mon[79930]: pgmap v14530: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:32:40 smithi083 ceph-mon[77242]: pgmap v14530: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:32:40 smithi153 ceph-mon[70044]: pgmap v14530: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:41.508 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:32:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:32:41.509 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:32:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:32:41.509 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:32:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:32:41.509 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:32:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:32:41.509 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:32:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:32:41.510 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:32:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:32:41.510 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:32:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:32:41.510 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:32:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:32:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:32:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:32:41.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:32:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:32:41.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:32:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:32:41.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:32:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:32:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:32:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:32:41] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T13:32:42.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:32:42 smithi083 ceph-mon[79930]: pgmap v14531: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:42.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:32:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:32:41.968Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:32:42.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:32:42 smithi083 ceph-mon[77242]: pgmap v14531: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:32:42 smithi153 ceph-mon[70044]: pgmap v14531: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:42.559 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:32:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:32:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:32:43.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:32:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:32:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:32:44.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:32:44 smithi153 ceph-mon[70044]: pgmap v14532: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:44.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:32:44 smithi083 ceph-mon[79930]: pgmap v14532: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:44.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:32:44 smithi083 ceph-mon[77242]: pgmap v14532: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:45.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:32:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:32:45.333Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:32:46.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:32:46 smithi083 ceph-mon[79930]: pgmap v14533: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:46.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:32:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:32:46.058Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:32:46.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:32:46 smithi083 ceph-mon[77242]: pgmap v14533: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:46.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:32:46 smithi153 ceph-mon[70044]: pgmap v14533: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:48.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:32:48 smithi153 ceph-mon[70044]: pgmap v14534: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:48.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:32:48 smithi083 ceph-mon[79930]: pgmap v14534: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:48.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:32:48 smithi083 ceph-mon[77242]: pgmap v14534: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:50.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:32:50 smithi153 ceph-mon[70044]: pgmap v14535: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:50.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:32:50 smithi083 ceph-mon[77242]: pgmap v14535: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:50.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:32:50 smithi083 ceph-mon[79930]: pgmap v14535: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:51.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:32:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:32:51] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T13:32:52.234 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:32:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:32:51.968Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:32:52.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:32:52 smithi153 ceph-mon[70044]: pgmap v14536: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:52.559 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:32:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:32:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:32:52.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:32:52 smithi083 ceph-mon[77242]: pgmap v14536: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:52.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:32:52 smithi083 ceph-mon[79930]: pgmap v14536: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:32:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:32:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:32:54.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:32:54 smithi153 ceph-mon[70044]: pgmap v14537: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:54.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:32:54 smithi083 ceph-mon[79930]: pgmap v14537: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:54.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:32:54 smithi083 ceph-mon[77242]: pgmap v14537: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:55.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:32:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:32:55.334Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:32:56.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:32:56 smithi083 ceph-mon[77242]: pgmap v14538: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:56.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:32:56 smithi083 ceph-mon[79930]: pgmap v14538: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:56.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:32:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:32:56.059Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:32:56.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:32:56 smithi153 ceph-mon[70044]: pgmap v14538: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:58.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:32:58 smithi153 ceph-mon[70044]: pgmap v14539: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:58.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:32:58 smithi083 ceph-mon[79930]: pgmap v14539: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:32:58.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:32:58 smithi083 ceph-mon[77242]: pgmap v14539: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:00.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:33:00 smithi153 ceph-mon[70044]: pgmap v14540: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:00.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:33:00 smithi083 ceph-mon[79930]: pgmap v14540: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:00.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:33:00 smithi083 ceph-mon[77242]: pgmap v14540: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:33:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:33:01] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T13:33:02.249 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:33:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:33:01.969Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:33:02.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:33:02 smithi153 ceph-mon[70044]: pgmap v14541: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:02.559 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:33:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:33:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:33:02.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:33:02 smithi083 ceph-mon[77242]: pgmap v14541: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:02.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:33:02 smithi083 ceph-mon[79930]: pgmap v14541: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:33:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:33:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:33:04.376 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:33:04.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:33:04 smithi153 ceph-mon[70044]: pgmap v14542: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:04.667 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:33:04.668 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 7m ago 8h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:33:04.668 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 4m ago 8h 63.7M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:33:04.668 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 7m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:33:04.668 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 4m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:33:04.669 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 4m ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:33:04.669 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 7m ago 8h 911M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:33:04.669 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 7m ago 8h 1772M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:33:04.669 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 4m ago 8h 766M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:33:04.669 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 7m ago 8h 1055M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:33:04.670 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 7m ago 8h 23.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:33:04.670 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 4m ago 8h 25.7M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:33:04.670 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 7m ago 8h 1282M 2320M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:33:04.671 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 7m ago 8h 1269M 2320M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:33:04.671 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 7m ago 8h 965M 2320M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:33:04.671 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 7m ago 8h 1243M 2320M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:33:04.671 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 4m ago 8h 1091M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:33:04.672 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (7h) 4m ago 8h 1368M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:33:04.672 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 4m ago 8h 1222M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:33:04.672 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 4m ago 8h 1452M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:33:04.672 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 4m ago 8h 182M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:33:04.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:33:04 smithi083 ceph-mon[77242]: pgmap v14542: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:04.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:33:04 smithi083 ceph-mon[79930]: pgmap v14542: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:05.024 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:33:05.024 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:33:05.025 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:33:05.025 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:33:05.025 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:33:05.025 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:33:05.026 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:33:05.026 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:33:05.026 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:33:05.026 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:33:05.027 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:33:05.027 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:33:05.027 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:33:05.027 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:33:05.027 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:33:05.028 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:33:05.028 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:33:05.028 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:33:05.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:33:05 smithi153 ceph-mon[70044]: from='client.60973 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:33:05.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:33:05 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1448757434' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:33:05.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:33:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:33:05.334Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:33:05.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:33:05 smithi083 ceph-mon[77242]: from='client.60973 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:33:05.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:33:05 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1448757434' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:33:05.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:33:05 smithi083 ceph-mon[79930]: from='client.60973 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:33:05.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:33:05 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1448757434' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:33:06.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:33:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:33:06.059Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:33:06.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:33:06 smithi083 ceph-mon[79930]: pgmap v14543: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:06.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:33:06 smithi083 ceph-mon[79930]: from='client.51048 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:33:06.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:33:06 smithi083 ceph-mon[77242]: pgmap v14543: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:06.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:33:06 smithi083 ceph-mon[77242]: from='client.51048 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:33:06.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:33:06 smithi153 ceph-mon[70044]: pgmap v14543: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:06.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:33:06 smithi153 ceph-mon[70044]: from='client.51048 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:33:08.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:33:08 smithi153 ceph-mon[70044]: pgmap v14544: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:08.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:33:08 smithi083 ceph-mon[77242]: pgmap v14544: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:08.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:33:08 smithi083 ceph-mon[79930]: pgmap v14544: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:10.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:33:10 smithi153 ceph-mon[70044]: pgmap v14545: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:10.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:33:10 smithi083 ceph-mon[79930]: pgmap v14545: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:10.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:33:10 smithi083 ceph-mon[77242]: pgmap v14545: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:33:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:33:11] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T13:33:12.268 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:33:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:33:11.970Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:33:12.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:33:12 smithi153 ceph-mon[70044]: pgmap v14546: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:12.559 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:33:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:33:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:33:12.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:33:12 smithi083 ceph-mon[77242]: pgmap v14546: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:12.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:33:12 smithi083 ceph-mon[79930]: pgmap v14546: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:33:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:33:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:33:14.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:33:14 smithi153 ceph-mon[70044]: pgmap v14547: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:14.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:33:14 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:33:14.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:33:14 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:33:14.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:33:14 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:33:14.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:33:14 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:33:14.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:33:14 smithi083 ceph-mon[79930]: pgmap v14547: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:14.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:33:14 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:33:14.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:33:14 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:33:14.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:33:14 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:33:14.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:33:14 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:33:14.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:33:14 smithi083 ceph-mon[77242]: pgmap v14547: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:14.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:33:14 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:33:14.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:33:14 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:33:14.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:33:14 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:33:14.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:33:14 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:33:15.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:33:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:33:15.335Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:33:16.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:33:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:33:16.059Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:33:16.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:33:16 smithi083 ceph-mon[77242]: pgmap v14548: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:16.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:33:16 smithi083 ceph-mon[79930]: pgmap v14548: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:16.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:33:16 smithi153 ceph-mon[70044]: pgmap v14548: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:18.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:33:18 smithi153 ceph-mon[70044]: pgmap v14549: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:18.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:33:18 smithi083 ceph-mon[79930]: pgmap v14549: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:18.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:33:18 smithi083 ceph-mon[77242]: pgmap v14549: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:20.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:33:20 smithi153 ceph-mon[70044]: pgmap v14550: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:20.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:33:20 smithi083 ceph-mon[79930]: pgmap v14550: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:20.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:33:20 smithi083 ceph-mon[77242]: pgmap v14550: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:21.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:33:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:33:21] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T13:33:22.285 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:33:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:33:21.971Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:33:22.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:33:22 smithi153 ceph-mon[70044]: pgmap v14551: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:22.559 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:33:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:33:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:33:22.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:33:22 smithi083 ceph-mon[77242]: pgmap v14551: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:22.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:33:22 smithi083 ceph-mon[79930]: pgmap v14551: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:33:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:33:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:33:24.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:33:24 smithi153 ceph-mon[70044]: pgmap v14552: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:24.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:33:24 smithi083 ceph-mon[79930]: pgmap v14552: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:24.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:33:24 smithi083 ceph-mon[77242]: pgmap v14552: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:25.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:33:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:33:25.336Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:33:26.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:33:26 smithi083 ceph-mon[79930]: pgmap v14553: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:26.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:33:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:33:26.060Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:33:26.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:33:26 smithi083 ceph-mon[77242]: pgmap v14553: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:26.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:33:26 smithi153 ceph-mon[70044]: pgmap v14553: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:28.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:33:28 smithi153 ceph-mon[70044]: pgmap v14554: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:28.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:33:28 smithi083 ceph-mon[79930]: pgmap v14554: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:28.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:33:28 smithi083 ceph-mon[77242]: pgmap v14554: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:30.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:33:30 smithi153 ceph-mon[70044]: pgmap v14555: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:30.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:33:30 smithi083 ceph-mon[79930]: pgmap v14555: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:30.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:33:30 smithi083 ceph-mon[77242]: pgmap v14555: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:31.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:33:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:33:31] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T13:33:32.300 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:33:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:33:31.972Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:33:32.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:33:32 smithi153 ceph-mon[70044]: pgmap v14556: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:32.559 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:33:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:33:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:33:32.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:33:32 smithi083 ceph-mon[77242]: pgmap v14556: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:32.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:33:32 smithi083 ceph-mon[79930]: pgmap v14556: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:33:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:33:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:33:34.563 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:33:34 smithi153 ceph-mon[70044]: pgmap v14557: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:34.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:33:34 smithi083 ceph-mon[77242]: pgmap v14557: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:34.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:33:34 smithi083 ceph-mon[79930]: pgmap v14557: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:35.354 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:33:35.641 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:33:35.641 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 7m ago 8h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:33:35.642 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 5m ago 8h 63.7M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:33:35.642 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 7m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:33:35.642 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 5m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:33:35.643 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 5m ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:33:35.643 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 7m ago 8h 911M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:33:35.643 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 7m ago 8h 1772M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:33:35.643 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 5m ago 8h 766M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:33:35.644 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 7m ago 8h 1055M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:33:35.644 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 7m ago 8h 23.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:33:35.644 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 5m ago 8h 25.7M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:33:35.644 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 7m ago 8h 1282M 2320M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:33:35.644 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 7m ago 8h 1269M 2320M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:33:35.645 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 7m ago 8h 965M 2320M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:33:35.645 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 7m ago 8h 1243M 2320M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:33:35.645 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 5m ago 8h 1091M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:33:35.645 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 5m ago 8h 1368M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:33:35.646 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (7h) 5m ago 8h 1222M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:33:35.646 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 5m ago 8h 1452M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:33:35.646 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 5m ago 8h 182M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:33:35.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:33:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:33:35.336Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:33:35.996 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:33:35.996 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:33:35.996 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:33:35.997 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:33:35.997 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:33:35.997 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:33:35.997 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:33:35.998 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:33:35.998 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:33:35.998 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:33:35.999 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:33:35.999 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:33:35.999 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:33:35.999 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:33:35.999 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:33:36.000 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:33:36.000 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:33:36.000 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:33:36.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:33:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:33:36.060Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:33:36.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:33:36 smithi083 ceph-mon[79930]: pgmap v14558: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:36.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:33:36 smithi083 ceph-mon[79930]: from='client.60991 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:33:36.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:33:36 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/979812838' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:33:36.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:33:36 smithi083 ceph-mon[77242]: pgmap v14558: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:36.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:33:36 smithi083 ceph-mon[77242]: from='client.60991 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:33:36.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:33:36 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/979812838' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:33:36.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:33:36 smithi153 ceph-mon[70044]: pgmap v14558: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:36.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:33:36 smithi153 ceph-mon[70044]: from='client.60991 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:33:36.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:33:36 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/979812838' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:33:37.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:33:37 smithi083 ceph-mon[79930]: from='client.60997 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:33:37.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:33:37 smithi083 ceph-mon[77242]: from='client.60997 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:33:37.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:33:37 smithi153 ceph-mon[70044]: from='client.60997 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:33:38.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:33:38 smithi083 ceph-mon[79930]: pgmap v14559: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:38.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:33:38 smithi083 ceph-mon[77242]: pgmap v14559: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:38.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:33:38 smithi153 ceph-mon[70044]: pgmap v14559: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:33:40 smithi083 ceph-mon[79930]: pgmap v14560: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:33:40 smithi083 ceph-mon[77242]: pgmap v14560: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:33:40 smithi153 ceph-mon[70044]: pgmap v14560: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:41.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:33:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:33:41] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T13:33:41.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:33:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:33:41.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:33:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:33:41.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:33:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:33:41.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:33:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:33:41.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:33:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:33:41.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:33:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:33:41.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:33:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:33:41.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:33:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:33:41.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:33:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:33:41.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:33:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:33:41.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:33:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:33:41.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:33:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:33:42.321 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:33:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:33:41.972Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:33:42.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:33:42 smithi083 ceph-mon[79930]: pgmap v14561: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:33:42 smithi083 ceph-mon[77242]: pgmap v14561: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:33:42 smithi153 ceph-mon[70044]: pgmap v14561: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:42.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:33:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:33:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:33:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:33:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:33:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:33:44.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:33:44 smithi083 ceph-mon[79930]: pgmap v14562: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:44.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:33:44 smithi083 ceph-mon[77242]: pgmap v14562: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:44.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:33:44 smithi153 ceph-mon[70044]: pgmap v14562: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:45.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:33:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:33:45.338Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:33:46.327 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:33:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:33:46.062Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:33:46.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:33:46 smithi083 ceph-mon[77242]: pgmap v14563: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:46.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:33:46 smithi083 ceph-mon[79930]: pgmap v14563: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:46.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:33:46 smithi153 ceph-mon[70044]: pgmap v14563: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:48.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:33:48 smithi083 ceph-mon[79930]: pgmap v14564: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:48.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:33:48 smithi083 ceph-mon[77242]: pgmap v14564: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:48.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:33:48 smithi153 ceph-mon[70044]: pgmap v14564: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:50.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:33:50 smithi083 ceph-mon[79930]: pgmap v14565: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:50.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:33:50 smithi083 ceph-mon[77242]: pgmap v14565: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:50.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:33:50 smithi153 ceph-mon[70044]: pgmap v14565: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:33:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:33:51] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T13:33:52.338 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:33:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:33:51.973Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:33:52.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:33:52 smithi083 ceph-mon[79930]: pgmap v14566: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:52.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:33:52 smithi083 ceph-mon[77242]: pgmap v14566: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:52.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:33:52 smithi153 ceph-mon[70044]: pgmap v14566: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:52.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:33:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:33:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:33:53.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:33:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:33:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:33:54.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:33:54 smithi083 ceph-mon[77242]: pgmap v14567: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:54.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:33:54 smithi083 ceph-mon[79930]: pgmap v14567: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:54.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:33:54 smithi153 ceph-mon[70044]: pgmap v14567: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:55.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:33:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:33:55.338Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:33:56.344 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:33:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:33:56.062Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:33:56.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:33:56 smithi083 ceph-mon[77242]: pgmap v14568: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:56.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:33:56 smithi083 ceph-mon[79930]: pgmap v14568: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:56.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:33:56 smithi153 ceph-mon[70044]: pgmap v14568: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:58.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:33:58 smithi083 ceph-mon[79930]: pgmap v14569: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:58.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:33:58 smithi083 ceph-mon[77242]: pgmap v14569: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:33:58.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:33:58 smithi153 ceph-mon[70044]: pgmap v14569: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:00.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:34:00 smithi083 ceph-mon[79930]: pgmap v14570: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:00.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:34:00 smithi083 ceph-mon[77242]: pgmap v14570: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:00.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:34:00 smithi153 ceph-mon[70044]: pgmap v14570: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:01.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:34:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:34:01] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T13:34:02.355 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:34:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:34:01.974Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:34:02.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:34:02 smithi083 ceph-mon[77242]: pgmap v14571: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:02.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:34:02 smithi083 ceph-mon[79930]: pgmap v14571: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:02.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:34:02 smithi153 ceph-mon[70044]: pgmap v14571: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:02.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:34:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:34:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:34:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:34:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:34:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:34:04.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:34:04 smithi083 ceph-mon[79930]: pgmap v14572: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:04.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:34:04 smithi083 ceph-mon[77242]: pgmap v14572: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:04.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:34:04 smithi153 ceph-mon[70044]: pgmap v14572: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:05.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:34:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:34:05.339Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:34:06.322 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:34:06.360 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:34:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:34:06.063Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:34:06.614 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:34:06.615 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 8m ago 8h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:34:06.615 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 5m ago 8h 63.7M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:34:06.615 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 8m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:34:06.616 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 5m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:34:06.616 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 5m ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:34:06.616 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 8m ago 8h 911M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:34:06.616 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 8m ago 8h 1772M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:34:06.617 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 5m ago 8h 766M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:34:06.617 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 8m ago 8h 1055M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:34:06.617 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 8m ago 8h 23.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:34:06.617 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 5m ago 8h 25.7M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:34:06.618 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 8m ago 8h 1282M 2320M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:34:06.618 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 8m ago 8h 1269M 2320M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:34:06.618 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 8m ago 8h 965M 2320M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:34:06.619 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 8m ago 8h 1243M 2320M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:34:06.619 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 5m ago 8h 1091M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:34:06.619 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 5m ago 8h 1368M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:34:06.619 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 5m ago 8h 1222M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:34:06.620 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (7h) 5m ago 8h 1452M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:34:06.620 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 5m ago 8h 182M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:34:06.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:34:06 smithi083 ceph-mon[79930]: pgmap v14573: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:06.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:34:06 smithi083 ceph-mon[77242]: pgmap v14573: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:06.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:34:06 smithi153 ceph-mon[70044]: pgmap v14573: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:06.975 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:34:06.975 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:34:06.976 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:34:06.976 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:34:06.976 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:34:06.976 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:34:06.977 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:34:06.977 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:34:06.977 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:34:06.977 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:34:06.978 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:34:06.978 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:34:06.978 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:34:06.978 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:34:06.979 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:34:06.979 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:34:06.979 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:34:06.979 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:34:07.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:34:07 smithi083 ceph-mon[79930]: from='client.61009 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:34:07.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:34:07 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3919490749' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:34:07.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:34:07 smithi083 ceph-mon[77242]: from='client.61009 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:34:07.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:34:07 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3919490749' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:34:07.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:34:07 smithi153 ceph-mon[70044]: from='client.61009 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:34:07.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:34:07 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3919490749' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:34:08.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:34:08 smithi083 ceph-mon[79930]: pgmap v14574: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:08.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:34:08 smithi083 ceph-mon[79930]: from='client.61015 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:34:08.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:34:08 smithi083 ceph-mon[77242]: pgmap v14574: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:08.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:34:08 smithi083 ceph-mon[77242]: from='client.61015 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:34:08.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:34:08 smithi153 ceph-mon[70044]: pgmap v14574: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:08.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:34:08 smithi153 ceph-mon[70044]: from='client.61015 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:34:10.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:34:10 smithi083 ceph-mon[79930]: pgmap v14575: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:10.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:34:10 smithi083 ceph-mon[77242]: pgmap v14575: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:10.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:34:10 smithi153 ceph-mon[70044]: pgmap v14575: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:34:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:34:11] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T13:34:12.372 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:34:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:34:11.974Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:34:12.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:34:12 smithi083 ceph-mon[79930]: pgmap v14576: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:12.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:34:12 smithi083 ceph-mon[77242]: pgmap v14576: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:12.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:34:12 smithi153 ceph-mon[70044]: pgmap v14576: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:12.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:34:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:34:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:34:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:34:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:34:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:34:14.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:34:14 smithi083 ceph-mon[79930]: pgmap v14577: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:14.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:34:14 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:34:14.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:34:14 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:34:14.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:34:14 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:34:14.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:34:14 smithi083 ceph-mon[77242]: pgmap v14577: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:14.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:34:14 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:34:14.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:34:14 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:34:14.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:34:14 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:34:14.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:34:14 smithi153 ceph-mon[70044]: pgmap v14577: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:14.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:34:14 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:34:14.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:34:14 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:34:14.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:34:14 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:34:15.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:34:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:34:15.339Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:34:15.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:34:15 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:34:15.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:34:15 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:34:15.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:34:15 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:34:15.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:34:15 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:34:15.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:34:15 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:34:15.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:34:15 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:34:15.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:34:15 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:34:15.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:34:15 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:34:15.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:34:15 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:34:15.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:34:15 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:34:15.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:34:15 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:34:15.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:34:15 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:34:15.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:34:15 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:34:15.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:34:15 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:34:15.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:34:15 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:34:15.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:34:15 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:34:15.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:34:15 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:34:15.718 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:34:15 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:34:15.718 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:34:15 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:34:15.718 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:34:15 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:34:15.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:34:15 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:34:15.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:34:15 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:34:15.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:34:15 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:34:15.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:34:15 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:34:15.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:34:15 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:34:15.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:34:15 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:34:15.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:34:15 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:34:15.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:34:15 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:34:15.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:34:15 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:34:15.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:34:15 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:34:16.380 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:34:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:34:16.064Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:34:16.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:34:16 smithi083 ceph-mon[79930]: pgmap v14578: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:16.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:34:16 smithi083 ceph-mon[79930]: Adjusting osd_memory_target on smithi083 to 2305M 2022-09-29T13:34:16.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:34:16 smithi083 ceph-mon[77242]: pgmap v14578: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:16.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:34:16 smithi083 ceph-mon[77242]: Adjusting osd_memory_target on smithi083 to 2305M 2022-09-29T13:34:16.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:34:16 smithi153 ceph-mon[70044]: pgmap v14578: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:16.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:34:16 smithi153 ceph-mon[70044]: Adjusting osd_memory_target on smithi083 to 2305M 2022-09-29T13:34:18.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:34:18 smithi083 ceph-mon[79930]: pgmap v14579: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:18.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:34:18 smithi083 ceph-mon[77242]: pgmap v14579: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:18.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:34:18 smithi153 ceph-mon[70044]: pgmap v14579: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:20.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:34:20 smithi083 ceph-mon[79930]: pgmap v14580: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:20.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:34:20 smithi083 ceph-mon[77242]: pgmap v14580: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:20.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:34:20 smithi153 ceph-mon[70044]: pgmap v14580: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:34:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:34:21] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T13:34:22.391 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:34:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:34:21.975Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:34:22.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:34:22 smithi083 ceph-mon[77242]: pgmap v14581: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:22.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:34:22 smithi083 ceph-mon[79930]: pgmap v14581: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:22.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:34:22 smithi153 ceph-mon[70044]: pgmap v14581: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:22.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:34:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:34:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:34:23.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:34:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:34:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:34:24.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:34:24 smithi083 ceph-mon[77242]: pgmap v14582: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:24.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:34:24 smithi083 ceph-mon[79930]: pgmap v14582: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:24.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:34:24 smithi153 ceph-mon[70044]: pgmap v14582: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:25.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:34:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:34:25.340Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:34:26.397 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:34:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:34:26.064Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:34:26.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:34:26 smithi083 ceph-mon[77242]: pgmap v14583: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:26.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:34:26 smithi083 ceph-mon[79930]: pgmap v14583: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:26.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:34:26 smithi153 ceph-mon[70044]: pgmap v14583: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:28.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:34:28 smithi083 ceph-mon[79930]: pgmap v14584: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:28.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:34:28 smithi083 ceph-mon[77242]: pgmap v14584: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:28.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:34:28 smithi153 ceph-mon[70044]: pgmap v14584: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:30.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:34:30 smithi083 ceph-mon[79930]: pgmap v14585: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:30.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:34:30 smithi083 ceph-mon[77242]: pgmap v14585: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:30.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:34:30 smithi153 ceph-mon[70044]: pgmap v14585: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:34:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:34:31] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T13:34:32.406 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:34:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:34:31.976Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:34:32.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:34:32 smithi083 ceph-mon[79930]: pgmap v14586: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:32.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:34:32 smithi083 ceph-mon[77242]: pgmap v14586: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:32.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:34:32 smithi153 ceph-mon[70044]: pgmap v14586: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:32.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:34:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:34:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:34:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:34:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:34:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:34:34.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:34:34 smithi083 ceph-mon[79930]: pgmap v14587: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:34.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:34:34 smithi083 ceph-mon[77242]: pgmap v14587: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:34.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:34:34 smithi153 ceph-mon[70044]: pgmap v14587: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:35.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:34:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:34:35.341Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:34:36.412 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:34:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:34:36.065Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:34:36.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:34:36 smithi083 ceph-mon[79930]: pgmap v14588: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:36.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:34:36 smithi083 ceph-mon[77242]: pgmap v14588: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:36.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:34:36 smithi153 ceph-mon[70044]: pgmap v14588: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:37.301 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:34:37.589 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:34:37.590 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 8m ago 8h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:34:37.590 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 6m ago 8h 63.7M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:34:37.590 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 8m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:34:37.591 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 6m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:34:37.591 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 6m ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:34:37.591 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 8m ago 8h 911M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:34:37.591 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 8m ago 8h 1772M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:34:37.592 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 6m ago 8h 766M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:34:37.592 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 8m ago 8h 1055M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:34:37.592 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 8m ago 8h 23.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:34:37.592 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 6m ago 8h 25.7M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:34:37.593 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 8m ago 8h 1282M 2305M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:34:37.593 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 8m ago 8h 1269M 2305M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:34:37.593 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 8m ago 8h 965M 2305M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:34:37.593 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 8m ago 8h 1243M 2305M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:34:37.594 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 6m ago 8h 1091M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:34:37.594 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 6m ago 8h 1368M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:34:37.594 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 6m ago 8h 1222M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:34:37.594 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 6m ago 8h 1452M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:34:37.595 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 6m ago 8h 182M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:34:37.944 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:34:37.944 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:34:37.944 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:34:37.945 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:34:37.945 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:34:37.945 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:34:37.945 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:34:37.946 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:34:37.946 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:34:37.946 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:34:37.946 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:34:37.947 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:34:37.947 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:34:37.947 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:34:37.947 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:34:37.947 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:34:37.948 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:34:37.948 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:34:38.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:34:38 smithi083 ceph-mon[79930]: pgmap v14589: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:38.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:34:38 smithi083 ceph-mon[79930]: from='client.51096 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:34:38.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:34:38 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/707586963' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:34:38.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:34:38 smithi083 ceph-mon[77242]: pgmap v14589: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:38.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:34:38 smithi083 ceph-mon[77242]: from='client.51096 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:34:38.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:34:38 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/707586963' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:34:38.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:34:38 smithi153 ceph-mon[70044]: pgmap v14589: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:38.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:34:38 smithi153 ceph-mon[70044]: from='client.51096 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:34:38.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:34:38 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/707586963' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:34:39.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:34:39 smithi083 ceph-mon[79930]: from='client.61033 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:34:39.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:34:39 smithi083 ceph-mon[77242]: from='client.61033 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:34:39.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:34:39 smithi153 ceph-mon[70044]: from='client.61033 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:34:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:34:40 smithi083 ceph-mon[79930]: pgmap v14590: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:34:40 smithi083 ceph-mon[77242]: pgmap v14590: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:34:40 smithi153 ceph-mon[70044]: pgmap v14590: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:41.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:34:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:34:41] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T13:34:41.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:34:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:34:41.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:34:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:34:41.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:34:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:34:41.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:34:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:34:41.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:34:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:34:41.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:34:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:34:41.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:34:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:34:41.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:34:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:34:41.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:34:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:34:41.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:34:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:34:41.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:34:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:34:41.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:34:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:34:42.433 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:34:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:34:41.977Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:34:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:34:42 smithi083 ceph-mon[79930]: pgmap v14591: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:34:42 smithi083 ceph-mon[77242]: pgmap v14591: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:34:42 smithi153 ceph-mon[70044]: pgmap v14591: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:42.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:34:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:34:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:34:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:34:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:34:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:34:44.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:34:44 smithi083 ceph-mon[79930]: pgmap v14592: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:44.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:34:44 smithi083 ceph-mon[77242]: pgmap v14592: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:44.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:34:44 smithi153 ceph-mon[70044]: pgmap v14592: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:45.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:34:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:34:45.342Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:34:46.439 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:34:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:34:46.066Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:34:46.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:34:46 smithi083 ceph-mon[77242]: pgmap v14593: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:46.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:34:46 smithi083 ceph-mon[79930]: pgmap v14593: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:46.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:34:46 smithi153 ceph-mon[70044]: pgmap v14593: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:48.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:34:48 smithi083 ceph-mon[79930]: pgmap v14594: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:48.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:34:48 smithi083 ceph-mon[77242]: pgmap v14594: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:48.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:34:48 smithi153 ceph-mon[70044]: pgmap v14594: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:50.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:34:50 smithi083 ceph-mon[79930]: pgmap v14595: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:50.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:34:50 smithi083 ceph-mon[77242]: pgmap v14595: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:50.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:34:50 smithi153 ceph-mon[70044]: pgmap v14595: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:34:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:34:51] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T13:34:52.448 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:34:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:34:51.977Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:34:52.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:34:52 smithi083 ceph-mon[79930]: pgmap v14596: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:52.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:34:52 smithi083 ceph-mon[77242]: pgmap v14596: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:52.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:34:52 smithi153 ceph-mon[70044]: pgmap v14596: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:52.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:34:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:34:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:34:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:34:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:34:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:34:54.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:34:54 smithi083 ceph-mon[79930]: pgmap v14597: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:54.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:34:54 smithi083 ceph-mon[77242]: pgmap v14597: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:54.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:34:54 smithi153 ceph-mon[70044]: pgmap v14597: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:55.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:34:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:34:55.342Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:34:56.453 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:34:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:34:56.067Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:34:56.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:34:56 smithi083 ceph-mon[77242]: pgmap v14598: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:56.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:34:56 smithi083 ceph-mon[79930]: pgmap v14598: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:56.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:34:56 smithi153 ceph-mon[70044]: pgmap v14598: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:58.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:34:58 smithi083 ceph-mon[79930]: pgmap v14599: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:58.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:34:58 smithi083 ceph-mon[77242]: pgmap v14599: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:34:58.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:34:58 smithi153 ceph-mon[70044]: pgmap v14599: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:00.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:35:00 smithi083 ceph-mon[77242]: pgmap v14600: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:00.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:35:00 smithi083 ceph-mon[79930]: pgmap v14600: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:00.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:35:00 smithi153 ceph-mon[70044]: pgmap v14600: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:01.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:35:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:35:01] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T13:35:02.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:35:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:35:01.978Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:35:02.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:35:02 smithi153 ceph-mon[70044]: pgmap v14601: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:02.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:35:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:35:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:35:02.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:35:02 smithi083 ceph-mon[77242]: pgmap v14601: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:02.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:35:02 smithi083 ceph-mon[79930]: pgmap v14601: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:35:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:35:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:35:04.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:35:04 smithi153 ceph-mon[70044]: pgmap v14602: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:04.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:35:04 smithi083 ceph-mon[79930]: pgmap v14602: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:04.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:35:04 smithi083 ceph-mon[77242]: pgmap v14602: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:05.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:35:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:35:05.343Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:35:06.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:35:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:35:06.067Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:35:06.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:35:06 smithi153 ceph-mon[70044]: pgmap v14603: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:06.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:35:06 smithi083 ceph-mon[79930]: pgmap v14603: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:06.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:35:06 smithi083 ceph-mon[77242]: pgmap v14603: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:08.269 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:35:08.559 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:35:08.559 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 9m ago 8h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:35:08.560 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 6m ago 8h 63.7M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:35:08.560 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 9m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:35:08.560 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 6m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:35:08.561 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 6m ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:35:08.561 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 9m ago 8h 911M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:35:08.561 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 9m ago 8h 1772M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:35:08.561 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 6m ago 8h 766M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:35:08.562 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 9m ago 8h 1055M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:35:08.562 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 9m ago 8h 23.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:35:08.562 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 6m ago 8h 25.7M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:35:08.562 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 9m ago 8h 1282M 2305M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:35:08.563 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 9m ago 8h 1269M 2305M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:35:08.563 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 9m ago 8h 965M 2305M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:35:08.563 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 9m ago 8h 1243M 2305M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:35:08.563 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 6m ago 8h 1091M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:35:08.563 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 6m ago 8h 1368M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:35:08.564 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 6m ago 8h 1222M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:35:08.564 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 6m ago 8h 1452M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:35:08.564 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 6m ago 8h 182M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:35:08.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:35:08 smithi153 ceph-mon[70044]: pgmap v14604: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:08.917 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:35:08.917 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:35:08.917 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:35:08.918 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:35:08.918 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:35:08.918 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:35:08.918 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:35:08.919 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:35:08.919 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:35:08.919 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:35:08.919 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:35:08.920 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:35:08.920 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:35:08.920 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:35:08.920 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:35:08.920 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:35:08.921 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:35:08.921 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:35:08.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:35:08 smithi083 ceph-mon[79930]: pgmap v14604: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:08.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:35:08 smithi083 ceph-mon[77242]: pgmap v14604: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:09.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:35:09 smithi153 ceph-mon[70044]: from='client.51114 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:35:09.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:35:09 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/991058843' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:35:09.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:35:09 smithi083 ceph-mon[79930]: from='client.51114 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:35:09.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:35:09 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/991058843' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:35:09.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:35:09 smithi083 ceph-mon[77242]: from='client.51114 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:35:09.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:35:09 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/991058843' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:35:10.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:35:10 smithi153 ceph-mon[70044]: from='client.51120 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:35:10.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:35:10 smithi153 ceph-mon[70044]: pgmap v14605: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:10.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:35:10 smithi083 ceph-mon[77242]: from='client.51120 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:35:10.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:35:10 smithi083 ceph-mon[77242]: pgmap v14605: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:10.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:35:10 smithi083 ceph-mon[79930]: from='client.51120 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:35:10.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:35:10 smithi083 ceph-mon[79930]: pgmap v14605: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:35:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:35:11] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T13:35:12.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:35:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:35:11.979Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:35:12.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:35:12 smithi153 ceph-mon[70044]: pgmap v14606: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:12.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:35:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:35:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:35:12.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:35:12 smithi083 ceph-mon[79930]: pgmap v14606: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:12.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:35:12 smithi083 ceph-mon[77242]: pgmap v14606: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:35:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:35:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:35:14.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:35:14 smithi153 ceph-mon[70044]: pgmap v14607: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:14.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:35:14 smithi083 ceph-mon[77242]: pgmap v14607: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:14.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:35:14 smithi083 ceph-mon[79930]: pgmap v14607: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:15.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:35:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:35:15.344Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:35:15.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:35:15 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:35:15.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:35:15 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:35:15.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:35:15 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:35:15.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:35:15 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:35:15.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:35:15 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:35:15.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:35:15 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:35:15.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:35:15 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:35:15.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:35:15 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:35:15.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:35:15 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:35:16.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:35:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:35:16.067Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:35:16.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:35:16 smithi153 ceph-mon[70044]: pgmap v14608: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:16.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:35:16 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:35:16.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:35:16 smithi083 ceph-mon[77242]: pgmap v14608: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:16.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:35:16 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:35:16.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:35:16 smithi083 ceph-mon[79930]: pgmap v14608: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:16.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:35:16 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:35:18.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:35:18 smithi153 ceph-mon[70044]: pgmap v14609: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:18.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:35:18 smithi083 ceph-mon[79930]: pgmap v14609: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:18.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:35:18 smithi083 ceph-mon[77242]: pgmap v14609: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:20.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:35:20 smithi153 ceph-mon[70044]: pgmap v14610: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:20.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:35:20 smithi083 ceph-mon[79930]: pgmap v14610: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:20.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:35:20 smithi083 ceph-mon[77242]: pgmap v14610: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:21.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:35:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:35:21] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T13:35:22.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:35:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:35:21.979Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:35:22.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:35:22 smithi153 ceph-mon[70044]: pgmap v14611: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:22.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:35:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:35:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:35:22.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:35:22 smithi083 ceph-mon[79930]: pgmap v14611: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:22.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:35:22 smithi083 ceph-mon[77242]: pgmap v14611: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:35:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:35:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:35:24.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:35:24 smithi153 ceph-mon[70044]: pgmap v14612: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:24.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:35:24 smithi083 ceph-mon[79930]: pgmap v14612: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:24.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:35:24 smithi083 ceph-mon[77242]: pgmap v14612: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:25.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:35:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:35:25.345Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:35:26.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:35:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:35:26.071Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:35:26.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:35:26 smithi153 ceph-mon[70044]: pgmap v14613: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:26.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:35:26 smithi083 ceph-mon[77242]: pgmap v14613: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:26.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:35:26 smithi083 ceph-mon[79930]: pgmap v14613: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:28.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:35:28 smithi153 ceph-mon[70044]: pgmap v14614: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:28.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:35:28 smithi083 ceph-mon[77242]: pgmap v14614: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:28.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:35:28 smithi083 ceph-mon[79930]: pgmap v14614: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:30.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:35:30 smithi153 ceph-mon[70044]: pgmap v14615: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:30.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:35:30 smithi083 ceph-mon[79930]: pgmap v14615: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:30.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:35:30 smithi083 ceph-mon[77242]: pgmap v14615: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:35:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:35:31] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T13:35:32.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:35:32 smithi083 ceph-mon[79930]: pgmap v14616: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:32.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:35:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:35:31.980Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:35:32.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:35:32 smithi083 ceph-mon[77242]: pgmap v14616: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:32.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:35:32 smithi153 ceph-mon[70044]: pgmap v14616: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:32.559 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:35:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:35:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:35:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:35:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:35:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:35:34.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:35:34 smithi083 ceph-mon[79930]: pgmap v14617: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:34.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:35:34 smithi083 ceph-mon[77242]: pgmap v14617: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:34.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:35:34 smithi153 ceph-mon[70044]: pgmap v14617: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:35.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:35:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:35:35.345Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:35:36.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:35:36 smithi083 ceph-mon[79930]: pgmap v14618: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:36.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:35:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:35:36.070Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:35:36.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:35:36 smithi083 ceph-mon[77242]: pgmap v14618: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:36.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:35:36 smithi153 ceph-mon[70044]: pgmap v14618: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:38.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:35:38 smithi083 ceph-mon[79930]: pgmap v14619: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:38.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:35:38 smithi083 ceph-mon[77242]: pgmap v14619: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:38.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:35:38 smithi153 ceph-mon[70044]: pgmap v14619: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:39.245 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:35:39.540 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:35:39.540 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 9m ago 8h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:35:39.540 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 7m ago 8h 63.7M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:35:39.541 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 9m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:35:39.541 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 7m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:35:39.541 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 7m ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:35:39.541 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 9m ago 8h 911M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:35:39.542 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 9m ago 8h 1772M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:35:39.542 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 7m ago 8h 766M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:35:39.542 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 9m ago 8h 1055M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:35:39.542 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 9m ago 8h 23.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:35:39.543 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 7m ago 8h 25.7M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:35:39.543 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 9m ago 8h 1282M 2305M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:35:39.543 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 9m ago 8h 1269M 2305M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:35:39.543 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 9m ago 8h 965M 2305M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:35:39.544 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 9m ago 8h 1243M 2305M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:35:39.544 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 7m ago 8h 1091M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:35:39.544 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 7m ago 8h 1368M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:35:39.544 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 7m ago 8h 1222M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:35:39.545 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 7m ago 8h 1452M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:35:39.545 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 7m ago 8h 182M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:35:39.899 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:35:39.900 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:35:39.900 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:35:39.900 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:35:39.901 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:35:39.901 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:35:39.901 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:35:39.901 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:35:39.902 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:35:39.902 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:35:39.902 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:35:39.902 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:35:39.903 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:35:39.903 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:35:39.903 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:35:39.904 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:35:39.904 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:35:39.904 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:35:40.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:35:40 smithi083 ceph-mon[77242]: pgmap v14620: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:35:40 smithi083 ceph-mon[77242]: from='client.51132 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:35:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:35:40 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2693412898' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:35:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:35:40 smithi083 ceph-mon[79930]: pgmap v14620: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:40.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:35:40 smithi083 ceph-mon[79930]: from='client.51132 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:35:40.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:35:40 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2693412898' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:35:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:35:40 smithi153 ceph-mon[70044]: pgmap v14620: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:40.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:35:40 smithi153 ceph-mon[70044]: from='client.51132 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:35:40.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:35:40 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2693412898' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:35:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:35:41 smithi083 ceph-mon[79930]: from='client.51138 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:35:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:35:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:35:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:35:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:35:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:35:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:35:41.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:35:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:35:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:35:41 smithi083 ceph-mon[77242]: from='client.51138 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:35:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:35:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:35:41.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:35:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:35:41.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:35:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:35:41.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:35:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:35:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:35:41 smithi153 ceph-mon[70044]: from='client.51138 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:35:41.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:35:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:35:41.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:35:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:35:41.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:35:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:35:41.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:35:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:35:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:35:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:35:41] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T13:35:42.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:35:42 smithi083 ceph-mon[79930]: pgmap v14621: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:42.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:35:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:35:41.982Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:35:42.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:35:42 smithi083 ceph-mon[77242]: pgmap v14621: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:35:42 smithi153 ceph-mon[70044]: pgmap v14621: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:42.559 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:35:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:35:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:35:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:35:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:35:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:35:44.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:35:44 smithi083 ceph-mon[79930]: pgmap v14622: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:44.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:35:44 smithi083 ceph-mon[77242]: pgmap v14622: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:44.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:35:44 smithi153 ceph-mon[70044]: pgmap v14622: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:45.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:35:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:35:45.346Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:35:46.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:35:46 smithi083 ceph-mon[79930]: pgmap v14623: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:46.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:35:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:35:46.070Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:35:46.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:35:46 smithi083 ceph-mon[77242]: pgmap v14623: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:46.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:35:46 smithi153 ceph-mon[70044]: pgmap v14623: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:48.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:35:48 smithi083 ceph-mon[77242]: pgmap v14624: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:48.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:35:48 smithi083 ceph-mon[79930]: pgmap v14624: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:48.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:35:48 smithi153 ceph-mon[70044]: pgmap v14624: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:50.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:35:50 smithi083 ceph-mon[79930]: pgmap v14625: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:50.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:35:50 smithi083 ceph-mon[77242]: pgmap v14625: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:50.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:35:50 smithi153 ceph-mon[70044]: pgmap v14625: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:51.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:35:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:35:51] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T13:35:52.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:35:52 smithi083 ceph-mon[79930]: pgmap v14626: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:52.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:35:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:35:51.982Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:35:52.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:35:52 smithi083 ceph-mon[77242]: pgmap v14626: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:52.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:35:52 smithi153 ceph-mon[70044]: pgmap v14626: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:52.559 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:35:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:35:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:35:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:35:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:35:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:35:54.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:35:54 smithi083 ceph-mon[79930]: pgmap v14627: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:54.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:35:54 smithi083 ceph-mon[77242]: pgmap v14627: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:54.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:35:54 smithi153 ceph-mon[70044]: pgmap v14627: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:55.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:35:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:35:55.346Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:35:56.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:35:56 smithi083 ceph-mon[79930]: pgmap v14628: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:56.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:35:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:35:56.070Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:35:56.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:35:56 smithi083 ceph-mon[77242]: pgmap v14628: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:56.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:35:56 smithi153 ceph-mon[70044]: pgmap v14628: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:58.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:35:58 smithi083 ceph-mon[79930]: pgmap v14629: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:58.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:35:58 smithi083 ceph-mon[77242]: pgmap v14629: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:35:58.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:35:58 smithi153 ceph-mon[70044]: pgmap v14629: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:00.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:36:00 smithi083 ceph-mon[79930]: pgmap v14630: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:00.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:36:00 smithi083 ceph-mon[77242]: pgmap v14630: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:00.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:36:00 smithi153 ceph-mon[70044]: pgmap v14630: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:01.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:36:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:36:01] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T13:36:02.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:36:02 smithi083 ceph-mon[79930]: pgmap v14631: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:02.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:36:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:36:01.983Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:36:02.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:36:02 smithi083 ceph-mon[77242]: pgmap v14631: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:02.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:36:02 smithi153 ceph-mon[70044]: pgmap v14631: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:02.559 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:36:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:36:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:36:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:36:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:36:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:36:04.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:36:04 smithi083 ceph-mon[79930]: pgmap v14632: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:04.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:36:04 smithi083 ceph-mon[77242]: pgmap v14632: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:04.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:36:04 smithi153 ceph-mon[70044]: pgmap v14632: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:05.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:36:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:36:05.347Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:36:06.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:36:06 smithi083 ceph-mon[79930]: pgmap v14633: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:06.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:36:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:36:06.071Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:36:06.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:36:06 smithi083 ceph-mon[77242]: pgmap v14633: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:06.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:36:06 smithi153 ceph-mon[70044]: pgmap v14633: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:08.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:36:08 smithi083 ceph-mon[79930]: pgmap v14634: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:08.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:36:08 smithi083 ceph-mon[77242]: pgmap v14634: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:08.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:36:08 smithi153 ceph-mon[70044]: pgmap v14634: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:10.224 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:36:10.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:36:10 smithi083 ceph-mon[79930]: pgmap v14635: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:10.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:36:10 smithi083 ceph-mon[77242]: pgmap v14635: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:10.511 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:36:10.511 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 10m ago 8h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:36:10.511 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 8m ago 8h 63.7M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:36:10.512 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 10m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:36:10.512 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 8m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:36:10.512 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 8m ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:36:10.512 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 10m ago 8h 911M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:36:10.513 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 10m ago 8h 1772M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:36:10.513 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 8m ago 8h 766M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:36:10.513 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 10m ago 8h 1055M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:36:10.514 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 10m ago 8h 23.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:36:10.514 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 8m ago 8h 25.7M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:36:10.515 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 10m ago 8h 1282M 2305M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:36:10.515 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 10m ago 8h 1269M 2305M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:36:10.515 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 10m ago 8h 965M 2305M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:36:10.515 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 10m ago 8h 1243M 2305M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:36:10.516 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 8m ago 8h 1091M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:36:10.516 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 8m ago 8h 1368M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:36:10.516 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 8m ago 8h 1222M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:36:10.516 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 8m ago 8h 1452M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:36:10.517 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 8m ago 8h 182M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:36:10.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:36:10 smithi153 ceph-mon[70044]: pgmap v14635: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:10.867 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:36:10.868 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:36:10.868 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:36:10.868 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:36:10.868 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:36:10.869 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:36:10.869 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:36:10.869 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:36:10.869 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:36:10.869 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:36:10.870 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:36:10.870 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:36:10.870 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:36:10.870 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:36:10.871 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:36:10.871 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:36:10.871 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:36:10.871 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:36:11.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:36:11 smithi083 ceph-mon[79930]: from='client.51150 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:36:11.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:36:11 smithi083 ceph-mon[79930]: from='client.51156 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:36:11.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:36:11 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4009547051' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:36:11.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:36:11 smithi083 ceph-mon[77242]: from='client.51150 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:36:11.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:36:11 smithi083 ceph-mon[77242]: from='client.51156 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:36:11.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:36:11 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4009547051' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:36:11.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:36:11 smithi153 ceph-mon[70044]: from='client.51150 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:36:11.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:36:11 smithi153 ceph-mon[70044]: from='client.51156 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:36:11.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:36:11 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4009547051' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:36:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:36:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:36:11] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T13:36:12.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:36:12 smithi083 ceph-mon[79930]: pgmap v14636: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:12.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:36:12 smithi083 ceph-mon[77242]: pgmap v14636: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:12.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:36:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:36:11.983Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:36:12.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:36:12 smithi153 ceph-mon[70044]: pgmap v14636: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:12.559 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:36:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:36:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:36:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:36:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:36:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:36:14.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:36:14 smithi083 ceph-mon[79930]: pgmap v14637: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:14.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:36:14 smithi083 ceph-mon[77242]: pgmap v14637: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:14.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:36:14 smithi153 ceph-mon[70044]: pgmap v14637: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:15.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:36:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:36:15.348Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:36:16.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:36:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:36:16.071Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:36:16.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:36:16 smithi083 ceph-mon[79930]: pgmap v14638: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:16.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:36:16 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:36:16.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:36:16 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:36:16.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:36:16 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:36:16.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:36:16 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:36:16.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:36:16 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:36:16.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:36:16 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:36:16.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:36:16 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:36:16.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:36:16 smithi083 ceph-mon[77242]: pgmap v14638: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:16.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:36:16 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:36:16.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:36:16 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:36:16.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:36:16 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:36:16.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:36:16 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:36:16.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:36:16 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:36:16.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:36:16 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:36:16.467 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:36:16 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:36:16.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:36:16 smithi153 ceph-mon[70044]: pgmap v14638: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:16.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:36:16 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:36:16.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:36:16 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:36:16.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:36:16 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:36:16.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:36:16 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:36:16.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:36:16 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:36:16.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:36:16 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:36:16.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:36:16 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:36:18.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:36:18 smithi083 ceph-mon[79930]: pgmap v14639: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:18.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:36:18 smithi083 ceph-mon[77242]: pgmap v14639: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:18.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:36:18 smithi153 ceph-mon[70044]: pgmap v14639: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:20.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:36:20 smithi083 ceph-mon[79930]: pgmap v14640: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:20.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:36:20 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:36:20.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:36:20 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:36:20.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:36:20 smithi083 ceph-mon[77242]: pgmap v14640: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:20.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:36:20 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:36:20.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:36:20 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:36:20.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:36:20 smithi153 ceph-mon[70044]: pgmap v14640: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:20.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:36:20 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:36:20.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:36:20 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:36:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:36:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:36:21] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T13:36:21.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:36:21 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:36:21.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:36:21 smithi083 ceph-mon[77242]: pgmap v14641: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:21.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:36:21 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:36:21.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:36:21 smithi083 ceph-mon[79930]: pgmap v14641: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:22.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:36:21 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:36:22.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:36:21 smithi153 ceph-mon[70044]: pgmap v14641: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:22.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:36:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:36:21.984Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:36:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:36:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:36:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:36:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:36:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:36:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:36:23.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:36:23 smithi083 ceph-mon[79930]: pgmap v14642: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:23.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:36:23 smithi083 ceph-mon[77242]: pgmap v14642: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:24.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:36:23 smithi153 ceph-mon[70044]: pgmap v14642: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:25.677 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:36:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:36:25.348Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:36:25.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:36:25 smithi083 ceph-mon[79930]: pgmap v14643: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:25.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:36:25 smithi083 ceph-mon[77242]: pgmap v14643: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:26.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:36:25 smithi153 ceph-mon[70044]: pgmap v14643: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:26.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:36:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:36:26.072Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:36:28.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:36:28 smithi083 ceph-mon[79930]: pgmap v14644: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:28.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:36:28 smithi083 ceph-mon[77242]: pgmap v14644: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:28.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:36:28 smithi153 ceph-mon[70044]: pgmap v14644: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:30.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:36:30 smithi083 ceph-mon[79930]: pgmap v14645: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:30.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:36:30 smithi083 ceph-mon[77242]: pgmap v14645: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:30.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:36:30 smithi153 ceph-mon[70044]: pgmap v14645: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:36:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:36:31] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T13:36:32.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:36:32 smithi083 ceph-mon[79930]: pgmap v14646: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:32.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:36:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:36:31.985Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:36:32.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:36:32 smithi083 ceph-mon[77242]: pgmap v14646: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:32.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:36:32 smithi153 ceph-mon[70044]: pgmap v14646: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:32.559 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:36:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:36:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:36:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:36:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:36:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:36:34.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:36:34 smithi083 ceph-mon[79930]: pgmap v14647: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:34.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:36:34 smithi083 ceph-mon[77242]: pgmap v14647: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:34.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:36:34 smithi153 ceph-mon[70044]: pgmap v14647: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:35.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:36:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:36:35.349Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:36:36.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:36:36 smithi083 ceph-mon[79930]: pgmap v14648: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:36.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:36:36 smithi083 ceph-mon[77242]: pgmap v14648: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:36.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:36:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:36:36.073Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:36:36.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:36:36 smithi153 ceph-mon[70044]: pgmap v14648: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:38.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:36:38 smithi083 ceph-mon[79930]: pgmap v14649: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:38.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:36:38 smithi083 ceph-mon[77242]: pgmap v14649: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:38.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:36:38 smithi153 ceph-mon[70044]: pgmap v14649: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:36:40 smithi083 ceph-mon[79930]: pgmap v14650: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:36:40 smithi083 ceph-mon[77242]: pgmap v14650: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:36:40 smithi153 ceph-mon[70044]: pgmap v14650: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:41.204 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:36:41.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:36:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:36:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:36:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:36:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:36:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:36:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:36:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:36:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:36:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:36:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:36:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:36:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:36:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:36:41.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:36:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:36:41.489 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:36:41.490 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 21s ago 8h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:36:41.490 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 8m ago 8h 63.7M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:36:41.490 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 21s ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:36:41.491 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 8m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:36:41.491 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 8m ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:36:41.491 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 21s ago 8h 919M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:36:41.491 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 21s ago 8h 1806M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:36:41.492 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 8m ago 8h 766M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:36:41.492 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 21s ago 8h 1075M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:36:41.492 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 21s ago 8h 24.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:36:41.492 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 8m ago 8h 25.7M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:36:41.493 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 21s ago 8h 1308M 2305M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:36:41.493 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 21s ago 8h 1295M 2305M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:36:41.493 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 21s ago 8h 984M 2305M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:36:41.493 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 21s ago 8h 1268M 2305M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:36:41.493 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 8m ago 8h 1091M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:36:41.494 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 8m ago 8h 1368M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:36:41.494 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 8m ago 8h 1222M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:36:41.494 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 8m ago 8h 1452M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:36:41.494 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 8m ago 8h 182M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:36:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:36:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:36:41.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:36:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:36:41.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:36:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:36:41.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:36:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:36:41.847 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:36:41.847 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:36:41.847 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:36:41.847 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:36:41.848 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:36:41.848 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:36:41.848 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:36:41.848 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:36:41.849 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:36:41.849 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:36:41.849 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:36:41.849 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:36:41.850 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:36:41.850 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:36:41.850 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:36:41.850 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:36:41.851 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:36:41.851 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:36:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:36:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:36:41] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T13:36:42.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:36:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:36:41.984Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:36:42.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:36:42 smithi083 ceph-mon[77242]: pgmap v14651: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:42.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:36:42 smithi083 ceph-mon[77242]: from='client.51168 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:36:42.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:36:42 smithi083 ceph-mon[77242]: from='client.51174 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:36:42.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:36:42 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2029068939' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:36:42.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:36:42 smithi083 ceph-mon[79930]: pgmap v14651: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:42.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:36:42 smithi083 ceph-mon[79930]: from='client.51168 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:36:42.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:36:42 smithi083 ceph-mon[79930]: from='client.51174 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:36:42.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:36:42 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2029068939' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:36:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:36:42 smithi153 ceph-mon[70044]: pgmap v14651: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:42.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:36:42 smithi153 ceph-mon[70044]: from='client.51168 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:36:42.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:36:42 smithi153 ceph-mon[70044]: from='client.51174 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:36:42.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:36:42 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2029068939' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:36:42.560 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:36:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:36:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:36:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:36:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:36:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:36:44.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:36:44 smithi083 ceph-mon[79930]: pgmap v14652: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:44.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:36:44 smithi083 ceph-mon[77242]: pgmap v14652: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:44.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:36:44 smithi153 ceph-mon[70044]: pgmap v14652: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:45.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:36:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:36:45.349Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:36:46.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:36:46 smithi083 ceph-mon[79930]: pgmap v14653: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:46.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:36:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:36:46.074Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:36:46.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:36:46 smithi083 ceph-mon[77242]: pgmap v14653: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:46.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:36:46 smithi153 ceph-mon[70044]: pgmap v14653: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:48.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:36:48 smithi083 ceph-mon[79930]: pgmap v14654: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:48.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:36:48 smithi083 ceph-mon[77242]: pgmap v14654: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:48.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:36:48 smithi153 ceph-mon[70044]: pgmap v14654: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:50.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:36:50 smithi083 ceph-mon[79930]: pgmap v14655: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:50.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:36:50 smithi083 ceph-mon[77242]: pgmap v14655: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:50.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:36:50 smithi153 ceph-mon[70044]: pgmap v14655: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:36:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:36:51] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T13:36:52.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:36:52 smithi083 ceph-mon[79930]: pgmap v14656: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:52.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:36:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:36:51.985Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:36:52.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:36:52 smithi083 ceph-mon[77242]: pgmap v14656: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:52.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:36:52 smithi153 ceph-mon[70044]: pgmap v14656: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:52.559 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:36:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:36:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:36:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:36:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:36:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:36:54.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:36:54 smithi083 ceph-mon[79930]: pgmap v14657: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:54.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:36:54 smithi083 ceph-mon[77242]: pgmap v14657: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:54.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:36:54 smithi153 ceph-mon[70044]: pgmap v14657: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:55.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:36:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:36:55.349Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:36:56.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:36:56 smithi083 ceph-mon[77242]: pgmap v14658: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:56.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:36:56 smithi083 ceph-mon[79930]: pgmap v14658: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:56.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:36:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:36:56.074Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:36:56.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:36:56 smithi153 ceph-mon[70044]: pgmap v14658: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:58.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:36:58 smithi083 ceph-mon[79930]: pgmap v14659: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:58.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:36:58 smithi083 ceph-mon[77242]: pgmap v14659: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:36:58.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:36:58 smithi153 ceph-mon[70044]: pgmap v14659: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:00.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:37:00 smithi083 ceph-mon[79930]: pgmap v14660: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:00.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:37:00 smithi083 ceph-mon[77242]: pgmap v14660: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:00.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:37:00 smithi153 ceph-mon[70044]: pgmap v14660: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:01.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:37:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:37:01] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T13:37:02.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:37:02 smithi083 ceph-mon[79930]: pgmap v14661: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:02.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:37:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:37:01.986Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:37:02.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:37:02 smithi083 ceph-mon[77242]: pgmap v14661: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:02.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:37:02 smithi153 ceph-mon[70044]: pgmap v14661: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:02.560 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:37:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:37:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:37:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:37:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:37:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:37:04.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:37:04 smithi083 ceph-mon[77242]: pgmap v14662: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:04.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:37:04 smithi083 ceph-mon[79930]: pgmap v14662: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:04.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:37:04 smithi153 ceph-mon[70044]: pgmap v14662: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:05.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:37:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:37:05.350Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:37:06.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:37:06 smithi083 ceph-mon[79930]: pgmap v14663: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:06.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:37:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:37:06.075Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:37:06.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:37:06 smithi083 ceph-mon[77242]: pgmap v14663: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:06.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:37:06 smithi153 ceph-mon[70044]: pgmap v14663: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:08.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:37:08 smithi083 ceph-mon[79930]: pgmap v14664: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:08.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:37:08 smithi083 ceph-mon[77242]: pgmap v14664: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:08.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:37:08 smithi153 ceph-mon[70044]: pgmap v14664: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:10.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:37:10 smithi083 ceph-mon[77242]: pgmap v14665: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:10.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:37:10 smithi083 ceph-mon[79930]: pgmap v14665: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:10.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:37:10 smithi153 ceph-mon[70044]: pgmap v14665: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:37:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:37:11] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T13:37:12.175 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:37:12.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:37:12 smithi083 ceph-mon[79930]: pgmap v14666: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:12.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:37:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:37:11.986Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:37:12.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:37:12 smithi083 ceph-mon[77242]: pgmap v14666: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:12.466 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:37:12.466 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 52s ago 8h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:37:12.467 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 9m ago 8h 63.7M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:37:12.467 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 52s ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:37:12.467 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 9m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:37:12.468 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 9m ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:37:12.468 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 52s ago 8h 919M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:37:12.469 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 52s ago 8h 1806M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:37:12.469 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 9m ago 8h 766M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:37:12.470 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 52s ago 8h 1075M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:37:12.470 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 52s ago 8h 24.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:37:12.471 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 9m ago 8h 25.7M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:37:12.471 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 52s ago 8h 1308M 2305M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:37:12.471 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 52s ago 8h 1295M 2305M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:37:12.472 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 52s ago 8h 984M 2305M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:37:12.472 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 52s ago 8h 1268M 2305M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:37:12.473 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 9m ago 8h 1091M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:37:12.473 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 9m ago 8h 1368M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:37:12.474 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 9m ago 8h 1222M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:37:12.474 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 9m ago 8h 1452M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:37:12.475 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 9m ago 8h 182M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:37:12.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:37:12 smithi153 ceph-mon[70044]: pgmap v14666: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:12.560 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:37:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:37:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:37:12.823 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:37:12.824 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:37:12.824 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:37:12.824 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:37:12.825 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:37:12.825 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:37:12.825 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:37:12.825 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:37:12.826 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:37:12.826 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:37:12.826 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:37:12.826 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:37:12.827 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:37:12.827 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:37:12.827 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:37:12.827 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:37:12.828 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:37:12.828 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:37:13.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:37:13 smithi153 ceph-mon[70044]: from='client.51186 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:37:13.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:37:13 smithi153 ceph-mon[70044]: from='client.51192 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:37:13.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:37:13 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1475538237' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:37:13.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:37:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:37:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:37:13.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:37:13 smithi083 ceph-mon[77242]: from='client.51186 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:37:13.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:37:13 smithi083 ceph-mon[77242]: from='client.51192 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:37:13.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:37:13 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1475538237' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:37:13.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:37:13 smithi083 ceph-mon[79930]: from='client.51186 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:37:13.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:37:13 smithi083 ceph-mon[79930]: from='client.51192 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:37:13.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:37:13 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1475538237' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:37:14.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:37:14 smithi153 ceph-mon[70044]: pgmap v14667: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:14.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:37:14 smithi083 ceph-mon[79930]: pgmap v14667: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:14.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:37:14 smithi083 ceph-mon[77242]: pgmap v14667: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:15.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:37:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:37:15.351Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:37:16.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:37:16 smithi083 ceph-mon[79930]: pgmap v14668: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:16.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:37:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:37:16.075Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:37:16.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:37:16 smithi083 ceph-mon[77242]: pgmap v14668: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:16.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:37:16 smithi153 ceph-mon[70044]: pgmap v14668: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:18.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:37:18 smithi153 ceph-mon[70044]: pgmap v14669: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:18.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:37:18 smithi083 ceph-mon[77242]: pgmap v14669: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:18.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:37:18 smithi083 ceph-mon[79930]: pgmap v14669: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:20.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:37:20 smithi153 ceph-mon[70044]: pgmap v14670: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:20.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:37:20 smithi083 ceph-mon[79930]: pgmap v14670: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:20.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:37:20 smithi083 ceph-mon[77242]: pgmap v14670: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:21.507 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:37:21 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:37:21.508 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:37:21 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:37:21.508 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:37:21 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:37:21.509 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:37:21 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:37:21.509 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:37:21 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:37:21.509 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:37:21 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:37:21.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:37:21 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:37:21.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:37:21 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:37:21.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:37:21 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:37:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:37:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:37:21] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T13:37:22.279 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:37:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:37:21.987Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:37:22.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:37:22 smithi153 ceph-mon[70044]: pgmap v14671: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:22.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:37:22 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:37:22.559 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:37:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:37:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:37:22.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:37:22 smithi083 ceph-mon[79930]: pgmap v14671: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:22.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:37:22 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:37:22.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:37:22 smithi083 ceph-mon[77242]: pgmap v14671: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:22.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:37:22 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:37:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:37:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:37:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:37:24.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:37:24 smithi153 ceph-mon[70044]: pgmap v14672: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:24.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:37:24 smithi083 ceph-mon[79930]: pgmap v14672: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:24.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:37:24 smithi083 ceph-mon[77242]: pgmap v14672: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:25.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:37:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:37:25.352Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:37:26.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:37:26 smithi083 ceph-mon[79930]: pgmap v14673: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:26.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:37:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:37:26.076Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:37:26.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:37:26 smithi083 ceph-mon[77242]: pgmap v14673: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:26.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:37:26 smithi153 ceph-mon[70044]: pgmap v14673: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:28.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:37:28 smithi153 ceph-mon[70044]: pgmap v14674: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:28.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:37:28 smithi083 ceph-mon[79930]: pgmap v14674: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:28.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:37:28 smithi083 ceph-mon[77242]: pgmap v14674: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:30.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:37:30 smithi153 ceph-mon[70044]: pgmap v14675: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:30.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:37:30 smithi083 ceph-mon[79930]: pgmap v14675: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:30.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:37:30 smithi083 ceph-mon[77242]: pgmap v14675: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:31.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:37:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:37:31] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T13:37:32.295 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:37:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:37:31.987Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:37:32.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:37:32 smithi153 ceph-mon[70044]: pgmap v14676: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:32.559 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:37:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:37:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:37:32.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:37:32 smithi083 ceph-mon[77242]: pgmap v14676: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:32.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:37:32 smithi083 ceph-mon[79930]: pgmap v14676: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:33.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:37:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:37:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:37:34.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:37:34 smithi153 ceph-mon[70044]: pgmap v14677: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:34.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:37:34 smithi083 ceph-mon[77242]: pgmap v14677: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:34.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:37:34 smithi083 ceph-mon[79930]: pgmap v14677: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:35.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:37:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:37:35.352Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:37:36.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:37:36 smithi083 ceph-mon[79930]: pgmap v14678: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:36.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:37:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:37:36.077Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:37:36.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:37:36 smithi083 ceph-mon[77242]: pgmap v14678: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:36.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:37:36 smithi153 ceph-mon[70044]: pgmap v14678: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:38.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:37:38 smithi153 ceph-mon[70044]: pgmap v14679: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:38.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:37:38 smithi083 ceph-mon[79930]: pgmap v14679: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:38.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:37:38 smithi083 ceph-mon[77242]: pgmap v14679: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:37:40 smithi153 ceph-mon[70044]: pgmap v14680: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:37:40 smithi083 ceph-mon[79930]: pgmap v14680: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:37:40 smithi083 ceph-mon[77242]: pgmap v14680: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:41.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:37:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:37:41] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T13:37:41.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:37:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:37:41.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:37:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:37:41.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:37:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:37:41.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:37:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:37:41.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:37:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:37:41.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:37:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:37:41.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:37:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:37:41.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:37:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:37:41.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:37:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:37:41.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:37:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:37:41.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:37:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:37:41.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:37:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:37:42.314 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:37:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:37:41.988Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:37:42.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:37:42 smithi083 ceph-mon[79930]: pgmap v14681: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:37:42 smithi083 ceph-mon[77242]: pgmap v14681: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:37:42 smithi153 ceph-mon[70044]: pgmap v14681: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:42.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:37:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:37:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:37:43.150 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:37:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:37:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:37:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:37:43.445 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:37:43.446 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 83s ago 8h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:37:43.446 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 9m ago 8h 63.7M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:37:43.446 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 83s ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:37:43.447 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 9m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:37:43.447 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 9m ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:37:43.447 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 83s ago 8h 919M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:37:43.447 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 83s ago 8h 1806M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:37:43.448 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 9m ago 8h 766M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:37:43.448 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 83s ago 8h 1075M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:37:43.448 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 83s ago 8h 24.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:37:43.448 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 9m ago 8h 25.7M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:37:43.448 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 83s ago 8h 1308M 2305M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:37:43.449 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 83s ago 8h 1295M 2305M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:37:43.449 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 83s ago 8h 984M 2305M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:37:43.449 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 83s ago 8h 1268M 2305M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:37:43.449 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 9m ago 8h 1091M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:37:43.450 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 9m ago 8h 1368M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:37:43.450 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 9m ago 8h 1222M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:37:43.450 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 9m ago 8h 1452M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:37:43.450 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 9m ago 8h 182M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:37:43.808 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:37:43.808 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:37:43.808 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:37:43.809 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:37:43.809 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:37:43.809 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:37:43.809 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:37:43.810 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:37:43.810 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:37:43.810 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:37:43.810 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:37:43.810 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:37:43.811 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:37:43.811 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:37:43.811 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:37:43.811 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:37:43.812 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:37:43.812 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:37:44.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:37:44 smithi083 ceph-mon[79930]: pgmap v14682: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:44.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:37:44 smithi083 ceph-mon[79930]: from='client.61135 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:37:44.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:37:44 smithi083 ceph-mon[79930]: from='client.61141 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:37:44.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:37:44 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2039771999' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:37:44.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:37:44 smithi083 ceph-mon[77242]: pgmap v14682: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:44.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:37:44 smithi083 ceph-mon[77242]: from='client.61135 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:37:44.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:37:44 smithi083 ceph-mon[77242]: from='client.61141 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:37:44.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:37:44 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2039771999' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:37:44.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:37:44 smithi153 ceph-mon[70044]: pgmap v14682: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:44.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:37:44 smithi153 ceph-mon[70044]: from='client.61135 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:37:44.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:37:44 smithi153 ceph-mon[70044]: from='client.61141 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:37:44.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:37:44 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2039771999' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:37:45.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:37:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:37:45.353Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:37:46.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:37:46 smithi083 ceph-mon[79930]: pgmap v14683: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:46.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:37:46 smithi083 ceph-mon[77242]: pgmap v14683: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:46.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:37:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:37:46.078Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:37:46.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:37:46 smithi153 ceph-mon[70044]: pgmap v14683: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:48.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:37:48 smithi083 ceph-mon[77242]: pgmap v14684: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:48.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:37:48 smithi083 ceph-mon[79930]: pgmap v14684: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:48.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:37:48 smithi153 ceph-mon[70044]: pgmap v14684: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:50.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:37:50 smithi083 ceph-mon[79930]: pgmap v14685: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:50.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:37:50 smithi083 ceph-mon[77242]: pgmap v14685: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:50.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:37:50 smithi153 ceph-mon[70044]: pgmap v14685: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:37:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:37:51] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T13:37:52.330 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:37:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:37:51.989Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:37:52.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:37:52 smithi083 ceph-mon[79930]: pgmap v14686: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:52.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:37:52 smithi083 ceph-mon[77242]: pgmap v14686: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:52.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:37:52 smithi153 ceph-mon[70044]: pgmap v14686: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:52.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:37:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:37:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:37:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:37:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:37:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:37:54.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:37:54 smithi083 ceph-mon[79930]: pgmap v14687: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:54.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:37:54 smithi083 ceph-mon[77242]: pgmap v14687: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:54.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:37:54 smithi153 ceph-mon[70044]: pgmap v14687: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:55.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:37:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:37:55.354Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:37:56.338 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:37:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:37:56.078Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:37:56.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:37:56 smithi083 ceph-mon[77242]: pgmap v14688: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:56.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:37:56 smithi083 ceph-mon[79930]: pgmap v14688: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:56.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:37:56 smithi153 ceph-mon[70044]: pgmap v14688: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:58.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:37:58 smithi083 ceph-mon[79930]: pgmap v14689: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:58.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:37:58 smithi083 ceph-mon[77242]: pgmap v14689: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:37:58.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:37:58 smithi153 ceph-mon[70044]: pgmap v14689: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:00.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:38:00 smithi083 ceph-mon[79930]: pgmap v14690: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:00.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:38:00 smithi083 ceph-mon[77242]: pgmap v14690: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:00.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:38:00 smithi153 ceph-mon[70044]: pgmap v14690: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:38:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:38:01] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T13:38:02.346 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:38:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:38:01.990Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:38:02.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:38:02 smithi083 ceph-mon[77242]: pgmap v14691: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:02.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:38:02 smithi083 ceph-mon[79930]: pgmap v14691: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:02.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:38:02 smithi153 ceph-mon[70044]: pgmap v14691: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:02.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:38:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:38:02.316Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:38:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:38:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:38:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:38:04.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:38:04 smithi083 ceph-mon[77242]: pgmap v14692: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:04.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:38:04 smithi083 ceph-mon[79930]: pgmap v14692: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:04.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:38:04 smithi153 ceph-mon[70044]: pgmap v14692: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:05.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:38:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:38:05.354Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:38:06.351 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:38:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:38:06.078Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:38:06.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:38:06 smithi083 ceph-mon[77242]: pgmap v14693: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:06.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:38:06 smithi083 ceph-mon[79930]: pgmap v14693: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:06.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:38:06 smithi153 ceph-mon[70044]: pgmap v14693: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:08.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:38:08 smithi083 ceph-mon[79930]: pgmap v14694: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:08.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:38:08 smithi083 ceph-mon[77242]: pgmap v14694: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:08.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:38:08 smithi153 ceph-mon[70044]: pgmap v14694: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:10.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:38:10 smithi083 ceph-mon[79930]: pgmap v14695: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:10.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:38:10 smithi083 ceph-mon[77242]: pgmap v14695: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:10.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:38:10 smithi153 ceph-mon[70044]: pgmap v14695: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:38:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:38:11] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T13:38:12.365 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:38:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:38:11.990Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:38:12.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:38:12 smithi083 ceph-mon[77242]: pgmap v14696: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:12.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:38:12 smithi083 ceph-mon[79930]: pgmap v14696: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:12.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:38:12 smithi153 ceph-mon[70044]: pgmap v14696: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:12.810 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:38:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:38:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:38:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:38:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:38:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:38:14.133 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:38:14.421 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:38:14.422 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 114s ago 8h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:38:14.422 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 10m ago 8h 63.7M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:38:14.422 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 114s ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:38:14.423 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 10m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:38:14.423 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 10m ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:38:14.423 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 114s ago 8h 919M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:38:14.423 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 114s ago 8h 1806M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:38:14.424 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 10m ago 8h 766M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:38:14.424 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 114s ago 8h 1075M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:38:14.424 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 114s ago 8h 24.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:38:14.424 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 10m ago 8h 25.7M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:38:14.425 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 114s ago 8h 1308M 2305M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:38:14.425 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 114s ago 8h 1295M 2305M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:38:14.425 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 114s ago 8h 984M 2305M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:38:14.425 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 114s ago 8h 1268M 2305M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:38:14.426 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 10m ago 8h 1091M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:38:14.426 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 10m ago 8h 1368M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:38:14.426 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 10m ago 8h 1222M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:38:14.426 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 10m ago 8h 1452M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:38:14.427 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 10m ago 8h 182M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:38:14.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:38:14 smithi083 ceph-mon[77242]: pgmap v14697: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:14.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:38:14 smithi083 ceph-mon[79930]: pgmap v14697: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:14.777 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:38:14.778 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:38:14.778 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:38:14.778 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:38:14.779 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:38:14.779 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:38:14.779 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:38:14.779 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:38:14.780 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:38:14.780 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:38:14.780 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:38:14.780 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:38:14.780 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:38:14.781 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:38:14.781 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:38:14.781 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:38:14.781 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:38:14.782 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:38:14.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:38:14 smithi153 ceph-mon[70044]: pgmap v14697: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:15.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:38:15 smithi083 ceph-mon[79930]: from='client.61153 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:38:15.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:38:15 smithi083 ceph-mon[79930]: from='client.51228 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:38:15.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:38:15 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/267516527' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:38:15.714 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:38:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:38:15.354Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:38:15.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:38:15 smithi083 ceph-mon[77242]: from='client.61153 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:38:15.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:38:15 smithi083 ceph-mon[77242]: from='client.51228 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:38:15.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:38:15 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/267516527' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:38:15.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:38:15 smithi153 ceph-mon[70044]: from='client.61153 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:38:15.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:38:15 smithi153 ceph-mon[70044]: from='client.51228 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:38:15.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:38:15 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/267516527' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:38:16.379 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:38:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:38:16.080Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:38:16.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:38:16 smithi083 ceph-mon[79930]: pgmap v14698: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:16.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:38:16 smithi083 ceph-mon[77242]: pgmap v14698: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:16.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:38:16 smithi153 ceph-mon[70044]: pgmap v14698: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:18.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:38:18 smithi083 ceph-mon[77242]: pgmap v14699: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:18.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:38:18 smithi083 ceph-mon[79930]: pgmap v14699: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:18.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:38:18 smithi153 ceph-mon[70044]: pgmap v14699: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:20.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:38:20 smithi083 ceph-mon[77242]: pgmap v14700: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:20.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:38:20 smithi083 ceph-mon[79930]: pgmap v14700: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:20.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:38:20 smithi153 ceph-mon[70044]: pgmap v14700: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:21.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:38:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:38:21] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T13:38:22.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:38:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:38:21.990Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:38:22.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:38:22 smithi083 ceph-mon[79930]: pgmap v14701: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:22.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:38:22 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:38:22.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:38:22 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:38:22.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:38:22 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:38:22.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:38:22 smithi083 ceph-mon[77242]: pgmap v14701: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:22.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:38:22 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:38:22.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:38:22 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:38:22.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:38:22 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:38:22.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:38:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:38:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:38:22.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:38:22 smithi153 ceph-mon[70044]: pgmap v14701: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:22.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:38:22 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:38:22.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:38:22 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:38:22.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:38:22 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:38:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:38:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:38:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:38:24.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:38:24 smithi083 ceph-mon[77242]: pgmap v14702: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:24.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:38:24 smithi083 ceph-mon[79930]: pgmap v14702: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:24.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:38:24 smithi153 ceph-mon[70044]: pgmap v14702: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:25.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:38:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:38:25.356Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:38:26.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:38:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:38:26.081Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:38:26.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:38:26 smithi083 ceph-mon[79930]: pgmap v14703: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:26.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:38:26 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:38:26.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:38:26 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:38:26.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:38:26 smithi083 ceph-mon[77242]: pgmap v14703: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:26.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:38:26 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:38:26.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:38:26 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:38:26.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:38:26 smithi153 ceph-mon[70044]: pgmap v14703: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:26.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:38:26 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:38:26.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:38:26 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:38:27.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:38:27 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:38:27.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:38:27 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:38:27.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:38:27 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:38:28.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:38:28 smithi083 ceph-mon[79930]: pgmap v14704: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:28.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:38:28 smithi083 ceph-mon[77242]: pgmap v14704: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:28.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:38:28 smithi153 ceph-mon[70044]: pgmap v14704: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:30.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:38:30 smithi083 ceph-mon[79930]: pgmap v14705: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:30.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:38:30 smithi083 ceph-mon[77242]: pgmap v14705: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:30.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:38:30 smithi153 ceph-mon[70044]: pgmap v14705: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:38:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:38:31] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T13:38:32.454 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:38:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:38:31.992Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:38:32.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:38:32 smithi083 ceph-mon[77242]: pgmap v14706: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:32.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:38:32 smithi083 ceph-mon[79930]: pgmap v14706: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:32.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:38:32 smithi153 ceph-mon[70044]: pgmap v14706: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:32.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:38:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:38:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:38:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:38:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:38:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:38:34.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:38:34 smithi083 ceph-mon[79930]: pgmap v14707: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:34.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:38:34 smithi083 ceph-mon[77242]: pgmap v14707: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:34.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:38:34 smithi153 ceph-mon[70044]: pgmap v14707: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:35.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:38:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:38:35.356Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:38:36.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:38:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:38:36.082Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:38:36.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:38:36 smithi083 ceph-mon[79930]: pgmap v14708: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:36.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:38:36 smithi083 ceph-mon[77242]: pgmap v14708: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:36.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:38:36 smithi153 ceph-mon[70044]: pgmap v14708: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:38.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:38:38 smithi153 ceph-mon[70044]: pgmap v14709: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:38.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:38:38 smithi083 ceph-mon[79930]: pgmap v14709: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:38.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:38:38 smithi083 ceph-mon[77242]: pgmap v14709: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:38:40 smithi153 ceph-mon[70044]: pgmap v14710: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:38:40 smithi083 ceph-mon[79930]: pgmap v14710: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:38:40 smithi083 ceph-mon[77242]: pgmap v14710: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:41.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:38:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:38:41.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:38:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:38:41.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:38:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:38:41.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:38:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:38:41.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:38:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:38:41] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T13:38:41.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:38:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:38:41.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:38:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:38:41.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:38:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:38:41.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:38:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:38:41.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:38:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:38:41.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:38:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:38:41.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:38:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:38:41.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:38:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:38:42.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:38:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:38:41.993Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:38:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:38:42 smithi153 ceph-mon[70044]: pgmap v14711: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:42.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:38:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:38:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:38:42.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:38:42 smithi083 ceph-mon[79930]: pgmap v14711: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:42.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:38:42 smithi083 ceph-mon[77242]: pgmap v14711: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:38:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:38:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:38:44.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:38:44 smithi153 ceph-mon[70044]: pgmap v14712: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:44.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:38:44 smithi083 ceph-mon[79930]: pgmap v14712: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:44.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:38:44 smithi083 ceph-mon[77242]: pgmap v14712: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:45.104 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:38:45.394 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:38:45.395 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 2m ago 8h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:38:45.395 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 19s ago 8h 64.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:38:45.395 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 2m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:38:45.396 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 19s ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:38:45.396 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 19s ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:38:45.396 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 2m ago 8h 919M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:38:45.396 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 2m ago 8h 1806M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:38:45.397 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 19s ago 8h 783M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:38:45.397 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 2m ago 8h 1075M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:38:45.397 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 2m ago 8h 24.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:38:45.397 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 19s ago 8h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:38:45.398 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 2m ago 8h 1308M 2305M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:38:45.398 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 2m ago 8h 1295M 2305M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:38:45.398 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 2m ago 8h 984M 2305M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:38:45.399 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 2m ago 8h 1268M 2305M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:38:45.399 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 19s ago 8h 1114M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:38:45.399 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 19s ago 8h 1395M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:38:45.399 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 19s ago 8h 1247M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:38:45.400 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 19s ago 8h 1481M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:38:45.400 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 19s ago 8h 183M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:38:45.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:38:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:38:45.356Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:38:45.753 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:38:45.753 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:38:45.753 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:38:45.753 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:38:45.754 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:38:45.754 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:38:45.754 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:38:45.755 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:38:45.755 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:38:45.755 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:38:45.755 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:38:45.756 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:38:45.756 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:38:45.756 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:38:45.756 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:38:45.757 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:38:45.757 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:38:45.757 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:38:46.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:38:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:38:46.081Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:38:46.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:38:46 smithi153 ceph-mon[70044]: pgmap v14713: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:46.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:38:46 smithi153 ceph-mon[70044]: from='client.51240 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:38:46.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:38:46 smithi153 ceph-mon[70044]: from='client.61177 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:38:46.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:38:46 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3847839009' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:38:46.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:38:46 smithi083 ceph-mon[79930]: pgmap v14713: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:46.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:38:46 smithi083 ceph-mon[79930]: from='client.51240 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:38:46.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:38:46 smithi083 ceph-mon[79930]: from='client.61177 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:38:46.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:38:46 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3847839009' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:38:46.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:38:46 smithi083 ceph-mon[77242]: pgmap v14713: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:46.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:38:46 smithi083 ceph-mon[77242]: from='client.51240 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:38:46.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:38:46 smithi083 ceph-mon[77242]: from='client.61177 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:38:46.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:38:46 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3847839009' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:38:48.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:38:48 smithi153 ceph-mon[70044]: pgmap v14714: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:48.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:38:48 smithi083 ceph-mon[79930]: pgmap v14714: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:48.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:38:48 smithi083 ceph-mon[77242]: pgmap v14714: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:50.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:38:50 smithi153 ceph-mon[70044]: pgmap v14715: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:50.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:38:50 smithi083 ceph-mon[79930]: pgmap v14715: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:50.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:38:50 smithi083 ceph-mon[77242]: pgmap v14715: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:51.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:38:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:38:51] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T13:38:52.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:38:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:38:51.993Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:38:52.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:38:52 smithi153 ceph-mon[70044]: pgmap v14716: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:52.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:38:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:38:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:38:52.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:38:52 smithi083 ceph-mon[79930]: pgmap v14716: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:52.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:38:52 smithi083 ceph-mon[77242]: pgmap v14716: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:38:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:38:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:38:54.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:38:54 smithi153 ceph-mon[70044]: pgmap v14717: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:54.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:38:54 smithi083 ceph-mon[79930]: pgmap v14717: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:54.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:38:54 smithi083 ceph-mon[77242]: pgmap v14717: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:55.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:38:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:38:55.356Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:38:56.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:38:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:38:56.083Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:38:56.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:38:56 smithi153 ceph-mon[70044]: pgmap v14718: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:56.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:38:56 smithi083 ceph-mon[79930]: pgmap v14718: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:56.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:38:56 smithi083 ceph-mon[77242]: pgmap v14718: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:58.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:38:58 smithi153 ceph-mon[70044]: pgmap v14719: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:58.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:38:58 smithi083 ceph-mon[77242]: pgmap v14719: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:38:58.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:38:58 smithi083 ceph-mon[79930]: pgmap v14719: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:00.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:39:00 smithi153 ceph-mon[70044]: pgmap v14720: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:00.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:39:00 smithi083 ceph-mon[77242]: pgmap v14720: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:00.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:39:00 smithi083 ceph-mon[79930]: pgmap v14720: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:39:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:39:01] "GET /metrics HTTP/1.1" 200 225232 "" "Prometheus/2.33.4" 2022-09-29T13:39:02.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:39:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:39:01.995Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:39:02.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:39:02 smithi153 ceph-mon[70044]: pgmap v14721: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:02.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:39:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:39:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:39:02.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:39:02 smithi083 ceph-mon[79930]: pgmap v14721: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:02.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:39:02 smithi083 ceph-mon[77242]: pgmap v14721: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:39:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:39:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:39:04.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:39:04 smithi153 ceph-mon[70044]: pgmap v14722: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:04.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:39:04 smithi083 ceph-mon[79930]: pgmap v14722: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:04.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:39:04 smithi083 ceph-mon[77242]: pgmap v14722: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:05.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:39:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:39:05.357Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:39:06.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:39:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:39:06.083Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:39:06.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:39:06 smithi153 ceph-mon[70044]: pgmap v14723: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:06.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:39:06 smithi083 ceph-mon[77242]: pgmap v14723: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:06.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:39:06 smithi083 ceph-mon[79930]: pgmap v14723: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:08.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:39:08 smithi153 ceph-mon[70044]: pgmap v14724: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:08.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:39:08 smithi083 ceph-mon[79930]: pgmap v14724: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:08.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:39:08 smithi083 ceph-mon[77242]: pgmap v14724: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:10.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:39:10 smithi153 ceph-mon[70044]: pgmap v14725: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:10.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:39:10 smithi083 ceph-mon[79930]: pgmap v14725: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:10.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:39:10 smithi083 ceph-mon[77242]: pgmap v14725: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:39:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:39:11] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T13:39:12.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:39:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:39:11.996Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:39:12.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:39:12 smithi153 ceph-mon[70044]: pgmap v14726: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:12.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:39:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:39:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:39:12.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:39:12 smithi083 ceph-mon[77242]: pgmap v14726: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:12.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:39:12 smithi083 ceph-mon[79930]: pgmap v14726: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:39:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:39:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:39:14.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:39:14 smithi153 ceph-mon[70044]: pgmap v14727: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:14.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:39:14 smithi083 ceph-mon[79930]: pgmap v14727: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:14.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:39:14 smithi083 ceph-mon[77242]: pgmap v14727: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:15.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:39:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:39:15.358Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:39:16.079 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:39:16.370 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:39:16.370 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 2m ago 8h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:39:16.371 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 50s ago 8h 64.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:39:16.371 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 2m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:39:16.371 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 50s ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:39:16.372 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 50s ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:39:16.372 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 2m ago 8h 919M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:39:16.372 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 2m ago 8h 1806M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:39:16.373 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 50s ago 8h 783M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:39:16.373 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 2m ago 8h 1075M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:39:16.373 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 2m ago 8h 24.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:39:16.373 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 50s ago 8h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:39:16.373 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 2m ago 8h 1308M 2305M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:39:16.374 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 2m ago 8h 1295M 2305M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:39:16.374 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 2m ago 8h 984M 2305M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:39:16.374 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 2m ago 8h 1268M 2305M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:39:16.375 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 50s ago 8h 1114M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:39:16.375 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 50s ago 8h 1395M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:39:16.375 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 50s ago 8h 1247M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:39:16.375 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 50s ago 8h 1481M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:39:16.375 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 50s ago 8h 183M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:39:16.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:39:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:39:16.083Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:39:16.729 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:39:16.729 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:39:16.729 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:39:16.730 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:39:16.730 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:39:16.730 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:39:16.730 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:39:16.731 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:39:16.731 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:39:16.731 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:39:16.731 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:39:16.731 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:39:16.732 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:39:16.732 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:39:16.732 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:39:16.732 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:39:16.733 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:39:16.733 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:39:16.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:39:16 smithi153 ceph-mon[70044]: pgmap v14728: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:16.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:39:16 smithi083 ceph-mon[79930]: pgmap v14728: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:16.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:39:16 smithi083 ceph-mon[77242]: pgmap v14728: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:17.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:39:17 smithi153 ceph-mon[70044]: from='client.51258 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:39:17.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:39:17 smithi153 ceph-mon[70044]: from='client.61195 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:39:17.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:39:17 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3000474226' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:39:17.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:39:17 smithi083 ceph-mon[79930]: from='client.51258 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:39:17.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:39:17 smithi083 ceph-mon[79930]: from='client.61195 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:39:17.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:39:17 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3000474226' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:39:17.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:39:17 smithi083 ceph-mon[77242]: from='client.51258 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:39:17.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:39:17 smithi083 ceph-mon[77242]: from='client.61195 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:39:17.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:39:17 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3000474226' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:39:18.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:39:18 smithi153 ceph-mon[70044]: pgmap v14729: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:18.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:39:18 smithi083 ceph-mon[79930]: pgmap v14729: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:18.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:39:18 smithi083 ceph-mon[77242]: pgmap v14729: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:20.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:39:20 smithi153 ceph-mon[70044]: pgmap v14730: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:20.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:39:20 smithi083 ceph-mon[79930]: pgmap v14730: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:20.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:39:20 smithi083 ceph-mon[77242]: pgmap v14730: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:21.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:39:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:39:21] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T13:39:22.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:39:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:39:21.996Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:39:22.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:39:22 smithi153 ceph-mon[70044]: pgmap v14731: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:22.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:39:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:39:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:39:22.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:39:22 smithi083 ceph-mon[77242]: pgmap v14731: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:22.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:39:22 smithi083 ceph-mon[79930]: pgmap v14731: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:39:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:39:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:39:24.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:39:24 smithi153 ceph-mon[70044]: pgmap v14732: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:24.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:39:24 smithi083 ceph-mon[79930]: pgmap v14732: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:24.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:39:24 smithi083 ceph-mon[77242]: pgmap v14732: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:25.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:39:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:39:25.358Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:39:26.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:39:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:39:26.084Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:39:26.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:39:26 smithi153 ceph-mon[70044]: pgmap v14733: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:26.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:39:26 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:39:26.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:39:26 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:39:26.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:39:26 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:39:26.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:39:26 smithi083 ceph-mon[79930]: pgmap v14733: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:26.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:39:26 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:39:26.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:39:26 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:39:26.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:39:26 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:39:26.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:39:26 smithi083 ceph-mon[77242]: pgmap v14733: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:26.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:39:26 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:39:26.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:39:26 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:39:26.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:39:26 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:39:28.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:39:28 smithi083 ceph-mon[77242]: pgmap v14734: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:28.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:39:28 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:39:28.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:39:28 smithi083 ceph-mon[79930]: pgmap v14734: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:28.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:39:28 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:39:28.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:39:28 smithi153 ceph-mon[70044]: pgmap v14734: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:28.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:39:28 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:39:30.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:39:30 smithi083 ceph-mon[79930]: pgmap v14735: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:30.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:39:30 smithi083 ceph-mon[77242]: pgmap v14735: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:30.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:39:30 smithi153 ceph-mon[70044]: pgmap v14735: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:39:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:39:31] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T13:39:32.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:39:32 smithi083 ceph-mon[79930]: pgmap v14736: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:32.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:39:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:39:31.997Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:39:32.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:39:32 smithi083 ceph-mon[77242]: pgmap v14736: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:32.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:39:32 smithi153 ceph-mon[70044]: pgmap v14736: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:32.559 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:39:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:39:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:39:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:39:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:39:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:39:34.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:39:34 smithi083 ceph-mon[79930]: pgmap v14737: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:34.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:39:34 smithi083 ceph-mon[77242]: pgmap v14737: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:34.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:39:34 smithi153 ceph-mon[70044]: pgmap v14737: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:35.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:39:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:39:35.359Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:39:36.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:39:36 smithi083 ceph-mon[79930]: pgmap v14738: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:36.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:39:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:39:36.085Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:39:36.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:39:36 smithi083 ceph-mon[77242]: pgmap v14738: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:36.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:39:36 smithi153 ceph-mon[70044]: pgmap v14738: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:38.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:39:38 smithi083 ceph-mon[77242]: pgmap v14739: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:38.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:39:38 smithi083 ceph-mon[79930]: pgmap v14739: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:38.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:39:38 smithi153 ceph-mon[70044]: pgmap v14739: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:39:40 smithi083 ceph-mon[79930]: pgmap v14740: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:39:40 smithi083 ceph-mon[77242]: pgmap v14740: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:39:40 smithi153 ceph-mon[70044]: pgmap v14740: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:41.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:39:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:39:41.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:39:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:39:41.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:39:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:39:41.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:39:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:39:41.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:39:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:39:41.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:39:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:39:41.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:39:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:39:41.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:39:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:39:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:39:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:39:41.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:39:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:39:41.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:39:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:39:41.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:39:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:39:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:39:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:39:41] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T13:39:42.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:39:42 smithi083 ceph-mon[79930]: pgmap v14741: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:42.518 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:39:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:39:41.997Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:39:42.519 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:39:42 smithi083 ceph-mon[77242]: pgmap v14741: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:39:42 smithi153 ceph-mon[70044]: pgmap v14741: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:42.559 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:39:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:39:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:39:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:39:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:39:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:39:44.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:39:44 smithi083 ceph-mon[77242]: pgmap v14742: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:44.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:39:44 smithi083 ceph-mon[79930]: pgmap v14742: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:44.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:39:44 smithi153 ceph-mon[70044]: pgmap v14742: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:45.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:39:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:39:45.360Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:39:46.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:39:46 smithi083 ceph-mon[79930]: pgmap v14743: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:46.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:39:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:39:46.085Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:39:46.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:39:46 smithi083 ceph-mon[77242]: pgmap v14743: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:46.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:39:46 smithi153 ceph-mon[70044]: pgmap v14743: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:47.049 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:39:47.338 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:39:47.339 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 3m ago 8h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:39:47.339 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 81s ago 8h 64.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:39:47.339 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 3m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:39:47.339 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 81s ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:39:47.340 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 81s ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:39:47.340 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 3m ago 8h 919M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:39:47.340 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 3m ago 8h 1806M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:39:47.340 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 81s ago 8h 783M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:39:47.341 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 3m ago 8h 1075M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:39:47.341 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 3m ago 8h 24.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:39:47.341 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 81s ago 8h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:39:47.342 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 3m ago 8h 1308M 2305M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:39:47.342 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 3m ago 8h 1295M 2305M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:39:47.342 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 3m ago 8h 984M 2305M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:39:47.342 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 3m ago 8h 1268M 2305M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:39:47.343 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 81s ago 8h 1114M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:39:47.343 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 81s ago 8h 1395M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:39:47.343 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 81s ago 8h 1247M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:39:47.343 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 81s ago 8h 1481M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:39:47.344 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 81s ago 8h 183M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:39:47.693 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:39:47.694 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:39:47.694 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:39:47.695 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:39:47.695 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:39:47.695 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:39:47.695 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:39:47.696 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:39:47.696 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:39:47.696 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:39:47.696 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:39:47.696 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:39:47.697 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:39:47.697 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:39:47.697 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:39:47.697 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:39:47.698 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:39:47.698 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:39:48.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:39:48 smithi083 ceph-mon[77242]: pgmap v14744: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:48.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:39:48 smithi083 ceph-mon[77242]: from='client.51276 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:39:48.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:39:48 smithi083 ceph-mon[77242]: from='client.51282 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:39:48.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:39:48 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2633776686' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:39:48.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:39:48 smithi083 ceph-mon[79930]: pgmap v14744: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:48.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:39:48 smithi083 ceph-mon[79930]: from='client.51276 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:39:48.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:39:48 smithi083 ceph-mon[79930]: from='client.51282 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:39:48.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:39:48 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2633776686' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:39:48.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:39:48 smithi153 ceph-mon[70044]: pgmap v14744: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:48.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:39:48 smithi153 ceph-mon[70044]: from='client.51276 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:39:48.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:39:48 smithi153 ceph-mon[70044]: from='client.51282 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:39:48.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:39:48 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2633776686' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:39:50.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:39:50 smithi083 ceph-mon[79930]: pgmap v14745: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:50.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:39:50 smithi083 ceph-mon[77242]: pgmap v14745: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:50.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:39:50 smithi153 ceph-mon[70044]: pgmap v14745: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:39:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:39:51] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T13:39:52.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:39:52 smithi083 ceph-mon[77242]: pgmap v14746: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:52.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:39:52 smithi083 ceph-mon[79930]: pgmap v14746: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:52.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:39:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:39:51.998Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:39:52.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:39:52 smithi153 ceph-mon[70044]: pgmap v14746: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:52.559 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:39:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:39:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:39:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:39:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:39:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:39:54.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:39:54 smithi153 ceph-mon[70044]: pgmap v14747: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:54.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:39:54 smithi083 ceph-mon[79930]: pgmap v14747: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:54.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:39:54 smithi083 ceph-mon[77242]: pgmap v14747: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:55.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:39:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:39:55.360Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:39:56.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:39:56 smithi083 ceph-mon[79930]: pgmap v14748: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:56.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:39:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:39:56.086Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:39:56.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:39:56 smithi083 ceph-mon[77242]: pgmap v14748: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:56.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:39:56 smithi153 ceph-mon[70044]: pgmap v14748: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:58.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:39:58 smithi153 ceph-mon[70044]: pgmap v14749: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:58.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:39:58 smithi083 ceph-mon[79930]: pgmap v14749: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:39:58.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:39:58 smithi083 ceph-mon[77242]: pgmap v14749: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:00.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:40:00 smithi083 ceph-mon[79930]: pgmap v14750: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:00.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:40:00 smithi083 ceph-mon[79930]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T13:40:00.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:40:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[77219]: 2022-09-29T13:39:59.999+0000 7f160f665700 -1 log_channel(cluster) log [ERR] : overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T13:40:00.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:40:00 smithi083 ceph-mon[77242]: pgmap v14750: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:00.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:40:00 smithi083 ceph-mon[77242]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T13:40:00.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:40:00 smithi153 ceph-mon[70044]: pgmap v14750: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:00.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:40:00 smithi153 ceph-mon[70044]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T13:40:01.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:40:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:40:01] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T13:40:02.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:40:02 smithi083 ceph-mon[79930]: pgmap v14751: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:02.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:40:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:40:02.001Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:40:02.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:40:02 smithi083 ceph-mon[77242]: pgmap v14751: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:02.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:40:02 smithi153 ceph-mon[70044]: pgmap v14751: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:02.559 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:40:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:40:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:40:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:40:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:40:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:40:04.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:40:04 smithi153 ceph-mon[70044]: pgmap v14752: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:04.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:40:04 smithi083 ceph-mon[79930]: pgmap v14752: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:04.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:40:04 smithi083 ceph-mon[77242]: pgmap v14752: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:05.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:40:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:40:05.360Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:40:06.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:40:06 smithi083 ceph-mon[79930]: pgmap v14753: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:06.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:40:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:40:06.086Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:40:06.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:40:06 smithi083 ceph-mon[77242]: pgmap v14753: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:06.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:40:06 smithi153 ceph-mon[70044]: pgmap v14753: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:08.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:40:08 smithi153 ceph-mon[70044]: pgmap v14754: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:08.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:40:08 smithi083 ceph-mon[79930]: pgmap v14754: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:08.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:40:08 smithi083 ceph-mon[77242]: pgmap v14754: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:10.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:40:10 smithi153 ceph-mon[70044]: pgmap v14755: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:10.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:40:10 smithi083 ceph-mon[79930]: pgmap v14755: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:10.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:40:10 smithi083 ceph-mon[77242]: pgmap v14755: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:11.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:40:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:40:11] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T13:40:12.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:40:12 smithi083 ceph-mon[79930]: pgmap v14756: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:12.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:40:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:40:11.999Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:40:12.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:40:12 smithi083 ceph-mon[77242]: pgmap v14756: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:12.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:40:12 smithi153 ceph-mon[70044]: pgmap v14756: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:12.559 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:40:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:40:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:40:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:40:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:40:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:40:14.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:40:14 smithi153 ceph-mon[70044]: pgmap v14757: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:14.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:40:14 smithi083 ceph-mon[79930]: pgmap v14757: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:14.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:40:14 smithi083 ceph-mon[77242]: pgmap v14757: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:15.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:40:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:40:15.361Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:40:16.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:40:16 smithi083 ceph-mon[79930]: pgmap v14758: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:16.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:40:16 smithi083 ceph-mon[77242]: pgmap v14758: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:16.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:40:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:40:16.087Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:40:16.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:40:16 smithi153 ceph-mon[70044]: pgmap v14758: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:18.019 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:40:18.311 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:40:18.380 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 3m ago 8h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:40:18.381 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 112s ago 8h 64.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:40:18.381 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 3m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:40:18.382 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 112s ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:40:18.382 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 112s ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:40:18.383 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 3m ago 8h 919M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:40:18.383 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 3m ago 8h 1806M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:40:18.383 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 112s ago 8h 783M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:40:18.384 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 3m ago 8h 1075M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:40:18.384 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 3m ago 8h 24.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:40:18.384 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 112s ago 8h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:40:18.384 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 3m ago 8h 1308M 2305M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:40:18.385 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 3m ago 8h 1295M 2305M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:40:18.385 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 3m ago 8h 984M 2305M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:40:18.385 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 3m ago 8h 1268M 2305M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:40:18.385 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 112s ago 8h 1114M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:40:18.386 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 112s ago 8h 1395M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:40:18.386 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 112s ago 8h 1247M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:40:18.386 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 112s ago 8h 1481M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:40:18.386 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 112s ago 8h 183M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:40:18.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:40:18 smithi153 ceph-mon[70044]: pgmap v14759: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:18.665 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:40:18.665 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:40:18.666 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:40:18.666 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:40:18.666 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:40:18.666 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:40:18.667 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:40:18.667 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:40:18.667 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:40:18.667 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:40:18.668 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:40:18.668 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:40:18.668 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:40:18.668 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:40:18.668 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:40:18.669 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:40:18.669 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:40:18.669 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:40:18.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:40:18 smithi083 ceph-mon[79930]: pgmap v14759: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:18.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:40:18 smithi083 ceph-mon[77242]: pgmap v14759: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:19.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:40:19 smithi153 ceph-mon[70044]: from='client.61225 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:40:19.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:40:19 smithi153 ceph-mon[70044]: from='client.51300 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:40:19.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:40:19 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3630340311' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:40:19.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:40:19 smithi083 ceph-mon[79930]: from='client.61225 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:40:19.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:40:19 smithi083 ceph-mon[79930]: from='client.51300 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:40:19.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:40:19 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3630340311' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:40:19.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:40:19 smithi083 ceph-mon[77242]: from='client.61225 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:40:19.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:40:19 smithi083 ceph-mon[77242]: from='client.51300 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:40:19.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:40:19 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3630340311' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:40:20.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:40:20 smithi153 ceph-mon[70044]: pgmap v14760: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:20.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:40:20 smithi083 ceph-mon[79930]: pgmap v14760: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:20.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:40:20 smithi083 ceph-mon[77242]: pgmap v14760: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:40:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:40:21] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T13:40:22.265 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:40:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:40:22.000Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:40:22.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:40:22 smithi153 ceph-mon[70044]: pgmap v14761: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:22.560 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:40:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:40:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:40:22.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:40:22 smithi083 ceph-mon[79930]: pgmap v14761: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:22.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:40:22 smithi083 ceph-mon[77242]: pgmap v14761: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:40:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:40:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:40:24.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:40:24 smithi153 ceph-mon[70044]: pgmap v14762: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:24.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:40:24 smithi083 ceph-mon[79930]: pgmap v14762: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:24.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:40:24 smithi083 ceph-mon[77242]: pgmap v14762: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:25.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:40:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:40:25.361Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:40:26.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:40:26 smithi083 ceph-mon[79930]: pgmap v14763: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:26.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:40:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:40:26.087Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:40:26.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:40:26 smithi083 ceph-mon[77242]: pgmap v14763: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:26.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:40:26 smithi153 ceph-mon[70044]: pgmap v14763: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:27.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:40:27 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:40:27.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:40:27 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:40:27.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:40:27 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:40:27.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:40:27 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:40:27.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:40:27 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:40:27.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:40:27 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:40:27.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:40:27 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:40:27.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:40:27 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:40:27.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:40:27 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:40:28.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:40:28 smithi153 ceph-mon[70044]: pgmap v14764: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:28.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:40:28 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:40:28.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:40:28 smithi083 ceph-mon[79930]: pgmap v14764: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:28.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:40:28 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:40:28.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:40:28 smithi083 ceph-mon[77242]: pgmap v14764: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:28.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:40:28 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:40:30.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:40:30 smithi153 ceph-mon[70044]: pgmap v14765: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:30.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:40:30 smithi083 ceph-mon[79930]: pgmap v14765: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:30.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:40:30 smithi083 ceph-mon[77242]: pgmap v14765: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:31.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:40:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:40:31] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T13:40:32.283 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:40:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:40:32.001Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:40:32.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:40:32 smithi153 ceph-mon[70044]: pgmap v14766: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:32.559 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:40:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:40:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:40:32.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:40:32 smithi083 ceph-mon[77242]: pgmap v14766: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:32.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:40:32 smithi083 ceph-mon[79930]: pgmap v14766: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:33.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:40:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:40:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:40:34.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:40:34 smithi153 ceph-mon[70044]: pgmap v14767: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:34.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:40:34 smithi083 ceph-mon[79930]: pgmap v14767: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:34.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:40:34 smithi083 ceph-mon[77242]: pgmap v14767: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:35.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:40:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:40:35.361Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:40:36.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:40:36 smithi083 ceph-mon[79930]: pgmap v14768: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:36.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:40:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:40:36.088Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:40:36.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:40:36 smithi083 ceph-mon[77242]: pgmap v14768: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:36.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:40:36 smithi153 ceph-mon[70044]: pgmap v14768: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:38.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:40:38 smithi153 ceph-mon[70044]: pgmap v14769: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:38.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:40:38 smithi083 ceph-mon[79930]: pgmap v14769: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:38.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:40:38 smithi083 ceph-mon[77242]: pgmap v14769: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:40:40 smithi153 ceph-mon[70044]: pgmap v14770: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:40:40 smithi083 ceph-mon[79930]: pgmap v14770: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:40:40 smithi083 ceph-mon[77242]: pgmap v14770: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:40:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:40:41.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:40:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:40:41.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:40:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:40:41.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:40:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:40:41.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:40:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:40:41] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T13:40:41.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:40:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:40:41.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:40:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:40:41.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:40:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:40:41.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:40:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:40:41.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:40:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:40:41.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:40:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:40:41.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:40:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:40:41.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:40:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:40:42.300 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:40:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:40:42.001Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:40:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:40:42 smithi153 ceph-mon[70044]: pgmap v14771: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:42.559 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:40:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:40:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:40:42.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:40:42 smithi083 ceph-mon[77242]: pgmap v14771: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:42.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:40:42 smithi083 ceph-mon[79930]: pgmap v14771: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:43.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:40:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:40:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:40:44.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:40:44 smithi153 ceph-mon[70044]: pgmap v14772: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:44.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:40:44 smithi083 ceph-mon[79930]: pgmap v14772: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:44.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:40:44 smithi083 ceph-mon[77242]: pgmap v14772: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:45.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:40:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:40:45.362Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:40:46.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:40:46 smithi083 ceph-mon[79930]: pgmap v14773: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:46.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:40:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:40:46.088Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:40:46.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:40:46 smithi083 ceph-mon[77242]: pgmap v14773: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:46.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:40:46 smithi153 ceph-mon[70044]: pgmap v14773: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:48.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:40:48 smithi083 ceph-mon[77242]: pgmap v14774: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:48.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:40:48 smithi083 ceph-mon[79930]: pgmap v14774: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:48.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:40:48 smithi153 ceph-mon[70044]: pgmap v14774: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:48.995 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:40:49.282 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:40:49.282 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 4m ago 8h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:40:49.283 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 2m ago 8h 64.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:40:49.283 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 4m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:40:49.284 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 2m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:40:49.284 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 2m ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:40:49.284 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 4m ago 8h 919M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:40:49.285 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 4m ago 8h 1806M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:40:49.285 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 2m ago 8h 783M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:40:49.285 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 4m ago 8h 1075M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:40:49.285 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 4m ago 8h 24.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:40:49.286 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 2m ago 8h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:40:49.286 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 4m ago 8h 1308M 2305M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:40:49.286 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 4m ago 8h 1295M 2305M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:40:49.287 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 4m ago 8h 984M 2305M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:40:49.287 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 4m ago 8h 1268M 2305M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:40:49.287 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 2m ago 8h 1114M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:40:49.287 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 2m ago 8h 1395M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:40:49.288 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 2m ago 8h 1247M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:40:49.288 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 2m ago 8h 1481M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:40:49.289 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 2m ago 8h 183M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:40:49.639 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:40:49.640 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:40:49.640 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:40:49.640 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:40:49.640 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:40:49.641 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:40:49.641 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:40:49.641 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:40:49.641 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:40:49.642 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:40:49.642 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:40:49.642 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:40:49.642 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:40:49.643 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:40:49.643 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:40:49.643 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:40:49.643 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:40:49.643 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:40:50.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:40:50 smithi083 ceph-mon[77242]: pgmap v14775: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:50.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:40:50 smithi083 ceph-mon[77242]: from='client.61243 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:40:50.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:40:50 smithi083 ceph-mon[77242]: from='client.51318 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:40:50.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:40:50 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4056811939' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:40:50.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:40:50 smithi083 ceph-mon[79930]: pgmap v14775: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:50.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:40:50 smithi083 ceph-mon[79930]: from='client.61243 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:40:50.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:40:50 smithi083 ceph-mon[79930]: from='client.51318 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:40:50.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:40:50 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4056811939' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:40:50.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:40:50 smithi153 ceph-mon[70044]: pgmap v14775: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:50.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:40:50 smithi153 ceph-mon[70044]: from='client.61243 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:40:50.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:40:50 smithi153 ceph-mon[70044]: from='client.51318 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:40:50.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:40:50 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4056811939' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:40:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:40:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:40:51] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T13:40:52.316 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:40:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:40:52.002Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:40:52.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:40:52 smithi083 ceph-mon[77242]: pgmap v14776: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:52.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:40:52 smithi083 ceph-mon[79930]: pgmap v14776: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:52.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:40:52 smithi153 ceph-mon[70044]: pgmap v14776: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:52.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:40:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:40:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:40:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:40:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:40:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:40:54.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:40:54 smithi083 ceph-mon[79930]: pgmap v14777: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:54.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:40:54 smithi083 ceph-mon[77242]: pgmap v14777: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:54.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:40:54 smithi153 ceph-mon[70044]: pgmap v14777: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:55.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:40:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:40:55.363Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:40:56.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:40:56 smithi083 ceph-mon[77242]: pgmap v14778: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:56.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:40:56 smithi083 ceph-mon[79930]: pgmap v14778: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:56.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:40:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:40:56.089Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:40:56.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:40:56 smithi153 ceph-mon[70044]: pgmap v14778: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:58.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:40:58 smithi083 ceph-mon[77242]: pgmap v14779: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:58.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:40:58 smithi083 ceph-mon[79930]: pgmap v14779: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:40:58.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:40:58 smithi153 ceph-mon[70044]: pgmap v14779: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:00.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:41:00 smithi083 ceph-mon[77242]: pgmap v14780: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:00.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:41:00 smithi083 ceph-mon[79930]: pgmap v14780: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:00.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:41:00 smithi153 ceph-mon[70044]: pgmap v14780: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:41:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:41:01] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T13:41:02.330 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:41:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:41:02.002Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:41:02.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:41:02 smithi083 ceph-mon[79930]: pgmap v14781: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:02.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:41:02 smithi083 ceph-mon[77242]: pgmap v14781: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:02.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:41:02 smithi153 ceph-mon[70044]: pgmap v14781: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:02.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:41:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:41:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:41:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:41:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:41:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:41:04.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:41:04 smithi083 ceph-mon[79930]: pgmap v14782: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:04.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:41:04 smithi083 ceph-mon[77242]: pgmap v14782: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:04.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:41:04 smithi153 ceph-mon[70044]: pgmap v14782: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:05.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:41:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:41:05.363Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:41:06.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:41:06 smithi083 ceph-mon[79930]: pgmap v14783: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:06.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:41:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:41:06.089Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:41:06.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:41:06 smithi083 ceph-mon[77242]: pgmap v14783: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:06.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:41:06 smithi153 ceph-mon[70044]: pgmap v14783: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:08.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:41:08 smithi083 ceph-mon[79930]: pgmap v14784: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:08.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:41:08 smithi083 ceph-mon[77242]: pgmap v14784: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:08.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:41:08 smithi153 ceph-mon[70044]: pgmap v14784: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:10.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:41:10 smithi083 ceph-mon[79930]: pgmap v14785: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:10.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:41:10 smithi083 ceph-mon[77242]: pgmap v14785: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:10.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:41:10 smithi153 ceph-mon[70044]: pgmap v14785: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:41:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:41:11] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T13:41:12.346 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:41:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:41:12.003Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:41:12.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:41:12 smithi083 ceph-mon[79930]: pgmap v14786: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:12.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:41:12 smithi083 ceph-mon[77242]: pgmap v14786: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:12.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:41:12 smithi153 ceph-mon[70044]: pgmap v14786: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:12.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:41:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:41:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:41:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:41:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:41:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:41:14.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:41:14 smithi083 ceph-mon[79930]: pgmap v14787: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:14.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:41:14 smithi083 ceph-mon[77242]: pgmap v14787: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:14.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:41:14 smithi153 ceph-mon[70044]: pgmap v14787: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:15.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:41:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:41:15.364Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:41:16.354 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:41:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:41:16.090Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:41:16.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:41:16 smithi083 ceph-mon[79930]: pgmap v14788: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:16.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:41:16 smithi083 ceph-mon[77242]: pgmap v14788: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:16.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:41:16 smithi153 ceph-mon[70044]: pgmap v14788: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:18.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:41:18 smithi083 ceph-mon[79930]: pgmap v14789: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:18.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:41:18 smithi083 ceph-mon[77242]: pgmap v14789: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:18.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:41:18 smithi153 ceph-mon[70044]: pgmap v14789: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:19.966 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:41:20.254 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:41:20.255 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 5m ago 8h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:41:20.255 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 2m ago 8h 64.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:41:20.255 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 5m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:41:20.255 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 2m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:41:20.256 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 2m ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:41:20.256 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 5m ago 8h 919M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:41:20.256 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 5m ago 8h 1806M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:41:20.256 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 2m ago 8h 783M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:41:20.257 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 5m ago 8h 1075M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:41:20.257 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 5m ago 8h 24.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:41:20.257 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 2m ago 8h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:41:20.258 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 5m ago 8h 1308M 2305M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:41:20.258 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 5m ago 8h 1295M 2305M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:41:20.258 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 5m ago 8h 984M 2305M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:41:20.258 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 5m ago 8h 1268M 2305M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:41:20.259 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 2m ago 8h 1114M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:41:20.259 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 2m ago 8h 1395M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:41:20.259 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 2m ago 8h 1247M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:41:20.259 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 2m ago 8h 1481M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:41:20.260 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 2m ago 8h 183M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:41:20.611 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:41:20.611 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:41:20.611 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:41:20.612 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:41:20.612 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:41:20.612 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:41:20.612 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:41:20.613 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:41:20.613 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:41:20.613 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:41:20.613 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:41:20.613 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:41:20.614 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:41:20.614 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:41:20.614 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:41:20.614 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:41:20.615 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:41:20.615 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:41:20.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:41:20 smithi083 ceph-mon[79930]: pgmap v14790: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:20.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:41:20 smithi083 ceph-mon[77242]: pgmap v14790: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:20.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:41:20 smithi153 ceph-mon[70044]: pgmap v14790: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:21.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:41:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:41:21] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T13:41:21.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:41:21 smithi083 ceph-mon[79930]: from='client.51330 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:41:21.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:41:21 smithi083 ceph-mon[79930]: from='client.51336 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:41:21.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:41:21 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3080119411' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:41:21.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:41:21 smithi083 ceph-mon[77242]: from='client.51330 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:41:21.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:41:21 smithi083 ceph-mon[77242]: from='client.51336 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:41:21.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:41:21 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3080119411' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:41:21.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:41:21 smithi153 ceph-mon[70044]: from='client.51330 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:41:21.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:41:21 smithi153 ceph-mon[70044]: from='client.51336 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:41:21.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:41:21 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3080119411' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:41:22.369 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:41:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:41:22.003Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:41:22.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:41:22 smithi083 ceph-mon[77242]: pgmap v14791: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:22.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:41:22 smithi083 ceph-mon[79930]: pgmap v14791: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:22.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:41:22 smithi153 ceph-mon[70044]: pgmap v14791: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:22.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:41:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:41:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:41:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:41:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:41:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:41:24.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:41:24 smithi083 ceph-mon[79930]: pgmap v14792: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:24.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:41:24 smithi083 ceph-mon[77242]: pgmap v14792: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:24.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:41:24 smithi153 ceph-mon[70044]: pgmap v14792: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:25.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:41:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:41:25.364Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:41:26.377 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:41:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:41:26.091Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:41:26.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:41:26 smithi083 ceph-mon[79930]: pgmap v14793: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:26.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:41:26 smithi083 ceph-mon[77242]: pgmap v14793: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:26.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:41:26 smithi153 ceph-mon[70044]: pgmap v14793: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:28.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:41:28 smithi083 ceph-mon[79930]: pgmap v14794: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:28.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:41:28 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:41:28.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:41:28 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:41:28.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:41:28 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:41:28.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:41:28 smithi083 ceph-mon[77242]: pgmap v14794: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:28.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:41:28 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:41:28.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:41:28 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:41:28.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:41:28 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:41:28.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:41:28 smithi153 ceph-mon[70044]: pgmap v14794: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:28.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:41:28 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:41:28.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:41:28 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:41:28.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:41:28 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:41:29.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:41:29 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:41:29.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:41:29 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:41:29.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:41:29 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:41:30.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:41:30 smithi083 ceph-mon[79930]: pgmap v14795: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:30.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:41:30 smithi083 ceph-mon[77242]: pgmap v14795: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:30.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:41:30 smithi153 ceph-mon[70044]: pgmap v14795: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:41:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:41:31] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T13:41:32.427 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:41:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:41:32.005Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:41:32.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:41:32 smithi083 ceph-mon[79930]: pgmap v14796: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:32.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:41:32 smithi083 ceph-mon[77242]: pgmap v14796: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:32.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:41:32 smithi153 ceph-mon[70044]: pgmap v14796: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:32.811 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:41:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:41:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:41:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:41:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:41:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:41:34.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:41:34 smithi083 ceph-mon[79930]: pgmap v14797: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:34.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:41:34 smithi083 ceph-mon[77242]: pgmap v14797: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:34.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:41:34 smithi153 ceph-mon[70044]: pgmap v14797: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:35.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:41:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:41:35.365Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:41:36.433 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:41:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:41:36.091Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:41:36.433 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:41:36 smithi083 ceph-mon[77242]: pgmap v14798: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:36.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:41:36 smithi083 ceph-mon[79930]: pgmap v14798: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:36.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:41:36 smithi153 ceph-mon[70044]: pgmap v14798: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:38.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:41:38 smithi083 ceph-mon[79930]: pgmap v14799: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:38.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:41:38 smithi083 ceph-mon[77242]: pgmap v14799: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:38.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:41:38 smithi153 ceph-mon[70044]: pgmap v14799: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:41:40 smithi083 ceph-mon[79930]: pgmap v14800: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:41:40 smithi083 ceph-mon[77242]: pgmap v14800: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:41:40 smithi153 ceph-mon[70044]: pgmap v14800: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:41.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:41:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:41:41] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T13:41:41.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:41:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:41:41.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:41:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:41:41.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:41:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:41:41.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:41:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:41:41.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:41:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:41:41.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:41:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:41:41.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:41:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:41:41.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:41:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:41:41.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:41:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:41:41.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:41:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:41:41.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:41:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:41:41.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:41:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:41:42.445 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:41:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:41:42.005Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:41:42.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:41:42 smithi083 ceph-mon[79930]: pgmap v14801: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:41:42 smithi083 ceph-mon[77242]: pgmap v14801: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:41:42 smithi153 ceph-mon[70044]: pgmap v14801: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:42.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:41:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:41:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:41:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:41:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:41:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:41:44.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:41:44 smithi083 ceph-mon[79930]: pgmap v14802: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:44.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:41:44 smithi083 ceph-mon[77242]: pgmap v14802: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:44.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:41:44 smithi153 ceph-mon[70044]: pgmap v14802: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:45.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:41:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:41:45.366Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:41:46.451 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:41:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:41:46.091Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:41:46.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:41:46 smithi083 ceph-mon[79930]: pgmap v14803: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:46.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:41:46 smithi083 ceph-mon[77242]: pgmap v14803: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:46.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:41:46 smithi153 ceph-mon[70044]: pgmap v14803: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:48.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:41:48 smithi083 ceph-mon[79930]: pgmap v14804: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:48.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:41:48 smithi083 ceph-mon[77242]: pgmap v14804: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:48.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:41:48 smithi153 ceph-mon[70044]: pgmap v14804: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:50.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:41:50 smithi083 ceph-mon[79930]: pgmap v14805: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:50.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:41:50 smithi083 ceph-mon[77242]: pgmap v14805: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:50.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:41:50 smithi153 ceph-mon[70044]: pgmap v14805: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:50.938 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:41:51.228 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:41:51.229 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 5m ago 8h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:41:51.229 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 3m ago 8h 64.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:41:51.229 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 5m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:41:51.229 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 3m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:41:51.230 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 3m ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:41:51.230 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 5m ago 8h 919M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:41:51.230 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 5m ago 8h 1806M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:41:51.230 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 3m ago 8h 783M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:41:51.231 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 5m ago 8h 1075M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:41:51.231 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 5m ago 8h 24.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:41:51.231 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 3m ago 8h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:41:51.231 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 5m ago 8h 1308M 2305M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:41:51.232 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 5m ago 8h 1295M 2305M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:41:51.232 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 5m ago 8h 984M 2305M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:41:51.232 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 5m ago 8h 1268M 2305M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:41:51.232 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 3m ago 8h 1114M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:41:51.232 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 3m ago 8h 1395M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:41:51.233 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 3m ago 8h 1247M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:41:51.233 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 3m ago 8h 1481M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:41:51.233 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 3m ago 8h 183M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:41:51.585 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:41:51.585 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:41:51.586 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:41:51.586 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:41:51.586 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:41:51.586 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:41:51.587 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:41:51.587 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:41:51.587 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:41:51.587 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:41:51.587 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:41:51.588 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:41:51.588 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:41:51.588 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:41:51.588 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:41:51.589 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:41:51.589 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:41:51.589 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:41:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:41:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:41:51] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T13:41:52.459 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:41:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:41:52.005Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:41:52.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:41:52 smithi083 ceph-mon[77242]: pgmap v14806: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:52.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:41:52 smithi083 ceph-mon[77242]: from='client.51348 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:41:52.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:41:52 smithi083 ceph-mon[77242]: from='client.51354 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:41:52.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:41:52 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/407906811' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:41:52.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:41:52 smithi083 ceph-mon[79930]: pgmap v14806: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:52.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:41:52 smithi083 ceph-mon[79930]: from='client.51348 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:41:52.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:41:52 smithi083 ceph-mon[79930]: from='client.51354 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:41:52.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:41:52 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/407906811' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:41:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:41:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:41:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:41:52.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:41:52 smithi153 ceph-mon[70044]: pgmap v14806: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:52.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:41:52 smithi153 ceph-mon[70044]: from='client.51348 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:41:52.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:41:52 smithi153 ceph-mon[70044]: from='client.51354 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:41:52.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:41:52 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/407906811' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:41:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:41:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:41:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:41:54.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:41:54 smithi153 ceph-mon[70044]: pgmap v14807: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:54.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:41:54 smithi083 ceph-mon[79930]: pgmap v14807: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:54.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:41:54 smithi083 ceph-mon[77242]: pgmap v14807: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:55.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:41:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:41:55.366Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:41:56.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:41:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:41:56.092Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:41:56.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:41:56 smithi153 ceph-mon[70044]: pgmap v14808: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:56.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:41:56 smithi083 ceph-mon[79930]: pgmap v14808: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:56.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:41:56 smithi083 ceph-mon[77242]: pgmap v14808: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:58.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:41:58 smithi153 ceph-mon[70044]: pgmap v14809: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:58.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:41:58 smithi083 ceph-mon[79930]: pgmap v14809: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:41:58.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:41:58 smithi083 ceph-mon[77242]: pgmap v14809: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:00.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:42:00 smithi153 ceph-mon[70044]: pgmap v14810: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:00.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:42:00 smithi083 ceph-mon[79930]: pgmap v14810: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:00.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:42:00 smithi083 ceph-mon[77242]: pgmap v14810: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:42:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:42:01] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T13:42:02.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:42:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:42:02.006Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:42:02.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:42:02 smithi153 ceph-mon[70044]: pgmap v14811: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:02.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:42:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:42:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:42:02.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:42:02 smithi083 ceph-mon[77242]: pgmap v14811: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:02.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:42:02 smithi083 ceph-mon[79930]: pgmap v14811: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:42:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:42:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:42:04.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:42:04 smithi153 ceph-mon[70044]: pgmap v14812: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:04.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:42:04 smithi083 ceph-mon[77242]: pgmap v14812: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:04.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:42:04 smithi083 ceph-mon[79930]: pgmap v14812: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:05.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:42:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:42:05.367Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:42:06.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:42:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:42:06.092Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:42:06.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:42:06 smithi153 ceph-mon[70044]: pgmap v14813: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:06.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:42:06 smithi083 ceph-mon[79930]: pgmap v14813: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:06.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:42:06 smithi083 ceph-mon[77242]: pgmap v14813: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:08.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:42:08 smithi153 ceph-mon[70044]: pgmap v14814: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:08.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:42:08 smithi083 ceph-mon[79930]: pgmap v14814: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:08.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:42:08 smithi083 ceph-mon[77242]: pgmap v14814: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:10.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:42:10 smithi153 ceph-mon[70044]: pgmap v14815: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:10.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:42:10 smithi083 ceph-mon[79930]: pgmap v14815: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:10.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:42:10 smithi083 ceph-mon[77242]: pgmap v14815: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:42:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:42:11] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T13:42:12.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:42:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:42:12.007Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:42:12.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:42:12 smithi153 ceph-mon[70044]: pgmap v14816: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:12.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:42:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:42:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:42:12.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:42:12 smithi083 ceph-mon[79930]: pgmap v14816: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:12.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:42:12 smithi083 ceph-mon[77242]: pgmap v14816: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:42:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:42:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:42:14.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:42:14 smithi153 ceph-mon[70044]: pgmap v14817: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:14.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:42:14 smithi083 ceph-mon[79930]: pgmap v14817: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:14.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:42:14 smithi083 ceph-mon[77242]: pgmap v14817: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:15.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:42:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:42:15.368Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:42:16.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:42:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:42:16.093Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:42:16.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:42:16 smithi153 ceph-mon[70044]: pgmap v14818: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:16.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:42:16 smithi083 ceph-mon[79930]: pgmap v14818: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:16.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:42:16 smithi083 ceph-mon[77242]: pgmap v14818: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:18.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:42:18 smithi153 ceph-mon[70044]: pgmap v14819: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:18.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:42:18 smithi083 ceph-mon[79930]: pgmap v14819: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:18.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:42:18 smithi083 ceph-mon[77242]: pgmap v14819: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:20.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:42:20 smithi153 ceph-mon[70044]: pgmap v14820: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:20.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:42:20 smithi083 ceph-mon[79930]: pgmap v14820: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:20.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:42:20 smithi083 ceph-mon[77242]: pgmap v14820: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:21.906 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:42:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:42:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:42:21] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T13:42:22.196 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:42:22.197 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 6m ago 8h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:42:22.197 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 3m ago 8h 64.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:42:22.197 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 6m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:42:22.198 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 3m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:42:22.198 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 3m ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:42:22.198 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 6m ago 8h 919M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:42:22.198 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 6m ago 8h 1806M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:42:22.199 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 3m ago 8h 783M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:42:22.199 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 6m ago 8h 1075M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:42:22.199 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 6m ago 8h 24.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:42:22.199 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 3m ago 8h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:42:22.200 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 6m ago 8h 1308M 2305M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:42:22.200 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 6m ago 8h 1295M 2305M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:42:22.200 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 6m ago 8h 984M 2305M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:42:22.200 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 6m ago 8h 1268M 2305M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:42:22.201 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 3m ago 8h 1114M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:42:22.201 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 3m ago 8h 1395M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:42:22.201 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 3m ago 8h 1247M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:42:22.201 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 3m ago 8h 1481M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:42:22.201 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 3m ago 8h 183M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:42:22.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:42:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:42:22.007Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:42:22.550 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:42:22.551 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:42:22.551 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:42:22.551 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:42:22.552 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:42:22.552 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:42:22.552 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:42:22.552 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:42:22.553 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:42:22.553 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:42:22.553 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:42:22.553 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:42:22.554 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:42:22.554 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:42:22.554 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:42:22.554 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:42:22.555 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:42:22.555 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:42:22.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:42:22 smithi153 ceph-mon[70044]: pgmap v14821: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:22.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:42:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:42:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:42:22.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:42:22 smithi083 ceph-mon[77242]: pgmap v14821: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:22.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:42:22 smithi083 ceph-mon[79930]: pgmap v14821: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:42:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:42:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:42:23.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:42:23 smithi153 ceph-mon[70044]: from='client.61297 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:42:23.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:42:23 smithi153 ceph-mon[70044]: from='client.61303 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:42:23.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:42:23 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2979279874' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:42:23.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:42:23 smithi083 ceph-mon[79930]: from='client.61297 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:42:23.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:42:23 smithi083 ceph-mon[79930]: from='client.61303 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:42:23.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:42:23 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2979279874' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:42:23.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:42:23 smithi083 ceph-mon[77242]: from='client.61297 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:42:23.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:42:23 smithi083 ceph-mon[77242]: from='client.61303 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:42:23.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:42:23 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2979279874' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:42:24.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:42:24 smithi153 ceph-mon[70044]: pgmap v14822: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:24.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:42:24 smithi083 ceph-mon[79930]: pgmap v14822: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:24.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:42:24 smithi083 ceph-mon[77242]: pgmap v14822: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:25.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:42:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:42:25.369Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:42:26.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:42:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:42:26.093Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:42:26.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:42:26 smithi153 ceph-mon[70044]: pgmap v14823: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:26.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:42:26 smithi083 ceph-mon[77242]: pgmap v14823: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:26.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:42:26 smithi083 ceph-mon[79930]: pgmap v14823: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:28.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:42:28 smithi153 ceph-mon[70044]: pgmap v14824: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:28.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:42:28 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:42:28.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:42:28 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:42:28.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:42:28 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:42:28.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:42:28 smithi083 ceph-mon[79930]: pgmap v14824: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:28.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:42:28 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:42:28.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:42:28 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:42:28.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:42:28 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:42:28.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:42:28 smithi083 ceph-mon[77242]: pgmap v14824: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:28.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:42:28 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:42:28.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:42:28 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:42:28.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:42:28 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:42:30.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:42:30 smithi083 ceph-mon[77242]: pgmap v14825: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:30.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:42:30 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:42:30.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:42:30 smithi083 ceph-mon[79930]: pgmap v14825: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:30.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:42:30 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:42:30.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:42:30 smithi153 ceph-mon[70044]: pgmap v14825: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:30.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:42:30 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:42:31.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:42:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:42:31] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T13:42:32.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:42:32 smithi083 ceph-mon[77242]: pgmap v14826: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:32.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:42:32 smithi083 ceph-mon[79930]: pgmap v14826: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:32.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:42:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:42:32.009Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:42:32.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:42:32 smithi153 ceph-mon[70044]: pgmap v14826: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:32.559 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:42:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:42:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:42:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:42:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:42:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:42:34.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:42:34 smithi153 ceph-mon[70044]: pgmap v14827: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:34.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:42:34 smithi083 ceph-mon[79930]: pgmap v14827: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:34.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:42:34 smithi083 ceph-mon[77242]: pgmap v14827: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:35.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:42:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:42:35.369Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:42:36.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:42:36 smithi083 ceph-mon[79930]: pgmap v14828: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:36.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:42:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:42:36.093Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:42:36.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:42:36 smithi083 ceph-mon[77242]: pgmap v14828: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:36.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:42:36 smithi153 ceph-mon[70044]: pgmap v14828: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:38.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:42:38 smithi153 ceph-mon[70044]: pgmap v14829: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:38.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:42:38 smithi083 ceph-mon[79930]: pgmap v14829: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:38.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:42:38 smithi083 ceph-mon[77242]: pgmap v14829: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:42:40 smithi153 ceph-mon[70044]: pgmap v14830: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:42:40 smithi083 ceph-mon[77242]: pgmap v14830: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:42:40 smithi083 ceph-mon[79930]: pgmap v14830: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:41.508 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:42:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:42:41.509 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:42:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:42:41.509 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:42:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:42:41.509 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:42:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:42:41.509 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:42:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:42:41.510 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:42:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:42:41.510 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:42:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:42:41.510 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:42:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:42:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:42:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:42:41.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:42:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:42:41.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:42:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:42:41.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:42:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:42:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:42:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:42:41] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T13:42:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:42:42 smithi083 ceph-mon[79930]: pgmap v14831: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:42.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:42:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:42:42.010Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:42:42.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:42:42 smithi083 ceph-mon[77242]: pgmap v14831: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:42:42 smithi153 ceph-mon[70044]: pgmap v14831: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:42.559 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:42:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:42:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:42:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:42:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:42:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:42:44.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:42:44 smithi153 ceph-mon[70044]: pgmap v14832: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:44.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:42:44 smithi083 ceph-mon[79930]: pgmap v14832: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:44.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:42:44 smithi083 ceph-mon[77242]: pgmap v14832: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:45.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:42:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:42:45.369Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:42:46.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:42:46 smithi083 ceph-mon[79930]: pgmap v14833: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:46.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:42:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:42:46.093Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:42:46.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:42:46 smithi083 ceph-mon[77242]: pgmap v14833: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:46.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:42:46 smithi153 ceph-mon[70044]: pgmap v14833: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:48.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:42:48 smithi153 ceph-mon[70044]: pgmap v14834: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:48.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:42:48 smithi083 ceph-mon[79930]: pgmap v14834: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:48.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:42:48 smithi083 ceph-mon[77242]: pgmap v14834: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:50.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:42:50 smithi153 ceph-mon[70044]: pgmap v14835: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:50.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:42:50 smithi083 ceph-mon[79930]: pgmap v14835: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:50.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:42:50 smithi083 ceph-mon[77242]: pgmap v14835: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:42:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:42:51] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T13:42:52.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:42:52 smithi083 ceph-mon[77242]: pgmap v14836: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:52.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:42:52 smithi083 ceph-mon[79930]: pgmap v14836: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:52.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:42:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:42:52.010Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:42:52.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:42:52 smithi153 ceph-mon[70044]: pgmap v14836: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:52.559 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:42:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:42:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:42:52.879 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:42:53.166 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:42:53.167 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 6m ago 8h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:42:53.167 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 4m ago 8h 64.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:42:53.167 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 6m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:42:53.167 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 4m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:42:53.168 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 4m ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:42:53.168 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 6m ago 8h 919M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:42:53.168 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 6m ago 8h 1806M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:42:53.168 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 4m ago 8h 783M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:42:53.169 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 6m ago 8h 1075M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:42:53.169 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 6m ago 8h 24.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:42:53.169 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 4m ago 8h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:42:53.169 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 6m ago 8h 1308M 2305M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:42:53.170 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 6m ago 8h 1295M 2305M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:42:53.170 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 6m ago 8h 984M 2305M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:42:53.170 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 6m ago 8h 1268M 2305M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:42:53.170 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 4m ago 8h 1114M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:42:53.171 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 4m ago 8h 1395M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:42:53.171 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 4m ago 8h 1247M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:42:53.171 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 4m ago 8h 1481M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:42:53.171 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 4m ago 8h 183M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:42:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:42:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:42:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:42:53.531 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:42:53.532 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:42:53.532 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:42:53.532 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:42:53.532 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:42:53.533 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:42:53.533 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:42:53.533 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:42:53.533 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:42:53.534 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:42:53.534 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:42:53.534 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:42:53.534 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:42:53.535 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:42:53.535 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:42:53.535 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:42:53.535 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:42:53.536 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:42:54.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:42:54 smithi153 ceph-mon[70044]: pgmap v14837: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:54.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:42:54 smithi153 ceph-mon[70044]: from='client.51384 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:42:54.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:42:54 smithi153 ceph-mon[70044]: from='client.51390 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:42:54.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:42:54 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2017757628' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:42:54.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:42:54 smithi083 ceph-mon[79930]: pgmap v14837: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:54.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:42:54 smithi083 ceph-mon[79930]: from='client.51384 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:42:54.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:42:54 smithi083 ceph-mon[79930]: from='client.51390 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:42:54.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:42:54 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2017757628' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:42:54.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:42:54 smithi083 ceph-mon[77242]: pgmap v14837: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:54.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:42:54 smithi083 ceph-mon[77242]: from='client.51384 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:42:54.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:42:54 smithi083 ceph-mon[77242]: from='client.51390 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:42:54.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:42:54 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2017757628' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:42:55.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:42:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:42:55.370Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:42:56.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:42:56 smithi083 ceph-mon[79930]: pgmap v14838: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:56.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:42:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:42:56.094Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:42:56.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:42:56 smithi083 ceph-mon[77242]: pgmap v14838: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:56.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:42:56 smithi153 ceph-mon[70044]: pgmap v14838: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:58.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:42:58 smithi153 ceph-mon[70044]: pgmap v14839: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:58.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:42:58 smithi083 ceph-mon[79930]: pgmap v14839: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:42:58.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:42:58 smithi083 ceph-mon[77242]: pgmap v14839: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:00.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:43:00 smithi153 ceph-mon[70044]: pgmap v14840: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:00.596 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:43:00 smithi083 ceph-mon[79930]: pgmap v14840: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:00.597 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:43:00 smithi083 ceph-mon[77242]: pgmap v14840: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:43:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:43:01] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T13:43:02.267 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:43:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:43:02.011Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:43:02.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:43:02 smithi153 ceph-mon[70044]: pgmap v14841: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:02.559 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:43:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:43:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:43:02.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:43:02 smithi083 ceph-mon[79930]: pgmap v14841: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:02.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:43:02 smithi083 ceph-mon[77242]: pgmap v14841: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:43:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:43:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:43:04.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:43:04 smithi153 ceph-mon[70044]: pgmap v14842: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:04.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:43:04 smithi083 ceph-mon[79930]: pgmap v14842: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:04.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:43:04 smithi083 ceph-mon[77242]: pgmap v14842: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:05.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:43:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:43:05.370Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:43:06.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:43:06 smithi083 ceph-mon[79930]: pgmap v14843: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:06.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:43:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:43:06.094Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:43:06.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:43:06 smithi083 ceph-mon[77242]: pgmap v14843: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:06.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:43:06 smithi153 ceph-mon[70044]: pgmap v14843: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:08.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:43:08 smithi153 ceph-mon[70044]: pgmap v14844: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:08.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:43:08 smithi083 ceph-mon[79930]: pgmap v14844: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:08.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:43:08 smithi083 ceph-mon[77242]: pgmap v14844: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:10.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:43:10 smithi153 ceph-mon[70044]: pgmap v14845: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:10.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:43:10 smithi083 ceph-mon[79930]: pgmap v14845: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:10.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:43:10 smithi083 ceph-mon[77242]: pgmap v14845: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:43:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:43:11] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T13:43:12.281 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:43:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:43:12.011Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:43:12.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:43:12 smithi153 ceph-mon[70044]: pgmap v14846: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:12.559 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:43:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:43:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:43:12.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:43:12 smithi083 ceph-mon[79930]: pgmap v14846: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:12.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:43:12 smithi083 ceph-mon[77242]: pgmap v14846: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:43:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:43:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:43:14.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:43:14 smithi153 ceph-mon[70044]: pgmap v14847: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:14.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:43:14 smithi083 ceph-mon[77242]: pgmap v14847: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:14.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:43:14 smithi083 ceph-mon[79930]: pgmap v14847: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:15.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:43:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:43:15.370Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:43:16.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:43:16 smithi083 ceph-mon[79930]: pgmap v14848: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:16.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:43:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:43:16.095Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:43:16.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:43:16 smithi083 ceph-mon[77242]: pgmap v14848: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:16.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:43:16 smithi153 ceph-mon[70044]: pgmap v14848: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:18.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:43:18 smithi153 ceph-mon[70044]: pgmap v14849: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:18.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:43:18 smithi083 ceph-mon[79930]: pgmap v14849: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:18.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:43:18 smithi083 ceph-mon[77242]: pgmap v14849: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:20.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:43:20 smithi153 ceph-mon[70044]: pgmap v14850: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:20.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:43:20 smithi083 ceph-mon[79930]: pgmap v14850: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:20.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:43:20 smithi083 ceph-mon[77242]: pgmap v14850: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:43:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:43:21] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T13:43:22.297 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:43:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:43:22.012Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:43:22.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:43:22 smithi153 ceph-mon[70044]: pgmap v14851: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:22.559 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:43:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:43:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:43:22.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:43:22 smithi083 ceph-mon[77242]: pgmap v14851: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:22.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:43:22 smithi083 ceph-mon[79930]: pgmap v14851: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:43:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:43:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:43:23.865 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:43:24.154 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:43:24.155 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 7m ago 8h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:43:24.155 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 4m ago 8h 64.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:43:24.155 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 7m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:43:24.156 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 4m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:43:24.156 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 4m ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:43:24.156 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 7m ago 8h 919M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:43:24.156 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 7m ago 8h 1806M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:43:24.157 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 4m ago 8h 783M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:43:24.157 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 7m ago 8h 1075M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:43:24.157 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 7m ago 8h 24.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:43:24.157 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 4m ago 8h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:43:24.158 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 7m ago 8h 1308M 2305M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:43:24.158 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 7m ago 8h 1295M 2305M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:43:24.158 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 7m ago 8h 984M 2305M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:43:24.159 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 7m ago 8h 1268M 2305M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:43:24.159 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 4m ago 8h 1114M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:43:24.159 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 4m ago 8h 1395M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:43:24.159 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 4m ago 8h 1247M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:43:24.160 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 4m ago 8h 1481M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:43:24.160 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 4m ago 8h 183M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:43:24.512 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:43:24.513 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:43:24.513 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:43:24.513 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:43:24.514 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:43:24.514 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:43:24.514 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:43:24.515 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:43:24.515 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:43:24.516 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:43:24.516 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:43:24.516 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:43:24.517 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:43:24.517 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:43:24.518 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:43:24.518 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:43:24.518 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:43:24.519 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:43:24.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:43:24 smithi153 ceph-mon[70044]: pgmap v14852: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:24.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:43:24 smithi083 ceph-mon[79930]: pgmap v14852: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:24.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:43:24 smithi083 ceph-mon[77242]: pgmap v14852: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:25.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:43:25 smithi153 ceph-mon[70044]: from='client.51402 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:43:25.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:43:25 smithi153 ceph-mon[70044]: from='client.61339 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:43:25.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:43:25 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2532837523' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:43:25.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:43:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:43:25.371Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:43:25.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:43:25 smithi083 ceph-mon[77242]: from='client.51402 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:43:25.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:43:25 smithi083 ceph-mon[77242]: from='client.61339 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:43:25.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:43:25 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2532837523' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:43:25.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:43:25 smithi083 ceph-mon[79930]: from='client.51402 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:43:25.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:43:25 smithi083 ceph-mon[79930]: from='client.61339 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:43:25.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:43:25 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2532837523' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:43:26.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:43:26 smithi083 ceph-mon[79930]: pgmap v14853: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:26.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:43:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:43:26.096Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:43:26.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:43:26 smithi083 ceph-mon[77242]: pgmap v14853: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:26.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:43:26 smithi153 ceph-mon[70044]: pgmap v14853: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:28.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:43:28 smithi083 ceph-mon[79930]: pgmap v14854: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:28.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:43:28 smithi083 ceph-mon[77242]: pgmap v14854: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:28.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:43:28 smithi153 ceph-mon[70044]: pgmap v14854: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:29.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:43:29 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:43:29.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:43:29 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:43:29.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:43:29 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:43:29.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:43:29 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:43:29.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:43:29 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:43:29.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:43:29 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:43:29.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:43:29 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:43:29.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:43:29 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:43:29.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:43:29 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:43:30.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:43:30 smithi083 ceph-mon[77242]: pgmap v14855: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:30.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:43:30 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:43:30.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:43:30 smithi083 ceph-mon[79930]: pgmap v14855: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:30.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:43:30 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:43:30.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:43:30 smithi153 ceph-mon[70044]: pgmap v14855: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:30.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:43:30 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:43:31.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:43:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:43:31] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T13:43:32.316 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:43:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:43:32.012Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:43:32.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:43:32 smithi083 ceph-mon[77242]: pgmap v14856: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:32.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:43:32 smithi083 ceph-mon[79930]: pgmap v14856: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:32.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:43:32 smithi153 ceph-mon[70044]: pgmap v14856: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:32.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:43:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:43:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:43:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:43:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:43:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:43:34.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:43:34 smithi083 ceph-mon[79930]: pgmap v14857: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:34.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:43:34 smithi083 ceph-mon[77242]: pgmap v14857: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:34.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:43:34 smithi153 ceph-mon[70044]: pgmap v14857: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:35.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:43:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:43:35.372Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:43:36.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:43:36 smithi083 ceph-mon[79930]: pgmap v14858: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:36.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:43:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:43:36.096Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:43:36.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:43:36 smithi083 ceph-mon[77242]: pgmap v14858: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:36.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:43:36 smithi153 ceph-mon[70044]: pgmap v14858: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:38.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:43:38 smithi083 ceph-mon[79930]: pgmap v14859: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:38.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:43:38 smithi083 ceph-mon[77242]: pgmap v14859: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:38.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:43:38 smithi153 ceph-mon[70044]: pgmap v14859: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:43:40 smithi083 ceph-mon[79930]: pgmap v14860: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:43:40 smithi083 ceph-mon[77242]: pgmap v14860: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:43:40 smithi153 ceph-mon[70044]: pgmap v14860: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:41.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:43:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:43:41] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T13:43:41.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:43:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:43:41.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:43:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:43:41.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:43:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:43:41.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:43:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:43:41.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:43:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:43:41.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:43:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:43:41.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:43:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:43:41.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:43:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:43:41.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:43:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:43:41.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:43:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:43:41.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:43:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:43:41.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:43:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:43:42.339 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:43:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:43:42.013Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:43:42.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:43:42 smithi083 ceph-mon[77242]: pgmap v14861: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:42.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:43:42 smithi083 ceph-mon[79930]: pgmap v14861: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:43:42 smithi153 ceph-mon[70044]: pgmap v14861: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:42.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:43:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:43:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:43:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:43:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:43:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:43:44.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:43:44 smithi083 ceph-mon[79930]: pgmap v14862: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:44.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:43:44 smithi083 ceph-mon[77242]: pgmap v14862: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:44.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:43:44 smithi153 ceph-mon[70044]: pgmap v14862: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:45.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:43:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:43:45.372Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:43:46.350 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:43:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:43:46.096Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:43:46.350 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:43:46 smithi083 ceph-mon[77242]: pgmap v14863: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:46.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:43:46 smithi083 ceph-mon[79930]: pgmap v14863: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:46.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:43:46 smithi153 ceph-mon[70044]: pgmap v14863: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:48.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:43:48 smithi083 ceph-mon[77242]: pgmap v14864: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:48.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:43:48 smithi083 ceph-mon[79930]: pgmap v14864: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:48.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:43:48 smithi153 ceph-mon[70044]: pgmap v14864: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:50.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:43:50 smithi083 ceph-mon[79930]: pgmap v14865: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:50.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:43:50 smithi083 ceph-mon[77242]: pgmap v14865: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:50.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:43:50 smithi153 ceph-mon[70044]: pgmap v14865: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:43:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:43:51] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T13:43:52.355 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:43:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:43:52.013Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:43:52.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:43:52 smithi083 ceph-mon[79930]: pgmap v14866: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:52.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:43:52 smithi083 ceph-mon[77242]: pgmap v14866: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:52.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:43:52 smithi153 ceph-mon[70044]: pgmap v14866: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:52.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:43:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:43:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:43:53.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:43:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:43:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:43:54.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:43:54 smithi083 ceph-mon[79930]: pgmap v14867: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:54.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:43:54 smithi083 ceph-mon[77242]: pgmap v14867: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:54.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:43:54 smithi153 ceph-mon[70044]: pgmap v14867: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:54.863 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:43:55.149 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:43:55.150 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 7m ago 8h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:43:55.151 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 5m ago 8h 64.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:43:55.151 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 7m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:43:55.151 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 5m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:43:55.152 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 5m ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:43:55.152 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 7m ago 8h 919M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:43:55.153 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 7m ago 8h 1806M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:43:55.153 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 5m ago 8h 783M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:43:55.153 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 7m ago 8h 1075M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:43:55.154 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 7m ago 8h 24.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:43:55.154 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 5m ago 8h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:43:55.154 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 7m ago 8h 1308M 2305M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:43:55.154 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 7m ago 8h 1295M 2305M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:43:55.155 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 7m ago 8h 984M 2305M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:43:55.155 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 7m ago 8h 1268M 2305M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:43:55.155 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 5m ago 8h 1114M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:43:55.155 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 5m ago 8h 1395M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:43:55.155 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 5m ago 8h 1247M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:43:55.156 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 5m ago 8h 1481M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:43:55.156 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 5m ago 8h 183M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:43:55.509 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:43:55.509 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:43:55.510 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:43:55.510 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:43:55.510 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:43:55.510 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:43:55.511 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:43:55.511 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:43:55.511 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:43:55.511 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:43:55.512 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:43:55.512 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:43:55.512 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:43:55.512 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:43:55.512 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:43:55.513 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:43:55.513 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:43:55.513 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:43:55.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:43:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:43:55.372Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:43:56.361 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:43:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:43:56.096Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:43:56.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:43:56 smithi083 ceph-mon[77242]: pgmap v14868: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:56.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:43:56 smithi083 ceph-mon[77242]: from='client.61351 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:43:56.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:43:56 smithi083 ceph-mon[77242]: from='client.51426 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:43:56.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:43:56 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2126016279' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:43:56.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:43:56 smithi083 ceph-mon[79930]: pgmap v14868: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:56.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:43:56 smithi083 ceph-mon[79930]: from='client.61351 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:43:56.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:43:56 smithi083 ceph-mon[79930]: from='client.51426 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:43:56.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:43:56 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2126016279' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:43:56.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:43:56 smithi153 ceph-mon[70044]: pgmap v14868: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:56.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:43:56 smithi153 ceph-mon[70044]: from='client.61351 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:43:56.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:43:56 smithi153 ceph-mon[70044]: from='client.51426 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:43:56.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:43:56 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2126016279' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:43:58.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:43:58 smithi083 ceph-mon[77242]: pgmap v14869: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:58.774 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:43:58 smithi083 ceph-mon[79930]: pgmap v14869: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:43:58.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:43:58 smithi153 ceph-mon[70044]: pgmap v14869: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:00.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:44:00 smithi083 ceph-mon[79930]: pgmap v14870: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:00.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:44:00 smithi083 ceph-mon[77242]: pgmap v14870: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:00.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:44:00 smithi153 ceph-mon[70044]: pgmap v14870: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:01.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:44:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:44:01] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T13:44:02.369 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:44:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:44:02.014Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:44:02.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:44:02 smithi083 ceph-mon[77242]: pgmap v14871: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:02.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:44:02 smithi083 ceph-mon[79930]: pgmap v14871: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:02.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:44:02 smithi153 ceph-mon[70044]: pgmap v14871: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:02.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:44:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:44:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:44:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:44:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:44:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:44:04.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:44:04 smithi083 ceph-mon[77242]: pgmap v14872: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:04.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:44:04 smithi083 ceph-mon[79930]: pgmap v14872: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:04.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:44:04 smithi153 ceph-mon[70044]: pgmap v14872: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:05.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:44:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:44:05.373Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:44:06.375 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:44:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:44:06.098Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:44:06.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:44:06 smithi083 ceph-mon[79930]: pgmap v14873: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:06.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:44:06 smithi083 ceph-mon[77242]: pgmap v14873: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:06.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:44:06 smithi153 ceph-mon[70044]: pgmap v14873: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:08.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:44:08 smithi083 ceph-mon[77242]: pgmap v14874: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:08.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:44:08 smithi083 ceph-mon[79930]: pgmap v14874: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:08.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:44:08 smithi153 ceph-mon[70044]: pgmap v14874: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:10.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:44:10 smithi083 ceph-mon[79930]: pgmap v14875: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:10.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:44:10 smithi083 ceph-mon[77242]: pgmap v14875: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:10.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:44:10 smithi153 ceph-mon[70044]: pgmap v14875: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:44:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:44:11] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T13:44:12.386 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:44:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:44:12.014Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:44:12.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:44:12 smithi083 ceph-mon[79930]: pgmap v14876: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:12.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:44:12 smithi083 ceph-mon[77242]: pgmap v14876: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:12.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:44:12 smithi153 ceph-mon[70044]: pgmap v14876: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:12.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:44:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:44:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:44:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:44:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:44:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:44:14.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:44:14 smithi083 ceph-mon[79930]: pgmap v14877: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:14.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:44:14 smithi083 ceph-mon[77242]: pgmap v14877: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:14.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:44:14 smithi153 ceph-mon[70044]: pgmap v14877: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:15.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:44:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:44:15.373Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:44:16.392 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:44:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:44:16.098Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:44:16.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:44:16 smithi083 ceph-mon[79930]: pgmap v14878: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:16.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:44:16 smithi083 ceph-mon[77242]: pgmap v14878: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:16.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:44:16 smithi153 ceph-mon[70044]: pgmap v14878: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:18.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:44:18 smithi083 ceph-mon[79930]: pgmap v14879: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:18.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:44:18 smithi083 ceph-mon[77242]: pgmap v14879: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:18.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:44:18 smithi153 ceph-mon[70044]: pgmap v14879: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:20.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:44:20 smithi083 ceph-mon[79930]: pgmap v14880: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:20.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:44:20 smithi083 ceph-mon[77242]: pgmap v14880: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:20.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:44:20 smithi153 ceph-mon[70044]: pgmap v14880: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:21.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:44:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:44:21] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T13:44:22.402 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:44:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:44:22.015Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:44:22.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:44:22 smithi083 ceph-mon[77242]: pgmap v14881: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:22.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:44:22 smithi083 ceph-mon[79930]: pgmap v14881: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:22.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:44:22 smithi153 ceph-mon[70044]: pgmap v14881: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:22.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:44:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:44:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:44:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:44:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:44:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:44:24.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:44:24 smithi083 ceph-mon[77242]: pgmap v14882: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:24.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:44:24 smithi083 ceph-mon[79930]: pgmap v14882: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:24.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:44:24 smithi153 ceph-mon[70044]: pgmap v14882: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:25.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:44:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:44:25.374Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:44:25.835 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:44:26.125 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:44:26.125 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 8m ago 8h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:44:26.126 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 6m ago 8h 64.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:44:26.126 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 8m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:44:26.127 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 6m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:44:26.127 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 6m ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:44:26.127 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 8m ago 8h 919M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:44:26.127 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 8m ago 8h 1806M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:44:26.128 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 6m ago 8h 783M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:44:26.128 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 8m ago 8h 1075M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:44:26.128 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 8m ago 8h 24.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:44:26.128 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 6m ago 8h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:44:26.129 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 8m ago 8h 1308M 2305M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:44:26.129 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 8m ago 8h 1295M 2305M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:44:26.129 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 8m ago 8h 984M 2305M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:44:26.129 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 8m ago 8h 1268M 2305M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:44:26.129 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 6m ago 8h 1114M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:44:26.130 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 6m ago 8h 1395M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:44:26.130 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 6m ago 8h 1247M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:44:26.130 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 6m ago 8h 1481M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:44:26.130 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 6m ago 8h 183M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:44:26.409 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:44:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:44:26.098Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:44:26.480 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:44:26.481 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:44:26.481 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:44:26.481 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:44:26.482 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:44:26.482 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:44:26.482 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:44:26.482 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:44:26.483 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:44:26.483 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:44:26.483 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:44:26.483 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:44:26.484 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:44:26.484 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:44:26.484 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:44:26.484 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:44:26.485 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:44:26.485 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:44:26.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:44:26 smithi083 ceph-mon[79930]: pgmap v14883: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:26.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:44:26 smithi083 ceph-mon[77242]: pgmap v14883: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:26.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:44:26 smithi153 ceph-mon[70044]: pgmap v14883: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:27.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:44:27 smithi083 ceph-mon[79930]: from='client.51438 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:44:27.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:44:27 smithi083 ceph-mon[79930]: from='client.51444 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:44:27.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:44:27 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/978887545' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:44:27.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:44:27 smithi083 ceph-mon[77242]: from='client.51438 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:44:27.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:44:27 smithi083 ceph-mon[77242]: from='client.51444 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:44:27.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:44:27 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/978887545' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:44:27.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:44:27 smithi153 ceph-mon[70044]: from='client.51438 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:44:27.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:44:27 smithi153 ceph-mon[70044]: from='client.51444 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:44:27.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:44:27 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/978887545' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:44:28.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:44:28 smithi083 ceph-mon[79930]: pgmap v14884: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:28.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:44:28 smithi083 ceph-mon[77242]: pgmap v14884: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:28.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:44:28 smithi153 ceph-mon[70044]: pgmap v14884: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:30.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:44:30 smithi083 ceph-mon[79930]: pgmap v14885: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:30.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:44:30 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:44:30.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:44:30 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:44:30.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:44:30 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:44:30.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:44:30 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:44:30.718 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:44:30 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:44:30.718 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:44:30 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:44:30.719 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:44:30 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:44:30.719 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:44:30 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:44:30.719 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:44:30 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:44:30.719 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:44:30 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:44:30.720 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:44:30 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:44:30.720 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:44:30 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:44:30.721 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:44:30 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:44:30.721 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:44:30 smithi083 ceph-mon[77242]: pgmap v14885: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:30.722 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:44:30 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:44:30.722 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:44:30 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:44:30.722 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:44:30 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:44:30.723 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:44:30 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:44:30.723 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:44:30 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:44:30.724 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:44:30 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:44:30.724 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:44:30 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:44:30.724 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:44:30 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:44:30.724 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:44:30 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:44:30.725 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:44:30 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:44:30.725 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:44:30 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:44:30.726 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:44:30 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:44:30.726 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:44:30 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:44:30.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:44:30 smithi153 ceph-mon[70044]: pgmap v14885: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:30.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:44:30 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:44:30.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:44:30 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:44:30.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:44:30 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:44:30.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:44:30 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:44:30.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:44:30 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:44:30.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:44:30 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:44:30.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:44:30 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:44:30.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:44:30 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:44:30.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:44:30 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:44:30.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:44:30 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:44:30.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:44:30 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:44:30.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:44:30 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:44:30.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:44:30 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:44:31.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:44:31 smithi083 ceph-mon[79930]: Adjusting osd_memory_target on smithi083 to 2291M 2022-09-29T13:44:31.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:44:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:44:31] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T13:44:31.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:44:31 smithi083 ceph-mon[77242]: Adjusting osd_memory_target on smithi083 to 2291M 2022-09-29T13:44:31.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:44:31 smithi153 ceph-mon[70044]: Adjusting osd_memory_target on smithi083 to 2291M 2022-09-29T13:44:32.425 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:44:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:44:32.015Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:44:32.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:44:32 smithi083 ceph-mon[79930]: pgmap v14886: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:32.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:44:32 smithi083 ceph-mon[77242]: pgmap v14886: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:32.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:44:32 smithi153 ceph-mon[70044]: pgmap v14886: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:32.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:44:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:44:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:44:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:44:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:44:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:44:34.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:44:34 smithi083 ceph-mon[79930]: pgmap v14887: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:34.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:44:34 smithi083 ceph-mon[77242]: pgmap v14887: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:34.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:44:34 smithi153 ceph-mon[70044]: pgmap v14887: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:35.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:44:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:44:35.374Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:44:36.433 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:44:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:44:36.099Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:44:36.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:44:36 smithi083 ceph-mon[79930]: pgmap v14888: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:36.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:44:36 smithi083 ceph-mon[77242]: pgmap v14888: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:36.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:44:36 smithi153 ceph-mon[70044]: pgmap v14888: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:38.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:44:38 smithi083 ceph-mon[79930]: pgmap v14889: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:38.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:44:38 smithi083 ceph-mon[77242]: pgmap v14889: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:38.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:44:38 smithi153 ceph-mon[70044]: pgmap v14889: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:44:40 smithi083 ceph-mon[79930]: pgmap v14890: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:44:40 smithi083 ceph-mon[77242]: pgmap v14890: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:40.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:44:40 smithi153 ceph-mon[70044]: pgmap v14890: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:41.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:44:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:44:41] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T13:44:41.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:44:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:44:41.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:44:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:44:41.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:44:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:44:41.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:44:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:44:41.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:44:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:44:41.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:44:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:44:41.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:44:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:44:41.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:44:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:44:41.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:44:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:44:41.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:44:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:44:41.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:44:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:44:41.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:44:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:44:42.446 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:44:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:44:42.015Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:44:42.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:44:42 smithi083 ceph-mon[77242]: pgmap v14891: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:42.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:44:42 smithi083 ceph-mon[79930]: pgmap v14891: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:44:42 smithi153 ceph-mon[70044]: pgmap v14891: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:42.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:44:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:44:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:44:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:44:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:44:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:44:44.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:44:44 smithi083 ceph-mon[79930]: pgmap v14892: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:44.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:44:44 smithi083 ceph-mon[77242]: pgmap v14892: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:44.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:44:44 smithi153 ceph-mon[70044]: pgmap v14892: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:45.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:44:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:44:45.374Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:44:46.459 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:44:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:44:46.100Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:44:46.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:44:46 smithi083 ceph-mon[79930]: pgmap v14893: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:46.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:44:46 smithi083 ceph-mon[77242]: pgmap v14893: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:46.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:44:46 smithi153 ceph-mon[70044]: pgmap v14893: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:48.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:44:48 smithi083 ceph-mon[79930]: pgmap v14894: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:48.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:44:48 smithi083 ceph-mon[77242]: pgmap v14894: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:48.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:44:48 smithi153 ceph-mon[70044]: pgmap v14894: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:50.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:44:50 smithi153 ceph-mon[70044]: pgmap v14895: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:50.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:44:50 smithi083 ceph-mon[79930]: pgmap v14895: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:50.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:44:50 smithi083 ceph-mon[77242]: pgmap v14895: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:44:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:44:51] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T13:44:52.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:44:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:44:52.016Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:44:52.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:44:52 smithi153 ceph-mon[70044]: pgmap v14896: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:52.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:44:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:44:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:44:52.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:44:52 smithi083 ceph-mon[79930]: pgmap v14896: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:52.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:44:52 smithi083 ceph-mon[77242]: pgmap v14896: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:44:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:44:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:44:54.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:44:54 smithi153 ceph-mon[70044]: pgmap v14897: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:54.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:44:54 smithi083 ceph-mon[79930]: pgmap v14897: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:54.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:44:54 smithi083 ceph-mon[77242]: pgmap v14897: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:55.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:44:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:44:55.376Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:44:56.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:44:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:44:56.101Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:44:56.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:44:56 smithi153 ceph-mon[70044]: pgmap v14898: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:56.809 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:44:56.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:44:56 smithi083 ceph-mon[79930]: pgmap v14898: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:56.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:44:56 smithi083 ceph-mon[77242]: pgmap v14898: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:57.098 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:44:57.099 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 8m ago 8h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:44:57.099 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 6m ago 8h 64.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:44:57.099 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 8m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:44:57.099 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 6m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:44:57.100 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 6m ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:44:57.100 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 8m ago 8h 919M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:44:57.100 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 8m ago 8h 1806M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:44:57.100 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 6m ago 8h 783M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:44:57.101 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 8m ago 8h 1075M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:44:57.101 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 8m ago 8h 24.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:44:57.101 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 6m ago 8h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:44:57.101 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 8m ago 8h 1308M 2291M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:44:57.102 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 8m ago 8h 1295M 2291M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:44:57.102 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 8m ago 8h 984M 2291M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:44:57.102 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 8m ago 8h 1268M 2291M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:44:57.102 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 6m ago 8h 1114M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:44:57.103 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 6m ago 8h 1395M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:44:57.103 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 6m ago 8h 1247M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:44:57.103 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 6m ago 8h 1481M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:44:57.103 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 6m ago 8h 183M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:44:57.452 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:44:57.452 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:44:57.453 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:44:57.453 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:44:57.453 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:44:57.453 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:44:57.454 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:44:57.454 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:44:57.454 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:44:57.454 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:44:57.455 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:44:57.455 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:44:57.455 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:44:57.456 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:44:57.456 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:44:57.456 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:44:57.456 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:44:57.457 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:44:57.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:44:57 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4221413371' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:44:57.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:44:57 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4221413371' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:44:57.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:44:57 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4221413371' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:44:58.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:44:58 smithi153 ceph-mon[70044]: pgmap v14899: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:58.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:44:58 smithi153 ceph-mon[70044]: from='client.51456 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:44:58.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:44:58 smithi153 ceph-mon[70044]: from='client.61393 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:44:58.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:44:58 smithi083 ceph-mon[79930]: pgmap v14899: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:58.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:44:58 smithi083 ceph-mon[79930]: from='client.51456 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:44:58.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:44:58 smithi083 ceph-mon[79930]: from='client.61393 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:44:58.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:44:58 smithi083 ceph-mon[77242]: pgmap v14899: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:44:58.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:44:58 smithi083 ceph-mon[77242]: from='client.51456 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:44:58.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:44:58 smithi083 ceph-mon[77242]: from='client.61393 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:45:00.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:45:00 smithi153 ceph-mon[70044]: pgmap v14900: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:00.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:45:00 smithi083 ceph-mon[79930]: pgmap v14900: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:00.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:45:00 smithi083 ceph-mon[77242]: pgmap v14900: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:45:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:45:01] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T13:45:02.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:45:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:45:02.016Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:45:02.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:45:02 smithi153 ceph-mon[70044]: pgmap v14901: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:02.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:45:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:45:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:45:02.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:45:02 smithi083 ceph-mon[79930]: pgmap v14901: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:02.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:45:02 smithi083 ceph-mon[77242]: pgmap v14901: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:45:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:45:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:45:04.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:45:04 smithi153 ceph-mon[70044]: pgmap v14902: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:04.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:45:04 smithi083 ceph-mon[79930]: pgmap v14902: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:04.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:45:04 smithi083 ceph-mon[77242]: pgmap v14902: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:05.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:45:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:45:05.376Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:45:06.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:45:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:45:06.102Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:45:06.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:45:06 smithi153 ceph-mon[70044]: pgmap v14903: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:06.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:45:06 smithi083 ceph-mon[79930]: pgmap v14903: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:06.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:45:06 smithi083 ceph-mon[77242]: pgmap v14903: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:08.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:45:08 smithi153 ceph-mon[70044]: pgmap v14904: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:08.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:45:08 smithi083 ceph-mon[79930]: pgmap v14904: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:08.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:45:08 smithi083 ceph-mon[77242]: pgmap v14904: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:10.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:45:10 smithi153 ceph-mon[70044]: pgmap v14905: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:10.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:45:10 smithi083 ceph-mon[77242]: pgmap v14905: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:10.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:45:10 smithi083 ceph-mon[79930]: pgmap v14905: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:45:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:45:11] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T13:45:12.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:45:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:45:12.017Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:45:12.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:45:12 smithi153 ceph-mon[70044]: pgmap v14906: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:12.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:45:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:45:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:45:12.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:45:12 smithi083 ceph-mon[79930]: pgmap v14906: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:12.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:45:12 smithi083 ceph-mon[77242]: pgmap v14906: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:45:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:45:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:45:14.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:45:14 smithi153 ceph-mon[70044]: pgmap v14907: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:14.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:45:14 smithi083 ceph-mon[79930]: pgmap v14907: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:14.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:45:14 smithi083 ceph-mon[77242]: pgmap v14907: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:15.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:45:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:45:15.377Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:45:16.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:45:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:45:16.103Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:45:16.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:45:16 smithi153 ceph-mon[70044]: pgmap v14908: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:16.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:45:16 smithi083 ceph-mon[77242]: pgmap v14908: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:16.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:45:16 smithi083 ceph-mon[79930]: pgmap v14908: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:18.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:45:18 smithi153 ceph-mon[70044]: pgmap v14909: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:18.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:45:18 smithi083 ceph-mon[77242]: pgmap v14909: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:18.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:45:18 smithi083 ceph-mon[79930]: pgmap v14909: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:20.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:45:20 smithi153 ceph-mon[70044]: pgmap v14910: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:20.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:45:20 smithi083 ceph-mon[79930]: pgmap v14910: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:20.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:45:20 smithi083 ceph-mon[77242]: pgmap v14910: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:45:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:45:21] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T13:45:22.472 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:45:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:45:22.017Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:45:22.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:45:22 smithi153 ceph-mon[70044]: pgmap v14911: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:22.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:45:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:45:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:45:22.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:45:22 smithi083 ceph-mon[79930]: pgmap v14911: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:22.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:45:22 smithi083 ceph-mon[77242]: pgmap v14911: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:45:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:45:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:45:24.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:45:24 smithi153 ceph-mon[70044]: pgmap v14912: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:24.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:45:24 smithi083 ceph-mon[79930]: pgmap v14912: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:24.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:45:24 smithi083 ceph-mon[77242]: pgmap v14912: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:25.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:45:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:45:25.378Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:45:26.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:45:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:45:26.103Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:45:26.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:45:26 smithi153 ceph-mon[70044]: pgmap v14913: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:26.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:45:26 smithi083 ceph-mon[79930]: pgmap v14913: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:26.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:45:26 smithi083 ceph-mon[77242]: pgmap v14913: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:27.781 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:45:28.068 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:45:28.068 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 9m ago 8h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:45:28.069 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 7m ago 8h 64.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:45:28.069 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 9m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:45:28.069 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 7m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:45:28.069 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 7m ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:45:28.070 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 9m ago 8h 919M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:45:28.070 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 9m ago 8h 1806M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:45:28.070 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 7m ago 8h 783M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:45:28.070 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 9m ago 8h 1075M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:45:28.071 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 9m ago 8h 24.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:45:28.071 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 7m ago 8h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:45:28.071 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 9m ago 8h 1308M 2291M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:45:28.071 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 9m ago 8h 1295M 2291M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:45:28.072 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 9m ago 8h 984M 2291M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:45:28.072 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 9m ago 8h 1268M 2291M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:45:28.072 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 7m ago 8h 1114M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:45:28.072 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 7m ago 8h 1395M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:45:28.073 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 7m ago 8h 1247M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:45:28.073 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 7m ago 8h 1481M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:45:28.073 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 7m ago 8h 183M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:45:28.424 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:45:28.424 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:45:28.424 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:45:28.425 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:45:28.425 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:45:28.425 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:45:28.425 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:45:28.426 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:45:28.426 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:45:28.426 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:45:28.426 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:45:28.427 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:45:28.427 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:45:28.427 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:45:28.427 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:45:28.428 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:45:28.428 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:45:28.428 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:45:28.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:45:28 smithi153 ceph-mon[70044]: pgmap v14914: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:28.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:45:28 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2890410118' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:45:28.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:45:28 smithi083 ceph-mon[79930]: pgmap v14914: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:28.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:45:28 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2890410118' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:45:28.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:45:28 smithi083 ceph-mon[77242]: pgmap v14914: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:28.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:45:28 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2890410118' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:45:29.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:45:29 smithi153 ceph-mon[70044]: from='client.51474 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:45:29.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:45:29 smithi153 ceph-mon[70044]: from='client.61411 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:45:29.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:45:29 smithi083 ceph-mon[79930]: from='client.51474 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:45:29.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:45:29 smithi083 ceph-mon[79930]: from='client.61411 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:45:29.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:45:29 smithi083 ceph-mon[77242]: from='client.51474 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:45:29.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:45:29 smithi083 ceph-mon[77242]: from='client.61411 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:45:30.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:45:30 smithi153 ceph-mon[70044]: pgmap v14915: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:30.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:45:30 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:45:30.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:45:30 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:45:30.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:45:30 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:45:30.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:45:30 smithi083 ceph-mon[79930]: pgmap v14915: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:30.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:45:30 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:45:30.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:45:30 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:45:30.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:45:30 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:45:30.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:45:30 smithi083 ceph-mon[77242]: pgmap v14915: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:30.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:45:30 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:45:30.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:45:30 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:45:30.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:45:30 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:45:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:45:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:45:31] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T13:45:32.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:45:32 smithi153 ceph-mon[70044]: pgmap v14916: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:32.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:45:32 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:45:32.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:45:32 smithi083 ceph-mon[79930]: pgmap v14916: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:32.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:45:32 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:45:32.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:45:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:45:32.017Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:45:32.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:45:32 smithi083 ceph-mon[77242]: pgmap v14916: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:32.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:45:32 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:45:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:45:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:45:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:45:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:45:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:45:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:45:34.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:45:34 smithi153 ceph-mon[70044]: pgmap v14917: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:34.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:45:34 smithi083 ceph-mon[79930]: pgmap v14917: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:34.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:45:34 smithi083 ceph-mon[77242]: pgmap v14917: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:35.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:45:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:45:35.378Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:45:36.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:45:36 smithi153 ceph-mon[70044]: pgmap v14918: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:36.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:45:36 smithi083 ceph-mon[79930]: pgmap v14918: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:36.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:45:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:45:36.103Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:45:36.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:45:36 smithi083 ceph-mon[77242]: pgmap v14918: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:38.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:45:38 smithi153 ceph-mon[70044]: pgmap v14919: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:38.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:45:38 smithi083 ceph-mon[77242]: pgmap v14919: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:38.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:45:38 smithi083 ceph-mon[79930]: pgmap v14919: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:45:40 smithi153 ceph-mon[70044]: pgmap v14920: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:45:40 smithi083 ceph-mon[77242]: pgmap v14920: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:45:40 smithi083 ceph-mon[79930]: pgmap v14920: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:45:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:45:41.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:45:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:45:41.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:45:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:45:41.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:45:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:45:41.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:45:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:45:41] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T13:45:41.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:45:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:45:41.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:45:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:45:41.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:45:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:45:41.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:45:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:45:41.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:45:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:45:41.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:45:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:45:41.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:45:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:45:41.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:45:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:45:42.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:45:42 smithi083 ceph-mon[79930]: pgmap v14921: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:42.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:45:42 smithi083 ceph-mon[77242]: pgmap v14921: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:42.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:45:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:45:42.017Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:45:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:45:42 smithi153 ceph-mon[70044]: pgmap v14921: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:42.559 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:45:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:45:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:45:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:45:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:45:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:45:44.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:45:44 smithi153 ceph-mon[70044]: pgmap v14922: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:44.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:45:44 smithi083 ceph-mon[77242]: pgmap v14922: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:44.745 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:45:44 smithi083 ceph-mon[79930]: pgmap v14922: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:45.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:45:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:45:45.379Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:45:46.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:45:46 smithi083 ceph-mon[79930]: pgmap v14923: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:46.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:45:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:45:46.104Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:45:46.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:45:46 smithi083 ceph-mon[77242]: pgmap v14923: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:46.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:45:46 smithi153 ceph-mon[70044]: pgmap v14923: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:48.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:45:48 smithi153 ceph-mon[70044]: pgmap v14924: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:48.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:45:48 smithi083 ceph-mon[79930]: pgmap v14924: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:48.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:45:48 smithi083 ceph-mon[77242]: pgmap v14924: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:50.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:45:50 smithi153 ceph-mon[70044]: pgmap v14925: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:50.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:45:50 smithi083 ceph-mon[79930]: pgmap v14925: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:50.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:45:50 smithi083 ceph-mon[77242]: pgmap v14925: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:45:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:45:51] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T13:45:52.283 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:45:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:45:52.018Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:45:52.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:45:52 smithi153 ceph-mon[70044]: pgmap v14926: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:52.559 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:45:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:45:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:45:52.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:45:52 smithi083 ceph-mon[77242]: pgmap v14926: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:52.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:45:52 smithi083 ceph-mon[79930]: pgmap v14926: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:45:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:45:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:45:54.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:45:54 smithi153 ceph-mon[70044]: pgmap v14927: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:54.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:45:54 smithi083 ceph-mon[79930]: pgmap v14927: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:54.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:45:54 smithi083 ceph-mon[77242]: pgmap v14927: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:55.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:45:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:45:55.379Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:45:56.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:45:56 smithi083 ceph-mon[79930]: pgmap v14928: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:56.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:45:56 smithi083 ceph-mon[77242]: pgmap v14928: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:56.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:45:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:45:56.105Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:45:56.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:45:56 smithi153 ceph-mon[70044]: pgmap v14928: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:58.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:45:58 smithi153 ceph-mon[70044]: pgmap v14929: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:58.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:45:58 smithi083 ceph-mon[79930]: pgmap v14929: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:58.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:45:58 smithi083 ceph-mon[77242]: pgmap v14929: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:45:58.757 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:45:59.046 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:45:59.047 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 9m ago 8h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:45:59.047 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 7m ago 8h 64.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:45:59.047 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 9m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:45:59.048 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 7m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:45:59.048 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 7m ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:45:59.048 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 9m ago 8h 919M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:45:59.048 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 9m ago 8h 1806M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:45:59.049 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 7m ago 8h 783M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:45:59.049 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 9m ago 8h 1075M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:45:59.049 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 9m ago 8h 24.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:45:59.049 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 7m ago 8h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:45:59.050 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 9m ago 8h 1308M 2291M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:45:59.050 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 9m ago 8h 1295M 2291M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:45:59.050 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 9m ago 8h 984M 2291M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:45:59.050 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 9m ago 8h 1268M 2291M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:45:59.051 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 7m ago 8h 1114M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:45:59.051 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 7m ago 8h 1395M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:45:59.051 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 7m ago 8h 1247M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:45:59.051 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 7m ago 8h 1481M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:45:59.052 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 7m ago 8h 183M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:45:59.408 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:45:59.409 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:45:59.409 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:45:59.409 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:45:59.410 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:45:59.410 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:45:59.410 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:45:59.411 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:45:59.411 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:45:59.411 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:45:59.411 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:45:59.411 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:45:59.412 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:45:59.412 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:45:59.412 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:45:59.412 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:45:59.413 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:45:59.413 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:46:00.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:46:00 smithi153 ceph-mon[70044]: from='client.51492 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:46:00.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:46:00 smithi153 ceph-mon[70044]: pgmap v14930: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:46:00.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:46:00 smithi153 ceph-mon[70044]: from='client.51498 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:46:00.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:46:00 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2655295971' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:46:00.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:46:00 smithi083 ceph-mon[79930]: from='client.51492 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:46:00.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:46:00 smithi083 ceph-mon[79930]: pgmap v14930: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:46:00.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:46:00 smithi083 ceph-mon[79930]: from='client.51498 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:46:00.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:46:00 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2655295971' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:46:00.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:46:00 smithi083 ceph-mon[77242]: from='client.51492 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:46:00.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:46:00 smithi083 ceph-mon[77242]: pgmap v14930: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:46:00.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:46:00 smithi083 ceph-mon[77242]: from='client.51498 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:46:00.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:46:00 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2655295971' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:46:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:46:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:46:01] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T13:46:02.298 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:46:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:46:02.018Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:46:02.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:46:02 smithi153 ceph-mon[70044]: pgmap v14931: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:46:02.559 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:46:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:46:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:46:02.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:46:02 smithi083 ceph-mon[77242]: pgmap v14931: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:46:02.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:46:02 smithi083 ceph-mon[79930]: pgmap v14931: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:46:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:46:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:46:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:46:04.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:46:04 smithi153 ceph-mon[70044]: pgmap v14932: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:46:04.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:46:04 smithi083 ceph-mon[79930]: pgmap v14932: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:46:04.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:46:04 smithi083 ceph-mon[77242]: pgmap v14932: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:46:05.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:46:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:46:05.380Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:46:06.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:46:06 smithi083 ceph-mon[79930]: pgmap v14933: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:46:06.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:46:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:46:06.107Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:46:06.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:46:06 smithi083 ceph-mon[77242]: pgmap v14933: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:46:06.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:46:06 smithi153 ceph-mon[70044]: pgmap v14933: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:46:08.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:46:08 smithi153 ceph-mon[70044]: pgmap v14934: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:46:08.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:46:08 smithi083 ceph-mon[79930]: pgmap v14934: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:46:08.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:46:08 smithi083 ceph-mon[77242]: pgmap v14934: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:46:10.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:46:10 smithi083 ceph-mon[79930]: pgmap v14935: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:46:10.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:46:10 smithi083 ceph-mon[77242]: pgmap v14935: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:46:10.814 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:46:10 smithi153 ceph-mon[70044]: pgmap v14935: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:46:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:46:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:46:11] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T13:46:12.314 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:46:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:46:12.020Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:46:12.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:46:12 smithi083 ceph-mon[79930]: pgmap v14936: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:46:12.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:46:12 smithi083 ceph-mon[77242]: pgmap v14936: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:46:12.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:46:12 smithi153 ceph-mon[70044]: pgmap v14936: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:46:12.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:46:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:46:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:46:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:46:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:46:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:46:14.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:46:14 smithi083 ceph-mon[79930]: pgmap v14937: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:46:14.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:46:14 smithi083 ceph-mon[77242]: pgmap v14937: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:46:14.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:46:14 smithi153 ceph-mon[70044]: pgmap v14937: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:46:15.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:46:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:46:15.381Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:46:16.468 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:46:16 smithi083 ceph-mon[79930]: pgmap v14938: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:46:16.469 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:46:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:46:16.106Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:46:16.470 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:46:16 smithi083 ceph-mon[77242]: pgmap v14938: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:46:16.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:46:16 smithi153 ceph-mon[70044]: pgmap v14938: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:46:18.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:46:18 smithi083 ceph-mon[77242]: pgmap v14939: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:46:18.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:46:18 smithi083 ceph-mon[79930]: pgmap v14939: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:46:18.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:46:18 smithi153 ceph-mon[70044]: pgmap v14939: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:46:20.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:46:20 smithi083 ceph-mon[79930]: pgmap v14940: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:46:20.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:46:20 smithi083 ceph-mon[77242]: pgmap v14940: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:46:20.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:46:20 smithi153 ceph-mon[70044]: pgmap v14940: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:46:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:46:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:46:21] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T13:46:22.333 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:46:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:46:22.020Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:46:22.334 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:46:22 smithi083 ceph-mon[77242]: pgmap v14941: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:46:22.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:46:22 smithi083 ceph-mon[79930]: pgmap v14941: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:46:22.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:46:22 smithi153 ceph-mon[70044]: pgmap v14941: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:46:22.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:46:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:46:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:46:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:46:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:46:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:46:24.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:46:24 smithi083 ceph-mon[79930]: pgmap v14942: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:46:24.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:46:24 smithi083 ceph-mon[77242]: pgmap v14942: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:46:24.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:46:24 smithi153 ceph-mon[70044]: pgmap v14942: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:46:25.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:46:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:46:25.382Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:46:26.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:46:26 smithi083 ceph-mon[79930]: pgmap v14943: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:46:26.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:46:26 smithi083 ceph-mon[77242]: pgmap v14943: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:46:26.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:46:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:46:26.107Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:46:26.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:46:26 smithi153 ceph-mon[70044]: pgmap v14943: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:46:28.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:46:28 smithi083 ceph-mon[79930]: pgmap v14944: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:46:28.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:46:28 smithi083 ceph-mon[77242]: pgmap v14944: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:46:28.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:46:28 smithi153 ceph-mon[70044]: pgmap v14944: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:46:29.734 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:46:30.024 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:46:30.025 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 10m ago 8h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:46:30.025 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 8m ago 8h 64.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:46:30.026 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 10m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:46:30.026 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 8m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:46:30.027 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 8m ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:46:30.027 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 10m ago 8h 919M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:46:30.027 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 10m ago 8h 1806M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:46:30.028 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 8m ago 8h 783M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:46:30.028 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 10m ago 8h 1075M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:46:30.029 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 10m ago 8h 24.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:46:30.029 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 8m ago 8h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:46:30.030 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 10m ago 8h 1308M 2291M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:46:30.030 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 10m ago 8h 1295M 2291M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:46:30.031 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 10m ago 8h 984M 2291M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:46:30.031 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 10m ago 8h 1268M 2291M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:46:30.032 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 8m ago 8h 1114M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:46:30.032 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 8m ago 8h 1395M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:46:30.032 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 8m ago 8h 1247M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:46:30.033 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 8m ago 8h 1481M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:46:30.033 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 8m ago 8h 183M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:46:30.381 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:46:30.382 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:46:30.382 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:46:30.383 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:46:30.383 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:46:30.383 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:46:30.384 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:46:30.385 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:46:30.385 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:46:30.386 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:46:30.386 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:46:30.387 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:46:30.387 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:46:30.388 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:46:30.388 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:46:30.389 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:46:30.389 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:46:30.389 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:46:30.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:46:30 smithi083 ceph-mon[79930]: pgmap v14945: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:46:30.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:46:30 smithi083 ceph-mon[77242]: pgmap v14945: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:46:30.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:46:30 smithi153 ceph-mon[70044]: pgmap v14945: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:46:31.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:46:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:46:31] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T13:46:31.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:46:31 smithi083 ceph-mon[77242]: from='client.61441 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:46:31.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:46:31 smithi083 ceph-mon[77242]: from='client.61447 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:46:31.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:46:31 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2303315662' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:46:31.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:46:31 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:46:31.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:46:31 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:46:31.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:46:31 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:46:31.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:46:31 smithi083 ceph-mon[79930]: from='client.61441 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:46:31.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:46:31 smithi083 ceph-mon[79930]: from='client.61447 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:46:31.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:46:31 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2303315662' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:46:31.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:46:31 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:46:31.718 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:46:31 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:46:31.718 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:46:31 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:46:31.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:46:31 smithi153 ceph-mon[70044]: from='client.61441 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:46:31.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:46:31 smithi153 ceph-mon[70044]: from='client.61447 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:46:31.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:46:31 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2303315662' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:46:31.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:46:31 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:46:31.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:46:31 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:46:31.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:46:31 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:46:32.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:46:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:46:32.020Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:46:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:46:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:46:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:46:32.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:46:32 smithi153 ceph-mon[70044]: pgmap v14946: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:46:32.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:46:32 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:46:32.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:46:32 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:46:32.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:46:32 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:46:32.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:46:32 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:46:32.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:46:32 smithi083 ceph-mon[79930]: pgmap v14946: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:46:32.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:46:32 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:46:32.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:46:32 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:46:32.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:46:32 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:46:32.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:46:32 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:46:32.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:46:32 smithi083 ceph-mon[77242]: pgmap v14946: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:46:32.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:46:32 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:46:32.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:46:32 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:46:32.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:46:32 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:46:32.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:46:32 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:46:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:46:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:46:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:46:34.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:46:34 smithi083 ceph-mon[79930]: pgmap v14947: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:46:34.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:46:34 smithi083 ceph-mon[77242]: pgmap v14947: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:46:35.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:46:34 smithi153 ceph-mon[70044]: pgmap v14947: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:46:35.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:46:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:46:35.382Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:46:36.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:46:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:46:36.108Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:46:36.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:46:36 smithi083 ceph-mon[79930]: pgmap v14948: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:46:36.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:46:36 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:46:36.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:46:36 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:46:36.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:46:36 smithi083 ceph-mon[77242]: pgmap v14948: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:46:36.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:46:36 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:46:36.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:46:36 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:46:37.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:46:36 smithi153 ceph-mon[70044]: pgmap v14948: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:46:37.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:46:36 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:46:37.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:46:36 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:46:37.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:46:37 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:46:37.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:46:37 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:46:37.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:46:37 smithi083 ceph-mon[79930]: pgmap v14949: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:46:37.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:46:37 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:46:37.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:46:37 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:46:37.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:46:37 smithi083 ceph-mon[77242]: pgmap v14949: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:46:38.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:46:37 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:46:38.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:46:37 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:46:38.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:46:37 smithi153 ceph-mon[70044]: pgmap v14949: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:46:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:46:40 smithi083 ceph-mon[79930]: pgmap v14950: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:46:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:46:40 smithi083 ceph-mon[79930]: Detected new or changed devices on smithi083 2022-09-29T13:46:40.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:46:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:46:40.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:46:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:46:40.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:46:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:46:40.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:46:40 smithi083 ceph-mon[77242]: pgmap v14950: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:46:40.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:46:40 smithi083 ceph-mon[77242]: Detected new or changed devices on smithi083 2022-09-29T13:46:40.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:46:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:46:40.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:46:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:46:40.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:46:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:46:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:46:40 smithi153 ceph-mon[70044]: pgmap v14950: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:46:40.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:46:40 smithi153 ceph-mon[70044]: Detected new or changed devices on smithi083 2022-09-29T13:46:40.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:46:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:46:40.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:46:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:46:40.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:46:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:46:41.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:46:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:46:41] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T13:46:41.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:46:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:46:41.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:46:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:46:41.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:46:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:46:41.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:46:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:46:41.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:46:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:46:41.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:46:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:46:41.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:46:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:46:41.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:46:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:46:41.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:46:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:46:41.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:46:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:46:41.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:46:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:46:41.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:46:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:46:42.402 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:46:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:46:42.021Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:46:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:46:42 smithi083 ceph-mon[79930]: pgmap v14951: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:46:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:46:42 smithi083 ceph-mon[77242]: pgmap v14951: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:46:42.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:46:42 smithi153 ceph-mon[70044]: pgmap v14951: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:46:42.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:46:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:46:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:46:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:46:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:46:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:46:44.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:46:44 smithi083 ceph-mon[79930]: pgmap v14952: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:46:44.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:46:44 smithi083 ceph-mon[77242]: pgmap v14952: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:46:44.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:46:44 smithi153 ceph-mon[70044]: pgmap v14952: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:46:45.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:46:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:46:45.384Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:46:46.407 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:46:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:46:46.109Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:46:46.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:46:46 smithi083 ceph-mon[79930]: pgmap v14953: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:46:46.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:46:46 smithi083 ceph-mon[77242]: pgmap v14953: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:46:46.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:46:46 smithi153 ceph-mon[70044]: pgmap v14953: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:46:48.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:46:48 smithi083 ceph-mon[79930]: pgmap v14954: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:46:48.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:46:48 smithi083 ceph-mon[77242]: pgmap v14954: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:46:48.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:46:48 smithi153 ceph-mon[70044]: pgmap v14954: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:46:50.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:46:50 smithi083 ceph-mon[79930]: pgmap v14955: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:46:50.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:46:50 smithi083 ceph-mon[77242]: pgmap v14955: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:46:50.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:46:50 smithi153 ceph-mon[70044]: pgmap v14955: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:46:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:46:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:46:51] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T13:46:52.415 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:46:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:46:52.022Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:46:52.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:46:52 smithi083 ceph-mon[79930]: pgmap v14956: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:46:52.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:46:52 smithi083 ceph-mon[77242]: pgmap v14956: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:46:52.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:46:52 smithi153 ceph-mon[70044]: pgmap v14956: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:46:52.810 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:46:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:46:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:46:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:46:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:46:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:46:54.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:46:54 smithi083 ceph-mon[79930]: pgmap v14957: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:46:54.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:46:54 smithi083 ceph-mon[77242]: pgmap v14957: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:46:54.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:46:54 smithi153 ceph-mon[70044]: pgmap v14957: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:46:55.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:46:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:46:55.385Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:46:56.423 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:46:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:46:56.110Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:46:56.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:46:56 smithi083 ceph-mon[79930]: pgmap v14958: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:46:56.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:46:56 smithi083 ceph-mon[77242]: pgmap v14958: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:46:56.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:46:56 smithi153 ceph-mon[70044]: pgmap v14958: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:46:58.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:46:58 smithi083 ceph-mon[79930]: pgmap v14959: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:46:58.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:46:58 smithi083 ceph-mon[77242]: pgmap v14959: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:46:58.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:46:58 smithi153 ceph-mon[70044]: pgmap v14959: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:00.712 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:47:00.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:47:00 smithi083 ceph-mon[79930]: pgmap v14960: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:00.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:47:00 smithi083 ceph-mon[77242]: pgmap v14960: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:00.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:47:00 smithi153 ceph-mon[70044]: pgmap v14960: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:01.001 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:47:01.002 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 25s ago 8h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:47:01.002 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 8m ago 8h 64.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:47:01.002 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 25s ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:47:01.003 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 8m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:47:01.003 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 8m ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:47:01.003 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 25s ago 8h 927M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:47:01.003 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 25s ago 8h 1842M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:47:01.004 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 8m ago 8h 783M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:47:01.004 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 25s ago 8h 1097M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:47:01.004 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 25s ago 8h 24.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:47:01.004 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 8m ago 8h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:47:01.004 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 25s ago 8h 1334M 2291M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:47:01.005 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 25s ago 8h 1320M 2291M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:47:01.005 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 25s ago 8h 1028M 2291M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:47:01.005 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 25s ago 8h 1294M 2291M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:47:01.005 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 8m ago 8h 1114M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:47:01.006 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 8m ago 8h 1395M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:47:01.006 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 8m ago 8h 1247M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:47:01.006 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 8m ago 8h 1481M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:47:01.006 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 8m ago 8h 183M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:47:01.358 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:47:01.359 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:47:01.359 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:47:01.359 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:47:01.360 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:47:01.360 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:47:01.360 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:47:01.360 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:47:01.360 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:47:01.361 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:47:01.361 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:47:01.361 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:47:01.361 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:47:01.362 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:47:01.362 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:47:01.362 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:47:01.362 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:47:01.363 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:47:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:47:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:47:01] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T13:47:02.279 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:47:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:47:02.023Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:47:02.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:47:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:47:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:47:02.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:47:02 smithi153 ceph-mon[70044]: from='client.61459 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:47:02.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:47:02 smithi153 ceph-mon[70044]: pgmap v14961: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:02.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:47:02 smithi153 ceph-mon[70044]: from='client.51534 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:47:02.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:47:02 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1486988141' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:47:02.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:47:02 smithi083 ceph-mon[77242]: from='client.61459 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:47:02.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:47:02 smithi083 ceph-mon[77242]: pgmap v14961: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:02.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:47:02 smithi083 ceph-mon[77242]: from='client.51534 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:47:02.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:47:02 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1486988141' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:47:02.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:47:02 smithi083 ceph-mon[79930]: from='client.61459 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:47:02.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:47:02 smithi083 ceph-mon[79930]: pgmap v14961: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:02.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:47:02 smithi083 ceph-mon[79930]: from='client.51534 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:47:02.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:47:02 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1486988141' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:47:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:47:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:47:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:47:04.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:47:04 smithi153 ceph-mon[70044]: pgmap v14962: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:04.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:47:04 smithi083 ceph-mon[79930]: pgmap v14962: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:04.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:47:04 smithi083 ceph-mon[77242]: pgmap v14962: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:05.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:47:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:47:05.385Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:47:06.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:47:06 smithi083 ceph-mon[79930]: pgmap v14963: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:06.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:47:06 smithi083 ceph-mon[77242]: pgmap v14963: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:06.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:47:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:47:06.110Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:47:06.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:47:06 smithi153 ceph-mon[70044]: pgmap v14963: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:08.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:47:08 smithi153 ceph-mon[70044]: pgmap v14964: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:08.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:47:08 smithi083 ceph-mon[79930]: pgmap v14964: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:08.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:47:08 smithi083 ceph-mon[77242]: pgmap v14964: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:10.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:47:10 smithi153 ceph-mon[70044]: pgmap v14965: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:10.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:47:10 smithi083 ceph-mon[79930]: pgmap v14965: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:10.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:47:10 smithi083 ceph-mon[77242]: pgmap v14965: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:47:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:47:11] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T13:47:12.292 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:47:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:47:12.023Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:47:12.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:47:12 smithi153 ceph-mon[70044]: pgmap v14966: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:12.559 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:47:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:47:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:47:12.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:47:12 smithi083 ceph-mon[77242]: pgmap v14966: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:12.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:47:12 smithi083 ceph-mon[79930]: pgmap v14966: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:47:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:47:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:47:14.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:47:14 smithi153 ceph-mon[70044]: pgmap v14967: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:14.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:47:14 smithi083 ceph-mon[79930]: pgmap v14967: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:14.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:47:14 smithi083 ceph-mon[77242]: pgmap v14967: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:15.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:47:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:47:15.386Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:47:16.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:47:16 smithi083 ceph-mon[79930]: pgmap v14968: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:16.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:47:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:47:16.111Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:47:16.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:47:16 smithi083 ceph-mon[77242]: pgmap v14968: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:16.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:47:16 smithi153 ceph-mon[70044]: pgmap v14968: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:18.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:47:18 smithi153 ceph-mon[70044]: pgmap v14969: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:18.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:47:18 smithi083 ceph-mon[79930]: pgmap v14969: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:18.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:47:18 smithi083 ceph-mon[77242]: pgmap v14969: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:20.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:47:20 smithi153 ceph-mon[70044]: pgmap v14970: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:20.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:47:20 smithi083 ceph-mon[77242]: pgmap v14970: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:20.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:47:20 smithi083 ceph-mon[79930]: pgmap v14970: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:47:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:47:21] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T13:47:22.309 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:47:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:47:22.024Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:47:22.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:47:22 smithi083 ceph-mon[77242]: pgmap v14971: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:22.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:47:22 smithi083 ceph-mon[79930]: pgmap v14971: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:22.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:47:22 smithi153 ceph-mon[70044]: pgmap v14971: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:22.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:47:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:47:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:47:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:47:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:47:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:47:24.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:47:24 smithi083 ceph-mon[77242]: pgmap v14972: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:24.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:47:24 smithi083 ceph-mon[79930]: pgmap v14972: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:24.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:47:24 smithi153 ceph-mon[70044]: pgmap v14972: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:25.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:47:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:47:25.386Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:47:26.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:47:26 smithi083 ceph-mon[79930]: pgmap v14973: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:26.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:47:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:47:26.112Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:47:26.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:47:26 smithi083 ceph-mon[77242]: pgmap v14973: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:26.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:47:26 smithi153 ceph-mon[70044]: pgmap v14973: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:28.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:47:28 smithi083 ceph-mon[77242]: pgmap v14974: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:28.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:47:28 smithi083 ceph-mon[79930]: pgmap v14974: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:28.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:47:28 smithi153 ceph-mon[70044]: pgmap v14974: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:30.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:47:30 smithi083 ceph-mon[79930]: pgmap v14975: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:30.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:47:30 smithi083 ceph-mon[77242]: pgmap v14975: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:30.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:47:30 smithi153 ceph-mon[70044]: pgmap v14975: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:31.686 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:47:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:47:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:47:31] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T13:47:31.975 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:47:31.975 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 56s ago 8h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:47:31.975 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 9m ago 8h 64.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:47:31.976 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 56s ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:47:31.976 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 9m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:47:31.976 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 9m ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:47:31.977 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 56s ago 8h 927M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:47:31.977 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 56s ago 8h 1842M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:47:31.978 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 9m ago 8h 783M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:47:31.978 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 56s ago 8h 1097M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:47:31.979 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 56s ago 8h 24.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:47:31.979 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 9m ago 8h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:47:31.980 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 56s ago 8h 1334M 2291M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:47:31.980 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 56s ago 8h 1320M 2291M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:47:31.981 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 56s ago 8h 1028M 2291M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:47:31.981 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 56s ago 8h 1294M 2291M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:47:31.982 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 9m ago 8h 1114M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:47:31.982 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 9m ago 8h 1395M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:47:31.983 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 9m ago 8h 1247M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:47:31.983 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 9m ago 8h 1481M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:47:31.983 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 9m ago 8h 183M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:47:32.325 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:47:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:47:32.026Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:47:32.333 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:47:32.333 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:47:32.334 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:47:32.334 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:47:32.334 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:47:32.334 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:47:32.335 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:47:32.335 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:47:32.335 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:47:32.335 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:47:32.336 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:47:32.336 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:47:32.336 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:47:32.337 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:47:32.337 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:47:32.337 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:47:32.337 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:47:32.337 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:47:32.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:47:32 smithi083 ceph-mon[79930]: pgmap v14976: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:32.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:47:32 smithi083 ceph-mon[77242]: pgmap v14976: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:32.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:47:32 smithi153 ceph-mon[70044]: pgmap v14976: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:32.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:47:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:47:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:47:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:47:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:47:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:47:33.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:47:33 smithi083 ceph-mon[79930]: from='client.51546 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:47:33.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:47:33 smithi083 ceph-mon[79930]: from='client.61483 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:47:33.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:47:33 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4049619470' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:47:33.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:47:33 smithi083 ceph-mon[77242]: from='client.51546 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:47:33.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:47:33 smithi083 ceph-mon[77242]: from='client.61483 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:47:33.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:47:33 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4049619470' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:47:33.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:47:33 smithi153 ceph-mon[70044]: from='client.51546 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:47:33.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:47:33 smithi153 ceph-mon[70044]: from='client.61483 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:47:33.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:47:33 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4049619470' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:47:34.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:47:34 smithi083 ceph-mon[79930]: pgmap v14977: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:34.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:47:34 smithi083 ceph-mon[77242]: pgmap v14977: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:34.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:47:34 smithi153 ceph-mon[70044]: pgmap v14977: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:35.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:47:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:47:35.387Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:47:36.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:47:36 smithi083 ceph-mon[77242]: pgmap v14978: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:36.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:47:36 smithi083 ceph-mon[79930]: pgmap v14978: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:36.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:47:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:47:36.113Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:47:36.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:47:36 smithi153 ceph-mon[70044]: pgmap v14978: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:38.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:47:38 smithi083 ceph-mon[79930]: pgmap v14979: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:38.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:47:38 smithi083 ceph-mon[77242]: pgmap v14979: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:38.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:47:38 smithi153 ceph-mon[70044]: pgmap v14979: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:40.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:47:40 smithi083 ceph-mon[79930]: pgmap v14980: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:40.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:47:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:47:40.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:47:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:47:40.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:47:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:47:40.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:47:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:47:40.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:47:40 smithi083 ceph-mon[77242]: pgmap v14980: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:40.718 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:47:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:47:40.718 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:47:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:47:40.718 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:47:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:47:40.719 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:47:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:47:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:47:40 smithi153 ceph-mon[70044]: pgmap v14980: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:40.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:47:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:47:40.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:47:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:47:40.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:47:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:47:40.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:47:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:47:41.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:47:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:47:41.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:47:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:47:41.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:47:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:47:41.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:47:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:47:41.714 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:47:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:47:41] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T13:47:41.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:47:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:47:41.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:47:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:47:41.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:47:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:47:41.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:47:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:47:41.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:47:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:47:41.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:47:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:47:41.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:47:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:47:41.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:47:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:47:42.349 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:47:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:47:42.026Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:47:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:47:42 smithi083 ceph-mon[79930]: pgmap v14981: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:47:42 smithi083 ceph-mon[77242]: pgmap v14981: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:47:42 smithi153 ceph-mon[70044]: pgmap v14981: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:42.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:47:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:47:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:47:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:47:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:47:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:47:44.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:47:44 smithi083 ceph-mon[79930]: pgmap v14982: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:44.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:47:44 smithi083 ceph-mon[77242]: pgmap v14982: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:44.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:47:44 smithi153 ceph-mon[70044]: pgmap v14982: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:45.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:47:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:47:45.387Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:47:46.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:47:46 smithi083 ceph-mon[79930]: pgmap v14983: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:46.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:47:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:47:46.114Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:47:46.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:47:46 smithi083 ceph-mon[77242]: pgmap v14983: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:46.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:47:46 smithi153 ceph-mon[70044]: pgmap v14983: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:48.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:47:48 smithi083 ceph-mon[77242]: pgmap v14984: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:48.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:47:48 smithi083 ceph-mon[79930]: pgmap v14984: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:48.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:47:48 smithi153 ceph-mon[70044]: pgmap v14984: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:50.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:47:50 smithi083 ceph-mon[79930]: pgmap v14985: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:50.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:47:50 smithi083 ceph-mon[77242]: pgmap v14985: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:50.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:47:50 smithi153 ceph-mon[70044]: pgmap v14985: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:47:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:47:51] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T13:47:52.364 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:47:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:47:52.026Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:47:52.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:47:52 smithi083 ceph-mon[79930]: pgmap v14986: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:52.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:47:52 smithi083 ceph-mon[77242]: pgmap v14986: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:52.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:47:52 smithi153 ceph-mon[70044]: pgmap v14986: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:52.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:47:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:47:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:47:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:47:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:47:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:47:54.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:47:54 smithi083 ceph-mon[79930]: pgmap v14987: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:54.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:47:54 smithi083 ceph-mon[77242]: pgmap v14987: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:54.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:47:54 smithi153 ceph-mon[70044]: pgmap v14987: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:55.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:47:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:47:55.387Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:47:56.377 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:47:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:47:56.114Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:47:56.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:47:56 smithi083 ceph-mon[79930]: pgmap v14988: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:56.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:47:56 smithi083 ceph-mon[77242]: pgmap v14988: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:56.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:47:56 smithi153 ceph-mon[70044]: pgmap v14988: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:58.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:47:58 smithi083 ceph-mon[79930]: pgmap v14989: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:58.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:47:58 smithi083 ceph-mon[77242]: pgmap v14989: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:47:58.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:47:58 smithi153 ceph-mon[70044]: pgmap v14989: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:00.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:48:00 smithi083 ceph-mon[79930]: pgmap v14990: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:00.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:48:00 smithi083 ceph-mon[77242]: pgmap v14990: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:00.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:48:00 smithi153 ceph-mon[70044]: pgmap v14990: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:01.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:48:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:48:01] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T13:48:02.389 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:48:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:48:02.027Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:48:02.662 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:48:02.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:48:02 smithi083 ceph-mon[79930]: pgmap v14991: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:02.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:48:02 smithi083 ceph-mon[77242]: pgmap v14991: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:02.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:48:02 smithi153 ceph-mon[70044]: pgmap v14991: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:02.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:48:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:48:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:48:02.952 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:48:02.953 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 87s ago 8h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:48:02.953 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 9m ago 8h 64.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:48:02.953 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 87s ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:48:02.953 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 9m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:48:02.954 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 9m ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:48:02.954 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 87s ago 8h 927M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:48:02.954 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 87s ago 8h 1842M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:48:02.955 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 9m ago 8h 783M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:48:02.955 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 87s ago 8h 1097M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:48:02.955 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 87s ago 8h 24.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:48:02.955 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 9m ago 8h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:48:02.956 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 87s ago 8h 1334M 2291M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:48:02.956 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 87s ago 8h 1320M 2291M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:48:02.956 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 87s ago 8h 1028M 2291M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:48:02.956 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 87s ago 8h 1294M 2291M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:48:02.957 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 9m ago 8h 1114M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:48:02.957 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 9m ago 8h 1395M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:48:02.957 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 9m ago 8h 1247M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:48:02.957 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 9m ago 8h 1481M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:48:02.958 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 9m ago 8h 183M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:48:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:48:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:48:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:48:03.309 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:48:03.309 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:48:03.309 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:48:03.310 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:48:03.310 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:48:03.310 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:48:03.311 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:48:03.311 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:48:03.311 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:48:03.311 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:48:03.311 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:48:03.312 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:48:03.312 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:48:03.312 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:48:03.312 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:48:03.313 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:48:03.313 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:48:03.313 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:48:03.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:48:03 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/470174154' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:48:03.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:48:03 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/470174154' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:48:03.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:48:03 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/470174154' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:48:04.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:48:04 smithi083 ceph-mon[79930]: from='client.51564 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:48:04.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:48:04 smithi083 ceph-mon[79930]: pgmap v14992: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:04.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:48:04 smithi083 ceph-mon[79930]: from='client.51570 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:48:04.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:48:04 smithi083 ceph-mon[77242]: from='client.51564 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:48:04.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:48:04 smithi083 ceph-mon[77242]: pgmap v14992: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:04.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:48:04 smithi083 ceph-mon[77242]: from='client.51570 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:48:04.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:48:04 smithi153 ceph-mon[70044]: from='client.51564 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:48:04.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:48:04 smithi153 ceph-mon[70044]: pgmap v14992: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:04.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:48:04 smithi153 ceph-mon[70044]: from='client.51570 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:48:05.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:48:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:48:05.388Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:48:06.396 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:48:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:48:06.115Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:48:06.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:48:06 smithi083 ceph-mon[79930]: pgmap v14993: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:06.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:48:06 smithi083 ceph-mon[77242]: pgmap v14993: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:06.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:48:06 smithi153 ceph-mon[70044]: pgmap v14993: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:08.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:48:08 smithi083 ceph-mon[79930]: pgmap v14994: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:08.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:48:08 smithi083 ceph-mon[77242]: pgmap v14994: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:08.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:48:08 smithi153 ceph-mon[70044]: pgmap v14994: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:10.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:48:10 smithi083 ceph-mon[79930]: pgmap v14995: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:10.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:48:10 smithi083 ceph-mon[77242]: pgmap v14995: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:10.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:48:10 smithi153 ceph-mon[70044]: pgmap v14995: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:48:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:48:11] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T13:48:12.405 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:48:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:48:12.028Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:48:12.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:48:12 smithi083 ceph-mon[77242]: pgmap v14996: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:12.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:48:12 smithi083 ceph-mon[79930]: pgmap v14996: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:12.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:48:12 smithi153 ceph-mon[70044]: pgmap v14996: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:12.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:48:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:48:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:48:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:48:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:48:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:48:14.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:48:14 smithi083 ceph-mon[79930]: pgmap v14997: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:14.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:48:14 smithi083 ceph-mon[77242]: pgmap v14997: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:14.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:48:14 smithi153 ceph-mon[70044]: pgmap v14997: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:15.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:48:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:48:15.389Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:48:16.410 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:48:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:48:16.115Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:48:16.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:48:16 smithi083 ceph-mon[79930]: pgmap v14998: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:16.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:48:16 smithi083 ceph-mon[77242]: pgmap v14998: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:16.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:48:16 smithi153 ceph-mon[70044]: pgmap v14998: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:18.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:48:18 smithi083 ceph-mon[77242]: pgmap v14999: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:18.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:48:18 smithi083 ceph-mon[79930]: pgmap v14999: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:18.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:48:18 smithi153 ceph-mon[70044]: pgmap v14999: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:20.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:48:20 smithi083 ceph-mon[79930]: pgmap v15000: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:20.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:48:20 smithi083 ceph-mon[77242]: pgmap v15000: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:20.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:48:20 smithi153 ceph-mon[70044]: pgmap v15000: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:48:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:48:21] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T13:48:22.419 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:48:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:48:22.029Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:48:22.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:48:22 smithi083 ceph-mon[77242]: pgmap v15001: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:22.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:48:22 smithi083 ceph-mon[79930]: pgmap v15001: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:22.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:48:22 smithi153 ceph-mon[70044]: pgmap v15001: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:22.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:48:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:48:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:48:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:48:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:48:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:48:24.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:48:24 smithi083 ceph-mon[79930]: pgmap v15002: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:24.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:48:24 smithi083 ceph-mon[77242]: pgmap v15002: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:24.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:48:24 smithi153 ceph-mon[70044]: pgmap v15002: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:25.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:48:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:48:25.389Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:48:26.426 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:48:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:48:26.115Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:48:26.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:48:26 smithi083 ceph-mon[79930]: pgmap v15003: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:26.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:48:26 smithi083 ceph-mon[77242]: pgmap v15003: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:26.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:48:26 smithi153 ceph-mon[70044]: pgmap v15003: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:28.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:48:28 smithi083 ceph-mon[79930]: pgmap v15004: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:28.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:48:28 smithi083 ceph-mon[77242]: pgmap v15004: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:28.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:48:28 smithi153 ceph-mon[70044]: pgmap v15004: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:30.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:48:30 smithi083 ceph-mon[79930]: pgmap v15005: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:30.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:48:30 smithi083 ceph-mon[77242]: pgmap v15005: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:30.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:48:30 smithi153 ceph-mon[70044]: pgmap v15005: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:48:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:48:31] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T13:48:32.434 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:48:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:48:32.029Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:48:32.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:48:32 smithi083 ceph-mon[77242]: pgmap v15006: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:32.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:48:32 smithi083 ceph-mon[79930]: pgmap v15006: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:32.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:48:32 smithi153 ceph-mon[70044]: pgmap v15006: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:32.810 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:48:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:48:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:48:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:48:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:48:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:48:33.641 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:48:33.935 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:48:33.936 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 118s ago 8h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:48:33.936 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 10m ago 8h 64.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:48:33.936 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 118s ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:48:33.936 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 10m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:48:33.937 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 10m ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:48:33.937 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 118s ago 8h 927M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:48:33.937 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 118s ago 8h 1842M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:48:33.937 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 10m ago 8h 783M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:48:33.937 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 118s ago 8h 1097M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:48:33.938 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 118s ago 8h 24.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:48:33.938 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 10m ago 8h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:48:33.938 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 118s ago 8h 1334M 2291M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:48:33.938 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 118s ago 8h 1320M 2291M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:48:33.939 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 118s ago 8h 1028M 2291M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:48:33.939 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 118s ago 8h 1294M 2291M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:48:33.939 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 10m ago 8h 1114M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:48:33.939 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 10m ago 8h 1395M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:48:33.942 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 10m ago 8h 1247M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:48:33.942 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 10m ago 8h 1481M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:48:33.942 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 10m ago 8h 183M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:48:34.290 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:48:34.290 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:48:34.290 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:48:34.291 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:48:34.291 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:48:34.291 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:48:34.291 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:48:34.291 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:48:34.292 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:48:34.292 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:48:34.292 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:48:34.292 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:48:34.293 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:48:34.293 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:48:34.293 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:48:34.293 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:48:34.293 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:48:34.294 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:48:34.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:48:34 smithi083 ceph-mon[77242]: pgmap v15007: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:34.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:48:34 smithi083 ceph-mon[77242]: from='client.61513 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:48:34.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:48:34 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3130594571' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:48:34.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:48:34 smithi083 ceph-mon[79930]: pgmap v15007: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:34.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:48:34 smithi083 ceph-mon[79930]: from='client.61513 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:48:34.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:48:34 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3130594571' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:48:34.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:48:34 smithi153 ceph-mon[70044]: pgmap v15007: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:34.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:48:34 smithi153 ceph-mon[70044]: from='client.61513 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:48:34.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:48:34 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3130594571' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:48:35.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:48:35 smithi083 ceph-mon[77242]: from='client.61519 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:48:35.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:48:35 smithi083 ceph-mon[79930]: from='client.61519 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:48:35.714 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:48:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:48:35.390Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:48:35.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:48:35 smithi153 ceph-mon[70044]: from='client.61519 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:48:36.445 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:48:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:48:36.117Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:48:36.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:48:36 smithi083 ceph-mon[77242]: pgmap v15008: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:36.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:48:36 smithi083 ceph-mon[79930]: pgmap v15008: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:36.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:48:36 smithi153 ceph-mon[70044]: pgmap v15008: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:38.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:48:38 smithi083 ceph-mon[77242]: pgmap v15009: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:38.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:48:38 smithi083 ceph-mon[79930]: pgmap v15009: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:38.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:48:38 smithi153 ceph-mon[70044]: pgmap v15009: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:48:40 smithi083 ceph-mon[79930]: pgmap v15010: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:48:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:48:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:48:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:48:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:48:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:48:40.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:48:40 smithi083 ceph-mon[77242]: pgmap v15010: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:40.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:48:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:48:40.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:48:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:48:40.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:48:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:48:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:48:40 smithi153 ceph-mon[70044]: pgmap v15010: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:48:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:48:40.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:48:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:48:40.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:48:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:48:41.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:48:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:48:41] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T13:48:41.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:48:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:48:41.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:48:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:48:41.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:48:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:48:41.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:48:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:48:41.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:48:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:48:41.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:48:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:48:41.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:48:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:48:41.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:48:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:48:41.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:48:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:48:41.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:48:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:48:41.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:48:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:48:41.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:48:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:48:42.460 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:48:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:48:42.030Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:48:42.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:48:42 smithi083 ceph-mon[77242]: pgmap v15011: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:42.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:48:42 smithi083 ceph-mon[79930]: pgmap v15011: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:48:42 smithi153 ceph-mon[70044]: pgmap v15011: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:42.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:48:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:48:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:48:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:48:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:48:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:48:44.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:48:44 smithi153 ceph-mon[70044]: pgmap v15012: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:44.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:48:44 smithi083 ceph-mon[79930]: pgmap v15012: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:44.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:48:44 smithi083 ceph-mon[77242]: pgmap v15012: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:45.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:48:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:48:45.391Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:48:45.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:48:45 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:48:45.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:48:45 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:48:45.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:48:45 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:48:45.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:48:45 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:48:45.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:48:45 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:48:45.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:48:45 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:48:45.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:48:45 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:48:45.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:48:45 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:48:46.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:48:45 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:48:46.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:48:45 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:48:46.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:48:45 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:48:46.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:48:45 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:48:46.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:48:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:48:46.117Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:48:46.746 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:48:46 smithi153 ceph-mon[70044]: pgmap v15013: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:46.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:48:46 smithi083 ceph-mon[77242]: pgmap v15013: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:46.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:48:46 smithi083 ceph-mon[79930]: pgmap v15013: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:48.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:48:48 smithi083 ceph-mon[79930]: pgmap v15014: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:48.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:48:48 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:48:48.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:48:48 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:48:48.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:48:48 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:48:48.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:48:48 smithi083 ceph-mon[77242]: pgmap v15014: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:48.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:48:48 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:48:48.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:48:48 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:48:48.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:48:48 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:48:49.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:48:48 smithi153 ceph-mon[70044]: pgmap v15014: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:49.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:48:48 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:48:49.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:48:48 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:48:49.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:48:48 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:48:49.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:48:49 smithi083 ceph-mon[79930]: Detected new or changed devices on smithi153 2022-09-29T13:48:49.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:48:49 smithi083 ceph-mon[77242]: Detected new or changed devices on smithi153 2022-09-29T13:48:50.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:48:49 smithi153 ceph-mon[70044]: Detected new or changed devices on smithi153 2022-09-29T13:48:50.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:48:50 smithi083 ceph-mon[79930]: pgmap v15015: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:50.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:48:50 smithi083 ceph-mon[77242]: pgmap v15015: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:51.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:48:50 smithi153 ceph-mon[70044]: pgmap v15015: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:51.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:48:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:48:51] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T13:48:52.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:48:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:48:52.031Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:48:52.651 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:48:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:48:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:48:52.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:48:52 smithi083 ceph-mon[79930]: pgmap v15016: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:52.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:48:52 smithi083 ceph-mon[77242]: pgmap v15016: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:53.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:48:52 smithi153 ceph-mon[70044]: pgmap v15016: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:48:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:48:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:48:53.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:48:53 smithi083 ceph-mon[77242]: pgmap v15017: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:53.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:48:53 smithi083 ceph-mon[79930]: pgmap v15017: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:54.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:48:53 smithi153 ceph-mon[70044]: pgmap v15017: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:55.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:48:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:48:55.391Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:48:56.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:48:55 smithi083 ceph-mon[79930]: pgmap v15018: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:56.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:48:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:48:56.117Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:48:56.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:48:55 smithi083 ceph-mon[77242]: pgmap v15018: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:56.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:48:55 smithi153 ceph-mon[70044]: pgmap v15018: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:58.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:48:58 smithi153 ceph-mon[70044]: pgmap v15019: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:58.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:48:58 smithi083 ceph-mon[79930]: pgmap v15019: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:48:58.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:48:58 smithi083 ceph-mon[77242]: pgmap v15019: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:00.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:49:00 smithi083 ceph-mon[79930]: pgmap v15020: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:00.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:49:00 smithi083 ceph-mon[77242]: pgmap v15020: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:00.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:49:00 smithi153 ceph-mon[70044]: pgmap v15020: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:49:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:49:01] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T13:49:02.313 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:49:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:49:02.031Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:49:02.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:49:02 smithi083 ceph-mon[79930]: pgmap v15021: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:02.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:49:02 smithi083 ceph-mon[77242]: pgmap v15021: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:02.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:49:02 smithi153 ceph-mon[70044]: pgmap v15021: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:02.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:49:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:49:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:49:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:49:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:49:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:49:04.620 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:49:04.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:49:04 smithi083 ceph-mon[79930]: pgmap v15022: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:04.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:49:04 smithi083 ceph-mon[77242]: pgmap v15022: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:04.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:49:04 smithi153 ceph-mon[70044]: pgmap v15022: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:04.912 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:49:04.912 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 2m ago 8h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:49:04.912 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 20s ago 8h 63.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:49:04.913 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 2m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:49:04.913 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 20s ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:49:04.913 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 20s ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:49:04.913 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 2m ago 8h 927M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:49:04.914 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 2m ago 8h 1842M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:49:04.914 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 20s ago 8h 799M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:49:04.914 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 2m ago 8h 1097M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:49:04.914 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 2m ago 8h 24.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:49:04.915 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 20s ago 8h 26.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:49:04.915 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 2m ago 8h 1334M 2291M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:49:04.915 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 2m ago 8h 1320M 2291M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:49:04.916 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 2m ago 8h 1028M 2291M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:49:04.916 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 2m ago 8h 1294M 2291M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:49:04.916 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 20s ago 8h 1135M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:49:04.916 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 20s ago 8h 1424M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:49:04.917 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 20s ago 8h 1272M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:49:04.917 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 20s ago 8h 1511M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:49:04.917 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 20s ago 8h 185M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:49:05.266 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:49:05.266 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:49:05.267 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:49:05.267 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:49:05.267 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:49:05.268 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:49:05.268 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:49:05.268 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:49:05.268 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:49:05.268 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:49:05.269 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:49:05.269 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:49:05.269 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:49:05.269 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:49:05.270 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:49:05.270 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:49:05.270 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:49:05.271 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:49:05.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:49:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:49:05.391Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:49:05.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:49:05 smithi083 ceph-mon[77242]: from='client.61531 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:49:05.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:49:05 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/711537893' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:49:05.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:49:05 smithi083 ceph-mon[79930]: from='client.61531 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:49:05.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:49:05 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/711537893' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:49:05.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:49:05 smithi153 ceph-mon[70044]: from='client.61531 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:49:05.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:49:05 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/711537893' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:49:06.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:49:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:49:06.118Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:49:06.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:49:06 smithi083 ceph-mon[77242]: pgmap v15023: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:06.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:49:06 smithi083 ceph-mon[77242]: from='client.51606 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:49:06.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:49:06 smithi083 ceph-mon[79930]: pgmap v15023: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:06.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:49:06 smithi083 ceph-mon[79930]: from='client.51606 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:49:06.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:49:06 smithi153 ceph-mon[70044]: pgmap v15023: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:06.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:49:06 smithi153 ceph-mon[70044]: from='client.51606 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:49:08.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:49:08 smithi083 ceph-mon[77242]: pgmap v15024: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:08.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:49:08 smithi083 ceph-mon[79930]: pgmap v15024: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:08.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:49:08 smithi153 ceph-mon[70044]: pgmap v15024: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:10.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:49:10 smithi083 ceph-mon[79930]: pgmap v15025: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:10.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:49:10 smithi083 ceph-mon[77242]: pgmap v15025: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:10.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:49:10 smithi153 ceph-mon[70044]: pgmap v15025: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:11.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:49:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:49:11] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T13:49:12.331 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:49:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:49:12.031Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:49:12.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:49:12 smithi083 ceph-mon[79930]: pgmap v15026: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:12.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:49:12 smithi083 ceph-mon[77242]: pgmap v15026: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:12.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:49:12 smithi153 ceph-mon[70044]: pgmap v15026: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:12.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:49:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:49:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:49:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:49:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:49:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:49:14.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:49:14 smithi083 ceph-mon[79930]: pgmap v15027: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:14.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:49:14 smithi083 ceph-mon[77242]: pgmap v15027: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:14.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:49:14 smithi153 ceph-mon[70044]: pgmap v15027: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:15.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:49:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:49:15.393Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:49:16.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:49:16 smithi083 ceph-mon[79930]: pgmap v15028: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:16.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:49:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:49:16.118Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:49:16.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:49:16 smithi083 ceph-mon[77242]: pgmap v15028: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:16.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:49:16 smithi153 ceph-mon[70044]: pgmap v15028: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:18.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:49:18 smithi083 ceph-mon[79930]: pgmap v15029: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:18.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:49:18 smithi083 ceph-mon[77242]: pgmap v15029: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:18.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:49:18 smithi153 ceph-mon[70044]: pgmap v15029: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:20.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:49:20 smithi083 ceph-mon[79930]: pgmap v15030: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:20.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:49:20 smithi083 ceph-mon[77242]: pgmap v15030: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:20.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:49:20 smithi153 ceph-mon[70044]: pgmap v15030: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:21.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:49:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:49:21] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T13:49:22.347 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:49:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:49:22.032Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:49:22.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:49:22 smithi083 ceph-mon[77242]: pgmap v15031: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:22.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:49:22 smithi083 ceph-mon[79930]: pgmap v15031: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:22.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:49:22 smithi153 ceph-mon[70044]: pgmap v15031: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:22.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:49:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:49:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:49:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:49:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:49:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:49:24.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:49:24 smithi083 ceph-mon[77242]: pgmap v15032: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:24.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:49:24 smithi083 ceph-mon[79930]: pgmap v15032: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:24.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:49:24 smithi153 ceph-mon[70044]: pgmap v15032: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:25.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:49:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:49:25.394Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:49:26.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:49:26 smithi083 ceph-mon[79930]: pgmap v15033: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:26.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:49:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:49:26.119Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:49:26.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:49:26 smithi083 ceph-mon[77242]: pgmap v15033: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:26.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:49:26 smithi153 ceph-mon[70044]: pgmap v15033: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:28.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:49:28 smithi083 ceph-mon[79930]: pgmap v15034: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:28.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:49:28 smithi083 ceph-mon[77242]: pgmap v15034: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:28.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:49:28 smithi153 ceph-mon[70044]: pgmap v15034: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:30.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:49:30 smithi083 ceph-mon[79930]: pgmap v15035: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:30.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:49:30 smithi083 ceph-mon[77242]: pgmap v15035: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:30.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:49:30 smithi153 ceph-mon[70044]: pgmap v15035: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:49:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:49:31] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T13:49:32.362 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:49:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:49:32.032Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:49:32.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:49:32 smithi083 ceph-mon[77242]: pgmap v15036: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:32.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:49:32 smithi083 ceph-mon[79930]: pgmap v15036: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:32.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:49:32 smithi153 ceph-mon[70044]: pgmap v15036: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:32.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:49:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:49:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:49:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:49:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:49:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:49:34.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:49:34 smithi083 ceph-mon[77242]: pgmap v15037: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:34.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:49:34 smithi083 ceph-mon[79930]: pgmap v15037: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:34.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:49:34 smithi153 ceph-mon[70044]: pgmap v15037: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:35.599 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:49:35.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:49:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:49:35.395Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:49:35.887 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:49:35.888 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 3m ago 8h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:49:35.888 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 51s ago 8h 63.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:49:35.889 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 3m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:49:35.889 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 51s ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:49:35.889 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 51s ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:49:35.890 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 3m ago 8h 927M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:49:35.890 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 3m ago 8h 1842M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:49:35.890 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 51s ago 8h 799M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:49:35.891 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 3m ago 8h 1097M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:49:35.891 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 3m ago 8h 24.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:49:35.892 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 51s ago 8h 26.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:49:35.892 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 3m ago 8h 1334M 2291M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:49:35.892 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 3m ago 8h 1320M 2291M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:49:35.893 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 3m ago 8h 1028M 2291M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:49:35.893 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 3m ago 8h 1294M 2291M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:49:35.893 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 51s ago 8h 1135M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:49:35.894 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 51s ago 8h 1424M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:49:35.894 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 51s ago 8h 1272M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:49:35.895 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 51s ago 8h 1511M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:49:35.895 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 51s ago 8h 185M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:49:36.242 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:49:36.242 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:49:36.242 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:49:36.243 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:49:36.243 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:49:36.243 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:49:36.243 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:49:36.243 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:49:36.244 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:49:36.244 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:49:36.244 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:49:36.244 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:49:36.245 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:49:36.245 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:49:36.245 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:49:36.245 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:49:36.246 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:49:36.246 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:49:36.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:49:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:49:36.119Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:49:36.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:49:36 smithi083 ceph-mon[77242]: pgmap v15038: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:36.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:49:36 smithi083 ceph-mon[77242]: from='client.61549 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:49:36.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:49:36 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2022005034' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:49:36.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:49:36 smithi083 ceph-mon[79930]: pgmap v15038: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:36.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:49:36 smithi083 ceph-mon[79930]: from='client.61549 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:49:36.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:49:36 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2022005034' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:49:36.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:49:36 smithi153 ceph-mon[70044]: pgmap v15038: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:36.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:49:36 smithi153 ceph-mon[70044]: from='client.61549 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:49:36.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:49:36 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2022005034' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:49:37.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:49:37 smithi083 ceph-mon[79930]: from='client.61555 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:49:37.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:49:37 smithi083 ceph-mon[77242]: from='client.61555 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:49:37.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:49:37 smithi153 ceph-mon[70044]: from='client.61555 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:49:38.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:49:38 smithi083 ceph-mon[77242]: pgmap v15039: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:38.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:49:38 smithi083 ceph-mon[79930]: pgmap v15039: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:38.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:49:38 smithi153 ceph-mon[70044]: pgmap v15039: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:49:40 smithi083 ceph-mon[79930]: pgmap v15040: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:40.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:49:40 smithi083 ceph-mon[77242]: pgmap v15040: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:49:40 smithi153 ceph-mon[70044]: pgmap v15040: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:41.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:49:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:49:41] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T13:49:41.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:49:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:49:41.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:49:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:49:41.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:49:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:49:41.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:49:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:49:41.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:49:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:49:41.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:49:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:49:41.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:49:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:49:41.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:49:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:49:41.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:49:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:49:41.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:49:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:49:41.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:49:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:49:41.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:49:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:49:42.390 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:49:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:49:42.034Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:49:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:49:42 smithi083 ceph-mon[79930]: pgmap v15041: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:49:42 smithi083 ceph-mon[77242]: pgmap v15041: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:49:42 smithi153 ceph-mon[70044]: pgmap v15041: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:42.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:49:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:49:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:49:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:49:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:49:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:49:44.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:49:44 smithi083 ceph-mon[79930]: pgmap v15042: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:44.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:49:44 smithi083 ceph-mon[77242]: pgmap v15042: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:44.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:49:44 smithi153 ceph-mon[70044]: pgmap v15042: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:45.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:49:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:49:45.395Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:49:46.397 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:49:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:49:46.120Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:49:46.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:49:46 smithi083 ceph-mon[77242]: pgmap v15043: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:46.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:49:46 smithi083 ceph-mon[79930]: pgmap v15043: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:46.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:49:46 smithi153 ceph-mon[70044]: pgmap v15043: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:48.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:49:48 smithi083 ceph-mon[79930]: pgmap v15044: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:48.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:49:48 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:49:48.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:49:48 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:49:48.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:49:48 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:49:48.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:49:48 smithi083 ceph-mon[77242]: pgmap v15044: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:48.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:49:48 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:49:48.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:49:48 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:49:48.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:49:48 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:49:48.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:49:48 smithi153 ceph-mon[70044]: pgmap v15044: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:48.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:49:48 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:49:48.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:49:48 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:49:48.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:49:48 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:49:50.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:49:49 smithi083 ceph-mon[79930]: pgmap v15045: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:50.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:49:49 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:49:50.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:49:49 smithi083 ceph-mon[77242]: pgmap v15045: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:50.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:49:49 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:49:50.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:49:49 smithi153 ceph-mon[70044]: pgmap v15045: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:50.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:49:49 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:49:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:49:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:49:51] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T13:49:52.322 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:49:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:49:52.034Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:49:52.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:49:52 smithi083 ceph-mon[77242]: pgmap v15046: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:52.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:49:52 smithi083 ceph-mon[79930]: pgmap v15046: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:52.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:49:52 smithi153 ceph-mon[70044]: pgmap v15046: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:52.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:49:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:49:52.317Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:49:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:49:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:49:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:49:54.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:49:54 smithi083 ceph-mon[77242]: pgmap v15047: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:54.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:49:54 smithi083 ceph-mon[79930]: pgmap v15047: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:54.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:49:54 smithi153 ceph-mon[70044]: pgmap v15047: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:55.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:49:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:49:55.395Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:49:56.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:49:56 smithi083 ceph-mon[79930]: pgmap v15048: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:56.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:49:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:49:56.120Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:49:56.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:49:56 smithi083 ceph-mon[77242]: pgmap v15048: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:56.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:49:56 smithi153 ceph-mon[70044]: pgmap v15048: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:58.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:49:58 smithi083 ceph-mon[79930]: pgmap v15049: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:58.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:49:58 smithi083 ceph-mon[77242]: pgmap v15049: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:49:58.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:49:58 smithi153 ceph-mon[70044]: pgmap v15049: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:00.333 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:50:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[77219]: 2022-09-29T13:49:59.999+0000 7f160f665700 -1 log_channel(cluster) log [ERR] : overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T13:50:00.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:50:00 smithi083 ceph-mon[77242]: pgmap v15050: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:00.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:50:00 smithi083 ceph-mon[77242]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T13:50:00.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:50:00 smithi083 ceph-mon[79930]: pgmap v15050: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:00.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:50:00 smithi083 ceph-mon[79930]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T13:50:00.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:50:00 smithi153 ceph-mon[70044]: pgmap v15050: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:00.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:50:00 smithi153 ceph-mon[70044]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T13:50:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:50:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:50:01] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T13:50:02.336 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:50:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:50:02.035Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:50:02.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:50:02 smithi083 ceph-mon[79930]: pgmap v15051: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:02.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:50:02 smithi083 ceph-mon[77242]: pgmap v15051: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:02.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:50:02 smithi153 ceph-mon[70044]: pgmap v15051: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:02.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:50:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:50:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:50:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:50:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:50:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:50:04.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:50:04 smithi083 ceph-mon[79930]: pgmap v15052: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:04.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:50:04 smithi083 ceph-mon[77242]: pgmap v15052: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:04.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:50:04 smithi153 ceph-mon[70044]: pgmap v15052: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:05.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:50:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:50:05.396Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:50:06.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:50:06 smithi083 ceph-mon[79930]: pgmap v15053: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:06.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:50:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:50:06.121Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:50:06.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:50:06 smithi083 ceph-mon[77242]: pgmap v15053: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:06.570 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:50:06.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:50:06 smithi153 ceph-mon[70044]: pgmap v15053: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:06.859 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:50:06.860 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 3m ago 8h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:50:06.860 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 82s ago 8h 63.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:50:06.860 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 3m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:50:06.860 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 82s ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:50:06.861 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 82s ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:50:06.861 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 3m ago 8h 927M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:50:06.861 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 3m ago 8h 1842M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:50:06.862 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 82s ago 8h 799M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:50:06.862 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 3m ago 8h 1097M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:50:06.862 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 3m ago 8h 24.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:50:06.863 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 82s ago 8h 26.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:50:06.863 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 3m ago 8h 1334M 2291M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:50:06.863 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 3m ago 8h 1320M 2291M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:50:06.864 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 3m ago 8h 1028M 2291M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:50:06.864 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 3m ago 8h 1294M 2291M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:50:06.864 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 82s ago 8h 1135M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:50:06.864 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 82s ago 8h 1424M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:50:06.865 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 82s ago 8h 1272M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:50:06.865 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 82s ago 8h 1511M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:50:06.865 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 82s ago 8h 185M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:50:07.217 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:50:07.217 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:50:07.218 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:50:07.218 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:50:07.219 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:50:07.219 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:50:07.219 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:50:07.219 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:50:07.220 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:50:07.220 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:50:07.220 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:50:07.220 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:50:07.221 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:50:07.221 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:50:07.222 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:50:07.222 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:50:07.223 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:50:07.223 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:50:07.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:50:07 smithi083 ceph-mon[79930]: from='client.51636 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:50:07.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:50:07 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1861604128' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:50:07.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:50:07 smithi083 ceph-mon[77242]: from='client.51636 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:50:07.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:50:07 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1861604128' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:50:07.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:50:07 smithi153 ceph-mon[70044]: from='client.51636 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:50:07.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:50:07 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1861604128' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:50:08.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:50:08 smithi083 ceph-mon[79930]: from='client.61573 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:50:08.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:50:08 smithi083 ceph-mon[79930]: pgmap v15054: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:08.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:50:08 smithi083 ceph-mon[77242]: from='client.61573 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:50:08.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:50:08 smithi083 ceph-mon[77242]: pgmap v15054: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:08.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:50:08 smithi153 ceph-mon[70044]: from='client.61573 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:50:08.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:50:08 smithi153 ceph-mon[70044]: pgmap v15054: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:10.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:50:10 smithi083 ceph-mon[79930]: pgmap v15055: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:10.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:50:10 smithi083 ceph-mon[77242]: pgmap v15055: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:10.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:50:10 smithi153 ceph-mon[70044]: pgmap v15055: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:50:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:50:11] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T13:50:12.357 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:50:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:50:12.036Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:50:12.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:50:12 smithi083 ceph-mon[77242]: pgmap v15056: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:12.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:50:12 smithi083 ceph-mon[79930]: pgmap v15056: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:12.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:50:12 smithi153 ceph-mon[70044]: pgmap v15056: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:12.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:50:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:50:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:50:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:50:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:50:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:50:14.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:50:14 smithi083 ceph-mon[77242]: pgmap v15057: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:14.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:50:14 smithi083 ceph-mon[79930]: pgmap v15057: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:14.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:50:14 smithi153 ceph-mon[70044]: pgmap v15057: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:15.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:50:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:50:15.397Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:50:16.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:50:16 smithi083 ceph-mon[79930]: pgmap v15058: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:16.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:50:16 smithi083 ceph-mon[77242]: pgmap v15058: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:16.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:50:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:50:16.122Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:50:16.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:50:16 smithi153 ceph-mon[70044]: pgmap v15058: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:18.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:50:18 smithi083 ceph-mon[79930]: pgmap v15059: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:18.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:50:18 smithi083 ceph-mon[77242]: pgmap v15059: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:18.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:50:18 smithi153 ceph-mon[70044]: pgmap v15059: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:20.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:50:20 smithi083 ceph-mon[79930]: pgmap v15060: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:20.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:50:20 smithi083 ceph-mon[77242]: pgmap v15060: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:20.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:50:20 smithi153 ceph-mon[70044]: pgmap v15060: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:50:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:50:21] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T13:50:22.371 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:50:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:50:22.036Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:50:22.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:50:22 smithi083 ceph-mon[79930]: pgmap v15061: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:22.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:50:22 smithi083 ceph-mon[77242]: pgmap v15061: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:22.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:50:22 smithi153 ceph-mon[70044]: pgmap v15061: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:22.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:50:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:50:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:50:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:50:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:50:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:50:24.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:50:24 smithi083 ceph-mon[79930]: pgmap v15062: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:24.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:50:24 smithi083 ceph-mon[77242]: pgmap v15062: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:24.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:50:24 smithi153 ceph-mon[70044]: pgmap v15062: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:25.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:50:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:50:25.398Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:50:26.379 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:50:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:50:26.122Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:50:26.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:50:26 smithi083 ceph-mon[79930]: pgmap v15063: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:26.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:50:26 smithi083 ceph-mon[77242]: pgmap v15063: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:26.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:50:26 smithi153 ceph-mon[70044]: pgmap v15063: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:28.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:50:28 smithi083 ceph-mon[79930]: pgmap v15064: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:28.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:50:28 smithi083 ceph-mon[77242]: pgmap v15064: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:28.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:50:28 smithi153 ceph-mon[70044]: pgmap v15064: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:30.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:50:30 smithi083 ceph-mon[77242]: pgmap v15065: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:30.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:50:30 smithi083 ceph-mon[79930]: pgmap v15065: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:30.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:50:30 smithi153 ceph-mon[70044]: pgmap v15065: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:50:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:50:31] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T13:50:32.388 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:50:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:50:32.037Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:50:32.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:50:32 smithi083 ceph-mon[79930]: pgmap v15066: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:32.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:50:32 smithi083 ceph-mon[77242]: pgmap v15066: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:32.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:50:32 smithi153 ceph-mon[70044]: pgmap v15066: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:32.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:50:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:50:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:50:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:50:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:50:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:50:34.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:50:34 smithi083 ceph-mon[79930]: pgmap v15067: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:34.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:50:34 smithi083 ceph-mon[77242]: pgmap v15067: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:34.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:50:34 smithi153 ceph-mon[70044]: pgmap v15067: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:35.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:50:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:50:35.398Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:50:36.397 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:50:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:50:36.122Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:50:36.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:50:36 smithi083 ceph-mon[79930]: pgmap v15068: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:36.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:50:36 smithi083 ceph-mon[77242]: pgmap v15068: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:36.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:50:36 smithi153 ceph-mon[70044]: pgmap v15068: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:37.544 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:50:37.833 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:50:37.834 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 4m ago 8h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:50:37.834 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 113s ago 8h 63.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:50:37.835 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 4m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:50:37.835 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 113s ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:50:37.835 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 113s ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:50:37.835 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 4m ago 8h 927M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:50:37.836 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 4m ago 8h 1842M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:50:37.836 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 113s ago 8h 799M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:50:37.836 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 4m ago 8h 1097M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:50:37.836 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 4m ago 8h 24.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:50:37.837 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 113s ago 8h 26.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:50:37.837 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 4m ago 8h 1334M 2291M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:50:37.837 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 4m ago 8h 1320M 2291M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:50:37.837 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 4m ago 8h 1028M 2291M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:50:37.838 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 4m ago 8h 1294M 2291M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:50:37.838 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 113s ago 8h 1135M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:50:37.838 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 113s ago 8h 1424M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:50:37.838 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 113s ago 8h 1272M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:50:37.838 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 113s ago 8h 1511M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:50:37.839 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 113s ago 8h 185M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:50:38.192 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:50:38.192 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:50:38.192 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:50:38.193 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:50:38.193 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:50:38.193 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:50:38.193 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:50:38.194 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:50:38.194 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:50:38.194 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:50:38.194 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:50:38.195 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:50:38.195 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:50:38.195 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:50:38.195 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:50:38.196 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:50:38.196 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:50:38.196 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:50:38.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:50:38 smithi083 ceph-mon[79930]: pgmap v15069: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:38.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:50:38 smithi083 ceph-mon[79930]: from='client.51654 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:50:38.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:50:38 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2557346188' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:50:38.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:50:38 smithi083 ceph-mon[77242]: pgmap v15069: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:38.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:50:38 smithi083 ceph-mon[77242]: from='client.51654 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:50:38.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:50:38 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2557346188' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:50:38.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:50:38 smithi153 ceph-mon[70044]: pgmap v15069: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:38.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:50:38 smithi153 ceph-mon[70044]: from='client.51654 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:50:38.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:50:38 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2557346188' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:50:39.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:50:39 smithi083 ceph-mon[79930]: from='client.51660 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:50:39.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:50:39 smithi083 ceph-mon[77242]: from='client.51660 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:50:39.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:50:39 smithi153 ceph-mon[70044]: from='client.51660 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:50:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:50:40 smithi083 ceph-mon[79930]: pgmap v15070: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:50:40 smithi083 ceph-mon[77242]: pgmap v15070: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:50:40 smithi153 ceph-mon[70044]: pgmap v15070: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:41.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:50:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:50:41] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T13:50:41.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:50:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:50:41.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:50:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:50:41.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:50:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:50:41.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:50:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:50:41.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:50:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:50:41.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:50:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:50:41.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:50:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:50:41.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:50:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:50:41.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:50:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:50:41.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:50:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:50:41.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:50:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:50:41.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:50:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:50:42.416 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:50:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:50:42.038Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:50:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:50:42 smithi083 ceph-mon[79930]: pgmap v15071: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:50:42 smithi083 ceph-mon[77242]: pgmap v15071: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:50:42 smithi153 ceph-mon[70044]: pgmap v15071: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:42.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:50:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:50:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:50:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:50:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:50:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:50:44.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:50:44 smithi083 ceph-mon[79930]: pgmap v15072: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:44.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:50:44 smithi083 ceph-mon[77242]: pgmap v15072: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:44.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:50:44 smithi153 ceph-mon[70044]: pgmap v15072: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:45.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:50:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:50:45.398Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:50:46.421 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:50:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:50:46.122Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:50:46.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:50:46 smithi083 ceph-mon[79930]: pgmap v15073: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:46.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:50:46 smithi083 ceph-mon[77242]: pgmap v15073: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:46.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:50:46 smithi153 ceph-mon[70044]: pgmap v15073: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:48.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:50:48 smithi083 ceph-mon[79930]: pgmap v15074: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:48.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:50:48 smithi083 ceph-mon[77242]: pgmap v15074: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:48.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:50:48 smithi153 ceph-mon[70044]: pgmap v15074: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:49.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:50:49 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:50:49.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:50:49 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:50:49.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:50:49 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:50:49.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:50:49 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:50:49.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:50:49 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:50:49.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:50:49 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:50:49.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:50:49 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:50:49.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:50:49 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:50:49.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:50:49 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:50:50.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:50:50 smithi153 ceph-mon[70044]: pgmap v15075: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:50.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:50:50 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:50:50.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:50:50 smithi083 ceph-mon[77242]: pgmap v15075: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:50.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:50:50 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:50:50.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:50:50 smithi083 ceph-mon[79930]: pgmap v15075: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:50.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:50:50 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:50:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:50:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:50:51] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T13:50:52.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:50:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:50:52.039Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:50:52.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:50:52 smithi153 ceph-mon[70044]: pgmap v15076: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:52.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:50:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:50:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:50:52.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:50:52 smithi083 ceph-mon[79930]: pgmap v15076: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:52.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:50:52 smithi083 ceph-mon[77242]: pgmap v15076: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:50:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:50:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:50:54.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:50:54 smithi153 ceph-mon[70044]: pgmap v15077: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:54.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:50:54 smithi083 ceph-mon[79930]: pgmap v15077: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:54.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:50:54 smithi083 ceph-mon[77242]: pgmap v15077: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:55.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:50:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:50:55.399Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:50:56.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:50:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:50:56.123Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:50:56.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:50:56 smithi153 ceph-mon[70044]: pgmap v15078: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:56.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:50:56 smithi083 ceph-mon[79930]: pgmap v15078: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:56.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:50:56 smithi083 ceph-mon[77242]: pgmap v15078: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:58.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:50:58 smithi153 ceph-mon[70044]: pgmap v15079: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:58.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:50:58 smithi083 ceph-mon[79930]: pgmap v15079: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:50:58.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:50:58 smithi083 ceph-mon[77242]: pgmap v15079: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:00.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:51:00 smithi083 ceph-mon[77242]: pgmap v15080: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:00.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:51:00 smithi083 ceph-mon[79930]: pgmap v15080: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:01.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:51:00 smithi153 ceph-mon[70044]: pgmap v15080: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:51:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:51:01] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T13:51:02.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:51:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:51:02.039Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:51:02.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:51:02 smithi153 ceph-mon[70044]: pgmap v15081: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:02.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:51:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:51:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:51:02.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:51:02 smithi083 ceph-mon[79930]: pgmap v15081: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:02.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:51:02 smithi083 ceph-mon[77242]: pgmap v15081: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:51:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:51:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:51:04.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:51:04 smithi083 ceph-mon[79930]: pgmap v15082: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:04.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:51:04 smithi083 ceph-mon[77242]: pgmap v15082: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:05.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:51:04 smithi153 ceph-mon[70044]: pgmap v15082: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:05.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:51:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:51:05.399Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:51:06.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:51:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:51:06.125Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:51:06.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:51:06 smithi083 ceph-mon[79930]: pgmap v15083: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:06.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:51:06 smithi083 ceph-mon[77242]: pgmap v15083: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:07.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:51:06 smithi153 ceph-mon[70044]: pgmap v15083: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:08.518 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:51:08.805 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:51:08.806 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 4m ago 8h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:51:08.806 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 2m ago 8h 63.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:51:08.806 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 4m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:51:08.807 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 2m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:51:08.807 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 2m ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:51:08.807 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 4m ago 8h 927M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:51:08.807 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 4m ago 8h 1842M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:51:08.808 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 2m ago 8h 799M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:51:08.808 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 4m ago 8h 1097M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:51:08.808 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 4m ago 8h 24.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:51:08.808 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 2m ago 8h 26.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:51:08.809 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 4m ago 8h 1334M 2291M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:51:08.809 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 4m ago 8h 1320M 2291M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:51:08.809 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 4m ago 8h 1028M 2291M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:51:08.810 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 4m ago 8h 1294M 2291M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:51:08.810 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 2m ago 8h 1135M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:51:08.810 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 2m ago 8h 1424M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:51:08.810 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 2m ago 8h 1272M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:51:08.811 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 2m ago 8h 1511M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:51:08.811 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 2m ago 8h 185M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:51:08.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:51:08 smithi083 ceph-mon[79930]: pgmap v15084: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:08.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:51:08 smithi083 ceph-mon[77242]: pgmap v15084: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:09.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:51:08 smithi153 ceph-mon[70044]: pgmap v15084: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:09.159 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:51:09.159 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:51:09.159 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:51:09.160 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:51:09.160 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:51:09.160 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:51:09.160 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:51:09.161 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:51:09.161 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:51:09.161 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:51:09.161 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:51:09.162 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:51:09.162 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:51:09.162 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:51:09.162 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:51:09.163 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:51:09.163 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:51:09.163 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:51:09.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:51:09 smithi083 ceph-mon[79930]: from='client.51672 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:51:09.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:51:09 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4202969400' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:51:09.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:51:09 smithi083 ceph-mon[77242]: from='client.51672 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:51:09.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:51:09 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4202969400' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:51:10.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:51:09 smithi153 ceph-mon[70044]: from='client.51672 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:51:10.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:51:09 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4202969400' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:51:10.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:51:10 smithi083 ceph-mon[79930]: from='client.51678 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:51:10.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:51:10 smithi083 ceph-mon[79930]: pgmap v15085: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:10.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:51:10 smithi083 ceph-mon[77242]: from='client.51678 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:51:10.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:51:10 smithi083 ceph-mon[77242]: pgmap v15085: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:11.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:51:10 smithi153 ceph-mon[70044]: from='client.51678 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:51:11.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:51:10 smithi153 ceph-mon[70044]: pgmap v15085: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:51:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:51:11] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T13:51:12.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:51:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:51:12.040Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:51:12.586 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:51:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:51:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:51:12.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:51:12 smithi083 ceph-mon[79930]: pgmap v15086: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:12.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:51:12 smithi083 ceph-mon[77242]: pgmap v15086: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:13.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:51:12 smithi153 ceph-mon[70044]: pgmap v15086: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:51:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:51:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:51:14.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:51:14 smithi083 ceph-mon[79930]: pgmap v15087: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:14.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:51:14 smithi083 ceph-mon[77242]: pgmap v15087: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:15.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:51:14 smithi153 ceph-mon[70044]: pgmap v15087: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:15.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:51:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:51:15.400Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:51:16.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:51:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:51:16.125Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:51:16.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:51:16 smithi083 ceph-mon[77242]: pgmap v15088: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:16.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:51:16 smithi083 ceph-mon[79930]: pgmap v15088: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:17.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:51:16 smithi153 ceph-mon[70044]: pgmap v15088: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:18.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:51:18 smithi083 ceph-mon[79930]: pgmap v15089: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:18.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:51:18 smithi083 ceph-mon[77242]: pgmap v15089: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:19.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:51:18 smithi153 ceph-mon[70044]: pgmap v15089: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:20.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:51:20 smithi083 ceph-mon[77242]: pgmap v15090: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:20.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:51:20 smithi083 ceph-mon[79930]: pgmap v15090: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:21.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:51:20 smithi153 ceph-mon[70044]: pgmap v15090: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:51:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:51:21] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T13:51:22.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:51:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:51:22.040Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:51:22.600 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:51:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:51:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:51:22.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:51:22 smithi083 ceph-mon[77242]: pgmap v15091: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:22.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:51:22 smithi083 ceph-mon[79930]: pgmap v15091: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:23.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:51:22 smithi153 ceph-mon[70044]: pgmap v15091: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:51:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:51:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:51:24.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:51:24 smithi083 ceph-mon[77242]: pgmap v15092: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:24.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:51:24 smithi083 ceph-mon[79930]: pgmap v15092: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:25.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:51:24 smithi153 ceph-mon[70044]: pgmap v15092: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:25.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:51:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:51:25.401Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:51:26.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:51:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:51:26.125Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:51:26.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:51:26 smithi083 ceph-mon[79930]: pgmap v15093: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:26.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:51:26 smithi083 ceph-mon[77242]: pgmap v15093: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:27.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:51:26 smithi153 ceph-mon[70044]: pgmap v15093: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:28.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:51:28 smithi083 ceph-mon[77242]: pgmap v15094: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:28.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:51:28 smithi083 ceph-mon[79930]: pgmap v15094: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:29.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:51:28 smithi153 ceph-mon[70044]: pgmap v15094: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:30.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:51:30 smithi083 ceph-mon[79930]: pgmap v15095: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:30.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:51:30 smithi083 ceph-mon[77242]: pgmap v15095: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:31.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:51:30 smithi153 ceph-mon[70044]: pgmap v15095: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:51:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:51:31] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T13:51:32.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:51:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:51:32.040Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:51:32.616 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:51:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:51:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:51:32.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:51:32 smithi083 ceph-mon[79930]: pgmap v15096: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:32.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:51:32 smithi083 ceph-mon[77242]: pgmap v15096: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:33.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:51:32 smithi153 ceph-mon[70044]: pgmap v15096: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:51:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:51:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:51:34.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:51:34 smithi083 ceph-mon[79930]: pgmap v15097: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:34.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:51:34 smithi083 ceph-mon[77242]: pgmap v15097: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:35.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:51:34 smithi153 ceph-mon[70044]: pgmap v15097: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:35.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:51:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:51:35.402Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:51:36.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:51:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:51:36.126Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:51:36.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:51:36 smithi083 ceph-mon[79930]: pgmap v15098: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:36.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:51:36 smithi083 ceph-mon[77242]: pgmap v15098: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:37.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:51:36 smithi153 ceph-mon[70044]: pgmap v15098: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:38.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:51:38 smithi083 ceph-mon[79930]: pgmap v15099: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:38.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:51:38 smithi083 ceph-mon[77242]: pgmap v15099: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:39.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:51:38 smithi153 ceph-mon[70044]: pgmap v15099: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:39.484 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:51:39.776 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:51:39.777 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 5m ago 8h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:51:39.777 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 2m ago 8h 63.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:51:39.777 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 5m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:51:39.778 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 2m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:51:39.778 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 2m ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:51:39.778 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 5m ago 8h 927M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:51:39.778 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 5m ago 8h 1842M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:51:39.778 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 2m ago 8h 799M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:51:39.779 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 5m ago 8h 1097M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:51:39.779 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 5m ago 8h 24.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:51:39.779 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 2m ago 8h 26.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:51:39.779 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 5m ago 8h 1334M 2291M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:51:39.780 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 5m ago 8h 1320M 2291M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:51:39.780 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 5m ago 8h 1028M 2291M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:51:39.780 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 5m ago 8h 1294M 2291M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:51:39.780 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 2m ago 8h 1135M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:51:39.781 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 2m ago 8h 1424M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:51:39.781 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 2m ago 8h 1272M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:51:39.781 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 2m ago 8h 1511M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:51:39.781 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 2m ago 8h 185M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:51:40.133 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:51:40.133 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:51:40.134 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:51:40.134 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:51:40.134 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:51:40.134 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:51:40.135 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:51:40.135 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:51:40.135 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:51:40.135 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:51:40.136 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:51:40.136 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:51:40.136 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:51:40.136 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:51:40.136 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:51:40.137 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:51:40.137 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:51:40.137 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:51:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:51:40 smithi083 ceph-mon[79930]: pgmap v15100: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:51:40 smithi083 ceph-mon[79930]: from='client.61621 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:51:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:51:40 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/43762631' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:51:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:51:40 smithi083 ceph-mon[77242]: pgmap v15100: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:40.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:51:40 smithi083 ceph-mon[77242]: from='client.61621 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:51:40.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:51:40 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/43762631' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:51:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:51:40 smithi153 ceph-mon[70044]: pgmap v15100: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:51:40 smithi153 ceph-mon[70044]: from='client.61621 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:51:41.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:51:40 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/43762631' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:51:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:51:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:51:41] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T13:51:41.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:51:41 smithi083 ceph-mon[79930]: from='client.51696 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:51:41.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:51:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:51:41.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:51:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:51:41.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:51:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:51:41.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:51:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:51:41.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:51:41 smithi083 ceph-mon[77242]: from='client.51696 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:51:41.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:51:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:51:41.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:51:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:51:41.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:51:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:51:41.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:51:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:51:42.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:51:41 smithi153 ceph-mon[70044]: from='client.51696 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:51:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:51:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:51:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:51:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:51:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:51:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:51:42.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:51:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:51:42.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:51:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:51:42.041Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:51:42.636 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:51:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:51:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:51:42.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:51:42 smithi083 ceph-mon[79930]: pgmap v15101: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:42.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:51:42 smithi083 ceph-mon[77242]: pgmap v15101: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:43.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:51:42 smithi153 ceph-mon[70044]: pgmap v15101: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:51:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:51:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:51:44.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:51:44 smithi083 ceph-mon[79930]: pgmap v15102: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:44.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:51:44 smithi083 ceph-mon[77242]: pgmap v15102: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:45.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:51:44 smithi153 ceph-mon[70044]: pgmap v15102: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:45.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:51:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:51:45.402Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:51:46.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:51:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:51:46.127Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:51:46.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:51:46 smithi083 ceph-mon[77242]: pgmap v15103: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:46.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:51:46 smithi083 ceph-mon[79930]: pgmap v15103: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:47.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:51:46 smithi153 ceph-mon[70044]: pgmap v15103: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:48.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:51:48 smithi083 ceph-mon[79930]: pgmap v15104: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:48.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:51:48 smithi083 ceph-mon[77242]: pgmap v15104: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:49.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:51:48 smithi153 ceph-mon[70044]: pgmap v15104: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:49.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:51:49 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:51:49.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:51:49 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:51:49.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:51:49 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:51:49.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:51:49 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:51:49.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:51:49 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:51:49.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:51:49 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:51:50.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:51:49 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:51:50.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:51:49 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:51:50.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:51:49 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:51:50.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:51:50 smithi083 ceph-mon[77242]: pgmap v15105: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:50.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:51:50 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:51:50.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:51:50 smithi083 ceph-mon[79930]: pgmap v15105: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:50.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:51:50 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:51:51.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:51:50 smithi153 ceph-mon[70044]: pgmap v15105: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:51.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:51:50 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:51:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:51:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:51:51] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T13:51:52.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:51:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:51:52.042Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:51:52.654 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:51:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:51:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:51:52.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:51:52 smithi083 ceph-mon[79930]: pgmap v15106: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:52.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:51:52 smithi083 ceph-mon[77242]: pgmap v15106: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:53.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:51:52 smithi153 ceph-mon[70044]: pgmap v15106: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:51:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:51:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:51:54.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:51:54 smithi083 ceph-mon[77242]: pgmap v15107: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:54.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:51:54 smithi083 ceph-mon[79930]: pgmap v15107: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:55.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:51:54 smithi153 ceph-mon[70044]: pgmap v15107: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:55.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:51:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:51:55.402Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:51:56.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:51:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:51:56.127Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:51:56.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:51:56 smithi083 ceph-mon[79930]: pgmap v15108: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:56.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:51:56 smithi083 ceph-mon[77242]: pgmap v15108: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:57.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:51:56 smithi153 ceph-mon[70044]: pgmap v15108: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:58.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:51:58 smithi083 ceph-mon[79930]: pgmap v15109: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:58.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:51:58 smithi083 ceph-mon[77242]: pgmap v15109: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:51:59.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:51:58 smithi153 ceph-mon[70044]: pgmap v15109: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:00.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:52:00 smithi083 ceph-mon[79930]: pgmap v15110: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:00.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:52:00 smithi083 ceph-mon[77242]: pgmap v15110: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:01.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:52:00 smithi153 ceph-mon[70044]: pgmap v15110: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:52:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:52:01] "GET /metrics HTTP/1.1" 200 225247 "" "Prometheus/2.33.4" 2022-09-29T13:52:02.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:52:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:52:02.042Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:52:02.668 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:52:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:52:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:52:02.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:52:02 smithi083 ceph-mon[79930]: pgmap v15111: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:02.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:52:02 smithi083 ceph-mon[77242]: pgmap v15111: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:03.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:52:02 smithi153 ceph-mon[70044]: pgmap v15111: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:52:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:52:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:52:04.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:52:04 smithi083 ceph-mon[79930]: pgmap v15112: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:04.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:52:04 smithi083 ceph-mon[77242]: pgmap v15112: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:05.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:52:04 smithi153 ceph-mon[70044]: pgmap v15112: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:05.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:52:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:52:05.403Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:52:06.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:52:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:52:06.128Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:52:06.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:52:06 smithi083 ceph-mon[79930]: pgmap v15113: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:06.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:52:06 smithi083 ceph-mon[77242]: pgmap v15113: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:07.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:52:06 smithi153 ceph-mon[70044]: pgmap v15113: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:08.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:52:08 smithi083 ceph-mon[79930]: pgmap v15114: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:08.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:52:08 smithi083 ceph-mon[77242]: pgmap v15114: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:09.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:52:08 smithi153 ceph-mon[70044]: pgmap v15114: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:10.457 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:52:10.748 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:52:10.748 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 5m ago 8h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:52:10.748 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 3m ago 8h 63.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:52:10.749 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 5m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:52:10.749 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 3m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:52:10.749 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 3m ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:52:10.749 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 5m ago 8h 927M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:52:10.750 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 5m ago 8h 1842M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:52:10.750 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 3m ago 8h 799M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:52:10.750 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 5m ago 8h 1097M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:52:10.750 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 5m ago 8h 24.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:52:10.751 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 3m ago 8h 26.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:52:10.751 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 5m ago 8h 1334M 2291M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:52:10.751 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 5m ago 8h 1320M 2291M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:52:10.751 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 5m ago 8h 1028M 2291M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:52:10.751 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 5m ago 8h 1294M 2291M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:52:10.752 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 3m ago 8h 1135M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:52:10.752 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 3m ago 8h 1424M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:52:10.752 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 3m ago 8h 1272M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:52:10.752 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 3m ago 8h 1511M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:52:10.753 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 3m ago 8h 185M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:52:10.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:52:10 smithi083 ceph-mon[77242]: pgmap v15115: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:10.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:52:10 smithi083 ceph-mon[79930]: pgmap v15115: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:11.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:52:10 smithi153 ceph-mon[70044]: pgmap v15115: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:11.106 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:52:11.106 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:52:11.107 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:52:11.107 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:52:11.107 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:52:11.107 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:52:11.108 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:52:11.108 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:52:11.108 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:52:11.108 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:52:11.108 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:52:11.109 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:52:11.109 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:52:11.109 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:52:11.109 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:52:11.110 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:52:11.110 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:52:11.110 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:52:11.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:52:11 smithi083 ceph-mon[77242]: from='client.51708 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:52:11.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:52:11 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/439182944' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:52:11.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:52:11 smithi083 ceph-mon[79930]: from='client.51708 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:52:11.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:52:11 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/439182944' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:52:11.964 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:52:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:52:11] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T13:52:12.063 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:52:11 smithi153 ceph-mon[70044]: from='client.51708 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:52:12.064 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:52:11 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/439182944' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:52:12.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:52:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:52:12.043Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:52:12.687 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:52:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:52:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:52:12.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:52:12 smithi083 ceph-mon[79930]: from='client.61645 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:52:12.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:52:12 smithi083 ceph-mon[79930]: pgmap v15116: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:12.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:52:12 smithi083 ceph-mon[77242]: from='client.61645 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:52:12.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:52:12 smithi083 ceph-mon[77242]: pgmap v15116: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:13.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:52:12 smithi153 ceph-mon[70044]: from='client.61645 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:52:13.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:52:12 smithi153 ceph-mon[70044]: pgmap v15116: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:52:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:52:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:52:13.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:52:13 smithi083 ceph-mon[77242]: pgmap v15117: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:13.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:52:13 smithi083 ceph-mon[79930]: pgmap v15117: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:14.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:52:13 smithi153 ceph-mon[70044]: pgmap v15117: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:15.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:52:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:52:15.403Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:52:16.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:52:15 smithi153 ceph-mon[70044]: pgmap v15118: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:16.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:52:15 smithi083 ceph-mon[79930]: pgmap v15118: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:16.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:52:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:52:16.129Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:52:16.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:52:15 smithi083 ceph-mon[77242]: pgmap v15118: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:18.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:52:18 smithi083 ceph-mon[79930]: pgmap v15119: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:18.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:52:18 smithi083 ceph-mon[77242]: pgmap v15119: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:18.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:52:18 smithi153 ceph-mon[70044]: pgmap v15119: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:20.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:52:20 smithi083 ceph-mon[79930]: pgmap v15120: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:20.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:52:20 smithi083 ceph-mon[77242]: pgmap v15120: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:20.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:52:20 smithi153 ceph-mon[70044]: pgmap v15120: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:21.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:52:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:52:21] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T13:52:22.355 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:52:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:52:22.043Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:52:22.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:52:22 smithi083 ceph-mon[77242]: pgmap v15121: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:22.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:52:22 smithi083 ceph-mon[79930]: pgmap v15121: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:22.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:52:22 smithi153 ceph-mon[70044]: pgmap v15121: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:22.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:52:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:52:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:52:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:52:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:52:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:52:24.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:52:24 smithi083 ceph-mon[79930]: pgmap v15122: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:24.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:52:24 smithi083 ceph-mon[77242]: pgmap v15122: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:24.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:52:24 smithi153 ceph-mon[70044]: pgmap v15122: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:25.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:52:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:52:25.404Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:52:26.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:52:26 smithi083 ceph-mon[79930]: pgmap v15123: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:26.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:52:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:52:26.130Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:52:26.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:52:26 smithi083 ceph-mon[77242]: pgmap v15123: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:26.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:52:26 smithi153 ceph-mon[70044]: pgmap v15123: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:28.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:52:28 smithi083 ceph-mon[77242]: pgmap v15124: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:28.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:52:28 smithi083 ceph-mon[79930]: pgmap v15124: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:28.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:52:28 smithi153 ceph-mon[70044]: pgmap v15124: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:30.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:52:30 smithi083 ceph-mon[79930]: pgmap v15125: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:30.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:52:30 smithi083 ceph-mon[77242]: pgmap v15125: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:30.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:52:30 smithi153 ceph-mon[70044]: pgmap v15125: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:52:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:52:31] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T13:52:32.369 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:52:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:52:32.044Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:52:32.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:52:32 smithi083 ceph-mon[79930]: pgmap v15126: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:32.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:52:32 smithi083 ceph-mon[77242]: pgmap v15126: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:32.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:52:32 smithi153 ceph-mon[70044]: pgmap v15126: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:32.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:52:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:52:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:52:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:52:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:52:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:52:34.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:52:34 smithi083 ceph-mon[77242]: pgmap v15127: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:34.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:52:34 smithi083 ceph-mon[79930]: pgmap v15127: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:34.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:52:34 smithi153 ceph-mon[70044]: pgmap v15127: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:35.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:52:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:52:35.405Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:52:36.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:52:36 smithi083 ceph-mon[77242]: pgmap v15128: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:36.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:52:36 smithi083 ceph-mon[79930]: pgmap v15128: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:36.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:52:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:52:36.131Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:52:36.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:52:36 smithi153 ceph-mon[70044]: pgmap v15128: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:38.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:52:38 smithi083 ceph-mon[77242]: pgmap v15129: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:38.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:52:38 smithi083 ceph-mon[79930]: pgmap v15129: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:38.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:52:38 smithi153 ceph-mon[70044]: pgmap v15129: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:52:40 smithi083 ceph-mon[77242]: pgmap v15130: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:52:40 smithi083 ceph-mon[79930]: pgmap v15130: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:52:40 smithi153 ceph-mon[70044]: pgmap v15130: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:41.438 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:52:41.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:52:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:52:41] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T13:52:41.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:52:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:52:41.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:52:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:52:41.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:52:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:52:41.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:52:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:52:41.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:52:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:52:41.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:52:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:52:41.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:52:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:52:41.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:52:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:52:41.726 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:52:41.726 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 6m ago 8h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:52:41.726 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 3m ago 8h 63.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:52:41.727 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 6m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:52:41.727 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 3m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:52:41.727 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 3m ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:52:41.727 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 6m ago 8h 927M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:52:41.727 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 6m ago 8h 1842M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:52:41.728 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 3m ago 8h 799M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:52:41.728 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 6m ago 8h 1097M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:52:41.728 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 6m ago 8h 24.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:52:41.729 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 3m ago 8h 26.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:52:41.729 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 6m ago 8h 1334M 2291M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:52:41.729 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 6m ago 8h 1320M 2291M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:52:41.729 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 6m ago 8h 1028M 2291M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:52:41.730 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 6m ago 8h 1294M 2291M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:52:41.730 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 3m ago 8h 1135M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:52:41.730 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 3m ago 8h 1424M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:52:41.730 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 3m ago 8h 1272M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:52:41.731 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 3m ago 8h 1511M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:52:41.731 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 3m ago 8h 185M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:52:41.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:52:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:52:41.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:52:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:52:41.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:52:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:52:41.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:52:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:52:42.082 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:52:42.083 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:52:42.083 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:52:42.083 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:52:42.083 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:52:42.084 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:52:42.084 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:52:42.084 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:52:42.084 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:52:42.085 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:52:42.085 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:52:42.085 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:52:42.085 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:52:42.085 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:52:42.086 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:52:42.086 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:52:42.086 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:52:42.086 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:52:42.388 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:52:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:52:42.046Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:52:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:52:42 smithi083 ceph-mon[79930]: pgmap v15131: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:42.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:52:42 smithi083 ceph-mon[79930]: from='client.51726 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:52:42.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:52:42 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1727049020' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:52:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:52:42 smithi083 ceph-mon[77242]: pgmap v15131: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:42.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:52:42 smithi083 ceph-mon[77242]: from='client.51726 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:52:42.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:52:42 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1727049020' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:52:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:52:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:52:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:52:42.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:52:42 smithi153 ceph-mon[70044]: pgmap v15131: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:42.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:52:42 smithi153 ceph-mon[70044]: from='client.51726 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:52:42.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:52:42 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1727049020' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:52:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:52:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:52:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:52:43.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:52:43 smithi083 ceph-mon[79930]: from='client.51732 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:52:43.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:52:43 smithi083 ceph-mon[77242]: from='client.51732 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:52:43.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:52:43 smithi153 ceph-mon[70044]: from='client.51732 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:52:44.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:52:44 smithi083 ceph-mon[79930]: pgmap v15132: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:44.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:52:44 smithi083 ceph-mon[77242]: pgmap v15132: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:44.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:52:44 smithi153 ceph-mon[70044]: pgmap v15132: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:45.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:52:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:52:45.406Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:52:46.398 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:52:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:52:46.131Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:52:46.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:52:46 smithi083 ceph-mon[79930]: pgmap v15133: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:46.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:52:46 smithi083 ceph-mon[77242]: pgmap v15133: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:46.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:52:46 smithi153 ceph-mon[70044]: pgmap v15133: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:48.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:52:48 smithi083 ceph-mon[79930]: pgmap v15134: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:48.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:52:48 smithi083 ceph-mon[77242]: pgmap v15134: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:48.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:52:48 smithi153 ceph-mon[70044]: pgmap v15134: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:50.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:52:50 smithi083 ceph-mon[79930]: pgmap v15135: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:50.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:52:50 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:52:50.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:52:50 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:52:50.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:52:50 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:52:50.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:52:50 smithi083 ceph-mon[77242]: pgmap v15135: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:50.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:52:50 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:52:50.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:52:50 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:52:50.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:52:50 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:52:50.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:52:50 smithi153 ceph-mon[70044]: pgmap v15135: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:50.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:52:50 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:52:50.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:52:50 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:52:50.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:52:50 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:52:51.861 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:52:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:52:51] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T13:52:52.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:52:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:52:52.046Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:52:52.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:52:51 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:52:52.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:52:51 smithi083 ceph-mon[77242]: pgmap v15136: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:52.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:52:51 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:52:52.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:52:51 smithi083 ceph-mon[79930]: pgmap v15136: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:52.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:52:51 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:52:52.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:52:51 smithi153 ceph-mon[70044]: pgmap v15136: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:52:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:52:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:52:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:52:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:52:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:52:54.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:52:53 smithi153 ceph-mon[70044]: pgmap v15137: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:54.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:52:53 smithi083 ceph-mon[79930]: pgmap v15137: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:54.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:52:53 smithi083 ceph-mon[77242]: pgmap v15137: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:55.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:52:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:52:55.406Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:52:56.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:52:55 smithi153 ceph-mon[70044]: pgmap v15138: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:56.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:52:55 smithi083 ceph-mon[79930]: pgmap v15138: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:56.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:52:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:52:56.132Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:52:56.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:52:55 smithi083 ceph-mon[77242]: pgmap v15138: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:58.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:52:58 smithi083 ceph-mon[79930]: pgmap v15139: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:58.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:52:58 smithi083 ceph-mon[77242]: pgmap v15139: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:52:58.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:52:58 smithi153 ceph-mon[70044]: pgmap v15139: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:00.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:53:00 smithi083 ceph-mon[77242]: pgmap v15140: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:00.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:53:00 smithi083 ceph-mon[79930]: pgmap v15140: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:00.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:53:00 smithi153 ceph-mon[70044]: pgmap v15140: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:53:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:53:01] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T13:53:02.365 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:53:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:53:02.047Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:53:02.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:53:02 smithi083 ceph-mon[77242]: pgmap v15141: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:02.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:53:02 smithi083 ceph-mon[79930]: pgmap v15141: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:02.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:53:02 smithi153 ceph-mon[70044]: pgmap v15141: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:02.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:53:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:53:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:53:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:53:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:53:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:53:04.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:53:04 smithi083 ceph-mon[79930]: pgmap v15142: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:04.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:53:04 smithi083 ceph-mon[77242]: pgmap v15142: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:04.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:53:04 smithi153 ceph-mon[70044]: pgmap v15142: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:05.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:53:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:53:05.407Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:53:06.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:53:06 smithi083 ceph-mon[79930]: pgmap v15143: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:06.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:53:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:53:06.133Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:53:06.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:53:06 smithi083 ceph-mon[77242]: pgmap v15143: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:06.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:53:06 smithi153 ceph-mon[70044]: pgmap v15143: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:08.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:53:08 smithi083 ceph-mon[79930]: pgmap v15144: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:08.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:53:08 smithi083 ceph-mon[77242]: pgmap v15144: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:08.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:53:08 smithi153 ceph-mon[70044]: pgmap v15144: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:10.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:53:10 smithi083 ceph-mon[79930]: pgmap v15145: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:10.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:53:10 smithi083 ceph-mon[77242]: pgmap v15145: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:10.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:53:10 smithi153 ceph-mon[70044]: pgmap v15145: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:53:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:53:11] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T13:53:12.381 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:53:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:53:12.048Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:53:12.409 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:53:12.699 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:53:12.699 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 6m ago 8h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:53:12.700 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 4m ago 8h 63.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:53:12.700 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 6m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:53:12.700 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 4m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:53:12.700 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 4m ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:53:12.700 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 6m ago 8h 927M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:53:12.701 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 6m ago 8h 1842M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:53:12.701 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 4m ago 8h 799M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:53:12.701 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 6m ago 8h 1097M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:53:12.701 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 6m ago 8h 24.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:53:12.702 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 4m ago 8h 26.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:53:12.702 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 6m ago 8h 1334M 2291M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:53:12.702 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 6m ago 8h 1320M 2291M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:53:12.702 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 6m ago 8h 1028M 2291M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:53:12.703 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 6m ago 8h 1294M 2291M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:53:12.703 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 4m ago 8h 1135M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:53:12.703 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 4m ago 8h 1424M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:53:12.703 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 4m ago 8h 1272M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:53:12.704 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 4m ago 8h 1511M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:53:12.704 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 4m ago 8h 185M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:53:12.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:53:12 smithi083 ceph-mon[79930]: pgmap v15146: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:12.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:53:12 smithi083 ceph-mon[77242]: pgmap v15146: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:12.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:53:12 smithi153 ceph-mon[70044]: pgmap v15146: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:12.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:53:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:53:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:53:13.057 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:53:13.057 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:53:13.058 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:53:13.058 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:53:13.058 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:53:13.058 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:53:13.059 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:53:13.059 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:53:13.059 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:53:13.059 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:53:13.060 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:53:13.060 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:53:13.060 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:53:13.060 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:53:13.060 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:53:13.061 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:53:13.061 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:53:13.061 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:53:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:53:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:53:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:53:13.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:53:13 smithi083 ceph-mon[79930]: from='client.61675 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:53:13.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:53:13 smithi083 ceph-mon[79930]: from='client.61681 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:53:13.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:53:13 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2072981711' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:53:13.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:53:13 smithi083 ceph-mon[77242]: from='client.61675 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:53:13.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:53:13 smithi083 ceph-mon[77242]: from='client.61681 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:53:13.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:53:13 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2072981711' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:53:13.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:53:13 smithi153 ceph-mon[70044]: from='client.61675 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:53:13.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:53:13 smithi153 ceph-mon[70044]: from='client.61681 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:53:13.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:53:13 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2072981711' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:53:14.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:53:14 smithi083 ceph-mon[79930]: pgmap v15147: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:14.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:53:14 smithi083 ceph-mon[77242]: pgmap v15147: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:14.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:53:14 smithi153 ceph-mon[70044]: pgmap v15147: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:15.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:53:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:53:15.407Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:53:16.400 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:53:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:53:16.135Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:53:16.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:53:16 smithi083 ceph-mon[79930]: pgmap v15148: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:16.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:53:16 smithi083 ceph-mon[77242]: pgmap v15148: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:16.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:53:16 smithi153 ceph-mon[70044]: pgmap v15148: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:18.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:53:18 smithi083 ceph-mon[79930]: pgmap v15149: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:18.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:53:18 smithi083 ceph-mon[77242]: pgmap v15149: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:18.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:53:18 smithi153 ceph-mon[70044]: pgmap v15149: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:20.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:53:20 smithi083 ceph-mon[79930]: pgmap v15150: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:20.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:53:20 smithi083 ceph-mon[77242]: pgmap v15150: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:20.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:53:20 smithi153 ceph-mon[70044]: pgmap v15150: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:53:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:53:21] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T13:53:22.411 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:53:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:53:22.048Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:53:22.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:53:22 smithi083 ceph-mon[79930]: pgmap v15151: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:22.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:53:22 smithi083 ceph-mon[77242]: pgmap v15151: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:22.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:53:22 smithi153 ceph-mon[70044]: pgmap v15151: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:22.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:53:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:53:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:53:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:53:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:53:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:53:24.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:53:24 smithi083 ceph-mon[79930]: pgmap v15152: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:24.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:53:24 smithi083 ceph-mon[77242]: pgmap v15152: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:24.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:53:24 smithi153 ceph-mon[70044]: pgmap v15152: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:25.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:53:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:53:25.407Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:53:26.416 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:53:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:53:26.135Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:53:26.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:53:26 smithi083 ceph-mon[79930]: pgmap v15153: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:26.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:53:26 smithi083 ceph-mon[77242]: pgmap v15153: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:26.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:53:26 smithi153 ceph-mon[70044]: pgmap v15153: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:28.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:53:28 smithi083 ceph-mon[79930]: pgmap v15154: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:28.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:53:28 smithi083 ceph-mon[77242]: pgmap v15154: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:28.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:53:28 smithi153 ceph-mon[70044]: pgmap v15154: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:30.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:53:30 smithi083 ceph-mon[79930]: pgmap v15155: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:30.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:53:30 smithi083 ceph-mon[77242]: pgmap v15155: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:30.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:53:30 smithi153 ceph-mon[70044]: pgmap v15155: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:53:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:53:31] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T13:53:32.425 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:53:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:53:32.049Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:53:32.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:53:32 smithi083 ceph-mon[79930]: pgmap v15156: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:32.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:53:32 smithi083 ceph-mon[77242]: pgmap v15156: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:32.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:53:32 smithi153 ceph-mon[70044]: pgmap v15156: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:32.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:53:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:53:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:53:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:53:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:53:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:53:34.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:53:34 smithi083 ceph-mon[77242]: pgmap v15157: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:34.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:53:34 smithi083 ceph-mon[79930]: pgmap v15157: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:34.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:53:34 smithi153 ceph-mon[70044]: pgmap v15157: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:35.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:53:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:53:35.408Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:53:36.431 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:53:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:53:36.137Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:53:36.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:53:36 smithi083 ceph-mon[79930]: pgmap v15158: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:36.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:53:36 smithi083 ceph-mon[77242]: pgmap v15158: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:36.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:53:36 smithi153 ceph-mon[70044]: pgmap v15158: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:38.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:53:38 smithi083 ceph-mon[79930]: pgmap v15159: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:38.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:53:38 smithi083 ceph-mon[77242]: pgmap v15159: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:38.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:53:38 smithi153 ceph-mon[70044]: pgmap v15159: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:53:40 smithi083 ceph-mon[79930]: pgmap v15160: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:53:40 smithi083 ceph-mon[77242]: pgmap v15160: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:53:40 smithi153 ceph-mon[70044]: pgmap v15160: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:41.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:53:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:53:41] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T13:53:41.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:53:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:53:41.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:53:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:53:41.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:53:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:53:41.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:53:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:53:41.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:53:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:53:41.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:53:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:53:41.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:53:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:53:41.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:53:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:53:41.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:53:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:53:41.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:53:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:53:41.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:53:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:53:41.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:53:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:53:42.443 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:53:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:53:42.049Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:53:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:53:42 smithi083 ceph-mon[79930]: pgmap v15161: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:53:42 smithi083 ceph-mon[77242]: pgmap v15161: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:53:42 smithi153 ceph-mon[70044]: pgmap v15161: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:42.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:53:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:53:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:53:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:53:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:53:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:53:43.390 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:53:43.683 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:53:43.684 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 7m ago 8h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:53:43.684 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 4m ago 8h 63.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:53:43.684 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 7m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:53:43.684 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 4m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:53:43.685 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 4m ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:53:43.685 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 7m ago 8h 927M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:53:43.685 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 7m ago 8h 1842M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:53:43.685 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 4m ago 8h 799M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:53:43.686 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 7m ago 8h 1097M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:53:43.686 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 7m ago 8h 24.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:53:43.686 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 4m ago 8h 26.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:53:43.686 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 7m ago 8h 1334M 2291M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:53:43.687 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 7m ago 8h 1320M 2291M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:53:43.687 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 7m ago 8h 1028M 2291M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:53:43.687 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 7m ago 8h 1294M 2291M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:53:43.687 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 4m ago 8h 1135M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:53:43.688 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 4m ago 8h 1424M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:53:43.688 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 4m ago 8h 1272M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:53:43.688 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 4m ago 8h 1511M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:53:43.688 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 4m ago 8h 185M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:53:44.041 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:53:44.041 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:53:44.041 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:53:44.042 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:53:44.042 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:53:44.042 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:53:44.042 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:53:44.043 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:53:44.043 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:53:44.043 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:53:44.044 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:53:44.044 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:53:44.044 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:53:44.045 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:53:44.045 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:53:44.045 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:53:44.045 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:53:44.046 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:53:44.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:53:44 smithi083 ceph-mon[79930]: pgmap v15162: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:44.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:53:44 smithi083 ceph-mon[79930]: from='client.51762 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:53:44.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:53:44 smithi083 ceph-mon[79930]: from='client.61699 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:53:44.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:53:44 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2777344548' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:53:44.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:53:44 smithi083 ceph-mon[77242]: pgmap v15162: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:44.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:53:44 smithi083 ceph-mon[77242]: from='client.51762 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:53:44.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:53:44 smithi083 ceph-mon[77242]: from='client.61699 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:53:44.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:53:44 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2777344548' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:53:44.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:53:44 smithi153 ceph-mon[70044]: pgmap v15162: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:44.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:53:44 smithi153 ceph-mon[70044]: from='client.51762 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:53:44.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:53:44 smithi153 ceph-mon[70044]: from='client.61699 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:53:44.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:53:44 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2777344548' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:53:45.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:53:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:53:45.409Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:53:46.448 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:53:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:53:46.137Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:53:46.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:53:46 smithi083 ceph-mon[79930]: pgmap v15163: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:46.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:53:46 smithi083 ceph-mon[77242]: pgmap v15163: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:46.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:53:46 smithi153 ceph-mon[70044]: pgmap v15163: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:48.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:53:48 smithi083 ceph-mon[77242]: pgmap v15164: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:48.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:53:48 smithi083 ceph-mon[79930]: pgmap v15164: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:48.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:53:48 smithi153 ceph-mon[70044]: pgmap v15164: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:50.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:53:50 smithi083 ceph-mon[79930]: pgmap v15165: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:50.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:53:50 smithi083 ceph-mon[77242]: pgmap v15165: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:50.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:53:50 smithi153 ceph-mon[70044]: pgmap v15165: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:51.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:53:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:53:51] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T13:53:51.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:53:51 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:53:51.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:53:51 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:53:51.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:53:51 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:53:51.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:53:51 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:53:51.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:53:51 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:53:51.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:53:51 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:53:51.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:53:51 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:53:51.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:53:51 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:53:51.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:53:51 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:53:52.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:53:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:53:52.049Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:53:52.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:53:52 smithi153 ceph-mon[70044]: pgmap v15166: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:52.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:53:52 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:53:52.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:53:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:53:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:53:52.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:53:52 smithi083 ceph-mon[77242]: pgmap v15166: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:52.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:53:52 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:53:52.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:53:52 smithi083 ceph-mon[79930]: pgmap v15166: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:52.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:53:52 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:53:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:53:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:53:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:53:54.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:53:54 smithi153 ceph-mon[70044]: pgmap v15167: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:54.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:53:54 smithi083 ceph-mon[79930]: pgmap v15167: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:54.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:53:54 smithi083 ceph-mon[77242]: pgmap v15167: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:55.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:53:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:53:55.409Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:53:56.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:53:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:53:56.138Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:53:56.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:53:56 smithi153 ceph-mon[70044]: pgmap v15168: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:56.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:53:56 smithi083 ceph-mon[79930]: pgmap v15168: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:56.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:53:56 smithi083 ceph-mon[77242]: pgmap v15168: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:58.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:53:58 smithi153 ceph-mon[70044]: pgmap v15169: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:58.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:53:58 smithi083 ceph-mon[77242]: pgmap v15169: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:53:58.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:53:58 smithi083 ceph-mon[79930]: pgmap v15169: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:00.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:54:00 smithi153 ceph-mon[70044]: pgmap v15170: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:00.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:54:00 smithi083 ceph-mon[79930]: pgmap v15170: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:00.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:54:00 smithi083 ceph-mon[77242]: pgmap v15170: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:54:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:54:01] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T13:54:02.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:54:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:54:02.051Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:54:02.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:54:02 smithi153 ceph-mon[70044]: pgmap v15171: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:02.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:54:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:54:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:54:02.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:54:02 smithi083 ceph-mon[79930]: pgmap v15171: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:02.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:54:02 smithi083 ceph-mon[77242]: pgmap v15171: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:54:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:54:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:54:04.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:54:04 smithi153 ceph-mon[70044]: pgmap v15172: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:04.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:54:04 smithi083 ceph-mon[79930]: pgmap v15172: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:04.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:54:04 smithi083 ceph-mon[77242]: pgmap v15172: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:05.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:54:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:54:05.410Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:54:06.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:54:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:54:06.139Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:54:06.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:54:06 smithi153 ceph-mon[70044]: pgmap v15173: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:06.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:54:06 smithi083 ceph-mon[79930]: pgmap v15173: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:06.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:54:06 smithi083 ceph-mon[77242]: pgmap v15173: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:08.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:54:08 smithi153 ceph-mon[70044]: pgmap v15174: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:08.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:54:08 smithi083 ceph-mon[77242]: pgmap v15174: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:08.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:54:08 smithi083 ceph-mon[79930]: pgmap v15174: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:10.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:54:10 smithi153 ceph-mon[70044]: pgmap v15175: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:10.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:54:10 smithi083 ceph-mon[77242]: pgmap v15175: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:10.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:54:10 smithi083 ceph-mon[79930]: pgmap v15175: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:54:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:54:11] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T13:54:12.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:54:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:54:12.052Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:54:12.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:54:12 smithi153 ceph-mon[70044]: pgmap v15176: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:12.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:54:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:54:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:54:12.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:54:12 smithi083 ceph-mon[77242]: pgmap v15176: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:12.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:54:12 smithi083 ceph-mon[79930]: pgmap v15176: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:54:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:54:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:54:14.367 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:54:14.655 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:54:14.656 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 7m ago 8h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:54:14.656 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 5m ago 8h 63.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:54:14.656 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 7m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:54:14.656 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 5m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:54:14.657 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 5m ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:54:14.657 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 7m ago 8h 927M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:54:14.657 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 7m ago 8h 1842M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:54:14.657 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 5m ago 8h 799M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:54:14.657 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 7m ago 8h 1097M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:54:14.658 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 7m ago 8h 24.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:54:14.658 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 5m ago 8h 26.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:54:14.658 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 7m ago 8h 1334M 2291M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:54:14.658 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 7m ago 8h 1320M 2291M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:54:14.659 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 7m ago 8h 1028M 2291M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:54:14.659 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 7m ago 8h 1294M 2291M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:54:14.659 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 5m ago 8h 1135M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:54:14.659 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 5m ago 8h 1424M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:54:14.660 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 5m ago 8h 1272M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:54:14.660 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 5m ago 8h 1511M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:54:14.660 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 5m ago 8h 185M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:54:14.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:54:14 smithi153 ceph-mon[70044]: pgmap v15177: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:14.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:54:14 smithi083 ceph-mon[77242]: pgmap v15177: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:14.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:54:14 smithi083 ceph-mon[79930]: pgmap v15177: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:15.013 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:54:15.013 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:54:15.013 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:54:15.014 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:54:15.014 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:54:15.014 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:54:15.014 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:54:15.015 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:54:15.015 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:54:15.015 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:54:15.015 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:54:15.016 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:54:15.016 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:54:15.016 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:54:15.016 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:54:15.017 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:54:15.017 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:54:15.017 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:54:15.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:54:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:54:15.410Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:54:15.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:54:15 smithi083 ceph-mon[79930]: from='client.51780 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:54:15.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:54:15 smithi083 ceph-mon[79930]: from='client.51786 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:54:15.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:54:15 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3173658828' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:54:15.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:54:15 smithi083 ceph-mon[77242]: from='client.51780 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:54:15.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:54:15 smithi083 ceph-mon[77242]: from='client.51786 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:54:15.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:54:15 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3173658828' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:54:16.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:54:15 smithi153 ceph-mon[70044]: from='client.51780 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:54:16.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:54:15 smithi153 ceph-mon[70044]: from='client.51786 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:54:16.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:54:15 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3173658828' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:54:16.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:54:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:54:16.139Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:54:16.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:54:16 smithi083 ceph-mon[79930]: pgmap v15178: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:16.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:54:16 smithi083 ceph-mon[77242]: pgmap v15178: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:17.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:54:16 smithi153 ceph-mon[70044]: pgmap v15178: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:18.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:54:18 smithi083 ceph-mon[79930]: pgmap v15179: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:18.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:54:18 smithi083 ceph-mon[77242]: pgmap v15179: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:19.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:54:18 smithi153 ceph-mon[70044]: pgmap v15179: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:20.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:54:20 smithi083 ceph-mon[77242]: pgmap v15180: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:20.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:54:20 smithi083 ceph-mon[79930]: pgmap v15180: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:21.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:54:20 smithi153 ceph-mon[70044]: pgmap v15180: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:54:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:54:21] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T13:54:22.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:54:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:54:22.053Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:54:22.579 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:54:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:54:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:54:22.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:54:22 smithi083 ceph-mon[79930]: pgmap v15181: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:22.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:54:22 smithi083 ceph-mon[77242]: pgmap v15181: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:23.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:54:22 smithi153 ceph-mon[70044]: pgmap v15181: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:54:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:54:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:54:24.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:54:24 smithi083 ceph-mon[79930]: pgmap v15182: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:24.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:54:24 smithi083 ceph-mon[77242]: pgmap v15182: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:25.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:54:24 smithi153 ceph-mon[70044]: pgmap v15182: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:25.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:54:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:54:25.411Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:54:26.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:54:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:54:26.139Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:54:26.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:54:26 smithi083 ceph-mon[79930]: pgmap v15183: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:26.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:54:26 smithi083 ceph-mon[77242]: pgmap v15183: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:27.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:54:26 smithi153 ceph-mon[70044]: pgmap v15183: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:28.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:54:28 smithi083 ceph-mon[77242]: pgmap v15184: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:28.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:54:28 smithi083 ceph-mon[79930]: pgmap v15184: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:29.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:54:28 smithi153 ceph-mon[70044]: pgmap v15184: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:30.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:54:30 smithi083 ceph-mon[79930]: pgmap v15185: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:30.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:54:30 smithi083 ceph-mon[77242]: pgmap v15185: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:31.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:54:30 smithi153 ceph-mon[70044]: pgmap v15185: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:54:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:54:31] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T13:54:32.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:54:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:54:32.053Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:54:32.595 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:54:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:54:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:54:32.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:54:32 smithi083 ceph-mon[79930]: pgmap v15186: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:32.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:54:32 smithi083 ceph-mon[77242]: pgmap v15186: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:33.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:54:32 smithi153 ceph-mon[70044]: pgmap v15186: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:54:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:54:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:54:34.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:54:34 smithi083 ceph-mon[79930]: pgmap v15187: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:34.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:54:34 smithi083 ceph-mon[77242]: pgmap v15187: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:35.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:54:34 smithi153 ceph-mon[70044]: pgmap v15187: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:35.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:54:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:54:35.412Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:54:36.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:54:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:54:36.140Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:54:36.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:54:36 smithi083 ceph-mon[77242]: pgmap v15188: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:36.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:54:36 smithi083 ceph-mon[79930]: pgmap v15188: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:37.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:54:36 smithi153 ceph-mon[70044]: pgmap v15188: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:38.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:54:38 smithi083 ceph-mon[79930]: pgmap v15189: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:38.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:54:38 smithi083 ceph-mon[77242]: pgmap v15189: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:39.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:54:38 smithi153 ceph-mon[70044]: pgmap v15189: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:54:40 smithi083 ceph-mon[79930]: pgmap v15190: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:54:40 smithi083 ceph-mon[77242]: pgmap v15190: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:54:40 smithi153 ceph-mon[70044]: pgmap v15190: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:54:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:54:41] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T13:54:41.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:54:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:54:41.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:54:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:54:41.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:54:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:54:41.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:54:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:54:41.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:54:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:54:41.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:54:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:54:41.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:54:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:54:41.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:54:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:54:42.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:54:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:54:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:54:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:54:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:54:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:54:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:54:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:54:42.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:54:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:54:42.054Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:54:42.614 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:54:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:54:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:54:42.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:54:42 smithi083 ceph-mon[79930]: pgmap v15191: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:42.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:54:42 smithi083 ceph-mon[77242]: pgmap v15191: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:43.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:54:42 smithi153 ceph-mon[70044]: pgmap v15191: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:54:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:54:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:54:44.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:54:44 smithi083 ceph-mon[79930]: pgmap v15192: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:44.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:54:44 smithi083 ceph-mon[77242]: pgmap v15192: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:45.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:54:44 smithi153 ceph-mon[70044]: pgmap v15192: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:45.338 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:54:45.631 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:54:45.631 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 8m ago 8h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:54:45.631 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 6m ago 8h 63.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:54:45.631 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 8m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:54:45.632 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 6m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:54:45.632 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 6m ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:54:45.632 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 8m ago 8h 927M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:54:45.632 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 8m ago 8h 1842M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:54:45.633 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 6m ago 8h 799M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:54:45.633 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 8m ago 8h 1097M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:54:45.633 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 8m ago 8h 24.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:54:45.633 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 6m ago 8h 26.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:54:45.634 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 8m ago 8h 1334M 2291M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:54:45.634 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 8m ago 8h 1320M 2291M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:54:45.634 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 8m ago 8h 1028M 2291M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:54:45.634 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 8m ago 8h 1294M 2291M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:54:45.635 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 6m ago 8h 1135M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:54:45.635 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 6m ago 8h 1424M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:54:45.635 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 6m ago 8h 1272M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:54:45.635 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 6m ago 8h 1511M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:54:45.635 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 6m ago 8h 185M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:54:45.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:54:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:54:45.411Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:54:45.988 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:54:45.989 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:54:45.989 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:54:45.989 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:54:45.990 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:54:45.990 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:54:45.990 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:54:45.990 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:54:45.991 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:54:45.991 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:54:45.991 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:54:45.991 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:54:45.992 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:54:45.992 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:54:45.992 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:54:45.992 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:54:45.992 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:54:45.993 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:54:46.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:54:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:54:46.139Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:54:46.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:54:46 smithi083 ceph-mon[77242]: pgmap v15193: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:46.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:54:46 smithi083 ceph-mon[77242]: from='client.51798 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:54:46.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:54:46 smithi083 ceph-mon[77242]: from='client.61735 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:54:46.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:54:46 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/113716511' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:54:46.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:54:46 smithi083 ceph-mon[79930]: pgmap v15193: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:46.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:54:46 smithi083 ceph-mon[79930]: from='client.51798 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:54:46.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:54:46 smithi083 ceph-mon[79930]: from='client.61735 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:54:46.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:54:46 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/113716511' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:54:47.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:54:46 smithi153 ceph-mon[70044]: pgmap v15193: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:47.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:54:46 smithi153 ceph-mon[70044]: from='client.51798 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:54:47.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:54:46 smithi153 ceph-mon[70044]: from='client.61735 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:54:47.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:54:46 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/113716511' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:54:48.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:54:48 smithi083 ceph-mon[79930]: pgmap v15194: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:48.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:54:48 smithi083 ceph-mon[77242]: pgmap v15194: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:49.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:54:48 smithi153 ceph-mon[70044]: pgmap v15194: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:50.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:54:50 smithi083 ceph-mon[79930]: pgmap v15195: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:50.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:54:50 smithi083 ceph-mon[77242]: pgmap v15195: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:51.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:54:50 smithi153 ceph-mon[70044]: pgmap v15195: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:54:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:54:51] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T13:54:52.384 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:54:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:54:52.055Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:54:52.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:54:52 smithi083 ceph-mon[79930]: pgmap v15196: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:52.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:54:52 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:54:52.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:54:52 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:54:52.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:54:52 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:54:52.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:54:52 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:54:52.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:54:52 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:54:52.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:54:52 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:54:52.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:54:52 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:54:52.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:54:52 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:54:52.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:54:52 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:54:52.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:54:52 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:54:52.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:54:52 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:54:52.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:54:52 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:54:52.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:54:52 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:54:52.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:54:52 smithi083 ceph-mon[77242]: pgmap v15196: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:52.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:54:52 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:54:52.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:54:52 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:54:52.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:54:52 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:54:52.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:54:52 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:54:52.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:54:52 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:54:52.718 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:54:52 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:54:52.718 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:54:52 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:54:52.718 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:54:52 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:54:52.718 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:54:52 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:54:52.719 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:54:52 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:54:52.719 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:54:52 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:54:52.719 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:54:52 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:54:52.719 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:54:52 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:54:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:54:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:54:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:54:52.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:54:52 smithi153 ceph-mon[70044]: pgmap v15196: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:52.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:54:52 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:54:52.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:54:52 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:54:52.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:54:52 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:54:52.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:54:52 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:54:52.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:54:52 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:54:52.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:54:52 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:54:52.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:54:52 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:54:52.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:54:52 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:54:52.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:54:52 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:54:52.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:54:52 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:54:52.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:54:52 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T13:54:52.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:54:52 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:54:52.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:54:52 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:54:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:54:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:54:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:54:53.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:54:53 smithi083 ceph-mon[77242]: Adjusting osd_memory_target on smithi083 to 2277M 2022-09-29T13:54:53.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:54:53 smithi083 ceph-mon[79930]: Adjusting osd_memory_target on smithi083 to 2277M 2022-09-29T13:54:53.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:54:53 smithi153 ceph-mon[70044]: Adjusting osd_memory_target on smithi083 to 2277M 2022-09-29T13:54:54.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:54:54 smithi083 ceph-mon[77242]: pgmap v15197: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:54.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:54:54 smithi083 ceph-mon[79930]: pgmap v15197: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:54.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:54:54 smithi153 ceph-mon[70044]: pgmap v15197: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:55.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:54:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:54:55.412Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:54:56.396 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:54:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:54:56.141Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:54:56.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:54:56 smithi083 ceph-mon[79930]: pgmap v15198: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:56.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:54:56 smithi083 ceph-mon[77242]: pgmap v15198: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:56.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:54:56 smithi153 ceph-mon[70044]: pgmap v15198: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:58.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:54:58 smithi083 ceph-mon[79930]: pgmap v15199: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:58.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:54:58 smithi083 ceph-mon[77242]: pgmap v15199: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:54:58.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:54:58 smithi153 ceph-mon[70044]: pgmap v15199: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:00.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:55:00 smithi083 ceph-mon[79930]: pgmap v15200: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:00.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:55:00 smithi083 ceph-mon[77242]: pgmap v15200: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:00.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:55:00 smithi153 ceph-mon[70044]: pgmap v15200: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:55:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:55:01] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T13:55:02.406 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:55:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:55:02.055Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:55:02.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:55:02 smithi083 ceph-mon[79930]: pgmap v15201: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:02.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:55:02 smithi083 ceph-mon[77242]: pgmap v15201: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:02.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:55:02 smithi153 ceph-mon[70044]: pgmap v15201: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:02.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:55:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:55:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:55:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:55:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:55:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:55:04.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:55:04 smithi083 ceph-mon[79930]: pgmap v15202: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:04.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:55:04 smithi083 ceph-mon[77242]: pgmap v15202: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:04.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:55:04 smithi153 ceph-mon[70044]: pgmap v15202: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:05.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:55:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:55:05.414Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:55:06.413 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:55:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:55:06.141Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:55:06.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:55:06 smithi083 ceph-mon[79930]: pgmap v15203: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:06.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:55:06 smithi083 ceph-mon[77242]: pgmap v15203: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:06.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:55:06 smithi153 ceph-mon[70044]: pgmap v15203: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:08.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:55:08 smithi083 ceph-mon[79930]: pgmap v15204: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:08.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:55:08 smithi083 ceph-mon[77242]: pgmap v15204: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:08.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:55:08 smithi153 ceph-mon[70044]: pgmap v15204: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:10.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:55:10 smithi083 ceph-mon[79930]: pgmap v15205: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:10.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:55:10 smithi083 ceph-mon[77242]: pgmap v15205: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:10.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:55:10 smithi153 ceph-mon[70044]: pgmap v15205: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:55:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:55:11] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T13:55:12.421 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:55:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:55:12.056Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:55:12.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:55:12 smithi083 ceph-mon[79930]: pgmap v15206: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:12.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:55:12 smithi083 ceph-mon[77242]: pgmap v15206: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:12.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:55:12 smithi153 ceph-mon[70044]: pgmap v15206: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:12.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:55:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:55:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:55:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:55:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:55:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:55:14.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:55:14 smithi083 ceph-mon[79930]: pgmap v15207: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:14.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:55:14 smithi083 ceph-mon[77242]: pgmap v15207: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:14.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:55:14 smithi153 ceph-mon[70044]: pgmap v15207: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:15.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:55:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:55:15.414Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:55:16.314 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:55:16.426 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:55:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:55:16.141Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:55:16.604 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:55:16.634 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 8m ago 8h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:55:16.634 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 6m ago 8h 63.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:55:16.635 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 8m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:55:16.635 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 6m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:55:16.635 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 6m ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:55:16.635 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 8m ago 8h 927M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:55:16.636 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 8m ago 8h 1842M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:55:16.636 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 6m ago 8h 799M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:55:16.636 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 8m ago 8h 1097M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:55:16.636 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 8m ago 8h 24.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:55:16.636 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 6m ago 8h 26.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:55:16.637 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 8m ago 8h 1334M 2277M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:55:16.637 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 8m ago 8h 1320M 2277M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:55:16.637 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 8m ago 8h 1028M 2277M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:55:16.637 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 8m ago 8h 1294M 2277M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:55:16.638 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 6m ago 8h 1135M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:55:16.638 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 6m ago 8h 1424M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:55:16.638 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 6m ago 8h 1272M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:55:16.638 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 6m ago 8h 1511M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:55:16.638 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 6m ago 8h 185M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:55:16.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:55:16 smithi083 ceph-mon[79930]: pgmap v15208: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:16.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:55:16 smithi083 ceph-mon[77242]: pgmap v15208: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:16.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:55:16 smithi153 ceph-mon[70044]: pgmap v15208: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:16.961 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:55:16.961 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:55:16.961 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:55:16.961 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:55:16.962 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:55:16.962 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:55:16.962 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:55:16.962 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:55:16.963 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:55:16.963 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:55:16.963 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:55:16.963 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:55:16.964 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:55:16.964 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:55:16.964 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:55:16.964 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:55:16.965 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:55:16.965 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:55:17.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:55:17 smithi083 ceph-mon[79930]: from='client.51816 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:55:17.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:55:17 smithi083 ceph-mon[79930]: from='client.61753 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:55:17.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:55:17 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1128284787' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:55:17.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:55:17 smithi083 ceph-mon[77242]: from='client.51816 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:55:17.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:55:17 smithi083 ceph-mon[77242]: from='client.61753 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:55:17.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:55:17 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1128284787' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:55:17.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:55:17 smithi153 ceph-mon[70044]: from='client.51816 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:55:17.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:55:17 smithi153 ceph-mon[70044]: from='client.61753 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:55:17.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:55:17 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1128284787' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:55:18.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:55:18 smithi083 ceph-mon[79930]: pgmap v15209: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:18.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:55:18 smithi083 ceph-mon[77242]: pgmap v15209: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:18.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:55:18 smithi153 ceph-mon[70044]: pgmap v15209: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:20.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:55:20 smithi083 ceph-mon[79930]: pgmap v15210: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:20.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:55:20 smithi083 ceph-mon[77242]: pgmap v15210: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:20.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:55:20 smithi153 ceph-mon[70044]: pgmap v15210: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:55:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:55:21] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T13:55:22.438 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:55:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:55:22.057Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:55:22.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:55:22 smithi083 ceph-mon[77242]: pgmap v15211: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:22.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:55:22 smithi083 ceph-mon[79930]: pgmap v15211: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:22.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:55:22 smithi153 ceph-mon[70044]: pgmap v15211: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:55:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:55:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:55:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:55:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:55:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:55:24.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:55:24 smithi083 ceph-mon[77242]: pgmap v15212: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:24.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:55:24 smithi083 ceph-mon[79930]: pgmap v15212: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:24.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:55:24 smithi153 ceph-mon[70044]: pgmap v15212: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:25.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:55:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:55:25.415Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:55:26.443 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:55:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:55:26.142Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:55:26.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:55:26 smithi083 ceph-mon[79930]: pgmap v15213: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:26.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:55:26 smithi083 ceph-mon[77242]: pgmap v15213: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:26.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:55:26 smithi153 ceph-mon[70044]: pgmap v15213: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:28.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:55:28 smithi083 ceph-mon[79930]: pgmap v15214: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:28.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:55:28 smithi083 ceph-mon[77242]: pgmap v15214: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:28.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:55:28 smithi153 ceph-mon[70044]: pgmap v15214: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:30.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:55:30 smithi083 ceph-mon[79930]: pgmap v15215: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:30.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:55:30 smithi083 ceph-mon[77242]: pgmap v15215: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:30.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:55:30 smithi153 ceph-mon[70044]: pgmap v15215: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:55:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:55:31] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T13:55:32.452 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:55:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:55:32.058Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:55:32.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:55:32 smithi083 ceph-mon[79930]: pgmap v15216: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:32.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:55:32 smithi083 ceph-mon[77242]: pgmap v15216: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:32.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:55:32 smithi153 ceph-mon[70044]: pgmap v15216: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:55:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:55:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:55:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:55:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:55:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:55:34.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:55:34 smithi083 ceph-mon[79930]: pgmap v15217: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:34.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:55:34 smithi083 ceph-mon[77242]: pgmap v15217: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:34.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:55:34 smithi153 ceph-mon[70044]: pgmap v15217: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:35.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:55:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:55:35.416Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:55:36.457 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:55:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:55:36.143Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:55:36.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:55:36 smithi083 ceph-mon[79930]: pgmap v15218: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:36.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:55:36 smithi083 ceph-mon[77242]: pgmap v15218: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:36.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:55:36 smithi153 ceph-mon[70044]: pgmap v15218: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:38.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:55:38 smithi083 ceph-mon[77242]: pgmap v15219: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:38.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:55:38 smithi083 ceph-mon[79930]: pgmap v15219: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:38.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:55:38 smithi153 ceph-mon[70044]: pgmap v15219: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:55:40 smithi153 ceph-mon[70044]: pgmap v15220: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:55:40 smithi083 ceph-mon[79930]: pgmap v15220: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:55:40 smithi083 ceph-mon[77242]: pgmap v15220: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:41.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:55:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:55:41.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:55:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:55:41.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:55:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:55:41.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:55:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:55:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:55:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:55:41] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T13:55:41.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:55:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:55:41.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:55:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:55:41.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:55:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:55:41.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:55:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:55:41.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:55:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:55:41.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:55:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:55:41.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:55:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:55:41.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:55:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:55:42.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:55:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:55:42.058Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:55:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:55:42 smithi153 ceph-mon[70044]: pgmap v15221: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:42.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:55:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:55:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:55:42.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:55:42 smithi083 ceph-mon[77242]: pgmap v15221: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:42.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:55:42 smithi083 ceph-mon[79930]: pgmap v15221: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:55:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:55:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:55:44.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:55:44 smithi153 ceph-mon[70044]: pgmap v15222: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:44.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:55:44 smithi083 ceph-mon[79930]: pgmap v15222: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:44.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:55:44 smithi083 ceph-mon[77242]: pgmap v15222: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:45.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:55:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:55:45.416Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:55:46.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:55:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:55:46.143Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:55:46.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:55:46 smithi153 ceph-mon[70044]: pgmap v15223: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:46.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:55:46 smithi083 ceph-mon[77242]: pgmap v15223: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:46.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:55:46 smithi083 ceph-mon[79930]: pgmap v15223: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:47.284 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:55:47.574 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:55:47.574 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 9m ago 8h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:55:47.575 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 7m ago 8h 63.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:55:47.575 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 9m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:55:47.575 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 7m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:55:47.575 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 7m ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:55:47.576 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 9m ago 8h 927M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:55:47.576 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 9m ago 8h 1842M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:55:47.576 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 7m ago 8h 799M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:55:47.576 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 9m ago 8h 1097M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:55:47.577 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 9m ago 8h 24.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:55:47.577 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 7m ago 8h 26.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:55:47.577 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 9m ago 8h 1334M 2277M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:55:47.577 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 9m ago 8h 1320M 2277M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:55:47.578 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 9m ago 8h 1028M 2277M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:55:47.578 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 9m ago 8h 1294M 2277M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:55:47.578 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 7m ago 8h 1135M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:55:47.578 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 7m ago 8h 1424M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:55:47.579 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 7m ago 8h 1272M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:55:47.579 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 7m ago 8h 1511M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:55:47.579 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 7m ago 8h 185M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:55:47.933 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:55:47.933 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:55:47.933 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:55:47.933 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:55:47.934 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:55:47.934 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:55:47.934 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:55:47.934 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:55:47.935 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:55:47.935 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:55:47.935 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:55:47.935 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:55:47.936 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:55:47.936 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:55:47.936 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:55:47.936 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:55:47.936 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:55:47.937 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:55:48.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:55:48 smithi153 ceph-mon[70044]: pgmap v15224: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:48.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:55:48 smithi153 ceph-mon[70044]: from='client.51834 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:55:48.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:55:48 smithi153 ceph-mon[70044]: from='client.61771 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:55:48.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:55:48 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4123678148' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:55:48.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:55:48 smithi083 ceph-mon[77242]: pgmap v15224: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:48.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:55:48 smithi083 ceph-mon[77242]: from='client.51834 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:55:48.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:55:48 smithi083 ceph-mon[77242]: from='client.61771 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:55:48.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:55:48 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4123678148' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:55:48.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:55:48 smithi083 ceph-mon[79930]: pgmap v15224: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:48.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:55:48 smithi083 ceph-mon[79930]: from='client.51834 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:55:48.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:55:48 smithi083 ceph-mon[79930]: from='client.61771 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:55:48.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:55:48 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4123678148' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:55:50.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:55:50 smithi153 ceph-mon[70044]: pgmap v15225: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:50.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:55:50 smithi083 ceph-mon[77242]: pgmap v15225: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:50.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:55:50 smithi083 ceph-mon[79930]: pgmap v15225: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:55:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:55:51] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T13:55:52.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:55:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:55:52.059Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:55:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:55:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:55:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:55:52.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:55:52 smithi153 ceph-mon[70044]: pgmap v15226: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:52.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:55:52 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:55:52.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:55:52 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:55:52.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:55:52 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:55:52.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:55:52 smithi083 ceph-mon[79930]: pgmap v15226: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:52.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:55:52 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:55:52.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:55:52 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:55:52.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:55:52 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:55:52.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:55:52 smithi083 ceph-mon[77242]: pgmap v15226: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:52.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:55:52 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:55:52.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:55:52 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:55:52.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:55:52 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:55:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:55:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:55:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:55:54.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:55:53 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:55:54.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:55:53 smithi083 ceph-mon[79930]: pgmap v15227: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:54.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:55:53 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:55:54.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:55:53 smithi083 ceph-mon[77242]: pgmap v15227: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:54.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:55:53 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:55:54.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:55:53 smithi153 ceph-mon[70044]: pgmap v15227: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:55.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:55:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:55:55.417Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:55:56.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:55:56 smithi153 ceph-mon[70044]: pgmap v15228: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:56.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:55:56 smithi083 ceph-mon[79930]: pgmap v15228: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:56.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:55:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:55:56.144Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:55:56.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:55:56 smithi083 ceph-mon[77242]: pgmap v15228: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:58.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:55:58 smithi083 ceph-mon[77242]: pgmap v15229: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:58.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:55:58 smithi083 ceph-mon[79930]: pgmap v15229: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:55:58.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:55:58 smithi153 ceph-mon[70044]: pgmap v15229: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:00.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:56:00 smithi083 ceph-mon[79930]: pgmap v15230: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:00.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:56:00 smithi083 ceph-mon[77242]: pgmap v15230: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:00.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:56:00 smithi153 ceph-mon[70044]: pgmap v15230: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:56:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:56:01] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T13:56:02.408 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:56:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:56:02.059Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:56:02.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:56:02 smithi083 ceph-mon[77242]: pgmap v15231: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:02.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:56:02 smithi083 ceph-mon[79930]: pgmap v15231: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:02.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:56:02 smithi153 ceph-mon[70044]: pgmap v15231: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:02.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:56:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:56:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:56:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:56:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:56:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:56:04.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:56:04 smithi083 ceph-mon[77242]: pgmap v15232: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:04.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:56:04 smithi083 ceph-mon[79930]: pgmap v15232: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:04.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:56:04 smithi153 ceph-mon[70044]: pgmap v15232: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:05.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:56:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:56:05.418Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:56:06.416 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:56:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:56:06.144Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:56:06.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:56:06 smithi083 ceph-mon[79930]: pgmap v15233: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:06.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:56:06 smithi083 ceph-mon[77242]: pgmap v15233: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:06.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:56:06 smithi153 ceph-mon[70044]: pgmap v15233: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:08.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:56:08 smithi083 ceph-mon[79930]: pgmap v15234: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:08.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:56:08 smithi083 ceph-mon[77242]: pgmap v15234: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:08.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:56:08 smithi153 ceph-mon[70044]: pgmap v15234: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:10.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:56:10 smithi083 ceph-mon[79930]: pgmap v15235: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:10.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:56:10 smithi083 ceph-mon[77242]: pgmap v15235: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:10.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:56:10 smithi153 ceph-mon[70044]: pgmap v15235: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:56:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:56:11] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T13:56:12.426 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:56:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:56:12.060Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:56:12.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:56:12 smithi083 ceph-mon[79930]: pgmap v15236: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:12.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:56:12 smithi083 ceph-mon[77242]: pgmap v15236: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:12.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:56:12 smithi153 ceph-mon[70044]: pgmap v15236: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:12.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:56:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:56:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:56:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:56:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:56:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:56:14.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:56:14 smithi083 ceph-mon[79930]: pgmap v15237: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:14.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:56:14 smithi083 ceph-mon[77242]: pgmap v15237: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:14.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:56:14 smithi153 ceph-mon[70044]: pgmap v15237: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:15.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:56:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:56:15.419Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:56:16.432 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:56:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:56:16.145Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:56:16.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:56:16 smithi083 ceph-mon[77242]: pgmap v15238: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:16.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:56:16 smithi083 ceph-mon[79930]: pgmap v15238: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:16.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:56:16 smithi153 ceph-mon[70044]: pgmap v15238: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:18.264 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:56:18.553 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:56:18.553 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 9m ago 8h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:56:18.553 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 7m ago 8h 63.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:56:18.553 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 9m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:56:18.554 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 7m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:56:18.554 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 7m ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:56:18.554 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 9m ago 8h 927M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:56:18.554 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 9m ago 8h 1842M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:56:18.555 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 7m ago 8h 799M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:56:18.555 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 9m ago 8h 1097M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:56:18.555 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 9m ago 8h 24.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:56:18.555 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 7m ago 8h 26.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:56:18.556 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 9m ago 8h 1334M 2277M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:56:18.556 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 9m ago 8h 1320M 2277M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:56:18.556 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 9m ago 8h 1028M 2277M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:56:18.556 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 9m ago 8h 1294M 2277M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:56:18.556 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 7m ago 8h 1135M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:56:18.557 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 7m ago 8h 1424M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:56:18.557 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 7m ago 8h 1272M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:56:18.557 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 7m ago 8h 1511M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:56:18.557 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 7m ago 8h 185M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:56:18.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:56:18 smithi083 ceph-mon[79930]: pgmap v15239: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:18.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:56:18 smithi083 ceph-mon[77242]: pgmap v15239: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:18.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:56:18 smithi153 ceph-mon[70044]: pgmap v15239: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:18.908 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:56:18.909 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:56:18.909 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:56:18.909 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:56:18.909 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:56:18.910 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:56:18.910 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:56:18.910 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:56:18.910 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:56:18.911 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:56:18.911 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:56:18.911 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:56:18.911 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:56:18.911 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:56:18.912 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:56:18.912 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:56:18.912 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:56:18.912 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:56:19.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:56:19 smithi083 ceph-mon[77242]: from='client.61783 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:56:19.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:56:19 smithi083 ceph-mon[77242]: from='client.61789 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:56:19.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:56:19 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4276408191' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:56:19.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:56:19 smithi083 ceph-mon[79930]: from='client.61783 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:56:19.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:56:19 smithi083 ceph-mon[79930]: from='client.61789 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:56:19.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:56:19 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4276408191' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:56:19.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:56:19 smithi153 ceph-mon[70044]: from='client.61783 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:56:19.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:56:19 smithi153 ceph-mon[70044]: from='client.61789 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:56:19.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:56:19 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4276408191' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:56:20.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:56:20 smithi083 ceph-mon[79930]: pgmap v15240: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:20.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:56:20 smithi083 ceph-mon[77242]: pgmap v15240: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:20.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:56:20 smithi153 ceph-mon[70044]: pgmap v15240: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:56:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:56:21] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T13:56:22.444 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:56:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:56:22.060Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:56:22.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:56:22 smithi083 ceph-mon[77242]: pgmap v15241: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:22.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:56:22 smithi083 ceph-mon[79930]: pgmap v15241: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:22.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:56:22 smithi153 ceph-mon[70044]: pgmap v15241: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:22.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:56:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:56:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:56:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:56:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:56:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:56:24.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:56:24 smithi083 ceph-mon[79930]: pgmap v15242: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:24.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:56:24 smithi083 ceph-mon[77242]: pgmap v15242: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:24.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:56:24 smithi153 ceph-mon[70044]: pgmap v15242: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:25.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:56:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:56:25.419Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:56:26.452 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:56:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:56:26.146Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:56:26.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:56:26 smithi083 ceph-mon[79930]: pgmap v15243: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:26.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:56:26 smithi083 ceph-mon[77242]: pgmap v15243: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:26.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:56:26 smithi153 ceph-mon[70044]: pgmap v15243: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:28.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:56:28 smithi083 ceph-mon[77242]: pgmap v15244: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:28.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:56:28 smithi083 ceph-mon[79930]: pgmap v15244: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:28.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:56:28 smithi153 ceph-mon[70044]: pgmap v15244: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:30.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:56:30 smithi083 ceph-mon[79930]: pgmap v15245: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:30.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:56:30 smithi083 ceph-mon[77242]: pgmap v15245: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:30.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:56:30 smithi153 ceph-mon[70044]: pgmap v15245: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:56:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:56:31] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T13:56:32.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:56:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:56:32.061Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:56:32.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:56:32 smithi153 ceph-mon[70044]: pgmap v15246: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:32.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:56:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:56:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:56:32.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:56:32 smithi083 ceph-mon[79930]: pgmap v15246: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:32.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:56:32 smithi083 ceph-mon[77242]: pgmap v15246: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:56:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:56:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:56:34.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:56:34 smithi153 ceph-mon[70044]: pgmap v15247: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:34.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:56:34 smithi083 ceph-mon[77242]: pgmap v15247: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:34.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:56:34 smithi083 ceph-mon[79930]: pgmap v15247: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:35.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:56:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:56:35.420Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:56:36.465 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:56:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:56:36.146Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:56:36.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:56:36 smithi153 ceph-mon[70044]: pgmap v15248: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:36.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:56:36 smithi083 ceph-mon[79930]: pgmap v15248: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:36.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:56:36 smithi083 ceph-mon[77242]: pgmap v15248: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:38.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:56:38 smithi153 ceph-mon[70044]: pgmap v15249: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:38.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:56:38 smithi083 ceph-mon[79930]: pgmap v15249: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:38.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:56:38 smithi083 ceph-mon[77242]: pgmap v15249: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:56:40 smithi153 ceph-mon[70044]: pgmap v15250: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:56:40 smithi083 ceph-mon[79930]: pgmap v15250: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:56:40 smithi083 ceph-mon[77242]: pgmap v15250: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:41.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:56:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:56:41.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:56:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:56:41.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:56:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:56:41.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:56:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:56:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:56:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:56:41] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T13:56:41.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:56:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:56:41.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:56:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:56:41.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:56:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:56:41.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:56:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:56:41.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:56:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:56:41.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:56:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:56:41.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:56:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:56:41.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:56:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:56:42.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:56:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:56:42.061Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:56:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:56:42 smithi153 ceph-mon[70044]: pgmap v15251: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:42.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:56:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:56:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:56:42.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:56:42 smithi083 ceph-mon[79930]: pgmap v15251: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:42.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:56:42 smithi083 ceph-mon[77242]: pgmap v15251: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:56:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:56:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:56:44.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:56:44 smithi153 ceph-mon[70044]: pgmap v15252: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:44.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:56:44 smithi083 ceph-mon[77242]: pgmap v15252: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:44.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:56:44 smithi083 ceph-mon[79930]: pgmap v15252: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:45.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:56:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:56:45.420Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:56:46.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:56:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:56:46.147Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:56:46.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:56:46 smithi153 ceph-mon[70044]: pgmap v15253: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:46.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:56:46 smithi083 ceph-mon[79930]: pgmap v15253: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:46.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:56:46 smithi083 ceph-mon[77242]: pgmap v15253: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:48.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:56:48 smithi153 ceph-mon[70044]: pgmap v15254: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:48.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:56:48 smithi083 ceph-mon[79930]: pgmap v15254: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:48.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:56:48 smithi083 ceph-mon[77242]: pgmap v15254: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:49.233 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:56:49.522 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:56:49.522 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 10m ago 8h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:56:49.523 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 8m ago 8h 63.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:56:49.523 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 10m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:56:49.523 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 8m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:56:49.523 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 8m ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:56:49.524 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 10m ago 8h 927M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:56:49.524 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 10m ago 8h 1842M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:56:49.524 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 8m ago 8h 799M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:56:49.524 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 10m ago 8h 1097M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:56:49.524 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 10m ago 8h 24.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:56:49.525 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 8m ago 8h 26.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:56:49.525 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 10m ago 8h 1334M 2277M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:56:49.525 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 10m ago 8h 1320M 2277M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:56:49.525 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 10m ago 8h 1028M 2277M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:56:49.525 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 10m ago 8h 1294M 2277M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:56:49.526 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 8m ago 8h 1135M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:56:49.526 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 8m ago 8h 1424M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:56:49.526 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 8m ago 8h 1272M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:56:49.527 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 8m ago 8h 1511M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:56:49.527 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 8m ago 8h 185M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:56:49.877 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:56:49.877 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:56:49.877 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:56:49.878 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:56:49.878 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:56:49.878 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:56:49.878 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:56:49.879 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:56:49.879 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:56:49.879 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:56:49.879 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:56:49.879 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:56:49.880 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:56:49.880 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:56:49.880 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:56:49.880 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:56:49.881 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:56:49.881 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:56:50.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:56:50 smithi153 ceph-mon[70044]: pgmap v15255: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:50.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:56:50 smithi153 ceph-mon[70044]: from='client.51870 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:56:50.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:56:50 smithi153 ceph-mon[70044]: from='client.51876 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:56:50.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:56:50 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/507206928' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:56:50.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:56:50 smithi083 ceph-mon[79930]: pgmap v15255: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:50.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:56:50 smithi083 ceph-mon[79930]: from='client.51870 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:56:50.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:56:50 smithi083 ceph-mon[79930]: from='client.51876 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:56:50.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:56:50 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/507206928' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:56:50.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:56:50 smithi083 ceph-mon[77242]: pgmap v15255: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:50.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:56:50 smithi083 ceph-mon[77242]: from='client.51870 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:56:50.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:56:50 smithi083 ceph-mon[77242]: from='client.51876 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:56:50.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:56:50 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/507206928' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:56:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:56:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:56:51] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T13:56:52.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:56:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:56:52.062Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:56:52.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:56:52 smithi153 ceph-mon[70044]: pgmap v15256: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:56:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:56:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:56:52.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:56:52 smithi083 ceph-mon[79930]: pgmap v15256: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:52.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:56:52 smithi083 ceph-mon[77242]: pgmap v15256: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:56:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:56:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:56:53.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:56:53 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:56:53.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:56:53 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:56:53.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:56:53 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:56:53.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:56:53 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:56:53.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:56:53 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:56:53.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:56:53 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:56:53.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:56:53 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:56:53.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:56:53 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:56:53.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:56:53 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:56:53.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:56:53 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:56:53.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:56:53 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:56:53.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:56:53 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:56:53.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:56:53 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:56:53.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:56:53 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:56:53.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:56:53 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:56:53.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:56:53 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:56:53.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:56:53 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:56:53.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:56:53 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:56:53.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:56:53 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:56:53.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:56:53 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:56:53.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:56:53 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:56:54.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:56:54 smithi153 ceph-mon[70044]: pgmap v15257: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:54.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:56:54 smithi083 ceph-mon[79930]: pgmap v15257: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:54.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:56:54 smithi083 ceph-mon[77242]: pgmap v15257: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:55.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:56:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:56:55.420Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:56:56.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:56:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:56:56.148Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:56:56.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:56:56 smithi153 ceph-mon[70044]: pgmap v15258: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:56.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:56:56 smithi083 ceph-mon[79930]: pgmap v15258: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:56.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:56:56 smithi083 ceph-mon[77242]: pgmap v15258: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:58.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:56:58 smithi083 ceph-mon[79930]: pgmap v15259: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:58.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:56:58 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:56:58.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:56:58 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:56:58.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:56:58 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:56:58.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:56:58 smithi083 ceph-mon[77242]: pgmap v15259: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:58.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:56:58 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:56:58.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:56:58 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:56:58.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:56:58 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:56:59.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:56:58 smithi153 ceph-mon[70044]: pgmap v15259: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:56:59.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:56:58 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:56:59.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:56:58 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:56:59.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:56:58 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:57:00.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:57:00 smithi083 ceph-mon[79930]: pgmap v15260: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:00.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:57:00 smithi083 ceph-mon[77242]: pgmap v15260: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:01.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:57:00 smithi153 ceph-mon[70044]: pgmap v15260: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:57:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:57:01] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T13:57:02.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:57:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:57:02.063Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:57:02.586 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:57:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:57:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:57:02.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:57:02 smithi083 ceph-mon[77242]: pgmap v15261: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:02.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:57:02 smithi083 ceph-mon[79930]: pgmap v15261: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:03.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:57:02 smithi153 ceph-mon[70044]: pgmap v15261: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:57:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:57:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:57:04.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:57:04 smithi083 ceph-mon[77242]: pgmap v15262: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:04.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:57:04 smithi083 ceph-mon[79930]: pgmap v15262: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:05.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:57:04 smithi153 ceph-mon[70044]: pgmap v15262: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:05.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:57:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:57:05.422Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:57:06.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:57:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:57:06.149Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:57:06.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:57:06 smithi083 ceph-mon[79930]: pgmap v15263: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:06.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:57:06 smithi083 ceph-mon[77242]: pgmap v15263: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:07.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:57:06 smithi153 ceph-mon[70044]: pgmap v15263: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:08.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:57:08 smithi083 ceph-mon[79930]: pgmap v15264: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:08.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:57:08 smithi083 ceph-mon[77242]: pgmap v15264: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:09.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:57:08 smithi153 ceph-mon[70044]: pgmap v15264: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:10.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:57:10 smithi083 ceph-mon[79930]: pgmap v15265: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:10.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:57:10 smithi083 ceph-mon[77242]: pgmap v15265: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:11.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:57:10 smithi153 ceph-mon[70044]: pgmap v15265: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:57:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:57:11] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T13:57:12.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:57:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:57:12.064Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:57:12.600 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:57:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:57:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:57:12.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:57:12 smithi083 ceph-mon[79930]: pgmap v15266: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:12.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:57:12 smithi083 ceph-mon[77242]: pgmap v15266: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:13.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:57:12 smithi153 ceph-mon[70044]: pgmap v15266: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:57:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:57:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:57:14.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:57:14 smithi083 ceph-mon[79930]: pgmap v15267: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:14.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:57:14 smithi083 ceph-mon[77242]: pgmap v15267: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:15.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:57:14 smithi153 ceph-mon[70044]: pgmap v15267: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:15.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:57:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:57:15.423Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:57:16.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:57:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:57:16.149Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:57:16.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:57:16 smithi083 ceph-mon[79930]: pgmap v15268: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:16.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:57:16 smithi083 ceph-mon[77242]: pgmap v15268: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:17.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:57:16 smithi153 ceph-mon[70044]: pgmap v15268: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:18.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:57:18 smithi083 ceph-mon[79930]: pgmap v15269: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:18.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:57:18 smithi083 ceph-mon[77242]: pgmap v15269: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:19.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:57:18 smithi153 ceph-mon[70044]: pgmap v15269: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:20.205 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:57:20.493 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:57:20.493 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 22s ago 8h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:57:20.493 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 8m ago 8h 63.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:57:20.494 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 22s ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:57:20.494 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 8m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:57:20.494 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 8m ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:57:20.494 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 22s ago 8h 935M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:57:20.495 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 22s ago 8h 1879M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:57:20.495 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 8m ago 8h 799M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:57:20.495 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 22s ago 8h 1121M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:57:20.495 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 22s ago 8h 24.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:57:20.496 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 8m ago 8h 26.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:57:20.496 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 22s ago 8h 1359M 2277M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:57:20.496 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 22s ago 8h 1347M 2277M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:57:20.496 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 22s ago 8h 1048M 2277M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:57:20.496 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 22s ago 8h 1319M 2277M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:57:20.497 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 8m ago 8h 1135M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:57:20.497 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 8m ago 8h 1424M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:57:20.497 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 8m ago 8h 1272M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:57:20.497 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 8m ago 8h 1511M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:57:20.498 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 8m ago 8h 185M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:57:20.848 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:57:20.849 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:57:20.849 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:57:20.849 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:57:20.849 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:57:20.849 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:57:20.850 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:57:20.850 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:57:20.850 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:57:20.851 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:57:20.851 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:57:20.851 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:57:20.851 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:57:20.852 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:57:20.852 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:57:20.852 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:57:20.852 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:57:20.853 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:57:20.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:57:20 smithi083 ceph-mon[79930]: pgmap v15270: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:20.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:57:20 smithi083 ceph-mon[77242]: pgmap v15270: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:21.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:57:20 smithi153 ceph-mon[70044]: pgmap v15270: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:57:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:57:21] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T13:57:21.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:57:21 smithi083 ceph-mon[79930]: from='client.51888 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:57:21.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:57:21 smithi083 ceph-mon[79930]: from='client.51894 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:57:21.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:57:21 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/332012972' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:57:21.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:57:21 smithi083 ceph-mon[77242]: from='client.51888 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:57:21.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:57:21 smithi083 ceph-mon[77242]: from='client.51894 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:57:21.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:57:21 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/332012972' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:57:22.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:57:21 smithi153 ceph-mon[70044]: from='client.51888 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:57:22.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:57:21 smithi153 ceph-mon[70044]: from='client.51894 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:57:22.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:57:21 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/332012972' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:57:22.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:57:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:57:22.064Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:57:22.618 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:57:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:57:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:57:22.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:57:22 smithi083 ceph-mon[79930]: pgmap v15271: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:22.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:57:22 smithi083 ceph-mon[77242]: pgmap v15271: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:23.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:57:22 smithi153 ceph-mon[70044]: pgmap v15271: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:57:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:57:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:57:24.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:57:24 smithi083 ceph-mon[79930]: pgmap v15272: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:24.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:57:24 smithi083 ceph-mon[77242]: pgmap v15272: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:25.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:57:24 smithi153 ceph-mon[70044]: pgmap v15272: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:25.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:57:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:57:25.424Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:57:26.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:57:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:57:26.150Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:57:26.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:57:26 smithi083 ceph-mon[79930]: pgmap v15273: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:26.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:57:26 smithi083 ceph-mon[77242]: pgmap v15273: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:27.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:57:26 smithi153 ceph-mon[70044]: pgmap v15273: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:28.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:57:28 smithi083 ceph-mon[79930]: pgmap v15274: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:28.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:57:28 smithi083 ceph-mon[77242]: pgmap v15274: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:29.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:57:28 smithi153 ceph-mon[70044]: pgmap v15274: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:30.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:57:30 smithi083 ceph-mon[77242]: pgmap v15275: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:30.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:57:30 smithi083 ceph-mon[79930]: pgmap v15275: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:31.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:57:30 smithi153 ceph-mon[70044]: pgmap v15275: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:57:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:57:31] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T13:57:32.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:57:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:57:32.066Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:57:32.642 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:57:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:57:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:57:32.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:57:32 smithi083 ceph-mon[77242]: pgmap v15276: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:32.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:57:32 smithi083 ceph-mon[79930]: pgmap v15276: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:33.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:57:32 smithi153 ceph-mon[70044]: pgmap v15276: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:57:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:57:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:57:34.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:57:34 smithi083 ceph-mon[77242]: pgmap v15277: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:34.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:57:34 smithi083 ceph-mon[79930]: pgmap v15277: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:35.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:57:34 smithi153 ceph-mon[70044]: pgmap v15277: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:35.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:57:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:57:35.424Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:57:36.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:57:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:57:36.150Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:57:36.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:57:36 smithi083 ceph-mon[79930]: pgmap v15278: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:36.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:57:36 smithi083 ceph-mon[77242]: pgmap v15278: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:37.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:57:36 smithi153 ceph-mon[70044]: pgmap v15278: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:38.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:57:38 smithi083 ceph-mon[77242]: pgmap v15279: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:38.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:57:38 smithi083 ceph-mon[79930]: pgmap v15279: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:39.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:57:38 smithi153 ceph-mon[70044]: pgmap v15279: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:57:40 smithi083 ceph-mon[79930]: pgmap v15280: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:57:40 smithi083 ceph-mon[77242]: pgmap v15280: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:57:40 smithi153 ceph-mon[70044]: pgmap v15280: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:57:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:57:41] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T13:57:41.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:57:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:57:41.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:57:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:57:41.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:57:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:57:41.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:57:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:57:41.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:57:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:57:41.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:57:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:57:41.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:57:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:57:41.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:57:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:57:42.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:57:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:57:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:57:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:57:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:57:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:57:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:57:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:57:42.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:57:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:57:42.066Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:57:42.662 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:57:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:57:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:57:42.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:57:42 smithi083 ceph-mon[79930]: pgmap v15281: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:42.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:57:42 smithi083 ceph-mon[77242]: pgmap v15281: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:43.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:57:42 smithi153 ceph-mon[70044]: pgmap v15281: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:57:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:57:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:57:44.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:57:44 smithi083 ceph-mon[79930]: pgmap v15282: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:44.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:57:44 smithi083 ceph-mon[77242]: pgmap v15282: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:45.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:57:44 smithi153 ceph-mon[70044]: pgmap v15282: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:45.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:57:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:57:45.425Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:57:46.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:57:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:57:46.152Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:57:46.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:57:46 smithi083 ceph-mon[79930]: pgmap v15283: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:46.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:57:46 smithi083 ceph-mon[77242]: pgmap v15283: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:47.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:57:46 smithi153 ceph-mon[70044]: pgmap v15283: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:48.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:57:48 smithi083 ceph-mon[79930]: pgmap v15284: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:48.968 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:57:48 smithi083 ceph-mon[77242]: pgmap v15284: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:49.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:57:48 smithi153 ceph-mon[70044]: pgmap v15284: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:50.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:57:50 smithi083 ceph-mon[79930]: pgmap v15285: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:50.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:57:50 smithi083 ceph-mon[77242]: pgmap v15285: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:51.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:57:50 smithi153 ceph-mon[70044]: pgmap v15285: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:51.177 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:57:51.464 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:57:51.464 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 53s ago 8h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:57:51.464 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 9m ago 8h 63.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:57:51.465 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 53s ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:57:51.465 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 9m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:57:51.465 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 9m ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:57:51.465 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 53s ago 8h 935M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:57:51.466 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 53s ago 8h 1879M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:57:51.466 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 9m ago 8h 799M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:57:51.466 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 53s ago 8h 1121M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:57:51.466 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 53s ago 8h 24.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:57:51.467 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 9m ago 8h 26.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:57:51.467 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 53s ago 8h 1359M 2277M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:57:51.467 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 53s ago 8h 1347M 2277M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:57:51.467 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 53s ago 8h 1048M 2277M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:57:51.468 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 53s ago 8h 1319M 2277M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:57:51.468 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 9m ago 8h 1135M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:57:51.468 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 9m ago 8h 1424M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:57:51.468 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 9m ago 8h 1272M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:57:51.468 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 9m ago 8h 1511M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:57:51.469 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 9m ago 8h 185M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:57:51.821 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:57:51.821 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:57:51.822 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:57:51.822 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:57:51.822 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:57:51.822 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:57:51.823 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:57:51.823 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:57:51.823 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:57:51.823 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:57:51.824 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:57:51.824 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:57:51.824 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:57:51.824 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:57:51.825 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:57:51.825 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:57:51.825 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:57:51.825 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:57:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:57:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:57:51] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T13:57:52.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:57:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:57:52.066Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:57:52.677 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:57:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:57:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:57:52.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:57:52 smithi083 ceph-mon[79930]: pgmap v15286: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:52.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:57:52 smithi083 ceph-mon[79930]: from='client.61837 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:57:52.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:57:52 smithi083 ceph-mon[79930]: from='client.51912 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:57:52.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:57:52 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/906690538' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:57:52.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:57:52 smithi083 ceph-mon[77242]: pgmap v15286: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:52.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:57:52 smithi083 ceph-mon[77242]: from='client.61837 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:57:52.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:57:52 smithi083 ceph-mon[77242]: from='client.51912 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:57:52.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:57:52 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/906690538' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:57:53.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:57:52 smithi153 ceph-mon[70044]: pgmap v15286: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:53.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:57:52 smithi153 ceph-mon[70044]: from='client.61837 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:57:53.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:57:52 smithi153 ceph-mon[70044]: from='client.51912 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:57:53.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:57:52 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/906690538' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:57:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:57:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:57:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:57:54.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:57:54 smithi083 ceph-mon[79930]: pgmap v15287: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:54.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:57:54 smithi083 ceph-mon[77242]: pgmap v15287: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:55.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:57:54 smithi153 ceph-mon[70044]: pgmap v15287: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:55.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:57:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:57:55.425Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:57:56.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:57:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:57:56.152Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:57:56.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:57:56 smithi083 ceph-mon[77242]: pgmap v15288: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:56.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:57:56 smithi083 ceph-mon[79930]: pgmap v15288: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:57.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:57:56 smithi153 ceph-mon[70044]: pgmap v15288: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:58.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:57:58 smithi083 ceph-mon[79930]: pgmap v15289: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:58.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:57:58 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:57:58.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:57:58 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:57:58.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:57:58 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:57:58.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:57:58 smithi083 ceph-mon[77242]: pgmap v15289: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:58.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:57:58 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:57:58.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:57:58 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:57:58.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:57:58 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:57:59.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:57:58 smithi153 ceph-mon[70044]: pgmap v15289: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:57:59.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:57:58 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:57:59.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:57:58 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:57:59.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:57:58 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:58:00.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:57:59 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:58:00.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:57:59 smithi153 ceph-mon[70044]: pgmap v15290: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:00.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:57:59 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:58:00.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:57:59 smithi083 ceph-mon[79930]: pgmap v15290: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:00.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:57:59 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:58:00.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:57:59 smithi083 ceph-mon[77242]: pgmap v15290: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:58:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:58:01] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T13:58:02.427 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:58:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:58:02.067Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:58:02.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:58:02 smithi083 ceph-mon[79930]: pgmap v15291: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:02.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:58:02 smithi083 ceph-mon[77242]: pgmap v15291: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:02.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:58:02 smithi153 ceph-mon[70044]: pgmap v15291: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:02.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:58:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:58:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:58:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:58:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:58:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:58:04.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:58:04 smithi083 ceph-mon[79930]: pgmap v15292: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:04.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:58:04 smithi083 ceph-mon[77242]: pgmap v15292: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:04.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:58:04 smithi153 ceph-mon[70044]: pgmap v15292: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:05.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:58:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:58:05.426Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:58:06.435 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:58:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:58:06.152Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:58:06.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:58:06 smithi083 ceph-mon[79930]: pgmap v15293: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:06.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:58:06 smithi083 ceph-mon[77242]: pgmap v15293: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:06.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:58:06 smithi153 ceph-mon[70044]: pgmap v15293: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:08.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:58:08 smithi083 ceph-mon[79930]: pgmap v15294: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:08.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:58:08 smithi083 ceph-mon[77242]: pgmap v15294: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:08.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:58:08 smithi153 ceph-mon[70044]: pgmap v15294: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:10.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:58:10 smithi083 ceph-mon[77242]: pgmap v15295: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:10.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:58:10 smithi083 ceph-mon[79930]: pgmap v15295: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:10.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:58:10 smithi153 ceph-mon[70044]: pgmap v15295: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:58:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:58:11] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T13:58:12.443 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:58:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:58:12.068Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:58:12.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:58:12 smithi083 ceph-mon[79930]: pgmap v15296: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:12.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:58:12 smithi083 ceph-mon[77242]: pgmap v15296: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:12.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:58:12 smithi153 ceph-mon[70044]: pgmap v15296: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:12.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:58:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:58:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:58:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:58:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:58:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:58:14.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:58:14 smithi083 ceph-mon[79930]: pgmap v15297: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:14.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:58:14 smithi083 ceph-mon[77242]: pgmap v15297: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:14.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:58:14 smithi153 ceph-mon[70044]: pgmap v15297: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:15.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:58:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:58:15.427Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:58:16.450 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:58:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:58:16.153Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:58:16.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:58:16 smithi083 ceph-mon[79930]: pgmap v15298: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:16.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:58:16 smithi083 ceph-mon[77242]: pgmap v15298: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:16.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:58:16 smithi153 ceph-mon[70044]: pgmap v15298: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:18.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:58:18 smithi083 ceph-mon[79930]: pgmap v15299: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:18.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:58:18 smithi083 ceph-mon[77242]: pgmap v15299: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:18.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:58:18 smithi153 ceph-mon[70044]: pgmap v15299: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:20.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:58:20 smithi083 ceph-mon[79930]: pgmap v15300: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:20.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:58:20 smithi083 ceph-mon[77242]: pgmap v15300: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:20.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:58:20 smithi153 ceph-mon[70044]: pgmap v15300: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:58:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:58:21] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T13:58:22.152 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:58:22.442 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:58:22.443 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 84s ago 8h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:58:22.443 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 9m ago 8h 63.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:58:22.444 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 84s ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:58:22.444 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 9m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:58:22.444 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 9m ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:58:22.445 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 84s ago 8h 935M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:58:22.445 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 84s ago 8h 1879M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:58:22.446 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 9m ago 8h 799M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:58:22.446 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 84s ago 8h 1121M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:58:22.447 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 84s ago 8h 24.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:58:22.447 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 9m ago 8h 26.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:58:22.448 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 84s ago 8h 1359M 2277M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:58:22.448 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 84s ago 8h 1347M 2277M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:58:22.449 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 84s ago 8h 1048M 2277M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:58:22.449 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 84s ago 8h 1319M 2277M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:58:22.449 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 9m ago 8h 1135M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:58:22.450 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 9m ago 8h 1424M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:58:22.450 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 9m ago 8h 1272M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:58:22.451 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 9m ago 8h 1511M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:58:22.451 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 9m ago 8h 185M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:58:22.458 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:58:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:58:22.067Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:58:22.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:58:22 smithi083 ceph-mon[79930]: pgmap v15301: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:22.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:58:22 smithi083 ceph-mon[77242]: pgmap v15301: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:22.800 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:58:22.800 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:58:22.801 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:58:22.801 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:58:22.802 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:58:22.802 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:58:22.803 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:58:22.803 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:58:22.804 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:58:22.804 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:58:22.805 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:58:22.805 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:58:22.806 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:58:22.806 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:58:22.807 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:58:22.807 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:58:22.808 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:58:22.808 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:58:22.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:58:22 smithi153 ceph-mon[70044]: pgmap v15301: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:22.810 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:58:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:58:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:58:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:58:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:58:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:58:23.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:58:23 smithi153 ceph-mon[70044]: from='client.61855 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:58:23.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:58:23 smithi153 ceph-mon[70044]: from='client.51930 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:58:23.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:58:23 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1664819147' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:58:23.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:58:23 smithi083 ceph-mon[79930]: from='client.61855 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:58:23.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:58:23 smithi083 ceph-mon[79930]: from='client.51930 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:58:23.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:58:23 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1664819147' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:58:23.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:58:23 smithi083 ceph-mon[77242]: from='client.61855 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:58:23.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:58:23 smithi083 ceph-mon[77242]: from='client.51930 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:58:23.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:58:23 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1664819147' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:58:24.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:58:24 smithi153 ceph-mon[70044]: pgmap v15302: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:24.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:58:24 smithi083 ceph-mon[79930]: pgmap v15302: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:24.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:58:24 smithi083 ceph-mon[77242]: pgmap v15302: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:25.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:58:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:58:25.427Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:58:26.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:58:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:58:26.153Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:58:26.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:58:26 smithi153 ceph-mon[70044]: pgmap v15303: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:26.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:58:26 smithi083 ceph-mon[77242]: pgmap v15303: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:26.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:58:26 smithi083 ceph-mon[79930]: pgmap v15303: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:28.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:58:28 smithi153 ceph-mon[70044]: pgmap v15304: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:28.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:58:28 smithi083 ceph-mon[79930]: pgmap v15304: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:28.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:58:28 smithi083 ceph-mon[77242]: pgmap v15304: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:30.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:58:30 smithi153 ceph-mon[70044]: pgmap v15305: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:30.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:58:30 smithi083 ceph-mon[79930]: pgmap v15305: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:30.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:58:30 smithi083 ceph-mon[77242]: pgmap v15305: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:58:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:58:31] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T13:58:32.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:58:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:58:32.068Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:58:32.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:58:32 smithi153 ceph-mon[70044]: pgmap v15306: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:32.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:58:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:58:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:58:32.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:58:32 smithi083 ceph-mon[79930]: pgmap v15306: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:32.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:58:32 smithi083 ceph-mon[77242]: pgmap v15306: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:58:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:58:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:58:34.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:58:34 smithi153 ceph-mon[70044]: pgmap v15307: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:34.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:58:34 smithi083 ceph-mon[79930]: pgmap v15307: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:34.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:58:34 smithi083 ceph-mon[77242]: pgmap v15307: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:35.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:58:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:58:35.427Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:58:36.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:58:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:58:36.154Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:58:36.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:58:36 smithi153 ceph-mon[70044]: pgmap v15308: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:36.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:58:36 smithi083 ceph-mon[79930]: pgmap v15308: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:36.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:58:36 smithi083 ceph-mon[77242]: pgmap v15308: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:38.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:58:38 smithi153 ceph-mon[70044]: pgmap v15309: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:38.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:58:38 smithi083 ceph-mon[79930]: pgmap v15309: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:38.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:58:38 smithi083 ceph-mon[77242]: pgmap v15309: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:58:40 smithi153 ceph-mon[70044]: pgmap v15310: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:58:40 smithi083 ceph-mon[79930]: pgmap v15310: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:58:40 smithi083 ceph-mon[77242]: pgmap v15310: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:41.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:58:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:58:41.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:58:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:58:41.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:58:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:58:41.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:58:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:58:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:58:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:58:41] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T13:58:41.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:58:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:58:41.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:58:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:58:41.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:58:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:58:41.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:58:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:58:41.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:58:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:58:41.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:58:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:58:41.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:58:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:58:41.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:58:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:58:42.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:58:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:58:42.069Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:58:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:58:42 smithi153 ceph-mon[70044]: pgmap v15311: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:42.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:58:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:58:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:58:42.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:58:42 smithi083 ceph-mon[79930]: pgmap v15311: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:42.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:58:42 smithi083 ceph-mon[77242]: pgmap v15311: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:58:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:58:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:58:44.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:58:44 smithi153 ceph-mon[70044]: pgmap v15312: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:44.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:58:44 smithi083 ceph-mon[79930]: pgmap v15312: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:44.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:58:44 smithi083 ceph-mon[77242]: pgmap v15312: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:45.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:58:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:58:45.428Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:58:46.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:58:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:58:46.155Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:58:46.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:58:46 smithi153 ceph-mon[70044]: pgmap v15313: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:46.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:58:46 smithi083 ceph-mon[77242]: pgmap v15313: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:46.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:58:46 smithi083 ceph-mon[79930]: pgmap v15313: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:48.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:58:48 smithi153 ceph-mon[70044]: pgmap v15314: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:48.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:58:48 smithi083 ceph-mon[77242]: pgmap v15314: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:48.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:58:48 smithi083 ceph-mon[79930]: pgmap v15314: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:50.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:58:50 smithi153 ceph-mon[70044]: pgmap v15315: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:50.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:58:50 smithi083 ceph-mon[79930]: pgmap v15315: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:50.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:58:50 smithi083 ceph-mon[77242]: pgmap v15315: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:58:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:58:51] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T13:58:52.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:58:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:58:52.069Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:58:52.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:58:52 smithi153 ceph-mon[70044]: pgmap v15316: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:52.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:58:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:58:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:58:52.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:58:52 smithi083 ceph-mon[79930]: pgmap v15316: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:52.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:58:52 smithi083 ceph-mon[77242]: pgmap v15316: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:53.127 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:58:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:58:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:58:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:58:53.422 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:58:53.422 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 115s ago 8h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:58:53.423 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 10m ago 8h 63.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:58:53.423 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 115s ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:58:53.423 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 10m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:58:53.423 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 10m ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:58:53.424 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 115s ago 8h 935M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:58:53.424 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 115s ago 8h 1879M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:58:53.424 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 10m ago 8h 799M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:58:53.424 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 115s ago 8h 1121M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:58:53.424 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 115s ago 8h 24.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:58:53.425 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 10m ago 8h 26.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:58:53.425 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 115s ago 8h 1359M 2277M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:58:53.425 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 115s ago 8h 1347M 2277M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:58:53.425 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 115s ago 8h 1048M 2277M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:58:53.426 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 115s ago 8h 1319M 2277M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:58:53.426 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 10m ago 8h 1135M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:58:53.426 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 10m ago 8h 1424M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:58:53.426 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 10m ago 8h 1272M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:58:53.426 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 10m ago 8h 1511M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:58:53.427 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 10m ago 8h 185M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:58:53.777 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:58:53.777 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:58:53.778 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:58:53.778 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:58:53.778 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:58:53.778 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:58:53.779 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:58:53.779 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:58:53.779 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:58:53.779 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:58:53.780 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:58:53.780 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:58:53.780 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:58:53.780 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:58:53.780 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:58:53.781 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:58:53.781 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:58:53.781 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:58:54.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:58:54 smithi153 ceph-mon[70044]: pgmap v15317: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:54.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:58:54 smithi153 ceph-mon[70044]: from='client.61873 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:58:54.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:58:54 smithi153 ceph-mon[70044]: from='client.51948 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:58:54.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:58:54 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4264100844' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:58:54.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:58:54 smithi083 ceph-mon[79930]: pgmap v15317: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:54.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:58:54 smithi083 ceph-mon[79930]: from='client.61873 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:58:54.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:58:54 smithi083 ceph-mon[79930]: from='client.51948 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:58:54.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:58:54 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4264100844' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:58:54.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:58:54 smithi083 ceph-mon[77242]: pgmap v15317: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:54.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:58:54 smithi083 ceph-mon[77242]: from='client.61873 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:58:54.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:58:54 smithi083 ceph-mon[77242]: from='client.51948 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:58:54.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:58:54 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4264100844' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:58:55.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:58:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:58:55.428Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:58:56.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:58:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:58:56.156Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:58:56.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:58:56 smithi153 ceph-mon[70044]: pgmap v15318: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:56.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:58:56 smithi083 ceph-mon[79930]: pgmap v15318: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:56.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:58:56 smithi083 ceph-mon[77242]: pgmap v15318: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:58.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:58:58 smithi153 ceph-mon[70044]: pgmap v15319: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:58.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:58:58 smithi083 ceph-mon[79930]: pgmap v15319: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:58.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:58:58 smithi083 ceph-mon[77242]: pgmap v15319: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:58:59.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:58:59 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:58:59.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:58:59 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:58:59.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:58:59 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:58:59.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:58:59 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:58:59.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:58:59 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:58:59.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:58:59 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:58:59.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:58:59 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T13:58:59.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:58:59 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T13:58:59.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:58:59 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T13:59:00.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:59:00 smithi153 ceph-mon[70044]: pgmap v15320: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:59:00.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:59:00 smithi083 ceph-mon[79930]: pgmap v15320: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:59:00.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:59:00 smithi083 ceph-mon[77242]: pgmap v15320: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:59:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:59:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:59:01] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T13:59:02.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:59:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:59:02.071Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:59:02.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:59:02 smithi153 ceph-mon[70044]: pgmap v15321: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:59:02.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:59:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:59:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:59:02.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:59:02 smithi083 ceph-mon[77242]: pgmap v15321: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:59:02.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:59:02 smithi083 ceph-mon[79930]: pgmap v15321: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:59:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:59:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:59:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:59:04.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:59:04 smithi083 ceph-mon[79930]: pgmap v15322: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:59:04.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:59:04 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:59:04.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:59:04 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:59:04.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:59:04 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:59:04.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:59:04 smithi083 ceph-mon[77242]: pgmap v15322: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:59:04.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:59:04 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:59:04.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:59:04 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:59:04.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:59:04 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:59:04.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:59:04 smithi153 ceph-mon[70044]: pgmap v15322: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:59:04.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:59:04 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:59:04.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:59:04 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:59:04.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:59:04 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T13:59:05.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:59:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:59:05.429Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:59:06.428 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:59:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:59:06.157Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:59:06.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:59:06 smithi083 ceph-mon[77242]: pgmap v15323: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:59:06.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:59:06 smithi083 ceph-mon[79930]: pgmap v15323: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:59:06.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:59:06 smithi153 ceph-mon[70044]: pgmap v15323: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:59:08.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:59:08 smithi083 ceph-mon[77242]: pgmap v15324: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:59:08.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:59:08 smithi083 ceph-mon[79930]: pgmap v15324: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:59:08.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:59:08 smithi153 ceph-mon[70044]: pgmap v15324: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:59:10.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:59:10 smithi083 ceph-mon[77242]: pgmap v15325: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:59:10.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:59:10 smithi083 ceph-mon[79930]: pgmap v15325: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:59:10.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:59:10 smithi153 ceph-mon[70044]: pgmap v15325: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:59:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:59:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:59:11] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T13:59:12.447 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:59:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:59:12.070Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:59:12.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:59:12 smithi083 ceph-mon[79930]: pgmap v15326: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:59:12.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:59:12 smithi083 ceph-mon[77242]: pgmap v15326: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:59:12.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:59:12 smithi153 ceph-mon[70044]: pgmap v15326: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:59:12.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:59:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:59:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:59:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:59:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:59:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:59:14.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:59:14 smithi083 ceph-mon[79930]: pgmap v15327: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:59:14.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:59:14 smithi083 ceph-mon[77242]: pgmap v15327: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:59:14.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:59:14 smithi153 ceph-mon[70044]: pgmap v15327: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:59:15.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:59:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:59:15.430Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:59:16.453 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:59:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:59:16.157Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:59:16.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:59:16 smithi083 ceph-mon[77242]: pgmap v15328: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:59:16.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:59:16 smithi083 ceph-mon[79930]: pgmap v15328: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:59:16.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:59:16 smithi153 ceph-mon[70044]: pgmap v15328: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:59:18.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:59:18 smithi083 ceph-mon[79930]: pgmap v15329: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:59:18.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:59:18 smithi083 ceph-mon[77242]: pgmap v15329: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:59:18.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:59:18 smithi153 ceph-mon[70044]: pgmap v15329: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:59:20.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:59:20 smithi083 ceph-mon[79930]: pgmap v15330: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:59:20.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:59:20 smithi083 ceph-mon[77242]: pgmap v15330: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:59:20.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:59:20 smithi153 ceph-mon[70044]: pgmap v15330: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:59:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:59:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:59:21] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T13:59:22.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:59:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:59:22.071Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:59:22.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:59:22 smithi083 ceph-mon[79930]: pgmap v15331: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:59:22.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:59:22 smithi083 ceph-mon[77242]: pgmap v15331: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:59:22.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:59:22 smithi153 ceph-mon[70044]: pgmap v15331: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:59:22.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:59:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:59:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:59:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:59:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:59:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:59:24.104 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:59:24.392 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:59:24.393 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 2m ago 8h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:59:24.393 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 20s ago 8h 64.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:59:24.393 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 2m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:59:24.393 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 20s ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:59:24.393 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 20s ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:59:24.394 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 2m ago 8h 935M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:59:24.394 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 2m ago 8h 1879M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:59:24.394 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 20s ago 8h 816M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:59:24.394 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 2m ago 8h 1121M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:59:24.395 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 2m ago 8h 24.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:59:24.395 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 20s ago 8h 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:59:24.395 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 2m ago 8h 1359M 2277M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:59:24.395 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 2m ago 8h 1347M 2277M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:59:24.395 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 2m ago 8h 1048M 2277M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:59:24.396 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 2m ago 8h 1319M 2277M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:59:24.396 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 20s ago 8h 1158M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:59:24.396 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 20s ago 8h 1452M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:59:24.396 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 20s ago 8h 1298M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:59:24.397 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 20s ago 8h 1542M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:59:24.397 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 20s ago 8h 186M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:59:24.752 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:59:24.753 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:59:24.753 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:59:24.753 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:59:24.753 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:59:24.754 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:59:24.754 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:59:24.754 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:59:24.754 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:59:24.755 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:59:24.755 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:59:24.755 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:59:24.755 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:59:24.756 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:59:24.756 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:59:24.756 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:59:24.756 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:59:24.756 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:59:24.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:59:24 smithi153 ceph-mon[70044]: pgmap v15332: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:59:24.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:59:24 smithi083 ceph-mon[77242]: pgmap v15332: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:59:24.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:59:24 smithi083 ceph-mon[79930]: pgmap v15332: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:59:25.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:59:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:59:25.430Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:59:25.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:59:25 smithi083 ceph-mon[77242]: from='client.61891 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:59:25.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:59:25 smithi083 ceph-mon[77242]: from='client.51966 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:59:25.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:59:25 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1283359231' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:59:25.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:59:25 smithi083 ceph-mon[79930]: from='client.61891 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:59:25.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:59:25 smithi083 ceph-mon[79930]: from='client.51966 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:59:25.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:59:25 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1283359231' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:59:25.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:59:25 smithi153 ceph-mon[70044]: from='client.61891 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:59:25.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:59:25 smithi153 ceph-mon[70044]: from='client.51966 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:59:25.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:59:25 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1283359231' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:59:26.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:59:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:59:26.159Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:59:26.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:59:26 smithi153 ceph-mon[70044]: pgmap v15333: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:59:26.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:59:26 smithi083 ceph-mon[79930]: pgmap v15333: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:59:26.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:59:26 smithi083 ceph-mon[77242]: pgmap v15333: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:59:28.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:59:28 smithi153 ceph-mon[70044]: pgmap v15334: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:59:28.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:59:28 smithi083 ceph-mon[79930]: pgmap v15334: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:59:28.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:59:28 smithi083 ceph-mon[77242]: pgmap v15334: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:59:30.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:59:30 smithi153 ceph-mon[70044]: pgmap v15335: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:59:30.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:59:30 smithi083 ceph-mon[79930]: pgmap v15335: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:59:30.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:59:30 smithi083 ceph-mon[77242]: pgmap v15335: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:59:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:59:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:59:31] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T13:59:32.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:59:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:59:32.071Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:59:32.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:59:32 smithi153 ceph-mon[70044]: pgmap v15336: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:59:32.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:59:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:59:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:59:32.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:59:32 smithi083 ceph-mon[79930]: pgmap v15336: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:59:32.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:59:32 smithi083 ceph-mon[77242]: pgmap v15336: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:59:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:59:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:59:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:59:34.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:59:34 smithi153 ceph-mon[70044]: pgmap v15337: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:59:34.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:59:34 smithi083 ceph-mon[79930]: pgmap v15337: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:59:34.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:59:34 smithi083 ceph-mon[77242]: pgmap v15337: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:59:35.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:59:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:59:35.432Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:59:36.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:59:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:59:36.159Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:59:36.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:59:36 smithi153 ceph-mon[70044]: pgmap v15338: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:59:36.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:59:36 smithi083 ceph-mon[79930]: pgmap v15338: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:59:36.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:59:36 smithi083 ceph-mon[77242]: pgmap v15338: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:59:38.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:59:38 smithi153 ceph-mon[70044]: pgmap v15339: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:59:38.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:59:38 smithi083 ceph-mon[79930]: pgmap v15339: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:59:38.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:59:38 smithi083 ceph-mon[77242]: pgmap v15339: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:59:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:59:40 smithi153 ceph-mon[70044]: pgmap v15340: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:59:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:59:40 smithi083 ceph-mon[79930]: pgmap v15340: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:59:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:59:40 smithi083 ceph-mon[77242]: pgmap v15340: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:59:41.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:59:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:59:41.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:59:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:59:41.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:59:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:59:41.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:59:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:59:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:59:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:59:41] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T13:59:41.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:59:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:59:41.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:59:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:59:41.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:59:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:59:41.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:59:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:59:41.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:59:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:59:41.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:59:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:59:41.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:59:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T13:59:41.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:59:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T13:59:42.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:59:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:59:42.072Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:59:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:59:42 smithi153 ceph-mon[70044]: pgmap v15341: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:59:42.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:59:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:59:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:59:42.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:59:42 smithi083 ceph-mon[77242]: pgmap v15341: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:59:42.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:59:42 smithi083 ceph-mon[79930]: pgmap v15341: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:59:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:59:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:59:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:59:44.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:59:44 smithi153 ceph-mon[70044]: pgmap v15342: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:59:44.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:59:44 smithi083 ceph-mon[79930]: pgmap v15342: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:59:44.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:59:44 smithi083 ceph-mon[77242]: pgmap v15342: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:59:45.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:59:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:59:45.432Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:59:46.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:59:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:59:46.159Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:59:46.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:59:46 smithi153 ceph-mon[70044]: pgmap v15343: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:59:46.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:59:46 smithi083 ceph-mon[79930]: pgmap v15343: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:59:46.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:59:46 smithi083 ceph-mon[77242]: pgmap v15343: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:59:48.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:59:48 smithi153 ceph-mon[70044]: pgmap v15344: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:59:48.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:59:48 smithi083 ceph-mon[79930]: pgmap v15344: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:59:48.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:59:48 smithi083 ceph-mon[77242]: pgmap v15344: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:59:50.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:59:50 smithi153 ceph-mon[70044]: pgmap v15345: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:59:50.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:59:50 smithi083 ceph-mon[79930]: pgmap v15345: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:59:50.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:59:50 smithi083 ceph-mon[77242]: pgmap v15345: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:59:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 13:59:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:59:51] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T13:59:52.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:59:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:59:52.073Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:59:52.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:59:52 smithi153 ceph-mon[70044]: pgmap v15346: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:59:52.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 13:59:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T13:59:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T13:59:52.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:59:52 smithi083 ceph-mon[79930]: pgmap v15346: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:59:52.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:59:52 smithi083 ceph-mon[77242]: pgmap v15346: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:59:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 13:59:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:13:59:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T13:59:54.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:59:54 smithi153 ceph-mon[70044]: pgmap v15347: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:59:54.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:59:54 smithi083 ceph-mon[79930]: pgmap v15347: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:59:54.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:59:54 smithi083 ceph-mon[77242]: pgmap v15347: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:59:55.083 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T13:59:55.372 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T13:59:55.372 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 2m ago 8h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T13:59:55.373 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 51s ago 8h 64.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T13:59:55.373 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 2m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T13:59:55.373 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 51s ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T13:59:55.373 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 51s ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T13:59:55.374 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 2m ago 8h 935M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T13:59:55.374 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 2m ago 8h 1879M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T13:59:55.374 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 51s ago 8h 816M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T13:59:55.374 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 2m ago 8h 1121M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T13:59:55.375 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 2m ago 8h 24.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T13:59:55.375 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 51s ago 8h 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T13:59:55.375 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 2m ago 8h 1359M 2277M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T13:59:55.375 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 2m ago 8h 1347M 2277M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T13:59:55.375 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 2m ago 8h 1048M 2277M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T13:59:55.376 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 2m ago 8h 1319M 2277M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T13:59:55.376 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 51s ago 8h 1158M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T13:59:55.376 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 51s ago 8h 1452M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T13:59:55.376 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 51s ago 8h 1298M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T13:59:55.377 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 51s ago 8h 1542M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T13:59:55.377 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 51s ago 8h 186M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T13:59:55.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:59:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:59:55.432Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:59:55.731 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T13:59:55.731 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T13:59:55.731 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T13:59:55.732 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:59:55.732 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T13:59:55.732 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T13:59:55.732 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:59:55.733 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T13:59:55.733 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T13:59:55.733 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:59:55.733 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T13:59:55.734 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T13:59:55.734 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T13:59:55.734 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T13:59:55.734 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T13:59:55.735 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T13:59:55.735 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T13:59:55.735 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T13:59:56.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 13:59:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T13:59:56.160Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T13:59:56.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:59:56 smithi153 ceph-mon[70044]: pgmap v15348: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:59:56.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:59:56 smithi153 ceph-mon[70044]: from='client.51978 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:59:56.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:59:56 smithi153 ceph-mon[70044]: from='client.51984 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:59:56.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:59:56 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2704129036' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:59:56.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:59:56 smithi083 ceph-mon[79930]: pgmap v15348: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:59:56.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:59:56 smithi083 ceph-mon[79930]: from='client.51978 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:59:56.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:59:56 smithi083 ceph-mon[79930]: from='client.51984 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:59:56.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:59:56 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2704129036' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:59:56.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:59:56 smithi083 ceph-mon[77242]: pgmap v15348: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:59:56.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:59:56 smithi083 ceph-mon[77242]: from='client.51978 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:59:56.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:59:56 smithi083 ceph-mon[77242]: from='client.51984 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T13:59:56.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:59:56 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2704129036' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T13:59:58.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 13:59:58 smithi153 ceph-mon[70044]: pgmap v15349: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:59:58.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 13:59:58 smithi083 ceph-mon[77242]: pgmap v15349: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T13:59:58.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 13:59:58 smithi083 ceph-mon[79930]: pgmap v15349: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:00.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:00:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[77219]: 2022-09-29T13:59:59.999+0000 7f160f665700 -1 log_channel(cluster) log [ERR] : overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T14:00:00.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:00:00 smithi153 ceph-mon[70044]: pgmap v15350: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:00.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:00:00 smithi153 ceph-mon[70044]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T14:00:00.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:00:00 smithi083 ceph-mon[77242]: pgmap v15350: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:00.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:00:00 smithi083 ceph-mon[77242]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T14:00:00.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:00:00 smithi083 ceph-mon[79930]: pgmap v15350: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:00.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:00:00 smithi083 ceph-mon[79930]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T14:00:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:00:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:00:01] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T14:00:02.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:00:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:00:02.074Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:00:02.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:00:02 smithi153 ceph-mon[70044]: pgmap v15351: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:02.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:00:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:00:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:00:02.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:00:02 smithi083 ceph-mon[79930]: pgmap v15351: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:02.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:00:02 smithi083 ceph-mon[77242]: pgmap v15351: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:00:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:00:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:00:04.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:00:04 smithi153 ceph-mon[70044]: pgmap v15352: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:04.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:00:04 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:00:04.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:00:04 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:00:04.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:00:04 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:00:04.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:00:04 smithi083 ceph-mon[79930]: pgmap v15352: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:04.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:00:04 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:00:04.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:00:04 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:00:04.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:00:04 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:00:04.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:00:04 smithi083 ceph-mon[77242]: pgmap v15352: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:04.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:00:04 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:00:04.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:00:04 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:00:04.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:00:04 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:00:05.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:00:05 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:00:05.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:00:05 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:00:05.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:00:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:00:05.434Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:00:06.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:00:05 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:00:06.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:00:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:00:06.161Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:00:06.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:00:06 smithi083 ceph-mon[79930]: pgmap v15353: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:06.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:00:06 smithi083 ceph-mon[77242]: pgmap v15353: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:07.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:00:06 smithi153 ceph-mon[70044]: pgmap v15353: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:08.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:00:08 smithi083 ceph-mon[79930]: pgmap v15354: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:08.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:00:08 smithi083 ceph-mon[77242]: pgmap v15354: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:09.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:00:08 smithi153 ceph-mon[70044]: pgmap v15354: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:10.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:00:10 smithi083 ceph-mon[79930]: pgmap v15355: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:10.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:00:10 smithi083 ceph-mon[77242]: pgmap v15355: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:11.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:00:10 smithi153 ceph-mon[70044]: pgmap v15355: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:00:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:00:11] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T14:00:12.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:00:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:00:12.074Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:00:12.648 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:00:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:00:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:00:12.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:00:12 smithi083 ceph-mon[79930]: pgmap v15356: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:12.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:00:12 smithi083 ceph-mon[77242]: pgmap v15356: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:13.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:00:12 smithi153 ceph-mon[70044]: pgmap v15356: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:00:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:00:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:00:14.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:00:14 smithi083 ceph-mon[79930]: pgmap v15357: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:14.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:00:14 smithi083 ceph-mon[77242]: pgmap v15357: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:15.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:00:14 smithi153 ceph-mon[70044]: pgmap v15357: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:15.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:00:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:00:15.435Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:00:16.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:00:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:00:16.162Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:00:16.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:00:16 smithi083 ceph-mon[79930]: pgmap v15358: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:16.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:00:16 smithi083 ceph-mon[77242]: pgmap v15358: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:17.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:00:16 smithi153 ceph-mon[70044]: pgmap v15358: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:18.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:00:18 smithi083 ceph-mon[79930]: pgmap v15359: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:18.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:00:18 smithi083 ceph-mon[77242]: pgmap v15359: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:19.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:00:18 smithi153 ceph-mon[70044]: pgmap v15359: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:20.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:00:20 smithi083 ceph-mon[79930]: pgmap v15360: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:20.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:00:20 smithi083 ceph-mon[77242]: pgmap v15360: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:21.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:00:20 smithi153 ceph-mon[70044]: pgmap v15360: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:00:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:00:21] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T14:00:22.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:00:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:00:22.075Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:00:22.665 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:00:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:00:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:00:22.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:00:22 smithi083 ceph-mon[79930]: pgmap v15361: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:22.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:00:22 smithi083 ceph-mon[77242]: pgmap v15361: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:23.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:00:22 smithi153 ceph-mon[70044]: pgmap v15361: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:00:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:00:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:00:24.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:00:24 smithi083 ceph-mon[79930]: pgmap v15362: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:24.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:00:24 smithi083 ceph-mon[77242]: pgmap v15362: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:25.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:00:24 smithi153 ceph-mon[70044]: pgmap v15362: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:25.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:00:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:00:25.436Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:00:26.055 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:00:26.347 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:00:26.347 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 3m ago 8h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:00:26.347 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 82s ago 8h 64.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:00:26.348 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 3m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:00:26.348 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 82s ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:00:26.348 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 82s ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:00:26.348 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 3m ago 8h 935M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:00:26.349 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 3m ago 8h 1879M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:00:26.349 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 82s ago 8h 816M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:00:26.349 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 3m ago 8h 1121M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:00:26.349 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 3m ago 8h 24.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:00:26.349 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 82s ago 8h 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:00:26.350 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 3m ago 8h 1359M 2277M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:00:26.350 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 3m ago 8h 1347M 2277M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:00:26.350 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 3m ago 8h 1048M 2277M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:00:26.350 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 3m ago 8h 1319M 2277M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:00:26.351 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 82s ago 8h 1158M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:00:26.351 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 82s ago 8h 1452M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:00:26.351 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 82s ago 8h 1298M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:00:26.351 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 82s ago 8h 1542M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:00:26.352 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 82s ago 8h 186M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:00:26.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:00:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:00:26.162Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:00:26.701 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:00:26.702 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:00:26.702 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:00:26.703 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:00:26.703 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:00:26.703 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:00:26.703 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:00:26.703 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:00:26.704 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:00:26.704 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:00:26.704 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:00:26.704 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:00:26.705 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:00:26.705 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:00:26.705 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:00:26.705 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:00:26.706 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:00:26.706 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:00:26.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:00:26 smithi083 ceph-mon[79930]: pgmap v15363: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:26.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:00:26 smithi083 ceph-mon[77242]: pgmap v15363: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:27.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:00:26 smithi153 ceph-mon[70044]: pgmap v15363: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:27.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:00:27 smithi083 ceph-mon[79930]: from='client.61927 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:00:27.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:00:27 smithi083 ceph-mon[79930]: from='client.52002 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:00:27.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:00:27 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/700010705' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:00:27.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:00:27 smithi083 ceph-mon[77242]: from='client.61927 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:00:27.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:00:27 smithi083 ceph-mon[77242]: from='client.52002 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:00:27.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:00:27 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/700010705' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:00:28.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:00:27 smithi153 ceph-mon[70044]: from='client.61927 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:00:28.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:00:27 smithi153 ceph-mon[70044]: from='client.52002 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:00:28.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:00:27 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/700010705' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:00:28.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:00:28 smithi083 ceph-mon[79930]: pgmap v15364: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:28.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:00:28 smithi083 ceph-mon[77242]: pgmap v15364: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:29.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:00:28 smithi153 ceph-mon[70044]: pgmap v15364: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:30.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:00:30 smithi083 ceph-mon[79930]: pgmap v15365: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:30.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:00:30 smithi083 ceph-mon[77242]: pgmap v15365: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:31.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:00:30 smithi153 ceph-mon[70044]: pgmap v15365: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:00:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:00:31] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T14:00:32.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:00:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:00:32.076Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:00:32.684 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:00:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:00:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:00:32.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:00:32 smithi083 ceph-mon[79930]: pgmap v15366: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:32.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:00:32 smithi083 ceph-mon[77242]: pgmap v15366: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:33.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:00:32 smithi153 ceph-mon[70044]: pgmap v15366: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:00:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:00:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:00:34.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:00:34 smithi083 ceph-mon[79930]: pgmap v15367: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:34.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:00:34 smithi083 ceph-mon[77242]: pgmap v15367: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:35.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:00:34 smithi153 ceph-mon[70044]: pgmap v15367: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:35.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:00:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:00:35.436Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:00:36.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:00:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:00:36.164Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:00:36.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:00:36 smithi083 ceph-mon[77242]: pgmap v15368: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:36.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:00:36 smithi083 ceph-mon[79930]: pgmap v15368: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:37.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:00:36 smithi153 ceph-mon[70044]: pgmap v15368: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:38.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:00:38 smithi083 ceph-mon[79930]: pgmap v15369: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:38.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:00:38 smithi083 ceph-mon[77242]: pgmap v15369: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:39.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:00:38 smithi153 ceph-mon[70044]: pgmap v15369: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:00:40 smithi083 ceph-mon[79930]: pgmap v15370: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:00:40 smithi083 ceph-mon[77242]: pgmap v15370: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:00:40 smithi153 ceph-mon[70044]: pgmap v15370: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:00:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:00:41] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T14:00:41.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:00:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:00:41.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:00:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:00:41.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:00:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:00:41.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:00:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:00:41.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:00:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:00:41.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:00:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:00:41.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:00:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:00:41.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:00:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:00:42.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:00:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:00:42.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:00:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:00:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:00:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:00:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:00:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:00:42.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:00:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:00:42.076Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:00:42.707 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:00:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:00:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:00:42.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:00:42 smithi083 ceph-mon[79930]: pgmap v15371: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:42.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:00:42 smithi083 ceph-mon[77242]: pgmap v15371: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:43.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:00:42 smithi153 ceph-mon[70044]: pgmap v15371: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:00:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:00:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:00:44.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:00:44 smithi083 ceph-mon[79930]: pgmap v15372: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:44.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:00:44 smithi083 ceph-mon[77242]: pgmap v15372: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:45.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:00:44 smithi153 ceph-mon[70044]: pgmap v15372: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:45.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:00:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:00:45.437Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:00:46.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:00:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:00:46.164Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:00:47.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:00:46 smithi153 ceph-mon[70044]: pgmap v15373: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:47.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:00:46 smithi083 ceph-mon[79930]: pgmap v15373: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:47.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:00:46 smithi083 ceph-mon[77242]: pgmap v15373: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:49.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:00:48 smithi153 ceph-mon[70044]: pgmap v15374: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:49.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:00:48 smithi083 ceph-mon[79930]: pgmap v15374: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:49.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:00:48 smithi083 ceph-mon[77242]: pgmap v15374: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:51.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:00:50 smithi153 ceph-mon[70044]: pgmap v15375: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:51.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:00:50 smithi083 ceph-mon[79930]: pgmap v15375: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:51.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:00:50 smithi083 ceph-mon[77242]: pgmap v15375: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:00:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:00:51] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T14:00:52.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:00:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:00:52.076Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:00:52.727 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:00:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:00:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:00:53.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:00:52 smithi153 ceph-mon[70044]: pgmap v15376: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:53.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:00:52 smithi083 ceph-mon[79930]: pgmap v15376: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:53.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:00:52 smithi083 ceph-mon[77242]: pgmap v15376: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:00:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:00:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:00:55.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:00:54 smithi153 ceph-mon[70044]: pgmap v15377: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:55.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:00:54 smithi083 ceph-mon[79930]: pgmap v15377: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:55.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:00:54 smithi083 ceph-mon[77242]: pgmap v15377: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:55.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:00:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:00:55.437Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:00:56.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:00:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:00:56.165Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:00:57.027 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:00:57.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:00:56 smithi153 ceph-mon[70044]: pgmap v15378: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:57.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:00:56 smithi083 ceph-mon[79930]: pgmap v15378: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:57.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:00:56 smithi083 ceph-mon[77242]: pgmap v15378: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:57.316 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:00:57.316 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 3m ago 8h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:00:57.316 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 113s ago 8h 64.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:00:57.316 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 3m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:00:57.317 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 113s ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:00:57.317 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 113s ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:00:57.317 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 3m ago 8h 935M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:00:57.317 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 3m ago 8h 1879M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:00:57.318 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 113s ago 8h 816M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:00:57.318 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 3m ago 8h 1121M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:00:57.318 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 3m ago 8h 24.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:00:57.318 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 113s ago 8h 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:00:57.319 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 3m ago 8h 1359M 2277M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:00:57.319 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 3m ago 8h 1347M 2277M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:00:57.319 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 3m ago 8h 1048M 2277M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:00:57.319 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 3m ago 8h 1319M 2277M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:00:57.319 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 113s ago 8h 1158M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:00:57.320 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 113s ago 8h 1452M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:00:57.320 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 113s ago 8h 1298M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:00:57.320 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 113s ago 8h 1542M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:00:57.320 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 113s ago 8h 186M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:00:57.668 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:00:57.669 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:00:57.669 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:00:57.669 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:00:57.669 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:00:57.670 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:00:57.670 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:00:57.670 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:00:57.670 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:00:57.671 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:00:57.671 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:00:57.671 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:00:57.671 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:00:57.671 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:00:57.672 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:00:57.672 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:00:57.672 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:00:57.673 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:00:58.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:00:57 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2029341321' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:00:58.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:00:57 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2029341321' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:00:58.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:00:57 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2029341321' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:00:59.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:00:58 smithi153 ceph-mon[70044]: from='client.61945 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:00:59.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:00:58 smithi153 ceph-mon[70044]: pgmap v15379: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:59.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:00:58 smithi153 ceph-mon[70044]: from='client.52020 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:00:59.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:00:58 smithi083 ceph-mon[79930]: from='client.61945 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:00:59.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:00:58 smithi083 ceph-mon[79930]: pgmap v15379: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:59.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:00:58 smithi083 ceph-mon[79930]: from='client.52020 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:00:59.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:00:58 smithi083 ceph-mon[77242]: from='client.61945 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:00:59.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:00:58 smithi083 ceph-mon[77242]: pgmap v15379: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:00:59.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:00:58 smithi083 ceph-mon[77242]: from='client.52020 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:01:01.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:01:00 smithi153 ceph-mon[70044]: pgmap v15380: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:01.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:01:00 smithi083 ceph-mon[79930]: pgmap v15380: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:01.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:01:00 smithi083 ceph-mon[77242]: pgmap v15380: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:01.772 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:01:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:01:01] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T14:01:02.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:01:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:01:02.077Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:01:02.745 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:01:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:01:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:01:03.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:01:02 smithi153 ceph-mon[70044]: pgmap v15381: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:03.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:01:02 smithi083 ceph-mon[79930]: pgmap v15381: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:03.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:01:02 smithi083 ceph-mon[77242]: pgmap v15381: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:01:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:01:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:01:05.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:01:04 smithi153 ceph-mon[70044]: pgmap v15382: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:05.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:01:04 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:01:05.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:01:04 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:01:05.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:01:04 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:01:05.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:01:04 smithi083 ceph-mon[79930]: pgmap v15382: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:05.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:01:04 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:01:05.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:01:04 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:01:05.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:01:04 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:01:05.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:01:04 smithi083 ceph-mon[77242]: pgmap v15382: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:05.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:01:04 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:01:05.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:01:04 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:01:05.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:01:04 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:01:05.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:01:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:01:05.438Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:01:06.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:01:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:01:06.166Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:01:06.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:01:06 smithi083 ceph-mon[77242]: pgmap v15383: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:06.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:01:06 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:01:06.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:01:06 smithi083 ceph-mon[79930]: pgmap v15383: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:06.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:01:06 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:01:06.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:01:06 smithi153 ceph-mon[70044]: pgmap v15383: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:06.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:01:06 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:01:08.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:01:08 smithi153 ceph-mon[70044]: pgmap v15384: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:08.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:01:08 smithi083 ceph-mon[77242]: pgmap v15384: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:08.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:01:08 smithi083 ceph-mon[79930]: pgmap v15384: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:10.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:01:10 smithi153 ceph-mon[70044]: pgmap v15385: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:10.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:01:10 smithi083 ceph-mon[77242]: pgmap v15385: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:10.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:01:10 smithi083 ceph-mon[79930]: pgmap v15385: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:01:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:01:11] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T14:01:12.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:01:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:01:12.077Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:01:12.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:01:12 smithi153 ceph-mon[70044]: pgmap v15386: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:12.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:01:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:01:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:01:12.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:01:12 smithi083 ceph-mon[79930]: pgmap v15386: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:12.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:01:12 smithi083 ceph-mon[77242]: pgmap v15386: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:01:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:01:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:01:14.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:01:14 smithi153 ceph-mon[70044]: pgmap v15387: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:14.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:01:14 smithi083 ceph-mon[79930]: pgmap v15387: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:14.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:01:14 smithi083 ceph-mon[77242]: pgmap v15387: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:15.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:01:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:01:15.440Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:01:16.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:01:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:01:16.166Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:01:16.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:01:16 smithi153 ceph-mon[70044]: pgmap v15388: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:16.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:01:16 smithi083 ceph-mon[79930]: pgmap v15388: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:16.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:01:16 smithi083 ceph-mon[77242]: pgmap v15388: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:18.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:01:18 smithi153 ceph-mon[70044]: pgmap v15389: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:18.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:01:18 smithi083 ceph-mon[77242]: pgmap v15389: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:18.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:01:18 smithi083 ceph-mon[79930]: pgmap v15389: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:20.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:01:20 smithi153 ceph-mon[70044]: pgmap v15390: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:20.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:01:20 smithi083 ceph-mon[77242]: pgmap v15390: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:20.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:01:20 smithi083 ceph-mon[79930]: pgmap v15390: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:01:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:01:21] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T14:01:22.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:01:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:01:22.078Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:01:22.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:01:22 smithi153 ceph-mon[70044]: pgmap v15391: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:22.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:01:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:01:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:01:22.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:01:22 smithi083 ceph-mon[77242]: pgmap v15391: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:22.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:01:22 smithi083 ceph-mon[79930]: pgmap v15391: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:01:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:01:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:01:24.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:01:24 smithi153 ceph-mon[70044]: pgmap v15392: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:24.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:01:24 smithi083 ceph-mon[77242]: pgmap v15392: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:24.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:01:24 smithi083 ceph-mon[79930]: pgmap v15392: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:25.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:01:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:01:25.440Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:01:26.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:01:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:01:26.167Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:01:26.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:01:26 smithi153 ceph-mon[70044]: pgmap v15393: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:26.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:01:26 smithi083 ceph-mon[79930]: pgmap v15393: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:26.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:01:26 smithi083 ceph-mon[77242]: pgmap v15393: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:27.997 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:01:28.284 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:01:28.285 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 4m ago 8h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:01:28.285 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 2m ago 8h 64.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:01:28.285 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 4m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:01:28.285 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 2m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:01:28.286 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 2m ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:01:28.286 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 4m ago 8h 935M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:01:28.286 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 4m ago 8h 1879M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:01:28.286 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 2m ago 8h 816M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:01:28.287 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 4m ago 8h 1121M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:01:28.287 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 4m ago 8h 24.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:01:28.287 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 2m ago 8h 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:01:28.287 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 4m ago 8h 1359M 2277M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:01:28.287 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 4m ago 8h 1347M 2277M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:01:28.288 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 4m ago 8h 1048M 2277M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:01:28.288 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 4m ago 8h 1319M 2277M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:01:28.288 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 2m ago 8h 1158M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:01:28.288 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 2m ago 8h 1452M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:01:28.289 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 2m ago 8h 1298M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:01:28.289 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 2m ago 8h 1542M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:01:28.289 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 2m ago 8h 186M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:01:28.642 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:01:28.642 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:01:28.643 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:01:28.643 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:01:28.643 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:01:28.643 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:01:28.644 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:01:28.644 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:01:28.644 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:01:28.644 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:01:28.645 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:01:28.645 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:01:28.645 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:01:28.645 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:01:28.645 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:01:28.646 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:01:28.646 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:01:28.646 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:01:28.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:01:28 smithi153 ceph-mon[70044]: pgmap v15394: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:28.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:01:28 smithi083 ceph-mon[79930]: pgmap v15394: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:28.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:01:28 smithi083 ceph-mon[77242]: pgmap v15394: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:29.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:01:29 smithi153 ceph-mon[70044]: from='client.52032 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:01:29.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:01:29 smithi153 ceph-mon[70044]: from='client.61969 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:01:29.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:01:29 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1237774494' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:01:29.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:01:29 smithi083 ceph-mon[79930]: from='client.52032 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:01:29.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:01:29 smithi083 ceph-mon[79930]: from='client.61969 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:01:29.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:01:29 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1237774494' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:01:29.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:01:29 smithi083 ceph-mon[77242]: from='client.52032 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:01:29.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:01:29 smithi083 ceph-mon[77242]: from='client.61969 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:01:29.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:01:29 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1237774494' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:01:30.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:01:30 smithi153 ceph-mon[70044]: pgmap v15395: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:30.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:01:30 smithi083 ceph-mon[79930]: pgmap v15395: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:30.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:01:30 smithi083 ceph-mon[77242]: pgmap v15395: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:01:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:01:31] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T14:01:32.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:01:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:01:32.079Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:01:32.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:01:32 smithi153 ceph-mon[70044]: pgmap v15396: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:32.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:01:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:01:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:01:32.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:01:32 smithi083 ceph-mon[77242]: pgmap v15396: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:32.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:01:32 smithi083 ceph-mon[79930]: pgmap v15396: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:01:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:01:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:01:34.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:01:34 smithi153 ceph-mon[70044]: pgmap v15397: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:34.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:01:34 smithi083 ceph-mon[79930]: pgmap v15397: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:34.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:01:34 smithi083 ceph-mon[77242]: pgmap v15397: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:35.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:01:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:01:35.441Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:01:36.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:01:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:01:36.169Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:01:36.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:01:36 smithi153 ceph-mon[70044]: pgmap v15398: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:36.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:01:36 smithi083 ceph-mon[79930]: pgmap v15398: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:36.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:01:36 smithi083 ceph-mon[77242]: pgmap v15398: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:38.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:01:38 smithi153 ceph-mon[70044]: pgmap v15399: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:38.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:01:38 smithi083 ceph-mon[77242]: pgmap v15399: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:38.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:01:38 smithi083 ceph-mon[79930]: pgmap v15399: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:01:40 smithi153 ceph-mon[70044]: pgmap v15400: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:01:40 smithi083 ceph-mon[79930]: pgmap v15400: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:01:40 smithi083 ceph-mon[77242]: pgmap v15400: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:41.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:01:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:01:41.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:01:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:01:41.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:01:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:01:41.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:01:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:01:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:01:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:01:41] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T14:01:41.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:01:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:01:41.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:01:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:01:41.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:01:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:01:41.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:01:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:01:41.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:01:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:01:41.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:01:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:01:41.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:01:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:01:41.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:01:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:01:42.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:01:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:01:42.080Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:01:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:01:42 smithi153 ceph-mon[70044]: pgmap v15401: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:42.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:01:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:01:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:01:42.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:01:42 smithi083 ceph-mon[79930]: pgmap v15401: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:42.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:01:42 smithi083 ceph-mon[77242]: pgmap v15401: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:01:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:01:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:01:44.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:01:44 smithi153 ceph-mon[70044]: pgmap v15402: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:44.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:01:44 smithi083 ceph-mon[79930]: pgmap v15402: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:44.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:01:44 smithi083 ceph-mon[77242]: pgmap v15402: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:45.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:01:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:01:45.442Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:01:46.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:01:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:01:46.169Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:01:46.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:01:46 smithi153 ceph-mon[70044]: pgmap v15403: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:46.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:01:46 smithi083 ceph-mon[79930]: pgmap v15403: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:46.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:01:46 smithi083 ceph-mon[77242]: pgmap v15403: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:48.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:01:48 smithi153 ceph-mon[70044]: pgmap v15404: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:48.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:01:48 smithi083 ceph-mon[79930]: pgmap v15404: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:48.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:01:48 smithi083 ceph-mon[77242]: pgmap v15404: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:50.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:01:50 smithi153 ceph-mon[70044]: pgmap v15405: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:50.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:01:50 smithi083 ceph-mon[79930]: pgmap v15405: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:50.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:01:50 smithi083 ceph-mon[77242]: pgmap v15405: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:01:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:01:51] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T14:01:52.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:01:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:01:52.080Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:01:52.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:01:52 smithi153 ceph-mon[70044]: pgmap v15406: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:52.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:01:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:01:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:01:52.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:01:52 smithi083 ceph-mon[79930]: pgmap v15406: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:52.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:01:52 smithi083 ceph-mon[77242]: pgmap v15406: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:01:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:01:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:01:54.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:01:54 smithi153 ceph-mon[70044]: pgmap v15407: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:54.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:01:54 smithi083 ceph-mon[77242]: pgmap v15407: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:54.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:01:54 smithi083 ceph-mon[79930]: pgmap v15407: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:55.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:01:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:01:55.441Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:01:56.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:01:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:01:56.170Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:01:56.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:01:56 smithi153 ceph-mon[70044]: pgmap v15408: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:56.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:01:56 smithi083 ceph-mon[79930]: pgmap v15408: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:56.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:01:56 smithi083 ceph-mon[77242]: pgmap v15408: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:58.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:01:58 smithi153 ceph-mon[70044]: pgmap v15409: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:58.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:01:58 smithi083 ceph-mon[77242]: pgmap v15409: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:58.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:01:58 smithi083 ceph-mon[79930]: pgmap v15409: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:01:58.968 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:01:59.260 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:01:59.261 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 5m ago 8h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:01:59.261 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 2m ago 8h 64.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:01:59.261 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 5m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:01:59.261 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 2m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:01:59.262 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 2m ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:01:59.262 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 5m ago 8h 935M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:01:59.262 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 5m ago 8h 1879M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:01:59.262 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 2m ago 8h 816M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:01:59.263 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 5m ago 8h 1121M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:01:59.263 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 5m ago 8h 24.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:01:59.263 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 2m ago 8h 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:01:59.263 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 5m ago 8h 1359M 2277M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:01:59.263 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 5m ago 8h 1347M 2277M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:01:59.264 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 5m ago 8h 1048M 2277M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:01:59.264 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 5m ago 8h 1319M 2277M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:01:59.264 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 2m ago 8h 1158M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:01:59.264 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 2m ago 8h 1452M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:01:59.265 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 2m ago 8h 1298M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:01:59.265 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 2m ago 8h 1542M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:01:59.265 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 2m ago 8h 186M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:01:59.619 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:01:59.619 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:01:59.619 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:01:59.620 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:01:59.620 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:01:59.620 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:01:59.620 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:01:59.621 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:01:59.621 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:01:59.621 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:01:59.621 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:01:59.622 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:01:59.622 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:01:59.622 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:01:59.622 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:01:59.622 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:01:59.623 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:01:59.623 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:02:00.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:02:00 smithi153 ceph-mon[70044]: from='client.52050 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:02:00.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:02:00 smithi153 ceph-mon[70044]: pgmap v15410: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:02:00.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:02:00 smithi153 ceph-mon[70044]: from='client.52056 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:02:00.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:02:00 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3722450142' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:02:00.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:02:00 smithi083 ceph-mon[79930]: from='client.52050 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:02:00.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:02:00 smithi083 ceph-mon[79930]: pgmap v15410: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:02:00.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:02:00 smithi083 ceph-mon[79930]: from='client.52056 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:02:00.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:02:00 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3722450142' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:02:00.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:02:00 smithi083 ceph-mon[77242]: from='client.52050 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:02:00.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:02:00 smithi083 ceph-mon[77242]: pgmap v15410: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:02:00.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:02:00 smithi083 ceph-mon[77242]: from='client.52056 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:02:00.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:02:00 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3722450142' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:02:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:02:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:02:01] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T14:02:02.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:02:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:02:02.082Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:02:02.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:02:02 smithi153 ceph-mon[70044]: pgmap v15411: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:02:02.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:02:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:02:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:02:02.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:02:02 smithi083 ceph-mon[79930]: pgmap v15411: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:02:02.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:02:02 smithi083 ceph-mon[77242]: pgmap v15411: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:02:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:02:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:02:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:02:04.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:02:04 smithi083 ceph-mon[79930]: pgmap v15412: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:02:04.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:02:04 smithi083 ceph-mon[77242]: pgmap v15412: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:02:05.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:02:04 smithi153 ceph-mon[70044]: pgmap v15412: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:02:05.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:02:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:02:05.441Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:02:05.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:02:05 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:02:05.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:02:05 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:02:05.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:02:05 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:02:05.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:02:05 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:02:05.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:02:05 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:02:05.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:02:05 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:02:06.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:02:05 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:02:06.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:02:05 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:02:06.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:02:05 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:02:06.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:02:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:02:06.170Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:02:06.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:02:06 smithi083 ceph-mon[77242]: pgmap v15413: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:02:06.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:02:06 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:02:06.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:02:06 smithi083 ceph-mon[79930]: pgmap v15413: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:02:06.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:02:06 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:02:07.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:02:06 smithi153 ceph-mon[70044]: pgmap v15413: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:02:07.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:02:06 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:02:08.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:02:08 smithi083 ceph-mon[79930]: pgmap v15414: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:02:08.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:02:08 smithi083 ceph-mon[77242]: pgmap v15414: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:02:09.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:02:08 smithi153 ceph-mon[70044]: pgmap v15414: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:02:10.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:02:10 smithi083 ceph-mon[77242]: pgmap v15415: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:02:10.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:02:10 smithi083 ceph-mon[79930]: pgmap v15415: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:02:11.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:02:10 smithi153 ceph-mon[70044]: pgmap v15415: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:02:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:02:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:02:11] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T14:02:12.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:02:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:02:12.082Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:02:12.579 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:02:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:02:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:02:12.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:02:12 smithi083 ceph-mon[77242]: pgmap v15416: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:02:12.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:02:12 smithi083 ceph-mon[79930]: pgmap v15416: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:02:13.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:02:12 smithi153 ceph-mon[70044]: pgmap v15416: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:02:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:02:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:02:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:02:14.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:02:14 smithi083 ceph-mon[79930]: pgmap v15417: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:02:14.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:02:14 smithi083 ceph-mon[77242]: pgmap v15417: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:02:15.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:02:14 smithi153 ceph-mon[70044]: pgmap v15417: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:02:15.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:02:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:02:15.443Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:02:16.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:02:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:02:16.171Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:02:16.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:02:16 smithi083 ceph-mon[79930]: pgmap v15418: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:02:16.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:02:16 smithi083 ceph-mon[77242]: pgmap v15418: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:02:17.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:02:16 smithi153 ceph-mon[70044]: pgmap v15418: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:02:18.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:02:18 smithi083 ceph-mon[79930]: pgmap v15419: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:02:18.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:02:18 smithi083 ceph-mon[77242]: pgmap v15419: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:02:19.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:02:18 smithi153 ceph-mon[70044]: pgmap v15419: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:02:20.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:02:20 smithi083 ceph-mon[79930]: pgmap v15420: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:02:20.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:02:20 smithi083 ceph-mon[77242]: pgmap v15420: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:02:21.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:02:20 smithi153 ceph-mon[70044]: pgmap v15420: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:02:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:02:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:02:21] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T14:02:22.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:02:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:02:22.083Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:02:22.598 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:02:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:02:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:02:22.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:02:22 smithi083 ceph-mon[79930]: pgmap v15421: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:02:22.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:02:22 smithi083 ceph-mon[77242]: pgmap v15421: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:02:23.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:02:22 smithi153 ceph-mon[70044]: pgmap v15421: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:02:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:02:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:02:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:02:24.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:02:24 smithi083 ceph-mon[79930]: pgmap v15422: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:02:24.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:02:24 smithi083 ceph-mon[77242]: pgmap v15422: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:02:25.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:02:24 smithi153 ceph-mon[70044]: pgmap v15422: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:02:25.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:02:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:02:25.444Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:02:26.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:02:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:02:26.172Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:02:26.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:02:26 smithi083 ceph-mon[79930]: pgmap v15423: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:02:26.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:02:26 smithi083 ceph-mon[77242]: pgmap v15423: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:02:27.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:02:26 smithi153 ceph-mon[70044]: pgmap v15423: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:02:28.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:02:28 smithi083 ceph-mon[77242]: pgmap v15424: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:02:28.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:02:28 smithi083 ceph-mon[79930]: pgmap v15424: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:02:29.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:02:28 smithi153 ceph-mon[70044]: pgmap v15424: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:02:29.945 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:02:30.235 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:02:30.235 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 5m ago 8h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:02:30.236 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 3m ago 8h 64.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:02:30.236 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 5m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:02:30.236 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 3m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:02:30.236 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 3m ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:02:30.237 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 5m ago 8h 935M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:02:30.237 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 5m ago 8h 1879M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:02:30.237 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 3m ago 8h 816M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:02:30.238 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 5m ago 8h 1121M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:02:30.238 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 5m ago 8h 24.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:02:30.239 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 3m ago 8h 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:02:30.239 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 5m ago 8h 1359M 2277M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:02:30.239 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 5m ago 8h 1347M 2277M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:02:30.239 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 5m ago 8h 1048M 2277M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:02:30.240 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 5m ago 8h 1319M 2277M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:02:30.240 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 3m ago 8h 1158M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:02:30.240 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 3m ago 8h 1452M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:02:30.240 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 3m ago 8h 1298M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:02:30.241 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 3m ago 8h 1542M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:02:30.241 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 3m ago 8h 186M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:02:30.589 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:02:30.589 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:02:30.590 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:02:30.590 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:02:30.591 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:02:30.591 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:02:30.592 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:02:30.592 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:02:30.593 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:02:30.593 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:02:30.594 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:02:30.594 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:02:30.595 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:02:30.595 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:02:30.595 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:02:30.596 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:02:30.596 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:02:30.597 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:02:30.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:02:30 smithi083 ceph-mon[79930]: pgmap v15425: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:02:30.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:02:30 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1540637470' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:02:30.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:02:30 smithi083 ceph-mon[77242]: pgmap v15425: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:02:30.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:02:30 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1540637470' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:02:31.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:02:30 smithi153 ceph-mon[70044]: pgmap v15425: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:02:31.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:02:30 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1540637470' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:02:31.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:02:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:02:31] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T14:02:31.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:02:31 smithi083 ceph-mon[79930]: from='client.52068 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:02:31.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:02:31 smithi083 ceph-mon[79930]: from='client.52074 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:02:31.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:02:31 smithi083 ceph-mon[77242]: from='client.52068 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:02:31.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:02:31 smithi083 ceph-mon[77242]: from='client.52074 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:02:32.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:02:31 smithi153 ceph-mon[70044]: from='client.52068 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:02:32.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:02:31 smithi153 ceph-mon[70044]: from='client.52074 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:02:32.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:02:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:02:32.083Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:02:32.619 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:02:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:02:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:02:32.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:02:32 smithi083 ceph-mon[79930]: pgmap v15426: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:02:32.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:02:32 smithi083 ceph-mon[77242]: pgmap v15426: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:02:33.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:02:32 smithi153 ceph-mon[70044]: pgmap v15426: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:02:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:02:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:02:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:02:34.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:02:34 smithi083 ceph-mon[79930]: pgmap v15427: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:02:34.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:02:34 smithi083 ceph-mon[77242]: pgmap v15427: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:02:35.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:02:34 smithi153 ceph-mon[70044]: pgmap v15427: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:02:35.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:02:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:02:35.445Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:02:36.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:02:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:02:36.173Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:02:36.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:02:36 smithi083 ceph-mon[79930]: pgmap v15428: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:02:36.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:02:36 smithi083 ceph-mon[77242]: pgmap v15428: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:02:37.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:02:36 smithi153 ceph-mon[70044]: pgmap v15428: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:02:38.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:02:38 smithi083 ceph-mon[77242]: pgmap v15429: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:02:38.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:02:38 smithi083 ceph-mon[79930]: pgmap v15429: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:02:39.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:02:38 smithi153 ceph-mon[70044]: pgmap v15429: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:02:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:02:40 smithi083 ceph-mon[79930]: pgmap v15430: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:02:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:02:40 smithi083 ceph-mon[77242]: pgmap v15430: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:02:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:02:40 smithi153 ceph-mon[70044]: pgmap v15430: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:02:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:02:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:02:41] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T14:02:41.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:02:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:02:41.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:02:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:02:41.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:02:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:02:41.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:02:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:02:41.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:02:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:02:41.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:02:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:02:41.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:02:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:02:41.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:02:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:02:42.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:02:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:02:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:02:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:02:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:02:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:02:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:02:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:02:42.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:02:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:02:42.084Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:02:42.636 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:02:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:02:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:02:42.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:02:42 smithi083 ceph-mon[79930]: pgmap v15431: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:02:42.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:02:42 smithi083 ceph-mon[77242]: pgmap v15431: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:02:43.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:02:42 smithi153 ceph-mon[70044]: pgmap v15431: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:02:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:02:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:02:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:02:44.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:02:44 smithi083 ceph-mon[79930]: pgmap v15432: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:02:44.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:02:44 smithi083 ceph-mon[77242]: pgmap v15432: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:02:45.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:02:44 smithi153 ceph-mon[70044]: pgmap v15432: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:02:45.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:02:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:02:45.446Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:02:46.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:02:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:02:46.173Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:02:46.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:02:46 smithi083 ceph-mon[79930]: pgmap v15433: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:02:46.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:02:46 smithi083 ceph-mon[77242]: pgmap v15433: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:02:47.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:02:46 smithi153 ceph-mon[70044]: pgmap v15433: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:02:48.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:02:48 smithi083 ceph-mon[79930]: pgmap v15434: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:02:48.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:02:48 smithi083 ceph-mon[77242]: pgmap v15434: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:02:49.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:02:48 smithi153 ceph-mon[70044]: pgmap v15434: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:02:50.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:02:50 smithi083 ceph-mon[79930]: pgmap v15435: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:02:50.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:02:50 smithi083 ceph-mon[77242]: pgmap v15435: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:02:51.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:02:50 smithi153 ceph-mon[70044]: pgmap v15435: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:02:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:02:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:02:51] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T14:02:52.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:02:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:02:52.086Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:02:52.652 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:02:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:02:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:02:52.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:02:52 smithi083 ceph-mon[79930]: pgmap v15436: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:02:52.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:02:52 smithi083 ceph-mon[77242]: pgmap v15436: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:02:53.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:02:52 smithi153 ceph-mon[70044]: pgmap v15436: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:02:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:02:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:02:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:02:54.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:02:54 smithi083 ceph-mon[79930]: pgmap v15437: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:02:54.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:02:54 smithi083 ceph-mon[77242]: pgmap v15437: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:02:55.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:02:54 smithi153 ceph-mon[70044]: pgmap v15437: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:02:55.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:02:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:02:55.447Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:02:56.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:02:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:02:56.174Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:02:56.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:02:56 smithi083 ceph-mon[79930]: pgmap v15438: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:02:56.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:02:56 smithi083 ceph-mon[77242]: pgmap v15438: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:02:57.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:02:56 smithi153 ceph-mon[70044]: pgmap v15438: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:02:58.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:02:58 smithi083 ceph-mon[77242]: pgmap v15439: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:02:58.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:02:58 smithi083 ceph-mon[79930]: pgmap v15439: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:02:59.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:02:58 smithi153 ceph-mon[70044]: pgmap v15439: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:00.918 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:03:00.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:03:00 smithi083 ceph-mon[79930]: pgmap v15440: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:00.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:03:00 smithi083 ceph-mon[77242]: pgmap v15440: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:01.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:03:00 smithi153 ceph-mon[70044]: pgmap v15440: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:01.208 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:03:01.208 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 6m ago 8h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:03:01.209 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 3m ago 8h 64.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:03:01.209 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 6m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:03:01.209 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 3m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:03:01.209 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 3m ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:03:01.210 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 6m ago 8h 935M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:03:01.210 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 6m ago 8h 1879M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:03:01.210 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 3m ago 8h 816M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:03:01.210 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 6m ago 8h 1121M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:03:01.211 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 6m ago 8h 24.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:03:01.215 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 3m ago 8h 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:03:01.215 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 6m ago 8h 1359M 2277M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:03:01.215 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 6m ago 8h 1347M 2277M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:03:01.216 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 6m ago 8h 1048M 2277M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:03:01.216 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 6m ago 8h 1319M 2277M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:03:01.216 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 3m ago 8h 1158M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:03:01.216 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 3m ago 8h 1452M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:03:01.217 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 3m ago 8h 1298M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:03:01.217 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 3m ago 8h 1542M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:03:01.217 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 3m ago 8h 186M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:03:01.565 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:03:01.565 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:03:01.565 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:03:01.566 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:03:01.566 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:03:01.566 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:03:01.566 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:03:01.567 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:03:01.567 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:03:01.567 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:03:01.567 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:03:01.568 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:03:01.568 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:03:01.568 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:03:01.568 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:03:01.569 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:03:01.569 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:03:01.569 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:03:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:03:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:03:01] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T14:03:02.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:03:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:03:02.086Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:03:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:03:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:03:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:03:02.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:03:02 smithi153 ceph-mon[70044]: from='client.52086 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:03:02.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:03:02 smithi153 ceph-mon[70044]: pgmap v15441: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:02.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:03:02 smithi153 ceph-mon[70044]: from='client.52092 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:03:02.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:03:02 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2838321430' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:03:02.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:03:02 smithi083 ceph-mon[77242]: from='client.52086 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:03:02.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:03:02 smithi083 ceph-mon[77242]: pgmap v15441: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:02.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:03:02 smithi083 ceph-mon[77242]: from='client.52092 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:03:02.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:03:02 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2838321430' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:03:02.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:03:02 smithi083 ceph-mon[79930]: from='client.52086 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:03:02.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:03:02 smithi083 ceph-mon[79930]: pgmap v15441: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:02.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:03:02 smithi083 ceph-mon[79930]: from='client.52092 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:03:02.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:03:02 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2838321430' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:03:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:03:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:03:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:03:04.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:03:04 smithi153 ceph-mon[70044]: pgmap v15442: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:04.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:03:04 smithi083 ceph-mon[77242]: pgmap v15442: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:04.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:03:04 smithi083 ceph-mon[79930]: pgmap v15442: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:05.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:03:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:03:05.447Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:03:06.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:03:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:03:06.174Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:03:06.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:03:06 smithi153 ceph-mon[70044]: pgmap v15443: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:06.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:03:06 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:03:06.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:03:06 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:03:06.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:03:06 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:03:06.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:03:06 smithi083 ceph-mon[79930]: pgmap v15443: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:06.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:03:06 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:03:06.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:03:06 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:03:06.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:03:06 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:03:06.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:03:06 smithi083 ceph-mon[77242]: pgmap v15443: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:06.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:03:06 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:03:06.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:03:06 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:03:06.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:03:06 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:03:07.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:03:07 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:03:07.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:03:07 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:03:08.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:03:07 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:03:08.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:03:08 smithi083 ceph-mon[79930]: pgmap v15444: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:08.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:03:08 smithi083 ceph-mon[77242]: pgmap v15444: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:09.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:03:08 smithi153 ceph-mon[70044]: pgmap v15444: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:10.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:03:10 smithi083 ceph-mon[79930]: pgmap v15445: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:10.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:03:10 smithi083 ceph-mon[77242]: pgmap v15445: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:11.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:03:10 smithi153 ceph-mon[70044]: pgmap v15445: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:03:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:03:11] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T14:03:12.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:03:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:03:12.087Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:03:12.633 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:03:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:03:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:03:12.927 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:03:12 smithi083 ceph-mon[79930]: pgmap v15446: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:12.927 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:03:12 smithi083 ceph-mon[77242]: pgmap v15446: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:13.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:03:12 smithi153 ceph-mon[70044]: pgmap v15446: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:13.311 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:03:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:03:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:03:14.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:03:14 smithi083 ceph-mon[77242]: pgmap v15447: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:14.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:03:14 smithi083 ceph-mon[79930]: pgmap v15447: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:15.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:03:14 smithi153 ceph-mon[70044]: pgmap v15447: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:15.478 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:03:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:03:15.447Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:03:16.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:03:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:03:16.175Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:03:16.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:03:16 smithi083 ceph-mon[79930]: pgmap v15448: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:16.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:03:16 smithi083 ceph-mon[77242]: pgmap v15448: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:17.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:03:16 smithi153 ceph-mon[70044]: pgmap v15448: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:18.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:03:18 smithi083 ceph-mon[79930]: pgmap v15449: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:18.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:03:18 smithi083 ceph-mon[77242]: pgmap v15449: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:19.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:03:18 smithi153 ceph-mon[70044]: pgmap v15449: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:20.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:03:20 smithi083 ceph-mon[79930]: pgmap v15450: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:20.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:03:20 smithi083 ceph-mon[77242]: pgmap v15450: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:21.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:03:20 smithi153 ceph-mon[70044]: pgmap v15450: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:03:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:03:21] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T14:03:22.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:03:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:03:22.088Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:03:22.648 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:03:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:03:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:03:22.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:03:22 smithi083 ceph-mon[77242]: pgmap v15451: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:22.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:03:22 smithi083 ceph-mon[79930]: pgmap v15451: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:23.026 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:03:22 smithi153 ceph-mon[70044]: pgmap v15451: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:03:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:03:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:03:24.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:03:24 smithi083 ceph-mon[79930]: pgmap v15452: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:24.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:03:24 smithi083 ceph-mon[77242]: pgmap v15452: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:25.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:03:24 smithi153 ceph-mon[70044]: pgmap v15452: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:25.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:03:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:03:25.449Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:03:26.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:03:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:03:26.175Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:03:26.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:03:26 smithi083 ceph-mon[77242]: pgmap v15453: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:26.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:03:26 smithi083 ceph-mon[79930]: pgmap v15453: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:27.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:03:26 smithi153 ceph-mon[70044]: pgmap v15453: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:28.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:03:28 smithi083 ceph-mon[77242]: pgmap v15454: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:28.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:03:28 smithi083 ceph-mon[79930]: pgmap v15454: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:29.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:03:28 smithi153 ceph-mon[70044]: pgmap v15454: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:30.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:03:30 smithi083 ceph-mon[77242]: pgmap v15455: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:30.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:03:30 smithi083 ceph-mon[79930]: pgmap v15455: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:31.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:03:30 smithi153 ceph-mon[70044]: pgmap v15455: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:31.894 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:03:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:03:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:03:31] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T14:03:32.180 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:03:32.180 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 6m ago 8h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:03:32.180 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 4m ago 8h 64.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:03:32.180 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 6m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:03:32.181 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 4m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:03:32.181 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 4m ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:03:32.181 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 6m ago 8h 935M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:03:32.181 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 6m ago 8h 1879M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:03:32.182 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 4m ago 8h 816M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:03:32.182 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 6m ago 8h 1121M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:03:32.182 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 6m ago 8h 24.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:03:32.182 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 4m ago 8h 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:03:32.183 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 6m ago 8h 1359M 2277M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:03:32.183 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 6m ago 8h 1347M 2277M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:03:32.183 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 6m ago 8h 1048M 2277M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:03:32.183 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 6m ago 8h 1319M 2277M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:03:32.183 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 4m ago 8h 1158M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:03:32.184 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 4m ago 8h 1452M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:03:32.184 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 4m ago 8h 1298M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:03:32.184 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 4m ago 8h 1542M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:03:32.184 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 4m ago 8h 186M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:03:32.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:03:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:03:32.088Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:03:32.536 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:03:32.537 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:03:32.537 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:03:32.537 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:03:32.538 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:03:32.538 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:03:32.538 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:03:32.538 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:03:32.539 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:03:32.539 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:03:32.539 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:03:32.539 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:03:32.540 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:03:32.540 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:03:32.540 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:03:32.540 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:03:32.541 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:03:32.541 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:03:32.662 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:03:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:03:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:03:32.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:03:32 smithi083 ceph-mon[77242]: pgmap v15456: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:32.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:03:32 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2755054557' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:03:32.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:03:32 smithi083 ceph-mon[79930]: pgmap v15456: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:32.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:03:32 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2755054557' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:03:33.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:03:32 smithi153 ceph-mon[70044]: pgmap v15456: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:33.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:03:32 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2755054557' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:03:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:03:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:03:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:03:33.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:03:33 smithi083 ceph-mon[77242]: from='client.62035 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:03:33.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:03:33 smithi083 ceph-mon[77242]: from='client.62041 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:03:33.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:03:33 smithi083 ceph-mon[79930]: from='client.62035 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:03:33.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:03:33 smithi083 ceph-mon[79930]: from='client.62041 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:03:34.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:03:33 smithi153 ceph-mon[70044]: from='client.62035 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:03:34.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:03:33 smithi153 ceph-mon[70044]: from='client.62041 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:03:34.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:03:34 smithi083 ceph-mon[79930]: pgmap v15457: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:34.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:03:34 smithi083 ceph-mon[77242]: pgmap v15457: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:35.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:03:34 smithi153 ceph-mon[70044]: pgmap v15457: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:35.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:03:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:03:35.449Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:03:36.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:03:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:03:36.177Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:03:36.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:03:36 smithi083 ceph-mon[79930]: pgmap v15458: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:36.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:03:36 smithi083 ceph-mon[77242]: pgmap v15458: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:37.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:03:36 smithi153 ceph-mon[70044]: pgmap v15458: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:38.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:03:38 smithi083 ceph-mon[79930]: pgmap v15459: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:38.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:03:38 smithi083 ceph-mon[77242]: pgmap v15459: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:39.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:03:38 smithi153 ceph-mon[70044]: pgmap v15459: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:03:40 smithi083 ceph-mon[79930]: pgmap v15460: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:03:40 smithi083 ceph-mon[77242]: pgmap v15460: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:03:40 smithi153 ceph-mon[70044]: pgmap v15460: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:03:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:03:41] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T14:03:41.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:03:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:03:41.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:03:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:03:41.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:03:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:03:41.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:03:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:03:41.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:03:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:03:41.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:03:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:03:41.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:03:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:03:41.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:03:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:03:42.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:03:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:03:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:03:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:03:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:03:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:03:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:03:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:03:42.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:03:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:03:42.088Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:03:42.684 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:03:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:03:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:03:42.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:03:42 smithi083 ceph-mon[79930]: pgmap v15461: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:42.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:03:42 smithi083 ceph-mon[77242]: pgmap v15461: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:43.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:03:42 smithi153 ceph-mon[70044]: pgmap v15461: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:03:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:03:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:03:44.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:03:44 smithi083 ceph-mon[79930]: pgmap v15462: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:44.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:03:44 smithi083 ceph-mon[77242]: pgmap v15462: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:45.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:03:44 smithi153 ceph-mon[70044]: pgmap v15462: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:45.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:03:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:03:45.449Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:03:46.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:03:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:03:46.178Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:03:46.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:03:46 smithi083 ceph-mon[79930]: pgmap v15463: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:46.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:03:46 smithi083 ceph-mon[77242]: pgmap v15463: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:47.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:03:46 smithi153 ceph-mon[70044]: pgmap v15463: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:48.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:03:48 smithi083 ceph-mon[79930]: pgmap v15464: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:48.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:03:48 smithi083 ceph-mon[77242]: pgmap v15464: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:49.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:03:48 smithi153 ceph-mon[70044]: pgmap v15464: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:50.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:03:50 smithi083 ceph-mon[79930]: pgmap v15465: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:50.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:03:50 smithi083 ceph-mon[77242]: pgmap v15465: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:51.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:03:50 smithi153 ceph-mon[70044]: pgmap v15465: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:03:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:03:51] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T14:03:52.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:03:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:03:52.089Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:03:52.703 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:03:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:03:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:03:52.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:03:52 smithi083 ceph-mon[77242]: pgmap v15466: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:52.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:03:52 smithi083 ceph-mon[79930]: pgmap v15466: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:53.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:03:52 smithi153 ceph-mon[70044]: pgmap v15466: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:03:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:03:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:03:54.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:03:54 smithi083 ceph-mon[79930]: pgmap v15467: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:54.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:03:54 smithi083 ceph-mon[77242]: pgmap v15467: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:55.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:03:54 smithi153 ceph-mon[70044]: pgmap v15467: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:55.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:03:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:03:55.450Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:03:56.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:03:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:03:56.178Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:03:56.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:03:56 smithi083 ceph-mon[79930]: pgmap v15468: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:56.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:03:56 smithi083 ceph-mon[77242]: pgmap v15468: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:57.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:03:56 smithi153 ceph-mon[70044]: pgmap v15468: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:59.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:03:58 smithi153 ceph-mon[70044]: pgmap v15469: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:59.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:03:58 smithi083 ceph-mon[79930]: pgmap v15469: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:03:59.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:03:58 smithi083 ceph-mon[77242]: pgmap v15469: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:01.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:04:00 smithi153 ceph-mon[70044]: pgmap v15470: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:01.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:04:00 smithi083 ceph-mon[79930]: pgmap v15470: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:01.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:04:00 smithi083 ceph-mon[77242]: pgmap v15470: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:04:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:04:01] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T14:04:02.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:04:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:04:02.090Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:04:02.725 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:04:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:04:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:04:02.867 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:04:03.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:04:02 smithi153 ceph-mon[70044]: pgmap v15471: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:03.153 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:04:03.154 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 7m ago 8h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:04:03.154 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 4m ago 8h 64.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:04:03.154 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 7m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:04:03.155 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 4m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:04:03.155 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 4m ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:04:03.155 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 7m ago 8h 935M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:04:03.155 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 7m ago 8h 1879M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:04:03.155 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 4m ago 8h 816M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:04:03.156 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 7m ago 8h 1121M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:04:03.156 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 7m ago 8h 24.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:04:03.156 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 4m ago 8h 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:04:03.156 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 7m ago 8h 1359M 2277M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:04:03.157 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 7m ago 8h 1347M 2277M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:04:03.157 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 7m ago 8h 1048M 2277M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:04:03.157 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 7m ago 8h 1319M 2277M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:04:03.157 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 4m ago 8h 1158M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:04:03.157 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 4m ago 8h 1452M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:04:03.158 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 4m ago 8h 1298M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:04:03.158 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 4m ago 8h 1542M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:04:03.158 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 4m ago 8h 186M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:04:03.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:04:02 smithi083 ceph-mon[77242]: pgmap v15471: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:03.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:04:02 smithi083 ceph-mon[79930]: pgmap v15471: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:04:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:04:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:04:03.519 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:04:03.519 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:04:03.520 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:04:03.520 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:04:03.520 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:04:03.520 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:04:03.520 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:04:03.521 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:04:03.521 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:04:03.521 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:04:03.521 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:04:03.522 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:04:03.522 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:04:03.522 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:04:03.522 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:04:03.523 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:04:03.523 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:04:03.523 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:04:04.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:04:03 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/603155948' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:04:04.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:04:03 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/603155948' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:04:04.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:04:03 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/603155948' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:04:05.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:04:04 smithi153 ceph-mon[70044]: from='client.62053 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:04:05.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:04:04 smithi153 ceph-mon[70044]: pgmap v15472: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:05.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:04:04 smithi153 ceph-mon[70044]: from='client.52128 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:04:05.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:04:04 smithi083 ceph-mon[79930]: from='client.62053 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:04:05.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:04:04 smithi083 ceph-mon[79930]: pgmap v15472: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:05.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:04:04 smithi083 ceph-mon[79930]: from='client.52128 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:04:05.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:04:04 smithi083 ceph-mon[77242]: from='client.62053 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:04:05.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:04:04 smithi083 ceph-mon[77242]: pgmap v15472: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:05.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:04:04 smithi083 ceph-mon[77242]: from='client.52128 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:04:05.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:04:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:04:05.450Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:04:06.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:04:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:04:06.179Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:04:07.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:04:06 smithi153 ceph-mon[70044]: pgmap v15473: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:07.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:04:06 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:04:07.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:04:06 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:04:07.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:04:06 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:04:07.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:04:06 smithi083 ceph-mon[79930]: pgmap v15473: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:07.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:04:06 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:04:07.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:04:06 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:04:07.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:04:06 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:04:07.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:04:06 smithi083 ceph-mon[77242]: pgmap v15473: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:07.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:04:06 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:04:07.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:04:06 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:04:07.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:04:06 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:04:08.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:04:08 smithi153 ceph-mon[70044]: pgmap v15474: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:08.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:04:08 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:04:08.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:04:08 smithi083 ceph-mon[79930]: pgmap v15474: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:08.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:04:08 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:04:08.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:04:08 smithi083 ceph-mon[77242]: pgmap v15474: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:08.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:04:08 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:04:10.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:04:10 smithi153 ceph-mon[70044]: pgmap v15475: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:10.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:04:10 smithi083 ceph-mon[79930]: pgmap v15475: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:10.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:04:10 smithi083 ceph-mon[77242]: pgmap v15475: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:04:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:04:11] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T14:04:12.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:04:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:04:12.090Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:04:12.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:04:12 smithi153 ceph-mon[70044]: pgmap v15476: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:12.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:04:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:04:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:04:12.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:04:12 smithi083 ceph-mon[79930]: pgmap v15476: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:12.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:04:12 smithi083 ceph-mon[77242]: pgmap v15476: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:04:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:04:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:04:14.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:04:14 smithi153 ceph-mon[70044]: pgmap v15477: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:14.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:04:14 smithi083 ceph-mon[79930]: pgmap v15477: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:14.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:04:14 smithi083 ceph-mon[77242]: pgmap v15477: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:15.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:04:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:04:15.452Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:04:16.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:04:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:04:16.179Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:04:16.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:04:16 smithi153 ceph-mon[70044]: pgmap v15478: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:16.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:04:16 smithi083 ceph-mon[77242]: pgmap v15478: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:16.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:04:16 smithi083 ceph-mon[79930]: pgmap v15478: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:18.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:04:18 smithi153 ceph-mon[70044]: pgmap v15479: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:18.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:04:18 smithi083 ceph-mon[77242]: pgmap v15479: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:18.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:04:18 smithi083 ceph-mon[79930]: pgmap v15479: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:20.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:04:20 smithi153 ceph-mon[70044]: pgmap v15480: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:20.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:04:20 smithi083 ceph-mon[77242]: pgmap v15480: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:20.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:04:20 smithi083 ceph-mon[79930]: pgmap v15480: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:04:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:04:21] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T14:04:22.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:04:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:04:22.091Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:04:22.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:04:22 smithi153 ceph-mon[70044]: pgmap v15481: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:22.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:04:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:04:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:04:22.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:04:22 smithi083 ceph-mon[77242]: pgmap v15481: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:22.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:04:22 smithi083 ceph-mon[79930]: pgmap v15481: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:04:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:04:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:04:24.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:04:24 smithi153 ceph-mon[70044]: pgmap v15482: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:24.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:04:24 smithi083 ceph-mon[77242]: pgmap v15482: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:24.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:04:24 smithi083 ceph-mon[79930]: pgmap v15482: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:25.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:04:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:04:25.452Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:04:26.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:04:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:04:26.179Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:04:26.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:04:26 smithi153 ceph-mon[70044]: pgmap v15483: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:26.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:04:26 smithi083 ceph-mon[79930]: pgmap v15483: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:26.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:04:26 smithi083 ceph-mon[77242]: pgmap v15483: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:28.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:04:28 smithi153 ceph-mon[70044]: pgmap v15484: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:28.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:04:28 smithi083 ceph-mon[79930]: pgmap v15484: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:28.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:04:28 smithi083 ceph-mon[77242]: pgmap v15484: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:30.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:04:30 smithi153 ceph-mon[70044]: pgmap v15485: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:30.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:04:30 smithi083 ceph-mon[79930]: pgmap v15485: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:30.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:04:30 smithi083 ceph-mon[77242]: pgmap v15485: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:04:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:04:31] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T14:04:32.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:04:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:04:32.091Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:04:32.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:04:32 smithi153 ceph-mon[70044]: pgmap v15486: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:32.810 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:04:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:04:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:04:32.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:04:32 smithi083 ceph-mon[79930]: pgmap v15486: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:32.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:04:32 smithi083 ceph-mon[77242]: pgmap v15486: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:04:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:04:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:04:33.845 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:04:34.135 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:04:34.136 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 7m ago 8h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:04:34.136 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 5m ago 8h 64.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:04:34.136 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 7m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:04:34.136 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 5m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:04:34.137 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 5m ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:04:34.137 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 7m ago 8h 935M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:04:34.137 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 7m ago 8h 1879M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:04:34.137 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 5m ago 8h 816M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:04:34.138 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 7m ago 8h 1121M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:04:34.138 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 7m ago 8h 24.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:04:34.138 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 5m ago 8h 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:04:34.139 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 7m ago 8h 1359M 2277M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:04:34.139 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 7m ago 8h 1347M 2277M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:04:34.139 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 7m ago 8h 1048M 2277M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:04:34.139 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 7m ago 8h 1319M 2277M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:04:34.139 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 5m ago 8h 1158M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:04:34.140 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 5m ago 8h 1452M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:04:34.140 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 5m ago 8h 1298M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:04:34.140 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 5m ago 8h 1542M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:04:34.140 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 5m ago 8h 186M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:04:34.491 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:04:34.492 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:04:34.492 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:04:34.492 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:04:34.492 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:04:34.493 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:04:34.493 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:04:34.493 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:04:34.493 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:04:34.494 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:04:34.494 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:04:34.494 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:04:34.494 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:04:34.494 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:04:34.495 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:04:34.495 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:04:34.495 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:04:34.496 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:04:34.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:04:34 smithi153 ceph-mon[70044]: pgmap v15487: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:34.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:04:34 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/291083995' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:04:34.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:04:34 smithi083 ceph-mon[79930]: pgmap v15487: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:34.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:04:34 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/291083995' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:04:34.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:04:34 smithi083 ceph-mon[77242]: pgmap v15487: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:34.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:04:34 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/291083995' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:04:35.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:04:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:04:35.453Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:04:35.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:04:35 smithi083 ceph-mon[77242]: from='client.62071 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:04:35.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:04:35 smithi083 ceph-mon[77242]: from='client.62077 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:04:35.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:04:35 smithi083 ceph-mon[79930]: from='client.62071 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:04:35.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:04:35 smithi083 ceph-mon[79930]: from='client.62077 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:04:35.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:04:35 smithi153 ceph-mon[70044]: from='client.62071 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:04:35.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:04:35 smithi153 ceph-mon[70044]: from='client.62077 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:04:36.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:04:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:04:36.181Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:04:36.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:04:36 smithi153 ceph-mon[70044]: pgmap v15488: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:36.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:04:36 smithi083 ceph-mon[79930]: pgmap v15488: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:36.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:04:36 smithi083 ceph-mon[77242]: pgmap v15488: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:38.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:04:38 smithi153 ceph-mon[70044]: pgmap v15489: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:38.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:04:38 smithi083 ceph-mon[79930]: pgmap v15489: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:38.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:04:38 smithi083 ceph-mon[77242]: pgmap v15489: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:04:40 smithi083 ceph-mon[79930]: pgmap v15490: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:04:40 smithi083 ceph-mon[77242]: pgmap v15490: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:04:40 smithi153 ceph-mon[70044]: pgmap v15490: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:04:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:04:41] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T14:04:41.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:04:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:04:41.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:04:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:04:41.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:04:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:04:41.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:04:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:04:41.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:04:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:04:41.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:04:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:04:41.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:04:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:04:41.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:04:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:04:42.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:04:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:04:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:04:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:04:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:04:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:04:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:04:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:04:42.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:04:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:04:42.092Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:04:42.568 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:04:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:04:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:04:42.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:04:42 smithi083 ceph-mon[79930]: pgmap v15491: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:42.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:04:42 smithi083 ceph-mon[77242]: pgmap v15491: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:43.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:04:42 smithi153 ceph-mon[70044]: pgmap v15491: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:04:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:04:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:04:44.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:04:44 smithi083 ceph-mon[79930]: pgmap v15492: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:44.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:04:44 smithi083 ceph-mon[77242]: pgmap v15492: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:45.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:04:44 smithi153 ceph-mon[70044]: pgmap v15492: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:45.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:04:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:04:45.454Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:04:46.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:04:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:04:46.181Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:04:46.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:04:46 smithi083 ceph-mon[77242]: pgmap v15493: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:46.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:04:46 smithi083 ceph-mon[79930]: pgmap v15493: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:47.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:04:46 smithi153 ceph-mon[70044]: pgmap v15493: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:48.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:04:48 smithi083 ceph-mon[79930]: pgmap v15494: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:48.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:04:48 smithi083 ceph-mon[77242]: pgmap v15494: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:49.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:04:48 smithi153 ceph-mon[70044]: pgmap v15494: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:50.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:04:50 smithi083 ceph-mon[79930]: pgmap v15495: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:50.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:04:50 smithi083 ceph-mon[77242]: pgmap v15495: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:51.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:04:50 smithi153 ceph-mon[70044]: pgmap v15495: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:51.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:04:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:04:51] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T14:04:52.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:04:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:04:52.092Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:04:52.583 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:04:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:04:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:04:52.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:04:52 smithi083 ceph-mon[79930]: pgmap v15496: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:52.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:04:52 smithi083 ceph-mon[77242]: pgmap v15496: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:53.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:04:52 smithi153 ceph-mon[70044]: pgmap v15496: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:04:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:04:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:04:54.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:04:54 smithi083 ceph-mon[79930]: pgmap v15497: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:54.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:04:54 smithi083 ceph-mon[77242]: pgmap v15497: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:55.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:04:54 smithi153 ceph-mon[70044]: pgmap v15497: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:55.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:04:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:04:55.454Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:04:56.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:04:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:04:56.182Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:04:56.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:04:56 smithi083 ceph-mon[77242]: pgmap v15498: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:56.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:04:56 smithi083 ceph-mon[79930]: pgmap v15498: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:57.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:04:56 smithi153 ceph-mon[70044]: pgmap v15498: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:58.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:04:58 smithi083 ceph-mon[79930]: pgmap v15499: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:58.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:04:58 smithi083 ceph-mon[77242]: pgmap v15499: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:04:59.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:04:58 smithi153 ceph-mon[70044]: pgmap v15499: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:00.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:05:00 smithi083 ceph-mon[79930]: pgmap v15500: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:00.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:05:00 smithi083 ceph-mon[77242]: pgmap v15500: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:01.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:05:00 smithi153 ceph-mon[70044]: pgmap v15500: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:01.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:05:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:05:01] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T14:05:02.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:05:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:05:02.092Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:05:02.597 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:05:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:05:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:05:02.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:05:02 smithi083 ceph-mon[77242]: pgmap v15501: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:02.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:05:02 smithi083 ceph-mon[79930]: pgmap v15501: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:03.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:05:02 smithi153 ceph-mon[70044]: pgmap v15501: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:05:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:05:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:05:04.820 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:05:04.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:05:04 smithi083 ceph-mon[79930]: pgmap v15502: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:04.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:05:04 smithi083 ceph-mon[77242]: pgmap v15502: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:05.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:05:04 smithi153 ceph-mon[70044]: pgmap v15502: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:05.109 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:05:05.110 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 8m ago 8h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:05:05.110 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 6m ago 8h 64.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:05:05.110 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 8m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:05:05.111 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 6m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:05:05.111 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 6m ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:05:05.111 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 8m ago 8h 935M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:05:05.111 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 8m ago 8h 1879M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:05:05.111 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 6m ago 8h 816M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:05:05.112 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 8m ago 8h 1121M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:05:05.112 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 8m ago 8h 24.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:05:05.112 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 6m ago 8h 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:05:05.112 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 8m ago 8h 1359M 2277M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:05:05.113 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 8m ago 8h 1347M 2277M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:05:05.113 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 8m ago 8h 1048M 2277M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:05:05.113 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 8m ago 8h 1319M 2277M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:05:05.113 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 6m ago 8h 1158M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:05:05.113 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 6m ago 8h 1452M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:05:05.114 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 6m ago 8h 1298M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:05:05.114 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 6m ago 8h 1542M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:05:05.114 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 6m ago 8h 186M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:05:05.468 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:05:05.469 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:05:05.469 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:05:05.469 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:05:05.470 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:05:05.470 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:05:05.470 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:05:05.470 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:05:05.471 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:05:05.471 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:05:05.471 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:05:05.471 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:05:05.471 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:05:05.472 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:05:05.472 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:05:05.472 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:05:05.472 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:05:05.473 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:05:05.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:05:05 smithi083 ceph-mon[79930]: from='client.62089 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:05:05.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:05:05 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2433628837' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:05:05.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:05:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:05:05.454Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:05:05.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:05:05 smithi083 ceph-mon[77242]: from='client.62089 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:05:05.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:05:05 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2433628837' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:05:06.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:05:05 smithi153 ceph-mon[70044]: from='client.62089 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:05:06.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:05:05 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2433628837' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:05:06.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:05:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:05:06.182Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:05:06.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:05:06 smithi083 ceph-mon[79930]: from='client.62095 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:05:06.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:05:06 smithi083 ceph-mon[79930]: pgmap v15503: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:06.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:05:06 smithi083 ceph-mon[77242]: from='client.62095 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:05:06.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:05:06 smithi083 ceph-mon[77242]: pgmap v15503: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:07.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:05:06 smithi153 ceph-mon[70044]: from='client.62095 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:05:07.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:05:06 smithi153 ceph-mon[70044]: pgmap v15503: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:07.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:05:07 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:05:07.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:05:07 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:05:07.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:05:07 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:05:07.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:05:07 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:05:07.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:05:07 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:05:07.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:05:07 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:05:08.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:05:07 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:05:08.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:05:07 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:05:08.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:05:07 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:05:08.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:05:08 smithi083 ceph-mon[79930]: pgmap v15504: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:08.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:05:08 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:05:08.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:05:08 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:05:08.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:05:08 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:05:08.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:05:08 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:05:08.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:05:08 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:05:08.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:05:08 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:05:08.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:05:08 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:05:08.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:05:08 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:05:08.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:05:08 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:05:08.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:05:08 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:05:08.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:05:08 smithi083 ceph-mon[77242]: pgmap v15504: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:08.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:05:08 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:05:08.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:05:08 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:05:08.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:05:08 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:05:08.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:05:08 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:05:08.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:05:08 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:05:08.968 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:05:08 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:05:08.968 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:05:08 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:05:08.968 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:05:08 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:05:08.968 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:05:08 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:05:08.969 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:05:08 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:05:09.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:05:08 smithi153 ceph-mon[70044]: pgmap v15504: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:09.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:05:08 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:05:09.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:05:08 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:05:09.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:05:08 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:05:09.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:05:08 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:05:09.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:05:08 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:05:09.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:05:08 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:05:09.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:05:08 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:05:09.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:05:08 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:05:09.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:05:08 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:05:09.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:05:08 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:05:09.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:05:09 smithi083 ceph-mon[79930]: Adjusting osd_memory_target on smithi083 to 2262M 2022-09-29T14:05:09.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:05:09 smithi083 ceph-mon[77242]: Adjusting osd_memory_target on smithi083 to 2262M 2022-09-29T14:05:10.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:05:09 smithi153 ceph-mon[70044]: Adjusting osd_memory_target on smithi083 to 2262M 2022-09-29T14:05:10.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:05:10 smithi083 ceph-mon[79930]: pgmap v15505: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:10.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:05:10 smithi083 ceph-mon[77242]: pgmap v15505: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:11.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:05:10 smithi153 ceph-mon[70044]: pgmap v15505: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:05:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:05:11] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T14:05:12.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:05:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:05:12.092Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:05:12.622 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:05:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:05:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:05:12.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:05:12 smithi083 ceph-mon[77242]: pgmap v15506: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:12.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:05:12 smithi083 ceph-mon[79930]: pgmap v15506: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:13.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:05:12 smithi153 ceph-mon[70044]: pgmap v15506: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:05:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:05:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:05:14.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:05:14 smithi083 ceph-mon[79930]: pgmap v15507: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:14.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:05:14 smithi083 ceph-mon[77242]: pgmap v15507: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:15.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:05:14 smithi153 ceph-mon[70044]: pgmap v15507: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:15.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:05:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:05:15.455Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:05:16.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:05:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:05:16.183Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:05:16.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:05:16 smithi083 ceph-mon[77242]: pgmap v15508: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:16.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:05:16 smithi083 ceph-mon[79930]: pgmap v15508: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:17.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:05:16 smithi153 ceph-mon[70044]: pgmap v15508: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:18.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:05:18 smithi083 ceph-mon[79930]: pgmap v15509: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:18.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:05:18 smithi083 ceph-mon[77242]: pgmap v15509: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:19.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:05:18 smithi153 ceph-mon[70044]: pgmap v15509: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:20.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:05:20 smithi083 ceph-mon[79930]: pgmap v15510: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:20.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:05:20 smithi083 ceph-mon[77242]: pgmap v15510: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:21.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:05:20 smithi153 ceph-mon[70044]: pgmap v15510: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:05:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:05:21] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T14:05:22.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:05:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:05:22.093Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:05:22.640 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:05:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:05:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:05:22.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:05:22 smithi083 ceph-mon[77242]: pgmap v15511: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:22.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:05:22 smithi083 ceph-mon[79930]: pgmap v15511: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:23.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:05:22 smithi153 ceph-mon[70044]: pgmap v15511: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:05:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:05:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:05:24.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:05:24 smithi083 ceph-mon[77242]: pgmap v15512: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:24.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:05:24 smithi083 ceph-mon[79930]: pgmap v15512: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:25.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:05:24 smithi153 ceph-mon[70044]: pgmap v15512: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:25.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:05:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:05:25.455Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:05:26.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:05:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:05:26.184Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:05:26.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:05:26 smithi083 ceph-mon[79930]: pgmap v15513: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:26.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:05:26 smithi083 ceph-mon[77242]: pgmap v15513: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:27.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:05:26 smithi153 ceph-mon[70044]: pgmap v15513: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:28.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:05:28 smithi083 ceph-mon[79930]: pgmap v15514: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:28.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:05:28 smithi083 ceph-mon[77242]: pgmap v15514: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:29.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:05:28 smithi153 ceph-mon[70044]: pgmap v15514: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:30.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:05:30 smithi083 ceph-mon[79930]: pgmap v15515: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:30.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:05:30 smithi083 ceph-mon[77242]: pgmap v15515: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:31.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:05:30 smithi153 ceph-mon[70044]: pgmap v15515: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:05:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:05:31] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T14:05:32.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:05:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:05:32.094Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:05:32.659 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:05:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:05:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:05:32.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:05:32 smithi083 ceph-mon[79930]: pgmap v15516: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:32.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:05:32 smithi083 ceph-mon[77242]: pgmap v15516: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:33.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:05:32 smithi153 ceph-mon[70044]: pgmap v15516: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:05:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:05:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:05:34.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:05:34 smithi083 ceph-mon[79930]: pgmap v15517: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:34.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:05:34 smithi083 ceph-mon[77242]: pgmap v15517: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:35.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:05:34 smithi153 ceph-mon[70044]: pgmap v15517: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:35.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:05:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:05:35.456Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:05:35.796 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:05:36.086 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:05:36.087 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 8m ago 8h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:05:36.087 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 6m ago 8h 64.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:05:36.087 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 8m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:05:36.087 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 6m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:05:36.088 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 6m ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:05:36.088 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 8m ago 8h 935M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:05:36.088 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 8m ago 8h 1879M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:05:36.088 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 6m ago 8h 816M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:05:36.089 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 8m ago 8h 1121M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:05:36.089 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 8m ago 8h 24.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:05:36.089 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 6m ago 8h 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:05:36.089 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 8m ago 8h 1359M 2262M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:05:36.090 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 8m ago 8h 1347M 2262M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:05:36.090 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 8m ago 8h 1048M 2262M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:05:36.090 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 8m ago 8h 1319M 2262M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:05:36.090 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 6m ago 8h 1158M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:05:36.091 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 6m ago 8h 1452M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:05:36.091 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 6m ago 8h 1298M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:05:36.091 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 6m ago 8h 1542M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:05:36.091 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 6m ago 8h 186M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:05:36.447 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:05:36.448 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:05:36.448 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:05:36.448 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:05:36.448 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:05:36.449 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:05:36.449 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:05:36.449 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:05:36.449 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:05:36.450 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:05:36.450 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:05:36.450 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:05:36.450 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:05:36.451 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:05:36.451 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:05:36.451 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:05:36.451 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:05:36.452 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:05:36.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:05:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:05:36.184Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:05:36.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:05:36 smithi083 ceph-mon[77242]: pgmap v15518: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:36.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:05:36 smithi083 ceph-mon[77242]: from='client.52176 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:05:36.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:05:36 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4045711825' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:05:36.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:05:36 smithi083 ceph-mon[79930]: pgmap v15518: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:36.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:05:36 smithi083 ceph-mon[79930]: from='client.52176 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:05:36.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:05:36 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4045711825' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:05:37.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:05:36 smithi153 ceph-mon[70044]: pgmap v15518: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:37.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:05:36 smithi153 ceph-mon[70044]: from='client.52176 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:05:37.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:05:36 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4045711825' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:05:37.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:05:37 smithi083 ceph-mon[79930]: from='client.62113 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:05:37.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:05:37 smithi083 ceph-mon[77242]: from='client.62113 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:05:38.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:05:37 smithi153 ceph-mon[70044]: from='client.62113 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:05:38.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:05:38 smithi083 ceph-mon[79930]: pgmap v15519: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:38.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:05:38 smithi083 ceph-mon[77242]: pgmap v15519: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:39.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:05:38 smithi153 ceph-mon[70044]: pgmap v15519: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:05:40 smithi083 ceph-mon[79930]: pgmap v15520: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:05:40 smithi083 ceph-mon[77242]: pgmap v15520: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:05:40 smithi153 ceph-mon[70044]: pgmap v15520: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:05:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:05:41] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T14:05:41.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:05:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:05:41.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:05:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:05:41.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:05:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:05:41.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:05:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:05:41.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:05:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:05:41.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:05:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:05:41.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:05:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:05:41.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:05:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:05:42.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:05:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:05:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:05:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:05:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:05:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:05:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:05:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:05:42.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:05:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:05:42.095Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:05:42.684 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:05:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:05:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:05:42.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:05:42 smithi083 ceph-mon[79930]: pgmap v15521: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:42.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:05:42 smithi083 ceph-mon[77242]: pgmap v15521: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:43.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:05:42 smithi153 ceph-mon[70044]: pgmap v15521: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:43.026 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:05:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:05:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:05:44.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:05:44 smithi083 ceph-mon[79930]: pgmap v15522: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:44.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:05:44 smithi083 ceph-mon[77242]: pgmap v15522: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:45.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:05:44 smithi153 ceph-mon[70044]: pgmap v15522: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:45.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:05:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:05:45.457Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:05:46.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:05:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:05:46.184Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:05:46.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:05:46 smithi083 ceph-mon[77242]: pgmap v15523: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:46.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:05:46 smithi083 ceph-mon[79930]: pgmap v15523: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:47.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:05:46 smithi153 ceph-mon[70044]: pgmap v15523: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:48.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:05:48 smithi083 ceph-mon[79930]: pgmap v15524: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:48.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:05:48 smithi083 ceph-mon[77242]: pgmap v15524: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:49.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:05:48 smithi153 ceph-mon[70044]: pgmap v15524: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:50.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:05:50 smithi083 ceph-mon[79930]: pgmap v15525: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:50.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:05:50 smithi083 ceph-mon[77242]: pgmap v15525: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:51.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:05:50 smithi153 ceph-mon[70044]: pgmap v15525: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:05:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:05:51] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T14:05:52.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:05:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:05:52.096Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:05:52.700 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:05:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:05:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:05:52.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:05:52 smithi083 ceph-mon[79930]: pgmap v15526: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:52.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:05:52 smithi083 ceph-mon[77242]: pgmap v15526: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:53.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:05:52 smithi153 ceph-mon[70044]: pgmap v15526: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:05:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:05:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:05:54.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:05:54 smithi083 ceph-mon[77242]: pgmap v15527: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:54.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:05:54 smithi083 ceph-mon[79930]: pgmap v15527: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:55.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:05:54 smithi153 ceph-mon[70044]: pgmap v15527: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:55.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:05:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:05:55.457Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:05:56.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:05:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:05:56.185Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:05:56.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:05:56 smithi083 ceph-mon[77242]: pgmap v15528: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:56.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:05:56 smithi083 ceph-mon[79930]: pgmap v15528: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:57.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:05:56 smithi153 ceph-mon[70044]: pgmap v15528: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:58.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:05:58 smithi083 ceph-mon[77242]: pgmap v15529: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:58.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:05:58 smithi083 ceph-mon[79930]: pgmap v15529: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:05:59.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:05:58 smithi153 ceph-mon[70044]: pgmap v15529: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:01.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:06:00 smithi153 ceph-mon[70044]: pgmap v15530: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:01.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:06:00 smithi083 ceph-mon[77242]: pgmap v15530: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:01.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:06:00 smithi083 ceph-mon[79930]: pgmap v15530: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:06:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:06:01] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T14:06:02.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:06:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:06:02.097Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:06:02.716 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:06:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:06:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:06:03.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:06:02 smithi153 ceph-mon[70044]: pgmap v15531: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:03.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:06:02 smithi083 ceph-mon[77242]: pgmap v15531: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:03.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:06:02 smithi083 ceph-mon[79930]: pgmap v15531: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:06:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:06:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:06:05.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:06:04 smithi153 ceph-mon[70044]: pgmap v15532: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:05.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:06:04 smithi083 ceph-mon[77242]: pgmap v15532: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:05.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:06:04 smithi083 ceph-mon[79930]: pgmap v15532: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:05.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:06:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:06:05.458Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:06:06.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:06:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:06:06.186Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:06:06.773 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:06:07.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:06:06 smithi153 ceph-mon[70044]: pgmap v15533: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:07.061 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:06:07.062 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 9m ago 8h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:06:07.062 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 7m ago 8h 64.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:06:07.062 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 9m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:06:07.063 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 7m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:06:07.063 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 7m ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:06:07.063 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 9m ago 8h 935M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:06:07.063 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 9m ago 8h 1879M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:06:07.063 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 7m ago 8h 816M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:06:07.064 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 9m ago 8h 1121M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:06:07.064 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 9m ago 8h 24.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:06:07.064 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 7m ago 8h 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:06:07.064 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 9m ago 8h 1359M 2262M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:06:07.065 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 9m ago 8h 1347M 2262M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:06:07.065 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 9m ago 8h 1048M 2262M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:06:07.065 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 9m ago 8h 1319M 2262M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:06:07.065 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 7m ago 8h 1158M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:06:07.066 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 7m ago 8h 1452M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:06:07.066 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 7m ago 8h 1298M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:06:07.066 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 7m ago 8h 1542M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:06:07.066 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 7m ago 8h 186M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:06:07.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:06:06 smithi083 ceph-mon[79930]: pgmap v15533: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:07.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:06:06 smithi083 ceph-mon[77242]: pgmap v15533: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:07.418 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:06:07.419 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:06:07.419 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:06:07.419 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:06:07.419 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:06:07.420 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:06:07.420 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:06:07.420 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:06:07.420 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:06:07.421 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:06:07.421 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:06:07.421 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:06:07.421 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:06:07.422 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:06:07.422 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:06:07.422 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:06:07.422 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:06:07.423 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:06:08.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:06:07 smithi153 ceph-mon[70044]: from='client.52194 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:06:08.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:06:07 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3938320141' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:06:08.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:06:07 smithi083 ceph-mon[79930]: from='client.52194 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:06:08.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:06:07 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3938320141' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:06:08.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:06:07 smithi083 ceph-mon[77242]: from='client.52194 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:06:08.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:06:07 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3938320141' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:06:09.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:06:08 smithi153 ceph-mon[70044]: from='client.62131 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:06:09.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:06:08 smithi153 ceph-mon[70044]: pgmap v15534: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:09.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:06:08 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:06:09.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:06:08 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:06:09.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:06:08 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:06:09.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:06:08 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:06:09.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:06:08 smithi083 ceph-mon[79930]: from='client.62131 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:06:09.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:06:08 smithi083 ceph-mon[79930]: pgmap v15534: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:09.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:06:08 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:06:09.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:06:08 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:06:09.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:06:08 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:06:09.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:06:08 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:06:09.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:06:08 smithi083 ceph-mon[77242]: from='client.62131 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:06:09.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:06:08 smithi083 ceph-mon[77242]: pgmap v15534: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:09.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:06:08 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:06:09.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:06:08 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:06:09.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:06:08 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:06:09.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:06:08 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:06:11.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:06:10 smithi153 ceph-mon[70044]: pgmap v15535: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:11.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:06:10 smithi083 ceph-mon[79930]: pgmap v15535: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:11.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:06:10 smithi083 ceph-mon[77242]: pgmap v15535: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:06:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:06:11] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T14:06:12.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:06:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:06:12.098Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:06:12.739 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:06:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:06:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:06:13.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:06:12 smithi153 ceph-mon[70044]: pgmap v15536: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:13.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:06:12 smithi083 ceph-mon[77242]: pgmap v15536: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:13.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:06:12 smithi083 ceph-mon[79930]: pgmap v15536: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:06:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:06:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:06:15.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:06:14 smithi153 ceph-mon[70044]: pgmap v15537: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:15.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:06:14 smithi083 ceph-mon[79930]: pgmap v15537: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:15.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:06:14 smithi083 ceph-mon[77242]: pgmap v15537: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:15.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:06:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:06:15.458Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:06:16.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:06:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:06:16.187Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:06:17.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:06:16 smithi153 ceph-mon[70044]: pgmap v15538: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:17.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:06:16 smithi083 ceph-mon[79930]: pgmap v15538: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:17.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:06:16 smithi083 ceph-mon[77242]: pgmap v15538: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:19.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:06:18 smithi153 ceph-mon[70044]: pgmap v15539: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:19.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:06:18 smithi083 ceph-mon[79930]: pgmap v15539: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:19.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:06:18 smithi083 ceph-mon[77242]: pgmap v15539: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:21.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:06:20 smithi153 ceph-mon[70044]: pgmap v15540: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:21.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:06:20 smithi083 ceph-mon[77242]: pgmap v15540: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:21.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:06:20 smithi083 ceph-mon[79930]: pgmap v15540: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:06:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:06:21] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T14:06:22.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:06:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:06:22.098Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:06:22.748 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:06:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:06:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:06:23.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:06:22 smithi153 ceph-mon[70044]: pgmap v15541: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:23.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:06:22 smithi083 ceph-mon[79930]: pgmap v15541: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:23.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:06:22 smithi083 ceph-mon[77242]: pgmap v15541: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:06:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:06:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:06:25.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:06:24 smithi153 ceph-mon[70044]: pgmap v15542: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:25.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:06:24 smithi083 ceph-mon[79930]: pgmap v15542: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:25.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:06:24 smithi083 ceph-mon[77242]: pgmap v15542: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:25.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:06:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:06:25.458Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:06:26.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:06:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:06:26.188Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:06:27.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:06:26 smithi153 ceph-mon[70044]: pgmap v15543: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:27.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:06:26 smithi083 ceph-mon[77242]: pgmap v15543: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:27.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:06:26 smithi083 ceph-mon[79930]: pgmap v15543: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:29.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:06:28 smithi153 ceph-mon[70044]: pgmap v15544: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:29.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:06:28 smithi083 ceph-mon[79930]: pgmap v15544: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:29.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:06:28 smithi083 ceph-mon[77242]: pgmap v15544: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:31.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:06:30 smithi153 ceph-mon[70044]: pgmap v15545: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:31.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:06:30 smithi083 ceph-mon[79930]: pgmap v15545: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:31.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:06:30 smithi083 ceph-mon[77242]: pgmap v15545: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:06:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:06:31] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T14:06:32.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:06:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:06:32.099Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:06:32.762 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:06:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:06:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:06:33.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:06:32 smithi153 ceph-mon[70044]: pgmap v15546: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:33.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:06:32 smithi083 ceph-mon[77242]: pgmap v15546: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:33.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:06:32 smithi083 ceph-mon[79930]: pgmap v15546: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:06:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:06:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:06:35.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:06:34 smithi153 ceph-mon[70044]: pgmap v15547: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:35.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:06:34 smithi083 ceph-mon[79930]: pgmap v15547: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:35.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:06:34 smithi083 ceph-mon[77242]: pgmap v15547: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:35.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:06:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:06:35.460Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:06:36.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:06:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:06:36.188Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:06:37.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:06:36 smithi153 ceph-mon[70044]: pgmap v15548: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:37.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:06:36 smithi083 ceph-mon[79930]: pgmap v15548: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:37.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:06:36 smithi083 ceph-mon[77242]: pgmap v15548: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:37.742 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:06:38.031 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:06:38.031 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 9m ago 8h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:06:38.032 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 7m ago 8h 64.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:06:38.032 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 9m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:06:38.032 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 7m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:06:38.032 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 7m ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:06:38.033 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 9m ago 8h 935M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:06:38.033 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 9m ago 8h 1879M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:06:38.033 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 7m ago 8h 816M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:06:38.033 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 9m ago 8h 1121M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:06:38.034 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 9m ago 8h 24.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:06:38.034 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 7m ago 8h 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:06:38.034 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 9m ago 8h 1359M 2262M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:06:38.034 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 9m ago 8h 1347M 2262M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:06:38.034 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 9m ago 8h 1048M 2262M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:06:38.035 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 9m ago 8h 1319M 2262M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:06:38.035 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 7m ago 8h 1158M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:06:38.035 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 7m ago 8h 1452M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:06:38.035 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 7m ago 8h 1298M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:06:38.036 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 7m ago 8h 1542M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:06:38.036 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 7m ago 8h 186M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:06:38.385 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:06:38.386 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:06:38.386 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:06:38.386 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:06:38.387 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:06:38.387 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:06:38.387 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:06:38.387 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:06:38.388 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:06:38.388 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:06:38.388 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:06:38.388 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:06:38.388 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:06:38.389 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:06:38.389 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:06:38.389 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:06:38.389 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:06:38.390 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:06:39.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:06:38 smithi153 ceph-mon[70044]: pgmap v15549: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:39.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:06:38 smithi153 ceph-mon[70044]: from='client.52212 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:06:39.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:06:38 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3104064444' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:06:39.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:06:38 smithi083 ceph-mon[79930]: pgmap v15549: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:39.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:06:38 smithi083 ceph-mon[79930]: from='client.52212 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:06:39.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:06:38 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3104064444' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:06:39.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:06:38 smithi083 ceph-mon[77242]: pgmap v15549: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:39.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:06:38 smithi083 ceph-mon[77242]: from='client.52212 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:06:39.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:06:38 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3104064444' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:06:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:06:39 smithi153 ceph-mon[70044]: from='client.52218 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:06:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:06:39 smithi083 ceph-mon[79930]: from='client.52218 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:06:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:06:39 smithi083 ceph-mon[77242]: from='client.52218 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:06:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:06:40 smithi153 ceph-mon[70044]: pgmap v15550: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:41.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:06:40 smithi083 ceph-mon[79930]: pgmap v15550: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:06:40 smithi083 ceph-mon[77242]: pgmap v15550: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:41.781 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:06:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:06:41] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T14:06:42.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:06:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:06:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:06:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:06:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:06:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:06:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:06:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:06:42.102 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:06:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:06:42.102 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:06:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:06:42.103 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:06:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:06:42.103 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:06:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:06:42.103 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:06:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:06:42.104 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:06:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:06:42.104 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:06:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:06:42.104 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:06:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:06:42.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:06:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:06:42.100Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:06:42.782 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:06:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:06:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:06:43.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:06:42 smithi153 ceph-mon[70044]: pgmap v15551: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:43.059 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:06:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:06:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:06:43.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:06:42 smithi083 ceph-mon[79930]: pgmap v15551: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:43.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:06:42 smithi083 ceph-mon[77242]: pgmap v15551: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:45.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:06:44 smithi153 ceph-mon[70044]: pgmap v15552: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:45.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:06:44 smithi083 ceph-mon[79930]: pgmap v15552: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:45.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:06:44 smithi083 ceph-mon[77242]: pgmap v15552: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:45.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:06:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:06:45.460Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:06:46.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:06:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:06:46.189Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:06:47.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:06:46 smithi153 ceph-mon[70044]: pgmap v15553: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:47.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:06:46 smithi083 ceph-mon[79930]: pgmap v15553: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:47.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:06:46 smithi083 ceph-mon[77242]: pgmap v15553: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:49.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:06:48 smithi153 ceph-mon[70044]: pgmap v15554: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:49.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:06:48 smithi083 ceph-mon[79930]: pgmap v15554: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:49.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:06:48 smithi083 ceph-mon[77242]: pgmap v15554: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:51.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:06:50 smithi153 ceph-mon[70044]: pgmap v15555: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:51.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:06:50 smithi083 ceph-mon[79930]: pgmap v15555: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:51.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:06:50 smithi083 ceph-mon[77242]: pgmap v15555: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:06:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:06:51] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T14:06:52.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:06:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:06:52.101Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:06:52.796 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:06:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:06:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:06:53.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:06:52 smithi153 ceph-mon[70044]: pgmap v15556: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:53.059 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:06:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:06:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:06:53.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:06:52 smithi083 ceph-mon[77242]: pgmap v15556: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:53.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:06:52 smithi083 ceph-mon[79930]: pgmap v15556: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:55.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:06:54 smithi153 ceph-mon[70044]: pgmap v15557: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:55.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:06:54 smithi083 ceph-mon[79930]: pgmap v15557: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:55.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:06:54 smithi083 ceph-mon[77242]: pgmap v15557: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:55.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:06:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:06:55.461Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:06:56.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:06:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:06:56.190Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:06:57.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:06:56 smithi153 ceph-mon[70044]: pgmap v15558: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:57.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:06:56 smithi083 ceph-mon[79930]: pgmap v15558: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:57.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:06:56 smithi083 ceph-mon[77242]: pgmap v15558: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:59.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:06:58 smithi153 ceph-mon[70044]: pgmap v15559: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:59.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:06:58 smithi083 ceph-mon[79930]: pgmap v15559: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:06:59.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:06:58 smithi083 ceph-mon[77242]: pgmap v15559: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:01.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:07:00 smithi083 ceph-mon[79930]: pgmap v15560: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:01.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:07:00 smithi083 ceph-mon[77242]: pgmap v15560: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:01.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:07:00 smithi153 ceph-mon[70044]: pgmap v15560: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:01.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:07:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:07:01] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T14:07:02.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:07:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:07:02.102Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:07:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:07:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:07:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:07:03.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:07:02 smithi083 ceph-mon[79930]: pgmap v15561: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:03.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:07:02 smithi083 ceph-mon[77242]: pgmap v15561: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:03.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:07:02 smithi153 ceph-mon[70044]: pgmap v15561: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:07:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:07:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:07:05.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:07:04 smithi083 ceph-mon[79930]: pgmap v15562: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:05.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:07:04 smithi083 ceph-mon[77242]: pgmap v15562: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:05.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:07:04 smithi153 ceph-mon[70044]: pgmap v15562: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:05.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:07:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:07:05.462Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:07:06.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:07:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:07:06.191Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:07:07.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:07:06 smithi083 ceph-mon[79930]: pgmap v15563: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:07.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:07:06 smithi083 ceph-mon[77242]: pgmap v15563: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:07.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:07:06 smithi153 ceph-mon[70044]: pgmap v15563: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:08.715 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:07:09.019 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:07:09.019 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 10m ago 8h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:07:09.019 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 8m ago 8h 64.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:07:09.020 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 10m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:07:09.020 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 8m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:07:09.020 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 8m ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:07:09.020 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 10m ago 8h 935M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:07:09.021 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 10m ago 8h 1879M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:07:09.021 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 8m ago 8h 816M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:07:09.021 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 10m ago 8h 1121M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:07:09.021 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 10m ago 8h 24.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:07:09.022 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 8m ago 8h 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:07:09.022 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 10m ago 8h 1359M 2262M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:07:09.022 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 10m ago 8h 1347M 2262M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:07:09.022 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 10m ago 8h 1048M 2262M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:07:09.023 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 10m ago 8h 1319M 2262M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:07:09.023 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 8m ago 8h 1158M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:07:09.023 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 8m ago 8h 1452M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:07:09.023 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 8m ago 8h 1298M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:07:09.023 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 8m ago 8h 1542M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:07:09.024 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 8m ago 8h 186M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:07:09.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:07:08 smithi083 ceph-mon[77242]: pgmap v15564: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:09.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:07:08 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:07:09.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:07:08 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:07:09.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:07:08 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:07:09.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:07:08 smithi083 ceph-mon[79930]: pgmap v15564: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:09.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:07:08 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:07:09.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:07:08 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:07:09.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:07:08 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:07:09.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:07:08 smithi153 ceph-mon[70044]: pgmap v15564: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:09.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:07:08 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:07:09.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:07:08 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:07:09.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:07:08 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:07:09.408 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:07:09.408 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:07:09.409 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:07:09.409 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:07:09.409 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:07:09.409 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:07:09.409 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:07:09.410 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:07:09.410 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:07:09.410 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:07:09.410 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:07:09.411 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:07:09.411 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:07:09.411 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:07:09.411 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:07:09.412 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:07:09.412 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:07:09.412 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:07:10.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:07:09 smithi083 ceph-mon[79930]: from='client.62161 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:07:10.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:07:09 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:07:10.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:07:09 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:07:10.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:07:09 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:07:10.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:07:09 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:07:10.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:07:09 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4266247079' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:07:10.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:07:09 smithi083 ceph-mon[77242]: from='client.62161 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:07:10.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:07:09 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:07:10.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:07:09 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:07:10.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:07:09 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:07:10.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:07:09 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:07:10.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:07:09 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4266247079' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:07:10.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:07:09 smithi153 ceph-mon[70044]: from='client.62161 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:07:10.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:07:09 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:07:10.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:07:09 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:07:10.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:07:09 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:07:10.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:07:09 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:07:10.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:07:09 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4266247079' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:07:11.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:07:10 smithi083 ceph-mon[79930]: from='client.52236 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:07:11.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:07:10 smithi083 ceph-mon[79930]: pgmap v15565: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:11.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:07:10 smithi083 ceph-mon[77242]: from='client.52236 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:07:11.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:07:10 smithi083 ceph-mon[77242]: pgmap v15565: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:11.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:07:10 smithi153 ceph-mon[70044]: from='client.52236 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:07:11.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:07:10 smithi153 ceph-mon[70044]: pgmap v15565: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:07:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:07:11] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T14:07:12.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:07:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:07:12.102Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:07:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:07:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:07:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:07:13.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:07:12 smithi083 ceph-mon[79930]: pgmap v15566: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:13.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:07:12 smithi083 ceph-mon[77242]: pgmap v15566: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:13.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:07:12 smithi153 ceph-mon[70044]: pgmap v15566: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:13.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:07:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:07:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:07:14.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:07:14 smithi083 ceph-mon[77242]: pgmap v15567: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:14.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:07:14 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:07:14.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:07:14 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:07:14.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:07:14 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:07:14.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:07:14 smithi083 ceph-mon[79930]: pgmap v15567: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:14.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:07:14 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:07:14.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:07:14 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:07:14.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:07:14 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:07:14.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:07:14 smithi153 ceph-mon[70044]: pgmap v15567: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:14.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:07:14 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:07:14.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:07:14 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:07:14.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:07:14 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:07:15.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:07:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:07:15.463Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:07:16.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:07:16 smithi083 ceph-mon[79930]: pgmap v15568: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:16.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:07:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:07:16.192Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:07:16.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:07:16 smithi083 ceph-mon[77242]: pgmap v15568: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:16.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:07:16 smithi153 ceph-mon[70044]: pgmap v15568: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:18.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:07:18 smithi153 ceph-mon[70044]: pgmap v15569: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:18.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:07:18 smithi083 ceph-mon[79930]: pgmap v15569: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:18.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:07:18 smithi083 ceph-mon[77242]: pgmap v15569: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:20.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:07:20 smithi153 ceph-mon[70044]: pgmap v15570: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:20.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:07:20 smithi083 ceph-mon[79930]: pgmap v15570: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:20.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:07:20 smithi083 ceph-mon[77242]: pgmap v15570: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:07:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:07:21] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T14:07:22.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:07:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:07:22.103Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:07:22.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:07:22 smithi153 ceph-mon[70044]: pgmap v15571: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:22.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:07:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:07:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:07:22.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:07:22 smithi083 ceph-mon[79930]: pgmap v15571: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:22.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:07:22 smithi083 ceph-mon[77242]: pgmap v15571: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:07:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:07:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:07:24.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:07:24 smithi083 ceph-mon[79930]: pgmap v15572: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:24.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:07:24 smithi083 ceph-mon[77242]: pgmap v15572: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:25.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:07:24 smithi153 ceph-mon[70044]: pgmap v15572: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:25.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:07:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:07:25.463Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:07:26.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:07:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:07:26.193Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:07:26.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:07:26 smithi083 ceph-mon[79930]: pgmap v15573: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:26.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:07:26 smithi083 ceph-mon[77242]: pgmap v15573: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:27.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:07:26 smithi153 ceph-mon[70044]: pgmap v15573: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:28.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:07:28 smithi083 ceph-mon[79930]: pgmap v15574: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:28.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:07:28 smithi083 ceph-mon[77242]: pgmap v15574: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:29.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:07:28 smithi153 ceph-mon[70044]: pgmap v15574: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:30.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:07:30 smithi083 ceph-mon[77242]: pgmap v15575: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:30.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:07:30 smithi083 ceph-mon[79930]: pgmap v15575: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:31.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:07:30 smithi153 ceph-mon[70044]: pgmap v15575: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:07:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:07:31] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T14:07:32.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:07:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:07:32.104Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:07:32.575 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:07:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:07:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:07:32.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:07:32 smithi083 ceph-mon[79930]: pgmap v15576: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:32.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:07:32 smithi083 ceph-mon[77242]: pgmap v15576: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:33.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:07:32 smithi153 ceph-mon[70044]: pgmap v15576: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:07:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:07:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:07:34.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:07:34 smithi083 ceph-mon[79930]: pgmap v15577: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:34.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:07:34 smithi083 ceph-mon[77242]: pgmap v15577: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:35.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:07:34 smithi153 ceph-mon[70044]: pgmap v15577: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:35.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:07:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:07:35.464Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:07:36.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:07:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:07:36.194Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:07:36.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:07:36 smithi083 ceph-mon[79930]: pgmap v15578: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:36.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:07:36 smithi083 ceph-mon[77242]: pgmap v15578: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:37.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:07:36 smithi153 ceph-mon[70044]: pgmap v15578: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:38.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:07:38 smithi083 ceph-mon[79930]: pgmap v15579: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:38.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:07:38 smithi083 ceph-mon[77242]: pgmap v15579: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:39.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:07:38 smithi153 ceph-mon[70044]: pgmap v15579: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:39.739 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:07:40.028 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:07:40.029 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 26s ago 8h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:07:40.029 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 8m ago 8h 64.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:07:40.029 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 26s ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:07:40.029 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 8m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:07:40.030 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 8m ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:07:40.030 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 26s ago 8h 943M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:07:40.030 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 26s ago 8h 1914M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:07:40.030 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 8m ago 8h 816M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:07:40.031 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 26s ago 8h 1143M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:07:40.031 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 26s ago 8h 24.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:07:40.031 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 8m ago 8h 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:07:40.031 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 26s ago 8h 1386M 2262M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:07:40.032 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 26s ago 8h 1373M 2262M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:07:40.032 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 26s ago 8h 1068M 2262M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:07:40.032 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 26s ago 8h 1345M 2262M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:07:40.032 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 8m ago 8h 1158M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:07:40.032 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 8m ago 8h 1452M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:07:40.033 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 8m ago 8h 1298M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:07:40.033 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 8m ago 8h 1542M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:07:40.033 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 8m ago 8h 186M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:07:40.381 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:07:40.382 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:07:40.382 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:07:40.382 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:07:40.383 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:07:40.383 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:07:40.383 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:07:40.384 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:07:40.384 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:07:40.384 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:07:40.384 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:07:40.385 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:07:40.385 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:07:40.385 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:07:40.385 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:07:40.385 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:07:40.386 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:07:40.386 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:07:40.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:07:40 smithi083 ceph-mon[77242]: pgmap v15580: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:07:40 smithi083 ceph-mon[77242]: from='client.52248 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:07:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:07:40 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2044160771' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:07:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:07:40 smithi083 ceph-mon[79930]: pgmap v15580: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:40.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:07:40 smithi083 ceph-mon[79930]: from='client.52248 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:07:40.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:07:40 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2044160771' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:07:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:07:40 smithi153 ceph-mon[70044]: pgmap v15580: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:41.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:07:40 smithi153 ceph-mon[70044]: from='client.52248 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:07:41.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:07:40 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2044160771' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:07:41.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:07:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:07:41] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T14:07:41.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:07:41 smithi083 ceph-mon[77242]: from='client.52254 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:07:41.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:07:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:07:41.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:07:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:07:41.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:07:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:07:41.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:07:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:07:41.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:07:41 smithi083 ceph-mon[79930]: from='client.52254 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:07:41.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:07:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:07:41.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:07:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:07:41.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:07:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:07:41.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:07:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:07:42.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:07:41 smithi153 ceph-mon[70044]: from='client.52254 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:07:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:07:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:07:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:07:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:07:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:07:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:07:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:07:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:07:42.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:07:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:07:42.105Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:07:42.591 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:07:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:07:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:07:42.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:07:42 smithi083 ceph-mon[79930]: pgmap v15581: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:42.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:07:42 smithi083 ceph-mon[77242]: pgmap v15581: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:43.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:07:42 smithi153 ceph-mon[70044]: pgmap v15581: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:07:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:07:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:07:44.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:07:44 smithi083 ceph-mon[79930]: pgmap v15582: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:44.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:07:44 smithi083 ceph-mon[77242]: pgmap v15582: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:45.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:07:44 smithi153 ceph-mon[70044]: pgmap v15582: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:45.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:07:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:07:45.464Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:07:46.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:07:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:07:46.194Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:07:46.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:07:46 smithi083 ceph-mon[79930]: pgmap v15583: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:46.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:07:46 smithi083 ceph-mon[77242]: pgmap v15583: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:47.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:07:46 smithi153 ceph-mon[70044]: pgmap v15583: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:48.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:07:48 smithi083 ceph-mon[79930]: pgmap v15584: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:48.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:07:48 smithi083 ceph-mon[77242]: pgmap v15584: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:49.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:07:48 smithi153 ceph-mon[70044]: pgmap v15584: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:50.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:07:50 smithi083 ceph-mon[79930]: pgmap v15585: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:50.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:07:50 smithi083 ceph-mon[77242]: pgmap v15585: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:51.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:07:50 smithi153 ceph-mon[70044]: pgmap v15585: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:07:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:07:51] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T14:07:52.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:07:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:07:52.106Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:07:52.609 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:07:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:07:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:07:52.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:07:52 smithi083 ceph-mon[79930]: pgmap v15586: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:52.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:07:52 smithi083 ceph-mon[77242]: pgmap v15586: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:53.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:07:52 smithi153 ceph-mon[70044]: pgmap v15586: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:07:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:07:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:07:54.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:07:54 smithi083 ceph-mon[79930]: pgmap v15587: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:54.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:07:54 smithi083 ceph-mon[77242]: pgmap v15587: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:55.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:07:54 smithi153 ceph-mon[70044]: pgmap v15587: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:55.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:07:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:07:55.464Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:07:56.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:07:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:07:56.195Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:07:56.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:07:56 smithi083 ceph-mon[77242]: pgmap v15588: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:56.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:07:56 smithi083 ceph-mon[79930]: pgmap v15588: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:57.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:07:56 smithi153 ceph-mon[70044]: pgmap v15588: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:58.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:07:58 smithi083 ceph-mon[77242]: pgmap v15589: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:58.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:07:58 smithi083 ceph-mon[79930]: pgmap v15589: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:07:59.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:07:58 smithi153 ceph-mon[70044]: pgmap v15589: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:00.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:08:00 smithi083 ceph-mon[79930]: pgmap v15590: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:00.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:08:00 smithi083 ceph-mon[77242]: pgmap v15590: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:01.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:08:00 smithi153 ceph-mon[70044]: pgmap v15590: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:01.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:08:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:08:01] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T14:08:02.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:08:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:08:02.106Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:08:02.625 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:08:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:08:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:08:02.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:08:02 smithi083 ceph-mon[79930]: pgmap v15591: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:02.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:08:02 smithi083 ceph-mon[77242]: pgmap v15591: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:03.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:08:02 smithi153 ceph-mon[70044]: pgmap v15591: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:08:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:08:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:08:04.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:08:04 smithi083 ceph-mon[79930]: pgmap v15592: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:04.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:08:04 smithi083 ceph-mon[77242]: pgmap v15592: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:05.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:08:04 smithi153 ceph-mon[70044]: pgmap v15592: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:05.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:08:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:08:05.465Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:08:06.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:08:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:08:06.196Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:08:06.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:08:06 smithi083 ceph-mon[79930]: pgmap v15593: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:06.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:08:06 smithi083 ceph-mon[77242]: pgmap v15593: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:07.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:08:06 smithi153 ceph-mon[70044]: pgmap v15593: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:08.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:08:08 smithi083 ceph-mon[79930]: pgmap v15594: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:08.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:08:08 smithi083 ceph-mon[77242]: pgmap v15594: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:09.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:08:08 smithi153 ceph-mon[70044]: pgmap v15594: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:10.712 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:08:10.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:08:10 smithi083 ceph-mon[77242]: pgmap v15595: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:10.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:08:10 smithi083 ceph-mon[79930]: pgmap v15595: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:11.001 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:08:11.001 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 57s ago 8h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:08:11.001 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 9m ago 8h 64.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:08:11.001 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 57s ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:08:11.002 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 9m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:08:11.002 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 9m ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:08:11.002 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 57s ago 8h 943M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:08:11.002 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 57s ago 8h 1914M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:08:11.003 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 9m ago 8h 816M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:08:11.003 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 57s ago 8h 1143M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:08:11.003 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 57s ago 8h 24.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:08:11.003 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 9m ago 8h 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:08:11.004 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 57s ago 8h 1386M 2262M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:08:11.004 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 57s ago 8h 1373M 2262M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:08:11.004 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 57s ago 8h 1068M 2262M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:08:11.004 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 57s ago 8h 1345M 2262M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:08:11.004 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 9m ago 8h 1158M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:08:11.005 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 9m ago 8h 1452M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:08:11.005 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 9m ago 8h 1298M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:08:11.005 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 9m ago 8h 1542M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:08:11.005 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 9m ago 8h 186M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:08:11.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:08:10 smithi153 ceph-mon[70044]: pgmap v15595: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:11.358 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:08:11.359 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:08:11.359 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:08:11.359 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:08:11.360 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:08:11.360 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:08:11.360 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:08:11.360 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:08:11.360 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:08:11.361 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:08:11.361 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:08:11.361 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:08:11.361 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:08:11.362 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:08:11.362 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:08:11.362 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:08:11.362 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:08:11.363 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:08:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:08:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:08:11] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T14:08:11.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:08:11 smithi083 ceph-mon[77242]: from='client.62197 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:08:11.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:08:11 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4101953526' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:08:11.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:08:11 smithi083 ceph-mon[79930]: from='client.62197 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:08:11.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:08:11 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4101953526' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:08:12.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:08:11 smithi153 ceph-mon[70044]: from='client.62197 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:08:12.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:08:11 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4101953526' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:08:12.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:08:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:08:12.107Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:08:12.647 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:08:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:08:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:08:12.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:08:12 smithi083 ceph-mon[77242]: from='client.52272 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:08:12.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:08:12 smithi083 ceph-mon[77242]: pgmap v15596: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:12.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:08:12 smithi083 ceph-mon[79930]: from='client.52272 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:08:12.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:08:12 smithi083 ceph-mon[79930]: pgmap v15596: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:13.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:08:12 smithi153 ceph-mon[70044]: from='client.52272 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:08:13.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:08:12 smithi153 ceph-mon[70044]: pgmap v15596: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:08:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:08:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:08:14.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:08:14 smithi083 ceph-mon[79930]: pgmap v15597: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:14.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:08:14 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:08:14.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:08:14 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:08:14.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:08:14 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:08:14.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:08:14 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:08:14.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:08:14 smithi083 ceph-mon[77242]: pgmap v15597: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:14.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:08:14 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:08:14.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:08:14 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:08:14.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:08:14 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:08:14.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:08:14 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:08:15.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:08:14 smithi153 ceph-mon[70044]: pgmap v15597: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:15.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:08:14 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:08:15.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:08:14 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:08:15.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:08:14 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:08:15.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:08:14 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:08:15.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:08:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:08:15.466Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:08:16.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:08:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:08:16.197Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:08:16.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:08:16 smithi083 ceph-mon[77242]: pgmap v15598: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:16.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:08:16 smithi083 ceph-mon[79930]: pgmap v15598: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:17.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:08:16 smithi153 ceph-mon[70044]: pgmap v15598: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:18.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:08:18 smithi083 ceph-mon[79930]: pgmap v15599: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:18.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:08:18 smithi083 ceph-mon[77242]: pgmap v15599: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:19.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:08:18 smithi153 ceph-mon[70044]: pgmap v15599: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:20.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:08:20 smithi083 ceph-mon[79930]: pgmap v15600: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:20.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:08:20 smithi083 ceph-mon[77242]: pgmap v15600: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:21.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:08:20 smithi153 ceph-mon[70044]: pgmap v15600: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:08:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:08:21] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T14:08:22.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:08:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:08:22.108Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:08:22.663 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:08:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:08:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:08:22.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:08:22 smithi083 ceph-mon[79930]: pgmap v15601: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:22.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:08:22 smithi083 ceph-mon[77242]: pgmap v15601: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:23.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:08:22 smithi153 ceph-mon[70044]: pgmap v15601: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:08:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:08:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:08:24.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:08:24 smithi083 ceph-mon[79930]: pgmap v15602: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:24.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:08:24 smithi083 ceph-mon[77242]: pgmap v15602: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:25.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:08:24 smithi153 ceph-mon[70044]: pgmap v15602: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:25.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:08:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:08:25.466Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:08:26.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:08:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:08:26.198Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:08:26.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:08:26 smithi083 ceph-mon[79930]: pgmap v15603: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:26.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:08:26 smithi083 ceph-mon[77242]: pgmap v15603: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:27.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:08:26 smithi153 ceph-mon[70044]: pgmap v15603: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:28.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:08:28 smithi083 ceph-mon[79930]: pgmap v15604: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:28.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:08:28 smithi083 ceph-mon[77242]: pgmap v15604: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:29.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:08:28 smithi153 ceph-mon[70044]: pgmap v15604: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:30.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:08:30 smithi083 ceph-mon[79930]: pgmap v15605: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:30.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:08:30 smithi083 ceph-mon[77242]: pgmap v15605: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:31.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:08:30 smithi153 ceph-mon[70044]: pgmap v15605: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:08:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:08:31] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T14:08:32.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:08:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:08:32.109Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:08:32.676 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:08:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:08:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:08:32.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:08:32 smithi083 ceph-mon[79930]: pgmap v15606: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:32.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:08:32 smithi083 ceph-mon[77242]: pgmap v15606: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:33.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:08:32 smithi153 ceph-mon[70044]: pgmap v15606: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:08:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:08:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:08:34.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:08:34 smithi083 ceph-mon[77242]: pgmap v15607: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:34.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:08:34 smithi083 ceph-mon[79930]: pgmap v15607: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:35.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:08:34 smithi153 ceph-mon[70044]: pgmap v15607: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:35.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:08:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:08:35.466Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:08:36.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:08:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:08:36.199Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:08:36.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:08:36 smithi083 ceph-mon[79930]: pgmap v15608: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:36.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:08:36 smithi083 ceph-mon[77242]: pgmap v15608: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:37.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:08:36 smithi153 ceph-mon[70044]: pgmap v15608: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:38.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:08:38 smithi083 ceph-mon[79930]: pgmap v15609: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:38.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:08:38 smithi083 ceph-mon[77242]: pgmap v15609: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:39.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:08:38 smithi153 ceph-mon[70044]: pgmap v15609: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:08:40 smithi083 ceph-mon[79930]: pgmap v15610: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:08:40 smithi083 ceph-mon[77242]: pgmap v15610: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:08:40 smithi153 ceph-mon[70044]: pgmap v15610: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:41.686 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:08:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:08:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:08:41] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T14:08:41.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:08:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:08:41.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:08:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:08:41.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:08:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:08:41.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:08:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:08:41.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:08:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:08:41.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:08:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:08:41.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:08:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:08:41.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:08:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:08:41.978 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:08:41.978 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 88s ago 8h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:08:41.978 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 9m ago 8h 64.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:08:41.979 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 88s ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:08:41.979 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 9m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:08:41.979 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 9m ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:08:41.979 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 88s ago 8h 943M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:08:41.980 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 88s ago 8h 1914M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:08:41.980 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 9m ago 8h 816M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:08:41.980 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 88s ago 8h 1143M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:08:41.980 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 88s ago 8h 24.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:08:41.980 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 9m ago 8h 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:08:41.981 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 88s ago 8h 1386M 2262M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:08:41.981 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 88s ago 8h 1373M 2262M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:08:41.981 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 88s ago 8h 1068M 2262M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:08:41.981 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 88s ago 8h 1345M 2262M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:08:41.982 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 9m ago 8h 1158M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:08:41.982 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 9m ago 8h 1452M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:08:41.982 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 9m ago 8h 1298M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:08:41.982 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 9m ago 8h 1542M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:08:41.983 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 9m ago 8h 186M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:08:42.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:08:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:08:42.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:08:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:08:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:08:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:08:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:08:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:08:42.331 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:08:42.332 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:08:42.332 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:08:42.332 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:08:42.332 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:08:42.333 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:08:42.333 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:08:42.333 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:08:42.333 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:08:42.334 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:08:42.334 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:08:42.334 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:08:42.334 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:08:42.335 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:08:42.335 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:08:42.335 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:08:42.335 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:08:42.336 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:08:42.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:08:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:08:42.109Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:08:42.696 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:08:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:08:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:08:42.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:08:42 smithi083 ceph-mon[77242]: pgmap v15611: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:42.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:08:42 smithi083 ceph-mon[77242]: from='client.52284 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:08:42.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:08:42 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1307608694' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:08:42.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:08:42 smithi083 ceph-mon[79930]: pgmap v15611: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:42.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:08:42 smithi083 ceph-mon[79930]: from='client.52284 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:08:42.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:08:42 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1307608694' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:08:43.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:08:42 smithi153 ceph-mon[70044]: pgmap v15611: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:43.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:08:42 smithi153 ceph-mon[70044]: from='client.52284 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:08:43.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:08:42 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1307608694' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:08:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:08:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:08:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:08:43.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:08:43 smithi083 ceph-mon[79930]: from='client.52290 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:08:43.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:08:43 smithi083 ceph-mon[77242]: from='client.52290 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:08:44.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:08:43 smithi153 ceph-mon[70044]: from='client.52290 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:08:44.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:08:44 smithi083 ceph-mon[79930]: pgmap v15612: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:44.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:08:44 smithi083 ceph-mon[77242]: pgmap v15612: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:45.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:08:44 smithi153 ceph-mon[70044]: pgmap v15612: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:45.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:08:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:08:45.467Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:08:46.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:08:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:08:46.199Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:08:46.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:08:46 smithi083 ceph-mon[77242]: pgmap v15613: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:46.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:08:46 smithi083 ceph-mon[79930]: pgmap v15613: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:47.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:08:46 smithi153 ceph-mon[70044]: pgmap v15613: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:48.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:08:48 smithi083 ceph-mon[79930]: pgmap v15614: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:48.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:08:48 smithi083 ceph-mon[77242]: pgmap v15614: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:49.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:08:48 smithi153 ceph-mon[70044]: pgmap v15614: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:50.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:08:50 smithi083 ceph-mon[77242]: pgmap v15615: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:50.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:08:50 smithi083 ceph-mon[79930]: pgmap v15615: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:51.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:08:50 smithi153 ceph-mon[70044]: pgmap v15615: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:08:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:08:51] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T14:08:52.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:08:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:08:52.110Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:08:52.715 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:08:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:08:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:08:53.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:08:52 smithi153 ceph-mon[70044]: pgmap v15616: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:53.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:08:52 smithi083 ceph-mon[79930]: pgmap v15616: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:53.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:08:52 smithi083 ceph-mon[77242]: pgmap v15616: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:08:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:08:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:08:55.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:08:54 smithi153 ceph-mon[70044]: pgmap v15617: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:55.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:08:54 smithi083 ceph-mon[77242]: pgmap v15617: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:55.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:08:54 smithi083 ceph-mon[79930]: pgmap v15617: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:55.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:08:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:08:55.468Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:08:56.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:08:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:08:56.200Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:08:57.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:08:56 smithi153 ceph-mon[70044]: pgmap v15618: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:57.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:08:56 smithi083 ceph-mon[77242]: pgmap v15618: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:57.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:08:56 smithi083 ceph-mon[79930]: pgmap v15618: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:59.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:08:58 smithi153 ceph-mon[70044]: pgmap v15619: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:59.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:08:58 smithi083 ceph-mon[79930]: pgmap v15619: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:08:59.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:08:58 smithi083 ceph-mon[77242]: pgmap v15619: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:01.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:09:00 smithi153 ceph-mon[70044]: pgmap v15620: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:01.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:09:00 smithi083 ceph-mon[77242]: pgmap v15620: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:01.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:09:00 smithi083 ceph-mon[79930]: pgmap v15620: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:09:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:09:01] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T14:09:02.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:09:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:09:02.110Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:09:02.732 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:09:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:09:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:09:03.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:09:02 smithi153 ceph-mon[70044]: pgmap v15621: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:03.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:09:02 smithi083 ceph-mon[79930]: pgmap v15621: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:03.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:09:02 smithi083 ceph-mon[77242]: pgmap v15621: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:09:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:09:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:09:05.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:09:04 smithi153 ceph-mon[70044]: pgmap v15622: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:05.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:09:04 smithi083 ceph-mon[79930]: pgmap v15622: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:05.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:09:04 smithi083 ceph-mon[77242]: pgmap v15622: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:05.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:09:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:09:05.468Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:09:06.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:09:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:09:06.201Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:09:07.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:09:06 smithi153 ceph-mon[70044]: pgmap v15623: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:07.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:09:06 smithi083 ceph-mon[79930]: pgmap v15623: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:07.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:09:06 smithi083 ceph-mon[77242]: pgmap v15623: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:09.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:09:08 smithi153 ceph-mon[70044]: pgmap v15624: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:09.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:09:08 smithi083 ceph-mon[79930]: pgmap v15624: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:09.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:09:08 smithi083 ceph-mon[77242]: pgmap v15624: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:11.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:09:10 smithi153 ceph-mon[70044]: pgmap v15625: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:11.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:09:10 smithi083 ceph-mon[79930]: pgmap v15625: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:11.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:09:10 smithi083 ceph-mon[77242]: pgmap v15625: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:09:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:09:11] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T14:09:12.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:09:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:09:12.112Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:09:12.658 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:09:12.747 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:09:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:09:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:09:12.951 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:09:12.952 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 119s ago 8h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:09:12.952 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 10m ago 8h 64.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:09:12.952 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 119s ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:09:12.953 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 10m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:09:12.953 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 10m ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:09:12.954 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 119s ago 8h 943M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:09:12.954 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 119s ago 8h 1914M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:09:12.955 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 10m ago 8h 816M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:09:12.955 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 119s ago 8h 1143M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:09:12.956 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 119s ago 8h 24.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:09:12.956 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 10m ago 8h 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:09:12.957 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 119s ago 8h 1386M 2262M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:09:12.957 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 119s ago 8h 1373M 2262M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:09:12.958 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 119s ago 8h 1068M 2262M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:09:12.958 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 119s ago 8h 1345M 2262M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:09:12.959 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 10m ago 8h 1158M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:09:12.959 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 10m ago 8h 1452M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:09:12.960 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 10m ago 8h 1298M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:09:12.960 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 10m ago 8h 1542M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:09:12.961 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 10m ago 8h 186M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:09:13.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:09:12 smithi153 ceph-mon[70044]: pgmap v15626: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:13.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:09:12 smithi083 ceph-mon[79930]: pgmap v15626: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:13.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:09:12 smithi083 ceph-mon[77242]: pgmap v15626: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:13.304 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:09:13.304 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:09:13.305 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:09:13.305 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:09:13.305 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:09:13.305 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:09:13.306 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:09:13.306 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:09:13.306 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:09:13.306 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:09:13.307 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:09:13.307 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:09:13.307 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:09:13.308 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:09:13.308 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:09:13.308 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:09:13.308 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:09:13.309 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:09:13.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:09:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:09:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:09:14.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:09:13 smithi153 ceph-mon[70044]: from='client.62233 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:09:14.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:09:13 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/940230406' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:09:14.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:09:13 smithi083 ceph-mon[79930]: from='client.62233 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:09:14.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:09:13 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/940230406' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:09:14.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:09:13 smithi083 ceph-mon[77242]: from='client.62233 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:09:14.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:09:13 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/940230406' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:09:15.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:09:14 smithi153 ceph-mon[70044]: from='client.52308 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:09:15.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:09:14 smithi153 ceph-mon[70044]: pgmap v15627: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:15.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:09:14 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:09:15.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:09:14 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:09:15.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:09:14 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:09:15.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:09:14 smithi083 ceph-mon[79930]: from='client.52308 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:09:15.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:09:14 smithi083 ceph-mon[79930]: pgmap v15627: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:15.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:09:14 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:09:15.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:09:14 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:09:15.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:09:14 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:09:15.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:09:14 smithi083 ceph-mon[77242]: from='client.52308 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:09:15.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:09:14 smithi083 ceph-mon[77242]: pgmap v15627: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:15.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:09:14 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:09:15.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:09:14 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:09:15.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:09:14 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:09:15.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:09:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:09:15.468Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:09:16.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:09:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:09:16.201Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:09:17.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:09:16 smithi153 ceph-mon[70044]: pgmap v15628: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:17.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:09:16 smithi083 ceph-mon[79930]: pgmap v15628: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:17.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:09:16 smithi083 ceph-mon[77242]: pgmap v15628: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:19.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:09:18 smithi153 ceph-mon[70044]: pgmap v15629: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:19.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:09:18 smithi083 ceph-mon[79930]: pgmap v15629: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:19.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:09:18 smithi083 ceph-mon[77242]: pgmap v15629: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:20.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:09:20 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:09:20.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:09:20 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:09:20.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:09:20 smithi083 ceph-mon[79930]: pgmap v15630: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:20.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:09:20 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:09:20.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:09:20 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:09:20.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:09:20 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:09:20.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:09:20 smithi083 ceph-mon[77242]: pgmap v15630: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:20.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:09:20 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:09:20.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:09:20 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:09:20.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:09:20 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:09:20.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:09:20 smithi153 ceph-mon[70044]: pgmap v15630: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:20.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:09:20 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:09:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:09:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:09:21] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T14:09:22.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:09:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:09:22.112Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:09:22.580 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:09:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:09:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:09:22.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:09:22 smithi083 ceph-mon[79930]: pgmap v15631: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:22.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:09:22 smithi083 ceph-mon[77242]: pgmap v15631: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:23.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:09:22 smithi153 ceph-mon[70044]: pgmap v15631: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:09:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:09:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:09:24.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:09:24 smithi083 ceph-mon[79930]: pgmap v15632: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:24.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:09:24 smithi083 ceph-mon[77242]: pgmap v15632: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:25.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:09:24 smithi153 ceph-mon[70044]: pgmap v15632: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:25.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:09:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:09:25.470Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:09:26.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:09:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:09:26.202Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:09:26.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:09:26 smithi083 ceph-mon[79930]: pgmap v15633: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:26.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:09:26 smithi083 ceph-mon[77242]: pgmap v15633: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:27.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:09:26 smithi153 ceph-mon[70044]: pgmap v15633: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:28.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:09:28 smithi083 ceph-mon[77242]: pgmap v15634: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:28.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:09:28 smithi083 ceph-mon[79930]: pgmap v15634: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:29.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:09:28 smithi153 ceph-mon[70044]: pgmap v15634: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:30.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:09:30 smithi083 ceph-mon[79930]: pgmap v15635: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:30.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:09:30 smithi083 ceph-mon[77242]: pgmap v15635: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:31.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:09:30 smithi153 ceph-mon[70044]: pgmap v15635: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:09:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:09:31] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T14:09:32.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:09:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:09:32.113Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:09:32.594 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:09:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:09:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:09:32.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:09:32 smithi083 ceph-mon[77242]: pgmap v15636: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:32.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:09:32 smithi083 ceph-mon[79930]: pgmap v15636: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:33.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:09:32 smithi153 ceph-mon[70044]: pgmap v15636: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:09:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:09:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:09:34.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:09:34 smithi083 ceph-mon[79930]: pgmap v15637: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:34.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:09:34 smithi083 ceph-mon[77242]: pgmap v15637: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:35.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:09:34 smithi153 ceph-mon[70044]: pgmap v15637: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:35.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:09:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:09:35.470Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:09:36.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:09:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:09:36.203Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:09:36.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:09:36 smithi083 ceph-mon[79930]: pgmap v15638: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:36.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:09:36 smithi083 ceph-mon[77242]: pgmap v15638: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:37.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:09:36 smithi153 ceph-mon[70044]: pgmap v15638: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:38.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:09:38 smithi083 ceph-mon[79930]: pgmap v15639: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:38.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:09:38 smithi083 ceph-mon[77242]: pgmap v15639: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:39.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:09:38 smithi153 ceph-mon[70044]: pgmap v15639: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:09:40 smithi083 ceph-mon[79930]: pgmap v15640: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:09:40 smithi083 ceph-mon[77242]: pgmap v15640: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:09:40 smithi153 ceph-mon[70044]: pgmap v15640: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:09:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:09:41] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T14:09:41.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:09:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:09:41.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:09:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:09:41.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:09:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:09:41.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:09:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:09:41.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:09:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:09:41.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:09:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:09:41.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:09:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:09:41.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:09:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:09:42.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:09:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:09:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:09:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:09:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:09:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:09:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:09:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:09:42.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:09:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:09:42.114Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:09:42.616 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:09:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:09:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:09:42.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:09:42 smithi083 ceph-mon[79930]: pgmap v15641: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:42.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:09:42 smithi083 ceph-mon[77242]: pgmap v15641: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:43.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:09:42 smithi153 ceph-mon[70044]: pgmap v15641: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:09:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:09:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:09:43.639 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:09:43.932 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:09:43.933 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 2m ago 8h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:09:43.933 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 24s ago 8h 63.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:09:43.933 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 2m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:09:43.933 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 24s ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:09:43.934 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 24s ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:09:43.934 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 2m ago 8h 943M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:09:43.934 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 2m ago 8h 1914M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:09:43.934 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 24s ago 8h 830M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:09:43.935 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 2m ago 8h 1143M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:09:43.935 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 2m ago 8h 24.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:09:43.935 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 24s ago 8h 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:09:43.936 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 2m ago 8h 1386M 2262M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:09:43.936 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 2m ago 8h 1373M 2262M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:09:43.936 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 2m ago 8h 1068M 2262M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:09:43.936 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 2m ago 8h 1345M 2262M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:09:43.936 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 24s ago 8h 1180M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:09:43.937 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 24s ago 8h 1480M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:09:43.937 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 24s ago 8h 1323M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:09:43.937 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 24s ago 8h 1572M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:09:43.938 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 24s ago 8h 189M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:09:44.290 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:09:44.291 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:09:44.291 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:09:44.291 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:09:44.292 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:09:44.292 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:09:44.292 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:09:44.292 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:09:44.293 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:09:44.293 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:09:44.293 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:09:44.293 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:09:44.294 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:09:44.294 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:09:44.294 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:09:44.294 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:09:44.295 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:09:44.295 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:09:44.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:09:44 smithi083 ceph-mon[79930]: pgmap v15642: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:44.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:09:44 smithi083 ceph-mon[79930]: from='client.62251 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:09:44.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:09:44 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/315614407' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:09:44.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:09:44 smithi083 ceph-mon[77242]: pgmap v15642: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:44.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:09:44 smithi083 ceph-mon[77242]: from='client.62251 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:09:44.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:09:44 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/315614407' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:09:45.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:09:44 smithi153 ceph-mon[70044]: pgmap v15642: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:45.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:09:44 smithi153 ceph-mon[70044]: from='client.62251 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:09:45.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:09:44 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/315614407' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:09:45.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:09:45 smithi083 ceph-mon[79930]: from='client.52326 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:09:45.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:09:45 smithi083 ceph-mon[77242]: from='client.52326 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:09:45.964 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:09:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:09:45.471Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:09:46.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:09:45 smithi153 ceph-mon[70044]: from='client.52326 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:09:46.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:09:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:09:46.204Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:09:46.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:09:46 smithi083 ceph-mon[79930]: pgmap v15643: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:46.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:09:46 smithi083 ceph-mon[77242]: pgmap v15643: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:47.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:09:46 smithi153 ceph-mon[70044]: pgmap v15643: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:48.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:09:48 smithi083 ceph-mon[79930]: pgmap v15644: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:48.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:09:48 smithi083 ceph-mon[77242]: pgmap v15644: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:49.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:09:48 smithi153 ceph-mon[70044]: pgmap v15644: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:50.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:09:50 smithi083 ceph-mon[79930]: pgmap v15645: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:50.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:09:50 smithi083 ceph-mon[77242]: pgmap v15645: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:51.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:09:50 smithi153 ceph-mon[70044]: pgmap v15645: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:09:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:09:51] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T14:09:52.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:09:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:09:52.114Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:09:52.634 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:09:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:09:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:09:52.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:09:52 smithi083 ceph-mon[79930]: pgmap v15646: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:52.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:09:52 smithi083 ceph-mon[77242]: pgmap v15646: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:53.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:09:52 smithi153 ceph-mon[70044]: pgmap v15646: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:09:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:09:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:09:54.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:09:54 smithi083 ceph-mon[79930]: pgmap v15647: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:54.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:09:54 smithi083 ceph-mon[77242]: pgmap v15647: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:55.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:09:54 smithi153 ceph-mon[70044]: pgmap v15647: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:55.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:09:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:09:55.471Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:09:56.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:09:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:09:56.204Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:09:56.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:09:56 smithi083 ceph-mon[79930]: pgmap v15648: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:56.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:09:56 smithi083 ceph-mon[77242]: pgmap v15648: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:57.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:09:56 smithi153 ceph-mon[70044]: pgmap v15648: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:58.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:09:58 smithi083 ceph-mon[79930]: pgmap v15649: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:58.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:09:58 smithi083 ceph-mon[77242]: pgmap v15649: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:09:59.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:09:58 smithi153 ceph-mon[70044]: pgmap v15649: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:00.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:10:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[77219]: 2022-09-29T14:09:59.999+0000 7f160f665700 -1 log_channel(cluster) log [ERR] : overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T14:10:00.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:10:00 smithi083 ceph-mon[77242]: pgmap v15650: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:00.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:10:00 smithi083 ceph-mon[77242]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T14:10:00.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:10:00 smithi083 ceph-mon[79930]: pgmap v15650: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:00.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:10:00 smithi083 ceph-mon[79930]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T14:10:01.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:10:00 smithi153 ceph-mon[70044]: pgmap v15650: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:01.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:10:00 smithi153 ceph-mon[70044]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T14:10:01.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:10:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:10:01] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T14:10:02.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:10:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:10:02.115Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:10:02.651 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:10:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:10:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:10:02.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:10:02 smithi083 ceph-mon[79930]: pgmap v15651: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:02.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:10:02 smithi083 ceph-mon[77242]: pgmap v15651: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:03.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:10:02 smithi153 ceph-mon[70044]: pgmap v15651: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:10:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:10:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:10:04.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:10:04 smithi083 ceph-mon[79930]: pgmap v15652: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:04.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:10:04 smithi083 ceph-mon[77242]: pgmap v15652: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:05.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:10:04 smithi153 ceph-mon[70044]: pgmap v15652: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:05.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:10:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:10:05.471Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:10:06.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:10:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:10:06.206Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:10:06.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:10:06 smithi083 ceph-mon[79930]: pgmap v15653: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:06.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:10:06 smithi083 ceph-mon[77242]: pgmap v15653: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:07.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:10:06 smithi153 ceph-mon[70044]: pgmap v15653: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:08.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:10:08 smithi083 ceph-mon[79930]: pgmap v15654: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:08.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:10:08 smithi083 ceph-mon[77242]: pgmap v15654: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:09.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:10:08 smithi153 ceph-mon[70044]: pgmap v15654: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:10.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:10:10 smithi083 ceph-mon[79930]: pgmap v15655: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:10.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:10:10 smithi083 ceph-mon[77242]: pgmap v15655: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:11.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:10:10 smithi153 ceph-mon[70044]: pgmap v15655: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:10:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:10:11] "GET /metrics HTTP/1.1" 200 225246 "" "Prometheus/2.33.4" 2022-09-29T14:10:12.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:10:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:10:12.116Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:10:12.664 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:10:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:10:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:10:12.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:10:12 smithi083 ceph-mon[79930]: pgmap v15656: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:12.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:10:12 smithi083 ceph-mon[77242]: pgmap v15656: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:13.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:10:12 smithi153 ceph-mon[70044]: pgmap v15656: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:10:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:10:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:10:14.616 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:10:14.905 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:10:14.906 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 3m ago 8h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:10:14.906 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 55s ago 8h 63.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:10:14.906 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 3m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:10:14.907 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 55s ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:10:14.907 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 55s ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:10:14.907 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 3m ago 8h 943M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:10:14.907 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 3m ago 8h 1914M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:10:14.908 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 55s ago 8h 830M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:10:14.908 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 3m ago 8h 1143M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:10:14.908 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 3m ago 8h 24.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:10:14.908 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 55s ago 8h 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:10:14.909 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 3m ago 8h 1386M 2262M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:10:14.909 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 3m ago 8h 1373M 2262M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:10:14.909 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 3m ago 8h 1068M 2262M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:10:14.909 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 3m ago 8h 1345M 2262M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:10:14.910 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 55s ago 8h 1180M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:10:14.910 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 55s ago 8h 1480M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:10:14.910 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 55s ago 8h 1323M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:10:14.910 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 55s ago 8h 1572M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:10:14.911 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 55s ago 8h 189M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:10:14.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:10:14 smithi083 ceph-mon[79930]: pgmap v15657: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:14.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:10:14 smithi083 ceph-mon[77242]: pgmap v15657: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:15.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:10:14 smithi153 ceph-mon[70044]: pgmap v15657: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:15.263 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:10:15.264 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:10:15.264 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:10:15.264 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:10:15.264 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:10:15.264 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:10:15.265 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:10:15.265 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:10:15.265 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:10:15.265 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:10:15.266 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:10:15.266 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:10:15.266 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:10:15.266 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:10:15.266 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:10:15.267 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:10:15.267 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:10:15.267 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:10:15.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:10:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:10:15.471Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:10:15.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:10:15 smithi083 ceph-mon[77242]: from='client.62269 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:10:15.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:10:15 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1700354429' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:10:15.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:10:15 smithi083 ceph-mon[79930]: from='client.62269 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:10:15.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:10:15 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1700354429' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:10:16.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:10:15 smithi153 ceph-mon[70044]: from='client.62269 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:10:16.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:10:15 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1700354429' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:10:16.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:10:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:10:16.206Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:10:16.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:10:16 smithi083 ceph-mon[79930]: from='client.52344 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:10:16.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:10:16 smithi083 ceph-mon[79930]: pgmap v15658: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:16.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:10:16 smithi083 ceph-mon[77242]: from='client.52344 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:10:16.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:10:16 smithi083 ceph-mon[77242]: pgmap v15658: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:17.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:10:16 smithi153 ceph-mon[70044]: from='client.52344 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:10:17.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:10:16 smithi153 ceph-mon[70044]: pgmap v15658: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:18.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:10:18 smithi083 ceph-mon[79930]: pgmap v15659: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:18.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:10:18 smithi083 ceph-mon[77242]: pgmap v15659: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:19.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:10:18 smithi153 ceph-mon[70044]: pgmap v15659: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:20.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:10:20 smithi083 ceph-mon[79930]: pgmap v15660: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:20.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:10:20 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:10:20.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:10:20 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:10:20.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:10:20 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:10:20.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:10:20 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:10:20.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:10:20 smithi083 ceph-mon[77242]: pgmap v15660: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:20.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:10:20 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:10:20.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:10:20 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:10:20.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:10:20 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:10:20.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:10:20 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:10:21.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:10:20 smithi153 ceph-mon[70044]: pgmap v15660: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:21.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:10:20 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:10:21.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:10:20 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:10:21.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:10:20 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:10:21.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:10:20 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:10:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:10:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:10:21] "GET /metrics HTTP/1.1" 200 225246 "" "Prometheus/2.33.4" 2022-09-29T14:10:22.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:10:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:10:22.116Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:10:22.686 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:10:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:10:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:10:22.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:10:22 smithi083 ceph-mon[79930]: pgmap v15661: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:22.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:10:22 smithi083 ceph-mon[77242]: pgmap v15661: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:23.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:10:22 smithi153 ceph-mon[70044]: pgmap v15661: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:10:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:10:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:10:24.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:10:24 smithi083 ceph-mon[79930]: pgmap v15662: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:24.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:10:24 smithi083 ceph-mon[77242]: pgmap v15662: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:25.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:10:24 smithi153 ceph-mon[70044]: pgmap v15662: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:25.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:10:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:10:25.471Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:10:26.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:10:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:10:26.207Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:10:26.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:10:26 smithi083 ceph-mon[79930]: pgmap v15663: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:26.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:10:26 smithi083 ceph-mon[77242]: pgmap v15663: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:27.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:10:26 smithi153 ceph-mon[70044]: pgmap v15663: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:28.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:10:28 smithi083 ceph-mon[79930]: pgmap v15664: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:28.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:10:28 smithi083 ceph-mon[77242]: pgmap v15664: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:29.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:10:28 smithi153 ceph-mon[70044]: pgmap v15664: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:30.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:10:30 smithi083 ceph-mon[79930]: pgmap v15665: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:30.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:10:30 smithi083 ceph-mon[77242]: pgmap v15665: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:31.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:10:30 smithi153 ceph-mon[70044]: pgmap v15665: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:10:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:10:31] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T14:10:32.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:10:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:10:32.117Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:10:32.708 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:10:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:10:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:10:32.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:10:32 smithi083 ceph-mon[79930]: pgmap v15666: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:32.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:10:32 smithi083 ceph-mon[77242]: pgmap v15666: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:33.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:10:32 smithi153 ceph-mon[70044]: pgmap v15666: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:33.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:10:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:10:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:10:34.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:10:34 smithi083 ceph-mon[77242]: pgmap v15667: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:34.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:10:34 smithi083 ceph-mon[79930]: pgmap v15667: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:35.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:10:34 smithi153 ceph-mon[70044]: pgmap v15667: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:35.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:10:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:10:35.472Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:10:36.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:10:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:10:36.208Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:10:37.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:10:36 smithi153 ceph-mon[70044]: pgmap v15668: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:37.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:10:36 smithi083 ceph-mon[79930]: pgmap v15668: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:37.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:10:36 smithi083 ceph-mon[77242]: pgmap v15668: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:39.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:10:38 smithi153 ceph-mon[70044]: pgmap v15669: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:39.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:10:38 smithi083 ceph-mon[79930]: pgmap v15669: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:39.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:10:38 smithi083 ceph-mon[77242]: pgmap v15669: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:10:40 smithi153 ceph-mon[70044]: pgmap v15670: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:41.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:10:40 smithi083 ceph-mon[79930]: pgmap v15670: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:10:40 smithi083 ceph-mon[77242]: pgmap v15670: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:10:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:10:41] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T14:10:41.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:10:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:10:41.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:10:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:10:41.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:10:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:10:41.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:10:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:10:41.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:10:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:10:41.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:10:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:10:41.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:10:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:10:41.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:10:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:10:42.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:10:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:10:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:10:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:10:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:10:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:10:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:10:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:10:42.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:10:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:10:42.117Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:10:42.727 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:10:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:10:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:10:43.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:10:42 smithi153 ceph-mon[70044]: pgmap v15671: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:43.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:10:42 smithi083 ceph-mon[79930]: pgmap v15671: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:43.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:10:42 smithi083 ceph-mon[77242]: pgmap v15671: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:10:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:10:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:10:45.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:10:44 smithi153 ceph-mon[70044]: pgmap v15672: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:45.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:10:44 smithi083 ceph-mon[79930]: pgmap v15672: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:45.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:10:44 smithi083 ceph-mon[77242]: pgmap v15672: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:45.590 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:10:45.879 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:10:45.879 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 3m ago 8h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:10:45.879 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 86s ago 8h 63.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:10:45.879 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 3m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:10:45.880 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 86s ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:10:45.880 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 86s ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:10:45.880 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 3m ago 8h 943M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:10:45.880 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 3m ago 8h 1914M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:10:45.881 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 86s ago 8h 830M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:10:45.881 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 3m ago 8h 1143M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:10:45.881 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 3m ago 8h 24.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:10:45.881 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 86s ago 8h 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:10:45.882 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 3m ago 8h 1386M 2262M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:10:45.882 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 3m ago 8h 1373M 2262M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:10:45.882 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 3m ago 8h 1068M 2262M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:10:45.882 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 3m ago 8h 1345M 2262M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:10:45.882 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 86s ago 8h 1180M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:10:45.883 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 86s ago 8h 1480M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:10:45.883 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 86s ago 8h 1323M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:10:45.883 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 86s ago 8h 1572M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:10:45.883 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 86s ago 8h 189M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:10:45.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:10:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:10:45.472Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:10:46.239 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:10:46.239 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:10:46.239 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:10:46.239 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:10:46.240 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:10:46.240 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:10:46.240 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:10:46.240 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:10:46.241 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:10:46.241 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:10:46.241 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:10:46.241 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:10:46.241 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:10:46.242 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:10:46.242 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:10:46.243 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:10:46.243 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:10:46.244 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:10:46.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:10:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:10:46.208Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:10:47.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:10:46 smithi153 ceph-mon[70044]: pgmap v15673: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:47.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:10:46 smithi153 ceph-mon[70044]: from='client.52356 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:10:47.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:10:46 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1173871726' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:10:47.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:10:46 smithi083 ceph-mon[79930]: pgmap v15673: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:47.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:10:46 smithi083 ceph-mon[79930]: from='client.52356 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:10:47.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:10:46 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1173871726' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:10:47.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:10:46 smithi083 ceph-mon[77242]: pgmap v15673: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:47.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:10:46 smithi083 ceph-mon[77242]: from='client.52356 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:10:47.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:10:46 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1173871726' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:10:48.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:10:47 smithi153 ceph-mon[70044]: from='client.52362 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:10:48.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:10:47 smithi083 ceph-mon[77242]: from='client.52362 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:10:48.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:10:47 smithi083 ceph-mon[79930]: from='client.52362 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:10:49.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:10:48 smithi153 ceph-mon[70044]: pgmap v15674: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:49.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:10:48 smithi083 ceph-mon[79930]: pgmap v15674: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:49.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:10:48 smithi083 ceph-mon[77242]: pgmap v15674: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:51.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:10:50 smithi153 ceph-mon[70044]: pgmap v15675: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:51.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:10:50 smithi083 ceph-mon[79930]: pgmap v15675: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:51.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:10:50 smithi083 ceph-mon[77242]: pgmap v15675: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:10:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:10:51] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T14:10:52.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:10:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:10:52.118Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:10:52.748 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:10:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:10:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:10:53.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:10:52 smithi153 ceph-mon[70044]: pgmap v15676: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:53.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:10:52 smithi083 ceph-mon[77242]: pgmap v15676: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:53.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:10:52 smithi083 ceph-mon[79930]: pgmap v15676: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:10:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:10:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:10:55.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:10:54 smithi153 ceph-mon[70044]: pgmap v15677: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:55.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:10:54 smithi083 ceph-mon[79930]: pgmap v15677: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:55.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:10:54 smithi083 ceph-mon[77242]: pgmap v15677: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:55.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:10:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:10:55.474Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:10:56.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:10:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:10:56.209Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:10:57.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:10:56 smithi153 ceph-mon[70044]: pgmap v15678: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:57.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:10:56 smithi083 ceph-mon[79930]: pgmap v15678: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:57.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:10:56 smithi083 ceph-mon[77242]: pgmap v15678: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:59.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:10:58 smithi153 ceph-mon[70044]: pgmap v15679: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:59.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:10:58 smithi083 ceph-mon[77242]: pgmap v15679: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:10:59.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:10:58 smithi083 ceph-mon[79930]: pgmap v15679: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:01.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:11:00 smithi153 ceph-mon[70044]: pgmap v15680: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:01.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:11:00 smithi083 ceph-mon[77242]: pgmap v15680: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:01.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:11:00 smithi083 ceph-mon[79930]: pgmap v15680: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:11:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:11:01] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T14:11:02.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:11:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:11:02.118Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:11:02.763 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:11:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:11:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:11:03.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:11:02 smithi153 ceph-mon[70044]: pgmap v15681: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:03.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:11:02 smithi083 ceph-mon[79930]: pgmap v15681: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:03.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:11:02 smithi083 ceph-mon[77242]: pgmap v15681: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:11:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:11:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:11:05.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:11:04 smithi153 ceph-mon[70044]: pgmap v15682: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:05.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:11:04 smithi083 ceph-mon[79930]: pgmap v15682: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:05.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:11:04 smithi083 ceph-mon[77242]: pgmap v15682: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:05.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:11:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:11:05.475Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:11:06.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:11:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:11:06.210Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:11:07.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:11:06 smithi153 ceph-mon[70044]: pgmap v15683: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:07.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:11:06 smithi083 ceph-mon[79930]: pgmap v15683: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:07.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:11:06 smithi083 ceph-mon[77242]: pgmap v15683: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:09.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:11:08 smithi153 ceph-mon[70044]: pgmap v15684: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:09.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:11:08 smithi083 ceph-mon[77242]: pgmap v15684: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:09.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:11:08 smithi083 ceph-mon[79930]: pgmap v15684: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:11.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:11:10 smithi153 ceph-mon[70044]: pgmap v15685: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:11.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:11:10 smithi083 ceph-mon[79930]: pgmap v15685: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:11.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:11:10 smithi083 ceph-mon[77242]: pgmap v15685: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:11:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:11:11] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T14:11:12.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:11:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:11:12.119Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:11:12.777 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:11:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:11:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:11:13.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:11:12 smithi153 ceph-mon[70044]: pgmap v15686: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:13.059 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:11:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:11:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:11:13.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:11:12 smithi083 ceph-mon[77242]: pgmap v15686: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:13.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:11:12 smithi083 ceph-mon[79930]: pgmap v15686: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:15.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:11:14 smithi153 ceph-mon[70044]: pgmap v15687: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:15.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:11:14 smithi083 ceph-mon[79930]: pgmap v15687: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:15.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:11:14 smithi083 ceph-mon[77242]: pgmap v15687: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:15.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:11:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:11:15.476Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:11:16.563 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:11:16.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:11:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:11:16.211Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:11:16.855 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:11:16.855 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 4m ago 8h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:11:16.856 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 117s ago 8h 63.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:11:16.856 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 4m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:11:16.856 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 117s ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:11:16.856 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 117s ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:11:16.857 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 4m ago 8h 943M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:11:16.857 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 4m ago 8h 1914M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:11:16.857 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 117s ago 8h 830M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:11:16.857 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 4m ago 8h 1143M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:11:16.858 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 4m ago 8h 24.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:11:16.858 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 117s ago 8h 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:11:16.858 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 4m ago 8h 1386M 2262M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:11:16.858 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 4m ago 8h 1373M 2262M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:11:16.859 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 4m ago 8h 1068M 2262M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:11:16.859 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 4m ago 8h 1345M 2262M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:11:16.859 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 117s ago 8h 1180M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:11:16.859 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 117s ago 8h 1480M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:11:16.860 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 117s ago 8h 1323M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:11:16.860 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 117s ago 8h 1572M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:11:16.860 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 117s ago 8h 189M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:11:17.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:11:16 smithi153 ceph-mon[70044]: pgmap v15688: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:17.208 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:11:17.209 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:11:17.209 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:11:17.209 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:11:17.209 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:11:17.210 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:11:17.210 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:11:17.210 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:11:17.210 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:11:17.211 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:11:17.211 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:11:17.211 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:11:17.211 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:11:17.212 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:11:17.212 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:11:17.212 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:11:17.212 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:11:17.213 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:11:17.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:11:16 smithi083 ceph-mon[79930]: pgmap v15688: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:17.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:11:16 smithi083 ceph-mon[77242]: pgmap v15688: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:18.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:11:17 smithi153 ceph-mon[70044]: from='client.62305 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:11:18.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:11:17 smithi153 ceph-mon[70044]: from='client.52380 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:11:18.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:11:17 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1270624596' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:11:18.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:11:17 smithi083 ceph-mon[79930]: from='client.62305 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:11:18.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:11:17 smithi083 ceph-mon[79930]: from='client.52380 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:11:18.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:11:17 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1270624596' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:11:18.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:11:17 smithi083 ceph-mon[77242]: from='client.62305 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:11:18.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:11:17 smithi083 ceph-mon[77242]: from='client.52380 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:11:18.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:11:17 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1270624596' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:11:19.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:11:18 smithi153 ceph-mon[70044]: pgmap v15689: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:19.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:11:18 smithi083 ceph-mon[79930]: pgmap v15689: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:19.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:11:18 smithi083 ceph-mon[77242]: pgmap v15689: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:21.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:11:20 smithi153 ceph-mon[70044]: pgmap v15690: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:21.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:11:20 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:11:21.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:11:20 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:11:21.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:11:20 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:11:21.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:11:20 smithi083 ceph-mon[79930]: pgmap v15690: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:21.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:11:20 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:11:21.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:11:20 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:11:21.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:11:20 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:11:21.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:11:20 smithi083 ceph-mon[77242]: pgmap v15690: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:21.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:11:20 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:11:21.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:11:20 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:11:21.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:11:20 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:11:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:11:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:11:21] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T14:11:22.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:11:21 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:11:22.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:11:21 smithi153 ceph-mon[70044]: pgmap v15691: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:22.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:11:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:11:22.120Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:11:22.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:11:21 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:11:22.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:11:21 smithi083 ceph-mon[77242]: pgmap v15691: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:22.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:11:21 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:11:22.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:11:21 smithi083 ceph-mon[79930]: pgmap v15691: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:11:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:11:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:11:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:11:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:11:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:11:24.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:11:24 smithi083 ceph-mon[79930]: pgmap v15692: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:24.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:11:24 smithi083 ceph-mon[77242]: pgmap v15692: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:24.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:11:24 smithi153 ceph-mon[70044]: pgmap v15692: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:25.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:11:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:11:25.477Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:11:26.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:11:26 smithi083 ceph-mon[79930]: pgmap v15693: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:26.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:11:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:11:26.211Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:11:26.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:11:26 smithi083 ceph-mon[77242]: pgmap v15693: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:26.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:11:26 smithi153 ceph-mon[70044]: pgmap v15693: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:28.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:11:28 smithi083 ceph-mon[79930]: pgmap v15694: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:28.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:11:28 smithi083 ceph-mon[77242]: pgmap v15694: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:29.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:11:28 smithi153 ceph-mon[70044]: pgmap v15694: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:30.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:11:30 smithi083 ceph-mon[79930]: pgmap v15695: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:30.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:11:30 smithi083 ceph-mon[77242]: pgmap v15695: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:31.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:11:30 smithi153 ceph-mon[70044]: pgmap v15695: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:11:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:11:31] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T14:11:32.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:11:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:11:32.121Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:11:32.618 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:11:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:11:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:11:32.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:11:32 smithi083 ceph-mon[79930]: pgmap v15696: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:32.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:11:32 smithi083 ceph-mon[77242]: pgmap v15696: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:33.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:11:32 smithi153 ceph-mon[70044]: pgmap v15696: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:11:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:11:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:11:34.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:11:34 smithi083 ceph-mon[79930]: pgmap v15697: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:34.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:11:34 smithi083 ceph-mon[77242]: pgmap v15697: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:35.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:11:34 smithi153 ceph-mon[70044]: pgmap v15697: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:35.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:11:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:11:35.477Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:11:36.624 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:11:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:11:36.212Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:11:36.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:11:36 smithi083 ceph-mon[79930]: pgmap v15698: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:36.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:11:36 smithi083 ceph-mon[77242]: pgmap v15698: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:37.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:11:36 smithi153 ceph-mon[70044]: pgmap v15698: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:38.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:11:38 smithi083 ceph-mon[79930]: pgmap v15699: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:38.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:11:38 smithi083 ceph-mon[77242]: pgmap v15699: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:39.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:11:38 smithi153 ceph-mon[70044]: pgmap v15699: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:11:40 smithi083 ceph-mon[79930]: pgmap v15700: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:11:40 smithi083 ceph-mon[77242]: pgmap v15700: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:11:40 smithi153 ceph-mon[70044]: pgmap v15700: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:41.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:11:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:11:41] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T14:11:41.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:11:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:11:41.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:11:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:11:41.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:11:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:11:41.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:11:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:11:41.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:11:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:11:41.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:11:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:11:41.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:11:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:11:41.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:11:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:11:42.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:11:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:11:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:11:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:11:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:11:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:11:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:11:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:11:42.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:11:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:11:42.121Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:11:42.638 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:11:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:11:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:11:42.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:11:42 smithi083 ceph-mon[79930]: pgmap v15701: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:42.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:11:42 smithi083 ceph-mon[77242]: pgmap v15701: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:43.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:11:42 smithi153 ceph-mon[70044]: pgmap v15701: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:11:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:11:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:11:44.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:11:44 smithi083 ceph-mon[79930]: pgmap v15702: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:44.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:11:44 smithi083 ceph-mon[77242]: pgmap v15702: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:45.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:11:44 smithi153 ceph-mon[70044]: pgmap v15702: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:45.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:11:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:11:45.478Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:11:46.643 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:11:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:11:46.212Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:11:46.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:11:46 smithi083 ceph-mon[79930]: pgmap v15703: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:46.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:11:46 smithi083 ceph-mon[77242]: pgmap v15703: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:47.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:11:46 smithi153 ceph-mon[70044]: pgmap v15703: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:47.535 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:11:47.821 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:11:47.822 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 4m ago 8h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:11:47.822 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 2m ago 8h 63.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:11:47.822 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 4m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:11:47.822 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 2m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:11:47.823 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 2m ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:11:47.823 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 4m ago 8h 943M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:11:47.823 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 4m ago 8h 1914M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:11:47.823 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 2m ago 8h 830M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:11:47.824 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 4m ago 8h 1143M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:11:47.824 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 4m ago 8h 24.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:11:47.824 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 2m ago 8h 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:11:47.824 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 4m ago 8h 1386M 2262M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:11:47.825 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 4m ago 8h 1373M 2262M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:11:47.825 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 4m ago 8h 1068M 2262M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:11:47.825 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 4m ago 8h 1345M 2262M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:11:47.825 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 2m ago 8h 1180M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:11:47.826 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 2m ago 8h 1480M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:11:47.826 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 2m ago 8h 1323M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:11:47.826 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 2m ago 8h 1572M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:11:47.826 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 2m ago 8h 189M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:11:48.178 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:11:48.178 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:11:48.179 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:11:48.179 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:11:48.179 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:11:48.179 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:11:48.179 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:11:48.180 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:11:48.180 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:11:48.180 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:11:48.180 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:11:48.181 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:11:48.181 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:11:48.181 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:11:48.181 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:11:48.182 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:11:48.182 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:11:48.182 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:11:48.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:11:48 smithi083 ceph-mon[79930]: pgmap v15704: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:48.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:11:48 smithi083 ceph-mon[79930]: from='client.52392 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:11:48.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:11:48 smithi083 ceph-mon[79930]: from='client.52398 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:11:48.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:11:48 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2037388311' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:11:48.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:11:48 smithi083 ceph-mon[77242]: pgmap v15704: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:48.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:11:48 smithi083 ceph-mon[77242]: from='client.52392 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:11:48.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:11:48 smithi083 ceph-mon[77242]: from='client.52398 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:11:48.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:11:48 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2037388311' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:11:49.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:11:48 smithi153 ceph-mon[70044]: pgmap v15704: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:49.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:11:48 smithi153 ceph-mon[70044]: from='client.52392 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:11:49.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:11:48 smithi153 ceph-mon[70044]: from='client.52398 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:11:49.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:11:48 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2037388311' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:11:50.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:11:50 smithi083 ceph-mon[79930]: pgmap v15705: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:50.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:11:50 smithi083 ceph-mon[77242]: pgmap v15705: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:51.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:11:50 smithi153 ceph-mon[70044]: pgmap v15705: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:11:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:11:51] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T14:11:52.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:11:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:11:52.122Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:11:52.652 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:11:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:11:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:11:52.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:11:52 smithi083 ceph-mon[77242]: pgmap v15706: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:52.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:11:52 smithi083 ceph-mon[79930]: pgmap v15706: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:53.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:11:52 smithi153 ceph-mon[70044]: pgmap v15706: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:11:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:11:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:11:54.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:11:54 smithi083 ceph-mon[79930]: pgmap v15707: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:54.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:11:54 smithi083 ceph-mon[77242]: pgmap v15707: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:55.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:11:54 smithi153 ceph-mon[70044]: pgmap v15707: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:55.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:11:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:11:55.479Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:11:56.656 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:11:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:11:56.213Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:11:56.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:11:56 smithi083 ceph-mon[79930]: pgmap v15708: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:56.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:11:56 smithi083 ceph-mon[77242]: pgmap v15708: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:57.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:11:56 smithi153 ceph-mon[70044]: pgmap v15708: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:58.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:11:58 smithi083 ceph-mon[79930]: pgmap v15709: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:58.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:11:58 smithi083 ceph-mon[77242]: pgmap v15709: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:11:59.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:11:58 smithi153 ceph-mon[70044]: pgmap v15709: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:00.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:12:00 smithi083 ceph-mon[79930]: pgmap v15710: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:00.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:12:00 smithi083 ceph-mon[77242]: pgmap v15710: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:01.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:12:00 smithi153 ceph-mon[70044]: pgmap v15710: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:12:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:12:01] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T14:12:02.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:12:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:12:02.122Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:12:02.667 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:12:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:12:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:12:02.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:12:02 smithi083 ceph-mon[79930]: pgmap v15711: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:02.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:12:02 smithi083 ceph-mon[77242]: pgmap v15711: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:03.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:12:02 smithi153 ceph-mon[70044]: pgmap v15711: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:12:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:12:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:12:04.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:12:04 smithi083 ceph-mon[79930]: pgmap v15712: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:04.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:12:04 smithi083 ceph-mon[77242]: pgmap v15712: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:05.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:12:04 smithi153 ceph-mon[70044]: pgmap v15712: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:05.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:12:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:12:05.479Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:12:06.672 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:12:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:12:06.213Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:12:06.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:12:06 smithi083 ceph-mon[77242]: pgmap v15713: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:06.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:12:06 smithi083 ceph-mon[79930]: pgmap v15713: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:07.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:12:06 smithi153 ceph-mon[70044]: pgmap v15713: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:08.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:12:08 smithi083 ceph-mon[79930]: pgmap v15714: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:08.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:12:08 smithi083 ceph-mon[77242]: pgmap v15714: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:09.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:12:08 smithi153 ceph-mon[70044]: pgmap v15714: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:10.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:12:10 smithi083 ceph-mon[79930]: pgmap v15715: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:10.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:12:10 smithi083 ceph-mon[77242]: pgmap v15715: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:11.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:12:10 smithi153 ceph-mon[70044]: pgmap v15715: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:12:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:12:11] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T14:12:12.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:12:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:12:12.123Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:12:12.684 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:12:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:12:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:12:12.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:12:12 smithi083 ceph-mon[79930]: pgmap v15716: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:12.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:12:12 smithi083 ceph-mon[77242]: pgmap v15716: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:13.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:12:12 smithi153 ceph-mon[70044]: pgmap v15716: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:12:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:12:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:12:14.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:12:14 smithi083 ceph-mon[79930]: pgmap v15717: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:14.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:12:14 smithi083 ceph-mon[77242]: pgmap v15717: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:15.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:12:14 smithi153 ceph-mon[70044]: pgmap v15717: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:15.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:12:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:12:15.480Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:12:16.688 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:12:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:12:16.214Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:12:16.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:12:16 smithi083 ceph-mon[79930]: pgmap v15718: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:16.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:12:16 smithi083 ceph-mon[77242]: pgmap v15718: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:17.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:12:16 smithi153 ceph-mon[70044]: pgmap v15718: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:18.502 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:12:18.791 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:12:18.792 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 5m ago 8h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:12:18.792 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 2m ago 8h 63.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:12:18.792 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 5m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:12:18.792 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 2m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:12:18.793 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 2m ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:12:18.793 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 5m ago 8h 943M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:12:18.793 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 5m ago 8h 1914M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:12:18.793 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 2m ago 8h 830M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:12:18.794 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 5m ago 8h 1143M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:12:18.794 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 5m ago 8h 24.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:12:18.794 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 2m ago 8h 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:12:18.794 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 5m ago 8h 1386M 2262M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:12:18.795 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 5m ago 8h 1373M 2262M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:12:18.795 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 5m ago 8h 1068M 2262M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:12:18.795 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 5m ago 8h 1345M 2262M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:12:18.795 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 2m ago 8h 1180M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:12:18.796 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 2m ago 8h 1480M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:12:18.796 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 2m ago 8h 1323M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:12:18.796 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 2m ago 8h 1572M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:12:18.796 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 2m ago 8h 189M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:12:18.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:12:18 smithi083 ceph-mon[79930]: pgmap v15719: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:18.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:12:18 smithi083 ceph-mon[77242]: pgmap v15719: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:19.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:12:18 smithi153 ceph-mon[70044]: pgmap v15719: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:19.150 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:12:19.150 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:12:19.150 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:12:19.151 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:12:19.151 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:12:19.151 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:12:19.151 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:12:19.151 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:12:19.152 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:12:19.152 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:12:19.152 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:12:19.152 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:12:19.153 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:12:19.153 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:12:19.153 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:12:19.153 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:12:19.153 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:12:19.154 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:12:19.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:12:19 smithi083 ceph-mon[77242]: from='client.52410 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:12:19.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:12:19 smithi083 ceph-mon[77242]: from='client.62347 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:12:19.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:12:19 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3466616125' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:12:19.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:12:19 smithi083 ceph-mon[79930]: from='client.52410 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:12:19.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:12:19 smithi083 ceph-mon[79930]: from='client.62347 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:12:19.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:12:19 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3466616125' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:12:20.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:12:19 smithi153 ceph-mon[70044]: from='client.52410 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:12:20.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:12:19 smithi153 ceph-mon[70044]: from='client.62347 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:12:20.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:12:19 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3466616125' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:12:20.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:12:20 smithi083 ceph-mon[79930]: pgmap v15720: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:20.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:12:20 smithi083 ceph-mon[77242]: pgmap v15720: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:21.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:12:20 smithi153 ceph-mon[70044]: pgmap v15720: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:12:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:12:21] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T14:12:21.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:12:21 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:12:21.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:12:21 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:12:21.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:12:21 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:12:21.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:12:21 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:12:21.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:12:21 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:12:21.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:12:21 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:12:21.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:12:21 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:12:21.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:12:21 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:12:22.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:12:21 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:12:22.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:12:21 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:12:22.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:12:21 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:12:22.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:12:21 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:12:22.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:12:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:12:22.122Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:12:22.705 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:12:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:12:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:12:22.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:12:22 smithi083 ceph-mon[79930]: pgmap v15721: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:22.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:12:22 smithi083 ceph-mon[77242]: pgmap v15721: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:23.026 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:12:22 smithi153 ceph-mon[70044]: pgmap v15721: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:12:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:12:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:12:24.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:12:24 smithi083 ceph-mon[79930]: pgmap v15722: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:24.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:12:24 smithi083 ceph-mon[77242]: pgmap v15722: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:25.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:12:24 smithi153 ceph-mon[70044]: pgmap v15722: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:25.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:12:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:12:25.481Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:12:26.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:12:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:12:26.215Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:12:26.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:12:26 smithi083 ceph-mon[79930]: pgmap v15723: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:26.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:12:26 smithi083 ceph-mon[77242]: pgmap v15723: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:27.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:12:26 smithi153 ceph-mon[70044]: pgmap v15723: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:29.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:12:28 smithi153 ceph-mon[70044]: pgmap v15724: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:29.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:12:28 smithi083 ceph-mon[77242]: pgmap v15724: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:29.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:12:28 smithi083 ceph-mon[79930]: pgmap v15724: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:31.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:12:30 smithi153 ceph-mon[70044]: pgmap v15725: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:31.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:12:30 smithi083 ceph-mon[79930]: pgmap v15725: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:31.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:12:30 smithi083 ceph-mon[77242]: pgmap v15725: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:12:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:12:31] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T14:12:32.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:12:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:12:32.124Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:12:32.720 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:12:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:12:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:12:33.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:12:32 smithi153 ceph-mon[70044]: pgmap v15726: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:33.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:12:32 smithi083 ceph-mon[79930]: pgmap v15726: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:33.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:12:32 smithi083 ceph-mon[77242]: pgmap v15726: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:12:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:12:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:12:35.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:12:34 smithi153 ceph-mon[70044]: pgmap v15727: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:35.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:12:34 smithi083 ceph-mon[79930]: pgmap v15727: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:35.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:12:34 smithi083 ceph-mon[77242]: pgmap v15727: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:35.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:12:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:12:35.481Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:12:36.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:12:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:12:36.216Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:12:37.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:12:36 smithi153 ceph-mon[70044]: pgmap v15728: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:37.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:12:36 smithi083 ceph-mon[77242]: pgmap v15728: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:37.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:12:36 smithi083 ceph-mon[79930]: pgmap v15728: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:39.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:12:38 smithi153 ceph-mon[70044]: pgmap v15729: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:39.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:12:38 smithi083 ceph-mon[79930]: pgmap v15729: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:39.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:12:38 smithi083 ceph-mon[77242]: pgmap v15729: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:12:40 smithi153 ceph-mon[70044]: pgmap v15730: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:41.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:12:40 smithi083 ceph-mon[77242]: pgmap v15730: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:41.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:12:40 smithi083 ceph-mon[79930]: pgmap v15730: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:41.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:12:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:12:41] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T14:12:41.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:12:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:12:41.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:12:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:12:41.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:12:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:12:41.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:12:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:12:41.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:12:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:12:41.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:12:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:12:41.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:12:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:12:41.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:12:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:12:42.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:12:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:12:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:12:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:12:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:12:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:12:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:12:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:12:42.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:12:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:12:42.124Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:12:42.737 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:12:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:12:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:12:43.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:12:42 smithi153 ceph-mon[70044]: pgmap v15731: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:43.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:12:42 smithi083 ceph-mon[79930]: pgmap v15731: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:43.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:12:42 smithi083 ceph-mon[77242]: pgmap v15731: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:12:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:12:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:12:45.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:12:44 smithi153 ceph-mon[70044]: pgmap v15732: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:45.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:12:44 smithi083 ceph-mon[79930]: pgmap v15732: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:45.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:12:44 smithi083 ceph-mon[77242]: pgmap v15732: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:45.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:12:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:12:45.483Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:12:46.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:12:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:12:46.217Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:12:47.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:12:46 smithi153 ceph-mon[70044]: pgmap v15733: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:47.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:12:46 smithi083 ceph-mon[77242]: pgmap v15733: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:47.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:12:46 smithi083 ceph-mon[79930]: pgmap v15733: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:49.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:12:48 smithi153 ceph-mon[70044]: pgmap v15734: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:49.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:12:48 smithi083 ceph-mon[79930]: pgmap v15734: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:49.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:12:48 smithi083 ceph-mon[77242]: pgmap v15734: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:49.476 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:12:49.768 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:12:49.769 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 5m ago 8h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:12:49.769 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 3m ago 8h 63.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:12:49.769 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 5m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:12:49.770 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 3m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:12:49.770 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 3m ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:12:49.770 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 5m ago 8h 943M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:12:49.770 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 5m ago 8h 1914M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:12:49.771 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 3m ago 8h 830M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:12:49.771 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 5m ago 8h 1143M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:12:49.771 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 5m ago 8h 24.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:12:49.771 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 3m ago 8h 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:12:49.772 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 5m ago 8h 1386M 2262M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:12:49.772 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 5m ago 8h 1373M 2262M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:12:49.772 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 5m ago 8h 1068M 2262M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:12:49.772 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 5m ago 8h 1345M 2262M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:12:49.772 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 3m ago 8h 1180M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:12:49.773 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 3m ago 8h 1480M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:12:49.773 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 3m ago 8h 1323M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:12:49.773 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 3m ago 8h 1572M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:12:49.773 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 3m ago 8h 189M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:12:50.123 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:12:50.124 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:12:50.124 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:12:50.124 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:12:50.124 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:12:50.125 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:12:50.125 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:12:50.125 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:12:50.125 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:12:50.126 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:12:50.126 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:12:50.126 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:12:50.126 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:12:50.127 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:12:50.127 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:12:50.127 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:12:50.127 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:12:50.128 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:12:51.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:12:50 smithi153 ceph-mon[70044]: pgmap v15735: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:51.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:12:50 smithi153 ceph-mon[70044]: from='client.52428 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:12:51.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:12:50 smithi153 ceph-mon[70044]: from='client.62365 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:12:51.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:12:50 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3773315051' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:12:51.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:12:50 smithi083 ceph-mon[79930]: pgmap v15735: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:51.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:12:50 smithi083 ceph-mon[79930]: from='client.52428 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:12:51.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:12:50 smithi083 ceph-mon[79930]: from='client.62365 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:12:51.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:12:50 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3773315051' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:12:51.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:12:50 smithi083 ceph-mon[77242]: pgmap v15735: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:51.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:12:50 smithi083 ceph-mon[77242]: from='client.52428 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:12:51.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:12:50 smithi083 ceph-mon[77242]: from='client.62365 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:12:51.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:12:50 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3773315051' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:12:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:12:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:12:51] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T14:12:52.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:12:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:12:52.125Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:12:52.751 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:12:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:12:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:12:53.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:12:52 smithi153 ceph-mon[70044]: pgmap v15736: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:53.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:12:52 smithi083 ceph-mon[79930]: pgmap v15736: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:53.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:12:52 smithi083 ceph-mon[77242]: pgmap v15736: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:12:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:12:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:12:55.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:12:54 smithi153 ceph-mon[70044]: pgmap v15737: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:55.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:12:54 smithi083 ceph-mon[77242]: pgmap v15737: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:55.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:12:54 smithi083 ceph-mon[79930]: pgmap v15737: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:55.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:12:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:12:55.483Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:12:56.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:12:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:12:56.217Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:12:57.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:12:56 smithi153 ceph-mon[70044]: pgmap v15738: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:57.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:12:56 smithi083 ceph-mon[79930]: pgmap v15738: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:57.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:12:56 smithi083 ceph-mon[77242]: pgmap v15738: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:59.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:12:58 smithi153 ceph-mon[70044]: pgmap v15739: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:59.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:12:58 smithi083 ceph-mon[79930]: pgmap v15739: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:12:59.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:12:58 smithi083 ceph-mon[77242]: pgmap v15739: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:01.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:13:00 smithi153 ceph-mon[70044]: pgmap v15740: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:01.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:13:00 smithi083 ceph-mon[79930]: pgmap v15740: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:01.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:13:00 smithi083 ceph-mon[77242]: pgmap v15740: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:13:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:13:01] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T14:13:02.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:13:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:13:02.125Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:13:02.769 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:13:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:13:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:13:03.026 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:13:02 smithi153 ceph-mon[70044]: pgmap v15741: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:03.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:13:02 smithi083 ceph-mon[77242]: pgmap v15741: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:03.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:13:02 smithi083 ceph-mon[79930]: pgmap v15741: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:13:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:13:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:13:05.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:13:04 smithi153 ceph-mon[70044]: pgmap v15742: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:05.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:13:04 smithi083 ceph-mon[77242]: pgmap v15742: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:05.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:13:04 smithi083 ceph-mon[79930]: pgmap v15742: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:05.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:13:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:13:05.483Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:13:06.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:13:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:13:06.218Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:13:07.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:13:06 smithi153 ceph-mon[70044]: pgmap v15743: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:07.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:13:06 smithi083 ceph-mon[77242]: pgmap v15743: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:07.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:13:06 smithi083 ceph-mon[79930]: pgmap v15743: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:09.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:13:08 smithi153 ceph-mon[70044]: pgmap v15744: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:09.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:13:08 smithi083 ceph-mon[77242]: pgmap v15744: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:09.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:13:08 smithi083 ceph-mon[79930]: pgmap v15744: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:11.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:13:10 smithi153 ceph-mon[70044]: pgmap v15745: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:11.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:13:10 smithi083 ceph-mon[77242]: pgmap v15745: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:11.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:13:10 smithi083 ceph-mon[79930]: pgmap v15745: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:13:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:13:11] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T14:13:12.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:13:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:13:12.126Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:13:12.786 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:13:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:13:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:13:13.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:13:12 smithi153 ceph-mon[70044]: pgmap v15746: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:13.059 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:13:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:13:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:13:13.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:13:12 smithi083 ceph-mon[77242]: pgmap v15746: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:13.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:13:12 smithi083 ceph-mon[79930]: pgmap v15746: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:15.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:13:14 smithi153 ceph-mon[70044]: pgmap v15747: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:15.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:13:14 smithi083 ceph-mon[79930]: pgmap v15747: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:15.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:13:14 smithi083 ceph-mon[77242]: pgmap v15747: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:15.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:13:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:13:15.484Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:13:16.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:13:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:13:16.218Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:13:17.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:13:16 smithi153 ceph-mon[70044]: pgmap v15748: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:17.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:13:16 smithi083 ceph-mon[79930]: pgmap v15748: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:17.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:13:16 smithi083 ceph-mon[77242]: pgmap v15748: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:19.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:13:18 smithi153 ceph-mon[70044]: pgmap v15749: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:19.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:13:18 smithi083 ceph-mon[79930]: pgmap v15749: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:19.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:13:18 smithi083 ceph-mon[77242]: pgmap v15749: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:20.449 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:13:20.737 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:13:20.738 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 6m ago 8h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:13:20.738 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 4m ago 8h 63.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:13:20.738 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 6m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:13:20.739 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 4m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:13:20.739 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 4m ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:13:20.739 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 6m ago 8h 943M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:13:20.739 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 6m ago 8h 1914M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:13:20.740 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 4m ago 8h 830M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:13:20.740 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 6m ago 8h 1143M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:13:20.740 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 6m ago 8h 24.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:13:20.740 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 4m ago 8h 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:13:20.741 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 6m ago 8h 1386M 2262M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:13:20.741 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 6m ago 8h 1373M 2262M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:13:20.741 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 6m ago 8h 1068M 2262M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:13:20.741 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 6m ago 8h 1345M 2262M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:13:20.741 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 4m ago 8h 1180M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:13:20.742 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 4m ago 8h 1480M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:13:20.742 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 4m ago 8h 1323M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:13:20.742 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 4m ago 8h 1572M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:13:20.742 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 4m ago 8h 189M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:13:21.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:13:20 smithi153 ceph-mon[70044]: pgmap v15750: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:21.096 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:13:21.096 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:13:21.096 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:13:21.097 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:13:21.097 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:13:21.097 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:13:21.097 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:13:21.097 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:13:21.098 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:13:21.098 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:13:21.098 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:13:21.099 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:13:21.099 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:13:21.099 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:13:21.099 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:13:21.099 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:13:21.100 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:13:21.100 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:13:21.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:13:20 smithi083 ceph-mon[77242]: pgmap v15750: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:21.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:13:20 smithi083 ceph-mon[79930]: pgmap v15750: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:21.801 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:13:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:13:21] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T14:13:22.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:13:21 smithi153 ceph-mon[70044]: from='client.52446 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:13:22.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:13:21 smithi153 ceph-mon[70044]: from='client.52452 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:13:22.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:13:21 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1157244019' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:13:22.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:13:21 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:13:22.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:13:21 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:13:22.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:13:21 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:13:22.129 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:13:21 smithi083 ceph-mon[79930]: from='client.52446 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:13:22.129 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:13:21 smithi083 ceph-mon[79930]: from='client.52452 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:13:22.129 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:13:21 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1157244019' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:13:22.129 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:13:21 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:13:22.130 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:13:21 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:13:22.130 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:13:21 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:13:22.130 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:13:21 smithi083 ceph-mon[77242]: from='client.52446 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:13:22.131 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:13:21 smithi083 ceph-mon[77242]: from='client.52452 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:13:22.131 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:13:21 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1157244019' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:13:22.131 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:13:21 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:13:22.131 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:13:21 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:13:22.132 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:13:21 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:13:22.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:13:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:13:22.126Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:13:22.803 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:13:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:13:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:13:23.058 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:13:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:13:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:13:23.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:13:22 smithi153 ceph-mon[70044]: pgmap v15751: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:23.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:13:22 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:13:23.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:13:22 smithi083 ceph-mon[79930]: pgmap v15751: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:23.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:13:22 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:13:23.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:13:22 smithi083 ceph-mon[77242]: pgmap v15751: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:23.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:13:22 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:13:25.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:13:24 smithi153 ceph-mon[70044]: pgmap v15752: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:25.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:13:24 smithi083 ceph-mon[79930]: pgmap v15752: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:25.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:13:24 smithi083 ceph-mon[77242]: pgmap v15752: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:25.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:13:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:13:25.485Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:13:26.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:13:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:13:26.218Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:13:27.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:13:26 smithi083 ceph-mon[79930]: pgmap v15753: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:27.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:13:26 smithi083 ceph-mon[77242]: pgmap v15753: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:27.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:13:26 smithi153 ceph-mon[70044]: pgmap v15753: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:29.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:13:28 smithi083 ceph-mon[79930]: pgmap v15754: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:29.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:13:28 smithi083 ceph-mon[77242]: pgmap v15754: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:29.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:13:28 smithi153 ceph-mon[70044]: pgmap v15754: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:31.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:13:30 smithi083 ceph-mon[79930]: pgmap v15755: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:31.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:13:30 smithi083 ceph-mon[77242]: pgmap v15755: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:31.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:13:30 smithi153 ceph-mon[70044]: pgmap v15755: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:31.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:13:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:13:31] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T14:13:32.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:13:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:13:32.128Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:13:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:13:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:13:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:13:33.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:13:32 smithi083 ceph-mon[79930]: pgmap v15756: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:33.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:13:32 smithi083 ceph-mon[77242]: pgmap v15756: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:33.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:13:32 smithi153 ceph-mon[70044]: pgmap v15756: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:33.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:13:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:13:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:13:35.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:13:34 smithi083 ceph-mon[79930]: pgmap v15757: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:35.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:13:34 smithi083 ceph-mon[77242]: pgmap v15757: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:35.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:13:34 smithi153 ceph-mon[70044]: pgmap v15757: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:35.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:13:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:13:35.485Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:13:36.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:13:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:13:36.219Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:13:37.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:13:36 smithi083 ceph-mon[79930]: pgmap v15758: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:37.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:13:36 smithi083 ceph-mon[77242]: pgmap v15758: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:37.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:13:36 smithi153 ceph-mon[70044]: pgmap v15758: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:39.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:13:38 smithi083 ceph-mon[77242]: pgmap v15759: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:39.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:13:38 smithi083 ceph-mon[79930]: pgmap v15759: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:39.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:13:38 smithi153 ceph-mon[70044]: pgmap v15759: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:41.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:13:40 smithi083 ceph-mon[79930]: pgmap v15760: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:13:40 smithi083 ceph-mon[77242]: pgmap v15760: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:13:40 smithi153 ceph-mon[70044]: pgmap v15760: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:41.834 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:13:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:13:41] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T14:13:42.131 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:13:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:13:42.132 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:13:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:13:42.132 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:13:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:13:42.132 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:13:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:13:42.133 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:13:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:13:42.133 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:13:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:13:42.133 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:13:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:13:42.133 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:13:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:13:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:13:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:13:42.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:13:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:13:42.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:13:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:13:42.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:13:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:13:42.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:13:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:13:42.128Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:13:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:13:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:13:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:13:43.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:13:42 smithi083 ceph-mon[77242]: pgmap v15761: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:43.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:13:42 smithi083 ceph-mon[79930]: pgmap v15761: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:43.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:13:42 smithi153 ceph-mon[70044]: pgmap v15761: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:43.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:13:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:13:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:13:45.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:13:44 smithi083 ceph-mon[79930]: pgmap v15762: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:45.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:13:44 smithi083 ceph-mon[77242]: pgmap v15762: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:45.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:13:44 smithi153 ceph-mon[70044]: pgmap v15762: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:45.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:13:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:13:45.486Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:13:46.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:13:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:13:46.220Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:13:47.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:13:46 smithi083 ceph-mon[79930]: pgmap v15763: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:47.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:13:46 smithi083 ceph-mon[77242]: pgmap v15763: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:47.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:13:46 smithi153 ceph-mon[70044]: pgmap v15763: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:49.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:13:48 smithi083 ceph-mon[79930]: pgmap v15764: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:49.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:13:48 smithi083 ceph-mon[77242]: pgmap v15764: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:49.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:13:48 smithi153 ceph-mon[70044]: pgmap v15764: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:51.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:13:50 smithi083 ceph-mon[79930]: pgmap v15765: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:51.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:13:50 smithi083 ceph-mon[77242]: pgmap v15765: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:51.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:13:50 smithi153 ceph-mon[70044]: pgmap v15765: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:51.422 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:13:51.714 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:13:51.714 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 6m ago 8h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:13:51.715 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 4m ago 8h 63.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:13:51.715 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 6m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:13:51.715 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 4m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:13:51.715 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 4m ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:13:51.716 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 6m ago 8h 943M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:13:51.716 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 6m ago 8h 1914M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:13:51.716 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 4m ago 8h 830M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:13:51.716 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 6m ago 8h 1143M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:13:51.717 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 6m ago 8h 24.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:13:51.717 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 4m ago 8h 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:13:51.717 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 6m ago 8h 1386M 2262M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:13:51.717 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 6m ago 8h 1373M 2262M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:13:51.718 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 6m ago 8h 1068M 2262M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:13:51.718 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 6m ago 8h 1345M 2262M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:13:51.718 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 4m ago 8h 1180M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:13:51.718 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 4m ago 8h 1480M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:13:51.718 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 4m ago 8h 1323M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:13:51.719 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 4m ago 8h 1572M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:13:51.719 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 4m ago 8h 189M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:13:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:13:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:13:51] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T14:13:52.071 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:13:52.071 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:13:52.072 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:13:52.072 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:13:52.073 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:13:52.073 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:13:52.074 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:13:52.074 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:13:52.075 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:13:52.075 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:13:52.076 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:13:52.076 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:13:52.076 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:13:52.077 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:13:52.077 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:13:52.078 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:13:52.078 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:13:52.079 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:13:52.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:13:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:13:52.129Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:13:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:13:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:13:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:13:53.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:13:52 smithi083 ceph-mon[79930]: pgmap v15766: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:53.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:13:52 smithi083 ceph-mon[79930]: from='client.62395 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:13:53.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:13:52 smithi083 ceph-mon[79930]: from='client.52470 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:13:53.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:13:52 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4269417385' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:13:53.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:13:52 smithi083 ceph-mon[77242]: pgmap v15766: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:53.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:13:52 smithi083 ceph-mon[77242]: from='client.62395 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:13:53.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:13:52 smithi083 ceph-mon[77242]: from='client.52470 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:13:53.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:13:52 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4269417385' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:13:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:13:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:13:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:13:53.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:13:52 smithi153 ceph-mon[70044]: pgmap v15766: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:53.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:13:52 smithi153 ceph-mon[70044]: from='client.62395 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:13:53.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:13:52 smithi153 ceph-mon[70044]: from='client.52470 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:13:53.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:13:52 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4269417385' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:13:55.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:13:54 smithi083 ceph-mon[79930]: pgmap v15767: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:55.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:13:54 smithi083 ceph-mon[77242]: pgmap v15767: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:55.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:13:54 smithi153 ceph-mon[70044]: pgmap v15767: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:55.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:13:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:13:55.487Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:13:56.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:13:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:13:56.220Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:13:57.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:13:56 smithi083 ceph-mon[77242]: pgmap v15768: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:57.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:13:56 smithi083 ceph-mon[79930]: pgmap v15768: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:57.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:13:56 smithi153 ceph-mon[70044]: pgmap v15768: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:59.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:13:58 smithi083 ceph-mon[77242]: pgmap v15769: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:59.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:13:58 smithi083 ceph-mon[79930]: pgmap v15769: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:13:59.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:13:58 smithi153 ceph-mon[70044]: pgmap v15769: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:01.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:14:00 smithi083 ceph-mon[79930]: pgmap v15770: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:01.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:14:00 smithi083 ceph-mon[77242]: pgmap v15770: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:01.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:14:00 smithi153 ceph-mon[70044]: pgmap v15770: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:14:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:14:01] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T14:14:02.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:14:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:14:02.130Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:14:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:14:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:14:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:14:03.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:14:02 smithi083 ceph-mon[79930]: pgmap v15771: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:03.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:14:02 smithi083 ceph-mon[77242]: pgmap v15771: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:03.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:14:02 smithi153 ceph-mon[70044]: pgmap v15771: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:14:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:14:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:14:05.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:14:04 smithi083 ceph-mon[79930]: pgmap v15772: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:05.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:14:04 smithi083 ceph-mon[77242]: pgmap v15772: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:05.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:14:04 smithi153 ceph-mon[70044]: pgmap v15772: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:05.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:14:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:14:05.488Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:14:06.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:14:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:14:06.221Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:14:07.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:14:06 smithi083 ceph-mon[79930]: pgmap v15773: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:07.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:14:06 smithi083 ceph-mon[77242]: pgmap v15773: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:07.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:14:06 smithi153 ceph-mon[70044]: pgmap v15773: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:09.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:14:08 smithi083 ceph-mon[79930]: pgmap v15774: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:09.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:14:08 smithi083 ceph-mon[77242]: pgmap v15774: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:09.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:14:08 smithi153 ceph-mon[70044]: pgmap v15774: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:11.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:14:10 smithi083 ceph-mon[79930]: pgmap v15775: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:11.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:14:10 smithi083 ceph-mon[77242]: pgmap v15775: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:11.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:14:10 smithi153 ceph-mon[70044]: pgmap v15775: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:14:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:14:11] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T14:14:12.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:14:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:14:12.131Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:14:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:14:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:14:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:14:13.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:14:12 smithi083 ceph-mon[77242]: pgmap v15776: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:13.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:14:12 smithi083 ceph-mon[79930]: pgmap v15776: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:13.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:14:12 smithi153 ceph-mon[70044]: pgmap v15776: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:13.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:14:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:14:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:14:15.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:14:14 smithi083 ceph-mon[79930]: pgmap v15777: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:15.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:14:14 smithi083 ceph-mon[77242]: pgmap v15777: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:15.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:14:14 smithi153 ceph-mon[70044]: pgmap v15777: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:15.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:14:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:14:15.488Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:14:16.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:14:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:14:16.222Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:14:17.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:14:16 smithi083 ceph-mon[77242]: pgmap v15778: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:17.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:14:16 smithi083 ceph-mon[79930]: pgmap v15778: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:17.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:14:16 smithi153 ceph-mon[70044]: pgmap v15778: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:19.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:14:18 smithi083 ceph-mon[79930]: pgmap v15779: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:19.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:14:18 smithi083 ceph-mon[77242]: pgmap v15779: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:19.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:14:18 smithi153 ceph-mon[70044]: pgmap v15779: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:21.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:14:20 smithi083 ceph-mon[79930]: pgmap v15780: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:21.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:14:20 smithi083 ceph-mon[77242]: pgmap v15780: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:21.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:14:20 smithi153 ceph-mon[70044]: pgmap v15780: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:14:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:14:21] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T14:14:22.411 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:14:22.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:14:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:14:22.131Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:14:22.714 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:14:22.714 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 7m ago 8h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:14:22.714 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 5m ago 8h 63.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:14:22.715 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 7m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:14:22.715 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 5m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:14:22.715 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 5m ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:14:22.715 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 7m ago 8h 943M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:14:22.715 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 7m ago 8h 1914M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:14:22.716 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 5m ago 8h 830M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:14:22.716 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 7m ago 8h 1143M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:14:22.716 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 7m ago 8h 24.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:14:22.716 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 5m ago 8h 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:14:22.716 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 7m ago 8h 1386M 2262M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:14:22.717 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 7m ago 8h 1373M 2262M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:14:22.717 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 7m ago 8h 1068M 2262M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:14:22.717 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 7m ago 8h 1345M 2262M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:14:22.717 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 5m ago 8h 1180M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:14:22.718 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 5m ago 8h 1480M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:14:22.718 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 5m ago 8h 1323M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:14:22.718 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 5m ago 8h 1572M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:14:22.718 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 5m ago 8h 189M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:14:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:14:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:14:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:14:23.091 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:14:23.091 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:14:23.091 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:14:23.091 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:14:23.092 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:14:23.092 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:14:23.092 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:14:23.092 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:14:23.093 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:14:23.093 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:14:23.093 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:14:23.093 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:14:23.093 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:14:23.094 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:14:23.094 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:14:23.094 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:14:23.094 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:14:23.094 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:14:23.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:14:22 smithi083 ceph-mon[79930]: pgmap v15781: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:23.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:14:22 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:14:23.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:14:22 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:14:23.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:14:22 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:14:23.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:14:22 smithi083 ceph-mon[77242]: pgmap v15781: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:23.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:14:22 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:14:23.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:14:22 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:14:23.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:14:22 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:14:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:14:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:14:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:14:23.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:14:22 smithi153 ceph-mon[70044]: pgmap v15781: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:23.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:14:22 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:14:23.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:14:22 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:14:23.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:14:22 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:14:24.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:14:23 smithi083 ceph-mon[77242]: from='client.62413 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:14:24.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:14:23 smithi083 ceph-mon[77242]: from='client.62419 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:14:24.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:14:23 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:14:24.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:14:23 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/513036328' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:14:24.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:14:23 smithi083 ceph-mon[79930]: from='client.62413 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:14:24.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:14:23 smithi083 ceph-mon[79930]: from='client.62419 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:14:24.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:14:23 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:14:24.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:14:23 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/513036328' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:14:24.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:14:23 smithi153 ceph-mon[70044]: from='client.62413 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:14:24.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:14:23 smithi153 ceph-mon[70044]: from='client.62419 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:14:24.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:14:23 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:14:24.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:14:23 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/513036328' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:14:25.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:14:24 smithi083 ceph-mon[79930]: pgmap v15782: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:25.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:14:24 smithi083 ceph-mon[77242]: pgmap v15782: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:25.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:14:24 smithi153 ceph-mon[70044]: pgmap v15782: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:25.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:14:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:14:25.490Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:14:26.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:14:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:14:26.223Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:14:27.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:14:26 smithi083 ceph-mon[79930]: pgmap v15783: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:27.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:14:26 smithi083 ceph-mon[77242]: pgmap v15783: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:27.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:14:26 smithi153 ceph-mon[70044]: pgmap v15783: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:28.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:14:27 smithi083 ceph-mon[77242]: pgmap v15784: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:28.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:14:27 smithi083 ceph-mon[79930]: pgmap v15784: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:28.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:14:27 smithi153 ceph-mon[70044]: pgmap v15784: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:30.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:14:30 smithi083 ceph-mon[79930]: pgmap v15785: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:30.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:14:30 smithi083 ceph-mon[77242]: pgmap v15785: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:30.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:14:30 smithi153 ceph-mon[70044]: pgmap v15785: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:14:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:14:31] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T14:14:32.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:14:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:14:32.132Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:14:32.649 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:14:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:14:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:14:32.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:14:32 smithi083 ceph-mon[79930]: pgmap v15786: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:32.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:14:32 smithi083 ceph-mon[77242]: pgmap v15786: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:33.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:14:32 smithi153 ceph-mon[70044]: pgmap v15786: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:14:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:14:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:14:34.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:14:34 smithi083 ceph-mon[77242]: pgmap v15787: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:34.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:14:34 smithi083 ceph-mon[79930]: pgmap v15787: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:35.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:14:34 smithi153 ceph-mon[70044]: pgmap v15787: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:35.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:14:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:14:35.491Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:14:36.654 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:14:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:14:36.224Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:14:36.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:14:36 smithi083 ceph-mon[77242]: pgmap v15788: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:36.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:14:36 smithi083 ceph-mon[79930]: pgmap v15788: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:37.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:14:36 smithi153 ceph-mon[70044]: pgmap v15788: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:38.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:14:38 smithi083 ceph-mon[79930]: pgmap v15789: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:38.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:14:38 smithi083 ceph-mon[77242]: pgmap v15789: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:39.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:14:38 smithi153 ceph-mon[70044]: pgmap v15789: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:14:40 smithi083 ceph-mon[79930]: pgmap v15790: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:14:40 smithi083 ceph-mon[77242]: pgmap v15790: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:14:40 smithi153 ceph-mon[70044]: pgmap v15790: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:41.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:14:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:14:41] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T14:14:41.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:14:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:14:41.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:14:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:14:41.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:14:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:14:41.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:14:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:14:41.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:14:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:14:41.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:14:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:14:41.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:14:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:14:41.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:14:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:14:42.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:14:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:14:42.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:14:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:14:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:14:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:14:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:14:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:14:42.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:14:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:14:42.133Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:14:42.668 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:14:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:14:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:14:42.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:14:42 smithi083 ceph-mon[79930]: pgmap v15791: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:42.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:14:42 smithi083 ceph-mon[77242]: pgmap v15791: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:43.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:14:42 smithi153 ceph-mon[70044]: pgmap v15791: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:14:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:14:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:14:44.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:14:44 smithi083 ceph-mon[79930]: pgmap v15792: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:44.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:14:44 smithi083 ceph-mon[77242]: pgmap v15792: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:45.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:14:44 smithi153 ceph-mon[70044]: pgmap v15792: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:45.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:14:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:14:45.492Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:14:46.675 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:14:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:14:46.225Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:14:46.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:14:46 smithi083 ceph-mon[79930]: pgmap v15793: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:46.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:14:46 smithi083 ceph-mon[77242]: pgmap v15793: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:47.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:14:46 smithi153 ceph-mon[70044]: pgmap v15793: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:48.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:14:48 smithi083 ceph-mon[77242]: pgmap v15794: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:48.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:14:48 smithi083 ceph-mon[79930]: pgmap v15794: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:49.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:14:48 smithi153 ceph-mon[70044]: pgmap v15794: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:50.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:14:50 smithi083 ceph-mon[77242]: pgmap v15795: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:50.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:14:50 smithi083 ceph-mon[79930]: pgmap v15795: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:51.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:14:50 smithi153 ceph-mon[70044]: pgmap v15795: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:14:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:14:51] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T14:14:52.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:14:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:14:52.133Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:14:52.684 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:14:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:14:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:14:52.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:14:52 smithi083 ceph-mon[77242]: pgmap v15796: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:52.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:14:52 smithi083 ceph-mon[79930]: pgmap v15796: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:53.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:14:52 smithi153 ceph-mon[70044]: pgmap v15796: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:14:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:14:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:14:53.425 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:14:53.718 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:14:53.719 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 7m ago 8h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:14:53.719 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 5m ago 8h 63.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:14:53.719 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 7m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:14:53.719 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 5m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:14:53.720 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 5m ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:14:53.720 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 7m ago 8h 943M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:14:53.720 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 7m ago 8h 1914M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:14:53.720 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 5m ago 8h 830M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:14:53.721 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 7m ago 8h 1143M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:14:53.721 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 7m ago 8h 24.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:14:53.721 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 5m ago 8h 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:14:53.721 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 7m ago 8h 1386M 2262M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:14:53.721 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 7m ago 8h 1373M 2262M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:14:53.722 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 7m ago 8h 1068M 2262M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:14:53.722 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 7m ago 8h 1345M 2262M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:14:53.722 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 5m ago 8h 1180M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:14:53.722 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 5m ago 8h 1480M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:14:53.723 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 5m ago 8h 1323M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:14:53.723 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 5m ago 8h 1572M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:14:53.723 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 5m ago 8h 189M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:14:54.085 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:14:54.085 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:14:54.085 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:14:54.086 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:14:54.086 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:14:54.086 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:14:54.086 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:14:54.087 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:14:54.087 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:14:54.087 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:14:54.087 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:14:54.088 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:14:54.088 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:14:54.088 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:14:54.088 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:14:54.089 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:14:54.089 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:14:54.089 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:14:54.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:14:54 smithi083 ceph-mon[77242]: pgmap v15797: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:54.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:14:54 smithi083 ceph-mon[77242]: from='client.62431 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:14:54.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:14:54 smithi083 ceph-mon[77242]: from='client.52506 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:14:54.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:14:54 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3600685010' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:14:54.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:14:54 smithi083 ceph-mon[79930]: pgmap v15797: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:54.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:14:54 smithi083 ceph-mon[79930]: from='client.62431 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:14:54.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:14:54 smithi083 ceph-mon[79930]: from='client.52506 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:14:54.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:14:54 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3600685010' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:14:55.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:14:54 smithi153 ceph-mon[70044]: pgmap v15797: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:55.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:14:54 smithi153 ceph-mon[70044]: from='client.62431 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:14:55.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:14:54 smithi153 ceph-mon[70044]: from='client.52506 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:14:55.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:14:54 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3600685010' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:14:55.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:14:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:14:55.492Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:14:56.691 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:14:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:14:56.226Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:14:56.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:14:56 smithi083 ceph-mon[79930]: pgmap v15798: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:56.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:14:56 smithi083 ceph-mon[77242]: pgmap v15798: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:57.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:14:56 smithi153 ceph-mon[70044]: pgmap v15798: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:58.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:14:58 smithi083 ceph-mon[79930]: pgmap v15799: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:58.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:14:58 smithi083 ceph-mon[77242]: pgmap v15799: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:14:59.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:14:58 smithi153 ceph-mon[70044]: pgmap v15799: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:00.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:15:00 smithi083 ceph-mon[79930]: pgmap v15800: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:00.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:15:00 smithi083 ceph-mon[77242]: pgmap v15800: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:01.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:15:00 smithi153 ceph-mon[70044]: pgmap v15800: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:15:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:15:01] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T14:15:02.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:15:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:15:02.134Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:15:02.701 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:15:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:15:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:15:02.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:15:02 smithi083 ceph-mon[79930]: pgmap v15801: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:02.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:15:02 smithi083 ceph-mon[77242]: pgmap v15801: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:03.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:15:02 smithi153 ceph-mon[70044]: pgmap v15801: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:15:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:15:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:15:04.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:15:04 smithi083 ceph-mon[77242]: pgmap v15802: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:04.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:15:04 smithi083 ceph-mon[79930]: pgmap v15802: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:05.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:15:04 smithi153 ceph-mon[70044]: pgmap v15802: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:05.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:15:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:15:05.493Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:15:06.706 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:15:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:15:06.226Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:15:06.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:15:06 smithi083 ceph-mon[77242]: pgmap v15803: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:06.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:15:06 smithi083 ceph-mon[79930]: pgmap v15803: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:07.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:15:06 smithi153 ceph-mon[70044]: pgmap v15803: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:08.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:15:08 smithi083 ceph-mon[79930]: pgmap v15804: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:08.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:15:08 smithi083 ceph-mon[77242]: pgmap v15804: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:09.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:15:08 smithi153 ceph-mon[70044]: pgmap v15804: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:10.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:15:10 smithi083 ceph-mon[79930]: pgmap v15805: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:10.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:15:10 smithi083 ceph-mon[77242]: pgmap v15805: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:11.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:15:10 smithi153 ceph-mon[70044]: pgmap v15805: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:15:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:15:11] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T14:15:12.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:15:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:15:12.134Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:15:12.715 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:15:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:15:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:15:13.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:15:12 smithi153 ceph-mon[70044]: pgmap v15806: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:13.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:15:12 smithi083 ceph-mon[79930]: pgmap v15806: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:13.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:15:12 smithi083 ceph-mon[77242]: pgmap v15806: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:15:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:15:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:15:15.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:15:14 smithi153 ceph-mon[70044]: pgmap v15807: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:15.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:15:14 smithi083 ceph-mon[79930]: pgmap v15807: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:15.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:15:14 smithi083 ceph-mon[77242]: pgmap v15807: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:15.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:15:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:15:15.494Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:15:16.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:15:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:15:16.227Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:15:17.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:15:16 smithi153 ceph-mon[70044]: pgmap v15808: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:17.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:15:16 smithi083 ceph-mon[79930]: pgmap v15808: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:17.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:15:16 smithi083 ceph-mon[77242]: pgmap v15808: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:19.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:15:18 smithi153 ceph-mon[70044]: pgmap v15809: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:19.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:15:18 smithi083 ceph-mon[79930]: pgmap v15809: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:19.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:15:18 smithi083 ceph-mon[77242]: pgmap v15809: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:21.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:15:20 smithi153 ceph-mon[70044]: pgmap v15810: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:21.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:15:20 smithi083 ceph-mon[79930]: pgmap v15810: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:21.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:15:20 smithi083 ceph-mon[77242]: pgmap v15810: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:15:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:15:21] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T14:15:22.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:15:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:15:22.135Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:15:22.733 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:15:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:15:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:15:23.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:15:22 smithi153 ceph-mon[70044]: pgmap v15811: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:23.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:15:22 smithi083 ceph-mon[77242]: pgmap v15811: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:23.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:15:22 smithi083 ceph-mon[79930]: pgmap v15811: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:15:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:15:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:15:24.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:15:23 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:15:24.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:15:23 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:15:24.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:15:23 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:15:24.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:15:23 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:15:24.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:15:23 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:15:24.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:15:23 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:15:24.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:15:23 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:15:24.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:15:23 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:15:24.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:15:23 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:15:24.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:15:23 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:15:24.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:15:23 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:15:24.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:15:23 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:15:24.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:15:23 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:15:24.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:15:23 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:15:24.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:15:23 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:15:24.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:15:23 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:15:24.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:15:23 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:15:24.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:15:23 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:15:24.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:15:23 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:15:24.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:15:23 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:15:24.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:15:23 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:15:24.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:15:23 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:15:24.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:15:23 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:15:24.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:15:23 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:15:24.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:15:23 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:15:24.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:15:23 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:15:24.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:15:23 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:15:24.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:15:23 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:15:24.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:15:23 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:15:24.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:15:23 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:15:24.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:15:23 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:15:24.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:15:23 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:15:24.218 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:15:23 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:15:24.218 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:15:23 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:15:24.218 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:15:23 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:15:24.218 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:15:23 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:15:24.218 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:15:23 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:15:24.219 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:15:23 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:15:24.219 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:15:23 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:15:24.402 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:15:24.691 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:15:24.692 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 8m ago 8h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:15:24.692 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 6m ago 8h 63.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:15:24.692 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 8m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:15:24.693 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 6m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:15:24.693 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 6m ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:15:24.693 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 8m ago 8h 943M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:15:24.693 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 8m ago 8h 1914M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:15:24.694 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 6m ago 8h 830M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:15:24.694 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 8m ago 8h 1143M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:15:24.694 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 8m ago 8h 24.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:15:24.694 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 6m ago 8h 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:15:24.695 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 8m ago 8h 1386M 2247M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:15:24.695 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 8m ago 8h 1373M 2247M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:15:24.695 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 8m ago 8h 1068M 2247M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:15:24.695 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 8m ago 8h 1345M 2247M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:15:24.696 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 6m ago 8h 1180M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:15:24.696 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 6m ago 8h 1480M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:15:24.696 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 6m ago 8h 1323M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:15:24.696 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 6m ago 8h 1572M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:15:24.696 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 6m ago 8h 189M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:15:25.048 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:15:25.048 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:15:25.049 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:15:25.049 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:15:25.049 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:15:25.050 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:15:25.050 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:15:25.050 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:15:25.050 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:15:25.050 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:15:25.051 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:15:25.051 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:15:25.051 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:15:25.051 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:15:25.052 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:15:25.052 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:15:25.052 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:15:25.052 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:15:25.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:15:24 smithi153 ceph-mon[70044]: pgmap v15812: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:25.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:15:24 smithi153 ceph-mon[70044]: Adjusting osd_memory_target on smithi083 to 2247M 2022-09-29T14:15:25.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:15:24 smithi083 ceph-mon[79930]: pgmap v15812: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:25.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:15:24 smithi083 ceph-mon[79930]: Adjusting osd_memory_target on smithi083 to 2247M 2022-09-29T14:15:25.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:15:24 smithi083 ceph-mon[77242]: pgmap v15812: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:25.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:15:24 smithi083 ceph-mon[77242]: Adjusting osd_memory_target on smithi083 to 2247M 2022-09-29T14:15:25.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:15:25 smithi083 ceph-mon[79930]: from='client.62449 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:15:25.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:15:25 smithi083 ceph-mon[79930]: from='client.62455 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:15:25.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:15:25 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3366976223' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:15:25.964 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:15:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:15:25.494Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:15:25.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:15:25 smithi083 ceph-mon[77242]: from='client.62449 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:15:25.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:15:25 smithi083 ceph-mon[77242]: from='client.62455 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:15:25.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:15:25 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3366976223' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:15:26.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:15:25 smithi153 ceph-mon[70044]: from='client.62449 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:15:26.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:15:25 smithi153 ceph-mon[70044]: from='client.62455 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:15:26.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:15:25 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3366976223' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:15:26.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:15:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:15:26.228Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:15:27.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:15:26 smithi153 ceph-mon[70044]: pgmap v15813: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:27.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:15:26 smithi083 ceph-mon[79930]: pgmap v15813: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:27.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:15:26 smithi083 ceph-mon[77242]: pgmap v15813: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:29.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:15:28 smithi153 ceph-mon[70044]: pgmap v15814: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:29.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:15:28 smithi083 ceph-mon[79930]: pgmap v15814: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:29.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:15:28 smithi083 ceph-mon[77242]: pgmap v15814: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:31.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:15:30 smithi153 ceph-mon[70044]: pgmap v15815: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:31.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:15:30 smithi083 ceph-mon[79930]: pgmap v15815: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:31.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:15:30 smithi083 ceph-mon[77242]: pgmap v15815: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:15:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:15:31] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T14:15:32.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:15:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:15:32.137Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:15:32.757 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:15:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:15:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:15:33.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:15:32 smithi153 ceph-mon[70044]: pgmap v15816: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:33.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:15:32 smithi083 ceph-mon[77242]: pgmap v15816: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:33.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:15:32 smithi083 ceph-mon[79930]: pgmap v15816: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:15:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:15:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:15:35.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:15:34 smithi153 ceph-mon[70044]: pgmap v15817: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:35.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:15:34 smithi083 ceph-mon[79930]: pgmap v15817: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:35.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:15:34 smithi083 ceph-mon[77242]: pgmap v15817: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:35.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:15:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:15:35.495Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:15:36.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:15:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:15:36.228Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:15:37.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:15:36 smithi153 ceph-mon[70044]: pgmap v15818: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:37.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:15:36 smithi083 ceph-mon[77242]: pgmap v15818: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:37.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:15:36 smithi083 ceph-mon[79930]: pgmap v15818: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:39.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:15:38 smithi153 ceph-mon[70044]: pgmap v15819: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:39.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:15:38 smithi083 ceph-mon[77242]: pgmap v15819: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:39.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:15:38 smithi083 ceph-mon[79930]: pgmap v15819: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:15:40 smithi153 ceph-mon[70044]: pgmap v15820: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:41.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:15:40 smithi083 ceph-mon[77242]: pgmap v15820: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:41.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:15:40 smithi083 ceph-mon[79930]: pgmap v15820: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:41.777 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:15:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:15:41] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T14:15:42.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:15:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:15:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:15:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:15:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:15:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:15:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:15:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:15:42.140 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:15:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:15:42.199 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:15:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:15:42.199 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:15:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:15:42.200 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:15:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:15:42.200 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:15:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:15:42.200 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:15:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:15:42.201 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:15:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:15:42.201 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:15:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:15:42.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:15:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:15:42.138Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:15:42.779 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:15:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:15:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:15:43.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:15:42 smithi153 ceph-mon[70044]: pgmap v15821: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:43.059 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:15:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:15:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:15:43.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:15:42 smithi083 ceph-mon[77242]: pgmap v15821: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:43.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:15:42 smithi083 ceph-mon[79930]: pgmap v15821: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:45.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:15:44 smithi153 ceph-mon[70044]: pgmap v15822: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:45.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:15:44 smithi083 ceph-mon[79930]: pgmap v15822: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:45.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:15:44 smithi083 ceph-mon[77242]: pgmap v15822: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:45.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:15:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:15:45.496Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:15:46.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:15:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:15:46.229Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:15:47.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:15:46 smithi153 ceph-mon[70044]: pgmap v15823: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:47.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:15:46 smithi083 ceph-mon[77242]: pgmap v15823: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:47.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:15:46 smithi083 ceph-mon[79930]: pgmap v15823: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:49.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:15:48 smithi153 ceph-mon[70044]: pgmap v15824: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:49.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:15:48 smithi083 ceph-mon[79930]: pgmap v15824: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:49.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:15:48 smithi083 ceph-mon[77242]: pgmap v15824: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:51.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:15:50 smithi153 ceph-mon[70044]: pgmap v15825: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:51.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:15:50 smithi083 ceph-mon[79930]: pgmap v15825: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:51.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:15:50 smithi083 ceph-mon[77242]: pgmap v15825: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:15:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:15:51] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T14:15:52.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:15:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:15:52.138Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:15:52.797 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:15:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:15:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:15:53.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:15:52 smithi153 ceph-mon[70044]: pgmap v15826: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:53.059 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:15:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:15:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:15:53.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:15:52 smithi083 ceph-mon[79930]: pgmap v15826: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:53.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:15:52 smithi083 ceph-mon[77242]: pgmap v15826: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:55.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:15:54 smithi153 ceph-mon[70044]: pgmap v15827: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:55.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:15:54 smithi083 ceph-mon[79930]: pgmap v15827: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:55.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:15:54 smithi083 ceph-mon[77242]: pgmap v15827: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:55.374 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:15:55.663 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:15:55.663 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 8m ago 8h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:15:55.663 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 6m ago 8h 63.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:15:55.664 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 8m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:15:55.664 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 6m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:15:55.664 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 6m ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:15:55.664 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 8m ago 8h 943M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:15:55.665 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 8m ago 8h 1914M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:15:55.665 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 6m ago 8h 830M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:15:55.665 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 8m ago 8h 1143M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:15:55.665 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 8m ago 8h 24.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:15:55.666 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 6m ago 8h 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:15:55.666 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 8m ago 8h 1386M 2247M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:15:55.666 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 8m ago 8h 1373M 2247M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:15:55.666 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 8m ago 8h 1068M 2247M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:15:55.667 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 8m ago 8h 1345M 2247M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:15:55.667 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 6m ago 8h 1180M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:15:55.667 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 6m ago 8h 1480M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:15:55.667 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 6m ago 8h 1323M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:15:55.667 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 6m ago 8h 1572M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:15:55.668 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 6m ago 8h 189M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:15:55.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:15:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:15:55.495Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:15:56.018 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:15:56.019 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:15:56.019 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:15:56.019 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:15:56.020 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:15:56.020 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:15:56.020 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:15:56.020 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:15:56.020 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:15:56.021 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:15:56.021 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:15:56.021 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:15:56.021 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:15:56.022 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:15:56.022 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:15:56.022 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:15:56.022 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:15:56.022 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:15:56.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:15:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:15:56.230Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:15:57.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:15:56 smithi153 ceph-mon[70044]: from='client.52536 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:15:57.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:15:56 smithi153 ceph-mon[70044]: pgmap v15828: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:57.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:15:56 smithi153 ceph-mon[70044]: from='client.62473 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:15:57.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:15:56 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1024915413' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:15:57.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:15:56 smithi083 ceph-mon[79930]: from='client.52536 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:15:57.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:15:56 smithi083 ceph-mon[79930]: pgmap v15828: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:57.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:15:56 smithi083 ceph-mon[79930]: from='client.62473 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:15:57.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:15:56 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1024915413' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:15:57.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:15:56 smithi083 ceph-mon[77242]: from='client.52536 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:15:57.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:15:56 smithi083 ceph-mon[77242]: pgmap v15828: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:57.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:15:56 smithi083 ceph-mon[77242]: from='client.62473 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:15:57.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:15:56 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1024915413' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:15:59.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:15:58 smithi153 ceph-mon[70044]: pgmap v15829: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:59.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:15:58 smithi083 ceph-mon[79930]: pgmap v15829: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:15:59.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:15:58 smithi083 ceph-mon[77242]: pgmap v15829: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:01.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:16:00 smithi083 ceph-mon[79930]: pgmap v15830: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:01.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:16:00 smithi083 ceph-mon[77242]: pgmap v15830: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:01.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:16:00 smithi153 ceph-mon[70044]: pgmap v15830: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:16:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:16:01] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T14:16:02.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:16:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:16:02.139Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:16:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:16:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:16:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:16:03.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:16:02 smithi083 ceph-mon[79930]: pgmap v15831: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:03.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:16:02 smithi083 ceph-mon[77242]: pgmap v15831: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:03.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:16:02 smithi153 ceph-mon[70044]: pgmap v15831: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:16:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:16:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:16:05.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:16:04 smithi083 ceph-mon[79930]: pgmap v15832: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:05.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:16:04 smithi083 ceph-mon[77242]: pgmap v15832: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:05.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:16:04 smithi153 ceph-mon[70044]: pgmap v15832: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:05.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:16:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:16:05.496Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:16:06.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:16:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:16:06.232Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:16:07.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:16:06 smithi083 ceph-mon[79930]: pgmap v15833: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:07.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:16:06 smithi083 ceph-mon[77242]: pgmap v15833: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:07.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:16:06 smithi153 ceph-mon[70044]: pgmap v15833: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:09.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:16:08 smithi083 ceph-mon[79930]: pgmap v15834: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:09.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:16:08 smithi083 ceph-mon[77242]: pgmap v15834: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:09.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:16:08 smithi153 ceph-mon[70044]: pgmap v15834: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:11.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:16:10 smithi083 ceph-mon[79930]: pgmap v15835: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:11.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:16:10 smithi083 ceph-mon[77242]: pgmap v15835: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:11.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:16:10 smithi153 ceph-mon[70044]: pgmap v15835: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:16:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:16:11] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T14:16:12.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:16:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:16:12.139Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:16:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:16:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:16:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:16:13.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:16:12 smithi083 ceph-mon[79930]: pgmap v15836: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:13.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:16:12 smithi083 ceph-mon[77242]: pgmap v15836: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:13.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:16:12 smithi153 ceph-mon[70044]: pgmap v15836: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:13.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:16:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:16:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:16:15.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:16:14 smithi083 ceph-mon[79930]: pgmap v15837: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:15.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:16:14 smithi083 ceph-mon[77242]: pgmap v15837: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:15.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:16:14 smithi153 ceph-mon[70044]: pgmap v15837: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:15.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:16:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:16:15.496Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:16:16.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:16:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:16:16.232Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:16:17.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:16:16 smithi083 ceph-mon[79930]: pgmap v15838: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:17.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:16:16 smithi083 ceph-mon[77242]: pgmap v15838: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:17.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:16:16 smithi153 ceph-mon[70044]: pgmap v15838: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:19.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:16:18 smithi083 ceph-mon[77242]: pgmap v15839: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:19.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:16:18 smithi083 ceph-mon[79930]: pgmap v15839: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:19.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:16:18 smithi153 ceph-mon[70044]: pgmap v15839: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:21.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:16:20 smithi083 ceph-mon[79930]: pgmap v15840: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:21.319 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:16:20 smithi083 ceph-mon[77242]: pgmap v15840: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:21.319 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:16:20 smithi153 ceph-mon[70044]: pgmap v15840: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:16:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:16:21] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T14:16:22.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:16:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:16:22.140Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:16:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:16:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:16:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:16:23.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:16:22 smithi083 ceph-mon[79930]: pgmap v15841: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:23.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:16:22 smithi083 ceph-mon[77242]: pgmap v15841: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:23.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:16:22 smithi153 ceph-mon[70044]: pgmap v15841: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:23.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:16:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:16:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:16:24.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:16:23 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:16:24.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:16:23 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:16:24.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:16:23 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:16:24.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:16:23 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:16:24.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:16:23 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:16:24.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:16:23 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:16:24.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:16:23 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:16:24.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:16:23 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:16:24.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:16:23 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:16:25.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:16:24 smithi083 ceph-mon[79930]: pgmap v15842: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:25.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:16:24 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:16:25.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:16:24 smithi083 ceph-mon[77242]: pgmap v15842: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:25.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:16:24 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:16:25.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:16:24 smithi153 ceph-mon[70044]: pgmap v15842: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:25.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:16:24 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:16:25.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:16:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:16:25.497Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:16:26.347 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:16:26.635 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:16:26.636 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 9m ago 8h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:16:26.636 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 7m ago 8h 63.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:16:26.636 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 9m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:16:26.637 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 7m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:16:26.637 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 7m ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:16:26.637 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 9m ago 8h 943M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:16:26.637 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 9m ago 8h 1914M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:16:26.637 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 7m ago 8h 830M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:16:26.638 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 9m ago 8h 1143M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:16:26.638 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 9m ago 8h 24.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:16:26.638 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 7m ago 8h 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:16:26.638 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 9m ago 8h 1386M 2247M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:16:26.639 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 9m ago 8h 1373M 2247M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:16:26.639 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 9m ago 8h 1068M 2247M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:16:26.639 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 9m ago 8h 1345M 2247M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:16:26.640 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 7m ago 8h 1180M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:16:26.640 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 7m ago 8h 1480M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:16:26.640 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 7m ago 8h 1323M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:16:26.640 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 7m ago 8h 1572M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:16:26.641 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 7m ago 8h 189M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:16:26.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:16:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:16:26.232Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:16:26.995 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:16:26.996 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:16:26.996 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:16:26.996 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:16:26.997 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:16:26.997 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:16:26.997 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:16:26.997 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:16:26.997 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:16:26.998 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:16:26.998 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:16:26.998 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:16:26.998 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:16:26.999 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:16:26.999 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:16:26.999 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:16:26.999 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:16:27.000 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:16:27.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:16:26 smithi083 ceph-mon[79930]: pgmap v15843: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:27.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:16:26 smithi083 ceph-mon[77242]: pgmap v15843: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:27.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:16:26 smithi153 ceph-mon[70044]: pgmap v15843: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:28.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:16:27 smithi083 ceph-mon[79930]: from='client.62485 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:16:28.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:16:27 smithi083 ceph-mon[79930]: from='client.52560 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:16:28.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:16:27 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/81354280' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:16:28.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:16:27 smithi083 ceph-mon[77242]: from='client.62485 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:16:28.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:16:27 smithi083 ceph-mon[77242]: from='client.52560 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:16:28.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:16:27 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/81354280' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:16:28.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:16:27 smithi153 ceph-mon[70044]: from='client.62485 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:16:28.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:16:27 smithi153 ceph-mon[70044]: from='client.52560 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:16:28.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:16:27 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/81354280' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:16:29.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:16:28 smithi083 ceph-mon[79930]: pgmap v15844: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:29.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:16:28 smithi083 ceph-mon[77242]: pgmap v15844: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:29.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:16:28 smithi153 ceph-mon[70044]: pgmap v15844: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:31.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:16:30 smithi083 ceph-mon[79930]: pgmap v15845: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:31.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:16:30 smithi083 ceph-mon[77242]: pgmap v15845: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:31.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:16:30 smithi153 ceph-mon[70044]: pgmap v15845: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:16:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:16:31] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T14:16:32.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:16:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:16:32.140Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:16:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:16:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:16:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:16:33.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:16:32 smithi083 ceph-mon[79930]: pgmap v15846: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:33.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:16:32 smithi083 ceph-mon[77242]: pgmap v15846: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:33.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:16:32 smithi153 ceph-mon[70044]: pgmap v15846: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:33.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:16:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:16:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:16:35.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:16:34 smithi083 ceph-mon[79930]: pgmap v15847: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:35.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:16:34 smithi083 ceph-mon[77242]: pgmap v15847: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:35.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:16:34 smithi153 ceph-mon[70044]: pgmap v15847: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:35.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:16:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:16:35.498Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:16:36.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:16:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:16:36.233Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:16:37.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:16:36 smithi083 ceph-mon[79930]: pgmap v15848: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:37.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:16:36 smithi083 ceph-mon[77242]: pgmap v15848: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:37.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:16:36 smithi153 ceph-mon[70044]: pgmap v15848: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:39.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:16:38 smithi083 ceph-mon[79930]: pgmap v15849: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:39.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:16:38 smithi083 ceph-mon[77242]: pgmap v15849: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:39.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:16:38 smithi153 ceph-mon[70044]: pgmap v15849: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:41.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:16:40 smithi083 ceph-mon[79930]: pgmap v15850: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:16:40 smithi083 ceph-mon[77242]: pgmap v15850: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:16:40 smithi153 ceph-mon[70044]: pgmap v15850: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:41.882 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:16:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:16:41] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T14:16:42.143 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:16:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:16:42.144 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:16:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:16:42.144 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:16:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:16:42.144 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:16:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:16:42.145 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:16:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:16:42.145 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:16:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:16:42.145 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:16:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:16:42.146 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:16:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:16:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:16:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:16:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:16:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:16:42.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:16:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:16:42.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:16:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:16:42.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:16:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:16:42.141Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:16:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:16:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:16:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:16:43.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:16:42 smithi083 ceph-mon[79930]: pgmap v15851: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:43.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:16:42 smithi083 ceph-mon[77242]: pgmap v15851: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:43.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:16:42 smithi153 ceph-mon[70044]: pgmap v15851: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:43.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:16:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:16:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:16:45.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:16:44 smithi083 ceph-mon[79930]: pgmap v15852: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:45.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:16:44 smithi083 ceph-mon[77242]: pgmap v15852: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:45.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:16:44 smithi153 ceph-mon[70044]: pgmap v15852: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:45.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:16:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:16:45.499Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:16:46.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:16:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:16:46.233Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:16:47.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:16:46 smithi083 ceph-mon[77242]: pgmap v15853: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:47.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:16:46 smithi083 ceph-mon[79930]: pgmap v15853: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:47.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:16:46 smithi153 ceph-mon[70044]: pgmap v15853: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:49.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:16:48 smithi083 ceph-mon[79930]: pgmap v15854: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:49.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:16:48 smithi083 ceph-mon[77242]: pgmap v15854: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:49.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:16:48 smithi153 ceph-mon[70044]: pgmap v15854: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:51.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:16:50 smithi083 ceph-mon[79930]: pgmap v15855: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:51.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:16:50 smithi083 ceph-mon[77242]: pgmap v15855: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:51.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:16:50 smithi153 ceph-mon[70044]: pgmap v15855: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:16:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:16:51] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T14:16:52.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:16:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:16:52.141Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:16:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:16:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:16:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:16:53.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:16:52 smithi083 ceph-mon[79930]: pgmap v15856: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:53.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:16:52 smithi083 ceph-mon[77242]: pgmap v15856: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:53.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:16:52 smithi153 ceph-mon[70044]: pgmap v15856: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:53.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:16:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:16:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:16:55.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:16:54 smithi083 ceph-mon[79930]: pgmap v15857: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:55.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:16:54 smithi083 ceph-mon[77242]: pgmap v15857: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:55.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:16:54 smithi153 ceph-mon[70044]: pgmap v15857: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:55.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:16:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:16:55.500Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:16:56.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:16:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:16:56.234Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:16:57.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:16:56 smithi083 ceph-mon[79930]: pgmap v15858: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:57.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:16:56 smithi083 ceph-mon[77242]: pgmap v15858: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:57.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:16:56 smithi153 ceph-mon[70044]: pgmap v15858: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:57.325 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:16:57.614 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:16:57.615 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 9m ago 8h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:16:57.615 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 7m ago 8h 63.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:16:57.615 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 9m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:16:57.616 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 7m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:16:57.616 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 7m ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:16:57.616 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 9m ago 8h 943M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:16:57.617 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 9m ago 8h 1914M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:16:57.617 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 7m ago 8h 830M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:16:57.617 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 9m ago 8h 1143M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:16:57.617 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 9m ago 8h 24.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:16:57.618 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 7m ago 8h 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:16:57.618 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 9m ago 8h 1386M 2247M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:16:57.618 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 9m ago 8h 1373M 2247M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:16:57.619 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 9m ago 8h 1068M 2247M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:16:57.619 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 9m ago 8h 1345M 2247M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:16:57.619 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 7m ago 8h 1180M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:16:57.620 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 7m ago 8h 1480M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:16:57.620 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 7m ago 8h 1323M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:16:57.620 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 7m ago 8h 1572M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:16:57.620 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 7m ago 8h 189M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:16:57.969 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:16:57.969 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:16:57.969 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:16:57.969 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:16:57.970 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:16:57.970 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:16:57.970 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:16:57.970 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:16:57.971 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:16:57.971 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:16:57.971 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:16:57.971 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:16:57.972 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:16:57.972 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:16:57.972 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:16:57.972 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:16:57.972 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:16:57.973 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:16:59.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:16:58 smithi083 ceph-mon[79930]: from='client.52572 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:16:59.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:16:58 smithi083 ceph-mon[79930]: pgmap v15859: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:59.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:16:58 smithi083 ceph-mon[79930]: from='client.52578 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:16:59.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:16:58 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4189919735' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:16:59.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:16:58 smithi083 ceph-mon[77242]: from='client.52572 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:16:59.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:16:58 smithi083 ceph-mon[77242]: pgmap v15859: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:59.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:16:58 smithi083 ceph-mon[77242]: from='client.52578 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:16:59.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:16:58 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4189919735' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:16:59.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:16:58 smithi153 ceph-mon[70044]: from='client.52572 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:16:59.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:16:58 smithi153 ceph-mon[70044]: pgmap v15859: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:16:59.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:16:58 smithi153 ceph-mon[70044]: from='client.52578 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:16:59.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:16:58 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4189919735' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:17:01.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:17:00 smithi083 ceph-mon[79930]: pgmap v15860: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:01.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:17:00 smithi083 ceph-mon[77242]: pgmap v15860: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:01.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:17:00 smithi153 ceph-mon[70044]: pgmap v15860: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:17:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:17:01] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T14:17:02.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:17:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:17:02.142Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:17:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:17:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:17:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:17:03.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:17:02 smithi083 ceph-mon[79930]: pgmap v15861: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:03.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:17:02 smithi083 ceph-mon[77242]: pgmap v15861: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:03.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:17:02 smithi153 ceph-mon[70044]: pgmap v15861: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:17:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:17:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:17:05.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:17:04 smithi083 ceph-mon[77242]: pgmap v15862: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:05.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:17:04 smithi083 ceph-mon[79930]: pgmap v15862: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:05.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:17:04 smithi153 ceph-mon[70044]: pgmap v15862: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:05.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:17:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:17:05.501Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:17:06.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:17:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:17:06.236Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:17:07.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:17:06 smithi083 ceph-mon[79930]: pgmap v15863: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:07.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:17:06 smithi083 ceph-mon[77242]: pgmap v15863: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:07.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:17:06 smithi153 ceph-mon[70044]: pgmap v15863: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:09.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:17:08 smithi083 ceph-mon[79930]: pgmap v15864: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:09.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:17:08 smithi083 ceph-mon[77242]: pgmap v15864: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:09.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:17:08 smithi153 ceph-mon[70044]: pgmap v15864: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:10.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:17:09 smithi083 ceph-mon[79930]: pgmap v15865: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:10.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:17:09 smithi083 ceph-mon[77242]: pgmap v15865: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:10.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:17:09 smithi153 ceph-mon[70044]: pgmap v15865: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:11.968 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:17:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:17:11] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T14:17:12.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:17:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:17:12.142Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:17:12.690 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:17:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:17:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:17:12.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:17:12 smithi083 ceph-mon[79930]: pgmap v15866: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:12.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:17:12 smithi083 ceph-mon[77242]: pgmap v15866: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:13.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:17:12 smithi153 ceph-mon[70044]: pgmap v15866: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:17:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:17:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:17:14.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:17:14 smithi083 ceph-mon[79930]: pgmap v15867: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:14.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:17:14 smithi083 ceph-mon[77242]: pgmap v15867: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:15.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:17:14 smithi153 ceph-mon[70044]: pgmap v15867: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:15.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:17:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:17:15.501Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:17:16.695 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:17:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:17:16.235Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:17:16.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:17:16 smithi083 ceph-mon[79930]: pgmap v15868: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:16.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:17:16 smithi083 ceph-mon[77242]: pgmap v15868: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:17.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:17:16 smithi153 ceph-mon[70044]: pgmap v15868: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:18.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:17:18 smithi083 ceph-mon[77242]: pgmap v15869: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:18.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:17:18 smithi083 ceph-mon[79930]: pgmap v15869: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:19.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:17:18 smithi153 ceph-mon[70044]: pgmap v15869: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:20.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:17:20 smithi083 ceph-mon[79930]: pgmap v15870: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:20.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:17:20 smithi083 ceph-mon[77242]: pgmap v15870: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:21.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:17:20 smithi153 ceph-mon[70044]: pgmap v15870: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:17:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:17:21] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T14:17:22.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:17:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:17:22.143Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:17:22.704 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:17:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:17:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:17:22.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:17:22 smithi083 ceph-mon[79930]: pgmap v15871: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:22.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:17:22 smithi083 ceph-mon[77242]: pgmap v15871: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:23.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:17:22 smithi153 ceph-mon[70044]: pgmap v15871: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:17:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:17:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:17:24.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:17:24 smithi083 ceph-mon[79930]: pgmap v15872: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:24.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:17:24 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:17:24.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:17:24 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:17:24.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:17:24 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:17:24.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:17:24 smithi083 ceph-mon[77242]: pgmap v15872: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:24.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:17:24 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:17:24.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:17:24 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:17:24.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:17:24 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:17:25.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:17:24 smithi153 ceph-mon[70044]: pgmap v15872: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:25.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:17:24 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:17:25.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:17:24 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:17:25.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:17:24 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:17:25.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:17:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:17:25.501Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:17:25.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:17:25 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:17:25.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:17:25 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:17:25.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:17:25 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:17:25.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:17:25 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:17:25.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:17:25 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:17:25.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:17:25 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:17:25.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:17:25 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:17:25.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:17:25 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:17:26.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:17:25 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:17:26.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:17:25 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:17:26.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:17:25 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:17:26.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:17:25 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:17:26.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:17:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:17:26.236Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:17:27.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:17:26 smithi153 ceph-mon[70044]: pgmap v15873: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:27.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:17:26 smithi083 ceph-mon[79930]: pgmap v15873: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:27.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:17:26 smithi083 ceph-mon[77242]: pgmap v15873: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:28.297 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:17:28.621 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:17:28.621 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 10m ago 8h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:17:28.622 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 8m ago 8h 63.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:17:28.622 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 10m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:17:28.622 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 8m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:17:28.622 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 8m ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:17:28.623 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 10m ago 8h 943M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:17:28.623 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 10m ago 8h 1914M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:17:28.623 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 8m ago 8h 830M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:17:28.623 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 10m ago 8h 1143M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:17:28.623 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 10m ago 8h 24.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:17:28.624 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 8m ago 8h 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:17:28.624 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 10m ago 8h 1386M 2247M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:17:28.624 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 10m ago 8h 1373M 2247M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:17:28.624 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 10m ago 8h 1068M 2247M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:17:28.625 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 10m ago 8h 1345M 2247M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:17:28.625 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 8m ago 8h 1180M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:17:28.625 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 8m ago 8h 1480M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:17:28.625 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 8m ago 8h 1323M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:17:28.625 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 8m ago 8h 1572M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:17:28.626 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 8m ago 8h 189M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:17:28.997 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:17:28.998 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:17:28.998 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:17:28.998 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:17:28.999 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:17:28.999 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:17:28.999 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:17:28.999 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:17:28.999 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:17:29.000 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:17:29.000 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:17:29.000 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:17:29.000 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:17:29.001 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:17:29.001 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:17:29.001 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:17:29.001 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:17:29.002 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:17:29.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:17:28 smithi153 ceph-mon[70044]: pgmap v15874: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:29.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:17:28 smithi083 ceph-mon[79930]: pgmap v15874: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:29.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:17:28 smithi083 ceph-mon[77242]: pgmap v15874: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:30.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:17:29 smithi153 ceph-mon[70044]: from='client.52590 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:17:30.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:17:29 smithi153 ceph-mon[70044]: from='client.52596 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:17:30.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:17:29 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:17:30.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:17:29 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:17:30.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:17:29 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1531403658' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:17:30.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:17:29 smithi083 ceph-mon[79930]: from='client.52590 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:17:30.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:17:29 smithi083 ceph-mon[79930]: from='client.52596 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:17:30.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:17:29 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:17:30.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:17:29 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:17:30.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:17:29 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1531403658' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:17:30.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:17:29 smithi083 ceph-mon[77242]: from='client.52590 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:17:30.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:17:29 smithi083 ceph-mon[77242]: from='client.52596 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:17:30.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:17:29 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:17:30.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:17:29 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:17:30.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:17:29 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1531403658' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:17:31.079 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:17:30 smithi083 ceph-mon[79930]: pgmap v15875: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:31.080 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:17:30 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:17:31.080 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:17:30 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:17:31.080 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:17:30 smithi083 ceph-mon[77242]: pgmap v15875: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:31.081 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:17:30 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:17:31.081 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:17:30 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:17:31.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:17:30 smithi153 ceph-mon[70044]: pgmap v15875: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:31.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:17:30 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:17:31.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:17:30 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:17:31.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:17:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:17:31] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T14:17:32.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:17:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:17:32.142Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:17:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:17:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:17:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:17:33.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:17:32 smithi083 ceph-mon[79930]: pgmap v15876: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:33.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:17:32 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:17:33.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:17:32 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:17:33.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:17:32 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:17:33.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:17:32 smithi083 ceph-mon[77242]: pgmap v15876: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:33.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:17:32 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:17:33.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:17:32 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:17:33.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:17:32 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:17:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:17:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:17:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:17:33.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:17:32 smithi153 ceph-mon[70044]: pgmap v15876: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:33.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:17:32 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:17:33.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:17:32 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:17:33.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:17:32 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:17:34.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:17:33 smithi083 ceph-mon[79930]: Detected new or changed devices on smithi083 2022-09-29T14:17:34.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:17:33 smithi083 ceph-mon[77242]: Detected new or changed devices on smithi083 2022-09-29T14:17:34.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:17:33 smithi153 ceph-mon[70044]: Detected new or changed devices on smithi083 2022-09-29T14:17:35.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:17:34 smithi083 ceph-mon[79930]: pgmap v15877: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:35.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:17:34 smithi083 ceph-mon[77242]: pgmap v15877: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:35.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:17:34 smithi153 ceph-mon[70044]: pgmap v15877: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:35.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:17:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:17:35.502Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:17:36.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:17:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:17:36.236Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:17:37.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:17:36 smithi083 ceph-mon[79930]: pgmap v15878: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:37.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:17:36 smithi083 ceph-mon[77242]: pgmap v15878: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:37.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:17:36 smithi153 ceph-mon[70044]: pgmap v15878: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:39.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:17:38 smithi083 ceph-mon[77242]: pgmap v15879: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:39.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:17:38 smithi083 ceph-mon[79930]: pgmap v15879: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:39.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:17:38 smithi153 ceph-mon[70044]: pgmap v15879: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:41.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:17:40 smithi083 ceph-mon[77242]: pgmap v15880: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:41.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:17:40 smithi083 ceph-mon[79930]: pgmap v15880: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:17:40 smithi153 ceph-mon[70044]: pgmap v15880: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:41.930 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:17:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:17:41] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T14:17:42.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:17:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:17:42.143Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:17:42.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:17:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:17:42.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:17:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:17:42.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:17:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:17:42.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:17:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:17:42.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:17:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:17:42.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:17:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:17:42.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:17:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:17:42.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:17:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:17:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:17:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:17:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:17:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:17:42.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:17:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:17:42.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:17:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:17:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:17:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:17:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:17:43.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:17:42 smithi083 ceph-mon[79930]: pgmap v15881: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:43.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:17:42 smithi083 ceph-mon[77242]: pgmap v15881: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:43.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:17:42 smithi153 ceph-mon[70044]: pgmap v15881: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:43.392 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:17:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:17:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:17:45.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:17:44 smithi083 ceph-mon[79930]: pgmap v15882: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:45.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:17:44 smithi083 ceph-mon[77242]: pgmap v15882: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:45.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:17:44 smithi153 ceph-mon[70044]: pgmap v15882: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:45.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:17:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:17:45.503Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:17:46.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:17:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:17:46.237Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:17:47.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:17:46 smithi083 ceph-mon[79930]: pgmap v15883: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:47.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:17:46 smithi083 ceph-mon[77242]: pgmap v15883: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:47.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:17:46 smithi153 ceph-mon[70044]: pgmap v15883: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:48.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:17:47 smithi083 ceph-mon[79930]: pgmap v15884: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:48.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:17:47 smithi083 ceph-mon[77242]: pgmap v15884: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:48.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:17:47 smithi153 ceph-mon[70044]: pgmap v15884: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:50.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:17:50 smithi083 ceph-mon[77242]: pgmap v15885: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:50.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:17:50 smithi083 ceph-mon[79930]: pgmap v15885: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:50.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:17:50 smithi153 ceph-mon[70044]: pgmap v15885: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:17:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:17:51] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T14:17:52.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:17:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:17:52.144Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:17:52.698 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:17:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:17:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:17:52.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:17:52 smithi083 ceph-mon[77242]: pgmap v15886: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:52.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:17:52 smithi083 ceph-mon[79930]: pgmap v15886: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:53.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:17:52 smithi153 ceph-mon[70044]: pgmap v15886: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:17:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:17:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:17:54.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:17:54 smithi083 ceph-mon[77242]: pgmap v15887: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:54.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:17:54 smithi083 ceph-mon[79930]: pgmap v15887: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:55.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:17:54 smithi153 ceph-mon[70044]: pgmap v15887: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:55.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:17:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:17:55.503Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:17:56.704 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:17:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:17:56.238Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:17:56.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:17:56 smithi083 ceph-mon[77242]: pgmap v15888: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:56.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:17:56 smithi083 ceph-mon[79930]: pgmap v15888: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:57.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:17:56 smithi153 ceph-mon[70044]: pgmap v15888: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:58.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:17:58 smithi083 ceph-mon[77242]: pgmap v15889: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:58.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:17:58 smithi083 ceph-mon[79930]: pgmap v15889: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:59.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:17:58 smithi153 ceph-mon[70044]: pgmap v15889: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:17:59.329 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:17:59.614 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:17:59.615 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 30s ago 8h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:17:59.615 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 8m ago 8h 63.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:17:59.616 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 30s ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:17:59.616 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 8m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:17:59.616 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 8m ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:17:59.616 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 30s ago 8h 951M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:17:59.616 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 30s ago 8h 1948M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:17:59.617 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 8m ago 8h 830M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:17:59.617 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 30s ago 8h 1163M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:17:59.617 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 30s ago 8h 23.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:17:59.617 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 8m ago 8h 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:17:59.618 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 30s ago 8h 1412M 2247M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:17:59.618 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 30s ago 8h 1398M 2247M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:17:59.618 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 30s ago 8h 1088M 2247M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:17:59.619 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 30s ago 8h 1371M 2247M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:17:59.619 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 8m ago 8h 1180M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:17:59.619 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 8m ago 8h 1480M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:17:59.619 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 8m ago 8h 1323M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:17:59.619 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 8m ago 8h 1572M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:17:59.620 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 8m ago 8h 189M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:17:59.969 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:17:59.970 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:17:59.970 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:17:59.970 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:17:59.970 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:17:59.971 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:17:59.971 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:17:59.971 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:17:59.971 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:17:59.972 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:17:59.972 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:17:59.972 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:17:59.972 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:17:59.972 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:17:59.973 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:17:59.973 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:17:59.973 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:17:59.973 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:18:00.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:18:00 smithi083 ceph-mon[79930]: from='client.52608 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:18:00.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:18:00 smithi083 ceph-mon[79930]: pgmap v15890: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:18:00.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:18:00 smithi083 ceph-mon[79930]: from='client.52614 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:18:00.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:18:00 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2633833702' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:18:00.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:18:00 smithi083 ceph-mon[77242]: from='client.52608 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:18:00.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:18:00 smithi083 ceph-mon[77242]: pgmap v15890: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:18:00.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:18:00 smithi083 ceph-mon[77242]: from='client.52614 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:18:00.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:18:00 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2633833702' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:18:01.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:18:00 smithi153 ceph-mon[70044]: from='client.52608 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:18:01.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:18:00 smithi153 ceph-mon[70044]: pgmap v15890: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:18:01.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:18:00 smithi153 ceph-mon[70044]: from='client.52614 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:18:01.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:18:00 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2633833702' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:18:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:18:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:18:01] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T14:18:02.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:18:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:18:02.144Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:18:02.713 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:18:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:18:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:18:03.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:18:02 smithi153 ceph-mon[70044]: pgmap v15891: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:18:03.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:18:02 smithi083 ceph-mon[79930]: pgmap v15891: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:18:03.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:18:02 smithi083 ceph-mon[77242]: pgmap v15891: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:18:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:18:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:18:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:18:05.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:18:04 smithi153 ceph-mon[70044]: pgmap v15892: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:18:05.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:18:04 smithi083 ceph-mon[79930]: pgmap v15892: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:18:05.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:18:04 smithi083 ceph-mon[77242]: pgmap v15892: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:18:05.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:18:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:18:05.504Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:18:06.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:18:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:18:06.240Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:18:07.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:18:06 smithi153 ceph-mon[70044]: pgmap v15893: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:18:07.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:18:06 smithi083 ceph-mon[79930]: pgmap v15893: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:18:07.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:18:06 smithi083 ceph-mon[77242]: pgmap v15893: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:18:09.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:18:08 smithi153 ceph-mon[70044]: pgmap v15894: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:18:09.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:18:08 smithi083 ceph-mon[79930]: pgmap v15894: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:18:09.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:18:08 smithi083 ceph-mon[77242]: pgmap v15894: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:18:11.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:18:10 smithi153 ceph-mon[70044]: pgmap v15895: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:18:11.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:18:10 smithi083 ceph-mon[79930]: pgmap v15895: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:18:11.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:18:10 smithi083 ceph-mon[77242]: pgmap v15895: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:18:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:18:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:18:11] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T14:18:12.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:18:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:18:12.145Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:18:12.730 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:18:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:18:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:18:13.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:18:12 smithi153 ceph-mon[70044]: pgmap v15896: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:18:13.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:18:12 smithi083 ceph-mon[79930]: pgmap v15896: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:18:13.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:18:12 smithi083 ceph-mon[77242]: pgmap v15896: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:18:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:18:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:18:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:18:15.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:18:14 smithi153 ceph-mon[70044]: pgmap v15897: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:18:15.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:18:14 smithi083 ceph-mon[79930]: pgmap v15897: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:18:15.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:18:14 smithi083 ceph-mon[77242]: pgmap v15897: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:18:15.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:18:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:18:15.504Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:18:16.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:18:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:18:16.241Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:18:17.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:18:16 smithi153 ceph-mon[70044]: pgmap v15898: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:18:17.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:18:16 smithi083 ceph-mon[77242]: pgmap v15898: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:18:17.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:18:16 smithi083 ceph-mon[79930]: pgmap v15898: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:18:19.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:18:18 smithi153 ceph-mon[70044]: pgmap v15899: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:18:19.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:18:18 smithi083 ceph-mon[79930]: pgmap v15899: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:18:19.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:18:18 smithi083 ceph-mon[77242]: pgmap v15899: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:18:21.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:18:20 smithi153 ceph-mon[70044]: pgmap v15900: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:18:21.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:18:20 smithi083 ceph-mon[79930]: pgmap v15900: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:18:21.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:18:20 smithi083 ceph-mon[77242]: pgmap v15900: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:18:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:18:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:18:21] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T14:18:22.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:18:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:18:22.145Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:18:22.744 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:18:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:18:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:18:23.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:18:22 smithi153 ceph-mon[70044]: pgmap v15901: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:18:23.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:18:22 smithi083 ceph-mon[79930]: pgmap v15901: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:18:23.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:18:22 smithi083 ceph-mon[77242]: pgmap v15901: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:18:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:18:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:18:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:18:25.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:18:24 smithi153 ceph-mon[70044]: pgmap v15902: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:18:25.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:18:24 smithi083 ceph-mon[79930]: pgmap v15902: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:18:25.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:18:24 smithi083 ceph-mon[77242]: pgmap v15902: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:18:25.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:18:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:18:25.505Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:18:26.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:18:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:18:26.241Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:18:27.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:18:26 smithi153 ceph-mon[70044]: pgmap v15903: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:18:27.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:18:26 smithi083 ceph-mon[77242]: pgmap v15903: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:18:27.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:18:26 smithi083 ceph-mon[79930]: pgmap v15903: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:18:29.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:18:28 smithi153 ceph-mon[70044]: pgmap v15904: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:18:29.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:18:28 smithi083 ceph-mon[79930]: pgmap v15904: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:18:29.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:18:28 smithi083 ceph-mon[77242]: pgmap v15904: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:18:30.297 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:18:30.584 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:18:30.584 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 61s ago 8h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:18:30.584 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 9m ago 8h 63.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:18:30.585 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 61s ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:18:30.585 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 9m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:18:30.585 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 9m ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:18:30.585 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 61s ago 8h 951M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:18:30.586 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 61s ago 8h 1948M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:18:30.586 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 9m ago 8h 830M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:18:30.586 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 61s ago 8h 1163M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:18:30.592 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 61s ago 8h 23.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:18:30.592 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 9m ago 8h 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:18:30.593 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 61s ago 8h 1412M 2247M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:18:30.593 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 61s ago 8h 1398M 2247M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:18:30.593 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 61s ago 8h 1088M 2247M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:18:30.593 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 61s ago 8h 1371M 2247M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:18:30.593 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 9m ago 8h 1180M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:18:30.594 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 9m ago 8h 1480M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:18:30.594 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 9m ago 8h 1323M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:18:30.594 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 9m ago 8h 1572M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:18:30.594 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 9m ago 8h 189M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:18:30.941 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:18:30.942 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:18:30.942 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:18:30.942 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:18:30.943 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:18:30.943 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:18:30.943 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:18:30.943 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:18:30.943 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:18:30.944 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:18:30.944 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:18:30.944 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:18:30.944 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:18:30.944 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:18:30.945 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:18:30.945 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:18:30.945 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:18:30.945 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:18:31.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:18:30 smithi153 ceph-mon[70044]: pgmap v15905: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:18:31.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:18:30 smithi083 ceph-mon[79930]: pgmap v15905: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:18:31.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:18:30 smithi083 ceph-mon[77242]: pgmap v15905: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:18:31.762 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:18:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:18:31] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T14:18:32.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:18:31 smithi153 ceph-mon[70044]: from='client.52626 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:18:32.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:18:31 smithi153 ceph-mon[70044]: from='client.62563 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:18:32.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:18:31 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1333715050' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:18:32.149 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:18:31 smithi083 ceph-mon[79930]: from='client.52626 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:18:32.149 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:18:31 smithi083 ceph-mon[79930]: from='client.62563 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:18:32.149 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:18:31 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1333715050' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:18:32.149 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:18:31 smithi083 ceph-mon[77242]: from='client.52626 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:18:32.150 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:18:31 smithi083 ceph-mon[77242]: from='client.62563 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:18:32.150 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:18:31 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1333715050' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:18:32.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:18:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:18:32.146Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:18:32.764 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:18:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:18:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:18:33.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:18:32 smithi153 ceph-mon[70044]: pgmap v15906: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:18:33.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:18:32 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:18:33.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:18:32 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:18:33.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:18:32 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:18:33.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:18:32 smithi083 ceph-mon[77242]: pgmap v15906: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:18:33.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:18:32 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:18:33.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:18:32 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:18:33.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:18:32 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:18:33.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:18:32 smithi083 ceph-mon[79930]: pgmap v15906: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:18:33.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:18:32 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:18:33.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:18:32 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:18:33.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:18:32 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:18:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:18:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:18:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:18:34.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:18:34 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:18:34.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:18:34 smithi083 ceph-mon[79930]: pgmap v15907: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:18:34.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:18:34 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:18:34.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:18:34 smithi083 ceph-mon[77242]: pgmap v15907: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:18:34.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:18:34 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:18:34.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:18:34 smithi153 ceph-mon[70044]: pgmap v15907: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:18:35.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:18:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:18:35.506Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:18:36.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:18:36 smithi083 ceph-mon[79930]: pgmap v15908: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:18:36.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:18:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:18:36.241Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:18:36.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:18:36 smithi083 ceph-mon[77242]: pgmap v15908: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:18:36.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:18:36 smithi153 ceph-mon[70044]: pgmap v15908: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:18:38.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:18:38 smithi083 ceph-mon[79930]: pgmap v15909: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:18:38.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:18:38 smithi083 ceph-mon[77242]: pgmap v15909: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:18:39.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:18:38 smithi153 ceph-mon[70044]: pgmap v15909: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:18:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:18:40 smithi153 ceph-mon[70044]: pgmap v15910: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:18:41.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:18:40 smithi083 ceph-mon[79930]: pgmap v15910: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:18:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:18:40 smithi083 ceph-mon[77242]: pgmap v15910: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:18:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:18:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:18:41] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T14:18:41.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:18:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:18:41.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:18:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:18:41.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:18:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:18:41.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:18:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:18:41.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:18:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:18:41.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:18:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:18:41.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:18:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:18:41.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:18:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:18:42.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:18:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:18:42.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:18:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:18:42.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:18:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:18:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:18:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:18:42.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:18:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:18:42.147Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:18:42.725 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:18:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:18:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:18:43.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:18:42 smithi153 ceph-mon[70044]: pgmap v15911: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:18:43.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:18:42 smithi083 ceph-mon[79930]: pgmap v15911: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:18:43.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:18:42 smithi083 ceph-mon[77242]: pgmap v15911: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:18:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:18:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:18:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:18:45.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:18:44 smithi153 ceph-mon[70044]: pgmap v15912: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:18:45.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:18:44 smithi083 ceph-mon[79930]: pgmap v15912: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:18:45.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:18:44 smithi083 ceph-mon[77242]: pgmap v15912: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:18:45.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:18:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:18:45.507Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:18:46.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:18:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:18:46.242Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:18:47.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:18:46 smithi153 ceph-mon[70044]: pgmap v15913: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:18:47.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:18:46 smithi083 ceph-mon[79930]: pgmap v15913: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:18:47.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:18:46 smithi083 ceph-mon[77242]: pgmap v15913: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:18:49.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:18:48 smithi153 ceph-mon[70044]: pgmap v15914: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:18:49.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:18:48 smithi083 ceph-mon[79930]: pgmap v15914: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:18:49.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:18:48 smithi083 ceph-mon[77242]: pgmap v15914: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:18:51.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:18:50 smithi153 ceph-mon[70044]: pgmap v15915: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:18:51.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:18:50 smithi083 ceph-mon[79930]: pgmap v15915: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:18:51.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:18:50 smithi083 ceph-mon[77242]: pgmap v15915: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:18:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:18:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:18:51] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T14:18:52.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:18:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:18:52.147Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:18:52.740 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:18:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:18:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:18:53.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:18:52 smithi153 ceph-mon[70044]: pgmap v15916: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:18:53.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:18:52 smithi083 ceph-mon[77242]: pgmap v15916: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:18:53.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:18:52 smithi083 ceph-mon[79930]: pgmap v15916: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:18:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:18:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:18:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:18:55.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:18:54 smithi153 ceph-mon[70044]: pgmap v15917: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:18:55.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:18:54 smithi083 ceph-mon[79930]: pgmap v15917: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:18:55.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:18:54 smithi083 ceph-mon[77242]: pgmap v15917: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:18:55.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:18:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:18:55.507Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:18:56.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:18:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:18:56.242Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:18:57.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:18:56 smithi153 ceph-mon[70044]: pgmap v15918: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:18:57.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:18:56 smithi083 ceph-mon[79930]: pgmap v15918: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:18:57.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:18:56 smithi083 ceph-mon[77242]: pgmap v15918: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:18:59.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:18:58 smithi153 ceph-mon[70044]: pgmap v15919: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:18:59.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:18:58 smithi083 ceph-mon[77242]: pgmap v15919: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:18:59.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:18:58 smithi083 ceph-mon[79930]: pgmap v15919: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:01.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:19:00 smithi153 ceph-mon[70044]: pgmap v15920: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:01.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:19:00 smithi083 ceph-mon[77242]: pgmap v15920: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:01.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:19:00 smithi083 ceph-mon[79930]: pgmap v15920: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:01.265 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:19:01.556 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:19:01.557 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 92s ago 8h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:19:01.557 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 9m ago 8h 63.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:19:01.557 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 92s ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:19:01.557 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 9m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:19:01.558 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 9m ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:19:01.558 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 92s ago 8h 951M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:19:01.558 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 92s ago 8h 1948M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:19:01.558 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 9m ago 8h 830M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:19:01.558 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 92s ago 8h 1163M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:19:01.559 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 92s ago 8h 23.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:19:01.559 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 9m ago 8h 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:19:01.559 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 92s ago 8h 1412M 2247M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:19:01.559 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 92s ago 8h 1398M 2247M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:19:01.560 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 92s ago 8h 1088M 2247M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:19:01.560 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 92s ago 8h 1371M 2247M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:19:01.560 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 9m ago 8h 1180M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:19:01.560 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 9m ago 8h 1480M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:19:01.561 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 9m ago 8h 1323M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:19:01.561 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 9m ago 8h 1572M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:19:01.561 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 9m ago 8h 189M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:19:01.915 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:19:01.915 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:19:01.916 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:19:01.916 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:19:01.916 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:19:01.916 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:19:01.916 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:19:01.917 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:19:01.917 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:19:01.917 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:19:01.917 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:19:01.918 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:19:01.918 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:19:01.918 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:19:01.918 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:19:01.919 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:19:01.919 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:19:01.919 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:19:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:19:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:19:01] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T14:19:02.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:19:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:19:02.148Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:19:02.755 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:19:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:19:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:19:03.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:19:02 smithi153 ceph-mon[70044]: from='client.62575 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:19:03.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:19:02 smithi153 ceph-mon[70044]: pgmap v15921: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:03.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:19:02 smithi153 ceph-mon[70044]: from='client.62581 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:19:03.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:19:02 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/963871809' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:19:03.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:19:02 smithi083 ceph-mon[79930]: from='client.62575 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:19:03.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:19:02 smithi083 ceph-mon[79930]: pgmap v15921: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:03.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:19:02 smithi083 ceph-mon[79930]: from='client.62581 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:19:03.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:19:02 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/963871809' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:19:03.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:19:02 smithi083 ceph-mon[77242]: from='client.62575 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:19:03.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:19:02 smithi083 ceph-mon[77242]: pgmap v15921: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:03.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:19:02 smithi083 ceph-mon[77242]: from='client.62581 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:19:03.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:19:02 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/963871809' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:19:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:19:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:19:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:19:05.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:19:04 smithi153 ceph-mon[70044]: pgmap v15922: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:05.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:19:04 smithi083 ceph-mon[79930]: pgmap v15922: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:05.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:19:04 smithi083 ceph-mon[77242]: pgmap v15922: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:05.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:19:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:19:05.508Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:19:06.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:19:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:19:06.243Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:19:07.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:19:06 smithi153 ceph-mon[70044]: pgmap v15923: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:07.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:19:06 smithi083 ceph-mon[79930]: pgmap v15923: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:07.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:19:06 smithi083 ceph-mon[77242]: pgmap v15923: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:09.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:19:08 smithi153 ceph-mon[70044]: pgmap v15924: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:09.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:19:08 smithi083 ceph-mon[79930]: pgmap v15924: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:09.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:19:08 smithi083 ceph-mon[77242]: pgmap v15924: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:11.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:19:10 smithi153 ceph-mon[70044]: pgmap v15925: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:11.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:19:10 smithi083 ceph-mon[77242]: pgmap v15925: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:11.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:19:10 smithi083 ceph-mon[79930]: pgmap v15925: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:19:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:19:11] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T14:19:12.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:19:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:19:12.148Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:19:12.770 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:19:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:19:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:19:13.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:19:12 smithi153 ceph-mon[70044]: pgmap v15926: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:13.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:19:12 smithi083 ceph-mon[79930]: pgmap v15926: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:13.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:19:12 smithi083 ceph-mon[77242]: pgmap v15926: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:19:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:19:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:19:15.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:19:14 smithi153 ceph-mon[70044]: pgmap v15927: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:15.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:19:14 smithi083 ceph-mon[77242]: pgmap v15927: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:15.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:19:14 smithi083 ceph-mon[79930]: pgmap v15927: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:15.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:19:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:19:15.508Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:19:16.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:19:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:19:16.244Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:19:17.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:19:16 smithi153 ceph-mon[70044]: pgmap v15928: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:17.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:19:16 smithi083 ceph-mon[79930]: pgmap v15928: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:17.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:19:16 smithi083 ceph-mon[77242]: pgmap v15928: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:19.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:19:18 smithi153 ceph-mon[70044]: pgmap v15929: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:19.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:19:18 smithi083 ceph-mon[79930]: pgmap v15929: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:19.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:19:18 smithi083 ceph-mon[77242]: pgmap v15929: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:21.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:19:20 smithi153 ceph-mon[70044]: pgmap v15930: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:21.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:19:20 smithi083 ceph-mon[77242]: pgmap v15930: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:21.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:19:20 smithi083 ceph-mon[79930]: pgmap v15930: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:19:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:19:21] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T14:19:22.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:19:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:19:22.149Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:19:22.785 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:19:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:19:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:19:23.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:19:22 smithi153 ceph-mon[70044]: pgmap v15931: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:23.059 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:19:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:19:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:19:23.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:19:22 smithi083 ceph-mon[79930]: pgmap v15931: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:23.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:19:22 smithi083 ceph-mon[77242]: pgmap v15931: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:25.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:19:24 smithi153 ceph-mon[70044]: pgmap v15932: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:25.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:19:24 smithi083 ceph-mon[79930]: pgmap v15932: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:25.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:19:24 smithi083 ceph-mon[77242]: pgmap v15932: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:25.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:19:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:19:25.509Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:19:26.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:19:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:19:26.244Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:19:27.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:19:26 smithi153 ceph-mon[70044]: pgmap v15933: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:27.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:19:26 smithi083 ceph-mon[77242]: pgmap v15933: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:27.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:19:26 smithi083 ceph-mon[79930]: pgmap v15933: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:29.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:19:28 smithi153 ceph-mon[70044]: pgmap v15934: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:29.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:19:28 smithi083 ceph-mon[77242]: pgmap v15934: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:29.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:19:28 smithi083 ceph-mon[79930]: pgmap v15934: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:31.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:19:30 smithi153 ceph-mon[70044]: pgmap v15935: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:31.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:19:30 smithi083 ceph-mon[77242]: pgmap v15935: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:31.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:19:30 smithi083 ceph-mon[79930]: pgmap v15935: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:19:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:19:31] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T14:19:32.243 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:19:32.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:19:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:19:32.149Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:19:32.529 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:19:32.530 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 2m ago 8h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:19:32.530 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 10m ago 8h 63.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:19:32.530 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 2m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:19:32.530 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 10m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:19:32.531 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 10m ago 8h 448M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:19:32.531 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 2m ago 8h 951M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:19:32.531 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 2m ago 8h 1948M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:19:32.531 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 10m ago 8h 830M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:19:32.532 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 2m ago 8h 1163M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:19:32.532 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 2m ago 8h 23.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:19:32.532 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 10m ago 8h 26.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:19:32.532 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 2m ago 8h 1412M 2247M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:19:32.532 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 2m ago 8h 1398M 2247M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:19:32.533 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 2m ago 8h 1088M 2247M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:19:32.533 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 2m ago 8h 1371M 2247M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:19:32.533 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 10m ago 8h 1180M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:19:32.533 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 10m ago 8h 1480M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:19:32.534 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 10m ago 8h 1323M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:19:32.534 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 10m ago 8h 1572M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:19:32.534 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 10m ago 8h 189M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:19:32.801 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:19:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:19:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:19:32.887 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:19:32.888 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:19:32.888 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:19:32.888 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:19:32.888 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:19:32.888 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:19:32.889 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:19:32.889 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:19:32.889 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:19:32.889 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:19:32.890 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:19:32.890 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:19:32.890 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:19:32.890 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:19:32.891 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:19:32.891 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:19:32.891 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:19:32.891 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:19:33.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:19:32 smithi153 ceph-mon[70044]: pgmap v15936: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:33.059 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:19:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:19:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:19:33.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:19:32 smithi083 ceph-mon[77242]: pgmap v15936: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:33.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:19:32 smithi083 ceph-mon[79930]: pgmap v15936: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:34.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:19:33 smithi153 ceph-mon[70044]: from='client.52662 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:19:34.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:19:33 smithi153 ceph-mon[70044]: from='client.52668 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:19:34.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:19:33 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/325468804' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:19:34.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:19:33 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:19:34.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:19:33 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:19:34.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:19:33 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:19:34.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:19:33 smithi083 ceph-mon[79930]: from='client.52662 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:19:34.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:19:33 smithi083 ceph-mon[79930]: from='client.52668 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:19:34.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:19:33 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/325468804' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:19:34.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:19:33 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:19:34.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:19:33 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:19:34.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:19:33 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:19:34.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:19:33 smithi083 ceph-mon[77242]: from='client.52662 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:19:34.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:19:33 smithi083 ceph-mon[77242]: from='client.52668 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:19:34.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:19:33 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/325468804' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:19:34.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:19:33 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:19:34.218 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:19:33 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:19:34.218 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:19:33 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:19:35.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:19:34 smithi083 ceph-mon[79930]: pgmap v15937: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:35.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:19:34 smithi083 ceph-mon[77242]: pgmap v15937: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:35.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:19:34 smithi153 ceph-mon[70044]: pgmap v15937: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:35.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:19:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:19:35.509Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:19:36.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:19:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:19:36.245Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:19:37.143 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:19:36 smithi153 ceph-mon[70044]: pgmap v15938: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:37.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:19:36 smithi083 ceph-mon[77242]: pgmap v15938: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:37.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:19:36 smithi083 ceph-mon[79930]: pgmap v15938: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:38.947 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:19:38 smithi153 ceph-mon[70044]: pgmap v15939: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:38.947 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:19:38 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:19:38.947 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:19:38 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:19:39.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:19:38 smithi083 ceph-mon[79930]: pgmap v15939: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:39.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:19:38 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:19:39.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:19:38 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:19:39.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:19:38 smithi083 ceph-mon[77242]: pgmap v15939: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:39.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:19:38 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:19:39.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:19:38 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:19:39.873 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:19:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:19:39.873 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:19:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:19:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:19:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:19:40.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:19:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:19:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:19:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:19:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:19:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:19:41.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:19:40 smithi153 ceph-mon[70044]: pgmap v15940: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:41.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:19:40 smithi083 ceph-mon[77242]: pgmap v15940: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:41.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:19:40 smithi083 ceph-mon[79930]: pgmap v15940: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:19:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:19:41] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T14:19:42.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:19:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:19:42.151Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:19:42.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:19:42 smithi083 ceph-mon[79930]: Detected new or changed devices on smithi153 2022-09-29T14:19:42.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:19:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:19:42.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:19:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:19:42.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:19:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:19:42.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:19:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:19:42.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:19:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:19:42.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:19:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:19:42.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:19:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:19:42.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:19:42 smithi083 ceph-mon[79930]: pgmap v15941: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:42.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:19:42 smithi083 ceph-mon[77242]: Detected new or changed devices on smithi153 2022-09-29T14:19:42.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:19:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:19:42.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:19:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:19:42.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:19:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:19:42.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:19:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:19:42.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:19:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:19:42.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:19:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:19:42.467 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:19:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:19:42.467 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:19:42 smithi083 ceph-mon[77242]: pgmap v15941: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:42.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:19:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:19:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:19:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:19:42 smithi153 ceph-mon[70044]: Detected new or changed devices on smithi153 2022-09-29T14:19:42.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:19:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:19:42.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:19:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:19:42.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:19:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:19:42.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:19:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:19:42.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:19:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:19:42.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:19:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:19:42.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:19:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:19:42.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:19:42 smithi153 ceph-mon[70044]: pgmap v15941: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:19:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:19:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:19:44.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:19:44 smithi153 ceph-mon[70044]: pgmap v15942: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:44.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:19:44 smithi083 ceph-mon[79930]: pgmap v15942: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:44.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:19:44 smithi083 ceph-mon[77242]: pgmap v15942: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:45.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:19:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:19:45.510Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:19:46.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:19:46 smithi083 ceph-mon[79930]: pgmap v15943: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:46.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:19:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:19:46.245Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:19:46.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:19:46 smithi083 ceph-mon[77242]: pgmap v15943: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:46.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:19:46 smithi153 ceph-mon[70044]: pgmap v15943: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:48.986 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:19:48 smithi153 ceph-mon[70044]: pgmap v15944: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:49.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:19:48 smithi083 ceph-mon[79930]: pgmap v15944: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:49.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:19:48 smithi083 ceph-mon[77242]: pgmap v15944: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:51.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:19:50 smithi153 ceph-mon[70044]: pgmap v15945: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:51.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:19:50 smithi083 ceph-mon[79930]: pgmap v15945: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:51.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:19:50 smithi083 ceph-mon[77242]: pgmap v15945: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:19:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:19:51] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T14:19:52.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:19:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:19:52.152Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:19:52.730 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:19:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:19:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:19:53.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:19:52 smithi153 ceph-mon[70044]: pgmap v15946: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:53.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:19:52 smithi083 ceph-mon[77242]: pgmap v15946: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:53.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:19:52 smithi083 ceph-mon[79930]: pgmap v15946: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:19:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:19:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:19:55.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:19:54 smithi153 ceph-mon[70044]: pgmap v15947: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:55.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:19:54 smithi083 ceph-mon[79930]: pgmap v15947: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:55.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:19:54 smithi083 ceph-mon[77242]: pgmap v15947: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:55.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:19:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:19:55.511Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:19:56.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:19:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:19:56.247Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:19:57.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:19:56 smithi153 ceph-mon[70044]: pgmap v15948: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:57.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:19:56 smithi083 ceph-mon[79930]: pgmap v15948: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:57.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:19:56 smithi083 ceph-mon[77242]: pgmap v15948: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:59.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:19:58 smithi153 ceph-mon[70044]: pgmap v15949: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:59.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:19:58 smithi083 ceph-mon[79930]: pgmap v15949: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:19:59.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:19:58 smithi083 ceph-mon[77242]: pgmap v15949: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:00.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:20:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[77219]: 2022-09-29T14:19:59.999+0000 7f160f665700 -1 log_channel(cluster) log [ERR] : overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T14:20:01.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:20:00 smithi153 ceph-mon[70044]: pgmap v15950: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:01.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:20:00 smithi153 ceph-mon[70044]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T14:20:01.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:20:00 smithi083 ceph-mon[79930]: pgmap v15950: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:01.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:20:00 smithi083 ceph-mon[79930]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T14:20:01.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:20:00 smithi083 ceph-mon[77242]: pgmap v15950: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:01.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:20:00 smithi083 ceph-mon[77242]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T14:20:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:20:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:20:01] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T14:20:02.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:20:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:20:02.153Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:20:02.747 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:20:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:20:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:20:03.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:20:02 smithi153 ceph-mon[70044]: pgmap v15951: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:03.207 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:20:03.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:20:02 smithi083 ceph-mon[79930]: pgmap v15951: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:03.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:20:02 smithi083 ceph-mon[77242]: pgmap v15951: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:20:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:20:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:20:03.504 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:20:03.504 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 2m ago 8h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:20:03.505 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 25s ago 8h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:20:03.505 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 2m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:20:03.505 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 25s ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:20:03.505 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 25s ago 8h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:20:03.506 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 2m ago 8h 951M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:20:03.506 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 2m ago 8h 1948M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:20:03.506 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 25s ago 8h 847M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:20:03.506 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 2m ago 8h 1163M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:20:03.507 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 2m ago 8h 23.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:20:03.507 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 25s ago 8h 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:20:03.507 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 2m ago 8h 1412M 2247M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:20:03.507 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 2m ago 8h 1398M 2247M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:20:03.508 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 2m ago 8h 1088M 2247M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:20:03.508 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 2m ago 8h 1371M 2247M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:20:03.508 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 25s ago 8h 1203M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:20:03.508 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 25s ago 8h 1508M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:20:03.509 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 25s ago 8h 1348M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:20:03.509 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 25s ago 8h 1602M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:20:03.509 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 25s ago 8h 192M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:20:03.867 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:20:03.867 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:20:03.867 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:20:03.868 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:20:03.868 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:20:03.868 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:20:03.868 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:20:03.869 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:20:03.869 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:20:03.869 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:20:03.869 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:20:03.869 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:20:03.870 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:20:03.870 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:20:03.870 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:20:03.870 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:20:03.871 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:20:03.871 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:20:05.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:20:04 smithi153 ceph-mon[70044]: from='client.52680 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:20:05.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:20:04 smithi153 ceph-mon[70044]: pgmap v15952: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:05.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:20:04 smithi153 ceph-mon[70044]: from='client.52686 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:20:05.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:20:04 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1882095371' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:20:05.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:20:04 smithi083 ceph-mon[79930]: from='client.52680 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:20:05.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:20:04 smithi083 ceph-mon[79930]: pgmap v15952: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:05.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:20:04 smithi083 ceph-mon[79930]: from='client.52686 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:20:05.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:20:04 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1882095371' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:20:05.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:20:04 smithi083 ceph-mon[77242]: from='client.52680 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:20:05.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:20:04 smithi083 ceph-mon[77242]: pgmap v15952: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:05.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:20:04 smithi083 ceph-mon[77242]: from='client.52686 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:20:05.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:20:04 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1882095371' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:20:05.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:20:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:20:05.512Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:20:06.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:20:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:20:06.247Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:20:07.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:20:06 smithi153 ceph-mon[70044]: pgmap v15953: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:07.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:20:06 smithi083 ceph-mon[77242]: pgmap v15953: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:07.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:20:06 smithi083 ceph-mon[79930]: pgmap v15953: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:09.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:20:08 smithi153 ceph-mon[70044]: pgmap v15954: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:09.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:20:08 smithi083 ceph-mon[79930]: pgmap v15954: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:09.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:20:08 smithi083 ceph-mon[77242]: pgmap v15954: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:11.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:20:10 smithi153 ceph-mon[70044]: pgmap v15955: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:11.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:20:10 smithi083 ceph-mon[79930]: pgmap v15955: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:11.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:20:10 smithi083 ceph-mon[77242]: pgmap v15955: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:20:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:20:11] "GET /metrics HTTP/1.1" 200 225246 "" "Prometheus/2.33.4" 2022-09-29T14:20:12.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:20:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:20:12.154Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:20:12.769 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:20:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:20:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:20:13.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:20:12 smithi153 ceph-mon[70044]: pgmap v15956: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:13.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:20:12 smithi083 ceph-mon[79930]: pgmap v15956: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:13.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:20:12 smithi083 ceph-mon[77242]: pgmap v15956: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:20:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:20:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:20:15.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:20:14 smithi153 ceph-mon[70044]: pgmap v15957: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:15.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:20:14 smithi083 ceph-mon[79930]: pgmap v15957: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:15.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:20:14 smithi083 ceph-mon[77242]: pgmap v15957: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:15.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:20:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:20:15.512Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:20:16.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:20:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:20:16.248Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:20:17.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:20:16 smithi153 ceph-mon[70044]: pgmap v15958: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:17.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:20:16 smithi083 ceph-mon[77242]: pgmap v15958: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:17.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:20:16 smithi083 ceph-mon[79930]: pgmap v15958: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:19.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:20:18 smithi153 ceph-mon[70044]: pgmap v15959: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:19.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:20:18 smithi083 ceph-mon[79930]: pgmap v15959: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:19.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:20:18 smithi083 ceph-mon[77242]: pgmap v15959: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:21.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:20:20 smithi153 ceph-mon[70044]: pgmap v15960: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:21.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:20:20 smithi083 ceph-mon[79930]: pgmap v15960: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:21.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:20:20 smithi083 ceph-mon[77242]: pgmap v15960: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:20:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:20:21] "GET /metrics HTTP/1.1" 200 225246 "" "Prometheus/2.33.4" 2022-09-29T14:20:22.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:20:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:20:22.154Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:20:22.784 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:20:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:20:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:20:23.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:20:22 smithi153 ceph-mon[70044]: pgmap v15961: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:23.059 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:20:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:20:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:20:23.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:20:22 smithi083 ceph-mon[79930]: pgmap v15961: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:23.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:20:22 smithi083 ceph-mon[77242]: pgmap v15961: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:25.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:20:24 smithi153 ceph-mon[70044]: pgmap v15962: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:25.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:20:24 smithi083 ceph-mon[79930]: pgmap v15962: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:25.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:20:24 smithi083 ceph-mon[77242]: pgmap v15962: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:25.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:20:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:20:25.513Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:20:26.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:20:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:20:26.248Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:20:27.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:20:26 smithi153 ceph-mon[70044]: pgmap v15963: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:27.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:20:26 smithi083 ceph-mon[79930]: pgmap v15963: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:27.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:20:26 smithi083 ceph-mon[77242]: pgmap v15963: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:29.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:20:28 smithi153 ceph-mon[70044]: pgmap v15964: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:29.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:20:28 smithi083 ceph-mon[79930]: pgmap v15964: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:29.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:20:28 smithi083 ceph-mon[77242]: pgmap v15964: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:31.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:20:30 smithi153 ceph-mon[70044]: pgmap v15965: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:31.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:20:30 smithi083 ceph-mon[79930]: pgmap v15965: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:31.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:20:30 smithi083 ceph-mon[77242]: pgmap v15965: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:20:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:20:31] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T14:20:32.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:20:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:20:32.154Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:20:32.800 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:20:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:20:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:20:33.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:20:32 smithi153 ceph-mon[70044]: pgmap v15966: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:33.059 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:20:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:20:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:20:33.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:20:32 smithi083 ceph-mon[79930]: pgmap v15966: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:33.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:20:32 smithi083 ceph-mon[77242]: pgmap v15966: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:34.194 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:20:34.481 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:20:34.482 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 3m ago 8h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:20:34.482 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 56s ago 8h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:20:34.482 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 3m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:20:34.483 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 56s ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:20:34.483 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 56s ago 8h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:20:34.483 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 3m ago 8h 951M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:20:34.483 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 3m ago 8h 1948M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:20:34.484 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 56s ago 8h 847M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:20:34.484 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 3m ago 8h 1163M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:20:34.484 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 3m ago 8h 23.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:20:34.484 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 56s ago 8h 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:20:34.485 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 3m ago 8h 1412M 2247M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:20:34.485 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 3m ago 8h 1398M 2247M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:20:34.485 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 3m ago 8h 1088M 2247M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:20:34.485 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 3m ago 8h 1371M 2247M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:20:34.486 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 56s ago 8h 1203M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:20:34.486 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 56s ago 8h 1508M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:20:34.486 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 56s ago 8h 1348M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:20:34.486 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 56s ago 8h 1602M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:20:34.487 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 56s ago 8h 192M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:20:34.837 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:20:34.838 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:20:34.838 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:20:34.838 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:20:34.838 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:20:34.838 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:20:34.839 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:20:34.839 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:20:34.839 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:20:34.839 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:20:34.840 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:20:34.840 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:20:34.840 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:20:34.840 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:20:34.840 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:20:34.841 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:20:34.841 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:20:34.841 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:20:35.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:20:34 smithi153 ceph-mon[70044]: pgmap v15967: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:35.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:20:34 smithi083 ceph-mon[79930]: pgmap v15967: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:35.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:20:34 smithi083 ceph-mon[77242]: pgmap v15967: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:35.807 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:20:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:20:35.512Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:20:36.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:20:35 smithi153 ceph-mon[70044]: from='client.52698 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:20:36.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:20:35 smithi153 ceph-mon[70044]: from='client.62635 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:20:36.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:20:35 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2494351550' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:20:36.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:20:35 smithi083 ceph-mon[77242]: from='client.52698 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:20:36.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:20:35 smithi083 ceph-mon[77242]: from='client.62635 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:20:36.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:20:35 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2494351550' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:20:36.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:20:35 smithi083 ceph-mon[79930]: from='client.52698 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:20:36.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:20:35 smithi083 ceph-mon[79930]: from='client.62635 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:20:36.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:20:35 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2494351550' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:20:36.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:20:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:20:36.249Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:20:37.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:20:36 smithi083 ceph-mon[79930]: pgmap v15968: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:37.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:20:36 smithi083 ceph-mon[77242]: pgmap v15968: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:37.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:20:36 smithi153 ceph-mon[70044]: pgmap v15968: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:39.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:20:38 smithi083 ceph-mon[79930]: pgmap v15969: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:39.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:20:38 smithi083 ceph-mon[77242]: pgmap v15969: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:39.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:20:38 smithi153 ceph-mon[70044]: pgmap v15969: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:41.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:20:40 smithi083 ceph-mon[79930]: pgmap v15970: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:20:40 smithi083 ceph-mon[77242]: pgmap v15970: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:20:40 smithi153 ceph-mon[70044]: pgmap v15970: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:41.823 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:20:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:20:41] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T14:20:42.157 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:20:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:20:42.157 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:20:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:20:42.157 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:20:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:20:42.158 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:20:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:20:42.158 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:20:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:20:42.158 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:20:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:20:42.158 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:20:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:20:42.159 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:20:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:20:42.159 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:20:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:20:42.159 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:20:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:20:42.160 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:20:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:20:42.160 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:20:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:20:42.160 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:20:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:20:42.160 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:20:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:20:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:20:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:20:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:20:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:20:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:20:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:20:42.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:20:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:20:42.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:20:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:20:42.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:20:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:20:42.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:20:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:20:42.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:20:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:20:42.154Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:20:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:20:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:20:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:20:43.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:20:42 smithi083 ceph-mon[79930]: pgmap v15971: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:43.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:20:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:20:43.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:20:42 smithi083 ceph-mon[77242]: pgmap v15971: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:43.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:20:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:20:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:20:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:20:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:20:43.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:20:42 smithi153 ceph-mon[70044]: pgmap v15971: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:43.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:20:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:20:45.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:20:44 smithi083 ceph-mon[79930]: pgmap v15972: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:45.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:20:44 smithi083 ceph-mon[77242]: pgmap v15972: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:45.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:20:44 smithi153 ceph-mon[70044]: pgmap v15972: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:45.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:20:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:20:45.514Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:20:46.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:20:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:20:46.249Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:20:47.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:20:46 smithi083 ceph-mon[77242]: pgmap v15973: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:47.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:20:46 smithi083 ceph-mon[79930]: pgmap v15973: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:47.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:20:46 smithi153 ceph-mon[70044]: pgmap v15973: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:49.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:20:48 smithi083 ceph-mon[77242]: pgmap v15974: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:49.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:20:48 smithi083 ceph-mon[79930]: pgmap v15974: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:49.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:20:48 smithi153 ceph-mon[70044]: pgmap v15974: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:51.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:20:50 smithi153 ceph-mon[70044]: pgmap v15975: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:51.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:20:50 smithi083 ceph-mon[77242]: pgmap v15975: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:51.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:20:50 smithi083 ceph-mon[79930]: pgmap v15975: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:20:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:20:51] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T14:20:52.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:20:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:20:52.155Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:20:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:20:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:20:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:20:53.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:20:52 smithi153 ceph-mon[70044]: pgmap v15976: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:20:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:20:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:20:53.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:20:52 smithi083 ceph-mon[77242]: pgmap v15976: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:53.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:20:52 smithi083 ceph-mon[79930]: pgmap v15976: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:55.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:20:54 smithi153 ceph-mon[70044]: pgmap v15977: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:55.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:20:54 smithi083 ceph-mon[77242]: pgmap v15977: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:55.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:20:54 smithi083 ceph-mon[79930]: pgmap v15977: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:55.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:20:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:20:55.514Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:20:56.252 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:20:55 smithi083 ceph-mon[79930]: pgmap v15978: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:56.252 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:20:55 smithi083 ceph-mon[77242]: pgmap v15978: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:56.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:20:55 smithi153 ceph-mon[70044]: pgmap v15978: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:56.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:20:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:20:56.250Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:20:59.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:20:58 smithi153 ceph-mon[70044]: pgmap v15979: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:59.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:20:58 smithi083 ceph-mon[77242]: pgmap v15979: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:20:59.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:20:58 smithi083 ceph-mon[79930]: pgmap v15979: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:01.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:21:00 smithi153 ceph-mon[70044]: pgmap v15980: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:01.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:21:00 smithi083 ceph-mon[79930]: pgmap v15980: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:01.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:21:00 smithi083 ceph-mon[77242]: pgmap v15980: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:21:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:21:01] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T14:21:02.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:21:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:21:02.156Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:21:02.748 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:21:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:21:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:21:03.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:21:02 smithi153 ceph-mon[70044]: pgmap v15981: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:03.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:21:02 smithi083 ceph-mon[77242]: pgmap v15981: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:03.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:21:02 smithi083 ceph-mon[79930]: pgmap v15981: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:21:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:21:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:21:04.751 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:21:04 smithi083 ceph-mon[77242]: pgmap v15982: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:05.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:21:04 smithi153 ceph-mon[70044]: pgmap v15982: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:05.161 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:21:05.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:21:04 smithi083 ceph-mon[79930]: pgmap v15982: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:05.451 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:21:05.451 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 3m ago 8h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:21:05.451 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 87s ago 8h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:21:05.451 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 3m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:21:05.452 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 87s ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:21:05.452 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 87s ago 8h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:21:05.452 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 3m ago 9h 951M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:21:05.452 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 3m ago 9h 1948M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:21:05.453 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 87s ago 8h 847M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:21:05.453 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 3m ago 8h 1163M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:21:05.453 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 3m ago 8h 23.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:21:05.453 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 87s ago 8h 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:21:05.453 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 3m ago 8h 1412M 2247M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:21:05.454 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 3m ago 8h 1398M 2247M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:21:05.454 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 3m ago 8h 1088M 2247M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:21:05.454 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 3m ago 8h 1371M 2247M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:21:05.455 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 87s ago 8h 1203M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:21:05.455 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 87s ago 8h 1508M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:21:05.455 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 87s ago 8h 1348M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:21:05.455 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 87s ago 8h 1602M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:21:05.456 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 87s ago 8h 192M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:21:05.811 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:21:05.811 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:21:05.811 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:21:05.812 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:21:05.812 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:21:05.812 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:21:05.812 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:21:05.813 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:21:05.813 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:21:05.813 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:21:05.813 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:21:05.813 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:21:05.814 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:21:05.814 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:21:05.814 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:21:05.815 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:21:05.815 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:21:05.815 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:21:05.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:21:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:21:05.514Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:21:06.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:21:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:21:06.250Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:21:07.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:21:06 smithi153 ceph-mon[70044]: from='client.52716 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:21:07.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:21:06 smithi153 ceph-mon[70044]: from='client.52722 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:21:07.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:21:06 smithi153 ceph-mon[70044]: pgmap v15983: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:07.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:21:06 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3974415121' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:21:07.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:21:06 smithi083 ceph-mon[79930]: from='client.52716 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:21:07.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:21:06 smithi083 ceph-mon[79930]: from='client.52722 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:21:07.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:21:06 smithi083 ceph-mon[79930]: pgmap v15983: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:07.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:21:06 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3974415121' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:21:07.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:21:06 smithi083 ceph-mon[77242]: from='client.52716 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:21:07.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:21:06 smithi083 ceph-mon[77242]: from='client.52722 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:21:07.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:21:06 smithi083 ceph-mon[77242]: pgmap v15983: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:07.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:21:06 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3974415121' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:21:09.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:21:08 smithi153 ceph-mon[70044]: pgmap v15984: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:09.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:21:08 smithi083 ceph-mon[79930]: pgmap v15984: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:09.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:21:08 smithi083 ceph-mon[77242]: pgmap v15984: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:11.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:21:10 smithi153 ceph-mon[70044]: pgmap v15985: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:11.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:21:10 smithi083 ceph-mon[77242]: pgmap v15985: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:11.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:21:10 smithi083 ceph-mon[79930]: pgmap v15985: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:21:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:21:11] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T14:21:12.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:21:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:21:12.156Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:21:12.764 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:21:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:21:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:21:13.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:21:12 smithi153 ceph-mon[70044]: pgmap v15986: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:13.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:21:12 smithi083 ceph-mon[77242]: pgmap v15986: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:13.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:21:12 smithi083 ceph-mon[79930]: pgmap v15986: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:21:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:21:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:21:15.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:21:14 smithi153 ceph-mon[70044]: pgmap v15987: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:15.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:21:14 smithi083 ceph-mon[79930]: pgmap v15987: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:15.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:21:14 smithi083 ceph-mon[77242]: pgmap v15987: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:15.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:21:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:21:15.517Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:21:16.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:21:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:21:16.252Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:21:17.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:21:16 smithi153 ceph-mon[70044]: pgmap v15988: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:17.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:21:16 smithi083 ceph-mon[79930]: pgmap v15988: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:17.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:21:16 smithi083 ceph-mon[77242]: pgmap v15988: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:19.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:21:18 smithi153 ceph-mon[70044]: pgmap v15989: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:19.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:21:18 smithi083 ceph-mon[79930]: pgmap v15989: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:19.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:21:18 smithi083 ceph-mon[77242]: pgmap v15989: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:21.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:21:20 smithi153 ceph-mon[70044]: pgmap v15990: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:21.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:21:20 smithi083 ceph-mon[79930]: pgmap v15990: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:21.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:21:20 smithi083 ceph-mon[77242]: pgmap v15990: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:21:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:21:21] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T14:21:22.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:21:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:21:22.157Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:21:22.778 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:21:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:21:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:21:23.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:21:22 smithi153 ceph-mon[70044]: pgmap v15991: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:23.059 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:21:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:21:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:21:23.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:21:22 smithi083 ceph-mon[79930]: pgmap v15991: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:23.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:21:22 smithi083 ceph-mon[77242]: pgmap v15991: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:25.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:21:24 smithi153 ceph-mon[70044]: pgmap v15992: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:25.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:21:24 smithi083 ceph-mon[79930]: pgmap v15992: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:25.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:21:24 smithi083 ceph-mon[77242]: pgmap v15992: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:25.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:21:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:21:25.517Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:21:26.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:21:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:21:26.252Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:21:27.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:21:26 smithi153 ceph-mon[70044]: pgmap v15993: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:27.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:21:26 smithi083 ceph-mon[79930]: pgmap v15993: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:27.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:21:26 smithi083 ceph-mon[77242]: pgmap v15993: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:29.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:21:28 smithi153 ceph-mon[70044]: pgmap v15994: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:29.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:21:28 smithi083 ceph-mon[79930]: pgmap v15994: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:29.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:21:28 smithi083 ceph-mon[77242]: pgmap v15994: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:31.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:21:30 smithi153 ceph-mon[70044]: pgmap v15995: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:31.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:21:30 smithi083 ceph-mon[79930]: pgmap v15995: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:31.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:21:30 smithi083 ceph-mon[77242]: pgmap v15995: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:21:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:21:31] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T14:21:32.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:21:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:21:32.158Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:21:32.795 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:21:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:21:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:21:33.058 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:21:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:21:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:21:33.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:21:32 smithi153 ceph-mon[70044]: pgmap v15996: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:33.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:21:32 smithi083 ceph-mon[77242]: pgmap v15996: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:33.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:21:32 smithi083 ceph-mon[79930]: pgmap v15996: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:35.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:21:34 smithi153 ceph-mon[70044]: pgmap v15997: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:35.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:21:34 smithi083 ceph-mon[79930]: pgmap v15997: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:35.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:21:34 smithi083 ceph-mon[77242]: pgmap v15997: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:35.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:21:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:21:35.518Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:21:36.137 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:21:36.428 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:21:36.428 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 4m ago 8h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:21:36.429 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 118s ago 8h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:21:36.429 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 4m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:21:36.429 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 118s ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:21:36.429 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 118s ago 8h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:21:36.430 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 4m ago 9h 951M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:21:36.430 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 4m ago 9h 1948M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:21:36.430 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 118s ago 8h 847M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:21:36.430 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 4m ago 8h 1163M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:21:36.431 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 4m ago 8h 23.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:21:36.431 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 118s ago 8h 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:21:36.431 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 4m ago 8h 1412M 2247M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:21:36.431 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 4m ago 8h 1398M 2247M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:21:36.431 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 4m ago 8h 1088M 2247M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:21:36.432 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 4m ago 8h 1371M 2247M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:21:36.432 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 118s ago 8h 1203M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:21:36.432 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 118s ago 8h 1508M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:21:36.432 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 118s ago 8h 1348M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:21:36.433 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 118s ago 8h 1602M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:21:36.433 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 118s ago 8h 192M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:21:36.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:21:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:21:36.254Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:21:36.786 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:21:36.786 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:21:36.787 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:21:36.787 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:21:36.787 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:21:36.787 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:21:36.788 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:21:36.788 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:21:36.788 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:21:36.788 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:21:36.789 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:21:36.789 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:21:36.789 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:21:36.789 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:21:36.790 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:21:36.790 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:21:36.790 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:21:36.790 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:21:37.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:21:36 smithi153 ceph-mon[70044]: pgmap v15998: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:37.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:21:36 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/687811166' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:21:37.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:21:36 smithi083 ceph-mon[79930]: pgmap v15998: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:37.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:21:36 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/687811166' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:21:37.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:21:36 smithi083 ceph-mon[77242]: pgmap v15998: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:37.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:21:36 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/687811166' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:21:38.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:21:37 smithi153 ceph-mon[70044]: from='client.62665 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:21:38.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:21:37 smithi153 ceph-mon[70044]: from='client.52740 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:21:38.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:21:37 smithi083 ceph-mon[79930]: from='client.62665 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:21:38.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:21:37 smithi083 ceph-mon[79930]: from='client.52740 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:21:38.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:21:37 smithi083 ceph-mon[77242]: from='client.62665 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:21:38.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:21:37 smithi083 ceph-mon[77242]: from='client.52740 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:21:39.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:21:38 smithi153 ceph-mon[70044]: pgmap v15999: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:39.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:21:38 smithi083 ceph-mon[79930]: pgmap v15999: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:39.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:21:38 smithi083 ceph-mon[77242]: pgmap v15999: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:41.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:21:40 smithi083 ceph-mon[79930]: pgmap v16000: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:21:40 smithi083 ceph-mon[77242]: pgmap v16000: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:21:40 smithi153 ceph-mon[70044]: pgmap v16000: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:41.817 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:21:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:21:41] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T14:21:42.160 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:21:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:21:42.161 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:21:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:21:42.161 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:21:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:21:42.161 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:21:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:21:42.162 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:21:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:21:42.162 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:21:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:21:42.162 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:21:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:21:42.162 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:21:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:21:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:21:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:21:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:21:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:21:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:21:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:21:42.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:21:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:21:42.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:21:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:21:42.158Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:21:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:21:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:21:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:21:43.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:21:42 smithi083 ceph-mon[79930]: pgmap v16001: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:43.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:21:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:21:43.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:21:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:21:43.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:21:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:21:43.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:21:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:21:43.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:21:42 smithi083 ceph-mon[77242]: pgmap v16001: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:43.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:21:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:21:43.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:21:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:21:43.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:21:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:21:43.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:21:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:21:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:21:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:21:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:21:43.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:21:42 smithi153 ceph-mon[70044]: pgmap v16001: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:43.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:21:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:21:43.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:21:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:21:43.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:21:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:21:43.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:21:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:21:45.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:21:44 smithi083 ceph-mon[79930]: pgmap v16002: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:45.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:21:44 smithi083 ceph-mon[77242]: pgmap v16002: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:45.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:21:44 smithi153 ceph-mon[70044]: pgmap v16002: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:45.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:21:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:21:45.519Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:21:46.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:21:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:21:46.254Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:21:47.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:21:46 smithi083 ceph-mon[79930]: pgmap v16003: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:47.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:21:46 smithi083 ceph-mon[77242]: pgmap v16003: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:47.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:21:46 smithi153 ceph-mon[70044]: pgmap v16003: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:49.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:21:48 smithi083 ceph-mon[79930]: pgmap v16004: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:49.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:21:48 smithi083 ceph-mon[77242]: pgmap v16004: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:49.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:21:48 smithi153 ceph-mon[70044]: pgmap v16004: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:51.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:21:50 smithi083 ceph-mon[79930]: pgmap v16005: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:51.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:21:50 smithi083 ceph-mon[77242]: pgmap v16005: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:51.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:21:50 smithi153 ceph-mon[70044]: pgmap v16005: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:21:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:21:51] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T14:21:52.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:21:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:21:52.158Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:21:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:21:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:21:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:21:53.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:21:52 smithi083 ceph-mon[77242]: pgmap v16006: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:53.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:21:52 smithi083 ceph-mon[79930]: pgmap v16006: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:53.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:21:52 smithi153 ceph-mon[70044]: pgmap v16006: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:53.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:21:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:21:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:21:55.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:21:54 smithi083 ceph-mon[77242]: pgmap v16007: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:55.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:21:54 smithi083 ceph-mon[79930]: pgmap v16007: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:55.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:21:54 smithi153 ceph-mon[70044]: pgmap v16007: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:55.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:21:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:21:55.520Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:21:56.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:21:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:21:56.255Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:21:57.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:21:56 smithi083 ceph-mon[77242]: pgmap v16008: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:57.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:21:56 smithi083 ceph-mon[79930]: pgmap v16008: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:57.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:21:56 smithi153 ceph-mon[70044]: pgmap v16008: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:59.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:21:58 smithi083 ceph-mon[77242]: pgmap v16009: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:59.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:21:58 smithi083 ceph-mon[79930]: pgmap v16009: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:21:59.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:21:58 smithi153 ceph-mon[70044]: pgmap v16009: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:01.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:22:00 smithi083 ceph-mon[77242]: pgmap v16010: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:01.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:22:00 smithi083 ceph-mon[79930]: pgmap v16010: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:01.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:22:00 smithi153 ceph-mon[70044]: pgmap v16010: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:22:01] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T14:22:02.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:22:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:22:02.159Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:22:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:22:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:22:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:22:03.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:22:02 smithi083 ceph-mon[77242]: pgmap v16011: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:03.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:22:02 smithi083 ceph-mon[79930]: pgmap v16011: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:03.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:22:02 smithi153 ceph-mon[70044]: pgmap v16011: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:22:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:22:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:22:05.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:22:04 smithi083 ceph-mon[79930]: pgmap v16012: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:05.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:22:04 smithi083 ceph-mon[77242]: pgmap v16012: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:05.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:22:04 smithi153 ceph-mon[70044]: pgmap v16012: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:05.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:22:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:22:05.520Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:22:06.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:22:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:22:06.255Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:22:07.117 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:22:07.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:22:06 smithi083 ceph-mon[79930]: pgmap v16013: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:07.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:22:06 smithi083 ceph-mon[77242]: pgmap v16013: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:07.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:22:06 smithi153 ceph-mon[70044]: pgmap v16013: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:07.405 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:22:07.405 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 4m ago 8h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:22:07.405 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 2m ago 8h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:22:07.406 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 4m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:22:07.406 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 2m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:22:07.406 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 2m ago 8h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:22:07.406 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 4m ago 9h 951M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:22:07.407 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 4m ago 9h 1948M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:22:07.407 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 2m ago 8h 847M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:22:07.407 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 4m ago 9h 1163M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:22:07.407 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 4m ago 8h 23.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:22:07.407 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 2m ago 8h 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:22:07.408 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 4m ago 8h 1412M 2247M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:22:07.408 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 4m ago 8h 1398M 2247M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:22:07.408 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 4m ago 8h 1088M 2247M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:22:07.408 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 4m ago 8h 1371M 2247M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:22:07.409 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 2m ago 8h 1203M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:22:07.409 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 2m ago 8h 1508M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:22:07.409 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 2m ago 8h 1348M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:22:07.409 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 2m ago 8h 1602M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:22:07.409 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 2m ago 8h 192M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:22:07.758 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:22:07.759 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:22:07.759 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:22:07.759 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:22:07.759 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:22:07.760 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:22:07.760 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:22:07.760 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:22:07.760 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:22:07.760 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:22:07.761 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:22:07.761 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:22:07.761 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:22:07.761 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:22:07.762 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:22:07.762 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:22:07.762 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:22:07.762 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:22:08.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:22:07 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1129643837' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:22:08.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:22:07 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1129643837' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:22:08.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:22:07 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1129643837' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:22:09.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:22:08 smithi083 ceph-mon[79930]: from='client.52752 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:22:09.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:22:08 smithi083 ceph-mon[79930]: from='client.52758 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:22:09.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:22:08 smithi083 ceph-mon[79930]: pgmap v16014: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:09.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:22:08 smithi083 ceph-mon[77242]: from='client.52752 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:22:09.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:22:08 smithi083 ceph-mon[77242]: from='client.52758 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:22:09.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:22:08 smithi083 ceph-mon[77242]: pgmap v16014: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:09.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:22:08 smithi153 ceph-mon[70044]: from='client.52752 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:22:09.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:22:08 smithi153 ceph-mon[70044]: from='client.52758 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:22:09.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:22:08 smithi153 ceph-mon[70044]: pgmap v16014: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:11.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:22:10 smithi083 ceph-mon[79930]: pgmap v16015: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:11.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:22:10 smithi083 ceph-mon[77242]: pgmap v16015: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:11.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:22:10 smithi153 ceph-mon[70044]: pgmap v16015: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:22:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:22:11] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T14:22:12.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:22:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:22:12.160Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:22:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:22:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:22:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:22:13.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:22:12 smithi083 ceph-mon[77242]: pgmap v16016: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:13.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:22:12 smithi083 ceph-mon[79930]: pgmap v16016: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:13.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:22:12 smithi153 ceph-mon[70044]: pgmap v16016: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:13.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:22:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:22:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:22:15.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:22:14 smithi083 ceph-mon[79930]: pgmap v16017: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:15.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:22:14 smithi083 ceph-mon[77242]: pgmap v16017: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:15.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:22:14 smithi153 ceph-mon[70044]: pgmap v16017: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:15.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:22:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:22:15.521Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:22:16.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:22:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:22:16.256Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:22:17.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:22:16 smithi083 ceph-mon[79930]: pgmap v16018: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:17.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:22:16 smithi083 ceph-mon[77242]: pgmap v16018: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:17.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:22:16 smithi153 ceph-mon[70044]: pgmap v16018: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:19.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:22:18 smithi083 ceph-mon[79930]: pgmap v16019: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:19.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:22:18 smithi083 ceph-mon[77242]: pgmap v16019: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:19.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:22:18 smithi153 ceph-mon[70044]: pgmap v16019: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:21.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:22:20 smithi083 ceph-mon[79930]: pgmap v16020: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:21.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:22:20 smithi083 ceph-mon[77242]: pgmap v16020: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:21.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:22:20 smithi153 ceph-mon[70044]: pgmap v16020: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:22:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:22:21] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T14:22:22.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:22:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:22:22.160Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:22:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:22:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:22:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:22:23.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:22:22 smithi083 ceph-mon[79930]: pgmap v16021: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:23.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:22:22 smithi083 ceph-mon[77242]: pgmap v16021: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:23.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:22:22 smithi153 ceph-mon[70044]: pgmap v16021: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:22:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:22:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:22:25.218 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:22:24 smithi083 ceph-mon[77242]: pgmap v16022: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:25.219 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:22:24 smithi083 ceph-mon[79930]: pgmap v16022: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:25.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:22:24 smithi153 ceph-mon[70044]: pgmap v16022: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:25.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:22:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:22:25.521Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:22:26.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:22:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:22:26.256Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:22:27.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:22:26 smithi083 ceph-mon[79930]: pgmap v16023: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:27.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:22:26 smithi083 ceph-mon[77242]: pgmap v16023: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:27.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:22:26 smithi153 ceph-mon[70044]: pgmap v16023: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:29.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:22:28 smithi083 ceph-mon[77242]: pgmap v16024: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:29.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:22:28 smithi083 ceph-mon[79930]: pgmap v16024: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:29.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:22:28 smithi153 ceph-mon[70044]: pgmap v16024: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:31.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:22:30 smithi083 ceph-mon[79930]: pgmap v16025: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:31.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:22:30 smithi083 ceph-mon[77242]: pgmap v16025: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:31.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:22:30 smithi153 ceph-mon[70044]: pgmap v16025: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:22:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:22:31] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T14:22:32.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:22:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:22:32.160Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:22:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:22:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:22:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:22:33.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:22:32 smithi083 ceph-mon[77242]: pgmap v16026: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:33.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:22:32 smithi083 ceph-mon[79930]: pgmap v16026: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:33.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:22:32 smithi153 ceph-mon[70044]: pgmap v16026: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:33.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:22:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:22:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:22:35.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:22:34 smithi083 ceph-mon[77242]: pgmap v16027: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:35.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:22:34 smithi083 ceph-mon[79930]: pgmap v16027: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:35.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:22:34 smithi153 ceph-mon[70044]: pgmap v16027: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:35.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:22:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:22:35.522Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:22:36.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:22:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:22:36.256Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:22:37.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:22:36 smithi083 ceph-mon[79930]: pgmap v16028: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:37.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:22:36 smithi083 ceph-mon[77242]: pgmap v16028: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:37.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:22:36 smithi153 ceph-mon[70044]: pgmap v16028: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:38.087 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:22:38.377 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:22:38.377 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 5m ago 8h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:22:38.378 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 3m ago 8h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:22:38.378 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 5m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:22:38.378 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 3m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:22:38.378 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 3m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:22:38.378 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 5m ago 9h 951M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:22:38.379 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 5m ago 9h 1948M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:22:38.379 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 3m ago 9h 847M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:22:38.379 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 5m ago 9h 1163M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:22:38.379 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 5m ago 8h 23.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:22:38.380 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 3m ago 8h 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:22:38.380 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 5m ago 8h 1412M 2247M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:22:38.380 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 5m ago 8h 1398M 2247M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:22:38.380 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 5m ago 8h 1088M 2247M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:22:38.380 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 5m ago 8h 1371M 2247M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:22:38.381 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 3m ago 8h 1203M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:22:38.381 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 3m ago 8h 1508M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:22:38.381 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 3m ago 8h 1348M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:22:38.382 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 3m ago 8h 1602M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:22:38.382 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 3m ago 8h 192M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:22:38.731 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:22:38.731 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:22:38.732 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:22:38.732 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:22:38.732 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:22:38.732 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:22:38.733 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:22:38.733 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:22:38.733 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:22:38.733 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:22:38.733 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:22:38.734 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:22:38.734 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:22:38.734 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:22:38.734 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:22:38.735 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:22:38.735 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:22:38.735 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:22:39.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:22:38 smithi083 ceph-mon[79930]: pgmap v16029: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:39.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:22:38 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4040538319' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:22:39.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:22:38 smithi083 ceph-mon[77242]: pgmap v16029: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:39.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:22:38 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4040538319' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:22:39.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:22:38 smithi153 ceph-mon[70044]: pgmap v16029: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:39.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:22:38 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4040538319' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:22:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:22:39 smithi083 ceph-mon[79930]: from='client.62701 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:22:40.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:22:39 smithi083 ceph-mon[79930]: from='client.52776 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:22:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:22:39 smithi083 ceph-mon[77242]: from='client.62701 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:22:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:22:39 smithi083 ceph-mon[77242]: from='client.52776 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:22:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:22:39 smithi153 ceph-mon[70044]: from='client.62701 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:22:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:22:39 smithi153 ceph-mon[70044]: from='client.52776 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:22:41.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:22:40 smithi083 ceph-mon[79930]: pgmap v16030: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:22:40 smithi083 ceph-mon[77242]: pgmap v16030: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:22:40 smithi153 ceph-mon[70044]: pgmap v16030: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:41.927 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:22:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:22:41] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T14:22:42.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:22:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:22:42.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:22:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:22:42.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:22:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:22:42.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:22:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:22:42.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:22:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:22:42.161Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:22:42.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:22:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:22:42.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:22:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:22:42.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:22:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:22:42.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:22:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:22:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:22:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:22:42.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:22:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:22:42.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:22:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:22:42.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:22:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:22:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:22:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:22:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:22:43.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:22:42 smithi083 ceph-mon[79930]: pgmap v16031: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:43.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:22:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:22:43.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:22:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:22:43.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:22:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:22:43.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:22:42 smithi083 ceph-mon[77242]: pgmap v16031: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:43.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:22:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:22:43.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:22:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:22:43.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:22:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:22:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:22:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:22:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:22:43.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:22:42 smithi153 ceph-mon[70044]: pgmap v16031: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:43.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:22:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:22:43.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:22:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:22:43.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:22:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:22:44.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:22:44 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:22:44.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:22:44 smithi153 ceph-mon[70044]: pgmap v16032: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:44.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:22:44 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:22:44.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:22:44 smithi083 ceph-mon[77242]: pgmap v16032: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:44.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:22:44 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:22:44.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:22:44 smithi083 ceph-mon[79930]: pgmap v16032: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:45.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:22:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:22:45.523Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:22:46.544 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:22:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:22:46.257Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:22:46.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:22:46 smithi153 ceph-mon[70044]: pgmap v16033: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:46.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:22:46 smithi083 ceph-mon[77242]: pgmap v16033: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:46.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:22:46 smithi083 ceph-mon[79930]: pgmap v16033: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:49.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:22:48 smithi153 ceph-mon[70044]: pgmap v16034: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:49.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:22:48 smithi083 ceph-mon[79930]: pgmap v16034: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:49.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:22:48 smithi083 ceph-mon[77242]: pgmap v16034: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:51.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:22:50 smithi153 ceph-mon[70044]: pgmap v16035: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:51.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:22:50 smithi083 ceph-mon[79930]: pgmap v16035: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:51.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:22:50 smithi083 ceph-mon[77242]: pgmap v16035: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:22:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:22:51] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T14:22:52.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:22:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:22:52.161Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:22:52.770 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:22:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:22:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:22:53.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:22:52 smithi153 ceph-mon[70044]: pgmap v16036: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:53.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:22:52 smithi083 ceph-mon[79930]: pgmap v16036: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:53.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:22:52 smithi083 ceph-mon[77242]: pgmap v16036: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:22:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:22:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:22:55.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:22:54 smithi153 ceph-mon[70044]: pgmap v16037: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:55.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:22:54 smithi083 ceph-mon[79930]: pgmap v16037: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:55.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:22:54 smithi083 ceph-mon[77242]: pgmap v16037: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:55.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:22:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:22:55.523Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:22:56.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:22:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:22:56.257Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:22:57.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:22:56 smithi153 ceph-mon[70044]: pgmap v16038: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:57.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:22:56 smithi083 ceph-mon[79930]: pgmap v16038: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:57.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:22:56 smithi083 ceph-mon[77242]: pgmap v16038: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:59.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:22:58 smithi153 ceph-mon[70044]: pgmap v16039: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:59.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:22:58 smithi083 ceph-mon[79930]: pgmap v16039: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:22:59.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:22:58 smithi083 ceph-mon[77242]: pgmap v16039: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:01.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:23:00 smithi153 ceph-mon[70044]: pgmap v16040: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:01.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:23:00 smithi083 ceph-mon[79930]: pgmap v16040: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:01.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:23:00 smithi083 ceph-mon[77242]: pgmap v16040: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:23:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:23:01] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T14:23:02.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:23:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:23:02.162Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:23:02.786 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:23:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:23:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:23:03.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:23:02 smithi153 ceph-mon[70044]: pgmap v16041: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:03.059 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:23:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:23:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:23:03.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:23:02 smithi083 ceph-mon[77242]: pgmap v16041: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:03.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:23:02 smithi083 ceph-mon[79930]: pgmap v16041: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:05.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:23:04 smithi153 ceph-mon[70044]: pgmap v16042: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:05.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:23:04 smithi083 ceph-mon[79930]: pgmap v16042: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:05.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:23:04 smithi083 ceph-mon[77242]: pgmap v16042: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:05.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:23:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:23:05.524Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:23:06.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:23:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:23:06.258Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:23:07.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:23:06 smithi153 ceph-mon[70044]: pgmap v16043: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:07.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:23:06 smithi083 ceph-mon[79930]: pgmap v16043: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:07.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:23:06 smithi083 ceph-mon[77242]: pgmap v16043: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:09.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:23:08 smithi153 ceph-mon[70044]: pgmap v16044: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:09.060 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:23:09.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:23:08 smithi083 ceph-mon[79930]: pgmap v16044: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:09.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:23:08 smithi083 ceph-mon[77242]: pgmap v16044: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:09.349 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:23:09.349 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 5m ago 8h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:23:09.350 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 3m ago 8h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:23:09.350 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 5m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:23:09.350 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 3m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:23:09.351 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 3m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:23:09.351 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 5m ago 9h 951M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:23:09.351 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 5m ago 9h 1948M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:23:09.351 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 3m ago 9h 847M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:23:09.351 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 5m ago 9h 1163M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:23:09.352 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 5m ago 8h 23.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:23:09.352 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 3m ago 8h 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:23:09.352 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 5m ago 9h 1412M 2247M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:23:09.352 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 5m ago 9h 1398M 2247M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:23:09.352 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 5m ago 8h 1088M 2247M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:23:09.353 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 5m ago 8h 1371M 2247M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:23:09.353 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 3m ago 8h 1203M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:23:09.353 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 3m ago 8h 1508M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:23:09.353 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 3m ago 8h 1348M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:23:09.354 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 3m ago 8h 1602M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:23:09.354 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 3m ago 8h 192M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:23:09.707 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:23:09.708 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:23:09.708 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:23:09.708 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:23:09.708 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:23:09.709 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:23:09.709 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:23:09.709 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:23:09.709 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:23:09.710 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:23:09.710 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:23:09.710 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:23:09.710 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:23:09.711 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:23:09.711 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:23:09.711 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:23:09.711 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:23:09.711 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:23:10.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:23:09 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2737890109' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:23:10.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:23:09 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2737890109' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:23:10.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:23:09 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2737890109' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:23:11.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:23:10 smithi153 ceph-mon[70044]: from='client.62719 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:23:11.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:23:10 smithi153 ceph-mon[70044]: from='client.62725 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:23:11.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:23:10 smithi153 ceph-mon[70044]: pgmap v16045: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:11.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:23:10 smithi083 ceph-mon[79930]: from='client.62719 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:23:11.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:23:10 smithi083 ceph-mon[79930]: from='client.62725 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:23:11.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:23:10 smithi083 ceph-mon[79930]: pgmap v16045: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:11.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:23:10 smithi083 ceph-mon[77242]: from='client.62719 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:23:11.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:23:10 smithi083 ceph-mon[77242]: from='client.62725 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:23:11.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:23:10 smithi083 ceph-mon[77242]: pgmap v16045: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:23:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:23:11] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T14:23:12.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:23:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:23:12.163Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:23:12.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:23:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:23:12.317Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:23:13.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:23:12 smithi153 ceph-mon[70044]: pgmap v16046: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:13.058 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:23:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:23:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:23:13.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:23:12 smithi083 ceph-mon[79930]: pgmap v16046: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:13.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:23:12 smithi083 ceph-mon[77242]: pgmap v16046: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:15.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:23:14 smithi083 ceph-mon[79930]: pgmap v16047: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:15.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:23:14 smithi083 ceph-mon[77242]: pgmap v16047: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:15.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:23:14 smithi153 ceph-mon[70044]: pgmap v16047: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:15.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:23:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:23:15.524Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:23:16.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:23:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:23:16.259Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:23:17.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:23:16 smithi083 ceph-mon[77242]: pgmap v16048: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:17.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:23:16 smithi083 ceph-mon[79930]: pgmap v16048: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:17.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:23:16 smithi153 ceph-mon[70044]: pgmap v16048: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:19.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:23:18 smithi083 ceph-mon[79930]: pgmap v16049: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:19.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:23:18 smithi083 ceph-mon[77242]: pgmap v16049: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:19.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:23:18 smithi153 ceph-mon[70044]: pgmap v16049: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:21.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:23:20 smithi083 ceph-mon[79930]: pgmap v16050: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:21.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:23:20 smithi083 ceph-mon[77242]: pgmap v16050: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:21.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:23:20 smithi153 ceph-mon[70044]: pgmap v16050: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:23:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:23:21] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T14:23:22.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:23:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:23:22.163Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:23:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:23:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:23:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:23:23.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:23:22 smithi083 ceph-mon[79930]: pgmap v16051: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:23.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:23:22 smithi083 ceph-mon[77242]: pgmap v16051: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:23.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:23:22 smithi153 ceph-mon[70044]: pgmap v16051: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:23:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:23:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:23:25.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:23:24 smithi083 ceph-mon[77242]: pgmap v16052: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:25.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:23:24 smithi083 ceph-mon[79930]: pgmap v16052: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:25.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:23:24 smithi153 ceph-mon[70044]: pgmap v16052: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:25.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:23:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:23:25.525Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:23:26.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:23:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:23:26.259Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:23:27.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:23:26 smithi083 ceph-mon[77242]: pgmap v16053: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:27.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:23:26 smithi083 ceph-mon[79930]: pgmap v16053: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:27.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:23:26 smithi153 ceph-mon[70044]: pgmap v16053: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:29.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:23:28 smithi083 ceph-mon[79930]: pgmap v16054: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:29.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:23:28 smithi083 ceph-mon[77242]: pgmap v16054: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:29.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:23:28 smithi153 ceph-mon[70044]: pgmap v16054: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:31.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:23:30 smithi083 ceph-mon[79930]: pgmap v16055: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:31.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:23:30 smithi083 ceph-mon[77242]: pgmap v16055: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:31.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:23:30 smithi153 ceph-mon[70044]: pgmap v16055: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:23:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:23:31] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T14:23:32.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:23:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:23:32.163Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:23:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:23:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:23:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:23:33.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:23:32 smithi083 ceph-mon[79930]: pgmap v16056: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:33.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:23:32 smithi083 ceph-mon[77242]: pgmap v16056: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:33.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:23:32 smithi153 ceph-mon[70044]: pgmap v16056: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:23:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:23:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:23:35.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:23:34 smithi083 ceph-mon[79930]: pgmap v16057: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:35.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:23:34 smithi083 ceph-mon[77242]: pgmap v16057: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:35.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:23:34 smithi153 ceph-mon[70044]: pgmap v16057: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:35.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:23:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:23:35.526Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:23:36.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:23:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:23:36.259Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:23:37.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:23:36 smithi083 ceph-mon[79930]: pgmap v16058: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:37.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:23:36 smithi083 ceph-mon[77242]: pgmap v16058: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:37.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:23:36 smithi153 ceph-mon[70044]: pgmap v16058: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:39.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:23:38 smithi083 ceph-mon[79930]: pgmap v16059: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:39.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:23:38 smithi083 ceph-mon[77242]: pgmap v16059: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:39.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:23:38 smithi153 ceph-mon[70044]: pgmap v16059: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:40.029 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:23:40.315 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:23:40.315 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 6m ago 8h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:23:40.315 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 4m ago 8h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:23:40.316 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 6m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:23:40.316 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 4m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:23:40.316 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 4m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:23:40.316 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 6m ago 9h 951M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:23:40.316 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 6m ago 9h 1948M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:23:40.317 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 4m ago 9h 847M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:23:40.317 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 6m ago 9h 1163M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:23:40.317 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 6m ago 8h 23.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:23:40.318 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 4m ago 8h 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:23:40.318 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 6m ago 9h 1412M 2247M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:23:40.318 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 6m ago 9h 1398M 2247M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:23:40.318 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 6m ago 9h 1088M 2247M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:23:40.318 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 6m ago 9h 1371M 2247M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:23:40.319 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 4m ago 8h 1203M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:23:40.319 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 4m ago 8h 1508M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:23:40.319 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 4m ago 8h 1348M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:23:40.319 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 4m ago 8h 1602M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:23:40.320 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 4m ago 8h 192M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:23:40.671 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:23:40.672 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:23:40.672 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:23:40.672 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:23:40.673 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:23:40.673 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:23:40.673 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:23:40.673 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:23:40.674 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:23:40.674 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:23:40.674 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:23:40.674 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:23:40.674 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:23:40.675 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:23:40.675 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:23:40.675 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:23:40.675 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:23:40.676 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:23:41.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:23:40 smithi083 ceph-mon[77242]: pgmap v16060: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:41.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:23:40 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1207300567' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:23:41.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:23:40 smithi083 ceph-mon[79930]: pgmap v16060: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:41.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:23:40 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1207300567' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:23:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:23:40 smithi153 ceph-mon[70044]: pgmap v16060: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:23:40 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1207300567' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:23:41.853 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:23:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:23:41] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T14:23:42.167 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:23:41 smithi083 ceph-mon[79930]: from='client.52806 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:23:42.167 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:23:41 smithi083 ceph-mon[79930]: from='client.62743 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:23:42.167 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:23:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:23:42.168 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:23:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:23:42.168 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:23:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:23:42.168 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:23:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:23:42.169 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:23:41 smithi083 ceph-mon[77242]: from='client.52806 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:23:42.169 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:23:41 smithi083 ceph-mon[77242]: from='client.62743 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:23:42.169 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:23:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:23:42.169 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:23:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:23:42.170 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:23:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:23:42.170 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:23:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:23:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:23:41 smithi153 ceph-mon[70044]: from='client.52806 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:23:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:23:41 smithi153 ceph-mon[70044]: from='client.62743 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:23:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:23:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:23:42.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:23:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:23:42.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:23:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:23:42.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:23:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:23:42.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:23:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:23:42.164Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:23:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:23:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:23:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:23:43.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:23:42 smithi083 ceph-mon[77242]: pgmap v16061: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:43.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:23:42 smithi083 ceph-mon[79930]: pgmap v16061: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:43.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:23:42 smithi153 ceph-mon[70044]: pgmap v16061: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:43.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:23:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:23:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:23:44.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:23:43 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:23:44.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:23:43 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:23:44.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:23:43 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:23:44.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:23:43 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:23:44.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:23:43 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:23:44.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:23:43 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:23:44.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:23:43 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:23:44.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:23:43 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:23:44.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:23:43 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:23:45.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:23:44 smithi083 ceph-mon[79930]: pgmap v16062: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:45.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:23:44 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:23:45.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:23:44 smithi083 ceph-mon[77242]: pgmap v16062: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:45.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:23:44 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:23:45.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:23:44 smithi153 ceph-mon[70044]: pgmap v16062: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:45.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:23:44 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:23:45.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:23:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:23:45.527Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:23:46.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:23:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:23:46.260Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:23:47.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:23:46 smithi083 ceph-mon[79930]: pgmap v16063: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:47.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:23:46 smithi083 ceph-mon[77242]: pgmap v16063: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:47.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:23:46 smithi153 ceph-mon[70044]: pgmap v16063: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:49.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:23:48 smithi083 ceph-mon[77242]: pgmap v16064: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:49.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:23:48 smithi083 ceph-mon[79930]: pgmap v16064: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:49.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:23:48 smithi153 ceph-mon[70044]: pgmap v16064: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:51.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:23:50 smithi083 ceph-mon[77242]: pgmap v16065: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:51.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:23:50 smithi083 ceph-mon[79930]: pgmap v16065: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:51.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:23:50 smithi153 ceph-mon[70044]: pgmap v16065: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:23:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:23:51] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T14:23:52.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:23:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:23:52.165Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:23:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:23:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:23:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:23:53.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:23:52 smithi083 ceph-mon[77242]: pgmap v16066: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:53.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:23:52 smithi083 ceph-mon[79930]: pgmap v16066: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:53.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:23:52 smithi153 ceph-mon[70044]: pgmap v16066: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:23:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:23:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:23:55.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:23:54 smithi083 ceph-mon[77242]: pgmap v16067: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:55.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:23:54 smithi083 ceph-mon[79930]: pgmap v16067: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:55.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:23:54 smithi153 ceph-mon[70044]: pgmap v16067: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:55.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:23:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:23:55.527Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:23:56.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:23:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:23:56.260Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:23:57.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:23:56 smithi083 ceph-mon[79930]: pgmap v16068: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:57.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:23:56 smithi083 ceph-mon[77242]: pgmap v16068: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:57.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:23:56 smithi153 ceph-mon[70044]: pgmap v16068: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:59.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:23:58 smithi083 ceph-mon[77242]: pgmap v16069: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:59.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:23:58 smithi083 ceph-mon[79930]: pgmap v16069: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:23:59.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:23:58 smithi153 ceph-mon[70044]: pgmap v16069: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:01.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:24:00 smithi083 ceph-mon[79930]: pgmap v16070: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:01.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:24:00 smithi083 ceph-mon[77242]: pgmap v16070: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:01.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:24:00 smithi153 ceph-mon[70044]: pgmap v16070: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:24:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:24:01] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T14:24:02.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:24:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:24:02.165Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:24:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:24:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:24:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:24:03.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:24:02 smithi083 ceph-mon[77242]: pgmap v16071: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:03.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:24:02 smithi083 ceph-mon[79930]: pgmap v16071: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:03.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:24:02 smithi153 ceph-mon[70044]: pgmap v16071: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:24:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:24:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:24:05.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:24:04 smithi083 ceph-mon[79930]: pgmap v16072: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:05.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:24:04 smithi083 ceph-mon[77242]: pgmap v16072: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:05.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:24:04 smithi153 ceph-mon[70044]: pgmap v16072: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:05.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:24:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:24:05.528Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:24:06.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:24:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:24:06.261Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:24:07.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:24:06 smithi153 ceph-mon[70044]: pgmap v16073: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:07.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:24:06 smithi083 ceph-mon[79930]: pgmap v16073: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:07.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:24:06 smithi083 ceph-mon[77242]: pgmap v16073: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:09.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:24:08 smithi153 ceph-mon[70044]: pgmap v16074: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:09.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:24:08 smithi083 ceph-mon[79930]: pgmap v16074: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:09.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:24:08 smithi083 ceph-mon[77242]: pgmap v16074: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:11.003 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:24:11.291 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:24:11.291 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 6m ago 8h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:24:11.292 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 4m ago 8h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:24:11.292 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 6m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:24:11.292 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 4m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:24:11.292 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 4m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:24:11.293 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 6m ago 9h 951M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:24:11.293 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 6m ago 9h 1948M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:24:11.293 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 4m ago 9h 847M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:24:11.293 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 6m ago 9h 1163M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:24:11.293 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 6m ago 8h 23.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:24:11.294 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 4m ago 8h 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:24:11.294 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 6m ago 9h 1412M 2247M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:24:11.294 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 6m ago 9h 1398M 2247M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:24:11.295 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 6m ago 9h 1088M 2247M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:24:11.295 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 6m ago 9h 1371M 2247M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:24:11.295 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 4m ago 9h 1203M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:24:11.295 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 4m ago 9h 1508M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:24:11.296 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 4m ago 8h 1348M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:24:11.296 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 4m ago 8h 1602M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:24:11.296 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 4m ago 8h 192M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:24:11.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:24:10 smithi153 ceph-mon[70044]: pgmap v16075: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:11.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:24:10 smithi083 ceph-mon[79930]: pgmap v16075: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:11.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:24:10 smithi083 ceph-mon[77242]: pgmap v16075: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:11.648 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:24:11.649 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:24:11.649 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:24:11.649 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:24:11.650 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:24:11.650 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:24:11.650 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:24:11.650 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:24:11.651 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:24:11.651 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:24:11.651 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:24:11.651 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:24:11.651 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:24:11.652 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:24:11.652 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:24:11.652 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:24:11.652 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:24:11.653 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:24:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:24:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:24:11] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T14:24:12.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:24:11 smithi153 ceph-mon[70044]: from='client.52824 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:24:12.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:24:11 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/799346894' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:24:12.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:24:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:24:12.165Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:24:12.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:24:11 smithi083 ceph-mon[77242]: from='client.52824 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:24:12.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:24:11 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/799346894' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:24:12.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:24:11 smithi083 ceph-mon[79930]: from='client.52824 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:24:12.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:24:11 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/799346894' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:24:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:24:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:24:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:24:13.312 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:24:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:24:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:24:13.312 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:24:12 smithi153 ceph-mon[70044]: from='client.62761 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:24:13.313 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:24:12 smithi153 ceph-mon[70044]: pgmap v16076: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:13.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:24:12 smithi083 ceph-mon[79930]: from='client.62761 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:24:13.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:24:12 smithi083 ceph-mon[79930]: pgmap v16076: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:13.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:24:12 smithi083 ceph-mon[77242]: from='client.62761 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:24:13.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:24:12 smithi083 ceph-mon[77242]: pgmap v16076: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:15.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:24:14 smithi153 ceph-mon[70044]: pgmap v16077: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:15.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:24:14 smithi083 ceph-mon[79930]: pgmap v16077: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:15.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:24:14 smithi083 ceph-mon[77242]: pgmap v16077: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:15.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:24:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:24:15.529Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:24:16.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:24:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:24:16.261Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:24:17.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:24:16 smithi153 ceph-mon[70044]: pgmap v16078: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:17.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:24:16 smithi083 ceph-mon[79930]: pgmap v16078: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:17.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:24:16 smithi083 ceph-mon[77242]: pgmap v16078: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:19.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:24:18 smithi153 ceph-mon[70044]: pgmap v16079: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:19.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:24:18 smithi083 ceph-mon[79930]: pgmap v16079: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:19.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:24:18 smithi083 ceph-mon[77242]: pgmap v16079: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:21.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:24:20 smithi153 ceph-mon[70044]: pgmap v16080: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:21.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:24:20 smithi083 ceph-mon[79930]: pgmap v16080: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:21.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:24:20 smithi083 ceph-mon[77242]: pgmap v16080: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:24:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:24:21] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T14:24:22.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:24:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:24:22.167Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:24:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:24:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:24:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:24:23.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:24:22 smithi153 ceph-mon[70044]: pgmap v16081: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:23.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:24:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:24:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:24:23.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:24:22 smithi083 ceph-mon[79930]: pgmap v16081: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:23.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:24:22 smithi083 ceph-mon[77242]: pgmap v16081: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:25.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:24:24 smithi153 ceph-mon[70044]: pgmap v16082: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:25.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:24:24 smithi083 ceph-mon[77242]: pgmap v16082: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:25.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:24:24 smithi083 ceph-mon[79930]: pgmap v16082: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:25.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:24:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:24:25.530Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:24:26.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:24:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:24:26.262Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:24:27.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:24:27 smithi153 ceph-mon[70044]: pgmap v16083: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:27.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:24:27 smithi083 ceph-mon[79930]: pgmap v16083: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:27.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:24:27 smithi083 ceph-mon[77242]: pgmap v16083: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:29.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:24:29 smithi153 ceph-mon[70044]: pgmap v16084: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:29.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:24:29 smithi083 ceph-mon[77242]: pgmap v16084: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:29.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:24:29 smithi083 ceph-mon[79930]: pgmap v16084: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:30.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:24:30 smithi153 ceph-mon[70044]: pgmap v16085: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:30.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:24:30 smithi083 ceph-mon[77242]: pgmap v16085: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:30.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:24:30 smithi083 ceph-mon[79930]: pgmap v16085: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:24:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:24:31] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T14:24:32.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:24:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:24:32.167Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:24:32.789 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:24:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:24:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:24:33.058 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:24:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:24:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:24:33.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:24:32 smithi153 ceph-mon[70044]: pgmap v16086: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:33.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:24:32 smithi083 ceph-mon[79930]: pgmap v16086: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:33.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:24:32 smithi083 ceph-mon[77242]: pgmap v16086: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:35.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:24:34 smithi153 ceph-mon[70044]: pgmap v16087: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:35.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:24:34 smithi083 ceph-mon[79930]: pgmap v16087: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:35.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:24:34 smithi083 ceph-mon[77242]: pgmap v16087: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:35.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:24:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:24:35.531Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:24:36.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:24:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:24:36.263Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:24:37.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:24:36 smithi153 ceph-mon[70044]: pgmap v16088: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:37.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:24:36 smithi083 ceph-mon[79930]: pgmap v16088: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:37.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:24:36 smithi083 ceph-mon[77242]: pgmap v16088: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:39.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:24:38 smithi153 ceph-mon[70044]: pgmap v16089: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:39.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:24:38 smithi083 ceph-mon[77242]: pgmap v16089: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:39.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:24:38 smithi083 ceph-mon[79930]: pgmap v16089: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:24:40 smithi153 ceph-mon[70044]: pgmap v16090: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:41.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:24:40 smithi083 ceph-mon[79930]: pgmap v16090: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:41.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:24:40 smithi083 ceph-mon[77242]: pgmap v16090: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:41.814 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:24:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:24:41] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T14:24:41.814 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:24:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:24:41.815 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:24:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:24:41.815 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:24:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:24:41.815 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:24:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:24:41.976 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:24:42.169 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:24:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:24:42.169 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:24:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:24:42.170 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:24:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:24:42.170 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:24:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:24:42.266 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:24:42.266 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 7m ago 8h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:24:42.266 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 5m ago 8h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:24:42.266 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (8h) 7m ago 8h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:24:42.267 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (8h) 5m ago 8h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:24:42.267 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 5m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:24:42.267 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 7m ago 9h 951M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:24:42.267 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 7m ago 9h 1948M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:24:42.267 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 5m ago 9h 847M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:24:42.268 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 7m ago 9h 1163M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:24:42.268 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 7m ago 8h 23.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:24:42.268 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 5m ago 8h 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:24:42.268 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 7m ago 9h 1412M 2247M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:24:42.269 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 7m ago 9h 1398M 2247M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:24:42.269 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 7m ago 9h 1088M 2247M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:24:42.269 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 7m ago 9h 1371M 2247M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:24:42.269 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 5m ago 9h 1203M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:24:42.270 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 5m ago 9h 1508M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:24:42.270 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 5m ago 9h 1348M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:24:42.270 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 5m ago 9h 1602M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:24:42.270 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 5m ago 8h 192M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:24:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:24:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:24:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:24:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:24:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:24:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:24:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:24:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:24:42.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:24:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:24:42.167Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:24:42.618 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:24:42.618 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:24:42.619 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:24:42.619 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:24:42.619 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:24:42.619 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:24:42.620 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:24:42.620 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:24:42.620 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:24:42.620 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:24:42.620 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:24:42.621 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:24:42.621 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:24:42.621 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:24:42.621 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:24:42.622 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:24:42.622 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:24:42.622 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:24:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:24:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:24:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:24:43.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:24:42 smithi083 ceph-mon[77242]: pgmap v16091: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:43.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:24:42 smithi083 ceph-mon[77242]: from='client.62773 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:24:43.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:24:42 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4142284728' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:24:43.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:24:42 smithi083 ceph-mon[79930]: pgmap v16091: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:43.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:24:42 smithi083 ceph-mon[79930]: from='client.62773 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:24:43.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:24:42 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4142284728' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:24:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:24:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:24:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:24:43.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:24:42 smithi153 ceph-mon[70044]: pgmap v16091: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:43.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:24:42 smithi153 ceph-mon[70044]: from='client.62773 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:24:43.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:24:42 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4142284728' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:24:44.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:24:43 smithi083 ceph-mon[79930]: from='client.62779 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:24:44.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:24:43 smithi083 ceph-mon[77242]: from='client.62779 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:24:44.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:24:43 smithi153 ceph-mon[70044]: from='client.62779 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:24:45.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:24:44 smithi083 ceph-mon[79930]: pgmap v16092: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:45.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:24:44 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:24:45.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:24:44 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:24:45.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:24:44 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:24:45.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:24:44 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:24:45.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:24:44 smithi083 ceph-mon[77242]: pgmap v16092: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:45.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:24:44 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:24:45.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:24:44 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:24:45.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:24:44 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:24:45.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:24:44 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:24:45.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:24:44 smithi153 ceph-mon[70044]: pgmap v16092: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:45.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:24:44 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:24:45.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:24:44 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:24:45.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:24:44 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:24:45.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:24:44 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:24:45.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:24:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:24:45.532Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:24:46.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:24:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:24:46.263Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:24:47.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:24:46 smithi083 ceph-mon[77242]: pgmap v16093: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:47.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:24:46 smithi083 ceph-mon[79930]: pgmap v16093: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:47.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:24:46 smithi153 ceph-mon[70044]: pgmap v16093: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:49.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:24:48 smithi083 ceph-mon[79930]: pgmap v16094: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:49.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:24:48 smithi083 ceph-mon[77242]: pgmap v16094: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:49.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:24:48 smithi153 ceph-mon[70044]: pgmap v16094: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:51.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:24:50 smithi083 ceph-mon[79930]: pgmap v16095: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:51.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:24:50 smithi083 ceph-mon[77242]: pgmap v16095: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:51.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:24:50 smithi153 ceph-mon[70044]: pgmap v16095: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:24:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:24:51] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T14:24:52.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:24:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:24:52.168Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:24:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:24:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:24:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:24:53.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:24:52 smithi083 ceph-mon[79930]: pgmap v16096: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:53.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:24:52 smithi083 ceph-mon[77242]: pgmap v16096: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:53.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:24:52 smithi153 ceph-mon[70044]: pgmap v16096: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:24:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:24:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:24:55.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:24:54 smithi083 ceph-mon[79930]: pgmap v16097: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:55.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:24:54 smithi083 ceph-mon[77242]: pgmap v16097: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:55.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:24:54 smithi153 ceph-mon[70044]: pgmap v16097: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:55.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:24:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:24:55.532Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:24:56.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:24:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:24:56.264Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:24:57.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:24:56 smithi083 ceph-mon[77242]: pgmap v16098: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:57.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:24:56 smithi083 ceph-mon[79930]: pgmap v16098: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:57.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:24:56 smithi153 ceph-mon[70044]: pgmap v16098: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:59.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:24:58 smithi083 ceph-mon[79930]: pgmap v16099: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:59.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:24:58 smithi083 ceph-mon[77242]: pgmap v16099: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:24:59.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:24:58 smithi153 ceph-mon[70044]: pgmap v16099: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:01.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:25:00 smithi083 ceph-mon[79930]: pgmap v16100: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:01.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:25:00 smithi083 ceph-mon[77242]: pgmap v16100: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:01.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:25:00 smithi153 ceph-mon[70044]: pgmap v16100: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:25:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:25:01] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T14:25:02.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:25:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:25:02.169Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:25:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:25:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:25:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:25:03.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:25:02 smithi083 ceph-mon[77242]: pgmap v16101: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:03.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:25:02 smithi083 ceph-mon[79930]: pgmap v16101: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:03.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:25:02 smithi153 ceph-mon[70044]: pgmap v16101: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:25:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:25:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:25:05.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:25:04 smithi083 ceph-mon[79930]: pgmap v16102: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:05.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:25:04 smithi083 ceph-mon[77242]: pgmap v16102: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:05.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:25:04 smithi153 ceph-mon[70044]: pgmap v16102: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:05.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:25:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:25:05.533Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:25:06.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:25:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:25:06.265Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:25:07.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:25:06 smithi083 ceph-mon[77242]: pgmap v16103: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:07.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:25:06 smithi083 ceph-mon[79930]: pgmap v16103: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:07.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:25:06 smithi153 ceph-mon[70044]: pgmap v16103: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:09.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:25:08 smithi083 ceph-mon[79930]: pgmap v16104: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:09.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:25:08 smithi083 ceph-mon[77242]: pgmap v16104: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:09.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:25:08 smithi153 ceph-mon[70044]: pgmap v16104: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:11.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:25:10 smithi083 ceph-mon[79930]: pgmap v16105: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:11.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:25:10 smithi083 ceph-mon[77242]: pgmap v16105: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:11.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:25:10 smithi153 ceph-mon[70044]: pgmap v16105: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:25:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:25:11] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T14:25:12.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:25:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:25:12.169Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:25:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:25:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:25:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:25:12.944 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:25:13.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:25:12 smithi083 ceph-mon[79930]: pgmap v16106: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:13.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:25:12 smithi083 ceph-mon[77242]: pgmap v16106: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:13.234 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:25:13.235 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 7m ago 8h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:25:13.235 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 5m ago 8h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:25:13.235 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 7m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:25:13.235 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 5m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:25:13.236 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 5m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:25:13.236 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 7m ago 9h 951M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:25:13.236 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 7m ago 9h 1948M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:25:13.236 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 5m ago 9h 847M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:25:13.237 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 7m ago 9h 1163M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:25:13.237 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8h) 7m ago 8h 23.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:25:13.237 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (8h) 5m ago 8h 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:25:13.237 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 7m ago 9h 1412M 2247M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:25:13.237 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 7m ago 9h 1398M 2247M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:25:13.238 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 7m ago 9h 1088M 2247M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:25:13.238 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 7m ago 9h 1371M 2247M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:25:13.238 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 5m ago 9h 1203M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:25:13.238 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 5m ago 9h 1508M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:25:13.239 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 5m ago 9h 1348M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:25:13.239 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 5m ago 9h 1602M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:25:13.239 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 5m ago 8h 192M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:25:13.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:25:12 smithi153 ceph-mon[70044]: pgmap v16106: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:25:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:25:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:25:13.599 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:25:13.599 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:25:13.599 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:25:13.599 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:25:13.600 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:25:13.600 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:25:13.600 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:25:13.600 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:25:13.601 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:25:13.601 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:25:13.601 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:25:13.601 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:25:13.601 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:25:13.602 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:25:13.602 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:25:13.602 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:25:13.602 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:25:13.603 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:25:14.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:25:13 smithi083 ceph-mon[77242]: from='client.52860 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:25:14.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:25:13 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1755830948' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:25:14.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:25:13 smithi083 ceph-mon[79930]: from='client.52860 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:25:14.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:25:13 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1755830948' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:25:14.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:25:13 smithi153 ceph-mon[70044]: from='client.52860 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:25:14.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:25:13 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1755830948' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:25:15.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:25:14 smithi083 ceph-mon[79930]: from='client.62797 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:25:15.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:25:14 smithi083 ceph-mon[79930]: pgmap v16107: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:15.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:25:14 smithi083 ceph-mon[77242]: from='client.62797 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:25:15.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:25:14 smithi083 ceph-mon[77242]: pgmap v16107: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:15.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:25:14 smithi153 ceph-mon[70044]: from='client.62797 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:25:15.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:25:14 smithi153 ceph-mon[70044]: pgmap v16107: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:15.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:25:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:25:15.534Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:25:16.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:25:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:25:16.266Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:25:17.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:25:16 smithi083 ceph-mon[79930]: pgmap v16108: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:17.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:25:16 smithi083 ceph-mon[77242]: pgmap v16108: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:17.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:25:16 smithi153 ceph-mon[70044]: pgmap v16108: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:19.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:25:18 smithi083 ceph-mon[79930]: pgmap v16109: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:19.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:25:18 smithi083 ceph-mon[77242]: pgmap v16109: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:19.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:25:18 smithi153 ceph-mon[70044]: pgmap v16109: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:21.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:25:20 smithi083 ceph-mon[79930]: pgmap v16110: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:21.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:25:20 smithi083 ceph-mon[77242]: pgmap v16110: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:21.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:25:20 smithi153 ceph-mon[70044]: pgmap v16110: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:25:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:25:21] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T14:25:22.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:25:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:25:22.170Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:25:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:25:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:25:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:25:23.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:25:22 smithi083 ceph-mon[77242]: pgmap v16111: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:23.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:25:22 smithi083 ceph-mon[79930]: pgmap v16111: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:23.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:25:22 smithi153 ceph-mon[70044]: pgmap v16111: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:25:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:25:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:25:25.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:25:24 smithi083 ceph-mon[77242]: pgmap v16112: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:25.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:25:24 smithi083 ceph-mon[79930]: pgmap v16112: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:25.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:25:24 smithi153 ceph-mon[70044]: pgmap v16112: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:25.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:25:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:25:25.534Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:25:26.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:25:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:25:26.266Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:25:27.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:25:26 smithi083 ceph-mon[79930]: pgmap v16113: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:27.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:25:26 smithi083 ceph-mon[77242]: pgmap v16113: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:27.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:25:26 smithi153 ceph-mon[70044]: pgmap v16113: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:29.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:25:28 smithi083 ceph-mon[79930]: pgmap v16114: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:29.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:25:28 smithi083 ceph-mon[77242]: pgmap v16114: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:29.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:25:28 smithi153 ceph-mon[70044]: pgmap v16114: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:31.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:25:30 smithi083 ceph-mon[79930]: pgmap v16115: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:31.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:25:30 smithi083 ceph-mon[77242]: pgmap v16115: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:31.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:25:30 smithi153 ceph-mon[70044]: pgmap v16115: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:25:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:25:31] "GET /metrics HTTP/1.1" 200 225246 "" "Prometheus/2.33.4" 2022-09-29T14:25:32.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:25:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:25:32.170Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:25:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:25:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:25:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:25:33.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:25:32 smithi083 ceph-mon[79930]: pgmap v16116: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:33.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:25:32 smithi083 ceph-mon[77242]: pgmap v16116: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:33.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:25:32 smithi153 ceph-mon[70044]: pgmap v16116: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:25:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:25:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:25:35.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:25:34 smithi083 ceph-mon[79930]: pgmap v16117: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:35.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:25:34 smithi083 ceph-mon[77242]: pgmap v16117: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:35.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:25:34 smithi153 ceph-mon[70044]: pgmap v16117: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:35.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:25:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:25:35.535Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:25:36.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:25:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:25:36.267Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:25:37.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:25:36 smithi083 ceph-mon[79930]: pgmap v16118: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:37.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:25:36 smithi083 ceph-mon[77242]: pgmap v16118: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:37.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:25:36 smithi153 ceph-mon[70044]: pgmap v16118: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:39.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:25:38 smithi083 ceph-mon[79930]: pgmap v16119: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:39.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:25:38 smithi083 ceph-mon[77242]: pgmap v16119: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:39.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:25:38 smithi153 ceph-mon[70044]: pgmap v16119: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:41.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:25:40 smithi083 ceph-mon[79930]: pgmap v16120: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:25:40 smithi083 ceph-mon[77242]: pgmap v16120: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:25:40 smithi153 ceph-mon[70044]: pgmap v16120: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:41.918 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:25:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:25:41] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T14:25:41.920 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:25:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:25:41.921 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:25:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:25:41.921 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:25:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:25:41.921 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:25:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:25:42.172 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:25:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:25:42.173 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:25:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:25:42.173 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:25:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:25:42.173 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:25:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:25:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:25:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:25:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:25:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:25:42.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:25:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:25:42.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:25:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:25:42.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:25:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:25:42.170Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:25:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:25:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:25:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:25:43.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:25:42 smithi083 ceph-mon[77242]: pgmap v16121: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:43.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:25:42 smithi083 ceph-mon[79930]: pgmap v16121: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:43.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:25:42 smithi153 ceph-mon[70044]: pgmap v16121: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:25:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:25:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:25:43.928 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:25:44.215 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:25:44.216 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 8m ago 8h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:25:44.216 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (8h) 6m ago 8h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:25:44.216 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 8m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:25:44.217 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 6m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:25:44.217 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 6m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:25:44.217 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 8m ago 9h 951M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:25:44.217 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 8m ago 9h 1948M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:25:44.218 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 6m ago 9h 847M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:25:44.218 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 8m ago 9h 1163M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:25:44.218 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 8m ago 9h 23.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:25:44.218 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 6m ago 9h 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:25:44.219 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 8m ago 9h 1412M 2247M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:25:44.219 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 8m ago 9h 1398M 2247M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:25:44.219 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 8m ago 9h 1088M 2247M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:25:44.219 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 8m ago 9h 1371M 2247M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:25:44.219 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 6m ago 9h 1203M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:25:44.220 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 6m ago 9h 1508M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:25:44.220 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 6m ago 9h 1348M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:25:44.220 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 6m ago 9h 1602M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:25:44.220 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 6m ago 9h 192M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:25:44.573 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:25:44.573 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:25:44.574 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:25:44.574 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:25:44.574 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:25:44.574 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:25:44.575 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:25:44.575 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:25:44.575 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:25:44.575 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:25:44.576 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:25:44.576 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:25:44.576 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:25:44.576 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:25:44.576 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:25:44.577 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:25:44.577 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:25:44.577 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:25:45.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:25:44 smithi083 ceph-mon[79930]: pgmap v16122: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:45.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:25:44 smithi083 ceph-mon[79930]: from='client.52878 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:25:45.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:25:44 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3414073137' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:25:45.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:25:44 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:25:45.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:25:44 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:25:45.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:25:44 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:25:45.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:25:44 smithi083 ceph-mon[77242]: pgmap v16122: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:45.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:25:44 smithi083 ceph-mon[77242]: from='client.52878 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:25:45.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:25:44 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3414073137' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:25:45.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:25:44 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:25:45.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:25:44 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:25:45.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:25:44 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:25:45.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:25:44 smithi153 ceph-mon[70044]: pgmap v16122: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:45.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:25:44 smithi153 ceph-mon[70044]: from='client.52878 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:25:45.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:25:44 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3414073137' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:25:45.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:25:44 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:25:45.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:25:44 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:25:45.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:25:44 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:25:45.923 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:25:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:25:45.535Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:25:46.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:25:45 smithi083 ceph-mon[77242]: from='client.62815 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:25:46.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:25:45 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:25:46.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:25:45 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:25:46.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:25:45 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:25:46.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:25:45 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:25:46.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:25:45 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:25:46.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:25:45 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:25:46.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:25:45 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:25:46.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:25:45 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:25:46.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:25:45 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:25:46.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:25:45 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:25:46.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:25:45 smithi083 ceph-mon[79930]: from='client.62815 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:25:46.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:25:45 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:25:46.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:25:45 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:25:46.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:25:45 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:25:46.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:25:45 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:25:46.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:25:45 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:25:46.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:25:45 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:25:46.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:25:45 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:25:46.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:25:45 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:25:46.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:25:45 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:25:46.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:25:45 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:25:46.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:25:45 smithi153 ceph-mon[70044]: from='client.62815 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:25:46.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:25:45 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:25:46.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:25:45 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:25:46.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:25:45 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:25:46.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:25:45 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:25:46.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:25:45 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:25:46.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:25:45 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:25:46.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:25:45 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:25:46.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:25:45 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:25:46.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:25:45 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:25:46.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:25:45 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:25:46.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:25:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:25:46.268Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:25:47.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:25:46 smithi083 ceph-mon[77242]: Adjusting osd_memory_target on smithi083 to 2234M 2022-09-29T14:25:47.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:25:46 smithi083 ceph-mon[77242]: pgmap v16123: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:47.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:25:46 smithi083 ceph-mon[79930]: Adjusting osd_memory_target on smithi083 to 2234M 2022-09-29T14:25:47.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:25:46 smithi083 ceph-mon[79930]: pgmap v16123: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:47.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:25:46 smithi153 ceph-mon[70044]: Adjusting osd_memory_target on smithi083 to 2234M 2022-09-29T14:25:47.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:25:46 smithi153 ceph-mon[70044]: pgmap v16123: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:49.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:25:48 smithi083 ceph-mon[79930]: pgmap v16124: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:49.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:25:48 smithi083 ceph-mon[77242]: pgmap v16124: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:49.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:25:48 smithi153 ceph-mon[70044]: pgmap v16124: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:51.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:25:50 smithi083 ceph-mon[77242]: pgmap v16125: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:51.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:25:50 smithi083 ceph-mon[79930]: pgmap v16125: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:51.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:25:50 smithi153 ceph-mon[70044]: pgmap v16125: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:25:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:25:51] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T14:25:52.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:25:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:25:52.171Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:25:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:25:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:25:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:25:53.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:25:52 smithi083 ceph-mon[77242]: pgmap v16126: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:53.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:25:52 smithi083 ceph-mon[79930]: pgmap v16126: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:53.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:25:52 smithi153 ceph-mon[70044]: pgmap v16126: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:25:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:25:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:25:55.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:25:54 smithi083 ceph-mon[79930]: pgmap v16127: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:55.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:25:54 smithi083 ceph-mon[77242]: pgmap v16127: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:55.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:25:54 smithi153 ceph-mon[70044]: pgmap v16127: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:55.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:25:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:25:55.536Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:25:56.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:25:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:25:56.269Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:25:57.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:25:56 smithi083 ceph-mon[77242]: pgmap v16128: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:57.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:25:56 smithi083 ceph-mon[79930]: pgmap v16128: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:57.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:25:56 smithi153 ceph-mon[70044]: pgmap v16128: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:59.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:25:58 smithi083 ceph-mon[79930]: pgmap v16129: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:59.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:25:58 smithi083 ceph-mon[77242]: pgmap v16129: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:25:59.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:25:58 smithi153 ceph-mon[70044]: pgmap v16129: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:01.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:26:00 smithi083 ceph-mon[79930]: pgmap v16130: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:01.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:26:00 smithi083 ceph-mon[77242]: pgmap v16130: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:01.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:26:00 smithi153 ceph-mon[70044]: pgmap v16130: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:26:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:26:01] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T14:26:02.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:26:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:26:02.172Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:26:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:26:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:26:03.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:26:02 smithi083 ceph-mon[79930]: pgmap v16131: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:03.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:26:02 smithi083 ceph-mon[77242]: pgmap v16131: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:03.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:26:02 smithi153 ceph-mon[70044]: pgmap v16131: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:26:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:26:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:26:05.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:26:04 smithi083 ceph-mon[77242]: pgmap v16132: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:05.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:26:04 smithi083 ceph-mon[79930]: pgmap v16132: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:05.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:26:04 smithi153 ceph-mon[70044]: pgmap v16132: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:05.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:26:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:26:05.536Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:26:06.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:26:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:26:06.270Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:26:07.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:26:06 smithi083 ceph-mon[79930]: pgmap v16133: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:07.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:26:06 smithi083 ceph-mon[77242]: pgmap v16133: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:07.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:26:06 smithi153 ceph-mon[70044]: pgmap v16133: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:09.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:26:08 smithi083 ceph-mon[79930]: pgmap v16134: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:09.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:26:08 smithi083 ceph-mon[77242]: pgmap v16134: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:09.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:26:08 smithi153 ceph-mon[70044]: pgmap v16134: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:11.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:26:10 smithi153 ceph-mon[70044]: pgmap v16135: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:11.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:26:10 smithi083 ceph-mon[79930]: pgmap v16135: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:11.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:26:10 smithi083 ceph-mon[77242]: pgmap v16135: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:26:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:26:11] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T14:26:12.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:26:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:26:12.172Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:26:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:26:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:26:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:26:13.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:26:12 smithi153 ceph-mon[70044]: pgmap v16136: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:26:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:26:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:26:13.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:26:12 smithi083 ceph-mon[77242]: pgmap v16136: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:13.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:26:12 smithi083 ceph-mon[79930]: pgmap v16136: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:14.900 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:26:15.190 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:26:15.190 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (8h) 8m ago 9h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:26:15.190 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 6m ago 9h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:26:15.191 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 8m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:26:15.191 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 6m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:26:15.191 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 6m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:26:15.191 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 8m ago 9h 951M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:26:15.192 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 8m ago 9h 1948M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:26:15.192 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 6m ago 9h 847M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:26:15.192 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 8m ago 9h 1163M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:26:15.192 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 8m ago 9h 23.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:26:15.193 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 6m ago 9h 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:26:15.193 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 8m ago 9h 1412M 2234M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:26:15.193 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 8m ago 9h 1398M 2234M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:26:15.193 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 8m ago 9h 1088M 2234M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:26:15.194 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 8m ago 9h 1371M 2234M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:26:15.194 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 6m ago 9h 1203M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:26:15.194 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 6m ago 9h 1508M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:26:15.194 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 6m ago 9h 1348M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:26:15.194 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 6m ago 9h 1602M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:26:15.195 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 6m ago 9h 192M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:26:15.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:26:14 smithi153 ceph-mon[70044]: pgmap v16137: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:15.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:26:14 smithi083 ceph-mon[79930]: pgmap v16137: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:15.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:26:14 smithi083 ceph-mon[77242]: pgmap v16137: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:15.543 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:26:15.544 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:26:15.544 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:26:15.544 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:26:15.544 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:26:15.545 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:26:15.545 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:26:15.545 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:26:15.545 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:26:15.546 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:26:15.546 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:26:15.546 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:26:15.546 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:26:15.546 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:26:15.547 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:26:15.547 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:26:15.547 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:26:15.547 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:26:15.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:26:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:26:15.536Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:26:16.271 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:26:15 smithi083 ceph-mon[79930]: from='client.62827 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:26:16.272 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:26:15 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2249375034' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:26:16.272 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:26:15 smithi083 ceph-mon[77242]: from='client.62827 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:26:16.272 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:26:15 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2249375034' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:26:16.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:26:15 smithi153 ceph-mon[70044]: from='client.62827 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:26:16.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:26:15 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2249375034' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:26:16.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:26:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:26:16.269Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:26:17.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:26:16 smithi153 ceph-mon[70044]: from='client.52902 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:26:17.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:26:16 smithi153 ceph-mon[70044]: pgmap v16138: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:17.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:26:16 smithi083 ceph-mon[77242]: from='client.52902 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:26:17.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:26:16 smithi083 ceph-mon[77242]: pgmap v16138: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:17.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:26:16 smithi083 ceph-mon[79930]: from='client.52902 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:26:17.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:26:16 smithi083 ceph-mon[79930]: pgmap v16138: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:19.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:26:18 smithi153 ceph-mon[70044]: pgmap v16139: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:19.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:26:18 smithi083 ceph-mon[79930]: pgmap v16139: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:19.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:26:18 smithi083 ceph-mon[77242]: pgmap v16139: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:21.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:26:20 smithi153 ceph-mon[70044]: pgmap v16140: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:21.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:26:20 smithi083 ceph-mon[79930]: pgmap v16140: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:21.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:26:20 smithi083 ceph-mon[77242]: pgmap v16140: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:26:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:26:21] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T14:26:22.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:26:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:26:22.173Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:26:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:26:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:26:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:26:23.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:26:22 smithi153 ceph-mon[70044]: pgmap v16141: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:26:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:26:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:26:23.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:26:22 smithi083 ceph-mon[79930]: pgmap v16141: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:23.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:26:22 smithi083 ceph-mon[77242]: pgmap v16141: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:25.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:26:24 smithi153 ceph-mon[70044]: pgmap v16142: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:25.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:26:24 smithi083 ceph-mon[79930]: pgmap v16142: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:25.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:26:24 smithi083 ceph-mon[77242]: pgmap v16142: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:25.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:26:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:26:25.537Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:26:26.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:26:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:26:26.271Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:26:27.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:26:26 smithi153 ceph-mon[70044]: pgmap v16143: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:27.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:26:26 smithi083 ceph-mon[77242]: pgmap v16143: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:27.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:26:26 smithi083 ceph-mon[79930]: pgmap v16143: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:29.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:26:28 smithi153 ceph-mon[70044]: pgmap v16144: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:29.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:26:28 smithi083 ceph-mon[79930]: pgmap v16144: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:29.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:26:28 smithi083 ceph-mon[77242]: pgmap v16144: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:31.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:26:30 smithi153 ceph-mon[70044]: pgmap v16145: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:31.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:26:30 smithi083 ceph-mon[77242]: pgmap v16145: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:31.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:26:30 smithi083 ceph-mon[79930]: pgmap v16145: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:26:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:26:31] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T14:26:32.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:26:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:26:32.174Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:26:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:26:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:26:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:26:33.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:26:32 smithi153 ceph-mon[70044]: pgmap v16146: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:26:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:26:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:26:33.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:26:32 smithi083 ceph-mon[79930]: pgmap v16146: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:33.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:26:32 smithi083 ceph-mon[77242]: pgmap v16146: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:35.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:26:35 smithi153 ceph-mon[70044]: pgmap v16147: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:35.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:26:35 smithi083 ceph-mon[79930]: pgmap v16147: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:35.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:26:35 smithi083 ceph-mon[77242]: pgmap v16147: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:35.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:26:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:26:35.538Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:26:36.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:26:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:26:36.271Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:26:37.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:26:37 smithi153 ceph-mon[70044]: pgmap v16148: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:37.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:26:37 smithi083 ceph-mon[79930]: pgmap v16148: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:37.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:26:37 smithi083 ceph-mon[77242]: pgmap v16148: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:39.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:26:39 smithi153 ceph-mon[70044]: pgmap v16149: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:39.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:26:39 smithi083 ceph-mon[77242]: pgmap v16149: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:39.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:26:39 smithi083 ceph-mon[79930]: pgmap v16149: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:26:41 smithi153 ceph-mon[70044]: pgmap v16150: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:41.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:26:41 smithi083 ceph-mon[77242]: pgmap v16150: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:41.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:26:41 smithi083 ceph-mon[79930]: pgmap v16150: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:26:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:26:41] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T14:26:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:26:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:26:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:26:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:26:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:26:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:26:42.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:26:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:26:42.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:26:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:26:42.174Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:26:42.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:26:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:26:42.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:26:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:26:42.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:26:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:26:42.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:26:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:26:42.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:26:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:26:42.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:26:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:26:42.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:26:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:26:42.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:26:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:26:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:26:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:26:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:26:43.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:26:43 smithi153 ceph-mon[70044]: pgmap v16151: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:26:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:26:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:26:43.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:26:43 smithi083 ceph-mon[77242]: pgmap v16151: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:43.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:26:43 smithi083 ceph-mon[79930]: pgmap v16151: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:45.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:26:45 smithi153 ceph-mon[70044]: pgmap v16152: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:45.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:26:45 smithi083 ceph-mon[79930]: pgmap v16152: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:45.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:26:45 smithi083 ceph-mon[77242]: pgmap v16152: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:45.907 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:26:45.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:26:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:26:45.538Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:26:46.206 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:26:46.206 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 9m ago 9h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:26:46.206 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 7m ago 9h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:26:46.207 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 9m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:26:46.207 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 7m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:26:46.207 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 7m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:26:46.207 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 9m ago 9h 951M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:26:46.207 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 9m ago 9h 1948M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:26:46.208 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 7m ago 9h 847M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:26:46.208 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 9m ago 9h 1163M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:26:46.208 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 9m ago 9h 23.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:26:46.208 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 7m ago 9h 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:26:46.208 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 9m ago 9h 1412M 2234M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:26:46.209 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 9m ago 9h 1398M 2234M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:26:46.209 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 9m ago 9h 1088M 2234M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:26:46.209 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 9m ago 9h 1371M 2234M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:26:46.209 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 7m ago 9h 1203M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:26:46.209 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 7m ago 9h 1508M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:26:46.210 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 7m ago 9h 1348M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:26:46.210 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 7m ago 9h 1602M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:26:46.210 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 7m ago 9h 192M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:26:46.275 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:26:46 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:26:46.275 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:26:46 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:26:46.275 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:26:46 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:26:46.275 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:26:46 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:26:46.276 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:26:46 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:26:46.276 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:26:46 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:26:46.276 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:26:46 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:26:46.276 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:26:46 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:26:46.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:26:46 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:26:46.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:26:46 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:26:46.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:26:46 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:26:46.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:26:46 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:26:46.562 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:26:46.563 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:26:46.563 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:26:46.563 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:26:46.564 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:26:46.564 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:26:46.564 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:26:46.564 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:26:46.565 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:26:46.565 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:26:46.565 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:26:46.565 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:26:46.565 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:26:46.566 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:26:46.566 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:26:46.566 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:26:46.566 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:26:46.566 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:26:46.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:26:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:26:46.273Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:26:47.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:26:47 smithi153 ceph-mon[70044]: pgmap v16153: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:47.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:26:47 smithi153 ceph-mon[70044]: from='client.52914 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:26:47.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:26:47 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/380450423' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:26:47.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:26:47 smithi083 ceph-mon[79930]: pgmap v16153: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:47.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:26:47 smithi083 ceph-mon[79930]: from='client.52914 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:26:47.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:26:47 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/380450423' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:26:47.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:26:47 smithi083 ceph-mon[77242]: pgmap v16153: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:47.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:26:47 smithi083 ceph-mon[77242]: from='client.52914 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:26:47.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:26:47 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/380450423' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:26:48.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:26:48 smithi153 ceph-mon[70044]: from='client.52920 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:26:48.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:26:48 smithi083 ceph-mon[79930]: from='client.52920 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:26:48.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:26:48 smithi083 ceph-mon[77242]: from='client.52920 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:26:49.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:26:49 smithi153 ceph-mon[70044]: pgmap v16154: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:49.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:26:49 smithi083 ceph-mon[79930]: pgmap v16154: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:49.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:26:49 smithi083 ceph-mon[77242]: pgmap v16154: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:50.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:26:50 smithi153 ceph-mon[70044]: pgmap v16155: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:50.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:26:50 smithi083 ceph-mon[79930]: pgmap v16155: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:50.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:26:50 smithi083 ceph-mon[77242]: pgmap v16155: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:26:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:26:51] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T14:26:52.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:26:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:26:52.174Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:26:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:26:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:26:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:26:53.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:26:52 smithi083 ceph-mon[79930]: pgmap v16156: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:53.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:26:52 smithi083 ceph-mon[77242]: pgmap v16156: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:53.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:26:52 smithi153 ceph-mon[70044]: pgmap v16156: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:26:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:26:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:26:55.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:26:54 smithi083 ceph-mon[79930]: pgmap v16157: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:55.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:26:54 smithi083 ceph-mon[77242]: pgmap v16157: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:55.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:26:54 smithi153 ceph-mon[70044]: pgmap v16157: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:55.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:26:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:26:55.539Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:26:56.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:26:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:26:56.272Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:26:57.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:26:56 smithi083 ceph-mon[79930]: pgmap v16158: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:57.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:26:56 smithi083 ceph-mon[77242]: pgmap v16158: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:57.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:26:56 smithi153 ceph-mon[70044]: pgmap v16158: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:59.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:26:58 smithi083 ceph-mon[79930]: pgmap v16159: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:59.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:26:58 smithi083 ceph-mon[77242]: pgmap v16159: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:26:59.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:26:58 smithi153 ceph-mon[70044]: pgmap v16159: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:01.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:27:00 smithi083 ceph-mon[77242]: pgmap v16160: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:01.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:27:00 smithi083 ceph-mon[79930]: pgmap v16160: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:01.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:27:00 smithi153 ceph-mon[70044]: pgmap v16160: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:27:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:27:01] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T14:27:02.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:27:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:27:02.175Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:27:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:27:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:27:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:27:03.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:27:02 smithi083 ceph-mon[77242]: pgmap v16161: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:03.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:27:02 smithi083 ceph-mon[79930]: pgmap v16161: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:03.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:27:02 smithi153 ceph-mon[70044]: pgmap v16161: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:27:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:27:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:27:05.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:27:04 smithi083 ceph-mon[79930]: pgmap v16162: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:05.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:27:04 smithi083 ceph-mon[77242]: pgmap v16162: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:05.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:27:04 smithi153 ceph-mon[70044]: pgmap v16162: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:05.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:27:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:27:05.539Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:27:06.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:27:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:27:06.273Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:27:07.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:27:06 smithi083 ceph-mon[77242]: pgmap v16163: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:07.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:27:06 smithi083 ceph-mon[79930]: pgmap v16163: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:07.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:27:06 smithi153 ceph-mon[70044]: pgmap v16163: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:09.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:27:08 smithi083 ceph-mon[79930]: pgmap v16164: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:09.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:27:08 smithi083 ceph-mon[77242]: pgmap v16164: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:09.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:27:08 smithi153 ceph-mon[70044]: pgmap v16164: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:11.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:27:10 smithi083 ceph-mon[79930]: pgmap v16165: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:11.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:27:10 smithi083 ceph-mon[77242]: pgmap v16165: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:11.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:27:10 smithi153 ceph-mon[70044]: pgmap v16165: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:27:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:27:11] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T14:27:12.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:27:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:27:12.175Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:27:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:27:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:27:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:27:13.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:27:12 smithi083 ceph-mon[79930]: pgmap v16166: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:13.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:27:12 smithi083 ceph-mon[77242]: pgmap v16166: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:13.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:27:12 smithi153 ceph-mon[70044]: pgmap v16166: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:27:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:27:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:27:15.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:27:14 smithi083 ceph-mon[79930]: pgmap v16167: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:15.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:27:14 smithi083 ceph-mon[77242]: pgmap v16167: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:15.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:27:14 smithi153 ceph-mon[70044]: pgmap v16167: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:15.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:27:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:27:15.540Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:27:16.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:27:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:27:16.273Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:27:16.890 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:27:17.177 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:27:17.178 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 9m ago 9h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:27:17.178 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 7m ago 9h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:27:17.178 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 9m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:27:17.179 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 7m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:27:17.179 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 7m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:27:17.179 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 9m ago 9h 951M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:27:17.179 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 9m ago 9h 1948M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:27:17.179 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 7m ago 9h 847M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:27:17.180 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 9m ago 9h 1163M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:27:17.180 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 9m ago 9h 23.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:27:17.180 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 7m ago 9h 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:27:17.180 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 9m ago 9h 1412M 2234M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:27:17.180 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 9m ago 9h 1398M 2234M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:27:17.181 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 9m ago 9h 1088M 2234M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:27:17.181 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 9m ago 9h 1371M 2234M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:27:17.181 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 7m ago 9h 1203M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:27:17.181 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 7m ago 9h 1508M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:27:17.182 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 7m ago 9h 1348M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:27:17.182 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 7m ago 9h 1602M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:27:17.182 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 7m ago 9h 192M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:27:17.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:27:16 smithi083 ceph-mon[79930]: pgmap v16168: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:17.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:27:16 smithi083 ceph-mon[77242]: pgmap v16168: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:17.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:27:16 smithi153 ceph-mon[70044]: pgmap v16168: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:17.530 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:27:17.531 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:27:17.531 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:27:17.531 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:27:17.531 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:27:17.532 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:27:17.532 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:27:17.532 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:27:17.532 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:27:17.533 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:27:17.533 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:27:17.533 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:27:17.533 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:27:17.533 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:27:17.534 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:27:17.534 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:27:17.534 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:27:17.534 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:27:18.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:27:17 smithi083 ceph-mon[77242]: from='client.52932 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:27:18.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:27:17 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1178949118' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:27:18.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:27:17 smithi083 ceph-mon[79930]: from='client.52932 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:27:18.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:27:17 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1178949118' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:27:18.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:27:17 smithi153 ceph-mon[70044]: from='client.52932 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:27:18.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:27:17 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1178949118' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:27:19.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:27:18 smithi083 ceph-mon[79930]: from='client.62869 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:27:19.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:27:18 smithi083 ceph-mon[79930]: pgmap v16169: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:19.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:27:18 smithi083 ceph-mon[77242]: from='client.62869 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:27:19.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:27:18 smithi083 ceph-mon[77242]: pgmap v16169: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:19.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:27:18 smithi153 ceph-mon[70044]: from='client.62869 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:27:19.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:27:18 smithi153 ceph-mon[70044]: pgmap v16169: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:21.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:27:20 smithi083 ceph-mon[79930]: pgmap v16170: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:21.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:27:20 smithi083 ceph-mon[77242]: pgmap v16170: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:21.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:27:20 smithi153 ceph-mon[70044]: pgmap v16170: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:27:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:27:21] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T14:27:22.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:27:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:27:22.176Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:27:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:27:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:27:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:27:23.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:27:22 smithi083 ceph-mon[79930]: pgmap v16171: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:23.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:27:22 smithi083 ceph-mon[77242]: pgmap v16171: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:23.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:27:22 smithi153 ceph-mon[70044]: pgmap v16171: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:27:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:27:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:27:25.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:27:24 smithi083 ceph-mon[79930]: pgmap v16172: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:25.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:27:24 smithi083 ceph-mon[77242]: pgmap v16172: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:25.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:27:24 smithi153 ceph-mon[70044]: pgmap v16172: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:25.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:27:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:27:25.541Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:27:26.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:27:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:27:26.273Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:27:27.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:27:26 smithi083 ceph-mon[79930]: pgmap v16173: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:27.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:27:26 smithi083 ceph-mon[77242]: pgmap v16173: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:27.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:27:26 smithi153 ceph-mon[70044]: pgmap v16173: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:29.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:27:28 smithi083 ceph-mon[79930]: pgmap v16174: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:29.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:27:28 smithi083 ceph-mon[77242]: pgmap v16174: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:29.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:27:28 smithi153 ceph-mon[70044]: pgmap v16174: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:31.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:27:30 smithi083 ceph-mon[79930]: pgmap v16175: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:31.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:27:30 smithi083 ceph-mon[77242]: pgmap v16175: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:31.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:27:30 smithi153 ceph-mon[70044]: pgmap v16175: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:27:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:27:31] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T14:27:32.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:27:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:27:32.176Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:27:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:27:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:27:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:27:33.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:27:32 smithi083 ceph-mon[79930]: pgmap v16176: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:33.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:27:32 smithi083 ceph-mon[77242]: pgmap v16176: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:27:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:27:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:27:33.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:27:32 smithi153 ceph-mon[70044]: pgmap v16176: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:35.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:27:34 smithi083 ceph-mon[77242]: pgmap v16177: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:35.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:27:34 smithi083 ceph-mon[79930]: pgmap v16177: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:35.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:27:34 smithi153 ceph-mon[70044]: pgmap v16177: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:35.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:27:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:27:35.541Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:27:36.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:27:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:27:36.274Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:27:37.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:27:36 smithi083 ceph-mon[79930]: pgmap v16178: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:37.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:27:36 smithi083 ceph-mon[77242]: pgmap v16178: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:37.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:27:36 smithi153 ceph-mon[70044]: pgmap v16178: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:39.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:27:38 smithi083 ceph-mon[77242]: pgmap v16179: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:39.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:27:38 smithi083 ceph-mon[79930]: pgmap v16179: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:39.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:27:38 smithi153 ceph-mon[70044]: pgmap v16179: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:41.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:27:40 smithi083 ceph-mon[79930]: pgmap v16180: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:27:40 smithi083 ceph-mon[77242]: pgmap v16180: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:27:40 smithi153 ceph-mon[70044]: pgmap v16180: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:41.906 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:27:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:27:41] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T14:27:42.180 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:27:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:27:42.180 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:27:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:27:42.180 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:27:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:27:42.180 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:27:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:27:42.181 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:27:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:27:42.181 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:27:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:27:42.181 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:27:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:27:42.182 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:27:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:27:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:27:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:27:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:27:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:27:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:27:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:27:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:27:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:27:42.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:27:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:27:42.177Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:27:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:27:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:27:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:27:43.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:27:42 smithi083 ceph-mon[79930]: pgmap v16181: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:43.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:27:42 smithi083 ceph-mon[77242]: pgmap v16181: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:43.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:27:42 smithi153 ceph-mon[70044]: pgmap v16181: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:27:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:27:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:27:45.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:27:44 smithi083 ceph-mon[79930]: pgmap v16182: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:45.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:27:44 smithi083 ceph-mon[77242]: pgmap v16182: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:45.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:27:44 smithi153 ceph-mon[70044]: pgmap v16182: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:45.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:27:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:27:45.542Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:27:46.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:27:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:27:46.274Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:27:47.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:27:46 smithi083 ceph-mon[79930]: pgmap v16183: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:47.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:27:46 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:27:47.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:27:46 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:27:47.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:27:46 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:27:47.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:27:46 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:27:47.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:27:46 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:27:47.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:27:46 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:27:47.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:27:46 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:27:47.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:27:46 smithi083 ceph-mon[77242]: pgmap v16183: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:47.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:27:46 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:27:47.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:27:46 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:27:47.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:27:46 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:27:47.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:27:46 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:27:47.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:27:46 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:27:47.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:27:46 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:27:47.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:27:46 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:27:47.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:27:46 smithi153 ceph-mon[70044]: pgmap v16183: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:47.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:27:46 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:27:47.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:27:46 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:27:47.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:27:46 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:27:47.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:27:46 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:27:47.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:27:46 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:27:47.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:27:46 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:27:47.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:27:46 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:27:47.860 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:27:48.170 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:27:48.171 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 10m ago 9h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:27:48.171 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 8m ago 9h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:27:48.171 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 10m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:27:48.172 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 8m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:27:48.172 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 8m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:27:48.172 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 10m ago 9h 951M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:27:48.172 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 10m ago 9h 1948M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:27:48.173 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 8m ago 9h 847M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:27:48.173 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 10m ago 9h 1163M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:27:48.173 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 10m ago 9h 23.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:27:48.173 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 8m ago 9h 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:27:48.174 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 10m ago 9h 1412M 2234M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:27:48.174 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 10m ago 9h 1398M 2234M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:27:48.174 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 10m ago 9h 1088M 2234M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:27:48.174 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 10m ago 9h 1371M 2234M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:27:48.174 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 8m ago 9h 1203M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:27:48.175 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 8m ago 9h 1508M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:27:48.175 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 8m ago 9h 1348M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:27:48.175 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 8m ago 9h 1602M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:27:48.175 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 8m ago 9h 192M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:27:48.548 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:27:48.548 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:27:48.548 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:27:48.549 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:27:48.549 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:27:48.549 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:27:48.550 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:27:48.550 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:27:48.550 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:27:48.550 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:27:48.551 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:27:48.551 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:27:48.551 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:27:48.551 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:27:48.552 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:27:48.552 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:27:48.552 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:27:48.552 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:27:49.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:27:48 smithi083 ceph-mon[79930]: pgmap v16184: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:49.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:27:48 smithi083 ceph-mon[79930]: from='client.62881 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:27:49.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:27:48 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3685301263' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:27:49.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:27:48 smithi083 ceph-mon[77242]: pgmap v16184: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:49.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:27:48 smithi083 ceph-mon[77242]: from='client.62881 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:27:49.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:27:48 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3685301263' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:27:49.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:27:48 smithi153 ceph-mon[70044]: pgmap v16184: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:49.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:27:48 smithi153 ceph-mon[70044]: from='client.62881 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:27:49.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:27:48 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3685301263' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:27:50.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:27:49 smithi083 ceph-mon[79930]: from='client.52956 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:27:50.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:27:49 smithi083 ceph-mon[77242]: from='client.52956 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:27:50.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:27:49 smithi153 ceph-mon[70044]: from='client.52956 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:27:51.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:27:50 smithi083 ceph-mon[79930]: pgmap v16185: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:51.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:27:50 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:27:51.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:27:50 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:27:51.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:27:50 smithi083 ceph-mon[77242]: pgmap v16185: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:51.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:27:50 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:27:51.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:27:50 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:27:51.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:27:50 smithi153 ceph-mon[70044]: pgmap v16185: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:51.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:27:50 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:27:51.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:27:50 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:27:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:27:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:27:51] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T14:27:52.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:27:52 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:27:52.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:27:52 smithi083 ceph-mon[79930]: pgmap v16186: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:52.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:27:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:27:52.178Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:27:52.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:27:52 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:27:52.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:27:52 smithi083 ceph-mon[77242]: pgmap v16186: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:52.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:27:52 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:27:52.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:27:52 smithi153 ceph-mon[70044]: pgmap v16186: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:52.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:27:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:27:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:27:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:27:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:27:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:27:54.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:27:54 smithi083 ceph-mon[79930]: pgmap v16187: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:54.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:27:54 smithi083 ceph-mon[77242]: pgmap v16187: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:55.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:27:54 smithi153 ceph-mon[70044]: pgmap v16187: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:55.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:27:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:27:55.542Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:27:56.644 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:27:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:27:56.276Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:27:56.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:27:56 smithi083 ceph-mon[79930]: pgmap v16188: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:56.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:27:56 smithi083 ceph-mon[77242]: pgmap v16188: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:57.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:27:56 smithi153 ceph-mon[70044]: pgmap v16188: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:59.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:27:58 smithi083 ceph-mon[79930]: pgmap v16189: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:59.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:27:58 smithi083 ceph-mon[77242]: pgmap v16189: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:27:59.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:27:58 smithi153 ceph-mon[70044]: pgmap v16189: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:01.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:28:00 smithi083 ceph-mon[79930]: pgmap v16190: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:01.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:28:00 smithi083 ceph-mon[77242]: pgmap v16190: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:01.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:28:00 smithi153 ceph-mon[70044]: pgmap v16190: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:28:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:28:01] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T14:28:02.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:28:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:28:02.179Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:28:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:28:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:28:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:28:03.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:28:02 smithi083 ceph-mon[79930]: pgmap v16191: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:03.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:28:02 smithi083 ceph-mon[77242]: pgmap v16191: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:28:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:28:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:28:03.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:28:02 smithi153 ceph-mon[70044]: pgmap v16191: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:05.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:28:04 smithi083 ceph-mon[79930]: pgmap v16192: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:05.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:28:04 smithi083 ceph-mon[77242]: pgmap v16192: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:05.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:28:04 smithi153 ceph-mon[70044]: pgmap v16192: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:05.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:28:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:28:05.543Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:28:06.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:28:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:28:06.277Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:28:07.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:28:06 smithi083 ceph-mon[79930]: pgmap v16193: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:07.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:28:06 smithi083 ceph-mon[77242]: pgmap v16193: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:07.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:28:06 smithi153 ceph-mon[70044]: pgmap v16193: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:09.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:28:08 smithi083 ceph-mon[79930]: pgmap v16194: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:09.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:28:08 smithi083 ceph-mon[77242]: pgmap v16194: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:09.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:28:08 smithi153 ceph-mon[70044]: pgmap v16194: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:11.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:28:10 smithi083 ceph-mon[79930]: pgmap v16195: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:11.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:28:10 smithi083 ceph-mon[77242]: pgmap v16195: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:11.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:28:10 smithi153 ceph-mon[70044]: pgmap v16195: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:28:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:28:11] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T14:28:12.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:28:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:28:12.180Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:28:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:28:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:28:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:28:13.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:28:12 smithi083 ceph-mon[77242]: pgmap v16196: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:13.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:28:12 smithi083 ceph-mon[79930]: pgmap v16196: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:28:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:28:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:28:13.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:28:12 smithi153 ceph-mon[70044]: pgmap v16196: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:15.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:28:14 smithi083 ceph-mon[77242]: pgmap v16197: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:15.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:28:14 smithi083 ceph-mon[79930]: pgmap v16197: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:15.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:28:14 smithi153 ceph-mon[70044]: pgmap v16197: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:15.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:28:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:28:15.544Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:28:16.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:28:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:28:16.277Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:28:17.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:28:16 smithi083 ceph-mon[79930]: pgmap v16198: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:17.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:28:16 smithi083 ceph-mon[77242]: pgmap v16198: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:17.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:28:16 smithi153 ceph-mon[70044]: pgmap v16198: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:18.874 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:28:19.162 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:28:19.162 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 28s ago 9h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:28:19.162 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 8m ago 9h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:28:19.163 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 28s ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:28:19.163 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 8m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:28:19.163 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 8m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:28:19.163 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (8h) 28s ago 9h 959M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:28:19.164 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 28s ago 9h 1984M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:28:19.164 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 8m ago 9h 847M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:28:19.164 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 28s ago 9h 1185M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:28:19.164 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 28s ago 9h 24.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:28:19.164 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 8m ago 9h 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:28:19.165 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 28s ago 9h 1438M 2234M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:28:19.165 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 28s ago 9h 1425M 2234M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:28:19.165 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 28s ago 9h 1107M 2234M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:28:19.165 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 28s ago 9h 1396M 2234M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:28:19.166 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 8m ago 9h 1203M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:28:19.166 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 8m ago 9h 1508M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:28:19.166 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 8m ago 9h 1348M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:28:19.166 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 8m ago 9h 1602M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:28:19.167 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 8m ago 9h 192M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:28:19.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:28:18 smithi083 ceph-mon[79930]: pgmap v16199: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:19.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:28:18 smithi083 ceph-mon[77242]: pgmap v16199: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:19.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:28:18 smithi153 ceph-mon[70044]: pgmap v16199: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:19.521 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:28:19.521 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:28:19.522 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:28:19.522 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:28:19.522 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:28:19.522 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:28:19.523 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:28:19.523 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:28:19.523 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:28:19.523 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:28:19.524 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:28:19.524 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:28:19.524 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:28:19.524 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:28:19.524 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:28:19.525 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:28:19.525 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:28:19.525 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:28:20.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:28:19 smithi083 ceph-mon[79930]: from='client.62899 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:28:20.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:28:19 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1002559414' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:28:20.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:28:19 smithi083 ceph-mon[77242]: from='client.62899 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:28:20.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:28:19 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1002559414' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:28:20.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:28:19 smithi153 ceph-mon[70044]: from='client.62899 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:28:20.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:28:19 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1002559414' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:28:21.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:28:20 smithi083 ceph-mon[79930]: from='client.62905 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:28:21.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:28:20 smithi083 ceph-mon[79930]: pgmap v16200: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:21.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:28:20 smithi083 ceph-mon[77242]: from='client.62905 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:28:21.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:28:20 smithi083 ceph-mon[77242]: pgmap v16200: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:21.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:28:20 smithi153 ceph-mon[70044]: from='client.62905 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:28:21.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:28:20 smithi153 ceph-mon[70044]: pgmap v16200: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:28:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:28:21] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T14:28:22.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:28:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:28:22.180Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:28:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:28:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:28:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:28:23.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:28:22 smithi083 ceph-mon[77242]: pgmap v16201: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:23.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:28:22 smithi083 ceph-mon[79930]: pgmap v16201: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:23.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:28:22 smithi153 ceph-mon[70044]: pgmap v16201: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:28:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:28:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:28:25.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:28:24 smithi083 ceph-mon[77242]: pgmap v16202: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:25.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:28:24 smithi083 ceph-mon[79930]: pgmap v16202: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:25.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:28:24 smithi153 ceph-mon[70044]: pgmap v16202: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:25.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:28:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:28:25.544Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:28:26.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:28:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:28:26.277Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:28:27.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:28:26 smithi083 ceph-mon[79930]: pgmap v16203: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:27.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:28:26 smithi083 ceph-mon[77242]: pgmap v16203: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:27.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:28:26 smithi153 ceph-mon[70044]: pgmap v16203: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:29.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:28:28 smithi083 ceph-mon[79930]: pgmap v16204: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:29.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:28:28 smithi083 ceph-mon[77242]: pgmap v16204: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:29.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:28:28 smithi153 ceph-mon[70044]: pgmap v16204: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:31.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:28:30 smithi083 ceph-mon[79930]: pgmap v16205: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:31.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:28:30 smithi083 ceph-mon[77242]: pgmap v16205: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:31.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:28:30 smithi153 ceph-mon[70044]: pgmap v16205: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:28:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:28:31] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T14:28:32.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:28:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:28:32.181Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:28:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:28:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:28:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:28:33.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:28:32 smithi083 ceph-mon[77242]: pgmap v16206: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:33.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:28:32 smithi083 ceph-mon[79930]: pgmap v16206: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:33.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:28:32 smithi153 ceph-mon[70044]: pgmap v16206: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:28:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:28:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:28:35.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:28:34 smithi083 ceph-mon[79930]: pgmap v16207: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:35.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:28:34 smithi083 ceph-mon[77242]: pgmap v16207: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:35.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:28:34 smithi153 ceph-mon[70044]: pgmap v16207: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:35.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:28:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:28:35.545Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:28:36.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:28:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:28:36.278Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:28:37.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:28:36 smithi083 ceph-mon[79930]: pgmap v16208: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:37.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:28:36 smithi083 ceph-mon[77242]: pgmap v16208: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:37.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:28:36 smithi153 ceph-mon[70044]: pgmap v16208: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:39.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:28:38 smithi083 ceph-mon[79930]: pgmap v16209: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:39.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:28:38 smithi083 ceph-mon[77242]: pgmap v16209: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:39.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:28:38 smithi153 ceph-mon[70044]: pgmap v16209: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:41.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:28:40 smithi083 ceph-mon[79930]: pgmap v16210: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:28:40 smithi083 ceph-mon[77242]: pgmap v16210: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:28:40 smithi153 ceph-mon[70044]: pgmap v16210: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:41.906 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:28:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:28:41] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T14:28:41.906 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:28:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:28:41.907 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:28:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:28:41.907 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:28:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:28:41.907 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:28:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:28:42.184 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:28:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:28:42.184 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:28:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:28:42.185 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:28:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:28:42.185 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:28:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:28:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:28:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:28:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:28:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:28:42.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:28:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:28:42.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:28:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:28:42.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:28:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:28:42.182Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:28:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:28:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:28:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:28:43.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:28:42 smithi083 ceph-mon[77242]: pgmap v16211: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:43.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:28:42 smithi083 ceph-mon[79930]: pgmap v16211: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:43.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:28:42 smithi153 ceph-mon[70044]: pgmap v16211: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:28:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:28:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:28:45.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:28:44 smithi083 ceph-mon[79930]: pgmap v16212: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:45.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:28:44 smithi083 ceph-mon[77242]: pgmap v16212: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:45.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:28:44 smithi153 ceph-mon[70044]: pgmap v16212: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:45.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:28:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:28:45.546Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:28:46.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:28:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:28:46.279Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:28:47.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:28:46 smithi083 ceph-mon[79930]: pgmap v16213: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:47.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:28:46 smithi083 ceph-mon[77242]: pgmap v16213: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:47.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:28:46 smithi153 ceph-mon[70044]: pgmap v16213: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:49.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:28:48 smithi083 ceph-mon[79930]: pgmap v16214: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:49.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:28:48 smithi083 ceph-mon[77242]: pgmap v16214: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:49.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:28:48 smithi153 ceph-mon[70044]: pgmap v16214: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:49.845 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:28:50.134 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:28:50.135 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 59s ago 9h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:28:50.135 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 9m ago 9h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:28:50.135 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 59s ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:28:50.135 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 9m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:28:50.136 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (8h) 9m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:28:50.136 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 59s ago 9h 959M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:28:50.136 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 59s ago 9h 1984M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:28:50.136 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 9m ago 9h 847M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:28:50.136 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 59s ago 9h 1185M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:28:50.137 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 59s ago 9h 24.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:28:50.137 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 9m ago 9h 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:28:50.137 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 59s ago 9h 1438M 2234M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:28:50.137 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 59s ago 9h 1425M 2234M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:28:50.138 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 59s ago 9h 1107M 2234M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:28:50.138 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 59s ago 9h 1396M 2234M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:28:50.138 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 9m ago 9h 1203M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:28:50.138 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 9m ago 9h 1508M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:28:50.139 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 9m ago 9h 1348M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:28:50.139 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 9m ago 9h 1602M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:28:50.139 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (8h) 9m ago 9h 192M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:28:50.492 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:28:50.493 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:28:50.493 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:28:50.493 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:28:50.493 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:28:50.494 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:28:50.494 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:28:50.494 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:28:50.494 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:28:50.495 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:28:50.495 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:28:50.495 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:28:50.495 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:28:50.496 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:28:50.496 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:28:50.496 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:28:50.496 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:28:50.496 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:28:51.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:28:50 smithi083 ceph-mon[77242]: pgmap v16215: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:51.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:28:50 smithi083 ceph-mon[77242]: from='client.52986 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:28:51.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:28:50 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3211326151' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:28:51.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:28:50 smithi083 ceph-mon[79930]: pgmap v16215: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:51.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:28:50 smithi083 ceph-mon[79930]: from='client.52986 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:28:51.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:28:50 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3211326151' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:28:51.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:28:50 smithi153 ceph-mon[70044]: pgmap v16215: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:51.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:28:50 smithi153 ceph-mon[70044]: from='client.52986 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:28:51.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:28:50 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3211326151' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:28:51.927 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:28:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:28:51] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T14:28:52.184 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:28:51 smithi083 ceph-mon[79930]: from='client.52992 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:28:52.184 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:28:51 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:28:52.185 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:28:51 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:28:52.185 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:28:51 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:28:52.185 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:28:51 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:28:52.185 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:28:51 smithi083 ceph-mon[77242]: from='client.52992 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:28:52.186 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:28:51 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:28:52.186 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:28:51 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:28:52.186 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:28:51 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:28:52.186 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:28:51 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:28:52.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:28:51 smithi153 ceph-mon[70044]: from='client.52992 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:28:52.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:28:51 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:28:52.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:28:51 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:28:52.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:28:51 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:28:52.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:28:51 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:28:52.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:28:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:28:52.182Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:28:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:28:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:28:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:28:53.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:28:52 smithi083 ceph-mon[79930]: pgmap v16216: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:53.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:28:52 smithi083 ceph-mon[77242]: pgmap v16216: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:53.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:28:52 smithi153 ceph-mon[70044]: pgmap v16216: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:28:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:28:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:28:55.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:28:54 smithi083 ceph-mon[79930]: pgmap v16217: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:55.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:28:54 smithi083 ceph-mon[77242]: pgmap v16217: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:55.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:28:54 smithi153 ceph-mon[70044]: pgmap v16217: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:55.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:28:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:28:55.546Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:28:56.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:28:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:28:56.279Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:28:57.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:28:56 smithi083 ceph-mon[79930]: pgmap v16218: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:57.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:28:56 smithi083 ceph-mon[77242]: pgmap v16218: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:57.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:28:56 smithi153 ceph-mon[70044]: pgmap v16218: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:59.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:28:58 smithi083 ceph-mon[77242]: pgmap v16219: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:59.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:28:58 smithi083 ceph-mon[79930]: pgmap v16219: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:28:59.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:28:58 smithi153 ceph-mon[70044]: pgmap v16219: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:01.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:29:00 smithi083 ceph-mon[79930]: pgmap v16220: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:01.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:29:00 smithi083 ceph-mon[77242]: pgmap v16220: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:01.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:29:00 smithi153 ceph-mon[70044]: pgmap v16220: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:29:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:29:01] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T14:29:02.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:29:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:29:02.182Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:29:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:29:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:29:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:29:03.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:29:02 smithi083 ceph-mon[79930]: pgmap v16221: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:03.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:29:02 smithi083 ceph-mon[77242]: pgmap v16221: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:03.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:29:02 smithi153 ceph-mon[70044]: pgmap v16221: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:29:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:29:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:29:05.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:29:04 smithi083 ceph-mon[79930]: pgmap v16222: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:05.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:29:04 smithi083 ceph-mon[77242]: pgmap v16222: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:05.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:29:04 smithi153 ceph-mon[70044]: pgmap v16222: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:05.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:29:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:29:05.547Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:29:06.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:29:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:29:06.280Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:29:07.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:29:06 smithi083 ceph-mon[77242]: pgmap v16223: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:07.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:29:06 smithi083 ceph-mon[79930]: pgmap v16223: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:07.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:29:06 smithi153 ceph-mon[70044]: pgmap v16223: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:09.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:29:08 smithi083 ceph-mon[77242]: pgmap v16224: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:09.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:29:08 smithi083 ceph-mon[79930]: pgmap v16224: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:09.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:29:08 smithi153 ceph-mon[70044]: pgmap v16224: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:11.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:29:10 smithi083 ceph-mon[79930]: pgmap v16225: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:11.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:29:10 smithi083 ceph-mon[77242]: pgmap v16225: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:11.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:29:10 smithi153 ceph-mon[70044]: pgmap v16225: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:29:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:29:11] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T14:29:12.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:29:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:29:12.183Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:29:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:29:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:29:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:29:13.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:29:12 smithi083 ceph-mon[79930]: pgmap v16226: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:13.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:29:12 smithi083 ceph-mon[77242]: pgmap v16226: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:13.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:29:12 smithi153 ceph-mon[70044]: pgmap v16226: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:29:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:29:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:29:15.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:29:14 smithi153 ceph-mon[70044]: pgmap v16227: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:15.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:29:14 smithi083 ceph-mon[79930]: pgmap v16227: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:15.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:29:14 smithi083 ceph-mon[77242]: pgmap v16227: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:15.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:29:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:29:15.547Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:29:16.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:29:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:29:16.280Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:29:17.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:29:16 smithi153 ceph-mon[70044]: pgmap v16228: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:17.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:29:16 smithi083 ceph-mon[79930]: pgmap v16228: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:17.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:29:16 smithi083 ceph-mon[77242]: pgmap v16228: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:19.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:29:18 smithi153 ceph-mon[70044]: pgmap v16229: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:19.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:29:18 smithi083 ceph-mon[79930]: pgmap v16229: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:19.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:29:18 smithi083 ceph-mon[77242]: pgmap v16229: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:20.820 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:29:21.110 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:29:21.111 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 90s ago 9h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:29:21.111 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 9m ago 9h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:29:21.111 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 90s ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:29:21.111 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 9m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:29:21.112 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 9m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:29:21.112 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 90s ago 9h 959M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:29:21.112 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (8h) 90s ago 9h 1984M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:29:21.112 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 9m ago 9h 847M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:29:21.112 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 90s ago 9h 1185M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:29:21.113 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 90s ago 9h 24.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:29:21.113 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 9m ago 9h 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:29:21.113 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 90s ago 9h 1438M 2234M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:29:21.113 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 90s ago 9h 1425M 2234M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:29:21.113 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 90s ago 9h 1107M 2234M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:29:21.114 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 90s ago 9h 1396M 2234M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:29:21.114 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 9m ago 9h 1203M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:29:21.114 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 9m ago 9h 1508M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:29:21.114 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 9m ago 9h 1348M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:29:21.114 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 9m ago 9h 1602M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:29:21.115 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 9m ago 9h 192M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:29:21.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:29:20 smithi153 ceph-mon[70044]: pgmap v16230: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:21.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:29:20 smithi083 ceph-mon[77242]: pgmap v16230: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:21.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:29:20 smithi083 ceph-mon[79930]: pgmap v16230: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:21.470 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:29:21.470 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:29:21.470 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:29:21.470 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:29:21.471 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:29:21.471 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:29:21.471 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:29:21.471 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:29:21.471 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:29:21.472 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:29:21.472 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:29:21.472 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:29:21.472 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:29:21.472 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:29:21.473 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:29:21.473 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:29:21.473 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:29:21.473 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:29:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:29:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:29:21] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T14:29:22.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:29:21 smithi153 ceph-mon[70044]: from='client.62935 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:29:22.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:29:21 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4238651654' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:29:22.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:29:21 smithi083 ceph-mon[79930]: from='client.62935 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:29:22.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:29:21 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4238651654' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:29:22.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:29:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:29:22.183Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:29:22.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:29:21 smithi083 ceph-mon[77242]: from='client.62935 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:29:22.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:29:21 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4238651654' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:29:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:29:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:29:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:29:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:29:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:29:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:29:23.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:29:22 smithi153 ceph-mon[70044]: from='client.53010 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:29:23.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:29:22 smithi153 ceph-mon[70044]: pgmap v16231: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:23.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:29:22 smithi083 ceph-mon[77242]: from='client.53010 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:29:23.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:29:22 smithi083 ceph-mon[77242]: pgmap v16231: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:23.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:29:22 smithi083 ceph-mon[79930]: from='client.53010 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:29:23.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:29:22 smithi083 ceph-mon[79930]: pgmap v16231: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:25.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:29:24 smithi153 ceph-mon[70044]: pgmap v16232: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:25.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:29:24 smithi083 ceph-mon[79930]: pgmap v16232: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:25.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:29:24 smithi083 ceph-mon[77242]: pgmap v16232: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:25.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:29:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:29:25.548Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:29:26.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:29:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:29:26.282Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:29:27.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:29:26 smithi153 ceph-mon[70044]: pgmap v16233: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:27.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:29:26 smithi083 ceph-mon[79930]: pgmap v16233: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:27.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:29:26 smithi083 ceph-mon[77242]: pgmap v16233: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:29.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:29:28 smithi153 ceph-mon[70044]: pgmap v16234: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:29.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:29:28 smithi083 ceph-mon[79930]: pgmap v16234: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:29.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:29:28 smithi083 ceph-mon[77242]: pgmap v16234: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:31.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:29:30 smithi153 ceph-mon[70044]: pgmap v16235: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:31.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:29:30 smithi083 ceph-mon[79930]: pgmap v16235: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:31.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:29:30 smithi083 ceph-mon[77242]: pgmap v16235: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:29:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:29:31] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T14:29:32.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:29:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:29:32.184Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:29:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:29:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:29:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:29:33.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:29:32 smithi153 ceph-mon[70044]: pgmap v16236: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:29:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:29:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:29:33.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:29:32 smithi083 ceph-mon[79930]: pgmap v16236: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:33.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:29:32 smithi083 ceph-mon[77242]: pgmap v16236: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:35.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:29:35 smithi153 ceph-mon[70044]: pgmap v16237: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:35.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:29:35 smithi083 ceph-mon[79930]: pgmap v16237: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:35.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:29:35 smithi083 ceph-mon[77242]: pgmap v16237: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:35.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:29:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:29:35.549Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:29:36.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:29:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:29:36.282Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:29:37.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:29:37 smithi153 ceph-mon[70044]: pgmap v16238: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:37.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:29:37 smithi083 ceph-mon[79930]: pgmap v16238: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:37.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:29:37 smithi083 ceph-mon[77242]: pgmap v16238: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:39.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:29:39 smithi153 ceph-mon[70044]: pgmap v16239: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:39.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:29:39 smithi083 ceph-mon[79930]: pgmap v16239: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:39.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:29:39 smithi083 ceph-mon[77242]: pgmap v16239: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:29:41 smithi153 ceph-mon[70044]: pgmap v16240: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:41.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:29:41 smithi083 ceph-mon[79930]: pgmap v16240: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:41.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:29:41 smithi083 ceph-mon[77242]: pgmap v16240: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:29:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:29:41] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T14:29:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:29:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:29:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:29:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:29:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:29:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:29:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:29:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:29:42.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:29:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:29:42.185Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:29:42.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:29:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:29:42.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:29:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:29:42.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:29:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:29:42.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:29:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:29:42.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:29:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:29:42.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:29:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:29:42.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:29:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:29:42.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:29:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:29:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:29:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:29:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:29:43.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:29:43 smithi153 ceph-mon[70044]: pgmap v16241: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:29:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:29:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:29:43.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:29:43 smithi083 ceph-mon[79930]: pgmap v16241: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:43.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:29:43 smithi083 ceph-mon[77242]: pgmap v16241: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:45.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:29:45 smithi153 ceph-mon[70044]: pgmap v16242: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:45.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:29:45 smithi083 ceph-mon[77242]: pgmap v16242: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:45.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:29:45 smithi083 ceph-mon[79930]: pgmap v16242: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:45.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:29:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:29:45.549Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:29:46.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:29:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:29:46.283Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:29:47.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:29:47 smithi153 ceph-mon[70044]: pgmap v16243: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:47.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:29:47 smithi083 ceph-mon[77242]: pgmap v16243: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:47.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:29:47 smithi083 ceph-mon[79930]: pgmap v16243: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:49.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:29:49 smithi153 ceph-mon[70044]: pgmap v16244: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:49.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:29:49 smithi083 ceph-mon[77242]: pgmap v16244: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:49.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:29:49 smithi083 ceph-mon[79930]: pgmap v16244: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:51.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:29:51 smithi153 ceph-mon[70044]: pgmap v16245: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:51.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:29:51 smithi083 ceph-mon[79930]: pgmap v16245: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:51.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:29:51 smithi083 ceph-mon[77242]: pgmap v16245: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:51.797 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:29:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:29:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:29:51] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T14:29:52.098 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:29:52.099 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 2m ago 9h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:29:52.099 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 10m ago 9h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:29:52.099 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 2m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:29:52.099 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 10m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:29:52.100 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 10m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:29:52.100 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 2m ago 9h 959M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:29:52.100 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 2m ago 9h 1984M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:29:52.100 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (8h) 10m ago 9h 847M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:29:52.100 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (8h) 2m ago 9h 1185M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:29:52.101 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 2m ago 9h 24.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:29:52.101 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 10m ago 9h 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:29:52.101 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 2m ago 9h 1438M 2234M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:29:52.101 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 2m ago 9h 1425M 2234M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:29:52.101 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 2m ago 9h 1107M 2234M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:29:52.102 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 2m ago 9h 1396M 2234M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:29:52.102 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 10m ago 9h 1203M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:29:52.102 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 10m ago 9h 1508M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:29:52.102 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 10m ago 9h 1348M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:29:52.102 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 10m ago 9h 1602M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:29:52.103 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 10m ago 9h 192M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:29:52.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:29:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:29:52.187Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:29:52.467 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:29:52.467 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:29:52.468 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:29:52.468 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:29:52.468 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:29:52.468 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:29:52.469 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:29:52.469 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:29:52.469 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:29:52.469 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:29:52.469 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:29:52.470 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:29:52.470 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:29:52.470 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:29:52.470 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:29:52.471 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:29:52.471 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:29:52.471 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:29:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:29:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:29:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:29:53.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:29:52 smithi083 ceph-mon[79930]: pgmap v16246: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:53.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:29:52 smithi083 ceph-mon[79930]: from='client.62953 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:29:53.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:29:52 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:29:53.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:29:52 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:29:53.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:29:52 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:29:53.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:29:52 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1202041613' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:29:53.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:29:52 smithi083 ceph-mon[77242]: pgmap v16246: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:53.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:29:52 smithi083 ceph-mon[77242]: from='client.62953 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:29:53.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:29:52 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:29:53.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:29:52 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:29:53.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:29:52 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:29:53.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:29:52 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1202041613' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:29:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:29:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:29:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:29:53.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:29:52 smithi153 ceph-mon[70044]: pgmap v16246: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:53.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:29:52 smithi153 ceph-mon[70044]: from='client.62953 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:29:53.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:29:52 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:29:53.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:29:52 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:29:53.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:29:52 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:29:53.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:29:52 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1202041613' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:29:54.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:29:53 smithi083 ceph-mon[79930]: from='client.53028 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:29:54.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:29:53 smithi083 ceph-mon[77242]: from='client.53028 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:29:54.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:29:53 smithi153 ceph-mon[70044]: from='client.53028 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:29:55.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:29:54 smithi083 ceph-mon[79930]: pgmap v16247: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:55.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:29:54 smithi083 ceph-mon[77242]: pgmap v16247: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:55.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:29:54 smithi153 ceph-mon[70044]: pgmap v16247: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:55.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:29:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:29:55.550Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:29:56.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:29:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:29:56.285Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:29:57.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:29:56 smithi083 ceph-mon[79930]: pgmap v16248: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:57.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:29:56 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:29:57.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:29:56 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:29:57.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:29:56 smithi083 ceph-mon[77242]: pgmap v16248: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:57.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:29:56 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:29:57.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:29:56 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:29:57.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:29:56 smithi153 ceph-mon[70044]: pgmap v16248: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:57.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:29:56 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:29:57.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:29:56 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:29:58.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:29:58 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:29:58.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:29:58 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:29:58.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:29:58 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:29:59.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:29:59 smithi153 ceph-mon[70044]: pgmap v16249: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:59.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:29:59 smithi083 ceph-mon[79930]: pgmap v16249: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:29:59.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:29:59 smithi083 ceph-mon[77242]: pgmap v16249: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:00.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:30:00 smithi153 ceph-mon[70044]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T14:30:00.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:30:00 smithi083 ceph-mon[79930]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T14:30:00.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:30:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[77219]: 2022-09-29T14:29:59.999+0000 7f160f665700 -1 log_channel(cluster) log [ERR] : overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T14:30:00.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:30:00 smithi083 ceph-mon[77242]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T14:30:01.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:30:01 smithi153 ceph-mon[70044]: pgmap v16250: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:01.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:30:01 smithi083 ceph-mon[79930]: pgmap v16250: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:01.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:30:01 smithi083 ceph-mon[77242]: pgmap v16250: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:30:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:30:01] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T14:30:02.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:30:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:30:02.187Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:30:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:30:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:30:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:30:03.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:30:03 smithi153 ceph-mon[70044]: pgmap v16251: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:30:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:30:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:30:03.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:30:03 smithi083 ceph-mon[79930]: pgmap v16251: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:03.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:30:03 smithi083 ceph-mon[77242]: pgmap v16251: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:05.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:30:05 smithi153 ceph-mon[70044]: pgmap v16252: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:05.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:30:05 smithi083 ceph-mon[79930]: pgmap v16252: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:05.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:30:05 smithi083 ceph-mon[77242]: pgmap v16252: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:05.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:30:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:30:05.551Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:30:06.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:30:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:30:06.286Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:30:07.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:30:07 smithi153 ceph-mon[70044]: pgmap v16253: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:07.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:30:07 smithi083 ceph-mon[79930]: pgmap v16253: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:07.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:30:07 smithi083 ceph-mon[77242]: pgmap v16253: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:09.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:30:09 smithi153 ceph-mon[70044]: pgmap v16254: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:09.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:30:09 smithi083 ceph-mon[79930]: pgmap v16254: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:09.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:30:09 smithi083 ceph-mon[77242]: pgmap v16254: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:11.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:30:11 smithi153 ceph-mon[70044]: pgmap v16255: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:11.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:30:11 smithi083 ceph-mon[79930]: pgmap v16255: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:11.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:30:11 smithi083 ceph-mon[77242]: pgmap v16255: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:30:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:30:11] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T14:30:12.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:30:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:30:12.188Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:30:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:30:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:30:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:30:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:30:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:30:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:30:13.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:30:13 smithi153 ceph-mon[70044]: pgmap v16256: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:13.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:30:13 smithi083 ceph-mon[79930]: pgmap v16256: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:13.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:30:13 smithi083 ceph-mon[77242]: pgmap v16256: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:15.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:30:15 smithi153 ceph-mon[70044]: pgmap v16257: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:15.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:30:15 smithi083 ceph-mon[79930]: pgmap v16257: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:15.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:30:15 smithi083 ceph-mon[77242]: pgmap v16257: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:15.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:30:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:30:15.552Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:30:16.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:30:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:30:16.287Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:30:17.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:30:17 smithi153 ceph-mon[70044]: pgmap v16258: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:17.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:30:17 smithi083 ceph-mon[79930]: pgmap v16258: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:17.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:30:17 smithi083 ceph-mon[77242]: pgmap v16258: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:19.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:30:19 smithi153 ceph-mon[70044]: pgmap v16259: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:19.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:30:19 smithi083 ceph-mon[79930]: pgmap v16259: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:19.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:30:19 smithi083 ceph-mon[77242]: pgmap v16259: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:21.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:30:21 smithi153 ceph-mon[70044]: pgmap v16260: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:21.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:30:21 smithi083 ceph-mon[79930]: pgmap v16260: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:21.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:30:21 smithi083 ceph-mon[77242]: pgmap v16260: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:30:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:30:21] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T14:30:22.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:30:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:30:22.189Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:30:22.791 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:30:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:30:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:30:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:30:23.081 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:30:23.082 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 2m ago 9h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:30:23.082 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 26s ago 9h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:30:23.082 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 2m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:30:23.083 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 26s ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:30:23.083 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 26s ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:30:23.083 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 2m ago 9h 959M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:30:23.083 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 2m ago 9h 1984M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:30:23.083 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 26s ago 9h 863M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:30:23.084 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 2m ago 9h 1185M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:30:23.084 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 2m ago 9h 24.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:30:23.084 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 26s ago 9h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:30:23.084 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8h) 2m ago 9h 1438M 2234M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:30:23.084 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 2m ago 9h 1425M 2234M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:30:23.085 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 2m ago 9h 1107M 2234M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:30:23.085 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 2m ago 9h 1396M 2234M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:30:23.085 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 26s ago 9h 1224M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:30:23.085 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 26s ago 9h 1537M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:30:23.086 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 26s ago 9h 1373M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:30:23.086 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 26s ago 9h 1633M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:30:23.086 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 26s ago 9h 194M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:30:23.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:30:23 smithi153 ceph-mon[70044]: pgmap v16261: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:30:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:30:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:30:23.447 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:30:23.448 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:30:23.448 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:30:23.448 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:30:23.448 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:30:23.449 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:30:23.449 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:30:23.449 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:30:23.449 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:30:23.449 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:30:23.450 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:30:23.450 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:30:23.450 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:30:23.450 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:30:23.450 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:30:23.451 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:30:23.451 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:30:23.451 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:30:23.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:30:23 smithi083 ceph-mon[79930]: pgmap v16261: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:23.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:30:23 smithi083 ceph-mon[77242]: pgmap v16261: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:24.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:30:24 smithi153 ceph-mon[70044]: from='client.62971 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:30:24.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:30:24 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2668673021' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:30:24.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:30:24 smithi083 ceph-mon[79930]: from='client.62971 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:30:24.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:30:24 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2668673021' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:30:24.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:30:24 smithi083 ceph-mon[77242]: from='client.62971 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:30:24.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:30:24 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2668673021' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:30:25.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:30:25 smithi153 ceph-mon[70044]: from='client.53046 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:30:25.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:30:25 smithi153 ceph-mon[70044]: pgmap v16262: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:25.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:30:25 smithi083 ceph-mon[79930]: from='client.53046 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:30:25.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:30:25 smithi083 ceph-mon[79930]: pgmap v16262: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:25.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:30:25 smithi083 ceph-mon[77242]: from='client.53046 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:30:25.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:30:25 smithi083 ceph-mon[77242]: pgmap v16262: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:25.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:30:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:30:25.553Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:30:26.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:30:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:30:26.288Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:30:27.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:30:27 smithi083 ceph-mon[77242]: pgmap v16263: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:27.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:30:27 smithi083 ceph-mon[79930]: pgmap v16263: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:27.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:30:27 smithi153 ceph-mon[70044]: pgmap v16263: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:29.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:30:29 smithi083 ceph-mon[79930]: pgmap v16264: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:29.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:30:29 smithi083 ceph-mon[77242]: pgmap v16264: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:29.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:30:29 smithi153 ceph-mon[70044]: pgmap v16264: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:31.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:30:31 smithi083 ceph-mon[79930]: pgmap v16265: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:31.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:30:31 smithi083 ceph-mon[77242]: pgmap v16265: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:31.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:30:31 smithi153 ceph-mon[70044]: pgmap v16265: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:30:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:30:31] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T14:30:32.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:30:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:30:32.190Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:30:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:30:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:30:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:30:33.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:30:32 smithi083 ceph-mon[79930]: pgmap v16266: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:33.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:30:32 smithi083 ceph-mon[77242]: pgmap v16266: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:33.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:30:32 smithi153 ceph-mon[70044]: pgmap v16266: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:30:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:30:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:30:35.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:30:34 smithi083 ceph-mon[77242]: pgmap v16267: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:35.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:30:34 smithi083 ceph-mon[79930]: pgmap v16267: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:35.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:30:34 smithi153 ceph-mon[70044]: pgmap v16267: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:35.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:30:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:30:35.554Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:30:36.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:30:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:30:36.288Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:30:37.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:30:36 smithi083 ceph-mon[79930]: pgmap v16268: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:37.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:30:36 smithi083 ceph-mon[77242]: pgmap v16268: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:37.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:30:36 smithi153 ceph-mon[70044]: pgmap v16268: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:39.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:30:38 smithi083 ceph-mon[79930]: pgmap v16269: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:39.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:30:38 smithi083 ceph-mon[77242]: pgmap v16269: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:39.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:30:38 smithi153 ceph-mon[70044]: pgmap v16269: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:41.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:30:40 smithi083 ceph-mon[79930]: pgmap v16270: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:30:40 smithi083 ceph-mon[77242]: pgmap v16270: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:30:40 smithi153 ceph-mon[70044]: pgmap v16270: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:41.879 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:30:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:30:41] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T14:30:42.193 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:30:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:30:42.193 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:30:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:30:42.194 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:30:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:30:42.194 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:30:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:30:42.194 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:30:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:30:42.195 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:30:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:30:42.195 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:30:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:30:42.195 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:30:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:30:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:30:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:30:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:30:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:30:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:30:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:30:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:30:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:30:42.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:30:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:30:42.191Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:30:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:30:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:30:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:30:43.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:30:42 smithi083 ceph-mon[79930]: pgmap v16271: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:43.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:30:42 smithi083 ceph-mon[77242]: pgmap v16271: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:43.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:30:42 smithi153 ceph-mon[70044]: pgmap v16271: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:30:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:30:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:30:45.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:30:44 smithi083 ceph-mon[79930]: pgmap v16272: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:45.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:30:44 smithi083 ceph-mon[77242]: pgmap v16272: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:45.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:30:44 smithi153 ceph-mon[70044]: pgmap v16272: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:45.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:30:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:30:45.555Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:30:46.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:30:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:30:46.288Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:30:47.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:30:46 smithi083 ceph-mon[79930]: pgmap v16273: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:47.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:30:46 smithi083 ceph-mon[77242]: pgmap v16273: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:47.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:30:46 smithi153 ceph-mon[70044]: pgmap v16273: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:49.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:30:48 smithi083 ceph-mon[77242]: pgmap v16274: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:49.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:30:48 smithi083 ceph-mon[79930]: pgmap v16274: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:49.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:30:48 smithi153 ceph-mon[70044]: pgmap v16274: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:51.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:30:50 smithi083 ceph-mon[79930]: pgmap v16275: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:51.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:30:50 smithi083 ceph-mon[77242]: pgmap v16275: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:51.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:30:50 smithi153 ceph-mon[70044]: pgmap v16275: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:30:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:30:51] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T14:30:52.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:30:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:30:52.191Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:30:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:30:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:30:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:30:53.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:30:52 smithi083 ceph-mon[79930]: pgmap v16276: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:53.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:30:52 smithi083 ceph-mon[77242]: pgmap v16276: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:53.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:30:52 smithi153 ceph-mon[70044]: pgmap v16276: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:30:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:30:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:30:53.786 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:30:54.079 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:30:54.079 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 3m ago 9h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:30:54.080 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 57s ago 9h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:30:54.080 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 3m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:30:54.080 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 57s ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:30:54.080 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 57s ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:30:54.080 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 3m ago 9h 959M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:30:54.081 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 3m ago 9h 1984M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:30:54.081 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 57s ago 9h 863M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:30:54.081 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 3m ago 9h 1185M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:30:54.081 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 3m ago 9h 24.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:30:54.081 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 57s ago 9h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:30:54.082 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 3m ago 9h 1438M 2234M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:30:54.082 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8h) 3m ago 9h 1425M 2234M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:30:54.082 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 3m ago 9h 1107M 2234M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:30:54.082 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 3m ago 9h 1396M 2234M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:30:54.083 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 57s ago 9h 1224M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:30:54.083 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 57s ago 9h 1537M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:30:54.083 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 57s ago 9h 1373M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:30:54.083 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 57s ago 9h 1633M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:30:54.083 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 57s ago 9h 194M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:30:54.434 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:30:54.434 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:30:54.435 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:30:54.435 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:30:54.435 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:30:54.435 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:30:54.436 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:30:54.436 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:30:54.436 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:30:54.436 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:30:54.437 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:30:54.437 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:30:54.437 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:30:54.437 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:30:54.437 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:30:54.438 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:30:54.438 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:30:54.438 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:30:55.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:30:54 smithi083 ceph-mon[77242]: pgmap v16277: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:55.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:30:54 smithi083 ceph-mon[77242]: from='client.62989 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:30:55.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:30:54 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2843359790' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:30:55.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:30:54 smithi083 ceph-mon[79930]: pgmap v16277: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:55.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:30:54 smithi083 ceph-mon[79930]: from='client.62989 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:30:55.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:30:54 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2843359790' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:30:55.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:30:54 smithi153 ceph-mon[70044]: pgmap v16277: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:55.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:30:54 smithi153 ceph-mon[70044]: from='client.62989 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:30:55.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:30:54 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2843359790' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:30:55.906 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:30:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:30:55.555Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:30:56.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:30:55 smithi083 ceph-mon[79930]: from='client.53064 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:30:56.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:30:55 smithi083 ceph-mon[77242]: from='client.53064 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:30:56.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:30:55 smithi153 ceph-mon[70044]: from='client.53064 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:30:56.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:30:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:30:56.289Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:30:57.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:30:56 smithi083 ceph-mon[79930]: pgmap v16278: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:57.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:30:56 smithi083 ceph-mon[77242]: pgmap v16278: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:57.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:30:56 smithi153 ceph-mon[70044]: pgmap v16278: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:58.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:30:57 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:30:58.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:30:57 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:30:58.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:30:57 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:30:58.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:30:57 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:30:58.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:30:57 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:30:58.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:30:57 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:30:58.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:30:57 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:30:58.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:30:57 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:30:58.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:30:57 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:30:58.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:30:57 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:30:58.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:30:57 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:30:58.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:30:57 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:30:59.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:30:58 smithi083 ceph-mon[79930]: pgmap v16279: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:59.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:30:58 smithi083 ceph-mon[77242]: pgmap v16279: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:30:59.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:30:58 smithi153 ceph-mon[70044]: pgmap v16279: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:01.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:31:00 smithi083 ceph-mon[79930]: pgmap v16280: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:01.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:31:00 smithi083 ceph-mon[77242]: pgmap v16280: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:01.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:31:00 smithi153 ceph-mon[70044]: pgmap v16280: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:31:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:31:01] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T14:31:02.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:31:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:31:02.192Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:31:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:31:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:31:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:31:03.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:31:02 smithi083 ceph-mon[79930]: pgmap v16281: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:03.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:31:02 smithi083 ceph-mon[77242]: pgmap v16281: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:03.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:31:02 smithi153 ceph-mon[70044]: pgmap v16281: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:31:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:31:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:31:05.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:31:04 smithi083 ceph-mon[79930]: pgmap v16282: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:05.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:31:04 smithi083 ceph-mon[77242]: pgmap v16282: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:05.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:31:04 smithi153 ceph-mon[70044]: pgmap v16282: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:05.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:31:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:31:05.556Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:31:06.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:31:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:31:06.290Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:31:07.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:31:06 smithi083 ceph-mon[79930]: pgmap v16283: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:07.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:31:06 smithi083 ceph-mon[77242]: pgmap v16283: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:07.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:31:06 smithi153 ceph-mon[70044]: pgmap v16283: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:09.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:31:08 smithi083 ceph-mon[77242]: pgmap v16284: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:09.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:31:08 smithi083 ceph-mon[79930]: pgmap v16284: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:09.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:31:08 smithi153 ceph-mon[70044]: pgmap v16284: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:11.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:31:10 smithi083 ceph-mon[79930]: pgmap v16285: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:11.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:31:10 smithi083 ceph-mon[77242]: pgmap v16285: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:11.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:31:10 smithi153 ceph-mon[70044]: pgmap v16285: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:31:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:31:11] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T14:31:12.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:31:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:31:12.193Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:31:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:31:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:31:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:31:13.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:31:12 smithi083 ceph-mon[79930]: pgmap v16286: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:13.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:31:12 smithi083 ceph-mon[77242]: pgmap v16286: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:13.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:31:12 smithi153 ceph-mon[70044]: pgmap v16286: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:13.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:31:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:31:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:31:15.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:31:14 smithi083 ceph-mon[79930]: pgmap v16287: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:15.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:31:14 smithi083 ceph-mon[77242]: pgmap v16287: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:15.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:31:14 smithi153 ceph-mon[70044]: pgmap v16287: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:15.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:31:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:31:15.557Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:31:16.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:31:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:31:16.290Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:31:17.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:31:16 smithi083 ceph-mon[79930]: pgmap v16288: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:17.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:31:16 smithi083 ceph-mon[77242]: pgmap v16288: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:17.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:31:16 smithi153 ceph-mon[70044]: pgmap v16288: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:19.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:31:18 smithi083 ceph-mon[77242]: pgmap v16289: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:19.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:31:18 smithi083 ceph-mon[79930]: pgmap v16289: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:19.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:31:18 smithi153 ceph-mon[70044]: pgmap v16289: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:21.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:31:20 smithi083 ceph-mon[77242]: pgmap v16290: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:21.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:31:20 smithi083 ceph-mon[79930]: pgmap v16290: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:21.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:31:20 smithi153 ceph-mon[70044]: pgmap v16290: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:31:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:31:21] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T14:31:22.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:31:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:31:22.193Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:31:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:31:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:31:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:31:23.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:31:22 smithi083 ceph-mon[77242]: pgmap v16291: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:23.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:31:22 smithi083 ceph-mon[79930]: pgmap v16291: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:23.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:31:22 smithi153 ceph-mon[70044]: pgmap v16291: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:31:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:31:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:31:24.761 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:31:25.051 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:31:25.052 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 3m ago 9h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:31:25.052 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 88s ago 9h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:31:25.052 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 3m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:31:25.053 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 88s ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:31:25.053 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 88s ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:31:25.053 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 3m ago 9h 959M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:31:25.053 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 3m ago 9h 1984M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:31:25.054 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 88s ago 9h 863M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:31:25.054 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 3m ago 9h 1185M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:31:25.054 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 3m ago 9h 24.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:31:25.054 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 88s ago 9h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:31:25.054 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 3m ago 9h 1438M 2234M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:31:25.055 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 3m ago 9h 1425M 2234M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:31:25.055 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8h) 3m ago 9h 1107M 2234M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:31:25.055 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 3m ago 9h 1396M 2234M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:31:25.055 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 88s ago 9h 1224M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:31:25.056 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 88s ago 9h 1537M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:31:25.056 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 88s ago 9h 1373M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:31:25.056 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 88s ago 9h 1633M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:31:25.056 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 88s ago 9h 194M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:31:25.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:31:24 smithi083 ceph-mon[77242]: pgmap v16292: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:25.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:31:24 smithi083 ceph-mon[79930]: pgmap v16292: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:25.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:31:24 smithi153 ceph-mon[70044]: pgmap v16292: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:25.406 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:31:25.407 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:31:25.407 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:31:25.407 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:31:25.407 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:31:25.408 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:31:25.408 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:31:25.408 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:31:25.408 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:31:25.408 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:31:25.409 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:31:25.409 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:31:25.409 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:31:25.409 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:31:25.410 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:31:25.410 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:31:25.410 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:31:25.410 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:31:25.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:31:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:31:25.556Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:31:26.292 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:31:25 smithi083 ceph-mon[77242]: from='client.63007 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:31:26.292 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:31:25 smithi083 ceph-mon[77242]: from='client.53082 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:31:26.293 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:31:25 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2712575850' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:31:26.293 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:31:25 smithi083 ceph-mon[79930]: from='client.63007 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:31:26.293 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:31:25 smithi083 ceph-mon[79930]: from='client.53082 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:31:26.293 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:31:25 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2712575850' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:31:26.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:31:25 smithi153 ceph-mon[70044]: from='client.63007 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:31:26.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:31:25 smithi153 ceph-mon[70044]: from='client.53082 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:31:26.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:31:25 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2712575850' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:31:26.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:31:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:31:26.290Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:31:27.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:31:26 smithi153 ceph-mon[70044]: pgmap v16293: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:27.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:31:26 smithi083 ceph-mon[77242]: pgmap v16293: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:27.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:31:26 smithi083 ceph-mon[79930]: pgmap v16293: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:29.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:31:28 smithi153 ceph-mon[70044]: pgmap v16294: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:29.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:31:28 smithi083 ceph-mon[79930]: pgmap v16294: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:29.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:31:28 smithi083 ceph-mon[77242]: pgmap v16294: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:31.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:31:30 smithi153 ceph-mon[70044]: pgmap v16295: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:31.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:31:30 smithi083 ceph-mon[79930]: pgmap v16295: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:31.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:31:30 smithi083 ceph-mon[77242]: pgmap v16295: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:31:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:31:31] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T14:31:32.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:31:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:31:32.195Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:31:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:31:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:31:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:31:33.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:31:32 smithi153 ceph-mon[70044]: pgmap v16296: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:31:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:31:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:31:33.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:31:32 smithi083 ceph-mon[79930]: pgmap v16296: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:33.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:31:32 smithi083 ceph-mon[77242]: pgmap v16296: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:35.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:31:34 smithi153 ceph-mon[70044]: pgmap v16297: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:35.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:31:34 smithi083 ceph-mon[79930]: pgmap v16297: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:35.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:31:34 smithi083 ceph-mon[77242]: pgmap v16297: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:35.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:31:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:31:35.557Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:31:36.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:31:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:31:36.292Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:31:37.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:31:36 smithi153 ceph-mon[70044]: pgmap v16298: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:37.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:31:36 smithi083 ceph-mon[79930]: pgmap v16298: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:37.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:31:36 smithi083 ceph-mon[77242]: pgmap v16298: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:39.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:31:38 smithi153 ceph-mon[70044]: pgmap v16299: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:39.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:31:38 smithi083 ceph-mon[79930]: pgmap v16299: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:39.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:31:38 smithi083 ceph-mon[77242]: pgmap v16299: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:31:40 smithi153 ceph-mon[70044]: pgmap v16300: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:41.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:31:40 smithi083 ceph-mon[79930]: pgmap v16300: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:41.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:31:40 smithi083 ceph-mon[77242]: pgmap v16300: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:31:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:31:41] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T14:31:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:31:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:31:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:31:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:31:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:31:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:31:42.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:31:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:31:42.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:31:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:31:42.195Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:31:42.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:31:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:31:42.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:31:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:31:42.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:31:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:31:42.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:31:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:31:42.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:31:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:31:42.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:31:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:31:42.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:31:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:31:42.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:31:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:31:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:31:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:31:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:31:43.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:31:43 smithi153 ceph-mon[70044]: pgmap v16301: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:31:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:31:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:31:43.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:31:43 smithi083 ceph-mon[79930]: pgmap v16301: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:43.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:31:42 smithi083 ceph-mon[77242]: pgmap v16301: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:45.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:31:45 smithi153 ceph-mon[70044]: pgmap v16302: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:45.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:31:45 smithi083 ceph-mon[77242]: pgmap v16302: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:45.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:31:45 smithi083 ceph-mon[79930]: pgmap v16302: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:45.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:31:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:31:45.558Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:31:46.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:31:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:31:46.292Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:31:47.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:31:47 smithi153 ceph-mon[70044]: pgmap v16303: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:47.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:31:47 smithi083 ceph-mon[79930]: pgmap v16303: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:47.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:31:47 smithi083 ceph-mon[77242]: pgmap v16303: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:49.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:31:49 smithi153 ceph-mon[70044]: pgmap v16304: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:49.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:31:49 smithi083 ceph-mon[79930]: pgmap v16304: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:49.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:31:49 smithi083 ceph-mon[77242]: pgmap v16304: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:51.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:31:51 smithi153 ceph-mon[70044]: pgmap v16305: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:51.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:31:51 smithi083 ceph-mon[77242]: pgmap v16305: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:51.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:31:51 smithi083 ceph-mon[79930]: pgmap v16305: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:31:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:31:51] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T14:31:52.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:31:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:31:52.195Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:31:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:31:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:31:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:31:53.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:31:53 smithi153 ceph-mon[70044]: pgmap v16306: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:31:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:31:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:31:53.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:31:53 smithi083 ceph-mon[79930]: pgmap v16306: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:53.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:31:53 smithi083 ceph-mon[77242]: pgmap v16306: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:55.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:31:55 smithi153 ceph-mon[70044]: pgmap v16307: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:55.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:31:55 smithi083 ceph-mon[79930]: pgmap v16307: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:55.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:31:55 smithi083 ceph-mon[77242]: pgmap v16307: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:55.732 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:31:55.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:31:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:31:55.557Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:31:56.025 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:31:56.026 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 4m ago 9h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:31:56.026 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 119s ago 9h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:31:56.026 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 4m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:31:56.026 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 119s ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:31:56.027 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 119s ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:31:56.027 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 4m ago 9h 959M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:31:56.027 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 4m ago 9h 1984M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:31:56.027 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 119s ago 9h 863M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:31:56.028 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 4m ago 9h 1185M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:31:56.028 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 4m ago 9h 24.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:31:56.028 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 119s ago 9h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:31:56.028 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 4m ago 9h 1438M 2234M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:31:56.029 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 4m ago 9h 1425M 2234M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:31:56.029 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 4m ago 9h 1107M 2234M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:31:56.029 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8h) 4m ago 9h 1396M 2234M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:31:56.029 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 119s ago 9h 1224M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:31:56.030 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 119s ago 9h 1537M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:31:56.030 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 119s ago 9h 1373M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:31:56.030 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 119s ago 9h 1633M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:31:56.030 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 119s ago 9h 194M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:31:56.382 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:31:56.383 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:31:56.383 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:31:56.383 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:31:56.383 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:31:56.384 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:31:56.384 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:31:56.384 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:31:56.384 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:31:56.385 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:31:56.385 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:31:56.385 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:31:56.385 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:31:56.385 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:31:56.386 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:31:56.386 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:31:56.386 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:31:56.386 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:31:56.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:31:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:31:56.291Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:31:57.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:31:57 smithi153 ceph-mon[70044]: pgmap v16308: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:57.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:31:57 smithi153 ceph-mon[70044]: from='client.63025 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:31:57.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:31:57 smithi153 ceph-mon[70044]: from='client.53100 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:31:57.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:31:57 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3912118228' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:31:57.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:31:57 smithi083 ceph-mon[79930]: pgmap v16308: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:57.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:31:57 smithi083 ceph-mon[79930]: from='client.63025 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:31:57.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:31:57 smithi083 ceph-mon[79930]: from='client.53100 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:31:57.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:31:57 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3912118228' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:31:57.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:31:57 smithi083 ceph-mon[77242]: pgmap v16308: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:57.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:31:57 smithi083 ceph-mon[77242]: from='client.63025 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:31:57.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:31:57 smithi083 ceph-mon[77242]: from='client.53100 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:31:57.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:31:57 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3912118228' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:31:58.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:31:58 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:31:58.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:31:58 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:31:58.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:31:58 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:31:58.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:31:58 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:31:58.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:31:58 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:31:58.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:31:58 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:31:58.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:31:58 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:31:58.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:31:58 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:31:58.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:31:58 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:31:59.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:31:59 smithi153 ceph-mon[70044]: pgmap v16309: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:59.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:31:59 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:31:59.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:31:59 smithi083 ceph-mon[79930]: pgmap v16309: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:59.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:31:59 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:31:59.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:31:59 smithi083 ceph-mon[77242]: pgmap v16309: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:31:59.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:31:59 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:32:01.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:32:01 smithi153 ceph-mon[70044]: pgmap v16310: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:01.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:32:01 smithi083 ceph-mon[79930]: pgmap v16310: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:01.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:32:01 smithi083 ceph-mon[77242]: pgmap v16310: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:32:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:32:01] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T14:32:02.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:32:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:32:02.195Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:32:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:32:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:32:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:32:03.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:32:03 smithi153 ceph-mon[70044]: pgmap v16311: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:32:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:32:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:32:03.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:32:03 smithi083 ceph-mon[79930]: pgmap v16311: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:03.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:32:03 smithi083 ceph-mon[77242]: pgmap v16311: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:05.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:32:05 smithi153 ceph-mon[70044]: pgmap v16312: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:05.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:32:05 smithi083 ceph-mon[79930]: pgmap v16312: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:05.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:32:05 smithi083 ceph-mon[77242]: pgmap v16312: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:05.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:32:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:32:05.558Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:32:06.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:32:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:32:06.293Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:32:07.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:32:07 smithi153 ceph-mon[70044]: pgmap v16313: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:07.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:32:07 smithi083 ceph-mon[77242]: pgmap v16313: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:07.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:32:07 smithi083 ceph-mon[79930]: pgmap v16313: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:09.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:32:09 smithi153 ceph-mon[70044]: pgmap v16314: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:09.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:32:09 smithi083 ceph-mon[77242]: pgmap v16314: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:09.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:32:09 smithi083 ceph-mon[79930]: pgmap v16314: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:11.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:32:11 smithi153 ceph-mon[70044]: pgmap v16315: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:11.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:32:11 smithi083 ceph-mon[79930]: pgmap v16315: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:11.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:32:11 smithi083 ceph-mon[77242]: pgmap v16315: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:32:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:32:11] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T14:32:12.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:32:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:32:12.196Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:32:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:32:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:32:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:32:13.312 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:32:13 smithi153 ceph-mon[70044]: pgmap v16316: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:13.312 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:32:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:32:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:32:13.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:32:13 smithi083 ceph-mon[79930]: pgmap v16316: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:13.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:32:13 smithi083 ceph-mon[77242]: pgmap v16316: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:15.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:32:15 smithi153 ceph-mon[70044]: pgmap v16317: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:15.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:32:15 smithi083 ceph-mon[79930]: pgmap v16317: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:15.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:32:15 smithi083 ceph-mon[77242]: pgmap v16317: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:15.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:32:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:32:15.560Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:32:16.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:32:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:32:16.294Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:32:17.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:32:17 smithi083 ceph-mon[77242]: pgmap v16318: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:17.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:32:17 smithi083 ceph-mon[79930]: pgmap v16318: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:17.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:32:17 smithi153 ceph-mon[70044]: pgmap v16318: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:19.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:32:19 smithi083 ceph-mon[77242]: pgmap v16319: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:19.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:32:19 smithi083 ceph-mon[79930]: pgmap v16319: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:19.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:32:19 smithi153 ceph-mon[70044]: pgmap v16319: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:21.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:32:21 smithi083 ceph-mon[79930]: pgmap v16320: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:21.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:32:21 smithi083 ceph-mon[77242]: pgmap v16320: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:21.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:32:21 smithi153 ceph-mon[70044]: pgmap v16320: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:32:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:32:21] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T14:32:22.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:32:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:32:22.197Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:32:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:32:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:32:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:32:23.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:32:23 smithi153 ceph-mon[70044]: pgmap v16321: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:23.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:32:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:32:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:32:23.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:32:23 smithi083 ceph-mon[77242]: pgmap v16321: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:23.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:32:23 smithi083 ceph-mon[79930]: pgmap v16321: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:25.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:32:25 smithi083 ceph-mon[77242]: pgmap v16322: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:25.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:32:25 smithi083 ceph-mon[79930]: pgmap v16322: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:25.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:32:25 smithi153 ceph-mon[70044]: pgmap v16322: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:25.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:32:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:32:25.561Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:32:26.709 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:32:26.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:32:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:32:26.294Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:32:26.998 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:32:26.998 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 4m ago 9h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:32:26.998 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 2m ago 9h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:32:26.999 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 4m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:32:26.999 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 2m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:32:26.999 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 2m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:32:26.999 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 4m ago 9h 959M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:32:27.000 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 4m ago 9h 1984M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:32:27.000 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 2m ago 9h 863M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:32:27.000 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 4m ago 9h 1185M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:32:27.000 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 4m ago 9h 24.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:32:27.001 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 2m ago 9h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:32:27.001 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 4m ago 9h 1438M 2234M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:32:27.001 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 4m ago 9h 1425M 2234M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:32:27.001 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 4m ago 9h 1107M 2234M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:32:27.002 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 4m ago 9h 1396M 2234M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:32:27.002 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (8h) 2m ago 9h 1224M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:32:27.002 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 2m ago 9h 1537M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:32:27.002 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 2m ago 9h 1373M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:32:27.002 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 2m ago 9h 1633M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:32:27.003 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 2m ago 9h 194M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:32:27.355 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:32:27.355 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:32:27.355 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:32:27.356 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:32:27.356 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:32:27.356 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:32:27.357 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:32:27.357 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:32:27.357 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:32:27.357 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:32:27.357 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:32:27.358 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:32:27.358 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:32:27.358 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:32:27.358 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:32:27.359 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:32:27.359 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:32:27.359 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:32:27.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:32:27 smithi083 ceph-mon[79930]: pgmap v16323: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:27.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:32:27 smithi083 ceph-mon[77242]: pgmap v16323: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:27.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:32:27 smithi153 ceph-mon[70044]: pgmap v16323: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:28.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:32:28 smithi083 ceph-mon[79930]: from='client.63043 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:32:28.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:32:28 smithi083 ceph-mon[79930]: from='client.53118 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:32:28.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:32:28 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3820098412' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:32:28.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:32:28 smithi083 ceph-mon[77242]: from='client.63043 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:32:28.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:32:28 smithi083 ceph-mon[77242]: from='client.53118 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:32:28.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:32:28 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3820098412' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:32:28.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:32:28 smithi153 ceph-mon[70044]: from='client.63043 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:32:28.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:32:28 smithi153 ceph-mon[70044]: from='client.53118 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:32:28.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:32:28 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3820098412' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:32:29.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:32:29 smithi083 ceph-mon[79930]: pgmap v16324: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:29.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:32:29 smithi083 ceph-mon[77242]: pgmap v16324: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:29.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:32:29 smithi153 ceph-mon[70044]: pgmap v16324: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:31.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:32:31 smithi083 ceph-mon[79930]: pgmap v16325: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:31.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:32:31 smithi083 ceph-mon[77242]: pgmap v16325: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:31.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:32:31 smithi153 ceph-mon[70044]: pgmap v16325: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:32:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:32:31] "GET /metrics HTTP/1.1" 200 225234 "" "Prometheus/2.33.4" 2022-09-29T14:32:32.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:32:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:32:32.198Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:32:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:32:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:32:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:32:33.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:32:33 smithi153 ceph-mon[70044]: pgmap v16326: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:32:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:32:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:32:33.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:32:33 smithi083 ceph-mon[79930]: pgmap v16326: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:33.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:32:33 smithi083 ceph-mon[77242]: pgmap v16326: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:34.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:32:34 smithi083 ceph-mon[77242]: pgmap v16327: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:34.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:32:34 smithi083 ceph-mon[79930]: pgmap v16327: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:32:34 smithi153 ceph-mon[70044]: pgmap v16327: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:35.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:32:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:32:35.561Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:32:36.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:32:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:32:36.296Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:32:37.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:32:36 smithi153 ceph-mon[70044]: pgmap v16328: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:37.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:32:36 smithi083 ceph-mon[77242]: pgmap v16328: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:37.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:32:36 smithi083 ceph-mon[79930]: pgmap v16328: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:39.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:32:38 smithi083 ceph-mon[79930]: pgmap v16329: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:39.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:32:38 smithi083 ceph-mon[77242]: pgmap v16329: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:39.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:32:38 smithi153 ceph-mon[70044]: pgmap v16329: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:41.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:32:40 smithi083 ceph-mon[79930]: pgmap v16330: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:32:40 smithi083 ceph-mon[77242]: pgmap v16330: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:32:40 smithi153 ceph-mon[70044]: pgmap v16330: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:41.906 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:32:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:32:41] "GET /metrics HTTP/1.1" 200 225247 "" "Prometheus/2.33.4" 2022-09-29T14:32:42.201 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:32:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:32:42.201 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:32:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:32:42.202 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:32:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:32:42.202 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:32:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:32:42.202 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:32:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:32:42.202 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:32:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:32:42.203 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:32:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:32:42.203 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:32:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:32:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:32:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:32:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:32:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:32:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:32:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:32:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:32:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:32:42.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:32:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:32:42.199Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:32:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:32:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:32:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:32:43.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:32:42 smithi083 ceph-mon[77242]: pgmap v16331: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:43.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:32:42 smithi083 ceph-mon[79930]: pgmap v16331: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:43.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:32:42 smithi153 ceph-mon[70044]: pgmap v16331: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:32:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:32:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:32:45.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:32:44 smithi083 ceph-mon[77242]: pgmap v16332: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:45.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:32:44 smithi083 ceph-mon[79930]: pgmap v16332: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:45.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:32:44 smithi153 ceph-mon[70044]: pgmap v16332: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:45.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:32:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:32:45.561Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:32:46.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:32:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:32:46.295Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:32:47.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:32:46 smithi083 ceph-mon[79930]: pgmap v16333: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:47.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:32:46 smithi083 ceph-mon[77242]: pgmap v16333: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:47.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:32:46 smithi153 ceph-mon[70044]: pgmap v16333: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:49.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:32:48 smithi083 ceph-mon[79930]: pgmap v16334: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:49.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:32:48 smithi083 ceph-mon[77242]: pgmap v16334: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:49.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:32:48 smithi153 ceph-mon[70044]: pgmap v16334: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:51.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:32:50 smithi083 ceph-mon[77242]: pgmap v16335: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:51.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:32:50 smithi083 ceph-mon[79930]: pgmap v16335: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:51.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:32:50 smithi153 ceph-mon[70044]: pgmap v16335: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:32:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:32:51] "GET /metrics HTTP/1.1" 200 225247 "" "Prometheus/2.33.4" 2022-09-29T14:32:52.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:32:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:32:52.200Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:32:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:32:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:32:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:32:53.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:32:52 smithi083 ceph-mon[79930]: pgmap v16336: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:53.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:32:52 smithi083 ceph-mon[77242]: pgmap v16336: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:53.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:32:52 smithi153 ceph-mon[70044]: pgmap v16336: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:32:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:32:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:32:55.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:32:54 smithi083 ceph-mon[77242]: pgmap v16337: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:55.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:32:54 smithi083 ceph-mon[79930]: pgmap v16337: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:55.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:32:54 smithi153 ceph-mon[70044]: pgmap v16337: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:55.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:32:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:32:55.562Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:32:56.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:32:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:32:56.295Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:32:57.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:32:56 smithi083 ceph-mon[79930]: pgmap v16338: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:57.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:32:56 smithi083 ceph-mon[77242]: pgmap v16338: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:57.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:32:56 smithi153 ceph-mon[70044]: pgmap v16338: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:57.679 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:32:57.982 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:32:57.983 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 5m ago 9h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:32:57.983 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 3m ago 9h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:32:57.983 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 5m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:32:57.983 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 3m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:32:57.984 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 3m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:32:57.984 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 5m ago 9h 959M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:32:57.984 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 5m ago 9h 1984M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:32:57.984 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 3m ago 9h 863M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:32:57.984 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 5m ago 9h 1185M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:32:57.985 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 5m ago 9h 24.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:32:57.985 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 3m ago 9h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:32:57.985 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 5m ago 9h 1438M 2234M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:32:57.985 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 5m ago 9h 1425M 2234M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:32:57.986 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 5m ago 9h 1107M 2234M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:32:57.986 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 5m ago 9h 1396M 2234M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:32:57.986 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 3m ago 9h 1224M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:32:57.986 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (8h) 3m ago 9h 1537M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:32:57.987 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 3m ago 9h 1373M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:32:57.987 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 3m ago 9h 1633M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:32:57.987 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 3m ago 9h 194M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:32:58.339 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:32:58.340 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:32:58.340 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:32:58.340 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:32:58.341 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:32:58.341 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:32:58.341 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:32:58.341 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:32:58.342 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:32:58.342 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:32:58.342 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:32:58.343 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:32:58.343 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:32:58.343 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:32:58.343 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:32:58.344 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:32:58.344 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:32:58.344 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:32:59.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:32:58 smithi083 ceph-mon[79930]: from='client.53130 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:32:59.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:32:58 smithi083 ceph-mon[79930]: pgmap v16339: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:59.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:32:58 smithi083 ceph-mon[79930]: from='client.63067 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:32:59.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:32:58 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1342792645' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:32:59.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:32:58 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:32:59.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:32:58 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:32:59.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:32:58 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:32:59.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:32:58 smithi083 ceph-mon[77242]: from='client.53130 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:32:59.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:32:58 smithi083 ceph-mon[77242]: pgmap v16339: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:59.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:32:58 smithi083 ceph-mon[77242]: from='client.63067 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:32:59.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:32:58 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1342792645' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:32:59.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:32:58 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:32:59.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:32:58 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:32:59.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:32:58 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:32:59.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:32:58 smithi153 ceph-mon[70044]: from='client.53130 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:32:59.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:32:58 smithi153 ceph-mon[70044]: pgmap v16339: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:32:59.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:32:58 smithi153 ceph-mon[70044]: from='client.63067 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:32:59.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:32:58 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1342792645' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:32:59.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:32:58 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:32:59.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:32:58 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:32:59.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:32:58 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:33:00.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:32:59 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:33:00.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:32:59 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:33:00.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:32:59 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:33:01.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:33:00 smithi153 ceph-mon[70044]: pgmap v16340: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:01.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:33:00 smithi083 ceph-mon[79930]: pgmap v16340: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:01.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:33:00 smithi083 ceph-mon[77242]: pgmap v16340: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:33:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:33:01] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T14:33:02.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:33:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:33:02.200Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:33:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:33:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:33:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:33:03.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:33:02 smithi153 ceph-mon[70044]: pgmap v16341: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:33:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:33:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:33:03.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:33:02 smithi083 ceph-mon[79930]: pgmap v16341: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:03.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:33:02 smithi083 ceph-mon[77242]: pgmap v16341: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:05.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:33:04 smithi153 ceph-mon[70044]: pgmap v16342: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:05.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:33:04 smithi083 ceph-mon[79930]: pgmap v16342: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:05.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:33:04 smithi083 ceph-mon[77242]: pgmap v16342: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:05.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:33:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:33:05.562Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:33:06.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:33:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:33:06.296Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:33:07.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:33:06 smithi153 ceph-mon[70044]: pgmap v16343: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:07.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:33:06 smithi083 ceph-mon[77242]: pgmap v16343: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:07.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:33:06 smithi083 ceph-mon[79930]: pgmap v16343: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:09.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:33:08 smithi153 ceph-mon[70044]: pgmap v16344: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:09.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:33:08 smithi083 ceph-mon[79930]: pgmap v16344: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:09.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:33:08 smithi083 ceph-mon[77242]: pgmap v16344: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:11.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:33:10 smithi153 ceph-mon[70044]: pgmap v16345: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:11.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:33:10 smithi083 ceph-mon[79930]: pgmap v16345: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:11.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:33:10 smithi083 ceph-mon[77242]: pgmap v16345: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:33:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:33:11] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T14:33:12.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:33:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:33:12.201Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:33:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:33:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:33:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:33:13.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:33:12 smithi153 ceph-mon[70044]: pgmap v16346: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:33:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:33:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:33:13.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:33:12 smithi083 ceph-mon[79930]: pgmap v16346: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:13.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:33:12 smithi083 ceph-mon[77242]: pgmap v16346: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:15.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:33:14 smithi153 ceph-mon[70044]: pgmap v16347: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:15.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:33:14 smithi083 ceph-mon[77242]: pgmap v16347: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:15.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:33:14 smithi083 ceph-mon[79930]: pgmap v16347: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:15.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:33:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:33:15.563Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:33:16.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:33:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:33:16.298Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:33:17.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:33:16 smithi153 ceph-mon[70044]: pgmap v16348: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:17.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:33:16 smithi083 ceph-mon[77242]: pgmap v16348: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:17.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:33:16 smithi083 ceph-mon[79930]: pgmap v16348: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:19.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:33:19 smithi153 ceph-mon[70044]: pgmap v16349: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:19.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:33:19 smithi083 ceph-mon[79930]: pgmap v16349: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:19.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:33:18 smithi083 ceph-mon[77242]: pgmap v16349: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:21.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:33:21 smithi153 ceph-mon[70044]: pgmap v16350: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:21.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:33:21 smithi083 ceph-mon[79930]: pgmap v16350: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:21.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:33:21 smithi083 ceph-mon[77242]: pgmap v16350: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:33:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:33:21] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T14:33:22.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:33:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:33:22.201Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:33:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:33:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:33:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:33:23.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:33:23 smithi153 ceph-mon[70044]: pgmap v16351: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:33:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:33:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:33:23.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:33:23 smithi083 ceph-mon[77242]: pgmap v16351: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:23.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:33:23 smithi083 ceph-mon[79930]: pgmap v16351: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:25.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:33:25 smithi153 ceph-mon[70044]: pgmap v16352: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:25.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:33:25 smithi083 ceph-mon[79930]: pgmap v16352: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:25.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:33:25 smithi083 ceph-mon[77242]: pgmap v16352: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:25.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:33:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:33:25.564Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:33:26.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:33:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:33:26.298Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:33:27.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:33:27 smithi153 ceph-mon[70044]: pgmap v16353: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:27.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:33:27 smithi083 ceph-mon[77242]: pgmap v16353: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:27.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:33:27 smithi083 ceph-mon[79930]: pgmap v16353: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:28.667 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:33:28.953 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:33:28.953 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 5m ago 9h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:33:28.953 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 3m ago 9h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:33:28.953 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 5m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:33:28.954 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 3m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:33:28.954 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 3m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:33:28.954 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 5m ago 9h 959M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:33:28.954 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 5m ago 9h 1984M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:33:28.955 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 3m ago 9h 863M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:33:28.955 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 5m ago 9h 1185M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:33:28.955 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 5m ago 9h 24.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:33:28.955 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 3m ago 9h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:33:28.955 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 5m ago 9h 1438M 2234M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:33:28.956 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 5m ago 9h 1425M 2234M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:33:28.956 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 5m ago 9h 1107M 2234M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:33:28.956 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 5m ago 9h 1396M 2234M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:33:28.956 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 3m ago 9h 1224M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:33:28.956 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 3m ago 9h 1537M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:33:28.957 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (8h) 3m ago 9h 1373M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:33:28.957 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 3m ago 9h 1633M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:33:28.957 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 3m ago 9h 194M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:33:29.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:33:29 smithi153 ceph-mon[70044]: pgmap v16354: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:29.309 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:33:29.310 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:33:29.310 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:33:29.310 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:33:29.310 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:33:29.311 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:33:29.311 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:33:29.311 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:33:29.311 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:33:29.311 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:33:29.312 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:33:29.312 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:33:29.312 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:33:29.312 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:33:29.312 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:33:29.313 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:33:29.313 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:33:29.313 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:33:29.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:33:29 smithi083 ceph-mon[79930]: pgmap v16354: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:29.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:33:29 smithi083 ceph-mon[77242]: pgmap v16354: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:30.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:33:30 smithi153 ceph-mon[70044]: from='client.63079 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:33:30.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:33:30 smithi153 ceph-mon[70044]: from='client.63085 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:33:30.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:33:30 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/452160255' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:33:30.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:33:30 smithi083 ceph-mon[79930]: from='client.63079 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:33:30.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:33:30 smithi083 ceph-mon[79930]: from='client.63085 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:33:30.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:33:30 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/452160255' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:33:30.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:33:30 smithi083 ceph-mon[77242]: from='client.63079 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:33:30.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:33:30 smithi083 ceph-mon[77242]: from='client.63085 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:33:30.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:33:30 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/452160255' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:33:31.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:33:31 smithi153 ceph-mon[70044]: pgmap v16355: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:31.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:33:31 smithi083 ceph-mon[79930]: pgmap v16355: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:31.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:33:31 smithi083 ceph-mon[77242]: pgmap v16355: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:33:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:33:31] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T14:33:32.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:33:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:33:32.202Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:33:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:33:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:33:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:33:33.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:33:33 smithi153 ceph-mon[70044]: pgmap v16356: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:33:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:33:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:33:33.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:33:33 smithi083 ceph-mon[77242]: pgmap v16356: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:33.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:33:33 smithi083 ceph-mon[79930]: pgmap v16356: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:35.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:33:35 smithi153 ceph-mon[70044]: pgmap v16357: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:35.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:33:35 smithi083 ceph-mon[79930]: pgmap v16357: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:35.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:33:35 smithi083 ceph-mon[77242]: pgmap v16357: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:35.967 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:33:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:33:35.564Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:33:36.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:33:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:33:36.299Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:33:37.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:33:37 smithi153 ceph-mon[70044]: pgmap v16358: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:37.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:33:37 smithi083 ceph-mon[77242]: pgmap v16358: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:37.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:33:37 smithi083 ceph-mon[79930]: pgmap v16358: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:39.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:33:39 smithi153 ceph-mon[70044]: pgmap v16359: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:39.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:33:39 smithi083 ceph-mon[77242]: pgmap v16359: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:39.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:33:39 smithi083 ceph-mon[79930]: pgmap v16359: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:33:41 smithi153 ceph-mon[70044]: pgmap v16360: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:41.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:33:41 smithi083 ceph-mon[79930]: pgmap v16360: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:41.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:33:41 smithi083 ceph-mon[77242]: pgmap v16360: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:33:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:33:41] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T14:33:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:33:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:33:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:33:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:33:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:33:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:33:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:33:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:33:42.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:33:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:33:42.202Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:33:42.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:33:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:33:42.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:33:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:33:42.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:33:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:33:42.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:33:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:33:42.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:33:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:33:42.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:33:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:33:42.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:33:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:33:42.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:33:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:33:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:33:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:33:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:33:43.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:33:43 smithi153 ceph-mon[70044]: pgmap v16361: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:33:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:33:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:33:43.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:33:43 smithi083 ceph-mon[79930]: pgmap v16361: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:43.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:33:43 smithi083 ceph-mon[77242]: pgmap v16361: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:45.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:33:45 smithi153 ceph-mon[70044]: pgmap v16362: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:45.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:33:45 smithi083 ceph-mon[79930]: pgmap v16362: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:45.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:33:45 smithi083 ceph-mon[77242]: pgmap v16362: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:45.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:33:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:33:45.565Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:33:46.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:33:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:33:46.299Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:33:47.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:33:47 smithi083 ceph-mon[79930]: pgmap v16363: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:47.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:33:47 smithi083 ceph-mon[77242]: pgmap v16363: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:47.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:33:47 smithi153 ceph-mon[70044]: pgmap v16363: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:49.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:33:49 smithi083 ceph-mon[79930]: pgmap v16364: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:49.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:33:49 smithi083 ceph-mon[77242]: pgmap v16364: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:49.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:33:49 smithi153 ceph-mon[70044]: pgmap v16364: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:51.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:33:51 smithi083 ceph-mon[77242]: pgmap v16365: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:51.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:33:51 smithi083 ceph-mon[79930]: pgmap v16365: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:51.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:33:51 smithi153 ceph-mon[70044]: pgmap v16365: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:33:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:33:51] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T14:33:52.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:33:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:33:52.203Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:33:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:33:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:33:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:33:53.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:33:53 smithi153 ceph-mon[70044]: pgmap v16366: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:33:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:33:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:33:53.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:33:53 smithi083 ceph-mon[79930]: pgmap v16366: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:53.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:33:53 smithi083 ceph-mon[77242]: pgmap v16366: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:55.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:33:55 smithi083 ceph-mon[79930]: pgmap v16367: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:55.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:33:55 smithi083 ceph-mon[77242]: pgmap v16367: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:55.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:33:55 smithi153 ceph-mon[70044]: pgmap v16367: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:55.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:33:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:33:55.565Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:33:56.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:33:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:33:56.300Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:33:57.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:33:57 smithi083 ceph-mon[79930]: pgmap v16368: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:57.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:33:57 smithi083 ceph-mon[77242]: pgmap v16368: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:57.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:33:57 smithi153 ceph-mon[70044]: pgmap v16368: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:59.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:33:59 smithi083 ceph-mon[79930]: pgmap v16369: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:59.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:33:59 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:33:59.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:33:59 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:33:59.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:33:59 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:33:59.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:33:59 smithi083 ceph-mon[77242]: pgmap v16369: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:59.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:33:59 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:33:59.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:33:59 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:33:59.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:33:59 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:33:59.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:33:59 smithi153 ceph-mon[70044]: pgmap v16369: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:33:59.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:33:59 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:33:59.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:33:59 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:33:59.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:33:59 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:33:59.651 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:33:59.944 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:33:59.945 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 6m ago 9h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:33:59.945 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 4m ago 9h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:33:59.945 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 6m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:33:59.945 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 4m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:33:59.946 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 4m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:33:59.946 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 6m ago 9h 959M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:33:59.946 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 6m ago 9h 1984M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:33:59.946 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 4m ago 9h 863M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:33:59.947 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 6m ago 9h 1185M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:33:59.947 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 6m ago 9h 24.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:33:59.947 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 4m ago 9h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:33:59.947 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 6m ago 9h 1438M 2234M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:33:59.948 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 6m ago 9h 1425M 2234M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:33:59.948 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 6m ago 9h 1107M 2234M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:33:59.948 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 6m ago 9h 1396M 2234M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:33:59.948 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 4m ago 9h 1224M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:33:59.949 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 4m ago 9h 1537M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:33:59.949 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 4m ago 9h 1373M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:33:59.949 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (8h) 4m ago 9h 1633M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:33:59.949 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 4m ago 9h 194M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:34:00.302 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:34:00.303 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:34:00.303 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:34:00.303 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:34:00.303 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:34:00.304 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:34:00.304 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:34:00.304 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:34:00.304 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:34:00.305 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:34:00.305 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:34:00.305 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:34:00.305 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:34:00.305 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:34:00.306 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:34:00.306 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:34:00.306 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:34:00.306 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:34:00.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:34:00 smithi083 ceph-mon[79930]: from='client.63097 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:34:00.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:34:00 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:34:00.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:34:00 smithi083 ceph-mon[79930]: pgmap v16370: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:34:00.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:34:00 smithi083 ceph-mon[79930]: from='client.63103 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:34:00.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:34:00 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3539495112' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:34:00.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:34:00 smithi083 ceph-mon[77242]: from='client.63097 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:34:00.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:34:00 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:34:00.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:34:00 smithi083 ceph-mon[77242]: pgmap v16370: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:34:00.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:34:00 smithi083 ceph-mon[77242]: from='client.63103 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:34:00.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:34:00 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3539495112' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:34:01.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:34:00 smithi153 ceph-mon[70044]: from='client.63097 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:34:01.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:34:00 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:34:01.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:34:00 smithi153 ceph-mon[70044]: pgmap v16370: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:34:01.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:34:00 smithi153 ceph-mon[70044]: from='client.63103 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:34:01.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:34:00 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3539495112' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:34:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:34:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:34:01] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T14:34:02.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:34:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:34:02.204Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:34:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:34:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:34:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:34:03.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:34:02 smithi083 ceph-mon[79930]: pgmap v16371: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:34:03.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:34:02 smithi083 ceph-mon[77242]: pgmap v16371: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:34:03.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:34:02 smithi153 ceph-mon[70044]: pgmap v16371: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:34:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:34:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:34:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:34:05.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:34:04 smithi083 ceph-mon[77242]: pgmap v16372: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:34:05.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:34:04 smithi083 ceph-mon[79930]: pgmap v16372: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:34:05.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:34:04 smithi153 ceph-mon[70044]: pgmap v16372: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:34:05.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:34:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:34:05.565Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:34:06.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:34:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:34:06.301Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:34:07.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:34:06 smithi083 ceph-mon[79930]: pgmap v16373: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:34:07.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:34:06 smithi083 ceph-mon[77242]: pgmap v16373: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:34:07.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:34:06 smithi153 ceph-mon[70044]: pgmap v16373: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:34:09.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:34:08 smithi083 ceph-mon[79930]: pgmap v16374: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:34:09.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:34:08 smithi083 ceph-mon[77242]: pgmap v16374: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:34:09.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:34:08 smithi153 ceph-mon[70044]: pgmap v16374: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:34:11.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:34:10 smithi083 ceph-mon[79930]: pgmap v16375: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:34:11.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:34:10 smithi083 ceph-mon[77242]: pgmap v16375: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:34:11.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:34:10 smithi153 ceph-mon[70044]: pgmap v16375: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:34:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:34:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:34:11] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T14:34:12.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:34:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:34:12.204Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:34:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:34:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:34:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:34:13.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:34:12 smithi083 ceph-mon[77242]: pgmap v16376: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:34:13.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:34:12 smithi083 ceph-mon[79930]: pgmap v16376: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:34:13.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:34:12 smithi153 ceph-mon[70044]: pgmap v16376: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:34:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:34:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:34:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:34:15.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:34:14 smithi083 ceph-mon[77242]: pgmap v16377: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:34:15.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:34:14 smithi083 ceph-mon[79930]: pgmap v16377: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:34:15.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:34:14 smithi153 ceph-mon[70044]: pgmap v16377: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:34:15.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:34:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:34:15.566Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:34:16.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:34:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:34:16.301Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:34:17.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:34:16 smithi083 ceph-mon[79930]: pgmap v16378: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:34:17.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:34:16 smithi083 ceph-mon[77242]: pgmap v16378: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:34:17.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:34:16 smithi153 ceph-mon[70044]: pgmap v16378: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:34:19.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:34:18 smithi083 ceph-mon[79930]: pgmap v16379: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:34:19.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:34:18 smithi083 ceph-mon[77242]: pgmap v16379: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:34:19.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:34:18 smithi153 ceph-mon[70044]: pgmap v16379: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:34:21.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:34:20 smithi083 ceph-mon[79930]: pgmap v16380: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:34:21.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:34:20 smithi083 ceph-mon[77242]: pgmap v16380: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:34:21.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:34:20 smithi153 ceph-mon[70044]: pgmap v16380: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:34:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:34:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:34:21] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T14:34:22.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:34:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:34:22.205Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:34:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:34:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:34:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:34:23.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:34:22 smithi083 ceph-mon[77242]: pgmap v16381: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:34:23.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:34:22 smithi083 ceph-mon[79930]: pgmap v16381: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:34:23.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:34:22 smithi153 ceph-mon[70044]: pgmap v16381: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:34:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:34:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:34:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:34:25.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:34:24 smithi083 ceph-mon[77242]: pgmap v16382: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:34:25.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:34:24 smithi083 ceph-mon[79930]: pgmap v16382: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:34:25.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:34:24 smithi153 ceph-mon[70044]: pgmap v16382: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:34:25.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:34:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:34:25.567Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:34:26.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:34:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:34:26.302Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:34:27.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:34:26 smithi083 ceph-mon[77242]: pgmap v16383: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:34:27.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:34:26 smithi083 ceph-mon[79930]: pgmap v16383: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:34:27.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:34:26 smithi153 ceph-mon[70044]: pgmap v16383: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:34:29.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:34:28 smithi083 ceph-mon[79930]: pgmap v16384: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:34:29.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:34:28 smithi083 ceph-mon[77242]: pgmap v16384: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:34:29.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:34:28 smithi153 ceph-mon[70044]: pgmap v16384: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:34:30.628 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:34:30.916 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:34:30.916 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 6m ago 9h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:34:30.916 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 4m ago 9h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:34:30.917 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 6m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:34:30.917 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 4m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:34:30.917 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 4m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:34:30.917 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 6m ago 9h 959M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:34:30.918 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 6m ago 9h 1984M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:34:30.918 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 4m ago 9h 863M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:34:30.918 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 6m ago 9h 1185M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:34:30.919 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 6m ago 9h 24.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:34:30.919 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 4m ago 9h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:34:30.919 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 6m ago 9h 1438M 2234M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:34:30.919 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 6m ago 9h 1425M 2234M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:34:30.919 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 6m ago 9h 1107M 2234M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:34:30.920 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 6m ago 9h 1396M 2234M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:34:30.920 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 4m ago 9h 1224M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:34:30.920 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 4m ago 9h 1537M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:34:30.920 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 4m ago 9h 1373M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:34:30.921 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 4m ago 9h 1633M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:34:30.921 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 4m ago 9h 194M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:34:31.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:34:30 smithi083 ceph-mon[79930]: pgmap v16385: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:34:31.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:34:30 smithi083 ceph-mon[77242]: pgmap v16385: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:34:31.275 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:34:31.275 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:34:31.275 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:34:31.275 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:34:31.276 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:34:31.276 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:34:31.276 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:34:31.276 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:34:31.277 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:34:31.277 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:34:31.277 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:34:31.277 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:34:31.278 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:34:31.278 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:34:31.278 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:34:31.278 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:34:31.278 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:34:31.279 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:34:31.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:34:30 smithi153 ceph-mon[70044]: pgmap v16385: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:34:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:34:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:34:31] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T14:34:32.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:34:31 smithi153 ceph-mon[70044]: from='client.53184 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:34:32.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:34:31 smithi153 ceph-mon[70044]: from='client.53190 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:34:32.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:34:31 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4068373439' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:34:32.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:34:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:34:32.205Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:34:32.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:34:31 smithi083 ceph-mon[77242]: from='client.53184 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:34:32.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:34:31 smithi083 ceph-mon[77242]: from='client.53190 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:34:32.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:34:31 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4068373439' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:34:32.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:34:31 smithi083 ceph-mon[79930]: from='client.53184 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:34:32.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:34:31 smithi083 ceph-mon[79930]: from='client.53190 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:34:32.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:34:31 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4068373439' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:34:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:34:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:34:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:34:33.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:34:32 smithi153 ceph-mon[70044]: pgmap v16386: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:34:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:34:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:34:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:34:33.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:34:32 smithi083 ceph-mon[79930]: pgmap v16386: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:34:33.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:34:32 smithi083 ceph-mon[77242]: pgmap v16386: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:34:35.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:34:34 smithi153 ceph-mon[70044]: pgmap v16387: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:34:35.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:34:34 smithi083 ceph-mon[77242]: pgmap v16387: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:34:35.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:34:34 smithi083 ceph-mon[79930]: pgmap v16387: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:34:35.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:34:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:34:35.567Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:34:36.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:34:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:34:36.303Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:34:37.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:34:36 smithi153 ceph-mon[70044]: pgmap v16388: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:34:37.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:34:36 smithi083 ceph-mon[79930]: pgmap v16388: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:34:37.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:34:36 smithi083 ceph-mon[77242]: pgmap v16388: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:34:39.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:34:38 smithi153 ceph-mon[70044]: pgmap v16389: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:34:39.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:34:38 smithi083 ceph-mon[79930]: pgmap v16389: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:34:39.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:34:38 smithi083 ceph-mon[77242]: pgmap v16389: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:34:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:34:40 smithi153 ceph-mon[70044]: pgmap v16390: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:34:41.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:34:40 smithi083 ceph-mon[79930]: pgmap v16390: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:34:41.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:34:40 smithi083 ceph-mon[77242]: pgmap v16390: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:34:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:34:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:34:41] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T14:34:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:34:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:34:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:34:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:34:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:34:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:34:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:34:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:34:42.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:34:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:34:42.207Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:34:42.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:34:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:34:42.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:34:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:34:42.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:34:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:34:42.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:34:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:34:42.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:34:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:34:42.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:34:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:34:42.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:34:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:34:42.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:34:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:34:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:34:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:34:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:34:43.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:34:42 smithi153 ceph-mon[70044]: pgmap v16391: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:34:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:34:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:34:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:34:43.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:34:42 smithi083 ceph-mon[79930]: pgmap v16391: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:34:43.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:34:42 smithi083 ceph-mon[77242]: pgmap v16391: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:34:45.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:34:44 smithi153 ceph-mon[70044]: pgmap v16392: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:34:45.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:34:44 smithi083 ceph-mon[79930]: pgmap v16392: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:34:45.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:34:44 smithi083 ceph-mon[77242]: pgmap v16392: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:34:45.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:34:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:34:45.568Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:34:46.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:34:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:34:46.303Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:34:47.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:34:46 smithi153 ceph-mon[70044]: pgmap v16393: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:34:47.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:34:46 smithi083 ceph-mon[79930]: pgmap v16393: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:34:47.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:34:46 smithi083 ceph-mon[77242]: pgmap v16393: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:34:49.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:34:48 smithi153 ceph-mon[70044]: pgmap v16394: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:34:49.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:34:48 smithi083 ceph-mon[79930]: pgmap v16394: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:34:49.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:34:48 smithi083 ceph-mon[77242]: pgmap v16394: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:34:51.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:34:51 smithi153 ceph-mon[70044]: pgmap v16395: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:34:51.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:34:51 smithi083 ceph-mon[79930]: pgmap v16395: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:34:51.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:34:51 smithi083 ceph-mon[77242]: pgmap v16395: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:34:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:34:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:34:51] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T14:34:52.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:34:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:34:52.207Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:34:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:34:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:34:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:34:53.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:34:53 smithi153 ceph-mon[70044]: pgmap v16396: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:34:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:34:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:34:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:34:53.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:34:53 smithi083 ceph-mon[79930]: pgmap v16396: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:34:53.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:34:53 smithi083 ceph-mon[77242]: pgmap v16396: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:34:55.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:34:55 smithi153 ceph-mon[70044]: pgmap v16397: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:34:55.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:34:55 smithi083 ceph-mon[79930]: pgmap v16397: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:34:55.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:34:55 smithi083 ceph-mon[77242]: pgmap v16397: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:34:55.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:34:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:34:55.569Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:34:56.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:34:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:34:56.303Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:34:57.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:34:57 smithi153 ceph-mon[70044]: pgmap v16398: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:34:57.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:34:57 smithi083 ceph-mon[79930]: pgmap v16398: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:34:57.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:34:57 smithi083 ceph-mon[77242]: pgmap v16398: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:34:59.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:34:59 smithi153 ceph-mon[70044]: pgmap v16399: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:34:59.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:34:59 smithi083 ceph-mon[79930]: pgmap v16399: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:34:59.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:34:59 smithi083 ceph-mon[77242]: pgmap v16399: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:35:00 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:35:00.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:35:00 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:35:00.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:35:00 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:35:00.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:35:00 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:35:00.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:35:00 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:35:00.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:35:00 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:35:00.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:35:00 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:35:00.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:35:00 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:35:00.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:35:00 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:35:01.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:35:01 smithi153 ceph-mon[70044]: pgmap v16400: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:01.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:35:01 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:35:01.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:35:01 smithi083 ceph-mon[77242]: pgmap v16400: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:01.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:35:01 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:35:01.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:35:01 smithi083 ceph-mon[79930]: pgmap v16400: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:01.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:35:01 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:35:01.598 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:35:01.885 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:35:01.886 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 7m ago 9h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:35:01.886 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 5m ago 9h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:35:01.886 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 7m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:35:01.886 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 5m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:35:01.887 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 5m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:35:01.887 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 7m ago 9h 959M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:35:01.887 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 7m ago 9h 1984M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:35:01.887 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 5m ago 9h 863M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:35:01.887 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 7m ago 9h 1185M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:35:01.888 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 7m ago 9h 24.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:35:01.888 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 5m ago 9h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:35:01.888 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 7m ago 9h 1438M 2234M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:35:01.888 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 7m ago 9h 1425M 2234M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:35:01.889 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 7m ago 9h 1107M 2234M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:35:01.889 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 7m ago 9h 1396M 2234M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:35:01.889 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 5m ago 9h 1224M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:35:01.889 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 5m ago 9h 1537M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:35:01.889 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 5m ago 9h 1373M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:35:01.890 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 5m ago 9h 1633M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:35:01.890 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 5m ago 9h 194M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:35:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:35:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:35:01] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T14:35:02.249 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:35:02.250 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:35:02.250 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:35:02.250 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:35:02.251 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:35:02.251 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:35:02.251 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:35:02.251 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:35:02.251 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:35:02.252 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:35:02.252 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:35:02.252 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:35:02.252 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:35:02.253 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:35:02.253 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:35:02.253 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:35:02.253 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:35:02.253 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:35:02.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:35:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:35:02.207Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:35:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:35:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:35:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:35:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:35:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:35:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:35:03.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:35:03 smithi153 ceph-mon[70044]: from='client.53202 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:35:03.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:35:03 smithi153 ceph-mon[70044]: pgmap v16401: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:03.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:35:03 smithi153 ceph-mon[70044]: from='client.63139 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:35:03.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:35:03 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2429582474' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:35:03.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:35:03 smithi083 ceph-mon[79930]: from='client.53202 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:35:03.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:35:03 smithi083 ceph-mon[79930]: pgmap v16401: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:03.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:35:03 smithi083 ceph-mon[79930]: from='client.63139 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:35:03.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:35:03 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2429582474' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:35:03.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:35:03 smithi083 ceph-mon[77242]: from='client.53202 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:35:03.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:35:03 smithi083 ceph-mon[77242]: pgmap v16401: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:03.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:35:03 smithi083 ceph-mon[77242]: from='client.63139 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:35:03.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:35:03 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2429582474' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:35:05.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:35:05 smithi153 ceph-mon[70044]: pgmap v16402: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:05.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:35:05 smithi083 ceph-mon[79930]: pgmap v16402: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:05.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:35:05 smithi083 ceph-mon[77242]: pgmap v16402: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:05.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:35:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:35:05.569Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:35:06.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:35:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:35:06.303Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:35:07.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:35:07 smithi153 ceph-mon[70044]: pgmap v16403: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:07.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:35:07 smithi083 ceph-mon[77242]: pgmap v16403: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:07.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:35:07 smithi083 ceph-mon[79930]: pgmap v16403: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:09.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:35:09 smithi153 ceph-mon[70044]: pgmap v16404: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:09.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:35:09 smithi083 ceph-mon[77242]: pgmap v16404: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:09.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:35:09 smithi083 ceph-mon[79930]: pgmap v16404: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:11.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:35:11 smithi153 ceph-mon[70044]: pgmap v16405: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:11.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:35:11 smithi083 ceph-mon[77242]: pgmap v16405: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:11.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:35:11 smithi083 ceph-mon[79930]: pgmap v16405: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:35:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:35:11] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T14:35:12.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:35:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:35:12.209Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:35:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:35:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:35:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:35:13.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:35:13 smithi153 ceph-mon[70044]: pgmap v16406: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:35:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:35:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:35:13.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:35:13 smithi083 ceph-mon[77242]: pgmap v16406: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:13.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:35:13 smithi083 ceph-mon[79930]: pgmap v16406: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:15.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:35:15 smithi153 ceph-mon[70044]: pgmap v16407: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:15.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:35:15 smithi083 ceph-mon[77242]: pgmap v16407: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:15.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:35:15 smithi083 ceph-mon[79930]: pgmap v16407: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:15.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:35:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:35:15.570Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:35:16.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:35:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:35:16.303Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:35:17.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:35:17 smithi153 ceph-mon[70044]: pgmap v16408: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:17.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:35:17 smithi083 ceph-mon[79930]: pgmap v16408: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:17.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:35:17 smithi083 ceph-mon[77242]: pgmap v16408: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:19.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:35:19 smithi153 ceph-mon[70044]: pgmap v16409: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:19.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:35:19 smithi083 ceph-mon[79930]: pgmap v16409: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:19.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:35:19 smithi083 ceph-mon[77242]: pgmap v16409: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:21.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:35:21 smithi153 ceph-mon[70044]: pgmap v16410: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:21.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:35:21 smithi083 ceph-mon[79930]: pgmap v16410: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:21.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:35:21 smithi083 ceph-mon[77242]: pgmap v16410: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:35:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:35:21] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T14:35:22.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:35:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:35:22.210Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:35:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:35:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:35:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:35:23.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:35:23 smithi153 ceph-mon[70044]: pgmap v16411: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:35:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:35:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:35:23.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:35:23 smithi083 ceph-mon[79930]: pgmap v16411: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:23.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:35:23 smithi083 ceph-mon[77242]: pgmap v16411: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:25.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:35:25 smithi083 ceph-mon[77242]: pgmap v16412: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:25.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:35:25 smithi083 ceph-mon[79930]: pgmap v16412: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:25.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:35:25 smithi153 ceph-mon[70044]: pgmap v16412: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:25.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:35:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:35:25.570Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:35:26.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:35:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:35:26.306Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:35:27.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:35:27 smithi083 ceph-mon[77242]: pgmap v16413: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:27.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:35:27 smithi083 ceph-mon[79930]: pgmap v16413: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:27.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:35:27 smithi153 ceph-mon[70044]: pgmap v16413: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:29.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:35:29 smithi083 ceph-mon[77242]: pgmap v16414: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:29.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:35:29 smithi083 ceph-mon[79930]: pgmap v16414: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:29.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:35:29 smithi153 ceph-mon[70044]: pgmap v16414: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:31.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:35:31 smithi083 ceph-mon[79930]: pgmap v16415: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:31.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:35:31 smithi083 ceph-mon[77242]: pgmap v16415: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:31.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:35:31 smithi153 ceph-mon[70044]: pgmap v16415: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:35:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:35:31] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T14:35:32.577 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:35:32.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:35:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:35:32.210Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:35:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:35:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:35:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:35:32.865 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:35:32.865 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 7m ago 9h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:35:32.866 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 5m ago 9h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:35:32.866 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 7m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:35:32.866 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 5m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:35:32.866 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 5m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:35:32.867 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 7m ago 9h 959M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:35:32.867 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 7m ago 9h 1984M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:35:32.867 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 5m ago 9h 863M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:35:32.867 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 7m ago 9h 1185M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:35:32.867 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 7m ago 9h 24.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:35:32.868 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 5m ago 9h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:35:32.868 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 7m ago 9h 1438M 2234M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:35:32.868 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 7m ago 9h 1425M 2234M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:35:32.868 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 7m ago 9h 1107M 2234M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:35:32.869 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 7m ago 9h 1396M 2234M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:35:32.869 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 5m ago 9h 1224M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:35:32.869 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 5m ago 9h 1537M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:35:32.869 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 5m ago 9h 1373M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:35:32.869 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 5m ago 9h 1633M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:35:32.870 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 5m ago 9h 194M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:35:33.220 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:35:33.221 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:35:33.221 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:35:33.221 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:35:33.221 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:35:33.222 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:35:33.222 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:35:33.222 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:35:33.222 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:35:33.223 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:35:33.223 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:35:33.223 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:35:33.223 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:35:33.223 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:35:33.224 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:35:33.224 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:35:33.224 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:35:33.224 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:35:33.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:35:33 smithi153 ceph-mon[70044]: pgmap v16416: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:35:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:35:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:35:33.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:35:33 smithi083 ceph-mon[77242]: pgmap v16416: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:33.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:35:33 smithi083 ceph-mon[79930]: pgmap v16416: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:34.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:35:34 smithi083 ceph-mon[77242]: from='client.53220 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:35:34.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:35:34 smithi083 ceph-mon[77242]: from='client.53226 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:35:34.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:35:34 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2519958205' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:35:34.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:35:34 smithi083 ceph-mon[79930]: from='client.53220 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:35:34.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:35:34 smithi083 ceph-mon[79930]: from='client.53226 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:35:34.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:35:34 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2519958205' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:35:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:35:34 smithi153 ceph-mon[70044]: from='client.53220 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:35:34.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:35:34 smithi153 ceph-mon[70044]: from='client.53226 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:35:34.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:35:34 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2519958205' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:35:35.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:35:35 smithi083 ceph-mon[77242]: pgmap v16417: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:35.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:35:35 smithi083 ceph-mon[79930]: pgmap v16417: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:35.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:35:35 smithi153 ceph-mon[70044]: pgmap v16417: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:35.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:35:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:35:35.571Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:35:36.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:35:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:35:36.305Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:35:37.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:35:37 smithi083 ceph-mon[79930]: pgmap v16418: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:37.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:35:37 smithi083 ceph-mon[77242]: pgmap v16418: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:37.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:35:37 smithi153 ceph-mon[70044]: pgmap v16418: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:39.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:35:39 smithi083 ceph-mon[79930]: pgmap v16419: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:39.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:35:39 smithi083 ceph-mon[77242]: pgmap v16419: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:39.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:35:39 smithi153 ceph-mon[70044]: pgmap v16419: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:41.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:35:41 smithi083 ceph-mon[77242]: pgmap v16420: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:41.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:35:41 smithi083 ceph-mon[79930]: pgmap v16420: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:35:41 smithi153 ceph-mon[70044]: pgmap v16420: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:35:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:35:41] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T14:35:42.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:35:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:35:42.211Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:35:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:35:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:35:42.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:35:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:35:42.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:35:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:35:42.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:35:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:35:42.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:35:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:35:42.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:35:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:35:42.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:35:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:35:42.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:35:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:35:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:35:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:35:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:35:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:35:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:35:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:35:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:35:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:35:42.559 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:35:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:35:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:35:43.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:35:43 smithi153 ceph-mon[70044]: pgmap v16421: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:35:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:35:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:35:43.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:35:43 smithi083 ceph-mon[79930]: pgmap v16421: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:43.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:35:43 smithi083 ceph-mon[77242]: pgmap v16421: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:45.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:35:45 smithi083 ceph-mon[77242]: pgmap v16422: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:45.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:35:45 smithi083 ceph-mon[79930]: pgmap v16422: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:45.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:35:45 smithi153 ceph-mon[70044]: pgmap v16422: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:45.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:35:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:35:45.572Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:35:46.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:35:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:35:46.305Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:35:47.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:35:47 smithi083 ceph-mon[79930]: pgmap v16423: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:47.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:35:47 smithi083 ceph-mon[77242]: pgmap v16423: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:47.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:35:47 smithi153 ceph-mon[70044]: pgmap v16423: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:49.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:35:49 smithi083 ceph-mon[79930]: pgmap v16424: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:49.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:35:49 smithi083 ceph-mon[77242]: pgmap v16424: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:49.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:35:49 smithi153 ceph-mon[70044]: pgmap v16424: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:51.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:35:51 smithi083 ceph-mon[77242]: pgmap v16425: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:51.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:35:51 smithi083 ceph-mon[79930]: pgmap v16425: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:51.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:35:51 smithi153 ceph-mon[70044]: pgmap v16425: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:35:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:35:51] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T14:35:52.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:35:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:35:52.212Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:35:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:35:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:35:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:35:53.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:35:53 smithi153 ceph-mon[70044]: pgmap v16426: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:35:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:35:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:35:53.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:35:53 smithi083 ceph-mon[79930]: pgmap v16426: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:53.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:35:53 smithi083 ceph-mon[77242]: pgmap v16426: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:55.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:35:55 smithi083 ceph-mon[77242]: pgmap v16427: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:55.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:35:55 smithi083 ceph-mon[79930]: pgmap v16427: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:55.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:35:55 smithi153 ceph-mon[70044]: pgmap v16427: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:55.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:35:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:35:55.573Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:35:56.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:35:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:35:56.306Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:35:57.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:35:57 smithi083 ceph-mon[77242]: pgmap v16428: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:57.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:35:57 smithi083 ceph-mon[79930]: pgmap v16428: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:57.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:35:57 smithi153 ceph-mon[70044]: pgmap v16428: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:59.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:35:59 smithi083 ceph-mon[77242]: pgmap v16429: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:59.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:35:59 smithi083 ceph-mon[79930]: pgmap v16429: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:35:59.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:35:59 smithi153 ceph-mon[70044]: pgmap v16429: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:00.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:36:00 smithi083 ceph-mon[79930]: pgmap v16430: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:00.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:36:00 smithi083 ceph-mon[77242]: pgmap v16430: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:36:00 smithi153 ceph-mon[70044]: pgmap v16430: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:01.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:36:01 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:36:01.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:36:01 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:36:01.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:36:01 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:36:01.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:36:01 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:36:01.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:36:01 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:36:01.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:36:01 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:36:01.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:36:01 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:36:01.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:36:01 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:36:01.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:36:01 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:36:01.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:36:01 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:36:01.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:36:01 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:36:01.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:36:01 smithi083 ceph-mon[77242]: Adjusting osd_memory_target on smithi083 to 2219M 2022-09-29T14:36:01.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:36:01 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:36:01.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:36:01 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:36:01.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:36:01 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:36:01.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:36:01 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:36:01.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:36:01 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:36:01.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:36:01 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:36:01.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:36:01 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:36:01.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:36:01 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:36:01.467 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:36:01 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:36:01.467 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:36:01 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:36:01.467 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:36:01 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:36:01.467 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:36:01 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:36:01.468 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:36:01 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:36:01.468 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:36:01 smithi083 ceph-mon[79930]: Adjusting osd_memory_target on smithi083 to 2219M 2022-09-29T14:36:01.468 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:36:01 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:36:01.468 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:36:01 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:36:01.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:36:01 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:36:01.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:36:01 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:36:01.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:36:01 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:36:01.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:36:01 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:36:01.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:36:01 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:36:01.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:36:01 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:36:01.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:36:01 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:36:01.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:36:01 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:36:01.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:36:01 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:36:01.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:36:01 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:36:01.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:36:01 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:36:01.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:36:01 smithi153 ceph-mon[70044]: Adjusting osd_memory_target on smithi083 to 2219M 2022-09-29T14:36:01.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:36:01 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:36:01.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:36:01 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:36:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:36:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:36:01] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T14:36:02.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:36:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:36:02.212Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:36:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:36:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:36:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:36:03.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:36:02 smithi083 ceph-mon[77242]: pgmap v16431: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:03.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:36:02 smithi083 ceph-mon[79930]: pgmap v16431: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:03.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:36:02 smithi153 ceph-mon[70044]: pgmap v16431: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:36:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:36:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:36:03.560 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:36:03.850 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:36:03.851 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 8m ago 9h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:36:03.851 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 6m ago 9h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:36:03.851 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 8m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:36:03.851 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 6m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:36:03.852 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 6m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:36:03.852 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 8m ago 9h 959M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:36:03.852 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 8m ago 9h 1984M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:36:03.852 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 6m ago 9h 863M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:36:03.852 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 8m ago 9h 1185M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:36:03.853 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 8m ago 9h 24.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:36:03.853 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 6m ago 9h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:36:03.853 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 8m ago 9h 1438M 2219M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:36:03.853 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 8m ago 9h 1425M 2219M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:36:03.854 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 8m ago 9h 1107M 2219M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:36:03.854 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 8m ago 9h 1396M 2219M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:36:03.854 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 6m ago 9h 1224M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:36:03.854 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 6m ago 9h 1537M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:36:03.854 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 6m ago 9h 1373M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:36:03.855 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 6m ago 9h 1633M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:36:03.855 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 6m ago 9h 194M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:36:04.205 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:36:04.205 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:36:04.206 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:36:04.206 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:36:04.206 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:36:04.206 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:36:04.206 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:36:04.207 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:36:04.207 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:36:04.207 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:36:04.207 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:36:04.207 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:36:04.208 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:36:04.208 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:36:04.208 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:36:04.208 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:36:04.209 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:36:04.209 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:36:05.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:36:04 smithi083 ceph-mon[79930]: from='client.53238 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:36:05.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:36:04 smithi083 ceph-mon[79930]: pgmap v16432: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:05.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:36:04 smithi083 ceph-mon[79930]: from='client.53244 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:36:05.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:36:04 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/733079031' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:36:05.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:36:04 smithi083 ceph-mon[77242]: from='client.53238 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:36:05.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:36:04 smithi083 ceph-mon[77242]: pgmap v16432: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:05.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:36:04 smithi083 ceph-mon[77242]: from='client.53244 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:36:05.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:36:04 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/733079031' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:36:05.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:36:04 smithi153 ceph-mon[70044]: from='client.53238 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:36:05.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:36:04 smithi153 ceph-mon[70044]: pgmap v16432: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:05.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:36:04 smithi153 ceph-mon[70044]: from='client.53244 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:36:05.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:36:04 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/733079031' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:36:05.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:36:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:36:05.574Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:36:06.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:36:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:36:06.306Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:36:07.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:36:06 smithi083 ceph-mon[77242]: pgmap v16433: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:07.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:36:06 smithi083 ceph-mon[79930]: pgmap v16433: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:07.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:36:06 smithi153 ceph-mon[70044]: pgmap v16433: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:09.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:36:08 smithi083 ceph-mon[79930]: pgmap v16434: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:09.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:36:08 smithi083 ceph-mon[77242]: pgmap v16434: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:09.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:36:08 smithi153 ceph-mon[70044]: pgmap v16434: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:11.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:36:10 smithi083 ceph-mon[79930]: pgmap v16435: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:11.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:36:10 smithi083 ceph-mon[77242]: pgmap v16435: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:11.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:36:10 smithi153 ceph-mon[70044]: pgmap v16435: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:36:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:36:11] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T14:36:12.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:36:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:36:12.213Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:36:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:36:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:36:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:36:13.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:36:12 smithi083 ceph-mon[77242]: pgmap v16436: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:13.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:36:12 smithi083 ceph-mon[79930]: pgmap v16436: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:13.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:36:12 smithi153 ceph-mon[70044]: pgmap v16436: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:36:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:36:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:36:15.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:36:14 smithi083 ceph-mon[77242]: pgmap v16437: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:15.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:36:14 smithi083 ceph-mon[79930]: pgmap v16437: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:15.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:36:14 smithi153 ceph-mon[70044]: pgmap v16437: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:15.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:36:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:36:15.574Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:36:16.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:36:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:36:16.307Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:36:17.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:36:16 smithi153 ceph-mon[70044]: pgmap v16438: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:17.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:36:16 smithi083 ceph-mon[77242]: pgmap v16438: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:17.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:36:16 smithi083 ceph-mon[79930]: pgmap v16438: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:19.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:36:18 smithi153 ceph-mon[70044]: pgmap v16439: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:19.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:36:18 smithi083 ceph-mon[79930]: pgmap v16439: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:19.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:36:18 smithi083 ceph-mon[77242]: pgmap v16439: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:21.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:36:20 smithi153 ceph-mon[70044]: pgmap v16440: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:21.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:36:20 smithi083 ceph-mon[79930]: pgmap v16440: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:21.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:36:20 smithi083 ceph-mon[77242]: pgmap v16440: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:36:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:36:21] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T14:36:22.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:36:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:36:22.213Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:36:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:36:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:36:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:36:23.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:36:22 smithi153 ceph-mon[70044]: pgmap v16441: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:36:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:36:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:36:23.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:36:22 smithi083 ceph-mon[79930]: pgmap v16441: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:23.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:36:22 smithi083 ceph-mon[77242]: pgmap v16441: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:25.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:36:24 smithi153 ceph-mon[70044]: pgmap v16442: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:25.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:36:24 smithi083 ceph-mon[77242]: pgmap v16442: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:25.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:36:24 smithi083 ceph-mon[79930]: pgmap v16442: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:25.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:36:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:36:25.575Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:36:26.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:36:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:36:26.307Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:36:27.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:36:26 smithi153 ceph-mon[70044]: pgmap v16443: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:27.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:36:26 smithi083 ceph-mon[77242]: pgmap v16443: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:27.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:36:26 smithi083 ceph-mon[79930]: pgmap v16443: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:29.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:36:28 smithi153 ceph-mon[70044]: pgmap v16444: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:29.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:36:28 smithi083 ceph-mon[79930]: pgmap v16444: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:29.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:36:28 smithi083 ceph-mon[77242]: pgmap v16444: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:31.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:36:30 smithi153 ceph-mon[70044]: pgmap v16445: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:31.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:36:30 smithi083 ceph-mon[79930]: pgmap v16445: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:31.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:36:30 smithi083 ceph-mon[77242]: pgmap v16445: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:36:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:36:31] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T14:36:32.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:36:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:36:32.214Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:36:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:36:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:36:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:36:33.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:36:32 smithi153 ceph-mon[70044]: pgmap v16446: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:36:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:36:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:36:33.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:36:32 smithi083 ceph-mon[79930]: pgmap v16446: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:33.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:36:32 smithi083 ceph-mon[77242]: pgmap v16446: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:34.530 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:36:34.817 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:36:34.818 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 8m ago 9h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:36:34.818 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 6m ago 9h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:36:34.818 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 8m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:36:34.819 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 6m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:36:34.819 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 6m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:36:34.819 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 8m ago 9h 959M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:36:34.819 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 8m ago 9h 1984M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:36:34.819 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 6m ago 9h 863M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:36:34.820 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 8m ago 9h 1185M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:36:34.820 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 8m ago 9h 24.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:36:34.820 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 6m ago 9h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:36:34.820 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 8m ago 9h 1438M 2219M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:36:34.820 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 8m ago 9h 1425M 2219M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:36:34.821 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 8m ago 9h 1107M 2219M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:36:34.821 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 8m ago 9h 1396M 2219M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:36:34.821 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 6m ago 9h 1224M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:36:34.821 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 6m ago 9h 1537M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:36:34.822 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 6m ago 9h 1373M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:36:34.822 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 6m ago 9h 1633M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:36:34.822 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 6m ago 9h 194M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:36:35.174 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:36:35.175 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:36:35.175 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:36:35.175 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:36:35.175 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:36:35.176 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:36:35.176 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:36:35.176 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:36:35.176 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:36:35.177 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:36:35.177 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:36:35.177 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:36:35.177 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:36:35.177 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:36:35.178 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:36:35.178 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:36:35.178 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:36:35.178 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:36:35.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:36:34 smithi153 ceph-mon[70044]: pgmap v16447: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:35.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:36:34 smithi083 ceph-mon[79930]: pgmap v16447: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:35.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:36:34 smithi083 ceph-mon[77242]: pgmap v16447: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:35.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:36:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:36:35.575Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:36:36.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:36:35 smithi153 ceph-mon[70044]: from='client.53256 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:36:36.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:36:35 smithi153 ceph-mon[70044]: from='client.53262 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:36:36.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:36:35 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2928117681' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:36:36.310 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:36:35 smithi083 ceph-mon[79930]: from='client.53256 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:36:36.310 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:36:35 smithi083 ceph-mon[79930]: from='client.53262 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:36:36.310 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:36:35 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2928117681' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:36:36.311 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:36:35 smithi083 ceph-mon[77242]: from='client.53256 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:36:36.311 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:36:35 smithi083 ceph-mon[77242]: from='client.53262 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:36:36.311 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:36:35 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2928117681' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:36:36.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:36:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:36:36.308Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:36:37.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:36:36 smithi153 ceph-mon[70044]: pgmap v16448: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:37.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:36:36 smithi083 ceph-mon[79930]: pgmap v16448: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:37.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:36:36 smithi083 ceph-mon[77242]: pgmap v16448: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:39.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:36:39 smithi153 ceph-mon[70044]: pgmap v16449: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:39.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:36:39 smithi083 ceph-mon[79930]: pgmap v16449: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:39.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:36:39 smithi083 ceph-mon[77242]: pgmap v16449: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:36:41 smithi153 ceph-mon[70044]: pgmap v16450: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:41.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:36:41 smithi083 ceph-mon[79930]: pgmap v16450: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:41.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:36:41 smithi083 ceph-mon[77242]: pgmap v16450: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:36:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:36:41] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T14:36:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:36:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:36:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:36:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:36:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:36:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:36:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:36:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:36:42.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:36:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:36:42.214Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:36:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:36:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:36:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:36:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:36:42.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:36:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:36:42.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:36:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:36:42.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:36:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:36:42.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:36:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:36:42.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:36:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:36:42.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:36:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:36:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:36:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:36:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:36:43.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:36:43 smithi153 ceph-mon[70044]: pgmap v16451: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:36:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:36:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:36:43.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:36:43 smithi083 ceph-mon[79930]: pgmap v16451: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:43.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:36:43 smithi083 ceph-mon[77242]: pgmap v16451: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:45.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:36:45 smithi153 ceph-mon[70044]: pgmap v16452: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:45.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:36:45 smithi083 ceph-mon[79930]: pgmap v16452: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:45.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:36:45 smithi083 ceph-mon[77242]: pgmap v16452: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:45.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:36:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:36:45.576Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:36:46.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:36:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:36:46.308Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:36:47.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:36:47 smithi153 ceph-mon[70044]: pgmap v16453: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:47.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:36:47 smithi083 ceph-mon[79930]: pgmap v16453: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:47.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:36:47 smithi083 ceph-mon[77242]: pgmap v16453: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:49.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:36:49 smithi153 ceph-mon[70044]: pgmap v16454: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:49.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:36:49 smithi083 ceph-mon[79930]: pgmap v16454: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:49.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:36:49 smithi083 ceph-mon[77242]: pgmap v16454: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:51.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:36:51 smithi153 ceph-mon[70044]: pgmap v16455: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:51.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:36:51 smithi083 ceph-mon[79930]: pgmap v16455: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:51.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:36:51 smithi083 ceph-mon[77242]: pgmap v16455: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:36:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:36:51] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T14:36:52.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:36:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:36:52.215Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:36:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:36:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:36:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:36:53.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:36:53 smithi153 ceph-mon[70044]: pgmap v16456: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:36:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:36:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:36:53.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:36:53 smithi083 ceph-mon[79930]: pgmap v16456: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:53.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:36:53 smithi083 ceph-mon[77242]: pgmap v16456: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:55.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:36:55 smithi153 ceph-mon[70044]: pgmap v16457: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:55.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:36:55 smithi083 ceph-mon[77242]: pgmap v16457: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:55.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:36:55 smithi083 ceph-mon[79930]: pgmap v16457: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:55.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:36:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:36:55.576Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:36:56.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:36:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:36:56.309Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:36:57.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:36:57 smithi153 ceph-mon[70044]: pgmap v16458: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:57.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:36:57 smithi083 ceph-mon[79930]: pgmap v16458: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:57.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:36:57 smithi083 ceph-mon[77242]: pgmap v16458: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:59.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:36:59 smithi153 ceph-mon[70044]: pgmap v16459: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:59.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:36:59 smithi083 ceph-mon[77242]: pgmap v16459: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:36:59.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:36:59 smithi083 ceph-mon[79930]: pgmap v16459: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:01.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:37:01 smithi153 ceph-mon[70044]: pgmap v16460: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:01.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:37:01 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:37:01.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:37:01 smithi083 ceph-mon[77242]: pgmap v16460: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:01.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:37:01 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:37:01.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:37:01 smithi083 ceph-mon[79930]: pgmap v16460: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:01.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:37:01 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:37:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:37:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:37:01] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T14:37:02.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:37:02 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:37:02.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:37:02 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:37:02.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:37:02 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:37:02.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:37:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:37:02.215Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:37:02.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:37:02 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:37:02.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:37:02 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:37:02.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:37:02 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:37:02.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:37:02 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:37:02.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:37:02 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:37:02.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:37:02 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:37:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:37:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:37:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:37:03.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:37:03 smithi153 ceph-mon[70044]: pgmap v16461: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:37:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:37:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:37:03.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:37:03 smithi083 ceph-mon[77242]: pgmap v16461: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:03.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:37:03 smithi083 ceph-mon[79930]: pgmap v16461: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:05.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:37:05 smithi153 ceph-mon[70044]: pgmap v16462: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:05.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:37:05 smithi083 ceph-mon[77242]: pgmap v16462: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:05.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:37:05 smithi083 ceph-mon[79930]: pgmap v16462: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:05.500 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:37:05.789 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:37:05.789 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 9m ago 9h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:37:05.790 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 7m ago 9h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:37:05.790 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 9m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:37:05.790 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 7m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:37:05.790 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 7m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:37:05.791 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 9m ago 9h 959M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:37:05.791 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 9m ago 9h 1984M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:37:05.791 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 7m ago 9h 863M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:37:05.791 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 9m ago 9h 1185M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:37:05.792 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 9m ago 9h 24.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:37:05.792 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 7m ago 9h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:37:05.792 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 9m ago 9h 1438M 2219M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:37:05.792 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 9m ago 9h 1425M 2219M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:37:05.793 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 9m ago 9h 1107M 2219M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:37:05.793 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 9m ago 9h 1396M 2219M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:37:05.793 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 7m ago 9h 1224M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:37:05.793 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 7m ago 9h 1537M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:37:05.793 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 7m ago 9h 1373M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:37:05.794 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 7m ago 9h 1633M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:37:05.794 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 7m ago 9h 194M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:37:05.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:37:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:37:05.576Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:37:06.146 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:37:06.146 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:37:06.146 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:37:06.147 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:37:06.147 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:37:06.147 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:37:06.147 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:37:06.148 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:37:06.148 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:37:06.148 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:37:06.148 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:37:06.148 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:37:06.149 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:37:06.149 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:37:06.149 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:37:06.149 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:37:06.149 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:37:06.150 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:37:06.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:37:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:37:06.309Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:37:07.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:37:07 smithi153 ceph-mon[70044]: from='client.63205 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:37:07.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:37:07 smithi153 ceph-mon[70044]: pgmap v16463: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:07.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:37:07 smithi153 ceph-mon[70044]: from='client.53280 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:37:07.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:37:07 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/289416851' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:37:07.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:37:07 smithi083 ceph-mon[79930]: from='client.63205 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:37:07.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:37:07 smithi083 ceph-mon[79930]: pgmap v16463: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:07.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:37:07 smithi083 ceph-mon[79930]: from='client.53280 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:37:07.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:37:07 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/289416851' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:37:07.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:37:07 smithi083 ceph-mon[77242]: from='client.63205 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:37:07.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:37:07 smithi083 ceph-mon[77242]: pgmap v16463: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:07.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:37:07 smithi083 ceph-mon[77242]: from='client.53280 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:37:07.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:37:07 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/289416851' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:37:09.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:37:09 smithi153 ceph-mon[70044]: pgmap v16464: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:09.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:37:09 smithi083 ceph-mon[79930]: pgmap v16464: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:09.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:37:09 smithi083 ceph-mon[77242]: pgmap v16464: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:11.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:37:11 smithi153 ceph-mon[70044]: pgmap v16465: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:11.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:37:11 smithi083 ceph-mon[77242]: pgmap v16465: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:11.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:37:11 smithi083 ceph-mon[79930]: pgmap v16465: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:37:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:37:11] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T14:37:12.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:37:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:37:12.217Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:37:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:37:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:37:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:37:13.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:37:13 smithi153 ceph-mon[70044]: pgmap v16466: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:37:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:37:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:37:13.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:37:13 smithi083 ceph-mon[79930]: pgmap v16466: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:13.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:37:13 smithi083 ceph-mon[77242]: pgmap v16466: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:15.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:37:15 smithi083 ceph-mon[79930]: pgmap v16467: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:15.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:37:15 smithi083 ceph-mon[77242]: pgmap v16467: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:15.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:37:15 smithi153 ceph-mon[70044]: pgmap v16467: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:15.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:37:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:37:15.578Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:37:16.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:37:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:37:16.310Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:37:17.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:37:17 smithi083 ceph-mon[79930]: pgmap v16468: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:17.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:37:17 smithi083 ceph-mon[77242]: pgmap v16468: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:17.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:37:17 smithi153 ceph-mon[70044]: pgmap v16468: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:19.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:37:19 smithi083 ceph-mon[79930]: pgmap v16469: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:19.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:37:19 smithi083 ceph-mon[77242]: pgmap v16469: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:19.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:37:19 smithi153 ceph-mon[70044]: pgmap v16469: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:21.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:37:21 smithi083 ceph-mon[79930]: pgmap v16470: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:21.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:37:21 smithi083 ceph-mon[77242]: pgmap v16470: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:21.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:37:21 smithi153 ceph-mon[70044]: pgmap v16470: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:37:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:37:21] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T14:37:22.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:37:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:37:22.217Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:37:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:37:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:37:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:37:23.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:37:23 smithi153 ceph-mon[70044]: pgmap v16471: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:37:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:37:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:37:23.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:37:23 smithi083 ceph-mon[79930]: pgmap v16471: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:23.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:37:23 smithi083 ceph-mon[77242]: pgmap v16471: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:25.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:37:25 smithi083 ceph-mon[79930]: pgmap v16472: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:25.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:37:25 smithi083 ceph-mon[77242]: pgmap v16472: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:25.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:37:25 smithi153 ceph-mon[70044]: pgmap v16472: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:25.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:37:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:37:25.578Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:37:26.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:37:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:37:26.310Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:37:27.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:37:27 smithi083 ceph-mon[79930]: pgmap v16473: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:27.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:37:27 smithi083 ceph-mon[77242]: pgmap v16473: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:27.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:37:27 smithi153 ceph-mon[70044]: pgmap v16473: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:29.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:37:29 smithi083 ceph-mon[79930]: pgmap v16474: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:29.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:37:29 smithi083 ceph-mon[77242]: pgmap v16474: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:29.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:37:29 smithi153 ceph-mon[70044]: pgmap v16474: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:31.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:37:31 smithi083 ceph-mon[79930]: pgmap v16475: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:31.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:37:31 smithi083 ceph-mon[77242]: pgmap v16475: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:31.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:37:31 smithi153 ceph-mon[70044]: pgmap v16475: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:37:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:37:31] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T14:37:32.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:37:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:37:32.218Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:37:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:37:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:37:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:37:33.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:37:33 smithi153 ceph-mon[70044]: pgmap v16476: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:37:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:37:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:37:33.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:37:33 smithi083 ceph-mon[79930]: pgmap v16476: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:33.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:37:33 smithi083 ceph-mon[77242]: pgmap v16476: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:35.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:37:35 smithi083 ceph-mon[79930]: pgmap v16477: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:35.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:37:35 smithi083 ceph-mon[77242]: pgmap v16477: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:35.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:37:35 smithi153 ceph-mon[70044]: pgmap v16477: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:35.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:37:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:37:35.579Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:37:36.472 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:37:36.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:37:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:37:36.311Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:37:36.765 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:37:36.765 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 9m ago 9h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:37:36.765 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 7m ago 9h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:37:36.765 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 9m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:37:36.766 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 7m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:37:36.766 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 7m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:37:36.766 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 9m ago 9h 959M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:37:36.766 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 9m ago 9h 1984M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:37:36.767 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 7m ago 9h 863M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:37:36.767 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 9m ago 9h 1185M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:37:36.767 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 9m ago 9h 24.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:37:36.767 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 7m ago 9h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:37:36.768 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 9m ago 9h 1438M 2219M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:37:36.768 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 9m ago 9h 1425M 2219M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:37:36.768 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 9m ago 9h 1107M 2219M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:37:36.768 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 9m ago 9h 1396M 2219M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:37:36.769 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 7m ago 9h 1224M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:37:36.769 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 7m ago 9h 1537M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:37:36.769 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 7m ago 9h 1373M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:37:36.769 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 7m ago 9h 1633M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:37:36.770 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 7m ago 9h 194M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:37:37.125 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:37:37.125 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:37:37.125 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:37:37.126 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:37:37.126 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:37:37.126 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:37:37.126 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:37:37.127 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:37:37.127 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:37:37.127 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:37:37.127 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:37:37.128 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:37:37.128 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:37:37.128 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:37:37.128 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:37:37.128 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:37:37.129 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:37:37.129 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:37:37.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:37:37 smithi083 ceph-mon[79930]: pgmap v16478: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:37.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:37:37 smithi083 ceph-mon[77242]: pgmap v16478: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:37.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:37:37 smithi153 ceph-mon[70044]: pgmap v16478: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:38.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:37:38 smithi083 ceph-mon[79930]: from='client.53292 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:37:38.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:37:38 smithi083 ceph-mon[79930]: from='client.63229 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:37:38.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:37:38 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2193641367' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:37:38.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:37:38 smithi083 ceph-mon[77242]: from='client.53292 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:37:38.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:37:38 smithi083 ceph-mon[77242]: from='client.63229 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:37:38.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:37:38 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2193641367' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:37:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:37:38 smithi153 ceph-mon[70044]: from='client.53292 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:37:38.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:37:38 smithi153 ceph-mon[70044]: from='client.63229 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:37:38.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:37:38 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2193641367' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:37:39.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:37:39 smithi083 ceph-mon[79930]: pgmap v16479: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:39.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:37:39 smithi083 ceph-mon[77242]: pgmap v16479: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:39.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:37:39 smithi153 ceph-mon[70044]: pgmap v16479: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:41.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:37:41 smithi083 ceph-mon[79930]: pgmap v16480: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:41.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:37:41 smithi083 ceph-mon[77242]: pgmap v16480: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:37:41 smithi153 ceph-mon[70044]: pgmap v16480: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:37:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:37:41] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T14:37:42.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:37:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:37:42.219Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:37:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:37:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:37:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:37:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:37:42.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:37:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:37:42.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:37:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:37:42.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:37:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:37:42.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:37:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:37:42.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:37:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:37:42.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:37:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:37:42.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:37:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:37:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:37:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:37:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:37:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:37:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:37:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:37:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:37:42.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:37:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:37:43.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:37:43 smithi153 ceph-mon[70044]: pgmap v16481: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:37:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:37:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:37:43.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:37:43 smithi083 ceph-mon[77242]: pgmap v16481: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:43.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:37:43 smithi083 ceph-mon[79930]: pgmap v16481: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:45.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:37:45 smithi083 ceph-mon[79930]: pgmap v16482: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:45.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:37:45 smithi083 ceph-mon[77242]: pgmap v16482: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:45.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:37:45 smithi153 ceph-mon[70044]: pgmap v16482: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:45.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:37:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:37:45.580Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:37:46.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:37:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:37:46.312Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:37:47.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:37:47 smithi083 ceph-mon[79930]: pgmap v16483: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:47.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:37:47 smithi083 ceph-mon[77242]: pgmap v16483: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:47.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:37:47 smithi153 ceph-mon[70044]: pgmap v16483: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:49.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:37:49 smithi083 ceph-mon[79930]: pgmap v16484: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:49.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:37:49 smithi083 ceph-mon[77242]: pgmap v16484: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:49.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:37:49 smithi153 ceph-mon[70044]: pgmap v16484: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:51.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:37:51 smithi083 ceph-mon[79930]: pgmap v16485: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:51.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:37:51 smithi083 ceph-mon[77242]: pgmap v16485: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:51.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:37:51 smithi153 ceph-mon[70044]: pgmap v16485: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:37:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:37:51] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T14:37:52.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:37:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:37:52.219Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:37:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:37:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:37:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:37:53.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:37:53 smithi153 ceph-mon[70044]: pgmap v16486: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:37:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:37:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:37:53.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:37:53 smithi083 ceph-mon[77242]: pgmap v16486: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:53.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:37:53 smithi083 ceph-mon[79930]: pgmap v16486: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:55.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:37:55 smithi083 ceph-mon[79930]: pgmap v16487: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:55.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:37:55 smithi083 ceph-mon[77242]: pgmap v16487: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:55.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:37:55 smithi153 ceph-mon[70044]: pgmap v16487: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:55.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:37:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:37:55.580Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:37:56.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:37:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:37:56.313Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:37:57.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:37:57 smithi083 ceph-mon[79930]: pgmap v16488: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:57.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:37:57 smithi083 ceph-mon[77242]: pgmap v16488: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:57.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:37:57 smithi153 ceph-mon[70044]: pgmap v16488: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:59.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:37:59 smithi083 ceph-mon[79930]: pgmap v16489: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:59.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:37:59 smithi083 ceph-mon[77242]: pgmap v16489: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:37:59.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:37:59 smithi153 ceph-mon[70044]: pgmap v16489: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:01.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:38:01 smithi083 ceph-mon[79930]: pgmap v16490: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:01.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:38:01 smithi083 ceph-mon[77242]: pgmap v16490: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:01.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:38:01 smithi153 ceph-mon[70044]: pgmap v16490: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:38:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:38:01] "GET /metrics HTTP/1.1" 200 225246 "" "Prometheus/2.33.4" 2022-09-29T14:38:02.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:38:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:38:02.220Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:38:02.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:38:02 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:38:02.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:38:02 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:38:02.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:38:02 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:38:02.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:38:02 smithi083 ceph-mon[79930]: pgmap v16491: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:02.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:38:02 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:38:02.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:38:02 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:38:02.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:38:02 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:38:02.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:38:02 smithi083 ceph-mon[77242]: pgmap v16491: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:02.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:38:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:38:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:38:02.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:38:02 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:38:02.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:38:02 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:38:02.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:38:02 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:38:02.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:38:02 smithi153 ceph-mon[70044]: pgmap v16491: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:38:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:38:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:38:03.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:38:03 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:38:03.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:38:03 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:38:03.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:38:03 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:38:03.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:38:03 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:38:03.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:38:03 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:38:03.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:38:03 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:38:03.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:38:03 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:38:03.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:38:03 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:38:03.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:38:03 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:38:03.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:38:03 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:38:03.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:38:03 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:38:03.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:38:03 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:38:04.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:38:04 smithi083 ceph-mon[79930]: pgmap v16492: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:04.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:38:04 smithi083 ceph-mon[77242]: pgmap v16492: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:38:04 smithi153 ceph-mon[70044]: pgmap v16492: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:05.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:38:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:38:05.579Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:38:06.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:38:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:38:06.313Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:38:07.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:38:06 smithi083 ceph-mon[79930]: pgmap v16493: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:07.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:38:06 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:38:07.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:38:06 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:38:07.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:38:06 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:38:07.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:38:06 smithi083 ceph-mon[77242]: pgmap v16493: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:07.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:38:06 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:38:07.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:38:06 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:38:07.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:38:06 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:38:07.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:38:06 smithi153 ceph-mon[70044]: pgmap v16493: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:07.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:38:06 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:38:07.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:38:06 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:38:07.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:38:06 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:38:07.439 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:38:07.726 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:38:07.727 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 1s ago 9h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:38:07.727 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 8m ago 9h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:38:07.727 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 1s ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:38:07.727 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 8m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:38:07.728 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 8m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:38:07.728 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 1s ago 9h 966M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:38:07.728 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 1s ago 9h 2019M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:38:07.728 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 8m ago 9h 863M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:38:07.729 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 1s ago 9h 1208M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:38:07.729 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 1s ago 9h 23.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:38:07.729 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 8m ago 9h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:38:07.729 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 1s ago 9h 1464M 2219M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:38:07.729 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 1s ago 9h 1451M 2219M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:38:07.730 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 1s ago 9h 1128M 2219M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:38:07.730 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 1s ago 9h 1422M 2219M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:38:07.730 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 8m ago 9h 1224M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:38:07.730 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 8m ago 9h 1537M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:38:07.730 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 8m ago 9h 1373M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:38:07.731 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 8m ago 9h 1633M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:38:07.731 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 8m ago 9h 194M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:38:08.084 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:38:08.085 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:38:08.085 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:38:08.085 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:38:08.085 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:38:08.085 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:38:08.086 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:38:08.086 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:38:08.086 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:38:08.086 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:38:08.087 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:38:08.087 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:38:08.087 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:38:08.087 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:38:08.087 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:38:08.088 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:38:08.088 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:38:08.088 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:38:09.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:38:08 smithi153 ceph-mon[70044]: from='client.53310 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:38:09.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:38:08 smithi153 ceph-mon[70044]: from='client.63247 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:38:09.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:38:08 smithi153 ceph-mon[70044]: pgmap v16494: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:09.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:38:08 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1663836250' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:38:09.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:38:08 smithi083 ceph-mon[79930]: from='client.53310 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:38:09.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:38:08 smithi083 ceph-mon[79930]: from='client.63247 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:38:09.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:38:08 smithi083 ceph-mon[79930]: pgmap v16494: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:09.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:38:08 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1663836250' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:38:09.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:38:08 smithi083 ceph-mon[77242]: from='client.53310 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:38:09.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:38:08 smithi083 ceph-mon[77242]: from='client.63247 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:38:09.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:38:08 smithi083 ceph-mon[77242]: pgmap v16494: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:09.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:38:08 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1663836250' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:38:11.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:38:10 smithi153 ceph-mon[70044]: pgmap v16495: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:11.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:38:10 smithi083 ceph-mon[79930]: pgmap v16495: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:11.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:38:10 smithi083 ceph-mon[77242]: pgmap v16495: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:38:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:38:11] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T14:38:12.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:38:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:38:12.220Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:38:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:38:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:38:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:38:13.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:38:12 smithi153 ceph-mon[70044]: pgmap v16496: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:38:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:38:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:38:13.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:38:12 smithi083 ceph-mon[79930]: pgmap v16496: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:13.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:38:12 smithi083 ceph-mon[77242]: pgmap v16496: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:15.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:38:14 smithi153 ceph-mon[70044]: pgmap v16497: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:15.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:38:14 smithi083 ceph-mon[79930]: pgmap v16497: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:15.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:38:14 smithi083 ceph-mon[77242]: pgmap v16497: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:15.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:38:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:38:15.580Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:38:16.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:38:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:38:16.314Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:38:17.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:38:16 smithi153 ceph-mon[70044]: pgmap v16498: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:17.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:38:16 smithi083 ceph-mon[77242]: pgmap v16498: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:17.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:38:16 smithi083 ceph-mon[79930]: pgmap v16498: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:19.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:38:18 smithi153 ceph-mon[70044]: pgmap v16499: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:19.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:38:18 smithi083 ceph-mon[79930]: pgmap v16499: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:19.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:38:18 smithi083 ceph-mon[77242]: pgmap v16499: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:21.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:38:20 smithi153 ceph-mon[70044]: pgmap v16500: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:21.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:38:20 smithi083 ceph-mon[79930]: pgmap v16500: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:21.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:38:20 smithi083 ceph-mon[77242]: pgmap v16500: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:38:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:38:21] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T14:38:22.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:38:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:38:22.221Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:38:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:38:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:38:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:38:23.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:38:22 smithi153 ceph-mon[70044]: pgmap v16501: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:38:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:38:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:38:23.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:38:22 smithi083 ceph-mon[79930]: pgmap v16501: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:23.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:38:22 smithi083 ceph-mon[77242]: pgmap v16501: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:25.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:38:24 smithi153 ceph-mon[70044]: pgmap v16502: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:25.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:38:24 smithi083 ceph-mon[79930]: pgmap v16502: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:25.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:38:24 smithi083 ceph-mon[77242]: pgmap v16502: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:25.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:38:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:38:25.581Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:38:26.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:38:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:38:26.315Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:38:27.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:38:26 smithi153 ceph-mon[70044]: pgmap v16503: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:27.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:38:26 smithi083 ceph-mon[79930]: pgmap v16503: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:27.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:38:26 smithi083 ceph-mon[77242]: pgmap v16503: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:29.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:38:28 smithi153 ceph-mon[70044]: pgmap v16504: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:29.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:38:28 smithi083 ceph-mon[79930]: pgmap v16504: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:29.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:38:28 smithi083 ceph-mon[77242]: pgmap v16504: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:31.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:38:31 smithi153 ceph-mon[70044]: pgmap v16505: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:31.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:38:31 smithi083 ceph-mon[79930]: pgmap v16505: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:31.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:38:31 smithi083 ceph-mon[77242]: pgmap v16505: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:38:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:38:31] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T14:38:32.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:38:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:38:32.222Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:38:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:38:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:38:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:38:33.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:38:33 smithi153 ceph-mon[70044]: pgmap v16506: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:38:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:38:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:38:33.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:38:33 smithi083 ceph-mon[77242]: pgmap v16506: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:33.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:38:33 smithi083 ceph-mon[79930]: pgmap v16506: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:35.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:38:35 smithi153 ceph-mon[70044]: pgmap v16507: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:35.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:38:35 smithi083 ceph-mon[79930]: pgmap v16507: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:35.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:38:35 smithi083 ceph-mon[77242]: pgmap v16507: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:35.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:38:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:38:35.582Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:38:36.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:38:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:38:36.315Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:38:37.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:38:37 smithi153 ceph-mon[70044]: pgmap v16508: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:37.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:38:37 smithi083 ceph-mon[79930]: pgmap v16508: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:37.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:38:37 smithi083 ceph-mon[77242]: pgmap v16508: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:38.408 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:38:38.694 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:38:38.695 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 32s ago 9h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:38:38.695 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 8m ago 9h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:38:38.695 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 32s ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:38:38.695 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 8m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:38:38.696 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 8m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:38:38.696 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 32s ago 9h 966M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:38:38.696 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 32s ago 9h 2019M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:38:38.696 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 8m ago 9h 863M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:38:38.696 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 32s ago 9h 1208M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:38:38.697 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 32s ago 9h 23.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:38:38.697 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 8m ago 9h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:38:38.697 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 32s ago 9h 1464M 2219M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:38:38.697 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 32s ago 9h 1451M 2219M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:38:38.697 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 32s ago 9h 1128M 2219M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:38:38.698 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 32s ago 9h 1422M 2219M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:38:38.698 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 8m ago 9h 1224M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:38:38.698 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 8m ago 9h 1537M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:38:38.698 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 8m ago 9h 1373M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:38:38.698 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 8m ago 9h 1633M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:38:38.699 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 8m ago 9h 194M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:38:39.051 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:38:39.052 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:38:39.052 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:38:39.052 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:38:39.052 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:38:39.052 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:38:39.053 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:38:39.053 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:38:39.053 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:38:39.053 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:38:39.053 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:38:39.054 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:38:39.054 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:38:39.054 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:38:39.054 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:38:39.055 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:38:39.055 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:38:39.055 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:38:39.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:38:39 smithi153 ceph-mon[70044]: pgmap v16509: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:39.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:38:39 smithi083 ceph-mon[79930]: pgmap v16509: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:39.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:38:39 smithi083 ceph-mon[77242]: pgmap v16509: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:38:40 smithi153 ceph-mon[70044]: from='client.53328 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:38:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:38:40 smithi153 ceph-mon[70044]: from='client.63265 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:38:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:38:40 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1265460760' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:38:40.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:38:40 smithi083 ceph-mon[77242]: from='client.53328 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:38:40.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:38:40 smithi083 ceph-mon[77242]: from='client.63265 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:38:40.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:38:40 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1265460760' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:38:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:38:40 smithi083 ceph-mon[79930]: from='client.53328 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:38:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:38:40 smithi083 ceph-mon[79930]: from='client.63265 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:38:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:38:40 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1265460760' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:38:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:38:41 smithi153 ceph-mon[70044]: pgmap v16510: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:41.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:38:41 smithi083 ceph-mon[79930]: pgmap v16510: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:41.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:38:41 smithi083 ceph-mon[77242]: pgmap v16510: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:38:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:38:41] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T14:38:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:38:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:38:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:38:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:38:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:38:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:38:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:38:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:38:42.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:38:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:38:42.222Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:38:42.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:38:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:38:42.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:38:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:38:42.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:38:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:38:42.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:38:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:38:42.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:38:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:38:42.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:38:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:38:42.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:38:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:38:42.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:38:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:38:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:38:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:38:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:38:43.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:38:43 smithi153 ceph-mon[70044]: pgmap v16511: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:38:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:38:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:38:43.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:38:43 smithi083 ceph-mon[79930]: pgmap v16511: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:43.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:38:43 smithi083 ceph-mon[77242]: pgmap v16511: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:45.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:38:45 smithi153 ceph-mon[70044]: pgmap v16512: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:45.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:38:45 smithi083 ceph-mon[79930]: pgmap v16512: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:45.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:38:45 smithi083 ceph-mon[77242]: pgmap v16512: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:45.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:38:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:38:45.582Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:38:46.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:38:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:38:46.316Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:38:47.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:38:47 smithi153 ceph-mon[70044]: pgmap v16513: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:47.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:38:47 smithi083 ceph-mon[79930]: pgmap v16513: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:47.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:38:47 smithi083 ceph-mon[77242]: pgmap v16513: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:49.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:38:49 smithi153 ceph-mon[70044]: pgmap v16514: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:49.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:38:49 smithi083 ceph-mon[79930]: pgmap v16514: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:49.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:38:49 smithi083 ceph-mon[77242]: pgmap v16514: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:51.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:38:51 smithi153 ceph-mon[70044]: pgmap v16515: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:51.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:38:51 smithi083 ceph-mon[79930]: pgmap v16515: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:51.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:38:51 smithi083 ceph-mon[77242]: pgmap v16515: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:38:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:38:51] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T14:38:52.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:38:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:38:52.222Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:38:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:38:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:38:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:38:53.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:38:53 smithi153 ceph-mon[70044]: pgmap v16516: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:38:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:38:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:38:53.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:38:53 smithi083 ceph-mon[79930]: pgmap v16516: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:53.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:38:53 smithi083 ceph-mon[77242]: pgmap v16516: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:55.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:38:55 smithi153 ceph-mon[70044]: pgmap v16517: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:55.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:38:55 smithi083 ceph-mon[79930]: pgmap v16517: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:55.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:38:55 smithi083 ceph-mon[77242]: pgmap v16517: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:55.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:38:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:38:55.583Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:38:56.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:38:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:38:56.317Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:38:57.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:38:57 smithi153 ceph-mon[70044]: pgmap v16518: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:57.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:38:57 smithi083 ceph-mon[79930]: pgmap v16518: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:57.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:38:57 smithi083 ceph-mon[77242]: pgmap v16518: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:59.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:38:59 smithi153 ceph-mon[70044]: pgmap v16519: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:59.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:38:59 smithi083 ceph-mon[79930]: pgmap v16519: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:38:59.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:38:59 smithi083 ceph-mon[77242]: pgmap v16519: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:01.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:39:01 smithi153 ceph-mon[70044]: pgmap v16520: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:01.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:39:01 smithi083 ceph-mon[79930]: pgmap v16520: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:01.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:39:01 smithi083 ceph-mon[77242]: pgmap v16520: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:39:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:39:01] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T14:39:02.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:39:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:39:02.223Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:39:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:39:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:39:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:39:03.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:39:03 smithi153 ceph-mon[70044]: pgmap v16521: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:39:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:39:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:39:03.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:39:03 smithi083 ceph-mon[77242]: pgmap v16521: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:03.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:39:03 smithi083 ceph-mon[79930]: pgmap v16521: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:05.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:39:05 smithi083 ceph-mon[77242]: pgmap v16522: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:05.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:39:05 smithi083 ceph-mon[79930]: pgmap v16522: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:05.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:39:05 smithi153 ceph-mon[70044]: pgmap v16522: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:05.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:39:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:39:05.584Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:39:06.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:39:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:39:06.317Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:39:07.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:39:07 smithi083 ceph-mon[79930]: pgmap v16523: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:07.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:39:07 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:39:07.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:39:07 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:39:07.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:39:07 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:39:07.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:39:07 smithi083 ceph-mon[77242]: pgmap v16523: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:07.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:39:07 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:39:07.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:39:07 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:39:07.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:39:07 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:39:07.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:39:07 smithi153 ceph-mon[70044]: pgmap v16523: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:07.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:39:07 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:39:07.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:39:07 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:39:07.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:39:07 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:39:08.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:39:08 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:39:08.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:39:08 smithi083 ceph-mon[77242]: pgmap v16524: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:08.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:39:08 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:39:08.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:39:08 smithi083 ceph-mon[79930]: pgmap v16524: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:08.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:39:08 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:39:08.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:39:08 smithi153 ceph-mon[70044]: pgmap v16524: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:09.377 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:39:09.666 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:39:09.666 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 63s ago 9h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:39:09.666 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 9m ago 9h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:39:09.667 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 63s ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:39:09.667 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 9m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:39:09.667 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 9m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:39:09.667 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 63s ago 9h 966M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:39:09.668 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 63s ago 9h 2019M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:39:09.668 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 9m ago 9h 863M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:39:09.668 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 63s ago 9h 1208M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:39:09.668 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 63s ago 9h 23.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:39:09.668 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 9m ago 9h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:39:09.669 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 63s ago 9h 1464M 2219M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:39:09.669 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 63s ago 9h 1451M 2219M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:39:09.669 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 63s ago 9h 1128M 2219M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:39:09.669 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 63s ago 9h 1422M 2219M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:39:09.669 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 9m ago 9h 1224M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:39:09.670 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 9m ago 9h 1537M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:39:09.670 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 9m ago 9h 1373M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:39:09.670 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 9m ago 9h 1633M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:39:09.670 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 9m ago 9h 194M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:39:10.023 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:39:10.023 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:39:10.024 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:39:10.024 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:39:10.024 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:39:10.024 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:39:10.025 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:39:10.025 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:39:10.025 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:39:10.025 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:39:10.026 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:39:10.026 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:39:10.026 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:39:10.026 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:39:10.026 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:39:10.027 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:39:10.027 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:39:10.027 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:39:11.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:39:10 smithi083 ceph-mon[79930]: from='client.53346 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:39:11.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:39:10 smithi083 ceph-mon[79930]: from='client.63283 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:39:11.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:39:10 smithi083 ceph-mon[79930]: pgmap v16525: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:11.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:39:10 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2956992272' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:39:11.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:39:10 smithi083 ceph-mon[77242]: from='client.53346 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:39:11.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:39:10 smithi083 ceph-mon[77242]: from='client.63283 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:39:11.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:39:10 smithi083 ceph-mon[77242]: pgmap v16525: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:11.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:39:10 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2956992272' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:39:11.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:39:10 smithi153 ceph-mon[70044]: from='client.53346 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:39:11.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:39:10 smithi153 ceph-mon[70044]: from='client.63283 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:39:11.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:39:10 smithi153 ceph-mon[70044]: pgmap v16525: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:11.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:39:10 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2956992272' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:39:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:39:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:39:11] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T14:39:12.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:39:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:39:12.224Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:39:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:39:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:39:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:39:13.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:39:12 smithi153 ceph-mon[70044]: pgmap v16526: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:39:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:39:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:39:13.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:39:12 smithi083 ceph-mon[79930]: pgmap v16526: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:13.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:39:12 smithi083 ceph-mon[77242]: pgmap v16526: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:15.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:39:14 smithi153 ceph-mon[70044]: pgmap v16527: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:15.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:39:14 smithi083 ceph-mon[79930]: pgmap v16527: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:15.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:39:14 smithi083 ceph-mon[77242]: pgmap v16527: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:15.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:39:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:39:15.584Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:39:16.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:39:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:39:16.319Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:39:17.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:39:16 smithi153 ceph-mon[70044]: pgmap v16528: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:17.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:39:16 smithi083 ceph-mon[79930]: pgmap v16528: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:17.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:39:16 smithi083 ceph-mon[77242]: pgmap v16528: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:19.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:39:18 smithi153 ceph-mon[70044]: pgmap v16529: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:19.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:39:18 smithi083 ceph-mon[79930]: pgmap v16529: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:19.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:39:18 smithi083 ceph-mon[77242]: pgmap v16529: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:21.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:39:20 smithi153 ceph-mon[70044]: pgmap v16530: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:21.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:39:20 smithi083 ceph-mon[79930]: pgmap v16530: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:21.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:39:20 smithi083 ceph-mon[77242]: pgmap v16530: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:39:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:39:21] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T14:39:22.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:39:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:39:22.225Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:39:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:39:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:39:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:39:23.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:39:22 smithi153 ceph-mon[70044]: pgmap v16531: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:39:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:39:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:39:23.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:39:22 smithi083 ceph-mon[77242]: pgmap v16531: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:23.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:39:22 smithi083 ceph-mon[79930]: pgmap v16531: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:25.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:39:25 smithi153 ceph-mon[70044]: pgmap v16532: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:25.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:39:25 smithi083 ceph-mon[79930]: pgmap v16532: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:25.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:39:25 smithi083 ceph-mon[77242]: pgmap v16532: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:25.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:39:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:39:25.585Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:39:26.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:39:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:39:26.319Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:39:27.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:39:27 smithi153 ceph-mon[70044]: pgmap v16533: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:27.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:39:27 smithi083 ceph-mon[77242]: pgmap v16533: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:27.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:39:27 smithi083 ceph-mon[79930]: pgmap v16533: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:29.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:39:29 smithi153 ceph-mon[70044]: pgmap v16534: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:29.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:39:29 smithi083 ceph-mon[79930]: pgmap v16534: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:29.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:39:29 smithi083 ceph-mon[77242]: pgmap v16534: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:31.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:39:31 smithi153 ceph-mon[70044]: pgmap v16535: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:31.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:39:31 smithi083 ceph-mon[79930]: pgmap v16535: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:31.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:39:31 smithi083 ceph-mon[77242]: pgmap v16535: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:39:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:39:31] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T14:39:32.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:39:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:39:32.225Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:39:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:39:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:39:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:39:33.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:39:33 smithi153 ceph-mon[70044]: pgmap v16536: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:39:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:39:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:39:33.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:39:33 smithi083 ceph-mon[79930]: pgmap v16536: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:33.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:39:33 smithi083 ceph-mon[77242]: pgmap v16536: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:35.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:39:35 smithi153 ceph-mon[70044]: pgmap v16537: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:35.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:39:35 smithi083 ceph-mon[79930]: pgmap v16537: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:35.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:39:35 smithi083 ceph-mon[77242]: pgmap v16537: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:35.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:39:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:39:35.585Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:39:36.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:39:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:39:36.320Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:39:37.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:39:37 smithi153 ceph-mon[70044]: pgmap v16538: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:37.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:39:37 smithi083 ceph-mon[77242]: pgmap v16538: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:37.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:39:37 smithi083 ceph-mon[79930]: pgmap v16538: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:39.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:39:39 smithi153 ceph-mon[70044]: pgmap v16539: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:39.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:39:39 smithi083 ceph-mon[79930]: pgmap v16539: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:39.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:39:39 smithi083 ceph-mon[77242]: pgmap v16539: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:40.350 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:39:40.636 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:39:40.636 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 94s ago 9h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:39:40.637 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 9m ago 9h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:39:40.637 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 94s ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:39:40.637 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 9m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:39:40.637 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 9m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:39:40.637 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 94s ago 9h 966M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:39:40.638 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 94s ago 9h 2019M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:39:40.638 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 9m ago 9h 863M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:39:40.638 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 94s ago 9h 1208M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:39:40.638 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 94s ago 9h 23.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:39:40.639 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 9m ago 9h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:39:40.639 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 94s ago 9h 1464M 2219M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:39:40.639 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 94s ago 9h 1451M 2219M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:39:40.639 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 94s ago 9h 1128M 2219M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:39:40.640 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 94s ago 9h 1422M 2219M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:39:40.640 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 9m ago 9h 1224M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:39:40.640 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 9m ago 9h 1537M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:39:40.640 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 9m ago 9h 1373M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:39:40.640 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 9m ago 9h 1633M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:39:40.641 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 9m ago 9h 194M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:39:40.989 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:39:40.989 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:39:40.989 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:39:40.989 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:39:40.990 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:39:40.990 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:39:40.990 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:39:40.991 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:39:40.991 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:39:40.991 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:39:40.991 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:39:40.991 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:39:40.992 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:39:40.992 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:39:40.992 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:39:40.992 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:39:40.993 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:39:40.993 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:39:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:39:41 smithi153 ceph-mon[70044]: pgmap v16540: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:39:41 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/224402915' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:39:41.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:39:41 smithi083 ceph-mon[79930]: pgmap v16540: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:41.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:39:41 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/224402915' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:39:41.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:39:41 smithi083 ceph-mon[77242]: pgmap v16540: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:41.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:39:41 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/224402915' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:39:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:39:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:39:41] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T14:39:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:39:42 smithi153 ceph-mon[70044]: from='client.53364 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:39:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:39:42 smithi153 ceph-mon[70044]: from='client.53370 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:39:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:39:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:39:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:39:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:39:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:39:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:39:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:39:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:39:42.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:39:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:39:42.226Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:39:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:39:42 smithi083 ceph-mon[79930]: from='client.53364 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:39:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:39:42 smithi083 ceph-mon[79930]: from='client.53370 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:39:42.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:39:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:39:42.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:39:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:39:42.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:39:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:39:42.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:39:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:39:42.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:39:42 smithi083 ceph-mon[77242]: from='client.53364 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:39:42.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:39:42 smithi083 ceph-mon[77242]: from='client.53370 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:39:42.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:39:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:39:42.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:39:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:39:42.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:39:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:39:42.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:39:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:39:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:39:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:39:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:39:43.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:39:43 smithi153 ceph-mon[70044]: pgmap v16541: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:39:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:39:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:39:43.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:39:43 smithi083 ceph-mon[79930]: pgmap v16541: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:43.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:39:43 smithi083 ceph-mon[77242]: pgmap v16541: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:45.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:39:45 smithi153 ceph-mon[70044]: pgmap v16542: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:45.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:39:45 smithi083 ceph-mon[79930]: pgmap v16542: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:45.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:39:45 smithi083 ceph-mon[77242]: pgmap v16542: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:45.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:39:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:39:45.586Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:39:46.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:39:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:39:46.321Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:39:47.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:39:47 smithi153 ceph-mon[70044]: pgmap v16543: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:47.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:39:47 smithi083 ceph-mon[77242]: pgmap v16543: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:47.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:39:47 smithi083 ceph-mon[79930]: pgmap v16543: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:49.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:39:49 smithi153 ceph-mon[70044]: pgmap v16544: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:49.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:39:49 smithi083 ceph-mon[79930]: pgmap v16544: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:49.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:39:49 smithi083 ceph-mon[77242]: pgmap v16544: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:51.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:39:51 smithi153 ceph-mon[70044]: pgmap v16545: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:51.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:39:51 smithi083 ceph-mon[79930]: pgmap v16545: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:51.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:39:51 smithi083 ceph-mon[77242]: pgmap v16545: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:39:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:39:51] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T14:39:52.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:39:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:39:52.226Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:39:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:39:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:39:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:39:53.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:39:53 smithi153 ceph-mon[70044]: pgmap v16546: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:39:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:39:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:39:53.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:39:53 smithi083 ceph-mon[79930]: pgmap v16546: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:53.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:39:53 smithi083 ceph-mon[77242]: pgmap v16546: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:55.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:39:55 smithi153 ceph-mon[70044]: pgmap v16547: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:55.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:39:55 smithi083 ceph-mon[79930]: pgmap v16547: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:55.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:39:55 smithi083 ceph-mon[77242]: pgmap v16547: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:55.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:39:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:39:55.586Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:39:56.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:39:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:39:56.321Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:39:57.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:39:57 smithi153 ceph-mon[70044]: pgmap v16548: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:57.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:39:57 smithi083 ceph-mon[77242]: pgmap v16548: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:57.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:39:57 smithi083 ceph-mon[79930]: pgmap v16548: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:59.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:39:59 smithi153 ceph-mon[70044]: pgmap v16549: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:59.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:39:59 smithi083 ceph-mon[79930]: pgmap v16549: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:39:59.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:39:59 smithi083 ceph-mon[77242]: pgmap v16549: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:00.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:40:00 smithi083 ceph-mon[79930]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T14:40:00.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:40:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[77219]: 2022-09-29T14:39:59.999+0000 7f160f665700 -1 log_channel(cluster) log [ERR] : overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T14:40:00.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:40:00 smithi083 ceph-mon[77242]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T14:40:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:40:00 smithi153 ceph-mon[70044]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T14:40:01.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:40:01 smithi083 ceph-mon[79930]: pgmap v16550: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:01.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:40:01 smithi083 ceph-mon[77242]: pgmap v16550: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:01.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:40:01 smithi153 ceph-mon[70044]: pgmap v16550: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:40:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:40:01] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T14:40:02.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:40:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:40:02.229Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:40:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:40:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:40:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:40:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:40:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:40:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:40:03.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:40:03 smithi153 ceph-mon[70044]: pgmap v16551: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:03.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:40:03 smithi083 ceph-mon[79930]: pgmap v16551: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:03.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:40:03 smithi083 ceph-mon[77242]: pgmap v16551: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:05.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:40:05 smithi083 ceph-mon[79930]: pgmap v16552: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:05.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:40:05 smithi083 ceph-mon[77242]: pgmap v16552: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:05.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:40:05 smithi153 ceph-mon[70044]: pgmap v16552: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:05.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:40:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:40:05.587Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:40:06.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:40:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:40:06.321Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:40:07.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:40:07 smithi083 ceph-mon[79930]: pgmap v16553: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:07.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:40:07 smithi083 ceph-mon[77242]: pgmap v16553: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:07.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:40:07 smithi153 ceph-mon[70044]: pgmap v16553: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:08.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:40:08 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:40:08.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:40:08 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:40:08.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:40:08 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:40:08.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:40:08 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:40:08.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:40:08 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:40:08.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:40:08 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:40:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:40:08 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:40:08.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:40:08 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:40:08.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:40:08 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:40:09.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:40:09 smithi083 ceph-mon[79930]: pgmap v16554: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:09.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:40:09 smithi083 ceph-mon[77242]: pgmap v16554: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:09.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:40:09 smithi153 ceph-mon[70044]: pgmap v16554: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:11.317 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:40:11.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:40:11 smithi083 ceph-mon[79930]: pgmap v16555: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:11.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:40:11 smithi083 ceph-mon[77242]: pgmap v16555: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:11.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:40:11 smithi153 ceph-mon[70044]: pgmap v16555: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:11.609 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:40:11.609 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 2m ago 9h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:40:11.609 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 10m ago 9h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:40:11.609 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 2m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:40:11.610 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 10m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:40:11.610 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 10m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:40:11.610 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 2m ago 9h 966M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:40:11.610 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 2m ago 9h 2019M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:40:11.611 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 10m ago 9h 863M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:40:11.611 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 2m ago 9h 1208M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:40:11.611 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 2m ago 9h 23.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:40:11.611 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 10m ago 9h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:40:11.612 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 2m ago 9h 1464M 2219M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:40:11.612 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 2m ago 9h 1451M 2219M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:40:11.612 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 2m ago 9h 1128M 2219M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:40:11.612 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 2m ago 9h 1422M 2219M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:40:11.613 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 10m ago 9h 1224M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:40:11.613 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 10m ago 9h 1537M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:40:11.613 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 10m ago 9h 1373M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:40:11.613 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 10m ago 9h 1633M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:40:11.614 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 10m ago 9h 194M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:40:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:40:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:40:11] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T14:40:11.976 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:40:11.977 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:40:11.977 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:40:11.977 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:40:11.978 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:40:11.978 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:40:11.978 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:40:11.978 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:40:11.979 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:40:11.979 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:40:11.979 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:40:11.979 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:40:11.980 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:40:11.980 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:40:11.980 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:40:11.980 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:40:11.981 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:40:11.981 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:40:12.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:40:12 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4169003117' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:40:12.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:40:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:40:12.227Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:40:12.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:40:12 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4169003117' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:40:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:40:12 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4169003117' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:40:12.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:40:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:40:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:40:13.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:40:13 smithi153 ceph-mon[70044]: from='client.63313 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:40:13.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:40:13 smithi153 ceph-mon[70044]: from='client.63319 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:40:13.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:40:13 smithi153 ceph-mon[70044]: pgmap v16556: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:13.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:40:13 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:40:13.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:40:13 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:40:13.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:40:13 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:40:13.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:40:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:40:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:40:13.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:40:13 smithi083 ceph-mon[79930]: from='client.63313 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:40:13.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:40:13 smithi083 ceph-mon[79930]: from='client.63319 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:40:13.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:40:13 smithi083 ceph-mon[79930]: pgmap v16556: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:13.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:40:13 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:40:13.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:40:13 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:40:13.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:40:13 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:40:13.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:40:13 smithi083 ceph-mon[77242]: from='client.63313 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:40:13.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:40:13 smithi083 ceph-mon[77242]: from='client.63319 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:40:13.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:40:13 smithi083 ceph-mon[77242]: pgmap v16556: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:13.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:40:13 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:40:13.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:40:13 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:40:13.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:40:13 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:40:15.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:40:15 smithi083 ceph-mon[79930]: pgmap v16557: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:15.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:40:15 smithi083 ceph-mon[77242]: pgmap v16557: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:15.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:40:15 smithi153 ceph-mon[70044]: pgmap v16557: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:15.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:40:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:40:15.587Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:40:16.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:40:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:40:16.321Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:40:17.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:40:17 smithi083 ceph-mon[79930]: pgmap v16558: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:17.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:40:17 smithi083 ceph-mon[77242]: pgmap v16558: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:17.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:40:17 smithi153 ceph-mon[70044]: pgmap v16558: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:19.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:40:19 smithi083 ceph-mon[79930]: pgmap v16559: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:19.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:40:19 smithi083 ceph-mon[77242]: pgmap v16559: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:19.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:40:19 smithi153 ceph-mon[70044]: pgmap v16559: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:21.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:40:21 smithi083 ceph-mon[79930]: pgmap v16560: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:21.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:40:21 smithi083 ceph-mon[77242]: pgmap v16560: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:21.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:40:21 smithi153 ceph-mon[70044]: pgmap v16560: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:40:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:40:21] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T14:40:22.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:40:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:40:22.229Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:40:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:40:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:40:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:40:23.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:40:23 smithi153 ceph-mon[70044]: pgmap v16561: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:40:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:40:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:40:23.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:40:23 smithi083 ceph-mon[79930]: pgmap v16561: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:23.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:40:23 smithi083 ceph-mon[77242]: pgmap v16561: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:25.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:40:25 smithi083 ceph-mon[77242]: pgmap v16562: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:25.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:40:25 smithi083 ceph-mon[79930]: pgmap v16562: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:25.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:40:25 smithi153 ceph-mon[70044]: pgmap v16562: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:25.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:40:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:40:25.588Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:40:26.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:40:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:40:26.322Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:40:27.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:40:27 smithi083 ceph-mon[79930]: pgmap v16563: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:27.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:40:27 smithi083 ceph-mon[77242]: pgmap v16563: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:27.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:40:27 smithi153 ceph-mon[70044]: pgmap v16563: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:29.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:40:29 smithi083 ceph-mon[79930]: pgmap v16564: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:29.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:40:29 smithi083 ceph-mon[77242]: pgmap v16564: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:29.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:40:29 smithi153 ceph-mon[70044]: pgmap v16564: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:31.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:40:31 smithi083 ceph-mon[79930]: pgmap v16565: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:31.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:40:31 smithi083 ceph-mon[77242]: pgmap v16565: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:31.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:40:31 smithi153 ceph-mon[70044]: pgmap v16565: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:40:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:40:31] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T14:40:32.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:40:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:40:32.230Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:40:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:40:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:40:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:40:33.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:40:33 smithi153 ceph-mon[70044]: pgmap v16566: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:40:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:40:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:40:33.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:40:33 smithi083 ceph-mon[77242]: pgmap v16566: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:33.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:40:33 smithi083 ceph-mon[79930]: pgmap v16566: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:35.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:40:35 smithi083 ceph-mon[77242]: pgmap v16567: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:35.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:40:35 smithi083 ceph-mon[79930]: pgmap v16567: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:35.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:40:35 smithi153 ceph-mon[70044]: pgmap v16567: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:35.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:40:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:40:35.588Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:40:36.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:40:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:40:36.323Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:40:37.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:40:37 smithi083 ceph-mon[79930]: pgmap v16568: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:37.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:40:37 smithi083 ceph-mon[77242]: pgmap v16568: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:37.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:40:37 smithi153 ceph-mon[70044]: pgmap v16568: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:39.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:40:39 smithi083 ceph-mon[79930]: pgmap v16569: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:39.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:40:39 smithi083 ceph-mon[77242]: pgmap v16569: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:39.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:40:39 smithi153 ceph-mon[70044]: pgmap v16569: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:41.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:40:41 smithi083 ceph-mon[79930]: pgmap v16570: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:41.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:40:41 smithi083 ceph-mon[77242]: pgmap v16570: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:40:41 smithi153 ceph-mon[70044]: pgmap v16570: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:40:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:40:41] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T14:40:42.311 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:40:42.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:40:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:40:42.230Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:40:42.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:40:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:40:42.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:40:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:40:42.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:40:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:40:42.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:40:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:40:42.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:40:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:40:42.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:40:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:40:42.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:40:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:40:42.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:40:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:40:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:40:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:40:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:40:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:40:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:40:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:40:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:40:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:40:42.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:40:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:40:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:40:42.599 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:40:42.599 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 2m ago 9h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:40:42.599 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 30s ago 9h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:40:42.600 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 2m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:40:42.600 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 30s ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:40:42.600 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 30s ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:40:42.601 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 2m ago 9h 966M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:40:42.601 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 2m ago 9h 2019M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:40:42.601 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 30s ago 9h 876M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:40:42.601 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 2m ago 9h 1208M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:40:42.601 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 2m ago 9h 23.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:40:42.602 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 30s ago 9h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:40:42.602 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 2m ago 9h 1464M 2219M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:40:42.602 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 2m ago 9h 1451M 2219M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:40:42.602 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 2m ago 9h 1128M 2219M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:40:42.603 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 2m ago 9h 1422M 2219M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:40:42.603 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 30s ago 9h 1247M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:40:42.603 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 30s ago 9h 1565M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:40:42.603 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 30s ago 9h 1398M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:40:42.604 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 30s ago 9h 1662M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:40:42.604 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 30s ago 9h 197M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:40:42.952 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:40:42.952 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:40:42.952 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:40:42.953 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:40:42.953 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:40:42.953 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:40:42.953 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:40:42.954 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:40:42.954 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:40:42.954 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:40:42.954 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:40:42.955 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:40:42.955 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:40:42.955 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:40:42.955 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:40:42.955 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:40:42.956 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:40:42.956 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:40:43.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:40:43 smithi153 ceph-mon[70044]: pgmap v16571: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:43.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:40:43 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3942983050' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:40:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:40:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:40:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:40:43.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:40:43 smithi083 ceph-mon[79930]: pgmap v16571: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:43.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:40:43 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3942983050' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:40:43.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:40:43 smithi083 ceph-mon[77242]: pgmap v16571: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:43.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:40:43 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3942983050' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:40:44.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:40:44 smithi083 ceph-mon[79930]: from='client.53400 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:40:44.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:40:44 smithi083 ceph-mon[79930]: from='client.53406 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:40:44.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:40:44 smithi083 ceph-mon[77242]: from='client.53400 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:40:44.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:40:44 smithi083 ceph-mon[77242]: from='client.53406 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:40:44.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:40:44 smithi153 ceph-mon[70044]: from='client.53400 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:40:44.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:40:44 smithi153 ceph-mon[70044]: from='client.53406 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:40:45.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:40:45 smithi083 ceph-mon[79930]: pgmap v16572: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:45.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:40:45 smithi083 ceph-mon[77242]: pgmap v16572: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:45.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:40:45 smithi153 ceph-mon[70044]: pgmap v16572: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:45.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:40:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:40:45.588Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:40:46.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:40:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:40:46.323Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:40:47.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:40:47 smithi083 ceph-mon[79930]: pgmap v16573: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:47.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:40:47 smithi083 ceph-mon[77242]: pgmap v16573: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:47.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:40:47 smithi153 ceph-mon[70044]: pgmap v16573: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:49.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:40:49 smithi083 ceph-mon[77242]: pgmap v16574: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:49.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:40:49 smithi083 ceph-mon[79930]: pgmap v16574: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:49.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:40:49 smithi153 ceph-mon[70044]: pgmap v16574: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:51.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:40:51 smithi083 ceph-mon[79930]: pgmap v16575: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:51.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:40:51 smithi083 ceph-mon[77242]: pgmap v16575: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:51.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:40:51 smithi153 ceph-mon[70044]: pgmap v16575: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:40:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:40:51] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T14:40:52.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:40:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:40:52.231Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:40:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:40:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:40:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:40:53.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:40:53 smithi153 ceph-mon[70044]: pgmap v16576: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:40:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:40:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:40:53.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:40:53 smithi083 ceph-mon[77242]: pgmap v16576: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:53.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:40:53 smithi083 ceph-mon[79930]: pgmap v16576: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:55.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:40:55 smithi083 ceph-mon[79930]: pgmap v16577: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:55.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:40:55 smithi083 ceph-mon[77242]: pgmap v16577: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:55.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:40:55 smithi153 ceph-mon[70044]: pgmap v16577: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:55.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:40:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:40:55.588Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:40:56.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:40:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:40:56.323Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:40:57.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:40:57 smithi083 ceph-mon[79930]: pgmap v16578: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:57.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:40:57 smithi083 ceph-mon[77242]: pgmap v16578: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:57.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:40:57 smithi153 ceph-mon[70044]: pgmap v16578: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:58.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:40:58 smithi083 ceph-mon[79930]: pgmap v16579: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:58.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:40:58 smithi083 ceph-mon[77242]: pgmap v16579: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:40:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:40:58 smithi153 ceph-mon[70044]: pgmap v16579: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:01.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:41:00 smithi083 ceph-mon[79930]: pgmap v16580: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:01.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:41:00 smithi083 ceph-mon[77242]: pgmap v16580: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:01.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:41:00 smithi153 ceph-mon[70044]: pgmap v16580: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:41:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:41:01] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T14:41:02.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:41:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:41:02.232Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:41:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:41:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:41:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:41:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:41:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:41:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:41:03.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:41:03 smithi153 ceph-mon[70044]: pgmap v16581: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:03.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:41:03 smithi083 ceph-mon[79930]: pgmap v16581: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:03.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:41:03 smithi083 ceph-mon[77242]: pgmap v16581: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:05.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:41:05 smithi153 ceph-mon[70044]: pgmap v16582: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:05.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:41:05 smithi083 ceph-mon[79930]: pgmap v16582: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:05.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:41:05 smithi083 ceph-mon[77242]: pgmap v16582: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:05.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:41:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:41:05.589Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:41:06.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:41:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:41:06.324Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:41:07.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:41:07 smithi153 ceph-mon[70044]: pgmap v16583: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:07.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:41:07 smithi083 ceph-mon[79930]: pgmap v16583: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:07.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:41:07 smithi083 ceph-mon[77242]: pgmap v16583: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:09.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:41:09 smithi153 ceph-mon[70044]: pgmap v16584: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:09.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:41:09 smithi083 ceph-mon[79930]: pgmap v16584: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:09.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:41:09 smithi083 ceph-mon[77242]: pgmap v16584: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:11.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:41:11 smithi153 ceph-mon[70044]: pgmap v16585: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:11.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:41:11 smithi083 ceph-mon[79930]: pgmap v16585: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:11.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:41:11 smithi083 ceph-mon[77242]: pgmap v16585: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:41:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:41:11] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T14:41:12.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:41:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:41:12.233Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:41:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:41:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:41:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:41:13.301 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:41:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:41:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:41:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:41:13.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:41:13 smithi153 ceph-mon[70044]: pgmap v16586: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:13.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:41:13 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:41:13.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:41:13 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:41:13.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:41:13 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:41:13.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:41:13 smithi083 ceph-mon[79930]: pgmap v16586: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:13.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:41:13 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:41:13.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:41:13 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:41:13.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:41:13 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:41:13.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:41:13 smithi083 ceph-mon[77242]: pgmap v16586: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:13.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:41:13 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:41:13.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:41:13 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:41:13.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:41:13 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:41:13.613 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:41:13.613 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 3m ago 9h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:41:13.614 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 61s ago 9h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:41:13.614 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 3m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:41:13.614 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 61s ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:41:13.614 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 61s ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:41:13.615 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 3m ago 9h 966M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:41:13.615 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 3m ago 9h 2019M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:41:13.615 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 61s ago 9h 876M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:41:13.615 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 3m ago 9h 1208M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:41:13.615 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 3m ago 9h 23.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:41:13.616 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 61s ago 9h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:41:13.616 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 3m ago 9h 1464M 2219M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:41:13.616 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 3m ago 9h 1451M 2219M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:41:13.616 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 3m ago 9h 1128M 2219M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:41:13.616 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 3m ago 9h 1422M 2219M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:41:13.617 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 61s ago 9h 1247M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:41:13.617 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 61s ago 9h 1565M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:41:13.617 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 61s ago 9h 1398M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:41:13.617 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 61s ago 9h 1662M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:41:13.618 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 61s ago 9h 197M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:41:13.976 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:41:13.977 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:41:13.977 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:41:13.977 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:41:13.977 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:41:13.978 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:41:13.978 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:41:13.978 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:41:13.978 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:41:13.979 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:41:13.979 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:41:13.979 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:41:13.979 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:41:13.979 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:41:13.980 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:41:13.980 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:41:13.980 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:41:13.980 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:41:14.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:41:14 smithi153 ceph-mon[70044]: from='client.53418 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:41:14.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:41:14 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:41:14.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:41:14 smithi153 ceph-mon[70044]: from='client.63355 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:41:14.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:41:14 smithi153 ceph-mon[70044]: pgmap v16587: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:14.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:41:14 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1650085078' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:41:14.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:41:14 smithi083 ceph-mon[77242]: from='client.53418 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:41:14.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:41:14 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:41:14.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:41:14 smithi083 ceph-mon[77242]: from='client.63355 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:41:14.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:41:14 smithi083 ceph-mon[77242]: pgmap v16587: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:14.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:41:14 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1650085078' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:41:14.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:41:14 smithi083 ceph-mon[79930]: from='client.53418 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:41:14.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:41:14 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:41:14.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:41:14 smithi083 ceph-mon[79930]: from='client.63355 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:41:14.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:41:14 smithi083 ceph-mon[79930]: pgmap v16587: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:14.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:41:14 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1650085078' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:41:15.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:41:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:41:15.590Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:41:16.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:41:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:41:16.325Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:41:17.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:41:16 smithi083 ceph-mon[79930]: pgmap v16588: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:17.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:41:16 smithi083 ceph-mon[77242]: pgmap v16588: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:17.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:41:16 smithi153 ceph-mon[70044]: pgmap v16588: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:19.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:41:19 smithi153 ceph-mon[70044]: pgmap v16589: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:19.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:41:19 smithi083 ceph-mon[77242]: pgmap v16589: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:19.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:41:19 smithi083 ceph-mon[79930]: pgmap v16589: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:21.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:41:21 smithi153 ceph-mon[70044]: pgmap v16590: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:21.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:41:21 smithi083 ceph-mon[77242]: pgmap v16590: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:21.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:41:21 smithi083 ceph-mon[79930]: pgmap v16590: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:41:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:41:21] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T14:41:22.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:41:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:41:22.234Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:41:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:41:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:41:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:41:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:41:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:41:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:41:23.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:41:23 smithi153 ceph-mon[70044]: pgmap v16591: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:23.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:41:23 smithi083 ceph-mon[79930]: pgmap v16591: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:23.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:41:23 smithi083 ceph-mon[77242]: pgmap v16591: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:25.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:41:25 smithi153 ceph-mon[70044]: pgmap v16592: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:25.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:41:25 smithi083 ceph-mon[79930]: pgmap v16592: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:25.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:41:25 smithi083 ceph-mon[77242]: pgmap v16592: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:25.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:41:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:41:25.591Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:41:26.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:41:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:41:26.325Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:41:27.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:41:27 smithi153 ceph-mon[70044]: pgmap v16593: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:27.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:41:27 smithi083 ceph-mon[77242]: pgmap v16593: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:27.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:41:27 smithi083 ceph-mon[79930]: pgmap v16593: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:29.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:41:29 smithi153 ceph-mon[70044]: pgmap v16594: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:29.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:41:29 smithi083 ceph-mon[77242]: pgmap v16594: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:29.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:41:29 smithi083 ceph-mon[79930]: pgmap v16594: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:31.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:41:31 smithi153 ceph-mon[70044]: pgmap v16595: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:31.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:41:31 smithi083 ceph-mon[79930]: pgmap v16595: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:31.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:41:31 smithi083 ceph-mon[77242]: pgmap v16595: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:41:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:41:31] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T14:41:32.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:41:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:41:32.234Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:41:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:41:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:41:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:41:33.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:41:33 smithi153 ceph-mon[70044]: pgmap v16596: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:41:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:41:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:41:33.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:41:33 smithi083 ceph-mon[79930]: pgmap v16596: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:33.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:41:33 smithi083 ceph-mon[77242]: pgmap v16596: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:35.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:41:35 smithi153 ceph-mon[70044]: pgmap v16597: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:35.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:41:35 smithi083 ceph-mon[79930]: pgmap v16597: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:35.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:41:35 smithi083 ceph-mon[77242]: pgmap v16597: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:35.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:41:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:41:35.591Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:41:36.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:41:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:41:36.326Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:41:37.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:41:37 smithi153 ceph-mon[70044]: pgmap v16598: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:37.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:41:37 smithi083 ceph-mon[79930]: pgmap v16598: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:37.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:41:37 smithi083 ceph-mon[77242]: pgmap v16598: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:39.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:41:39 smithi153 ceph-mon[70044]: pgmap v16599: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:39.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:41:39 smithi083 ceph-mon[77242]: pgmap v16599: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:39.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:41:39 smithi083 ceph-mon[79930]: pgmap v16599: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:41:41 smithi153 ceph-mon[70044]: pgmap v16600: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:41.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:41:41 smithi083 ceph-mon[79930]: pgmap v16600: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:41.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:41:41 smithi083 ceph-mon[77242]: pgmap v16600: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:41:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:41:41] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T14:41:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:41:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:41:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:41:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:41:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:41:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:41:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:41:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:41:42.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:41:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:41:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:41:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:41:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:41:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:41:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:41:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:41:42.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:41:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:41:42.234Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:41:42.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:41:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:41:42.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:41:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:41:42.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:41:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:41:42.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:41:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:41:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:41:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:41:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:41:43.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:41:43 smithi153 ceph-mon[70044]: pgmap v16601: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:41:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:41:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:41:43.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:41:43 smithi083 ceph-mon[79930]: pgmap v16601: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:43.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:41:43 smithi083 ceph-mon[77242]: pgmap v16601: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:44.300 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:41:44.589 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:41:44.589 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 3m ago 9h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:41:44.589 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 92s ago 9h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:41:44.589 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 3m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:41:44.590 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 92s ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:41:44.590 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 92s ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:41:44.590 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 3m ago 9h 966M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:41:44.590 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 3m ago 9h 2019M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:41:44.591 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 92s ago 9h 876M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:41:44.591 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 3m ago 9h 1208M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:41:44.591 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 3m ago 9h 23.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:41:44.591 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 92s ago 9h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:41:44.592 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 3m ago 9h 1464M 2219M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:41:44.592 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 3m ago 9h 1451M 2219M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:41:44.592 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 3m ago 9h 1128M 2219M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:41:44.592 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 3m ago 9h 1422M 2219M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:41:44.592 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 92s ago 9h 1247M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:41:44.593 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 92s ago 9h 1565M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:41:44.593 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 92s ago 9h 1398M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:41:44.593 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 92s ago 9h 1662M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:41:44.593 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 92s ago 9h 197M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:41:44.944 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:41:44.944 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:41:44.945 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:41:44.945 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:41:44.945 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:41:44.945 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:41:44.946 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:41:44.946 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:41:44.946 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:41:44.946 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:41:44.947 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:41:44.947 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:41:44.947 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:41:44.947 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:41:44.948 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:41:44.948 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:41:44.948 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:41:44.948 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:41:45.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:41:45 smithi083 ceph-mon[77242]: pgmap v16602: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:45.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:41:45 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4068389306' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:41:45.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:41:45 smithi083 ceph-mon[79930]: pgmap v16602: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:45.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:41:45 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4068389306' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:41:45.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:41:45 smithi153 ceph-mon[70044]: pgmap v16602: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:45.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:41:45 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4068389306' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:41:45.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:41:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:41:45.591Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:41:46.328 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:41:46 smithi083 ceph-mon[79930]: from='client.63367 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:41:46.329 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:41:46 smithi083 ceph-mon[79930]: from='client.63373 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:41:46.329 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:41:46 smithi083 ceph-mon[77242]: from='client.63367 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:41:46.329 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:41:46 smithi083 ceph-mon[77242]: from='client.63373 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:41:46.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:41:46 smithi153 ceph-mon[70044]: from='client.63367 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:41:46.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:41:46 smithi153 ceph-mon[70044]: from='client.63373 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:41:46.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:41:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:41:46.327Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:41:47.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:41:47 smithi083 ceph-mon[79930]: pgmap v16603: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:47.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:41:47 smithi083 ceph-mon[77242]: pgmap v16603: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:47.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:41:47 smithi153 ceph-mon[70044]: pgmap v16603: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:49.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:41:49 smithi083 ceph-mon[79930]: pgmap v16604: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:49.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:41:49 smithi083 ceph-mon[77242]: pgmap v16604: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:49.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:41:49 smithi153 ceph-mon[70044]: pgmap v16604: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:51.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:41:51 smithi083 ceph-mon[79930]: pgmap v16605: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:51.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:41:51 smithi083 ceph-mon[77242]: pgmap v16605: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:51.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:41:51 smithi153 ceph-mon[70044]: pgmap v16605: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:41:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:41:51] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T14:41:52.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:41:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:41:52.235Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:41:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:41:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:41:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:41:53.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:41:53 smithi153 ceph-mon[70044]: pgmap v16606: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:41:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:41:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:41:53.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:41:53 smithi083 ceph-mon[79930]: pgmap v16606: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:53.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:41:53 smithi083 ceph-mon[77242]: pgmap v16606: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:55.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:41:55 smithi083 ceph-mon[79930]: pgmap v16607: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:55.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:41:55 smithi083 ceph-mon[77242]: pgmap v16607: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:55.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:41:55 smithi153 ceph-mon[70044]: pgmap v16607: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:55.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:41:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:41:55.592Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:41:56.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:41:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:41:56.328Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:41:57.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:41:57 smithi083 ceph-mon[79930]: pgmap v16608: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:57.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:41:57 smithi083 ceph-mon[77242]: pgmap v16608: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:57.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:41:57 smithi153 ceph-mon[70044]: pgmap v16608: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:59.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:41:59 smithi083 ceph-mon[77242]: pgmap v16609: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:59.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:41:59 smithi083 ceph-mon[79930]: pgmap v16609: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:41:59.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:41:59 smithi153 ceph-mon[70044]: pgmap v16609: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:01.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:42:01 smithi083 ceph-mon[79930]: pgmap v16610: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:01.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:42:01 smithi083 ceph-mon[77242]: pgmap v16610: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:01.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:42:01 smithi153 ceph-mon[70044]: pgmap v16610: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:42:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:42:01] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T14:42:02.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:42:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:42:02.236Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:42:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:42:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:42:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:42:03.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:42:03 smithi153 ceph-mon[70044]: pgmap v16611: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:42:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:42:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:42:03.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:42:03 smithi083 ceph-mon[77242]: pgmap v16611: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:03.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:42:03 smithi083 ceph-mon[79930]: pgmap v16611: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:05.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:42:05 smithi083 ceph-mon[77242]: pgmap v16612: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:05.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:42:05 smithi083 ceph-mon[79930]: pgmap v16612: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:05.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:42:05 smithi153 ceph-mon[70044]: pgmap v16612: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:05.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:42:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:42:05.593Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:42:06.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:42:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:42:06.328Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:42:07.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:42:07 smithi083 ceph-mon[79930]: pgmap v16613: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:07.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:42:07 smithi083 ceph-mon[77242]: pgmap v16613: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:07.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:42:07 smithi153 ceph-mon[70044]: pgmap v16613: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:09.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:42:09 smithi083 ceph-mon[77242]: pgmap v16614: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:09.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:42:09 smithi083 ceph-mon[79930]: pgmap v16614: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:09.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:42:09 smithi153 ceph-mon[70044]: pgmap v16614: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:11.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:42:11 smithi083 ceph-mon[79930]: pgmap v16615: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:11.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:42:11 smithi083 ceph-mon[77242]: pgmap v16615: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:11.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:42:11 smithi153 ceph-mon[70044]: pgmap v16615: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:42:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:42:11] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T14:42:12.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:42:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:42:12.236Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:42:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:42:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:42:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:42:13.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:42:13 smithi153 ceph-mon[70044]: pgmap v16616: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:42:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:42:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:42:13.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:42:13 smithi083 ceph-mon[79930]: pgmap v16616: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:13.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:42:13 smithi083 ceph-mon[77242]: pgmap v16616: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:14.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:42:14 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:42:14.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:42:14 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:42:14.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:42:14 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:42:14.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:42:14 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:42:14.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:42:14 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:42:14.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:42:14 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:42:14.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:42:14 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:42:14.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:42:14 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:42:14.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:42:14 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:42:15.269 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:42:15.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:42:15 smithi083 ceph-mon[79930]: pgmap v16617: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:15.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:42:15 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:42:15.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:42:15 smithi083 ceph-mon[77242]: pgmap v16617: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:15.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:42:15 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:42:15.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:42:15 smithi153 ceph-mon[70044]: pgmap v16617: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:15.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:42:15 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:42:15.558 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:42:15.559 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 4m ago 9h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:42:15.559 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 2m ago 9h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:42:15.559 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 4m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:42:15.559 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 2m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:42:15.560 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 2m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:42:15.560 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 4m ago 9h 966M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:42:15.560 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 4m ago 9h 2019M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:42:15.560 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 2m ago 9h 876M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:42:15.561 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 4m ago 9h 1208M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:42:15.561 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 4m ago 9h 23.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:42:15.561 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 2m ago 9h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:42:15.561 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 4m ago 9h 1464M 2219M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:42:15.561 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 4m ago 9h 1451M 2219M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:42:15.562 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 4m ago 9h 1128M 2219M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:42:15.562 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 4m ago 9h 1422M 2219M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:42:15.562 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 2m ago 9h 1247M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:42:15.562 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 2m ago 9h 1565M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:42:15.563 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 2m ago 9h 1398M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:42:15.563 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 2m ago 9h 1662M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:42:15.563 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 2m ago 9h 197M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:42:15.917 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:42:15.918 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:42:15.918 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:42:15.918 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:42:15.918 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:42:15.919 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:42:15.919 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:42:15.919 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:42:15.919 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:42:15.919 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:42:15.920 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:42:15.920 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:42:15.920 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:42:15.920 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:42:15.921 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:42:15.921 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:42:15.921 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:42:15.921 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:42:15.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:42:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:42:15.592Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:42:16.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:42:16 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/623827538' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:42:16.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:42:16 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/623827538' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:42:16.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:42:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:42:16.328Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:42:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:42:16 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/623827538' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:42:17.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:42:17 smithi083 ceph-mon[79930]: from='client.53454 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:42:17.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:42:17 smithi083 ceph-mon[79930]: from='client.63391 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:42:17.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:42:17 smithi083 ceph-mon[79930]: pgmap v16618: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:17.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:42:17 smithi083 ceph-mon[77242]: from='client.53454 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:42:17.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:42:17 smithi083 ceph-mon[77242]: from='client.63391 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:42:17.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:42:17 smithi083 ceph-mon[77242]: pgmap v16618: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:17.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:42:17 smithi153 ceph-mon[70044]: from='client.53454 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:42:17.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:42:17 smithi153 ceph-mon[70044]: from='client.63391 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:42:17.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:42:17 smithi153 ceph-mon[70044]: pgmap v16618: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:19.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:42:19 smithi083 ceph-mon[79930]: pgmap v16619: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:19.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:42:19 smithi083 ceph-mon[77242]: pgmap v16619: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:19.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:42:19 smithi153 ceph-mon[70044]: pgmap v16619: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:21.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:42:21 smithi083 ceph-mon[79930]: pgmap v16620: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:21.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:42:21 smithi083 ceph-mon[77242]: pgmap v16620: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:21.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:42:21 smithi153 ceph-mon[70044]: pgmap v16620: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:42:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:42:21] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T14:42:22.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:42:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:42:22.237Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:42:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:42:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:42:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:42:23.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:42:23 smithi153 ceph-mon[70044]: pgmap v16621: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:42:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:42:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:42:23.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:42:23 smithi083 ceph-mon[79930]: pgmap v16621: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:23.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:42:23 smithi083 ceph-mon[77242]: pgmap v16621: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:25.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:42:25 smithi153 ceph-mon[70044]: pgmap v16622: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:25.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:42:25 smithi083 ceph-mon[79930]: pgmap v16622: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:25.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:42:25 smithi083 ceph-mon[77242]: pgmap v16622: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:25.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:42:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:42:25.594Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:42:26.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:42:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:42:26.329Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:42:27.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:42:27 smithi153 ceph-mon[70044]: pgmap v16623: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:27.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:42:27 smithi083 ceph-mon[79930]: pgmap v16623: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:27.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:42:27 smithi083 ceph-mon[77242]: pgmap v16623: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:29.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:42:29 smithi153 ceph-mon[70044]: pgmap v16624: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:29.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:42:29 smithi083 ceph-mon[79930]: pgmap v16624: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:29.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:42:29 smithi083 ceph-mon[77242]: pgmap v16624: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:31.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:42:31 smithi153 ceph-mon[70044]: pgmap v16625: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:31.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:42:31 smithi083 ceph-mon[79930]: pgmap v16625: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:31.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:42:31 smithi083 ceph-mon[77242]: pgmap v16625: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:42:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:42:31] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T14:42:32.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:42:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:42:32.237Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:42:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:42:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:42:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:42:33.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:42:33 smithi153 ceph-mon[70044]: pgmap v16626: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:42:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:42:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:42:33.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:42:33 smithi083 ceph-mon[77242]: pgmap v16626: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:33.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:42:33 smithi083 ceph-mon[79930]: pgmap v16626: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:35.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:42:35 smithi153 ceph-mon[70044]: pgmap v16627: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:35.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:42:35 smithi083 ceph-mon[79930]: pgmap v16627: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:35.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:42:35 smithi083 ceph-mon[77242]: pgmap v16627: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:35.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:42:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:42:35.595Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:42:36.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:42:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:42:36.330Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:42:37.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:42:37 smithi153 ceph-mon[70044]: pgmap v16628: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:37.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:42:37 smithi083 ceph-mon[79930]: pgmap v16628: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:37.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:42:37 smithi083 ceph-mon[77242]: pgmap v16628: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:39.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:42:39 smithi153 ceph-mon[70044]: pgmap v16629: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:39.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:42:39 smithi083 ceph-mon[79930]: pgmap v16629: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:39.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:42:39 smithi083 ceph-mon[77242]: pgmap v16629: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:42:41 smithi153 ceph-mon[70044]: pgmap v16630: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:41.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:42:41 smithi083 ceph-mon[79930]: pgmap v16630: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:41.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:42:41 smithi083 ceph-mon[77242]: pgmap v16630: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:42:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:42:41] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T14:42:42.315 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:42:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:42:42.316 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:42:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:42:42.316 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:42:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:42:42.316 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:42:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:42:42.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:42:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:42:42.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:42:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:42:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:42:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:42:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:42:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:42:42.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:42:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:42:42.237Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:42:42.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:42:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:42:42.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:42:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:42:42.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:42:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:42:42.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:42:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:42:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:42:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:42:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:42:43.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:42:43 smithi153 ceph-mon[70044]: pgmap v16631: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:42:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:42:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:42:43.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:42:43 smithi083 ceph-mon[79930]: pgmap v16631: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:43.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:42:43 smithi083 ceph-mon[77242]: pgmap v16631: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:45.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:42:45 smithi083 ceph-mon[79930]: pgmap v16632: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:45.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:42:45 smithi083 ceph-mon[77242]: pgmap v16632: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:45.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:42:45 smithi153 ceph-mon[70044]: pgmap v16632: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:45.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:42:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:42:45.596Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:42:46.249 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:42:46.534 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:42:46.534 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 4m ago 9h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:42:46.535 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 2m ago 9h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:42:46.535 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 4m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:42:46.535 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 2m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:42:46.535 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 2m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:42:46.536 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 4m ago 9h 966M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:42:46.536 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 4m ago 9h 2019M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:42:46.536 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 2m ago 9h 876M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:42:46.536 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 4m ago 9h 1208M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:42:46.537 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 4m ago 9h 23.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:42:46.537 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 2m ago 9h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:42:46.537 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 4m ago 9h 1464M 2219M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:42:46.537 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 4m ago 9h 1451M 2219M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:42:46.537 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 4m ago 9h 1128M 2219M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:42:46.538 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 4m ago 9h 1422M 2219M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:42:46.538 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 2m ago 9h 1247M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:42:46.538 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 2m ago 9h 1565M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:42:46.538 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 2m ago 9h 1398M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:42:46.539 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 2m ago 9h 1662M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:42:46.539 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 2m ago 9h 197M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:42:46.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:42:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:42:46.330Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:42:46.889 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:42:46.890 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:42:46.890 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:42:46.890 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:42:46.890 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:42:46.891 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:42:46.891 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:42:46.891 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:42:46.891 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:42:46.892 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:42:46.892 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:42:46.892 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:42:46.892 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:42:46.892 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:42:46.893 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:42:46.893 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:42:46.893 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:42:46.893 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:42:47.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:42:47 smithi083 ceph-mon[79930]: pgmap v16633: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:47.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:42:47 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3698322662' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:42:47.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:42:47 smithi083 ceph-mon[77242]: pgmap v16633: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:47.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:42:47 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3698322662' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:42:47.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:42:47 smithi153 ceph-mon[70044]: pgmap v16633: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:47.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:42:47 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3698322662' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:42:48.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:42:48 smithi083 ceph-mon[77242]: from='client.53472 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:42:48.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:42:48 smithi083 ceph-mon[77242]: from='client.53478 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:42:48.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:42:48 smithi083 ceph-mon[79930]: from='client.53472 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:42:48.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:42:48 smithi083 ceph-mon[79930]: from='client.53478 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:42:48.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:42:48 smithi153 ceph-mon[70044]: from='client.53472 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:42:48.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:42:48 smithi153 ceph-mon[70044]: from='client.53478 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:42:49.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:42:49 smithi083 ceph-mon[79930]: pgmap v16634: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:49.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:42:49 smithi083 ceph-mon[77242]: pgmap v16634: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:49.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:42:49 smithi153 ceph-mon[70044]: pgmap v16634: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:51.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:42:51 smithi083 ceph-mon[79930]: pgmap v16635: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:51.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:42:51 smithi083 ceph-mon[77242]: pgmap v16635: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:51.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:42:51 smithi153 ceph-mon[70044]: pgmap v16635: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:42:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:42:51] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T14:42:52.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:42:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:42:52.238Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:42:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:42:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:42:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:42:53.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:42:53 smithi153 ceph-mon[70044]: pgmap v16636: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:42:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:42:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:42:53.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:42:53 smithi083 ceph-mon[77242]: pgmap v16636: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:53.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:42:53 smithi083 ceph-mon[79930]: pgmap v16636: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:55.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:42:55 smithi083 ceph-mon[79930]: pgmap v16637: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:55.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:42:55 smithi083 ceph-mon[77242]: pgmap v16637: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:55.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:42:55 smithi153 ceph-mon[70044]: pgmap v16637: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:55.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:42:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:42:55.596Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:42:56.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:42:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:42:56.332Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:42:57.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:42:57 smithi083 ceph-mon[79930]: pgmap v16638: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:57.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:42:57 smithi083 ceph-mon[77242]: pgmap v16638: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:57.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:42:57 smithi153 ceph-mon[70044]: pgmap v16638: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:59.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:42:59 smithi083 ceph-mon[79930]: pgmap v16639: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:59.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:42:59 smithi083 ceph-mon[77242]: pgmap v16639: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:42:59.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:42:59 smithi153 ceph-mon[70044]: pgmap v16639: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:01.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:43:01 smithi083 ceph-mon[79930]: pgmap v16640: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:01.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:43:01 smithi083 ceph-mon[77242]: pgmap v16640: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:01.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:43:01 smithi153 ceph-mon[70044]: pgmap v16640: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:43:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:43:01] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T14:43:02.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:43:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:43:02.238Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:43:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:43:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:43:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:43:03.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:43:03 smithi153 ceph-mon[70044]: pgmap v16641: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:43:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:43:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:43:03.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:43:03 smithi083 ceph-mon[77242]: pgmap v16641: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:03.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:43:03 smithi083 ceph-mon[79930]: pgmap v16641: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:05.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:43:05 smithi083 ceph-mon[79930]: pgmap v16642: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:05.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:43:05 smithi083 ceph-mon[77242]: pgmap v16642: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:05.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:43:05 smithi153 ceph-mon[70044]: pgmap v16642: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:05.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:43:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:43:05.597Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:43:06.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:43:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:43:06.333Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:43:07.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:43:07 smithi083 ceph-mon[79930]: pgmap v16643: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:07.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:43:07 smithi083 ceph-mon[77242]: pgmap v16643: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:07.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:43:07 smithi153 ceph-mon[70044]: pgmap v16643: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:09.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:43:09 smithi083 ceph-mon[79930]: pgmap v16644: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:09.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:43:09 smithi083 ceph-mon[77242]: pgmap v16644: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:09.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:43:09 smithi153 ceph-mon[70044]: pgmap v16644: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:11.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:43:11 smithi083 ceph-mon[79930]: pgmap v16645: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:11.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:43:11 smithi083 ceph-mon[77242]: pgmap v16645: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:11.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:43:11 smithi153 ceph-mon[70044]: pgmap v16645: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:43:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:43:11] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T14:43:12.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:43:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:43:12.239Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:43:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:43:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:43:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:43:13.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:43:13 smithi153 ceph-mon[70044]: pgmap v16646: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:43:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:43:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:43:13.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:43:13 smithi083 ceph-mon[79930]: pgmap v16646: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:13.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:43:13 smithi083 ceph-mon[77242]: pgmap v16646: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:15.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:43:15 smithi083 ceph-mon[77242]: pgmap v16647: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:15.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:43:15 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:43:15.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:43:15 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:43:15.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:43:15 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:43:15.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:43:15 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:43:15.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:43:15 smithi083 ceph-mon[79930]: pgmap v16647: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:15.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:43:15 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:43:15.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:43:15 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:43:15.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:43:15 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:43:15.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:43:15 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:43:15.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:43:15 smithi153 ceph-mon[70044]: pgmap v16647: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:15.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:43:15 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:43:15.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:43:15 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:43:15.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:43:15 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:43:15.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:43:15 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:43:15.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:43:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:43:15.597Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:43:16.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:43:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:43:16.333Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:43:17.220 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:43:17.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:43:17 smithi083 ceph-mon[79930]: pgmap v16648: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:17.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:43:17 smithi083 ceph-mon[77242]: pgmap v16648: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:17.510 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:43:17.511 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 5m ago 9h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:43:17.511 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 3m ago 9h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:43:17.511 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 5m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:43:17.511 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 3m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:43:17.511 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 3m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:43:17.512 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 5m ago 9h 966M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:43:17.512 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 5m ago 9h 2019M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:43:17.512 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 3m ago 9h 876M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:43:17.512 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 5m ago 9h 1208M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:43:17.512 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 5m ago 9h 23.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:43:17.513 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 3m ago 9h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:43:17.513 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 5m ago 9h 1464M 2219M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:43:17.513 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 5m ago 9h 1451M 2219M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:43:17.513 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 5m ago 9h 1128M 2219M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:43:17.513 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 5m ago 9h 1422M 2219M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:43:17.514 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 3m ago 9h 1247M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:43:17.514 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 3m ago 9h 1565M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:43:17.514 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 3m ago 9h 1398M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:43:17.514 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 3m ago 9h 1662M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:43:17.515 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 3m ago 9h 197M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:43:17.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:43:17 smithi153 ceph-mon[70044]: pgmap v16648: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:17.865 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:43:17.865 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:43:17.866 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:43:17.866 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:43:17.866 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:43:17.866 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:43:17.866 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:43:17.867 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:43:17.867 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:43:17.867 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:43:17.867 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:43:17.867 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:43:17.868 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:43:17.868 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:43:17.868 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:43:17.868 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:43:17.868 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:43:17.869 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:43:18.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:43:18 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/143259750' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:43:18.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:43:18 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/143259750' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:43:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:43:18 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/143259750' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:43:19.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:43:19 smithi083 ceph-mon[77242]: from='client.53490 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:43:19.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:43:19 smithi083 ceph-mon[77242]: from='client.53496 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:43:19.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:43:19 smithi083 ceph-mon[77242]: pgmap v16649: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:19.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:43:19 smithi083 ceph-mon[79930]: from='client.53490 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:43:19.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:43:19 smithi083 ceph-mon[79930]: from='client.53496 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:43:19.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:43:19 smithi083 ceph-mon[79930]: pgmap v16649: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:19.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:43:19 smithi153 ceph-mon[70044]: from='client.53490 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:43:19.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:43:19 smithi153 ceph-mon[70044]: from='client.53496 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:43:19.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:43:19 smithi153 ceph-mon[70044]: pgmap v16649: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:21.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:43:21 smithi083 ceph-mon[77242]: pgmap v16650: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:21.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:43:21 smithi083 ceph-mon[79930]: pgmap v16650: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:21.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:43:21 smithi153 ceph-mon[70044]: pgmap v16650: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:43:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:43:21] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T14:43:22.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:43:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:43:22.240Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:43:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:43:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:43:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:43:23.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:43:23 smithi153 ceph-mon[70044]: pgmap v16651: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:43:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:43:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:43:23.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:43:23 smithi083 ceph-mon[79930]: pgmap v16651: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:23.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:43:23 smithi083 ceph-mon[77242]: pgmap v16651: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:25.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:43:25 smithi083 ceph-mon[79930]: pgmap v16652: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:25.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:43:25 smithi083 ceph-mon[77242]: pgmap v16652: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:25.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:43:25 smithi153 ceph-mon[70044]: pgmap v16652: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:25.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:43:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:43:25.599Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:43:26.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:43:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:43:26.334Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:43:27.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:43:27 smithi083 ceph-mon[79930]: pgmap v16653: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:27.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:43:27 smithi083 ceph-mon[77242]: pgmap v16653: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:27.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:43:27 smithi153 ceph-mon[70044]: pgmap v16653: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:29.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:43:29 smithi083 ceph-mon[79930]: pgmap v16654: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:29.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:43:29 smithi083 ceph-mon[77242]: pgmap v16654: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:29.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:43:29 smithi153 ceph-mon[70044]: pgmap v16654: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:31.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:43:31 smithi083 ceph-mon[79930]: pgmap v16655: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:31.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:43:31 smithi083 ceph-mon[77242]: pgmap v16655: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:31.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:43:31 smithi153 ceph-mon[70044]: pgmap v16655: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:43:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:43:31] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T14:43:32.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:43:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:43:32.240Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:43:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:43:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:43:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:43:33.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:43:33 smithi153 ceph-mon[70044]: pgmap v16656: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:43:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:43:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:43:33.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:43:33 smithi083 ceph-mon[79930]: pgmap v16656: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:33.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:43:33 smithi083 ceph-mon[77242]: pgmap v16656: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:35.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:43:35 smithi083 ceph-mon[79930]: pgmap v16657: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:35.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:43:35 smithi083 ceph-mon[77242]: pgmap v16657: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:35.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:43:35 smithi153 ceph-mon[70044]: pgmap v16657: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:35.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:43:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:43:35.599Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:43:36.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:43:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:43:36.334Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:43:37.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:43:37 smithi083 ceph-mon[77242]: pgmap v16658: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:37.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:43:37 smithi083 ceph-mon[79930]: pgmap v16658: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:37.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:43:37 smithi153 ceph-mon[70044]: pgmap v16658: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:39.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:43:39 smithi083 ceph-mon[77242]: pgmap v16659: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:39.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:43:39 smithi083 ceph-mon[79930]: pgmap v16659: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:39.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:43:39 smithi153 ceph-mon[70044]: pgmap v16659: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:41.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:43:41 smithi083 ceph-mon[79930]: pgmap v16660: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:41.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:43:41 smithi083 ceph-mon[77242]: pgmap v16660: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:43:41 smithi153 ceph-mon[70044]: pgmap v16660: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:43:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:43:41] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T14:43:42.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:43:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:43:42.242Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:43:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:43:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:43:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:43:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:43:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:43:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:43:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:43:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:43:42.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:43:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:43:42.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:43:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:43:42.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:43:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:43:42.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:43:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:43:42.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:43:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:43:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:43:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:43:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:43:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:43:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:43:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:43:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:43:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:43:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:43:43.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:43:43 smithi153 ceph-mon[70044]: pgmap v16661: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:43:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:43:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:43:43.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:43:43 smithi083 ceph-mon[79930]: pgmap v16661: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:43.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:43:43 smithi083 ceph-mon[77242]: pgmap v16661: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:45.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:43:45 smithi083 ceph-mon[79930]: pgmap v16662: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:45.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:43:45 smithi083 ceph-mon[77242]: pgmap v16662: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:45.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:43:45 smithi153 ceph-mon[70044]: pgmap v16662: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:45.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:43:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:43:45.600Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:43:46.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:43:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:43:46.335Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:43:47.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:43:47 smithi083 ceph-mon[79930]: pgmap v16663: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:47.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:43:47 smithi083 ceph-mon[77242]: pgmap v16663: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:47.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:43:47 smithi153 ceph-mon[70044]: pgmap v16663: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:48.192 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:43:48.481 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:43:48.481 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 5m ago 9h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:43:48.482 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 3m ago 9h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:43:48.482 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 5m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:43:48.482 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 3m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:43:48.482 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 3m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:43:48.483 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 5m ago 9h 966M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:43:48.483 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 5m ago 9h 2019M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:43:48.483 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 3m ago 9h 876M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:43:48.483 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 5m ago 9h 1208M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:43:48.484 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 5m ago 9h 23.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:43:48.484 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 3m ago 9h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:43:48.484 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 5m ago 9h 1464M 2219M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:43:48.484 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 5m ago 9h 1451M 2219M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:43:48.485 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 5m ago 9h 1128M 2219M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:43:48.485 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 5m ago 9h 1422M 2219M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:43:48.485 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 3m ago 9h 1247M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:43:48.485 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 3m ago 9h 1565M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:43:48.486 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 3m ago 9h 1398M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:43:48.486 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 3m ago 9h 1662M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:43:48.486 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 3m ago 9h 197M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:43:48.836 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:43:48.836 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:43:48.837 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:43:48.837 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:43:48.837 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:43:48.837 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:43:48.838 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:43:48.838 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:43:48.838 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:43:48.839 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:43:48.839 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:43:48.839 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:43:48.839 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:43:48.839 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:43:48.840 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:43:48.840 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:43:48.840 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:43:48.840 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:43:49.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:43:49 smithi083 ceph-mon[77242]: pgmap v16664: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:49.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:43:49 smithi083 ceph-mon[77242]: from='client.63439 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:43:49.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:43:49 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1373840192' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:43:49.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:43:49 smithi083 ceph-mon[79930]: pgmap v16664: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:49.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:43:49 smithi083 ceph-mon[79930]: from='client.63439 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:43:49.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:43:49 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1373840192' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:43:49.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:43:49 smithi153 ceph-mon[70044]: pgmap v16664: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:49.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:43:49 smithi153 ceph-mon[70044]: from='client.63439 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:43:49.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:43:49 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1373840192' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:43:50.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:43:50 smithi083 ceph-mon[77242]: from='client.63445 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:43:50.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:43:50 smithi083 ceph-mon[79930]: from='client.63445 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:43:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:43:50 smithi153 ceph-mon[70044]: from='client.63445 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:43:51.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:43:51 smithi083 ceph-mon[79930]: pgmap v16665: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:51.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:43:51 smithi083 ceph-mon[77242]: pgmap v16665: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:51.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:43:51 smithi153 ceph-mon[70044]: pgmap v16665: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:43:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:43:51] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T14:43:52.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:43:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:43:52.242Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:43:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:43:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:43:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:43:53.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:43:53 smithi153 ceph-mon[70044]: pgmap v16666: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:43:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:43:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:43:53.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:43:53 smithi083 ceph-mon[79930]: pgmap v16666: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:53.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:43:53 smithi083 ceph-mon[77242]: pgmap v16666: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:55.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:43:55 smithi083 ceph-mon[77242]: pgmap v16667: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:55.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:43:55 smithi083 ceph-mon[79930]: pgmap v16667: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:55.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:43:55 smithi153 ceph-mon[70044]: pgmap v16667: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:55.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:43:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:43:55.601Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:43:56.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:43:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:43:56.335Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:43:57.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:43:57 smithi083 ceph-mon[79930]: pgmap v16668: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:57.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:43:57 smithi083 ceph-mon[77242]: pgmap v16668: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:57.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:43:57 smithi153 ceph-mon[70044]: pgmap v16668: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:59.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:43:59 smithi083 ceph-mon[79930]: pgmap v16669: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:59.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:43:59 smithi083 ceph-mon[77242]: pgmap v16669: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:43:59.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:43:59 smithi153 ceph-mon[70044]: pgmap v16669: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:01.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:44:01 smithi083 ceph-mon[79930]: pgmap v16670: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:01.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:44:01 smithi083 ceph-mon[77242]: pgmap v16670: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:01.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:44:01 smithi153 ceph-mon[70044]: pgmap v16670: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:44:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:44:01] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T14:44:02.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:44:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:44:02.242Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:44:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:44:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:44:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:44:03.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:44:03 smithi153 ceph-mon[70044]: pgmap v16671: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:44:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:44:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:44:03.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:44:03 smithi083 ceph-mon[77242]: pgmap v16671: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:03.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:44:03 smithi083 ceph-mon[79930]: pgmap v16671: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:05.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:44:05 smithi083 ceph-mon[79930]: pgmap v16672: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:05.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:44:05 smithi083 ceph-mon[77242]: pgmap v16672: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:05.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:44:05 smithi153 ceph-mon[70044]: pgmap v16672: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:05.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:44:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:44:05.601Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:44:06.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:44:06 smithi083 ceph-mon[77242]: pgmap v16673: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:06.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:44:06 smithi083 ceph-mon[79930]: pgmap v16673: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:06.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:44:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:44:06.336Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:44:06.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:44:06 smithi153 ceph-mon[70044]: pgmap v16673: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:09.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:44:09 smithi153 ceph-mon[70044]: pgmap v16674: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:09.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:44:09 smithi083 ceph-mon[79930]: pgmap v16674: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:09.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:44:09 smithi083 ceph-mon[77242]: pgmap v16674: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:11.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:44:11 smithi153 ceph-mon[70044]: pgmap v16675: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:11.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:44:11 smithi083 ceph-mon[77242]: pgmap v16675: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:11.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:44:11 smithi083 ceph-mon[79930]: pgmap v16675: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:44:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:44:11] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T14:44:12.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:44:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:44:12.243Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:44:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:44:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:44:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:44:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:44:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:44:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:44:13.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:44:13 smithi153 ceph-mon[70044]: pgmap v16676: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:13.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:44:13 smithi083 ceph-mon[79930]: pgmap v16676: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:13.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:44:13 smithi083 ceph-mon[77242]: pgmap v16676: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:15.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:44:15 smithi153 ceph-mon[70044]: pgmap v16677: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:15.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:44:15 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:44:15.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:44:15 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:44:15.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:44:15 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:44:15.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:44:15 smithi083 ceph-mon[77242]: pgmap v16677: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:15.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:44:15 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:44:15.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:44:15 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:44:15.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:44:15 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:44:15.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:44:15 smithi083 ceph-mon[79930]: pgmap v16677: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:15.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:44:15 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:44:15.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:44:15 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:44:15.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:44:15 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:44:15.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:44:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:44:15.601Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:44:16.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:44:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:44:16.336Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:44:16.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:44:16 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:44:16.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:44:16 smithi083 ceph-mon[77242]: pgmap v16678: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:16.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:44:16 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:44:16.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:44:16 smithi083 ceph-mon[79930]: pgmap v16678: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:16.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:44:16 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:44:16.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:44:16 smithi153 ceph-mon[70044]: pgmap v16678: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:19.160 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:44:19.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:44:19 smithi153 ceph-mon[70044]: pgmap v16679: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:19.449 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:44:19.449 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 6m ago 9h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:44:19.450 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 4m ago 9h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:44:19.450 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 6m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:44:19.450 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 4m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:44:19.450 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 4m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:44:19.451 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 6m ago 9h 966M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:44:19.451 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 6m ago 9h 2019M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:44:19.451 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 4m ago 9h 876M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:44:19.451 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 6m ago 9h 1208M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:44:19.452 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 6m ago 9h 23.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:44:19.452 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 4m ago 9h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:44:19.452 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 6m ago 9h 1464M 2219M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:44:19.452 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 6m ago 9h 1451M 2219M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:44:19.452 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 6m ago 9h 1128M 2219M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:44:19.453 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 6m ago 9h 1422M 2219M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:44:19.453 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 4m ago 9h 1247M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:44:19.453 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 4m ago 9h 1565M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:44:19.453 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 4m ago 9h 1398M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:44:19.454 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 4m ago 9h 1662M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:44:19.454 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 4m ago 9h 197M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:44:19.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:44:19 smithi083 ceph-mon[77242]: pgmap v16679: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:19.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:44:19 smithi083 ceph-mon[79930]: pgmap v16679: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:19.804 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:44:19.804 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:44:19.805 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:44:19.805 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:44:19.805 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:44:19.805 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:44:19.806 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:44:19.806 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:44:19.806 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:44:19.806 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:44:19.806 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:44:19.807 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:44:19.807 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:44:19.807 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:44:19.807 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:44:19.808 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:44:19.808 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:44:19.808 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:44:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:44:20 smithi153 ceph-mon[70044]: from='client.63457 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:44:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:44:20 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1295146486' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:44:20.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:44:20 smithi083 ceph-mon[79930]: from='client.63457 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:44:20.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:44:20 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1295146486' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:44:20.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:44:20 smithi083 ceph-mon[77242]: from='client.63457 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:44:20.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:44:20 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1295146486' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:44:21.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:44:21 smithi153 ceph-mon[70044]: from='client.63463 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:44:21.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:44:21 smithi153 ceph-mon[70044]: pgmap v16680: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:21.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:44:21 smithi083 ceph-mon[77242]: from='client.63463 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:44:21.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:44:21 smithi083 ceph-mon[77242]: pgmap v16680: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:21.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:44:21 smithi083 ceph-mon[79930]: from='client.63463 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:44:21.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:44:21 smithi083 ceph-mon[79930]: pgmap v16680: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:44:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:44:21] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T14:44:22.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:44:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:44:22.243Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:44:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:44:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:44:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:44:23.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:44:23 smithi153 ceph-mon[70044]: pgmap v16681: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:44:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:44:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:44:23.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:44:23 smithi083 ceph-mon[77242]: pgmap v16681: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:23.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:44:23 smithi083 ceph-mon[79930]: pgmap v16681: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:25.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:44:25 smithi083 ceph-mon[77242]: pgmap v16682: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:25.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:44:25 smithi083 ceph-mon[79930]: pgmap v16682: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:25.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:44:25 smithi153 ceph-mon[70044]: pgmap v16682: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:25.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:44:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:44:25.603Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:44:26.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:44:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:44:26.337Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:44:27.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:44:27 smithi083 ceph-mon[77242]: pgmap v16683: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:27.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:44:27 smithi083 ceph-mon[79930]: pgmap v16683: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:27.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:44:27 smithi153 ceph-mon[70044]: pgmap v16683: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:29.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:44:29 smithi083 ceph-mon[77242]: pgmap v16684: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:29.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:44:29 smithi083 ceph-mon[79930]: pgmap v16684: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:29.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:44:29 smithi153 ceph-mon[70044]: pgmap v16684: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:31.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:44:31 smithi083 ceph-mon[79930]: pgmap v16685: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:31.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:44:31 smithi083 ceph-mon[77242]: pgmap v16685: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:31.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:44:31 smithi153 ceph-mon[70044]: pgmap v16685: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:44:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:44:31] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T14:44:32.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:44:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:44:32.244Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:44:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:44:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:44:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:44:33.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:44:33 smithi153 ceph-mon[70044]: pgmap v16686: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:44:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:44:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:44:33.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:44:33 smithi083 ceph-mon[79930]: pgmap v16686: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:33.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:44:33 smithi083 ceph-mon[77242]: pgmap v16686: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:35.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:44:35 smithi083 ceph-mon[79930]: pgmap v16687: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:35.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:44:35 smithi083 ceph-mon[77242]: pgmap v16687: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:35.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:44:35 smithi153 ceph-mon[70044]: pgmap v16687: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:35.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:44:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:44:35.603Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:44:36.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:44:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:44:36.338Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:44:37.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:44:37 smithi083 ceph-mon[79930]: pgmap v16688: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:37.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:44:37 smithi083 ceph-mon[77242]: pgmap v16688: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:37.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:44:37 smithi153 ceph-mon[70044]: pgmap v16688: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:39.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:44:39 smithi083 ceph-mon[79930]: pgmap v16689: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:39.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:44:39 smithi083 ceph-mon[77242]: pgmap v16689: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:39.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:44:39 smithi153 ceph-mon[70044]: pgmap v16689: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:41.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:44:41 smithi083 ceph-mon[79930]: pgmap v16690: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:41.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:44:41 smithi083 ceph-mon[77242]: pgmap v16690: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:44:41 smithi153 ceph-mon[70044]: pgmap v16690: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:44:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:44:41] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T14:44:42.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:44:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:44:42.244Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:44:42.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:44:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:44:42.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:44:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:44:42.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:44:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:44:42.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:44:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:44:42.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:44:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:44:42.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:44:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:44:42.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:44:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:44:42.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:44:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:44:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:44:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:44:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:44:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:44:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:44:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:44:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:44:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:44:42.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:44:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:44:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:44:43.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:44:43 smithi153 ceph-mon[70044]: pgmap v16691: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:44:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:44:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:44:43.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:44:43 smithi083 ceph-mon[79930]: pgmap v16691: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:43.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:44:43 smithi083 ceph-mon[77242]: pgmap v16691: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:45.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:44:45 smithi083 ceph-mon[79930]: pgmap v16692: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:45.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:44:45 smithi083 ceph-mon[77242]: pgmap v16692: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:45.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:44:45 smithi153 ceph-mon[70044]: pgmap v16692: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:45.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:44:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:44:45.604Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:44:46.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:44:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:44:46.338Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:44:47.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:44:47 smithi083 ceph-mon[77242]: pgmap v16693: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:47.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:44:47 smithi083 ceph-mon[79930]: pgmap v16693: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:47.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:44:47 smithi153 ceph-mon[70044]: pgmap v16693: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:49.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:44:49 smithi083 ceph-mon[79930]: pgmap v16694: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:49.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:44:49 smithi083 ceph-mon[77242]: pgmap v16694: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:49.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:44:49 smithi153 ceph-mon[70044]: pgmap v16694: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:50.132 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:44:50.418 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:44:50.419 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 6m ago 9h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:44:50.419 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 4m ago 9h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:44:50.419 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 6m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:44:50.419 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 4m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:44:50.420 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 4m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:44:50.420 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 6m ago 9h 966M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:44:50.420 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 6m ago 9h 2019M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:44:50.420 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 4m ago 9h 876M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:44:50.420 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 6m ago 9h 1208M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:44:50.421 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 6m ago 9h 23.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:44:50.421 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 4m ago 9h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:44:50.421 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 6m ago 9h 1464M 2219M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:44:50.421 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 6m ago 9h 1451M 2219M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:44:50.422 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 6m ago 9h 1128M 2219M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:44:50.422 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 6m ago 9h 1422M 2219M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:44:50.422 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 4m ago 9h 1247M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:44:50.422 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 4m ago 9h 1565M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:44:50.423 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 4m ago 9h 1398M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:44:50.423 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 4m ago 9h 1662M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:44:50.423 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 4m ago 9h 197M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:44:50.774 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:44:50.775 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:44:50.775 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:44:50.775 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:44:50.775 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:44:50.776 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:44:50.776 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:44:50.776 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:44:50.776 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:44:50.777 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:44:50.777 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:44:50.777 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:44:50.777 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:44:50.777 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:44:50.778 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:44:50.778 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:44:50.778 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:44:50.778 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:44:51.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:44:51 smithi083 ceph-mon[79930]: pgmap v16695: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:51.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:44:51 smithi083 ceph-mon[79930]: from='client.63475 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:44:51.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:44:51 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/960718451' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:44:51.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:44:51 smithi083 ceph-mon[77242]: pgmap v16695: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:51.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:44:51 smithi083 ceph-mon[77242]: from='client.63475 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:44:51.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:44:51 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/960718451' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:44:51.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:44:51 smithi153 ceph-mon[70044]: pgmap v16695: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:51.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:44:51 smithi153 ceph-mon[70044]: from='client.63475 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:44:51.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:44:51 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/960718451' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:44:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:44:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:44:51] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T14:44:52.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:44:52 smithi083 ceph-mon[77242]: from='client.53550 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:44:52.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:44:52 smithi083 ceph-mon[79930]: from='client.53550 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:44:52.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:44:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:44:52.245Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:44:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:44:52 smithi153 ceph-mon[70044]: from='client.53550 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:44:52.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:44:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:44:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:44:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:44:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:44:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:44:53.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:44:53 smithi153 ceph-mon[70044]: pgmap v16696: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:53.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:44:53 smithi083 ceph-mon[79930]: pgmap v16696: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:53.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:44:53 smithi083 ceph-mon[77242]: pgmap v16696: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:55.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:44:55 smithi083 ceph-mon[77242]: pgmap v16697: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:55.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:44:55 smithi083 ceph-mon[79930]: pgmap v16697: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:55.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:44:55 smithi153 ceph-mon[70044]: pgmap v16697: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:55.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:44:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:44:55.605Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:44:56.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:44:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:44:56.339Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:44:57.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:44:57 smithi083 ceph-mon[79930]: pgmap v16698: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:57.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:44:57 smithi083 ceph-mon[77242]: pgmap v16698: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:57.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:44:57 smithi153 ceph-mon[70044]: pgmap v16698: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:59.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:44:59 smithi083 ceph-mon[79930]: pgmap v16699: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:59.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:44:59 smithi083 ceph-mon[77242]: pgmap v16699: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:44:59.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:44:59 smithi153 ceph-mon[70044]: pgmap v16699: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:01.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:45:01 smithi083 ceph-mon[79930]: pgmap v16700: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:01.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:45:01 smithi083 ceph-mon[77242]: pgmap v16700: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:01.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:45:01 smithi153 ceph-mon[70044]: pgmap v16700: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:45:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:45:01] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T14:45:02.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:45:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:45:02.245Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:45:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:45:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:45:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:45:03.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:45:03 smithi153 ceph-mon[70044]: pgmap v16701: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:45:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:45:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:45:03.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:45:03 smithi083 ceph-mon[79930]: pgmap v16701: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:03.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:45:03 smithi083 ceph-mon[77242]: pgmap v16701: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:05.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:45:05 smithi083 ceph-mon[77242]: pgmap v16702: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:05.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:45:05 smithi083 ceph-mon[79930]: pgmap v16702: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:05.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:45:05 smithi153 ceph-mon[70044]: pgmap v16702: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:05.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:45:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:45:05.605Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:45:06.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:45:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:45:06.340Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:45:07.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:45:07 smithi083 ceph-mon[79930]: pgmap v16703: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:07.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:45:07 smithi083 ceph-mon[77242]: pgmap v16703: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:07.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:45:07 smithi153 ceph-mon[70044]: pgmap v16703: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:09.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:45:09 smithi083 ceph-mon[79930]: pgmap v16704: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:09.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:45:09 smithi083 ceph-mon[77242]: pgmap v16704: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:09.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:45:09 smithi153 ceph-mon[70044]: pgmap v16704: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:11.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:45:11 smithi083 ceph-mon[79930]: pgmap v16705: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:11.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:45:11 smithi083 ceph-mon[77242]: pgmap v16705: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:11.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:45:11 smithi153 ceph-mon[70044]: pgmap v16705: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:45:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:45:11] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T14:45:12.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:45:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:45:12.246Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:45:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:45:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:45:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:45:13.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:45:13 smithi153 ceph-mon[70044]: pgmap v16706: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:45:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:45:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:45:13.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:45:13 smithi083 ceph-mon[79930]: pgmap v16706: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:13.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:45:13 smithi083 ceph-mon[77242]: pgmap v16706: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:15.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:45:15 smithi083 ceph-mon[79930]: pgmap v16707: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:15.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:45:15 smithi083 ceph-mon[77242]: pgmap v16707: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:15.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:45:15 smithi153 ceph-mon[70044]: pgmap v16707: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:15.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:45:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:45:15.605Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:45:16.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:45:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:45:16.340Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:45:16.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:45:16 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:45:16.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:45:16 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:45:16.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:45:16 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:45:16.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:45:16 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:45:16.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:45:16 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:45:16.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:45:16 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:45:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:45:16 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:45:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:45:16 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:45:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:45:16 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:45:17.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:45:17 smithi083 ceph-mon[79930]: pgmap v16708: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:17.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:45:17 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:45:17.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:45:17 smithi083 ceph-mon[77242]: pgmap v16708: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:17.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:45:17 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:45:17.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:45:17 smithi153 ceph-mon[70044]: pgmap v16708: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:17.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:45:17 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:45:19.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:45:19 smithi083 ceph-mon[79930]: pgmap v16709: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:19.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:45:19 smithi083 ceph-mon[77242]: pgmap v16709: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:19.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:45:19 smithi153 ceph-mon[70044]: pgmap v16709: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:21.099 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:45:21.389 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:45:21.390 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 7m ago 9h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:45:21.390 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 5m ago 9h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:45:21.390 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 7m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:45:21.390 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 5m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:45:21.391 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 5m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:45:21.391 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 7m ago 9h 966M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:45:21.391 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 7m ago 9h 2019M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:45:21.391 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 5m ago 9h 876M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:45:21.392 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 7m ago 9h 1208M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:45:21.392 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 7m ago 9h 23.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:45:21.392 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 5m ago 9h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:45:21.392 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 7m ago 9h 1464M 2219M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:45:21.393 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 7m ago 9h 1451M 2219M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:45:21.393 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 7m ago 9h 1128M 2219M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:45:21.393 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 7m ago 9h 1422M 2219M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:45:21.393 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 5m ago 9h 1247M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:45:21.394 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 5m ago 9h 1565M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:45:21.394 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 5m ago 9h 1398M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:45:21.394 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 5m ago 9h 1662M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:45:21.394 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 5m ago 9h 197M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:45:21.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:45:21 smithi083 ceph-mon[79930]: pgmap v16710: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:21.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:45:21 smithi083 ceph-mon[77242]: pgmap v16710: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:21.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:45:21 smithi153 ceph-mon[70044]: pgmap v16710: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:21.744 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:45:21.745 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:45:21.745 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:45:21.745 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:45:21.746 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:45:21.746 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:45:21.746 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:45:21.746 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:45:21.746 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:45:21.747 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:45:21.747 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:45:21.747 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:45:21.747 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:45:21.748 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:45:21.748 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:45:21.748 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:45:21.748 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:45:21.748 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:45:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:45:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:45:21] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T14:45:22.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:45:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:45:22.246Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:45:22.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:45:22 smithi083 ceph-mon[77242]: from='client.63493 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:45:22.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:45:22 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/225742458' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:45:22.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:45:22 smithi083 ceph-mon[79930]: from='client.63493 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:45:22.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:45:22 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/225742458' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:45:22.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:45:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:45:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:45:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:45:22 smithi153 ceph-mon[70044]: from='client.63493 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:45:22.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:45:22 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/225742458' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:45:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:45:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:45:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:45:23.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:45:23 smithi153 ceph-mon[70044]: from='client.53568 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:45:23.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:45:23 smithi153 ceph-mon[70044]: pgmap v16711: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:23.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:45:23 smithi083 ceph-mon[79930]: from='client.53568 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:45:23.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:45:23 smithi083 ceph-mon[79930]: pgmap v16711: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:23.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:45:23 smithi083 ceph-mon[77242]: from='client.53568 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:45:23.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:45:23 smithi083 ceph-mon[77242]: pgmap v16711: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:25.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:45:25 smithi083 ceph-mon[79930]: pgmap v16712: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:25.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:45:25 smithi083 ceph-mon[77242]: pgmap v16712: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:25.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:45:25 smithi153 ceph-mon[70044]: pgmap v16712: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:25.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:45:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:45:25.606Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:45:26.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:45:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:45:26.341Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:45:27.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:45:27 smithi083 ceph-mon[77242]: pgmap v16713: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:27.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:45:27 smithi083 ceph-mon[79930]: pgmap v16713: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:27.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:45:27 smithi153 ceph-mon[70044]: pgmap v16713: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:29.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:45:29 smithi083 ceph-mon[79930]: pgmap v16714: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:29.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:45:29 smithi083 ceph-mon[77242]: pgmap v16714: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:29.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:45:29 smithi153 ceph-mon[70044]: pgmap v16714: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:31.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:45:31 smithi083 ceph-mon[79930]: pgmap v16715: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:31.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:45:31 smithi083 ceph-mon[77242]: pgmap v16715: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:31.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:45:31 smithi153 ceph-mon[70044]: pgmap v16715: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:45:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:45:31] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T14:45:32.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:45:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:45:32.247Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:45:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:45:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:45:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:45:33.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:45:33 smithi153 ceph-mon[70044]: pgmap v16716: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:45:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:45:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:45:33.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:45:33 smithi083 ceph-mon[79930]: pgmap v16716: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:33.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:45:33 smithi083 ceph-mon[77242]: pgmap v16716: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:35.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:45:35 smithi083 ceph-mon[79930]: pgmap v16717: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:35.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:45:35 smithi083 ceph-mon[77242]: pgmap v16717: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:35.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:45:35 smithi153 ceph-mon[70044]: pgmap v16717: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:35.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:45:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:45:35.607Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:45:36.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:45:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:45:36.341Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:45:37.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:45:37 smithi083 ceph-mon[77242]: pgmap v16718: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:37.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:45:37 smithi083 ceph-mon[79930]: pgmap v16718: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:37.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:45:37 smithi153 ceph-mon[70044]: pgmap v16718: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:39.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:45:39 smithi083 ceph-mon[79930]: pgmap v16719: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:39.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:45:39 smithi083 ceph-mon[77242]: pgmap v16719: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:39.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:45:39 smithi153 ceph-mon[70044]: pgmap v16719: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:41.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:45:41 smithi083 ceph-mon[79930]: pgmap v16720: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:41.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:45:41 smithi083 ceph-mon[77242]: pgmap v16720: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:45:41 smithi153 ceph-mon[70044]: pgmap v16720: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:45:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:45:41] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T14:45:42.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:45:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:45:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:45:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:45:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:45:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:45:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:45:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:45:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:45:42.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:45:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:45:42.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:45:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:45:42.248Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:45:42.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:45:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:45:42.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:45:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:45:42.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:45:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:45:42.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:45:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:45:42.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:45:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:45:42.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:45:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:45:42.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:45:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:45:42.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:45:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:45:43.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:45:43 smithi153 ceph-mon[70044]: pgmap v16721: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:45:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:45:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:45:43.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:45:43 smithi083 ceph-mon[79930]: pgmap v16721: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:43.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:45:43 smithi083 ceph-mon[77242]: pgmap v16721: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:45.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:45:45 smithi153 ceph-mon[70044]: pgmap v16722: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:45.608 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:45:45 smithi083 ceph-mon[79930]: pgmap v16722: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:45.609 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:45:45 smithi083 ceph-mon[77242]: pgmap v16722: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:45.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:45:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:45:45.607Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:45:46.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:45:46 smithi153 ceph-mon[70044]: pgmap v16723: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:46.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:45:46 smithi083 ceph-mon[77242]: pgmap v16723: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:46.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:45:46 smithi083 ceph-mon[79930]: pgmap v16723: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:46.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:45:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:45:46.342Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:45:49.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:45:49 smithi083 ceph-mon[79930]: pgmap v16724: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:49.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:45:49 smithi083 ceph-mon[77242]: pgmap v16724: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:49.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:45:49 smithi153 ceph-mon[70044]: pgmap v16724: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:51.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:45:51 smithi083 ceph-mon[79930]: pgmap v16725: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:51.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:45:51 smithi083 ceph-mon[77242]: pgmap v16725: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:51.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:45:51 smithi153 ceph-mon[70044]: pgmap v16725: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:45:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:45:51] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T14:45:52.075 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:45:52.366 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:45:52.367 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 7m ago 9h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:45:52.367 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 5m ago 9h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:45:52.367 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 7m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:45:52.367 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 5m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:45:52.367 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 5m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:45:52.368 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 7m ago 9h 966M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:45:52.368 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 7m ago 9h 2019M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:45:52.368 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 5m ago 9h 876M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:45:52.368 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 7m ago 9h 1208M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:45:52.369 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 7m ago 9h 23.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:45:52.369 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 5m ago 9h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:45:52.369 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 7m ago 9h 1464M 2219M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:45:52.369 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 7m ago 9h 1451M 2219M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:45:52.370 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 7m ago 9h 1128M 2219M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:45:52.370 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 7m ago 9h 1422M 2219M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:45:52.370 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 5m ago 9h 1247M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:45:52.370 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 5m ago 9h 1565M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:45:52.371 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 5m ago 9h 1398M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:45:52.371 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 5m ago 9h 1662M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:45:52.371 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 5m ago 9h 197M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:45:52.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:45:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:45:52.247Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:45:52.726 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:45:52.727 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:45:52.727 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:45:52.727 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:45:52.727 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:45:52.728 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:45:52.728 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:45:52.728 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:45:52.728 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:45:52.728 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:45:52.729 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:45:52.729 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:45:52.729 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:45:52.729 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:45:52.730 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:45:52.730 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:45:52.730 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:45:52.730 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:45:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:45:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:45:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:45:53.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:45:53 smithi153 ceph-mon[70044]: pgmap v16726: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:53.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:45:53 smithi153 ceph-mon[70044]: from='client.63511 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:45:53.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:45:53 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2640739886' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:45:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:45:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:45:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:45:53.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:45:53 smithi083 ceph-mon[77242]: pgmap v16726: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:53.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:45:53 smithi083 ceph-mon[77242]: from='client.63511 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:45:53.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:45:53 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2640739886' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:45:53.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:45:53 smithi083 ceph-mon[79930]: pgmap v16726: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:53.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:45:53 smithi083 ceph-mon[79930]: from='client.63511 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:45:53.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:45:53 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2640739886' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:45:54.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:45:54 smithi083 ceph-mon[77242]: from='client.53586 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:45:54.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:45:54 smithi083 ceph-mon[79930]: from='client.53586 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:45:54.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:45:54 smithi153 ceph-mon[70044]: from='client.53586 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:45:55.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:45:55 smithi083 ceph-mon[79930]: pgmap v16727: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:55.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:45:55 smithi083 ceph-mon[77242]: pgmap v16727: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:55.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:45:55 smithi153 ceph-mon[70044]: pgmap v16727: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:55.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:45:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:45:55.608Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:45:56.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:45:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:45:56.343Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:45:57.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:45:57 smithi083 ceph-mon[77242]: pgmap v16728: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:57.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:45:57 smithi083 ceph-mon[79930]: pgmap v16728: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:57.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:45:57 smithi153 ceph-mon[70044]: pgmap v16728: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:59.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:45:59 smithi083 ceph-mon[79930]: pgmap v16729: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:59.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:45:59 smithi083 ceph-mon[77242]: pgmap v16729: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:45:59.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:45:59 smithi153 ceph-mon[70044]: pgmap v16729: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:01.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:46:01 smithi083 ceph-mon[79930]: pgmap v16730: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:01.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:46:01 smithi083 ceph-mon[77242]: pgmap v16730: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:01.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:46:01 smithi153 ceph-mon[70044]: pgmap v16730: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:46:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:46:01] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T14:46:02.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:46:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:46:02.249Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:46:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:46:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:46:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:46:03.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:46:03 smithi153 ceph-mon[70044]: pgmap v16731: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:46:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:46:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:46:03.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:46:03 smithi083 ceph-mon[79930]: pgmap v16731: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:03.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:46:03 smithi083 ceph-mon[77242]: pgmap v16731: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:05.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:46:05 smithi083 ceph-mon[79930]: pgmap v16732: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:05.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:46:05 smithi083 ceph-mon[77242]: pgmap v16732: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:05.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:46:05 smithi153 ceph-mon[70044]: pgmap v16732: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:05.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:46:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:46:05.608Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:46:06.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:46:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:46:06.344Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:46:07.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:46:07 smithi083 ceph-mon[77242]: pgmap v16733: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:07.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:46:07 smithi083 ceph-mon[79930]: pgmap v16733: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:07.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:46:07 smithi153 ceph-mon[70044]: pgmap v16733: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:09.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:46:09 smithi083 ceph-mon[79930]: pgmap v16734: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:09.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:46:09 smithi083 ceph-mon[77242]: pgmap v16734: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:09.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:46:09 smithi153 ceph-mon[70044]: pgmap v16734: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:11.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:46:11 smithi083 ceph-mon[79930]: pgmap v16735: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:11.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:46:11 smithi083 ceph-mon[77242]: pgmap v16735: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:11.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:46:11 smithi153 ceph-mon[70044]: pgmap v16735: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:46:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:46:11] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T14:46:12.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:46:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:46:12.250Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:46:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:46:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:46:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:46:13.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:46:13 smithi153 ceph-mon[70044]: pgmap v16736: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:46:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:46:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:46:13.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:46:13 smithi083 ceph-mon[79930]: pgmap v16736: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:13.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:46:13 smithi083 ceph-mon[77242]: pgmap v16736: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:15.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:46:15 smithi083 ceph-mon[79930]: pgmap v16737: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:15.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:46:15 smithi083 ceph-mon[77242]: pgmap v16737: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:15.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:46:15 smithi153 ceph-mon[70044]: pgmap v16737: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:15.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:46:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:46:15.609Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:46:16.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:46:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:46:16.344Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:46:17.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:46:17 smithi083 ceph-mon[79930]: pgmap v16738: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:17.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:46:17 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:46:17.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:46:17 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:46:17.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:46:17 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:46:17.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:46:17 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:46:17.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:46:17 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:46:17.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:46:17 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:46:17.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:46:17 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:46:17.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:46:17 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:46:17.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:46:17 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:46:17.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:46:17 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:46:17.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:46:17 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:46:17.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:46:17 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:46:17.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:46:17 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:46:17.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:46:17 smithi083 ceph-mon[77242]: pgmap v16738: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:17.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:46:17 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:46:17.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:46:17 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:46:17.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:46:17 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:46:17.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:46:17 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:46:17.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:46:17 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:46:17.467 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:46:17 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:46:17.467 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:46:17 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:46:17.467 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:46:17 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:46:17.467 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:46:17 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:46:17.468 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:46:17 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:46:17.468 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:46:17 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:46:17.468 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:46:17 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:46:17.468 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:46:17 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:46:17.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:46:17 smithi153 ceph-mon[70044]: pgmap v16738: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:17.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:46:17 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:46:17.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:46:17 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:46:17.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:46:17 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:46:17.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:46:17 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:46:17.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:46:17 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:46:17.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:46:17 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:46:17.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:46:17 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:46:17.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:46:17 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:46:17.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:46:17 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:46:17.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:46:17 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:46:17.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:46:17 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:46:17.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:46:17 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:46:17.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:46:17 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:46:18.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:46:18 smithi083 ceph-mon[79930]: Adjusting osd_memory_target on smithi083 to 2205M 2022-09-29T14:46:18.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:46:18 smithi083 ceph-mon[77242]: Adjusting osd_memory_target on smithi083 to 2205M 2022-09-29T14:46:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:46:18 smithi153 ceph-mon[70044]: Adjusting osd_memory_target on smithi083 to 2205M 2022-09-29T14:46:19.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:46:19 smithi083 ceph-mon[77242]: pgmap v16739: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:19.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:46:19 smithi083 ceph-mon[79930]: pgmap v16739: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:19.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:46:19 smithi153 ceph-mon[70044]: pgmap v16739: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:21.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:46:21 smithi083 ceph-mon[79930]: pgmap v16740: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:21.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:46:21 smithi083 ceph-mon[77242]: pgmap v16740: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:21.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:46:21 smithi153 ceph-mon[70044]: pgmap v16740: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:46:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:46:21] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T14:46:22.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:46:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:46:22.251Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:46:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:46:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:46:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:46:23.053 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:46:23.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:46:23 smithi153 ceph-mon[70044]: pgmap v16741: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:46:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:46:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:46:23.349 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:46:23.349 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 8m ago 9h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:46:23.349 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 6m ago 9h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:46:23.350 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 8m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:46:23.350 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 6m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:46:23.350 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 6m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:46:23.350 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 8m ago 9h 966M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:46:23.351 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 8m ago 9h 2019M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:46:23.351 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 6m ago 9h 876M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:46:23.351 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 8m ago 9h 1208M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:46:23.351 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 8m ago 9h 23.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:46:23.351 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 6m ago 9h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:46:23.352 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 8m ago 9h 1464M 2205M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:46:23.352 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 8m ago 9h 1451M 2205M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:46:23.352 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 8m ago 9h 1128M 2205M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:46:23.352 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 8m ago 9h 1422M 2205M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:46:23.353 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 6m ago 9h 1247M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:46:23.353 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 6m ago 9h 1565M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:46:23.353 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 6m ago 9h 1398M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:46:23.353 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 6m ago 9h 1662M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:46:23.353 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 6m ago 9h 197M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:46:23.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:46:23 smithi083 ceph-mon[79930]: pgmap v16741: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:23.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:46:23 smithi083 ceph-mon[77242]: pgmap v16741: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:23.710 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:46:23.711 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:46:23.711 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:46:23.711 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:46:23.711 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:46:23.711 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:46:23.712 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:46:23.712 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:46:23.712 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:46:23.712 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:46:23.713 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:46:23.713 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:46:23.713 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:46:23.713 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:46:23.714 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:46:23.714 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:46:23.714 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:46:23.714 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:46:24.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:46:24 smithi083 ceph-mon[79930]: from='client.53598 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:46:24.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:46:24 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1928113209' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:46:24.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:46:24 smithi083 ceph-mon[77242]: from='client.53598 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:46:24.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:46:24 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1928113209' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:46:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:46:24 smithi153 ceph-mon[70044]: from='client.53598 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:46:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:46:24 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1928113209' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:46:25.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:46:25 smithi083 ceph-mon[79930]: from='client.63535 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:46:25.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:46:25 smithi083 ceph-mon[79930]: pgmap v16742: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:25.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:46:25 smithi083 ceph-mon[77242]: from='client.63535 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:46:25.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:46:25 smithi083 ceph-mon[77242]: pgmap v16742: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:25.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:46:25 smithi153 ceph-mon[70044]: from='client.63535 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:46:25.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:46:25 smithi153 ceph-mon[70044]: pgmap v16742: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:25.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:46:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:46:25.610Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:46:26.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:46:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:46:26.345Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:46:27.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:46:27 smithi083 ceph-mon[79930]: pgmap v16743: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:27.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:46:27 smithi083 ceph-mon[77242]: pgmap v16743: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:27.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:46:27 smithi153 ceph-mon[70044]: pgmap v16743: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:29.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:46:29 smithi083 ceph-mon[79930]: pgmap v16744: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:29.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:46:29 smithi083 ceph-mon[77242]: pgmap v16744: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:29.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:46:29 smithi153 ceph-mon[70044]: pgmap v16744: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:31.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:46:31 smithi083 ceph-mon[79930]: pgmap v16745: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:31.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:46:31 smithi083 ceph-mon[77242]: pgmap v16745: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:31.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:46:31 smithi153 ceph-mon[70044]: pgmap v16745: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:46:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:46:31] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T14:46:32.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:46:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:46:32.251Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:46:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:46:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:46:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:46:33.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:46:33 smithi153 ceph-mon[70044]: pgmap v16746: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:46:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:46:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:46:33.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:46:33 smithi083 ceph-mon[79930]: pgmap v16746: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:33.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:46:33 smithi083 ceph-mon[77242]: pgmap v16746: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:35.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:46:35 smithi083 ceph-mon[79930]: pgmap v16747: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:35.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:46:35 smithi083 ceph-mon[77242]: pgmap v16747: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:35.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:46:35 smithi153 ceph-mon[70044]: pgmap v16747: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:35.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:46:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:46:35.610Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:46:36.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:46:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:46:36.346Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:46:37.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:46:37 smithi083 ceph-mon[79930]: pgmap v16748: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:37.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:46:37 smithi083 ceph-mon[77242]: pgmap v16748: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:37.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:46:37 smithi153 ceph-mon[70044]: pgmap v16748: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:39.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:46:39 smithi083 ceph-mon[79930]: pgmap v16749: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:39.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:46:39 smithi083 ceph-mon[77242]: pgmap v16749: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:39.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:46:39 smithi153 ceph-mon[70044]: pgmap v16749: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:41.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:46:41 smithi083 ceph-mon[79930]: pgmap v16750: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:41.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:46:41 smithi083 ceph-mon[77242]: pgmap v16750: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:46:41 smithi153 ceph-mon[70044]: pgmap v16750: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:46:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:46:41] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T14:46:42.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:46:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:46:42.252Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:46:42.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:46:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:46:42.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:46:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:46:42.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:46:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:46:42.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:46:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:46:42.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:46:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:46:42.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:46:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:46:42.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:46:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:46:42.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:46:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:46:42.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:46:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:46:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:46:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:46:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:46:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:46:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:46:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:46:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:46:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:46:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:46:43.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:46:43 smithi153 ceph-mon[70044]: pgmap v16751: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:46:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:46:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:46:43.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:46:43 smithi083 ceph-mon[79930]: pgmap v16751: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:43.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:46:43 smithi083 ceph-mon[77242]: pgmap v16751: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:45.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:46:45 smithi083 ceph-mon[79930]: pgmap v16752: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:45.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:46:45 smithi083 ceph-mon[77242]: pgmap v16752: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:45.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:46:45 smithi153 ceph-mon[70044]: pgmap v16752: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:45.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:46:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:46:45.611Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:46:46.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:46:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:46:46.347Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:46:47.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:46:47 smithi083 ceph-mon[77242]: pgmap v16753: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:47.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:46:47 smithi083 ceph-mon[79930]: pgmap v16753: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:47.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:46:47 smithi153 ceph-mon[70044]: pgmap v16753: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:49.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:46:49 smithi083 ceph-mon[79930]: pgmap v16754: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:49.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:46:49 smithi083 ceph-mon[77242]: pgmap v16754: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:49.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:46:49 smithi153 ceph-mon[70044]: pgmap v16754: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:51.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:46:51 smithi083 ceph-mon[77242]: pgmap v16755: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:51.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:46:51 smithi083 ceph-mon[79930]: pgmap v16755: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:51.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:46:51 smithi153 ceph-mon[70044]: pgmap v16755: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:46:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:46:51] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T14:46:52.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:46:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:46:52.252Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:46:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:46:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:46:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:46:53.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:46:53 smithi153 ceph-mon[70044]: pgmap v16756: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:46:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:46:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:46:53.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:46:53 smithi083 ceph-mon[79930]: pgmap v16756: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:53.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:46:53 smithi083 ceph-mon[77242]: pgmap v16756: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:54.036 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:46:54.328 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:46:54.328 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 8m ago 9h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:46:54.328 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 6m ago 9h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:46:54.328 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 8m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:46:54.329 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 6m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:46:54.329 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 6m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:46:54.329 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 8m ago 9h 966M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:46:54.329 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 8m ago 9h 2019M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:46:54.330 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 6m ago 9h 876M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:46:54.330 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 8m ago 9h 1208M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:46:54.330 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 8m ago 9h 23.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:46:54.330 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 6m ago 9h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:46:54.331 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 8m ago 9h 1464M 2205M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:46:54.331 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 8m ago 9h 1451M 2205M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:46:54.331 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 8m ago 9h 1128M 2205M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:46:54.331 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 8m ago 9h 1422M 2205M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:46:54.331 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 6m ago 9h 1247M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:46:54.332 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 6m ago 9h 1565M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:46:54.332 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 6m ago 9h 1398M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:46:54.332 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 6m ago 9h 1662M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:46:54.332 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 6m ago 9h 197M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:46:54.688 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:46:54.688 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:46:54.688 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:46:54.689 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:46:54.689 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:46:54.689 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:46:54.689 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:46:54.690 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:46:54.690 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:46:54.690 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:46:54.690 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:46:54.691 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:46:54.691 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:46:54.691 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:46:54.691 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:46:54.692 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:46:54.692 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:46:54.692 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:46:55.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:46:55 smithi083 ceph-mon[79930]: pgmap v16757: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:55.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:46:55 smithi083 ceph-mon[79930]: from='client.53616 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:46:55.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:46:55 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1401531941' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:46:55.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:46:55 smithi083 ceph-mon[77242]: pgmap v16757: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:55.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:46:55 smithi083 ceph-mon[77242]: from='client.53616 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:46:55.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:46:55 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1401531941' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:46:55.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:46:55 smithi153 ceph-mon[70044]: pgmap v16757: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:55.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:46:55 smithi153 ceph-mon[70044]: from='client.53616 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:46:55.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:46:55 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1401531941' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:46:55.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:46:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:46:55.611Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:46:56.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:46:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:46:56.348Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:46:56.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:46:56 smithi083 ceph-mon[77242]: from='client.63553 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:46:56.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:46:56 smithi083 ceph-mon[79930]: from='client.63553 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:46:56.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:46:56 smithi153 ceph-mon[70044]: from='client.63553 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:46:57.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:46:57 smithi083 ceph-mon[77242]: pgmap v16758: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:57.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:46:57 smithi083 ceph-mon[79930]: pgmap v16758: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:57.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:46:57 smithi153 ceph-mon[70044]: pgmap v16758: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:59.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:46:59 smithi083 ceph-mon[77242]: pgmap v16759: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:59.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:46:59 smithi083 ceph-mon[79930]: pgmap v16759: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:46:59.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:46:59 smithi153 ceph-mon[70044]: pgmap v16759: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:01.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:47:01 smithi083 ceph-mon[79930]: pgmap v16760: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:01.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:47:01 smithi083 ceph-mon[77242]: pgmap v16760: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:01.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:47:01 smithi153 ceph-mon[70044]: pgmap v16760: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:47:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:47:01] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T14:47:02.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:47:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:47:02.253Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:47:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:47:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:47:02.317Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:47:03.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:47:03 smithi153 ceph-mon[70044]: pgmap v16761: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:47:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:47:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:47:03.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:47:03 smithi083 ceph-mon[79930]: pgmap v16761: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:03.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:47:03 smithi083 ceph-mon[77242]: pgmap v16761: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:05.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:47:05 smithi083 ceph-mon[77242]: pgmap v16762: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:05.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:47:05 smithi083 ceph-mon[79930]: pgmap v16762: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:05.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:47:05 smithi153 ceph-mon[70044]: pgmap v16762: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:05.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:47:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:47:05.612Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:47:06.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:47:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:47:06.348Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:47:07.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:47:07 smithi083 ceph-mon[79930]: pgmap v16763: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:07.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:47:07 smithi083 ceph-mon[77242]: pgmap v16763: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:07.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:47:07 smithi153 ceph-mon[70044]: pgmap v16763: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:09.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:47:09 smithi153 ceph-mon[70044]: pgmap v16764: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:09.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:47:09 smithi083 ceph-mon[77242]: pgmap v16764: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:09.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:47:09 smithi083 ceph-mon[79930]: pgmap v16764: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:11.507 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:47:11 smithi083 ceph-mon[77242]: pgmap v16765: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:11.508 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:47:11 smithi083 ceph-mon[79930]: pgmap v16765: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:11.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:47:11 smithi153 ceph-mon[70044]: pgmap v16765: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:47:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:47:11] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T14:47:12.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:47:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:47:12.253Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:47:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:47:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:47:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:47:13.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:47:13 smithi153 ceph-mon[70044]: pgmap v16766: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:47:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:47:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:47:13.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:47:13 smithi083 ceph-mon[79930]: pgmap v16766: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:13.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:47:13 smithi083 ceph-mon[77242]: pgmap v16766: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:15.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:47:15 smithi153 ceph-mon[70044]: pgmap v16767: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:15.613 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:47:15 smithi083 ceph-mon[79930]: pgmap v16767: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:15.613 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:47:15 smithi083 ceph-mon[77242]: pgmap v16767: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:15.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:47:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:47:15.612Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:47:16.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:47:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:47:16.349Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:47:17.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:47:17 smithi153 ceph-mon[70044]: pgmap v16768: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:17.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:47:17 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:47:17.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:47:17 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:47:17.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:47:17 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:47:17.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:47:17 smithi083 ceph-mon[79930]: pgmap v16768: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:17.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:47:17 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:47:17.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:47:17 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:47:17.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:47:17 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:47:17.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:47:17 smithi083 ceph-mon[77242]: pgmap v16768: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:17.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:47:17 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:47:17.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:47:17 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:47:17.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:47:17 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:47:18.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:47:18 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:47:18.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:47:18 smithi153 ceph-mon[70044]: pgmap v16769: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:18.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:47:18 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:47:18.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:47:18 smithi083 ceph-mon[79930]: pgmap v16769: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:18.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:47:18 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:47:18.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:47:18 smithi083 ceph-mon[77242]: pgmap v16769: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:21.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:47:21 smithi153 ceph-mon[70044]: pgmap v16770: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:21.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:47:21 smithi083 ceph-mon[79930]: pgmap v16770: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:21.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:47:21 smithi083 ceph-mon[77242]: pgmap v16770: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:47:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:47:21] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T14:47:22.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:47:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:47:22.253Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:47:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:47:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:47:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:47:23.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:47:23 smithi153 ceph-mon[70044]: pgmap v16771: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:47:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:47:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:47:23.460 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:47:23 smithi083 ceph-mon[77242]: pgmap v16771: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:23.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:47:23 smithi083 ceph-mon[79930]: pgmap v16771: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:25.006 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:47:25.300 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:47:25.300 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 9m ago 9h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:47:25.300 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 7m ago 9h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:47:25.301 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 9m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:47:25.301 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 7m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:47:25.301 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 7m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:47:25.301 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 9m ago 9h 966M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:47:25.302 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 9m ago 9h 2019M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:47:25.302 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 7m ago 9h 876M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:47:25.302 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 9m ago 9h 1208M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:47:25.302 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 9m ago 9h 23.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:47:25.303 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 7m ago 9h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:47:25.303 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 9m ago 9h 1464M 2205M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:47:25.303 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 9m ago 9h 1451M 2205M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:47:25.303 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 9m ago 9h 1128M 2205M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:47:25.303 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 9m ago 9h 1422M 2205M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:47:25.304 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 7m ago 9h 1247M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:47:25.304 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 7m ago 9h 1565M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:47:25.304 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 7m ago 9h 1398M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:47:25.304 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 7m ago 9h 1662M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:47:25.305 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 7m ago 9h 197M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:47:25.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:47:25 smithi083 ceph-mon[79930]: pgmap v16772: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:25.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:47:25 smithi083 ceph-mon[77242]: pgmap v16772: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:25.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:47:25 smithi153 ceph-mon[70044]: pgmap v16772: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:25.659 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:47:25.659 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:47:25.659 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:47:25.660 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:47:25.660 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:47:25.660 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:47:25.660 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:47:25.661 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:47:25.661 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:47:25.661 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:47:25.661 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:47:25.661 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:47:25.662 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:47:25.662 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:47:25.662 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:47:25.662 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:47:25.663 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:47:25.663 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:47:25.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:47:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:47:25.611Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:47:26.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:47:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:47:26.349Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:47:26.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:47:26 smithi083 ceph-mon[77242]: from='client.63565 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:47:26.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:47:26 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1618961584' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:47:26.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:47:26 smithi083 ceph-mon[79930]: from='client.63565 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:47:26.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:47:26 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1618961584' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:47:26.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:47:26 smithi153 ceph-mon[70044]: from='client.63565 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:47:26.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:47:26 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1618961584' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:47:27.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:47:27 smithi083 ceph-mon[79930]: from='client.53640 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:47:27.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:47:27 smithi083 ceph-mon[79930]: pgmap v16773: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:27.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:47:27 smithi083 ceph-mon[77242]: from='client.53640 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:47:27.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:47:27 smithi083 ceph-mon[77242]: pgmap v16773: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:27.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:47:27 smithi153 ceph-mon[70044]: from='client.53640 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:47:27.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:47:27 smithi153 ceph-mon[70044]: pgmap v16773: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:29.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:47:29 smithi083 ceph-mon[79930]: pgmap v16774: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:29.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:47:29 smithi083 ceph-mon[77242]: pgmap v16774: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:29.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:47:29 smithi153 ceph-mon[70044]: pgmap v16774: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:31.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:47:31 smithi083 ceph-mon[79930]: pgmap v16775: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:31.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:47:31 smithi083 ceph-mon[77242]: pgmap v16775: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:31.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:47:31 smithi153 ceph-mon[70044]: pgmap v16775: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:31.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:47:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:47:31] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T14:47:32.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:47:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:47:32.254Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:47:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:47:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:47:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:47:33.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:47:33 smithi153 ceph-mon[70044]: pgmap v16776: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:47:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:47:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:47:33.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:47:33 smithi083 ceph-mon[79930]: pgmap v16776: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:33.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:47:33 smithi083 ceph-mon[77242]: pgmap v16776: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:35.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:47:35 smithi083 ceph-mon[79930]: pgmap v16777: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:35.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:47:35 smithi083 ceph-mon[77242]: pgmap v16777: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:35.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:47:35 smithi153 ceph-mon[70044]: pgmap v16777: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:35.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:47:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:47:35.613Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:47:36.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:47:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:47:36.351Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:47:37.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:47:37 smithi083 ceph-mon[77242]: pgmap v16778: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:37.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:47:37 smithi083 ceph-mon[79930]: pgmap v16778: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:37.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:47:37 smithi153 ceph-mon[70044]: pgmap v16778: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:39.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:47:39 smithi083 ceph-mon[79930]: pgmap v16779: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:39.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:47:39 smithi083 ceph-mon[77242]: pgmap v16779: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:39.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:47:39 smithi153 ceph-mon[70044]: pgmap v16779: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:41.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:47:41 smithi083 ceph-mon[79930]: pgmap v16780: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:41.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:47:41 smithi083 ceph-mon[77242]: pgmap v16780: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:47:41 smithi153 ceph-mon[70044]: pgmap v16780: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:47:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:47:41] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T14:47:42.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:47:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:47:42.255Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:47:42.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:47:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:47:42.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:47:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:47:42.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:47:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:47:42.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:47:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:47:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:47:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:47:42.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:47:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:47:42.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:47:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:47:42.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:47:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:47:42.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:47:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:47:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:47:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:47:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:47:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:47:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:47:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:47:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:47:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:47:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:47:43.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:47:43 smithi153 ceph-mon[70044]: pgmap v16781: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:47:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:47:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:47:43.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:47:43 smithi083 ceph-mon[79930]: pgmap v16781: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:43.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:47:43 smithi083 ceph-mon[77242]: pgmap v16781: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:45.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:47:45 smithi083 ceph-mon[79930]: pgmap v16782: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:45.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:47:45 smithi083 ceph-mon[77242]: pgmap v16782: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:45.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:47:45 smithi153 ceph-mon[70044]: pgmap v16782: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:45.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:47:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:47:45.614Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:47:46.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:47:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:47:46.351Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:47:47.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:47:47 smithi083 ceph-mon[77242]: pgmap v16783: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:47.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:47:47 smithi083 ceph-mon[79930]: pgmap v16783: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:47.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:47:47 smithi153 ceph-mon[70044]: pgmap v16783: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:49.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:47:49 smithi083 ceph-mon[79930]: pgmap v16784: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:49.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:47:49 smithi083 ceph-mon[77242]: pgmap v16784: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:49.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:47:49 smithi153 ceph-mon[70044]: pgmap v16784: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:51.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:47:51 smithi083 ceph-mon[79930]: pgmap v16785: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:51.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:47:51 smithi083 ceph-mon[77242]: pgmap v16785: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:51.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:47:51 smithi153 ceph-mon[70044]: pgmap v16785: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:47:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:47:51] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T14:47:52.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:47:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:47:52.256Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:47:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:47:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:47:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:47:53.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:47:53 smithi153 ceph-mon[70044]: pgmap v16786: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:47:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:47:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:47:53.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:47:53 smithi083 ceph-mon[79930]: pgmap v16786: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:53.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:47:53 smithi083 ceph-mon[77242]: pgmap v16786: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:55.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:47:55 smithi083 ceph-mon[79930]: pgmap v16787: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:55.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:47:55 smithi083 ceph-mon[77242]: pgmap v16787: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:55.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:47:55 smithi153 ceph-mon[70044]: pgmap v16787: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:55.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:47:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:47:55.614Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:47:55.990 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:47:56.280 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:47:56.280 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 9m ago 9h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:47:56.281 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 7m ago 9h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:47:56.281 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 9m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:47:56.282 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 7m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:47:56.282 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 7m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:47:56.283 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 9m ago 9h 966M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:47:56.283 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 9m ago 9h 2019M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:47:56.284 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 7m ago 9h 876M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:47:56.284 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 9m ago 9h 1208M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:47:56.285 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 9m ago 9h 23.9M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:47:56.285 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 7m ago 9h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:47:56.285 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 9m ago 9h 1464M 2205M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:47:56.286 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 9m ago 9h 1451M 2205M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:47:56.286 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 9m ago 9h 1128M 2205M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:47:56.287 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 9m ago 9h 1422M 2205M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:47:56.287 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 7m ago 9h 1247M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:47:56.288 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 7m ago 9h 1565M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:47:56.288 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 7m ago 9h 1398M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:47:56.288 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 7m ago 9h 1662M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:47:56.289 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 7m ago 9h 197M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:47:56.640 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:47:56.641 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:47:56.641 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:47:56.642 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:47:56.642 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:47:56.643 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:47:56.643 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:47:56.644 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:47:56.644 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:47:56.644 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:47:56.645 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:47:56.645 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:47:56.646 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:47:56.646 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:47:56.647 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:47:56.647 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:47:56.647 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:47:56.648 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:47:56.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:47:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:47:56.351Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:47:57.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:47:57 smithi083 ceph-mon[79930]: pgmap v16788: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:57.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:47:57 smithi083 ceph-mon[79930]: from='client.63583 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:47:57.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:47:57 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1332450902' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:47:57.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:47:57 smithi083 ceph-mon[77242]: pgmap v16788: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:57.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:47:57 smithi083 ceph-mon[77242]: from='client.63583 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:47:57.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:47:57 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1332450902' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:47:57.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:47:57 smithi153 ceph-mon[70044]: pgmap v16788: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:57.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:47:57 smithi153 ceph-mon[70044]: from='client.63583 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:47:57.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:47:57 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1332450902' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:47:58.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:47:58 smithi083 ceph-mon[79930]: from='client.53658 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:47:58.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:47:58 smithi083 ceph-mon[77242]: from='client.53658 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:47:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:47:58 smithi153 ceph-mon[70044]: from='client.53658 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:47:59.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:47:59 smithi083 ceph-mon[77242]: pgmap v16789: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:59.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:47:59 smithi083 ceph-mon[79930]: pgmap v16789: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:47:59.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:47:59 smithi153 ceph-mon[70044]: pgmap v16789: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:01.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:48:01 smithi083 ceph-mon[77242]: pgmap v16790: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:01.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:48:01 smithi083 ceph-mon[79930]: pgmap v16790: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:01.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:48:01 smithi153 ceph-mon[70044]: pgmap v16790: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:48:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:48:01] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T14:48:02.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:48:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:48:02.256Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:48:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:48:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:48:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:48:03.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:48:03 smithi153 ceph-mon[70044]: pgmap v16791: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:48:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:48:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:48:03.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:48:03 smithi083 ceph-mon[79930]: pgmap v16791: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:03.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:48:03 smithi083 ceph-mon[77242]: pgmap v16791: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:05.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:48:05 smithi083 ceph-mon[77242]: pgmap v16792: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:05.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:48:05 smithi083 ceph-mon[79930]: pgmap v16792: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:05.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:48:05 smithi153 ceph-mon[70044]: pgmap v16792: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:05.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:48:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:48:05.615Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:48:06.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:48:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:48:06.352Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:48:07.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:48:07 smithi083 ceph-mon[77242]: pgmap v16793: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:07.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:48:07 smithi083 ceph-mon[79930]: pgmap v16793: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:07.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:48:07 smithi153 ceph-mon[70044]: pgmap v16793: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:09.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:48:09 smithi083 ceph-mon[77242]: pgmap v16794: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:09.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:48:09 smithi083 ceph-mon[79930]: pgmap v16794: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:09.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:48:09 smithi153 ceph-mon[70044]: pgmap v16794: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:11.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:48:11 smithi083 ceph-mon[77242]: pgmap v16795: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:11.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:48:11 smithi083 ceph-mon[79930]: pgmap v16795: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:11.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:48:11 smithi153 ceph-mon[70044]: pgmap v16795: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:48:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:48:11] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T14:48:12.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:48:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:48:12.257Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:48:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:48:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:48:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:48:13.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:48:13 smithi153 ceph-mon[70044]: pgmap v16796: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:48:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:48:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:48:13.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:48:13 smithi083 ceph-mon[79930]: pgmap v16796: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:13.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:48:13 smithi083 ceph-mon[77242]: pgmap v16796: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:15.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:48:15 smithi083 ceph-mon[79930]: pgmap v16797: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:15.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:48:15 smithi083 ceph-mon[77242]: pgmap v16797: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:15.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:48:15 smithi153 ceph-mon[70044]: pgmap v16797: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:15.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:48:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:48:15.616Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:48:16.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:48:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:48:16.353Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:48:17.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:48:17 smithi083 ceph-mon[79930]: pgmap v16798: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:17.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:48:17 smithi083 ceph-mon[77242]: pgmap v16798: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:17.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:48:17 smithi153 ceph-mon[70044]: pgmap v16798: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:18.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:48:18 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:48:18.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:48:18 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:48:18.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:48:18 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:48:18.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:48:18 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:48:18.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:48:18 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:48:18.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:48:18 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:48:18.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:48:18 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:48:18.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:48:18 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:48:18.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:48:18 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:48:18.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:48:18 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:48:18.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:48:18 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:48:18.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:48:18 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:48:18.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:48:18 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:48:18.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:48:18 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:48:18.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:48:18 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:48:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:48:18 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:48:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:48:18 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:48:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:48:18 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:48:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:48:18 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:48:18.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:48:18 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:48:18.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:48:18 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:48:19.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:48:19 smithi083 ceph-mon[79930]: pgmap v16799: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:19.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:48:19 smithi083 ceph-mon[77242]: pgmap v16799: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:19.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:48:19 smithi153 ceph-mon[70044]: pgmap v16799: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:21.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:48:21 smithi083 ceph-mon[79930]: pgmap v16800: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:21.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:48:21 smithi083 ceph-mon[77242]: pgmap v16800: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:21.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:48:21 smithi153 ceph-mon[70044]: pgmap v16800: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:21.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:48:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:48:21] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T14:48:22.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:48:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:48:22.256Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:48:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:48:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:48:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:48:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:48:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:48:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:48:23.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:48:23 smithi153 ceph-mon[70044]: pgmap v16801: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:23.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:48:23 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:48:23.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:48:23 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:48:23.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:48:23 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:48:23.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:48:23 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:48:23.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:48:23 smithi083 ceph-mon[79930]: pgmap v16801: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:23.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:48:23 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:48:23.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:48:23 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:48:23.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:48:23 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:48:23.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:48:23 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:48:23.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:48:23 smithi083 ceph-mon[77242]: pgmap v16801: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:23.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:48:23 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:48:23.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:48:23 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:48:23.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:48:23 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:48:23.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:48:23 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:48:25.522 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:48:25 smithi083 ceph-mon[79930]: pgmap v16802: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:25.523 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:48:25 smithi083 ceph-mon[77242]: pgmap v16802: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:25.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:48:25 smithi153 ceph-mon[70044]: pgmap v16802: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:25.783 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:48:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:48:25.616Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:48:26.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:48:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:48:26.353Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:48:26.971 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:48:27.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:48:26 smithi153 ceph-mon[70044]: Detected new or changed devices on smithi083 2022-09-29T14:48:27.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:48:26 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:48:27.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:48:26 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:48:27.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:48:26 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:48:27.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:48:26 smithi153 ceph-mon[70044]: pgmap v16803: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:27.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:48:26 smithi083 ceph-mon[79930]: Detected new or changed devices on smithi083 2022-09-29T14:48:27.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:48:26 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:48:27.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:48:26 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:48:27.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:48:26 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:48:27.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:48:26 smithi083 ceph-mon[79930]: pgmap v16803: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:27.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:48:26 smithi083 ceph-mon[77242]: Detected new or changed devices on smithi083 2022-09-29T14:48:27.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:48:26 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:48:27.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:48:26 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:48:27.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:48:26 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:48:27.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:48:26 smithi083 ceph-mon[77242]: pgmap v16803: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:27.261 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:48:27.262 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 5s ago 9h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:48:27.262 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 8m ago 9h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:48:27.262 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 5s ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:48:27.262 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 8m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:48:27.263 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 8m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:48:27.263 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 5s ago 9h 974M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:48:27.263 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 5s ago 9h 2055M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:48:27.263 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 8m ago 9h 876M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:48:27.264 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 5s ago 9h 1230M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:48:27.264 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 5s ago 9h 24.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:48:27.264 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 8m ago 9h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:48:27.264 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 5s ago 9h 1490M 2205M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:48:27.265 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 5s ago 9h 1476M 2205M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:48:27.265 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 5s ago 9h 1147M 2205M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:48:27.265 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 5s ago 9h 1447M 2205M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:48:27.265 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 8m ago 9h 1247M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:48:27.266 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 8m ago 9h 1565M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:48:27.266 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 8m ago 9h 1398M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:48:27.266 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 8m ago 9h 1662M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:48:27.266 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 8m ago 9h 197M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:48:27.618 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:48:27.619 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:48:27.619 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:48:27.619 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:48:27.619 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:48:27.620 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:48:27.620 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:48:27.620 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:48:27.620 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:48:27.620 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:48:27.621 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:48:27.621 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:48:27.621 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:48:27.621 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:48:27.622 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:48:27.622 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:48:27.622 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:48:27.622 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:48:28.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:48:28 smithi083 ceph-mon[79930]: from='client.63601 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:48:28.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:48:28 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2932569350' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:48:28.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:48:28 smithi083 ceph-mon[77242]: from='client.63601 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:48:28.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:48:28 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2932569350' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:48:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:48:28 smithi153 ceph-mon[70044]: from='client.63601 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:48:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:48:28 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2932569350' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:48:29.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:48:29 smithi083 ceph-mon[77242]: from='client.63607 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:48:29.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:48:29 smithi083 ceph-mon[77242]: pgmap v16804: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:29.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:48:29 smithi083 ceph-mon[79930]: from='client.63607 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:48:29.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:48:29 smithi083 ceph-mon[79930]: pgmap v16804: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:29.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:48:29 smithi153 ceph-mon[70044]: from='client.63607 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:48:29.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:48:29 smithi153 ceph-mon[70044]: pgmap v16804: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:31.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:48:31 smithi083 ceph-mon[79930]: pgmap v16805: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:31.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:48:31 smithi083 ceph-mon[77242]: pgmap v16805: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:31.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:48:31 smithi153 ceph-mon[70044]: pgmap v16805: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:48:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:48:31] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T14:48:32.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:48:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:48:32.257Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:48:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:48:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:48:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:48:33.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:48:33 smithi153 ceph-mon[70044]: pgmap v16806: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:48:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:48:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:48:33.460 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:48:33 smithi083 ceph-mon[77242]: pgmap v16806: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:33.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:48:33 smithi083 ceph-mon[79930]: pgmap v16806: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:35.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:48:35 smithi083 ceph-mon[77242]: pgmap v16807: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:35.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:48:35 smithi083 ceph-mon[79930]: pgmap v16807: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:35.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:48:35 smithi153 ceph-mon[70044]: pgmap v16807: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:35.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:48:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:48:35.618Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:48:36.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:48:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:48:36.355Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:48:37.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:48:37 smithi083 ceph-mon[79930]: pgmap v16808: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:37.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:48:37 smithi083 ceph-mon[77242]: pgmap v16808: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:37.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:48:37 smithi153 ceph-mon[70044]: pgmap v16808: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:39.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:48:39 smithi083 ceph-mon[79930]: pgmap v16809: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:39.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:48:39 smithi083 ceph-mon[77242]: pgmap v16809: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:39.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:48:39 smithi153 ceph-mon[70044]: pgmap v16809: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:41.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:48:41 smithi083 ceph-mon[79930]: pgmap v16810: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:41.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:48:41 smithi083 ceph-mon[77242]: pgmap v16810: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:48:41 smithi153 ceph-mon[70044]: pgmap v16810: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:41.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:48:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:48:41] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T14:48:42.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:48:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:48:42.258Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:48:42.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:48:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:48:42.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:48:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:48:42.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:48:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:48:42.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:48:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:48:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:48:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:48:42.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:48:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:48:42.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:48:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:48:42.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:48:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:48:42.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:48:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:48:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:48:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:48:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:48:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:48:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:48:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:48:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:48:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:48:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:48:43.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:48:43 smithi153 ceph-mon[70044]: pgmap v16811: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:48:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:48:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:48:43.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:48:43 smithi083 ceph-mon[79930]: pgmap v16811: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:43.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:48:43 smithi083 ceph-mon[77242]: pgmap v16811: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:45.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:48:45 smithi083 ceph-mon[79930]: pgmap v16812: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:45.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:48:45 smithi083 ceph-mon[77242]: pgmap v16812: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:45.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:48:45 smithi153 ceph-mon[70044]: pgmap v16812: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:45.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:48:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:48:45.619Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:48:46.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:48:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:48:46.356Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:48:47.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:48:47 smithi083 ceph-mon[79930]: pgmap v16813: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:47.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:48:47 smithi083 ceph-mon[77242]: pgmap v16813: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:47.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:48:47 smithi153 ceph-mon[70044]: pgmap v16813: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:49.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:48:49 smithi083 ceph-mon[79930]: pgmap v16814: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:49.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:48:49 smithi083 ceph-mon[77242]: pgmap v16814: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:49.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:48:49 smithi153 ceph-mon[70044]: pgmap v16814: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:51.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:48:51 smithi083 ceph-mon[79930]: pgmap v16815: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:51.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:48:51 smithi083 ceph-mon[77242]: pgmap v16815: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:51.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:48:51 smithi153 ceph-mon[70044]: pgmap v16815: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:48:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:48:51] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T14:48:52.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:48:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:48:52.258Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:48:52.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:48:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:48:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:48:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:48:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:48:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:48:53.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:48:53 smithi153 ceph-mon[70044]: pgmap v16816: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:53.460 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:48:53 smithi083 ceph-mon[77242]: pgmap v16816: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:53.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:48:53 smithi083 ceph-mon[79930]: pgmap v16816: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:55.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:48:55 smithi083 ceph-mon[77242]: pgmap v16817: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:55.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:48:55 smithi083 ceph-mon[79930]: pgmap v16817: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:55.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:48:55 smithi153 ceph-mon[70044]: pgmap v16817: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:55.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:48:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:48:55.620Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:48:56.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:48:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:48:56.356Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:48:57.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:48:57 smithi083 ceph-mon[79930]: pgmap v16818: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:57.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:48:57 smithi083 ceph-mon[77242]: pgmap v16818: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:57.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:48:57 smithi153 ceph-mon[70044]: pgmap v16818: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:57.943 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:48:58.233 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:48:58.234 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 36s ago 9h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:48:58.234 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 8m ago 9h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:48:58.234 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 36s ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:48:58.235 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 8m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:48:58.235 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 8m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:48:58.235 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 36s ago 9h 974M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:48:58.235 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 36s ago 9h 2055M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:48:58.236 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 8m ago 9h 876M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:48:58.236 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 36s ago 9h 1230M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:48:58.236 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 36s ago 9h 24.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:48:58.236 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 8m ago 9h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:48:58.237 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 36s ago 9h 1490M 2205M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:48:58.237 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 36s ago 9h 1476M 2205M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:48:58.237 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 36s ago 9h 1147M 2205M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:48:58.237 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 36s ago 9h 1447M 2205M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:48:58.238 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 8m ago 9h 1247M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:48:58.238 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 8m ago 9h 1565M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:48:58.238 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 8m ago 9h 1398M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:48:58.238 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 8m ago 9h 1662M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:48:58.238 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 8m ago 9h 197M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:48:58.590 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:48:58.590 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:48:58.591 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:48:58.591 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:48:58.591 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:48:58.591 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:48:58.592 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:48:58.592 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:48:58.592 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:48:58.592 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:48:58.593 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:48:58.593 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:48:58.593 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:48:58.593 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:48:58.593 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:48:58.594 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:48:58.594 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:48:58.594 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:48:59.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:48:59 smithi083 ceph-mon[79930]: from='client.63619 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:48:59.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:48:59 smithi083 ceph-mon[79930]: pgmap v16819: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:59.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:48:59 smithi083 ceph-mon[79930]: from='client.53694 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:48:59.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:48:59 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2251900910' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:48:59.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:48:59 smithi083 ceph-mon[77242]: from='client.63619 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:48:59.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:48:59 smithi083 ceph-mon[77242]: pgmap v16819: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:59.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:48:59 smithi083 ceph-mon[77242]: from='client.53694 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:48:59.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:48:59 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2251900910' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:48:59.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:48:59 smithi153 ceph-mon[70044]: from='client.63619 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:48:59.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:48:59 smithi153 ceph-mon[70044]: pgmap v16819: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:48:59.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:48:59 smithi153 ceph-mon[70044]: from='client.53694 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:48:59.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:48:59 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2251900910' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:49:01.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:49:01 smithi083 ceph-mon[79930]: pgmap v16820: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:01.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:49:01 smithi083 ceph-mon[77242]: pgmap v16820: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:01.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:49:01 smithi153 ceph-mon[70044]: pgmap v16820: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:49:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:49:01] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T14:49:02.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:49:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:49:02.259Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:49:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:49:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:49:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:49:03.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:49:03 smithi153 ceph-mon[70044]: pgmap v16821: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:49:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:49:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:49:03.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:49:03 smithi083 ceph-mon[79930]: pgmap v16821: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:03.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:49:03 smithi083 ceph-mon[77242]: pgmap v16821: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:05.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:49:05 smithi083 ceph-mon[79930]: pgmap v16822: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:05.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:49:05 smithi083 ceph-mon[77242]: pgmap v16822: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:05.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:49:05 smithi153 ceph-mon[70044]: pgmap v16822: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:05.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:49:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:49:05.620Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:49:06.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:49:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:49:06.357Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:49:07.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:49:07 smithi083 ceph-mon[79930]: pgmap v16823: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:07.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:49:07 smithi083 ceph-mon[77242]: pgmap v16823: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:07.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:49:07 smithi153 ceph-mon[70044]: pgmap v16823: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:09.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:49:09 smithi083 ceph-mon[79930]: pgmap v16824: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:09.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:49:09 smithi083 ceph-mon[77242]: pgmap v16824: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:09.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:49:09 smithi153 ceph-mon[70044]: pgmap v16824: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:11.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:49:11 smithi083 ceph-mon[79930]: pgmap v16825: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:11.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:49:11 smithi083 ceph-mon[77242]: pgmap v16825: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:11.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:49:11 smithi153 ceph-mon[70044]: pgmap v16825: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:11.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:49:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:49:11] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T14:49:12.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:49:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:49:12.260Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:49:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:49:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:49:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:49:13.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:49:13 smithi153 ceph-mon[70044]: pgmap v16826: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:49:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:49:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:49:13.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:49:13 smithi083 ceph-mon[79930]: pgmap v16826: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:13.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:49:13 smithi083 ceph-mon[77242]: pgmap v16826: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:15.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:49:15 smithi083 ceph-mon[79930]: pgmap v16827: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:15.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:49:15 smithi083 ceph-mon[77242]: pgmap v16827: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:15.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:49:15 smithi153 ceph-mon[70044]: pgmap v16827: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:15.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:49:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:49:15.621Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:49:16.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:49:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:49:16.357Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:49:17.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:49:17 smithi083 ceph-mon[77242]: pgmap v16828: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:17.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:49:17 smithi083 ceph-mon[79930]: pgmap v16828: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:17.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:49:17 smithi153 ceph-mon[70044]: pgmap v16828: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:19.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:49:19 smithi083 ceph-mon[79930]: pgmap v16829: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:19.469 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:49:19 smithi083 ceph-mon[77242]: pgmap v16829: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:19.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:49:19 smithi153 ceph-mon[70044]: pgmap v16829: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:21.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:49:21 smithi083 ceph-mon[77242]: pgmap v16830: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:21.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:49:21 smithi083 ceph-mon[79930]: pgmap v16830: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:21.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:49:21 smithi153 ceph-mon[70044]: pgmap v16830: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:49:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:49:21] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T14:49:22.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:49:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:49:22.261Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:49:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:49:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:49:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:49:23.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:49:23 smithi153 ceph-mon[70044]: pgmap v16831: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:49:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:49:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:49:23.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:49:23 smithi083 ceph-mon[79930]: pgmap v16831: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:23.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:49:23 smithi083 ceph-mon[77242]: pgmap v16831: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:25.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:49:25 smithi083 ceph-mon[77242]: pgmap v16832: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:25.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:49:25 smithi083 ceph-mon[79930]: pgmap v16832: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:25.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:49:25 smithi153 ceph-mon[70044]: pgmap v16832: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:25.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:49:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:49:25.622Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:49:26.460 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:49:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:49:26.357Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:49:26.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:49:26 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:49:26.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:49:26 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:49:26.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:49:26 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:49:26.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:49:26 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:49:26.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:49:26 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:49:26.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:49:26 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:49:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:49:26 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:49:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:49:26 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:49:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:49:26 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:49:27.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:49:27 smithi083 ceph-mon[79930]: pgmap v16833: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:27.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:49:27 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:49:27.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:49:27 smithi083 ceph-mon[77242]: pgmap v16833: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:27.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:49:27 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:49:27.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:49:27 smithi153 ceph-mon[70044]: pgmap v16833: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:27.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:49:27 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:49:28.914 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:49:29.202 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:49:29.202 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 66s ago 9h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:49:29.202 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 9m ago 9h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:49:29.203 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 66s ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:49:29.203 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 9m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:49:29.203 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 9m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:49:29.203 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 66s ago 9h 974M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:49:29.204 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 66s ago 9h 2055M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:49:29.204 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 9m ago 9h 876M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:49:29.204 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 66s ago 9h 1230M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:49:29.204 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 66s ago 9h 24.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:49:29.205 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 9m ago 9h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:49:29.205 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 66s ago 9h 1490M 2205M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:49:29.205 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 66s ago 9h 1476M 2205M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:49:29.205 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 66s ago 9h 1147M 2205M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:49:29.206 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 66s ago 9h 1447M 2205M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:49:29.206 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 9m ago 9h 1247M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:49:29.206 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 9m ago 9h 1565M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:49:29.206 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 9m ago 9h 1398M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:49:29.207 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 9m ago 9h 1662M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:49:29.207 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 9m ago 9h 197M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:49:29.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:49:29 smithi153 ceph-mon[70044]: pgmap v16834: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:29.558 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:49:29.558 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:49:29.558 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:49:29.558 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:49:29.559 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:49:29.559 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:49:29.559 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:49:29.559 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:49:29.559 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:49:29.560 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:49:29.560 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:49:29.560 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:49:29.560 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:49:29.561 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:49:29.561 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:49:29.561 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:49:29.561 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:49:29.561 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:49:29.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:49:29 smithi083 ceph-mon[79930]: pgmap v16834: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:29.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:49:29 smithi083 ceph-mon[77242]: pgmap v16834: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:49:30 smithi153 ceph-mon[70044]: from='client.63637 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:49:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:49:30 smithi153 ceph-mon[70044]: from='client.63643 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:49:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:49:30 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/724514634' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:49:30.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:49:30 smithi083 ceph-mon[79930]: from='client.63637 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:49:30.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:49:30 smithi083 ceph-mon[79930]: from='client.63643 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:49:30.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:49:30 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/724514634' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:49:30.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:49:30 smithi083 ceph-mon[77242]: from='client.63637 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:49:30.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:49:30 smithi083 ceph-mon[77242]: from='client.63643 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:49:30.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:49:30 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/724514634' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:49:31.507 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:49:31 smithi083 ceph-mon[79930]: pgmap v16835: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:31.507 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:49:31 smithi083 ceph-mon[77242]: pgmap v16835: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:31.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:49:31 smithi153 ceph-mon[70044]: pgmap v16835: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:49:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:49:31] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T14:49:32.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:49:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:49:32.262Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:49:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:49:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:49:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:49:33.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:49:33 smithi153 ceph-mon[70044]: pgmap v16836: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:49:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:49:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:49:33.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:49:33 smithi083 ceph-mon[79930]: pgmap v16836: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:33.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:49:33 smithi083 ceph-mon[77242]: pgmap v16836: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:35.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:49:35 smithi153 ceph-mon[70044]: pgmap v16837: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:35.622 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:49:35 smithi083 ceph-mon[79930]: pgmap v16837: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:35.623 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:49:35 smithi083 ceph-mon[77242]: pgmap v16837: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:35.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:49:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:49:35.621Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:49:36.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:49:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:49:36.358Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:49:37.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:49:37 smithi153 ceph-mon[70044]: pgmap v16838: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:37.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:49:37 smithi083 ceph-mon[77242]: pgmap v16838: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:37.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:49:37 smithi083 ceph-mon[79930]: pgmap v16838: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:39.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:49:39 smithi153 ceph-mon[70044]: pgmap v16839: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:39.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:49:39 smithi083 ceph-mon[79930]: pgmap v16839: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:39.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:49:39 smithi083 ceph-mon[77242]: pgmap v16839: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:41.507 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:49:41 smithi083 ceph-mon[77242]: pgmap v16840: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:41.508 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:49:41 smithi083 ceph-mon[79930]: pgmap v16840: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:49:41 smithi153 ceph-mon[70044]: pgmap v16840: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:41.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:49:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:49:41] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T14:49:42.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:49:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:49:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:49:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:49:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:49:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:49:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:49:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:49:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:49:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:49:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:49:42.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:49:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:49:42.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:49:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:49:42.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:49:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:49:42.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:49:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:49:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:49:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:49:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:49:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:49:42.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:49:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:49:42.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:49:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:49:42.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:49:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:49:42.263Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:49:43.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:49:43 smithi153 ceph-mon[70044]: pgmap v16841: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:49:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:49:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:49:43.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:49:43 smithi083 ceph-mon[77242]: pgmap v16841: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:43.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:49:43 smithi083 ceph-mon[79930]: pgmap v16841: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:45.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:49:45 smithi153 ceph-mon[70044]: pgmap v16842: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:45.623 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:49:45 smithi083 ceph-mon[79930]: pgmap v16842: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:45.623 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:49:45 smithi083 ceph-mon[77242]: pgmap v16842: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:45.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:49:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:49:45.622Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:49:46.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:49:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:49:46.359Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:49:47.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:49:47 smithi153 ceph-mon[70044]: pgmap v16843: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:47.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:49:47 smithi083 ceph-mon[79930]: pgmap v16843: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:47.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:49:47 smithi083 ceph-mon[77242]: pgmap v16843: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:49.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:49:49 smithi153 ceph-mon[70044]: pgmap v16844: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:49.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:49:49 smithi083 ceph-mon[79930]: pgmap v16844: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:49.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:49:49 smithi083 ceph-mon[77242]: pgmap v16844: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:51.507 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:49:51 smithi083 ceph-mon[79930]: pgmap v16845: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:51.507 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:49:51 smithi083 ceph-mon[77242]: pgmap v16845: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:51.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:49:51 smithi153 ceph-mon[70044]: pgmap v16845: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:49:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:49:51] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T14:49:52.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:49:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:49:52.264Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:49:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:49:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:49:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:49:53.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:49:53 smithi153 ceph-mon[70044]: pgmap v16846: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:49:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:49:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:49:53.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:49:53 smithi083 ceph-mon[79930]: pgmap v16846: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:53.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:49:53 smithi083 ceph-mon[77242]: pgmap v16846: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:54.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:49:54 smithi153 ceph-mon[70044]: pgmap v16847: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:54.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:49:54 smithi083 ceph-mon[79930]: pgmap v16847: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:54.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:49:54 smithi083 ceph-mon[77242]: pgmap v16847: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:55.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:49:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:49:55.623Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:49:56.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:49:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:49:56.359Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:49:57.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:49:57 smithi153 ceph-mon[70044]: pgmap v16848: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:57.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:49:57 smithi083 ceph-mon[79930]: pgmap v16848: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:57.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:49:57 smithi083 ceph-mon[77242]: pgmap v16848: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:59.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:49:59 smithi083 ceph-mon[79930]: pgmap v16849: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:59.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:49:59 smithi083 ceph-mon[77242]: pgmap v16849: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:59.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:49:59 smithi153 ceph-mon[70044]: pgmap v16849: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:49:59.882 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:50:00.172 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:50:00.173 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 97s ago 9h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:50:00.173 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 9m ago 9h 63.9M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:50:00.173 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 97s ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:50:00.173 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 9m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:50:00.174 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 9m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:50:00.174 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 97s ago 9h 974M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:50:00.174 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 97s ago 9h 2055M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:50:00.174 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 9m ago 9h 876M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:50:00.175 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 97s ago 9h 1230M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:50:00.175 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 97s ago 9h 24.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:50:00.175 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 9m ago 9h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:50:00.175 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 97s ago 9h 1490M 2205M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:50:00.176 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 97s ago 9h 1476M 2205M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:50:00.176 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 97s ago 9h 1147M 2205M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:50:00.176 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 97s ago 9h 1447M 2205M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:50:00.176 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 9m ago 9h 1247M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:50:00.177 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 9m ago 9h 1565M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:50:00.177 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 9m ago 9h 1398M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:50:00.177 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 9m ago 9h 1662M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:50:00.177 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 9m ago 9h 197M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:50:00.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:50:00 smithi083 ceph-mon[79930]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T14:50:00.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:50:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[77219]: 2022-09-29T14:49:59.999+0000 7f160f665700 -1 log_channel(cluster) log [ERR] : overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T14:50:00.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:50:00 smithi083 ceph-mon[77242]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T14:50:00.530 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:50:00.531 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:50:00.531 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:50:00.532 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:50:00.532 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:50:00.533 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:50:00.533 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:50:00.534 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:50:00.534 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:50:00.535 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:50:00.535 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:50:00.536 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:50:00.536 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:50:00.537 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:50:00.537 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:50:00.537 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:50:00.538 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:50:00.538 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:50:00.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:50:00 smithi153 ceph-mon[70044]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T14:50:01.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:50:01 smithi083 ceph-mon[79930]: from='client.63655 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:50:01.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:50:01 smithi083 ceph-mon[79930]: pgmap v16850: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:50:01.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:50:01 smithi083 ceph-mon[79930]: from='client.63661 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:50:01.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:50:01 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/664325659' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:50:01.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:50:01 smithi083 ceph-mon[77242]: from='client.63655 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:50:01.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:50:01 smithi083 ceph-mon[77242]: pgmap v16850: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:50:01.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:50:01 smithi083 ceph-mon[77242]: from='client.63661 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:50:01.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:50:01 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/664325659' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:50:01.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:50:01 smithi153 ceph-mon[70044]: from='client.63655 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:50:01.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:50:01 smithi153 ceph-mon[70044]: pgmap v16850: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:50:01.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:50:01 smithi153 ceph-mon[70044]: from='client.63661 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:50:01.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:50:01 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/664325659' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:50:01.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:50:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:50:01] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T14:50:02.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:50:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:50:02.265Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:50:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:50:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:50:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:50:03.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:50:03 smithi153 ceph-mon[70044]: pgmap v16851: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:50:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:50:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:50:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:50:03.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:50:03 smithi083 ceph-mon[79930]: pgmap v16851: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:50:03.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:50:03 smithi083 ceph-mon[77242]: pgmap v16851: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:50:05.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:50:05 smithi083 ceph-mon[79930]: pgmap v16852: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:50:05.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:50:05 smithi083 ceph-mon[77242]: pgmap v16852: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:50:05.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:50:05 smithi153 ceph-mon[70044]: pgmap v16852: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:50:05.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:50:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:50:05.624Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:50:06.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:50:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:50:06.360Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:50:07.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:50:07 smithi083 ceph-mon[79930]: pgmap v16853: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:50:07.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:50:07 smithi083 ceph-mon[77242]: pgmap v16853: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:50:07.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:50:07 smithi153 ceph-mon[70044]: pgmap v16853: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:50:09.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:50:09 smithi083 ceph-mon[77242]: pgmap v16854: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:50:09.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:50:09 smithi083 ceph-mon[79930]: pgmap v16854: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:50:09.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:50:09 smithi153 ceph-mon[70044]: pgmap v16854: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:50:11.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:50:11 smithi083 ceph-mon[79930]: pgmap v16855: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:50:11.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:50:11 smithi083 ceph-mon[77242]: pgmap v16855: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:50:11.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:50:11 smithi153 ceph-mon[70044]: pgmap v16855: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:50:11.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:50:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:50:11] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T14:50:12.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:50:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:50:12.265Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:50:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:50:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:50:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:50:13.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:50:13 smithi153 ceph-mon[70044]: pgmap v16856: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:50:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:50:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:50:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:50:13.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:50:13 smithi083 ceph-mon[79930]: pgmap v16856: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:50:13.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:50:13 smithi083 ceph-mon[77242]: pgmap v16856: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:50:15.460 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:50:15 smithi083 ceph-mon[77242]: pgmap v16857: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:50:15.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:50:15 smithi083 ceph-mon[79930]: pgmap v16857: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:50:15.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:50:15 smithi153 ceph-mon[70044]: pgmap v16857: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:50:15.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:50:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:50:15.624Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:50:16.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:50:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:50:16.361Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:50:17.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:50:17 smithi083 ceph-mon[77242]: pgmap v16858: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:50:17.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:50:17 smithi083 ceph-mon[79930]: pgmap v16858: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:50:17.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:50:17 smithi153 ceph-mon[70044]: pgmap v16858: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:50:19.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:50:19 smithi083 ceph-mon[77242]: pgmap v16859: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:50:19.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:50:19 smithi083 ceph-mon[79930]: pgmap v16859: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:50:19.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:50:19 smithi153 ceph-mon[70044]: pgmap v16859: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:50:21.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:50:21 smithi083 ceph-mon[79930]: pgmap v16860: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:50:21.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:50:21 smithi083 ceph-mon[77242]: pgmap v16860: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:50:21.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:50:21 smithi153 ceph-mon[70044]: pgmap v16860: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:50:21.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:50:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:50:21] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T14:50:22.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:50:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:50:22.266Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:50:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:50:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:50:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:50:23.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:50:23 smithi153 ceph-mon[70044]: pgmap v16861: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:50:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:50:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:50:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:50:23.460 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:50:23 smithi083 ceph-mon[77242]: pgmap v16861: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:50:23.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:50:23 smithi083 ceph-mon[79930]: pgmap v16861: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:50:25.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:50:25 smithi083 ceph-mon[79930]: pgmap v16862: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:50:25.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:50:25 smithi083 ceph-mon[77242]: pgmap v16862: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:50:25.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:50:25 smithi153 ceph-mon[70044]: pgmap v16862: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:50:25.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:50:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:50:25.625Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:50:26.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:50:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:50:26.361Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:50:27.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:50:27 smithi083 ceph-mon[79930]: pgmap v16863: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:50:27.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:50:27 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:50:27.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:50:27 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:50:27.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:50:27 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:50:27.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:50:27 smithi083 ceph-mon[77242]: pgmap v16863: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:50:27.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:50:27 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:50:27.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:50:27 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:50:27.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:50:27 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:50:27.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:50:27 smithi153 ceph-mon[70044]: pgmap v16863: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:50:27.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:50:27 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:50:27.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:50:27 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:50:27.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:50:27 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:50:29.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:50:29 smithi083 ceph-mon[79930]: pgmap v16864: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:50:29.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:50:29 smithi083 ceph-mon[77242]: pgmap v16864: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:50:29.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:50:29 smithi153 ceph-mon[70044]: pgmap v16864: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:50:30.856 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:50:31.151 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:50:31.152 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 2m ago 9h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:50:31.152 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 0s ago 9h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:50:31.153 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 2m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:50:31.153 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 0s ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:50:31.153 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 0s ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:50:31.154 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 2m ago 9h 974M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:50:31.154 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 2m ago 9h 2055M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:50:31.155 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 0s ago 9h 893M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:50:31.155 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 2m ago 9h 1230M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:50:31.156 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 2m ago 9h 24.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:50:31.156 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 0s ago 9h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:50:31.157 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 2m ago 9h 1490M 2205M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:50:31.157 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 2m ago 9h 1476M 2205M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:50:31.157 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 2m ago 9h 1147M 2205M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:50:31.158 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 2m ago 9h 1447M 2205M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:50:31.158 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 0s ago 9h 1269M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:50:31.159 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 0s ago 9h 1593M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:50:31.159 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 0s ago 9h 1424M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:50:31.160 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 0s ago 9h 1693M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:50:31.160 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 0s ago 9h 201M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:50:31.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:50:31 smithi083 ceph-mon[79930]: pgmap v16865: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:50:31.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:50:31 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:50:31.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:50:31 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:50:31.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:50:31 smithi083 ceph-mon[77242]: pgmap v16865: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:50:31.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:50:31 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:50:31.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:50:31 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:50:31.510 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:50:31.510 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:50:31.510 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:50:31.511 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:50:31.511 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:50:31.511 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:50:31.511 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:50:31.512 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:50:31.512 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:50:31.512 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:50:31.512 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:50:31.513 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:50:31.513 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:50:31.513 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:50:31.513 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:50:31.514 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:50:31.514 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:50:31.514 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:50:31.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:50:31 smithi153 ceph-mon[70044]: pgmap v16865: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:50:31.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:50:31 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:50:31.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:50:31 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:50:31.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:50:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:50:31] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T14:50:32.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:50:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:50:32.266Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:50:32.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:50:32 smithi083 ceph-mon[79930]: from='client.63673 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:50:32.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:50:32 smithi083 ceph-mon[79930]: from='client.63679 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:50:32.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:50:32 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2503719428' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:50:32.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:50:32 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:50:32.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:50:32 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:50:32.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:50:32 smithi083 ceph-mon[77242]: from='client.63673 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:50:32.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:50:32 smithi083 ceph-mon[77242]: from='client.63679 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:50:32.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:50:32 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2503719428' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:50:32.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:50:32 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:50:32.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:50:32 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:50:32.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:50:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:50:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:50:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:50:32 smithi153 ceph-mon[70044]: from='client.63673 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:50:32.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:50:32 smithi153 ceph-mon[70044]: from='client.63679 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:50:32.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:50:32 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2503719428' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:50:32.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:50:32 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:50:32.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:50:32 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:50:33.124 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:50:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:50:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:50:33.394 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:50:33 smithi153 ceph-mon[70044]: pgmap v16866: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:50:33.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:50:33 smithi083 ceph-mon[77242]: pgmap v16866: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:50:33.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:50:33 smithi083 ceph-mon[79930]: pgmap v16866: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:50:35.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:50:35 smithi083 ceph-mon[77242]: pgmap v16867: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:50:35.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:50:35 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:50:35.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:50:35 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:50:35.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:50:35 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:50:35.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:50:35 smithi083 ceph-mon[79930]: pgmap v16867: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:50:35.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:50:35 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:50:35.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:50:35 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:50:35.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:50:35 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:50:35.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:50:35 smithi153 ceph-mon[70044]: pgmap v16867: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:50:35.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:50:35 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:50:35.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:50:35 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:50:35.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:50:35 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:50:35.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:50:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:50:35.625Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:50:36.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:50:36 smithi083 ceph-mon[79930]: Detected new or changed devices on smithi153 2022-09-29T14:50:36.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:50:36 smithi083 ceph-mon[77242]: Detected new or changed devices on smithi153 2022-09-29T14:50:36.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:50:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:50:36.362Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:50:36.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:50:36 smithi153 ceph-mon[70044]: Detected new or changed devices on smithi153 2022-09-29T14:50:37.460 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:50:37 smithi083 ceph-mon[77242]: pgmap v16868: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:50:37.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:50:37 smithi083 ceph-mon[79930]: pgmap v16868: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:50:37.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:50:37 smithi153 ceph-mon[70044]: pgmap v16868: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:50:39.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:50:39 smithi083 ceph-mon[77242]: pgmap v16869: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:50:39.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:50:39 smithi083 ceph-mon[79930]: pgmap v16869: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:50:39.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:50:39 smithi153 ceph-mon[70044]: pgmap v16869: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:50:41.460 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:50:41 smithi083 ceph-mon[77242]: pgmap v16870: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:50:41.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:50:41 smithi083 ceph-mon[79930]: pgmap v16870: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:50:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:50:41 smithi153 ceph-mon[70044]: pgmap v16870: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:50:41.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:50:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:50:41] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T14:50:42.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:50:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:50:42.267Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:50:42.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:50:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:50:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:50:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:50:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:50:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:50:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:50:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:50:42.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:50:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:50:42.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:50:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:50:42.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:50:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:50:42.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:50:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:50:42.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:50:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:50:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:50:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:50:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:50:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:50:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:50:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:50:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:50:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:50:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:50:43.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:50:43 smithi153 ceph-mon[70044]: pgmap v16871: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:50:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:50:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:50:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:50:43.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:50:43 smithi083 ceph-mon[79930]: pgmap v16871: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:50:43.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:50:43 smithi083 ceph-mon[77242]: pgmap v16871: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:50:45.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:50:45 smithi083 ceph-mon[79930]: pgmap v16872: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:50:45.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:50:45 smithi083 ceph-mon[77242]: pgmap v16872: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:50:45.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:50:45 smithi153 ceph-mon[70044]: pgmap v16872: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:50:45.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:50:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:50:45.626Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:50:46.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:50:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:50:46.362Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:50:47.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:50:47 smithi153 ceph-mon[70044]: pgmap v16873: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:50:47.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:50:47 smithi083 ceph-mon[79930]: pgmap v16873: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:50:47.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:50:47 smithi083 ceph-mon[77242]: pgmap v16873: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:50:49.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:50:49 smithi153 ceph-mon[70044]: pgmap v16874: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:50:49.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:50:49 smithi083 ceph-mon[79930]: pgmap v16874: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:50:49.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:50:49 smithi083 ceph-mon[77242]: pgmap v16874: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:50:51.506 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:50:51 smithi083 ceph-mon[79930]: pgmap v16875: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:50:51.507 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:50:51 smithi083 ceph-mon[77242]: pgmap v16875: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:50:51.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:50:51 smithi153 ceph-mon[70044]: pgmap v16875: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:50:51.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:50:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:50:51] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T14:50:52.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:50:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:50:52.268Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:50:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:50:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:50:52.317Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:50:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:50:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:50:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:50:53.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:50:53 smithi153 ceph-mon[70044]: pgmap v16876: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:50:53.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:50:53 smithi083 ceph-mon[79930]: pgmap v16876: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:50:53.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:50:53 smithi083 ceph-mon[77242]: pgmap v16876: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:50:55.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:50:55 smithi153 ceph-mon[70044]: pgmap v16877: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:50:55.628 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:50:55 smithi083 ceph-mon[79930]: pgmap v16877: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:50:55.628 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:50:55 smithi083 ceph-mon[77242]: pgmap v16877: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:50:55.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:50:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:50:55.627Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:50:56.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:50:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:50:56.363Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:50:57.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:50:57 smithi153 ceph-mon[70044]: pgmap v16878: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:50:57.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:50:57 smithi083 ceph-mon[79930]: pgmap v16878: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:50:57.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:50:57 smithi083 ceph-mon[77242]: pgmap v16878: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:50:59.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:50:59 smithi153 ceph-mon[70044]: pgmap v16879: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:50:59.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:50:59 smithi083 ceph-mon[79930]: pgmap v16879: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:50:59.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:50:59 smithi083 ceph-mon[77242]: pgmap v16879: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:01.507 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:51:01 smithi083 ceph-mon[79930]: pgmap v16880: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:01.507 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:51:01 smithi083 ceph-mon[77242]: pgmap v16880: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:01.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:51:01 smithi153 ceph-mon[70044]: pgmap v16880: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:01.836 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:51:01.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:51:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:51:01] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T14:51:02.129 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:51:02.129 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 2m ago 9h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:51:02.129 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 31s ago 9h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:51:02.130 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 2m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:51:02.130 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 31s ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:51:02.130 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 31s ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:51:02.130 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 2m ago 9h 974M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:51:02.131 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 2m ago 9h 2055M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:51:02.131 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 31s ago 9h 893M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:51:02.131 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 2m ago 9h 1230M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:51:02.131 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 2m ago 9h 24.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:51:02.132 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 31s ago 9h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:51:02.132 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 2m ago 9h 1490M 2205M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:51:02.132 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 2m ago 9h 1476M 2205M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:51:02.133 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 2m ago 9h 1147M 2205M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:51:02.133 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 2m ago 9h 1447M 2205M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:51:02.133 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 31s ago 9h 1269M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:51:02.133 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 31s ago 9h 1593M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:51:02.134 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 31s ago 9h 1424M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:51:02.134 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 31s ago 9h 1693M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:51:02.134 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 31s ago 9h 201M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:51:02.485 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:51:02.486 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:51:02.486 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:51:02.486 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:51:02.486 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:51:02.487 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:51:02.487 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:51:02.487 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:51:02.487 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:51:02.488 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:51:02.488 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:51:02.488 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:51:02.488 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:51:02.488 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:51:02.489 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:51:02.489 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:51:02.489 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:51:02.489 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:51:02.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:51:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:51:02.267Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:51:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:51:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:51:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:51:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:51:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:51:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:51:03.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:51:03 smithi153 ceph-mon[70044]: from='client.53760 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:51:03.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:51:03 smithi153 ceph-mon[70044]: pgmap v16881: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:03.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:51:03 smithi153 ceph-mon[70044]: from='client.63697 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:51:03.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:51:03 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1794317870' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:51:03.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:51:03 smithi083 ceph-mon[79930]: from='client.53760 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:51:03.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:51:03 smithi083 ceph-mon[79930]: pgmap v16881: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:03.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:51:03 smithi083 ceph-mon[79930]: from='client.63697 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:51:03.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:51:03 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1794317870' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:51:03.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:51:03 smithi083 ceph-mon[77242]: from='client.53760 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:51:03.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:51:03 smithi083 ceph-mon[77242]: pgmap v16881: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:03.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:51:03 smithi083 ceph-mon[77242]: from='client.63697 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:51:03.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:51:03 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1794317870' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:51:05.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:51:05 smithi153 ceph-mon[70044]: pgmap v16882: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:05.628 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:51:05 smithi083 ceph-mon[79930]: pgmap v16882: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:05.628 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:51:05 smithi083 ceph-mon[77242]: pgmap v16882: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:05.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:51:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:51:05.628Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:51:06.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:51:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:51:06.364Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:51:07.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:51:07 smithi153 ceph-mon[70044]: pgmap v16883: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:07.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:51:07 smithi083 ceph-mon[79930]: pgmap v16883: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:07.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:51:07 smithi083 ceph-mon[77242]: pgmap v16883: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:09.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:51:09 smithi153 ceph-mon[70044]: pgmap v16884: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:09.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:51:09 smithi083 ceph-mon[79930]: pgmap v16884: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:09.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:51:09 smithi083 ceph-mon[77242]: pgmap v16884: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:11.507 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:51:11 smithi083 ceph-mon[79930]: pgmap v16885: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:11.507 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:51:11 smithi083 ceph-mon[77242]: pgmap v16885: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:11.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:51:11 smithi153 ceph-mon[70044]: pgmap v16885: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:11.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:51:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:51:11] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T14:51:12.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:51:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:51:12.269Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:51:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:51:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:51:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:51:13.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:51:13 smithi153 ceph-mon[70044]: pgmap v16886: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:51:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:51:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:51:13.710 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:51:13 smithi083 ceph-mon[77242]: pgmap v16886: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:13.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:51:13 smithi083 ceph-mon[79930]: pgmap v16886: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:15.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:51:15 smithi153 ceph-mon[70044]: pgmap v16887: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:15.628 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:51:15 smithi083 ceph-mon[79930]: pgmap v16887: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:15.629 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:51:15 smithi083 ceph-mon[77242]: pgmap v16887: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:15.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:51:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:51:15.628Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:51:16.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:51:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:51:16.365Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:51:17.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:51:17 smithi153 ceph-mon[70044]: pgmap v16888: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:17.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:51:17 smithi083 ceph-mon[79930]: pgmap v16888: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:17.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:51:17 smithi083 ceph-mon[77242]: pgmap v16888: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:19.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:51:19 smithi153 ceph-mon[70044]: pgmap v16889: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:19.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:51:19 smithi083 ceph-mon[79930]: pgmap v16889: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:19.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:51:19 smithi083 ceph-mon[77242]: pgmap v16889: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:21.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:51:21 smithi153 ceph-mon[70044]: pgmap v16890: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:21.710 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:51:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:51:21] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T14:51:21.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:51:21 smithi083 ceph-mon[79930]: pgmap v16890: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:21.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:51:21 smithi083 ceph-mon[77242]: pgmap v16890: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:22.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:51:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:51:22.270Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:51:22.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:51:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:51:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:51:23.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:51:23 smithi153 ceph-mon[70044]: pgmap v16891: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:51:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:51:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:51:23.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:51:23 smithi083 ceph-mon[79930]: pgmap v16891: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:23.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:51:23 smithi083 ceph-mon[77242]: pgmap v16891: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:25.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:51:25 smithi083 ceph-mon[79930]: pgmap v16892: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:25.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:51:25 smithi083 ceph-mon[77242]: pgmap v16892: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:25.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:51:25 smithi153 ceph-mon[70044]: pgmap v16892: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:25.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:51:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:51:25.628Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:51:26.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:51:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:51:26.366Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:51:27.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:51:27 smithi083 ceph-mon[79930]: pgmap v16893: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:27.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:51:27 smithi083 ceph-mon[77242]: pgmap v16893: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:27.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:51:27 smithi153 ceph-mon[70044]: pgmap v16893: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:29.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:51:29 smithi083 ceph-mon[77242]: pgmap v16894: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:29.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:51:29 smithi083 ceph-mon[79930]: pgmap v16894: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:29.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:51:29 smithi153 ceph-mon[70044]: pgmap v16894: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:31.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:51:31 smithi083 ceph-mon[79930]: pgmap v16895: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:31.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:51:31 smithi083 ceph-mon[77242]: pgmap v16895: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:31.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:51:31 smithi153 ceph-mon[70044]: pgmap v16895: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:31.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:51:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:51:31] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T14:51:32.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:51:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:51:32.271Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:51:32.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:51:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:51:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:51:32.812 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:51:33.099 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:51:33.099 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 3m ago 9h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:51:33.099 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 62s ago 9h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:51:33.100 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 3m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:51:33.100 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 62s ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:51:33.100 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 62s ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:51:33.100 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 3m ago 9h 974M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:51:33.101 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 3m ago 9h 2055M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:51:33.101 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 62s ago 9h 893M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:51:33.101 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 3m ago 9h 1230M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:51:33.101 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 3m ago 9h 24.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:51:33.101 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 62s ago 9h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:51:33.102 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 3m ago 9h 1490M 2205M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:51:33.102 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 3m ago 9h 1476M 2205M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:51:33.102 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 3m ago 9h 1147M 2205M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:51:33.102 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 3m ago 9h 1447M 2205M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:51:33.102 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 62s ago 9h 1269M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:51:33.103 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 62s ago 9h 1593M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:51:33.103 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 62s ago 9h 1424M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:51:33.103 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 62s ago 9h 1693M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:51:33.103 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 62s ago 9h 201M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:51:33.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:51:33 smithi153 ceph-mon[70044]: pgmap v16896: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:51:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:51:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:51:33.458 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:51:33.458 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:51:33.459 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:51:33.459 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:51:33.459 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:51:33.459 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:51:33.459 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:51:33.460 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:51:33.460 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:51:33.460 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:51:33.460 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:51:33.461 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:51:33.461 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:51:33.461 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:51:33.461 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:51:33.461 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:51:33.462 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:51:33.462 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:51:33.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:51:33 smithi083 ceph-mon[77242]: pgmap v16896: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:33.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:51:33 smithi083 ceph-mon[79930]: pgmap v16896: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:34.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:51:34 smithi083 ceph-mon[79930]: from='client.53778 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:51:34.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:51:34 smithi083 ceph-mon[79930]: from='client.53784 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:51:34.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:51:34 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/786884158' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:51:34.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:51:34 smithi083 ceph-mon[77242]: from='client.53778 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:51:34.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:51:34 smithi083 ceph-mon[77242]: from='client.53784 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:51:34.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:51:34 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/786884158' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:51:34.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:51:34 smithi153 ceph-mon[70044]: from='client.53778 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:51:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:51:34 smithi153 ceph-mon[70044]: from='client.53784 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:51:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:51:34 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/786884158' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:51:35.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:51:35 smithi083 ceph-mon[79930]: pgmap v16897: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:35.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:51:35 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:51:35.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:51:35 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:51:35.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:51:35 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:51:35.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:51:35 smithi083 ceph-mon[77242]: pgmap v16897: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:35.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:51:35 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:51:35.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:51:35 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:51:35.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:51:35 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:51:35.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:51:35 smithi153 ceph-mon[70044]: pgmap v16897: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:35.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:51:35 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:51:35.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:51:35 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:51:35.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:51:35 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:51:35.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:51:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:51:35.628Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:51:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:51:36 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:51:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:51:36 smithi153 ceph-mon[70044]: pgmap v16898: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:36.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:51:36 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:51:36.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:51:36 smithi083 ceph-mon[79930]: pgmap v16898: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:36.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:51:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:51:36.366Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:51:36.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:51:36 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:51:36.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:51:36 smithi083 ceph-mon[77242]: pgmap v16898: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:39.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:51:39 smithi083 ceph-mon[79930]: pgmap v16899: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:39.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:51:39 smithi083 ceph-mon[77242]: pgmap v16899: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:39.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:51:39 smithi153 ceph-mon[70044]: pgmap v16899: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:41.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:51:41 smithi083 ceph-mon[79930]: pgmap v16900: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:41.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:51:41 smithi083 ceph-mon[77242]: pgmap v16900: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:41.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:51:41 smithi153 ceph-mon[70044]: pgmap v16900: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:41.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:51:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:51:41] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T14:51:42.460 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:51:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:51:42.272Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:51:42.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:51:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:51:42.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:51:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:51:42.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:51:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:51:42.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:51:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:51:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:51:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:51:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:51:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:51:42.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:51:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:51:42.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:51:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:51:42.556 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:51:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:51:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:51:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:51:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:51:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:51:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:51:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:51:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:51:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:51:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:51:43.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:51:43 smithi153 ceph-mon[70044]: pgmap v16901: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:51:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:51:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:51:43.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:51:43 smithi083 ceph-mon[79930]: pgmap v16901: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:43.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:51:43 smithi083 ceph-mon[77242]: pgmap v16901: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:45.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:51:45 smithi083 ceph-mon[79930]: pgmap v16902: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:45.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:51:45 smithi083 ceph-mon[77242]: pgmap v16902: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:45.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:51:45 smithi153 ceph-mon[70044]: pgmap v16902: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:45.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:51:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:51:45.630Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:51:46.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:51:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:51:46.366Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:51:47.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:51:47 smithi083 ceph-mon[79930]: pgmap v16903: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:47.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:51:47 smithi083 ceph-mon[77242]: pgmap v16903: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:47.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:51:47 smithi153 ceph-mon[70044]: pgmap v16903: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:49.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:51:49 smithi083 ceph-mon[79930]: pgmap v16904: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:49.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:51:49 smithi083 ceph-mon[77242]: pgmap v16904: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:49.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:51:49 smithi153 ceph-mon[70044]: pgmap v16904: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:51.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:51:51 smithi083 ceph-mon[79930]: pgmap v16905: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:51.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:51:51 smithi083 ceph-mon[77242]: pgmap v16905: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:51.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:51:51 smithi153 ceph-mon[70044]: pgmap v16905: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:51.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:51:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:51:51] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T14:51:52.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:51:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:51:52.273Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:51:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:51:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:51:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:51:53.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:51:53 smithi153 ceph-mon[70044]: pgmap v16906: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:51:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:51:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:51:53.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:51:53 smithi083 ceph-mon[79930]: pgmap v16906: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:53.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:51:53 smithi083 ceph-mon[77242]: pgmap v16906: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:55.460 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:51:55 smithi083 ceph-mon[77242]: pgmap v16907: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:55.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:51:55 smithi083 ceph-mon[79930]: pgmap v16907: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:55.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:51:55 smithi153 ceph-mon[70044]: pgmap v16907: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:55.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:51:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:51:55.630Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:51:56.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:51:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:51:56.367Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:51:57.460 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:51:57 smithi083 ceph-mon[77242]: pgmap v16908: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:57.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:51:57 smithi083 ceph-mon[79930]: pgmap v16908: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:57.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:51:57 smithi153 ceph-mon[70044]: pgmap v16908: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:59.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:51:59 smithi083 ceph-mon[79930]: pgmap v16909: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:59.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:51:59 smithi083 ceph-mon[77242]: pgmap v16909: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:51:59.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:51:59 smithi153 ceph-mon[70044]: pgmap v16909: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:01.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:52:01 smithi083 ceph-mon[79930]: pgmap v16910: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:01.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:52:01 smithi083 ceph-mon[77242]: pgmap v16910: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:01.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:52:01 smithi153 ceph-mon[70044]: pgmap v16910: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:01.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:52:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:52:01] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T14:52:02.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:52:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:52:02.273Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:52:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:52:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:52:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:52:03.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:52:03 smithi153 ceph-mon[70044]: pgmap v16911: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:52:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:52:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:52:03.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:52:03 smithi083 ceph-mon[79930]: pgmap v16911: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:03.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:52:03 smithi083 ceph-mon[77242]: pgmap v16911: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:03.795 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:52:04.084 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:52:04.085 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 3m ago 9h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:52:04.085 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 93s ago 9h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:52:04.085 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 3m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:52:04.085 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 93s ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:52:04.086 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 93s ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:52:04.086 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 3m ago 9h 974M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:52:04.086 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 3m ago 9h 2055M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:52:04.086 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 93s ago 9h 893M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:52:04.087 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 3m ago 9h 1230M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:52:04.087 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 3m ago 9h 24.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:52:04.087 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 93s ago 9h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:52:04.087 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 3m ago 9h 1490M 2205M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:52:04.088 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 3m ago 9h 1476M 2205M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:52:04.088 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 3m ago 9h 1147M 2205M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:52:04.088 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 3m ago 9h 1447M 2205M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:52:04.088 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 93s ago 9h 1269M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:52:04.088 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 93s ago 9h 1593M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:52:04.089 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 93s ago 9h 1424M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:52:04.089 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 93s ago 9h 1693M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:52:04.089 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 93s ago 9h 201M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:52:04.438 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:52:04.439 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:52:04.439 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:52:04.439 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:52:04.440 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:52:04.440 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:52:04.440 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:52:04.440 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:52:04.440 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:52:04.441 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:52:04.441 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:52:04.441 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:52:04.441 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:52:04.442 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:52:04.442 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:52:04.442 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:52:04.442 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:52:04.443 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:52:05.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:52:05 smithi083 ceph-mon[79930]: from='client.53796 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:52:05.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:52:05 smithi083 ceph-mon[79930]: pgmap v16912: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:05.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:52:05 smithi083 ceph-mon[79930]: from='client.53802 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:52:05.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:52:05 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4085258100' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:52:05.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:52:05 smithi083 ceph-mon[77242]: from='client.53796 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:52:05.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:52:05 smithi083 ceph-mon[77242]: pgmap v16912: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:05.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:52:05 smithi083 ceph-mon[77242]: from='client.53802 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:52:05.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:52:05 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4085258100' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:52:05.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:52:05 smithi153 ceph-mon[70044]: from='client.53796 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:52:05.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:52:05 smithi153 ceph-mon[70044]: pgmap v16912: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:05.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:52:05 smithi153 ceph-mon[70044]: from='client.53802 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:52:05.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:52:05 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4085258100' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:52:05.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:52:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:52:05.631Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:52:06.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:52:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:52:06.368Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:52:07.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:52:07 smithi083 ceph-mon[79930]: pgmap v16913: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:07.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:52:07 smithi083 ceph-mon[77242]: pgmap v16913: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:07.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:52:07 smithi153 ceph-mon[70044]: pgmap v16913: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:09.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:52:09 smithi083 ceph-mon[79930]: pgmap v16914: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:09.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:52:09 smithi083 ceph-mon[77242]: pgmap v16914: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:09.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:52:09 smithi153 ceph-mon[70044]: pgmap v16914: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:11.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:52:11 smithi083 ceph-mon[79930]: pgmap v16915: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:11.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:52:11 smithi083 ceph-mon[77242]: pgmap v16915: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:11.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:52:11 smithi153 ceph-mon[70044]: pgmap v16915: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:11.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:52:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:52:11] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T14:52:12.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:52:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:52:12.274Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:52:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:52:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:52:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:52:13.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:52:13 smithi153 ceph-mon[70044]: pgmap v16916: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:52:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:52:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:52:13.460 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:52:13 smithi083 ceph-mon[77242]: pgmap v16916: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:13.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:52:13 smithi083 ceph-mon[79930]: pgmap v16916: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:15.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:52:15 smithi153 ceph-mon[70044]: pgmap v16917: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:15.631 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:52:15 smithi083 ceph-mon[79930]: pgmap v16917: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:15.632 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:52:15 smithi083 ceph-mon[77242]: pgmap v16917: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:15.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:52:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:52:15.631Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:52:16.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:52:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:52:16.368Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:52:17.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:52:17 smithi153 ceph-mon[70044]: pgmap v16918: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:17.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:52:17 smithi083 ceph-mon[79930]: pgmap v16918: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:17.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:52:17 smithi083 ceph-mon[77242]: pgmap v16918: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:19.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:52:19 smithi153 ceph-mon[70044]: pgmap v16919: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:19.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:52:19 smithi083 ceph-mon[79930]: pgmap v16919: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:19.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:52:19 smithi083 ceph-mon[77242]: pgmap v16919: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:21.507 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:52:21 smithi083 ceph-mon[79930]: pgmap v16920: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:21.507 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:52:21 smithi083 ceph-mon[77242]: pgmap v16920: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:21.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:52:21 smithi153 ceph-mon[70044]: pgmap v16920: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:21.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:52:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:52:21] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T14:52:22.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:52:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:52:22.275Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:52:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:52:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:52:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:52:23.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:52:23 smithi153 ceph-mon[70044]: pgmap v16921: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:52:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:52:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:52:23.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:52:23 smithi083 ceph-mon[79930]: pgmap v16921: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:23.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:52:23 smithi083 ceph-mon[77242]: pgmap v16921: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:25.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:52:25 smithi153 ceph-mon[70044]: pgmap v16922: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:25.632 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:52:25 smithi083 ceph-mon[79930]: pgmap v16922: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:25.633 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:52:25 smithi083 ceph-mon[77242]: pgmap v16922: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:25.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:52:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:52:25.632Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:52:26.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:52:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:52:26.369Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:52:27.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:52:27 smithi153 ceph-mon[70044]: pgmap v16923: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:27.710 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:52:27 smithi083 ceph-mon[77242]: pgmap v16923: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:27.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:52:27 smithi083 ceph-mon[79930]: pgmap v16923: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:29.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:52:29 smithi153 ceph-mon[70044]: pgmap v16924: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:29.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:52:29 smithi083 ceph-mon[79930]: pgmap v16924: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:29.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:52:29 smithi083 ceph-mon[77242]: pgmap v16924: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:31.506 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:52:31 smithi083 ceph-mon[79930]: pgmap v16925: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:31.506 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:52:31 smithi083 ceph-mon[77242]: pgmap v16925: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:31.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:52:31 smithi153 ceph-mon[70044]: pgmap v16925: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:31.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:52:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:52:31] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T14:52:32.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:52:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:52:32.275Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:52:32.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:52:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:52:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:52:33.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:52:33 smithi153 ceph-mon[70044]: pgmap v16926: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:52:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:52:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:52:33.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:52:33 smithi083 ceph-mon[79930]: pgmap v16926: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:33.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:52:33 smithi083 ceph-mon[77242]: pgmap v16926: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:34.761 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:52:35.049 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:52:35.049 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 4m ago 9h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:52:35.049 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 2m ago 9h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:52:35.050 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 4m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:52:35.050 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 2m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:52:35.050 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 2m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:52:35.050 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 4m ago 9h 974M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:52:35.051 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 4m ago 9h 2055M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:52:35.051 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 2m ago 9h 893M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:52:35.051 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 4m ago 9h 1230M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:52:35.051 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 4m ago 9h 24.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:52:35.052 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 2m ago 9h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:52:35.052 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 4m ago 9h 1490M 2205M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:52:35.052 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 4m ago 9h 1476M 2205M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:52:35.052 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 4m ago 9h 1147M 2205M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:52:35.052 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 4m ago 9h 1447M 2205M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:52:35.053 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 2m ago 9h 1269M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:52:35.053 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 2m ago 9h 1593M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:52:35.053 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 2m ago 9h 1424M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:52:35.053 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 2m ago 9h 1693M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:52:35.053 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 2m ago 9h 201M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:52:35.407 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:52:35.408 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:52:35.408 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:52:35.408 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:52:35.408 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:52:35.409 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:52:35.409 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:52:35.409 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:52:35.409 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:52:35.409 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:52:35.410 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:52:35.410 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:52:35.410 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:52:35.410 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:52:35.411 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:52:35.411 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:52:35.411 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:52:35.411 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:52:35.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:52:35 smithi153 ceph-mon[70044]: pgmap v16927: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:35.634 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:52:35 smithi083 ceph-mon[79930]: pgmap v16927: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:35.634 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:52:35 smithi083 ceph-mon[77242]: pgmap v16927: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:35.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:52:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:52:35.634Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:52:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:52:36 smithi153 ceph-mon[70044]: from='client.63745 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:52:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:52:36 smithi153 ceph-mon[70044]: from='client.63751 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:52:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:52:36 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:52:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:52:36 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:52:36.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:52:36 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:52:36.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:52:36 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3068249123' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:52:36.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:52:36 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:52:36.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:52:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:52:36.369Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:52:36.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:52:36 smithi083 ceph-mon[79930]: from='client.63745 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:52:36.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:52:36 smithi083 ceph-mon[79930]: from='client.63751 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:52:36.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:52:36 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:52:36.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:52:36 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:52:36.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:52:36 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:52:36.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:52:36 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3068249123' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:52:36.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:52:36 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:52:36.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:52:36 smithi083 ceph-mon[77242]: from='client.63745 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:52:36.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:52:36 smithi083 ceph-mon[77242]: from='client.63751 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:52:36.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:52:36 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:52:36.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:52:36 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:52:36.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:52:36 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:52:36.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:52:36 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3068249123' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:52:36.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:52:36 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:52:37.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:52:37 smithi153 ceph-mon[70044]: pgmap v16928: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:37.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:52:37 smithi083 ceph-mon[79930]: pgmap v16928: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:37.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:52:37 smithi083 ceph-mon[77242]: pgmap v16928: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:39.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:52:39 smithi153 ceph-mon[70044]: pgmap v16929: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:39.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:52:39 smithi083 ceph-mon[79930]: pgmap v16929: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:39.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:52:39 smithi083 ceph-mon[77242]: pgmap v16929: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:52:41 smithi153 ceph-mon[70044]: pgmap v16930: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:41.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:52:41 smithi083 ceph-mon[79930]: pgmap v16930: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:41.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:52:41 smithi083 ceph-mon[77242]: pgmap v16930: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:41.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:52:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:52:41] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T14:52:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:52:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:52:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:52:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:52:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:52:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:52:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:52:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:52:42.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:52:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:52:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:52:42.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:52:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:52:42.276Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:52:42.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:52:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:52:42.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:52:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:52:42.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:52:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:52:42.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:52:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:52:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:52:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:52:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:52:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:52:42.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:52:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:52:42.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:52:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:52:43.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:52:43 smithi153 ceph-mon[70044]: pgmap v16931: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:52:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:52:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:52:43.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:52:43 smithi083 ceph-mon[79930]: pgmap v16931: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:43.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:52:43 smithi083 ceph-mon[77242]: pgmap v16931: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:45.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:52:45 smithi153 ceph-mon[70044]: pgmap v16932: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:45.634 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:52:45 smithi083 ceph-mon[79930]: pgmap v16932: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:45.635 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:52:45 smithi083 ceph-mon[77242]: pgmap v16932: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:45.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:52:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:52:45.634Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:52:46.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:52:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:52:46.370Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:52:47.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:52:47 smithi153 ceph-mon[70044]: pgmap v16933: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:47.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:52:47 smithi083 ceph-mon[79930]: pgmap v16933: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:47.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:52:47 smithi083 ceph-mon[77242]: pgmap v16933: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:49.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:52:49 smithi153 ceph-mon[70044]: pgmap v16934: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:49.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:52:49 smithi083 ceph-mon[79930]: pgmap v16934: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:49.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:52:49 smithi083 ceph-mon[77242]: pgmap v16934: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:51.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:52:51 smithi153 ceph-mon[70044]: pgmap v16935: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:51.710 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:52:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:52:51] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T14:52:51.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:52:51 smithi083 ceph-mon[79930]: pgmap v16935: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:51.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:52:51 smithi083 ceph-mon[77242]: pgmap v16935: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:52.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:52:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:52:52.277Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:52:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:52:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:52:52.316Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:52:53.283 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:52:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:52:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:52:53.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:52:53 smithi153 ceph-mon[70044]: pgmap v16936: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:53.710 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:52:53 smithi083 ceph-mon[77242]: pgmap v16936: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:53.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:52:53 smithi083 ceph-mon[79930]: pgmap v16936: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:55.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:52:55 smithi153 ceph-mon[70044]: pgmap v16937: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:55.635 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:52:55 smithi083 ceph-mon[79930]: pgmap v16937: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:55.635 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:52:55 smithi083 ceph-mon[77242]: pgmap v16937: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:55.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:52:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:52:55.634Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:52:56.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:52:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:52:56.371Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:52:57.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:52:57 smithi153 ceph-mon[70044]: pgmap v16938: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:57.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:52:57 smithi083 ceph-mon[79930]: pgmap v16938: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:57.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:52:57 smithi083 ceph-mon[77242]: pgmap v16938: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:58.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:52:58 smithi153 ceph-mon[70044]: pgmap v16939: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:58.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:52:58 smithi083 ceph-mon[79930]: pgmap v16939: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:52:58.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:52:58 smithi083 ceph-mon[77242]: pgmap v16939: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:01.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:53:01 smithi083 ceph-mon[79930]: pgmap v16940: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:01.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:53:01 smithi083 ceph-mon[77242]: pgmap v16940: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:01.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:53:01 smithi153 ceph-mon[70044]: pgmap v16940: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:01.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:53:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:53:01] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T14:53:02.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:53:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:53:02.277Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:53:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:53:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:53:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:53:03.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:53:03 smithi153 ceph-mon[70044]: pgmap v16941: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:53:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:53:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:53:03.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:53:03 smithi083 ceph-mon[79930]: pgmap v16941: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:03.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:53:03 smithi083 ceph-mon[77242]: pgmap v16941: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:05.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:53:05 smithi083 ceph-mon[79930]: pgmap v16942: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:05.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:53:05 smithi083 ceph-mon[77242]: pgmap v16942: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:05.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:53:05 smithi153 ceph-mon[70044]: pgmap v16942: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:05.736 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:53:05.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:53:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:53:05.634Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:53:06.025 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:53:06.026 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 4m ago 9h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:53:06.026 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 2m ago 9h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:53:06.026 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 4m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:53:06.027 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 2m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:53:06.027 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 2m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:53:06.027 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 4m ago 9h 974M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:53:06.027 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 4m ago 9h 2055M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:53:06.028 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 2m ago 9h 893M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:53:06.028 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 4m ago 9h 1230M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:53:06.028 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 4m ago 9h 24.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:53:06.028 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 2m ago 9h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:53:06.029 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 4m ago 9h 1490M 2205M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:53:06.029 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 4m ago 9h 1476M 2205M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:53:06.029 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 4m ago 9h 1147M 2205M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:53:06.029 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 4m ago 9h 1447M 2205M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:53:06.030 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 2m ago 9h 1269M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:53:06.030 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 2m ago 9h 1593M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:53:06.030 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 2m ago 9h 1424M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:53:06.030 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 2m ago 9h 1693M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:53:06.031 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 2m ago 9h 201M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:53:06.382 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:53:06.383 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:53:06.383 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:53:06.383 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:53:06.383 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:53:06.384 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:53:06.384 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:53:06.384 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:53:06.384 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:53:06.385 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:53:06.385 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:53:06.385 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:53:06.385 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:53:06.385 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:53:06.386 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:53:06.386 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:53:06.386 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:53:06.386 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:53:06.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:53:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:53:06.371Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:53:07.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:53:07 smithi083 ceph-mon[79930]: from='client.63763 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:53:07.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:53:07 smithi083 ceph-mon[79930]: from='client.63769 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:53:07.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:53:07 smithi083 ceph-mon[79930]: pgmap v16943: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:07.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:53:07 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3085197481' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:53:07.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:53:07 smithi083 ceph-mon[77242]: from='client.63763 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:53:07.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:53:07 smithi083 ceph-mon[77242]: from='client.63769 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:53:07.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:53:07 smithi083 ceph-mon[77242]: pgmap v16943: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:07.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:53:07 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3085197481' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:53:07.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:53:07 smithi153 ceph-mon[70044]: from='client.63763 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:53:07.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:53:07 smithi153 ceph-mon[70044]: from='client.63769 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:53:07.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:53:07 smithi153 ceph-mon[70044]: pgmap v16943: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:07.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:53:07 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3085197481' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:53:09.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:53:09 smithi083 ceph-mon[79930]: pgmap v16944: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:09.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:53:09 smithi083 ceph-mon[77242]: pgmap v16944: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:09.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:53:09 smithi153 ceph-mon[70044]: pgmap v16944: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:11.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:53:11 smithi083 ceph-mon[79930]: pgmap v16945: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:11.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:53:11 smithi083 ceph-mon[77242]: pgmap v16945: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:11.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:53:11 smithi153 ceph-mon[70044]: pgmap v16945: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:11.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:53:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:53:11] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T14:53:12.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:53:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:53:12.278Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:53:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:53:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:53:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:53:13.312 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:53:13 smithi153 ceph-mon[70044]: pgmap v16946: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:13.313 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:53:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:53:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:53:13.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:53:13 smithi083 ceph-mon[79930]: pgmap v16946: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:13.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:53:13 smithi083 ceph-mon[77242]: pgmap v16946: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:15.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:53:15 smithi083 ceph-mon[79930]: pgmap v16947: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:15.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:53:15 smithi083 ceph-mon[77242]: pgmap v16947: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:15.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:53:15 smithi153 ceph-mon[70044]: pgmap v16947: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:15.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:53:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:53:15.637Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:53:16.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:53:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:53:16.372Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:53:17.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:53:17 smithi083 ceph-mon[79930]: pgmap v16948: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:17.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:53:17 smithi083 ceph-mon[77242]: pgmap v16948: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:17.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:53:17 smithi153 ceph-mon[70044]: pgmap v16948: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:19.460 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:53:19 smithi083 ceph-mon[77242]: pgmap v16949: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:19.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:53:19 smithi083 ceph-mon[79930]: pgmap v16949: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:19.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:53:19 smithi153 ceph-mon[70044]: pgmap v16949: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:21.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:53:21 smithi083 ceph-mon[79930]: pgmap v16950: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:21.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:53:21 smithi083 ceph-mon[77242]: pgmap v16950: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:21.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:53:21 smithi153 ceph-mon[70044]: pgmap v16950: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:21.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:53:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:53:21] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T14:53:22.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:53:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:53:22.278Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:53:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:53:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:53:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:53:23.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:53:23 smithi153 ceph-mon[70044]: pgmap v16951: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:53:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:53:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:53:23.460 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:53:23 smithi083 ceph-mon[77242]: pgmap v16951: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:23.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:53:23 smithi083 ceph-mon[79930]: pgmap v16951: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:25.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:53:25 smithi083 ceph-mon[79930]: pgmap v16952: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:25.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:53:25 smithi083 ceph-mon[77242]: pgmap v16952: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:25.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:53:25 smithi153 ceph-mon[70044]: pgmap v16952: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:25.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:53:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:53:25.638Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:53:26.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:53:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:53:26.373Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:53:27.460 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:53:27 smithi083 ceph-mon[77242]: pgmap v16953: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:27.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:53:27 smithi083 ceph-mon[79930]: pgmap v16953: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:27.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:53:27 smithi153 ceph-mon[70044]: pgmap v16953: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:29.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:53:29 smithi083 ceph-mon[79930]: pgmap v16954: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:29.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:53:29 smithi083 ceph-mon[77242]: pgmap v16954: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:29.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:53:29 smithi153 ceph-mon[70044]: pgmap v16954: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:31.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:53:31 smithi083 ceph-mon[79930]: pgmap v16955: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:31.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:53:31 smithi083 ceph-mon[77242]: pgmap v16955: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:31.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:53:31 smithi153 ceph-mon[70044]: pgmap v16955: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:31.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:53:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:53:31] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T14:53:32.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:53:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:53:32.280Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:53:32.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:53:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:53:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:53:33.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:53:33 smithi153 ceph-mon[70044]: pgmap v16956: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:53:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:53:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:53:33.710 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:53:33 smithi083 ceph-mon[77242]: pgmap v16956: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:33.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:53:33 smithi083 ceph-mon[79930]: pgmap v16956: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:35.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:53:35 smithi153 ceph-mon[70044]: pgmap v16957: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:35.639 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:53:35 smithi083 ceph-mon[79930]: pgmap v16957: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:35.639 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:53:35 smithi083 ceph-mon[77242]: pgmap v16957: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:35.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:53:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:53:35.638Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:53:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:53:36 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:53:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:53:36 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:53:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:53:36 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:53:36.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:53:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:53:36.373Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:53:36.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:53:36 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:53:36.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:53:36 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:53:36.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:53:36 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:53:36.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:53:36 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:53:36.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:53:36 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:53:36.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:53:36 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:53:36.738 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:53:37.026 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:53:37.027 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 5m ago 9h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:53:37.027 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 3m ago 9h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:53:37.027 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 5m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:53:37.027 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 3m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:53:37.028 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 3m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:53:37.028 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 5m ago 9h 974M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:53:37.028 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 5m ago 9h 2055M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:53:37.028 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 3m ago 9h 893M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:53:37.029 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 5m ago 9h 1230M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:53:37.029 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 5m ago 9h 24.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:53:37.029 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 3m ago 9h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:53:37.029 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 5m ago 9h 1490M 2205M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:53:37.029 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 5m ago 9h 1476M 2205M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:53:37.030 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 5m ago 9h 1147M 2205M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:53:37.030 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 5m ago 9h 1447M 2205M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:53:37.030 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 3m ago 9h 1269M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:53:37.030 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 3m ago 9h 1593M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:53:37.031 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 3m ago 9h 1424M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:53:37.031 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 3m ago 9h 1693M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:53:37.031 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 3m ago 9h 201M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:53:37.386 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:53:37.387 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:53:37.387 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:53:37.387 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:53:37.388 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:53:37.388 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:53:37.388 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:53:37.388 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:53:37.388 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:53:37.389 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:53:37.389 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:53:37.389 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:53:37.389 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:53:37.390 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:53:37.390 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:53:37.390 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:53:37.390 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:53:37.391 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:53:37.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:53:37 smithi153 ceph-mon[70044]: pgmap v16958: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:37.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:53:37 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:53:37.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:53:37 smithi083 ceph-mon[79930]: pgmap v16958: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:37.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:53:37 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:53:37.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:53:37 smithi083 ceph-mon[77242]: pgmap v16958: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:37.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:53:37 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:53:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:53:38 smithi153 ceph-mon[70044]: from='client.63781 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:53:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:53:38 smithi153 ceph-mon[70044]: from='client.63787 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:53:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:53:38 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3581971427' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:53:38.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:53:38 smithi083 ceph-mon[79930]: from='client.63781 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:53:38.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:53:38 smithi083 ceph-mon[79930]: from='client.63787 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:53:38.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:53:38 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3581971427' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:53:38.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:53:38 smithi083 ceph-mon[77242]: from='client.63781 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:53:38.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:53:38 smithi083 ceph-mon[77242]: from='client.63787 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:53:38.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:53:38 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3581971427' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:53:39.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:53:39 smithi153 ceph-mon[70044]: pgmap v16959: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:39.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:53:39 smithi083 ceph-mon[79930]: pgmap v16959: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:39.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:53:39 smithi083 ceph-mon[77242]: pgmap v16959: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:41.506 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:53:41 smithi083 ceph-mon[79930]: pgmap v16960: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:41.507 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:53:41 smithi083 ceph-mon[77242]: pgmap v16960: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:41.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:53:41 smithi153 ceph-mon[70044]: pgmap v16960: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:41.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:53:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:53:41] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T14:53:42.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:53:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:53:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:53:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:53:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:53:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:53:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:53:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:53:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:53:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:53:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:53:42.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:53:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:53:42.280Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:53:42.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:53:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:53:42.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:53:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:53:42.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:53:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:53:42.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:53:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:53:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:53:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:53:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:53:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:53:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:53:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:53:42.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:53:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:53:43.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:53:43 smithi153 ceph-mon[70044]: pgmap v16961: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:53:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:53:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:53:43.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:53:43 smithi083 ceph-mon[79930]: pgmap v16961: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:43.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:53:43 smithi083 ceph-mon[77242]: pgmap v16961: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:45.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:53:45 smithi153 ceph-mon[70044]: pgmap v16962: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:45.640 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:53:45 smithi083 ceph-mon[79930]: pgmap v16962: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:45.640 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:53:45 smithi083 ceph-mon[77242]: pgmap v16962: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:45.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:53:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:53:45.639Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:53:46.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:53:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:53:46.374Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:53:47.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:53:47 smithi153 ceph-mon[70044]: pgmap v16963: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:47.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:53:47 smithi083 ceph-mon[79930]: pgmap v16963: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:47.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:53:47 smithi083 ceph-mon[77242]: pgmap v16963: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:49.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:53:49 smithi153 ceph-mon[70044]: pgmap v16964: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:49.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:53:49 smithi083 ceph-mon[79930]: pgmap v16964: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:49.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:53:49 smithi083 ceph-mon[77242]: pgmap v16964: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:51.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:53:51 smithi153 ceph-mon[70044]: pgmap v16965: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:51.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:53:51 smithi083 ceph-mon[79930]: pgmap v16965: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:51.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:53:51 smithi083 ceph-mon[77242]: pgmap v16965: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:51.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:53:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:53:51] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T14:53:52.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:53:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:53:52.281Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:53:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:53:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:53:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:53:53.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:53:53 smithi153 ceph-mon[70044]: pgmap v16966: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:53.373 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:53:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:53:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:53:53.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:53:53 smithi083 ceph-mon[79930]: pgmap v16966: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:53.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:53:53 smithi083 ceph-mon[77242]: pgmap v16966: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:55.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:53:55 smithi153 ceph-mon[70044]: pgmap v16967: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:55.640 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:53:55 smithi083 ceph-mon[79930]: pgmap v16967: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:55.640 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:53:55 smithi083 ceph-mon[77242]: pgmap v16967: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:55.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:53:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:53:55.640Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:53:56.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:53:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:53:56.375Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:53:57.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:53:57 smithi153 ceph-mon[70044]: pgmap v16968: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:57.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:53:57 smithi083 ceph-mon[79930]: pgmap v16968: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:57.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:53:57 smithi083 ceph-mon[77242]: pgmap v16968: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:59.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:53:59 smithi153 ceph-mon[70044]: pgmap v16969: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:59.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:53:59 smithi083 ceph-mon[79930]: pgmap v16969: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:53:59.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:53:59 smithi083 ceph-mon[77242]: pgmap v16969: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:01.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:54:01 smithi153 ceph-mon[70044]: pgmap v16970: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:01.710 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:54:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:54:01] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T14:54:01.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:54:01 smithi083 ceph-mon[79930]: pgmap v16970: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:01.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:54:01 smithi083 ceph-mon[77242]: pgmap v16970: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:02.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:54:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:54:02.281Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:54:02.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:54:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:54:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:54:03.277 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:54:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:54:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:54:03.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:54:03 smithi153 ceph-mon[70044]: pgmap v16971: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:03.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:54:03 smithi083 ceph-mon[79930]: pgmap v16971: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:03.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:54:03 smithi083 ceph-mon[77242]: pgmap v16971: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:05.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:54:05 smithi153 ceph-mon[70044]: pgmap v16972: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:05.641 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:54:05 smithi083 ceph-mon[79930]: pgmap v16972: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:05.642 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:54:05 smithi083 ceph-mon[77242]: pgmap v16972: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:05.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:54:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:54:05.641Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:54:06.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:54:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:54:06.376Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:54:07.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:54:07 smithi153 ceph-mon[70044]: pgmap v16973: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:07.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:54:07 smithi083 ceph-mon[79930]: pgmap v16973: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:07.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:54:07 smithi083 ceph-mon[77242]: pgmap v16973: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:07.713 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:54:08.003 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:54:08.004 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 5m ago 9h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:54:08.004 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 3m ago 9h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:54:08.004 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 5m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:54:08.005 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 3m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:54:08.005 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 3m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:54:08.005 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 5m ago 9h 974M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:54:08.005 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 5m ago 9h 2055M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:54:08.006 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 3m ago 9h 893M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:54:08.006 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 5m ago 9h 1230M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:54:08.006 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 5m ago 9h 24.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:54:08.006 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 3m ago 9h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:54:08.007 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 5m ago 9h 1490M 2205M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:54:08.007 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 5m ago 9h 1476M 2205M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:54:08.007 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 5m ago 9h 1147M 2205M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:54:08.007 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 5m ago 9h 1447M 2205M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:54:08.007 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 3m ago 9h 1269M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:54:08.008 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 3m ago 9h 1593M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:54:08.008 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 3m ago 9h 1424M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:54:08.008 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 3m ago 9h 1693M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:54:08.008 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 3m ago 9h 201M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:54:08.359 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:54:08.360 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:54:08.360 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:54:08.360 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:54:08.360 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:54:08.361 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:54:08.361 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:54:08.361 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:54:08.361 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:54:08.362 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:54:08.362 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:54:08.362 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:54:08.362 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:54:08.363 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:54:08.363 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:54:08.363 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:54:08.363 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:54:08.364 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:54:09.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:54:09 smithi153 ceph-mon[70044]: from='client.53868 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:54:09.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:54:09 smithi153 ceph-mon[70044]: from='client.63805 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:54:09.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:54:09 smithi153 ceph-mon[70044]: pgmap v16974: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:09.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:54:09 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3903105173' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:54:09.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:54:09 smithi083 ceph-mon[79930]: from='client.53868 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:54:09.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:54:09 smithi083 ceph-mon[79930]: from='client.63805 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:54:09.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:54:09 smithi083 ceph-mon[79930]: pgmap v16974: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:09.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:54:09 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3903105173' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:54:09.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:54:09 smithi083 ceph-mon[77242]: from='client.53868 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:54:09.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:54:09 smithi083 ceph-mon[77242]: from='client.63805 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:54:09.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:54:09 smithi083 ceph-mon[77242]: pgmap v16974: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:09.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:54:09 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3903105173' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:54:11.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:54:11 smithi153 ceph-mon[70044]: pgmap v16975: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:11.710 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:54:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:54:11] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T14:54:11.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:54:11 smithi083 ceph-mon[79930]: pgmap v16975: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:11.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:54:11 smithi083 ceph-mon[77242]: pgmap v16975: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:12.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:54:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:54:12.281Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:54:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:54:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:54:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:54:13.291 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:54:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:54:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:54:13.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:54:13 smithi153 ceph-mon[70044]: pgmap v16976: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:13.710 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:54:13 smithi083 ceph-mon[77242]: pgmap v16976: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:13.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:54:13 smithi083 ceph-mon[79930]: pgmap v16976: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:15.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:54:15 smithi153 ceph-mon[70044]: pgmap v16977: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:15.641 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:54:15 smithi083 ceph-mon[79930]: pgmap v16977: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:15.641 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:54:15 smithi083 ceph-mon[77242]: pgmap v16977: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:15.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:54:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:54:15.641Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:54:16.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:54:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:54:16.377Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:54:17.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:54:17 smithi153 ceph-mon[70044]: pgmap v16978: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:17.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:54:17 smithi083 ceph-mon[79930]: pgmap v16978: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:17.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:54:17 smithi083 ceph-mon[77242]: pgmap v16978: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:19.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:54:19 smithi153 ceph-mon[70044]: pgmap v16979: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:19.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:54:19 smithi083 ceph-mon[79930]: pgmap v16979: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:19.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:54:19 smithi083 ceph-mon[77242]: pgmap v16979: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:20.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:54:20 smithi083 ceph-mon[79930]: pgmap v16980: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:20.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:54:20 smithi083 ceph-mon[77242]: pgmap v16980: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:20.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:54:20 smithi153 ceph-mon[70044]: pgmap v16980: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:21.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:54:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:54:21] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T14:54:22.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:54:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:54:22.282Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:54:22.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:54:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:54:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:54:23.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:54:23 smithi153 ceph-mon[70044]: pgmap v16981: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:54:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:54:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:54:23.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:54:23 smithi083 ceph-mon[79930]: pgmap v16981: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:23.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:54:23 smithi083 ceph-mon[77242]: pgmap v16981: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:25.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:54:25 smithi083 ceph-mon[79930]: pgmap v16982: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:25.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:54:25 smithi083 ceph-mon[77242]: pgmap v16982: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:25.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:54:25 smithi153 ceph-mon[70044]: pgmap v16982: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:25.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:54:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:54:25.641Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:54:26.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:54:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:54:26.378Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:54:27.460 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:54:27 smithi083 ceph-mon[77242]: pgmap v16983: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:27.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:54:27 smithi083 ceph-mon[79930]: pgmap v16983: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:27.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:54:27 smithi153 ceph-mon[70044]: pgmap v16983: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:29.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:54:29 smithi083 ceph-mon[77242]: pgmap v16984: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:29.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:54:29 smithi083 ceph-mon[79930]: pgmap v16984: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:29.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:54:29 smithi153 ceph-mon[70044]: pgmap v16984: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:31.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:54:31 smithi083 ceph-mon[77242]: pgmap v16985: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:31.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:54:31 smithi083 ceph-mon[79930]: pgmap v16985: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:31.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:54:31 smithi153 ceph-mon[70044]: pgmap v16985: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:31.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:54:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:54:31] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T14:54:32.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:54:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:54:32.282Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:54:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:54:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:54:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:54:33.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:54:33 smithi153 ceph-mon[70044]: pgmap v16986: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:54:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:54:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:54:33.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:54:33 smithi083 ceph-mon[79930]: pgmap v16986: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:33.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:54:33 smithi083 ceph-mon[77242]: pgmap v16986: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:35.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:54:35 smithi083 ceph-mon[79930]: pgmap v16987: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:35.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:54:35 smithi083 ceph-mon[77242]: pgmap v16987: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:35.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:54:35 smithi153 ceph-mon[70044]: pgmap v16987: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:35.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:54:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:54:35.642Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:54:36.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:54:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:54:36.379Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:54:37.460 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:54:37 smithi083 ceph-mon[77242]: pgmap v16988: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:37.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:54:37 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:54:37.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:54:37 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:54:37.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:54:37 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:54:37.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:54:37 smithi083 ceph-mon[79930]: pgmap v16988: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:37.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:54:37 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:54:37.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:54:37 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:54:37.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:54:37 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:54:37.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:54:37 smithi153 ceph-mon[70044]: pgmap v16988: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:37.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:54:37 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:54:37.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:54:37 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:54:37.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:54:37 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:54:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:54:38 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:54:38.688 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:54:38.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:54:38 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:54:38.710 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:54:38 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:54:38.977 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:54:38.977 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 6m ago 9h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:54:38.978 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 4m ago 9h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:54:38.978 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 6m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:54:38.978 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 4m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:54:38.978 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 4m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:54:38.979 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 6m ago 9h 974M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:54:38.979 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 6m ago 9h 2055M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:54:38.979 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 4m ago 9h 893M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:54:38.979 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 6m ago 9h 1230M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:54:38.980 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 6m ago 9h 24.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:54:38.980 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 4m ago 9h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:54:38.980 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 6m ago 9h 1490M 2205M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:54:38.980 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 6m ago 9h 1476M 2205M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:54:38.981 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 6m ago 9h 1147M 2205M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:54:38.981 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 6m ago 9h 1447M 2205M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:54:38.981 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 4m ago 9h 1269M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:54:38.981 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 4m ago 9h 1593M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:54:38.982 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 4m ago 9h 1424M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:54:38.982 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 4m ago 9h 1693M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:54:38.982 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 4m ago 9h 201M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:54:39.332 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:54:39.333 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:54:39.333 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:54:39.333 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:54:39.334 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:54:39.334 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:54:39.334 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:54:39.334 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:54:39.335 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:54:39.335 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:54:39.335 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:54:39.335 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:54:39.336 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:54:39.336 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:54:39.336 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:54:39.336 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:54:39.337 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:54:39.337 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:54:39.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:54:39 smithi153 ceph-mon[70044]: pgmap v16989: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:39.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:54:39 smithi083 ceph-mon[79930]: pgmap v16989: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:39.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:54:39 smithi083 ceph-mon[77242]: pgmap v16989: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:54:40 smithi153 ceph-mon[70044]: from='client.53886 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:54:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:54:40 smithi153 ceph-mon[70044]: from='client.63823 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:54:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:54:40 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1218644205' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:54:40.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:54:40 smithi083 ceph-mon[79930]: from='client.53886 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:54:40.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:54:40 smithi083 ceph-mon[79930]: from='client.63823 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:54:40.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:54:40 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1218644205' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:54:40.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:54:40 smithi083 ceph-mon[77242]: from='client.53886 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:54:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:54:40 smithi083 ceph-mon[77242]: from='client.63823 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:54:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:54:40 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1218644205' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:54:41.506 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:54:41 smithi083 ceph-mon[79930]: pgmap v16990: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:41.507 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:54:41 smithi083 ceph-mon[77242]: pgmap v16990: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:41.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:54:41 smithi153 ceph-mon[70044]: pgmap v16990: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:41.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:54:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:54:41] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T14:54:42.556 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:54:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:54:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:54:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:54:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:54:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:54:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:54:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:54:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:54:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:54:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:54:42.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:54:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:54:42.283Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:54:42.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:54:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:54:42.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:54:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:54:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:54:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:54:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:54:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:54:42.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:54:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:54:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:54:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:54:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:54:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:54:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:54:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:54:43.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:54:43 smithi153 ceph-mon[70044]: pgmap v16991: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:54:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:54:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:54:43.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:54:43 smithi083 ceph-mon[79930]: pgmap v16991: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:43.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:54:43 smithi083 ceph-mon[77242]: pgmap v16991: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:45.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:54:45 smithi153 ceph-mon[70044]: pgmap v16992: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:45.643 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:54:45 smithi083 ceph-mon[79930]: pgmap v16992: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:45.643 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:54:45 smithi083 ceph-mon[77242]: pgmap v16992: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:45.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:54:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:54:45.642Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:54:46.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:54:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:54:46.380Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:54:47.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:54:47 smithi153 ceph-mon[70044]: pgmap v16993: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:47.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:54:47 smithi083 ceph-mon[79930]: pgmap v16993: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:47.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:54:47 smithi083 ceph-mon[77242]: pgmap v16993: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:49.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:54:49 smithi153 ceph-mon[70044]: pgmap v16994: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:49.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:54:49 smithi083 ceph-mon[79930]: pgmap v16994: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:49.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:54:49 smithi083 ceph-mon[77242]: pgmap v16994: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:51.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:54:51 smithi153 ceph-mon[70044]: pgmap v16995: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:51.710 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:54:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:54:51] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T14:54:51.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:54:51 smithi083 ceph-mon[77242]: pgmap v16995: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:51.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:54:51 smithi083 ceph-mon[79930]: pgmap v16995: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:52.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:54:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:54:52.284Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:54:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:54:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:54:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:54:53.278 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:54:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:54:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:54:53.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:54:53 smithi153 ceph-mon[70044]: pgmap v16996: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:53.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:54:53 smithi083 ceph-mon[79930]: pgmap v16996: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:53.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:54:53 smithi083 ceph-mon[77242]: pgmap v16996: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:55.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:54:55 smithi153 ceph-mon[70044]: pgmap v16997: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:55.644 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:54:55 smithi083 ceph-mon[79930]: pgmap v16997: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:55.644 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:54:55 smithi083 ceph-mon[77242]: pgmap v16997: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:55.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:54:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:54:55.643Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:54:56.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:54:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:54:56.380Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:54:57.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:54:57 smithi153 ceph-mon[70044]: pgmap v16998: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:57.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:54:57 smithi083 ceph-mon[77242]: pgmap v16998: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:57.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:54:57 smithi083 ceph-mon[79930]: pgmap v16998: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:59.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:54:59 smithi153 ceph-mon[70044]: pgmap v16999: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:59.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:54:59 smithi083 ceph-mon[79930]: pgmap v16999: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:54:59.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:54:59 smithi083 ceph-mon[77242]: pgmap v16999: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:01.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:55:01 smithi153 ceph-mon[70044]: pgmap v17000: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:01.710 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:55:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:55:01] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T14:55:01.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:55:01 smithi083 ceph-mon[79930]: pgmap v17000: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:01.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:55:01 smithi083 ceph-mon[77242]: pgmap v17000: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:02.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:55:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:55:02.285Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:55:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:55:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:55:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:55:03.293 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:55:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:55:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:55:03.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:55:03 smithi153 ceph-mon[70044]: pgmap v17001: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:03.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:55:03 smithi083 ceph-mon[79930]: pgmap v17001: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:03.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:55:03 smithi083 ceph-mon[77242]: pgmap v17001: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:05.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:55:05 smithi153 ceph-mon[70044]: pgmap v17002: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:05.645 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:55:05 smithi083 ceph-mon[79930]: pgmap v17002: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:05.646 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:55:05 smithi083 ceph-mon[77242]: pgmap v17002: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:05.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:55:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:55:05.645Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:55:06.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:55:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:55:06.381Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:55:07.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:55:07 smithi153 ceph-mon[70044]: pgmap v17003: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:07.710 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:55:07 smithi083 ceph-mon[77242]: pgmap v17003: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:07.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:55:07 smithi083 ceph-mon[79930]: pgmap v17003: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:09.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:55:09 smithi153 ceph-mon[70044]: pgmap v17004: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:09.681 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:55:09.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:55:09 smithi083 ceph-mon[79930]: pgmap v17004: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:09.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:55:09 smithi083 ceph-mon[77242]: pgmap v17004: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:09.969 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:55:09.970 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 6m ago 9h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:55:09.970 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 4m ago 9h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:55:09.970 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 6m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:55:09.970 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 4m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:55:09.971 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 4m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:55:09.971 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 6m ago 9h 974M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:55:09.971 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 6m ago 9h 2055M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:55:09.971 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 4m ago 9h 893M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:55:09.972 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 6m ago 9h 1230M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:55:09.972 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 6m ago 9h 24.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:55:09.972 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 4m ago 9h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:55:09.972 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 6m ago 9h 1490M 2205M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:55:09.972 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 6m ago 9h 1476M 2205M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:55:09.973 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 6m ago 9h 1147M 2205M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:55:09.973 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 6m ago 9h 1447M 2205M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:55:09.973 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 4m ago 9h 1269M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:55:09.973 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 4m ago 9h 1593M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:55:09.974 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 4m ago 9h 1424M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:55:09.974 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 4m ago 9h 1693M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:55:09.974 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 4m ago 9h 201M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:55:10.327 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:55:10.328 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:55:10.328 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:55:10.328 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:55:10.328 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:55:10.329 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:55:10.329 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:55:10.329 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:55:10.329 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:55:10.330 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:55:10.330 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:55:10.330 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:55:10.330 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:55:10.331 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:55:10.331 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:55:10.331 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:55:10.331 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:55:10.331 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:55:11.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:55:11 smithi153 ceph-mon[70044]: from='client.63835 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:55:11.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:55:11 smithi153 ceph-mon[70044]: from='client.63841 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:55:11.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:55:11 smithi153 ceph-mon[70044]: pgmap v17005: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:11.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:55:11 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1688453337' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:55:11.710 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:55:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:55:11] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T14:55:11.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:55:11 smithi083 ceph-mon[79930]: from='client.63835 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:55:11.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:55:11 smithi083 ceph-mon[79930]: from='client.63841 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:55:11.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:55:11 smithi083 ceph-mon[79930]: pgmap v17005: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:11.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:55:11 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1688453337' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:55:11.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:55:11 smithi083 ceph-mon[77242]: from='client.63835 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:55:11.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:55:11 smithi083 ceph-mon[77242]: from='client.63841 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:55:11.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:55:11 smithi083 ceph-mon[77242]: pgmap v17005: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:11.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:55:11 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1688453337' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:55:12.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:55:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:55:12.285Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:55:12.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:55:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:55:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:55:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:55:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:55:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:55:13.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:55:13 smithi083 ceph-mon[79930]: pgmap v17006: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:13.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:55:13 smithi083 ceph-mon[77242]: pgmap v17006: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:13.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:55:13 smithi153 ceph-mon[70044]: pgmap v17006: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:15.646 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:55:15 smithi083 ceph-mon[79930]: pgmap v17007: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:15.646 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:55:15 smithi083 ceph-mon[77242]: pgmap v17007: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:15.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:55:15 smithi153 ceph-mon[70044]: pgmap v17007: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:15.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:55:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:55:15.645Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:55:16.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:55:16 smithi083 ceph-mon[79930]: pgmap v17008: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:16.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:55:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:55:16.381Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:55:16.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:55:16 smithi083 ceph-mon[77242]: pgmap v17008: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:16.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:55:16 smithi153 ceph-mon[70044]: pgmap v17008: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:19.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:55:19 smithi083 ceph-mon[79930]: pgmap v17009: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:19.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:55:19 smithi083 ceph-mon[77242]: pgmap v17009: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:19.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:55:19 smithi153 ceph-mon[70044]: pgmap v17009: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:21.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:55:21 smithi083 ceph-mon[79930]: pgmap v17010: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:21.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:55:21 smithi083 ceph-mon[77242]: pgmap v17010: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:21.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:55:21 smithi153 ceph-mon[70044]: pgmap v17010: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:21.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:55:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:55:21] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T14:55:22.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:55:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:55:22.286Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:55:22.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:55:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:55:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:55:23.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:55:23 smithi153 ceph-mon[70044]: pgmap v17011: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:55:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:55:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:55:23.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:55:23 smithi083 ceph-mon[79930]: pgmap v17011: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:23.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:55:23 smithi083 ceph-mon[77242]: pgmap v17011: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:25.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:55:25 smithi083 ceph-mon[79930]: pgmap v17012: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:25.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:55:25 smithi083 ceph-mon[77242]: pgmap v17012: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:25.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:55:25 smithi153 ceph-mon[70044]: pgmap v17012: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:25.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:55:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:55:25.645Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:55:26.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:55:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:55:26.381Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:55:27.460 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:55:27 smithi083 ceph-mon[77242]: pgmap v17013: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:27.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:55:27 smithi083 ceph-mon[79930]: pgmap v17013: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:27.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:55:27 smithi153 ceph-mon[70044]: pgmap v17013: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:29.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:55:29 smithi083 ceph-mon[79930]: pgmap v17014: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:29.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:55:29 smithi083 ceph-mon[77242]: pgmap v17014: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:29.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:55:29 smithi153 ceph-mon[70044]: pgmap v17014: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:31.506 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:55:31 smithi083 ceph-mon[79930]: pgmap v17015: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:31.506 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:55:31 smithi083 ceph-mon[77242]: pgmap v17015: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:31.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:55:31 smithi153 ceph-mon[70044]: pgmap v17015: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:31.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:55:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:55:31] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T14:55:32.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:55:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:55:32.287Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:55:32.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:55:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:55:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:55:33.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:55:33 smithi153 ceph-mon[70044]: pgmap v17016: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:55:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:55:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:55:33.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:55:33 smithi083 ceph-mon[79930]: pgmap v17016: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:33.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:55:33 smithi083 ceph-mon[77242]: pgmap v17016: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:35.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:55:35 smithi153 ceph-mon[70044]: pgmap v17017: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:35.646 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:55:35 smithi083 ceph-mon[79930]: pgmap v17017: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:35.646 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:55:35 smithi083 ceph-mon[77242]: pgmap v17017: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:35.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:55:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:55:35.645Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:55:36.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:55:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:55:36.382Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:55:37.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:55:37 smithi153 ceph-mon[70044]: pgmap v17018: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:37.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:55:37 smithi083 ceph-mon[79930]: pgmap v17018: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:37.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:55:37 smithi083 ceph-mon[77242]: pgmap v17018: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:38.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:55:38 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:55:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:55:38 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:55:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:55:38 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:55:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:55:38 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:55:38.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:55:38 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:55:38.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:55:38 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:55:38.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:55:38 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:55:38.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:55:38 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:55:38.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:55:38 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:55:38.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:55:38 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:55:38.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:55:38 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:55:38.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:55:38 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:55:39.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:55:39 smithi153 ceph-mon[70044]: pgmap v17019: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:39.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:55:39 smithi083 ceph-mon[79930]: pgmap v17019: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:39.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:55:39 smithi083 ceph-mon[77242]: pgmap v17019: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:40.648 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:55:40.938 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:55:40.938 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 7m ago 9h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:55:40.938 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 5m ago 9h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:55:40.939 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 7m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:55:40.939 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 5m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:55:40.939 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 5m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:55:40.939 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 7m ago 9h 974M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:55:40.939 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 7m ago 9h 2055M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:55:40.940 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 5m ago 9h 893M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:55:40.940 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 7m ago 9h 1230M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:55:40.940 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 7m ago 9h 24.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:55:40.940 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 5m ago 9h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:55:40.941 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 7m ago 9h 1490M 2205M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:55:40.941 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 7m ago 9h 1476M 2205M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:55:40.941 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 7m ago 9h 1147M 2205M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:55:40.941 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 7m ago 9h 1447M 2205M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:55:40.942 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 5m ago 9h 1269M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:55:40.942 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 5m ago 9h 1593M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:55:40.942 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 5m ago 9h 1424M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:55:40.942 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 5m ago 9h 1693M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:55:40.942 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 5m ago 9h 201M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:55:41.294 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:55:41.294 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:55:41.294 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:55:41.295 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:55:41.295 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:55:41.295 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:55:41.295 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:55:41.296 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:55:41.296 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:55:41.296 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:55:41.296 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:55:41.297 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:55:41.297 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:55:41.297 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:55:41.297 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:55:41.297 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:55:41.298 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:55:41.298 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:55:41.506 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:55:41 smithi083 ceph-mon[79930]: pgmap v17020: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:41.507 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:55:41 smithi083 ceph-mon[77242]: pgmap v17020: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:41.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:55:41 smithi153 ceph-mon[70044]: pgmap v17020: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:41.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:55:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:55:41] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T14:55:42.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:55:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:55:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:55:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:55:42 smithi153 ceph-mon[70044]: from='client.53922 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:55:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:55:42 smithi153 ceph-mon[70044]: from='client.53928 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:55:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:55:42 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/317067599' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:55:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:55:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:55:42.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:55:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:55:42.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:55:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:55:42.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:55:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:55:42.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:55:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:55:42.287Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:55:42.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:55:42 smithi083 ceph-mon[79930]: from='client.53922 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:55:42.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:55:42 smithi083 ceph-mon[79930]: from='client.53928 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:55:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:55:42 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/317067599' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:55:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:55:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:55:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:55:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:55:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:55:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:55:42.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:55:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:55:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:55:42 smithi083 ceph-mon[77242]: from='client.53922 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:55:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:55:42 smithi083 ceph-mon[77242]: from='client.53928 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:55:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:55:42 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/317067599' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:55:42.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:55:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:55:42.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:55:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:55:42.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:55:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:55:42.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:55:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:55:43.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:55:43 smithi153 ceph-mon[70044]: pgmap v17021: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:55:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:55:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:55:43.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:55:43 smithi083 ceph-mon[77242]: pgmap v17021: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:43.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:55:43 smithi083 ceph-mon[79930]: pgmap v17021: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:45.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:55:45 smithi153 ceph-mon[70044]: pgmap v17022: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:45.647 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:55:45 smithi083 ceph-mon[79930]: pgmap v17022: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:45.648 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:55:45 smithi083 ceph-mon[77242]: pgmap v17022: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:45.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:55:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:55:45.646Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:55:46.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:55:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:55:46.383Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:55:47.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:55:47 smithi153 ceph-mon[70044]: pgmap v17023: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:47.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:55:47 smithi083 ceph-mon[79930]: pgmap v17023: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:47.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:55:47 smithi083 ceph-mon[77242]: pgmap v17023: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:49.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:55:49 smithi153 ceph-mon[70044]: pgmap v17024: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:49.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:55:49 smithi083 ceph-mon[79930]: pgmap v17024: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:49.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:55:49 smithi083 ceph-mon[77242]: pgmap v17024: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:51.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:55:51 smithi153 ceph-mon[70044]: pgmap v17025: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:51.710 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:55:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:55:51] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T14:55:51.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:55:51 smithi083 ceph-mon[79930]: pgmap v17025: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:51.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:55:51 smithi083 ceph-mon[77242]: pgmap v17025: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:52.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:55:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:55:52.288Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:55:52.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:55:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:55:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:55:53.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:55:53 smithi153 ceph-mon[70044]: pgmap v17026: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:55:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:55:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:55:53.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:55:53 smithi083 ceph-mon[79930]: pgmap v17026: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:53.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:55:53 smithi083 ceph-mon[77242]: pgmap v17026: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:55.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:55:55 smithi153 ceph-mon[70044]: pgmap v17027: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:55.647 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:55:55 smithi083 ceph-mon[79930]: pgmap v17027: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:55.648 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:55:55 smithi083 ceph-mon[77242]: pgmap v17027: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:55.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:55:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:55:55.647Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:55:56.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:55:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:55:56.384Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:55:57.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:55:57 smithi153 ceph-mon[70044]: pgmap v17028: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:57.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:55:57 smithi083 ceph-mon[79930]: pgmap v17028: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:57.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:55:57 smithi083 ceph-mon[77242]: pgmap v17028: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:59.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:55:59 smithi153 ceph-mon[70044]: pgmap v17029: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:59.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:55:59 smithi083 ceph-mon[79930]: pgmap v17029: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:55:59.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:55:59 smithi083 ceph-mon[77242]: pgmap v17029: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:01.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:56:01 smithi153 ceph-mon[70044]: pgmap v17030: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:01.710 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:56:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:56:01] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T14:56:01.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:56:01 smithi083 ceph-mon[79930]: pgmap v17030: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:01.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:56:01 smithi083 ceph-mon[77242]: pgmap v17030: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:02.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:56:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:56:02.288Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:56:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:56:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:56:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:56:03.283 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:56:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:56:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:56:03.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:56:03 smithi153 ceph-mon[70044]: pgmap v17031: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:03.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:56:03 smithi083 ceph-mon[79930]: pgmap v17031: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:03.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:56:03 smithi083 ceph-mon[77242]: pgmap v17031: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:05.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:56:05 smithi153 ceph-mon[70044]: pgmap v17032: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:05.648 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:56:05 smithi083 ceph-mon[79930]: pgmap v17032: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:05.648 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:56:05 smithi083 ceph-mon[77242]: pgmap v17032: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:05.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:56:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:56:05.647Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:56:06.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:56:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:56:06.385Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:56:07.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:56:07 smithi153 ceph-mon[70044]: pgmap v17033: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:07.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:56:07 smithi083 ceph-mon[79930]: pgmap v17033: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:07.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:56:07 smithi083 ceph-mon[77242]: pgmap v17033: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:09.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:56:09 smithi153 ceph-mon[70044]: pgmap v17034: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:09.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:56:09 smithi083 ceph-mon[79930]: pgmap v17034: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:09.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:56:09 smithi083 ceph-mon[77242]: pgmap v17034: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:11.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:56:11 smithi153 ceph-mon[70044]: pgmap v17035: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:11.619 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:56:11.710 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:56:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:56:11] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T14:56:11.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:56:11 smithi083 ceph-mon[79930]: pgmap v17035: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:11.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:56:11 smithi083 ceph-mon[77242]: pgmap v17035: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:11.908 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:56:11.909 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 7m ago 9h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:56:11.909 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 5m ago 9h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:56:11.910 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 7m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:56:11.910 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 5m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:56:11.911 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 5m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:56:11.911 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 7m ago 9h 974M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:56:11.911 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 7m ago 9h 2055M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:56:11.912 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 5m ago 9h 893M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:56:11.912 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 7m ago 9h 1230M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:56:11.913 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 7m ago 9h 24.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:56:11.913 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 5m ago 9h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:56:11.913 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 7m ago 9h 1490M 2205M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:56:11.914 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 7m ago 9h 1476M 2205M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:56:11.914 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 7m ago 9h 1147M 2205M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:56:11.914 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 7m ago 9h 1447M 2205M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:56:11.914 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 5m ago 9h 1269M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:56:11.915 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 5m ago 9h 1593M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:56:11.915 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 5m ago 9h 1424M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:56:11.915 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 5m ago 9h 1693M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:56:11.915 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 5m ago 9h 201M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:56:12.265 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:56:12.265 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:56:12.265 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:56:12.266 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:56:12.266 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:56:12.266 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:56:12.267 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:56:12.267 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:56:12.267 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:56:12.267 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:56:12.267 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:56:12.268 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:56:12.268 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:56:12.268 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:56:12.268 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:56:12.269 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:56:12.269 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:56:12.269 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:56:12.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:56:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:56:12.287Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:56:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:56:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:56:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:56:13.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:56:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:56:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:56:13.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:56:13 smithi153 ceph-mon[70044]: from='client.53940 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:56:13.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:56:13 smithi153 ceph-mon[70044]: from='client.63877 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:56:13.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:56:13 smithi153 ceph-mon[70044]: pgmap v17036: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:13.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:56:13 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4009838346' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:56:13.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:56:13 smithi083 ceph-mon[77242]: from='client.53940 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:56:13.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:56:13 smithi083 ceph-mon[77242]: from='client.63877 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:56:13.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:56:13 smithi083 ceph-mon[77242]: pgmap v17036: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:13.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:56:13 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4009838346' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:56:13.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:56:13 smithi083 ceph-mon[79930]: from='client.53940 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:56:13.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:56:13 smithi083 ceph-mon[79930]: from='client.63877 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:56:13.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:56:13 smithi083 ceph-mon[79930]: pgmap v17036: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:13.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:56:13 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4009838346' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:56:15.460 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:56:15 smithi083 ceph-mon[77242]: pgmap v17037: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:15.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:56:15 smithi083 ceph-mon[79930]: pgmap v17037: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:15.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:56:15 smithi153 ceph-mon[70044]: pgmap v17037: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:15.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:56:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:56:15.648Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:56:16.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:56:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:56:16.385Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:56:17.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:56:17 smithi153 ceph-mon[70044]: pgmap v17038: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:17.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:56:17 smithi083 ceph-mon[79930]: pgmap v17038: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:17.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:56:17 smithi083 ceph-mon[77242]: pgmap v17038: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:19.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:56:19 smithi153 ceph-mon[70044]: pgmap v17039: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:19.710 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:56:19 smithi083 ceph-mon[77242]: pgmap v17039: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:19.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:56:19 smithi083 ceph-mon[79930]: pgmap v17039: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:21.506 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:56:21 smithi083 ceph-mon[79930]: pgmap v17040: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:21.507 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:56:21 smithi083 ceph-mon[77242]: pgmap v17040: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:21.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:56:21 smithi153 ceph-mon[70044]: pgmap v17040: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:21.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:56:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:56:21] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T14:56:22.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:56:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:56:22.289Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:56:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:56:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:56:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:56:23.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:56:23 smithi153 ceph-mon[70044]: pgmap v17041: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:56:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:56:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:56:23.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:56:23 smithi083 ceph-mon[77242]: pgmap v17041: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:23.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:56:23 smithi083 ceph-mon[79930]: pgmap v17041: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:25.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:56:25 smithi153 ceph-mon[70044]: pgmap v17042: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:25.649 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:56:25 smithi083 ceph-mon[79930]: pgmap v17042: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:25.650 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:56:25 smithi083 ceph-mon[77242]: pgmap v17042: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:25.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:56:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:56:25.649Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:56:26.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:56:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:56:26.386Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:56:27.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:56:27 smithi153 ceph-mon[70044]: pgmap v17043: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:27.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:56:27 smithi083 ceph-mon[79930]: pgmap v17043: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:27.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:56:27 smithi083 ceph-mon[77242]: pgmap v17043: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:29.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:56:29 smithi153 ceph-mon[70044]: pgmap v17044: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:29.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:56:29 smithi083 ceph-mon[79930]: pgmap v17044: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:29.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:56:29 smithi083 ceph-mon[77242]: pgmap v17044: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:31.507 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:56:31 smithi083 ceph-mon[79930]: pgmap v17045: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:31.508 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:56:31 smithi083 ceph-mon[77242]: pgmap v17045: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:31.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:56:31 smithi153 ceph-mon[70044]: pgmap v17045: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:31.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:56:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:56:31] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T14:56:32.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:56:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:56:32.290Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:56:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:56:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:56:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:56:33.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:56:33 smithi153 ceph-mon[70044]: pgmap v17046: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:56:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:56:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:56:33.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:56:33 smithi083 ceph-mon[79930]: pgmap v17046: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:33.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:56:33 smithi083 ceph-mon[77242]: pgmap v17046: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:35.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:56:35 smithi153 ceph-mon[70044]: pgmap v17047: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:35.650 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:56:35 smithi083 ceph-mon[77242]: pgmap v17047: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:35.651 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:56:35 smithi083 ceph-mon[79930]: pgmap v17047: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:35.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:56:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:56:35.649Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:56:36.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:56:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:56:36.387Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:56:37.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:56:37 smithi153 ceph-mon[70044]: pgmap v17048: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:37.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:56:37 smithi083 ceph-mon[79930]: pgmap v17048: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:37.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:56:37 smithi083 ceph-mon[77242]: pgmap v17048: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:56:38 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:56:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:56:38 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:56:38.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:56:38 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:56:38.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:56:38 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:56:38.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:56:38 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:56:38.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:56:38 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:56:38.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:56:38 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:56:38.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:56:38 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:56:38.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:56:38 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:56:39.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:56:39 smithi153 ceph-mon[70044]: pgmap v17049: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:39.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:56:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:56:39.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:56:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:56:39.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:56:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:56:39.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:56:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:56:39.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:56:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:56:39.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:56:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:56:39.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:56:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:56:39.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:56:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:56:39.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:56:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:56:39.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:56:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:56:39.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:56:39 smithi083 ceph-mon[79930]: pgmap v17049: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:39.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:56:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:56:39.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:56:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:56:39.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:56:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:56:39.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:56:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:56:39.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:56:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:56:39.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:56:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:56:39.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:56:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:56:39.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:56:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:56:39.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:56:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:56:39.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:56:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:56:39.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:56:39 smithi083 ceph-mon[77242]: pgmap v17049: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:39.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:56:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:56:39.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:56:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:56:39.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:56:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:56:39.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:56:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:56:39.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:56:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:56:39.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:56:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:56:39.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:56:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:56:39.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:56:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T14:56:39.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:56:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:56:39.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:56:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:56:40.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:56:40 smithi153 ceph-mon[70044]: Adjusting osd_memory_target on smithi083 to 2190M 2022-09-29T14:56:40.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:56:40 smithi083 ceph-mon[77242]: Adjusting osd_memory_target on smithi083 to 2190M 2022-09-29T14:56:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:56:40 smithi083 ceph-mon[79930]: Adjusting osd_memory_target on smithi083 to 2190M 2022-09-29T14:56:41.506 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:56:41 smithi083 ceph-mon[79930]: pgmap v17050: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:41.507 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:56:41 smithi083 ceph-mon[77242]: pgmap v17050: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:56:41 smithi153 ceph-mon[70044]: pgmap v17050: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:56:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:56:41] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T14:56:42.556 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:56:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:56:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:56:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:56:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:56:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:56:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:56:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:56:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:56:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:56:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:56:42.591 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:56:42.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:56:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:56:42.292Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:56:42.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:56:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:56:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:56:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:56:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:56:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:56:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:56:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:56:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:56:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:56:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:56:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:56:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:56:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:56:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:56:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:56:42.880 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:56:42.880 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 8m ago 9h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:56:42.880 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 6m ago 9h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:56:42.881 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 8m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:56:42.881 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 6m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:56:42.881 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 6m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:56:42.881 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 8m ago 9h 974M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:56:42.882 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 8m ago 9h 2055M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:56:42.882 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 6m ago 9h 893M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:56:42.882 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 8m ago 9h 1230M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:56:42.882 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 8m ago 9h 24.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:56:42.883 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 6m ago 9h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:56:42.883 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 8m ago 9h 1490M 2190M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:56:42.884 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 8m ago 9h 1476M 2190M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:56:42.884 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 8m ago 9h 1147M 2190M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:56:42.885 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 8m ago 9h 1447M 2190M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:56:42.885 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 6m ago 9h 1269M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:56:42.886 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 6m ago 9h 1593M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:56:42.886 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 6m ago 9h 1424M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:56:42.887 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 6m ago 9h 1693M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:56:42.887 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 6m ago 9h 201M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:56:43.235 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:56:43.235 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:56:43.236 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:56:43.236 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:56:43.237 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:56:43.237 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:56:43.238 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:56:43.238 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:56:43.238 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:56:43.239 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:56:43.239 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:56:43.240 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:56:43.240 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:56:43.241 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:56:43.241 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:56:43.241 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:56:43.242 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:56:43.242 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:56:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:56:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:56:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:56:43.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:56:43 smithi153 ceph-mon[70044]: pgmap v17051: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:43.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:56:43 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1492030118' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:56:43.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:56:43 smithi083 ceph-mon[79930]: pgmap v17051: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:43.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:56:43 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1492030118' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:56:43.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:56:43 smithi083 ceph-mon[77242]: pgmap v17051: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:43.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:56:43 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1492030118' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:56:44.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:56:44 smithi153 ceph-mon[70044]: from='client.63889 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:56:44.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:56:44 smithi153 ceph-mon[70044]: from='client.63895 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:56:44.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:56:44 smithi083 ceph-mon[79930]: from='client.63889 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:56:44.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:56:44 smithi083 ceph-mon[79930]: from='client.63895 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:56:44.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:56:44 smithi083 ceph-mon[77242]: from='client.63889 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:56:44.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:56:44 smithi083 ceph-mon[77242]: from='client.63895 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:56:45.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:56:45 smithi153 ceph-mon[70044]: pgmap v17052: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:45.651 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:56:45 smithi083 ceph-mon[79930]: pgmap v17052: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:45.651 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:56:45 smithi083 ceph-mon[77242]: pgmap v17052: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:45.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:56:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:56:45.650Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:56:46.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:56:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:56:46.388Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:56:47.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:56:47 smithi153 ceph-mon[70044]: pgmap v17053: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:47.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:56:47 smithi083 ceph-mon[79930]: pgmap v17053: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:47.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:56:47 smithi083 ceph-mon[77242]: pgmap v17053: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:49.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:56:49 smithi153 ceph-mon[70044]: pgmap v17054: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:49.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:56:49 smithi083 ceph-mon[79930]: pgmap v17054: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:49.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:56:49 smithi083 ceph-mon[77242]: pgmap v17054: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:51.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:56:51 smithi153 ceph-mon[70044]: pgmap v17055: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:51.710 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:56:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:56:51] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T14:56:51.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:56:51 smithi083 ceph-mon[79930]: pgmap v17055: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:51.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:56:51 smithi083 ceph-mon[77242]: pgmap v17055: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:52.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:56:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:56:52.291Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:56:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:56:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:56:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:56:53.285 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:56:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:56:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:56:53.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:56:53 smithi153 ceph-mon[70044]: pgmap v17056: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:53.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:56:53 smithi083 ceph-mon[79930]: pgmap v17056: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:53.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:56:53 smithi083 ceph-mon[77242]: pgmap v17056: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:55.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:56:55 smithi153 ceph-mon[70044]: pgmap v17057: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:55.658 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:56:55 smithi083 ceph-mon[79930]: pgmap v17057: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:55.659 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:56:55 smithi083 ceph-mon[77242]: pgmap v17057: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:55.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:56:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:56:55.651Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:56:56.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:56:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:56:56.389Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:56:57.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:56:57 smithi153 ceph-mon[70044]: pgmap v17058: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:57.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:56:57 smithi083 ceph-mon[79930]: pgmap v17058: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:57.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:56:57 smithi083 ceph-mon[77242]: pgmap v17058: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:59.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:56:59 smithi153 ceph-mon[70044]: pgmap v17059: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:59.710 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:56:59 smithi083 ceph-mon[77242]: pgmap v17059: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:56:59.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:56:59 smithi083 ceph-mon[79930]: pgmap v17059: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:01.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:57:01 smithi153 ceph-mon[70044]: pgmap v17060: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:01.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:57:01 smithi083 ceph-mon[79930]: pgmap v17060: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:01.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:57:01 smithi083 ceph-mon[77242]: pgmap v17060: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:01.714 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:57:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:57:01] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T14:57:02.719 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:57:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:57:02.292Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:57:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:57:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:57:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:57:03.302 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:57:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:57:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:57:03.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:57:03 smithi153 ceph-mon[70044]: pgmap v17061: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:03.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:57:03 smithi083 ceph-mon[79930]: pgmap v17061: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:03.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:57:03 smithi083 ceph-mon[77242]: pgmap v17061: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:05.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:57:05 smithi153 ceph-mon[70044]: pgmap v17062: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:05.652 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:57:05 smithi083 ceph-mon[79930]: pgmap v17062: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:05.652 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:57:05 smithi083 ceph-mon[77242]: pgmap v17062: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:05.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:57:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:57:05.651Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:57:06.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:57:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:57:06.389Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:57:07.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:57:07 smithi083 ceph-mon[79930]: pgmap v17063: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:07.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:57:07 smithi083 ceph-mon[77242]: pgmap v17063: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:07.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:57:07 smithi153 ceph-mon[70044]: pgmap v17063: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:09.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:57:09 smithi083 ceph-mon[79930]: pgmap v17064: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:09.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:57:09 smithi083 ceph-mon[77242]: pgmap v17064: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:09.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:57:09 smithi153 ceph-mon[70044]: pgmap v17064: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:11.710 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:57:11 smithi083 ceph-mon[77242]: pgmap v17065: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:11.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:57:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:57:11] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T14:57:11.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:57:11 smithi083 ceph-mon[79930]: pgmap v17065: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:11.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:57:11 smithi153 ceph-mon[70044]: pgmap v17065: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:12.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:57:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:57:12.293Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:57:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:57:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:57:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:57:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:57:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:57:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:57:13.568 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:57:13.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:57:13 smithi083 ceph-mon[79930]: pgmap v17066: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:13.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:57:13 smithi083 ceph-mon[77242]: pgmap v17066: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:13.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:57:13 smithi153 ceph-mon[70044]: pgmap v17066: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:13.865 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:57:13.865 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 8m ago 9h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:57:13.866 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 6m ago 9h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:57:13.866 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 8m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:57:13.866 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 6m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:57:13.867 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 6m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:57:13.867 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 8m ago 9h 974M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:57:13.867 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 8m ago 9h 2055M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:57:13.868 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 6m ago 9h 893M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:57:13.868 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 8m ago 9h 1230M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:57:13.869 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 8m ago 9h 24.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:57:13.870 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 6m ago 9h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:57:13.870 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 8m ago 9h 1490M 2190M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:57:13.870 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 8m ago 9h 1476M 2190M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:57:13.870 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 8m ago 9h 1147M 2190M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:57:13.871 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 8m ago 9h 1447M 2190M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:57:13.871 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 6m ago 9h 1269M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:57:13.871 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 6m ago 9h 1593M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:57:13.871 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 6m ago 9h 1424M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:57:13.872 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 6m ago 9h 1693M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:57:13.872 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 6m ago 9h 201M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:57:14.222 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:57:14.223 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:57:14.223 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:57:14.223 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:57:14.223 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:57:14.224 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:57:14.224 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:57:14.224 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:57:14.224 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:57:14.225 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:57:14.225 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:57:14.225 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:57:14.225 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:57:14.228 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:57:14.229 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:57:14.229 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:57:14.230 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:57:14.230 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:57:14.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:57:14 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2482707989' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:57:14.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:57:14 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2482707989' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:57:14.815 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:57:14 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2482707989' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:57:15.653 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:57:15 smithi083 ceph-mon[79930]: from='client.53976 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:57:15.653 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:57:15 smithi083 ceph-mon[79930]: from='client.53982 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:57:15.654 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:57:15 smithi083 ceph-mon[79930]: pgmap v17067: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:15.654 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:57:15 smithi083 ceph-mon[77242]: from='client.53976 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:57:15.655 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:57:15 smithi083 ceph-mon[77242]: from='client.53982 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:57:15.655 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:57:15 smithi083 ceph-mon[77242]: pgmap v17067: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:15.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:57:15 smithi153 ceph-mon[70044]: from='client.53976 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:57:15.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:57:15 smithi153 ceph-mon[70044]: from='client.53982 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:57:15.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:57:15 smithi153 ceph-mon[70044]: pgmap v17067: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:15.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:57:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:57:15.652Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:57:16.716 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:57:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:57:16.390Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:57:17.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:57:17 smithi083 ceph-mon[79930]: pgmap v17068: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:17.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:57:17 smithi083 ceph-mon[77242]: pgmap v17068: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:17.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:57:17 smithi153 ceph-mon[70044]: pgmap v17068: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:19.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:57:19 smithi083 ceph-mon[79930]: pgmap v17069: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:19.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:57:19 smithi083 ceph-mon[77242]: pgmap v17069: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:19.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:57:19 smithi153 ceph-mon[70044]: pgmap v17069: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:20.722 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:57:20 smithi083 ceph-mon[79930]: pgmap v17070: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:20.723 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:57:20 smithi083 ceph-mon[77242]: pgmap v17070: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:20.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:57:20 smithi153 ceph-mon[70044]: pgmap v17070: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:21.972 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:57:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:57:21] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T14:57:22.718 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:57:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:57:22.293Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:57:22.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:57:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:57:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:57:23.314 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:57:23 smithi153 ceph-mon[70044]: pgmap v17071: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:23.315 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:57:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:57:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:57:23.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:57:23 smithi083 ceph-mon[79930]: pgmap v17071: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:23.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:57:23 smithi083 ceph-mon[77242]: pgmap v17071: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:25.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:57:25 smithi153 ceph-mon[70044]: pgmap v17072: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:25.653 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:57:25 smithi083 ceph-mon[77242]: pgmap v17072: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:25.654 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:57:25 smithi083 ceph-mon[79930]: pgmap v17072: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:25.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:57:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:57:25.652Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:57:26.717 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:57:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:57:26.391Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:57:27.567 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:57:27 smithi153 ceph-mon[70044]: pgmap v17073: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:27.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:57:27 smithi083 ceph-mon[79930]: pgmap v17073: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:27.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:57:27 smithi083 ceph-mon[77242]: pgmap v17073: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:29.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:57:29 smithi153 ceph-mon[70044]: pgmap v17074: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:29.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:57:29 smithi083 ceph-mon[79930]: pgmap v17074: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:29.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:57:29 smithi083 ceph-mon[77242]: pgmap v17074: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:31.513 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:57:31 smithi083 ceph-mon[77242]: pgmap v17075: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:31.514 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:57:31 smithi083 ceph-mon[79930]: pgmap v17075: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:31.562 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:57:31 smithi153 ceph-mon[70044]: pgmap v17075: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:57:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:57:31] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T14:57:32.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:57:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:57:32.294Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:57:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:57:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:57:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:57:33.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:57:33 smithi153 ceph-mon[70044]: pgmap v17076: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:33.311 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:57:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:57:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:57:33.719 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:57:33 smithi083 ceph-mon[79930]: pgmap v17076: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:33.720 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:57:33 smithi083 ceph-mon[77242]: pgmap v17076: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:35.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:57:35 smithi153 ceph-mon[70044]: pgmap v17077: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:35.655 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:57:35 smithi083 ceph-mon[79930]: pgmap v17077: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:35.656 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:57:35 smithi083 ceph-mon[77242]: pgmap v17077: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:35.965 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:57:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:57:35.653Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:57:36.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:57:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:57:36.392Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:57:37.563 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:57:37 smithi153 ceph-mon[70044]: pgmap v17078: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:37.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:57:37 smithi083 ceph-mon[79930]: pgmap v17078: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:37.726 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:57:37 smithi083 ceph-mon[77242]: pgmap v17078: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:39.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:57:39 smithi153 ceph-mon[70044]: pgmap v17079: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:39.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:57:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:57:39.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:57:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:57:39.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:57:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:57:39.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:57:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:57:39.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:57:39 smithi083 ceph-mon[79930]: pgmap v17079: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:39.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:57:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:57:39.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:57:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:57:39.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:57:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:57:39.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:57:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:57:39.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:57:39 smithi083 ceph-mon[77242]: pgmap v17079: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:39.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:57:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:57:39.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:57:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:57:39.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:57:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:57:39.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:57:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:57:41.507 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:57:41 smithi083 ceph-mon[79930]: pgmap v17080: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:41.507 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:57:41 smithi083 ceph-mon[77242]: pgmap v17080: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:57:41 smithi153 ceph-mon[70044]: pgmap v17080: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:41.964 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:57:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:57:41] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T14:57:42.569 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:57:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:57:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:57:42.589 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:57:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:57:42.589 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:57:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:57:42.589 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:57:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:57:42.590 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:57:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:57:42.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:57:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:57:42.295Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:57:42.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:57:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:57:42.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:57:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:57:42.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:57:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:57:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:57:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:57:42.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:57:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:57:42.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:57:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:57:42.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:57:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:57:42.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:57:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:57:43.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:57:43 smithi153 ceph-mon[70044]: pgmap v17081: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:57:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:57:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:57:43.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:57:43 smithi083 ceph-mon[79930]: pgmap v17081: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:43.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:57:43 smithi083 ceph-mon[77242]: pgmap v17081: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:44.551 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:57:44.839 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:57:44.846 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 9m ago 9h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:57:44.846 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 7m ago 9h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:57:44.847 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 9m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:57:44.847 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 7m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:57:44.848 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 7m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:57:44.849 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 9m ago 9h 974M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:57:44.854 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 9m ago 9h 2055M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:57:44.855 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 7m ago 9h 893M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:57:44.855 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 9m ago 9h 1230M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:57:44.856 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 9m ago 9h 24.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:57:44.857 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 7m ago 9h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:57:44.857 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 9m ago 9h 1490M 2190M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:57:44.858 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 9m ago 9h 1476M 2190M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:57:44.858 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 9m ago 9h 1147M 2190M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:57:44.859 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 9m ago 9h 1447M 2190M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:57:44.859 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 7m ago 9h 1269M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:57:44.860 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 7m ago 9h 1593M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:57:44.860 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 7m ago 9h 1424M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:57:44.861 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 7m ago 9h 1693M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:57:44.861 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 7m ago 9h 201M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:57:45.195 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:57:45.196 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:57:45.196 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:57:45.197 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:57:45.197 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:57:45.197 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:57:45.198 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:57:45.198 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:57:45.198 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:57:45.198 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:57:45.199 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:57:45.199 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:57:45.199 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:57:45.199 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:57:45.200 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:57:45.200 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:57:45.200 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:57:45.200 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:57:45.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:57:45 smithi153 ceph-mon[70044]: pgmap v17082: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:45.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:57:45 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/427241739' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:57:45.653 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:57:45 smithi083 ceph-mon[79930]: pgmap v17082: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:45.654 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:57:45 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/427241739' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:57:45.655 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:57:45 smithi083 ceph-mon[77242]: pgmap v17082: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:45.655 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:57:45 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/427241739' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:57:45.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:57:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:57:45.653Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:57:46.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:57:46 smithi153 ceph-mon[70044]: from='client.63925 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:57:46.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:57:46 smithi153 ceph-mon[70044]: from='client.63931 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:57:46.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:57:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:57:46.393Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:57:46.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:57:46 smithi083 ceph-mon[77242]: from='client.63925 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:57:46.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:57:46 smithi083 ceph-mon[77242]: from='client.63931 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:57:46.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:57:46 smithi083 ceph-mon[79930]: from='client.63925 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:57:46.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:57:46 smithi083 ceph-mon[79930]: from='client.63931 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:57:47.561 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:57:47 smithi153 ceph-mon[70044]: pgmap v17083: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:47.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:57:47 smithi083 ceph-mon[79930]: pgmap v17083: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:47.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:57:47 smithi083 ceph-mon[77242]: pgmap v17083: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:49.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:57:49 smithi153 ceph-mon[70044]: pgmap v17084: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:49.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:57:49 smithi083 ceph-mon[79930]: pgmap v17084: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:49.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:57:49 smithi083 ceph-mon[77242]: pgmap v17084: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:51.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:57:51 smithi153 ceph-mon[70044]: pgmap v17085: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:51.718 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:57:51 smithi083 ceph-mon[79930]: pgmap v17085: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:51.718 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:57:51 smithi083 ceph-mon[77242]: pgmap v17085: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:51.719 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:57:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:57:51] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T14:57:52.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:57:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:57:52.295Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:57:52.815 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:57:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:57:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:57:53.278 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:57:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:57:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:57:53.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:57:53 smithi153 ceph-mon[70044]: pgmap v17086: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:53.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:57:53 smithi083 ceph-mon[79930]: pgmap v17086: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:53.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:57:53 smithi083 ceph-mon[77242]: pgmap v17086: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:55.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:57:55 smithi153 ceph-mon[70044]: pgmap v17087: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:55.656 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:57:55 smithi083 ceph-mon[79930]: pgmap v17087: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:55.657 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:57:55 smithi083 ceph-mon[77242]: pgmap v17087: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:55.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:57:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:57:55.654Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:57:56.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:57:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:57:56.394Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:57:57.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:57:57 smithi153 ceph-mon[70044]: pgmap v17088: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:57.710 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:57:57 smithi083 ceph-mon[77242]: pgmap v17088: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:57.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:57:57 smithi083 ceph-mon[79930]: pgmap v17088: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:59.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:57:59 smithi153 ceph-mon[70044]: pgmap v17089: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:59.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:57:59 smithi083 ceph-mon[79930]: pgmap v17089: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:57:59.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:57:59 smithi083 ceph-mon[77242]: pgmap v17089: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:01.561 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:58:01 smithi153 ceph-mon[70044]: pgmap v17090: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:01.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:58:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:58:01] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T14:58:01.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:58:01 smithi083 ceph-mon[79930]: pgmap v17090: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:01.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:58:01 smithi083 ceph-mon[77242]: pgmap v17090: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:02.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:58:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:58:02.296Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:58:02.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:58:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:58:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:58:03.294 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:58:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:58:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:58:03.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:58:03 smithi153 ceph-mon[70044]: pgmap v17091: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:03.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:58:03 smithi083 ceph-mon[79930]: pgmap v17091: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:03.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:58:03 smithi083 ceph-mon[77242]: pgmap v17091: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:05.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:58:05 smithi153 ceph-mon[70044]: pgmap v17092: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:05.656 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:58:05 smithi083 ceph-mon[79930]: pgmap v17092: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:05.657 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:58:05 smithi083 ceph-mon[77242]: pgmap v17092: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:05.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:58:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:58:05.655Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:58:06.714 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:58:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:58:06.394Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:58:07.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:58:07 smithi153 ceph-mon[70044]: pgmap v17093: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:07.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:58:07 smithi083 ceph-mon[77242]: pgmap v17093: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:07.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:58:07 smithi083 ceph-mon[79930]: pgmap v17093: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:09.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:58:09 smithi153 ceph-mon[70044]: pgmap v17094: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:09.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:58:09 smithi083 ceph-mon[79930]: pgmap v17094: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:09.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:58:09 smithi083 ceph-mon[77242]: pgmap v17094: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:11.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:58:11 smithi153 ceph-mon[70044]: pgmap v17095: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:11.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:58:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:58:11] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T14:58:11.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:58:11 smithi083 ceph-mon[79930]: pgmap v17095: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:11.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:58:11 smithi083 ceph-mon[77242]: pgmap v17095: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:12.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:58:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:58:12.296Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:58:12.810 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:58:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:58:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:58:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:58:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:58:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:58:13.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:58:13 smithi083 ceph-mon[79930]: pgmap v17096: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:13.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:58:13 smithi083 ceph-mon[77242]: pgmap v17096: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:13.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:58:13 smithi153 ceph-mon[70044]: pgmap v17096: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:15.529 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:58:15.658 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:58:15 smithi083 ceph-mon[79930]: pgmap v17097: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:15.659 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:58:15 smithi083 ceph-mon[77242]: pgmap v17097: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:15.811 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:58:15.812 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 9m ago 9h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:58:15.813 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 7m ago 9h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:58:15.813 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 9m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:58:15.813 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 7m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:58:15.814 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 7m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:58:15.814 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 9m ago 9h 974M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:58:15.815 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 9m ago 9h 2055M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:58:15.815 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 7m ago 9h 893M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:58:15.816 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 9m ago 9h 1230M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:58:15.816 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 9m ago 9h 24.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:58:15.817 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 7m ago 9h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:58:15.817 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 9m ago 9h 1490M 2190M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:58:15.818 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 9m ago 9h 1476M 2190M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:58:15.818 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 9m ago 9h 1147M 2190M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:58:15.819 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 9m ago 9h 1447M 2190M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:58:15.819 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 7m ago 9h 1269M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:58:15.819 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 7m ago 9h 1593M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:58:15.820 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 7m ago 9h 1424M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:58:15.820 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 7m ago 9h 1693M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:58:15.823 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 7m ago 9h 201M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:58:15.824 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:58:15 smithi153 ceph-mon[70044]: pgmap v17097: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:15.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:58:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:58:15.655Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:58:16.173 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:58:16.173 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:58:16.173 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:58:16.174 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:58:16.174 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:58:16.174 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:58:16.175 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:58:16.175 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:58:16.175 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:58:16.175 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:58:16.176 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:58:16.176 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:58:16.176 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:58:16.176 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:58:16.177 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:58:16.177 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:58:16.177 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:58:16.177 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:58:16.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:58:16 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4176123546' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:58:16.715 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:58:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:58:16.394Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:58:16.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:58:16 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4176123546' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:58:16.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:58:16 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4176123546' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:58:17.722 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:58:17 smithi083 ceph-mon[79930]: from='client.63943 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:58:17.723 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:58:17 smithi083 ceph-mon[79930]: from='client.63949 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:58:17.723 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:58:17 smithi083 ceph-mon[79930]: pgmap v17098: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:17.724 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:58:17 smithi083 ceph-mon[77242]: from='client.63943 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:58:17.724 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:58:17 smithi083 ceph-mon[77242]: from='client.63949 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:58:17.725 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:58:17 smithi083 ceph-mon[77242]: pgmap v17098: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:17.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:58:17 smithi153 ceph-mon[70044]: from='client.63943 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:58:17.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:58:17 smithi153 ceph-mon[70044]: from='client.63949 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:58:17.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:58:17 smithi153 ceph-mon[70044]: pgmap v17098: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:19.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:58:19 smithi083 ceph-mon[79930]: pgmap v17099: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:19.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:58:19 smithi083 ceph-mon[77242]: pgmap v17099: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:19.814 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:58:19 smithi153 ceph-mon[70044]: pgmap v17099: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:21.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:58:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:58:21] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T14:58:21.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:58:21 smithi083 ceph-mon[79930]: pgmap v17100: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:21.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:58:21 smithi083 ceph-mon[77242]: pgmap v17100: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:21.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:58:21 smithi153 ceph-mon[70044]: pgmap v17100: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:22.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:58:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:58:22.297Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:58:22.811 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:58:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:58:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:58:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:58:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:58:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:58:23.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:58:23 smithi083 ceph-mon[79930]: pgmap v17101: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:23.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:58:23 smithi083 ceph-mon[77242]: pgmap v17101: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:23.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:58:23 smithi153 ceph-mon[70044]: pgmap v17101: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:25.658 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:58:25 smithi083 ceph-mon[79930]: pgmap v17102: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:25.659 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:58:25 smithi083 ceph-mon[77242]: pgmap v17102: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:25.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:58:25 smithi153 ceph-mon[70044]: pgmap v17102: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:25.967 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:58:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:58:25.657Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:58:26.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:58:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:58:26.396Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:58:27.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:58:27 smithi083 ceph-mon[79930]: pgmap v17103: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:27.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:58:27 smithi083 ceph-mon[77242]: pgmap v17103: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:27.814 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:58:27 smithi153 ceph-mon[70044]: pgmap v17103: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:29.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:58:29 smithi083 ceph-mon[77242]: pgmap v17104: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:29.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:58:29 smithi083 ceph-mon[79930]: pgmap v17104: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:29.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:58:29 smithi153 ceph-mon[70044]: pgmap v17104: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:31.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:58:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:58:31] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T14:58:31.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:58:31 smithi083 ceph-mon[79930]: pgmap v17105: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:31.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:58:31 smithi083 ceph-mon[77242]: pgmap v17105: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:31.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:58:31 smithi153 ceph-mon[70044]: pgmap v17105: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:32.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:58:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:58:32.297Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:58:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:58:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:58:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:58:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:58:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:58:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:58:33.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:58:33 smithi083 ceph-mon[77242]: pgmap v17106: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:33.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:58:33 smithi083 ceph-mon[79930]: pgmap v17106: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:33.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:58:33 smithi153 ceph-mon[70044]: pgmap v17106: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:35.658 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:58:35 smithi083 ceph-mon[79930]: pgmap v17107: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:35.659 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:58:35 smithi083 ceph-mon[77242]: pgmap v17107: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:35.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:58:35 smithi153 ceph-mon[70044]: pgmap v17107: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:35.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:58:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:58:35.657Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:58:36.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:58:36 smithi083 ceph-mon[77242]: pgmap v17108: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:36.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:58:36 smithi083 ceph-mon[79930]: pgmap v17108: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:36.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:58:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:58:36.397Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:58:36.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:58:36 smithi153 ceph-mon[70044]: pgmap v17108: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:39.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:58:39 smithi153 ceph-mon[70044]: pgmap v17109: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:39.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:58:39 smithi083 ceph-mon[79930]: pgmap v17109: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:39.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:58:39 smithi083 ceph-mon[77242]: pgmap v17109: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:40.563 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:58:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:58:40.563 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:58:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:58:40.564 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:58:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:58:40.564 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:58:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:58:40.565 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:58:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:58:40.565 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:58:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:58:40.565 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:58:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:58:40.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:58:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:58:40.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:58:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:58:40.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:58:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:58:40.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:58:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:58:40.718 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:58:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:58:40.718 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:58:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:58:40.729 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:58:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:58:40.730 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:58:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:58:40.730 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:58:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:58:40.731 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:58:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:58:40.731 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:58:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:58:40.731 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:58:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:58:40.732 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:58:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:58:40.732 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:58:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:58:41.507 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:58:41 smithi083 ceph-mon[79930]: pgmap v17110: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:41.508 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:58:41 smithi083 ceph-mon[77242]: pgmap v17110: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:58:41 smithi153 ceph-mon[70044]: pgmap v17110: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:41.964 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:58:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:58:41] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T14:58:42.556 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:58:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:58:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:58:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:58:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:58:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:58:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:58:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:58:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:58:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:58:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:58:42.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:58:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:58:42.297Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:58:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:58:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:58:42.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:58:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:58:42.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:58:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:58:42.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:58:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:58:42.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:58:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:58:42.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:58:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:58:42.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:58:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:58:42.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:58:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:58:43.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:58:43 smithi153 ceph-mon[70044]: pgmap v17111: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:58:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:58:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:58:43.723 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:58:43 smithi083 ceph-mon[79930]: pgmap v17111: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:43.724 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:58:43 smithi083 ceph-mon[77242]: pgmap v17111: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:45.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:58:44 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:58:45.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:58:44 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:58:45.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:58:44 smithi083 ceph-mon[77242]: pgmap v17112: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:45.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:58:44 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:58:45.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:58:44 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:58:45.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:58:44 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:58:45.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:58:44 smithi083 ceph-mon[79930]: pgmap v17112: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:45.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:58:44 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:58:45.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:58:44 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:58:45.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:58:44 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:58:45.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:58:44 smithi153 ceph-mon[70044]: pgmap v17112: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:45.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:58:44 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:58:45.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:58:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:58:45.658Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:58:46.498 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:58:46.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:58:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:58:46.397Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:58:46.789 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:58:46.789 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 2s ago 9h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:58:46.790 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 8m ago 9h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:58:46.790 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 2s ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:58:46.791 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 8m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:58:46.791 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 8m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:58:46.791 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 2s ago 9h 982M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:58:46.792 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 2s ago 9h 2088M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:58:46.792 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 8m ago 9h 893M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:58:46.793 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 2s ago 9h 1251M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:58:46.793 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 2s ago 9h 24.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:58:46.793 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 8m ago 9h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:58:46.806 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 2s ago 9h 1516M 2190M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:58:46.807 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 2s ago 9h 1503M 2190M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:58:46.807 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 2s ago 9h 1168M 2190M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:58:46.808 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 2s ago 9h 1473M 2190M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:58:46.808 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 8m ago 9h 1269M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:58:46.808 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 8m ago 9h 1593M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:58:46.809 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 8m ago 9h 1424M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:58:46.809 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 8m ago 9h 1693M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:58:46.809 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 8m ago 9h 201M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:58:47.147 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:58:47.148 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:58:47.148 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:58:47.149 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:58:47.149 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:58:47.150 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:58:47.150 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:58:47.151 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:58:47.151 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:58:47.151 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:58:47.152 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:58:47.152 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:58:47.153 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:58:47.153 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:58:47.165 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:58:47.169 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:58:47.169 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:58:47.170 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:58:47.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:58:47 smithi153 ceph-mon[70044]: pgmap v17113: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:47.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:58:47 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3907691417' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:58:47.719 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:58:47 smithi083 ceph-mon[77242]: pgmap v17113: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:47.719 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:58:47 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3907691417' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:58:47.720 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:58:47 smithi083 ceph-mon[79930]: pgmap v17113: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:47.720 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:58:47 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3907691417' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:58:48.562 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:58:48 smithi153 ceph-mon[70044]: from='client.54030 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:58:48.563 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:58:48 smithi153 ceph-mon[70044]: from='client.54036 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:58:48.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:58:48 smithi083 ceph-mon[79930]: from='client.54030 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:58:48.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:58:48 smithi083 ceph-mon[79930]: from='client.54036 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:58:48.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:58:48 smithi083 ceph-mon[77242]: from='client.54030 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:58:48.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:58:48 smithi083 ceph-mon[77242]: from='client.54036 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:58:49.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:58:49 smithi153 ceph-mon[70044]: pgmap v17114: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:49.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:58:49 smithi083 ceph-mon[79930]: pgmap v17114: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:49.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:58:49 smithi083 ceph-mon[77242]: pgmap v17114: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:51.509 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:58:51 smithi083 ceph-mon[79930]: pgmap v17115: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:51.510 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:58:51 smithi083 ceph-mon[77242]: pgmap v17115: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:51.563 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:58:51 smithi153 ceph-mon[70044]: pgmap v17115: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:51.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:58:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:58:51] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T14:58:52.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:58:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:58:52.299Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:58:52.810 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:58:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:58:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:58:53.323 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:58:53 smithi153 ceph-mon[70044]: pgmap v17116: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:53.323 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:58:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:58:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:58:53.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:58:53 smithi083 ceph-mon[79930]: pgmap v17116: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:53.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:58:53 smithi083 ceph-mon[77242]: pgmap v17116: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:55.563 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:58:55 smithi153 ceph-mon[70044]: pgmap v17117: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:55.659 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:58:55 smithi083 ceph-mon[79930]: pgmap v17117: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:55.661 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:58:55 smithi083 ceph-mon[77242]: pgmap v17117: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:55.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:58:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:58:55.659Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:58:56.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:58:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:58:56.399Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:58:57.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:58:57 smithi153 ceph-mon[70044]: pgmap v17118: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:57.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:58:57 smithi083 ceph-mon[79930]: pgmap v17118: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:57.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:58:57 smithi083 ceph-mon[77242]: pgmap v17118: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:59.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:58:59 smithi153 ceph-mon[70044]: pgmap v17119: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:59.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:58:59 smithi083 ceph-mon[79930]: pgmap v17119: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:58:59.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:58:59 smithi083 ceph-mon[77242]: pgmap v17119: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:01.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:59:01 smithi153 ceph-mon[70044]: pgmap v17120: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:01.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:59:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:59:01] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T14:59:01.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:59:01 smithi083 ceph-mon[79930]: pgmap v17120: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:01.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:59:01 smithi083 ceph-mon[77242]: pgmap v17120: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:02.714 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:59:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:59:02.299Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:59:02.810 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:59:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:59:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:59:03.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:59:03 smithi153 ceph-mon[70044]: pgmap v17121: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:59:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:59:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:59:03.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:59:03 smithi083 ceph-mon[79930]: pgmap v17121: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:03.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:59:03 smithi083 ceph-mon[77242]: pgmap v17121: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:05.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:59:05 smithi153 ceph-mon[70044]: pgmap v17122: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:05.675 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:59:05 smithi083 ceph-mon[79930]: pgmap v17122: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:05.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:59:05 smithi083 ceph-mon[77242]: pgmap v17122: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:05.971 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:59:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:59:05.660Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:59:06.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:59:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:59:06.399Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:59:07.562 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:59:07 smithi153 ceph-mon[70044]: pgmap v17123: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:07.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:59:07 smithi083 ceph-mon[79930]: pgmap v17123: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:07.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:59:07 smithi083 ceph-mon[77242]: pgmap v17123: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:09.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:59:09 smithi153 ceph-mon[70044]: pgmap v17124: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:09.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:59:09 smithi083 ceph-mon[79930]: pgmap v17124: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:09.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:59:09 smithi083 ceph-mon[77242]: pgmap v17124: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:11.565 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:59:11 smithi153 ceph-mon[70044]: pgmap v17125: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:11.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:59:11 smithi083 ceph-mon[77242]: pgmap v17125: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:11.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:59:11 smithi083 ceph-mon[79930]: pgmap v17125: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:11.717 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:59:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:59:11] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T14:59:12.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:59:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:59:12.300Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:59:12.811 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:59:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:59:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:59:13.284 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:59:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:59:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:59:13.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:59:13 smithi153 ceph-mon[70044]: pgmap v17126: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:13.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:59:13 smithi083 ceph-mon[79930]: pgmap v17126: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:13.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:59:13 smithi083 ceph-mon[77242]: pgmap v17126: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:15.566 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:59:15 smithi153 ceph-mon[70044]: pgmap v17127: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:15.663 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:59:15 smithi083 ceph-mon[79930]: pgmap v17127: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:15.663 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:59:15 smithi083 ceph-mon[77242]: pgmap v17127: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:15.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:59:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:59:15.661Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:59:16.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:59:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:59:16.400Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:59:17.473 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:59:17.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:59:17 smithi153 ceph-mon[70044]: pgmap v17128: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:17.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:59:17 smithi083 ceph-mon[79930]: pgmap v17128: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:17.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:59:17 smithi083 ceph-mon[77242]: pgmap v17128: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:17.760 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:59:17.760 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 33s ago 9h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:59:17.760 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 8m ago 9h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:59:17.761 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 33s ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:59:17.761 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 8m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:59:17.761 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 8m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:59:17.762 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 33s ago 9h 982M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:59:17.762 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 33s ago 9h 2088M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:59:17.762 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 8m ago 9h 893M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:59:17.762 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 33s ago 9h 1251M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:59:17.763 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 33s ago 9h 24.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:59:17.763 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 8m ago 9h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:59:17.763 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 33s ago 9h 1516M 2190M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:59:17.763 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 33s ago 9h 1503M 2190M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:59:17.764 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 33s ago 9h 1168M 2190M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:59:17.764 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 33s ago 9h 1473M 2190M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:59:17.764 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 8m ago 9h 1269M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:59:17.764 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 8m ago 9h 1593M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:59:17.764 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 8m ago 9h 1424M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:59:17.765 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 8m ago 9h 1693M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:59:17.773 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 8m ago 9h 201M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:59:18.115 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:59:18.115 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:59:18.116 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:59:18.117 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:59:18.117 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:59:18.117 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:59:18.131 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:59:18.133 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:59:18.134 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:59:18.137 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:59:18.137 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:59:18.137 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:59:18.145 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:59:18.146 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:59:18.149 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:59:18.150 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:59:18.150 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:59:18.151 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:59:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:59:18 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3095582574' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:59:18.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:59:18 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3095582574' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:59:18.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:59:18 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3095582574' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:59:19.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:59:19 smithi153 ceph-mon[70044]: from='client.54048 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:59:19.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:59:19 smithi153 ceph-mon[70044]: from='client.63985 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:59:19.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:59:19 smithi153 ceph-mon[70044]: pgmap v17129: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:19.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:59:19 smithi083 ceph-mon[79930]: from='client.54048 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:59:19.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:59:19 smithi083 ceph-mon[79930]: from='client.63985 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:59:19.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:59:19 smithi083 ceph-mon[79930]: pgmap v17129: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:19.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:59:19 smithi083 ceph-mon[77242]: from='client.54048 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:59:19.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:59:19 smithi083 ceph-mon[77242]: from='client.63985 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:59:19.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:59:19 smithi083 ceph-mon[77242]: pgmap v17129: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:21.563 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:59:21 smithi153 ceph-mon[70044]: pgmap v17130: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:21.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:59:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:59:21] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T14:59:21.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:59:21 smithi083 ceph-mon[79930]: pgmap v17130: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:21.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:59:21 smithi083 ceph-mon[77242]: pgmap v17130: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:22.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:59:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:59:22.301Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:59:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:59:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:59:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:59:23.314 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:59:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:59:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:59:23.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:59:23 smithi153 ceph-mon[70044]: pgmap v17131: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:23.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:59:23 smithi083 ceph-mon[79930]: pgmap v17131: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:23.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:59:23 smithi083 ceph-mon[77242]: pgmap v17131: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:25.664 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:59:25 smithi083 ceph-mon[79930]: pgmap v17132: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:25.665 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:59:25 smithi083 ceph-mon[77242]: pgmap v17132: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:25.819 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:59:25 smithi153 ceph-mon[70044]: pgmap v17132: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:25.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:59:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:59:25.662Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:59:26.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:59:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:59:26.401Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:59:27.710 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:59:27 smithi083 ceph-mon[77242]: pgmap v17133: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:27.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:59:27 smithi083 ceph-mon[79930]: pgmap v17133: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:27.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:59:27 smithi153 ceph-mon[70044]: pgmap v17133: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:29.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:59:29 smithi083 ceph-mon[79930]: pgmap v17134: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:29.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:59:29 smithi083 ceph-mon[77242]: pgmap v17134: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:29.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:59:29 smithi153 ceph-mon[70044]: pgmap v17134: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:31.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:59:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:59:31] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T14:59:31.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:59:31 smithi083 ceph-mon[79930]: pgmap v17135: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:31.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:59:31 smithi083 ceph-mon[77242]: pgmap v17135: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:31.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:59:31 smithi153 ceph-mon[70044]: pgmap v17135: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:32.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:59:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:59:32.301Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:59:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:59:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:59:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:59:33.312 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:59:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:59:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:59:33.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:59:33 smithi083 ceph-mon[79930]: pgmap v17136: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:33.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:59:33 smithi083 ceph-mon[77242]: pgmap v17136: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:33.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:59:33 smithi153 ceph-mon[70044]: pgmap v17136: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:35.667 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:59:35 smithi083 ceph-mon[79930]: pgmap v17137: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:35.667 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:59:35 smithi083 ceph-mon[77242]: pgmap v17137: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:35.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:59:35 smithi153 ceph-mon[70044]: pgmap v17137: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:35.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:59:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:59:35.662Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:59:36.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:59:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:59:36.402Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:59:37.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:59:37 smithi083 ceph-mon[79930]: pgmap v17138: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:37.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:59:37 smithi083 ceph-mon[77242]: pgmap v17138: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:37.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:59:37 smithi153 ceph-mon[70044]: pgmap v17138: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:39.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:59:39 smithi083 ceph-mon[79930]: pgmap v17139: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:39.718 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:59:39 smithi083 ceph-mon[77242]: pgmap v17139: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:39.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:59:39 smithi153 ceph-mon[70044]: pgmap v17139: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:41.710 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:59:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:59:41] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T14:59:41.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:59:41 smithi083 ceph-mon[79930]: pgmap v17140: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:41.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:59:41 smithi083 ceph-mon[77242]: pgmap v17140: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:41.814 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:59:41 smithi153 ceph-mon[70044]: pgmap v17140: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:42.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:59:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:59:42.301Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:59:42.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:59:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:59:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:59:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:59:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:59:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:59:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:59:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:59:42.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:59:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:59:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:59:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:59:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:59:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:59:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:59:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:59:42.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:59:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:59:42.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:59:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:59:42.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:59:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T14:59:42.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:59:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T14:59:42.813 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:59:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:59:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:59:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:59:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:59:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:59:43.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:59:43 smithi083 ceph-mon[79930]: pgmap v17141: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:43.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:59:43 smithi083 ceph-mon[77242]: pgmap v17141: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:43.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:59:43 smithi153 ceph-mon[70044]: pgmap v17141: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:45.667 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:59:45 smithi083 ceph-mon[79930]: pgmap v17142: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:45.668 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:59:45 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:59:45.669 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:59:45 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:59:45.669 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:59:45 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:59:45.669 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:59:45 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:59:45.674 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:59:45 smithi083 ceph-mon[77242]: pgmap v17142: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:45.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:59:45 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:59:45.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:59:45 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:59:45.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:59:45 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:59:45.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:59:45 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:59:45.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:59:45 smithi153 ceph-mon[70044]: pgmap v17142: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:45.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:59:45 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T14:59:45.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:59:45 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T14:59:45.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:59:45 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T14:59:45.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:59:45 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T14:59:45.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:59:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:59:45.662Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:59:46.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:59:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:59:46.403Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:59:47.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:59:47 smithi083 ceph-mon[79930]: pgmap v17143: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:47.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:59:47 smithi083 ceph-mon[77242]: pgmap v17143: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:47.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:59:47 smithi153 ceph-mon[70044]: pgmap v17143: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:48.442 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T14:59:48.730 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T14:59:48.730 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 64s ago 9h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T14:59:48.731 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 9m ago 9h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T14:59:48.731 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 64s ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T14:59:48.731 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 9m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T14:59:48.732 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 9m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T14:59:48.732 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 64s ago 9h 982M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T14:59:48.732 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 64s ago 9h 2088M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T14:59:48.732 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 9m ago 9h 893M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T14:59:48.732 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 64s ago 9h 1251M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T14:59:48.733 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 64s ago 9h 24.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T14:59:48.733 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 9m ago 9h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T14:59:48.733 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 64s ago 9h 1516M 2190M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T14:59:48.733 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 64s ago 9h 1503M 2190M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T14:59:48.734 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 64s ago 9h 1168M 2190M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T14:59:48.734 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 64s ago 9h 1473M 2190M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T14:59:48.734 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 9m ago 9h 1269M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T14:59:48.735 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 9m ago 9h 1593M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T14:59:48.743 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 9m ago 9h 1424M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T14:59:48.757 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 9m ago 9h 1693M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T14:59:48.757 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 9m ago 9h 201M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T14:59:49.090 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T14:59:49.091 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T14:59:49.092 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T14:59:49.092 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:59:49.093 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T14:59:49.093 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T14:59:49.093 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:59:49.104 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T14:59:49.104 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T14:59:49.104 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:59:49.105 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T14:59:49.105 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T14:59:49.149 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T14:59:49.150 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T14:59:49.150 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T14:59:49.151 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T14:59:49.151 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T14:59:49.152 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T14:59:49.722 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:59:49 smithi083 ceph-mon[79930]: pgmap v17144: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:49.725 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:59:49 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3678883945' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:59:49.733 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:59:49 smithi083 ceph-mon[77242]: pgmap v17144: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:49.733 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:59:49 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3678883945' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:59:49.813 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:59:49 smithi153 ceph-mon[70044]: pgmap v17144: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:49.814 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:59:49 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3678883945' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T14:59:50.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:59:50 smithi083 ceph-mon[79930]: from='client.63997 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:59:50.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:59:50 smithi083 ceph-mon[79930]: from='client.54072 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:59:50.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:59:50 smithi083 ceph-mon[77242]: from='client.63997 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:59:50.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:59:50 smithi083 ceph-mon[77242]: from='client.54072 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:59:50.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:59:50 smithi153 ceph-mon[70044]: from='client.63997 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:59:50.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:59:50 smithi153 ceph-mon[70044]: from='client.54072 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T14:59:51.714 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 14:59:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:59:51] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T14:59:51.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:59:51 smithi083 ceph-mon[79930]: pgmap v17145: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:51.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:59:51 smithi083 ceph-mon[77242]: pgmap v17145: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:51.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:59:51 smithi153 ceph-mon[70044]: pgmap v17145: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:52.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:59:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:59:52.301Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:59:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 14:59:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T14:59:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T14:59:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 14:59:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:14:59:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T14:59:53.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:59:53 smithi083 ceph-mon[77242]: pgmap v17146: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:53.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:59:53 smithi083 ceph-mon[79930]: pgmap v17146: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:53.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:59:53 smithi153 ceph-mon[70044]: pgmap v17146: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:54.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:59:54 smithi083 ceph-mon[79930]: pgmap v17147: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:54.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:59:54 smithi083 ceph-mon[77242]: pgmap v17147: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:54.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:59:54 smithi153 ceph-mon[70044]: pgmap v17147: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:55.965 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:59:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:59:55.663Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:59:56.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 14:59:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T14:59:56.404Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T14:59:57.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:59:57 smithi153 ceph-mon[70044]: pgmap v17148: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:57.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:59:57 smithi083 ceph-mon[79930]: pgmap v17148: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:57.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:59:57 smithi083 ceph-mon[77242]: pgmap v17148: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:59.562 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 14:59:59 smithi153 ceph-mon[70044]: pgmap v17149: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:59.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 14:59:59 smithi083 ceph-mon[79930]: pgmap v17149: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T14:59:59.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 14:59:59 smithi083 ceph-mon[77242]: pgmap v17149: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:00.265 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:00:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[77219]: 2022-09-29T14:59:59.999+0000 7f160f665700 -1 log_channel(cluster) log [ERR] : overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T15:00:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:00:00 smithi153 ceph-mon[70044]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T15:00:00.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:00:00 smithi083 ceph-mon[79930]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T15:00:00.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:00:00 smithi083 ceph-mon[77242]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T15:00:01.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:00:01 smithi153 ceph-mon[70044]: pgmap v17150: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:01.710 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:00:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:00:01] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T15:00:01.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:00:01 smithi083 ceph-mon[79930]: pgmap v17150: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:01.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:00:01 smithi083 ceph-mon[77242]: pgmap v17150: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:02.059 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:00:01 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:00:01.569Z caller=compact.go:519 level=info component=tsdb msg="write block" mint=1664452800138 maxt=1664460000000 ulid=01GE4W5ZVB0TA9HKYCPFQETFNK duration=54.574801ms 2022-09-29T15:00:02.060 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:00:01 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:00:01.573Z caller=head.go:827 level=info component=tsdb msg="Head GC completed" duration=2.974026ms 2022-09-29T15:00:02.060 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:00:01 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:00:01.573Z caller=checkpoint.go:98 level=info component=tsdb msg="Creating checkpoint" from_segment=4 to_segment=5 mint=1664460000000 2022-09-29T15:00:02.061 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:00:01 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:00:01.683Z caller=head.go:996 level=info component=tsdb msg="WAL checkpoint complete" first=4 last=5 duration=109.707507ms 2022-09-29T15:00:02.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:00:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:00:02.302Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:00:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:00:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:00:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:00:03.313 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:00:03 smithi153 ceph-mon[70044]: pgmap v17151: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:03.314 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:00:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:00:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:00:03.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:00:03 smithi083 ceph-mon[79930]: pgmap v17151: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:03.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:00:03 smithi083 ceph-mon[77242]: pgmap v17151: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:05.561 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:00:05 smithi153 ceph-mon[70044]: pgmap v17152: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:05.670 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:00:05 smithi083 ceph-mon[79930]: pgmap v17152: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:05.671 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:00:05 smithi083 ceph-mon[77242]: pgmap v17152: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:05.966 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:00:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:00:05.664Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:00:06.718 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:00:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:00:06.405Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:00:07.569 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:00:07 smithi153 ceph-mon[70044]: pgmap v17153: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:07.721 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:00:07 smithi083 ceph-mon[79930]: pgmap v17153: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:07.722 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:00:07 smithi083 ceph-mon[77242]: pgmap v17153: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:09.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:00:09 smithi153 ceph-mon[70044]: pgmap v17154: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:09.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:00:09 smithi083 ceph-mon[79930]: pgmap v17154: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:09.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:00:09 smithi083 ceph-mon[77242]: pgmap v17154: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:11.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:00:11 smithi153 ceph-mon[70044]: pgmap v17155: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:11.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:00:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:00:11] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T15:00:11.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:00:11 smithi083 ceph-mon[79930]: pgmap v17155: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:11.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:00:11 smithi083 ceph-mon[77242]: pgmap v17155: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:12.714 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:00:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:00:12.303Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:00:12.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:00:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:00:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:00:13.282 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:00:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:00:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:00:13.561 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:00:13 smithi153 ceph-mon[70044]: pgmap v17156: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:13.710 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:00:13 smithi083 ceph-mon[77242]: pgmap v17156: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:13.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:00:13 smithi083 ceph-mon[79930]: pgmap v17156: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:15.563 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:00:15 smithi153 ceph-mon[70044]: pgmap v17157: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:15.671 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:00:15 smithi083 ceph-mon[79930]: pgmap v17157: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:15.672 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:00:15 smithi083 ceph-mon[77242]: pgmap v17157: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:15.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:00:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:00:15.665Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:00:16.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:00:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:00:16.405Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:00:17.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:00:17 smithi153 ceph-mon[70044]: pgmap v17158: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:17.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:00:17 smithi083 ceph-mon[79930]: pgmap v17158: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:17.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:00:17 smithi083 ceph-mon[77242]: pgmap v17158: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:19.414 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:00:19.574 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:00:19 smithi153 ceph-mon[70044]: pgmap v17159: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:19.699 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:00:19.700 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 95s ago 9h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:00:19.701 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 9m ago 9h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:00:19.701 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 95s ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:00:19.701 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 9m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:00:19.702 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 9m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:00:19.702 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 95s ago 9h 982M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:00:19.703 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 95s ago 9h 2088M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:00:19.703 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 9m ago 9h 893M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:00:19.704 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 95s ago 9h 1251M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:00:19.704 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 95s ago 9h 24.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:00:19.705 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 9m ago 9h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:00:19.705 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 95s ago 9h 1516M 2190M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:00:19.705 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 95s ago 9h 1503M 2190M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:00:19.723 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 95s ago 9h 1168M 2190M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:00:19.724 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 95s ago 9h 1473M 2190M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:00:19.724 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 9m ago 9h 1269M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:00:19.724 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 9m ago 9h 1593M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:00:19.725 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 9m ago 9h 1424M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:00:19.725 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 9m ago 9h 1693M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:00:19.726 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 9m ago 9h 201M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:00:19.735 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:00:19 smithi083 ceph-mon[79930]: pgmap v17159: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:19.736 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:00:19 smithi083 ceph-mon[77242]: pgmap v17159: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:20.052 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:00:20.053 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:00:20.054 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:00:20.054 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:00:20.054 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:00:20.055 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:00:20.055 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:00:20.056 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:00:20.056 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:00:20.056 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:00:20.057 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:00:20.057 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:00:20.058 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:00:20.082 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:00:20.083 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:00:20.083 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:00:20.083 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:00:20.084 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:00:20.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:00:20 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1799009552' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:00:20.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:00:20 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1799009552' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:00:20.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:00:20 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1799009552' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:00:21.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:00:21 smithi153 ceph-mon[70044]: from='client.64015 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:00:21.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:00:21 smithi153 ceph-mon[70044]: from='client.64021 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:00:21.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:00:21 smithi153 ceph-mon[70044]: pgmap v17160: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:21.710 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:00:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:00:21] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T15:00:21.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:00:21 smithi083 ceph-mon[77242]: from='client.64015 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:00:21.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:00:21 smithi083 ceph-mon[77242]: from='client.64021 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:00:21.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:00:21 smithi083 ceph-mon[77242]: pgmap v17160: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:21.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:00:21 smithi083 ceph-mon[79930]: from='client.64015 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:00:21.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:00:21 smithi083 ceph-mon[79930]: from='client.64021 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:00:21.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:00:21 smithi083 ceph-mon[79930]: pgmap v17160: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:22.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:00:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:00:22.304Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:00:22.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:00:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:00:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:00:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:00:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:00:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:00:23.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:00:23 smithi153 ceph-mon[70044]: pgmap v17161: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:23.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:00:23 smithi083 ceph-mon[79930]: pgmap v17161: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:23.735 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:00:23 smithi083 ceph-mon[77242]: pgmap v17161: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:25.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:00:25 smithi153 ceph-mon[70044]: pgmap v17162: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:25.666 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:00:25 smithi083 ceph-mon[79930]: pgmap v17162: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:25.667 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:00:25 smithi083 ceph-mon[77242]: pgmap v17162: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:25.967 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:00:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:00:25.665Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:00:26.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:00:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:00:26.406Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:00:27.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:00:27 smithi083 ceph-mon[77242]: pgmap v17163: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:27.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:00:27 smithi083 ceph-mon[79930]: pgmap v17163: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:27.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:00:27 smithi153 ceph-mon[70044]: pgmap v17163: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:29.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:00:29 smithi083 ceph-mon[77242]: pgmap v17164: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:29.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:00:29 smithi083 ceph-mon[79930]: pgmap v17164: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:29.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:00:29 smithi153 ceph-mon[70044]: pgmap v17164: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:31.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:00:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:00:31] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T15:00:31.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:00:31 smithi083 ceph-mon[79930]: pgmap v17165: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:31.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:00:31 smithi083 ceph-mon[77242]: pgmap v17165: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:31.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:00:31 smithi153 ceph-mon[70044]: pgmap v17165: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:32.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:00:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:00:32.304Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:00:32.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:00:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:00:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:00:33.310 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:00:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:00:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:00:33.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:00:33 smithi083 ceph-mon[79930]: pgmap v17166: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:33.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:00:33 smithi083 ceph-mon[77242]: pgmap v17166: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:33.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:00:33 smithi153 ceph-mon[70044]: pgmap v17166: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:35.671 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:00:35 smithi083 ceph-mon[79930]: pgmap v17167: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:35.672 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:00:35 smithi083 ceph-mon[77242]: pgmap v17167: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:35.815 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:00:35 smithi153 ceph-mon[70044]: pgmap v17167: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:35.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:00:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:00:35.666Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:00:36.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:00:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:00:36.407Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:00:37.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:00:37 smithi083 ceph-mon[79930]: pgmap v17168: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:37.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:00:37 smithi083 ceph-mon[77242]: pgmap v17168: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:37.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:00:37 smithi153 ceph-mon[70044]: pgmap v17168: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:39.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:00:39 smithi083 ceph-mon[79930]: pgmap v17169: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:39.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:00:39 smithi083 ceph-mon[77242]: pgmap v17169: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:39.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:00:39 smithi153 ceph-mon[70044]: pgmap v17169: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:41.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:00:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:00:41] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T15:00:41.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:00:41 smithi083 ceph-mon[79930]: pgmap v17170: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:41.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:00:41 smithi083 ceph-mon[77242]: pgmap v17170: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:41.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:00:41 smithi153 ceph-mon[70044]: pgmap v17170: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:42.722 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:00:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:00:42.723 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:00:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:00:42.724 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:00:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:00:42.724 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:00:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:00:42.725 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:00:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:00:42.305Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:00:42.738 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:00:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:00:42.738 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:00:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:00:42.739 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:00:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:00:42.739 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:00:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:00:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:00:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:00:42.317Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:00:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:00:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:00:42.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:00:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:00:42.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:00:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:00:42.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:00:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:00:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:00:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:00:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:00:43.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:00:43 smithi083 ceph-mon[79930]: pgmap v17171: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:43.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:00:43 smithi083 ceph-mon[77242]: pgmap v17171: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:43.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:00:43 smithi153 ceph-mon[70044]: pgmap v17171: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:45.669 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:00:45 smithi083 ceph-mon[79930]: pgmap v17172: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:45.670 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:00:45 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:00:45.670 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:00:45 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:00:45.670 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:00:45 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:00:45.671 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:00:45 smithi083 ceph-mon[77242]: pgmap v17172: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:45.671 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:00:45 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:00:45.671 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:00:45 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:00:45.672 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:00:45 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:00:45.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:00:45 smithi153 ceph-mon[70044]: pgmap v17172: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:45.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:00:45 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:00:45.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:00:45 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:00:45.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:00:45 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:00:45.967 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:00:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:00:45.667Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:00:46.715 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:00:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:00:46.407Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:00:47.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:00:47 smithi083 ceph-mon[77242]: pgmap v17173: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:47.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:00:47 smithi083 ceph-mon[79930]: pgmap v17173: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:47.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:00:47 smithi153 ceph-mon[70044]: pgmap v17173: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:49.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:00:49 smithi083 ceph-mon[79930]: pgmap v17174: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:49.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:00:49 smithi083 ceph-mon[77242]: pgmap v17174: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:49.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:00:49 smithi153 ceph-mon[70044]: pgmap v17174: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:50.381 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:00:50.669 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:00:50.670 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 2m ago 9h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:00:50.670 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 0s ago 9h 63.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:00:50.671 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 2m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:00:50.671 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 0s ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:00:50.672 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 0s ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:00:50.672 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 2m ago 9h 982M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:00:50.673 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 2m ago 9h 2088M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:00:50.673 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 0s ago 9h 909M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:00:50.674 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 2m ago 9h 1251M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:00:50.674 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 2m ago 9h 24.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:00:50.675 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 0s ago 9h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:00:50.676 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 2m ago 9h 1516M 2190M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:00:50.676 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 2m ago 9h 1503M 2190M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:00:50.677 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 2m ago 9h 1168M 2190M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:00:50.677 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 2m ago 9h 1473M 2190M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:00:50.677 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 0s ago 9h 1292M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:00:50.678 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 0s ago 9h 1622M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:00:50.678 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 0s ago 9h 1449M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:00:50.679 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 0s ago 9h 1724M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:00:50.679 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 0s ago 9h 173M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:00:51.024 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:00:51.025 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:00:51.025 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:00:51.026 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:00:51.026 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:00:51.026 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:00:51.027 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:00:51.027 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:00:51.028 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:00:51.028 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:00:51.029 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:00:51.029 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:00:51.029 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:00:51.055 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:00:51.055 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:00:51.056 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:00:51.056 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:00:51.057 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:00:51.079 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:00:50 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:00:51.079 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:00:50 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:00:51.080 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:00:50 smithi153 ceph-mon[70044]: pgmap v17175: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:51.080 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:00:50 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:00:51.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:00:50 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:00:51.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:00:50 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:00:51.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:00:50 smithi083 ceph-mon[77242]: pgmap v17175: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:51.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:00:50 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:00:51.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:00:50 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:00:51.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:00:50 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:00:51.228 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:00:50 smithi083 ceph-mon[79930]: pgmap v17175: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:51.228 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:00:50 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:00:51.803 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:00:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:00:51] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T15:00:52.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:00:51 smithi153 ceph-mon[70044]: from='client.54102 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:00:52.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:00:51 smithi153 ceph-mon[70044]: from='client.64039 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:00:52.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:00:51 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/678407875' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:00:52.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:00:51 smithi083 ceph-mon[77242]: from='client.54102 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:00:52.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:00:51 smithi083 ceph-mon[77242]: from='client.64039 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:00:52.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:00:51 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/678407875' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:00:52.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:00:51 smithi083 ceph-mon[79930]: from='client.54102 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:00:52.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:00:51 smithi083 ceph-mon[79930]: from='client.64039 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:00:52.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:00:51 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/678407875' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:00:52.715 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:00:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:00:52.305Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:00:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:00:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:00:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:00:53.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:00:53 smithi153 ceph-mon[70044]: pgmap v17176: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:00:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:00:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:00:53.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:00:53 smithi083 ceph-mon[79930]: pgmap v17176: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:53.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:00:53 smithi083 ceph-mon[77242]: pgmap v17176: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:55.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:00:55 smithi153 ceph-mon[70044]: pgmap v17177: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:55.671 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:00:55 smithi083 ceph-mon[79930]: pgmap v17177: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:55.672 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:00:55 smithi083 ceph-mon[77242]: pgmap v17177: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:55.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:00:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:00:55.667Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:00:56.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:00:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:00:56.408Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:00:57.562 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:00:57 smithi153 ceph-mon[70044]: pgmap v17178: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:57.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:00:57 smithi083 ceph-mon[79930]: pgmap v17178: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:57.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:00:57 smithi083 ceph-mon[77242]: pgmap v17178: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:59.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:00:59 smithi153 ceph-mon[70044]: pgmap v17179: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:59.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:00:59 smithi083 ceph-mon[79930]: pgmap v17179: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:00:59.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:00:59 smithi083 ceph-mon[77242]: pgmap v17179: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:01.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:01:01 smithi153 ceph-mon[70044]: pgmap v17180: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:01.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:01:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:01:01] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T15:01:01.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:01:01 smithi083 ceph-mon[79930]: pgmap v17180: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:01.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:01:01 smithi083 ceph-mon[77242]: pgmap v17180: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:02.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:01:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:01:02.306Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:01:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:01:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:01:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:01:03.286 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:01:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:01:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:01:03.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:01:03 smithi153 ceph-mon[70044]: pgmap v17181: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:03.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:01:03 smithi083 ceph-mon[79930]: pgmap v17181: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:03.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:01:03 smithi083 ceph-mon[77242]: pgmap v17181: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:05.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:01:05 smithi153 ceph-mon[70044]: pgmap v17182: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:05.670 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:01:05 smithi083 ceph-mon[79930]: pgmap v17182: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:05.671 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:01:05 smithi083 ceph-mon[77242]: pgmap v17182: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:05.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:01:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:01:05.668Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:01:06.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:01:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:01:06.408Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:01:07.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:01:07 smithi153 ceph-mon[70044]: pgmap v17183: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:07.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:01:07 smithi083 ceph-mon[79930]: pgmap v17183: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:07.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:01:07 smithi083 ceph-mon[77242]: pgmap v17183: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:09.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:01:09 smithi153 ceph-mon[70044]: pgmap v17184: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:09.710 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:01:09 smithi083 ceph-mon[77242]: pgmap v17184: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:09.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:01:09 smithi083 ceph-mon[79930]: pgmap v17184: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:11.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:01:11 smithi153 ceph-mon[70044]: pgmap v17185: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:11.710 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:01:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:01:11] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T15:01:11.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:01:11 smithi083 ceph-mon[79930]: pgmap v17185: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:11.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:01:11 smithi083 ceph-mon[77242]: pgmap v17185: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:12.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:01:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:01:12.307Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:01:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:01:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:01:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:01:13.298 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:01:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:01:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:01:13.562 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:01:13 smithi153 ceph-mon[70044]: pgmap v17186: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:13.710 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:01:13 smithi083 ceph-mon[77242]: pgmap v17186: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:13.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:01:13 smithi083 ceph-mon[79930]: pgmap v17186: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:15.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:01:15 smithi153 ceph-mon[70044]: pgmap v17187: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:15.669 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:01:15 smithi083 ceph-mon[79930]: pgmap v17187: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:15.670 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:01:15 smithi083 ceph-mon[77242]: pgmap v17187: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:15.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:01:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:01:15.669Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:01:16.718 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:01:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:01:16.409Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:01:17.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:01:17 smithi153 ceph-mon[70044]: pgmap v17188: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:17.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:01:17 smithi083 ceph-mon[79930]: pgmap v17188: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:17.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:01:17 smithi083 ceph-mon[77242]: pgmap v17188: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:19.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:01:19 smithi083 ceph-mon[79930]: pgmap v17189: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:19.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:01:19 smithi083 ceph-mon[77242]: pgmap v17189: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:19.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:01:19 smithi153 ceph-mon[70044]: pgmap v17189: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:21.354 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:01:21.641 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:01:21.641 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 2m ago 9h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:01:21.642 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 31s ago 9h 63.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:01:21.642 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 2m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:01:21.643 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 31s ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:01:21.643 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 31s ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:01:21.643 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 2m ago 9h 982M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:01:21.644 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 2m ago 9h 2088M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:01:21.644 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 31s ago 9h 909M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:01:21.644 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 2m ago 9h 1251M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:01:21.645 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 2m ago 9h 24.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:01:21.645 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 31s ago 9h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:01:21.645 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 2m ago 9h 1516M 2190M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:01:21.650 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 2m ago 9h 1503M 2190M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:01:21.651 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 2m ago 9h 1168M 2190M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:01:21.651 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 2m ago 9h 1473M 2190M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:01:21.652 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 31s ago 9h 1292M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:01:21.652 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 31s ago 9h 1622M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:01:21.653 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 31s ago 9h 1449M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:01:21.653 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 31s ago 9h 1724M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:01:21.671 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 31s ago 9h 173M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:01:21.710 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:01:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:01:21] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T15:01:21.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:01:21 smithi083 ceph-mon[79930]: pgmap v17190: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:21.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:01:21 smithi083 ceph-mon[77242]: pgmap v17190: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:21.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:01:21 smithi153 ceph-mon[70044]: pgmap v17190: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:21.994 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:01:21.995 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:01:21.995 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:01:21.996 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:01:22.002 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:01:22.003 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:01:22.003 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:01:22.015 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:01:22.016 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:01:22.016 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:01:22.017 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:01:22.017 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:01:22.030 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:01:22.030 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:01:22.030 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:01:22.045 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:01:22.046 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:01:22.054 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:01:22.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:01:22 smithi083 ceph-mon[79930]: from='client.54120 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:01:22.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:01:22 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1629254073' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:01:22.725 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:01:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:01:22.306Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:01:22.726 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:01:22 smithi083 ceph-mon[77242]: from='client.54120 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:01:22.726 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:01:22 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1629254073' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:01:22.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:01:22 smithi153 ceph-mon[70044]: from='client.54120 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:01:22.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:01:22 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1629254073' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:01:22.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:01:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:01:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:01:23.311 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:01:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:01:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:01:23.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:01:23 smithi083 ceph-mon[79930]: from='client.64057 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:01:23.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:01:23 smithi083 ceph-mon[79930]: pgmap v17191: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:23.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:01:23 smithi083 ceph-mon[77242]: from='client.64057 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:01:23.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:01:23 smithi083 ceph-mon[77242]: pgmap v17191: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:23.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:01:23 smithi153 ceph-mon[70044]: from='client.64057 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:01:23.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:01:23 smithi153 ceph-mon[70044]: pgmap v17191: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:25.678 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:01:25 smithi083 ceph-mon[79930]: pgmap v17192: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:25.679 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:01:25 smithi083 ceph-mon[77242]: pgmap v17192: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:25.813 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:01:25 smithi153 ceph-mon[70044]: pgmap v17192: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:25.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:01:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:01:25.669Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:01:26.715 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:01:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:01:26.409Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:01:27.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:01:27 smithi083 ceph-mon[79930]: pgmap v17193: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:27.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:01:27 smithi083 ceph-mon[77242]: pgmap v17193: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:27.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:01:27 smithi153 ceph-mon[70044]: pgmap v17193: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:29.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:01:29 smithi083 ceph-mon[77242]: pgmap v17194: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:29.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:01:29 smithi083 ceph-mon[79930]: pgmap v17194: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:29.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:01:29 smithi153 ceph-mon[70044]: pgmap v17194: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:31.710 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:01:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:01:31] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T15:01:31.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:01:31 smithi083 ceph-mon[79930]: pgmap v17195: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:31.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:01:31 smithi083 ceph-mon[77242]: pgmap v17195: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:31.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:01:31 smithi153 ceph-mon[70044]: pgmap v17195: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:32.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:01:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:01:32.308Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:01:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:01:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:01:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:01:33.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:01:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:01:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:01:33.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:01:33 smithi083 ceph-mon[79930]: pgmap v17196: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:33.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:01:33 smithi083 ceph-mon[77242]: pgmap v17196: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:33.814 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:01:33 smithi153 ceph-mon[70044]: pgmap v17196: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:35.670 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:01:35 smithi083 ceph-mon[79930]: pgmap v17197: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:35.671 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:01:35 smithi083 ceph-mon[77242]: pgmap v17197: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:35.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:01:35 smithi153 ceph-mon[70044]: pgmap v17197: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:35.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:01:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:01:35.670Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:01:36.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:01:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:01:36.410Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:01:37.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:01:37 smithi083 ceph-mon[79930]: pgmap v17198: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:37.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:01:37 smithi083 ceph-mon[77242]: pgmap v17198: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:37.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:01:37 smithi153 ceph-mon[70044]: pgmap v17198: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:39.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:01:39 smithi083 ceph-mon[77242]: pgmap v17199: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:39.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:01:39 smithi083 ceph-mon[79930]: pgmap v17199: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:39.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:01:39 smithi153 ceph-mon[70044]: pgmap v17199: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:41.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:01:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:01:41] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T15:01:41.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:01:41 smithi083 ceph-mon[79930]: pgmap v17200: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:41.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:01:41 smithi083 ceph-mon[77242]: pgmap v17200: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:41.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:01:41 smithi153 ceph-mon[70044]: pgmap v17200: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:42.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:01:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:01:42.308Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:01:42.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:01:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:01:42.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:01:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:01:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:01:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:01:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:01:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:01:42.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:01:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:01:42.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:01:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:01:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:01:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:01:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:01:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:01:42.810 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:01:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:01:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:01:42.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:01:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:01:42.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:01:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:01:42.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:01:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:01:42.813 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:01:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:01:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:01:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:01:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:01:43.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:01:43 smithi083 ceph-mon[79930]: pgmap v17201: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:43.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:01:43 smithi083 ceph-mon[77242]: pgmap v17201: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:43.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:01:43 smithi153 ceph-mon[70044]: pgmap v17201: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:45.671 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:01:45 smithi083 ceph-mon[79930]: pgmap v17202: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:45.672 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:01:45 smithi083 ceph-mon[77242]: pgmap v17202: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:45.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:01:45 smithi153 ceph-mon[70044]: pgmap v17202: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:45.978 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:01:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:01:45.670Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:01:46.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:01:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:01:46.411Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:01:47.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:01:47 smithi083 ceph-mon[79930]: pgmap v17203: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:47.721 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:01:47 smithi083 ceph-mon[77242]: pgmap v17203: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:47.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:01:47 smithi153 ceph-mon[70044]: pgmap v17203: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:49.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:01:49 smithi083 ceph-mon[79930]: pgmap v17204: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:49.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:01:49 smithi083 ceph-mon[77242]: pgmap v17204: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:49.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:01:49 smithi153 ceph-mon[70044]: pgmap v17204: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:51.710 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:01:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:01:51] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T15:01:51.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:01:51 smithi083 ceph-mon[77242]: pgmap v17205: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:51.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:01:51 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:01:51.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:01:51 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:01:51.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:01:51 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:01:51.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:01:51 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:01:51.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:01:51 smithi083 ceph-mon[79930]: pgmap v17205: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:51.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:01:51 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:01:51.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:01:51 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:01:51.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:01:51 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:01:51.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:01:51 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:01:51.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:01:51 smithi153 ceph-mon[70044]: pgmap v17205: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:51.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:01:51 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:01:51.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:01:51 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:01:51.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:01:51 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:01:51.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:01:51 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:01:52.317 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:01:52.611 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:01:52.611 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 3m ago 9h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:01:52.612 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 62s ago 9h 63.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:01:52.612 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 3m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:01:52.613 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 62s ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:01:52.613 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 62s ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:01:52.613 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 3m ago 9h 982M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:01:52.614 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 3m ago 9h 2088M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:01:52.614 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 62s ago 9h 909M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:01:52.615 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 3m ago 9h 1251M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:01:52.615 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 3m ago 9h 24.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:01:52.616 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 62s ago 9h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:01:52.616 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 3m ago 9h 1516M 2190M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:01:52.616 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 3m ago 9h 1503M 2190M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:01:52.617 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 3m ago 9h 1168M 2190M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:01:52.617 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 3m ago 9h 1473M 2190M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:01:52.618 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 62s ago 9h 1292M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:01:52.618 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 62s ago 9h 1622M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:01:52.619 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 62s ago 9h 1449M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:01:52.619 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 62s ago 9h 1724M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:01:52.619 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 62s ago 9h 173M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:01:52.718 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:01:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:01:52.308Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:01:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:01:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:01:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:01:52.967 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:01:52.968 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:01:52.968 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:01:52.969 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:01:52.978 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:01:52.978 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:01:52.979 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:01:52.979 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:01:52.980 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:01:52.981 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:01:52.982 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:01:52.982 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:01:52.983 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:01:52.983 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:01:52.984 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:01:52.984 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:01:52.985 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:01:52.985 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:01:53.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:01:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:01:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:01:53.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:01:53 smithi083 ceph-mon[79930]: pgmap v17206: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:53.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:01:53 smithi083 ceph-mon[79930]: from='client.54138 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:01:53.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:01:53 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/362243122' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:01:53.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:01:53 smithi083 ceph-mon[77242]: pgmap v17206: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:53.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:01:53 smithi083 ceph-mon[77242]: from='client.54138 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:01:53.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:01:53 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/362243122' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:01:53.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:01:53 smithi153 ceph-mon[70044]: pgmap v17206: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:53.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:01:53 smithi153 ceph-mon[70044]: from='client.54138 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:01:53.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:01:53 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/362243122' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:01:54.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:01:54 smithi083 ceph-mon[79930]: from='client.54144 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:01:54.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:01:54 smithi083 ceph-mon[79930]: pgmap v17207: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:54.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:01:54 smithi083 ceph-mon[77242]: from='client.54144 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:01:54.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:01:54 smithi083 ceph-mon[77242]: pgmap v17207: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:54.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:01:54 smithi153 ceph-mon[70044]: from='client.54144 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:01:54.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:01:54 smithi153 ceph-mon[70044]: pgmap v17207: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:55.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:01:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:01:55.671Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:01:56.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:01:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:01:56.412Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:01:57.563 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:01:57 smithi153 ceph-mon[70044]: pgmap v17208: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:57.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:01:57 smithi083 ceph-mon[79930]: pgmap v17208: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:57.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:01:57 smithi083 ceph-mon[77242]: pgmap v17208: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:59.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:01:59 smithi153 ceph-mon[70044]: pgmap v17209: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:59.710 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:01:59 smithi083 ceph-mon[77242]: pgmap v17209: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:01:59.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:01:59 smithi083 ceph-mon[79930]: pgmap v17209: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:01.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:02:01 smithi153 ceph-mon[70044]: pgmap v17210: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:01.710 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:02:01 smithi083 ceph-mon[77242]: pgmap v17210: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:01.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:02:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:02:01] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T15:02:01.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:02:01 smithi083 ceph-mon[79930]: pgmap v17210: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:02.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:02:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:02:02.309Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:02:02.818 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:02:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:02:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:02:03.294 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:02:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:02:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:02:03.563 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:02:03 smithi153 ceph-mon[70044]: pgmap v17211: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:03.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:02:03 smithi083 ceph-mon[79930]: pgmap v17211: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:03.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:02:03 smithi083 ceph-mon[77242]: pgmap v17211: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:05.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:02:05 smithi153 ceph-mon[70044]: pgmap v17212: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:05.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:02:05 smithi083 ceph-mon[79930]: pgmap v17212: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:05.677 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:02:05 smithi083 ceph-mon[77242]: pgmap v17212: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:05.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:02:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:02:05.672Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:02:06.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:02:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:02:06.413Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:02:07.563 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:02:07 smithi153 ceph-mon[70044]: pgmap v17213: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:07.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:02:07 smithi083 ceph-mon[79930]: pgmap v17213: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:07.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:02:07 smithi083 ceph-mon[77242]: pgmap v17213: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:09.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:02:09 smithi153 ceph-mon[70044]: pgmap v17214: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:09.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:02:09 smithi083 ceph-mon[79930]: pgmap v17214: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:09.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:02:09 smithi083 ceph-mon[77242]: pgmap v17214: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:11.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:02:11 smithi153 ceph-mon[70044]: pgmap v17215: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:11.718 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:02:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:02:11] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T15:02:11.719 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:02:11 smithi083 ceph-mon[79930]: pgmap v17215: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:11.720 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:02:11 smithi083 ceph-mon[77242]: pgmap v17215: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:12.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:02:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:02:12.310Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:02:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:02:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:02:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:02:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:02:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:02:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:02:13.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:02:13 smithi153 ceph-mon[70044]: pgmap v17216: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:13.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:02:13 smithi083 ceph-mon[79930]: pgmap v17216: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:13.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:02:13 smithi083 ceph-mon[77242]: pgmap v17216: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:15.674 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:02:15 smithi083 ceph-mon[79930]: pgmap v17217: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:15.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:02:15 smithi083 ceph-mon[77242]: pgmap v17217: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:15.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:02:15 smithi153 ceph-mon[70044]: pgmap v17217: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:15.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:02:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:02:15.673Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:02:16.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:02:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:02:16.414Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:02:17.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:02:17 smithi083 ceph-mon[79930]: pgmap v17218: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:17.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:02:17 smithi083 ceph-mon[77242]: pgmap v17218: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:17.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:02:17 smithi153 ceph-mon[70044]: pgmap v17218: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:19.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:02:19 smithi083 ceph-mon[79930]: pgmap v17219: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:19.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:02:19 smithi083 ceph-mon[77242]: pgmap v17219: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:19.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:02:19 smithi153 ceph-mon[70044]: pgmap v17219: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:21.710 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:02:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:02:21] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T15:02:21.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:02:21 smithi083 ceph-mon[79930]: pgmap v17220: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:21.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:02:21 smithi083 ceph-mon[77242]: pgmap v17220: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:21.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:02:21 smithi153 ceph-mon[70044]: pgmap v17220: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:22.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:02:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:02:22.310Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:02:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:02:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:02:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:02:23.298 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:02:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:02:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:02:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:02:23.598 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:02:23.599 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 3m ago 9h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:02:23.600 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 93s ago 9h 63.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:02:23.600 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 3m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:02:23.600 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 93s ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:02:23.601 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 93s ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:02:23.601 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 3m ago 9h 982M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:02:23.605 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 3m ago 9h 2088M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:02:23.605 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 93s ago 9h 909M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:02:23.606 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 3m ago 9h 1251M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:02:23.606 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 3m ago 9h 24.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:02:23.607 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 93s ago 9h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:02:23.607 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 3m ago 9h 1516M 2190M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:02:23.608 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 3m ago 9h 1503M 2190M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:02:23.608 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 3m ago 9h 1168M 2190M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:02:23.609 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 3m ago 9h 1473M 2190M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:02:23.610 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 93s ago 9h 1292M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:02:23.610 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 93s ago 9h 1622M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:02:23.613 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 93s ago 9h 1449M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:02:23.614 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 93s ago 9h 1724M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:02:23.615 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 93s ago 9h 173M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:02:23.722 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:02:23 smithi083 ceph-mon[79930]: pgmap v17221: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:23.723 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:02:23 smithi083 ceph-mon[77242]: pgmap v17221: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:23.814 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:02:23 smithi153 ceph-mon[70044]: pgmap v17221: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:23.960 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:02:23.961 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:02:23.961 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:02:23.961 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:02:23.961 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:02:23.962 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:02:23.962 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:02:23.963 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:02:23.963 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:02:23.963 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:02:23.963 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:02:23.964 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:02:23.964 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:02:23.964 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:02:23.964 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:02:23.990 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:02:23.990 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:02:23.990 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:02:24.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:02:24 smithi083 ceph-mon[77242]: from='client.54156 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:02:24.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:02:24 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/740788612' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:02:24.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:02:24 smithi083 ceph-mon[79930]: from='client.54156 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:02:24.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:02:24 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/740788612' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:02:24.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:02:24 smithi153 ceph-mon[70044]: from='client.54156 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:02:24.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:02:24 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/740788612' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:02:25.673 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:02:25 smithi083 ceph-mon[79930]: from='client.54162 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:02:25.674 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:02:25 smithi083 ceph-mon[79930]: pgmap v17222: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:25.674 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:02:25 smithi083 ceph-mon[77242]: from='client.54162 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:02:25.674 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:02:25 smithi083 ceph-mon[77242]: pgmap v17222: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:25.815 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:02:25 smithi153 ceph-mon[70044]: from='client.54162 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:02:25.816 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:02:25 smithi153 ceph-mon[70044]: pgmap v17222: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:25.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:02:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:02:25.673Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:02:26.717 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:02:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:02:26.414Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:02:27.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:02:27 smithi083 ceph-mon[79930]: pgmap v17223: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:27.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:02:27 smithi083 ceph-mon[77242]: pgmap v17223: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:27.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:02:27 smithi153 ceph-mon[70044]: pgmap v17223: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:29.718 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:02:29 smithi083 ceph-mon[79930]: pgmap v17224: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:29.719 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:02:29 smithi083 ceph-mon[77242]: pgmap v17224: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:29.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:02:29 smithi153 ceph-mon[70044]: pgmap v17224: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:31.710 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:02:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:02:31] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T15:02:31.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:02:31 smithi083 ceph-mon[79930]: pgmap v17225: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:31.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:02:31 smithi083 ceph-mon[77242]: pgmap v17225: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:31.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:02:31 smithi153 ceph-mon[70044]: pgmap v17225: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:32.714 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:02:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:02:32.310Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:02:32.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:02:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:02:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:02:33.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:02:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:02:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:02:33.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:02:33 smithi083 ceph-mon[79930]: pgmap v17226: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:33.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:02:33 smithi083 ceph-mon[77242]: pgmap v17226: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:33.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:02:33 smithi153 ceph-mon[70044]: pgmap v17226: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:35.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:02:35 smithi083 ceph-mon[79930]: pgmap v17227: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:35.677 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:02:35 smithi083 ceph-mon[77242]: pgmap v17227: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:35.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:02:35 smithi153 ceph-mon[70044]: pgmap v17227: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:35.966 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:02:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:02:35.674Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:02:36.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:02:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:02:36.415Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:02:37.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:02:37 smithi083 ceph-mon[79930]: pgmap v17228: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:37.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:02:37 smithi083 ceph-mon[77242]: pgmap v17228: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:37.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:02:37 smithi153 ceph-mon[70044]: pgmap v17228: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:39.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:02:39 smithi083 ceph-mon[79930]: pgmap v17229: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:39.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:02:39 smithi083 ceph-mon[77242]: pgmap v17229: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:39.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:02:39 smithi153 ceph-mon[70044]: pgmap v17229: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:41.710 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:02:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:02:41] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T15:02:41.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:02:41 smithi083 ceph-mon[79930]: pgmap v17230: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:41.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:02:41 smithi083 ceph-mon[77242]: pgmap v17230: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:41.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:02:41 smithi153 ceph-mon[70044]: pgmap v17230: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:42.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:02:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:02:42.311Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:02:42.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:02:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:02:42.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:02:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:02:42.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:02:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:02:42.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:02:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:02:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:02:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:02:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:02:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:02:42.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:02:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:02:42.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:02:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:02:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:02:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:02:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:02:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:02:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:02:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:02:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:02:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:02:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:02:42.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:02:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:02:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:02:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:02:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:02:43.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:02:43 smithi083 ceph-mon[77242]: pgmap v17231: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:43.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:02:43 smithi083 ceph-mon[79930]: pgmap v17231: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:43.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:02:43 smithi153 ceph-mon[70044]: pgmap v17231: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:45.674 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:02:45 smithi083 ceph-mon[79930]: pgmap v17232: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:45.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:02:45 smithi083 ceph-mon[77242]: pgmap v17232: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:45.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:02:45 smithi153 ceph-mon[70044]: pgmap v17232: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:45.965 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:02:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:02:45.674Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:02:46.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:02:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:02:46.415Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:02:47.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:02:47 smithi083 ceph-mon[77242]: pgmap v17233: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:47.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:02:47 smithi083 ceph-mon[79930]: pgmap v17233: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:47.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:02:47 smithi153 ceph-mon[70044]: pgmap v17233: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:49.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:02:49 smithi083 ceph-mon[79930]: pgmap v17234: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:49.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:02:49 smithi083 ceph-mon[77242]: pgmap v17234: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:49.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:02:49 smithi153 ceph-mon[70044]: pgmap v17234: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:51.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:02:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:02:51] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T15:02:51.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:02:51 smithi083 ceph-mon[77242]: pgmap v17235: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:51.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:02:51 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:02:51.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:02:51 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:02:51.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:02:51 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:02:51.718 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:02:51 smithi083 ceph-mon[79930]: pgmap v17235: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:51.719 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:02:51 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:02:51.719 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:02:51 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:02:51.720 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:02:51 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:02:51.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:02:51 smithi153 ceph-mon[70044]: pgmap v17235: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:51.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:02:51 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:02:51.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:02:51 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:02:51.813 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:02:51 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:02:52.633 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:02:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:02:52.311Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:02:52.634 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:02:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:02:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:02:52.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:02:52 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:02:52.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:02:52 smithi083 ceph-mon[77242]: pgmap v17236: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:52.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:02:52 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:02:52.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:02:52 smithi083 ceph-mon[79930]: pgmap v17236: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:53.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:02:52 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:02:53.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:02:52 smithi153 ceph-mon[70044]: pgmap v17236: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:02:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:02:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:02:54.287 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:02:54.584 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:02:54.585 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 4m ago 9h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:02:54.585 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 2m ago 9h 63.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:02:54.586 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 4m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:02:54.586 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 2m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:02:54.587 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 2m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:02:54.587 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 4m ago 9h 982M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:02:54.588 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 4m ago 9h 2088M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:02:54.596 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 2m ago 9h 909M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:02:54.596 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 4m ago 9h 1251M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:02:54.597 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 4m ago 9h 24.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:02:54.597 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 2m ago 9h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:02:54.598 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 4m ago 9h 1516M 2190M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:02:54.598 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 4m ago 9h 1503M 2190M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:02:54.599 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 4m ago 9h 1168M 2190M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:02:54.599 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 4m ago 9h 1473M 2190M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:02:54.600 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 2m ago 9h 1292M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:02:54.600 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 2m ago 9h 1622M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:02:54.601 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 2m ago 9h 1449M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:02:54.601 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 2m ago 9h 1724M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:02:54.602 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 2m ago 9h 173M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:02:54.934 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:02:54.935 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:02:54.936 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:02:54.936 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:02:54.937 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:02:54.937 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:02:54.938 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:02:54.938 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:02:54.939 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:02:54.939 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:02:54.939 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:02:54.940 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:02:54.940 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:02:54.941 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:02:54.941 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:02:54.941 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:02:54.947 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:02:54.947 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:02:55.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:02:55 smithi153 ceph-mon[70044]: pgmap v17237: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:55.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:02:55 smithi153 ceph-mon[70044]: from='client.64105 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:02:55.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:02:55 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/449806908' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:02:55.674 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:02:55 smithi083 ceph-mon[79930]: pgmap v17237: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:55.675 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:02:55 smithi083 ceph-mon[79930]: from='client.64105 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:02:55.675 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:02:55 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/449806908' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:02:55.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:02:55 smithi083 ceph-mon[77242]: pgmap v17237: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:55.677 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:02:55 smithi083 ceph-mon[77242]: from='client.64105 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:02:55.677 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:02:55 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/449806908' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:02:55.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:02:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:02:55.674Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:02:56.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:02:56 smithi153 ceph-mon[70044]: from='client.64111 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:02:56.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:02:56 smithi083 ceph-mon[79930]: from='client.64111 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:02:56.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:02:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:02:56.416Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:02:56.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:02:56 smithi083 ceph-mon[77242]: from='client.64111 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:02:57.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:02:57 smithi153 ceph-mon[70044]: pgmap v17238: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:57.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:02:57 smithi083 ceph-mon[79930]: pgmap v17238: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:57.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:02:57 smithi083 ceph-mon[77242]: pgmap v17238: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:59.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:02:59 smithi153 ceph-mon[70044]: pgmap v17239: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:59.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:02:59 smithi083 ceph-mon[79930]: pgmap v17239: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:02:59.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:02:59 smithi083 ceph-mon[77242]: pgmap v17239: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:01.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:03:01 smithi153 ceph-mon[70044]: pgmap v17240: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:01.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:03:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:03:01] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T15:03:01.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:03:01 smithi083 ceph-mon[79930]: pgmap v17240: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:01.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:03:01 smithi083 ceph-mon[77242]: pgmap v17240: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:02.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:03:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:03:02.313Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:03:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:03:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:03:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:03:03.310 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:03:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:03:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:03:03.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:03:03 smithi083 ceph-mon[79930]: pgmap v17241: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:03.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:03:03 smithi083 ceph-mon[77242]: pgmap v17241: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:03.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:03:03 smithi153 ceph-mon[70044]: pgmap v17241: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:05.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:03:05 smithi083 ceph-mon[79930]: pgmap v17242: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:05.678 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:03:05 smithi083 ceph-mon[77242]: pgmap v17242: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:05.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:03:05 smithi153 ceph-mon[70044]: pgmap v17242: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:05.965 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:03:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:03:05.675Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:03:06.714 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:03:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:03:06.417Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:03:07.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:03:07 smithi083 ceph-mon[79930]: pgmap v17243: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:07.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:03:07 smithi083 ceph-mon[77242]: pgmap v17243: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:07.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:03:07 smithi153 ceph-mon[70044]: pgmap v17243: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:09.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:03:09 smithi083 ceph-mon[79930]: pgmap v17244: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:09.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:03:09 smithi083 ceph-mon[77242]: pgmap v17244: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:09.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:03:09 smithi153 ceph-mon[70044]: pgmap v17244: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:11.714 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:03:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:03:11] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T15:03:11.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:03:11 smithi083 ceph-mon[79930]: pgmap v17245: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:11.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:03:11 smithi083 ceph-mon[77242]: pgmap v17245: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:11.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:03:11 smithi153 ceph-mon[70044]: pgmap v17245: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:12.714 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:03:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:03:12.314Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:03:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:03:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:03:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:03:13.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:03:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:03:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:03:13.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:03:13 smithi083 ceph-mon[79930]: pgmap v17246: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:13.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:03:13 smithi083 ceph-mon[77242]: pgmap v17246: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:13.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:03:13 smithi153 ceph-mon[70044]: pgmap v17246: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:15.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:03:15 smithi083 ceph-mon[79930]: pgmap v17247: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:15.678 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:03:15 smithi083 ceph-mon[77242]: pgmap v17247: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:15.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:03:15 smithi153 ceph-mon[70044]: pgmap v17247: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:15.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:03:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:03:15.677Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:03:16.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:03:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:03:16.417Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:03:17.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:03:17 smithi083 ceph-mon[79930]: pgmap v17248: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:17.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:03:17 smithi083 ceph-mon[77242]: pgmap v17248: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:17.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:03:17 smithi153 ceph-mon[70044]: pgmap v17248: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:19.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:03:19 smithi083 ceph-mon[79930]: pgmap v17249: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:19.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:03:19 smithi083 ceph-mon[77242]: pgmap v17249: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:19.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:03:19 smithi153 ceph-mon[70044]: pgmap v17249: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:21.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:03:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:03:21] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T15:03:21.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:03:21 smithi083 ceph-mon[79930]: pgmap v17250: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:21.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:03:21 smithi083 ceph-mon[77242]: pgmap v17250: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:21.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:03:21 smithi153 ceph-mon[70044]: pgmap v17250: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:22.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:03:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:03:22.314Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:03:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:03:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:03:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:03:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:03:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:03:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:03:23.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:03:23 smithi083 ceph-mon[79930]: pgmap v17251: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:23.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:03:23 smithi083 ceph-mon[77242]: pgmap v17251: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:23.814 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:03:23 smithi153 ceph-mon[70044]: pgmap v17251: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:25.263 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:03:25.558 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:03:25.558 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 4m ago 9h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:03:25.559 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 2m ago 9h 63.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:03:25.560 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 4m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:03:25.561 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 2m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:03:25.561 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 2m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:03:25.562 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 4m ago 9h 982M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:03:25.562 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 4m ago 9h 2088M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:03:25.562 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 2m ago 9h 909M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:03:25.563 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 4m ago 9h 1251M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:03:25.563 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 4m ago 9h 24.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:03:25.564 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 2m ago 9h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:03:25.564 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 4m ago 9h 1516M 2190M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:03:25.565 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 4m ago 9h 1503M 2190M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:03:25.565 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 4m ago 9h 1168M 2190M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:03:25.565 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 4m ago 9h 1473M 2190M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:03:25.566 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 2m ago 9h 1292M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:03:25.566 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 2m ago 9h 1622M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:03:25.567 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 2m ago 9h 1449M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:03:25.583 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 2m ago 9h 1724M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:03:25.584 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 2m ago 9h 173M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:03:25.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:03:25 smithi083 ceph-mon[79930]: pgmap v17252: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:25.678 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:03:25 smithi083 ceph-mon[77242]: pgmap v17252: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:25.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:03:25 smithi153 ceph-mon[70044]: pgmap v17252: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:25.919 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:03:25.919 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:03:25.920 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:03:25.920 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:03:25.921 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:03:25.921 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:03:25.933 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:03:25.933 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:03:25.934 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:03:25.934 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:03:25.942 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:03:25.942 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:03:25.943 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:03:25.943 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:03:25.944 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:03:25.944 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:03:25.945 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:03:25.945 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:03:25.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:03:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:03:25.677Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:03:26.710 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:03:26 smithi083 ceph-mon[77242]: from='client.54192 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:03:26.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:03:26 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1627520653' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:03:26.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:03:26 smithi083 ceph-mon[79930]: from='client.54192 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:03:26.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:03:26 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1627520653' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:03:26.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:03:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:03:26.417Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:03:26.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:03:26 smithi153 ceph-mon[70044]: from='client.54192 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:03:26.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:03:26 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1627520653' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:03:27.710 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:03:27 smithi083 ceph-mon[77242]: from='client.64129 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:03:27.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:03:27 smithi083 ceph-mon[77242]: pgmap v17253: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:27.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:03:27 smithi083 ceph-mon[79930]: from='client.64129 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:03:27.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:03:27 smithi083 ceph-mon[79930]: pgmap v17253: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:27.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:03:27 smithi153 ceph-mon[70044]: from='client.64129 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:03:27.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:03:27 smithi153 ceph-mon[70044]: pgmap v17253: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:29.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:03:29 smithi083 ceph-mon[77242]: pgmap v17254: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:29.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:03:29 smithi083 ceph-mon[79930]: pgmap v17254: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:29.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:03:29 smithi153 ceph-mon[70044]: pgmap v17254: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:31.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:03:31 smithi083 ceph-mon[77242]: pgmap v17255: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:31.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:03:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:03:31] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T15:03:31.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:03:31 smithi083 ceph-mon[79930]: pgmap v17255: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:31.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:03:31 smithi153 ceph-mon[70044]: pgmap v17255: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:32.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:03:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:03:32.315Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:03:32.816 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:03:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:03:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:03:33.310 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:03:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:03:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:03:33.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:03:33 smithi083 ceph-mon[79930]: pgmap v17256: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:33.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:03:33 smithi083 ceph-mon[77242]: pgmap v17256: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:33.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:03:33 smithi153 ceph-mon[70044]: pgmap v17256: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:35.679 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:03:35 smithi083 ceph-mon[79930]: pgmap v17257: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:35.680 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:03:35 smithi083 ceph-mon[77242]: pgmap v17257: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:35.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:03:35 smithi153 ceph-mon[70044]: pgmap v17257: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:35.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:03:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:03:35.679Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:03:36.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:03:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:03:36.420Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:03:37.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:03:37 smithi083 ceph-mon[79930]: pgmap v17258: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:37.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:03:37 smithi083 ceph-mon[77242]: pgmap v17258: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:37.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:03:37 smithi153 ceph-mon[70044]: pgmap v17258: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:39.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:03:39 smithi083 ceph-mon[79930]: pgmap v17259: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:39.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:03:39 smithi083 ceph-mon[77242]: pgmap v17259: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:39.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:03:39 smithi153 ceph-mon[70044]: pgmap v17259: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:41.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:03:41 smithi083 ceph-mon[77242]: pgmap v17260: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:41.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:03:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:03:41] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T15:03:41.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:03:41 smithi083 ceph-mon[79930]: pgmap v17260: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:41.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:03:41 smithi153 ceph-mon[70044]: pgmap v17260: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:42.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:03:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:03:42.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:03:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:03:42.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:03:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:03:42.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:03:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:03:42.718 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:03:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:03:42.316Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:03:42.719 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:03:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:03:42.719 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:03:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:03:42.720 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:03:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:03:42.720 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:03:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:03:42.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:03:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:03:42.317Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:03:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:03:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:03:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:03:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:03:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:03:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:03:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:03:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:03:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:03:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:03:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:03:43.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:03:43 smithi083 ceph-mon[79930]: pgmap v17261: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:43.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:03:43 smithi083 ceph-mon[77242]: pgmap v17261: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:43.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:03:43 smithi153 ceph-mon[70044]: pgmap v17261: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:45.680 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:03:45 smithi083 ceph-mon[79930]: pgmap v17262: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:45.680 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:03:45 smithi083 ceph-mon[77242]: pgmap v17262: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:45.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:03:45 smithi153 ceph-mon[70044]: pgmap v17262: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:45.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:03:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:03:45.679Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:03:46.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:03:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:03:46.420Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:03:47.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:03:47 smithi083 ceph-mon[79930]: pgmap v17263: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:47.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:03:47 smithi083 ceph-mon[77242]: pgmap v17263: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:47.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:03:47 smithi153 ceph-mon[70044]: pgmap v17263: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:49.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:03:49 smithi083 ceph-mon[79930]: pgmap v17264: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:49.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:03:49 smithi083 ceph-mon[77242]: pgmap v17264: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:49.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:03:49 smithi153 ceph-mon[70044]: pgmap v17264: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:51.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:03:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:03:51] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T15:03:51.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:03:51 smithi083 ceph-mon[79930]: pgmap v17265: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:51.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:03:51 smithi083 ceph-mon[77242]: pgmap v17265: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:51.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:03:51 smithi153 ceph-mon[70044]: pgmap v17265: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:52.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:03:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:03:52.316Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:03:52.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:03:52 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:03:52.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:03:52 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:03:52.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:03:52 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:03:52.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:03:52 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:03:52.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:03:52 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:03:52.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:03:52 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:03:52.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:03:52 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:03:52.719 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:03:52 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:03:52.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:03:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:03:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:03:52.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:03:52 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:03:52.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:03:52 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:03:52.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:03:52 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:03:52.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:03:52 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:03:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:03:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:03:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:03:53.710 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:03:53 smithi083 ceph-mon[77242]: pgmap v17266: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:53.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:03:53 smithi083 ceph-mon[79930]: pgmap v17266: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:53.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:03:53 smithi153 ceph-mon[70044]: pgmap v17266: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:54.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:03:54 smithi083 ceph-mon[79930]: pgmap v17267: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:54.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:03:54 smithi083 ceph-mon[77242]: pgmap v17267: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:54.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:03:54 smithi153 ceph-mon[70044]: pgmap v17267: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:55.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:03:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:03:55.680Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:03:56.236 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:03:56.531 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:03:56.531 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 5m ago 9h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:03:56.531 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 3m ago 9h 63.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:03:56.532 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 5m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:03:56.532 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 3m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:03:56.532 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 3m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:03:56.532 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 5m ago 9h 982M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:03:56.533 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 5m ago 9h 2088M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:03:56.533 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 3m ago 9h 909M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:03:56.533 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 5m ago 9h 1251M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:03:56.533 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 5m ago 9h 24.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:03:56.533 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 3m ago 9h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:03:56.534 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 5m ago 9h 1516M 2190M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:03:56.534 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 5m ago 9h 1503M 2190M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:03:56.535 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 5m ago 9h 1168M 2190M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:03:56.535 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 5m ago 9h 1473M 2190M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:03:56.535 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 3m ago 9h 1292M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:03:56.535 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 3m ago 9h 1622M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:03:56.536 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 3m ago 9h 1449M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:03:56.536 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 3m ago 9h 1724M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:03:56.538 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 3m ago 9h 173M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:03:56.716 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:03:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:03:56.420Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:03:56.881 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:03:56.882 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:03:56.882 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:03:56.882 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:03:56.883 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:03:56.883 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:03:56.883 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:03:56.883 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:03:56.884 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:03:56.884 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:03:56.884 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:03:56.884 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:03:56.885 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:03:56.885 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:03:56.885 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:03:56.885 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:03:56.886 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:03:56.886 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:03:57.710 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:03:57 smithi083 ceph-mon[77242]: from='client.64141 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:03:57.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:03:57 smithi083 ceph-mon[77242]: pgmap v17268: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:57.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:03:57 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3725710926' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:03:57.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:03:57 smithi083 ceph-mon[79930]: from='client.64141 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:03:57.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:03:57 smithi083 ceph-mon[79930]: pgmap v17268: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:57.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:03:57 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3725710926' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:03:57.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:03:57 smithi153 ceph-mon[70044]: from='client.64141 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:03:57.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:03:57 smithi153 ceph-mon[70044]: pgmap v17268: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:57.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:03:57 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3725710926' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:03:58.710 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:03:58 smithi083 ceph-mon[77242]: from='client.54216 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:03:58.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:03:58 smithi083 ceph-mon[79930]: from='client.54216 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:03:58.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:03:58 smithi153 ceph-mon[70044]: from='client.54216 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:03:59.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:03:59 smithi083 ceph-mon[77242]: pgmap v17269: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:59.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:03:59 smithi083 ceph-mon[79930]: pgmap v17269: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:03:59.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:03:59 smithi153 ceph-mon[70044]: pgmap v17269: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:01.710 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:04:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:04:01] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T15:04:01.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:04:01 smithi083 ceph-mon[79930]: pgmap v17270: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:01.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:04:01 smithi083 ceph-mon[77242]: pgmap v17270: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:01.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:04:01 smithi153 ceph-mon[70044]: pgmap v17270: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:02.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:04:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:04:02.317Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:04:02.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:04:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:04:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:04:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:04:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:04:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:04:03.710 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:04:03 smithi083 ceph-mon[77242]: pgmap v17271: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:03.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:04:03 smithi083 ceph-mon[79930]: pgmap v17271: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:03.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:04:03 smithi153 ceph-mon[70044]: pgmap v17271: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:05.681 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:04:05 smithi083 ceph-mon[79930]: pgmap v17272: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:05.682 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:04:05 smithi083 ceph-mon[77242]: pgmap v17272: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:05.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:04:05 smithi153 ceph-mon[70044]: pgmap v17272: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:05.964 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:04:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:04:05.680Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:04:06.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:04:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:04:06.422Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:04:07.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:04:07 smithi083 ceph-mon[79930]: pgmap v17273: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:07.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:04:07 smithi083 ceph-mon[77242]: pgmap v17273: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:07.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:04:07 smithi153 ceph-mon[70044]: pgmap v17273: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:09.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:04:09 smithi083 ceph-mon[79930]: pgmap v17274: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:09.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:04:09 smithi083 ceph-mon[77242]: pgmap v17274: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:09.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:04:09 smithi153 ceph-mon[70044]: pgmap v17274: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:11.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:04:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:04:11] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T15:04:11.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:04:11 smithi083 ceph-mon[79930]: pgmap v17275: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:11.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:04:11 smithi083 ceph-mon[77242]: pgmap v17275: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:11.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:04:11 smithi153 ceph-mon[70044]: pgmap v17275: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:12.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:04:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:04:12.317Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:04:12.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:04:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:04:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:04:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:04:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:04:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:04:13.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:04:13 smithi083 ceph-mon[79930]: pgmap v17276: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:13.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:04:13 smithi083 ceph-mon[77242]: pgmap v17276: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:13.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:04:13 smithi153 ceph-mon[70044]: pgmap v17276: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:15.681 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:04:15 smithi083 ceph-mon[79930]: pgmap v17277: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:15.682 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:04:15 smithi083 ceph-mon[77242]: pgmap v17277: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:15.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:04:15 smithi153 ceph-mon[70044]: pgmap v17277: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:15.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:04:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:04:15.681Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:04:16.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:04:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:04:16.422Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:04:17.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:04:17 smithi083 ceph-mon[79930]: pgmap v17278: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:17.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:04:17 smithi083 ceph-mon[77242]: pgmap v17278: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:17.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:04:17 smithi153 ceph-mon[70044]: pgmap v17278: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:19.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:04:19 smithi083 ceph-mon[79930]: pgmap v17279: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:19.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:04:19 smithi083 ceph-mon[77242]: pgmap v17279: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:19.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:04:19 smithi153 ceph-mon[70044]: pgmap v17279: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:21.710 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:04:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:04:21] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T15:04:21.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:04:21 smithi083 ceph-mon[79930]: pgmap v17280: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:21.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:04:21 smithi083 ceph-mon[77242]: pgmap v17280: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:21.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:04:21 smithi153 ceph-mon[70044]: pgmap v17280: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:22.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:04:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:04:22.318Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:04:22.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:04:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:04:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:04:23.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:04:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:04:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:04:23.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:04:23 smithi083 ceph-mon[79930]: pgmap v17281: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:23.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:04:23 smithi083 ceph-mon[77242]: pgmap v17281: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:23.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:04:23 smithi153 ceph-mon[70044]: pgmap v17281: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:25.682 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:04:25 smithi083 ceph-mon[79930]: pgmap v17282: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:25.683 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:04:25 smithi083 ceph-mon[77242]: pgmap v17282: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:25.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:04:25 smithi153 ceph-mon[70044]: pgmap v17282: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:25.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:04:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:04:25.682Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:04:26.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:04:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:04:26.423Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:04:27.208 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:04:27.498 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:04:27.498 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 5m ago 9h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:04:27.499 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 3m ago 9h 63.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:04:27.499 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 5m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:04:27.499 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 3m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:04:27.499 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 3m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:04:27.500 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 5m ago 9h 982M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:04:27.500 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 5m ago 9h 2088M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:04:27.500 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 3m ago 9h 909M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:04:27.500 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 5m ago 9h 1251M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:04:27.501 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 5m ago 9h 24.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:04:27.501 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 3m ago 9h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:04:27.501 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 5m ago 9h 1516M 2190M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:04:27.501 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 5m ago 9h 1503M 2190M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:04:27.502 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 5m ago 9h 1168M 2190M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:04:27.502 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 5m ago 9h 1473M 2190M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:04:27.502 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 3m ago 9h 1292M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:04:27.503 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 3m ago 9h 1622M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:04:27.503 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 3m ago 9h 1449M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:04:27.503 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 3m ago 9h 1724M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:04:27.503 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 3m ago 9h 173M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:04:27.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:04:27 smithi083 ceph-mon[79930]: pgmap v17283: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:27.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:04:27 smithi083 ceph-mon[77242]: pgmap v17283: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:27.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:04:27 smithi153 ceph-mon[70044]: pgmap v17283: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:27.856 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:04:27.857 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:04:27.857 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:04:27.858 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:04:27.858 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:04:27.859 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:04:27.860 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:04:27.861 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:04:27.861 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:04:27.862 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:04:27.862 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:04:27.863 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:04:27.863 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:04:27.864 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:04:27.864 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:04:27.865 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:04:27.865 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:04:27.866 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:04:28.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:04:28 smithi083 ceph-mon[79930]: from='client.64159 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:04:28.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:04:28 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2571421294' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:04:28.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:04:28 smithi083 ceph-mon[77242]: from='client.64159 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:04:28.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:04:28 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2571421294' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:04:28.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:04:28 smithi153 ceph-mon[70044]: from='client.64159 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:04:28.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:04:28 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2571421294' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:04:29.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:04:29 smithi083 ceph-mon[77242]: from='client.54234 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:04:29.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:04:29 smithi083 ceph-mon[77242]: pgmap v17284: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:29.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:04:29 smithi083 ceph-mon[79930]: from='client.54234 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:04:29.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:04:29 smithi083 ceph-mon[79930]: pgmap v17284: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:29.814 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:04:29 smithi153 ceph-mon[70044]: from='client.54234 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:04:29.815 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:04:29 smithi153 ceph-mon[70044]: pgmap v17284: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:31.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:04:31 smithi083 ceph-mon[77242]: pgmap v17285: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:31.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:04:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:04:31] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T15:04:31.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:04:31 smithi083 ceph-mon[79930]: pgmap v17285: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:31.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:04:31 smithi153 ceph-mon[70044]: pgmap v17285: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:32.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:04:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:04:32.319Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:04:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:04:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:04:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:04:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:04:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:04:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:04:33.710 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:04:33 smithi083 ceph-mon[77242]: pgmap v17286: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:33.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:04:33 smithi083 ceph-mon[79930]: pgmap v17286: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:33.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:04:33 smithi153 ceph-mon[70044]: pgmap v17286: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:35.683 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:04:35 smithi083 ceph-mon[79930]: pgmap v17287: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:35.684 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:04:35 smithi083 ceph-mon[77242]: pgmap v17287: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:35.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:04:35 smithi153 ceph-mon[70044]: pgmap v17287: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:35.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:04:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:04:35.682Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:04:36.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:04:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:04:36.425Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:04:37.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:04:37 smithi083 ceph-mon[77242]: pgmap v17288: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:37.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:04:37 smithi083 ceph-mon[79930]: pgmap v17288: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:37.815 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:04:37 smithi153 ceph-mon[70044]: pgmap v17288: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:39.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:04:39 smithi083 ceph-mon[77242]: pgmap v17289: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:39.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:04:39 smithi083 ceph-mon[79930]: pgmap v17289: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:39.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:04:39 smithi153 ceph-mon[70044]: pgmap v17289: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:41.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:04:41 smithi083 ceph-mon[77242]: pgmap v17290: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:41.715 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:04:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:04:41] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T15:04:41.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:04:41 smithi083 ceph-mon[79930]: pgmap v17290: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:41.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:04:41 smithi153 ceph-mon[70044]: pgmap v17290: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:42.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:04:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:04:42.320Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:04:42.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:04:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:04:42.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:04:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:04:42.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:04:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:04:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:04:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:04:42.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:04:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:04:42.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:04:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:04:42.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:04:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:04:42.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:04:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:04:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:04:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:04:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:04:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:04:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:04:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:04:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:04:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:04:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:04:42.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:04:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:04:43.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:04:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:04:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:04:43.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:04:43 smithi083 ceph-mon[77242]: pgmap v17291: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:43.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:04:43 smithi083 ceph-mon[79930]: pgmap v17291: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:43.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:04:43 smithi153 ceph-mon[70044]: pgmap v17291: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:45.684 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:04:45 smithi083 ceph-mon[77242]: pgmap v17292: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:45.685 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:04:45 smithi083 ceph-mon[79930]: pgmap v17292: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:45.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:04:45 smithi153 ceph-mon[70044]: pgmap v17292: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:45.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:04:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:04:45.682Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:04:46.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:04:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:04:46.424Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:04:47.710 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:04:47 smithi083 ceph-mon[77242]: pgmap v17293: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:47.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:04:47 smithi083 ceph-mon[79930]: pgmap v17293: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:47.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:04:47 smithi153 ceph-mon[70044]: pgmap v17293: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:48.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:04:48 smithi083 ceph-mon[77242]: pgmap v17294: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:48.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:04:48 smithi083 ceph-mon[79930]: pgmap v17294: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:48.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:04:48 smithi153 ceph-mon[70044]: pgmap v17294: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:51.710 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:04:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:04:51] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T15:04:51.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:04:51 smithi083 ceph-mon[79930]: pgmap v17295: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:51.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:04:51 smithi083 ceph-mon[77242]: pgmap v17295: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:51.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:04:51 smithi153 ceph-mon[70044]: pgmap v17295: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:52.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:04:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:04:52.321Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:04:52.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:04:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:04:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:04:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:04:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:04:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:04:53.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:04:53 smithi083 ceph-mon[79930]: pgmap v17296: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:53.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:04:53 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:04:53.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:04:53 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:04:53.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:04:53 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:04:53.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:04:53 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:04:53.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:04:53 smithi083 ceph-mon[77242]: pgmap v17296: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:53.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:04:53 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:04:53.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:04:53 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:04:53.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:04:53 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:04:53.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:04:53 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:04:53.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:04:53 smithi153 ceph-mon[70044]: pgmap v17296: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:53.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:04:53 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:04:53.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:04:53 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:04:53.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:04:53 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:04:53.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:04:53 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:04:55.683 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:04:55 smithi083 ceph-mon[79930]: pgmap v17297: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:55.684 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:04:55 smithi083 ceph-mon[77242]: pgmap v17297: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:55.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:04:55 smithi153 ceph-mon[70044]: pgmap v17297: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:55.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:04:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:04:55.682Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:04:56.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:04:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:04:56.425Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:04:57.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:04:57 smithi083 ceph-mon[79930]: pgmap v17298: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:57.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:04:57 smithi083 ceph-mon[77242]: pgmap v17298: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:57.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:04:57 smithi153 ceph-mon[70044]: pgmap v17298: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:58.183 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:04:58.475 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:04:58.475 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 6m ago 9h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:04:58.475 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 4m ago 9h 63.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:04:58.476 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 6m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:04:58.476 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 4m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:04:58.476 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 4m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:04:58.476 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 6m ago 9h 982M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:04:58.476 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 6m ago 9h 2088M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:04:58.477 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 4m ago 9h 909M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:04:58.477 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 6m ago 9h 1251M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:04:58.477 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 6m ago 9h 24.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:04:58.477 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 4m ago 9h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:04:58.478 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 6m ago 9h 1516M 2190M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:04:58.478 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 6m ago 9h 1503M 2190M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:04:58.478 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 6m ago 9h 1168M 2190M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:04:58.478 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 6m ago 9h 1473M 2190M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:04:58.480 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 4m ago 9h 1292M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:04:58.480 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 4m ago 9h 1622M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:04:58.481 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 4m ago 9h 1449M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:04:58.481 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 4m ago 9h 1724M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:04:58.481 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 4m ago 9h 173M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:04:58.832 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:04:58.832 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:04:58.833 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:04:58.833 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:04:58.833 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:04:58.834 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:04:58.834 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:04:58.834 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:04:58.835 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:04:58.835 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:04:58.835 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:04:58.835 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:04:58.836 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:04:58.836 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:04:58.836 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:04:58.836 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:04:58.837 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:04:58.837 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:04:59.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:04:59 smithi083 ceph-mon[79930]: from='client.64177 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:04:59.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:04:59 smithi083 ceph-mon[79930]: pgmap v17299: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:59.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:04:59 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1251439985' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:04:59.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:04:59 smithi083 ceph-mon[77242]: from='client.64177 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:04:59.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:04:59 smithi083 ceph-mon[77242]: pgmap v17299: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:59.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:04:59 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1251439985' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:04:59.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:04:59 smithi153 ceph-mon[70044]: from='client.64177 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:04:59.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:04:59 smithi153 ceph-mon[70044]: pgmap v17299: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:04:59.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:04:59 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1251439985' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:05:00.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:05:00 smithi083 ceph-mon[79930]: from='client.64183 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:05:00.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:05:00 smithi083 ceph-mon[77242]: from='client.64183 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:05:00.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:05:00 smithi153 ceph-mon[70044]: from='client.64183 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:05:01.710 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:05:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:05:01] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T15:05:01.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:05:01 smithi083 ceph-mon[79930]: pgmap v17300: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:05:01.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:05:01 smithi083 ceph-mon[77242]: pgmap v17300: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:05:01.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:05:01 smithi153 ceph-mon[70044]: pgmap v17300: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:05:02.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:05:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:05:02.322Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:05:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:05:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:05:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:05:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:05:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:05:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:05:03.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:05:03 smithi083 ceph-mon[79930]: pgmap v17301: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:05:03.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:05:03 smithi083 ceph-mon[77242]: pgmap v17301: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:05:03.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:05:03 smithi153 ceph-mon[70044]: pgmap v17301: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:05:05.683 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:05:05 smithi083 ceph-mon[79930]: pgmap v17302: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:05:05.684 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:05:05 smithi083 ceph-mon[77242]: pgmap v17302: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:05:05.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:05:05 smithi153 ceph-mon[70044]: pgmap v17302: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:05:05.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:05:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:05:05.682Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:05:06.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:05:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:05:06.426Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:05:07.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:05:07 smithi083 ceph-mon[79930]: pgmap v17303: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:05:07.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:05:07 smithi083 ceph-mon[77242]: pgmap v17303: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:05:07.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:05:07 smithi153 ceph-mon[70044]: pgmap v17303: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:05:09.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:05:09 smithi083 ceph-mon[77242]: pgmap v17304: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:05:09.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:05:09 smithi083 ceph-mon[79930]: pgmap v17304: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:05:09.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:05:09 smithi153 ceph-mon[70044]: pgmap v17304: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:05:11.710 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:05:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:05:11] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T15:05:11.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:05:11 smithi083 ceph-mon[79930]: pgmap v17305: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:05:11.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:05:11 smithi083 ceph-mon[77242]: pgmap v17305: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:05:11.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:05:11 smithi153 ceph-mon[70044]: pgmap v17305: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:05:12.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:05:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:05:12.323Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:05:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:05:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:05:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:05:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:05:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:05:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:05:13.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:05:13 smithi083 ceph-mon[79930]: pgmap v17306: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:05:13.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:05:13 smithi083 ceph-mon[77242]: pgmap v17306: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:05:13.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:05:13 smithi153 ceph-mon[70044]: pgmap v17306: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:05:15.683 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:05:15 smithi083 ceph-mon[79930]: pgmap v17307: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:05:15.684 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:05:15 smithi083 ceph-mon[77242]: pgmap v17307: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:05:15.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:05:15 smithi153 ceph-mon[70044]: pgmap v17307: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:05:15.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:05:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:05:15.683Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:05:16.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:05:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:05:16.426Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:05:17.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:05:17 smithi083 ceph-mon[79930]: pgmap v17308: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:05:17.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:05:17 smithi083 ceph-mon[77242]: pgmap v17308: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:05:17.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:05:17 smithi153 ceph-mon[70044]: pgmap v17308: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:05:19.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:05:19 smithi083 ceph-mon[79930]: pgmap v17309: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:05:19.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:05:19 smithi083 ceph-mon[77242]: pgmap v17309: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:05:19.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:05:19 smithi153 ceph-mon[70044]: pgmap v17309: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:05:21.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:05:21 smithi083 ceph-mon[79930]: pgmap v17310: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:05:21.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:05:21 smithi083 ceph-mon[77242]: pgmap v17310: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:05:21.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:05:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:05:21] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T15:05:21.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:05:21 smithi153 ceph-mon[70044]: pgmap v17310: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:05:22.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:05:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:05:22.322Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:05:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:05:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:05:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:05:23.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:05:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:05:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:05:23.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:05:23 smithi083 ceph-mon[77242]: pgmap v17311: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:05:23.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:05:23 smithi083 ceph-mon[79930]: pgmap v17311: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:05:23.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:05:23 smithi153 ceph-mon[70044]: pgmap v17311: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:05:25.689 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:05:25 smithi083 ceph-mon[79930]: pgmap v17312: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:05:25.689 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:05:25 smithi083 ceph-mon[77242]: pgmap v17312: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:05:25.814 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:05:25 smithi153 ceph-mon[70044]: pgmap v17312: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:05:25.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:05:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:05:25.683Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:05:26.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:05:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:05:26.427Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:05:27.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:05:27 smithi083 ceph-mon[79930]: pgmap v17313: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:05:27.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:05:27 smithi083 ceph-mon[77242]: pgmap v17313: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:05:27.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:05:27 smithi153 ceph-mon[70044]: pgmap v17313: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:05:29.158 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:05:29.448 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:05:29.449 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 6m ago 9h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:05:29.450 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 4m ago 9h 63.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:05:29.450 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 6m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:05:29.451 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 4m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:05:29.451 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 4m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:05:29.452 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 6m ago 9h 982M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:05:29.452 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 6m ago 9h 2088M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:05:29.453 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 4m ago 9h 909M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:05:29.455 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 6m ago 9h 1251M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:05:29.455 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 6m ago 9h 24.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:05:29.455 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 4m ago 9h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:05:29.456 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 6m ago 9h 1516M 2190M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:05:29.456 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 6m ago 9h 1503M 2190M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:05:29.456 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 6m ago 9h 1168M 2190M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:05:29.457 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 6m ago 9h 1473M 2190M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:05:29.457 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 4m ago 9h 1292M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:05:29.457 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 4m ago 9h 1622M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:05:29.457 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 4m ago 9h 1449M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:05:29.458 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 4m ago 9h 1724M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:05:29.458 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 4m ago 9h 173M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:05:29.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:05:29 smithi083 ceph-mon[79930]: pgmap v17314: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:05:29.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:05:29 smithi083 ceph-mon[77242]: pgmap v17314: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:05:29.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:05:29 smithi153 ceph-mon[70044]: pgmap v17314: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:05:29.808 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:05:29.808 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:05:29.809 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:05:29.810 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:05:29.811 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:05:29.811 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:05:29.812 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:05:29.812 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:05:29.812 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:05:29.813 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:05:29.813 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:05:29.814 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:05:29.814 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:05:29.815 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:05:29.815 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:05:29.816 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:05:29.816 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:05:29.817 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:05:30.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:05:30 smithi083 ceph-mon[77242]: from='client.64195 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:05:30.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:05:30 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4164209444' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:05:30.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:05:30 smithi083 ceph-mon[79930]: from='client.64195 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:05:30.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:05:30 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4164209444' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:05:30.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:05:30 smithi153 ceph-mon[70044]: from='client.64195 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:05:30.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:05:30 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4164209444' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:05:31.710 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:05:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:05:31] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T15:05:31.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:05:31 smithi083 ceph-mon[77242]: from='client.54270 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:05:31.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:05:31 smithi083 ceph-mon[77242]: pgmap v17315: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:05:31.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:05:31 smithi083 ceph-mon[79930]: from='client.54270 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:05:31.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:05:31 smithi083 ceph-mon[79930]: pgmap v17315: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:05:31.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:05:31 smithi153 ceph-mon[70044]: from='client.54270 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:05:31.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:05:31 smithi153 ceph-mon[70044]: pgmap v17315: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:05:32.714 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:05:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:05:32.323Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:05:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:05:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:05:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:05:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:05:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:05:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:05:33.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:05:33 smithi083 ceph-mon[79930]: pgmap v17316: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:05:33.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:05:33 smithi083 ceph-mon[77242]: pgmap v17316: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:05:33.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:05:33 smithi153 ceph-mon[70044]: pgmap v17316: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:05:35.684 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:05:35 smithi083 ceph-mon[79930]: pgmap v17317: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:05:35.684 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:05:35 smithi083 ceph-mon[77242]: pgmap v17317: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:05:35.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:05:35 smithi153 ceph-mon[70044]: pgmap v17317: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:05:35.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:05:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:05:35.684Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:05:36.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:05:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:05:36.427Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:05:37.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:05:37 smithi083 ceph-mon[79930]: pgmap v17318: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:05:37.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:05:37 smithi083 ceph-mon[77242]: pgmap v17318: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:05:37.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:05:37 smithi153 ceph-mon[70044]: pgmap v17318: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:05:38.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:05:38 smithi083 ceph-mon[79930]: pgmap v17319: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:05:38.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:05:38 smithi083 ceph-mon[77242]: pgmap v17319: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:05:38.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:05:38 smithi153 ceph-mon[70044]: pgmap v17319: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:05:41.710 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:05:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:05:41] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T15:05:41.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:05:41 smithi083 ceph-mon[79930]: pgmap v17320: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:05:41.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:05:41 smithi083 ceph-mon[77242]: pgmap v17320: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:05:41.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:05:41 smithi153 ceph-mon[70044]: pgmap v17320: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:05:42.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:05:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:05:42.323Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:05:42.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:05:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:05:42.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:05:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:05:42.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:05:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:05:42.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:05:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:05:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:05:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:05:42.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:05:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:05:42.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:05:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:05:42.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:05:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:05:42.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:05:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:05:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:05:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:05:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:05:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:05:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:05:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:05:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:05:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:05:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:05:43.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:05:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:05:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:05:43.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:05:43 smithi083 ceph-mon[79930]: pgmap v17321: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:05:43.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:05:43 smithi083 ceph-mon[77242]: pgmap v17321: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:05:43.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:05:43 smithi153 ceph-mon[70044]: pgmap v17321: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:05:45.685 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:05:45 smithi083 ceph-mon[79930]: pgmap v17322: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:05:45.686 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:05:45 smithi083 ceph-mon[77242]: pgmap v17322: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:05:45.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:05:45 smithi153 ceph-mon[70044]: pgmap v17322: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:05:45.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:05:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:05:45.685Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:05:46.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:05:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:05:46.427Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:05:47.710 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:05:47 smithi083 ceph-mon[77242]: pgmap v17323: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:05:47.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:05:47 smithi083 ceph-mon[79930]: pgmap v17323: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:05:47.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:05:47 smithi153 ceph-mon[70044]: pgmap v17323: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:05:49.710 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:05:49 smithi083 ceph-mon[77242]: pgmap v17324: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:05:49.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:05:49 smithi083 ceph-mon[79930]: pgmap v17324: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:05:49.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:05:49 smithi153 ceph-mon[70044]: pgmap v17324: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:05:51.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:05:51 smithi083 ceph-mon[77242]: pgmap v17325: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:05:51.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:05:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:05:51] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T15:05:51.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:05:51 smithi083 ceph-mon[79930]: pgmap v17325: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:05:51.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:05:51 smithi153 ceph-mon[70044]: pgmap v17325: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:05:52.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:05:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:05:52.325Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:05:52.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:05:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:05:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:05:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:05:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:05:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:05:53.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:05:53 smithi083 ceph-mon[79930]: pgmap v17326: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:05:53.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:05:53 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:05:53.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:05:53 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:05:53.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:05:53 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:05:53.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:05:53 smithi083 ceph-mon[77242]: pgmap v17326: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:05:53.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:05:53 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:05:53.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:05:53 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:05:53.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:05:53 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:05:53.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:05:53 smithi153 ceph-mon[70044]: pgmap v17326: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:05:53.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:05:53 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:05:53.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:05:53 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:05:53.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:05:53 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:05:54.960 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:05:54 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:05:54.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:05:54 smithi083 ceph-mon[77242]: pgmap v17327: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:05:54.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:05:54 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:05:54.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:05:54 smithi083 ceph-mon[79930]: pgmap v17327: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:05:55.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:05:54 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:05:55.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:05:54 smithi153 ceph-mon[70044]: pgmap v17327: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:05:55.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:05:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:05:55.684Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:05:56.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:05:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:05:56.428Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:05:57.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:05:57 smithi083 ceph-mon[79930]: pgmap v17328: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:05:57.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:05:57 smithi083 ceph-mon[77242]: pgmap v17328: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:05:57.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:05:57 smithi153 ceph-mon[70044]: pgmap v17328: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:05:59.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:05:59 smithi083 ceph-mon[79930]: pgmap v17329: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:05:59.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:05:59 smithi083 ceph-mon[77242]: pgmap v17329: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:05:59.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:05:59 smithi153 ceph-mon[70044]: pgmap v17329: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:00.134 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:06:00.424 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:06:00.425 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 7m ago 9h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:06:00.425 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 5m ago 9h 63.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:06:00.425 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 7m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:06:00.426 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 5m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:06:00.426 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 5m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:06:00.426 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 7m ago 9h 982M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:06:00.427 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 7m ago 9h 2088M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:06:00.427 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 5m ago 9h 909M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:06:00.427 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 7m ago 9h 1251M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:06:00.428 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 7m ago 9h 24.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:06:00.428 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 5m ago 9h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:06:00.428 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 7m ago 9h 1516M 2190M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:06:00.429 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 7m ago 9h 1503M 2190M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:06:00.429 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 7m ago 9h 1168M 2190M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:06:00.430 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 7m ago 9h 1473M 2190M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:06:00.430 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 5m ago 9h 1292M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:06:00.431 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 5m ago 9h 1622M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:06:00.431 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 5m ago 9h 1449M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:06:00.431 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 5m ago 9h 1724M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:06:00.432 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 5m ago 9h 173M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:06:00.781 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:06:00.781 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:06:00.782 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:06:00.782 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:06:00.782 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:06:00.782 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:06:00.783 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:06:00.783 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:06:00.783 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:06:00.783 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:06:00.784 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:06:00.784 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:06:00.784 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:06:00.784 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:06:00.784 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:06:00.785 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:06:00.785 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:06:00.785 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:06:01.710 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:06:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:06:01] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T15:06:01.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:06:01 smithi083 ceph-mon[77242]: from='client.64213 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:06:01.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:06:01 smithi083 ceph-mon[77242]: pgmap v17330: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:01.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:06:01 smithi083 ceph-mon[77242]: from='client.64219 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:06:01.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:06:01 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4099071660' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:06:01.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:06:01 smithi083 ceph-mon[79930]: from='client.64213 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:06:01.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:06:01 smithi083 ceph-mon[79930]: pgmap v17330: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:01.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:06:01 smithi083 ceph-mon[79930]: from='client.64219 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:06:01.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:06:01 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4099071660' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:06:01.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:06:01 smithi153 ceph-mon[70044]: from='client.64213 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:06:01.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:06:01 smithi153 ceph-mon[70044]: pgmap v17330: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:01.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:06:01 smithi153 ceph-mon[70044]: from='client.64219 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:06:01.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:06:01 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4099071660' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:06:02.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:06:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:06:02.326Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:06:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:06:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:06:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:06:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:06:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:06:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:06:03.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:06:03 smithi083 ceph-mon[79930]: pgmap v17331: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:03.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:06:03 smithi083 ceph-mon[77242]: pgmap v17331: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:03.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:06:03 smithi153 ceph-mon[70044]: pgmap v17331: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:05.685 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:06:05 smithi083 ceph-mon[79930]: pgmap v17332: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:05.686 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:06:05 smithi083 ceph-mon[77242]: pgmap v17332: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:05.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:06:05 smithi153 ceph-mon[70044]: pgmap v17332: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:05.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:06:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:06:05.684Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:06:06.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:06:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:06:06.428Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:06:07.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:06:07 smithi083 ceph-mon[79930]: pgmap v17333: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:07.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:06:07 smithi083 ceph-mon[77242]: pgmap v17333: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:07.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:06:07 smithi153 ceph-mon[70044]: pgmap v17333: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:09.710 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:06:09 smithi083 ceph-mon[77242]: pgmap v17334: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:09.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:06:09 smithi083 ceph-mon[79930]: pgmap v17334: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:09.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:06:09 smithi153 ceph-mon[70044]: pgmap v17334: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:11.710 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:06:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:06:11] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T15:06:11.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:06:11 smithi083 ceph-mon[79930]: pgmap v17335: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:11.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:06:11 smithi083 ceph-mon[77242]: pgmap v17335: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:11.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:06:11 smithi153 ceph-mon[70044]: pgmap v17335: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:12.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:06:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:06:12.326Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:06:12.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:06:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:06:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:06:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:06:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:06:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:06:13.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:06:13 smithi083 ceph-mon[79930]: pgmap v17336: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:13.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:06:13 smithi083 ceph-mon[77242]: pgmap v17336: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:13.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:06:13 smithi153 ceph-mon[70044]: pgmap v17336: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:15.687 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:06:15 smithi083 ceph-mon[79930]: pgmap v17337: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:15.688 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:06:15 smithi083 ceph-mon[77242]: pgmap v17337: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:15.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:06:15 smithi153 ceph-mon[70044]: pgmap v17337: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:15.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:06:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:06:15.686Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:06:16.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:06:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:06:16.428Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:06:17.719 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:06:17 smithi083 ceph-mon[79930]: pgmap v17338: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:17.720 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:06:17 smithi083 ceph-mon[77242]: pgmap v17338: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:17.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:06:17 smithi153 ceph-mon[70044]: pgmap v17338: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:19.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:06:19 smithi083 ceph-mon[79930]: pgmap v17339: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:19.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:06:19 smithi083 ceph-mon[77242]: pgmap v17339: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:19.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:06:19 smithi153 ceph-mon[70044]: pgmap v17339: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:21.710 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:06:21 smithi083 ceph-mon[77242]: pgmap v17340: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:21.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:06:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:06:21] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T15:06:21.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:06:21 smithi083 ceph-mon[79930]: pgmap v17340: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:21.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:06:21 smithi153 ceph-mon[70044]: pgmap v17340: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:22.714 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:06:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:06:22.327Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:06:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:06:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:06:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:06:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:06:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:06:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:06:23.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:06:23 smithi083 ceph-mon[79930]: pgmap v17341: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:23.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:06:23 smithi083 ceph-mon[77242]: pgmap v17341: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:23.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:06:23 smithi153 ceph-mon[70044]: pgmap v17341: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:25.688 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:06:25 smithi083 ceph-mon[79930]: pgmap v17342: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:25.688 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:06:25 smithi083 ceph-mon[77242]: pgmap v17342: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:25.814 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:06:25 smithi153 ceph-mon[70044]: pgmap v17342: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:25.966 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:06:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:06:25.687Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:06:26.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:06:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:06:26.429Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:06:27.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:06:27 smithi083 ceph-mon[79930]: pgmap v17343: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:27.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:06:27 smithi083 ceph-mon[77242]: pgmap v17343: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:27.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:06:27 smithi153 ceph-mon[70044]: pgmap v17343: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:29.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:06:29 smithi083 ceph-mon[79930]: pgmap v17344: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:29.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:06:29 smithi083 ceph-mon[77242]: pgmap v17344: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:29.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:06:29 smithi153 ceph-mon[70044]: pgmap v17344: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:31.106 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:06:31.396 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:06:31.397 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 7m ago 9h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:06:31.397 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 5m ago 9h 63.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:06:31.397 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 7m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:06:31.397 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 5m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:06:31.397 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 5m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:06:31.398 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 7m ago 9h 982M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:06:31.398 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 7m ago 9h 2088M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:06:31.398 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 5m ago 9h 909M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:06:31.399 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 7m ago 9h 1251M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:06:31.399 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 7m ago 9h 24.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:06:31.399 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 5m ago 9h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:06:31.399 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 7m ago 9h 1516M 2190M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:06:31.399 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 7m ago 9h 1503M 2190M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:06:31.400 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 7m ago 9h 1168M 2190M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:06:31.400 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 7m ago 9h 1473M 2190M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:06:31.400 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 5m ago 9h 1292M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:06:31.400 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 5m ago 9h 1622M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:06:31.401 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 5m ago 9h 1449M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:06:31.401 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 5m ago 9h 1724M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:06:31.401 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 5m ago 9h 173M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:06:31.710 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:06:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:06:31] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T15:06:31.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:06:31 smithi083 ceph-mon[79930]: pgmap v17345: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:31.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:06:31 smithi083 ceph-mon[77242]: pgmap v17345: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:31.754 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:06:31.754 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:06:31.755 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:06:31.755 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:06:31.756 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:06:31.756 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:06:31.756 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:06:31.756 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:06:31.757 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:06:31.757 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:06:31.757 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:06:31.757 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:06:31.758 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:06:31.758 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:06:31.758 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:06:31.758 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:06:31.759 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:06:31.759 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:06:31.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:06:31 smithi153 ceph-mon[70044]: pgmap v17345: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:32.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:06:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:06:32.327Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:06:32.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:06:32 smithi083 ceph-mon[77242]: from='client.64231 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:06:32.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:06:32 smithi083 ceph-mon[77242]: from='client.54306 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:06:32.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:06:32 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3330255007' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:06:32.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:06:32 smithi083 ceph-mon[79930]: from='client.64231 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:06:32.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:06:32 smithi083 ceph-mon[79930]: from='client.54306 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:06:32.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:06:32 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3330255007' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:06:32.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:06:32 smithi153 ceph-mon[70044]: from='client.64231 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:06:32.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:06:32 smithi153 ceph-mon[70044]: from='client.54306 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:06:32.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:06:32 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3330255007' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:06:32.810 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:06:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:06:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:06:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:06:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:06:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:06:33.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:06:33 smithi083 ceph-mon[79930]: pgmap v17346: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:33.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:06:33 smithi083 ceph-mon[77242]: pgmap v17346: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:33.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:06:33 smithi153 ceph-mon[70044]: pgmap v17346: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:35.689 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:06:35 smithi083 ceph-mon[79930]: pgmap v17347: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:35.690 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:06:35 smithi083 ceph-mon[77242]: pgmap v17347: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:35.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:06:35 smithi153 ceph-mon[70044]: pgmap v17347: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:35.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:06:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:06:35.688Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:06:36.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:06:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:06:36.430Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:06:37.710 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:06:37 smithi083 ceph-mon[77242]: pgmap v17348: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:37.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:06:37 smithi083 ceph-mon[79930]: pgmap v17348: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:37.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:06:37 smithi153 ceph-mon[70044]: pgmap v17348: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:39.710 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:06:39 smithi083 ceph-mon[77242]: pgmap v17349: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:39.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:06:39 smithi083 ceph-mon[79930]: pgmap v17349: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:39.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:06:39 smithi153 ceph-mon[70044]: pgmap v17349: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:41.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:06:41 smithi083 ceph-mon[79930]: pgmap v17350: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:41.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:06:41 smithi083 ceph-mon[77242]: pgmap v17350: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:41.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:06:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:06:41] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T15:06:41.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:06:41 smithi153 ceph-mon[70044]: pgmap v17350: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:42.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:06:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:06:42.329Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:06:42.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:06:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:06:42.725 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:06:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:06:42.725 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:06:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:06:42.726 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:06:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:06:42.726 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:06:42 smithi083 ceph-mon[79930]: pgmap v17351: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:42.727 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:06:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:06:42.727 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:06:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:06:42.727 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:06:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:06:42.727 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:06:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:06:42.728 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:06:42 smithi083 ceph-mon[77242]: pgmap v17351: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:06:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:06:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:06:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:06:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:06:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:06:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:06:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:06:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:06:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:06:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:06:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:06:42 smithi153 ceph-mon[70044]: pgmap v17351: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:06:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:06:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:06:45.689 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:06:45 smithi083 ceph-mon[79930]: pgmap v17352: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:45.690 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:06:45 smithi083 ceph-mon[77242]: pgmap v17352: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:45.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:06:45 smithi153 ceph-mon[70044]: pgmap v17352: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:45.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:06:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:06:45.689Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:06:46.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:06:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:06:46.430Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:06:47.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:06:47 smithi083 ceph-mon[79930]: pgmap v17353: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:47.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:06:47 smithi083 ceph-mon[77242]: pgmap v17353: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:47.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:06:47 smithi153 ceph-mon[70044]: pgmap v17353: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:49.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:06:49 smithi083 ceph-mon[79930]: pgmap v17354: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:49.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:06:49 smithi083 ceph-mon[77242]: pgmap v17354: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:49.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:06:49 smithi153 ceph-mon[70044]: pgmap v17354: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:51.710 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:06:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:06:51] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T15:06:51.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:06:51 smithi083 ceph-mon[79930]: pgmap v17355: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:51.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:06:51 smithi083 ceph-mon[77242]: pgmap v17355: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:51.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:06:51 smithi153 ceph-mon[70044]: pgmap v17355: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:52.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:06:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:06:52.329Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:06:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:06:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:06:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:06:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:06:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:06:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:06:53.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:06:53 smithi083 ceph-mon[79930]: pgmap v17356: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:53.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:06:53 smithi083 ceph-mon[77242]: pgmap v17356: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:53.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:06:53 smithi153 ceph-mon[70044]: pgmap v17356: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:54.991 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:06:54 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:06:54.994 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:06:54 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:06:54.994 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:06:54 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:06:54.995 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:06:54 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:06:54.995 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:06:54 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:06:54.995 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:06:54 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:06:54.995 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:06:54 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:06:54.996 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:06:54 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:06:54.996 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:06:54 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:06:54.996 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:06:54 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:06:54.996 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:06:54 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:06:54.996 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:06:54 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:06:54.997 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:06:54 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:06:54.997 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:06:54 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:06:54.997 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:06:54 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:06:54.998 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:06:54 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:06:54.998 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:06:54 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:06:54.998 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:06:54 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:06:54.998 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:06:54 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:06:54.999 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:06:54 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:06:54.999 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:06:54 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:06:54.999 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:06:54 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:06:54.999 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:06:54 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:06:55.000 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:06:54 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:06:55.000 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:06:54 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:06:55.000 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:06:54 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:06:55.001 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:06:54 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:06:55.001 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:06:54 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:06:55.001 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:06:54 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:06:55.001 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:06:54 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:06:55.002 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:06:54 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:06:55.002 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:06:54 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:06:55.002 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:06:54 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:06:55.003 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:06:54 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:06:55.003 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:06:54 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:06:55.003 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:06:54 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:06:55.003 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:06:54 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:06:55.004 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:06:54 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:06:55.004 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:06:54 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:06:55.690 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:06:55 smithi083 ceph-mon[79930]: Adjusting osd_memory_target on smithi083 to 2177M 2022-09-29T15:06:55.690 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:06:55 smithi083 ceph-mon[79930]: pgmap v17357: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:55.691 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:06:55 smithi083 ceph-mon[77242]: Adjusting osd_memory_target on smithi083 to 2177M 2022-09-29T15:06:55.691 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:06:55 smithi083 ceph-mon[77242]: pgmap v17357: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:55.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:06:55 smithi153 ceph-mon[70044]: Adjusting osd_memory_target on smithi083 to 2177M 2022-09-29T15:06:55.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:06:55 smithi153 ceph-mon[70044]: pgmap v17357: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:55.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:06:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:06:55.689Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:06:56.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:06:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:06:56.430Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:06:57.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:06:57 smithi083 ceph-mon[79930]: pgmap v17358: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:57.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:06:57 smithi083 ceph-mon[77242]: pgmap v17358: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:57.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:06:57 smithi153 ceph-mon[70044]: pgmap v17358: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:59.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:06:59 smithi083 ceph-mon[79930]: pgmap v17359: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:59.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:06:59 smithi083 ceph-mon[77242]: pgmap v17359: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:06:59.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:06:59 smithi153 ceph-mon[70044]: pgmap v17359: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:01.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:07:01 smithi083 ceph-mon[79930]: pgmap v17360: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:01.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:07:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:07:01] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T15:07:01.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:07:01 smithi083 ceph-mon[77242]: pgmap v17360: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:01.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:07:01 smithi153 ceph-mon[70044]: pgmap v17360: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:02.078 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:07:02.369 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:07:02.370 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 8m ago 9h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:07:02.371 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 6m ago 9h 63.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:07:02.371 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 8m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:07:02.372 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 6m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:07:02.372 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 6m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:07:02.373 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 8m ago 9h 982M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:07:02.373 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 8m ago 9h 2088M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:07:02.373 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 6m ago 9h 909M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:07:02.374 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 8m ago 9h 1251M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:07:02.374 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 8m ago 9h 24.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:07:02.375 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 6m ago 9h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:07:02.375 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 8m ago 9h 1516M 2177M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:07:02.376 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 8m ago 9h 1503M 2177M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:07:02.376 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 8m ago 9h 1168M 2177M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:07:02.377 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 8m ago 9h 1473M 2177M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:07:02.377 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 6m ago 9h 1292M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:07:02.378 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 6m ago 9h 1622M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:07:02.378 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 6m ago 9h 1449M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:07:02.379 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 6m ago 9h 1724M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:07:02.379 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 6m ago 9h 173M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:07:02.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:07:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:07:02.328Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:07:02.727 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:07:02.728 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:07:02.728 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:07:02.728 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:07:02.729 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:07:02.729 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:07:02.729 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:07:02.730 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:07:02.730 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:07:02.730 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:07:02.730 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:07:02.731 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:07:02.731 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:07:02.731 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:07:02.731 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:07:02.732 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:07:02.732 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:07:02.732 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:07:02.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:07:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:07:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:07:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:07:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:07:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:07:03.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:07:03 smithi083 ceph-mon[79930]: from='client.64249 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:07:03.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:07:03 smithi083 ceph-mon[79930]: pgmap v17361: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:03.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:07:03 smithi083 ceph-mon[79930]: from='client.64255 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:07:03.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:07:03 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1752898689' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:07:03.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:07:03 smithi083 ceph-mon[77242]: from='client.64249 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:07:03.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:07:03 smithi083 ceph-mon[77242]: pgmap v17361: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:03.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:07:03 smithi083 ceph-mon[77242]: from='client.64255 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:07:03.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:07:03 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1752898689' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:07:03.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:07:03 smithi153 ceph-mon[70044]: from='client.64249 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:07:03.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:07:03 smithi153 ceph-mon[70044]: pgmap v17361: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:03.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:07:03 smithi153 ceph-mon[70044]: from='client.64255 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:07:03.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:07:03 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1752898689' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:07:05.692 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:07:05 smithi083 ceph-mon[79930]: pgmap v17362: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:05.693 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:07:05 smithi083 ceph-mon[77242]: pgmap v17362: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:05.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:07:05 smithi153 ceph-mon[70044]: pgmap v17362: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:05.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:07:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:07:05.691Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:07:06.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:07:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:07:06.432Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:07:07.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:07:07 smithi083 ceph-mon[79930]: pgmap v17363: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:07.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:07:07 smithi083 ceph-mon[77242]: pgmap v17363: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:07.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:07:07 smithi153 ceph-mon[70044]: pgmap v17363: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:09.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:07:09 smithi083 ceph-mon[79930]: pgmap v17364: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:09.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:07:09 smithi083 ceph-mon[77242]: pgmap v17364: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:09.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:07:09 smithi153 ceph-mon[70044]: pgmap v17364: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:11.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:07:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:07:11] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T15:07:11.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:07:11 smithi083 ceph-mon[79930]: pgmap v17365: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:11.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:07:11 smithi083 ceph-mon[77242]: pgmap v17365: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:11.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:07:11 smithi153 ceph-mon[70044]: pgmap v17365: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:12.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:07:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:07:12.330Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:07:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:07:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:07:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:07:13.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:07:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:07:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:07:13.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:07:13 smithi083 ceph-mon[79930]: pgmap v17366: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:13.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:07:13 smithi083 ceph-mon[77242]: pgmap v17366: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:13.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:07:13 smithi153 ceph-mon[70044]: pgmap v17366: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:15.692 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:07:15 smithi083 ceph-mon[79930]: pgmap v17367: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:15.693 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:07:15 smithi083 ceph-mon[77242]: pgmap v17367: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:15.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:07:15 smithi153 ceph-mon[70044]: pgmap v17367: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:15.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:07:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:07:15.691Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:07:16.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:07:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:07:16.432Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:07:17.710 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:07:17 smithi083 ceph-mon[77242]: pgmap v17368: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:17.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:07:17 smithi083 ceph-mon[79930]: pgmap v17368: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:17.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:07:17 smithi153 ceph-mon[70044]: pgmap v17368: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:19.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:07:19 smithi083 ceph-mon[79930]: pgmap v17369: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:19.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:07:19 smithi083 ceph-mon[77242]: pgmap v17369: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:19.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:07:19 smithi153 ceph-mon[70044]: pgmap v17369: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:21.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:07:21 smithi083 ceph-mon[79930]: pgmap v17370: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:21.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:07:21 smithi083 ceph-mon[77242]: pgmap v17370: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:21.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:07:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:07:21] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T15:07:21.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:07:21 smithi153 ceph-mon[70044]: pgmap v17370: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:22.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:07:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:07:22.331Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:07:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:07:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:07:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:07:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:07:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:07:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:07:23.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:07:23 smithi083 ceph-mon[79930]: pgmap v17371: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:23.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:07:23 smithi083 ceph-mon[77242]: pgmap v17371: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:23.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:07:23 smithi153 ceph-mon[70044]: pgmap v17371: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:25.692 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:07:25 smithi083 ceph-mon[77242]: pgmap v17372: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:25.693 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:07:25 smithi083 ceph-mon[79930]: pgmap v17372: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:25.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:07:25 smithi153 ceph-mon[70044]: pgmap v17372: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:25.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:07:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:07:25.692Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:07:26.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:07:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:07:26.433Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:07:27.710 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:07:27 smithi083 ceph-mon[77242]: pgmap v17373: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:27.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:07:27 smithi083 ceph-mon[79930]: pgmap v17373: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:27.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:07:27 smithi153 ceph-mon[70044]: pgmap v17373: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:28.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:07:28 smithi083 ceph-mon[79930]: pgmap v17374: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:28.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:07:28 smithi083 ceph-mon[77242]: pgmap v17374: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:28.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:07:28 smithi153 ceph-mon[70044]: pgmap v17374: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:31.710 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:07:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:07:31] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T15:07:31.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:07:31 smithi083 ceph-mon[79930]: pgmap v17375: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:31.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:07:31 smithi083 ceph-mon[77242]: pgmap v17375: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:31.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:07:31 smithi153 ceph-mon[70044]: pgmap v17375: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:32.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:07:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:07:32.331Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:07:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:07:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:07:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:07:33.051 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:07:33.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:07:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:07:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:07:33.342 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:07:33.343 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 8m ago 9h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:07:33.343 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 6m ago 9h 63.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:07:33.343 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 8m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:07:33.343 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 6m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:07:33.344 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 6m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:07:33.344 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 8m ago 9h 982M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:07:33.344 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 8m ago 9h 2088M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:07:33.344 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 6m ago 9h 909M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:07:33.345 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 8m ago 9h 1251M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:07:33.345 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 8m ago 9h 24.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:07:33.345 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 6m ago 9h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:07:33.345 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 8m ago 9h 1516M 2177M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:07:33.346 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 8m ago 9h 1503M 2177M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:07:33.346 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 8m ago 9h 1168M 2177M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:07:33.346 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 8m ago 9h 1473M 2177M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:07:33.346 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 6m ago 9h 1292M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:07:33.347 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 6m ago 9h 1622M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:07:33.347 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 6m ago 9h 1449M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:07:33.347 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 6m ago 9h 1724M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:07:33.347 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 6m ago 9h 173M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:07:33.706 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:07:33.707 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:07:33.707 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:07:33.707 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:07:33.708 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:07:33.708 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:07:33.708 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:07:33.709 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:07:33.709 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:07:33.709 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:07:33.709 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:07:33.710 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:07:33.710 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:07:33.711 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:07:33.711 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:07:33.711 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:07:33.711 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:07:33.712 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:07:33.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:07:33 smithi083 ceph-mon[79930]: pgmap v17376: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:33.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:07:33 smithi083 ceph-mon[77242]: pgmap v17376: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:33.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:07:33 smithi153 ceph-mon[70044]: pgmap v17376: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:34.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:07:34 smithi083 ceph-mon[79930]: from='client.64267 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:07:34.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:07:34 smithi083 ceph-mon[79930]: from='client.54342 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:07:34.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:07:34 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3926682265' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:07:34.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:07:34 smithi083 ceph-mon[77242]: from='client.64267 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:07:34.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:07:34 smithi083 ceph-mon[77242]: from='client.54342 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:07:34.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:07:34 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3926682265' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:07:34.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:07:34 smithi153 ceph-mon[70044]: from='client.64267 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:07:34.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:07:34 smithi153 ceph-mon[70044]: from='client.54342 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:07:34.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:07:34 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3926682265' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:07:35.692 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:07:35 smithi083 ceph-mon[79930]: pgmap v17377: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:35.693 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:07:35 smithi083 ceph-mon[77242]: pgmap v17377: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:35.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:07:35 smithi153 ceph-mon[70044]: pgmap v17377: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:35.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:07:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:07:35.692Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:07:36.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:07:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:07:36.434Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:07:37.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:07:37 smithi083 ceph-mon[79930]: pgmap v17378: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:37.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:07:37 smithi083 ceph-mon[77242]: pgmap v17378: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:37.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:07:37 smithi153 ceph-mon[70044]: pgmap v17378: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:39.710 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:07:39 smithi083 ceph-mon[77242]: pgmap v17379: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:39.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:07:39 smithi083 ceph-mon[79930]: pgmap v17379: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:39.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:07:39 smithi153 ceph-mon[70044]: pgmap v17379: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:41.710 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:07:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:07:41] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T15:07:41.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:07:41 smithi083 ceph-mon[79930]: pgmap v17380: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:41.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:07:41 smithi083 ceph-mon[77242]: pgmap v17380: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:41.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:07:41 smithi153 ceph-mon[70044]: pgmap v17380: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:42.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:07:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:07:42.332Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:07:42.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:07:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:07:42.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:07:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:07:42.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:07:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:07:42.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:07:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:07:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:07:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:07:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:07:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:07:42.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:07:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:07:42.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:07:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:07:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:07:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:07:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:07:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:07:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:07:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:07:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:07:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:07:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:07:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:07:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:07:43.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:07:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:07:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:07:43.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:07:43 smithi083 ceph-mon[79930]: pgmap v17381: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:43.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:07:43 smithi083 ceph-mon[77242]: pgmap v17381: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:43.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:07:43 smithi153 ceph-mon[70044]: pgmap v17381: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:45.695 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:07:45 smithi083 ceph-mon[79930]: pgmap v17382: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:45.695 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:07:45 smithi083 ceph-mon[77242]: pgmap v17382: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:45.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:07:45 smithi153 ceph-mon[70044]: pgmap v17382: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:45.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:07:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:07:45.693Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:07:46.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:07:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:07:46.435Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:07:47.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:07:47 smithi083 ceph-mon[79930]: pgmap v17383: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:47.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:07:47 smithi083 ceph-mon[77242]: pgmap v17383: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:47.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:07:47 smithi153 ceph-mon[70044]: pgmap v17383: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:49.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:07:49 smithi083 ceph-mon[79930]: pgmap v17384: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:49.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:07:49 smithi083 ceph-mon[77242]: pgmap v17384: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:49.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:07:49 smithi153 ceph-mon[70044]: pgmap v17384: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:51.710 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:07:51 smithi083 ceph-mon[77242]: pgmap v17385: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:51.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:07:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:07:51] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T15:07:51.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:07:51 smithi083 ceph-mon[79930]: pgmap v17385: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:51.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:07:51 smithi153 ceph-mon[70044]: pgmap v17385: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:52.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:07:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:07:52.333Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:07:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:07:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:07:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:07:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:07:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:07:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:07:53.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:07:53 smithi083 ceph-mon[79930]: pgmap v17386: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:53.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:07:53 smithi083 ceph-mon[77242]: pgmap v17386: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:53.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:07:53 smithi153 ceph-mon[70044]: pgmap v17386: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:54.710 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:07:54 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:07:54.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:07:54 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:07:54.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:07:54 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:07:54.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:07:54 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:07:54.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:07:54 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:07:54.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:07:54 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:07:54.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:07:54 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:07:54.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:07:54 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:07:54.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:07:54 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:07:55.693 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:07:55 smithi083 ceph-mon[79930]: pgmap v17387: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:55.693 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:07:55 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:07:55.694 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:07:55 smithi083 ceph-mon[77242]: pgmap v17387: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:55.694 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:07:55 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:07:55.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:07:55 smithi153 ceph-mon[70044]: pgmap v17387: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:55.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:07:55 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:07:55.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:07:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:07:55.693Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:07:56.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:07:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:07:56.435Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:07:57.710 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:07:57 smithi083 ceph-mon[77242]: pgmap v17388: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:57.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:07:57 smithi083 ceph-mon[79930]: pgmap v17388: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:57.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:07:57 smithi153 ceph-mon[70044]: pgmap v17388: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:59.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:07:59 smithi083 ceph-mon[79930]: pgmap v17389: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:59.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:07:59 smithi083 ceph-mon[77242]: pgmap v17389: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:07:59.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:07:59 smithi153 ceph-mon[70044]: pgmap v17389: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:00.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:08:00 smithi153 ceph-mon[70044]: pgmap v17390: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:00.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:08:00 smithi083 ceph-mon[79930]: pgmap v17390: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:00.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:08:00 smithi083 ceph-mon[77242]: pgmap v17390: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:01.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:08:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:08:01] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T15:08:02.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:08:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:08:02.334Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:08:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:08:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:08:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:08:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:08:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:08:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:08:03.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:08:03 smithi083 ceph-mon[79930]: pgmap v17391: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:03.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:08:03 smithi083 ceph-mon[77242]: pgmap v17391: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:03.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:08:03 smithi153 ceph-mon[70044]: pgmap v17391: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:04.034 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:08:04.321 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:08:04.321 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 9m ago 9h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:08:04.321 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 7m ago 9h 63.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:08:04.322 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 9m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:08:04.322 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 7m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:08:04.322 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 7m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:08:04.322 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 9m ago 9h 982M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:08:04.322 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 9m ago 9h 2088M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:08:04.323 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 7m ago 9h 909M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:08:04.323 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 9m ago 9h 1251M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:08:04.323 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 9m ago 9h 24.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:08:04.323 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 7m ago 9h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:08:04.324 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 9m ago 9h 1516M 2177M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:08:04.324 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 9m ago 9h 1503M 2177M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:08:04.324 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 9m ago 9h 1168M 2177M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:08:04.324 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 9m ago 9h 1473M 2177M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:08:04.325 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 7m ago 9h 1292M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:08:04.325 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 7m ago 9h 1622M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:08:04.325 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 7m ago 9h 1449M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:08:04.325 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 7m ago 9h 1724M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:08:04.326 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 7m ago 9h 173M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:08:04.679 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:08:04.679 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:08:04.679 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:08:04.680 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:08:04.680 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:08:04.680 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:08:04.681 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:08:04.681 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:08:04.681 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:08:04.681 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:08:04.682 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:08:04.682 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:08:04.682 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:08:04.682 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:08:04.683 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:08:04.683 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:08:04.683 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:08:04.683 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:08:05.694 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:08:05 smithi083 ceph-mon[79930]: from='client.64285 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:08:05.695 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:08:05 smithi083 ceph-mon[79930]: from='client.54360 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:08:05.695 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:08:05 smithi083 ceph-mon[79930]: pgmap v17392: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:05.695 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:08:05 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/351168604' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:08:05.695 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:08:05 smithi083 ceph-mon[77242]: from='client.64285 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:08:05.696 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:08:05 smithi083 ceph-mon[77242]: from='client.54360 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:08:05.696 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:08:05 smithi083 ceph-mon[77242]: pgmap v17392: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:05.696 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:08:05 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/351168604' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:08:05.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:08:05 smithi153 ceph-mon[70044]: from='client.64285 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:08:05.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:08:05 smithi153 ceph-mon[70044]: from='client.54360 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:08:05.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:08:05 smithi153 ceph-mon[70044]: pgmap v17392: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:05.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:08:05 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/351168604' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:08:05.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:08:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:08:05.694Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:08:06.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:08:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:08:06.435Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:08:07.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:08:07 smithi083 ceph-mon[79930]: pgmap v17393: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:07.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:08:07 smithi083 ceph-mon[77242]: pgmap v17393: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:07.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:08:07 smithi153 ceph-mon[70044]: pgmap v17393: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:09.710 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:08:09 smithi083 ceph-mon[77242]: pgmap v17394: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:09.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:08:09 smithi083 ceph-mon[79930]: pgmap v17394: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:09.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:08:09 smithi153 ceph-mon[70044]: pgmap v17394: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:11.710 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:08:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:08:11] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T15:08:11.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:08:11 smithi083 ceph-mon[79930]: pgmap v17395: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:11.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:08:11 smithi083 ceph-mon[77242]: pgmap v17395: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:11.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:08:11 smithi153 ceph-mon[70044]: pgmap v17395: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:12.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:08:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:08:12.335Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:08:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:08:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:08:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:08:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:08:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:08:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:08:13.710 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:08:13 smithi083 ceph-mon[77242]: pgmap v17396: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:13.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:08:13 smithi083 ceph-mon[79930]: pgmap v17396: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:13.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:08:13 smithi153 ceph-mon[70044]: pgmap v17396: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:15.695 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:08:15 smithi083 ceph-mon[79930]: pgmap v17397: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:15.696 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:08:15 smithi083 ceph-mon[77242]: pgmap v17397: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:15.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:08:15 smithi153 ceph-mon[70044]: pgmap v17397: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:15.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:08:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:08:15.695Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:08:16.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:08:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:08:16.436Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:08:17.710 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:08:17 smithi083 ceph-mon[77242]: pgmap v17398: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:17.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:08:17 smithi083 ceph-mon[79930]: pgmap v17398: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:17.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:08:17 smithi153 ceph-mon[70044]: pgmap v17398: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:19.710 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:08:19 smithi083 ceph-mon[77242]: pgmap v17399: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:19.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:08:19 smithi083 ceph-mon[79930]: pgmap v17399: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:19.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:08:19 smithi153 ceph-mon[70044]: pgmap v17399: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:21.710 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:08:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:08:21] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T15:08:21.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:08:21 smithi083 ceph-mon[79930]: pgmap v17400: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:21.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:08:21 smithi083 ceph-mon[77242]: pgmap v17400: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:21.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:08:21 smithi153 ceph-mon[70044]: pgmap v17400: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:22.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:08:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:08:22.335Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:08:22.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:08:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:08:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:08:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:08:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:08:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:08:23.710 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:08:23 smithi083 ceph-mon[77242]: pgmap v17401: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:23.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:08:23 smithi083 ceph-mon[79930]: pgmap v17401: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:23.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:08:23 smithi153 ceph-mon[70044]: pgmap v17401: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:25.696 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:08:25 smithi083 ceph-mon[79930]: pgmap v17402: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:25.697 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:08:25 smithi083 ceph-mon[77242]: pgmap v17402: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:25.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:08:25 smithi153 ceph-mon[70044]: pgmap v17402: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:25.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:08:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:08:25.696Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:08:26.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:08:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:08:26.436Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:08:27.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:08:27 smithi083 ceph-mon[79930]: pgmap v17403: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:27.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:08:27 smithi083 ceph-mon[77242]: pgmap v17403: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:27.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:08:27 smithi153 ceph-mon[70044]: pgmap v17403: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:29.710 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:08:29 smithi083 ceph-mon[77242]: pgmap v17404: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:29.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:08:29 smithi083 ceph-mon[79930]: pgmap v17404: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:29.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:08:29 smithi153 ceph-mon[70044]: pgmap v17404: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:31.710 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:08:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:08:31] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T15:08:31.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:08:31 smithi083 ceph-mon[79930]: pgmap v17405: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:31.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:08:31 smithi083 ceph-mon[77242]: pgmap v17405: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:31.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:08:31 smithi153 ceph-mon[70044]: pgmap v17405: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:32.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:08:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:08:32.335Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:08:32.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:08:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:08:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:08:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:08:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:08:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:08:33.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:08:33 smithi083 ceph-mon[79930]: pgmap v17406: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:33.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:08:33 smithi083 ceph-mon[77242]: pgmap v17406: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:33.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:08:33 smithi153 ceph-mon[70044]: pgmap v17406: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:35.006 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:08:35.295 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:08:35.296 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 9m ago 9h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:08:35.296 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 7m ago 9h 63.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:08:35.296 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 9m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:08:35.296 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 7m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:08:35.297 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 7m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:08:35.297 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 9m ago 9h 982M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:08:35.297 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 9m ago 9h 2088M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:08:35.297 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 7m ago 9h 909M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:08:35.298 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 9m ago 9h 1251M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:08:35.298 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 9m ago 9h 24.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:08:35.298 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 7m ago 9h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:08:35.298 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 9m ago 9h 1516M 2177M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:08:35.299 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 9m ago 9h 1503M 2177M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:08:35.299 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 9m ago 9h 1168M 2177M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:08:35.299 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 9m ago 9h 1473M 2177M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:08:35.299 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 7m ago 9h 1292M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:08:35.299 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 7m ago 9h 1622M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:08:35.300 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 7m ago 9h 1449M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:08:35.300 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 7m ago 9h 1724M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:08:35.300 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 7m ago 9h 173M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:08:35.653 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:08:35.653 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:08:35.653 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:08:35.654 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:08:35.654 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:08:35.654 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:08:35.654 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:08:35.655 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:08:35.655 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:08:35.655 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:08:35.656 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:08:35.656 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:08:35.656 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:08:35.656 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:08:35.657 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:08:35.657 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:08:35.657 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:08:35.657 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:08:35.696 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:08:35 smithi083 ceph-mon[79930]: pgmap v17407: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:35.697 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:08:35 smithi083 ceph-mon[77242]: pgmap v17407: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:35.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:08:35 smithi153 ceph-mon[70044]: pgmap v17407: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:35.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:08:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:08:35.696Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:08:36.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:08:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:08:36.439Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:08:36.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:08:36 smithi083 ceph-mon[77242]: from='client.54372 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:08:36.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:08:36 smithi083 ceph-mon[77242]: from='client.54378 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:08:36.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:08:36 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2201292707' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:08:36.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:08:36 smithi083 ceph-mon[79930]: from='client.54372 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:08:36.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:08:36 smithi083 ceph-mon[79930]: from='client.54378 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:08:36.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:08:36 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2201292707' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:08:36.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:08:36 smithi153 ceph-mon[70044]: from='client.54372 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:08:36.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:08:36 smithi153 ceph-mon[70044]: from='client.54378 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:08:36.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:08:36 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2201292707' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:08:37.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:08:37 smithi083 ceph-mon[79930]: pgmap v17408: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:37.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:08:37 smithi083 ceph-mon[77242]: pgmap v17408: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:37.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:08:37 smithi153 ceph-mon[70044]: pgmap v17408: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:39.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:08:39 smithi083 ceph-mon[79930]: pgmap v17409: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:39.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:08:39 smithi083 ceph-mon[77242]: pgmap v17409: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:39.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:08:39 smithi153 ceph-mon[70044]: pgmap v17409: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:41.710 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:08:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:08:41] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T15:08:41.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:08:41 smithi083 ceph-mon[79930]: pgmap v17410: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:41.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:08:41 smithi083 ceph-mon[77242]: pgmap v17410: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:41.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:08:41 smithi153 ceph-mon[70044]: pgmap v17410: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:42.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:08:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:08:42.336Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:08:42.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:08:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:08:42.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:08:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:08:42.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:08:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:08:42.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:08:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:08:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:08:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:08:42.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:08:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:08:42.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:08:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:08:42.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:08:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:08:42.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:08:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:08:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:08:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:08:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:08:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:08:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:08:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:08:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:08:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:08:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:08:43.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:08:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:08:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:08:43.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:08:43 smithi083 ceph-mon[79930]: pgmap v17411: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:43.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:08:43 smithi083 ceph-mon[77242]: pgmap v17411: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:43.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:08:43 smithi153 ceph-mon[70044]: pgmap v17411: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:44.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:08:44 smithi153 ceph-mon[70044]: pgmap v17412: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:44.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:08:44 smithi083 ceph-mon[79930]: pgmap v17412: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:44.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:08:44 smithi083 ceph-mon[77242]: pgmap v17412: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:45.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:08:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:08:45.697Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:08:46.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:08:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:08:46.438Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:08:47.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:08:47 smithi083 ceph-mon[79930]: pgmap v17413: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:47.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:08:47 smithi083 ceph-mon[77242]: pgmap v17413: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:47.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:08:47 smithi153 ceph-mon[70044]: pgmap v17413: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:49.710 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:08:49 smithi083 ceph-mon[77242]: pgmap v17414: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:49.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:08:49 smithi083 ceph-mon[79930]: pgmap v17414: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:49.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:08:49 smithi153 ceph-mon[70044]: pgmap v17414: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:51.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:08:51 smithi083 ceph-mon[79930]: pgmap v17415: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:51.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:08:51 smithi083 ceph-mon[77242]: pgmap v17415: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:51.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:08:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:08:51] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T15:08:51.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:08:51 smithi153 ceph-mon[70044]: pgmap v17415: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:52.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:08:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:08:52.337Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:08:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:08:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:08:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:08:53.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:08:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:08:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:08:53.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:08:53 smithi083 ceph-mon[79930]: pgmap v17416: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:53.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:08:53 smithi083 ceph-mon[77242]: pgmap v17416: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:53.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:08:53 smithi153 ceph-mon[70044]: pgmap v17416: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:55.697 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:08:55 smithi083 ceph-mon[79930]: pgmap v17417: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:55.698 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:08:55 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:08:55.698 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:08:55 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:08:55.698 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:08:55 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:08:55.699 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:08:55 smithi083 ceph-mon[77242]: pgmap v17417: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:55.699 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:08:55 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:08:55.699 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:08:55 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:08:55.700 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:08:55 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:08:55.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:08:55 smithi153 ceph-mon[70044]: pgmap v17417: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:55.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:08:55 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:08:55.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:08:55 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:08:55.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:08:55 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:08:55.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:08:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:08:55.697Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:08:56.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:08:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:08:56.437Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:08:56.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:08:56 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:08:56.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:08:56 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:08:56.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:08:56 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:08:56.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:08:56 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:08:56.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:08:56 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:08:56.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:08:56 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:08:56.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:08:56 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:08:56.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:08:56 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:08:56.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:08:56 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:08:56.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:08:56 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:08:56.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:08:56 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:08:56.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:08:56 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:08:57.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:08:57 smithi083 ceph-mon[79930]: pgmap v17418: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:57.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:08:57 smithi083 ceph-mon[77242]: pgmap v17418: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:57.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:08:57 smithi153 ceph-mon[70044]: pgmap v17418: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:59.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:08:59 smithi083 ceph-mon[79930]: pgmap v17419: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:59.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:08:59 smithi083 ceph-mon[77242]: pgmap v17419: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:08:59.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:08:59 smithi153 ceph-mon[70044]: pgmap v17419: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:00.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:09:00 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:09:00.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:09:00 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:09:00.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:09:00 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:09:00.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:09:00 smithi083 ceph-mon[79930]: pgmap v17420: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:00.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:09:00 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:09:00.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:09:00 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:09:00.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:09:00 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:09:00.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:09:00 smithi083 ceph-mon[77242]: pgmap v17420: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:01.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:09:00 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:09:01.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:09:00 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:09:01.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:09:00 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:09:01.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:09:00 smithi153 ceph-mon[70044]: pgmap v17420: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:01.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:09:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:09:01] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T15:09:02.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:09:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:09:02.338Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:09:02.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:09:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:09:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:09:03.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:09:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:09:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:09:03.710 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:09:03 smithi083 ceph-mon[77242]: pgmap v17421: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:03.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:09:03 smithi083 ceph-mon[79930]: pgmap v17421: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:03.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:09:03 smithi153 ceph-mon[70044]: pgmap v17421: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:05.701 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:09:05 smithi083 ceph-mon[77242]: pgmap v17422: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:05.701 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:09:05 smithi083 ceph-mon[79930]: pgmap v17422: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:05.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:09:05 smithi153 ceph-mon[70044]: pgmap v17422: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:05.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:09:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:09:05.700Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:09:05.983 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:09:06.273 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:09:06.273 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 6s ago 9h 27.3M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:09:06.274 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 8m ago 9h 63.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:09:06.274 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 6s ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:09:06.274 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 8m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:09:06.274 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 8m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:09:06.275 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 6s ago 9h 990M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:09:06.275 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 6s ago 9h 2124M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:09:06.275 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 8m ago 9h 909M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:09:06.275 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 6s ago 9h 1273M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:09:06.276 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 6s ago 9h 24.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:09:06.276 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 8m ago 9h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:09:06.276 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 6s ago 9h 1543M 2177M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:09:06.276 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 6s ago 9h 1529M 2177M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:09:06.277 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 6s ago 9h 1187M 2177M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:09:06.277 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 6s ago 9h 1499M 2177M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:09:06.277 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 8m ago 9h 1292M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:09:06.277 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 8m ago 9h 1622M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:09:06.277 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 8m ago 9h 1449M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:09:06.278 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 8m ago 9h 1724M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:09:06.278 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 8m ago 9h 173M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:09:06.630 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:09:06.630 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:09:06.631 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:09:06.631 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:09:06.631 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:09:06.631 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:09:06.632 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:09:06.632 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:09:06.632 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:09:06.632 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:09:06.633 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:09:06.633 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:09:06.633 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:09:06.633 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:09:06.634 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:09:06.634 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:09:06.634 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:09:06.634 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:09:06.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:09:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:09:06.438Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:09:07.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:09:07 smithi083 ceph-mon[79930]: from='client.54390 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:09:07.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:09:07 smithi083 ceph-mon[79930]: from='client.64327 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:09:07.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:09:07 smithi083 ceph-mon[79930]: pgmap v17423: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:07.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:09:07 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4083351118' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:09:07.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:09:07 smithi083 ceph-mon[77242]: from='client.54390 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:09:07.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:09:07 smithi083 ceph-mon[77242]: from='client.64327 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:09:07.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:09:07 smithi083 ceph-mon[77242]: pgmap v17423: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:07.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:09:07 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4083351118' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:09:07.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:09:07 smithi153 ceph-mon[70044]: from='client.54390 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:09:07.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:09:07 smithi153 ceph-mon[70044]: from='client.64327 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:09:07.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:09:07 smithi153 ceph-mon[70044]: pgmap v17423: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:07.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:09:07 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4083351118' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:09:09.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:09:09 smithi083 ceph-mon[79930]: pgmap v17424: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:09.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:09:09 smithi083 ceph-mon[77242]: pgmap v17424: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:09.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:09:09 smithi153 ceph-mon[70044]: pgmap v17424: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:11.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:09:11 smithi083 ceph-mon[79930]: pgmap v17425: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:11.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:09:11 smithi083 ceph-mon[77242]: pgmap v17425: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:11.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:09:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:09:11] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T15:09:11.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:09:11 smithi153 ceph-mon[70044]: pgmap v17425: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:12.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:09:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:09:12.339Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:09:12.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:09:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:09:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:09:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:09:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:09:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:09:13.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:09:13 smithi083 ceph-mon[79930]: pgmap v17426: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:13.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:09:13 smithi083 ceph-mon[77242]: pgmap v17426: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:13.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:09:13 smithi153 ceph-mon[70044]: pgmap v17426: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:15.699 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:09:15 smithi083 ceph-mon[79930]: pgmap v17427: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:15.699 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:09:15 smithi083 ceph-mon[77242]: pgmap v17427: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:15.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:09:15 smithi153 ceph-mon[70044]: pgmap v17427: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:15.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:09:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:09:15.698Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:09:16.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:09:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:09:16.440Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:09:17.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:09:17 smithi083 ceph-mon[77242]: pgmap v17428: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:17.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:09:17 smithi083 ceph-mon[79930]: pgmap v17428: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:17.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:09:17 smithi153 ceph-mon[70044]: pgmap v17428: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:19.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:09:19 smithi083 ceph-mon[79930]: pgmap v17429: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:19.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:09:19 smithi083 ceph-mon[77242]: pgmap v17429: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:19.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:09:19 smithi153 ceph-mon[70044]: pgmap v17429: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:21.710 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:09:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:09:21] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T15:09:21.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:09:21 smithi083 ceph-mon[79930]: pgmap v17430: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:21.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:09:21 smithi083 ceph-mon[77242]: pgmap v17430: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:21.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:09:21 smithi153 ceph-mon[70044]: pgmap v17430: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:22.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:09:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:09:22.340Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:09:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:09:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:09:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:09:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:09:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:09:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:09:23.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:09:23 smithi083 ceph-mon[79930]: pgmap v17431: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:23.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:09:23 smithi083 ceph-mon[77242]: pgmap v17431: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:23.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:09:23 smithi153 ceph-mon[70044]: pgmap v17431: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:25.699 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:09:25 smithi083 ceph-mon[79930]: pgmap v17432: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:25.700 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:09:25 smithi083 ceph-mon[77242]: pgmap v17432: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:25.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:09:25 smithi153 ceph-mon[70044]: pgmap v17432: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:25.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:09:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:09:25.699Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:09:26.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:09:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:09:26.440Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:09:27.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:09:27 smithi083 ceph-mon[79930]: pgmap v17433: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:27.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:09:27 smithi083 ceph-mon[77242]: pgmap v17433: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:27.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:09:27 smithi153 ceph-mon[70044]: pgmap v17433: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:29.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:09:29 smithi083 ceph-mon[79930]: pgmap v17434: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:29.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:09:29 smithi083 ceph-mon[77242]: pgmap v17434: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:29.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:09:29 smithi153 ceph-mon[70044]: pgmap v17434: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:31.710 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:09:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:09:31] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T15:09:31.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:09:31 smithi083 ceph-mon[79930]: pgmap v17435: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:31.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:09:31 smithi083 ceph-mon[77242]: pgmap v17435: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:31.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:09:31 smithi153 ceph-mon[70044]: pgmap v17435: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:32.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:09:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:09:32.341Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:09:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:09:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:09:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:09:33.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:09:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:09:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:09:33.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:09:33 smithi083 ceph-mon[79930]: pgmap v17436: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:33.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:09:33 smithi083 ceph-mon[77242]: pgmap v17436: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:33.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:09:33 smithi153 ceph-mon[70044]: pgmap v17436: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:35.700 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:09:35 smithi083 ceph-mon[79930]: pgmap v17437: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:35.700 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:09:35 smithi083 ceph-mon[77242]: pgmap v17437: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:35.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:09:35 smithi153 ceph-mon[70044]: pgmap v17437: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:35.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:09:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:09:35.699Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:09:36.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:09:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:09:36.440Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:09:36.958 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:09:37.246 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:09:37.247 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 37s ago 9h 27.3M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:09:37.247 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 8m ago 9h 63.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:09:37.247 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 37s ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:09:37.247 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 8m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:09:37.248 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 8m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:09:37.248 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 37s ago 9h 990M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:09:37.248 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 37s ago 9h 2124M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:09:37.248 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 8m ago 9h 909M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:09:37.249 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 37s ago 9h 1273M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:09:37.249 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 37s ago 9h 24.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:09:37.249 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 8m ago 9h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:09:37.249 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 37s ago 9h 1543M 2177M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:09:37.250 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 37s ago 9h 1529M 2177M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:09:37.250 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 37s ago 9h 1187M 2177M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:09:37.250 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 37s ago 9h 1499M 2177M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:09:37.250 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 8m ago 9h 1292M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:09:37.251 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 8m ago 9h 1622M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:09:37.251 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 8m ago 9h 1449M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:09:37.251 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 8m ago 9h 1724M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:09:37.251 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 8m ago 9h 173M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:09:37.602 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:09:37.603 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:09:37.603 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:09:37.603 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:09:37.604 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:09:37.604 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:09:37.604 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:09:37.604 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:09:37.604 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:09:37.605 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:09:37.605 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:09:37.605 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:09:37.605 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:09:37.606 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:09:37.606 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:09:37.606 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:09:37.606 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:09:37.607 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:09:37.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:09:37 smithi083 ceph-mon[79930]: pgmap v17438: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:37.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:09:37 smithi083 ceph-mon[77242]: pgmap v17438: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:37.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:09:37 smithi153 ceph-mon[70044]: pgmap v17438: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:38.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:09:38 smithi083 ceph-mon[79930]: from='client.54408 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:09:38.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:09:38 smithi083 ceph-mon[79930]: from='client.54414 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:09:38.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:09:38 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2292983885' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:09:38.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:09:38 smithi083 ceph-mon[77242]: from='client.54408 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:09:38.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:09:38 smithi083 ceph-mon[77242]: from='client.54414 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:09:38.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:09:38 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2292983885' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:09:38.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:09:38 smithi153 ceph-mon[70044]: from='client.54408 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:09:38.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:09:38 smithi153 ceph-mon[70044]: from='client.54414 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:09:38.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:09:38 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2292983885' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:09:39.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:09:39 smithi083 ceph-mon[79930]: pgmap v17439: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:39.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:09:39 smithi083 ceph-mon[77242]: pgmap v17439: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:39.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:09:39 smithi153 ceph-mon[70044]: pgmap v17439: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:40.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:09:40 smithi153 ceph-mon[70044]: pgmap v17440: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:40.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:09:40 smithi083 ceph-mon[79930]: pgmap v17440: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:40.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:09:40 smithi083 ceph-mon[77242]: pgmap v17440: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:09:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:09:41] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T15:09:41.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:09:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:09:41.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:09:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:09:41.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:09:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:09:41.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:09:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:09:41.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:09:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:09:41.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:09:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:09:41.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:09:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:09:41.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:09:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:09:42.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:09:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:09:42.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:09:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:09:42.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:09:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:09:42.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:09:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:09:42.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:09:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:09:42.341Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:09:42.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:09:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:09:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:09:43.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:09:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:09:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:09:43.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:09:43 smithi083 ceph-mon[79930]: pgmap v17441: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:43.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:09:43 smithi083 ceph-mon[77242]: pgmap v17441: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:43.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:09:43 smithi153 ceph-mon[70044]: pgmap v17441: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:45.701 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:09:45 smithi083 ceph-mon[79930]: pgmap v17442: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:45.701 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:09:45 smithi083 ceph-mon[77242]: pgmap v17442: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:45.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:09:45 smithi153 ceph-mon[70044]: pgmap v17442: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:45.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:09:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:09:45.700Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:09:46.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:09:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:09:46.441Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:09:47.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:09:47 smithi083 ceph-mon[79930]: pgmap v17443: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:47.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:09:47 smithi083 ceph-mon[77242]: pgmap v17443: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:47.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:09:47 smithi153 ceph-mon[70044]: pgmap v17443: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:49.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:09:49 smithi083 ceph-mon[79930]: pgmap v17444: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:49.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:09:49 smithi083 ceph-mon[77242]: pgmap v17444: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:49.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:09:49 smithi153 ceph-mon[70044]: pgmap v17444: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:51.710 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:09:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:09:51] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T15:09:51.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:09:51 smithi083 ceph-mon[79930]: pgmap v17445: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:51.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:09:51 smithi083 ceph-mon[77242]: pgmap v17445: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:51.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:09:51 smithi153 ceph-mon[70044]: pgmap v17445: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:52.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:09:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:09:52.342Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:09:52.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:09:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:09:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:09:53.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:09:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:09:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:09:53.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:09:53 smithi083 ceph-mon[79930]: pgmap v17446: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:53.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:09:53 smithi083 ceph-mon[77242]: pgmap v17446: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:53.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:09:53 smithi153 ceph-mon[70044]: pgmap v17446: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:55.702 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:09:55 smithi083 ceph-mon[79930]: pgmap v17447: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:55.702 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:09:55 smithi083 ceph-mon[77242]: pgmap v17447: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:55.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:09:55 smithi153 ceph-mon[70044]: pgmap v17447: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:55.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:09:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:09:55.701Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:09:56.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:09:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:09:56.442Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:09:57.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:09:57 smithi083 ceph-mon[79930]: pgmap v17448: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:57.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:09:57 smithi083 ceph-mon[77242]: pgmap v17448: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:57.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:09:57 smithi153 ceph-mon[70044]: pgmap v17448: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:59.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:09:59 smithi083 ceph-mon[79930]: pgmap v17449: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:59.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:09:59 smithi083 ceph-mon[77242]: pgmap v17449: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:09:59.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:09:59 smithi153 ceph-mon[70044]: pgmap v17449: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:00.424 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:10:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[77219]: 2022-09-29T15:09:59.999+0000 7f160f665700 -1 log_channel(cluster) log [ERR] : overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T15:10:00.424 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:10:00 smithi083 ceph-mon[77242]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T15:10:00.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:10:00 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:10:00.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:10:00 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:10:00.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:10:00 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:10:00.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:10:00 smithi083 ceph-mon[79930]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T15:10:00.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:10:00 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:10:00.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:10:00 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:10:00.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:10:00 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:10:00.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:10:00 smithi153 ceph-mon[70044]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T15:10:00.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:10:00 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:10:00.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:10:00 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:10:00.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:10:00 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:10:01.710 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:10:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:10:01] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T15:10:01.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:10:01 smithi083 ceph-mon[77242]: pgmap v17450: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:01.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:10:01 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:10:01.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:10:01 smithi083 ceph-mon[79930]: pgmap v17450: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:01.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:10:01 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:10:01.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:10:01 smithi153 ceph-mon[70044]: pgmap v17450: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:01.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:10:01 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:10:02.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:10:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:10:02.343Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:10:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:10:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:10:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:10:03.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:10:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:10:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:10:03.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:10:03 smithi083 ceph-mon[79930]: pgmap v17451: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:03.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:10:03 smithi083 ceph-mon[77242]: pgmap v17451: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:03.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:10:03 smithi153 ceph-mon[70044]: pgmap v17451: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:05.703 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:10:05 smithi083 ceph-mon[79930]: pgmap v17452: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:05.703 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:10:05 smithi083 ceph-mon[77242]: pgmap v17452: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:05.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:10:05 smithi153 ceph-mon[70044]: pgmap v17452: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:05.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:10:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:10:05.702Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:10:06.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:10:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:10:06.443Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:10:07.710 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:10:07 smithi083 ceph-mon[77242]: pgmap v17453: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:07.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:10:07 smithi083 ceph-mon[79930]: pgmap v17453: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:07.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:10:07 smithi153 ceph-mon[70044]: pgmap v17453: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:07.930 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:10:08.215 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:10:08.215 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 68s ago 9h 27.3M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:10:08.215 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 9m ago 9h 63.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:10:08.216 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 68s ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:10:08.216 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 9m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:10:08.216 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 9m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:10:08.216 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 68s ago 9h 990M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:10:08.217 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 68s ago 9h 2124M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:10:08.217 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 9m ago 9h 909M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:10:08.217 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 68s ago 9h 1273M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:10:08.218 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 68s ago 9h 24.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:10:08.218 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 9m ago 9h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:10:08.218 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 68s ago 9h 1543M 2177M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:10:08.219 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 68s ago 9h 1529M 2177M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:10:08.219 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 68s ago 9h 1187M 2177M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:10:08.219 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 68s ago 9h 1499M 2177M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:10:08.219 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 9m ago 9h 1292M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:10:08.220 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 9m ago 9h 1622M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:10:08.220 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 9m ago 9h 1449M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:10:08.220 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 9m ago 9h 1724M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:10:08.221 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 9m ago 9h 173M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:10:08.571 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:10:08.571 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:10:08.572 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:10:08.572 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:10:08.572 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:10:08.572 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:10:08.572 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:10:08.573 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:10:08.573 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:10:08.573 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:10:08.573 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:10:08.574 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:10:08.574 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:10:08.574 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:10:08.575 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:10:08.575 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:10:08.575 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:10:08.575 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:10:09.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:10:09 smithi083 ceph-mon[79930]: from='client.54426 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:10:09.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:10:09 smithi083 ceph-mon[79930]: from='client.64363 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:10:09.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:10:09 smithi083 ceph-mon[79930]: pgmap v17454: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:09.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:10:09 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2190675979' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:10:09.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:10:09 smithi083 ceph-mon[77242]: from='client.54426 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:10:09.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:10:09 smithi083 ceph-mon[77242]: from='client.64363 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:10:09.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:10:09 smithi083 ceph-mon[77242]: pgmap v17454: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:09.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:10:09 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2190675979' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:10:09.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:10:09 smithi153 ceph-mon[70044]: from='client.54426 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:10:09.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:10:09 smithi153 ceph-mon[70044]: from='client.64363 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:10:09.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:10:09 smithi153 ceph-mon[70044]: pgmap v17454: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:09.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:10:09 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2190675979' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:10:11.710 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:10:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:10:11] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T15:10:11.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:10:11 smithi083 ceph-mon[79930]: pgmap v17455: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:11.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:10:11 smithi083 ceph-mon[77242]: pgmap v17455: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:11.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:10:11 smithi153 ceph-mon[70044]: pgmap v17455: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:12.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:10:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:10:12.344Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:10:12.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:10:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:10:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:10:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:10:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:10:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:10:13.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:10:13 smithi083 ceph-mon[79930]: pgmap v17456: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:13.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:10:13 smithi083 ceph-mon[77242]: pgmap v17456: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:13.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:10:13 smithi153 ceph-mon[70044]: pgmap v17456: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:15.703 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:10:15 smithi083 ceph-mon[79930]: pgmap v17457: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:15.703 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:10:15 smithi083 ceph-mon[77242]: pgmap v17457: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:15.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:10:15 smithi153 ceph-mon[70044]: pgmap v17457: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:15.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:10:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:10:15.702Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:10:16.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:10:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:10:16.443Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:10:17.710 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:10:17 smithi083 ceph-mon[77242]: pgmap v17458: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:17.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:10:17 smithi083 ceph-mon[79930]: pgmap v17458: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:17.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:10:17 smithi153 ceph-mon[70044]: pgmap v17458: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:19.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:10:19 smithi083 ceph-mon[79930]: pgmap v17459: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:19.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:10:19 smithi083 ceph-mon[77242]: pgmap v17459: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:19.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:10:19 smithi153 ceph-mon[70044]: pgmap v17459: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:21.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:10:21 smithi083 ceph-mon[79930]: pgmap v17460: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:21.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:10:21 smithi083 ceph-mon[77242]: pgmap v17460: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:21.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:10:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:10:21] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T15:10:21.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:10:21 smithi153 ceph-mon[70044]: pgmap v17460: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:22.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:10:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:10:22.344Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:10:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:10:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:10:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:10:23.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:10:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:10:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:10:23.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:10:23 smithi153 ceph-mon[70044]: pgmap v17461: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:23.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:10:23 smithi083 ceph-mon[79930]: pgmap v17461: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:23.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:10:23 smithi083 ceph-mon[77242]: pgmap v17461: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:24.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:10:24 smithi153 ceph-mon[70044]: pgmap v17462: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:24.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:10:24 smithi083 ceph-mon[79930]: pgmap v17462: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:24.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:10:24 smithi083 ceph-mon[77242]: pgmap v17462: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:25.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:10:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:10:25.703Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:10:26.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:10:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:10:26.444Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:10:27.710 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:10:27 smithi083 ceph-mon[77242]: pgmap v17463: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:27.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:10:27 smithi083 ceph-mon[79930]: pgmap v17463: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:27.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:10:27 smithi153 ceph-mon[70044]: pgmap v17463: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:29.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:10:29 smithi083 ceph-mon[79930]: pgmap v17464: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:29.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:10:29 smithi083 ceph-mon[77242]: pgmap v17464: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:29.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:10:29 smithi153 ceph-mon[70044]: pgmap v17464: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:31.710 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:10:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:10:31] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T15:10:31.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:10:31 smithi083 ceph-mon[79930]: pgmap v17465: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:31.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:10:31 smithi083 ceph-mon[77242]: pgmap v17465: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:31.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:10:31 smithi153 ceph-mon[70044]: pgmap v17465: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:32.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:10:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:10:32.344Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:10:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:10:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:10:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:10:33.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:10:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:10:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:10:33.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:10:33 smithi083 ceph-mon[79930]: pgmap v17466: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:33.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:10:33 smithi083 ceph-mon[77242]: pgmap v17466: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:33.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:10:33 smithi153 ceph-mon[70044]: pgmap v17466: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:35.705 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:10:35 smithi083 ceph-mon[79930]: pgmap v17467: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:35.706 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:10:35 smithi083 ceph-mon[77242]: pgmap v17467: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:35.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:10:35 smithi153 ceph-mon[70044]: pgmap v17467: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:35.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:10:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:10:35.705Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:10:36.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:10:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:10:36.445Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:10:37.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:10:37 smithi083 ceph-mon[79930]: pgmap v17468: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:37.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:10:37 smithi083 ceph-mon[77242]: pgmap v17468: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:37.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:10:37 smithi153 ceph-mon[70044]: pgmap v17468: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:38.898 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:10:39.187 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:10:39.188 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 99s ago 9h 27.3M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:10:39.188 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 9m ago 9h 63.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:10:39.188 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 99s ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:10:39.188 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 9m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:10:39.189 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 9m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:10:39.189 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 99s ago 9h 990M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:10:39.189 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 99s ago 9h 2124M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:10:39.189 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 9m ago 9h 909M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:10:39.190 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 99s ago 9h 1273M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:10:39.190 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 99s ago 9h 24.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:10:39.190 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 9m ago 9h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:10:39.190 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 99s ago 9h 1543M 2177M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:10:39.191 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 99s ago 9h 1529M 2177M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:10:39.191 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 99s ago 9h 1187M 2177M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:10:39.191 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 99s ago 9h 1499M 2177M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:10:39.191 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 9m ago 9h 1292M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:10:39.192 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 9m ago 9h 1622M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:10:39.192 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 9m ago 9h 1449M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:10:39.192 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 9m ago 9h 1724M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:10:39.192 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 9m ago 9h 173M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:10:39.545 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:10:39.546 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:10:39.546 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:10:39.546 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:10:39.546 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:10:39.547 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:10:39.547 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:10:39.547 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:10:39.547 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:10:39.548 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:10:39.548 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:10:39.548 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:10:39.548 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:10:39.549 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:10:39.549 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:10:39.549 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:10:39.549 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:10:39.550 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:10:39.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:10:39 smithi083 ceph-mon[79930]: pgmap v17469: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:39.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:10:39 smithi083 ceph-mon[77242]: pgmap v17469: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:39.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:10:39 smithi153 ceph-mon[70044]: pgmap v17469: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:40.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:10:40 smithi083 ceph-mon[79930]: from='client.64375 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:10:40.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:10:40 smithi083 ceph-mon[79930]: from='client.54450 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:10:40.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:10:40 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2311911686' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:10:40.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:10:40 smithi083 ceph-mon[77242]: from='client.64375 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:10:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:10:40 smithi083 ceph-mon[77242]: from='client.54450 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:10:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:10:40 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2311911686' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:10:40.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:10:40 smithi153 ceph-mon[70044]: from='client.64375 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:10:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:10:40 smithi153 ceph-mon[70044]: from='client.54450 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:10:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:10:40 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2311911686' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:10:41.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:10:41 smithi083 ceph-mon[79930]: pgmap v17470: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:41.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:10:41 smithi083 ceph-mon[77242]: pgmap v17470: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:41.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:10:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:10:41] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T15:10:41.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:10:41 smithi153 ceph-mon[70044]: pgmap v17470: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:42.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:10:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:10:42.346Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:10:42.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:10:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:10:42.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:10:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:10:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:10:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:10:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:10:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:10:42.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:10:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:10:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:10:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:10:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:10:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:10:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:10:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:10:42.811 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:10:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:10:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:10:42.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:10:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:10:42.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:10:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:10:42.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:10:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:10:42.813 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:10:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:10:43.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:10:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:10:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:10:43.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:10:43 smithi083 ceph-mon[79930]: pgmap v17471: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:43.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:10:43 smithi083 ceph-mon[77242]: pgmap v17471: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:43.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:10:43 smithi153 ceph-mon[70044]: pgmap v17471: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:45.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:10:45 smithi153 ceph-mon[70044]: pgmap v17472: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:45.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:10:45 smithi083 ceph-mon[79930]: pgmap v17472: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:45.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:10:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:10:45.705Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:10:45.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:10:45 smithi083 ceph-mon[77242]: pgmap v17472: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:46.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:10:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:10:46.446Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:10:47.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:10:47 smithi153 ceph-mon[70044]: pgmap v17473: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:47.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:10:47 smithi083 ceph-mon[79930]: pgmap v17473: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:47.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:10:47 smithi083 ceph-mon[77242]: pgmap v17473: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:48.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:10:48 smithi153 ceph-mon[70044]: pgmap v17474: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:48.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:10:48 smithi083 ceph-mon[79930]: pgmap v17474: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:48.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:10:48 smithi083 ceph-mon[77242]: pgmap v17474: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:51.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:10:51 smithi083 ceph-mon[79930]: pgmap v17475: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:51.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:10:51 smithi083 ceph-mon[77242]: pgmap v17475: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:51.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:10:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:10:51] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T15:10:51.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:10:51 smithi153 ceph-mon[70044]: pgmap v17475: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:52.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:10:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:10:52.346Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:10:52.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:10:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:10:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:10:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:10:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:10:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:10:53.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:10:53 smithi083 ceph-mon[79930]: pgmap v17476: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:53.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:10:53 smithi083 ceph-mon[77242]: pgmap v17476: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:53.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:10:53 smithi153 ceph-mon[70044]: pgmap v17476: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:55.706 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:10:55 smithi083 ceph-mon[79930]: pgmap v17477: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:55.707 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:10:55 smithi083 ceph-mon[77242]: pgmap v17477: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:55.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:10:55 smithi153 ceph-mon[70044]: pgmap v17477: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:55.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:10:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:10:55.705Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:10:56.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:10:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:10:56.447Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:10:57.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:10:57 smithi083 ceph-mon[79930]: pgmap v17478: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:57.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:10:57 smithi083 ceph-mon[77242]: pgmap v17478: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:57.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:10:57 smithi153 ceph-mon[70044]: pgmap v17478: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:59.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:10:59 smithi083 ceph-mon[79930]: pgmap v17479: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:59.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:10:59 smithi083 ceph-mon[77242]: pgmap v17479: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:10:59.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:10:59 smithi153 ceph-mon[70044]: pgmap v17479: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:01.710 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:11:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:11:01] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T15:11:01.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:11:01 smithi083 ceph-mon[77242]: pgmap v17480: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:01.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:11:01 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:11:01.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:11:01 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:11:01.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:11:01 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:11:01.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:11:01 smithi083 ceph-mon[79930]: pgmap v17480: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:01.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:11:01 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:11:01.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:11:01 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:11:01.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:11:01 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:11:01.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:11:01 smithi153 ceph-mon[70044]: pgmap v17480: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:01.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:11:01 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:11:01.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:11:01 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:11:01.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:11:01 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:11:02.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:11:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:11:02.347Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:11:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:11:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:11:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:11:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:11:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:11:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:11:03.710 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:11:03 smithi083 ceph-mon[77242]: pgmap v17481: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:03.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:11:03 smithi083 ceph-mon[79930]: pgmap v17481: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:03.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:11:03 smithi153 ceph-mon[70044]: pgmap v17481: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:05.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:11:05 smithi153 ceph-mon[70044]: pgmap v17482: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:05.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:11:05 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:11:05.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:11:05 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:11:05.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:11:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:11:05.707Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:11:05.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:11:05 smithi083 ceph-mon[77242]: pgmap v17482: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:05.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:11:05 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:11:05.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:11:05 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:11:05.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:11:05 smithi083 ceph-mon[79930]: pgmap v17482: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:05.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:11:05 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:11:05.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:11:05 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:11:06.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:11:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:11:06.447Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:11:07.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:11:06 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:11:07.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:11:06 smithi153 ceph-mon[70044]: pgmap v17483: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:07.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:11:06 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:11:07.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:11:06 smithi083 ceph-mon[79930]: pgmap v17483: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:07.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:11:06 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:11:07.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:11:06 smithi083 ceph-mon[77242]: pgmap v17483: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:09.710 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:11:09 smithi083 ceph-mon[77242]: pgmap v17484: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:09.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:11:09 smithi083 ceph-mon[79930]: pgmap v17484: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:09.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:11:09 smithi153 ceph-mon[70044]: pgmap v17484: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:09.875 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:11:10.163 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:11:10.163 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 2m ago 9h 27.3M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:11:10.163 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 4s ago 9h 64.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:11:10.163 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 2m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:11:10.164 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 4s ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:11:10.164 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 4s ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:11:10.164 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 2m ago 9h 990M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:11:10.164 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 2m ago 9h 2124M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:11:10.165 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 4s ago 9h 926M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:11:10.165 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 2m ago 9h 1273M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:11:10.165 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 2m ago 9h 24.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:11:10.165 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 4s ago 9h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:11:10.166 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 2m ago 9h 1543M 2177M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:11:10.166 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 2m ago 9h 1529M 2177M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:11:10.166 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 2m ago 9h 1187M 2177M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:11:10.166 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 2m ago 9h 1499M 2177M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:11:10.166 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 4s ago 9h 1313M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:11:10.167 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 4s ago 9h 1649M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:11:10.167 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 4s ago 9h 1474M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:11:10.167 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 4s ago 9h 1754M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:11:10.167 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 4s ago 9h 174M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:11:10.517 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:11:10.517 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:11:10.518 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:11:10.518 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:11:10.518 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:11:10.519 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:11:10.519 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:11:10.519 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:11:10.519 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:11:10.520 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:11:10.520 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:11:10.520 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:11:10.521 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:11:10.521 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:11:10.521 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:11:10.521 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:11:10.522 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:11:10.522 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:11:11.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:11:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:11:11] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T15:11:11.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:11:11 smithi083 ceph-mon[79930]: from='client.54462 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:11:11.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:11:11 smithi083 ceph-mon[79930]: from='client.64399 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:11:11.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:11:11 smithi083 ceph-mon[79930]: pgmap v17485: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:11.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:11:11 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/463414449' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:11:11.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:11:11 smithi083 ceph-mon[77242]: from='client.54462 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:11:11.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:11:11 smithi083 ceph-mon[77242]: from='client.64399 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:11:11.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:11:11 smithi083 ceph-mon[77242]: pgmap v17485: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:11.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:11:11 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/463414449' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:11:11.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:11:11 smithi153 ceph-mon[70044]: from='client.54462 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:11:11.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:11:11 smithi153 ceph-mon[70044]: from='client.64399 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:11:11.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:11:11 smithi153 ceph-mon[70044]: pgmap v17485: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:11.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:11:11 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/463414449' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:11:12.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:11:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:11:12.347Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:11:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:11:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:11:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:11:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:11:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:11:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:11:13.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:11:13 smithi083 ceph-mon[79930]: pgmap v17486: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:13.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:11:13 smithi083 ceph-mon[77242]: pgmap v17486: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:13.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:11:13 smithi153 ceph-mon[70044]: pgmap v17486: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:15.708 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:11:15 smithi083 ceph-mon[79930]: pgmap v17487: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:15.709 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:11:15 smithi083 ceph-mon[77242]: pgmap v17487: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:15.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:11:15 smithi153 ceph-mon[70044]: pgmap v17487: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:15.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:11:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:11:15.708Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:11:16.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:11:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:11:16.448Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:11:17.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:11:17 smithi083 ceph-mon[79930]: pgmap v17488: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:17.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:11:17 smithi083 ceph-mon[77242]: pgmap v17488: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:17.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:11:17 smithi153 ceph-mon[70044]: pgmap v17488: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:19.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:11:19 smithi083 ceph-mon[79930]: pgmap v17489: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:19.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:11:19 smithi083 ceph-mon[77242]: pgmap v17489: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:19.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:11:19 smithi153 ceph-mon[70044]: pgmap v17489: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:21.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:11:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:11:21] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T15:11:21.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:11:21 smithi083 ceph-mon[79930]: pgmap v17490: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:21.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:11:21 smithi083 ceph-mon[77242]: pgmap v17490: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:21.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:11:21 smithi153 ceph-mon[70044]: pgmap v17490: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:22.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:11:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:11:22.348Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:11:22.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:11:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:11:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:11:23.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:11:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:11:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:11:23.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:11:23 smithi083 ceph-mon[79930]: pgmap v17491: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:23.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:11:23 smithi083 ceph-mon[77242]: pgmap v17491: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:23.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:11:23 smithi153 ceph-mon[70044]: pgmap v17491: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:25.709 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:11:25 smithi083 ceph-mon[79930]: pgmap v17492: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:25.710 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:11:25 smithi083 ceph-mon[77242]: pgmap v17492: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:25.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:11:25 smithi153 ceph-mon[70044]: pgmap v17492: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:25.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:11:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:11:25.709Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:11:26.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:11:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:11:26.449Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:11:27.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:11:27 smithi083 ceph-mon[79930]: pgmap v17493: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:27.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:11:27 smithi083 ceph-mon[77242]: pgmap v17493: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:27.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:11:27 smithi153 ceph-mon[70044]: pgmap v17493: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:29.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:11:29 smithi083 ceph-mon[79930]: pgmap v17494: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:29.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:11:29 smithi083 ceph-mon[77242]: pgmap v17494: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:29.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:11:29 smithi153 ceph-mon[70044]: pgmap v17494: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:31.710 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:11:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:11:31] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T15:11:31.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:11:31 smithi083 ceph-mon[79930]: pgmap v17495: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:31.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:11:31 smithi083 ceph-mon[77242]: pgmap v17495: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:31.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:11:31 smithi153 ceph-mon[70044]: pgmap v17495: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:32.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:11:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:11:32.348Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:11:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:11:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:11:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:11:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:11:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:11:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:11:33.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:11:33 smithi083 ceph-mon[79930]: pgmap v17496: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:33.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:11:33 smithi083 ceph-mon[77242]: pgmap v17496: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:33.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:11:33 smithi153 ceph-mon[70044]: pgmap v17496: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:35.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:11:35 smithi083 ceph-mon[79930]: pgmap v17497: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:35.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:11:35 smithi083 ceph-mon[77242]: pgmap v17497: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:35.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:11:35 smithi153 ceph-mon[70044]: pgmap v17497: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:36.210 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:11:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:11:35.710Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:11:36.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:11:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:11:36.450Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:11:37.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:11:37 smithi083 ceph-mon[79930]: pgmap v17498: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:37.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:11:37 smithi083 ceph-mon[77242]: pgmap v17498: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:37.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:11:37 smithi153 ceph-mon[70044]: pgmap v17498: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:39.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:11:39 smithi083 ceph-mon[77242]: pgmap v17499: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:39.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:11:39 smithi083 ceph-mon[79930]: pgmap v17499: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:39.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:11:39 smithi153 ceph-mon[70044]: pgmap v17499: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:40.844 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:11:41.134 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:11:41.135 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 2m ago 9h 27.3M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:11:41.135 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 35s ago 9h 64.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:11:41.136 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 2m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:11:41.136 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 35s ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:11:41.136 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 35s ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:11:41.137 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 2m ago 9h 990M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:11:41.137 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 2m ago 9h 2124M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:11:41.138 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 35s ago 9h 926M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:11:41.138 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 2m ago 9h 1273M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:11:41.139 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 2m ago 9h 24.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:11:41.139 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 35s ago 9h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:11:41.140 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 2m ago 9h 1543M 2177M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:11:41.140 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 2m ago 9h 1529M 2177M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:11:41.140 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 2m ago 9h 1187M 2177M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:11:41.141 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 2m ago 9h 1499M 2177M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:11:41.141 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 35s ago 9h 1313M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:11:41.142 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 35s ago 9h 1649M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:11:41.142 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 35s ago 9h 1474M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:11:41.143 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 35s ago 9h 1754M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:11:41.143 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 35s ago 9h 174M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:11:41.491 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:11:41.491 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:11:41.492 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:11:41.492 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:11:41.492 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:11:41.492 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:11:41.492 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:11:41.493 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:11:41.493 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:11:41.493 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:11:41.493 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:11:41.494 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:11:41.494 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:11:41.494 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:11:41.494 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:11:41.494 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:11:41.495 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:11:41.495 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:11:41.710 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:11:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:11:41] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T15:11:41.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:11:41 smithi083 ceph-mon[79930]: pgmap v17500: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:41.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:11:41 smithi083 ceph-mon[77242]: pgmap v17500: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:41.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:11:41 smithi153 ceph-mon[70044]: pgmap v17500: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:42.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:11:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:11:42.350Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:11:42.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:11:42 smithi083 ceph-mon[79930]: from='client.54480 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:11:42.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:11:42 smithi083 ceph-mon[79930]: from='client.54486 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:11:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:11:42 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/15423093' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:11:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:11:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:11:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:11:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:11:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:11:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:11:42.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:11:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:11:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:11:42 smithi083 ceph-mon[77242]: from='client.54480 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:11:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:11:42 smithi083 ceph-mon[77242]: from='client.54486 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:11:42.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:11:42 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/15423093' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:11:42.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:11:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:11:42.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:11:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:11:42.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:11:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:11:42.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:11:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:11:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:11:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:11:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:11:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:11:42 smithi153 ceph-mon[70044]: from='client.54480 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:11:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:11:42 smithi153 ceph-mon[70044]: from='client.54486 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:11:42.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:11:42 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/15423093' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:11:42.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:11:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:11:42.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:11:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:11:42.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:11:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:11:42.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:11:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:11:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:11:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:11:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:11:43.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:11:43 smithi153 ceph-mon[70044]: pgmap v17501: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:43.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:11:43 smithi083 ceph-mon[79930]: pgmap v17501: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:43.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:11:43 smithi083 ceph-mon[77242]: pgmap v17501: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:45.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:11:45 smithi153 ceph-mon[70044]: pgmap v17502: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:45.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:11:45 smithi083 ceph-mon[79930]: pgmap v17502: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:45.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:11:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:11:45.711Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:11:45.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:11:45 smithi083 ceph-mon[77242]: pgmap v17502: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:46.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:11:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:11:46.450Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:11:47.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:11:47 smithi153 ceph-mon[70044]: pgmap v17503: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:47.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:11:47 smithi083 ceph-mon[79930]: pgmap v17503: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:47.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:11:47 smithi083 ceph-mon[77242]: pgmap v17503: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:49.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:11:49 smithi153 ceph-mon[70044]: pgmap v17504: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:49.960 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:11:49 smithi083 ceph-mon[77242]: pgmap v17504: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:49.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:11:49 smithi083 ceph-mon[79930]: pgmap v17504: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:50.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:11:50 smithi153 ceph-mon[70044]: pgmap v17505: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:50.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:11:50 smithi083 ceph-mon[79930]: pgmap v17505: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:50.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:11:50 smithi083 ceph-mon[77242]: pgmap v17505: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:11:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:11:51] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T15:11:52.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:11:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:11:52.350Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:11:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:11:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:11:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:11:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:11:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:11:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:11:53.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:11:53 smithi083 ceph-mon[79930]: pgmap v17506: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:53.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:11:53 smithi083 ceph-mon[77242]: pgmap v17506: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:53.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:11:53 smithi153 ceph-mon[70044]: pgmap v17506: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:55.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:11:55 smithi083 ceph-mon[79930]: pgmap v17507: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:55.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:11:55 smithi083 ceph-mon[77242]: pgmap v17507: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:55.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:11:55 smithi153 ceph-mon[70044]: pgmap v17507: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:56.210 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:11:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:11:55.711Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:11:56.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:11:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:11:56.450Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:11:57.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:11:57 smithi153 ceph-mon[70044]: pgmap v17508: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:57.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:11:57 smithi083 ceph-mon[79930]: pgmap v17508: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:57.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:11:57 smithi083 ceph-mon[77242]: pgmap v17508: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:59.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:11:59 smithi153 ceph-mon[70044]: pgmap v17509: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:59.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:11:59 smithi083 ceph-mon[79930]: pgmap v17509: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:11:59.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:11:59 smithi083 ceph-mon[77242]: pgmap v17509: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:01.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:12:01 smithi153 ceph-mon[70044]: pgmap v17510: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:01.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:12:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:12:01] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T15:12:01.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:12:01 smithi083 ceph-mon[79930]: pgmap v17510: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:01.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:12:01 smithi083 ceph-mon[77242]: pgmap v17510: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:02.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:12:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:12:02.351Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:12:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:12:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:12:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:12:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:12:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:12:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:12:03.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:12:03 smithi153 ceph-mon[70044]: pgmap v17511: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:03.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:12:03 smithi083 ceph-mon[79930]: pgmap v17511: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:03.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:12:03 smithi083 ceph-mon[77242]: pgmap v17511: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:05.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:12:05 smithi153 ceph-mon[70044]: pgmap v17512: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:05.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:12:05 smithi083 ceph-mon[79930]: pgmap v17512: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:05.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:12:05 smithi083 ceph-mon[77242]: pgmap v17512: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:05.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:12:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:12:05.711Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:12:06.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:12:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:12:06.451Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:12:06.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:12:06 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:12:06.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:12:06 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:12:06.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:12:06 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:12:06.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:12:06 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:12:06.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:12:06 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:12:06.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:12:06 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:12:06.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:12:06 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:12:06.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:12:06 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:12:06.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:12:06 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:12:06.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:12:06 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:12:06.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:12:06 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:12:06.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:12:06 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:12:07.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:12:07 smithi153 ceph-mon[70044]: pgmap v17513: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:07.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:12:07 smithi083 ceph-mon[77242]: pgmap v17513: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:07.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:12:07 smithi083 ceph-mon[79930]: pgmap v17513: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:08.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:12:08 smithi153 ceph-mon[70044]: pgmap v17514: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:08.960 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:12:08 smithi083 ceph-mon[77242]: pgmap v17514: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:08.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:12:08 smithi083 ceph-mon[79930]: pgmap v17514: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:11.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:12:11 smithi153 ceph-mon[70044]: pgmap v17515: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:11.813 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:12:11.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:12:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:12:11] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T15:12:11.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:12:11 smithi083 ceph-mon[79930]: pgmap v17515: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:11.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:12:11 smithi083 ceph-mon[77242]: pgmap v17515: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:12.104 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:12:12.104 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 3m ago 9h 27.3M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:12:12.105 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 66s ago 9h 64.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:12:12.105 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 3m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:12:12.106 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 66s ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:12:12.106 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 66s ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:12:12.107 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 3m ago 9h 990M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:12:12.107 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 3m ago 9h 2124M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:12:12.108 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 66s ago 9h 926M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:12:12.108 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 3m ago 9h 1273M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:12:12.109 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 3m ago 9h 24.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:12:12.109 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 66s ago 9h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:12:12.110 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 3m ago 9h 1543M 2177M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:12:12.110 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 3m ago 9h 1529M 2177M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:12:12.111 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 3m ago 9h 1187M 2177M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:12:12.111 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 3m ago 9h 1499M 2177M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:12:12.112 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 66s ago 9h 1313M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:12:12.112 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 66s ago 9h 1649M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:12:12.112 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 66s ago 9h 1474M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:12:12.113 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 66s ago 9h 1754M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:12:12.113 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 66s ago 9h 174M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:12:12.466 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:12:12.467 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:12:12.467 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:12:12.467 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:12:12.467 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:12:12.468 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:12:12.468 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:12:12.468 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:12:12.468 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:12:12.469 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:12:12.469 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:12:12.469 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:12:12.469 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:12:12.469 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:12:12.470 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:12:12.470 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:12:12.470 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:12:12.470 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:12:12.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:12:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:12:12.351Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:12:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:12:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:12:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:12:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:12:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:12:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:12:13.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:12:13 smithi153 ceph-mon[70044]: from='client.54498 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:12:13.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:12:13 smithi153 ceph-mon[70044]: from='client.64435 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:12:13.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:12:13 smithi153 ceph-mon[70044]: pgmap v17516: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:13.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:12:13 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2181857243' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:12:13.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:12:13 smithi083 ceph-mon[79930]: from='client.54498 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:12:13.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:12:13 smithi083 ceph-mon[79930]: from='client.64435 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:12:13.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:12:13 smithi083 ceph-mon[79930]: pgmap v17516: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:13.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:12:13 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2181857243' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:12:13.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:12:13 smithi083 ceph-mon[77242]: from='client.54498 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:12:13.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:12:13 smithi083 ceph-mon[77242]: from='client.64435 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:12:13.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:12:13 smithi083 ceph-mon[77242]: pgmap v17516: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:13.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:12:13 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2181857243' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:12:15.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:12:15 smithi153 ceph-mon[70044]: pgmap v17517: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:15.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:12:15 smithi083 ceph-mon[79930]: pgmap v17517: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:15.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:12:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:12:15.712Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:12:15.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:12:15 smithi083 ceph-mon[77242]: pgmap v17517: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:16.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:12:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:12:16.452Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:12:17.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:12:17 smithi153 ceph-mon[70044]: pgmap v17518: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:17.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:12:17 smithi083 ceph-mon[79930]: pgmap v17518: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:17.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:12:17 smithi083 ceph-mon[77242]: pgmap v17518: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:19.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:12:19 smithi153 ceph-mon[70044]: pgmap v17519: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:19.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:12:19 smithi083 ceph-mon[77242]: pgmap v17519: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:19.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:12:19 smithi083 ceph-mon[79930]: pgmap v17519: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:21.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:12:21 smithi153 ceph-mon[70044]: pgmap v17520: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:21.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:12:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:12:21] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T15:12:21.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:12:21 smithi083 ceph-mon[79930]: pgmap v17520: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:21.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:12:21 smithi083 ceph-mon[77242]: pgmap v17520: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:22.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:12:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:12:22.352Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:12:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:12:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:12:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:12:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:12:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:12:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:12:23.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:12:23 smithi153 ceph-mon[70044]: pgmap v17521: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:23.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:12:23 smithi083 ceph-mon[79930]: pgmap v17521: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:23.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:12:23 smithi083 ceph-mon[77242]: pgmap v17521: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:25.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:12:25 smithi153 ceph-mon[70044]: pgmap v17522: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:25.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:12:25 smithi083 ceph-mon[79930]: pgmap v17522: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:25.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:12:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:12:25.712Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:12:25.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:12:25 smithi083 ceph-mon[77242]: pgmap v17522: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:26.710 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:12:26 smithi083 ceph-mon[77242]: pgmap v17523: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:26.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:12:26 smithi083 ceph-mon[79930]: pgmap v17523: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:26.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:12:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:12:26.453Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:12:26.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:12:26 smithi153 ceph-mon[70044]: pgmap v17523: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:29.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:12:29 smithi083 ceph-mon[79930]: pgmap v17524: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:29.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:12:29 smithi083 ceph-mon[77242]: pgmap v17524: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:29.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:12:29 smithi153 ceph-mon[70044]: pgmap v17524: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:31.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:12:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:12:31] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T15:12:31.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:12:31 smithi083 ceph-mon[79930]: pgmap v17525: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:31.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:12:31 smithi083 ceph-mon[77242]: pgmap v17525: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:31.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:12:31 smithi153 ceph-mon[70044]: pgmap v17525: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:32.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:12:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:12:32.353Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:12:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:12:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:12:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:12:33.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:12:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:12:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:12:33.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:12:33 smithi083 ceph-mon[79930]: pgmap v17526: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:33.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:12:33 smithi083 ceph-mon[77242]: pgmap v17526: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:33.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:12:33 smithi153 ceph-mon[70044]: pgmap v17526: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:35.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:12:35 smithi083 ceph-mon[79930]: pgmap v17527: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:35.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:12:35 smithi083 ceph-mon[77242]: pgmap v17527: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:35.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:12:35 smithi153 ceph-mon[70044]: pgmap v17527: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:36.210 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:12:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:12:35.713Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:12:36.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:12:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:12:36.453Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:12:37.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:12:37 smithi083 ceph-mon[79930]: pgmap v17528: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:37.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:12:37 smithi083 ceph-mon[77242]: pgmap v17528: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:37.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:12:37 smithi153 ceph-mon[70044]: pgmap v17528: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:39.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:12:39 smithi083 ceph-mon[79930]: pgmap v17529: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:39.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:12:39 smithi083 ceph-mon[77242]: pgmap v17529: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:39.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:12:39 smithi153 ceph-mon[70044]: pgmap v17529: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:41.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:12:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:12:41] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T15:12:41.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:12:41 smithi083 ceph-mon[79930]: pgmap v17530: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:41.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:12:41 smithi083 ceph-mon[77242]: pgmap v17530: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:41.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:12:41 smithi153 ceph-mon[70044]: pgmap v17530: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:42.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:12:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:12:42.354Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:12:42.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:12:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:12:42.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:12:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:12:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:12:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:12:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:12:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:12:42.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:12:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:12:42.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:12:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:12:42.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:12:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:12:42.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:12:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:12:42.793 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:12:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:12:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:12:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:12:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:12:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:12:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:12:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:12:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:12:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:12:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:12:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:12:43.080 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:12:43.080 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 3m ago 9h 27.3M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:12:43.081 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 97s ago 9h 64.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:12:43.081 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 3m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:12:43.081 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 97s ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:12:43.081 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 97s ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:12:43.082 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 3m ago 9h 990M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:12:43.082 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 3m ago 9h 2124M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:12:43.083 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 97s ago 9h 926M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:12:43.083 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 3m ago 9h 1273M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:12:43.084 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 3m ago 9h 24.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:12:43.084 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 97s ago 9h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:12:43.085 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 3m ago 9h 1543M 2177M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:12:43.085 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 3m ago 9h 1529M 2177M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:12:43.085 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 3m ago 9h 1187M 2177M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:12:43.086 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 3m ago 9h 1499M 2177M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:12:43.086 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 97s ago 9h 1313M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:12:43.087 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 97s ago 9h 1649M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:12:43.087 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 97s ago 9h 1474M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:12:43.088 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 97s ago 9h 1754M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:12:43.088 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 97s ago 9h 174M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:12:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:12:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:12:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:12:43.438 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:12:43.439 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:12:43.439 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:12:43.439 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:12:43.439 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:12:43.440 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:12:43.440 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:12:43.440 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:12:43.440 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:12:43.441 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:12:43.441 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:12:43.441 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:12:43.441 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:12:43.441 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:12:43.442 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:12:43.442 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:12:43.442 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:12:43.442 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:12:43.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:12:43 smithi083 ceph-mon[79930]: pgmap v17531: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:43.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:12:43 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1945037475' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:12:43.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:12:43 smithi083 ceph-mon[77242]: pgmap v17531: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:43.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:12:43 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1945037475' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:12:43.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:12:43 smithi153 ceph-mon[70044]: pgmap v17531: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:43.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:12:43 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1945037475' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:12:44.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:12:44 smithi083 ceph-mon[79930]: from='client.54516 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:12:44.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:12:44 smithi083 ceph-mon[79930]: from='client.54522 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:12:44.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:12:44 smithi083 ceph-mon[77242]: from='client.54516 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:12:44.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:12:44 smithi083 ceph-mon[77242]: from='client.54522 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:12:44.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:12:44 smithi153 ceph-mon[70044]: from='client.54516 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:12:44.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:12:44 smithi153 ceph-mon[70044]: from='client.54522 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:12:45.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:12:45 smithi083 ceph-mon[79930]: pgmap v17532: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:45.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:12:45 smithi083 ceph-mon[77242]: pgmap v17532: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:45.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:12:45 smithi153 ceph-mon[70044]: pgmap v17532: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:46.210 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:12:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:12:45.714Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:12:46.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:12:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:12:46.454Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:12:47.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:12:47 smithi153 ceph-mon[70044]: pgmap v17533: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:47.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:12:47 smithi083 ceph-mon[79930]: pgmap v17533: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:47.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:12:47 smithi083 ceph-mon[77242]: pgmap v17533: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:49.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:12:49 smithi153 ceph-mon[70044]: pgmap v17534: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:49.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:12:49 smithi083 ceph-mon[79930]: pgmap v17534: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:49.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:12:49 smithi083 ceph-mon[77242]: pgmap v17534: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:51.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:12:51 smithi153 ceph-mon[70044]: pgmap v17535: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:12:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:12:51] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T15:12:51.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:12:51 smithi083 ceph-mon[79930]: pgmap v17535: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:51.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:12:51 smithi083 ceph-mon[77242]: pgmap v17535: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:52.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:12:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:12:52.355Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:12:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:12:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:12:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:12:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:12:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:12:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:12:53.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:12:53 smithi153 ceph-mon[70044]: pgmap v17536: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:53.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:12:53 smithi083 ceph-mon[79930]: pgmap v17536: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:53.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:12:53 smithi083 ceph-mon[77242]: pgmap v17536: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:55.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:12:55 smithi153 ceph-mon[70044]: pgmap v17537: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:55.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:12:55 smithi083 ceph-mon[77242]: pgmap v17537: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:55.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:12:55 smithi083 ceph-mon[79930]: pgmap v17537: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:55.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:12:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:12:55.714Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:12:56.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:12:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:12:56.454Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:12:57.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:12:57 smithi153 ceph-mon[70044]: pgmap v17538: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:57.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:12:57 smithi083 ceph-mon[79930]: pgmap v17538: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:57.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:12:57 smithi083 ceph-mon[77242]: pgmap v17538: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:59.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:12:59 smithi153 ceph-mon[70044]: pgmap v17539: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:59.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:12:59 smithi083 ceph-mon[77242]: pgmap v17539: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:12:59.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:12:59 smithi083 ceph-mon[79930]: pgmap v17539: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:01.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:13:01 smithi153 ceph-mon[70044]: pgmap v17540: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:13:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:13:01] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T15:13:01.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:13:01 smithi083 ceph-mon[79930]: pgmap v17540: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:01.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:13:01 smithi083 ceph-mon[77242]: pgmap v17540: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:02.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:13:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:13:02.356Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:13:02.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:13:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:13:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:13:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:13:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:13:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:13:03.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:13:03 smithi153 ceph-mon[70044]: pgmap v17541: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:03.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:13:03 smithi083 ceph-mon[79930]: pgmap v17541: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:03.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:13:03 smithi083 ceph-mon[77242]: pgmap v17541: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:05.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:13:05 smithi153 ceph-mon[70044]: pgmap v17542: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:05.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:13:05 smithi083 ceph-mon[79930]: pgmap v17542: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:05.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:13:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:13:05.715Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:13:05.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:13:05 smithi083 ceph-mon[77242]: pgmap v17542: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:06.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:13:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:13:06.455Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:13:06.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:13:06 smithi083 ceph-mon[77242]: pgmap v17543: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:06.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:13:06 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:13:06.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:13:06 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:13:06.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:13:06 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:13:06.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:13:06 smithi083 ceph-mon[79930]: pgmap v17543: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:06.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:13:06 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:13:06.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:13:06 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:13:06.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:13:06 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:13:06.814 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:13:06 smithi153 ceph-mon[70044]: pgmap v17543: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:06.814 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:13:06 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:13:06.815 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:13:06 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:13:06.815 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:13:06 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:13:08.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:13:08 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:13:08.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:13:08 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:13:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:13:08 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:13:09.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:13:09 smithi083 ceph-mon[79930]: pgmap v17544: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:09.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:13:09 smithi083 ceph-mon[77242]: pgmap v17544: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:09.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:13:09 smithi153 ceph-mon[70044]: pgmap v17544: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:11.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:13:11 smithi153 ceph-mon[70044]: pgmap v17545: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:11.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:13:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:13:11] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T15:13:11.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:13:11 smithi083 ceph-mon[79930]: pgmap v17545: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:11.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:13:11 smithi083 ceph-mon[77242]: pgmap v17545: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:12.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:13:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:13:12.356Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:13:12.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:13:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:13:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:13:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:13:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:13:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:13:13.774 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:13:13.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:13:13 smithi153 ceph-mon[70044]: pgmap v17546: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:13.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:13:13 smithi083 ceph-mon[79930]: pgmap v17546: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:13.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:13:13 smithi083 ceph-mon[77242]: pgmap v17546: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:14.068 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:13:14.068 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 4m ago 9h 27.3M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:13:14.069 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 2m ago 9h 64.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:13:14.069 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 4m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:13:14.069 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 2m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:13:14.070 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 2m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:13:14.070 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 4m ago 9h 990M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:13:14.070 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 4m ago 9h 2124M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:13:14.070 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 2m ago 9h 926M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:13:14.071 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 4m ago 9h 1273M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:13:14.071 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 4m ago 9h 24.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:13:14.071 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 2m ago 9h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:13:14.071 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 4m ago 9h 1543M 2177M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:13:14.072 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 4m ago 9h 1529M 2177M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:13:14.072 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 4m ago 9h 1187M 2177M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:13:14.072 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 4m ago 9h 1499M 2177M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:13:14.072 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 2m ago 9h 1313M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:13:14.073 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 2m ago 9h 1649M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:13:14.073 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 2m ago 9h 1474M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:13:14.073 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 2m ago 9h 1754M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:13:14.073 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 2m ago 9h 174M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:13:14.423 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:13:14.424 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:13:14.424 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:13:14.424 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:13:14.424 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:13:14.425 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:13:14.425 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:13:14.425 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:13:14.425 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:13:14.426 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:13:14.426 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:13:14.426 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:13:14.426 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:13:14.427 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:13:14.427 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:13:14.427 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:13:14.428 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:13:14.428 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:13:14.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:13:14 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/249403368' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:13:14.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:13:14 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/249403368' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:13:14.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:13:14 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/249403368' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:13:15.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:13:15 smithi153 ceph-mon[70044]: from='client.54534 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:13:15.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:13:15 smithi153 ceph-mon[70044]: from='client.64471 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:13:15.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:13:15 smithi153 ceph-mon[70044]: pgmap v17547: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:15.960 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:13:15 smithi083 ceph-mon[77242]: from='client.54534 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:13:15.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:13:15 smithi083 ceph-mon[77242]: from='client.64471 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:13:15.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:13:15 smithi083 ceph-mon[77242]: pgmap v17547: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:15.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:13:15 smithi083 ceph-mon[79930]: from='client.54534 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:13:15.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:13:15 smithi083 ceph-mon[79930]: from='client.64471 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:13:15.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:13:15 smithi083 ceph-mon[79930]: pgmap v17547: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:15.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:13:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:13:15.715Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:13:16.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:13:16 smithi083 ceph-mon[79930]: pgmap v17548: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:16.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:13:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:13:16.455Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:13:16.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:13:16 smithi083 ceph-mon[77242]: pgmap v17548: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:16.816 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:13:16 smithi153 ceph-mon[70044]: pgmap v17548: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:19.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:13:19 smithi083 ceph-mon[79930]: pgmap v17549: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:19.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:13:19 smithi083 ceph-mon[77242]: pgmap v17549: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:19.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:13:19 smithi153 ceph-mon[70044]: pgmap v17549: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:21.710 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:13:21 smithi083 ceph-mon[77242]: pgmap v17550: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:21.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:13:21 smithi083 ceph-mon[79930]: pgmap v17550: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:21.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:13:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:13:21] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T15:13:21.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:13:21 smithi153 ceph-mon[70044]: pgmap v17550: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:22.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:13:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:13:22.357Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:13:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:13:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:13:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:13:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:13:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:13:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:13:23.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:13:23 smithi083 ceph-mon[79930]: pgmap v17551: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:23.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:13:23 smithi083 ceph-mon[77242]: pgmap v17551: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:23.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:13:23 smithi153 ceph-mon[70044]: pgmap v17551: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:25.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:13:25 smithi083 ceph-mon[79930]: pgmap v17552: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:25.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:13:25 smithi083 ceph-mon[77242]: pgmap v17552: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:25.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:13:25 smithi153 ceph-mon[70044]: pgmap v17552: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:26.210 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:13:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:13:25.716Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:13:26.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:13:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:13:26.456Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:13:27.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:13:27 smithi083 ceph-mon[79930]: pgmap v17553: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:27.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:13:27 smithi083 ceph-mon[77242]: pgmap v17553: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:27.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:13:27 smithi153 ceph-mon[70044]: pgmap v17553: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:29.710 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:13:29 smithi083 ceph-mon[77242]: pgmap v17554: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:29.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:13:29 smithi083 ceph-mon[79930]: pgmap v17554: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:29.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:13:29 smithi153 ceph-mon[70044]: pgmap v17554: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:31.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:13:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:13:31] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T15:13:31.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:13:31 smithi083 ceph-mon[79930]: pgmap v17555: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:31.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:13:31 smithi083 ceph-mon[77242]: pgmap v17555: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:31.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:13:31 smithi153 ceph-mon[70044]: pgmap v17555: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:32.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:13:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:13:32.358Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:13:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:13:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:13:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:13:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:13:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:13:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:13:33.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:13:33 smithi083 ceph-mon[79930]: pgmap v17556: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:33.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:13:33 smithi083 ceph-mon[77242]: pgmap v17556: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:33.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:13:33 smithi153 ceph-mon[70044]: pgmap v17556: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:35.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:13:35 smithi083 ceph-mon[79930]: pgmap v17557: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:35.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:13:35 smithi083 ceph-mon[77242]: pgmap v17557: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:35.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:13:35 smithi153 ceph-mon[70044]: pgmap v17557: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:36.210 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:13:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:13:35.717Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:13:36.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:13:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:13:36.456Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:13:37.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:13:37 smithi153 ceph-mon[70044]: pgmap v17558: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:37.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:13:37 smithi083 ceph-mon[77242]: pgmap v17558: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:37.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:13:37 smithi083 ceph-mon[79930]: pgmap v17558: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:39.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:13:39 smithi153 ceph-mon[70044]: pgmap v17559: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:39.960 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:13:39 smithi083 ceph-mon[77242]: pgmap v17559: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:39.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:13:39 smithi083 ceph-mon[79930]: pgmap v17559: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:41.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:13:41 smithi153 ceph-mon[70044]: pgmap v17560: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:13:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:13:41] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T15:13:41.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:13:41 smithi083 ceph-mon[79930]: pgmap v17560: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:41.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:13:41 smithi083 ceph-mon[77242]: pgmap v17560: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:42.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:13:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:13:42.358Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:13:42.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:13:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:13:42.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:13:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:13:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:13:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:13:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:13:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:13:42.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:13:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:13:42.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:13:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:13:42.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:13:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:13:42.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:13:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:13:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:13:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:13:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:13:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:13:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:13:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:13:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:13:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:13:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:13:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:13:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:13:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:13:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:13:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:13:43.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:13:43 smithi153 ceph-mon[70044]: pgmap v17561: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:43.960 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:13:43 smithi083 ceph-mon[77242]: pgmap v17561: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:43.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:13:43 smithi083 ceph-mon[79930]: pgmap v17561: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:44.753 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:13:45.045 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:13:45.046 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 4m ago 9h 27.3M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:13:45.046 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 2m ago 9h 64.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:13:45.046 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 4m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:13:45.047 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 2m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:13:45.047 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 2m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:13:45.048 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 4m ago 9h 990M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:13:45.048 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 4m ago 9h 2124M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:13:45.049 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 2m ago 9h 926M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:13:45.049 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 4m ago 9h 1273M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:13:45.050 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 4m ago 9h 24.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:13:45.050 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 2m ago 9h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:13:45.051 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 4m ago 9h 1543M 2177M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:13:45.051 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 4m ago 9h 1529M 2177M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:13:45.052 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 4m ago 9h 1187M 2177M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:13:45.052 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 4m ago 9h 1499M 2177M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:13:45.053 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 2m ago 9h 1313M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:13:45.053 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 2m ago 9h 1649M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:13:45.054 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 2m ago 9h 1474M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:13:45.054 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 2m ago 9h 1754M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:13:45.055 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 2m ago 9h 174M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:13:45.403 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:13:45.403 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:13:45.404 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:13:45.404 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:13:45.404 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:13:45.405 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:13:45.405 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:13:45.406 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:13:45.406 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:13:45.407 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:13:45.407 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:13:45.408 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:13:45.408 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:13:45.409 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:13:45.409 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:13:45.409 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:13:45.410 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:13:45.410 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:13:45.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:13:45 smithi153 ceph-mon[70044]: pgmap v17562: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:45.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:13:45 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2870434508' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:13:45.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:13:45 smithi083 ceph-mon[79930]: pgmap v17562: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:45.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:13:45 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2870434508' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:13:45.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:13:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:13:45.717Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:13:45.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:13:45 smithi083 ceph-mon[77242]: pgmap v17562: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:45.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:13:45 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2870434508' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:13:46.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:13:46 smithi083 ceph-mon[79930]: from='client.64483 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:13:46.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:13:46 smithi083 ceph-mon[79930]: from='client.64489 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:13:46.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:13:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:13:46.457Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:13:46.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:13:46 smithi083 ceph-mon[77242]: from='client.64483 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:13:46.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:13:46 smithi083 ceph-mon[77242]: from='client.64489 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:13:46.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:13:46 smithi153 ceph-mon[70044]: from='client.64483 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:13:46.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:13:46 smithi153 ceph-mon[70044]: from='client.64489 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:13:47.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:13:47 smithi153 ceph-mon[70044]: pgmap v17563: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:47.974 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:13:47 smithi083 ceph-mon[79930]: pgmap v17563: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:47.975 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:13:47 smithi083 ceph-mon[77242]: pgmap v17563: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:49.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:13:49 smithi153 ceph-mon[70044]: pgmap v17564: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:49.960 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:13:49 smithi083 ceph-mon[77242]: pgmap v17564: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:49.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:13:49 smithi083 ceph-mon[79930]: pgmap v17564: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:51.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:13:51 smithi153 ceph-mon[70044]: pgmap v17565: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:13:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:13:51] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T15:13:51.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:13:51 smithi083 ceph-mon[79930]: pgmap v17565: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:51.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:13:51 smithi083 ceph-mon[77242]: pgmap v17565: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:52.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:13:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:13:52.360Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:13:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:13:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:13:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:13:53.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:13:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:13:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:13:53.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:13:53 smithi083 ceph-mon[79930]: pgmap v17566: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:53.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:13:53 smithi083 ceph-mon[77242]: pgmap v17566: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:53.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:13:53 smithi153 ceph-mon[70044]: pgmap v17566: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:55.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:13:55 smithi083 ceph-mon[79930]: pgmap v17567: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:55.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:13:55 smithi083 ceph-mon[77242]: pgmap v17567: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:55.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:13:55 smithi153 ceph-mon[70044]: pgmap v17567: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:56.210 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:13:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:13:55.718Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:13:56.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:13:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:13:56.457Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:13:57.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:13:57 smithi083 ceph-mon[79930]: pgmap v17568: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:57.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:13:57 smithi083 ceph-mon[77242]: pgmap v17568: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:57.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:13:57 smithi153 ceph-mon[70044]: pgmap v17568: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:59.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:13:59 smithi083 ceph-mon[77242]: pgmap v17569: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:59.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:13:59 smithi083 ceph-mon[79930]: pgmap v17569: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:13:59.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:13:59 smithi153 ceph-mon[70044]: pgmap v17569: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:01.710 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:14:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:14:01] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T15:14:01.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:14:01 smithi083 ceph-mon[79930]: pgmap v17570: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:01.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:14:01 smithi083 ceph-mon[77242]: pgmap v17570: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:01.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:14:01 smithi153 ceph-mon[70044]: pgmap v17570: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:02.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:14:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:14:02.360Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:14:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:14:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:14:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:14:03.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:14:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:14:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:14:03.710 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:14:03 smithi083 ceph-mon[77242]: pgmap v17571: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:03.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:14:03 smithi083 ceph-mon[79930]: pgmap v17571: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:03.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:14:03 smithi153 ceph-mon[70044]: pgmap v17571: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:05.710 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:14:05 smithi083 ceph-mon[77242]: pgmap v17572: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:05.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:14:05 smithi083 ceph-mon[79930]: pgmap v17572: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:05.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:14:05 smithi153 ceph-mon[70044]: pgmap v17572: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:06.210 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:14:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:14:05.719Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:14:06.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:14:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:14:06.458Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:14:07.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:14:07 smithi083 ceph-mon[79930]: pgmap v17573: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:07.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:14:07 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:14:07.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:14:07 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:14:07.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:14:07 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:14:07.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:14:07 smithi083 ceph-mon[77242]: pgmap v17573: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:07.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:14:07 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:14:07.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:14:07 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:14:07.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:14:07 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:14:07.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:14:07 smithi153 ceph-mon[70044]: pgmap v17573: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:07.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:14:07 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:14:07.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:14:07 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:14:07.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:14:07 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:14:09.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:14:08 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:14:09.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:14:08 smithi153 ceph-mon[70044]: pgmap v17574: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:09.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:14:08 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:14:09.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:14:08 smithi083 ceph-mon[79930]: pgmap v17574: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:09.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:14:08 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:14:09.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:14:08 smithi083 ceph-mon[77242]: pgmap v17574: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:10.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:14:10 smithi153 ceph-mon[70044]: pgmap v17575: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:10.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:14:10 smithi083 ceph-mon[79930]: pgmap v17575: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:10.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:14:10 smithi083 ceph-mon[77242]: pgmap v17575: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:11.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:14:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:14:11] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T15:14:12.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:14:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:14:12.361Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:14:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:14:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:14:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:14:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:14:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:14:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:14:13.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:14:13 smithi083 ceph-mon[79930]: pgmap v17576: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:13.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:14:13 smithi083 ceph-mon[77242]: pgmap v17576: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:13.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:14:13 smithi153 ceph-mon[70044]: pgmap v17576: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:15.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:14:15 smithi083 ceph-mon[79930]: pgmap v17577: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:15.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:14:15 smithi083 ceph-mon[77242]: pgmap v17577: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:15.732 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:14:15.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:14:15 smithi153 ceph-mon[70044]: pgmap v17577: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:16.021 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:14:16.021 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 5m ago 9h 27.3M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:14:16.021 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 3m ago 9h 64.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:14:16.022 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 5m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:14:16.022 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 3m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:14:16.022 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 3m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:14:16.023 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 5m ago 9h 990M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:14:16.023 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 5m ago 9h 2124M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:14:16.023 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 3m ago 9h 926M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:14:16.023 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 5m ago 9h 1273M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:14:16.023 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 5m ago 9h 24.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:14:16.024 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 3m ago 9h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:14:16.024 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 5m ago 9h 1543M 2177M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:14:16.024 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 5m ago 9h 1529M 2177M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:14:16.024 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 5m ago 9h 1187M 2177M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:14:16.025 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 5m ago 9h 1499M 2177M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:14:16.025 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 3m ago 9h 1313M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:14:16.025 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 3m ago 9h 1649M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:14:16.026 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 3m ago 9h 1474M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:14:16.026 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 3m ago 9h 1754M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:14:16.026 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 3m ago 9h 174M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:14:16.210 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:14:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:14:15.718Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:14:16.380 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:14:16.380 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:14:16.381 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:14:16.381 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:14:16.382 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:14:16.382 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:14:16.383 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:14:16.383 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:14:16.384 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:14:16.384 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:14:16.384 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:14:16.384 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:14:16.385 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:14:16.385 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:14:16.385 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:14:16.386 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:14:16.386 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:14:16.386 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:14:16.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:14:16 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/928371007' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:14:16.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:14:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:14:16.458Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:14:16.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:14:16 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/928371007' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:14:16.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:14:16 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/928371007' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:14:17.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:14:17 smithi153 ceph-mon[70044]: from='client.64501 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:14:17.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:14:17 smithi153 ceph-mon[70044]: from='client.64507 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:14:17.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:14:17 smithi153 ceph-mon[70044]: pgmap v17578: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:17.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:14:17 smithi083 ceph-mon[79930]: from='client.64501 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:14:17.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:14:17 smithi083 ceph-mon[79930]: from='client.64507 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:14:17.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:14:17 smithi083 ceph-mon[79930]: pgmap v17578: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:17.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:14:17 smithi083 ceph-mon[77242]: from='client.64501 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:14:17.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:14:17 smithi083 ceph-mon[77242]: from='client.64507 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:14:17.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:14:17 smithi083 ceph-mon[77242]: pgmap v17578: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:19.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:14:19 smithi153 ceph-mon[70044]: pgmap v17579: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:19.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:14:19 smithi083 ceph-mon[79930]: pgmap v17579: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:19.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:14:19 smithi083 ceph-mon[77242]: pgmap v17579: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:21.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:14:21 smithi153 ceph-mon[70044]: pgmap v17580: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:21.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:14:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:14:21] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T15:14:21.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:14:21 smithi083 ceph-mon[79930]: pgmap v17580: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:21.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:14:21 smithi083 ceph-mon[77242]: pgmap v17580: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:22.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:14:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:14:22.362Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:14:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:14:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:14:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:14:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:14:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:14:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:14:23.710 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:14:23 smithi083 ceph-mon[77242]: pgmap v17581: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:23.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:14:23 smithi083 ceph-mon[79930]: pgmap v17581: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:23.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:14:23 smithi153 ceph-mon[70044]: pgmap v17581: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:25.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:14:25 smithi083 ceph-mon[79930]: pgmap v17582: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:25.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:14:25 smithi083 ceph-mon[77242]: pgmap v17582: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:25.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:14:25 smithi153 ceph-mon[70044]: pgmap v17582: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:26.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:14:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:14:25.719Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:14:26.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:14:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:14:26.458Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:14:27.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:14:27 smithi083 ceph-mon[79930]: pgmap v17583: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:27.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:14:27 smithi083 ceph-mon[77242]: pgmap v17583: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:27.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:14:27 smithi153 ceph-mon[70044]: pgmap v17583: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:29.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:14:29 smithi153 ceph-mon[70044]: pgmap v17584: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:29.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:14:29 smithi083 ceph-mon[79930]: pgmap v17584: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:29.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:14:29 smithi083 ceph-mon[77242]: pgmap v17584: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:31.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:14:31 smithi153 ceph-mon[70044]: pgmap v17585: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:31.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:14:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:14:31] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T15:14:31.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:14:31 smithi083 ceph-mon[79930]: pgmap v17585: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:31.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:14:31 smithi083 ceph-mon[77242]: pgmap v17585: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:32.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:14:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:14:32.362Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:14:32.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:14:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:14:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:14:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:14:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:14:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:14:33.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:14:33 smithi083 ceph-mon[79930]: pgmap v17586: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:33.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:14:33 smithi083 ceph-mon[77242]: pgmap v17586: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:33.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:14:33 smithi153 ceph-mon[70044]: pgmap v17586: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:34.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:14:34 smithi153 ceph-mon[70044]: pgmap v17587: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:34.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:14:34 smithi083 ceph-mon[77242]: pgmap v17587: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:34.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:14:34 smithi083 ceph-mon[79930]: pgmap v17587: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:36.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:14:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:14:35.719Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:14:36.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:14:36 smithi083 ceph-mon[79930]: pgmap v17588: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:36.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:14:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:14:36.458Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:14:36.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:14:36 smithi083 ceph-mon[77242]: pgmap v17588: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:36.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:14:36 smithi153 ceph-mon[70044]: pgmap v17588: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:39.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:14:39 smithi153 ceph-mon[70044]: pgmap v17589: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:39.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:14:39 smithi083 ceph-mon[79930]: pgmap v17589: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:39.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:14:39 smithi083 ceph-mon[77242]: pgmap v17589: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:41.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:14:41 smithi153 ceph-mon[70044]: pgmap v17590: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:41.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:14:41 smithi083 ceph-mon[79930]: pgmap v17590: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:41.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:14:41 smithi083 ceph-mon[77242]: pgmap v17590: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:14:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:14:41] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T15:14:42.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:14:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:14:42.364Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:14:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:14:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:14:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:14:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:14:42.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:14:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:14:42.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:14:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:14:42.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:14:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:14:42.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:14:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:14:42.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:14:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:14:42.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:14:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:14:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:14:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:14:42.317Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:14:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:14:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:14:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:14:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:14:42.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:14:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:14:42.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:14:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:14:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:14:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:14:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:14:43.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:14:43 smithi153 ceph-mon[70044]: pgmap v17591: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:43.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:14:43 smithi083 ceph-mon[79930]: pgmap v17591: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:43.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:14:43 smithi083 ceph-mon[77242]: pgmap v17591: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:45.721 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:14:45 smithi083 ceph-mon[79930]: pgmap v17592: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:45.722 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:14:45 smithi083 ceph-mon[77242]: pgmap v17592: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:45.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:14:45 smithi153 ceph-mon[70044]: pgmap v17592: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:46.210 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:14:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:14:45.720Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:14:46.711 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:14:46.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:14:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:14:46.460Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:14:46.999 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:14:47.000 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 5m ago 9h 27.3M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:14:47.000 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 3m ago 9h 64.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:14:47.000 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 5m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:14:47.000 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 3m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:14:47.000 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 3m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:14:47.001 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 5m ago 9h 990M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:14:47.001 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 5m ago 9h 2124M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:14:47.001 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 3m ago 9h 926M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:14:47.001 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 5m ago 9h 1273M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:14:47.002 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 5m ago 9h 24.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:14:47.002 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 3m ago 9h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:14:47.002 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 5m ago 9h 1543M 2177M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:14:47.002 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 5m ago 9h 1529M 2177M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:14:47.003 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 5m ago 9h 1187M 2177M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:14:47.003 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 5m ago 9h 1499M 2177M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:14:47.003 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 3m ago 9h 1313M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:14:47.003 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 3m ago 9h 1649M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:14:47.004 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 3m ago 9h 1474M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:14:47.004 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 3m ago 9h 1754M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:14:47.004 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 3m ago 9h 174M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:14:47.359 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:14:47.359 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:14:47.360 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:14:47.360 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:14:47.360 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:14:47.360 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:14:47.361 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:14:47.361 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:14:47.361 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:14:47.361 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:14:47.362 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:14:47.362 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:14:47.362 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:14:47.362 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:14:47.363 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:14:47.363 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:14:47.363 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:14:47.363 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:14:47.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:14:47 smithi153 ceph-mon[70044]: pgmap v17593: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:47.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:14:47 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1421736962' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:14:47.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:14:47 smithi083 ceph-mon[79930]: pgmap v17593: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:47.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:14:47 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1421736962' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:14:47.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:14:47 smithi083 ceph-mon[77242]: pgmap v17593: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:47.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:14:47 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1421736962' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:14:48.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:14:48 smithi153 ceph-mon[70044]: from='client.54588 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:14:48.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:14:48 smithi153 ceph-mon[70044]: from='client.64525 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:14:48.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:14:48 smithi083 ceph-mon[79930]: from='client.54588 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:14:48.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:14:48 smithi083 ceph-mon[79930]: from='client.64525 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:14:48.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:14:48 smithi083 ceph-mon[77242]: from='client.54588 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:14:48.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:14:48 smithi083 ceph-mon[77242]: from='client.64525 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:14:49.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:14:49 smithi153 ceph-mon[70044]: pgmap v17594: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:49.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:14:49 smithi083 ceph-mon[79930]: pgmap v17594: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:49.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:14:49 smithi083 ceph-mon[77242]: pgmap v17594: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:51.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:14:51 smithi153 ceph-mon[70044]: pgmap v17595: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:51.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:14:51 smithi083 ceph-mon[77242]: pgmap v17595: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:51.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:14:51 smithi083 ceph-mon[79930]: pgmap v17595: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:51.967 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:14:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:14:51] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T15:14:52.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:14:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:14:52.363Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:14:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:14:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:14:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:14:53.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:14:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:14:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:14:53.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:14:53 smithi153 ceph-mon[70044]: pgmap v17596: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:53.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:14:53 smithi083 ceph-mon[79930]: pgmap v17596: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:53.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:14:53 smithi083 ceph-mon[77242]: pgmap v17596: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:55.722 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:14:55 smithi083 ceph-mon[79930]: pgmap v17597: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:55.723 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:14:55 smithi083 ceph-mon[77242]: pgmap v17597: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:55.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:14:55 smithi153 ceph-mon[70044]: pgmap v17597: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:56.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:14:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:14:55.721Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:14:56.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:14:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:14:56.459Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:14:57.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:14:57 smithi153 ceph-mon[70044]: pgmap v17598: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:57.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:14:57 smithi083 ceph-mon[79930]: pgmap v17598: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:57.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:14:57 smithi083 ceph-mon[77242]: pgmap v17598: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:59.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:14:59 smithi153 ceph-mon[70044]: pgmap v17599: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:59.960 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:14:59 smithi083 ceph-mon[77242]: pgmap v17599: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:14:59.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:14:59 smithi083 ceph-mon[79930]: pgmap v17599: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:01.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:15:01 smithi153 ceph-mon[70044]: pgmap v17600: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:01.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:15:01 smithi083 ceph-mon[77242]: pgmap v17600: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:01.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:15:01 smithi083 ceph-mon[79930]: pgmap v17600: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:01.964 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:15:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:15:01] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T15:15:02.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:15:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:15:02.364Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:15:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:15:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:15:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:15:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:15:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:15:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:15:03.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:15:03 smithi153 ceph-mon[70044]: pgmap v17601: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:03.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:15:03 smithi083 ceph-mon[79930]: pgmap v17601: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:03.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:15:03 smithi083 ceph-mon[77242]: pgmap v17601: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:04.960 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:15:04 smithi083 ceph-mon[77242]: pgmap v17602: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:04.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:15:04 smithi083 ceph-mon[79930]: pgmap v17602: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:05.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:15:04 smithi153 ceph-mon[70044]: pgmap v17602: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:06.210 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:15:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:15:05.722Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:15:06.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:15:06 smithi083 ceph-mon[79930]: pgmap v17603: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:06.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:15:06 smithi083 ceph-mon[77242]: pgmap v17603: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:06.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:15:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:15:06.460Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:15:07.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:15:06 smithi153 ceph-mon[70044]: pgmap v17603: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:08.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:15:08 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:15:08.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:15:08 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:15:08.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:15:08 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:15:08.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:15:08 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:15:08.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:15:08 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:15:08.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:15:08 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:15:08.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:15:08 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:15:08.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:15:08 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:15:08.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:15:08 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:15:08.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:15:08 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:15:08.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:15:08 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:15:08.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:15:08 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:15:09.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:15:09 smithi153 ceph-mon[70044]: pgmap v17604: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:09.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:15:09 smithi083 ceph-mon[79930]: pgmap v17604: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:09.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:15:09 smithi083 ceph-mon[77242]: pgmap v17604: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:11.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:15:11 smithi153 ceph-mon[70044]: pgmap v17605: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:11.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:15:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:15:11] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T15:15:11.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:15:11 smithi083 ceph-mon[79930]: pgmap v17605: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:11.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:15:11 smithi083 ceph-mon[77242]: pgmap v17605: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:12.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:15:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:15:12.364Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:15:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:15:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:15:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:15:13.316 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:15:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:15:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:15:13.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:15:13 smithi153 ceph-mon[70044]: pgmap v17606: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:13.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:15:13 smithi083 ceph-mon[79930]: pgmap v17606: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:13.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:15:13 smithi083 ceph-mon[77242]: pgmap v17606: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:15.724 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:15:15 smithi083 ceph-mon[79930]: pgmap v17607: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:15.725 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:15:15 smithi083 ceph-mon[77242]: pgmap v17607: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:15.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:15:15 smithi153 ceph-mon[70044]: pgmap v17607: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:16.210 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:15:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:15:15.722Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:15:16.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:15:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:15:16.460Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:15:17.686 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:15:17.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:15:17 smithi153 ceph-mon[70044]: pgmap v17608: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:17.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:15:17 smithi083 ceph-mon[79930]: pgmap v17608: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:17.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:15:17 smithi083 ceph-mon[77242]: pgmap v17608: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:17.974 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:15:17.975 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 6m ago 9h 27.3M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:15:17.975 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 4m ago 9h 64.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:15:17.975 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 6m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:15:17.975 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 4m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:15:17.976 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 4m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:15:17.976 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 6m ago 9h 990M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:15:17.976 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 6m ago 9h 2124M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:15:17.976 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 4m ago 9h 926M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:15:17.977 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 6m ago 9h 1273M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:15:17.977 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 6m ago 9h 24.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:15:17.977 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 4m ago 9h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:15:17.977 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 6m ago 9h 1543M 2177M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:15:17.977 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 6m ago 9h 1529M 2177M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:15:17.978 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 6m ago 9h 1187M 2177M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:15:17.978 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 6m ago 9h 1499M 2177M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:15:17.978 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 4m ago 9h 1313M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:15:17.978 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 4m ago 9h 1649M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:15:17.979 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 4m ago 9h 1474M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:15:17.979 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 4m ago 9h 1754M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:15:17.979 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 4m ago 9h 174M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:15:18.328 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:15:18.328 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:15:18.329 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:15:18.329 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:15:18.329 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:15:18.329 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:15:18.330 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:15:18.330 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:15:18.330 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:15:18.330 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:15:18.331 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:15:18.331 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:15:18.331 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:15:18.331 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:15:18.332 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:15:18.332 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:15:18.332 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:15:18.332 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:15:18.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:15:18 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/850816332' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:15:18.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:15:18 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/850816332' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:15:18.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:15:18 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/850816332' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:15:19.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:15:19 smithi153 ceph-mon[70044]: from='client.54606 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:15:19.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:15:19 smithi153 ceph-mon[70044]: from='client.54612 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:15:19.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:15:19 smithi153 ceph-mon[70044]: pgmap v17609: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:19.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:15:19 smithi083 ceph-mon[79930]: from='client.54606 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:15:19.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:15:19 smithi083 ceph-mon[79930]: from='client.54612 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:15:19.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:15:19 smithi083 ceph-mon[79930]: pgmap v17609: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:19.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:15:19 smithi083 ceph-mon[77242]: from='client.54606 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:15:19.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:15:19 smithi083 ceph-mon[77242]: from='client.54612 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:15:19.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:15:19 smithi083 ceph-mon[77242]: pgmap v17609: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:21.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:15:21 smithi153 ceph-mon[70044]: pgmap v17610: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:15:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:15:21] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T15:15:21.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:15:21 smithi083 ceph-mon[79930]: pgmap v17610: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:21.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:15:21 smithi083 ceph-mon[77242]: pgmap v17610: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:22.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:15:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:15:22.365Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:15:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:15:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:15:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:15:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:15:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:15:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:15:23.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:15:23 smithi153 ceph-mon[70044]: pgmap v17611: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:23.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:15:23 smithi083 ceph-mon[79930]: pgmap v17611: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:23.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:15:23 smithi083 ceph-mon[77242]: pgmap v17611: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:24.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:15:24 smithi083 ceph-mon[79930]: pgmap v17612: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:24.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:15:24 smithi083 ceph-mon[77242]: pgmap v17612: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:25.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:15:24 smithi153 ceph-mon[70044]: pgmap v17612: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:26.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:15:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:15:25.722Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:15:26.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:15:26 smithi083 ceph-mon[79930]: pgmap v17613: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:26.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:15:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:15:26.461Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:15:26.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:15:26 smithi083 ceph-mon[77242]: pgmap v17613: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:27.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:15:26 smithi153 ceph-mon[70044]: pgmap v17613: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:28.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:15:28 smithi083 ceph-mon[79930]: pgmap v17614: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:28.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:15:28 smithi083 ceph-mon[77242]: pgmap v17614: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:29.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:15:28 smithi153 ceph-mon[70044]: pgmap v17614: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:30.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:15:30 smithi083 ceph-mon[79930]: pgmap v17615: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:30.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:15:30 smithi083 ceph-mon[77242]: pgmap v17615: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:31.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:15:30 smithi153 ceph-mon[70044]: pgmap v17615: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:31.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:15:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:15:31] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T15:15:32.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:15:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:15:32.365Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:15:32.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:15:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:15:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:15:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:15:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:15:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:15:33.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:15:33 smithi153 ceph-mon[70044]: pgmap v17616: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:33.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:15:33 smithi083 ceph-mon[79930]: pgmap v17616: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:33.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:15:33 smithi083 ceph-mon[77242]: pgmap v17616: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:34.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:15:34 smithi083 ceph-mon[79930]: pgmap v17617: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:34.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:15:34 smithi083 ceph-mon[77242]: pgmap v17617: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:35.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:15:34 smithi153 ceph-mon[70044]: pgmap v17617: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:36.210 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:15:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:15:35.722Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:15:36.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:15:36 smithi083 ceph-mon[79930]: pgmap v17618: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:36.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:15:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:15:36.461Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:15:36.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:15:36 smithi083 ceph-mon[77242]: pgmap v17618: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:37.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:15:36 smithi153 ceph-mon[70044]: pgmap v17618: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:38.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:15:38 smithi083 ceph-mon[79930]: pgmap v17619: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:38.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:15:38 smithi083 ceph-mon[77242]: pgmap v17619: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:39.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:15:38 smithi153 ceph-mon[70044]: pgmap v17619: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:40.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:15:40 smithi083 ceph-mon[79930]: pgmap v17620: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:40.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:15:40 smithi083 ceph-mon[77242]: pgmap v17620: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:15:40 smithi153 ceph-mon[70044]: pgmap v17620: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:41.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:15:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:15:41] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T15:15:41.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:15:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:15:41.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:15:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:15:41.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:15:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:15:41.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:15:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:15:41.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:15:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:15:41.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:15:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:15:41.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:15:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:15:41.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:15:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:15:42.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:15:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:15:42.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:15:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:15:42.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:15:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:15:42.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:15:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:15:42.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:15:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:15:42.365Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:15:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:15:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:15:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:15:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:15:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:15:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:15:43.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:15:43 smithi153 ceph-mon[70044]: pgmap v17621: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:43.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:15:43 smithi083 ceph-mon[79930]: pgmap v17621: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:43.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:15:43 smithi083 ceph-mon[77242]: pgmap v17621: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:44.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:15:44 smithi083 ceph-mon[79930]: pgmap v17622: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:44.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:15:44 smithi083 ceph-mon[77242]: pgmap v17622: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:45.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:15:44 smithi153 ceph-mon[70044]: pgmap v17622: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:46.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:15:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:15:45.723Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:15:46.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:15:46 smithi083 ceph-mon[79930]: pgmap v17623: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:46.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:15:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:15:46.462Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:15:46.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:15:46 smithi083 ceph-mon[77242]: pgmap v17623: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:47.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:15:46 smithi153 ceph-mon[70044]: pgmap v17623: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:48.657 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:15:48.944 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:15:48.945 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 6m ago 9h 27.3M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:15:48.945 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 4m ago 9h 64.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:15:48.946 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 6m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:15:48.946 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 4m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:15:48.947 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 4m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:15:48.947 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 6m ago 9h 990M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:15:48.947 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 6m ago 9h 2124M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:15:48.948 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 4m ago 9h 926M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:15:48.948 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 6m ago 9h 1273M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:15:48.949 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 6m ago 9h 24.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:15:48.949 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 4m ago 9h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:15:48.950 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 6m ago 9h 1543M 2177M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:15:48.950 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 6m ago 9h 1529M 2177M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:15:48.951 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 6m ago 9h 1187M 2177M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:15:48.951 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 6m ago 9h 1499M 2177M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:15:48.951 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 4m ago 9h 1313M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:15:48.952 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 4m ago 9h 1649M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:15:48.952 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 4m ago 9h 1474M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:15:48.953 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 4m ago 9h 1754M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:15:48.953 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 4m ago 9h 174M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:15:48.960 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:15:48 smithi083 ceph-mon[77242]: pgmap v17624: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:48.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:15:48 smithi083 ceph-mon[79930]: pgmap v17624: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:49.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:15:48 smithi153 ceph-mon[70044]: pgmap v17624: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:49.296 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:15:49.297 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:15:49.297 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:15:49.297 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:15:49.297 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:15:49.298 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:15:49.298 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:15:49.298 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:15:49.298 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:15:49.299 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:15:49.299 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:15:49.299 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:15:49.299 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:15:49.299 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:15:49.300 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:15:49.300 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:15:49.300 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:15:49.300 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:15:49.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:15:49 smithi083 ceph-mon[77242]: from='client.64555 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:15:49.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:15:49 smithi083 ceph-mon[77242]: from='client.64561 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:15:49.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:15:49 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4205042777' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:15:49.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:15:49 smithi083 ceph-mon[79930]: from='client.64555 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:15:49.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:15:49 smithi083 ceph-mon[79930]: from='client.64561 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:15:49.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:15:49 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4205042777' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:15:50.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:15:49 smithi153 ceph-mon[70044]: from='client.64555 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:15:50.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:15:49 smithi153 ceph-mon[70044]: from='client.64561 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:15:50.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:15:49 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4205042777' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:15:50.960 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:15:50 smithi083 ceph-mon[77242]: pgmap v17625: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:50.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:15:50 smithi083 ceph-mon[79930]: pgmap v17625: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:51.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:15:50 smithi153 ceph-mon[70044]: pgmap v17625: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:51.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:15:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:15:51] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T15:15:52.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:15:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:15:52.366Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:15:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:15:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:15:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:15:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:15:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:15:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:15:53.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:15:53 smithi153 ceph-mon[70044]: pgmap v17626: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:53.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:15:53 smithi083 ceph-mon[77242]: pgmap v17626: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:53.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:15:53 smithi083 ceph-mon[79930]: pgmap v17626: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:54.960 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:15:54 smithi083 ceph-mon[77242]: pgmap v17627: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:54.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:15:54 smithi083 ceph-mon[79930]: pgmap v17627: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:55.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:15:54 smithi153 ceph-mon[70044]: pgmap v17627: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:56.210 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:15:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:15:55.724Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:15:56.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:15:56 smithi083 ceph-mon[79930]: pgmap v17628: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:56.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:15:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:15:56.463Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:15:56.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:15:56 smithi083 ceph-mon[77242]: pgmap v17628: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:57.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:15:56 smithi153 ceph-mon[70044]: pgmap v17628: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:58.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:15:58 smithi083 ceph-mon[77242]: pgmap v17629: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:58.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:15:58 smithi083 ceph-mon[79930]: pgmap v17629: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:15:59.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:15:58 smithi153 ceph-mon[70044]: pgmap v17629: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:00.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:16:00 smithi083 ceph-mon[79930]: pgmap v17630: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:01.007 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:16:00 smithi083 ceph-mon[77242]: pgmap v17630: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:01.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:16:00 smithi153 ceph-mon[70044]: pgmap v17630: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:01.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:16:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:16:01] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T15:16:02.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:16:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:16:02.367Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:16:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:16:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:16:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:16:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:16:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:16:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:16:03.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:16:03 smithi153 ceph-mon[70044]: pgmap v17631: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:03.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:16:03 smithi083 ceph-mon[79930]: pgmap v17631: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:03.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:16:03 smithi083 ceph-mon[77242]: pgmap v17631: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:04.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:16:04 smithi083 ceph-mon[79930]: pgmap v17632: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:04.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:16:04 smithi083 ceph-mon[77242]: pgmap v17632: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:05.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:16:04 smithi153 ceph-mon[70044]: pgmap v17632: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:06.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:16:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:16:05.725Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:16:06.960 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:16:06 smithi083 ceph-mon[77242]: pgmap v17633: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:06.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:16:06 smithi083 ceph-mon[79930]: pgmap v17633: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:06.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:16:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:16:06.464Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:16:07.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:16:06 smithi153 ceph-mon[70044]: pgmap v17633: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:08.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:16:08 smithi153 ceph-mon[70044]: pgmap v17634: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:08.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:16:08 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:16:08.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:16:08 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:16:08.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:16:08 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:16:08.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:16:08 smithi083 ceph-mon[79930]: pgmap v17634: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:08.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:16:08 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:16:08.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:16:08 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:16:08.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:16:08 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:16:08.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:16:08 smithi083 ceph-mon[77242]: pgmap v17634: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:08.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:16:08 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:16:08.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:16:08 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:16:08.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:16:08 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:16:10.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:16:10 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:16:10.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:16:10 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:16:10.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:16:10 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:16:11.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:16:11 smithi083 ceph-mon[79930]: pgmap v17635: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:11.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:16:11 smithi083 ceph-mon[77242]: pgmap v17635: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:11.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:16:11 smithi153 ceph-mon[70044]: pgmap v17635: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:16:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:16:11] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T15:16:12.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:16:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:16:12.367Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:16:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:16:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:16:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:16:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:16:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:16:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:16:13.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:16:13 smithi153 ceph-mon[70044]: pgmap v17636: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:13.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:16:13 smithi083 ceph-mon[79930]: pgmap v17636: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:13.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:16:13 smithi083 ceph-mon[77242]: pgmap v17636: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:14.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:16:14 smithi083 ceph-mon[79930]: pgmap v17637: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:14.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:16:14 smithi083 ceph-mon[77242]: pgmap v17637: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:15.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:16:14 smithi153 ceph-mon[70044]: pgmap v17637: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:16.210 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:16:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:16:15.725Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:16:16.960 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:16:16 smithi083 ceph-mon[77242]: pgmap v17638: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:16.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:16:16 smithi083 ceph-mon[79930]: pgmap v17638: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:16.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:16:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:16:16.464Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:16:17.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:16:16 smithi153 ceph-mon[70044]: pgmap v17638: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:18.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:16:18 smithi083 ceph-mon[79930]: pgmap v17639: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:18.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:16:18 smithi083 ceph-mon[77242]: pgmap v17639: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:19.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:16:18 smithi153 ceph-mon[70044]: pgmap v17639: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:19.623 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:16:19.913 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:16:19.914 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 7m ago 9h 27.3M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:16:19.914 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 5m ago 9h 64.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:16:19.915 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 7m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:16:19.915 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 5m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:16:19.916 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 5m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:16:19.916 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 7m ago 9h 990M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:16:19.916 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 7m ago 9h 2124M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:16:19.917 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 5m ago 9h 926M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:16:19.917 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 7m ago 9h 1273M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:16:19.918 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 7m ago 9h 24.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:16:19.918 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 5m ago 9h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:16:19.919 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 7m ago 9h 1543M 2177M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:16:19.919 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 7m ago 9h 1529M 2177M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:16:19.920 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 7m ago 9h 1187M 2177M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:16:19.920 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 7m ago 9h 1499M 2177M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:16:19.921 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 5m ago 9h 1313M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:16:19.921 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 5m ago 9h 1649M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:16:19.921 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 5m ago 9h 1474M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:16:19.922 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 5m ago 9h 1754M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:16:19.922 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 5m ago 9h 174M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:16:20.265 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:16:20.266 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:16:20.266 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:16:20.266 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:16:20.266 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:16:20.267 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:16:20.267 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:16:20.267 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:16:20.268 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:16:20.268 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:16:20.269 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:16:20.269 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:16:20.269 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:16:20.270 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:16:20.270 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:16:20.270 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:16:20.270 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:16:20.271 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:16:20.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:16:20 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3589250401' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:16:20.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:16:20 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3589250401' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:16:20.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:16:20 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3589250401' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:16:21.710 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:16:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:16:21] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T15:16:21.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:16:21 smithi083 ceph-mon[77242]: from='client.54642 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:16:21.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:16:21 smithi083 ceph-mon[77242]: from='client.54648 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:16:21.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:16:21 smithi083 ceph-mon[77242]: pgmap v17640: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:21.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:16:21 smithi083 ceph-mon[79930]: from='client.54642 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:16:21.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:16:21 smithi083 ceph-mon[79930]: from='client.54648 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:16:21.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:16:21 smithi083 ceph-mon[79930]: pgmap v17640: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:21.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:16:21 smithi153 ceph-mon[70044]: from='client.54642 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:16:21.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:16:21 smithi153 ceph-mon[70044]: from='client.54648 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:16:21.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:16:21 smithi153 ceph-mon[70044]: pgmap v17640: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:22.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:16:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:16:22.368Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:16:22.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:16:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:16:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:16:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:16:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:16:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:16:23.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:16:23 smithi153 ceph-mon[70044]: pgmap v17641: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:23.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:16:23 smithi083 ceph-mon[79930]: pgmap v17641: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:23.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:16:23 smithi083 ceph-mon[77242]: pgmap v17641: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:24.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:16:24 smithi083 ceph-mon[79930]: pgmap v17642: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:24.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:16:24 smithi083 ceph-mon[77242]: pgmap v17642: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:25.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:16:24 smithi153 ceph-mon[70044]: pgmap v17642: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:26.210 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:16:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:16:25.727Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:16:26.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:16:26 smithi083 ceph-mon[79930]: pgmap v17643: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:26.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:16:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:16:26.466Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:16:26.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:16:26 smithi083 ceph-mon[77242]: pgmap v17643: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:27.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:16:26 smithi153 ceph-mon[70044]: pgmap v17643: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:28.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:16:28 smithi083 ceph-mon[79930]: pgmap v17644: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:28.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:16:28 smithi083 ceph-mon[77242]: pgmap v17644: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:29.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:16:28 smithi153 ceph-mon[70044]: pgmap v17644: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:30.960 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:16:30 smithi083 ceph-mon[77242]: pgmap v17645: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:30.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:16:30 smithi083 ceph-mon[79930]: pgmap v17645: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:31.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:16:30 smithi153 ceph-mon[70044]: pgmap v17645: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:31.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:16:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:16:31] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T15:16:32.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:16:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:16:32.369Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:16:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:16:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:16:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:16:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:16:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:16:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:16:33.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:16:33 smithi153 ceph-mon[70044]: pgmap v17646: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:33.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:16:33 smithi083 ceph-mon[79930]: pgmap v17646: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:33.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:16:33 smithi083 ceph-mon[77242]: pgmap v17646: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:34.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:16:34 smithi083 ceph-mon[79930]: pgmap v17647: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:34.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:16:34 smithi083 ceph-mon[77242]: pgmap v17647: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:35.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:16:34 smithi153 ceph-mon[70044]: pgmap v17647: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:36.210 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:16:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:16:35.728Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:16:36.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:16:36 smithi083 ceph-mon[79930]: pgmap v17648: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:36.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:16:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:16:36.467Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:16:36.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:16:36 smithi083 ceph-mon[77242]: pgmap v17648: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:37.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:16:36 smithi153 ceph-mon[70044]: pgmap v17648: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:38.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:16:38 smithi083 ceph-mon[79930]: pgmap v17649: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:38.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:16:38 smithi083 ceph-mon[77242]: pgmap v17649: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:39.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:16:38 smithi153 ceph-mon[70044]: pgmap v17649: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:40.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:16:40 smithi083 ceph-mon[79930]: pgmap v17650: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:40.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:16:40 smithi083 ceph-mon[77242]: pgmap v17650: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:16:40 smithi153 ceph-mon[70044]: pgmap v17650: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:41.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:16:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:16:41] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T15:16:41.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:16:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:16:41.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:16:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:16:41.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:16:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:16:41.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:16:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:16:41.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:16:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:16:41.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:16:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:16:41.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:16:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:16:41.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:16:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:16:42.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:16:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:16:42.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:16:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:16:42.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:16:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:16:42.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:16:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:16:42.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:16:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:16:42.370Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:16:42.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:16:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:16:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:16:43.310 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:16:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:16:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:16:43.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:16:43 smithi153 ceph-mon[70044]: pgmap v17651: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:43.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:16:43 smithi083 ceph-mon[79930]: pgmap v17651: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:43.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:16:43 smithi083 ceph-mon[77242]: pgmap v17651: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:44.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:16:44 smithi083 ceph-mon[79930]: pgmap v17652: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:44.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:16:44 smithi083 ceph-mon[77242]: pgmap v17652: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:45.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:16:44 smithi153 ceph-mon[70044]: pgmap v17652: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:46.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:16:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:16:45.728Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:16:46.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:16:46 smithi083 ceph-mon[79930]: pgmap v17653: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:46.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:16:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:16:46.467Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:16:46.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:16:46 smithi083 ceph-mon[77242]: pgmap v17653: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:47.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:16:46 smithi153 ceph-mon[70044]: pgmap v17653: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:48.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:16:48 smithi083 ceph-mon[79930]: pgmap v17654: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:48.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:16:48 smithi083 ceph-mon[77242]: pgmap v17654: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:49.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:16:48 smithi153 ceph-mon[70044]: pgmap v17654: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:50.594 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:16:50.888 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:16:50.889 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 7m ago 9h 27.3M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:16:50.889 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 5m ago 9h 64.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:16:50.890 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 7m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:16:50.890 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 5m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:16:50.891 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 5m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:16:50.891 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 7m ago 9h 990M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:16:50.892 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 7m ago 9h 2124M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:16:50.892 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 5m ago 9h 926M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:16:50.893 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 7m ago 9h 1273M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:16:50.893 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 7m ago 9h 24.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:16:50.894 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 5m ago 9h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:16:50.894 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 7m ago 9h 1543M 2177M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:16:50.895 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 7m ago 9h 1529M 2177M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:16:50.895 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 7m ago 9h 1187M 2177M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:16:50.896 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 7m ago 9h 1499M 2177M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:16:50.896 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 5m ago 9h 1313M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:16:50.897 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 5m ago 9h 1649M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:16:50.897 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 5m ago 9h 1474M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:16:50.897 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 5m ago 9h 1754M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:16:50.898 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 5m ago 9h 174M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:16:50.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:16:50 smithi083 ceph-mon[79930]: pgmap v17655: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:50.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:16:50 smithi083 ceph-mon[77242]: pgmap v17655: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:51.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:16:50 smithi153 ceph-mon[70044]: pgmap v17655: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:51.238 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:16:51.238 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:16:51.239 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:16:51.239 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:16:51.239 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:16:51.239 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:16:51.240 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:16:51.240 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:16:51.240 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:16:51.241 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:16:51.241 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:16:51.241 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:16:51.242 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:16:51.242 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:16:51.242 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:16:51.243 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:16:51.243 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:16:51.244 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:16:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:16:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:16:51] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T15:16:51.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:16:51 smithi083 ceph-mon[77242]: from='client.54660 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:16:51.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:16:51 smithi083 ceph-mon[77242]: from='client.64597 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:16:51.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:16:51 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4038582971' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:16:51.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:16:51 smithi083 ceph-mon[79930]: from='client.54660 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:16:51.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:16:51 smithi083 ceph-mon[79930]: from='client.64597 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:16:51.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:16:51 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4038582971' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:16:52.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:16:51 smithi153 ceph-mon[70044]: from='client.54660 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:16:52.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:16:51 smithi153 ceph-mon[70044]: from='client.64597 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:16:52.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:16:51 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4038582971' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:16:52.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:16:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:16:52.371Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:16:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:16:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:16:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:16:53.314 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:16:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:16:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:16:53.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:16:53 smithi153 ceph-mon[70044]: pgmap v17656: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:53.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:16:53 smithi083 ceph-mon[79930]: pgmap v17656: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:53.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:16:53 smithi083 ceph-mon[77242]: pgmap v17656: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:54.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:16:54 smithi083 ceph-mon[79930]: pgmap v17657: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:54.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:16:54 smithi083 ceph-mon[77242]: pgmap v17657: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:55.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:16:54 smithi153 ceph-mon[70044]: pgmap v17657: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:56.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:16:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:16:55.729Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:16:56.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:16:56 smithi083 ceph-mon[79930]: pgmap v17658: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:56.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:16:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:16:56.467Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:16:56.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:16:56 smithi083 ceph-mon[77242]: pgmap v17658: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:57.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:16:56 smithi153 ceph-mon[70044]: pgmap v17658: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:58.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:16:58 smithi083 ceph-mon[79930]: pgmap v17659: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:58.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:16:58 smithi083 ceph-mon[77242]: pgmap v17659: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:16:59.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:16:58 smithi153 ceph-mon[70044]: pgmap v17659: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:00.960 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:17:00 smithi083 ceph-mon[77242]: pgmap v17660: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:00.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:17:00 smithi083 ceph-mon[79930]: pgmap v17660: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:01.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:17:00 smithi153 ceph-mon[70044]: pgmap v17660: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:01.966 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:17:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:17:01] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T15:17:02.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:17:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:17:02.371Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:17:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:17:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:17:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:17:03.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:17:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:17:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:17:03.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:17:03 smithi153 ceph-mon[70044]: pgmap v17661: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:03.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:17:03 smithi083 ceph-mon[79930]: pgmap v17661: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:03.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:17:03 smithi083 ceph-mon[77242]: pgmap v17661: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:04.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:17:04 smithi083 ceph-mon[79930]: pgmap v17662: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:04.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:17:04 smithi083 ceph-mon[77242]: pgmap v17662: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:05.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:17:04 smithi153 ceph-mon[70044]: pgmap v17662: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:06.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:17:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:17:05.729Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:17:06.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:17:06 smithi083 ceph-mon[79930]: pgmap v17663: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:06.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:17:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:17:06.468Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:17:06.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:17:06 smithi083 ceph-mon[77242]: pgmap v17663: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:07.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:17:06 smithi153 ceph-mon[70044]: pgmap v17663: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:08.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:17:08 smithi083 ceph-mon[79930]: pgmap v17664: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:08.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:17:08 smithi083 ceph-mon[77242]: pgmap v17664: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:09.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:17:08 smithi153 ceph-mon[70044]: pgmap v17664: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:09.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:17:09 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:17:09.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:17:09 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:17:09.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:17:09 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:17:09.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:17:09 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:17:09.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:17:09 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:17:09.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:17:09 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:17:10.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:17:09 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:17:10.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:17:09 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:17:10.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:17:09 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:17:10.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:17:10 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:17:10.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:17:10 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:17:10.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:17:10 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:17:10.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:17:10 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:17:10.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:17:10 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:17:10.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:17:10 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:17:10.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:17:10 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:17:10.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:17:10 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:17:10.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:17:10 smithi083 ceph-mon[79930]: Adjusting osd_memory_target on smithi083 to 2162M 2022-09-29T15:17:10.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:17:10 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:17:10.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:17:10 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:17:10.968 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:17:10 smithi083 ceph-mon[79930]: pgmap v17665: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:10.968 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:17:10 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:17:10.969 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:17:10 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:17:10.969 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:17:10 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:17:10.970 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:17:10 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:17:10.970 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:17:10 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:17:10.971 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:17:10 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:17:10.971 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:17:10 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:17:10.971 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:17:10 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:17:10.972 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:17:10 smithi083 ceph-mon[77242]: Adjusting osd_memory_target on smithi083 to 2162M 2022-09-29T15:17:10.972 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:17:10 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:17:10.974 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:17:10 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:17:10.974 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:17:10 smithi083 ceph-mon[77242]: pgmap v17665: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:11.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:17:10 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:17:11.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:17:10 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:17:11.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:17:10 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:17:11.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:17:10 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:17:11.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:17:10 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:17:11.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:17:10 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:17:11.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:17:10 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:17:11.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:17:10 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:17:11.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:17:10 smithi153 ceph-mon[70044]: Adjusting osd_memory_target on smithi083 to 2162M 2022-09-29T15:17:11.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:17:10 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:17:11.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:17:10 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:17:11.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:17:10 smithi153 ceph-mon[70044]: pgmap v17665: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:17:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:17:11] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T15:17:12.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:17:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:17:12.372Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:17:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:17:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:17:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:17:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:17:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:17:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:17:13.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:17:13 smithi153 ceph-mon[70044]: pgmap v17666: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:13.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:17:13 smithi083 ceph-mon[79930]: pgmap v17666: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:13.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:17:13 smithi083 ceph-mon[77242]: pgmap v17666: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:14.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:17:14 smithi083 ceph-mon[79930]: pgmap v17667: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:14.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:17:14 smithi083 ceph-mon[77242]: pgmap v17667: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:15.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:17:14 smithi153 ceph-mon[70044]: pgmap v17667: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:16.210 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:17:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:17:15.730Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:17:16.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:17:16 smithi083 ceph-mon[79930]: pgmap v17668: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:16.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:17:16 smithi083 ceph-mon[77242]: pgmap v17668: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:16.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:17:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:17:16.468Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:17:17.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:17:16 smithi153 ceph-mon[70044]: pgmap v17668: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:18.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:17:18 smithi083 ceph-mon[79930]: pgmap v17669: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:18.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:17:18 smithi083 ceph-mon[77242]: pgmap v17669: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:19.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:17:18 smithi153 ceph-mon[70044]: pgmap v17669: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:20.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:17:20 smithi083 ceph-mon[79930]: pgmap v17670: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:20.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:17:20 smithi083 ceph-mon[77242]: pgmap v17670: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:21.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:17:20 smithi153 ceph-mon[70044]: pgmap v17670: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:21.568 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:17:21.857 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:17:21.858 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 8m ago 9h 27.3M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:17:21.859 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 6m ago 9h 64.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:17:21.859 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 8m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:17:21.859 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 6m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:17:21.860 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 6m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:17:21.860 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 8m ago 9h 990M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:17:21.861 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 8m ago 9h 2124M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:17:21.861 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 6m ago 9h 926M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:17:21.862 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 8m ago 9h 1273M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:17:21.862 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 8m ago 9h 24.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:17:21.863 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 6m ago 9h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:17:21.863 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 8m ago 9h 1543M 2162M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:17:21.864 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 8m ago 9h 1529M 2162M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:17:21.864 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 8m ago 9h 1187M 2162M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:17:21.865 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 8m ago 9h 1499M 2162M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:17:21.865 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 6m ago 9h 1313M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:17:21.866 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 6m ago 9h 1649M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:17:21.866 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 6m ago 9h 1474M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:17:21.866 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 6m ago 9h 1754M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:17:21.867 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 6m ago 9h 174M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:17:21.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:17:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:17:21] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T15:17:22.211 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:17:22.211 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:17:22.212 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:17:22.212 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:17:22.212 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:17:22.213 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:17:22.213 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:17:22.213 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:17:22.213 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:17:22.214 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:17:22.214 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:17:22.214 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:17:22.214 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:17:22.215 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:17:22.215 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:17:22.215 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:17:22.215 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:17:22.216 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:17:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:17:22 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/613505410' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:17:22.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:17:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:17:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:17:22.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:17:22 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/613505410' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:17:22.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:17:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:17:22.372Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:17:22.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:17:22 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/613505410' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:17:23.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:17:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:17:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:17:23.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:17:23 smithi153 ceph-mon[70044]: from='client.54678 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:17:23.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:17:23 smithi153 ceph-mon[70044]: from='client.54684 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:17:23.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:17:23 smithi153 ceph-mon[70044]: pgmap v17671: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:23.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:17:23 smithi083 ceph-mon[79930]: from='client.54678 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:17:23.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:17:23 smithi083 ceph-mon[79930]: from='client.54684 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:17:23.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:17:23 smithi083 ceph-mon[79930]: pgmap v17671: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:23.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:17:23 smithi083 ceph-mon[77242]: from='client.54678 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:17:23.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:17:23 smithi083 ceph-mon[77242]: from='client.54684 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:17:23.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:17:23 smithi083 ceph-mon[77242]: pgmap v17671: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:24.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:17:24 smithi083 ceph-mon[79930]: pgmap v17672: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:24.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:17:24 smithi083 ceph-mon[77242]: pgmap v17672: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:25.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:17:24 smithi153 ceph-mon[70044]: pgmap v17672: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:26.210 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:17:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:17:25.730Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:17:26.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:17:26 smithi083 ceph-mon[79930]: pgmap v17673: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:26.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:17:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:17:26.469Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:17:26.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:17:26 smithi083 ceph-mon[77242]: pgmap v17673: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:27.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:17:26 smithi153 ceph-mon[70044]: pgmap v17673: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:28.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:17:28 smithi083 ceph-mon[79930]: pgmap v17674: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:28.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:17:28 smithi083 ceph-mon[77242]: pgmap v17674: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:29.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:17:28 smithi153 ceph-mon[70044]: pgmap v17674: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:30.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:17:30 smithi083 ceph-mon[79930]: pgmap v17675: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:30.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:17:30 smithi083 ceph-mon[77242]: pgmap v17675: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:31.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:17:30 smithi153 ceph-mon[70044]: pgmap v17675: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:17:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:17:31] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T15:17:32.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:17:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:17:32.373Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:17:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:17:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:17:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:17:33.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:17:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:17:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:17:33.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:17:33 smithi153 ceph-mon[70044]: pgmap v17676: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:33.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:17:33 smithi083 ceph-mon[79930]: pgmap v17676: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:33.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:17:33 smithi083 ceph-mon[77242]: pgmap v17676: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:34.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:17:34 smithi083 ceph-mon[79930]: pgmap v17677: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:34.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:17:34 smithi083 ceph-mon[77242]: pgmap v17677: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:35.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:17:34 smithi153 ceph-mon[70044]: pgmap v17677: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:36.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:17:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:17:35.731Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:17:36.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:17:36 smithi083 ceph-mon[79930]: pgmap v17678: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:36.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:17:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:17:36.469Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:17:36.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:17:36 smithi083 ceph-mon[77242]: pgmap v17678: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:37.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:17:36 smithi153 ceph-mon[70044]: pgmap v17678: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:38.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:17:38 smithi083 ceph-mon[79930]: pgmap v17679: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:38.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:17:38 smithi083 ceph-mon[77242]: pgmap v17679: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:39.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:17:38 smithi153 ceph-mon[70044]: pgmap v17679: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:40.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:17:40 smithi083 ceph-mon[79930]: pgmap v17680: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:40.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:17:40 smithi083 ceph-mon[77242]: pgmap v17680: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:17:40 smithi153 ceph-mon[70044]: pgmap v17680: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:17:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:17:41] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T15:17:41.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:17:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:17:41.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:17:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:17:41.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:17:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:17:41.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:17:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:17:41.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:17:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:17:41.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:17:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:17:41.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:17:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:17:41.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:17:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:17:42.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:17:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:17:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:17:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:17:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:17:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:17:42.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:17:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:17:42.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:17:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:17:42.374Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:17:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:17:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:17:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:17:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:17:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:17:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:17:43.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:17:43 smithi153 ceph-mon[70044]: pgmap v17681: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:43.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:17:43 smithi083 ceph-mon[79930]: pgmap v17681: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:43.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:17:43 smithi083 ceph-mon[77242]: pgmap v17681: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:44.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:17:44 smithi083 ceph-mon[79930]: pgmap v17682: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:44.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:17:44 smithi083 ceph-mon[77242]: pgmap v17682: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:45.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:17:44 smithi153 ceph-mon[70044]: pgmap v17682: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:46.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:17:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:17:45.732Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:17:46.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:17:46 smithi083 ceph-mon[79930]: pgmap v17683: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:46.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:17:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:17:46.469Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:17:46.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:17:46 smithi083 ceph-mon[77242]: pgmap v17683: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:47.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:17:46 smithi153 ceph-mon[70044]: pgmap v17683: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:48.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:17:48 smithi083 ceph-mon[77242]: pgmap v17684: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:48.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:17:48 smithi083 ceph-mon[79930]: pgmap v17684: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:49.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:17:48 smithi153 ceph-mon[70044]: pgmap v17684: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:50.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:17:50 smithi083 ceph-mon[79930]: pgmap v17685: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:50.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:17:50 smithi083 ceph-mon[77242]: pgmap v17685: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:51.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:17:50 smithi153 ceph-mon[70044]: pgmap v17685: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:51.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:17:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:17:51] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T15:17:52.536 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:17:52.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:17:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:17:52.374Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:17:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:17:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:17:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:17:52.827 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:17:52.827 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 8m ago 9h 27.3M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:17:52.828 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 6m ago 9h 64.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:17:52.828 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 8m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:17:52.829 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 6m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:17:52.829 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 6m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:17:52.830 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 8m ago 9h 990M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:17:52.831 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 8m ago 9h 2124M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:17:52.831 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 6m ago 9h 926M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:17:52.831 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 8m ago 9h 1273M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:17:52.831 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 8m ago 9h 24.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:17:52.832 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 6m ago 9h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:17:52.832 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 8m ago 9h 1543M 2162M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:17:52.832 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 8m ago 9h 1529M 2162M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:17:52.832 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 8m ago 9h 1187M 2162M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:17:52.833 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 8m ago 9h 1499M 2162M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:17:52.833 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 6m ago 9h 1313M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:17:52.833 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 6m ago 9h 1649M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:17:52.833 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 6m ago 9h 1474M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:17:52.834 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 6m ago 9h 1754M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:17:52.834 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 6m ago 9h 174M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:17:53.185 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:17:53.185 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:17:53.186 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:17:53.186 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:17:53.186 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:17:53.187 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:17:53.187 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:17:53.187 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:17:53.187 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:17:53.188 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:17:53.188 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:17:53.188 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:17:53.188 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:17:53.188 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:17:53.189 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:17:53.189 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:17:53.189 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:17:53.189 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:17:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:17:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:17:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:17:53.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:17:53 smithi153 ceph-mon[70044]: pgmap v17686: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:53.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:17:53 smithi153 ceph-mon[70044]: from='client.64627 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:17:53.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:17:53 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2645635066' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:17:53.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:17:53 smithi083 ceph-mon[79930]: pgmap v17686: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:53.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:17:53 smithi083 ceph-mon[79930]: from='client.64627 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:17:53.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:17:53 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2645635066' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:17:53.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:17:53 smithi083 ceph-mon[77242]: pgmap v17686: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:53.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:17:53 smithi083 ceph-mon[77242]: from='client.64627 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:17:53.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:17:53 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2645635066' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:17:54.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:17:54 smithi153 ceph-mon[70044]: from='client.64633 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:17:54.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:17:54 smithi083 ceph-mon[79930]: from='client.64633 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:17:54.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:17:54 smithi083 ceph-mon[77242]: from='client.64633 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:17:55.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:17:55 smithi153 ceph-mon[70044]: pgmap v17687: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:55.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:17:55 smithi083 ceph-mon[79930]: pgmap v17687: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:55.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:17:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:17:55.733Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:17:55.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:17:55 smithi083 ceph-mon[77242]: pgmap v17687: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:56.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:17:56 smithi083 ceph-mon[79930]: pgmap v17688: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:56.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:17:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:17:56.470Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:17:56.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:17:56 smithi083 ceph-mon[77242]: pgmap v17688: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:57.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:17:56 smithi153 ceph-mon[70044]: pgmap v17688: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:58.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:17:58 smithi083 ceph-mon[77242]: pgmap v17689: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:58.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:17:58 smithi083 ceph-mon[79930]: pgmap v17689: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:17:59.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:17:58 smithi153 ceph-mon[70044]: pgmap v17689: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:00.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:18:00 smithi083 ceph-mon[79930]: pgmap v17690: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:00.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:18:00 smithi083 ceph-mon[77242]: pgmap v17690: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:01.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:18:00 smithi153 ceph-mon[70044]: pgmap v17690: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:18:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:18:01] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T15:18:02.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:18:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:18:02.375Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:18:02.812 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:18:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:18:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:18:03.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:18:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:18:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:18:03.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:18:03 smithi153 ceph-mon[70044]: pgmap v17691: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:03.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:18:03 smithi083 ceph-mon[79930]: pgmap v17691: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:03.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:18:03 smithi083 ceph-mon[77242]: pgmap v17691: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:04.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:18:04 smithi083 ceph-mon[79930]: pgmap v17692: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:04.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:18:04 smithi083 ceph-mon[77242]: pgmap v17692: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:05.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:18:04 smithi153 ceph-mon[70044]: pgmap v17692: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:06.215 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:18:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:18:05.734Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:18:06.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:18:06 smithi083 ceph-mon[79930]: pgmap v17693: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:06.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:18:06 smithi083 ceph-mon[77242]: pgmap v17693: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:06.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:18:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:18:06.471Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:18:07.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:18:06 smithi153 ceph-mon[70044]: pgmap v17693: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:08.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:18:08 smithi083 ceph-mon[79930]: pgmap v17694: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:08.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:18:08 smithi083 ceph-mon[77242]: pgmap v17694: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:09.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:18:08 smithi153 ceph-mon[70044]: pgmap v17694: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:10.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:18:09 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:18:10.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:18:09 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:18:10.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:18:09 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:18:10.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:18:09 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:18:10.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:18:09 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:18:10.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:18:09 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:18:10.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:18:09 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:18:10.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:18:09 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:18:10.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:18:09 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:18:11.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:18:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:18:11] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T15:18:11.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:18:11 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:18:11.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:18:11 smithi083 ceph-mon[77242]: pgmap v17695: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:11.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:18:11 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:18:11.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:18:11 smithi083 ceph-mon[79930]: pgmap v17695: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:11.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:18:11 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:18:11.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:18:11 smithi153 ceph-mon[70044]: pgmap v17695: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:12.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:18:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:18:12.376Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:18:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:18:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:18:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:18:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:18:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:18:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:18:13.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:18:13 smithi153 ceph-mon[70044]: pgmap v17696: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:13.960 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:18:13 smithi083 ceph-mon[77242]: pgmap v17696: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:13.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:18:13 smithi083 ceph-mon[79930]: pgmap v17696: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:14.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:18:14 smithi083 ceph-mon[79930]: pgmap v17697: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:14.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:18:14 smithi083 ceph-mon[77242]: pgmap v17697: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:15.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:18:14 smithi153 ceph-mon[70044]: pgmap v17697: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:16.210 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:18:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:18:15.734Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:18:16.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:18:16 smithi083 ceph-mon[79930]: pgmap v17698: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:16.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:18:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:18:16.472Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:18:16.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:18:16 smithi083 ceph-mon[77242]: pgmap v17698: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:17.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:18:16 smithi153 ceph-mon[70044]: pgmap v17698: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:18.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:18:18 smithi083 ceph-mon[79930]: pgmap v17699: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:18.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:18:18 smithi083 ceph-mon[77242]: pgmap v17699: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:19.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:18:18 smithi153 ceph-mon[70044]: pgmap v17699: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:20.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:18:20 smithi083 ceph-mon[77242]: pgmap v17700: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:20.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:18:20 smithi083 ceph-mon[79930]: pgmap v17700: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:21.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:18:20 smithi153 ceph-mon[70044]: pgmap v17700: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:21.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:18:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:18:21] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T15:18:22.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:18:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:18:22.376Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:18:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:18:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:18:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:18:23.310 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:18:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:18:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:18:23.518 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:18:23.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:18:23 smithi153 ceph-mon[70044]: pgmap v17701: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:23.814 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:18:23.814 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 9m ago 9h 27.3M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:18:23.814 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 7m ago 9h 64.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:18:23.815 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 9m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:18:23.815 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 7m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:18:23.815 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 7m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:18:23.815 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 9m ago 9h 990M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:18:23.816 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 9m ago 9h 2124M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:18:23.816 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 7m ago 9h 926M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:18:23.816 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 9m ago 9h 1273M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:18:23.817 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 9m ago 9h 24.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:18:23.817 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 7m ago 9h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:18:23.817 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 9m ago 9h 1543M 2162M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:18:23.817 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 9m ago 9h 1529M 2162M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:18:23.818 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 9m ago 9h 1187M 2162M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:18:23.818 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 9m ago 9h 1499M 2162M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:18:23.818 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 7m ago 9h 1313M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:18:23.818 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 7m ago 9h 1649M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:18:23.819 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 7m ago 9h 1474M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:18:23.819 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 7m ago 9h 1754M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:18:23.819 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 7m ago 9h 174M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:18:23.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:18:23 smithi083 ceph-mon[79930]: pgmap v17701: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:23.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:18:23 smithi083 ceph-mon[77242]: pgmap v17701: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:24.172 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:18:24.172 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:18:24.173 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:18:24.173 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:18:24.174 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:18:24.175 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:18:24.175 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:18:24.175 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:18:24.176 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:18:24.176 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:18:24.177 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:18:24.177 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:18:24.178 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:18:24.178 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:18:24.179 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:18:24.179 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:18:24.180 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:18:24.180 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:18:24.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:18:24 smithi153 ceph-mon[70044]: from='client.54714 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:18:24.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:18:24 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1596742632' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:18:24.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:18:24 smithi083 ceph-mon[79930]: from='client.54714 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:18:24.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:18:24 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1596742632' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:18:24.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:18:24 smithi083 ceph-mon[77242]: from='client.54714 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:18:24.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:18:24 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1596742632' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:18:25.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:18:25 smithi153 ceph-mon[70044]: from='client.54720 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:18:25.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:18:25 smithi153 ceph-mon[70044]: pgmap v17702: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:25.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:18:25 smithi083 ceph-mon[79930]: from='client.54720 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:18:25.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:18:25 smithi083 ceph-mon[79930]: pgmap v17702: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:25.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:18:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:18:25.734Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:18:25.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:18:25 smithi083 ceph-mon[77242]: from='client.54720 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:18:25.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:18:25 smithi083 ceph-mon[77242]: pgmap v17702: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:26.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:18:26 smithi083 ceph-mon[79930]: pgmap v17703: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:26.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:18:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:18:26.472Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:18:26.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:18:26 smithi083 ceph-mon[77242]: pgmap v17703: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:27.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:18:26 smithi153 ceph-mon[70044]: pgmap v17703: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:28.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:18:28 smithi083 ceph-mon[79930]: pgmap v17704: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:28.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:18:28 smithi083 ceph-mon[77242]: pgmap v17704: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:29.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:18:28 smithi153 ceph-mon[70044]: pgmap v17704: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:30.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:18:30 smithi083 ceph-mon[79930]: pgmap v17705: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:30.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:18:30 smithi083 ceph-mon[77242]: pgmap v17705: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:31.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:18:30 smithi153 ceph-mon[70044]: pgmap v17705: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:18:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:18:31] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T15:18:32.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:18:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:18:32.377Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:18:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:18:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:18:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:18:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:18:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:18:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:18:33.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:18:33 smithi153 ceph-mon[70044]: pgmap v17706: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:33.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:18:33 smithi083 ceph-mon[79930]: pgmap v17706: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:33.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:18:33 smithi083 ceph-mon[77242]: pgmap v17706: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:34.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:18:34 smithi083 ceph-mon[79930]: pgmap v17707: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:34.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:18:34 smithi083 ceph-mon[77242]: pgmap v17707: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:35.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:18:34 smithi153 ceph-mon[70044]: pgmap v17707: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:36.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:18:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:18:35.735Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:18:36.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:18:36 smithi083 ceph-mon[79930]: pgmap v17708: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:36.965 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:18:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:18:36.473Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:18:36.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:18:36 smithi083 ceph-mon[77242]: pgmap v17708: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:37.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:18:36 smithi153 ceph-mon[70044]: pgmap v17708: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:38.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:18:38 smithi083 ceph-mon[79930]: pgmap v17709: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:38.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:18:38 smithi083 ceph-mon[77242]: pgmap v17709: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:39.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:18:38 smithi153 ceph-mon[70044]: pgmap v17709: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:40.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:18:40 smithi083 ceph-mon[79930]: pgmap v17710: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:40.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:18:40 smithi083 ceph-mon[77242]: pgmap v17710: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:18:40 smithi153 ceph-mon[70044]: pgmap v17710: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:41.969 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:18:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:18:41] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T15:18:41.971 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:18:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:18:41.971 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:18:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:18:41.972 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:18:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:18:41.972 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:18:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:18:41.973 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:18:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:18:41.973 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:18:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:18:41.974 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:18:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:18:41.974 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:18:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:18:42.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:18:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:18:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:18:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:18:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:18:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:18:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:18:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:18:42.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:18:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:18:42.378Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:18:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:18:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:18:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:18:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:18:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:18:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:18:43.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:18:43 smithi153 ceph-mon[70044]: pgmap v17711: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:43.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:18:43 smithi083 ceph-mon[79930]: pgmap v17711: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:43.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:18:43 smithi083 ceph-mon[77242]: pgmap v17711: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:44.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:18:44 smithi083 ceph-mon[79930]: pgmap v17712: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:44.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:18:44 smithi083 ceph-mon[77242]: pgmap v17712: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:45.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:18:44 smithi153 ceph-mon[70044]: pgmap v17712: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:46.210 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:18:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:18:45.735Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:18:46.987 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:18:46 smithi083 ceph-mon[79930]: pgmap v17713: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:46.988 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:18:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:18:46.473Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:18:46.988 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:18:46 smithi083 ceph-mon[77242]: pgmap v17713: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:47.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:18:46 smithi153 ceph-mon[70044]: pgmap v17713: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:48.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:18:48 smithi083 ceph-mon[79930]: pgmap v17714: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:48.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:18:48 smithi083 ceph-mon[77242]: pgmap v17714: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:49.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:18:48 smithi153 ceph-mon[70044]: pgmap v17714: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:50.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:18:50 smithi083 ceph-mon[79930]: pgmap v17715: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:50.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:18:50 smithi083 ceph-mon[77242]: pgmap v17715: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:51.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:18:50 smithi153 ceph-mon[70044]: pgmap v17715: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:51.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:18:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:18:51] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T15:18:52.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:18:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:18:52.378Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:18:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:18:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:18:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:18:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:18:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:18:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:18:53.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:18:53 smithi153 ceph-mon[70044]: pgmap v17716: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:53.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:18:53 smithi083 ceph-mon[79930]: pgmap v17716: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:53.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:18:53 smithi083 ceph-mon[77242]: pgmap v17716: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:54.500 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:18:54.816 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:18:54.816 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 9m ago 9h 27.3M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:18:54.816 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 7m ago 9h 64.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:18:54.817 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 9m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:18:54.817 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 7m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:18:54.817 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 7m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:18:54.817 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 9m ago 9h 990M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:18:54.818 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 9m ago 9h 2124M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:18:54.818 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 7m ago 9h 926M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:18:54.818 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 9m ago 9h 1273M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:18:54.818 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 9m ago 9h 24.0M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:18:54.819 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 7m ago 9h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:18:54.819 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 9m ago 9h 1543M 2162M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:18:54.819 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 9m ago 9h 1529M 2162M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:18:54.819 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 9m ago 9h 1187M 2162M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:18:54.820 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 9m ago 9h 1499M 2162M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:18:54.820 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 7m ago 9h 1313M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:18:54.820 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 7m ago 9h 1649M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:18:54.820 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 7m ago 9h 1474M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:18:54.821 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 7m ago 9h 1754M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:18:54.821 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 7m ago 9h 174M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:18:54.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:18:54 smithi083 ceph-mon[79930]: from='client.64663 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:18:54.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:18:54 smithi083 ceph-mon[79930]: pgmap v17717: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:54.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:18:54 smithi083 ceph-mon[77242]: from='client.64663 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:18:54.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:18:54 smithi083 ceph-mon[77242]: pgmap v17717: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:55.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:18:54 smithi153 ceph-mon[70044]: from='client.64663 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:18:55.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:18:54 smithi153 ceph-mon[70044]: pgmap v17717: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:55.177 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:18:55.177 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:18:55.178 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:18:55.178 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:18:55.179 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:18:55.179 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:18:55.180 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:18:55.180 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:18:55.181 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:18:55.181 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:18:55.182 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:18:55.182 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:18:55.183 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:18:55.183 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:18:55.184 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:18:55.184 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:18:55.185 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:18:55.185 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:18:55.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:18:55 smithi083 ceph-mon[79930]: from='client.54738 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:18:55.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:18:55 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3637696101' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:18:55.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:18:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:18:55.735Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:18:55.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:18:55 smithi083 ceph-mon[77242]: from='client.54738 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:18:55.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:18:55 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3637696101' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:18:56.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:18:55 smithi153 ceph-mon[70044]: from='client.54738 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:18:56.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:18:55 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3637696101' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:18:56.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:18:56 smithi083 ceph-mon[79930]: pgmap v17718: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:56.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:18:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:18:56.474Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:18:56.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:18:56 smithi083 ceph-mon[77242]: pgmap v17718: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:57.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:18:56 smithi153 ceph-mon[70044]: pgmap v17718: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:58.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:18:58 smithi083 ceph-mon[79930]: pgmap v17719: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:58.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:18:58 smithi083 ceph-mon[77242]: pgmap v17719: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:18:59.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:18:58 smithi153 ceph-mon[70044]: pgmap v17719: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:19:00.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:19:00 smithi083 ceph-mon[79930]: pgmap v17720: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:19:00.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:19:00 smithi083 ceph-mon[77242]: pgmap v17720: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:19:01.064 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:19:00 smithi153 ceph-mon[70044]: pgmap v17720: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:19:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:19:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:19:01] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T15:19:02.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:19:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:19:02.379Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:19:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:19:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:19:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:19:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:19:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:19:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:19:03.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:19:03 smithi153 ceph-mon[70044]: pgmap v17721: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:19:03.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:19:03 smithi083 ceph-mon[79930]: pgmap v17721: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:19:03.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:19:03 smithi083 ceph-mon[77242]: pgmap v17721: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:19:04.971 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:19:04 smithi083 ceph-mon[79930]: pgmap v17722: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:19:04.971 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:19:04 smithi083 ceph-mon[77242]: pgmap v17722: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:19:05.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:19:04 smithi153 ceph-mon[70044]: pgmap v17722: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:19:06.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:19:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:19:05.736Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:19:06.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:19:06 smithi083 ceph-mon[79930]: pgmap v17723: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:19:06.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:19:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:19:06.476Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:19:06.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:19:06 smithi083 ceph-mon[77242]: pgmap v17723: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:19:07.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:19:06 smithi153 ceph-mon[70044]: pgmap v17723: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:19:08.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:19:08 smithi083 ceph-mon[79930]: pgmap v17724: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:19:08.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:19:08 smithi083 ceph-mon[77242]: pgmap v17724: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:19:09.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:19:08 smithi153 ceph-mon[70044]: pgmap v17724: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:19:10.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:19:10 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:19:10.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:19:10 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:19:10.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:19:10 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:19:10.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:19:10 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:19:10.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:19:10 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:19:10.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:19:10 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:19:10.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:19:10 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:19:10.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:19:10 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:19:10.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:19:10 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:19:11.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:19:11 smithi153 ceph-mon[70044]: pgmap v17725: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:19:11.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:19:11 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:19:11.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:19:11 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:19:11.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:19:11 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:19:11.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:19:11 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:19:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:19:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:19:11] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T15:19:11.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:19:11 smithi083 ceph-mon[79930]: pgmap v17725: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:19:11.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:19:11 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:19:11.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:19:11 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:19:11.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:19:11 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:19:11.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:19:11 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:19:11.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:19:11 smithi083 ceph-mon[77242]: pgmap v17725: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:19:11.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:19:11 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:19:11.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:19:11 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:19:11.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:19:11 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:19:11.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:19:11 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:19:12.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:19:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:19:12.379Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:19:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:19:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:19:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:19:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:19:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:19:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:19:13.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:19:13 smithi153 ceph-mon[70044]: pgmap v17726: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:19:13.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:19:13 smithi083 ceph-mon[79930]: pgmap v17726: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:19:13.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:19:13 smithi083 ceph-mon[77242]: pgmap v17726: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:19:15.743 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:19:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:19:15.737Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:19:16.118 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:19:16 smithi083 ceph-mon[77242]: pgmap v17727: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:19:16.460 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:19:16 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:19:16.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:19:16 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:19:16.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:19:16 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:19:16.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:19:16 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:19:16.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:19:16 smithi083 ceph-mon[79930]: pgmap v17727: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:19:16.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:19:16 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:19:16.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:19:16 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:19:16.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:19:16 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:19:16.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:19:16 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:19:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:19:16 smithi153 ceph-mon[70044]: pgmap v17727: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:19:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:19:16 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:19:16.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:19:16 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:19:16.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:19:16 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:19:16.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:19:16 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:19:16.917 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:19:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:19:16.476Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:19:18.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:19:18 smithi083 ceph-mon[79930]: pgmap v17728: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:19:18.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:19:18 smithi083 ceph-mon[77242]: pgmap v17728: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:19:18.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:19:18 smithi153 ceph-mon[70044]: pgmap v17728: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:19:19.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:19:19 smithi083 ceph-mon[79930]: pgmap v17729: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:19:19.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:19:19 smithi083 ceph-mon[79930]: Detected new or changed devices on smithi083 2022-09-29T15:19:19.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:19:19 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:19:19.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:19:19 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:19:19.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:19:19 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:19:19.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:19:19 smithi083 ceph-mon[77242]: pgmap v17729: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:19:19.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:19:19 smithi083 ceph-mon[77242]: Detected new or changed devices on smithi083 2022-09-29T15:19:19.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:19:19 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:19:19.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:19:19 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:19:19.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:19:19 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:19:20.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:19:19 smithi153 ceph-mon[70044]: pgmap v17729: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:19:20.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:19:19 smithi153 ceph-mon[70044]: Detected new or changed devices on smithi083 2022-09-29T15:19:20.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:19:19 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:19:20.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:19:19 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:19:20.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:19:19 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:19:21.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:19:21 smithi083 ceph-mon[79930]: pgmap v17730: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:19:21.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:19:21 smithi083 ceph-mon[77242]: pgmap v17730: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:19:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:19:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:19:21] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T15:19:22.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:19:21 smithi153 ceph-mon[70044]: pgmap v17730: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:19:22.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:19:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:19:22.380Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:19:22.817 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:19:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:19:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:19:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:19:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:19:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:19:24.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:19:24 smithi153 ceph-mon[70044]: pgmap v17731: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:19:24.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:19:24 smithi083 ceph-mon[79930]: pgmap v17731: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:19:24.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:19:24 smithi083 ceph-mon[77242]: pgmap v17731: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:19:25.501 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:19:25.791 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:19:25.791 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 10s ago 9h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:19:25.791 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 8m ago 9h 64.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:19:25.792 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 10s ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:19:25.792 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 8m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:19:25.792 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 8m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:19:25.792 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 10s ago 9h 997M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:19:25.793 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 10s ago 9h 2159M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:19:25.793 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 8m ago 9h 926M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:19:25.793 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 10s ago 9h 1295M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:19:25.793 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 10s ago 9h 24.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:19:25.793 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 8m ago 9h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:19:25.794 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 10s ago 9h 1568M 2162M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:19:25.794 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 10s ago 9h 1555M 2162M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:19:25.794 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 10s ago 9h 1208M 2162M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:19:25.794 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 10s ago 9h 1524M 2162M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:19:25.795 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 8m ago 9h 1313M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:19:25.795 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 8m ago 9h 1649M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:19:25.795 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 8m ago 9h 1474M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:19:25.795 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 8m ago 9h 1754M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:19:25.796 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 8m ago 9h 174M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:19:26.147 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:19:26.147 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:19:26.148 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:19:26.148 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:19:26.148 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:19:26.149 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:19:26.149 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:19:26.149 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:19:26.150 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:19:26.150 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:19:26.151 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:19:26.151 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:19:26.151 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:19:26.152 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:19:26.152 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:19:26.153 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:19:26.153 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:19:26.153 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:19:26.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:19:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:19:25.736Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:19:26.816 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:19:26 smithi153 ceph-mon[70044]: pgmap v17732: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:19:26.817 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:19:26 smithi153 ceph-mon[70044]: from='client.54750 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:19:26.818 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:19:26 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3918277719' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:19:26.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:19:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:19:26.477Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:19:26.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:19:26 smithi083 ceph-mon[79930]: pgmap v17732: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:19:26.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:19:26 smithi083 ceph-mon[79930]: from='client.54750 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:19:26.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:19:26 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3918277719' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:19:26.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:19:26 smithi083 ceph-mon[77242]: pgmap v17732: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:19:26.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:19:26 smithi083 ceph-mon[77242]: from='client.54750 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:19:26.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:19:26 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3918277719' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:19:27.832 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:19:27 smithi153 ceph-mon[70044]: from='client.64687 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:19:27.970 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:19:27 smithi083 ceph-mon[79930]: from='client.64687 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:19:27.971 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:19:27 smithi083 ceph-mon[77242]: from='client.64687 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:19:28.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:19:28 smithi153 ceph-mon[70044]: pgmap v17733: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:19:28.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:19:28 smithi083 ceph-mon[79930]: pgmap v17733: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:19:28.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:19:28 smithi083 ceph-mon[77242]: pgmap v17733: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:19:30.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:19:30 smithi153 ceph-mon[70044]: pgmap v17734: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:19:30.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:19:30 smithi083 ceph-mon[79930]: pgmap v17734: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:19:30.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:19:30 smithi083 ceph-mon[77242]: pgmap v17734: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:19:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:19:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:19:31] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T15:19:32.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:19:32 smithi083 ceph-mon[79930]: pgmap v17735: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:19:32.717 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:19:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:19:32.381Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:19:32.719 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:19:32 smithi083 ceph-mon[77242]: pgmap v17735: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:19:32.813 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:19:32 smithi153 ceph-mon[70044]: pgmap v17735: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:19:32.813 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:19:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:19:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:19:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:19:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:19:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:19:34.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:19:34 smithi153 ceph-mon[70044]: pgmap v17736: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:19:34.969 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:19:34 smithi083 ceph-mon[79930]: pgmap v17736: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:19:34.970 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:19:34 smithi083 ceph-mon[77242]: pgmap v17736: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:19:36.210 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:19:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:19:35.738Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:19:36.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:19:36 smithi153 ceph-mon[70044]: pgmap v17737: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:19:36.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:19:36 smithi083 ceph-mon[79930]: pgmap v17737: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:19:36.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:19:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:19:36.479Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:19:36.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:19:36 smithi083 ceph-mon[77242]: pgmap v17737: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:19:38.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:19:38 smithi153 ceph-mon[70044]: pgmap v17738: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:19:38.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:19:38 smithi083 ceph-mon[79930]: pgmap v17738: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:19:38.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:19:38 smithi083 ceph-mon[77242]: pgmap v17738: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:19:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:19:40 smithi153 ceph-mon[70044]: pgmap v17739: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:19:40.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:19:40 smithi083 ceph-mon[79930]: pgmap v17739: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:19:40.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:19:40 smithi083 ceph-mon[77242]: pgmap v17739: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:19:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:19:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:19:41] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T15:19:42.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:19:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:19:42.382Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:19:42.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:19:42 smithi083 ceph-mon[79930]: pgmap v17740: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:19:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:19:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:19:42.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:19:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:19:42.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:19:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:19:42.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:19:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:19:42.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:19:42 smithi083 ceph-mon[77242]: pgmap v17740: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:19:42.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:19:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:19:42.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:19:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:19:42.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:19:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:19:42.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:19:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:19:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:19:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:19:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:19:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:19:42 smithi153 ceph-mon[70044]: pgmap v17740: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:19:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:19:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:19:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:19:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:19:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:19:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:19:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:19:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:19:43.311 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:19:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:19:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:19:44.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:19:44 smithi153 ceph-mon[70044]: pgmap v17741: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:19:44.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:19:44 smithi083 ceph-mon[77242]: pgmap v17741: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:19:44.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:19:44 smithi083 ceph-mon[79930]: pgmap v17741: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:19:46.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:19:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:19:45.739Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:19:46.816 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:19:46 smithi153 ceph-mon[70044]: pgmap v17742: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:19:46.975 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:19:46 smithi083 ceph-mon[79930]: pgmap v17742: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:19:46.976 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:19:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:19:46.478Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:19:46.976 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:19:46 smithi083 ceph-mon[77242]: pgmap v17742: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:19:48.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:19:48 smithi153 ceph-mon[70044]: pgmap v17743: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:19:48.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:19:48 smithi083 ceph-mon[79930]: pgmap v17743: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:19:48.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:19:48 smithi083 ceph-mon[77242]: pgmap v17743: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:19:50.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:19:50 smithi153 ceph-mon[70044]: pgmap v17744: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:19:50.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:19:50 smithi083 ceph-mon[79930]: pgmap v17744: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:19:50.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:19:50 smithi083 ceph-mon[77242]: pgmap v17744: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:19:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:19:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:19:51] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T15:19:52.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:19:52 smithi083 ceph-mon[79930]: pgmap v17745: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:19:52.718 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:19:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:19:52.382Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:19:52.718 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:19:52 smithi083 ceph-mon[77242]: pgmap v17745: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:19:52.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:19:52 smithi153 ceph-mon[70044]: pgmap v17745: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:19:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:19:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:19:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:19:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:19:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:19:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:19:53.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:19:53 smithi083 ceph-mon[79930]: pgmap v17746: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:19:53.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:19:53 smithi083 ceph-mon[77242]: pgmap v17746: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:19:54.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:19:53 smithi153 ceph-mon[70044]: pgmap v17746: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:19:55.969 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:19:55 smithi083 ceph-mon[79930]: pgmap v17747: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:19:55.970 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:19:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:19:55.738Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:19:55.971 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:19:55 smithi083 ceph-mon[77242]: pgmap v17747: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:19:56.063 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:19:55 smithi153 ceph-mon[70044]: pgmap v17747: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:19:56.474 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:19:56.766 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:19:56.766 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 41s ago 9h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:19:56.767 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 8m ago 9h 64.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:19:56.767 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 41s ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:19:56.768 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 8m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:19:56.768 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 8m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:19:56.769 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 41s ago 9h 997M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:19:56.769 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 41s ago 9h 2159M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:19:56.770 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 8m ago 9h 926M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:19:56.770 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 41s ago 9h 1295M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:19:56.771 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 41s ago 9h 24.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:19:56.771 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 8m ago 9h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:19:56.772 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 41s ago 9h 1568M 2162M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:19:56.772 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 41s ago 9h 1555M 2162M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:19:56.773 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 41s ago 9h 1208M 2162M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:19:56.773 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 41s ago 9h 1524M 2162M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:19:56.774 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 8m ago 9h 1313M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:19:56.774 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 8m ago 9h 1649M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:19:56.775 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 8m ago 9h 1474M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:19:56.775 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 8m ago 9h 1754M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:19:56.776 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 8m ago 9h 174M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:19:56.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:19:56 smithi083 ceph-mon[79930]: from='client.64699 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:19:56.968 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:19:56 smithi083 ceph-mon[77242]: from='client.64699 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:19:56.969 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:19:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:19:56.479Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:19:57.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:19:56 smithi153 ceph-mon[70044]: from='client.64699 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:19:57.137 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:19:57.137 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:19:57.137 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:19:57.138 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:19:57.138 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:19:57.138 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:19:57.139 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:19:57.139 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:19:57.139 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:19:57.139 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:19:57.140 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:19:57.140 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:19:57.140 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:19:57.141 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:19:57.141 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:19:57.141 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:19:57.149 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:19:57.149 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:19:57.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:19:57 smithi083 ceph-mon[79930]: pgmap v17748: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:19:57.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:19:57 smithi083 ceph-mon[79930]: from='client.64705 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:19:57.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:19:57 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1931485850' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:19:57.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:19:57 smithi083 ceph-mon[77242]: pgmap v17748: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:19:57.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:19:57 smithi083 ceph-mon[77242]: from='client.64705 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:19:57.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:19:57 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1931485850' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:19:58.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:19:57 smithi153 ceph-mon[70044]: pgmap v17748: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:19:58.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:19:57 smithi153 ceph-mon[70044]: from='client.64705 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:19:58.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:19:57 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1931485850' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:19:59.969 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:19:59 smithi083 ceph-mon[79930]: pgmap v17749: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:19:59.970 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:19:59 smithi083 ceph-mon[77242]: pgmap v17749: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:20:00.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:19:59 smithi153 ceph-mon[70044]: pgmap v17749: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:20:00.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:20:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[77219]: 2022-09-29T15:19:59.999+0000 7f160f665700 -1 log_channel(cluster) log [ERR] : overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T15:20:00.971 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:20:00 smithi083 ceph-mon[77242]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T15:20:00.971 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:20:00 smithi083 ceph-mon[79930]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T15:20:01.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:20:00 smithi153 ceph-mon[70044]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T15:20:01.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:20:01 smithi083 ceph-mon[77242]: pgmap v17750: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:20:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:20:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:20:01] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T15:20:01.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:20:01 smithi083 ceph-mon[79930]: pgmap v17750: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:20:02.066 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:20:01 smithi153 ceph-mon[70044]: pgmap v17750: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:20:02.718 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:20:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:20:02.383Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:20:02.842 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:20:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:20:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:20:03.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:20:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:20:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:20:04.820 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:20:04 smithi153 ceph-mon[70044]: pgmap v17751: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:20:04.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:20:04 smithi083 ceph-mon[79930]: pgmap v17751: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:20:04.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:20:04 smithi083 ceph-mon[77242]: pgmap v17751: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:20:06.218 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:20:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:20:05.739Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:20:06.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:20:06 smithi153 ceph-mon[70044]: pgmap v17752: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:20:06.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:20:06 smithi083 ceph-mon[77242]: pgmap v17752: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:20:06.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:20:06 smithi083 ceph-mon[79930]: pgmap v17752: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:20:06.967 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:20:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:20:06.480Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:20:08.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:20:08 smithi153 ceph-mon[70044]: pgmap v17753: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:20:08.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:20:08 smithi083 ceph-mon[79930]: pgmap v17753: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:20:08.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:20:08 smithi083 ceph-mon[77242]: pgmap v17753: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:20:10.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:20:10 smithi153 ceph-mon[70044]: pgmap v17754: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:20:10.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:20:10 smithi083 ceph-mon[77242]: pgmap v17754: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:20:10.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:20:10 smithi083 ceph-mon[79930]: pgmap v17754: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:20:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:20:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:20:11] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T15:20:12.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:20:12 smithi083 ceph-mon[79930]: pgmap v17755: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:20:12.714 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:20:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:20:12.384Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:20:12.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:20:12 smithi083 ceph-mon[77242]: pgmap v17755: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:20:12.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:20:12 smithi153 ceph-mon[70044]: pgmap v17755: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:20:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:20:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:20:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:20:13.311 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:20:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:20:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:20:14.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:20:14 smithi153 ceph-mon[70044]: pgmap v17756: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:20:14.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:20:14 smithi083 ceph-mon[77242]: pgmap v17756: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:20:14.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:20:14 smithi083 ceph-mon[79930]: pgmap v17756: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:20:16.210 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:20:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:20:15.740Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:20:16.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:20:16 smithi153 ceph-mon[70044]: pgmap v17757: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:20:16.984 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:20:16 smithi083 ceph-mon[79930]: pgmap v17757: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:20:16.985 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:20:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:20:16.480Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:20:16.985 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:20:16 smithi083 ceph-mon[77242]: pgmap v17757: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:20:18.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:20:18 smithi153 ceph-mon[70044]: pgmap v17758: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:20:18.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:20:18 smithi083 ceph-mon[79930]: pgmap v17758: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:20:18.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:20:18 smithi083 ceph-mon[77242]: pgmap v17758: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:20:19.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:20:19 smithi153 ceph-mon[70044]: pgmap v17759: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:20:19.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:20:19 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:20:19.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:20:19 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:20:19.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:20:19 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:20:19.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:20:19 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:20:19.969 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:20:19 smithi083 ceph-mon[77242]: pgmap v17759: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:20:19.969 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:20:19 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:20:19.970 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:20:19 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:20:19.970 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:20:19 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:20:19.971 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:20:19 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:20:19.971 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:20:19 smithi083 ceph-mon[79930]: pgmap v17759: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:20:19.972 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:20:19 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:20:19.972 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:20:19 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:20:19.973 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:20:19 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:20:19.973 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:20:19 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:20:21.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:20:21 smithi083 ceph-mon[77242]: pgmap v17760: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:20:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:20:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:20:21] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T15:20:21.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:20:21 smithi083 ceph-mon[79930]: pgmap v17760: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:20:22.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:20:21 smithi153 ceph-mon[70044]: pgmap v17760: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:20:22.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:20:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:20:22.385Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:20:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:20:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:20:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:20:23.310 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:20:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:20:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:20:24.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:20:24 smithi153 ceph-mon[70044]: pgmap v17761: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:20:24.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:20:24 smithi083 ceph-mon[77242]: pgmap v17761: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:20:24.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:20:24 smithi083 ceph-mon[79930]: pgmap v17761: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:20:26.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:20:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:20:25.741Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:20:26.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:20:26 smithi153 ceph-mon[70044]: pgmap v17762: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:20:26.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:20:26 smithi083 ceph-mon[79930]: pgmap v17762: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:20:26.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:20:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:20:26.481Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:20:26.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:20:26 smithi083 ceph-mon[77242]: pgmap v17762: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:20:27.453 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:20:27.740 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:20:27.741 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 72s ago 9h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:20:27.741 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 9m ago 9h 64.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:20:27.742 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 72s ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:20:27.743 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 9m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:20:27.743 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 9m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:20:27.743 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 72s ago 9h 997M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:20:27.744 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 72s ago 9h 2159M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:20:27.744 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 9m ago 9h 926M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:20:27.745 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 72s ago 9h 1295M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:20:27.745 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 72s ago 9h 24.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:20:27.747 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 9m ago 9h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:20:27.747 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 72s ago 9h 1568M 2162M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:20:27.747 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 72s ago 9h 1555M 2162M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:20:27.748 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 72s ago 9h 1208M 2162M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:20:27.751 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 72s ago 9h 1524M 2162M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:20:27.752 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 9m ago 9h 1313M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:20:27.752 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 9m ago 9h 1649M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:20:27.752 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 9m ago 9h 1474M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:20:27.753 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 9m ago 9h 1754M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:20:27.753 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 9m ago 9h 174M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:20:28.097 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:20:28.097 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:20:28.098 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:20:28.098 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:20:28.099 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:20:28.099 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:20:28.100 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:20:28.100 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:20:28.101 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:20:28.101 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:20:28.102 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:20:28.102 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:20:28.103 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:20:28.103 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:20:28.103 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:20:28.104 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:20:28.106 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:20:28.106 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:20:28.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:20:28 smithi153 ceph-mon[70044]: pgmap v17763: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:20:28.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:20:28 smithi153 ceph-mon[70044]: from='client.54786 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:20:28.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:20:28 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2765169129' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:20:28.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:20:28 smithi083 ceph-mon[79930]: pgmap v17763: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:20:28.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:20:28 smithi083 ceph-mon[79930]: from='client.54786 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:20:28.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:20:28 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2765169129' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:20:28.970 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:20:28 smithi083 ceph-mon[77242]: pgmap v17763: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:20:28.971 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:20:28 smithi083 ceph-mon[77242]: from='client.54786 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:20:28.972 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:20:28 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2765169129' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:20:29.817 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:20:29 smithi153 ceph-mon[70044]: from='client.54792 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:20:29.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:20:29 smithi083 ceph-mon[79930]: from='client.54792 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:20:29.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:20:29 smithi083 ceph-mon[77242]: from='client.54792 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:20:30.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:20:30 smithi153 ceph-mon[70044]: pgmap v17764: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:20:30.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:20:30 smithi083 ceph-mon[77242]: pgmap v17764: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:20:30.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:20:30 smithi083 ceph-mon[79930]: pgmap v17764: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:20:31.968 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:20:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:20:31] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T15:20:32.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:20:32 smithi083 ceph-mon[79930]: pgmap v17765: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:20:32.718 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:20:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:20:32.386Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:20:32.719 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:20:32 smithi083 ceph-mon[77242]: pgmap v17765: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:20:32.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:20:32 smithi153 ceph-mon[70044]: pgmap v17765: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:20:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:20:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:20:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:20:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:20:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:20:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:20:34.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:20:34 smithi153 ceph-mon[70044]: pgmap v17766: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:20:34.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:20:34 smithi083 ceph-mon[79930]: pgmap v17766: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:20:34.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:20:34 smithi083 ceph-mon[77242]: pgmap v17766: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:20:35.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:20:35 smithi083 ceph-mon[79930]: pgmap v17767: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:20:35.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:20:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:20:35.742Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:20:35.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:20:35 smithi083 ceph-mon[77242]: pgmap v17767: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:20:36.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:20:35 smithi153 ceph-mon[70044]: pgmap v17767: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:20:36.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:20:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:20:36.483Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:20:38.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:20:38 smithi153 ceph-mon[70044]: pgmap v17768: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:20:38.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:20:38 smithi083 ceph-mon[79930]: pgmap v17768: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:20:38.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:20:38 smithi083 ceph-mon[77242]: pgmap v17768: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:20:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:20:40 smithi153 ceph-mon[70044]: pgmap v17769: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:20:40.968 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:20:40 smithi083 ceph-mon[77242]: pgmap v17769: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:20:40.970 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:20:40 smithi083 ceph-mon[79930]: pgmap v17769: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:20:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:20:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:20:41] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T15:20:42.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:20:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:20:42.387Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:20:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:20:42 smithi083 ceph-mon[77242]: pgmap v17770: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:20:42.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:20:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:20:42.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:20:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:20:42.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:20:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:20:42.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:20:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:20:42.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:20:42 smithi083 ceph-mon[79930]: pgmap v17770: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:20:42.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:20:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:20:42.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:20:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:20:42.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:20:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:20:42.718 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:20:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:20:42.811 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:20:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:20:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:20:42.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:20:42 smithi153 ceph-mon[70044]: pgmap v17770: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:20:42.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:20:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:20:42.814 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:20:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:20:42.814 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:20:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:20:42.815 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:20:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:20:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:20:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:20:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:20:44.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:20:44 smithi153 ceph-mon[70044]: pgmap v17771: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:20:44.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:20:44 smithi083 ceph-mon[79930]: pgmap v17771: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:20:44.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:20:44 smithi083 ceph-mon[77242]: pgmap v17771: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:20:46.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:20:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:20:45.743Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:20:46.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:20:46 smithi153 ceph-mon[70044]: pgmap v17772: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:20:46.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:20:46 smithi083 ceph-mon[79930]: pgmap v17772: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:20:46.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:20:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:20:46.483Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:20:46.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:20:46 smithi083 ceph-mon[77242]: pgmap v17772: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:20:48.818 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:20:48 smithi153 ceph-mon[70044]: pgmap v17773: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:20:48.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:20:48 smithi083 ceph-mon[79930]: pgmap v17773: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:20:48.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:20:48 smithi083 ceph-mon[77242]: pgmap v17773: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:20:50.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:20:50 smithi153 ceph-mon[70044]: pgmap v17774: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:20:50.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:20:50 smithi083 ceph-mon[77242]: pgmap v17774: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:20:50.968 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:20:50 smithi083 ceph-mon[79930]: pgmap v17774: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:20:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:20:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:20:51] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T15:20:52.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:20:52 smithi083 ceph-mon[79930]: pgmap v17775: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:20:52.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:20:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:20:52.388Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:20:52.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:20:52 smithi083 ceph-mon[77242]: pgmap v17775: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:20:52.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:20:52 smithi153 ceph-mon[70044]: pgmap v17775: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:20:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:20:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:20:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:20:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:20:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:20:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:20:53.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:20:53 smithi083 ceph-mon[77242]: pgmap v17776: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:20:53.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:20:53 smithi083 ceph-mon[79930]: pgmap v17776: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:20:54.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:20:53 smithi153 ceph-mon[70044]: pgmap v17776: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:20:55.975 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:20:55 smithi083 ceph-mon[79930]: pgmap v17777: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:20:55.976 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:20:55 smithi083 ceph-mon[77242]: pgmap v17777: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:20:55.977 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:20:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:20:55.744Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:20:56.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:20:55 smithi153 ceph-mon[70044]: pgmap v17777: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:20:56.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:20:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:20:56.483Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:20:58.423 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:20:58.720 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:20:58.721 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 103s ago 9h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:20:58.721 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 9m ago 9h 64.3M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:20:58.722 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 103s ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:20:58.722 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 9m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:20:58.722 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 9m ago 9h 449M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:20:58.723 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 103s ago 10h 997M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:20:58.723 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 103s ago 10h 2159M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:20:58.726 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 9m ago 9h 926M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:20:58.727 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 103s ago 9h 1295M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:20:58.727 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 103s ago 9h 24.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:20:58.727 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 9m ago 9h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:20:58.728 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 103s ago 9h 1568M 2162M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:20:58.728 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 103s ago 9h 1555M 2162M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:20:58.728 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 103s ago 9h 1208M 2162M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:20:58.728 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 103s ago 9h 1524M 2162M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:20:58.729 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 9m ago 9h 1313M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:20:58.729 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 9m ago 9h 1649M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:20:58.729 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 9m ago 9h 1474M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:20:58.729 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 9m ago 9h 1754M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:20:58.730 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 9m ago 9h 174M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:20:58.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:20:58 smithi153 ceph-mon[70044]: pgmap v17778: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:20:58.970 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:20:58 smithi083 ceph-mon[79930]: pgmap v17778: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:20:58.970 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:20:58 smithi083 ceph-mon[77242]: pgmap v17778: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:20:59.066 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:20:59.067 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:20:59.067 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:20:59.067 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:20:59.068 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:20:59.068 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:20:59.068 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:20:59.068 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:20:59.069 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:20:59.070 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:20:59.070 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:20:59.071 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:20:59.071 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:20:59.072 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:20:59.072 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:20:59.073 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:20:59.073 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:20:59.075 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:20:59.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:20:59 smithi153 ceph-mon[70044]: from='client.54804 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:20:59.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:20:59 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2271305483' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:20:59.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:20:59 smithi083 ceph-mon[79930]: from='client.54804 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:20:59.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:20:59 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2271305483' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:20:59.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:20:59 smithi083 ceph-mon[77242]: from='client.54804 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:20:59.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:20:59 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2271305483' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:21:00.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:21:00 smithi153 ceph-mon[70044]: pgmap v17779: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:00.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:21:00 smithi153 ceph-mon[70044]: from='client.54810 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:21:00.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:21:00 smithi083 ceph-mon[79930]: pgmap v17779: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:00.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:21:00 smithi083 ceph-mon[79930]: from='client.54810 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:21:00.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:21:00 smithi083 ceph-mon[77242]: pgmap v17779: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:00.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:21:00 smithi083 ceph-mon[77242]: from='client.54810 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:21:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:21:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:21:01] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T15:21:02.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:21:02 smithi083 ceph-mon[79930]: pgmap v17780: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:02.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:21:02 smithi083 ceph-mon[77242]: pgmap v17780: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:02.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:21:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:21:02.389Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:21:02.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:21:02 smithi153 ceph-mon[70044]: pgmap v17780: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:21:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:21:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:21:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:21:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:21:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:21:04.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:21:04 smithi153 ceph-mon[70044]: pgmap v17781: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:04.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:21:04 smithi083 ceph-mon[79930]: pgmap v17781: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:04.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:21:04 smithi083 ceph-mon[77242]: pgmap v17781: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:06.210 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:21:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:21:05.745Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:21:06.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:21:06 smithi153 ceph-mon[70044]: pgmap v17782: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:06.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:21:06 smithi083 ceph-mon[79930]: pgmap v17782: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:06.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:21:06 smithi083 ceph-mon[77242]: pgmap v17782: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:06.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:21:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:21:06.484Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:21:08.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:21:08 smithi153 ceph-mon[70044]: pgmap v17783: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:08.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:21:08 smithi083 ceph-mon[79930]: pgmap v17783: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:08.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:21:08 smithi083 ceph-mon[77242]: pgmap v17783: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:09.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:21:09 smithi083 ceph-mon[77242]: pgmap v17784: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:09.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:21:09 smithi083 ceph-mon[79930]: pgmap v17784: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:10.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:21:09 smithi153 ceph-mon[70044]: pgmap v17784: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:11.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:21:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:21:11] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T15:21:11.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:21:11 smithi083 ceph-mon[79930]: pgmap v17785: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:11.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:21:11 smithi083 ceph-mon[77242]: pgmap v17785: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:12.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:21:11 smithi153 ceph-mon[70044]: pgmap v17785: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:12.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:21:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:21:12.388Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:21:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:21:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:21:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:21:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:21:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:21:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:21:14.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:21:14 smithi153 ceph-mon[70044]: pgmap v17786: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:14.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:21:14 smithi083 ceph-mon[77242]: pgmap v17786: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:14.968 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:21:14 smithi083 ceph-mon[79930]: pgmap v17786: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:16.214 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:21:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:21:15.745Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:21:16.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:21:16 smithi153 ceph-mon[70044]: pgmap v17787: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:16.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:21:16 smithi083 ceph-mon[79930]: pgmap v17787: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:16.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:21:16 smithi083 ceph-mon[77242]: pgmap v17787: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:16.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:21:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:21:16.484Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:21:18.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:21:18 smithi153 ceph-mon[70044]: pgmap v17788: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:18.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:21:18 smithi083 ceph-mon[79930]: pgmap v17788: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:18.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:21:18 smithi083 ceph-mon[77242]: pgmap v17788: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:19.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:21:19 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:21:19.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:21:19 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:21:19.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:21:19 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:21:19.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:21:19 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:21:19.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:21:19 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:21:19.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:21:19 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:21:20.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:21:19 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:21:20.114 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:21:19 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:21:20.115 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:21:19 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:21:20.985 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:21:20 smithi083 ceph-mon[79930]: pgmap v17789: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:20.986 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:21:20 smithi083 ceph-mon[77242]: pgmap v17789: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:21.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:21:20 smithi153 ceph-mon[70044]: pgmap v17789: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:21.966 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:21:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:21:21] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T15:21:21.969 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:21:21 smithi083 ceph-mon[79930]: pgmap v17790: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:21.969 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:21:21 smithi083 ceph-mon[77242]: pgmap v17790: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:22.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:21:21 smithi153 ceph-mon[70044]: pgmap v17790: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:22.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:21:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:21:22.389Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:21:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:21:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:21:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:21:23.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:21:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:21:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:21:24.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:21:24 smithi153 ceph-mon[70044]: pgmap v17791: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:24.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:21:24 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:21:24.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:21:24 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:21:24.960 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:21:24 smithi083 ceph-mon[77242]: pgmap v17791: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:24.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:21:24 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:21:24.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:21:24 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:21:24.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:21:24 smithi083 ceph-mon[79930]: pgmap v17791: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:24.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:21:24 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:21:24.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:21:24 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:21:26.039 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:21:25 smithi153 ceph-mon[70044]: pgmap v17792: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:26.039 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:21:25 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:21:26.039 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:21:25 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:21:26.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:21:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:21:25.745Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:21:26.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:21:25 smithi083 ceph-mon[77242]: pgmap v17792: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:26.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:21:25 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:21:26.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:21:25 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:21:26.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:21:25 smithi083 ceph-mon[79930]: pgmap v17792: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:26.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:21:25 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:21:26.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:21:25 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:21:26.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:21:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:21:26.485Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:21:28.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:21:28 smithi083 ceph-mon[79930]: pgmap v17793: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:28.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:21:28 smithi083 ceph-mon[79930]: Detected new or changed devices on smithi153 2022-09-29T15:21:28.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:21:28 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:21:28.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:21:28 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:21:28.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:21:28 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:21:28.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:21:28 smithi083 ceph-mon[77242]: pgmap v17793: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:28.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:21:28 smithi083 ceph-mon[77242]: Detected new or changed devices on smithi153 2022-09-29T15:21:28.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:21:28 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:21:28.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:21:28 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:21:28.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:21:28 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:21:28.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:21:28 smithi153 ceph-mon[70044]: pgmap v17793: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:28.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:21:28 smithi153 ceph-mon[70044]: Detected new or changed devices on smithi153 2022-09-29T15:21:28.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:21:28 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:21:28.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:21:28 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:21:28.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:21:28 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:21:29.393 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:21:29.697 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:21:29.698 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 2m ago 9h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:21:29.699 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 5s ago 9h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:21:29.699 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 2m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:21:29.700 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 5s ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:21:29.700 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 5s ago 9h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:21:29.701 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 2m ago 10h 997M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:21:29.702 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 2m ago 10h 2159M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:21:29.702 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 5s ago 9h 942M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:21:29.703 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 2m ago 9h 1295M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:21:29.703 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 2m ago 9h 24.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:21:29.703 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 5s ago 9h 27.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:21:29.703 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 2m ago 9h 1568M 2162M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:21:29.704 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 2m ago 9h 1555M 2162M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:21:29.704 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 2m ago 9h 1208M 2162M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:21:29.705 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 2m ago 9h 1524M 2162M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:21:29.706 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 5s ago 9h 1336M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:21:29.707 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 5s ago 9h 1678M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:21:29.707 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 5s ago 9h 1500M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:21:29.708 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 5s ago 9h 1784M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:21:29.708 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 5s ago 9h 175M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:21:30.037 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:21:30.037 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:21:30.038 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:21:30.038 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:21:30.039 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:21:30.039 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:21:30.040 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:21:30.040 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:21:30.041 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:21:30.041 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:21:30.041 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:21:30.042 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:21:30.042 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:21:30.043 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:21:30.043 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:21:30.044 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:21:30.044 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:21:30.045 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:21:30.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:21:30 smithi083 ceph-mon[79930]: pgmap v17794: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:30.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:21:30 smithi083 ceph-mon[79930]: from='client.54822 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:21:30.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:21:30 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2074190405' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:21:30.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:21:30 smithi083 ceph-mon[77242]: pgmap v17794: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:30.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:21:30 smithi083 ceph-mon[77242]: from='client.54822 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:21:30.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:21:30 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2074190405' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:21:30.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:21:30 smithi153 ceph-mon[70044]: pgmap v17794: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:30.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:21:30 smithi153 ceph-mon[70044]: from='client.54822 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:21:30.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:21:30 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2074190405' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:21:31.717 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:21:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:21:31] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T15:21:31.718 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:21:31 smithi083 ceph-mon[79930]: from='client.54828 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:21:31.719 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:21:31 smithi083 ceph-mon[77242]: from='client.54828 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:21:31.818 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:21:31 smithi153 ceph-mon[70044]: from='client.54828 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:21:32.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:21:32 smithi083 ceph-mon[79930]: pgmap v17795: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:32.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:21:32 smithi083 ceph-mon[77242]: pgmap v17795: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:32.714 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:21:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:21:32.390Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:21:32.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:21:32 smithi153 ceph-mon[70044]: pgmap v17795: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:32.817 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:21:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:21:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:21:33.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:21:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:21:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:21:34.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:21:34 smithi153 ceph-mon[70044]: pgmap v17796: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:34.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:21:34 smithi083 ceph-mon[79930]: pgmap v17796: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:34.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:21:34 smithi083 ceph-mon[77242]: pgmap v17796: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:36.210 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:21:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:21:35.746Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:21:36.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:21:36 smithi153 ceph-mon[70044]: pgmap v17797: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:36.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:21:36 smithi083 ceph-mon[77242]: pgmap v17797: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:36.969 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:21:36 smithi083 ceph-mon[79930]: pgmap v17797: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:36.969 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:21:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:21:36.484Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:21:38.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:21:38 smithi153 ceph-mon[70044]: pgmap v17798: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:38.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:21:38 smithi083 ceph-mon[77242]: pgmap v17798: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:38.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:21:38 smithi083 ceph-mon[79930]: pgmap v17798: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:40.970 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:21:40 smithi083 ceph-mon[79930]: pgmap v17799: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:40.971 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:21:40 smithi083 ceph-mon[77242]: pgmap v17799: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:41.073 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:21:40 smithi153 ceph-mon[70044]: pgmap v17799: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:41.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:21:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:21:41] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T15:21:42.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:21:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:21:42.390Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:21:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:21:42 smithi083 ceph-mon[79930]: pgmap v17800: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:42.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:21:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:21:42.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:21:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:21:42.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:21:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:21:42.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:21:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:21:42.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:21:42 smithi083 ceph-mon[77242]: pgmap v17800: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:42.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:21:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:21:42.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:21:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:21:42.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:21:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:21:42.718 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:21:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:21:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:21:42 smithi153 ceph-mon[70044]: pgmap v17800: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:21:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:21:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:21:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:21:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:21:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:21:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:21:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:21:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:21:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:21:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:21:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:21:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:21:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:21:44.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:21:44 smithi083 ceph-mon[79930]: pgmap v17801: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:44.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:21:44 smithi083 ceph-mon[77242]: pgmap v17801: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:45.063 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:21:44 smithi153 ceph-mon[70044]: pgmap v17801: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:45.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:21:45 smithi083 ceph-mon[79930]: pgmap v17802: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:45.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:21:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:21:45.746Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:21:45.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:21:45 smithi083 ceph-mon[77242]: pgmap v17802: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:46.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:21:45 smithi153 ceph-mon[70044]: pgmap v17802: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:46.966 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:21:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:21:46.485Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:21:48.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:21:48 smithi153 ceph-mon[70044]: pgmap v17803: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:48.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:21:48 smithi083 ceph-mon[79930]: pgmap v17803: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:48.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:21:48 smithi083 ceph-mon[77242]: pgmap v17803: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:50.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:21:50 smithi153 ceph-mon[70044]: pgmap v17804: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:50.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:21:50 smithi083 ceph-mon[77242]: pgmap v17804: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:50.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:21:50 smithi083 ceph-mon[79930]: pgmap v17804: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:51.987 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:21:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:21:51] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T15:21:52.710 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:21:52 smithi083 ceph-mon[77242]: pgmap v17805: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:52.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:21:52 smithi083 ceph-mon[79930]: pgmap v17805: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:52.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:21:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:21:52.390Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:21:52.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:21:52 smithi153 ceph-mon[70044]: pgmap v17805: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:21:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:21:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:21:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:21:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:21:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:21:54.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:21:54 smithi083 ceph-mon[79930]: pgmap v17806: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:54.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:21:54 smithi083 ceph-mon[77242]: pgmap v17806: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:55.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:21:54 smithi153 ceph-mon[70044]: pgmap v17806: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:56.214 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:21:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:21:55.747Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:21:56.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:21:56 smithi083 ceph-mon[79930]: pgmap v17807: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:56.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:21:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:21:56.486Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:21:56.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:21:56 smithi083 ceph-mon[77242]: pgmap v17807: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:57.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:21:56 smithi153 ceph-mon[70044]: pgmap v17807: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:58.960 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:21:58 smithi083 ceph-mon[77242]: pgmap v17808: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:58.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:21:58 smithi083 ceph-mon[79930]: pgmap v17808: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:59.069 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:21:58 smithi153 ceph-mon[70044]: pgmap v17808: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:21:59.970 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:21:59 smithi083 ceph-mon[79930]: pgmap v17809: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:00.042 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:21:59 smithi083 ceph-mon[77242]: pgmap v17809: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:00.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:21:59 smithi153 ceph-mon[70044]: pgmap v17809: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:00.365 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:22:00.652 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:22:00.653 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 2m ago 9h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:22:00.653 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 36s ago 9h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:22:00.653 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 2m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:22:00.654 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 36s ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:22:00.654 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 36s ago 9h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:22:00.654 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 2m ago 10h 997M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:22:00.655 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 2m ago 10h 2159M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:22:00.655 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 36s ago 9h 942M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:22:00.655 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 2m ago 9h 1295M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:22:00.656 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 2m ago 9h 24.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:22:00.656 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 36s ago 9h 27.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:22:00.657 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 2m ago 9h 1568M 2162M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:22:00.657 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 2m ago 9h 1555M 2162M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:22:00.657 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 2m ago 9h 1208M 2162M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:22:00.657 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 2m ago 9h 1524M 2162M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:22:00.659 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 36s ago 9h 1336M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:22:00.660 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 36s ago 9h 1678M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:22:00.660 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 36s ago 9h 1500M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:22:00.661 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 36s ago 9h 1784M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:22:00.661 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 36s ago 9h 175M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:22:00.986 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:22:00 smithi083 ceph-mon[79930]: from='client.54840 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:22:00.986 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:22:00 smithi083 ceph-mon[77242]: from='client.54840 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:22:01.008 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:22:01.008 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:22:01.008 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:22:01.009 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:22:01.009 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:22:01.009 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:22:01.010 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:22:01.010 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:22:01.010 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:22:01.011 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:22:01.011 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:22:01.011 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:22:01.011 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:22:01.012 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:22:01.012 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:22:01.012 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:22:01.012 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:22:01.013 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:22:01.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:22:00 smithi153 ceph-mon[70044]: from='client.54840 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:22:01.964 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:22:01] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T15:22:01.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:22:01 smithi083 ceph-mon[77242]: pgmap v17810: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:01.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:22:01 smithi083 ceph-mon[77242]: from='client.64777 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:22:01.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:22:01 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4228841092' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:22:01.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:22:01 smithi083 ceph-mon[79930]: pgmap v17810: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:01.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:22:01 smithi083 ceph-mon[79930]: from='client.64777 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:22:01.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:22:01 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4228841092' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:22:02.063 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:22:01 smithi153 ceph-mon[70044]: pgmap v17810: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:02.064 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:22:01 smithi153 ceph-mon[70044]: from='client.64777 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:22:02.064 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:22:01 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4228841092' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:22:02.719 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:22:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:22:02.391Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:22:02.814 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:22:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:22:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:22:03.310 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:22:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:22:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:22:04.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:22:04 smithi153 ceph-mon[70044]: pgmap v17811: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:04.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:22:04 smithi083 ceph-mon[79930]: pgmap v17811: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:04.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:22:04 smithi083 ceph-mon[77242]: pgmap v17811: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:06.217 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:22:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:22:05.748Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:22:06.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:22:06 smithi083 ceph-mon[79930]: pgmap v17812: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:06.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:22:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:22:06.487Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:22:06.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:22:06 smithi083 ceph-mon[77242]: pgmap v17812: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:07.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:22:06 smithi153 ceph-mon[70044]: pgmap v17812: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:08.979 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:22:08 smithi083 ceph-mon[79930]: pgmap v17813: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:08.979 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:22:08 smithi083 ceph-mon[77242]: pgmap v17813: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:09.066 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:22:08 smithi153 ceph-mon[70044]: pgmap v17813: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:10.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:22:10 smithi083 ceph-mon[79930]: pgmap v17814: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:10.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:22:10 smithi083 ceph-mon[77242]: pgmap v17814: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:11.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:22:10 smithi153 ceph-mon[70044]: pgmap v17814: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:22:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:22:11] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T15:22:12.574 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:22:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:22:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:22:12.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:22:12 smithi083 ceph-mon[79930]: pgmap v17815: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:12.715 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:22:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:22:12.391Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:22:12.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:22:12 smithi083 ceph-mon[77242]: pgmap v17815: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:13.022 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:22:12 smithi153 ceph-mon[70044]: pgmap v17815: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:13.312 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:22:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:22:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:22:13.969 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:22:13 smithi083 ceph-mon[77242]: pgmap v17816: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:13.970 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:22:13 smithi083 ceph-mon[79930]: pgmap v17816: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:14.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:22:13 smithi153 ceph-mon[70044]: pgmap v17816: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:15.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:22:15 smithi083 ceph-mon[79930]: pgmap v17817: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:15.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:22:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:22:15.749Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:22:15.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:22:15 smithi083 ceph-mon[77242]: pgmap v17817: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:16.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:22:15 smithi153 ceph-mon[70044]: pgmap v17817: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:16.972 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:22:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:22:16.486Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:22:18.981 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:22:18 smithi083 ceph-mon[79930]: pgmap v17818: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:18.995 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:22:18 smithi083 ceph-mon[77242]: pgmap v17818: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:19.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:22:18 smithi153 ceph-mon[70044]: pgmap v17818: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:20.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:22:20 smithi083 ceph-mon[79930]: pgmap v17819: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:20.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:22:20 smithi083 ceph-mon[77242]: pgmap v17819: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:21.078 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:22:20 smithi153 ceph-mon[70044]: pgmap v17819: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:21.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:22:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:22:21] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T15:22:22.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:22:22 smithi083 ceph-mon[77242]: pgmap v17820: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:22.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:22:22 smithi083 ceph-mon[79930]: pgmap v17820: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:22.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:22:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:22:22.393Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:22:22.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:22:22 smithi153 ceph-mon[70044]: pgmap v17820: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:22.811 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:22:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:22:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:22:23.310 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:22:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:22:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:22:24.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:22:24 smithi083 ceph-mon[77242]: pgmap v17821: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:24.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:22:24 smithi083 ceph-mon[79930]: pgmap v17821: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:25.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:22:24 smithi153 ceph-mon[70044]: pgmap v17821: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:25.981 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:22:25 smithi083 ceph-mon[77242]: pgmap v17822: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:25.982 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:22:25 smithi083 ceph-mon[79930]: pgmap v17822: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:25.983 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:22:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:22:25.749Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:22:26.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:22:25 smithi153 ceph-mon[70044]: pgmap v17822: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:26.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:22:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:22:26.488Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:22:27.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:22:27 smithi083 ceph-mon[79930]: pgmap v17823: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:27.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:22:27 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:22:27.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:22:27 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:22:27.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:22:27 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:22:27.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:22:27 smithi083 ceph-mon[77242]: pgmap v17823: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:27.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:22:27 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:22:27.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:22:27 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:22:27.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:22:27 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:22:28.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:22:27 smithi153 ceph-mon[70044]: pgmap v17823: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:28.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:22:27 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:22:28.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:22:27 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:22:28.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:22:27 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:22:29.328 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:22:29 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:22:29.475 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:22:29 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:22:29.506 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:22:29 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:22:30.313 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:22:30 smithi153 ceph-mon[70044]: pgmap v17824: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:30.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:22:30 smithi083 ceph-mon[77242]: pgmap v17824: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:30.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:22:30 smithi083 ceph-mon[79930]: pgmap v17824: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:31.340 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:22:31.631 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:22:31.631 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 3m ago 9h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:22:31.632 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 67s ago 9h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:22:31.639 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 3m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:22:31.639 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 67s ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:22:31.639 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 67s ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:22:31.640 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 3m ago 10h 997M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:22:31.640 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 3m ago 10h 2159M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:22:31.641 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 67s ago 10h 942M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:22:31.641 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 3m ago 10h 1295M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:22:31.642 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 3m ago 9h 24.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:22:31.642 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 67s ago 9h 27.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:22:31.643 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 3m ago 9h 1568M 2162M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:22:31.643 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 3m ago 9h 1555M 2162M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:22:31.643 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 3m ago 9h 1208M 2162M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:22:31.644 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 3m ago 9h 1524M 2162M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:22:31.645 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 67s ago 9h 1336M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:22:31.645 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 67s ago 9h 1678M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:22:31.646 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 67s ago 9h 1500M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:22:31.646 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 67s ago 9h 1784M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:22:31.647 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 67s ago 9h 175M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:22:31.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:22:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:22:31] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T15:22:31.990 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:22:31.991 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:22:31.991 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:22:31.991 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:22:31.992 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:22:31.992 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:22:31.992 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:22:31.992 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:22:31.993 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:22:31.993 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:22:31.993 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:22:31.993 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:22:31.994 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:22:31.994 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:22:31.994 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:22:31.994 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:22:31.995 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:22:31.995 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:22:32.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:22:32 smithi153 ceph-mon[70044]: pgmap v17825: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:32.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:22:32 smithi153 ceph-mon[70044]: from='client.54858 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:22:32.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:22:32 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2961986964' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:22:32.394 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:22:32 smithi083 ceph-mon[79930]: pgmap v17825: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:32.395 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:22:32 smithi083 ceph-mon[79930]: from='client.54858 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:22:32.395 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:22:32 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2961986964' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:22:32.396 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:22:32 smithi083 ceph-mon[77242]: pgmap v17825: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:32.396 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:22:32 smithi083 ceph-mon[77242]: from='client.54858 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:22:32.397 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:22:32 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2961986964' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:22:32.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:22:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:22:32.393Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:22:32.824 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:22:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:22:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:22:33.340 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:22:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:22:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:22:33.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:22:33 smithi083 ceph-mon[77242]: from='client.54864 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:22:33.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:22:33 smithi083 ceph-mon[79930]: from='client.54864 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:22:34.065 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:22:33 smithi153 ceph-mon[70044]: from='client.54864 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:22:34.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:22:34 smithi083 ceph-mon[79930]: pgmap v17826: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:34.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:22:34 smithi083 ceph-mon[77242]: pgmap v17826: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:35.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:22:34 smithi153 ceph-mon[70044]: pgmap v17826: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:36.218 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:22:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:22:35.750Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:22:36.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:22:36 smithi083 ceph-mon[79930]: pgmap v17827: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:36.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:22:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:22:36.488Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:22:36.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:22:36 smithi083 ceph-mon[77242]: pgmap v17827: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:37.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:22:36 smithi153 ceph-mon[70044]: pgmap v17827: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:38.975 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:22:38 smithi083 ceph-mon[79930]: pgmap v17828: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:38.975 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:22:38 smithi083 ceph-mon[77242]: pgmap v17828: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:39.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:22:38 smithi153 ceph-mon[70044]: pgmap v17828: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:40.968 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:22:40 smithi083 ceph-mon[79930]: pgmap v17829: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:40.969 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:22:40 smithi083 ceph-mon[77242]: pgmap v17829: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:22:40 smithi153 ceph-mon[70044]: pgmap v17829: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:22:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:22:41] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T15:22:42.567 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:22:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:22:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:22:42.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:22:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:22:42.395Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:22:42.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:22:42 smithi083 ceph-mon[77242]: pgmap v17830: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:42.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:22:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:22:42.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:22:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:22:42.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:22:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:22:42.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:22:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:22:42.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:22:42 smithi083 ceph-mon[79930]: pgmap v17830: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:42.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:22:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:22:42.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:22:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:22:42.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:22:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:22:42.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:22:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:22:43.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:22:42 smithi153 ceph-mon[70044]: pgmap v17830: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:43.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:22:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:22:43.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:22:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:22:43.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:22:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:22:43.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:22:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:22:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:22:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:22:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:22:44.970 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:22:44 smithi083 ceph-mon[79930]: pgmap v17831: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:44.972 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:22:44 smithi083 ceph-mon[77242]: pgmap v17831: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:45.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:22:44 smithi153 ceph-mon[70044]: pgmap v17831: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:46.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:22:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:22:45.751Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:22:46.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:22:46 smithi083 ceph-mon[79930]: pgmap v17832: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:46.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:22:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:22:46.489Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:22:46.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:22:46 smithi083 ceph-mon[77242]: pgmap v17832: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:47.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:22:46 smithi153 ceph-mon[70044]: pgmap v17832: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:48.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:22:48 smithi083 ceph-mon[79930]: pgmap v17833: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:48.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:22:48 smithi083 ceph-mon[77242]: pgmap v17833: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:49.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:22:48 smithi153 ceph-mon[70044]: pgmap v17833: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:50.975 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:22:50 smithi083 ceph-mon[79930]: pgmap v17834: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:50.975 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:22:50 smithi083 ceph-mon[77242]: pgmap v17834: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:51.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:22:50 smithi153 ceph-mon[70044]: pgmap v17834: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:22:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:22:51] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T15:22:52.588 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:22:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:22:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:22:52.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:22:52 smithi083 ceph-mon[77242]: pgmap v17835: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:52.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:22:52 smithi083 ceph-mon[79930]: pgmap v17835: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:52.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:22:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:22:52.396Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:22:53.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:22:52 smithi153 ceph-mon[70044]: pgmap v17835: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:53.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:22:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:22:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:22:54.972 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:22:54 smithi083 ceph-mon[79930]: pgmap v17836: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:54.973 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:22:54 smithi083 ceph-mon[77242]: pgmap v17836: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:55.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:22:54 smithi153 ceph-mon[70044]: pgmap v17836: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:55.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:22:55 smithi083 ceph-mon[79930]: pgmap v17837: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:55.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:22:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:22:55.752Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:22:55.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:22:55 smithi083 ceph-mon[77242]: pgmap v17837: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:56.065 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:22:55 smithi153 ceph-mon[70044]: pgmap v17837: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:56.964 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:22:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:22:56.489Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:22:58.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:22:58 smithi083 ceph-mon[79930]: pgmap v17838: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:58.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:22:58 smithi083 ceph-mon[77242]: pgmap v17838: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:59.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:22:58 smithi153 ceph-mon[70044]: pgmap v17838: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:59.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:22:59 smithi083 ceph-mon[79930]: pgmap v17839: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:22:59.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:22:59 smithi083 ceph-mon[77242]: pgmap v17839: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:00.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:22:59 smithi153 ceph-mon[70044]: pgmap v17839: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:01.967 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:23:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:23:01] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T15:23:01.968 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:23:01 smithi083 ceph-mon[79930]: pgmap v17840: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:01.969 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:23:01 smithi083 ceph-mon[77242]: pgmap v17840: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:02.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:23:01 smithi153 ceph-mon[70044]: pgmap v17840: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:02.315 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:23:02.604 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:23:02.605 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 3m ago 9h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:23:02.606 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 98s ago 9h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:23:02.606 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 3m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:23:02.608 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 98s ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:23:02.608 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 98s ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:23:02.612 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 3m ago 10h 997M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:23:02.612 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 3m ago 10h 2159M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:23:02.613 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 98s ago 10h 942M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:23:02.613 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 3m ago 10h 1295M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:23:02.614 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 3m ago 9h 24.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:23:02.614 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 98s ago 9h 27.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:23:02.615 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 3m ago 10h 1568M 2162M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:23:02.624 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 3m ago 10h 1555M 2162M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:23:02.627 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 3m ago 9h 1208M 2162M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:23:02.627 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 3m ago 9h 1524M 2162M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:23:02.628 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 98s ago 9h 1336M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:23:02.629 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 98s ago 9h 1678M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:23:02.629 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 98s ago 9h 1500M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:23:02.630 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 98s ago 9h 1784M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:23:02.630 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 98s ago 9h 175M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:23:02.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:23:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:23:02.396Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:23:02.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:23:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:23:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:23:02.956 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:23:02.957 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:23:02.957 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:23:02.957 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:23:02.958 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:23:02.958 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:23:02.958 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:23:02.958 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:23:02.959 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:23:02.960 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:23:02.961 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:23:02.961 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:23:02.963 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:23:02.963 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:23:02.964 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:23:02.964 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:23:02.964 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:23:02.965 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:23:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:23:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:23:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:23:03.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:23:03 smithi083 ceph-mon[77242]: from='client.64807 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:23:03.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:23:03 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3191390416' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:23:03.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:23:03 smithi083 ceph-mon[79930]: from='client.64807 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:23:03.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:23:03 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3191390416' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:23:04.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:23:03 smithi153 ceph-mon[70044]: from='client.64807 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:23:04.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:23:03 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3191390416' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:23:04.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:23:04 smithi083 ceph-mon[79930]: from='client.54882 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:23:04.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:23:04 smithi083 ceph-mon[79930]: pgmap v17841: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:04.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:23:04 smithi083 ceph-mon[77242]: from='client.54882 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:23:04.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:23:04 smithi083 ceph-mon[77242]: pgmap v17841: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:05.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:23:04 smithi153 ceph-mon[70044]: from='client.54882 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:23:05.063 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:23:04 smithi153 ceph-mon[70044]: pgmap v17841: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:06.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:23:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:23:05.753Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:23:06.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:23:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:23:06.491Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:23:06.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:23:06 smithi083 ceph-mon[77242]: pgmap v17842: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:06.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:23:06 smithi083 ceph-mon[79930]: pgmap v17842: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:07.063 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:23:06 smithi153 ceph-mon[70044]: pgmap v17842: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:08.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:23:08 smithi083 ceph-mon[77242]: pgmap v17843: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:08.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:23:08 smithi083 ceph-mon[79930]: pgmap v17843: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:09.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:23:08 smithi153 ceph-mon[70044]: pgmap v17843: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:10.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:23:10 smithi083 ceph-mon[77242]: pgmap v17844: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:10.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:23:10 smithi083 ceph-mon[79930]: pgmap v17844: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:11.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:23:10 smithi153 ceph-mon[70044]: pgmap v17844: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:11.999 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:23:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:23:11] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T15:23:12.578 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:23:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:23:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:23:12.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:23:12 smithi083 ceph-mon[77242]: pgmap v17845: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:12.718 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:23:12 smithi083 ceph-mon[79930]: pgmap v17845: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:12.719 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:23:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:23:12.398Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:23:13.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:23:12 smithi153 ceph-mon[70044]: pgmap v17845: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:13.318 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:23:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:23:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:23:13.969 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:23:13 smithi083 ceph-mon[77242]: pgmap v17846: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:13.969 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:23:13 smithi083 ceph-mon[79930]: pgmap v17846: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:14.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:23:13 smithi153 ceph-mon[70044]: pgmap v17846: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:15.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:23:15 smithi083 ceph-mon[77242]: pgmap v17847: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:15.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:23:15 smithi083 ceph-mon[79930]: pgmap v17847: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:15.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:23:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:23:15.754Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:23:16.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:23:15 smithi153 ceph-mon[70044]: pgmap v17847: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:16.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:23:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:23:16.490Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:23:18.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:23:18 smithi083 ceph-mon[79930]: pgmap v17848: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:18.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:23:18 smithi083 ceph-mon[77242]: pgmap v17848: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:19.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:23:18 smithi153 ceph-mon[70044]: pgmap v17848: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:20.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:23:20 smithi083 ceph-mon[79930]: pgmap v17849: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:20.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:23:20 smithi083 ceph-mon[77242]: pgmap v17849: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:21.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:23:20 smithi153 ceph-mon[70044]: pgmap v17849: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:21.966 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:23:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:23:21] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T15:23:22.577 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:23:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:23:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:23:22.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:23:22 smithi083 ceph-mon[79930]: pgmap v17850: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:22.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:23:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:23:22.398Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:23:22.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:23:22 smithi083 ceph-mon[77242]: pgmap v17850: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:23.027 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:23:22 smithi153 ceph-mon[70044]: pgmap v17850: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:23:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:23:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:23:23.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:23:23 smithi083 ceph-mon[79930]: pgmap v17851: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:23.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:23:23 smithi083 ceph-mon[77242]: pgmap v17851: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:24.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:23:23 smithi153 ceph-mon[70044]: pgmap v17851: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:25.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:23:25 smithi083 ceph-mon[79930]: pgmap v17852: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:25.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:23:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:23:25.754Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:23:25.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:23:25 smithi083 ceph-mon[77242]: pgmap v17852: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:26.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:23:25 smithi153 ceph-mon[70044]: pgmap v17852: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:26.972 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:23:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:23:26.491Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:23:28.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:23:28 smithi083 ceph-mon[77242]: pgmap v17853: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:28.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:23:28 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:23:28.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:23:28 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:23:28.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:23:28 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:23:28.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:23:28 smithi083 ceph-mon[79930]: pgmap v17853: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:28.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:23:28 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:23:28.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:23:28 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:23:28.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:23:28 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:23:29.067 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:23:28 smithi153 ceph-mon[70044]: pgmap v17853: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:29.067 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:23:28 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:23:29.077 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:23:28 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:23:29.077 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:23:28 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:23:29.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:23:29 smithi083 ceph-mon[79930]: pgmap v17854: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:29.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:23:29 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:23:29.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:23:29 smithi083 ceph-mon[77242]: pgmap v17854: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:29.968 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:23:29 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:23:30.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:23:29 smithi153 ceph-mon[70044]: pgmap v17854: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:30.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:23:29 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:23:31.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:23:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:23:31] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T15:23:31.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:23:31 smithi083 ceph-mon[79930]: pgmap v17855: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:31.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:23:31 smithi083 ceph-mon[77242]: pgmap v17855: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:32.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:23:31 smithi153 ceph-mon[70044]: pgmap v17855: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:32.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:23:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:23:32.399Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:23:32.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:23:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:23:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:23:33.283 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:23:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:23:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:23:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:23:33.581 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:23:33.581 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 4m ago 9h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:23:33.582 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 2m ago 9h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:23:33.582 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 4m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:23:33.582 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 2m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:23:33.583 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 2m ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:23:33.583 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 4m ago 10h 997M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:23:33.583 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 4m ago 10h 2159M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:23:33.584 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 2m ago 10h 942M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:23:33.584 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 4m ago 10h 1295M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:23:33.584 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 4m ago 9h 24.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:23:33.584 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 2m ago 9h 27.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:23:33.585 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 4m ago 10h 1568M 2162M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:23:33.585 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 4m ago 10h 1555M 2162M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:23:33.585 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 4m ago 10h 1208M 2162M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:23:33.585 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 4m ago 9h 1524M 2162M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:23:33.586 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 2m ago 9h 1336M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:23:33.586 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 2m ago 9h 1678M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:23:33.586 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 2m ago 9h 1500M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:23:33.586 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 2m ago 9h 1784M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:23:33.587 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 2m ago 9h 175M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:23:33.948 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:23:33.949 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:23:33.949 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:23:33.950 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:23:33.950 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:23:33.951 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:23:33.951 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:23:33.952 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:23:33.952 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:23:33.952 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:23:33.952 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:23:33.953 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:23:33.953 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:23:33.953 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:23:33.954 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:23:33.954 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:23:33.954 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:23:33.954 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:23:34.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:23:34 smithi083 ceph-mon[77242]: pgmap v17856: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:34.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:23:34 smithi083 ceph-mon[77242]: from='client.54894 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:23:34.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:23:34 smithi083 ceph-mon[77242]: from='client.64831 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:23:34.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:23:34 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/486084844' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:23:34.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:23:34 smithi083 ceph-mon[79930]: pgmap v17856: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:34.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:23:34 smithi083 ceph-mon[79930]: from='client.54894 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:23:34.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:23:34 smithi083 ceph-mon[79930]: from='client.64831 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:23:34.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:23:34 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/486084844' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:23:35.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:23:34 smithi153 ceph-mon[70044]: pgmap v17856: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:35.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:23:34 smithi153 ceph-mon[70044]: from='client.54894 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:23:35.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:23:34 smithi153 ceph-mon[70044]: from='client.64831 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:23:35.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:23:34 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/486084844' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:23:36.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:23:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:23:35.754Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:23:36.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:23:36 smithi083 ceph-mon[77242]: pgmap v17857: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:36.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:23:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:23:36.492Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:23:36.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:23:36 smithi083 ceph-mon[79930]: pgmap v17857: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:37.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:23:36 smithi153 ceph-mon[70044]: pgmap v17857: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:38.960 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:23:38 smithi083 ceph-mon[77242]: pgmap v17858: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:38.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:23:38 smithi083 ceph-mon[79930]: pgmap v17858: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:39.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:23:38 smithi153 ceph-mon[70044]: pgmap v17858: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:40.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:23:40 smithi083 ceph-mon[77242]: pgmap v17859: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:40.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:23:40 smithi083 ceph-mon[79930]: pgmap v17859: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:41.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:23:40 smithi153 ceph-mon[70044]: pgmap v17859: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:41.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:23:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:23:41] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T15:23:41.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:23:41 smithi083 ceph-mon[77242]: pgmap v17860: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:41.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:23:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:23:41.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:23:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:23:41.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:23:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:23:41.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:23:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:23:41.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:23:41 smithi083 ceph-mon[79930]: pgmap v17860: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:41.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:23:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:23:41.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:23:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:23:41.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:23:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:23:41.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:23:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:23:42.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:23:41 smithi153 ceph-mon[70044]: pgmap v17860: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:42.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:23:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:23:42.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:23:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:23:42.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:23:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:23:42.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:23:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:23:42.714 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:23:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:23:42.400Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:23:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:23:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:23:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:23:43.314 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:23:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:23:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:23:44.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:23:44 smithi083 ceph-mon[79930]: pgmap v17861: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:44.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:23:44 smithi083 ceph-mon[77242]: pgmap v17861: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:45.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:23:44 smithi153 ceph-mon[70044]: pgmap v17861: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:46.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:23:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:23:45.756Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:23:46.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:23:46 smithi083 ceph-mon[79930]: pgmap v17862: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:46.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:23:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:23:46.493Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:23:46.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:23:46 smithi083 ceph-mon[77242]: pgmap v17862: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:47.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:23:46 smithi153 ceph-mon[70044]: pgmap v17862: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:48.960 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:23:48 smithi083 ceph-mon[77242]: pgmap v17863: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:48.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:23:48 smithi083 ceph-mon[79930]: pgmap v17863: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:49.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:23:48 smithi153 ceph-mon[70044]: pgmap v17863: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:50.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:23:50 smithi083 ceph-mon[77242]: pgmap v17864: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:50.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:23:50 smithi083 ceph-mon[79930]: pgmap v17864: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:51.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:23:50 smithi153 ceph-mon[70044]: pgmap v17864: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:51.960 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:23:51 smithi083 ceph-mon[77242]: pgmap v17865: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:23:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:23:51] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T15:23:51.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:23:51 smithi083 ceph-mon[79930]: pgmap v17865: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:52.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:23:51 smithi153 ceph-mon[70044]: pgmap v17865: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:52.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:23:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:23:52.400Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:23:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:23:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:23:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:23:53.314 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:23:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:23:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:23:54.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:23:54 smithi083 ceph-mon[79930]: pgmap v17866: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:54.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:23:54 smithi083 ceph-mon[77242]: pgmap v17866: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:55.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:23:54 smithi153 ceph-mon[70044]: pgmap v17866: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:55.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:23:55 smithi083 ceph-mon[79930]: pgmap v17867: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:55.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:23:55 smithi083 ceph-mon[77242]: pgmap v17867: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:55.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:23:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:23:55.755Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:23:56.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:23:55 smithi153 ceph-mon[70044]: pgmap v17867: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:56.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:23:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:23:56.494Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:23:58.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:23:58 smithi083 ceph-mon[79930]: pgmap v17868: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:58.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:23:58 smithi083 ceph-mon[77242]: pgmap v17868: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:59.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:23:58 smithi153 ceph-mon[70044]: pgmap v17868: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:59.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:23:59 smithi083 ceph-mon[77242]: pgmap v17869: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:23:59.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:23:59 smithi083 ceph-mon[79930]: pgmap v17869: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:00.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:23:59 smithi153 ceph-mon[70044]: pgmap v17869: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:24:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:24:01] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T15:24:01.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:24:01 smithi083 ceph-mon[79930]: pgmap v17870: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:01.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:24:01 smithi083 ceph-mon[77242]: pgmap v17870: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:02.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:24:01 smithi153 ceph-mon[70044]: pgmap v17870: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:02.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:24:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:24:02.401Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:24:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:24:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:24:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:24:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:24:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:24:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:24:04.277 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:24:04.567 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:24:04.567 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 4m ago 9h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:24:04.567 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 2m ago 9h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:24:04.568 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 4m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:24:04.568 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 2m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:24:04.568 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 2m ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:24:04.568 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 4m ago 10h 997M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:24:04.569 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 4m ago 10h 2159M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:24:04.569 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 2m ago 10h 942M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:24:04.569 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 4m ago 10h 1295M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:24:04.569 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 4m ago 9h 24.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:24:04.570 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 2m ago 9h 27.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:24:04.570 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 4m ago 10h 1568M 2162M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:24:04.570 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 4m ago 10h 1555M 2162M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:24:04.571 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 4m ago 10h 1208M 2162M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:24:04.571 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 4m ago 10h 1524M 2162M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:24:04.571 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 2m ago 10h 1336M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:24:04.571 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 2m ago 9h 1678M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:24:04.571 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 2m ago 9h 1500M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:24:04.572 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 2m ago 9h 1784M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:24:04.572 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 2m ago 9h 175M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:24:04.919 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:24:04.919 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:24:04.920 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:24:04.920 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:24:04.924 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:24:04.924 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:24:04.924 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:24:04.925 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:24:04.925 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:24:04.925 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:24:04.925 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:24:04.926 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:24:04.926 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:24:04.926 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:24:04.926 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:24:04.927 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:24:04.927 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:24:04.927 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:24:04.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:24:04 smithi083 ceph-mon[79930]: pgmap v17871: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:04.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:24:04 smithi083 ceph-mon[77242]: pgmap v17871: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:05.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:24:04 smithi153 ceph-mon[70044]: pgmap v17871: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:05.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:24:05 smithi083 ceph-mon[77242]: from='client.64843 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:24:05.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:24:05 smithi083 ceph-mon[77242]: from='client.64849 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:24:05.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:24:05 smithi083 ceph-mon[77242]: pgmap v17872: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:05.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:24:05 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3262342029' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:24:05.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:24:05 smithi083 ceph-mon[79930]: from='client.64843 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:24:05.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:24:05 smithi083 ceph-mon[79930]: from='client.64849 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:24:05.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:24:05 smithi083 ceph-mon[79930]: pgmap v17872: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:05.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:24:05 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3262342029' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:24:05.965 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:24:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:24:05.756Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:24:06.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:24:05 smithi153 ceph-mon[70044]: from='client.64843 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:24:06.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:24:05 smithi153 ceph-mon[70044]: from='client.64849 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:24:06.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:24:05 smithi153 ceph-mon[70044]: pgmap v17872: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:06.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:24:05 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3262342029' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:24:06.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:24:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:24:06.495Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:24:08.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:24:08 smithi083 ceph-mon[79930]: pgmap v17873: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:08.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:24:08 smithi083 ceph-mon[77242]: pgmap v17873: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:09.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:24:08 smithi153 ceph-mon[70044]: pgmap v17873: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:09.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:24:09 smithi083 ceph-mon[79930]: pgmap v17874: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:09.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:24:09 smithi083 ceph-mon[77242]: pgmap v17874: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:10.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:24:09 smithi153 ceph-mon[70044]: pgmap v17874: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:24:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:24:11] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T15:24:11.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:24:11 smithi083 ceph-mon[79930]: pgmap v17875: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:11.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:24:11 smithi083 ceph-mon[77242]: pgmap v17875: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:12.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:24:11 smithi153 ceph-mon[70044]: pgmap v17875: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:12.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:24:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:24:12.401Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:24:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:24:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:24:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:24:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:24:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:24:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:24:14.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:24:14 smithi083 ceph-mon[79930]: pgmap v17876: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:14.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:24:14 smithi083 ceph-mon[77242]: pgmap v17876: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:15.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:24:14 smithi153 ceph-mon[70044]: pgmap v17876: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:15.960 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:24:15 smithi083 ceph-mon[77242]: pgmap v17877: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:15.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:24:15 smithi083 ceph-mon[79930]: pgmap v17877: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:15.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:24:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:24:15.756Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:24:16.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:24:15 smithi153 ceph-mon[70044]: pgmap v17877: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:16.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:24:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:24:16.494Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:24:18.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:24:18 smithi083 ceph-mon[77242]: pgmap v17878: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:18.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:24:18 smithi083 ceph-mon[79930]: pgmap v17878: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:19.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:24:18 smithi153 ceph-mon[70044]: pgmap v17878: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:19.960 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:24:19 smithi083 ceph-mon[77242]: pgmap v17879: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:19.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:24:19 smithi083 ceph-mon[79930]: pgmap v17879: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:20.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:24:19 smithi153 ceph-mon[70044]: pgmap v17879: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:21.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:24:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:24:21] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T15:24:21.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:24:21 smithi083 ceph-mon[79930]: pgmap v17880: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:21.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:24:21 smithi083 ceph-mon[77242]: pgmap v17880: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:22.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:24:21 smithi153 ceph-mon[70044]: pgmap v17880: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:22.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:24:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:24:22.402Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:24:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:24:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:24:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:24:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:24:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:24:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:24:23.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:24:23 smithi083 ceph-mon[79930]: pgmap v17881: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:23.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:24:23 smithi083 ceph-mon[77242]: pgmap v17881: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:24.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:24:23 smithi153 ceph-mon[70044]: pgmap v17881: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:25.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:24:25 smithi083 ceph-mon[79930]: pgmap v17882: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:25.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:24:25 smithi083 ceph-mon[77242]: pgmap v17882: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:25.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:24:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:24:25.757Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:24:26.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:24:25 smithi153 ceph-mon[70044]: pgmap v17882: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:26.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:24:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:24:26.495Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:24:28.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:24:28 smithi083 ceph-mon[79930]: pgmap v17883: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:28.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:24:28 smithi083 ceph-mon[77242]: pgmap v17883: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:29.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:24:28 smithi153 ceph-mon[70044]: pgmap v17883: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:29.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:24:29 smithi083 ceph-mon[79930]: pgmap v17884: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:29.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:24:29 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:24:29.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:24:29 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:24:29.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:24:29 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:24:29.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:24:29 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:24:29.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:24:29 smithi083 ceph-mon[77242]: pgmap v17884: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:29.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:24:29 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:24:29.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:24:29 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:24:29.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:24:29 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:24:29.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:24:29 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:24:30.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:24:29 smithi153 ceph-mon[70044]: pgmap v17884: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:30.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:24:29 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:24:30.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:24:29 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:24:30.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:24:29 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:24:30.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:24:29 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:24:31.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:24:31 smithi083 ceph-mon[77242]: pgmap v17885: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:24:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:24:31] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T15:24:31.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:24:31 smithi083 ceph-mon[79930]: pgmap v17885: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:32.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:24:31 smithi153 ceph-mon[70044]: pgmap v17885: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:32.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:24:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:24:32.402Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:24:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:24:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:24:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:24:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:24:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:24:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:24:33.960 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:24:33 smithi083 ceph-mon[77242]: pgmap v17886: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:33.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:24:33 smithi083 ceph-mon[79930]: pgmap v17886: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:34.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:24:33 smithi153 ceph-mon[70044]: pgmap v17886: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:35.238 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:24:35.528 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:24:35.528 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 5m ago 9h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:24:35.529 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 3m ago 9h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:24:35.529 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (9h) 5m ago 9h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:24:35.530 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (9h) 3m ago 9h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:24:35.530 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 3m ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:24:35.531 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 5m ago 10h 997M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:24:35.531 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 5m ago 10h 2159M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:24:35.532 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 3m ago 10h 942M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:24:35.532 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 5m ago 10h 1295M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:24:35.533 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 5m ago 9h 24.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:24:35.533 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 3m ago 9h 27.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:24:35.534 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 5m ago 10h 1568M 2162M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:24:35.534 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 5m ago 10h 1555M 2162M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:24:35.535 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 5m ago 10h 1208M 2162M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:24:35.535 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 5m ago 10h 1524M 2162M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:24:35.536 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 3m ago 10h 1336M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:24:35.536 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 3m ago 10h 1678M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:24:35.537 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 3m ago 10h 1500M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:24:35.537 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 3m ago 9h 1784M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:24:35.538 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 3m ago 9h 175M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:24:35.885 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:24:35.886 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:24:35.886 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:24:35.886 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:24:35.887 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:24:35.887 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:24:35.887 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:24:35.887 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:24:35.888 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:24:35.888 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:24:35.888 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:24:35.888 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:24:35.889 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:24:35.889 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:24:35.889 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:24:35.890 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:24:35.890 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:24:35.890 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:24:35.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:24:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:24:35.756Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:24:35.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:24:35 smithi083 ceph-mon[79930]: pgmap v17887: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:35.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:24:35 smithi083 ceph-mon[79930]: from='client.54930 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:24:35.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:24:35 smithi083 ceph-mon[79930]: from='client.54936 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:24:35.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:24:35 smithi083 ceph-mon[77242]: pgmap v17887: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:35.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:24:35 smithi083 ceph-mon[77242]: from='client.54930 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:24:35.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:24:35 smithi083 ceph-mon[77242]: from='client.54936 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:24:36.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:24:35 smithi153 ceph-mon[70044]: pgmap v17887: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:36.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:24:35 smithi153 ceph-mon[70044]: from='client.54930 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:24:36.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:24:35 smithi153 ceph-mon[70044]: from='client.54936 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:24:36.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:24:36 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2481496140' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:24:36.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:24:36 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2481496140' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:24:36.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:24:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:24:36.496Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:24:37.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:24:36 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2481496140' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:24:37.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:24:37 smithi083 ceph-mon[79930]: pgmap v17888: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:37.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:24:37 smithi083 ceph-mon[77242]: pgmap v17888: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:38.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:24:37 smithi153 ceph-mon[70044]: pgmap v17888: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:39.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:24:39 smithi083 ceph-mon[79930]: pgmap v17889: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:24:39 smithi083 ceph-mon[77242]: pgmap v17889: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:24:39 smithi153 ceph-mon[70044]: pgmap v17889: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:24:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:24:41] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T15:24:41.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:24:41 smithi083 ceph-mon[79930]: pgmap v17890: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:41.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:24:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:24:41.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:24:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:24:41.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:24:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:24:41.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:24:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:24:41.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:24:41 smithi083 ceph-mon[77242]: pgmap v17890: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:41.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:24:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:24:41.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:24:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:24:41.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:24:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:24:41.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:24:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:24:42.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:24:41 smithi153 ceph-mon[70044]: pgmap v17890: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:42.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:24:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:24:42.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:24:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:24:42.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:24:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:24:42.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:24:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:24:42.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:24:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:24:42.403Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:24:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:24:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:24:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:24:43.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:24:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:24:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:24:43.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:24:43 smithi083 ceph-mon[79930]: pgmap v17891: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:43.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:24:43 smithi083 ceph-mon[77242]: pgmap v17891: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:44.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:24:43 smithi153 ceph-mon[70044]: pgmap v17891: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:45.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:24:45 smithi083 ceph-mon[79930]: pgmap v17892: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:45.964 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:24:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:24:45.758Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:24:45.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:24:45 smithi083 ceph-mon[77242]: pgmap v17892: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:46.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:24:45 smithi153 ceph-mon[70044]: pgmap v17892: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:46.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:24:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:24:46.497Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:24:47.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:24:47 smithi083 ceph-mon[79930]: pgmap v17893: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:47.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:24:47 smithi083 ceph-mon[77242]: pgmap v17893: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:48.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:24:47 smithi153 ceph-mon[70044]: pgmap v17893: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:49.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:24:49 smithi083 ceph-mon[79930]: pgmap v17894: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:49.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:24:49 smithi083 ceph-mon[77242]: pgmap v17894: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:50.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:24:49 smithi153 ceph-mon[70044]: pgmap v17894: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:51.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:24:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:24:51] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T15:24:51.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:24:51 smithi083 ceph-mon[79930]: pgmap v17895: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:51.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:24:51 smithi083 ceph-mon[77242]: pgmap v17895: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:52.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:24:51 smithi153 ceph-mon[70044]: pgmap v17895: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:52.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:24:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:24:52.403Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:24:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:24:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:24:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:24:53.310 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:24:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:24:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:24:53.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:24:53 smithi083 ceph-mon[77242]: pgmap v17896: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:53.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:24:53 smithi083 ceph-mon[79930]: pgmap v17896: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:54.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:24:53 smithi153 ceph-mon[70044]: pgmap v17896: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:55.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:24:55 smithi083 ceph-mon[79930]: pgmap v17897: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:55.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:24:55 smithi083 ceph-mon[77242]: pgmap v17897: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:55.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:24:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:24:55.758Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:24:56.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:24:55 smithi153 ceph-mon[70044]: pgmap v17897: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:56.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:24:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:24:56.497Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:24:57.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:24:57 smithi083 ceph-mon[77242]: pgmap v17898: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:57.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:24:57 smithi083 ceph-mon[79930]: pgmap v17898: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:58.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:24:57 smithi153 ceph-mon[70044]: pgmap v17898: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:59.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:24:59 smithi083 ceph-mon[79930]: pgmap v17899: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:24:59.974 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:24:59 smithi083 ceph-mon[77242]: pgmap v17899: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:25:00.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:24:59 smithi153 ceph-mon[70044]: pgmap v17899: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:25:01.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:25:01 smithi083 ceph-mon[79930]: pgmap v17900: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:25:01.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:25:01 smithi083 ceph-mon[77242]: pgmap v17900: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:25:01.964 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:25:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:25:01] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T15:25:02.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:25:01 smithi153 ceph-mon[70044]: pgmap v17900: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:25:02.714 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:25:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:25:02.404Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:25:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:25:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:25:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:25:03.310 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:25:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:25:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:25:03.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:25:03 smithi083 ceph-mon[79930]: pgmap v17901: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:25:03.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:25:03 smithi083 ceph-mon[77242]: pgmap v17901: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:25:04.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:25:03 smithi153 ceph-mon[70044]: pgmap v17901: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:25:05.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:25:05 smithi083 ceph-mon[79930]: pgmap v17902: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:25:05.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:25:05 smithi083 ceph-mon[77242]: pgmap v17902: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:25:05.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:25:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:25:05.759Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:25:06.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:25:05 smithi153 ceph-mon[70044]: pgmap v17902: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:25:06.212 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:25:06.502 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:25:06.503 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 5m ago 9h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:25:06.503 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 3m ago 9h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:25:06.503 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 5m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:25:06.503 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 3m ago 10h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:25:06.504 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 3m ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:25:06.504 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 5m ago 10h 997M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:25:06.504 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 5m ago 10h 2159M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:25:06.504 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 3m ago 10h 942M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:25:06.505 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 5m ago 10h 1295M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:25:06.505 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (9h) 5m ago 9h 24.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:25:06.505 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 3m ago 9h 27.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:25:06.505 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 5m ago 10h 1568M 2162M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:25:06.506 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 5m ago 10h 1555M 2162M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:25:06.506 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 5m ago 10h 1208M 2162M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:25:06.506 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 5m ago 10h 1524M 2162M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:25:06.506 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 3m ago 10h 1336M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:25:06.507 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 3m ago 10h 1678M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:25:06.507 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 3m ago 10h 1500M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:25:06.507 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 3m ago 10h 1784M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:25:06.508 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 3m ago 9h 175M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:25:06.858 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:25:06.858 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:25:06.859 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:25:06.859 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:25:06.859 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:25:06.859 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:25:06.860 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:25:06.860 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:25:06.860 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:25:06.860 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:25:06.861 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:25:06.861 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:25:06.861 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:25:06.861 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:25:06.862 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:25:06.862 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:25:06.862 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:25:06.862 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:25:06.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:25:06 smithi083 ceph-mon[79930]: from='client.64879 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:25:06.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:25:06 smithi083 ceph-mon[79930]: from='client.54954 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:25:06.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:25:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:25:06.502Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:25:06.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:25:06 smithi083 ceph-mon[77242]: from='client.64879 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:25:06.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:25:06 smithi083 ceph-mon[77242]: from='client.54954 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:25:07.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:25:06 smithi153 ceph-mon[70044]: from='client.64879 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:25:07.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:25:06 smithi153 ceph-mon[70044]: from='client.54954 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:25:07.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:25:07 smithi083 ceph-mon[79930]: pgmap v17903: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:25:07.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:25:07 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1531821442' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:25:07.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:25:07 smithi083 ceph-mon[77242]: pgmap v17903: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:25:07.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:25:07 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1531821442' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:25:08.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:25:07 smithi153 ceph-mon[70044]: pgmap v17903: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:25:08.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:25:07 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1531821442' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:25:09.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:25:09 smithi083 ceph-mon[79930]: pgmap v17904: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:25:09.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:25:09 smithi083 ceph-mon[77242]: pgmap v17904: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:25:10.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:25:09 smithi153 ceph-mon[70044]: pgmap v17904: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:25:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:25:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:25:11] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T15:25:11.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:25:11 smithi083 ceph-mon[79930]: pgmap v17905: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:25:11.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:25:11 smithi083 ceph-mon[77242]: pgmap v17905: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:25:12.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:25:11 smithi153 ceph-mon[70044]: pgmap v17905: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:25:12.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:25:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:25:12.405Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:25:12.810 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:25:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:25:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:25:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:25:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:25:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:25:13.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:25:13 smithi083 ceph-mon[79930]: pgmap v17906: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:25:13.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:25:13 smithi083 ceph-mon[77242]: pgmap v17906: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:25:14.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:25:13 smithi153 ceph-mon[70044]: pgmap v17906: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:25:15.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:25:15 smithi083 ceph-mon[79930]: pgmap v17907: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:25:15.964 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:25:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:25:15.760Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:25:15.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:25:15 smithi083 ceph-mon[77242]: pgmap v17907: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:25:16.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:25:15 smithi153 ceph-mon[70044]: pgmap v17907: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:25:16.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:25:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:25:16.498Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:25:17.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:25:17 smithi083 ceph-mon[77242]: pgmap v17908: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:25:17.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:25:17 smithi083 ceph-mon[79930]: pgmap v17908: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:25:18.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:25:17 smithi153 ceph-mon[70044]: pgmap v17908: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:25:19.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:25:19 smithi083 ceph-mon[79930]: pgmap v17909: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:25:19.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:25:19 smithi083 ceph-mon[77242]: pgmap v17909: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:25:20.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:25:19 smithi153 ceph-mon[70044]: pgmap v17909: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:25:21.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:25:21 smithi083 ceph-mon[77242]: pgmap v17910: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:25:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:25:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:25:21] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T15:25:21.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:25:21 smithi083 ceph-mon[79930]: pgmap v17910: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:25:22.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:25:21 smithi153 ceph-mon[70044]: pgmap v17910: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:25:22.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:25:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:25:22.405Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:25:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:25:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:25:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:25:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:25:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:25:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:25:23.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:25:23 smithi083 ceph-mon[79930]: pgmap v17911: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:25:23.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:25:23 smithi083 ceph-mon[77242]: pgmap v17911: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:25:24.066 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:25:23 smithi153 ceph-mon[70044]: pgmap v17911: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:25:25.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:25:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:25:25.761Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:25:25.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:25:25 smithi083 ceph-mon[77242]: pgmap v17912: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:25:25.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:25:25 smithi083 ceph-mon[79930]: pgmap v17912: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:25:26.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:25:25 smithi153 ceph-mon[70044]: pgmap v17912: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:25:26.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:25:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:25:26.499Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:25:27.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:25:27 smithi083 ceph-mon[79930]: pgmap v17913: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:25:27.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:25:27 smithi083 ceph-mon[77242]: pgmap v17913: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:25:28.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:25:27 smithi153 ceph-mon[70044]: pgmap v17913: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:25:30.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:25:29 smithi153 ceph-mon[70044]: pgmap v17914: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:25:30.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:25:29 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:25:30.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:25:29 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:25:30.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:25:29 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:25:30.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:25:29 smithi083 ceph-mon[79930]: pgmap v17914: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:25:30.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:25:29 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:25:30.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:25:29 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:25:30.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:25:29 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:25:30.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:25:29 smithi083 ceph-mon[77242]: pgmap v17914: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:25:30.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:25:29 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:25:30.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:25:29 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:25:30.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:25:29 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:25:31.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:25:31 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:25:31.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:25:31 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:25:31.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:25:31 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:25:31.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:25:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:25:31] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T15:25:32.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:25:32 smithi153 ceph-mon[70044]: pgmap v17915: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:25:32.407 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:25:32 smithi083 ceph-mon[77242]: pgmap v17915: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:25:32.409 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:25:32 smithi083 ceph-mon[79930]: pgmap v17915: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:25:32.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:25:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:25:32.405Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:25:32.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:25:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:25:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:25:33.310 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:25:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:25:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:25:33.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:25:33 smithi083 ceph-mon[77242]: pgmap v17916: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:25:33.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:25:33 smithi083 ceph-mon[79930]: pgmap v17916: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:25:34.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:25:33 smithi153 ceph-mon[70044]: pgmap v17916: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:25:35.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:25:35 smithi083 ceph-mon[79930]: pgmap v17917: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:25:35.965 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:25:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:25:35.761Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:25:35.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:25:35 smithi083 ceph-mon[77242]: pgmap v17917: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:25:36.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:25:35 smithi153 ceph-mon[70044]: pgmap v17917: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:25:36.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:25:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:25:36.500Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:25:37.184 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:25:37.471 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:25:37.472 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 6m ago 9h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:25:37.472 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (9h) 4m ago 9h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:25:37.473 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 6m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:25:37.473 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 4m ago 10h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:25:37.473 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 4m ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:25:37.474 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 6m ago 10h 997M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:25:37.474 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 6m ago 10h 2159M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:25:37.474 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 4m ago 10h 942M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:25:37.474 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 6m ago 10h 1295M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:25:37.475 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 6m ago 10h 24.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:25:37.475 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (9h) 4m ago 9h 27.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:25:37.475 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 6m ago 10h 1568M 2162M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:25:37.475 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 6m ago 10h 1555M 2162M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:25:37.476 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 6m ago 10h 1208M 2162M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:25:37.476 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 6m ago 10h 1524M 2162M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:25:37.476 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 4m ago 10h 1336M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:25:37.476 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 4m ago 10h 1678M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:25:37.477 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 4m ago 10h 1500M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:25:37.477 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 4m ago 10h 1784M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:25:37.478 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 4m ago 10h 175M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:25:37.831 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:25:37.832 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:25:37.832 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:25:37.832 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:25:37.833 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:25:37.833 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:25:37.833 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:25:37.833 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:25:37.834 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:25:37.835 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:25:37.835 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:25:37.836 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:25:37.836 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:25:37.837 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:25:37.837 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:25:37.839 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:25:37.839 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:25:37.840 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:25:38.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:25:37 smithi153 ceph-mon[70044]: pgmap v17918: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:25:38.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:25:37 smithi153 ceph-mon[70044]: from='client.54966 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:25:38.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:25:37 smithi153 ceph-mon[70044]: from='client.64903 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:25:38.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:25:37 smithi083 ceph-mon[77242]: pgmap v17918: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:25:38.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:25:37 smithi083 ceph-mon[77242]: from='client.54966 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:25:38.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:25:37 smithi083 ceph-mon[77242]: from='client.64903 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:25:38.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:25:37 smithi083 ceph-mon[79930]: pgmap v17918: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:25:38.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:25:37 smithi083 ceph-mon[79930]: from='client.54966 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:25:38.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:25:37 smithi083 ceph-mon[79930]: from='client.64903 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:25:39.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:25:38 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2866145633' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:25:39.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:25:38 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2866145633' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:25:39.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:25:38 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2866145633' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:25:40.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:25:39 smithi153 ceph-mon[70044]: pgmap v17919: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:25:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:25:39 smithi083 ceph-mon[79930]: pgmap v17919: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:25:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:25:39 smithi083 ceph-mon[77242]: pgmap v17919: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:25:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:25:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:25:41] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T15:25:41.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:25:41 smithi083 ceph-mon[77242]: pgmap v17920: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:25:41.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:25:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:25:41.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:25:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:25:41.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:25:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:25:41.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:25:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:25:41.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:25:41 smithi083 ceph-mon[79930]: pgmap v17920: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:25:41.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:25:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:25:41.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:25:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:25:41.968 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:25:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:25:41.968 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:25:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:25:42.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:25:41 smithi153 ceph-mon[70044]: pgmap v17920: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:25:42.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:25:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:25:42.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:25:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:25:42.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:25:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:25:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:25:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:25:42.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:25:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:25:42.407Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:25:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:25:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:25:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:25:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:25:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:25:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:25:43.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:25:43 smithi083 ceph-mon[79930]: pgmap v17921: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:25:43.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:25:43 smithi083 ceph-mon[77242]: pgmap v17921: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:25:44.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:25:43 smithi153 ceph-mon[70044]: pgmap v17921: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:25:46.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:25:45 smithi153 ceph-mon[70044]: pgmap v17922: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:25:46.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:25:45 smithi083 ceph-mon[79930]: pgmap v17922: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:25:46.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:25:45 smithi083 ceph-mon[77242]: pgmap v17922: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:25:46.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:25:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:25:45.762Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:25:46.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:25:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:25:46.501Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:25:48.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:25:47 smithi153 ceph-mon[70044]: pgmap v17923: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:25:48.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:25:47 smithi083 ceph-mon[77242]: pgmap v17923: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:25:48.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:25:47 smithi083 ceph-mon[79930]: pgmap v17923: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:25:50.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:25:49 smithi153 ceph-mon[70044]: pgmap v17924: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:25:50.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:25:49 smithi083 ceph-mon[79930]: pgmap v17924: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:25:50.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:25:49 smithi083 ceph-mon[77242]: pgmap v17924: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:25:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:25:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:25:51] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T15:25:51.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:25:51 smithi083 ceph-mon[79930]: pgmap v17925: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:25:51.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:25:51 smithi083 ceph-mon[77242]: pgmap v17925: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:25:52.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:25:51 smithi153 ceph-mon[70044]: pgmap v17925: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:25:52.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:25:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:25:52.407Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:25:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:25:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:25:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:25:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:25:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:25:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:25:53.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:25:53 smithi083 ceph-mon[79930]: pgmap v17926: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:25:53.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:25:53 smithi083 ceph-mon[77242]: pgmap v17926: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:25:54.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:25:53 smithi153 ceph-mon[70044]: pgmap v17926: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:25:56.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:25:55 smithi153 ceph-mon[70044]: pgmap v17927: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:25:56.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:25:55 smithi083 ceph-mon[79930]: pgmap v17927: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:25:56.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:25:55 smithi083 ceph-mon[77242]: pgmap v17927: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:25:56.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:25:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:25:55.763Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:25:56.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:25:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:25:56.501Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:25:58.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:25:57 smithi153 ceph-mon[70044]: pgmap v17928: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:25:58.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:25:57 smithi083 ceph-mon[77242]: pgmap v17928: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:25:58.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:25:57 smithi083 ceph-mon[79930]: pgmap v17928: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:00.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:25:59 smithi153 ceph-mon[70044]: pgmap v17929: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:00.210 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:25:59 smithi083 ceph-mon[77242]: pgmap v17929: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:00.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:25:59 smithi083 ceph-mon[79930]: pgmap v17929: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:26:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:26:01] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T15:26:01.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:26:01 smithi083 ceph-mon[79930]: pgmap v17930: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:01.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:26:01 smithi083 ceph-mon[77242]: pgmap v17930: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:02.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:26:01 smithi153 ceph-mon[70044]: pgmap v17930: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:02.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:26:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:26:02.407Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:26:02.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:26:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:26:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:26:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:26:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:26:03.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:26:03 smithi083 ceph-mon[79930]: pgmap v17931: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:03.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:26:03 smithi083 ceph-mon[77242]: pgmap v17931: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:04.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:26:03 smithi153 ceph-mon[70044]: pgmap v17931: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:06.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:26:05 smithi153 ceph-mon[70044]: pgmap v17932: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:06.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:26:05 smithi083 ceph-mon[79930]: pgmap v17932: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:06.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:26:05 smithi083 ceph-mon[77242]: pgmap v17932: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:06.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:26:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:26:05.762Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:26:06.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:26:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:26:06.503Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:26:08.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:26:07 smithi153 ceph-mon[70044]: pgmap v17933: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:08.156 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:26:08.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:26:07 smithi083 ceph-mon[79930]: pgmap v17933: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:08.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:26:07 smithi083 ceph-mon[77242]: pgmap v17933: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:08.446 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:26:08.446 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (9h) 6m ago 10h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:26:08.446 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 4m ago 10h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:26:08.447 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 6m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:26:08.447 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 4m ago 10h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:26:08.447 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 4m ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:26:08.447 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 6m ago 10h 997M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:26:08.448 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 6m ago 10h 2159M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:26:08.448 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 4m ago 10h 942M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:26:08.448 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 6m ago 10h 1295M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:26:08.448 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 6m ago 10h 24.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:26:08.449 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 4m ago 10h 27.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:26:08.449 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 6m ago 10h 1568M 2162M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:26:08.449 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 6m ago 10h 1555M 2162M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:26:08.449 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 6m ago 10h 1208M 2162M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:26:08.449 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 6m ago 10h 1524M 2162M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:26:08.450 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 4m ago 10h 1336M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:26:08.450 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 4m ago 10h 1678M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:26:08.450 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 4m ago 10h 1500M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:26:08.451 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 4m ago 10h 1784M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:26:08.451 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 4m ago 10h 175M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:26:08.805 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:26:08.806 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:26:08.806 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:26:08.806 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:26:08.806 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:26:08.807 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:26:08.807 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:26:08.807 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:26:08.808 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:26:08.808 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:26:08.808 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:26:08.808 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:26:08.809 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:26:08.809 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:26:08.809 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:26:08.809 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:26:08.810 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:26:08.810 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:26:09.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:26:08 smithi153 ceph-mon[70044]: from='client.54984 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:26:09.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:26:08 smithi153 ceph-mon[70044]: from='client.54990 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:26:09.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:26:08 smithi083 ceph-mon[79930]: from='client.54984 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:26:09.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:26:08 smithi083 ceph-mon[79930]: from='client.54990 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:26:09.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:26:08 smithi083 ceph-mon[77242]: from='client.54984 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:26:09.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:26:08 smithi083 ceph-mon[77242]: from='client.54990 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:26:10.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:26:09 smithi153 ceph-mon[70044]: pgmap v17934: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:10.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:26:09 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/690081821' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:26:10.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:26:09 smithi083 ceph-mon[79930]: pgmap v17934: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:10.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:26:09 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/690081821' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:26:10.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:26:09 smithi083 ceph-mon[77242]: pgmap v17934: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:10.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:26:09 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/690081821' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:26:11.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:26:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:26:11] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T15:26:11.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:26:11 smithi083 ceph-mon[79930]: pgmap v17935: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:11.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:26:11 smithi083 ceph-mon[77242]: pgmap v17935: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:12.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:26:11 smithi153 ceph-mon[70044]: pgmap v17935: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:12.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:26:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:26:12.409Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:26:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:26:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:26:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:26:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:26:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:26:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:26:13.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:26:13 smithi083 ceph-mon[79930]: pgmap v17936: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:13.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:26:13 smithi083 ceph-mon[77242]: pgmap v17936: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:14.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:26:13 smithi153 ceph-mon[70044]: pgmap v17936: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:16.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:26:15 smithi153 ceph-mon[70044]: pgmap v17937: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:16.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:26:15 smithi083 ceph-mon[79930]: pgmap v17937: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:16.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:26:15 smithi083 ceph-mon[77242]: pgmap v17937: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:16.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:26:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:26:15.764Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:26:16.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:26:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:26:16.503Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:26:18.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:26:17 smithi153 ceph-mon[70044]: pgmap v17938: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:18.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:26:17 smithi083 ceph-mon[79930]: pgmap v17938: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:18.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:26:17 smithi083 ceph-mon[77242]: pgmap v17938: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:20.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:26:19 smithi153 ceph-mon[70044]: pgmap v17939: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:20.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:26:19 smithi083 ceph-mon[79930]: pgmap v17939: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:20.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:26:19 smithi083 ceph-mon[77242]: pgmap v17939: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:26:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:26:21] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T15:26:21.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:26:21 smithi083 ceph-mon[79930]: pgmap v17940: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:21.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:26:21 smithi083 ceph-mon[77242]: pgmap v17940: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:22.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:26:21 smithi153 ceph-mon[70044]: pgmap v17940: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:22.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:26:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:26:22.409Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:26:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:26:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:26:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:26:23.315 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:26:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:26:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:26:23.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:26:23 smithi083 ceph-mon[79930]: pgmap v17941: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:23.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:26:23 smithi083 ceph-mon[77242]: pgmap v17941: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:24.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:26:23 smithi153 ceph-mon[70044]: pgmap v17941: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:26.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:26:25 smithi153 ceph-mon[70044]: pgmap v17942: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:26.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:26:25 smithi083 ceph-mon[79930]: pgmap v17942: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:26.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:26:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:26:25.765Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:26:26.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:26:25 smithi083 ceph-mon[77242]: pgmap v17942: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:26.966 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:26:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:26:26.504Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:26:28.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:26:27 smithi153 ceph-mon[70044]: pgmap v17943: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:28.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:26:27 smithi083 ceph-mon[79930]: pgmap v17943: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:28.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:26:27 smithi083 ceph-mon[77242]: pgmap v17943: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:30.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:26:29 smithi153 ceph-mon[70044]: pgmap v17944: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:30.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:26:29 smithi083 ceph-mon[79930]: pgmap v17944: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:30.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:26:29 smithi083 ceph-mon[77242]: pgmap v17944: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:31.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:26:30 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:26:31.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:26:30 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:26:31.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:26:30 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:26:31.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:26:30 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:26:31.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:26:30 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:26:31.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:26:30 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:26:31.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:26:30 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:26:31.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:26:30 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:26:31.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:26:30 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:26:31.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:26:30 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:26:31.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:26:30 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:26:31.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:26:30 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:26:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:26:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:26:31] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T15:26:31.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:26:31 smithi083 ceph-mon[79930]: pgmap v17945: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:31.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:26:31 smithi083 ceph-mon[77242]: pgmap v17945: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:32.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:26:31 smithi153 ceph-mon[70044]: pgmap v17945: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:32.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:26:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:26:32.410Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:26:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:26:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:26:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:26:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:26:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:26:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:26:33.960 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:26:33 smithi083 ceph-mon[77242]: pgmap v17946: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:33.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:26:33 smithi083 ceph-mon[79930]: pgmap v17946: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:34.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:26:33 smithi153 ceph-mon[70044]: pgmap v17946: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:36.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:26:35 smithi153 ceph-mon[70044]: pgmap v17947: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:36.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:26:35 smithi083 ceph-mon[79930]: pgmap v17947: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:36.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:26:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:26:35.765Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:26:36.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:26:35 smithi083 ceph-mon[77242]: pgmap v17947: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:36.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:26:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:26:36.504Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:26:38.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:26:37 smithi153 ceph-mon[70044]: pgmap v17948: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:38.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:26:37 smithi083 ceph-mon[79930]: pgmap v17948: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:38.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:26:37 smithi083 ceph-mon[77242]: pgmap v17948: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:39.134 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:26:39.424 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:26:39.424 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 7m ago 10h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:26:39.424 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 5m ago 10h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:26:39.424 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 7m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:26:39.425 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 5m ago 10h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:26:39.425 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 5m ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:26:39.425 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 7m ago 10h 997M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:26:39.425 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 7m ago 10h 2159M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:26:39.426 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 5m ago 10h 942M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:26:39.426 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 7m ago 10h 1295M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:26:39.426 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 7m ago 10h 24.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:26:39.426 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 5m ago 10h 27.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:26:39.427 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 7m ago 10h 1568M 2162M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:26:39.427 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 7m ago 10h 1555M 2162M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:26:39.427 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 7m ago 10h 1208M 2162M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:26:39.427 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 7m ago 10h 1524M 2162M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:26:39.428 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 5m ago 10h 1336M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:26:39.428 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 5m ago 10h 1678M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:26:39.428 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 5m ago 10h 1500M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:26:39.428 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 5m ago 10h 1784M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:26:39.429 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 5m ago 10h 175M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:26:39.784 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:26:39.785 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:26:39.785 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:26:39.786 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:26:39.786 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:26:39.787 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:26:39.787 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:26:39.788 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:26:39.790 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:26:39.790 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:26:39.790 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:26:39.790 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:26:39.791 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:26:39.791 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:26:39.791 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:26:39.792 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:26:39.793 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:26:39.793 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:26:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:26:39 smithi153 ceph-mon[70044]: pgmap v17949: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:26:39 smithi153 ceph-mon[70044]: from='client.64933 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:26:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:26:39 smithi153 ceph-mon[70044]: from='client.55008 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:26:40.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:26:39 smithi083 ceph-mon[79930]: pgmap v17949: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:40.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:26:39 smithi083 ceph-mon[79930]: from='client.64933 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:26:40.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:26:39 smithi083 ceph-mon[79930]: from='client.55008 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:26:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:26:39 smithi083 ceph-mon[77242]: pgmap v17949: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:40.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:26:39 smithi083 ceph-mon[77242]: from='client.64933 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:26:40.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:26:39 smithi083 ceph-mon[77242]: from='client.55008 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:26:41.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:26:40 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1343349447' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:26:41.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:26:40 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1343349447' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:26:41.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:26:40 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1343349447' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:26:41.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:26:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:26:41] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T15:26:41.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:26:41 smithi083 ceph-mon[77242]: pgmap v17950: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:41.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:26:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:26:41.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:26:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:26:41.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:26:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:26:41.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:26:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:26:41.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:26:41 smithi083 ceph-mon[79930]: pgmap v17950: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:41.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:26:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:26:41.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:26:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:26:41.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:26:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:26:41.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:26:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:26:42.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:26:41 smithi153 ceph-mon[70044]: pgmap v17950: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:42.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:26:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:26:42.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:26:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:26:42.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:26:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:26:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:26:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:26:42.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:26:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:26:42.411Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:26:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:26:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:26:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:26:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:26:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:26:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:26:43.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:26:43 smithi083 ceph-mon[79930]: pgmap v17951: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:43.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:26:43 smithi083 ceph-mon[77242]: pgmap v17951: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:44.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:26:43 smithi153 ceph-mon[70044]: pgmap v17951: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:46.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:26:45 smithi153 ceph-mon[70044]: pgmap v17952: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:46.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:26:45 smithi083 ceph-mon[79930]: pgmap v17952: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:46.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:26:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:26:45.766Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:26:46.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:26:45 smithi083 ceph-mon[77242]: pgmap v17952: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:46.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:26:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:26:46.505Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:26:48.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:26:47 smithi153 ceph-mon[70044]: pgmap v17953: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:48.226 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:26:47 smithi083 ceph-mon[77242]: pgmap v17953: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:48.227 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:26:47 smithi083 ceph-mon[79930]: pgmap v17953: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:50.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:26:49 smithi153 ceph-mon[70044]: pgmap v17954: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:50.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:26:49 smithi083 ceph-mon[79930]: pgmap v17954: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:50.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:26:49 smithi083 ceph-mon[77242]: pgmap v17954: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:51.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:26:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:26:51] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T15:26:51.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:26:51 smithi083 ceph-mon[79930]: pgmap v17955: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:51.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:26:51 smithi083 ceph-mon[77242]: pgmap v17955: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:52.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:26:51 smithi153 ceph-mon[70044]: pgmap v17955: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:52.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:26:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:26:52.412Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:26:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:26:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:26:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:26:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:26:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:26:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:26:53.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:26:53 smithi083 ceph-mon[79930]: pgmap v17956: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:53.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:26:53 smithi083 ceph-mon[77242]: pgmap v17956: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:54.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:26:53 smithi153 ceph-mon[70044]: pgmap v17956: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:56.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:26:55 smithi153 ceph-mon[70044]: pgmap v17957: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:56.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:26:55 smithi083 ceph-mon[79930]: pgmap v17957: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:56.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:26:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:26:55.767Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:26:56.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:26:55 smithi083 ceph-mon[77242]: pgmap v17957: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:56.965 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:26:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:26:56.505Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:26:58.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:26:57 smithi153 ceph-mon[70044]: pgmap v17958: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:58.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:26:57 smithi083 ceph-mon[79930]: pgmap v17958: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:26:58.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:26:57 smithi083 ceph-mon[77242]: pgmap v17958: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:00.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:26:59 smithi153 ceph-mon[70044]: pgmap v17959: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:00.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:26:59 smithi083 ceph-mon[79930]: pgmap v17959: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:00.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:26:59 smithi083 ceph-mon[77242]: pgmap v17959: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:27:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:27:01] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T15:27:01.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:27:01 smithi083 ceph-mon[79930]: pgmap v17960: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:01.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:27:01 smithi083 ceph-mon[77242]: pgmap v17960: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:02.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:27:01 smithi153 ceph-mon[70044]: pgmap v17960: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:02.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:27:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:27:02.412Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:27:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:27:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:27:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:27:03.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:27:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:27:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:27:03.960 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:27:03 smithi083 ceph-mon[77242]: pgmap v17961: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:03.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:27:03 smithi083 ceph-mon[79930]: pgmap v17961: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:04.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:27:03 smithi153 ceph-mon[70044]: pgmap v17961: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:06.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:27:05 smithi153 ceph-mon[70044]: pgmap v17962: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:06.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:27:05 smithi083 ceph-mon[79930]: pgmap v17962: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:06.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:27:05 smithi083 ceph-mon[77242]: pgmap v17962: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:06.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:27:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:27:05.767Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:27:06.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:27:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:27:06.505Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:27:08.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:27:07 smithi153 ceph-mon[70044]: pgmap v17963: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:08.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:27:07 smithi083 ceph-mon[79930]: pgmap v17963: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:08.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:27:07 smithi083 ceph-mon[77242]: pgmap v17963: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:10.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:27:09 smithi153 ceph-mon[70044]: pgmap v17964: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:10.106 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:27:10.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:27:09 smithi083 ceph-mon[79930]: pgmap v17964: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:10.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:27:09 smithi083 ceph-mon[77242]: pgmap v17964: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:10.395 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:27:10.395 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 7m ago 10h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:27:10.395 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 5m ago 10h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:27:10.396 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 7m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:27:10.396 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 5m ago 10h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:27:10.396 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 5m ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:27:10.396 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 7m ago 10h 997M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:27:10.397 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 7m ago 10h 2159M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:27:10.397 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 5m ago 10h 942M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:27:10.397 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 7m ago 10h 1295M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:27:10.397 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 7m ago 10h 24.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:27:10.398 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 5m ago 10h 27.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:27:10.398 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 7m ago 10h 1568M 2162M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:27:10.398 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 7m ago 10h 1555M 2162M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:27:10.398 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 7m ago 10h 1208M 2162M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:27:10.399 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 7m ago 10h 1524M 2162M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:27:10.399 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 5m ago 10h 1336M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:27:10.399 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 5m ago 10h 1678M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:27:10.399 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 5m ago 10h 1500M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:27:10.400 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 5m ago 10h 1784M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:27:10.400 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 5m ago 10h 175M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:27:10.757 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:27:10.758 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:27:10.758 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:27:10.758 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:27:10.759 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:27:10.759 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:27:10.759 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:27:10.759 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:27:10.760 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:27:10.760 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:27:10.760 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:27:10.761 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:27:10.761 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:27:10.761 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:27:10.762 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:27:10.762 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:27:10.763 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:27:10.763 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:27:11.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:27:10 smithi153 ceph-mon[70044]: from='client.55020 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:27:11.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:27:10 smithi153 ceph-mon[70044]: from='client.64957 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:27:11.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:27:10 smithi083 ceph-mon[79930]: from='client.55020 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:27:11.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:27:10 smithi083 ceph-mon[79930]: from='client.64957 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:27:11.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:27:10 smithi083 ceph-mon[77242]: from='client.55020 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:27:11.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:27:10 smithi083 ceph-mon[77242]: from='client.64957 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:27:11.757 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:27:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:27:11] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T15:27:11.758 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:27:11 smithi083 ceph-mon[77242]: pgmap v17965: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:11.758 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:27:11 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3421773531' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:27:12.063 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:27:11 smithi153 ceph-mon[70044]: pgmap v17965: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:12.064 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:27:11 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3421773531' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:27:12.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:27:11 smithi083 ceph-mon[79930]: pgmap v17965: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:12.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:27:11 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3421773531' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:27:12.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:27:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:27:12.413Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:27:12.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:27:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:27:12.317Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:27:13.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:27:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:27:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:27:13.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:27:13 smithi083 ceph-mon[79930]: pgmap v17966: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:13.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:27:13 smithi083 ceph-mon[77242]: pgmap v17966: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:14.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:27:13 smithi153 ceph-mon[70044]: pgmap v17966: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:16.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:27:15 smithi153 ceph-mon[70044]: pgmap v17967: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:16.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:27:15 smithi083 ceph-mon[79930]: pgmap v17967: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:16.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:27:15 smithi083 ceph-mon[77242]: pgmap v17967: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:16.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:27:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:27:15.768Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:27:16.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:27:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:27:16.506Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:27:18.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:27:17 smithi153 ceph-mon[70044]: pgmap v17968: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:18.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:27:17 smithi083 ceph-mon[79930]: pgmap v17968: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:18.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:27:17 smithi083 ceph-mon[77242]: pgmap v17968: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:20.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:27:19 smithi153 ceph-mon[70044]: pgmap v17969: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:20.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:27:19 smithi083 ceph-mon[79930]: pgmap v17969: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:20.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:27:19 smithi083 ceph-mon[77242]: pgmap v17969: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:21.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:27:21 smithi083 ceph-mon[79930]: pgmap v17970: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:21.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:27:21 smithi083 ceph-mon[77242]: pgmap v17970: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:27:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:27:21] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T15:27:22.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:27:21 smithi153 ceph-mon[70044]: pgmap v17970: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:22.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:27:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:27:22.414Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:27:22.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:27:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:27:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:27:23.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:27:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:27:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:27:23.960 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:27:23 smithi083 ceph-mon[77242]: pgmap v17971: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:23.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:27:23 smithi083 ceph-mon[79930]: pgmap v17971: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:24.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:27:23 smithi153 ceph-mon[70044]: pgmap v17971: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:26.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:27:25 smithi153 ceph-mon[70044]: pgmap v17972: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:26.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:27:25 smithi083 ceph-mon[79930]: pgmap v17972: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:26.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:27:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:27:25.768Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:27:26.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:27:25 smithi083 ceph-mon[77242]: pgmap v17972: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:26.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:27:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:27:26.507Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:27:28.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:27:27 smithi153 ceph-mon[70044]: pgmap v17973: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:28.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:27:27 smithi083 ceph-mon[77242]: pgmap v17973: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:28.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:27:27 smithi083 ceph-mon[79930]: pgmap v17973: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:30.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:27:29 smithi153 ceph-mon[70044]: pgmap v17974: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:30.210 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:27:29 smithi083 ceph-mon[77242]: pgmap v17974: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:30.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:27:29 smithi083 ceph-mon[79930]: pgmap v17974: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:31.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:27:30 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:27:31.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:27:30 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:27:31.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:27:30 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:27:31.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:27:30 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:27:31.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:27:30 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:27:31.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:27:30 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:27:31.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:27:30 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:27:31.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:27:30 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:27:31.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:27:30 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:27:31.759 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:27:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:27:31] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T15:27:32.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:27:31 smithi153 ceph-mon[70044]: pgmap v17975: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:32.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:27:31 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:27:32.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:27:31 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:27:32.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:27:31 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:27:32.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:27:31 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:27:32.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:27:31 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:27:32.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:27:31 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:27:32.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:27:31 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:27:32.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:27:31 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:27:32.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:27:31 smithi153 ceph-mon[70044]: Adjusting osd_memory_target on smithi083 to 2148M 2022-09-29T15:27:32.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:27:31 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:27:32.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:27:31 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:27:32.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:27:31 smithi083 ceph-mon[77242]: pgmap v17975: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:32.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:27:31 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:27:32.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:27:31 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:27:32.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:27:31 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:27:32.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:27:31 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:27:32.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:27:31 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:27:32.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:27:31 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:27:32.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:27:31 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:27:32.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:27:31 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:27:32.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:27:31 smithi083 ceph-mon[77242]: Adjusting osd_memory_target on smithi083 to 2148M 2022-09-29T15:27:32.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:27:31 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:27:32.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:27:31 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:27:32.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:27:31 smithi083 ceph-mon[79930]: pgmap v17975: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:32.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:27:31 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:27:32.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:27:31 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:27:32.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:27:31 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:27:32.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:27:31 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:27:32.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:27:31 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:27:32.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:27:31 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:27:32.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:27:31 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:27:32.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:27:31 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:27:32.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:27:31 smithi083 ceph-mon[79930]: Adjusting osd_memory_target on smithi083 to 2148M 2022-09-29T15:27:32.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:27:31 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:27:32.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:27:31 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:27:32.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:27:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:27:32.415Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:27:32.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:27:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:27:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:27:33.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:27:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:27:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:27:33.960 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:27:33 smithi083 ceph-mon[77242]: pgmap v17976: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:33.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:27:33 smithi083 ceph-mon[79930]: pgmap v17976: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:34.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:27:33 smithi153 ceph-mon[70044]: pgmap v17976: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:36.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:27:35 smithi153 ceph-mon[70044]: pgmap v17977: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:36.210 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:27:35 smithi083 ceph-mon[77242]: pgmap v17977: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:36.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:27:35 smithi083 ceph-mon[79930]: pgmap v17977: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:36.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:27:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:27:35.769Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:27:36.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:27:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:27:36.506Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:27:38.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:27:37 smithi153 ceph-mon[70044]: pgmap v17978: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:38.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:27:37 smithi083 ceph-mon[79930]: pgmap v17978: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:38.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:27:37 smithi083 ceph-mon[77242]: pgmap v17978: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:27:39 smithi153 ceph-mon[70044]: pgmap v17979: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:40.210 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:27:39 smithi083 ceph-mon[77242]: pgmap v17979: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:40.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:27:39 smithi083 ceph-mon[79930]: pgmap v17979: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:41.082 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:27:41.373 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:27:41.373 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 8m ago 10h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:27:41.373 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 6m ago 10h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:27:41.374 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 8m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:27:41.374 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 6m ago 10h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:27:41.375 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 6m ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:27:41.375 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 8m ago 10h 997M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:27:41.375 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 8m ago 10h 2159M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:27:41.375 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 6m ago 10h 942M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:27:41.376 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 8m ago 10h 1295M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:27:41.376 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 8m ago 10h 24.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:27:41.376 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 6m ago 10h 27.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:27:41.376 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 8m ago 10h 1568M 2148M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:27:41.376 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 8m ago 10h 1555M 2148M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:27:41.377 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 8m ago 10h 1208M 2148M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:27:41.377 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 8m ago 10h 1524M 2148M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:27:41.377 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 6m ago 10h 1336M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:27:41.377 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 6m ago 10h 1678M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:27:41.378 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 6m ago 10h 1500M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:27:41.378 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 6m ago 10h 1784M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:27:41.378 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 6m ago 10h 175M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:27:41.730 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:27:41.731 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:27:41.731 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:27:41.731 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:27:41.732 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:27:41.732 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:27:41.732 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:27:41.732 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:27:41.732 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:27:41.733 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:27:41.733 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:27:41.733 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:27:41.733 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:27:41.734 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:27:41.734 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:27:41.734 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:27:41.734 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:27:41.735 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:27:41.758 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:27:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:27:41] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T15:27:42.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:27:41 smithi153 ceph-mon[70044]: pgmap v17980: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:42.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:27:41 smithi153 ceph-mon[70044]: from='client.55038 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:27:42.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:27:41 smithi153 ceph-mon[70044]: from='client.64975 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:27:42.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:27:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:27:42.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:27:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:27:42.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:27:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:27:42.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:27:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:27:42.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:27:41 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1732105047' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:27:42.210 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:27:41 smithi083 ceph-mon[77242]: pgmap v17980: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:42.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:27:41 smithi083 ceph-mon[77242]: from='client.55038 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:27:42.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:27:41 smithi083 ceph-mon[77242]: from='client.64975 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:27:42.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:27:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:27:42.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:27:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:27:42.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:27:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:27:42.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:27:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:27:42.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:27:41 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1732105047' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:27:42.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:27:41 smithi083 ceph-mon[79930]: pgmap v17980: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:42.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:27:41 smithi083 ceph-mon[79930]: from='client.55038 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:27:42.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:27:41 smithi083 ceph-mon[79930]: from='client.64975 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:27:42.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:27:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:27:42.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:27:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:27:42.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:27:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:27:42.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:27:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:27:42.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:27:41 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1732105047' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:27:42.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:27:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:27:42.415Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:27:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:27:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:27:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:27:43.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:27:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:27:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:27:43.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:27:43 smithi083 ceph-mon[79930]: pgmap v17981: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:43.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:27:43 smithi083 ceph-mon[77242]: pgmap v17981: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:44.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:27:43 smithi153 ceph-mon[70044]: pgmap v17981: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:46.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:27:45 smithi153 ceph-mon[70044]: pgmap v17982: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:46.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:27:45 smithi083 ceph-mon[79930]: pgmap v17982: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:46.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:27:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:27:45.769Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:27:46.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:27:45 smithi083 ceph-mon[77242]: pgmap v17982: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:46.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:27:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:27:46.507Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:27:48.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:27:47 smithi153 ceph-mon[70044]: pgmap v17983: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:48.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:27:47 smithi083 ceph-mon[79930]: pgmap v17983: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:48.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:27:47 smithi083 ceph-mon[77242]: pgmap v17983: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:50.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:27:49 smithi153 ceph-mon[70044]: pgmap v17984: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:50.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:27:49 smithi083 ceph-mon[79930]: pgmap v17984: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:50.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:27:49 smithi083 ceph-mon[77242]: pgmap v17984: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:51.763 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:27:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:27:51] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T15:27:52.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:27:51 smithi153 ceph-mon[70044]: pgmap v17985: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:52.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:27:51 smithi083 ceph-mon[79930]: pgmap v17985: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:52.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:27:51 smithi083 ceph-mon[77242]: pgmap v17985: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:52.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:27:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:27:52.416Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:27:52.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:27:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:27:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:27:53.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:27:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:27:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:27:53.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:27:53 smithi083 ceph-mon[79930]: pgmap v17986: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:53.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:27:53 smithi083 ceph-mon[77242]: pgmap v17986: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:54.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:27:53 smithi153 ceph-mon[70044]: pgmap v17986: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:56.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:27:55 smithi153 ceph-mon[70044]: pgmap v17987: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:56.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:27:55 smithi083 ceph-mon[79930]: pgmap v17987: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:56.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:27:55 smithi083 ceph-mon[77242]: pgmap v17987: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:56.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:27:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:27:55.770Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:27:56.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:27:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:27:56.509Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:27:58.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:27:57 smithi153 ceph-mon[70044]: pgmap v17988: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:58.210 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:27:57 smithi083 ceph-mon[77242]: pgmap v17988: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:27:58.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:27:57 smithi083 ceph-mon[79930]: pgmap v17988: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:00.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:27:59 smithi153 ceph-mon[70044]: pgmap v17989: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:00.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:27:59 smithi083 ceph-mon[79930]: pgmap v17989: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:00.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:27:59 smithi083 ceph-mon[77242]: pgmap v17989: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:01.765 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:28:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:28:01] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T15:28:02.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:28:01 smithi153 ceph-mon[70044]: pgmap v17990: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:02.210 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:28:01 smithi083 ceph-mon[77242]: pgmap v17990: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:02.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:28:01 smithi083 ceph-mon[79930]: pgmap v17990: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:02.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:28:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:28:02.416Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:28:02.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:28:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:28:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:28:03.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:28:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:28:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:28:03.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:28:03 smithi083 ceph-mon[79930]: pgmap v17991: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:03.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:28:03 smithi083 ceph-mon[77242]: pgmap v17991: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:04.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:28:03 smithi153 ceph-mon[70044]: pgmap v17991: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:06.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:28:05 smithi153 ceph-mon[70044]: pgmap v17992: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:06.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:28:05 smithi083 ceph-mon[79930]: pgmap v17992: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:06.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:28:05 smithi083 ceph-mon[77242]: pgmap v17992: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:06.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:28:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:28:05.771Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:28:06.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:28:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:28:06.509Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:28:08.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:28:07 smithi153 ceph-mon[70044]: pgmap v17993: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:08.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:28:07 smithi083 ceph-mon[79930]: pgmap v17993: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:08.219 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:28:07 smithi083 ceph-mon[77242]: pgmap v17993: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:10.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:28:09 smithi153 ceph-mon[70044]: pgmap v17994: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:10.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:28:09 smithi083 ceph-mon[79930]: pgmap v17994: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:10.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:28:09 smithi083 ceph-mon[77242]: pgmap v17994: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:11.768 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:28:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:28:11] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T15:28:12.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:28:11 smithi153 ceph-mon[70044]: pgmap v17995: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:12.062 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:28:12.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:28:11 smithi083 ceph-mon[79930]: pgmap v17995: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:12.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:28:11 smithi083 ceph-mon[77242]: pgmap v17995: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:12.351 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:28:12.351 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 8m ago 10h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:28:12.351 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 6m ago 10h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:28:12.352 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 8m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:28:12.352 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 6m ago 10h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:28:12.352 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 6m ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:28:12.352 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 8m ago 10h 997M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:28:12.353 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 8m ago 10h 2159M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:28:12.353 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 6m ago 10h 942M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:28:12.353 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 8m ago 10h 1295M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:28:12.353 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 8m ago 10h 24.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:28:12.354 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 6m ago 10h 27.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:28:12.354 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 8m ago 10h 1568M 2148M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:28:12.354 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 8m ago 10h 1555M 2148M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:28:12.354 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 8m ago 10h 1208M 2148M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:28:12.355 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 8m ago 10h 1524M 2148M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:28:12.355 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 6m ago 10h 1336M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:28:12.355 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 6m ago 10h 1678M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:28:12.355 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 6m ago 10h 1500M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:28:12.355 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 6m ago 10h 1784M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:28:12.356 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 6m ago 10h 175M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:28:12.707 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:28:12.708 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:28:12.708 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:28:12.708 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:28:12.708 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:28:12.709 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:28:12.709 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:28:12.709 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:28:12.709 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:28:12.710 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:28:12.710 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:28:12.710 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:28:12.710 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:28:12.711 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:28:12.711 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:28:12.711 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:28:12.711 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:28:12.711 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:28:12.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:28:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:28:12.416Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:28:12.770 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:28:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:28:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:28:13.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:28:12 smithi153 ceph-mon[70044]: from='client.64987 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:28:13.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:28:12 smithi153 ceph-mon[70044]: from='client.64993 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:28:13.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:28:12 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1368868029' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:28:13.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:28:12 smithi083 ceph-mon[79930]: from='client.64987 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:28:13.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:28:12 smithi083 ceph-mon[79930]: from='client.64993 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:28:13.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:28:12 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1368868029' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:28:13.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:28:12 smithi083 ceph-mon[77242]: from='client.64987 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:28:13.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:28:12 smithi083 ceph-mon[77242]: from='client.64993 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:28:13.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:28:12 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1368868029' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:28:13.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:28:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:28:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:28:14.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:28:13 smithi153 ceph-mon[70044]: pgmap v17996: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:14.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:28:13 smithi083 ceph-mon[79930]: pgmap v17996: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:14.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:28:13 smithi083 ceph-mon[77242]: pgmap v17996: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:16.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:28:15 smithi153 ceph-mon[70044]: pgmap v17997: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:16.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:28:15 smithi083 ceph-mon[79930]: pgmap v17997: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:16.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:28:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:28:15.771Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:28:16.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:28:15 smithi083 ceph-mon[77242]: pgmap v17997: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:16.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:28:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:28:16.510Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:28:18.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:28:17 smithi153 ceph-mon[70044]: pgmap v17998: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:18.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:28:17 smithi083 ceph-mon[79930]: pgmap v17998: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:18.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:28:17 smithi083 ceph-mon[77242]: pgmap v17998: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:20.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:28:19 smithi153 ceph-mon[70044]: pgmap v17999: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:20.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:28:19 smithi083 ceph-mon[79930]: pgmap v17999: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:20.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:28:19 smithi083 ceph-mon[77242]: pgmap v17999: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:21.785 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:28:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:28:21] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T15:28:22.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:28:21 smithi153 ceph-mon[70044]: pgmap v18000: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:22.210 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:28:21 smithi083 ceph-mon[77242]: pgmap v18000: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:22.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:28:21 smithi083 ceph-mon[79930]: pgmap v18000: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:22.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:28:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:28:22.417Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:28:22.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:28:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:28:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:28:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:28:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:28:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:28:23.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:28:23 smithi083 ceph-mon[79930]: pgmap v18001: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:23.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:28:23 smithi083 ceph-mon[77242]: pgmap v18001: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:24.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:28:23 smithi153 ceph-mon[70044]: pgmap v18001: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:26.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:28:25 smithi153 ceph-mon[70044]: pgmap v18002: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:26.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:28:25 smithi083 ceph-mon[79930]: pgmap v18002: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:26.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:28:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:28:25.771Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:28:26.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:28:25 smithi083 ceph-mon[77242]: pgmap v18002: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:26.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:28:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:28:26.511Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:28:28.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:28:27 smithi153 ceph-mon[70044]: pgmap v18003: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:28.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:28:27 smithi083 ceph-mon[79930]: pgmap v18003: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:28.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:28:27 smithi083 ceph-mon[77242]: pgmap v18003: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:30.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:28:29 smithi153 ceph-mon[70044]: pgmap v18004: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:30.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:28:29 smithi083 ceph-mon[79930]: pgmap v18004: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:30.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:28:29 smithi083 ceph-mon[77242]: pgmap v18004: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:31.774 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:28:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:28:31] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T15:28:32.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:28:31 smithi153 ceph-mon[70044]: pgmap v18005: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:32.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:28:31 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:28:32.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:28:31 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:28:32.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:28:31 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:28:32.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:28:31 smithi083 ceph-mon[79930]: pgmap v18005: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:32.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:28:31 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:28:32.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:28:31 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:28:32.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:28:31 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:28:32.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:28:31 smithi083 ceph-mon[77242]: pgmap v18005: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:32.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:28:31 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:28:32.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:28:31 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:28:32.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:28:31 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:28:32.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:28:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:28:32.418Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:28:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:28:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:28:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:28:33.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:28:32 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:28:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:28:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:28:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:28:33.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:28:32 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:28:33.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:28:32 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:28:34.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:28:33 smithi153 ceph-mon[70044]: pgmap v18006: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:34.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:28:33 smithi083 ceph-mon[79930]: pgmap v18006: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:34.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:28:33 smithi083 ceph-mon[77242]: pgmap v18006: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:36.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:28:35 smithi083 ceph-mon[79930]: pgmap v18007: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:36.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:28:35 smithi083 ceph-mon[77242]: pgmap v18007: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:36.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:28:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:28:35.772Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:28:36.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:28:35 smithi153 ceph-mon[70044]: pgmap v18007: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:36.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:28:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:28:36.511Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:28:38.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:28:37 smithi153 ceph-mon[70044]: pgmap v18008: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:38.460 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:28:37 smithi083 ceph-mon[77242]: pgmap v18008: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:38.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:28:37 smithi083 ceph-mon[79930]: pgmap v18008: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:40.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:28:39 smithi153 ceph-mon[70044]: pgmap v18009: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:40.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:28:39 smithi083 ceph-mon[79930]: pgmap v18009: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:40.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:28:39 smithi083 ceph-mon[77242]: pgmap v18009: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:28:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:28:41] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T15:28:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:28:41 smithi153 ceph-mon[70044]: pgmap v18010: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:28:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:28:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:28:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:28:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:28:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:28:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:28:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:28:42.420 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:28:41 smithi083 ceph-mon[79930]: pgmap v18010: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:42.420 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:28:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:28:42.420 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:28:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:28:42.421 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:28:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:28:42.421 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:28:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:28:42.422 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:28:41 smithi083 ceph-mon[77242]: pgmap v18010: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:42.422 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:28:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:28:42.423 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:28:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:28:42.423 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:28:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:28:42.423 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:28:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:28:42.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:28:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:28:42.419Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:28:42.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:28:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:28:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:28:43.033 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:28:43.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:28:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:28:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:28:43.319 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:28:43.319 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 9m ago 10h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:28:43.319 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 7m ago 10h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:28:43.320 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 9m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:28:43.320 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 7m ago 10h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:28:43.320 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 7m ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:28:43.320 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (9h) 9m ago 10h 997M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:28:43.321 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 9m ago 10h 2159M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:28:43.321 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 7m ago 10h 942M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:28:43.321 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 9m ago 10h 1295M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:28:43.321 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 9m ago 10h 24.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:28:43.322 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 7m ago 10h 27.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:28:43.322 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 9m ago 10h 1568M 2148M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:28:43.322 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 9m ago 10h 1555M 2148M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:28:43.322 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 9m ago 10h 1208M 2148M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:28:43.322 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 9m ago 10h 1524M 2148M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:28:43.323 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 7m ago 10h 1336M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:28:43.323 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 7m ago 10h 1678M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:28:43.323 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 7m ago 10h 1500M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:28:43.323 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 7m ago 10h 1784M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:28:43.324 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (9h) 7m ago 10h 175M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:28:43.689 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:28:43.690 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:28:43.690 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:28:43.690 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:28:43.691 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:28:43.691 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:28:43.691 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:28:43.691 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:28:43.692 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:28:43.692 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:28:43.692 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:28:43.692 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:28:43.693 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:28:43.693 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:28:43.693 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:28:43.693 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:28:43.694 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:28:43.694 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:28:43.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:28:43 smithi083 ceph-mon[79930]: pgmap v18011: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:43.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:28:43 smithi083 ceph-mon[79930]: from='client.65005 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:28:43.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:28:43 smithi083 ceph-mon[79930]: from='client.55080 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:28:43.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:28:43 smithi083 ceph-mon[77242]: pgmap v18011: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:43.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:28:43 smithi083 ceph-mon[77242]: from='client.65005 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:28:43.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:28:43 smithi083 ceph-mon[77242]: from='client.55080 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:28:44.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:28:43 smithi153 ceph-mon[70044]: pgmap v18011: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:44.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:28:43 smithi153 ceph-mon[70044]: from='client.65005 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:28:44.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:28:43 smithi153 ceph-mon[70044]: from='client.55080 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:28:44.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:28:44 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4146668210' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:28:44.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:28:44 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4146668210' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:28:45.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:28:44 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4146668210' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:28:45.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:28:45 smithi083 ceph-mon[79930]: pgmap v18012: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:45.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:28:45 smithi083 ceph-mon[77242]: pgmap v18012: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:45.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:28:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:28:45.773Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:28:46.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:28:45 smithi153 ceph-mon[70044]: pgmap v18012: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:46.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:28:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:28:46.512Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:28:48.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:28:47 smithi153 ceph-mon[70044]: pgmap v18013: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:48.210 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:28:47 smithi083 ceph-mon[77242]: pgmap v18013: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:48.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:28:47 smithi083 ceph-mon[79930]: pgmap v18013: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:50.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:28:49 smithi153 ceph-mon[70044]: pgmap v18014: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:50.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:28:49 smithi083 ceph-mon[79930]: pgmap v18014: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:50.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:28:49 smithi083 ceph-mon[77242]: pgmap v18014: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:51.780 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:28:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:28:51] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T15:28:52.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:28:51 smithi153 ceph-mon[70044]: pgmap v18015: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:52.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:28:51 smithi083 ceph-mon[79930]: pgmap v18015: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:52.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:28:51 smithi083 ceph-mon[77242]: pgmap v18015: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:52.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:28:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:28:52.419Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:28:52.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:28:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:28:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:28:53.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:28:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:28:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:28:53.960 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:28:53 smithi083 ceph-mon[77242]: pgmap v18016: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:53.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:28:53 smithi083 ceph-mon[79930]: pgmap v18016: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:54.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:28:53 smithi153 ceph-mon[70044]: pgmap v18016: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:56.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:28:55 smithi153 ceph-mon[70044]: pgmap v18017: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:56.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:28:55 smithi083 ceph-mon[79930]: pgmap v18017: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:56.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:28:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:28:55.774Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:28:56.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:28:55 smithi083 ceph-mon[77242]: pgmap v18017: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:56.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:28:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:28:56.513Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:28:58.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:28:57 smithi153 ceph-mon[70044]: pgmap v18018: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:58.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:28:57 smithi083 ceph-mon[79930]: pgmap v18018: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:28:58.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:28:57 smithi083 ceph-mon[77242]: pgmap v18018: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:00.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:28:59 smithi153 ceph-mon[70044]: pgmap v18019: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:00.210 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:28:59 smithi083 ceph-mon[77242]: pgmap v18019: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:00.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:28:59 smithi083 ceph-mon[79930]: pgmap v18019: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:01.784 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:29:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:29:01] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T15:29:02.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:29:01 smithi153 ceph-mon[70044]: pgmap v18020: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:02.210 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:29:01 smithi083 ceph-mon[77242]: pgmap v18020: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:02.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:29:01 smithi083 ceph-mon[79930]: pgmap v18020: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:02.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:29:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:29:02.419Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:29:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:29:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:29:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:29:03.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:29:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:29:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:29:03.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:29:03 smithi083 ceph-mon[79930]: pgmap v18021: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:03.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:29:03 smithi083 ceph-mon[77242]: pgmap v18021: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:04.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:29:03 smithi153 ceph-mon[70044]: pgmap v18021: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:06.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:29:05 smithi153 ceph-mon[70044]: pgmap v18022: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:06.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:29:05 smithi083 ceph-mon[79930]: pgmap v18022: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:06.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:29:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:29:05.774Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:29:06.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:29:05 smithi083 ceph-mon[77242]: pgmap v18022: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:06.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:29:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:29:06.513Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:29:08.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:29:07 smithi153 ceph-mon[70044]: pgmap v18023: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:08.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:29:07 smithi083 ceph-mon[79930]: pgmap v18023: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:08.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:29:07 smithi083 ceph-mon[77242]: pgmap v18023: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:10.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:29:09 smithi153 ceph-mon[70044]: pgmap v18024: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:10.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:29:09 smithi083 ceph-mon[79930]: pgmap v18024: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:10.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:29:09 smithi083 ceph-mon[77242]: pgmap v18024: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:11.786 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:29:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:29:11] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T15:29:12.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:29:11 smithi153 ceph-mon[70044]: pgmap v18025: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:12.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:29:11 smithi083 ceph-mon[79930]: pgmap v18025: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:12.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:29:11 smithi083 ceph-mon[77242]: pgmap v18025: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:12.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:29:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:29:12.420Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:29:12.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:29:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:29:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:29:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:29:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:29:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:29:13.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:29:13 smithi083 ceph-mon[79930]: pgmap v18026: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:13.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:29:13 smithi083 ceph-mon[77242]: pgmap v18026: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:14.023 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:29:14.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:29:13 smithi153 ceph-mon[70044]: pgmap v18026: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:14.310 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:29:14.310 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 9m ago 10h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:29:14.310 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 7m ago 10h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:29:14.311 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 9m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:29:14.311 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 7m ago 10h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:29:14.311 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (9h) 7m ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:29:14.311 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 9m ago 10h 997M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:29:14.311 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (9h) 9m ago 10h 2159M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:29:14.312 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 7m ago 10h 942M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:29:14.312 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 9m ago 10h 1295M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:29:14.312 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 9m ago 10h 24.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:29:14.313 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 7m ago 10h 27.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:29:14.313 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 9m ago 10h 1568M 2148M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:29:14.313 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 9m ago 10h 1555M 2148M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:29:14.313 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 9m ago 10h 1208M 2148M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:29:14.313 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 9m ago 10h 1524M 2148M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:29:14.314 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 7m ago 10h 1336M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:29:14.314 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 7m ago 10h 1678M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:29:14.314 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 7m ago 10h 1500M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:29:14.315 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 7m ago 10h 1784M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:29:14.315 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 7m ago 10h 175M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:29:14.668 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:29:14.668 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:29:14.669 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:29:14.669 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:29:14.669 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:29:14.670 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:29:14.670 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:29:14.670 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:29:14.670 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:29:14.671 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:29:14.671 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:29:14.671 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:29:14.672 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:29:14.672 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:29:14.672 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:29:14.672 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:29:14.673 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:29:14.673 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:29:14.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:29:14 smithi083 ceph-mon[79930]: from='client.65023 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:29:14.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:29:14 smithi083 ceph-mon[79930]: from='client.55098 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:29:14.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:29:14 smithi083 ceph-mon[77242]: from='client.65023 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:29:14.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:29:14 smithi083 ceph-mon[77242]: from='client.55098 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:29:15.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:29:14 smithi153 ceph-mon[70044]: from='client.65023 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:29:15.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:29:14 smithi153 ceph-mon[70044]: from='client.55098 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:29:15.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:29:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:29:15.774Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:29:15.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:29:15 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1344193394' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:29:15.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:29:15 smithi083 ceph-mon[77242]: pgmap v18027: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:15.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:29:15 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1344193394' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:29:15.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:29:15 smithi083 ceph-mon[79930]: pgmap v18027: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:16.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:29:15 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1344193394' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:29:16.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:29:15 smithi153 ceph-mon[70044]: pgmap v18027: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:16.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:29:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:29:16.513Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:29:18.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:29:17 smithi153 ceph-mon[70044]: pgmap v18028: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:18.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:29:17 smithi083 ceph-mon[77242]: pgmap v18028: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:18.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:29:17 smithi083 ceph-mon[79930]: pgmap v18028: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:20.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:29:19 smithi153 ceph-mon[70044]: pgmap v18029: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:20.210 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:29:19 smithi083 ceph-mon[77242]: pgmap v18029: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:20.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:29:19 smithi083 ceph-mon[79930]: pgmap v18029: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:21.786 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:29:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:29:21] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T15:29:22.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:29:21 smithi153 ceph-mon[70044]: pgmap v18030: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:22.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:29:21 smithi083 ceph-mon[77242]: pgmap v18030: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:22.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:29:21 smithi083 ceph-mon[79930]: pgmap v18030: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:22.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:29:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:29:22.421Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:29:22.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:29:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:29:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:29:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:29:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:29:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:29:23.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:29:23 smithi083 ceph-mon[77242]: pgmap v18031: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:23.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:29:23 smithi083 ceph-mon[79930]: pgmap v18031: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:24.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:29:23 smithi153 ceph-mon[70044]: pgmap v18031: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:26.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:29:25 smithi153 ceph-mon[70044]: pgmap v18032: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:26.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:29:25 smithi083 ceph-mon[79930]: pgmap v18032: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:26.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:29:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:29:25.775Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:29:26.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:29:25 smithi083 ceph-mon[77242]: pgmap v18032: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:26.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:29:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:29:26.514Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:29:28.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:29:27 smithi153 ceph-mon[70044]: pgmap v18033: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:28.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:29:27 smithi083 ceph-mon[79930]: pgmap v18033: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:28.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:29:27 smithi083 ceph-mon[77242]: pgmap v18033: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:30.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:29:29 smithi153 ceph-mon[70044]: pgmap v18034: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:30.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:29:29 smithi083 ceph-mon[79930]: pgmap v18034: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:30.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:29:29 smithi083 ceph-mon[77242]: pgmap v18034: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:31.792 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:29:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:29:31] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T15:29:32.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:29:31 smithi153 ceph-mon[70044]: pgmap v18035: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:32.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:29:31 smithi083 ceph-mon[79930]: pgmap v18035: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:32.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:29:31 smithi083 ceph-mon[77242]: pgmap v18035: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:32.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:29:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:29:32.420Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:29:32.794 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:29:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:29:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:29:33.060 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:29:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:29:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:29:33.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:29:32 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:29:33.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:29:32 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:29:33.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:29:32 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:29:33.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:29:32 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:29:33.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:29:32 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:29:33.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:29:32 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:29:33.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:29:32 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:29:33.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:29:32 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:29:33.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:29:32 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:29:33.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:29:32 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:29:33.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:29:32 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:29:33.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:29:32 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:29:33.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:29:32 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:29:33.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:29:32 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:29:33.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:29:32 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:29:33.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:29:32 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:29:33.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:29:32 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:29:33.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:29:32 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:29:33.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:29:32 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:29:33.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:29:32 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:29:33.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:29:32 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:29:34.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:29:33 smithi153 ceph-mon[70044]: pgmap v18036: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:34.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:29:33 smithi083 ceph-mon[79930]: pgmap v18036: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:34.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:29:33 smithi083 ceph-mon[77242]: pgmap v18036: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:36.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:29:35 smithi153 ceph-mon[70044]: pgmap v18037: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:36.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:29:35 smithi083 ceph-mon[79930]: pgmap v18037: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:36.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:29:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:29:35.775Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:29:36.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:29:35 smithi083 ceph-mon[77242]: pgmap v18037: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:36.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:29:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:29:36.515Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:29:37.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:29:37 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:29:37.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:29:37 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:29:37.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:29:37 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:29:37.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:29:37 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:29:37.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:29:37 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:29:37.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:29:37 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:29:37.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:29:37 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:29:37.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:29:37 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:29:37.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:29:37 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:29:38.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:29:38 smithi153 ceph-mon[70044]: pgmap v18038: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:38.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:29:38 smithi083 ceph-mon[79930]: pgmap v18038: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:38.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:29:38 smithi083 ceph-mon[77242]: pgmap v18038: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:40.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:29:40 smithi153 ceph-mon[70044]: pgmap v18039: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:40.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:29:40 smithi083 ceph-mon[79930]: pgmap v18039: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:40.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:29:40 smithi083 ceph-mon[77242]: pgmap v18039: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:41.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:29:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:29:41] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T15:29:42.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:29:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:29:42.421Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:29:42.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:29:42 smithi083 ceph-mon[79930]: pgmap v18040: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:42.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:29:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:29:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:29:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:29:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:29:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:29:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:29:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:29:42.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:29:42 smithi083 ceph-mon[77242]: pgmap v18040: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:29:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:29:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:29:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:29:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:29:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:29:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:29:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:29:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:29:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:29:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:29:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:29:42 smithi153 ceph-mon[70044]: pgmap v18040: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:29:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:29:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:29:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:29:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:29:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:29:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:29:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:29:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:29:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:29:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:29:43.960 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:29:43 smithi083 ceph-mon[77242]: pgmap v18041: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:43.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:29:43 smithi083 ceph-mon[79930]: pgmap v18041: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:44.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:29:43 smithi153 ceph-mon[70044]: pgmap v18041: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:45.000 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:29:45.290 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:29:45.290 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 8s ago 10h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:29:45.290 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 8m ago 10h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:29:45.291 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 8s ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:29:45.291 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 8m ago 10h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:29:45.291 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 8m ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:29:45.291 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 8s ago 10h 1029M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:29:45.292 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 8s ago 10h 2196M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:29:45.292 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (9h) 8m ago 10h 942M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:29:45.292 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9h) 8s ago 10h 1319M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:29:45.292 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 8s ago 10h 24.3M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:29:45.292 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 8m ago 10h 27.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:29:45.293 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 8s ago 10h 1595M 2148M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:29:45.293 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 8s ago 10h 1581M 2148M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:29:45.293 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 8s ago 10h 1228M 2148M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:29:45.293 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 8s ago 10h 1550M 2148M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:29:45.294 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 8m ago 10h 1336M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:29:45.294 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 8m ago 10h 1678M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:29:45.294 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 8m ago 10h 1500M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:29:45.294 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 8m ago 10h 1784M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:29:45.295 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 8m ago 10h 175M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:29:45.648 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:29:45.648 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:29:45.648 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:29:45.649 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:29:45.649 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:29:45.649 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:29:45.649 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:29:45.650 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:29:45.650 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:29:45.650 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:29:45.650 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:29:45.651 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:29:45.651 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:29:45.651 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:29:45.651 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:29:45.652 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:29:45.652 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:29:45.652 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:29:46.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:29:45 smithi153 ceph-mon[70044]: pgmap v18042: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:46.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:29:45 smithi153 ceph-mon[70044]: from='client.65041 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:29:46.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:29:45 smithi153 ceph-mon[70044]: from='client.65047 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:29:46.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:29:45 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3279303021' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:29:46.210 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:29:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:29:45.775Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:29:46.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:29:45 smithi083 ceph-mon[77242]: pgmap v18042: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:46.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:29:45 smithi083 ceph-mon[77242]: from='client.65041 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:29:46.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:29:45 smithi083 ceph-mon[77242]: from='client.65047 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:29:46.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:29:45 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3279303021' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:29:46.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:29:45 smithi083 ceph-mon[79930]: pgmap v18042: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:46.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:29:45 smithi083 ceph-mon[79930]: from='client.65041 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:29:46.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:29:45 smithi083 ceph-mon[79930]: from='client.65047 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:29:46.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:29:45 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3279303021' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:29:46.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:29:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:29:46.515Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:29:48.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:29:47 smithi153 ceph-mon[70044]: pgmap v18043: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:48.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:29:47 smithi083 ceph-mon[79930]: pgmap v18043: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:48.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:29:47 smithi083 ceph-mon[77242]: pgmap v18043: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:50.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:29:49 smithi153 ceph-mon[70044]: pgmap v18044: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:50.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:29:49 smithi083 ceph-mon[79930]: pgmap v18044: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:50.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:29:49 smithi083 ceph-mon[77242]: pgmap v18044: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:51.798 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:29:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:29:51] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T15:29:52.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:29:51 smithi153 ceph-mon[70044]: pgmap v18045: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:52.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:29:51 smithi083 ceph-mon[79930]: pgmap v18045: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:52.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:29:51 smithi083 ceph-mon[77242]: pgmap v18045: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:52.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:29:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:29:52.422Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:29:52.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:29:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:29:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:29:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:29:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:29:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:29:53.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:29:53 smithi083 ceph-mon[79930]: pgmap v18046: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:53.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:29:53 smithi083 ceph-mon[77242]: pgmap v18046: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:54.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:29:53 smithi153 ceph-mon[70044]: pgmap v18046: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:56.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:29:55 smithi153 ceph-mon[70044]: pgmap v18047: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:56.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:29:55 smithi083 ceph-mon[79930]: pgmap v18047: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:56.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:29:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:29:55.777Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:29:56.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:29:55 smithi083 ceph-mon[77242]: pgmap v18047: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:56.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:29:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:29:56.516Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:29:58.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:29:57 smithi153 ceph-mon[70044]: pgmap v18048: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:58.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:29:57 smithi083 ceph-mon[79930]: pgmap v18048: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:29:58.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:29:57 smithi083 ceph-mon[77242]: pgmap v18048: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:00.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:29:59 smithi153 ceph-mon[70044]: pgmap v18049: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:00.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:29:59 smithi083 ceph-mon[79930]: pgmap v18049: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:00.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:29:59 smithi083 ceph-mon[77242]: pgmap v18049: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:00.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:30:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[77219]: 2022-09-29T15:29:59.999+0000 7f160f665700 -1 log_channel(cluster) log [ERR] : overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T15:30:01.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:30:00 smithi083 ceph-mon[79930]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T15:30:01.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:30:00 smithi083 ceph-mon[77242]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T15:30:01.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:30:00 smithi153 ceph-mon[70044]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T15:30:01.806 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:30:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:30:01] "GET /metrics HTTP/1.1" 200 225247 "" "Prometheus/2.33.4" 2022-09-29T15:30:02.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:30:01 smithi083 ceph-mon[79930]: pgmap v18050: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:02.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:30:01 smithi083 ceph-mon[77242]: pgmap v18050: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:02.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:30:01 smithi153 ceph-mon[70044]: pgmap v18050: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:02.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:30:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:30:02.422Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:30:02.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:30:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:30:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:30:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:30:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:30:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:30:03.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:30:03 smithi083 ceph-mon[79930]: pgmap v18051: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:03.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:30:03 smithi083 ceph-mon[77242]: pgmap v18051: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:04.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:30:03 smithi153 ceph-mon[70044]: pgmap v18051: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:06.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:30:05 smithi153 ceph-mon[70044]: pgmap v18052: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:06.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:30:05 smithi083 ceph-mon[77242]: pgmap v18052: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:06.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:30:05 smithi083 ceph-mon[79930]: pgmap v18052: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:06.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:30:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:30:05.777Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:30:06.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:30:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:30:06.517Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:30:08.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:30:07 smithi153 ceph-mon[70044]: pgmap v18053: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:08.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:30:07 smithi083 ceph-mon[79930]: pgmap v18053: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:08.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:30:07 smithi083 ceph-mon[77242]: pgmap v18053: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:10.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:30:09 smithi153 ceph-mon[70044]: pgmap v18054: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:10.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:30:09 smithi083 ceph-mon[79930]: pgmap v18054: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:10.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:30:09 smithi083 ceph-mon[77242]: pgmap v18054: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:11.806 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:30:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:30:11] "GET /metrics HTTP/1.1" 200 225246 "" "Prometheus/2.33.4" 2022-09-29T15:30:12.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:30:11 smithi153 ceph-mon[70044]: pgmap v18055: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:12.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:30:11 smithi083 ceph-mon[79930]: pgmap v18055: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:12.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:30:11 smithi083 ceph-mon[77242]: pgmap v18055: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:12.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:30:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:30:12.423Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:30:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:30:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:30:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:30:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:30:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:30:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:30:13.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:30:13 smithi083 ceph-mon[79930]: pgmap v18056: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:13.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:30:13 smithi083 ceph-mon[77242]: pgmap v18056: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:14.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:30:13 smithi153 ceph-mon[70044]: pgmap v18056: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:15.971 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:30:16.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:30:15 smithi153 ceph-mon[70044]: pgmap v18057: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:16.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:30:15 smithi083 ceph-mon[77242]: pgmap v18057: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:16.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:30:15 smithi083 ceph-mon[79930]: pgmap v18057: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:16.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:30:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:30:15.776Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:30:16.262 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:30:16.263 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 39s ago 10h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:30:16.263 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 8m ago 10h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:30:16.264 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 39s ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:30:16.264 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 8m ago 10h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:30:16.264 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 8m ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:30:16.265 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 39s ago 10h 1029M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:30:16.265 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 39s ago 10h 2196M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:30:16.266 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 8m ago 10h 942M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:30:16.266 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 39s ago 10h 1319M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:30:16.266 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 39s ago 10h 24.3M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:30:16.267 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 8m ago 10h 27.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:30:16.267 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 39s ago 10h 1595M 2148M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:30:16.267 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 39s ago 10h 1581M 2148M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:30:16.267 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 39s ago 10h 1228M 2148M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:30:16.268 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 39s ago 10h 1550M 2148M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:30:16.268 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 8m ago 10h 1336M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:30:16.269 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 8m ago 10h 1678M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:30:16.269 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 8m ago 10h 1500M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:30:16.269 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 8m ago 10h 1784M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:30:16.270 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 8m ago 10h 175M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:30:16.619 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:30:16.619 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:30:16.619 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:30:16.620 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:30:16.620 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:30:16.620 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:30:16.621 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:30:16.621 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:30:16.621 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:30:16.621 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:30:16.622 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:30:16.622 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:30:16.622 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:30:16.622 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:30:16.623 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:30:16.623 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:30:16.623 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:30:16.623 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:30:16.804 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:30:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:30:16.517Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:30:17.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:30:16 smithi153 ceph-mon[70044]: from='client.55128 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:30:17.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:30:16 smithi153 ceph-mon[70044]: from='client.55134 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:30:17.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:30:16 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2672720911' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:30:17.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:30:16 smithi083 ceph-mon[79930]: from='client.55128 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:30:17.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:30:16 smithi083 ceph-mon[79930]: from='client.55134 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:30:17.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:30:16 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2672720911' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:30:17.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:30:16 smithi083 ceph-mon[77242]: from='client.55128 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:30:17.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:30:16 smithi083 ceph-mon[77242]: from='client.55134 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:30:17.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:30:16 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2672720911' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:30:18.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:30:17 smithi083 ceph-mon[79930]: pgmap v18058: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:18.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:30:17 smithi083 ceph-mon[77242]: pgmap v18058: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:18.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:30:17 smithi153 ceph-mon[70044]: pgmap v18058: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:20.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:30:19 smithi083 ceph-mon[77242]: pgmap v18059: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:20.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:30:19 smithi083 ceph-mon[79930]: pgmap v18059: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:30:19 smithi153 ceph-mon[70044]: pgmap v18059: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:21.814 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:30:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:30:21] "GET /metrics HTTP/1.1" 200 225246 "" "Prometheus/2.33.4" 2022-09-29T15:30:22.210 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:30:21 smithi083 ceph-mon[77242]: pgmap v18060: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:22.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:30:21 smithi083 ceph-mon[79930]: pgmap v18060: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:22.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:30:21 smithi153 ceph-mon[70044]: pgmap v18060: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:22.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:30:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:30:22.424Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:30:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:30:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:30:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:30:23.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:30:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:30:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:30:23.960 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:30:23 smithi083 ceph-mon[77242]: pgmap v18061: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:23.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:30:23 smithi083 ceph-mon[79930]: pgmap v18061: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:24.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:30:23 smithi153 ceph-mon[70044]: pgmap v18061: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:26.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:30:25 smithi153 ceph-mon[70044]: pgmap v18062: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:26.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:30:25 smithi083 ceph-mon[79930]: pgmap v18062: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:26.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:30:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:30:25.778Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:30:26.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:30:25 smithi083 ceph-mon[77242]: pgmap v18062: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:26.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:30:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:30:26.518Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:30:28.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:30:27 smithi153 ceph-mon[70044]: pgmap v18063: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:28.210 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:30:27 smithi083 ceph-mon[77242]: pgmap v18063: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:28.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:30:27 smithi083 ceph-mon[79930]: pgmap v18063: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:30.210 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:30:29 smithi083 ceph-mon[77242]: pgmap v18064: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:30.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:30:29 smithi083 ceph-mon[79930]: pgmap v18064: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:30.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:30:29 smithi153 ceph-mon[70044]: pgmap v18064: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:31.811 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:30:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:30:31] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T15:30:31.812 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:30:31 smithi083 ceph-mon[77242]: pgmap v18065: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:32.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:30:31 smithi083 ceph-mon[79930]: pgmap v18065: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:32.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:30:31 smithi153 ceph-mon[70044]: pgmap v18065: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:32.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:30:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:30:32.424Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:30:32.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:30:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:30:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:30:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:30:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:30:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:30:33.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:30:33 smithi083 ceph-mon[79930]: pgmap v18066: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:33.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:30:33 smithi083 ceph-mon[77242]: pgmap v18066: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:34.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:30:33 smithi153 ceph-mon[70044]: pgmap v18066: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:36.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:30:35 smithi153 ceph-mon[70044]: pgmap v18067: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:36.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:30:35 smithi083 ceph-mon[77242]: pgmap v18067: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:36.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:30:35 smithi083 ceph-mon[79930]: pgmap v18067: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:36.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:30:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:30:35.779Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:30:36.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:30:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:30:36.518Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:30:38.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:30:37 smithi083 ceph-mon[79930]: pgmap v18068: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:38.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:30:37 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:30:38.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:30:37 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:30:38.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:30:37 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:30:38.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:30:37 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:30:38.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:30:37 smithi083 ceph-mon[77242]: pgmap v18068: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:38.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:30:37 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:30:38.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:30:37 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:30:38.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:30:37 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:30:38.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:30:37 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:30:38.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:30:37 smithi153 ceph-mon[70044]: pgmap v18068: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:38.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:30:37 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:30:38.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:30:37 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:30:38.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:30:37 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:30:38.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:30:37 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:30:40.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:30:39 smithi083 ceph-mon[79930]: pgmap v18069: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:40.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:30:39 smithi083 ceph-mon[77242]: pgmap v18069: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:40.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:30:39 smithi153 ceph-mon[70044]: pgmap v18069: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:41.815 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:30:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:30:41] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T15:30:42.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:30:41 smithi083 ceph-mon[79930]: pgmap v18070: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:42.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:30:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:30:42.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:30:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:30:42.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:30:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:30:42.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:30:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:30:42.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:30:41 smithi083 ceph-mon[77242]: pgmap v18070: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:42.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:30:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:30:42.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:30:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:30:42.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:30:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:30:42.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:30:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:30:42.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:30:41 smithi153 ceph-mon[70044]: pgmap v18070: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:30:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:30:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:30:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:30:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:30:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:30:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:30:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:30:42.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:30:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:30:42.425Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:30:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:30:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:30:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:30:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:30:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:30:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:30:43.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:30:43 smithi083 ceph-mon[79930]: pgmap v18071: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:43.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:30:43 smithi083 ceph-mon[77242]: pgmap v18071: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:44.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:30:43 smithi153 ceph-mon[70044]: pgmap v18071: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:46.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:30:45 smithi083 ceph-mon[77242]: pgmap v18072: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:46.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:30:45 smithi083 ceph-mon[79930]: pgmap v18072: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:46.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:30:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:30:45.780Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:30:46.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:30:45 smithi153 ceph-mon[70044]: pgmap v18072: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:46.947 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:30:46.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:30:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:30:46.519Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:30:47.239 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:30:47.240 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 70s ago 10h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:30:47.240 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 9m ago 10h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:30:47.240 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 70s ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:30:47.240 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 9m ago 10h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:30:47.241 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 9m ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:30:47.241 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 70s ago 10h 1029M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:30:47.241 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 70s ago 10h 2196M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:30:47.241 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 9m ago 10h 942M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:30:47.242 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 70s ago 10h 1319M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:30:47.242 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 70s ago 10h 24.3M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:30:47.242 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 9m ago 10h 27.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:30:47.242 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9h) 70s ago 10h 1595M 2148M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:30:47.242 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 70s ago 10h 1581M 2148M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:30:47.243 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 70s ago 10h 1228M 2148M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:30:47.243 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 70s ago 10h 1550M 2148M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:30:47.243 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 9m ago 10h 1336M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:30:47.243 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 9m ago 10h 1678M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:30:47.244 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 9m ago 10h 1500M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:30:47.244 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 9m ago 10h 1784M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:30:47.244 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 9m ago 10h 175M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:30:47.597 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:30:47.597 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:30:47.597 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:30:47.598 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:30:47.598 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:30:47.598 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:30:47.599 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:30:47.599 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:30:47.599 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:30:47.599 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:30:47.600 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:30:47.600 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:30:47.600 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:30:47.600 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:30:47.601 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:30:47.601 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:30:47.601 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:30:47.601 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:30:48.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:30:47 smithi083 ceph-mon[79930]: pgmap v18073: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:48.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:30:47 smithi083 ceph-mon[79930]: from='client.55146 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:30:48.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:30:47 smithi083 ceph-mon[79930]: from='client.55152 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:30:48.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:30:47 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3547023684' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:30:48.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:30:47 smithi083 ceph-mon[77242]: pgmap v18073: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:48.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:30:47 smithi083 ceph-mon[77242]: from='client.55146 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:30:48.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:30:47 smithi083 ceph-mon[77242]: from='client.55152 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:30:48.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:30:47 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3547023684' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:30:48.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:30:47 smithi153 ceph-mon[70044]: pgmap v18073: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:48.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:30:47 smithi153 ceph-mon[70044]: from='client.55146 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:30:48.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:30:47 smithi153 ceph-mon[70044]: from='client.55152 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:30:48.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:30:47 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3547023684' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:30:50.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:30:49 smithi083 ceph-mon[79930]: pgmap v18074: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:50.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:30:49 smithi083 ceph-mon[77242]: pgmap v18074: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:50.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:30:49 smithi153 ceph-mon[70044]: pgmap v18074: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:51.819 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:30:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:30:51] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T15:30:52.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:30:51 smithi083 ceph-mon[77242]: pgmap v18075: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:52.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:30:51 smithi083 ceph-mon[79930]: pgmap v18075: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:52.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:30:51 smithi153 ceph-mon[70044]: pgmap v18075: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:52.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:30:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:30:52.426Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:30:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:30:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:30:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:30:53.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:30:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:30:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:30:53.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:30:53 smithi083 ceph-mon[79930]: pgmap v18076: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:53.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:30:53 smithi083 ceph-mon[77242]: pgmap v18076: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:54.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:30:53 smithi153 ceph-mon[70044]: pgmap v18076: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:56.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:30:55 smithi083 ceph-mon[79930]: pgmap v18077: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:56.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:30:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:30:55.781Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:30:56.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:30:55 smithi083 ceph-mon[77242]: pgmap v18077: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:56.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:30:55 smithi153 ceph-mon[70044]: pgmap v18077: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:56.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:30:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:30:56.521Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:30:58.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:30:57 smithi083 ceph-mon[79930]: pgmap v18078: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:58.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:30:57 smithi083 ceph-mon[77242]: pgmap v18078: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:30:58.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:30:57 smithi153 ceph-mon[70044]: pgmap v18078: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:00.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:30:59 smithi083 ceph-mon[79930]: pgmap v18079: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:00.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:30:59 smithi083 ceph-mon[77242]: pgmap v18079: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:30:59 smithi153 ceph-mon[70044]: pgmap v18079: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:01.827 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:31:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:31:01] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T15:31:02.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:31:01 smithi083 ceph-mon[79930]: pgmap v18080: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:02.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:31:01 smithi083 ceph-mon[77242]: pgmap v18080: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:02.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:31:01 smithi153 ceph-mon[70044]: pgmap v18080: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:02.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:31:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:31:02.427Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:31:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:31:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:31:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:31:03.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:31:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:31:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:31:03.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:31:03 smithi083 ceph-mon[77242]: pgmap v18081: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:03.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:31:03 smithi083 ceph-mon[79930]: pgmap v18081: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:04.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:31:03 smithi153 ceph-mon[70044]: pgmap v18081: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:06.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:31:05 smithi083 ceph-mon[79930]: pgmap v18082: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:06.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:31:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:31:05.782Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:31:06.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:31:05 smithi083 ceph-mon[77242]: pgmap v18082: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:06.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:31:05 smithi153 ceph-mon[70044]: pgmap v18082: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:06.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:31:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:31:06.521Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:31:08.210 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:31:07 smithi083 ceph-mon[77242]: pgmap v18083: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:08.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:31:07 smithi083 ceph-mon[79930]: pgmap v18083: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:08.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:31:07 smithi153 ceph-mon[70044]: pgmap v18083: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:10.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:31:09 smithi083 ceph-mon[79930]: pgmap v18084: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:10.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:31:09 smithi083 ceph-mon[77242]: pgmap v18084: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:10.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:31:09 smithi153 ceph-mon[70044]: pgmap v18084: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:11.825 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:31:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:31:11] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T15:31:12.210 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:31:11 smithi083 ceph-mon[77242]: pgmap v18085: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:12.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:31:11 smithi083 ceph-mon[79930]: pgmap v18085: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:12.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:31:11 smithi153 ceph-mon[70044]: pgmap v18085: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:12.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:31:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:31:12.427Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:31:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:31:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:31:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:31:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:31:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:31:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:31:13.960 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:31:13 smithi083 ceph-mon[77242]: pgmap v18086: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:13.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:31:13 smithi083 ceph-mon[79930]: pgmap v18086: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:14.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:31:13 smithi153 ceph-mon[70044]: pgmap v18086: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:16.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:31:15 smithi083 ceph-mon[79930]: pgmap v18087: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:16.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:31:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:31:15.783Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:31:16.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:31:15 smithi083 ceph-mon[77242]: pgmap v18087: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:16.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:31:15 smithi153 ceph-mon[70044]: pgmap v18087: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:16.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:31:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:31:16.522Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:31:17.929 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:31:18.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:31:17 smithi083 ceph-mon[79930]: pgmap v18088: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:18.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:31:17 smithi083 ceph-mon[77242]: pgmap v18088: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:18.220 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:31:18.221 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 101s ago 10h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:31:18.221 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 9m ago 10h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:31:18.222 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 101s ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:31:18.222 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 9m ago 10h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:31:18.222 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 9m ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:31:18.223 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 101s ago 10h 1029M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:31:18.223 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 101s ago 10h 2196M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:31:18.224 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 9m ago 10h 942M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:31:18.224 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 101s ago 10h 1319M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:31:18.225 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 101s ago 10h 24.3M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:31:18.225 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 9m ago 10h 27.4M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:31:18.226 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 101s ago 10h 1595M 2148M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:31:18.226 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9h) 101s ago 10h 1581M 2148M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:31:18.227 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9h) 101s ago 10h 1228M 2148M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:31:18.227 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 101s ago 10h 1550M 2148M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:31:18.228 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 9m ago 10h 1336M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:31:18.228 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 9m ago 10h 1678M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:31:18.228 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 9m ago 10h 1500M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:31:18.229 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 9m ago 10h 1784M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:31:18.229 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 9m ago 10h 175M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:31:18.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:31:17 smithi153 ceph-mon[70044]: pgmap v18088: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:18.578 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:31:18.579 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:31:18.579 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:31:18.579 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:31:18.579 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:31:18.580 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:31:18.580 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:31:18.580 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:31:18.580 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:31:18.580 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:31:18.581 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:31:18.581 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:31:18.581 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:31:18.581 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:31:18.582 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:31:18.582 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:31:18.582 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:31:18.582 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:31:19.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:31:18 smithi083 ceph-mon[79930]: from='client.55164 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:31:19.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:31:18 smithi083 ceph-mon[79930]: from='client.55170 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:31:19.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:31:18 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3047949215' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:31:19.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:31:18 smithi083 ceph-mon[77242]: from='client.55164 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:31:19.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:31:18 smithi083 ceph-mon[77242]: from='client.55170 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:31:19.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:31:18 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3047949215' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:31:19.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:31:18 smithi153 ceph-mon[70044]: from='client.55164 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:31:19.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:31:18 smithi153 ceph-mon[70044]: from='client.55170 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:31:19.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:31:18 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3047949215' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:31:20.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:31:19 smithi083 ceph-mon[79930]: pgmap v18089: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:20.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:31:19 smithi083 ceph-mon[77242]: pgmap v18089: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:20.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:31:19 smithi153 ceph-mon[70044]: pgmap v18089: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:21.836 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:31:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:31:21] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T15:31:22.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:31:21 smithi083 ceph-mon[77242]: pgmap v18090: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:22.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:31:21 smithi083 ceph-mon[79930]: pgmap v18090: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:22.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:31:21 smithi153 ceph-mon[70044]: pgmap v18090: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:22.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:31:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:31:22.428Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:31:22.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:31:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:31:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:31:23.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:31:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:31:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:31:23.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:31:23 smithi083 ceph-mon[79930]: pgmap v18091: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:23.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:31:23 smithi083 ceph-mon[77242]: pgmap v18091: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:24.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:31:23 smithi153 ceph-mon[70044]: pgmap v18091: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:26.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:31:25 smithi083 ceph-mon[79930]: pgmap v18092: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:26.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:31:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:31:25.784Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:31:26.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:31:25 smithi083 ceph-mon[77242]: pgmap v18092: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:26.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:31:25 smithi153 ceph-mon[70044]: pgmap v18092: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:26.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:31:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:31:26.522Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:31:28.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:31:27 smithi083 ceph-mon[79930]: pgmap v18093: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:28.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:31:27 smithi083 ceph-mon[77242]: pgmap v18093: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:28.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:31:27 smithi153 ceph-mon[70044]: pgmap v18093: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:30.210 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:31:29 smithi083 ceph-mon[77242]: pgmap v18094: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:30.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:31:29 smithi083 ceph-mon[79930]: pgmap v18094: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:30.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:31:29 smithi153 ceph-mon[70044]: pgmap v18094: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:31.831 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:31:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:31:31] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T15:31:32.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:31:31 smithi083 ceph-mon[77242]: pgmap v18095: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:32.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:31:31 smithi083 ceph-mon[79930]: pgmap v18095: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:32.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:31:31 smithi153 ceph-mon[70044]: pgmap v18095: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:32.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:31:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:31:32.428Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:31:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:31:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:31:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:31:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:31:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:31:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:31:33.960 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:31:33 smithi083 ceph-mon[77242]: pgmap v18096: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:33.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:31:33 smithi083 ceph-mon[79930]: pgmap v18096: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:34.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:31:33 smithi153 ceph-mon[70044]: pgmap v18096: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:36.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:31:35 smithi083 ceph-mon[77242]: pgmap v18097: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:36.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:31:35 smithi083 ceph-mon[79930]: pgmap v18097: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:36.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:31:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:31:35.785Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:31:36.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:31:35 smithi153 ceph-mon[70044]: pgmap v18097: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:36.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:31:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:31:36.523Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:31:38.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:31:37 smithi083 ceph-mon[79930]: pgmap v18098: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:38.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:31:37 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:31:38.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:31:37 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:31:38.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:31:37 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:31:38.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:31:37 smithi083 ceph-mon[77242]: pgmap v18098: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:38.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:31:37 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:31:38.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:31:37 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:31:38.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:31:37 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:31:38.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:31:37 smithi153 ceph-mon[70044]: pgmap v18098: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:38.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:31:37 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:31:38.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:31:37 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:31:38.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:31:37 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:31:40.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:31:39 smithi083 ceph-mon[79930]: pgmap v18099: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:40.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:31:39 smithi083 ceph-mon[77242]: pgmap v18099: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:40.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:31:39 smithi153 ceph-mon[70044]: pgmap v18099: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:41.832 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:31:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:31:41] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T15:31:42.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:31:41 smithi083 ceph-mon[79930]: pgmap v18100: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:42.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:31:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:31:42.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:31:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:31:42.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:31:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:31:42.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:31:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:31:42.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:31:41 smithi083 ceph-mon[77242]: pgmap v18100: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:42.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:31:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:31:42.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:31:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:31:42.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:31:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:31:42.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:31:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:31:42.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:31:41 smithi153 ceph-mon[70044]: pgmap v18100: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:31:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:31:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:31:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:31:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:31:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:31:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:31:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:31:42.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:31:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:31:42.429Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:31:42.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:31:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:31:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:31:43.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:31:43 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:31:43.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:31:43 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:31:43.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:31:43 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:31:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:31:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:31:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:31:43.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:31:43 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:31:43.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:31:43 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:31:43.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:31:43 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:31:43.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:31:43 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:31:43.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:31:43 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:31:43.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:31:43 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:31:44.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:31:44 smithi153 ceph-mon[70044]: pgmap v18101: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:44.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:31:44 smithi083 ceph-mon[79930]: pgmap v18101: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:44.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:31:44 smithi083 ceph-mon[77242]: pgmap v18101: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:46.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:31:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:31:45.786Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:31:46.524 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:31:46 smithi083 ceph-mon[79930]: pgmap v18102: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:46.525 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:31:46 smithi083 ceph-mon[77242]: pgmap v18102: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:46.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:31:46 smithi153 ceph-mon[70044]: pgmap v18102: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:46.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:31:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:31:46.524Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:31:48.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:31:48 smithi153 ceph-mon[70044]: pgmap v18103: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:48.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:31:48 smithi083 ceph-mon[79930]: pgmap v18103: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:48.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:31:48 smithi083 ceph-mon[77242]: pgmap v18103: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:48.906 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:31:49.193 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:31:49.193 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 2m ago 10h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:31:49.194 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 6s ago 10h 63.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:31:49.194 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 2m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:31:49.194 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 6s ago 10h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:31:49.194 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 6s ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:31:49.195 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 2m ago 10h 1029M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:31:49.195 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 2m ago 10h 2196M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:31:49.195 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 6s ago 10h 957M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:31:49.195 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 2m ago 10h 1319M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:31:49.196 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 2m ago 10h 24.3M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:31:49.196 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 6s ago 10h 29.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:31:49.196 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 2m ago 10h 1595M 2148M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:31:49.196 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 2m ago 10h 1581M 2148M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:31:49.197 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 2m ago 10h 1228M 2148M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:31:49.197 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9h) 2m ago 10h 1550M 2148M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:31:49.197 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 6s ago 10h 1358M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:31:49.197 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 6s ago 10h 1707M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:31:49.198 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 6s ago 10h 1524M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:31:49.198 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 6s ago 10h 1814M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:31:49.199 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 6s ago 10h 177M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:31:49.555 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:31:49.556 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:31:49.556 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:31:49.556 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:31:49.557 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:31:49.557 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:31:49.557 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:31:49.557 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:31:49.558 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:31:49.558 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:31:49.558 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:31:49.558 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:31:49.559 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:31:49.559 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:31:49.559 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:31:49.559 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:31:49.559 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:31:49.560 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:31:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:31:50 smithi153 ceph-mon[70044]: pgmap v18104: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:31:50 smithi153 ceph-mon[70044]: from='client.65113 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:31:50.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:31:50 smithi153 ceph-mon[70044]: from='client.65119 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:31:50.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:31:50 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/737946751' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:31:50.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:31:50 smithi083 ceph-mon[79930]: pgmap v18104: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:50.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:31:50 smithi083 ceph-mon[79930]: from='client.65113 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:31:50.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:31:50 smithi083 ceph-mon[79930]: from='client.65119 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:31:50.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:31:50 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/737946751' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:31:50.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:31:50 smithi083 ceph-mon[77242]: pgmap v18104: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:50.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:31:50 smithi083 ceph-mon[77242]: from='client.65113 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:31:50.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:31:50 smithi083 ceph-mon[77242]: from='client.65119 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:31:50.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:31:50 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/737946751' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:31:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:31:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:31:51] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T15:31:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:31:52 smithi153 ceph-mon[70044]: pgmap v18105: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:52.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:31:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:31:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:31:52.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:31:52 smithi083 ceph-mon[79930]: pgmap v18105: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:52.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:31:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:31:52.430Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:31:52.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:31:52 smithi083 ceph-mon[77242]: pgmap v18105: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:31:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:31:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:31:53.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:31:53 smithi083 ceph-mon[79930]: pgmap v18106: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:53.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:31:53 smithi083 ceph-mon[77242]: pgmap v18106: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:54.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:31:53 smithi153 ceph-mon[70044]: pgmap v18106: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:56.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:31:55 smithi083 ceph-mon[77242]: pgmap v18107: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:56.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:31:55 smithi083 ceph-mon[79930]: pgmap v18107: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:56.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:31:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:31:55.786Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:31:56.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:31:55 smithi153 ceph-mon[70044]: pgmap v18107: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:56.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:31:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:31:56.524Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:31:58.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:31:57 smithi083 ceph-mon[79930]: pgmap v18108: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:58.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:31:57 smithi083 ceph-mon[77242]: pgmap v18108: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:31:58.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:31:57 smithi153 ceph-mon[70044]: pgmap v18108: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:00.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:31:59 smithi083 ceph-mon[79930]: pgmap v18109: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:00.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:31:59 smithi083 ceph-mon[77242]: pgmap v18109: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:31:59 smithi153 ceph-mon[70044]: pgmap v18109: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:01.841 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:32:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:32:01] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T15:32:02.210 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:32:01 smithi083 ceph-mon[77242]: pgmap v18110: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:02.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:32:01 smithi083 ceph-mon[79930]: pgmap v18110: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:02.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:32:01 smithi153 ceph-mon[70044]: pgmap v18110: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:02.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:32:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:32:02.431Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:32:02.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:32:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:32:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:32:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:32:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:32:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:32:03.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:32:03 smithi083 ceph-mon[77242]: pgmap v18111: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:03.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:32:03 smithi083 ceph-mon[79930]: pgmap v18111: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:04.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:32:03 smithi153 ceph-mon[70044]: pgmap v18111: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:06.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:32:05 smithi083 ceph-mon[79930]: pgmap v18112: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:06.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:32:05 smithi083 ceph-mon[77242]: pgmap v18112: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:06.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:32:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:32:05.786Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:32:06.313 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:32:05 smithi153 ceph-mon[70044]: pgmap v18112: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:06.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:32:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:32:06.525Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:32:08.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:32:07 smithi083 ceph-mon[79930]: pgmap v18113: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:08.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:32:07 smithi083 ceph-mon[77242]: pgmap v18113: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:08.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:32:07 smithi153 ceph-mon[70044]: pgmap v18113: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:10.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:32:09 smithi083 ceph-mon[79930]: pgmap v18114: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:10.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:32:09 smithi083 ceph-mon[77242]: pgmap v18114: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:10.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:32:09 smithi153 ceph-mon[70044]: pgmap v18114: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:11.842 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:32:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:32:11] "GET /metrics HTTP/1.1" 200 225246 "" "Prometheus/2.33.4" 2022-09-29T15:32:12.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:32:11 smithi083 ceph-mon[77242]: pgmap v18115: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:12.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:32:11 smithi083 ceph-mon[79930]: pgmap v18115: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:12.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:32:11 smithi153 ceph-mon[70044]: pgmap v18115: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:12.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:32:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:32:12.431Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:32:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:32:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:32:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:32:13.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:32:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:32:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:32:13.960 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:32:13 smithi083 ceph-mon[77242]: pgmap v18116: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:13.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:32:13 smithi083 ceph-mon[79930]: pgmap v18116: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:14.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:32:13 smithi153 ceph-mon[70044]: pgmap v18116: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:16.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:32:15 smithi083 ceph-mon[77242]: pgmap v18117: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:16.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:32:15 smithi083 ceph-mon[79930]: pgmap v18117: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:16.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:32:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:32:15.787Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:32:16.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:32:15 smithi153 ceph-mon[70044]: pgmap v18117: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:16.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:32:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:32:16.526Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:32:18.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:32:17 smithi083 ceph-mon[79930]: pgmap v18118: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:18.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:32:17 smithi083 ceph-mon[77242]: pgmap v18118: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:18.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:32:17 smithi153 ceph-mon[70044]: pgmap v18118: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:19.878 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:32:20.164 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:32:20.165 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 2m ago 10h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:32:20.165 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 37s ago 10h 63.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:32:20.165 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 2m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:32:20.165 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 37s ago 10h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:32:20.166 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 37s ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:32:20.166 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 2m ago 10h 1029M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:32:20.166 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 2m ago 10h 2196M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:32:20.166 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 37s ago 10h 957M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:32:20.167 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 2m ago 10h 1319M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:32:20.167 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 2m ago 10h 24.3M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:32:20.167 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 37s ago 10h 29.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:32:20.167 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 2m ago 10h 1595M 2148M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:32:20.168 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 2m ago 10h 1581M 2148M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:32:20.168 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 2m ago 10h 1228M 2148M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:32:20.168 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 2m ago 10h 1550M 2148M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:32:20.168 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (9h) 37s ago 10h 1358M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:32:20.169 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 37s ago 10h 1707M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:32:20.169 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 37s ago 10h 1524M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:32:20.169 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 37s ago 10h 1814M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:32:20.169 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 37s ago 10h 177M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:32:20.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:32:19 smithi083 ceph-mon[79930]: pgmap v18119: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:20.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:32:19 smithi083 ceph-mon[77242]: pgmap v18119: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:32:19 smithi153 ceph-mon[70044]: pgmap v18119: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:20.519 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:32:20.519 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:32:20.519 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:32:20.520 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:32:20.520 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:32:20.520 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:32:20.520 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:32:20.521 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:32:20.521 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:32:20.521 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:32:20.522 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:32:20.522 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:32:20.522 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:32:20.522 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:32:20.523 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:32:20.523 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:32:20.523 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:32:20.524 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:32:21.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:32:20 smithi083 ceph-mon[79930]: from='client.65131 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:32:21.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:32:20 smithi083 ceph-mon[79930]: from='client.65137 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:32:21.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:32:20 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1305883210' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:32:21.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:32:20 smithi083 ceph-mon[77242]: from='client.65131 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:32:21.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:32:20 smithi083 ceph-mon[77242]: from='client.65137 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:32:21.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:32:20 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1305883210' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:32:21.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:32:20 smithi153 ceph-mon[70044]: from='client.65131 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:32:21.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:32:20 smithi153 ceph-mon[70044]: from='client.65137 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:32:21.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:32:20 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1305883210' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:32:21.850 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:32:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:32:21] "GET /metrics HTTP/1.1" 200 225246 "" "Prometheus/2.33.4" 2022-09-29T15:32:22.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:32:21 smithi083 ceph-mon[77242]: pgmap v18120: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:22.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:32:21 smithi083 ceph-mon[79930]: pgmap v18120: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:22.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:32:21 smithi153 ceph-mon[70044]: pgmap v18120: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:22.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:32:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:32:22.432Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:32:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:32:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:32:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:32:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:32:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:32:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:32:23.960 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:32:23 smithi083 ceph-mon[77242]: pgmap v18121: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:23.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:32:23 smithi083 ceph-mon[79930]: pgmap v18121: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:24.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:32:23 smithi153 ceph-mon[70044]: pgmap v18121: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:26.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:32:25 smithi083 ceph-mon[77242]: pgmap v18122: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:26.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:32:25 smithi083 ceph-mon[79930]: pgmap v18122: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:26.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:32:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:32:25.788Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:32:26.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:32:25 smithi153 ceph-mon[70044]: pgmap v18122: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:26.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:32:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:32:26.527Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:32:28.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:32:27 smithi083 ceph-mon[79930]: pgmap v18123: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:28.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:32:27 smithi083 ceph-mon[77242]: pgmap v18123: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:28.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:32:27 smithi153 ceph-mon[70044]: pgmap v18123: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:30.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:32:29 smithi083 ceph-mon[79930]: pgmap v18124: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:30.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:32:29 smithi083 ceph-mon[77242]: pgmap v18124: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:30.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:32:29 smithi153 ceph-mon[70044]: pgmap v18124: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:31.849 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:32:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:32:31] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T15:32:32.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:32:31 smithi083 ceph-mon[79930]: pgmap v18125: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:32.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:32:31 smithi083 ceph-mon[77242]: pgmap v18125: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:32.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:32:31 smithi153 ceph-mon[70044]: pgmap v18125: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:32.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:32:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:32:32.432Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:32:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:32:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:32:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:32:33.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:32:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:32:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:32:33.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:32:33 smithi083 ceph-mon[79930]: pgmap v18126: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:33.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:32:33 smithi083 ceph-mon[77242]: pgmap v18126: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:34.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:32:33 smithi153 ceph-mon[70044]: pgmap v18126: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:36.210 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:32:35 smithi083 ceph-mon[77242]: pgmap v18127: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:36.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:32:35 smithi083 ceph-mon[79930]: pgmap v18127: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:36.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:32:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:32:35.789Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:32:36.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:32:35 smithi153 ceph-mon[70044]: pgmap v18127: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:36.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:32:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:32:36.527Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:32:38.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:32:37 smithi083 ceph-mon[77242]: pgmap v18128: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:38.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:32:37 smithi083 ceph-mon[79930]: pgmap v18128: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:38.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:32:37 smithi153 ceph-mon[70044]: pgmap v18128: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:40.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:32:39 smithi083 ceph-mon[77242]: pgmap v18129: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:40.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:32:39 smithi083 ceph-mon[79930]: pgmap v18129: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:40.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:32:39 smithi153 ceph-mon[70044]: pgmap v18129: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:41.853 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:32:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:32:41] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T15:32:42.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:32:41 smithi083 ceph-mon[79930]: pgmap v18130: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:42.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:32:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:32:42.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:32:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:32:42.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:32:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:32:42.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:32:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:32:42.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:32:41 smithi083 ceph-mon[77242]: pgmap v18130: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:42.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:32:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:32:42.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:32:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:32:42.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:32:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:32:42.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:32:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:32:42.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:32:41 smithi153 ceph-mon[70044]: pgmap v18130: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:32:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:32:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:32:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:32:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:32:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:32:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:32:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:32:42.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:32:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:32:42.433Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:32:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:32:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:32:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:32:43.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:32:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:32:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:32:43.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:32:43 smithi083 ceph-mon[79930]: pgmap v18131: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:43.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:32:43 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:32:43.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:32:43 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:32:43.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:32:43 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:32:43.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:32:43 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:32:43.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:32:43 smithi083 ceph-mon[77242]: pgmap v18131: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:43.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:32:43 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:32:43.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:32:43 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:32:43.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:32:43 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:32:43.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:32:43 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:32:44.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:32:43 smithi153 ceph-mon[70044]: pgmap v18131: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:44.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:32:43 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:32:44.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:32:43 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:32:44.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:32:43 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:32:44.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:32:43 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:32:46.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:32:45 smithi083 ceph-mon[79930]: pgmap v18132: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:46.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:32:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:32:45.790Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:32:46.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:32:45 smithi083 ceph-mon[77242]: pgmap v18132: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:46.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:32:45 smithi153 ceph-mon[70044]: pgmap v18132: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:46.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:32:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:32:46.528Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:32:48.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:32:47 smithi083 ceph-mon[79930]: pgmap v18133: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:48.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:32:47 smithi083 ceph-mon[77242]: pgmap v18133: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:48.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:32:47 smithi153 ceph-mon[70044]: pgmap v18133: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:50.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:32:49 smithi083 ceph-mon[77242]: pgmap v18134: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:50.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:32:49 smithi083 ceph-mon[79930]: pgmap v18134: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:50.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:32:49 smithi153 ceph-mon[70044]: pgmap v18134: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:50.847 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:32:51.138 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:32:51.139 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 3m ago 10h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:32:51.139 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 68s ago 10h 63.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:32:51.139 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 3m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:32:51.140 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 68s ago 10h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:32:51.140 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 68s ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:32:51.140 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 3m ago 10h 1029M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:32:51.140 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 3m ago 10h 2196M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:32:51.140 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 68s ago 10h 957M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:32:51.141 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 3m ago 10h 1319M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:32:51.141 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 3m ago 10h 24.3M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:32:51.141 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 68s ago 10h 29.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:32:51.141 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 3m ago 10h 1595M 2148M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:32:51.142 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 3m ago 10h 1581M 2148M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:32:51.142 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 3m ago 10h 1228M 2148M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:32:51.142 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 3m ago 10h 1550M 2148M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:32:51.142 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 68s ago 10h 1358M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:32:51.143 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (9h) 68s ago 10h 1707M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:32:51.143 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 68s ago 10h 1524M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:32:51.143 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 68s ago 10h 1814M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:32:51.143 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 68s ago 10h 177M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:32:51.493 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:32:51.494 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:32:51.494 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:32:51.494 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:32:51.494 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:32:51.495 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:32:51.495 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:32:51.495 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:32:51.495 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:32:51.495 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:32:51.496 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:32:51.496 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:32:51.496 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:32:51.496 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:32:51.497 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:32:51.497 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:32:51.497 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:32:51.497 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:32:51.854 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:32:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:32:51] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T15:32:52.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:32:51 smithi083 ceph-mon[79930]: pgmap v18135: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:52.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:32:51 smithi083 ceph-mon[79930]: from='client.55218 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:32:52.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:32:51 smithi083 ceph-mon[79930]: from='client.55224 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:32:52.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:32:51 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1014033142' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:32:52.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:32:51 smithi083 ceph-mon[77242]: pgmap v18135: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:52.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:32:51 smithi083 ceph-mon[77242]: from='client.55218 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:32:52.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:32:51 smithi083 ceph-mon[77242]: from='client.55224 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:32:52.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:32:51 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1014033142' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:32:52.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:32:51 smithi153 ceph-mon[70044]: pgmap v18135: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:52.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:32:51 smithi153 ceph-mon[70044]: from='client.55218 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:32:52.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:32:51 smithi153 ceph-mon[70044]: from='client.55224 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:32:52.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:32:51 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1014033142' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:32:52.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:32:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:32:52.433Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:32:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:32:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:32:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:32:53.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:32:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:32:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:32:53.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:32:53 smithi083 ceph-mon[79930]: pgmap v18136: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:53.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:32:53 smithi083 ceph-mon[77242]: pgmap v18136: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:54.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:32:53 smithi153 ceph-mon[70044]: pgmap v18136: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:56.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:32:55 smithi083 ceph-mon[79930]: pgmap v18137: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:56.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:32:55 smithi083 ceph-mon[77242]: pgmap v18137: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:56.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:32:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:32:55.790Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:32:56.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:32:55 smithi153 ceph-mon[70044]: pgmap v18137: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:56.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:32:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:32:56.529Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:32:58.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:32:57 smithi083 ceph-mon[79930]: pgmap v18138: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:58.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:32:57 smithi083 ceph-mon[77242]: pgmap v18138: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:32:58.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:32:57 smithi153 ceph-mon[70044]: pgmap v18138: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:00.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:32:59 smithi083 ceph-mon[79930]: pgmap v18139: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:00.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:32:59 smithi083 ceph-mon[77242]: pgmap v18139: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:00.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:32:59 smithi153 ceph-mon[70044]: pgmap v18139: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:01.857 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:33:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:33:01] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T15:33:02.210 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:33:01 smithi083 ceph-mon[77242]: pgmap v18140: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:02.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:33:01 smithi083 ceph-mon[79930]: pgmap v18140: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:02.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:33:01 smithi153 ceph-mon[70044]: pgmap v18140: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:02.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:33:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:33:02.434Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:33:02.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:33:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:33:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:33:03.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:33:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:33:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:33:04.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:33:03 smithi153 ceph-mon[70044]: pgmap v18141: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:04.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:33:03 smithi083 ceph-mon[79930]: pgmap v18141: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:04.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:33:03 smithi083 ceph-mon[77242]: pgmap v18141: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:06.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:33:05 smithi083 ceph-mon[79930]: pgmap v18142: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:06.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:33:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:33:05.791Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:33:06.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:33:05 smithi083 ceph-mon[77242]: pgmap v18142: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:06.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:33:05 smithi153 ceph-mon[70044]: pgmap v18142: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:06.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:33:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:33:06.529Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:33:08.210 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:33:07 smithi083 ceph-mon[77242]: pgmap v18143: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:08.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:33:07 smithi083 ceph-mon[79930]: pgmap v18143: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:08.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:33:07 smithi153 ceph-mon[70044]: pgmap v18143: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:10.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:33:09 smithi083 ceph-mon[79930]: pgmap v18144: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:10.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:33:09 smithi083 ceph-mon[77242]: pgmap v18144: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:10.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:33:09 smithi153 ceph-mon[70044]: pgmap v18144: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:11.861 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:33:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:33:11] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T15:33:12.210 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:33:11 smithi083 ceph-mon[77242]: pgmap v18145: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:12.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:33:11 smithi083 ceph-mon[79930]: pgmap v18145: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:12.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:33:11 smithi153 ceph-mon[70044]: pgmap v18145: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:12.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:33:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:33:12.434Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:33:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:33:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:33:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:33:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:33:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:33:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:33:13.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:33:13 smithi083 ceph-mon[79930]: pgmap v18146: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:13.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:33:13 smithi083 ceph-mon[77242]: pgmap v18146: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:14.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:33:13 smithi153 ceph-mon[70044]: pgmap v18146: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:16.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:33:15 smithi083 ceph-mon[77242]: pgmap v18147: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:16.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:33:15 smithi083 ceph-mon[79930]: pgmap v18147: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:16.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:33:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:33:15.792Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:33:16.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:33:15 smithi153 ceph-mon[70044]: pgmap v18147: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:16.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:33:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:33:16.530Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:33:18.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:33:17 smithi083 ceph-mon[79930]: pgmap v18148: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:18.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:33:17 smithi083 ceph-mon[77242]: pgmap v18148: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:18.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:33:17 smithi153 ceph-mon[70044]: pgmap v18148: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:20.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:33:19 smithi083 ceph-mon[79930]: pgmap v18149: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:20.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:33:19 smithi083 ceph-mon[77242]: pgmap v18149: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:20.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:33:19 smithi153 ceph-mon[70044]: pgmap v18149: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:21.818 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:33:21.865 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:33:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:33:21] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T15:33:22.105 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:33:22.106 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 3m ago 10h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:33:22.106 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 99s ago 10h 63.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:33:22.106 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 3m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:33:22.106 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 99s ago 10h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:33:22.107 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 99s ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:33:22.107 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 3m ago 10h 1029M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:33:22.107 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 3m ago 10h 2196M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:33:22.107 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 99s ago 10h 957M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:33:22.108 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 3m ago 10h 1319M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:33:22.108 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 3m ago 10h 24.3M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:33:22.108 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 99s ago 10h 29.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:33:22.108 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 3m ago 10h 1595M 2148M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:33:22.109 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 3m ago 10h 1581M 2148M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:33:22.109 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 3m ago 10h 1228M 2148M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:33:22.109 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 3m ago 10h 1550M 2148M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:33:22.109 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 99s ago 10h 1358M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:33:22.109 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 99s ago 10h 1707M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:33:22.110 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (9h) 99s ago 10h 1524M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:33:22.110 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 99s ago 10h 1814M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:33:22.110 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 99s ago 10h 177M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:33:22.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:33:21 smithi083 ceph-mon[79930]: pgmap v18150: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:22.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:33:21 smithi083 ceph-mon[77242]: pgmap v18150: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:22.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:33:21 smithi153 ceph-mon[70044]: pgmap v18150: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:22.462 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:33:22.463 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:33:22.463 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:33:22.463 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:33:22.463 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:33:22.464 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:33:22.464 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:33:22.464 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:33:22.464 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:33:22.464 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:33:22.465 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:33:22.465 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:33:22.465 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:33:22.465 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:33:22.466 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:33:22.466 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:33:22.466 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:33:22.466 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:33:22.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:33:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:33:22.434Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:33:22.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:33:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:33:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:33:23.210 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:33:22 smithi083 ceph-mon[77242]: from='client.65167 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:33:23.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:33:22 smithi083 ceph-mon[77242]: from='client.65173 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:33:23.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:33:22 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3914604328' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:33:23.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:33:22 smithi083 ceph-mon[79930]: from='client.65167 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:33:23.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:33:22 smithi083 ceph-mon[79930]: from='client.65173 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:33:23.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:33:22 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3914604328' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:33:23.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:33:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:33:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:33:23.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:33:22 smithi153 ceph-mon[70044]: from='client.65167 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:33:23.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:33:22 smithi153 ceph-mon[70044]: from='client.65173 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:33:23.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:33:22 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3914604328' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:33:24.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:33:23 smithi083 ceph-mon[79930]: pgmap v18151: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:24.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:33:23 smithi083 ceph-mon[77242]: pgmap v18151: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:24.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:33:23 smithi153 ceph-mon[70044]: pgmap v18151: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:26.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:33:25 smithi083 ceph-mon[79930]: pgmap v18152: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:26.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:33:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:33:25.791Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:33:26.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:33:25 smithi083 ceph-mon[77242]: pgmap v18152: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:26.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:33:25 smithi153 ceph-mon[70044]: pgmap v18152: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:26.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:33:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:33:26.531Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:33:28.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:33:27 smithi083 ceph-mon[79930]: pgmap v18153: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:28.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:33:27 smithi083 ceph-mon[77242]: pgmap v18153: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:28.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:33:27 smithi153 ceph-mon[70044]: pgmap v18153: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:30.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:33:29 smithi083 ceph-mon[79930]: pgmap v18154: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:30.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:33:29 smithi083 ceph-mon[77242]: pgmap v18154: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:30.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:33:29 smithi153 ceph-mon[70044]: pgmap v18154: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:31.886 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:33:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:33:31] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T15:33:32.210 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:33:31 smithi083 ceph-mon[77242]: pgmap v18155: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:32.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:33:31 smithi083 ceph-mon[79930]: pgmap v18155: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:32.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:33:31 smithi153 ceph-mon[70044]: pgmap v18155: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:32.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:33:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:33:32.436Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:33:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:33:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:33:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:33:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:33:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:33:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:33:34.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:33:33 smithi153 ceph-mon[70044]: pgmap v18156: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:34.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:33:33 smithi083 ceph-mon[77242]: pgmap v18156: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:34.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:33:33 smithi083 ceph-mon[79930]: pgmap v18156: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:36.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:33:35 smithi083 ceph-mon[79930]: pgmap v18157: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:36.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:33:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:33:35.793Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:33:36.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:33:35 smithi083 ceph-mon[77242]: pgmap v18157: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:36.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:33:35 smithi153 ceph-mon[70044]: pgmap v18157: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:36.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:33:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:33:36.531Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:33:38.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:33:37 smithi083 ceph-mon[79930]: pgmap v18158: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:38.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:33:37 smithi083 ceph-mon[77242]: pgmap v18158: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:38.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:33:37 smithi153 ceph-mon[70044]: pgmap v18158: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:40.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:33:39 smithi083 ceph-mon[77242]: pgmap v18159: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:40.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:33:39 smithi083 ceph-mon[79930]: pgmap v18159: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:40.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:33:39 smithi153 ceph-mon[70044]: pgmap v18159: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:41.870 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:33:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:33:41] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T15:33:42.210 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:33:41 smithi083 ceph-mon[77242]: pgmap v18160: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:42.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:33:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:33:42.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:33:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:33:42.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:33:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:33:42.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:33:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:33:42.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:33:41 smithi083 ceph-mon[79930]: pgmap v18160: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:42.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:33:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:33:42.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:33:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:33:42.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:33:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:33:42.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:33:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:33:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:33:41 smithi153 ceph-mon[70044]: pgmap v18160: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:33:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:33:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:33:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:33:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:33:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:33:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:33:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:33:42.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:33:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:33:42.436Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:33:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:33:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:33:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:33:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:33:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:33:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:33:44.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:33:43 smithi153 ceph-mon[70044]: pgmap v18161: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:44.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:33:43 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:33:44.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:33:43 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:33:44.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:33:43 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:33:44.210 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:33:43 smithi083 ceph-mon[77242]: pgmap v18161: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:44.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:33:43 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:33:44.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:33:43 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:33:44.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:33:43 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:33:44.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:33:43 smithi083 ceph-mon[79930]: pgmap v18161: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:44.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:33:43 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:33:44.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:33:43 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:33:44.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:33:43 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:33:45.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:33:45 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:33:45.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:33:45 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:33:45.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:33:45 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:33:46.097 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:33:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:33:45.794Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:33:46.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:33:46 smithi083 ceph-mon[79930]: pgmap v18162: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:46.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:33:46 smithi083 ceph-mon[77242]: pgmap v18162: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:46.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:33:46 smithi153 ceph-mon[70044]: pgmap v18162: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:46.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:33:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:33:46.532Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:33:48.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:33:48 smithi083 ceph-mon[79930]: pgmap v18163: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:48.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:33:48 smithi083 ceph-mon[77242]: pgmap v18163: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:48.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:33:48 smithi153 ceph-mon[70044]: pgmap v18163: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:50.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:33:50 smithi083 ceph-mon[79930]: pgmap v18164: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:50.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:33:50 smithi083 ceph-mon[77242]: pgmap v18164: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:50.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:33:50 smithi153 ceph-mon[70044]: pgmap v18164: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:51.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:33:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:33:51] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T15:33:52.438 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:33:52 smithi083 ceph-mon[79930]: pgmap v18165: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:52.439 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:33:52 smithi083 ceph-mon[77242]: pgmap v18165: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:33:52 smithi153 ceph-mon[70044]: pgmap v18165: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:52.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:33:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:33:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:33:52.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:33:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:33:52.437Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:33:52.796 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:33:53.085 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:33:53.085 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 4m ago 10h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:33:53.086 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 2m ago 10h 63.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:33:53.087 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 4m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:33:53.087 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 2m ago 10h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:33:53.088 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 2m ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:33:53.088 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 4m ago 10h 1029M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:33:53.089 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 4m ago 10h 2196M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:33:53.089 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 2m ago 10h 957M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:33:53.089 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 4m ago 10h 1319M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:33:53.090 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 4m ago 10h 24.3M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:33:53.090 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 2m ago 10h 29.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:33:53.091 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 4m ago 10h 1595M 2148M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:33:53.091 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 4m ago 10h 1581M 2148M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:33:53.092 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 4m ago 10h 1228M 2148M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:33:53.092 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 4m ago 10h 1550M 2148M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:33:53.094 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 2m ago 10h 1358M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:33:53.094 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 2m ago 10h 1707M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:33:53.094 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 2m ago 10h 1524M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:33:53.095 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 2m ago 10h 1814M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:33:53.095 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 2m ago 10h 177M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:33:53.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:33:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:33:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:33:53.449 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:33:53.449 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:33:53.450 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:33:53.450 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:33:53.450 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:33:53.450 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:33:53.451 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:33:53.451 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:33:53.451 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:33:53.451 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:33:53.451 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:33:53.452 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:33:53.452 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:33:53.452 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:33:53.452 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:33:53.453 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:33:53.453 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:33:53.453 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:33:54.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:33:53 smithi153 ceph-mon[70044]: from='client.55254 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:33:54.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:33:53 smithi153 ceph-mon[70044]: pgmap v18166: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:54.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:33:53 smithi153 ceph-mon[70044]: from='client.55260 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:33:54.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:33:53 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/669745162' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:33:54.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:33:53 smithi083 ceph-mon[79930]: from='client.55254 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:33:54.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:33:53 smithi083 ceph-mon[79930]: pgmap v18166: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:54.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:33:53 smithi083 ceph-mon[79930]: from='client.55260 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:33:54.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:33:53 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/669745162' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:33:54.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:33:53 smithi083 ceph-mon[77242]: from='client.55254 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:33:54.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:33:53 smithi083 ceph-mon[77242]: pgmap v18166: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:54.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:33:53 smithi083 ceph-mon[77242]: from='client.55260 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:33:54.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:33:53 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/669745162' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:33:56.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:33:55 smithi083 ceph-mon[79930]: pgmap v18167: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:56.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:33:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:33:55.794Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:33:56.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:33:55 smithi083 ceph-mon[77242]: pgmap v18167: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:56.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:33:55 smithi153 ceph-mon[70044]: pgmap v18167: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:56.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:33:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:33:56.532Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:33:58.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:33:57 smithi083 ceph-mon[79930]: pgmap v18168: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:58.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:33:57 smithi083 ceph-mon[77242]: pgmap v18168: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:33:58.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:33:57 smithi153 ceph-mon[70044]: pgmap v18168: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:00.210 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:33:59 smithi083 ceph-mon[77242]: pgmap v18169: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:00.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:33:59 smithi083 ceph-mon[79930]: pgmap v18169: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:33:59 smithi153 ceph-mon[70044]: pgmap v18169: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:01.876 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:34:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:34:01] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T15:34:02.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:34:01 smithi083 ceph-mon[79930]: pgmap v18170: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:02.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:34:01 smithi083 ceph-mon[77242]: pgmap v18170: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:02.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:34:01 smithi153 ceph-mon[70044]: pgmap v18170: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:02.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:34:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:34:02.437Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:34:02.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:34:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:34:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:34:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:34:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:34:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:34:04.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:34:03 smithi153 ceph-mon[70044]: pgmap v18171: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:04.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:34:03 smithi083 ceph-mon[79930]: pgmap v18171: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:04.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:34:03 smithi083 ceph-mon[77242]: pgmap v18171: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:06.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:34:05 smithi083 ceph-mon[79930]: pgmap v18172: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:06.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:34:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:34:05.795Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:34:06.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:34:05 smithi083 ceph-mon[77242]: pgmap v18172: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:06.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:34:05 smithi153 ceph-mon[70044]: pgmap v18172: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:06.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:34:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:34:06.534Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:34:08.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:34:07 smithi083 ceph-mon[79930]: pgmap v18173: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:08.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:34:07 smithi083 ceph-mon[77242]: pgmap v18173: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:08.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:34:07 smithi153 ceph-mon[70044]: pgmap v18173: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:10.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:34:09 smithi083 ceph-mon[79930]: pgmap v18174: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:10.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:34:09 smithi083 ceph-mon[77242]: pgmap v18174: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:10.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:34:09 smithi153 ceph-mon[70044]: pgmap v18174: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:11.881 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:34:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:34:11] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T15:34:12.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:34:11 smithi083 ceph-mon[79930]: pgmap v18175: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:12.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:34:11 smithi083 ceph-mon[77242]: pgmap v18175: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:12.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:34:11 smithi153 ceph-mon[70044]: pgmap v18175: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:12.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:34:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:34:12.438Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:34:12.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:34:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:34:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:34:13.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:34:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:34:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:34:14.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:34:13 smithi153 ceph-mon[70044]: pgmap v18176: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:14.210 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:34:13 smithi083 ceph-mon[77242]: pgmap v18176: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:14.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:34:13 smithi083 ceph-mon[79930]: pgmap v18176: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:16.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:34:15 smithi083 ceph-mon[77242]: pgmap v18177: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:16.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:34:15 smithi083 ceph-mon[79930]: pgmap v18177: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:16.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:34:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:34:15.795Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:34:16.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:34:15 smithi153 ceph-mon[70044]: pgmap v18177: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:16.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:34:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:34:16.534Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:34:18.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:34:17 smithi083 ceph-mon[79930]: pgmap v18178: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:18.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:34:17 smithi083 ceph-mon[77242]: pgmap v18178: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:18.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:34:17 smithi153 ceph-mon[70044]: pgmap v18178: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:20.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:34:19 smithi083 ceph-mon[79930]: pgmap v18179: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:20.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:34:19 smithi083 ceph-mon[77242]: pgmap v18179: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:34:19 smithi153 ceph-mon[70044]: pgmap v18179: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:21.882 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:34:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:34:21] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T15:34:22.218 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:34:21 smithi083 ceph-mon[79930]: pgmap v18180: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:22.219 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:34:21 smithi083 ceph-mon[77242]: pgmap v18180: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:22.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:34:21 smithi153 ceph-mon[70044]: pgmap v18180: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:22.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:34:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:34:22.439Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:34:22.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:34:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:34:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:34:23.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:34:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:34:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:34:23.778 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:34:24.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:34:23 smithi153 ceph-mon[70044]: pgmap v18181: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:24.067 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:34:24.068 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 4m ago 10h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:34:24.068 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 2m ago 10h 63.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:34:24.069 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 4m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:34:24.069 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 2m ago 10h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:34:24.069 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 2m ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:34:24.070 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 4m ago 10h 1029M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:34:24.070 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 4m ago 10h 2196M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:34:24.071 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 2m ago 10h 957M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:34:24.071 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 4m ago 10h 1319M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:34:24.071 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 4m ago 10h 24.3M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:34:24.072 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 2m ago 10h 29.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:34:24.072 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 4m ago 10h 1595M 2148M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:34:24.073 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 4m ago 10h 1581M 2148M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:34:24.073 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 4m ago 10h 1228M 2148M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:34:24.073 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 4m ago 10h 1550M 2148M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:34:24.073 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 2m ago 10h 1358M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:34:24.074 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 2m ago 10h 1707M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:34:24.074 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 2m ago 10h 1524M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:34:24.074 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (9h) 2m ago 10h 1814M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:34:24.074 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 2m ago 10h 177M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:34:24.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:34:23 smithi083 ceph-mon[79930]: pgmap v18181: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:24.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:34:23 smithi083 ceph-mon[77242]: pgmap v18181: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:24.421 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:34:24.421 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:34:24.422 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:34:24.422 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:34:24.423 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:34:24.423 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:34:24.424 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:34:24.424 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:34:24.425 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:34:24.425 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:34:24.425 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:34:24.426 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:34:24.426 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:34:24.427 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:34:24.427 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:34:24.428 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:34:24.428 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:34:24.428 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:34:25.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:34:24 smithi153 ceph-mon[70044]: from='client.65203 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:34:25.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:34:24 smithi153 ceph-mon[70044]: from='client.55278 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:34:25.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:34:24 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/153319907' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:34:25.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:34:24 smithi083 ceph-mon[79930]: from='client.65203 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:34:25.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:34:24 smithi083 ceph-mon[79930]: from='client.55278 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:34:25.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:34:24 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/153319907' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:34:25.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:34:24 smithi083 ceph-mon[77242]: from='client.65203 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:34:25.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:34:24 smithi083 ceph-mon[77242]: from='client.55278 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:34:25.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:34:24 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/153319907' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:34:26.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:34:25 smithi153 ceph-mon[70044]: pgmap v18182: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:26.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:34:25 smithi083 ceph-mon[79930]: pgmap v18182: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:26.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:34:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:34:25.796Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:34:26.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:34:25 smithi083 ceph-mon[77242]: pgmap v18182: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:26.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:34:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:34:26.534Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:34:28.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:34:27 smithi083 ceph-mon[79930]: pgmap v18183: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:28.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:34:27 smithi083 ceph-mon[77242]: pgmap v18183: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:28.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:34:27 smithi153 ceph-mon[70044]: pgmap v18183: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:30.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:34:29 smithi083 ceph-mon[79930]: pgmap v18184: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:30.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:34:29 smithi083 ceph-mon[77242]: pgmap v18184: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:30.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:34:29 smithi153 ceph-mon[70044]: pgmap v18184: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:31.885 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:34:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:34:31] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T15:34:32.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:34:31 smithi083 ceph-mon[79930]: pgmap v18185: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:32.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:34:31 smithi083 ceph-mon[77242]: pgmap v18185: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:32.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:34:31 smithi153 ceph-mon[70044]: pgmap v18185: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:32.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:34:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:34:32.439Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:34:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:34:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:34:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:34:33.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:34:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:34:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:34:34.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:34:33 smithi153 ceph-mon[70044]: pgmap v18186: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:34.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:34:33 smithi083 ceph-mon[77242]: pgmap v18186: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:34.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:34:33 smithi083 ceph-mon[79930]: pgmap v18186: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:36.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:34:35 smithi083 ceph-mon[79930]: pgmap v18187: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:36.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:34:35 smithi083 ceph-mon[77242]: pgmap v18187: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:36.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:34:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:34:35.796Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:34:36.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:34:35 smithi153 ceph-mon[70044]: pgmap v18187: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:36.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:34:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:34:36.535Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:34:38.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:34:37 smithi083 ceph-mon[77242]: pgmap v18188: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:38.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:34:37 smithi083 ceph-mon[79930]: pgmap v18188: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:38.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:34:37 smithi153 ceph-mon[70044]: pgmap v18188: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:40.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:34:39 smithi083 ceph-mon[79930]: pgmap v18189: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:40.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:34:39 smithi083 ceph-mon[77242]: pgmap v18189: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:34:39 smithi153 ceph-mon[70044]: pgmap v18189: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:41.888 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:34:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:34:41] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T15:34:42.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:34:41 smithi083 ceph-mon[77242]: pgmap v18190: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:42.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:34:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:34:42.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:34:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:34:42.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:34:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:34:42.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:34:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:34:42.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:34:41 smithi083 ceph-mon[79930]: pgmap v18190: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:42.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:34:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:34:42.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:34:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:34:42.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:34:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:34:42.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:34:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:34:42.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:34:41 smithi153 ceph-mon[70044]: pgmap v18190: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:34:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:34:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:34:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:34:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:34:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:34:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:34:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:34:42.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:34:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:34:42.440Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:34:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:34:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:34:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:34:43.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:34:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:34:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:34:44.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:34:43 smithi153 ceph-mon[70044]: pgmap v18191: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:44.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:34:43 smithi083 ceph-mon[79930]: pgmap v18191: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:44.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:34:43 smithi083 ceph-mon[77242]: pgmap v18191: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:45.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:34:44 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:34:45.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:34:44 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:34:45.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:34:44 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:34:45.210 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:34:44 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:34:45.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:34:44 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:34:45.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:34:44 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:34:45.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:34:44 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:34:45.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:34:44 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:34:45.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:34:44 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:34:46.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:34:45 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:34:46.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:34:45 smithi153 ceph-mon[70044]: pgmap v18192: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:46.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:34:45 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:34:46.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:34:45 smithi083 ceph-mon[79930]: pgmap v18192: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:46.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:34:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:34:45.797Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:34:46.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:34:45 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:34:46.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:34:45 smithi083 ceph-mon[77242]: pgmap v18192: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:46.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:34:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:34:46.536Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:34:48.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:34:47 smithi083 ceph-mon[79930]: pgmap v18193: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:48.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:34:47 smithi083 ceph-mon[77242]: pgmap v18193: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:48.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:34:47 smithi153 ceph-mon[70044]: pgmap v18193: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:50.210 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:34:49 smithi083 ceph-mon[77242]: pgmap v18194: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:50.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:34:49 smithi083 ceph-mon[79930]: pgmap v18194: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:50.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:34:49 smithi153 ceph-mon[70044]: pgmap v18194: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:51.891 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:34:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:34:51] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T15:34:52.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:34:51 smithi083 ceph-mon[79930]: pgmap v18195: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:52.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:34:51 smithi083 ceph-mon[77242]: pgmap v18195: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:52.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:34:51 smithi153 ceph-mon[70044]: pgmap v18195: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:52.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:34:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:34:52.441Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:34:52.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:34:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:34:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:34:53.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:34:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:34:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:34:54.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:34:53 smithi153 ceph-mon[70044]: pgmap v18196: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:54.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:34:53 smithi083 ceph-mon[79930]: pgmap v18196: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:54.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:34:53 smithi083 ceph-mon[77242]: pgmap v18196: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:54.750 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:34:55.040 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:34:55.041 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 5m ago 10h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:34:55.041 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 3m ago 10h 63.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:34:55.041 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 5m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:34:55.041 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 3m ago 10h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:34:55.042 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 3m ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:34:55.042 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 5m ago 10h 1029M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:34:55.042 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 5m ago 10h 2196M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:34:55.042 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 3m ago 10h 957M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:34:55.043 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 5m ago 10h 1319M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:34:55.043 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 5m ago 10h 24.3M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:34:55.043 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 3m ago 10h 29.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:34:55.043 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 5m ago 10h 1595M 2148M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:34:55.044 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 5m ago 10h 1581M 2148M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:34:55.044 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 5m ago 10h 1228M 2148M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:34:55.044 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 5m ago 10h 1550M 2148M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:34:55.044 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 3m ago 10h 1358M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:34:55.045 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 3m ago 10h 1707M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:34:55.045 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 3m ago 10h 1524M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:34:55.045 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 3m ago 10h 1814M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:34:55.045 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 3m ago 10h 177M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:34:55.399 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:34:55.399 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:34:55.399 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:34:55.400 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:34:55.400 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:34:55.400 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:34:55.401 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:34:55.401 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:34:55.401 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:34:55.401 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:34:55.402 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:34:55.402 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:34:55.402 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:34:55.402 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:34:55.402 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:34:55.403 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:34:55.403 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:34:55.403 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:34:56.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:34:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:34:55.797Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:34:56.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:34:55 smithi083 ceph-mon[79930]: from='client.65221 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:34:56.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:34:55 smithi083 ceph-mon[79930]: pgmap v18197: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:56.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:34:55 smithi083 ceph-mon[79930]: from='client.65227 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:34:56.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:34:55 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3873515477' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:34:56.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:34:55 smithi083 ceph-mon[77242]: from='client.65221 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:34:56.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:34:55 smithi083 ceph-mon[77242]: pgmap v18197: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:56.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:34:55 smithi083 ceph-mon[77242]: from='client.65227 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:34:56.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:34:55 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3873515477' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:34:56.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:34:55 smithi153 ceph-mon[70044]: from='client.65221 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:34:56.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:34:55 smithi153 ceph-mon[70044]: pgmap v18197: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:56.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:34:55 smithi153 ceph-mon[70044]: from='client.65227 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:34:56.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:34:55 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3873515477' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:34:56.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:34:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:34:56.537Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:34:58.210 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:34:57 smithi083 ceph-mon[77242]: pgmap v18198: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:58.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:34:57 smithi083 ceph-mon[79930]: pgmap v18198: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:34:58.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:34:57 smithi153 ceph-mon[70044]: pgmap v18198: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:00.210 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:34:59 smithi083 ceph-mon[77242]: pgmap v18199: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:00.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:34:59 smithi083 ceph-mon[79930]: pgmap v18199: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:34:59 smithi153 ceph-mon[70044]: pgmap v18199: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:01.894 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:35:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:35:01] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T15:35:02.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:35:01 smithi083 ceph-mon[79930]: pgmap v18200: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:02.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:35:01 smithi083 ceph-mon[77242]: pgmap v18200: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:02.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:35:01 smithi153 ceph-mon[70044]: pgmap v18200: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:02.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:35:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:35:02.442Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:35:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:35:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:35:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:35:03.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:35:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:35:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:35:04.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:35:03 smithi153 ceph-mon[70044]: pgmap v18201: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:04.210 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:35:03 smithi083 ceph-mon[77242]: pgmap v18201: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:04.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:35:03 smithi083 ceph-mon[79930]: pgmap v18201: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:06.210 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:35:05 smithi083 ceph-mon[77242]: pgmap v18202: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:06.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:35:05 smithi083 ceph-mon[79930]: pgmap v18202: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:06.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:35:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:35:05.798Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:35:06.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:35:05 smithi153 ceph-mon[70044]: pgmap v18202: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:06.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:35:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:35:06.539Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:35:08.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:35:07 smithi083 ceph-mon[79930]: pgmap v18203: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:08.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:35:07 smithi083 ceph-mon[77242]: pgmap v18203: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:08.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:35:07 smithi153 ceph-mon[70044]: pgmap v18203: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:10.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:35:09 smithi083 ceph-mon[79930]: pgmap v18204: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:10.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:35:09 smithi083 ceph-mon[77242]: pgmap v18204: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:10.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:35:09 smithi153 ceph-mon[70044]: pgmap v18204: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:11.898 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:35:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:35:11] "GET /metrics HTTP/1.1" 200 225246 "" "Prometheus/2.33.4" 2022-09-29T15:35:12.210 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:35:11 smithi083 ceph-mon[77242]: pgmap v18205: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:12.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:35:11 smithi083 ceph-mon[79930]: pgmap v18205: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:12.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:35:11 smithi153 ceph-mon[70044]: pgmap v18205: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:12.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:35:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:35:12.442Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:35:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:35:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:35:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:35:13.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:35:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:35:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:35:14.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:35:13 smithi153 ceph-mon[70044]: pgmap v18206: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:14.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:35:13 smithi083 ceph-mon[79930]: pgmap v18206: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:14.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:35:13 smithi083 ceph-mon[77242]: pgmap v18206: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:16.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:35:15 smithi083 ceph-mon[77242]: pgmap v18207: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:16.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:35:15 smithi083 ceph-mon[79930]: pgmap v18207: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:16.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:35:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:35:15.799Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:35:16.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:35:15 smithi153 ceph-mon[70044]: pgmap v18207: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:16.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:35:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:35:16.539Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:35:18.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:35:17 smithi083 ceph-mon[79930]: pgmap v18208: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:18.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:35:17 smithi083 ceph-mon[77242]: pgmap v18208: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:18.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:35:17 smithi153 ceph-mon[70044]: pgmap v18208: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:20.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:35:19 smithi083 ceph-mon[79930]: pgmap v18209: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:20.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:35:19 smithi083 ceph-mon[77242]: pgmap v18209: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:20.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:35:19 smithi153 ceph-mon[70044]: pgmap v18209: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:21.901 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:35:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:35:21] "GET /metrics HTTP/1.1" 200 225246 "" "Prometheus/2.33.4" 2022-09-29T15:35:22.210 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:35:21 smithi083 ceph-mon[77242]: pgmap v18210: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:22.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:35:21 smithi083 ceph-mon[79930]: pgmap v18210: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:22.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:35:21 smithi153 ceph-mon[70044]: pgmap v18210: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:22.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:35:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:35:22.443Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:35:22.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:35:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:35:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:35:23.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:35:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:35:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:35:24.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:35:23 smithi153 ceph-mon[70044]: pgmap v18211: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:24.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:35:23 smithi083 ceph-mon[79930]: pgmap v18211: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:24.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:35:23 smithi083 ceph-mon[77242]: pgmap v18211: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:25.725 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:35:26.017 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:35:26.018 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 5m ago 10h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:35:26.018 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 3m ago 10h 63.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:35:26.018 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 5m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:35:26.018 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 3m ago 10h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:35:26.019 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 3m ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:35:26.019 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 5m ago 10h 1029M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:35:26.019 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 5m ago 10h 2196M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:35:26.019 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 3m ago 10h 957M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:35:26.019 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 5m ago 10h 1319M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:35:26.020 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 5m ago 10h 24.3M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:35:26.020 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 3m ago 10h 29.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:35:26.020 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 5m ago 10h 1595M 2148M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:35:26.020 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 5m ago 10h 1581M 2148M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:35:26.021 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 5m ago 10h 1228M 2148M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:35:26.021 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 5m ago 10h 1550M 2148M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:35:26.021 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 3m ago 10h 1358M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:35:26.021 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 3m ago 10h 1707M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:35:26.021 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 3m ago 10h 1524M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:35:26.022 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 3m ago 10h 1814M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:35:26.022 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 3m ago 10h 177M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:35:26.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:35:25 smithi083 ceph-mon[79930]: pgmap v18212: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:26.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:35:25 smithi083 ceph-mon[77242]: pgmap v18212: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:26.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:35:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:35:25.799Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:35:26.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:35:25 smithi153 ceph-mon[70044]: pgmap v18212: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:26.375 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:35:26.376 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:35:26.376 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:35:26.376 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:35:26.376 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:35:26.377 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:35:26.377 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:35:26.377 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:35:26.377 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:35:26.378 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:35:26.378 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:35:26.378 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:35:26.378 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:35:26.379 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:35:26.379 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:35:26.379 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:35:26.379 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:35:26.379 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:35:26.899 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:35:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:35:26.540Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:35:27.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:35:26 smithi083 ceph-mon[79930]: from='client.65239 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:35:27.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:35:26 smithi083 ceph-mon[79930]: from='client.55314 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:35:27.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:35:26 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3046371712' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:35:27.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:35:26 smithi083 ceph-mon[77242]: from='client.65239 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:35:27.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:35:26 smithi083 ceph-mon[77242]: from='client.55314 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:35:27.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:35:26 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3046371712' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:35:27.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:35:26 smithi153 ceph-mon[70044]: from='client.65239 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:35:27.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:35:26 smithi153 ceph-mon[70044]: from='client.55314 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:35:27.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:35:26 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3046371712' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:35:28.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:35:27 smithi083 ceph-mon[79930]: pgmap v18213: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:28.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:35:27 smithi083 ceph-mon[77242]: pgmap v18213: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:28.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:35:27 smithi153 ceph-mon[70044]: pgmap v18213: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:30.210 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:35:29 smithi083 ceph-mon[77242]: pgmap v18214: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:30.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:35:29 smithi083 ceph-mon[79930]: pgmap v18214: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:30.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:35:29 smithi153 ceph-mon[70044]: pgmap v18214: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:31.906 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:35:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:35:31] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T15:35:32.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:35:31 smithi083 ceph-mon[79930]: pgmap v18215: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:32.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:35:31 smithi083 ceph-mon[77242]: pgmap v18215: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:32.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:35:31 smithi153 ceph-mon[70044]: pgmap v18215: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:32.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:35:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:35:32.443Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:35:32.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:35:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:35:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:35:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:35:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:35:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:35:33.742 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:35:33 smithi153 ceph-mon[70044]: pgmap v18216: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:34.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:35:33 smithi083 ceph-mon[77242]: pgmap v18216: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:34.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:35:33 smithi083 ceph-mon[79930]: pgmap v18216: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:36.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:35:35 smithi083 ceph-mon[79930]: pgmap v18217: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:36.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:35:35 smithi083 ceph-mon[77242]: pgmap v18217: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:36.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:35:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:35:35.801Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:35:36.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:35:35 smithi153 ceph-mon[70044]: pgmap v18217: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:36.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:35:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:35:36.540Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:35:38.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:35:37 smithi083 ceph-mon[77242]: pgmap v18218: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:38.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:35:37 smithi083 ceph-mon[79930]: pgmap v18218: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:38.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:35:37 smithi153 ceph-mon[70044]: pgmap v18218: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:40.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:35:39 smithi083 ceph-mon[77242]: pgmap v18219: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:40.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:35:39 smithi083 ceph-mon[79930]: pgmap v18219: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:40.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:35:39 smithi153 ceph-mon[70044]: pgmap v18219: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:41.907 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:35:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:35:41] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T15:35:42.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:35:41 smithi083 ceph-mon[79930]: pgmap v18220: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:42.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:35:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:35:42.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:35:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:35:42.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:35:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:35:42.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:35:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:35:42.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:35:41 smithi083 ceph-mon[77242]: pgmap v18220: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:42.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:35:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:35:42.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:35:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:35:42.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:35:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:35:42.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:35:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:35:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:35:41 smithi153 ceph-mon[70044]: pgmap v18220: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:35:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:35:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:35:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:35:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:35:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:35:42.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:35:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:35:42.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:35:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:35:42.444Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:35:42.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:35:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:35:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:35:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:35:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:35:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:35:44.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:35:43 smithi153 ceph-mon[70044]: pgmap v18221: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:44.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:35:43 smithi083 ceph-mon[79930]: pgmap v18221: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:44.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:35:43 smithi083 ceph-mon[77242]: pgmap v18221: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:45.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:35:44 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:35:45.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:35:44 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:35:45.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:35:44 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:35:45.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:35:44 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:35:45.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:35:44 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:35:45.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:35:44 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:35:45.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:35:44 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:35:45.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:35:44 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:35:45.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:35:44 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:35:46.210 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:35:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:35:45.801Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:35:46.710 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:35:46 smithi083 ceph-mon[77242]: pgmap v18222: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:46.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:35:46 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:35:46.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:35:46 smithi083 ceph-mon[79930]: pgmap v18222: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:46.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:35:46 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:35:46.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:35:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:35:46.540Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:35:46.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:35:46 smithi153 ceph-mon[70044]: pgmap v18222: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:46.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:35:46 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:35:48.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:35:48 smithi083 ceph-mon[77242]: pgmap v18223: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:48.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:35:48 smithi083 ceph-mon[79930]: pgmap v18223: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:48.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:35:48 smithi153 ceph-mon[70044]: pgmap v18223: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:50.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:35:50 smithi083 ceph-mon[79930]: pgmap v18224: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:50.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:35:50 smithi083 ceph-mon[77242]: pgmap v18224: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:50.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:35:50 smithi153 ceph-mon[70044]: pgmap v18224: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:51.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:35:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:35:51] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T15:35:52.710 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:35:52 smithi083 ceph-mon[77242]: pgmap v18225: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:52.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:35:52 smithi083 ceph-mon[79930]: pgmap v18225: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:52.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:35:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:35:52.445Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:35:52.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:35:52 smithi153 ceph-mon[70044]: pgmap v18225: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:35:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:35:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:35:53.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:35:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:35:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:35:54.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:35:53 smithi153 ceph-mon[70044]: pgmap v18226: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:54.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:35:53 smithi083 ceph-mon[79930]: pgmap v18226: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:54.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:35:53 smithi083 ceph-mon[77242]: pgmap v18226: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:56.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:35:55 smithi083 ceph-mon[79930]: pgmap v18227: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:56.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:35:55 smithi083 ceph-mon[77242]: pgmap v18227: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:56.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:35:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:35:55.802Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:35:56.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:35:55 smithi153 ceph-mon[70044]: pgmap v18227: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:56.701 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:35:56.911 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:35:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:35:56.540Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:35:56.994 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:35:56.995 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 6m ago 10h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:35:56.995 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 4m ago 10h 63.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:35:56.995 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 6m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:35:56.995 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 4m ago 10h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:35:56.996 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 4m ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:35:56.996 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 6m ago 10h 1029M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:35:56.996 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 6m ago 10h 2196M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:35:56.997 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 4m ago 10h 957M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:35:56.997 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 6m ago 10h 1319M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:35:56.997 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 6m ago 10h 24.3M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:35:56.997 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 4m ago 10h 29.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:35:56.998 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 6m ago 10h 1595M 2148M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:35:56.998 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 6m ago 10h 1581M 2148M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:35:56.998 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 6m ago 10h 1228M 2148M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:35:56.999 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 6m ago 10h 1550M 2148M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:35:56.999 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 4m ago 10h 1358M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:35:57.000 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 4m ago 10h 1707M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:35:57.000 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 4m ago 10h 1524M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:35:57.001 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 4m ago 10h 1814M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:35:57.001 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 4m ago 10h 177M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:35:57.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:35:56 smithi083 ceph-mon[79930]: from='client.55326 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:35:57.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:35:56 smithi083 ceph-mon[77242]: from='client.55326 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:35:57.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:35:56 smithi153 ceph-mon[70044]: from='client.55326 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:35:57.352 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:35:57.353 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:35:57.353 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:35:57.353 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:35:57.353 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:35:57.354 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:35:57.354 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:35:57.354 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:35:57.354 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:35:57.355 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:35:57.355 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:35:57.355 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:35:57.355 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:35:57.356 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:35:57.356 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:35:57.356 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:35:57.356 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:35:57.357 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:35:58.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:35:57 smithi083 ceph-mon[79930]: pgmap v18228: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:58.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:35:57 smithi083 ceph-mon[79930]: from='client.65263 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:35:58.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:35:57 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2664064102' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:35:58.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:35:57 smithi083 ceph-mon[77242]: pgmap v18228: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:58.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:35:57 smithi083 ceph-mon[77242]: from='client.65263 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:35:58.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:35:57 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2664064102' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:35:58.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:35:57 smithi153 ceph-mon[70044]: pgmap v18228: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:35:58.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:35:57 smithi153 ceph-mon[70044]: from='client.65263 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:35:58.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:35:57 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2664064102' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:36:00.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:35:59 smithi083 ceph-mon[79930]: pgmap v18229: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:00.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:35:59 smithi083 ceph-mon[77242]: pgmap v18229: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:35:59 smithi153 ceph-mon[70044]: pgmap v18229: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:01.920 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:36:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:36:01] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T15:36:02.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:36:01 smithi083 ceph-mon[77242]: pgmap v18230: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:02.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:36:01 smithi083 ceph-mon[79930]: pgmap v18230: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:02.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:36:01 smithi153 ceph-mon[70044]: pgmap v18230: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:02.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:36:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:36:02.445Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:36:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:36:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:36:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:36:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:36:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:36:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:36:04.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:36:03 smithi153 ceph-mon[70044]: pgmap v18231: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:04.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:36:03 smithi083 ceph-mon[79930]: pgmap v18231: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:04.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:36:03 smithi083 ceph-mon[77242]: pgmap v18231: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:06.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:36:05 smithi083 ceph-mon[79930]: pgmap v18232: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:06.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:36:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:36:05.803Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:36:06.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:36:05 smithi083 ceph-mon[77242]: pgmap v18232: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:06.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:36:05 smithi153 ceph-mon[70044]: pgmap v18232: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:06.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:36:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:36:06.543Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:36:08.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:36:07 smithi083 ceph-mon[77242]: pgmap v18233: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:08.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:36:07 smithi083 ceph-mon[79930]: pgmap v18233: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:08.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:36:07 smithi153 ceph-mon[70044]: pgmap v18233: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:10.210 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:36:09 smithi083 ceph-mon[77242]: pgmap v18234: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:10.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:36:09 smithi083 ceph-mon[79930]: pgmap v18234: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:10.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:36:09 smithi153 ceph-mon[70044]: pgmap v18234: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:11.916 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:36:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:36:11] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T15:36:12.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:36:11 smithi083 ceph-mon[79930]: pgmap v18235: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:12.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:36:11 smithi083 ceph-mon[77242]: pgmap v18235: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:12.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:36:11 smithi153 ceph-mon[70044]: pgmap v18235: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:12.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:36:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:36:12.445Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:36:12.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:36:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:36:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:36:13.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:36:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:36:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:36:14.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:36:13 smithi153 ceph-mon[70044]: pgmap v18236: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:14.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:36:13 smithi083 ceph-mon[77242]: pgmap v18236: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:14.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:36:13 smithi083 ceph-mon[79930]: pgmap v18236: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:16.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:36:15 smithi083 ceph-mon[79930]: pgmap v18237: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:16.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:36:15 smithi083 ceph-mon[77242]: pgmap v18237: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:16.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:36:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:36:15.804Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:36:16.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:36:15 smithi153 ceph-mon[70044]: pgmap v18237: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:16.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:36:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:36:16.543Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:36:18.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:36:17 smithi083 ceph-mon[79930]: pgmap v18238: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:18.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:36:17 smithi083 ceph-mon[77242]: pgmap v18238: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:18.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:36:17 smithi153 ceph-mon[70044]: pgmap v18238: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:20.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:36:19 smithi083 ceph-mon[79930]: pgmap v18239: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:20.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:36:19 smithi083 ceph-mon[77242]: pgmap v18239: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:20.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:36:19 smithi153 ceph-mon[70044]: pgmap v18239: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:21.919 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:36:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:36:21] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T15:36:22.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:36:21 smithi083 ceph-mon[77242]: pgmap v18240: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:22.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:36:21 smithi083 ceph-mon[79930]: pgmap v18240: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:22.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:36:21 smithi153 ceph-mon[70044]: pgmap v18240: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:22.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:36:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:36:22.447Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:36:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:36:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:36:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:36:23.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:36:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:36:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:36:24.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:36:23 smithi153 ceph-mon[70044]: pgmap v18241: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:24.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:36:23 smithi083 ceph-mon[79930]: pgmap v18241: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:24.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:36:23 smithi083 ceph-mon[77242]: pgmap v18241: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:26.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:36:25 smithi083 ceph-mon[79930]: pgmap v18242: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:26.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:36:25 smithi083 ceph-mon[77242]: pgmap v18242: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:26.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:36:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:36:25.804Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:36:26.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:36:25 smithi153 ceph-mon[70044]: pgmap v18242: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:26.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:36:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:36:26.544Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:36:27.680 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:36:27.971 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:36:27.972 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 6m ago 10h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:36:27.972 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 4m ago 10h 63.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:36:27.972 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 6m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:36:27.972 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 4m ago 10h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:36:27.973 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 4m ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:36:27.973 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 6m ago 10h 1029M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:36:27.973 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 6m ago 10h 2196M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:36:27.973 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 4m ago 10h 957M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:36:27.974 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 6m ago 10h 1319M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:36:27.974 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 6m ago 10h 24.3M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:36:27.974 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 4m ago 10h 29.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:36:27.974 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 6m ago 10h 1595M 2148M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:36:27.975 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 6m ago 10h 1581M 2148M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:36:27.975 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 6m ago 10h 1228M 2148M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:36:27.975 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 6m ago 10h 1550M 2148M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:36:27.975 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 4m ago 10h 1358M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:36:27.975 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 4m ago 10h 1707M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:36:27.976 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 4m ago 10h 1524M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:36:27.976 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 4m ago 10h 1814M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:36:27.976 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 4m ago 10h 177M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:36:28.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:36:27 smithi083 ceph-mon[79930]: pgmap v18243: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:28.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:36:27 smithi083 ceph-mon[79930]: from='client.65275 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:36:28.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:36:27 smithi083 ceph-mon[77242]: pgmap v18243: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:28.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:36:27 smithi083 ceph-mon[77242]: from='client.65275 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:36:28.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:36:27 smithi153 ceph-mon[70044]: pgmap v18243: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:28.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:36:27 smithi153 ceph-mon[70044]: from='client.65275 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:36:28.333 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:36:28.333 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:36:28.334 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:36:28.334 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:36:28.334 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:36:28.334 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:36:28.335 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:36:28.335 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:36:28.335 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:36:28.335 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:36:28.336 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:36:28.336 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:36:28.336 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:36:28.336 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:36:28.337 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:36:28.337 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:36:28.337 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:36:28.337 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:36:29.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:36:28 smithi083 ceph-mon[79930]: from='client.65281 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:36:29.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:36:28 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2250840201' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:36:29.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:36:28 smithi083 ceph-mon[77242]: from='client.65281 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:36:29.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:36:28 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2250840201' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:36:29.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:36:28 smithi153 ceph-mon[70044]: from='client.65281 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:36:29.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:36:28 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2250840201' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:36:30.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:36:29 smithi083 ceph-mon[79930]: pgmap v18244: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:30.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:36:29 smithi083 ceph-mon[77242]: pgmap v18244: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:30.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:36:29 smithi153 ceph-mon[70044]: pgmap v18244: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:31.927 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:36:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:36:31] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T15:36:32.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:36:31 smithi083 ceph-mon[79930]: pgmap v18245: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:32.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:36:31 smithi083 ceph-mon[77242]: pgmap v18245: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:32.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:36:31 smithi153 ceph-mon[70044]: pgmap v18245: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:32.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:36:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:36:32.447Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:36:32.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:36:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:36:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:36:33.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:36:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:36:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:36:34.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:36:33 smithi153 ceph-mon[70044]: pgmap v18246: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:34.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:36:33 smithi083 ceph-mon[79930]: pgmap v18246: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:34.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:36:33 smithi083 ceph-mon[77242]: pgmap v18246: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:36.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:36:35 smithi083 ceph-mon[79930]: pgmap v18247: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:36.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:36:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:36:35.805Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:36:36.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:36:35 smithi083 ceph-mon[77242]: pgmap v18247: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:36.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:36:35 smithi153 ceph-mon[70044]: pgmap v18247: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:36.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:36:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:36:36.545Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:36:38.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:36:37 smithi083 ceph-mon[79930]: pgmap v18248: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:38.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:36:37 smithi083 ceph-mon[77242]: pgmap v18248: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:38.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:36:37 smithi153 ceph-mon[70044]: pgmap v18248: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:40.210 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:36:39 smithi083 ceph-mon[77242]: pgmap v18249: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:40.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:36:39 smithi083 ceph-mon[79930]: pgmap v18249: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:40.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:36:39 smithi153 ceph-mon[70044]: pgmap v18249: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:41.928 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:36:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:36:41] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T15:36:42.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:36:41 smithi083 ceph-mon[79930]: pgmap v18250: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:42.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:36:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:36:42.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:36:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:36:42.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:36:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:36:42.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:36:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:36:42.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:36:41 smithi083 ceph-mon[77242]: pgmap v18250: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:42.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:36:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:36:42.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:36:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:36:42.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:36:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:36:42.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:36:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:36:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:36:41 smithi153 ceph-mon[70044]: pgmap v18250: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:36:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:36:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:36:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:36:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:36:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:36:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:36:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:36:42.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:36:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:36:42.448Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:36:42.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:36:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:36:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:36:43.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:36:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:36:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:36:44.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:36:43 smithi153 ceph-mon[70044]: pgmap v18251: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:44.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:36:43 smithi083 ceph-mon[79930]: pgmap v18251: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:44.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:36:43 smithi083 ceph-mon[77242]: pgmap v18251: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:46.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:36:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:36:45.804Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:36:46.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:36:45 smithi083 ceph-mon[79930]: pgmap v18252: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:46.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:36:45 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:36:46.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:36:45 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:36:46.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:36:45 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:36:46.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:36:45 smithi083 ceph-mon[77242]: pgmap v18252: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:46.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:36:45 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:36:46.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:36:45 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:36:46.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:36:45 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:36:46.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:36:45 smithi153 ceph-mon[70044]: pgmap v18252: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:46.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:36:45 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:36:46.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:36:45 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:36:46.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:36:45 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:36:46.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:36:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:36:46.545Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:36:47.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:36:47 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:36:47.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:36:47 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:36:47.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:36:47 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:36:48.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:36:48 smithi083 ceph-mon[77242]: pgmap v18253: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:48.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:36:48 smithi083 ceph-mon[79930]: pgmap v18253: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:48.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:36:48 smithi153 ceph-mon[70044]: pgmap v18253: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:50.460 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:36:50 smithi083 ceph-mon[77242]: pgmap v18254: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:50.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:36:50 smithi083 ceph-mon[79930]: pgmap v18254: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:50.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:36:50 smithi153 ceph-mon[70044]: pgmap v18254: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:36:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:36:51] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T15:36:52.449 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:36:52 smithi083 ceph-mon[79930]: pgmap v18255: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:52.450 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:36:52 smithi083 ceph-mon[77242]: pgmap v18255: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:36:52 smithi153 ceph-mon[70044]: pgmap v18255: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:52.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:36:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:36:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:36:52.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:36:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:36:52.449Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:36:53.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:36:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:36:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:36:54.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:36:53 smithi153 ceph-mon[70044]: pgmap v18256: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:54.210 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:36:53 smithi083 ceph-mon[77242]: pgmap v18256: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:54.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:36:53 smithi083 ceph-mon[79930]: pgmap v18256: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:56.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:36:55 smithi083 ceph-mon[79930]: pgmap v18257: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:56.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:36:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:36:55.806Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:36:56.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:36:55 smithi083 ceph-mon[77242]: pgmap v18257: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:56.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:36:55 smithi153 ceph-mon[70044]: pgmap v18257: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:56.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:36:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:36:56.546Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:36:58.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:36:57 smithi083 ceph-mon[79930]: pgmap v18258: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:58.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:36:57 smithi083 ceph-mon[77242]: pgmap v18258: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:58.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:36:57 smithi153 ceph-mon[70044]: pgmap v18258: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:36:58.663 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:36:58.944 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:36:58.945 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 7m ago 10h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:36:58.945 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 5m ago 10h 63.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:36:58.945 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 7m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:36:58.946 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 5m ago 10h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:36:58.946 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 5m ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:36:58.946 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 7m ago 10h 1029M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:36:58.947 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 7m ago 10h 2196M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:36:58.947 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 5m ago 10h 957M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:36:58.947 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 7m ago 10h 1319M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:36:58.947 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 7m ago 10h 24.3M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:36:58.948 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 5m ago 10h 29.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:36:58.948 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 7m ago 10h 1595M 2148M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:36:58.948 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 7m ago 10h 1581M 2148M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:36:58.948 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 7m ago 10h 1228M 2148M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:36:58.949 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 7m ago 10h 1550M 2148M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:36:58.949 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 5m ago 10h 1358M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:36:58.949 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 5m ago 10h 1707M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:36:58.950 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 5m ago 10h 1524M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:36:58.950 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 5m ago 10h 1814M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:36:58.950 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 5m ago 10h 177M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:36:59.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:36:58 smithi083 ceph-mon[79930]: from='client.55362 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:36:59.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:36:58 smithi083 ceph-mon[77242]: from='client.55362 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:36:59.298 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:36:59.299 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:36:59.299 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:36:59.299 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:36:59.299 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:36:59.300 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:36:59.300 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:36:59.300 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:36:59.300 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:36:59.301 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:36:59.301 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:36:59.301 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:36:59.301 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:36:59.302 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:36:59.302 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:36:59.302 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:36:59.302 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:36:59.302 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:36:59.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:36:58 smithi153 ceph-mon[70044]: from='client.55362 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:37:00.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:36:59 smithi083 ceph-mon[79930]: pgmap v18259: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:37:00.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:36:59 smithi083 ceph-mon[79930]: from='client.65299 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:37:00.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:36:59 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3196931591' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:37:00.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:36:59 smithi083 ceph-mon[77242]: pgmap v18259: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:37:00.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:36:59 smithi083 ceph-mon[77242]: from='client.65299 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:37:00.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:36:59 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3196931591' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:37:00.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:36:59 smithi153 ceph-mon[70044]: pgmap v18259: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:37:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:36:59 smithi153 ceph-mon[70044]: from='client.65299 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:37:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:36:59 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3196931591' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:37:01.935 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:37:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:37:01] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T15:37:02.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:37:01 smithi083 ceph-mon[77242]: pgmap v18260: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:37:02.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:37:01 smithi083 ceph-mon[79930]: pgmap v18260: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:37:02.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:37:01 smithi153 ceph-mon[70044]: pgmap v18260: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:37:02.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:37:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:37:02.449Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:37:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:37:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:37:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:37:03.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:37:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:37:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:37:04.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:37:03 smithi153 ceph-mon[70044]: pgmap v18261: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:37:04.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:37:03 smithi083 ceph-mon[79930]: pgmap v18261: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:37:04.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:37:03 smithi083 ceph-mon[77242]: pgmap v18261: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:37:06.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:37:05 smithi083 ceph-mon[79930]: pgmap v18262: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:37:06.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:37:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:37:05.807Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:37:06.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:37:05 smithi083 ceph-mon[77242]: pgmap v18262: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:37:06.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:37:05 smithi153 ceph-mon[70044]: pgmap v18262: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:37:06.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:37:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:37:06.548Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:37:08.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:37:07 smithi083 ceph-mon[79930]: pgmap v18263: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:37:08.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:37:07 smithi083 ceph-mon[77242]: pgmap v18263: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:37:08.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:37:07 smithi153 ceph-mon[70044]: pgmap v18263: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:37:10.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:37:09 smithi083 ceph-mon[79930]: pgmap v18264: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:37:10.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:37:09 smithi083 ceph-mon[77242]: pgmap v18264: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:37:10.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:37:09 smithi153 ceph-mon[70044]: pgmap v18264: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:37:11.936 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:37:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:37:11] "GET /metrics HTTP/1.1" 200 225248 "" "Prometheus/2.33.4" 2022-09-29T15:37:12.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:37:11 smithi083 ceph-mon[79930]: pgmap v18265: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:37:12.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:37:11 smithi083 ceph-mon[77242]: pgmap v18265: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:37:12.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:37:11 smithi153 ceph-mon[70044]: pgmap v18265: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:37:12.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:37:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:37:12.450Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:37:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:37:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:37:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:37:13.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:37:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:37:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:37:14.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:37:13 smithi153 ceph-mon[70044]: pgmap v18266: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:37:14.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:37:13 smithi083 ceph-mon[79930]: pgmap v18266: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:37:14.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:37:13 smithi083 ceph-mon[77242]: pgmap v18266: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:37:16.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:37:15 smithi083 ceph-mon[77242]: pgmap v18267: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:37:16.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:37:15 smithi083 ceph-mon[79930]: pgmap v18267: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:37:16.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:37:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:37:15.808Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:37:16.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:37:15 smithi153 ceph-mon[70044]: pgmap v18267: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:37:16.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:37:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:37:16.548Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:37:18.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:37:17 smithi083 ceph-mon[79930]: pgmap v18268: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:37:18.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:37:17 smithi083 ceph-mon[77242]: pgmap v18268: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:37:18.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:37:17 smithi153 ceph-mon[70044]: pgmap v18268: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:37:20.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:37:19 smithi083 ceph-mon[79930]: pgmap v18269: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:37:20.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:37:19 smithi083 ceph-mon[77242]: pgmap v18269: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:37:20.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:37:19 smithi153 ceph-mon[70044]: pgmap v18269: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:37:21.939 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:37:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:37:21] "GET /metrics HTTP/1.1" 200 225248 "" "Prometheus/2.33.4" 2022-09-29T15:37:22.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:37:21 smithi083 ceph-mon[79930]: pgmap v18270: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:37:22.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:37:21 smithi083 ceph-mon[77242]: pgmap v18270: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:37:22.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:37:21 smithi153 ceph-mon[70044]: pgmap v18270: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:37:22.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:37:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:37:22.451Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:37:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:37:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:37:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:37:23.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:37:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:37:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:37:24.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:37:23 smithi153 ceph-mon[70044]: pgmap v18271: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:37:24.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:37:23 smithi083 ceph-mon[77242]: pgmap v18271: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:37:24.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:37:23 smithi083 ceph-mon[79930]: pgmap v18271: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:37:26.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:37:25 smithi083 ceph-mon[79930]: pgmap v18272: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:37:26.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:37:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:37:25.808Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:37:26.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:37:25 smithi083 ceph-mon[77242]: pgmap v18272: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:37:26.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:37:25 smithi153 ceph-mon[70044]: pgmap v18272: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:37:26.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:37:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:37:26.549Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:37:28.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:37:27 smithi083 ceph-mon[79930]: pgmap v18273: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:37:28.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:37:27 smithi083 ceph-mon[77242]: pgmap v18273: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:37:28.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:37:27 smithi153 ceph-mon[70044]: pgmap v18273: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:37:29.621 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:37:29.918 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:37:29.919 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 7m ago 10h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:37:29.920 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 5m ago 10h 63.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:37:29.920 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 7m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:37:29.920 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 5m ago 10h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:37:29.921 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 5m ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:37:29.921 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 7m ago 10h 1029M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:37:29.922 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 7m ago 10h 2196M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:37:29.922 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 5m ago 10h 957M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:37:29.923 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 7m ago 10h 1319M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:37:29.923 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 7m ago 10h 24.3M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:37:29.923 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 5m ago 10h 29.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:37:29.924 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 7m ago 10h 1595M 2148M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:37:29.924 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 7m ago 10h 1581M 2148M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:37:29.925 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 7m ago 10h 1228M 2148M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:37:29.926 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 7m ago 10h 1550M 2148M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:37:29.926 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 5m ago 10h 1358M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:37:29.926 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 5m ago 10h 1707M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:37:29.927 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 5m ago 10h 1524M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:37:29.927 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 5m ago 10h 1814M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:37:29.927 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 5m ago 10h 177M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:37:30.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:37:29 smithi083 ceph-mon[79930]: pgmap v18274: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:37:30.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:37:29 smithi083 ceph-mon[79930]: from='client.65311 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:37:30.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:37:29 smithi083 ceph-mon[77242]: pgmap v18274: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:37:30.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:37:29 smithi083 ceph-mon[77242]: from='client.65311 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:37:30.274 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:37:30.275 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:37:30.275 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:37:30.275 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:37:30.276 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:37:30.276 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:37:30.276 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:37:30.277 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:37:30.277 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:37:30.277 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:37:30.278 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:37:30.278 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:37:30.278 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:37:30.278 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:37:30.279 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:37:30.279 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:37:30.279 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:37:30.279 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:37:30.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:37:29 smithi153 ceph-mon[70044]: pgmap v18274: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:37:30.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:37:29 smithi153 ceph-mon[70044]: from='client.65311 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:37:31.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:37:30 smithi083 ceph-mon[79930]: from='client.55386 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:37:31.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:37:30 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1333766797' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:37:31.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:37:30 smithi083 ceph-mon[77242]: from='client.55386 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:37:31.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:37:30 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1333766797' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:37:31.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:37:30 smithi153 ceph-mon[70044]: from='client.55386 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:37:31.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:37:30 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1333766797' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:37:31.954 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:37:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:37:31] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T15:37:32.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:37:31 smithi083 ceph-mon[79930]: pgmap v18275: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:37:32.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:37:31 smithi083 ceph-mon[77242]: pgmap v18275: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:37:32.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:37:31 smithi153 ceph-mon[70044]: pgmap v18275: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:37:32.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:37:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:37:32.452Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:37:32.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:37:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:37:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:37:33.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:37:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:37:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:37:34.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:37:33 smithi153 ceph-mon[70044]: pgmap v18276: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:37:34.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:37:33 smithi083 ceph-mon[79930]: pgmap v18276: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:37:34.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:37:33 smithi083 ceph-mon[77242]: pgmap v18276: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:37:36.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:37:35 smithi083 ceph-mon[79930]: pgmap v18277: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:37:36.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:37:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:37:35.809Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:37:36.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:37:35 smithi083 ceph-mon[77242]: pgmap v18277: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:37:36.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:37:35 smithi153 ceph-mon[70044]: pgmap v18277: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:37:36.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:37:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:37:36.549Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:37:38.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:37:37 smithi083 ceph-mon[79930]: pgmap v18278: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:37:38.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:37:37 smithi083 ceph-mon[77242]: pgmap v18278: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:37:38.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:37:37 smithi153 ceph-mon[70044]: pgmap v18278: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:37:40.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:37:39 smithi083 ceph-mon[79930]: pgmap v18279: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:37:40.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:37:39 smithi083 ceph-mon[77242]: pgmap v18279: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:37:40.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:37:39 smithi153 ceph-mon[70044]: pgmap v18279: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:37:41.945 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:37:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:37:41] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T15:37:42.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:37:41 smithi083 ceph-mon[79930]: pgmap v18280: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:37:42.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:37:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:37:42.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:37:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:37:42.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:37:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:37:42.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:37:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:37:42.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:37:41 smithi083 ceph-mon[77242]: pgmap v18280: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:37:42.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:37:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:37:42.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:37:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:37:42.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:37:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:37:42.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:37:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:37:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:37:41 smithi153 ceph-mon[70044]: pgmap v18280: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:37:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:37:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:37:42.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:37:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:37:42.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:37:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:37:42.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:37:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:37:42.721 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:37:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:37:42.452Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:37:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:37:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:37:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:37:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:37:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:37:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:37:44.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:37:43 smithi153 ceph-mon[70044]: pgmap v18281: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:37:44.210 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:37:43 smithi083 ceph-mon[77242]: pgmap v18281: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:37:44.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:37:43 smithi083 ceph-mon[79930]: pgmap v18281: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:37:46.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:37:45 smithi083 ceph-mon[79930]: pgmap v18282: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:37:46.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:37:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:37:45.809Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:37:46.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:37:45 smithi083 ceph-mon[77242]: pgmap v18282: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:37:46.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:37:45 smithi153 ceph-mon[70044]: pgmap v18282: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:37:46.941 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:37:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:37:46.549Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:37:47.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:37:46 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:37:47.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:37:46 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:37:47.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:37:46 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:37:47.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:37:46 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:37:47.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:37:46 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:37:47.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:37:46 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:37:47.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:37:46 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:37:47.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:37:46 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:37:47.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:37:46 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:37:47.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:37:46 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:37:47.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:37:46 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:37:47.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:37:46 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:37:47.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:37:46 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:37:47.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:37:46 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:37:47.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:37:46 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:37:47.218 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:37:46 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:37:47.218 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:37:46 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:37:47.218 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:37:46 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:37:47.218 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:37:46 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:37:47.219 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:37:46 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:37:47.219 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:37:46 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:37:47.219 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:37:46 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:37:47.220 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:37:46 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:37:47.220 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:37:46 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:37:47.221 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:37:46 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:37:47.221 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:37:46 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:37:47.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:37:46 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:37:47.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:37:46 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:37:47.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:37:46 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:37:47.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:37:46 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:37:47.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:37:46 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:37:47.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:37:46 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:37:47.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:37:46 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:37:47.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:37:46 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:37:47.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:37:46 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:37:47.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:37:46 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:37:47.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:37:46 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:37:47.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:37:46 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:37:47.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:37:46 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:37:48.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:37:47 smithi083 ceph-mon[79930]: Adjusting osd_memory_target on smithi083 to 2133M 2022-09-29T15:37:48.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:37:47 smithi083 ceph-mon[79930]: pgmap v18283: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:37:48.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:37:47 smithi083 ceph-mon[77242]: Adjusting osd_memory_target on smithi083 to 2133M 2022-09-29T15:37:48.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:37:47 smithi083 ceph-mon[77242]: pgmap v18283: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:37:48.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:37:47 smithi153 ceph-mon[70044]: Adjusting osd_memory_target on smithi083 to 2133M 2022-09-29T15:37:48.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:37:47 smithi153 ceph-mon[70044]: pgmap v18283: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:37:50.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:37:49 smithi083 ceph-mon[79930]: pgmap v18284: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:37:50.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:37:49 smithi083 ceph-mon[77242]: pgmap v18284: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:37:50.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:37:49 smithi153 ceph-mon[70044]: pgmap v18284: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:37:51.951 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:37:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:37:51] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T15:37:52.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:37:51 smithi083 ceph-mon[77242]: pgmap v18285: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:37:52.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:37:51 smithi083 ceph-mon[79930]: pgmap v18285: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:37:52.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:37:51 smithi153 ceph-mon[70044]: pgmap v18285: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:37:52.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:37:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:37:52.453Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:37:52.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:37:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:37:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:37:53.313 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:37:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:37:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:37:54.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:37:53 smithi153 ceph-mon[70044]: pgmap v18286: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:37:54.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:37:53 smithi083 ceph-mon[79930]: pgmap v18286: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:37:54.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:37:53 smithi083 ceph-mon[77242]: pgmap v18286: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:37:56.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:37:55 smithi083 ceph-mon[79930]: pgmap v18287: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:37:56.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:37:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:37:55.810Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:37:56.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:37:55 smithi083 ceph-mon[77242]: pgmap v18287: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:37:56.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:37:55 smithi153 ceph-mon[70044]: pgmap v18287: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:37:56.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:37:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:37:56.550Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:37:58.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:37:57 smithi083 ceph-mon[77242]: pgmap v18288: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:37:58.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:37:57 smithi083 ceph-mon[79930]: pgmap v18288: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:37:58.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:37:57 smithi153 ceph-mon[70044]: pgmap v18288: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:00.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:37:59 smithi083 ceph-mon[77242]: pgmap v18289: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:00.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:37:59 smithi083 ceph-mon[79930]: pgmap v18289: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:00.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:37:59 smithi153 ceph-mon[70044]: pgmap v18289: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:00.600 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:38:00.889 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:38:00.890 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 8m ago 10h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:38:00.890 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 6m ago 10h 63.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:38:00.891 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 8m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:38:00.891 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 6m ago 10h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:38:00.891 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 6m ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:38:00.891 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 8m ago 10h 1029M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:38:00.892 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 8m ago 10h 2196M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:38:00.892 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 6m ago 10h 957M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:38:00.892 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 8m ago 10h 1319M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:38:00.892 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 8m ago 10h 24.3M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:38:00.893 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 6m ago 10h 29.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:38:00.893 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 8m ago 10h 1595M 2133M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:38:00.893 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 8m ago 10h 1581M 2133M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:38:00.893 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 8m ago 10h 1228M 2133M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:38:00.893 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 8m ago 10h 1550M 2133M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:38:00.894 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 6m ago 10h 1358M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:38:00.894 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 6m ago 10h 1707M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:38:00.894 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 6m ago 10h 1524M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:38:00.895 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 6m ago 10h 1814M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:38:00.895 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 6m ago 10h 177M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:38:01.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:38:00 smithi083 ceph-mon[79930]: from='client.55398 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:38:01.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:38:00 smithi083 ceph-mon[77242]: from='client.55398 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:38:01.247 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:38:01.248 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:38:01.248 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:38:01.249 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:38:01.249 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:38:01.250 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:38:01.250 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:38:01.251 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:38:01.251 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:38:01.251 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:38:01.252 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:38:01.252 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:38:01.253 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:38:01.253 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:38:01.254 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:38:01.255 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:38:01.255 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:38:01.257 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:38:01.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:38:00 smithi153 ceph-mon[70044]: from='client.55398 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:38:01.958 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:38:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:38:01] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T15:38:02.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:38:01 smithi083 ceph-mon[79930]: from='client.65335 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:38:02.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:38:01 smithi083 ceph-mon[79930]: pgmap v18290: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:02.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:38:01 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4115738968' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:38:02.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:38:01 smithi083 ceph-mon[77242]: from='client.65335 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:38:02.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:38:01 smithi083 ceph-mon[77242]: pgmap v18290: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:02.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:38:01 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4115738968' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:38:02.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:38:01 smithi153 ceph-mon[70044]: from='client.65335 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:38:02.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:38:01 smithi153 ceph-mon[70044]: pgmap v18290: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:02.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:38:01 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4115738968' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:38:02.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:38:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:38:02.453Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:38:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:38:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:38:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:38:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:38:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:38:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:38:04.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:38:03 smithi153 ceph-mon[70044]: pgmap v18291: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:04.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:38:03 smithi083 ceph-mon[79930]: pgmap v18291: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:04.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:38:03 smithi083 ceph-mon[77242]: pgmap v18291: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:06.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:38:05 smithi083 ceph-mon[79930]: pgmap v18292: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:06.216 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:38:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:38:05.811Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:38:06.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:38:05 smithi083 ceph-mon[77242]: pgmap v18292: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:06.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:38:05 smithi153 ceph-mon[70044]: pgmap v18292: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:06.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:38:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:38:06.551Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:38:08.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:38:07 smithi083 ceph-mon[79930]: pgmap v18293: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:08.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:38:07 smithi083 ceph-mon[77242]: pgmap v18293: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:08.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:38:07 smithi153 ceph-mon[70044]: pgmap v18293: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:10.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:38:09 smithi083 ceph-mon[79930]: pgmap v18294: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:10.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:38:09 smithi083 ceph-mon[77242]: pgmap v18294: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:10.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:38:09 smithi153 ceph-mon[70044]: pgmap v18294: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:11.955 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:38:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:38:11] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T15:38:12.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:38:11 smithi083 ceph-mon[79930]: pgmap v18295: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:12.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:38:11 smithi083 ceph-mon[77242]: pgmap v18295: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:12.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:38:11 smithi153 ceph-mon[70044]: pgmap v18295: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:12.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:38:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:38:12.454Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:38:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:38:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:38:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:38:13.316 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:38:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:38:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:38:14.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:38:13 smithi153 ceph-mon[70044]: pgmap v18296: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:14.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:38:13 smithi083 ceph-mon[79930]: pgmap v18296: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:14.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:38:13 smithi083 ceph-mon[77242]: pgmap v18296: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:16.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:38:15 smithi083 ceph-mon[79930]: pgmap v18297: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:16.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:38:15 smithi083 ceph-mon[77242]: pgmap v18297: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:16.214 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:38:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:38:15.812Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:38:16.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:38:15 smithi153 ceph-mon[70044]: pgmap v18297: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:16.964 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:38:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:38:16.552Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:38:18.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:38:17 smithi083 ceph-mon[79930]: pgmap v18298: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:18.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:38:17 smithi083 ceph-mon[77242]: pgmap v18298: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:18.313 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:38:17 smithi153 ceph-mon[70044]: pgmap v18298: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:20.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:38:19 smithi083 ceph-mon[79930]: pgmap v18299: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:20.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:38:19 smithi083 ceph-mon[77242]: pgmap v18299: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:20.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:38:19 smithi153 ceph-mon[70044]: pgmap v18299: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:21.958 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:38:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:38:21] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T15:38:22.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:38:21 smithi083 ceph-mon[79930]: pgmap v18300: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:22.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:38:21 smithi083 ceph-mon[77242]: pgmap v18300: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:22.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:38:21 smithi153 ceph-mon[70044]: pgmap v18300: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:22.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:38:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:38:22.455Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:38:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:38:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:38:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:38:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:38:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:38:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:38:24.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:38:23 smithi153 ceph-mon[70044]: pgmap v18301: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:24.218 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:38:23 smithi083 ceph-mon[77242]: pgmap v18301: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:24.218 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:38:23 smithi083 ceph-mon[79930]: pgmap v18301: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:26.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:38:25 smithi083 ceph-mon[79930]: pgmap v18302: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:26.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:38:25 smithi083 ceph-mon[77242]: pgmap v18302: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:26.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:38:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:38:25.813Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:38:26.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:38:25 smithi153 ceph-mon[70044]: pgmap v18302: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:26.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:38:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:38:26.551Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:38:28.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:38:27 smithi083 ceph-mon[77242]: pgmap v18303: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:28.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:38:27 smithi083 ceph-mon[79930]: pgmap v18303: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:28.317 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:38:27 smithi153 ceph-mon[70044]: pgmap v18303: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:30.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:38:29 smithi083 ceph-mon[77242]: pgmap v18304: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:30.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:38:29 smithi083 ceph-mon[79930]: pgmap v18304: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:30.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:38:29 smithi153 ceph-mon[70044]: pgmap v18304: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:31.579 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:38:31.859 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:38:31.860 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 8m ago 10h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:38:31.860 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 6m ago 10h 63.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:38:31.860 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 8m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:38:31.861 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 6m ago 10h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:38:31.862 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 6m ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:38:31.862 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 8m ago 10h 1029M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:38:31.862 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 8m ago 10h 2196M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:38:31.863 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 6m ago 10h 957M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:38:31.863 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 8m ago 10h 1319M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:38:31.863 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 8m ago 10h 24.3M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:38:31.863 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 6m ago 10h 29.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:38:31.864 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 8m ago 10h 1595M 2133M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:38:31.864 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 8m ago 10h 1581M 2133M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:38:31.864 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 8m ago 10h 1228M 2133M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:38:31.865 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 8m ago 10h 1550M 2133M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:38:31.865 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 6m ago 10h 1358M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:38:31.867 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 6m ago 10h 1707M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:38:31.868 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 6m ago 10h 1524M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:38:31.868 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 6m ago 10h 1814M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:38:31.869 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 6m ago 10h 177M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:38:31.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:38:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:38:31] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T15:38:32.209 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:38:32.210 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:38:32.210 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:38:32.211 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:38:32.212 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:38:32.212 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:38:32.213 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:38:32.213 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:38:32.214 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:38:32.215 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:38:32.230 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:38:32.230 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:38:32.243 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:38:32.247 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:38:32.247 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:38:32.248 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:38:32.250 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:38:32.250 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:38:32.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:38:31 smithi153 ceph-mon[70044]: pgmap v18305: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:32.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:38:31 smithi153 ceph-mon[70044]: from='client.65347 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:38:32.457 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:38:31 smithi083 ceph-mon[79930]: pgmap v18305: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:32.457 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:38:31 smithi083 ceph-mon[79930]: from='client.65347 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:38:32.457 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:38:31 smithi083 ceph-mon[77242]: pgmap v18305: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:32.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:38:31 smithi083 ceph-mon[77242]: from='client.65347 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:38:32.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:38:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:38:32.456Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:38:32.818 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:38:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:38:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:38:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:38:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:38:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:38:33.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:38:32 smithi153 ceph-mon[70044]: from='client.55422 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:38:33.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:38:32 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/495720996' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:38:33.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:38:32 smithi083 ceph-mon[79930]: from='client.55422 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:38:33.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:38:32 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/495720996' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:38:33.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:38:32 smithi083 ceph-mon[77242]: from='client.55422 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:38:33.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:38:32 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/495720996' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:38:34.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:38:33 smithi153 ceph-mon[70044]: pgmap v18306: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:34.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:38:33 smithi083 ceph-mon[79930]: pgmap v18306: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:34.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:38:33 smithi083 ceph-mon[77242]: pgmap v18306: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:36.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:38:35 smithi083 ceph-mon[79930]: pgmap v18307: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:36.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:38:35 smithi083 ceph-mon[77242]: pgmap v18307: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:36.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:38:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:38:35.814Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:38:36.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:38:35 smithi153 ceph-mon[70044]: pgmap v18307: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:36.970 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:38:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:38:36.553Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:38:38.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:38:37 smithi153 ceph-mon[70044]: pgmap v18308: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:38.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:38:37 smithi083 ceph-mon[77242]: pgmap v18308: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:38.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:38:37 smithi083 ceph-mon[79930]: pgmap v18308: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:40.314 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:38:39 smithi153 ceph-mon[70044]: pgmap v18309: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:40.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:38:39 smithi083 ceph-mon[77242]: pgmap v18309: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:38:39 smithi083 ceph-mon[79930]: pgmap v18309: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:41.968 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:38:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:38:41] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T15:38:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:38:41 smithi153 ceph-mon[70044]: pgmap v18310: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:42.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:38:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:38:42.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:38:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:38:42.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:38:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:38:42.312 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:38:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:38:42.478 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:38:41 smithi083 ceph-mon[79930]: pgmap v18310: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:42.479 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:38:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:38:42.480 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:38:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:38:42.480 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:38:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:38:42.481 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:38:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:38:42.481 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:38:41 smithi083 ceph-mon[77242]: pgmap v18310: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:42.484 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:38:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:38:42.485 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:38:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:38:42.488 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:38:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:38:42.489 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:38:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:38:42.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:38:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:38:42.457Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:38:42.825 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:38:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:38:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:38:43.310 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:38:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:38:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:38:44.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:38:43 smithi153 ceph-mon[70044]: pgmap v18311: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:44.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:38:43 smithi083 ceph-mon[79930]: pgmap v18311: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:44.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:38:43 smithi083 ceph-mon[77242]: pgmap v18311: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:46.232 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:38:45 smithi083 ceph-mon[79930]: pgmap v18312: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:46.233 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:38:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:38:45.814Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:38:46.239 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:38:45 smithi083 ceph-mon[77242]: pgmap v18312: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:46.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:38:45 smithi153 ceph-mon[70044]: pgmap v18312: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:46.964 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:38:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:38:46.553Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:38:47.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:38:46 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:38:47.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:38:46 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:38:47.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:38:46 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:38:47.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:38:46 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:38:47.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:38:46 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:38:47.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:38:46 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:38:47.468 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:38:46 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:38:47.468 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:38:46 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:38:47.469 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:38:46 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:38:48.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:38:48 smithi083 ceph-mon[77242]: pgmap v18313: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:48.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:38:48 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:38:48.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:38:48 smithi083 ceph-mon[79930]: pgmap v18313: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:48.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:38:48 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:38:48.842 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:38:48 smithi153 ceph-mon[70044]: pgmap v18313: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:48.843 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:38:48 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:38:50.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:38:50 smithi083 ceph-mon[79930]: pgmap v18314: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:50.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:38:50 smithi083 ceph-mon[77242]: pgmap v18314: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:50.816 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:38:50 smithi153 ceph-mon[70044]: pgmap v18314: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:51.967 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:38:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:38:51] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T15:38:52.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:38:52 smithi083 ceph-mon[79930]: pgmap v18315: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:52.714 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:38:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:38:52.458Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:38:52.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:38:52 smithi083 ceph-mon[77242]: pgmap v18315: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:52.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:38:52 smithi153 ceph-mon[70044]: pgmap v18315: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:52.811 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:38:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:38:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:38:53.315 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:38:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:38:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:38:54.064 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:38:53 smithi153 ceph-mon[70044]: pgmap v18316: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:54.231 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:38:53 smithi083 ceph-mon[79930]: pgmap v18316: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:54.232 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:38:53 smithi083 ceph-mon[77242]: pgmap v18316: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:56.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:38:55 smithi083 ceph-mon[77242]: pgmap v18317: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:56.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:38:55 smithi083 ceph-mon[79930]: pgmap v18317: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:56.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:38:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:38:55.815Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:38:56.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:38:55 smithi153 ceph-mon[70044]: pgmap v18317: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:56.984 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:38:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:38:56.553Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:38:58.317 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:38:57 smithi153 ceph-mon[70044]: pgmap v18318: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:58.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:38:57 smithi083 ceph-mon[79930]: pgmap v18318: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:38:58.498 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:38:57 smithi083 ceph-mon[77242]: pgmap v18318: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:00.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:38:59 smithi153 ceph-mon[70044]: pgmap v18319: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:00.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:38:59 smithi083 ceph-mon[79930]: pgmap v18319: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:00.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:38:59 smithi083 ceph-mon[77242]: pgmap v18319: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:39:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:39:01] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T15:39:02.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:39:01 smithi153 ceph-mon[70044]: pgmap v18320: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:02.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:39:01 smithi083 ceph-mon[79930]: pgmap v18320: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:02.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:39:01 smithi083 ceph-mon[77242]: pgmap v18320: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:02.540 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:39:02.747 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:39:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:39:02.458Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:39:02.839 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:39:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:39:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:39:02.840 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:39:02.841 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 9m ago 10h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:39:02.841 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 7m ago 10h 63.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:39:02.849 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 9m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:39:02.850 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 7m ago 10h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:39:02.850 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 7m ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:39:02.851 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 9m ago 10h 1029M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:39:02.851 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 9m ago 10h 2196M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:39:02.852 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 7m ago 10h 957M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:39:02.852 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 9m ago 10h 1319M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:39:02.852 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 9m ago 10h 24.3M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:39:02.853 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 7m ago 10h 29.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:39:02.853 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 9m ago 10h 1595M 2133M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:39:02.853 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 9m ago 10h 1581M 2133M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:39:02.878 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 9m ago 10h 1228M 2133M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:39:02.879 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 9m ago 10h 1550M 2133M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:39:02.879 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 7m ago 10h 1358M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:39:02.905 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 7m ago 10h 1707M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:39:02.906 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 7m ago 10h 1524M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:39:02.906 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 7m ago 10h 1814M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:39:02.907 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 7m ago 10h 177M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:39:03.208 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:39:03.208 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:39:03.209 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:39:03.209 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:39:03.226 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:39:03.226 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:39:03.227 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:39:03.227 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:39:03.228 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:39:03.228 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:39:03.229 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:39:03.229 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:39:03.250 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:39:03.251 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:39:03.251 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:39:03.252 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:39:03.278 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:39:03.279 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:39:03.319 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:39:02 smithi153 ceph-mon[70044]: from='client.55434 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:39:03.319 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:39:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:39:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:39:03.468 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:39:02 smithi083 ceph-mon[79930]: from='client.55434 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:39:03.469 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:39:02 smithi083 ceph-mon[77242]: from='client.55434 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:39:04.317 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:39:03 smithi153 ceph-mon[70044]: from='client.65371 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:39:04.318 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:39:03 smithi153 ceph-mon[70044]: pgmap v18321: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:04.318 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:39:03 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/71878213' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:39:04.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:39:03 smithi083 ceph-mon[79930]: from='client.65371 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:39:04.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:39:03 smithi083 ceph-mon[79930]: pgmap v18321: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:04.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:39:03 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/71878213' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:39:04.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:39:03 smithi083 ceph-mon[77242]: from='client.65371 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:39:04.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:39:03 smithi083 ceph-mon[77242]: pgmap v18321: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:04.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:39:03 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/71878213' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:39:06.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:39:05 smithi083 ceph-mon[77242]: pgmap v18322: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:06.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:39:05 smithi083 ceph-mon[79930]: pgmap v18322: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:06.216 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:39:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:39:05.815Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:39:06.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:39:05 smithi153 ceph-mon[70044]: pgmap v18322: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:06.966 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:39:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:39:06.554Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:39:08.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:39:07 smithi153 ceph-mon[70044]: pgmap v18323: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:08.470 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:39:07 smithi083 ceph-mon[79930]: pgmap v18323: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:08.471 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:39:07 smithi083 ceph-mon[77242]: pgmap v18323: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:10.312 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:39:09 smithi153 ceph-mon[70044]: pgmap v18324: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:10.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:39:09 smithi083 ceph-mon[79930]: pgmap v18324: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:10.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:39:09 smithi083 ceph-mon[77242]: pgmap v18324: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:11.984 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:39:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:39:11] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T15:39:12.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:39:11 smithi153 ceph-mon[70044]: pgmap v18325: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:12.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:39:11 smithi083 ceph-mon[79930]: pgmap v18325: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:12.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:39:11 smithi083 ceph-mon[77242]: pgmap v18325: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:12.727 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:39:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:39:12.459Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:39:12.824 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:39:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:39:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:39:13.314 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:39:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:39:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:39:14.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:39:13 smithi153 ceph-mon[70044]: pgmap v18326: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:14.222 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:39:13 smithi083 ceph-mon[79930]: pgmap v18326: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:14.223 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:39:13 smithi083 ceph-mon[77242]: pgmap v18326: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:16.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:39:15 smithi083 ceph-mon[79930]: pgmap v18327: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:16.218 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:39:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:39:15.816Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:39:16.219 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:39:15 smithi083 ceph-mon[77242]: pgmap v18327: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:16.331 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:39:15 smithi153 ceph-mon[70044]: pgmap v18327: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:16.971 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:39:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:39:16.554Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:39:18.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:39:17 smithi153 ceph-mon[70044]: pgmap v18328: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:18.471 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:39:17 smithi083 ceph-mon[79930]: pgmap v18328: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:18.471 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:39:17 smithi083 ceph-mon[77242]: pgmap v18328: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:20.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:39:19 smithi153 ceph-mon[70044]: pgmap v18329: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:20.494 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:39:19 smithi083 ceph-mon[79930]: pgmap v18329: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:20.495 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:39:19 smithi083 ceph-mon[77242]: pgmap v18329: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:21.973 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:39:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:39:21] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T15:39:22.315 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:39:21 smithi153 ceph-mon[70044]: pgmap v18330: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:22.483 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:39:21 smithi083 ceph-mon[79930]: pgmap v18330: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:22.497 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:39:21 smithi083 ceph-mon[77242]: pgmap v18330: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:22.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:39:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:39:22.459Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:39:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:39:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:39:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:39:23.324 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:39:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:39:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:39:24.078 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:39:23 smithi153 ceph-mon[70044]: pgmap v18331: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:24.223 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:39:23 smithi083 ceph-mon[79930]: pgmap v18331: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:24.224 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:39:23 smithi083 ceph-mon[77242]: pgmap v18331: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:26.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:39:25 smithi083 ceph-mon[79930]: pgmap v18332: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:26.215 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:39:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:39:25.816Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:39:26.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:39:25 smithi083 ceph-mon[77242]: pgmap v18332: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:26.314 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:39:25 smithi153 ceph-mon[70044]: pgmap v18332: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:26.972 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:39:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:39:26.555Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:39:28.313 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:39:27 smithi153 ceph-mon[70044]: pgmap v18333: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:28.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:39:27 smithi083 ceph-mon[79930]: pgmap v18333: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:28.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:39:27 smithi083 ceph-mon[77242]: pgmap v18333: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:30.312 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:39:29 smithi153 ceph-mon[70044]: pgmap v18334: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:30.467 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:39:29 smithi083 ceph-mon[79930]: pgmap v18334: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:30.468 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:39:29 smithi083 ceph-mon[77242]: pgmap v18334: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:39:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:39:31] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T15:39:32.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:39:31 smithi153 ceph-mon[70044]: pgmap v18335: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:32.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:39:31 smithi083 ceph-mon[79930]: pgmap v18335: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:32.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:39:31 smithi083 ceph-mon[77242]: pgmap v18335: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:32.815 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:39:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:39:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:39:32.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:39:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:39:32.461Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:39:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:39:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:39:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:39:33.531 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:39:33.823 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:39:33.824 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 9m ago 10h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:39:33.824 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 7m ago 10h 63.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:39:33.825 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 9m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:39:33.825 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 7m ago 10h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:39:33.863 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 7m ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:39:33.863 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 9m ago 10h 1029M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:39:33.875 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 9m ago 10h 2196M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:39:33.876 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 7m ago 10h 957M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:39:33.876 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 9m ago 10h 1319M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:39:33.876 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 9m ago 10h 24.3M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:39:33.877 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 7m ago 10h 29.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:39:33.877 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 9m ago 10h 1595M 2133M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:39:33.880 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 9m ago 10h 1581M 2133M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:39:33.881 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 9m ago 10h 1228M 2133M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:39:33.882 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 9m ago 10h 1550M 2133M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:39:33.904 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 7m ago 10h 1358M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:39:33.905 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 7m ago 10h 1707M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:39:33.905 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 7m ago 10h 1524M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:39:33.926 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 7m ago 10h 1814M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:39:33.927 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 7m ago 10h 177M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:39:34.063 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:39:33 smithi153 ceph-mon[70044]: pgmap v18336: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:34.073 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:39:33 smithi153 ceph-mon[70044]: from='client.55452 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:39:34.178 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:39:34.179 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:39:34.179 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:39:34.179 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:39:34.180 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:39:34.181 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:39:34.181 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:39:34.181 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:39:34.182 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:39:34.191 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:39:34.192 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:39:34.192 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:39:34.192 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:39:34.192 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:39:34.193 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:39:34.193 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:39:34.193 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:39:34.194 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:39:34.235 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:39:33 smithi083 ceph-mon[79930]: pgmap v18336: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:34.236 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:39:33 smithi083 ceph-mon[79930]: from='client.55452 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:39:34.237 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:39:33 smithi083 ceph-mon[77242]: pgmap v18336: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:34.237 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:39:33 smithi083 ceph-mon[77242]: from='client.55452 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:39:35.067 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:39:34 smithi153 ceph-mon[70044]: from='client.65389 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:39:35.069 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:39:34 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1366974243' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:39:35.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:39:34 smithi083 ceph-mon[79930]: from='client.65389 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:39:35.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:39:34 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1366974243' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:39:35.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:39:34 smithi083 ceph-mon[77242]: from='client.65389 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:39:35.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:39:34 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1366974243' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:39:36.068 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:39:35 smithi153 ceph-mon[70044]: pgmap v18337: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:36.221 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:39:35 smithi083 ceph-mon[79930]: pgmap v18337: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:36.222 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:39:35 smithi083 ceph-mon[77242]: pgmap v18337: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:36.231 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:39:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:39:35.816Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:39:36.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:39:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:39:36.556Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:39:38.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:39:37 smithi153 ceph-mon[70044]: pgmap v18338: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:38.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:39:37 smithi083 ceph-mon[77242]: pgmap v18338: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:38.502 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:39:37 smithi083 ceph-mon[79930]: pgmap v18338: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:40.317 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:39:39 smithi153 ceph-mon[70044]: pgmap v18339: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:39:39 smithi083 ceph-mon[79930]: pgmap v18339: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:39:39 smithi083 ceph-mon[77242]: pgmap v18339: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:39:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:39:41] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T15:39:42.316 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:39:41 smithi153 ceph-mon[70044]: pgmap v18340: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:42.316 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:39:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:39:42.326 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:39:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:39:42.327 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:39:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:39:42.327 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:39:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:39:42.467 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:39:41 smithi083 ceph-mon[79930]: pgmap v18340: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:42.467 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:39:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:39:42.468 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:39:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:39:42.468 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:39:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:39:42.469 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:39:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:39:42.486 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:39:41 smithi083 ceph-mon[77242]: pgmap v18340: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:42.486 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:39:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:39:42.487 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:39:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:39:42.487 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:39:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:39:42.488 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:39:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:39:42.824 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:39:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:39:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:39:42.973 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:39:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:39:42.461Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:39:43.310 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:39:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:39:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:39:44.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:39:43 smithi153 ceph-mon[70044]: pgmap v18341: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:44.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:39:43 smithi083 ceph-mon[77242]: pgmap v18341: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:44.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:39:43 smithi083 ceph-mon[79930]: pgmap v18341: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:46.219 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:39:45 smithi083 ceph-mon[79930]: pgmap v18342: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:46.220 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:39:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:39:45.817Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:39:46.233 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:39:45 smithi083 ceph-mon[77242]: pgmap v18342: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:46.327 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:39:45 smithi153 ceph-mon[70044]: pgmap v18342: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:46.972 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:39:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:39:46.557Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:39:48.328 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:39:47 smithi153 ceph-mon[70044]: pgmap v18343: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:48.329 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:39:47 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:39:48.329 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:39:47 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:39:48.329 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:39:47 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:39:48.330 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:39:47 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:39:48.330 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:39:47 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:39:48.331 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:39:47 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:39:48.331 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:39:47 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:39:48.477 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:39:47 smithi083 ceph-mon[77242]: pgmap v18343: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:48.477 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:39:47 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:39:48.477 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:39:47 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:39:48.482 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:39:47 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:39:48.482 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:39:47 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:39:48.482 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:39:47 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:39:48.483 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:39:47 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:39:48.483 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:39:47 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:39:48.483 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:39:47 smithi083 ceph-mon[79930]: pgmap v18343: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:48.484 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:39:47 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:39:48.484 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:39:47 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:39:48.484 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:39:47 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:39:48.484 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:39:47 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:39:48.485 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:39:47 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:39:48.485 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:39:47 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:39:48.485 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:39:47 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:39:50.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:39:49 smithi153 ceph-mon[70044]: pgmap v18344: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:50.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:39:49 smithi083 ceph-mon[77242]: pgmap v18344: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:50.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:39:49 smithi083 ceph-mon[79930]: pgmap v18344: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:39:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:39:51] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T15:39:52.322 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:39:51 smithi153 ceph-mon[70044]: pgmap v18345: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:52.322 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:39:51 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:39:52.323 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:39:51 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:39:52.476 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:39:51 smithi083 ceph-mon[79930]: pgmap v18345: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:52.476 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:39:51 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:39:52.477 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:39:51 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:39:52.478 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:39:51 smithi083 ceph-mon[77242]: pgmap v18345: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:52.478 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:39:51 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:39:52.479 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:39:51 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:39:52.814 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:39:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:39:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:39:52.969 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:39:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:39:52.462Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:39:53.310 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:39:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:39:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:39:53.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:39:53 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:39:53.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:39:53 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:39:53.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:39:53 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:39:54.814 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:39:54 smithi153 ceph-mon[70044]: pgmap v18346: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:54.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:39:54 smithi083 ceph-mon[79930]: pgmap v18346: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:54.968 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:39:54 smithi083 ceph-mon[77242]: pgmap v18346: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:56.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:39:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:39:55.817Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:39:56.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:39:56 smithi153 ceph-mon[70044]: pgmap v18347: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:56.972 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:39:56 smithi083 ceph-mon[77242]: pgmap v18347: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:56.973 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:39:56 smithi083 ceph-mon[79930]: pgmap v18347: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:56.974 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:39:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:39:56.558Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:39:58.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:39:58 smithi153 ceph-mon[70044]: pgmap v18348: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:58.979 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:39:58 smithi083 ceph-mon[79930]: pgmap v18348: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:39:58.980 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:39:58 smithi083 ceph-mon[77242]: pgmap v18348: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:00.475 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:40:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[77219]: 2022-09-29T15:39:59.999+0000 7f160f665700 -1 log_channel(cluster) log [ERR] : overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T15:40:00.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:40:00 smithi153 ceph-mon[70044]: pgmap v18349: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:00.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:40:00 smithi153 ceph-mon[70044]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T15:40:00.999 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:40:00 smithi083 ceph-mon[79930]: pgmap v18349: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:00.999 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:40:00 smithi083 ceph-mon[79930]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T15:40:01.000 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:40:00 smithi083 ceph-mon[77242]: pgmap v18349: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:01.001 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:40:00 smithi083 ceph-mon[77242]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T15:40:01.967 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:40:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:40:01] "GET /metrics HTTP/1.1" 200 225246 "" "Prometheus/2.33.4" 2022-09-29T15:40:02.815 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:40:02 smithi153 ceph-mon[70044]: pgmap v18350: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:02.816 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:40:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:40:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:40:02.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:40:02 smithi083 ceph-mon[79930]: pgmap v18350: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:02.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:40:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:40:02.464Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:40:02.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:40:02 smithi083 ceph-mon[77242]: pgmap v18350: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:03.310 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:40:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:40:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:40:04.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:40:03 smithi153 ceph-mon[70044]: pgmap v18351: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:04.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:40:03 smithi083 ceph-mon[79930]: pgmap v18351: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:04.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:40:03 smithi083 ceph-mon[77242]: pgmap v18351: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:04.511 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:40:04.795 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:40:04.795 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 12s ago 10h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:40:04.795 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 8m ago 10h 63.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:40:04.796 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 12s ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:40:04.796 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 8m ago 10h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:40:04.796 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 8m ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:40:04.796 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 12s ago 10h 1037M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:40:04.797 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 12s ago 10h 2229M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:40:04.797 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 8m ago 10h 957M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:40:04.797 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 12s ago 10h 1339M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:40:04.797 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 12s ago 10h 23.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:40:04.818 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 8m ago 10h 29.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:40:04.819 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 12s ago 10h 1620M 2133M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:40:04.844 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 12s ago 10h 1607M 2133M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:40:04.845 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 12s ago 10h 1248M 2133M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:40:04.845 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 12s ago 10h 1574M 2133M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:40:04.845 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 8m ago 10h 1358M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:40:04.863 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 8m ago 10h 1707M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:40:04.864 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 8m ago 10h 1524M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:40:04.875 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 8m ago 10h 1814M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:40:04.882 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 8m ago 10h 177M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:40:05.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:40:04 smithi153 ceph-mon[70044]: from='client.55470 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:40:05.155 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:40:05.156 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:40:05.156 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:40:05.157 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:40:05.157 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:40:05.203 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:40:05.204 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:40:05.204 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:40:05.205 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:40:05.205 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:40:05.205 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:40:05.206 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:40:05.206 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:40:05.235 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:40:05.235 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:40:05.236 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:40:05.236 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:40:05.237 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:40:05.279 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:40:04 smithi083 ceph-mon[79930]: from='client.55470 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:40:05.280 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:40:04 smithi083 ceph-mon[77242]: from='client.55470 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:40:06.221 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:40:05 smithi083 ceph-mon[79930]: from='client.55476 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:40:06.221 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:40:05 smithi083 ceph-mon[79930]: pgmap v18352: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:06.222 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:40:05 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2714301234' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:40:06.222 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:40:05 smithi083 ceph-mon[77242]: from='client.55476 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:40:06.223 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:40:05 smithi083 ceph-mon[77242]: pgmap v18352: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:06.223 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:40:05 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2714301234' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:40:06.224 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:40:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:40:05.817Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:40:06.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:40:05 smithi153 ceph-mon[70044]: from='client.55476 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:40:06.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:40:05 smithi153 ceph-mon[70044]: pgmap v18352: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:06.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:40:05 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2714301234' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:40:06.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:40:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:40:06.559Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:40:08.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:40:07 smithi153 ceph-mon[70044]: pgmap v18353: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:08.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:40:07 smithi083 ceph-mon[79930]: pgmap v18353: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:08.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:40:07 smithi083 ceph-mon[77242]: pgmap v18353: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:10.327 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:40:09 smithi153 ceph-mon[70044]: pgmap v18354: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:10.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:40:09 smithi083 ceph-mon[79930]: pgmap v18354: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:10.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:40:09 smithi083 ceph-mon[77242]: pgmap v18354: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:40:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:40:11] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T15:40:12.317 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:40:11 smithi153 ceph-mon[70044]: pgmap v18355: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:12.469 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:40:11 smithi083 ceph-mon[79930]: pgmap v18355: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:12.479 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:40:11 smithi083 ceph-mon[77242]: pgmap v18355: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:12.811 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:40:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:40:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:40:12.967 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:40:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:40:12.463Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:40:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:40:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:40:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:40:14.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:40:13 smithi153 ceph-mon[70044]: pgmap v18356: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:14.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:40:13 smithi083 ceph-mon[79930]: pgmap v18356: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:14.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:40:13 smithi083 ceph-mon[77242]: pgmap v18356: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:16.227 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:40:15 smithi083 ceph-mon[79930]: pgmap v18357: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:16.229 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:40:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:40:15.818Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:40:16.229 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:40:15 smithi083 ceph-mon[77242]: pgmap v18357: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:16.323 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:40:15 smithi153 ceph-mon[70044]: pgmap v18357: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:16.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:40:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:40:16.558Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:40:18.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:40:17 smithi153 ceph-mon[70044]: pgmap v18358: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:18.484 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:40:17 smithi083 ceph-mon[79930]: pgmap v18358: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:18.485 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:40:17 smithi083 ceph-mon[77242]: pgmap v18358: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:20.320 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:40:19 smithi153 ceph-mon[70044]: pgmap v18359: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:20.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:40:19 smithi083 ceph-mon[79930]: pgmap v18359: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:20.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:40:19 smithi083 ceph-mon[77242]: pgmap v18359: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:21.966 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:40:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:40:21] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T15:40:22.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:40:21 smithi153 ceph-mon[70044]: pgmap v18360: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:22.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:40:21 smithi083 ceph-mon[79930]: pgmap v18360: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:22.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:40:21 smithi083 ceph-mon[77242]: pgmap v18360: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:40:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:40:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:40:22.986 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:40:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:40:22.463Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:40:23.311 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:40:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:40:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:40:24.219 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:40:23 smithi083 ceph-mon[79930]: pgmap v18361: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:24.220 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:40:23 smithi083 ceph-mon[77242]: pgmap v18361: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:24.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:40:23 smithi153 ceph-mon[70044]: pgmap v18361: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:26.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:40:25 smithi083 ceph-mon[79930]: pgmap v18362: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:26.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:40:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:40:25.819Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:40:26.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:40:25 smithi083 ceph-mon[77242]: pgmap v18362: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:26.324 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:40:25 smithi153 ceph-mon[70044]: pgmap v18362: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:26.966 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:40:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:40:26.559Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:40:28.314 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:40:27 smithi153 ceph-mon[70044]: pgmap v18363: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:28.471 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:40:27 smithi083 ceph-mon[79930]: pgmap v18363: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:28.471 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:40:27 smithi083 ceph-mon[77242]: pgmap v18363: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:30.319 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:40:29 smithi153 ceph-mon[70044]: pgmap v18364: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:30.468 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:40:29 smithi083 ceph-mon[79930]: pgmap v18364: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:30.469 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:40:29 smithi083 ceph-mon[77242]: pgmap v18364: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:31.987 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:40:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:40:31] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T15:40:32.318 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:40:31 smithi153 ceph-mon[70044]: pgmap v18365: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:32.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:40:32 smithi083 ceph-mon[79930]: pgmap v18365: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:32.467 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:40:31 smithi083 ceph-mon[77242]: pgmap v18365: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:40:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:40:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:40:32.968 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:40:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:40:32.464Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:40:33.318 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:40:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:40:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:40:34.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:40:33 smithi083 ceph-mon[79930]: pgmap v18366: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:34.230 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:40:33 smithi083 ceph-mon[77242]: pgmap v18366: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:34.316 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:40:33 smithi153 ceph-mon[70044]: pgmap v18366: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:35.479 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:40:35.790 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:40:35.791 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 43s ago 10h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:40:35.822 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 8m ago 10h 63.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:40:35.823 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 43s ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:40:35.823 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 8m ago 10h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:40:35.823 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 8m ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:40:35.824 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 43s ago 10h 1037M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:40:35.824 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 43s ago 10h 2229M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:40:35.824 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 8m ago 10h 957M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:40:35.824 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 43s ago 10h 1339M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:40:35.825 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 43s ago 10h 23.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:40:35.825 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 8m ago 10h 29.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:40:35.825 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 43s ago 10h 1620M 2133M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:40:35.826 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 43s ago 10h 1607M 2133M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:40:35.833 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 43s ago 10h 1248M 2133M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:40:35.834 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 43s ago 10h 1574M 2133M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:40:35.834 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 8m ago 10h 1358M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:40:35.834 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 8m ago 10h 1707M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:40:35.834 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 8m ago 10h 1524M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:40:35.835 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 8m ago 10h 1814M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:40:35.835 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 8m ago 10h 177M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:40:36.131 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:40:36.132 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:40:36.132 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:40:36.133 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:40:36.133 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:40:36.136 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:40:36.137 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:40:36.137 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:40:36.138 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:40:36.138 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:40:36.139 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:40:36.139 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:40:36.140 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:40:36.140 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:40:36.140 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:40:36.141 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:40:36.141 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:40:36.142 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:40:36.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:40:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:40:35.819Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:40:36.223 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:40:35 smithi083 ceph-mon[79930]: pgmap v18367: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:36.223 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:40:35 smithi083 ceph-mon[79930]: from='client.55488 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:40:36.223 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:40:35 smithi083 ceph-mon[77242]: pgmap v18367: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:36.224 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:40:35 smithi083 ceph-mon[77242]: from='client.55488 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:40:36.321 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:40:35 smithi153 ceph-mon[70044]: pgmap v18367: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:36.322 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:40:35 smithi153 ceph-mon[70044]: from='client.55488 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:40:36.969 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:40:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:40:36.560Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:40:37.319 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:40:36 smithi153 ceph-mon[70044]: from='client.55494 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:40:37.320 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:40:36 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1420088543' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:40:37.485 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:40:36 smithi083 ceph-mon[79930]: from='client.55494 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:40:37.486 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:40:36 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1420088543' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:40:37.487 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:40:36 smithi083 ceph-mon[77242]: from='client.55494 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:40:37.487 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:40:36 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1420088543' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:40:38.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:40:37 smithi153 ceph-mon[70044]: pgmap v18368: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:38.474 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:40:37 smithi083 ceph-mon[79930]: pgmap v18368: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:38.497 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:40:37 smithi083 ceph-mon[77242]: pgmap v18368: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:40.318 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:40:40 smithi153 ceph-mon[70044]: pgmap v18369: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:40.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:40:40 smithi083 ceph-mon[79930]: pgmap v18369: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:40:40 smithi083 ceph-mon[77242]: pgmap v18369: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:41.970 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:40:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:40:41] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T15:40:42.318 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:40:42 smithi153 ceph-mon[70044]: pgmap v18370: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:42.319 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:40:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:40:42.320 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:40:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:40:42.320 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:40:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:40:42.329 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:40:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:40:42.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:40:42 smithi083 ceph-mon[79930]: pgmap v18370: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:42.468 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:40:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:40:42.469 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:40:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:40:42.470 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:40:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:40:42.470 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:40:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:40:42.487 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:40:42 smithi083 ceph-mon[77242]: pgmap v18370: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:42.487 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:40:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:40:42.488 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:40:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:40:42.488 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:40:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:40:42.489 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:40:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:40:42.810 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:40:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:40:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:40:42.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:40:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:40:42.465Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:40:43.310 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:40:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:40:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:40:44.219 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:40:43 smithi083 ceph-mon[77242]: pgmap v18371: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:44.220 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:40:43 smithi083 ceph-mon[79930]: pgmap v18371: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:44.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:40:43 smithi153 ceph-mon[70044]: pgmap v18371: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:46.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:40:45 smithi083 ceph-mon[79930]: pgmap v18372: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:46.218 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:40:45 smithi083 ceph-mon[77242]: pgmap v18372: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:46.219 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:40:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:40:45.821Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:40:46.314 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:40:45 smithi153 ceph-mon[70044]: pgmap v18372: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:46.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:40:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:40:46.560Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:40:48.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:40:48 smithi153 ceph-mon[70044]: pgmap v18373: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:48.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:40:48 smithi083 ceph-mon[79930]: pgmap v18373: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:48.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:40:47 smithi083 ceph-mon[77242]: pgmap v18373: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:50.318 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:40:50 smithi153 ceph-mon[70044]: pgmap v18374: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:50.471 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:40:50 smithi083 ceph-mon[79930]: pgmap v18374: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:50.472 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:40:50 smithi083 ceph-mon[77242]: pgmap v18374: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:51.964 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:40:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:40:51] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T15:40:52.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:40:52 smithi153 ceph-mon[70044]: pgmap v18375: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:52.474 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:40:52 smithi083 ceph-mon[79930]: pgmap v18375: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:52.475 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:40:52 smithi083 ceph-mon[77242]: pgmap v18375: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:52.823 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:40:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:40:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:40:52.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:40:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:40:52.465Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:40:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:40:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:40:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:40:53.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:40:53 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:40:53.336 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:40:53 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:40:53.337 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:40:53 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:40:53.476 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:40:53 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:40:53.496 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:40:53 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:40:53.497 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:40:53 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:40:53.497 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:40:53 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:40:53.501 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:40:53 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:40:53.501 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:40:53 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:40:54.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:40:54 smithi083 ceph-mon[79930]: pgmap v18376: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:54.467 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:40:54 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:40:54.467 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:40:54 smithi083 ceph-mon[77242]: pgmap v18376: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:54.467 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:40:54 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:40:54.566 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:40:54 smithi153 ceph-mon[70044]: pgmap v18376: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:54.567 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:40:54 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:40:56.162 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:40:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:40:55.821Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:40:56.474 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:40:56 smithi083 ceph-mon[79930]: pgmap v18377: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:56.475 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:40:56 smithi083 ceph-mon[77242]: pgmap v18377: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:56.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:40:56 smithi153 ceph-mon[70044]: pgmap v18377: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:56.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:40:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:40:56.561Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:40:58.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:40:58 smithi083 ceph-mon[79930]: pgmap v18378: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:58.467 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:40:58 smithi083 ceph-mon[77242]: pgmap v18378: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:40:58.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:40:58 smithi153 ceph-mon[70044]: pgmap v18378: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:00.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:41:00 smithi083 ceph-mon[79930]: pgmap v18379: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:00.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:41:00 smithi083 ceph-mon[77242]: pgmap v18379: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:00.562 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:41:00 smithi153 ceph-mon[70044]: pgmap v18379: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:41:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:41:01] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T15:41:02.471 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:41:02 smithi083 ceph-mon[77242]: pgmap v18380: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:02.472 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:41:02 smithi083 ceph-mon[79930]: pgmap v18380: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:02.564 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:41:02 smithi153 ceph-mon[70044]: pgmap v18380: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:02.570 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:41:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:41:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:41:02.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:41:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:41:02.466Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:41:03.311 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:41:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:41:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:41:04.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:41:03 smithi083 ceph-mon[79930]: pgmap v18381: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:04.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:41:03 smithi083 ceph-mon[77242]: pgmap v18381: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:04.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:41:03 smithi153 ceph-mon[70044]: pgmap v18381: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:06.223 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:41:06 smithi083 ceph-mon[77242]: pgmap v18382: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:06.235 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:41:06 smithi083 ceph-mon[79930]: pgmap v18382: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:06.236 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:41:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:41:05.822Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:41:06.323 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:41:06 smithi153 ceph-mon[70044]: pgmap v18382: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:06.466 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:41:06.746 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:41:06.747 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 74s ago 10h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:41:06.747 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 9m ago 10h 63.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:41:06.748 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 74s ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:41:06.748 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 9m ago 10h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:41:06.748 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 9m ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:41:06.761 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 74s ago 10h 1037M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:41:06.771 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 74s ago 10h 2229M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:41:06.772 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 9m ago 10h 957M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:41:06.772 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 74s ago 10h 1339M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:41:06.772 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 74s ago 10h 23.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:41:06.773 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 9m ago 10h 29.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:41:06.773 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 74s ago 10h 1620M 2133M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:41:06.773 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 74s ago 10h 1607M 2133M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:41:06.774 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 74s ago 10h 1248M 2133M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:41:06.774 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 74s ago 10h 1574M 2133M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:41:06.775 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 9m ago 10h 1358M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:41:06.775 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 9m ago 10h 1707M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:41:06.775 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 9m ago 10h 1524M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:41:06.776 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 9m ago 10h 1814M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:41:06.780 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 9m ago 10h 177M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:41:06.975 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:41:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:41:06.561Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:41:07.107 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:41:07.107 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:41:07.108 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:41:07.108 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:41:07.109 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:41:07.109 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:41:07.127 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:41:07.128 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:41:07.128 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:41:07.128 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:41:07.129 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:41:07.150 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:41:07.151 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:41:07.151 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:41:07.152 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:41:07.152 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:41:07.174 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:41:07.175 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:41:07.312 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:41:07 smithi153 ceph-mon[70044]: from='client.55506 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:41:07.313 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:41:07 smithi153 ceph-mon[70044]: from='client.65443 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:41:07.467 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:41:07 smithi083 ceph-mon[79930]: from='client.55506 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:41:07.467 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:41:07 smithi083 ceph-mon[79930]: from='client.65443 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:41:07.468 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:41:07 smithi083 ceph-mon[77242]: from='client.55506 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:41:07.478 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:41:07 smithi083 ceph-mon[77242]: from='client.65443 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:41:08.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:41:08 smithi153 ceph-mon[70044]: pgmap v18383: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:08.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:41:08 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2668285047' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:41:08.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:41:08 smithi083 ceph-mon[79930]: pgmap v18383: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:08.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:41:08 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2668285047' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:41:08.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:41:08 smithi083 ceph-mon[77242]: pgmap v18383: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:08.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:41:08 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2668285047' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:41:10.314 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:41:10 smithi153 ceph-mon[70044]: pgmap v18384: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:10.483 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:41:10 smithi083 ceph-mon[79930]: pgmap v18384: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:10.484 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:41:10 smithi083 ceph-mon[77242]: pgmap v18384: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:41:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:41:11] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T15:41:12.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:41:12 smithi153 ceph-mon[70044]: pgmap v18385: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:12.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:41:12 smithi083 ceph-mon[77242]: pgmap v18385: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:12.467 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:41:12 smithi083 ceph-mon[79930]: pgmap v18385: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:12.826 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:41:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:41:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:41:12.976 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:41:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:41:12.466Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:41:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:41:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:41:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:41:14.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:41:13 smithi083 ceph-mon[79930]: pgmap v18386: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:14.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:41:13 smithi083 ceph-mon[77242]: pgmap v18386: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:14.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:41:13 smithi153 ceph-mon[70044]: pgmap v18386: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:16.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:41:16 smithi083 ceph-mon[79930]: pgmap v18387: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:16.227 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:41:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:41:15.822Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:41:16.228 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:41:16 smithi083 ceph-mon[77242]: pgmap v18387: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:16.316 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:41:16 smithi153 ceph-mon[70044]: pgmap v18387: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:16.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:41:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:41:16.562Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:41:18.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:41:18 smithi153 ceph-mon[70044]: pgmap v18388: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:18.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:41:18 smithi083 ceph-mon[77242]: pgmap v18388: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:18.467 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:41:18 smithi083 ceph-mon[79930]: pgmap v18388: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:20.318 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:41:20 smithi153 ceph-mon[70044]: pgmap v18389: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:20.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:41:20 smithi083 ceph-mon[79930]: pgmap v18389: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:20.467 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:41:20 smithi083 ceph-mon[77242]: pgmap v18389: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:21.979 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:41:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:41:21] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T15:41:22.313 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:41:22 smithi153 ceph-mon[70044]: pgmap v18390: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:22.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:41:22 smithi083 ceph-mon[79930]: pgmap v18390: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:22.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:41:22 smithi083 ceph-mon[77242]: pgmap v18390: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:41:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:41:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:41:22.966 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:41:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:41:22.468Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:41:23.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:41:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:41:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:41:24.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:41:23 smithi083 ceph-mon[79930]: pgmap v18391: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:24.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:41:23 smithi083 ceph-mon[77242]: pgmap v18391: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:24.320 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:41:23 smithi153 ceph-mon[70044]: pgmap v18391: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:26.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:41:26 smithi083 ceph-mon[79930]: pgmap v18392: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:26.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:41:26 smithi083 ceph-mon[77242]: pgmap v18392: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:26.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:41:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:41:25.823Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:41:26.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:41:26 smithi153 ceph-mon[70044]: pgmap v18392: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:26.967 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:41:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:41:26.563Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:41:28.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:41:28 smithi153 ceph-mon[70044]: pgmap v18393: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:28.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:41:28 smithi083 ceph-mon[79930]: pgmap v18393: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:28.486 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:41:28 smithi083 ceph-mon[77242]: pgmap v18393: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:30.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:41:30 smithi153 ceph-mon[70044]: pgmap v18394: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:30.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:41:30 smithi083 ceph-mon[79930]: pgmap v18394: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:30.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:41:30 smithi083 ceph-mon[77242]: pgmap v18394: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:31.964 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:41:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:41:31] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T15:41:32.335 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:41:32 smithi153 ceph-mon[70044]: pgmap v18395: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:32.474 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:41:32 smithi083 ceph-mon[77242]: pgmap v18395: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:32.475 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:41:32 smithi083 ceph-mon[79930]: pgmap v18395: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:41:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:41:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:41:32.966 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:41:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:41:32.468Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:41:33.315 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:41:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:41:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:41:34.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:41:33 smithi083 ceph-mon[77242]: pgmap v18396: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:34.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:41:33 smithi083 ceph-mon[79930]: pgmap v18396: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:34.326 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:41:33 smithi153 ceph-mon[70044]: pgmap v18396: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:36.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:41:36 smithi083 ceph-mon[77242]: pgmap v18397: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:36.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:41:36 smithi083 ceph-mon[79930]: pgmap v18397: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:36.231 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:41:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:41:35.824Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:41:36.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:41:36 smithi153 ceph-mon[70044]: pgmap v18397: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:36.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:41:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:41:36.564Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:41:37.430 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:41:37.726 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:41:37.728 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 105s ago 10h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:41:37.728 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 9m ago 10h 63.6M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:41:37.729 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 105s ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:41:37.729 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 9m ago 10h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:41:37.730 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 9m ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:41:37.730 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 105s ago 10h 1037M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:41:37.731 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 105s ago 10h 2229M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:41:37.731 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 9m ago 10h 957M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:41:37.731 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 105s ago 10h 1339M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:41:37.732 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 105s ago 10h 23.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:41:37.732 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 9m ago 10h 29.3M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:41:37.733 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 105s ago 10h 1620M 2133M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:41:37.733 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 105s ago 10h 1607M 2133M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:41:37.746 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 105s ago 10h 1248M 2133M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:41:37.746 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 105s ago 10h 1574M 2133M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:41:37.747 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 9m ago 10h 1358M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:41:37.747 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 9m ago 10h 1707M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:41:37.747 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 9m ago 10h 1524M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:41:37.747 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 9m ago 10h 1814M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:41:37.748 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 9m ago 10h 177M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:41:38.074 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:41:38.075 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:41:38.075 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:41:38.076 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:41:38.076 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:41:38.077 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:41:38.077 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:41:38.077 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:41:38.122 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:41:38.123 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:41:38.123 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:41:38.124 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:41:38.124 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:41:38.128 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:41:38.129 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:41:38.129 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:41:38.129 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:41:38.130 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:41:38.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:41:38 smithi153 ceph-mon[70044]: pgmap v18398: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:38.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:41:38 smithi153 ceph-mon[70044]: from='client.55524 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:41:38.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:41:38 smithi153 ceph-mon[70044]: from='client.55530 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:41:38.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:41:38 smithi083 ceph-mon[79930]: pgmap v18398: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:38.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:41:38 smithi083 ceph-mon[79930]: from='client.55524 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:41:38.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:41:38 smithi083 ceph-mon[79930]: from='client.55530 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:41:38.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:41:38 smithi083 ceph-mon[77242]: pgmap v18398: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:38.477 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:41:38 smithi083 ceph-mon[77242]: from='client.55524 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:41:38.478 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:41:38 smithi083 ceph-mon[77242]: from='client.55530 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:41:39.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:41:39 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1820150401' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:41:39.474 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:41:39 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1820150401' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:41:39.475 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:41:39 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1820150401' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:41:40.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:41:40 smithi153 ceph-mon[70044]: pgmap v18399: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:40.471 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:41:40 smithi083 ceph-mon[79930]: pgmap v18399: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:40.475 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:41:40 smithi083 ceph-mon[77242]: pgmap v18399: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:41.974 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:41:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:41:41] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T15:41:42.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:41:42 smithi153 ceph-mon[70044]: pgmap v18400: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:42.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:41:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:41:42.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:41:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:41:42.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:41:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:41:42.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:41:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:41:42.470 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:41:42 smithi083 ceph-mon[77242]: pgmap v18400: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:42.471 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:41:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:41:42.471 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:41:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:41:42.472 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:41:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:41:42.472 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:41:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:41:42.480 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:41:42 smithi083 ceph-mon[79930]: pgmap v18400: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:42.483 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:41:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:41:42.484 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:41:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:41:42.484 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:41:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:41:42.484 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:41:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:41:42.812 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:41:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:41:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:41:42.974 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:41:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:41:42.468Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:41:43.318 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:41:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:41:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:41:44.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:41:43 smithi083 ceph-mon[77242]: pgmap v18401: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:44.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:41:43 smithi083 ceph-mon[79930]: pgmap v18401: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:44.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:41:43 smithi153 ceph-mon[70044]: pgmap v18401: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:46.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:41:46 smithi083 ceph-mon[79930]: pgmap v18402: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:46.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:41:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:41:45.824Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:41:46.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:41:46 smithi083 ceph-mon[77242]: pgmap v18402: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:46.313 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:41:46 smithi153 ceph-mon[70044]: pgmap v18402: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:46.970 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:41:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:41:46.565Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:41:48.320 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:41:48 smithi153 ceph-mon[70044]: pgmap v18403: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:48.471 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:41:48 smithi083 ceph-mon[77242]: pgmap v18403: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:48.483 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:41:48 smithi083 ceph-mon[79930]: pgmap v18403: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:50.314 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:41:50 smithi153 ceph-mon[70044]: pgmap v18404: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:50.467 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:41:50 smithi083 ceph-mon[77242]: pgmap v18404: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:50.468 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:41:50 smithi083 ceph-mon[79930]: pgmap v18404: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:51.970 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:41:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:41:51] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T15:41:52.315 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:41:52 smithi153 ceph-mon[70044]: pgmap v18405: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:52.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:41:52 smithi083 ceph-mon[77242]: pgmap v18405: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:52.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:41:52 smithi083 ceph-mon[79930]: pgmap v18405: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:41:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:41:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:41:52.976 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:41:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:41:52.469Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:41:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:41:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:41:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:41:54.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:41:53 smithi083 ceph-mon[79930]: pgmap v18406: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:54.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:41:53 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:41:54.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:41:53 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:41:54.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:41:53 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:41:54.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:41:53 smithi083 ceph-mon[77242]: pgmap v18406: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:54.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:41:53 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:41:54.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:41:53 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:41:54.221 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:41:53 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:41:54.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:41:53 smithi153 ceph-mon[70044]: pgmap v18406: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:54.312 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:41:53 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:41:54.313 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:41:53 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:41:54.313 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:41:53 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:41:56.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:41:56 smithi083 ceph-mon[79930]: pgmap v18407: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:56.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:41:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:41:55.824Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:41:56.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:41:56 smithi083 ceph-mon[77242]: pgmap v18407: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:56.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:41:56 smithi153 ceph-mon[70044]: pgmap v18407: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:56.981 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:41:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:41:56.564Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:41:58.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:41:58 smithi083 ceph-mon[79930]: pgmap v18408: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:58.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:41:58 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:41:58.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:41:58 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:41:58.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:41:58 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:41:58.968 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:41:58 smithi083 ceph-mon[77242]: pgmap v18408: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:58.968 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:41:58 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:41:58.971 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:41:58 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:41:58.972 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:41:58 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:41:59.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:41:58 smithi153 ceph-mon[70044]: pgmap v18408: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:41:59.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:41:58 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:41:59.063 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:41:58 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:41:59.063 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:41:58 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:42:00.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:42:00 smithi083 ceph-mon[79930]: pgmap v18409: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:00.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:42:00 smithi083 ceph-mon[77242]: pgmap v18409: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:01.063 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:42:00 smithi153 ceph-mon[70044]: pgmap v18409: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:42:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:42:01] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T15:42:02.687 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:42:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:42:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:42:02.968 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:42:02 smithi083 ceph-mon[79930]: pgmap v18410: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:02.969 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:42:02 smithi083 ceph-mon[77242]: pgmap v18410: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:02.969 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:42:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:42:02.470Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:42:03.026 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:42:02 smithi153 ceph-mon[70044]: pgmap v18410: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:42:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:42:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:42:04.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:42:03 smithi083 ceph-mon[79930]: pgmap v18411: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:04.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:42:03 smithi083 ceph-mon[77242]: pgmap v18411: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:04.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:42:03 smithi153 ceph-mon[70044]: pgmap v18411: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:06.230 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:42:06 smithi083 ceph-mon[79930]: pgmap v18412: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:06.231 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:42:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:42:05.825Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:42:06.232 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:42:06 smithi083 ceph-mon[77242]: pgmap v18412: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:06.316 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:42:06 smithi153 ceph-mon[70044]: pgmap v18412: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:07.015 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:42:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:42:06.565Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:42:08.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:42:08 smithi153 ceph-mon[70044]: pgmap v18413: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:08.402 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:42:08.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:42:08 smithi083 ceph-mon[79930]: pgmap v18413: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:08.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:42:08 smithi083 ceph-mon[77242]: pgmap v18413: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:08.689 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:42:08.690 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 2m ago 10h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:42:08.691 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 10s ago 10h 64.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:42:08.691 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 2m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:42:08.722 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 10s ago 10h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:42:08.734 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 10s ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:42:08.735 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 2m ago 10h 1037M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:42:08.735 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 2m ago 10h 2229M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:42:08.735 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 10s ago 10h 973M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:42:08.736 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 2m ago 10h 1339M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:42:08.736 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 2m ago 10h 23.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:42:08.736 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 10s ago 10h 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:42:08.737 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 2m ago 10h 1620M 2133M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:42:08.737 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 2m ago 10h 1607M 2133M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:42:08.737 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 2m ago 10h 1248M 2133M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:42:08.740 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 2m ago 10h 1574M 2133M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:42:08.740 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 10s ago 10h 1381M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:42:08.740 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 10s ago 10h 1734M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:42:08.741 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 10s ago 10h 1550M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:42:08.741 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 10s ago 10h 1845M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:42:08.742 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 10s ago 10h 180M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:42:09.050 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:42:09.051 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:42:09.051 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:42:09.075 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:42:09.078 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:42:09.085 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:42:09.085 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:42:09.191 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:42:09.192 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:42:09.192 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:42:09.193 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:42:09.193 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:42:09.194 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:42:09.194 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:42:09.195 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:42:09.195 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:42:09.195 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:42:09.196 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:42:09.313 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:42:09 smithi153 ceph-mon[70044]: from='client.55542 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:42:09.313 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:42:09 smithi153 ceph-mon[70044]: from='client.55548 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:42:09.524 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:42:09 smithi083 ceph-mon[77242]: from='client.55542 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:42:09.524 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:42:09 smithi083 ceph-mon[77242]: from='client.55548 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:42:09.525 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:42:09 smithi083 ceph-mon[79930]: from='client.55542 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:42:09.525 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:42:09 smithi083 ceph-mon[79930]: from='client.55548 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:42:10.319 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:42:10 smithi153 ceph-mon[70044]: pgmap v18414: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:10.320 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:42:10 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3323928216' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:42:10.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:42:10 smithi083 ceph-mon[79930]: pgmap v18414: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:10.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:42:10 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3323928216' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:42:10.496 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:42:10 smithi083 ceph-mon[77242]: pgmap v18414: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:10.497 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:42:10 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3323928216' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:42:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:42:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:42:11] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T15:42:12.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:42:12 smithi153 ceph-mon[70044]: pgmap v18415: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:12.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:42:12 smithi083 ceph-mon[79930]: pgmap v18415: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:12.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:42:12 smithi083 ceph-mon[77242]: pgmap v18415: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:42:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:42:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:42:12.969 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:42:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:42:12.470Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:42:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:42:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:42:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:42:14.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:42:13 smithi083 ceph-mon[79930]: pgmap v18416: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:14.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:42:13 smithi083 ceph-mon[77242]: pgmap v18416: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:14.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:42:13 smithi153 ceph-mon[70044]: pgmap v18416: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:16.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:42:16 smithi083 ceph-mon[79930]: pgmap v18417: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:16.219 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:42:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:42:15.825Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:42:16.220 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:42:16 smithi083 ceph-mon[77242]: pgmap v18417: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:16.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:42:16 smithi153 ceph-mon[70044]: pgmap v18417: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:16.973 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:42:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:42:16.566Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:42:18.355 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:42:18 smithi153 ceph-mon[70044]: pgmap v18418: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:18.474 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:42:18 smithi083 ceph-mon[79930]: pgmap v18418: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:18.475 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:42:18 smithi083 ceph-mon[77242]: pgmap v18418: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:42:20 smithi153 ceph-mon[70044]: pgmap v18419: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:20.471 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:42:20 smithi083 ceph-mon[79930]: pgmap v18419: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:20.471 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:42:20 smithi083 ceph-mon[77242]: pgmap v18419: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:42:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:42:21] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T15:42:22.314 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:42:22 smithi153 ceph-mon[70044]: pgmap v18420: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:22.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:42:22 smithi083 ceph-mon[79930]: pgmap v18420: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:22.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:42:22 smithi083 ceph-mon[77242]: pgmap v18420: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:42:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:42:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:42:22.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:42:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:42:22.470Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:42:23.315 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:42:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:42:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:42:24.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:42:23 smithi083 ceph-mon[77242]: pgmap v18421: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:24.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:42:23 smithi083 ceph-mon[79930]: pgmap v18421: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:24.315 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:42:23 smithi153 ceph-mon[70044]: pgmap v18421: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:26.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:42:26 smithi083 ceph-mon[79930]: pgmap v18422: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:26.214 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:42:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:42:25.825Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:42:26.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:42:26 smithi083 ceph-mon[77242]: pgmap v18422: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:26.314 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:42:26 smithi153 ceph-mon[70044]: pgmap v18422: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:26.966 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:42:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:42:26.567Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:42:28.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:42:28 smithi153 ceph-mon[70044]: pgmap v18423: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:28.468 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:42:28 smithi083 ceph-mon[79930]: pgmap v18423: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:28.469 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:42:28 smithi083 ceph-mon[77242]: pgmap v18423: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:30.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:42:30 smithi153 ceph-mon[70044]: pgmap v18424: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:30.468 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:42:30 smithi083 ceph-mon[79930]: pgmap v18424: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:30.469 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:42:30 smithi083 ceph-mon[77242]: pgmap v18424: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:31.970 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:42:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:42:31] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T15:42:32.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:42:32 smithi153 ceph-mon[70044]: pgmap v18425: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:32.468 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:42:32 smithi083 ceph-mon[79930]: pgmap v18425: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:32.469 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:42:32 smithi083 ceph-mon[77242]: pgmap v18425: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:32.810 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:42:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:42:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:42:32.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:42:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:42:32.471Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:42:33.318 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:42:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:42:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:42:34.218 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:42:33 smithi083 ceph-mon[79930]: pgmap v18426: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:34.219 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:42:33 smithi083 ceph-mon[77242]: pgmap v18426: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:34.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:42:33 smithi153 ceph-mon[70044]: pgmap v18426: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:36.220 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:42:36 smithi083 ceph-mon[77242]: pgmap v18427: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:36.221 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:42:36 smithi083 ceph-mon[79930]: pgmap v18427: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:36.221 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:42:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:42:35.826Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:42:36.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:42:36 smithi153 ceph-mon[70044]: pgmap v18427: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:36.969 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:42:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:42:36.568Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:42:38.325 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:42:38 smithi153 ceph-mon[70044]: pgmap v18428: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:38.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:42:38 smithi083 ceph-mon[79930]: pgmap v18428: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:38.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:42:38 smithi083 ceph-mon[77242]: pgmap v18428: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:39.366 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:42:39.659 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:42:39.660 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 2m ago 10h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:42:39.660 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 41s ago 10h 64.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:42:39.660 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 2m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:42:39.661 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 41s ago 10h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:42:39.661 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 41s ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:42:39.677 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 2m ago 10h 1037M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:42:39.677 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 2m ago 10h 2229M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:42:39.690 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 41s ago 10h 973M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:42:39.690 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 2m ago 10h 1339M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:42:39.691 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 2m ago 10h 23.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:42:39.691 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 41s ago 10h 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:42:39.692 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 2m ago 10h 1620M 2133M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:42:39.692 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 2m ago 10h 1607M 2133M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:42:39.693 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 2m ago 10h 1248M 2133M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:42:39.695 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 2m ago 10h 1574M 2133M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:42:39.696 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 41s ago 10h 1381M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:42:39.696 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 41s ago 10h 1734M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:42:39.696 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 41s ago 10h 1550M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:42:39.697 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 41s ago 10h 1845M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:42:39.697 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 41s ago 10h 180M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:42:40.016 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:42:40.017 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:42:40.017 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:42:40.017 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:42:40.017 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:42:40.019 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:42:40.019 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:42:40.019 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:42:40.019 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:42:40.020 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:42:40.020 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:42:40.020 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:42:40.020 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:42:40.020 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:42:40.024 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:42:40.024 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:42:40.025 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:42:40.025 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:42:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:42:40 smithi153 ceph-mon[70044]: pgmap v18429: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:40.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:42:40 smithi153 ceph-mon[70044]: from='client.55560 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:42:40.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:42:40 smithi153 ceph-mon[70044]: from='client.65497 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:42:40.331 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:42:40 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/890456053' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:42:40.484 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:42:40 smithi083 ceph-mon[77242]: pgmap v18429: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:40.485 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:42:40 smithi083 ceph-mon[77242]: from='client.55560 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:42:40.485 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:42:40 smithi083 ceph-mon[77242]: from='client.65497 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:42:40.493 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:42:40 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/890456053' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:42:40.495 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:42:40 smithi083 ceph-mon[79930]: pgmap v18429: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:40.495 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:42:40 smithi083 ceph-mon[79930]: from='client.55560 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:42:40.496 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:42:40 smithi083 ceph-mon[79930]: from='client.65497 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:42:40.496 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:42:40 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/890456053' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:42:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:42:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:42:41] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T15:42:42.316 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:42:42 smithi153 ceph-mon[70044]: pgmap v18430: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:42.316 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:42:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:42:42.317 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:42:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:42:42.317 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:42:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:42:42.317 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:42:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:42:42.467 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:42:42 smithi083 ceph-mon[79930]: pgmap v18430: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:42.468 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:42:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:42:42.468 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:42:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:42:42.469 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:42:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:42:42.469 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:42:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:42:42.488 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:42:42 smithi083 ceph-mon[77242]: pgmap v18430: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:42.489 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:42:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:42:42.489 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:42:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:42:42.490 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:42:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:42:42.490 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:42:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:42:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:42:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:42:42.316Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:42:42.965 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:42:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:42:42.472Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:42:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:42:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:42:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:42:44.221 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:42:43 smithi083 ceph-mon[79930]: pgmap v18431: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:44.222 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:42:43 smithi083 ceph-mon[77242]: pgmap v18431: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:44.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:42:43 smithi153 ceph-mon[70044]: pgmap v18431: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:46.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:42:46 smithi083 ceph-mon[79930]: pgmap v18432: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:46.219 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:42:46 smithi083 ceph-mon[77242]: pgmap v18432: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:46.220 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:42:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:42:45.827Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:42:46.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:42:46 smithi153 ceph-mon[70044]: pgmap v18432: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:46.977 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:42:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:42:46.568Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:42:48.312 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:42:48 smithi153 ceph-mon[70044]: pgmap v18433: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:48.471 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:42:48 smithi083 ceph-mon[79930]: pgmap v18433: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:48.472 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:42:48 smithi083 ceph-mon[77242]: pgmap v18433: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:50.327 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:42:50 smithi153 ceph-mon[70044]: pgmap v18434: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:50.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:42:50 smithi083 ceph-mon[79930]: pgmap v18434: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:50.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:42:50 smithi083 ceph-mon[77242]: pgmap v18434: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:51.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:42:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:42:51] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T15:42:52.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:42:52 smithi153 ceph-mon[70044]: pgmap v18435: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:52.486 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:42:52 smithi083 ceph-mon[79930]: pgmap v18435: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:52.487 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:42:52 smithi083 ceph-mon[77242]: pgmap v18435: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:52.855 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:42:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:42:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:42:52.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:42:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:42:52.472Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:42:53.318 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:42:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:42:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:42:54.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:42:53 smithi083 ceph-mon[77242]: pgmap v18436: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:54.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:42:53 smithi083 ceph-mon[79930]: pgmap v18436: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:54.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:42:53 smithi153 ceph-mon[70044]: pgmap v18436: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:56.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:42:56 smithi083 ceph-mon[79930]: pgmap v18437: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:56.216 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:42:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:42:55.827Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:42:56.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:42:56 smithi083 ceph-mon[77242]: pgmap v18437: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:56.315 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:42:56 smithi153 ceph-mon[70044]: pgmap v18437: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:56.966 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:42:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:42:56.569Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:42:58.319 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:42:58 smithi153 ceph-mon[70044]: pgmap v18438: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:58.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:42:58 smithi083 ceph-mon[79930]: pgmap v18438: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:58.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:42:58 smithi083 ceph-mon[77242]: pgmap v18438: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:42:59.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:42:59 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:42:59.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:42:59 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:42:59.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:42:59 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:42:59.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:42:59 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:42:59.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:42:59 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:42:59.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:42:59 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:42:59.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:42:59 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:42:59.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:42:59 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:42:59.472 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:42:59 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:42:59.491 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:42:59 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:42:59.492 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:42:59 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:42:59.492 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:42:59 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:43:00.340 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:43:00 smithi153 ceph-mon[70044]: pgmap v18439: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:00.482 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:43:00 smithi083 ceph-mon[79930]: pgmap v18439: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:00.483 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:43:00 smithi083 ceph-mon[77242]: pgmap v18439: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:01.964 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:43:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:43:01] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T15:43:02.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:43:02 smithi153 ceph-mon[70044]: pgmap v18440: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:02.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:43:02 smithi083 ceph-mon[77242]: pgmap v18440: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:02.498 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:43:02 smithi083 ceph-mon[79930]: pgmap v18440: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:43:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:43:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:43:03.006 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:43:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:43:02.473Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:43:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:43:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:43:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:43:04.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:43:03 smithi083 ceph-mon[79930]: pgmap v18441: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:04.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:43:03 smithi083 ceph-mon[77242]: pgmap v18441: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:04.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:43:03 smithi153 ceph-mon[70044]: pgmap v18441: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:06.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:43:06 smithi083 ceph-mon[79930]: pgmap v18442: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:06.215 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:43:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:43:05.828Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:43:06.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:43:06 smithi083 ceph-mon[77242]: pgmap v18442: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:06.332 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:43:06 smithi153 ceph-mon[70044]: pgmap v18442: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:06.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:43:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:43:06.570Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:43:08.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:43:08 smithi153 ceph-mon[70044]: pgmap v18443: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:08.469 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:43:08 smithi083 ceph-mon[77242]: pgmap v18443: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:08.469 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:43:08 smithi083 ceph-mon[79930]: pgmap v18443: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:10.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:43:10 smithi153 ceph-mon[70044]: pgmap v18444: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:10.338 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:43:10.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:43:10 smithi083 ceph-mon[77242]: pgmap v18444: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:10.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:43:10 smithi083 ceph-mon[79930]: pgmap v18444: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:10.627 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:43:10.628 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 3m ago 10h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:43:10.628 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 72s ago 10h 64.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:43:10.628 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 3m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:43:10.629 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 72s ago 10h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:43:10.629 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 72s ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:43:10.629 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 3m ago 10h 1037M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:43:10.629 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 3m ago 10h 2229M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:43:10.663 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 72s ago 10h 973M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:43:10.664 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 3m ago 10h 1339M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:43:10.664 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 3m ago 10h 23.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:43:10.665 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 72s ago 10h 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:43:10.665 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 3m ago 10h 1620M 2133M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:43:10.672 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 3m ago 10h 1607M 2133M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:43:10.672 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 3m ago 10h 1248M 2133M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:43:10.673 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 3m ago 10h 1574M 2133M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:43:10.673 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 72s ago 10h 1381M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:43:10.674 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 72s ago 10h 1734M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:43:10.674 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 72s ago 10h 1550M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:43:10.675 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 72s ago 10h 1845M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:43:10.675 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 72s ago 10h 180M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:43:10.982 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:43:10.983 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:43:10.983 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:43:10.984 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:43:10.984 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:43:10.985 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:43:10.985 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:43:10.999 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:43:11.000 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:43:11.000 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:43:11.001 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:43:11.001 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:43:11.002 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:43:11.002 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:43:11.002 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:43:11.003 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:43:11.031 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:43:11.031 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:43:11.318 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:43:11 smithi153 ceph-mon[70044]: from='client.65509 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:43:11.319 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:43:11 smithi153 ceph-mon[70044]: from='client.55584 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:43:11.319 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:43:11 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3086119475' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:43:11.475 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:43:11 smithi083 ceph-mon[79930]: from='client.65509 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:43:11.475 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:43:11 smithi083 ceph-mon[79930]: from='client.55584 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:43:11.476 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:43:11 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3086119475' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:43:11.476 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:43:11 smithi083 ceph-mon[77242]: from='client.65509 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:43:11.477 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:43:11 smithi083 ceph-mon[77242]: from='client.55584 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:43:11.477 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:43:11 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3086119475' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:43:11.967 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:43:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:43:11] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T15:43:12.332 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:43:12 smithi153 ceph-mon[70044]: pgmap v18445: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:12.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:43:12 smithi083 ceph-mon[79930]: pgmap v18445: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:12.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:43:12 smithi083 ceph-mon[77242]: pgmap v18445: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:43:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:43:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:43:12.984 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:43:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:43:12.474Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:43:13.313 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:43:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:43:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:43:14.219 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:43:13 smithi083 ceph-mon[77242]: pgmap v18446: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:14.220 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:43:13 smithi083 ceph-mon[79930]: pgmap v18446: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:14.312 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:43:13 smithi153 ceph-mon[70044]: pgmap v18446: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:16.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:43:16 smithi083 ceph-mon[79930]: pgmap v18447: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:16.228 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:43:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:43:15.828Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:43:16.241 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:43:16 smithi083 ceph-mon[77242]: pgmap v18447: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:16.318 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:43:16 smithi153 ceph-mon[70044]: pgmap v18447: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:16.967 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:43:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:43:16.571Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:43:18.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:43:18 smithi153 ceph-mon[70044]: pgmap v18448: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:18.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:43:18 smithi083 ceph-mon[79930]: pgmap v18448: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:18.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:43:18 smithi083 ceph-mon[77242]: pgmap v18448: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:20.316 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:43:20 smithi153 ceph-mon[70044]: pgmap v18449: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:20.471 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:43:20 smithi083 ceph-mon[79930]: pgmap v18449: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:20.472 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:43:20 smithi083 ceph-mon[77242]: pgmap v18449: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:21.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:43:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:43:21] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T15:43:22.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:43:22 smithi153 ceph-mon[70044]: pgmap v18450: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:22.469 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:43:22 smithi083 ceph-mon[77242]: pgmap v18450: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:22.470 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:43:22 smithi083 ceph-mon[79930]: pgmap v18450: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:43:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:43:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:43:22.966 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:43:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:43:22.475Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:43:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:43:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:43:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:43:24.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:43:23 smithi083 ceph-mon[79930]: pgmap v18451: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:24.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:43:23 smithi083 ceph-mon[77242]: pgmap v18451: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:24.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:43:23 smithi153 ceph-mon[70044]: pgmap v18451: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:26.242 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:43:26 smithi083 ceph-mon[77242]: pgmap v18452: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:26.243 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:43:26 smithi083 ceph-mon[79930]: pgmap v18452: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:26.258 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:43:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:43:25.829Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:43:26.325 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:43:26 smithi153 ceph-mon[70044]: pgmap v18452: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:26.978 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:43:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:43:26.571Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:43:28.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:43:28 smithi153 ceph-mon[70044]: pgmap v18453: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:28.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:43:28 smithi083 ceph-mon[77242]: pgmap v18453: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:28.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:43:28 smithi083 ceph-mon[79930]: pgmap v18453: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:30.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:43:30 smithi153 ceph-mon[70044]: pgmap v18454: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:30.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:43:30 smithi083 ceph-mon[79930]: pgmap v18454: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:30.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:43:30 smithi083 ceph-mon[77242]: pgmap v18454: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:43:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:43:31] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T15:43:32.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:43:32 smithi153 ceph-mon[70044]: pgmap v18455: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:32.470 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:43:32 smithi083 ceph-mon[79930]: pgmap v18455: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:32.471 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:43:32 smithi083 ceph-mon[77242]: pgmap v18455: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:32.810 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:43:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:43:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:43:32.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:43:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:43:32.475Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:43:33.319 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:43:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:43:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:43:34.221 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:43:33 smithi083 ceph-mon[77242]: pgmap v18456: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:34.222 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:43:33 smithi083 ceph-mon[79930]: pgmap v18456: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:34.323 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:43:33 smithi153 ceph-mon[70044]: pgmap v18456: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:36.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:43:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:43:35.829Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:43:36.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:43:36 smithi083 ceph-mon[77242]: pgmap v18457: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:36.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:43:36 smithi083 ceph-mon[79930]: pgmap v18457: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:36.316 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:43:36 smithi153 ceph-mon[70044]: pgmap v18457: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:36.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:43:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:43:36.572Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:43:38.319 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:43:38 smithi153 ceph-mon[70044]: pgmap v18458: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:38.469 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:43:38 smithi083 ceph-mon[79930]: pgmap v18458: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:38.515 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:43:38 smithi083 ceph-mon[77242]: pgmap v18458: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:40.324 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:43:40 smithi153 ceph-mon[70044]: pgmap v18459: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:40.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:43:40 smithi083 ceph-mon[77242]: pgmap v18459: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:40.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:43:40 smithi083 ceph-mon[79930]: pgmap v18459: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:41.334 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:43:41.606 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:43:41.607 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 3m ago 10h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:43:41.607 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 103s ago 10h 64.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:43:41.607 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 3m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:43:41.607 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 103s ago 10h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:43:41.608 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 103s ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:43:41.608 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 3m ago 10h 1037M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:43:41.608 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 3m ago 10h 2229M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:43:41.654 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 103s ago 10h 973M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:43:41.655 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 3m ago 10h 1339M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:43:41.655 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 3m ago 10h 23.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:43:41.655 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 103s ago 10h 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:43:41.656 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 3m ago 10h 1620M 2133M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:43:41.656 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 3m ago 10h 1607M 2133M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:43:41.656 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 3m ago 10h 1248M 2133M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:43:41.656 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 3m ago 10h 1574M 2133M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:43:41.657 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 103s ago 10h 1381M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:43:41.657 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 103s ago 10h 1734M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:43:41.657 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 103s ago 10h 1550M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:43:41.657 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 103s ago 10h 1845M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:43:41.673 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 103s ago 10h 180M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:43:41.971 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:43:41.972 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:43:41.973 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:43:41.973 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:43:41.974 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:43:41.975 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:43:41.976 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:43:41.976 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:43:41.977 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:43:41.977 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:43:41.978 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:43:41.978 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:43:41.979 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:43:41.979 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:43:41.980 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:43:41.980 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:43:41.981 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:43:41.981 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:43:42.004 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:43:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:43:41] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T15:43:42.322 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:43:42 smithi153 ceph-mon[70044]: pgmap v18460: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:42.323 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:43:42 smithi153 ceph-mon[70044]: from='client.65527 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:43:42.324 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:43:42 smithi153 ceph-mon[70044]: from='client.65533 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:43:42.324 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:43:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:43:42.324 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:43:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:43:42.325 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:43:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:43:42.325 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:43:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:43:42.347 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:43:42 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3747477567' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:43:42.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:43:42 smithi083 ceph-mon[77242]: pgmap v18460: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:42.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:43:42 smithi083 ceph-mon[77242]: from='client.65527 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:43:42.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:43:42 smithi083 ceph-mon[77242]: from='client.65533 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:43:42.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:43:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:43:42.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:43:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:43:42.492 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:43:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:43:42.493 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:43:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:43:42.493 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:43:42 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3747477567' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:43:42.494 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:43:42 smithi083 ceph-mon[79930]: pgmap v18460: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:42.495 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:43:42 smithi083 ceph-mon[79930]: from='client.65527 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:43:42.495 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:43:42 smithi083 ceph-mon[79930]: from='client.65533 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:43:42.496 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:43:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:43:42.496 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:43:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:43:42.497 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:43:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:43:42.497 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:43:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:43:42.509 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:43:42 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3747477567' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:43:42.838 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:43:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:43:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:43:42.973 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:43:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:43:42.476Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:43:43.311 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:43:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:43:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:43:44.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:43:43 smithi083 ceph-mon[79930]: pgmap v18461: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:44.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:43:43 smithi083 ceph-mon[77242]: pgmap v18461: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:44.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:43:43 smithi153 ceph-mon[70044]: pgmap v18461: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:46.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:43:46 smithi083 ceph-mon[79930]: pgmap v18462: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:46.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:43:46 smithi083 ceph-mon[77242]: pgmap v18462: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:46.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:43:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:43:45.830Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:43:46.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:43:46 smithi153 ceph-mon[70044]: pgmap v18462: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:46.974 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:43:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:43:46.573Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:43:48.316 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:43:48 smithi153 ceph-mon[70044]: pgmap v18463: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:48.474 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:43:48 smithi083 ceph-mon[79930]: pgmap v18463: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:48.475 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:43:48 smithi083 ceph-mon[77242]: pgmap v18463: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:50.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:43:50 smithi083 ceph-mon[77242]: pgmap v18464: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:50.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:43:50 smithi083 ceph-mon[79930]: pgmap v18464: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:50.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:43:50 smithi153 ceph-mon[70044]: pgmap v18464: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:51.965 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:43:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:43:51] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T15:43:52.318 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:43:52 smithi153 ceph-mon[70044]: pgmap v18465: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:52.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:43:52 smithi083 ceph-mon[79930]: pgmap v18465: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:52.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:43:52 smithi083 ceph-mon[77242]: pgmap v18465: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:52.812 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:43:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:43:52.316Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:43:52.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:43:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:43:52.477Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:43:53.311 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:43:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:43:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:43:54.220 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:43:53 smithi083 ceph-mon[79930]: pgmap v18466: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:54.221 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:43:53 smithi083 ceph-mon[77242]: pgmap v18466: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:54.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:43:53 smithi153 ceph-mon[70044]: pgmap v18466: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:56.218 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:43:56 smithi083 ceph-mon[79930]: pgmap v18467: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:56.219 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:43:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:43:55.831Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:43:56.219 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:43:56 smithi083 ceph-mon[77242]: pgmap v18467: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:56.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:43:56 smithi153 ceph-mon[70044]: pgmap v18467: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:56.966 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:43:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:43:56.573Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:43:58.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:43:58 smithi153 ceph-mon[70044]: pgmap v18468: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:58.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:43:58 smithi083 ceph-mon[77242]: pgmap v18468: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:58.467 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:43:58 smithi083 ceph-mon[79930]: pgmap v18468: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:43:59.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:43:59 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:43:59.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:43:59 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:43:59.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:43:59 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:43:59.483 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:43:59 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:43:59.483 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:43:59 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:43:59.484 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:43:59 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:43:59.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:43:59 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:43:59.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:43:59 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:43:59.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:43:59 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:44:00.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:44:00 smithi153 ceph-mon[70044]: pgmap v18469: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:44:00.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:44:00 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:44:00.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:44:00 smithi083 ceph-mon[79930]: pgmap v18469: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:44:00.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:44:00 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:44:00.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:44:00 smithi083 ceph-mon[77242]: pgmap v18469: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:44:00.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:44:00 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:44:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:44:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:44:01] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T15:44:02.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:44:02 smithi153 ceph-mon[70044]: pgmap v18470: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:44:02.811 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:44:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:44:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:44:02.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:44:02 smithi083 ceph-mon[79930]: pgmap v18470: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:44:02.967 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:44:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:44:02.477Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:44:02.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:44:02 smithi083 ceph-mon[77242]: pgmap v18470: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:44:03.310 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:44:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:44:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:44:04.219 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:44:03 smithi083 ceph-mon[79930]: pgmap v18471: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:44:04.220 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:44:03 smithi083 ceph-mon[77242]: pgmap v18471: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:44:04.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:44:03 smithi153 ceph-mon[70044]: pgmap v18471: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:44:06.228 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:44:06 smithi083 ceph-mon[79930]: pgmap v18472: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:44:06.229 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:44:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:44:05.831Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:44:06.240 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:44:06 smithi083 ceph-mon[77242]: pgmap v18472: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:44:06.327 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:44:06 smithi153 ceph-mon[70044]: pgmap v18472: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:44:06.972 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:44:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:44:06.574Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:44:08.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:44:08 smithi083 ceph-mon[79930]: pgmap v18473: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:44:08.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:44:08 smithi083 ceph-mon[77242]: pgmap v18473: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:44:08.561 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:44:08 smithi153 ceph-mon[70044]: pgmap v18473: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:44:10.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:44:10 smithi083 ceph-mon[79930]: pgmap v18474: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:44:10.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:44:10 smithi083 ceph-mon[77242]: pgmap v18474: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:44:10.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:44:10 smithi153 ceph-mon[70044]: pgmap v18474: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:44:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:44:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:44:11] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T15:44:12.284 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:44:12.322 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:44:12 smithi153 ceph-mon[70044]: pgmap v18475: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:44:12.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:44:12 smithi083 ceph-mon[79930]: pgmap v18475: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:44:12.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:44:12 smithi083 ceph-mon[77242]: pgmap v18475: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:44:12.572 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:44:12.572 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 4m ago 10h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:44:12.573 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 2m ago 10h 64.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:44:12.573 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 4m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:44:12.573 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 2m ago 10h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:44:12.573 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 2m ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:44:12.575 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 4m ago 10h 1037M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:44:12.578 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 4m ago 10h 2229M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:44:12.578 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 2m ago 10h 973M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:44:12.578 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 4m ago 10h 1339M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:44:12.579 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 4m ago 10h 23.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:44:12.579 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 2m ago 10h 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:44:12.586 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 4m ago 10h 1620M 2133M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:44:12.587 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 4m ago 10h 1607M 2133M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:44:12.587 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 4m ago 10h 1248M 2133M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:44:12.587 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 4m ago 10h 1574M 2133M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:44:12.588 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 2m ago 10h 1381M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:44:12.588 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 2m ago 10h 1734M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:44:12.588 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 2m ago 10h 1550M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:44:12.588 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 2m ago 10h 1845M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:44:12.589 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 2m ago 10h 180M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:44:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:44:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:44:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:44:12.928 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:44:12.929 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:44:12.929 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:44:12.929 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:44:12.931 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:44:12.932 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:44:12.932 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:44:12.933 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:44:12.957 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:44:12.958 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:44:12.958 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:44:12.960 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:44:12.961 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:44:12.961 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:44:12.962 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:44:12.962 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:44:12.979 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:44:12.979 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:44:12.981 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:44:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:44:12.477Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:44:13.313 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:44:13 smithi153 ceph-mon[70044]: from='client.55614 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:44:13.316 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:44:13 smithi153 ceph-mon[70044]: from='client.65551 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:44:13.317 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:44:13 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1086572492' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:44:13.330 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:44:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:44:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:44:13.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:44:13 smithi083 ceph-mon[79930]: from='client.55614 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:44:13.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:44:13 smithi083 ceph-mon[79930]: from='client.65551 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:44:13.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:44:13 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1086572492' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:44:13.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:44:13 smithi083 ceph-mon[77242]: from='client.55614 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:44:13.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:44:13 smithi083 ceph-mon[77242]: from='client.65551 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:44:13.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:44:13 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1086572492' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:44:14.476 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:44:14 smithi083 ceph-mon[77242]: pgmap v18476: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:44:14.483 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:44:14 smithi083 ceph-mon[79930]: pgmap v18476: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:44:14.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:44:14 smithi153 ceph-mon[70044]: pgmap v18476: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:44:16.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:44:16 smithi083 ceph-mon[79930]: pgmap v18477: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:44:16.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:44:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:44:15.832Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:44:16.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:44:16 smithi083 ceph-mon[77242]: pgmap v18477: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:44:16.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:44:16 smithi153 ceph-mon[70044]: pgmap v18477: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:44:16.971 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:44:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:44:16.575Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:44:18.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:44:18 smithi083 ceph-mon[79930]: pgmap v18478: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:44:18.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:44:18 smithi083 ceph-mon[77242]: pgmap v18478: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:44:18.563 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:44:18 smithi153 ceph-mon[70044]: pgmap v18478: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:44:20.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:44:20 smithi083 ceph-mon[77242]: pgmap v18479: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:44:20.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:44:20 smithi083 ceph-mon[79930]: pgmap v18479: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:44:20.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:44:20 smithi153 ceph-mon[70044]: pgmap v18479: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:44:21.970 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:44:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:44:21] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T15:44:22.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:44:22 smithi083 ceph-mon[79930]: pgmap v18480: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:44:22.467 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:44:22 smithi083 ceph-mon[77242]: pgmap v18480: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:44:22.567 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:44:22 smithi153 ceph-mon[70044]: pgmap v18480: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:44:22.568 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:44:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:44:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:44:22.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:44:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:44:22.478Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:44:23.310 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:44:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:44:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:44:24.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:44:23 smithi083 ceph-mon[79930]: pgmap v18481: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:44:24.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:44:23 smithi083 ceph-mon[77242]: pgmap v18481: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:44:24.320 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:44:23 smithi153 ceph-mon[70044]: pgmap v18481: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:44:26.218 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:44:26 smithi083 ceph-mon[79930]: pgmap v18482: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:44:26.220 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:44:26 smithi083 ceph-mon[77242]: pgmap v18482: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:44:26.220 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:44:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:44:25.833Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:44:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:44:26 smithi153 ceph-mon[70044]: pgmap v18482: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:44:26.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:44:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:44:26.576Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:44:28.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:44:28 smithi083 ceph-mon[79930]: pgmap v18483: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:44:28.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:44:28 smithi083 ceph-mon[77242]: pgmap v18483: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:44:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:44:28 smithi153 ceph-mon[70044]: pgmap v18483: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:44:30.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:44:30 smithi083 ceph-mon[79930]: pgmap v18484: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:44:30.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:44:30 smithi083 ceph-mon[77242]: pgmap v18484: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:44:30.582 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:44:30 smithi153 ceph-mon[70044]: pgmap v18484: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:44:31.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:44:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:44:31] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T15:44:32.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:44:32 smithi083 ceph-mon[79930]: pgmap v18485: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:44:32.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:44:32 smithi083 ceph-mon[77242]: pgmap v18485: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:44:32.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:44:32 smithi153 ceph-mon[70044]: pgmap v18485: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:44:32.559 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:44:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:44:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:44:32.971 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:44:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:44:32.479Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:44:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:44:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:44:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:44:34.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:44:33 smithi083 ceph-mon[79930]: pgmap v18486: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:44:34.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:44:33 smithi083 ceph-mon[77242]: pgmap v18486: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:44:34.318 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:44:33 smithi153 ceph-mon[70044]: pgmap v18486: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:44:36.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:44:36 smithi083 ceph-mon[79930]: pgmap v18487: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:44:36.215 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:44:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:44:35.834Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:44:36.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:44:36 smithi083 ceph-mon[77242]: pgmap v18487: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:44:36.562 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:44:36 smithi153 ceph-mon[70044]: pgmap v18487: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:44:36.967 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:44:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:44:36.577Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:44:38.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:44:38 smithi083 ceph-mon[79930]: pgmap v18488: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:44:38.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:44:38 smithi083 ceph-mon[77242]: pgmap v18488: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:44:38.570 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:44:38 smithi153 ceph-mon[70044]: pgmap v18488: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:44:40.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:44:40 smithi083 ceph-mon[79930]: pgmap v18489: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:44:40.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:44:40 smithi083 ceph-mon[77242]: pgmap v18489: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:44:40.564 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:44:40 smithi153 ceph-mon[70044]: pgmap v18489: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:44:41.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:44:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:44:41] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T15:44:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:44:42 smithi083 ceph-mon[79930]: pgmap v18490: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:44:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:44:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:44:42.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:44:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:44:42.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:44:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:44:42.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:44:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:44:42.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:44:42 smithi083 ceph-mon[77242]: pgmap v18490: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:44:42.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:44:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:44:42.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:44:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:44:42.474 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:44:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:44:42.475 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:44:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:44:42.562 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:44:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:44:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:44:42.563 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:44:42 smithi153 ceph-mon[70044]: pgmap v18490: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:44:42.564 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:44:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:44:42.564 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:44:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:44:42.569 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:44:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:44:42.570 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:44:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:44:42.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:44:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:44:42.480Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:44:43.259 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:44:43.312 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:44:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:44:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:44:43.558 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:44:43.575 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 4m ago 10h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:44:43.582 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 2m ago 10h 64.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:44:43.583 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 4m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:44:43.583 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 2m ago 10h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:44:43.584 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 2m ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:44:43.584 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 4m ago 10h 1037M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:44:43.585 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 4m ago 10h 2229M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:44:43.585 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 2m ago 10h 973M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:44:43.586 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 4m ago 10h 1339M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:44:43.586 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 4m ago 10h 23.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:44:43.587 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 2m ago 10h 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:44:43.587 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 4m ago 10h 1620M 2133M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:44:43.587 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 4m ago 10h 1607M 2133M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:44:43.616 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 4m ago 10h 1248M 2133M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:44:43.616 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 4m ago 10h 1574M 2133M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:44:43.617 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 2m ago 10h 1381M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:44:43.617 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 2m ago 10h 1734M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:44:43.617 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 2m ago 10h 1550M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:44:43.618 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 2m ago 10h 1845M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:44:43.618 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 2m ago 10h 180M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:44:43.934 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:44:43.935 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:44:43.936 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:44:43.936 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:44:43.937 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:44:43.937 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:44:43.938 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:44:43.938 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:44:43.939 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:44:43.939 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:44:43.940 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:44:43.940 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:44:43.941 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:44:43.941 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:44:43.942 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:44:43.942 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:44:43.943 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:44:43.943 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:44:44.218 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:44:43 smithi083 ceph-mon[79930]: pgmap v18491: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:44:44.218 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:44:43 smithi083 ceph-mon[79930]: from='client.55632 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:44:44.219 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:44:43 smithi083 ceph-mon[79930]: from='client.55638 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:44:44.220 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:44:43 smithi083 ceph-mon[77242]: pgmap v18491: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:44:44.220 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:44:43 smithi083 ceph-mon[77242]: from='client.55632 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:44:44.220 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:44:43 smithi083 ceph-mon[77242]: from='client.55638 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:44:44.316 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:44:43 smithi153 ceph-mon[70044]: pgmap v18491: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:44:44.317 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:44:43 smithi153 ceph-mon[70044]: from='client.55632 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:44:44.317 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:44:43 smithi153 ceph-mon[70044]: from='client.55638 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:44:45.247 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:44:44 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2711773624' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:44:45.248 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:44:44 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2711773624' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:44:45.319 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:44:44 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2711773624' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:44:46.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:44:45 smithi083 ceph-mon[79930]: pgmap v18492: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:44:46.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:44:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:44:45.834Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:44:46.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:44:45 smithi083 ceph-mon[77242]: pgmap v18492: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:44:46.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:44:45 smithi153 ceph-mon[70044]: pgmap v18492: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:44:46.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:44:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:44:46.577Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:44:48.470 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:44:48 smithi083 ceph-mon[77242]: pgmap v18493: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:44:48.474 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:44:48 smithi083 ceph-mon[79930]: pgmap v18493: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:44:48.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:44:48 smithi153 ceph-mon[70044]: pgmap v18493: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:44:50.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:44:50 smithi083 ceph-mon[79930]: pgmap v18494: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:44:50.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:44:50 smithi083 ceph-mon[77242]: pgmap v18494: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:44:50.563 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:44:50 smithi153 ceph-mon[70044]: pgmap v18494: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:44:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:44:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:44:51] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T15:44:52.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:44:52 smithi083 ceph-mon[79930]: pgmap v18495: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:44:52.470 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:44:52 smithi083 ceph-mon[77242]: pgmap v18495: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:44:52.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:44:52 smithi153 ceph-mon[70044]: pgmap v18495: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:44:52.560 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:44:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:44:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:44:52.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:44:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:44:52.481Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:44:53.310 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:44:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:44:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:44:54.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:44:53 smithi083 ceph-mon[79930]: pgmap v18496: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:44:54.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:44:53 smithi083 ceph-mon[77242]: pgmap v18496: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:44:54.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:44:53 smithi153 ceph-mon[70044]: pgmap v18496: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:44:56.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:44:56 smithi083 ceph-mon[79930]: pgmap v18497: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:44:56.216 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:44:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:44:55.835Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:44:56.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:44:56 smithi083 ceph-mon[77242]: pgmap v18497: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:44:56.564 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:44:56 smithi153 ceph-mon[70044]: pgmap v18497: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:44:56.970 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:44:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:44:56.578Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:44:58.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:44:58 smithi083 ceph-mon[79930]: pgmap v18498: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:44:58.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:44:58 smithi083 ceph-mon[77242]: pgmap v18498: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:44:58.563 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:44:58 smithi153 ceph-mon[70044]: pgmap v18498: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:00.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:45:00 smithi083 ceph-mon[79930]: pgmap v18499: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:00.472 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:45:00 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:45:00.473 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:45:00 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:45:00.473 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:45:00 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:45:00.474 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:45:00 smithi083 ceph-mon[77242]: pgmap v18499: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:00.474 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:45:00 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:45:00.475 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:45:00 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:45:00.475 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:45:00 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:45:00.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:45:00 smithi153 ceph-mon[70044]: pgmap v18499: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:00.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:45:00 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:45:00.561 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:45:00 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:45:00.561 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:45:00 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:45:01.512 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:45:01 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:45:01.512 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:45:01 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:45:01.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:45:01 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:45:01.976 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:45:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:45:01] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T15:45:02.492 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:45:02 smithi083 ceph-mon[79930]: pgmap v18500: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:02.493 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:45:02 smithi083 ceph-mon[77242]: pgmap v18500: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:02.561 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:45:02 smithi153 ceph-mon[70044]: pgmap v18500: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:02.562 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:45:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:45:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:45:02.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:45:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:45:02.481Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:45:03.310 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:45:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:45:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:45:04.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:45:03 smithi083 ceph-mon[79930]: pgmap v18501: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:04.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:45:03 smithi083 ceph-mon[77242]: pgmap v18501: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:04.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:45:03 smithi153 ceph-mon[70044]: pgmap v18501: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:06.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:45:06 smithi083 ceph-mon[79930]: pgmap v18502: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:06.215 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:45:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:45:05.835Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:45:06.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:45:06 smithi083 ceph-mon[77242]: pgmap v18502: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:06.587 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:45:06 smithi153 ceph-mon[70044]: pgmap v18502: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:06.968 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:45:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:45:06.579Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:45:08.475 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:45:08 smithi083 ceph-mon[79930]: pgmap v18503: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:08.476 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:45:08 smithi083 ceph-mon[77242]: pgmap v18503: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:08.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:45:08 smithi153 ceph-mon[70044]: pgmap v18503: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:10.468 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:45:10 smithi083 ceph-mon[79930]: pgmap v18504: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:10.470 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:45:10 smithi083 ceph-mon[77242]: pgmap v18504: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:45:10 smithi153 ceph-mon[70044]: pgmap v18504: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:45:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:45:11] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T15:45:12.468 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:45:12 smithi083 ceph-mon[79930]: pgmap v18505: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:12.468 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:45:12 smithi083 ceph-mon[77242]: pgmap v18505: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:12.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:45:12 smithi153 ceph-mon[70044]: pgmap v18505: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:12.560 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:45:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:45:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:45:12.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:45:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:45:12.482Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:45:13.311 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:45:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:45:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:45:14.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:45:13 smithi083 ceph-mon[79930]: pgmap v18506: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:14.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:45:13 smithi083 ceph-mon[77242]: pgmap v18506: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:14.262 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:45:14.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:45:13 smithi153 ceph-mon[70044]: pgmap v18506: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:14.540 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:45:14.541 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 5m ago 10h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:45:14.541 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 3m ago 10h 64.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:45:14.541 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 5m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:45:14.541 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 3m ago 10h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:45:14.542 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 3m ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:45:14.542 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 5m ago 10h 1037M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:45:14.542 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 5m ago 10h 2229M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:45:14.543 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 3m ago 10h 973M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:45:14.543 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 5m ago 10h 1339M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:45:14.543 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 5m ago 10h 23.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:45:14.543 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 3m ago 10h 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:45:14.544 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 5m ago 10h 1620M 2133M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:45:14.544 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 5m ago 10h 1607M 2133M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:45:14.544 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 5m ago 10h 1248M 2133M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:45:14.545 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 5m ago 10h 1574M 2133M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:45:14.545 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 3m ago 10h 1381M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:45:14.545 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 3m ago 10h 1734M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:45:14.545 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 3m ago 10h 1550M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:45:14.545 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 3m ago 10h 1845M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:45:14.546 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 3m ago 10h 180M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:45:14.900 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:45:14.901 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:45:14.901 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:45:14.901 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:45:14.902 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:45:14.902 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:45:14.903 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:45:14.903 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:45:14.904 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:45:14.904 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:45:14.905 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:45:14.905 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:45:14.931 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:45:14.931 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:45:14.932 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:45:14.932 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:45:14.933 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:45:14.933 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:45:15.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:45:14 smithi083 ceph-mon[79930]: from='client.55650 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:45:15.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:45:14 smithi083 ceph-mon[79930]: from='client.65587 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:45:15.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:45:14 smithi083 ceph-mon[77242]: from='client.55650 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:45:15.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:45:14 smithi083 ceph-mon[77242]: from='client.65587 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:45:15.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:45:14 smithi153 ceph-mon[70044]: from='client.55650 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:45:15.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:45:14 smithi153 ceph-mon[70044]: from='client.65587 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:45:16.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:45:15 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2763077393' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:45:16.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:45:15 smithi083 ceph-mon[77242]: pgmap v18507: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:16.220 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:45:15 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2763077393' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:45:16.220 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:45:15 smithi083 ceph-mon[79930]: pgmap v18507: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:16.221 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:45:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:45:15.835Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:45:16.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:45:15 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2763077393' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:45:16.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:45:15 smithi153 ceph-mon[70044]: pgmap v18507: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:16.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:45:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:45:16.579Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:45:18.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:45:18 smithi083 ceph-mon[79930]: pgmap v18508: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:18.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:45:18 smithi083 ceph-mon[77242]: pgmap v18508: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:18.566 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:45:18 smithi153 ceph-mon[70044]: pgmap v18508: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:20.471 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:45:20 smithi083 ceph-mon[79930]: pgmap v18509: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:20.472 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:45:20 smithi083 ceph-mon[77242]: pgmap v18509: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:20.573 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:45:20 smithi153 ceph-mon[70044]: pgmap v18509: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:21.976 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:45:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:45:21] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T15:45:22.467 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:45:22 smithi083 ceph-mon[77242]: pgmap v18510: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:22.469 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:45:22 smithi083 ceph-mon[79930]: pgmap v18510: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:22.562 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:45:22 smithi153 ceph-mon[70044]: pgmap v18510: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:22.562 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:45:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:45:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:45:22.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:45:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:45:22.483Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:45:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:45:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:45:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:45:24.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:45:23 smithi083 ceph-mon[79930]: pgmap v18511: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:24.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:45:23 smithi083 ceph-mon[77242]: pgmap v18511: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:24.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:45:23 smithi153 ceph-mon[70044]: pgmap v18511: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:26.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:45:26 smithi083 ceph-mon[79930]: pgmap v18512: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:26.218 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:45:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:45:25.837Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:45:26.219 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:45:26 smithi083 ceph-mon[77242]: pgmap v18512: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:45:26 smithi153 ceph-mon[70044]: pgmap v18512: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:26.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:45:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:45:26.579Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:45:28.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:45:28 smithi083 ceph-mon[79930]: pgmap v18513: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:28.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:45:28 smithi083 ceph-mon[77242]: pgmap v18513: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:28.577 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:45:28 smithi153 ceph-mon[70044]: pgmap v18513: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:30.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:45:30 smithi083 ceph-mon[79930]: pgmap v18514: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:30.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:45:30 smithi083 ceph-mon[77242]: pgmap v18514: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:30.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:45:30 smithi153 ceph-mon[70044]: pgmap v18514: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:45:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:45:31] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T15:45:32.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:45:32 smithi083 ceph-mon[77242]: pgmap v18515: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:32.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:45:32 smithi083 ceph-mon[79930]: pgmap v18515: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:45:32 smithi153 ceph-mon[70044]: pgmap v18515: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:32.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:45:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:45:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:45:32.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:45:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:45:32.484Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:45:33.310 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:45:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:45:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:45:34.218 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:45:33 smithi083 ceph-mon[77242]: pgmap v18516: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:34.218 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:45:33 smithi083 ceph-mon[79930]: pgmap v18516: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:34.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:45:33 smithi153 ceph-mon[70044]: pgmap v18516: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:36.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:45:36 smithi083 ceph-mon[79930]: pgmap v18517: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:36.216 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:45:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:45:35.837Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:45:36.224 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:45:36 smithi083 ceph-mon[77242]: pgmap v18517: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:36.567 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:45:36 smithi153 ceph-mon[70044]: pgmap v18517: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:36.971 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:45:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:45:36.580Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:45:38.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:45:38 smithi083 ceph-mon[79930]: pgmap v18518: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:38.486 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:45:38 smithi083 ceph-mon[77242]: pgmap v18518: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:38.576 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:45:38 smithi153 ceph-mon[70044]: pgmap v18518: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:40.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:45:40 smithi083 ceph-mon[77242]: pgmap v18519: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:40.479 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:45:40 smithi083 ceph-mon[79930]: pgmap v18519: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:40.562 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:45:40 smithi153 ceph-mon[70044]: pgmap v18519: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:41.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:45:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:45:41] "GET /metrics HTTP/1.1" 200 225246 "" "Prometheus/2.33.4" 2022-09-29T15:45:42.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:45:42 smithi083 ceph-mon[77242]: pgmap v18520: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:42.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:45:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:45:42.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:45:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:45:42.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:45:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:45:42.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:45:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:45:42.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:45:42 smithi083 ceph-mon[79930]: pgmap v18520: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:42.467 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:45:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:45:42.468 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:45:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:45:42.469 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:45:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:45:42.474 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:45:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:45:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:45:42 smithi153 ceph-mon[70044]: pgmap v18520: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:45:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:45:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:45:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:45:42.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:45:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:45:42.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:45:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:45:42.561 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:45:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:45:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:45:42.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:45:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:45:42.484Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:45:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:45:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:45:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:45:44.223 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:45:43 smithi083 ceph-mon[77242]: pgmap v18521: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:44.224 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:45:43 smithi083 ceph-mon[79930]: pgmap v18521: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:44.314 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:45:43 smithi153 ceph-mon[70044]: pgmap v18521: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:45.230 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:45:45.516 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:45:45.516 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 5m ago 10h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:45:45.516 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 3m ago 10h 64.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:45:45.517 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 5m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:45:45.522 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 3m ago 10h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:45:45.523 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 3m ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:45:45.527 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 5m ago 10h 1037M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:45:45.528 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 5m ago 10h 2229M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:45:45.528 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 3m ago 10h 973M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:45:45.529 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 5m ago 10h 1339M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:45:45.529 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 5m ago 10h 23.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:45:45.529 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 3m ago 10h 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:45:45.530 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 5m ago 10h 1620M 2133M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:45:45.530 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 5m ago 10h 1607M 2133M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:45:45.531 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 5m ago 10h 1248M 2133M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:45:45.531 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 5m ago 10h 1574M 2133M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:45:45.531 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 3m ago 10h 1381M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:45:45.532 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 3m ago 10h 1734M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:45:45.532 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 3m ago 10h 1550M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:45:45.532 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 3m ago 10h 1845M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:45:45.533 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 3m ago 10h 180M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:45:45.874 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:45:45.874 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:45:45.875 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:45:45.875 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:45:45.875 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:45:45.875 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:45:45.876 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:45:45.876 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:45:45.876 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:45:45.876 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:45:45.877 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:45:45.877 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:45:45.877 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:45:45.892 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:45:45.892 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:45:45.893 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:45:45.893 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:45:45.898 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:45:46.089 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:45:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:45:45.837Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:45:46.091 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:45:46 smithi083 ceph-mon[77242]: pgmap v18522: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:46.091 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:45:46 smithi083 ceph-mon[77242]: from='client.65599 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:45:46.091 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:45:46 smithi083 ceph-mon[77242]: from='client.55674 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:45:46.091 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:45:46 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3116383559' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:45:46.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:45:46 smithi083 ceph-mon[79930]: pgmap v18522: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:46.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:45:46 smithi083 ceph-mon[79930]: from='client.65599 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:45:46.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:45:46 smithi083 ceph-mon[79930]: from='client.55674 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:45:46.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:45:46 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3116383559' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:45:46.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:45:46 smithi153 ceph-mon[70044]: pgmap v18522: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:46.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:45:46 smithi153 ceph-mon[70044]: from='client.65599 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:45:46.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:45:46 smithi153 ceph-mon[70044]: from='client.55674 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:45:46.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:45:46 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3116383559' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:45:46.970 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:45:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:45:46.580Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:45:48.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:45:48 smithi083 ceph-mon[77242]: pgmap v18523: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:48.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:45:48 smithi083 ceph-mon[79930]: pgmap v18523: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:48.565 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:45:48 smithi153 ceph-mon[70044]: pgmap v18523: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:50.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:45:50 smithi083 ceph-mon[79930]: pgmap v18524: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:50.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:45:50 smithi083 ceph-mon[77242]: pgmap v18524: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:50.562 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:45:50 smithi153 ceph-mon[70044]: pgmap v18524: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:51.966 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:45:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:45:51] "GET /metrics HTTP/1.1" 200 225246 "" "Prometheus/2.33.4" 2022-09-29T15:45:52.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:45:52 smithi083 ceph-mon[79930]: pgmap v18525: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:52.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:45:52 smithi083 ceph-mon[77242]: pgmap v18525: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:52.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:45:52 smithi153 ceph-mon[70044]: pgmap v18525: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:52.559 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:45:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:45:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:45:52.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:45:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:45:52.485Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:45:53.317 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:45:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:45:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:45:54.222 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:45:53 smithi083 ceph-mon[79930]: pgmap v18526: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:54.238 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:45:53 smithi083 ceph-mon[77242]: pgmap v18526: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:54.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:45:53 smithi153 ceph-mon[70044]: pgmap v18526: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:56.094 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:45:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:45:55.838Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:45:56.473 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:45:56 smithi083 ceph-mon[77242]: pgmap v18527: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:56.474 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:45:56 smithi083 ceph-mon[79930]: pgmap v18527: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:56.567 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:45:56 smithi153 ceph-mon[70044]: pgmap v18527: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:56.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:45:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:45:56.581Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:45:58.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:45:58 smithi083 ceph-mon[79930]: pgmap v18528: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:58.475 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:45:58 smithi083 ceph-mon[77242]: pgmap v18528: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:45:58.565 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:45:58 smithi153 ceph-mon[70044]: pgmap v18528: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:00.486 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:46:00 smithi083 ceph-mon[79930]: pgmap v18529: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:00.487 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:46:00 smithi083 ceph-mon[77242]: pgmap v18529: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:00.561 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:46:00 smithi153 ceph-mon[70044]: pgmap v18529: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:01.495 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:46:01 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:46:01.496 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:46:01 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:46:01.514 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:46:01 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:46:01.515 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:46:01 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:46:01.516 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:46:01 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:46:01.517 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:46:01 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:46:01.517 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:46:01 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:46:01.520 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:46:01 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:46:01.565 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:46:01 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:46:01.565 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:46:01 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:46:01.566 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:46:01 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:46:01.566 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:46:01 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:46:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:46:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:46:01] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T15:46:02.471 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:46:02 smithi083 ceph-mon[79930]: pgmap v18530: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:02.472 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:46:02 smithi083 ceph-mon[77242]: pgmap v18530: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:02.571 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:46:02 smithi153 ceph-mon[70044]: pgmap v18530: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:02.572 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:46:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:46:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:46:02.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:46:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:46:02.486Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:46:03.311 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:46:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:46:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:46:04.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:46:03 smithi083 ceph-mon[79930]: pgmap v18531: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:04.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:46:03 smithi083 ceph-mon[77242]: pgmap v18531: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:04.314 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:46:03 smithi153 ceph-mon[70044]: pgmap v18531: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:06.093 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:46:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:46:05.839Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:46:06.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:46:06 smithi083 ceph-mon[77242]: pgmap v18532: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:06.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:46:06 smithi083 ceph-mon[79930]: pgmap v18532: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:06.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:46:06 smithi153 ceph-mon[70044]: pgmap v18532: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:06.978 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:46:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:46:06.581Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:46:08.485 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:46:08 smithi083 ceph-mon[79930]: pgmap v18533: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:08.486 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:46:08 smithi083 ceph-mon[77242]: pgmap v18533: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:08.571 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:46:08 smithi153 ceph-mon[70044]: pgmap v18533: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:10.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:46:10 smithi083 ceph-mon[79930]: pgmap v18534: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:10.482 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:46:10 smithi083 ceph-mon[77242]: pgmap v18534: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:10.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:46:10 smithi153 ceph-mon[70044]: pgmap v18534: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:46:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:46:11] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T15:46:12.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:46:12 smithi083 ceph-mon[79930]: pgmap v18535: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:12.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:46:12 smithi083 ceph-mon[77242]: pgmap v18535: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:12.575 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:46:12 smithi153 ceph-mon[70044]: pgmap v18535: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:12.576 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:46:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:46:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:46:12.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:46:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:46:12.486Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:46:13.310 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:46:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:46:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:46:14.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:46:13 smithi083 ceph-mon[77242]: pgmap v18536: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:14.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:46:13 smithi083 ceph-mon[79930]: pgmap v18536: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:14.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:46:13 smithi153 ceph-mon[70044]: pgmap v18536: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:16.104 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:46:16 smithi083 ceph-mon[77242]: pgmap v18537: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:16.114 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:46:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:46:15.840Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:46:16.210 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:46:16.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:46:16 smithi083 ceph-mon[79930]: pgmap v18537: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:16.492 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:46:16.492 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 6m ago 10h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:46:16.493 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 4m ago 10h 64.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:46:16.493 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 6m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:46:16.493 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 4m ago 10h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:46:16.494 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 4m ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:46:16.494 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 6m ago 10h 1037M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:46:16.495 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 6m ago 10h 2229M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:46:16.495 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 4m ago 10h 973M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:46:16.496 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 6m ago 10h 1339M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:46:16.496 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 6m ago 10h 23.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:46:16.497 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 4m ago 10h 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:46:16.497 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 6m ago 10h 1620M 2133M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:46:16.506 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 6m ago 10h 1607M 2133M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:46:16.506 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 6m ago 10h 1248M 2133M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:46:16.530 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 6m ago 10h 1574M 2133M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:46:16.531 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 4m ago 10h 1381M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:46:16.531 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 4m ago 10h 1734M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:46:16.531 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 4m ago 10h 1550M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:46:16.532 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 4m ago 10h 1845M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:46:16.532 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 4m ago 10h 180M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:46:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:46:16 smithi153 ceph-mon[70044]: pgmap v18537: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:16.848 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:46:16.849 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:46:16.849 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:46:16.869 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:46:16.870 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:46:16.870 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:46:16.871 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:46:16.871 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:46:16.872 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:46:16.872 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:46:16.873 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:46:16.873 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:46:16.895 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:46:16.895 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:46:16.896 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:46:16.896 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:46:16.897 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:46:16.927 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:46:16.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:46:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:46:16.582Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:46:17.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:46:17 smithi083 ceph-mon[77242]: from='client.65617 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:46:17.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:46:17 smithi083 ceph-mon[77242]: from='client.65623 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:46:17.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:46:17 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2761449072' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:46:17.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:46:17 smithi083 ceph-mon[79930]: from='client.65617 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:46:17.467 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:46:17 smithi083 ceph-mon[79930]: from='client.65623 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:46:17.467 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:46:17 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2761449072' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:46:17.577 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:46:17 smithi153 ceph-mon[70044]: from='client.65617 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:46:17.579 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:46:17 smithi153 ceph-mon[70044]: from='client.65623 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:46:17.580 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:46:17 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2761449072' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:46:18.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:46:18 smithi083 ceph-mon[79930]: pgmap v18538: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:18.475 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:46:18 smithi083 ceph-mon[77242]: pgmap v18538: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:18.566 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:46:18 smithi153 ceph-mon[70044]: pgmap v18538: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:20.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:46:20 smithi083 ceph-mon[77242]: pgmap v18539: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:20.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:46:20 smithi083 ceph-mon[79930]: pgmap v18539: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:20.564 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:46:20 smithi153 ceph-mon[70044]: pgmap v18539: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:21.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:46:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:46:21] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T15:46:22.469 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:46:22 smithi083 ceph-mon[77242]: pgmap v18540: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:22.494 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:46:22 smithi083 ceph-mon[79930]: pgmap v18540: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:22.561 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:46:22 smithi153 ceph-mon[70044]: pgmap v18540: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:22.562 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:46:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:46:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:46:22.966 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:46:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:46:22.486Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:46:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:46:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:46:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:46:24.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:46:23 smithi083 ceph-mon[77242]: pgmap v18541: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:24.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:46:23 smithi083 ceph-mon[79930]: pgmap v18541: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:24.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:46:23 smithi153 ceph-mon[70044]: pgmap v18541: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:26.111 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:46:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:46:25.841Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:46:26.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:46:26 smithi083 ceph-mon[77242]: pgmap v18542: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:26.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:46:26 smithi083 ceph-mon[79930]: pgmap v18542: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:46:26 smithi153 ceph-mon[70044]: pgmap v18542: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:26.966 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:46:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:46:26.583Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:46:28.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:46:28 smithi083 ceph-mon[79930]: pgmap v18543: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:28.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:46:28 smithi083 ceph-mon[77242]: pgmap v18543: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:28.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:46:28 smithi153 ceph-mon[70044]: pgmap v18543: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:30.501 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:46:30 smithi083 ceph-mon[79930]: pgmap v18544: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:30.502 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:46:30 smithi083 ceph-mon[77242]: pgmap v18544: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:30.568 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:46:30 smithi153 ceph-mon[70044]: pgmap v18544: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:31.967 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:46:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:46:31] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T15:46:32.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:46:32 smithi083 ceph-mon[79930]: pgmap v18545: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:32.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:46:32 smithi083 ceph-mon[77242]: pgmap v18545: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:32.571 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:46:32 smithi153 ceph-mon[70044]: pgmap v18545: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:32.572 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:46:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:46:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:46:32.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:46:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:46:32.487Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:46:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:46:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:46:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:46:34.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:46:33 smithi083 ceph-mon[79930]: pgmap v18546: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:34.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:46:33 smithi083 ceph-mon[77242]: pgmap v18546: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:34.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:46:33 smithi153 ceph-mon[70044]: pgmap v18546: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:36.111 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:46:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:46:35.841Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:46:36.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:46:36 smithi083 ceph-mon[77242]: pgmap v18547: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:36.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:46:36 smithi083 ceph-mon[79930]: pgmap v18547: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:46:36 smithi153 ceph-mon[70044]: pgmap v18547: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:36.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:46:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:46:36.583Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:46:38.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:46:38 smithi083 ceph-mon[79930]: pgmap v18548: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:38.524 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:46:38 smithi083 ceph-mon[77242]: pgmap v18548: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:38.562 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:46:38 smithi153 ceph-mon[70044]: pgmap v18548: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:46:40 smithi083 ceph-mon[79930]: pgmap v18549: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:40.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:46:40 smithi083 ceph-mon[77242]: pgmap v18549: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:40.571 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:46:40 smithi153 ceph-mon[70044]: pgmap v18549: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:46:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:46:41] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T15:46:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:46:42 smithi083 ceph-mon[79930]: pgmap v18550: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:42.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:46:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:46:42.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:46:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:46:42.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:46:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:46:42.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:46:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:46:42.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:46:42 smithi083 ceph-mon[77242]: pgmap v18550: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:42.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:46:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:46:42.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:46:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:46:42.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:46:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:46:42.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:46:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:46:42.562 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:46:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:46:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:46:42.563 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:46:42 smithi153 ceph-mon[70044]: pgmap v18550: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:42.563 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:46:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:46:42.564 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:46:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:46:42.564 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:46:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:46:42.564 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:46:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:46:42.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:46:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:46:42.488Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:46:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:46:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:46:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:46:44.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:46:43 smithi083 ceph-mon[77242]: pgmap v18551: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:44.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:46:43 smithi083 ceph-mon[79930]: pgmap v18551: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:44.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:46:43 smithi153 ceph-mon[70044]: pgmap v18551: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:46.106 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:46:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:46:45.841Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:46:46.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:46:46 smithi083 ceph-mon[79930]: pgmap v18552: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:46.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:46:46 smithi083 ceph-mon[77242]: pgmap v18552: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:46.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:46:46 smithi153 ceph-mon[70044]: pgmap v18552: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:46.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:46:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:46:46.585Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:46:47.170 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:46:47.457 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:46:47.457 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 6m ago 10h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:46:47.458 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 4m ago 10h 64.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:46:47.458 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 6m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:46:47.459 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 4m ago 10h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:46:47.460 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 4m ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:46:47.460 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 6m ago 10h 1037M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:46:47.461 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 6m ago 10h 2229M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:46:47.462 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 4m ago 10h 973M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:46:47.462 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 6m ago 10h 1339M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:46:47.463 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 6m ago 10h 23.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:46:47.463 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 4m ago 10h 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:46:47.463 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 6m ago 10h 1620M 2133M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:46:47.464 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 6m ago 10h 1607M 2133M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:46:47.464 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 6m ago 10h 1248M 2133M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:46:47.464 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 6m ago 10h 1574M 2133M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:46:47.464 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 4m ago 10h 1381M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:46:47.465 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 4m ago 10h 1734M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:46:47.465 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 4m ago 10h 1550M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:46:47.465 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 4m ago 10h 1845M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:46:47.466 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 4m ago 10h 180M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:46:47.812 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:46:47.813 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:46:47.813 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:46:47.814 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:46:47.814 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:46:47.815 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:46:47.815 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:46:47.815 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:46:47.816 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:46:47.816 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:46:47.817 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:46:47.817 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:46:47.818 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:46:47.818 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:46:47.819 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:46:47.827 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:46:47.828 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:46:47.840 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:46:48.468 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:46:48 smithi083 ceph-mon[79930]: pgmap v18553: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:48.469 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:46:48 smithi083 ceph-mon[79930]: from='client.65635 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:46:48.469 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:46:48 smithi083 ceph-mon[79930]: from='client.65641 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:46:48.469 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:46:48 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1554489977' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:46:48.470 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:46:48 smithi083 ceph-mon[77242]: pgmap v18553: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:48.470 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:46:48 smithi083 ceph-mon[77242]: from='client.65635 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:46:48.470 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:46:48 smithi083 ceph-mon[77242]: from='client.65641 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:46:48.470 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:46:48 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1554489977' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:46:48.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:46:48 smithi153 ceph-mon[70044]: pgmap v18553: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:48.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:46:48 smithi153 ceph-mon[70044]: from='client.65635 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:46:48.561 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:46:48 smithi153 ceph-mon[70044]: from='client.65641 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:46:48.561 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:46:48 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1554489977' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:46:50.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:46:50 smithi083 ceph-mon[77242]: pgmap v18554: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:50.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:46:50 smithi083 ceph-mon[79930]: pgmap v18554: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:50.578 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:46:50 smithi153 ceph-mon[70044]: pgmap v18554: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:51.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:46:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:46:51] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T15:46:52.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:46:52 smithi083 ceph-mon[79930]: pgmap v18555: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:52.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:46:52 smithi083 ceph-mon[77242]: pgmap v18555: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:52.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:46:52 smithi153 ceph-mon[70044]: pgmap v18555: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:52.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:46:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:46:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:46:52.964 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:46:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:46:52.490Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:46:53.311 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:46:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:46:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:46:54.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:46:53 smithi083 ceph-mon[77242]: pgmap v18556: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:54.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:46:53 smithi083 ceph-mon[79930]: pgmap v18556: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:54.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:46:53 smithi153 ceph-mon[70044]: pgmap v18556: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:56.109 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:46:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:46:55.841Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:46:56.110 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:46:56 smithi083 ceph-mon[77242]: pgmap v18557: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:56.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:46:56 smithi083 ceph-mon[79930]: pgmap v18557: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:56.586 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:46:56 smithi153 ceph-mon[70044]: pgmap v18557: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:56.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:46:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:46:56.585Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:46:58.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:46:58 smithi083 ceph-mon[79930]: pgmap v18558: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:58.498 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:46:58 smithi083 ceph-mon[77242]: pgmap v18558: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:46:58.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:46:58 smithi153 ceph-mon[70044]: pgmap v18558: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:00.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:47:00 smithi083 ceph-mon[79930]: pgmap v18559: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:00.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:47:00 smithi083 ceph-mon[77242]: pgmap v18559: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:47:00 smithi153 ceph-mon[70044]: pgmap v18559: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:01.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:47:01 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:47:01.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:47:01 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:47:01.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:47:01 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:47:01.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:47:01 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:47:01.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:47:01 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:47:01.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:47:01 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:47:01.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:47:01 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:47:01.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:47:01 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:47:01.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:47:01 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:47:01.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:47:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:47:01] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T15:47:02.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:47:02 smithi153 ceph-mon[70044]: pgmap v18560: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:02.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:47:02 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:47:02.812 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:47:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:47:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:47:02.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:47:02 smithi083 ceph-mon[79930]: pgmap v18560: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:02.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:47:02 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:47:02.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:47:02 smithi083 ceph-mon[77242]: pgmap v18560: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:02.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:47:02 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:47:02.964 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:47:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:47:02.490Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:47:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:47:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:47:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:47:04.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:47:03 smithi083 ceph-mon[77242]: pgmap v18561: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:04.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:47:03 smithi083 ceph-mon[79930]: pgmap v18561: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:04.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:47:03 smithi153 ceph-mon[70044]: pgmap v18561: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:06.111 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:47:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:47:05.842Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:47:06.460 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:47:06 smithi083 ceph-mon[77242]: pgmap v18562: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:06.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:47:06 smithi083 ceph-mon[79930]: pgmap v18562: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:06.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:47:06 smithi153 ceph-mon[70044]: pgmap v18562: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:06.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:47:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:47:06.585Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:47:08.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:47:08 smithi083 ceph-mon[79930]: pgmap v18563: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:08.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:47:08 smithi083 ceph-mon[77242]: pgmap v18563: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:08.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:47:08 smithi153 ceph-mon[70044]: pgmap v18563: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:10.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:47:10 smithi083 ceph-mon[79930]: pgmap v18564: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:10.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:47:10 smithi083 ceph-mon[77242]: pgmap v18564: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:10.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:47:10 smithi153 ceph-mon[70044]: pgmap v18564: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:47:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:47:11] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T15:47:12.460 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:47:12 smithi083 ceph-mon[77242]: pgmap v18565: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:12.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:47:12 smithi083 ceph-mon[79930]: pgmap v18565: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:12.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:47:12 smithi153 ceph-mon[70044]: pgmap v18565: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:12.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:47:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:47:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:47:12.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:47:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:47:12.490Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:47:13.313 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:47:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:47:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:47:14.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:47:13 smithi083 ceph-mon[79930]: pgmap v18566: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:14.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:47:13 smithi083 ceph-mon[77242]: pgmap v18566: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:14.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:47:13 smithi153 ceph-mon[70044]: pgmap v18566: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:16.114 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:47:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:47:15.843Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:47:16.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:47:16 smithi083 ceph-mon[79930]: pgmap v18567: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:16.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:47:16 smithi083 ceph-mon[77242]: pgmap v18567: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:16.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:47:16 smithi153 ceph-mon[70044]: pgmap v18567: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:16.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:47:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:47:16.587Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:47:18.140 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:47:18.431 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:47:18.432 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 7m ago 10h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:47:18.432 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 5m ago 10h 64.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:47:18.432 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 7m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:47:18.433 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 5m ago 10h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:47:18.433 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 5m ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:47:18.433 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 7m ago 10h 1037M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:47:18.434 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 7m ago 10h 2229M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:47:18.434 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 5m ago 10h 973M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:47:18.434 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 7m ago 10h 1339M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:47:18.435 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 7m ago 10h 23.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:47:18.435 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 5m ago 10h 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:47:18.436 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 7m ago 10h 1620M 2133M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:47:18.436 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 7m ago 10h 1607M 2133M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:47:18.436 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 7m ago 10h 1248M 2133M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:47:18.436 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 7m ago 10h 1574M 2133M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:47:18.437 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 5m ago 10h 1381M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:47:18.437 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 5m ago 10h 1734M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:47:18.437 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 5m ago 10h 1550M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:47:18.437 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 5m ago 10h 1845M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:47:18.439 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 5m ago 10h 180M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:47:18.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:47:18 smithi083 ceph-mon[79930]: pgmap v18568: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:18.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:47:18 smithi083 ceph-mon[77242]: pgmap v18568: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:47:18 smithi153 ceph-mon[70044]: pgmap v18568: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:18.784 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:47:18.785 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:47:18.785 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:47:18.785 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:47:18.785 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:47:18.786 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:47:18.786 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:47:18.786 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:47:18.786 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:47:18.787 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:47:18.787 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:47:18.787 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:47:18.787 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:47:18.787 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:47:18.788 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:47:18.788 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:47:18.788 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:47:18.788 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:47:19.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:47:19 smithi083 ceph-mon[79930]: from='client.55722 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:47:19.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:47:19 smithi083 ceph-mon[79930]: from='client.55728 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:47:19.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:47:19 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1204878208' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:47:19.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:47:19 smithi083 ceph-mon[77242]: from='client.55722 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:47:19.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:47:19 smithi083 ceph-mon[77242]: from='client.55728 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:47:19.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:47:19 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1204878208' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:47:19.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:47:19 smithi153 ceph-mon[70044]: from='client.55722 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:47:19.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:47:19 smithi153 ceph-mon[70044]: from='client.55728 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:47:19.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:47:19 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1204878208' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:47:20.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:47:20 smithi083 ceph-mon[79930]: pgmap v18569: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:20.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:47:20 smithi083 ceph-mon[77242]: pgmap v18569: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:20.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:47:20 smithi153 ceph-mon[70044]: pgmap v18569: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:47:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:47:21] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T15:47:22.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:47:22 smithi083 ceph-mon[79930]: pgmap v18570: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:22.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:47:22 smithi083 ceph-mon[77242]: pgmap v18570: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:47:22 smithi153 ceph-mon[70044]: pgmap v18570: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:22.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:47:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:47:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:47:22.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:47:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:47:22.490Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:47:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:47:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:47:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:47:24.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:47:23 smithi083 ceph-mon[77242]: pgmap v18571: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:24.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:47:23 smithi083 ceph-mon[79930]: pgmap v18571: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:24.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:47:23 smithi153 ceph-mon[70044]: pgmap v18571: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:26.117 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:47:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:47:25.844Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:47:26.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:47:26 smithi083 ceph-mon[77242]: pgmap v18572: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:26.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:47:26 smithi083 ceph-mon[79930]: pgmap v18572: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:47:26 smithi153 ceph-mon[70044]: pgmap v18572: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:26.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:47:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:47:26.587Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:47:28.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:47:28 smithi083 ceph-mon[79930]: pgmap v18573: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:28.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:47:28 smithi083 ceph-mon[77242]: pgmap v18573: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:47:28 smithi153 ceph-mon[70044]: pgmap v18573: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:30.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:47:30 smithi083 ceph-mon[77242]: pgmap v18574: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:30.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:47:30 smithi083 ceph-mon[79930]: pgmap v18574: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:47:30 smithi153 ceph-mon[70044]: pgmap v18574: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:47:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:47:31] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T15:47:32.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:47:32 smithi083 ceph-mon[79930]: pgmap v18575: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:32.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:47:32 smithi083 ceph-mon[77242]: pgmap v18575: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:47:32 smithi153 ceph-mon[70044]: pgmap v18575: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:32.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:47:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:47:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:47:32.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:47:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:47:32.492Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:47:33.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:47:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:47:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:47:34.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:47:33 smithi083 ceph-mon[77242]: pgmap v18576: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:34.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:47:33 smithi083 ceph-mon[79930]: pgmap v18576: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:34.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:47:33 smithi153 ceph-mon[70044]: pgmap v18576: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:36.119 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:47:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:47:35.843Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:47:36.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:47:36 smithi083 ceph-mon[77242]: pgmap v18577: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:36.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:47:36 smithi083 ceph-mon[79930]: pgmap v18577: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:36.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:47:36 smithi153 ceph-mon[70044]: pgmap v18577: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:36.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:47:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:47:36.588Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:47:38.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:47:38 smithi083 ceph-mon[79930]: pgmap v18578: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:38.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:47:38 smithi083 ceph-mon[77242]: pgmap v18578: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:38.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:47:38 smithi153 ceph-mon[70044]: pgmap v18578: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:40.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:47:40 smithi083 ceph-mon[77242]: pgmap v18579: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:40.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:47:40 smithi083 ceph-mon[79930]: pgmap v18579: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:47:40 smithi153 ceph-mon[70044]: pgmap v18579: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:47:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:47:41] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T15:47:42.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:47:42 smithi083 ceph-mon[79930]: pgmap v18580: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:42.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:47:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:47:42.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:47:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:47:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:47:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:47:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:47:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:47:42.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:47:42 smithi083 ceph-mon[77242]: pgmap v18580: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:42.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:47:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:47:42.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:47:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:47:42.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:47:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:47:42.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:47:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:47:42.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:47:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:47:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:47:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:47:42 smithi153 ceph-mon[70044]: pgmap v18580: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:47:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:47:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:47:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:47:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:47:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:47:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:47:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:47:42.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:47:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:47:42.493Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:47:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:47:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:47:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:47:44.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:47:43 smithi083 ceph-mon[79930]: pgmap v18581: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:44.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:47:43 smithi083 ceph-mon[77242]: pgmap v18581: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:44.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:47:43 smithi153 ceph-mon[70044]: pgmap v18581: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:46.123 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:47:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:47:45.845Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:47:46.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:47:46 smithi083 ceph-mon[77242]: pgmap v18582: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:46.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:47:46 smithi083 ceph-mon[79930]: pgmap v18582: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:46.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:47:46 smithi153 ceph-mon[70044]: pgmap v18582: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:46.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:47:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:47:46.589Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:47:48.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:47:48 smithi083 ceph-mon[77242]: pgmap v18583: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:48.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:47:48 smithi083 ceph-mon[79930]: pgmap v18583: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:48.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:47:48 smithi153 ceph-mon[70044]: pgmap v18583: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:49.110 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:47:49.396 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:47:49.397 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 7m ago 10h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:47:49.397 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 5m ago 10h 64.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:47:49.398 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 7m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:47:49.398 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 5m ago 10h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:47:49.398 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 5m ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:47:49.399 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 7m ago 10h 1037M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:47:49.399 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 7m ago 10h 2229M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:47:49.399 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 5m ago 10h 973M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:47:49.399 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 7m ago 10h 1339M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:47:49.400 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 7m ago 10h 23.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:47:49.400 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 5m ago 10h 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:47:49.400 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 7m ago 10h 1620M 2133M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:47:49.400 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 7m ago 10h 1607M 2133M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:47:49.401 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 7m ago 10h 1248M 2133M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:47:49.401 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 7m ago 10h 1574M 2133M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:47:49.401 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 5m ago 10h 1381M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:47:49.401 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 5m ago 10h 1734M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:47:49.402 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 5m ago 10h 1550M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:47:49.402 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 5m ago 10h 1845M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:47:49.402 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 5m ago 10h 180M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:47:49.750 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:47:49.751 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:47:49.751 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:47:49.751 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:47:49.751 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:47:49.752 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:47:49.752 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:47:49.752 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:47:49.752 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:47:49.753 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:47:49.753 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:47:49.753 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:47:49.753 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:47:49.754 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:47:49.754 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:47:49.754 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:47:49.754 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:47:49.755 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:47:50.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:47:50 smithi083 ceph-mon[79930]: pgmap v18584: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:50.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:47:50 smithi083 ceph-mon[79930]: from='client.55740 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:47:50.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:47:50 smithi083 ceph-mon[79930]: from='client.55746 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:47:50.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:47:50 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/727660015' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:47:50.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:47:50 smithi083 ceph-mon[77242]: pgmap v18584: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:50.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:47:50 smithi083 ceph-mon[77242]: from='client.55740 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:47:50.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:47:50 smithi083 ceph-mon[77242]: from='client.55746 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:47:50.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:47:50 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/727660015' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:47:50.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:47:50 smithi153 ceph-mon[70044]: pgmap v18584: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:47:50 smithi153 ceph-mon[70044]: from='client.55740 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:47:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:47:50 smithi153 ceph-mon[70044]: from='client.55746 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:47:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:47:50 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/727660015' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:47:51.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:47:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:47:51] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T15:47:52.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:47:52 smithi083 ceph-mon[79930]: pgmap v18585: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:52.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:47:52 smithi083 ceph-mon[77242]: pgmap v18585: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:52.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:47:52 smithi153 ceph-mon[70044]: pgmap v18585: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:52.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:47:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:47:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:47:52.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:47:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:47:52.493Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:47:53.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:47:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:47:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:47:54.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:47:53 smithi083 ceph-mon[77242]: pgmap v18586: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:54.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:47:53 smithi083 ceph-mon[79930]: pgmap v18586: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:54.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:47:53 smithi153 ceph-mon[70044]: pgmap v18586: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:56.126 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:47:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:47:55.844Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:47:56.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:47:56 smithi083 ceph-mon[79930]: pgmap v18587: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:56.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:47:56 smithi083 ceph-mon[77242]: pgmap v18587: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:56.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:47:56 smithi153 ceph-mon[70044]: pgmap v18587: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:56.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:47:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:47:56.589Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:47:58.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:47:58 smithi083 ceph-mon[79930]: pgmap v18588: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:58.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:47:58 smithi083 ceph-mon[77242]: pgmap v18588: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:47:58.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:47:58 smithi153 ceph-mon[70044]: pgmap v18588: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:00.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:48:00 smithi083 ceph-mon[79930]: pgmap v18589: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:00.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:48:00 smithi083 ceph-mon[77242]: pgmap v18589: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:00.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:48:00 smithi153 ceph-mon[70044]: pgmap v18589: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:01.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:48:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:48:01] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T15:48:02.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:48:02 smithi083 ceph-mon[79930]: pgmap v18590: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:02.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:48:02 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:48:02.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:48:02 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:48:02.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:48:02 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:48:02.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:48:02 smithi083 ceph-mon[77242]: pgmap v18590: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:02.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:48:02 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:48:02.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:48:02 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:48:02.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:48:02 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:48:02.556 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:48:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:48:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:48:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:48:02 smithi153 ceph-mon[70044]: pgmap v18590: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:48:02 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:48:02.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:48:02 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:48:02.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:48:02 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:48:02.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:48:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:48:02.494Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:48:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:48:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:48:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:48:03.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:48:03 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:48:03.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:48:03 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:48:03.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:48:03 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:48:03.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:48:03 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:48:03.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:48:03 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:48:03.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:48:03 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:48:03.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:48:03 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:48:03.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:48:03 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:48:03.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:48:03 smithi153 ceph-mon[70044]: Adjusting osd_memory_target on smithi083 to 2120M 2022-09-29T15:48:03.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:48:03 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:48:03.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:48:03 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:48:03.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:48:03 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:48:03.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:48:03 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:48:03.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:48:03 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:48:03.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:48:03 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:48:03.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:48:03 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:48:03.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:48:03 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:48:03.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:48:03 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:48:03.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:48:03 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:48:03.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:48:03 smithi083 ceph-mon[77242]: Adjusting osd_memory_target on smithi083 to 2120M 2022-09-29T15:48:03.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:48:03 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:48:03.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:48:03 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:48:03.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:48:03 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:48:03.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:48:03 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:48:03.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:48:03 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:48:03.467 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:48:03 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:48:03.467 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:48:03 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:48:03.467 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:48:03 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:48:03.467 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:48:03 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:48:03.468 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:48:03 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:48:03.468 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:48:03 smithi083 ceph-mon[79930]: Adjusting osd_memory_target on smithi083 to 2120M 2022-09-29T15:48:03.468 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:48:03 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:48:03.468 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:48:03 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:48:04.460 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:48:04 smithi083 ceph-mon[77242]: pgmap v18591: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:04.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:48:04 smithi083 ceph-mon[79930]: pgmap v18591: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:04.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:48:04 smithi153 ceph-mon[70044]: pgmap v18591: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:06.140 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:48:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:48:05.845Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:48:06.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:48:06 smithi083 ceph-mon[77242]: pgmap v18592: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:06.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:48:06 smithi083 ceph-mon[79930]: pgmap v18592: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:06.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:48:06 smithi153 ceph-mon[70044]: pgmap v18592: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:06.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:48:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:48:06.590Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:48:08.460 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:48:08 smithi083 ceph-mon[77242]: pgmap v18593: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:08.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:48:08 smithi083 ceph-mon[79930]: pgmap v18593: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:48:08 smithi153 ceph-mon[70044]: pgmap v18593: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:10.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:48:10 smithi083 ceph-mon[79930]: pgmap v18594: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:10.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:48:10 smithi083 ceph-mon[77242]: pgmap v18594: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:10.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:48:10 smithi153 ceph-mon[70044]: pgmap v18594: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:48:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:48:11] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T15:48:12.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:48:12 smithi083 ceph-mon[79930]: pgmap v18595: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:12.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:48:12 smithi083 ceph-mon[77242]: pgmap v18595: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:48:12 smithi153 ceph-mon[70044]: pgmap v18595: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:12.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:48:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:48:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:48:12.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:48:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:48:12.495Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:48:13.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:48:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:48:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:48:14.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:48:13 smithi083 ceph-mon[77242]: pgmap v18596: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:14.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:48:13 smithi083 ceph-mon[79930]: pgmap v18596: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:14.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:48:13 smithi153 ceph-mon[70044]: pgmap v18596: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:16.132 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:48:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:48:15.846Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:48:16.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:48:16 smithi083 ceph-mon[79930]: pgmap v18597: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:16.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:48:16 smithi083 ceph-mon[77242]: pgmap v18597: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:16.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:48:16 smithi153 ceph-mon[70044]: pgmap v18597: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:16.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:48:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:48:16.590Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:48:18.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:48:18 smithi083 ceph-mon[77242]: pgmap v18598: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:18.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:48:18 smithi083 ceph-mon[79930]: pgmap v18598: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:48:18 smithi153 ceph-mon[70044]: pgmap v18598: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:20.073 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:48:20.365 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:48:20.380 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 8m ago 10h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:48:20.380 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 6m ago 10h 64.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:48:20.381 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 8m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:48:20.381 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 6m ago 10h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:48:20.381 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 6m ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:48:20.382 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 8m ago 10h 1037M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:48:20.382 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 8m ago 10h 2229M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:48:20.383 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 6m ago 10h 973M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:48:20.383 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 8m ago 10h 1339M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:48:20.384 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 8m ago 10h 23.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:48:20.384 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 6m ago 10h 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:48:20.385 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 8m ago 10h 1620M 2120M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:48:20.385 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 8m ago 10h 1607M 2120M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:48:20.386 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 8m ago 10h 1248M 2120M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:48:20.386 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 8m ago 10h 1574M 2120M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:48:20.386 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 6m ago 10h 1381M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:48:20.387 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 6m ago 10h 1734M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:48:20.387 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 6m ago 10h 1550M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:48:20.388 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 6m ago 10h 1845M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:48:20.388 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 6m ago 10h 180M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:48:20.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:48:20 smithi083 ceph-mon[79930]: pgmap v18599: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:20.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:48:20 smithi083 ceph-mon[77242]: pgmap v18599: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:48:20 smithi153 ceph-mon[70044]: pgmap v18599: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:20.723 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:48:20.724 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:48:20.724 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:48:20.724 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:48:20.724 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:48:20.725 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:48:20.725 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:48:20.725 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:48:20.725 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:48:20.726 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:48:20.726 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:48:20.726 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:48:20.726 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:48:20.727 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:48:20.727 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:48:20.727 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:48:20.727 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:48:20.728 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:48:21.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:48:21 smithi083 ceph-mon[79930]: from='client.55758 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:48:21.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:48:21 smithi083 ceph-mon[79930]: from='client.65695 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:48:21.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:48:21 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/66216538' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:48:21.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:48:21 smithi083 ceph-mon[77242]: from='client.55758 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:48:21.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:48:21 smithi083 ceph-mon[77242]: from='client.65695 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:48:21.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:48:21 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/66216538' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:48:21.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:48:21 smithi153 ceph-mon[70044]: from='client.55758 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:48:21.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:48:21 smithi153 ceph-mon[70044]: from='client.65695 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:48:21.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:48:21 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/66216538' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:48:21.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:48:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:48:21] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T15:48:22.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:48:22 smithi083 ceph-mon[79930]: pgmap v18600: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:22.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:48:22 smithi083 ceph-mon[77242]: pgmap v18600: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:48:22 smithi153 ceph-mon[70044]: pgmap v18600: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:22.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:48:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:48:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:48:22.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:48:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:48:22.496Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:48:23.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:48:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:48:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:48:24.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:48:23 smithi083 ceph-mon[79930]: pgmap v18601: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:24.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:48:23 smithi083 ceph-mon[77242]: pgmap v18601: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:24.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:48:23 smithi153 ceph-mon[70044]: pgmap v18601: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:26.135 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:48:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:48:25.847Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:48:26.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:48:26 smithi083 ceph-mon[79930]: pgmap v18602: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:26.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:48:26 smithi083 ceph-mon[77242]: pgmap v18602: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:26.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:48:26 smithi153 ceph-mon[70044]: pgmap v18602: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:26.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:48:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:48:26.591Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:48:28.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:48:28 smithi083 ceph-mon[79930]: pgmap v18603: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:28.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:48:28 smithi083 ceph-mon[77242]: pgmap v18603: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:28.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:48:28 smithi153 ceph-mon[70044]: pgmap v18603: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:30.468 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:48:30 smithi083 ceph-mon[79930]: pgmap v18604: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:30.469 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:48:30 smithi083 ceph-mon[77242]: pgmap v18604: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:48:30 smithi153 ceph-mon[70044]: pgmap v18604: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:31.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:48:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:48:31] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T15:48:32.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:48:32 smithi083 ceph-mon[79930]: pgmap v18605: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:32.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:48:32 smithi083 ceph-mon[77242]: pgmap v18605: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:48:32 smithi153 ceph-mon[70044]: pgmap v18605: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:32.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:48:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:48:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:48:32.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:48:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:48:32.496Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:48:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:48:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:48:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:48:34.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:48:33 smithi083 ceph-mon[79930]: pgmap v18606: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:34.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:48:33 smithi083 ceph-mon[77242]: pgmap v18606: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:34.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:48:33 smithi153 ceph-mon[70044]: pgmap v18606: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:36.143 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:48:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:48:35.848Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:48:36.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:48:36 smithi083 ceph-mon[79930]: pgmap v18607: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:36.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:48:36 smithi083 ceph-mon[77242]: pgmap v18607: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:48:36 smithi153 ceph-mon[70044]: pgmap v18607: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:36.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:48:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:48:36.592Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:48:38.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:48:38 smithi083 ceph-mon[79930]: pgmap v18608: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:38.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:48:38 smithi083 ceph-mon[77242]: pgmap v18608: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:48:38 smithi153 ceph-mon[70044]: pgmap v18608: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:40.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:48:40 smithi083 ceph-mon[79930]: pgmap v18609: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:40.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:48:40 smithi083 ceph-mon[77242]: pgmap v18609: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:40.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:48:40 smithi153 ceph-mon[70044]: pgmap v18609: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:41.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:48:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:48:41] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T15:48:42.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:48:42 smithi083 ceph-mon[79930]: pgmap v18610: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:42.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:48:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:48:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:48:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:48:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:48:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:48:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:48:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:48:42.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:48:42 smithi083 ceph-mon[77242]: pgmap v18610: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:42.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:48:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:48:42.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:48:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:48:42.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:48:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:48:42.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:48:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:48:42.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:48:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:48:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:48:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:48:42 smithi153 ceph-mon[70044]: pgmap v18610: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:48:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:48:42.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:48:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:48:42.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:48:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:48:42.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:48:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:48:42.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:48:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:48:42.497Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:48:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:48:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:48:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:48:44.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:48:43 smithi083 ceph-mon[79930]: pgmap v18611: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:44.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:48:43 smithi083 ceph-mon[77242]: pgmap v18611: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:44.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:48:43 smithi153 ceph-mon[70044]: pgmap v18611: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:46.141 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:48:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:48:45.848Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:48:46.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:48:46 smithi083 ceph-mon[79930]: pgmap v18612: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:46.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:48:46 smithi083 ceph-mon[77242]: pgmap v18612: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:46.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:48:46 smithi153 ceph-mon[70044]: pgmap v18612: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:46.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:48:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:48:46.593Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:48:48.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:48:48 smithi083 ceph-mon[77242]: pgmap v18613: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:48.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:48:48 smithi083 ceph-mon[79930]: pgmap v18613: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:48.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:48:48 smithi153 ceph-mon[70044]: pgmap v18613: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:50.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:48:50 smithi083 ceph-mon[79930]: pgmap v18614: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:50.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:48:50 smithi083 ceph-mon[77242]: pgmap v18614: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:48:50 smithi153 ceph-mon[70044]: pgmap v18614: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:51.054 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:48:51.343 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:48:51.344 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 8m ago 10h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:48:51.344 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 6m ago 10h 64.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:48:51.344 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 8m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:48:51.345 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 6m ago 10h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:48:51.345 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 6m ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:48:51.345 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 8m ago 10h 1037M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:48:51.345 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 8m ago 10h 2229M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:48:51.345 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 6m ago 10h 973M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:48:51.346 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 8m ago 10h 1339M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:48:51.346 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 8m ago 10h 23.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:48:51.346 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 6m ago 10h 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:48:51.346 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 8m ago 10h 1620M 2120M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:48:51.347 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 8m ago 10h 1607M 2120M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:48:51.347 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 8m ago 10h 1248M 2120M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:48:51.347 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 8m ago 10h 1574M 2120M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:48:51.347 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 6m ago 10h 1381M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:48:51.348 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 6m ago 10h 1734M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:48:51.348 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 6m ago 10h 1550M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:48:51.348 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 6m ago 10h 1845M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:48:51.348 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 6m ago 10h 180M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:48:51.699 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:48:51.700 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:48:51.700 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:48:51.700 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:48:51.701 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:48:51.701 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:48:51.701 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:48:51.701 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:48:51.702 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:48:51.702 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:48:51.702 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:48:51.702 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:48:51.702 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:48:51.703 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:48:51.703 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:48:51.703 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:48:51.704 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:48:51.704 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:48:51.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:48:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:48:51] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T15:48:52.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:48:52 smithi083 ceph-mon[79930]: from='client.55776 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:48:52.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:48:52 smithi083 ceph-mon[79930]: pgmap v18615: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:52.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:48:52 smithi083 ceph-mon[79930]: from='client.65713 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:48:52.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:48:52 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1323783625' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:48:52.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:48:52 smithi083 ceph-mon[77242]: from='client.55776 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:48:52.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:48:52 smithi083 ceph-mon[77242]: pgmap v18615: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:52.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:48:52 smithi083 ceph-mon[77242]: from='client.65713 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:48:52.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:48:52 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1323783625' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:48:52.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:48:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:48:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:48:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:48:52 smithi153 ceph-mon[70044]: from='client.55776 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:48:52.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:48:52 smithi153 ceph-mon[70044]: pgmap v18615: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:52.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:48:52 smithi153 ceph-mon[70044]: from='client.65713 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:48:52.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:48:52 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1323783625' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:48:52.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:48:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:48:52.498Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:48:53.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:48:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:48:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:48:54.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:48:53 smithi083 ceph-mon[79930]: pgmap v18616: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:54.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:48:53 smithi083 ceph-mon[77242]: pgmap v18616: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:54.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:48:53 smithi153 ceph-mon[70044]: pgmap v18616: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:56.144 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:48:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:48:55.849Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:48:56.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:48:56 smithi083 ceph-mon[77242]: pgmap v18617: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:56.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:48:56 smithi083 ceph-mon[79930]: pgmap v18617: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:56.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:48:56 smithi153 ceph-mon[70044]: pgmap v18617: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:56.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:48:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:48:56.593Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:48:58.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:48:58 smithi083 ceph-mon[79930]: pgmap v18618: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:58.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:48:58 smithi083 ceph-mon[77242]: pgmap v18618: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:48:58.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:48:58 smithi153 ceph-mon[70044]: pgmap v18618: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:00.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:49:00 smithi083 ceph-mon[77242]: pgmap v18619: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:00.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:49:00 smithi083 ceph-mon[79930]: pgmap v18619: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:00.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:49:00 smithi153 ceph-mon[70044]: pgmap v18619: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:49:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:49:01] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T15:49:02.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:49:02 smithi083 ceph-mon[79930]: pgmap v18620: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:02.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:49:02 smithi083 ceph-mon[77242]: pgmap v18620: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:02.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:49:02 smithi153 ceph-mon[70044]: pgmap v18620: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:02.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:49:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:49:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:49:02.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:49:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:49:02.498Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:49:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:49:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:49:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:49:03.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:49:03 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:49:03.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:49:03 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:49:03.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:49:03 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:49:03.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:49:03 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:49:03.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:49:03 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:49:03.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:49:03 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:49:03.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:49:03 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:49:03.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:49:03 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:49:03.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:49:03 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:49:03.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:49:03 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:49:03.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:49:03 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:49:03.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:49:03 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:49:04.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:49:04 smithi083 ceph-mon[77242]: pgmap v18621: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:04.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:49:04 smithi083 ceph-mon[79930]: pgmap v18621: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:04.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:49:04 smithi153 ceph-mon[70044]: pgmap v18621: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:06.161 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:49:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:49:05.850Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:49:06.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:49:06 smithi083 ceph-mon[79930]: pgmap v18622: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:06.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:49:06 smithi083 ceph-mon[77242]: pgmap v18622: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:06.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:49:06 smithi153 ceph-mon[70044]: pgmap v18622: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:06.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:49:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:49:06.594Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:49:08.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:49:08 smithi083 ceph-mon[79930]: pgmap v18623: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:08.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:49:08 smithi083 ceph-mon[77242]: pgmap v18623: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:08.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:49:08 smithi153 ceph-mon[70044]: pgmap v18623: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:10.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:49:10 smithi083 ceph-mon[79930]: pgmap v18624: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:10.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:49:10 smithi083 ceph-mon[77242]: pgmap v18624: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:10.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:49:10 smithi153 ceph-mon[70044]: pgmap v18624: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:49:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:49:11] "GET /metrics HTTP/1.1" 200 225246 "" "Prometheus/2.33.4" 2022-09-29T15:49:12.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:49:12 smithi083 ceph-mon[79930]: pgmap v18625: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:12.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:49:12 smithi083 ceph-mon[77242]: pgmap v18625: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:49:12 smithi153 ceph-mon[70044]: pgmap v18625: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:12.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:49:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:49:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:49:12.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:49:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:49:12.500Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:49:13.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:49:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:49:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:49:14.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:49:13 smithi083 ceph-mon[79930]: pgmap v18626: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:14.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:49:13 smithi083 ceph-mon[77242]: pgmap v18626: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:14.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:49:13 smithi153 ceph-mon[70044]: pgmap v18626: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:16.150 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:49:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:49:15.850Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:49:16.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:49:16 smithi083 ceph-mon[79930]: pgmap v18627: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:16.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:49:16 smithi083 ceph-mon[77242]: pgmap v18627: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:16.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:49:16 smithi153 ceph-mon[70044]: pgmap v18627: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:16.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:49:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:49:16.594Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:49:18.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:49:18 smithi083 ceph-mon[79930]: pgmap v18628: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:18.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:49:18 smithi083 ceph-mon[77242]: pgmap v18628: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:49:18 smithi153 ceph-mon[70044]: pgmap v18628: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:20.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:49:20 smithi083 ceph-mon[79930]: pgmap v18629: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:20.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:49:20 smithi083 ceph-mon[77242]: pgmap v18629: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:20.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:49:20 smithi153 ceph-mon[70044]: pgmap v18629: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:21.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:49:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:49:21] "GET /metrics HTTP/1.1" 200 225246 "" "Prometheus/2.33.4" 2022-09-29T15:49:22.024 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:49:22.316 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:49:22.316 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 9m ago 10h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:49:22.317 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 7m ago 10h 64.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:49:22.317 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 9m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:49:22.317 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 7m ago 10h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:49:22.318 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 7m ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:49:22.318 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 9m ago 10h 1037M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:49:22.318 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 9m ago 10h 2229M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:49:22.318 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 7m ago 10h 973M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:49:22.318 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 9m ago 10h 1339M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:49:22.319 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 9m ago 10h 23.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:49:22.319 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 7m ago 10h 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:49:22.319 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 9m ago 10h 1620M 2120M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:49:22.319 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 9m ago 10h 1607M 2120M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:49:22.320 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 9m ago 10h 1248M 2120M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:49:22.320 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 9m ago 10h 1574M 2120M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:49:22.320 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 7m ago 10h 1381M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:49:22.320 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 7m ago 10h 1734M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:49:22.321 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 7m ago 10h 1550M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:49:22.321 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 7m ago 10h 1845M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:49:22.321 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 7m ago 10h 180M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:49:22.460 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:49:22 smithi083 ceph-mon[77242]: pgmap v18630: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:22.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:49:22 smithi083 ceph-mon[79930]: pgmap v18630: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:22.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:49:22 smithi153 ceph-mon[70044]: pgmap v18630: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:22.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:49:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:49:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:49:22.672 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:49:22.672 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:49:22.672 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:49:22.672 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:49:22.673 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:49:22.673 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:49:22.673 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:49:22.673 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:49:22.674 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:49:22.674 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:49:22.674 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:49:22.674 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:49:22.675 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:49:22.675 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:49:22.675 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:49:22.675 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:49:22.675 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:49:22.676 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:49:22.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:49:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:49:22.499Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:49:23.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:49:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:49:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:49:23.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:49:23 smithi153 ceph-mon[70044]: from='client.55794 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:49:23.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:49:23 smithi153 ceph-mon[70044]: from='client.65731 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:49:23.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:49:23 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2684995088' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:49:23.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:49:23 smithi083 ceph-mon[79930]: from='client.55794 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:49:23.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:49:23 smithi083 ceph-mon[79930]: from='client.65731 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:49:23.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:49:23 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2684995088' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:49:23.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:49:23 smithi083 ceph-mon[77242]: from='client.55794 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:49:23.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:49:23 smithi083 ceph-mon[77242]: from='client.65731 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:49:23.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:49:23 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2684995088' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:49:24.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:49:24 smithi083 ceph-mon[79930]: pgmap v18631: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:24.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:49:24 smithi083 ceph-mon[77242]: pgmap v18631: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:49:24 smithi153 ceph-mon[70044]: pgmap v18631: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:26.170 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:49:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:49:25.850Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:49:26.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:49:26 smithi083 ceph-mon[79930]: pgmap v18632: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:26.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:49:26 smithi083 ceph-mon[77242]: pgmap v18632: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:26.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:49:26 smithi153 ceph-mon[70044]: pgmap v18632: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:26.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:49:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:49:26.595Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:49:28.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:49:28 smithi083 ceph-mon[77242]: pgmap v18633: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:28.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:49:28 smithi083 ceph-mon[79930]: pgmap v18633: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:28.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:49:28 smithi153 ceph-mon[70044]: pgmap v18633: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:30.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:49:30 smithi083 ceph-mon[77242]: pgmap v18634: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:30.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:49:30 smithi083 ceph-mon[79930]: pgmap v18634: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:30.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:49:30 smithi153 ceph-mon[70044]: pgmap v18634: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:31.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:49:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:49:31] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T15:49:32.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:49:32 smithi083 ceph-mon[79930]: pgmap v18635: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:32.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:49:32 smithi083 ceph-mon[77242]: pgmap v18635: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:32.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:49:32 smithi153 ceph-mon[70044]: pgmap v18635: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:32.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:49:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:49:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:49:32.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:49:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:49:32.501Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:49:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:49:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:49:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:49:34.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:49:33 smithi083 ceph-mon[79930]: pgmap v18636: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:34.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:49:33 smithi083 ceph-mon[77242]: pgmap v18636: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:34.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:49:33 smithi153 ceph-mon[70044]: pgmap v18636: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:36.156 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:49:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:49:35.850Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:49:36.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:49:36 smithi083 ceph-mon[77242]: pgmap v18637: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:36.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:49:36 smithi083 ceph-mon[79930]: pgmap v18637: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:49:36 smithi153 ceph-mon[70044]: pgmap v18637: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:36.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:49:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:49:36.596Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:49:38.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:49:38 smithi083 ceph-mon[79930]: pgmap v18638: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:38.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:49:38 smithi083 ceph-mon[77242]: pgmap v18638: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:38.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:49:38 smithi153 ceph-mon[70044]: pgmap v18638: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:40.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:49:40 smithi083 ceph-mon[77242]: pgmap v18639: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:40.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:49:40 smithi083 ceph-mon[79930]: pgmap v18639: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:40.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:49:40 smithi153 ceph-mon[70044]: pgmap v18639: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:49:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:49:41] "GET /metrics HTTP/1.1" 200 225246 "" "Prometheus/2.33.4" 2022-09-29T15:49:42.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:49:42 smithi083 ceph-mon[79930]: pgmap v18640: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:42.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:49:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:49:42.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:49:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:49:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:49:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:49:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:49:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:49:42.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:49:42 smithi083 ceph-mon[77242]: pgmap v18640: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:42.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:49:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:49:42.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:49:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:49:42.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:49:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:49:42.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:49:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:49:42.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:49:42 smithi153 ceph-mon[70044]: pgmap v18640: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:49:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:49:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:49:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:49:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:49:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:49:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:49:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:49:42.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:49:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:49:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:49:42.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:49:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:49:42.502Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:49:43.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:49:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:49:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:49:44.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:49:43 smithi083 ceph-mon[79930]: pgmap v18641: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:44.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:49:43 smithi083 ceph-mon[77242]: pgmap v18641: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:44.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:49:43 smithi153 ceph-mon[70044]: pgmap v18641: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:46.160 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:49:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:49:45.851Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:49:46.469 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:49:46 smithi083 ceph-mon[79930]: pgmap v18642: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:46.470 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:49:46 smithi083 ceph-mon[77242]: pgmap v18642: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:46.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:49:46 smithi153 ceph-mon[70044]: pgmap v18642: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:46.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:49:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:49:46.597Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:49:48.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:49:48 smithi083 ceph-mon[77242]: pgmap v18643: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:48.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:49:48 smithi083 ceph-mon[79930]: pgmap v18643: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:48.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:49:48 smithi153 ceph-mon[70044]: pgmap v18643: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:50.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:49:50 smithi083 ceph-mon[79930]: pgmap v18644: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:50.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:49:50 smithi083 ceph-mon[77242]: pgmap v18644: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:49:50 smithi153 ceph-mon[70044]: pgmap v18644: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:51.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:49:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:49:51] "GET /metrics HTTP/1.1" 200 225246 "" "Prometheus/2.33.4" 2022-09-29T15:49:52.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:49:52 smithi083 ceph-mon[79930]: pgmap v18645: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:52.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:49:52 smithi083 ceph-mon[77242]: pgmap v18645: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:49:52 smithi153 ceph-mon[70044]: pgmap v18645: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:52.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:49:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:49:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:49:52.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:49:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:49:52.502Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:49:53.001 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:49:53.290 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:49:53.290 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 10m ago 10h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:49:53.290 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 7m ago 10h 64.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:49:53.291 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 10m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:49:53.291 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 7m ago 10h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:49:53.291 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 7m ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:49:53.291 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 10m ago 10h 1037M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:49:53.292 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 10m ago 10h 2229M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:49:53.292 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 7m ago 10h 973M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:49:53.292 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 10m ago 10h 1339M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:49:53.292 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 10m ago 10h 23.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:49:53.292 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 7m ago 10h 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:49:53.293 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 10m ago 10h 1620M 2120M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:49:53.293 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 10m ago 10h 1607M 2120M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:49:53.293 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 10m ago 10h 1248M 2120M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:49:53.293 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 10m ago 10h 1574M 2120M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:49:53.294 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 7m ago 10h 1381M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:49:53.294 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 7m ago 10h 1734M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:49:53.294 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 7m ago 10h 1550M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:49:53.294 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 7m ago 10h 1845M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:49:53.294 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 7m ago 10h 180M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:49:53.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:49:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:49:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:49:53.665 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:49:53.665 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:49:53.665 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:49:53.665 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:49:53.666 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:49:53.666 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:49:53.666 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:49:53.666 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:49:53.667 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:49:53.667 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:49:53.667 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:49:53.667 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:49:53.668 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:49:53.668 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:49:53.668 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:49:53.668 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:49:53.669 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:49:53.669 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:49:54.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:49:53 smithi083 ceph-mon[79930]: from='client.55812 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:49:54.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:49:53 smithi083 ceph-mon[79930]: pgmap v18646: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:54.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:49:53 smithi083 ceph-mon[79930]: from='client.55818 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:49:54.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:49:53 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1270404873' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:49:54.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:49:53 smithi083 ceph-mon[77242]: from='client.55812 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:49:54.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:49:53 smithi083 ceph-mon[77242]: pgmap v18646: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:54.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:49:53 smithi083 ceph-mon[77242]: from='client.55818 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:49:54.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:49:53 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1270404873' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:49:54.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:49:53 smithi153 ceph-mon[70044]: from='client.55812 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:49:54.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:49:53 smithi153 ceph-mon[70044]: pgmap v18646: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:54.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:49:53 smithi153 ceph-mon[70044]: from='client.55818 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:49:54.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:49:53 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1270404873' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:49:56.164 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:49:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:49:55.851Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:49:56.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:49:56 smithi083 ceph-mon[79930]: pgmap v18647: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:56.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:49:56 smithi083 ceph-mon[77242]: pgmap v18647: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:56.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:49:56 smithi153 ceph-mon[70044]: pgmap v18647: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:56.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:49:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:49:56.598Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:49:58.460 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:49:58 smithi083 ceph-mon[77242]: pgmap v18648: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:58.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:49:58 smithi083 ceph-mon[79930]: pgmap v18648: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:49:58.563 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:49:58 smithi153 ceph-mon[70044]: pgmap v18648: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:00.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:50:00 smithi083 ceph-mon[79930]: pgmap v18649: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:00.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:50:00 smithi083 ceph-mon[79930]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T15:50:00.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:50:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[77219]: 2022-09-29T15:49:59.999+0000 7f160f665700 -1 log_channel(cluster) log [ERR] : overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T15:50:00.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:50:00 smithi083 ceph-mon[77242]: pgmap v18649: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:00.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:50:00 smithi083 ceph-mon[77242]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T15:50:00.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:50:00 smithi153 ceph-mon[70044]: pgmap v18649: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:50:00 smithi153 ceph-mon[70044]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T15:50:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:50:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:50:01] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T15:50:02.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:50:02 smithi083 ceph-mon[79930]: pgmap v18650: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:02.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:50:02 smithi083 ceph-mon[77242]: pgmap v18650: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:50:02 smithi153 ceph-mon[70044]: pgmap v18650: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:02.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:50:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:50:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:50:02.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:50:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:50:02.502Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:50:03.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:50:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:50:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:50:04.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:50:03 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:50:04.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:50:03 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:50:04.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:50:03 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:50:04.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:50:03 smithi083 ceph-mon[79930]: pgmap v18651: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:04.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:50:03 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:50:04.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:50:03 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:50:04.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:50:03 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:50:04.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:50:03 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:50:04.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:50:03 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:50:04.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:50:03 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:50:04.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:50:03 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:50:04.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:50:03 smithi083 ceph-mon[77242]: pgmap v18651: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:04.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:50:03 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:50:04.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:50:03 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:50:04.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:50:03 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:50:04.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:50:03 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:50:04.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:50:03 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:50:04.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:50:03 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:50:04.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:50:03 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:50:04.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:50:03 smithi153 ceph-mon[70044]: pgmap v18651: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:04.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:50:03 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:50:04.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:50:03 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:50:04.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:50:03 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:50:04.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:50:03 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:50:06.165 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:50:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:50:05.851Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:50:06.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:50:06 smithi083 ceph-mon[77242]: pgmap v18652: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:06.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:50:06 smithi083 ceph-mon[79930]: pgmap v18652: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:06.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:50:06 smithi153 ceph-mon[70044]: pgmap v18652: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:06.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:50:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:50:06.598Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:50:08.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:50:08 smithi083 ceph-mon[79930]: pgmap v18653: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:08.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:50:08 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:50:08.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:50:08 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:50:08.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:50:08 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:50:08.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:50:08 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:50:08.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:50:08 smithi083 ceph-mon[77242]: pgmap v18653: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:08.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:50:08 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:50:08.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:50:08 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:50:08.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:50:08 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:50:08.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:50:08 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:50:09.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:50:08 smithi153 ceph-mon[70044]: pgmap v18653: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:09.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:50:08 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:50:09.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:50:08 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:50:09.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:50:08 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:50:09.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:50:08 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:50:10.886 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:50:10 smithi083 ceph-mon[79930]: pgmap v18654: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:10.886 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:50:10 smithi083 ceph-mon[77242]: pgmap v18654: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:11.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:50:10 smithi153 ceph-mon[70044]: pgmap v18654: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:50:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:50:11] "GET /metrics HTTP/1.1" 200 225247 "" "Prometheus/2.33.4" 2022-09-29T15:50:12.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:50:12 smithi083 ceph-mon[79930]: pgmap v18655: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:12.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:50:12 smithi083 ceph-mon[79930]: Detected new or changed devices on smithi083 2022-09-29T15:50:12.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:50:12 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:50:12.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:50:12 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:50:12.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:50:12 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:50:12.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:50:12 smithi083 ceph-mon[77242]: pgmap v18655: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:12.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:50:12 smithi083 ceph-mon[77242]: Detected new or changed devices on smithi083 2022-09-29T15:50:12.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:50:12 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:50:12.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:50:12 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:50:12.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:50:12 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:50:12.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:50:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:50:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:50:12.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:50:12 smithi153 ceph-mon[70044]: pgmap v18655: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:12.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:50:12 smithi153 ceph-mon[70044]: Detected new or changed devices on smithi083 2022-09-29T15:50:12.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:50:12 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:50:12.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:50:12 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:50:12.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:50:12 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:50:12.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:50:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:50:12.503Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:50:13.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:50:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:50:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:50:14.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:50:13 smithi083 ceph-mon[79930]: pgmap v18656: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:14.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:50:13 smithi083 ceph-mon[77242]: pgmap v18656: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:14.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:50:13 smithi153 ceph-mon[70044]: pgmap v18656: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:16.168 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:50:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:50:15.853Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:50:16.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:50:16 smithi083 ceph-mon[79930]: pgmap v18657: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:16.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:50:16 smithi083 ceph-mon[77242]: pgmap v18657: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:16.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:50:16 smithi153 ceph-mon[70044]: pgmap v18657: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:16.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:50:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:50:16.599Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:50:18.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:50:18 smithi083 ceph-mon[77242]: pgmap v18658: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:18.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:50:18 smithi083 ceph-mon[79930]: pgmap v18658: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:50:18 smithi153 ceph-mon[70044]: pgmap v18658: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:20.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:50:20 smithi083 ceph-mon[79930]: pgmap v18659: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:20.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:50:20 smithi083 ceph-mon[77242]: pgmap v18659: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:50:20 smithi153 ceph-mon[70044]: pgmap v18659: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:21.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:50:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:50:21] "GET /metrics HTTP/1.1" 200 225247 "" "Prometheus/2.33.4" 2022-09-29T15:50:22.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:50:22 smithi083 ceph-mon[77242]: pgmap v18660: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:22.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:50:22 smithi083 ceph-mon[79930]: pgmap v18660: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:22.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:50:22 smithi153 ceph-mon[70044]: pgmap v18660: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:22.568 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:50:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:50:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:50:22.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:50:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:50:22.504Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:50:23.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:50:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:50:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:50:23.993 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:50:24.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:50:23 smithi083 ceph-mon[79930]: pgmap v18661: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:24.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:50:23 smithi083 ceph-mon[77242]: pgmap v18661: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:24.284 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:50:24.284 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 16s ago 10h 27.3M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:50:24.284 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 8m ago 10h 64.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:50:24.285 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 16s ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:50:24.285 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 8m ago 10h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:50:24.285 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 8m ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:50:24.285 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 16s ago 10h 1044M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:50:24.286 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 16s ago 10h 2266M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:50:24.286 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 8m ago 10h 973M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:50:24.286 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 16s ago 10h 1361M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:50:24.286 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 16s ago 10h 24.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:50:24.287 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 8m ago 10h 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:50:24.287 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 16s ago 10h 1646M 2120M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:50:24.287 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 16s ago 10h 1633M 2120M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:50:24.287 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 16s ago 10h 1268M 2120M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:50:24.287 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 16s ago 10h 1600M 2120M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:50:24.288 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 8m ago 10h 1381M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:50:24.288 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 8m ago 10h 1734M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:50:24.288 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 8m ago 10h 1550M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:50:24.288 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 8m ago 10h 1845M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:50:24.289 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 8m ago 10h 180M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:50:24.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:50:23 smithi153 ceph-mon[70044]: pgmap v18661: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:24.638 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:50:24.638 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:50:24.639 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:50:24.639 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:50:24.639 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:50:24.639 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:50:24.640 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:50:24.640 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:50:24.640 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:50:24.641 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:50:24.641 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:50:24.641 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:50:24.641 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:50:24.641 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:50:24.642 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:50:24.642 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:50:24.642 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:50:24.643 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:50:25.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:50:24 smithi083 ceph-mon[79930]: from='client.65761 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:50:25.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:50:24 smithi083 ceph-mon[79930]: from='client.55836 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:50:25.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:50:24 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3746784888' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:50:25.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:50:24 smithi083 ceph-mon[77242]: from='client.65761 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:50:25.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:50:24 smithi083 ceph-mon[77242]: from='client.55836 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:50:25.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:50:24 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3746784888' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:50:25.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:50:24 smithi153 ceph-mon[70044]: from='client.65761 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:50:25.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:50:24 smithi153 ceph-mon[70044]: from='client.55836 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:50:25.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:50:24 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3746784888' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:50:26.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:50:25 smithi083 ceph-mon[79930]: pgmap v18662: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:26.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:50:25 smithi083 ceph-mon[77242]: pgmap v18662: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:26.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:50:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:50:25.852Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:50:26.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:50:25 smithi153 ceph-mon[70044]: pgmap v18662: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:26.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:50:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:50:26.600Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:50:28.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:50:28 smithi083 ceph-mon[79930]: pgmap v18663: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:28.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:50:28 smithi083 ceph-mon[77242]: pgmap v18663: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:28.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:50:28 smithi153 ceph-mon[70044]: pgmap v18663: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:30.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:50:30 smithi083 ceph-mon[79930]: pgmap v18664: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:30.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:50:30 smithi083 ceph-mon[77242]: pgmap v18664: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:30.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:50:30 smithi153 ceph-mon[70044]: pgmap v18664: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:50:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:50:31] "GET /metrics HTTP/1.1" 200 225235 "" "Prometheus/2.33.4" 2022-09-29T15:50:32.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:50:32 smithi083 ceph-mon[79930]: pgmap v18665: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:32.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:50:32 smithi083 ceph-mon[77242]: pgmap v18665: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:32.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:50:32 smithi153 ceph-mon[70044]: pgmap v18665: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:32.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:50:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:50:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:50:32.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:50:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:50:32.504Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:50:33.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:50:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:50:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:50:34.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:50:33 smithi083 ceph-mon[79930]: pgmap v18666: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:34.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:50:33 smithi083 ceph-mon[77242]: pgmap v18666: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:34.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:50:33 smithi153 ceph-mon[70044]: pgmap v18666: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:36.174 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:50:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:50:35.853Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:50:36.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:50:36 smithi083 ceph-mon[79930]: pgmap v18667: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:36.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:50:36 smithi083 ceph-mon[77242]: pgmap v18667: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:36.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:50:36 smithi153 ceph-mon[70044]: pgmap v18667: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:36.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:50:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:50:36.600Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:50:38.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:50:38 smithi083 ceph-mon[77242]: pgmap v18668: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:38.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:50:38 smithi083 ceph-mon[79930]: pgmap v18668: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:38.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:50:38 smithi153 ceph-mon[70044]: pgmap v18668: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:40.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:50:40 smithi083 ceph-mon[79930]: pgmap v18669: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:40.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:50:40 smithi083 ceph-mon[77242]: pgmap v18669: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:40.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:50:40 smithi153 ceph-mon[70044]: pgmap v18669: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:50:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:50:41] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T15:50:42.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:50:42 smithi083 ceph-mon[79930]: pgmap v18670: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:42.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:50:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:50:42.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:50:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:50:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:50:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:50:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:50:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:50:42.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:50:42 smithi083 ceph-mon[77242]: pgmap v18670: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:42.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:50:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:50:42.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:50:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:50:42.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:50:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:50:42.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:50:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:50:42.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:50:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:50:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:50:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:50:42 smithi153 ceph-mon[70044]: pgmap v18670: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:50:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:50:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:50:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:50:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:50:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:50:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:50:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:50:42.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:50:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:50:42.506Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:50:43.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:50:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:50:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:50:44.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:50:43 smithi083 ceph-mon[79930]: pgmap v18671: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:44.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:50:43 smithi083 ceph-mon[77242]: pgmap v18671: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:44.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:50:43 smithi153 ceph-mon[70044]: pgmap v18671: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:46.177 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:50:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:50:45.854Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:50:46.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:50:46 smithi083 ceph-mon[77242]: pgmap v18672: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:46.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:50:46 smithi083 ceph-mon[79930]: pgmap v18672: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:46.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:50:46 smithi153 ceph-mon[70044]: pgmap v18672: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:46.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:50:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:50:46.601Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:50:48.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:50:48 smithi083 ceph-mon[77242]: pgmap v18673: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:48.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:50:48 smithi083 ceph-mon[79930]: pgmap v18673: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:48.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:50:48 smithi153 ceph-mon[70044]: pgmap v18673: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:50.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:50:50 smithi083 ceph-mon[77242]: pgmap v18674: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:50.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:50:50 smithi083 ceph-mon[79930]: pgmap v18674: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:50:50 smithi153 ceph-mon[70044]: pgmap v18674: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:50:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:50:51] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T15:50:52.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:50:52 smithi083 ceph-mon[77242]: pgmap v18675: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:52.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:50:52 smithi083 ceph-mon[79930]: pgmap v18675: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:52.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:50:52 smithi153 ceph-mon[70044]: pgmap v18675: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:52.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:50:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:50:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:50:52.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:50:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:50:52.507Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:50:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:50:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:50:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:50:54.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:50:53 smithi083 ceph-mon[79930]: pgmap v18676: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:54.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:50:53 smithi083 ceph-mon[77242]: pgmap v18676: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:54.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:50:53 smithi153 ceph-mon[70044]: pgmap v18676: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:54.977 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:50:55.266 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:50:55.267 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 47s ago 10h 27.3M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:50:55.267 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 8m ago 10h 64.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:50:55.267 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 47s ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:50:55.268 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 8m ago 10h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:50:55.268 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 8m ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:50:55.269 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 47s ago 10h 1044M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:50:55.269 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 47s ago 10h 2266M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:50:55.270 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 8m ago 10h 973M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:50:55.270 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 47s ago 10h 1361M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:50:55.271 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 47s ago 10h 24.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:50:55.271 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 8m ago 10h 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:50:55.271 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 47s ago 10h 1646M 2120M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:50:55.272 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 47s ago 10h 1633M 2120M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:50:55.272 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 47s ago 10h 1268M 2120M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:50:55.273 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 47s ago 10h 1600M 2120M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:50:55.273 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 8m ago 10h 1381M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:50:55.274 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 8m ago 10h 1734M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:50:55.274 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 8m ago 10h 1550M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:50:55.275 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 8m ago 10h 1845M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:50:55.275 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 8m ago 10h 180M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:50:55.618 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:50:55.618 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:50:55.619 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:50:55.619 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:50:55.620 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:50:55.620 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:50:55.621 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:50:55.621 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:50:55.622 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:50:55.622 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:50:55.623 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:50:55.623 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:50:55.624 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:50:55.624 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:50:55.625 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:50:55.625 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:50:55.626 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:50:55.626 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:50:56.180 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:50:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:50:55.854Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:50:56.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:50:56 smithi083 ceph-mon[77242]: from='client.65779 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:50:56.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:50:56 smithi083 ceph-mon[77242]: pgmap v18677: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:56.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:50:56 smithi083 ceph-mon[77242]: from='client.65785 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:50:56.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:50:56 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/970919212' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:50:56.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:50:56 smithi083 ceph-mon[79930]: from='client.65779 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:50:56.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:50:56 smithi083 ceph-mon[79930]: pgmap v18677: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:56.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:50:56 smithi083 ceph-mon[79930]: from='client.65785 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:50:56.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:50:56 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/970919212' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:50:56.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:50:56 smithi153 ceph-mon[70044]: from='client.65779 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:50:56.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:50:56 smithi153 ceph-mon[70044]: pgmap v18677: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:56.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:50:56 smithi153 ceph-mon[70044]: from='client.65785 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:50:56.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:50:56 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/970919212' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:50:56.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:50:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:50:56.602Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:50:58.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:50:58 smithi083 ceph-mon[79930]: pgmap v18678: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:58.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:50:58 smithi083 ceph-mon[77242]: pgmap v18678: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:50:58.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:50:58 smithi153 ceph-mon[70044]: pgmap v18678: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:00.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:51:00 smithi083 ceph-mon[79930]: pgmap v18679: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:00.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:51:00 smithi083 ceph-mon[77242]: pgmap v18679: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:00.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:51:00 smithi153 ceph-mon[70044]: pgmap v18679: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:51:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:51:01] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T15:51:02.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:51:02 smithi083 ceph-mon[79930]: pgmap v18680: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:02.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:51:02 smithi083 ceph-mon[77242]: pgmap v18680: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:02.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:51:02 smithi153 ceph-mon[70044]: pgmap v18680: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:02.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:51:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:51:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:51:02.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:51:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:51:02.507Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:51:03.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:51:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:51:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:51:04.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:51:03 smithi083 ceph-mon[77242]: pgmap v18681: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:04.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:51:03 smithi083 ceph-mon[79930]: pgmap v18681: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:04.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:51:03 smithi153 ceph-mon[70044]: pgmap v18681: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:06.184 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:51:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:51:05.855Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:51:06.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:51:06 smithi083 ceph-mon[77242]: pgmap v18682: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:06.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:51:06 smithi083 ceph-mon[79930]: pgmap v18682: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:06.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:51:06 smithi153 ceph-mon[70044]: pgmap v18682: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:06.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:51:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:51:06.604Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:51:08.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:51:08 smithi083 ceph-mon[79930]: pgmap v18683: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:08.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:51:08 smithi083 ceph-mon[77242]: pgmap v18683: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:08.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:51:08 smithi153 ceph-mon[70044]: pgmap v18683: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:10.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:51:10 smithi083 ceph-mon[79930]: pgmap v18684: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:10.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:51:10 smithi083 ceph-mon[77242]: pgmap v18684: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:10.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:51:10 smithi153 ceph-mon[70044]: pgmap v18684: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:11.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:51:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:51:11] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T15:51:12.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:51:12 smithi083 ceph-mon[79930]: pgmap v18685: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:12.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:51:12 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:51:12.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:51:12 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:51:12.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:51:12 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:51:12.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:51:12 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:51:12.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:51:12 smithi083 ceph-mon[77242]: pgmap v18685: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:12.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:51:12 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:51:12.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:51:12 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:51:12.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:51:12 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:51:12.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:51:12 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:51:12.556 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:51:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:51:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:51:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:51:12 smithi153 ceph-mon[70044]: pgmap v18685: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:51:12 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:51:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:51:12 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:51:12.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:51:12 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:51:12.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:51:12 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:51:12.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:51:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:51:12.507Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:51:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:51:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:51:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:51:14.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:51:13 smithi083 ceph-mon[79930]: pgmap v18686: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:14.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:51:13 smithi083 ceph-mon[77242]: pgmap v18686: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:14.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:51:13 smithi153 ceph-mon[70044]: pgmap v18686: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:16.186 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:51:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:51:15.856Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:51:16.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:51:16 smithi083 ceph-mon[79930]: pgmap v18687: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:16.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:51:16 smithi083 ceph-mon[77242]: pgmap v18687: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:16.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:51:16 smithi153 ceph-mon[70044]: pgmap v18687: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:16.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:51:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:51:16.604Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:51:18.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:51:18 smithi083 ceph-mon[79930]: pgmap v18688: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:18.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:51:18 smithi083 ceph-mon[77242]: pgmap v18688: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:18.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:51:18 smithi153 ceph-mon[70044]: pgmap v18688: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:20.460 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:51:20 smithi083 ceph-mon[77242]: pgmap v18689: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:20.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:51:20 smithi083 ceph-mon[79930]: pgmap v18689: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:20.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:51:20 smithi153 ceph-mon[70044]: pgmap v18689: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:21.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:51:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:51:21] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T15:51:22.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:51:22 smithi083 ceph-mon[79930]: pgmap v18690: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:22.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:51:22 smithi083 ceph-mon[77242]: pgmap v18690: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:22.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:51:22 smithi153 ceph-mon[70044]: pgmap v18690: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:22.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:51:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:51:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:51:22.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:51:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:51:22.509Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:51:23.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:51:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:51:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:51:24.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:51:23 smithi083 ceph-mon[79930]: pgmap v18691: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:24.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:51:23 smithi083 ceph-mon[77242]: pgmap v18691: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:24.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:51:23 smithi153 ceph-mon[70044]: pgmap v18691: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:25.946 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:51:26.189 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:51:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:51:25.855Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:51:26.235 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:51:26.236 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 78s ago 10h 27.3M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:51:26.236 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 9m ago 10h 64.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:51:26.237 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 78s ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:51:26.237 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 9m ago 10h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:51:26.238 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 9m ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:51:26.238 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 78s ago 10h 1044M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:51:26.239 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 78s ago 10h 2266M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:51:26.239 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 9m ago 10h 973M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:51:26.239 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 78s ago 10h 1361M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:51:26.240 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 78s ago 10h 24.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:51:26.240 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 9m ago 10h 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:51:26.240 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 78s ago 10h 1646M 2120M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:51:26.241 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 78s ago 10h 1633M 2120M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:51:26.241 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 78s ago 10h 1268M 2120M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:51:26.241 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 78s ago 10h 1600M 2120M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:51:26.242 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 9m ago 10h 1381M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:51:26.242 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 9m ago 10h 1734M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:51:26.243 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 9m ago 10h 1550M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:51:26.243 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 9m ago 10h 1845M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:51:26.243 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 9m ago 10h 180M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:51:26.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:51:26 smithi083 ceph-mon[79930]: pgmap v18692: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:26.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:51:26 smithi083 ceph-mon[77242]: pgmap v18692: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:26.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:51:26 smithi153 ceph-mon[70044]: pgmap v18692: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:26.590 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:51:26.590 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:51:26.591 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:51:26.591 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:51:26.591 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:51:26.592 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:51:26.592 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:51:26.592 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:51:26.593 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:51:26.593 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:51:26.593 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:51:26.593 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:51:26.594 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:51:26.594 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:51:26.594 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:51:26.594 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:51:26.595 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:51:26.595 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:51:26.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:51:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:51:26.604Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:51:27.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:51:27 smithi083 ceph-mon[79930]: from='client.55866 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:51:27.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:51:27 smithi083 ceph-mon[79930]: from='client.65803 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:51:27.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:51:27 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1917509455' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:51:27.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:51:27 smithi083 ceph-mon[77242]: from='client.55866 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:51:27.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:51:27 smithi083 ceph-mon[77242]: from='client.65803 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:51:27.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:51:27 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1917509455' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:51:27.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:51:27 smithi153 ceph-mon[70044]: from='client.55866 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:51:27.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:51:27 smithi153 ceph-mon[70044]: from='client.65803 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:51:27.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:51:27 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1917509455' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:51:28.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:51:28 smithi083 ceph-mon[77242]: pgmap v18693: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:28.467 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:51:28 smithi083 ceph-mon[79930]: pgmap v18693: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:51:28 smithi153 ceph-mon[70044]: pgmap v18693: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:30.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:51:30 smithi083 ceph-mon[79930]: pgmap v18694: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:30.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:51:30 smithi083 ceph-mon[77242]: pgmap v18694: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:30.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:51:30 smithi153 ceph-mon[70044]: pgmap v18694: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:51:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:51:31] "GET /metrics HTTP/1.1" 200 225247 "" "Prometheus/2.33.4" 2022-09-29T15:51:32.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:51:32 smithi083 ceph-mon[79930]: pgmap v18695: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:32.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:51:32 smithi083 ceph-mon[77242]: pgmap v18695: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:32.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:51:32 smithi153 ceph-mon[70044]: pgmap v18695: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:32.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:51:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:51:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:51:32.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:51:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:51:32.509Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:51:33.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:51:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:51:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:51:34.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:51:33 smithi083 ceph-mon[79930]: pgmap v18696: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:34.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:51:33 smithi083 ceph-mon[77242]: pgmap v18696: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:34.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:51:33 smithi153 ceph-mon[70044]: pgmap v18696: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:36.193 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:51:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:51:35.858Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:51:36.460 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:51:36 smithi083 ceph-mon[77242]: pgmap v18697: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:36.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:51:36 smithi083 ceph-mon[79930]: pgmap v18697: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:36.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:51:36 smithi153 ceph-mon[70044]: pgmap v18697: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:36.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:51:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:51:36.606Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:51:38.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:51:38 smithi083 ceph-mon[79930]: pgmap v18698: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:38.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:51:38 smithi083 ceph-mon[77242]: pgmap v18698: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:38.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:51:38 smithi153 ceph-mon[70044]: pgmap v18698: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:40.460 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:51:40 smithi083 ceph-mon[77242]: pgmap v18699: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:40.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:51:40 smithi083 ceph-mon[79930]: pgmap v18699: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:40.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:51:40 smithi153 ceph-mon[70044]: pgmap v18699: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:51:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:51:41] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T15:51:42.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:51:42 smithi083 ceph-mon[77242]: pgmap v18700: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:42.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:51:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:51:42.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:51:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:51:42.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:51:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:51:42.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:51:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:51:42.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:51:42 smithi083 ceph-mon[79930]: pgmap v18700: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:42.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:51:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:51:42.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:51:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:51:42.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:51:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:51:42.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:51:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:51:42.556 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:51:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:51:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:51:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:51:42 smithi153 ceph-mon[70044]: pgmap v18700: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:51:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:51:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:51:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:51:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:51:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:51:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:51:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:51:42.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:51:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:51:42.510Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:51:43.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:51:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:51:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:51:44.210 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:51:43 smithi083 ceph-mon[77242]: pgmap v18701: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:44.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:51:43 smithi083 ceph-mon[79930]: pgmap v18701: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:44.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:51:43 smithi153 ceph-mon[70044]: pgmap v18701: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:46.196 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:51:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:51:45.858Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:51:46.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:51:46 smithi083 ceph-mon[79930]: pgmap v18702: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:46.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:51:46 smithi083 ceph-mon[77242]: pgmap v18702: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:46.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:51:46 smithi153 ceph-mon[70044]: pgmap v18702: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:46.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:51:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:51:46.606Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:51:48.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:51:48 smithi083 ceph-mon[79930]: pgmap v18703: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:48.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:51:48 smithi083 ceph-mon[77242]: pgmap v18703: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:48.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:51:48 smithi153 ceph-mon[70044]: pgmap v18703: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:50.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:51:50 smithi083 ceph-mon[79930]: pgmap v18704: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:50.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:51:50 smithi083 ceph-mon[77242]: pgmap v18704: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:50.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:51:50 smithi153 ceph-mon[70044]: pgmap v18704: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:51:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:51:51] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T15:51:52.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:51:52 smithi083 ceph-mon[77242]: pgmap v18705: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:52.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:51:52 smithi083 ceph-mon[79930]: pgmap v18705: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:51:52 smithi153 ceph-mon[70044]: pgmap v18705: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:52.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:51:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:51:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:51:52.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:51:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:51:52.511Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:51:53.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:51:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:51:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:51:54.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:51:53 smithi083 ceph-mon[79930]: pgmap v18706: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:54.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:51:53 smithi083 ceph-mon[77242]: pgmap v18706: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:54.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:51:53 smithi153 ceph-mon[70044]: pgmap v18706: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:56.199 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:51:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:51:55.859Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:51:56.460 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:51:56 smithi083 ceph-mon[77242]: pgmap v18707: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:56.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:51:56 smithi083 ceph-mon[79930]: pgmap v18707: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:56.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:51:56 smithi153 ceph-mon[70044]: pgmap v18707: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:56.916 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:51:56.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:51:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:51:56.607Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:51:57.205 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:51:57.205 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 109s ago 10h 27.3M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:51:57.206 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 9m ago 10h 64.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:51:57.206 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 109s ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:51:57.207 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 9m ago 10h 28.2M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:51:57.207 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 9m ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:51:57.208 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 109s ago 10h 1044M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:51:57.208 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 109s ago 10h 2266M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:51:57.209 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 9m ago 10h 973M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:51:57.209 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 109s ago 10h 1361M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:51:57.210 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 109s ago 10h 24.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:51:57.210 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 9m ago 10h 26.1M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:51:57.211 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 109s ago 10h 1646M 2120M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:51:57.211 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 109s ago 10h 1633M 2120M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:51:57.212 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 109s ago 10h 1268M 2120M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:51:57.212 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 109s ago 10h 1600M 2120M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:51:57.212 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 9m ago 10h 1381M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:51:57.213 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 9m ago 10h 1734M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:51:57.213 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 9m ago 10h 1550M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:51:57.214 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 9m ago 10h 1845M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:51:57.214 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 9m ago 10h 180M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:51:57.561 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:51:57.561 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:51:57.562 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:51:57.562 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:51:57.563 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:51:57.563 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:51:57.563 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:51:57.564 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:51:57.564 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:51:57.565 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:51:57.565 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:51:57.566 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:51:57.566 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:51:57.567 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:51:57.567 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:51:57.567 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:51:57.568 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:51:57.568 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:51:58.460 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:51:58 smithi083 ceph-mon[77242]: from='client.55884 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:51:58.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:51:58 smithi083 ceph-mon[77242]: pgmap v18708: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:58.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:51:58 smithi083 ceph-mon[77242]: from='client.55890 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:51:58.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:51:58 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2245604795' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:51:58.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:51:58 smithi083 ceph-mon[79930]: from='client.55884 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:51:58.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:51:58 smithi083 ceph-mon[79930]: pgmap v18708: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:58.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:51:58 smithi083 ceph-mon[79930]: from='client.55890 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:51:58.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:51:58 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2245604795' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:51:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:51:58 smithi153 ceph-mon[70044]: from='client.55884 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:51:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:51:58 smithi153 ceph-mon[70044]: pgmap v18708: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:51:58.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:51:58 smithi153 ceph-mon[70044]: from='client.55890 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:51:58.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:51:58 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2245604795' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:52:00.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:52:00 smithi083 ceph-mon[77242]: pgmap v18709: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:00.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:52:00 smithi083 ceph-mon[79930]: pgmap v18709: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:00.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:52:00 smithi153 ceph-mon[70044]: pgmap v18709: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:52:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:52:01] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T15:52:02.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:52:02 smithi083 ceph-mon[79930]: pgmap v18710: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:02.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:52:02 smithi083 ceph-mon[77242]: pgmap v18710: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:52:02 smithi153 ceph-mon[70044]: pgmap v18710: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:02.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:52:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:52:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:52:02.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:52:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:52:02.511Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:52:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:52:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:52:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:52:04.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:52:03 smithi153 ceph-mon[70044]: pgmap v18711: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:04.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:52:03 smithi083 ceph-mon[79930]: pgmap v18711: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:04.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:52:03 smithi083 ceph-mon[77242]: pgmap v18711: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:06.202 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:52:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:52:05.860Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:52:06.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:52:06 smithi083 ceph-mon[79930]: pgmap v18712: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:06.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:52:06 smithi083 ceph-mon[77242]: pgmap v18712: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:06.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:52:06 smithi153 ceph-mon[70044]: pgmap v18712: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:06.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:52:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:52:06.608Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:52:08.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:52:08 smithi083 ceph-mon[79930]: pgmap v18713: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:08.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:52:08 smithi083 ceph-mon[77242]: pgmap v18713: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:52:08 smithi153 ceph-mon[70044]: pgmap v18713: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:10.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:52:10 smithi083 ceph-mon[79930]: pgmap v18714: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:10.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:52:10 smithi083 ceph-mon[77242]: pgmap v18714: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:10.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:52:10 smithi153 ceph-mon[70044]: pgmap v18714: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:11.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:52:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:52:11] "GET /metrics HTTP/1.1" 200 225247 "" "Prometheus/2.33.4" 2022-09-29T15:52:12.513 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:52:12 smithi083 ceph-mon[79930]: pgmap v18715: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:12.513 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:52:12 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:52:12.514 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:52:12 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:52:12.514 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:52:12 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:52:12.515 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:52:12 smithi083 ceph-mon[77242]: pgmap v18715: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:12.515 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:52:12 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:52:12.516 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:52:12 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:52:12.516 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:52:12 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:52:12.556 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:52:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:52:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:52:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:52:12 smithi153 ceph-mon[70044]: pgmap v18715: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:52:12 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:52:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:52:12 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:52:12.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:52:12 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:52:12.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:52:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:52:12.511Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:52:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:52:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:52:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:52:14.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:52:13 smithi153 ceph-mon[70044]: pgmap v18716: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:14.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:52:13 smithi083 ceph-mon[79930]: pgmap v18716: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:14.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:52:13 smithi083 ceph-mon[77242]: pgmap v18716: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:16.205 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:52:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:52:15.860Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:52:16.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:52:16 smithi083 ceph-mon[79930]: pgmap v18717: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:16.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:52:16 smithi083 ceph-mon[77242]: pgmap v18717: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:52:16 smithi153 ceph-mon[70044]: pgmap v18717: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:16.964 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:52:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:52:16.608Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:52:17.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:52:17 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:52:17.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:52:17 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:52:17.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:52:17 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:52:17.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:52:17 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:52:17.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:52:17 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:52:17.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:52:17 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:52:17.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:52:17 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:52:17.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:52:17 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:52:17.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:52:17 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:52:17.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:52:17 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:52:17.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:52:17 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:52:17.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:52:17 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:52:18.410 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:52:18 smithi153 ceph-mon[70044]: pgmap v18718: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:18.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:52:18 smithi083 ceph-mon[79930]: pgmap v18718: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:18.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:52:18 smithi083 ceph-mon[77242]: pgmap v18718: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:21.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:52:20 smithi153 ceph-mon[70044]: pgmap v18719: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:21.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:52:20 smithi153 ceph-mon[70044]: Detected new or changed devices on smithi153 2022-09-29T15:52:21.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:52:20 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:52:21.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:52:20 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:52:21.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:52:20 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:52:21.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:52:20 smithi083 ceph-mon[79930]: pgmap v18719: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:21.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:52:20 smithi083 ceph-mon[79930]: Detected new or changed devices on smithi153 2022-09-29T15:52:21.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:52:20 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:52:21.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:52:20 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:52:21.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:52:20 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:52:21.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:52:20 smithi083 ceph-mon[77242]: pgmap v18719: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:21.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:52:20 smithi083 ceph-mon[77242]: Detected new or changed devices on smithi153 2022-09-29T15:52:21.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:52:20 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:52:21.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:52:20 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:52:21.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:52:20 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:52:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:52:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:52:21] "GET /metrics HTTP/1.1" 200 225247 "" "Prometheus/2.33.4" 2022-09-29T15:52:22.754 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:52:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:52:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:52:22.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:52:22 smithi083 ceph-mon[79930]: pgmap v18720: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:22.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:52:22 smithi083 ceph-mon[77242]: pgmap v18720: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:22.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:52:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:52:22.513Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:52:23.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:52:22 smithi153 ceph-mon[70044]: pgmap v18720: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:23.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:52:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:52:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:52:24.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:52:23 smithi153 ceph-mon[70044]: pgmap v18721: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:24.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:52:23 smithi083 ceph-mon[79930]: pgmap v18721: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:24.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:52:23 smithi083 ceph-mon[77242]: pgmap v18721: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:26.208 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:52:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:52:25.861Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:52:26.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:52:26 smithi083 ceph-mon[79930]: pgmap v18722: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:26.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:52:26 smithi083 ceph-mon[77242]: pgmap v18722: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:52:26 smithi153 ceph-mon[70044]: pgmap v18722: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:26.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:52:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:52:26.609Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:52:27.885 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:52:28.174 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:52:28.175 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 2m ago 10h 27.3M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:52:28.175 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 11s ago 10h 63.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:52:28.175 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 2m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:52:28.176 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 11s ago 10h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:52:28.176 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 11s ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:52:28.176 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 2m ago 10h 1044M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:52:28.176 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 2m ago 10h 2266M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:52:28.177 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 11s ago 10h 989M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:52:28.177 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 2m ago 10h 1361M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:52:28.177 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 2m ago 10h 24.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:52:28.177 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 11s ago 10h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:52:28.177 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 2m ago 10h 1646M 2120M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:52:28.178 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 2m ago 10h 1633M 2120M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:52:28.178 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 2m ago 10h 1268M 2120M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:52:28.178 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 2m ago 10h 1600M 2120M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:52:28.178 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 11s ago 10h 1402M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:52:28.179 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 11s ago 10h 1763M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:52:28.179 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 11s ago 10h 1575M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:52:28.179 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 11s ago 10h 1874M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:52:28.179 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 11s ago 10h 182M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:52:28.531 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:52:28.531 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:52:28.531 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:52:28.531 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:52:28.532 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:52:28.532 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:52:28.532 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:52:28.532 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:52:28.533 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:52:28.533 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:52:28.533 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:52:28.533 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:52:28.534 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:52:28.534 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:52:28.534 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:52:28.534 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:52:28.535 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:52:28.535 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:52:28.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:52:28 smithi153 ceph-mon[70044]: pgmap v18723: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:28.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:52:28 smithi083 ceph-mon[77242]: pgmap v18723: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:28.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:52:28 smithi083 ceph-mon[79930]: pgmap v18723: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:29.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:52:29 smithi153 ceph-mon[70044]: from='client.65833 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:52:29.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:52:29 smithi153 ceph-mon[70044]: from='client.55908 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:52:29.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:52:29 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/255969560' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:52:29.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:52:29 smithi083 ceph-mon[79930]: from='client.65833 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:52:29.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:52:29 smithi083 ceph-mon[79930]: from='client.55908 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:52:29.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:52:29 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/255969560' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:52:29.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:52:29 smithi083 ceph-mon[77242]: from='client.65833 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:52:29.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:52:29 smithi083 ceph-mon[77242]: from='client.55908 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:52:29.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:52:29 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/255969560' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:52:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:52:30 smithi153 ceph-mon[70044]: pgmap v18724: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:30.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:52:30 smithi083 ceph-mon[79930]: pgmap v18724: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:30.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:52:30 smithi083 ceph-mon[77242]: pgmap v18724: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:52:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:52:31] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T15:52:32.514 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:52:32 smithi083 ceph-mon[79930]: pgmap v18725: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:32.515 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:52:32 smithi083 ceph-mon[77242]: pgmap v18725: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:32.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:52:32 smithi153 ceph-mon[70044]: pgmap v18725: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:32.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:52:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:52:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:52:32.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:52:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:52:32.513Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:52:33.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:52:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:52:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:52:34.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:52:33 smithi153 ceph-mon[70044]: pgmap v18726: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:34.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:52:33 smithi083 ceph-mon[79930]: pgmap v18726: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:34.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:52:33 smithi083 ceph-mon[77242]: pgmap v18726: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:36.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:52:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:52:35.861Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:52:36.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:52:36 smithi153 ceph-mon[70044]: pgmap v18727: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:36.611 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:52:36 smithi083 ceph-mon[79930]: pgmap v18727: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:36.612 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:52:36 smithi083 ceph-mon[77242]: pgmap v18727: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:36.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:52:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:52:36.610Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:52:38.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:52:38 smithi153 ceph-mon[70044]: pgmap v18728: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:38.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:52:38 smithi083 ceph-mon[79930]: pgmap v18728: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:38.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:52:38 smithi083 ceph-mon[77242]: pgmap v18728: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:40.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:52:40 smithi153 ceph-mon[70044]: pgmap v18729: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:40.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:52:40 smithi083 ceph-mon[79930]: pgmap v18729: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:52:40 smithi083 ceph-mon[77242]: pgmap v18729: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:52:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:52:41] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T15:52:42.515 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:52:42 smithi083 ceph-mon[79930]: pgmap v18730: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:42.516 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:52:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:52:42.517 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:52:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:52:42.517 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:52:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:52:42.518 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:52:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:52:42.518 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:52:42 smithi083 ceph-mon[77242]: pgmap v18730: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:42.519 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:52:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:52:42.519 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:52:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:52:42.520 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:52:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:52:42.520 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:52:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:52:42.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:52:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:52:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:52:42.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:52:42 smithi153 ceph-mon[70044]: pgmap v18730: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:42.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:52:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:52:42.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:52:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:52:42.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:52:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:52:42.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:52:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:52:42.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:52:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:52:42.514Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:52:43.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:52:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:52:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:52:44.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:52:43 smithi153 ceph-mon[70044]: pgmap v18731: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:44.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:52:43 smithi083 ceph-mon[79930]: pgmap v18731: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:44.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:52:43 smithi083 ceph-mon[77242]: pgmap v18731: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:46.210 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:52:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:52:45.862Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:52:46.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:52:46 smithi153 ceph-mon[70044]: pgmap v18732: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:46.611 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:52:46 smithi083 ceph-mon[79930]: pgmap v18732: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:46.612 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:52:46 smithi083 ceph-mon[77242]: pgmap v18732: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:46.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:52:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:52:46.610Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:52:48.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:52:48 smithi153 ceph-mon[70044]: pgmap v18733: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:48.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:52:48 smithi083 ceph-mon[79930]: pgmap v18733: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:48.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:52:48 smithi083 ceph-mon[77242]: pgmap v18733: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:50.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:52:50 smithi153 ceph-mon[70044]: pgmap v18734: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:50.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:52:50 smithi083 ceph-mon[79930]: pgmap v18734: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:50.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:52:50 smithi083 ceph-mon[77242]: pgmap v18734: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:51.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:52:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:52:51] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T15:52:52.516 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:52:52 smithi083 ceph-mon[77242]: pgmap v18735: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:52.517 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:52:52 smithi083 ceph-mon[79930]: pgmap v18735: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:52.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:52:52 smithi153 ceph-mon[70044]: pgmap v18735: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:52.559 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:52:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:52:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:52:52.966 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:52:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:52:52.515Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:52:53.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:52:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:52:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:52:54.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:52:53 smithi153 ceph-mon[70044]: pgmap v18736: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:54.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:52:53 smithi083 ceph-mon[77242]: pgmap v18736: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:54.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:52:53 smithi083 ceph-mon[79930]: pgmap v18736: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:56.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:52:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:52:55.862Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:52:56.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:52:56 smithi153 ceph-mon[70044]: pgmap v18737: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:56.611 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:52:56 smithi083 ceph-mon[79930]: pgmap v18737: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:56.612 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:52:56 smithi083 ceph-mon[77242]: pgmap v18737: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:56.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:52:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:52:56.610Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:52:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:52:58 smithi153 ceph-mon[70044]: pgmap v18738: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:58.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:52:58 smithi083 ceph-mon[79930]: pgmap v18738: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:58.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:52:58 smithi083 ceph-mon[77242]: pgmap v18738: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:52:58.859 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:52:59.143 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:52:59.144 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 2m ago 10h 27.3M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:52:59.145 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 42s ago 10h 63.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:52:59.145 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 2m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:52:59.146 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 42s ago 10h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:52:59.146 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 42s ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:52:59.147 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 2m ago 10h 1044M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:52:59.147 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 2m ago 10h 2266M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:52:59.147 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 42s ago 10h 989M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:52:59.148 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 2m ago 10h 1361M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:52:59.148 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 2m ago 10h 24.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:52:59.149 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 42s ago 10h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:52:59.149 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 2m ago 10h 1646M 2120M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:52:59.150 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 2m ago 10h 1633M 2120M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:52:59.150 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 2m ago 10h 1268M 2120M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:52:59.151 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 2m ago 10h 1600M 2120M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:52:59.151 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 42s ago 10h 1402M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:52:59.152 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 42s ago 10h 1763M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:52:59.152 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 42s ago 10h 1575M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:52:59.153 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 42s ago 10h 1874M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:52:59.153 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 42s ago 10h 182M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:52:59.503 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:52:59.503 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:52:59.504 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:52:59.504 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:52:59.505 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:52:59.505 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:52:59.506 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:52:59.506 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:52:59.507 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:52:59.507 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:52:59.507 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:52:59.508 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:52:59.508 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:52:59.509 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:52:59.509 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:52:59.510 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:52:59.510 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:52:59.511 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:52:59.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:52:59 smithi153 ceph-mon[70044]: from='client.55920 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:52:59.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:52:59 smithi083 ceph-mon[77242]: from='client.55920 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:52:59.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:52:59 smithi083 ceph-mon[79930]: from='client.55920 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:53:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:53:00 smithi153 ceph-mon[70044]: from='client.55926 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:53:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:53:00 smithi153 ceph-mon[70044]: pgmap v18739: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:53:00.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:53:00 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1165369691' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:53:00.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:53:00 smithi083 ceph-mon[79930]: from='client.55926 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:53:00.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:53:00 smithi083 ceph-mon[79930]: pgmap v18739: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:53:00.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:53:00 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1165369691' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:53:00.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:53:00 smithi083 ceph-mon[77242]: from='client.55926 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:53:00.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:53:00 smithi083 ceph-mon[77242]: pgmap v18739: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:53:00.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:53:00 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1165369691' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:53:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:53:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:53:01] "GET /metrics HTTP/1.1" 200 225246 "" "Prometheus/2.33.4" 2022-09-29T15:53:02.516 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:53:02 smithi083 ceph-mon[77242]: pgmap v18740: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:53:02.517 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:53:02 smithi083 ceph-mon[79930]: pgmap v18740: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:53:02.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:53:02 smithi153 ceph-mon[70044]: pgmap v18740: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:53:02.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:53:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:53:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:53:02.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:53:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:53:02.516Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:53:03.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:53:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:53:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:53:04.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:53:03 smithi153 ceph-mon[70044]: pgmap v18741: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:53:04.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:53:03 smithi083 ceph-mon[79930]: pgmap v18741: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:53:04.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:53:03 smithi083 ceph-mon[77242]: pgmap v18741: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:53:06.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:53:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:53:05.864Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:53:06.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:53:06 smithi153 ceph-mon[70044]: pgmap v18742: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:53:06.612 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:53:06 smithi083 ceph-mon[79930]: pgmap v18742: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:53:06.613 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:53:06 smithi083 ceph-mon[77242]: pgmap v18742: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:53:06.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:53:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:53:06.612Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:53:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:53:08 smithi153 ceph-mon[70044]: pgmap v18743: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:53:08.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:53:08 smithi083 ceph-mon[77242]: pgmap v18743: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:53:08.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:53:08 smithi083 ceph-mon[79930]: pgmap v18743: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:53:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:53:10 smithi153 ceph-mon[70044]: pgmap v18744: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:53:10.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:53:10 smithi083 ceph-mon[77242]: pgmap v18744: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:53:10.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:53:10 smithi083 ceph-mon[79930]: pgmap v18744: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:53:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:53:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:53:11] "GET /metrics HTTP/1.1" 200 225246 "" "Prometheus/2.33.4" 2022-09-29T15:53:12.517 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:53:12 smithi083 ceph-mon[79930]: pgmap v18745: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:53:12.518 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:53:12 smithi083 ceph-mon[77242]: pgmap v18745: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:53:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:53:12 smithi153 ceph-mon[70044]: pgmap v18745: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:53:12.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:53:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:53:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:53:12.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:53:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:53:12.516Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:53:13.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:53:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:53:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:53:14.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:53:13 smithi153 ceph-mon[70044]: pgmap v18746: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:53:14.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:53:13 smithi083 ceph-mon[77242]: pgmap v18746: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:53:14.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:53:13 smithi083 ceph-mon[79930]: pgmap v18746: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:53:16.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:53:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:53:15.864Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:53:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:53:16 smithi153 ceph-mon[70044]: pgmap v18747: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:53:16.613 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:53:16 smithi083 ceph-mon[79930]: pgmap v18747: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:53:16.614 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:53:16 smithi083 ceph-mon[77242]: pgmap v18747: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:53:16.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:53:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:53:16.612Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:53:18.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:53:18 smithi153 ceph-mon[70044]: pgmap v18748: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:53:18.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:53:18 smithi083 ceph-mon[77242]: pgmap v18748: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:53:18.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:53:18 smithi083 ceph-mon[79930]: pgmap v18748: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:53:20.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:53:20 smithi153 ceph-mon[70044]: pgmap v18749: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:53:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:53:20 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:53:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:53:20 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:53:20.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:53:20 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:53:20.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:53:20 smithi083 ceph-mon[77242]: pgmap v18749: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:53:20.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:53:20 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:53:20.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:53:20 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:53:20.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:53:20 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:53:20.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:53:20 smithi083 ceph-mon[79930]: pgmap v18749: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:53:20.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:53:20 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:53:20.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:53:20 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:53:20.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:53:20 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:53:21.714 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:53:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:53:21] "GET /metrics HTTP/1.1" 200 225246 "" "Prometheus/2.33.4" 2022-09-29T15:53:21.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:53:21 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:53:21.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:53:21 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:53:21.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:53:21 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:53:22.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:53:22 smithi083 ceph-mon[77242]: pgmap v18750: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:53:22.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:53:22 smithi083 ceph-mon[79930]: pgmap v18750: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:53:22.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:53:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:53:22.516Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:53:22.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:53:22 smithi153 ceph-mon[70044]: pgmap v18750: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:53:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:53:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:53:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:53:23.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:53:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:53:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:53:24.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:53:23 smithi153 ceph-mon[70044]: pgmap v18751: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:53:24.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:53:23 smithi083 ceph-mon[79930]: pgmap v18751: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:53:24.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:53:23 smithi083 ceph-mon[77242]: pgmap v18751: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:53:26.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:53:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:53:25.865Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:53:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:53:26 smithi153 ceph-mon[70044]: pgmap v18752: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:53:26.614 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:53:26 smithi083 ceph-mon[79930]: pgmap v18752: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:53:26.615 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:53:26 smithi083 ceph-mon[77242]: pgmap v18752: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:53:26.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:53:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:53:26.613Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:53:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:53:28 smithi153 ceph-mon[70044]: pgmap v18753: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:53:28.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:53:28 smithi083 ceph-mon[79930]: pgmap v18753: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:53:28.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:53:28 smithi083 ceph-mon[77242]: pgmap v18753: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:53:29.829 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:53:30.117 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:53:30.117 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 3m ago 10h 27.3M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:53:30.118 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 73s ago 10h 63.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:53:30.118 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 3m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:53:30.118 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 73s ago 10h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:53:30.119 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 73s ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:53:30.119 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 3m ago 10h 1044M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:53:30.120 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 3m ago 10h 2266M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:53:30.120 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 73s ago 10h 989M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:53:30.120 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 3m ago 10h 1361M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:53:30.120 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 3m ago 10h 24.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:53:30.121 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 73s ago 10h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:53:30.121 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 3m ago 10h 1646M 2120M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:53:30.121 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 3m ago 10h 1633M 2120M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:53:30.121 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 3m ago 10h 1268M 2120M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:53:30.121 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 3m ago 10h 1600M 2120M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:53:30.122 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 73s ago 10h 1402M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:53:30.122 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 73s ago 10h 1763M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:53:30.122 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 73s ago 10h 1575M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:53:30.122 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 73s ago 10h 1874M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:53:30.123 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 73s ago 10h 182M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:53:30.478 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:53:30.479 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:53:30.479 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:53:30.479 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:53:30.479 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:53:30.480 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:53:30.480 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:53:30.480 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:53:30.480 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:53:30.480 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:53:30.481 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:53:30.481 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:53:30.481 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:53:30.481 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:53:30.482 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:53:30.482 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:53:30.482 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:53:30.482 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:53:30.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:53:30 smithi153 ceph-mon[70044]: pgmap v18754: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:53:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:53:30 smithi153 ceph-mon[70044]: from='client.65869 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:53:30.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:53:30 smithi083 ceph-mon[79930]: pgmap v18754: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:53:30.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:53:30 smithi083 ceph-mon[79930]: from='client.65869 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:53:30.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:53:30 smithi083 ceph-mon[77242]: pgmap v18754: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:53:30.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:53:30 smithi083 ceph-mon[77242]: from='client.65869 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:53:31.245 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:53:31 smithi083 ceph-mon[79930]: from='client.65875 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:53:31.245 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:53:31 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3596937300' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:53:31.246 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:53:31 smithi083 ceph-mon[77242]: from='client.65875 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:53:31.246 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:53:31 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3596937300' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:53:31.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:53:31 smithi153 ceph-mon[70044]: from='client.65875 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:53:31.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:53:31 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3596937300' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:53:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:53:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:53:31] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T15:53:32.518 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:53:32 smithi083 ceph-mon[79930]: pgmap v18755: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:53:32.519 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:53:32 smithi083 ceph-mon[77242]: pgmap v18755: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:53:32.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:53:32 smithi153 ceph-mon[70044]: pgmap v18755: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:53:32.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:53:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:53:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:53:32.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:53:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:53:32.517Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:53:33.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:53:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:53:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:53:34.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:53:33 smithi153 ceph-mon[70044]: pgmap v18756: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:53:34.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:53:33 smithi083 ceph-mon[77242]: pgmap v18756: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:53:34.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:53:33 smithi083 ceph-mon[79930]: pgmap v18756: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:53:36.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:53:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:53:35.865Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:53:36.561 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:53:36 smithi153 ceph-mon[70044]: pgmap v18757: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:53:36.614 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:53:36 smithi083 ceph-mon[79930]: pgmap v18757: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:53:36.615 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:53:36 smithi083 ceph-mon[77242]: pgmap v18757: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:53:36.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:53:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:53:36.613Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:53:38.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:53:38 smithi153 ceph-mon[70044]: pgmap v18758: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:53:38.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:53:38 smithi083 ceph-mon[77242]: pgmap v18758: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:53:38.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:53:38 smithi083 ceph-mon[79930]: pgmap v18758: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:53:40.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:53:40 smithi153 ceph-mon[70044]: pgmap v18759: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:53:40.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:53:40 smithi083 ceph-mon[77242]: pgmap v18759: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:53:40.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:53:40 smithi083 ceph-mon[79930]: pgmap v18759: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:53:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:53:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:53:41] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T15:53:42.518 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:53:42 smithi083 ceph-mon[79930]: pgmap v18760: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:53:42.519 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:53:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:53:42.519 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:53:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:53:42.520 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:53:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:53:42.520 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:53:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:53:42.521 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:53:42 smithi083 ceph-mon[77242]: pgmap v18760: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:53:42.521 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:53:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:53:42.521 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:53:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:53:42.521 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:53:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:53:42.521 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:53:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:53:42.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:53:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:53:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:53:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:53:42 smithi153 ceph-mon[70044]: pgmap v18760: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:53:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:53:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:53:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:53:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:53:42.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:53:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:53:42.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:53:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:53:42.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:53:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:53:42.517Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:53:43.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:53:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:53:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:53:44.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:53:43 smithi153 ceph-mon[70044]: pgmap v18761: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:53:44.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:53:43 smithi083 ceph-mon[79930]: pgmap v18761: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:53:44.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:53:43 smithi083 ceph-mon[77242]: pgmap v18761: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:53:46.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:53:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:53:45.866Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:53:46.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:53:46 smithi153 ceph-mon[70044]: pgmap v18762: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:53:46.614 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:53:46 smithi083 ceph-mon[79930]: pgmap v18762: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:53:46.615 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:53:46 smithi083 ceph-mon[77242]: pgmap v18762: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:53:46.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:53:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:53:46.614Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:53:48.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:53:48 smithi153 ceph-mon[70044]: pgmap v18763: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:53:48.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:53:48 smithi083 ceph-mon[79930]: pgmap v18763: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:53:48.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:53:48 smithi083 ceph-mon[77242]: pgmap v18763: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:53:50.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:53:50 smithi153 ceph-mon[70044]: pgmap v18764: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:53:50.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:53:50 smithi083 ceph-mon[79930]: pgmap v18764: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:53:50.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:53:50 smithi083 ceph-mon[77242]: pgmap v18764: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:53:51.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:53:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:53:51] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T15:53:52.520 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:53:52 smithi083 ceph-mon[79930]: pgmap v18765: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:53:52.520 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:53:52 smithi083 ceph-mon[77242]: pgmap v18765: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:53:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:53:52 smithi153 ceph-mon[70044]: pgmap v18765: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:53:52.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:53:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:53:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:53:52.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:53:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:53:52.518Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:53:53.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:53:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:53:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:53:54.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:53:53 smithi153 ceph-mon[70044]: pgmap v18766: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:53:54.460 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:53:53 smithi083 ceph-mon[77242]: pgmap v18766: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:53:54.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:53:53 smithi083 ceph-mon[79930]: pgmap v18766: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:53:56.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:53:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:53:55.867Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:53:56.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:53:56 smithi153 ceph-mon[70044]: pgmap v18767: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:53:56.615 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:53:56 smithi083 ceph-mon[79930]: pgmap v18767: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:53:56.615 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:53:56 smithi083 ceph-mon[77242]: pgmap v18767: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:53:56.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:53:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:53:56.614Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:53:58.565 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:53:58 smithi153 ceph-mon[70044]: pgmap v18768: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:53:58.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:53:58 smithi083 ceph-mon[77242]: pgmap v18768: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:53:58.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:53:58 smithi083 ceph-mon[79930]: pgmap v18768: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:00.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:54:00 smithi153 ceph-mon[70044]: pgmap v18769: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:00.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:54:00 smithi083 ceph-mon[79930]: pgmap v18769: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:00.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:54:00 smithi083 ceph-mon[77242]: pgmap v18769: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:00.805 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:54:01.095 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:54:01.096 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 3m ago 10h 27.3M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:54:01.096 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 104s ago 10h 63.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:54:01.096 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 3m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:54:01.097 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 104s ago 10h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:54:01.097 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 104s ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:54:01.097 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 3m ago 10h 1044M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:54:01.097 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 3m ago 10h 2266M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:54:01.097 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 104s ago 10h 989M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:54:01.098 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 3m ago 10h 1361M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:54:01.098 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 3m ago 10h 24.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:54:01.099 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 104s ago 10h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:54:01.099 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 3m ago 10h 1646M 2120M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:54:01.100 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 3m ago 10h 1633M 2120M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:54:01.100 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 3m ago 10h 1268M 2120M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:54:01.101 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 3m ago 10h 1600M 2120M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:54:01.101 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 104s ago 10h 1402M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:54:01.101 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 104s ago 10h 1763M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:54:01.102 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 104s ago 10h 1575M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:54:01.102 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 104s ago 10h 1874M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:54:01.103 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 104s ago 10h 182M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:54:01.450 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:54:01.451 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:54:01.451 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:54:01.451 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:54:01.452 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:54:01.452 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:54:01.452 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:54:01.453 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:54:01.453 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:54:01.453 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:54:01.453 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:54:01.454 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:54:01.454 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:54:01.454 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:54:01.454 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:54:01.455 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:54:01.455 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:54:01.455 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:54:01.505 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:54:01 smithi083 ceph-mon[79930]: from='client.65887 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:54:01.506 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:54:01 smithi083 ceph-mon[77242]: from='client.65887 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:54:01.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:54:01 smithi153 ceph-mon[70044]: from='client.65887 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:54:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:54:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:54:01] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T15:54:02.520 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:54:02 smithi083 ceph-mon[77242]: from='client.55962 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:54:02.521 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:54:02 smithi083 ceph-mon[77242]: pgmap v18770: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:02.522 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:54:02 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2962710622' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:54:02.523 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:54:02 smithi083 ceph-mon[79930]: from='client.55962 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:54:02.523 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:54:02 smithi083 ceph-mon[79930]: pgmap v18770: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:02.524 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:54:02 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2962710622' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:54:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:54:02 smithi153 ceph-mon[70044]: from='client.55962 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:54:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:54:02 smithi153 ceph-mon[70044]: pgmap v18770: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:02.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:54:02 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2962710622' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:54:02.559 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:54:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:54:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:54:02.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:54:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:54:02.519Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:54:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:54:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:54:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:54:04.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:54:03 smithi153 ceph-mon[70044]: pgmap v18771: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:04.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:54:03 smithi083 ceph-mon[79930]: pgmap v18771: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:04.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:54:03 smithi083 ceph-mon[77242]: pgmap v18771: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:06.215 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:54:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:54:05.867Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:54:06.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:54:06 smithi153 ceph-mon[70044]: pgmap v18772: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:06.616 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:54:06 smithi083 ceph-mon[79930]: pgmap v18772: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:06.616 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:54:06 smithi083 ceph-mon[77242]: pgmap v18772: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:06.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:54:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:54:06.615Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:54:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:54:08 smithi153 ceph-mon[70044]: pgmap v18773: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:08.710 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:54:08 smithi083 ceph-mon[77242]: pgmap v18773: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:08.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:54:08 smithi083 ceph-mon[79930]: pgmap v18773: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:10.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:54:10 smithi153 ceph-mon[70044]: pgmap v18774: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:10.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:54:10 smithi083 ceph-mon[79930]: pgmap v18774: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:10.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:54:10 smithi083 ceph-mon[77242]: pgmap v18774: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:54:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:54:11] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T15:54:12.520 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:54:12 smithi083 ceph-mon[79930]: pgmap v18775: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:12.521 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:54:12 smithi083 ceph-mon[77242]: pgmap v18775: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:54:12 smithi153 ceph-mon[70044]: pgmap v18775: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:12.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:54:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:54:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:54:12.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:54:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:54:12.520Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:54:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:54:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:54:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:54:14.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:54:13 smithi153 ceph-mon[70044]: pgmap v18776: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:14.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:54:13 smithi083 ceph-mon[77242]: pgmap v18776: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:14.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:54:13 smithi083 ceph-mon[79930]: pgmap v18776: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:16.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:54:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:54:15.868Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:54:16.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:54:16 smithi153 ceph-mon[70044]: pgmap v18777: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:16.617 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:54:16 smithi083 ceph-mon[79930]: pgmap v18777: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:16.618 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:54:16 smithi083 ceph-mon[77242]: pgmap v18777: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:16.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:54:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:54:16.616Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:54:18.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:54:18 smithi153 ceph-mon[70044]: pgmap v18778: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:18.710 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:54:18 smithi083 ceph-mon[77242]: pgmap v18778: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:18.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:54:18 smithi083 ceph-mon[79930]: pgmap v18778: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:20.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:54:20 smithi153 ceph-mon[70044]: pgmap v18779: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:20.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:54:20 smithi083 ceph-mon[79930]: pgmap v18779: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:20.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:54:20 smithi083 ceph-mon[77242]: pgmap v18779: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:21.506 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:54:21 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:54:21.507 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:54:21 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:54:21.507 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:54:21 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:54:21.507 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:54:21 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:54:21.508 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:54:21 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:54:21.508 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:54:21 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:54:21.508 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:54:21 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:54:21.508 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:54:21 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:54:21.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:54:21 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:54:21.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:54:21 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:54:21.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:54:21 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:54:21.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:54:21 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:54:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:54:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:54:21] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T15:54:22.521 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:54:22 smithi083 ceph-mon[79930]: pgmap v18780: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:22.522 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:54:22 smithi083 ceph-mon[77242]: pgmap v18780: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:22.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:54:22 smithi153 ceph-mon[70044]: pgmap v18780: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:22.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:54:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:54:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:54:22.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:54:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:54:22.521Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:54:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:54:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:54:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:54:24.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:54:23 smithi153 ceph-mon[70044]: pgmap v18781: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:24.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:54:23 smithi083 ceph-mon[79930]: pgmap v18781: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:24.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:54:23 smithi083 ceph-mon[77242]: pgmap v18781: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:26.210 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:54:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:54:25.869Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:54:26.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:54:26 smithi153 ceph-mon[70044]: pgmap v18782: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:26.617 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:54:26 smithi083 ceph-mon[79930]: pgmap v18782: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:26.617 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:54:26 smithi083 ceph-mon[77242]: pgmap v18782: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:26.965 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:54:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:54:26.616Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:54:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:54:28 smithi153 ceph-mon[70044]: pgmap v18783: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:28.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:54:28 smithi083 ceph-mon[79930]: pgmap v18783: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:28.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:54:28 smithi083 ceph-mon[77242]: pgmap v18783: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:54:30 smithi153 ceph-mon[70044]: pgmap v18784: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:30.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:54:30 smithi083 ceph-mon[79930]: pgmap v18784: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:30.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:54:30 smithi083 ceph-mon[77242]: pgmap v18784: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:31.773 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:54:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:54:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:54:31] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T15:54:32.063 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:54:32.064 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 4m ago 10h 27.3M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:54:32.064 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 2m ago 10h 63.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:54:32.065 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 4m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:54:32.065 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 2m ago 10h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:54:32.066 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 2m ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:54:32.066 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 4m ago 10h 1044M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:54:32.067 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 4m ago 10h 2266M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:54:32.067 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 2m ago 10h 989M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:54:32.068 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 4m ago 10h 1361M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:54:32.068 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 4m ago 10h 24.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:54:32.068 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 2m ago 10h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:54:32.069 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 4m ago 10h 1646M 2120M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:54:32.069 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 4m ago 10h 1633M 2120M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:54:32.069 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 4m ago 10h 1268M 2120M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:54:32.070 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 4m ago 10h 1600M 2120M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:54:32.070 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 2m ago 10h 1402M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:54:32.070 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 2m ago 10h 1763M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:54:32.071 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 2m ago 10h 1575M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:54:32.071 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 2m ago 10h 1874M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:54:32.071 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 2m ago 10h 182M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:54:32.419 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:54:32.420 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:54:32.420 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:54:32.420 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:54:32.420 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:54:32.421 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:54:32.421 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:54:32.421 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:54:32.421 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:54:32.422 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:54:32.422 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:54:32.422 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:54:32.423 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:54:32.423 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:54:32.423 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:54:32.424 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:54:32.424 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:54:32.424 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:54:32.521 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:54:32 smithi083 ceph-mon[79930]: pgmap v18785: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:32.522 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:54:32 smithi083 ceph-mon[79930]: from='client.65905 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:54:32.523 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:54:32 smithi083 ceph-mon[77242]: pgmap v18785: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:32.526 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:54:32 smithi083 ceph-mon[77242]: from='client.65905 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:54:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:54:32 smithi153 ceph-mon[70044]: pgmap v18785: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:54:32 smithi153 ceph-mon[70044]: from='client.65905 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:54:32.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:54:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:54:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:54:32.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:54:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:54:32.520Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:54:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:54:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:54:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:54:33.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:54:33 smithi153 ceph-mon[70044]: from='client.65911 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:54:33.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:54:33 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/517606154' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:54:33.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:54:33 smithi083 ceph-mon[79930]: from='client.65911 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:54:33.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:54:33 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/517606154' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:54:33.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:54:33 smithi083 ceph-mon[77242]: from='client.65911 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:54:33.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:54:33 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/517606154' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:54:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:54:34 smithi153 ceph-mon[70044]: pgmap v18786: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:34.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:54:34 smithi083 ceph-mon[77242]: pgmap v18786: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:34.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:54:34 smithi083 ceph-mon[79930]: pgmap v18786: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:36.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:54:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:54:35.869Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:54:36.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:54:36 smithi153 ceph-mon[70044]: pgmap v18787: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:36.618 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:54:36 smithi083 ceph-mon[77242]: pgmap v18787: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:36.619 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:54:36 smithi083 ceph-mon[79930]: pgmap v18787: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:36.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:54:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:54:36.617Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:54:38.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:54:38 smithi153 ceph-mon[70044]: pgmap v18788: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:38.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:54:38 smithi083 ceph-mon[77242]: pgmap v18788: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:38.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:54:38 smithi083 ceph-mon[79930]: pgmap v18788: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:40.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:54:40 smithi153 ceph-mon[70044]: pgmap v18789: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:40.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:54:40 smithi083 ceph-mon[77242]: pgmap v18789: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:40.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:54:40 smithi083 ceph-mon[79930]: pgmap v18789: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:54:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:54:41] "GET /metrics HTTP/1.1" 200 225246 "" "Prometheus/2.33.4" 2022-09-29T15:54:42.556 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:54:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:54:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:54:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:54:42 smithi153 ceph-mon[70044]: pgmap v18790: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:54:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:54:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:54:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:54:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:54:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:54:42.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:54:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:54:42.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:54:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:54:42.522Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:54:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:54:42 smithi083 ceph-mon[77242]: pgmap v18790: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:54:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:54:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:54:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:54:42.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:54:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:54:42.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:54:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:54:42.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:54:42 smithi083 ceph-mon[79930]: pgmap v18790: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:42.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:54:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:54:42.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:54:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:54:42.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:54:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:54:42.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:54:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:54:43.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:54:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:54:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:54:44.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:54:43 smithi153 ceph-mon[70044]: pgmap v18791: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:44.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:54:43 smithi083 ceph-mon[79930]: pgmap v18791: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:44.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:54:43 smithi083 ceph-mon[77242]: pgmap v18791: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:46.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:54:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:54:45.870Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:54:46.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:54:46 smithi153 ceph-mon[70044]: pgmap v18792: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:46.618 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:54:46 smithi083 ceph-mon[79930]: pgmap v18792: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:46.618 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:54:46 smithi083 ceph-mon[77242]: pgmap v18792: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:46.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:54:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:54:46.617Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:54:48.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:54:48 smithi153 ceph-mon[70044]: pgmap v18793: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:48.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:54:48 smithi083 ceph-mon[79930]: pgmap v18793: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:48.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:54:48 smithi083 ceph-mon[77242]: pgmap v18793: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:50.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:54:50 smithi153 ceph-mon[70044]: pgmap v18794: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:50.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:54:50 smithi083 ceph-mon[79930]: pgmap v18794: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:50.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:54:50 smithi083 ceph-mon[77242]: pgmap v18794: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:51.969 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:54:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:54:51] "GET /metrics HTTP/1.1" 200 225246 "" "Prometheus/2.33.4" 2022-09-29T15:54:52.523 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:54:52 smithi083 ceph-mon[79930]: pgmap v18795: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:52.523 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:54:52 smithi083 ceph-mon[77242]: pgmap v18795: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:52.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:54:52 smithi153 ceph-mon[70044]: pgmap v18795: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:52.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:54:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:54:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:54:52.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:54:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:54:52.522Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:54:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:54:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:54:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:54:54.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:54:53 smithi153 ceph-mon[70044]: pgmap v18796: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:54.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:54:53 smithi083 ceph-mon[79930]: pgmap v18796: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:54.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:54:53 smithi083 ceph-mon[77242]: pgmap v18796: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:56.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:54:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:54:55.871Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:54:56.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:54:56 smithi153 ceph-mon[70044]: pgmap v18797: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:56.621 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:54:56 smithi083 ceph-mon[79930]: pgmap v18797: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:56.621 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:54:56 smithi083 ceph-mon[77242]: pgmap v18797: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:56.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:54:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:54:56.618Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:54:58.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:54:58 smithi153 ceph-mon[70044]: pgmap v18798: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:58.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:54:58 smithi083 ceph-mon[79930]: pgmap v18798: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:54:58.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:54:58 smithi083 ceph-mon[77242]: pgmap v18798: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:00.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:55:00 smithi153 ceph-mon[70044]: pgmap v18799: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:00.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:55:00 smithi083 ceph-mon[79930]: pgmap v18799: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:00.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:55:00 smithi083 ceph-mon[77242]: pgmap v18799: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:01.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:55:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:55:01] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T15:55:02.524 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:55:02 smithi083 ceph-mon[79930]: pgmap v18800: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:02.524 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:55:02 smithi083 ceph-mon[77242]: pgmap v18800: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:02.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:55:02 smithi153 ceph-mon[70044]: pgmap v18800: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:02.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:55:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:55:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:55:02.743 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:55:02.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:55:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:55:02.523Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:55:03.033 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:55:03.034 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 4m ago 10h 27.3M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:55:03.034 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 2m ago 10h 63.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:55:03.035 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 4m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:55:03.035 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 2m ago 10h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:55:03.035 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 2m ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:55:03.036 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 4m ago 10h 1044M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:55:03.036 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 4m ago 10h 2266M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:55:03.037 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 2m ago 10h 989M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:55:03.037 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 4m ago 10h 1361M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:55:03.038 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 4m ago 10h 24.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:55:03.038 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 2m ago 10h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:55:03.039 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 4m ago 10h 1646M 2120M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:55:03.039 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 4m ago 10h 1633M 2120M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:55:03.039 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 4m ago 10h 1268M 2120M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:55:03.040 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 4m ago 10h 1600M 2120M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:55:03.040 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 2m ago 10h 1402M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:55:03.041 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 2m ago 10h 1763M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:55:03.041 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 2m ago 10h 1575M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:55:03.041 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 2m ago 10h 1874M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:55:03.042 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 2m ago 10h 182M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:55:03.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:55:03 smithi153 ceph-mon[70044]: from='client.55992 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:55:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:55:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:55:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:55:03.400 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:55:03.400 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:55:03.402 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:55:03.402 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:55:03.403 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:55:03.403 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:55:03.404 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:55:03.404 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:55:03.404 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:55:03.405 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:55:03.405 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:55:03.406 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:55:03.406 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:55:03.407 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:55:03.407 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:55:03.408 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:55:03.408 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:55:03.409 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:55:03.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:55:03 smithi083 ceph-mon[79930]: from='client.55992 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:55:03.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:55:03 smithi083 ceph-mon[77242]: from='client.55992 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:55:04.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:55:04 smithi153 ceph-mon[70044]: from='client.65929 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:55:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:55:04 smithi153 ceph-mon[70044]: pgmap v18801: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:55:04 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2762069843' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:55:04.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:55:04 smithi083 ceph-mon[79930]: from='client.65929 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:55:04.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:55:04 smithi083 ceph-mon[79930]: pgmap v18801: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:04.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:55:04 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2762069843' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:55:04.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:55:04 smithi083 ceph-mon[77242]: from='client.65929 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:55:04.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:55:04 smithi083 ceph-mon[77242]: pgmap v18801: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:04.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:55:04 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2762069843' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:55:06.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:55:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:55:05.872Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:55:06.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:55:06 smithi153 ceph-mon[70044]: pgmap v18802: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:06.620 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:55:06 smithi083 ceph-mon[79930]: pgmap v18802: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:06.621 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:55:06 smithi083 ceph-mon[77242]: pgmap v18802: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:06.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:55:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:55:06.620Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:55:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:55:08 smithi153 ceph-mon[70044]: pgmap v18803: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:08.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:55:08 smithi083 ceph-mon[79930]: pgmap v18803: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:08.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:55:08 smithi083 ceph-mon[77242]: pgmap v18803: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:55:10 smithi153 ceph-mon[70044]: pgmap v18804: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:10.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:55:10 smithi083 ceph-mon[79930]: pgmap v18804: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:10.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:55:10 smithi083 ceph-mon[77242]: pgmap v18804: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:55:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:55:11] "GET /metrics HTTP/1.1" 200 225246 "" "Prometheus/2.33.4" 2022-09-29T15:55:12.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:55:12 smithi153 ceph-mon[70044]: pgmap v18805: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:12.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:55:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:55:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:55:12.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:55:12 smithi083 ceph-mon[79930]: pgmap v18805: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:12.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:55:12 smithi083 ceph-mon[77242]: pgmap v18805: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:12.714 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:55:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:55:12.524Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:55:13.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:55:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:55:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:55:14.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:55:14 smithi153 ceph-mon[70044]: pgmap v18806: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:14.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:55:14 smithi083 ceph-mon[79930]: pgmap v18806: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:14.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:55:14 smithi083 ceph-mon[77242]: pgmap v18806: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:16.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:55:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:55:15.872Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:55:16.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:55:16 smithi153 ceph-mon[70044]: pgmap v18807: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:16.620 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:55:16 smithi083 ceph-mon[79930]: pgmap v18807: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:16.621 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:55:16 smithi083 ceph-mon[77242]: pgmap v18807: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:16.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:55:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:55:16.619Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:55:18.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:55:18 smithi153 ceph-mon[70044]: pgmap v18808: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:18.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:55:18 smithi083 ceph-mon[79930]: pgmap v18808: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:18.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:55:18 smithi083 ceph-mon[77242]: pgmap v18808: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:20.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:55:20 smithi153 ceph-mon[70044]: pgmap v18809: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:20.718 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:55:20 smithi083 ceph-mon[79930]: pgmap v18809: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:20.719 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:55:20 smithi083 ceph-mon[77242]: pgmap v18809: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:21.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:55:21 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:55:21.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:55:21 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:55:21.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:55:21 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:55:21.714 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:55:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:55:21] "GET /metrics HTTP/1.1" 200 225246 "" "Prometheus/2.33.4" 2022-09-29T15:55:21.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:55:21 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:55:21.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:55:21 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:55:21.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:55:21 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:55:21.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:55:21 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:55:21.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:55:21 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:55:21.718 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:55:21 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:55:22.749 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:55:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:55:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:55:22.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:55:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:55:22.524Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:55:22.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:55:22 smithi083 ceph-mon[77242]: pgmap v18810: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:22.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:55:22 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:55:22.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:55:22 smithi083 ceph-mon[79930]: pgmap v18810: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:22.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:55:22 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:55:23.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:55:22 smithi153 ceph-mon[70044]: pgmap v18810: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:23.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:55:22 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:55:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:55:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:55:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:55:24.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:55:24 smithi153 ceph-mon[70044]: pgmap v18811: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:24.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:55:24 smithi083 ceph-mon[79930]: pgmap v18811: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:24.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:55:24 smithi083 ceph-mon[77242]: pgmap v18811: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:26.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:55:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:55:25.873Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:55:26.562 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:55:26 smithi153 ceph-mon[70044]: pgmap v18812: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:26.621 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:55:26 smithi083 ceph-mon[79930]: pgmap v18812: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:26.622 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:55:26 smithi083 ceph-mon[77242]: pgmap v18812: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:26.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:55:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:55:26.620Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:55:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:55:28 smithi153 ceph-mon[70044]: pgmap v18813: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:28.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:55:28 smithi083 ceph-mon[79930]: pgmap v18813: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:28.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:55:28 smithi083 ceph-mon[77242]: pgmap v18813: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:55:30 smithi153 ceph-mon[70044]: pgmap v18814: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:30.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:55:30 smithi083 ceph-mon[77242]: pgmap v18814: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:30.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:55:30 smithi083 ceph-mon[79930]: pgmap v18814: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:55:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:55:31] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T15:55:32.526 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:55:32 smithi083 ceph-mon[79930]: pgmap v18815: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:32.527 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:55:32 smithi083 ceph-mon[77242]: pgmap v18815: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:55:32 smithi153 ceph-mon[70044]: pgmap v18815: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:32.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:55:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:55:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:55:32.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:55:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:55:32.525Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:55:33.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:55:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:55:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:55:33.729 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:55:34.028 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:55:34.028 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 5m ago 10h 27.3M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:55:34.029 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 3m ago 10h 63.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:55:34.029 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 5m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:55:34.030 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 3m ago 10h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:55:34.030 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 3m ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:55:34.031 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 5m ago 10h 1044M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:55:34.031 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 5m ago 10h 2266M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:55:34.032 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 3m ago 10h 989M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:55:34.032 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 5m ago 10h 1361M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:55:34.033 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 5m ago 10h 24.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:55:34.033 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 3m ago 10h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:55:34.034 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 5m ago 10h 1646M 2120M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:55:34.034 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 5m ago 10h 1633M 2120M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:55:34.035 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 5m ago 10h 1268M 2120M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:55:34.035 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 5m ago 10h 1600M 2120M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:55:34.036 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 3m ago 10h 1402M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:55:34.036 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 3m ago 10h 1763M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:55:34.036 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 3m ago 10h 1575M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:55:34.037 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 3m ago 10h 1874M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:55:34.037 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 3m ago 10h 182M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:55:34.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:55:34 smithi153 ceph-mon[70044]: pgmap v18816: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:34.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:55:34 smithi153 ceph-mon[70044]: from='client.56010 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:55:34.384 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:55:34.385 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:55:34.385 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:55:34.386 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:55:34.386 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:55:34.387 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:55:34.387 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:55:34.388 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:55:34.388 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:55:34.389 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:55:34.389 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:55:34.390 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:55:34.390 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:55:34.390 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:55:34.391 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:55:34.391 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:55:34.392 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:55:34.392 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:55:34.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:55:34 smithi083 ceph-mon[79930]: pgmap v18816: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:34.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:55:34 smithi083 ceph-mon[79930]: from='client.56010 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:55:34.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:55:34 smithi083 ceph-mon[77242]: pgmap v18816: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:34.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:55:34 smithi083 ceph-mon[77242]: from='client.56010 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:55:35.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:55:35 smithi153 ceph-mon[70044]: from='client.65947 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:55:35.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:55:35 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/220506018' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:55:35.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:55:35 smithi083 ceph-mon[79930]: from='client.65947 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:55:35.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:55:35 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/220506018' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:55:35.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:55:35 smithi083 ceph-mon[77242]: from='client.65947 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:55:35.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:55:35 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/220506018' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:55:36.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:55:36 smithi083 ceph-mon[79930]: pgmap v18817: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:36.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:55:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:55:35.873Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:55:36.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:55:36 smithi083 ceph-mon[77242]: pgmap v18817: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:36.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:55:36 smithi153 ceph-mon[70044]: pgmap v18817: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:36.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:55:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:55:36.621Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:55:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:55:38 smithi153 ceph-mon[70044]: pgmap v18818: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:38.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:55:38 smithi083 ceph-mon[77242]: pgmap v18818: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:38.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:55:38 smithi083 ceph-mon[79930]: pgmap v18818: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:55:40 smithi153 ceph-mon[70044]: pgmap v18819: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:40.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:55:40 smithi083 ceph-mon[77242]: pgmap v18819: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:40.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:55:40 smithi083 ceph-mon[79930]: pgmap v18819: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:55:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:55:41] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T15:55:42.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:55:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:55:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:55:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:55:42 smithi153 ceph-mon[70044]: pgmap v18820: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:42.562 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:55:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:55:42.562 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:55:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:55:42.563 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:55:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:55:42.563 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:55:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:55:42.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:55:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:55:42.526Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:55:42.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:55:42 smithi083 ceph-mon[79930]: pgmap v18820: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:42.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:55:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:55:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:55:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:55:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:55:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:55:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:55:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:55:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:55:42 smithi083 ceph-mon[77242]: pgmap v18820: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:55:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:55:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:55:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:55:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:55:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:55:42.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:55:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:55:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:55:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:55:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:55:44.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:55:44 smithi153 ceph-mon[70044]: pgmap v18821: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:44.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:55:44 smithi083 ceph-mon[79930]: pgmap v18821: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:44.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:55:44 smithi083 ceph-mon[77242]: pgmap v18821: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:46.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:55:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:55:45.874Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:55:46.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:55:46 smithi153 ceph-mon[70044]: pgmap v18822: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:46.621 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:55:46 smithi083 ceph-mon[79930]: pgmap v18822: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:46.622 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:55:46 smithi083 ceph-mon[77242]: pgmap v18822: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:46.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:55:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:55:46.621Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:55:48.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:55:48 smithi153 ceph-mon[70044]: pgmap v18823: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:48.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:55:48 smithi083 ceph-mon[79930]: pgmap v18823: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:48.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:55:48 smithi083 ceph-mon[77242]: pgmap v18823: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:55:50 smithi153 ceph-mon[70044]: pgmap v18824: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:50.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:55:50 smithi083 ceph-mon[79930]: pgmap v18824: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:50.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:55:50 smithi083 ceph-mon[77242]: pgmap v18824: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:55:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:55:51] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T15:55:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:55:52 smithi153 ceph-mon[70044]: pgmap v18825: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:52.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:55:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:55:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:55:52.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:55:52 smithi083 ceph-mon[79930]: pgmap v18825: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:52.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:55:52 smithi083 ceph-mon[77242]: pgmap v18825: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:52.714 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:55:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:55:52.526Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:55:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:55:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:55:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:55:54.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:55:54 smithi153 ceph-mon[70044]: pgmap v18826: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:54.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:55:54 smithi083 ceph-mon[79930]: pgmap v18826: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:54.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:55:54 smithi083 ceph-mon[77242]: pgmap v18826: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:56.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:55:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:55:55.874Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:55:56.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:55:56 smithi153 ceph-mon[70044]: pgmap v18827: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:56.622 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:55:56 smithi083 ceph-mon[79930]: pgmap v18827: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:56.623 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:55:56 smithi083 ceph-mon[77242]: pgmap v18827: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:56.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:55:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:55:56.622Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:55:58.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:55:58 smithi153 ceph-mon[70044]: pgmap v18828: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:58.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:55:58 smithi083 ceph-mon[77242]: pgmap v18828: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:55:58.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:55:58 smithi083 ceph-mon[79930]: pgmap v18828: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:00.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:56:00 smithi153 ceph-mon[70044]: pgmap v18829: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:00.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:56:00 smithi083 ceph-mon[77242]: pgmap v18829: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:00.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:56:00 smithi083 ceph-mon[79930]: pgmap v18829: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:01.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:56:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:56:01] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T15:56:02.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:56:02 smithi153 ceph-mon[70044]: pgmap v18830: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:02.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:56:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:56:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:56:02.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:56:02 smithi083 ceph-mon[77242]: pgmap v18830: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:02.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:56:02 smithi083 ceph-mon[79930]: pgmap v18830: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:02.716 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:56:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:56:02.527Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:56:03.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:56:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:56:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:56:04.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:56:04 smithi153 ceph-mon[70044]: pgmap v18831: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:04.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:56:04 smithi083 ceph-mon[79930]: pgmap v18831: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:04.467 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:56:04 smithi083 ceph-mon[77242]: pgmap v18831: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:04.708 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:56:04.997 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:56:04.997 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 5m ago 10h 27.3M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:56:04.998 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 3m ago 10h 63.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:56:04.998 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 5m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:56:04.999 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 3m ago 10h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:56:04.999 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 3m ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:56:05.000 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 5m ago 10h 1044M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:56:05.000 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 5m ago 10h 2266M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:56:05.001 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 3m ago 10h 989M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:56:05.001 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 5m ago 10h 1361M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:56:05.001 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 5m ago 10h 24.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:56:05.002 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 3m ago 10h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:56:05.003 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 5m ago 10h 1646M 2120M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:56:05.003 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 5m ago 10h 1633M 2120M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:56:05.003 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 5m ago 10h 1268M 2120M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:56:05.004 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 5m ago 10h 1600M 2120M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:56:05.004 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 3m ago 10h 1402M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:56:05.004 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 3m ago 10h 1763M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:56:05.005 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 3m ago 10h 1575M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:56:05.005 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 3m ago 10h 1874M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:56:05.006 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 3m ago 10h 182M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:56:05.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:56:05 smithi153 ceph-mon[70044]: from='client.56028 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:56:05.352 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:56:05.352 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:56:05.353 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:56:05.353 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:56:05.354 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:56:05.354 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:56:05.355 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:56:05.355 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:56:05.355 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:56:05.356 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:56:05.356 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:56:05.358 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:56:05.358 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:56:05.359 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:56:05.359 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:56:05.359 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:56:05.359 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:56:05.360 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:56:05.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:56:05 smithi083 ceph-mon[77242]: from='client.56028 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:56:05.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:56:05 smithi083 ceph-mon[79930]: from='client.56028 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:56:06.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:56:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:56:05.874Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:56:06.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:56:06 smithi083 ceph-mon[77242]: from='client.56034 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:56:06.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:56:06 smithi083 ceph-mon[77242]: pgmap v18832: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:06.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:56:06 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1707152238' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:56:06.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:56:06 smithi083 ceph-mon[79930]: from='client.56034 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:56:06.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:56:06 smithi083 ceph-mon[79930]: pgmap v18832: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:06.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:56:06 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1707152238' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:56:06.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:56:06 smithi153 ceph-mon[70044]: from='client.56034 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:56:06.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:56:06 smithi153 ceph-mon[70044]: pgmap v18832: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:06.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:56:06 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1707152238' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:56:06.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:56:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:56:06.623Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:56:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:56:08 smithi153 ceph-mon[70044]: pgmap v18833: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:08.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:56:08 smithi083 ceph-mon[79930]: pgmap v18833: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:08.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:56:08 smithi083 ceph-mon[77242]: pgmap v18833: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:10.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:56:10 smithi153 ceph-mon[70044]: pgmap v18834: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:10.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:56:10 smithi083 ceph-mon[79930]: pgmap v18834: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:10.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:56:10 smithi083 ceph-mon[77242]: pgmap v18834: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:56:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:56:11] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T15:56:12.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:56:12 smithi153 ceph-mon[70044]: pgmap v18835: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:12.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:56:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:56:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:56:12.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:56:12 smithi083 ceph-mon[79930]: pgmap v18835: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:12.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:56:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:56:12.528Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:56:12.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:56:12 smithi083 ceph-mon[77242]: pgmap v18835: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:56:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:56:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:56:14.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:56:14 smithi153 ceph-mon[70044]: pgmap v18836: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:14.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:56:14 smithi083 ceph-mon[79930]: pgmap v18836: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:14.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:56:14 smithi083 ceph-mon[77242]: pgmap v18836: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:16.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:56:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:56:15.876Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:56:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:56:16 smithi153 ceph-mon[70044]: pgmap v18837: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:16.624 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:56:16 smithi083 ceph-mon[79930]: pgmap v18837: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:16.625 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:56:16 smithi083 ceph-mon[77242]: pgmap v18837: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:16.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:56:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:56:16.623Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:56:18.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:56:18 smithi153 ceph-mon[70044]: pgmap v18838: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:18.710 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:56:18 smithi083 ceph-mon[77242]: pgmap v18838: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:18.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:56:18 smithi083 ceph-mon[79930]: pgmap v18838: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:20.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:56:20 smithi153 ceph-mon[70044]: pgmap v18839: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:20.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:56:20 smithi083 ceph-mon[79930]: pgmap v18839: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:20.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:56:20 smithi083 ceph-mon[77242]: pgmap v18839: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:21.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:56:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:56:21] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T15:56:22.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:56:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:56:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:56:22.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:56:22 smithi153 ceph-mon[70044]: pgmap v18840: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:22.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:56:22 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:56:22.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:56:22 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:56:22.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:56:22 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:56:22.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:56:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:56:22.527Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:56:22.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:56:22 smithi083 ceph-mon[77242]: pgmap v18840: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:22.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:56:22 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:56:22.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:56:22 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:56:22.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:56:22 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:56:22.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:56:22 smithi083 ceph-mon[79930]: pgmap v18840: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:22.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:56:22 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:56:22.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:56:22 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:56:22.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:56:22 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:56:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:56:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:56:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:56:23.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:56:23 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:56:23.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:56:23 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:56:23.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:56:23 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:56:24.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:56:24 smithi083 ceph-mon[79930]: pgmap v18841: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:24.718 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:56:24 smithi083 ceph-mon[77242]: pgmap v18841: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:24.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:56:24 smithi153 ceph-mon[70044]: pgmap v18841: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:26.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:56:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:56:25.876Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:56:26.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:56:26 smithi083 ceph-mon[79930]: pgmap v18842: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:26.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:56:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:56:26.624Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:56:26.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:56:26 smithi083 ceph-mon[77242]: pgmap v18842: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:26.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:56:26 smithi153 ceph-mon[70044]: pgmap v18842: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:28.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:56:28 smithi083 ceph-mon[79930]: pgmap v18843: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:28.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:56:28 smithi083 ceph-mon[77242]: pgmap v18843: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:28.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:56:28 smithi153 ceph-mon[70044]: pgmap v18843: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:30.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:56:30 smithi083 ceph-mon[79930]: pgmap v18844: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:30.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:56:30 smithi083 ceph-mon[77242]: pgmap v18844: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:30.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:56:30 smithi153 ceph-mon[70044]: pgmap v18844: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:31.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:56:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:56:31] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T15:56:32.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:56:32 smithi083 ceph-mon[79930]: pgmap v18845: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:32.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:56:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:56:32.529Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:56:32.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:56:32 smithi083 ceph-mon[77242]: pgmap v18845: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:32.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:56:32 smithi153 ceph-mon[70044]: pgmap v18845: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:56:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:56:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:56:33.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:56:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:56:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:56:34.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:56:34 smithi153 ceph-mon[70044]: pgmap v18846: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:34.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:56:34 smithi083 ceph-mon[79930]: pgmap v18846: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:34.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:56:34 smithi083 ceph-mon[77242]: pgmap v18846: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:35.676 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:56:35.965 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:56:35.966 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 6m ago 10h 27.3M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:56:35.966 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 4m ago 10h 63.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:56:35.967 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 6m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:56:35.967 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 4m ago 10h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:56:35.967 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 4m ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:56:35.968 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 6m ago 10h 1044M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:56:35.968 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 6m ago 10h 2266M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:56:35.969 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 4m ago 10h 989M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:56:35.969 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 6m ago 10h 1361M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:56:35.970 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 6m ago 10h 24.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:56:35.971 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 4m ago 10h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:56:35.972 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 6m ago 10h 1646M 2120M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:56:35.973 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 6m ago 10h 1633M 2120M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:56:35.973 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 6m ago 10h 1268M 2120M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:56:35.974 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 6m ago 10h 1600M 2120M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:56:35.974 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 4m ago 10h 1402M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:56:35.975 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 4m ago 10h 1763M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:56:35.978 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 4m ago 10h 1575M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:56:35.979 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 4m ago 10h 1874M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:56:35.979 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 4m ago 10h 182M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:56:36.210 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:56:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:56:35.875Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:56:36.323 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:56:36.323 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:56:36.324 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:56:36.324 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:56:36.325 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:56:36.325 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:56:36.326 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:56:36.326 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:56:36.327 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:56:36.327 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:56:36.328 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:56:36.328 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:56:36.328 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:56:36.329 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:56:36.329 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:56:36.330 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:56:36.330 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:56:36.331 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:56:36.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:56:36 smithi153 ceph-mon[70044]: pgmap v18847: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:56:36 smithi153 ceph-mon[70044]: from='client.56046 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:56:36.625 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:56:36 smithi083 ceph-mon[79930]: pgmap v18847: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:36.626 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:56:36 smithi083 ceph-mon[79930]: from='client.56046 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:56:36.627 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:56:36 smithi083 ceph-mon[77242]: pgmap v18847: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:36.627 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:56:36 smithi083 ceph-mon[77242]: from='client.56046 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:56:36.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:56:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:56:36.625Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:56:37.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:56:37 smithi153 ceph-mon[70044]: from='client.65983 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:56:37.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:56:37 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1636033096' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:56:37.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:56:37 smithi083 ceph-mon[79930]: from='client.65983 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:56:37.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:56:37 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1636033096' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:56:37.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:56:37 smithi083 ceph-mon[77242]: from='client.65983 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:56:37.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:56:37 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1636033096' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:56:38.565 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:56:38 smithi153 ceph-mon[70044]: pgmap v18848: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:38.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:56:38 smithi083 ceph-mon[79930]: pgmap v18848: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:38.718 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:56:38 smithi083 ceph-mon[77242]: pgmap v18848: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:56:40 smithi153 ceph-mon[70044]: pgmap v18849: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:40.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:56:40 smithi083 ceph-mon[79930]: pgmap v18849: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:56:40 smithi083 ceph-mon[77242]: pgmap v18849: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:56:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:56:41] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T15:56:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:56:42 smithi153 ceph-mon[70044]: pgmap v18850: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:56:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:56:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:56:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:56:42.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:56:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:56:42.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:56:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:56:42.560 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:56:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:56:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:56:42.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:56:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:56:42.529Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:56:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:56:42 smithi083 ceph-mon[79930]: pgmap v18850: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:42.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:56:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:56:42.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:56:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:56:42.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:56:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:56:42.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:56:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:56:42.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:56:42 smithi083 ceph-mon[77242]: pgmap v18850: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:42.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:56:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:56:42.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:56:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:56:42.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:56:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:56:42.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:56:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:56:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:56:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:56:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:56:44.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:56:44 smithi153 ceph-mon[70044]: pgmap v18851: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:44.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:56:44 smithi083 ceph-mon[79930]: pgmap v18851: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:44.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:56:44 smithi083 ceph-mon[77242]: pgmap v18851: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:46.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:56:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:56:45.877Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:56:46.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:56:46 smithi153 ceph-mon[70044]: pgmap v18852: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:46.626 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:56:46 smithi083 ceph-mon[79930]: pgmap v18852: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:46.627 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:56:46 smithi083 ceph-mon[77242]: pgmap v18852: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:46.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:56:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:56:46.626Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:56:48.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:56:48 smithi153 ceph-mon[70044]: pgmap v18853: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:48.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:56:48 smithi083 ceph-mon[79930]: pgmap v18853: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:48.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:56:48 smithi083 ceph-mon[77242]: pgmap v18853: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:50.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:56:50 smithi153 ceph-mon[70044]: pgmap v18854: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:50.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:56:50 smithi083 ceph-mon[79930]: pgmap v18854: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:50.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:56:50 smithi083 ceph-mon[77242]: pgmap v18854: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:51.969 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:56:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:56:51] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T15:56:52.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:56:52 smithi153 ceph-mon[70044]: pgmap v18855: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:52.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:56:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:56:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:56:52.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:56:52 smithi083 ceph-mon[79930]: pgmap v18855: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:52.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:56:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:56:52.529Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:56:52.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:56:52 smithi083 ceph-mon[77242]: pgmap v18855: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:56:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:56:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:56:54.314 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:56:54 smithi153 ceph-mon[70044]: pgmap v18856: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:54.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:56:54 smithi083 ceph-mon[79930]: pgmap v18856: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:54.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:56:54 smithi083 ceph-mon[77242]: pgmap v18856: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:56.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:56:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:56:55.877Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:56:56.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:56:56 smithi153 ceph-mon[70044]: pgmap v18857: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:56.627 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:56:56 smithi083 ceph-mon[79930]: pgmap v18857: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:56.628 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:56:56 smithi083 ceph-mon[77242]: pgmap v18857: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:56.969 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:56:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:56:56.626Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:56:58.563 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:56:58 smithi153 ceph-mon[70044]: pgmap v18858: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:58.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:56:58 smithi083 ceph-mon[79930]: pgmap v18858: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:56:58.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:56:58 smithi083 ceph-mon[77242]: pgmap v18858: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:00.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:57:00 smithi153 ceph-mon[70044]: pgmap v18859: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:00.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:57:00 smithi083 ceph-mon[79930]: pgmap v18859: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:00.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:57:00 smithi083 ceph-mon[77242]: pgmap v18859: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:57:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:57:01] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T15:57:02.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:57:02 smithi083 ceph-mon[79930]: pgmap v18860: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:02.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:57:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:57:02.530Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:57:02.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:57:02 smithi083 ceph-mon[77242]: pgmap v18860: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:02.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:57:02 smithi153 ceph-mon[70044]: pgmap v18860: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:57:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:57:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:57:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:57:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:57:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:57:04.318 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:57:04 smithi153 ceph-mon[70044]: pgmap v18861: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:04.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:57:04 smithi083 ceph-mon[77242]: pgmap v18861: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:04.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:57:04 smithi083 ceph-mon[79930]: pgmap v18861: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:06.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:57:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:57:05.878Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:57:06.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:57:06 smithi153 ceph-mon[70044]: pgmap v18862: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:06.626 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:57:06 smithi083 ceph-mon[79930]: pgmap v18862: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:06.627 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:57:06 smithi083 ceph-mon[77242]: pgmap v18862: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:06.650 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:57:06.938 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:57:06.938 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 6m ago 10h 27.3M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:57:06.938 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 4m ago 10h 63.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:57:06.939 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 6m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:57:06.939 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 4m ago 10h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:57:06.939 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 4m ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:57:06.939 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 6m ago 10h 1044M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:57:06.940 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 6m ago 10h 2266M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:57:06.940 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 4m ago 10h 989M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:57:06.940 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 6m ago 10h 1361M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:57:06.941 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 6m ago 10h 24.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:57:06.941 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 4m ago 10h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:57:06.941 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 6m ago 10h 1646M 2120M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:57:06.941 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 6m ago 10h 1633M 2120M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:57:06.941 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 6m ago 10h 1268M 2120M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:57:06.942 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 6m ago 10h 1600M 2120M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:57:06.942 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 4m ago 10h 1402M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:57:06.942 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 4m ago 10h 1763M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:57:06.943 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 4m ago 10h 1575M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:57:06.943 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 4m ago 10h 1874M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:57:06.943 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 4m ago 10h 182M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:57:06.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:57:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:57:06.625Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:57:07.294 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:57:07.294 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:57:07.294 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:57:07.295 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:57:07.295 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:57:07.295 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:57:07.295 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:57:07.296 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:57:07.296 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:57:07.297 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:57:07.297 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:57:07.297 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:57:07.297 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:57:07.298 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:57:07.298 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:57:07.298 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:57:07.299 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:57:07.299 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:57:07.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:57:07 smithi153 ceph-mon[70044]: from='client.56064 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:57:07.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:57:07 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3084127569' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:57:07.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:57:07 smithi083 ceph-mon[77242]: from='client.56064 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:57:07.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:57:07 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3084127569' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:57:07.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:57:07 smithi083 ceph-mon[79930]: from='client.56064 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:57:07.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:57:07 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3084127569' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:57:08.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:57:08 smithi153 ceph-mon[70044]: from='client.56070 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:57:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:57:08 smithi153 ceph-mon[70044]: pgmap v18863: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:08.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:57:08 smithi083 ceph-mon[79930]: from='client.56070 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:57:08.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:57:08 smithi083 ceph-mon[79930]: pgmap v18863: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:08.718 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:57:08 smithi083 ceph-mon[77242]: from='client.56070 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:57:08.719 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:57:08 smithi083 ceph-mon[77242]: pgmap v18863: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:10.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:57:10 smithi083 ceph-mon[79930]: pgmap v18864: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:10.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:57:10 smithi083 ceph-mon[77242]: pgmap v18864: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:10.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:57:10 smithi153 ceph-mon[70044]: pgmap v18864: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:57:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:57:11] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T15:57:12.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:57:12 smithi083 ceph-mon[79930]: pgmap v18865: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:12.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:57:12 smithi083 ceph-mon[77242]: pgmap v18865: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:12.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:57:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:57:12.531Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:57:12.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:57:12 smithi153 ceph-mon[70044]: pgmap v18865: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:57:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:57:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:57:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:57:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:57:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:57:14.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:57:14 smithi153 ceph-mon[70044]: pgmap v18866: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:14.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:57:14 smithi083 ceph-mon[77242]: pgmap v18866: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:14.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:57:14 smithi083 ceph-mon[79930]: pgmap v18866: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:16.210 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:57:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:57:15.879Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:57:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:57:16 smithi153 ceph-mon[70044]: pgmap v18867: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:16.627 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:57:16 smithi083 ceph-mon[79930]: pgmap v18867: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:16.628 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:57:16 smithi083 ceph-mon[77242]: pgmap v18867: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:16.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:57:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:57:16.626Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:57:18.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:57:18 smithi153 ceph-mon[70044]: pgmap v18868: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:18.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:57:18 smithi083 ceph-mon[79930]: pgmap v18868: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:18.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:57:18 smithi083 ceph-mon[77242]: pgmap v18868: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:20.561 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:57:20 smithi153 ceph-mon[70044]: pgmap v18869: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:20.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:57:20 smithi083 ceph-mon[79930]: pgmap v18869: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:20.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:57:20 smithi083 ceph-mon[77242]: pgmap v18869: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:21.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:57:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:57:21] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T15:57:22.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:57:22 smithi083 ceph-mon[79930]: pgmap v18870: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:22.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:57:22 smithi083 ceph-mon[77242]: pgmap v18870: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:22.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:57:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:57:22.532Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:57:22.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:57:22 smithi153 ceph-mon[70044]: pgmap v18870: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:57:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:57:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:57:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:57:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:57:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:57:23.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:57:23 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:57:23.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:57:23 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:57:23.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:57:23 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:57:23.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:57:23 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:57:23.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:57:23 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:57:23.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:57:23 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:57:23.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:57:23 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:57:23.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:57:23 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:57:23.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:57:23 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:57:23.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:57:23 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:57:23.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:57:23 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:57:23.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:57:23 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:57:24.718 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:57:24 smithi083 ceph-mon[79930]: pgmap v18871: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:24.719 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:57:24 smithi083 ceph-mon[77242]: pgmap v18871: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:24.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:57:24 smithi153 ceph-mon[70044]: pgmap v18871: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:26.215 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:57:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:57:25.879Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:57:26.628 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:57:26 smithi083 ceph-mon[79930]: pgmap v18872: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:26.628 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:57:26 smithi083 ceph-mon[77242]: pgmap v18872: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:26.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:57:26 smithi153 ceph-mon[70044]: pgmap v18872: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:26.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:57:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:57:26.627Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:57:28.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:57:28 smithi083 ceph-mon[79930]: pgmap v18873: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:28.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:57:28 smithi083 ceph-mon[77242]: pgmap v18873: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:28.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:57:28 smithi153 ceph-mon[70044]: pgmap v18873: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:30.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:57:30 smithi083 ceph-mon[79930]: pgmap v18874: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:30.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:57:30 smithi083 ceph-mon[77242]: pgmap v18874: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:30.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:57:30 smithi153 ceph-mon[70044]: pgmap v18874: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:31.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:57:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:57:31] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T15:57:32.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:57:32 smithi083 ceph-mon[79930]: pgmap v18875: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:32.714 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:57:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:57:32.533Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:57:32.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:57:32 smithi083 ceph-mon[77242]: pgmap v18875: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:32.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:57:32 smithi153 ceph-mon[70044]: pgmap v18875: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:57:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:57:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:57:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:57:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:57:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:57:34.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:57:34 smithi153 ceph-mon[70044]: pgmap v18876: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:34.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:57:34 smithi083 ceph-mon[77242]: pgmap v18876: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:34.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:57:34 smithi083 ceph-mon[79930]: pgmap v18876: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:36.217 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:57:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:57:35.880Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:57:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:57:36 smithi153 ceph-mon[70044]: pgmap v18877: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:36.629 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:57:36 smithi083 ceph-mon[79930]: pgmap v18877: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:36.630 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:57:36 smithi083 ceph-mon[77242]: pgmap v18877: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:36.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:57:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:57:36.628Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:57:37.622 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:57:37.911 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:57:37.911 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 7m ago 10h 27.3M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:57:37.911 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 5m ago 10h 63.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:57:37.911 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 7m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:57:37.912 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 5m ago 10h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:57:37.912 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 5m ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:57:37.912 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 7m ago 10h 1044M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:57:37.912 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 7m ago 10h 2266M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:57:37.912 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 5m ago 10h 989M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:57:37.913 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 7m ago 10h 1361M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:57:37.913 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 7m ago 10h 24.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:57:37.913 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 5m ago 10h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:57:37.914 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 7m ago 10h 1646M 2120M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:57:37.914 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 7m ago 10h 1633M 2120M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:57:37.914 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 7m ago 10h 1268M 2120M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:57:37.914 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 7m ago 10h 1600M 2120M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:57:37.915 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 5m ago 10h 1402M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:57:37.915 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 5m ago 10h 1763M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:57:37.915 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 5m ago 10h 1575M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:57:37.915 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 5m ago 10h 1874M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:57:37.916 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 5m ago 10h 182M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:57:38.265 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:57:38.265 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:57:38.265 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:57:38.266 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:57:38.266 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:57:38.266 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:57:38.267 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:57:38.267 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:57:38.267 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:57:38.267 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:57:38.267 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:57:38.268 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:57:38.268 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:57:38.268 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:57:38.269 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:57:38.269 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:57:38.269 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:57:38.269 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:57:38.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:57:38 smithi153 ceph-mon[70044]: pgmap v18878: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:38.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:57:38 smithi153 ceph-mon[70044]: from='client.66013 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:57:38.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:57:38 smithi153 ceph-mon[70044]: from='client.56088 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:57:38.561 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:57:38 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/46615619' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:57:38.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:57:38 smithi083 ceph-mon[77242]: pgmap v18878: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:38.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:57:38 smithi083 ceph-mon[77242]: from='client.66013 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:57:38.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:57:38 smithi083 ceph-mon[77242]: from='client.56088 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:57:38.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:57:38 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/46615619' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:57:38.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:57:38 smithi083 ceph-mon[79930]: pgmap v18878: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:38.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:57:38 smithi083 ceph-mon[79930]: from='client.66013 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:57:38.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:57:38 smithi083 ceph-mon[79930]: from='client.56088 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:57:38.718 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:57:38 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/46615619' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:57:40.721 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:57:40 smithi083 ceph-mon[79930]: pgmap v18879: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:40.722 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:57:40 smithi083 ceph-mon[77242]: pgmap v18879: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:40.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:57:40 smithi153 ceph-mon[70044]: pgmap v18879: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:57:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:57:41] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T15:57:42.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:57:42 smithi083 ceph-mon[79930]: pgmap v18880: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:42.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:57:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:57:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:57:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:57:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:57:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:57:42.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:57:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:57:42.714 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:57:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:57:42.534Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:57:42.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:57:42 smithi083 ceph-mon[77242]: pgmap v18880: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:42.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:57:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:57:42.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:57:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:57:42.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:57:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:57:42.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:57:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:57:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:57:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:57:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:57:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:57:42 smithi153 ceph-mon[70044]: pgmap v18880: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:57:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:57:42.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:57:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:57:42.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:57:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:57:42.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:57:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:57:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:57:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:57:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:57:44.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:57:44 smithi153 ceph-mon[70044]: pgmap v18881: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:44.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:57:44 smithi083 ceph-mon[79930]: pgmap v18881: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:44.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:57:44 smithi083 ceph-mon[77242]: pgmap v18881: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:46.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:57:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:57:45.881Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:57:46.630 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:57:46 smithi083 ceph-mon[79930]: pgmap v18882: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:46.631 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:57:46 smithi083 ceph-mon[77242]: pgmap v18882: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:46.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:57:46 smithi153 ceph-mon[70044]: pgmap v18882: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:46.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:57:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:57:46.629Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:57:48.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:57:48 smithi083 ceph-mon[79930]: pgmap v18883: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:48.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:57:48 smithi083 ceph-mon[77242]: pgmap v18883: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:48.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:57:48 smithi153 ceph-mon[70044]: pgmap v18883: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:50.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:57:50 smithi083 ceph-mon[79930]: pgmap v18884: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:50.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:57:50 smithi083 ceph-mon[77242]: pgmap v18884: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:50.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:57:50 smithi153 ceph-mon[70044]: pgmap v18884: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:57:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:57:51] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T15:57:52.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:57:52 smithi083 ceph-mon[79930]: pgmap v18885: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:52.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:57:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:57:52.535Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:57:52.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:57:52 smithi083 ceph-mon[77242]: pgmap v18885: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:52.815 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:57:52 smithi153 ceph-mon[70044]: pgmap v18885: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:52.816 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:57:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:57:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:57:53.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:57:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:57:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:57:54.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:57:54 smithi153 ceph-mon[70044]: pgmap v18886: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:54.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:57:54 smithi083 ceph-mon[79930]: pgmap v18886: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:54.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:57:54 smithi083 ceph-mon[77242]: pgmap v18886: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:56.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:57:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:57:55.881Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:57:56.635 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:57:56 smithi083 ceph-mon[79930]: pgmap v18887: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:56.636 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:57:56 smithi083 ceph-mon[77242]: pgmap v18887: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:56.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:57:56 smithi153 ceph-mon[70044]: pgmap v18887: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:56.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:57:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:57:56.629Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:57:58.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:57:58 smithi083 ceph-mon[79930]: pgmap v18888: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:58.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:57:58 smithi083 ceph-mon[77242]: pgmap v18888: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:57:58.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:57:58 smithi153 ceph-mon[70044]: pgmap v18888: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:00.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:58:00 smithi083 ceph-mon[79930]: pgmap v18889: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:00.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:58:00 smithi083 ceph-mon[77242]: pgmap v18889: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:00.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:58:00 smithi153 ceph-mon[70044]: pgmap v18889: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:01.967 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:58:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:58:01] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T15:58:02.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:58:02 smithi083 ceph-mon[77242]: pgmap v18890: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:02.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:58:02 smithi083 ceph-mon[79930]: pgmap v18890: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:02.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:58:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:58:02.536Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:58:02.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:58:02 smithi153 ceph-mon[70044]: pgmap v18890: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:58:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:58:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:58:03.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:58:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:58:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:58:04.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:58:04 smithi153 ceph-mon[70044]: pgmap v18891: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:04.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:58:04 smithi083 ceph-mon[79930]: pgmap v18891: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:04.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:58:04 smithi083 ceph-mon[77242]: pgmap v18891: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:06.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:58:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:58:05.881Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:58:06.631 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:58:06 smithi083 ceph-mon[79930]: pgmap v18892: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:06.631 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:58:06 smithi083 ceph-mon[77242]: pgmap v18892: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:06.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:58:06 smithi153 ceph-mon[70044]: pgmap v18892: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:06.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:58:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:58:06.630Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:58:08.589 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:58:08.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:58:08 smithi083 ceph-mon[79930]: pgmap v18893: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:08.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:58:08 smithi083 ceph-mon[77242]: pgmap v18893: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:08.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:58:08 smithi153 ceph-mon[70044]: pgmap v18893: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:08.880 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:58:08.880 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 8m ago 10h 27.3M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:58:08.881 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 5m ago 10h 63.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:58:08.881 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 8m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:58:08.882 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 5m ago 10h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:58:08.882 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 5m ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:58:08.883 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 8m ago 10h 1044M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:58:08.883 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 8m ago 10h 2266M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:58:08.883 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 5m ago 10h 989M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:58:08.884 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 8m ago 10h 1361M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:58:08.884 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 8m ago 10h 24.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:58:08.885 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 5m ago 10h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:58:08.885 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 8m ago 10h 1646M 2120M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:58:08.886 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 8m ago 10h 1633M 2120M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:58:08.886 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 8m ago 10h 1268M 2120M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:58:08.887 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 8m ago 10h 1600M 2120M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:58:08.887 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 5m ago 10h 1402M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:58:08.887 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 5m ago 10h 1763M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:58:08.888 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 5m ago 10h 1575M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:58:08.888 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 5m ago 10h 1874M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:58:08.889 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 5m ago 10h 182M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:58:09.236 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:58:09.237 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:58:09.237 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:58:09.237 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:58:09.237 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:58:09.238 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:58:09.238 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:58:09.238 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:58:09.238 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:58:09.239 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:58:09.239 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:58:09.239 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:58:09.240 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:58:09.240 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:58:09.240 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:58:09.241 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:58:09.241 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:58:09.241 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:58:09.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:58:09 smithi083 ceph-mon[79930]: from='client.66031 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:58:09.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:58:09 smithi083 ceph-mon[79930]: from='client.66037 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:58:09.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:58:09 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3575045884' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:58:09.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:58:09 smithi083 ceph-mon[77242]: from='client.66031 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:58:09.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:58:09 smithi083 ceph-mon[77242]: from='client.66037 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:58:09.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:58:09 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3575045884' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:58:09.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:58:09 smithi153 ceph-mon[70044]: from='client.66031 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:58:09.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:58:09 smithi153 ceph-mon[70044]: from='client.66037 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:58:09.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:58:09 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3575045884' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:58:10.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:58:10 smithi083 ceph-mon[79930]: pgmap v18894: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:10.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:58:10 smithi083 ceph-mon[77242]: pgmap v18894: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:10.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:58:10 smithi153 ceph-mon[70044]: pgmap v18894: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:58:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:58:11] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T15:58:12.710 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:58:12 smithi083 ceph-mon[77242]: pgmap v18895: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:12.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:58:12 smithi083 ceph-mon[79930]: pgmap v18895: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:12.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:58:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:58:12.537Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:58:12.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:58:12 smithi153 ceph-mon[70044]: pgmap v18895: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:12.811 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:58:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:58:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:58:13.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:58:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:58:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:58:14.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:58:14 smithi153 ceph-mon[70044]: pgmap v18896: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:14.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:58:14 smithi083 ceph-mon[79930]: pgmap v18896: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:14.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:58:14 smithi083 ceph-mon[77242]: pgmap v18896: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:16.210 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:58:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:58:15.881Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:58:16.632 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:58:16 smithi083 ceph-mon[79930]: pgmap v18897: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:16.632 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:58:16 smithi083 ceph-mon[77242]: pgmap v18897: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:16.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:58:16 smithi153 ceph-mon[70044]: pgmap v18897: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:16.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:58:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:58:16.630Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:58:18.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:58:18 smithi083 ceph-mon[77242]: pgmap v18898: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:18.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:58:18 smithi083 ceph-mon[79930]: pgmap v18898: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:18.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:58:18 smithi153 ceph-mon[70044]: pgmap v18898: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:20.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:58:20 smithi083 ceph-mon[77242]: pgmap v18899: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:20.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:58:20 smithi083 ceph-mon[79930]: pgmap v18899: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:20.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:58:20 smithi153 ceph-mon[70044]: pgmap v18899: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:21.966 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:58:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:58:21] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T15:58:22.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:58:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:58:22.537Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:58:22.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:58:22 smithi083 ceph-mon[77242]: pgmap v18900: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:22.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:58:22 smithi083 ceph-mon[79930]: pgmap v18900: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:22.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:58:22 smithi153 ceph-mon[70044]: pgmap v18900: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:58:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:58:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:58:23.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:58:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:58:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:58:24.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:58:24 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:58:24.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:58:24 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:58:24.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:58:24 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:58:24.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:58:24 smithi153 ceph-mon[70044]: pgmap v18901: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:24.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:58:24 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:58:24.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:58:24 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:58:24.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:58:24 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:58:24.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:58:24 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:58:24.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:58:24 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:58:24.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:58:24 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:58:24.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:58:24 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:58:24.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:58:24 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:58:24.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:58:24 smithi153 ceph-mon[70044]: Adjusting osd_memory_target on smithi083 to 2105M 2022-09-29T15:58:24.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:58:24 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:58:24.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:58:24 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:58:24.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:58:24 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:58:24.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:58:24 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:58:24.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:58:24 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:58:24.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:58:24 smithi083 ceph-mon[79930]: pgmap v18901: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:24.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:58:24 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:58:24.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:58:24 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:58:24.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:58:24 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:58:24.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:58:24 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:58:24.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:58:24 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:58:24.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:58:24 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:58:24.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:58:24 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:58:24.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:58:24 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:58:24.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:58:24 smithi083 ceph-mon[79930]: Adjusting osd_memory_target on smithi083 to 2105M 2022-09-29T15:58:24.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:58:24 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:58:24.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:58:24 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:58:24.467 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:58:24 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:58:24.467 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:58:24 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:58:24.467 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:58:24 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:58:24.468 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:58:24 smithi083 ceph-mon[77242]: pgmap v18901: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:24.468 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:58:24 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:58:24.468 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:58:24 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:58:24.468 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:58:24 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:58:24.468 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:58:24 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:58:24.469 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:58:24 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:58:24.469 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:58:24 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:58:24.469 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:58:24 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:58:24.469 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:58:24 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T15:58:24.470 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:58:24 smithi083 ceph-mon[77242]: Adjusting osd_memory_target on smithi083 to 2105M 2022-09-29T15:58:24.470 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:58:24 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:58:24.470 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:58:24 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:58:26.215 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:58:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:58:25.882Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:58:26.632 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:58:26 smithi083 ceph-mon[79930]: pgmap v18902: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:26.633 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:58:26 smithi083 ceph-mon[77242]: pgmap v18902: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:26.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:58:26 smithi153 ceph-mon[70044]: pgmap v18902: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:26.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:58:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:58:26.631Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:58:28.719 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:58:28 smithi083 ceph-mon[77242]: pgmap v18903: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:28.719 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:58:28 smithi083 ceph-mon[79930]: pgmap v18903: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:28.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:58:28 smithi153 ceph-mon[70044]: pgmap v18903: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:30.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:58:30 smithi083 ceph-mon[79930]: pgmap v18904: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:30.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:58:30 smithi083 ceph-mon[77242]: pgmap v18904: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:30.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:58:30 smithi153 ceph-mon[70044]: pgmap v18904: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:31.973 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:58:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:58:31] "GET /metrics HTTP/1.1" 200 225233 "" "Prometheus/2.33.4" 2022-09-29T15:58:32.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:58:32 smithi083 ceph-mon[79930]: pgmap v18905: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:32.718 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:58:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:58:32.538Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:58:32.719 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:58:32 smithi083 ceph-mon[77242]: pgmap v18905: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:32.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:58:32 smithi153 ceph-mon[70044]: pgmap v18905: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:32.810 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:58:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:58:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:58:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:58:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:58:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:58:34.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:58:34 smithi153 ceph-mon[70044]: pgmap v18906: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:34.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:58:34 smithi083 ceph-mon[79930]: pgmap v18906: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:34.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:58:34 smithi083 ceph-mon[77242]: pgmap v18906: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:36.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:58:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:58:35.882Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:58:36.643 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:58:36 smithi083 ceph-mon[79930]: pgmap v18907: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:36.644 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:58:36 smithi083 ceph-mon[77242]: pgmap v18907: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:36.817 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:58:36 smithi153 ceph-mon[70044]: pgmap v18907: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:36.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:58:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:58:36.632Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:58:38.722 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:58:38 smithi083 ceph-mon[77242]: pgmap v18908: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:38.723 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:58:38 smithi083 ceph-mon[79930]: pgmap v18908: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:38.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:58:38 smithi153 ceph-mon[70044]: pgmap v18908: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:39.565 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:58:39.861 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:58:39.862 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 8m ago 10h 27.3M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:58:39.862 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 6m ago 10h 63.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:58:39.862 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 8m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:58:39.863 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 6m ago 10h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:58:39.863 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 6m ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:58:39.863 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 8m ago 10h 1044M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:58:39.863 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 8m ago 10h 2266M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:58:39.864 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 6m ago 10h 989M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:58:39.864 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 8m ago 10h 1361M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:58:39.864 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 8m ago 10h 24.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:58:39.864 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 6m ago 10h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:58:39.864 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 8m ago 10h 1646M 2105M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:58:39.865 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 8m ago 10h 1633M 2105M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:58:39.865 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 8m ago 10h 1268M 2105M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:58:39.865 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 8m ago 10h 1600M 2105M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:58:39.865 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 6m ago 10h 1402M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:58:39.866 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 6m ago 10h 1763M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:58:39.866 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 6m ago 10h 1575M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:58:39.866 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 6m ago 10h 1874M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:58:39.866 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 6m ago 10h 182M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:58:40.215 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:58:40.215 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:58:40.216 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:58:40.217 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:58:40.217 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:58:40.218 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:58:40.219 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:58:40.219 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:58:40.220 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:58:40.220 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:58:40.221 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:58:40.221 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:58:40.222 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:58:40.222 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:58:40.222 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:58:40.224 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:58:40.225 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:58:40.225 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:58:40.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:58:40 smithi083 ceph-mon[77242]: pgmap v18909: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:40.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:58:40 smithi083 ceph-mon[77242]: from='client.66049 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:58:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:58:40 smithi083 ceph-mon[77242]: from='client.56124 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:58:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:58:40 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3325054848' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:58:40.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:58:40 smithi083 ceph-mon[79930]: pgmap v18909: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:40.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:58:40 smithi083 ceph-mon[79930]: from='client.66049 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:58:40.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:58:40 smithi083 ceph-mon[79930]: from='client.56124 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:58:40.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:58:40 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3325054848' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:58:40.816 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:58:40 smithi153 ceph-mon[70044]: pgmap v18909: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:40.817 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:58:40 smithi153 ceph-mon[70044]: from='client.66049 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:58:40.817 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:58:40 smithi153 ceph-mon[70044]: from='client.56124 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:58:40.817 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:58:40 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3325054848' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:58:41.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:58:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:58:41] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T15:58:42.718 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:58:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:58:42.538Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:58:42.719 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:58:42 smithi083 ceph-mon[77242]: pgmap v18910: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:42.720 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:58:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:58:42.720 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:58:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:58:42.720 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:58:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:58:42.721 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:58:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:58:42.721 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:58:42 smithi083 ceph-mon[79930]: pgmap v18910: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:42.721 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:58:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:58:42.722 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:58:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:58:42.723 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:58:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:58:42.723 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:58:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:58:42.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:58:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:58:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:58:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:58:42 smithi153 ceph-mon[70044]: pgmap v18910: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:58:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:58:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:58:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:58:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:58:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:58:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:58:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:58:43.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:58:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:58:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:58:44.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:58:44 smithi153 ceph-mon[70044]: pgmap v18911: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:44.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:58:44 smithi083 ceph-mon[79930]: pgmap v18911: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:44.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:58:44 smithi083 ceph-mon[77242]: pgmap v18911: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:46.218 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:58:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:58:45.883Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:58:46.636 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:58:46 smithi083 ceph-mon[79930]: pgmap v18912: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:46.637 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:58:46 smithi083 ceph-mon[77242]: pgmap v18912: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:46.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:58:46 smithi153 ceph-mon[70044]: pgmap v18912: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:46.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:58:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:58:46.633Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:58:48.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:58:48 smithi083 ceph-mon[79930]: pgmap v18913: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:48.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:58:48 smithi083 ceph-mon[77242]: pgmap v18913: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:48.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:58:48 smithi153 ceph-mon[70044]: pgmap v18913: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:50.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:58:50 smithi083 ceph-mon[79930]: pgmap v18914: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:50.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:58:50 smithi083 ceph-mon[77242]: pgmap v18914: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:50.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:58:50 smithi153 ceph-mon[70044]: pgmap v18914: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:58:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:58:51] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T15:58:52.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:58:52 smithi083 ceph-mon[79930]: pgmap v18915: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:52.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:58:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:58:52.539Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:58:52.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:58:52 smithi083 ceph-mon[77242]: pgmap v18915: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:52.814 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:58:52 smithi153 ceph-mon[70044]: pgmap v18915: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:52.815 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:58:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:58:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:58:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:58:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:58:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:58:54.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:58:54 smithi153 ceph-mon[70044]: pgmap v18916: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:54.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:58:54 smithi083 ceph-mon[77242]: pgmap v18916: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:54.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:58:54 smithi083 ceph-mon[79930]: pgmap v18916: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:56.217 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:58:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:58:55.883Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:58:56.634 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:58:56 smithi083 ceph-mon[79930]: pgmap v18917: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:56.635 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:58:56 smithi083 ceph-mon[77242]: pgmap v18917: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:56.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:58:56 smithi153 ceph-mon[70044]: pgmap v18917: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:56.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:58:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:58:56.633Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:58:58.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:58:58 smithi083 ceph-mon[79930]: pgmap v18918: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:58.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:58:58 smithi083 ceph-mon[77242]: pgmap v18918: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:58:58.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:58:58 smithi153 ceph-mon[70044]: pgmap v18918: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:00.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:59:00 smithi083 ceph-mon[79930]: pgmap v18919: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:00.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:59:00 smithi083 ceph-mon[77242]: pgmap v18919: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:00.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:59:00 smithi153 ceph-mon[70044]: pgmap v18919: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:59:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:59:01] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T15:59:02.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:59:02 smithi083 ceph-mon[79930]: pgmap v18920: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:02.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:59:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:59:02.540Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:59:02.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:59:02 smithi083 ceph-mon[77242]: pgmap v18920: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:02.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:59:02 smithi153 ceph-mon[70044]: pgmap v18920: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:02.812 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:59:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:59:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:59:03.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:59:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:59:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:59:04.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:59:04 smithi083 ceph-mon[79930]: pgmap v18921: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:04.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:59:04 smithi083 ceph-mon[77242]: pgmap v18921: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:59:04 smithi153 ceph-mon[70044]: pgmap v18921: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:06.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:59:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:59:05.883Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:59:06.634 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:59:06 smithi083 ceph-mon[79930]: pgmap v18922: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:06.635 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:59:06 smithi083 ceph-mon[77242]: pgmap v18922: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:06.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:59:06 smithi153 ceph-mon[70044]: pgmap v18922: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:06.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:59:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:59:06.634Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:59:08.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:59:08 smithi083 ceph-mon[79930]: pgmap v18923: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:08.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:59:08 smithi083 ceph-mon[77242]: pgmap v18923: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:08.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:59:08 smithi153 ceph-mon[70044]: pgmap v18923: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:10.544 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:59:10.710 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:59:10 smithi083 ceph-mon[77242]: pgmap v18924: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:10.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:59:10 smithi083 ceph-mon[79930]: pgmap v18924: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:10.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:59:10 smithi153 ceph-mon[70044]: pgmap v18924: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:10.832 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:59:10.832 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 9m ago 10h 27.3M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:59:10.832 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 6m ago 10h 63.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:59:10.832 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 9m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:59:10.833 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 6m ago 10h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:59:10.833 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 6m ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:59:10.833 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 9m ago 10h 1044M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:59:10.833 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 9m ago 10h 2266M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:59:10.834 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 6m ago 10h 989M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:59:10.834 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 9m ago 10h 1361M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:59:10.834 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 9m ago 10h 24.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:59:10.834 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 6m ago 10h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:59:10.835 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 9m ago 10h 1646M 2105M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:59:10.835 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 9m ago 10h 1633M 2105M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:59:10.835 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 9m ago 10h 1268M 2105M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:59:10.835 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 9m ago 10h 1600M 2105M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:59:10.836 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 6m ago 10h 1402M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:59:10.836 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 6m ago 10h 1763M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:59:10.836 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 6m ago 10h 1575M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:59:10.836 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 6m ago 10h 1874M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:59:10.837 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 6m ago 10h 182M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:59:11.188 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:59:11.188 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:59:11.188 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:59:11.189 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:59:11.189 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:59:11.189 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:59:11.189 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:59:11.190 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:59:11.190 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:59:11.190 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:59:11.190 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:59:11.191 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:59:11.191 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:59:11.191 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:59:11.191 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:59:11.191 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:59:11.192 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:59:11.192 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:59:11.714 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:59:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:59:11] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T15:59:11.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:59:11 smithi083 ceph-mon[77242]: from='client.66067 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:59:11.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:59:11 smithi083 ceph-mon[77242]: from='client.66073 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:59:11.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:59:11 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2859309872' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:59:11.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:59:11 smithi083 ceph-mon[79930]: from='client.66067 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:59:11.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:59:11 smithi083 ceph-mon[79930]: from='client.66073 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:59:11.733 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:59:11 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2859309872' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:59:11.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:59:11 smithi153 ceph-mon[70044]: from='client.66067 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:59:11.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:59:11 smithi153 ceph-mon[70044]: from='client.66073 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:59:11.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:59:11 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2859309872' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:59:12.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:59:12 smithi083 ceph-mon[77242]: pgmap v18925: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:12.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:59:12 smithi083 ceph-mon[79930]: pgmap v18925: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:12.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:59:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:59:12.541Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:59:12.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:59:12 smithi153 ceph-mon[70044]: pgmap v18925: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:59:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:59:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:59:13.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:59:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:59:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:59:14.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:59:14 smithi083 ceph-mon[77242]: pgmap v18926: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:14.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:59:14 smithi083 ceph-mon[79930]: pgmap v18926: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:14.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:59:14 smithi153 ceph-mon[70044]: pgmap v18926: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:16.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:59:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:59:15.884Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:59:16.635 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:59:16 smithi083 ceph-mon[77242]: pgmap v18927: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:16.636 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:59:16 smithi083 ceph-mon[79930]: pgmap v18927: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:16.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:59:16 smithi153 ceph-mon[70044]: pgmap v18927: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:16.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:59:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:59:16.634Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:59:18.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:59:18 smithi083 ceph-mon[77242]: pgmap v18928: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:18.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:59:18 smithi083 ceph-mon[79930]: pgmap v18928: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:18.815 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:59:18 smithi153 ceph-mon[70044]: pgmap v18928: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:20.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:59:20 smithi083 ceph-mon[79930]: pgmap v18929: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:20.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:59:20 smithi083 ceph-mon[77242]: pgmap v18929: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:20.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:59:20 smithi153 ceph-mon[70044]: pgmap v18929: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:21.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:59:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:59:21] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T15:59:22.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:59:22 smithi083 ceph-mon[79930]: pgmap v18930: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:22.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:59:22 smithi083 ceph-mon[77242]: pgmap v18930: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:22.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:59:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:59:22.541Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:59:22.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:59:22 smithi153 ceph-mon[70044]: pgmap v18930: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:22.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:59:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:59:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:59:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:59:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:59:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:59:24.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:59:24 smithi083 ceph-mon[79930]: pgmap v18931: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:24.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:59:24 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:59:24.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:59:24 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:59:24.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:59:24 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:59:24.470 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:59:24 smithi083 ceph-mon[77242]: pgmap v18931: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:24.471 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:59:24 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:59:24.471 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:59:24 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:59:24.472 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:59:24 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:59:24.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:59:24 smithi153 ceph-mon[70044]: pgmap v18931: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:59:24 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T15:59:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:59:24 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T15:59:24.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:59:24 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T15:59:25.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:59:25 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:59:25.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:59:25 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:59:25.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:59:25 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T15:59:26.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:59:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:59:25.885Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:59:26.636 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:59:26 smithi083 ceph-mon[77242]: pgmap v18932: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:26.636 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:59:26 smithi083 ceph-mon[79930]: pgmap v18932: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:26.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:59:26 smithi153 ceph-mon[70044]: pgmap v18932: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:26.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:59:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:59:26.634Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:59:28.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:59:28 smithi083 ceph-mon[79930]: pgmap v18933: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:28.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:59:28 smithi083 ceph-mon[77242]: pgmap v18933: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:28.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:59:28 smithi153 ceph-mon[70044]: pgmap v18933: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:30.718 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:59:30 smithi083 ceph-mon[79930]: pgmap v18934: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:30.719 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:59:30 smithi083 ceph-mon[77242]: pgmap v18934: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:30.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:59:30 smithi153 ceph-mon[70044]: pgmap v18934: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:59:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:59:31] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T15:59:32.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:59:32 smithi083 ceph-mon[79930]: pgmap v18935: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:32.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:59:32 smithi083 ceph-mon[77242]: pgmap v18935: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:32.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:59:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:59:32.542Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:59:32.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:59:32 smithi153 ceph-mon[70044]: pgmap v18935: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:59:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:59:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:59:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:59:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:59:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:59:34.467 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:59:34 smithi083 ceph-mon[79930]: pgmap v18936: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:34.467 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:59:34 smithi083 ceph-mon[77242]: pgmap v18936: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:59:34 smithi153 ceph-mon[70044]: pgmap v18936: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:36.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:59:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:59:35.886Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:59:36.643 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:59:36 smithi083 ceph-mon[79930]: pgmap v18937: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:36.643 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:59:36 smithi083 ceph-mon[77242]: pgmap v18937: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:36.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:59:36 smithi153 ceph-mon[70044]: pgmap v18937: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:36.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:59:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:59:36.635Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:59:38.721 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:59:38 smithi083 ceph-mon[79930]: pgmap v18938: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:38.722 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:59:38 smithi083 ceph-mon[77242]: pgmap v18938: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:38.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:59:38 smithi153 ceph-mon[70044]: pgmap v18938: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:40.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:59:40 smithi083 ceph-mon[79930]: pgmap v18939: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:40.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:59:40 smithi083 ceph-mon[77242]: pgmap v18939: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:40.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:59:40 smithi153 ceph-mon[70044]: pgmap v18939: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:41.517 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T15:59:41.808 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T15:59:41.808 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 9m ago 10h 27.3M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T15:59:41.809 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 7m ago 10h 63.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T15:59:41.809 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 9m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T15:59:41.809 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 7m ago 10h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T15:59:41.809 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 7m ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T15:59:41.810 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 9m ago 10h 1044M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T15:59:41.810 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 9m ago 10h 2266M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T15:59:41.810 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 7m ago 10h 989M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T15:59:41.810 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 9m ago 10h 1361M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T15:59:41.811 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 9m ago 10h 24.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T15:59:41.811 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 7m ago 10h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T15:59:41.812 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 9m ago 10h 1646M 2105M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T15:59:41.812 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 9m ago 10h 1633M 2105M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T15:59:41.812 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 9m ago 10h 1268M 2105M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T15:59:41.812 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 9m ago 10h 1600M 2105M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T15:59:41.813 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 7m ago 10h 1402M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T15:59:41.813 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 7m ago 10h 1763M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T15:59:41.813 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 7m ago 10h 1575M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T15:59:41.813 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 7m ago 10h 1874M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T15:59:41.814 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 7m ago 10h 182M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T15:59:41.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:59:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:59:41] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T15:59:42.178 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T15:59:42.179 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T15:59:42.179 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T15:59:42.179 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:59:42.179 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T15:59:42.180 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T15:59:42.180 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:59:42.180 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T15:59:42.180 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T15:59:42.181 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:59:42.181 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T15:59:42.181 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T15:59:42.181 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T15:59:42.181 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T15:59:42.195 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T15:59:42.196 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T15:59:42.196 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T15:59:42.196 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T15:59:42.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:59:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:59:42.542Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:59:42.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:59:42 smithi083 ceph-mon[77242]: pgmap v18940: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:59:42 smithi083 ceph-mon[77242]: from='client.56154 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:59:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:59:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:59:42.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:59:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:59:42.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:59:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:59:42.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:59:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:59:42.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:59:42 smithi083 ceph-mon[77242]: from='client.66091 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:59:42.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:59:42 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2259557327' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:59:42.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:59:42 smithi083 ceph-mon[79930]: pgmap v18940: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:42.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:59:42 smithi083 ceph-mon[79930]: from='client.56154 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:59:42.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:59:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:59:42.718 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:59:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:59:42.718 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:59:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:59:42.719 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:59:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:59:42.719 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:59:42 smithi083 ceph-mon[79930]: from='client.66091 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:59:42.720 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:59:42 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2259557327' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:59:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:59:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:59:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:59:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:59:42 smithi153 ceph-mon[70044]: pgmap v18940: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:59:42 smithi153 ceph-mon[70044]: from='client.56154 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:59:42.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:59:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:59:42.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:59:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:59:42.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:59:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T15:59:42.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:59:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T15:59:42.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:59:42 smithi153 ceph-mon[70044]: from='client.66091 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T15:59:42.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:59:42 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2259557327' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T15:59:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:59:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:59:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:59:44.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:59:44 smithi083 ceph-mon[79930]: pgmap v18941: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:44.469 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:59:44 smithi083 ceph-mon[77242]: pgmap v18941: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:44.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:59:44 smithi153 ceph-mon[70044]: pgmap v18941: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:46.215 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:59:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:59:45.886Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:59:46.636 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:59:46 smithi083 ceph-mon[79930]: pgmap v18942: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:46.638 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:59:46 smithi083 ceph-mon[77242]: pgmap v18942: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:46.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:59:46 smithi153 ceph-mon[70044]: pgmap v18942: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:46.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:59:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:59:46.636Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:59:48.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:59:48 smithi083 ceph-mon[79930]: pgmap v18943: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:48.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:59:48 smithi083 ceph-mon[77242]: pgmap v18943: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:48.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:59:48 smithi153 ceph-mon[70044]: pgmap v18943: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:50.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:59:50 smithi083 ceph-mon[79930]: pgmap v18944: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:50.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:59:50 smithi083 ceph-mon[77242]: pgmap v18944: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:50.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:59:50 smithi153 ceph-mon[70044]: pgmap v18944: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 15:59:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:59:51] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T15:59:52.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:59:52 smithi083 ceph-mon[79930]: pgmap v18945: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:52.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:59:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:59:52.543Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:59:52.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:59:52 smithi083 ceph-mon[77242]: pgmap v18945: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:52.819 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:59:52 smithi153 ceph-mon[70044]: pgmap v18945: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:52.820 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 15:59:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T15:59:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T15:59:53.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 15:59:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:15:59:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T15:59:54.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:59:54 smithi083 ceph-mon[79930]: pgmap v18946: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:54.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:59:54 smithi083 ceph-mon[77242]: pgmap v18946: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:54.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:59:54 smithi153 ceph-mon[70044]: pgmap v18946: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:56.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:59:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:59:55.886Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:59:56.638 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:59:56 smithi083 ceph-mon[79930]: pgmap v18947: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:56.638 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:59:56 smithi083 ceph-mon[77242]: pgmap v18947: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:56.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:59:56 smithi153 ceph-mon[70044]: pgmap v18947: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:56.964 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 15:59:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T15:59:56.637Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T15:59:58.735 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 15:59:58 smithi083 ceph-mon[79930]: pgmap v18948: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:58.736 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 15:59:58 smithi083 ceph-mon[77242]: pgmap v18948: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T15:59:58.814 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 15:59:58 smithi153 ceph-mon[70044]: pgmap v18948: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:00.356 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:00:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[77219]: 2022-09-29T15:59:59.999+0000 7f160f665700 -1 log_channel(cluster) log [ERR] : overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T16:00:00.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:00:00 smithi083 ceph-mon[77242]: pgmap v18949: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:00.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:00:00 smithi083 ceph-mon[77242]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T16:00:00.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:00:00 smithi083 ceph-mon[79930]: pgmap v18949: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:00.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:00:00 smithi083 ceph-mon[79930]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T16:00:00.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:00:00 smithi153 ceph-mon[70044]: pgmap v18949: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:00.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:00:00 smithi153 ceph-mon[70044]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T16:00:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:00:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:00:01] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T16:00:02.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:00:02 smithi083 ceph-mon[79930]: pgmap v18950: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:02.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:00:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:00:02.543Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:00:02.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:00:02 smithi083 ceph-mon[77242]: pgmap v18950: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:02.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:00:02 smithi153 ceph-mon[70044]: pgmap v18950: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:00:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:00:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:00:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:00:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:00:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:00:04.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:00:04 smithi083 ceph-mon[79930]: pgmap v18951: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:04.467 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:00:04 smithi083 ceph-mon[77242]: pgmap v18951: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:04.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:00:04 smithi153 ceph-mon[70044]: pgmap v18951: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:06.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:00:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:00:05.886Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:00:06.639 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:00:06 smithi083 ceph-mon[79930]: pgmap v18952: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:06.639 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:00:06 smithi083 ceph-mon[77242]: pgmap v18952: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:06.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:00:06 smithi153 ceph-mon[70044]: pgmap v18952: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:06.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:00:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:00:06.638Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:00:08.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:00:08 smithi083 ceph-mon[79930]: pgmap v18953: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:08.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:00:08 smithi083 ceph-mon[77242]: pgmap v18953: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:08.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:00:08 smithi153 ceph-mon[70044]: pgmap v18953: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:10.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:00:10 smithi083 ceph-mon[79930]: pgmap v18954: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:10.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:00:10 smithi083 ceph-mon[77242]: pgmap v18954: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:10.814 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:00:10 smithi153 ceph-mon[70044]: pgmap v18954: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:00:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:00:11] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T16:00:12.489 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:00:12.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:00:12 smithi083 ceph-mon[77242]: pgmap v18955: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:12.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:00:12 smithi083 ceph-mon[79930]: pgmap v18955: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:12.714 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:00:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:00:12.544Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:00:12.777 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:00:12.778 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 10m ago 10h 27.3M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:00:12.778 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 7m ago 10h 63.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:00:12.779 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 10m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:00:12.779 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 7m ago 10h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:00:12.779 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 7m ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:00:12.780 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 10m ago 10h 1044M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:00:12.780 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 10m ago 10h 2266M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:00:12.789 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 7m ago 10h 989M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:00:12.790 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 10m ago 10h 1361M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:00:12.791 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 10m ago 10h 24.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:00:12.791 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 7m ago 10h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:00:12.792 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 10m ago 10h 1646M 2105M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:00:12.792 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 10m ago 10h 1633M 2105M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:00:12.793 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 10m ago 10h 1268M 2105M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:00:12.793 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 10m ago 10h 1600M 2105M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:00:12.794 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 7m ago 10h 1402M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:00:12.794 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 7m ago 10h 1763M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:00:12.795 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 7m ago 10h 1575M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:00:12.795 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 7m ago 10h 1874M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:00:12.796 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 7m ago 10h 182M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:00:12.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:00:12 smithi153 ceph-mon[70044]: pgmap v18955: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:00:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:00:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:00:13.140 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:00:13.220 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:00:13.220 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:00:13.221 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:00:13.221 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:00:13.221 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:00:13.222 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:00:13.222 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:00:13.222 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:00:13.223 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:00:13.223 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:00:13.223 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:00:13.224 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:00:13.224 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:00:13.224 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:00:13.225 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:00:13.225 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:00:13.225 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:00:13.318 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:00:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:00:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:00:13.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:00:13 smithi083 ceph-mon[77242]: from='client.66103 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:00:13.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:00:13 smithi083 ceph-mon[77242]: from='client.56178 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:00:13.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:00:13 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3463640918' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:00:13.718 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:00:13 smithi083 ceph-mon[79930]: from='client.66103 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:00:13.718 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:00:13 smithi083 ceph-mon[79930]: from='client.56178 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:00:13.719 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:00:13 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3463640918' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:00:13.820 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:00:13 smithi153 ceph-mon[70044]: from='client.66103 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:00:13.820 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:00:13 smithi153 ceph-mon[70044]: from='client.56178 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:00:13.821 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:00:13 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3463640918' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:00:14.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:00:14 smithi083 ceph-mon[79930]: pgmap v18956: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:14.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:00:14 smithi083 ceph-mon[77242]: pgmap v18956: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:14.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:00:14 smithi153 ceph-mon[70044]: pgmap v18956: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:16.224 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:00:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:00:15.887Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:00:16.650 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:00:16 smithi083 ceph-mon[79930]: pgmap v18957: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:16.651 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:00:16 smithi083 ceph-mon[77242]: pgmap v18957: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:16.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:00:16 smithi153 ceph-mon[70044]: pgmap v18957: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:16.966 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:00:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:00:16.639Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:00:18.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:00:18 smithi083 ceph-mon[79930]: pgmap v18958: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:18.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:00:18 smithi083 ceph-mon[77242]: pgmap v18958: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:18.815 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:00:18 smithi153 ceph-mon[70044]: pgmap v18958: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:20.710 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:00:20 smithi083 ceph-mon[77242]: pgmap v18959: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:20.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:00:20 smithi083 ceph-mon[79930]: pgmap v18959: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:20.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:00:20 smithi153 ceph-mon[70044]: pgmap v18959: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:00:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:00:21] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T16:00:22.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:00:22 smithi083 ceph-mon[79930]: pgmap v18960: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:22.733 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:00:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:00:22.545Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:00:22.734 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:00:22 smithi083 ceph-mon[77242]: pgmap v18960: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:22.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:00:22 smithi153 ceph-mon[70044]: pgmap v18960: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:00:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:00:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:00:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:00:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:00:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:00:24.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:00:24 smithi083 ceph-mon[79930]: pgmap v18961: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:24.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:00:24 smithi083 ceph-mon[77242]: pgmap v18961: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:24.562 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:00:24 smithi153 ceph-mon[70044]: pgmap v18961: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:25.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:00:25 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:00:25.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:00:25 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:00:25.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:00:25 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:00:25.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:00:25 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:00:25.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:00:25 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:00:25.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:00:25 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:00:25.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:00:25 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:00:25.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:00:25 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:00:25.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:00:25 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:00:25.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:00:25 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:00:25.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:00:25 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:00:25.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:00:25 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:00:25.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:00:25 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:00:25.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:00:25 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:00:25.564 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:00:25 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:00:25.565 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:00:25 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:00:25.565 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:00:25 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:00:25.566 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:00:25 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:00:25.566 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:00:25 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:00:25.568 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:00:25 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:00:25.568 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:00:25 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:00:26.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:00:26 smithi083 ceph-mon[79930]: pgmap v18962: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:26.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:00:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:00:25.887Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:00:26.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:00:26 smithi083 ceph-mon[77242]: pgmap v18962: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:00:26 smithi153 ceph-mon[70044]: pgmap v18962: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:26.973 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:00:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:00:26.639Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:00:28.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:00:28 smithi083 ceph-mon[79930]: pgmap v18963: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:28.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:00:28 smithi083 ceph-mon[77242]: pgmap v18963: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:28.815 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:00:28 smithi153 ceph-mon[70044]: pgmap v18963: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:30.316 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:00:30 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:00:30.317 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:00:30 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:00:30.317 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:00:30 smithi153 ceph-mon[70044]: pgmap v18964: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:30.318 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:00:30 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:00:30.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:00:30 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:00:30.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:00:30 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:00:30.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:00:30 smithi083 ceph-mon[79930]: pgmap v18964: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:30.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:00:30 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:00:30.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:00:30 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:00:30.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:00:30 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:00:30.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:00:30 smithi083 ceph-mon[77242]: pgmap v18964: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:30.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:00:30 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:00:31.970 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:00:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:00:31] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T16:00:32.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:00:32 smithi083 ceph-mon[77242]: pgmap v18965: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:32.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:00:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:00:32.546Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:00:32.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:00:32 smithi083 ceph-mon[79930]: pgmap v18965: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:32.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:00:32 smithi153 ceph-mon[70044]: pgmap v18965: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:32.810 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:00:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:00:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:00:33.325 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:00:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:00:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:00:34.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:00:34 smithi083 ceph-mon[77242]: pgmap v18966: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:34.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:00:34 smithi083 ceph-mon[79930]: pgmap v18966: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:34.564 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:00:34 smithi153 ceph-mon[70044]: pgmap v18966: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:36.215 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:00:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:00:35.889Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:00:36.641 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:00:36 smithi083 ceph-mon[79930]: pgmap v18967: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:36.643 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:00:36 smithi083 ceph-mon[77242]: pgmap v18967: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:36.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:00:36 smithi153 ceph-mon[70044]: pgmap v18967: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:36.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:00:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:00:36.640Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:00:38.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:00:38 smithi083 ceph-mon[77242]: pgmap v18968: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:38.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:00:38 smithi083 ceph-mon[79930]: pgmap v18968: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:38.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:00:38 smithi153 ceph-mon[70044]: pgmap v18968: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:40.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:00:40 smithi083 ceph-mon[79930]: pgmap v18969: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:00:40 smithi083 ceph-mon[77242]: pgmap v18969: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:40.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:00:40 smithi153 ceph-mon[70044]: pgmap v18969: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:41.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:00:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:00:41] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T16:00:42.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:00:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:00:42.547Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:00:42.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:00:42 smithi083 ceph-mon[77242]: pgmap v18970: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:42.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:00:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:00:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:00:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:00:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:00:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:00:42.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:00:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:00:42.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:00:42 smithi083 ceph-mon[79930]: pgmap v18970: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:42.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:00:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:00:42.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:00:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:00:42.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:00:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:00:42.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:00:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:00:42.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:00:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:00:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:00:42.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:00:42 smithi153 ceph-mon[70044]: pgmap v18970: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:42.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:00:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:00:42.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:00:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:00:42.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:00:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:00:42.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:00:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:00:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:00:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:00:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:00:43.486 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:00:43.771 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:00:43.772 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 14s ago 10h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:00:43.772 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 8m ago 10h 63.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:00:43.772 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 14s ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:00:43.772 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 8m ago 10h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:00:43.773 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 8m ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:00:43.773 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 14s ago 10h 1052M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:00:43.773 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 14s ago 10h 2300M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:00:43.773 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 8m ago 10h 989M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:00:43.774 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 14s ago 10h 1384M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:00:43.774 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 14s ago 10h 24.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:00:43.775 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 8m ago 10h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:00:43.775 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 14s ago 10h 1673M 2105M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:00:43.775 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 14s ago 10h 1658M 2105M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:00:43.775 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 14s ago 10h 1288M 2105M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:00:43.776 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 14s ago 10h 1626M 2105M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:00:43.776 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 8m ago 10h 1402M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:00:43.776 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 8m ago 10h 1763M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:00:43.776 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 8m ago 10h 1575M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:00:43.777 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 8m ago 10h 1874M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:00:43.777 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 8m ago 10h 182M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:00:44.129 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:00:44.129 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:00:44.130 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:00:44.130 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:00:44.130 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:00:44.131 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:00:44.131 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:00:44.132 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:00:44.132 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:00:44.132 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:00:44.133 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:00:44.133 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:00:44.134 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:00:44.136 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:00:44.136 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:00:44.137 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:00:44.137 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:00:44.138 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:00:44.471 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:00:44 smithi083 ceph-mon[79930]: pgmap v18971: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:44.472 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:00:44 smithi083 ceph-mon[79930]: from='client.56190 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:00:44.472 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:00:44 smithi083 ceph-mon[79930]: from='client.66127 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:00:44.473 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:00:44 smithi083 ceph-mon[77242]: pgmap v18971: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:44.473 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:00:44 smithi083 ceph-mon[77242]: from='client.56190 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:00:44.474 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:00:44 smithi083 ceph-mon[77242]: from='client.66127 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:00:44.568 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:00:44 smithi153 ceph-mon[70044]: pgmap v18971: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:44.569 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:00:44 smithi153 ceph-mon[70044]: from='client.56190 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:00:44.569 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:00:44 smithi153 ceph-mon[70044]: from='client.66127 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:00:45.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:00:45 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4273570161' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:00:45.467 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:00:45 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4273570161' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:00:45.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:00:45 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4273570161' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:00:46.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:00:46 smithi083 ceph-mon[79930]: pgmap v18972: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:46.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:00:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:00:45.889Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:00:46.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:00:46 smithi083 ceph-mon[77242]: pgmap v18972: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:46.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:00:46 smithi153 ceph-mon[70044]: pgmap v18972: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:46.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:00:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:00:46.641Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:00:48.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:00:48 smithi083 ceph-mon[79930]: pgmap v18973: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:48.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:00:48 smithi083 ceph-mon[77242]: pgmap v18973: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:48.816 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:00:48 smithi153 ceph-mon[70044]: pgmap v18973: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:50.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:00:50 smithi083 ceph-mon[77242]: pgmap v18974: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:50.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:00:50 smithi083 ceph-mon[79930]: pgmap v18974: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:50.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:00:50 smithi153 ceph-mon[70044]: pgmap v18974: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:00:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:00:51] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T16:00:52.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:00:52 smithi083 ceph-mon[79930]: pgmap v18975: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:52.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:00:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:00:52.548Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:00:52.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:00:52 smithi083 ceph-mon[77242]: pgmap v18975: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:52.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:00:52 smithi153 ceph-mon[70044]: pgmap v18975: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:52.822 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:00:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:00:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:00:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:00:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:00:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:00:54.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:00:54 smithi083 ceph-mon[79930]: pgmap v18976: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:54.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:00:54 smithi083 ceph-mon[77242]: pgmap v18976: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:54.564 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:00:54 smithi153 ceph-mon[70044]: pgmap v18976: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:56.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:00:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:00:55.890Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:00:56.642 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:00:56 smithi083 ceph-mon[79930]: pgmap v18977: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:56.643 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:00:56 smithi083 ceph-mon[77242]: pgmap v18977: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:56.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:00:56 smithi153 ceph-mon[70044]: pgmap v18977: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:56.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:00:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:00:56.641Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:00:58.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:00:58 smithi083 ceph-mon[79930]: pgmap v18978: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:58.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:00:58 smithi083 ceph-mon[77242]: pgmap v18978: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:00:58.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:00:58 smithi153 ceph-mon[70044]: pgmap v18978: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:00.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:01:00 smithi083 ceph-mon[79930]: pgmap v18979: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:00.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:01:00 smithi083 ceph-mon[77242]: pgmap v18979: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:00.826 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:01:00 smithi153 ceph-mon[70044]: pgmap v18979: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:01.842 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:01:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:01:01] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T16:01:02.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:01:02 smithi083 ceph-mon[77242]: pgmap v18980: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:02.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:01:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:01:02.549Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:01:02.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:01:02 smithi083 ceph-mon[79930]: pgmap v18980: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:02.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:01:02 smithi153 ceph-mon[70044]: pgmap v18980: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:01:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:01:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:01:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:01:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:01:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:01:04.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:01:04 smithi083 ceph-mon[79930]: pgmap v18981: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:04.467 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:01:04 smithi083 ceph-mon[77242]: pgmap v18981: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:04.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:01:04 smithi153 ceph-mon[70044]: pgmap v18981: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:06.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:01:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:01:05.890Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:01:06.649 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:01:06 smithi083 ceph-mon[79930]: pgmap v18982: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:06.650 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:01:06 smithi083 ceph-mon[77242]: pgmap v18982: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:06.816 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:01:06 smithi153 ceph-mon[70044]: pgmap v18982: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:06.966 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:01:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:01:06.642Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:01:08.720 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:01:08 smithi083 ceph-mon[79930]: pgmap v18983: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:08.721 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:01:08 smithi083 ceph-mon[77242]: pgmap v18983: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:08.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:01:08 smithi153 ceph-mon[70044]: pgmap v18983: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:10.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:01:10 smithi083 ceph-mon[77242]: pgmap v18984: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:10.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:01:10 smithi083 ceph-mon[79930]: pgmap v18984: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:10.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:01:10 smithi153 ceph-mon[70044]: pgmap v18984: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:11.967 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:01:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:01:11] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T16:01:12.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:01:12 smithi083 ceph-mon[77242]: pgmap v18985: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:12.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:01:12 smithi083 ceph-mon[79930]: pgmap v18985: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:12.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:01:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:01:12.549Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:01:12.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:01:12 smithi153 ceph-mon[70044]: pgmap v18985: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:01:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:01:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:01:13.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:01:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:01:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:01:14.462 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:01:14.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:01:14 smithi083 ceph-mon[79930]: pgmap v18986: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:14.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:01:14 smithi083 ceph-mon[77242]: pgmap v18986: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:14.561 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:01:14 smithi153 ceph-mon[70044]: pgmap v18986: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:14.743 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:01:14.743 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 45s ago 10h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:01:14.744 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 8m ago 10h 63.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:01:14.744 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 45s ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:01:14.745 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 8m ago 10h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:01:14.745 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 8m ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:01:14.745 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 45s ago 10h 1052M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:01:14.746 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 45s ago 10h 2300M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:01:14.746 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 8m ago 10h 989M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:01:14.747 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 45s ago 10h 1384M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:01:14.747 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 45s ago 10h 24.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:01:14.748 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 8m ago 10h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:01:14.748 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 45s ago 10h 1673M 2105M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:01:14.749 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 45s ago 10h 1658M 2105M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:01:14.749 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 45s ago 10h 1288M 2105M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:01:14.750 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 45s ago 10h 1626M 2105M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:01:14.750 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 8m ago 10h 1402M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:01:14.751 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 8m ago 10h 1763M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:01:14.751 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 8m ago 10h 1575M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:01:14.751 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 8m ago 10h 1874M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:01:14.752 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 8m ago 10h 182M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:01:15.100 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:01:15.101 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:01:15.101 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:01:15.102 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:01:15.102 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:01:15.102 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:01:15.102 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:01:15.103 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:01:15.103 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:01:15.104 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:01:15.104 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:01:15.104 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:01:15.106 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:01:15.106 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:01:15.106 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:01:15.107 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:01:15.107 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:01:15.107 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:01:15.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:01:15 smithi083 ceph-mon[79930]: from='client.56208 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:01:15.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:01:15 smithi083 ceph-mon[79930]: from='client.56214 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:01:15.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:01:15 smithi083 ceph-mon[77242]: from='client.56208 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:01:15.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:01:15 smithi083 ceph-mon[77242]: from='client.56214 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:01:15.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:01:15 smithi153 ceph-mon[70044]: from='client.56208 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:01:15.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:01:15 smithi153 ceph-mon[70044]: from='client.56214 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:01:16.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:01:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:01:15.889Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:01:16.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:01:16 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/975257017' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:01:16.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:01:16 smithi083 ceph-mon[77242]: pgmap v18987: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:16.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:01:16 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/975257017' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:01:16.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:01:16 smithi083 ceph-mon[79930]: pgmap v18987: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:16.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:01:16 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/975257017' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:01:16.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:01:16 smithi153 ceph-mon[70044]: pgmap v18987: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:16.972 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:01:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:01:16.643Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:01:18.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:01:18 smithi083 ceph-mon[79930]: pgmap v18988: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:18.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:01:18 smithi083 ceph-mon[77242]: pgmap v18988: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:18.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:01:18 smithi153 ceph-mon[70044]: pgmap v18988: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:20.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:01:20 smithi083 ceph-mon[77242]: pgmap v18989: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:20.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:01:20 smithi083 ceph-mon[79930]: pgmap v18989: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:20.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:01:20 smithi153 ceph-mon[70044]: pgmap v18989: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:21.968 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:01:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:01:21] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T16:01:22.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:01:22 smithi083 ceph-mon[79930]: pgmap v18990: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:22.716 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:01:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:01:22.549Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:01:22.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:01:22 smithi083 ceph-mon[77242]: pgmap v18990: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:22.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:01:22 smithi153 ceph-mon[70044]: pgmap v18990: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:22.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:01:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:01:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:01:23.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:01:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:01:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:01:24.467 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:01:24 smithi083 ceph-mon[79930]: pgmap v18991: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:24.468 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:01:24 smithi083 ceph-mon[77242]: pgmap v18991: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:24.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:01:24 smithi153 ceph-mon[70044]: pgmap v18991: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:26.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:01:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:01:25.891Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:01:26.644 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:01:26 smithi083 ceph-mon[79930]: pgmap v18992: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:26.644 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:01:26 smithi083 ceph-mon[77242]: pgmap v18992: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:26.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:01:26 smithi153 ceph-mon[70044]: pgmap v18992: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:26.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:01:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:01:26.643Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:01:28.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:01:28 smithi083 ceph-mon[79930]: pgmap v18993: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:28.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:01:28 smithi083 ceph-mon[77242]: pgmap v18993: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:28.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:01:28 smithi153 ceph-mon[70044]: pgmap v18993: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:30.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:01:30 smithi083 ceph-mon[77242]: pgmap v18994: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:30.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:01:30 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:01:30.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:01:30 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:01:30.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:01:30 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:01:30.718 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:01:30 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:01:30.719 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:01:30 smithi083 ceph-mon[79930]: pgmap v18994: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:30.719 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:01:30 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:01:30.720 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:01:30 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:01:30.720 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:01:30 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:01:30.721 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:01:30 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:01:30.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:01:30 smithi153 ceph-mon[70044]: pgmap v18994: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:30.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:01:30 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:01:30.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:01:30 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:01:30.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:01:30 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:01:30.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:01:30 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:01:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:01:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:01:31] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T16:01:32.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:01:32 smithi083 ceph-mon[79930]: pgmap v18995: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:32.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:01:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:01:32.551Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:01:32.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:01:32 smithi083 ceph-mon[77242]: pgmap v18995: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:32.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:01:32 smithi153 ceph-mon[70044]: pgmap v18995: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:01:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:01:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:01:33.318 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:01:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:01:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:01:34.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:01:34 smithi083 ceph-mon[79930]: pgmap v18996: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:34.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:01:34 smithi083 ceph-mon[77242]: pgmap v18996: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:34.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:01:34 smithi153 ceph-mon[70044]: pgmap v18996: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:36.218 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:01:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:01:35.891Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:01:36.646 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:01:36 smithi083 ceph-mon[79930]: pgmap v18997: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:36.647 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:01:36 smithi083 ceph-mon[77242]: pgmap v18997: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:36.814 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:01:36 smithi153 ceph-mon[70044]: pgmap v18997: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:36.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:01:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:01:36.644Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:01:38.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:01:38 smithi083 ceph-mon[77242]: pgmap v18998: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:38.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:01:38 smithi083 ceph-mon[79930]: pgmap v18998: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:38.817 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:01:38 smithi153 ceph-mon[70044]: pgmap v18998: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:40.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:01:40 smithi083 ceph-mon[79930]: pgmap v18999: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:01:40 smithi083 ceph-mon[77242]: pgmap v18999: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:01:40 smithi153 ceph-mon[70044]: pgmap v18999: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:01:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:01:41] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T16:01:42.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:01:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:01:42.552Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:01:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:01:42 smithi083 ceph-mon[77242]: pgmap v19000: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:01:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:01:42.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:01:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:01:42.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:01:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:01:42.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:01:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:01:42.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:01:42 smithi083 ceph-mon[79930]: pgmap v19000: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:42.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:01:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:01:42.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:01:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:01:42.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:01:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:01:42.719 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:01:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:01:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:01:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:01:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:01:42.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:01:42 smithi153 ceph-mon[70044]: pgmap v19000: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:42.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:01:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:01:42.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:01:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:01:42.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:01:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:01:42.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:01:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:01:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:01:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:01:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:01:44.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:01:44 smithi083 ceph-mon[79930]: pgmap v19001: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:44.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:01:44 smithi083 ceph-mon[77242]: pgmap v19001: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:44.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:01:44 smithi153 ceph-mon[70044]: pgmap v19001: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:45.432 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:01:45.719 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:01:45.720 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 76s ago 10h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:01:45.720 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 9m ago 10h 63.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:01:45.720 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 76s ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:01:45.721 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 9m ago 10h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:01:45.721 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 9m ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:01:45.721 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 76s ago 10h 1052M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:01:45.721 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 76s ago 10h 2300M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:01:45.722 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 9m ago 10h 989M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:01:45.722 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 76s ago 10h 1384M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:01:45.722 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 76s ago 10h 24.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:01:45.723 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 9m ago 10h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:01:45.724 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 76s ago 10h 1673M 2105M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:01:45.724 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 76s ago 10h 1658M 2105M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:01:45.724 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 76s ago 10h 1288M 2105M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:01:45.724 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 76s ago 10h 1626M 2105M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:01:45.725 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 9m ago 10h 1402M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:01:45.725 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 9m ago 10h 1763M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:01:45.725 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 9m ago 10h 1575M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:01:45.729 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 9m ago 10h 1874M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:01:45.729 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 9m ago 10h 182M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:01:46.095 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:01:46.096 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:01:46.102 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:01:46.108 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:01:46.109 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:01:46.109 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:01:46.110 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:01:46.111 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:01:46.111 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:01:46.111 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:01:46.112 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:01:46.112 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:01:46.112 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:01:46.113 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:01:46.113 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:01:46.114 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:01:46.114 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:01:46.115 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:01:46.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:01:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:01:45.892Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:01:46.644 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:01:46 smithi083 ceph-mon[79930]: pgmap v19002: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:46.645 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:01:46 smithi083 ceph-mon[79930]: from='client.66157 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:01:46.645 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:01:46 smithi083 ceph-mon[79930]: from='client.66163 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:01:46.645 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:01:46 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3553917050' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:01:46.646 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:01:46 smithi083 ceph-mon[77242]: pgmap v19002: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:46.647 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:01:46 smithi083 ceph-mon[77242]: from='client.66157 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:01:46.647 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:01:46 smithi083 ceph-mon[77242]: from='client.66163 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:01:46.647 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:01:46 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3553917050' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:01:46.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:01:46 smithi153 ceph-mon[70044]: pgmap v19002: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:46.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:01:46 smithi153 ceph-mon[70044]: from='client.66157 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:01:46.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:01:46 smithi153 ceph-mon[70044]: from='client.66163 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:01:46.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:01:46 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3553917050' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:01:46.970 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:01:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:01:46.643Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:01:48.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:01:48 smithi083 ceph-mon[79930]: pgmap v19003: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:48.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:01:48 smithi083 ceph-mon[77242]: pgmap v19003: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:48.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:01:48 smithi153 ceph-mon[70044]: pgmap v19003: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:50.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:01:50 smithi083 ceph-mon[77242]: pgmap v19004: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:50.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:01:50 smithi083 ceph-mon[79930]: pgmap v19004: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:50.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:01:50 smithi153 ceph-mon[70044]: pgmap v19004: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:01:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:01:51] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T16:01:52.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:01:52 smithi083 ceph-mon[79930]: pgmap v19005: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:52.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:01:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:01:52.552Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:01:52.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:01:52 smithi083 ceph-mon[77242]: pgmap v19005: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:52.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:01:52 smithi153 ceph-mon[70044]: pgmap v19005: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:01:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:01:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:01:53.314 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:01:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:01:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:01:54.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:01:54 smithi083 ceph-mon[79930]: pgmap v19006: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:54.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:01:54 smithi083 ceph-mon[77242]: pgmap v19006: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:54.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:01:54 smithi153 ceph-mon[70044]: pgmap v19006: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:56.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:01:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:01:55.893Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:01:56.646 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:01:56 smithi083 ceph-mon[79930]: pgmap v19007: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:56.647 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:01:56 smithi083 ceph-mon[77242]: pgmap v19007: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:56.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:01:56 smithi153 ceph-mon[70044]: pgmap v19007: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:56.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:01:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:01:56.645Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:01:58.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:01:58 smithi083 ceph-mon[77242]: pgmap v19008: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:58.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:01:58 smithi083 ceph-mon[79930]: pgmap v19008: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:01:58.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:01:58 smithi153 ceph-mon[70044]: pgmap v19008: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:00.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:02:00 smithi083 ceph-mon[79930]: pgmap v19009: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:00.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:02:00 smithi083 ceph-mon[77242]: pgmap v19009: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:00.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:02:00 smithi153 ceph-mon[70044]: pgmap v19009: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:02:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:02:01] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T16:02:02.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:02:02 smithi083 ceph-mon[77242]: pgmap v19010: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:02.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:02:02 smithi083 ceph-mon[79930]: pgmap v19010: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:02.718 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:02:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:02:02.553Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:02:02.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:02:02 smithi153 ceph-mon[70044]: pgmap v19010: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:02.813 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:02:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:02:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:02:03.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:02:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:02:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:02:04.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:02:04 smithi083 ceph-mon[77242]: pgmap v19011: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:04.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:02:04 smithi083 ceph-mon[79930]: pgmap v19011: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:04.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:02:04 smithi153 ceph-mon[70044]: pgmap v19011: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:06.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:02:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:02:05.894Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:02:06.649 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:02:06 smithi083 ceph-mon[79930]: pgmap v19012: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:06.650 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:02:06 smithi083 ceph-mon[77242]: pgmap v19012: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:06.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:02:06 smithi153 ceph-mon[70044]: pgmap v19012: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:06.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:02:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:02:06.646Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:02:08.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:02:08 smithi083 ceph-mon[77242]: pgmap v19013: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:08.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:02:08 smithi083 ceph-mon[79930]: pgmap v19013: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:08.818 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:02:08 smithi153 ceph-mon[70044]: pgmap v19013: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:10.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:02:10 smithi083 ceph-mon[79930]: pgmap v19014: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:10.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:02:10 smithi083 ceph-mon[77242]: pgmap v19014: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:10.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:02:10 smithi153 ceph-mon[70044]: pgmap v19014: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:02:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:02:11] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T16:02:12.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:02:12 smithi083 ceph-mon[79930]: pgmap v19015: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:12.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:02:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:02:12.553Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:02:12.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:02:12 smithi083 ceph-mon[77242]: pgmap v19015: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:12.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:02:12 smithi153 ceph-mon[70044]: pgmap v19015: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:02:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:02:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:02:13.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:02:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:02:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:02:14.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:02:14 smithi083 ceph-mon[79930]: pgmap v19016: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:14.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:02:14 smithi083 ceph-mon[77242]: pgmap v19016: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:14.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:02:14 smithi153 ceph-mon[70044]: pgmap v19016: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:16.210 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:02:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:02:15.894Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:02:16.404 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:02:16.653 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:02:16 smithi083 ceph-mon[79930]: pgmap v19017: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:16.653 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:02:16 smithi083 ceph-mon[77242]: pgmap v19017: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:16.698 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:02:16.698 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 107s ago 10h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:02:16.699 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 10m ago 10h 63.8M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:02:16.699 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 107s ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:02:16.699 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 10m ago 10h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:02:16.699 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 10m ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:02:16.700 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 107s ago 10h 1052M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:02:16.700 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 107s ago 10h 2300M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:02:16.700 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 10m ago 10h 989M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:02:16.700 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 107s ago 10h 1384M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:02:16.701 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 107s ago 10h 24.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:02:16.701 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 10m ago 10h 26.2M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:02:16.701 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 107s ago 10h 1673M 2105M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:02:16.701 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 107s ago 10h 1658M 2105M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:02:16.701 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 107s ago 10h 1288M 2105M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:02:16.710 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 107s ago 10h 1626M 2105M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:02:16.711 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 10m ago 10h 1402M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:02:16.711 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 10m ago 10h 1763M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:02:16.712 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 10m ago 10h 1575M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:02:16.712 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 10m ago 10h 1874M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:02:16.713 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 10m ago 10h 182M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:02:16.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:02:16 smithi153 ceph-mon[70044]: pgmap v19017: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:16.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:02:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:02:16.647Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:02:17.064 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:02:17.064 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:02:17.065 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:02:17.065 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:02:17.066 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:02:17.068 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:02:17.068 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:02:17.069 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:02:17.069 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:02:17.069 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:02:17.070 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:02:17.070 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:02:17.071 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:02:17.097 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:02:17.097 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:02:17.097 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:02:17.098 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:02:17.098 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:02:17.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:02:17 smithi083 ceph-mon[79930]: from='client.56244 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:02:17.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:02:17 smithi083 ceph-mon[79930]: from='client.66181 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:02:17.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:02:17 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/318434392' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:02:17.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:02:17 smithi083 ceph-mon[77242]: from='client.56244 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:02:17.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:02:17 smithi083 ceph-mon[77242]: from='client.66181 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:02:17.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:02:17 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/318434392' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:02:17.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:02:17 smithi153 ceph-mon[70044]: from='client.56244 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:02:17.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:02:17 smithi153 ceph-mon[70044]: from='client.66181 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:02:17.813 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:02:17 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/318434392' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:02:18.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:02:18 smithi083 ceph-mon[77242]: pgmap v19018: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:18.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:02:18 smithi083 ceph-mon[79930]: pgmap v19018: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:18.814 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:02:18 smithi153 ceph-mon[70044]: pgmap v19018: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:20.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:02:20 smithi083 ceph-mon[79930]: pgmap v19019: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:20.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:02:20 smithi083 ceph-mon[77242]: pgmap v19019: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:20.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:02:20 smithi153 ceph-mon[70044]: pgmap v19019: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:21.966 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:02:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:02:21] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T16:02:22.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:02:22 smithi083 ceph-mon[79930]: pgmap v19020: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:22.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:02:22 smithi083 ceph-mon[77242]: pgmap v19020: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:22.717 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:02:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:02:22.554Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:02:22.814 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:02:22 smithi153 ceph-mon[70044]: pgmap v19020: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:22.815 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:02:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:02:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:02:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:02:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:02:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:02:24.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:02:24 smithi083 ceph-mon[79930]: pgmap v19021: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:24.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:02:24 smithi083 ceph-mon[77242]: pgmap v19021: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:02:24 smithi153 ceph-mon[70044]: pgmap v19021: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:26.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:02:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:02:25.894Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:02:26.656 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:02:26 smithi083 ceph-mon[79930]: pgmap v19022: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:26.657 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:02:26 smithi083 ceph-mon[77242]: pgmap v19022: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:26.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:02:26 smithi153 ceph-mon[70044]: pgmap v19022: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:26.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:02:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:02:26.647Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:02:28.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:02:28 smithi083 ceph-mon[79930]: pgmap v19023: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:28.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:02:28 smithi083 ceph-mon[77242]: pgmap v19023: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:28.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:02:28 smithi153 ceph-mon[70044]: pgmap v19023: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:30.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:02:30 smithi083 ceph-mon[79930]: pgmap v19024: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:30.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:02:30 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:02:30.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:02:30 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:02:30.718 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:02:30 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:02:30.718 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:02:30 smithi083 ceph-mon[77242]: pgmap v19024: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:30.719 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:02:30 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:02:30.719 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:02:30 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:02:30.720 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:02:30 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:02:30.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:02:30 smithi153 ceph-mon[70044]: pgmap v19024: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:30.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:02:30 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:02:30.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:02:30 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:02:30.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:02:30 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:02:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:02:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:02:31] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T16:02:32.727 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:02:32 smithi083 ceph-mon[79930]: pgmap v19025: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:32.728 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:02:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:02:32.554Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:02:32.728 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:02:32 smithi083 ceph-mon[77242]: pgmap v19025: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:32.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:02:32 smithi153 ceph-mon[70044]: pgmap v19025: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:02:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:02:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:02:33.316 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:02:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:02:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:02:34.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:02:34 smithi083 ceph-mon[77242]: pgmap v19026: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:34.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:02:34 smithi083 ceph-mon[79930]: pgmap v19026: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:34.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:02:34 smithi153 ceph-mon[70044]: pgmap v19026: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:36.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:02:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:02:35.894Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:02:36.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:02:36 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:02:36.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:02:36 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:02:36.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:02:36 smithi083 ceph-mon[77242]: pgmap v19027: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:36.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:02:36 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:02:36.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:02:36 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:02:36.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:02:36 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:02:36.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:02:36 smithi083 ceph-mon[79930]: pgmap v19027: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:36.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:02:36 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:02:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:02:36 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:02:36.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:02:36 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:02:36.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:02:36 smithi153 ceph-mon[70044]: pgmap v19027: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:36.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:02:36 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:02:36.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:02:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:02:36.648Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:02:38.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:02:38 smithi083 ceph-mon[77242]: pgmap v19028: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:38.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:02:38 smithi083 ceph-mon[79930]: pgmap v19028: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:38.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:02:38 smithi153 ceph-mon[70044]: pgmap v19028: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:40.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:02:40 smithi083 ceph-mon[77242]: pgmap v19029: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:40.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:02:40 smithi083 ceph-mon[79930]: pgmap v19029: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:40.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:02:40 smithi153 ceph-mon[70044]: pgmap v19029: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:02:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:02:41] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T16:02:42.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:02:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:02:42.555Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:02:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:02:42 smithi083 ceph-mon[77242]: pgmap v19030: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:02:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:02:42.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:02:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:02:42.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:02:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:02:42.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:02:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:02:42.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:02:42 smithi083 ceph-mon[79930]: pgmap v19030: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:42.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:02:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:02:42.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:02:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:02:42.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:02:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:02:42.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:02:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:02:42.821 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:02:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:02:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:02:42.822 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:02:42 smithi153 ceph-mon[70044]: pgmap v19030: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:42.822 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:02:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:02:42.823 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:02:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:02:42.823 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:02:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:02:42.824 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:02:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:02:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:02:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:02:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:02:44.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:02:44 smithi083 ceph-mon[79930]: pgmap v19031: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:44.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:02:44 smithi083 ceph-mon[77242]: pgmap v19031: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:44.571 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:02:44 smithi153 ceph-mon[70044]: pgmap v19031: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:46.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:02:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:02:45.896Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:02:46.649 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:02:46 smithi083 ceph-mon[79930]: pgmap v19032: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:46.650 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:02:46 smithi083 ceph-mon[77242]: pgmap v19032: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:46.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:02:46 smithi153 ceph-mon[70044]: pgmap v19032: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:46.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:02:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:02:46.648Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:02:47.385 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:02:47.675 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:02:47.676 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 2m ago 10h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:02:47.676 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 12s ago 10h 64.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:02:47.676 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 2m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:02:47.677 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 12s ago 10h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:02:47.677 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 12s ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:02:47.678 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 2m ago 10h 1052M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:02:47.678 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 2m ago 10h 2300M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:02:47.679 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 12s ago 10h 1030M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:02:47.679 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 2m ago 10h 1384M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:02:47.680 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 2m ago 10h 24.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:02:47.680 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 12s ago 10h 26.7M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:02:47.681 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 2m ago 10h 1673M 2105M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:02:47.681 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 2m ago 10h 1658M 2105M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:02:47.682 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 2m ago 10h 1288M 2105M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:02:47.683 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 2m ago 10h 1626M 2105M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:02:47.683 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 12s ago 10h 1425M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:02:47.684 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 12s ago 10h 1790M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:02:47.685 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 12s ago 10h 1600M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:02:47.686 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 12s ago 10h 1904M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:02:47.686 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 12s ago 10h 185M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:02:48.034 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:02:48.035 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:02:48.035 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:02:48.036 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:02:48.036 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:02:48.037 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:02:48.037 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:02:48.037 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:02:48.038 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:02:48.038 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:02:48.038 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:02:48.039 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:02:48.039 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:02:48.040 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:02:48.040 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:02:48.040 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:02:48.041 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:02:48.041 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:02:48.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:02:48 smithi083 ceph-mon[79930]: pgmap v19033: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:48.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:02:48 smithi083 ceph-mon[79930]: from='client.66193 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:02:48.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:02:48 smithi083 ceph-mon[79930]: from='client.56268 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:02:48.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:02:48 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/541244891' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:02:48.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:02:48 smithi083 ceph-mon[77242]: pgmap v19033: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:48.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:02:48 smithi083 ceph-mon[77242]: from='client.66193 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:02:48.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:02:48 smithi083 ceph-mon[77242]: from='client.56268 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:02:48.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:02:48 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/541244891' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:02:48.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:02:48 smithi153 ceph-mon[70044]: pgmap v19033: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:48.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:02:48 smithi153 ceph-mon[70044]: from='client.66193 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:02:48.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:02:48 smithi153 ceph-mon[70044]: from='client.56268 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:02:48.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:02:48 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/541244891' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:02:50.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:02:50 smithi083 ceph-mon[79930]: pgmap v19034: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:50.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:02:50 smithi083 ceph-mon[77242]: pgmap v19034: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:50.817 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:02:50 smithi153 ceph-mon[70044]: pgmap v19034: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:02:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:02:51] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T16:02:52.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:02:52 smithi083 ceph-mon[79930]: pgmap v19035: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:52.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:02:52 smithi083 ceph-mon[77242]: pgmap v19035: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:52.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:02:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:02:52.555Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:02:52.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:02:52 smithi153 ceph-mon[70044]: pgmap v19035: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:02:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:02:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:02:53.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:02:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:02:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:02:54.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:02:54 smithi083 ceph-mon[79930]: pgmap v19036: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:54.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:02:54 smithi083 ceph-mon[77242]: pgmap v19036: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:54.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:02:54 smithi153 ceph-mon[70044]: pgmap v19036: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:56.210 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:02:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:02:55.897Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:02:56.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:02:56 smithi083 ceph-mon[79930]: pgmap v19037: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:56.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:02:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:02:56.649Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:02:56.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:02:56 smithi083 ceph-mon[77242]: pgmap v19037: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:56.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:02:56 smithi153 ceph-mon[70044]: pgmap v19037: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:58.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:02:58 smithi083 ceph-mon[79930]: pgmap v19038: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:58.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:02:58 smithi083 ceph-mon[77242]: pgmap v19038: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:02:58.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:02:58 smithi153 ceph-mon[70044]: pgmap v19038: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:00.723 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:03:00 smithi083 ceph-mon[79930]: pgmap v19039: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:00.723 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:03:00 smithi083 ceph-mon[77242]: pgmap v19039: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:00.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:03:00 smithi153 ceph-mon[70044]: pgmap v19039: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:01.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:03:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:03:01] "GET /metrics HTTP/1.1" 200 225246 "" "Prometheus/2.33.4" 2022-09-29T16:03:02.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:03:02 smithi083 ceph-mon[79930]: pgmap v19040: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:02.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:03:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:03:02.556Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:03:02.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:03:02 smithi083 ceph-mon[77242]: pgmap v19040: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:02.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:03:02 smithi153 ceph-mon[70044]: pgmap v19040: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:03:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:03:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:03:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:03:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:03:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:03:04.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:03:04 smithi083 ceph-mon[79930]: pgmap v19041: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:04.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:03:04 smithi083 ceph-mon[77242]: pgmap v19041: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:04.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:03:04 smithi153 ceph-mon[70044]: pgmap v19041: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:06.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:03:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:03:05.898Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:03:06.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:03:06 smithi083 ceph-mon[79930]: pgmap v19042: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:06.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:03:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:03:06.649Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:03:06.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:03:06 smithi083 ceph-mon[77242]: pgmap v19042: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:06.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:03:06 smithi153 ceph-mon[70044]: pgmap v19042: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:08.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:03:08 smithi083 ceph-mon[79930]: pgmap v19043: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:08.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:03:08 smithi083 ceph-mon[77242]: pgmap v19043: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:08.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:03:08 smithi153 ceph-mon[70044]: pgmap v19043: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:10.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:03:10 smithi083 ceph-mon[79930]: pgmap v19044: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:10.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:03:10 smithi083 ceph-mon[77242]: pgmap v19044: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:10.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:03:10 smithi153 ceph-mon[70044]: pgmap v19044: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:03:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:03:11] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T16:03:12.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:03:12 smithi083 ceph-mon[79930]: pgmap v19045: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:12.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:03:12 smithi083 ceph-mon[77242]: pgmap v19045: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:12.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:03:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:03:12.557Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:03:12.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:03:12 smithi153 ceph-mon[70044]: pgmap v19045: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:03:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:03:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:03:13.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:03:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:03:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:03:14.399 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:03:14 smithi083 ceph-mon[77242]: pgmap v19046: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:14.400 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:03:14 smithi083 ceph-mon[79930]: pgmap v19046: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:14.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:03:14 smithi153 ceph-mon[70044]: pgmap v19046: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:16.219 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:03:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:03:15.898Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:03:16.650 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:03:16 smithi083 ceph-mon[79930]: pgmap v19047: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:16.651 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:03:16 smithi083 ceph-mon[77242]: pgmap v19047: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:16.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:03:16 smithi153 ceph-mon[70044]: pgmap v19047: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:16.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:03:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:03:16.649Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:03:18.362 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:03:18.650 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:03:18.651 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 2m ago 10h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:03:18.651 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 43s ago 10h 64.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:03:18.651 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 2m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:03:18.651 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 43s ago 10h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:03:18.652 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 43s ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:03:18.652 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 2m ago 10h 1052M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:03:18.652 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 2m ago 10h 2300M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:03:18.652 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 43s ago 10h 1030M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:03:18.653 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 2m ago 10h 1384M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:03:18.653 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 2m ago 10h 24.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:03:18.653 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 43s ago 10h 26.7M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:03:18.654 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 2m ago 10h 1673M 2105M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:03:18.654 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 2m ago 10h 1658M 2105M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:03:18.654 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 2m ago 10h 1288M 2105M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:03:18.654 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 2m ago 10h 1626M 2105M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:03:18.655 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 43s ago 10h 1425M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:03:18.655 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 43s ago 10h 1790M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:03:18.655 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 43s ago 10h 1600M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:03:18.655 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 43s ago 10h 1904M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:03:18.656 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 43s ago 10h 185M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:03:18.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:03:18 smithi083 ceph-mon[79930]: pgmap v19048: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:18.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:03:18 smithi083 ceph-mon[77242]: pgmap v19048: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:18.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:03:18 smithi153 ceph-mon[70044]: pgmap v19048: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:19.005 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:03:19.005 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:03:19.005 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:03:19.006 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:03:19.006 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:03:19.006 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:03:19.006 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:03:19.007 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:03:19.007 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:03:19.007 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:03:19.007 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:03:19.008 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:03:19.008 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:03:19.008 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:03:19.008 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:03:19.009 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:03:19.009 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:03:19.009 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:03:19.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:03:19 smithi083 ceph-mon[79930]: from='client.56280 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:03:19.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:03:19 smithi083 ceph-mon[79930]: from='client.56286 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:03:19.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:03:19 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1565561404' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:03:19.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:03:19 smithi083 ceph-mon[77242]: from='client.56280 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:03:19.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:03:19 smithi083 ceph-mon[77242]: from='client.56286 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:03:19.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:03:19 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1565561404' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:03:19.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:03:19 smithi153 ceph-mon[70044]: from='client.56280 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:03:19.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:03:19 smithi153 ceph-mon[70044]: from='client.56286 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:03:19.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:03:19 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1565561404' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:03:20.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:03:20 smithi083 ceph-mon[79930]: pgmap v19049: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:20.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:03:20 smithi083 ceph-mon[77242]: pgmap v19049: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:20.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:03:20 smithi153 ceph-mon[70044]: pgmap v19049: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:03:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:03:21] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T16:03:22.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:03:22 smithi083 ceph-mon[79930]: pgmap v19050: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:22.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:03:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:03:22.558Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:03:22.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:03:22 smithi083 ceph-mon[77242]: pgmap v19050: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:22.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:03:22 smithi153 ceph-mon[70044]: pgmap v19050: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:03:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:03:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:03:23.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:03:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:03:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:03:24.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:03:24 smithi083 ceph-mon[77242]: pgmap v19051: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:24.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:03:24 smithi083 ceph-mon[79930]: pgmap v19051: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:24.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:03:24 smithi153 ceph-mon[70044]: pgmap v19051: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:26.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:03:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:03:25.899Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:03:26.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:03:26 smithi083 ceph-mon[77242]: pgmap v19052: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:26.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:03:26 smithi083 ceph-mon[79930]: pgmap v19052: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:26.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:03:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:03:26.650Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:03:26.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:03:26 smithi153 ceph-mon[70044]: pgmap v19052: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:28.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:03:28 smithi083 ceph-mon[79930]: pgmap v19053: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:28.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:03:28 smithi083 ceph-mon[77242]: pgmap v19053: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:28.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:03:28 smithi153 ceph-mon[70044]: pgmap v19053: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:30.718 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:03:30 smithi083 ceph-mon[79930]: pgmap v19054: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:30.718 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:03:30 smithi083 ceph-mon[77242]: pgmap v19054: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:30.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:03:30 smithi153 ceph-mon[70044]: pgmap v19054: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:31.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:03:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:03:31] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T16:03:32.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:03:32 smithi083 ceph-mon[77242]: pgmap v19055: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:32.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:03:32 smithi083 ceph-mon[79930]: pgmap v19055: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:32.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:03:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:03:32.558Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:03:32.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:03:32 smithi153 ceph-mon[70044]: pgmap v19055: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:03:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:03:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:03:33.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:03:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:03:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:03:34.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:03:34 smithi083 ceph-mon[79930]: pgmap v19056: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:34.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:03:34 smithi083 ceph-mon[77242]: pgmap v19056: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:34.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:03:34 smithi153 ceph-mon[70044]: pgmap v19056: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:36.210 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:03:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:03:35.899Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:03:36.710 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:03:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:03:36.650Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:03:36.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:03:36 smithi083 ceph-mon[79930]: pgmap v19057: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:36.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:03:36 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:03:36.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:03:36 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:03:36.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:03:36 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:03:36.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:03:36 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:03:36.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:03:36 smithi083 ceph-mon[77242]: pgmap v19057: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:36.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:03:36 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:03:36.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:03:36 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:03:36.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:03:36 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:03:36.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:03:36 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:03:36.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:03:36 smithi153 ceph-mon[70044]: pgmap v19057: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:36.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:03:36 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:03:36.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:03:36 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:03:36.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:03:36 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:03:36.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:03:36 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:03:38.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:03:38 smithi083 ceph-mon[77242]: pgmap v19058: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:38.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:03:38 smithi083 ceph-mon[79930]: pgmap v19058: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:38.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:03:38 smithi153 ceph-mon[70044]: pgmap v19058: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:40.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:03:40 smithi083 ceph-mon[77242]: pgmap v19059: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:40.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:03:40 smithi083 ceph-mon[79930]: pgmap v19059: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:03:40 smithi153 ceph-mon[70044]: pgmap v19059: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:03:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:03:41] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T16:03:42.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:03:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:03:42.560Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:03:42.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:03:42 smithi083 ceph-mon[77242]: pgmap v19060: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:42.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:03:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:03:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:03:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:03:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:03:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:03:42.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:03:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:03:42.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:03:42 smithi083 ceph-mon[79930]: pgmap v19060: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:42.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:03:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:03:42.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:03:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:03:42.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:03:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:03:42.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:03:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:03:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:03:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:03:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:03:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:03:42 smithi153 ceph-mon[70044]: pgmap v19060: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:03:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:03:42.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:03:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:03:42.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:03:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:03:42.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:03:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:03:43.310 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:03:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:03:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:03:44.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:03:44 smithi083 ceph-mon[77242]: pgmap v19061: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:44.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:03:44 smithi083 ceph-mon[79930]: pgmap v19061: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:44.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:03:44 smithi153 ceph-mon[70044]: pgmap v19061: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:46.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:03:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:03:45.900Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:03:46.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:03:46 smithi083 ceph-mon[79930]: pgmap v19062: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:46.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:03:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:03:46.652Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:03:46.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:03:46 smithi083 ceph-mon[77242]: pgmap v19062: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:46.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:03:46 smithi153 ceph-mon[70044]: pgmap v19062: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:48.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:03:48 smithi083 ceph-mon[79930]: pgmap v19063: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:48.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:03:48 smithi083 ceph-mon[77242]: pgmap v19063: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:48.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:03:48 smithi153 ceph-mon[70044]: pgmap v19063: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:49.332 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:03:49.622 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:03:49.622 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 3m ago 10h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:03:49.622 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 74s ago 10h 64.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:03:49.623 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 3m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:03:49.623 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 74s ago 10h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:03:49.623 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 74s ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:03:49.623 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 3m ago 10h 1052M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:03:49.624 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 3m ago 10h 2300M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:03:49.624 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 74s ago 10h 1030M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:03:49.624 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 3m ago 10h 1384M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:03:49.624 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 3m ago 10h 24.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:03:49.624 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 74s ago 10h 26.7M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:03:49.625 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 3m ago 10h 1673M 2105M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:03:49.625 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 3m ago 10h 1658M 2105M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:03:49.625 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 3m ago 10h 1288M 2105M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:03:49.625 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 3m ago 10h 1626M 2105M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:03:49.626 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 74s ago 10h 1425M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:03:49.626 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 74s ago 10h 1790M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:03:49.626 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 74s ago 10h 1600M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:03:49.627 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 74s ago 10h 1904M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:03:49.627 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 74s ago 10h 185M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:03:49.977 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:03:49.977 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:03:49.978 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:03:49.978 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:03:49.978 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:03:49.978 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:03:49.979 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:03:49.979 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:03:49.979 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:03:49.979 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:03:49.980 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:03:49.982 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:03:49.982 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:03:49.983 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:03:49.983 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:03:49.983 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:03:49.984 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:03:49.984 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:03:50.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:03:50 smithi083 ceph-mon[79930]: from='client.56298 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:03:50.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:03:50 smithi083 ceph-mon[79930]: pgmap v19064: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:50.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:03:50 smithi083 ceph-mon[79930]: from='client.66235 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:03:50.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:03:50 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3929913160' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:03:50.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:03:50 smithi083 ceph-mon[77242]: from='client.56298 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:03:50.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:03:50 smithi083 ceph-mon[77242]: pgmap v19064: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:50.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:03:50 smithi083 ceph-mon[77242]: from='client.66235 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:03:50.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:03:50 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3929913160' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:03:50.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:03:50 smithi153 ceph-mon[70044]: from='client.56298 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:03:50.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:03:50 smithi153 ceph-mon[70044]: pgmap v19064: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:50.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:03:50 smithi153 ceph-mon[70044]: from='client.66235 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:03:50.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:03:50 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3929913160' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:03:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:03:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:03:51] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T16:03:52.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:03:52 smithi083 ceph-mon[77242]: pgmap v19065: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:52.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:03:52 smithi083 ceph-mon[79930]: pgmap v19065: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:52.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:03:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:03:52.561Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:03:52.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:03:52 smithi153 ceph-mon[70044]: pgmap v19065: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:03:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:03:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:03:53.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:03:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:03:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:03:54.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:03:54 smithi083 ceph-mon[79930]: pgmap v19066: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:54.467 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:03:54 smithi083 ceph-mon[77242]: pgmap v19066: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:54.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:03:54 smithi153 ceph-mon[70044]: pgmap v19066: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:56.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:03:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:03:55.900Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:03:56.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:03:56 smithi083 ceph-mon[79930]: pgmap v19067: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:56.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:03:56 smithi083 ceph-mon[77242]: pgmap v19067: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:56.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:03:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:03:56.653Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:03:56.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:03:56 smithi153 ceph-mon[70044]: pgmap v19067: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:58.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:03:58 smithi083 ceph-mon[79930]: pgmap v19068: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:58.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:03:58 smithi083 ceph-mon[77242]: pgmap v19068: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:03:58.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:03:58 smithi153 ceph-mon[70044]: pgmap v19068: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:00.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:04:00 smithi083 ceph-mon[79930]: pgmap v19069: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:00.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:04:00 smithi083 ceph-mon[77242]: pgmap v19069: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:00.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:04:00 smithi153 ceph-mon[70044]: pgmap v19069: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:04:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:04:01] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T16:04:02.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:04:02 smithi083 ceph-mon[79930]: pgmap v19070: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:02.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:04:02 smithi083 ceph-mon[77242]: pgmap v19070: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:02.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:04:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:04:02.561Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:04:02.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:04:02 smithi153 ceph-mon[70044]: pgmap v19070: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:02.811 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:04:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:04:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:04:03.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:04:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:04:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:04:04.460 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:04:04 smithi083 ceph-mon[77242]: pgmap v19071: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:04.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:04:04 smithi083 ceph-mon[79930]: pgmap v19071: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:04.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:04:04 smithi153 ceph-mon[70044]: pgmap v19071: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:06.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:04:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:04:05.901Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:04:06.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:04:06 smithi083 ceph-mon[79930]: pgmap v19072: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:06.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:04:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:04:06.653Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:04:06.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:04:06 smithi083 ceph-mon[77242]: pgmap v19072: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:06.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:04:06 smithi153 ceph-mon[70044]: pgmap v19072: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:08.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:04:08 smithi083 ceph-mon[77242]: pgmap v19073: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:08.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:04:08 smithi083 ceph-mon[79930]: pgmap v19073: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:08.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:04:08 smithi153 ceph-mon[70044]: pgmap v19073: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:10.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:04:10 smithi083 ceph-mon[79930]: pgmap v19074: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:10.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:04:10 smithi083 ceph-mon[77242]: pgmap v19074: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:10.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:04:10 smithi153 ceph-mon[70044]: pgmap v19074: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:04:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:04:11] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T16:04:12.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:04:12 smithi083 ceph-mon[79930]: pgmap v19075: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:12.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:04:12 smithi083 ceph-mon[77242]: pgmap v19075: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:12.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:04:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:04:12.561Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:04:12.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:04:12 smithi153 ceph-mon[70044]: pgmap v19075: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:04:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:04:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:04:13.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:04:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:04:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:04:14.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:04:14 smithi083 ceph-mon[79930]: pgmap v19076: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:14.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:04:14 smithi083 ceph-mon[77242]: pgmap v19076: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:14.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:04:14 smithi153 ceph-mon[70044]: pgmap v19076: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:16.210 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:04:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:04:15.901Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:04:16.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:04:16 smithi083 ceph-mon[77242]: pgmap v19077: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:16.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:04:16 smithi083 ceph-mon[79930]: pgmap v19077: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:16.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:04:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:04:16.654Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:04:16.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:04:16 smithi153 ceph-mon[70044]: pgmap v19077: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:18.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:04:18 smithi083 ceph-mon[79930]: pgmap v19078: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:18.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:04:18 smithi083 ceph-mon[77242]: pgmap v19078: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:18.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:04:18 smithi153 ceph-mon[70044]: pgmap v19078: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:20.302 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:04:20.592 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:04:20.593 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 3m ago 10h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:04:20.593 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 105s ago 10h 64.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:04:20.593 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 3m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:04:20.593 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 105s ago 10h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:04:20.594 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 105s ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:04:20.594 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 3m ago 10h 1052M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:04:20.594 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 3m ago 10h 2300M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:04:20.594 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 105s ago 10h 1030M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:04:20.595 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 3m ago 10h 1384M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:04:20.595 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 3m ago 10h 24.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:04:20.595 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 105s ago 10h 26.7M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:04:20.595 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 3m ago 10h 1673M 2105M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:04:20.596 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 3m ago 10h 1658M 2105M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:04:20.596 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 3m ago 10h 1288M 2105M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:04:20.596 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 3m ago 10h 1626M 2105M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:04:20.596 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 105s ago 10h 1425M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:04:20.597 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 105s ago 10h 1790M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:04:20.597 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 105s ago 10h 1600M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:04:20.597 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 105s ago 10h 1904M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:04:20.597 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 105s ago 10h 185M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:04:20.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:04:20 smithi083 ceph-mon[79930]: pgmap v19079: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:20.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:04:20 smithi083 ceph-mon[77242]: pgmap v19079: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:20.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:04:20 smithi153 ceph-mon[70044]: pgmap v19079: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:20.948 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:04:20.949 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:04:20.949 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:04:20.949 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:04:20.949 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:04:20.950 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:04:20.950 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:04:20.950 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:04:20.951 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:04:20.951 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:04:20.951 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:04:20.951 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:04:20.952 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:04:20.952 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:04:20.952 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:04:20.952 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:04:20.953 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:04:20.953 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:04:21.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:04:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:04:21] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T16:04:21.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:04:21 smithi083 ceph-mon[79930]: from='client.56316 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:04:21.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:04:21 smithi083 ceph-mon[79930]: from='client.66253 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:04:21.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:04:21 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1857244829' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:04:21.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:04:21 smithi083 ceph-mon[77242]: from='client.56316 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:04:21.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:04:21 smithi083 ceph-mon[77242]: from='client.66253 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:04:21.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:04:21 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1857244829' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:04:21.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:04:21 smithi153 ceph-mon[70044]: from='client.56316 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:04:21.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:04:21 smithi153 ceph-mon[70044]: from='client.66253 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:04:21.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:04:21 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1857244829' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:04:22.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:04:22 smithi083 ceph-mon[79930]: pgmap v19080: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:22.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:04:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:04:22.561Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:04:22.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:04:22 smithi083 ceph-mon[77242]: pgmap v19080: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:22.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:04:22 smithi153 ceph-mon[70044]: pgmap v19080: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:04:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:04:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:04:23.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:04:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:04:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:04:24.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:04:24 smithi083 ceph-mon[77242]: pgmap v19081: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:24.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:04:24 smithi083 ceph-mon[79930]: pgmap v19081: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:24.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:04:24 smithi153 ceph-mon[70044]: pgmap v19081: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:26.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:04:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:04:25.901Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:04:26.710 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:04:26 smithi083 ceph-mon[77242]: pgmap v19082: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:26.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:04:26 smithi083 ceph-mon[79930]: pgmap v19082: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:26.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:04:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:04:26.655Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:04:26.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:04:26 smithi153 ceph-mon[70044]: pgmap v19082: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:28.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:04:28 smithi083 ceph-mon[79930]: pgmap v19083: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:28.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:04:28 smithi083 ceph-mon[77242]: pgmap v19083: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:28.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:04:28 smithi153 ceph-mon[70044]: pgmap v19083: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:30.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:04:30 smithi083 ceph-mon[79930]: pgmap v19084: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:30.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:04:30 smithi083 ceph-mon[77242]: pgmap v19084: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:30.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:04:30 smithi153 ceph-mon[70044]: pgmap v19084: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:04:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:04:31] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T16:04:32.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:04:32 smithi083 ceph-mon[79930]: pgmap v19085: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:32.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:04:32 smithi083 ceph-mon[77242]: pgmap v19085: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:32.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:04:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:04:32.562Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:04:32.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:04:32 smithi153 ceph-mon[70044]: pgmap v19085: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:04:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:04:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:04:33.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:04:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:04:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:04:34.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:04:34 smithi083 ceph-mon[79930]: pgmap v19086: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:34.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:04:34 smithi083 ceph-mon[77242]: pgmap v19086: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:34.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:04:34 smithi153 ceph-mon[70044]: pgmap v19086: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:36.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:04:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:04:35.902Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:04:36.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:04:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:04:36.654Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:04:36.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:04:36 smithi083 ceph-mon[77242]: pgmap v19087: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:36.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:04:36 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:04:36.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:04:36 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:04:36.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:04:36 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:04:36.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:04:36 smithi083 ceph-mon[79930]: pgmap v19087: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:36.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:04:36 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:04:36.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:04:36 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:04:36.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:04:36 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:04:36.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:04:36 smithi153 ceph-mon[70044]: pgmap v19087: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:36.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:04:36 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:04:36.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:04:36 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:04:36.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:04:36 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:04:38.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:04:37 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:04:38.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:04:37 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:04:38.467 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:04:37 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:04:39.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:04:38 smithi153 ceph-mon[70044]: pgmap v19088: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:39.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:04:38 smithi083 ceph-mon[79930]: pgmap v19088: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:39.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:04:38 smithi083 ceph-mon[77242]: pgmap v19088: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:41.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:04:40 smithi153 ceph-mon[70044]: pgmap v19089: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:41.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:04:40 smithi083 ceph-mon[79930]: pgmap v19089: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:41.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:04:40 smithi083 ceph-mon[77242]: pgmap v19089: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:04:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:04:41] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T16:04:42.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:04:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:04:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:04:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:04:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:04:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:04:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:04:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:04:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:04:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:04:42.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:04:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:04:42.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:04:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:04:42.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:04:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:04:42.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:04:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:04:42.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:04:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:04:42.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:04:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:04:42.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:04:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:04:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:04:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:04:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:04:42.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:04:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:04:42.563Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:04:43.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:04:42 smithi153 ceph-mon[70044]: pgmap v19090: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:04:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:04:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:04:43.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:04:42 smithi083 ceph-mon[77242]: pgmap v19090: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:43.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:04:42 smithi083 ceph-mon[79930]: pgmap v19090: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:44.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:04:44 smithi083 ceph-mon[79930]: pgmap v19091: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:44.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:04:44 smithi083 ceph-mon[77242]: pgmap v19091: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:44.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:04:44 smithi153 ceph-mon[70044]: pgmap v19091: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:46.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:04:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:04:45.903Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:04:46.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:04:46 smithi083 ceph-mon[77242]: pgmap v19092: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:46.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:04:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:04:46.655Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:04:46.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:04:46 smithi083 ceph-mon[79930]: pgmap v19092: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:46.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:04:46 smithi153 ceph-mon[70044]: pgmap v19092: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:48.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:04:48 smithi083 ceph-mon[79930]: pgmap v19093: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:48.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:04:48 smithi083 ceph-mon[77242]: pgmap v19093: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:48.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:04:48 smithi153 ceph-mon[70044]: pgmap v19093: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:50.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:04:50 smithi083 ceph-mon[79930]: pgmap v19094: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:50.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:04:50 smithi083 ceph-mon[77242]: pgmap v19094: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:50.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:04:50 smithi153 ceph-mon[70044]: pgmap v19094: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:51.275 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:04:51.569 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:04:51.570 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 4m ago 10h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:04:51.570 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 2m ago 10h 64.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:04:51.571 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 4m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:04:51.571 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 2m ago 10h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:04:51.572 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 2m ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:04:51.572 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 4m ago 10h 1052M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:04:51.573 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 4m ago 10h 2300M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:04:51.573 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 2m ago 10h 1030M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:04:51.573 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 4m ago 10h 1384M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:04:51.574 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 4m ago 10h 24.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:04:51.574 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 2m ago 10h 26.7M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:04:51.575 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 4m ago 10h 1673M 2105M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:04:51.575 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 4m ago 10h 1658M 2105M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:04:51.576 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 4m ago 10h 1288M 2105M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:04:51.576 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 4m ago 10h 1626M 2105M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:04:51.577 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 2m ago 10h 1425M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:04:51.577 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 2m ago 10h 1790M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:04:51.578 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 2m ago 10h 1600M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:04:51.578 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 2m ago 10h 1904M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:04:51.579 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 2m ago 10h 185M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:04:51.926 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:04:51.927 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:04:51.927 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:04:51.927 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:04:51.928 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:04:51.928 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:04:51.928 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:04:51.929 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:04:51.929 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:04:51.929 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:04:51.930 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:04:51.930 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:04:51.930 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:04:51.930 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:04:51.930 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:04:51.931 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:04:51.931 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:04:51.931 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:04:51.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:04:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:04:51] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T16:04:52.711 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:04:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:04:52.563Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:04:52.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:04:52 smithi083 ceph-mon[79930]: from='client.56334 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:04:52.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:04:52 smithi083 ceph-mon[79930]: pgmap v19095: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:52.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:04:52 smithi083 ceph-mon[79930]: from='client.56340 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:04:52.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:04:52 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1577717847' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:04:52.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:04:52 smithi083 ceph-mon[77242]: from='client.56334 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:04:52.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:04:52 smithi083 ceph-mon[77242]: pgmap v19095: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:52.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:04:52 smithi083 ceph-mon[77242]: from='client.56340 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:04:52.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:04:52 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1577717847' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:04:52.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:04:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:04:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:04:52.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:04:52 smithi153 ceph-mon[70044]: from='client.56334 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:04:52.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:04:52 smithi153 ceph-mon[70044]: pgmap v19095: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:52.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:04:52 smithi153 ceph-mon[70044]: from='client.56340 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:04:52.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:04:52 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1577717847' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:04:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:04:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:04:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:04:54.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:04:54 smithi083 ceph-mon[79930]: pgmap v19096: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:54.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:04:54 smithi083 ceph-mon[77242]: pgmap v19096: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:54.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:04:54 smithi153 ceph-mon[70044]: pgmap v19096: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:56.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:04:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:04:55.904Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:04:56.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:04:56 smithi083 ceph-mon[79930]: pgmap v19097: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:56.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:04:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:04:56.656Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:04:56.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:04:56 smithi083 ceph-mon[77242]: pgmap v19097: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:56.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:04:56 smithi153 ceph-mon[70044]: pgmap v19097: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:58.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:04:58 smithi083 ceph-mon[77242]: pgmap v19098: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:58.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:04:58 smithi083 ceph-mon[79930]: pgmap v19098: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:04:58.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:04:58 smithi153 ceph-mon[70044]: pgmap v19098: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:00.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:05:00 smithi083 ceph-mon[79930]: pgmap v19099: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:00.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:05:00 smithi083 ceph-mon[77242]: pgmap v19099: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:00.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:05:00 smithi153 ceph-mon[70044]: pgmap v19099: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:01.969 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:05:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:05:01] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T16:05:02.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:05:02 smithi083 ceph-mon[79930]: pgmap v19100: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:02.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:05:02 smithi083 ceph-mon[77242]: pgmap v19100: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:02.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:05:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:05:02.564Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:05:02.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:05:02 smithi153 ceph-mon[70044]: pgmap v19100: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:05:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:05:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:05:03.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:05:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:05:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:05:04.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:05:04 smithi083 ceph-mon[79930]: pgmap v19101: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:04.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:05:04 smithi083 ceph-mon[77242]: pgmap v19101: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:04.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:05:04 smithi153 ceph-mon[70044]: pgmap v19101: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:06.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:05:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:05:05.905Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:05:06.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:05:06 smithi083 ceph-mon[79930]: pgmap v19102: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:06.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:05:06 smithi083 ceph-mon[77242]: pgmap v19102: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:06.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:05:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:05:06.657Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:05:06.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:05:06 smithi153 ceph-mon[70044]: pgmap v19102: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:08.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:05:08 smithi083 ceph-mon[79930]: pgmap v19103: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:08.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:05:08 smithi083 ceph-mon[77242]: pgmap v19103: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:08.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:05:08 smithi153 ceph-mon[70044]: pgmap v19103: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:10.710 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:05:10 smithi083 ceph-mon[77242]: pgmap v19104: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:10.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:05:10 smithi083 ceph-mon[79930]: pgmap v19104: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:10.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:05:10 smithi153 ceph-mon[70044]: pgmap v19104: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:05:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:05:11] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T16:05:12.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:05:12 smithi083 ceph-mon[79930]: pgmap v19105: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:12.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:05:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:05:12.564Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:05:12.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:05:12 smithi083 ceph-mon[77242]: pgmap v19105: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:12.813 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:05:12 smithi153 ceph-mon[70044]: pgmap v19105: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:12.813 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:05:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:05:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:05:13.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:05:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:05:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:05:14.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:05:14 smithi083 ceph-mon[79930]: pgmap v19106: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:14.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:05:14 smithi083 ceph-mon[77242]: pgmap v19106: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:14.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:05:14 smithi153 ceph-mon[70044]: pgmap v19106: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:16.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:05:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:05:15.905Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:05:16.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:05:16 smithi083 ceph-mon[79930]: pgmap v19107: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:16.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:05:16 smithi083 ceph-mon[77242]: pgmap v19107: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:16.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:05:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:05:16.658Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:05:16.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:05:16 smithi153 ceph-mon[70044]: pgmap v19107: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:18.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:05:18 smithi083 ceph-mon[79930]: pgmap v19108: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:18.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:05:18 smithi083 ceph-mon[77242]: pgmap v19108: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:18.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:05:18 smithi153 ceph-mon[70044]: pgmap v19108: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:20.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:05:20 smithi083 ceph-mon[79930]: pgmap v19109: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:20.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:05:20 smithi083 ceph-mon[77242]: pgmap v19109: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:20.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:05:20 smithi153 ceph-mon[70044]: pgmap v19109: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:05:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:05:21] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T16:05:22.256 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:05:22.561 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:05:22.562 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 4m ago 10h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:05:22.563 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 2m ago 10h 64.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:05:22.563 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 4m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:05:22.564 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 2m ago 10h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:05:22.564 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 2m ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:05:22.565 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 4m ago 10h 1052M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:05:22.565 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 4m ago 10h 2300M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:05:22.565 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 2m ago 10h 1030M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:05:22.566 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 4m ago 10h 1384M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:05:22.566 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 4m ago 10h 24.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:05:22.567 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 2m ago 10h 26.7M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:05:22.567 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 4m ago 10h 1673M 2105M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:05:22.568 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 4m ago 10h 1658M 2105M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:05:22.568 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 4m ago 10h 1288M 2105M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:05:22.569 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 4m ago 10h 1626M 2105M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:05:22.569 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 2m ago 10h 1425M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:05:22.569 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 2m ago 10h 1790M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:05:22.570 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 2m ago 10h 1600M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:05:22.570 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 2m ago 10h 1904M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:05:22.571 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 2m ago 10h 185M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:05:22.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:05:22 smithi083 ceph-mon[79930]: pgmap v19110: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:22.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:05:22 smithi083 ceph-mon[77242]: pgmap v19110: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:22.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:05:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:05:22.564Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:05:22.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:05:22 smithi153 ceph-mon[70044]: pgmap v19110: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:05:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:05:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:05:22.902 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:05:22.903 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:05:22.903 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:05:22.904 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:05:22.904 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:05:22.905 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:05:22.905 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:05:22.906 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:05:22.906 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:05:22.906 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:05:22.907 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:05:22.907 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:05:22.908 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:05:22.908 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:05:22.909 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:05:22.909 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:05:22.910 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:05:22.910 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:05:23.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:05:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:05:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:05:23.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:05:23 smithi083 ceph-mon[79930]: from='client.56352 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:05:23.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:05:23 smithi083 ceph-mon[79930]: from='client.56358 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:05:23.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:05:23 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/430692530' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:05:23.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:05:23 smithi083 ceph-mon[77242]: from='client.56352 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:05:23.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:05:23 smithi083 ceph-mon[77242]: from='client.56358 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:05:23.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:05:23 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/430692530' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:05:23.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:05:23 smithi153 ceph-mon[70044]: from='client.56352 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:05:23.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:05:23 smithi153 ceph-mon[70044]: from='client.56358 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:05:23.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:05:23 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/430692530' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:05:24.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:05:24 smithi083 ceph-mon[79930]: pgmap v19111: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:24.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:05:24 smithi083 ceph-mon[77242]: pgmap v19111: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:24.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:05:24 smithi153 ceph-mon[70044]: pgmap v19111: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:26.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:05:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:05:25.906Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:05:26.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:05:26 smithi083 ceph-mon[77242]: pgmap v19112: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:26.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:05:26 smithi083 ceph-mon[79930]: pgmap v19112: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:26.713 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:05:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:05:26.658Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:05:26.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:05:26 smithi153 ceph-mon[70044]: pgmap v19112: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:28.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:05:28 smithi153 ceph-mon[70044]: pgmap v19113: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:28.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:05:28 smithi083 ceph-mon[77242]: pgmap v19113: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:28.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:05:28 smithi083 ceph-mon[79930]: pgmap v19113: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:30.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:05:30 smithi153 ceph-mon[70044]: pgmap v19114: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:30.968 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:05:30 smithi083 ceph-mon[77242]: pgmap v19114: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:30.969 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:05:30 smithi083 ceph-mon[79930]: pgmap v19114: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:31.967 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:05:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:05:31] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T16:05:32.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:05:32 smithi153 ceph-mon[70044]: pgmap v19115: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:05:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:05:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:05:32.968 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:05:32 smithi083 ceph-mon[79930]: pgmap v19115: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:32.969 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:05:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:05:32.566Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:05:32.970 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:05:32 smithi083 ceph-mon[77242]: pgmap v19115: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:33.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:05:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:05:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:05:34.468 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:05:34 smithi083 ceph-mon[79930]: pgmap v19116: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:34.469 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:05:34 smithi083 ceph-mon[77242]: pgmap v19116: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:34.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:05:34 smithi153 ceph-mon[70044]: pgmap v19116: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:36.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:05:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:05:35.907Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:05:36.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:05:36 smithi083 ceph-mon[79930]: pgmap v19117: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:36.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:05:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:05:36.658Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:05:36.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:05:36 smithi083 ceph-mon[77242]: pgmap v19117: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:36.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:05:36 smithi153 ceph-mon[70044]: pgmap v19117: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:37.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:05:37 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:05:37.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:05:37 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:05:37.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:05:37 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:05:37.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:05:37 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:05:37.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:05:37 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:05:37.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:05:37 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:05:37.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:05:37 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:05:37.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:05:37 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:05:37.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:05:37 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:05:38.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:05:38 smithi083 ceph-mon[79930]: pgmap v19118: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:38.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:05:38 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:05:38.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:05:38 smithi083 ceph-mon[77242]: pgmap v19118: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:38.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:05:38 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:05:39.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:05:38 smithi153 ceph-mon[70044]: pgmap v19118: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:39.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:05:38 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:05:40.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:05:40 smithi083 ceph-mon[79930]: pgmap v19119: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:40.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:05:40 smithi083 ceph-mon[77242]: pgmap v19119: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:41.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:05:40 smithi153 ceph-mon[70044]: pgmap v19119: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:05:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:05:41] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T16:05:42.641 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:05:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:05:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:05:42.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:05:42 smithi083 ceph-mon[79930]: pgmap v19120: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:42.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:05:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:05:42.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:05:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:05:42.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:05:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:05:42.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:05:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:05:42.964 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:05:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:05:42.567Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:05:42.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:05:42 smithi083 ceph-mon[77242]: pgmap v19120: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:42.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:05:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:05:42.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:05:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:05:42.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:05:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:05:42.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:05:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:05:43.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:05:42 smithi153 ceph-mon[70044]: pgmap v19120: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:43.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:05:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:05:43.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:05:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:05:43.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:05:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:05:43.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:05:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:05:43.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:05:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:05:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:05:44.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:05:44 smithi083 ceph-mon[77242]: pgmap v19121: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:44.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:05:44 smithi083 ceph-mon[79930]: pgmap v19121: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:44.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:05:44 smithi153 ceph-mon[70044]: pgmap v19121: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:46.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:05:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:05:45.907Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:05:46.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:05:46 smithi083 ceph-mon[79930]: pgmap v19122: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:46.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:05:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:05:46.659Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:05:46.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:05:46 smithi083 ceph-mon[77242]: pgmap v19122: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:46.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:05:46 smithi153 ceph-mon[70044]: pgmap v19122: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:48.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:05:48 smithi083 ceph-mon[79930]: pgmap v19123: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:48.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:05:48 smithi083 ceph-mon[77242]: pgmap v19123: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:48.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:05:48 smithi153 ceph-mon[70044]: pgmap v19123: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:50.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:05:50 smithi083 ceph-mon[79930]: pgmap v19124: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:50.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:05:50 smithi083 ceph-mon[77242]: pgmap v19124: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:50.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:05:50 smithi153 ceph-mon[70044]: pgmap v19124: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:05:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:05:51] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T16:05:52.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:05:52 smithi153 ceph-mon[70044]: pgmap v19125: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:52.811 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:05:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:05:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:05:52.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:05:52 smithi083 ceph-mon[79930]: pgmap v19125: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:52.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:05:52 smithi083 ceph-mon[77242]: pgmap v19125: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:52.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:05:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:05:52.567Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:05:53.228 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:05:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:05:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:05:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:05:53.527 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:05:53.527 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 5m ago 10h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:05:53.528 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 3m ago 10h 64.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:05:53.528 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 5m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:05:53.529 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 3m ago 10h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:05:53.529 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 3m ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:05:53.530 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 5m ago 10h 1052M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:05:53.530 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 5m ago 10h 2300M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:05:53.531 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 3m ago 10h 1030M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:05:53.531 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 5m ago 10h 1384M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:05:53.532 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 5m ago 10h 24.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:05:53.532 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 3m ago 10h 26.7M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:05:53.532 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 5m ago 10h 1673M 2105M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:05:53.533 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 5m ago 10h 1658M 2105M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:05:53.533 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 5m ago 10h 1288M 2105M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:05:53.533 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 5m ago 10h 1626M 2105M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:05:53.533 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 3m ago 10h 1425M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:05:53.534 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 3m ago 10h 1790M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:05:53.534 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 3m ago 10h 1600M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:05:53.534 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 3m ago 10h 1904M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:05:53.535 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 3m ago 10h 185M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:05:53.895 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:05:53.896 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:05:53.896 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:05:53.896 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:05:53.897 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:05:53.897 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:05:53.897 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:05:53.897 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:05:53.897 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:05:53.898 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:05:53.898 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:05:53.898 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:05:53.898 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:05:53.899 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:05:53.899 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:05:53.899 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:05:53.899 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:05:53.899 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:05:54.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:05:54 smithi083 ceph-mon[79930]: from='client.56370 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:05:54.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:05:54 smithi083 ceph-mon[79930]: pgmap v19126: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:54.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:05:54 smithi083 ceph-mon[79930]: from='client.56376 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:05:54.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:05:54 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4008504300' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:05:54.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:05:54 smithi083 ceph-mon[77242]: from='client.56370 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:05:54.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:05:54 smithi083 ceph-mon[77242]: pgmap v19126: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:54.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:05:54 smithi083 ceph-mon[77242]: from='client.56376 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:05:54.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:05:54 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4008504300' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:05:54.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:05:54 smithi153 ceph-mon[70044]: from='client.56370 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:05:54.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:05:54 smithi153 ceph-mon[70044]: pgmap v19126: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:54.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:05:54 smithi153 ceph-mon[70044]: from='client.56376 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:05:54.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:05:54 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4008504300' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:05:56.210 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:05:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:05:55.908Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:05:56.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:05:56 smithi083 ceph-mon[79930]: pgmap v19127: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:56.740 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:05:56 smithi083 ceph-mon[77242]: pgmap v19127: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:56.741 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:05:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:05:56.660Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:05:56.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:05:56 smithi153 ceph-mon[70044]: pgmap v19127: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:58.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:05:58 smithi083 ceph-mon[79930]: pgmap v19128: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:58.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:05:58 smithi083 ceph-mon[77242]: pgmap v19128: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:05:58.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:05:58 smithi153 ceph-mon[70044]: pgmap v19128: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:00.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:06:00 smithi153 ceph-mon[70044]: pgmap v19129: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:00.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:06:00 smithi083 ceph-mon[79930]: pgmap v19129: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:00.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:06:00 smithi083 ceph-mon[77242]: pgmap v19129: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:01.965 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:06:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:06:01] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T16:06:02.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:06:02 smithi153 ceph-mon[70044]: pgmap v19130: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:06:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:06:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:06:02.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:06:02 smithi083 ceph-mon[79930]: pgmap v19130: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:02.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:06:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:06:02.568Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:06:02.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:06:02 smithi083 ceph-mon[77242]: pgmap v19130: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:06:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:06:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:06:04.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:06:04 smithi083 ceph-mon[79930]: pgmap v19131: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:04.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:06:04 smithi083 ceph-mon[77242]: pgmap v19131: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:04.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:06:04 smithi153 ceph-mon[70044]: pgmap v19131: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:06.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:06:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:06:05.910Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:06:06.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:06:06 smithi083 ceph-mon[79930]: pgmap v19132: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:06.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:06:06 smithi083 ceph-mon[77242]: pgmap v19132: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:06.712 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:06:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:06:06.660Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:06:06.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:06:06 smithi153 ceph-mon[70044]: pgmap v19132: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:08.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:06:08 smithi153 ceph-mon[70044]: pgmap v19133: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:08.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:06:08 smithi083 ceph-mon[79930]: pgmap v19133: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:08.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:06:08 smithi083 ceph-mon[77242]: pgmap v19133: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:10.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:06:10 smithi153 ceph-mon[70044]: pgmap v19134: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:10.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:06:10 smithi083 ceph-mon[79930]: pgmap v19134: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:10.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:06:10 smithi083 ceph-mon[77242]: pgmap v19134: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:06:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:06:11] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T16:06:12.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:06:12 smithi153 ceph-mon[70044]: pgmap v19135: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:06:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:06:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:06:12.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:06:12 smithi083 ceph-mon[79930]: pgmap v19135: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:12.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:06:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:06:12.569Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:06:12.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:06:12 smithi083 ceph-mon[77242]: pgmap v19135: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:13.316 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:06:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:06:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:06:14.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:06:14 smithi083 ceph-mon[77242]: pgmap v19136: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:14.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:06:14 smithi083 ceph-mon[79930]: pgmap v19136: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:14.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:06:14 smithi153 ceph-mon[70044]: pgmap v19136: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:16.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:06:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:06:15.910Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:06:16.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:06:16 smithi153 ceph-mon[70044]: pgmap v19137: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:16.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:06:16 smithi083 ceph-mon[79930]: pgmap v19137: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:16.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:06:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:06:16.661Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:06:16.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:06:16 smithi083 ceph-mon[77242]: pgmap v19137: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:18.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:06:18 smithi153 ceph-mon[70044]: pgmap v19138: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:18.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:06:18 smithi083 ceph-mon[79930]: pgmap v19138: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:18.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:06:18 smithi083 ceph-mon[77242]: pgmap v19138: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:20.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:06:20 smithi153 ceph-mon[70044]: pgmap v19139: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:20.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:06:20 smithi083 ceph-mon[79930]: pgmap v19139: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:20.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:06:20 smithi083 ceph-mon[77242]: pgmap v19139: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:06:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:06:21] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T16:06:22.815 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:06:22 smithi153 ceph-mon[70044]: pgmap v19140: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:22.816 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:06:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:06:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:06:22.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:06:22 smithi083 ceph-mon[79930]: pgmap v19140: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:22.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:06:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:06:22.570Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:06:22.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:06:22 smithi083 ceph-mon[77242]: pgmap v19140: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:06:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:06:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:06:24.221 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:06:24.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:06:24 smithi083 ceph-mon[79930]: pgmap v19141: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:24.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:06:24 smithi083 ceph-mon[77242]: pgmap v19141: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:24.510 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:06:24.510 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 5m ago 10h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:06:24.511 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 3m ago 10h 64.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:06:24.511 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 5m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:06:24.511 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 3m ago 10h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:06:24.511 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 3m ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:06:24.512 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 5m ago 10h 1052M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:06:24.512 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 5m ago 10h 2300M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:06:24.513 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 3m ago 10h 1030M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:06:24.513 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 5m ago 10h 1384M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:06:24.513 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 5m ago 10h 24.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:06:24.513 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 3m ago 10h 26.7M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:06:24.514 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 5m ago 10h 1673M 2105M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:06:24.514 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 5m ago 10h 1658M 2105M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:06:24.514 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 5m ago 10h 1288M 2105M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:06:24.515 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 5m ago 10h 1626M 2105M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:06:24.515 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 3m ago 10h 1425M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:06:24.515 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 3m ago 10h 1790M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:06:24.516 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 3m ago 10h 1600M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:06:24.517 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 3m ago 10h 1904M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:06:24.517 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 3m ago 10h 185M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:06:24.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:06:24 smithi153 ceph-mon[70044]: pgmap v19141: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:24.867 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:06:24.867 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:06:24.868 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:06:24.868 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:06:24.869 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:06:24.869 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:06:24.870 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:06:24.870 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:06:24.871 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:06:24.871 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:06:24.872 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:06:24.872 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:06:24.872 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:06:24.873 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:06:24.873 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:06:24.874 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:06:24.874 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:06:24.875 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:06:25.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:06:25 smithi083 ceph-mon[79930]: from='client.56388 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:06:25.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:06:25 smithi083 ceph-mon[79930]: from='client.66325 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:06:25.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:06:25 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2997792007' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:06:25.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:06:25 smithi083 ceph-mon[77242]: from='client.56388 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:06:25.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:06:25 smithi083 ceph-mon[77242]: from='client.66325 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:06:25.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:06:25 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2997792007' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:06:25.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:06:25 smithi153 ceph-mon[70044]: from='client.56388 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:06:25.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:06:25 smithi153 ceph-mon[70044]: from='client.66325 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:06:25.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:06:25 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2997792007' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:06:26.163 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:06:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:06:25.911Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:06:26.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:06:26 smithi083 ceph-mon[79930]: pgmap v19142: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:26.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:06:26 smithi083 ceph-mon[77242]: pgmap v19142: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:06:26 smithi153 ceph-mon[70044]: pgmap v19142: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:26.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:06:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:06:26.661Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:06:28.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:06:28 smithi153 ceph-mon[70044]: pgmap v19143: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:28.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:06:28 smithi083 ceph-mon[79930]: pgmap v19143: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:28.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:06:28 smithi083 ceph-mon[77242]: pgmap v19143: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:30.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:06:30 smithi153 ceph-mon[70044]: pgmap v19144: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:30.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:06:30 smithi083 ceph-mon[79930]: pgmap v19144: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:30.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:06:30 smithi083 ceph-mon[77242]: pgmap v19144: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:31.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:06:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:06:31] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T16:06:32.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:06:32 smithi153 ceph-mon[70044]: pgmap v19145: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:06:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:06:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:06:32.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:06:32 smithi083 ceph-mon[79930]: pgmap v19145: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:32.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:06:32 smithi083 ceph-mon[77242]: pgmap v19145: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:32.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:06:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:06:32.571Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:06:33.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:06:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:06:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:06:34.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:06:34 smithi083 ceph-mon[79930]: pgmap v19146: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:34.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:06:34 smithi083 ceph-mon[77242]: pgmap v19146: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:06:34 smithi153 ceph-mon[70044]: pgmap v19146: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:36.210 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:06:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:06:35.911Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:06:36.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:06:36 smithi153 ceph-mon[70044]: pgmap v19147: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:36.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:06:36 smithi083 ceph-mon[77242]: pgmap v19147: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:36.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:06:36 smithi083 ceph-mon[79930]: pgmap v19147: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:36.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:06:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:06:36.661Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:06:38.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:06:38 smithi153 ceph-mon[70044]: pgmap v19148: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:38.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:06:38 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:06:38.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:06:38 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:06:38.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:06:38 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:06:38.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:06:38 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:06:38.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:06:38 smithi083 ceph-mon[77242]: pgmap v19148: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:38.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:06:38 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:06:38.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:06:38 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:06:38.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:06:38 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:06:38.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:06:38 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:06:38.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:06:38 smithi083 ceph-mon[79930]: pgmap v19148: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:38.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:06:38 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:06:38.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:06:38 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:06:38.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:06:38 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:06:38.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:06:38 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:06:40.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:06:40 smithi153 ceph-mon[70044]: pgmap v19149: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:40.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:06:40 smithi083 ceph-mon[77242]: pgmap v19149: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:40.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:06:40 smithi083 ceph-mon[79930]: pgmap v19149: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:41.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:06:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:06:41] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T16:06:42.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:06:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:06:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:06:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:06:42 smithi153 ceph-mon[70044]: pgmap v19150: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:06:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:06:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:06:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:06:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:06:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:06:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:06:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:06:42.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:06:42 smithi083 ceph-mon[79930]: pgmap v19150: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:42.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:06:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:06:42.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:06:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:06:42.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:06:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:06:42.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:06:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:06:42.964 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:06:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:06:42.571Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:06:42.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:06:42 smithi083 ceph-mon[77242]: pgmap v19150: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:42.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:06:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:06:42.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:06:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:06:42.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:06:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:06:42.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:06:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:06:43.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:06:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:06:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:06:44.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:06:44 smithi083 ceph-mon[79930]: pgmap v19151: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:44.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:06:44 smithi083 ceph-mon[77242]: pgmap v19151: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:44.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:06:44 smithi153 ceph-mon[70044]: pgmap v19151: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:46.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:06:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:06:45.912Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:06:46.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:06:46 smithi153 ceph-mon[70044]: pgmap v19152: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:46.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:06:46 smithi083 ceph-mon[79930]: pgmap v19152: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:46.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:06:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:06:46.662Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:06:46.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:06:46 smithi083 ceph-mon[77242]: pgmap v19152: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:48.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:06:48 smithi153 ceph-mon[70044]: pgmap v19153: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:48.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:06:48 smithi083 ceph-mon[79930]: pgmap v19153: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:48.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:06:48 smithi083 ceph-mon[77242]: pgmap v19153: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:50.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:06:50 smithi153 ceph-mon[70044]: pgmap v19154: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:50.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:06:50 smithi083 ceph-mon[79930]: pgmap v19154: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:50.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:06:50 smithi083 ceph-mon[77242]: pgmap v19154: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:06:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:06:51] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T16:06:52.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:06:52 smithi153 ceph-mon[70044]: pgmap v19155: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:06:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:06:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:06:52.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:06:52 smithi083 ceph-mon[77242]: pgmap v19155: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:52.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:06:52 smithi083 ceph-mon[79930]: pgmap v19155: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:52.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:06:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:06:52.572Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:06:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:06:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:06:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:06:54.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:06:54 smithi083 ceph-mon[79930]: pgmap v19156: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:54.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:06:54 smithi083 ceph-mon[77242]: pgmap v19156: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:54.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:06:54 smithi153 ceph-mon[70044]: pgmap v19156: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:55.192 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:06:55.483 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:06:55.484 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 6m ago 10h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:06:55.484 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 4m ago 10h 64.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:06:55.484 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 6m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:06:55.484 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 4m ago 10h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:06:55.485 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 4m ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:06:55.485 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 6m ago 10h 1052M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:06:55.485 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 6m ago 10h 2300M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:06:55.485 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 4m ago 10h 1030M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:06:55.486 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 6m ago 10h 1384M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:06:55.486 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 6m ago 10h 24.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:06:55.486 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 4m ago 10h 26.7M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:06:55.486 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 6m ago 10h 1673M 2105M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:06:55.486 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 6m ago 10h 1658M 2105M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:06:55.487 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 6m ago 10h 1288M 2105M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:06:55.487 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 6m ago 10h 1626M 2105M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:06:55.487 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 4m ago 10h 1425M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:06:55.487 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 4m ago 10h 1790M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:06:55.488 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 4m ago 10h 1600M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:06:55.488 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 4m ago 10h 1904M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:06:55.488 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 4m ago 10h 185M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:06:55.839 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:06:55.842 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:06:55.843 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:06:55.843 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:06:55.843 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:06:55.844 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:06:55.844 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:06:55.844 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:06:55.845 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:06:55.845 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:06:55.845 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:06:55.845 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:06:55.847 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:06:55.847 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:06:55.848 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:06:55.848 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:06:55.848 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:06:55.848 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:06:56.210 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:06:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:06:55.911Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:06:56.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:06:56 smithi153 ceph-mon[70044]: from='client.66337 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:06:56.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:06:56 smithi153 ceph-mon[70044]: pgmap v19157: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:56.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:06:56 smithi153 ceph-mon[70044]: from='client.66343 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:06:56.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:06:56 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2945180642' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:06:56.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:06:56 smithi083 ceph-mon[79930]: from='client.66337 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:06:56.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:06:56 smithi083 ceph-mon[79930]: pgmap v19157: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:56.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:06:56 smithi083 ceph-mon[79930]: from='client.66343 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:06:56.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:06:56 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2945180642' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:06:56.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:06:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:06:56.662Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:06:56.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:06:56 smithi083 ceph-mon[77242]: from='client.66337 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:06:56.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:06:56 smithi083 ceph-mon[77242]: pgmap v19157: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:56.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:06:56 smithi083 ceph-mon[77242]: from='client.66343 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:06:56.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:06:56 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2945180642' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:06:58.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:06:58 smithi153 ceph-mon[70044]: pgmap v19158: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:58.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:06:58 smithi083 ceph-mon[77242]: pgmap v19158: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:06:58.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:06:58 smithi083 ceph-mon[79930]: pgmap v19158: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:00.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:07:00 smithi153 ceph-mon[70044]: pgmap v19159: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:00.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:07:00 smithi083 ceph-mon[79930]: pgmap v19159: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:00.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:07:00 smithi083 ceph-mon[77242]: pgmap v19159: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:07:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:07:01] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T16:07:02.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:07:02 smithi153 ceph-mon[70044]: pgmap v19160: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:07:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:07:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:07:02.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:07:02 smithi083 ceph-mon[79930]: pgmap v19160: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:02.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:07:02 smithi083 ceph-mon[77242]: pgmap v19160: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:02.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:07:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:07:02.573Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:07:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:07:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:07:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:07:04.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:07:04 smithi083 ceph-mon[79930]: pgmap v19161: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:04.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:07:04 smithi083 ceph-mon[77242]: pgmap v19161: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:04.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:07:04 smithi153 ceph-mon[70044]: pgmap v19161: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:06.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:07:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:07:05.913Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:07:06.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:07:06 smithi153 ceph-mon[70044]: pgmap v19162: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:06.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:07:06 smithi083 ceph-mon[79930]: pgmap v19162: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:06.967 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:07:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:07:06.663Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:07:06.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:07:06 smithi083 ceph-mon[77242]: pgmap v19162: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:08.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:07:08 smithi153 ceph-mon[70044]: pgmap v19163: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:08.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:07:08 smithi083 ceph-mon[79930]: pgmap v19163: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:08.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:07:08 smithi083 ceph-mon[77242]: pgmap v19163: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:10.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:07:10 smithi153 ceph-mon[70044]: pgmap v19164: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:10.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:07:10 smithi083 ceph-mon[79930]: pgmap v19164: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:10.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:07:10 smithi083 ceph-mon[77242]: pgmap v19164: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:07:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:07:11] "GET /metrics HTTP/1.1" 200 225246 "" "Prometheus/2.33.4" 2022-09-29T16:07:12.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:07:12 smithi153 ceph-mon[70044]: pgmap v19165: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:07:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:07:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:07:12.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:07:12 smithi083 ceph-mon[79930]: pgmap v19165: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:12.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:07:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:07:12.573Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:07:12.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:07:12 smithi083 ceph-mon[77242]: pgmap v19165: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:13.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:07:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:07:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:07:14.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:07:14 smithi083 ceph-mon[77242]: pgmap v19166: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:14.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:07:14 smithi083 ceph-mon[79930]: pgmap v19166: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:14.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:07:14 smithi153 ceph-mon[70044]: pgmap v19166: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:16.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:07:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:07:15.914Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:07:16.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:07:16 smithi153 ceph-mon[70044]: pgmap v19167: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:16.960 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:07:16 smithi083 ceph-mon[77242]: pgmap v19167: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:16.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:07:16 smithi083 ceph-mon[79930]: pgmap v19167: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:16.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:07:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:07:16.664Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:07:18.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:07:18 smithi153 ceph-mon[70044]: pgmap v19168: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:18.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:07:18 smithi083 ceph-mon[79930]: pgmap v19168: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:18.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:07:18 smithi083 ceph-mon[77242]: pgmap v19168: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:20.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:07:20 smithi153 ceph-mon[70044]: pgmap v19169: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:20.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:07:20 smithi083 ceph-mon[77242]: pgmap v19169: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:20.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:07:20 smithi083 ceph-mon[79930]: pgmap v19169: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:21.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:07:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:07:21] "GET /metrics HTTP/1.1" 200 225246 "" "Prometheus/2.33.4" 2022-09-29T16:07:22.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:07:22 smithi153 ceph-mon[70044]: pgmap v19170: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:07:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:07:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:07:22.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:07:22 smithi083 ceph-mon[79930]: pgmap v19170: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:22.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:07:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:07:22.574Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:07:22.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:07:22 smithi083 ceph-mon[77242]: pgmap v19170: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:23.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:07:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:07:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:07:24.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:07:24 smithi083 ceph-mon[79930]: pgmap v19171: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:24.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:07:24 smithi083 ceph-mon[77242]: pgmap v19171: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:24.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:07:24 smithi153 ceph-mon[70044]: pgmap v19171: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:26.165 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:07:26.210 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:07:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:07:25.913Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:07:26.455 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:07:26.456 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 6m ago 10h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:07:26.456 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 4m ago 10h 64.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:07:26.456 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 6m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:07:26.456 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 4m ago 10h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:07:26.457 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 4m ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:07:26.457 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 6m ago 10h 1052M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:07:26.457 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 6m ago 10h 2300M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:07:26.457 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 4m ago 10h 1030M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:07:26.458 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 6m ago 10h 1384M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:07:26.458 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 6m ago 10h 24.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:07:26.458 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 4m ago 10h 26.7M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:07:26.458 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 6m ago 10h 1673M 2105M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:07:26.459 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 6m ago 10h 1658M 2105M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:07:26.459 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 6m ago 10h 1288M 2105M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:07:26.459 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 6m ago 10h 1626M 2105M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:07:26.459 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 4m ago 10h 1425M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:07:26.460 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 4m ago 10h 1790M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:07:26.460 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 4m ago 10h 1600M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:07:26.460 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 4m ago 10h 1904M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:07:26.460 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 4m ago 10h 185M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:07:26.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:07:26 smithi153 ceph-mon[70044]: pgmap v19172: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:26.812 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:07:26.812 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:07:26.813 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:07:26.813 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:07:26.813 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:07:26.814 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:07:26.814 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:07:26.814 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:07:26.814 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:07:26.815 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:07:26.815 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:07:26.815 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:07:26.815 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:07:26.816 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:07:26.816 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:07:26.817 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:07:26.817 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:07:26.817 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:07:26.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:07:26 smithi083 ceph-mon[79930]: pgmap v19172: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:26.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:07:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:07:26.664Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:07:26.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:07:26 smithi083 ceph-mon[77242]: pgmap v19172: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:27.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:07:27 smithi153 ceph-mon[70044]: from='client.56424 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:07:27.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:07:27 smithi153 ceph-mon[70044]: from='client.66361 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:07:27.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:07:27 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/285340721' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:07:27.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:07:27 smithi083 ceph-mon[79930]: from='client.56424 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:07:27.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:07:27 smithi083 ceph-mon[79930]: from='client.66361 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:07:27.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:07:27 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/285340721' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:07:27.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:07:27 smithi083 ceph-mon[77242]: from='client.56424 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:07:27.972 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:07:27 smithi083 ceph-mon[77242]: from='client.66361 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:07:27.972 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:07:27 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/285340721' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:07:28.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:07:28 smithi153 ceph-mon[70044]: pgmap v19173: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:28.960 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:07:28 smithi083 ceph-mon[77242]: pgmap v19173: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:28.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:07:28 smithi083 ceph-mon[79930]: pgmap v19173: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:30.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:07:30 smithi153 ceph-mon[70044]: pgmap v19174: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:30.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:07:30 smithi083 ceph-mon[77242]: pgmap v19174: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:30.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:07:30 smithi083 ceph-mon[79930]: pgmap v19174: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:07:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:07:31] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T16:07:32.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:07:32 smithi153 ceph-mon[70044]: pgmap v19175: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:07:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:07:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:07:32.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:07:32 smithi083 ceph-mon[79930]: pgmap v19175: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:32.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:07:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:07:32.575Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:07:32.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:07:32 smithi083 ceph-mon[77242]: pgmap v19175: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:33.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:07:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:07:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:07:34.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:07:34 smithi083 ceph-mon[79930]: pgmap v19176: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:34.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:07:34 smithi083 ceph-mon[77242]: pgmap v19176: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:34.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:07:34 smithi153 ceph-mon[70044]: pgmap v19176: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:36.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:07:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:07:35.915Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:07:36.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:07:36 smithi153 ceph-mon[70044]: pgmap v19177: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:36.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:07:36 smithi083 ceph-mon[77242]: pgmap v19177: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:36.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:07:36 smithi083 ceph-mon[79930]: pgmap v19177: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:36.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:07:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:07:36.665Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:07:38.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:07:38 smithi153 ceph-mon[70044]: pgmap v19178: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:38.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:07:38 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:07:38.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:07:38 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:07:38.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:07:38 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:07:38.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:07:38 smithi083 ceph-mon[79930]: pgmap v19178: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:38.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:07:38 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:07:38.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:07:38 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:07:38.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:07:38 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:07:38.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:07:38 smithi083 ceph-mon[77242]: pgmap v19178: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:38.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:07:38 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:07:38.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:07:38 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:07:38.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:07:38 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:07:40.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:07:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:07:40.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:07:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:07:40.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:07:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:07:41.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:07:40 smithi083 ceph-mon[79930]: pgmap v19179: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:41.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:07:40 smithi083 ceph-mon[77242]: pgmap v19179: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:41.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:07:40 smithi153 ceph-mon[70044]: pgmap v19179: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:41.924 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:07:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:07:41] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T16:07:42.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:07:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:07:42.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:07:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:07:42.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:07:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:07:42.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:07:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:07:42.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:07:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:07:42.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:07:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:07:42.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:07:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:07:42.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:07:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:07:42.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:07:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:07:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:07:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:07:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:07:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:07:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:07:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:07:42.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:07:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:07:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:07:42.926 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:07:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:07:42.575Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:07:43.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:07:42 smithi083 ceph-mon[79930]: pgmap v19180: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:43.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:07:42 smithi083 ceph-mon[77242]: pgmap v19180: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:43.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:07:42 smithi153 ceph-mon[70044]: pgmap v19180: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:07:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:07:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:07:44.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:07:44 smithi083 ceph-mon[79930]: pgmap v19181: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:44.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:07:44 smithi083 ceph-mon[77242]: pgmap v19181: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:44.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:07:44 smithi153 ceph-mon[70044]: pgmap v19181: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:46.210 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:07:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:07:45.916Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:07:46.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:07:46 smithi153 ceph-mon[70044]: pgmap v19182: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:46.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:07:46 smithi083 ceph-mon[79930]: pgmap v19182: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:46.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:07:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:07:46.666Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:07:46.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:07:46 smithi083 ceph-mon[77242]: pgmap v19182: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:48.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:07:48 smithi153 ceph-mon[70044]: pgmap v19183: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:48.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:07:48 smithi083 ceph-mon[79930]: pgmap v19183: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:48.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:07:48 smithi083 ceph-mon[77242]: pgmap v19183: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:50.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:07:50 smithi153 ceph-mon[70044]: pgmap v19184: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:50.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:07:50 smithi083 ceph-mon[79930]: pgmap v19184: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:50.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:07:50 smithi083 ceph-mon[77242]: pgmap v19184: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:07:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:07:51] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T16:07:52.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:07:52 smithi153 ceph-mon[70044]: pgmap v19185: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:07:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:07:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:07:52.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:07:52 smithi083 ceph-mon[79930]: pgmap v19185: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:52.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:07:52 smithi083 ceph-mon[77242]: pgmap v19185: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:52.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:07:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:07:52.576Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:07:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:07:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:07:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:07:54.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:07:54 smithi083 ceph-mon[77242]: pgmap v19186: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:54.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:07:54 smithi083 ceph-mon[79930]: pgmap v19186: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:54.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:07:54 smithi153 ceph-mon[70044]: pgmap v19186: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:56.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:07:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:07:55.917Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:07:56.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:07:56 smithi153 ceph-mon[70044]: pgmap v19187: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:56.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:07:56 smithi083 ceph-mon[79930]: pgmap v19187: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:56.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:07:56 smithi083 ceph-mon[77242]: pgmap v19187: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:56.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:07:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:07:56.666Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:07:57.138 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:07:57.425 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:07:57.427 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 7m ago 10h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:07:57.427 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 5m ago 10h 64.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:07:57.427 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 7m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:07:57.427 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 5m ago 10h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:07:57.427 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 5m ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:07:57.428 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 7m ago 10h 1052M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:07:57.428 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 7m ago 10h 2300M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:07:57.428 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 5m ago 10h 1030M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:07:57.428 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 7m ago 10h 1384M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:07:57.429 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 7m ago 10h 24.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:07:57.429 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 5m ago 10h 26.7M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:07:57.429 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 7m ago 10h 1673M 2105M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:07:57.429 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 7m ago 10h 1658M 2105M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:07:57.430 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 7m ago 10h 1288M 2105M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:07:57.430 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 7m ago 10h 1626M 2105M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:07:57.430 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 5m ago 10h 1425M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:07:57.431 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 5m ago 10h 1790M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:07:57.431 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 5m ago 10h 1600M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:07:57.431 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 5m ago 10h 1904M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:07:57.431 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 5m ago 10h 185M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:07:57.786 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:07:57.787 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:07:57.787 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:07:57.787 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:07:57.787 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:07:57.788 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:07:57.788 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:07:57.788 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:07:57.789 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:07:57.789 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:07:57.790 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:07:57.790 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:07:57.791 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:07:57.791 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:07:57.792 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:07:57.792 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:07:57.793 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:07:57.793 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:07:58.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:07:58 smithi153 ceph-mon[70044]: from='client.56442 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:07:58.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:07:58 smithi153 ceph-mon[70044]: from='client.56448 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:07:58.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:07:58 smithi153 ceph-mon[70044]: pgmap v19188: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:58.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:07:58 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3211624364' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:07:58.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:07:58 smithi083 ceph-mon[79930]: from='client.56442 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:07:58.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:07:58 smithi083 ceph-mon[79930]: from='client.56448 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:07:58.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:07:58 smithi083 ceph-mon[79930]: pgmap v19188: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:58.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:07:58 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3211624364' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:07:58.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:07:58 smithi083 ceph-mon[77242]: from='client.56442 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:07:58.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:07:58 smithi083 ceph-mon[77242]: from='client.56448 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:07:58.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:07:58 smithi083 ceph-mon[77242]: pgmap v19188: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:07:58.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:07:58 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3211624364' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:08:00.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:08:00 smithi153 ceph-mon[70044]: pgmap v19189: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:00.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:08:00 smithi083 ceph-mon[79930]: pgmap v19189: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:00.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:08:00 smithi083 ceph-mon[77242]: pgmap v19189: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:01.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:08:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:08:01] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T16:08:02.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:08:02 smithi153 ceph-mon[70044]: pgmap v19190: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:08:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:08:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:08:02.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:08:02 smithi083 ceph-mon[77242]: pgmap v19190: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:02.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:08:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:08:02.576Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:08:02.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:08:02 smithi083 ceph-mon[79930]: pgmap v19190: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:03.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:08:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:08:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:08:04.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:08:04 smithi083 ceph-mon[79930]: pgmap v19191: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:04.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:08:04 smithi083 ceph-mon[77242]: pgmap v19191: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:04.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:08:04 smithi153 ceph-mon[70044]: pgmap v19191: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:06.210 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:08:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:08:05.917Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:08:06.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:08:06 smithi153 ceph-mon[70044]: pgmap v19192: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:06.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:08:06 smithi083 ceph-mon[79930]: pgmap v19192: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:06.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:08:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:08:06.667Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:08:06.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:08:06 smithi083 ceph-mon[77242]: pgmap v19192: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:08.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:08:08 smithi153 ceph-mon[70044]: pgmap v19193: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:08.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:08:08 smithi083 ceph-mon[79930]: pgmap v19193: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:08.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:08:08 smithi083 ceph-mon[77242]: pgmap v19193: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:10.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:08:10 smithi153 ceph-mon[70044]: pgmap v19194: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:10.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:08:10 smithi083 ceph-mon[79930]: pgmap v19194: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:10.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:08:10 smithi083 ceph-mon[77242]: pgmap v19194: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:08:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:08:11] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T16:08:12.814 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:08:12 smithi153 ceph-mon[70044]: pgmap v19195: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:12.815 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:08:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:08:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:08:12.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:08:12 smithi083 ceph-mon[79930]: pgmap v19195: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:12.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:08:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:08:12.577Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:08:12.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:08:12 smithi083 ceph-mon[77242]: pgmap v19195: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:13.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:08:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:08:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:08:14.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:08:14 smithi083 ceph-mon[79930]: pgmap v19196: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:14.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:08:14 smithi083 ceph-mon[77242]: pgmap v19196: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:14.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:08:14 smithi153 ceph-mon[70044]: pgmap v19196: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:16.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:08:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:08:15.919Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:08:16.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:08:16 smithi153 ceph-mon[70044]: pgmap v19197: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:16.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:08:16 smithi083 ceph-mon[79930]: pgmap v19197: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:16.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:08:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:08:16.667Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:08:16.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:08:16 smithi083 ceph-mon[77242]: pgmap v19197: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:18.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:08:18 smithi153 ceph-mon[70044]: pgmap v19198: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:18.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:08:18 smithi083 ceph-mon[79930]: pgmap v19198: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:18.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:08:18 smithi083 ceph-mon[77242]: pgmap v19198: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:20.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:08:20 smithi153 ceph-mon[70044]: pgmap v19199: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:20.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:08:20 smithi083 ceph-mon[77242]: pgmap v19199: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:20.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:08:20 smithi083 ceph-mon[79930]: pgmap v19199: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:08:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:08:21] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T16:08:22.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:08:22 smithi153 ceph-mon[70044]: pgmap v19200: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:08:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:08:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:08:22.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:08:22 smithi083 ceph-mon[79930]: pgmap v19200: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:22.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:08:22 smithi083 ceph-mon[77242]: pgmap v19200: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:22.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:08:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:08:22.577Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:08:23.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:08:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:08:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:08:24.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:08:24 smithi083 ceph-mon[79930]: pgmap v19201: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:24.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:08:24 smithi083 ceph-mon[77242]: pgmap v19201: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:24.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:08:24 smithi153 ceph-mon[70044]: pgmap v19201: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:26.210 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:08:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:08:25.919Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:08:26.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:08:26 smithi153 ceph-mon[70044]: pgmap v19202: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:26.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:08:26 smithi083 ceph-mon[79930]: pgmap v19202: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:26.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:08:26 smithi083 ceph-mon[77242]: pgmap v19202: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:26.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:08:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:08:26.669Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:08:28.109 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:08:28.399 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:08:28.399 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 7m ago 10h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:08:28.400 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 5m ago 10h 64.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:08:28.400 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 7m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:08:28.400 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 5m ago 10h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:08:28.400 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 5m ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:08:28.401 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 7m ago 10h 1052M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:08:28.401 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 7m ago 10h 2300M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:08:28.401 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 5m ago 10h 1030M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:08:28.401 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 7m ago 10h 1384M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:08:28.401 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 7m ago 10h 24.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:08:28.402 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 5m ago 10h 26.7M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:08:28.402 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 7m ago 10h 1673M 2105M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:08:28.402 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 7m ago 10h 1658M 2105M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:08:28.402 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 7m ago 10h 1288M 2105M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:08:28.403 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 7m ago 10h 1626M 2105M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:08:28.403 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 5m ago 10h 1425M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:08:28.403 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 5m ago 10h 1790M 2500M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:08:28.403 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 5m ago 10h 1600M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:08:28.404 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 5m ago 10h 1904M 2500M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:08:28.404 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 5m ago 10h 185M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:08:28.760 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:08:28.760 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:08:28.761 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:08:28.761 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:08:28.761 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:08:28.761 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:08:28.762 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:08:28.762 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:08:28.762 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:08:28.763 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:08:28.763 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:08:28.763 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:08:28.763 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:08:28.764 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:08:28.764 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:08:28.764 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:08:28.764 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:08:28.764 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:08:28.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:08:28 smithi153 ceph-mon[70044]: pgmap v19203: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:28.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:08:28 smithi083 ceph-mon[79930]: pgmap v19203: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:28.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:08:28 smithi083 ceph-mon[77242]: pgmap v19203: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:29.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:08:29 smithi153 ceph-mon[70044]: from='client.56460 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:08:29.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:08:29 smithi153 ceph-mon[70044]: from='client.56466 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:08:29.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:08:29 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3060203059' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:08:29.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:08:29 smithi083 ceph-mon[79930]: from='client.56460 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:08:29.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:08:29 smithi083 ceph-mon[79930]: from='client.56466 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:08:29.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:08:29 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3060203059' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:08:29.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:08:29 smithi083 ceph-mon[77242]: from='client.56460 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:08:29.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:08:29 smithi083 ceph-mon[77242]: from='client.56466 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:08:29.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:08:29 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3060203059' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:08:30.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:08:30 smithi153 ceph-mon[70044]: pgmap v19204: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:30.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:08:30 smithi083 ceph-mon[79930]: pgmap v19204: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:30.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:08:30 smithi083 ceph-mon[77242]: pgmap v19204: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:08:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:08:31] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T16:08:32.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:08:32 smithi153 ceph-mon[70044]: pgmap v19205: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:08:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:08:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:08:32.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:08:32 smithi083 ceph-mon[79930]: pgmap v19205: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:32.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:08:32 smithi083 ceph-mon[77242]: pgmap v19205: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:32.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:08:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:08:32.578Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:08:33.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:08:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:08:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:08:34.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:08:34 smithi083 ceph-mon[79930]: pgmap v19206: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:34.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:08:34 smithi083 ceph-mon[77242]: pgmap v19206: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:34.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:08:34 smithi153 ceph-mon[70044]: pgmap v19206: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:36.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:08:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:08:35.920Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:08:36.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:08:36 smithi153 ceph-mon[70044]: pgmap v19207: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:36.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:08:36 smithi083 ceph-mon[79930]: pgmap v19207: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:36.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:08:36 smithi083 ceph-mon[77242]: pgmap v19207: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:36.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:08:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:08:36.670Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:08:38.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:08:38 smithi153 ceph-mon[70044]: pgmap v19208: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:38.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:08:38 smithi083 ceph-mon[77242]: pgmap v19208: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:38.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:08:38 smithi083 ceph-mon[79930]: pgmap v19208: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:39.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:08:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:08:39.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:08:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:08:39.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:08:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:08:39.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:08:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:08:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:08:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:08:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:08:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:08:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:08:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:08:39.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:08:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:08:39.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:08:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:08:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:08:40 smithi153 ceph-mon[70044]: pgmap v19209: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:08:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:08:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:08:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:08:40.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:08:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:08:40.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:08:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:08:40.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:08:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:08:40.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:08:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:08:40.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:08:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:08:40.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:08:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:08:40.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:08:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:08:40.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:08:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:08:40.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:08:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:08:40.813 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:08:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:08:40.813 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:08:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:08:40.814 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:08:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:08:40.814 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:08:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:08:40.815 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:08:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:08:40.815 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:08:40 smithi153 ceph-mon[70044]: Adjusting osd_memory_target on smithi153 to 2498M 2022-09-29T16:08:40.815 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:08:40 smithi153 ceph-mon[70044]: Adjusting osd_memory_target on smithi083 to 2090M 2022-09-29T16:08:40.816 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:08:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:08:40.816 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:08:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:08:40.817 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:08:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:08:40.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:08:40 smithi083 ceph-mon[79930]: pgmap v19209: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:08:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:08:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:08:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:08:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:08:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:08:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:08:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:08:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:08:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:08:40.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:08:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:08:40.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:08:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:08:40.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:08:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:08:40.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:08:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:08:40.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:08:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:08:40.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:08:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:08:40.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:08:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:08:40.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:08:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:08:40.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:08:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:08:40.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:08:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:08:40.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:08:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:08:40.968 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:08:40 smithi083 ceph-mon[79930]: Adjusting osd_memory_target on smithi153 to 2498M 2022-09-29T16:08:40.968 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:08:40 smithi083 ceph-mon[79930]: Adjusting osd_memory_target on smithi083 to 2090M 2022-09-29T16:08:40.968 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:08:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:08:40.968 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:08:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:08:40.969 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:08:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:08:40.969 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:08:40 smithi083 ceph-mon[77242]: pgmap v19209: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:40.969 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:08:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:08:40.970 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:08:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:08:40.970 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:08:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:08:40.970 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:08:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:08:40.970 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:08:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:08:40.971 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:08:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:08:40.971 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:08:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:08:40.971 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:08:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:08:40.971 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:08:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:08:40.971 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:08:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:08:40.972 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:08:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:08:40.972 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:08:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:08:40.972 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:08:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:08:40.972 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:08:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:08:40.973 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:08:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:08:40.973 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:08:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:08:40.973 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:08:40 smithi083 ceph-mon[77242]: Adjusting osd_memory_target on smithi153 to 2498M 2022-09-29T16:08:40.973 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:08:40 smithi083 ceph-mon[77242]: Adjusting osd_memory_target on smithi083 to 2090M 2022-09-29T16:08:40.973 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:08:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:08:40.974 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:08:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:08:40.974 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:08:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:08:41.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:08:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:08:41] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T16:08:42.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:08:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:08:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:08:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:08:42 smithi153 ceph-mon[70044]: pgmap v19210: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:08:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:08:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:08:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:08:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:08:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:08:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:08:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:08:42.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:08:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:08:42.578Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:08:42.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:08:42 smithi083 ceph-mon[77242]: pgmap v19210: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:42.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:08:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:08:42.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:08:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:08:42.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:08:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:08:42.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:08:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:08:42.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:08:42 smithi083 ceph-mon[79930]: pgmap v19210: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:42.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:08:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:08:42.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:08:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:08:42.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:08:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:08:42.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:08:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:08:43.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:08:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:08:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:08:44.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:08:44 smithi083 ceph-mon[77242]: pgmap v19211: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:44.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:08:44 smithi083 ceph-mon[79930]: pgmap v19211: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:44.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:08:44 smithi153 ceph-mon[70044]: pgmap v19211: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:46.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:08:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:08:45.921Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:08:46.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:08:46 smithi153 ceph-mon[70044]: pgmap v19212: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:46.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:08:46 smithi083 ceph-mon[77242]: pgmap v19212: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:46.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:08:46 smithi083 ceph-mon[79930]: pgmap v19212: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:46.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:08:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:08:46.670Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:08:48.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:08:48 smithi153 ceph-mon[70044]: pgmap v19213: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:48.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:08:48 smithi083 ceph-mon[79930]: pgmap v19213: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:48.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:08:48 smithi083 ceph-mon[77242]: pgmap v19213: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:50.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:08:50 smithi153 ceph-mon[70044]: pgmap v19214: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:50.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:08:50 smithi083 ceph-mon[79930]: pgmap v19214: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:50.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:08:50 smithi083 ceph-mon[77242]: pgmap v19214: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:51.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:08:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:08:51] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T16:08:52.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:08:52 smithi153 ceph-mon[70044]: pgmap v19215: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:08:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:08:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:08:52.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:08:52 smithi083 ceph-mon[79930]: pgmap v19215: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:52.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:08:52 smithi083 ceph-mon[77242]: pgmap v19215: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:52.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:08:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:08:52.579Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:08:53.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:08:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:08:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:08:54.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:08:54 smithi083 ceph-mon[79930]: pgmap v19216: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:54.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:08:54 smithi083 ceph-mon[77242]: pgmap v19216: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:54.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:08:54 smithi153 ceph-mon[70044]: pgmap v19216: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:56.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:08:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:08:55.923Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:08:56.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:08:56 smithi153 ceph-mon[70044]: pgmap v19217: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:56.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:08:56 smithi083 ceph-mon[79930]: pgmap v19217: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:56.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:08:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:08:56.670Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:08:56.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:08:56 smithi083 ceph-mon[77242]: pgmap v19217: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:58.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:08:58 smithi153 ceph-mon[70044]: pgmap v19218: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:58.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:08:58 smithi083 ceph-mon[77242]: pgmap v19218: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:58.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:08:58 smithi083 ceph-mon[79930]: pgmap v19218: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:08:59.088 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:08:59.375 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:08:59.375 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 8m ago 10h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:08:59.376 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 6m ago 10h 64.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:08:59.376 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 8m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:08:59.376 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 6m ago 10h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:08:59.377 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 6m ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:08:59.377 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 8m ago 10h 1052M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:08:59.377 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 8m ago 10h 2300M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:08:59.378 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 6m ago 10h 1030M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:08:59.378 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 8m ago 10h 1384M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:08:59.379 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 8m ago 10h 24.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:08:59.379 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 6m ago 10h 26.7M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:08:59.380 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 8m ago 10h 1673M 2090M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:08:59.380 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 8m ago 10h 1658M 2090M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:08:59.381 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 8m ago 10h 1288M 2090M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:08:59.381 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 8m ago 10h 1626M 2090M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:08:59.382 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 6m ago 10h 1425M 2498M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:08:59.382 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 6m ago 10h 1790M 2498M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:08:59.383 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 6m ago 10h 1600M 2498M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:08:59.383 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 6m ago 10h 1904M 2498M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:08:59.384 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 6m ago 10h 185M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:08:59.734 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:08:59.734 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:08:59.734 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:08:59.734 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:08:59.735 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:08:59.735 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:08:59.735 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:08:59.735 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:08:59.736 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:08:59.736 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:08:59.736 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:08:59.736 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:08:59.736 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:08:59.737 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:08:59.737 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:08:59.737 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:08:59.737 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:08:59.738 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:09:00.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:09:00 smithi153 ceph-mon[70044]: from='client.56478 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:09:00.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:09:00 smithi153 ceph-mon[70044]: from='client.66415 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:09:00.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:09:00 smithi153 ceph-mon[70044]: pgmap v19219: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:09:00.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:09:00 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1454735369' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:09:00.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:09:00 smithi083 ceph-mon[79930]: from='client.56478 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:09:00.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:09:00 smithi083 ceph-mon[79930]: from='client.66415 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:09:00.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:09:00 smithi083 ceph-mon[79930]: pgmap v19219: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:09:00.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:09:00 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1454735369' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:09:00.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:09:00 smithi083 ceph-mon[77242]: from='client.56478 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:09:00.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:09:00 smithi083 ceph-mon[77242]: from='client.66415 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:09:00.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:09:00 smithi083 ceph-mon[77242]: pgmap v19219: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:09:00.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:09:00 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1454735369' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:09:01.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:09:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:09:01] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T16:09:02.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:09:02 smithi153 ceph-mon[70044]: pgmap v19220: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:09:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:09:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:09:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:09:02.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:09:02 smithi083 ceph-mon[79930]: pgmap v19220: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:09:02.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:09:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:09:02.579Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:09:02.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:09:02 smithi083 ceph-mon[77242]: pgmap v19220: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:09:03.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:09:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:09:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:09:04.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:09:04 smithi083 ceph-mon[79930]: pgmap v19221: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:09:04.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:09:04 smithi083 ceph-mon[77242]: pgmap v19221: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:09:04.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:09:04 smithi153 ceph-mon[70044]: pgmap v19221: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:09:06.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:09:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:09:05.923Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:09:06.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:09:06 smithi153 ceph-mon[70044]: pgmap v19222: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:09:06.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:09:06 smithi083 ceph-mon[79930]: pgmap v19222: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:09:06.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:09:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:09:06.671Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:09:06.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:09:06 smithi083 ceph-mon[77242]: pgmap v19222: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:09:08.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:09:08 smithi153 ceph-mon[70044]: pgmap v19223: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:09:08.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:09:08 smithi083 ceph-mon[79930]: pgmap v19223: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:09:08.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:09:08 smithi083 ceph-mon[77242]: pgmap v19223: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:09:10.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:09:10 smithi153 ceph-mon[70044]: pgmap v19224: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:09:10.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:09:10 smithi083 ceph-mon[79930]: pgmap v19224: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:09:10.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:09:10 smithi083 ceph-mon[77242]: pgmap v19224: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:09:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:09:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:09:11] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T16:09:12.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:09:12 smithi153 ceph-mon[70044]: pgmap v19225: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:09:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:09:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:09:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:09:12.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:09:12 smithi083 ceph-mon[79930]: pgmap v19225: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:09:12.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:09:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:09:12.580Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:09:12.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:09:12 smithi083 ceph-mon[77242]: pgmap v19225: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:09:13.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:09:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:09:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:09:14.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:09:14 smithi083 ceph-mon[79930]: pgmap v19226: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:09:14.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:09:14 smithi083 ceph-mon[77242]: pgmap v19226: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:09:14.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:09:14 smithi153 ceph-mon[70044]: pgmap v19226: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:09:16.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:09:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:09:15.923Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:09:16.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:09:16 smithi153 ceph-mon[70044]: pgmap v19227: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:09:16.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:09:16 smithi083 ceph-mon[79930]: pgmap v19227: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:09:16.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:09:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:09:16.672Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:09:16.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:09:16 smithi083 ceph-mon[77242]: pgmap v19227: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:09:18.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:09:18 smithi153 ceph-mon[70044]: pgmap v19228: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:09:18.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:09:18 smithi083 ceph-mon[77242]: pgmap v19228: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:09:18.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:09:18 smithi083 ceph-mon[79930]: pgmap v19228: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:09:20.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:09:20 smithi153 ceph-mon[70044]: pgmap v19229: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:09:20.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:09:20 smithi083 ceph-mon[79930]: pgmap v19229: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:09:20.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:09:20 smithi083 ceph-mon[77242]: pgmap v19229: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:09:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:09:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:09:21] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T16:09:22.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:09:22 smithi153 ceph-mon[70044]: pgmap v19230: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:09:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:09:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:09:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:09:22.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:09:22 smithi083 ceph-mon[79930]: pgmap v19230: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:09:22.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:09:22 smithi083 ceph-mon[77242]: pgmap v19230: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:09:22.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:09:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:09:22.580Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:09:23.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:09:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:09:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:09:24.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:09:24 smithi083 ceph-mon[79930]: pgmap v19231: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:09:24.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:09:24 smithi083 ceph-mon[77242]: pgmap v19231: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:09:24.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:09:24 smithi153 ceph-mon[70044]: pgmap v19231: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:09:26.210 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:09:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:09:25.925Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:09:26.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:09:26 smithi153 ceph-mon[70044]: pgmap v19232: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:09:26.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:09:26 smithi083 ceph-mon[79930]: pgmap v19232: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:09:26.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:09:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:09:26.672Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:09:26.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:09:26 smithi083 ceph-mon[77242]: pgmap v19232: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:09:28.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:09:28 smithi153 ceph-mon[70044]: pgmap v19233: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:09:28.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:09:28 smithi083 ceph-mon[79930]: pgmap v19233: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:09:28.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:09:28 smithi083 ceph-mon[77242]: pgmap v19233: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:09:30.061 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:09:30.349 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:09:30.349 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 9m ago 10h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:09:30.350 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 6m ago 10h 64.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:09:30.350 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 9m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:09:30.351 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 6m ago 10h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:09:30.351 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 6m ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:09:30.351 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 9m ago 10h 1052M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:09:30.352 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 9m ago 10h 2300M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:09:30.352 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 6m ago 10h 1030M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:09:30.353 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 9m ago 10h 1384M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:09:30.353 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 9m ago 10h 24.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:09:30.354 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 6m ago 10h 26.7M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:09:30.354 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 9m ago 10h 1673M 2090M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:09:30.354 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 9m ago 10h 1658M 2090M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:09:30.355 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 9m ago 10h 1288M 2090M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:09:30.355 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 9m ago 10h 1626M 2090M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:09:30.355 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 6m ago 10h 1425M 2498M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:09:30.356 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 6m ago 10h 1790M 2498M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:09:30.356 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 6m ago 10h 1600M 2498M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:09:30.356 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 6m ago 10h 1904M 2498M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:09:30.356 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 6m ago 10h 185M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:09:30.705 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:09:30.706 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:09:30.706 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:09:30.706 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:09:30.707 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:09:30.707 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:09:30.707 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:09:30.708 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:09:30.708 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:09:30.708 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:09:30.708 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:09:30.709 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:09:30.709 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:09:30.709 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:09:30.710 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:09:30.710 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:09:30.710 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:09:30.711 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:09:30.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:09:30 smithi153 ceph-mon[70044]: pgmap v19234: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:09:30.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:09:30 smithi083 ceph-mon[79930]: pgmap v19234: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:09:30.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:09:30 smithi083 ceph-mon[77242]: pgmap v19234: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:09:31.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:09:31 smithi153 ceph-mon[70044]: from='client.56496 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:09:31.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:09:31 smithi153 ceph-mon[70044]: from='client.56502 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:09:31.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:09:31 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1030308282' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:09:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:09:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:09:31] "GET /metrics HTTP/1.1" 200 225246 "" "Prometheus/2.33.4" 2022-09-29T16:09:31.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:09:31 smithi083 ceph-mon[77242]: from='client.56496 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:09:31.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:09:31 smithi083 ceph-mon[77242]: from='client.56502 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:09:31.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:09:31 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1030308282' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:09:31.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:09:31 smithi083 ceph-mon[79930]: from='client.56496 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:09:31.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:09:31 smithi083 ceph-mon[79930]: from='client.56502 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:09:31.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:09:31 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1030308282' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:09:32.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:09:32 smithi153 ceph-mon[70044]: pgmap v19235: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:09:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:09:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:09:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:09:32.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:09:32 smithi083 ceph-mon[79930]: pgmap v19235: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:09:32.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:09:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:09:32.582Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:09:32.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:09:32 smithi083 ceph-mon[77242]: pgmap v19235: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:09:33.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:09:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:09:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:09:34.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:09:34 smithi083 ceph-mon[79930]: pgmap v19236: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:09:34.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:09:34 smithi083 ceph-mon[77242]: pgmap v19236: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:09:34.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:09:34 smithi153 ceph-mon[70044]: pgmap v19236: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:09:36.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:09:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:09:35.925Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:09:36.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:09:36 smithi153 ceph-mon[70044]: pgmap v19237: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:09:36.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:09:36 smithi083 ceph-mon[77242]: pgmap v19237: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:09:36.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:09:36 smithi083 ceph-mon[79930]: pgmap v19237: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:09:36.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:09:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:09:36.673Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:09:38.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:09:38 smithi153 ceph-mon[70044]: pgmap v19238: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:09:38.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:09:38 smithi083 ceph-mon[79930]: pgmap v19238: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:09:38.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:09:38 smithi083 ceph-mon[77242]: pgmap v19238: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:09:40.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:09:40 smithi153 ceph-mon[70044]: pgmap v19239: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:09:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:09:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:09:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:09:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:09:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:09:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:09:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:09:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:09:40.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:09:40 smithi083 ceph-mon[79930]: pgmap v19239: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:09:40.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:09:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:09:40.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:09:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:09:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:09:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:09:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:09:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:09:40.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:09:40 smithi083 ceph-mon[77242]: pgmap v19239: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:09:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:09:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:09:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:09:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:09:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:09:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:09:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:09:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:09:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:09:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:09:41] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T16:09:42.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:09:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:09:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:09:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:09:42 smithi153 ceph-mon[70044]: pgmap v19240: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:09:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:09:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:09:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:09:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:09:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:09:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:09:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:09:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:09:42.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:09:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:09:42.582Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:09:42.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:09:42 smithi083 ceph-mon[79930]: pgmap v19240: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:09:42.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:09:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:09:42.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:09:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:09:42.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:09:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:09:42.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:09:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:09:42.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:09:42 smithi083 ceph-mon[77242]: pgmap v19240: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:09:42.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:09:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:09:42.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:09:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:09:42.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:09:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:09:42.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:09:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:09:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:09:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:09:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:09:44.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:09:44 smithi083 ceph-mon[79930]: pgmap v19241: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:09:44.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:09:44 smithi083 ceph-mon[77242]: pgmap v19241: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:09:44.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:09:44 smithi153 ceph-mon[70044]: pgmap v19241: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:09:46.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:09:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:09:45.926Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:09:46.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:09:46 smithi153 ceph-mon[70044]: pgmap v19242: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:09:46.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:09:46 smithi083 ceph-mon[79930]: pgmap v19242: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:09:46.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:09:46 smithi083 ceph-mon[77242]: pgmap v19242: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:09:46.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:09:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:09:46.674Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:09:48.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:09:48 smithi153 ceph-mon[70044]: pgmap v19243: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:09:48.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:09:48 smithi083 ceph-mon[79930]: pgmap v19243: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:09:48.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:09:48 smithi083 ceph-mon[77242]: pgmap v19243: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:09:50.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:09:50 smithi153 ceph-mon[70044]: pgmap v19244: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:09:50.960 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:09:50 smithi083 ceph-mon[77242]: pgmap v19244: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:09:50.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:09:50 smithi083 ceph-mon[79930]: pgmap v19244: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:09:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:09:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:09:51] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T16:09:52.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:09:52 smithi153 ceph-mon[70044]: pgmap v19245: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:09:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:09:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:09:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:09:52.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:09:52 smithi083 ceph-mon[79930]: pgmap v19245: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:09:52.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:09:52 smithi083 ceph-mon[77242]: pgmap v19245: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:09:52.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:09:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:09:52.583Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:09:53.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:09:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:09:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:09:54.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:09:54 smithi083 ceph-mon[77242]: pgmap v19246: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:09:54.467 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:09:54 smithi083 ceph-mon[79930]: pgmap v19246: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:09:54.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:09:54 smithi153 ceph-mon[70044]: pgmap v19246: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:09:56.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:09:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:09:55.926Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:09:56.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:09:56 smithi153 ceph-mon[70044]: pgmap v19247: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:09:56.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:09:56 smithi083 ceph-mon[79930]: pgmap v19247: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:09:56.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:09:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:09:56.674Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:09:56.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:09:56 smithi083 ceph-mon[77242]: pgmap v19247: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:09:58.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:09:58 smithi153 ceph-mon[70044]: pgmap v19248: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:09:58.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:09:58 smithi083 ceph-mon[77242]: pgmap v19248: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:09:58.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:09:58 smithi083 ceph-mon[79930]: pgmap v19248: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:00.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:10:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[77219]: 2022-09-29T16:10:00.000+0000 7f160f665700 -1 log_channel(cluster) log [ERR] : overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T16:10:00.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:10:00 smithi153 ceph-mon[70044]: pgmap v19249: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:00.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:10:00 smithi153 ceph-mon[70044]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T16:10:00.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:10:00 smithi083 ceph-mon[79930]: pgmap v19249: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:00.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:10:00 smithi083 ceph-mon[79930]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T16:10:00.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:10:00 smithi083 ceph-mon[77242]: pgmap v19249: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:00.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:10:00 smithi083 ceph-mon[77242]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T16:10:01.036 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:10:01.327 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:10:01.327 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 9m ago 10h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:10:01.327 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 7m ago 10h 64.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:10:01.328 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 9m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:10:01.328 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 7m ago 10h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:10:01.328 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 7m ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:10:01.329 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 9m ago 10h 1052M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:10:01.329 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 9m ago 10h 2300M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:10:01.329 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 7m ago 10h 1030M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:10:01.329 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 9m ago 10h 1384M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:10:01.330 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 9m ago 10h 24.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:10:01.330 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 7m ago 10h 26.7M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:10:01.330 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 9m ago 10h 1673M 2090M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:10:01.331 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 9m ago 10h 1658M 2090M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:10:01.331 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 9m ago 10h 1288M 2090M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:10:01.331 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 9m ago 10h 1626M 2090M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:10:01.331 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 7m ago 10h 1425M 2498M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:10:01.332 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 7m ago 10h 1790M 2498M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:10:01.332 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 7m ago 10h 1600M 2498M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:10:01.332 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 7m ago 10h 1904M 2498M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:10:01.332 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 7m ago 10h 185M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:10:01.686 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:10:01.686 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:10:01.687 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:10:01.687 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:10:01.687 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:10:01.688 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:10:01.688 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:10:01.689 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:10:01.689 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:10:01.690 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:10:01.690 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:10:01.690 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:10:01.691 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:10:01.691 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:10:01.691 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:10:01.691 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:10:01.692 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:10:01.692 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:10:01.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:10:01 smithi153 ceph-mon[70044]: from='client.66445 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:10:01.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:10:01 smithi083 ceph-mon[77242]: from='client.66445 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:10:01.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:10:01 smithi083 ceph-mon[79930]: from='client.66445 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:10:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:10:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:10:01] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T16:10:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:10:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:10:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:10:02.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:10:02 smithi153 ceph-mon[70044]: from='client.56520 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:10:02.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:10:02 smithi153 ceph-mon[70044]: pgmap v19250: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:02.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:10:02 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1315954959' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:10:02.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:10:02 smithi083 ceph-mon[79930]: from='client.56520 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:10:02.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:10:02 smithi083 ceph-mon[79930]: pgmap v19250: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:02.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:10:02 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1315954959' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:10:02.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:10:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:10:02.584Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:10:02.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:10:02 smithi083 ceph-mon[77242]: from='client.56520 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:10:02.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:10:02 smithi083 ceph-mon[77242]: pgmap v19250: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:02.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:10:02 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1315954959' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:10:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:10:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:10:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:10:04.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:10:04 smithi083 ceph-mon[79930]: pgmap v19251: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:04.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:10:04 smithi083 ceph-mon[77242]: pgmap v19251: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:04.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:10:04 smithi153 ceph-mon[70044]: pgmap v19251: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:06.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:10:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:10:05.926Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:10:06.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:10:06 smithi153 ceph-mon[70044]: pgmap v19252: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:06.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:10:06 smithi083 ceph-mon[79930]: pgmap v19252: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:06.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:10:06 smithi083 ceph-mon[77242]: pgmap v19252: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:06.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:10:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:10:06.674Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:10:08.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:10:08 smithi153 ceph-mon[70044]: pgmap v19253: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:08.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:10:08 smithi083 ceph-mon[77242]: pgmap v19253: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:08.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:10:08 smithi083 ceph-mon[79930]: pgmap v19253: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:10.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:10:10 smithi153 ceph-mon[70044]: pgmap v19254: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:10.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:10:10 smithi083 ceph-mon[79930]: pgmap v19254: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:10.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:10:10 smithi083 ceph-mon[77242]: pgmap v19254: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:11.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:10:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:10:11] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T16:10:12.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:10:12 smithi153 ceph-mon[70044]: pgmap v19255: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:10:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:10:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:10:12.971 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:10:12 smithi083 ceph-mon[79930]: pgmap v19255: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:12.971 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:10:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:10:12.585Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:10:12.972 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:10:12 smithi083 ceph-mon[77242]: pgmap v19255: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:13.314 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:10:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:10:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:10:14.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:10:14 smithi083 ceph-mon[79930]: pgmap v19256: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:14.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:10:14 smithi083 ceph-mon[77242]: pgmap v19256: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:14.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:10:14 smithi153 ceph-mon[70044]: pgmap v19256: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:16.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:10:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:10:15.927Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:10:16.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:10:16 smithi153 ceph-mon[70044]: pgmap v19257: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:16.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:10:16 smithi083 ceph-mon[79930]: pgmap v19257: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:16.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:10:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:10:16.675Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:10:16.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:10:16 smithi083 ceph-mon[77242]: pgmap v19257: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:18.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:10:18 smithi153 ceph-mon[70044]: pgmap v19258: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:18.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:10:18 smithi083 ceph-mon[79930]: pgmap v19258: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:18.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:10:18 smithi083 ceph-mon[77242]: pgmap v19258: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:20.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:10:20 smithi153 ceph-mon[70044]: pgmap v19259: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:20.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:10:20 smithi083 ceph-mon[79930]: pgmap v19259: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:20.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:10:20 smithi083 ceph-mon[77242]: pgmap v19259: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:10:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:10:21] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T16:10:22.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:10:22 smithi153 ceph-mon[70044]: pgmap v19260: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:10:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:10:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:10:22.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:10:22 smithi083 ceph-mon[79930]: pgmap v19260: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:22.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:10:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:10:22.585Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:10:22.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:10:22 smithi083 ceph-mon[77242]: pgmap v19260: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:23.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:10:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:10:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:10:24.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:10:24 smithi083 ceph-mon[79930]: pgmap v19261: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:24.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:10:24 smithi083 ceph-mon[77242]: pgmap v19261: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:10:24 smithi153 ceph-mon[70044]: pgmap v19261: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:26.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:10:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:10:25.928Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:10:26.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:10:26 smithi153 ceph-mon[70044]: pgmap v19262: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:26.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:10:26 smithi083 ceph-mon[79930]: pgmap v19262: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:26.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:10:26 smithi083 ceph-mon[77242]: pgmap v19262: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:26.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:10:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:10:26.676Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:10:28.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:10:28 smithi153 ceph-mon[70044]: pgmap v19263: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:28.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:10:28 smithi083 ceph-mon[79930]: pgmap v19263: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:28.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:10:28 smithi083 ceph-mon[77242]: pgmap v19263: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:30.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:10:30 smithi153 ceph-mon[70044]: pgmap v19264: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:30.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:10:30 smithi083 ceph-mon[79930]: pgmap v19264: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:30.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:10:30 smithi083 ceph-mon[77242]: pgmap v19264: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:10:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:10:31] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T16:10:32.014 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:10:32.301 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:10:32.301 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 10m ago 10h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:10:32.302 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 7m ago 10h 64.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:10:32.302 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 10m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:10:32.303 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 7m ago 10h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:10:32.303 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 7m ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:10:32.304 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 10m ago 10h 1052M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:10:32.304 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 10m ago 10h 2300M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:10:32.304 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 7m ago 10h 1030M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:10:32.305 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 10m ago 10h 1384M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:10:32.305 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 10m ago 10h 24.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:10:32.306 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 7m ago 10h 26.7M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:10:32.306 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 10m ago 10h 1673M 2090M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:10:32.307 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 10m ago 10h 1658M 2090M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:10:32.307 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 10m ago 10h 1288M 2090M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:10:32.307 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 10m ago 10h 1626M 2090M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:10:32.308 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 7m ago 10h 1425M 2498M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:10:32.308 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 7m ago 10h 1790M 2498M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:10:32.309 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 7m ago 10h 1600M 2498M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:10:32.309 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 7m ago 10h 1904M 2498M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:10:32.310 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 7m ago 10h 185M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:10:32.657 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:10:32.657 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:10:32.657 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:10:32.658 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:10:32.658 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:10:32.658 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:10:32.658 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:10:32.659 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:10:32.659 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:10:32.659 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:10:32.659 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:10:32.660 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:10:32.660 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:10:32.660 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:10:32.660 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:10:32.661 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:10:32.661 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:10:32.661 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:10:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:10:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:10:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:10:32.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:10:32 smithi153 ceph-mon[70044]: pgmap v19265: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:32.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:10:32 smithi153 ceph-mon[70044]: from='client.56532 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:10:32.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:10:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:10:32.585Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:10:32.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:10:32 smithi083 ceph-mon[77242]: pgmap v19265: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:32.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:10:32 smithi083 ceph-mon[77242]: from='client.56532 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:10:32.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:10:32 smithi083 ceph-mon[79930]: pgmap v19265: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:32.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:10:32 smithi083 ceph-mon[79930]: from='client.56532 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:10:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:10:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:10:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:10:33.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:10:33 smithi153 ceph-mon[70044]: from='client.56538 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:10:33.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:10:33 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3242810710' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:10:33.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:10:33 smithi083 ceph-mon[77242]: from='client.56538 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:10:33.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:10:33 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3242810710' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:10:33.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:10:33 smithi083 ceph-mon[79930]: from='client.56538 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:10:33.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:10:33 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3242810710' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:10:34.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:10:34 smithi153 ceph-mon[70044]: pgmap v19266: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:34.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:10:34 smithi083 ceph-mon[79930]: pgmap v19266: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:34.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:10:34 smithi083 ceph-mon[77242]: pgmap v19266: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:36.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:10:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:10:35.928Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:10:36.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:10:36 smithi153 ceph-mon[70044]: pgmap v19267: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:36.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:10:36 smithi083 ceph-mon[79930]: pgmap v19267: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:36.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:10:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:10:36.677Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:10:36.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:10:36 smithi083 ceph-mon[77242]: pgmap v19267: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:38.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:10:38 smithi083 ceph-mon[77242]: pgmap v19268: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:38.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:10:38 smithi083 ceph-mon[79930]: pgmap v19268: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:39.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:10:38 smithi153 ceph-mon[70044]: pgmap v19268: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:40.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:10:40 smithi083 ceph-mon[79930]: pgmap v19269: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:40.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:10:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:10:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:10:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:10:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:10:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:10:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:10:40 smithi083 ceph-mon[77242]: pgmap v19269: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:10:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:10:40.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:10:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:10:40.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:10:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:10:41.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:10:40 smithi153 ceph-mon[70044]: pgmap v19269: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:10:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:10:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:10:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:10:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:10:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:10:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:10:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:10:41] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T16:10:41.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:10:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:10:41.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:10:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:10:41.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:10:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:10:41.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:10:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:10:41.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:10:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:10:41.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:10:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:10:41.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:10:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:10:41.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:10:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:10:42.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:10:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:10:42.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:10:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:10:42.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:10:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:10:42.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:10:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:10:42.756 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:10:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:10:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:10:42.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:10:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:10:42.585Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:10:42.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:10:42 smithi083 ceph-mon[77242]: pgmap v19270: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:42.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:10:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:10:42.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:10:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:10:42.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:10:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:10:42.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:10:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:10:42.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:10:42 smithi083 ceph-mon[79930]: pgmap v19270: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:42.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:10:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:10:42.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:10:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:10:42.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:10:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:10:42.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:10:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:10:43.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:10:42 smithi153 ceph-mon[70044]: pgmap v19270: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:43.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:10:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:10:43.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:10:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:10:43.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:10:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:10:43.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:10:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:10:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:10:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:10:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:10:44.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:10:44 smithi153 ceph-mon[70044]: pgmap v19271: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:44.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:10:44 smithi083 ceph-mon[79930]: pgmap v19271: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:44.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:10:44 smithi083 ceph-mon[77242]: pgmap v19271: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:46.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:10:45 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:10:46.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:10:45 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:10:46.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:10:45 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:10:46.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:10:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:10:45.928Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:10:46.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:10:45 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:10:46.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:10:45 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:10:46.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:10:45 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:10:46.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:10:45 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:10:46.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:10:45 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:10:46.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:10:45 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:10:46.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:10:46 smithi083 ceph-mon[79930]: pgmap v19272: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:46.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:10:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:10:46.677Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:10:46.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:10:46 smithi083 ceph-mon[77242]: pgmap v19272: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:47.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:10:46 smithi153 ceph-mon[70044]: pgmap v19272: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:49.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:10:48 smithi153 ceph-mon[70044]: pgmap v19273: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:49.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:10:48 smithi083 ceph-mon[77242]: pgmap v19273: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:49.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:10:48 smithi083 ceph-mon[79930]: pgmap v19273: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:51.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:10:50 smithi153 ceph-mon[70044]: pgmap v19274: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:51.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:10:50 smithi083 ceph-mon[77242]: pgmap v19274: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:51.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:10:50 smithi083 ceph-mon[79930]: pgmap v19274: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:10:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:10:51] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T16:10:52.736 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:10:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:10:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:10:52.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:10:52 smithi083 ceph-mon[77242]: pgmap v19275: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:52.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:10:52 smithi083 ceph-mon[79930]: pgmap v19275: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:52.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:10:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:10:52.586Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:10:53.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:10:52 smithi153 ceph-mon[70044]: pgmap v19275: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:53.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:10:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:10:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:10:54.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:10:54 smithi153 ceph-mon[70044]: pgmap v19276: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:54.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:10:54 smithi083 ceph-mon[79930]: pgmap v19276: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:54.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:10:54 smithi083 ceph-mon[77242]: pgmap v19276: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:56.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:10:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:10:55.929Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:10:56.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:10:56 smithi153 ceph-mon[70044]: pgmap v19277: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:56.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:10:56 smithi083 ceph-mon[77242]: pgmap v19277: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:56.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:10:56 smithi083 ceph-mon[79930]: pgmap v19277: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:56.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:10:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:10:56.677Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:10:58.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:10:58 smithi153 ceph-mon[70044]: pgmap v19278: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:58.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:10:58 smithi083 ceph-mon[79930]: pgmap v19278: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:10:58.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:10:58 smithi083 ceph-mon[77242]: pgmap v19278: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:00.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:11:00 smithi083 ceph-mon[79930]: pgmap v19279: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:00.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:11:00 smithi083 ceph-mon[77242]: pgmap v19279: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:01.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:11:00 smithi153 ceph-mon[70044]: pgmap v19279: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:11:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:11:01] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T16:11:02.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:11:02 smithi153 ceph-mon[70044]: pgmap v19280: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:11:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:11:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:11:02.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:11:02 smithi083 ceph-mon[79930]: pgmap v19280: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:02.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:11:02 smithi083 ceph-mon[77242]: pgmap v19280: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:02.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:11:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:11:02.587Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:11:02.991 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:11:03.279 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:11:03.279 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 18s ago 10h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:11:03.280 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 8m ago 10h 64.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:11:03.280 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 18s ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:11:03.280 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 8m ago 10h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:11:03.280 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 8m ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:11:03.281 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 18s ago 10h 1060M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:11:03.281 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 18s ago 10h 2335M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:11:03.281 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 8m ago 10h 1030M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:11:03.281 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 18s ago 10h 1404M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:11:03.282 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 18s ago 10h 24.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:11:03.282 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 8m ago 10h 26.7M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:11:03.282 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 18s ago 10h 1698M 2090M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:11:03.282 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 18s ago 10h 1685M 2090M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:11:03.282 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 18s ago 10h 1308M 2090M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:11:03.283 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 18s ago 10h 1651M 2090M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:11:03.283 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 8m ago 10h 1425M 2498M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:11:03.283 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 8m ago 10h 1790M 2498M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:11:03.284 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 8m ago 10h 1600M 2498M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:11:03.284 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 8m ago 10h 1904M 2498M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:11:03.284 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 8m ago 10h 185M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:11:03.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:11:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:11:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:11:03.655 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:11:03.655 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:11:03.656 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:11:03.656 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:11:03.656 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:11:03.656 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:11:03.656 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:11:03.657 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:11:03.657 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:11:03.657 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:11:03.657 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:11:03.658 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:11:03.658 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:11:03.658 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:11:03.659 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:11:03.659 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:11:03.659 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:11:03.660 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:11:03.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:11:03 smithi083 ceph-mon[77242]: from='client.56550 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:11:03.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:11:03 smithi083 ceph-mon[79930]: from='client.56550 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:11:04.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:11:03 smithi153 ceph-mon[70044]: from='client.56550 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:11:04.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:11:04 smithi083 ceph-mon[77242]: from='client.56556 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:11:04.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:11:04 smithi083 ceph-mon[77242]: pgmap v19281: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:04.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:11:04 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3431162438' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:11:04.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:11:04 smithi083 ceph-mon[79930]: from='client.56556 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:11:04.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:11:04 smithi083 ceph-mon[79930]: pgmap v19281: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:04.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:11:04 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3431162438' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:11:05.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:11:04 smithi153 ceph-mon[70044]: from='client.56556 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:11:05.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:11:04 smithi153 ceph-mon[70044]: pgmap v19281: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:05.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:11:04 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3431162438' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:11:06.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:11:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:11:05.930Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:11:06.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:11:06 smithi083 ceph-mon[79930]: pgmap v19282: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:06.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:11:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:11:06.678Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:11:06.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:11:06 smithi083 ceph-mon[77242]: pgmap v19282: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:07.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:11:06 smithi153 ceph-mon[70044]: pgmap v19282: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:08.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:11:08 smithi083 ceph-mon[79930]: pgmap v19283: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:08.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:11:08 smithi083 ceph-mon[77242]: pgmap v19283: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:09.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:11:08 smithi153 ceph-mon[70044]: pgmap v19283: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:10.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:11:10 smithi083 ceph-mon[77242]: pgmap v19284: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:10.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:11:10 smithi083 ceph-mon[79930]: pgmap v19284: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:11.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:11:10 smithi153 ceph-mon[70044]: pgmap v19284: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:11:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:11:11] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T16:11:12.580 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:11:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:11:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:11:12.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:11:12 smithi083 ceph-mon[79930]: pgmap v19285: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:12.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:11:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:11:12.587Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:11:12.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:11:12 smithi083 ceph-mon[77242]: pgmap v19285: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:13.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:11:12 smithi153 ceph-mon[70044]: pgmap v19285: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:13.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:11:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:11:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:11:14.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:11:14 smithi153 ceph-mon[70044]: pgmap v19286: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:14.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:11:14 smithi083 ceph-mon[79930]: pgmap v19286: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:14.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:11:14 smithi083 ceph-mon[77242]: pgmap v19286: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:16.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:11:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:11:15.930Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:11:16.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:11:16 smithi083 ceph-mon[79930]: pgmap v19287: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:16.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:11:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:11:16.679Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:11:16.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:11:16 smithi083 ceph-mon[77242]: pgmap v19287: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:17.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:11:16 smithi153 ceph-mon[70044]: pgmap v19287: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:18.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:11:18 smithi083 ceph-mon[79930]: pgmap v19288: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:18.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:11:18 smithi083 ceph-mon[77242]: pgmap v19288: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:19.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:11:18 smithi153 ceph-mon[70044]: pgmap v19288: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:20.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:11:20 smithi083 ceph-mon[79930]: pgmap v19289: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:20.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:11:20 smithi083 ceph-mon[77242]: pgmap v19289: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:21.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:11:20 smithi153 ceph-mon[70044]: pgmap v19289: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:11:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:11:21] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T16:11:22.565 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:11:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:11:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:11:22.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:11:22 smithi083 ceph-mon[79930]: pgmap v19290: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:22.968 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:11:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:11:22.588Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:11:22.969 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:11:22 smithi083 ceph-mon[77242]: pgmap v19290: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:23.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:11:22 smithi153 ceph-mon[70044]: pgmap v19290: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:11:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:11:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:11:24.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:11:24 smithi153 ceph-mon[70044]: pgmap v19291: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:24.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:11:24 smithi083 ceph-mon[79930]: pgmap v19291: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:24.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:11:24 smithi083 ceph-mon[77242]: pgmap v19291: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:26.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:11:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:11:25.931Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:11:26.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:11:26 smithi083 ceph-mon[79930]: pgmap v19292: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:26.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:11:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:11:26.679Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:11:26.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:11:26 smithi083 ceph-mon[77242]: pgmap v19292: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:27.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:11:26 smithi153 ceph-mon[70044]: pgmap v19292: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:28.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:11:28 smithi083 ceph-mon[77242]: pgmap v19293: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:28.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:11:28 smithi083 ceph-mon[79930]: pgmap v19293: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:29.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:11:28 smithi153 ceph-mon[70044]: pgmap v19293: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:30.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:11:30 smithi083 ceph-mon[79930]: pgmap v19294: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:30.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:11:30 smithi083 ceph-mon[77242]: pgmap v19294: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:31.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:11:30 smithi153 ceph-mon[70044]: pgmap v19294: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:11:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:11:31] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T16:11:32.568 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:11:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:11:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:11:32.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:11:32 smithi083 ceph-mon[79930]: pgmap v19295: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:32.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:11:32 smithi083 ceph-mon[77242]: pgmap v19295: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:32.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:11:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:11:32.589Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:11:33.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:11:32 smithi153 ceph-mon[70044]: pgmap v19295: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:11:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:11:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:11:33.984 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:11:34.272 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:11:34.272 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 49s ago 10h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:11:34.273 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 8m ago 10h 64.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:11:34.273 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 49s ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:11:34.273 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 8m ago 10h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:11:34.273 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 8m ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:11:34.274 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 49s ago 10h 1060M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:11:34.274 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 49s ago 10h 2335M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:11:34.274 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 8m ago 10h 1030M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:11:34.274 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 49s ago 10h 1404M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:11:34.275 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 49s ago 10h 24.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:11:34.275 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 8m ago 10h 26.7M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:11:34.275 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 49s ago 10h 1698M 2090M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:11:34.275 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 49s ago 10h 1685M 2090M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:11:34.276 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 49s ago 10h 1308M 2090M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:11:34.276 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 49s ago 10h 1651M 2090M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:11:34.276 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 8m ago 10h 1425M 2498M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:11:34.276 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 8m ago 10h 1790M 2498M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:11:34.277 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 8m ago 10h 1600M 2498M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:11:34.277 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 8m ago 10h 1904M 2498M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:11:34.277 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 8m ago 10h 185M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:11:34.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:11:34 smithi153 ceph-mon[70044]: pgmap v19296: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:11:34 smithi153 ceph-mon[70044]: from='client.66499 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:11:34.630 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:11:34.631 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:11:34.631 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:11:34.631 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:11:34.631 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:11:34.632 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:11:34.632 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:11:34.632 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:11:34.632 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:11:34.635 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:11:34.635 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:11:34.635 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:11:34.636 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:11:34.643 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:11:34.643 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:11:34.644 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:11:34.644 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:11:34.645 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:11:34.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:11:34 smithi083 ceph-mon[79930]: pgmap v19296: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:34.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:11:34 smithi083 ceph-mon[79930]: from='client.66499 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:11:34.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:11:34 smithi083 ceph-mon[77242]: pgmap v19296: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:34.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:11:34 smithi083 ceph-mon[77242]: from='client.66499 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:11:35.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:11:35 smithi153 ceph-mon[70044]: from='client.56574 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:11:35.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:11:35 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2177061584' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:11:35.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:11:35 smithi083 ceph-mon[79930]: from='client.56574 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:11:35.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:11:35 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2177061584' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:11:35.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:11:35 smithi083 ceph-mon[77242]: from='client.56574 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:11:35.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:11:35 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2177061584' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:11:36.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:11:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:11:35.931Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:11:36.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:11:36 smithi153 ceph-mon[70044]: pgmap v19297: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:36.681 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:11:36 smithi083 ceph-mon[79930]: pgmap v19297: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:36.681 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:11:36 smithi083 ceph-mon[77242]: pgmap v19297: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:36.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:11:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:11:36.680Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:11:38.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:11:38 smithi083 ceph-mon[79930]: pgmap v19298: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:38.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:11:38 smithi083 ceph-mon[77242]: pgmap v19298: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:39.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:11:38 smithi153 ceph-mon[70044]: pgmap v19298: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:40.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:11:40 smithi083 ceph-mon[79930]: pgmap v19299: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:40.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:11:40 smithi083 ceph-mon[77242]: pgmap v19299: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:11:40 smithi153 ceph-mon[70044]: pgmap v19299: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:11:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:11:41] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T16:11:42.570 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:11:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:11:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:11:42.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:11:42 smithi083 ceph-mon[79930]: pgmap v19300: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:42.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:11:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:11:42.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:11:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:11:42.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:11:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:11:42.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:11:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:11:42.964 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:11:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:11:42.589Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:11:42.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:11:42 smithi083 ceph-mon[77242]: pgmap v19300: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:42.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:11:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:11:42.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:11:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:11:42.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:11:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:11:42.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:11:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:11:43.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:11:42 smithi153 ceph-mon[70044]: pgmap v19300: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:43.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:11:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:11:43.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:11:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:11:43.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:11:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:11:43.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:11:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:11:43.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:11:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:11:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:11:44.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:11:44 smithi153 ceph-mon[70044]: pgmap v19301: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:44.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:11:44 smithi083 ceph-mon[77242]: pgmap v19301: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:44.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:11:44 smithi083 ceph-mon[79930]: pgmap v19301: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:45.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:11:45 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:11:45.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:11:45 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:11:45.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:11:45 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:11:45.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:11:45 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:11:45.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:11:45 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:11:45.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:11:45 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:11:45.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:11:45 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:11:45.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:11:45 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:11:45.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:11:45 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:11:46.210 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:11:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:11:45.931Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:11:46.969 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:11:46 smithi083 ceph-mon[77242]: pgmap v19302: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:46.970 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:11:46 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:11:46.970 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:11:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:11:46.680Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:11:46.971 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:11:46 smithi083 ceph-mon[79930]: pgmap v19302: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:46.971 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:11:46 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:11:47.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:11:46 smithi153 ceph-mon[70044]: pgmap v19302: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:47.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:11:46 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:11:49.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:11:48 smithi083 ceph-mon[79930]: pgmap v19303: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:49.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:11:48 smithi083 ceph-mon[77242]: pgmap v19303: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:49.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:11:48 smithi153 ceph-mon[70044]: pgmap v19303: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:51.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:11:50 smithi083 ceph-mon[79930]: pgmap v19304: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:51.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:11:50 smithi083 ceph-mon[77242]: pgmap v19304: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:51.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:11:50 smithi153 ceph-mon[70044]: pgmap v19304: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:51.965 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:11:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:11:51] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T16:11:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:11:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:11:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:11:52.910 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:11:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:11:52.590Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:11:53.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:11:52 smithi083 ceph-mon[79930]: pgmap v19305: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:53.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:11:52 smithi083 ceph-mon[77242]: pgmap v19305: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:53.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:11:52 smithi153 ceph-mon[70044]: pgmap v19305: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:11:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:11:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:11:54.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:11:54 smithi153 ceph-mon[70044]: pgmap v19306: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:54.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:11:54 smithi083 ceph-mon[77242]: pgmap v19306: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:54.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:11:54 smithi083 ceph-mon[79930]: pgmap v19306: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:56.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:11:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:11:55.932Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:11:56.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:11:56 smithi083 ceph-mon[79930]: pgmap v19307: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:56.964 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:11:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:11:56.681Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:11:56.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:11:56 smithi083 ceph-mon[77242]: pgmap v19307: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:57.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:11:56 smithi153 ceph-mon[70044]: pgmap v19307: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:58.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:11:58 smithi083 ceph-mon[79930]: pgmap v19308: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:58.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:11:58 smithi083 ceph-mon[77242]: pgmap v19308: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:11:59.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:11:58 smithi153 ceph-mon[70044]: pgmap v19308: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:00.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:12:00 smithi083 ceph-mon[79930]: pgmap v19309: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:00.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:12:00 smithi083 ceph-mon[77242]: pgmap v19309: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:01.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:12:00 smithi153 ceph-mon[70044]: pgmap v19309: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:12:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:12:01] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T16:12:02.580 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:12:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:12:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:12:02.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:12:02 smithi083 ceph-mon[79930]: pgmap v19310: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:02.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:12:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:12:02.591Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:12:02.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:12:02 smithi083 ceph-mon[77242]: pgmap v19310: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:03.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:12:02 smithi153 ceph-mon[70044]: pgmap v19310: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:12:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:12:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:12:04.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:12:04 smithi153 ceph-mon[70044]: pgmap v19311: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:04.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:12:04 smithi083 ceph-mon[79930]: pgmap v19311: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:04.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:12:04 smithi083 ceph-mon[77242]: pgmap v19311: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:04.959 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:12:05.249 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:12:05.249 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 80s ago 10h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:12:05.250 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 9m ago 10h 64.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:12:05.250 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 80s ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:12:05.251 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 9m ago 10h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:12:05.251 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 9m ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:12:05.251 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 80s ago 10h 1060M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:12:05.252 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 80s ago 10h 2335M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:12:05.252 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 9m ago 10h 1030M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:12:05.253 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 80s ago 10h 1404M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:12:05.253 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 80s ago 10h 24.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:12:05.254 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 9m ago 10h 26.7M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:12:05.258 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 80s ago 10h 1698M 2090M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:12:05.259 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 80s ago 10h 1685M 2090M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:12:05.259 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 80s ago 10h 1308M 2090M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:12:05.260 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 80s ago 10h 1651M 2090M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:12:05.261 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 9m ago 10h 1425M 2498M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:12:05.261 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 9m ago 10h 1790M 2498M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:12:05.261 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 9m ago 10h 1600M 2498M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:12:05.261 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 9m ago 10h 1904M 2498M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:12:05.261 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 9m ago 10h 185M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:12:05.561 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:12:05 smithi153 ceph-mon[70044]: from='client.66517 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:12:05.606 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:12:05.606 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:12:05.607 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:12:05.607 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:12:05.608 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:12:05.608 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:12:05.609 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:12:05.609 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:12:05.610 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:12:05.610 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:12:05.611 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:12:05.611 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:12:05.611 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:12:05.612 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:12:05.612 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:12:05.613 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:12:05.613 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:12:05.614 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:12:05.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:12:05 smithi083 ceph-mon[77242]: from='client.66517 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:12:05.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:12:05 smithi083 ceph-mon[79930]: from='client.66517 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:12:06.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:12:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:12:05.932Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:12:06.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:12:06 smithi153 ceph-mon[70044]: from='client.66523 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:12:06.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:12:06 smithi153 ceph-mon[70044]: pgmap v19312: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:06.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:12:06 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2388087810' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:12:06.683 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:12:06 smithi083 ceph-mon[79930]: from='client.66523 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:12:06.683 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:12:06 smithi083 ceph-mon[79930]: pgmap v19312: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:06.683 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:12:06 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2388087810' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:12:06.684 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:12:06 smithi083 ceph-mon[77242]: from='client.66523 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:12:06.684 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:12:06 smithi083 ceph-mon[77242]: pgmap v19312: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:06.684 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:12:06 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2388087810' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:12:06.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:12:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:12:06.682Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:12:08.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:12:08 smithi083 ceph-mon[79930]: pgmap v19313: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:08.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:12:08 smithi083 ceph-mon[77242]: pgmap v19313: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:09.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:12:08 smithi153 ceph-mon[70044]: pgmap v19313: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:10.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:12:10 smithi083 ceph-mon[79930]: pgmap v19314: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:10.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:12:10 smithi083 ceph-mon[77242]: pgmap v19314: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:11.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:12:10 smithi153 ceph-mon[70044]: pgmap v19314: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:12:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:12:11] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T16:12:12.579 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:12:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:12:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:12:12.968 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:12:12 smithi083 ceph-mon[77242]: pgmap v19315: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:12.969 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:12:12 smithi083 ceph-mon[79930]: pgmap v19315: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:12.970 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:12:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:12:12.591Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:12:13.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:12:12 smithi153 ceph-mon[70044]: pgmap v19315: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:12:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:12:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:12:14.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:12:14 smithi153 ceph-mon[70044]: pgmap v19316: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:14.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:12:14 smithi083 ceph-mon[79930]: pgmap v19316: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:14.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:12:14 smithi083 ceph-mon[77242]: pgmap v19316: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:16.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:12:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:12:15.934Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:12:16.968 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:12:16 smithi083 ceph-mon[79930]: pgmap v19317: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:16.970 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:12:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:12:16.682Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:12:16.971 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:12:16 smithi083 ceph-mon[77242]: pgmap v19317: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:17.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:12:16 smithi153 ceph-mon[70044]: pgmap v19317: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:18.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:12:18 smithi083 ceph-mon[79930]: pgmap v19318: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:18.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:12:18 smithi083 ceph-mon[77242]: pgmap v19318: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:19.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:12:18 smithi153 ceph-mon[70044]: pgmap v19318: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:20.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:12:20 smithi083 ceph-mon[77242]: pgmap v19319: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:20.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:12:20 smithi083 ceph-mon[79930]: pgmap v19319: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:21.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:12:20 smithi153 ceph-mon[70044]: pgmap v19319: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:12:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:12:21] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T16:12:22.586 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:12:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:12:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:12:22.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:12:22 smithi083 ceph-mon[79930]: pgmap v19320: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:22.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:12:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:12:22.592Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:12:22.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:12:22 smithi083 ceph-mon[77242]: pgmap v19320: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:23.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:12:22 smithi153 ceph-mon[70044]: pgmap v19320: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:23.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:12:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:12:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:12:24.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:12:24 smithi153 ceph-mon[70044]: pgmap v19321: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:24.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:12:24 smithi083 ceph-mon[77242]: pgmap v19321: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:24.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:12:24 smithi083 ceph-mon[79930]: pgmap v19321: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:26.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:12:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:12:25.935Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:12:26.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:12:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:12:26.683Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:12:26.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:12:26 smithi083 ceph-mon[77242]: pgmap v19322: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:26.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:12:26 smithi083 ceph-mon[79930]: pgmap v19322: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:27.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:12:26 smithi153 ceph-mon[70044]: pgmap v19322: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:28.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:12:28 smithi083 ceph-mon[79930]: pgmap v19323: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:28.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:12:28 smithi083 ceph-mon[77242]: pgmap v19323: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:29.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:12:28 smithi153 ceph-mon[70044]: pgmap v19323: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:30.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:12:30 smithi083 ceph-mon[77242]: pgmap v19324: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:30.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:12:30 smithi083 ceph-mon[79930]: pgmap v19324: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:31.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:12:30 smithi153 ceph-mon[70044]: pgmap v19324: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:12:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:12:31] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T16:12:32.594 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:12:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:12:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:12:32.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:12:32 smithi083 ceph-mon[77242]: pgmap v19325: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:32.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:12:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:12:32.592Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:12:32.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:12:32 smithi083 ceph-mon[79930]: pgmap v19325: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:33.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:12:32 smithi153 ceph-mon[70044]: pgmap v19325: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:33.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:12:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:12:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:12:34.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:12:34 smithi153 ceph-mon[70044]: pgmap v19326: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:34.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:12:34 smithi083 ceph-mon[77242]: pgmap v19326: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:34.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:12:34 smithi083 ceph-mon[79930]: pgmap v19326: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:35.933 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:12:36.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:12:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:12:35.935Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:12:36.221 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:12:36.221 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 111s ago 10h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:12:36.221 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 10m ago 10h 64.0M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:12:36.222 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 111s ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:12:36.222 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 10m ago 10h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:12:36.222 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 10m ago 10h 450M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:12:36.223 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 111s ago 10h 1060M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:12:36.223 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 111s ago 10h 2335M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:12:36.224 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 10m ago 10h 1030M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:12:36.224 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 111s ago 10h 1404M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:12:36.224 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 111s ago 10h 24.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:12:36.224 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 10m ago 10h 26.7M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:12:36.225 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 111s ago 10h 1698M 2090M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:12:36.226 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 111s ago 10h 1685M 2090M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:12:36.226 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 111s ago 10h 1308M 2090M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:12:36.226 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 111s ago 10h 1651M 2090M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:12:36.226 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 10m ago 10h 1425M 2498M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:12:36.227 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 10m ago 10h 1790M 2498M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:12:36.227 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 10m ago 10h 1600M 2498M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:12:36.227 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 10m ago 10h 1904M 2498M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:12:36.227 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 10m ago 10h 185M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:12:36.587 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:12:36.596 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:12:36.597 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:12:36.597 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:12:36.598 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:12:36.598 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:12:36.599 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:12:36.599 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:12:36.599 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:12:36.600 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:12:36.600 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:12:36.601 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:12:36.602 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:12:36.602 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:12:36.603 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:12:36.604 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:12:36.604 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:12:36.605 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:12:36.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:12:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:12:36.683Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:12:36.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:12:36 smithi083 ceph-mon[77242]: pgmap v19327: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:36.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:12:36 smithi083 ceph-mon[77242]: from='client.56604 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:12:36.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:12:36 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3784873400' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:12:36.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:12:36 smithi083 ceph-mon[79930]: pgmap v19327: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:36.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:12:36 smithi083 ceph-mon[79930]: from='client.56604 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:12:36.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:12:36 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3784873400' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:12:37.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:12:36 smithi153 ceph-mon[70044]: pgmap v19327: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:37.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:12:36 smithi153 ceph-mon[70044]: from='client.56604 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:12:37.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:12:36 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3784873400' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:12:37.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:12:37 smithi083 ceph-mon[79930]: from='client.56610 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:12:37.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:12:37 smithi083 ceph-mon[77242]: from='client.56610 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:12:38.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:12:37 smithi153 ceph-mon[70044]: from='client.56610 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:12:38.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:12:38 smithi083 ceph-mon[79930]: pgmap v19328: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:38.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:12:38 smithi083 ceph-mon[77242]: pgmap v19328: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:39.066 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:12:38 smithi153 ceph-mon[70044]: pgmap v19328: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:12:40 smithi083 ceph-mon[77242]: pgmap v19329: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:40.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:12:40 smithi083 ceph-mon[79930]: pgmap v19329: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:41.072 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:12:40 smithi153 ceph-mon[70044]: pgmap v19329: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:12:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:12:41] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T16:12:42.599 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:12:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:12:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:12:42.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:12:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:12:42.592Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:12:42.969 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:12:42 smithi083 ceph-mon[79930]: pgmap v19330: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:42.969 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:12:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:12:42.970 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:12:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:12:42.970 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:12:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:12:42.971 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:12:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:12:42.972 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:12:42 smithi083 ceph-mon[77242]: pgmap v19330: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:42.972 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:12:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:12:42.972 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:12:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:12:42.972 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:12:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:12:42.973 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:12:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:12:43.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:12:42 smithi153 ceph-mon[70044]: pgmap v19330: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:43.026 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:12:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:12:43.031 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:12:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:12:43.031 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:12:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:12:43.032 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:12:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:12:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:12:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:12:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:12:44.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:12:44 smithi153 ceph-mon[70044]: pgmap v19331: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:44.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:12:44 smithi083 ceph-mon[77242]: pgmap v19331: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:44.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:12:44 smithi083 ceph-mon[79930]: pgmap v19331: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:46.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:12:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:12:45.937Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:12:46.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:12:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:12:46.684Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:12:46.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:12:46 smithi083 ceph-mon[77242]: pgmap v19332: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:46.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:12:46 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:12:46.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:12:46 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:12:46.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:12:46 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:12:46.971 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:12:46 smithi083 ceph-mon[79930]: pgmap v19332: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:46.971 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:12:46 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:12:46.972 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:12:46 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:12:46.972 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:12:46 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:12:47.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:12:46 smithi153 ceph-mon[70044]: pgmap v19332: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:47.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:12:46 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:12:47.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:12:46 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:12:47.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:12:46 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:12:48.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:12:48 smithi083 ceph-mon[79930]: pgmap v19333: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:48.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:12:48 smithi083 ceph-mon[77242]: pgmap v19333: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:49.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:12:48 smithi153 ceph-mon[70044]: pgmap v19333: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:50.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:12:50 smithi083 ceph-mon[77242]: pgmap v19334: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:50.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:12:50 smithi083 ceph-mon[79930]: pgmap v19334: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:51.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:12:50 smithi153 ceph-mon[70044]: pgmap v19334: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:51.965 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:12:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:12:51] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T16:12:52.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:12:52 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:12:52.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:12:52 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:12:52.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:12:52 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:12:52.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:12:52 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:12:52.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:12:52 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:12:52.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:12:52 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:12:52.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:12:52 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:12:52.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:12:52 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:12:52.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:12:52 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:12:52.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:12:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:12:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:12:52.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:12:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:12:52.593Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:12:53.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:12:53 smithi153 ceph-mon[70044]: pgmap v19335: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:53.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:12:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:12:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:12:53.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:12:53 smithi083 ceph-mon[79930]: pgmap v19335: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:53.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:12:53 smithi083 ceph-mon[77242]: pgmap v19335: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:54.564 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:12:54 smithi153 ceph-mon[70044]: pgmap v19336: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:54.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:12:54 smithi083 ceph-mon[79930]: pgmap v19336: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:54.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:12:54 smithi083 ceph-mon[77242]: pgmap v19336: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:56.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:12:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:12:55.937Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:12:56.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:12:56 smithi083 ceph-mon[79930]: pgmap v19337: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:56.964 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:12:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:12:56.685Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:12:56.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:12:56 smithi083 ceph-mon[77242]: pgmap v19337: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:57.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:12:56 smithi153 ceph-mon[70044]: pgmap v19337: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:58.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:12:58 smithi083 ceph-mon[79930]: pgmap v19338: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:58.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:12:58 smithi083 ceph-mon[77242]: pgmap v19338: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:12:59.069 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:12:58 smithi153 ceph-mon[70044]: pgmap v19338: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:00.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:13:00 smithi083 ceph-mon[79930]: pgmap v19339: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:00.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:13:00 smithi083 ceph-mon[77242]: pgmap v19339: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:01.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:13:00 smithi153 ceph-mon[70044]: pgmap v19339: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:01.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:13:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:13:01] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T16:13:02.605 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:13:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:13:02.316Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:13:02.987 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:13:02 smithi083 ceph-mon[79930]: pgmap v19340: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:02.989 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:13:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:13:02.594Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:13:02.990 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:13:02 smithi083 ceph-mon[77242]: pgmap v19340: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:03.028 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:13:02 smithi153 ceph-mon[70044]: pgmap v19340: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:13:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:13:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:13:04.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:13:04 smithi153 ceph-mon[70044]: pgmap v19341: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:04.719 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:13:04 smithi083 ceph-mon[77242]: pgmap v19341: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:04.719 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:13:04 smithi083 ceph-mon[79930]: pgmap v19341: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:06.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:13:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:13:05.938Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:13:06.904 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:13:06.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:13:06 smithi083 ceph-mon[79930]: pgmap v19342: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:06.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:13:06 smithi083 ceph-mon[77242]: pgmap v19342: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:06.964 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:13:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:13:06.685Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:13:07.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:13:06 smithi153 ceph-mon[70044]: pgmap v19342: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:07.193 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:13:07.198 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 2m ago 10h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:13:07.198 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 16s ago 10h 63.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:13:07.203 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 2m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:13:07.204 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 16s ago 10h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:13:07.204 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 16s ago 10h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:13:07.205 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 2m ago 10h 1060M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:13:07.205 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 2m ago 10h 2335M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:13:07.205 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 16s ago 10h 1045M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:13:07.206 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 2m ago 10h 1404M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:13:07.206 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 2m ago 10h 24.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:13:07.207 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 16s ago 10h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:13:07.207 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 2m ago 10h 1698M 2090M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:13:07.208 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 2m ago 10h 1685M 2090M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:13:07.208 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 2m ago 10h 1308M 2090M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:13:07.209 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 2m ago 10h 1651M 2090M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:13:07.209 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 16s ago 10h 1447M 2498M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:13:07.210 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 16s ago 10h 1819M 2498M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:13:07.211 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 16s ago 10h 1626M 2498M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:13:07.211 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 16s ago 10h 1935M 2498M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:13:07.212 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 16s ago 10h 188M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:13:07.548 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:13:07.549 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:13:07.549 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:13:07.552 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:13:07.552 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:13:07.552 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:13:07.553 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:13:07.553 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:13:07.555 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:13:07.555 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:13:07.555 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:13:07.556 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:13:07.556 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:13:07.557 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:13:07.557 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:13:07.557 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:13:07.558 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:13:07.558 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:13:07.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:13:07 smithi083 ceph-mon[79930]: from='client.66553 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:13:07.970 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:13:07 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2665876182' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:13:07.975 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:13:07 smithi083 ceph-mon[77242]: from='client.66553 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:13:07.975 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:13:07 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2665876182' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:13:08.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:13:07 smithi153 ceph-mon[70044]: from='client.66553 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:13:08.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:13:07 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2665876182' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:13:08.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:13:08 smithi083 ceph-mon[79930]: from='client.56628 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:13:08.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:13:08 smithi083 ceph-mon[79930]: pgmap v19343: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:08.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:13:08 smithi083 ceph-mon[77242]: from='client.56628 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:13:08.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:13:08 smithi083 ceph-mon[77242]: pgmap v19343: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:09.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:13:08 smithi153 ceph-mon[70044]: from='client.56628 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:13:09.063 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:13:08 smithi153 ceph-mon[70044]: pgmap v19343: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:10.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:13:10 smithi083 ceph-mon[79930]: pgmap v19344: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:10.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:13:10 smithi083 ceph-mon[77242]: pgmap v19344: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:11.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:13:10 smithi153 ceph-mon[70044]: pgmap v19344: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:11.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:13:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:13:11] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T16:13:12.609 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:13:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:13:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:13:12.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:13:12 smithi083 ceph-mon[79930]: pgmap v19345: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:12.966 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:13:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:13:12.595Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:13:12.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:13:12 smithi083 ceph-mon[77242]: pgmap v19345: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:13.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:13:12 smithi153 ceph-mon[70044]: pgmap v19345: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:13.311 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:13:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:13:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:13:14.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:13:14 smithi153 ceph-mon[70044]: pgmap v19346: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:14.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:13:14 smithi083 ceph-mon[79930]: pgmap v19346: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:14.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:13:14 smithi083 ceph-mon[77242]: pgmap v19346: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:16.235 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:13:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:13:15.938Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:13:16.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:13:16 smithi083 ceph-mon[79930]: pgmap v19347: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:16.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:13:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:13:16.686Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:13:16.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:13:16 smithi083 ceph-mon[77242]: pgmap v19347: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:17.066 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:13:16 smithi153 ceph-mon[70044]: pgmap v19347: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:18.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:13:18 smithi083 ceph-mon[79930]: pgmap v19348: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:18.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:13:18 smithi083 ceph-mon[77242]: pgmap v19348: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:19.065 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:13:18 smithi153 ceph-mon[70044]: pgmap v19348: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:20.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:13:20 smithi083 ceph-mon[79930]: pgmap v19349: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:20.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:13:20 smithi083 ceph-mon[77242]: pgmap v19349: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:21.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:13:20 smithi153 ceph-mon[70044]: pgmap v19349: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:13:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:13:21] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T16:13:22.625 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:13:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:13:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:13:22.966 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:13:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:13:22.595Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:13:22.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:13:22 smithi083 ceph-mon[77242]: pgmap v19350: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:22.968 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:13:22 smithi083 ceph-mon[79930]: pgmap v19350: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:23.029 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:13:22 smithi153 ceph-mon[70044]: pgmap v19350: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:13:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:13:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:13:24.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:13:24 smithi153 ceph-mon[70044]: pgmap v19351: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:24.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:13:24 smithi083 ceph-mon[79930]: pgmap v19351: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:24.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:13:24 smithi083 ceph-mon[77242]: pgmap v19351: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:26.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:13:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:13:25.939Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:13:26.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:13:26 smithi083 ceph-mon[79930]: pgmap v19352: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:26.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:13:26 smithi083 ceph-mon[77242]: pgmap v19352: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:26.967 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:13:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:13:26.686Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:13:27.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:13:26 smithi153 ceph-mon[70044]: pgmap v19352: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:28.968 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:13:28 smithi083 ceph-mon[79930]: pgmap v19353: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:28.969 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:13:28 smithi083 ceph-mon[77242]: pgmap v19353: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:29.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:13:28 smithi153 ceph-mon[70044]: pgmap v19353: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:30.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:13:30 smithi083 ceph-mon[77242]: pgmap v19354: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:30.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:13:30 smithi083 ceph-mon[79930]: pgmap v19354: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:31.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:13:30 smithi153 ceph-mon[70044]: pgmap v19354: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:31.965 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:13:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:13:31] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T16:13:32.610 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:13:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:13:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:13:32.969 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:13:32 smithi083 ceph-mon[77242]: pgmap v19355: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:32.970 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:13:32 smithi083 ceph-mon[79930]: pgmap v19355: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:32.970 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:13:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:13:32.596Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:13:33.026 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:13:32 smithi153 ceph-mon[70044]: pgmap v19355: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:13:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:13:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:13:34.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:13:34 smithi153 ceph-mon[70044]: pgmap v19356: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:34.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:13:34 smithi083 ceph-mon[79930]: pgmap v19356: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:34.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:13:34 smithi083 ceph-mon[77242]: pgmap v19356: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:36.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:13:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:13:35.939Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:13:36.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:13:36 smithi083 ceph-mon[77242]: pgmap v19357: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:36.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:13:36 smithi083 ceph-mon[79930]: pgmap v19357: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:36.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:13:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:13:36.687Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:13:37.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:13:36 smithi153 ceph-mon[70044]: pgmap v19357: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:37.875 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:13:38.174 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:13:38.174 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 2m ago 10h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:13:38.175 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 47s ago 10h 63.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:13:38.175 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 2m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:13:38.175 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 47s ago 10h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:13:38.175 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 47s ago 10h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:13:38.176 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 2m ago 10h 1060M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:13:38.177 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 2m ago 10h 2335M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:13:38.177 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 47s ago 10h 1045M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:13:38.178 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 2m ago 10h 1404M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:13:38.179 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 2m ago 10h 24.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:13:38.179 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 47s ago 10h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:13:38.180 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 2m ago 10h 1698M 2090M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:13:38.180 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 2m ago 10h 1685M 2090M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:13:38.180 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 2m ago 10h 1308M 2090M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:13:38.180 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 2m ago 10h 1651M 2090M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:13:38.181 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 47s ago 10h 1447M 2498M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:13:38.181 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 47s ago 10h 1819M 2498M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:13:38.181 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 47s ago 10h 1626M 2498M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:13:38.181 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 47s ago 10h 1935M 2498M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:13:38.182 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 47s ago 10h 188M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:13:38.524 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:13:38.524 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:13:38.525 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:13:38.525 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:13:38.526 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:13:38.526 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:13:38.526 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:13:38.526 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:13:38.527 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:13:38.527 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:13:38.527 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:13:38.527 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:13:38.528 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:13:38.528 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:13:38.528 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:13:38.528 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:13:38.529 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:13:38.529 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:13:38.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:13:38 smithi083 ceph-mon[77242]: pgmap v19358: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:38.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:13:38 smithi083 ceph-mon[77242]: from='client.66571 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:13:38.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:13:38 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1744411677' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:13:38.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:13:38 smithi083 ceph-mon[79930]: pgmap v19358: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:38.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:13:38 smithi083 ceph-mon[79930]: from='client.66571 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:13:38.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:13:38 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1744411677' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:13:39.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:13:38 smithi153 ceph-mon[70044]: pgmap v19358: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:39.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:13:38 smithi153 ceph-mon[70044]: from='client.66571 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:13:39.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:13:38 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1744411677' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:13:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:13:39 smithi083 ceph-mon[79930]: from='client.56646 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:13:39.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:13:39 smithi083 ceph-mon[77242]: from='client.56646 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:13:40.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:13:39 smithi153 ceph-mon[70044]: from='client.56646 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:13:40.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:13:40 smithi083 ceph-mon[79930]: pgmap v19359: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:40.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:13:40 smithi083 ceph-mon[77242]: pgmap v19359: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:41.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:13:40 smithi153 ceph-mon[70044]: pgmap v19359: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:41.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:13:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:13:41] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T16:13:42.619 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:13:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:13:42.317Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:13:42.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:13:42 smithi083 ceph-mon[77242]: pgmap v19360: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:42.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:13:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:13:42.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:13:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:13:42.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:13:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:13:42.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:13:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:13:42.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:13:42 smithi083 ceph-mon[79930]: pgmap v19360: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:42.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:13:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:13:42.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:13:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:13:42.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:13:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:13:42.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:13:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:13:42.967 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:13:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:13:42.597Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:13:43.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:13:42 smithi153 ceph-mon[70044]: pgmap v19360: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:43.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:13:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:13:43.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:13:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:13:43.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:13:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:13:43.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:13:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:13:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:13:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:13:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:13:44.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:13:44 smithi153 ceph-mon[70044]: pgmap v19361: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:44.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:13:44 smithi083 ceph-mon[79930]: pgmap v19361: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:44.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:13:44 smithi083 ceph-mon[77242]: pgmap v19361: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:46.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:13:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:13:45.940Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:13:46.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:13:46 smithi083 ceph-mon[79930]: pgmap v19362: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:46.970 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:13:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:13:46.689Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:13:46.971 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:13:46 smithi083 ceph-mon[77242]: pgmap v19362: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:47.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:13:46 smithi153 ceph-mon[70044]: pgmap v19362: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:48.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:13:48 smithi083 ceph-mon[79930]: pgmap v19363: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:48.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:13:48 smithi083 ceph-mon[77242]: pgmap v19363: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:49.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:13:48 smithi153 ceph-mon[70044]: pgmap v19363: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:50.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:13:50 smithi083 ceph-mon[79930]: pgmap v19364: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:50.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:13:50 smithi083 ceph-mon[77242]: pgmap v19364: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:51.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:13:50 smithi153 ceph-mon[70044]: pgmap v19364: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:13:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:13:51] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T16:13:52.615 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:13:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:13:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:13:52.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:13:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:13:52.598Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:13:52.979 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:13:52 smithi083 ceph-mon[79930]: pgmap v19365: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:52.979 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:13:52 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:13:52.980 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:13:52 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:13:52.980 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:13:52 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:13:52.981 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:13:52 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:13:52.981 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:13:52 smithi083 ceph-mon[77242]: pgmap v19365: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:52.995 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:13:52 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:13:53.007 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:13:52 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:13:53.007 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:13:52 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:13:53.008 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:13:52 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:13:53.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:13:52 smithi153 ceph-mon[70044]: pgmap v19365: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:53.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:13:52 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:13:53.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:13:52 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:13:53.026 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:13:52 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:13:53.026 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:13:52 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:13:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:13:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:13:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:13:54.563 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:13:54 smithi153 ceph-mon[70044]: pgmap v19366: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:54.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:13:54 smithi083 ceph-mon[79930]: pgmap v19366: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:54.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:13:54 smithi083 ceph-mon[77242]: pgmap v19366: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:56.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:13:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:13:55.941Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:13:56.968 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:13:56 smithi083 ceph-mon[77242]: pgmap v19367: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:56.969 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:13:56 smithi083 ceph-mon[79930]: pgmap v19367: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:56.969 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:13:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:13:56.689Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:13:57.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:13:56 smithi153 ceph-mon[70044]: pgmap v19367: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:58.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:13:58 smithi083 ceph-mon[79930]: pgmap v19368: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:58.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:13:58 smithi083 ceph-mon[77242]: pgmap v19368: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:13:59.079 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:13:58 smithi153 ceph-mon[70044]: pgmap v19368: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:00.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:14:00 smithi083 ceph-mon[79930]: pgmap v19369: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:00.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:14:00 smithi083 ceph-mon[77242]: pgmap v19369: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:01.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:14:00 smithi153 ceph-mon[70044]: pgmap v19369: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:01.967 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:14:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:14:01] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T16:14:02.624 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:14:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:14:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:14:02.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:14:02 smithi083 ceph-mon[79930]: pgmap v19370: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:02.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:14:02 smithi083 ceph-mon[77242]: pgmap v19370: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:02.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:14:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:14:02.598Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:14:03.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:14:02 smithi153 ceph-mon[70044]: pgmap v19370: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:03.312 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:14:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:14:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:14:04.563 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:14:04 smithi153 ceph-mon[70044]: pgmap v19371: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:04.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:14:04 smithi083 ceph-mon[79930]: pgmap v19371: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:04.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:14:04 smithi083 ceph-mon[77242]: pgmap v19371: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:06.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:14:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:14:05.941Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:14:06.975 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:14:06 smithi083 ceph-mon[79930]: pgmap v19372: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:06.975 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:14:06 smithi083 ceph-mon[77242]: pgmap v19372: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:06.976 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:14:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:14:06.690Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:14:07.068 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:14:06 smithi153 ceph-mon[70044]: pgmap v19372: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:08.863 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:14:08.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:14:08 smithi083 ceph-mon[79930]: pgmap v19373: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:08.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:14:08 smithi083 ceph-mon[77242]: pgmap v19373: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:09.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:14:08 smithi153 ceph-mon[70044]: pgmap v19373: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:09.144 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:14:09.144 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 3m ago 10h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:14:09.145 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 78s ago 10h 63.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:14:09.145 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 3m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:14:09.146 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 78s ago 10h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:14:09.146 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 78s ago 10h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:14:09.146 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 3m ago 10h 1060M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:14:09.147 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 3m ago 10h 2335M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:14:09.149 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 78s ago 10h 1045M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:14:09.153 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 3m ago 10h 1404M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:14:09.154 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 3m ago 10h 24.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:14:09.154 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 78s ago 10h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:14:09.155 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 3m ago 10h 1698M 2090M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:14:09.155 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 3m ago 10h 1685M 2090M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:14:09.155 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 3m ago 10h 1308M 2090M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:14:09.156 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 3m ago 10h 1651M 2090M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:14:09.156 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 78s ago 10h 1447M 2498M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:14:09.157 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 78s ago 10h 1819M 2498M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:14:09.157 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 78s ago 10h 1626M 2498M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:14:09.158 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 78s ago 10h 1935M 2498M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:14:09.158 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 78s ago 10h 188M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:14:09.505 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:14:09.506 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:14:09.506 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:14:09.507 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:14:09.507 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:14:09.508 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:14:09.508 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:14:09.522 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:14:09.523 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:14:09.523 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:14:09.524 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:14:09.531 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:14:09.532 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:14:09.532 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:14:09.533 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:14:09.541 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:14:09.542 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:14:09.542 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:14:09.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:14:09 smithi083 ceph-mon[79930]: from='client.56658 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:14:09.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:14:09 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1005285108' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:14:09.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:14:09 smithi083 ceph-mon[77242]: from='client.56658 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:14:09.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:14:09 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1005285108' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:14:10.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:14:09 smithi153 ceph-mon[70044]: from='client.56658 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:14:10.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:14:09 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1005285108' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:14:10.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:14:10 smithi083 ceph-mon[79930]: from='client.56664 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:14:10.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:14:10 smithi083 ceph-mon[79930]: pgmap v19374: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:10.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:14:10 smithi083 ceph-mon[77242]: from='client.56664 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:14:10.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:14:10 smithi083 ceph-mon[77242]: pgmap v19374: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:11.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:14:10 smithi153 ceph-mon[70044]: from='client.56664 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:14:11.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:14:10 smithi153 ceph-mon[70044]: pgmap v19374: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:14:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:14:11] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T16:14:12.625 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:14:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:14:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:14:12.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:14:12 smithi083 ceph-mon[79930]: pgmap v19375: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:12.966 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:14:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:14:12.599Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:14:12.969 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:14:12 smithi083 ceph-mon[77242]: pgmap v19375: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:13.035 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:14:12 smithi153 ceph-mon[70044]: pgmap v19375: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:13.313 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:14:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:14:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:14:14.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:14:14 smithi153 ceph-mon[70044]: pgmap v19376: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:14.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:14:14 smithi083 ceph-mon[79930]: pgmap v19376: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:14.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:14:14 smithi083 ceph-mon[77242]: pgmap v19376: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:16.214 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:14:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:14:15.942Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:14:16.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:14:16 smithi083 ceph-mon[79930]: pgmap v19377: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:16.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:14:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:14:16.691Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:14:16.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:14:16 smithi083 ceph-mon[77242]: pgmap v19377: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:17.067 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:14:16 smithi153 ceph-mon[70044]: pgmap v19377: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:18.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:14:18 smithi083 ceph-mon[79930]: pgmap v19378: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:18.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:14:18 smithi083 ceph-mon[77242]: pgmap v19378: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:19.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:14:18 smithi153 ceph-mon[70044]: pgmap v19378: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:20.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:14:20 smithi083 ceph-mon[79930]: pgmap v19379: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:20.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:14:20 smithi083 ceph-mon[77242]: pgmap v19379: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:21.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:14:20 smithi153 ceph-mon[70044]: pgmap v19379: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:14:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:14:21] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T16:14:22.642 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:14:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:14:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:14:22.971 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:14:22 smithi083 ceph-mon[79930]: pgmap v19380: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:22.973 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:14:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:14:22.600Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:14:22.979 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:14:22 smithi083 ceph-mon[77242]: pgmap v19380: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:23.047 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:14:22 smithi153 ceph-mon[70044]: pgmap v19380: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:14:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:14:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:14:24.570 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:14:24 smithi153 ceph-mon[70044]: pgmap v19381: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:24.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:14:24 smithi083 ceph-mon[79930]: pgmap v19381: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:24.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:14:24 smithi083 ceph-mon[77242]: pgmap v19381: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:26.222 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:14:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:14:25.943Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:14:26.990 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:14:26 smithi083 ceph-mon[79930]: pgmap v19382: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:26.991 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:14:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:14:26.691Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:14:26.991 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:14:26 smithi083 ceph-mon[77242]: pgmap v19382: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:27.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:14:26 smithi153 ceph-mon[70044]: pgmap v19382: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:28.971 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:14:28 smithi083 ceph-mon[79930]: pgmap v19383: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:28.981 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:14:28 smithi083 ceph-mon[77242]: pgmap v19383: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:29.070 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:14:28 smithi153 ceph-mon[70044]: pgmap v19383: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:30.969 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:14:30 smithi083 ceph-mon[79930]: pgmap v19384: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:30.971 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:14:30 smithi083 ceph-mon[77242]: pgmap v19384: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:31.072 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:14:30 smithi153 ceph-mon[70044]: pgmap v19384: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:31.970 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:14:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:14:31] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T16:14:32.628 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:14:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:14:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:14:32.971 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:14:32 smithi083 ceph-mon[79930]: pgmap v19385: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:32.972 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:14:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:14:32.601Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:14:32.972 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:14:32 smithi083 ceph-mon[77242]: pgmap v19385: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:33.029 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:14:32 smithi153 ceph-mon[70044]: pgmap v19385: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:33.314 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:14:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:14:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:14:34.568 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:14:34 smithi153 ceph-mon[70044]: pgmap v19386: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:34.723 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:14:34 smithi083 ceph-mon[79930]: pgmap v19386: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:34.723 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:14:34 smithi083 ceph-mon[77242]: pgmap v19386: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:36.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:14:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:14:35.943Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:14:36.968 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:14:36 smithi083 ceph-mon[79930]: pgmap v19387: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:36.968 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:14:36 smithi083 ceph-mon[77242]: pgmap v19387: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:36.969 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:14:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:14:36.692Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:14:37.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:14:36 smithi153 ceph-mon[70044]: pgmap v19387: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:38.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:14:38 smithi083 ceph-mon[79930]: pgmap v19388: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:38.968 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:14:38 smithi083 ceph-mon[77242]: pgmap v19388: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:39.064 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:14:38 smithi153 ceph-mon[70044]: pgmap v19388: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:39.829 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:14:40.134 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:14:40.135 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 3m ago 10h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:14:40.136 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 109s ago 10h 63.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:14:40.138 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 3m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:14:40.138 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 109s ago 10h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:14:40.148 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 109s ago 10h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:14:40.149 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 3m ago 10h 1060M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:14:40.149 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 3m ago 10h 2335M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:14:40.150 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 109s ago 10h 1045M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:14:40.150 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 3m ago 10h 1404M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:14:40.151 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 3m ago 10h 24.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:14:40.151 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 109s ago 10h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:14:40.151 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 3m ago 10h 1698M 2090M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:14:40.152 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 3m ago 10h 1685M 2090M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:14:40.152 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 3m ago 10h 1308M 2090M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:14:40.153 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 3m ago 10h 1651M 2090M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:14:40.153 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 109s ago 10h 1447M 2498M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:14:40.153 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 109s ago 10h 1819M 2498M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:14:40.154 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 109s ago 10h 1626M 2498M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:14:40.155 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 109s ago 10h 1935M 2498M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:14:40.156 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 109s ago 10h 188M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:14:40.498 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:14:40.499 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:14:40.502 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:14:40.515 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:14:40.515 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:14:40.516 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:14:40.517 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:14:40.518 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:14:40.518 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:14:40.519 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:14:40.519 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:14:40.519 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:14:40.521 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:14:40.521 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:14:40.521 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:14:40.522 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:14:40.522 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:14:40.523 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:14:40.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:14:40 smithi083 ceph-mon[77242]: pgmap v19389: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:14:40 smithi083 ceph-mon[77242]: from='client.56676 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:14:40.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:14:40 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1508598523' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:14:40.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:14:40 smithi083 ceph-mon[79930]: pgmap v19389: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:40.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:14:40 smithi083 ceph-mon[79930]: from='client.56676 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:14:40.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:14:40 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1508598523' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:14:41.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:14:40 smithi153 ceph-mon[70044]: pgmap v19389: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:41.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:14:40 smithi153 ceph-mon[70044]: from='client.56676 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:14:41.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:14:40 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1508598523' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:14:41.977 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:14:41 smithi083 ceph-mon[77242]: from='client.56682 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:14:41.978 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:14:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:14:41] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T16:14:41.979 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:14:41 smithi083 ceph-mon[79930]: from='client.56682 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:14:42.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:14:41 smithi153 ceph-mon[70044]: from='client.56682 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:14:42.643 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:14:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:14:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:14:42.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:14:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:14:42.602Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:14:42.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:14:42 smithi083 ceph-mon[79930]: pgmap v19390: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:42.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:14:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:14:42.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:14:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:14:42.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:14:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:14:42.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:14:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:14:42.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:14:42 smithi083 ceph-mon[77242]: pgmap v19390: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:42.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:14:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:14:42.976 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:14:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:14:42.988 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:14:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:14:42.997 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:14:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:14:43.031 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:14:42 smithi153 ceph-mon[70044]: pgmap v19390: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:43.031 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:14:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:14:43.032 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:14:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:14:43.032 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:14:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:14:43.033 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:14:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:14:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:14:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:14:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:14:44.563 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:14:44 smithi153 ceph-mon[70044]: pgmap v19391: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:44.720 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:14:44 smithi083 ceph-mon[79930]: pgmap v19391: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:44.721 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:14:44 smithi083 ceph-mon[77242]: pgmap v19391: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:46.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:14:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:14:45.944Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:14:46.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:14:46 smithi083 ceph-mon[77242]: pgmap v19392: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:46.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:14:46 smithi083 ceph-mon[79930]: pgmap v19392: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:46.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:14:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:14:46.692Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:14:47.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:14:46 smithi153 ceph-mon[70044]: pgmap v19392: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:48.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:14:48 smithi083 ceph-mon[79930]: pgmap v19393: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:48.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:14:48 smithi083 ceph-mon[77242]: pgmap v19393: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:49.067 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:14:48 smithi153 ceph-mon[70044]: pgmap v19393: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:50.970 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:14:50 smithi083 ceph-mon[79930]: pgmap v19394: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:50.971 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:14:50 smithi083 ceph-mon[77242]: pgmap v19394: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:51.066 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:14:50 smithi153 ceph-mon[70044]: pgmap v19394: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:51.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:14:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:14:51] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T16:14:52.641 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:14:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:14:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:14:52.964 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:14:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:14:52.602Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:14:52.984 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:14:52 smithi083 ceph-mon[77242]: pgmap v19395: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:52.985 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:14:52 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:14:52.985 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:14:52 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:14:52.986 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:14:52 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:14:52.986 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:14:52 smithi083 ceph-mon[79930]: pgmap v19395: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:52.986 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:14:52 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:14:52.987 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:14:52 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:14:52.987 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:14:52 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:14:53.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:14:52 smithi153 ceph-mon[70044]: pgmap v19395: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:53.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:14:52 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:14:53.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:14:52 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:14:53.026 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:14:52 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:14:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:14:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:14:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:14:54.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:14:53 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:14:54.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:14:53 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:14:54.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:14:53 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:14:55.219 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:14:54 smithi083 ceph-mon[79930]: pgmap v19396: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:55.222 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:14:54 smithi083 ceph-mon[77242]: pgmap v19396: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:55.313 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:14:54 smithi153 ceph-mon[70044]: pgmap v19396: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:56.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:14:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:14:55.945Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:14:56.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:14:56 smithi083 ceph-mon[79930]: pgmap v19397: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:56.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:14:56 smithi083 ceph-mon[77242]: pgmap v19397: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:56.966 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:14:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:14:56.693Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:14:57.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:14:56 smithi153 ceph-mon[70044]: pgmap v19397: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:59.219 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:14:58 smithi083 ceph-mon[79930]: pgmap v19398: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:59.222 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:14:58 smithi083 ceph-mon[77242]: pgmap v19398: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:14:59.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:14:58 smithi153 ceph-mon[70044]: pgmap v19398: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:01.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:15:00 smithi083 ceph-mon[79930]: pgmap v19399: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:01.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:15:00 smithi083 ceph-mon[77242]: pgmap v19399: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:01.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:15:00 smithi153 ceph-mon[70044]: pgmap v19399: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:15:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:15:01] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T16:15:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:15:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:15:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:15:02.902 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:15:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:15:02.603Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:15:03.223 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:15:02 smithi083 ceph-mon[79930]: pgmap v19400: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:03.224 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:15:02 smithi083 ceph-mon[77242]: pgmap v19400: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:03.314 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:15:02 smithi153 ceph-mon[70044]: pgmap v19400: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:03.315 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:15:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:15:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:15:04.561 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:15:04 smithi153 ceph-mon[70044]: pgmap v19401: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:04.719 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:15:04 smithi083 ceph-mon[77242]: pgmap v19401: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:04.720 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:15:04 smithi083 ceph-mon[79930]: pgmap v19401: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:06.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:15:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:15:05.945Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:15:06.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:15:06 smithi083 ceph-mon[79930]: pgmap v19402: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:06.964 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:15:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:15:06.694Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:15:06.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:15:06 smithi083 ceph-mon[77242]: pgmap v19402: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:07.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:15:06 smithi153 ceph-mon[70044]: pgmap v19402: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:08.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:15:08 smithi083 ceph-mon[79930]: pgmap v19403: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:08.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:15:08 smithi083 ceph-mon[77242]: pgmap v19403: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:09.067 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:15:08 smithi153 ceph-mon[70044]: pgmap v19403: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:10.811 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:15:10.979 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:15:10 smithi083 ceph-mon[79930]: pgmap v19404: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:10.980 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:15:10 smithi083 ceph-mon[77242]: pgmap v19404: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:11.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:15:10 smithi153 ceph-mon[70044]: pgmap v19404: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:11.091 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:15:11.092 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 4m ago 10h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:15:11.092 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 2m ago 10h 63.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:15:11.093 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 4m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:15:11.093 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 2m ago 10h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:15:11.093 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 2m ago 10h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:15:11.094 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 4m ago 10h 1060M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:15:11.094 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 4m ago 10h 2335M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:15:11.095 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 2m ago 10h 1045M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:15:11.095 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 4m ago 10h 1404M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:15:11.096 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 4m ago 10h 24.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:15:11.096 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 2m ago 10h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:15:11.097 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 4m ago 10h 1698M 2090M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:15:11.097 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 4m ago 10h 1685M 2090M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:15:11.098 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 4m ago 10h 1308M 2090M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:15:11.099 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 4m ago 10h 1651M 2090M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:15:11.099 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 2m ago 10h 1447M 2498M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:15:11.100 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 2m ago 10h 1819M 2498M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:15:11.100 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 2m ago 10h 1626M 2498M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:15:11.100 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 2m ago 10h 1935M 2498M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:15:11.101 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 2m ago 10h 188M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:15:11.447 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:15:11.448 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:15:11.448 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:15:11.469 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:15:11.469 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:15:11.470 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:15:11.470 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:15:11.470 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:15:11.470 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:15:11.471 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:15:11.471 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:15:11.471 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:15:11.471 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:15:11.472 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:15:11.472 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:15:11.472 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:15:11.472 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:15:11.472 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:15:11.966 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:15:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:15:11] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T16:15:11.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:15:11 smithi083 ceph-mon[79930]: from='client.56694 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:15:11.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:15:11 smithi083 ceph-mon[79930]: from='client.56700 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:15:11.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:15:11 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2313990606' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:15:11.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:15:11 smithi083 ceph-mon[77242]: from='client.56694 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:15:11.968 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:15:11 smithi083 ceph-mon[77242]: from='client.56700 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:15:11.968 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:15:11 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2313990606' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:15:12.063 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:15:11 smithi153 ceph-mon[70044]: from='client.56694 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:15:12.067 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:15:11 smithi153 ceph-mon[70044]: from='client.56700 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:15:12.068 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:15:11 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2313990606' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:15:12.647 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:15:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:15:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:15:12.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:15:12 smithi083 ceph-mon[79930]: pgmap v19405: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:12.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:15:12 smithi083 ceph-mon[77242]: pgmap v19405: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:12.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:15:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:15:12.603Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:15:13.034 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:15:12 smithi153 ceph-mon[70044]: pgmap v19405: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:15:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:15:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:15:14.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:15:14 smithi153 ceph-mon[70044]: pgmap v19406: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:14.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:15:14 smithi083 ceph-mon[79930]: pgmap v19406: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:14.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:15:14 smithi083 ceph-mon[77242]: pgmap v19406: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:16.221 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:15:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:15:15.946Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:15:16.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:15:16 smithi083 ceph-mon[77242]: pgmap v19407: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:16.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:15:16 smithi083 ceph-mon[79930]: pgmap v19407: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:16.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:15:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:15:16.695Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:15:17.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:15:16 smithi153 ceph-mon[70044]: pgmap v19407: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:18.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:15:18 smithi083 ceph-mon[77242]: pgmap v19408: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:18.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:15:18 smithi083 ceph-mon[79930]: pgmap v19408: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:19.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:15:18 smithi153 ceph-mon[70044]: pgmap v19408: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:20.974 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:15:20 smithi083 ceph-mon[77242]: pgmap v19409: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:20.975 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:15:20 smithi083 ceph-mon[79930]: pgmap v19409: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:21.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:15:20 smithi153 ceph-mon[70044]: pgmap v19409: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:21.971 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:15:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:15:21] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T16:15:22.643 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:15:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:15:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:15:22.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:15:22 smithi083 ceph-mon[79930]: pgmap v19410: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:22.968 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:15:22 smithi083 ceph-mon[77242]: pgmap v19410: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:22.969 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:15:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:15:22.603Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:15:23.053 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:15:22 smithi153 ceph-mon[70044]: pgmap v19410: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:23.312 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:15:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:15:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:15:24.568 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:15:24 smithi153 ceph-mon[70044]: pgmap v19411: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:24.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:15:24 smithi083 ceph-mon[79930]: pgmap v19411: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:24.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:15:24 smithi083 ceph-mon[77242]: pgmap v19411: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:26.218 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:15:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:15:25.947Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:15:26.968 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:15:26 smithi083 ceph-mon[79930]: pgmap v19412: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:26.968 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:15:26 smithi083 ceph-mon[77242]: pgmap v19412: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:26.969 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:15:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:15:26.696Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:15:27.067 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:15:26 smithi153 ceph-mon[70044]: pgmap v19412: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:28.971 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:15:28 smithi083 ceph-mon[77242]: pgmap v19413: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:28.972 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:15:28 smithi083 ceph-mon[79930]: pgmap v19413: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:29.064 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:15:28 smithi153 ceph-mon[70044]: pgmap v19413: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:30.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:15:30 smithi083 ceph-mon[79930]: pgmap v19414: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:30.990 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:15:30 smithi083 ceph-mon[77242]: pgmap v19414: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:31.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:15:30 smithi153 ceph-mon[70044]: pgmap v19414: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:15:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:15:31] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T16:15:32.647 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:15:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:15:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:15:32.971 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:15:32 smithi083 ceph-mon[79930]: pgmap v19415: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:32.972 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:15:32 smithi083 ceph-mon[77242]: pgmap v19415: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:32.972 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:15:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:15:32.604Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:15:33.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:15:32 smithi153 ceph-mon[70044]: pgmap v19415: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:33.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:15:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:15:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:15:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:15:34 smithi153 ceph-mon[70044]: pgmap v19416: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:34.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:15:34 smithi083 ceph-mon[79930]: pgmap v19416: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:34.729 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:15:34 smithi083 ceph-mon[77242]: pgmap v19416: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:36.223 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:15:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:15:35.947Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:15:36.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:15:36 smithi083 ceph-mon[79930]: pgmap v19417: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:36.964 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:15:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:15:36.695Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:15:36.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:15:36 smithi083 ceph-mon[77242]: pgmap v19417: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:37.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:15:36 smithi153 ceph-mon[70044]: pgmap v19417: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:38.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:15:38 smithi083 ceph-mon[79930]: pgmap v19418: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:38.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:15:38 smithi083 ceph-mon[77242]: pgmap v19418: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:39.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:15:38 smithi153 ceph-mon[70044]: pgmap v19418: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:40.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:15:40 smithi083 ceph-mon[79930]: pgmap v19419: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:40.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:15:40 smithi083 ceph-mon[77242]: pgmap v19419: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:15:40 smithi153 ceph-mon[70044]: pgmap v19419: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:41.775 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:15:41.970 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:15:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:15:41] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T16:15:42.074 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:15:42.075 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 4m ago 10h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:15:42.076 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 2m ago 10h 63.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:15:42.076 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 4m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:15:42.076 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 2m ago 10h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:15:42.077 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 2m ago 10h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:15:42.077 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 4m ago 10h 1060M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:15:42.080 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 4m ago 10h 2335M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:15:42.080 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 2m ago 10h 1045M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:15:42.080 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 4m ago 10h 1404M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:15:42.081 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 4m ago 10h 24.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:15:42.081 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 2m ago 10h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:15:42.082 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 4m ago 10h 1698M 2090M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:15:42.082 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 4m ago 10h 1685M 2090M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:15:42.083 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 4m ago 10h 1308M 2090M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:15:42.083 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 4m ago 10h 1651M 2090M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:15:42.084 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 2m ago 10h 1447M 2498M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:15:42.084 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 2m ago 10h 1819M 2498M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:15:42.091 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 2m ago 10h 1626M 2498M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:15:42.091 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 2m ago 10h 1935M 2498M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:15:42.092 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 2m ago 10h 188M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:15:42.461 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:15:42.461 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:15:42.462 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:15:42.462 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:15:42.463 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:15:42.463 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:15:42.466 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:15:42.468 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:15:42.477 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:15:42.478 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:15:42.478 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:15:42.479 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:15:42.479 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:15:42.480 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:15:42.480 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:15:42.490 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:15:42.491 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:15:42.491 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:15:42.650 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:15:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:15:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:15:42.970 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:15:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:15:42.604Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:15:42.971 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:15:42 smithi083 ceph-mon[79930]: pgmap v19420: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:42.971 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:15:42 smithi083 ceph-mon[79930]: from='client.66643 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:15:42.971 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:15:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:15:42.972 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:15:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:15:42.972 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:15:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:15:42.972 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:15:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:15:42.972 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:15:42 smithi083 ceph-mon[79930]: from='client.66649 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:15:42.973 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:15:42 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3747073555' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:15:42.973 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:15:42 smithi083 ceph-mon[77242]: pgmap v19420: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:42.973 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:15:42 smithi083 ceph-mon[77242]: from='client.66643 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:15:42.986 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:15:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:15:43.006 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:15:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:15:43.007 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:15:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:15:43.007 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:15:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:15:43.007 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:15:42 smithi083 ceph-mon[77242]: from='client.66649 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:15:43.007 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:15:42 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3747073555' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:15:43.028 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:15:42 smithi153 ceph-mon[70044]: pgmap v19420: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:43.028 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:15:42 smithi153 ceph-mon[70044]: from='client.66643 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:15:43.028 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:15:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:15:43.029 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:15:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:15:43.029 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:15:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:15:43.029 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:15:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:15:43.029 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:15:42 smithi153 ceph-mon[70044]: from='client.66649 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:15:43.066 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:15:42 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3747073555' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:15:43.314 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:15:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:15:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:15:44.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:15:44 smithi153 ceph-mon[70044]: pgmap v19421: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:44.721 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:15:44 smithi083 ceph-mon[79930]: pgmap v19421: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:44.756 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:15:44 smithi083 ceph-mon[77242]: pgmap v19421: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:46.214 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:15:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:15:45.948Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:15:46.969 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:15:46 smithi083 ceph-mon[79930]: pgmap v19422: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:46.969 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:15:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:15:46.697Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:15:46.970 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:15:46 smithi083 ceph-mon[77242]: pgmap v19422: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:47.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:15:46 smithi153 ceph-mon[70044]: pgmap v19422: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:48.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:15:48 smithi083 ceph-mon[79930]: pgmap v19423: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:48.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:15:48 smithi083 ceph-mon[77242]: pgmap v19423: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:49.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:15:48 smithi153 ceph-mon[70044]: pgmap v19423: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:50.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:15:50 smithi083 ceph-mon[79930]: pgmap v19424: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:50.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:15:50 smithi083 ceph-mon[77242]: pgmap v19424: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:51.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:15:50 smithi153 ceph-mon[70044]: pgmap v19424: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:15:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:15:51] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T16:15:52.658 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:15:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:15:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:15:52.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:15:52 smithi083 ceph-mon[79930]: pgmap v19425: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:52.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:15:52 smithi083 ceph-mon[77242]: pgmap v19425: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:52.964 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:15:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:15:52.606Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:15:53.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:15:52 smithi153 ceph-mon[70044]: pgmap v19425: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:15:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:15:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:15:53.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:15:53 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:15:53.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:15:53 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:15:53.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:15:53 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:15:53.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:15:53 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:15:53.977 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:15:53 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:15:53.977 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:15:53 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:15:53.979 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:15:53 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:15:53.980 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:15:53 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:15:54.063 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:15:53 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:15:54.064 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:15:53 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:15:54.064 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:15:53 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:15:54.064 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:15:53 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:15:54.968 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:15:54 smithi083 ceph-mon[79930]: pgmap v19426: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:54.969 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:15:54 smithi083 ceph-mon[77242]: pgmap v19426: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:55.095 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:15:54 smithi153 ceph-mon[70044]: pgmap v19426: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:56.221 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:15:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:15:55.948Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:15:56.977 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:15:56 smithi083 ceph-mon[79930]: pgmap v19427: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:56.979 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:15:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:15:56.697Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:15:56.979 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:15:56 smithi083 ceph-mon[77242]: pgmap v19427: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:57.067 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:15:56 smithi153 ceph-mon[70044]: pgmap v19427: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:58.970 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:15:58 smithi083 ceph-mon[79930]: pgmap v19428: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:58.971 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:15:58 smithi083 ceph-mon[77242]: pgmap v19428: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:15:59.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:15:58 smithi153 ceph-mon[70044]: pgmap v19428: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:00.977 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:16:00 smithi083 ceph-mon[79930]: pgmap v19429: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:00.978 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:16:00 smithi083 ceph-mon[77242]: pgmap v19429: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:01.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:16:00 smithi153 ceph-mon[70044]: pgmap v19429: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:16:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:16:01] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T16:16:02.668 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:16:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:16:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:16:02.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:16:02 smithi083 ceph-mon[79930]: pgmap v19430: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:02.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:16:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:16:02.606Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:16:02.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:16:02 smithi083 ceph-mon[77242]: pgmap v19430: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:03.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:16:02 smithi153 ceph-mon[70044]: pgmap v19430: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:03.317 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:16:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:16:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:16:04.569 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:16:04 smithi153 ceph-mon[70044]: pgmap v19431: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:04.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:16:04 smithi083 ceph-mon[79930]: pgmap v19431: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:04.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:16:04 smithi083 ceph-mon[77242]: pgmap v19431: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:06.216 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:16:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:16:05.948Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:16:06.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:16:06 smithi083 ceph-mon[79930]: pgmap v19432: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:06.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:16:06 smithi083 ceph-mon[77242]: pgmap v19432: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:06.965 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:16:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:16:06.697Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:16:07.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:16:06 smithi153 ceph-mon[70044]: pgmap v19432: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:08.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:16:08 smithi083 ceph-mon[79930]: pgmap v19433: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:08.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:16:08 smithi083 ceph-mon[77242]: pgmap v19433: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:09.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:16:08 smithi153 ceph-mon[70044]: pgmap v19433: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:10.975 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:16:10 smithi083 ceph-mon[79930]: pgmap v19434: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:10.981 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:16:10 smithi083 ceph-mon[77242]: pgmap v19434: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:11.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:16:10 smithi153 ceph-mon[70044]: pgmap v19434: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:11.972 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:16:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:16:11] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T16:16:12.663 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:16:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:16:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:16:12.750 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:16:12.978 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:16:12 smithi083 ceph-mon[79930]: pgmap v19435: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:12.979 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:16:12 smithi083 ceph-mon[77242]: pgmap v19435: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:12.979 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:16:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:16:12.606Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:16:13.027 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:16:12 smithi153 ceph-mon[70044]: pgmap v19435: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:13.088 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:16:13.089 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 5m ago 10h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:16:13.089 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 3m ago 10h 63.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:16:13.089 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 5m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:16:13.093 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 3m ago 10h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:16:13.093 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 3m ago 10h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:16:13.093 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 5m ago 10h 1060M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:16:13.094 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 5m ago 10h 2335M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:16:13.094 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 3m ago 10h 1045M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:16:13.095 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 5m ago 10h 1404M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:16:13.095 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 5m ago 10h 24.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:16:13.096 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 3m ago 10h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:16:13.096 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 5m ago 10h 1698M 2090M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:16:13.097 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 5m ago 10h 1685M 2090M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:16:13.097 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 5m ago 10h 1308M 2090M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:16:13.098 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 5m ago 10h 1651M 2090M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:16:13.112 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 3m ago 10h 1447M 2498M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:16:13.113 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 3m ago 10h 1819M 2498M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:16:13.113 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 3m ago 10h 1626M 2498M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:16:13.113 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 3m ago 10h 1935M 2498M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:16:13.115 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 3m ago 10h 188M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:16:13.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:16:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:16:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:16:13.431 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:16:13.431 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:16:13.432 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:16:13.433 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:16:13.434 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:16:13.434 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:16:13.435 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:16:13.435 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:16:13.437 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:16:13.437 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:16:13.438 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:16:13.438 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:16:13.450 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:16:13.450 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:16:13.451 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:16:13.451 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:16:13.452 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:16:13.452 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:16:13.980 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:16:13 smithi083 ceph-mon[79930]: from='client.56730 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:16:13.981 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:16:13 smithi083 ceph-mon[79930]: from='client.66667 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:16:13.981 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:16:13 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/713590428' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:16:14.020 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:16:13 smithi083 ceph-mon[77242]: from='client.56730 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:16:14.020 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:16:13 smithi083 ceph-mon[77242]: from='client.66667 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:16:14.041 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:16:13 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/713590428' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:16:14.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:16:13 smithi153 ceph-mon[70044]: from='client.56730 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:16:14.063 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:16:13 smithi153 ceph-mon[70044]: from='client.66667 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:16:14.063 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:16:13 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/713590428' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:16:14.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:16:14 smithi083 ceph-mon[79930]: pgmap v19436: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:14.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:16:14 smithi083 ceph-mon[77242]: pgmap v19436: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:15.075 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:16:14 smithi153 ceph-mon[70044]: pgmap v19436: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:16.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:16:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:16:15.949Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:16:16.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:16:16 smithi083 ceph-mon[79930]: pgmap v19437: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:16.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:16:16 smithi083 ceph-mon[77242]: pgmap v19437: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:16.976 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:16:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:16:16.698Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:16:17.072 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:16:16 smithi153 ceph-mon[70044]: pgmap v19437: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:18.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:16:18 smithi083 ceph-mon[79930]: pgmap v19438: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:18.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:16:18 smithi083 ceph-mon[77242]: pgmap v19438: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:19.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:16:18 smithi153 ceph-mon[70044]: pgmap v19438: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:20.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:16:20 smithi083 ceph-mon[79930]: pgmap v19439: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:20.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:16:20 smithi083 ceph-mon[77242]: pgmap v19439: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:21.068 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:16:20 smithi153 ceph-mon[70044]: pgmap v19439: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:16:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:16:21] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T16:16:22.676 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:16:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:16:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:16:22.968 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:16:22 smithi083 ceph-mon[79930]: pgmap v19440: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:22.969 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:16:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:16:22.607Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:16:22.970 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:16:22 smithi083 ceph-mon[77242]: pgmap v19440: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:23.030 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:16:22 smithi153 ceph-mon[70044]: pgmap v19440: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:23.319 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:16:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:16:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:16:24.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:16:24 smithi153 ceph-mon[70044]: pgmap v19441: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:24.732 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:16:24 smithi083 ceph-mon[79930]: pgmap v19441: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:24.733 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:16:24 smithi083 ceph-mon[77242]: pgmap v19441: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:26.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:16:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:16:25.950Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:16:26.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:16:26 smithi083 ceph-mon[79930]: pgmap v19442: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:26.967 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:16:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:16:26.698Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:16:26.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:16:26 smithi083 ceph-mon[77242]: pgmap v19442: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:27.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:16:26 smithi153 ceph-mon[70044]: pgmap v19442: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:28.969 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:16:28 smithi083 ceph-mon[77242]: pgmap v19443: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:28.970 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:16:28 smithi083 ceph-mon[79930]: pgmap v19443: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:29.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:16:28 smithi153 ceph-mon[70044]: pgmap v19443: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:30.969 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:16:30 smithi083 ceph-mon[79930]: pgmap v19444: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:30.970 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:16:30 smithi083 ceph-mon[77242]: pgmap v19444: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:31.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:16:30 smithi153 ceph-mon[70044]: pgmap v19444: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:16:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:16:31] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T16:16:32.667 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:16:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:16:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:16:32.977 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:16:32 smithi083 ceph-mon[77242]: pgmap v19445: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:32.978 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:16:32 smithi083 ceph-mon[79930]: pgmap v19445: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:32.979 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:16:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:16:32.609Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:16:33.030 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:16:32 smithi153 ceph-mon[70044]: pgmap v19445: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:33.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:16:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:16:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:16:34.572 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:16:34 smithi153 ceph-mon[70044]: pgmap v19446: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:34.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:16:34 smithi083 ceph-mon[79930]: pgmap v19446: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:34.738 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:16:34 smithi083 ceph-mon[77242]: pgmap v19446: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:36.247 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:16:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:16:35.950Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:16:36.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:16:36 smithi083 ceph-mon[79930]: pgmap v19447: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:36.966 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:16:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:16:36.698Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:16:36.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:16:36 smithi083 ceph-mon[77242]: pgmap v19447: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:37.082 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:16:36 smithi153 ceph-mon[70044]: pgmap v19447: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:38.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:16:38 smithi083 ceph-mon[79930]: pgmap v19448: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:38.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:16:38 smithi083 ceph-mon[77242]: pgmap v19448: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:39.065 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:16:38 smithi153 ceph-mon[70044]: pgmap v19448: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:40.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:16:40 smithi083 ceph-mon[79930]: pgmap v19449: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:40.968 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:16:40 smithi083 ceph-mon[77242]: pgmap v19449: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:16:40 smithi153 ceph-mon[70044]: pgmap v19449: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:16:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:16:41] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T16:16:42.679 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:16:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:16:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:16:42.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:16:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:16:42.609Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:16:42.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:16:42 smithi083 ceph-mon[79930]: pgmap v19450: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:42.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:16:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:16:42.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:16:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:16:42.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:16:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:16:42.968 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:16:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:16:42.978 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:16:42 smithi083 ceph-mon[77242]: pgmap v19450: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:42.979 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:16:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:16:42.979 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:16:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:16:42.979 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:16:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:16:42.980 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:16:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:16:43.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:16:42 smithi153 ceph-mon[70044]: pgmap v19450: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:43.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:16:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:16:43.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:16:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:16:43.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:16:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:16:43.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:16:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:16:43.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:16:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:16:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:16:43.736 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:16:44.027 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:16:44.028 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 5m ago 10h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:16:44.029 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 3m ago 10h 63.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:16:44.029 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 5m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:16:44.030 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 3m ago 10h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:16:44.030 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 3m ago 10h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:16:44.035 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 5m ago 10h 1060M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:16:44.035 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 5m ago 10h 2335M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:16:44.037 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 3m ago 10h 1045M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:16:44.038 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 5m ago 10h 1404M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:16:44.038 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 5m ago 10h 24.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:16:44.038 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 3m ago 10h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:16:44.039 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 5m ago 10h 1698M 2090M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:16:44.039 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 5m ago 10h 1685M 2090M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:16:44.040 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 5m ago 10h 1308M 2090M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:16:44.041 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 5m ago 10h 1651M 2090M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:16:44.041 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 3m ago 10h 1447M 2498M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:16:44.042 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 3m ago 10h 1819M 2498M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:16:44.042 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 3m ago 10h 1626M 2498M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:16:44.044 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 3m ago 10h 1935M 2498M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:16:44.045 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 3m ago 10h 188M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:16:44.393 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:16:44.412 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:16:44.412 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:16:44.413 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:16:44.449 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:16:44.450 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:16:44.450 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:16:44.463 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:16:44.464 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:16:44.464 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:16:44.465 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:16:44.465 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:16:44.492 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:16:44.492 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:16:44.492 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:16:44.493 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:16:44.493 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:16:44.493 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:16:44.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:16:44 smithi153 ceph-mon[70044]: pgmap v19451: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:44.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:16:44 smithi153 ceph-mon[70044]: from='client.56748 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:16:44.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:16:44 smithi153 ceph-mon[70044]: from='client.66685 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:16:44.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:16:44 smithi083 ceph-mon[79930]: pgmap v19451: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:44.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:16:44 smithi083 ceph-mon[79930]: from='client.56748 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:16:44.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:16:44 smithi083 ceph-mon[79930]: from='client.66685 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:16:44.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:16:44 smithi083 ceph-mon[77242]: pgmap v19451: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:44.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:16:44 smithi083 ceph-mon[77242]: from='client.56748 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:16:44.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:16:44 smithi083 ceph-mon[77242]: from='client.66685 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:16:45.582 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:16:45 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2655647412' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:16:45.723 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:16:45 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2655647412' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:16:45.724 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:16:45 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2655647412' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:16:46.217 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:16:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:16:45.951Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:16:46.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:16:46 smithi153 ceph-mon[70044]: pgmap v19452: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:46.704 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:16:46 smithi083 ceph-mon[79930]: pgmap v19452: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:46.704 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:16:46 smithi083 ceph-mon[77242]: pgmap v19452: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:47.000 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:16:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:16:46.699Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:16:48.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:16:48 smithi083 ceph-mon[77242]: pgmap v19453: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:48.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:16:48 smithi083 ceph-mon[79930]: pgmap v19453: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:49.070 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:16:48 smithi153 ceph-mon[70044]: pgmap v19453: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:50.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:16:50 smithi083 ceph-mon[79930]: pgmap v19454: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:50.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:16:50 smithi083 ceph-mon[77242]: pgmap v19454: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:51.075 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:16:50 smithi153 ceph-mon[70044]: pgmap v19454: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:16:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:16:51] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T16:16:52.672 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:16:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:16:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:16:52.968 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:16:52 smithi083 ceph-mon[77242]: pgmap v19455: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:52.973 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:16:52 smithi083 ceph-mon[79930]: pgmap v19455: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:52.974 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:16:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:16:52.610Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:16:53.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:16:52 smithi153 ceph-mon[70044]: pgmap v19455: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:53.311 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:16:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:16:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:16:54.561 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:16:54 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:16:54.562 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:16:54 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:16:54.562 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:16:54 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:16:54.563 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:16:54 smithi153 ceph-mon[70044]: pgmap v19456: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:54.563 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:16:54 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:16:54.728 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:16:54 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:16:54.728 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:16:54 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:16:54.729 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:16:54 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:16:54.729 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:16:54 smithi083 ceph-mon[77242]: pgmap v19456: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:54.729 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:16:54 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:16:54.730 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:16:54 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:16:54.730 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:16:54 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:16:54.730 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:16:54 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:16:54.731 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:16:54 smithi083 ceph-mon[79930]: pgmap v19456: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:54.731 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:16:54 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:16:56.239 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:16:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:16:55.951Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:16:56.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:16:56 smithi083 ceph-mon[79930]: pgmap v19457: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:56.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:16:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:16:56.699Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:16:56.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:16:56 smithi083 ceph-mon[77242]: pgmap v19457: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:57.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:16:56 smithi153 ceph-mon[70044]: pgmap v19457: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:58.970 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:16:58 smithi083 ceph-mon[79930]: pgmap v19458: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:58.976 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:16:58 smithi083 ceph-mon[77242]: pgmap v19458: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:16:59.065 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:16:58 smithi153 ceph-mon[70044]: pgmap v19458: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:00.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:17:00 smithi083 ceph-mon[79930]: pgmap v19459: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:00.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:17:00 smithi083 ceph-mon[77242]: pgmap v19459: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:01.063 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:17:00 smithi153 ceph-mon[70044]: pgmap v19459: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:01.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:17:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:17:01] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T16:17:02.692 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:17:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:17:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:17:02.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:17:02 smithi083 ceph-mon[79930]: pgmap v19460: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:02.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:17:02 smithi083 ceph-mon[77242]: pgmap v19460: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:02.965 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:17:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:17:02.610Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:17:03.028 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:17:02 smithi153 ceph-mon[70044]: pgmap v19460: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:03.323 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:17:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:17:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:17:04.565 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:17:04 smithi153 ceph-mon[70044]: pgmap v19461: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:04.719 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:17:04 smithi083 ceph-mon[77242]: pgmap v19461: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:04.727 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:17:04 smithi083 ceph-mon[79930]: pgmap v19461: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:06.246 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:17:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:17:05.951Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:17:06.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:17:06 smithi083 ceph-mon[79930]: pgmap v19462: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:06.986 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:17:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:17:06.699Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:17:06.987 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:17:06 smithi083 ceph-mon[77242]: pgmap v19462: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:07.072 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:17:06 smithi153 ceph-mon[70044]: pgmap v19462: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:08.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:17:08 smithi083 ceph-mon[79930]: pgmap v19463: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:08.970 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:17:08 smithi083 ceph-mon[77242]: pgmap v19463: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:09.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:17:08 smithi153 ceph-mon[70044]: pgmap v19463: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:10.980 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:17:10 smithi083 ceph-mon[79930]: pgmap v19464: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:10.981 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:17:10 smithi083 ceph-mon[77242]: pgmap v19464: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:11.064 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:17:10 smithi153 ceph-mon[70044]: pgmap v19464: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:11.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:17:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:17:11] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T16:17:12.691 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:17:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:17:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:17:12.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:17:12 smithi083 ceph-mon[79930]: pgmap v19465: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:12.988 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:17:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:17:12.610Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:17:12.989 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:17:12 smithi083 ceph-mon[77242]: pgmap v19465: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:13.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:17:12 smithi153 ceph-mon[70044]: pgmap v19465: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:13.324 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:17:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:17:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:17:14.566 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:17:14 smithi153 ceph-mon[70044]: pgmap v19466: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:14.724 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:17:14.756 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:17:14 smithi083 ceph-mon[79930]: pgmap v19466: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:14.757 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:17:14 smithi083 ceph-mon[77242]: pgmap v19466: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:15.001 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:17:15.002 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 6m ago 10h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:17:15.002 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 4m ago 10h 63.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:17:15.002 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 6m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:17:15.003 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 4m ago 10h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:17:15.003 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 4m ago 10h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:17:15.003 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 6m ago 10h 1060M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:17:15.004 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 6m ago 10h 2335M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:17:15.004 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 4m ago 10h 1045M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:17:15.004 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 6m ago 10h 1404M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:17:15.004 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 6m ago 10h 24.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:17:15.005 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 4m ago 10h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:17:15.005 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 6m ago 10h 1698M 2090M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:17:15.005 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 6m ago 10h 1685M 2090M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:17:15.005 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 6m ago 10h 1308M 2090M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:17:15.006 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 6m ago 10h 1651M 2090M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:17:15.006 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 4m ago 10h 1447M 2498M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:17:15.006 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 4m ago 10h 1819M 2498M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:17:15.006 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 4m ago 10h 1626M 2498M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:17:15.007 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 4m ago 10h 1935M 2498M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:17:15.007 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 4m ago 10h 188M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:17:15.362 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:17:15.373 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:17:15.376 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:17:15.377 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:17:15.377 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:17:15.379 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:17:15.379 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:17:15.380 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:17:15.380 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:17:15.380 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:17:15.381 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:17:15.382 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:17:15.382 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:17:15.383 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:17:15.383 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:17:15.383 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:17:15.384 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:17:15.384 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:17:15.575 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:17:15 smithi153 ceph-mon[70044]: from='client.56766 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:17:15.575 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:17:15 smithi153 ceph-mon[70044]: from='client.56772 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:17:15.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:17:15 smithi083 ceph-mon[77242]: from='client.56766 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:17:15.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:17:15 smithi083 ceph-mon[77242]: from='client.56772 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:17:15.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:17:15 smithi083 ceph-mon[79930]: from='client.56766 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:17:15.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:17:15 smithi083 ceph-mon[79930]: from='client.56772 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:17:16.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:17:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:17:15.951Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:17:16.703 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:17:16 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2665550257' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:17:16.704 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:17:16 smithi083 ceph-mon[79930]: pgmap v19467: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:16.704 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:17:16 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2665550257' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:17:16.705 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:17:16 smithi083 ceph-mon[77242]: pgmap v19467: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:16.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:17:16 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2665550257' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:17:16.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:17:16 smithi153 ceph-mon[70044]: pgmap v19467: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:16.973 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:17:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:17:16.699Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:17:18.975 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:17:18 smithi083 ceph-mon[79930]: pgmap v19468: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:18.975 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:17:18 smithi083 ceph-mon[77242]: pgmap v19468: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:19.069 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:17:18 smithi153 ceph-mon[70044]: pgmap v19468: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:20.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:17:20 smithi083 ceph-mon[79930]: pgmap v19469: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:20.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:17:20 smithi083 ceph-mon[77242]: pgmap v19469: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:21.064 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:17:20 smithi153 ceph-mon[70044]: pgmap v19469: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:21.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:17:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:17:21] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T16:17:22.676 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:17:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:17:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:17:22.973 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:17:22 smithi083 ceph-mon[77242]: pgmap v19470: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:22.974 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:17:22 smithi083 ceph-mon[79930]: pgmap v19470: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:22.975 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:17:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:17:22.611Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:17:23.026 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:17:22 smithi153 ceph-mon[70044]: pgmap v19470: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:17:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:17:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:17:24.561 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:17:24 smithi153 ceph-mon[70044]: pgmap v19471: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:24.731 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:17:24 smithi083 ceph-mon[77242]: pgmap v19471: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:24.732 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:17:24 smithi083 ceph-mon[79930]: pgmap v19471: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:26.217 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:17:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:17:25.952Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:17:26.975 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:17:26 smithi083 ceph-mon[79930]: pgmap v19472: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:26.978 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:17:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:17:26.701Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:17:26.979 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:17:26 smithi083 ceph-mon[77242]: pgmap v19472: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:27.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:17:26 smithi153 ceph-mon[70044]: pgmap v19472: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:28.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:17:28 smithi083 ceph-mon[77242]: pgmap v19473: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:28.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:17:28 smithi083 ceph-mon[79930]: pgmap v19473: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:29.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:17:28 smithi153 ceph-mon[70044]: pgmap v19473: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:30.978 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:17:30 smithi083 ceph-mon[77242]: pgmap v19474: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:30.979 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:17:30 smithi083 ceph-mon[79930]: pgmap v19474: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:31.066 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:17:30 smithi153 ceph-mon[70044]: pgmap v19474: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:31.967 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:17:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:17:31] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T16:17:32.695 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:17:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:17:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:17:32.985 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:17:32 smithi083 ceph-mon[79930]: pgmap v19475: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:32.986 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:17:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:17:32.612Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:17:33.003 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:17:32 smithi083 ceph-mon[77242]: pgmap v19475: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:33.030 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:17:32 smithi153 ceph-mon[70044]: pgmap v19475: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:33.310 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:17:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:17:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:17:34.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:17:34 smithi153 ceph-mon[70044]: pgmap v19476: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:34.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:17:34 smithi083 ceph-mon[79930]: pgmap v19476: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:34.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:17:34 smithi083 ceph-mon[77242]: pgmap v19476: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:36.232 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:17:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:17:35.952Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:17:36.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:17:36 smithi083 ceph-mon[77242]: pgmap v19477: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:36.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:17:36 smithi083 ceph-mon[79930]: pgmap v19477: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:36.967 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:17:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:17:36.701Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:17:37.071 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:17:36 smithi153 ceph-mon[70044]: pgmap v19477: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:38.988 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:17:38 smithi083 ceph-mon[77242]: pgmap v19478: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:38.989 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:17:38 smithi083 ceph-mon[79930]: pgmap v19478: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:39.063 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:17:38 smithi153 ceph-mon[70044]: pgmap v19478: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:40.970 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:17:40 smithi083 ceph-mon[77242]: pgmap v19479: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:40.971 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:17:40 smithi083 ceph-mon[79930]: pgmap v19479: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:41.080 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:17:40 smithi153 ceph-mon[70044]: pgmap v19479: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:41.970 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:17:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:17:41] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T16:17:42.700 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:17:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:17:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:17:42.971 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:17:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:17:42.613Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:17:42.972 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:17:42 smithi083 ceph-mon[77242]: pgmap v19480: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:42.975 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:17:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:17:42.975 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:17:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:17:42.976 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:17:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:17:42.976 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:17:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:17:42.977 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:17:42 smithi083 ceph-mon[79930]: pgmap v19480: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:42.977 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:17:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:17:42.978 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:17:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:17:42.978 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:17:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:17:42.979 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:17:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:17:43.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:17:42 smithi153 ceph-mon[70044]: pgmap v19480: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:43.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:17:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:17:43.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:17:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:17:43.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:17:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:17:43.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:17:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:17:43.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:17:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:17:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:17:44.564 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:17:44 smithi153 ceph-mon[70044]: pgmap v19481: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:44.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:17:44 smithi083 ceph-mon[79930]: pgmap v19481: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:44.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:17:44 smithi083 ceph-mon[77242]: pgmap v19481: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:45.682 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:17:45.998 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:17:45.999 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 7m ago 10h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:17:45.999 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 4m ago 10h 63.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:17:46.000 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 7m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:17:46.000 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 4m ago 10h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:17:46.001 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 4m ago 10h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:17:46.001 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 7m ago 10h 1060M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:17:46.001 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 7m ago 10h 2335M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:17:46.023 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 4m ago 10h 1045M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:17:46.024 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 7m ago 10h 1404M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:17:46.024 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 7m ago 10h 24.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:17:46.025 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 4m ago 10h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:17:46.025 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 7m ago 10h 1698M 2090M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:17:46.037 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 7m ago 10h 1685M 2090M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:17:46.037 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 7m ago 10h 1308M 2090M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:17:46.038 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 7m ago 10h 1651M 2090M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:17:46.038 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 4m ago 10h 1447M 2498M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:17:46.039 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 4m ago 10h 1819M 2498M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:17:46.039 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 4m ago 10h 1626M 2498M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:17:46.040 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 4m ago 10h 1935M 2498M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:17:46.040 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 4m ago 10h 188M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:17:46.210 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:17:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:17:45.952Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:17:46.333 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:17:46.337 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:17:46.337 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:17:46.351 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:17:46.352 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:17:46.352 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:17:46.353 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:17:46.353 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:17:46.357 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:17:46.357 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:17:46.358 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:17:46.359 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:17:46.359 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:17:46.360 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:17:46.360 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:17:46.361 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:17:46.361 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:17:46.362 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:17:46.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:17:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:17:46.702Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:17:46.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:17:46 smithi083 ceph-mon[77242]: pgmap v19482: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:46.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:17:46 smithi083 ceph-mon[77242]: from='client.66715 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:17:46.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:17:46 smithi083 ceph-mon[77242]: from='client.56790 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:17:46.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:17:46 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3342605496' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:17:46.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:17:46 smithi083 ceph-mon[79930]: pgmap v19482: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:46.984 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:17:46 smithi083 ceph-mon[79930]: from='client.66715 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:17:46.985 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:17:46 smithi083 ceph-mon[79930]: from='client.56790 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:17:46.990 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:17:46 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3342605496' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:17:47.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:17:46 smithi153 ceph-mon[70044]: pgmap v19482: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:47.063 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:17:46 smithi153 ceph-mon[70044]: from='client.66715 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:17:47.063 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:17:46 smithi153 ceph-mon[70044]: from='client.56790 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:17:47.064 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:17:46 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3342605496' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:17:48.970 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:17:48 smithi083 ceph-mon[77242]: pgmap v19483: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:48.972 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:17:48 smithi083 ceph-mon[79930]: pgmap v19483: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:49.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:17:48 smithi153 ceph-mon[70044]: pgmap v19483: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:50.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:17:50 smithi083 ceph-mon[79930]: pgmap v19484: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:50.968 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:17:50 smithi083 ceph-mon[77242]: pgmap v19484: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:51.095 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:17:50 smithi153 ceph-mon[70044]: pgmap v19484: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:51.982 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:17:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:17:51] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T16:17:52.718 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:17:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:17:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:17:52.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:17:52 smithi083 ceph-mon[79930]: pgmap v19485: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:52.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:17:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:17:52.613Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:17:52.968 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:17:52 smithi083 ceph-mon[77242]: pgmap v19485: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:53.042 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:17:52 smithi153 ceph-mon[70044]: pgmap v19485: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:17:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:17:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:17:54.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:17:54 smithi153 ceph-mon[70044]: pgmap v19486: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:54.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:17:54 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:17:54.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:17:54 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:17:54.561 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:17:54 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:17:54.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:17:54 smithi083 ceph-mon[79930]: pgmap v19486: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:54.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:17:54 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:17:54.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:17:54 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:17:54.718 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:17:54 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:17:54.718 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:17:54 smithi083 ceph-mon[77242]: pgmap v19486: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:54.719 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:17:54 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:17:54.719 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:17:54 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:17:54.719 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:17:54 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:17:56.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:17:55 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:17:56.230 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:17:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:17:55.953Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:17:56.232 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:17:55 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:17:56.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:17:55 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:17:56.971 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:17:56 smithi083 ceph-mon[79930]: pgmap v19487: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:56.972 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:17:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:17:56.702Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:17:56.972 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:17:56 smithi083 ceph-mon[77242]: pgmap v19487: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:57.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:17:56 smithi153 ceph-mon[70044]: pgmap v19487: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:59.219 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:17:58 smithi083 ceph-mon[77242]: pgmap v19488: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:59.230 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:17:58 smithi083 ceph-mon[79930]: pgmap v19488: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:17:59.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:17:58 smithi153 ceph-mon[70044]: pgmap v19488: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:01.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:18:00 smithi083 ceph-mon[79930]: pgmap v19489: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:01.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:18:00 smithi083 ceph-mon[77242]: pgmap v19489: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:01.314 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:18:00 smithi153 ceph-mon[70044]: pgmap v19489: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:01.964 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:18:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:18:01] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T16:18:02.811 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:18:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:18:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:18:02.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:18:02 smithi083 ceph-mon[77242]: pgmap v19490: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:02.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:18:02 smithi083 ceph-mon[79930]: pgmap v19490: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:02.966 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:18:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:18:02.614Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:18:03.313 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:18:02 smithi153 ceph-mon[70044]: pgmap v19490: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:03.314 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:18:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:18:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:18:04.718 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:18:04 smithi083 ceph-mon[77242]: pgmap v19491: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:04.719 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:18:04 smithi083 ceph-mon[79930]: pgmap v19491: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:04.815 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:18:04 smithi153 ceph-mon[70044]: pgmap v19491: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:06.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:18:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:18:05.954Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:18:06.969 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:18:06 smithi083 ceph-mon[79930]: pgmap v19492: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:06.970 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:18:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:18:06.704Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:18:06.972 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:18:06 smithi083 ceph-mon[77242]: pgmap v19492: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:07.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:18:06 smithi153 ceph-mon[70044]: pgmap v19492: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:08.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:18:08 smithi083 ceph-mon[79930]: pgmap v19493: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:08.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:18:08 smithi083 ceph-mon[77242]: pgmap v19493: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:09.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:18:08 smithi153 ceph-mon[70044]: pgmap v19493: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:10.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:18:10 smithi083 ceph-mon[77242]: pgmap v19494: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:10.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:18:10 smithi083 ceph-mon[79930]: pgmap v19494: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:11.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:18:10 smithi153 ceph-mon[70044]: pgmap v19494: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:11.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:18:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:18:11] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T16:18:12.696 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:18:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:18:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:18:12.999 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:18:12 smithi083 ceph-mon[79930]: pgmap v19495: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:13.000 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:18:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:18:12.615Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:18:13.001 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:18:12 smithi083 ceph-mon[77242]: pgmap v19495: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:13.027 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:18:12 smithi153 ceph-mon[70044]: pgmap v19495: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:13.320 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:18:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:18:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:18:14.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:18:14 smithi083 ceph-mon[79930]: pgmap v19496: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:14.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:18:14 smithi083 ceph-mon[77242]: pgmap v19496: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:14.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:18:14 smithi153 ceph-mon[70044]: pgmap v19496: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:16.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:18:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:18:15.955Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:18:16.673 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:18:16.954 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:18:16.955 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 7m ago 10h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:18:16.956 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 5m ago 10h 63.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:18:16.957 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 7m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:18:16.960 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 5m ago 10h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:18:16.961 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 5m ago 10h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:18:16.961 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 7m ago 10h 1060M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:18:16.963 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 7m ago 10h 2335M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:18:16.964 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 5m ago 10h 1045M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:18:16.965 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 7m ago 10h 1404M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:18:16.966 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 7m ago 10h 24.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:18:16.966 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 5m ago 10h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:18:16.967 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 7m ago 10h 1698M 2090M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:18:16.967 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 7m ago 10h 1685M 2090M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:18:16.967 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 7m ago 10h 1308M 2090M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:18:16.968 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 7m ago 10h 1651M 2090M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:18:16.968 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 5m ago 10h 1447M 2498M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:18:16.970 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 5m ago 10h 1819M 2498M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:18:16.971 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 5m ago 10h 1626M 2498M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:18:16.971 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 5m ago 10h 1935M 2498M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:18:16.972 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 5m ago 10h 188M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:18:16.975 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:18:16 smithi083 ceph-mon[79930]: pgmap v19497: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:16.977 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:18:16 smithi083 ceph-mon[77242]: pgmap v19497: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:16.979 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:18:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:18:16.705Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:18:17.067 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:18:16 smithi153 ceph-mon[70044]: pgmap v19497: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:17.314 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:18:17.314 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:18:17.315 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:18:17.315 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:18:17.316 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:18:17.317 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:18:17.318 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:18:17.318 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:18:17.319 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:18:17.319 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:18:17.320 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:18:17.320 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:18:17.321 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:18:17.321 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:18:17.325 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:18:17.326 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:18:17.326 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:18:17.326 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:18:17.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:18:17 smithi083 ceph-mon[79930]: from='client.56802 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:18:17.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:18:17 smithi083 ceph-mon[79930]: from='client.66739 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:18:17.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:18:17 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2954190902' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:18:17.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:18:17 smithi083 ceph-mon[77242]: from='client.56802 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:18:17.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:18:17 smithi083 ceph-mon[77242]: from='client.66739 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:18:17.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:18:17 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2954190902' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:18:18.064 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:18:17 smithi153 ceph-mon[70044]: from='client.56802 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:18:18.065 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:18:17 smithi153 ceph-mon[70044]: from='client.66739 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:18:18.066 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:18:17 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2954190902' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:18:18.969 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:18:18 smithi083 ceph-mon[79930]: pgmap v19498: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:18.970 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:18:18 smithi083 ceph-mon[77242]: pgmap v19498: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:19.067 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:18:18 smithi153 ceph-mon[70044]: pgmap v19498: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:20.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:18:20 smithi083 ceph-mon[79930]: pgmap v19499: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:20.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:18:20 smithi083 ceph-mon[77242]: pgmap v19499: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:21.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:18:20 smithi153 ceph-mon[70044]: pgmap v19499: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:21.969 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:18:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:18:21] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T16:18:22.704 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:18:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:18:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:18:22.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:18:22 smithi083 ceph-mon[79930]: pgmap v19500: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:22.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:18:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:18:22.616Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:18:22.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:18:22 smithi083 ceph-mon[77242]: pgmap v19500: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:23.027 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:18:22 smithi153 ceph-mon[70044]: pgmap v19500: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:23.310 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:18:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:18:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:18:24.719 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:18:24 smithi083 ceph-mon[79930]: pgmap v19501: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:24.720 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:18:24 smithi083 ceph-mon[77242]: pgmap v19501: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:24.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:18:24 smithi153 ceph-mon[70044]: pgmap v19501: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:26.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:18:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:18:25.955Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:18:26.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:18:26 smithi083 ceph-mon[79930]: pgmap v19502: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:26.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:18:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:18:26.704Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:18:26.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:18:26 smithi083 ceph-mon[77242]: pgmap v19502: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:27.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:18:26 smithi153 ceph-mon[70044]: pgmap v19502: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:28.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:18:28 smithi083 ceph-mon[79930]: pgmap v19503: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:28.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:18:28 smithi083 ceph-mon[77242]: pgmap v19503: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:29.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:18:28 smithi153 ceph-mon[70044]: pgmap v19503: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:30.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:18:30 smithi083 ceph-mon[79930]: pgmap v19504: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:30.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:18:30 smithi083 ceph-mon[77242]: pgmap v19504: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:31.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:18:30 smithi153 ceph-mon[70044]: pgmap v19504: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:18:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:18:31] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T16:18:32.708 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:18:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:18:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:18:32.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:18:32 smithi083 ceph-mon[77242]: pgmap v19505: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:32.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:18:32 smithi083 ceph-mon[79930]: pgmap v19505: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:32.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:18:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:18:32.616Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:18:33.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:18:32 smithi153 ceph-mon[70044]: pgmap v19505: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:33.335 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:18:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:18:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:18:34.720 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:18:34 smithi083 ceph-mon[77242]: pgmap v19506: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:34.730 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:18:34 smithi083 ceph-mon[79930]: pgmap v19506: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:34.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:18:34 smithi153 ceph-mon[70044]: pgmap v19506: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:36.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:18:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:18:35.955Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:18:36.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:18:36 smithi083 ceph-mon[77242]: pgmap v19507: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:36.968 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:18:36 smithi083 ceph-mon[79930]: pgmap v19507: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:36.988 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:18:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:18:36.704Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:18:37.065 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:18:36 smithi153 ceph-mon[70044]: pgmap v19507: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:38.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:18:38 smithi083 ceph-mon[79930]: pgmap v19508: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:38.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:18:38 smithi083 ceph-mon[77242]: pgmap v19508: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:39.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:18:38 smithi153 ceph-mon[70044]: pgmap v19508: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:18:40 smithi083 ceph-mon[79930]: pgmap v19509: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:40.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:18:40 smithi083 ceph-mon[77242]: pgmap v19509: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:41.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:18:40 smithi153 ceph-mon[70044]: pgmap v19509: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:18:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:18:41] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T16:18:42.706 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:18:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:18:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:18:42.975 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:18:42 smithi083 ceph-mon[79930]: pgmap v19510: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:42.975 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:18:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:18:42.976 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:18:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:18:42.976 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:18:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:18:42.986 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:18:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:18:42.987 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:18:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:18:42.616Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:18:42.989 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:18:42 smithi083 ceph-mon[77242]: pgmap v19510: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:42.989 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:18:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:18:42.997 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:18:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:18:42.998 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:18:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:18:43.008 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:18:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:18:43.040 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:18:42 smithi153 ceph-mon[70044]: pgmap v19510: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:43.041 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:18:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:18:43.041 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:18:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:18:43.078 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:18:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:18:43.079 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:18:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:18:43.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:18:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:18:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:18:44.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:18:44 smithi083 ceph-mon[79930]: pgmap v19511: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:44.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:18:44 smithi083 ceph-mon[77242]: pgmap v19511: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:44.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:18:44 smithi153 ceph-mon[70044]: pgmap v19511: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:46.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:18:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:18:45.956Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:18:46.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:18:46 smithi083 ceph-mon[79930]: pgmap v19512: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:46.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:18:46 smithi083 ceph-mon[77242]: pgmap v19512: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:46.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:18:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:18:46.705Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:18:47.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:18:46 smithi153 ceph-mon[70044]: pgmap v19512: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:47.654 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:18:47.942 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:18:47.943 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 8m ago 10h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:18:47.943 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 5m ago 10h 63.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:18:47.944 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 8m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:18:47.944 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 5m ago 10h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:18:47.945 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 5m ago 10h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:18:47.945 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 8m ago 10h 1060M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:18:47.945 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 8m ago 10h 2335M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:18:47.968 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 5m ago 10h 1045M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:18:47.969 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 8m ago 10h 1404M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:18:47.969 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 8m ago 10h 24.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:18:47.969 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 5m ago 10h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:18:47.970 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 8m ago 10h 1698M 2090M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:18:47.970 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 8m ago 10h 1685M 2090M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:18:47.971 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 8m ago 10h 1308M 2090M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:18:47.971 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 8m ago 10h 1651M 2090M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:18:47.972 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 5m ago 10h 1447M 2498M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:18:47.972 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 5m ago 10h 1819M 2498M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:18:47.997 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 5m ago 10h 1626M 2498M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:18:47.997 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 5m ago 10h 1935M 2498M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:18:48.007 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 5m ago 10h 188M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:18:48.295 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:18:48.296 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:18:48.296 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:18:48.296 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:18:48.296 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:18:48.297 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:18:48.297 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:18:48.297 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:18:48.302 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:18:48.302 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:18:48.302 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:18:48.303 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:18:48.303 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:18:48.303 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:18:48.303 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:18:48.304 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:18:48.304 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:18:48.304 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:18:48.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:18:48 smithi083 ceph-mon[79930]: from='client.66751 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:18:48.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:18:48 smithi083 ceph-mon[79930]: pgmap v19513: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:48.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:18:48 smithi083 ceph-mon[79930]: from='client.66757 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:18:48.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:18:48 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1310179083' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:18:48.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:18:48 smithi083 ceph-mon[77242]: from='client.66751 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:18:48.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:18:48 smithi083 ceph-mon[77242]: pgmap v19513: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:48.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:18:48 smithi083 ceph-mon[77242]: from='client.66757 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:18:48.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:18:48 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1310179083' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:18:49.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:18:48 smithi153 ceph-mon[70044]: from='client.66751 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:18:49.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:18:48 smithi153 ceph-mon[70044]: pgmap v19513: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:49.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:18:48 smithi153 ceph-mon[70044]: from='client.66757 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:18:49.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:18:48 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1310179083' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:18:50.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:18:50 smithi083 ceph-mon[79930]: pgmap v19514: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:50.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:18:50 smithi083 ceph-mon[77242]: pgmap v19514: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:51.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:18:50 smithi153 ceph-mon[70044]: pgmap v19514: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:18:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:18:51] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T16:18:52.718 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:18:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:18:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:18:52.971 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:18:52 smithi083 ceph-mon[79930]: pgmap v19515: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:52.979 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:18:52 smithi083 ceph-mon[77242]: pgmap v19515: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:52.980 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:18:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:18:52.617Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:18:53.026 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:18:52 smithi153 ceph-mon[70044]: pgmap v19515: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:53.318 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:18:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:18:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:18:54.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:18:54 smithi083 ceph-mon[79930]: pgmap v19516: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:54.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:18:54 smithi083 ceph-mon[77242]: pgmap v19516: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:54.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:18:54 smithi153 ceph-mon[70044]: pgmap v19516: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:55.723 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:18:55 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:18:55.723 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:18:55 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:18:55.723 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:18:55 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:18:55.724 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:18:55 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:18:55.724 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:18:55 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:18:55.724 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:18:55 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:18:55.822 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:18:55 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:18:55.823 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:18:55 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:18:55.823 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:18:55 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:18:56.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:18:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:18:55.955Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:18:56.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:18:56 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:18:56.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:18:56 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:18:56.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:18:56 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:18:56.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:18:56 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:18:56.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:18:56 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:18:56.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:18:56 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:18:56.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:18:56 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:18:56.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:18:56 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:18:56.721 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:18:56 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:18:56.721 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:18:56 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:18:56.731 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:18:56 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:18:56.731 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:18:56 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:18:56.732 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:18:56 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:18:56.732 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:18:56 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:18:56.732 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:18:56 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:18:56.733 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:18:56 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:18:56.733 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:18:56 smithi083 ceph-mon[77242]: Adjusting osd_memory_target on smithi153 to 2494M 2022-09-29T16:18:56.756 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:18:56 smithi083 ceph-mon[77242]: Adjusting osd_memory_target on smithi083 to 2077M 2022-09-29T16:18:56.756 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:18:56 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:18:56.757 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:18:56 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:18:56.761 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:18:56 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:18:56.762 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:18:56 smithi083 ceph-mon[77242]: pgmap v19517: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:56.763 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:18:56 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:18:56.764 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:18:56 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:18:56.764 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:18:56 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:18:56.765 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:18:56 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:18:56.765 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:18:56 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:18:56.765 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:18:56 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:18:56.817 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:18:56 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:18:56.837 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:18:56 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:18:56.837 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:18:56 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:18:56.838 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:18:56 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:18:56.838 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:18:56 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:18:56.839 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:18:56 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:18:56.841 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:18:56 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:18:56.902 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:18:56 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:18:56.904 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:18:56 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:18:56.918 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:18:56 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:18:56.919 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:18:56 smithi083 ceph-mon[79930]: Adjusting osd_memory_target on smithi153 to 2494M 2022-09-29T16:18:56.919 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:18:56 smithi083 ceph-mon[79930]: Adjusting osd_memory_target on smithi083 to 2077M 2022-09-29T16:18:56.920 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:18:56 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:18:56.990 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:18:56 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:18:56.990 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:18:56 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:18:56.991 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:18:56 smithi083 ceph-mon[79930]: pgmap v19517: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:56.992 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:18:56 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:18:56.992 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:18:56 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:18:56.992 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:18:56 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:18:56.992 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:18:56 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:18:56.993 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:18:56 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:18:56.993 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:18:56 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:18:56.993 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:18:56 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:18:56.993 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:18:56 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:18:56.994 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:18:56 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:18:57.008 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:18:56 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:18:57.008 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:18:56 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:18:57.008 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:18:56 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:18:57.008 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:18:56 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:18:57.009 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:18:56 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:18:57.009 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:18:56 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:18:57.009 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:18:56 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:18:57.009 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:18:56 smithi153 ceph-mon[70044]: Adjusting osd_memory_target on smithi153 to 2494M 2022-09-29T16:18:57.010 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:18:56 smithi153 ceph-mon[70044]: Adjusting osd_memory_target on smithi083 to 2077M 2022-09-29T16:18:57.010 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:18:56 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:18:57.010 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:18:56 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:18:57.011 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:18:56 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:18:57.011 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:18:56 smithi153 ceph-mon[70044]: pgmap v19517: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:57.011 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:18:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:18:56.706Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:18:58.989 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:18:58 smithi083 ceph-mon[79930]: pgmap v19518: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:58.990 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:18:58 smithi083 ceph-mon[77242]: pgmap v19518: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:18:59.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:18:58 smithi153 ceph-mon[70044]: pgmap v19518: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:00.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:19:00 smithi083 ceph-mon[79930]: pgmap v19519: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:00.979 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:19:00 smithi083 ceph-mon[77242]: pgmap v19519: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:01.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:19:00 smithi153 ceph-mon[70044]: pgmap v19519: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:01.967 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:19:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:19:01] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T16:19:02.714 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:19:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:19:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:19:02.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:19:02 smithi083 ceph-mon[77242]: pgmap v19520: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:02.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:19:02 smithi083 ceph-mon[79930]: pgmap v19520: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:02.965 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:19:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:19:02.617Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:19:03.047 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:19:02 smithi153 ceph-mon[70044]: pgmap v19520: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:19:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:19:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:19:04.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:19:04 smithi083 ceph-mon[79930]: pgmap v19521: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:04.723 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:19:04 smithi083 ceph-mon[77242]: pgmap v19521: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:04.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:19:04 smithi153 ceph-mon[70044]: pgmap v19521: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:06.214 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:19:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:19:05.957Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:19:06.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:19:06 smithi083 ceph-mon[77242]: pgmap v19522: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:06.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:19:06 smithi083 ceph-mon[79930]: pgmap v19522: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:06.965 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:19:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:19:06.708Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:19:07.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:19:06 smithi153 ceph-mon[70044]: pgmap v19522: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:08.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:19:08 smithi083 ceph-mon[79930]: pgmap v19523: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:08.968 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:19:08 smithi083 ceph-mon[77242]: pgmap v19523: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:09.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:19:08 smithi153 ceph-mon[70044]: pgmap v19523: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:10.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:19:10 smithi083 ceph-mon[79930]: pgmap v19524: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:10.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:19:10 smithi083 ceph-mon[77242]: pgmap v19524: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:11.063 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:19:10 smithi153 ceph-mon[70044]: pgmap v19524: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:19:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:19:11] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T16:19:12.718 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:19:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:19:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:19:12.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:19:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:19:12.618Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:19:12.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:19:12 smithi083 ceph-mon[77242]: pgmap v19525: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:12.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:19:12 smithi083 ceph-mon[79930]: pgmap v19525: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:13.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:19:12 smithi153 ceph-mon[70044]: pgmap v19525: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:19:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:19:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:19:14.725 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:19:14 smithi083 ceph-mon[77242]: pgmap v19526: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:14.726 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:19:14 smithi083 ceph-mon[79930]: pgmap v19526: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:14.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:19:14 smithi153 ceph-mon[70044]: pgmap v19526: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:16.214 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:19:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:19:15.958Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:19:16.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:19:16 smithi083 ceph-mon[79930]: pgmap v19527: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:16.968 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:19:16 smithi083 ceph-mon[77242]: pgmap v19527: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:16.969 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:19:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:19:16.707Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:19:17.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:19:16 smithi153 ceph-mon[70044]: pgmap v19527: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:18.614 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:19:18.902 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:19:18.903 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 8m ago 10h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:19:18.903 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 6m ago 10h 63.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:19:18.903 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 8m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:19:18.904 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 6m ago 10h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:19:18.904 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 6m ago 10h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:19:18.904 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 8m ago 10h 1060M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:19:18.904 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 8m ago 10h 2335M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:19:18.905 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 6m ago 10h 1045M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:19:18.905 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 8m ago 10h 1404M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:19:18.905 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 8m ago 10h 24.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:19:18.905 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 6m ago 10h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:19:18.906 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 8m ago 10h 1698M 2077M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:19:18.906 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 8m ago 10h 1685M 2077M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:19:18.907 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 8m ago 10h 1308M 2077M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:19:18.907 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 8m ago 10h 1651M 2077M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:19:18.907 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 6m ago 10h 1447M 2494M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:19:18.907 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 6m ago 10h 1819M 2494M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:19:18.908 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 6m ago 10h 1626M 2494M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:19:18.908 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 6m ago 10h 1935M 2494M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:19:18.908 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 6m ago 10h 188M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:19:18.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:19:18 smithi083 ceph-mon[79930]: pgmap v19528: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:19.011 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:19:18 smithi083 ceph-mon[77242]: pgmap v19528: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:19.066 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:19:18 smithi153 ceph-mon[70044]: pgmap v19528: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:19.258 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:19:19.259 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:19:19.259 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:19:19.260 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:19:19.260 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:19:19.260 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:19:19.261 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:19:19.261 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:19:19.262 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:19:19.305 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:19:19.318 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:19:19.319 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:19:19.319 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:19:19.320 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:19:19.320 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:19:19.320 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:19:19.321 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:19:19.321 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:19:20.063 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:19:19 smithi153 ceph-mon[70044]: from='client.66769 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:19:20.064 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:19:19 smithi153 ceph-mon[70044]: from='client.56844 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:19:20.064 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:19:19 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2543926254' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:19:20.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:19:19 smithi083 ceph-mon[79930]: from='client.66769 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:19:20.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:19:19 smithi083 ceph-mon[79930]: from='client.56844 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:19:20.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:19:19 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2543926254' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:19:20.222 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:19:19 smithi083 ceph-mon[77242]: from='client.66769 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:19:20.223 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:19:19 smithi083 ceph-mon[77242]: from='client.56844 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:19:20.223 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:19:19 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2543926254' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:19:21.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:19:20 smithi153 ceph-mon[70044]: pgmap v19529: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:21.218 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:19:20 smithi083 ceph-mon[79930]: pgmap v19529: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:21.219 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:19:20 smithi083 ceph-mon[77242]: pgmap v19529: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:19:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:19:21] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T16:19:22.722 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:19:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:19:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:19:22.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:19:22 smithi083 ceph-mon[77242]: pgmap v19530: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:22.979 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:19:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:19:22.618Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:19:22.980 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:19:22 smithi083 ceph-mon[79930]: pgmap v19530: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:23.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:19:22 smithi153 ceph-mon[70044]: pgmap v19530: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:23.310 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:19:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:19:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:19:24.721 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:19:24 smithi083 ceph-mon[79930]: pgmap v19531: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:24.721 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:19:24 smithi083 ceph-mon[77242]: pgmap v19531: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:24.819 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:19:24 smithi153 ceph-mon[70044]: pgmap v19531: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:26.214 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:19:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:19:25.959Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:19:26.971 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:19:26 smithi083 ceph-mon[79930]: pgmap v19532: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:26.972 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:19:26 smithi083 ceph-mon[77242]: pgmap v19532: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:26.973 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:19:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:19:26.708Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:19:27.076 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:19:26 smithi153 ceph-mon[70044]: pgmap v19532: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:29.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:19:28 smithi153 ceph-mon[70044]: pgmap v19533: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:29.218 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:19:28 smithi083 ceph-mon[79930]: pgmap v19533: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:29.219 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:19:28 smithi083 ceph-mon[77242]: pgmap v19533: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:31.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:19:30 smithi153 ceph-mon[70044]: pgmap v19534: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:31.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:19:30 smithi083 ceph-mon[79930]: pgmap v19534: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:31.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:19:30 smithi083 ceph-mon[77242]: pgmap v19534: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:31.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:19:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:19:31] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T16:19:32.722 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:19:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:19:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:19:32.968 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:19:32 smithi083 ceph-mon[79930]: pgmap v19535: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:32.969 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:19:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:19:32.619Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:19:32.970 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:19:32 smithi083 ceph-mon[77242]: pgmap v19535: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:33.022 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:19:32 smithi153 ceph-mon[70044]: pgmap v19535: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:33.318 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:19:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:19:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:19:34.723 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:19:34 smithi083 ceph-mon[77242]: pgmap v19536: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:34.724 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:19:34 smithi083 ceph-mon[79930]: pgmap v19536: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:34.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:19:34 smithi153 ceph-mon[70044]: pgmap v19536: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:36.218 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:19:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:19:35.959Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:19:36.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:19:36 smithi083 ceph-mon[79930]: pgmap v19537: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:36.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:19:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:19:36.708Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:19:36.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:19:36 smithi083 ceph-mon[77242]: pgmap v19537: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:37.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:19:36 smithi153 ceph-mon[70044]: pgmap v19537: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:39.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:19:38 smithi153 ceph-mon[70044]: pgmap v19538: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:39.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:19:38 smithi083 ceph-mon[79930]: pgmap v19538: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:39.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:19:38 smithi083 ceph-mon[77242]: pgmap v19538: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:19:40 smithi153 ceph-mon[70044]: pgmap v19539: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:41.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:19:40 smithi083 ceph-mon[77242]: pgmap v19539: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:41.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:19:40 smithi083 ceph-mon[79930]: pgmap v19539: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:19:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:19:41] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T16:19:42.724 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:19:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:19:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:19:42.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:19:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:19:42.620Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:19:42.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:19:42 smithi083 ceph-mon[77242]: pgmap v19540: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:42.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:19:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:19:42.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:19:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:19:42.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:19:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:19:42.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:19:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:19:42.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:19:42 smithi083 ceph-mon[79930]: pgmap v19540: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:42.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:19:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:19:42.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:19:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:19:42.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:19:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:19:42.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:19:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:19:43.036 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:19:42 smithi153 ceph-mon[70044]: pgmap v19540: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:43.036 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:19:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:19:43.037 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:19:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:19:43.037 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:19:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:19:43.047 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:19:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:19:43.316 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:19:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:19:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:19:44.720 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:19:44 smithi083 ceph-mon[79930]: pgmap v19541: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:44.721 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:19:44 smithi083 ceph-mon[77242]: pgmap v19541: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:44.817 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:19:44 smithi153 ceph-mon[70044]: pgmap v19541: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:46.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:19:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:19:45.960Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:19:46.974 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:19:46 smithi083 ceph-mon[79930]: pgmap v19542: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:46.975 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:19:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:19:46.709Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:19:46.976 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:19:46 smithi083 ceph-mon[77242]: pgmap v19542: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:47.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:19:46 smithi153 ceph-mon[70044]: pgmap v19542: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:49.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:19:48 smithi153 ceph-mon[70044]: pgmap v19543: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:49.224 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:19:48 smithi083 ceph-mon[79930]: pgmap v19543: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:49.225 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:19:48 smithi083 ceph-mon[77242]: pgmap v19543: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:49.581 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:19:49.874 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:19:49.875 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 9m ago 10h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:19:49.876 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 6m ago 10h 63.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:19:49.876 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 9m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:19:49.876 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 6m ago 10h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:19:49.877 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 6m ago 10h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:19:49.877 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 9m ago 10h 1060M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:19:49.891 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 9m ago 10h 2335M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:19:49.892 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 6m ago 10h 1045M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:19:49.892 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 9m ago 10h 1404M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:19:49.903 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 9m ago 10h 24.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:19:49.904 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 6m ago 10h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:19:49.904 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 9m ago 10h 1698M 2077M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:19:49.905 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 9m ago 10h 1685M 2077M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:19:49.905 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 9m ago 10h 1308M 2077M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:19:49.906 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 9m ago 10h 1651M 2077M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:19:49.906 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 6m ago 10h 1447M 2494M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:19:49.907 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 6m ago 10h 1819M 2494M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:19:49.907 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 6m ago 10h 1626M 2494M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:19:49.908 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 6m ago 10h 1935M 2494M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:19:49.908 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 6m ago 10h 188M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:19:50.226 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:19:50.227 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:19:50.227 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:19:50.228 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:19:50.228 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:19:50.229 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:19:50.229 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:19:50.229 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:19:50.229 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:19:50.230 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:19:50.231 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:19:50.231 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:19:50.232 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:19:50.232 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:19:50.234 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:19:50.234 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:19:50.238 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:19:50.238 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:19:51.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:19:50 smithi153 ceph-mon[70044]: from='client.56856 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:19:51.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:19:50 smithi153 ceph-mon[70044]: pgmap v19544: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:51.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:19:50 smithi153 ceph-mon[70044]: from='client.56862 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:19:51.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:19:50 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2321505552' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:19:51.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:19:50 smithi083 ceph-mon[77242]: from='client.56856 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:19:51.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:19:50 smithi083 ceph-mon[77242]: pgmap v19544: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:51.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:19:50 smithi083 ceph-mon[77242]: from='client.56862 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:19:51.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:19:50 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2321505552' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:19:51.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:19:50 smithi083 ceph-mon[79930]: from='client.56856 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:19:51.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:19:50 smithi083 ceph-mon[79930]: pgmap v19544: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:51.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:19:50 smithi083 ceph-mon[79930]: from='client.56862 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:19:51.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:19:50 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2321505552' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:19:51.970 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:19:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:19:51] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T16:19:52.735 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:19:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:19:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:19:52.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:19:52 smithi083 ceph-mon[77242]: pgmap v19545: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:52.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:19:52 smithi083 ceph-mon[79930]: pgmap v19545: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:52.967 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:19:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:19:52.620Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:19:53.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:19:52 smithi153 ceph-mon[70044]: pgmap v19545: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:53.310 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:19:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:19:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:19:54.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:19:54 smithi083 ceph-mon[79930]: pgmap v19546: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:54.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:19:54 smithi083 ceph-mon[77242]: pgmap v19546: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:54.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:19:54 smithi153 ceph-mon[70044]: pgmap v19546: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:55.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:19:55 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:19:55.968 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:19:55 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:19:55.969 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:19:55 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:19:55.975 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:19:55 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:19:55.975 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:19:55 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:19:55.976 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:19:55 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:19:56.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:19:55 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:19:56.063 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:19:55 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:19:56.063 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:19:55 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:19:56.470 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:19:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:19:55.960Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:19:56.973 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:19:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:19:56.709Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:19:56.975 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:19:56 smithi083 ceph-mon[77242]: pgmap v19547: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:56.976 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:19:56 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:19:56.977 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:19:56 smithi083 ceph-mon[79930]: pgmap v19547: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:56.977 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:19:56 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:19:57.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:19:56 smithi153 ceph-mon[70044]: pgmap v19547: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:57.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:19:56 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:19:59.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:19:58 smithi153 ceph-mon[70044]: pgmap v19548: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:59.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:19:58 smithi083 ceph-mon[79930]: pgmap v19548: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:19:59.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:19:58 smithi083 ceph-mon[77242]: pgmap v19548: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:00.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:20:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[77219]: 2022-09-29T16:19:59.999+0000 7f160f665700 -1 log_channel(cluster) log [ERR] : overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T16:20:01.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:20:00 smithi153 ceph-mon[70044]: pgmap v19549: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:01.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:20:00 smithi153 ceph-mon[70044]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T16:20:01.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:20:00 smithi083 ceph-mon[79930]: pgmap v19549: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:01.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:20:00 smithi083 ceph-mon[79930]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T16:20:01.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:20:00 smithi083 ceph-mon[77242]: pgmap v19549: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:01.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:20:00 smithi083 ceph-mon[77242]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T16:20:01.984 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:20:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:20:01] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T16:20:02.727 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:20:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:20:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:20:02.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:20:02 smithi083 ceph-mon[79930]: pgmap v19550: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:02.964 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:20:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:20:02.620Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:20:02.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:20:02 smithi083 ceph-mon[77242]: pgmap v19550: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:03.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:20:02 smithi153 ceph-mon[70044]: pgmap v19550: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:20:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:20:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:20:04.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:20:04 smithi083 ceph-mon[79930]: pgmap v19551: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:04.749 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:20:04 smithi083 ceph-mon[77242]: pgmap v19551: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:04.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:20:04 smithi153 ceph-mon[70044]: pgmap v19551: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:06.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:20:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:20:05.961Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:20:06.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:20:06 smithi083 ceph-mon[79930]: pgmap v19552: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:06.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:20:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:20:06.710Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:20:06.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:20:06 smithi083 ceph-mon[77242]: pgmap v19552: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:07.063 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:20:06 smithi153 ceph-mon[70044]: pgmap v19552: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:09.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:20:08 smithi153 ceph-mon[70044]: pgmap v19553: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:09.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:20:08 smithi083 ceph-mon[79930]: pgmap v19553: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:09.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:20:08 smithi083 ceph-mon[77242]: pgmap v19553: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:11.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:20:10 smithi153 ceph-mon[70044]: pgmap v19554: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:11.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:20:10 smithi083 ceph-mon[79930]: pgmap v19554: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:11.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:20:10 smithi083 ceph-mon[77242]: pgmap v19554: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:20:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:20:11] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T16:20:12.732 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:20:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:20:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:20:12.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:20:12 smithi083 ceph-mon[79930]: pgmap v19555: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:12.967 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:20:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:20:12.621Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:20:12.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:20:12 smithi083 ceph-mon[77242]: pgmap v19555: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:13.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:20:12 smithi153 ceph-mon[70044]: pgmap v19555: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:20:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:20:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:20:14.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:20:14 smithi083 ceph-mon[77242]: pgmap v19556: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:14.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:20:14 smithi083 ceph-mon[79930]: pgmap v19556: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:14.824 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:20:14 smithi153 ceph-mon[70044]: pgmap v19556: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:16.469 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:20:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:20:15.962Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:20:16.727 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:20:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:20:16.710Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:20:17.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:20:16 smithi153 ceph-mon[70044]: pgmap v19557: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:17.219 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:20:16 smithi083 ceph-mon[79930]: pgmap v19557: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:17.223 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:20:16 smithi083 ceph-mon[77242]: pgmap v19557: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:19.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:20:18 smithi153 ceph-mon[70044]: pgmap v19558: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:19.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:20:18 smithi083 ceph-mon[79930]: pgmap v19558: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:19.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:20:18 smithi083 ceph-mon[77242]: pgmap v19558: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:20.552 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:20:20.840 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:20:20.841 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 9m ago 10h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:20:20.841 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 7m ago 10h 63.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:20:20.841 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 9m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:20:20.842 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 7m ago 10h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:20:20.842 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 7m ago 10h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:20:20.842 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 9m ago 10h 1060M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:20:20.843 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 9m ago 10h 2335M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:20:20.843 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 7m ago 10h 1045M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:20:20.843 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 9m ago 10h 1404M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:20:20.843 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 9m ago 10h 24.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:20:20.844 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 7m ago 10h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:20:20.844 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 9m ago 10h 1698M 2077M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:20:20.844 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 9m ago 10h 1685M 2077M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:20:20.845 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 9m ago 10h 1308M 2077M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:20:20.846 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 9m ago 10h 1651M 2077M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:20:20.846 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 7m ago 10h 1447M 2494M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:20:20.846 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 7m ago 10h 1819M 2494M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:20:20.851 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 7m ago 10h 1626M 2494M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:20:20.852 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 7m ago 10h 1935M 2494M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:20:20.852 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 7m ago 10h 188M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:20:21.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:20:20 smithi153 ceph-mon[70044]: pgmap v19559: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:21.195 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:20:21.196 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:20:21.196 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:20:21.197 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:20:21.197 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:20:21.198 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:20:21.200 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:20:21.200 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:20:21.201 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:20:21.201 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:20:21.202 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:20:21.202 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:20:21.203 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:20:21.203 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:20:21.204 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:20:21.204 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:20:21.205 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:20:21.205 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:20:21.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:20:20 smithi083 ceph-mon[77242]: pgmap v19559: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:21.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:20:20 smithi083 ceph-mon[79930]: pgmap v19559: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:20:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:20:21] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T16:20:21.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:20:21 smithi083 ceph-mon[79930]: from='client.66805 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:20:21.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:20:21 smithi083 ceph-mon[79930]: from='client.56880 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:20:21.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:20:21 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1237278900' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:20:21.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:20:21 smithi083 ceph-mon[77242]: from='client.66805 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:20:21.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:20:21 smithi083 ceph-mon[77242]: from='client.56880 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:20:21.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:20:21 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1237278900' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:20:22.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:20:21 smithi153 ceph-mon[70044]: from='client.66805 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:20:22.063 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:20:21 smithi153 ceph-mon[70044]: from='client.56880 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:20:22.063 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:20:21 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1237278900' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:20:22.744 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:20:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:20:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:20:22.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:20:22 smithi083 ceph-mon[79930]: pgmap v19560: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:22.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:20:22 smithi083 ceph-mon[77242]: pgmap v19560: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:22.966 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:20:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:20:22.621Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:20:23.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:20:22 smithi153 ceph-mon[70044]: pgmap v19560: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:23.314 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:20:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:20:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:20:24.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:20:24 smithi083 ceph-mon[79930]: pgmap v19561: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:24.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:20:24 smithi083 ceph-mon[77242]: pgmap v19561: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:24.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:20:24 smithi153 ceph-mon[70044]: pgmap v19561: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:26.465 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:20:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:20:25.962Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:20:27.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:20:26 smithi153 ceph-mon[70044]: pgmap v19562: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:27.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:20:26 smithi083 ceph-mon[79930]: pgmap v19562: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:27.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:20:26 smithi083 ceph-mon[77242]: pgmap v19562: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:27.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:20:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:20:26.711Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:20:29.064 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:20:28 smithi153 ceph-mon[70044]: pgmap v19563: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:29.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:20:28 smithi083 ceph-mon[79930]: pgmap v19563: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:29.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:20:28 smithi083 ceph-mon[77242]: pgmap v19563: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:31.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:20:30 smithi153 ceph-mon[70044]: pgmap v19564: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:31.239 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:20:30 smithi083 ceph-mon[79930]: pgmap v19564: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:31.240 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:20:30 smithi083 ceph-mon[77242]: pgmap v19564: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:20:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:20:31] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T16:20:32.743 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:20:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:20:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:20:32.968 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:20:32 smithi083 ceph-mon[77242]: pgmap v19565: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:32.969 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:20:32 smithi083 ceph-mon[79930]: pgmap v19565: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:32.971 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:20:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:20:32.623Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:20:33.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:20:32 smithi153 ceph-mon[70044]: pgmap v19565: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:20:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:20:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:20:34.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:20:34 smithi083 ceph-mon[79930]: pgmap v19566: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:34.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:20:34 smithi083 ceph-mon[77242]: pgmap v19566: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:34.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:20:34 smithi153 ceph-mon[70044]: pgmap v19566: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:36.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:20:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:20:35.963Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:20:37.067 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:20:36 smithi153 ceph-mon[70044]: pgmap v19567: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:37.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:20:36 smithi083 ceph-mon[79930]: pgmap v19567: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:37.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:20:36 smithi083 ceph-mon[77242]: pgmap v19567: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:37.214 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:20:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:20:36.712Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:20:39.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:20:38 smithi153 ceph-mon[70044]: pgmap v19568: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:39.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:20:38 smithi083 ceph-mon[79930]: pgmap v19568: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:39.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:20:38 smithi083 ceph-mon[77242]: pgmap v19568: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:41.067 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:20:40 smithi153 ceph-mon[70044]: pgmap v19569: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:41.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:20:40 smithi083 ceph-mon[77242]: pgmap v19569: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:41.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:20:40 smithi083 ceph-mon[79930]: pgmap v19569: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:41.974 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:20:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:20:41] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T16:20:42.741 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:20:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:20:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:20:42.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:20:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:20:42.623Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:20:42.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:20:42 smithi083 ceph-mon[77242]: pgmap v19570: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:42.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:20:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:20:42.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:20:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:20:42.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:20:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:20:42.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:20:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:20:42.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:20:42 smithi083 ceph-mon[79930]: pgmap v19570: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:42.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:20:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:20:42.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:20:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:20:42.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:20:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:20:42.968 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:20:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:20:43.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:20:42 smithi153 ceph-mon[70044]: pgmap v19570: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:43.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:20:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:20:43.038 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:20:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:20:43.039 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:20:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:20:43.039 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:20:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:20:43.312 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:20:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:20:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:20:44.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:20:44 smithi083 ceph-mon[79930]: pgmap v19571: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:44.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:20:44 smithi083 ceph-mon[77242]: pgmap v19571: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:44.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:20:44 smithi153 ceph-mon[70044]: pgmap v19571: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:46.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:20:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:20:45.964Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:20:47.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:20:46 smithi153 ceph-mon[70044]: pgmap v19572: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:47.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:20:46 smithi083 ceph-mon[79930]: pgmap v19572: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:47.214 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:20:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:20:46.713Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:20:47.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:20:46 smithi083 ceph-mon[77242]: pgmap v19572: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:49.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:20:48 smithi153 ceph-mon[70044]: pgmap v19573: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:49.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:20:48 smithi083 ceph-mon[79930]: pgmap v19573: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:49.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:20:48 smithi083 ceph-mon[77242]: pgmap v19573: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:51.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:20:50 smithi153 ceph-mon[70044]: pgmap v19574: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:51.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:20:50 smithi083 ceph-mon[79930]: pgmap v19574: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:51.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:20:50 smithi083 ceph-mon[77242]: pgmap v19574: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:51.519 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:20:51.810 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:20:51.811 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 10m ago 10h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:20:51.814 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 8m ago 10h 63.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:20:51.814 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 10m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:20:51.815 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 8m ago 10h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:20:51.815 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 8m ago 10h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:20:51.816 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 10m ago 10h 1060M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:20:51.816 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 10m ago 11h 2335M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:20:51.817 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 8m ago 10h 1045M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:20:51.817 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 10m ago 10h 1404M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:20:51.817 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 10m ago 10h 24.8M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:20:51.818 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 8m ago 10h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:20:51.818 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 10m ago 10h 1698M 2077M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:20:51.819 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 10m ago 10h 1685M 2077M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:20:51.819 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 10m ago 10h 1308M 2077M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:20:51.820 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 10m ago 10h 1651M 2077M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:20:51.820 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 8m ago 10h 1447M 2494M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:20:51.821 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 8m ago 10h 1819M 2494M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:20:51.821 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 8m ago 10h 1626M 2494M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:20:51.822 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 8m ago 10h 1935M 2494M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:20:51.822 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 8m ago 10h 188M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:20:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:20:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:20:51] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T16:20:52.163 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:20:52.163 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:20:52.164 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:20:52.164 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:20:52.165 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:20:52.165 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:20:52.169 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:20:52.169 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:20:52.169 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:20:52.170 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:20:52.171 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:20:52.171 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:20:52.171 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:20:52.171 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:20:52.172 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:20:52.172 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:20:52.172 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:20:52.173 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:20:52.744 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:20:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:20:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:20:52.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:20:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:20:52.623Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:20:52.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:20:52 smithi083 ceph-mon[77242]: from='client.66823 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:20:52.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:20:52 smithi083 ceph-mon[77242]: pgmap v19575: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:52.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:20:52 smithi083 ceph-mon[77242]: from='client.56898 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:20:52.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:20:52 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/581428488' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:20:52.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:20:52 smithi083 ceph-mon[79930]: from='client.66823 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:20:52.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:20:52 smithi083 ceph-mon[79930]: pgmap v19575: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:52.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:20:52 smithi083 ceph-mon[79930]: from='client.56898 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:20:52.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:20:52 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/581428488' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:20:53.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:20:52 smithi153 ceph-mon[70044]: from='client.66823 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:20:53.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:20:52 smithi153 ceph-mon[70044]: pgmap v19575: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:53.026 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:20:52 smithi153 ceph-mon[70044]: from='client.56898 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:20:53.026 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:20:52 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/581428488' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:20:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:20:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:20:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:20:54.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:20:54 smithi083 ceph-mon[79930]: pgmap v19576: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:54.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:20:54 smithi083 ceph-mon[77242]: pgmap v19576: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:54.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:20:54 smithi153 ceph-mon[70044]: pgmap v19576: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:56.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:20:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:20:55.964Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:20:57.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:20:56 smithi153 ceph-mon[70044]: pgmap v19577: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:57.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:20:56 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:20:57.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:20:56 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:20:57.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:20:56 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:20:57.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:20:56 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:20:57.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:20:56 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:20:57.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:20:56 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:20:57.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:20:56 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:20:57.219 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:20:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:20:56.713Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:20:57.219 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:20:56 smithi083 ceph-mon[77242]: pgmap v19577: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:57.220 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:20:56 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:20:57.220 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:20:56 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:20:57.220 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:20:56 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:20:57.220 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:20:56 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:20:57.221 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:20:56 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:20:57.221 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:20:56 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:20:57.222 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:20:56 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:20:57.222 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:20:56 smithi083 ceph-mon[79930]: pgmap v19577: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:57.222 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:20:56 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:20:57.223 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:20:56 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:20:57.223 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:20:56 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:20:57.223 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:20:56 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:20:57.223 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:20:56 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:20:57.224 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:20:56 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:20:57.224 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:20:56 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:20:59.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:20:58 smithi153 ceph-mon[70044]: pgmap v19578: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:59.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:20:58 smithi083 ceph-mon[79930]: pgmap v19578: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:20:59.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:20:58 smithi083 ceph-mon[77242]: pgmap v19578: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:01.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:21:00 smithi153 ceph-mon[70044]: pgmap v19579: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:01.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:21:00 smithi083 ceph-mon[79930]: pgmap v19579: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:01.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:21:00 smithi083 ceph-mon[77242]: pgmap v19579: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:01.769 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:21:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:21:01] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T16:21:01.770 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:21:01 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:21:02.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:21:01 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:21:02.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:21:01 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:21:02.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:21:01 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:21:02.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:21:01 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:21:02.218 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:21:01 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:21:02.219 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:21:01 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:21:02.219 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:21:01 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:21:02.220 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:21:01 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:21:02.220 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:21:01 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:21:02.220 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:21:01 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:21:02.220 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:21:01 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:21:02.777 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:21:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:21:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:21:02.780 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:21:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:21:02.623Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:21:02.780 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:21:02 smithi083 ceph-mon[77242]: pgmap v19580: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:03.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:21:02 smithi153 ceph-mon[70044]: pgmap v19580: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:03.057 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:21:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:21:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:21:03.078 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:21:02 smithi083 ceph-mon[79930]: pgmap v19580: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:04.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:21:04 smithi083 ceph-mon[79930]: pgmap v19581: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:04.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:21:04 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:21:04.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:21:04 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:21:04.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:21:04 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:21:04.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:21:04 smithi083 ceph-mon[77242]: pgmap v19581: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:04.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:21:04 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:21:04.476 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:21:04 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:21:04.476 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:21:04 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:21:04.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:21:04 smithi153 ceph-mon[70044]: pgmap v19581: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:04.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:21:04 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:21:04.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:21:04 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:21:04.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:21:04 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:21:05.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:21:05 smithi083 ceph-mon[79930]: Detected new or changed devices on smithi083 2022-09-29T16:21:05.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:21:05 smithi083 ceph-mon[77242]: Detected new or changed devices on smithi083 2022-09-29T16:21:05.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:21:05 smithi153 ceph-mon[70044]: Detected new or changed devices on smithi083 2022-09-29T16:21:06.364 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:21:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:21:05.965Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:21:06.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:21:06 smithi083 ceph-mon[77242]: pgmap v19582: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:06.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:21:06 smithi083 ceph-mon[79930]: pgmap v19582: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:06.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:21:06 smithi153 ceph-mon[70044]: pgmap v19582: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:07.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:21:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:21:06.714Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:21:09.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:21:08 smithi153 ceph-mon[70044]: pgmap v19583: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:09.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:21:08 smithi083 ceph-mon[77242]: pgmap v19583: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:09.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:21:08 smithi083 ceph-mon[79930]: pgmap v19583: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:11.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:21:10 smithi153 ceph-mon[70044]: pgmap v19584: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:11.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:21:10 smithi083 ceph-mon[79930]: pgmap v19584: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:11.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:21:10 smithi083 ceph-mon[77242]: pgmap v19584: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:11.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:21:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:21:11] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T16:21:12.748 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:21:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:21:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:21:12.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:21:12 smithi083 ceph-mon[79930]: pgmap v19585: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:12.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:21:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:21:12.625Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:21:12.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:21:12 smithi083 ceph-mon[77242]: pgmap v19585: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:13.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:21:12 smithi153 ceph-mon[70044]: pgmap v19585: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:13.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:21:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:21:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:21:14.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:21:14 smithi083 ceph-mon[79930]: pgmap v19586: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:14.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:21:14 smithi083 ceph-mon[77242]: pgmap v19586: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:14.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:21:14 smithi153 ceph-mon[70044]: pgmap v19586: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:16.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:21:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:21:15.965Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:21:17.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:21:16 smithi153 ceph-mon[70044]: pgmap v19587: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:17.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:21:16 smithi083 ceph-mon[79930]: pgmap v19587: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:17.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:21:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:21:16.714Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:21:17.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:21:16 smithi083 ceph-mon[77242]: pgmap v19587: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:19.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:21:18 smithi153 ceph-mon[70044]: pgmap v19588: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:19.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:21:18 smithi083 ceph-mon[79930]: pgmap v19588: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:19.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:21:18 smithi083 ceph-mon[77242]: pgmap v19588: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:21.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:21:20 smithi153 ceph-mon[70044]: pgmap v19589: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:21.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:21:20 smithi083 ceph-mon[79930]: pgmap v19589: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:21.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:21:20 smithi083 ceph-mon[77242]: pgmap v19589: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:21:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:21:21] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T16:21:22.493 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:21:22.752 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:21:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:21:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:21:22.781 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:21:22.782 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 22s ago 10h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:21:22.782 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 8m ago 10h 63.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:21:22.783 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 22s ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:21:22.783 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 8m ago 10h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:21:22.784 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 8m ago 10h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:21:22.784 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 22s ago 11h 1069M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:21:22.785 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 22s ago 11h 2369M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:21:22.785 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 8m ago 10h 1045M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:21:22.786 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 22s ago 10h 1428M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:21:22.786 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 22s ago 10h 24.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:21:22.787 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 8m ago 10h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:21:22.789 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 22s ago 10h 1725M 2077M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:21:22.790 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 22s ago 10h 1711M 2077M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:21:22.791 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 22s ago 10h 1328M 2077M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:21:22.792 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 22s ago 10h 1677M 2077M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:21:22.793 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 8m ago 10h 1447M 2494M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:21:22.793 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 8m ago 10h 1819M 2494M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:21:22.794 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 8m ago 10h 1626M 2494M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:21:22.794 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 8m ago 10h 1935M 2494M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:21:22.795 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 8m ago 10h 188M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:21:22.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:21:22 smithi083 ceph-mon[79930]: pgmap v19590: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:22.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:21:22 smithi083 ceph-mon[77242]: pgmap v19590: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:22.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:21:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:21:22.625Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:21:23.022 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:21:22 smithi153 ceph-mon[70044]: pgmap v19590: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:23.134 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:21:23.134 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:21:23.135 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:21:23.135 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:21:23.135 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:21:23.135 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:21:23.136 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:21:23.136 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:21:23.136 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:21:23.136 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:21:23.137 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:21:23.137 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:21:23.137 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:21:23.137 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:21:23.137 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:21:23.138 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:21:23.138 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:21:23.138 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:21:23.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:21:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:21:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:21:24.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:21:23 smithi153 ceph-mon[70044]: from='client.66841 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:21:24.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:21:23 smithi153 ceph-mon[70044]: from='client.56916 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:21:24.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:21:23 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2061705054' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:21:24.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:21:23 smithi083 ceph-mon[79930]: from='client.66841 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:21:24.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:21:23 smithi083 ceph-mon[79930]: from='client.56916 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:21:24.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:21:23 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2061705054' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:21:24.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:21:23 smithi083 ceph-mon[77242]: from='client.66841 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:21:24.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:21:23 smithi083 ceph-mon[77242]: from='client.56916 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:21:24.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:21:23 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2061705054' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:21:25.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:21:24 smithi153 ceph-mon[70044]: pgmap v19591: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:25.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:21:24 smithi083 ceph-mon[79930]: pgmap v19591: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:25.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:21:24 smithi083 ceph-mon[77242]: pgmap v19591: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:26.460 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:21:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:21:25.966Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:21:27.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:21:26 smithi153 ceph-mon[70044]: pgmap v19592: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:27.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:21:26 smithi083 ceph-mon[77242]: pgmap v19592: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:27.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:21:26 smithi083 ceph-mon[79930]: pgmap v19592: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:27.214 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:21:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:21:26.715Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:21:29.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:21:28 smithi153 ceph-mon[70044]: pgmap v19593: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:29.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:21:28 smithi083 ceph-mon[79930]: pgmap v19593: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:29.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:21:28 smithi083 ceph-mon[77242]: pgmap v19593: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:31.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:21:30 smithi153 ceph-mon[70044]: pgmap v19594: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:31.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:21:30 smithi083 ceph-mon[79930]: pgmap v19594: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:31.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:21:30 smithi083 ceph-mon[77242]: pgmap v19594: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:21:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:21:31] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T16:21:32.773 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:21:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:21:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:21:32.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:21:32 smithi083 ceph-mon[77242]: pgmap v19595: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:32.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:21:32 smithi083 ceph-mon[79930]: pgmap v19595: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:32.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:21:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:21:32.626Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:21:33.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:21:32 smithi153 ceph-mon[70044]: pgmap v19595: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:33.057 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:21:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:21:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:21:34.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:21:34 smithi083 ceph-mon[77242]: pgmap v19596: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:34.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:21:34 smithi083 ceph-mon[79930]: pgmap v19596: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:34.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:21:34 smithi153 ceph-mon[70044]: pgmap v19596: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:36.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:21:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:21:35.966Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:21:37.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:21:36 smithi153 ceph-mon[70044]: pgmap v19597: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:37.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:21:36 smithi083 ceph-mon[77242]: pgmap v19597: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:37.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:21:36 smithi083 ceph-mon[79930]: pgmap v19597: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:37.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:21:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:21:36.715Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:21:39.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:21:38 smithi153 ceph-mon[70044]: pgmap v19598: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:39.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:21:38 smithi083 ceph-mon[77242]: pgmap v19598: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:39.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:21:38 smithi083 ceph-mon[79930]: pgmap v19598: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:41.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:21:40 smithi153 ceph-mon[70044]: pgmap v19599: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:41.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:21:40 smithi083 ceph-mon[79930]: pgmap v19599: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:41.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:21:40 smithi083 ceph-mon[77242]: pgmap v19599: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:21:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:21:41] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T16:21:42.759 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:21:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:21:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:21:42.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:21:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:21:42.627Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:21:42.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:21:42 smithi083 ceph-mon[77242]: pgmap v19600: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:42.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:21:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:21:42.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:21:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:21:42.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:21:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:21:42.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:21:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:21:42.968 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:21:42 smithi083 ceph-mon[79930]: pgmap v19600: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:42.969 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:21:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:21:42.969 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:21:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:21:42.969 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:21:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:21:42.970 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:21:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:21:43.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:21:42 smithi153 ceph-mon[70044]: pgmap v19600: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:43.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:21:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:21:43.026 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:21:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:21:43.026 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:21:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:21:43.027 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:21:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:21:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:21:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:21:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:21:44.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:21:44 smithi083 ceph-mon[79930]: pgmap v19601: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:44.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:21:44 smithi083 ceph-mon[77242]: pgmap v19601: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:44.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:21:44 smithi153 ceph-mon[70044]: pgmap v19601: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:46.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:21:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:21:45.966Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:21:47.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:21:46 smithi153 ceph-mon[70044]: pgmap v19602: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:47.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:21:46 smithi083 ceph-mon[79930]: pgmap v19602: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:47.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:21:46 smithi083 ceph-mon[77242]: pgmap v19602: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:47.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:21:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:21:46.716Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:21:49.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:21:48 smithi153 ceph-mon[70044]: pgmap v19603: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:49.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:21:48 smithi083 ceph-mon[79930]: pgmap v19603: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:49.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:21:48 smithi083 ceph-mon[77242]: pgmap v19603: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:51.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:21:50 smithi153 ceph-mon[70044]: pgmap v19604: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:51.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:21:50 smithi083 ceph-mon[77242]: pgmap v19604: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:51.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:21:50 smithi083 ceph-mon[79930]: pgmap v19604: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:21:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:21:51] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T16:21:52.767 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:21:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:21:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:21:52.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:21:52 smithi083 ceph-mon[79930]: pgmap v19605: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:52.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:21:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:21:52.627Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:21:52.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:21:52 smithi083 ceph-mon[77242]: pgmap v19605: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:53.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:21:52 smithi153 ceph-mon[70044]: pgmap v19605: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:53.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:21:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:21:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:21:53.474 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:21:53.771 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:21:53.771 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 53s ago 10h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:21:53.771 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 9m ago 10h 63.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:21:53.771 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 53s ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:21:53.772 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 9m ago 10h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:21:53.772 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 9m ago 10h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:21:53.772 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 53s ago 11h 1069M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:21:53.772 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 53s ago 11h 2369M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:21:53.773 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 9m ago 10h 1045M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:21:53.773 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 53s ago 10h 1428M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:21:53.773 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 53s ago 10h 24.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:21:53.773 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 9m ago 10h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:21:53.774 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 53s ago 10h 1725M 2077M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:21:53.774 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 53s ago 10h 1711M 2077M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:21:53.774 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 53s ago 10h 1328M 2077M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:21:53.774 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 53s ago 10h 1677M 2077M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:21:53.774 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 9m ago 10h 1447M 2494M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:21:53.775 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 9m ago 10h 1819M 2494M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:21:53.775 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 9m ago 10h 1626M 2494M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:21:53.775 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 9m ago 10h 1935M 2494M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:21:53.775 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 9m ago 10h 188M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:21:54.129 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:21:54.129 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:21:54.130 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:21:54.130 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:21:54.130 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:21:54.130 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:21:54.131 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:21:54.131 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:21:54.131 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:21:54.131 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:21:54.132 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:21:54.132 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:21:54.132 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:21:54.132 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:21:54.133 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:21:54.133 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:21:54.133 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:21:54.133 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:21:54.710 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:21:54 smithi083 ceph-mon[77242]: from='client.66859 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:21:54.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:21:54 smithi083 ceph-mon[77242]: pgmap v19606: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:54.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:21:54 smithi083 ceph-mon[77242]: from='client.56934 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:21:54.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:21:54 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/862755042' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:21:54.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:21:54 smithi083 ceph-mon[79930]: from='client.66859 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:21:54.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:21:54 smithi083 ceph-mon[79930]: pgmap v19606: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:54.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:21:54 smithi083 ceph-mon[79930]: from='client.56934 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:21:54.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:21:54 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/862755042' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:21:54.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:21:54 smithi153 ceph-mon[70044]: from='client.66859 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:21:54.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:21:54 smithi153 ceph-mon[70044]: pgmap v19606: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:54.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:21:54 smithi153 ceph-mon[70044]: from='client.56934 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:21:54.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:21:54 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/862755042' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:21:56.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:21:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:21:55.967Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:21:57.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:21:56 smithi153 ceph-mon[70044]: pgmap v19607: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:57.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:21:56 smithi083 ceph-mon[79930]: pgmap v19607: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:57.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:21:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:21:56.716Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:21:57.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:21:56 smithi083 ceph-mon[77242]: pgmap v19607: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:59.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:21:58 smithi153 ceph-mon[70044]: pgmap v19608: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:59.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:21:58 smithi083 ceph-mon[79930]: pgmap v19608: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:21:59.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:21:58 smithi083 ceph-mon[77242]: pgmap v19608: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:01.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:22:00 smithi153 ceph-mon[70044]: pgmap v19609: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:01.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:22:00 smithi083 ceph-mon[79930]: pgmap v19609: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:01.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:22:00 smithi083 ceph-mon[77242]: pgmap v19609: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:22:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:22:01] "GET /metrics HTTP/1.1" 200 225246 "" "Prometheus/2.33.4" 2022-09-29T16:22:02.765 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:22:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:22:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:22:02.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:22:02 smithi083 ceph-mon[79930]: pgmap v19610: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:02.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:22:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:22:02.628Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:22:02.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:22:02 smithi083 ceph-mon[77242]: pgmap v19610: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:03.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:22:02 smithi153 ceph-mon[70044]: pgmap v19610: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:03.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:22:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:22:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:22:04.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:22:04 smithi083 ceph-mon[79930]: pgmap v19611: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:04.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:22:04 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:22:04.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:22:04 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:22:04.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:22:04 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:22:04.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:22:04 smithi083 ceph-mon[77242]: pgmap v19611: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:04.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:22:04 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:22:04.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:22:04 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:22:04.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:22:04 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:22:04.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:22:04 smithi153 ceph-mon[70044]: pgmap v19611: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:04.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:22:04 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:22:04.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:22:04 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:22:04.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:22:04 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:22:06.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:22:05 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:22:06.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:22:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:22:05.968Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:22:06.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:22:05 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:22:06.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:22:05 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:22:07.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:22:06 smithi083 ceph-mon[79930]: pgmap v19612: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:07.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:22:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:22:06.716Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:22:07.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:22:06 smithi083 ceph-mon[77242]: pgmap v19612: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:07.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:22:06 smithi153 ceph-mon[70044]: pgmap v19612: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:09.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:22:08 smithi083 ceph-mon[79930]: pgmap v19613: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:09.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:22:08 smithi083 ceph-mon[77242]: pgmap v19613: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:09.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:22:08 smithi153 ceph-mon[70044]: pgmap v19613: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:11.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:22:10 smithi083 ceph-mon[79930]: pgmap v19614: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:11.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:22:10 smithi083 ceph-mon[77242]: pgmap v19614: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:11.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:22:10 smithi153 ceph-mon[70044]: pgmap v19614: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:22:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:22:11] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T16:22:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:22:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:22:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:22:12.929 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:22:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:22:12.629Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:22:13.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:22:12 smithi083 ceph-mon[79930]: pgmap v19615: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:13.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:22:12 smithi083 ceph-mon[77242]: pgmap v19615: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:13.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:22:12 smithi153 ceph-mon[70044]: pgmap v19615: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:22:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:22:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:22:14.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:22:14 smithi083 ceph-mon[79930]: pgmap v19616: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:14.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:22:14 smithi083 ceph-mon[77242]: pgmap v19616: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:14.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:22:14 smithi153 ceph-mon[70044]: pgmap v19616: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:16.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:22:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:22:15.968Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:22:17.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:22:16 smithi153 ceph-mon[70044]: pgmap v19617: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:17.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:22:16 smithi083 ceph-mon[77242]: pgmap v19617: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:17.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:22:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:22:16.716Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:22:17.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:22:16 smithi083 ceph-mon[79930]: pgmap v19617: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:19.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:22:18 smithi153 ceph-mon[70044]: pgmap v19618: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:19.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:22:18 smithi083 ceph-mon[77242]: pgmap v19618: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:19.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:22:18 smithi083 ceph-mon[79930]: pgmap v19618: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:21.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:22:20 smithi153 ceph-mon[70044]: pgmap v19619: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:21.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:22:20 smithi083 ceph-mon[77242]: pgmap v19619: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:21.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:22:20 smithi083 ceph-mon[79930]: pgmap v19619: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:22:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:22:21] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T16:22:22.778 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:22:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:22:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:22:22.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:22:22 smithi083 ceph-mon[79930]: pgmap v19620: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:22.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:22:22 smithi083 ceph-mon[77242]: pgmap v19620: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:22.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:22:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:22:22.629Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:22:23.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:22:22 smithi153 ceph-mon[70044]: pgmap v19620: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:23.057 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:22:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:22:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:22:24.454 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:22:24.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:22:24 smithi083 ceph-mon[79930]: pgmap v19621: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:24.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:22:24 smithi083 ceph-mon[77242]: pgmap v19621: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:24.745 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:22:24.745 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 83s ago 10h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:22:24.746 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 9m ago 10h 63.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:22:24.746 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 83s ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:22:24.746 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 9m ago 10h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:22:24.747 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 9m ago 11h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:22:24.747 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 83s ago 11h 1069M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:22:24.747 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 83s ago 11h 2369M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:22:24.747 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 9m ago 11h 1045M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:22:24.748 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 83s ago 11h 1428M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:22:24.748 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 83s ago 10h 24.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:22:24.748 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 9m ago 10h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:22:24.749 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 83s ago 10h 1725M 2077M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:22:24.749 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 83s ago 10h 1711M 2077M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:22:24.749 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 83s ago 10h 1328M 2077M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:22:24.749 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 83s ago 10h 1677M 2077M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:22:24.750 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 9m ago 10h 1447M 2494M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:22:24.750 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 9m ago 10h 1819M 2494M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:22:24.750 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 9m ago 10h 1626M 2494M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:22:24.751 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 9m ago 10h 1935M 2494M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:22:24.752 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 9m ago 10h 188M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:22:24.813 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:22:24 smithi153 ceph-mon[70044]: pgmap v19621: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:25.104 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:22:25.104 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:22:25.105 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:22:25.105 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:22:25.110 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:22:25.110 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:22:25.110 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:22:25.111 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:22:25.111 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:22:25.111 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:22:25.111 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:22:25.112 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:22:25.112 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:22:25.112 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:22:25.112 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:22:25.113 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:22:25.113 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:22:25.113 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:22:25.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:22:25 smithi083 ceph-mon[79930]: from='client.56946 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:22:25.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:22:25 smithi083 ceph-mon[79930]: from='client.66883 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:22:25.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:22:25 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1350518178' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:22:25.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:22:25 smithi083 ceph-mon[77242]: from='client.56946 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:22:25.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:22:25 smithi083 ceph-mon[77242]: from='client.66883 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:22:25.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:22:25 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1350518178' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:22:25.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:22:25 smithi153 ceph-mon[70044]: from='client.56946 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:22:25.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:22:25 smithi153 ceph-mon[70044]: from='client.66883 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:22:25.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:22:25 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1350518178' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:22:26.377 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:22:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:22:25.968Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:22:26.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:22:26 smithi083 ceph-mon[79930]: pgmap v19622: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:26.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:22:26 smithi083 ceph-mon[77242]: pgmap v19622: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:26.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:22:26 smithi153 ceph-mon[70044]: pgmap v19622: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:27.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:22:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:22:26.718Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:22:29.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:22:28 smithi153 ceph-mon[70044]: pgmap v19623: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:29.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:22:28 smithi083 ceph-mon[79930]: pgmap v19623: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:29.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:22:28 smithi083 ceph-mon[77242]: pgmap v19623: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:31.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:22:30 smithi153 ceph-mon[70044]: pgmap v19624: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:31.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:22:30 smithi083 ceph-mon[79930]: pgmap v19624: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:31.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:22:30 smithi083 ceph-mon[77242]: pgmap v19624: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:31.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:22:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:22:31] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T16:22:32.774 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:22:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:22:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:22:32.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:22:32 smithi083 ceph-mon[79930]: pgmap v19625: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:32.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:22:32 smithi083 ceph-mon[77242]: pgmap v19625: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:32.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:22:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:22:32.629Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:22:33.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:22:32 smithi153 ceph-mon[70044]: pgmap v19625: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:33.057 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:22:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:22:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:22:34.722 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:22:34 smithi083 ceph-mon[79930]: pgmap v19626: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:34.723 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:22:34 smithi083 ceph-mon[77242]: pgmap v19626: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:34.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:22:34 smithi153 ceph-mon[70044]: pgmap v19626: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:36.460 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:22:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:22:35.969Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:22:37.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:22:36 smithi153 ceph-mon[70044]: pgmap v19627: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:37.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:22:36 smithi083 ceph-mon[79930]: pgmap v19627: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:37.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:22:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:22:36.718Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:22:37.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:22:36 smithi083 ceph-mon[77242]: pgmap v19627: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:39.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:22:38 smithi153 ceph-mon[70044]: pgmap v19628: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:39.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:22:38 smithi083 ceph-mon[79930]: pgmap v19628: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:39.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:22:38 smithi083 ceph-mon[77242]: pgmap v19628: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:41.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:22:40 smithi153 ceph-mon[70044]: pgmap v19629: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:41.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:22:40 smithi083 ceph-mon[79930]: pgmap v19629: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:41.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:22:40 smithi083 ceph-mon[77242]: pgmap v19629: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:41.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:22:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:22:41] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T16:22:42.778 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:22:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:22:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:22:42.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:22:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:22:42.631Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:22:42.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:22:42 smithi083 ceph-mon[77242]: pgmap v19630: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:42.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:22:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:22:42.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:22:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:22:42.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:22:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:22:42.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:22:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:22:42.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:22:42 smithi083 ceph-mon[79930]: pgmap v19630: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:42.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:22:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:22:42.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:22:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:22:42.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:22:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:22:42.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:22:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:22:43.056 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:22:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:22:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:22:43.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:22:42 smithi153 ceph-mon[70044]: pgmap v19630: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:43.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:22:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:22:43.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:22:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:22:43.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:22:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:22:43.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:22:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:22:44.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:22:44 smithi083 ceph-mon[79930]: pgmap v19631: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:44.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:22:44 smithi083 ceph-mon[77242]: pgmap v19631: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:44.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:22:44 smithi153 ceph-mon[70044]: pgmap v19631: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:46.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:22:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:22:45.970Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:22:47.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:22:46 smithi153 ceph-mon[70044]: pgmap v19632: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:47.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:22:46 smithi083 ceph-mon[79930]: pgmap v19632: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:47.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:22:46 smithi083 ceph-mon[77242]: pgmap v19632: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:47.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:22:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:22:46.719Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:22:49.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:22:48 smithi153 ceph-mon[70044]: pgmap v19633: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:49.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:22:48 smithi083 ceph-mon[79930]: pgmap v19633: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:49.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:22:48 smithi083 ceph-mon[77242]: pgmap v19633: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:51.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:22:50 smithi153 ceph-mon[70044]: pgmap v19634: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:51.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:22:50 smithi083 ceph-mon[79930]: pgmap v19634: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:51.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:22:50 smithi083 ceph-mon[77242]: pgmap v19634: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:51.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:22:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:22:51] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T16:22:52.780 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:22:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:22:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:22:52.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:22:52 smithi083 ceph-mon[79930]: pgmap v19635: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:52.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:22:52 smithi083 ceph-mon[77242]: pgmap v19635: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:52.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:22:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:22:52.631Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:22:53.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:22:52 smithi153 ceph-mon[70044]: pgmap v19635: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:53.057 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:22:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:22:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:22:54.710 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:22:54 smithi083 ceph-mon[77242]: pgmap v19636: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:54.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:22:54 smithi083 ceph-mon[79930]: pgmap v19636: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:54.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:22:54 smithi153 ceph-mon[70044]: pgmap v19636: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:55.426 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:22:55.715 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:22:55.716 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 114s ago 10h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:22:55.716 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 10m ago 10h 63.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:22:55.716 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 114s ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:22:55.716 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 10m ago 10h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:22:55.716 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 10m ago 11h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:22:55.717 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 114s ago 11h 1069M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:22:55.717 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 114s ago 11h 2369M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:22:55.717 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 10m ago 11h 1045M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:22:55.717 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 114s ago 11h 1428M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:22:55.718 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 114s ago 10h 24.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:22:55.718 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 10m ago 10h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:22:55.718 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 114s ago 11h 1725M 2077M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:22:55.718 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 114s ago 10h 1711M 2077M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:22:55.719 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 114s ago 10h 1328M 2077M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:22:55.719 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 114s ago 10h 1677M 2077M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:22:55.719 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 10m ago 10h 1447M 2494M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:22:55.719 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 10m ago 10h 1819M 2494M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:22:55.719 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 10m ago 10h 1626M 2494M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:22:55.720 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 10m ago 10h 1935M 2494M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:22:55.720 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 10m ago 10h 188M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:22:56.076 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:22:56.076 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:22:56.077 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:22:56.077 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:22:56.077 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:22:56.077 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:22:56.077 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:22:56.078 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:22:56.078 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:22:56.078 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:22:56.079 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:22:56.079 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:22:56.079 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:22:56.079 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:22:56.080 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:22:56.080 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:22:56.080 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:22:56.080 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:22:56.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:22:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:22:55.970Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:22:57.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:22:56 smithi153 ceph-mon[70044]: from='client.66895 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:22:57.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:22:56 smithi153 ceph-mon[70044]: from='client.56970 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:22:57.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:22:56 smithi153 ceph-mon[70044]: pgmap v19637: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:57.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:22:56 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4016092634' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:22:57.210 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:22:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:22:56.719Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:22:57.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:22:56 smithi083 ceph-mon[77242]: from='client.66895 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:22:57.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:22:56 smithi083 ceph-mon[77242]: from='client.56970 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:22:57.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:22:56 smithi083 ceph-mon[77242]: pgmap v19637: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:57.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:22:56 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4016092634' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:22:57.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:22:56 smithi083 ceph-mon[79930]: from='client.66895 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:22:57.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:22:56 smithi083 ceph-mon[79930]: from='client.56970 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:22:57.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:22:56 smithi083 ceph-mon[79930]: pgmap v19637: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:57.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:22:56 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4016092634' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:22:59.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:22:58 smithi153 ceph-mon[70044]: pgmap v19638: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:59.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:22:58 smithi083 ceph-mon[79930]: pgmap v19638: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:22:59.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:22:58 smithi083 ceph-mon[77242]: pgmap v19638: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:01.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:23:00 smithi153 ceph-mon[70044]: pgmap v19639: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:01.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:23:00 smithi083 ceph-mon[77242]: pgmap v19639: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:01.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:23:00 smithi083 ceph-mon[79930]: pgmap v19639: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:01.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:23:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:23:01] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T16:23:02.789 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:23:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:23:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:23:02.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:23:02 smithi083 ceph-mon[77242]: pgmap v19640: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:02.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:23:02 smithi083 ceph-mon[79930]: pgmap v19640: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:02.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:23:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:23:02.633Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:23:03.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:23:02 smithi153 ceph-mon[70044]: pgmap v19640: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:03.057 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:23:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:23:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:23:04.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:23:04 smithi083 ceph-mon[77242]: pgmap v19641: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:04.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:23:04 smithi083 ceph-mon[79930]: pgmap v19641: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:04.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:23:04 smithi153 ceph-mon[70044]: pgmap v19641: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:05.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:23:05 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:23:05.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:23:05 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:23:05.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:23:05 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:23:05.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:23:05 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:23:05.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:23:05 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:23:05.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:23:05 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:23:05.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:23:05 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:23:05.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:23:05 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:23:05.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:23:05 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:23:06.389 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:23:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:23:05.970Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:23:06.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:23:06 smithi083 ceph-mon[79930]: pgmap v19642: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:06.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:23:06 smithi083 ceph-mon[77242]: pgmap v19642: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:06.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:23:06 smithi153 ceph-mon[70044]: pgmap v19642: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:07.210 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:23:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:23:06.720Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:23:09.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:23:08 smithi153 ceph-mon[70044]: pgmap v19643: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:09.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:23:08 smithi083 ceph-mon[79930]: pgmap v19643: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:09.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:23:08 smithi083 ceph-mon[77242]: pgmap v19643: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:11.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:23:10 smithi083 ceph-mon[79930]: pgmap v19644: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:11.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:23:10 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:23:11.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:23:10 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:23:11.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:23:10 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:23:11.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:23:10 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:23:11.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:23:10 smithi083 ceph-mon[77242]: pgmap v19644: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:11.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:23:10 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:23:11.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:23:10 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:23:11.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:23:10 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:23:11.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:23:10 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:23:11.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:23:10 smithi153 ceph-mon[70044]: pgmap v19644: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:11.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:23:10 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:23:11.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:23:10 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:23:11.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:23:10 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:23:11.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:23:10 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:23:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:23:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:23:11] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T16:23:12.334 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:23:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:23:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:23:12.950 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:23:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:23:12.633Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:23:13.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:23:12 smithi083 ceph-mon[79930]: pgmap v19645: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:13.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:23:12 smithi083 ceph-mon[77242]: pgmap v19645: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:13.212 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:23:12 smithi153 ceph-mon[70044]: pgmap v19645: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:13.212 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:23:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:23:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:23:14.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:23:14 smithi083 ceph-mon[77242]: Detected new or changed devices on smithi153 2022-09-29T16:23:14.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:23:14 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:23:14.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:23:14 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:23:14.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:23:14 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:23:14.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:23:14 smithi083 ceph-mon[77242]: pgmap v19646: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:14.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:23:14 smithi083 ceph-mon[79930]: Detected new or changed devices on smithi153 2022-09-29T16:23:14.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:23:14 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:23:14.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:23:14 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:23:14.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:23:14 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:23:14.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:23:14 smithi083 ceph-mon[79930]: pgmap v19646: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:14.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:23:14 smithi153 ceph-mon[70044]: Detected new or changed devices on smithi153 2022-09-29T16:23:14.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:23:14 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:23:14.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:23:14 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:23:14.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:23:14 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:23:14.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:23:14 smithi153 ceph-mon[70044]: pgmap v19646: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:16.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:23:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:23:15.972Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:23:17.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:23:16 smithi153 ceph-mon[70044]: pgmap v19647: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:17.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:23:16 smithi083 ceph-mon[79930]: pgmap v19647: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:17.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:23:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:23:16.721Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:23:17.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:23:16 smithi083 ceph-mon[77242]: pgmap v19647: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:19.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:23:18 smithi153 ceph-mon[70044]: pgmap v19648: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:19.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:23:18 smithi083 ceph-mon[79930]: pgmap v19648: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:19.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:23:18 smithi083 ceph-mon[77242]: pgmap v19648: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:21.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:23:20 smithi153 ceph-mon[70044]: pgmap v19649: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:21.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:23:20 smithi083 ceph-mon[79930]: pgmap v19649: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:21.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:23:20 smithi083 ceph-mon[77242]: pgmap v19649: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:23:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:23:21] "GET /metrics HTTP/1.1" 200 225236 "" "Prometheus/2.33.4" 2022-09-29T16:23:22.790 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:23:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:23:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:23:22.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:23:22 smithi083 ceph-mon[79930]: pgmap v19650: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:22.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:23:22 smithi083 ceph-mon[77242]: pgmap v19650: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:22.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:23:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:23:22.634Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:23:23.056 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:23:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:23:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:23:23.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:23:22 smithi153 ceph-mon[70044]: pgmap v19650: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:24.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:23:24 smithi083 ceph-mon[79930]: pgmap v19651: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:24.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:23:24 smithi083 ceph-mon[77242]: pgmap v19651: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:24.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:23:24 smithi153 ceph-mon[70044]: pgmap v19651: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:26.403 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:23:26.460 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:23:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:23:25.972Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:23:26.690 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:23:26.691 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 2m ago 10h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:23:26.691 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 16s ago 10h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:23:26.691 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 2m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:23:26.692 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 16s ago 10h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:23:26.692 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 16s ago 11h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:23:26.692 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 2m ago 11h 1069M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:23:26.693 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 2m ago 11h 2369M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:23:26.693 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 16s ago 11h 1060M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:23:26.693 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 2m ago 11h 1428M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:23:26.693 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 2m ago 10h 24.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:23:26.693 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 16s ago 10h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:23:26.694 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 2m ago 11h 1725M 2077M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:23:26.694 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 2m ago 11h 1711M 2077M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:23:26.694 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 2m ago 11h 1328M 2077M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:23:26.694 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 2m ago 10h 1677M 2077M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:23:26.695 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 16s ago 10h 1469M 2494M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:23:26.695 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 16s ago 10h 1847M 2494M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:23:26.695 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 16s ago 10h 1651M 2494M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:23:26.695 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 16s ago 10h 1965M 2494M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:23:26.696 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 16s ago 10h 190M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:23:27.047 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:23:27.048 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:23:27.048 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:23:27.048 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:23:27.048 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:23:27.049 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:23:27.049 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:23:27.049 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:23:27.049 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:23:27.049 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:23:27.050 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:23:27.050 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:23:27.050 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:23:27.050 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:23:27.051 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:23:27.051 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:23:27.051 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:23:27.051 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:23:27.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:23:26 smithi153 ceph-mon[70044]: pgmap v19652: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:27.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:23:26 smithi083 ceph-mon[79930]: pgmap v19652: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:27.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:23:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:23:26.723Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:23:27.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:23:26 smithi083 ceph-mon[77242]: pgmap v19652: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:28.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:23:27 smithi153 ceph-mon[70044]: from='client.56982 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:23:28.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:23:27 smithi153 ceph-mon[70044]: from='client.56988 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:23:28.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:23:27 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/300122644' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:23:28.210 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:23:27 smithi083 ceph-mon[77242]: from='client.56982 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:23:28.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:23:27 smithi083 ceph-mon[77242]: from='client.56988 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:23:28.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:23:27 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/300122644' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:23:28.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:23:27 smithi083 ceph-mon[79930]: from='client.56982 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:23:28.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:23:27 smithi083 ceph-mon[79930]: from='client.56988 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:23:28.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:23:27 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/300122644' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:23:29.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:23:28 smithi153 ceph-mon[70044]: pgmap v19653: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:29.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:23:28 smithi083 ceph-mon[79930]: pgmap v19653: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:29.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:23:28 smithi083 ceph-mon[77242]: pgmap v19653: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:31.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:23:30 smithi153 ceph-mon[70044]: pgmap v19654: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:31.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:23:30 smithi083 ceph-mon[79930]: pgmap v19654: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:31.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:23:30 smithi083 ceph-mon[77242]: pgmap v19654: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:31.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:23:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:23:31] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T16:23:32.799 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:23:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:23:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:23:32.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:23:32 smithi083 ceph-mon[79930]: pgmap v19655: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:32.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:23:32 smithi083 ceph-mon[77242]: pgmap v19655: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:32.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:23:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:23:32.634Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:23:33.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:23:32 smithi153 ceph-mon[70044]: pgmap v19655: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:33.057 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:23:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:23:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:23:34.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:23:34 smithi083 ceph-mon[77242]: pgmap v19656: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:34.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:23:34 smithi083 ceph-mon[79930]: pgmap v19656: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:34.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:23:34 smithi153 ceph-mon[70044]: pgmap v19656: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:36.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:23:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:23:35.972Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:23:37.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:23:36 smithi153 ceph-mon[70044]: pgmap v19657: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:37.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:23:36 smithi083 ceph-mon[77242]: pgmap v19657: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:37.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:23:36 smithi083 ceph-mon[79930]: pgmap v19657: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:37.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:23:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:23:36.723Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:23:39.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:23:38 smithi153 ceph-mon[70044]: pgmap v19658: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:39.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:23:38 smithi083 ceph-mon[77242]: pgmap v19658: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:39.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:23:38 smithi083 ceph-mon[79930]: pgmap v19658: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:41.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:23:40 smithi153 ceph-mon[70044]: pgmap v19659: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:41.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:23:40 smithi083 ceph-mon[79930]: pgmap v19659: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:41.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:23:40 smithi083 ceph-mon[77242]: pgmap v19659: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:23:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:23:41] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T16:23:42.798 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:23:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:23:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:23:42.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:23:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:23:42.635Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:23:42.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:23:42 smithi083 ceph-mon[79930]: pgmap v19660: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:42.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:23:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:23:42.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:23:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:23:42.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:23:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:23:42.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:23:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:23:42.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:23:42 smithi083 ceph-mon[77242]: pgmap v19660: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:42.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:23:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:23:42.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:23:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:23:42.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:23:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:23:42.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:23:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:23:43.056 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:23:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:23:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:23:43.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:23:42 smithi153 ceph-mon[70044]: pgmap v19660: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:43.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:23:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:23:43.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:23:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:23:43.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:23:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:23:43.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:23:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:23:44.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:23:44 smithi083 ceph-mon[79930]: pgmap v19661: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:44.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:23:44 smithi083 ceph-mon[77242]: pgmap v19661: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:44.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:23:44 smithi153 ceph-mon[70044]: pgmap v19661: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:46.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:23:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:23:45.973Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:23:47.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:23:46 smithi153 ceph-mon[70044]: pgmap v19662: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:47.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:23:46 smithi083 ceph-mon[79930]: pgmap v19662: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:47.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:23:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:23:46.723Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:23:47.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:23:46 smithi083 ceph-mon[77242]: pgmap v19662: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:49.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:23:48 smithi153 ceph-mon[70044]: pgmap v19663: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:49.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:23:48 smithi083 ceph-mon[79930]: pgmap v19663: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:49.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:23:48 smithi083 ceph-mon[77242]: pgmap v19663: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:51.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:23:50 smithi153 ceph-mon[70044]: pgmap v19664: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:51.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:23:50 smithi083 ceph-mon[79930]: pgmap v19664: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:51.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:23:50 smithi083 ceph-mon[77242]: pgmap v19664: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:23:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:23:51] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T16:23:52.802 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:23:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:23:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:23:52.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:23:52 smithi083 ceph-mon[79930]: pgmap v19665: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:52.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:23:52 smithi083 ceph-mon[77242]: pgmap v19665: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:52.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:23:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:23:52.636Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:23:53.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:23:52 smithi153 ceph-mon[70044]: pgmap v19665: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:53.058 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:23:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:23:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:23:54.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:23:54 smithi083 ceph-mon[77242]: pgmap v19666: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:54.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:23:54 smithi083 ceph-mon[79930]: pgmap v19666: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:54.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:23:54 smithi153 ceph-mon[70044]: pgmap v19666: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:56.460 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:23:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:23:55.973Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:23:57.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:23:56 smithi153 ceph-mon[70044]: pgmap v19667: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:57.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:23:56 smithi083 ceph-mon[79930]: pgmap v19667: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:57.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:23:56 smithi083 ceph-mon[77242]: pgmap v19667: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:57.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:23:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:23:56.724Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:23:57.373 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:23:57.661 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:23:57.662 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 2m ago 10h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:23:57.662 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 47s ago 10h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:23:57.663 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 2m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:23:57.664 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 47s ago 10h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:23:57.664 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 47s ago 11h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:23:57.665 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 2m ago 11h 1069M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:23:57.665 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 2m ago 11h 2369M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:23:57.666 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 47s ago 11h 1060M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:23:57.666 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 2m ago 11h 1428M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:23:57.667 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 2m ago 10h 24.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:23:57.667 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 47s ago 10h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:23:57.667 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 2m ago 11h 1725M 2077M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:23:57.668 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 2m ago 11h 1711M 2077M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:23:57.668 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 2m ago 11h 1328M 2077M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:23:57.669 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 2m ago 11h 1677M 2077M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:23:57.669 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 47s ago 11h 1469M 2494M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:23:57.669 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 47s ago 10h 1847M 2494M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:23:57.670 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 47s ago 10h 1651M 2494M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:23:57.670 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 47s ago 10h 1965M 2494M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:23:57.671 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 47s ago 10h 190M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:23:58.017 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:23:58.017 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:23:58.017 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:23:58.017 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:23:58.018 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:23:58.018 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:23:58.018 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:23:58.018 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:23:58.019 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:23:58.019 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:23:58.019 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:23:58.019 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:23:58.020 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:23:58.020 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:23:58.020 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:23:58.020 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:23:58.021 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:23:58.021 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:23:59.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:23:58 smithi153 ceph-mon[70044]: from='client.66931 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:23:59.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:23:58 smithi153 ceph-mon[70044]: from='client.66937 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:23:59.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:23:58 smithi153 ceph-mon[70044]: pgmap v19668: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:59.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:23:58 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2301147421' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:23:59.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:23:58 smithi083 ceph-mon[79930]: from='client.66931 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:23:59.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:23:58 smithi083 ceph-mon[79930]: from='client.66937 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:23:59.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:23:58 smithi083 ceph-mon[79930]: pgmap v19668: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:59.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:23:58 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2301147421' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:23:59.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:23:58 smithi083 ceph-mon[77242]: from='client.66931 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:23:59.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:23:58 smithi083 ceph-mon[77242]: from='client.66937 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:23:59.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:23:58 smithi083 ceph-mon[77242]: pgmap v19668: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:23:59.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:23:58 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2301147421' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:24:01.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:24:00 smithi153 ceph-mon[70044]: pgmap v19669: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:01.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:24:00 smithi083 ceph-mon[79930]: pgmap v19669: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:01.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:24:00 smithi083 ceph-mon[77242]: pgmap v19669: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:24:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:24:01] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T16:24:02.804 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:24:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:24:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:24:02.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:24:02 smithi083 ceph-mon[77242]: pgmap v19670: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:02.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:24:02 smithi083 ceph-mon[79930]: pgmap v19670: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:02.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:24:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:24:02.637Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:24:03.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:24:02 smithi153 ceph-mon[70044]: pgmap v19670: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:03.058 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:24:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:24:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:24:04.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:24:04 smithi083 ceph-mon[79930]: pgmap v19671: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:04.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:24:04 smithi083 ceph-mon[77242]: pgmap v19671: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:04.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:24:04 smithi153 ceph-mon[70044]: pgmap v19671: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:06.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:24:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:24:05.973Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:24:07.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:24:06 smithi153 ceph-mon[70044]: pgmap v19672: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:07.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:24:06 smithi083 ceph-mon[79930]: pgmap v19672: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:07.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:24:06 smithi083 ceph-mon[77242]: pgmap v19672: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:07.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:24:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:24:06.726Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:24:09.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:24:08 smithi153 ceph-mon[70044]: pgmap v19673: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:09.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:24:08 smithi083 ceph-mon[79930]: pgmap v19673: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:09.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:24:08 smithi083 ceph-mon[77242]: pgmap v19673: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:11.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:24:10 smithi153 ceph-mon[70044]: pgmap v19674: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:11.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:24:10 smithi083 ceph-mon[79930]: pgmap v19674: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:11.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:24:10 smithi083 ceph-mon[77242]: pgmap v19674: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:11.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:24:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:24:11] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T16:24:12.805 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:24:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:24:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:24:12.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:24:12 smithi083 ceph-mon[79930]: pgmap v19675: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:12.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:24:12 smithi083 ceph-mon[77242]: pgmap v19675: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:12.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:24:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:24:12.638Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:24:13.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:24:12 smithi153 ceph-mon[70044]: pgmap v19675: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:13.057 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:24:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:24:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:24:14.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:24:14 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:24:14.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:24:14 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:24:14.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:24:14 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:24:14.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:24:14 smithi083 ceph-mon[79930]: pgmap v19676: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:14.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:24:14 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:24:14.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:24:14 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:24:14.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:24:14 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:24:14.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:24:14 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:24:14.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:24:14 smithi083 ceph-mon[77242]: pgmap v19676: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:14.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:24:14 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:24:14.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:24:14 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:24:14.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:24:14 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:24:14.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:24:14 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:24:14.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:24:14 smithi153 ceph-mon[70044]: pgmap v19676: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:14.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:24:14 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:24:16.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:24:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:24:15.973Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:24:17.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:24:16 smithi153 ceph-mon[70044]: pgmap v19677: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:17.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:24:16 smithi083 ceph-mon[79930]: pgmap v19677: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:17.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:24:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:24:16.726Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:24:17.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:24:16 smithi083 ceph-mon[77242]: pgmap v19677: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:19.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:24:18 smithi153 ceph-mon[70044]: pgmap v19678: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:19.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:24:18 smithi083 ceph-mon[77242]: pgmap v19678: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:19.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:24:18 smithi083 ceph-mon[79930]: pgmap v19678: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:21.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:24:20 smithi153 ceph-mon[70044]: pgmap v19679: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:21.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:24:20 smithi083 ceph-mon[79930]: pgmap v19679: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:21.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:24:20 smithi083 ceph-mon[77242]: pgmap v19679: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:21.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:24:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:24:21] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T16:24:22.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:24:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:24:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:24:22.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:24:22 smithi083 ceph-mon[79930]: pgmap v19680: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:22.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:24:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:24:22.639Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:24:22.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:24:22 smithi083 ceph-mon[77242]: pgmap v19680: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:23.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:24:22 smithi153 ceph-mon[70044]: pgmap v19680: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:24:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:24:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:24:24.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:24:24 smithi083 ceph-mon[79930]: pgmap v19681: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:24.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:24:24 smithi083 ceph-mon[77242]: pgmap v19681: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:24.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:24:24 smithi153 ceph-mon[70044]: pgmap v19681: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:26.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:24:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:24:25.973Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:24:27.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:24:26 smithi153 ceph-mon[70044]: pgmap v19682: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:27.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:24:26 smithi083 ceph-mon[79930]: pgmap v19682: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:27.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:24:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:24:26.727Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:24:27.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:24:26 smithi083 ceph-mon[77242]: pgmap v19682: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:28.342 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:24:28.630 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:24:28.630 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 3m ago 10h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:24:28.631 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 78s ago 10h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:24:28.631 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (10h) 3m ago 10h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:24:28.631 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (10h) 78s ago 10h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:24:28.631 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 78s ago 11h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:24:28.632 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 3m ago 11h 1069M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:24:28.632 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 3m ago 11h 2369M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:24:28.632 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 78s ago 11h 1060M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:24:28.632 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 3m ago 11h 1428M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:24:28.633 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 3m ago 10h 24.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:24:28.633 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 78s ago 10h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:24:28.633 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 3m ago 11h 1725M 2077M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:24:28.633 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 3m ago 11h 1711M 2077M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:24:28.634 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 3m ago 11h 1328M 2077M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:24:28.634 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 3m ago 11h 1677M 2077M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:24:28.634 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 78s ago 11h 1469M 2494M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:24:28.634 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 78s ago 11h 1847M 2494M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:24:28.635 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 78s ago 11h 1651M 2494M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:24:28.635 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 78s ago 10h 1965M 2494M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:24:28.635 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 78s ago 10h 190M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:24:28.984 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:24:28.985 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:24:28.985 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:24:28.985 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:24:28.986 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:24:28.986 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:24:28.986 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:24:28.987 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:24:28.987 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:24:28.987 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:24:28.987 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:24:28.988 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:24:28.988 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:24:28.988 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:24:28.989 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:24:28.989 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:24:28.989 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:24:28.989 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:24:29.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:24:28 smithi153 ceph-mon[70044]: pgmap v19683: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:29.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:24:28 smithi083 ceph-mon[79930]: pgmap v19683: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:29.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:24:28 smithi083 ceph-mon[77242]: pgmap v19683: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:30.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:24:29 smithi083 ceph-mon[79930]: from='client.57018 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:24:30.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:24:29 smithi083 ceph-mon[79930]: from='client.57024 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:24:30.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:24:29 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2637483227' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:24:30.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:24:29 smithi083 ceph-mon[77242]: from='client.57018 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:24:30.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:24:29 smithi083 ceph-mon[77242]: from='client.57024 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:24:30.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:24:29 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2637483227' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:24:30.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:24:29 smithi153 ceph-mon[70044]: from='client.57018 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:24:30.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:24:29 smithi153 ceph-mon[70044]: from='client.57024 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:24:30.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:24:29 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2637483227' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:24:31.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:24:30 smithi083 ceph-mon[79930]: pgmap v19684: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:31.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:24:30 smithi083 ceph-mon[77242]: pgmap v19684: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:31.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:24:30 smithi153 ceph-mon[70044]: pgmap v19684: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:24:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:24:31] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T16:24:32.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:24:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:24:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:24:32.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:24:32 smithi083 ceph-mon[79930]: pgmap v19685: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:32.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:24:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:24:32.639Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:24:32.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:24:32 smithi083 ceph-mon[77242]: pgmap v19685: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:33.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:24:32 smithi153 ceph-mon[70044]: pgmap v19685: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:24:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:24:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:24:34.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:24:34 smithi083 ceph-mon[79930]: pgmap v19686: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:34.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:24:34 smithi083 ceph-mon[77242]: pgmap v19686: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:34.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:24:34 smithi153 ceph-mon[70044]: pgmap v19686: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:36.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:24:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:24:35.974Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:24:37.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:24:36 smithi153 ceph-mon[70044]: pgmap v19687: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:37.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:24:36 smithi083 ceph-mon[79930]: pgmap v19687: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:37.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:24:36 smithi083 ceph-mon[77242]: pgmap v19687: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:37.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:24:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:24:36.729Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:24:39.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:24:38 smithi083 ceph-mon[79930]: pgmap v19688: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:39.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:24:38 smithi083 ceph-mon[77242]: pgmap v19688: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:39.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:24:38 smithi153 ceph-mon[70044]: pgmap v19688: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:41.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:24:40 smithi083 ceph-mon[79930]: pgmap v19689: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:41.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:24:40 smithi083 ceph-mon[77242]: pgmap v19689: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:41.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:24:40 smithi153 ceph-mon[70044]: pgmap v19689: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:41.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:24:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:24:41] "GET /metrics HTTP/1.1" 200 225246 "" "Prometheus/2.33.4" 2022-09-29T16:24:42.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:24:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:24:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:24:42.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:24:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:24:42.640Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:24:42.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:24:42 smithi083 ceph-mon[77242]: pgmap v19690: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:42.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:24:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:24:42.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:24:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:24:42.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:24:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:24:42.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:24:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:24:42.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:24:42 smithi083 ceph-mon[79930]: pgmap v19690: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:42.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:24:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:24:42.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:24:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:24:42.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:24:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:24:42.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:24:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:24:43.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:24:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:24:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:24:43.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:24:42 smithi153 ceph-mon[70044]: pgmap v19690: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:43.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:24:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:24:43.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:24:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:24:43.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:24:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:24:43.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:24:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:24:44.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:24:44 smithi083 ceph-mon[79930]: pgmap v19691: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:44.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:24:44 smithi083 ceph-mon[77242]: pgmap v19691: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:44.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:24:44 smithi153 ceph-mon[70044]: pgmap v19691: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:46.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:24:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:24:45.976Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:24:47.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:24:46 smithi083 ceph-mon[79930]: pgmap v19692: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:47.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:24:46 smithi083 ceph-mon[77242]: pgmap v19692: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:47.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:24:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:24:46.729Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:24:47.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:24:46 smithi153 ceph-mon[70044]: pgmap v19692: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:49.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:24:48 smithi083 ceph-mon[77242]: pgmap v19693: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:49.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:24:48 smithi083 ceph-mon[79930]: pgmap v19693: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:49.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:24:48 smithi153 ceph-mon[70044]: pgmap v19693: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:51.210 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:24:50 smithi083 ceph-mon[77242]: pgmap v19694: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:51.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:24:50 smithi083 ceph-mon[79930]: pgmap v19694: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:51.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:24:50 smithi153 ceph-mon[70044]: pgmap v19694: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:24:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:24:51] "GET /metrics HTTP/1.1" 200 225246 "" "Prometheus/2.33.4" 2022-09-29T16:24:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:24:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:24:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:24:52.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:24:52 smithi083 ceph-mon[79930]: pgmap v19695: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:52.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:24:52 smithi083 ceph-mon[77242]: pgmap v19695: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:52.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:24:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:24:52.640Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:24:53.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:24:52 smithi153 ceph-mon[70044]: pgmap v19695: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:53.315 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:24:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:24:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:24:54.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:24:54 smithi083 ceph-mon[79930]: pgmap v19696: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:54.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:24:54 smithi083 ceph-mon[77242]: pgmap v19696: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:54.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:24:54 smithi153 ceph-mon[70044]: pgmap v19696: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:56.460 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:24:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:24:55.977Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:24:57.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:24:56 smithi083 ceph-mon[79930]: pgmap v19697: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:57.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:24:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:24:56.730Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:24:57.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:24:56 smithi083 ceph-mon[77242]: pgmap v19697: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:57.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:24:56 smithi153 ceph-mon[70044]: pgmap v19697: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:59.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:24:58 smithi083 ceph-mon[77242]: pgmap v19698: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:59.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:24:58 smithi083 ceph-mon[79930]: pgmap v19698: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:59.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:24:58 smithi153 ceph-mon[70044]: pgmap v19698: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:24:59.309 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:24:59.599 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:24:59.599 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 3m ago 10h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:24:59.600 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 109s ago 10h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:24:59.600 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 3m ago 11h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:24:59.600 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 109s ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:24:59.600 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 109s ago 11h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:24:59.600 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 3m ago 11h 1069M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:24:59.601 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 3m ago 11h 2369M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:24:59.601 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 109s ago 11h 1060M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:24:59.601 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 3m ago 11h 1428M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:24:59.602 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 3m ago 10h 24.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:24:59.602 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 109s ago 10h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:24:59.602 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 3m ago 11h 1725M 2077M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:24:59.602 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 3m ago 11h 1711M 2077M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:24:59.603 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 3m ago 11h 1328M 2077M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:24:59.603 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 3m ago 11h 1677M 2077M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:24:59.603 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 109s ago 11h 1469M 2494M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:24:59.603 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 109s ago 11h 1847M 2494M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:24:59.604 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 109s ago 11h 1651M 2494M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:24:59.604 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 109s ago 11h 1965M 2494M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:24:59.604 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 109s ago 10h 190M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:24:59.959 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:24:59.959 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:24:59.959 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:24:59.959 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:24:59.960 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:24:59.960 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:24:59.960 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:24:59.960 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:24:59.961 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:24:59.961 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:24:59.961 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:24:59.961 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:24:59.962 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:24:59.962 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:24:59.962 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:24:59.962 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:24:59.963 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:24:59.963 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:25:01.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:25:00 smithi083 ceph-mon[79930]: from='client.66967 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:25:01.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:25:00 smithi083 ceph-mon[79930]: from='client.57042 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:25:01.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:25:00 smithi083 ceph-mon[79930]: pgmap v19699: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:25:01.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:25:00 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2556011532' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:25:01.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:25:00 smithi083 ceph-mon[77242]: from='client.66967 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:25:01.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:25:00 smithi083 ceph-mon[77242]: from='client.57042 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:25:01.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:25:00 smithi083 ceph-mon[77242]: pgmap v19699: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:25:01.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:25:00 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2556011532' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:25:01.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:25:00 smithi153 ceph-mon[70044]: from='client.66967 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:25:01.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:25:00 smithi153 ceph-mon[70044]: from='client.57042 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:25:01.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:25:00 smithi153 ceph-mon[70044]: pgmap v19699: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:25:01.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:25:00 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2556011532' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:25:01.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:25:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:25:01] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T16:25:02.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:25:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:25:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:25:02.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:25:02 smithi083 ceph-mon[79930]: pgmap v19700: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:25:02.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:25:02 smithi083 ceph-mon[77242]: pgmap v19700: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:25:02.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:25:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:25:02.641Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:25:03.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:25:02 smithi153 ceph-mon[70044]: pgmap v19700: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:25:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:25:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:25:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:25:04.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:25:04 smithi083 ceph-mon[79930]: pgmap v19701: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:25:04.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:25:04 smithi083 ceph-mon[77242]: pgmap v19701: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:25:04.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:25:04 smithi153 ceph-mon[70044]: pgmap v19701: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:25:06.460 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:25:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:25:05.977Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:25:07.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:25:06 smithi083 ceph-mon[77242]: pgmap v19702: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:25:07.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:25:06 smithi083 ceph-mon[79930]: pgmap v19702: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:25:07.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:25:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:25:06.731Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:25:07.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:25:06 smithi153 ceph-mon[70044]: pgmap v19702: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:25:09.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:25:08 smithi083 ceph-mon[79930]: pgmap v19703: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:25:09.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:25:08 smithi083 ceph-mon[77242]: pgmap v19703: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:25:09.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:25:08 smithi153 ceph-mon[70044]: pgmap v19703: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:25:11.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:25:10 smithi083 ceph-mon[79930]: pgmap v19704: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:25:11.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:25:10 smithi083 ceph-mon[77242]: pgmap v19704: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:25:11.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:25:10 smithi153 ceph-mon[70044]: pgmap v19704: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:25:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:25:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:25:11] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T16:25:12.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:25:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:25:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:25:12.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:25:12 smithi083 ceph-mon[79930]: pgmap v19705: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:25:12.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:25:12 smithi083 ceph-mon[77242]: pgmap v19705: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:25:12.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:25:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:25:12.641Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:25:13.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:25:12 smithi153 ceph-mon[70044]: pgmap v19705: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:25:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:25:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:25:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:25:14.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:25:14 smithi083 ceph-mon[79930]: pgmap v19706: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:25:14.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:25:14 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:25:14.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:25:14 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:25:14.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:25:14 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:25:14.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:25:14 smithi083 ceph-mon[77242]: pgmap v19706: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:25:14.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:25:14 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:25:14.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:25:14 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:25:14.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:25:14 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:25:14.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:25:14 smithi153 ceph-mon[70044]: pgmap v19706: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:25:14.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:25:14 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:25:14.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:25:14 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:25:14.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:25:14 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:25:16.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:25:15 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:25:16.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:25:15 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:25:16.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:25:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:25:15.977Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:25:16.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:25:15 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:25:17.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:25:16 smithi153 ceph-mon[70044]: pgmap v19707: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:25:17.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:25:16 smithi083 ceph-mon[77242]: pgmap v19707: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:25:17.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:25:16 smithi083 ceph-mon[79930]: pgmap v19707: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:25:17.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:25:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:25:16.731Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:25:19.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:25:18 smithi083 ceph-mon[79930]: pgmap v19708: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:25:19.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:25:18 smithi083 ceph-mon[77242]: pgmap v19708: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:25:19.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:25:18 smithi153 ceph-mon[70044]: pgmap v19708: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:25:21.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:25:20 smithi083 ceph-mon[79930]: pgmap v19709: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:25:21.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:25:20 smithi083 ceph-mon[77242]: pgmap v19709: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:25:21.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:25:20 smithi153 ceph-mon[70044]: pgmap v19709: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:25:21.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:25:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:25:21] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T16:25:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:25:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:25:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:25:22.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:25:22 smithi083 ceph-mon[79930]: pgmap v19710: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:25:22.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:25:22 smithi083 ceph-mon[77242]: pgmap v19710: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:25:22.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:25:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:25:22.642Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:25:23.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:25:22 smithi153 ceph-mon[70044]: pgmap v19710: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:25:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:25:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:25:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:25:24.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:25:24 smithi083 ceph-mon[79930]: pgmap v19711: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:25:24.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:25:24 smithi083 ceph-mon[77242]: pgmap v19711: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:25:24.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:25:24 smithi153 ceph-mon[70044]: pgmap v19711: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:25:26.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:25:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:25:25.977Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:25:27.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:25:26 smithi083 ceph-mon[79930]: pgmap v19712: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:25:27.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:25:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:25:26.732Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:25:27.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:25:26 smithi083 ceph-mon[77242]: pgmap v19712: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:25:27.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:25:26 smithi153 ceph-mon[70044]: pgmap v19712: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:25:29.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:25:28 smithi083 ceph-mon[79930]: pgmap v19713: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:25:29.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:25:28 smithi083 ceph-mon[77242]: pgmap v19713: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:25:29.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:25:28 smithi153 ceph-mon[70044]: pgmap v19713: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:25:30.282 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:25:30.570 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:25:30.570 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 4m ago 10h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:25:30.570 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (10h) 2m ago 10h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:25:30.570 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 4m ago 11h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:25:30.571 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 2m ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:25:30.571 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 2m ago 11h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:25:30.571 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 4m ago 11h 1069M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:25:30.571 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 4m ago 11h 2369M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:25:30.572 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 2m ago 11h 1060M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:25:30.572 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 4m ago 11h 1428M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:25:30.572 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (10h) 4m ago 10h 24.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:25:30.572 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (10h) 2m ago 10h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:25:30.572 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 4m ago 11h 1725M 2077M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:25:30.573 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 4m ago 11h 1711M 2077M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:25:30.573 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 4m ago 11h 1328M 2077M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:25:30.573 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 4m ago 11h 1677M 2077M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:25:30.573 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 2m ago 11h 1469M 2494M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:25:30.574 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 2m ago 11h 1847M 2494M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:25:30.574 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 2m ago 11h 1651M 2494M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:25:30.574 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 2m ago 11h 1965M 2494M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:25:30.574 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 2m ago 10h 190M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:25:30.926 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:25:30.926 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:25:30.927 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:25:30.927 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:25:30.927 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:25:30.927 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:25:30.928 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:25:30.928 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:25:30.928 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:25:30.929 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:25:30.929 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:25:30.930 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:25:30.930 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:25:30.931 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:25:30.931 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:25:30.932 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:25:30.932 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:25:30.932 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:25:31.210 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:25:30 smithi083 ceph-mon[77242]: pgmap v19714: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:25:31.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:25:30 smithi083 ceph-mon[79930]: pgmap v19714: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:25:31.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:25:30 smithi153 ceph-mon[70044]: pgmap v19714: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:25:31.833 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:25:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:25:31] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T16:25:31.834 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:25:31 smithi083 ceph-mon[77242]: from='client.57054 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:25:31.834 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:25:31 smithi083 ceph-mon[77242]: from='client.66991 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:25:31.834 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:25:31 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2044210995' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:25:32.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:25:31 smithi083 ceph-mon[79930]: from='client.57054 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:25:32.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:25:31 smithi083 ceph-mon[79930]: from='client.66991 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:25:32.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:25:31 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2044210995' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:25:32.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:25:31 smithi153 ceph-mon[70044]: from='client.57054 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:25:32.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:25:31 smithi153 ceph-mon[70044]: from='client.66991 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:25:32.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:25:31 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2044210995' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:25:32.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:25:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:25:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:25:32.960 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:25:32 smithi083 ceph-mon[77242]: pgmap v19715: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:25:32.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:25:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:25:32.643Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:25:32.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:25:32 smithi083 ceph-mon[79930]: pgmap v19715: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:25:33.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:25:32 smithi153 ceph-mon[70044]: pgmap v19715: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:25:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:25:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:25:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:25:34.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:25:34 smithi083 ceph-mon[77242]: pgmap v19716: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:25:34.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:25:34 smithi083 ceph-mon[79930]: pgmap v19716: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:25:34.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:25:34 smithi153 ceph-mon[70044]: pgmap v19716: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:25:36.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:25:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:25:35.978Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:25:37.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:25:36 smithi083 ceph-mon[77242]: pgmap v19717: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:25:37.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:25:36 smithi083 ceph-mon[79930]: pgmap v19717: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:25:37.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:25:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:25:36.732Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:25:37.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:25:36 smithi153 ceph-mon[70044]: pgmap v19717: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:25:39.210 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:25:38 smithi083 ceph-mon[77242]: pgmap v19718: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:25:39.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:25:38 smithi083 ceph-mon[79930]: pgmap v19718: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:25:39.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:25:38 smithi153 ceph-mon[70044]: pgmap v19718: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:25:41.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:25:40 smithi083 ceph-mon[79930]: pgmap v19719: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:25:41.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:25:40 smithi083 ceph-mon[77242]: pgmap v19719: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:25:41.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:25:40 smithi153 ceph-mon[70044]: pgmap v19719: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:25:41.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:25:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:25:41] "GET /metrics HTTP/1.1" 200 225246 "" "Prometheus/2.33.4" 2022-09-29T16:25:42.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:25:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:25:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:25:42.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:25:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:25:42.643Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:25:42.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:25:42 smithi083 ceph-mon[77242]: pgmap v19720: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:25:42.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:25:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:25:42.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:25:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:25:42.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:25:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:25:42.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:25:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:25:42.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:25:42 smithi083 ceph-mon[79930]: pgmap v19720: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:25:42.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:25:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:25:42.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:25:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:25:42.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:25:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:25:42.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:25:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:25:43.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:25:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:25:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:25:43.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:25:42 smithi153 ceph-mon[70044]: pgmap v19720: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:25:43.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:25:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:25:43.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:25:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:25:43.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:25:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:25:43.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:25:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:25:44.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:25:44 smithi083 ceph-mon[79930]: pgmap v19721: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:25:44.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:25:44 smithi083 ceph-mon[77242]: pgmap v19721: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:25:44.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:25:44 smithi153 ceph-mon[70044]: pgmap v19721: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:25:46.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:25:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:25:45.979Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:25:47.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:25:46 smithi083 ceph-mon[79930]: pgmap v19722: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:25:47.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:25:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:25:46.732Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:25:47.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:25:46 smithi083 ceph-mon[77242]: pgmap v19722: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:25:47.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:25:46 smithi153 ceph-mon[70044]: pgmap v19722: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:25:49.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:25:48 smithi083 ceph-mon[79930]: pgmap v19723: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:25:49.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:25:48 smithi083 ceph-mon[77242]: pgmap v19723: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:25:49.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:25:48 smithi153 ceph-mon[70044]: pgmap v19723: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:25:51.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:25:50 smithi083 ceph-mon[79930]: pgmap v19724: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:25:51.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:25:50 smithi083 ceph-mon[77242]: pgmap v19724: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:25:51.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:25:50 smithi153 ceph-mon[70044]: pgmap v19724: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:25:51.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:25:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:25:51] "GET /metrics HTTP/1.1" 200 225246 "" "Prometheus/2.33.4" 2022-09-29T16:25:52.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:25:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:25:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:25:52.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:25:52 smithi083 ceph-mon[77242]: pgmap v19725: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:25:52.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:25:52 smithi083 ceph-mon[79930]: pgmap v19725: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:25:52.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:25:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:25:52.643Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:25:53.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:25:52 smithi153 ceph-mon[70044]: pgmap v19725: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:25:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:25:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:25:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:25:54.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:25:54 smithi083 ceph-mon[79930]: pgmap v19726: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:25:54.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:25:54 smithi083 ceph-mon[77242]: pgmap v19726: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:25:54.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:25:54 smithi153 ceph-mon[70044]: pgmap v19726: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:25:56.460 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:25:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:25:55.980Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:25:57.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:25:56 smithi083 ceph-mon[77242]: pgmap v19727: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:25:57.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:25:56 smithi083 ceph-mon[79930]: pgmap v19727: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:25:57.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:25:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:25:56.733Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:25:57.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:25:56 smithi153 ceph-mon[70044]: pgmap v19727: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:25:59.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:25:58 smithi083 ceph-mon[79930]: pgmap v19728: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:25:59.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:25:58 smithi083 ceph-mon[77242]: pgmap v19728: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:25:59.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:25:58 smithi153 ceph-mon[70044]: pgmap v19728: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:01.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:26:00 smithi083 ceph-mon[79930]: pgmap v19729: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:01.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:26:00 smithi083 ceph-mon[77242]: pgmap v19729: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:01.252 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:26:01.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:26:00 smithi153 ceph-mon[70044]: pgmap v19729: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:01.541 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:26:01.541 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (10h) 5m ago 11h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:26:01.541 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 2m ago 10h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:26:01.542 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 5m ago 11h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:26:01.542 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 2m ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:26:01.542 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 2m ago 11h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:26:01.542 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 5m ago 11h 1069M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:26:01.543 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 5m ago 11h 2369M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:26:01.543 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 2m ago 11h 1060M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:26:01.543 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 5m ago 11h 1428M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:26:01.543 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 5m ago 11h 24.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:26:01.544 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 2m ago 11h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:26:01.544 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 5m ago 11h 1725M 2077M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:26:01.544 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 5m ago 11h 1711M 2077M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:26:01.544 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 5m ago 11h 1328M 2077M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:26:01.544 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 5m ago 11h 1677M 2077M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:26:01.545 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 2m ago 11h 1469M 2494M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:26:01.545 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 2m ago 11h 1847M 2494M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:26:01.545 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 2m ago 11h 1651M 2494M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:26:01.545 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 2m ago 11h 1965M 2494M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:26:01.546 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 2m ago 11h 190M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:26:01.897 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:26:01.897 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:26:01.898 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:26:01.898 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:26:01.898 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:26:01.898 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:26:01.899 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:26:01.899 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:26:01.899 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:26:01.900 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:26:01.900 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:26:01.900 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:26:01.900 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:26:01.901 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:26:01.901 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:26:01.901 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:26:01.901 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:26:01.902 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:26:01.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:26:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:26:01] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T16:26:02.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:26:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:26:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:26:02.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:26:02 smithi083 ceph-mon[79930]: from='client.67003 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:26:02.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:26:02 smithi083 ceph-mon[79930]: from='client.57078 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:26:02.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:26:02 smithi083 ceph-mon[79930]: pgmap v19730: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:02.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:26:02 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1838025581' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:26:02.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:26:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:26:02.644Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:26:02.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:26:02 smithi083 ceph-mon[77242]: from='client.67003 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:26:02.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:26:02 smithi083 ceph-mon[77242]: from='client.57078 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:26:02.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:26:02 smithi083 ceph-mon[77242]: pgmap v19730: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:02.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:26:02 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1838025581' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:26:03.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:26:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:26:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:26:03.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:26:02 smithi153 ceph-mon[70044]: from='client.67003 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:26:03.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:26:02 smithi153 ceph-mon[70044]: from='client.57078 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:26:03.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:26:02 smithi153 ceph-mon[70044]: pgmap v19730: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:03.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:26:02 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1838025581' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:26:04.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:26:04 smithi083 ceph-mon[79930]: pgmap v19731: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:04.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:26:04 smithi083 ceph-mon[77242]: pgmap v19731: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:04.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:26:04 smithi153 ceph-mon[70044]: pgmap v19731: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:06.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:26:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:26:05.981Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:26:07.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:26:06 smithi083 ceph-mon[79930]: pgmap v19732: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:07.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:26:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:26:06.733Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:26:07.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:26:06 smithi083 ceph-mon[77242]: pgmap v19732: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:07.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:26:06 smithi153 ceph-mon[70044]: pgmap v19732: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:09.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:26:08 smithi083 ceph-mon[79930]: pgmap v19733: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:09.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:26:08 smithi083 ceph-mon[77242]: pgmap v19733: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:09.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:26:08 smithi153 ceph-mon[70044]: pgmap v19733: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:11.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:26:10 smithi083 ceph-mon[79930]: pgmap v19734: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:11.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:26:10 smithi083 ceph-mon[77242]: pgmap v19734: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:11.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:26:10 smithi153 ceph-mon[70044]: pgmap v19734: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:11.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:26:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:26:11] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T16:26:12.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:26:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:26:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:26:12.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:26:12 smithi083 ceph-mon[79930]: pgmap v19735: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:12.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:26:12 smithi083 ceph-mon[77242]: pgmap v19735: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:12.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:26:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:26:12.645Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:26:13.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:26:12 smithi153 ceph-mon[70044]: pgmap v19735: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:26:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:26:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:26:14.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:26:14 smithi083 ceph-mon[79930]: pgmap v19736: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:14.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:26:14 smithi083 ceph-mon[77242]: pgmap v19736: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:14.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:26:14 smithi153 ceph-mon[70044]: pgmap v19736: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:15.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:26:15 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:26:15.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:26:15 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:26:15.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:26:15 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:26:15.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:26:15 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:26:15.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:26:15 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:26:15.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:26:15 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:26:15.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:26:15 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:26:15.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:26:15 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:26:15.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:26:15 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:26:15.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:26:15 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:26:15.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:26:15 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:26:15.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:26:15 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:26:16.428 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:26:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:26:15.981Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:26:16.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:26:16 smithi083 ceph-mon[79930]: pgmap v19737: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:16.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:26:16 smithi083 ceph-mon[77242]: pgmap v19737: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:16.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:26:16 smithi153 ceph-mon[70044]: pgmap v19737: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:17.210 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:26:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:26:16.734Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:26:19.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:26:18 smithi083 ceph-mon[79930]: pgmap v19738: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:19.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:26:18 smithi083 ceph-mon[77242]: pgmap v19738: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:19.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:26:18 smithi153 ceph-mon[70044]: pgmap v19738: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:21.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:26:20 smithi083 ceph-mon[79930]: pgmap v19739: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:21.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:26:20 smithi083 ceph-mon[77242]: pgmap v19739: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:21.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:26:20 smithi153 ceph-mon[70044]: pgmap v19739: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:21.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:26:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:26:21] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T16:26:22.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:26:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:26:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:26:22.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:26:22 smithi083 ceph-mon[79930]: pgmap v19740: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:22.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:26:22 smithi083 ceph-mon[77242]: pgmap v19740: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:22.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:26:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:26:22.646Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:26:23.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:26:22 smithi153 ceph-mon[70044]: pgmap v19740: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:26:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:26:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:26:24.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:26:24 smithi083 ceph-mon[79930]: pgmap v19741: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:24.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:26:24 smithi083 ceph-mon[77242]: pgmap v19741: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:24.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:26:24 smithi153 ceph-mon[70044]: pgmap v19741: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:26.460 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:26:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:26:25.982Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:26:27.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:26:26 smithi083 ceph-mon[79930]: pgmap v19742: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:27.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:26:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:26:26.735Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:26:27.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:26:26 smithi083 ceph-mon[77242]: pgmap v19742: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:27.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:26:26 smithi153 ceph-mon[70044]: pgmap v19742: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:29.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:26:28 smithi083 ceph-mon[77242]: pgmap v19743: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:29.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:26:28 smithi083 ceph-mon[79930]: pgmap v19743: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:29.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:26:28 smithi153 ceph-mon[70044]: pgmap v19743: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:31.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:26:30 smithi083 ceph-mon[77242]: pgmap v19744: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:31.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:26:30 smithi083 ceph-mon[79930]: pgmap v19744: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:31.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:26:30 smithi153 ceph-mon[70044]: pgmap v19744: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:31.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:26:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:26:31] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T16:26:32.224 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:26:32.512 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:26:32.513 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 5m ago 11h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:26:32.513 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 3m ago 11h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:26:32.513 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 5m ago 11h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:26:32.513 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 3m ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:26:32.514 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 3m ago 11h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:26:32.514 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 5m ago 11h 1069M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:26:32.514 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 5m ago 11h 2369M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:26:32.514 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 3m ago 11h 1060M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:26:32.515 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 5m ago 11h 1428M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:26:32.515 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 5m ago 11h 24.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:26:32.515 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 3m ago 11h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:26:32.515 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 5m ago 11h 1725M 2077M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:26:32.515 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 5m ago 11h 1711M 2077M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:26:32.516 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 5m ago 11h 1328M 2077M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:26:32.516 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 5m ago 11h 1677M 2077M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:26:32.516 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 3m ago 11h 1469M 2494M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:26:32.516 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 3m ago 11h 1847M 2494M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:26:32.517 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 3m ago 11h 1651M 2494M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:26:32.517 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 3m ago 11h 1965M 2494M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:26:32.517 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 3m ago 11h 190M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:26:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:26:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:26:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:26:32.873 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:26:32.873 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:26:32.873 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:26:32.873 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:26:32.874 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:26:32.874 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:26:32.874 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:26:32.874 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:26:32.875 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:26:32.875 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:26:32.875 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:26:32.875 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:26:32.875 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:26:32.876 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:26:32.876 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:26:32.876 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:26:32.876 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:26:32.877 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:26:32.960 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:26:32 smithi083 ceph-mon[77242]: pgmap v19745: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:32.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:26:32 smithi083 ceph-mon[77242]: from='client.57090 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:26:32.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:26:32 smithi083 ceph-mon[79930]: pgmap v19745: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:32.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:26:32 smithi083 ceph-mon[79930]: from='client.57090 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:26:32.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:26:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:26:32.647Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:26:33.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:26:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:26:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:26:33.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:26:32 smithi153 ceph-mon[70044]: pgmap v19745: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:33.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:26:32 smithi153 ceph-mon[70044]: from='client.57090 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:26:34.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:26:33 smithi083 ceph-mon[77242]: from='client.67027 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:26:34.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:26:33 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1645173440' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:26:34.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:26:33 smithi083 ceph-mon[79930]: from='client.67027 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:26:34.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:26:33 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1645173440' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:26:34.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:26:33 smithi153 ceph-mon[70044]: from='client.67027 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:26:34.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:26:33 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1645173440' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:26:35.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:26:34 smithi083 ceph-mon[79930]: pgmap v19746: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:35.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:26:34 smithi083 ceph-mon[77242]: pgmap v19746: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:35.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:26:34 smithi153 ceph-mon[70044]: pgmap v19746: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:36.460 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:26:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:26:35.983Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:26:37.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:26:36 smithi083 ceph-mon[79930]: pgmap v19747: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:37.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:26:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:26:36.736Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:26:37.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:26:36 smithi083 ceph-mon[77242]: pgmap v19747: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:37.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:26:36 smithi153 ceph-mon[70044]: pgmap v19747: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:39.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:26:38 smithi083 ceph-mon[79930]: pgmap v19748: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:39.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:26:38 smithi083 ceph-mon[77242]: pgmap v19748: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:39.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:26:38 smithi153 ceph-mon[70044]: pgmap v19748: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:41.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:26:40 smithi083 ceph-mon[79930]: pgmap v19749: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:41.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:26:40 smithi083 ceph-mon[77242]: pgmap v19749: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:41.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:26:40 smithi153 ceph-mon[70044]: pgmap v19749: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:41.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:26:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:26:41] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T16:26:42.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:26:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:26:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:26:42.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:26:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:26:42.647Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:26:42.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:26:42 smithi083 ceph-mon[77242]: pgmap v19750: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:42.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:26:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:26:42.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:26:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:26:42.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:26:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:26:42.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:26:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:26:42.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:26:42 smithi083 ceph-mon[79930]: pgmap v19750: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:42.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:26:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:26:42.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:26:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:26:42.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:26:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:26:42.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:26:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:26:43.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:26:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:26:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:26:43.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:26:42 smithi153 ceph-mon[70044]: pgmap v19750: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:43.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:26:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:26:43.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:26:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:26:43.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:26:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:26:43.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:26:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:26:44.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:26:44 smithi083 ceph-mon[77242]: pgmap v19751: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:44.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:26:44 smithi083 ceph-mon[79930]: pgmap v19751: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:44.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:26:44 smithi153 ceph-mon[70044]: pgmap v19751: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:46.460 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:26:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:26:45.984Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:26:47.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:26:46 smithi083 ceph-mon[79930]: pgmap v19752: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:47.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:26:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:26:46.737Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:26:47.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:26:46 smithi083 ceph-mon[77242]: pgmap v19752: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:47.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:26:46 smithi153 ceph-mon[70044]: pgmap v19752: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:49.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:26:48 smithi083 ceph-mon[79930]: pgmap v19753: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:49.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:26:48 smithi083 ceph-mon[77242]: pgmap v19753: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:49.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:26:48 smithi153 ceph-mon[70044]: pgmap v19753: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:51.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:26:50 smithi083 ceph-mon[79930]: pgmap v19754: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:51.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:26:50 smithi083 ceph-mon[77242]: pgmap v19754: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:51.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:26:50 smithi153 ceph-mon[70044]: pgmap v19754: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:51.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:26:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:26:51] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T16:26:52.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:26:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:26:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:26:52.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:26:52 smithi083 ceph-mon[79930]: pgmap v19755: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:52.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:26:52 smithi083 ceph-mon[77242]: pgmap v19755: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:52.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:26:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:26:52.647Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:26:53.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:26:52 smithi153 ceph-mon[70044]: pgmap v19755: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:26:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:26:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:26:54.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:26:54 smithi083 ceph-mon[77242]: pgmap v19756: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:54.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:26:54 smithi083 ceph-mon[79930]: pgmap v19756: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:54.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:26:54 smithi153 ceph-mon[70044]: pgmap v19756: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:56.460 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:26:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:26:55.984Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:26:57.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:26:56 smithi083 ceph-mon[79930]: pgmap v19757: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:57.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:26:56 smithi083 ceph-mon[77242]: pgmap v19757: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:57.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:26:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:26:56.737Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:26:57.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:26:56 smithi153 ceph-mon[70044]: pgmap v19757: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:59.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:26:58 smithi083 ceph-mon[79930]: pgmap v19758: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:59.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:26:58 smithi083 ceph-mon[77242]: pgmap v19758: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:26:59.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:26:58 smithi153 ceph-mon[70044]: pgmap v19758: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:01.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:27:00 smithi083 ceph-mon[79930]: pgmap v19759: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:01.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:27:00 smithi083 ceph-mon[77242]: pgmap v19759: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:01.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:27:00 smithi153 ceph-mon[70044]: pgmap v19759: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:01.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:27:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:27:01] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T16:27:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:27:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:27:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:27:02.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:27:02 smithi083 ceph-mon[79930]: pgmap v19760: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:02.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:27:02 smithi083 ceph-mon[77242]: pgmap v19760: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:02.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:27:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:27:02.648Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:27:03.198 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:27:03.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:27:02 smithi153 ceph-mon[70044]: pgmap v19760: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:27:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:27:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:27:03.502 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:27:03.502 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 6m ago 11h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:27:03.503 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 3m ago 11h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:27:03.503 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 6m ago 11h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:27:03.504 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 3m ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:27:03.504 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 3m ago 11h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:27:03.504 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 6m ago 11h 1069M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:27:03.505 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 6m ago 11h 2369M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:27:03.505 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 3m ago 11h 1060M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:27:03.505 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 6m ago 11h 1428M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:27:03.506 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 6m ago 11h 24.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:27:03.506 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 3m ago 11h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:27:03.507 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 6m ago 11h 1725M 2077M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:27:03.507 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 6m ago 11h 1711M 2077M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:27:03.507 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 6m ago 11h 1328M 2077M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:27:03.507 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 6m ago 11h 1677M 2077M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:27:03.508 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 3m ago 11h 1469M 2494M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:27:03.508 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 3m ago 11h 1847M 2494M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:27:03.508 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 3m ago 11h 1651M 2494M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:27:03.508 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 3m ago 11h 1965M 2494M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:27:03.509 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 3m ago 11h 190M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:27:03.869 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:27:03.869 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:27:03.870 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:27:03.870 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:27:03.870 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:27:03.870 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:27:03.871 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:27:03.871 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:27:03.871 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:27:03.871 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:27:03.872 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:27:03.872 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:27:03.872 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:27:03.872 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:27:03.872 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:27:03.873 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:27:03.873 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:27:03.873 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:27:04.210 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:27:03 smithi083 ceph-mon[77242]: from='client.57108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:27:04.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:27:03 smithi083 ceph-mon[79930]: from='client.57108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:27:04.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:27:03 smithi153 ceph-mon[70044]: from='client.57108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:27:05.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:27:04 smithi083 ceph-mon[79930]: from='client.67045 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:27:05.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:27:04 smithi083 ceph-mon[79930]: pgmap v19761: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:05.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:27:04 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1217327955' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:27:05.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:27:04 smithi083 ceph-mon[77242]: from='client.67045 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:27:05.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:27:04 smithi083 ceph-mon[77242]: pgmap v19761: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:05.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:27:04 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1217327955' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:27:05.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:27:04 smithi153 ceph-mon[70044]: from='client.67045 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:27:05.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:27:04 smithi153 ceph-mon[70044]: pgmap v19761: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:05.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:27:04 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1217327955' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:27:06.460 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:27:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:27:05.986Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:27:07.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:27:06 smithi083 ceph-mon[79930]: pgmap v19762: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:07.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:27:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:27:06.738Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:27:07.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:27:06 smithi083 ceph-mon[77242]: pgmap v19762: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:07.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:27:06 smithi153 ceph-mon[70044]: pgmap v19762: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:09.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:27:08 smithi083 ceph-mon[79930]: pgmap v19763: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:09.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:27:08 smithi083 ceph-mon[77242]: pgmap v19763: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:09.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:27:08 smithi153 ceph-mon[70044]: pgmap v19763: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:11.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:27:10 smithi083 ceph-mon[77242]: pgmap v19764: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:11.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:27:10 smithi083 ceph-mon[79930]: pgmap v19764: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:11.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:27:10 smithi153 ceph-mon[70044]: pgmap v19764: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:11.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:27:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:27:11] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T16:27:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:27:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:27:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:27:12.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:27:12 smithi083 ceph-mon[77242]: pgmap v19765: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:12.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:27:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:27:12.649Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:27:12.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:27:12 smithi083 ceph-mon[79930]: pgmap v19765: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:13.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:27:12 smithi153 ceph-mon[70044]: pgmap v19765: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:27:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:27:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:27:14.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:27:14 smithi083 ceph-mon[77242]: pgmap v19766: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:14.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:27:14 smithi083 ceph-mon[79930]: pgmap v19766: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:14.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:27:14 smithi153 ceph-mon[70044]: pgmap v19766: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:15.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:27:15 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:27:15.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:27:15 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:27:15.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:27:15 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:27:15.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:27:15 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:27:15.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:27:15 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:27:15.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:27:15 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:27:15.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:27:15 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:27:15.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:27:15 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:27:15.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:27:15 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:27:16.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:27:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:27:15.986Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:27:17.078 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:27:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:27:16.738Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:27:17.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:27:17 smithi083 ceph-mon[79930]: pgmap v19767: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:17.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:27:17 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:27:17.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:27:17 smithi083 ceph-mon[77242]: pgmap v19767: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:17.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:27:17 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:27:17.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:27:17 smithi153 ceph-mon[70044]: pgmap v19767: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:17.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:27:17 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:27:19.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:27:19 smithi083 ceph-mon[79930]: pgmap v19768: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:19.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:27:19 smithi083 ceph-mon[77242]: pgmap v19768: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:19.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:27:19 smithi153 ceph-mon[70044]: pgmap v19768: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:21.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:27:21 smithi083 ceph-mon[77242]: pgmap v19769: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:21.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:27:21 smithi083 ceph-mon[79930]: pgmap v19769: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:21.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:27:21 smithi153 ceph-mon[70044]: pgmap v19769: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:21.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:27:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:27:21] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T16:27:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:27:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:27:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:27:22.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:27:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:27:22.650Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:27:23.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:27:23 smithi153 ceph-mon[70044]: pgmap v19770: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:27:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:27:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:27:23.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:27:23 smithi083 ceph-mon[79930]: pgmap v19770: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:23.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:27:23 smithi083 ceph-mon[77242]: pgmap v19770: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:24.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:27:24 smithi083 ceph-mon[79930]: pgmap v19771: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:24.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:27:24 smithi083 ceph-mon[77242]: pgmap v19771: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:24.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:27:24 smithi153 ceph-mon[70044]: pgmap v19771: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:26.460 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:27:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:27:25.987Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:27:27.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:27:26 smithi083 ceph-mon[79930]: pgmap v19772: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:27.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:27:26 smithi083 ceph-mon[77242]: pgmap v19772: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:27.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:27:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:27:26.738Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:27:27.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:27:26 smithi153 ceph-mon[70044]: pgmap v19772: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:29.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:27:28 smithi083 ceph-mon[79930]: pgmap v19773: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:29.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:27:28 smithi083 ceph-mon[77242]: pgmap v19773: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:29.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:27:28 smithi153 ceph-mon[70044]: pgmap v19773: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:31.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:27:30 smithi083 ceph-mon[79930]: pgmap v19774: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:31.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:27:30 smithi083 ceph-mon[77242]: pgmap v19774: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:31.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:27:30 smithi153 ceph-mon[70044]: pgmap v19774: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:31.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:27:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:27:31] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T16:27:32.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:27:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:27:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:27:32.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:27:32 smithi083 ceph-mon[79930]: pgmap v19775: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:32.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:27:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:27:32.651Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:27:32.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:27:32 smithi083 ceph-mon[77242]: pgmap v19775: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:33.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:27:32 smithi153 ceph-mon[70044]: pgmap v19775: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:27:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:27:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:27:34.194 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:27:34.485 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:27:34.485 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 6m ago 11h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:27:34.486 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 4m ago 11h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:27:34.486 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 6m ago 11h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:27:34.486 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 4m ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:27:34.486 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 4m ago 11h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:27:34.487 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 6m ago 11h 1069M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:27:34.487 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 6m ago 11h 2369M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:27:34.487 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 4m ago 11h 1060M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:27:34.487 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 6m ago 11h 1428M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:27:34.488 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 6m ago 11h 24.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:27:34.488 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 4m ago 11h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:27:34.488 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 6m ago 11h 1725M 2077M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:27:34.488 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 6m ago 11h 1711M 2077M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:27:34.489 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 6m ago 11h 1328M 2077M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:27:34.489 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 6m ago 11h 1677M 2077M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:27:34.489 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 4m ago 11h 1469M 2494M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:27:34.489 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 4m ago 11h 1847M 2494M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:27:34.489 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 4m ago 11h 1651M 2494M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:27:34.490 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 4m ago 11h 1965M 2494M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:27:34.490 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 4m ago 11h 190M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:27:34.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:27:34 smithi083 ceph-mon[77242]: pgmap v19776: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:34.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:27:34 smithi083 ceph-mon[77242]: from='client.57126 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:27:34.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:27:34 smithi083 ceph-mon[79930]: pgmap v19776: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:34.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:27:34 smithi083 ceph-mon[79930]: from='client.57126 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:27:34.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:27:34 smithi153 ceph-mon[70044]: pgmap v19776: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:34.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:27:34 smithi153 ceph-mon[70044]: from='client.57126 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:27:34.839 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:27:34.839 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:27:34.839 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:27:34.840 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:27:34.840 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:27:34.840 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:27:34.841 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:27:34.841 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:27:34.841 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:27:34.841 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:27:34.842 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:27:34.842 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:27:34.842 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:27:34.842 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:27:34.842 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:27:34.843 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:27:34.843 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:27:34.843 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:27:35.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:27:35 smithi083 ceph-mon[79930]: from='client.57132 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:27:35.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:27:35 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/951717478' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:27:35.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:27:35 smithi083 ceph-mon[77242]: from='client.57132 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:27:35.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:27:35 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/951717478' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:27:35.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:27:35 smithi153 ceph-mon[70044]: from='client.57132 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:27:35.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:27:35 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/951717478' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:27:36.442 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:27:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:27:35.988Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:27:36.442 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:27:36 smithi083 ceph-mon[77242]: pgmap v19777: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:36.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:27:36 smithi083 ceph-mon[79930]: pgmap v19777: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:36.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:27:36 smithi153 ceph-mon[70044]: pgmap v19777: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:37.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:27:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:27:36.739Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:27:39.210 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:27:38 smithi083 ceph-mon[77242]: pgmap v19778: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:39.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:27:38 smithi083 ceph-mon[79930]: pgmap v19778: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:39.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:27:38 smithi153 ceph-mon[70044]: pgmap v19778: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:41.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:27:40 smithi083 ceph-mon[79930]: pgmap v19779: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:41.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:27:40 smithi083 ceph-mon[77242]: pgmap v19779: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:41.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:27:40 smithi153 ceph-mon[70044]: pgmap v19779: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:41.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:27:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:27:41] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T16:27:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:27:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:27:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:27:42.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:27:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:27:42.651Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:27:42.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:27:42 smithi083 ceph-mon[79930]: pgmap v19780: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:42.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:27:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:27:42.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:27:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:27:42.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:27:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:27:42.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:27:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:27:42.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:27:42 smithi083 ceph-mon[77242]: pgmap v19780: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:42.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:27:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:27:42.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:27:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:27:42.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:27:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:27:42.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:27:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:27:43.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:27:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:27:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:27:43.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:27:42 smithi153 ceph-mon[70044]: pgmap v19780: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:43.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:27:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:27:43.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:27:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:27:43.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:27:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:27:43.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:27:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:27:44.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:27:44 smithi083 ceph-mon[79930]: pgmap v19781: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:44.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:27:44 smithi083 ceph-mon[77242]: pgmap v19781: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:44.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:27:44 smithi153 ceph-mon[70044]: pgmap v19781: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:46.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:27:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:27:45.989Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:27:47.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:27:46 smithi083 ceph-mon[79930]: pgmap v19782: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:47.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:27:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:27:46.739Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:27:47.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:27:46 smithi083 ceph-mon[77242]: pgmap v19782: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:47.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:27:46 smithi153 ceph-mon[70044]: pgmap v19782: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:49.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:27:48 smithi083 ceph-mon[79930]: pgmap v19783: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:49.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:27:48 smithi083 ceph-mon[77242]: pgmap v19783: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:49.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:27:48 smithi153 ceph-mon[70044]: pgmap v19783: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:51.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:27:50 smithi083 ceph-mon[79930]: pgmap v19784: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:51.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:27:50 smithi083 ceph-mon[77242]: pgmap v19784: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:51.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:27:50 smithi153 ceph-mon[70044]: pgmap v19784: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:51.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:27:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:27:51] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T16:27:52.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:27:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:27:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:27:52.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:27:52 smithi083 ceph-mon[79930]: pgmap v19785: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:52.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:27:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:27:52.652Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:27:52.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:27:52 smithi083 ceph-mon[77242]: pgmap v19785: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:53.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:27:52 smithi153 ceph-mon[70044]: pgmap v19785: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:27:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:27:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:27:54.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:27:54 smithi083 ceph-mon[79930]: pgmap v19786: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:54.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:27:54 smithi083 ceph-mon[77242]: pgmap v19786: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:54.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:27:54 smithi153 ceph-mon[70044]: pgmap v19786: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:56.460 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:27:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:27:55.989Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:27:57.210 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:27:56 smithi083 ceph-mon[77242]: pgmap v19787: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:57.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:27:56 smithi083 ceph-mon[79930]: pgmap v19787: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:57.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:27:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:27:56.740Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:27:57.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:27:56 smithi153 ceph-mon[70044]: pgmap v19787: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:59.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:27:58 smithi083 ceph-mon[79930]: pgmap v19788: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:59.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:27:58 smithi083 ceph-mon[77242]: pgmap v19788: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:27:59.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:27:58 smithi153 ceph-mon[70044]: pgmap v19788: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:01.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:28:00 smithi083 ceph-mon[79930]: pgmap v19789: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:01.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:28:00 smithi083 ceph-mon[77242]: pgmap v19789: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:01.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:28:00 smithi153 ceph-mon[70044]: pgmap v19789: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:01.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:28:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:28:01] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T16:28:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:28:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:28:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:28:02.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:28:02 smithi083 ceph-mon[79930]: pgmap v19790: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:02.965 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:28:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:28:02.653Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:28:02.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:28:02 smithi083 ceph-mon[77242]: pgmap v19790: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:03.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:28:02 smithi153 ceph-mon[70044]: pgmap v19790: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:28:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:28:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:28:04.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:28:04 smithi083 ceph-mon[79930]: pgmap v19791: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:04.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:28:04 smithi083 ceph-mon[77242]: pgmap v19791: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:04.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:28:04 smithi153 ceph-mon[70044]: pgmap v19791: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:05.165 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:28:05.453 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:28:05.453 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 7m ago 11h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:28:05.453 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 4m ago 11h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:28:05.453 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 7m ago 11h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:28:05.454 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 4m ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:28:05.454 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 4m ago 11h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:28:05.455 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 7m ago 11h 1069M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:28:05.455 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 7m ago 11h 2369M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:28:05.455 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 4m ago 11h 1060M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:28:05.455 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 7m ago 11h 1428M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:28:05.455 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 7m ago 11h 24.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:28:05.456 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 4m ago 11h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:28:05.456 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 7m ago 11h 1725M 2077M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:28:05.456 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 7m ago 11h 1711M 2077M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:28:05.457 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 7m ago 11h 1328M 2077M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:28:05.457 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 7m ago 11h 1677M 2077M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:28:05.458 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 4m ago 11h 1469M 2494M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:28:05.458 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 4m ago 11h 1847M 2494M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:28:05.458 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 4m ago 11h 1651M 2494M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:28:05.458 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 4m ago 11h 1965M 2494M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:28:05.458 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 4m ago 11h 190M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:28:05.710 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:28:05 smithi083 ceph-mon[77242]: from='client.57144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:28:05.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:28:05 smithi083 ceph-mon[79930]: from='client.57144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:28:05.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:28:05 smithi153 ceph-mon[70044]: from='client.57144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:28:05.808 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:28:05.808 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:28:05.808 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:28:05.809 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:28:05.809 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:28:05.809 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:28:05.809 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:28:05.810 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:28:05.810 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:28:05.810 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:28:05.810 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:28:05.811 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:28:05.811 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:28:05.811 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:28:05.812 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:28:05.812 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:28:05.812 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:28:05.812 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:28:06.448 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:28:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:28:05.989Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:28:06.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:28:06 smithi083 ceph-mon[79930]: from='client.57150 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:28:06.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:28:06 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2240184740' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:28:06.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:28:06 smithi083 ceph-mon[79930]: pgmap v19792: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:06.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:28:06 smithi083 ceph-mon[77242]: from='client.57150 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:28:06.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:28:06 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2240184740' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:28:06.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:28:06 smithi083 ceph-mon[77242]: pgmap v19792: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:06.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:28:06 smithi153 ceph-mon[70044]: from='client.57150 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:28:06.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:28:06 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2240184740' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:28:06.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:28:06 smithi153 ceph-mon[70044]: pgmap v19792: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:07.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:28:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:28:06.740Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:28:09.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:28:08 smithi083 ceph-mon[77242]: pgmap v19793: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:09.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:28:08 smithi083 ceph-mon[79930]: pgmap v19793: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:09.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:28:08 smithi153 ceph-mon[70044]: pgmap v19793: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:11.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:28:10 smithi083 ceph-mon[77242]: pgmap v19794: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:11.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:28:10 smithi083 ceph-mon[79930]: pgmap v19794: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:11.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:28:10 smithi153 ceph-mon[70044]: pgmap v19794: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:28:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:28:11] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T16:28:12.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:28:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:28:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:28:12.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:28:12 smithi083 ceph-mon[79930]: pgmap v19795: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:12.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:28:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:28:12.654Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:28:12.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:28:12 smithi083 ceph-mon[77242]: pgmap v19795: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:13.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:28:12 smithi153 ceph-mon[70044]: pgmap v19795: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:28:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:28:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:28:14.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:28:14 smithi083 ceph-mon[77242]: pgmap v19796: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:14.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:28:14 smithi083 ceph-mon[79930]: pgmap v19796: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:14.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:28:14 smithi153 ceph-mon[70044]: pgmap v19796: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:16.460 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:28:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:28:15.991Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:28:17.219 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:28:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:28:16.743Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:28:17.220 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:28:16 smithi083 ceph-mon[79930]: pgmap v19797: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:17.220 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:28:16 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:28:17.220 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:28:16 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:28:17.221 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:28:16 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:28:17.221 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:28:16 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:28:17.221 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:28:16 smithi083 ceph-mon[77242]: pgmap v19797: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:17.221 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:28:16 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:28:17.222 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:28:16 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:28:17.222 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:28:16 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:28:17.222 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:28:16 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:28:17.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:28:16 smithi153 ceph-mon[70044]: pgmap v19797: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:17.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:28:16 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:28:17.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:28:16 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:28:17.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:28:16 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:28:17.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:28:16 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:28:19.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:28:18 smithi083 ceph-mon[79930]: pgmap v19798: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:19.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:28:18 smithi083 ceph-mon[77242]: pgmap v19798: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:19.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:28:18 smithi153 ceph-mon[70044]: pgmap v19798: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:21.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:28:20 smithi083 ceph-mon[79930]: pgmap v19799: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:21.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:28:20 smithi083 ceph-mon[77242]: pgmap v19799: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:21.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:28:20 smithi153 ceph-mon[70044]: pgmap v19799: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:28:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:28:21] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T16:28:22.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:28:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:28:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:28:22.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:28:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:28:22.654Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:28:22.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:28:22 smithi083 ceph-mon[77242]: pgmap v19800: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:22.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:28:22 smithi083 ceph-mon[79930]: pgmap v19800: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:23.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:28:22 smithi153 ceph-mon[70044]: pgmap v19800: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:28:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:28:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:28:24.726 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:28:24 smithi083 ceph-mon[77242]: pgmap v19801: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:24.727 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:28:24 smithi083 ceph-mon[79930]: pgmap v19801: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:24.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:28:24 smithi153 ceph-mon[70044]: pgmap v19801: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:26.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:28:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:28:25.992Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:28:27.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:28:26 smithi083 ceph-mon[79930]: pgmap v19802: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:27.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:28:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:28:26.742Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:28:27.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:28:26 smithi083 ceph-mon[77242]: pgmap v19802: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:27.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:28:26 smithi153 ceph-mon[70044]: pgmap v19802: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:29.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:28:28 smithi083 ceph-mon[77242]: pgmap v19803: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:29.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:28:28 smithi083 ceph-mon[79930]: pgmap v19803: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:29.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:28:28 smithi153 ceph-mon[70044]: pgmap v19803: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:31.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:28:30 smithi083 ceph-mon[77242]: pgmap v19804: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:31.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:28:30 smithi083 ceph-mon[79930]: pgmap v19804: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:31.317 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:28:30 smithi153 ceph-mon[70044]: pgmap v19804: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:28:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:28:31] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T16:28:32.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:28:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:28:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:28:32.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:28:32 smithi083 ceph-mon[77242]: pgmap v19805: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:32.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:28:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:28:32.655Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:28:32.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:28:32 smithi083 ceph-mon[79930]: pgmap v19805: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:33.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:28:32 smithi153 ceph-mon[70044]: pgmap v19805: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:28:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:28:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:28:34.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:28:34 smithi083 ceph-mon[79930]: pgmap v19806: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:34.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:28:34 smithi083 ceph-mon[77242]: pgmap v19806: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:34.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:28:34 smithi153 ceph-mon[70044]: pgmap v19806: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:36.143 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:28:36.429 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:28:36.429 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 7m ago 11h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:28:36.430 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 5m ago 11h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:28:36.430 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 7m ago 11h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:28:36.432 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 5m ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:28:36.432 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 5m ago 11h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:28:36.433 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (10h) 7m ago 11h 1069M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:28:36.433 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 7m ago 11h 2369M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:28:36.434 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 5m ago 11h 1060M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:28:36.435 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 7m ago 11h 1428M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:28:36.435 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 7m ago 11h 24.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:28:36.436 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 5m ago 11h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:28:36.436 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 7m ago 11h 1725M 2077M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:28:36.437 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 7m ago 11h 1711M 2077M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:28:36.437 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 7m ago 11h 1328M 2077M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:28:36.437 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 7m ago 11h 1677M 2077M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:28:36.438 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 5m ago 11h 1469M 2494M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:28:36.438 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 5m ago 11h 1847M 2494M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:28:36.438 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 5m ago 11h 1651M 2494M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:28:36.438 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 5m ago 11h 1965M 2494M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:28:36.439 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 5m ago 11h 190M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:28:36.460 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:28:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:28:35.991Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:28:36.784 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:28:36.785 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:28:36.785 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:28:36.786 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:28:36.786 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:28:36.786 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:28:36.787 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:28:36.787 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:28:36.787 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:28:36.787 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:28:36.788 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:28:36.788 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:28:36.788 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:28:36.789 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:28:36.789 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:28:36.790 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:28:36.790 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:28:36.791 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:28:37.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:28:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:28:36.742Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:28:37.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:28:36 smithi083 ceph-mon[77242]: pgmap v19807: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:37.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:28:36 smithi083 ceph-mon[77242]: from='client.57162 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:28:37.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:28:36 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1188294930' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:28:37.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:28:36 smithi083 ceph-mon[79930]: pgmap v19807: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:37.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:28:36 smithi083 ceph-mon[79930]: from='client.57162 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:28:37.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:28:36 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1188294930' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:28:37.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:28:36 smithi153 ceph-mon[70044]: pgmap v19807: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:37.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:28:36 smithi153 ceph-mon[70044]: from='client.57162 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:28:37.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:28:36 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1188294930' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:28:38.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:28:37 smithi083 ceph-mon[77242]: from='client.57168 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:28:38.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:28:37 smithi083 ceph-mon[79930]: from='client.57168 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:28:38.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:28:37 smithi153 ceph-mon[70044]: from='client.57168 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:28:39.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:28:38 smithi083 ceph-mon[79930]: pgmap v19808: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:39.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:28:38 smithi083 ceph-mon[77242]: pgmap v19808: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:39.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:28:38 smithi153 ceph-mon[70044]: pgmap v19808: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:41.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:28:40 smithi083 ceph-mon[77242]: pgmap v19809: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:41.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:28:40 smithi083 ceph-mon[79930]: pgmap v19809: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:41.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:28:40 smithi153 ceph-mon[70044]: pgmap v19809: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:41.898 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:28:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:28:41] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T16:28:41.899 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:28:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:28:41.899 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:28:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:28:41.899 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:28:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:28:41.900 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:28:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:28:42.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:28:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:28:42.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:28:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:28:42.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:28:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:28:42.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:28:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:28:42.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:28:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:28:42.314 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:28:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:28:42.314 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:28:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:28:42.314 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:28:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:28:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:28:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:28:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:28:42.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:28:42 smithi083 ceph-mon[79930]: pgmap v19810: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:42.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:28:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:28:42.656Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:28:42.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:28:42 smithi083 ceph-mon[77242]: pgmap v19810: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:43.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:28:42 smithi153 ceph-mon[70044]: pgmap v19810: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:28:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:28:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:28:44.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:28:44 smithi083 ceph-mon[77242]: pgmap v19811: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:44.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:28:44 smithi083 ceph-mon[79930]: pgmap v19811: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:44.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:28:44 smithi153 ceph-mon[70044]: pgmap v19811: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:46.466 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:28:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:28:45.993Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:28:47.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:28:46 smithi083 ceph-mon[79930]: pgmap v19812: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:47.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:28:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:28:46.743Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:28:47.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:28:46 smithi083 ceph-mon[77242]: pgmap v19812: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:47.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:28:46 smithi153 ceph-mon[70044]: pgmap v19812: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:49.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:28:48 smithi083 ceph-mon[79930]: pgmap v19813: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:49.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:28:48 smithi083 ceph-mon[77242]: pgmap v19813: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:49.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:28:48 smithi153 ceph-mon[70044]: pgmap v19813: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:51.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:28:50 smithi083 ceph-mon[79930]: pgmap v19814: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:51.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:28:50 smithi083 ceph-mon[77242]: pgmap v19814: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:51.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:28:50 smithi153 ceph-mon[70044]: pgmap v19814: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:51.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:28:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:28:51] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T16:28:52.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:28:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:28:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:28:52.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:28:52 smithi083 ceph-mon[77242]: pgmap v19815: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:52.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:28:52 smithi083 ceph-mon[79930]: pgmap v19815: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:52.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:28:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:28:52.657Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:28:53.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:28:52 smithi153 ceph-mon[70044]: pgmap v19815: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:28:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:28:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:28:54.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:28:54 smithi083 ceph-mon[79930]: pgmap v19816: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:54.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:28:54 smithi083 ceph-mon[77242]: pgmap v19816: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:54.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:28:54 smithi153 ceph-mon[70044]: pgmap v19816: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:56.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:28:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:28:55.994Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:28:57.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:28:56 smithi083 ceph-mon[79930]: pgmap v19817: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:57.215 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:28:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:28:56.744Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:28:57.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:28:56 smithi083 ceph-mon[77242]: pgmap v19817: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:57.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:28:56 smithi153 ceph-mon[70044]: pgmap v19817: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:59.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:28:58 smithi083 ceph-mon[79930]: pgmap v19818: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:59.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:28:58 smithi083 ceph-mon[77242]: pgmap v19818: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:28:59.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:28:58 smithi153 ceph-mon[70044]: pgmap v19818: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:01.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:29:00 smithi083 ceph-mon[79930]: pgmap v19819: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:01.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:29:00 smithi083 ceph-mon[77242]: pgmap v19819: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:01.314 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:29:00 smithi153 ceph-mon[70044]: pgmap v19819: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:29:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:29:01] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T16:29:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:29:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:29:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:29:02.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:29:02 smithi083 ceph-mon[79930]: pgmap v19820: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:02.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:29:02 smithi083 ceph-mon[77242]: pgmap v19820: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:02.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:29:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:29:02.657Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:29:03.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:29:02 smithi153 ceph-mon[70044]: pgmap v19820: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:29:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:29:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:29:04.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:29:04 smithi153 ceph-mon[70044]: pgmap v19821: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:04.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:29:04 smithi083 ceph-mon[79930]: pgmap v19821: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:04.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:29:04 smithi083 ceph-mon[77242]: pgmap v19821: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:06.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:29:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:29:05.995Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:29:07.113 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:29:07.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:29:06 smithi083 ceph-mon[79930]: pgmap v19822: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:07.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:29:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:29:06.745Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:29:07.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:29:06 smithi083 ceph-mon[77242]: pgmap v19822: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:07.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:29:06 smithi153 ceph-mon[70044]: pgmap v19822: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:07.403 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:29:07.404 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 8m ago 11h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:29:07.404 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 5m ago 11h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:29:07.404 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 8m ago 11h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:29:07.405 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 5m ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:29:07.405 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (10h) 5m ago 11h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:29:07.406 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11h) 8m ago 11h 1069M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:29:07.406 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10h) 8m ago 11h 2369M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:29:07.407 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 5m ago 11h 1060M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:29:07.407 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 8m ago 11h 1428M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:29:07.408 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 8m ago 11h 24.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:29:07.408 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 5m ago 11h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:29:07.409 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 8m ago 11h 1725M 2077M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:29:07.409 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 8m ago 11h 1711M 2077M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:29:07.410 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 8m ago 11h 1328M 2077M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:29:07.411 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 8m ago 11h 1677M 2077M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:29:07.411 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 5m ago 11h 1469M 2494M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:29:07.412 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 5m ago 11h 1847M 2494M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:29:07.412 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 5m ago 11h 1651M 2494M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:29:07.413 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 5m ago 11h 1965M 2494M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:29:07.413 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (10h) 5m ago 11h 190M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:29:07.757 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:29:07.758 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:29:07.759 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:29:07.759 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:29:07.760 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:29:07.760 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:29:07.761 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:29:07.761 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:29:07.761 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:29:07.762 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:29:07.762 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:29:07.763 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:29:07.763 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:29:07.764 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:29:07.764 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:29:07.765 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:29:07.765 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:29:07.766 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:29:08.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:29:07 smithi083 ceph-mon[79930]: from='client.57180 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:29:08.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:29:07 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/85498257' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:29:08.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:29:07 smithi083 ceph-mon[77242]: from='client.57180 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:29:08.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:29:07 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/85498257' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:29:08.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:29:07 smithi153 ceph-mon[70044]: from='client.57180 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:29:08.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:29:07 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/85498257' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:29:09.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:29:08 smithi083 ceph-mon[79930]: from='client.57186 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:29:09.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:29:08 smithi083 ceph-mon[79930]: pgmap v19823: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:09.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:29:08 smithi083 ceph-mon[77242]: from='client.57186 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:29:09.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:29:08 smithi083 ceph-mon[77242]: pgmap v19823: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:09.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:29:08 smithi153 ceph-mon[70044]: from='client.57186 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:29:09.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:29:08 smithi153 ceph-mon[70044]: pgmap v19823: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:11.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:29:10 smithi083 ceph-mon[79930]: pgmap v19824: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:11.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:29:10 smithi083 ceph-mon[77242]: pgmap v19824: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:11.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:29:10 smithi153 ceph-mon[70044]: pgmap v19824: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:29:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:29:11] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T16:29:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:29:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:29:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:29:12.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:29:12 smithi083 ceph-mon[79930]: pgmap v19825: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:12.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:29:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:29:12.658Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:29:12.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:29:12 smithi083 ceph-mon[77242]: pgmap v19825: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:13.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:29:12 smithi153 ceph-mon[70044]: pgmap v19825: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:29:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:29:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:29:14.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:29:14 smithi083 ceph-mon[77242]: pgmap v19826: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:14.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:29:14 smithi083 ceph-mon[79930]: pgmap v19826: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:14.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:29:14 smithi153 ceph-mon[70044]: pgmap v19826: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:16.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:29:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:29:15.995Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:29:17.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:29:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:29:16.745Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:29:17.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:29:16 smithi083 ceph-mon[77242]: pgmap v19827: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:17.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:29:16 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:29:17.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:29:16 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:29:17.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:29:16 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:29:17.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:29:16 smithi083 ceph-mon[79930]: pgmap v19827: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:17.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:29:16 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:29:17.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:29:16 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:29:17.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:29:16 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:29:17.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:29:16 smithi153 ceph-mon[70044]: pgmap v19827: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:17.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:29:16 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:29:17.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:29:16 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:29:17.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:29:16 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:29:18.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:29:17 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:29:18.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:29:17 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:29:18.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:29:17 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:29:18.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:29:17 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:29:18.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:29:17 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:29:18.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:29:17 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:29:18.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:29:17 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:29:18.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:29:17 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:29:18.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:29:17 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:29:18.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:29:17 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:29:18.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:29:17 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:29:18.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:29:17 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:29:18.218 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:29:17 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:29:18.218 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:29:17 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:29:18.218 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:29:17 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:29:18.218 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:29:17 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:29:18.219 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:29:17 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:29:18.219 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:29:17 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:29:18.219 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:29:17 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:29:18.220 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:29:17 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:29:18.220 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:29:17 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:29:18.220 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:29:17 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:29:18.220 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:29:17 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:29:18.222 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:29:17 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:29:18.222 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:29:17 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:29:18.223 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:29:17 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:29:18.223 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:29:17 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:29:18.223 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:29:17 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:29:18.223 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:29:17 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:29:18.224 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:29:17 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:29:18.224 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:29:17 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:29:18.224 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:29:17 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:29:18.224 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:29:17 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:29:18.225 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:29:17 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:29:18.225 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:29:17 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:29:18.225 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:29:17 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:29:18.225 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:29:17 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:29:18.225 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:29:17 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:29:18.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:29:17 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:29:18.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:29:17 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:29:18.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:29:17 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:29:18.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:29:17 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:29:18.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:29:17 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:29:18.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:29:17 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:29:18.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:29:17 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:29:18.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:29:17 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:29:18.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:29:17 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:29:18.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:29:17 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:29:18.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:29:17 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:29:18.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:29:17 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:29:18.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:29:17 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:29:18.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:29:17 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:29:18.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:29:17 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:29:18.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:29:17 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:29:18.312 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:29:17 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:29:18.312 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:29:17 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:29:18.313 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:29:17 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:29:19.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:29:18 smithi083 ceph-mon[77242]: Adjusting osd_memory_target on smithi083 to 2062M 2022-09-29T16:29:19.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:29:18 smithi083 ceph-mon[77242]: Adjusting osd_memory_target on smithi153 to 2491M 2022-09-29T16:29:19.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:29:18 smithi083 ceph-mon[77242]: pgmap v19828: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:19.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:29:18 smithi083 ceph-mon[79930]: Adjusting osd_memory_target on smithi083 to 2062M 2022-09-29T16:29:19.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:29:18 smithi083 ceph-mon[79930]: Adjusting osd_memory_target on smithi153 to 2491M 2022-09-29T16:29:19.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:29:18 smithi083 ceph-mon[79930]: pgmap v19828: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:19.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:29:18 smithi153 ceph-mon[70044]: Adjusting osd_memory_target on smithi083 to 2062M 2022-09-29T16:29:19.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:29:18 smithi153 ceph-mon[70044]: Adjusting osd_memory_target on smithi153 to 2491M 2022-09-29T16:29:19.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:29:18 smithi153 ceph-mon[70044]: pgmap v19828: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:21.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:29:20 smithi083 ceph-mon[79930]: pgmap v19829: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:21.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:29:20 smithi083 ceph-mon[77242]: pgmap v19829: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:21.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:29:20 smithi153 ceph-mon[70044]: pgmap v19829: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:21.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:29:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:29:21] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T16:29:22.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:29:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:29:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:29:22.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:29:22 smithi083 ceph-mon[79930]: pgmap v19830: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:22.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:29:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:29:22.659Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:29:22.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:29:22 smithi083 ceph-mon[77242]: pgmap v19830: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:23.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:29:22 smithi153 ceph-mon[70044]: pgmap v19830: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:29:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:29:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:29:24.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:29:24 smithi083 ceph-mon[79930]: pgmap v19831: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:24.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:29:24 smithi083 ceph-mon[77242]: pgmap v19831: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:24.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:29:24 smithi153 ceph-mon[70044]: pgmap v19831: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:26.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:29:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:29:25.996Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:29:27.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:29:26 smithi083 ceph-mon[79930]: pgmap v19832: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:27.249 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:29:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:29:26.746Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:29:27.250 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:29:26 smithi083 ceph-mon[77242]: pgmap v19832: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:27.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:29:26 smithi153 ceph-mon[70044]: pgmap v19832: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:29.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:29:28 smithi083 ceph-mon[79930]: pgmap v19833: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:29.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:29:28 smithi083 ceph-mon[77242]: pgmap v19833: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:29.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:29:28 smithi153 ceph-mon[70044]: pgmap v19833: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:31.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:29:30 smithi083 ceph-mon[79930]: pgmap v19834: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:31.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:29:30 smithi083 ceph-mon[77242]: pgmap v19834: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:31.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:29:30 smithi153 ceph-mon[70044]: pgmap v19834: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:31.964 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:29:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:29:31] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T16:29:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:29:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:29:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:29:32.910 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:29:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:29:32.659Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:29:33.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:29:32 smithi083 ceph-mon[77242]: pgmap v19835: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:33.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:29:32 smithi083 ceph-mon[79930]: pgmap v19835: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:33.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:29:32 smithi153 ceph-mon[70044]: pgmap v19835: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:29:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:29:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:29:34.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:29:34 smithi153 ceph-mon[70044]: pgmap v19836: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:34.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:29:34 smithi083 ceph-mon[79930]: pgmap v19836: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:34.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:29:34 smithi083 ceph-mon[77242]: pgmap v19836: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:36.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:29:35 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:29:35.996Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:29:37.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:29:36 smithi083 ceph-mon[77242]: pgmap v19837: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:37.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:29:36 smithi083 ceph-mon[79930]: pgmap v19837: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:37.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:29:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:29:36.746Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:29:37.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:29:36 smithi153 ceph-mon[70044]: pgmap v19837: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:38.085 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:29:38.372 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:29:38.372 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 8m ago 11h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:29:38.373 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 6m ago 11h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:29:38.377 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 8m ago 11h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:29:38.377 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 6m ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:29:38.378 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (11h) 6m ago 11h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:29:38.379 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11h) 8m ago 11h 1069M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:29:38.379 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11h) 8m ago 11h 2369M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:29:38.379 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (10h) 6m ago 11h 1060M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:29:38.380 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (10h) 8m ago 11h 1428M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:29:38.381 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 8m ago 11h 24.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:29:38.381 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 6m ago 11h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:29:38.381 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 8m ago 11h 1725M 2062M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:29:38.381 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 8m ago 11h 1711M 2062M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:29:38.382 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 8m ago 11h 1328M 2062M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:29:38.382 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 8m ago 11h 1677M 2062M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:29:38.382 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 6m ago 11h 1469M 2491M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:29:38.382 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 6m ago 11h 1847M 2491M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:29:38.382 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 6m ago 11h 1651M 2491M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:29:38.383 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 6m ago 11h 1965M 2491M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:29:38.383 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (11h) 6m ago 11h 190M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:29:38.734 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:29:38.735 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:29:38.735 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:29:38.735 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:29:38.735 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:29:38.736 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:29:38.736 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:29:38.736 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:29:38.736 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:29:38.737 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:29:38.737 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:29:38.737 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:29:38.737 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:29:38.738 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:29:38.738 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:29:38.738 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:29:38.738 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:29:38.739 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:29:39.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:29:38 smithi083 ceph-mon[79930]: pgmap v19838: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:39.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:29:38 smithi083 ceph-mon[79930]: from='client.67129 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:29:39.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:29:38 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3932803339' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:29:39.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:29:38 smithi083 ceph-mon[77242]: pgmap v19838: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:39.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:29:38 smithi083 ceph-mon[77242]: from='client.67129 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:29:39.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:29:38 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3932803339' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:29:39.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:29:38 smithi153 ceph-mon[70044]: pgmap v19838: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:39.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:29:38 smithi153 ceph-mon[70044]: from='client.67129 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:29:39.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:29:38 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3932803339' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:29:40.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:29:39 smithi083 ceph-mon[79930]: from='client.57204 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:29:40.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:29:39 smithi083 ceph-mon[77242]: from='client.57204 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:29:40.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:29:39 smithi153 ceph-mon[70044]: from='client.57204 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:29:41.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:29:40 smithi083 ceph-mon[79930]: pgmap v19839: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:41.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:29:40 smithi083 ceph-mon[77242]: pgmap v19839: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:29:40 smithi153 ceph-mon[70044]: pgmap v19839: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:41.914 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:29:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:29:41] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T16:29:42.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:29:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:29:42.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:29:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:29:42.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:29:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:29:42.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:29:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:29:42.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:29:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:29:42.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:29:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:29:42.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:29:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:29:42.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:29:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:29:42.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:29:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:29:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:29:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:29:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:29:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:29:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:29:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:29:42.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:29:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:29:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:29:42.918 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:29:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:29:42.660Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:29:43.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:29:42 smithi083 ceph-mon[79930]: pgmap v19840: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:43.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:29:42 smithi083 ceph-mon[77242]: pgmap v19840: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:43.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:29:42 smithi153 ceph-mon[70044]: pgmap v19840: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:43.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:29:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:29:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:29:44.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:29:44 smithi153 ceph-mon[70044]: pgmap v19841: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:44.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:29:44 smithi083 ceph-mon[77242]: pgmap v19841: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:44.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:29:44 smithi083 ceph-mon[79930]: pgmap v19841: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:46.465 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:29:45 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:29:45.997Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:29:47.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:29:46 smithi083 ceph-mon[77242]: pgmap v19842: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:47.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:29:46 smithi083 ceph-mon[79930]: pgmap v19842: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:47.217 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:29:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:29:46.747Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:29:47.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:29:46 smithi153 ceph-mon[70044]: pgmap v19842: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:49.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:29:48 smithi083 ceph-mon[79930]: pgmap v19843: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:49.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:29:48 smithi083 ceph-mon[77242]: pgmap v19843: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:49.314 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:29:48 smithi153 ceph-mon[70044]: pgmap v19843: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:51.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:29:50 smithi083 ceph-mon[79930]: pgmap v19844: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:51.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:29:50 smithi083 ceph-mon[77242]: pgmap v19844: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:51.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:29:50 smithi153 ceph-mon[70044]: pgmap v19844: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:51.969 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:29:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:29:51] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T16:29:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:29:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:29:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:29:52.915 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:29:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:29:52.660Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:29:53.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:29:52 smithi083 ceph-mon[79930]: pgmap v19845: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:53.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:29:52 smithi083 ceph-mon[77242]: pgmap v19845: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:53.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:29:52 smithi153 ceph-mon[70044]: pgmap v19845: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:29:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:29:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:29:54.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:29:54 smithi153 ceph-mon[70044]: pgmap v19846: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:54.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:29:54 smithi083 ceph-mon[77242]: pgmap v19846: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:54.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:29:54 smithi083 ceph-mon[79930]: pgmap v19846: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:56.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:29:55 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:29:55.998Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:29:57.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:29:56 smithi083 ceph-mon[79930]: pgmap v19847: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:57.216 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:29:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:29:56.748Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:29:57.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:29:56 smithi083 ceph-mon[77242]: pgmap v19847: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:57.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:29:56 smithi153 ceph-mon[70044]: pgmap v19847: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:59.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:29:58 smithi083 ceph-mon[79930]: pgmap v19848: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:59.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:29:58 smithi083 ceph-mon[77242]: pgmap v19848: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:29:59.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:29:58 smithi153 ceph-mon[70044]: pgmap v19848: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:00.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:30:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[77219]: 2022-09-29T16:29:59.999+0000 7f160f665700 -1 log_channel(cluster) log [ERR] : overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T16:30:01.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:30:00 smithi083 ceph-mon[79930]: pgmap v19849: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:01.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:30:00 smithi083 ceph-mon[79930]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T16:30:01.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:30:00 smithi083 ceph-mon[77242]: pgmap v19849: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:01.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:30:00 smithi083 ceph-mon[77242]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T16:30:01.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:30:00 smithi153 ceph-mon[70044]: pgmap v19849: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:01.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:30:00 smithi153 ceph-mon[70044]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T16:30:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:30:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:30:01] "GET /metrics HTTP/1.1" 200 225246 "" "Prometheus/2.33.4" 2022-09-29T16:30:02.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:30:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:30:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:30:02.926 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:30:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:30:02.661Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:30:03.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:30:02 smithi083 ceph-mon[79930]: pgmap v19850: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:03.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:30:02 smithi083 ceph-mon[77242]: pgmap v19850: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:03.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:30:02 smithi153 ceph-mon[70044]: pgmap v19850: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:30:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:30:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:30:04.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:30:04 smithi153 ceph-mon[70044]: pgmap v19851: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:04.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:30:04 smithi083 ceph-mon[79930]: pgmap v19851: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:04.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:30:04 smithi083 ceph-mon[77242]: pgmap v19851: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:06.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:30:05 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:30:05.998Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:30:07.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:30:06 smithi083 ceph-mon[79930]: pgmap v19852: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:07.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:30:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:30:06.749Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:30:07.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:30:06 smithi083 ceph-mon[77242]: pgmap v19852: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:07.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:30:06 smithi153 ceph-mon[70044]: pgmap v19852: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:09.092 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:30:09.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:30:08 smithi083 ceph-mon[79930]: pgmap v19853: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:09.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:30:08 smithi083 ceph-mon[77242]: pgmap v19853: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:09.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:30:08 smithi153 ceph-mon[70044]: pgmap v19853: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:09.349 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:30:09.350 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 9m ago 11h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:30:09.350 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 6m ago 11h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:30:09.351 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 9m ago 11h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:30:09.351 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 6m ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:30:09.352 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (11h) 6m ago 11h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:30:09.352 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11h) 9m ago 11h 1069M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:30:09.352 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11h) 9m ago 11h 2369M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:30:09.353 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (11h) 6m ago 11h 1060M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:30:09.353 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (11h) 9m ago 11h 1428M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:30:09.354 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 9m ago 11h 24.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:30:09.354 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 6m ago 11h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:30:09.355 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 9m ago 11h 1725M 2062M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:30:09.355 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 9m ago 11h 1711M 2062M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:30:09.356 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 9m ago 11h 1328M 2062M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:30:09.356 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 9m ago 11h 1677M 2062M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:30:09.356 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 6m ago 11h 1469M 2491M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:30:09.357 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 6m ago 11h 1847M 2491M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:30:09.357 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 6m ago 11h 1651M 2491M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:30:09.433 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 6m ago 11h 1965M 2491M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:30:09.433 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (11h) 6m ago 11h 190M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:30:09.705 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:30:09.706 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:30:09.706 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:30:09.707 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:30:09.707 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:30:09.708 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:30:09.708 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:30:09.709 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:30:09.709 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:30:09.709 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:30:09.710 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:30:09.710 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:30:09.711 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:30:09.711 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:30:09.711 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:30:09.712 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:30:09.712 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:30:09.713 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:30:10.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:30:09 smithi083 ceph-mon[79930]: from='client.67147 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:30:10.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:30:09 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2676545691' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:30:10.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:30:09 smithi083 ceph-mon[77242]: from='client.67147 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:30:10.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:30:09 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2676545691' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:30:10.324 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:30:09 smithi153 ceph-mon[70044]: from='client.67147 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:30:10.325 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:30:09 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2676545691' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:30:11.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:30:10 smithi083 ceph-mon[79930]: from='client.67153 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:30:11.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:30:10 smithi083 ceph-mon[79930]: pgmap v19854: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:11.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:30:10 smithi083 ceph-mon[77242]: from='client.67153 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:30:11.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:30:10 smithi083 ceph-mon[77242]: pgmap v19854: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:11.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:30:10 smithi153 ceph-mon[70044]: from='client.67153 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:30:11.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:30:10 smithi153 ceph-mon[70044]: pgmap v19854: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:11.966 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:30:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:30:11] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T16:30:12.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:30:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:30:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:30:12.926 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:30:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:30:12.661Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:30:13.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:30:12 smithi083 ceph-mon[79930]: pgmap v19855: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:13.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:30:12 smithi083 ceph-mon[77242]: pgmap v19855: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:13.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:30:12 smithi153 ceph-mon[70044]: pgmap v19855: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:30:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:30:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:30:14.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:30:14 smithi153 ceph-mon[70044]: pgmap v19856: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:14.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:30:14 smithi083 ceph-mon[79930]: pgmap v19856: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:14.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:30:14 smithi083 ceph-mon[77242]: pgmap v19856: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:16.460 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:30:15 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:30:15.999Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:30:17.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:30:16 smithi083 ceph-mon[77242]: pgmap v19857: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:17.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:30:16 smithi083 ceph-mon[79930]: pgmap v19857: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:17.217 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:30:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:30:16.750Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:30:17.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:30:16 smithi153 ceph-mon[70044]: pgmap v19857: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:18.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:30:17 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:30:18.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:30:17 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:30:18.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:30:17 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:30:18.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:30:17 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:30:18.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:30:17 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:30:18.218 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:30:17 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:30:18.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:30:17 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:30:18.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:30:17 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:30:18.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:30:17 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:30:19.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:30:19 smithi083 ceph-mon[79930]: pgmap v19858: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:19.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:30:19 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:30:19.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:30:19 smithi083 ceph-mon[77242]: pgmap v19858: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:19.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:30:19 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:30:19.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:30:19 smithi153 ceph-mon[70044]: pgmap v19858: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:19.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:30:19 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:30:21.472 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:30:21 smithi083 ceph-mon[79930]: pgmap v19859: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:21.473 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:30:21 smithi083 ceph-mon[77242]: pgmap v19859: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:21.562 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:30:21 smithi153 ceph-mon[70044]: pgmap v19859: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:21.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:30:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:30:21] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T16:30:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:30:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:30:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:30:22.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:30:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:30:22.662Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:30:23.317 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:30:23 smithi153 ceph-mon[70044]: pgmap v19860: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:23.317 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:30:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:30:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:30:23.460 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:30:23 smithi083 ceph-mon[77242]: pgmap v19860: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:23.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:30:23 smithi083 ceph-mon[79930]: pgmap v19860: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:24.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:30:24 smithi153 ceph-mon[70044]: pgmap v19861: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:24.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:30:24 smithi083 ceph-mon[77242]: pgmap v19861: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:24.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:30:24 smithi083 ceph-mon[79930]: pgmap v19861: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:26.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:30:25 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:30:25.999Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:30:27.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:30:26 smithi083 ceph-mon[79930]: pgmap v19862: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:27.218 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:30:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:30:26.750Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:30:27.219 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:30:26 smithi083 ceph-mon[77242]: pgmap v19862: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:27.313 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:30:26 smithi153 ceph-mon[70044]: pgmap v19862: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:29.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:30:28 smithi083 ceph-mon[79930]: pgmap v19863: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:29.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:30:28 smithi083 ceph-mon[77242]: pgmap v19863: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:29.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:30:28 smithi153 ceph-mon[70044]: pgmap v19863: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:31.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:30:30 smithi083 ceph-mon[79930]: pgmap v19864: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:31.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:30:30 smithi083 ceph-mon[77242]: pgmap v19864: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:31.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:30:30 smithi153 ceph-mon[70044]: pgmap v19864: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:31.969 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:30:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:30:31] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T16:30:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:30:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:30:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:30:32.926 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:30:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:30:32.662Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:30:33.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:30:32 smithi083 ceph-mon[77242]: pgmap v19865: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:33.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:30:32 smithi083 ceph-mon[79930]: pgmap v19865: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:33.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:30:32 smithi153 ceph-mon[70044]: pgmap v19865: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:30:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:30:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:30:34.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:30:34 smithi153 ceph-mon[70044]: pgmap v19866: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:34.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:30:34 smithi083 ceph-mon[77242]: pgmap v19866: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:34.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:30:34 smithi083 ceph-mon[79930]: pgmap v19866: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:36.469 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:30:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:30:36.000Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:30:37.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:30:36 smithi083 ceph-mon[79930]: pgmap v19867: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:37.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:30:36 smithi083 ceph-mon[77242]: pgmap v19867: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:37.215 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:30:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:30:36.751Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:30:37.312 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:30:36 smithi153 ceph-mon[70044]: pgmap v19867: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:39.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:30:38 smithi083 ceph-mon[79930]: pgmap v19868: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:39.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:30:38 smithi083 ceph-mon[77242]: pgmap v19868: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:39.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:30:38 smithi153 ceph-mon[70044]: pgmap v19868: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:40.040 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:30:40.328 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:30:40.328 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 9m ago 11h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:30:40.329 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 7m ago 11h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:30:40.329 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 9m ago 11h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:30:40.330 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 7m ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:30:40.330 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (11h) 7m ago 11h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:30:40.331 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11h) 9m ago 11h 1069M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:30:40.331 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11h) 9m ago 11h 2369M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:30:40.332 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (11h) 7m ago 11h 1060M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:30:40.332 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (11h) 9m ago 11h 1428M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:30:40.333 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 9m ago 11h 24.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:30:40.333 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 7m ago 11h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:30:40.334 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10h) 9m ago 11h 1725M 2062M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:30:40.334 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 9m ago 11h 1711M 2062M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:30:40.335 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 9m ago 11h 1328M 2062M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:30:40.336 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 9m ago 11h 1677M 2062M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:30:40.336 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 7m ago 11h 1469M 2491M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:30:40.337 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 7m ago 11h 1847M 2491M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:30:40.337 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 7m ago 11h 1651M 2491M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:30:40.339 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 7m ago 11h 1965M 2491M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:30:40.344 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (11h) 7m ago 11h 190M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:30:40.682 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:30:40.683 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:30:40.684 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:30:40.684 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:30:40.684 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:30:40.685 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:30:40.685 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:30:40.694 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:30:40.694 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:30:40.695 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:30:40.695 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:30:40.695 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:30:40.696 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:30:40.696 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:30:40.697 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:30:40.697 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:30:40.701 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:30:40.701 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:30:41.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:30:40 smithi083 ceph-mon[79930]: pgmap v19869: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:41.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:30:40 smithi083 ceph-mon[79930]: from='client.57234 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:30:41.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:30:40 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3238277529' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:30:41.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:30:40 smithi083 ceph-mon[77242]: pgmap v19869: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:41.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:30:40 smithi083 ceph-mon[77242]: from='client.57234 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:30:41.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:30:40 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3238277529' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:30:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:30:40 smithi153 ceph-mon[70044]: pgmap v19869: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:41.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:30:40 smithi153 ceph-mon[70044]: from='client.57234 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:30:41.312 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:30:40 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3238277529' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:30:41.935 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:30:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:30:41] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T16:30:41.936 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:30:41 smithi083 ceph-mon[77242]: from='client.57240 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:30:41.936 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:30:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:30:41.936 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:30:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:30:41.937 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:30:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:30:41.937 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:30:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:30:42.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:30:41 smithi083 ceph-mon[79930]: from='client.57240 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:30:42.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:30:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:30:42.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:30:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:30:42.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:30:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:30:42.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:30:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:30:42.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:30:41 smithi153 ceph-mon[70044]: from='client.57240 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:30:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:30:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:30:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:30:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:30:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:30:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:30:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:30:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:30:42.839 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:30:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:30:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:30:42.941 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:30:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:30:42.663Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:30:43.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:30:42 smithi083 ceph-mon[79930]: pgmap v19870: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:43.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:30:42 smithi083 ceph-mon[77242]: pgmap v19870: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:43.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:30:42 smithi153 ceph-mon[70044]: pgmap v19870: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:30:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:30:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:30:44.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:30:44 smithi153 ceph-mon[70044]: pgmap v19871: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:44.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:30:44 smithi083 ceph-mon[79930]: pgmap v19871: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:44.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:30:44 smithi083 ceph-mon[77242]: pgmap v19871: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:46.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:30:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:30:46.000Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:30:47.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:30:46 smithi083 ceph-mon[79930]: pgmap v19872: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:47.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:30:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:30:46.751Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:30:47.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:30:46 smithi083 ceph-mon[77242]: pgmap v19872: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:47.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:30:46 smithi153 ceph-mon[70044]: pgmap v19872: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:49.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:30:48 smithi083 ceph-mon[79930]: pgmap v19873: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:49.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:30:48 smithi083 ceph-mon[77242]: pgmap v19873: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:49.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:30:48 smithi153 ceph-mon[70044]: pgmap v19873: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:51.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:30:50 smithi083 ceph-mon[79930]: pgmap v19874: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:51.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:30:50 smithi083 ceph-mon[77242]: pgmap v19874: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:51.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:30:50 smithi153 ceph-mon[70044]: pgmap v19874: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:30:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:30:51] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T16:30:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:30:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:30:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:30:52.942 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:30:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:30:52.664Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:30:53.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:30:52 smithi083 ceph-mon[79930]: pgmap v19875: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:53.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:30:52 smithi083 ceph-mon[77242]: pgmap v19875: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:53.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:30:52 smithi153 ceph-mon[70044]: pgmap v19875: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:30:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:30:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:30:54.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:30:54 smithi153 ceph-mon[70044]: pgmap v19876: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:54.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:30:54 smithi083 ceph-mon[79930]: pgmap v19876: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:54.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:30:54 smithi083 ceph-mon[77242]: pgmap v19876: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:56.460 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:30:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:30:56.001Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:30:57.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:30:56 smithi083 ceph-mon[79930]: pgmap v19877: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:57.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:30:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:30:56.752Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:30:57.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:30:56 smithi083 ceph-mon[77242]: pgmap v19877: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:57.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:30:56 smithi153 ceph-mon[70044]: pgmap v19877: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:59.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:30:58 smithi083 ceph-mon[79930]: pgmap v19878: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:59.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:30:58 smithi083 ceph-mon[77242]: pgmap v19878: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:30:59.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:30:58 smithi153 ceph-mon[70044]: pgmap v19878: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:01.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:31:00 smithi083 ceph-mon[79930]: pgmap v19879: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:01.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:31:00 smithi083 ceph-mon[77242]: pgmap v19879: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:01.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:31:00 smithi153 ceph-mon[70044]: pgmap v19879: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:01.964 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:31:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:31:01] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T16:31:02.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:31:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:31:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:31:02.936 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:31:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:31:02.664Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:31:03.220 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:31:02 smithi083 ceph-mon[77242]: pgmap v19880: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:03.221 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:31:02 smithi083 ceph-mon[79930]: pgmap v19880: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:03.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:31:02 smithi153 ceph-mon[70044]: pgmap v19880: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:31:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:31:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:31:04.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:31:04 smithi153 ceph-mon[70044]: pgmap v19881: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:04.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:31:04 smithi083 ceph-mon[79930]: pgmap v19881: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:04.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:31:04 smithi083 ceph-mon[77242]: pgmap v19881: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:06.460 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:31:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:31:06.001Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:31:07.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:31:06 smithi083 ceph-mon[77242]: pgmap v19882: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:07.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:31:06 smithi083 ceph-mon[79930]: pgmap v19882: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:07.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:31:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:31:06.752Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:31:07.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:31:06 smithi153 ceph-mon[70044]: pgmap v19882: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:09.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:31:08 smithi083 ceph-mon[79930]: pgmap v19883: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:09.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:31:08 smithi083 ceph-mon[77242]: pgmap v19883: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:09.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:31:08 smithi153 ceph-mon[70044]: pgmap v19883: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:11.009 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:31:11.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:31:10 smithi083 ceph-mon[79930]: pgmap v19884: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:11.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:31:10 smithi083 ceph-mon[77242]: pgmap v19884: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:11.297 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:31:11.298 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 10m ago 11h 27.1M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:31:11.298 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 8m ago 11h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:31:11.299 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 10m ago 11h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:31:11.299 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 8m ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:31:11.299 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (11h) 8m ago 11h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:31:11.299 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11h) 10m ago 11h 1069M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:31:11.300 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11h) 10m ago 11h 2369M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:31:11.300 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (11h) 8m ago 11h 1060M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:31:11.301 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (11h) 10m ago 11h 1428M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:31:11.301 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 10m ago 11h 24.6M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:31:11.301 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 8m ago 11h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:31:11.303 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11h) 10m ago 11h 1725M 2062M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:31:11.303 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10h) 10m ago 11h 1711M 2062M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:31:11.304 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10h) 10m ago 11h 1328M 2062M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:31:11.305 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 10m ago 11h 1677M 2062M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:31:11.305 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 8m ago 11h 1469M 2491M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:31:11.305 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 8m ago 11h 1847M 2491M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:31:11.306 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 8m ago 11h 1651M 2491M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:31:11.307 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 8m ago 11h 1965M 2491M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:31:11.307 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (11h) 8m ago 11h 190M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:31:11.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:31:10 smithi153 ceph-mon[70044]: pgmap v19884: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:11.662 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:31:11.662 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:31:11.662 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:31:11.663 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:31:11.663 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:31:11.663 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:31:11.663 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:31:11.664 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:31:11.664 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:31:11.664 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:31:11.664 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:31:11.665 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:31:11.665 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:31:11.665 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:31:11.665 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:31:11.666 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:31:11.666 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:31:11.667 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:31:11.944 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:31:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:31:11] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T16:31:12.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:31:11 smithi083 ceph-mon[79930]: from='client.67183 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:31:12.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:31:11 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1220854735' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:31:12.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:31:11 smithi083 ceph-mon[77242]: from='client.67183 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:31:12.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:31:11 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1220854735' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:31:12.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:31:11 smithi153 ceph-mon[70044]: from='client.67183 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:31:12.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:31:11 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1220854735' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:31:12.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:31:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:31:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:31:12.945 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:31:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:31:12.665Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:31:13.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:31:12 smithi083 ceph-mon[79930]: from='client.57258 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:31:13.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:31:12 smithi083 ceph-mon[79930]: pgmap v19885: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:13.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:31:12 smithi083 ceph-mon[77242]: from='client.57258 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:31:13.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:31:12 smithi083 ceph-mon[77242]: pgmap v19885: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:31:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:31:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:31:13.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:31:12 smithi153 ceph-mon[70044]: from='client.57258 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:31:13.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:31:12 smithi153 ceph-mon[70044]: pgmap v19885: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:14.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:31:14 smithi153 ceph-mon[70044]: pgmap v19886: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:14.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:31:14 smithi083 ceph-mon[79930]: pgmap v19886: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:14.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:31:14 smithi083 ceph-mon[77242]: pgmap v19886: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:16.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:31:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:31:16.002Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:31:17.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:31:16 smithi083 ceph-mon[79930]: pgmap v19887: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:17.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:31:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:31:16.754Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:31:17.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:31:16 smithi083 ceph-mon[77242]: pgmap v19887: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:17.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:31:16 smithi153 ceph-mon[70044]: pgmap v19887: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:19.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:31:18 smithi083 ceph-mon[79930]: pgmap v19888: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:19.218 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:31:18 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:31:19.219 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:31:18 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:31:19.219 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:31:18 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:31:19.220 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:31:18 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:31:19.220 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:31:18 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:31:19.220 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:31:18 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:31:19.221 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:31:18 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:31:19.222 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:31:18 smithi083 ceph-mon[77242]: pgmap v19888: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:19.222 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:31:18 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:31:19.223 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:31:18 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:31:19.223 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:31:18 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:31:19.224 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:31:18 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:31:19.224 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:31:18 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:31:19.224 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:31:18 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:31:19.225 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:31:18 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:31:19.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:31:18 smithi153 ceph-mon[70044]: pgmap v19888: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:19.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:31:18 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:31:19.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:31:18 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:31:19.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:31:18 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:31:19.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:31:18 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:31:19.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:31:18 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:31:19.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:31:18 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:31:19.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:31:18 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:31:21.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:31:20 smithi083 ceph-mon[79930]: pgmap v19889: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:21.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:31:20 smithi083 ceph-mon[77242]: pgmap v19889: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:21.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:31:20 smithi153 ceph-mon[70044]: pgmap v19889: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:21.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:31:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:31:21] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T16:31:22.810 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:31:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:31:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:31:22.940 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:31:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:31:22.665Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:31:23.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:31:22 smithi083 ceph-mon[77242]: pgmap v19890: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:23.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:31:22 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:31:23.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:31:22 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:31:23.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:31:22 smithi083 ceph-mon[79930]: pgmap v19890: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:23.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:31:22 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:31:23.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:31:22 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:31:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:31:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:31:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:31:23.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:31:22 smithi153 ceph-mon[70044]: pgmap v19890: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:23.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:31:22 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:31:23.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:31:22 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:31:24.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:31:24 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:31:24.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:31:24 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:31:24.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:31:24 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:31:25.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:31:25 smithi153 ceph-mon[70044]: pgmap v19891: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:25.723 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:31:25 smithi083 ceph-mon[79930]: pgmap v19891: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:25.724 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:31:25 smithi083 ceph-mon[77242]: pgmap v19891: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:26.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:31:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:31:26.003Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:31:27.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:31:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:31:26.754Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:31:27.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:31:27 smithi153 ceph-mon[70044]: pgmap v19892: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:27.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:31:27 smithi083 ceph-mon[79930]: pgmap v19892: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:27.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:31:27 smithi083 ceph-mon[77242]: pgmap v19892: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:29.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:31:29 smithi153 ceph-mon[70044]: pgmap v19893: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:29.718 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:31:29 smithi083 ceph-mon[79930]: pgmap v19893: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:29.718 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:31:29 smithi083 ceph-mon[77242]: pgmap v19893: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:31.508 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:31:31 smithi083 ceph-mon[79930]: pgmap v19894: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:31.508 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:31:31 smithi083 ceph-mon[77242]: pgmap v19894: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:31.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:31:31 smithi153 ceph-mon[70044]: pgmap v19894: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:31.968 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:31:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:31:31] "GET /metrics HTTP/1.1" 200 225247 "" "Prometheus/2.33.4" 2022-09-29T16:31:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:31:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:31:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:31:32.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:31:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:31:32.666Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:31:33.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:31:33 smithi153 ceph-mon[70044]: pgmap v19895: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:31:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:31:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:31:33.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:31:33 smithi083 ceph-mon[77242]: pgmap v19895: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:33.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:31:33 smithi083 ceph-mon[79930]: pgmap v19895: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:34.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:31:34 smithi153 ceph-mon[70044]: pgmap v19896: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:34.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:31:34 smithi083 ceph-mon[79930]: pgmap v19896: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:34.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:31:34 smithi083 ceph-mon[77242]: pgmap v19896: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:36.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:31:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:31:36.003Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:31:37.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:31:36 smithi083 ceph-mon[79930]: pgmap v19897: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:37.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:31:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:31:36.754Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:31:37.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:31:36 smithi083 ceph-mon[77242]: pgmap v19897: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:37.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:31:36 smithi153 ceph-mon[70044]: pgmap v19897: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:39.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:31:38 smithi083 ceph-mon[79930]: pgmap v19898: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:39.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:31:38 smithi083 ceph-mon[77242]: pgmap v19898: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:39.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:31:38 smithi153 ceph-mon[70044]: pgmap v19898: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:41.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:31:40 smithi083 ceph-mon[79930]: pgmap v19899: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:41.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:31:40 smithi083 ceph-mon[77242]: pgmap v19899: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:41.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:31:40 smithi153 ceph-mon[70044]: pgmap v19899: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:41.953 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:31:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:31:41] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T16:31:41.996 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:31:42.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:31:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:31:42.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:31:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:31:42.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:31:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:31:42.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:31:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:31:42.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:31:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:31:42.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:31:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:31:42.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:31:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:31:42.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:31:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:31:42.286 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:31:42.287 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 19s ago 11h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:31:42.287 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 8m ago 11h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:31:42.288 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 19s ago 11h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:31:42.288 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 8m ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:31:42.289 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (11h) 8m ago 11h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:31:42.289 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11h) 19s ago 11h 1076M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:31:42.292 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11h) 19s ago 11h 2403M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:31:42.293 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (11h) 8m ago 11h 1060M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:31:42.293 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (11h) 19s ago 11h 1447M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:31:42.293 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 19s ago 11h 24.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:31:42.293 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 8m ago 11h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:31:42.294 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11h) 19s ago 11h 1751M 2062M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:31:42.294 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11h) 19s ago 11h 1736M 2062M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:31:42.294 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11h) 19s ago 11h 1348M 2062M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:31:42.294 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 19s ago 11h 1702M 2062M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:31:42.295 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 8m ago 11h 1469M 2491M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:31:42.295 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 8m ago 11h 1847M 2491M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:31:42.295 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 8m ago 11h 1651M 2491M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:31:42.295 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 8m ago 11h 1965M 2491M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:31:42.296 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (11h) 8m ago 11h 190M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:31:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:31:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:31:42.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:31:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:31:42.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:31:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:31:42.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:31:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:31:42.643 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:31:42.644 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:31:42.644 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:31:42.645 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:31:42.645 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:31:42.645 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:31:42.646 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:31:42.646 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:31:42.647 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:31:42.647 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:31:42.648 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:31:42.648 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:31:42.649 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:31:42.650 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:31:42.650 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:31:42.651 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:31:42.651 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:31:42.652 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:31:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:31:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:31:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:31:42.954 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:31:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:31:42.667Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:31:43.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:31:42 smithi083 ceph-mon[79930]: pgmap v19900: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:43.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:31:42 smithi083 ceph-mon[79930]: from='client.57270 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:31:43.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:31:42 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3787381019' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:31:43.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:31:42 smithi083 ceph-mon[77242]: pgmap v19900: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:43.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:31:42 smithi083 ceph-mon[77242]: from='client.57270 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:31:43.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:31:42 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3787381019' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:31:43.311 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:31:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:31:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:31:43.312 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:31:42 smithi153 ceph-mon[70044]: pgmap v19900: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:43.312 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:31:42 smithi153 ceph-mon[70044]: from='client.57270 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:31:43.312 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:31:42 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3787381019' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:31:44.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:31:43 smithi083 ceph-mon[79930]: from='client.57276 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:31:44.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:31:43 smithi083 ceph-mon[77242]: from='client.57276 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:31:44.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:31:43 smithi153 ceph-mon[70044]: from='client.57276 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:31:45.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:31:44 smithi083 ceph-mon[79930]: pgmap v19901: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:45.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:31:44 smithi083 ceph-mon[77242]: pgmap v19901: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:45.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:31:44 smithi153 ceph-mon[70044]: pgmap v19901: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:46.460 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:31:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:31:46.004Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:31:47.219 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:31:46 smithi083 ceph-mon[79930]: pgmap v19902: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:47.219 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:31:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:31:46.755Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:31:47.220 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:31:46 smithi083 ceph-mon[77242]: pgmap v19902: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:47.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:31:46 smithi153 ceph-mon[70044]: pgmap v19902: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:49.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:31:48 smithi153 ceph-mon[70044]: pgmap v19903: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:49.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:31:48 smithi083 ceph-mon[77242]: pgmap v19903: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:49.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:31:48 smithi083 ceph-mon[79930]: pgmap v19903: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:51.316 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:31:50 smithi153 ceph-mon[70044]: pgmap v19904: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:51.467 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:31:50 smithi083 ceph-mon[77242]: pgmap v19904: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:51.468 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:31:50 smithi083 ceph-mon[79930]: pgmap v19904: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:31:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:31:51] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T16:31:52.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:31:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:31:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:31:52.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:31:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:31:52.668Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:31:53.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:31:52 smithi153 ceph-mon[70044]: pgmap v19905: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:53.310 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:31:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:31:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:31:53.460 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:31:52 smithi083 ceph-mon[77242]: pgmap v19905: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:53.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:31:52 smithi083 ceph-mon[79930]: pgmap v19905: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:54.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:31:54 smithi153 ceph-mon[70044]: pgmap v19906: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:54.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:31:54 smithi083 ceph-mon[79930]: pgmap v19906: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:54.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:31:54 smithi083 ceph-mon[77242]: pgmap v19906: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:56.460 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:31:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:31:56.005Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:31:57.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:31:56 smithi083 ceph-mon[79930]: pgmap v19907: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:57.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:31:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:31:56.756Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:31:57.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:31:56 smithi083 ceph-mon[77242]: pgmap v19907: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:57.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:31:56 smithi153 ceph-mon[70044]: pgmap v19907: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:59.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:31:58 smithi083 ceph-mon[77242]: pgmap v19908: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:59.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:31:58 smithi083 ceph-mon[79930]: pgmap v19908: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:31:59.314 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:31:58 smithi153 ceph-mon[70044]: pgmap v19908: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:01.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:32:00 smithi083 ceph-mon[79930]: pgmap v19909: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:01.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:32:00 smithi083 ceph-mon[77242]: pgmap v19909: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:01.317 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:32:00 smithi153 ceph-mon[70044]: pgmap v19909: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:01.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:32:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:32:01] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T16:32:02.810 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:32:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:32:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:32:02.953 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:32:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:32:02.668Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:32:03.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:32:02 smithi083 ceph-mon[77242]: pgmap v19910: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:03.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:32:02 smithi083 ceph-mon[79930]: pgmap v19910: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:03.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:32:02 smithi153 ceph-mon[70044]: pgmap v19910: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:32:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:32:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:32:04.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:32:04 smithi153 ceph-mon[70044]: pgmap v19911: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:04.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:32:04 smithi083 ceph-mon[77242]: pgmap v19911: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:04.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:32:04 smithi083 ceph-mon[79930]: pgmap v19911: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:06.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:32:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:32:06.006Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:32:07.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:32:06 smithi083 ceph-mon[77242]: pgmap v19912: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:07.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:32:06 smithi083 ceph-mon[79930]: pgmap v19912: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:07.214 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:32:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:32:06.757Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:32:07.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:32:06 smithi153 ceph-mon[70044]: pgmap v19912: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:09.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:32:08 smithi083 ceph-mon[79930]: pgmap v19913: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:09.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:32:08 smithi083 ceph-mon[77242]: pgmap v19913: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:09.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:32:08 smithi153 ceph-mon[70044]: pgmap v19913: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:11.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:32:10 smithi083 ceph-mon[79930]: pgmap v19914: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:11.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:32:10 smithi083 ceph-mon[77242]: pgmap v19914: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:11.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:32:10 smithi153 ceph-mon[70044]: pgmap v19914: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:32:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:32:11] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T16:32:12.811 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:32:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:32:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:32:12.956 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:32:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:32:12.669Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:32:12.970 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:32:13.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:32:12 smithi083 ceph-mon[79930]: pgmap v19915: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:13.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:32:12 smithi083 ceph-mon[77242]: pgmap v19915: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:13.265 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:32:13.266 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 50s ago 11h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:32:13.267 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 9m ago 11h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:32:13.267 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 50s ago 11h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:32:13.268 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 9m ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:32:13.268 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (11h) 9m ago 11h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:32:13.269 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11h) 50s ago 11h 1076M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:32:13.269 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11h) 50s ago 11h 2403M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:32:13.270 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (11h) 9m ago 11h 1060M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:32:13.270 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (11h) 50s ago 11h 1447M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:32:13.271 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 50s ago 11h 24.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:32:13.271 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 9m ago 11h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:32:13.272 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11h) 50s ago 11h 1751M 2062M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:32:13.272 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11h) 50s ago 11h 1736M 2062M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:32:13.272 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11h) 50s ago 11h 1348M 2062M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:32:13.274 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10h) 50s ago 11h 1702M 2062M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:32:13.278 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 9m ago 11h 1469M 2491M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:32:13.279 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 9m ago 11h 1847M 2491M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:32:13.279 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 9m ago 11h 1651M 2491M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:32:13.280 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 9m ago 11h 1965M 2491M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:32:13.280 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (11h) 9m ago 11h 190M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:32:13.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:32:12 smithi153 ceph-mon[70044]: pgmap v19915: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:32:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:32:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:32:13.629 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:32:13.629 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:32:13.629 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:32:13.630 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:32:13.630 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:32:13.630 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:32:13.630 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:32:13.631 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:32:13.631 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:32:13.631 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:32:13.631 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:32:13.632 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:32:13.632 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:32:13.632 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:32:13.632 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:32:13.633 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:32:13.633 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:32:13.633 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:32:14.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:32:13 smithi083 ceph-mon[79930]: from='client.67219 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:32:14.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:32:13 smithi083 ceph-mon[79930]: from='client.57294 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:32:14.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:32:13 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2038943644' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:32:14.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:32:13 smithi083 ceph-mon[77242]: from='client.67219 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:32:14.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:32:13 smithi083 ceph-mon[77242]: from='client.57294 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:32:14.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:32:13 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2038943644' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:32:14.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:32:13 smithi153 ceph-mon[70044]: from='client.67219 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:32:14.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:32:13 smithi153 ceph-mon[70044]: from='client.57294 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:32:14.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:32:13 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2038943644' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:32:15.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:32:14 smithi153 ceph-mon[70044]: pgmap v19916: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:15.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:32:14 smithi083 ceph-mon[77242]: pgmap v19916: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:15.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:32:14 smithi083 ceph-mon[79930]: pgmap v19916: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:16.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:32:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:32:16.006Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:32:17.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:32:16 smithi083 ceph-mon[79930]: pgmap v19917: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:17.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:32:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:32:16.758Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:32:17.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:32:16 smithi083 ceph-mon[77242]: pgmap v19917: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:17.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:32:16 smithi153 ceph-mon[70044]: pgmap v19917: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:19.312 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:32:18 smithi153 ceph-mon[70044]: pgmap v19918: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:19.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:32:18 smithi083 ceph-mon[79930]: pgmap v19918: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:19.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:32:18 smithi083 ceph-mon[77242]: pgmap v19918: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:21.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:32:20 smithi153 ceph-mon[70044]: pgmap v19919: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:21.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:32:20 smithi083 ceph-mon[79930]: pgmap v19919: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:21.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:32:20 smithi083 ceph-mon[77242]: pgmap v19919: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:32:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:32:21] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T16:32:22.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:32:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:32:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:32:22.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:32:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:32:22.670Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:32:23.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:32:22 smithi153 ceph-mon[70044]: pgmap v19920: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:32:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:32:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:32:23.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:32:22 smithi083 ceph-mon[77242]: pgmap v19920: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:23.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:32:22 smithi083 ceph-mon[79930]: pgmap v19920: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:24.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:32:23 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:32:24.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:32:23 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:32:24.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:32:23 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:32:24.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:32:23 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:32:24.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:32:23 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:32:24.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:32:23 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:32:24.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:32:23 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:32:24.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:32:23 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:32:24.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:32:23 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:32:24.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:32:23 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:32:24.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:32:23 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:32:24.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:32:23 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:32:25.314 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:32:24 smithi153 ceph-mon[70044]: pgmap v19921: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:25.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:32:24 smithi083 ceph-mon[79930]: pgmap v19921: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:25.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:32:24 smithi083 ceph-mon[77242]: pgmap v19921: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:26.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:32:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:32:26.007Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:32:27.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:32:26 smithi083 ceph-mon[79930]: pgmap v19922: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:27.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:32:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:32:26.759Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:32:27.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:32:26 smithi083 ceph-mon[77242]: pgmap v19922: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:27.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:32:26 smithi153 ceph-mon[70044]: pgmap v19922: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:29.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:32:28 smithi153 ceph-mon[70044]: pgmap v19923: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:29.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:32:28 smithi083 ceph-mon[79930]: pgmap v19923: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:29.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:32:28 smithi083 ceph-mon[77242]: pgmap v19923: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:31.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:32:30 smithi153 ceph-mon[70044]: pgmap v19924: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:31.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:32:30 smithi083 ceph-mon[79930]: pgmap v19924: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:31.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:32:30 smithi083 ceph-mon[77242]: pgmap v19924: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:32:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:32:31] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T16:32:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:32:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:32:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:32:32.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:32:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:32:32.671Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:32:33.315 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:32:32 smithi153 ceph-mon[70044]: pgmap v19925: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:33.316 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:32:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:32:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:32:33.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:32:32 smithi083 ceph-mon[79930]: pgmap v19925: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:33.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:32:32 smithi083 ceph-mon[77242]: pgmap v19925: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:34.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:32:34 smithi153 ceph-mon[70044]: pgmap v19926: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:34.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:32:34 smithi083 ceph-mon[79930]: pgmap v19926: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:34.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:32:34 smithi083 ceph-mon[77242]: pgmap v19926: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:36.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:32:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:32:36.008Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:32:37.215 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:32:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:32:36.759Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:32:37.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:32:36 smithi083 ceph-mon[77242]: pgmap v19927: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:37.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:32:36 smithi083 ceph-mon[79930]: pgmap v19927: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:37.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:32:36 smithi153 ceph-mon[70044]: pgmap v19927: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:39.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:32:38 smithi083 ceph-mon[77242]: pgmap v19928: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:39.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:32:38 smithi083 ceph-mon[79930]: pgmap v19928: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:39.314 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:32:38 smithi153 ceph-mon[70044]: pgmap v19928: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:41.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:32:40 smithi153 ceph-mon[70044]: pgmap v19929: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:41.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:32:40 smithi083 ceph-mon[77242]: pgmap v19929: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:41.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:32:40 smithi083 ceph-mon[79930]: pgmap v19929: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:32:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:32:41] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T16:32:42.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:32:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:32:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:32:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:32:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:32:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:32:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:32:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:32:42.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:32:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:32:42.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:32:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:32:42.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:32:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:32:42.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:32:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:32:42.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:32:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:32:42.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:32:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:32:42.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:32:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:32:42.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:32:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:32:42.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:32:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:32:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:32:42.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:32:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:32:42.672Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:32:43.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:32:42 smithi153 ceph-mon[70044]: pgmap v19930: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:43.311 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:32:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:32:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:32:43.460 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:32:42 smithi083 ceph-mon[77242]: pgmap v19930: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:43.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:32:42 smithi083 ceph-mon[79930]: pgmap v19930: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:43.955 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:32:44.244 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:32:44.245 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 81s ago 11h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:32:44.245 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 9m ago 11h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:32:44.246 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 81s ago 11h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:32:44.246 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 9m ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:32:44.247 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (11h) 9m ago 11h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:32:44.247 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11h) 81s ago 11h 1076M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:32:44.248 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11h) 81s ago 11h 2403M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:32:44.248 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (11h) 9m ago 11h 1060M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:32:44.249 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (11h) 81s ago 11h 1447M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:32:44.249 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 81s ago 11h 24.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:32:44.250 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 9m ago 11h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:32:44.250 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11h) 81s ago 11h 1751M 2062M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:32:44.251 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11h) 81s ago 11h 1736M 2062M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:32:44.251 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11h) 81s ago 11h 1348M 2062M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:32:44.252 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (11h) 81s ago 11h 1702M 2062M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:32:44.252 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (10h) 9m ago 11h 1469M 2491M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:32:44.252 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 9m ago 11h 1847M 2491M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:32:44.253 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 9m ago 11h 1651M 2491M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:32:44.253 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 9m ago 11h 1965M 2491M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:32:44.254 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (11h) 9m ago 11h 190M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:32:44.602 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:32:44.603 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:32:44.603 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:32:44.604 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:32:44.604 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:32:44.605 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:32:44.606 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:32:44.606 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:32:44.607 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:32:44.607 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:32:44.608 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:32:44.608 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:32:44.609 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:32:44.609 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:32:44.610 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:32:44.610 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:32:44.610 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:32:44.611 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:32:44.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:32:44 smithi153 ceph-mon[70044]: pgmap v19931: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:44.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:32:44 smithi153 ceph-mon[70044]: from='client.57306 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:32:44.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:32:44 smithi153 ceph-mon[70044]: from='client.67243 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:32:44.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:32:44 smithi083 ceph-mon[77242]: pgmap v19931: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:44.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:32:44 smithi083 ceph-mon[77242]: from='client.57306 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:32:44.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:32:44 smithi083 ceph-mon[77242]: from='client.67243 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:32:44.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:32:44 smithi083 ceph-mon[79930]: pgmap v19931: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:44.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:32:44 smithi083 ceph-mon[79930]: from='client.57306 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:32:44.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:32:44 smithi083 ceph-mon[79930]: from='client.67243 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:32:45.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:32:45 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3637096294' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:32:45.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:32:45 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3637096294' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:32:45.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:32:45 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3637096294' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:32:46.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:32:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:32:46.008Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:32:46.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:32:46 smithi153 ceph-mon[70044]: pgmap v19932: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:46.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:32:46 smithi083 ceph-mon[79930]: pgmap v19932: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:46.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:32:46 smithi083 ceph-mon[77242]: pgmap v19932: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:46.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:32:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:32:46.759Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:32:49.312 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:32:48 smithi153 ceph-mon[70044]: pgmap v19933: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:49.472 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:32:48 smithi083 ceph-mon[79930]: pgmap v19933: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:49.485 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:32:48 smithi083 ceph-mon[77242]: pgmap v19933: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:51.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:32:50 smithi153 ceph-mon[70044]: pgmap v19934: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:51.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:32:50 smithi083 ceph-mon[79930]: pgmap v19934: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:51.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:32:50 smithi083 ceph-mon[77242]: pgmap v19934: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:32:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:32:51] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T16:32:52.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:32:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:32:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:32:52.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:32:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:32:52.672Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:32:53.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:32:52 smithi153 ceph-mon[70044]: pgmap v19935: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:32:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:32:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:32:53.475 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:32:52 smithi083 ceph-mon[79930]: pgmap v19935: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:53.476 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:32:52 smithi083 ceph-mon[77242]: pgmap v19935: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:54.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:32:54 smithi153 ceph-mon[70044]: pgmap v19936: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:54.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:32:54 smithi083 ceph-mon[77242]: pgmap v19936: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:54.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:32:54 smithi083 ceph-mon[79930]: pgmap v19936: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:56.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:32:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:32:56.009Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:32:57.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:32:56 smithi083 ceph-mon[77242]: pgmap v19937: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:57.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:32:56 smithi083 ceph-mon[79930]: pgmap v19937: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:57.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:32:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:32:56.760Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:32:57.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:32:56 smithi153 ceph-mon[70044]: pgmap v19937: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:59.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:32:58 smithi153 ceph-mon[70044]: pgmap v19938: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:59.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:32:58 smithi083 ceph-mon[79930]: pgmap v19938: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:32:59.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:32:58 smithi083 ceph-mon[77242]: pgmap v19938: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:01.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:33:00 smithi153 ceph-mon[70044]: pgmap v19939: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:01.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:33:00 smithi083 ceph-mon[79930]: pgmap v19939: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:01.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:33:00 smithi083 ceph-mon[77242]: pgmap v19939: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:33:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:33:01] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T16:33:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:33:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:33:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:33:02.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:33:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:33:02.673Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:33:03.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:33:02 smithi153 ceph-mon[70044]: pgmap v19940: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:33:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:33:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:33:03.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:33:02 smithi083 ceph-mon[79930]: pgmap v19940: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:03.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:33:02 smithi083 ceph-mon[77242]: pgmap v19940: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:04.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:33:04 smithi153 ceph-mon[70044]: pgmap v19941: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:04.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:33:04 smithi083 ceph-mon[79930]: pgmap v19941: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:04.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:33:04 smithi083 ceph-mon[77242]: pgmap v19941: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:06.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:33:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:33:06.009Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:33:07.219 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:33:06 smithi083 ceph-mon[79930]: pgmap v19942: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:07.220 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:33:06 smithi083 ceph-mon[77242]: pgmap v19942: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:07.221 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:33:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:33:06.760Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:33:07.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:33:06 smithi153 ceph-mon[70044]: pgmap v19942: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:09.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:33:08 smithi153 ceph-mon[70044]: pgmap v19943: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:09.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:33:08 smithi083 ceph-mon[79930]: pgmap v19943: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:09.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:33:08 smithi083 ceph-mon[77242]: pgmap v19943: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:11.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:33:10 smithi153 ceph-mon[70044]: pgmap v19944: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:11.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:33:10 smithi083 ceph-mon[77242]: pgmap v19944: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:11.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:33:10 smithi083 ceph-mon[79930]: pgmap v19944: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:33:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:33:11] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T16:33:12.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:33:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:33:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:33:12.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:33:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:33:12.674Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:33:13.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:33:12 smithi153 ceph-mon[70044]: pgmap v19945: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:33:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:33:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:33:13.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:33:12 smithi083 ceph-mon[79930]: pgmap v19945: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:13.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:33:12 smithi083 ceph-mon[77242]: pgmap v19945: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:14.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:33:14 smithi153 ceph-mon[70044]: pgmap v19946: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:14.926 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:33:14.960 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:33:14 smithi083 ceph-mon[77242]: pgmap v19946: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:14.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:33:14 smithi083 ceph-mon[79930]: pgmap v19946: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:15.217 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:33:15.218 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 112s ago 11h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:33:15.218 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 10m ago 11h 63.5M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:33:15.218 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 112s ago 11h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:33:15.218 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 10m ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:33:15.219 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (11h) 10m ago 11h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:33:15.219 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11h) 112s ago 11h 1076M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:33:15.219 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11h) 112s ago 11h 2403M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:33:15.219 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (11h) 10m ago 11h 1060M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:33:15.220 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (11h) 112s ago 11h 1447M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:33:15.220 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 112s ago 11h 24.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:33:15.220 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 10m ago 11h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:33:15.220 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11h) 112s ago 11h 1751M 2062M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:33:15.220 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11h) 112s ago 11h 1736M 2062M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:33:15.221 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11h) 112s ago 11h 1348M 2062M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:33:15.221 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (11h) 112s ago 11h 1702M 2062M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:33:15.221 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (11h) 10m ago 11h 1469M 2491M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:33:15.221 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (10h) 10m ago 11h 1847M 2491M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:33:15.222 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 10m ago 11h 1651M 2491M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:33:15.222 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 10m ago 11h 1965M 2491M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:33:15.222 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (11h) 10m ago 11h 190M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:33:15.575 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:33:15.575 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:33:15.576 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:33:15.576 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:33:15.576 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:33:15.576 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:33:15.577 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:33:15.577 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:33:15.577 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:33:15.577 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:33:15.578 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:33:15.578 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:33:15.578 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:33:15.578 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:33:15.579 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:33:15.579 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:33:15.579 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:33:15.579 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:33:15.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:33:15 smithi153 ceph-mon[70044]: from='client.57324 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:33:15.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:33:15 smithi153 ceph-mon[70044]: from='client.67261 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:33:15.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:33:15 smithi083 ceph-mon[79930]: from='client.57324 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:33:15.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:33:15 smithi083 ceph-mon[79930]: from='client.67261 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:33:15.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:33:15 smithi083 ceph-mon[77242]: from='client.57324 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:33:15.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:33:15 smithi083 ceph-mon[77242]: from='client.67261 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:33:16.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:33:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:33:16.010Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:33:16.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:33:16 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2174308808' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:33:16.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:33:16 smithi153 ceph-mon[70044]: pgmap v19947: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:16.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:33:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:33:16.761Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:33:16.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:33:16 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2174308808' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:33:16.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:33:16 smithi083 ceph-mon[77242]: pgmap v19947: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:16.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:33:16 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2174308808' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:33:16.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:33:16 smithi083 ceph-mon[79930]: pgmap v19947: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:19.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:33:18 smithi153 ceph-mon[70044]: pgmap v19948: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:19.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:33:18 smithi083 ceph-mon[77242]: pgmap v19948: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:19.467 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:33:18 smithi083 ceph-mon[79930]: pgmap v19948: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:21.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:33:20 smithi153 ceph-mon[70044]: pgmap v19949: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:21.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:33:20 smithi083 ceph-mon[77242]: pgmap v19949: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:21.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:33:20 smithi083 ceph-mon[79930]: pgmap v19949: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:21.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:33:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:33:21] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T16:33:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:33:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:33:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:33:22.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:33:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:33:22.675Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:33:23.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:33:22 smithi153 ceph-mon[70044]: pgmap v19950: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:33:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:33:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:33:23.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:33:22 smithi083 ceph-mon[77242]: pgmap v19950: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:23.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:33:22 smithi083 ceph-mon[79930]: pgmap v19950: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:24.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:33:24 smithi153 ceph-mon[70044]: pgmap v19951: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:24.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:33:24 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:33:24.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:33:24 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:33:24.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:33:24 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:33:24.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:33:24 smithi083 ceph-mon[77242]: pgmap v19951: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:24.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:33:24 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:33:24.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:33:24 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:33:24.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:33:24 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:33:24.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:33:24 smithi083 ceph-mon[79930]: pgmap v19951: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:24.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:33:24 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:33:24.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:33:24 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:33:24.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:33:24 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:33:26.460 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:33:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:33:26.012Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:33:27.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:33:26 smithi083 ceph-mon[77242]: pgmap v19952: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:27.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:33:26 smithi083 ceph-mon[79930]: pgmap v19952: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:27.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:33:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:33:26.762Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:33:27.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:33:26 smithi153 ceph-mon[70044]: pgmap v19952: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:29.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:33:28 smithi153 ceph-mon[70044]: pgmap v19953: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:29.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:33:28 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:33:29.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:33:28 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:33:29.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:33:28 smithi083 ceph-mon[79930]: pgmap v19953: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:29.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:33:28 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:33:29.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:33:28 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:33:29.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:33:28 smithi083 ceph-mon[77242]: pgmap v19953: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:29.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:33:28 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:33:29.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:33:28 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:33:30.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:33:30 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:33:30.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:33:30 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:33:30.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:33:30 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:33:31.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:33:31 smithi083 ceph-mon[79930]: pgmap v19954: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:31.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:33:31 smithi083 ceph-mon[77242]: pgmap v19954: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:31.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:33:31 smithi153 ceph-mon[70044]: pgmap v19954: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:31.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:33:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:33:31] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T16:33:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:33:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:33:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:33:32.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:33:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:33:32.675Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:33:33.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:33:33 smithi153 ceph-mon[70044]: pgmap v19955: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:33:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:33:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:33:33.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:33:33 smithi083 ceph-mon[79930]: pgmap v19955: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:33.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:33:33 smithi083 ceph-mon[77242]: pgmap v19955: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:34.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:33:34 smithi153 ceph-mon[70044]: pgmap v19956: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:34.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:33:34 smithi083 ceph-mon[77242]: pgmap v19956: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:34.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:33:34 smithi083 ceph-mon[79930]: pgmap v19956: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:36.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:33:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:33:36.012Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:33:37.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:33:36 smithi083 ceph-mon[79930]: pgmap v19957: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:37.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:33:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:33:36.763Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:33:37.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:33:36 smithi083 ceph-mon[77242]: pgmap v19957: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:37.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:33:36 smithi153 ceph-mon[70044]: pgmap v19957: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:39.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:33:38 smithi153 ceph-mon[70044]: pgmap v19958: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:39.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:33:38 smithi083 ceph-mon[77242]: pgmap v19958: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:39.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:33:38 smithi083 ceph-mon[79930]: pgmap v19958: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:41.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:33:40 smithi153 ceph-mon[70044]: pgmap v19959: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:41.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:33:40 smithi083 ceph-mon[79930]: pgmap v19959: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:41.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:33:40 smithi083 ceph-mon[77242]: pgmap v19959: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:41.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:33:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:33:41] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T16:33:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:33:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:33:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:33:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:33:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:33:41 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:33:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:33:41 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:33:42.460 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:33:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:33:42.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:33:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:33:42.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:33:41 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:33:42.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:33:41 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:33:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:33:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:33:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:33:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:33:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:33:41 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:33:42.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:33:41 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:33:42.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:33:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:33:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:33:42.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:33:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:33:42.676Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:33:43.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:33:42 smithi153 ceph-mon[70044]: pgmap v19960: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:43.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:33:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:33:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:33:43.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:33:42 smithi083 ceph-mon[79930]: pgmap v19960: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:43.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:33:42 smithi083 ceph-mon[77242]: pgmap v19960: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:44.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:33:44 smithi153 ceph-mon[70044]: pgmap v19961: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:44.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:33:44 smithi083 ceph-mon[79930]: pgmap v19961: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:44.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:33:44 smithi083 ceph-mon[77242]: pgmap v19961: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:45.902 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:33:46.192 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:33:46.193 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 2m ago 11h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:33:46.193 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 17s ago 11h 64.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:33:46.193 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 2m ago 11h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:33:46.193 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 17s ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:33:46.194 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (11h) 17s ago 11h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:33:46.194 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11h) 2m ago 11h 1076M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:33:46.194 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11h) 2m ago 11h 2403M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:33:46.194 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (11h) 17s ago 11h 1077M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:33:46.195 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (11h) 2m ago 11h 1447M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:33:46.195 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 2m ago 11h 24.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:33:46.195 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 17s ago 11h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:33:46.195 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11h) 2m ago 11h 1751M 2062M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:33:46.196 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11h) 2m ago 11h 1736M 2062M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:33:46.196 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11h) 2m ago 11h 1348M 2062M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:33:46.196 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (11h) 2m ago 11h 1702M 2062M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:33:46.197 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (11h) 17s ago 11h 1491M 2491M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:33:46.197 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (11h) 17s ago 11h 1875M 2491M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:33:46.198 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (10h) 17s ago 11h 1676M 2491M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:33:46.198 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 17s ago 11h 1995M 2491M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:33:46.198 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (11h) 17s ago 11h 193M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:33:46.460 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:33:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:33:46.012Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:33:46.550 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:33:46.550 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:33:46.551 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:33:46.551 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:33:46.552 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:33:46.552 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:33:46.553 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:33:46.553 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:33:46.553 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:33:46.554 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:33:46.554 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:33:46.555 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:33:46.555 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:33:46.556 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:33:46.557 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:33:46.563 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:33:46.564 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:33:46.564 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:33:47.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:33:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:33:46.764Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:33:47.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:33:46 smithi083 ceph-mon[77242]: from='client.67273 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:33:47.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:33:46 smithi083 ceph-mon[77242]: pgmap v19962: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:47.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:33:46 smithi083 ceph-mon[77242]: from='client.57348 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:33:47.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:33:46 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3927232336' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:33:47.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:33:46 smithi083 ceph-mon[79930]: from='client.67273 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:33:47.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:33:46 smithi083 ceph-mon[79930]: pgmap v19962: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:47.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:33:46 smithi083 ceph-mon[79930]: from='client.57348 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:33:47.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:33:46 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3927232336' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:33:47.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:33:46 smithi153 ceph-mon[70044]: from='client.67273 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:33:47.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:33:46 smithi153 ceph-mon[70044]: pgmap v19962: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:47.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:33:46 smithi153 ceph-mon[70044]: from='client.57348 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:33:47.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:33:46 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3927232336' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:33:49.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:33:48 smithi153 ceph-mon[70044]: pgmap v19963: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:49.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:33:48 smithi083 ceph-mon[79930]: pgmap v19963: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:49.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:33:48 smithi083 ceph-mon[77242]: pgmap v19963: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:51.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:33:50 smithi153 ceph-mon[70044]: pgmap v19964: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:51.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:33:50 smithi083 ceph-mon[79930]: pgmap v19964: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:51.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:33:50 smithi083 ceph-mon[77242]: pgmap v19964: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:33:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:33:51] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T16:33:52.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:33:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:33:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:33:52.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:33:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:33:52.677Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:33:53.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:33:52 smithi153 ceph-mon[70044]: pgmap v19965: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:33:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:33:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:33:53.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:33:52 smithi083 ceph-mon[79930]: pgmap v19965: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:53.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:33:52 smithi083 ceph-mon[77242]: pgmap v19965: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:54.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:33:54 smithi153 ceph-mon[70044]: pgmap v19966: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:54.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:33:54 smithi083 ceph-mon[77242]: pgmap v19966: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:54.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:33:54 smithi083 ceph-mon[79930]: pgmap v19966: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:56.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:33:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:33:56.013Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:33:57.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:33:56 smithi083 ceph-mon[79930]: pgmap v19967: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:57.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:33:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:33:56.765Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:33:57.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:33:56 smithi083 ceph-mon[77242]: pgmap v19967: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:57.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:33:56 smithi153 ceph-mon[70044]: pgmap v19967: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:59.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:33:58 smithi153 ceph-mon[70044]: pgmap v19968: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:59.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:33:58 smithi083 ceph-mon[79930]: pgmap v19968: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:33:59.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:33:58 smithi083 ceph-mon[77242]: pgmap v19968: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:01.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:34:00 smithi153 ceph-mon[70044]: pgmap v19969: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:01.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:34:00 smithi083 ceph-mon[79930]: pgmap v19969: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:01.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:34:00 smithi083 ceph-mon[77242]: pgmap v19969: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:01.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:34:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:34:01] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T16:34:02.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:34:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:34:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:34:02.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:34:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:34:02.677Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:34:03.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:34:02 smithi153 ceph-mon[70044]: pgmap v19970: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:34:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:34:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:34:03.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:34:02 smithi083 ceph-mon[79930]: pgmap v19970: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:03.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:34:02 smithi083 ceph-mon[77242]: pgmap v19970: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:04.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:34:04 smithi153 ceph-mon[70044]: pgmap v19971: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:04.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:34:04 smithi083 ceph-mon[79930]: pgmap v19971: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:04.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:34:04 smithi083 ceph-mon[77242]: pgmap v19971: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:06.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:34:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:34:06.014Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:34:07.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:34:06 smithi083 ceph-mon[79930]: pgmap v19972: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:07.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:34:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:34:06.766Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:34:07.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:34:06 smithi083 ceph-mon[77242]: pgmap v19972: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:07.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:34:06 smithi153 ceph-mon[70044]: pgmap v19972: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:09.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:34:08 smithi153 ceph-mon[70044]: pgmap v19973: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:09.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:34:08 smithi083 ceph-mon[79930]: pgmap v19973: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:09.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:34:08 smithi083 ceph-mon[77242]: pgmap v19973: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:11.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:34:10 smithi153 ceph-mon[70044]: pgmap v19974: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:11.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:34:10 smithi083 ceph-mon[79930]: pgmap v19974: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:11.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:34:10 smithi083 ceph-mon[77242]: pgmap v19974: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:34:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:34:11] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T16:34:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:34:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:34:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:34:12.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:34:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:34:12.677Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:34:13.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:34:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:34:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:34:13.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:34:12 smithi153 ceph-mon[70044]: pgmap v19975: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:13.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:34:12 smithi083 ceph-mon[77242]: pgmap v19975: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:13.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:34:12 smithi083 ceph-mon[79930]: pgmap v19975: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:14.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:34:14 smithi153 ceph-mon[70044]: pgmap v19976: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:14.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:34:14 smithi083 ceph-mon[77242]: pgmap v19976: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:14.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:34:14 smithi083 ceph-mon[79930]: pgmap v19976: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:16.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:34:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:34:16.014Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:34:16.915 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:34:17.170 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:34:17.171 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 2m ago 11h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:34:17.171 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 48s ago 11h 64.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:34:17.172 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 2m ago 11h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:34:17.172 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 48s ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:34:17.173 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (11h) 48s ago 11h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:34:17.173 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11h) 2m ago 11h 1076M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:34:17.174 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11h) 2m ago 11h 2403M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:34:17.178 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (11h) 48s ago 11h 1077M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:34:17.179 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (11h) 2m ago 11h 1447M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:34:17.180 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 2m ago 11h 24.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:34:17.180 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 48s ago 11h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:34:17.181 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11h) 2m ago 11h 1751M 2062M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:34:17.181 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11h) 2m ago 11h 1736M 2062M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:34:17.181 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11h) 2m ago 11h 1348M 2062M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:34:17.182 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (11h) 2m ago 11h 1702M 2062M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:34:17.182 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (11h) 48s ago 11h 1491M 2491M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:34:17.183 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (11h) 48s ago 11h 1875M 2491M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:34:17.183 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (11h) 48s ago 11h 1676M 2491M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:34:17.184 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (10h) 48s ago 11h 1995M 2491M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:34:17.184 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (11h) 48s ago 11h 193M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:34:17.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:34:16 smithi083 ceph-mon[79930]: pgmap v19977: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:17.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:34:16 smithi083 ceph-mon[77242]: pgmap v19977: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:17.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:34:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:34:16.766Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:34:17.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:34:16 smithi153 ceph-mon[70044]: pgmap v19977: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:17.527 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:34:17.528 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:34:17.528 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:34:17.529 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:34:17.529 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:34:17.529 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:34:17.529 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:34:17.530 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:34:17.530 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:34:17.530 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:34:17.530 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:34:17.531 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:34:17.531 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:34:17.531 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:34:17.531 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:34:17.532 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:34:17.532 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:34:17.532 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:34:18.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:34:17 smithi153 ceph-mon[70044]: from='client.57360 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:34:18.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:34:17 smithi153 ceph-mon[70044]: from='client.57366 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:34:18.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:34:17 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1565606802' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:34:18.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:34:17 smithi083 ceph-mon[79930]: from='client.57360 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:34:18.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:34:17 smithi083 ceph-mon[79930]: from='client.57366 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:34:18.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:34:17 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1565606802' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:34:18.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:34:17 smithi083 ceph-mon[77242]: from='client.57360 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:34:18.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:34:17 smithi083 ceph-mon[77242]: from='client.57366 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:34:18.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:34:17 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1565606802' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:34:19.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:34:18 smithi153 ceph-mon[70044]: pgmap v19978: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:19.460 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:34:18 smithi083 ceph-mon[77242]: pgmap v19978: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:19.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:34:18 smithi083 ceph-mon[79930]: pgmap v19978: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:21.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:34:21 smithi153 ceph-mon[70044]: pgmap v19979: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:21.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:34:21 smithi083 ceph-mon[79930]: pgmap v19979: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:21.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:34:21 smithi083 ceph-mon[77242]: pgmap v19979: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:34:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:34:21] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T16:34:22.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:34:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:34:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:34:22.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:34:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:34:22.678Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:34:23.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:34:23 smithi153 ceph-mon[70044]: pgmap v19980: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:34:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:34:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:34:23.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:34:23 smithi083 ceph-mon[79930]: pgmap v19980: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:23.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:34:23 smithi083 ceph-mon[77242]: pgmap v19980: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:24.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:34:24 smithi153 ceph-mon[70044]: pgmap v19981: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:24.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:34:24 smithi083 ceph-mon[79930]: pgmap v19981: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:24.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:34:24 smithi083 ceph-mon[77242]: pgmap v19981: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:26.460 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:34:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:34:26.015Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:34:27.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:34:26 smithi083 ceph-mon[79930]: pgmap v19982: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:27.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:34:26 smithi083 ceph-mon[77242]: pgmap v19982: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:27.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:34:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:34:26.768Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:34:27.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:34:26 smithi153 ceph-mon[70044]: pgmap v19982: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:29.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:34:28 smithi153 ceph-mon[70044]: pgmap v19983: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:29.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:34:28 smithi083 ceph-mon[77242]: pgmap v19983: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:29.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:34:28 smithi083 ceph-mon[79930]: pgmap v19983: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:30.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:34:29 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:34:30.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:34:29 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:34:30.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:34:29 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:34:30.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:34:29 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:34:30.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:34:30 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:34:30.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:34:30 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:34:30.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:34:30 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:34:30.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:34:30 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:34:30.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:34:29 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:34:30.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:34:29 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:34:30.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:34:29 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:34:30.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:34:29 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:34:31.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:34:31 smithi153 ceph-mon[70044]: pgmap v19984: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:31.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:34:31 smithi083 ceph-mon[79930]: pgmap v19984: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:31.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:34:31 smithi083 ceph-mon[77242]: pgmap v19984: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:31.965 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:34:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:34:31] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T16:34:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:34:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:34:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:34:32.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:34:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:34:32.679Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:34:33.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:34:33 smithi153 ceph-mon[70044]: pgmap v19985: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:34:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:34:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:34:33.460 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:34:33 smithi083 ceph-mon[77242]: pgmap v19985: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:33.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:34:33 smithi083 ceph-mon[79930]: pgmap v19985: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:34.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:34:34 smithi153 ceph-mon[70044]: pgmap v19986: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:34.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:34:34 smithi083 ceph-mon[77242]: pgmap v19986: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:34.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:34:34 smithi083 ceph-mon[79930]: pgmap v19986: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:36.460 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:34:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:34:36.016Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:34:37.220 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:34:36 smithi083 ceph-mon[79930]: pgmap v19987: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:37.221 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:34:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:34:36.768Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:34:37.223 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:34:36 smithi083 ceph-mon[77242]: pgmap v19987: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:37.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:34:36 smithi153 ceph-mon[70044]: pgmap v19987: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:39.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:34:38 smithi153 ceph-mon[70044]: pgmap v19988: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:39.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:34:38 smithi083 ceph-mon[79930]: pgmap v19988: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:39.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:34:38 smithi083 ceph-mon[77242]: pgmap v19988: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:41.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:34:41 smithi153 ceph-mon[70044]: pgmap v19989: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:41.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:34:41 smithi083 ceph-mon[79930]: pgmap v19989: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:41.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:34:41 smithi083 ceph-mon[77242]: pgmap v19989: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:41.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:34:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:34:41] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T16:34:42.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:34:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:34:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:34:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:34:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:34:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:34:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:34:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:34:42.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:34:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:34:42.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:34:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:34:42.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:34:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:34:42.467 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:34:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:34:42.467 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:34:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:34:42.468 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:34:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:34:42.468 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:34:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:34:42.469 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:34:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:34:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:34:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:34:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:34:42.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:34:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:34:42.680Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:34:43.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:34:43 smithi153 ceph-mon[70044]: pgmap v19990: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:34:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:34:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:34:43.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:34:43 smithi083 ceph-mon[79930]: pgmap v19990: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:43.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:34:43 smithi083 ceph-mon[77242]: pgmap v19990: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:44.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:34:44 smithi153 ceph-mon[70044]: pgmap v19991: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:44.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:34:44 smithi083 ceph-mon[79930]: pgmap v19991: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:44.968 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:34:44 smithi083 ceph-mon[77242]: pgmap v19991: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:46.460 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:34:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:34:46.017Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:34:47.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:34:46 smithi083 ceph-mon[79930]: pgmap v19992: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:47.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:34:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:34:46.768Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:34:47.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:34:46 smithi083 ceph-mon[77242]: pgmap v19992: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:47.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:34:46 smithi153 ceph-mon[70044]: pgmap v19992: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:47.852 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:34:48.142 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:34:48.143 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 3m ago 11h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:34:48.143 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 79s ago 11h 64.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:34:48.143 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 3m ago 11h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:34:48.144 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 79s ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:34:48.144 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (11h) 79s ago 11h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:34:48.145 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11h) 3m ago 11h 1076M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:34:48.145 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11h) 3m ago 11h 2403M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:34:48.146 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (11h) 79s ago 11h 1077M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:34:48.146 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (11h) 3m ago 11h 1447M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:34:48.147 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 3m ago 11h 24.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:34:48.147 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 79s ago 11h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:34:48.148 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11h) 3m ago 11h 1751M 2062M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:34:48.148 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11h) 3m ago 11h 1736M 2062M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:34:48.149 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11h) 3m ago 11h 1348M 2062M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:34:48.149 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (11h) 3m ago 11h 1702M 2062M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:34:48.150 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (11h) 79s ago 11h 1491M 2491M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:34:48.150 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (11h) 79s ago 11h 1875M 2491M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:34:48.150 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (11h) 79s ago 11h 1676M 2491M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:34:48.152 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (11h) 79s ago 11h 1995M 2491M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:34:48.153 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (11h) 79s ago 11h 193M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:34:48.496 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:34:48.496 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:34:48.497 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:34:48.497 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:34:48.497 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:34:48.498 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:34:48.498 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:34:48.499 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:34:48.499 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:34:48.499 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:34:48.500 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:34:48.500 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:34:48.501 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:34:48.501 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:34:48.502 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:34:48.502 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:34:48.503 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:34:48.503 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:34:49.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:34:49 smithi153 ceph-mon[70044]: from='client.67309 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:34:49.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:34:49 smithi153 ceph-mon[70044]: pgmap v19993: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:49.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:34:49 smithi153 ceph-mon[70044]: from='client.67315 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:34:49.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:34:49 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1453436508' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:34:49.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:34:49 smithi083 ceph-mon[79930]: from='client.67309 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:34:49.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:34:49 smithi083 ceph-mon[79930]: pgmap v19993: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:49.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:34:49 smithi083 ceph-mon[79930]: from='client.67315 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:34:49.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:34:49 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1453436508' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:34:49.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:34:49 smithi083 ceph-mon[77242]: from='client.67309 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:34:49.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:34:49 smithi083 ceph-mon[77242]: pgmap v19993: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:49.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:34:49 smithi083 ceph-mon[77242]: from='client.67315 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:34:49.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:34:49 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1453436508' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:34:51.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:34:51 smithi153 ceph-mon[70044]: pgmap v19994: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:51.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:34:51 smithi083 ceph-mon[79930]: pgmap v19994: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:51.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:34:51 smithi083 ceph-mon[77242]: pgmap v19994: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:51.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:34:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:34:51] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T16:34:52.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:34:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:34:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:34:52.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:34:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:34:52.680Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:34:53.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:34:53 smithi153 ceph-mon[70044]: pgmap v19995: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:34:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:34:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:34:53.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:34:53 smithi083 ceph-mon[79930]: pgmap v19995: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:53.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:34:53 smithi083 ceph-mon[77242]: pgmap v19995: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:54.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:34:54 smithi153 ceph-mon[70044]: pgmap v19996: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:54.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:34:54 smithi083 ceph-mon[79930]: pgmap v19996: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:54.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:34:54 smithi083 ceph-mon[77242]: pgmap v19996: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:56.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:34:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:34:56.017Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:34:57.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:34:57 smithi083 ceph-mon[79930]: pgmap v19997: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:57.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:34:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:34:56.769Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:34:57.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:34:57 smithi083 ceph-mon[77242]: pgmap v19997: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:57.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:34:57 smithi153 ceph-mon[70044]: pgmap v19997: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:59.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:34:59 smithi153 ceph-mon[70044]: pgmap v19998: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:59.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:34:59 smithi083 ceph-mon[79930]: pgmap v19998: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:34:59.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:34:59 smithi083 ceph-mon[77242]: pgmap v19998: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:01.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:35:01 smithi153 ceph-mon[70044]: pgmap v19999: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:01.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:35:01 smithi083 ceph-mon[79930]: pgmap v19999: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:01.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:35:01 smithi083 ceph-mon[77242]: pgmap v19999: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:01.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:35:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:35:01] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T16:35:02.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:35:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:35:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:35:02.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:35:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:35:02.680Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:35:03.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:35:03 smithi153 ceph-mon[70044]: pgmap v20000: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:35:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:35:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:35:03.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:35:03 smithi083 ceph-mon[79930]: pgmap v20000: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:03.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:35:03 smithi083 ceph-mon[77242]: pgmap v20000: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:04.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:35:04 smithi153 ceph-mon[70044]: pgmap v20001: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:04.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:35:04 smithi083 ceph-mon[79930]: pgmap v20001: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:04.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:35:04 smithi083 ceph-mon[77242]: pgmap v20001: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:06.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:35:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:35:06.018Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:35:07.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:35:07 smithi083 ceph-mon[79930]: pgmap v20002: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:07.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:35:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:35:06.769Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:35:07.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:35:07 smithi083 ceph-mon[77242]: pgmap v20002: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:07.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:35:07 smithi153 ceph-mon[70044]: pgmap v20002: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:09.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:35:09 smithi153 ceph-mon[70044]: pgmap v20003: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:09.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:35:09 smithi083 ceph-mon[79930]: pgmap v20003: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:09.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:35:09 smithi083 ceph-mon[77242]: pgmap v20003: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:11.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:35:11 smithi153 ceph-mon[70044]: pgmap v20004: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:11.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:35:11 smithi083 ceph-mon[79930]: pgmap v20004: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:11.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:35:11 smithi083 ceph-mon[77242]: pgmap v20004: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:11.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:35:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:35:11] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T16:35:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:35:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:35:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:35:12.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:35:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:35:12.680Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:35:13.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:35:13 smithi153 ceph-mon[70044]: pgmap v20005: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:35:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:35:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:35:13.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:35:13 smithi083 ceph-mon[79930]: pgmap v20005: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:13.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:35:13 smithi083 ceph-mon[77242]: pgmap v20005: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:14.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:35:14 smithi153 ceph-mon[70044]: pgmap v20006: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:14.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:35:14 smithi083 ceph-mon[79930]: pgmap v20006: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:14.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:35:14 smithi083 ceph-mon[77242]: pgmap v20006: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:16.460 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:35:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:35:16.019Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:35:17.210 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:35:17 smithi083 ceph-mon[77242]: pgmap v20007: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:17.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:35:17 smithi083 ceph-mon[79930]: pgmap v20007: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:17.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:35:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:35:16.770Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:35:17.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:35:17 smithi153 ceph-mon[70044]: pgmap v20007: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:18.824 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:35:19.115 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:35:19.115 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 3m ago 11h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:35:19.116 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 110s ago 11h 64.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:35:19.116 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 3m ago 11h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:35:19.116 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 110s ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:35:19.116 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (11h) 110s ago 11h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:35:19.117 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11h) 3m ago 11h 1076M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:35:19.117 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11h) 3m ago 11h 2403M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:35:19.117 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (11h) 110s ago 11h 1077M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:35:19.117 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (11h) 3m ago 11h 1447M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:35:19.120 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 3m ago 11h 24.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:35:19.120 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 110s ago 11h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:35:19.121 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11h) 3m ago 11h 1751M 2062M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:35:19.121 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11h) 3m ago 11h 1736M 2062M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:35:19.121 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11h) 3m ago 11h 1348M 2062M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:35:19.121 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (11h) 3m ago 11h 1702M 2062M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:35:19.122 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (11h) 110s ago 11h 1491M 2491M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:35:19.122 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (11h) 110s ago 11h 1875M 2491M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:35:19.123 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (11h) 110s ago 11h 1676M 2491M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:35:19.123 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (11h) 110s ago 11h 1995M 2491M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:35:19.124 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (11h) 110s ago 11h 193M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:35:19.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:35:19 smithi153 ceph-mon[70044]: pgmap v20008: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:19.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:35:19 smithi083 ceph-mon[79930]: pgmap v20008: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:19.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:35:19 smithi083 ceph-mon[77242]: pgmap v20008: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:19.477 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:35:19.477 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:35:19.477 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:35:19.478 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:35:19.478 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:35:19.479 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:35:19.479 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:35:19.480 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:35:19.480 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:35:19.481 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:35:19.481 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:35:19.481 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:35:19.482 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:35:19.482 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:35:19.483 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:35:19.483 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:35:19.484 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:35:19.485 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:35:20.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:35:20 smithi153 ceph-mon[70044]: from='client.57396 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:35:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:35:20 smithi153 ceph-mon[70044]: from='client.57402 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:35:20.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:35:20 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2050165722' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:35:20.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:35:20 smithi083 ceph-mon[79930]: from='client.57396 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:35:20.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:35:20 smithi083 ceph-mon[79930]: from='client.57402 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:35:20.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:35:20 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2050165722' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:35:20.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:35:20 smithi083 ceph-mon[77242]: from='client.57396 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:35:20.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:35:20 smithi083 ceph-mon[77242]: from='client.57402 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:35:20.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:35:20 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2050165722' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:35:21.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:35:21 smithi153 ceph-mon[70044]: pgmap v20009: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:21.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:35:21 smithi083 ceph-mon[79930]: pgmap v20009: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:21.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:35:21 smithi083 ceph-mon[77242]: pgmap v20009: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:21.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:35:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:35:21] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T16:35:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:35:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:35:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:35:22.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:35:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:35:22.681Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:35:23.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:35:23 smithi153 ceph-mon[70044]: pgmap v20010: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:35:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:35:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:35:23.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:35:23 smithi083 ceph-mon[77242]: pgmap v20010: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:23.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:35:23 smithi083 ceph-mon[79930]: pgmap v20010: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:24.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:35:24 smithi153 ceph-mon[70044]: pgmap v20011: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:24.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:35:24 smithi083 ceph-mon[79930]: pgmap v20011: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:24.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:35:24 smithi083 ceph-mon[77242]: pgmap v20011: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:26.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:35:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:35:26.019Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:35:27.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:35:27 smithi083 ceph-mon[79930]: pgmap v20012: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:27.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:35:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:35:26.772Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:35:27.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:35:27 smithi083 ceph-mon[77242]: pgmap v20012: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:27.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:35:27 smithi153 ceph-mon[70044]: pgmap v20012: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:29.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:35:29 smithi153 ceph-mon[70044]: pgmap v20013: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:29.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:35:29 smithi083 ceph-mon[79930]: pgmap v20013: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:29.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:35:29 smithi083 ceph-mon[77242]: pgmap v20013: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:30.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:35:30 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:35:30.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:35:30 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:35:30.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:35:30 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:35:30.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:35:30 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:35:30.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:35:30 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:35:30.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:35:30 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:35:30.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:35:30 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:35:30.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:35:30 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:35:30.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:35:30 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:35:31.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:35:31 smithi153 ceph-mon[70044]: pgmap v20014: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:31.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:35:31 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:35:31.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:35:31 smithi083 ceph-mon[79930]: pgmap v20014: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:31.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:35:31 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:35:31.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:35:31 smithi083 ceph-mon[77242]: pgmap v20014: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:31.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:35:31 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:35:31.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:35:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:35:31] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T16:35:32.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:35:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:35:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:35:32.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:35:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:35:32.681Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:35:33.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:35:33 smithi153 ceph-mon[70044]: pgmap v20015: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:35:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:35:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:35:33.460 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:35:33 smithi083 ceph-mon[77242]: pgmap v20015: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:33.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:35:33 smithi083 ceph-mon[79930]: pgmap v20015: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:34.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:35:34 smithi153 ceph-mon[70044]: pgmap v20016: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:34.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:35:34 smithi083 ceph-mon[77242]: pgmap v20016: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:34.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:35:34 smithi083 ceph-mon[79930]: pgmap v20016: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:36.460 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:35:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:35:36.019Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:35:37.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:35:37 smithi083 ceph-mon[79930]: pgmap v20017: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:37.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:35:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:35:36.771Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:35:37.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:35:37 smithi083 ceph-mon[77242]: pgmap v20017: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:37.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:35:37 smithi153 ceph-mon[70044]: pgmap v20017: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:39.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:35:39 smithi153 ceph-mon[70044]: pgmap v20018: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:39.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:35:39 smithi083 ceph-mon[79930]: pgmap v20018: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:39.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:35:39 smithi083 ceph-mon[77242]: pgmap v20018: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:41.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:35:41 smithi153 ceph-mon[70044]: pgmap v20019: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:41.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:35:41 smithi083 ceph-mon[79930]: pgmap v20019: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:41.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:35:41 smithi083 ceph-mon[77242]: pgmap v20019: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:35:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:35:41] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T16:35:42.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:35:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:35:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:35:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:35:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:35:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:35:42.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:35:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:35:42.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:35:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:35:42.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:35:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:35:42.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:35:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:35:42.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:35:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:35:42.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:35:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:35:42.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:35:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:35:42.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:35:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:35:42.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:35:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:35:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:35:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:35:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:35:42.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:35:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:35:42.681Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:35:43.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:35:43 smithi153 ceph-mon[70044]: pgmap v20020: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:35:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:35:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:35:43.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:35:43 smithi083 ceph-mon[79930]: pgmap v20020: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:43.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:35:43 smithi083 ceph-mon[77242]: pgmap v20020: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:44.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:35:44 smithi153 ceph-mon[70044]: pgmap v20021: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:44.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:35:44 smithi083 ceph-mon[77242]: pgmap v20021: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:44.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:35:44 smithi083 ceph-mon[79930]: pgmap v20021: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:46.460 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:35:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:35:46.020Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:35:47.210 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:35:47 smithi083 ceph-mon[79930]: pgmap v20022: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:47.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:35:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:35:46.771Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:35:47.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:35:47 smithi083 ceph-mon[77242]: pgmap v20022: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:47.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:35:47 smithi153 ceph-mon[70044]: pgmap v20022: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:49.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:35:49 smithi153 ceph-mon[70044]: pgmap v20023: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:49.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:35:49 smithi083 ceph-mon[79930]: pgmap v20023: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:49.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:35:49 smithi083 ceph-mon[77242]: pgmap v20023: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:49.802 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:35:50.092 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:35:50.092 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 4m ago 11h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:35:50.092 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 2m ago 11h 64.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:35:50.093 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 4m ago 11h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:35:50.093 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 2m ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:35:50.093 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (11h) 2m ago 11h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:35:50.093 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11h) 4m ago 11h 1076M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:35:50.094 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11h) 4m ago 11h 2403M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:35:50.094 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (11h) 2m ago 11h 1077M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:35:50.094 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (11h) 4m ago 11h 1447M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:35:50.094 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 4m ago 11h 24.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:35:50.095 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 2m ago 11h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:35:50.095 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11h) 4m ago 11h 1751M 2062M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:35:50.095 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11h) 4m ago 11h 1736M 2062M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:35:50.095 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11h) 4m ago 11h 1348M 2062M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:35:50.095 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (11h) 4m ago 11h 1702M 2062M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:35:50.096 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (11h) 2m ago 11h 1491M 2491M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:35:50.096 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (11h) 2m ago 11h 1875M 2491M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:35:50.096 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (11h) 2m ago 11h 1676M 2491M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:35:50.096 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (11h) 2m ago 11h 1995M 2491M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:35:50.097 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (11h) 2m ago 11h 193M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:35:50.450 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:35:50.451 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:35:50.451 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:35:50.451 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:35:50.451 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:35:50.452 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:35:50.452 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:35:50.452 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:35:50.453 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:35:50.453 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:35:50.453 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:35:50.453 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:35:50.453 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:35:50.454 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:35:50.454 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:35:50.454 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:35:50.454 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:35:50.455 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:35:51.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:35:51 smithi153 ceph-mon[70044]: from='client.57414 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:35:51.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:35:51 smithi153 ceph-mon[70044]: pgmap v20024: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:51.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:35:51 smithi153 ceph-mon[70044]: from='client.57420 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:35:51.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:35:51 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1278516203' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:35:51.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:35:51 smithi083 ceph-mon[79930]: from='client.57414 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:35:51.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:35:51 smithi083 ceph-mon[79930]: pgmap v20024: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:51.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:35:51 smithi083 ceph-mon[79930]: from='client.57420 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:35:51.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:35:51 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1278516203' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:35:51.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:35:51 smithi083 ceph-mon[77242]: from='client.57414 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:35:51.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:35:51 smithi083 ceph-mon[77242]: pgmap v20024: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:51.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:35:51 smithi083 ceph-mon[77242]: from='client.57420 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:35:51.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:35:51 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1278516203' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:35:51.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:35:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:35:51] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T16:35:52.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:35:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:35:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:35:52.964 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:35:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:35:52.681Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:35:53.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:35:53 smithi153 ceph-mon[70044]: pgmap v20025: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:35:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:35:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:35:53.460 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:35:53 smithi083 ceph-mon[77242]: pgmap v20025: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:53.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:35:53 smithi083 ceph-mon[79930]: pgmap v20025: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:54.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:35:54 smithi153 ceph-mon[70044]: pgmap v20026: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:54.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:35:54 smithi083 ceph-mon[79930]: pgmap v20026: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:54.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:35:54 smithi083 ceph-mon[77242]: pgmap v20026: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:56.460 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:35:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:35:56.020Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:35:57.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:35:57 smithi083 ceph-mon[79930]: pgmap v20027: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:57.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:35:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:35:56.772Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:35:57.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:35:57 smithi083 ceph-mon[77242]: pgmap v20027: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:57.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:35:57 smithi153 ceph-mon[70044]: pgmap v20027: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:59.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:35:59 smithi153 ceph-mon[70044]: pgmap v20028: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:59.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:35:59 smithi083 ceph-mon[79930]: pgmap v20028: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:35:59.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:35:59 smithi083 ceph-mon[77242]: pgmap v20028: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:01.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:36:01 smithi153 ceph-mon[70044]: pgmap v20029: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:01.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:36:01 smithi083 ceph-mon[79930]: pgmap v20029: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:01.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:36:01 smithi083 ceph-mon[77242]: pgmap v20029: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:01.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:36:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:36:01] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T16:36:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:36:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:36:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:36:02.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:36:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:36:02.682Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:36:03.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:36:03 smithi153 ceph-mon[70044]: pgmap v20030: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:36:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:36:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:36:03.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:36:03 smithi083 ceph-mon[79930]: pgmap v20030: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:03.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:36:03 smithi083 ceph-mon[77242]: pgmap v20030: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:04.816 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:36:04 smithi153 ceph-mon[70044]: pgmap v20031: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:04.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:36:04 smithi083 ceph-mon[77242]: pgmap v20031: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:04.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:36:04 smithi083 ceph-mon[79930]: pgmap v20031: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:06.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:36:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:36:06.020Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:36:07.023 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:36:07 smithi083 ceph-mon[77242]: pgmap v20032: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:07.024 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:36:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:36:06.772Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:36:07.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:36:07 smithi153 ceph-mon[70044]: pgmap v20032: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:07.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:36:07 smithi083 ceph-mon[79930]: pgmap v20032: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:09.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:36:09 smithi153 ceph-mon[70044]: pgmap v20033: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:09.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:36:09 smithi083 ceph-mon[79930]: pgmap v20033: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:09.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:36:09 smithi083 ceph-mon[77242]: pgmap v20033: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:11.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:36:11 smithi153 ceph-mon[70044]: pgmap v20034: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:11.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:36:11 smithi083 ceph-mon[79930]: pgmap v20034: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:11.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:36:11 smithi083 ceph-mon[77242]: pgmap v20034: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:36:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:36:11] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T16:36:12.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:36:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:36:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:36:12.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:36:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:36:12.682Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:36:13.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:36:13 smithi153 ceph-mon[70044]: pgmap v20035: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:36:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:36:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:36:13.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:36:13 smithi083 ceph-mon[77242]: pgmap v20035: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:13.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:36:13 smithi083 ceph-mon[79930]: pgmap v20035: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:14.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:36:14 smithi083 ceph-mon[77242]: pgmap v20036: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:14.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:36:14 smithi083 ceph-mon[79930]: pgmap v20036: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:15.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:36:14 smithi153 ceph-mon[70044]: pgmap v20036: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:16.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:36:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:36:16.021Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:36:17.026 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:36:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:36:16.774Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:36:17.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:36:17 smithi153 ceph-mon[70044]: pgmap v20037: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:17.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:36:17 smithi083 ceph-mon[79930]: pgmap v20037: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:17.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:36:17 smithi083 ceph-mon[77242]: pgmap v20037: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:19.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:36:19 smithi153 ceph-mon[70044]: pgmap v20038: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:19.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:36:19 smithi083 ceph-mon[79930]: pgmap v20038: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:19.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:36:19 smithi083 ceph-mon[77242]: pgmap v20038: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:20.772 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:36:21.060 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:36:21.061 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 4m ago 11h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:36:21.061 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 2m ago 11h 64.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:36:21.062 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 4m ago 11h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:36:21.062 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 2m ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:36:21.063 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (11h) 2m ago 11h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:36:21.063 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11h) 4m ago 11h 1076M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:36:21.064 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11h) 4m ago 11h 2403M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:36:21.064 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (11h) 2m ago 11h 1077M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:36:21.064 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (11h) 4m ago 11h 1447M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:36:21.065 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 4m ago 11h 24.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:36:21.066 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 2m ago 11h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:36:21.066 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11h) 4m ago 11h 1751M 2062M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:36:21.067 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11h) 4m ago 11h 1736M 2062M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:36:21.067 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11h) 4m ago 11h 1348M 2062M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:36:21.067 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (11h) 4m ago 11h 1702M 2062M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:36:21.068 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (11h) 2m ago 11h 1491M 2491M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:36:21.068 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (11h) 2m ago 11h 1875M 2491M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:36:21.069 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (11h) 2m ago 11h 1676M 2491M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:36:21.069 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (11h) 2m ago 11h 1995M 2491M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:36:21.071 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (11h) 2m ago 11h 193M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:36:21.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:36:21 smithi153 ceph-mon[70044]: pgmap v20039: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:21.415 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:36:21.415 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:36:21.416 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:36:21.416 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:36:21.417 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:36:21.417 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:36:21.417 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:36:21.417 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:36:21.418 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:36:21.418 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:36:21.418 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:36:21.419 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:36:21.419 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:36:21.420 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:36:21.421 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:36:21.421 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:36:21.422 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:36:21.422 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:36:21.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:36:21 smithi083 ceph-mon[79930]: pgmap v20039: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:21.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:36:21 smithi083 ceph-mon[77242]: pgmap v20039: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:36:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:36:21] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T16:36:22.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:36:22 smithi153 ceph-mon[70044]: from='client.57432 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:36:22.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:36:22 smithi153 ceph-mon[70044]: from='client.57438 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:36:22.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:36:22 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/678546214' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:36:22.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:36:22 smithi083 ceph-mon[77242]: from='client.57432 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:36:22.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:36:22 smithi083 ceph-mon[77242]: from='client.57438 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:36:22.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:36:22 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/678546214' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:36:22.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:36:22 smithi083 ceph-mon[79930]: from='client.57432 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:36:22.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:36:22 smithi083 ceph-mon[79930]: from='client.57438 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:36:22.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:36:22 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/678546214' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:36:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:36:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:36:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:36:22.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:36:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:36:22.683Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:36:23.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:36:23 smithi153 ceph-mon[70044]: pgmap v20040: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:36:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:36:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:36:23.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:36:23 smithi083 ceph-mon[79930]: pgmap v20040: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:23.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:36:23 smithi083 ceph-mon[77242]: pgmap v20040: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:24.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:36:24 smithi083 ceph-mon[79930]: pgmap v20041: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:24.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:36:24 smithi083 ceph-mon[77242]: pgmap v20041: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:25.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:36:24 smithi153 ceph-mon[70044]: pgmap v20041: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:26.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:36:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:36:26.022Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:36:27.028 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:36:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:36:26.774Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:36:27.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:36:27 smithi153 ceph-mon[70044]: pgmap v20042: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:27.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:36:27 smithi083 ceph-mon[79930]: pgmap v20042: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:27.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:36:27 smithi083 ceph-mon[77242]: pgmap v20042: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:29.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:36:29 smithi153 ceph-mon[70044]: pgmap v20043: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:29.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:36:29 smithi083 ceph-mon[79930]: pgmap v20043: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:29.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:36:29 smithi083 ceph-mon[77242]: pgmap v20043: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:31.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:36:31 smithi153 ceph-mon[70044]: pgmap v20044: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:31.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:36:31 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:36:31.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:36:31 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:36:31.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:36:31 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:36:31.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:36:31 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:36:31.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:36:31 smithi083 ceph-mon[79930]: pgmap v20044: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:31.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:36:31 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:36:31.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:36:31 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:36:31.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:36:31 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:36:31.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:36:31 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:36:31.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:36:31 smithi083 ceph-mon[77242]: pgmap v20044: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:31.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:36:31 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:36:31.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:36:31 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:36:31.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:36:31 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:36:31.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:36:31 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:36:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:36:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:36:31] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T16:36:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:36:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:36:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:36:32.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:36:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:36:32.683Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:36:33.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:36:33 smithi153 ceph-mon[70044]: pgmap v20045: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:33.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:36:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:36:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:36:33.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:36:33 smithi083 ceph-mon[77242]: pgmap v20045: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:33.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:36:33 smithi083 ceph-mon[79930]: pgmap v20045: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:34.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:36:34 smithi083 ceph-mon[79930]: pgmap v20046: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:34.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:36:34 smithi083 ceph-mon[77242]: pgmap v20046: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:35.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:36:34 smithi153 ceph-mon[70044]: pgmap v20046: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:36.460 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:36:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:36:36.024Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:36:37.031 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:36:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:36:36.775Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:36:37.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:36:37 smithi153 ceph-mon[70044]: pgmap v20047: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:37.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:36:37 smithi083 ceph-mon[79930]: pgmap v20047: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:37.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:36:37 smithi083 ceph-mon[77242]: pgmap v20047: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:39.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:36:39 smithi153 ceph-mon[70044]: pgmap v20048: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:39.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:36:39 smithi083 ceph-mon[79930]: pgmap v20048: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:39.467 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:36:39 smithi083 ceph-mon[77242]: pgmap v20048: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:41.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:36:41 smithi153 ceph-mon[70044]: pgmap v20049: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:41.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:36:41 smithi083 ceph-mon[79930]: pgmap v20049: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:41.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:36:41 smithi083 ceph-mon[77242]: pgmap v20049: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:36:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:36:41] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T16:36:42.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:36:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:36:42.312 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:36:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:36:42.312 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:36:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:36:42.313 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:36:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:36:42.460 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:36:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:36:42.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:36:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:36:42.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:36:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:36:42.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:36:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:36:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:36:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:36:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:36:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:36:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:36:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:36:42.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:36:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:36:42.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:36:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:36:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:36:42.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:36:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:36:42.683Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:36:43.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:36:43 smithi153 ceph-mon[70044]: pgmap v20050: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:36:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:36:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:36:43.460 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:36:43 smithi083 ceph-mon[77242]: pgmap v20050: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:43.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:36:43 smithi083 ceph-mon[79930]: pgmap v20050: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:44.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:36:44 smithi083 ceph-mon[77242]: pgmap v20051: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:44.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:36:44 smithi083 ceph-mon[79930]: pgmap v20051: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:45.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:36:44 smithi153 ceph-mon[70044]: pgmap v20051: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:46.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:36:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:36:46.024Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:36:47.034 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:36:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:36:46.776Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:36:47.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:36:47 smithi153 ceph-mon[70044]: pgmap v20052: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:47.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:36:47 smithi083 ceph-mon[79930]: pgmap v20052: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:47.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:36:47 smithi083 ceph-mon[77242]: pgmap v20052: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:49.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:36:49 smithi153 ceph-mon[70044]: pgmap v20053: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:49.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:36:49 smithi083 ceph-mon[79930]: pgmap v20053: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:49.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:36:49 smithi083 ceph-mon[77242]: pgmap v20053: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:51.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:36:51 smithi153 ceph-mon[70044]: pgmap v20054: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:51.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:36:51 smithi083 ceph-mon[79930]: pgmap v20054: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:51.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:36:51 smithi083 ceph-mon[77242]: pgmap v20054: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:51.742 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:36:51.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:36:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:36:51] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T16:36:52.031 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:36:52.032 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 5m ago 11h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:36:52.032 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 3m ago 11h 64.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:36:52.033 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 5m ago 11h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:36:52.033 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 3m ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:36:52.034 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (11h) 3m ago 11h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:36:52.034 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11h) 5m ago 11h 1076M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:36:52.034 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11h) 5m ago 11h 2403M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:36:52.035 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (11h) 3m ago 11h 1077M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:36:52.035 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (11h) 5m ago 11h 1447M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:36:52.035 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 5m ago 11h 24.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:36:52.035 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 3m ago 11h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:36:52.036 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11h) 5m ago 11h 1751M 2062M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:36:52.036 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11h) 5m ago 11h 1736M 2062M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:36:52.036 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11h) 5m ago 11h 1348M 2062M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:36:52.037 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (11h) 5m ago 11h 1702M 2062M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:36:52.037 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (11h) 3m ago 11h 1491M 2491M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:36:52.037 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (11h) 3m ago 11h 1875M 2491M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:36:52.038 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (11h) 3m ago 11h 1676M 2491M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:36:52.038 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (11h) 3m ago 11h 1995M 2491M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:36:52.038 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (11h) 3m ago 11h 193M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:36:52.387 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:36:52.387 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:36:52.387 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:36:52.388 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:36:52.388 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:36:52.388 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:36:52.389 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:36:52.389 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:36:52.389 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:36:52.390 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:36:52.390 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:36:52.390 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:36:52.390 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:36:52.391 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:36:52.391 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:36:52.391 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:36:52.391 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:36:52.392 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:36:52.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:36:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:36:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:36:52.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:36:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:36:52.683Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:36:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:36:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:36:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:36:53.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:36:53 smithi153 ceph-mon[70044]: from='client.57450 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:36:53.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:36:53 smithi153 ceph-mon[70044]: pgmap v20055: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:53.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:36:53 smithi153 ceph-mon[70044]: from='client.57456 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:36:53.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:36:53 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/527323480' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:36:53.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:36:53 smithi083 ceph-mon[77242]: from='client.57450 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:36:53.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:36:53 smithi083 ceph-mon[77242]: pgmap v20055: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:53.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:36:53 smithi083 ceph-mon[77242]: from='client.57456 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:36:53.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:36:53 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/527323480' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:36:53.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:36:53 smithi083 ceph-mon[79930]: from='client.57450 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:36:53.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:36:53 smithi083 ceph-mon[79930]: pgmap v20055: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:53.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:36:53 smithi083 ceph-mon[79930]: from='client.57456 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:36:53.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:36:53 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/527323480' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:36:54.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:36:54 smithi083 ceph-mon[79930]: pgmap v20056: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:54.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:36:54 smithi083 ceph-mon[77242]: pgmap v20056: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:55.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:36:54 smithi153 ceph-mon[70044]: pgmap v20056: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:56.460 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:36:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:36:56.025Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:36:57.038 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:36:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:36:56.776Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:36:57.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:36:57 smithi153 ceph-mon[70044]: pgmap v20057: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:57.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:36:57 smithi083 ceph-mon[77242]: pgmap v20057: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:57.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:36:57 smithi083 ceph-mon[79930]: pgmap v20057: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:59.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:36:59 smithi153 ceph-mon[70044]: pgmap v20058: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:59.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:36:59 smithi083 ceph-mon[79930]: pgmap v20058: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:36:59.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:36:59 smithi083 ceph-mon[77242]: pgmap v20058: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:01.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:37:01 smithi153 ceph-mon[70044]: pgmap v20059: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:01.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:37:01 smithi083 ceph-mon[79930]: pgmap v20059: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:01.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:37:01 smithi083 ceph-mon[77242]: pgmap v20059: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:37:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:37:01] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T16:37:02.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:37:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:37:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:37:02.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:37:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:37:02.684Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:37:03.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:37:03 smithi153 ceph-mon[70044]: pgmap v20060: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:37:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:37:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:37:03.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:37:03 smithi083 ceph-mon[79930]: pgmap v20060: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:03.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:37:03 smithi083 ceph-mon[77242]: pgmap v20060: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:04.960 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:37:04 smithi083 ceph-mon[77242]: pgmap v20061: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:04.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:37:04 smithi083 ceph-mon[79930]: pgmap v20061: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:05.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:37:04 smithi153 ceph-mon[70044]: pgmap v20061: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:06.460 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:37:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:37:06.026Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:37:07.042 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:37:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:37:06.777Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:37:07.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:37:07 smithi153 ceph-mon[70044]: pgmap v20062: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:07.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:37:07 smithi083 ceph-mon[77242]: pgmap v20062: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:07.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:37:07 smithi083 ceph-mon[79930]: pgmap v20062: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:09.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:37:09 smithi153 ceph-mon[70044]: pgmap v20063: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:09.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:37:09 smithi083 ceph-mon[79930]: pgmap v20063: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:09.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:37:09 smithi083 ceph-mon[77242]: pgmap v20063: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:11.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:37:11 smithi153 ceph-mon[70044]: pgmap v20064: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:11.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:37:11 smithi083 ceph-mon[79930]: pgmap v20064: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:11.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:37:11 smithi083 ceph-mon[77242]: pgmap v20064: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:11.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:37:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:37:11] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T16:37:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:37:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:37:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:37:12.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:37:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:37:12.685Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:37:13.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:37:13 smithi153 ceph-mon[70044]: pgmap v20065: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:37:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:37:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:37:13.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:37:13 smithi083 ceph-mon[79930]: pgmap v20065: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:13.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:37:13 smithi083 ceph-mon[77242]: pgmap v20065: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:14.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:37:14 smithi083 ceph-mon[79930]: pgmap v20066: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:14.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:37:14 smithi083 ceph-mon[77242]: pgmap v20066: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:15.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:37:14 smithi153 ceph-mon[70044]: pgmap v20066: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:16.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:37:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:37:16.027Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:37:17.043 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:37:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:37:16.777Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:37:17.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:37:17 smithi153 ceph-mon[70044]: pgmap v20067: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:17.460 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:37:17 smithi083 ceph-mon[77242]: pgmap v20067: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:17.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:37:17 smithi083 ceph-mon[79930]: pgmap v20067: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:19.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:37:19 smithi153 ceph-mon[70044]: pgmap v20068: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:19.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:37:19 smithi083 ceph-mon[79930]: pgmap v20068: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:19.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:37:19 smithi083 ceph-mon[77242]: pgmap v20068: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:21.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:37:21 smithi153 ceph-mon[70044]: pgmap v20069: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:21.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:37:21 smithi083 ceph-mon[79930]: pgmap v20069: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:21.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:37:21 smithi083 ceph-mon[77242]: pgmap v20069: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:37:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:37:21] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T16:37:22.712 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:37:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:37:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:37:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:37:22.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:37:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:37:22.684Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:37:23.001 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:37:23.002 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 6m ago 11h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:37:23.002 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 3m ago 11h 64.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:37:23.002 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 6m ago 11h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:37:23.003 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 3m ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:37:23.003 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (11h) 3m ago 11h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:37:23.003 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11h) 6m ago 11h 1076M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:37:23.003 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11h) 6m ago 11h 2403M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:37:23.003 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (11h) 3m ago 11h 1077M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:37:23.004 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (11h) 6m ago 11h 1447M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:37:23.004 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 6m ago 11h 24.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:37:23.004 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 3m ago 11h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:37:23.004 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11h) 6m ago 11h 1751M 2062M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:37:23.005 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11h) 6m ago 11h 1736M 2062M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:37:23.005 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11h) 6m ago 11h 1348M 2062M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:37:23.005 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (11h) 6m ago 11h 1702M 2062M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:37:23.005 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (11h) 3m ago 11h 1491M 2491M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:37:23.006 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (11h) 3m ago 11h 1875M 2491M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:37:23.006 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (11h) 3m ago 11h 1676M 2491M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:37:23.006 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (11h) 3m ago 11h 1995M 2491M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:37:23.006 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (11h) 3m ago 11h 193M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:37:23.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:37:23 smithi153 ceph-mon[70044]: pgmap v20070: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:37:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:37:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:37:23.363 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:37:23.363 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:37:23.363 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:37:23.364 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:37:23.364 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:37:23.364 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:37:23.364 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:37:23.365 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:37:23.365 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:37:23.365 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:37:23.365 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:37:23.366 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:37:23.366 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:37:23.367 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:37:23.367 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:37:23.368 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:37:23.368 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:37:23.369 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:37:23.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:37:23 smithi083 ceph-mon[79930]: pgmap v20070: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:23.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:37:23 smithi083 ceph-mon[77242]: pgmap v20070: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:24.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:37:24 smithi153 ceph-mon[70044]: from='client.67399 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:37:24.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:37:24 smithi153 ceph-mon[70044]: from='client.67405 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:37:24.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:37:24 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3617744525' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:37:24.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:37:24 smithi083 ceph-mon[79930]: from='client.67399 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:37:24.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:37:24 smithi083 ceph-mon[79930]: from='client.67405 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:37:24.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:37:24 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3617744525' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:37:24.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:37:24 smithi083 ceph-mon[77242]: from='client.67399 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:37:24.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:37:24 smithi083 ceph-mon[77242]: from='client.67405 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:37:24.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:37:24 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3617744525' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:37:25.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:37:25 smithi153 ceph-mon[70044]: pgmap v20071: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:25.460 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:37:25 smithi083 ceph-mon[77242]: pgmap v20071: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:25.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:37:25 smithi083 ceph-mon[79930]: pgmap v20071: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:26.460 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:37:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:37:26.026Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:37:27.058 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:37:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:37:26.778Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:37:27.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:37:27 smithi083 ceph-mon[79930]: pgmap v20072: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:27.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:37:27 smithi083 ceph-mon[77242]: pgmap v20072: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:27.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:37:27 smithi153 ceph-mon[70044]: pgmap v20072: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:29.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:37:29 smithi083 ceph-mon[79930]: pgmap v20073: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:29.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:37:29 smithi083 ceph-mon[77242]: pgmap v20073: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:29.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:37:29 smithi153 ceph-mon[70044]: pgmap v20073: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:31.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:37:31 smithi083 ceph-mon[79930]: pgmap v20074: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:31.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:37:31 smithi083 ceph-mon[77242]: pgmap v20074: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:31.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:37:31 smithi153 ceph-mon[70044]: pgmap v20074: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:31.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:37:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:37:31] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T16:37:32.315 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:37:32 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:37:32.316 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:37:32 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:37:32.316 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:37:32 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:37:32.317 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:37:32 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:37:32.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:37:32 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:37:32.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:37:32 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:37:32.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:37:32 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:37:32.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:37:32 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:37:32.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:37:32 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:37:32.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:37:32 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:37:32.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:37:32 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:37:32.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:37:32 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:37:32.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:37:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:37:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:37:32.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:37:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:37:32.685Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:37:33.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:37:33 smithi153 ceph-mon[70044]: pgmap v20075: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:37:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:37:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:37:33.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:37:33 smithi083 ceph-mon[79930]: pgmap v20075: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:33.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:37:33 smithi083 ceph-mon[77242]: pgmap v20075: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:34.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:37:34 smithi083 ceph-mon[79930]: pgmap v20076: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:34.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:37:34 smithi083 ceph-mon[77242]: pgmap v20076: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:35.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:37:34 smithi153 ceph-mon[70044]: pgmap v20076: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:36.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:37:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:37:36.028Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:37:37.050 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:37:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:37:36.778Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:37:37.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:37:37 smithi153 ceph-mon[70044]: pgmap v20077: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:37.460 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:37:37 smithi083 ceph-mon[77242]: pgmap v20077: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:37.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:37:37 smithi083 ceph-mon[79930]: pgmap v20077: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:39.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:37:39 smithi153 ceph-mon[70044]: pgmap v20078: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:39.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:37:39 smithi083 ceph-mon[77242]: pgmap v20078: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:39.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:37:39 smithi083 ceph-mon[79930]: pgmap v20078: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:41.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:37:41 smithi083 ceph-mon[77242]: pgmap v20079: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:41.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:37:41 smithi083 ceph-mon[79930]: pgmap v20079: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:41.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:37:41 smithi153 ceph-mon[70044]: pgmap v20079: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:41.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:37:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:37:41] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T16:37:42.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:37:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:37:42.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:37:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:37:42.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:37:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:37:42.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:37:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:37:42.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:37:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:37:42.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:37:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:37:42.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:37:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:37:42.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:37:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:37:42.556 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:37:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:37:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:37:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:37:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:37:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:37:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:37:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:37:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:37:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:37:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:37:42.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:37:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:37:42.686Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:37:43.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:37:43 smithi153 ceph-mon[70044]: pgmap v20080: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:37:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:37:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:37:43.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:37:43 smithi083 ceph-mon[79930]: pgmap v20080: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:43.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:37:43 smithi083 ceph-mon[77242]: pgmap v20080: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:44.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:37:44 smithi083 ceph-mon[79930]: pgmap v20081: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:44.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:37:44 smithi083 ceph-mon[77242]: pgmap v20081: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:45.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:37:44 smithi153 ceph-mon[70044]: pgmap v20081: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:46.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:37:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:37:46.029Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:37:47.053 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:37:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:37:46.779Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:37:47.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:37:47 smithi153 ceph-mon[70044]: pgmap v20082: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:47.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:37:47 smithi083 ceph-mon[77242]: pgmap v20082: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:47.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:37:47 smithi083 ceph-mon[79930]: pgmap v20082: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:49.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:37:49 smithi153 ceph-mon[70044]: pgmap v20083: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:49.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:37:49 smithi083 ceph-mon[79930]: pgmap v20083: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:49.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:37:49 smithi083 ceph-mon[77242]: pgmap v20083: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:51.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:37:51 smithi083 ceph-mon[79930]: pgmap v20084: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:51.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:37:51 smithi083 ceph-mon[77242]: pgmap v20084: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:51.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:37:51 smithi153 ceph-mon[70044]: pgmap v20084: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:51.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:37:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:37:51] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T16:37:52.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:37:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:37:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:37:52.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:37:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:37:52.686Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:37:53.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:37:53 smithi153 ceph-mon[70044]: pgmap v20085: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:37:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:37:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:37:53.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:37:53 smithi083 ceph-mon[79930]: pgmap v20085: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:53.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:37:53 smithi083 ceph-mon[77242]: pgmap v20085: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:53.694 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:37:53.982 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:37:53.982 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 6m ago 11h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:37:53.983 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 4m ago 11h 64.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:37:53.983 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 6m ago 11h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:37:53.983 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 4m ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:37:53.983 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (11h) 4m ago 11h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:37:53.984 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11h) 6m ago 11h 1076M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:37:53.984 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11h) 6m ago 11h 2403M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:37:53.984 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (11h) 4m ago 11h 1077M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:37:53.985 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (11h) 6m ago 11h 1447M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:37:53.985 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 6m ago 11h 24.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:37:53.985 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 4m ago 11h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:37:53.985 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11h) 6m ago 11h 1751M 2062M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:37:53.986 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11h) 6m ago 11h 1736M 2062M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:37:53.986 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11h) 6m ago 11h 1348M 2062M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:37:53.986 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (11h) 6m ago 11h 1702M 2062M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:37:53.986 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (11h) 4m ago 11h 1491M 2491M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:37:53.987 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (11h) 4m ago 11h 1875M 2491M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:37:53.987 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (11h) 4m ago 11h 1676M 2491M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:37:53.987 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (11h) 4m ago 11h 1995M 2491M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:37:53.987 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (11h) 4m ago 11h 193M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:37:54.342 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:37:54.342 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:37:54.342 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:37:54.343 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:37:54.343 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:37:54.343 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:37:54.343 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:37:54.344 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:37:54.344 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:37:54.344 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:37:54.344 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:37:54.345 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:37:54.345 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:37:54.345 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:37:54.345 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:37:54.346 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:37:54.346 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:37:54.346 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:37:54.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:37:54 smithi083 ceph-mon[79930]: from='client.67417 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:37:54.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:37:54 smithi083 ceph-mon[79930]: from='client.57492 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:37:54.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:37:54 smithi083 ceph-mon[79930]: pgmap v20086: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:54.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:37:54 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3973171540' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:37:54.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:37:54 smithi083 ceph-mon[77242]: from='client.67417 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:37:54.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:37:54 smithi083 ceph-mon[77242]: from='client.57492 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:37:54.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:37:54 smithi083 ceph-mon[77242]: pgmap v20086: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:54.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:37:54 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3973171540' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:37:55.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:37:54 smithi153 ceph-mon[70044]: from='client.67417 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:37:55.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:37:54 smithi153 ceph-mon[70044]: from='client.57492 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:37:55.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:37:54 smithi153 ceph-mon[70044]: pgmap v20086: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:55.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:37:54 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3973171540' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:37:56.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:37:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:37:56.028Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:37:57.056 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:37:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:37:56.780Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:37:57.460 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:37:57 smithi083 ceph-mon[77242]: pgmap v20087: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:57.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:37:57 smithi083 ceph-mon[79930]: pgmap v20087: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:57.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:37:57 smithi153 ceph-mon[70044]: pgmap v20087: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:59.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:37:59 smithi083 ceph-mon[77242]: pgmap v20088: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:59.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:37:59 smithi083 ceph-mon[79930]: pgmap v20088: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:37:59.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:37:59 smithi153 ceph-mon[70044]: pgmap v20088: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:01.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:38:01 smithi083 ceph-mon[79930]: pgmap v20089: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:01.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:38:01 smithi083 ceph-mon[77242]: pgmap v20089: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:01.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:38:01 smithi153 ceph-mon[70044]: pgmap v20089: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:01.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:38:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:38:01] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T16:38:02.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:38:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:38:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:38:02.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:38:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:38:02.687Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:38:03.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:38:03 smithi153 ceph-mon[70044]: pgmap v20090: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:38:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:38:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:38:03.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:38:03 smithi083 ceph-mon[79930]: pgmap v20090: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:03.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:38:03 smithi083 ceph-mon[77242]: pgmap v20090: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:04.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:38:04 smithi083 ceph-mon[79930]: pgmap v20091: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:04.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:38:04 smithi083 ceph-mon[77242]: pgmap v20091: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:05.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:38:04 smithi153 ceph-mon[70044]: pgmap v20091: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:06.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:38:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:38:06.029Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:38:07.059 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:38:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:38:06.781Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:38:07.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:38:07 smithi083 ceph-mon[77242]: pgmap v20092: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:07.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:38:07 smithi083 ceph-mon[79930]: pgmap v20092: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:07.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:38:07 smithi153 ceph-mon[70044]: pgmap v20092: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:09.460 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:38:09 smithi083 ceph-mon[77242]: pgmap v20093: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:09.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:38:09 smithi083 ceph-mon[79930]: pgmap v20093: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:09.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:38:09 smithi153 ceph-mon[70044]: pgmap v20093: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:11.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:38:11 smithi083 ceph-mon[79930]: pgmap v20094: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:11.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:38:11 smithi083 ceph-mon[77242]: pgmap v20094: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:11.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:38:11 smithi153 ceph-mon[70044]: pgmap v20094: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:11.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:38:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:38:11] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T16:38:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:38:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:38:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:38:12.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:38:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:38:12.687Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:38:13.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:38:13 smithi153 ceph-mon[70044]: pgmap v20095: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:38:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:38:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:38:13.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:38:13 smithi083 ceph-mon[79930]: pgmap v20095: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:13.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:38:13 smithi083 ceph-mon[77242]: pgmap v20095: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:14.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:38:14 smithi083 ceph-mon[77242]: pgmap v20096: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:14.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:38:14 smithi083 ceph-mon[79930]: pgmap v20096: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:15.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:38:14 smithi153 ceph-mon[70044]: pgmap v20096: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:16.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:38:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:38:16.030Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:38:17.063 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:38:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:38:16.781Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:38:17.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:38:17 smithi083 ceph-mon[77242]: pgmap v20097: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:17.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:38:17 smithi083 ceph-mon[79930]: pgmap v20097: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:17.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:38:17 smithi153 ceph-mon[70044]: pgmap v20097: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:19.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:38:19 smithi083 ceph-mon[79930]: pgmap v20098: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:19.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:38:19 smithi083 ceph-mon[77242]: pgmap v20098: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:19.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:38:19 smithi153 ceph-mon[70044]: pgmap v20098: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:21.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:38:21 smithi083 ceph-mon[79930]: pgmap v20099: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:21.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:38:21 smithi083 ceph-mon[77242]: pgmap v20099: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:21.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:38:21 smithi153 ceph-mon[70044]: pgmap v20099: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:21.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:38:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:38:21] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T16:38:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:38:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:38:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:38:22.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:38:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:38:22.689Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:38:23.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:38:23 smithi153 ceph-mon[70044]: pgmap v20100: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:38:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:38:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:38:23.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:38:23 smithi083 ceph-mon[79930]: pgmap v20100: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:23.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:38:23 smithi083 ceph-mon[77242]: pgmap v20100: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:24.671 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:38:24.960 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:38:24 smithi083 ceph-mon[77242]: pgmap v20101: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:24.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:38:24 smithi083 ceph-mon[79930]: pgmap v20101: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:24.962 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:38:24.962 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 7m ago 11h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:38:24.962 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 4m ago 11h 64.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:38:24.963 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 7m ago 11h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:38:24.963 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 4m ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:38:24.963 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (11h) 4m ago 11h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:38:24.963 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11h) 7m ago 11h 1076M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:38:24.963 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11h) 7m ago 11h 2403M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:38:24.964 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (11h) 4m ago 11h 1077M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:38:24.964 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (11h) 7m ago 11h 1447M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:38:24.964 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 7m ago 11h 24.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:38:24.964 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 4m ago 11h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:38:24.965 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11h) 7m ago 11h 1751M 2062M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:38:24.965 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11h) 7m ago 11h 1736M 2062M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:38:24.965 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11h) 7m ago 11h 1348M 2062M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:38:24.965 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (11h) 7m ago 11h 1702M 2062M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:38:24.966 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (11h) 4m ago 11h 1491M 2491M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:38:24.966 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (11h) 4m ago 11h 1875M 2491M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:38:24.966 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (11h) 4m ago 11h 1676M 2491M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:38:24.966 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (11h) 4m ago 11h 1995M 2491M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:38:24.967 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (11h) 4m ago 11h 193M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:38:25.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:38:24 smithi153 ceph-mon[70044]: pgmap v20101: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:25.318 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:38:25.319 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:38:25.319 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:38:25.320 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:38:25.320 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:38:25.321 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:38:25.321 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:38:25.321 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:38:25.322 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:38:25.322 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:38:25.323 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:38:25.323 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:38:25.324 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:38:25.324 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:38:25.325 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:38:25.325 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:38:25.326 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:38:25.326 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:38:25.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:38:25 smithi083 ceph-mon[79930]: from='client.67435 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:38:25.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:38:25 smithi083 ceph-mon[79930]: from='client.67441 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:38:25.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:38:25 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3243338442' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:38:25.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:38:25 smithi083 ceph-mon[77242]: from='client.67435 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:38:25.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:38:25 smithi083 ceph-mon[77242]: from='client.67441 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:38:25.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:38:25 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3243338442' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:38:26.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:38:25 smithi153 ceph-mon[70044]: from='client.67435 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:38:26.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:38:25 smithi153 ceph-mon[70044]: from='client.67441 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:38:26.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:38:25 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3243338442' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:38:26.460 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:38:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:38:26.029Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:38:26.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:38:26 smithi083 ceph-mon[79930]: pgmap v20102: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:26.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:38:26 smithi083 ceph-mon[77242]: pgmap v20102: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:26.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:38:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:38:26.781Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:38:27.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:38:26 smithi153 ceph-mon[70044]: pgmap v20102: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:29.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:38:29 smithi083 ceph-mon[79930]: pgmap v20103: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:29.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:38:29 smithi083 ceph-mon[77242]: pgmap v20103: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:29.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:38:29 smithi153 ceph-mon[70044]: pgmap v20103: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:31.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:38:31 smithi083 ceph-mon[79930]: pgmap v20104: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:31.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:38:31 smithi083 ceph-mon[77242]: pgmap v20104: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:31.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:38:31 smithi153 ceph-mon[70044]: pgmap v20104: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:31.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:38:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:38:31] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T16:38:32.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:38:32 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:38:32.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:38:32 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:38:32.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:38:32 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:38:32.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:38:32 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:38:32.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:38:32 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:38:32.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:38:32 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:38:32.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:38:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:38:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:38:32.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:38:32 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:38:32.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:38:32 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:38:32.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:38:32 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:38:32.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:38:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:38:32.688Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:38:33.302 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:38:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:38:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:38:33.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:38:33 smithi153 ceph-mon[70044]: pgmap v20105: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:33.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:38:33 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:38:33.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:38:33 smithi083 ceph-mon[79930]: pgmap v20105: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:33.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:38:33 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:38:33.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:38:33 smithi083 ceph-mon[77242]: pgmap v20105: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:33.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:38:33 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:38:34.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:38:34 smithi083 ceph-mon[79930]: pgmap v20106: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:34.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:38:34 smithi083 ceph-mon[77242]: pgmap v20106: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:35.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:38:34 smithi153 ceph-mon[70044]: pgmap v20106: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:36.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:38:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:38:36.031Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:38:37.069 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:38:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:38:36.782Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:38:37.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:38:37 smithi083 ceph-mon[79930]: pgmap v20107: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:37.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:38:37 smithi083 ceph-mon[77242]: pgmap v20107: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:37.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:38:37 smithi153 ceph-mon[70044]: pgmap v20107: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:39.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:38:39 smithi083 ceph-mon[79930]: pgmap v20108: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:39.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:38:39 smithi083 ceph-mon[77242]: pgmap v20108: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:39.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:38:39 smithi153 ceph-mon[70044]: pgmap v20108: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:41.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:38:41 smithi083 ceph-mon[79930]: pgmap v20109: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:41.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:38:41 smithi083 ceph-mon[77242]: pgmap v20109: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:38:41 smithi153 ceph-mon[70044]: pgmap v20109: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:41.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:38:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:38:41] "GET /metrics HTTP/1.1" 200 225246 "" "Prometheus/2.33.4" 2022-09-29T16:38:42.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:38:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:38:42.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:38:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:38:42.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:38:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:38:42.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:38:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:38:42.467 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:38:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:38:42.467 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:38:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:38:42.468 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:38:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:38:42.468 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:38:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:38:42.556 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:38:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:38:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:38:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:38:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:38:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:38:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:38:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:38:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:38:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:38:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:38:42.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:38:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:38:42.689Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:38:43.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:38:43 smithi153 ceph-mon[70044]: pgmap v20110: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:38:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:38:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:38:43.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:38:43 smithi083 ceph-mon[79930]: pgmap v20110: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:43.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:38:43 smithi083 ceph-mon[77242]: pgmap v20110: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:44.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:38:44 smithi083 ceph-mon[79930]: pgmap v20111: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:44.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:38:44 smithi083 ceph-mon[77242]: pgmap v20111: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:45.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:38:44 smithi153 ceph-mon[70044]: pgmap v20111: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:46.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:38:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:38:46.032Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:38:47.072 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:38:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:38:46.783Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:38:47.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:38:47 smithi083 ceph-mon[79930]: pgmap v20112: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:47.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:38:47 smithi083 ceph-mon[77242]: pgmap v20112: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:47.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:38:47 smithi153 ceph-mon[70044]: pgmap v20112: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:49.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:38:49 smithi083 ceph-mon[79930]: pgmap v20113: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:49.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:38:49 smithi083 ceph-mon[77242]: pgmap v20113: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:49.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:38:49 smithi153 ceph-mon[70044]: pgmap v20113: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:51.467 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:38:51 smithi083 ceph-mon[79930]: pgmap v20114: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:51.467 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:38:51 smithi083 ceph-mon[77242]: pgmap v20114: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:51.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:38:51 smithi153 ceph-mon[70044]: pgmap v20114: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:38:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:38:51] "GET /metrics HTTP/1.1" 200 225246 "" "Prometheus/2.33.4" 2022-09-29T16:38:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:38:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:38:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:38:52.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:38:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:38:52.689Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:38:53.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:38:53 smithi153 ceph-mon[70044]: pgmap v20115: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:38:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:38:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:38:53.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:38:53 smithi083 ceph-mon[79930]: pgmap v20115: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:53.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:38:53 smithi083 ceph-mon[77242]: pgmap v20115: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:54.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:38:54 smithi083 ceph-mon[77242]: pgmap v20116: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:54.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:38:54 smithi083 ceph-mon[79930]: pgmap v20116: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:55.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:38:54 smithi153 ceph-mon[70044]: pgmap v20116: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:55.646 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:38:55.934 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:38:55.934 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 7m ago 11h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:38:55.935 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 5m ago 11h 64.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:38:55.935 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 7m ago 11h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:38:55.935 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 5m ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:38:55.936 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (11h) 5m ago 11h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:38:55.936 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11h) 7m ago 11h 1076M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:38:55.936 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11h) 7m ago 11h 2403M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:38:55.936 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (11h) 5m ago 11h 1077M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:38:55.936 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (11h) 7m ago 11h 1447M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:38:55.937 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 7m ago 11h 24.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:38:55.937 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 5m ago 11h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:38:55.937 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11h) 7m ago 11h 1751M 2062M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:38:55.938 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11h) 7m ago 11h 1736M 2062M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:38:55.938 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11h) 7m ago 11h 1348M 2062M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:38:55.938 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (11h) 7m ago 11h 1702M 2062M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:38:55.938 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (11h) 5m ago 11h 1491M 2491M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:38:55.939 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (11h) 5m ago 11h 1875M 2491M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:38:55.939 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (11h) 5m ago 11h 1676M 2491M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:38:55.939 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (11h) 5m ago 11h 1995M 2491M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:38:55.939 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (11h) 5m ago 11h 193M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:38:56.294 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:38:56.295 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:38:56.295 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:38:56.295 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:38:56.295 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:38:56.295 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:38:56.296 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:38:56.296 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:38:56.296 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:38:56.296 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:38:56.297 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:38:56.297 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:38:56.297 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:38:56.297 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:38:56.297 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:38:56.298 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:38:56.298 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:38:56.298 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:38:56.460 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:38:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:38:56.031Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:38:57.074 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:38:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:38:56.783Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:38:57.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:38:57 smithi083 ceph-mon[79930]: from='client.57522 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:38:57.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:38:57 smithi083 ceph-mon[79930]: from='client.57528 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:38:57.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:38:57 smithi083 ceph-mon[79930]: pgmap v20117: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:57.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:38:57 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3781969323' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:38:57.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:38:57 smithi083 ceph-mon[77242]: from='client.57522 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:38:57.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:38:57 smithi083 ceph-mon[77242]: from='client.57528 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:38:57.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:38:57 smithi083 ceph-mon[77242]: pgmap v20117: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:57.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:38:57 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3781969323' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:38:57.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:38:57 smithi153 ceph-mon[70044]: from='client.57522 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:38:57.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:38:57 smithi153 ceph-mon[70044]: from='client.57528 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:38:57.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:38:57 smithi153 ceph-mon[70044]: pgmap v20117: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:57.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:38:57 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3781969323' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:38:59.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:38:59 smithi083 ceph-mon[79930]: pgmap v20118: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:59.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:38:59 smithi083 ceph-mon[77242]: pgmap v20118: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:38:59.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:38:59 smithi153 ceph-mon[70044]: pgmap v20118: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:01.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:39:01 smithi083 ceph-mon[79930]: pgmap v20119: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:01.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:39:01 smithi083 ceph-mon[77242]: pgmap v20119: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:01.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:39:01 smithi153 ceph-mon[70044]: pgmap v20119: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:01.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:39:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:39:01] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T16:39:02.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:39:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:39:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:39:02.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:39:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:39:02.690Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:39:03.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:39:03 smithi153 ceph-mon[70044]: pgmap v20120: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:39:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:39:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:39:03.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:39:03 smithi083 ceph-mon[79930]: pgmap v20120: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:03.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:39:03 smithi083 ceph-mon[77242]: pgmap v20120: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:04.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:39:04 smithi083 ceph-mon[79930]: pgmap v20121: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:04.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:39:04 smithi083 ceph-mon[77242]: pgmap v20121: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:05.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:39:04 smithi153 ceph-mon[70044]: pgmap v20121: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:06.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:39:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:39:06.033Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:39:07.079 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:39:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:39:06.785Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:39:07.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:39:07 smithi083 ceph-mon[79930]: pgmap v20122: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:07.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:39:07 smithi083 ceph-mon[77242]: pgmap v20122: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:07.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:39:07 smithi153 ceph-mon[70044]: pgmap v20122: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:09.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:39:09 smithi083 ceph-mon[79930]: pgmap v20123: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:09.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:39:09 smithi083 ceph-mon[77242]: pgmap v20123: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:09.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:39:09 smithi153 ceph-mon[70044]: pgmap v20123: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:11.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:39:11 smithi083 ceph-mon[79930]: pgmap v20124: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:11.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:39:11 smithi083 ceph-mon[77242]: pgmap v20124: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:11.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:39:11 smithi153 ceph-mon[70044]: pgmap v20124: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:39:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:39:11] "GET /metrics HTTP/1.1" 200 225246 "" "Prometheus/2.33.4" 2022-09-29T16:39:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:39:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:39:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:39:12.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:39:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:39:12.690Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:39:13.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:39:13 smithi153 ceph-mon[70044]: pgmap v20125: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:39:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:39:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:39:13.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:39:13 smithi083 ceph-mon[79930]: pgmap v20125: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:13.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:39:13 smithi083 ceph-mon[77242]: pgmap v20125: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:14.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:39:14 smithi083 ceph-mon[79930]: pgmap v20126: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:14.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:39:14 smithi083 ceph-mon[77242]: pgmap v20126: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:15.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:39:14 smithi153 ceph-mon[70044]: pgmap v20126: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:16.460 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:39:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:39:16.034Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:39:17.081 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:39:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:39:16.785Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:39:17.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:39:17 smithi083 ceph-mon[77242]: pgmap v20127: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:17.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:39:17 smithi083 ceph-mon[79930]: pgmap v20127: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:17.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:39:17 smithi153 ceph-mon[70044]: pgmap v20127: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:19.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:39:19 smithi083 ceph-mon[79930]: pgmap v20128: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:19.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:39:19 smithi083 ceph-mon[77242]: pgmap v20128: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:19.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:39:19 smithi153 ceph-mon[70044]: pgmap v20128: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:21.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:39:21 smithi083 ceph-mon[79930]: pgmap v20129: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:21.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:39:21 smithi083 ceph-mon[77242]: pgmap v20129: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:21.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:39:21 smithi153 ceph-mon[70044]: pgmap v20129: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:21.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:39:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:39:21] "GET /metrics HTTP/1.1" 200 225246 "" "Prometheus/2.33.4" 2022-09-29T16:39:22.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:39:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:39:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:39:22.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:39:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:39:22.690Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:39:23.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:39:23 smithi153 ceph-mon[70044]: pgmap v20130: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:39:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:39:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:39:23.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:39:23 smithi083 ceph-mon[79930]: pgmap v20130: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:23.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:39:23 smithi083 ceph-mon[77242]: pgmap v20130: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:24.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:39:24 smithi083 ceph-mon[79930]: pgmap v20131: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:24.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:39:24 smithi083 ceph-mon[77242]: pgmap v20131: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:25.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:39:24 smithi153 ceph-mon[70044]: pgmap v20131: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:26.460 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:39:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:39:26.034Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:39:26.622 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:39:26.909 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:39:26.910 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 8m ago 11h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:39:26.910 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 5m ago 11h 64.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:39:26.911 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 8m ago 11h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:39:26.911 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 5m ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:39:26.912 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (11h) 5m ago 11h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:39:26.912 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11h) 8m ago 11h 1076M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:39:26.913 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11h) 8m ago 11h 2403M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:39:26.913 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (11h) 5m ago 11h 1077M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:39:26.914 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (11h) 8m ago 11h 1447M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:39:26.914 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 8m ago 11h 24.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:39:26.915 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 5m ago 11h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:39:26.915 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11h) 8m ago 11h 1751M 2062M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:39:26.916 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11h) 8m ago 11h 1736M 2062M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:39:26.916 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11h) 8m ago 11h 1348M 2062M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:39:26.917 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (11h) 8m ago 11h 1702M 2062M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:39:26.917 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (11h) 5m ago 11h 1491M 2491M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:39:26.918 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (11h) 5m ago 11h 1875M 2491M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:39:26.918 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (11h) 5m ago 11h 1676M 2491M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:39:26.919 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (11h) 5m ago 11h 1995M 2491M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:39:26.919 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (11h) 5m ago 11h 193M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:39:27.084 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:39:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:39:26.785Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:39:27.085 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:39:27 smithi083 ceph-mon[77242]: pgmap v20132: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:27.268 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:39:27.269 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:39:27.269 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:39:27.269 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:39:27.269 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:39:27.270 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:39:27.270 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:39:27.270 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:39:27.270 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:39:27.271 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:39:27.271 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:39:27.271 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:39:27.271 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:39:27.272 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:39:27.272 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:39:27.272 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:39:27.272 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:39:27.273 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:39:27.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:39:27 smithi083 ceph-mon[79930]: pgmap v20132: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:27.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:39:27 smithi153 ceph-mon[70044]: pgmap v20132: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:28.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:39:28 smithi083 ceph-mon[77242]: from='client.57540 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:39:28.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:39:28 smithi083 ceph-mon[77242]: from='client.57546 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:39:28.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:39:28 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2725997434' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:39:28.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:39:28 smithi083 ceph-mon[79930]: from='client.57540 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:39:28.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:39:28 smithi083 ceph-mon[79930]: from='client.57546 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:39:28.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:39:28 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2725997434' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:39:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:39:28 smithi153 ceph-mon[70044]: from='client.57540 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:39:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:39:28 smithi153 ceph-mon[70044]: from='client.57546 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:39:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:39:28 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2725997434' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:39:29.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:39:29 smithi083 ceph-mon[77242]: pgmap v20133: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:29.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:39:29 smithi083 ceph-mon[79930]: pgmap v20133: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:29.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:39:29 smithi153 ceph-mon[70044]: pgmap v20133: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:31.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:39:31 smithi083 ceph-mon[77242]: pgmap v20134: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:31.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:39:31 smithi083 ceph-mon[79930]: pgmap v20134: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:31.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:39:31 smithi153 ceph-mon[70044]: pgmap v20134: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:31.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:39:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:39:31] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T16:39:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:39:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:39:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:39:32.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:39:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:39:32.690Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:39:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:39:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:39:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:39:33.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:39:33 smithi153 ceph-mon[70044]: pgmap v20135: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:33.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:39:33 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:39:33.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:39:33 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:39:33.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:39:33 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:39:33.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:39:33 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:39:33.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:39:33 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:39:33.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:39:33 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:39:33.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:39:33 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:39:33.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:39:33 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:39:33.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:39:33 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:39:33.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:39:33 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:39:33.312 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:39:33 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:39:33.312 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:39:33 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:39:33.313 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:39:33 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:39:33.313 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:39:33 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:39:33.314 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:39:33 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:39:33.314 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:39:33 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:39:33.314 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:39:33 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:39:33.315 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:39:33 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:39:33.315 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:39:33 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:39:33.315 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:39:33 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:39:33.315 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:39:33 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:39:33.316 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:39:33 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:39:33.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:39:33 smithi083 ceph-mon[79930]: pgmap v20135: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:33.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:39:33 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:39:33.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:39:33 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:39:33.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:39:33 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:39:33.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:39:33 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:39:33.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:39:33 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:39:33.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:39:33 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:39:33.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:39:33 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:39:33.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:39:33 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:39:33.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:39:33 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:39:33.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:39:33 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:39:33.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:39:33 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:39:33.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:39:33 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:39:33.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:39:33 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:39:33.467 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:39:33 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:39:33.468 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:39:33 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:39:33.468 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:39:33 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:39:33.468 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:39:33 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:39:33.469 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:39:33 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:39:33.469 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:39:33 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:39:33.470 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:39:33 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:39:33.470 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:39:33 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:39:33.470 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:39:33 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:39:33.470 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:39:33 smithi083 ceph-mon[77242]: pgmap v20135: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:33.471 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:39:33 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:39:33.471 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:39:33 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:39:33.471 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:39:33 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:39:33.471 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:39:33 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:39:33.472 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:39:33 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:39:33.472 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:39:33 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:39:33.472 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:39:33 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:39:33.472 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:39:33 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:39:33.473 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:39:33 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:39:33.473 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:39:33 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:39:33.473 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:39:33 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:39:33.473 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:39:33 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:39:33.474 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:39:33 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:39:33.474 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:39:33 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:39:33.474 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:39:33 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:39:33.475 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:39:33 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:39:33.475 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:39:33 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:39:33.475 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:39:33 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:39:33.475 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:39:33 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:39:33.476 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:39:33 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:39:33.476 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:39:33 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:39:33.476 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:39:33 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:39:34.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:39:34 smithi083 ceph-mon[79930]: Adjusting osd_memory_target on smithi153 to 2486M 2022-09-29T16:39:34.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:39:34 smithi083 ceph-mon[79930]: Adjusting osd_memory_target on smithi083 to 2049M 2022-09-29T16:39:34.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:39:34 smithi083 ceph-mon[77242]: Adjusting osd_memory_target on smithi153 to 2486M 2022-09-29T16:39:34.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:39:34 smithi083 ceph-mon[77242]: Adjusting osd_memory_target on smithi083 to 2049M 2022-09-29T16:39:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:39:34 smithi153 ceph-mon[70044]: Adjusting osd_memory_target on smithi153 to 2486M 2022-09-29T16:39:34.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:39:34 smithi153 ceph-mon[70044]: Adjusting osd_memory_target on smithi083 to 2049M 2022-09-29T16:39:35.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:39:35 smithi083 ceph-mon[79930]: pgmap v20136: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:35.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:39:35 smithi083 ceph-mon[77242]: pgmap v20136: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:35.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:39:35 smithi153 ceph-mon[70044]: pgmap v20136: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:36.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:39:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:39:36.035Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:39:37.108 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:39:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:39:36.786Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:39:37.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:39:37 smithi083 ceph-mon[79930]: pgmap v20137: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:37.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:39:37 smithi083 ceph-mon[77242]: pgmap v20137: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:37.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:39:37 smithi153 ceph-mon[70044]: pgmap v20137: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:39.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:39:39 smithi083 ceph-mon[79930]: pgmap v20138: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:39.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:39:39 smithi083 ceph-mon[77242]: pgmap v20138: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:39.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:39:39 smithi153 ceph-mon[70044]: pgmap v20138: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:41.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:39:41 smithi083 ceph-mon[79930]: pgmap v20139: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:41.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:39:41 smithi083 ceph-mon[77242]: pgmap v20139: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:41.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:39:41 smithi153 ceph-mon[70044]: pgmap v20139: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:41.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:39:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:39:41] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T16:39:42.460 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:39:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:39:42.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:39:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:39:42.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:39:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:39:42.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:39:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:39:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:39:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:39:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:39:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:39:42.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:39:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:39:42.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:39:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:39:42.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:39:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:39:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:39:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:39:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:39:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:39:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:39:42.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:39:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:39:42.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:39:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:39:42.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:39:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:39:42.692Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:39:43.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:39:43 smithi153 ceph-mon[70044]: pgmap v20140: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:39:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:39:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:39:43.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:39:43 smithi083 ceph-mon[79930]: pgmap v20140: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:43.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:39:43 smithi083 ceph-mon[77242]: pgmap v20140: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:44.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:39:44 smithi083 ceph-mon[79930]: pgmap v20141: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:44.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:39:44 smithi083 ceph-mon[77242]: pgmap v20141: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:45.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:39:44 smithi153 ceph-mon[70044]: pgmap v20141: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:46.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:39:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:39:46.035Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:39:47.089 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:39:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:39:46.787Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:39:47.460 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:39:47 smithi083 ceph-mon[77242]: pgmap v20142: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:47.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:39:47 smithi083 ceph-mon[79930]: pgmap v20142: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:47.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:39:47 smithi153 ceph-mon[70044]: pgmap v20142: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:49.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:39:49 smithi083 ceph-mon[77242]: pgmap v20143: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:49.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:39:49 smithi083 ceph-mon[79930]: pgmap v20143: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:49.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:39:49 smithi153 ceph-mon[70044]: pgmap v20143: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:51.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:39:51 smithi083 ceph-mon[77242]: pgmap v20144: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:51.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:39:51 smithi083 ceph-mon[79930]: pgmap v20144: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:51.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:39:51 smithi153 ceph-mon[70044]: pgmap v20144: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:51.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:39:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:39:51] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T16:39:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:39:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:39:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:39:52.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:39:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:39:52.693Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:39:53.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:39:53 smithi153 ceph-mon[70044]: pgmap v20145: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:39:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:39:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:39:53.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:39:53 smithi083 ceph-mon[79930]: pgmap v20145: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:53.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:39:53 smithi083 ceph-mon[77242]: pgmap v20145: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:54.960 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:39:54 smithi083 ceph-mon[77242]: pgmap v20146: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:54.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:39:54 smithi083 ceph-mon[79930]: pgmap v20146: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:55.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:39:54 smithi153 ceph-mon[70044]: pgmap v20146: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:56.460 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:39:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:39:56.036Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:39:57.093 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:39:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:39:56.787Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:39:57.460 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:39:57 smithi083 ceph-mon[77242]: pgmap v20147: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:57.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:39:57 smithi083 ceph-mon[79930]: pgmap v20147: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:57.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:39:57 smithi153 ceph-mon[70044]: pgmap v20147: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:57.595 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:39:57.880 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:39:57.880 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 8m ago 11h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:39:57.881 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 6m ago 11h 64.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:39:57.881 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 8m ago 11h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:39:57.881 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 6m ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:39:57.881 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (11h) 6m ago 11h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:39:57.882 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11h) 8m ago 11h 1076M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:39:57.882 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11h) 8m ago 11h 2403M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:39:57.882 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (11h) 6m ago 11h 1077M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:39:57.882 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (11h) 8m ago 11h 1447M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:39:57.883 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 8m ago 11h 24.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:39:57.883 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 6m ago 11h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:39:57.883 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11h) 8m ago 11h 1751M 2049M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:39:57.883 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11h) 8m ago 11h 1736M 2049M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:39:57.883 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11h) 8m ago 11h 1348M 2049M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:39:57.884 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (11h) 8m ago 11h 1702M 2049M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:39:57.884 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (11h) 6m ago 11h 1491M 2486M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:39:57.884 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (11h) 6m ago 11h 1875M 2486M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:39:57.884 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (11h) 6m ago 11h 1676M 2486M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:39:57.885 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (11h) 6m ago 11h 1995M 2486M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:39:57.885 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (11h) 6m ago 11h 193M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:39:58.239 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:39:58.239 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:39:58.239 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:39:58.240 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:39:58.240 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:39:58.240 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:39:58.240 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:39:58.240 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:39:58.241 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:39:58.241 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:39:58.241 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:39:58.241 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:39:58.242 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:39:58.242 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:39:58.242 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:39:58.242 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:39:58.242 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:39:58.243 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:39:59.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:39:59 smithi083 ceph-mon[79930]: from='client.67489 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:39:59.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:39:59 smithi083 ceph-mon[79930]: from='client.67495 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:39:59.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:39:59 smithi083 ceph-mon[79930]: pgmap v20148: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:59.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:39:59 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2535879452' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:39:59.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:39:59 smithi083 ceph-mon[77242]: from='client.67489 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:39:59.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:39:59 smithi083 ceph-mon[77242]: from='client.67495 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:39:59.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:39:59 smithi083 ceph-mon[77242]: pgmap v20148: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:59.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:39:59 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2535879452' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:39:59.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:39:59 smithi153 ceph-mon[70044]: from='client.67489 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:39:59.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:39:59 smithi153 ceph-mon[70044]: from='client.67495 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:39:59.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:39:59 smithi153 ceph-mon[70044]: pgmap v20148: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:39:59.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:39:59 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2535879452' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:40:00.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:40:00 smithi083 ceph-mon[79930]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T16:40:00.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:40:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[77219]: 2022-09-29T16:39:59.999+0000 7f160f665700 -1 log_channel(cluster) log [ERR] : overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T16:40:00.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:40:00 smithi083 ceph-mon[77242]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T16:40:00.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:40:00 smithi153 ceph-mon[70044]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T16:40:01.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:40:01 smithi083 ceph-mon[79930]: pgmap v20149: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:01.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:40:01 smithi083 ceph-mon[77242]: pgmap v20149: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:01.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:40:01 smithi153 ceph-mon[70044]: pgmap v20149: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:40:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:40:01] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T16:40:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:40:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:40:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:40:02.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:40:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:40:02.695Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:40:03.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:40:03 smithi153 ceph-mon[70044]: pgmap v20150: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:03.348 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:40:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:40:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:40:03.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:40:03 smithi083 ceph-mon[79930]: pgmap v20150: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:03.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:40:03 smithi083 ceph-mon[77242]: pgmap v20150: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:04.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:40:04 smithi083 ceph-mon[79930]: pgmap v20151: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:04.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:40:04 smithi083 ceph-mon[77242]: pgmap v20151: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:05.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:40:04 smithi153 ceph-mon[70044]: pgmap v20151: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:06.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:40:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:40:06.036Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:40:07.097 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:40:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:40:06.788Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:40:07.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:40:07 smithi083 ceph-mon[77242]: pgmap v20152: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:07.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:40:07 smithi083 ceph-mon[79930]: pgmap v20152: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:07.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:40:07 smithi153 ceph-mon[70044]: pgmap v20152: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:09.460 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:40:09 smithi083 ceph-mon[77242]: pgmap v20153: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:09.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:40:09 smithi083 ceph-mon[79930]: pgmap v20153: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:09.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:40:09 smithi153 ceph-mon[70044]: pgmap v20153: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:11.460 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:40:11 smithi083 ceph-mon[77242]: pgmap v20154: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:11.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:40:11 smithi083 ceph-mon[79930]: pgmap v20154: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:11.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:40:11 smithi153 ceph-mon[70044]: pgmap v20154: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:11.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:40:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:40:11] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T16:40:12.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:40:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:40:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:40:12.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:40:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:40:12.694Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:40:13.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:40:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:40:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:40:13.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:40:13 smithi153 ceph-mon[70044]: pgmap v20155: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:13.460 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:40:13 smithi083 ceph-mon[77242]: pgmap v20155: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:13.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:40:13 smithi083 ceph-mon[79930]: pgmap v20155: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:14.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:40:14 smithi083 ceph-mon[79930]: pgmap v20156: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:14.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:40:14 smithi083 ceph-mon[77242]: pgmap v20156: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:15.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:40:14 smithi153 ceph-mon[70044]: pgmap v20156: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:16.460 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:40:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:40:16.037Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:40:17.099 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:40:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:40:16.789Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:40:17.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:40:17 smithi083 ceph-mon[77242]: pgmap v20157: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:17.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:40:17 smithi083 ceph-mon[79930]: pgmap v20157: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:17.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:40:17 smithi153 ceph-mon[70044]: pgmap v20157: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:19.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:40:19 smithi083 ceph-mon[79930]: pgmap v20158: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:19.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:40:19 smithi083 ceph-mon[77242]: pgmap v20158: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:19.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:40:19 smithi153 ceph-mon[70044]: pgmap v20158: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:21.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:40:21 smithi083 ceph-mon[79930]: pgmap v20159: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:21.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:40:21 smithi083 ceph-mon[77242]: pgmap v20159: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:21.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:40:21 smithi153 ceph-mon[70044]: pgmap v20159: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:21.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:40:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:40:21] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T16:40:22.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:40:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:40:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:40:22.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:40:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:40:22.695Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:40:23.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:40:23 smithi153 ceph-mon[70044]: pgmap v20160: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:40:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:40:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:40:23.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:40:23 smithi083 ceph-mon[79930]: pgmap v20160: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:23.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:40:23 smithi083 ceph-mon[77242]: pgmap v20160: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:24.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:40:24 smithi083 ceph-mon[79930]: pgmap v20161: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:24.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:40:24 smithi083 ceph-mon[77242]: pgmap v20161: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:25.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:40:24 smithi153 ceph-mon[70044]: pgmap v20161: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:26.460 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:40:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:40:26.038Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:40:27.103 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:40:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:40:26.790Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:40:27.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:40:27 smithi083 ceph-mon[79930]: pgmap v20162: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:27.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:40:27 smithi083 ceph-mon[77242]: pgmap v20162: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:27.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:40:27 smithi153 ceph-mon[70044]: pgmap v20162: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:28.567 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:40:28.856 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:40:28.856 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 9m ago 11h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:40:28.856 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 7m ago 11h 64.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:40:28.856 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 9m ago 11h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:40:28.857 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 7m ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:40:28.857 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (11h) 7m ago 11h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:40:28.857 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11h) 9m ago 11h 1076M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:40:28.857 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11h) 9m ago 11h 2403M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:40:28.858 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (11h) 7m ago 11h 1077M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:40:28.858 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (11h) 9m ago 11h 1447M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:40:28.858 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 9m ago 11h 24.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:40:28.858 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 7m ago 11h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:40:28.859 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11h) 9m ago 11h 1751M 2049M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:40:28.859 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11h) 9m ago 11h 1736M 2049M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:40:28.859 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11h) 9m ago 11h 1348M 2049M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:40:28.859 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (11h) 9m ago 11h 1702M 2049M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:40:28.860 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (11h) 7m ago 11h 1491M 2486M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:40:28.860 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (11h) 7m ago 11h 1875M 2486M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:40:28.860 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (11h) 7m ago 11h 1676M 2486M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:40:28.860 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (11h) 7m ago 11h 1995M 2486M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:40:28.861 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (11h) 7m ago 11h 193M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:40:29.211 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:40:29.211 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:40:29.211 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:40:29.212 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:40:29.212 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:40:29.212 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:40:29.212 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:40:29.213 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:40:29.213 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:40:29.213 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:40:29.213 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:40:29.213 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:40:29.214 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:40:29.214 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:40:29.214 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:40:29.214 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:40:29.215 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:40:29.215 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:40:29.460 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:40:29 smithi083 ceph-mon[77242]: pgmap v20163: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:29.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:40:29 smithi083 ceph-mon[79930]: pgmap v20163: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:29.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:40:29 smithi153 ceph-mon[70044]: pgmap v20163: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:30.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:40:30 smithi083 ceph-mon[79930]: from='client.67507 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:40:30.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:40:30 smithi083 ceph-mon[79930]: from='client.67513 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:40:30.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:40:30 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3977450242' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:40:30.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:40:30 smithi083 ceph-mon[77242]: from='client.67507 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:40:30.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:40:30 smithi083 ceph-mon[77242]: from='client.67513 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:40:30.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:40:30 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3977450242' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:40:30.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:40:30 smithi153 ceph-mon[70044]: from='client.67507 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:40:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:40:30 smithi153 ceph-mon[70044]: from='client.67513 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:40:30.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:40:30 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3977450242' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:40:31.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:40:31 smithi083 ceph-mon[77242]: pgmap v20164: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:31.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:40:31 smithi083 ceph-mon[79930]: pgmap v20164: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:31.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:40:31 smithi153 ceph-mon[70044]: pgmap v20164: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:31.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:40:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:40:31] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T16:40:32.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:40:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:40:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:40:32.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:40:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:40:32.696Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:40:33.306 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:40:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:40:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:40:33.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:40:33 smithi153 ceph-mon[70044]: pgmap v20165: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:33.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:40:33 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:40:33.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:40:33 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:40:33.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:40:33 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:40:33.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:40:33 smithi083 ceph-mon[79930]: pgmap v20165: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:33.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:40:33 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:40:33.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:40:33 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:40:33.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:40:33 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:40:33.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:40:33 smithi083 ceph-mon[77242]: pgmap v20165: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:33.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:40:33 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:40:33.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:40:33 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:40:33.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:40:33 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:40:34.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:40:34 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:40:34.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:40:34 smithi083 ceph-mon[79930]: pgmap v20166: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:34.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:40:34 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:40:34.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:40:34 smithi083 ceph-mon[77242]: pgmap v20166: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:35.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:40:34 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:40:35.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:40:34 smithi153 ceph-mon[70044]: pgmap v20166: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:36.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:40:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:40:36.038Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:40:37.105 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:40:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:40:36.791Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:40:37.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:40:37 smithi083 ceph-mon[79930]: pgmap v20167: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:37.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:40:37 smithi083 ceph-mon[77242]: pgmap v20167: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:37.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:40:37 smithi153 ceph-mon[70044]: pgmap v20167: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:39.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:40:39 smithi083 ceph-mon[79930]: pgmap v20168: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:39.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:40:39 smithi083 ceph-mon[77242]: pgmap v20168: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:39.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:40:39 smithi153 ceph-mon[70044]: pgmap v20168: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:41.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:40:41 smithi083 ceph-mon[79930]: pgmap v20169: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:41.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:40:41 smithi083 ceph-mon[77242]: pgmap v20169: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:41.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:40:41 smithi153 ceph-mon[70044]: pgmap v20169: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:41.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:40:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:40:41] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T16:40:42.460 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:40:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:40:42.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:40:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:40:42.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:40:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:40:42.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:40:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:40:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:40:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:40:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:40:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:40:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:40:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:40:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:40:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:40:42.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:40:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:40:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:40:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:40:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:40:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:40:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:40:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:40:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:40:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:40:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:40:42.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:40:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:40:42.696Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:40:43.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:40:43 smithi153 ceph-mon[70044]: pgmap v20170: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:40:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:40:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:40:43.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:40:43 smithi083 ceph-mon[79930]: pgmap v20170: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:43.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:40:43 smithi083 ceph-mon[77242]: pgmap v20170: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:44.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:40:44 smithi083 ceph-mon[79930]: pgmap v20171: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:44.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:40:44 smithi083 ceph-mon[77242]: pgmap v20171: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:45.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:40:44 smithi153 ceph-mon[70044]: pgmap v20171: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:46.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:40:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:40:46.039Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:40:47.108 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:40:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:40:46.791Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:40:47.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:40:47 smithi083 ceph-mon[79930]: pgmap v20172: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:47.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:40:47 smithi083 ceph-mon[77242]: pgmap v20172: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:47.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:40:47 smithi153 ceph-mon[70044]: pgmap v20172: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:49.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:40:49 smithi083 ceph-mon[79930]: pgmap v20173: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:49.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:40:49 smithi083 ceph-mon[77242]: pgmap v20173: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:49.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:40:49 smithi153 ceph-mon[70044]: pgmap v20173: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:51.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:40:51 smithi083 ceph-mon[79930]: pgmap v20174: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:51.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:40:51 smithi083 ceph-mon[77242]: pgmap v20174: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:51.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:40:51 smithi153 ceph-mon[70044]: pgmap v20174: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:51.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:40:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:40:51] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T16:40:52.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:40:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:40:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:40:52.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:40:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:40:52.698Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:40:53.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:40:53 smithi153 ceph-mon[70044]: pgmap v20175: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:40:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:40:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:40:53.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:40:53 smithi083 ceph-mon[79930]: pgmap v20175: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:53.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:40:53 smithi083 ceph-mon[77242]: pgmap v20175: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:54.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:40:54 smithi083 ceph-mon[79930]: pgmap v20176: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:54.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:40:54 smithi083 ceph-mon[77242]: pgmap v20176: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:55.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:40:54 smithi153 ceph-mon[70044]: pgmap v20176: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:56.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:40:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:40:56.039Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:40:57.111 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:40:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:40:56.792Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:40:57.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:40:57 smithi083 ceph-mon[79930]: pgmap v20177: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:57.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:40:57 smithi083 ceph-mon[77242]: pgmap v20177: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:57.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:40:57 smithi153 ceph-mon[70044]: pgmap v20177: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:59.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:40:59 smithi083 ceph-mon[79930]: pgmap v20178: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:59.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:40:59 smithi083 ceph-mon[77242]: pgmap v20178: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:59.539 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:40:59.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:40:59 smithi153 ceph-mon[70044]: pgmap v20178: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:40:59.827 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:40:59.828 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 9m ago 11h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:40:59.828 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 7m ago 11h 64.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:40:59.828 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 9m ago 11h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:40:59.828 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 7m ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:40:59.829 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (11h) 7m ago 11h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:40:59.829 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11h) 9m ago 11h 1076M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:40:59.829 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11h) 9m ago 11h 2403M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:40:59.830 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (11h) 7m ago 11h 1077M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:40:59.830 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (11h) 9m ago 11h 1447M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:40:59.831 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 9m ago 11h 24.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:40:59.831 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 7m ago 11h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:40:59.831 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11h) 9m ago 11h 1751M 2049M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:40:59.832 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11h) 9m ago 11h 1736M 2049M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:40:59.832 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11h) 9m ago 11h 1348M 2049M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:40:59.832 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (11h) 9m ago 11h 1702M 2049M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:40:59.832 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (11h) 7m ago 11h 1491M 2486M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:40:59.833 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (11h) 7m ago 11h 1875M 2486M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:40:59.833 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (11h) 7m ago 11h 1676M 2486M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:40:59.833 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (11h) 7m ago 11h 1995M 2486M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:40:59.833 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (11h) 7m ago 11h 193M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:41:00.182 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:41:00.183 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:41:00.183 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:41:00.183 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:41:00.184 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:41:00.184 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:41:00.184 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:41:00.184 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:41:00.185 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:41:00.185 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:41:00.185 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:41:00.185 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:41:00.186 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:41:00.186 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:41:00.186 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:41:00.186 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:41:00.187 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:41:00.187 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:41:01.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:41:01 smithi083 ceph-mon[79930]: from='client.67525 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:41:01.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:41:01 smithi083 ceph-mon[79930]: from='client.57600 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:41:01.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:41:01 smithi083 ceph-mon[79930]: pgmap v20179: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:41:01.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:41:01 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2733139173' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:41:01.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:41:01 smithi083 ceph-mon[77242]: from='client.67525 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:41:01.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:41:01 smithi083 ceph-mon[77242]: from='client.57600 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:41:01.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:41:01 smithi083 ceph-mon[77242]: pgmap v20179: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:41:01.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:41:01 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2733139173' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:41:01.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:41:01 smithi153 ceph-mon[70044]: from='client.67525 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:41:01.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:41:01 smithi153 ceph-mon[70044]: from='client.57600 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:41:01.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:41:01 smithi153 ceph-mon[70044]: pgmap v20179: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:41:01.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:41:01 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2733139173' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:41:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:41:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:41:01] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T16:41:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:41:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:41:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:41:02.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:41:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:41:02.698Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:41:03.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:41:03 smithi153 ceph-mon[70044]: pgmap v20180: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:41:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:41:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:41:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:41:03.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:41:03 smithi083 ceph-mon[77242]: pgmap v20180: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:41:03.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:41:03 smithi083 ceph-mon[79930]: pgmap v20180: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:41:04.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:41:04 smithi083 ceph-mon[77242]: pgmap v20181: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:41:04.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:41:04 smithi083 ceph-mon[79930]: pgmap v20181: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:41:05.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:41:04 smithi153 ceph-mon[70044]: pgmap v20181: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:41:06.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:41:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:41:06.039Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:41:07.114 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:41:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:41:06.793Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:41:07.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:41:07 smithi083 ceph-mon[77242]: pgmap v20182: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:41:07.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:41:07 smithi083 ceph-mon[79930]: pgmap v20182: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:41:07.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:41:07 smithi153 ceph-mon[70044]: pgmap v20182: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:41:09.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:41:09 smithi083 ceph-mon[79930]: pgmap v20183: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:41:09.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:41:09 smithi083 ceph-mon[77242]: pgmap v20183: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:41:09.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:41:09 smithi153 ceph-mon[70044]: pgmap v20183: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:41:11.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:41:11 smithi083 ceph-mon[79930]: pgmap v20184: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:41:11.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:41:11 smithi083 ceph-mon[77242]: pgmap v20184: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:41:11.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:41:11 smithi153 ceph-mon[70044]: pgmap v20184: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:41:11.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:41:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:41:11] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T16:41:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:41:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:41:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:41:12.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:41:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:41:12.698Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:41:13.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:41:13 smithi153 ceph-mon[70044]: pgmap v20185: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:41:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:41:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:41:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:41:13.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:41:13 smithi083 ceph-mon[79930]: pgmap v20185: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:41:13.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:41:13 smithi083 ceph-mon[77242]: pgmap v20185: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:41:14.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:41:14 smithi083 ceph-mon[79930]: pgmap v20186: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:41:14.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:41:14 smithi083 ceph-mon[77242]: pgmap v20186: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:41:15.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:41:14 smithi153 ceph-mon[70044]: pgmap v20186: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:41:16.460 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:41:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:41:16.040Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:41:17.117 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:41:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:41:16.793Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:41:17.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:41:17 smithi083 ceph-mon[77242]: pgmap v20187: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:41:17.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:41:17 smithi083 ceph-mon[79930]: pgmap v20187: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:41:17.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:41:17 smithi153 ceph-mon[70044]: pgmap v20187: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:41:19.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:41:19 smithi083 ceph-mon[79930]: pgmap v20188: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:41:19.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:41:19 smithi083 ceph-mon[77242]: pgmap v20188: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:41:19.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:41:19 smithi153 ceph-mon[70044]: pgmap v20188: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:41:21.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:41:21 smithi083 ceph-mon[79930]: pgmap v20189: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:41:21.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:41:21 smithi083 ceph-mon[77242]: pgmap v20189: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:41:21.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:41:21 smithi153 ceph-mon[70044]: pgmap v20189: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:41:21.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:41:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:41:21] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T16:41:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:41:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:41:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:41:22.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:41:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:41:22.698Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:41:23.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:41:23 smithi153 ceph-mon[70044]: pgmap v20190: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:41:23.312 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:41:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:41:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:41:23.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:41:23 smithi083 ceph-mon[79930]: pgmap v20190: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:41:23.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:41:23 smithi083 ceph-mon[77242]: pgmap v20190: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:41:24.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:41:24 smithi083 ceph-mon[79930]: pgmap v20191: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:41:24.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:41:24 smithi083 ceph-mon[77242]: pgmap v20191: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:41:25.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:41:24 smithi153 ceph-mon[70044]: pgmap v20191: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:41:26.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:41:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:41:26.041Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:41:27.120 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:41:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:41:26.794Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:41:27.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:41:27 smithi083 ceph-mon[79930]: pgmap v20192: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:41:27.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:41:27 smithi083 ceph-mon[77242]: pgmap v20192: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:41:27.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:41:27 smithi153 ceph-mon[70044]: pgmap v20192: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:41:29.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:41:29 smithi083 ceph-mon[79930]: pgmap v20193: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:41:29.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:41:29 smithi083 ceph-mon[77242]: pgmap v20193: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:41:29.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:41:29 smithi153 ceph-mon[70044]: pgmap v20193: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:41:30.510 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:41:30.799 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:41:30.815 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 10m ago 11h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:41:30.815 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 8m ago 11h 64.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:41:30.816 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 10m ago 11h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:41:30.816 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 8m ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:41:30.816 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (11h) 8m ago 11h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:41:30.817 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11h) 10m ago 11h 1076M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:41:30.817 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11h) 10m ago 11h 2403M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:41:30.817 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (11h) 8m ago 11h 1077M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:41:30.817 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (11h) 10m ago 11h 1447M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:41:30.818 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 10m ago 11h 24.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:41:30.818 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 8m ago 11h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:41:30.818 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11h) 10m ago 11h 1751M 2049M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:41:30.818 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11h) 10m ago 11h 1736M 2049M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:41:30.819 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11h) 10m ago 11h 1348M 2049M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:41:30.819 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (11h) 10m ago 11h 1702M 2049M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:41:30.820 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (11h) 8m ago 11h 1491M 2486M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:41:30.820 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (11h) 8m ago 11h 1875M 2486M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:41:30.820 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (11h) 8m ago 11h 1676M 2486M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:41:30.821 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (11h) 8m ago 11h 1995M 2486M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:41:30.821 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (11h) 8m ago 11h 193M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:41:31.157 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:41:31.158 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:41:31.158 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:41:31.158 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:41:31.158 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:41:31.159 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:41:31.159 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:41:31.159 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:41:31.159 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:41:31.159 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:41:31.160 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:41:31.160 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:41:31.161 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:41:31.161 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:41:31.162 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:41:31.162 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:41:31.163 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:41:31.163 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:41:31.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:41:31 smithi083 ceph-mon[79930]: pgmap v20194: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:41:31.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:41:31 smithi083 ceph-mon[77242]: pgmap v20194: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:41:31.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:41:31 smithi153 ceph-mon[70044]: pgmap v20194: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:41:31.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:41:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:41:31] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T16:41:32.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:41:32 smithi083 ceph-mon[79930]: from='client.57612 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:41:32.467 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:41:32 smithi083 ceph-mon[79930]: from='client.67549 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:41:32.467 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:41:32 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4109383020' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:41:32.468 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:41:32 smithi083 ceph-mon[77242]: from='client.57612 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:41:32.469 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:41:32 smithi083 ceph-mon[77242]: from='client.67549 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:41:32.469 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:41:32 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4109383020' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:41:32.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:41:32 smithi153 ceph-mon[70044]: from='client.57612 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:41:32.574 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:41:32 smithi153 ceph-mon[70044]: from='client.67549 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:41:32.574 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:41:32 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4109383020' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:41:32.575 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:41:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:41:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:41:32.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:41:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:41:32.700Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:41:33.332 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:41:33 smithi153 ceph-mon[70044]: pgmap v20195: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:41:33.333 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:41:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:41:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:41:33.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:41:33 smithi083 ceph-mon[79930]: pgmap v20195: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:41:33.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:41:33 smithi083 ceph-mon[77242]: pgmap v20195: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:41:34.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:41:34 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:41:34.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:41:34 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:41:34.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:41:34 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:41:34.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:41:34 smithi083 ceph-mon[77242]: pgmap v20196: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:41:34.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:41:34 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:41:34.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:41:34 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:41:34.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:41:34 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:41:34.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:41:34 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:41:34.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:41:34 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:41:34.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:41:34 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:41:34.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:41:34 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:41:34.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:41:34 smithi083 ceph-mon[79930]: pgmap v20196: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:41:34.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:41:34 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:41:34.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:41:34 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:41:34.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:41:34 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:41:34.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:41:34 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:41:35.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:41:34 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:41:35.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:41:34 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:41:35.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:41:34 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:41:35.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:41:34 smithi153 ceph-mon[70044]: pgmap v20196: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:41:35.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:41:34 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:41:35.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:41:34 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:41:35.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:41:34 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:41:35.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:41:34 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:41:36.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:41:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:41:36.040Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:41:37.123 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:41:37 smithi083 ceph-mon[77242]: pgmap v20197: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:41:37.124 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:41:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:41:36.793Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:41:37.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:41:37 smithi083 ceph-mon[79930]: pgmap v20197: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:41:37.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:41:37 smithi153 ceph-mon[70044]: pgmap v20197: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:41:39.789 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:41:39 smithi083 ceph-mon[79930]: pgmap v20198: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:41:39.937 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:41:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:41:39.938 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:41:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:41:39.939 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:41:39 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:41:39.940 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:41:39 smithi083 ceph-mon[77242]: pgmap v20198: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:41:39.940 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:41:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:41:39.941 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:41:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:41:39.941 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:41:39 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:41:39.942 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:41:39 smithi153 ceph-mon[70044]: pgmap v20198: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:41:39.943 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:41:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:41:39.943 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:41:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:41:39.944 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:41:39 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:41:41.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:41:41 smithi083 ceph-mon[79930]: pgmap v20199: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:41:41.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:41:41 smithi083 ceph-mon[77242]: pgmap v20199: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:41:41.715 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:41:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:41:41] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T16:41:41.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:41:41 smithi153 ceph-mon[70044]: pgmap v20199: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:41:42.702 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:41:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:41:42.704 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:41:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:41:42.704 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:41:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:41:42.704 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:41:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:41:42.705 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:41:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:41:42.705 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:41:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:41:42.705 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:41:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:41:42.706 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:41:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:41:42.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:41:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:41:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:41:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:41:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:41:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:41:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:41:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:41:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:41:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:41:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:41:42.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:41:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:41:42.701Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:41:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:41:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:41:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:41:43.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:41:43 smithi083 ceph-mon[79930]: pgmap v20200: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:41:43.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:41:43 smithi083 ceph-mon[77242]: pgmap v20200: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:41:43.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:41:43 smithi153 ceph-mon[70044]: pgmap v20200: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:41:44.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:41:44 smithi083 ceph-mon[79930]: pgmap v20201: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:41:44.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:41:44 smithi083 ceph-mon[77242]: pgmap v20201: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:41:45.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:41:44 smithi153 ceph-mon[70044]: pgmap v20201: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:41:46.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:41:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:41:46.042Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:41:47.126 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:41:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:41:46.795Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:41:47.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:41:47 smithi083 ceph-mon[77242]: pgmap v20202: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:41:47.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:41:47 smithi083 ceph-mon[79930]: pgmap v20202: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:41:47.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:41:47 smithi153 ceph-mon[70044]: pgmap v20202: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:41:49.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:41:49 smithi083 ceph-mon[79930]: pgmap v20203: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:41:49.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:41:49 smithi083 ceph-mon[77242]: pgmap v20203: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:41:49.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:41:49 smithi153 ceph-mon[70044]: pgmap v20203: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:41:51.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:41:51 smithi083 ceph-mon[77242]: pgmap v20204: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:41:51.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:41:51 smithi083 ceph-mon[79930]: pgmap v20204: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:41:51.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:41:51 smithi153 ceph-mon[70044]: pgmap v20204: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:41:51.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:41:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:41:51] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T16:41:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:41:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:41:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:41:52.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:41:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:41:52.701Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:41:53.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:41:53 smithi153 ceph-mon[70044]: pgmap v20205: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:41:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:41:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:41:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:41:53.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:41:53 smithi083 ceph-mon[77242]: pgmap v20205: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:41:53.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:41:53 smithi083 ceph-mon[79930]: pgmap v20205: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:41:54.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:41:54 smithi083 ceph-mon[79930]: pgmap v20206: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:41:54.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:41:54 smithi083 ceph-mon[77242]: pgmap v20206: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:41:55.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:41:54 smithi153 ceph-mon[70044]: pgmap v20206: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:41:56.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:41:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:41:56.043Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:41:57.129 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:41:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:41:56.795Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:41:57.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:41:57 smithi083 ceph-mon[79930]: pgmap v20207: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:41:57.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:41:57 smithi083 ceph-mon[77242]: pgmap v20207: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:41:57.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:41:57 smithi153 ceph-mon[70044]: pgmap v20207: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:41:59.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:41:59 smithi083 ceph-mon[79930]: pgmap v20208: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:41:59.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:41:59 smithi083 ceph-mon[77242]: pgmap v20208: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:41:59.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:41:59 smithi153 ceph-mon[70044]: pgmap v20208: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:01.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:42:01 smithi083 ceph-mon[79930]: pgmap v20209: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:01.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:42:01 smithi083 ceph-mon[77242]: pgmap v20209: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:01.486 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:42:01.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:42:01 smithi153 ceph-mon[70044]: pgmap v20209: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:01.774 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:42:01.775 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 23s ago 11h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:42:01.775 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 8m ago 11h 64.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:42:01.775 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 23s ago 11h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:42:01.776 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 8m ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:42:01.776 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (11h) 8m ago 11h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:42:01.776 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11h) 23s ago 11h 1084M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:42:01.776 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11h) 23s ago 11h 2437M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:42:01.777 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (11h) 8m ago 11h 1077M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:42:01.777 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (11h) 23s ago 11h 1469M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:42:01.777 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 23s ago 11h 24.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:42:01.777 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 8m ago 11h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:42:01.778 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11h) 23s ago 11h 1777M 2049M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:42:01.778 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11h) 23s ago 11h 1763M 2049M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:42:01.778 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11h) 23s ago 11h 1369M 2049M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:42:01.778 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (11h) 23s ago 11h 1728M 2049M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:42:01.778 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (11h) 8m ago 11h 1491M 2486M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:42:01.779 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (11h) 8m ago 11h 1875M 2486M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:42:01.779 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (11h) 8m ago 11h 1676M 2486M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:42:01.779 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (11h) 8m ago 11h 1995M 2486M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:42:01.779 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (11h) 8m ago 11h 193M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:42:01.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:42:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:42:01] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T16:42:02.132 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:42:02.133 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:42:02.133 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:42:02.133 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:42:02.133 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:42:02.134 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:42:02.134 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:42:02.134 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:42:02.134 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:42:02.135 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:42:02.135 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:42:02.135 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:42:02.135 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:42:02.135 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:42:02.136 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:42:02.136 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:42:02.136 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:42:02.136 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:42:02.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:42:02 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3478345540' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:42:02.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:42:02 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3478345540' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:42:02.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:42:02 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3478345540' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:42:02.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:42:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:42:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:42:02.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:42:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:42:02.701Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:42:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:42:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:42:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:42:03.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:42:03 smithi153 ceph-mon[70044]: from='client.57630 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:42:03.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:42:03 smithi153 ceph-mon[70044]: from='client.57636 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:42:03.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:42:03 smithi153 ceph-mon[70044]: pgmap v20210: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:03.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:42:03 smithi083 ceph-mon[79930]: from='client.57630 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:42:03.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:42:03 smithi083 ceph-mon[79930]: from='client.57636 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:42:03.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:42:03 smithi083 ceph-mon[79930]: pgmap v20210: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:03.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:42:03 smithi083 ceph-mon[77242]: from='client.57630 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:42:03.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:42:03 smithi083 ceph-mon[77242]: from='client.57636 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:42:03.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:42:03 smithi083 ceph-mon[77242]: pgmap v20210: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:04.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:42:04 smithi083 ceph-mon[79930]: pgmap v20211: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:04.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:42:04 smithi083 ceph-mon[77242]: pgmap v20211: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:05.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:42:04 smithi153 ceph-mon[70044]: pgmap v20211: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:06.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:42:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:42:06.043Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:42:07.133 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:42:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:42:06.796Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:42:07.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:42:07 smithi083 ceph-mon[79930]: pgmap v20212: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:07.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:42:07 smithi083 ceph-mon[77242]: pgmap v20212: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:07.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:42:07 smithi153 ceph-mon[70044]: pgmap v20212: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:09.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:42:09 smithi083 ceph-mon[79930]: pgmap v20213: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:09.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:42:09 smithi083 ceph-mon[77242]: pgmap v20213: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:09.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:42:09 smithi153 ceph-mon[70044]: pgmap v20213: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:11.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:42:11 smithi083 ceph-mon[79930]: pgmap v20214: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:11.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:42:11 smithi083 ceph-mon[77242]: pgmap v20214: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:11.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:42:11 smithi153 ceph-mon[70044]: pgmap v20214: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:42:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:42:11] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T16:42:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:42:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:42:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:42:12.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:42:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:42:12.703Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:42:13.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:42:13 smithi153 ceph-mon[70044]: pgmap v20215: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:42:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:42:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:42:13.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:42:13 smithi083 ceph-mon[79930]: pgmap v20215: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:13.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:42:13 smithi083 ceph-mon[77242]: pgmap v20215: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:14.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:42:14 smithi083 ceph-mon[79930]: pgmap v20216: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:14.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:42:14 smithi083 ceph-mon[77242]: pgmap v20216: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:15.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:42:14 smithi153 ceph-mon[70044]: pgmap v20216: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:16.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:42:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:42:16.043Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:42:17.136 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:42:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:42:16.796Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:42:17.460 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:42:17 smithi083 ceph-mon[77242]: pgmap v20217: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:17.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:42:17 smithi083 ceph-mon[79930]: pgmap v20217: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:17.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:42:17 smithi153 ceph-mon[70044]: pgmap v20217: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:19.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:42:19 smithi083 ceph-mon[77242]: pgmap v20218: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:19.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:42:19 smithi083 ceph-mon[79930]: pgmap v20218: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:19.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:42:19 smithi153 ceph-mon[70044]: pgmap v20218: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:21.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:42:21 smithi083 ceph-mon[79930]: pgmap v20219: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:21.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:42:21 smithi083 ceph-mon[77242]: pgmap v20219: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:21.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:42:21 smithi153 ceph-mon[70044]: pgmap v20219: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:21.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:42:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:42:21] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T16:42:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:42:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:42:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:42:22.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:42:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:42:22.703Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:42:23.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:42:23 smithi153 ceph-mon[70044]: pgmap v20220: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:42:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:42:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:42:23.460 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:42:23 smithi083 ceph-mon[77242]: pgmap v20220: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:23.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:42:23 smithi083 ceph-mon[79930]: pgmap v20220: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:24.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:42:24 smithi083 ceph-mon[79930]: pgmap v20221: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:24.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:42:24 smithi083 ceph-mon[77242]: pgmap v20221: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:25.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:42:24 smithi153 ceph-mon[70044]: pgmap v20221: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:26.460 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:42:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:42:26.044Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:42:27.140 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:42:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:42:26.797Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:42:27.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:42:27 smithi083 ceph-mon[79930]: pgmap v20222: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:27.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:42:27 smithi083 ceph-mon[77242]: pgmap v20222: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:27.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:42:27 smithi153 ceph-mon[70044]: pgmap v20222: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:29.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:42:29 smithi083 ceph-mon[79930]: pgmap v20223: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:29.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:42:29 smithi083 ceph-mon[77242]: pgmap v20223: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:29.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:42:29 smithi153 ceph-mon[70044]: pgmap v20223: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:31.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:42:31 smithi083 ceph-mon[79930]: pgmap v20224: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:31.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:42:31 smithi083 ceph-mon[77242]: pgmap v20224: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:31.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:42:31 smithi153 ceph-mon[70044]: pgmap v20224: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:31.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:42:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:42:31] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T16:42:32.462 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:42:32.749 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:42:32.750 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 54s ago 11h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:42:32.750 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 9m ago 11h 64.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:42:32.750 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 54s ago 11h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:42:32.751 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 9m ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:42:32.751 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (11h) 9m ago 11h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:42:32.751 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11h) 54s ago 11h 1084M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:42:32.751 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11h) 54s ago 11h 2437M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:42:32.752 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (11h) 9m ago 11h 1077M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:42:32.752 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (11h) 54s ago 11h 1469M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:42:32.752 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 54s ago 11h 24.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:42:32.752 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 9m ago 11h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:42:32.752 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11h) 54s ago 11h 1777M 2049M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:42:32.753 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11h) 54s ago 11h 1763M 2049M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:42:32.753 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11h) 54s ago 11h 1369M 2049M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:42:32.753 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (11h) 54s ago 11h 1728M 2049M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:42:32.753 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (11h) 9m ago 11h 1491M 2486M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:42:32.754 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (11h) 9m ago 11h 1875M 2486M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:42:32.754 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (11h) 9m ago 11h 1676M 2486M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:42:32.754 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (11h) 9m ago 11h 1995M 2486M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:42:32.754 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (11h) 9m ago 11h 193M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:42:32.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:42:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:42:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:42:32.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:42:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:42:32.703Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:42:33.107 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:42:33.107 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:42:33.107 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:42:33.108 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:42:33.108 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:42:33.108 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:42:33.108 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:42:33.109 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:42:33.109 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:42:33.109 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:42:33.109 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:42:33.109 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:42:33.110 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:42:33.110 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:42:33.110 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:42:33.110 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:42:33.111 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:42:33.111 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:42:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:42:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:42:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:42:33.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:42:33 smithi153 ceph-mon[70044]: pgmap v20225: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:33.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:42:33 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/88721894' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:42:33.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:42:33 smithi083 ceph-mon[79930]: pgmap v20225: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:33.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:42:33 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/88721894' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:42:33.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:42:33 smithi083 ceph-mon[77242]: pgmap v20225: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:33.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:42:33 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/88721894' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:42:34.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:42:34 smithi083 ceph-mon[79930]: from='client.67579 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:42:34.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:42:34 smithi083 ceph-mon[79930]: from='client.57654 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:42:34.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:42:34 smithi083 ceph-mon[77242]: from='client.67579 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:42:34.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:42:34 smithi083 ceph-mon[77242]: from='client.57654 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:42:34.562 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:42:34 smithi153 ceph-mon[70044]: from='client.67579 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:42:34.562 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:42:34 smithi153 ceph-mon[70044]: from='client.57654 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:42:35.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:42:35 smithi083 ceph-mon[79930]: pgmap v20226: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:35.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:42:35 smithi083 ceph-mon[77242]: pgmap v20226: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:35.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:42:35 smithi153 ceph-mon[70044]: pgmap v20226: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:36.460 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:42:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:42:36.044Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:42:37.156 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:42:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:42:36.798Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:42:37.460 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:42:37 smithi083 ceph-mon[77242]: pgmap v20227: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:37.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:42:37 smithi083 ceph-mon[79930]: pgmap v20227: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:37.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:42:37 smithi153 ceph-mon[70044]: pgmap v20227: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:39.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:42:39 smithi083 ceph-mon[79930]: pgmap v20228: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:39.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:42:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:42:39.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:42:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:42:39.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:42:39 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:42:39.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:42:39 smithi083 ceph-mon[77242]: pgmap v20228: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:39.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:42:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:42:39.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:42:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:42:39.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:42:39 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:42:39.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:42:39 smithi153 ceph-mon[70044]: pgmap v20228: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:39.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:42:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:42:39.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:42:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:42:39.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:42:39 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:42:40.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:42:40 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:42:40.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:42:40 smithi083 ceph-mon[77242]: pgmap v20229: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:42:40 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:42:40.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:42:40 smithi083 ceph-mon[79930]: pgmap v20229: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:42:40 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:42:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:42:40 smithi153 ceph-mon[70044]: pgmap v20229: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:42:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:42:41] "GET /metrics HTTP/1.1" 200 225246 "" "Prometheus/2.33.4" 2022-09-29T16:42:42.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:42:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:42:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:42:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:42:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:42:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:42:42.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:42:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:42:42.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:42:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:42:42.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:42:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:42:42.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:42:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:42:42.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:42:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:42:42.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:42:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:42:42.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:42:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:42:42.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:42:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:42:42.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:42:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:42:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:42:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:42:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:42:42.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:42:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:42:42.704Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:42:43.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:42:43 smithi153 ceph-mon[70044]: pgmap v20230: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:42:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:42:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:42:43.460 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:42:43 smithi083 ceph-mon[77242]: pgmap v20230: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:43.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:42:43 smithi083 ceph-mon[79930]: pgmap v20230: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:44.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:42:44 smithi083 ceph-mon[79930]: pgmap v20231: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:44.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:42:44 smithi083 ceph-mon[77242]: pgmap v20231: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:45.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:42:44 smithi153 ceph-mon[70044]: pgmap v20231: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:46.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:42:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:42:46.044Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:42:47.147 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:42:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:42:46.799Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:42:47.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:42:47 smithi083 ceph-mon[79930]: pgmap v20232: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:47.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:42:47 smithi083 ceph-mon[77242]: pgmap v20232: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:47.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:42:47 smithi153 ceph-mon[70044]: pgmap v20232: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:49.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:42:49 smithi083 ceph-mon[79930]: pgmap v20233: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:49.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:42:49 smithi083 ceph-mon[77242]: pgmap v20233: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:49.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:42:49 smithi153 ceph-mon[70044]: pgmap v20233: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:51.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:42:51 smithi083 ceph-mon[79930]: pgmap v20234: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:51.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:42:51 smithi083 ceph-mon[77242]: pgmap v20234: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:51.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:42:51 smithi153 ceph-mon[70044]: pgmap v20234: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:51.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:42:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:42:51] "GET /metrics HTTP/1.1" 200 225246 "" "Prometheus/2.33.4" 2022-09-29T16:42:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:42:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:42:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:42:52.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:42:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:42:52.705Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:42:53.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:42:53 smithi153 ceph-mon[70044]: pgmap v20235: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:42:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:42:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:42:53.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:42:53 smithi083 ceph-mon[79930]: pgmap v20235: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:53.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:42:53 smithi083 ceph-mon[77242]: pgmap v20235: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:54.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:42:54 smithi083 ceph-mon[79930]: pgmap v20236: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:54.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:42:54 smithi083 ceph-mon[77242]: pgmap v20236: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:55.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:42:54 smithi153 ceph-mon[70044]: pgmap v20236: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:56.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:42:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:42:56.045Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:42:57.150 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:42:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:42:56.800Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:42:57.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:42:57 smithi083 ceph-mon[79930]: pgmap v20237: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:57.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:42:57 smithi083 ceph-mon[77242]: pgmap v20237: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:57.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:42:57 smithi153 ceph-mon[70044]: pgmap v20237: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:59.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:42:59 smithi083 ceph-mon[79930]: pgmap v20238: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:59.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:42:59 smithi083 ceph-mon[77242]: pgmap v20238: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:42:59.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:42:59 smithi153 ceph-mon[70044]: pgmap v20238: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:01.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:43:01 smithi083 ceph-mon[79930]: pgmap v20239: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:01.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:43:01 smithi083 ceph-mon[77242]: pgmap v20239: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:01.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:43:01 smithi153 ceph-mon[70044]: pgmap v20239: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:01.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:43:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:43:01] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T16:43:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:43:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:43:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:43:02.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:43:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:43:02.706Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:43:03.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:43:03 smithi153 ceph-mon[70044]: pgmap v20240: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:43:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:43:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:43:03.443 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:43:03.460 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:43:03 smithi083 ceph-mon[77242]: pgmap v20240: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:03.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:43:03 smithi083 ceph-mon[79930]: pgmap v20240: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:03.733 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:43:03.733 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 85s ago 11h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:43:03.734 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 9m ago 11h 64.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:43:03.735 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 85s ago 11h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:43:03.735 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 9m ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:43:03.736 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (11h) 9m ago 11h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:43:03.736 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11h) 85s ago 11h 1084M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:43:03.737 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11h) 85s ago 11h 2437M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:43:03.738 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (11h) 9m ago 11h 1077M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:43:03.738 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (11h) 85s ago 11h 1469M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:43:03.739 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 85s ago 11h 24.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:43:03.740 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 9m ago 11h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:43:03.741 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11h) 85s ago 11h 1777M 2049M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:43:03.742 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11h) 85s ago 11h 1763M 2049M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:43:03.742 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11h) 85s ago 11h 1369M 2049M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:43:03.742 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (11h) 85s ago 11h 1728M 2049M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:43:03.743 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (11h) 9m ago 11h 1491M 2486M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:43:03.743 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (11h) 9m ago 11h 1875M 2486M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:43:03.744 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (11h) 9m ago 11h 1676M 2486M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:43:03.744 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (11h) 9m ago 11h 1995M 2486M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:43:03.744 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (11h) 9m ago 11h 193M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:43:04.090 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:43:04.091 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:43:04.091 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:43:04.092 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:43:04.093 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:43:04.093 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:43:04.094 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:43:04.094 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:43:04.095 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:43:04.095 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:43:04.096 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:43:04.096 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:43:04.097 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:43:04.097 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:43:04.098 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:43:04.098 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:43:04.099 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:43:04.100 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:43:04.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:43:04 smithi083 ceph-mon[77242]: from='client.57666 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:43:04.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:43:04 smithi083 ceph-mon[77242]: from='client.57672 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:43:04.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:43:04 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3713509761' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:43:04.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:43:04 smithi083 ceph-mon[77242]: pgmap v20241: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:04.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:43:04 smithi083 ceph-mon[79930]: from='client.57666 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:43:04.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:43:04 smithi083 ceph-mon[79930]: from='client.57672 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:43:04.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:43:04 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3713509761' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:43:04.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:43:04 smithi083 ceph-mon[79930]: pgmap v20241: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:05.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:43:04 smithi153 ceph-mon[70044]: from='client.57666 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:43:05.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:43:04 smithi153 ceph-mon[70044]: from='client.57672 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:43:05.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:43:04 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3713509761' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:43:05.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:43:04 smithi153 ceph-mon[70044]: pgmap v20241: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:06.460 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:43:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:43:06.046Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:43:07.153 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:43:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:43:06.801Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:43:07.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:43:07 smithi083 ceph-mon[79930]: pgmap v20242: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:07.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:43:07 smithi083 ceph-mon[77242]: pgmap v20242: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:07.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:43:07 smithi153 ceph-mon[70044]: pgmap v20242: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:09.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:43:09 smithi083 ceph-mon[79930]: pgmap v20243: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:09.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:43:09 smithi083 ceph-mon[77242]: pgmap v20243: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:09.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:43:09 smithi153 ceph-mon[70044]: pgmap v20243: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:11.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:43:11 smithi083 ceph-mon[79930]: pgmap v20244: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:11.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:43:11 smithi083 ceph-mon[77242]: pgmap v20244: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:11.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:43:11 smithi153 ceph-mon[70044]: pgmap v20244: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:11.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:43:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:43:11] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T16:43:12.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:43:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:43:12.316Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:43:12.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:43:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:43:12.706Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:43:13.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:43:13 smithi153 ceph-mon[70044]: pgmap v20245: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:43:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:43:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:43:13.460 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:43:13 smithi083 ceph-mon[77242]: pgmap v20245: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:13.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:43:13 smithi083 ceph-mon[79930]: pgmap v20245: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:14.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:43:14 smithi083 ceph-mon[79930]: pgmap v20246: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:14.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:43:14 smithi083 ceph-mon[77242]: pgmap v20246: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:15.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:43:14 smithi153 ceph-mon[70044]: pgmap v20246: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:16.460 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:43:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:43:16.047Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:43:17.155 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:43:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:43:16.801Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:43:17.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:43:17 smithi083 ceph-mon[79930]: pgmap v20247: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:17.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:43:17 smithi083 ceph-mon[77242]: pgmap v20247: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:17.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:43:17 smithi153 ceph-mon[70044]: pgmap v20247: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:19.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:43:19 smithi083 ceph-mon[79930]: pgmap v20248: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:19.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:43:19 smithi083 ceph-mon[77242]: pgmap v20248: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:19.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:43:19 smithi153 ceph-mon[70044]: pgmap v20248: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:21.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:43:21 smithi083 ceph-mon[79930]: pgmap v20249: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:21.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:43:21 smithi083 ceph-mon[77242]: pgmap v20249: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:21.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:43:21 smithi153 ceph-mon[70044]: pgmap v20249: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:43:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:43:21] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T16:43:22.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:43:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:43:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:43:22.966 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:43:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:43:22.706Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:43:23.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:43:23 smithi153 ceph-mon[70044]: pgmap v20250: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:43:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:43:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:43:23.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:43:23 smithi083 ceph-mon[79930]: pgmap v20250: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:23.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:43:23 smithi083 ceph-mon[77242]: pgmap v20250: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:24.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:43:24 smithi083 ceph-mon[79930]: pgmap v20251: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:24.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:43:24 smithi083 ceph-mon[77242]: pgmap v20251: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:25.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:43:24 smithi153 ceph-mon[70044]: pgmap v20251: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:26.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:43:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:43:26.048Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:43:27.159 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:43:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:43:26.802Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:43:27.460 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:43:27 smithi083 ceph-mon[77242]: pgmap v20252: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:27.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:43:27 smithi083 ceph-mon[79930]: pgmap v20252: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:27.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:43:27 smithi153 ceph-mon[70044]: pgmap v20252: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:29.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:43:29 smithi083 ceph-mon[79930]: pgmap v20253: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:29.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:43:29 smithi083 ceph-mon[77242]: pgmap v20253: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:29.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:43:29 smithi153 ceph-mon[70044]: pgmap v20253: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:31.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:43:31 smithi083 ceph-mon[79930]: pgmap v20254: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:31.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:43:31 smithi083 ceph-mon[77242]: pgmap v20254: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:31.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:43:31 smithi153 ceph-mon[70044]: pgmap v20254: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:31.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:43:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:43:31] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T16:43:32.806 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:43:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:43:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:43:32.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:43:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:43:32.708Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:43:33.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:43:33 smithi153 ceph-mon[70044]: pgmap v20255: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:43:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:43:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:43:33.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:43:33 smithi083 ceph-mon[79930]: pgmap v20255: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:33.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:43:33 smithi083 ceph-mon[77242]: pgmap v20255: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:34.415 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:43:34.704 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:43:34.705 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 116s ago 11h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:43:34.705 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 10m ago 11h 64.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:43:34.705 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 116s ago 11h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:43:34.705 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 10m ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:43:34.706 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (11h) 10m ago 11h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:43:34.706 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11h) 116s ago 11h 1084M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:43:34.706 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11h) 116s ago 11h 2437M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:43:34.706 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (11h) 10m ago 11h 1077M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:43:34.706 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (11h) 116s ago 11h 1469M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:43:34.707 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 116s ago 11h 24.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:43:34.707 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 10m ago 11h 25.9M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:43:34.707 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11h) 116s ago 11h 1777M 2049M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:43:34.707 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11h) 116s ago 11h 1763M 2049M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:43:34.708 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11h) 116s ago 11h 1369M 2049M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:43:34.708 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (11h) 116s ago 11h 1728M 2049M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:43:34.708 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (11h) 10m ago 11h 1491M 2486M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:43:34.708 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (11h) 10m ago 11h 1875M 2486M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:43:34.708 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (11h) 10m ago 11h 1676M 2486M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:43:34.709 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (11h) 10m ago 11h 1995M 2486M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:43:34.709 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (11h) 10m ago 11h 193M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:43:34.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:43:34 smithi083 ceph-mon[79930]: pgmap v20256: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:34.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:43:34 smithi083 ceph-mon[77242]: pgmap v20256: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:35.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:43:34 smithi153 ceph-mon[70044]: pgmap v20256: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:35.060 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:43:35.061 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:43:35.061 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:43:35.061 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:43:35.061 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:43:35.062 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:43:35.062 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:43:35.062 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:43:35.062 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:43:35.062 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:43:35.063 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:43:35.063 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:43:35.063 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:43:35.063 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:43:35.064 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:43:35.064 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:43:35.064 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:43:35.064 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:43:35.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:43:35 smithi083 ceph-mon[79930]: from='client.67615 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:43:35.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:43:35 smithi083 ceph-mon[79930]: from='client.57690 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:43:35.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:43:35 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/263697989' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:43:35.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:43:35 smithi083 ceph-mon[77242]: from='client.67615 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:43:35.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:43:35 smithi083 ceph-mon[77242]: from='client.57690 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:43:35.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:43:35 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/263697989' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:43:36.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:43:35 smithi153 ceph-mon[70044]: from='client.67615 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:43:36.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:43:35 smithi153 ceph-mon[70044]: from='client.57690 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:43:36.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:43:35 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/263697989' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:43:36.460 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:43:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:43:36.047Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:43:36.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:43:36 smithi083 ceph-mon[79930]: pgmap v20257: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:36.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:43:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:43:36.803Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:43:36.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:43:36 smithi083 ceph-mon[77242]: pgmap v20257: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:37.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:43:36 smithi153 ceph-mon[70044]: pgmap v20257: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:39.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:43:39 smithi083 ceph-mon[79930]: pgmap v20258: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:39.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:43:39 smithi083 ceph-mon[77242]: pgmap v20258: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:39.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:43:39 smithi153 ceph-mon[70044]: pgmap v20258: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:40.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:43:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:43:40.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:43:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:43:40.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:43:40 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:43:40.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:43:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:43:40.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:43:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:43:40.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:43:40 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:43:40.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:43:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:43:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:43:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:43:40.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:43:40 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:43:41.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:43:41 smithi083 ceph-mon[79930]: pgmap v20259: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:41.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:43:41 smithi083 ceph-mon[77242]: pgmap v20259: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:43:41 smithi153 ceph-mon[70044]: pgmap v20259: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:41.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:43:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:43:41] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T16:43:42.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:43:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:43:42.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:43:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:43:42.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:43:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:43:42.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:43:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:43:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:43:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:43:42.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:43:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:43:42.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:43:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:43:42.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:43:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:43:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:43:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:43:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:43:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:43:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:43:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:43:42.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:43:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:43:42.559 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:43:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:43:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:43:42.960 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:43:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:43:42.708Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:43:43.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:43:43 smithi153 ceph-mon[70044]: pgmap v20260: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:43:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:43:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:43:43.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:43:43 smithi083 ceph-mon[79930]: pgmap v20260: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:43.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:43:43 smithi083 ceph-mon[77242]: pgmap v20260: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:45.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:43:45 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:43:45.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:43:45 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:43:45.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:43:45 smithi083 ceph-mon[79930]: pgmap v20261: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:45.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:43:45 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:43:45.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:43:45 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:43:45.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:43:45 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:43:45.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:43:45 smithi083 ceph-mon[77242]: pgmap v20261: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:45.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:43:45 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:43:45.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:43:45 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:43:45.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:43:45 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:43:45.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:43:45 smithi153 ceph-mon[70044]: pgmap v20261: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:45.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:43:45 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:43:46.460 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:43:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:43:46.049Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:43:47.173 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:43:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:43:46.803Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:43:47.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:43:47 smithi083 ceph-mon[79930]: pgmap v20262: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:47.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:43:47 smithi083 ceph-mon[77242]: pgmap v20262: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:47.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:43:47 smithi153 ceph-mon[70044]: pgmap v20262: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:49.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:43:49 smithi083 ceph-mon[79930]: pgmap v20263: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:49.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:43:49 smithi083 ceph-mon[77242]: pgmap v20263: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:49.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:43:49 smithi153 ceph-mon[70044]: pgmap v20263: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:51.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:43:51 smithi083 ceph-mon[79930]: pgmap v20264: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:51.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:43:51 smithi083 ceph-mon[77242]: pgmap v20264: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:51.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:43:51 smithi153 ceph-mon[70044]: pgmap v20264: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:51.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:43:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:43:51] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T16:43:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:43:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:43:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:43:52.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:43:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:43:52.708Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:43:53.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:43:53 smithi153 ceph-mon[70044]: pgmap v20265: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:53.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:43:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:43:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:43:53.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:43:53 smithi083 ceph-mon[79930]: pgmap v20265: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:53.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:43:53 smithi083 ceph-mon[77242]: pgmap v20265: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:54.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:43:54 smithi083 ceph-mon[79930]: pgmap v20266: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:54.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:43:54 smithi083 ceph-mon[77242]: pgmap v20266: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:55.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:43:54 smithi153 ceph-mon[70044]: pgmap v20266: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:56.460 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:43:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:43:56.049Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:43:57.168 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:43:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:43:56.804Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:43:57.460 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:43:57 smithi083 ceph-mon[77242]: pgmap v20267: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:57.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:43:57 smithi083 ceph-mon[79930]: pgmap v20267: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:57.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:43:57 smithi153 ceph-mon[70044]: pgmap v20267: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:59.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:43:59 smithi083 ceph-mon[79930]: pgmap v20268: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:59.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:43:59 smithi083 ceph-mon[77242]: pgmap v20268: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:43:59.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:43:59 smithi153 ceph-mon[70044]: pgmap v20268: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:01.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:44:01 smithi083 ceph-mon[77242]: pgmap v20269: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:01.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:44:01 smithi083 ceph-mon[79930]: pgmap v20269: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:01.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:44:01 smithi153 ceph-mon[70044]: pgmap v20269: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:44:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:44:01] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T16:44:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:44:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:44:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:44:02.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:44:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:44:02.709Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:44:03.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:44:03 smithi153 ceph-mon[70044]: pgmap v20270: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:44:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:44:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:44:03.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:44:03 smithi083 ceph-mon[77242]: pgmap v20270: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:03.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:44:03 smithi083 ceph-mon[79930]: pgmap v20270: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:04.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:44:04 smithi083 ceph-mon[77242]: pgmap v20271: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:04.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:44:04 smithi083 ceph-mon[79930]: pgmap v20271: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:05.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:44:04 smithi153 ceph-mon[70044]: pgmap v20271: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:05.390 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:44:05.677 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:44:05.677 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 2m ago 11h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:44:05.678 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 21s ago 11h 64.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:44:05.678 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 2m ago 11h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:44:05.678 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 21s ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:44:05.678 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (11h) 21s ago 11h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:44:05.679 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11h) 2m ago 11h 1084M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:44:05.679 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11h) 2m ago 11h 2437M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:44:05.679 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (11h) 21s ago 11h 1093M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:44:05.680 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (11h) 2m ago 11h 1469M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:44:05.680 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 2m ago 11h 24.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:44:05.680 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 21s ago 11h 26.6M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:44:05.680 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11h) 2m ago 11h 1777M 2049M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:44:05.680 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11h) 2m ago 11h 1763M 2049M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:44:05.681 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11h) 2m ago 11h 1369M 2049M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:44:05.681 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (11h) 2m ago 11h 1728M 2049M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:44:05.681 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (11h) 21s ago 11h 1514M 2486M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:44:05.681 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (11h) 21s ago 11h 1903M 2486M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:44:05.682 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (11h) 21s ago 11h 1701M 2486M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:44:05.682 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (11h) 21s ago 11h 2025M 2486M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:44:05.682 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (11h) 21s ago 11h 196M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:44:05.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:44:05 smithi083 ceph-mon[79930]: from='client.57702 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:44:05.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:44:05 smithi083 ceph-mon[77242]: from='client.57702 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:44:06.033 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:44:06.034 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:44:06.034 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:44:06.034 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:44:06.034 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:44:06.035 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:44:06.035 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:44:06.036 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:44:06.036 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:44:06.036 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:44:06.037 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:44:06.037 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:44:06.037 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:44:06.037 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:44:06.037 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:44:06.038 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:44:06.038 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:44:06.038 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:44:06.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:44:05 smithi153 ceph-mon[70044]: from='client.57702 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:44:06.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:44:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:44:06.049Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:44:06.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:44:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:44:06.804Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:44:06.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:44:06 smithi083 ceph-mon[77242]: from='client.57708 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:44:06.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:44:06 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3394732413' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:44:06.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:44:06 smithi083 ceph-mon[77242]: pgmap v20272: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:06.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:44:06 smithi083 ceph-mon[79930]: from='client.57708 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:44:06.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:44:06 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3394732413' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:44:06.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:44:06 smithi083 ceph-mon[79930]: pgmap v20272: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:07.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:44:06 smithi153 ceph-mon[70044]: from='client.57708 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:44:07.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:44:06 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3394732413' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:44:07.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:44:06 smithi153 ceph-mon[70044]: pgmap v20272: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:09.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:44:09 smithi083 ceph-mon[79930]: pgmap v20273: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:09.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:44:09 smithi083 ceph-mon[77242]: pgmap v20273: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:09.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:44:09 smithi153 ceph-mon[70044]: pgmap v20273: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:11.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:44:11 smithi083 ceph-mon[79930]: pgmap v20274: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:11.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:44:11 smithi083 ceph-mon[77242]: pgmap v20274: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:11.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:44:11 smithi153 ceph-mon[70044]: pgmap v20274: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:44:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:44:11] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T16:44:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:44:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:44:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:44:13.177 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:44:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:44:12.710Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:44:13.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:44:13 smithi153 ceph-mon[70044]: pgmap v20275: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:44:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:44:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:44:13.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:44:13 smithi083 ceph-mon[79930]: pgmap v20275: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:13.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:44:13 smithi083 ceph-mon[77242]: pgmap v20275: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:14.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:44:14 smithi083 ceph-mon[79930]: pgmap v20276: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:14.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:44:14 smithi083 ceph-mon[77242]: pgmap v20276: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:15.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:44:14 smithi153 ceph-mon[70044]: pgmap v20276: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:16.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:44:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:44:16.050Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:44:17.174 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:44:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:44:16.805Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:44:17.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:44:17 smithi083 ceph-mon[77242]: pgmap v20277: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:17.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:44:17 smithi083 ceph-mon[79930]: pgmap v20277: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:17.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:44:17 smithi153 ceph-mon[70044]: pgmap v20277: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:19.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:44:19 smithi083 ceph-mon[79930]: pgmap v20278: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:19.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:44:19 smithi083 ceph-mon[77242]: pgmap v20278: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:19.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:44:19 smithi153 ceph-mon[70044]: pgmap v20278: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:21.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:44:21 smithi083 ceph-mon[79930]: pgmap v20279: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:21.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:44:21 smithi083 ceph-mon[77242]: pgmap v20279: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:21.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:44:21 smithi153 ceph-mon[70044]: pgmap v20279: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:21.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:44:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:44:21] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T16:44:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:44:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:44:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:44:23.183 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:44:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:44:22.710Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:44:23.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:44:23 smithi153 ceph-mon[70044]: pgmap v20280: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:44:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:44:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:44:23.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:44:23 smithi083 ceph-mon[79930]: pgmap v20280: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:23.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:44:23 smithi083 ceph-mon[77242]: pgmap v20280: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:24.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:44:24 smithi083 ceph-mon[79930]: pgmap v20281: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:24.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:44:24 smithi083 ceph-mon[77242]: pgmap v20281: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:25.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:44:24 smithi153 ceph-mon[70044]: pgmap v20281: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:26.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:44:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:44:26.050Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:44:27.177 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:44:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:44:26.805Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:44:27.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:44:27 smithi083 ceph-mon[79930]: pgmap v20282: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:27.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:44:27 smithi083 ceph-mon[77242]: pgmap v20282: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:27.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:44:27 smithi153 ceph-mon[70044]: pgmap v20282: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:29.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:44:29 smithi083 ceph-mon[79930]: pgmap v20283: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:29.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:44:29 smithi083 ceph-mon[77242]: pgmap v20283: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:29.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:44:29 smithi153 ceph-mon[70044]: pgmap v20283: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:31.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:44:31 smithi083 ceph-mon[79930]: pgmap v20284: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:31.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:44:31 smithi083 ceph-mon[77242]: pgmap v20284: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:31.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:44:31 smithi153 ceph-mon[70044]: pgmap v20284: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:31.966 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:44:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:44:31] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T16:44:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:44:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:44:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:44:33.186 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:44:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:44:32.711Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:44:33.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:44:33 smithi153 ceph-mon[70044]: pgmap v20285: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:44:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:44:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:44:33.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:44:33 smithi083 ceph-mon[79930]: pgmap v20285: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:33.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:44:33 smithi083 ceph-mon[77242]: pgmap v20285: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:34.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:44:34 smithi083 ceph-mon[77242]: pgmap v20286: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:34.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:44:34 smithi083 ceph-mon[79930]: pgmap v20286: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:35.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:44:34 smithi153 ceph-mon[70044]: pgmap v20286: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:36.363 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:44:36.460 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:44:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:44:36.052Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:44:36.652 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:44:36.653 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 2m ago 11h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:44:36.653 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 52s ago 11h 64.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:44:36.654 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 2m ago 11h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:44:36.655 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 52s ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:44:36.655 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (11h) 52s ago 11h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:44:36.655 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11h) 2m ago 11h 1084M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:44:36.656 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11h) 2m ago 11h 2437M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:44:36.656 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (11h) 52s ago 11h 1093M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:44:36.657 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (11h) 2m ago 11h 1469M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:44:36.657 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 2m ago 11h 24.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:44:36.658 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 52s ago 11h 26.6M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:44:36.658 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11h) 2m ago 11h 1777M 2049M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:44:36.658 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11h) 2m ago 11h 1763M 2049M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:44:36.659 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11h) 2m ago 11h 1369M 2049M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:44:36.659 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (11h) 2m ago 11h 1728M 2049M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:44:36.659 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (11h) 52s ago 11h 1514M 2486M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:44:36.659 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (11h) 52s ago 11h 1903M 2486M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:44:36.660 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (11h) 52s ago 11h 1701M 2486M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:44:36.660 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (11h) 52s ago 11h 2025M 2486M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:44:36.660 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (11h) 52s ago 11h 196M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:44:37.007 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:44:37.008 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:44:37.008 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:44:37.008 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:44:37.009 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:44:37.009 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:44:37.009 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:44:37.009 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:44:37.009 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:44:37.010 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:44:37.010 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:44:37.010 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:44:37.011 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:44:37.011 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:44:37.011 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:44:37.011 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:44:37.013 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:44:37.013 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:44:37.179 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:44:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:44:36.805Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:44:37.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:44:37 smithi083 ceph-mon[79930]: pgmap v20287: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:37.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:44:37 smithi083 ceph-mon[79930]: from='client.57720 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:44:37.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:44:37 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3268834173' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:44:37.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:44:37 smithi083 ceph-mon[77242]: pgmap v20287: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:37.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:44:37 smithi083 ceph-mon[77242]: from='client.57720 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:44:37.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:44:37 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3268834173' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:44:37.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:44:37 smithi153 ceph-mon[70044]: pgmap v20287: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:37.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:44:37 smithi153 ceph-mon[70044]: from='client.57720 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:44:37.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:44:37 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3268834173' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:44:38.460 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:44:38 smithi083 ceph-mon[77242]: from='client.57726 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:44:38.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:44:38 smithi083 ceph-mon[79930]: from='client.57726 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:44:38.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:44:38 smithi153 ceph-mon[70044]: from='client.57726 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:44:39.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:44:39 smithi083 ceph-mon[77242]: pgmap v20288: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:39.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:44:39 smithi083 ceph-mon[79930]: pgmap v20288: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:39.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:44:39 smithi153 ceph-mon[70044]: pgmap v20288: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:41.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:44:41 smithi083 ceph-mon[77242]: pgmap v20289: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:41.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:44:41 smithi083 ceph-mon[79930]: pgmap v20289: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:44:41 smithi153 ceph-mon[70044]: pgmap v20289: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:41.964 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:44:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:44:41] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T16:44:42.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:44:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:44:42.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:44:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:44:42.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:44:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:44:42.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:44:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:44:42.467 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:44:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:44:42.467 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:44:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:44:42.467 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:44:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:44:42.467 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:44:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:44:42.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:44:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:44:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:44:42.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:44:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:44:42.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:44:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:44:42.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:44:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:44:42.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:44:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:44:43.198 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:44:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:44:42.712Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:44:43.198 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:44:43 smithi083 ceph-mon[77242]: pgmap v20290: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:43.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:44:43 smithi153 ceph-mon[70044]: pgmap v20290: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:44:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:44:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:44:43.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:44:43 smithi083 ceph-mon[79930]: pgmap v20290: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:44.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:44:44 smithi083 ceph-mon[79930]: pgmap v20291: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:44.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:44:44 smithi083 ceph-mon[77242]: pgmap v20291: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:45.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:44:44 smithi153 ceph-mon[70044]: pgmap v20291: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:45.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:44:45 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:44:45.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:44:45 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:44:45.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:44:45 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:44:45.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:44:45 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:44:45.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:44:45 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:44:45.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:44:45 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:44:45.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:44:45 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:44:45.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:44:45 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:44:46.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:44:45 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:44:46.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:44:45 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:44:46.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:44:45 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:44:46.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:44:45 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:44:46.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:44:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:44:46.052Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:44:46.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:44:46 smithi083 ceph-mon[77242]: pgmap v20292: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:46.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:44:46 smithi083 ceph-mon[79930]: pgmap v20292: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:46.963 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:44:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:44:46.806Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:44:47.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:44:46 smithi153 ceph-mon[70044]: pgmap v20292: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:49.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:44:49 smithi083 ceph-mon[79930]: pgmap v20293: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:49.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:44:49 smithi083 ceph-mon[77242]: pgmap v20293: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:49.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:44:49 smithi153 ceph-mon[70044]: pgmap v20293: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:51.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:44:51 smithi083 ceph-mon[79930]: pgmap v20294: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:51.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:44:51 smithi083 ceph-mon[77242]: pgmap v20294: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:51.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:44:51 smithi153 ceph-mon[70044]: pgmap v20294: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:51.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:44:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:44:51] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T16:44:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:44:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:44:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:44:53.189 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:44:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:44:52.713Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:44:53.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:44:53 smithi153 ceph-mon[70044]: pgmap v20295: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:44:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:44:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:44:53.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:44:53 smithi083 ceph-mon[79930]: pgmap v20295: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:53.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:44:53 smithi083 ceph-mon[77242]: pgmap v20295: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:54.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:44:54 smithi083 ceph-mon[79930]: pgmap v20296: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:54.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:44:54 smithi083 ceph-mon[77242]: pgmap v20296: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:55.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:44:54 smithi153 ceph-mon[70044]: pgmap v20296: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:56.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:44:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:44:56.053Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:44:57.190 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:44:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:44:56.806Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:44:57.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:44:57 smithi083 ceph-mon[77242]: pgmap v20297: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:57.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:44:57 smithi083 ceph-mon[79930]: pgmap v20297: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:57.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:44:57 smithi153 ceph-mon[70044]: pgmap v20297: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:59.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:44:59 smithi083 ceph-mon[77242]: pgmap v20298: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:59.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:44:59 smithi083 ceph-mon[79930]: pgmap v20298: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:44:59.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:44:59 smithi153 ceph-mon[70044]: pgmap v20298: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:01.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:45:01 smithi083 ceph-mon[79930]: pgmap v20299: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:01.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:45:01 smithi083 ceph-mon[77242]: pgmap v20299: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:01.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:45:01 smithi153 ceph-mon[70044]: pgmap v20299: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:45:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:45:01] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T16:45:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:45:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:45:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:45:03.195 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:45:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:45:02.714Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:45:03.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:45:03 smithi153 ceph-mon[70044]: pgmap v20300: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:45:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:45:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:45:03.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:45:03 smithi083 ceph-mon[79930]: pgmap v20300: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:03.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:45:03 smithi083 ceph-mon[77242]: pgmap v20300: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:04.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:45:04 smithi083 ceph-mon[79930]: pgmap v20301: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:04.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:45:04 smithi083 ceph-mon[77242]: pgmap v20301: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:05.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:45:04 smithi153 ceph-mon[70044]: pgmap v20301: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:06.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:45:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:45:06.054Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:45:07.188 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:45:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:45:06.807Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:45:07.336 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:45:07.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:45:07 smithi083 ceph-mon[79930]: pgmap v20302: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:07.467 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:45:07 smithi083 ceph-mon[77242]: pgmap v20302: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:07.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:45:07 smithi153 ceph-mon[70044]: pgmap v20302: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:07.621 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:45:07.621 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 3m ago 11h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:45:07.622 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 83s ago 11h 64.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:45:07.622 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 3m ago 11h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:45:07.622 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 83s ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:45:07.622 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (11h) 83s ago 11h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:45:07.623 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11h) 3m ago 11h 1084M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:45:07.623 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11h) 3m ago 11h 2437M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:45:07.623 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (11h) 83s ago 11h 1093M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:45:07.623 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (11h) 3m ago 11h 1469M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:45:07.624 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 3m ago 11h 24.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:45:07.624 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 83s ago 11h 26.6M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:45:07.624 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11h) 3m ago 11h 1777M 2049M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:45:07.624 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11h) 3m ago 11h 1763M 2049M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:45:07.625 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11h) 3m ago 11h 1369M 2049M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:45:07.625 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (11h) 3m ago 11h 1728M 2049M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:45:07.625 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (11h) 83s ago 11h 1514M 2486M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:45:07.626 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (11h) 83s ago 11h 1903M 2486M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:45:07.626 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (11h) 83s ago 11h 1701M 2486M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:45:07.626 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (11h) 83s ago 11h 2025M 2486M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:45:07.626 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (11h) 83s ago 11h 196M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:45:07.979 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:45:07.980 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:45:07.980 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:45:07.980 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:45:07.981 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:45:07.981 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:45:07.986 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:45:07.986 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:45:07.987 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:45:07.987 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:45:07.988 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:45:07.988 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:45:07.989 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:45:07.989 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:45:07.989 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:45:07.990 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:45:07.990 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:45:07.990 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:45:08.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:45:08 smithi083 ceph-mon[79930]: from='client.67669 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:45:08.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:45:08 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3927700009' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:45:08.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:45:08 smithi083 ceph-mon[77242]: from='client.67669 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:45:08.468 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:45:08 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3927700009' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:45:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:45:08 smithi153 ceph-mon[70044]: from='client.67669 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:45:08.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:45:08 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3927700009' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:45:09.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:45:09 smithi083 ceph-mon[79930]: from='client.67675 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:45:09.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:45:09 smithi083 ceph-mon[79930]: pgmap v20303: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:09.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:45:09 smithi083 ceph-mon[77242]: from='client.67675 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:45:09.467 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:45:09 smithi083 ceph-mon[77242]: pgmap v20303: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:09.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:45:09 smithi153 ceph-mon[70044]: from='client.67675 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:45:09.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:45:09 smithi153 ceph-mon[70044]: pgmap v20303: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:11.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:45:11 smithi083 ceph-mon[79930]: pgmap v20304: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:11.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:45:11 smithi083 ceph-mon[77242]: pgmap v20304: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:11.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:45:11 smithi153 ceph-mon[70044]: pgmap v20304: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:45:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:45:11] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T16:45:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:45:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:45:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:45:13.202 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:45:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:45:12.714Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:45:13.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:45:13 smithi153 ceph-mon[70044]: pgmap v20305: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:13.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:45:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:45:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:45:13.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:45:13 smithi083 ceph-mon[79930]: pgmap v20305: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:13.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:45:13 smithi083 ceph-mon[77242]: pgmap v20305: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:14.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:45:14 smithi083 ceph-mon[79930]: pgmap v20306: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:14.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:45:14 smithi083 ceph-mon[77242]: pgmap v20306: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:15.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:45:14 smithi153 ceph-mon[70044]: pgmap v20306: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:16.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:45:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:45:16.055Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:45:17.192 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:45:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:45:16.808Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:45:17.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:45:17 smithi083 ceph-mon[77242]: pgmap v20307: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:17.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:45:17 smithi083 ceph-mon[79930]: pgmap v20307: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:17.561 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:45:17 smithi153 ceph-mon[70044]: pgmap v20307: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:19.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:45:19 smithi083 ceph-mon[79930]: pgmap v20308: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:19.467 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:45:19 smithi083 ceph-mon[77242]: pgmap v20308: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:19.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:45:19 smithi153 ceph-mon[70044]: pgmap v20308: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:21.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:45:21 smithi083 ceph-mon[79930]: pgmap v20309: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:21.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:45:21 smithi083 ceph-mon[77242]: pgmap v20309: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:21.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:45:21 smithi153 ceph-mon[70044]: pgmap v20309: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:45:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:45:21] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T16:45:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:45:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:45:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:45:23.201 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:45:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:45:22.715Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:45:23.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:45:23 smithi153 ceph-mon[70044]: pgmap v20310: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:45:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:45:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:45:23.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:45:23 smithi083 ceph-mon[79930]: pgmap v20310: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:23.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:45:23 smithi083 ceph-mon[77242]: pgmap v20310: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:24.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:45:24 smithi083 ceph-mon[77242]: pgmap v20311: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:24.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:45:24 smithi083 ceph-mon[79930]: pgmap v20311: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:25.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:45:24 smithi153 ceph-mon[70044]: pgmap v20311: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:26.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:45:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:45:26.055Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:45:27.205 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:45:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:45:26.808Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:45:27.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:45:27 smithi083 ceph-mon[79930]: pgmap v20312: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:27.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:45:27 smithi083 ceph-mon[77242]: pgmap v20312: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:27.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:45:27 smithi153 ceph-mon[70044]: pgmap v20312: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:29.470 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:45:29 smithi083 ceph-mon[79930]: pgmap v20313: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:29.471 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:45:29 smithi083 ceph-mon[77242]: pgmap v20313: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:29.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:45:29 smithi153 ceph-mon[70044]: pgmap v20313: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:31.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:45:31 smithi083 ceph-mon[79930]: pgmap v20314: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:31.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:45:31 smithi083 ceph-mon[77242]: pgmap v20314: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:31.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:45:31 smithi153 ceph-mon[70044]: pgmap v20314: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:31.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:45:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:45:31] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T16:45:32.814 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:45:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:45:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:45:33.205 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:45:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:45:32.716Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:45:33.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:45:33 smithi153 ceph-mon[70044]: pgmap v20315: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:45:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:45:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:45:33.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:45:33 smithi083 ceph-mon[79930]: pgmap v20315: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:33.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:45:33 smithi083 ceph-mon[77242]: pgmap v20315: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:34.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:45:34 smithi083 ceph-mon[79930]: pgmap v20316: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:34.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:45:34 smithi083 ceph-mon[77242]: pgmap v20316: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:35.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:45:34 smithi153 ceph-mon[70044]: pgmap v20316: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:36.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:45:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:45:36.056Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:45:37.199 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:45:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:45:36.809Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:45:37.460 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:45:37 smithi083 ceph-mon[79930]: pgmap v20317: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:37.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:45:37 smithi083 ceph-mon[77242]: pgmap v20317: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:37.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:45:37 smithi153 ceph-mon[70044]: pgmap v20317: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:38.310 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:45:38.598 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:45:38.599 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 4m ago 11h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:45:38.599 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 114s ago 11h 64.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:45:38.600 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 4m ago 11h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:45:38.600 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 114s ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:45:38.601 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (11h) 114s ago 11h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:45:38.601 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11h) 4m ago 11h 1084M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:45:38.614 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11h) 4m ago 11h 2437M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:45:38.615 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (11h) 114s ago 11h 1093M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:45:38.616 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (11h) 4m ago 11h 1469M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:45:38.616 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 4m ago 11h 24.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:45:38.617 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 114s ago 11h 26.6M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:45:38.617 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11h) 4m ago 11h 1777M 2049M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:45:38.617 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11h) 4m ago 11h 1763M 2049M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:45:38.619 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11h) 4m ago 11h 1369M 2049M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:45:38.620 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (11h) 4m ago 11h 1728M 2049M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:45:38.620 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (11h) 114s ago 11h 1514M 2486M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:45:38.621 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (11h) 114s ago 11h 1903M 2486M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:45:38.623 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (11h) 114s ago 11h 1701M 2486M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:45:38.623 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (11h) 114s ago 11h 2025M 2486M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:45:38.623 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (11h) 114s ago 11h 196M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:45:38.953 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:45:38.954 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:45:38.954 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:45:38.955 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:45:38.955 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:45:38.955 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:45:38.955 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:45:38.955 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:45:38.956 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:45:38.956 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:45:38.957 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:45:38.957 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:45:38.957 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:45:38.957 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:45:38.958 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:45:38.958 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:45:38.958 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:45:38.958 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:45:39.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:45:39 smithi083 ceph-mon[79930]: pgmap v20318: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:39.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:45:39 smithi083 ceph-mon[79930]: from='client.67687 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:45:39.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:45:39 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1762731277' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:45:39.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:45:39 smithi083 ceph-mon[77242]: pgmap v20318: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:39.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:45:39 smithi083 ceph-mon[77242]: from='client.67687 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:45:39.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:45:39 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1762731277' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:45:39.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:45:39 smithi153 ceph-mon[70044]: pgmap v20318: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:39.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:45:39 smithi153 ceph-mon[70044]: from='client.67687 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:45:39.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:45:39 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1762731277' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:45:40.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:45:40 smithi153 ceph-mon[70044]: from='client.57762 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:45:40.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:45:40 smithi083 ceph-mon[79930]: from='client.57762 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:45:40.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:45:40 smithi083 ceph-mon[77242]: from='client.57762 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:45:41.506 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:45:41 smithi083 ceph-mon[79930]: pgmap v20319: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:41.507 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:45:41 smithi083 ceph-mon[77242]: pgmap v20319: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:45:41 smithi153 ceph-mon[70044]: pgmap v20319: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:45:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:45:41] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T16:45:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:45:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:45:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:45:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:45:42.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:45:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:45:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:45:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:45:42.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:45:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:45:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:45:42.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:45:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:45:42.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:45:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:45:42.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:45:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:45:42.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:45:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:45:42.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:45:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:45:42.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:45:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:45:42.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:45:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:45:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:45:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:45:43.214 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:45:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:45:42.717Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:45:43.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:45:43 smithi153 ceph-mon[70044]: pgmap v20320: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:45:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:45:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:45:43.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:45:43 smithi083 ceph-mon[79930]: pgmap v20320: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:43.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:45:43 smithi083 ceph-mon[77242]: pgmap v20320: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:44.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:45:44 smithi083 ceph-mon[79930]: pgmap v20321: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:44.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:45:44 smithi083 ceph-mon[77242]: pgmap v20321: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:45.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:45:44 smithi153 ceph-mon[70044]: pgmap v20321: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:45.960 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:45:45 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:45:45.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:45:45 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:45:45.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:45:45 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:45:45.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:45:45 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:45:45.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:45:45 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:45:45.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:45:45 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:45:46.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:45:45 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:45:46.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:45:45 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:45:46.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:45:45 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:45:46.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:45:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:45:46.055Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:45:47.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:45:46 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:45:47.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:45:46 smithi083 ceph-mon[79930]: pgmap v20322: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:47.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:45:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:45:46.809Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:45:47.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:45:46 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:45:47.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:45:46 smithi083 ceph-mon[77242]: pgmap v20322: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:47.312 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:45:46 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:45:47.312 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:45:46 smithi153 ceph-mon[70044]: pgmap v20322: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:49.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:45:49 smithi083 ceph-mon[77242]: pgmap v20323: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:49.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:45:49 smithi083 ceph-mon[79930]: pgmap v20323: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:49.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:45:49 smithi153 ceph-mon[70044]: pgmap v20323: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:51.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:45:51 smithi083 ceph-mon[79930]: pgmap v20324: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:51.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:45:51 smithi083 ceph-mon[77242]: pgmap v20324: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:51.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:45:51 smithi153 ceph-mon[70044]: pgmap v20324: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:45:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:45:51] "GET /metrics HTTP/1.1" 200 225237 "" "Prometheus/2.33.4" 2022-09-29T16:45:52.818 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:45:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:45:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:45:53.208 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:45:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:45:52.717Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:45:53.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:45:53 smithi153 ceph-mon[70044]: pgmap v20325: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:45:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:45:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:45:53.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:45:53 smithi083 ceph-mon[79930]: pgmap v20325: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:53.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:45:53 smithi083 ceph-mon[77242]: pgmap v20325: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:54.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:45:54 smithi083 ceph-mon[79930]: pgmap v20326: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:54.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:45:54 smithi083 ceph-mon[77242]: pgmap v20326: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:55.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:45:54 smithi153 ceph-mon[70044]: pgmap v20326: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:56.466 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:45:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:45:56.056Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:45:57.205 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:45:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:45:56.810Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:45:57.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:45:57 smithi083 ceph-mon[77242]: pgmap v20327: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:57.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:45:57 smithi083 ceph-mon[79930]: pgmap v20327: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:57.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:45:57 smithi153 ceph-mon[70044]: pgmap v20327: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:59.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:45:59 smithi083 ceph-mon[77242]: pgmap v20328: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:59.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:45:59 smithi083 ceph-mon[79930]: pgmap v20328: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:45:59.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:45:59 smithi153 ceph-mon[70044]: pgmap v20328: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:01.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:46:01 smithi083 ceph-mon[79930]: pgmap v20329: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:01.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:46:01 smithi083 ceph-mon[77242]: pgmap v20329: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:01.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:46:01 smithi153 ceph-mon[70044]: pgmap v20329: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:46:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:46:01] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T16:46:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:46:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:46:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:46:03.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:46:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:46:02.717Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:46:03.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:46:03 smithi153 ceph-mon[70044]: pgmap v20330: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:46:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:46:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:46:03.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:46:03 smithi083 ceph-mon[77242]: pgmap v20330: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:03.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:46:03 smithi083 ceph-mon[79930]: pgmap v20330: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:04.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:46:04 smithi083 ceph-mon[79930]: pgmap v20331: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:04.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:46:04 smithi083 ceph-mon[77242]: pgmap v20331: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:05.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:46:04 smithi153 ceph-mon[70044]: pgmap v20331: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:06.466 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:46:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:46:06.057Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:46:07.210 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:46:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:46:06.811Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:46:07.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:46:07 smithi083 ceph-mon[79930]: pgmap v20332: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:07.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:46:07 smithi083 ceph-mon[77242]: pgmap v20332: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:07.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:46:07 smithi153 ceph-mon[70044]: pgmap v20332: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:09.280 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:46:09.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:46:09 smithi153 ceph-mon[70044]: pgmap v20333: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:09.568 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:46:09.569 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 4m ago 11h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:46:09.569 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 2m ago 11h 64.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:46:09.570 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 4m ago 11h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:46:09.570 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 2m ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:46:09.572 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (11h) 2m ago 11h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:46:09.572 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11h) 4m ago 11h 1084M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:46:09.572 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11h) 4m ago 11h 2437M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:46:09.572 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (11h) 2m ago 11h 1093M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:46:09.573 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (11h) 4m ago 11h 1469M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:46:09.573 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 4m ago 11h 24.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:46:09.573 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 2m ago 11h 26.6M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:46:09.573 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11h) 4m ago 11h 1777M 2049M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:46:09.573 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11h) 4m ago 11h 1763M 2049M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:46:09.574 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11h) 4m ago 11h 1369M 2049M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:46:09.574 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (11h) 4m ago 11h 1728M 2049M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:46:09.574 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (11h) 2m ago 11h 1514M 2486M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:46:09.574 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (11h) 2m ago 11h 1903M 2486M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:46:09.575 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (11h) 2m ago 11h 1701M 2486M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:46:09.575 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (11h) 2m ago 11h 2025M 2486M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:46:09.575 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (11h) 2m ago 11h 196M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:46:09.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:46:09 smithi083 ceph-mon[79930]: pgmap v20333: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:09.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:46:09 smithi083 ceph-mon[77242]: pgmap v20333: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:09.947 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:46:09.948 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:46:09.948 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:46:09.949 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:46:09.949 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:46:09.949 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:46:09.949 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:46:09.950 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:46:09.950 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:46:09.950 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:46:09.951 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:46:09.951 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:46:09.951 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:46:09.952 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:46:09.952 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:46:09.953 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:46:09.954 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:46:09.954 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:46:10.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:46:10 smithi153 ceph-mon[70044]: from='client.57774 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:46:10.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:46:10 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2931018990' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:46:10.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:46:10 smithi083 ceph-mon[79930]: from='client.57774 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:46:10.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:46:10 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2931018990' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:46:10.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:46:10 smithi083 ceph-mon[77242]: from='client.57774 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:46:10.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:46:10 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2931018990' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:46:11.506 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:46:11 smithi083 ceph-mon[79930]: from='client.67711 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:46:11.507 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:46:11 smithi083 ceph-mon[79930]: pgmap v20334: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:11.507 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:46:11 smithi083 ceph-mon[77242]: from='client.67711 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:46:11.516 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:46:11 smithi083 ceph-mon[77242]: pgmap v20334: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:11.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:46:11 smithi153 ceph-mon[70044]: from='client.67711 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:46:11.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:46:11 smithi153 ceph-mon[70044]: pgmap v20334: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:46:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:46:11] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T16:46:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:46:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:46:12.317Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:46:13.210 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:46:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:46:12.718Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:46:13.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:46:13 smithi153 ceph-mon[70044]: pgmap v20335: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:13.311 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:46:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:46:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:46:13.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:46:13 smithi083 ceph-mon[79930]: pgmap v20335: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:13.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:46:13 smithi083 ceph-mon[77242]: pgmap v20335: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:14.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:46:14 smithi083 ceph-mon[79930]: pgmap v20336: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:14.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:46:14 smithi083 ceph-mon[77242]: pgmap v20336: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:15.072 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:46:14 smithi153 ceph-mon[70044]: pgmap v20336: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:16.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:46:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:46:16.059Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:46:17.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:46:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:46:16.812Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:46:17.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:46:17 smithi153 ceph-mon[70044]: pgmap v20337: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:17.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:46:17 smithi083 ceph-mon[79930]: pgmap v20337: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:17.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:46:17 smithi083 ceph-mon[77242]: pgmap v20337: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:19.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:46:19 smithi153 ceph-mon[70044]: pgmap v20338: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:19.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:46:19 smithi083 ceph-mon[79930]: pgmap v20338: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:19.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:46:19 smithi083 ceph-mon[77242]: pgmap v20338: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:21.508 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:46:21 smithi083 ceph-mon[77242]: pgmap v20339: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:21.508 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:46:21 smithi083 ceph-mon[79930]: pgmap v20339: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:21.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:46:21 smithi153 ceph-mon[70044]: pgmap v20339: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:46:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:46:21] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T16:46:22.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:46:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:46:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:46:23.215 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:46:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:46:22.719Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:46:23.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:46:23 smithi153 ceph-mon[70044]: pgmap v20340: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:46:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:46:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:46:23.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:46:23 smithi083 ceph-mon[77242]: pgmap v20340: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:23.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:46:23 smithi083 ceph-mon[79930]: pgmap v20340: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:24.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:46:24 smithi083 ceph-mon[77242]: pgmap v20341: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:24.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:46:24 smithi083 ceph-mon[79930]: pgmap v20341: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:25.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:46:24 smithi153 ceph-mon[70044]: pgmap v20341: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:26.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:46:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:46:26.060Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:46:27.214 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:46:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:46:26.812Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:46:27.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:46:27 smithi153 ceph-mon[70044]: pgmap v20342: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:27.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:46:27 smithi083 ceph-mon[77242]: pgmap v20342: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:27.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:46:27 smithi083 ceph-mon[79930]: pgmap v20342: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:29.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:46:29 smithi153 ceph-mon[70044]: pgmap v20343: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:29.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:46:29 smithi083 ceph-mon[77242]: pgmap v20343: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:29.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:46:29 smithi083 ceph-mon[79930]: pgmap v20343: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:31.507 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:46:31 smithi083 ceph-mon[79930]: pgmap v20344: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:31.508 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:46:31 smithi083 ceph-mon[77242]: pgmap v20344: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:31.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:46:31 smithi153 ceph-mon[70044]: pgmap v20344: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:46:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:46:31] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T16:46:32.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:46:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:46:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:46:33.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:46:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:46:32.720Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:46:33.316 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:46:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:46:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:46:33.317 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:46:33 smithi153 ceph-mon[70044]: pgmap v20345: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:33.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:46:33 smithi083 ceph-mon[79930]: pgmap v20345: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:33.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:46:33 smithi083 ceph-mon[77242]: pgmap v20345: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:34.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:46:34 smithi083 ceph-mon[79930]: pgmap v20346: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:34.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:46:34 smithi083 ceph-mon[77242]: pgmap v20346: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:35.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:46:34 smithi153 ceph-mon[70044]: pgmap v20346: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:36.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:46:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:46:36.059Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:46:37.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:46:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:46:36.813Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:46:37.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:46:37 smithi153 ceph-mon[70044]: pgmap v20347: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:37.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:46:37 smithi083 ceph-mon[79930]: pgmap v20347: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:37.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:46:37 smithi083 ceph-mon[77242]: pgmap v20347: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:39.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:46:39 smithi153 ceph-mon[70044]: pgmap v20348: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:39.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:46:39 smithi083 ceph-mon[79930]: pgmap v20348: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:39.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:46:39 smithi083 ceph-mon[77242]: pgmap v20348: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:40.275 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:46:40.564 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:46:40.564 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 5m ago 11h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:46:40.564 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 2m ago 11h 64.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:46:40.564 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 5m ago 11h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:46:40.565 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 2m ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:46:40.565 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (11h) 2m ago 11h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:46:40.565 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11h) 5m ago 11h 1084M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:46:40.565 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11h) 5m ago 11h 2437M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:46:40.566 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (11h) 2m ago 11h 1093M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:46:40.566 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (11h) 5m ago 11h 1469M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:46:40.566 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 5m ago 11h 24.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:46:40.566 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 2m ago 11h 26.6M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:46:40.567 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11h) 5m ago 11h 1777M 2049M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:46:40.567 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11h) 5m ago 11h 1763M 2049M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:46:40.567 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11h) 5m ago 11h 1369M 2049M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:46:40.567 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (11h) 5m ago 11h 1728M 2049M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:46:40.568 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (11h) 2m ago 11h 1514M 2486M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:46:40.568 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (11h) 2m ago 11h 1903M 2486M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:46:40.568 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (11h) 2m ago 11h 1701M 2486M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:46:40.569 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (11h) 2m ago 11h 2025M 2486M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:46:40.569 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (11h) 2m ago 11h 196M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:46:40.922 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:46:40.922 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:46:40.923 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:46:40.923 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:46:40.923 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:46:40.923 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:46:40.924 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:46:40.924 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:46:40.924 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:46:40.924 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:46:40.924 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:46:40.925 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:46:40.925 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:46:40.925 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:46:40.925 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:46:40.926 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:46:40.926 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:46:40.927 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:46:41.511 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:46:41 smithi083 ceph-mon[77242]: pgmap v20349: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:41.512 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:46:41 smithi083 ceph-mon[77242]: from='client.57792 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:46:41.512 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:46:41 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2093054044' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:46:41.513 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:46:41 smithi083 ceph-mon[79930]: pgmap v20349: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:41.513 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:46:41 smithi083 ceph-mon[79930]: from='client.57792 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:46:41.513 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:46:41 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2093054044' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:46:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:46:41 smithi153 ceph-mon[70044]: pgmap v20349: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:46:41 smithi153 ceph-mon[70044]: from='client.57792 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:46:41.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:46:41 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2093054044' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:46:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:46:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:46:41] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T16:46:42.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:46:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:46:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:46:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:46:42 smithi153 ceph-mon[70044]: from='client.57798 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:46:42.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:46:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:46:42.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:46:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:46:42.561 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:46:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:46:42.561 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:46:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:46:42.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:46:42 smithi083 ceph-mon[77242]: from='client.57798 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:46:42.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:46:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:46:42.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:46:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:46:42.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:46:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:46:42.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:46:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:46:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:46:42 smithi083 ceph-mon[79930]: from='client.57798 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:46:42.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:46:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:46:42.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:46:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:46:42.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:46:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:46:42.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:46:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:46:43.215 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:46:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:46:42.720Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:46:43.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:46:43 smithi153 ceph-mon[70044]: pgmap v20350: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:43.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:46:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:46:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:46:43.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:46:43 smithi083 ceph-mon[79930]: pgmap v20350: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:43.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:46:43 smithi083 ceph-mon[77242]: pgmap v20350: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:44.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:46:44 smithi083 ceph-mon[79930]: pgmap v20351: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:44.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:46:44 smithi083 ceph-mon[77242]: pgmap v20351: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:45.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:46:44 smithi153 ceph-mon[70044]: pgmap v20351: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:46.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:46:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:46:46.060Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:46:46.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:46:46 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:46:46.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:46:46 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:46:46.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:46:46 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:46:46.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:46:46 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:46:46.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:46:46 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:46:46.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:46:46 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:46:46.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:46:46 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:46:46.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:46:46 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:46:46.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:46:46 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:46:47.083 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:46:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:46:46.812Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:46:47.083 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:46:47 smithi083 ceph-mon[77242]: pgmap v20352: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:47.084 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:46:47 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:46:47.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:46:47 smithi083 ceph-mon[79930]: pgmap v20352: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:47.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:46:47 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:46:47.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:46:47 smithi153 ceph-mon[70044]: pgmap v20352: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:47.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:46:47 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:46:49.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:46:49 smithi153 ceph-mon[70044]: pgmap v20353: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:49.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:46:49 smithi083 ceph-mon[79930]: pgmap v20353: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:49.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:46:49 smithi083 ceph-mon[77242]: pgmap v20353: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:51.506 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:46:51 smithi083 ceph-mon[77242]: pgmap v20354: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:51.507 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:46:51 smithi083 ceph-mon[79930]: pgmap v20354: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:51.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:46:51 smithi153 ceph-mon[70044]: pgmap v20354: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:51.960 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:46:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:46:51] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T16:46:52.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:46:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:46:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:46:53.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:46:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:46:52.720Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:46:53.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:46:53 smithi153 ceph-mon[70044]: pgmap v20355: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:46:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:46:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:46:53.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:46:53 smithi083 ceph-mon[77242]: pgmap v20355: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:53.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:46:53 smithi083 ceph-mon[79930]: pgmap v20355: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:54.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:46:54 smithi083 ceph-mon[79930]: pgmap v20356: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:54.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:46:54 smithi083 ceph-mon[77242]: pgmap v20356: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:55.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:46:54 smithi153 ceph-mon[70044]: pgmap v20356: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:56.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:46:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:46:56.060Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:46:57.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:46:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:46:56.814Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:46:57.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:46:57 smithi153 ceph-mon[70044]: pgmap v20357: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:57.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:46:57 smithi083 ceph-mon[79930]: pgmap v20357: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:57.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:46:57 smithi083 ceph-mon[77242]: pgmap v20357: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:59.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:46:59 smithi153 ceph-mon[70044]: pgmap v20358: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:59.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:46:59 smithi083 ceph-mon[79930]: pgmap v20358: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:46:59.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:46:59 smithi083 ceph-mon[77242]: pgmap v20358: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:01.514 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:47:01 smithi083 ceph-mon[79930]: pgmap v20359: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:01.515 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:47:01 smithi083 ceph-mon[77242]: pgmap v20359: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:01.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:47:01 smithi153 ceph-mon[70044]: pgmap v20359: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:01.965 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:47:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:47:01] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T16:47:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:47:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:47:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:47:03.210 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:47:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:47:02.721Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:47:03.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:47:03 smithi153 ceph-mon[70044]: pgmap v20360: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:47:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:47:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:47:03.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:47:03 smithi083 ceph-mon[79930]: pgmap v20360: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:03.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:47:03 smithi083 ceph-mon[77242]: pgmap v20360: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:04.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:47:04 smithi083 ceph-mon[77242]: pgmap v20361: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:04.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:47:04 smithi083 ceph-mon[79930]: pgmap v20361: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:05.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:47:04 smithi153 ceph-mon[70044]: pgmap v20361: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:06.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:47:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:47:06.060Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:47:07.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:47:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:47:06.814Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:47:07.572 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:47:07 smithi153 ceph-mon[70044]: pgmap v20362: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:07.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:47:07 smithi083 ceph-mon[77242]: pgmap v20362: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:07.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:47:07 smithi083 ceph-mon[79930]: pgmap v20362: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:09.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:47:09 smithi153 ceph-mon[70044]: pgmap v20363: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:09.720 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:47:09 smithi083 ceph-mon[77242]: pgmap v20363: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:09.721 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:47:09 smithi083 ceph-mon[79930]: pgmap v20363: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:11.249 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:47:11.508 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:47:11 smithi083 ceph-mon[79930]: pgmap v20364: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:11.509 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:47:11 smithi083 ceph-mon[77242]: pgmap v20364: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:11.542 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:47:11.542 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 5m ago 11h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:47:11.543 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 3m ago 11h 64.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:47:11.545 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 5m ago 11h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:47:11.545 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 3m ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:47:11.545 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (11h) 3m ago 11h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:47:11.546 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11h) 5m ago 11h 1084M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:47:11.546 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11h) 5m ago 11h 2437M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:47:11.547 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (11h) 3m ago 11h 1093M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:47:11.547 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (11h) 5m ago 11h 1469M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:47:11.548 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 5m ago 11h 24.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:47:11.548 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 3m ago 11h 26.6M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:47:11.548 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11h) 5m ago 11h 1777M 2049M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:47:11.549 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11h) 5m ago 11h 1763M 2049M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:47:11.549 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11h) 5m ago 11h 1369M 2049M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:47:11.550 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (11h) 5m ago 11h 1728M 2049M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:47:11.572 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (11h) 3m ago 11h 1514M 2486M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:47:11.573 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (11h) 3m ago 11h 1903M 2486M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:47:11.573 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (11h) 3m ago 11h 1701M 2486M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:47:11.573 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (11h) 3m ago 11h 2025M 2486M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:47:11.574 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (11h) 3m ago 11h 196M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:47:11.575 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:47:11 smithi153 ceph-mon[70044]: pgmap v20364: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:11.897 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:47:11.897 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:47:11.898 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:47:11.899 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:47:11.899 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:47:11.899 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:47:11.900 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:47:11.900 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:47:11.901 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:47:11.902 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:47:11.902 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:47:11.903 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:47:11.913 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:47:11.914 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:47:11.914 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:47:11.914 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:47:11.915 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:47:11.915 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:47:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:47:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:47:11] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T16:47:12.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:47:12 smithi153 ceph-mon[70044]: from='client.67741 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:47:12.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:47:12 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2750043911' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:47:12.558 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:47:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:47:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:47:12.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:47:12 smithi083 ceph-mon[79930]: from='client.67741 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:47:12.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:47:12 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2750043911' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:47:12.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:47:12 smithi083 ceph-mon[77242]: from='client.67741 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:47:12.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:47:12 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2750043911' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:47:13.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:47:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:47:12.721Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:47:13.310 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:47:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:47:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:47:13.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:47:13 smithi153 ceph-mon[70044]: from='client.57816 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:47:13.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:47:13 smithi153 ceph-mon[70044]: pgmap v20365: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:13.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:47:13 smithi083 ceph-mon[79930]: from='client.57816 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:47:13.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:47:13 smithi083 ceph-mon[79930]: pgmap v20365: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:13.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:47:13 smithi083 ceph-mon[77242]: from='client.57816 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:47:13.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:47:13 smithi083 ceph-mon[77242]: pgmap v20365: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:14.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:47:14 smithi083 ceph-mon[77242]: pgmap v20366: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:14.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:47:14 smithi083 ceph-mon[79930]: pgmap v20366: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:15.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:47:14 smithi153 ceph-mon[70044]: pgmap v20366: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:16.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:47:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:47:16.061Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:47:17.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:47:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:47:16.815Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:47:17.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:47:17 smithi153 ceph-mon[70044]: pgmap v20367: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:17.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:47:17 smithi083 ceph-mon[77242]: pgmap v20367: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:17.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:47:17 smithi083 ceph-mon[79930]: pgmap v20367: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:19.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:47:19 smithi153 ceph-mon[70044]: pgmap v20368: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:19.718 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:47:19 smithi083 ceph-mon[79930]: pgmap v20368: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:19.719 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:47:19 smithi083 ceph-mon[77242]: pgmap v20368: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:21.509 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:47:21 smithi083 ceph-mon[79930]: pgmap v20369: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:21.510 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:47:21 smithi083 ceph-mon[77242]: pgmap v20369: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:21.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:47:21 smithi153 ceph-mon[70044]: pgmap v20369: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:47:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:47:21] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T16:47:22.812 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:47:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:47:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:47:23.216 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:47:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:47:22.722Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:47:23.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:47:23 smithi153 ceph-mon[70044]: pgmap v20370: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:47:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:47:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:47:23.710 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:47:23 smithi083 ceph-mon[79930]: pgmap v20370: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:23.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:47:23 smithi083 ceph-mon[77242]: pgmap v20370: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:25.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:47:24 smithi153 ceph-mon[70044]: pgmap v20371: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:25.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:47:24 smithi083 ceph-mon[79930]: pgmap v20371: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:25.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:47:24 smithi083 ceph-mon[77242]: pgmap v20371: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:26.467 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:47:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:47:26.062Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:47:27.218 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:47:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:47:26.816Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:47:27.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:47:27 smithi153 ceph-mon[70044]: pgmap v20372: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:27.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:47:27 smithi083 ceph-mon[77242]: pgmap v20372: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:27.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:47:27 smithi083 ceph-mon[79930]: pgmap v20372: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:29.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:47:29 smithi153 ceph-mon[70044]: pgmap v20373: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:29.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:47:29 smithi083 ceph-mon[77242]: pgmap v20373: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:29.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:47:29 smithi083 ceph-mon[79930]: pgmap v20373: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:31.508 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:47:31 smithi083 ceph-mon[79930]: pgmap v20374: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:31.509 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:47:31 smithi083 ceph-mon[77242]: pgmap v20374: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:31.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:47:31 smithi153 ceph-mon[70044]: pgmap v20374: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:47:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:47:31] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T16:47:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:47:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:47:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:47:33.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:47:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:47:32.723Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:47:33.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:47:33 smithi153 ceph-mon[70044]: pgmap v20375: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:47:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:47:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:47:33.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:47:33 smithi083 ceph-mon[77242]: pgmap v20375: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:33.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:47:33 smithi083 ceph-mon[79930]: pgmap v20375: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:35.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:47:34 smithi153 ceph-mon[70044]: pgmap v20376: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:35.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:47:34 smithi083 ceph-mon[77242]: pgmap v20376: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:35.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:47:34 smithi083 ceph-mon[79930]: pgmap v20376: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:36.467 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:47:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:47:36.064Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:47:37.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:47:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:47:36.816Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:47:37.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:47:37 smithi153 ceph-mon[70044]: pgmap v20377: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:37.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:47:37 smithi083 ceph-mon[77242]: pgmap v20377: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:37.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:47:37 smithi083 ceph-mon[79930]: pgmap v20377: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:39.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:47:39 smithi153 ceph-mon[70044]: pgmap v20378: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:39.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:47:39 smithi083 ceph-mon[79930]: pgmap v20378: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:39.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:47:39 smithi083 ceph-mon[77242]: pgmap v20378: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:41.513 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:47:41 smithi083 ceph-mon[77242]: pgmap v20379: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:41.513 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:47:41 smithi083 ceph-mon[79930]: pgmap v20379: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:41.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:47:41 smithi153 ceph-mon[70044]: pgmap v20379: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:47:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:47:41] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T16:47:42.226 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:47:42.522 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:47:42.522 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 6m ago 11h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:47:42.523 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 3m ago 11h 64.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:47:42.523 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 6m ago 11h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:47:42.524 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 3m ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:47:42.524 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (11h) 3m ago 11h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:47:42.530 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11h) 6m ago 11h 1084M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:47:42.531 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11h) 6m ago 11h 2437M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:47:42.532 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (11h) 3m ago 11h 1093M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:47:42.533 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (11h) 6m ago 11h 1469M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:47:42.533 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 6m ago 11h 24.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:47:42.534 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 3m ago 11h 26.6M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:47:42.534 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11h) 6m ago 11h 1777M 2049M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:47:42.535 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11h) 6m ago 11h 1763M 2049M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:47:42.535 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11h) 6m ago 11h 1369M 2049M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:47:42.537 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (11h) 6m ago 11h 1728M 2049M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:47:42.537 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (11h) 3m ago 11h 1514M 2486M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:47:42.538 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (11h) 3m ago 11h 1903M 2486M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:47:42.538 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (11h) 3m ago 11h 1701M 2486M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:47:42.539 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (11h) 3m ago 11h 2025M 2486M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:47:42.540 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (11h) 3m ago 11h 196M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:47:42.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:47:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:47:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:47:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:47:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:47:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:47:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:47:42.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:47:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:47:42.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:47:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:47:42.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:47:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:47:42.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:47:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:47:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:47:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:47:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:47:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:47:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:47:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:47:42.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:47:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:47:42.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:47:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:47:42.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:47:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:47:42.885 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:47:42.886 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:47:42.886 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:47:42.887 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:47:42.887 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:47:42.888 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:47:42.888 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:47:42.888 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:47:42.889 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:47:42.899 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:47:42.899 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:47:42.899 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:47:42.900 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:47:42.901 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:47:42.901 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:47:42.902 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:47:42.902 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:47:42.903 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:47:43.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:47:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:47:42.722Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:47:43.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:47:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:47:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:47:43.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:47:43 smithi153 ceph-mon[70044]: pgmap v20380: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:43.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:47:43 smithi153 ceph-mon[70044]: from='client.57828 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:47:43.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:47:43 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2747525162' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:47:43.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:47:43 smithi083 ceph-mon[77242]: pgmap v20380: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:43.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:47:43 smithi083 ceph-mon[77242]: from='client.57828 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:47:43.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:47:43 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2747525162' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:47:43.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:47:43 smithi083 ceph-mon[79930]: pgmap v20380: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:43.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:47:43 smithi083 ceph-mon[79930]: from='client.57828 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:47:43.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:47:43 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2747525162' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:47:44.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:47:44 smithi153 ceph-mon[70044]: from='client.67765 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:47:44.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:47:44 smithi083 ceph-mon[79930]: from='client.67765 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:47:44.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:47:44 smithi083 ceph-mon[77242]: from='client.67765 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:47:45.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:47:45 smithi153 ceph-mon[70044]: pgmap v20381: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:45.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:47:45 smithi083 ceph-mon[79930]: pgmap v20381: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:45.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:47:45 smithi083 ceph-mon[77242]: pgmap v20381: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:46.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:47:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:47:46.064Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:47:47.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:47:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:47:46.816Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:47:47.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:47:47 smithi153 ceph-mon[70044]: pgmap v20382: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:47.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:47:47 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:47:47.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:47:47 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:47:47.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:47:47 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:47:47.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:47:47 smithi083 ceph-mon[79930]: pgmap v20382: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:47.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:47:47 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:47:47.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:47:47 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:47:47.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:47:47 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:47:47.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:47:47 smithi083 ceph-mon[77242]: pgmap v20382: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:47.718 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:47:47 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:47:47.718 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:47:47 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:47:47.719 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:47:47 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:47:48.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:47:48 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:47:48.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:47:48 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:47:48.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:47:48 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:47:49.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:47:49 smithi153 ceph-mon[70044]: pgmap v20383: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:49.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:47:49 smithi083 ceph-mon[79930]: pgmap v20383: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:49.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:47:49 smithi083 ceph-mon[77242]: pgmap v20383: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:51.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:47:51 smithi153 ceph-mon[70044]: pgmap v20384: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:51.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:47:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:47:51] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T16:47:51.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:47:51 smithi083 ceph-mon[79930]: pgmap v20384: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:51.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:47:51 smithi083 ceph-mon[77242]: pgmap v20384: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:52.813 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:47:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:47:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:47:53.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:47:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:47:52.723Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:47:53.295 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:47:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:47:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:47:53.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:47:53 smithi153 ceph-mon[70044]: pgmap v20385: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:53.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:47:53 smithi083 ceph-mon[79930]: pgmap v20385: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:53.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:47:53 smithi083 ceph-mon[77242]: pgmap v20385: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:55.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:47:54 smithi153 ceph-mon[70044]: pgmap v20386: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:55.232 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:47:54 smithi083 ceph-mon[79930]: pgmap v20386: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:55.232 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:47:54 smithi083 ceph-mon[77242]: pgmap v20386: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:56.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:47:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:47:56.064Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:47:57.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:47:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:47:56.817Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:47:57.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:47:57 smithi153 ceph-mon[70044]: pgmap v20387: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:57.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:47:57 smithi083 ceph-mon[79930]: pgmap v20387: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:57.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:47:57 smithi083 ceph-mon[77242]: pgmap v20387: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:59.562 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:47:59 smithi153 ceph-mon[70044]: pgmap v20388: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:59.719 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:47:59 smithi083 ceph-mon[79930]: pgmap v20388: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:47:59.720 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:47:59 smithi083 ceph-mon[77242]: pgmap v20388: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:01.507 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:48:01 smithi083 ceph-mon[77242]: pgmap v20389: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:01.508 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:48:01 smithi083 ceph-mon[79930]: pgmap v20389: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:01.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:48:01 smithi153 ceph-mon[70044]: pgmap v20389: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:48:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:48:01] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T16:48:02.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:48:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:48:02.317Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:48:03.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:48:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:48:02.724Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:48:03.312 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:48:03 smithi153 ceph-mon[70044]: pgmap v20390: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:03.322 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:48:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:48:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:48:03.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:48:03 smithi083 ceph-mon[79930]: pgmap v20390: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:03.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:48:03 smithi083 ceph-mon[77242]: pgmap v20390: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:05.063 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:48:04 smithi153 ceph-mon[70044]: pgmap v20391: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:05.232 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:48:04 smithi083 ceph-mon[79930]: pgmap v20391: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:05.233 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:48:04 smithi083 ceph-mon[77242]: pgmap v20391: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:06.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:48:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:48:06.065Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:48:07.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:48:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:48:06.817Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:48:07.562 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:48:07 smithi153 ceph-mon[70044]: pgmap v20392: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:07.718 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:48:07 smithi083 ceph-mon[77242]: pgmap v20392: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:07.719 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:48:07 smithi083 ceph-mon[79930]: pgmap v20392: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:09.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:48:09 smithi153 ceph-mon[70044]: pgmap v20393: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:09.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:48:09 smithi083 ceph-mon[79930]: pgmap v20393: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:09.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:48:09 smithi083 ceph-mon[77242]: pgmap v20393: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:11.507 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:48:11 smithi083 ceph-mon[79930]: pgmap v20394: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:11.508 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:48:11 smithi083 ceph-mon[77242]: pgmap v20394: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:11.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:48:11 smithi153 ceph-mon[70044]: pgmap v20394: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:11.965 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:48:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:48:11] "GET /metrics HTTP/1.1" 200 225246 "" "Prometheus/2.33.4" 2022-09-29T16:48:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:48:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:48:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:48:13.206 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:48:13.219 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:48:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:48:12.724Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:48:13.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:48:13 smithi153 ceph-mon[70044]: pgmap v20395: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:48:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:48:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:48:13.515 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:48:13.515 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 6m ago 11h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:48:13.516 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 4m ago 11h 64.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:48:13.516 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 6m ago 11h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:48:13.516 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 4m ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:48:13.517 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (11h) 4m ago 11h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:48:13.517 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11h) 6m ago 11h 1084M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:48:13.518 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11h) 6m ago 11h 2437M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:48:13.530 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (11h) 4m ago 11h 1093M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:48:13.530 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (11h) 6m ago 11h 1469M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:48:13.531 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 6m ago 11h 24.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:48:13.531 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 4m ago 11h 26.6M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:48:13.531 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11h) 6m ago 11h 1777M 2049M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:48:13.532 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11h) 6m ago 11h 1763M 2049M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:48:13.532 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11h) 6m ago 11h 1369M 2049M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:48:13.532 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (11h) 6m ago 11h 1728M 2049M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:48:13.533 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (11h) 4m ago 11h 1514M 2486M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:48:13.533 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (11h) 4m ago 11h 1903M 2486M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:48:13.534 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (11h) 4m ago 11h 1701M 2486M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:48:13.534 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (11h) 4m ago 11h 2025M 2486M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:48:13.536 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (11h) 4m ago 11h 196M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:48:13.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:48:13 smithi083 ceph-mon[77242]: pgmap v20395: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:13.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:48:13 smithi083 ceph-mon[79930]: pgmap v20395: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:13.873 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:48:13.875 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:48:13.878 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:48:13.878 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:48:13.878 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:48:13.879 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:48:13.880 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:48:13.881 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:48:13.881 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:48:13.882 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:48:13.882 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:48:13.883 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:48:13.889 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:48:13.890 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:48:13.890 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:48:13.891 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:48:13.891 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:48:13.892 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:48:14.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:48:14 smithi153 ceph-mon[70044]: from='client.57846 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:48:14.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:48:14 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3253216491' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:48:14.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:48:14 smithi083 ceph-mon[79930]: from='client.57846 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:48:14.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:48:14 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3253216491' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:48:14.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:48:14 smithi083 ceph-mon[77242]: from='client.57846 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:48:14.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:48:14 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3253216491' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:48:15.565 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:48:15 smithi153 ceph-mon[70044]: from='client.67783 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:48:15.566 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:48:15 smithi153 ceph-mon[70044]: pgmap v20396: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:15.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:48:15 smithi083 ceph-mon[77242]: from='client.67783 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:48:15.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:48:15 smithi083 ceph-mon[77242]: pgmap v20396: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:15.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:48:15 smithi083 ceph-mon[79930]: from='client.67783 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:48:15.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:48:15 smithi083 ceph-mon[79930]: pgmap v20396: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:16.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:48:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:48:16.065Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:48:17.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:48:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:48:16.818Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:48:17.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:48:17 smithi153 ceph-mon[70044]: pgmap v20397: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:17.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:48:17 smithi083 ceph-mon[77242]: pgmap v20397: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:17.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:48:17 smithi083 ceph-mon[79930]: pgmap v20397: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:19.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:48:19 smithi153 ceph-mon[70044]: pgmap v20398: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:19.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:48:19 smithi083 ceph-mon[79930]: pgmap v20398: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:19.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:48:19 smithi083 ceph-mon[77242]: pgmap v20398: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:21.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:48:21 smithi153 ceph-mon[70044]: pgmap v20399: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:21.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:48:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:48:21] "GET /metrics HTTP/1.1" 200 225246 "" "Prometheus/2.33.4" 2022-09-29T16:48:21.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:48:21 smithi083 ceph-mon[79930]: pgmap v20399: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:21.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:48:21 smithi083 ceph-mon[77242]: pgmap v20399: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:48:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:48:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:48:23.222 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:48:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:48:22.724Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:48:23.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:48:23 smithi153 ceph-mon[70044]: pgmap v20400: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:23.311 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:48:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:48:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:48:23.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:48:23 smithi083 ceph-mon[79930]: pgmap v20400: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:23.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:48:23 smithi083 ceph-mon[77242]: pgmap v20400: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:25.066 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:48:24 smithi153 ceph-mon[70044]: pgmap v20401: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:25.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:48:24 smithi083 ceph-mon[79930]: pgmap v20401: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:25.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:48:24 smithi083 ceph-mon[77242]: pgmap v20401: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:26.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:48:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:48:26.066Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:48:27.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:48:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:48:26.818Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:48:27.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:48:27 smithi153 ceph-mon[70044]: pgmap v20402: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:27.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:48:27 smithi083 ceph-mon[77242]: pgmap v20402: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:27.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:48:27 smithi083 ceph-mon[79930]: pgmap v20402: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:29.561 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:48:29 smithi153 ceph-mon[70044]: pgmap v20403: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:29.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:48:29 smithi083 ceph-mon[79930]: pgmap v20403: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:29.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:48:29 smithi083 ceph-mon[77242]: pgmap v20403: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:31.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:48:31 smithi153 ceph-mon[70044]: pgmap v20404: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:31.719 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:48:31 smithi083 ceph-mon[79930]: pgmap v20404: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:31.719 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:48:31 smithi083 ceph-mon[77242]: pgmap v20404: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:31.720 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:48:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:48:31] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T16:48:32.810 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:48:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:48:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:48:33.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:48:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:48:32.725Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:48:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:48:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:48:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:48:33.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:48:33 smithi153 ceph-mon[70044]: pgmap v20405: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:33.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:48:33 smithi083 ceph-mon[77242]: pgmap v20405: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:33.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:48:33 smithi083 ceph-mon[79930]: pgmap v20405: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:35.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:48:34 smithi153 ceph-mon[70044]: pgmap v20406: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:35.219 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:48:34 smithi083 ceph-mon[77242]: pgmap v20406: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:35.220 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:48:34 smithi083 ceph-mon[79930]: pgmap v20406: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:36.471 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:48:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:48:36.066Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:48:37.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:48:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:48:36.818Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:48:37.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:48:37 smithi153 ceph-mon[70044]: pgmap v20407: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:37.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:48:37 smithi083 ceph-mon[77242]: pgmap v20407: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:37.718 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:48:37 smithi083 ceph-mon[79930]: pgmap v20407: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:39.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:48:39 smithi153 ceph-mon[70044]: pgmap v20408: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:39.719 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:48:39 smithi083 ceph-mon[77242]: pgmap v20408: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:39.720 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:48:39 smithi083 ceph-mon[79930]: pgmap v20408: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:41.590 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:48:41 smithi153 ceph-mon[70044]: pgmap v20409: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:41.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:48:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:48:41] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T16:48:41.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:48:41 smithi083 ceph-mon[79930]: pgmap v20409: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:41.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:48:41 smithi083 ceph-mon[77242]: pgmap v20409: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:42.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:48:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:48:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:48:42.576 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:48:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:48:42.576 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:48:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:48:42.577 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:48:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:48:42.577 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:48:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:48:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:48:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:48:42.724 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:48:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:48:42.725 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:48:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:48:42.725 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:48:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:48:42.726 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:48:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:48:42.726 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:48:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:48:42.727 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:48:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:48:42.731 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:48:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:48:43.221 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:48:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:48:42.726Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:48:43.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:48:43 smithi153 ceph-mon[70044]: pgmap v20410: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:48:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:48:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:48:43.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:48:43 smithi083 ceph-mon[77242]: pgmap v20410: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:43.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:48:43 smithi083 ceph-mon[79930]: pgmap v20410: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:44.215 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:48:44.490 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:48:44.491 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 7m ago 11h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:48:44.514 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 5m ago 11h 64.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:48:44.514 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 7m ago 11h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:48:44.515 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 5m ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:48:44.515 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (11h) 5m ago 11h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:48:44.516 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11h) 7m ago 11h 1084M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:48:44.516 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11h) 7m ago 11h 2437M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:48:44.516 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (11h) 5m ago 11h 1093M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:48:44.517 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (11h) 7m ago 11h 1469M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:48:44.517 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 7m ago 11h 24.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:48:44.524 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 5m ago 11h 26.6M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:48:44.525 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11h) 7m ago 11h 1777M 2049M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:48:44.526 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11h) 7m ago 11h 1763M 2049M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:48:44.526 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11h) 7m ago 11h 1369M 2049M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:48:44.527 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (11h) 7m ago 11h 1728M 2049M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:48:44.527 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (11h) 5m ago 11h 1514M 2486M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:48:44.528 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (11h) 5m ago 11h 1903M 2486M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:48:44.528 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (11h) 5m ago 11h 1701M 2486M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:48:44.528 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (11h) 5m ago 11h 2025M 2486M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:48:44.529 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (11h) 5m ago 11h 196M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:48:44.843 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:48:44.844 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:48:44.857 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:48:44.857 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:48:44.858 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:48:44.858 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:48:44.859 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:48:44.859 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:48:44.859 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:48:44.860 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:48:44.860 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:48:44.860 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:48:44.860 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:48:44.861 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:48:44.861 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:48:44.863 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:48:44.864 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:48:44.864 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:48:45.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:48:44 smithi153 ceph-mon[70044]: from='client.57864 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:48:45.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:48:44 smithi153 ceph-mon[70044]: pgmap v20411: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:45.219 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:48:44 smithi083 ceph-mon[77242]: from='client.57864 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:48:45.219 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:48:44 smithi083 ceph-mon[77242]: pgmap v20411: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:45.220 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:48:44 smithi083 ceph-mon[79930]: from='client.57864 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:48:45.220 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:48:44 smithi083 ceph-mon[79930]: pgmap v20411: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:46.083 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:48:45 smithi083 ceph-mon[79930]: from='client.57870 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:48:46.083 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:48:45 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3200910404' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:48:46.084 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:48:45 smithi153 ceph-mon[70044]: from='client.57870 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:48:46.084 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:48:45 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3200910404' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:48:46.111 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:48:45 smithi083 ceph-mon[77242]: from='client.57870 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:48:46.112 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:48:45 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3200910404' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:48:46.482 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:48:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:48:46.067Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:48:47.064 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:48:46 smithi153 ceph-mon[70044]: pgmap v20412: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:47.227 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:48:46 smithi083 ceph-mon[77242]: pgmap v20412: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:47.240 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:48:46 smithi083 ceph-mon[79930]: pgmap v20412: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:47.241 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:48:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:48:46.818Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:48:48.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:48:47 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:48:48.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:48:47 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:48:48.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:48:47 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:48:48.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:48:47 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:48:48.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:48:47 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:48:48.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:48:47 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:48:48.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:48:47 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:48:48.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:48:47 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:48:48.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:48:47 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:48:49.218 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:48:48 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:48:49.219 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:48:48 smithi083 ceph-mon[79930]: pgmap v20413: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:49.219 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:48:48 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:48:49.220 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:48:48 smithi083 ceph-mon[77242]: pgmap v20413: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:49.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:48:48 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:48:49.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:48:48 smithi153 ceph-mon[70044]: pgmap v20413: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:51.565 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:48:51 smithi153 ceph-mon[70044]: pgmap v20414: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:51.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:48:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:48:51] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T16:48:51.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:48:51 smithi083 ceph-mon[79930]: pgmap v20414: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:51.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:48:51 smithi083 ceph-mon[77242]: pgmap v20414: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:52.810 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:48:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:48:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:48:53.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:48:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:48:52.727Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:48:53.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:48:53 smithi153 ceph-mon[70044]: pgmap v20415: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:48:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:48:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:48:53.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:48:53 smithi083 ceph-mon[79930]: pgmap v20415: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:53.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:48:53 smithi083 ceph-mon[77242]: pgmap v20415: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:55.074 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:48:54 smithi153 ceph-mon[70044]: pgmap v20416: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:55.220 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:48:54 smithi083 ceph-mon[79930]: pgmap v20416: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:55.220 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:48:54 smithi083 ceph-mon[77242]: pgmap v20416: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:56.483 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:48:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:48:56.067Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:48:57.222 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:48:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:48:56.819Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:48:57.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:48:57 smithi153 ceph-mon[70044]: pgmap v20417: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:57.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:48:57 smithi083 ceph-mon[77242]: pgmap v20417: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:57.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:48:57 smithi083 ceph-mon[79930]: pgmap v20417: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:59.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:48:59 smithi153 ceph-mon[70044]: pgmap v20418: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:59.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:48:59 smithi083 ceph-mon[79930]: pgmap v20418: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:48:59.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:48:59 smithi083 ceph-mon[77242]: pgmap v20418: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:01.564 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:49:01 smithi153 ceph-mon[70044]: pgmap v20419: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:01.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:49:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:49:01] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T16:49:01.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:49:01 smithi083 ceph-mon[79930]: pgmap v20419: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:01.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:49:01 smithi083 ceph-mon[77242]: pgmap v20419: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:49:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:49:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:49:03.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:49:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:49:02.727Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:49:03.315 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:49:03 smithi153 ceph-mon[70044]: pgmap v20420: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:03.315 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:49:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:49:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:49:03.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:49:03 smithi083 ceph-mon[79930]: pgmap v20420: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:03.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:49:03 smithi083 ceph-mon[77242]: pgmap v20420: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:05.090 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:49:04 smithi153 ceph-mon[70044]: pgmap v20421: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:05.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:49:04 smithi083 ceph-mon[77242]: pgmap v20421: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:05.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:49:04 smithi083 ceph-mon[79930]: pgmap v20421: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:06.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:49:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:49:06.069Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:49:07.220 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:49:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:49:06.819Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:49:07.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:49:07 smithi153 ceph-mon[70044]: pgmap v20422: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:07.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:49:07 smithi083 ceph-mon[79930]: pgmap v20422: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:07.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:49:07 smithi083 ceph-mon[77242]: pgmap v20422: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:09.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:49:09 smithi153 ceph-mon[70044]: pgmap v20423: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:09.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:49:09 smithi083 ceph-mon[79930]: pgmap v20423: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:09.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:49:09 smithi083 ceph-mon[77242]: pgmap v20423: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:11.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:49:11 smithi153 ceph-mon[70044]: pgmap v20424: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:11.723 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:49:11 smithi083 ceph-mon[77242]: pgmap v20424: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:11.724 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:49:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:49:11] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T16:49:11.724 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:49:11 smithi083 ceph-mon[79930]: pgmap v20424: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:12.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:49:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:49:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:49:13.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:49:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:49:12.729Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:49:13.274 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:49:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:49:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:49:13.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:49:13 smithi153 ceph-mon[70044]: pgmap v20425: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:13.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:49:13 smithi083 ceph-mon[79930]: pgmap v20425: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:13.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:49:13 smithi083 ceph-mon[77242]: pgmap v20425: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:15.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:49:14 smithi153 ceph-mon[70044]: pgmap v20426: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:15.175 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:49:15.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:49:14 smithi083 ceph-mon[77242]: pgmap v20426: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:15.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:49:14 smithi083 ceph-mon[79930]: pgmap v20426: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:15.472 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:49:15.473 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 7m ago 11h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:49:15.473 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 5m ago 11h 64.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:49:15.474 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 7m ago 11h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:49:15.474 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 5m ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:49:15.475 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (11h) 5m ago 11h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:49:15.475 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11h) 7m ago 11h 1084M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:49:15.475 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11h) 7m ago 11h 2437M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:49:15.487 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (11h) 5m ago 11h 1093M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:49:15.490 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (11h) 7m ago 11h 1469M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:49:15.490 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 7m ago 11h 24.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:49:15.491 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 5m ago 11h 26.6M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:49:15.491 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11h) 7m ago 11h 1777M 2049M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:49:15.491 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11h) 7m ago 11h 1763M 2049M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:49:15.491 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11h) 7m ago 11h 1369M 2049M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:49:15.492 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (11h) 7m ago 11h 1728M 2049M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:49:15.492 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (11h) 5m ago 11h 1514M 2486M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:49:15.492 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (11h) 5m ago 11h 1903M 2486M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:49:15.492 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (11h) 5m ago 11h 1701M 2486M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:49:15.493 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (11h) 5m ago 11h 2025M 2486M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:49:15.493 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (11h) 5m ago 11h 196M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:49:15.813 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:49:15.813 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:49:15.814 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:49:15.814 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:49:15.815 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:49:15.815 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:49:15.816 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:49:15.816 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:49:15.816 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:49:15.817 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:49:15.817 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:49:15.828 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:49:15.828 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:49:15.828 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:49:15.828 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:49:15.829 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:49:15.829 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:49:15.829 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:49:16.056 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:49:15 smithi153 ceph-mon[70044]: from='client.57882 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:49:16.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:49:15 smithi153 ceph-mon[70044]: from='client.67819 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:49:16.070 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:49:15 smithi083 ceph-mon[79930]: from='client.57882 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:49:16.071 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:49:15 smithi083 ceph-mon[79930]: from='client.67819 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:49:16.073 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:49:15 smithi083 ceph-mon[77242]: from='client.57882 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:49:16.073 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:49:15 smithi083 ceph-mon[77242]: from='client.67819 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:49:16.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:49:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:49:16.069Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:49:17.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:49:16 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/503785146' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:49:17.063 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:49:16 smithi153 ceph-mon[70044]: pgmap v20427: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:17.220 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:49:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:49:16.820Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:49:17.221 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:49:16 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/503785146' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:49:17.221 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:49:16 smithi083 ceph-mon[77242]: pgmap v20427: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:17.228 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:49:16 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/503785146' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:49:17.228 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:49:16 smithi083 ceph-mon[79930]: pgmap v20427: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:19.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:49:19 smithi153 ceph-mon[70044]: pgmap v20428: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:19.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:49:19 smithi083 ceph-mon[79930]: pgmap v20428: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:19.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:49:19 smithi083 ceph-mon[77242]: pgmap v20428: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:21.565 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:49:21 smithi153 ceph-mon[70044]: pgmap v20429: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:21.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:49:21 smithi083 ceph-mon[79930]: pgmap v20429: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:21.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:49:21 smithi083 ceph-mon[77242]: pgmap v20429: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:21.715 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:49:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:49:21] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T16:49:22.812 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:49:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:49:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:49:23.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:49:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:49:22.729Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:49:23.274 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:49:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:49:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:49:23.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:49:23 smithi153 ceph-mon[70044]: pgmap v20430: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:23.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:49:23 smithi083 ceph-mon[79930]: pgmap v20430: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:23.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:49:23 smithi083 ceph-mon[77242]: pgmap v20430: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:25.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:49:24 smithi153 ceph-mon[70044]: pgmap v20431: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:25.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:49:24 smithi083 ceph-mon[79930]: pgmap v20431: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:25.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:49:24 smithi083 ceph-mon[77242]: pgmap v20431: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:26.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:49:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:49:26.070Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:49:27.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:49:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:49:26.821Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:49:27.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:49:27 smithi153 ceph-mon[70044]: pgmap v20432: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:27.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:49:27 smithi083 ceph-mon[79930]: pgmap v20432: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:27.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:49:27 smithi083 ceph-mon[77242]: pgmap v20432: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:29.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:49:29 smithi153 ceph-mon[70044]: pgmap v20433: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:29.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:49:29 smithi083 ceph-mon[79930]: pgmap v20433: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:29.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:49:29 smithi083 ceph-mon[77242]: pgmap v20433: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:31.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:49:31 smithi153 ceph-mon[70044]: pgmap v20434: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:31.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:49:31 smithi083 ceph-mon[77242]: pgmap v20434: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:31.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:49:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:49:31] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T16:49:31.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:49:31 smithi083 ceph-mon[79930]: pgmap v20434: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:32.810 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:49:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:49:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:49:33.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:49:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:49:32.730Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:49:33.283 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:49:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:49:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:49:33.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:49:33 smithi153 ceph-mon[70044]: pgmap v20435: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:33.719 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:49:33 smithi083 ceph-mon[77242]: pgmap v20435: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:33.760 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:49:33 smithi083 ceph-mon[79930]: pgmap v20435: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:35.064 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:49:34 smithi153 ceph-mon[70044]: pgmap v20436: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:35.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:49:34 smithi083 ceph-mon[77242]: pgmap v20436: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:35.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:49:34 smithi083 ceph-mon[79930]: pgmap v20436: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:36.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:49:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:49:36.071Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:49:37.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:49:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:49:36.822Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:49:37.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:49:37 smithi153 ceph-mon[70044]: pgmap v20437: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:37.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:49:37 smithi083 ceph-mon[79930]: pgmap v20437: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:37.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:49:37 smithi083 ceph-mon[77242]: pgmap v20437: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:39.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:49:39 smithi153 ceph-mon[70044]: pgmap v20438: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:39.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:49:39 smithi083 ceph-mon[79930]: pgmap v20438: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:39.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:49:39 smithi083 ceph-mon[77242]: pgmap v20438: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:41.565 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:49:41 smithi153 ceph-mon[70044]: pgmap v20439: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:41.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:49:41 smithi083 ceph-mon[79930]: pgmap v20439: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:41.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:49:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:49:41] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T16:49:41.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:49:41 smithi083 ceph-mon[77242]: pgmap v20439: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:42.563 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:49:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:49:42.564 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:49:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:49:42.564 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:49:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:49:42.565 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:49:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:49:42.566 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:49:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:49:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:49:42.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:49:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:49:42.719 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:49:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:49:42.720 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:49:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:49:42.720 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:49:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:49:42.720 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:49:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:49:42.721 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:49:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:49:42.721 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:49:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:49:42.721 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:49:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:49:43.225 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:49:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:49:42.730Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:49:43.287 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:49:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:49:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:49:43.562 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:49:43 smithi153 ceph-mon[70044]: pgmap v20440: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:43.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:49:43 smithi083 ceph-mon[79930]: pgmap v20440: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:43.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:49:43 smithi083 ceph-mon[77242]: pgmap v20440: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:45.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:49:44 smithi153 ceph-mon[70044]: pgmap v20441: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:45.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:49:44 smithi083 ceph-mon[79930]: pgmap v20441: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:45.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:49:44 smithi083 ceph-mon[77242]: pgmap v20441: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:46.141 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:49:46.428 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:49:46.429 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 8m ago 11h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:49:46.429 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 6m ago 11h 64.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:49:46.430 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 8m ago 11h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:49:46.430 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 6m ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:49:46.430 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (11h) 6m ago 11h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:49:46.433 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11h) 8m ago 11h 1084M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:49:46.434 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11h) 8m ago 11h 2437M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:49:46.434 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (11h) 6m ago 11h 1093M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:49:46.435 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (11h) 8m ago 11h 1469M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:49:46.435 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 8m ago 11h 24.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:49:46.436 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 6m ago 11h 26.6M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:49:46.436 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11h) 8m ago 11h 1777M 2049M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:49:46.437 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11h) 8m ago 11h 1763M 2049M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:49:46.437 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11h) 8m ago 11h 1369M 2049M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:49:46.440 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (11h) 8m ago 11h 1728M 2049M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:49:46.440 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (11h) 6m ago 11h 1514M 2486M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:49:46.441 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (11h) 6m ago 11h 1903M 2486M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:49:46.441 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (11h) 6m ago 11h 1701M 2486M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:49:46.442 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (11h) 6m ago 11h 2025M 2486M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:49:46.442 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (11h) 6m ago 11h 196M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:49:46.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:49:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:49:46.071Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:49:46.784 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:49:46.784 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:49:46.785 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:49:46.785 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:49:46.786 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:49:46.786 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:49:46.787 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:49:46.787 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:49:46.788 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:49:46.788 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:49:46.789 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:49:46.789 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:49:46.790 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:49:46.790 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:49:46.790 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:49:46.791 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:49:46.792 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:49:46.793 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:49:47.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:49:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:49:46.821Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:49:47.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:49:47 smithi153 ceph-mon[70044]: from='client.57900 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:49:47.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:49:47 smithi153 ceph-mon[70044]: pgmap v20442: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:47.561 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:49:47 smithi153 ceph-mon[70044]: from='client.67837 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:49:47.561 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:49:47 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/796393689' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:49:47.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:49:47 smithi083 ceph-mon[79930]: from='client.57900 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:49:47.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:49:47 smithi083 ceph-mon[79930]: pgmap v20442: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:47.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:49:47 smithi083 ceph-mon[79930]: from='client.67837 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:49:47.726 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:49:47 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/796393689' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:49:47.727 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:49:47 smithi083 ceph-mon[77242]: from='client.57900 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:49:47.727 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:49:47 smithi083 ceph-mon[77242]: pgmap v20442: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:47.728 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:49:47 smithi083 ceph-mon[77242]: from='client.67837 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:49:47.728 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:49:47 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/796393689' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:49:48.572 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:49:48 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:49:48.575 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:49:48 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:49:48.575 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:49:48 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:49:48.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:49:48 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:49:48.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:49:48 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:49:48.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:49:48 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:49:48.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:49:48 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:49:48.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:49:48 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:49:48.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:49:48 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:49:49.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:49:49 smithi153 ceph-mon[70044]: pgmap v20443: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:49.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:49:49 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:49:49.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:49:49 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:49:49.561 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:49:49 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:49:49.561 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:49:49 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:49:49.561 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:49:49 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:49:49.561 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:49:49 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:49:49.562 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:49:49 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:49:49.563 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:49:49 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:49:49.563 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:49:49 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:49:49.563 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:49:49 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:49:49.563 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:49:49 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:49:49.564 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:49:49 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:49:49.564 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:49:49 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:49:49.565 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:49:49 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:49:49.565 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:49:49 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:49:49.565 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:49:49 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:49:49.565 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:49:49 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:49:49.566 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:49:49 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:49:49.566 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:49:49 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:49:49.720 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:49:49 smithi083 ceph-mon[79930]: pgmap v20443: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:49.721 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:49:49 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:49:49.722 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:49:49 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:49:49.722 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:49:49 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:49:49.722 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:49:49 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:49:49.723 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:49:49 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:49:49.723 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:49:49 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:49:49.724 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:49:49 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:49:49.726 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:49:49 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:49:49.726 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:49:49 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:49:49.727 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:49:49 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:49:49.727 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:49:49 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:49:49.727 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:49:49 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:49:49.728 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:49:49 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:49:49.728 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:49:49 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:49:49.729 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:49:49 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:49:49.729 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:49:49 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:49:49.729 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:49:49 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:49:49.730 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:49:49 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:49:49.731 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:49:49 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:49:49.732 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:49:49 smithi083 ceph-mon[77242]: pgmap v20443: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:49.732 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:49:49 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:49:49.733 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:49:49 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:49:49.733 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:49:49 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:49:49.733 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:49:49 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:49:49.734 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:49:49 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:49:49.735 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:49:49 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:49:49.735 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:49:49 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:49:49.736 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:49:49 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:49:49.737 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:49:49 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:49:49.738 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:49:49 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:49:49.738 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:49:49 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:49:49.739 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:49:49 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:49:49.739 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:49:49 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:49:49.740 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:49:49 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:49:49.740 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:49:49 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:49:49.740 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:49:49 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T16:49:49.741 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:49:49 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:49:49.741 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:49:49 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:49:49.741 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:49:49 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:49:50.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:49:50 smithi153 ceph-mon[70044]: Adjusting osd_memory_target on smithi153 to 2482M 2022-09-29T16:49:50.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:49:50 smithi153 ceph-mon[70044]: Adjusting osd_memory_target on smithi083 to 2035M 2022-09-29T16:49:50.723 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:49:50 smithi083 ceph-mon[79930]: Adjusting osd_memory_target on smithi153 to 2482M 2022-09-29T16:49:50.723 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:49:50 smithi083 ceph-mon[79930]: Adjusting osd_memory_target on smithi083 to 2035M 2022-09-29T16:49:50.724 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:49:50 smithi083 ceph-mon[77242]: Adjusting osd_memory_target on smithi153 to 2482M 2022-09-29T16:49:50.727 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:49:50 smithi083 ceph-mon[77242]: Adjusting osd_memory_target on smithi083 to 2035M 2022-09-29T16:49:51.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:49:51 smithi153 ceph-mon[70044]: pgmap v20444: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:51.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:49:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:49:51] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T16:49:51.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:49:51 smithi083 ceph-mon[79930]: pgmap v20444: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:51.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:49:51 smithi083 ceph-mon[77242]: pgmap v20444: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:52.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:49:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:49:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:49:53.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:49:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:49:52.731Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:49:53.302 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:49:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:49:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:49:53.565 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:49:53 smithi153 ceph-mon[70044]: pgmap v20445: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:53.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:49:53 smithi083 ceph-mon[79930]: pgmap v20445: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:53.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:49:53 smithi083 ceph-mon[77242]: pgmap v20445: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:55.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:49:54 smithi153 ceph-mon[70044]: pgmap v20446: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:55.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:49:54 smithi083 ceph-mon[79930]: pgmap v20446: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:55.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:49:54 smithi083 ceph-mon[77242]: pgmap v20446: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:56.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:49:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:49:56.072Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:49:57.220 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:49:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:49:56.823Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:49:57.561 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:49:57 smithi153 ceph-mon[70044]: pgmap v20447: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:57.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:49:57 smithi083 ceph-mon[79930]: pgmap v20447: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:57.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:49:57 smithi083 ceph-mon[77242]: pgmap v20447: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:59.561 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:49:59 smithi153 ceph-mon[70044]: pgmap v20448: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:59.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:49:59 smithi083 ceph-mon[79930]: pgmap v20448: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:49:59.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:49:59 smithi083 ceph-mon[77242]: pgmap v20448: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:00.290 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:50:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[77219]: 2022-09-29T16:49:59.999+0000 7f160f665700 -1 log_channel(cluster) log [ERR] : overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T16:50:00.562 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:50:00 smithi153 ceph-mon[70044]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T16:50:00.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:50:00 smithi083 ceph-mon[79930]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T16:50:00.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:50:00 smithi083 ceph-mon[77242]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T16:50:01.561 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:50:01 smithi153 ceph-mon[70044]: pgmap v20449: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:01.715 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:50:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:50:01] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T16:50:01.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:50:01 smithi083 ceph-mon[79930]: pgmap v20449: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:01.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:50:01 smithi083 ceph-mon[77242]: pgmap v20449: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:50:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:50:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:50:03.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:50:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:50:02.732Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:50:03.304 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:50:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:50:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:50:03.574 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:50:03 smithi153 ceph-mon[70044]: pgmap v20450: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:03.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:50:03 smithi083 ceph-mon[77242]: pgmap v20450: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:03.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:50:03 smithi083 ceph-mon[79930]: pgmap v20450: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:05.063 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:50:04 smithi153 ceph-mon[70044]: pgmap v20451: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:05.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:50:04 smithi083 ceph-mon[79930]: pgmap v20451: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:05.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:50:04 smithi083 ceph-mon[77242]: pgmap v20451: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:06.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:50:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:50:06.073Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:50:07.230 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:50:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:50:06.824Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:50:07.564 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:50:07 smithi153 ceph-mon[70044]: pgmap v20452: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:07.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:50:07 smithi083 ceph-mon[79930]: pgmap v20452: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:07.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:50:07 smithi083 ceph-mon[77242]: pgmap v20452: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:09.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:50:09 smithi153 ceph-mon[70044]: pgmap v20453: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:09.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:50:09 smithi083 ceph-mon[79930]: pgmap v20453: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:09.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:50:09 smithi083 ceph-mon[77242]: pgmap v20453: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:11.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:50:11 smithi153 ceph-mon[70044]: pgmap v20454: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:11.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:50:11 smithi083 ceph-mon[77242]: pgmap v20454: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:11.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:50:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:50:11] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T16:50:11.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:50:11 smithi083 ceph-mon[79930]: pgmap v20454: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:12.832 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:50:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:50:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:50:13.217 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:50:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:50:12.733Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:50:13.302 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:50:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:50:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:50:13.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:50:13 smithi153 ceph-mon[70044]: pgmap v20455: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:13.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:50:13 smithi083 ceph-mon[79930]: pgmap v20455: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:13.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:50:13 smithi083 ceph-mon[77242]: pgmap v20455: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:15.064 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:50:14 smithi153 ceph-mon[70044]: pgmap v20456: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:15.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:50:14 smithi083 ceph-mon[79930]: pgmap v20456: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:15.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:50:14 smithi083 ceph-mon[77242]: pgmap v20456: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:16.469 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:50:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:50:16.073Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:50:17.115 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:50:17.217 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:50:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:50:16.826Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:50:17.420 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:50:17.421 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 8m ago 11h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:50:17.421 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 6m ago 11h 64.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:50:17.435 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 8m ago 11h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:50:17.487 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 6m ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:50:17.487 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (11h) 6m ago 11h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:50:17.488 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11h) 8m ago 11h 1084M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:50:17.488 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11h) 8m ago 11h 2437M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:50:17.488 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (11h) 6m ago 11h 1093M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:50:17.488 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (11h) 8m ago 11h 1469M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:50:17.489 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 8m ago 11h 24.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:50:17.489 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 6m ago 11h 26.6M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:50:17.489 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11h) 8m ago 11h 1777M 2035M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:50:17.489 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11h) 8m ago 11h 1763M 2035M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:50:17.510 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11h) 8m ago 11h 1369M 2035M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:50:17.523 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (11h) 8m ago 11h 1728M 2035M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:50:17.524 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (11h) 6m ago 11h 1514M 2482M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:50:17.524 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (11h) 6m ago 11h 1903M 2482M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:50:17.524 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (11h) 6m ago 11h 1701M 2482M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:50:17.524 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (11h) 6m ago 11h 2025M 2482M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:50:17.525 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (11h) 6m ago 11h 196M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:50:17.567 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:50:17 smithi153 ceph-mon[70044]: pgmap v20457: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:17.723 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:50:17 smithi083 ceph-mon[79930]: pgmap v20457: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:17.724 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:50:17 smithi083 ceph-mon[77242]: pgmap v20457: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:17.762 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:50:17.763 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:50:17.763 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:50:17.764 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:50:17.764 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:50:17.765 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:50:17.765 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:50:17.767 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:50:17.768 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:50:17.769 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:50:17.819 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:50:17.820 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:50:17.820 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:50:17.820 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:50:17.821 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:50:17.821 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:50:17.833 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:50:17.833 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:50:18.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:50:18 smithi153 ceph-mon[70044]: from='client.67849 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:50:18.570 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:50:18 smithi153 ceph-mon[70044]: from='client.67855 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:50:18.571 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:50:18 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2125272124' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:50:18.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:50:18 smithi083 ceph-mon[77242]: from='client.67849 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:50:18.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:50:18 smithi083 ceph-mon[77242]: from='client.67855 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:50:18.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:50:18 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2125272124' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:50:18.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:50:18 smithi083 ceph-mon[79930]: from='client.67849 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:50:18.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:50:18 smithi083 ceph-mon[79930]: from='client.67855 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:50:18.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:50:18 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2125272124' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:50:19.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:50:19 smithi153 ceph-mon[70044]: pgmap v20458: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:19.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:50:19 smithi083 ceph-mon[77242]: pgmap v20458: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:19.727 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:50:19 smithi083 ceph-mon[79930]: pgmap v20458: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:21.592 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:50:21 smithi153 ceph-mon[70044]: pgmap v20459: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:21.720 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:50:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:50:21] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T16:50:21.721 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:50:21 smithi083 ceph-mon[79930]: pgmap v20459: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:21.722 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:50:21 smithi083 ceph-mon[77242]: pgmap v20459: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:50:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:50:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:50:23.229 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:50:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:50:22.734Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:50:23.318 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:50:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:50:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:50:23.564 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:50:23 smithi153 ceph-mon[70044]: pgmap v20460: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:23.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:50:23 smithi083 ceph-mon[79930]: pgmap v20460: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:23.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:50:23 smithi083 ceph-mon[77242]: pgmap v20460: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:25.094 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:50:24 smithi153 ceph-mon[70044]: pgmap v20461: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:25.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:50:24 smithi083 ceph-mon[77242]: pgmap v20461: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:25.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:50:24 smithi083 ceph-mon[79930]: pgmap v20461: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:26.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:50:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:50:26.073Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:50:27.210 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:50:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:50:26.825Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:50:27.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:50:27 smithi153 ceph-mon[70044]: pgmap v20462: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:27.755 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:50:27 smithi083 ceph-mon[79930]: pgmap v20462: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:27.767 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:50:27 smithi083 ceph-mon[77242]: pgmap v20462: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:29.565 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:50:29 smithi153 ceph-mon[70044]: pgmap v20463: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:29.738 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:50:29 smithi083 ceph-mon[79930]: pgmap v20463: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:29.739 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:50:29 smithi083 ceph-mon[77242]: pgmap v20463: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:31.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:50:31 smithi153 ceph-mon[70044]: pgmap v20464: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:31.720 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:50:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:50:31] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T16:50:31.721 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:50:31 smithi083 ceph-mon[79930]: pgmap v20464: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:31.724 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:50:31 smithi083 ceph-mon[77242]: pgmap v20464: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:32.849 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:50:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:50:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:50:33.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:50:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:50:32.735Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:50:33.314 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:50:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:50:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:50:33.577 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:50:33 smithi153 ceph-mon[70044]: pgmap v20465: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:33.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:50:33 smithi083 ceph-mon[79930]: pgmap v20465: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:33.718 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:50:33 smithi083 ceph-mon[77242]: pgmap v20465: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:35.067 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:50:34 smithi153 ceph-mon[70044]: pgmap v20466: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:35.220 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:50:34 smithi083 ceph-mon[79930]: pgmap v20466: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:35.220 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:50:34 smithi083 ceph-mon[77242]: pgmap v20466: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:36.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:50:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:50:36.074Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:50:37.210 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:50:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:50:36.826Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:50:37.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:50:37 smithi153 ceph-mon[70044]: pgmap v20467: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:37.718 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:50:37 smithi083 ceph-mon[77242]: pgmap v20467: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:37.726 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:50:37 smithi083 ceph-mon[79930]: pgmap v20467: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:39.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:50:39 smithi153 ceph-mon[70044]: pgmap v20468: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:39.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:50:39 smithi083 ceph-mon[79930]: pgmap v20468: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:39.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:50:39 smithi083 ceph-mon[77242]: pgmap v20468: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:41.587 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:50:41 smithi153 ceph-mon[70044]: pgmap v20469: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:41.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:50:41 smithi083 ceph-mon[79930]: pgmap v20469: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:41.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:50:41 smithi083 ceph-mon[77242]: pgmap v20469: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:41.716 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:50:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:50:41] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T16:50:42.563 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:50:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:50:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:50:42.564 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:50:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:50:42.564 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:50:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:50:42.565 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:50:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:50:42.565 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:50:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:50:42.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:50:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:50:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:50:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:50:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:50:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:50:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:50:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:50:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:50:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:50:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:50:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:50:42.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:50:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:50:42.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:50:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:50:43.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:50:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:50:42.735Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:50:43.316 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:50:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:50:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:50:43.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:50:43 smithi153 ceph-mon[70044]: pgmap v20470: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:43.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:50:43 smithi083 ceph-mon[79930]: pgmap v20470: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:43.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:50:43 smithi083 ceph-mon[77242]: pgmap v20470: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:45.070 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:50:44 smithi153 ceph-mon[70044]: pgmap v20471: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:45.225 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:50:44 smithi083 ceph-mon[79930]: pgmap v20471: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:45.226 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:50:44 smithi083 ceph-mon[77242]: pgmap v20471: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:46.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:50:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:50:46.075Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:50:47.214 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:50:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:50:46.827Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:50:47.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:50:47 smithi153 ceph-mon[70044]: pgmap v20472: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:47.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:50:47 smithi083 ceph-mon[79930]: pgmap v20472: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:47.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:50:47 smithi083 ceph-mon[77242]: pgmap v20472: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:48.081 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:50:48.376 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:50:48.376 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 9m ago 11h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:50:48.377 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 7m ago 11h 64.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:50:48.377 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 9m ago 11h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:50:48.377 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 7m ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:50:48.390 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (11h) 7m ago 11h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:50:48.391 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11h) 9m ago 11h 1084M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:50:48.392 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11h) 9m ago 11h 2437M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:50:48.392 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (11h) 7m ago 11h 1093M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:50:48.393 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (11h) 9m ago 11h 1469M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:50:48.395 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 9m ago 11h 24.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:50:48.395 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 7m ago 11h 26.6M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:50:48.395 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11h) 9m ago 11h 1777M 2035M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:50:48.396 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11h) 9m ago 11h 1763M 2035M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:50:48.396 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11h) 9m ago 11h 1369M 2035M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:50:48.396 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (11h) 9m ago 11h 1728M 2035M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:50:48.396 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (11h) 7m ago 11h 1514M 2482M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:50:48.396 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (11h) 7m ago 11h 1903M 2482M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:50:48.398 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (11h) 7m ago 11h 1701M 2482M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:50:48.399 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (11h) 7m ago 11h 2025M 2482M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:50:48.399 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (11h) 7m ago 11h 196M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:50:48.742 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:50:48.743 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:50:48.743 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:50:48.744 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:50:48.744 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:50:48.745 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:50:48.775 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:50:48.775 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:50:48.776 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:50:48.776 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:50:48.777 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:50:48.791 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:50:48.792 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:50:48.792 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:50:48.793 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:50:48.793 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:50:48.793 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:50:48.809 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:50:49.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:50:49 smithi153 ceph-mon[70044]: from='client.57936 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:50:49.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:50:49 smithi153 ceph-mon[70044]: pgmap v20473: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:49.561 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:50:49 smithi153 ceph-mon[70044]: from='client.67873 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:50:49.561 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:50:49 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:50:49.571 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:50:49 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:50:49.572 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:50:49 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:50:49.572 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:50:49 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/802566470' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:50:49.573 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:50:49 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:50:49.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:50:49 smithi083 ceph-mon[79930]: from='client.57936 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:50:49.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:50:49 smithi083 ceph-mon[79930]: pgmap v20473: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:49.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:50:49 smithi083 ceph-mon[79930]: from='client.67873 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:50:49.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:50:49 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:50:49.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:50:49 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:50:49.718 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:50:49 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:50:49.718 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:50:49 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/802566470' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:50:49.719 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:50:49 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:50:49.721 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:50:49 smithi083 ceph-mon[77242]: from='client.57936 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:50:49.721 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:50:49 smithi083 ceph-mon[77242]: pgmap v20473: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:49.722 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:50:49 smithi083 ceph-mon[77242]: from='client.67873 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:50:49.722 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:50:49 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:50:49.723 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:50:49 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:50:49.724 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:50:49 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:50:49.734 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:50:49 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/802566470' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:50:49.735 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:50:49 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:50:51.562 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:50:51 smithi153 ceph-mon[70044]: pgmap v20474: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:51.726 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:50:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:50:51] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T16:50:51.727 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:50:51 smithi083 ceph-mon[79930]: pgmap v20474: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:51.727 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:50:51 smithi083 ceph-mon[77242]: pgmap v20474: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:52.813 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:50:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:50:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:50:53.222 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:50:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:50:52.735Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:50:53.304 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:50:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:50:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:50:53.562 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:50:53 smithi153 ceph-mon[70044]: pgmap v20475: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:53.725 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:50:53 smithi083 ceph-mon[77242]: pgmap v20475: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:53.726 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:50:53 smithi083 ceph-mon[79930]: pgmap v20475: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:55.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:50:54 smithi153 ceph-mon[70044]: pgmap v20476: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:55.254 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:50:54 smithi083 ceph-mon[79930]: pgmap v20476: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:55.255 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:50:54 smithi083 ceph-mon[77242]: pgmap v20476: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:56.488 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:50:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:50:56.076Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:50:57.231 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:50:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:50:56.827Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:50:57.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:50:57 smithi153 ceph-mon[70044]: pgmap v20477: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:57.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:50:57 smithi083 ceph-mon[77242]: pgmap v20477: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:57.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:50:57 smithi083 ceph-mon[79930]: pgmap v20477: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:59.568 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:50:59 smithi153 ceph-mon[70044]: pgmap v20478: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:59.718 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:50:59 smithi083 ceph-mon[79930]: pgmap v20478: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:50:59.719 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:50:59 smithi083 ceph-mon[77242]: pgmap v20478: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:01.560 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:51:01 smithi153 ceph-mon[70044]: pgmap v20479: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:01.718 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:51:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:51:01] "GET /metrics HTTP/1.1" 200 225247 "" "Prometheus/2.33.4" 2022-09-29T16:51:01.718 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:51:01 smithi083 ceph-mon[79930]: pgmap v20479: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:01.719 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:51:01 smithi083 ceph-mon[77242]: pgmap v20479: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:02.837 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:51:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:51:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:51:03.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:51:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:51:02.736Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:51:03.314 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:51:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:51:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:51:03.719 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:51:03 smithi083 ceph-mon[79930]: pgmap v20480: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:03.724 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:51:03 smithi083 ceph-mon[77242]: pgmap v20480: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:03.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:51:03 smithi153 ceph-mon[70044]: pgmap v20480: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:05.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:51:04 smithi153 ceph-mon[70044]: pgmap v20481: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:05.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:51:04 smithi083 ceph-mon[79930]: pgmap v20481: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:05.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:51:04 smithi083 ceph-mon[77242]: pgmap v20481: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:06.466 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:51:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:51:06.077Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:51:07.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:51:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:51:06.828Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:51:07.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:51:07 smithi153 ceph-mon[70044]: pgmap v20482: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:07.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:51:07 smithi083 ceph-mon[77242]: pgmap v20482: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:07.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:51:07 smithi083 ceph-mon[79930]: pgmap v20482: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:09.566 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:51:09 smithi153 ceph-mon[70044]: pgmap v20483: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:09.719 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:51:09 smithi083 ceph-mon[79930]: pgmap v20483: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:09.720 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:51:09 smithi083 ceph-mon[77242]: pgmap v20483: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:11.733 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:51:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:51:11] "GET /metrics HTTP/1.1" 200 225246 "" "Prometheus/2.33.4" 2022-09-29T16:51:11.735 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:51:11 smithi083 ceph-mon[79930]: pgmap v20484: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:11.735 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:51:11 smithi083 ceph-mon[77242]: pgmap v20484: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:11.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:51:11 smithi153 ceph-mon[70044]: pgmap v20484: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:12.812 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:51:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:51:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:51:13.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:51:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:51:12.736Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:51:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:51:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:51:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:51:13.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:51:13 smithi083 ceph-mon[79930]: pgmap v20485: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:13.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:51:13 smithi083 ceph-mon[77242]: pgmap v20485: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:13.816 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:51:13 smithi153 ceph-mon[70044]: pgmap v20485: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:15.063 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:51:14 smithi153 ceph-mon[70044]: pgmap v20486: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:15.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:51:14 smithi083 ceph-mon[79930]: pgmap v20486: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:15.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:51:14 smithi083 ceph-mon[77242]: pgmap v20486: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:16.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:51:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:51:16.077Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:51:17.224 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:51:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:51:16.829Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:51:17.562 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:51:17 smithi153 ceph-mon[70044]: pgmap v20487: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:17.723 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:51:17 smithi083 ceph-mon[77242]: pgmap v20487: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:17.724 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:51:17 smithi083 ceph-mon[79930]: pgmap v20487: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:19.072 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:51:19.361 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:51:19.372 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 9m ago 11h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:51:19.372 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 7m ago 11h 64.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:51:19.373 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 9m ago 11h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:51:19.373 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 7m ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:51:19.384 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (11h) 7m ago 11h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:51:19.385 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11h) 9m ago 11h 1084M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:51:19.385 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11h) 9m ago 11h 2437M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:51:19.385 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (11h) 7m ago 11h 1093M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:51:19.391 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (11h) 9m ago 11h 1469M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:51:19.392 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 9m ago 11h 24.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:51:19.392 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 7m ago 11h 26.6M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:51:19.393 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11h) 9m ago 11h 1777M 2035M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:51:19.394 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11h) 9m ago 11h 1763M 2035M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:51:19.394 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11h) 9m ago 11h 1369M 2035M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:51:19.395 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (11h) 9m ago 11h 1728M 2035M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:51:19.395 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (11h) 7m ago 11h 1514M 2482M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:51:19.395 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (11h) 7m ago 11h 1903M 2482M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:51:19.396 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (11h) 7m ago 11h 1701M 2482M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:51:19.409 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (11h) 7m ago 11h 2025M 2482M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:51:19.420 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (11h) 7m ago 11h 196M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:51:19.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:51:19 smithi083 ceph-mon[79930]: pgmap v20488: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:19.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:51:19 smithi083 ceph-mon[77242]: pgmap v20488: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:19.716 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:51:19.733 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:51:19.733 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:51:19.734 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:51:19.734 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:51:19.735 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:51:19.747 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:51:19.749 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:51:19.749 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:51:19.750 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:51:19.750 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:51:19.750 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:51:19.751 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:51:19.751 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:51:19.751 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:51:19.752 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:51:19.752 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:51:19.753 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:51:19.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:51:19 smithi153 ceph-mon[70044]: pgmap v20488: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:20.722 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:51:20 smithi083 ceph-mon[77242]: from='client.67885 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:51:20.723 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:51:20 smithi083 ceph-mon[77242]: from='client.57960 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:51:20.724 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:51:20 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2194625767' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:51:20.725 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:51:20 smithi083 ceph-mon[79930]: from='client.67885 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:51:20.735 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:51:20 smithi083 ceph-mon[79930]: from='client.57960 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:51:20.735 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:51:20 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2194625767' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:51:20.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:51:20 smithi153 ceph-mon[70044]: from='client.67885 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:51:20.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:51:20 smithi153 ceph-mon[70044]: from='client.57960 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:51:20.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:51:20 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2194625767' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:51:21.714 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:51:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:51:21] "GET /metrics HTTP/1.1" 200 225246 "" "Prometheus/2.33.4" 2022-09-29T16:51:21.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:51:21 smithi083 ceph-mon[77242]: pgmap v20489: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:21.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:51:21 smithi083 ceph-mon[79930]: pgmap v20489: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:21.815 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:51:21 smithi153 ceph-mon[70044]: pgmap v20489: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:51:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:51:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:51:23.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:51:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:51:22.736Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:51:23.320 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:51:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:51:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:51:23.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:51:23 smithi083 ceph-mon[79930]: pgmap v20490: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:23.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:51:23 smithi083 ceph-mon[77242]: pgmap v20490: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:23.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:51:23 smithi153 ceph-mon[70044]: pgmap v20490: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:25.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:51:24 smithi153 ceph-mon[70044]: pgmap v20491: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:25.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:51:24 smithi083 ceph-mon[77242]: pgmap v20491: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:25.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:51:24 smithi083 ceph-mon[79930]: pgmap v20491: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:26.466 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:51:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:51:26.079Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:51:27.214 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:51:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:51:26.829Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:51:27.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:51:27 smithi083 ceph-mon[79930]: pgmap v20492: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:27.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:51:27 smithi083 ceph-mon[77242]: pgmap v20492: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:27.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:51:27 smithi153 ceph-mon[70044]: pgmap v20492: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:29.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:51:29 smithi083 ceph-mon[79930]: pgmap v20493: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:29.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:51:29 smithi083 ceph-mon[77242]: pgmap v20493: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:29.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:51:29 smithi153 ceph-mon[70044]: pgmap v20493: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:31.724 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:51:31 smithi083 ceph-mon[77242]: pgmap v20494: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:31.729 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:51:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:51:31] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T16:51:31.735 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:51:31 smithi083 ceph-mon[79930]: pgmap v20494: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:31.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:51:31 smithi153 ceph-mon[70044]: pgmap v20494: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:51:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:51:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:51:33.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:51:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:51:32.737Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:51:33.312 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:51:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:51:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:51:33.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:51:33 smithi083 ceph-mon[79930]: pgmap v20495: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:33.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:51:33 smithi083 ceph-mon[77242]: pgmap v20495: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:33.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:51:33 smithi153 ceph-mon[70044]: pgmap v20495: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:35.066 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:51:34 smithi153 ceph-mon[70044]: pgmap v20496: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:35.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:51:34 smithi083 ceph-mon[79930]: pgmap v20496: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:35.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:51:34 smithi083 ceph-mon[77242]: pgmap v20496: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:36.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:51:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:51:36.079Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:51:37.231 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:51:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:51:36.830Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:51:37.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:51:37 smithi083 ceph-mon[79930]: pgmap v20497: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:37.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:51:37 smithi083 ceph-mon[77242]: pgmap v20497: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:37.847 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:51:37 smithi153 ceph-mon[70044]: pgmap v20497: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:39.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:51:39 smithi083 ceph-mon[79930]: pgmap v20498: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:39.718 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:51:39 smithi083 ceph-mon[77242]: pgmap v20498: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:39.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:51:39 smithi153 ceph-mon[70044]: pgmap v20498: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:41.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:51:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:51:41] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T16:51:41.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:51:41 smithi083 ceph-mon[79930]: pgmap v20499: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:41.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:51:41 smithi083 ceph-mon[77242]: pgmap v20499: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:41.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:51:41 smithi153 ceph-mon[70044]: pgmap v20499: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:42.718 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:51:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:51:42.718 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:51:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:51:42.718 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:51:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:51:42.719 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:51:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:51:42.719 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:51:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:51:42.720 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:51:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:51:42.720 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:51:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:51:42.720 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:51:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:51:42.816 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:51:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:51:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:51:42.828 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:51:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:51:42.828 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:51:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:51:42.829 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:51:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:51:42.829 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:51:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:51:43.221 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:51:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:51:42.737Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:51:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:51:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:51:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:51:43.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:51:43 smithi083 ceph-mon[79930]: pgmap v20500: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:43.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:51:43 smithi083 ceph-mon[77242]: pgmap v20500: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:43.813 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:51:43 smithi153 ceph-mon[70044]: pgmap v20500: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:45.067 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:51:44 smithi153 ceph-mon[70044]: pgmap v20501: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:45.219 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:51:44 smithi083 ceph-mon[77242]: pgmap v20501: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:45.220 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:51:44 smithi083 ceph-mon[79930]: pgmap v20501: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:46.466 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:51:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:51:46.080Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:51:47.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:51:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:51:46.830Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:51:47.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:51:47 smithi083 ceph-mon[77242]: pgmap v20502: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:47.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:51:47 smithi083 ceph-mon[79930]: pgmap v20502: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:47.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:51:47 smithi153 ceph-mon[70044]: pgmap v20502: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:49.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:51:49 smithi083 ceph-mon[77242]: pgmap v20503: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:49.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:51:49 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:51:49.740 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:51:49 smithi083 ceph-mon[79930]: pgmap v20503: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:49.740 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:51:49 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:51:49.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:51:49 smithi153 ceph-mon[70044]: pgmap v20503: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:49.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:51:49 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:51:50.063 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:51:50.367 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:51:50.368 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 10m ago 11h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:51:50.368 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 8m ago 11h 64.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:51:50.369 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 10m ago 11h 26.0M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:51:50.371 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 8m ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:51:50.372 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (11h) 8m ago 11h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:51:50.372 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11h) 10m ago 11h 1084M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:51:50.373 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11h) 10m ago 11h 2437M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:51:50.373 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (11h) 8m ago 11h 1093M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:51:50.374 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (11h) 10m ago 11h 1469M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:51:50.374 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 10m ago 11h 24.7M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:51:50.375 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 8m ago 11h 26.6M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:51:50.375 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11h) 10m ago 11h 1777M 2035M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:51:50.376 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11h) 10m ago 11h 1763M 2035M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:51:50.376 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11h) 10m ago 11h 1369M 2035M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:51:50.380 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (11h) 10m ago 11h 1728M 2035M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:51:50.381 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (11h) 8m ago 11h 1514M 2482M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:51:50.381 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (11h) 8m ago 11h 1903M 2482M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:51:50.382 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (11h) 8m ago 11h 1701M 2482M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:51:50.416 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (11h) 8m ago 11h 2025M 2482M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:51:50.417 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (11h) 8m ago 11h 196M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:51:50.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:51:50 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:51:50.738 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:51:50 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:51:50.739 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:51:50 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:51:50.739 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:51:50 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:51:50.740 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:51:50 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:51:50.740 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:51:50 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:51:50.741 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:51:50 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:51:50.741 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:51:50 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:51:50.741 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:51:50 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:51:50.743 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:51:50 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:51:50.743 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:51:50 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:51:50.743 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:51:50 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:51:50.788 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:51:50.789 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:51:50.789 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:51:50.803 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:51:50.804 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:51:50.818 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:51:50.818 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:51:50.819 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:51:50.826 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:51:50.826 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:51:50.826 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:51:50.827 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:51:50.827 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:51:50.827 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:51:50.827 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:51:50.827 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:51:50.828 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:51:50.828 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:51:50.829 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:51:50 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:51:50.830 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:51:50 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:51:50.831 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:51:50 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:51:50.831 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:51:50 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:51:50.831 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:51:50 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:51:50.832 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:51:50 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:51:51.714 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:51:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:51:51] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T16:51:51.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:51:51 smithi083 ceph-mon[77242]: from='client.67903 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:51:51.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:51:51 smithi083 ceph-mon[77242]: pgmap v20504: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:51.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:51:51 smithi083 ceph-mon[77242]: from='client.57978 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:51:51.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:51:51 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1091873651' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:51:51.726 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:51:51 smithi083 ceph-mon[79930]: from='client.67903 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:51:51.727 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:51:51 smithi083 ceph-mon[79930]: pgmap v20504: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:51.727 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:51:51 smithi083 ceph-mon[79930]: from='client.57978 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:51:51.727 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:51:51 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1091873651' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:51:51.814 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:51:51 smithi153 ceph-mon[70044]: from='client.67903 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:51:51.815 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:51:51 smithi153 ceph-mon[70044]: pgmap v20504: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:51.815 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:51:51 smithi153 ceph-mon[70044]: from='client.57978 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:51:51.815 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:51:51 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1091873651' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:51:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:51:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:51:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:51:53.216 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:51:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:51:52.738Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:51:53.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:51:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:51:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:51:53.720 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:51:53 smithi083 ceph-mon[79930]: pgmap v20505: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:53.721 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:51:53 smithi083 ceph-mon[77242]: pgmap v20505: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:53.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:51:53 smithi153 ceph-mon[70044]: pgmap v20505: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:55.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:51:55 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:51:55.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:51:55 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:51:55.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:51:55 smithi153 ceph-mon[70044]: pgmap v20506: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:55.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:51:55 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:51:55.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:51:55 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:51:55.467 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:51:55 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:51:55.467 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:51:55 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:51:55.468 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:51:55 smithi083 ceph-mon[79930]: pgmap v20506: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:55.468 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:51:55 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:51:55.486 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:51:55 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:51:55.487 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:51:55 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:51:55.488 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:51:55 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:51:55.488 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:51:55 smithi083 ceph-mon[77242]: pgmap v20506: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:55.488 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:51:55 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:51:55.489 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:51:55 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:51:56.435 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:51:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:51:56.080Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:51:57.156 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:51:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:51:56.830Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:51:57.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:51:57 smithi083 ceph-mon[79930]: pgmap v20507: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:57.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:51:57 smithi083 ceph-mon[77242]: pgmap v20507: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:57.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:51:57 smithi153 ceph-mon[70044]: pgmap v20507: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:58.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:51:58 smithi083 ceph-mon[79930]: Detected new or changed devices on smithi083 2022-09-29T16:51:58.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:51:58 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:51:58.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:51:58 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:51:58.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:51:58 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:51:58.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:51:58 smithi083 ceph-mon[79930]: pgmap v20508: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:58.968 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:51:58 smithi083 ceph-mon[77242]: Detected new or changed devices on smithi083 2022-09-29T16:51:58.968 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:51:58 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:51:58.969 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:51:58 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:51:58.969 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:51:58 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:51:58.974 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:51:58 smithi083 ceph-mon[77242]: pgmap v20508: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:51:59.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:51:58 smithi153 ceph-mon[70044]: Detected new or changed devices on smithi083 2022-09-29T16:51:59.064 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:51:58 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:51:59.065 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:51:58 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:51:59.065 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:51:58 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:51:59.066 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:51:58 smithi153 ceph-mon[70044]: pgmap v20508: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:01.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:52:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:52:01] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T16:52:01.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:52:01 smithi083 ceph-mon[79930]: pgmap v20509: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:01.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:52:01 smithi083 ceph-mon[77242]: pgmap v20509: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:01.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:52:01 smithi153 ceph-mon[70044]: pgmap v20509: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:52:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:52:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:52:03.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:52:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:52:02.739Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:52:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:52:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:52:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:52:03.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:52:03 smithi083 ceph-mon[79930]: pgmap v20510: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:03.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:52:03 smithi083 ceph-mon[77242]: pgmap v20510: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:03.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:52:03 smithi153 ceph-mon[70044]: pgmap v20510: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:05.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:52:04 smithi153 ceph-mon[70044]: pgmap v20511: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:05.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:52:04 smithi083 ceph-mon[77242]: pgmap v20511: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:05.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:52:04 smithi083 ceph-mon[79930]: pgmap v20511: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:06.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:52:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:52:06.081Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:52:07.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:52:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:52:06.832Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:52:07.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:52:07 smithi083 ceph-mon[79930]: pgmap v20512: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:07.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:52:07 smithi083 ceph-mon[77242]: pgmap v20512: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:07.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:52:07 smithi153 ceph-mon[70044]: pgmap v20512: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:09.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:52:09 smithi083 ceph-mon[79930]: pgmap v20513: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:09.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:52:09 smithi083 ceph-mon[77242]: pgmap v20513: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:09.827 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:52:09 smithi153 ceph-mon[70044]: pgmap v20513: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:11.720 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:52:11 smithi083 ceph-mon[77242]: pgmap v20514: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:11.721 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:52:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:52:11] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T16:52:11.733 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:52:11 smithi083 ceph-mon[79930]: pgmap v20514: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:11.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:52:11 smithi153 ceph-mon[70044]: pgmap v20514: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:52:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:52:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:52:13.216 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:52:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:52:12.739Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:52:13.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:52:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:52:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:52:13.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:52:13 smithi083 ceph-mon[79930]: pgmap v20515: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:13.718 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:52:13 smithi083 ceph-mon[77242]: pgmap v20515: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:13.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:52:13 smithi153 ceph-mon[70044]: pgmap v20515: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:15.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:52:14 smithi153 ceph-mon[70044]: pgmap v20516: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:15.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:52:14 smithi083 ceph-mon[79930]: pgmap v20516: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:15.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:52:14 smithi083 ceph-mon[77242]: pgmap v20516: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:16.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:52:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:52:16.082Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:52:17.222 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:52:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:52:16.833Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:52:17.726 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:52:17 smithi083 ceph-mon[77242]: pgmap v20517: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:17.734 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:52:17 smithi083 ceph-mon[79930]: pgmap v20517: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:17.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:52:17 smithi153 ceph-mon[70044]: pgmap v20517: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:19.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:52:19 smithi083 ceph-mon[77242]: pgmap v20518: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:19.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:52:19 smithi083 ceph-mon[79930]: pgmap v20518: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:19.817 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:52:19 smithi153 ceph-mon[70044]: pgmap v20518: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:21.106 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:52:21.391 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:52:21.392 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 27s ago 11h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:52:21.392 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 8m ago 11h 64.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:52:21.393 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 27s ago 11h 26.1M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:52:21.394 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 8m ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:52:21.394 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (11h) 8m ago 11h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:52:21.395 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11h) 27s ago 11h 1092M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:52:21.395 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11h) 27s ago 11h 2471M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:52:21.396 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (11h) 8m ago 11h 1093M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:52:21.396 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (11h) 27s ago 11h 1491M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:52:21.396 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 27s ago 11h 24.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:52:21.397 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 8m ago 11h 26.6M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:52:21.397 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11h) 27s ago 11h 1803M 2035M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:52:21.398 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11h) 27s ago 11h 1788M 2035M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:52:21.398 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11h) 27s ago 11h 1388M 2035M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:52:21.405 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (11h) 27s ago 11h 1754M 2035M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:52:21.406 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (11h) 8m ago 11h 1514M 2482M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:52:21.407 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (11h) 8m ago 11h 1903M 2482M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:52:21.407 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (11h) 8m ago 11h 1701M 2482M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:52:21.407 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (11h) 8m ago 11h 2025M 2482M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:52:21.407 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (11h) 8m ago 11h 196M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:52:21.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:52:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:52:21] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T16:52:21.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:52:21 smithi083 ceph-mon[79930]: pgmap v20519: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:21.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:52:21 smithi083 ceph-mon[77242]: pgmap v20519: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:21.753 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:52:21.753 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:52:21.753 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:52:21.754 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:52:21.754 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:52:21.754 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:52:21.755 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:52:21.755 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:52:21.755 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:52:21.755 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:52:21.756 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:52:21.756 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:52:21.756 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:52:21.756 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:52:21.756 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:52:21.757 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:52:21.757 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:52:21.757 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:52:21.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:52:21 smithi153 ceph-mon[70044]: pgmap v20519: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:22.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:52:22 smithi083 ceph-mon[79930]: from='client.57990 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:52:22.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:52:22 smithi083 ceph-mon[79930]: from='client.57996 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:52:22.719 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:52:22 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3498750443' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:52:22.726 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:52:22 smithi083 ceph-mon[77242]: from='client.57990 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:52:22.727 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:52:22 smithi083 ceph-mon[77242]: from='client.57996 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:52:22.727 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:52:22 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3498750443' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:52:22.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:52:22 smithi153 ceph-mon[70044]: from='client.57990 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:52:22.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:52:22 smithi153 ceph-mon[70044]: from='client.57996 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:52:22.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:52:22 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3498750443' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:52:22.811 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:52:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:52:22.316Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:52:23.218 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:52:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:52:22.740Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:52:23.310 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:52:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:52:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:52:23.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:52:23 smithi083 ceph-mon[79930]: pgmap v20520: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:23.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:52:23 smithi083 ceph-mon[77242]: pgmap v20520: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:23.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:52:23 smithi153 ceph-mon[70044]: pgmap v20520: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:25.063 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:52:24 smithi153 ceph-mon[70044]: pgmap v20521: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:25.238 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:52:24 smithi083 ceph-mon[79930]: pgmap v20521: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:25.239 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:52:24 smithi083 ceph-mon[77242]: pgmap v20521: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:26.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:52:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:52:26.082Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:52:27.217 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:52:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:52:26.834Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:52:27.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:52:27 smithi083 ceph-mon[79930]: pgmap v20522: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:27.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:52:27 smithi083 ceph-mon[77242]: pgmap v20522: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:27.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:52:27 smithi153 ceph-mon[70044]: pgmap v20522: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:29.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:52:29 smithi083 ceph-mon[79930]: pgmap v20523: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:29.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:52:29 smithi083 ceph-mon[77242]: pgmap v20523: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:29.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:52:29 smithi153 ceph-mon[70044]: pgmap v20523: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:31.715 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:52:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:52:31] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T16:52:31.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:52:31 smithi083 ceph-mon[79930]: pgmap v20524: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:31.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:52:31 smithi083 ceph-mon[77242]: pgmap v20524: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:31.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:52:31 smithi153 ceph-mon[70044]: pgmap v20524: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:32.825 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:52:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:52:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:52:33.217 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:52:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:52:32.740Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:52:33.312 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:52:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:52:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:52:33.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:52:33 smithi083 ceph-mon[79930]: pgmap v20525: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:33.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:52:33 smithi083 ceph-mon[77242]: pgmap v20525: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:33.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:52:33 smithi153 ceph-mon[70044]: pgmap v20525: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:35.066 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:52:34 smithi153 ceph-mon[70044]: pgmap v20526: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:35.219 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:52:34 smithi083 ceph-mon[79930]: pgmap v20526: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:35.230 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:52:34 smithi083 ceph-mon[77242]: pgmap v20526: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:36.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:52:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:52:36.084Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:52:37.230 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:52:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:52:36.834Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:52:37.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:52:37 smithi083 ceph-mon[77242]: pgmap v20527: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:37.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:52:37 smithi083 ceph-mon[79930]: pgmap v20527: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:37.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:52:37 smithi153 ceph-mon[70044]: pgmap v20527: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:39.737 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:52:39 smithi083 ceph-mon[79930]: pgmap v20528: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:39.738 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:52:39 smithi083 ceph-mon[77242]: pgmap v20528: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:39.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:52:39 smithi153 ceph-mon[70044]: pgmap v20528: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:41.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:52:41 smithi083 ceph-mon[79930]: pgmap v20529: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:41.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:52:41 smithi083 ceph-mon[77242]: pgmap v20529: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:41.718 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:52:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:52:41] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T16:52:41.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:52:41 smithi153 ceph-mon[70044]: pgmap v20529: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:42.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:52:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:52:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:52:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:52:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:52:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:52:42.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:52:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:52:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:52:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:52:42.731 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:52:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:52:42.731 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:52:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:52:42.732 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:52:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:52:42.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:52:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:52:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:52:42.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:52:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:52:42.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:52:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:52:42.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:52:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:52:42.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:52:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:52:43.216 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:52:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:52:42.741Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:52:43.317 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:52:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:52:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:52:43.728 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:52:43 smithi083 ceph-mon[79930]: pgmap v20530: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:43.729 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:52:43 smithi083 ceph-mon[77242]: pgmap v20530: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:43.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:52:43 smithi153 ceph-mon[70044]: pgmap v20530: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:45.064 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:52:44 smithi153 ceph-mon[70044]: pgmap v20531: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:45.250 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:52:44 smithi083 ceph-mon[77242]: pgmap v20531: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:45.252 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:52:44 smithi083 ceph-mon[79930]: pgmap v20531: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:46.466 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:52:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:52:46.085Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:52:47.221 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:52:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:52:46.835Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:52:47.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:52:47 smithi083 ceph-mon[79930]: pgmap v20532: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:47.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:52:47 smithi083 ceph-mon[77242]: pgmap v20532: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:47.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:52:47 smithi153 ceph-mon[70044]: pgmap v20532: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:49.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:52:49 smithi083 ceph-mon[79930]: pgmap v20533: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:49.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:52:49 smithi083 ceph-mon[77242]: pgmap v20533: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:49.835 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:52:49 smithi153 ceph-mon[70044]: pgmap v20533: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:51.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:52:51 smithi083 ceph-mon[79930]: pgmap v20534: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:51.718 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:52:51 smithi083 ceph-mon[77242]: pgmap v20534: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:51.719 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:52:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:52:51] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T16:52:51.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:52:51 smithi153 ceph-mon[70044]: pgmap v20534: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:52.081 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:52:52.370 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:52:52.371 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 58s ago 11h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:52:52.371 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 9m ago 11h 64.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:52:52.371 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 58s ago 11h 26.1M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:52:52.371 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 9m ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:52:52.372 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (11h) 9m ago 11h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:52:52.372 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11h) 58s ago 11h 1092M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:52:52.372 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11h) 58s ago 11h 2471M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:52:52.372 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (11h) 9m ago 11h 1093M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:52:52.373 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (11h) 58s ago 11h 1491M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:52:52.373 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 58s ago 11h 24.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:52:52.373 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 9m ago 11h 26.6M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:52:52.373 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11h) 58s ago 11h 1803M 2035M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:52:52.373 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11h) 58s ago 11h 1788M 2035M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:52:52.378 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11h) 58s ago 11h 1388M 2035M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:52:52.379 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (11h) 58s ago 11h 1754M 2035M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:52:52.379 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (11h) 9m ago 11h 1514M 2482M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:52:52.380 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (11h) 9m ago 11h 1903M 2482M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:52:52.380 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (11h) 9m ago 11h 1701M 2482M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:52:52.381 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (11h) 9m ago 11h 2025M 2482M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:52:52.381 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (11h) 9m ago 11h 196M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:52:52.726 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:52:52.726 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:52:52.727 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:52:52.728 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:52:52.728 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:52:52.729 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:52:52.729 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:52:52.730 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:52:52.730 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:52:52.731 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:52:52.732 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:52:52.733 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:52:52.735 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:52:52.735 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:52:52.736 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:52:52.736 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:52:52.736 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:52:52.737 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:52:52.811 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:52:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:52:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:52:53.214 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:52:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:52:52.741Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:52:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:52:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:52:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:52:53.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:52:53 smithi083 ceph-mon[79930]: from='client.67939 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:52:53.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:52:53 smithi083 ceph-mon[79930]: pgmap v20535: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:53.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:52:53 smithi083 ceph-mon[79930]: from='client.67945 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:52:53.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:52:53 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/148265551' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:52:53.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:52:53 smithi083 ceph-mon[77242]: from='client.67939 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:52:53.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:52:53 smithi083 ceph-mon[77242]: pgmap v20535: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:53.718 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:52:53 smithi083 ceph-mon[77242]: from='client.67945 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:52:53.718 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:52:53 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/148265551' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:52:53.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:52:53 smithi153 ceph-mon[70044]: from='client.67939 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:52:53.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:52:53 smithi153 ceph-mon[70044]: pgmap v20535: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:53.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:52:53 smithi153 ceph-mon[70044]: from='client.67945 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:52:53.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:52:53 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/148265551' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:52:55.065 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:52:54 smithi153 ceph-mon[70044]: pgmap v20536: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:55.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:52:54 smithi083 ceph-mon[79930]: pgmap v20536: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:55.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:52:54 smithi083 ceph-mon[77242]: pgmap v20536: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:56.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:52:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:52:56.086Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:52:57.227 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:52:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:52:56.836Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:52:57.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:52:57 smithi083 ceph-mon[79930]: pgmap v20537: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:57.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:52:57 smithi083 ceph-mon[77242]: pgmap v20537: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:57.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:52:57 smithi153 ceph-mon[70044]: pgmap v20537: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:58.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:52:58 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:52:58.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:52:58 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:52:58.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:52:58 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:52:58.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:52:58 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:52:58.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:52:58 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:52:58.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:52:58 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:52:58.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:52:58 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:52:58.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:52:58 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:52:58.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:52:58 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:52:58.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:52:58 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:52:58.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:52:58 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:52:58.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:52:58 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:52:59.721 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:52:59 smithi083 ceph-mon[79930]: pgmap v20538: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:59.734 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:52:59 smithi083 ceph-mon[77242]: pgmap v20538: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:52:59.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:52:59 smithi153 ceph-mon[70044]: pgmap v20538: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:01.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:53:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:53:01] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T16:53:01.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:53:01 smithi083 ceph-mon[79930]: pgmap v20539: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:01.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:53:01 smithi083 ceph-mon[77242]: pgmap v20539: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:01.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:53:01 smithi153 ceph-mon[70044]: pgmap v20539: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:02.810 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:53:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:53:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:53:03.217 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:53:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:53:02.743Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:53:03.310 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:53:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:53:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:53:03.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:53:03 smithi083 ceph-mon[79930]: pgmap v20540: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:03.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:53:03 smithi083 ceph-mon[77242]: pgmap v20540: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:03.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:53:03 smithi153 ceph-mon[70044]: pgmap v20540: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:05.064 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:53:04 smithi153 ceph-mon[70044]: pgmap v20541: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:05.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:53:04 smithi083 ceph-mon[77242]: pgmap v20541: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:05.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:53:04 smithi083 ceph-mon[79930]: pgmap v20541: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:06.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:53:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:53:06.087Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:53:07.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:53:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:53:06.837Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:53:07.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:53:07 smithi083 ceph-mon[79930]: pgmap v20542: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:07.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:53:07 smithi083 ceph-mon[77242]: pgmap v20542: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:07.814 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:53:07 smithi153 ceph-mon[70044]: pgmap v20542: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:09.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:53:09 smithi083 ceph-mon[79930]: pgmap v20543: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:09.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:53:09 smithi083 ceph-mon[77242]: pgmap v20543: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:09.813 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:53:09 smithi153 ceph-mon[70044]: pgmap v20543: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:11.714 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:53:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:53:11] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T16:53:11.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:53:11 smithi083 ceph-mon[79930]: pgmap v20544: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:11.729 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:53:11 smithi083 ceph-mon[77242]: pgmap v20544: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:11.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:53:11 smithi153 ceph-mon[70044]: pgmap v20544: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:12.812 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:53:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:53:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:53:13.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:53:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:53:12.743Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:53:13.310 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:53:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:53:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:53:13.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:53:13 smithi083 ceph-mon[79930]: pgmap v20545: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:13.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:53:13 smithi083 ceph-mon[77242]: pgmap v20545: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:13.818 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:53:13 smithi153 ceph-mon[70044]: pgmap v20545: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:15.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:53:14 smithi153 ceph-mon[70044]: pgmap v20546: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:15.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:53:14 smithi083 ceph-mon[79930]: pgmap v20546: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:15.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:53:14 smithi083 ceph-mon[77242]: pgmap v20546: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:16.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:53:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:53:16.088Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:53:17.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:53:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:53:16.838Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:53:17.734 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:53:17 smithi083 ceph-mon[77242]: pgmap v20547: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:17.736 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:53:17 smithi083 ceph-mon[79930]: pgmap v20547: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:17.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:53:17 smithi153 ceph-mon[70044]: pgmap v20547: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:19.731 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:53:19 smithi083 ceph-mon[79930]: pgmap v20548: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:19.732 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:53:19 smithi083 ceph-mon[77242]: pgmap v20548: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:19.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:53:19 smithi153 ceph-mon[70044]: pgmap v20548: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:21.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:53:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:53:21] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T16:53:21.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:53:21 smithi083 ceph-mon[79930]: pgmap v20549: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:21.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:53:21 smithi083 ceph-mon[77242]: pgmap v20549: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:21.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:53:21 smithi153 ceph-mon[70044]: pgmap v20549: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:22.819 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:53:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:53:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:53:23.049 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:53:23.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:53:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:53:22.745Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:53:23.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:53:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:53:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:53:23.348 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:53:23.349 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 89s ago 11h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:53:23.349 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 9m ago 11h 64.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:53:23.349 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 89s ago 11h 26.1M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:53:23.369 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 9m ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:53:23.370 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (11h) 9m ago 11h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:53:23.390 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11h) 89s ago 11h 1092M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:53:23.391 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11h) 89s ago 11h 2471M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:53:23.391 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (11h) 9m ago 11h 1093M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:53:23.391 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (11h) 89s ago 11h 1491M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:53:23.392 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 89s ago 11h 24.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:53:23.392 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 9m ago 11h 26.6M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:53:23.392 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11h) 89s ago 11h 1803M 2035M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:53:23.393 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11h) 89s ago 11h 1788M 2035M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:53:23.393 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11h) 89s ago 11h 1388M 2035M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:53:23.394 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (11h) 89s ago 11h 1754M 2035M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:53:23.394 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (11h) 9m ago 11h 1514M 2482M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:53:23.395 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (11h) 9m ago 11h 1903M 2482M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:53:23.395 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (11h) 9m ago 11h 1701M 2482M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:53:23.396 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (11h) 9m ago 11h 2025M 2482M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:53:23.396 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (11h) 9m ago 11h 196M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:53:23.709 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:53:23.710 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:53:23.710 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:53:23.711 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:53:23.711 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:53:23.712 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:53:23.712 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:53:23.713 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:53:23.713 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:53:23.713 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:53:23.730 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:53:23.731 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:53:23.731 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:53:23.731 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:53:23.731 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:53:23.732 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:53:23.732 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:53:23.732 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:53:23.733 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:53:23 smithi083 ceph-mon[79930]: pgmap v20550: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:23.733 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:53:23 smithi083 ceph-mon[77242]: pgmap v20550: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:23.813 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:53:23 smithi153 ceph-mon[70044]: pgmap v20550: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:24.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:53:24 smithi083 ceph-mon[79930]: from='client.58026 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:53:24.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:53:24 smithi083 ceph-mon[79930]: from='client.58032 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:53:24.722 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:53:24 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1103361426' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:53:24.723 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:53:24 smithi083 ceph-mon[77242]: from='client.58026 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:53:24.724 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:53:24 smithi083 ceph-mon[77242]: from='client.58032 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:53:24.724 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:53:24 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1103361426' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:53:24.826 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:53:24 smithi153 ceph-mon[70044]: from='client.58026 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:53:24.827 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:53:24 smithi153 ceph-mon[70044]: from='client.58032 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:53:24.827 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:53:24 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1103361426' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:53:25.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:53:25 smithi083 ceph-mon[79930]: pgmap v20551: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:25.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:53:25 smithi083 ceph-mon[77242]: pgmap v20551: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:25.817 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:53:25 smithi153 ceph-mon[70044]: pgmap v20551: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:26.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:53:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:53:26.088Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:53:27.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:53:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:53:26.838Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:53:27.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:53:27 smithi083 ceph-mon[79930]: pgmap v20552: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:27.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:53:27 smithi083 ceph-mon[77242]: pgmap v20552: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:27.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:53:27 smithi153 ceph-mon[70044]: pgmap v20552: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:29.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:53:29 smithi083 ceph-mon[79930]: pgmap v20553: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:29.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:53:29 smithi083 ceph-mon[77242]: pgmap v20553: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:29.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:53:29 smithi153 ceph-mon[70044]: pgmap v20553: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:31.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:53:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:53:31] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T16:53:31.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:53:31 smithi083 ceph-mon[79930]: pgmap v20554: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:31.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:53:31 smithi083 ceph-mon[77242]: pgmap v20554: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:31.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:53:31 smithi153 ceph-mon[70044]: pgmap v20554: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:32.814 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:53:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:53:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:53:33.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:53:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:53:32.744Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:53:33.311 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:53:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:53:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:53:33.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:53:33 smithi083 ceph-mon[79930]: pgmap v20555: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:33.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:53:33 smithi083 ceph-mon[77242]: pgmap v20555: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:33.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:53:33 smithi153 ceph-mon[70044]: pgmap v20555: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:35.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:53:34 smithi153 ceph-mon[70044]: pgmap v20556: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:35.222 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:53:34 smithi083 ceph-mon[77242]: pgmap v20556: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:35.227 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:53:34 smithi083 ceph-mon[79930]: pgmap v20556: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:36.475 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:53:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:53:36.089Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:53:37.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:53:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:53:36.839Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:53:37.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:53:37 smithi083 ceph-mon[79930]: pgmap v20557: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:37.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:53:37 smithi083 ceph-mon[77242]: pgmap v20557: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:37.813 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:53:37 smithi153 ceph-mon[70044]: pgmap v20557: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:39.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:53:39 smithi083 ceph-mon[79930]: pgmap v20558: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:39.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:53:39 smithi083 ceph-mon[77242]: pgmap v20558: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:39.815 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:53:39 smithi153 ceph-mon[70044]: pgmap v20558: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:41.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:53:41 smithi083 ceph-mon[79930]: pgmap v20559: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:41.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:53:41 smithi083 ceph-mon[77242]: pgmap v20559: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:41.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:53:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:53:41] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T16:53:41.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:53:41 smithi153 ceph-mon[70044]: pgmap v20559: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:53:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:53:42.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:53:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:53:42.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:53:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:53:42.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:53:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:53:42.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:53:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:53:42.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:53:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:53:42.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:53:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:53:42.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:53:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:53:42.814 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:53:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:53:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:53:42.815 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:53:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:53:42.816 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:53:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:53:42.816 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:53:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:53:42.816 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:53:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:53:43.216 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:53:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:53:42.745Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:53:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:53:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:53:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:53:43.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:53:43 smithi083 ceph-mon[77242]: pgmap v20560: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:43.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:53:43 smithi083 ceph-mon[79930]: pgmap v20560: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:43.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:53:43 smithi153 ceph-mon[70044]: pgmap v20560: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:45.063 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:53:44 smithi153 ceph-mon[70044]: pgmap v20561: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:45.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:53:44 smithi083 ceph-mon[79930]: pgmap v20561: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:45.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:53:44 smithi083 ceph-mon[77242]: pgmap v20561: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:46.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:53:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:53:46.089Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:53:47.223 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:53:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:53:46.840Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:53:47.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:53:47 smithi083 ceph-mon[77242]: pgmap v20562: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:47.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:53:47 smithi083 ceph-mon[79930]: pgmap v20562: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:47.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:53:47 smithi153 ceph-mon[70044]: pgmap v20562: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:49.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:53:49 smithi083 ceph-mon[77242]: pgmap v20563: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:49.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:53:49 smithi083 ceph-mon[79930]: pgmap v20563: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:49.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:53:49 smithi153 ceph-mon[70044]: pgmap v20563: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:51.717 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:53:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:53:51] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T16:53:51.718 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:53:51 smithi083 ceph-mon[79930]: pgmap v20564: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:51.719 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:53:51 smithi083 ceph-mon[77242]: pgmap v20564: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:51.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:53:51 smithi153 ceph-mon[70044]: pgmap v20564: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:53:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:53:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:53:53.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:53:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:53:52.745Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:53:53.311 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:53:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:53:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:53:53.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:53:53 smithi083 ceph-mon[77242]: pgmap v20565: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:53.718 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:53:53 smithi083 ceph-mon[79930]: pgmap v20565: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:53.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:53:53 smithi153 ceph-mon[70044]: pgmap v20565: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:54.053 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:53:54.323 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:53:54.323 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 2m ago 11h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:53:54.324 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 10m ago 11h 64.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:53:54.324 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 2m ago 11h 26.1M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:53:54.324 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 10m ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:53:54.324 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (11h) 10m ago 11h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:53:54.325 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11h) 2m ago 11h 1092M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:53:54.325 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11h) 2m ago 11h 2471M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:53:54.325 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (11h) 10m ago 11h 1093M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:53:54.325 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (11h) 2m ago 11h 1491M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:53:54.326 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 2m ago 11h 24.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:53:54.326 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 10m ago 11h 26.6M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:53:54.326 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11h) 2m ago 11h 1803M 2035M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:53:54.326 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11h) 2m ago 11h 1788M 2035M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:53:54.327 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11h) 2m ago 11h 1388M 2035M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:53:54.327 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (11h) 2m ago 11h 1754M 2035M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:53:54.327 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (11h) 10m ago 11h 1514M 2482M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:53:54.327 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (11h) 10m ago 11h 1903M 2482M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:53:54.328 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (11h) 10m ago 11h 1701M 2482M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:53:54.328 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (11h) 10m ago 11h 2025M 2482M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:53:54.328 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (11h) 10m ago 11h 196M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:53:54.689 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:53:54.690 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:53:54.691 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:53:54.691 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:53:54.691 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:53:54.692 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:53:54.692 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:53:54.693 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:53:54.693 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:53:54.715 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:53:54.715 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:53:54.716 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:53:54.716 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:53:54.717 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:53:54.717 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:53:54.717 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:53:54.718 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:53:54.718 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:53:55.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:53:54 smithi153 ceph-mon[70044]: from='client.67975 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:53:55.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:53:54 smithi153 ceph-mon[70044]: pgmap v20566: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:55.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:53:54 smithi153 ceph-mon[70044]: from='client.58050 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:53:55.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:53:54 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2687366866' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:53:55.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:53:54 smithi083 ceph-mon[79930]: from='client.67975 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:53:55.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:53:54 smithi083 ceph-mon[79930]: pgmap v20566: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:55.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:53:54 smithi083 ceph-mon[79930]: from='client.58050 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:53:55.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:53:54 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2687366866' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:53:55.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:53:54 smithi083 ceph-mon[77242]: from='client.67975 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:53:55.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:53:54 smithi083 ceph-mon[77242]: pgmap v20566: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:55.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:53:54 smithi083 ceph-mon[77242]: from='client.58050 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:53:55.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:53:54 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2687366866' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:53:56.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:53:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:53:56.090Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:53:57.214 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:53:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:53:56.841Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:53:57.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:53:57 smithi083 ceph-mon[77242]: pgmap v20567: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:57.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:53:57 smithi083 ceph-mon[79930]: pgmap v20567: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:57.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:53:57 smithi153 ceph-mon[70044]: pgmap v20567: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:58.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:53:58 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:53:58.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:53:58 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:53:58.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:53:58 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:53:58.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:53:58 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:53:58.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:53:58 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:53:58.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:53:58 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:53:58.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:53:58 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:53:58.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:53:58 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:53:58.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:53:58 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:53:59.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:53:59 smithi083 ceph-mon[79930]: pgmap v20568: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:59.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:53:59 smithi083 ceph-mon[77242]: pgmap v20568: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:53:59.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:53:59 smithi153 ceph-mon[70044]: pgmap v20568: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:01.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:54:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:54:01] "GET /metrics HTTP/1.1" 200 225246 "" "Prometheus/2.33.4" 2022-09-29T16:54:01.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:54:01 smithi083 ceph-mon[79930]: pgmap v20569: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:01.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:54:01 smithi083 ceph-mon[77242]: pgmap v20569: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:01.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:54:01 smithi153 ceph-mon[70044]: pgmap v20569: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:54:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:54:02.316Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:54:03.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:54:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:54:02.746Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:54:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:54:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:54:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:54:03.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:54:03 smithi083 ceph-mon[79930]: pgmap v20570: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:03.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:54:03 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:54:03.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:54:03 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:54:03.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:54:03 smithi083 ceph-mon[77242]: pgmap v20570: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:03.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:54:03 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:54:03.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:54:03 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:54:03.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:54:03 smithi153 ceph-mon[70044]: pgmap v20570: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:03.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:54:03 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:54:03.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:54:03 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:54:05.179 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:54:05 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:54:05.180 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:54:05 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:54:05.180 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:54:05 smithi153 ceph-mon[70044]: pgmap v20571: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:05.468 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:54:05 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:54:05.469 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:54:05 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:54:05.469 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:54:05 smithi083 ceph-mon[79930]: pgmap v20571: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:05.481 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:54:05 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:54:05.482 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:54:05 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:54:05.482 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:54:05 smithi083 ceph-mon[77242]: pgmap v20571: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:06.490 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:54:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:54:06.091Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:54:07.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:54:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:54:06.841Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:54:07.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:54:07 smithi083 ceph-mon[79930]: pgmap v20572: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:07.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:54:07 smithi083 ceph-mon[79930]: Detected new or changed devices on smithi153 2022-09-29T16:54:07.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:54:07 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:54:07.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:54:07 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:54:07.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:54:07 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:54:07.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:54:07 smithi083 ceph-mon[77242]: pgmap v20572: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:07.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:54:07 smithi083 ceph-mon[77242]: Detected new or changed devices on smithi153 2022-09-29T16:54:07.718 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:54:07 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:54:07.718 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:54:07 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:54:07.718 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:54:07 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:54:07.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:54:07 smithi153 ceph-mon[70044]: pgmap v20572: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:07.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:54:07 smithi153 ceph-mon[70044]: Detected new or changed devices on smithi153 2022-09-29T16:54:07.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:54:07 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:54:07.819 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:54:07 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:54:07.820 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:54:07 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:54:09.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:54:09 smithi153 ceph-mon[70044]: pgmap v20573: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:09.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:54:09 smithi083 ceph-mon[77242]: pgmap v20573: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:09.974 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:54:09 smithi083 ceph-mon[79930]: pgmap v20573: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:11.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:54:11 smithi153 ceph-mon[70044]: pgmap v20574: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:54:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:54:11] "GET /metrics HTTP/1.1" 200 225246 "" "Prometheus/2.33.4" 2022-09-29T16:54:11.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:54:11 smithi083 ceph-mon[79930]: pgmap v20574: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:11.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:54:11 smithi083 ceph-mon[77242]: pgmap v20574: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:54:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:54:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:54:13.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:54:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:54:12.746Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:54:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:54:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:54:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:54:13.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:54:13 smithi153 ceph-mon[70044]: pgmap v20575: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:13.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:54:13 smithi083 ceph-mon[79930]: pgmap v20575: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:13.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:54:13 smithi083 ceph-mon[77242]: pgmap v20575: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:15.065 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:54:14 smithi153 ceph-mon[70044]: pgmap v20576: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:15.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:54:14 smithi083 ceph-mon[77242]: pgmap v20576: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:15.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:54:14 smithi083 ceph-mon[79930]: pgmap v20576: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:16.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:54:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:54:16.092Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:54:17.216 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:54:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:54:16.842Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:54:17.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:54:17 smithi083 ceph-mon[77242]: pgmap v20577: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:17.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:54:17 smithi083 ceph-mon[79930]: pgmap v20577: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:17.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:54:17 smithi153 ceph-mon[70044]: pgmap v20577: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:19.716 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:54:19 smithi083 ceph-mon[79930]: pgmap v20578: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:19.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:54:19 smithi083 ceph-mon[77242]: pgmap v20578: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:19.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:54:19 smithi153 ceph-mon[70044]: pgmap v20578: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:21.720 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:54:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:54:21] "GET /metrics HTTP/1.1" 200 225246 "" "Prometheus/2.33.4" 2022-09-29T16:54:21.721 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:54:21 smithi083 ceph-mon[79930]: pgmap v20579: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:21.722 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:54:21 smithi083 ceph-mon[77242]: pgmap v20579: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:21.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:54:21 smithi153 ceph-mon[70044]: pgmap v20579: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:54:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:54:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:54:23.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:54:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:54:22.747Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:54:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:54:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:54:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:54:23.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:54:23 smithi083 ceph-mon[79930]: pgmap v20580: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:23.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:54:23 smithi083 ceph-mon[77242]: pgmap v20580: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:23.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:54:23 smithi153 ceph-mon[70044]: pgmap v20580: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:25.011 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:54:25.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:54:24 smithi153 ceph-mon[70044]: pgmap v20581: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:25.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:54:24 smithi083 ceph-mon[79930]: pgmap v20581: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:25.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:54:24 smithi083 ceph-mon[77242]: pgmap v20581: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:25.300 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:54:25.301 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 2m ago 11h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:54:25.301 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 22s ago 11h 63.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:54:25.302 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 2m ago 11h 26.1M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:54:25.302 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 22s ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:54:25.303 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (11h) 22s ago 11h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:54:25.303 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11h) 2m ago 11h 1092M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:54:25.303 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11h) 2m ago 11h 2471M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:54:25.304 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (11h) 22s ago 11h 1111M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:54:25.305 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (11h) 2m ago 11h 1491M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:54:25.305 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 2m ago 11h 24.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:54:25.306 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 22s ago 11h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:54:25.306 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11h) 2m ago 11h 1803M 2035M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:54:25.307 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11h) 2m ago 11h 1788M 2035M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:54:25.308 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11h) 2m ago 11h 1388M 2035M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:54:25.308 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (11h) 2m ago 11h 1754M 2035M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:54:25.309 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (11h) 22s ago 11h 1536M 2482M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:54:25.311 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (11h) 22s ago 11h 1932M 2482M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:54:25.312 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (11h) 22s ago 11h 1726M 2482M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:54:25.312 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (11h) 22s ago 11h 2056M 2482M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:54:25.313 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (11h) 22s ago 11h 199M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:54:25.651 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:54:25.651 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:54:25.652 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:54:25.653 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:54:25.653 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:54:25.653 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:54:25.653 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:54:25.654 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:54:25.654 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:54:25.655 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:54:25.655 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:54:25.660 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:54:25.661 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:54:25.662 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:54:25.668 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:54:25.668 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:54:25.669 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:54:25.669 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:54:26.092 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:54:25 smithi083 ceph-mon[77242]: from='client.58062 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:54:26.093 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:54:25 smithi083 ceph-mon[77242]: from='client.67999 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:54:26.093 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:54:25 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2261421743' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:54:26.094 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:54:25 smithi083 ceph-mon[79930]: from='client.58062 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:54:26.094 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:54:25 smithi083 ceph-mon[79930]: from='client.67999 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:54:26.094 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:54:25 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2261421743' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:54:26.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:54:25 smithi153 ceph-mon[70044]: from='client.58062 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:54:26.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:54:25 smithi153 ceph-mon[70044]: from='client.67999 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:54:26.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:54:25 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2261421743' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:54:26.475 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:54:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:54:26.092Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:54:27.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:54:26 smithi083 ceph-mon[79930]: pgmap v20582: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:27.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:54:26 smithi083 ceph-mon[77242]: pgmap v20582: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:27.216 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:54:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:54:26.842Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:54:27.312 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:54:26 smithi153 ceph-mon[70044]: pgmap v20582: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:29.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:54:29 smithi083 ceph-mon[77242]: pgmap v20583: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:29.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:54:29 smithi083 ceph-mon[79930]: pgmap v20583: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:29.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:54:29 smithi153 ceph-mon[70044]: pgmap v20583: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:31.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:54:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:54:31] "GET /metrics HTTP/1.1" 200 225246 "" "Prometheus/2.33.4" 2022-09-29T16:54:31.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:54:31 smithi083 ceph-mon[79930]: pgmap v20584: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:31.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:54:31 smithi083 ceph-mon[77242]: pgmap v20584: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:31.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:54:31 smithi153 ceph-mon[70044]: pgmap v20584: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:54:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:54:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:54:33.214 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:54:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:54:32.747Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:54:33.310 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:54:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:54:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:54:33.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:54:33 smithi083 ceph-mon[79930]: pgmap v20585: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:33.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:54:33 smithi083 ceph-mon[77242]: pgmap v20585: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:33.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:54:33 smithi153 ceph-mon[70044]: pgmap v20585: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:35.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:54:34 smithi153 ceph-mon[70044]: pgmap v20586: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:35.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:54:34 smithi083 ceph-mon[77242]: pgmap v20586: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:35.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:54:34 smithi083 ceph-mon[79930]: pgmap v20586: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:36.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:54:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:54:36.093Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:54:37.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:54:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:54:36.844Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:54:37.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:54:37 smithi083 ceph-mon[77242]: pgmap v20587: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:37.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:54:37 smithi083 ceph-mon[79930]: pgmap v20587: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:37.817 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:54:37 smithi153 ceph-mon[70044]: pgmap v20587: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:39.736 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:54:39 smithi083 ceph-mon[77242]: pgmap v20588: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:39.737 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:54:39 smithi083 ceph-mon[79930]: pgmap v20588: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:39.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:54:39 smithi153 ceph-mon[70044]: pgmap v20588: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:41.714 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:54:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:54:41] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T16:54:41.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:54:41 smithi083 ceph-mon[79930]: pgmap v20589: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:41.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:54:41 smithi083 ceph-mon[77242]: pgmap v20589: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:41.813 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:54:41 smithi153 ceph-mon[70044]: pgmap v20589: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:42.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:54:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:54:42.731 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:54:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:54:42.732 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:54:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:54:42.732 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:54:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:54:42.732 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:54:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:54:42.733 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:54:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:54:42.733 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:54:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:54:42.733 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:54:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:54:42.811 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:54:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:54:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:54:42.815 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:54:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:54:42.816 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:54:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:54:42.816 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:54:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:54:42.816 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:54:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:54:43.219 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:54:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:54:42.748Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:54:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:54:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:54:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:54:43.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:54:43 smithi083 ceph-mon[79930]: pgmap v20590: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:43.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:54:43 smithi083 ceph-mon[77242]: pgmap v20590: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:43.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:54:43 smithi153 ceph-mon[70044]: pgmap v20590: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:45.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:54:44 smithi083 ceph-mon[79930]: pgmap v20591: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:45.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:54:44 smithi083 ceph-mon[77242]: pgmap v20591: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:45.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:54:44 smithi153 ceph-mon[70044]: pgmap v20591: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:46.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:54:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:54:46.094Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:54:47.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:54:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:54:46.844Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:54:47.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:54:47 smithi083 ceph-mon[79930]: pgmap v20592: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:47.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:54:47 smithi083 ceph-mon[77242]: pgmap v20592: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:47.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:54:47 smithi153 ceph-mon[70044]: pgmap v20592: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:49.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:54:49 smithi083 ceph-mon[79930]: pgmap v20593: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:49.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:54:49 smithi083 ceph-mon[77242]: pgmap v20593: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:49.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:54:49 smithi153 ceph-mon[70044]: pgmap v20593: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:51.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:54:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:54:51] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T16:54:51.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:54:51 smithi083 ceph-mon[79930]: pgmap v20594: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:51.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:54:51 smithi083 ceph-mon[77242]: pgmap v20594: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:51.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:54:51 smithi153 ceph-mon[70044]: pgmap v20594: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:52.810 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:54:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:54:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:54:53.215 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:54:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:54:52.748Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:54:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:54:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:54:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:54:53.717 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:54:53 smithi083 ceph-mon[79930]: pgmap v20595: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:53.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:54:53 smithi083 ceph-mon[77242]: pgmap v20595: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:53.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:54:53 smithi153 ceph-mon[70044]: pgmap v20595: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:55.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:54:54 smithi083 ceph-mon[77242]: pgmap v20596: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:55.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:54:54 smithi083 ceph-mon[79930]: pgmap v20596: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:55.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:54:54 smithi153 ceph-mon[70044]: pgmap v20596: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:55.978 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:54:56.267 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:54:56.267 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 3m ago 11h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:54:56.267 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 53s ago 11h 63.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:54:56.268 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 3m ago 11h 26.1M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:54:56.268 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 53s ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:54:56.268 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (11h) 53s ago 11h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:54:56.268 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11h) 3m ago 11h 1092M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:54:56.269 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11h) 3m ago 11h 2471M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:54:56.270 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (11h) 53s ago 11h 1111M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:54:56.270 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (11h) 3m ago 11h 1491M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:54:56.270 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 3m ago 11h 24.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:54:56.271 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 53s ago 11h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:54:56.271 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11h) 3m ago 11h 1803M 2035M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:54:56.272 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11h) 3m ago 11h 1788M 2035M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:54:56.272 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11h) 3m ago 11h 1388M 2035M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:54:56.272 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (11h) 3m ago 11h 1754M 2035M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:54:56.272 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (11h) 53s ago 11h 1536M 2482M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:54:56.273 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (11h) 53s ago 11h 1932M 2482M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:54:56.273 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (11h) 53s ago 11h 1726M 2482M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:54:56.273 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (11h) 53s ago 11h 2056M 2482M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:54:56.273 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (11h) 53s ago 11h 199M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:54:56.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:54:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:54:56.094Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:54:56.624 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:54:56.625 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:54:56.625 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:54:56.625 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:54:56.626 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:54:56.627 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:54:56.627 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:54:56.628 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:54:56.628 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:54:56.629 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:54:56.629 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:54:56.630 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:54:56.630 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:54:56.631 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:54:56.631 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:54:56.631 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:54:56.632 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:54:56.632 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:54:57.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:54:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:54:56.844Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:54:57.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:54:57 smithi083 ceph-mon[77242]: from='client.68011 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:54:57.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:54:57 smithi083 ceph-mon[77242]: from='client.58086 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:54:57.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:54:57 smithi083 ceph-mon[77242]: pgmap v20597: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:57.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:54:57 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2560186078' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:54:57.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:54:57 smithi083 ceph-mon[79930]: from='client.68011 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:54:57.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:54:57 smithi083 ceph-mon[79930]: from='client.58086 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:54:57.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:54:57 smithi083 ceph-mon[79930]: pgmap v20597: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:57.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:54:57 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2560186078' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:54:57.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:54:57 smithi153 ceph-mon[70044]: from='client.68011 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:54:57.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:54:57 smithi153 ceph-mon[70044]: from='client.58086 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:54:57.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:54:57 smithi153 ceph-mon[70044]: pgmap v20597: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:57.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:54:57 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2560186078' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:54:59.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:54:59 smithi083 ceph-mon[79930]: pgmap v20598: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:59.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:54:59 smithi083 ceph-mon[77242]: pgmap v20598: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:54:59.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:54:59 smithi153 ceph-mon[70044]: pgmap v20598: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:01.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:55:01 smithi083 ceph-mon[77242]: pgmap v20599: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:01.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:55:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:55:01] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T16:55:01.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:55:01 smithi083 ceph-mon[79930]: pgmap v20599: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:01.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:55:01 smithi153 ceph-mon[70044]: pgmap v20599: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:55:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:55:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:55:03.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:55:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:55:02.748Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:55:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:55:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:55:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:55:03.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:55:03 smithi083 ceph-mon[77242]: pgmap v20600: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:03.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:55:03 smithi083 ceph-mon[79930]: pgmap v20600: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:03.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:55:03 smithi153 ceph-mon[70044]: pgmap v20600: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:05.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:55:04 smithi083 ceph-mon[77242]: pgmap v20601: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:05.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:55:04 smithi083 ceph-mon[79930]: pgmap v20601: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:05.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:55:04 smithi153 ceph-mon[70044]: pgmap v20601: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:06.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:55:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:55:06.096Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:55:07.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:55:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:55:06.844Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:55:07.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:55:07 smithi083 ceph-mon[79930]: pgmap v20602: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:07.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:55:07 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:55:07.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:55:07 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:55:07.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:55:07 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:55:07.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:55:07 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:55:07.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:55:07 smithi083 ceph-mon[77242]: pgmap v20602: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:07.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:55:07 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:55:07.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:55:07 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:55:07.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:55:07 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:55:07.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:55:07 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:55:07.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:55:07 smithi153 ceph-mon[70044]: pgmap v20602: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:07.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:55:07 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:55:07.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:55:07 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:55:07.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:55:07 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:55:07.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:55:07 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:55:09.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:55:09 smithi083 ceph-mon[79930]: pgmap v20603: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:09.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:55:09 smithi083 ceph-mon[77242]: pgmap v20603: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:09.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:55:09 smithi153 ceph-mon[70044]: pgmap v20603: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:11.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:55:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:55:11] "GET /metrics HTTP/1.1" 200 225247 "" "Prometheus/2.33.4" 2022-09-29T16:55:11.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:55:11 smithi083 ceph-mon[79930]: pgmap v20604: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:11.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:55:11 smithi083 ceph-mon[77242]: pgmap v20604: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:11.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:55:11 smithi153 ceph-mon[70044]: pgmap v20604: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:55:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:55:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:55:13.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:55:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:55:12.749Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:55:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:55:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:55:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:55:13.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:55:13 smithi083 ceph-mon[79930]: pgmap v20605: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:13.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:55:13 smithi083 ceph-mon[77242]: pgmap v20605: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:13.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:55:13 smithi153 ceph-mon[70044]: pgmap v20605: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:15.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:55:14 smithi083 ceph-mon[77242]: pgmap v20606: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:15.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:55:14 smithi083 ceph-mon[79930]: pgmap v20606: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:15.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:55:14 smithi153 ceph-mon[70044]: pgmap v20606: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:16.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:55:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:55:16.097Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:55:17.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:55:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:55:16.845Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:55:17.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:55:17 smithi083 ceph-mon[79930]: pgmap v20607: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:17.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:55:17 smithi083 ceph-mon[77242]: pgmap v20607: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:17.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:55:17 smithi153 ceph-mon[70044]: pgmap v20607: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:19.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:55:19 smithi083 ceph-mon[79930]: pgmap v20608: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:19.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:55:19 smithi083 ceph-mon[77242]: pgmap v20608: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:19.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:55:19 smithi153 ceph-mon[70044]: pgmap v20608: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:21.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:55:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:55:21] "GET /metrics HTTP/1.1" 200 225247 "" "Prometheus/2.33.4" 2022-09-29T16:55:21.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:55:21 smithi083 ceph-mon[79930]: pgmap v20609: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:21.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:55:21 smithi083 ceph-mon[77242]: pgmap v20609: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:21.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:55:21 smithi153 ceph-mon[70044]: pgmap v20609: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:55:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:55:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:55:23.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:55:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:55:22.751Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:55:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:55:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:55:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:55:23.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:55:23 smithi083 ceph-mon[79930]: pgmap v20610: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:23.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:55:23 smithi083 ceph-mon[77242]: pgmap v20610: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:23.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:55:23 smithi153 ceph-mon[70044]: pgmap v20610: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:25.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:55:24 smithi083 ceph-mon[79930]: pgmap v20611: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:25.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:55:24 smithi083 ceph-mon[77242]: pgmap v20611: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:25.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:55:24 smithi153 ceph-mon[70044]: pgmap v20611: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:26.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:55:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:55:26.096Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:55:26.954 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:55:27.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:55:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:55:26.846Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:55:27.242 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:55:27.242 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 3m ago 11h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:55:27.243 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 84s ago 11h 63.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:55:27.243 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 3m ago 11h 26.1M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:55:27.243 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 84s ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:55:27.243 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (11h) 84s ago 11h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:55:27.244 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11h) 3m ago 11h 1092M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:55:27.244 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11h) 3m ago 11h 2471M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:55:27.244 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (11h) 84s ago 11h 1111M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:55:27.244 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (11h) 3m ago 11h 1491M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:55:27.245 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 3m ago 11h 24.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:55:27.245 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 84s ago 11h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:55:27.245 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11h) 3m ago 11h 1803M 2035M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:55:27.246 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11h) 3m ago 11h 1788M 2035M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:55:27.246 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11h) 3m ago 11h 1388M 2035M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:55:27.246 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (11h) 3m ago 11h 1754M 2035M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:55:27.247 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (11h) 84s ago 11h 1536M 2482M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:55:27.247 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (11h) 84s ago 11h 1932M 2482M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:55:27.247 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (11h) 84s ago 11h 1726M 2482M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:55:27.248 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (11h) 84s ago 11h 2056M 2482M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:55:27.248 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (11h) 84s ago 11h 199M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:55:27.598 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:55:27.599 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:55:27.600 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:55:27.600 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:55:27.600 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:55:27.601 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:55:27.601 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:55:27.602 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:55:27.602 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:55:27.602 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:55:27.603 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:55:27.603 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:55:27.603 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:55:27.603 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:55:27.604 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:55:27.604 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:55:27.604 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:55:27.604 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:55:27.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:55:27 smithi083 ceph-mon[79930]: pgmap v20612: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:27.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:55:27 smithi083 ceph-mon[77242]: pgmap v20612: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:27.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:55:27 smithi153 ceph-mon[70044]: pgmap v20612: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:28.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:55:28 smithi083 ceph-mon[79930]: from='client.58098 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:55:28.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:55:28 smithi083 ceph-mon[79930]: from='client.68035 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:55:28.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:55:28 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2451057713' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:55:28.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:55:28 smithi083 ceph-mon[77242]: from='client.58098 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:55:28.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:55:28 smithi083 ceph-mon[77242]: from='client.68035 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:55:28.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:55:28 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2451057713' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:55:28.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:55:28 smithi153 ceph-mon[70044]: from='client.58098 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:55:28.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:55:28 smithi153 ceph-mon[70044]: from='client.68035 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:55:28.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:55:28 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2451057713' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:55:29.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:55:29 smithi083 ceph-mon[79930]: pgmap v20613: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:29.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:55:29 smithi083 ceph-mon[77242]: pgmap v20613: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:29.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:55:29 smithi153 ceph-mon[70044]: pgmap v20613: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:31.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:55:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:55:31] "GET /metrics HTTP/1.1" 200 225246 "" "Prometheus/2.33.4" 2022-09-29T16:55:31.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:55:31 smithi083 ceph-mon[79930]: pgmap v20614: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:31.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:55:31 smithi083 ceph-mon[77242]: pgmap v20614: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:31.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:55:31 smithi153 ceph-mon[70044]: pgmap v20614: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:55:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:55:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:55:33.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:55:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:55:32.750Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:55:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:55:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:55:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:55:33.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:55:33 smithi083 ceph-mon[79930]: pgmap v20615: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:33.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:55:33 smithi083 ceph-mon[77242]: pgmap v20615: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:33.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:55:33 smithi153 ceph-mon[70044]: pgmap v20615: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:35.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:55:34 smithi083 ceph-mon[77242]: pgmap v20616: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:35.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:55:34 smithi083 ceph-mon[79930]: pgmap v20616: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:35.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:55:34 smithi153 ceph-mon[70044]: pgmap v20616: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:36.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:55:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:55:36.097Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:55:37.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:55:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:55:36.846Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:55:37.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:55:37 smithi083 ceph-mon[77242]: pgmap v20617: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:37.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:55:37 smithi083 ceph-mon[79930]: pgmap v20617: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:37.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:55:37 smithi153 ceph-mon[70044]: pgmap v20617: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:39.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:55:39 smithi083 ceph-mon[79930]: pgmap v20618: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:39.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:55:39 smithi083 ceph-mon[77242]: pgmap v20618: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:39.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:55:39 smithi153 ceph-mon[70044]: pgmap v20618: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:41.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:55:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:55:41] "GET /metrics HTTP/1.1" 200 225246 "" "Prometheus/2.33.4" 2022-09-29T16:55:41.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:55:41 smithi083 ceph-mon[77242]: pgmap v20619: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:41.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:55:41 smithi083 ceph-mon[79930]: pgmap v20619: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:41.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:55:41 smithi153 ceph-mon[70044]: pgmap v20619: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:42.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:55:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:55:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:55:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:55:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:55:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:55:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:55:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:55:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:55:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:55:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:55:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:55:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:55:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:55:42.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:55:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:55:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:55:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:55:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:55:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:55:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:55:42.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:55:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:55:42.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:55:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:55:42.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:55:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:55:43.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:55:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:55:42.751Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:55:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:55:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:55:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:55:43.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:55:43 smithi083 ceph-mon[79930]: pgmap v20620: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:43.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:55:43 smithi083 ceph-mon[77242]: pgmap v20620: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:43.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:55:43 smithi153 ceph-mon[70044]: pgmap v20620: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:45.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:55:44 smithi083 ceph-mon[79930]: pgmap v20621: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:45.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:55:44 smithi083 ceph-mon[77242]: pgmap v20621: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:45.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:55:44 smithi153 ceph-mon[70044]: pgmap v20621: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:46.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:55:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:55:46.098Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:55:47.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:55:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:55:46.846Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:55:47.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:55:47 smithi083 ceph-mon[77242]: pgmap v20622: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:47.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:55:47 smithi083 ceph-mon[79930]: pgmap v20622: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:47.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:55:47 smithi153 ceph-mon[70044]: pgmap v20622: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:49.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:55:49 smithi083 ceph-mon[79930]: pgmap v20623: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:49.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:55:49 smithi083 ceph-mon[77242]: pgmap v20623: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:49.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:55:49 smithi153 ceph-mon[70044]: pgmap v20623: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:51.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:55:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:55:51] "GET /metrics HTTP/1.1" 200 225246 "" "Prometheus/2.33.4" 2022-09-29T16:55:51.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:55:51 smithi083 ceph-mon[79930]: pgmap v20624: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:51.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:55:51 smithi083 ceph-mon[77242]: pgmap v20624: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:51.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:55:51 smithi153 ceph-mon[70044]: pgmap v20624: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:55:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:55:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:55:53.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:55:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:55:52.752Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:55:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:55:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:55:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:55:53.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:55:53 smithi083 ceph-mon[79930]: pgmap v20625: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:53.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:55:53 smithi083 ceph-mon[77242]: pgmap v20625: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:53.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:55:53 smithi153 ceph-mon[70044]: pgmap v20625: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:55.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:55:54 smithi083 ceph-mon[79930]: pgmap v20626: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:55.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:55:54 smithi083 ceph-mon[77242]: pgmap v20626: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:55.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:55:54 smithi153 ceph-mon[70044]: pgmap v20626: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:56.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:55:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:55:56.097Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:55:57.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:55:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:55:56.847Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:55:57.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:55:57 smithi083 ceph-mon[79930]: pgmap v20627: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:57.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:55:57 smithi083 ceph-mon[77242]: pgmap v20627: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:57.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:55:57 smithi153 ceph-mon[70044]: pgmap v20627: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:57.928 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:55:58.214 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:55:58.214 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 4m ago 11h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:55:58.215 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 115s ago 11h 63.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:55:58.215 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 4m ago 11h 26.1M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:55:58.215 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 115s ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:55:58.215 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (11h) 115s ago 11h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:55:58.216 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11h) 4m ago 11h 1092M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:55:58.216 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11h) 4m ago 11h 2471M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:55:58.216 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (11h) 115s ago 11h 1111M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:55:58.216 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (11h) 4m ago 11h 1491M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:55:58.217 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 4m ago 11h 24.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:55:58.218 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 115s ago 11h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:55:58.218 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11h) 4m ago 11h 1803M 2035M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:55:58.219 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11h) 4m ago 11h 1788M 2035M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:55:58.219 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11h) 4m ago 11h 1388M 2035M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:55:58.219 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (11h) 4m ago 11h 1754M 2035M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:55:58.220 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (11h) 115s ago 11h 1536M 2482M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:55:58.220 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (11h) 115s ago 11h 1932M 2482M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:55:58.220 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (11h) 115s ago 11h 1726M 2482M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:55:58.220 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (11h) 115s ago 11h 2056M 2482M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:55:58.221 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (11h) 115s ago 11h 199M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:55:58.571 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:55:58.571 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:55:58.571 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:55:58.572 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:55:58.572 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:55:58.572 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:55:58.572 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:55:58.573 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:55:58.573 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:55:58.573 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:55:58.573 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:55:58.573 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:55:58.574 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:55:58.574 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:55:58.574 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:55:58.574 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:55:58.575 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:55:58.575 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:55:59.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:55:59 smithi083 ceph-mon[79930]: from='client.68047 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:55:59.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:55:59 smithi083 ceph-mon[79930]: from='client.68053 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:55:59.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:55:59 smithi083 ceph-mon[79930]: pgmap v20628: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:59.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:55:59 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2636966961' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:55:59.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:55:59 smithi083 ceph-mon[77242]: from='client.68047 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:55:59.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:55:59 smithi083 ceph-mon[77242]: from='client.68053 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:55:59.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:55:59 smithi083 ceph-mon[77242]: pgmap v20628: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:59.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:55:59 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2636966961' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:55:59.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:55:59 smithi153 ceph-mon[70044]: from='client.68047 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:55:59.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:55:59 smithi153 ceph-mon[70044]: from='client.68053 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:55:59.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:55:59 smithi153 ceph-mon[70044]: pgmap v20628: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:55:59.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:55:59 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2636966961' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:56:01.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:56:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:56:01] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T16:56:01.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:56:01 smithi083 ceph-mon[79930]: pgmap v20629: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:01.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:56:01 smithi083 ceph-mon[77242]: pgmap v20629: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:01.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:56:01 smithi153 ceph-mon[70044]: pgmap v20629: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:56:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:56:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:56:03.216 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:56:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:56:02.753Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:56:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:56:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:56:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:56:03.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:56:03 smithi083 ceph-mon[79930]: pgmap v20630: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:03.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:56:03 smithi083 ceph-mon[77242]: pgmap v20630: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:03.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:56:03 smithi153 ceph-mon[70044]: pgmap v20630: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:05.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:56:04 smithi083 ceph-mon[79930]: pgmap v20631: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:05.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:56:04 smithi083 ceph-mon[77242]: pgmap v20631: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:05.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:56:04 smithi153 ceph-mon[70044]: pgmap v20631: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:06.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:56:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:56:06.098Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:56:07.216 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:56:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:56:06.848Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:56:07.714 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:56:07 smithi083 ceph-mon[79930]: pgmap v20632: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:07.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:56:07 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:56:07.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:56:07 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:56:07.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:56:07 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:56:07.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:56:07 smithi083 ceph-mon[77242]: pgmap v20632: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:07.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:56:07 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:56:07.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:56:07 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:56:07.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:56:07 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:56:07.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:56:07 smithi153 ceph-mon[70044]: pgmap v20632: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:07.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:56:07 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:56:07.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:56:07 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:56:07.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:56:07 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:56:09.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:56:08 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:56:09.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:56:08 smithi153 ceph-mon[70044]: pgmap v20633: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:09.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:56:08 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:56:09.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:56:08 smithi083 ceph-mon[79930]: pgmap v20633: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:09.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:56:08 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:56:09.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:56:08 smithi083 ceph-mon[77242]: pgmap v20633: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:11.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:56:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:56:11] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T16:56:11.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:56:11 smithi083 ceph-mon[79930]: pgmap v20634: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:11.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:56:11 smithi083 ceph-mon[77242]: pgmap v20634: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:11.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:56:11 smithi153 ceph-mon[70044]: pgmap v20634: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:56:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:56:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:56:13.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:56:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:56:12.754Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:56:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:56:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:56:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:56:13.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:56:13 smithi083 ceph-mon[79930]: pgmap v20635: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:13.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:56:13 smithi083 ceph-mon[77242]: pgmap v20635: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:13.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:56:13 smithi153 ceph-mon[70044]: pgmap v20635: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:15.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:56:14 smithi083 ceph-mon[79930]: pgmap v20636: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:15.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:56:14 smithi083 ceph-mon[77242]: pgmap v20636: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:15.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:56:14 smithi153 ceph-mon[70044]: pgmap v20636: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:16.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:56:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:56:16.098Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:56:17.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:56:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:56:16.849Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:56:17.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:56:17 smithi083 ceph-mon[79930]: pgmap v20637: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:17.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:56:17 smithi083 ceph-mon[77242]: pgmap v20637: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:17.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:56:17 smithi153 ceph-mon[70044]: pgmap v20637: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:19.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:56:19 smithi083 ceph-mon[79930]: pgmap v20638: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:19.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:56:19 smithi083 ceph-mon[77242]: pgmap v20638: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:19.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:56:19 smithi153 ceph-mon[70044]: pgmap v20638: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:21.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:56:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:56:21] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T16:56:21.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:56:21 smithi083 ceph-mon[79930]: pgmap v20639: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:21.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:56:21 smithi083 ceph-mon[77242]: pgmap v20639: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:21.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:56:21 smithi153 ceph-mon[70044]: pgmap v20639: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:56:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:56:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:56:23.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:56:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:56:22.755Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:56:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:56:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:56:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:56:23.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:56:23 smithi083 ceph-mon[79930]: pgmap v20640: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:23.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:56:23 smithi083 ceph-mon[77242]: pgmap v20640: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:23.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:56:23 smithi153 ceph-mon[70044]: pgmap v20640: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:25.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:56:24 smithi083 ceph-mon[79930]: pgmap v20641: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:25.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:56:24 smithi083 ceph-mon[77242]: pgmap v20641: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:25.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:56:24 smithi153 ceph-mon[70044]: pgmap v20641: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:26.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:56:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:56:26.099Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:56:27.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:56:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:56:26.849Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:56:27.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:56:27 smithi083 ceph-mon[77242]: pgmap v20642: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:27.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:56:27 smithi083 ceph-mon[79930]: pgmap v20642: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:27.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:56:27 smithi153 ceph-mon[70044]: pgmap v20642: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:28.895 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:56:29.184 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:56:29.184 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 4m ago 11h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:56:29.185 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 2m ago 11h 63.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:56:29.185 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 4m ago 11h 26.1M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:56:29.186 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 2m ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:56:29.186 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (11h) 2m ago 11h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:56:29.187 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11h) 4m ago 11h 1092M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:56:29.187 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11h) 4m ago 11h 2471M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:56:29.188 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (11h) 2m ago 11h 1111M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:56:29.188 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (11h) 4m ago 11h 1491M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:56:29.189 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 4m ago 11h 24.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:56:29.189 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 2m ago 11h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:56:29.190 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11h) 4m ago 11h 1803M 2035M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:56:29.190 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11h) 4m ago 11h 1788M 2035M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:56:29.191 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11h) 4m ago 11h 1388M 2035M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:56:29.191 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (11h) 4m ago 11h 1754M 2035M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:56:29.192 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (11h) 2m ago 11h 1536M 2482M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:56:29.192 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (11h) 2m ago 11h 1932M 2482M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:56:29.193 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (11h) 2m ago 11h 1726M 2482M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:56:29.194 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (11h) 2m ago 11h 2056M 2482M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:56:29.194 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (11h) 2m ago 11h 199M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:56:29.539 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:56:29.539 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:56:29.540 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:56:29.540 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:56:29.540 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:56:29.540 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:56:29.541 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:56:29.541 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:56:29.541 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:56:29.541 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:56:29.542 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:56:29.542 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:56:29.542 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:56:29.546 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:56:29.547 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:56:29.547 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:56:29.547 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:56:29.547 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:56:29.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:56:29 smithi083 ceph-mon[77242]: pgmap v20643: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:29.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:56:29 smithi083 ceph-mon[79930]: pgmap v20643: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:29.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:56:29 smithi153 ceph-mon[70044]: pgmap v20643: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:30.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:56:30 smithi083 ceph-mon[79930]: from='client.58134 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:56:30.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:56:30 smithi083 ceph-mon[79930]: from='client.68071 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:56:30.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:56:30 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/18199835' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:56:30.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:56:30 smithi083 ceph-mon[77242]: from='client.58134 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:56:30.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:56:30 smithi083 ceph-mon[77242]: from='client.68071 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:56:30.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:56:30 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/18199835' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:56:30.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:56:30 smithi153 ceph-mon[70044]: from='client.58134 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:56:30.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:56:30 smithi153 ceph-mon[70044]: from='client.68071 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:56:30.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:56:30 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/18199835' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:56:31.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:56:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:56:31] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T16:56:31.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:56:31 smithi083 ceph-mon[79930]: pgmap v20644: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:31.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:56:31 smithi083 ceph-mon[77242]: pgmap v20644: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:31.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:56:31 smithi153 ceph-mon[70044]: pgmap v20644: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:32.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:56:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:56:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:56:33.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:56:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:56:32.756Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:56:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:56:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:56:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:56:33.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:56:33 smithi083 ceph-mon[79930]: pgmap v20645: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:33.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:56:33 smithi083 ceph-mon[77242]: pgmap v20645: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:33.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:56:33 smithi153 ceph-mon[70044]: pgmap v20645: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:35.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:56:34 smithi083 ceph-mon[77242]: pgmap v20646: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:35.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:56:34 smithi083 ceph-mon[79930]: pgmap v20646: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:35.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:56:34 smithi153 ceph-mon[70044]: pgmap v20646: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:36.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:56:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:56:36.099Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:56:37.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:56:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:56:36.851Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:56:37.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:56:37 smithi083 ceph-mon[79930]: pgmap v20647: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:37.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:56:37 smithi083 ceph-mon[77242]: pgmap v20647: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:37.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:56:37 smithi153 ceph-mon[70044]: pgmap v20647: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:39.721 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:56:39 smithi083 ceph-mon[79930]: pgmap v20648: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:39.722 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:56:39 smithi083 ceph-mon[77242]: pgmap v20648: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:39.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:56:39 smithi153 ceph-mon[70044]: pgmap v20648: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:41.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:56:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:56:41] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T16:56:41.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:56:41 smithi083 ceph-mon[79930]: pgmap v20649: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:41.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:56:41 smithi083 ceph-mon[77242]: pgmap v20649: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:41.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:56:41 smithi153 ceph-mon[70044]: pgmap v20649: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:42.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:56:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:56:42.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:56:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:56:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:56:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:56:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:56:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:56:42.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:56:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:56:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:56:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:56:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:56:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:56:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:56:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:56:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:56:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:56:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:56:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:56:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:56:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:56:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:56:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:56:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:56:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:56:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:56:43.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:56:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:56:42.757Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:56:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:56:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:56:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:56:43.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:56:43 smithi083 ceph-mon[79930]: pgmap v20650: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:43.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:56:43 smithi083 ceph-mon[77242]: pgmap v20650: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:43.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:56:43 smithi153 ceph-mon[70044]: pgmap v20650: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:45.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:56:44 smithi083 ceph-mon[79930]: pgmap v20651: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:45.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:56:44 smithi083 ceph-mon[77242]: pgmap v20651: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:45.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:56:44 smithi153 ceph-mon[70044]: pgmap v20651: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:46.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:56:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:56:46.100Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:56:47.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:56:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:56:46.851Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:56:47.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:56:47 smithi083 ceph-mon[79930]: pgmap v20652: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:47.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:56:47 smithi083 ceph-mon[77242]: pgmap v20652: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:47.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:56:47 smithi153 ceph-mon[70044]: pgmap v20652: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:49.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:56:49 smithi083 ceph-mon[79930]: pgmap v20653: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:49.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:56:49 smithi083 ceph-mon[77242]: pgmap v20653: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:49.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:56:49 smithi153 ceph-mon[70044]: pgmap v20653: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:51.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:56:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:56:51] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T16:56:51.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:56:51 smithi083 ceph-mon[79930]: pgmap v20654: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:51.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:56:51 smithi083 ceph-mon[77242]: pgmap v20654: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:51.813 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:56:51 smithi153 ceph-mon[70044]: pgmap v20654: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:56:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:56:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:56:53.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:56:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:56:52.757Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:56:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:56:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:56:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:56:53.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:56:53 smithi083 ceph-mon[79930]: pgmap v20655: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:53.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:56:53 smithi083 ceph-mon[77242]: pgmap v20655: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:53.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:56:53 smithi153 ceph-mon[70044]: pgmap v20655: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:55.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:56:54 smithi083 ceph-mon[79930]: pgmap v20656: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:55.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:56:54 smithi083 ceph-mon[77242]: pgmap v20656: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:55.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:56:54 smithi153 ceph-mon[70044]: pgmap v20656: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:56.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:56:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:56:56.100Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:56:57.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:56:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:56:56.852Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:56:57.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:56:57 smithi083 ceph-mon[77242]: pgmap v20657: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:57.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:56:57 smithi083 ceph-mon[79930]: pgmap v20657: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:57.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:56:57 smithi153 ceph-mon[70044]: pgmap v20657: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:59.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:56:59 smithi083 ceph-mon[79930]: pgmap v20658: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:59.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:56:59 smithi083 ceph-mon[77242]: pgmap v20658: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:59.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:56:59 smithi153 ceph-mon[70044]: pgmap v20658: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:56:59.865 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:57:00.158 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:57:00.159 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 5m ago 11h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:57:00.159 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 2m ago 11h 63.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:57:00.160 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 5m ago 11h 26.1M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:57:00.161 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 2m ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:57:00.161 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (11h) 2m ago 11h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:57:00.161 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11h) 5m ago 11h 1092M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:57:00.162 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11h) 5m ago 11h 2471M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:57:00.162 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (11h) 2m ago 11h 1111M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:57:00.163 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (11h) 5m ago 11h 1491M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:57:00.164 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 5m ago 11h 24.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:57:00.166 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 2m ago 11h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:57:00.167 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11h) 5m ago 11h 1803M 2035M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:57:00.167 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11h) 5m ago 11h 1788M 2035M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:57:00.168 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11h) 5m ago 11h 1388M 2035M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:57:00.168 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (11h) 5m ago 11h 1754M 2035M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:57:00.168 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (11h) 2m ago 11h 1536M 2482M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:57:00.169 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (11h) 2m ago 11h 1932M 2482M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:57:00.169 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (11h) 2m ago 11h 1726M 2482M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:57:00.169 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (11h) 2m ago 11h 2056M 2482M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:57:00.169 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (11h) 2m ago 11h 199M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:57:00.512 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:57:00.512 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:57:00.513 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:57:00.513 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:57:00.513 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:57:00.513 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:57:00.514 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:57:00.514 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:57:00.514 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:57:00.514 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:57:00.515 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:57:00.515 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:57:00.515 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:57:00.515 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:57:00.515 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:57:00.516 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:57:00.516 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:57:00.516 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:57:01.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:57:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:57:01] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T16:57:01.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:57:01 smithi083 ceph-mon[77242]: from='client.58152 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:57:01.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:57:01 smithi083 ceph-mon[77242]: from='client.68089 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:57:01.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:57:01 smithi083 ceph-mon[77242]: pgmap v20659: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:57:01.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:57:01 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3921846639' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:57:01.720 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:57:01 smithi083 ceph-mon[79930]: from='client.58152 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:57:01.721 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:57:01 smithi083 ceph-mon[79930]: from='client.68089 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:57:01.722 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:57:01 smithi083 ceph-mon[79930]: pgmap v20659: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:57:01.723 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:57:01 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3921846639' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:57:01.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:57:01 smithi153 ceph-mon[70044]: from='client.58152 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:57:01.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:57:01 smithi153 ceph-mon[70044]: from='client.68089 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:57:01.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:57:01 smithi153 ceph-mon[70044]: pgmap v20659: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:57:01.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:57:01 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3921846639' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:57:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:57:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:57:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:57:03.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:57:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:57:02.758Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:57:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:57:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:57:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:57:03.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:57:03 smithi083 ceph-mon[79930]: pgmap v20660: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:57:03.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:57:03 smithi083 ceph-mon[77242]: pgmap v20660: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:57:03.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:57:03 smithi153 ceph-mon[70044]: pgmap v20660: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:57:05.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:57:04 smithi083 ceph-mon[79930]: pgmap v20661: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:57:05.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:57:04 smithi083 ceph-mon[77242]: pgmap v20661: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:57:05.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:57:04 smithi153 ceph-mon[70044]: pgmap v20661: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:57:06.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:57:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:57:06.101Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:57:07.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:57:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:57:06.852Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:57:07.723 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:57:07 smithi083 ceph-mon[79930]: pgmap v20662: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:57:07.724 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:57:07 smithi083 ceph-mon[77242]: pgmap v20662: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:57:07.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:57:07 smithi153 ceph-mon[70044]: pgmap v20662: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:57:08.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:57:08 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:57:08.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:57:08 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:57:08.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:57:08 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:57:08.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:57:08 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:57:08.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:57:08 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:57:08.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:57:08 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:57:08.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:57:08 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:57:08.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:57:08 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:57:08.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:57:08 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:57:09.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:57:09 smithi083 ceph-mon[79930]: pgmap v20663: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:57:09.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:57:09 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:57:09.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:57:09 smithi083 ceph-mon[77242]: pgmap v20663: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:57:09.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:57:09 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:57:09.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:57:09 smithi153 ceph-mon[70044]: pgmap v20663: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:57:09.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:57:09 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:57:11.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:57:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:57:11] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T16:57:11.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:57:11 smithi083 ceph-mon[79930]: pgmap v20664: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:57:11.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:57:11 smithi083 ceph-mon[77242]: pgmap v20664: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:57:11.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:57:11 smithi153 ceph-mon[70044]: pgmap v20664: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:57:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:57:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:57:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:57:13.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:57:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:57:12.759Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:57:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:57:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:57:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:57:13.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:57:13 smithi083 ceph-mon[79930]: pgmap v20665: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:57:13.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:57:13 smithi083 ceph-mon[77242]: pgmap v20665: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:57:13.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:57:13 smithi153 ceph-mon[70044]: pgmap v20665: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:57:15.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:57:14 smithi083 ceph-mon[79930]: pgmap v20666: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:57:15.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:57:14 smithi083 ceph-mon[77242]: pgmap v20666: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:57:15.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:57:14 smithi153 ceph-mon[70044]: pgmap v20666: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:57:16.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:57:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:57:16.102Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:57:17.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:57:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:57:16.853Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:57:17.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:57:17 smithi083 ceph-mon[79930]: pgmap v20667: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:57:17.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:57:17 smithi083 ceph-mon[77242]: pgmap v20667: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:57:17.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:57:17 smithi153 ceph-mon[70044]: pgmap v20667: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:57:19.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:57:19 smithi083 ceph-mon[79930]: pgmap v20668: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:57:19.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:57:19 smithi083 ceph-mon[77242]: pgmap v20668: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:57:19.813 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:57:19 smithi153 ceph-mon[70044]: pgmap v20668: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:57:21.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:57:21 smithi083 ceph-mon[79930]: pgmap v20669: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:57:21.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:57:21 smithi083 ceph-mon[77242]: pgmap v20669: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:57:21.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:57:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:57:21] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T16:57:21.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:57:21 smithi153 ceph-mon[70044]: pgmap v20669: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:57:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:57:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:57:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:57:23.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:57:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:57:22.760Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:57:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:57:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:57:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:57:23.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:57:23 smithi083 ceph-mon[79930]: pgmap v20670: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:57:23.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:57:23 smithi083 ceph-mon[77242]: pgmap v20670: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:57:23.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:57:23 smithi153 ceph-mon[70044]: pgmap v20670: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:57:25.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:57:24 smithi083 ceph-mon[79930]: pgmap v20671: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:57:25.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:57:24 smithi083 ceph-mon[77242]: pgmap v20671: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:57:25.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:57:24 smithi153 ceph-mon[70044]: pgmap v20671: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:57:26.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:57:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:57:26.103Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:57:27.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:57:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:57:26.854Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:57:27.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:57:27 smithi083 ceph-mon[79930]: pgmap v20672: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:57:27.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:57:27 smithi083 ceph-mon[77242]: pgmap v20672: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:57:27.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:57:27 smithi153 ceph-mon[70044]: pgmap v20672: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:57:29.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:57:29 smithi083 ceph-mon[79930]: pgmap v20673: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:57:29.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:57:29 smithi083 ceph-mon[77242]: pgmap v20673: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:57:29.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:57:29 smithi153 ceph-mon[70044]: pgmap v20673: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:57:30.838 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:57:31.127 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:57:31.127 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 5m ago 11h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:57:31.128 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 3m ago 11h 63.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:57:31.128 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 5m ago 11h 26.1M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:57:31.128 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 3m ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:57:31.129 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (11h) 3m ago 11h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:57:31.129 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11h) 5m ago 11h 1092M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:57:31.129 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11h) 5m ago 11h 2471M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:57:31.129 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (11h) 3m ago 11h 1111M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:57:31.131 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (11h) 5m ago 11h 1491M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:57:31.132 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 5m ago 11h 24.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:57:31.132 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 3m ago 11h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:57:31.132 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11h) 5m ago 11h 1803M 2035M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:57:31.132 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11h) 5m ago 11h 1788M 2035M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:57:31.133 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11h) 5m ago 11h 1388M 2035M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:57:31.133 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (11h) 5m ago 11h 1754M 2035M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:57:31.133 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (11h) 3m ago 11h 1536M 2482M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:57:31.133 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (11h) 3m ago 11h 1932M 2482M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:57:31.133 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (11h) 3m ago 11h 1726M 2482M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:57:31.134 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (11h) 3m ago 11h 2056M 2482M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:57:31.259 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (11h) 3m ago 11h 199M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:57:31.486 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:57:31.486 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:57:31.487 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:57:31.487 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:57:31.488 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:57:31.488 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:57:31.488 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:57:31.489 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:57:31.489 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:57:31.489 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:57:31.489 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:57:31.490 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:57:31.490 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:57:31.491 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:57:31.492 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:57:31.493 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:57:31.493 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:57:31.493 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:57:31.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:57:31 smithi083 ceph-mon[79930]: pgmap v20674: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:57:31.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:57:31 smithi083 ceph-mon[77242]: pgmap v20674: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:57:31.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:57:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:57:31] "GET /metrics HTTP/1.1" 200 225247 "" "Prometheus/2.33.4" 2022-09-29T16:57:31.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:57:31 smithi153 ceph-mon[70044]: pgmap v20674: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:57:32.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:57:32 smithi083 ceph-mon[79930]: from='client.68101 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:57:32.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:57:32 smithi083 ceph-mon[79930]: from='client.58176 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:57:32.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:57:32 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1237866246' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:57:32.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:57:32 smithi083 ceph-mon[77242]: from='client.68101 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:57:32.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:57:32 smithi083 ceph-mon[77242]: from='client.58176 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:57:32.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:57:32 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1237866246' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:57:32.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:57:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:57:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:57:32.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:57:32 smithi153 ceph-mon[70044]: from='client.68101 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:57:32.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:57:32 smithi153 ceph-mon[70044]: from='client.58176 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:57:32.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:57:32 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1237866246' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:57:33.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:57:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:57:32.761Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:57:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:57:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:57:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:57:33.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:57:33 smithi083 ceph-mon[79930]: pgmap v20675: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:57:33.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:57:33 smithi083 ceph-mon[77242]: pgmap v20675: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:57:33.819 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:57:33 smithi153 ceph-mon[70044]: pgmap v20675: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:57:35.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:57:34 smithi083 ceph-mon[79930]: pgmap v20676: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:57:35.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:57:34 smithi083 ceph-mon[77242]: pgmap v20676: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:57:35.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:57:34 smithi153 ceph-mon[70044]: pgmap v20676: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:57:36.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:57:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:57:36.104Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:57:37.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:57:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:57:36.855Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:57:37.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:57:37 smithi083 ceph-mon[79930]: pgmap v20677: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:57:37.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:57:37 smithi083 ceph-mon[77242]: pgmap v20677: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:57:37.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:57:37 smithi153 ceph-mon[70044]: pgmap v20677: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:57:39.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:57:39 smithi083 ceph-mon[79930]: pgmap v20678: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:57:39.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:57:39 smithi083 ceph-mon[77242]: pgmap v20678: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:57:39.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:57:39 smithi153 ceph-mon[70044]: pgmap v20678: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:57:41.719 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:57:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:57:41] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T16:57:41.720 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:57:41 smithi083 ceph-mon[79930]: pgmap v20679: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:57:41.721 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:57:41 smithi083 ceph-mon[77242]: pgmap v20679: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:57:41.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:57:41 smithi153 ceph-mon[70044]: pgmap v20679: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:57:42.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:57:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:57:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:57:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:57:42.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:57:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:57:42.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:57:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:57:42.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:57:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:57:42.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:57:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:57:42.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:57:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:57:42.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:57:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:57:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:57:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:57:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:57:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:57:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:57:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:57:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:57:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:57:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:57:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:57:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:57:43.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:57:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:57:42.761Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:57:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:57:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:57:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:57:43.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:57:43 smithi083 ceph-mon[79930]: pgmap v20680: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:57:43.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:57:43 smithi083 ceph-mon[77242]: pgmap v20680: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:57:43.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:57:43 smithi153 ceph-mon[70044]: pgmap v20680: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:57:45.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:57:44 smithi083 ceph-mon[77242]: pgmap v20681: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:57:45.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:57:44 smithi083 ceph-mon[79930]: pgmap v20681: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:57:45.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:57:44 smithi153 ceph-mon[70044]: pgmap v20681: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:57:46.470 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:57:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:57:46.105Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:57:47.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:57:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:57:46.856Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:57:47.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:57:47 smithi083 ceph-mon[77242]: pgmap v20682: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:57:47.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:57:47 smithi083 ceph-mon[79930]: pgmap v20682: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:57:47.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:57:47 smithi153 ceph-mon[70044]: pgmap v20682: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:57:49.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:57:49 smithi083 ceph-mon[79930]: pgmap v20683: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:57:49.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:57:49 smithi083 ceph-mon[77242]: pgmap v20683: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:57:49.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:57:49 smithi153 ceph-mon[70044]: pgmap v20683: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:57:51.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:57:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:57:51] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T16:57:51.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:57:51 smithi083 ceph-mon[79930]: pgmap v20684: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:57:51.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:57:51 smithi083 ceph-mon[77242]: pgmap v20684: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:57:51.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:57:51 smithi153 ceph-mon[70044]: pgmap v20684: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:57:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:57:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:57:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:57:53.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:57:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:57:52.762Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:57:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:57:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:57:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:57:53.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:57:53 smithi083 ceph-mon[79930]: pgmap v20685: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:57:53.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:57:53 smithi083 ceph-mon[77242]: pgmap v20685: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:57:53.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:57:53 smithi153 ceph-mon[70044]: pgmap v20685: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:57:55.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:57:54 smithi083 ceph-mon[79930]: pgmap v20686: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:57:55.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:57:54 smithi083 ceph-mon[77242]: pgmap v20686: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:57:55.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:57:54 smithi153 ceph-mon[70044]: pgmap v20686: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:57:56.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:57:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:57:56.105Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:57:57.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:57:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:57:56.857Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:57:57.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:57:57 smithi083 ceph-mon[79930]: pgmap v20687: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:57:57.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:57:57 smithi083 ceph-mon[77242]: pgmap v20687: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:57:57.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:57:57 smithi153 ceph-mon[70044]: pgmap v20687: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:57:59.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:57:59 smithi083 ceph-mon[77242]: pgmap v20688: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:57:59.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:57:59 smithi083 ceph-mon[79930]: pgmap v20688: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:57:59.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:57:59 smithi153 ceph-mon[70044]: pgmap v20688: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:01.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:58:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:58:01] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T16:58:01.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:58:01 smithi083 ceph-mon[79930]: pgmap v20689: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:01.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:58:01 smithi083 ceph-mon[77242]: pgmap v20689: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:01.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:58:01 smithi153 ceph-mon[70044]: pgmap v20689: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:01.812 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:58:02.100 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:58:02.101 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 6m ago 11h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:58:02.101 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 3m ago 11h 63.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:58:02.101 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 6m ago 11h 26.1M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:58:02.101 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 3m ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:58:02.101 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (11h) 3m ago 11h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:58:02.102 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11h) 6m ago 11h 1092M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:58:02.102 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11h) 6m ago 11h 2471M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:58:02.102 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (11h) 3m ago 11h 1111M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:58:02.102 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (11h) 6m ago 11h 1491M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:58:02.103 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 6m ago 11h 24.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:58:02.103 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 3m ago 11h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:58:02.103 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11h) 6m ago 11h 1803M 2035M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:58:02.103 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11h) 6m ago 11h 1788M 2035M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:58:02.104 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11h) 6m ago 11h 1388M 2035M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:58:02.104 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (11h) 6m ago 11h 1754M 2035M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:58:02.104 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (11h) 3m ago 11h 1536M 2482M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:58:02.104 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (11h) 3m ago 11h 1932M 2482M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:58:02.104 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (11h) 3m ago 11h 1726M 2482M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:58:02.105 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (11h) 3m ago 11h 2056M 2482M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:58:02.105 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (11h) 3m ago 11h 199M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:58:02.460 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:58:02.461 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:58:02.461 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:58:02.462 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:58:02.463 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:58:02.463 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:58:02.464 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:58:02.464 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:58:02.465 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:58:02.465 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:58:02.465 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:58:02.465 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:58:02.465 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:58:02.466 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:58:02.466 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:58:02.466 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:58:02.467 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:58:02.467 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:58:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:58:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:58:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:58:03.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:58:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:58:02.761Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:58:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:58:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:58:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:58:03.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:58:03 smithi083 ceph-mon[79930]: from='client.58188 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:58:03.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:58:03 smithi083 ceph-mon[79930]: from='client.68125 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:58:03.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:58:03 smithi083 ceph-mon[79930]: pgmap v20690: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:03.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:58:03 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3693376305' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:58:03.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:58:03 smithi083 ceph-mon[77242]: from='client.58188 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:58:03.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:58:03 smithi083 ceph-mon[77242]: from='client.68125 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:58:03.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:58:03 smithi083 ceph-mon[77242]: pgmap v20690: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:03.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:58:03 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3693376305' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:58:03.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:58:03 smithi153 ceph-mon[70044]: from='client.58188 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:58:03.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:58:03 smithi153 ceph-mon[70044]: from='client.68125 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:58:03.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:58:03 smithi153 ceph-mon[70044]: pgmap v20690: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:03.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:58:03 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3693376305' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:58:05.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:58:04 smithi083 ceph-mon[79930]: pgmap v20691: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:05.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:58:04 smithi083 ceph-mon[77242]: pgmap v20691: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:05.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:58:04 smithi153 ceph-mon[70044]: pgmap v20691: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:06.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:58:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:58:06.105Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:58:07.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:58:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:58:06.858Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:58:07.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:58:07 smithi083 ceph-mon[79930]: pgmap v20692: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:07.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:58:07 smithi083 ceph-mon[77242]: pgmap v20692: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:07.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:58:07 smithi153 ceph-mon[70044]: pgmap v20692: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:09.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:58:09 smithi083 ceph-mon[79930]: pgmap v20693: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:09.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:58:09 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:58:09.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:58:09 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:58:09.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:58:09 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:58:09.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:58:09 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:58:09.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:58:09 smithi083 ceph-mon[77242]: pgmap v20693: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:09.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:58:09 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:58:09.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:58:09 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:58:09.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:58:09 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:58:09.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:58:09 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:58:09.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:58:09 smithi153 ceph-mon[70044]: pgmap v20693: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:09.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:58:09 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:58:09.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:58:09 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:58:09.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:58:09 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:58:09.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:58:09 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:58:11.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:58:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:58:11] "GET /metrics HTTP/1.1" 200 225246 "" "Prometheus/2.33.4" 2022-09-29T16:58:11.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:58:11 smithi083 ceph-mon[79930]: pgmap v20694: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:11.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:58:11 smithi083 ceph-mon[77242]: pgmap v20694: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:11.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:58:11 smithi153 ceph-mon[70044]: pgmap v20694: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:58:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:58:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:58:13.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:58:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:58:12.763Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:58:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:58:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:58:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:58:13.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:58:13 smithi083 ceph-mon[79930]: pgmap v20695: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:13.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:58:13 smithi083 ceph-mon[77242]: pgmap v20695: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:13.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:58:13 smithi153 ceph-mon[70044]: pgmap v20695: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:15.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:58:14 smithi083 ceph-mon[79930]: pgmap v20696: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:15.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:58:14 smithi083 ceph-mon[77242]: pgmap v20696: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:15.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:58:14 smithi153 ceph-mon[70044]: pgmap v20696: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:16.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:58:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:58:16.107Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:58:17.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:58:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:58:16.859Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:58:17.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:58:17 smithi083 ceph-mon[79930]: pgmap v20697: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:17.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:58:17 smithi083 ceph-mon[77242]: pgmap v20697: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:17.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:58:17 smithi153 ceph-mon[70044]: pgmap v20697: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:19.715 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:58:19 smithi083 ceph-mon[79930]: pgmap v20698: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:19.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:58:19 smithi083 ceph-mon[77242]: pgmap v20698: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:19.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:58:19 smithi153 ceph-mon[70044]: pgmap v20698: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:21.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:58:21 smithi083 ceph-mon[79930]: pgmap v20699: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:21.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:58:21 smithi083 ceph-mon[77242]: pgmap v20699: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:21.715 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:58:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:58:21] "GET /metrics HTTP/1.1" 200 225246 "" "Prometheus/2.33.4" 2022-09-29T16:58:21.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:58:21 smithi153 ceph-mon[70044]: pgmap v20699: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:58:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:58:22.317Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:58:23.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:58:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:58:22.764Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:58:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:58:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:58:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:58:23.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:58:23 smithi083 ceph-mon[79930]: pgmap v20700: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:23.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:58:23 smithi083 ceph-mon[77242]: pgmap v20700: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:23.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:58:23 smithi153 ceph-mon[70044]: pgmap v20700: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:25.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:58:24 smithi083 ceph-mon[79930]: pgmap v20701: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:25.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:58:24 smithi083 ceph-mon[77242]: pgmap v20701: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:25.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:58:24 smithi153 ceph-mon[70044]: pgmap v20701: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:26.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:58:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:58:26.109Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:58:27.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:58:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:58:26.860Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:58:27.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:58:27 smithi083 ceph-mon[79930]: pgmap v20702: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:27.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:58:27 smithi083 ceph-mon[77242]: pgmap v20702: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:27.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:58:27 smithi153 ceph-mon[70044]: pgmap v20702: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:29.718 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:58:29 smithi083 ceph-mon[79930]: pgmap v20703: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:29.719 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:58:29 smithi083 ceph-mon[77242]: pgmap v20703: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:29.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:58:29 smithi153 ceph-mon[70044]: pgmap v20703: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:31.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:58:31 smithi083 ceph-mon[79930]: pgmap v20704: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:31.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:58:31 smithi083 ceph-mon[77242]: pgmap v20704: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:31.713 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:58:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:58:31] "GET /metrics HTTP/1.1" 200 225247 "" "Prometheus/2.33.4" 2022-09-29T16:58:31.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:58:31 smithi153 ceph-mon[70044]: pgmap v20704: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:32.785 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:58:32.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:58:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:58:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:58:33.075 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:58:33.075 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 6m ago 11h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:58:33.076 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 4m ago 11h 63.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:58:33.076 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 6m ago 11h 26.1M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:58:33.077 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 4m ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:58:33.077 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (11h) 4m ago 11h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:58:33.078 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11h) 6m ago 11h 1092M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:58:33.078 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11h) 6m ago 11h 2471M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:58:33.079 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (11h) 4m ago 11h 1111M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:58:33.079 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (11h) 6m ago 11h 1491M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:58:33.079 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 6m ago 11h 24.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:58:33.080 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 4m ago 11h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:58:33.080 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11h) 6m ago 11h 1803M 2035M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:58:33.080 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11h) 6m ago 11h 1788M 2035M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:58:33.080 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11h) 6m ago 11h 1388M 2035M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:58:33.081 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (11h) 6m ago 11h 1754M 2035M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:58:33.081 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (11h) 4m ago 11h 1536M 2482M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:58:33.081 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (11h) 4m ago 11h 1932M 2482M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:58:33.081 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (11h) 4m ago 11h 1726M 2482M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:58:33.082 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (11h) 4m ago 11h 2056M 2482M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:58:33.082 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (11h) 4m ago 11h 199M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:58:33.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:58:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:58:32.763Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:58:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:58:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:58:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:58:33.437 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:58:33.437 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:58:33.438 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:58:33.438 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:58:33.439 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:58:33.439 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:58:33.439 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:58:33.440 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:58:33.440 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:58:33.440 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:58:33.441 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:58:33.441 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:58:33.442 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:58:33.442 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:58:33.443 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:58:33.443 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:58:33.443 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:58:33.444 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:58:33.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:58:33 smithi083 ceph-mon[79930]: pgmap v20705: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:33.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:58:33 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2688703662' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:58:33.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:58:33 smithi083 ceph-mon[77242]: pgmap v20705: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:33.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:58:33 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2688703662' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:58:33.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:58:33 smithi153 ceph-mon[70044]: pgmap v20705: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:33.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:58:33 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2688703662' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:58:34.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:58:34 smithi083 ceph-mon[79930]: from='client.68137 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:58:34.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:58:34 smithi083 ceph-mon[79930]: from='client.68143 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:58:34.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:58:34 smithi083 ceph-mon[77242]: from='client.68137 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:58:34.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:58:34 smithi083 ceph-mon[77242]: from='client.68143 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:58:34.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:58:34 smithi153 ceph-mon[70044]: from='client.68137 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:58:34.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:58:34 smithi153 ceph-mon[70044]: from='client.68143 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:58:35.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:58:35 smithi083 ceph-mon[79930]: pgmap v20706: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:35.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:58:35 smithi083 ceph-mon[77242]: pgmap v20706: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:35.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:58:35 smithi153 ceph-mon[70044]: pgmap v20706: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:36.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:58:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:58:36.108Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:58:37.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:58:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:58:36.861Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:58:37.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:58:37 smithi083 ceph-mon[79930]: pgmap v20707: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:37.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:58:37 smithi083 ceph-mon[77242]: pgmap v20707: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:37.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:58:37 smithi153 ceph-mon[70044]: pgmap v20707: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:39.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:58:39 smithi083 ceph-mon[79930]: pgmap v20708: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:39.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:58:39 smithi083 ceph-mon[77242]: pgmap v20708: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:39.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:58:39 smithi153 ceph-mon[70044]: pgmap v20708: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:41.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:58:41 smithi153 ceph-mon[70044]: pgmap v20709: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:58:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:58:41] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T16:58:41.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:58:41 smithi083 ceph-mon[79930]: pgmap v20709: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:41.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:58:41 smithi083 ceph-mon[77242]: pgmap v20709: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:42.766 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:58:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:58:42.767 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:58:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:58:42.767 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:58:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:58:42.768 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:58:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:58:42.768 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:58:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:58:42.769 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:58:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:58:42.769 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:58:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:58:42.770 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:58:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:58:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:58:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:58:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:58:42.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:58:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:58:42.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:58:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:58:42.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:58:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:58:42.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:58:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:58:43.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:58:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:58:42.765Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:58:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:58:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:58:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:58:43.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:58:43 smithi153 ceph-mon[70044]: pgmap v20710: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:43.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:58:43 smithi083 ceph-mon[77242]: pgmap v20710: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:43.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:58:43 smithi083 ceph-mon[79930]: pgmap v20710: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:45.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:58:44 smithi083 ceph-mon[79930]: pgmap v20711: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:45.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:58:44 smithi083 ceph-mon[77242]: pgmap v20711: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:45.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:58:44 smithi153 ceph-mon[70044]: pgmap v20711: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:46.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:58:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:58:46.109Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:58:47.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:58:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:58:46.861Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:58:47.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:58:47 smithi083 ceph-mon[79930]: pgmap v20712: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:47.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:58:47 smithi083 ceph-mon[77242]: pgmap v20712: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:47.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:58:47 smithi153 ceph-mon[70044]: pgmap v20712: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:49.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:58:49 smithi083 ceph-mon[79930]: pgmap v20713: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:49.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:58:49 smithi083 ceph-mon[77242]: pgmap v20713: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:49.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:58:49 smithi153 ceph-mon[70044]: pgmap v20713: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:51.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:58:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:58:51] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T16:58:51.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:58:51 smithi083 ceph-mon[79930]: pgmap v20714: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:51.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:58:51 smithi083 ceph-mon[77242]: pgmap v20714: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:51.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:58:51 smithi153 ceph-mon[70044]: pgmap v20714: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:58:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:58:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:58:53.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:58:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:58:52.766Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:58:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:58:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:58:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:58:53.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:58:53 smithi083 ceph-mon[79930]: pgmap v20715: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:53.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:58:53 smithi083 ceph-mon[77242]: pgmap v20715: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:53.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:58:53 smithi153 ceph-mon[70044]: pgmap v20715: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:55.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:58:54 smithi083 ceph-mon[79930]: pgmap v20716: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:55.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:58:54 smithi083 ceph-mon[77242]: pgmap v20716: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:55.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:58:54 smithi153 ceph-mon[70044]: pgmap v20716: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:56.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:58:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:58:56.110Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:58:57.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:58:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:58:56.862Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:58:57.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:58:57 smithi083 ceph-mon[79930]: pgmap v20717: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:57.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:58:57 smithi083 ceph-mon[77242]: pgmap v20717: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:57.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:58:57 smithi153 ceph-mon[70044]: pgmap v20717: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:59.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:58:59 smithi083 ceph-mon[79930]: pgmap v20718: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:59.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:58:59 smithi083 ceph-mon[77242]: pgmap v20718: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:58:59.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:58:59 smithi153 ceph-mon[70044]: pgmap v20718: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:01.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:59:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:59:01] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T16:59:01.713 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:59:01 smithi083 ceph-mon[79930]: pgmap v20719: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:01.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:59:01 smithi083 ceph-mon[77242]: pgmap v20719: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:01.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:59:01 smithi153 ceph-mon[70044]: pgmap v20719: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:59:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:59:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:59:03.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:59:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:59:02.767Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:59:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:59:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:59:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:59:03.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:59:03 smithi083 ceph-mon[79930]: pgmap v20720: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:03.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:59:03 smithi083 ceph-mon[77242]: pgmap v20720: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:03.765 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:59:03.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:59:03 smithi153 ceph-mon[70044]: pgmap v20720: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:04.060 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:59:04.060 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 7m ago 11h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:59:04.060 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 5m ago 11h 63.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:59:04.061 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 7m ago 11h 26.1M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:59:04.061 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 5m ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:59:04.061 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (11h) 5m ago 11h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:59:04.061 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11h) 7m ago 11h 1092M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:59:04.062 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11h) 7m ago 11h 2471M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:59:04.062 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (11h) 5m ago 11h 1111M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:59:04.063 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (11h) 7m ago 11h 1491M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:59:04.063 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 7m ago 11h 24.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:59:04.063 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 5m ago 11h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:59:04.064 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11h) 7m ago 11h 1803M 2035M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:59:04.064 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11h) 7m ago 11h 1788M 2035M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:59:04.064 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11h) 7m ago 11h 1388M 2035M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:59:04.064 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (11h) 7m ago 11h 1754M 2035M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:59:04.065 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (11h) 5m ago 11h 1536M 2482M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:59:04.065 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (11h) 5m ago 11h 1932M 2482M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:59:04.065 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (11h) 5m ago 11h 1726M 2482M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:59:04.066 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (11h) 5m ago 11h 2056M 2482M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:59:04.066 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (11h) 5m ago 11h 199M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:59:04.416 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:59:04.416 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:59:04.417 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:59:04.417 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:59:04.417 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:59:04.418 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:59:04.418 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:59:04.418 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:59:04.419 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:59:04.419 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:59:04.420 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:59:04.420 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:59:04.421 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:59:04.421 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:59:04.421 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:59:04.421 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:59:04.422 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:59:04.422 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:59:05.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:59:04 smithi083 ceph-mon[79930]: from='client.58224 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:59:05.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:59:04 smithi083 ceph-mon[79930]: from='client.68161 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:59:05.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:59:04 smithi083 ceph-mon[79930]: pgmap v20721: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:05.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:59:04 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2083838535' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:59:05.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:59:04 smithi083 ceph-mon[77242]: from='client.58224 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:59:05.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:59:04 smithi083 ceph-mon[77242]: from='client.68161 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:59:05.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:59:04 smithi083 ceph-mon[77242]: pgmap v20721: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:05.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:59:04 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2083838535' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:59:05.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:59:04 smithi153 ceph-mon[70044]: from='client.58224 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:59:05.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:59:04 smithi153 ceph-mon[70044]: from='client.68161 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:59:05.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:59:04 smithi153 ceph-mon[70044]: pgmap v20721: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:05.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:59:04 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2083838535' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:59:06.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:59:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:59:06.111Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:59:07.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:59:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:59:06.863Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:59:07.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:59:07 smithi083 ceph-mon[79930]: pgmap v20722: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:07.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:59:07 smithi083 ceph-mon[77242]: pgmap v20722: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:07.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:59:07 smithi153 ceph-mon[70044]: pgmap v20722: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:09.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:59:09 smithi083 ceph-mon[79930]: pgmap v20723: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:09.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:59:09 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:59:09.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:59:09 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:59:09.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:59:09 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:59:09.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:59:09 smithi083 ceph-mon[77242]: pgmap v20723: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:09.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:59:09 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:59:09.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:59:09 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:59:09.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:59:09 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:59:09.814 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:59:09 smithi153 ceph-mon[70044]: pgmap v20723: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:09.814 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:59:09 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T16:59:09.815 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:59:09 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T16:59:09.815 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:59:09 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T16:59:11.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:59:10 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:59:11.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:59:10 smithi153 ceph-mon[70044]: pgmap v20724: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:11.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:59:10 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:59:11.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:59:10 smithi083 ceph-mon[79930]: pgmap v20724: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:11.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:59:10 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T16:59:11.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:59:10 smithi083 ceph-mon[77242]: pgmap v20724: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:11.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:59:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:59:11] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T16:59:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:59:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:59:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:59:13.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:59:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:59:12.768Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:59:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:59:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:59:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:59:13.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:59:13 smithi083 ceph-mon[79930]: pgmap v20725: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:13.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:59:13 smithi083 ceph-mon[77242]: pgmap v20725: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:13.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:59:13 smithi153 ceph-mon[70044]: pgmap v20725: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:15.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:59:14 smithi083 ceph-mon[79930]: pgmap v20726: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:15.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:59:14 smithi083 ceph-mon[77242]: pgmap v20726: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:15.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:59:14 smithi153 ceph-mon[70044]: pgmap v20726: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:16.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:59:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:59:16.111Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:59:17.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:59:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:59:16.863Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:59:17.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:59:17 smithi083 ceph-mon[79930]: pgmap v20727: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:17.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:59:17 smithi083 ceph-mon[77242]: pgmap v20727: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:17.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:59:17 smithi153 ceph-mon[70044]: pgmap v20727: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:19.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:59:19 smithi083 ceph-mon[79930]: pgmap v20728: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:19.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:59:19 smithi083 ceph-mon[77242]: pgmap v20728: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:19.814 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:59:19 smithi153 ceph-mon[70044]: pgmap v20728: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:21.712 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:59:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:59:21] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T16:59:21.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:59:21 smithi083 ceph-mon[79930]: pgmap v20729: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:21.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:59:21 smithi083 ceph-mon[77242]: pgmap v20729: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:21.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:59:21 smithi153 ceph-mon[70044]: pgmap v20729: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:59:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:59:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:59:23.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:59:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:59:22.768Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:59:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:59:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:59:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:59:23.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:59:23 smithi153 ceph-mon[70044]: pgmap v20730: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:23.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:59:23 smithi083 ceph-mon[79930]: pgmap v20730: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:23.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:59:23 smithi083 ceph-mon[77242]: pgmap v20730: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:25.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:59:24 smithi083 ceph-mon[79930]: pgmap v20731: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:25.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:59:24 smithi083 ceph-mon[77242]: pgmap v20731: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:25.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:59:24 smithi153 ceph-mon[70044]: pgmap v20731: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:26.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:59:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:59:26.111Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:59:27.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:59:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:59:26.864Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:59:27.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:59:27 smithi083 ceph-mon[79930]: pgmap v20732: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:27.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:59:27 smithi083 ceph-mon[77242]: pgmap v20732: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:27.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:59:27 smithi153 ceph-mon[70044]: pgmap v20732: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:29.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:59:29 smithi083 ceph-mon[77242]: pgmap v20733: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:29.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:59:29 smithi083 ceph-mon[79930]: pgmap v20733: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:29.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:59:29 smithi153 ceph-mon[70044]: pgmap v20733: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:31.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:59:31 smithi153 ceph-mon[70044]: pgmap v20734: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:59:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:59:31] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T16:59:31.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:59:31 smithi083 ceph-mon[79930]: pgmap v20734: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:31.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:59:31 smithi083 ceph-mon[77242]: pgmap v20734: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:59:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:59:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:59:33.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:59:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:59:32.769Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:59:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:59:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:59:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:59:33.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:59:33 smithi153 ceph-mon[70044]: pgmap v20735: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:33.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:59:33 smithi083 ceph-mon[79930]: pgmap v20735: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:33.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:59:33 smithi083 ceph-mon[77242]: pgmap v20735: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:34.743 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T16:59:35.035 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T16:59:35.035 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 7m ago 11h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T16:59:35.036 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 5m ago 11h 63.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T16:59:35.036 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 7m ago 11h 26.1M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T16:59:35.036 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 5m ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T16:59:35.037 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (11h) 5m ago 11h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T16:59:35.037 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11h) 7m ago 11h 1092M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T16:59:35.038 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11h) 7m ago 11h 2471M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T16:59:35.038 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (11h) 5m ago 11h 1111M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T16:59:35.039 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (11h) 7m ago 11h 1491M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T16:59:35.039 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 7m ago 11h 24.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T16:59:35.040 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 5m ago 11h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T16:59:35.040 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11h) 7m ago 11h 1803M 2035M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T16:59:35.041 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11h) 7m ago 11h 1788M 2035M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T16:59:35.041 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11h) 7m ago 11h 1388M 2035M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T16:59:35.041 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (11h) 7m ago 11h 1754M 2035M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T16:59:35.042 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (11h) 5m ago 11h 1536M 2482M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T16:59:35.042 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (11h) 5m ago 11h 1932M 2482M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T16:59:35.043 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (11h) 5m ago 11h 1726M 2482M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T16:59:35.043 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (11h) 5m ago 11h 2056M 2482M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T16:59:35.044 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (11h) 5m ago 11h 199M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T16:59:35.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:59:34 smithi083 ceph-mon[79930]: pgmap v20736: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:35.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:59:34 smithi083 ceph-mon[77242]: pgmap v20736: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:35.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:59:34 smithi153 ceph-mon[70044]: pgmap v20736: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:35.388 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T16:59:35.389 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T16:59:35.389 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T16:59:35.390 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:59:35.390 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T16:59:35.391 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T16:59:35.391 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:59:35.391 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T16:59:35.393 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T16:59:35.393 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:59:35.394 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T16:59:35.395 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T16:59:35.395 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T16:59:35.395 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T16:59:35.395 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T16:59:35.396 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T16:59:35.396 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T16:59:35.396 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T16:59:36.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:59:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:59:36.111Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:59:36.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:59:35 smithi083 ceph-mon[77242]: from='client.58242 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:59:36.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:59:35 smithi083 ceph-mon[77242]: from='client.58248 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:59:36.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:59:35 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4214776818' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:59:36.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:59:35 smithi083 ceph-mon[79930]: from='client.58242 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:59:36.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:59:35 smithi083 ceph-mon[79930]: from='client.58248 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:59:36.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:59:35 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4214776818' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:59:36.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:59:35 smithi153 ceph-mon[70044]: from='client.58242 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:59:36.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:59:35 smithi153 ceph-mon[70044]: from='client.58248 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T16:59:36.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:59:35 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4214776818' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T16:59:37.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:59:36 smithi083 ceph-mon[77242]: pgmap v20737: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:37.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:59:36 smithi083 ceph-mon[79930]: pgmap v20737: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:37.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:59:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:59:36.865Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:59:37.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:59:36 smithi153 ceph-mon[70044]: pgmap v20737: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:39.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:59:39 smithi083 ceph-mon[79930]: pgmap v20738: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:39.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:59:39 smithi083 ceph-mon[77242]: pgmap v20738: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:39.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:59:39 smithi153 ceph-mon[70044]: pgmap v20738: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:41.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:59:41 smithi153 ceph-mon[70044]: pgmap v20739: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:59:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:59:41] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T16:59:41.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:59:41 smithi083 ceph-mon[79930]: pgmap v20739: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:41.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:59:41 smithi083 ceph-mon[77242]: pgmap v20739: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:42.771 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:59:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:59:42.771 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:59:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:59:42.772 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:59:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:59:42.772 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:59:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:59:42.772 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:59:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:59:42.772 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:59:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:59:42.773 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:59:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:59:42.773 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:59:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:59:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:59:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:59:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:59:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:59:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:59:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T16:59:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:59:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T16:59:42.809 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:59:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:59:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:59:43.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:59:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:59:42.770Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:59:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:59:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:59:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:59:43.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:59:43 smithi153 ceph-mon[70044]: pgmap v20740: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:43.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:59:43 smithi083 ceph-mon[79930]: pgmap v20740: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:43.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:59:43 smithi083 ceph-mon[77242]: pgmap v20740: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:45.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:59:44 smithi083 ceph-mon[79930]: pgmap v20741: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:45.218 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:59:44 smithi083 ceph-mon[77242]: pgmap v20741: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:45.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:59:44 smithi153 ceph-mon[70044]: pgmap v20741: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:46.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:59:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:59:46.112Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:59:47.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:59:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:59:46.865Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:59:47.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:59:47 smithi153 ceph-mon[70044]: pgmap v20742: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:47.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:59:47 smithi083 ceph-mon[79930]: pgmap v20742: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:47.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:59:47 smithi083 ceph-mon[77242]: pgmap v20742: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:49.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:59:49 smithi153 ceph-mon[70044]: pgmap v20743: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:49.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:59:49 smithi083 ceph-mon[79930]: pgmap v20743: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:49.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:59:49 smithi083 ceph-mon[77242]: pgmap v20743: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:51.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:59:51 smithi153 ceph-mon[70044]: pgmap v20744: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 16:59:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:59:51] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T16:59:51.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:59:51 smithi083 ceph-mon[79930]: pgmap v20744: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:51.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:59:51 smithi083 ceph-mon[77242]: pgmap v20744: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 16:59:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T16:59:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T16:59:53.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:59:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:59:52.771Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:59:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 16:59:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:16:59:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T16:59:53.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:59:53 smithi153 ceph-mon[70044]: pgmap v20745: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:53.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:59:53 smithi083 ceph-mon[79930]: pgmap v20745: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:53.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:59:53 smithi083 ceph-mon[77242]: pgmap v20745: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:55.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:59:54 smithi083 ceph-mon[79930]: pgmap v20746: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:55.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:59:54 smithi083 ceph-mon[77242]: pgmap v20746: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:55.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:59:54 smithi153 ceph-mon[70044]: pgmap v20746: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:56.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:59:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:59:56.113Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:59:57.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 16:59:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T16:59:56.866Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T16:59:57.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:59:57 smithi153 ceph-mon[70044]: pgmap v20747: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:57.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:59:57 smithi083 ceph-mon[79930]: pgmap v20747: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:57.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:59:57 smithi083 ceph-mon[77242]: pgmap v20747: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:59.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 16:59:59 smithi153 ceph-mon[70044]: pgmap v20748: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:59.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 16:59:59 smithi083 ceph-mon[79930]: pgmap v20748: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T16:59:59.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 16:59:59 smithi083 ceph-mon[77242]: pgmap v20748: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:00.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:00:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[77219]: 2022-09-29T16:59:59.999+0000 7f160f665700 -1 log_channel(cluster) log [ERR] : overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T17:00:00.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:00:00 smithi153 ceph-mon[70044]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T17:00:00.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:00:00 smithi083 ceph-mon[77242]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T17:00:00.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:00:00 smithi083 ceph-mon[79930]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T17:00:01.769 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:00:01 smithi153 ceph-mon[70044]: pgmap v20749: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:01.769 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:00:01 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:00:01.563Z caller=compact.go:519 level=info component=tsdb msg="write block" mint=1664460000138 maxt=1664467200000 ulid=01GE531Q3E8H05FSA4Y1K7RM91 duration=45.948765ms 2022-09-29T17:00:01.770 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:00:01 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:00:01.567Z caller=head.go:827 level=info component=tsdb msg="Head GC completed" duration=3.005302ms 2022-09-29T17:00:01.771 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:00:01 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:00:01.666Z caller=compact.go:460 level=info component=tsdb msg="compact blocks" count=3 mint=1664431200138 maxt=1664452800000 ulid=01GE531Q50Q4H4KWS11H89YZM5 sources="[01GE47JT3BCDM70Z9EJNTEMVGZ 01GE4EEHBAVN0KVW1PZWGGFA8X 01GE4NA8KA55YEQREYPDKD6DBD]" duration=98.346654ms 2022-09-29T17:00:01.771 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:00:01 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:00:01.667Z caller=db.go:1287 level=info component=tsdb msg="Deleting obsolete block" block=01GE47JT3BCDM70Z9EJNTEMVGZ 2022-09-29T17:00:01.771 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:00:01 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:00:01.668Z caller=db.go:1287 level=info component=tsdb msg="Deleting obsolete block" block=01GE4EEHBAVN0KVW1PZWGGFA8X 2022-09-29T17:00:01.772 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:00:01 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:00:01.668Z caller=db.go:1287 level=info component=tsdb msg="Deleting obsolete block" block=01GE4NA8KA55YEQREYPDKD6DBD 2022-09-29T17:00:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 17:00:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:00:01] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T17:00:01.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:00:01 smithi083 ceph-mon[79930]: pgmap v20749: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:01.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:00:01 smithi083 ceph-mon[77242]: pgmap v20749: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:02.058 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:00:01 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:00:01.766Z caller=compact.go:460 level=info component=tsdb msg="compact blocks" count=2 mint=1664429142218 maxt=1664452800000 ulid=01GE531Q84CZGBVH374VH8JXN4 sources="[01GE45KZVE75F92ZBKVQQ0RAXZ 01GE531Q50Q4H4KWS11H89YZM5]" duration=97.611013ms 2022-09-29T17:00:02.059 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:00:01 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:00:01.767Z caller=db.go:1287 level=info component=tsdb msg="Deleting obsolete block" block=01GE45KZVE75F92ZBKVQQ0RAXZ 2022-09-29T17:00:02.059 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:00:01 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:00:01.768Z caller=db.go:1287 level=info component=tsdb msg="Deleting obsolete block" block=01GE531Q50Q4H4KWS11H89YZM5 2022-09-29T17:00:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:00:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:00:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T17:00:03.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:00:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:00:02.771Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:00:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 17:00:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:00:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T17:00:03.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:00:03 smithi153 ceph-mon[70044]: pgmap v20750: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:03.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:00:03 smithi083 ceph-mon[79930]: pgmap v20750: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:03.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:00:03 smithi083 ceph-mon[77242]: pgmap v20750: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:05.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:00:04 smithi083 ceph-mon[79930]: pgmap v20751: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:05.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:00:04 smithi083 ceph-mon[77242]: pgmap v20751: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:05.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:00:04 smithi153 ceph-mon[70044]: pgmap v20751: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:05.718 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T17:00:06.007 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T17:00:06.008 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 8m ago 11h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T17:00:06.008 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 6m ago 11h 63.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T17:00:06.008 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 8m ago 11h 26.1M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T17:00:06.008 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 6m ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T17:00:06.009 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (11h) 6m ago 11h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T17:00:06.009 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11h) 8m ago 11h 1092M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T17:00:06.009 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11h) 8m ago 11h 2471M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T17:00:06.010 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (11h) 6m ago 11h 1111M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T17:00:06.011 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (11h) 8m ago 11h 1491M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T17:00:06.011 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 8m ago 11h 24.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T17:00:06.012 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 6m ago 11h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T17:00:06.012 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11h) 8m ago 11h 1803M 2035M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T17:00:06.012 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11h) 8m ago 11h 1788M 2035M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T17:00:06.013 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11h) 8m ago 11h 1388M 2035M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T17:00:06.013 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (11h) 8m ago 11h 1754M 2035M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T17:00:06.013 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (11h) 6m ago 11h 1536M 2482M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T17:00:06.014 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (11h) 6m ago 11h 1932M 2482M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T17:00:06.014 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (11h) 6m ago 11h 1726M 2482M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T17:00:06.014 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (11h) 6m ago 11h 2056M 2482M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T17:00:06.015 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (11h) 6m ago 11h 199M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T17:00:06.362 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T17:00:06.363 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T17:00:06.363 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T17:00:06.364 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:00:06.364 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T17:00:06.365 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T17:00:06.365 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:00:06.365 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T17:00:06.365 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T17:00:06.366 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:00:06.366 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T17:00:06.366 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T17:00:06.367 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:00:06.367 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T17:00:06.367 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T17:00:06.367 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T17:00:06.368 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T17:00:06.368 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T17:00:06.414 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:00:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:00:06.114Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:00:06.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:00:06 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1995543943' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T17:00:06.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:00:06 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1995543943' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T17:00:06.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:00:06 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1995543943' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T17:00:07.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:00:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:00:06.867Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:00:07.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:00:07 smithi083 ceph-mon[79930]: from='client.58260 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:00:07.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:00:07 smithi083 ceph-mon[79930]: from='client.68197 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:00:07.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:00:07 smithi083 ceph-mon[79930]: pgmap v20752: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:07.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:00:07 smithi083 ceph-mon[77242]: from='client.58260 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:00:07.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:00:07 smithi083 ceph-mon[77242]: from='client.68197 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:00:07.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:00:07 smithi083 ceph-mon[77242]: pgmap v20752: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:07.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:00:07 smithi153 ceph-mon[70044]: from='client.58260 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:00:07.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:00:07 smithi153 ceph-mon[70044]: from='client.68197 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:00:07.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:00:07 smithi153 ceph-mon[70044]: pgmap v20752: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:09.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:00:09 smithi083 ceph-mon[79930]: pgmap v20753: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:09.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:00:09 smithi083 ceph-mon[77242]: pgmap v20753: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:09.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:00:09 smithi153 ceph-mon[70044]: pgmap v20753: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:10.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:00:10 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T17:00:10.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:00:10 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T17:00:10.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:00:10 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T17:00:10.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:00:10 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T17:00:10.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:00:10 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T17:00:10.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:00:10 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T17:00:10.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:00:10 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T17:00:10.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:00:10 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T17:00:10.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:00:10 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T17:00:11.711 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 17:00:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:00:11] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T17:00:11.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:00:11 smithi083 ceph-mon[77242]: pgmap v20754: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:11.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:00:11 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:00:11.712 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:00:11 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:00:11.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:00:11 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:00:11.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:00:11 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:00:11.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:00:11 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:00:11.713 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:00:11 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:00:11.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:00:11 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:00:11.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:00:11 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:00:11.714 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:00:11 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:00:11.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:00:11 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:00:11.715 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:00:11 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:00:11.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:00:11 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:00:11.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:00:11 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:00:11.716 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:00:11 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:00:11.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:00:11 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:00:11.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:00:11 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:00:11.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:00:11 smithi083 ceph-mon[77242]: Adjusting osd_memory_target on smithi153 to 2478M 2022-09-29T17:00:11.717 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:00:11 smithi083 ceph-mon[77242]: Adjusting osd_memory_target on smithi083 to 2021M 2022-09-29T17:00:11.718 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:00:11 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:00:11.718 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:00:11 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:00:11.718 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:00:11 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:00:11.719 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:00:11 smithi083 ceph-mon[79930]: pgmap v20754: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:11.719 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:00:11 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:00:11.720 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:00:11 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:00:11.720 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:00:11 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:00:11.720 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:00:11 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:00:11.720 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:00:11 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:00:11.721 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:00:11 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:00:11.721 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:00:11 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:00:11.721 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:00:11 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:00:11.721 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:00:11 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:00:11.722 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:00:11 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:00:11.722 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:00:11 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:00:11.722 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:00:11 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:00:11.723 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:00:11 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:00:11.723 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:00:11 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:00:11.723 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:00:11 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:00:11.723 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:00:11 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:00:11.724 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:00:11 smithi083 ceph-mon[79930]: Adjusting osd_memory_target on smithi153 to 2478M 2022-09-29T17:00:11.724 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:00:11 smithi083 ceph-mon[79930]: Adjusting osd_memory_target on smithi083 to 2021M 2022-09-29T17:00:11.724 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:00:11 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:00:11.724 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:00:11 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:00:11.725 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:00:11 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:00:11.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:00:11 smithi153 ceph-mon[70044]: pgmap v20754: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:11.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:00:11 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:00:11.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:00:11 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:00:11.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:00:11 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:00:11.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:00:11 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:00:11.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:00:11 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:00:11.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:00:11 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:00:11.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:00:11 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:00:11.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:00:11 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:00:11.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:00:11 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:00:11.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:00:11 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:00:11.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:00:11 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:00:11.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:00:11 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:00:11.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:00:11 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:00:11.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:00:11 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:00:11.813 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:00:11 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:00:11.813 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:00:11 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:00:11.813 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:00:11 smithi153 ceph-mon[70044]: Adjusting osd_memory_target on smithi153 to 2478M 2022-09-29T17:00:11.821 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:00:11 smithi153 ceph-mon[70044]: Adjusting osd_memory_target on smithi083 to 2021M 2022-09-29T17:00:11.821 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:00:11 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:00:11.822 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:00:11 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:00:11.822 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:00:11 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:00:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:00:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:00:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T17:00:13.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:00:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:00:12.772Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:00:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 17:00:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:00:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T17:00:13.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:00:13 smithi083 ceph-mon[77242]: pgmap v20755: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:13.712 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:00:13 smithi083 ceph-mon[79930]: pgmap v20755: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:13.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:00:13 smithi153 ceph-mon[70044]: pgmap v20755: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:15.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:00:14 smithi083 ceph-mon[77242]: pgmap v20756: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:15.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:00:14 smithi083 ceph-mon[79930]: pgmap v20756: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:15.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:00:14 smithi153 ceph-mon[70044]: pgmap v20756: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:16.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:00:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:00:16.114Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:00:17.217 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:00:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:00:16.868Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:00:17.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:00:17 smithi153 ceph-mon[70044]: pgmap v20757: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:17.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:00:17 smithi083 ceph-mon[77242]: pgmap v20757: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:17.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:00:17 smithi083 ceph-mon[79930]: pgmap v20757: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:19.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:00:19 smithi153 ceph-mon[70044]: pgmap v20758: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:19.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:00:19 smithi083 ceph-mon[77242]: pgmap v20758: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:19.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:00:19 smithi083 ceph-mon[79930]: pgmap v20758: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:21.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:00:21 smithi153 ceph-mon[70044]: pgmap v20759: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 17:00:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:00:21] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T17:00:21.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:00:21 smithi083 ceph-mon[79930]: pgmap v20759: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:21.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:00:21 smithi083 ceph-mon[77242]: pgmap v20759: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:00:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:00:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T17:00:23.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:00:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:00:22.772Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:00:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 17:00:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:00:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T17:00:23.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:00:23 smithi153 ceph-mon[70044]: pgmap v20760: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:23.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:00:23 smithi083 ceph-mon[77242]: pgmap v20760: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:23.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:00:23 smithi083 ceph-mon[79930]: pgmap v20760: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:25.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:00:24 smithi083 ceph-mon[79930]: pgmap v20761: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:25.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:00:24 smithi083 ceph-mon[77242]: pgmap v20761: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:25.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:00:24 smithi153 ceph-mon[70044]: pgmap v20761: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:26.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:00:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:00:26.115Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:00:27.214 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:00:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:00:26.869Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:00:27.817 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:00:27 smithi153 ceph-mon[70044]: pgmap v20762: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:27.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:00:27 smithi083 ceph-mon[79930]: pgmap v20762: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:27.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:00:27 smithi083 ceph-mon[77242]: pgmap v20762: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:29.817 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:00:29 smithi153 ceph-mon[70044]: pgmap v20763: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:29.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:00:29 smithi083 ceph-mon[79930]: pgmap v20763: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:29.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:00:29 smithi083 ceph-mon[77242]: pgmap v20763: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:31.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:00:31 smithi153 ceph-mon[70044]: pgmap v20764: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 17:00:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:00:31] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T17:00:31.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:00:31 smithi083 ceph-mon[79930]: pgmap v20764: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:31.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:00:31 smithi083 ceph-mon[77242]: pgmap v20764: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:00:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:00:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T17:00:33.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:00:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:00:32.772Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:00:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 17:00:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:00:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T17:00:33.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:00:33 smithi153 ceph-mon[70044]: pgmap v20765: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:33.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:00:33 smithi083 ceph-mon[77242]: pgmap v20765: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:33.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:00:33 smithi083 ceph-mon[79930]: pgmap v20765: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:35.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:00:34 smithi083 ceph-mon[77242]: pgmap v20766: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:35.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:00:34 smithi083 ceph-mon[79930]: pgmap v20766: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:35.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:00:34 smithi153 ceph-mon[70044]: pgmap v20766: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:36.465 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:00:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:00:36.115Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:00:36.693 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T17:00:36.981 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T17:00:36.982 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 8m ago 11h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T17:00:36.982 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 6m ago 11h 63.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T17:00:36.982 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 8m ago 11h 26.1M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T17:00:36.982 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 6m ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T17:00:36.983 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (11h) 6m ago 11h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T17:00:36.983 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11h) 8m ago 11h 1092M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T17:00:36.983 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11h) 8m ago 11h 2471M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T17:00:36.983 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (11h) 6m ago 11h 1111M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T17:00:36.984 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (11h) 8m ago 11h 1491M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T17:00:36.984 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 8m ago 11h 24.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T17:00:36.984 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 6m ago 11h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T17:00:36.984 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11h) 8m ago 11h 1803M 2021M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T17:00:36.985 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11h) 8m ago 11h 1788M 2021M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T17:00:36.985 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11h) 8m ago 11h 1388M 2021M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T17:00:36.985 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (11h) 8m ago 11h 1754M 2021M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T17:00:36.985 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (11h) 6m ago 11h 1536M 2478M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T17:00:36.985 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (11h) 6m ago 11h 1932M 2478M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T17:00:36.986 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (11h) 6m ago 11h 1726M 2478M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T17:00:36.986 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (11h) 6m ago 11h 2056M 2478M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T17:00:36.986 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (11h) 6m ago 11h 199M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T17:00:37.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:00:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:00:36.869Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:00:37.342 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T17:00:37.343 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T17:00:37.343 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T17:00:37.343 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:00:37.343 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T17:00:37.343 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T17:00:37.344 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:00:37.344 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T17:00:37.344 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T17:00:37.344 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:00:37.345 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T17:00:37.345 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T17:00:37.345 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:00:37.345 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T17:00:37.346 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T17:00:37.346 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T17:00:37.346 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T17:00:37.346 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T17:00:37.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:00:37 smithi153 ceph-mon[70044]: pgmap v20767: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:37.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:00:37 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3286365442' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T17:00:37.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:00:37 smithi083 ceph-mon[77242]: pgmap v20767: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:37.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:00:37 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3286365442' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T17:00:37.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:00:37 smithi083 ceph-mon[79930]: pgmap v20767: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:37.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:00:37 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3286365442' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T17:00:38.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:00:38 smithi153 ceph-mon[70044]: from='client.58278 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:00:38.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:00:38 smithi153 ceph-mon[70044]: from='client.68215 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:00:38.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:00:38 smithi083 ceph-mon[79930]: from='client.58278 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:00:38.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:00:38 smithi083 ceph-mon[79930]: from='client.68215 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:00:38.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:00:38 smithi083 ceph-mon[77242]: from='client.58278 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:00:38.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:00:38 smithi083 ceph-mon[77242]: from='client.68215 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:00:39.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:00:39 smithi153 ceph-mon[70044]: pgmap v20768: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:39.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:00:39 smithi083 ceph-mon[79930]: pgmap v20768: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:39.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:00:39 smithi083 ceph-mon[77242]: pgmap v20768: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:41.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:00:41 smithi153 ceph-mon[70044]: pgmap v20769: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 17:00:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:00:41] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T17:00:41.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:00:41 smithi083 ceph-mon[79930]: pgmap v20769: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:41.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:00:41 smithi083 ceph-mon[77242]: pgmap v20769: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:42.775 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:00:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T17:00:42.775 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:00:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T17:00:42.776 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:00:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T17:00:42.776 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:00:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T17:00:42.777 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:00:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T17:00:42.777 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:00:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T17:00:42.778 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:00:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T17:00:42.778 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:00:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T17:00:42.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:00:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T17:00:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:00:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T17:00:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:00:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T17:00:42.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:00:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T17:00:42.810 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:00:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:00:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T17:00:43.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:00:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:00:42.773Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:00:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 17:00:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:00:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T17:00:43.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:00:43 smithi153 ceph-mon[70044]: pgmap v20770: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:43.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:00:43 smithi083 ceph-mon[77242]: pgmap v20770: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:43.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:00:43 smithi083 ceph-mon[79930]: pgmap v20770: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:45.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:00:44 smithi083 ceph-mon[79930]: pgmap v20771: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:45.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:00:44 smithi083 ceph-mon[77242]: pgmap v20771: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:45.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:00:44 smithi153 ceph-mon[70044]: pgmap v20771: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:46.470 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:00:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:00:46.116Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:00:47.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:00:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:00:46.870Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:00:47.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:00:47 smithi153 ceph-mon[70044]: pgmap v20772: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:47.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:00:47 smithi083 ceph-mon[77242]: pgmap v20772: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:47.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:00:47 smithi083 ceph-mon[79930]: pgmap v20772: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:49.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:00:49 smithi153 ceph-mon[70044]: pgmap v20773: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:49.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:00:49 smithi083 ceph-mon[77242]: pgmap v20773: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:49.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:00:49 smithi083 ceph-mon[79930]: pgmap v20773: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:51.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:00:51 smithi153 ceph-mon[70044]: pgmap v20774: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 17:00:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:00:51] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T17:00:51.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:00:51 smithi083 ceph-mon[77242]: pgmap v20774: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:51.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:00:51 smithi083 ceph-mon[79930]: pgmap v20774: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:00:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:00:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T17:00:53.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:00:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:00:52.773Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:00:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 17:00:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:00:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T17:00:53.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:00:53 smithi153 ceph-mon[70044]: pgmap v20775: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:53.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:00:53 smithi083 ceph-mon[79930]: pgmap v20775: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:53.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:00:53 smithi083 ceph-mon[77242]: pgmap v20775: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:55.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:00:54 smithi083 ceph-mon[79930]: pgmap v20776: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:55.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:00:54 smithi083 ceph-mon[77242]: pgmap v20776: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:55.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:00:54 smithi153 ceph-mon[70044]: pgmap v20776: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:56.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:00:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:00:56.118Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:00:57.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:00:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:00:56.871Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:00:57.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:00:57 smithi153 ceph-mon[70044]: pgmap v20777: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:57.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:00:57 smithi083 ceph-mon[79930]: pgmap v20777: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:57.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:00:57 smithi083 ceph-mon[77242]: pgmap v20777: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:59.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:00:59 smithi153 ceph-mon[70044]: pgmap v20778: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:59.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:00:59 smithi083 ceph-mon[79930]: pgmap v20778: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:00:59.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:00:59 smithi083 ceph-mon[77242]: pgmap v20778: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:01.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:01:01 smithi153 ceph-mon[70044]: pgmap v20779: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:01.871 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:01:01 smithi083 ceph-mon[77242]: pgmap v20779: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:01.872 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 17:01:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:01:01] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T17:01:01.873 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:01:01 smithi083 ceph-mon[79930]: pgmap v20779: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:01:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:01:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T17:01:03.218 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:01:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:01:02.774Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:01:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 17:01:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:01:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T17:01:03.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:01:03 smithi153 ceph-mon[70044]: pgmap v20780: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:03.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:01:03 smithi083 ceph-mon[79930]: pgmap v20780: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:03.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:01:03 smithi083 ceph-mon[77242]: pgmap v20780: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:05.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:01:04 smithi083 ceph-mon[79930]: pgmap v20781: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:05.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:01:04 smithi083 ceph-mon[77242]: pgmap v20781: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:05.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:01:04 smithi153 ceph-mon[70044]: pgmap v20781: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:06.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:01:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:01:06.118Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:01:07.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:01:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:01:06.871Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:01:07.671 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T17:01:07.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:01:07 smithi153 ceph-mon[70044]: pgmap v20782: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:07.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:01:07 smithi083 ceph-mon[79930]: pgmap v20782: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:07.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:01:07 smithi083 ceph-mon[77242]: pgmap v20782: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:07.964 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T17:01:07.964 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 9m ago 11h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T17:01:07.965 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 7m ago 11h 63.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T17:01:07.965 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 9m ago 11h 26.1M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T17:01:07.965 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 7m ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T17:01:07.966 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (11h) 7m ago 11h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T17:01:07.967 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11h) 9m ago 11h 1092M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T17:01:07.967 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11h) 9m ago 11h 2471M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T17:01:07.967 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (11h) 7m ago 11h 1111M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T17:01:07.968 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (11h) 9m ago 11h 1491M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T17:01:07.968 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 9m ago 11h 24.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T17:01:07.969 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 7m ago 11h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T17:01:07.969 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11h) 9m ago 11h 1803M 2021M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T17:01:07.970 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11h) 9m ago 11h 1788M 2021M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T17:01:07.970 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11h) 9m ago 11h 1388M 2021M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T17:01:07.971 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (11h) 9m ago 11h 1754M 2021M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T17:01:07.971 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (11h) 7m ago 11h 1536M 2478M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T17:01:07.972 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (11h) 7m ago 11h 1932M 2478M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T17:01:07.972 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (11h) 7m ago 11h 1726M 2478M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T17:01:07.972 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (11h) 7m ago 11h 2056M 2478M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T17:01:07.973 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (11h) 7m ago 11h 199M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T17:01:08.316 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T17:01:08.317 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T17:01:08.317 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T17:01:08.318 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:01:08.318 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T17:01:08.319 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T17:01:08.320 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:01:08.320 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T17:01:08.320 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T17:01:08.321 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:01:08.321 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T17:01:08.322 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T17:01:08.323 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:01:08.324 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T17:01:08.324 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T17:01:08.324 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T17:01:08.325 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T17:01:08.326 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T17:01:08.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:01:08 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/552665705' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T17:01:08.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:01:08 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/552665705' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T17:01:08.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:01:08 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/552665705' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T17:01:09.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:01:09 smithi153 ceph-mon[70044]: from='client.68227 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:01:09.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:01:09 smithi153 ceph-mon[70044]: from='client.68233 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:01:09.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:01:09 smithi153 ceph-mon[70044]: pgmap v20783: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:09.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:01:09 smithi083 ceph-mon[79930]: from='client.68227 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:01:09.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:01:09 smithi083 ceph-mon[79930]: from='client.68233 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:01:09.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:01:09 smithi083 ceph-mon[79930]: pgmap v20783: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:09.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:01:09 smithi083 ceph-mon[77242]: from='client.68227 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:01:09.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:01:09 smithi083 ceph-mon[77242]: from='client.68233 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:01:09.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:01:09 smithi083 ceph-mon[77242]: pgmap v20783: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:11.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:01:11 smithi153 ceph-mon[70044]: pgmap v20784: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:11.811 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:01:11 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T17:01:11.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:01:11 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T17:01:11.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:01:11 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T17:01:11.813 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:01:11 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:01:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 17:01:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:01:11] "GET /metrics HTTP/1.1" 200 225246 "" "Prometheus/2.33.4" 2022-09-29T17:01:11.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:01:11 smithi083 ceph-mon[79930]: pgmap v20784: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:11.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:01:11 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T17:01:11.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:01:11 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T17:01:11.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:01:11 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T17:01:11.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:01:11 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:01:11.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:01:11 smithi083 ceph-mon[77242]: pgmap v20784: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:11.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:01:11 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T17:01:11.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:01:11 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T17:01:11.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:01:11 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T17:01:11.968 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:01:11 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:01:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:01:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:01:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T17:01:13.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:01:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:01:12.775Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:01:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 17:01:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:01:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T17:01:13.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:01:13 smithi153 ceph-mon[70044]: pgmap v20785: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:13.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:01:13 smithi083 ceph-mon[77242]: pgmap v20785: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:13.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:01:13 smithi083 ceph-mon[79930]: pgmap v20785: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:15.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:01:14 smithi083 ceph-mon[77242]: pgmap v20786: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:15.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:01:14 smithi083 ceph-mon[79930]: pgmap v20786: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:15.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:01:14 smithi153 ceph-mon[70044]: pgmap v20786: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:16.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:01:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:01:16.118Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:01:17.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:01:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:01:16.872Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:01:17.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:01:17 smithi153 ceph-mon[70044]: pgmap v20787: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:17.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:01:17 smithi083 ceph-mon[77242]: pgmap v20787: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:17.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:01:17 smithi083 ceph-mon[79930]: pgmap v20787: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:19.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:01:19 smithi153 ceph-mon[70044]: pgmap v20788: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:19.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:01:19 smithi083 ceph-mon[79930]: pgmap v20788: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:19.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:01:19 smithi083 ceph-mon[77242]: pgmap v20788: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:21.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:01:21 smithi153 ceph-mon[70044]: pgmap v20789: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 17:01:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:01:21] "GET /metrics HTTP/1.1" 200 225246 "" "Prometheus/2.33.4" 2022-09-29T17:01:21.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:01:21 smithi083 ceph-mon[79930]: pgmap v20789: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:21.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:01:21 smithi083 ceph-mon[77242]: pgmap v20789: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:01:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:01:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T17:01:23.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:01:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:01:22.775Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:01:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 17:01:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:01:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T17:01:23.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:01:23 smithi153 ceph-mon[70044]: pgmap v20790: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:23.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:01:23 smithi083 ceph-mon[79930]: pgmap v20790: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:23.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:01:23 smithi083 ceph-mon[77242]: pgmap v20790: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:25.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:01:24 smithi083 ceph-mon[77242]: pgmap v20791: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:25.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:01:24 smithi083 ceph-mon[79930]: pgmap v20791: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:25.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:01:24 smithi153 ceph-mon[70044]: pgmap v20791: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:26.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:01:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:01:26.119Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:01:27.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:01:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:01:26.872Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:01:27.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:01:27 smithi153 ceph-mon[70044]: pgmap v20792: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:27.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:01:27 smithi083 ceph-mon[77242]: pgmap v20792: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:27.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:01:27 smithi083 ceph-mon[79930]: pgmap v20792: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:29.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:01:29 smithi153 ceph-mon[70044]: pgmap v20793: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:29.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:01:29 smithi083 ceph-mon[79930]: pgmap v20793: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:29.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:01:29 smithi083 ceph-mon[77242]: pgmap v20793: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:31.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:01:31 smithi153 ceph-mon[70044]: pgmap v20794: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 17:01:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:01:31] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T17:01:31.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:01:31 smithi083 ceph-mon[79930]: pgmap v20794: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:31.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:01:31 smithi083 ceph-mon[77242]: pgmap v20794: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:01:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:01:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T17:01:33.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:01:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:01:32.776Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:01:33.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 17:01:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:01:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T17:01:33.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:01:33 smithi153 ceph-mon[70044]: pgmap v20795: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:33.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:01:33 smithi083 ceph-mon[79930]: pgmap v20795: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:33.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:01:33 smithi083 ceph-mon[77242]: pgmap v20795: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:35.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:01:34 smithi083 ceph-mon[79930]: pgmap v20796: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:35.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:01:34 smithi083 ceph-mon[77242]: pgmap v20796: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:35.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:01:34 smithi153 ceph-mon[70044]: pgmap v20796: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:36.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:01:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:01:36.119Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:01:37.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:01:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:01:36.873Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:01:37.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:01:37 smithi153 ceph-mon[70044]: pgmap v20797: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:37.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:01:37 smithi083 ceph-mon[79930]: pgmap v20797: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:37.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:01:37 smithi083 ceph-mon[77242]: pgmap v20797: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:38.650 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T17:01:38.942 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T17:01:38.942 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 9m ago 11h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T17:01:38.943 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 7m ago 11h 63.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T17:01:38.943 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 9m ago 11h 26.1M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T17:01:38.943 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 7m ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T17:01:38.943 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (11h) 7m ago 11h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T17:01:38.944 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11h) 9m ago 11h 1092M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T17:01:38.944 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11h) 9m ago 11h 2471M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T17:01:38.944 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (11h) 7m ago 11h 1111M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T17:01:38.944 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (11h) 9m ago 11h 1491M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T17:01:38.945 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 9m ago 11h 24.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T17:01:38.945 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 7m ago 11h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T17:01:38.945 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11h) 9m ago 11h 1803M 2021M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T17:01:38.945 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11h) 9m ago 11h 1788M 2021M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T17:01:38.947 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11h) 9m ago 11h 1388M 2021M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T17:01:38.947 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (11h) 9m ago 11h 1754M 2021M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T17:01:38.947 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (11h) 7m ago 11h 1536M 2478M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T17:01:38.947 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (11h) 7m ago 11h 1932M 2478M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T17:01:38.948 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (11h) 7m ago 11h 1726M 2478M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T17:01:38.948 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (11h) 7m ago 11h 2056M 2478M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T17:01:38.948 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (11h) 7m ago 11h 199M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T17:01:39.295 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T17:01:39.295 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T17:01:39.296 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T17:01:39.296 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:01:39.297 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T17:01:39.297 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T17:01:39.298 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:01:39.298 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T17:01:39.299 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T17:01:39.299 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:01:39.300 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T17:01:39.300 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T17:01:39.301 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:01:39.301 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T17:01:39.301 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T17:01:39.302 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T17:01:39.303 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T17:01:39.303 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T17:01:39.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:01:39 smithi153 ceph-mon[70044]: pgmap v20798: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:39.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:01:39 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2336864371' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T17:01:39.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:01:39 smithi083 ceph-mon[79930]: pgmap v20798: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:39.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:01:39 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2336864371' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T17:01:39.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:01:39 smithi083 ceph-mon[77242]: pgmap v20798: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:39.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:01:39 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2336864371' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T17:01:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:01:40 smithi153 ceph-mon[70044]: from='client.58314 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:01:40.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:01:40 smithi153 ceph-mon[70044]: from='client.58320 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:01:40.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:01:40 smithi083 ceph-mon[79930]: from='client.58314 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:01:40.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:01:40 smithi083 ceph-mon[79930]: from='client.58320 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:01:40.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:01:40 smithi083 ceph-mon[77242]: from='client.58314 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:01:40.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:01:40 smithi083 ceph-mon[77242]: from='client.58320 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:01:41.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:01:41 smithi153 ceph-mon[70044]: pgmap v20799: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 17:01:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:01:41] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T17:01:41.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:01:41 smithi083 ceph-mon[79930]: pgmap v20799: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:41.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:01:41 smithi083 ceph-mon[77242]: pgmap v20799: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:42.778 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:01:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T17:01:42.779 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:01:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T17:01:42.779 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:01:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T17:01:42.780 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:01:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T17:01:42.780 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:01:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T17:01:42.781 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:01:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T17:01:42.781 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:01:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T17:01:42.781 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:01:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T17:01:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:01:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:01:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T17:01:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:01:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T17:01:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:01:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T17:01:42.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:01:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T17:01:42.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:01:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T17:01:43.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:01:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:01:42.777Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:01:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 17:01:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:01:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T17:01:43.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:01:43 smithi153 ceph-mon[70044]: pgmap v20800: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:43.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:01:43 smithi083 ceph-mon[77242]: pgmap v20800: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:43.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:01:43 smithi083 ceph-mon[79930]: pgmap v20800: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:45.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:01:44 smithi083 ceph-mon[77242]: pgmap v20801: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:45.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:01:44 smithi083 ceph-mon[79930]: pgmap v20801: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:45.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:01:44 smithi153 ceph-mon[70044]: pgmap v20801: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:46.467 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:01:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:01:46.120Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:01:47.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:01:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:01:46.873Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:01:47.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:01:47 smithi153 ceph-mon[70044]: pgmap v20802: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:47.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:01:47 smithi083 ceph-mon[77242]: pgmap v20802: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:47.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:01:47 smithi083 ceph-mon[79930]: pgmap v20802: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:49.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:01:49 smithi153 ceph-mon[70044]: pgmap v20803: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:49.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:01:49 smithi083 ceph-mon[77242]: pgmap v20803: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:49.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:01:49 smithi083 ceph-mon[79930]: pgmap v20803: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:51.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:01:51 smithi153 ceph-mon[70044]: pgmap v20804: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 17:01:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:01:51] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T17:01:51.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:01:51 smithi083 ceph-mon[79930]: pgmap v20804: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:51.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:01:51 smithi083 ceph-mon[77242]: pgmap v20804: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:01:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:01:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T17:01:53.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:01:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:01:52.777Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:01:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 17:01:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:01:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T17:01:53.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:01:53 smithi153 ceph-mon[70044]: pgmap v20805: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:53.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:01:53 smithi083 ceph-mon[77242]: pgmap v20805: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:53.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:01:53 smithi083 ceph-mon[79930]: pgmap v20805: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:55.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:01:54 smithi083 ceph-mon[77242]: pgmap v20806: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:55.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:01:54 smithi083 ceph-mon[79930]: pgmap v20806: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:55.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:01:54 smithi153 ceph-mon[70044]: pgmap v20806: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:56.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:01:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:01:56.120Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:01:57.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:01:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:01:56.873Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:01:57.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:01:57 smithi153 ceph-mon[70044]: pgmap v20807: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:57.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:01:57 smithi083 ceph-mon[79930]: pgmap v20807: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:57.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:01:57 smithi083 ceph-mon[77242]: pgmap v20807: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:59.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:01:59 smithi153 ceph-mon[70044]: pgmap v20808: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:59.968 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:01:59 smithi083 ceph-mon[79930]: pgmap v20808: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:01:59.969 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:01:59 smithi083 ceph-mon[77242]: pgmap v20808: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:01.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:02:01 smithi153 ceph-mon[70044]: pgmap v20809: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 17:02:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:02:01] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T17:02:01.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:02:01 smithi083 ceph-mon[79930]: pgmap v20809: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:01.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:02:01 smithi083 ceph-mon[77242]: pgmap v20809: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:02:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:02:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T17:02:03.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:02:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:02:02.778Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:02:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 17:02:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:02:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T17:02:03.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:02:03 smithi153 ceph-mon[70044]: pgmap v20810: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:03.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:02:03 smithi083 ceph-mon[79930]: pgmap v20810: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:03.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:02:03 smithi083 ceph-mon[77242]: pgmap v20810: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:05.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:02:04 smithi083 ceph-mon[79930]: pgmap v20811: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:05.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:02:04 smithi083 ceph-mon[77242]: pgmap v20811: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:05.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:02:04 smithi153 ceph-mon[70044]: pgmap v20811: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:06.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:02:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:02:06.121Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:02:07.219 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:02:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:02:06.873Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:02:07.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:02:07 smithi153 ceph-mon[70044]: pgmap v20812: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:07.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:02:07 smithi083 ceph-mon[79930]: pgmap v20812: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:07.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:02:07 smithi083 ceph-mon[77242]: pgmap v20812: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:09.623 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T17:02:09.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:02:09 smithi153 ceph-mon[70044]: pgmap v20813: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:09.936 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T17:02:09.936 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 10m ago 11h 27.2M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T17:02:09.938 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 8m ago 11h 63.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T17:02:09.938 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 10m ago 11h 26.1M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T17:02:09.939 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 8m ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T17:02:09.939 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (11h) 8m ago 11h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T17:02:09.940 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11h) 10m ago 11h 1092M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T17:02:09.940 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11h) 10m ago 11h 2471M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T17:02:09.940 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (11h) 8m ago 11h 1111M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T17:02:09.941 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (11h) 10m ago 11h 1491M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T17:02:09.941 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 10m ago 11h 24.4M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T17:02:09.942 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 8m ago 11h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T17:02:09.942 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11h) 10m ago 11h 1803M 2021M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T17:02:09.943 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11h) 10m ago 11h 1788M 2021M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T17:02:09.943 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11h) 10m ago 11h 1388M 2021M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T17:02:09.944 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (11h) 10m ago 11h 1754M 2021M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T17:02:09.944 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (11h) 8m ago 11h 1536M 2478M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T17:02:09.945 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (11h) 8m ago 11h 1932M 2478M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T17:02:09.945 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (11h) 8m ago 11h 1726M 2478M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T17:02:09.946 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (11h) 8m ago 11h 2056M 2478M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T17:02:09.946 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (11h) 8m ago 11h 199M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T17:02:09.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:02:09 smithi083 ceph-mon[79930]: pgmap v20813: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:09.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:02:09 smithi083 ceph-mon[77242]: pgmap v20813: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:10.289 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T17:02:10.290 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T17:02:10.290 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T17:02:10.291 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:02:10.291 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T17:02:10.292 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T17:02:10.293 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:02:10.293 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T17:02:10.294 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T17:02:10.296 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:02:10.296 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T17:02:10.296 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T17:02:10.297 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:02:10.297 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T17:02:10.297 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T17:02:10.297 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T17:02:10.298 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T17:02:10.298 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T17:02:10.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:02:10 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1500919937' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T17:02:10.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:02:10 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1500919937' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T17:02:10.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:02:10 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1500919937' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T17:02:11.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:02:11 smithi153 ceph-mon[70044]: from='client.68263 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:02:11.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:02:11 smithi153 ceph-mon[70044]: from='client.68269 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:02:11.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:02:11 smithi153 ceph-mon[70044]: pgmap v20814: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:11.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:02:11 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T17:02:11.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:02:11 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T17:02:11.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:02:11 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T17:02:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 17:02:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:02:11] "GET /metrics HTTP/1.1" 200 225246 "" "Prometheus/2.33.4" 2022-09-29T17:02:11.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:02:11 smithi083 ceph-mon[79930]: from='client.68263 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:02:11.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:02:11 smithi083 ceph-mon[79930]: from='client.68269 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:02:11.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:02:11 smithi083 ceph-mon[79930]: pgmap v20814: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:11.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:02:11 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T17:02:11.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:02:11 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T17:02:11.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:02:11 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T17:02:11.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:02:11 smithi083 ceph-mon[77242]: from='client.68263 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:02:11.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:02:11 smithi083 ceph-mon[77242]: from='client.68269 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:02:11.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:02:11 smithi083 ceph-mon[77242]: pgmap v20814: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:11.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:02:11 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T17:02:11.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:02:11 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T17:02:11.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:02:11 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T17:02:12.643 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:02:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:02:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T17:02:12.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:02:12 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:02:12.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:02:12 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:02:12.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:02:12 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:02:12.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:02:12 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:02:12.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:02:12 smithi083 ceph-mon[79930]: pgmap v20815: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:12.966 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:02:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:02:12.779Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:02:12.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:02:12 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:02:12.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:02:12 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:02:12.968 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:02:12 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:02:12.968 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:02:12 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:02:12.969 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:02:12 smithi083 ceph-mon[77242]: pgmap v20815: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:13.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:02:12 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:02:13.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:02:12 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:02:13.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:02:12 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:02:13.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:02:12 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:02:13.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:02:12 smithi153 ceph-mon[70044]: pgmap v20815: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 17:02:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:02:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T17:02:15.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:02:14 smithi083 ceph-mon[79930]: pgmap v20816: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:15.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:02:14 smithi083 ceph-mon[77242]: pgmap v20816: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:15.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:02:14 smithi153 ceph-mon[70044]: pgmap v20816: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:16.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:02:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:02:16.121Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:02:17.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:02:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:02:16.873Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:02:17.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:02:16 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:02:17.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:02:16 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:02:17.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:02:16 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:02:17.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:02:16 smithi083 ceph-mon[77242]: pgmap v20817: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:17.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:02:16 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:02:17.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:02:16 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:02:17.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:02:16 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:02:17.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:02:16 smithi083 ceph-mon[79930]: pgmap v20817: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:17.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:02:16 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:02:17.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:02:16 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:02:17.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:02:16 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:02:17.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:02:16 smithi153 ceph-mon[70044]: pgmap v20817: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:19.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:02:19 smithi153 ceph-mon[70044]: pgmap v20818: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:19.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:02:19 smithi083 ceph-mon[79930]: pgmap v20818: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:19.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:02:19 smithi083 ceph-mon[77242]: pgmap v20818: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:21.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:02:21 smithi153 ceph-mon[70044]: pgmap v20819: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:21.969 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 17:02:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:02:21] "GET /metrics HTTP/1.1" 200 225246 "" "Prometheus/2.33.4" 2022-09-29T17:02:21.971 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:02:21 smithi083 ceph-mon[79930]: pgmap v20819: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:21.971 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:02:21 smithi083 ceph-mon[77242]: pgmap v20819: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:02:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:02:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T17:02:23.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:02:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:02:22.779Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:02:23.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 17:02:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:02:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T17:02:23.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:02:23 smithi153 ceph-mon[70044]: pgmap v20820: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:23.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:02:23 smithi083 ceph-mon[79930]: pgmap v20820: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:23.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:02:23 smithi083 ceph-mon[77242]: pgmap v20820: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:25.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:02:24 smithi083 ceph-mon[79930]: pgmap v20821: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:25.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:02:24 smithi083 ceph-mon[77242]: pgmap v20821: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:25.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:02:24 smithi153 ceph-mon[70044]: pgmap v20821: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:26.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:02:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:02:26.122Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:02:27.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:02:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:02:26.875Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:02:27.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:02:27 smithi153 ceph-mon[70044]: pgmap v20822: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:27.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:02:27 smithi083 ceph-mon[79930]: pgmap v20822: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:27.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:02:27 smithi083 ceph-mon[77242]: pgmap v20822: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:29.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:02:29 smithi153 ceph-mon[70044]: pgmap v20823: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:29.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:02:29 smithi083 ceph-mon[79930]: pgmap v20823: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:29.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:02:29 smithi083 ceph-mon[77242]: pgmap v20823: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:31.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:02:31 smithi153 ceph-mon[70044]: pgmap v20824: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 17:02:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:02:31] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T17:02:31.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:02:31 smithi083 ceph-mon[79930]: pgmap v20824: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:31.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:02:31 smithi083 ceph-mon[77242]: pgmap v20824: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:02:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:02:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T17:02:33.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:02:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:02:32.780Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:02:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 17:02:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:02:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T17:02:33.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:02:33 smithi153 ceph-mon[70044]: pgmap v20825: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:33.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:02:33 smithi083 ceph-mon[79930]: pgmap v20825: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:33.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:02:33 smithi083 ceph-mon[77242]: pgmap v20825: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:35.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:02:34 smithi083 ceph-mon[79930]: pgmap v20826: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:35.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:02:34 smithi083 ceph-mon[77242]: pgmap v20826: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:35.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:02:34 smithi153 ceph-mon[70044]: pgmap v20826: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:36.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:02:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:02:36.122Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:02:37.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:02:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:02:36.875Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:02:37.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:02:37 smithi153 ceph-mon[70044]: pgmap v20827: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:37.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:02:37 smithi083 ceph-mon[77242]: pgmap v20827: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:37.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:02:37 smithi083 ceph-mon[79930]: pgmap v20827: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:39.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:02:39 smithi153 ceph-mon[70044]: pgmap v20828: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:39.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:02:39 smithi083 ceph-mon[77242]: pgmap v20828: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:02:39 smithi083 ceph-mon[79930]: pgmap v20828: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:40.624 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T17:02:40.904 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T17:02:40.905 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 25s ago 11h 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T17:02:40.906 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 8m ago 11h 63.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T17:02:40.906 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 25s ago 11h 26.1M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T17:02:40.907 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 8m ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T17:02:40.907 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (11h) 8m ago 11h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T17:02:40.907 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11h) 25s ago 11h 1100M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T17:02:40.908 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11h) 25s ago 11h 2503M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T17:02:40.909 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (11h) 8m ago 11h 1111M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T17:02:40.910 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (11h) 25s ago 11h 1513M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T17:02:40.910 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 25s ago 11h 24.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T17:02:40.911 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 8m ago 11h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T17:02:40.911 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11h) 25s ago 11h 1829M 2021M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T17:02:40.912 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11h) 25s ago 11h 1814M 2021M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T17:02:40.912 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11h) 25s ago 11h 1409M 2021M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T17:02:40.914 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (11h) 25s ago 11h 1779M 2021M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T17:02:40.917 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (11h) 8m ago 11h 1536M 2478M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T17:02:40.917 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (11h) 8m ago 11h 1932M 2478M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T17:02:40.918 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (11h) 8m ago 11h 1726M 2478M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T17:02:40.918 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (11h) 8m ago 11h 2056M 2478M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T17:02:40.918 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (11h) 8m ago 11h 199M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T17:02:41.259 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T17:02:41.259 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T17:02:41.260 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T17:02:41.260 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:02:41.261 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T17:02:41.261 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T17:02:41.261 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:02:41.262 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T17:02:41.262 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T17:02:41.263 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:02:41.264 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T17:02:41.264 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T17:02:41.265 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:02:41.265 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T17:02:41.266 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T17:02:41.266 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T17:02:41.266 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T17:02:41.267 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T17:02:41.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:02:41 smithi153 ceph-mon[70044]: pgmap v20829: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:41.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:02:41 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4217488780' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T17:02:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 17:02:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:02:41] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T17:02:41.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:02:41 smithi083 ceph-mon[77242]: pgmap v20829: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:41.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:02:41 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4217488780' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T17:02:41.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:02:41 smithi083 ceph-mon[79930]: pgmap v20829: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:41.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:02:41 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4217488780' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T17:02:42.783 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:02:42 smithi083 ceph-mon[79930]: from='client.68281 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:02:42.783 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:02:42 smithi083 ceph-mon[79930]: from='client.68287 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:02:42.783 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:02:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T17:02:42.783 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:02:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T17:02:42.784 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:02:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T17:02:42.784 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:02:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T17:02:42.784 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:02:42 smithi083 ceph-mon[77242]: from='client.68281 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:02:42.785 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:02:42 smithi083 ceph-mon[77242]: from='client.68287 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:02:42.785 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:02:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T17:02:42.785 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:02:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T17:02:42.785 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:02:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T17:02:42.786 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:02:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T17:02:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:02:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:02:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T17:02:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:02:42 smithi153 ceph-mon[70044]: from='client.68281 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:02:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:02:42 smithi153 ceph-mon[70044]: from='client.68287 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:02:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:02:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T17:02:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:02:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T17:02:42.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:02:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T17:02:42.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:02:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T17:02:43.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:02:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:02:42.781Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:02:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 17:02:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:02:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T17:02:43.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:02:43 smithi153 ceph-mon[70044]: pgmap v20830: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:43.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:02:43 smithi083 ceph-mon[79930]: pgmap v20830: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:43.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:02:43 smithi083 ceph-mon[77242]: pgmap v20830: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:45.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:02:44 smithi083 ceph-mon[79930]: pgmap v20831: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:45.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:02:44 smithi083 ceph-mon[77242]: pgmap v20831: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:45.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:02:44 smithi153 ceph-mon[70044]: pgmap v20831: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:46.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:02:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:02:46.123Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:02:47.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:02:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:02:46.876Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:02:47.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:02:47 smithi153 ceph-mon[70044]: pgmap v20832: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:47.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:02:47 smithi083 ceph-mon[79930]: pgmap v20832: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:47.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:02:47 smithi083 ceph-mon[77242]: pgmap v20832: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:49.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:02:49 smithi153 ceph-mon[70044]: pgmap v20833: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:49.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:02:49 smithi083 ceph-mon[79930]: pgmap v20833: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:49.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:02:49 smithi083 ceph-mon[77242]: pgmap v20833: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:51.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:02:51 smithi153 ceph-mon[70044]: pgmap v20834: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:51.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 17:02:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:02:51] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T17:02:51.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:02:51 smithi083 ceph-mon[79930]: pgmap v20834: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:51.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:02:51 smithi083 ceph-mon[77242]: pgmap v20834: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:52.810 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:02:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:02:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T17:02:53.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:02:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:02:52.781Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:02:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 17:02:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:02:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T17:02:53.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:02:53 smithi153 ceph-mon[70044]: pgmap v20835: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:53.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:02:53 smithi083 ceph-mon[79930]: pgmap v20835: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:53.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:02:53 smithi083 ceph-mon[77242]: pgmap v20835: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:55.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:02:54 smithi083 ceph-mon[79930]: pgmap v20836: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:55.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:02:54 smithi083 ceph-mon[77242]: pgmap v20836: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:55.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:02:54 smithi153 ceph-mon[70044]: pgmap v20836: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:56.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:02:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:02:56.124Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:02:57.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:02:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:02:56.876Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:02:57.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:02:57 smithi153 ceph-mon[70044]: pgmap v20837: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:57.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:02:57 smithi083 ceph-mon[79930]: pgmap v20837: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:57.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:02:57 smithi083 ceph-mon[77242]: pgmap v20837: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:59.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:02:59 smithi153 ceph-mon[70044]: pgmap v20838: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:59.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:02:59 smithi083 ceph-mon[79930]: pgmap v20838: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:02:59.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:02:59 smithi083 ceph-mon[77242]: pgmap v20838: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:01.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:03:01 smithi153 ceph-mon[70044]: pgmap v20839: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 17:03:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:03:01] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T17:03:01.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:03:01 smithi083 ceph-mon[79930]: pgmap v20839: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:01.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:03:01 smithi083 ceph-mon[77242]: pgmap v20839: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:02.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:03:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:03:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T17:03:03.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:03:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:03:02.782Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:03:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 17:03:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:03:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T17:03:03.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:03:03 smithi153 ceph-mon[70044]: pgmap v20840: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:03.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:03:03 smithi083 ceph-mon[79930]: pgmap v20840: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:03.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:03:03 smithi083 ceph-mon[77242]: pgmap v20840: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:05.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:03:04 smithi083 ceph-mon[79930]: pgmap v20841: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:05.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:03:04 smithi083 ceph-mon[77242]: pgmap v20841: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:05.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:03:04 smithi153 ceph-mon[70044]: pgmap v20841: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:06.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:03:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:03:06.124Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:03:07.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:03:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:03:06.877Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:03:07.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:03:07 smithi153 ceph-mon[70044]: pgmap v20842: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:07.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:03:07 smithi083 ceph-mon[77242]: pgmap v20842: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:07.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:03:07 smithi083 ceph-mon[79930]: pgmap v20842: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:09.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:03:09 smithi153 ceph-mon[70044]: pgmap v20843: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:09.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:03:09 smithi083 ceph-mon[79930]: pgmap v20843: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:09.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:03:09 smithi083 ceph-mon[77242]: pgmap v20843: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:11.591 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T17:03:11.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:03:11 smithi153 ceph-mon[70044]: pgmap v20844: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:11.884 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T17:03:11.884 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 56s ago 11h 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T17:03:11.884 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 9m ago 11h 63.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T17:03:11.885 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 56s ago 11h 26.1M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T17:03:11.885 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 9m ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T17:03:11.885 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (11h) 9m ago 11h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T17:03:11.885 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11h) 56s ago 11h 1100M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T17:03:11.886 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11h) 56s ago 11h 2503M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T17:03:11.886 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (11h) 9m ago 11h 1111M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T17:03:11.886 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (11h) 56s ago 11h 1513M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T17:03:11.886 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 56s ago 11h 24.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T17:03:11.887 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 9m ago 11h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T17:03:11.888 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11h) 56s ago 11h 1829M 2021M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T17:03:11.890 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11h) 56s ago 11h 1814M 2021M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T17:03:11.890 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11h) 56s ago 11h 1409M 2021M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T17:03:11.891 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (11h) 56s ago 11h 1779M 2021M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T17:03:11.891 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (11h) 9m ago 11h 1536M 2478M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T17:03:11.892 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (11h) 9m ago 11h 1932M 2478M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T17:03:11.892 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (11h) 9m ago 11h 1726M 2478M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T17:03:11.893 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (11h) 9m ago 11h 2056M 2478M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T17:03:11.893 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (11h) 9m ago 11h 199M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T17:03:11.970 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 17:03:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:03:11] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T17:03:11.971 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:03:11 smithi083 ceph-mon[77242]: pgmap v20844: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:11.972 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:03:11 smithi083 ceph-mon[79930]: pgmap v20844: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:12.241 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T17:03:12.241 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T17:03:12.242 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T17:03:12.242 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:03:12.242 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T17:03:12.243 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T17:03:12.243 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:03:12.243 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T17:03:12.243 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T17:03:12.244 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:03:12.244 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T17:03:12.244 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T17:03:12.244 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:03:12.244 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T17:03:12.245 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T17:03:12.245 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T17:03:12.245 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T17:03:12.245 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T17:03:12.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:03:12 smithi153 ceph-mon[70044]: from='client.58368 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:03:12.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:03:12 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/522966588' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T17:03:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:03:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:03:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T17:03:12.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:03:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:03:12.782Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:03:12.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:03:12 smithi083 ceph-mon[77242]: from='client.58368 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:03:12.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:03:12 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/522966588' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T17:03:12.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:03:12 smithi083 ceph-mon[79930]: from='client.58368 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:03:12.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:03:12 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/522966588' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T17:03:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 17:03:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:03:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T17:03:13.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:03:13 smithi153 ceph-mon[70044]: from='client.58374 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:03:13.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:03:13 smithi153 ceph-mon[70044]: pgmap v20845: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:13.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:03:13 smithi083 ceph-mon[79930]: from='client.58374 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:03:13.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:03:13 smithi083 ceph-mon[79930]: pgmap v20845: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:13.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:03:13 smithi083 ceph-mon[77242]: from='client.58374 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:03:13.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:03:13 smithi083 ceph-mon[77242]: pgmap v20845: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:15.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:03:14 smithi083 ceph-mon[77242]: pgmap v20846: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:15.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:03:14 smithi083 ceph-mon[79930]: pgmap v20846: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:15.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:03:14 smithi153 ceph-mon[70044]: pgmap v20846: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:16.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:03:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:03:16.125Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:03:16.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:03:16 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T17:03:16.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:03:16 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T17:03:16.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:03:16 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T17:03:16.878 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:03:16 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T17:03:16.880 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:03:16 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T17:03:16.880 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:03:16 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T17:03:16.881 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:03:16 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T17:03:16.881 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:03:16 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T17:03:16.882 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:03:16 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T17:03:17.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:03:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:03:16.877Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:03:17.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:03:17 smithi153 ceph-mon[70044]: pgmap v20847: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:17.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:03:17 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:03:17.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:03:17 smithi083 ceph-mon[77242]: pgmap v20847: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:17.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:03:17 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:03:17.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:03:17 smithi083 ceph-mon[79930]: pgmap v20847: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:17.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:03:17 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:03:19.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:03:19 smithi153 ceph-mon[70044]: pgmap v20848: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:19.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:03:19 smithi083 ceph-mon[79930]: pgmap v20848: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:19.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:03:19 smithi083 ceph-mon[77242]: pgmap v20848: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:21.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:03:21 smithi153 ceph-mon[70044]: pgmap v20849: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 17:03:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:03:21] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T17:03:21.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:03:21 smithi083 ceph-mon[79930]: pgmap v20849: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:21.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:03:21 smithi083 ceph-mon[77242]: pgmap v20849: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:03:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:03:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T17:03:23.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:03:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:03:22.783Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:03:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 17:03:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:03:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T17:03:23.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:03:23 smithi153 ceph-mon[70044]: pgmap v20850: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:23.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:03:23 smithi083 ceph-mon[77242]: pgmap v20850: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:23.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:03:23 smithi083 ceph-mon[79930]: pgmap v20850: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:25.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:03:24 smithi083 ceph-mon[79930]: pgmap v20851: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:25.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:03:24 smithi083 ceph-mon[77242]: pgmap v20851: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:25.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:03:24 smithi153 ceph-mon[70044]: pgmap v20851: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:26.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:03:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:03:26.127Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:03:27.214 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:03:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:03:26.878Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:03:27.815 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:03:27 smithi153 ceph-mon[70044]: pgmap v20852: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:27.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:03:27 smithi083 ceph-mon[79930]: pgmap v20852: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:27.968 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:03:27 smithi083 ceph-mon[77242]: pgmap v20852: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:29.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:03:29 smithi153 ceph-mon[70044]: pgmap v20853: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:29.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:03:29 smithi083 ceph-mon[79930]: pgmap v20853: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:29.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:03:29 smithi083 ceph-mon[77242]: pgmap v20853: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:31.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:03:31 smithi153 ceph-mon[70044]: pgmap v20854: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 17:03:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:03:31] "GET /metrics HTTP/1.1" 200 225246 "" "Prometheus/2.33.4" 2022-09-29T17:03:31.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:03:31 smithi083 ceph-mon[79930]: pgmap v20854: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:31.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:03:31 smithi083 ceph-mon[77242]: pgmap v20854: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:03:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:03:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T17:03:33.215 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:03:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:03:32.784Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:03:33.315 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 17:03:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:03:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T17:03:33.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:03:33 smithi153 ceph-mon[70044]: pgmap v20855: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:33.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:03:33 smithi083 ceph-mon[79930]: pgmap v20855: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:33.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:03:33 smithi083 ceph-mon[77242]: pgmap v20855: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:35.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:03:34 smithi083 ceph-mon[79930]: pgmap v20856: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:35.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:03:34 smithi083 ceph-mon[77242]: pgmap v20856: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:35.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:03:34 smithi153 ceph-mon[70044]: pgmap v20856: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:36.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:03:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:03:36.127Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:03:37.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:03:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:03:36.878Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:03:37.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:03:37 smithi153 ceph-mon[70044]: pgmap v20857: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:37.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:03:37 smithi083 ceph-mon[79930]: pgmap v20857: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:37.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:03:37 smithi083 ceph-mon[77242]: pgmap v20857: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:39.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:03:39 smithi153 ceph-mon[70044]: pgmap v20858: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:39.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:03:39 smithi083 ceph-mon[79930]: pgmap v20858: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:39.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:03:39 smithi083 ceph-mon[77242]: pgmap v20858: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:41.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:03:41 smithi153 ceph-mon[70044]: pgmap v20859: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 17:03:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:03:41] "GET /metrics HTTP/1.1" 200 225247 "" "Prometheus/2.33.4" 2022-09-29T17:03:41.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:03:41 smithi083 ceph-mon[79930]: pgmap v20859: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:41.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:03:41 smithi083 ceph-mon[77242]: pgmap v20859: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:42.569 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T17:03:42.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:03:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:03:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T17:03:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:03:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T17:03:42.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:03:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T17:03:42.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:03:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T17:03:42.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:03:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T17:03:42.860 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T17:03:42.860 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 87s ago 11h 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T17:03:42.861 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 9m ago 11h 63.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T17:03:42.861 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 87s ago 11h 26.1M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T17:03:42.862 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 9m ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T17:03:42.863 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (11h) 9m ago 11h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T17:03:42.863 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11h) 87s ago 11h 1100M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T17:03:42.863 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11h) 87s ago 11h 2503M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T17:03:42.863 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (11h) 9m ago 11h 1111M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T17:03:42.864 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (11h) 87s ago 11h 1513M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T17:03:42.864 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 87s ago 11h 24.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T17:03:42.864 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 9m ago 11h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T17:03:42.864 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11h) 87s ago 11h 1829M 2021M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T17:03:42.865 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11h) 87s ago 11h 1814M 2021M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T17:03:42.865 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11h) 87s ago 11h 1409M 2021M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T17:03:42.865 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (11h) 87s ago 11h 1779M 2021M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T17:03:42.866 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (11h) 9m ago 11h 1536M 2478M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T17:03:42.866 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (11h) 9m ago 11h 1932M 2478M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T17:03:42.866 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (11h) 9m ago 11h 1726M 2478M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T17:03:42.866 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (11h) 9m ago 11h 2056M 2478M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T17:03:42.867 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (11h) 9m ago 11h 199M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T17:03:42.968 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:03:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T17:03:42.968 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:03:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T17:03:42.969 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:03:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T17:03:42.969 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:03:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T17:03:42.970 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:03:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T17:03:42.971 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:03:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T17:03:42.971 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:03:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T17:03:42.972 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:03:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T17:03:42.973 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:03:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:03:42.783Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:03:43.214 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T17:03:43.214 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T17:03:43.215 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T17:03:43.215 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:03:43.215 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T17:03:43.215 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T17:03:43.216 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:03:43.216 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T17:03:43.216 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T17:03:43.217 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:03:43.217 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T17:03:43.217 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T17:03:43.217 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:03:43.218 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T17:03:43.218 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T17:03:43.218 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T17:03:43.218 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T17:03:43.219 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T17:03:43.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 17:03:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:03:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T17:03:43.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:03:43 smithi153 ceph-mon[70044]: pgmap v20860: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:43.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:03:43 smithi153 ceph-mon[70044]: from='client.58386 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:03:43.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:03:43 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1994197796' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T17:03:43.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:03:43 smithi083 ceph-mon[79930]: pgmap v20860: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:43.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:03:43 smithi083 ceph-mon[79930]: from='client.58386 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:03:43.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:03:43 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1994197796' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T17:03:43.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:03:43 smithi083 ceph-mon[77242]: pgmap v20860: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:43.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:03:43 smithi083 ceph-mon[77242]: from='client.58386 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:03:43.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:03:43 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1994197796' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T17:03:44.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:03:44 smithi153 ceph-mon[70044]: from='client.58392 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:03:44.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:03:44 smithi083 ceph-mon[79930]: from='client.58392 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:03:44.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:03:44 smithi083 ceph-mon[77242]: from='client.58392 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:03:45.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:03:45 smithi153 ceph-mon[70044]: pgmap v20861: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:45.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:03:45 smithi083 ceph-mon[79930]: pgmap v20861: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:45.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:03:45 smithi083 ceph-mon[77242]: pgmap v20861: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:46.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:03:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:03:46.128Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:03:47.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:03:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:03:46.879Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:03:47.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:03:47 smithi153 ceph-mon[70044]: pgmap v20862: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:47.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:03:47 smithi083 ceph-mon[79930]: pgmap v20862: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:47.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:03:47 smithi083 ceph-mon[77242]: pgmap v20862: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:49.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:03:49 smithi153 ceph-mon[70044]: pgmap v20863: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:49.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:03:49 smithi083 ceph-mon[77242]: pgmap v20863: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:49.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:03:49 smithi083 ceph-mon[79930]: pgmap v20863: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 17:03:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:03:51] "GET /metrics HTTP/1.1" 200 225247 "" "Prometheus/2.33.4" 2022-09-29T17:03:51.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:03:51 smithi083 ceph-mon[79930]: pgmap v20864: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:51.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:03:51 smithi083 ceph-mon[77242]: pgmap v20864: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:52.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:03:51 smithi153 ceph-mon[70044]: pgmap v20864: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:52.844 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:03:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:03:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T17:03:53.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:03:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:03:52.785Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:03:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 17:03:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:03:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T17:03:53.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:03:53 smithi083 ceph-mon[79930]: pgmap v20865: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:53.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:03:53 smithi083 ceph-mon[77242]: pgmap v20865: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:54.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:03:53 smithi153 ceph-mon[70044]: pgmap v20865: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:55.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:03:54 smithi083 ceph-mon[79930]: pgmap v20866: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:55.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:03:54 smithi083 ceph-mon[77242]: pgmap v20866: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:55.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:03:54 smithi153 ceph-mon[70044]: pgmap v20866: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:56.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:03:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:03:56.128Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:03:57.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:03:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:03:56.880Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:03:57.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:03:57 smithi153 ceph-mon[70044]: pgmap v20867: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:57.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:03:57 smithi083 ceph-mon[79930]: pgmap v20867: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:57.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:03:57 smithi083 ceph-mon[77242]: pgmap v20867: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:59.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:03:59 smithi153 ceph-mon[70044]: pgmap v20868: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:59.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:03:59 smithi083 ceph-mon[79930]: pgmap v20868: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:03:59.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:03:59 smithi083 ceph-mon[77242]: pgmap v20868: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:04:01.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:04:01 smithi153 ceph-mon[70044]: pgmap v20869: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:04:01.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:04:01 smithi083 ceph-mon[77242]: pgmap v20869: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:04:01.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:04:01 smithi083 ceph-mon[79930]: pgmap v20869: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:04:01.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 17:04:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:04:01] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T17:04:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:04:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:04:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T17:04:03.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:04:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:04:02.785Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:04:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 17:04:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:04:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T17:04:03.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:04:03 smithi153 ceph-mon[70044]: pgmap v20870: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:04:03.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:04:03 smithi083 ceph-mon[79930]: pgmap v20870: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:04:03.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:04:03 smithi083 ceph-mon[77242]: pgmap v20870: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:04:05.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:04:04 smithi083 ceph-mon[79930]: pgmap v20871: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:04:05.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:04:04 smithi083 ceph-mon[77242]: pgmap v20871: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:04:05.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:04:04 smithi153 ceph-mon[70044]: pgmap v20871: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:04:06.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:04:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:04:06.128Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:04:07.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:04:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:04:06.880Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:04:07.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:04:07 smithi153 ceph-mon[70044]: pgmap v20872: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:04:07.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:04:07 smithi083 ceph-mon[79930]: pgmap v20872: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:04:07.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:04:07 smithi083 ceph-mon[77242]: pgmap v20872: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:04:09.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:04:09 smithi153 ceph-mon[70044]: pgmap v20873: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:04:09.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:04:09 smithi083 ceph-mon[79930]: pgmap v20873: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:04:09.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:04:09 smithi083 ceph-mon[77242]: pgmap v20873: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:04:11.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:04:11 smithi153 ceph-mon[70044]: pgmap v20874: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:04:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 17:04:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:04:11] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T17:04:11.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:04:11 smithi083 ceph-mon[79930]: pgmap v20874: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:04:11.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:04:11 smithi083 ceph-mon[77242]: pgmap v20874: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:04:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:04:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:04:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T17:04:13.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:04:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:04:12.786Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:04:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 17:04:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:04:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T17:04:13.552 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T17:04:13.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:04:13 smithi153 ceph-mon[70044]: pgmap v20875: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:04:13.848 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T17:04:13.848 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 118s ago 11h 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T17:04:13.849 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 10m ago 11h 63.2M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T17:04:13.849 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 118s ago 11h 26.1M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T17:04:13.849 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 10m ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T17:04:13.849 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (11h) 10m ago 11h 451M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T17:04:13.850 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11h) 118s ago 11h 1100M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T17:04:13.850 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11h) 118s ago 11h 2503M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T17:04:13.850 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (11h) 10m ago 11h 1111M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T17:04:13.851 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (11h) 118s ago 11h 1513M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T17:04:13.851 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 118s ago 11h 24.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T17:04:13.851 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 10m ago 11h 26.0M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T17:04:13.852 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11h) 118s ago 11h 1829M 2021M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T17:04:13.852 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11h) 118s ago 11h 1814M 2021M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T17:04:13.852 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11h) 118s ago 11h 1409M 2021M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T17:04:13.852 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (11h) 118s ago 11h 1779M 2021M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T17:04:13.853 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (11h) 10m ago 11h 1536M 2478M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T17:04:13.853 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (11h) 10m ago 11h 1932M 2478M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T17:04:13.853 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (11h) 10m ago 11h 1726M 2478M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T17:04:13.853 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (11h) 10m ago 11h 2056M 2478M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T17:04:13.854 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (11h) 10m ago 11h 199M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T17:04:13.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:04:13 smithi083 ceph-mon[79930]: pgmap v20875: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:04:13.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:04:13 smithi083 ceph-mon[77242]: pgmap v20875: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:04:14.205 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T17:04:14.206 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T17:04:14.206 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T17:04:14.207 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:04:14.207 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T17:04:14.207 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T17:04:14.207 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:04:14.208 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T17:04:14.208 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T17:04:14.208 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:04:14.208 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T17:04:14.209 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T17:04:14.209 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:04:14.209 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T17:04:14.209 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T17:04:14.210 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T17:04:14.210 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T17:04:14.210 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T17:04:14.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:04:14 smithi153 ceph-mon[70044]: from='client.68335 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:04:14.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:04:14 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4219785139' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T17:04:14.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:04:14 smithi083 ceph-mon[79930]: from='client.68335 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:04:14.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:04:14 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4219785139' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T17:04:14.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:04:14 smithi083 ceph-mon[77242]: from='client.68335 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:04:14.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:04:14 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4219785139' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T17:04:15.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:04:15 smithi153 ceph-mon[70044]: from='client.58410 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:04:15.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:04:15 smithi153 ceph-mon[70044]: pgmap v20876: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:04:15.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:04:15 smithi083 ceph-mon[79930]: from='client.58410 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:04:15.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:04:15 smithi083 ceph-mon[79930]: pgmap v20876: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:04:15.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:04:15 smithi083 ceph-mon[77242]: from='client.58410 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:04:15.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:04:15 smithi083 ceph-mon[77242]: pgmap v20876: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:04:16.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:04:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:04:16.128Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:04:17.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:04:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:04:16.881Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:04:17.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:04:17 smithi153 ceph-mon[70044]: pgmap v20877: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:04:17.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:04:17 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T17:04:17.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:04:17 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T17:04:17.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:04:17 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T17:04:17.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:04:17 smithi083 ceph-mon[77242]: pgmap v20877: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:04:17.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:04:17 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T17:04:17.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:04:17 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T17:04:17.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:04:17 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T17:04:17.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:04:17 smithi083 ceph-mon[79930]: pgmap v20877: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:04:17.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:04:17 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T17:04:17.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:04:17 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T17:04:17.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:04:17 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T17:04:19.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:04:19 smithi083 ceph-mon[79930]: pgmap v20878: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:04:19.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:04:19 smithi083 ceph-mon[77242]: pgmap v20878: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:04:20.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:04:19 smithi153 ceph-mon[70044]: pgmap v20878: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:04:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 17:04:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:04:21] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T17:04:21.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:04:21 smithi083 ceph-mon[79930]: pgmap v20879: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:04:21.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:04:21 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:04:21.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:04:21 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:04:21.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:04:21 smithi083 ceph-mon[77242]: pgmap v20879: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:04:21.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:04:21 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:04:21.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:04:21 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:04:22.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:04:21 smithi153 ceph-mon[70044]: pgmap v20879: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:04:22.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:04:21 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:04:22.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:04:21 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:04:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:04:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:04:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T17:04:23.092 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:04:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:04:22.787Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:04:23.093 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:04:23 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:04:23.094 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:04:23 smithi083 ceph-mon[77242]: pgmap v20880: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:04:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 17:04:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:04:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T17:04:23.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:04:23 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:04:23.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:04:23 smithi153 ceph-mon[70044]: pgmap v20880: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:04:23.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:04:23 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:04:23.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:04:23 smithi083 ceph-mon[79930]: pgmap v20880: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:04:25.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:04:24 smithi083 ceph-mon[77242]: pgmap v20881: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:04:25.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:04:24 smithi083 ceph-mon[79930]: pgmap v20881: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:04:25.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:04:24 smithi153 ceph-mon[70044]: pgmap v20881: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:04:26.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:04:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:04:26.129Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:04:27.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:04:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:04:26.882Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:04:27.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:04:27 smithi153 ceph-mon[70044]: pgmap v20882: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:04:27.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:04:27 smithi083 ceph-mon[77242]: pgmap v20882: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:04:27.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:04:27 smithi083 ceph-mon[79930]: pgmap v20882: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:04:29.817 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:04:29 smithi153 ceph-mon[70044]: pgmap v20883: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:04:29.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:04:29 smithi083 ceph-mon[79930]: pgmap v20883: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:04:29.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:04:29 smithi083 ceph-mon[77242]: pgmap v20883: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:04:31.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:04:31 smithi153 ceph-mon[70044]: pgmap v20884: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:04:31.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:04:31 smithi083 ceph-mon[79930]: pgmap v20884: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:04:31.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:04:31 smithi083 ceph-mon[77242]: pgmap v20884: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:04:31.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 17:04:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:04:31] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T17:04:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:04:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:04:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T17:04:33.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:04:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:04:32.787Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:04:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 17:04:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:04:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T17:04:33.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:04:33 smithi083 ceph-mon[79930]: pgmap v20885: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:04:33.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:04:33 smithi083 ceph-mon[77242]: pgmap v20885: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:04:34.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:04:33 smithi153 ceph-mon[70044]: pgmap v20885: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:04:35.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:04:34 smithi083 ceph-mon[79930]: pgmap v20886: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:04:35.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:04:34 smithi083 ceph-mon[77242]: pgmap v20886: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:04:35.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:04:34 smithi153 ceph-mon[70044]: pgmap v20886: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:04:36.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:04:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:04:36.130Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:04:37.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:04:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:04:36.882Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:04:37.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:04:37 smithi153 ceph-mon[70044]: pgmap v20887: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:04:37.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:04:37 smithi083 ceph-mon[77242]: pgmap v20887: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:04:37.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:04:37 smithi083 ceph-mon[79930]: pgmap v20887: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:04:39.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:04:39 smithi153 ceph-mon[70044]: pgmap v20888: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:04:39.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:04:39 smithi083 ceph-mon[79930]: pgmap v20888: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:04:39.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:04:39 smithi083 ceph-mon[77242]: pgmap v20888: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:04:41.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:04:41 smithi083 ceph-mon[79930]: pgmap v20889: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:04:41.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:04:41 smithi083 ceph-mon[77242]: pgmap v20889: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:04:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 17:04:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:04:41] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T17:04:42.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:04:41 smithi153 ceph-mon[70044]: pgmap v20889: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:04:42.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:04:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T17:04:42.809 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:04:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T17:04:42.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:04:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T17:04:42.810 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:04:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T17:04:42.811 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:04:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:04:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T17:04:42.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:04:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:04:42.788Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:04:42.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:04:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T17:04:42.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:04:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T17:04:42.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:04:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T17:04:42.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:04:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T17:04:42.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:04:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T17:04:42.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:04:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T17:04:42.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:04:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T17:04:42.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:04:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T17:04:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 17:04:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:04:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T17:04:43.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:04:43 smithi083 ceph-mon[79930]: pgmap v20890: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:04:43.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:04:43 smithi083 ceph-mon[77242]: pgmap v20890: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:04:44.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:04:43 smithi153 ceph-mon[70044]: pgmap v20890: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:04:44.531 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T17:04:44.822 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T17:04:44.823 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 2m ago 11h 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T17:04:44.823 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 23s ago 11h 64.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T17:04:44.823 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 2m ago 11h 26.1M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T17:04:44.824 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 23s ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T17:04:44.824 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (11h) 23s ago 11h 452M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T17:04:44.824 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11h) 2m ago 11h 1100M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T17:04:44.825 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11h) 2m ago 11h 2503M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T17:04:44.825 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (11h) 23s ago 11h 1123M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T17:04:44.825 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (11h) 2m ago 11h 1513M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T17:04:44.825 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 2m ago 11h 24.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T17:04:44.826 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 23s ago 11h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T17:04:44.826 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11h) 2m ago 11h 1829M 2021M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T17:04:44.826 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11h) 2m ago 11h 1814M 2021M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T17:04:44.827 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11h) 2m ago 11h 1409M 2021M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T17:04:44.827 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (11h) 2m ago 11h 1779M 2021M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T17:04:44.827 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (11h) 23s ago 11h 1558M 2478M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T17:04:44.828 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (11h) 23s ago 11h 1959M 2478M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T17:04:44.828 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (11h) 23s ago 11h 1752M 2478M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T17:04:44.828 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (11h) 23s ago 11h 2085M 2478M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T17:04:44.829 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (11h) 23s ago 11h 190M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T17:04:45.178 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T17:04:45.179 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T17:04:45.180 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T17:04:45.180 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:04:45.180 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T17:04:45.181 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T17:04:45.181 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:04:45.181 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T17:04:45.181 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T17:04:45.182 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:04:45.182 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T17:04:45.183 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T17:04:45.183 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:04:45.184 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T17:04:45.184 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T17:04:45.185 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T17:04:45.185 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T17:04:45.186 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T17:04:45.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:04:44 smithi083 ceph-mon[79930]: pgmap v20891: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:04:45.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:04:44 smithi083 ceph-mon[79930]: from='client.68353 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:04:45.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:04:44 smithi083 ceph-mon[77242]: pgmap v20891: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:04:45.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:04:44 smithi083 ceph-mon[77242]: from='client.68353 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:04:45.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:04:44 smithi153 ceph-mon[70044]: pgmap v20891: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:04:45.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:04:44 smithi153 ceph-mon[70044]: from='client.68353 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:04:46.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:04:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:04:46.131Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:04:46.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:04:45 smithi083 ceph-mon[77242]: from='client.58428 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:04:46.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:04:45 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2317158075' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T17:04:46.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:04:45 smithi083 ceph-mon[79930]: from='client.58428 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:04:46.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:04:45 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2317158075' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T17:04:46.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:04:45 smithi153 ceph-mon[70044]: from='client.58428 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:04:46.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:04:45 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2317158075' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T17:04:47.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:04:46 smithi083 ceph-mon[79930]: pgmap v20892: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:04:47.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:04:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:04:46.884Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:04:47.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:04:46 smithi083 ceph-mon[77242]: pgmap v20892: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:04:47.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:04:46 smithi153 ceph-mon[70044]: pgmap v20892: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:04:49.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:04:49 smithi153 ceph-mon[70044]: pgmap v20893: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:04:49.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:04:49 smithi083 ceph-mon[79930]: pgmap v20893: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:04:49.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:04:49 smithi083 ceph-mon[77242]: pgmap v20893: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:04:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 17:04:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:04:51] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T17:04:51.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:04:51 smithi083 ceph-mon[79930]: pgmap v20894: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:04:51.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:04:51 smithi083 ceph-mon[77242]: pgmap v20894: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:04:52.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:04:51 smithi153 ceph-mon[70044]: pgmap v20894: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:04:52.812 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:04:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:04:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T17:04:53.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:04:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:04:52.789Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:04:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 17:04:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:04:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T17:04:53.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:04:53 smithi083 ceph-mon[79930]: pgmap v20895: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:04:53.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:04:53 smithi083 ceph-mon[77242]: pgmap v20895: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:04:54.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:04:53 smithi153 ceph-mon[70044]: pgmap v20895: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:04:55.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:04:54 smithi083 ceph-mon[79930]: pgmap v20896: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:04:55.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:04:54 smithi083 ceph-mon[77242]: pgmap v20896: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:04:55.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:04:54 smithi153 ceph-mon[70044]: pgmap v20896: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:04:56.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:04:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:04:56.132Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:04:57.221 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:04:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:04:56.884Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:04:57.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:04:57 smithi083 ceph-mon[79930]: pgmap v20897: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:04:57.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:04:57 smithi083 ceph-mon[77242]: pgmap v20897: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:04:58.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:04:57 smithi153 ceph-mon[70044]: pgmap v20897: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:04:59.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:04:59 smithi083 ceph-mon[79930]: pgmap v20898: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:04:59.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:04:59 smithi083 ceph-mon[77242]: pgmap v20898: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:00.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:04:59 smithi153 ceph-mon[70044]: pgmap v20898: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 17:05:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:05:01] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T17:05:01.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:05:01 smithi083 ceph-mon[77242]: pgmap v20899: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:01.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:05:01 smithi083 ceph-mon[79930]: pgmap v20899: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:02.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:05:01 smithi153 ceph-mon[70044]: pgmap v20899: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:05:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:05:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T17:05:03.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:05:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:05:02.790Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:05:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 17:05:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:05:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T17:05:03.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:05:03 smithi083 ceph-mon[79930]: pgmap v20900: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:03.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:05:03 smithi083 ceph-mon[77242]: pgmap v20900: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:04.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:05:03 smithi153 ceph-mon[70044]: pgmap v20900: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:05.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:05:04 smithi083 ceph-mon[79930]: pgmap v20901: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:05.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:05:04 smithi083 ceph-mon[77242]: pgmap v20901: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:05.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:05:04 smithi153 ceph-mon[70044]: pgmap v20901: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:06.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:05:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:05:06.132Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:05:07.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:05:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:05:06.885Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:05:07.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:05:07 smithi083 ceph-mon[79930]: pgmap v20902: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:07.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:05:07 smithi083 ceph-mon[77242]: pgmap v20902: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:08.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:05:07 smithi153 ceph-mon[70044]: pgmap v20902: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:09.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:05:09 smithi083 ceph-mon[79930]: pgmap v20903: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:09.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:05:09 smithi083 ceph-mon[77242]: pgmap v20903: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:10.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:05:09 smithi153 ceph-mon[70044]: pgmap v20903: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 17:05:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:05:11] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T17:05:11.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:05:11 smithi083 ceph-mon[79930]: pgmap v20904: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:11.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:05:11 smithi083 ceph-mon[77242]: pgmap v20904: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:12.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:05:11 smithi153 ceph-mon[70044]: pgmap v20904: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:05:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:05:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T17:05:13.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:05:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:05:12.790Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:05:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 17:05:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:05:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T17:05:13.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:05:13 smithi083 ceph-mon[79930]: pgmap v20905: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:13.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:05:13 smithi083 ceph-mon[77242]: pgmap v20905: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:14.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:05:13 smithi153 ceph-mon[70044]: pgmap v20905: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:15.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:05:14 smithi083 ceph-mon[79930]: pgmap v20906: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:15.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:05:14 smithi083 ceph-mon[77242]: pgmap v20906: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:15.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:05:14 smithi153 ceph-mon[70044]: pgmap v20906: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:15.505 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T17:05:15.793 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T17:05:15.793 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 2m ago 11h 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T17:05:15.794 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 54s ago 11h 64.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T17:05:15.794 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 2m ago 11h 26.1M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T17:05:15.794 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 54s ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T17:05:15.794 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (11h) 54s ago 11h 452M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T17:05:15.795 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11h) 2m ago 11h 1100M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T17:05:15.795 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11h) 2m ago 11h 2503M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T17:05:15.795 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (11h) 54s ago 11h 1123M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T17:05:15.795 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (11h) 2m ago 11h 1513M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T17:05:15.796 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 2m ago 11h 24.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T17:05:15.796 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 54s ago 11h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T17:05:15.797 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11h) 2m ago 11h 1829M 2021M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T17:05:15.797 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11h) 2m ago 11h 1814M 2021M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T17:05:15.798 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11h) 2m ago 11h 1409M 2021M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T17:05:15.798 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (11h) 2m ago 11h 1779M 2021M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T17:05:15.799 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (11h) 54s ago 11h 1558M 2478M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T17:05:15.799 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (11h) 54s ago 11h 1959M 2478M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T17:05:15.799 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (11h) 54s ago 11h 1752M 2478M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T17:05:15.799 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (11h) 54s ago 11h 2085M 2478M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T17:05:15.799 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (11h) 54s ago 11h 190M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T17:05:16.155 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T17:05:16.156 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T17:05:16.157 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T17:05:16.157 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:05:16.157 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T17:05:16.158 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T17:05:16.158 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:05:16.159 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T17:05:16.159 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T17:05:16.159 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:05:16.160 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T17:05:16.160 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T17:05:16.161 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:05:16.161 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T17:05:16.161 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T17:05:16.161 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T17:05:16.162 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T17:05:16.162 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T17:05:16.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:05:15 smithi083 ceph-mon[79930]: from='client.58440 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:05:16.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:05:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:05:16.133Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:05:16.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:05:15 smithi083 ceph-mon[77242]: from='client.58440 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:05:16.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:05:15 smithi153 ceph-mon[70044]: from='client.58440 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:05:17.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:05:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:05:16.885Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:05:17.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:05:16 smithi083 ceph-mon[77242]: from='client.58446 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:05:17.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:05:16 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3958070992' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T17:05:17.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:05:16 smithi083 ceph-mon[77242]: pgmap v20907: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:17.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:05:16 smithi083 ceph-mon[79930]: from='client.58446 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:05:17.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:05:16 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3958070992' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T17:05:17.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:05:16 smithi083 ceph-mon[79930]: pgmap v20907: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:17.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:05:16 smithi153 ceph-mon[70044]: from='client.58446 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:05:17.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:05:16 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3958070992' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T17:05:17.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:05:16 smithi153 ceph-mon[70044]: pgmap v20907: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:19.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:05:19 smithi083 ceph-mon[79930]: pgmap v20908: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:19.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:05:19 smithi083 ceph-mon[77242]: pgmap v20908: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:20.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:05:19 smithi153 ceph-mon[70044]: pgmap v20908: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 17:05:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:05:21] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T17:05:21.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:05:21 smithi083 ceph-mon[79930]: pgmap v20909: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:21.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:05:21 smithi083 ceph-mon[77242]: pgmap v20909: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:22.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:05:21 smithi153 ceph-mon[70044]: pgmap v20909: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:22.573 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:05:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:05:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T17:05:22.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:05:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:05:22.790Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:05:22.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:05:22 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T17:05:22.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:05:22 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T17:05:22.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:05:22 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T17:05:22.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:05:22 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T17:05:22.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:05:22 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T17:05:22.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:05:22 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T17:05:23.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:05:22 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T17:05:23.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:05:22 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T17:05:23.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:05:22 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T17:05:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 17:05:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:05:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T17:05:23.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:05:23 smithi083 ceph-mon[79930]: pgmap v20910: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:23.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:05:23 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:05:23.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:05:23 smithi083 ceph-mon[77242]: pgmap v20910: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:23.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:05:23 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:05:24.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:05:23 smithi153 ceph-mon[70044]: pgmap v20910: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:24.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:05:23 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:05:25.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:05:24 smithi083 ceph-mon[79930]: pgmap v20911: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:25.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:05:24 smithi083 ceph-mon[77242]: pgmap v20911: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:25.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:05:24 smithi153 ceph-mon[70044]: pgmap v20911: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:26.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:05:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:05:26.134Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:05:27.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:05:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:05:26.886Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:05:27.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:05:27 smithi083 ceph-mon[79930]: pgmap v20912: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:27.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:05:27 smithi083 ceph-mon[77242]: pgmap v20912: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:28.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:05:27 smithi153 ceph-mon[70044]: pgmap v20912: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:29.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:05:29 smithi083 ceph-mon[79930]: pgmap v20913: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:29.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:05:29 smithi083 ceph-mon[77242]: pgmap v20913: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:30.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:05:29 smithi153 ceph-mon[70044]: pgmap v20913: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 17:05:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:05:31] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T17:05:31.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:05:31 smithi083 ceph-mon[79930]: pgmap v20914: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:31.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:05:31 smithi083 ceph-mon[77242]: pgmap v20914: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:32.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:05:31 smithi153 ceph-mon[70044]: pgmap v20914: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:05:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:05:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T17:05:33.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:05:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:05:32.791Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:05:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 17:05:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:05:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T17:05:33.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:05:33 smithi083 ceph-mon[77242]: pgmap v20915: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:33.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:05:33 smithi083 ceph-mon[79930]: pgmap v20915: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:34.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:05:33 smithi153 ceph-mon[70044]: pgmap v20915: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:35.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:05:34 smithi083 ceph-mon[79930]: pgmap v20916: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:35.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:05:34 smithi083 ceph-mon[77242]: pgmap v20916: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:35.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:05:34 smithi153 ceph-mon[70044]: pgmap v20916: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:36.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:05:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:05:36.135Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:05:37.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:05:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:05:36.887Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:05:37.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:05:37 smithi083 ceph-mon[79930]: pgmap v20917: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:37.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:05:37 smithi083 ceph-mon[77242]: pgmap v20917: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:38.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:05:37 smithi153 ceph-mon[70044]: pgmap v20917: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:39.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:05:39 smithi083 ceph-mon[79930]: pgmap v20918: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:39.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:05:39 smithi083 ceph-mon[77242]: pgmap v20918: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:05:39 smithi153 ceph-mon[70044]: pgmap v20918: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:41.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:05:41 smithi083 ceph-mon[77242]: pgmap v20919: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 17:05:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:05:41] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T17:05:41.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:05:41 smithi083 ceph-mon[79930]: pgmap v20919: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:42.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:05:41 smithi153 ceph-mon[70044]: pgmap v20919: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:42.583 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:05:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:05:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T17:05:42.971 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:05:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T17:05:42.972 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:05:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T17:05:42.972 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:05:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T17:05:42.973 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:05:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T17:05:42.974 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:05:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:05:42.792Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:05:42.974 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:05:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T17:05:42.975 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:05:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T17:05:42.975 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:05:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T17:05:42.976 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:05:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T17:05:43.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:05:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T17:05:43.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:05:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T17:05:43.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:05:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T17:05:43.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:05:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T17:05:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 17:05:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:05:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T17:05:43.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:05:43 smithi083 ceph-mon[79930]: pgmap v20920: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:43.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:05:43 smithi083 ceph-mon[77242]: pgmap v20920: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:44.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:05:43 smithi153 ceph-mon[70044]: pgmap v20920: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:45.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:05:44 smithi083 ceph-mon[79930]: pgmap v20921: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:45.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:05:44 smithi083 ceph-mon[77242]: pgmap v20921: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:45.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:05:44 smithi153 ceph-mon[70044]: pgmap v20921: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:46.467 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:05:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:05:46.136Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:05:46.477 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T17:05:46.768 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T17:05:46.769 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 3m ago 11h 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T17:05:46.769 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 85s ago 11h 64.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T17:05:46.769 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 3m ago 11h 26.1M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T17:05:46.769 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 85s ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T17:05:46.770 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (11h) 85s ago 11h 452M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T17:05:46.770 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11h) 3m ago 11h 1100M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T17:05:46.770 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11h) 3m ago 11h 2503M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T17:05:46.770 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (11h) 85s ago 11h 1123M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T17:05:46.771 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (11h) 3m ago 11h 1513M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T17:05:46.771 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 3m ago 11h 24.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T17:05:46.771 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 85s ago 11h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T17:05:46.771 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11h) 3m ago 11h 1829M 2021M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T17:05:46.772 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11h) 3m ago 11h 1814M 2021M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T17:05:46.772 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11h) 3m ago 11h 1409M 2021M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T17:05:46.772 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (11h) 3m ago 11h 1779M 2021M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T17:05:46.772 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (11h) 85s ago 11h 1558M 2478M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T17:05:46.773 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (11h) 85s ago 11h 1959M 2478M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T17:05:46.773 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (11h) 85s ago 11h 1752M 2478M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T17:05:46.773 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (11h) 85s ago 11h 2085M 2478M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T17:05:46.773 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (11h) 85s ago 11h 190M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T17:05:47.125 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T17:05:47.125 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T17:05:47.126 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T17:05:47.126 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:05:47.126 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T17:05:47.126 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T17:05:47.127 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:05:47.127 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T17:05:47.127 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T17:05:47.127 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:05:47.128 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T17:05:47.128 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T17:05:47.128 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:05:47.128 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T17:05:47.128 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T17:05:47.129 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T17:05:47.129 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T17:05:47.129 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T17:05:47.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:05:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:05:46.888Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:05:47.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:05:47 smithi083 ceph-mon[79930]: from='client.58458 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:05:47.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:05:47 smithi083 ceph-mon[79930]: pgmap v20922: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:47.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:05:47 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1901159675' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T17:05:47.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:05:47 smithi083 ceph-mon[77242]: from='client.58458 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:05:47.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:05:47 smithi083 ceph-mon[77242]: pgmap v20922: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:47.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:05:47 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1901159675' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T17:05:48.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:05:47 smithi153 ceph-mon[70044]: from='client.58458 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:05:48.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:05:47 smithi153 ceph-mon[70044]: pgmap v20922: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:48.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:05:47 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1901159675' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T17:05:48.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:05:48 smithi083 ceph-mon[79930]: from='client.68395 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:05:48.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:05:48 smithi083 ceph-mon[77242]: from='client.68395 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:05:49.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:05:48 smithi153 ceph-mon[70044]: from='client.68395 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:05:49.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:05:49 smithi083 ceph-mon[79930]: pgmap v20923: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:49.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:05:49 smithi083 ceph-mon[77242]: pgmap v20923: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:50.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:05:49 smithi153 ceph-mon[70044]: pgmap v20923: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 17:05:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:05:51] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T17:05:51.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:05:51 smithi083 ceph-mon[77242]: pgmap v20924: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:51.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:05:51 smithi083 ceph-mon[79930]: pgmap v20924: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:52.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:05:51 smithi153 ceph-mon[70044]: pgmap v20924: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:05:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:05:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T17:05:53.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:05:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:05:52.793Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:05:53.316 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 17:05:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:05:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T17:05:53.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:05:53 smithi083 ceph-mon[79930]: pgmap v20925: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:53.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:05:53 smithi083 ceph-mon[77242]: pgmap v20925: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:54.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:05:53 smithi153 ceph-mon[70044]: pgmap v20925: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:55.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:05:54 smithi083 ceph-mon[77242]: pgmap v20926: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:55.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:05:54 smithi083 ceph-mon[79930]: pgmap v20926: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:55.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:05:54 smithi153 ceph-mon[70044]: pgmap v20926: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:56.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:05:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:05:56.135Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:05:57.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:05:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:05:56.888Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:05:57.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:05:57 smithi083 ceph-mon[79930]: pgmap v20927: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:57.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:05:57 smithi083 ceph-mon[77242]: pgmap v20927: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:58.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:05:57 smithi153 ceph-mon[70044]: pgmap v20927: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:59.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:05:59 smithi083 ceph-mon[77242]: pgmap v20928: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:05:59.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:05:59 smithi083 ceph-mon[79930]: pgmap v20928: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:00.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:05:59 smithi153 ceph-mon[70044]: pgmap v20928: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 17:06:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:06:01] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T17:06:01.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:06:01 smithi083 ceph-mon[79930]: pgmap v20929: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:01.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:06:01 smithi083 ceph-mon[77242]: pgmap v20929: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:02.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:06:01 smithi153 ceph-mon[70044]: pgmap v20929: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:06:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:06:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T17:06:03.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:06:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:06:02.794Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:06:03.312 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 17:06:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:06:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T17:06:03.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:06:03 smithi083 ceph-mon[77242]: pgmap v20930: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:03.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:06:03 smithi083 ceph-mon[79930]: pgmap v20930: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:04.064 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:06:03 smithi153 ceph-mon[70044]: pgmap v20930: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:05.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:06:04 smithi153 ceph-mon[70044]: pgmap v20931: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:05.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:06:04 smithi083 ceph-mon[79930]: pgmap v20931: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:05.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:06:04 smithi083 ceph-mon[77242]: pgmap v20931: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:06.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:06:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:06:06.136Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:06:07.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:06:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:06:06.889Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:06:07.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:06:07 smithi083 ceph-mon[79930]: pgmap v20932: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:07.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:06:07 smithi083 ceph-mon[77242]: pgmap v20932: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:08.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:06:07 smithi153 ceph-mon[70044]: pgmap v20932: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:09.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:06:09 smithi083 ceph-mon[79930]: pgmap v20933: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:09.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:06:09 smithi083 ceph-mon[77242]: pgmap v20933: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:10.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:06:09 smithi153 ceph-mon[70044]: pgmap v20933: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:11.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:06:11 smithi083 ceph-mon[77242]: pgmap v20934: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:11.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:06:11 smithi083 ceph-mon[79930]: pgmap v20934: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:11.970 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 17:06:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:06:11] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T17:06:12.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:06:11 smithi153 ceph-mon[70044]: pgmap v20934: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:12.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:06:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:06:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T17:06:13.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:06:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:06:12.794Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:06:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 17:06:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:06:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T17:06:13.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:06:13 smithi083 ceph-mon[77242]: pgmap v20935: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:13.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:06:13 smithi083 ceph-mon[79930]: pgmap v20935: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:14.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:06:13 smithi153 ceph-mon[70044]: pgmap v20935: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:15.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:06:14 smithi153 ceph-mon[70044]: pgmap v20936: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:15.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:06:14 smithi083 ceph-mon[77242]: pgmap v20936: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:15.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:06:14 smithi083 ceph-mon[79930]: pgmap v20936: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:16.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:06:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:06:16.137Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:06:17.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:06:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:06:16.889Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:06:17.454 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T17:06:17.745 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T17:06:17.745 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 4m ago 11h 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T17:06:17.747 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 116s ago 11h 64.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T17:06:17.747 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 4m ago 11h 26.1M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T17:06:17.747 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 116s ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T17:06:17.748 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (11h) 116s ago 11h 452M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T17:06:17.748 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11h) 4m ago 11h 1100M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T17:06:17.748 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11h) 4m ago 11h 2503M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T17:06:17.748 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (11h) 116s ago 11h 1123M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T17:06:17.749 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (11h) 4m ago 11h 1513M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T17:06:17.749 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 4m ago 11h 24.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T17:06:17.749 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 116s ago 11h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T17:06:17.749 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11h) 4m ago 11h 1829M 2021M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T17:06:17.750 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11h) 4m ago 11h 1814M 2021M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T17:06:17.750 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11h) 4m ago 11h 1409M 2021M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T17:06:17.750 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (11h) 4m ago 11h 1779M 2021M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T17:06:17.750 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (11h) 116s ago 11h 1558M 2478M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T17:06:17.751 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (11h) 116s ago 11h 1959M 2478M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T17:06:17.751 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (11h) 116s ago 11h 1752M 2478M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T17:06:17.751 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (11h) 116s ago 11h 2085M 2478M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T17:06:17.751 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (11h) 116s ago 11h 190M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T17:06:17.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:06:17 smithi083 ceph-mon[77242]: pgmap v20937: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:17.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:06:17 smithi083 ceph-mon[79930]: pgmap v20937: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:18.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:06:17 smithi153 ceph-mon[70044]: pgmap v20937: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:18.100 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T17:06:18.100 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T17:06:18.100 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T17:06:18.101 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:06:18.101 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T17:06:18.101 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T17:06:18.102 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:06:18.103 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T17:06:18.103 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T17:06:18.104 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:06:18.104 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T17:06:18.105 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T17:06:18.105 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:06:18.105 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T17:06:18.106 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T17:06:18.107 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T17:06:18.107 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T17:06:18.108 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T17:06:18.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:06:18 smithi083 ceph-mon[79930]: from='client.68407 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:06:18.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:06:18 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3345327379' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T17:06:18.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:06:18 smithi083 ceph-mon[77242]: from='client.68407 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:06:18.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:06:18 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3345327379' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T17:06:19.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:06:18 smithi153 ceph-mon[70044]: from='client.68407 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:06:19.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:06:18 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3345327379' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T17:06:19.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:06:19 smithi083 ceph-mon[79930]: from='client.68413 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:06:19.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:06:19 smithi083 ceph-mon[79930]: pgmap v20938: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:19.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:06:19 smithi083 ceph-mon[77242]: from='client.68413 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:06:19.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:06:19 smithi083 ceph-mon[77242]: pgmap v20938: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:20.065 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:06:19 smithi153 ceph-mon[70044]: from='client.68413 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:06:20.066 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:06:19 smithi153 ceph-mon[70044]: pgmap v20938: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:21.966 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 17:06:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:06:21] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T17:06:21.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:06:21 smithi083 ceph-mon[79930]: pgmap v20939: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:21.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:06:21 smithi083 ceph-mon[77242]: pgmap v20939: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:22.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:06:21 smithi153 ceph-mon[70044]: pgmap v20939: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:06:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:06:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T17:06:23.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:06:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:06:22.795Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:06:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 17:06:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:06:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T17:06:23.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:06:23 smithi083 ceph-mon[79930]: pgmap v20940: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:23.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:06:23 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T17:06:23.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:06:23 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T17:06:23.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:06:23 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T17:06:23.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:06:23 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:06:23.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:06:23 smithi083 ceph-mon[77242]: pgmap v20940: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:23.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:06:23 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T17:06:23.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:06:23 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T17:06:23.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:06:23 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T17:06:23.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:06:23 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:06:24.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:06:23 smithi153 ceph-mon[70044]: pgmap v20940: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:24.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:06:23 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T17:06:24.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:06:23 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T17:06:24.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:06:23 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T17:06:24.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:06:23 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:06:25.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:06:24 smithi153 ceph-mon[70044]: pgmap v20941: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:25.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:06:24 smithi083 ceph-mon[79930]: pgmap v20941: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:25.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:06:24 smithi083 ceph-mon[77242]: pgmap v20941: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:26.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:06:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:06:26.137Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:06:27.214 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:06:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:06:26.890Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:06:27.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:06:27 smithi083 ceph-mon[79930]: pgmap v20942: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:27.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:06:27 smithi083 ceph-mon[77242]: pgmap v20942: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:28.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:06:27 smithi153 ceph-mon[70044]: pgmap v20942: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:29.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:06:29 smithi083 ceph-mon[79930]: pgmap v20943: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:29.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:06:29 smithi083 ceph-mon[77242]: pgmap v20943: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:30.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:06:29 smithi153 ceph-mon[70044]: pgmap v20943: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 17:06:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:06:31] "GET /metrics HTTP/1.1" 200 225247 "" "Prometheus/2.33.4" 2022-09-29T17:06:31.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:06:31 smithi083 ceph-mon[79930]: pgmap v20944: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:31.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:06:31 smithi083 ceph-mon[77242]: pgmap v20944: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:32.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:06:31 smithi153 ceph-mon[70044]: pgmap v20944: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:06:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:06:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T17:06:33.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:06:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:06:32.795Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:06:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 17:06:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:06:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T17:06:33.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:06:33 smithi083 ceph-mon[77242]: pgmap v20945: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:33.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:06:33 smithi083 ceph-mon[79930]: pgmap v20945: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:34.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:06:33 smithi153 ceph-mon[70044]: pgmap v20945: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:35.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:06:34 smithi153 ceph-mon[70044]: pgmap v20946: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:35.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:06:34 smithi083 ceph-mon[79930]: pgmap v20946: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:35.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:06:34 smithi083 ceph-mon[77242]: pgmap v20946: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:36.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:06:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:06:36.139Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:06:37.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:06:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:06:36.890Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:06:37.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:06:37 smithi083 ceph-mon[79930]: pgmap v20947: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:37.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:06:37 smithi083 ceph-mon[77242]: pgmap v20947: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:38.068 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:06:37 smithi153 ceph-mon[70044]: pgmap v20947: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:39.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:06:39 smithi083 ceph-mon[79930]: pgmap v20948: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:39.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:06:39 smithi083 ceph-mon[77242]: pgmap v20948: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:06:39 smithi153 ceph-mon[70044]: pgmap v20948: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:41.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:06:41 smithi083 ceph-mon[77242]: pgmap v20949: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:41.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:06:41 smithi083 ceph-mon[79930]: pgmap v20949: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 17:06:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:06:41] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T17:06:42.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:06:41 smithi153 ceph-mon[70044]: pgmap v20949: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:42.600 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:06:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:06:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T17:06:42.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:06:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:06:42.796Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:06:42.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:06:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T17:06:42.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:06:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T17:06:42.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:06:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T17:06:42.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:06:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T17:06:42.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:06:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T17:06:42.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:06:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T17:06:42.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:06:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T17:06:42.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:06:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T17:06:43.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:06:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T17:06:43.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:06:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T17:06:43.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:06:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T17:06:43.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:06:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T17:06:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 17:06:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:06:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T17:06:43.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:06:43 smithi083 ceph-mon[79930]: pgmap v20950: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:43.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:06:43 smithi083 ceph-mon[77242]: pgmap v20950: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:44.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:06:43 smithi153 ceph-mon[70044]: pgmap v20950: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:45.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:06:44 smithi153 ceph-mon[70044]: pgmap v20951: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:45.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:06:44 smithi083 ceph-mon[77242]: pgmap v20951: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:45.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:06:44 smithi083 ceph-mon[79930]: pgmap v20951: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:46.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:06:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:06:46.139Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:06:47.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:06:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:06:46.891Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:06:47.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:06:47 smithi083 ceph-mon[77242]: pgmap v20952: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:47.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:06:47 smithi083 ceph-mon[79930]: pgmap v20952: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:48.067 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:06:47 smithi153 ceph-mon[70044]: pgmap v20952: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:48.425 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T17:06:48.714 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T17:06:48.714 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 4m ago 11h 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T17:06:48.715 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 2m ago 11h 64.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T17:06:48.715 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 4m ago 11h 26.1M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T17:06:48.715 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 2m ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T17:06:48.715 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (11h) 2m ago 11h 452M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T17:06:48.716 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11h) 4m ago 11h 1100M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T17:06:48.716 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11h) 4m ago 11h 2503M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T17:06:48.716 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (11h) 2m ago 11h 1123M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T17:06:48.716 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (11h) 4m ago 11h 1513M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T17:06:48.717 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 4m ago 11h 24.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T17:06:48.717 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 2m ago 11h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T17:06:48.717 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11h) 4m ago 11h 1829M 2021M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T17:06:48.717 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11h) 4m ago 11h 1814M 2021M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T17:06:48.718 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11h) 4m ago 11h 1409M 2021M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T17:06:48.718 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (11h) 4m ago 11h 1779M 2021M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T17:06:48.718 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (11h) 2m ago 11h 1558M 2478M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T17:06:48.718 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (11h) 2m ago 11h 1959M 2478M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T17:06:48.719 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (11h) 2m ago 11h 1752M 2478M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T17:06:48.719 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (11h) 2m ago 11h 2085M 2478M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T17:06:48.719 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (11h) 2m ago 11h 190M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T17:06:49.071 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T17:06:49.072 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T17:06:49.072 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T17:06:49.073 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:06:49.073 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T17:06:49.074 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T17:06:49.074 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:06:49.075 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T17:06:49.075 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T17:06:49.075 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:06:49.076 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T17:06:49.076 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T17:06:49.077 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:06:49.077 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T17:06:49.078 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T17:06:49.078 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T17:06:49.079 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T17:06:49.079 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T17:06:49.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:06:49 smithi083 ceph-mon[79930]: from='client.68425 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:06:49.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:06:49 smithi083 ceph-mon[79930]: pgmap v20953: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:49.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:06:49 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1426730981' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T17:06:49.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:06:49 smithi083 ceph-mon[77242]: from='client.68425 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:06:49.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:06:49 smithi083 ceph-mon[77242]: pgmap v20953: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:49.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:06:49 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1426730981' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T17:06:50.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:06:49 smithi153 ceph-mon[70044]: from='client.68425 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:06:50.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:06:49 smithi153 ceph-mon[70044]: pgmap v20953: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:50.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:06:49 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1426730981' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T17:06:50.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:06:50 smithi083 ceph-mon[79930]: from='client.68431 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:06:50.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:06:50 smithi083 ceph-mon[77242]: from='client.68431 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:06:51.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:06:50 smithi153 ceph-mon[70044]: from='client.68431 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:06:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 17:06:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:06:51] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T17:06:51.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:06:51 smithi083 ceph-mon[79930]: pgmap v20954: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:51.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:06:51 smithi083 ceph-mon[77242]: pgmap v20954: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:52.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:06:51 smithi153 ceph-mon[70044]: pgmap v20954: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:06:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:06:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T17:06:53.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:06:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:06:52.797Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:06:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 17:06:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:06:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T17:06:53.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:06:53 smithi083 ceph-mon[79930]: pgmap v20955: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:53.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:06:53 smithi083 ceph-mon[77242]: pgmap v20955: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:54.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:06:53 smithi153 ceph-mon[70044]: pgmap v20955: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:55.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:06:54 smithi153 ceph-mon[70044]: pgmap v20956: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:55.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:06:54 smithi083 ceph-mon[79930]: pgmap v20956: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:55.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:06:54 smithi083 ceph-mon[77242]: pgmap v20956: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:56.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:06:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:06:56.139Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:06:57.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:06:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:06:56.891Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:06:57.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:06:57 smithi083 ceph-mon[79930]: pgmap v20957: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:57.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:06:57 smithi083 ceph-mon[77242]: pgmap v20957: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:58.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:06:57 smithi153 ceph-mon[70044]: pgmap v20957: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:59.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:06:59 smithi083 ceph-mon[79930]: pgmap v20958: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:06:59.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:06:59 smithi083 ceph-mon[77242]: pgmap v20958: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:00.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:06:59 smithi153 ceph-mon[70044]: pgmap v20958: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:01.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:07:01 smithi083 ceph-mon[79930]: pgmap v20959: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:01.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:07:01 smithi083 ceph-mon[77242]: pgmap v20959: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 17:07:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:07:01] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T17:07:02.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:07:01 smithi153 ceph-mon[70044]: pgmap v20959: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:02.810 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:07:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:07:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T17:07:03.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:07:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:07:02.797Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:07:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 17:07:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:07:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T17:07:03.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:07:03 smithi083 ceph-mon[79930]: pgmap v20960: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:03.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:07:03 smithi083 ceph-mon[77242]: pgmap v20960: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:04.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:07:03 smithi153 ceph-mon[70044]: pgmap v20960: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:05.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:07:04 smithi153 ceph-mon[70044]: pgmap v20961: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:05.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:07:04 smithi083 ceph-mon[77242]: pgmap v20961: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:05.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:07:04 smithi083 ceph-mon[79930]: pgmap v20961: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:06.465 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:07:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:07:06.140Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:07:07.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:07:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:07:06.891Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:07:07.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:07:07 smithi083 ceph-mon[79930]: pgmap v20962: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:07.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:07:07 smithi083 ceph-mon[77242]: pgmap v20962: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:08.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:07:07 smithi153 ceph-mon[70044]: pgmap v20962: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:09.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:07:09 smithi083 ceph-mon[79930]: pgmap v20963: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:09.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:07:09 smithi083 ceph-mon[77242]: pgmap v20963: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:10.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:07:09 smithi153 ceph-mon[70044]: pgmap v20963: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 17:07:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:07:11] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T17:07:11.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:07:11 smithi083 ceph-mon[79930]: pgmap v20964: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:11.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:07:11 smithi083 ceph-mon[77242]: pgmap v20964: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:12.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:07:11 smithi153 ceph-mon[70044]: pgmap v20964: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:07:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:07:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T17:07:13.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:07:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:07:12.798Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:07:13.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 17:07:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:07:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T17:07:13.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:07:13 smithi083 ceph-mon[79930]: pgmap v20965: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:13.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:07:13 smithi083 ceph-mon[77242]: pgmap v20965: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:14.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:07:13 smithi153 ceph-mon[70044]: pgmap v20965: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:15.313 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:07:14 smithi153 ceph-mon[70044]: pgmap v20966: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:15.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:07:14 smithi083 ceph-mon[79930]: pgmap v20966: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:15.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:07:14 smithi083 ceph-mon[77242]: pgmap v20966: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:16.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:07:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:07:16.140Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:07:17.218 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:07:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:07:16.891Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:07:17.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:07:17 smithi083 ceph-mon[79930]: pgmap v20967: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:17.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:07:17 smithi083 ceph-mon[77242]: pgmap v20967: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:18.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:07:17 smithi153 ceph-mon[70044]: pgmap v20967: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:19.398 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T17:07:19.695 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T17:07:19.695 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 5m ago 11h 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T17:07:19.696 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 2m ago 11h 64.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T17:07:19.696 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 5m ago 11h 26.1M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T17:07:19.696 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 2m ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T17:07:19.697 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (11h) 2m ago 11h 452M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T17:07:19.697 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11h) 5m ago 11h 1100M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T17:07:19.703 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11h) 5m ago 11h 2503M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T17:07:19.704 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (11h) 2m ago 11h 1123M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T17:07:19.704 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (11h) 5m ago 11h 1513M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T17:07:19.705 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 5m ago 11h 24.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T17:07:19.705 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 2m ago 11h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T17:07:19.705 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11h) 5m ago 11h 1829M 2021M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T17:07:19.706 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11h) 5m ago 11h 1814M 2021M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T17:07:19.706 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11h) 5m ago 11h 1409M 2021M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T17:07:19.707 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (11h) 5m ago 11h 1779M 2021M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T17:07:19.707 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (11h) 2m ago 11h 1558M 2478M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T17:07:19.708 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (11h) 2m ago 11h 1959M 2478M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T17:07:19.708 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (11h) 2m ago 11h 1752M 2478M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T17:07:19.709 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (11h) 2m ago 11h 2085M 2478M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T17:07:19.709 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (11h) 2m ago 11h 190M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T17:07:19.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:07:19 smithi083 ceph-mon[79930]: pgmap v20968: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:19.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:07:19 smithi083 ceph-mon[77242]: pgmap v20968: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:20.044 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T17:07:20.044 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T17:07:20.044 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T17:07:20.045 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:07:20.045 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T17:07:20.045 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T17:07:20.045 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:07:20.046 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T17:07:20.046 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T17:07:20.046 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:07:20.047 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T17:07:20.047 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T17:07:20.048 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:07:20.048 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T17:07:20.048 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T17:07:20.049 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T17:07:20.049 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T17:07:20.049 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T17:07:20.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:07:19 smithi153 ceph-mon[70044]: pgmap v20968: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:20.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:07:20 smithi083 ceph-mon[79930]: from='client.58512 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:07:20.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:07:20 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2152147571' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T17:07:20.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:07:20 smithi083 ceph-mon[77242]: from='client.58512 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:07:20.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:07:20 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2152147571' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T17:07:21.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:07:20 smithi153 ceph-mon[70044]: from='client.58512 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:07:21.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:07:20 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2152147571' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T17:07:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 17:07:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:07:21] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T17:07:21.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:07:21 smithi083 ceph-mon[79930]: from='client.58518 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:07:21.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:07:21 smithi083 ceph-mon[79930]: pgmap v20969: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:21.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:07:21 smithi083 ceph-mon[77242]: from='client.58518 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:07:21.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:07:21 smithi083 ceph-mon[77242]: pgmap v20969: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:22.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:07:21 smithi153 ceph-mon[70044]: from='client.58518 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:07:22.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:07:21 smithi153 ceph-mon[70044]: pgmap v20969: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:22.810 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:07:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:07:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T17:07:23.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:07:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:07:22.799Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:07:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 17:07:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:07:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T17:07:23.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:07:23 smithi083 ceph-mon[79930]: pgmap v20970: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:23.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:07:23 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T17:07:23.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:07:23 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T17:07:23.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:07:23 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T17:07:23.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:07:23 smithi083 ceph-mon[77242]: pgmap v20970: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:23.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:07:23 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T17:07:23.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:07:23 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T17:07:23.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:07:23 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T17:07:24.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:07:23 smithi153 ceph-mon[70044]: pgmap v20970: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:24.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:07:23 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T17:07:24.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:07:23 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T17:07:24.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:07:23 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T17:07:25.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:07:25 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:07:25.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:07:25 smithi083 ceph-mon[79930]: pgmap v20971: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:25.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:07:25 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:07:25.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:07:25 smithi083 ceph-mon[77242]: pgmap v20971: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:25.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:07:25 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:07:25.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:07:25 smithi153 ceph-mon[70044]: pgmap v20971: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:26.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:07:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:07:26.141Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:07:27.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:07:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:07:26.892Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:07:27.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:07:27 smithi083 ceph-mon[79930]: pgmap v20972: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:27.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:07:27 smithi083 ceph-mon[77242]: pgmap v20972: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:28.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:07:27 smithi153 ceph-mon[70044]: pgmap v20972: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:29.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:07:29 smithi083 ceph-mon[79930]: pgmap v20973: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:29.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:07:29 smithi083 ceph-mon[77242]: pgmap v20973: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:30.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:07:29 smithi153 ceph-mon[70044]: pgmap v20973: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:31.973 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 17:07:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:07:31] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T17:07:31.974 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:07:31 smithi083 ceph-mon[79930]: pgmap v20974: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:31.975 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:07:31 smithi083 ceph-mon[77242]: pgmap v20974: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:32.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:07:31 smithi153 ceph-mon[70044]: pgmap v20974: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:07:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:07:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T17:07:33.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:07:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:07:32.800Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:07:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 17:07:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:07:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T17:07:33.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:07:33 smithi083 ceph-mon[79930]: pgmap v20975: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:33.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:07:33 smithi083 ceph-mon[77242]: pgmap v20975: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:34.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:07:33 smithi153 ceph-mon[70044]: pgmap v20975: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:35.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:07:34 smithi153 ceph-mon[70044]: pgmap v20976: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:35.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:07:34 smithi083 ceph-mon[79930]: pgmap v20976: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:35.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:07:34 smithi083 ceph-mon[77242]: pgmap v20976: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:36.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:07:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:07:36.142Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:07:37.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:07:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:07:36.893Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:07:37.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:07:37 smithi083 ceph-mon[77242]: pgmap v20977: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:37.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:07:37 smithi083 ceph-mon[79930]: pgmap v20977: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:38.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:07:37 smithi153 ceph-mon[70044]: pgmap v20977: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:39.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:07:39 smithi083 ceph-mon[79930]: pgmap v20978: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:39.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:07:39 smithi083 ceph-mon[77242]: pgmap v20978: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:07:39 smithi153 ceph-mon[70044]: pgmap v20978: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 17:07:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:07:41] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T17:07:41.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:07:41 smithi083 ceph-mon[79930]: pgmap v20979: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:41.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:07:41 smithi083 ceph-mon[77242]: pgmap v20979: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:42.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:07:41 smithi153 ceph-mon[70044]: pgmap v20979: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:42.615 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:07:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:07:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T17:07:42.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:07:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T17:07:42.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:07:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T17:07:42.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:07:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T17:07:42.966 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:07:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T17:07:42.967 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:07:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:07:42.800Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:07:42.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:07:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T17:07:42.968 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:07:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T17:07:42.968 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:07:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T17:07:42.969 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:07:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T17:07:43.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:07:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T17:07:43.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:07:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T17:07:43.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:07:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T17:07:43.026 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:07:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T17:07:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 17:07:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:07:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T17:07:43.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:07:43 smithi083 ceph-mon[79930]: pgmap v20980: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:43.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:07:43 smithi083 ceph-mon[77242]: pgmap v20980: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:44.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:07:43 smithi153 ceph-mon[70044]: pgmap v20980: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:45.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:07:44 smithi153 ceph-mon[70044]: pgmap v20981: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:45.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:07:44 smithi083 ceph-mon[79930]: pgmap v20981: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:45.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:07:44 smithi083 ceph-mon[77242]: pgmap v20981: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:46.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:07:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:07:46.142Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:07:47.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:07:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:07:46.894Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:07:47.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:07:47 smithi083 ceph-mon[77242]: pgmap v20982: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:47.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:07:47 smithi083 ceph-mon[79930]: pgmap v20982: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:48.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:07:47 smithi153 ceph-mon[70044]: pgmap v20982: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:49.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:07:49 smithi083 ceph-mon[79930]: pgmap v20983: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:49.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:07:49 smithi083 ceph-mon[77242]: pgmap v20983: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:50.065 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:07:49 smithi153 ceph-mon[70044]: pgmap v20983: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:50.369 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T17:07:50.661 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T17:07:50.661 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 5m ago 11h 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T17:07:50.661 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 3m ago 11h 64.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T17:07:50.661 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 5m ago 11h 26.1M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T17:07:50.662 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 3m ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T17:07:50.662 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (11h) 3m ago 11h 452M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T17:07:50.662 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11h) 5m ago 11h 1100M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T17:07:50.662 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11h) 5m ago 11h 2503M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T17:07:50.663 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (11h) 3m ago 11h 1123M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T17:07:50.663 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (11h) 5m ago 11h 1513M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T17:07:50.663 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 5m ago 11h 24.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T17:07:50.663 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 3m ago 11h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T17:07:50.664 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11h) 5m ago 11h 1829M 2021M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T17:07:50.664 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11h) 5m ago 11h 1814M 2021M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T17:07:50.664 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11h) 5m ago 11h 1409M 2021M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T17:07:50.665 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (11h) 5m ago 11h 1779M 2021M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T17:07:50.665 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (11h) 3m ago 11h 1558M 2478M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T17:07:50.665 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (11h) 3m ago 11h 1959M 2478M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T17:07:50.665 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (11h) 3m ago 11h 1752M 2478M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T17:07:50.666 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (11h) 3m ago 11h 2085M 2478M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T17:07:50.666 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (11h) 3m ago 11h 190M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T17:07:51.017 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T17:07:51.017 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T17:07:51.018 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T17:07:51.018 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:07:51.019 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T17:07:51.019 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T17:07:51.020 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:07:51.020 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T17:07:51.021 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T17:07:51.021 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:07:51.021 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T17:07:51.022 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T17:07:51.022 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:07:51.023 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T17:07:51.023 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T17:07:51.024 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T17:07:51.024 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T17:07:51.025 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T17:07:51.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:07:51 smithi083 ceph-mon[77242]: from='client.68461 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:07:51.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:07:51 smithi083 ceph-mon[77242]: pgmap v20984: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:51.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:07:51 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/542154610' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T17:07:51.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:07:51 smithi083 ceph-mon[79930]: from='client.68461 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:07:51.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:07:51 smithi083 ceph-mon[79930]: pgmap v20984: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:51.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:07:51 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/542154610' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T17:07:51.964 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 17:07:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:07:51] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T17:07:52.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:07:51 smithi153 ceph-mon[70044]: from='client.68461 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:07:52.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:07:51 smithi153 ceph-mon[70044]: pgmap v20984: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:52.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:07:51 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/542154610' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T17:07:52.621 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:07:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:07:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T17:07:52.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:07:52 smithi083 ceph-mon[79930]: from='client.58536 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:07:52.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:07:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:07:52.801Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:07:52.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:07:52 smithi083 ceph-mon[77242]: from='client.58536 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:07:53.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:07:52 smithi153 ceph-mon[70044]: from='client.58536 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:07:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 17:07:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:07:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T17:07:53.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:07:53 smithi083 ceph-mon[79930]: pgmap v20985: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:53.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:07:53 smithi083 ceph-mon[77242]: pgmap v20985: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:54.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:07:53 smithi153 ceph-mon[70044]: pgmap v20985: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:55.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:07:54 smithi153 ceph-mon[70044]: pgmap v20986: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:55.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:07:54 smithi083 ceph-mon[79930]: pgmap v20986: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:55.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:07:54 smithi083 ceph-mon[77242]: pgmap v20986: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:56.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:07:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:07:56.142Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:07:57.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:07:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:07:56.894Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:07:57.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:07:57 smithi083 ceph-mon[77242]: pgmap v20987: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:57.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:07:57 smithi083 ceph-mon[79930]: pgmap v20987: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:58.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:07:57 smithi153 ceph-mon[70044]: pgmap v20987: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:59.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:07:59 smithi083 ceph-mon[79930]: pgmap v20988: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:07:59.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:07:59 smithi083 ceph-mon[77242]: pgmap v20988: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:00.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:07:59 smithi153 ceph-mon[70044]: pgmap v20988: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:01.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:08:01 smithi083 ceph-mon[79930]: pgmap v20989: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:01.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:08:01 smithi083 ceph-mon[77242]: pgmap v20989: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:01.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 17:08:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:08:01] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T17:08:02.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:08:01 smithi153 ceph-mon[70044]: pgmap v20989: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:08:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:08:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T17:08:03.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:08:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:08:02.802Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:08:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 17:08:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:08:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T17:08:03.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:08:03 smithi083 ceph-mon[79930]: pgmap v20990: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:03.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:08:03 smithi083 ceph-mon[77242]: pgmap v20990: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:04.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:08:03 smithi153 ceph-mon[70044]: pgmap v20990: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:05.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:08:04 smithi153 ceph-mon[70044]: pgmap v20991: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:05.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:08:04 smithi083 ceph-mon[79930]: pgmap v20991: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:05.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:08:04 smithi083 ceph-mon[77242]: pgmap v20991: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:06.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:08:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:08:06.143Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:08:07.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:08:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:08:06.895Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:08:07.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:08:07 smithi083 ceph-mon[79930]: pgmap v20992: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:07.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:08:07 smithi083 ceph-mon[77242]: pgmap v20992: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:08.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:08:07 smithi153 ceph-mon[70044]: pgmap v20992: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:09.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:08:09 smithi083 ceph-mon[79930]: pgmap v20993: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:09.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:08:09 smithi083 ceph-mon[77242]: pgmap v20993: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:10.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:08:09 smithi153 ceph-mon[70044]: pgmap v20993: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 17:08:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:08:11] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T17:08:11.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:08:11 smithi083 ceph-mon[79930]: pgmap v20994: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:11.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:08:11 smithi083 ceph-mon[77242]: pgmap v20994: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:12.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:08:11 smithi153 ceph-mon[70044]: pgmap v20994: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:08:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:08:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T17:08:13.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:08:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:08:12.803Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:08:13.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 17:08:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:08:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T17:08:13.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:08:13 smithi083 ceph-mon[79930]: pgmap v20995: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:13.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:08:13 smithi083 ceph-mon[77242]: pgmap v20995: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:14.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:08:13 smithi153 ceph-mon[70044]: pgmap v20995: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:15.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:08:14 smithi153 ceph-mon[70044]: pgmap v20996: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:15.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:08:14 smithi083 ceph-mon[79930]: pgmap v20996: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:15.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:08:14 smithi083 ceph-mon[77242]: pgmap v20996: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:16.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:08:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:08:16.144Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:08:17.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:08:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:08:16.895Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:08:17.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:08:17 smithi083 ceph-mon[77242]: pgmap v20997: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:17.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:08:17 smithi083 ceph-mon[79930]: pgmap v20997: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:18.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:08:17 smithi153 ceph-mon[70044]: pgmap v20997: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:19.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:08:19 smithi083 ceph-mon[79930]: pgmap v20998: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:19.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:08:19 smithi083 ceph-mon[77242]: pgmap v20998: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:20.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:08:19 smithi153 ceph-mon[70044]: pgmap v20998: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:21.342 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T17:08:21.634 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T17:08:21.635 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 6m ago 11h 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T17:08:21.635 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 4m ago 11h 64.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T17:08:21.635 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 6m ago 11h 26.1M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T17:08:21.635 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 4m ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T17:08:21.636 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (11h) 4m ago 11h 452M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T17:08:21.636 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11h) 6m ago 11h 1100M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T17:08:21.636 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11h) 6m ago 11h 2503M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T17:08:21.636 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (11h) 4m ago 11h 1123M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T17:08:21.637 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (11h) 6m ago 11h 1513M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T17:08:21.637 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 6m ago 11h 24.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T17:08:21.637 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 4m ago 11h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T17:08:21.637 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11h) 6m ago 11h 1829M 2021M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T17:08:21.638 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11h) 6m ago 11h 1814M 2021M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T17:08:21.638 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11h) 6m ago 11h 1409M 2021M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T17:08:21.639 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (11h) 6m ago 11h 1779M 2021M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T17:08:21.639 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (11h) 4m ago 11h 1558M 2478M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T17:08:21.639 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (11h) 4m ago 11h 1959M 2478M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T17:08:21.639 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (11h) 4m ago 11h 1752M 2478M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T17:08:21.640 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (11h) 4m ago 11h 2085M 2478M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T17:08:21.640 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (11h) 4m ago 11h 190M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T17:08:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 17:08:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:08:21] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T17:08:21.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:08:21 smithi083 ceph-mon[79930]: pgmap v20999: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:21.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:08:21 smithi083 ceph-mon[77242]: pgmap v20999: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:21.990 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T17:08:21.991 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T17:08:21.991 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T17:08:21.992 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:08:21.992 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T17:08:21.993 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T17:08:21.994 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:08:21.994 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T17:08:21.995 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T17:08:21.996 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:08:21.996 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T17:08:21.997 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T17:08:21.997 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:08:21.997 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T17:08:21.998 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T17:08:21.998 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T17:08:21.999 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T17:08:21.999 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T17:08:22.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:08:21 smithi153 ceph-mon[70044]: pgmap v20999: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:22.634 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:08:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:08:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T17:08:22.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:08:22 smithi083 ceph-mon[79930]: from='client.58548 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:08:22.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:08:22 smithi083 ceph-mon[79930]: from='client.68485 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:08:22.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:08:22 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/865084125' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T17:08:22.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:08:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:08:22.802Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:08:22.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:08:22 smithi083 ceph-mon[77242]: from='client.58548 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:08:22.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:08:22 smithi083 ceph-mon[77242]: from='client.68485 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:08:22.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:08:22 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/865084125' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T17:08:23.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:08:22 smithi153 ceph-mon[70044]: from='client.58548 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:08:23.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:08:22 smithi153 ceph-mon[70044]: from='client.68485 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:08:23.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:08:22 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/865084125' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T17:08:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 17:08:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:08:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T17:08:23.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:08:23 smithi083 ceph-mon[79930]: pgmap v21000: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:23.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:08:23 smithi083 ceph-mon[77242]: pgmap v21000: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:24.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:08:23 smithi153 ceph-mon[70044]: pgmap v21000: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:25.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:08:24 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T17:08:25.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:08:24 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T17:08:25.312 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:08:24 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T17:08:25.312 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:08:24 smithi153 ceph-mon[70044]: pgmap v21001: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:25.313 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:08:24 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:08:25.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:08:24 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T17:08:25.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:08:24 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T17:08:25.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:08:24 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T17:08:25.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:08:24 smithi083 ceph-mon[77242]: pgmap v21001: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:25.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:08:24 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:08:25.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:08:24 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T17:08:25.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:08:24 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T17:08:25.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:08:24 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T17:08:25.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:08:24 smithi083 ceph-mon[79930]: pgmap v21001: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:25.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:08:24 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:08:26.463 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:08:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:08:26.144Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:08:27.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:08:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:08:26.896Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:08:27.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:08:27 smithi083 ceph-mon[77242]: pgmap v21002: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:27.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:08:27 smithi083 ceph-mon[79930]: pgmap v21002: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:28.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:08:27 smithi153 ceph-mon[70044]: pgmap v21002: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:29.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:08:29 smithi083 ceph-mon[79930]: pgmap v21003: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:29.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:08:29 smithi083 ceph-mon[77242]: pgmap v21003: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:30.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:08:29 smithi153 ceph-mon[70044]: pgmap v21003: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 17:08:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:08:31] "GET /metrics HTTP/1.1" 200 225239 "" "Prometheus/2.33.4" 2022-09-29T17:08:31.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:08:31 smithi083 ceph-mon[79930]: pgmap v21004: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:31.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:08:31 smithi083 ceph-mon[77242]: pgmap v21004: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:32.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:08:31 smithi153 ceph-mon[70044]: pgmap v21004: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:08:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:08:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T17:08:33.215 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:08:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:08:32.804Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:08:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 17:08:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:08:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T17:08:33.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:08:33 smithi083 ceph-mon[79930]: pgmap v21005: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:33.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:08:33 smithi083 ceph-mon[77242]: pgmap v21005: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:34.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:08:33 smithi153 ceph-mon[70044]: pgmap v21005: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:35.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:08:34 smithi153 ceph-mon[70044]: pgmap v21006: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:35.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:08:34 smithi083 ceph-mon[79930]: pgmap v21006: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:35.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:08:34 smithi083 ceph-mon[77242]: pgmap v21006: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:36.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:08:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:08:36.145Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:08:37.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:08:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:08:36.896Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:08:37.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:08:37 smithi083 ceph-mon[77242]: pgmap v21007: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:37.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:08:37 smithi083 ceph-mon[79930]: pgmap v21007: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:38.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:08:37 smithi153 ceph-mon[70044]: pgmap v21007: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:39.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:08:39 smithi083 ceph-mon[79930]: pgmap v21008: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:39.971 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:08:39 smithi083 ceph-mon[77242]: pgmap v21008: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:08:39 smithi153 ceph-mon[70044]: pgmap v21008: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 17:08:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:08:41] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T17:08:41.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:08:41 smithi083 ceph-mon[79930]: pgmap v21009: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:41.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:08:41 smithi083 ceph-mon[77242]: pgmap v21009: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:42.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:08:41 smithi153 ceph-mon[70044]: pgmap v21009: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:42.639 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:08:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:08:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T17:08:42.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:08:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:08:42.804Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:08:42.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:08:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T17:08:42.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:08:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T17:08:42.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:08:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T17:08:42.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:08:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T17:08:42.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:08:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T17:08:42.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:08:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T17:08:42.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:08:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T17:08:42.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:08:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T17:08:43.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:08:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T17:08:43.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:08:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T17:08:43.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:08:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T17:08:43.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:08:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T17:08:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 17:08:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:08:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T17:08:43.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:08:43 smithi083 ceph-mon[79930]: pgmap v21010: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:43.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:08:43 smithi083 ceph-mon[77242]: pgmap v21010: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:44.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:08:43 smithi153 ceph-mon[70044]: pgmap v21010: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:45.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:08:44 smithi153 ceph-mon[70044]: pgmap v21011: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:45.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:08:44 smithi083 ceph-mon[79930]: pgmap v21011: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:45.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:08:44 smithi083 ceph-mon[77242]: pgmap v21011: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:46.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:08:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:08:46.145Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:08:47.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:08:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:08:46.897Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:08:47.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:08:47 smithi083 ceph-mon[79930]: pgmap v21012: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:47.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:08:47 smithi083 ceph-mon[77242]: pgmap v21012: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:48.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:08:47 smithi153 ceph-mon[70044]: pgmap v21012: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:49.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:08:49 smithi083 ceph-mon[79930]: pgmap v21013: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:49.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:08:49 smithi083 ceph-mon[77242]: pgmap v21013: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:50.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:08:49 smithi153 ceph-mon[70044]: pgmap v21013: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 17:08:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:08:51] "GET /metrics HTTP/1.1" 200 225238 "" "Prometheus/2.33.4" 2022-09-29T17:08:51.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:08:51 smithi083 ceph-mon[79930]: pgmap v21014: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:51.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:08:51 smithi083 ceph-mon[77242]: pgmap v21014: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:52.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:08:51 smithi153 ceph-mon[70044]: pgmap v21014: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:52.313 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T17:08:52.604 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T17:08:52.604 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 6m ago 11h 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T17:08:52.605 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 4m ago 11h 64.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T17:08:52.605 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 6m ago 11h 26.1M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T17:08:52.606 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 4m ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T17:08:52.606 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (11h) 4m ago 11h 452M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T17:08:52.607 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11h) 6m ago 11h 1100M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T17:08:52.607 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11h) 6m ago 11h 2503M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T17:08:52.607 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (11h) 4m ago 11h 1123M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T17:08:52.608 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (11h) 6m ago 11h 1513M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T17:08:52.608 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 6m ago 11h 24.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T17:08:52.609 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 4m ago 11h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T17:08:52.609 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11h) 6m ago 11h 1829M 2021M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T17:08:52.610 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11h) 6m ago 11h 1814M 2021M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T17:08:52.610 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11h) 6m ago 11h 1409M 2021M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T17:08:52.611 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (11h) 6m ago 11h 1779M 2021M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T17:08:52.611 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (11h) 4m ago 11h 1558M 2478M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T17:08:52.612 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (11h) 4m ago 11h 1959M 2478M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T17:08:52.612 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (11h) 4m ago 11h 1752M 2478M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T17:08:52.612 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (11h) 4m ago 11h 2085M 2478M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T17:08:52.613 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (11h) 4m ago 11h 190M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T17:08:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:08:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:08:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T17:08:52.958 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T17:08:52.959 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T17:08:52.959 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T17:08:52.959 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:08:52.959 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T17:08:52.960 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T17:08:52.960 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:08:52.960 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T17:08:52.960 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T17:08:52.961 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:08:52.961 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T17:08:52.961 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T17:08:52.962 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:08:52.962 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T17:08:52.962 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T17:08:52.963 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T17:08:52.963 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T17:08:52.964 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T17:08:53.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:08:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:08:52.804Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:08:53.312 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 17:08:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:08:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T17:08:53.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:08:53 smithi083 ceph-mon[79930]: from='client.58566 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:08:53.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:08:53 smithi083 ceph-mon[79930]: pgmap v21015: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:53.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:08:53 smithi083 ceph-mon[79930]: from='client.68503 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:08:53.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:08:53 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/533289671' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T17:08:53.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:08:53 smithi083 ceph-mon[77242]: from='client.58566 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:08:53.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:08:53 smithi083 ceph-mon[77242]: pgmap v21015: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:53.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:08:53 smithi083 ceph-mon[77242]: from='client.68503 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:08:53.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:08:53 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/533289671' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T17:08:54.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:08:53 smithi153 ceph-mon[70044]: from='client.58566 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:08:54.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:08:53 smithi153 ceph-mon[70044]: pgmap v21015: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:54.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:08:53 smithi153 ceph-mon[70044]: from='client.68503 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:08:54.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:08:53 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/533289671' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T17:08:55.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:08:54 smithi153 ceph-mon[70044]: pgmap v21016: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:55.469 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:08:54 smithi083 ceph-mon[79930]: pgmap v21016: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:55.470 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:08:54 smithi083 ceph-mon[77242]: pgmap v21016: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:56.464 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:08:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:08:56.146Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:08:57.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:08:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:08:56.897Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:08:57.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:08:57 smithi083 ceph-mon[79930]: pgmap v21017: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:57.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:08:57 smithi083 ceph-mon[77242]: pgmap v21017: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:58.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:08:57 smithi153 ceph-mon[70044]: pgmap v21017: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:59.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:08:59 smithi083 ceph-mon[79930]: pgmap v21018: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:08:59.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:08:59 smithi083 ceph-mon[77242]: pgmap v21018: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:00.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:08:59 smithi153 ceph-mon[70044]: pgmap v21018: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 17:09:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:09:01] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T17:09:01.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:09:01 smithi083 ceph-mon[79930]: pgmap v21019: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:01.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:09:01 smithi083 ceph-mon[77242]: pgmap v21019: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:02.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:09:01 smithi153 ceph-mon[70044]: pgmap v21019: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:09:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:09:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T17:09:03.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:09:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:09:02.805Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:09:03.309 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 17:09:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:09:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T17:09:03.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:09:03 smithi083 ceph-mon[79930]: pgmap v21020: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:03.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:09:03 smithi083 ceph-mon[77242]: pgmap v21020: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:04.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:09:03 smithi153 ceph-mon[70044]: pgmap v21020: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:05.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:09:05 smithi153 ceph-mon[70044]: pgmap v21021: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:05.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:09:05 smithi083 ceph-mon[79930]: pgmap v21021: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:05.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:09:05 smithi083 ceph-mon[77242]: pgmap v21021: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:06.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:09:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:09:06.146Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:09:07.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:09:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:09:06.897Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:09:07.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:09:07 smithi083 ceph-mon[77242]: pgmap v21022: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:07.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:09:07 smithi083 ceph-mon[79930]: pgmap v21022: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:08.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:09:07 smithi153 ceph-mon[70044]: pgmap v21022: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:09.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:09:09 smithi083 ceph-mon[79930]: pgmap v21023: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:09.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:09:09 smithi083 ceph-mon[77242]: pgmap v21023: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:10.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:09:09 smithi153 ceph-mon[70044]: pgmap v21023: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 17:09:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:09:11] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T17:09:11.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:09:11 smithi083 ceph-mon[79930]: pgmap v21024: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:11.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:09:11 smithi083 ceph-mon[77242]: pgmap v21024: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:12.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:09:11 smithi153 ceph-mon[70044]: pgmap v21024: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:09:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:09:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T17:09:13.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:09:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:09:12.805Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:09:13.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 17:09:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:09:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T17:09:13.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:09:13 smithi083 ceph-mon[79930]: pgmap v21025: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:13.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:09:13 smithi083 ceph-mon[77242]: pgmap v21025: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:14.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:09:13 smithi153 ceph-mon[70044]: pgmap v21025: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:15.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:09:15 smithi153 ceph-mon[70044]: pgmap v21026: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:15.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:09:15 smithi083 ceph-mon[79930]: pgmap v21026: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:15.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:09:15 smithi083 ceph-mon[77242]: pgmap v21026: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:16.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:09:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:09:16.147Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:09:17.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:09:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:09:16.899Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:09:17.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:09:17 smithi083 ceph-mon[77242]: pgmap v21027: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:17.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:09:17 smithi083 ceph-mon[79930]: pgmap v21027: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:18.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:09:17 smithi153 ceph-mon[70044]: pgmap v21027: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:19.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:09:19 smithi083 ceph-mon[77242]: pgmap v21028: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:19.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:09:19 smithi083 ceph-mon[79930]: pgmap v21028: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:20.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:09:19 smithi153 ceph-mon[70044]: pgmap v21028: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 17:09:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:09:21] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T17:09:21.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:09:21 smithi083 ceph-mon[77242]: pgmap v21029: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:21.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:09:21 smithi083 ceph-mon[79930]: pgmap v21029: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:22.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:09:21 smithi153 ceph-mon[70044]: pgmap v21029: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:22.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:09:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:09:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T17:09:23.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:09:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:09:22.806Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:09:23.298 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T17:09:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 17:09:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:09:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T17:09:23.579 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T17:09:23.580 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 7m ago 11h 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T17:09:23.582 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 5m ago 11h 64.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T17:09:23.582 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 7m ago 11h 26.1M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T17:09:23.583 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 5m ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T17:09:23.583 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (11h) 5m ago 11h 452M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T17:09:23.584 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11h) 7m ago 11h 1100M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T17:09:23.584 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11h) 7m ago 11h 2503M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T17:09:23.585 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (11h) 5m ago 11h 1123M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T17:09:23.585 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (11h) 7m ago 11h 1513M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T17:09:23.586 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 7m ago 11h 24.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T17:09:23.586 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 5m ago 11h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T17:09:23.587 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11h) 7m ago 11h 1829M 2021M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T17:09:23.587 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11h) 7m ago 11h 1814M 2021M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T17:09:23.588 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11h) 7m ago 11h 1409M 2021M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T17:09:23.588 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (11h) 7m ago 11h 1779M 2021M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T17:09:23.589 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (11h) 5m ago 11h 1558M 2478M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T17:09:23.589 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (11h) 5m ago 11h 1959M 2478M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T17:09:23.589 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (11h) 5m ago 11h 1752M 2478M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T17:09:23.590 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (11h) 5m ago 11h 2085M 2478M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T17:09:23.590 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (11h) 5m ago 11h 190M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T17:09:23.940 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T17:09:23.940 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T17:09:23.941 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T17:09:23.941 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:09:23.941 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T17:09:23.941 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T17:09:23.942 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:09:23.942 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T17:09:23.942 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T17:09:23.942 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:09:23.943 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T17:09:23.943 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T17:09:23.943 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:09:23.943 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T17:09:23.944 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T17:09:23.944 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T17:09:23.944 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T17:09:23.944 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T17:09:23.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:09:23 smithi083 ceph-mon[79930]: pgmap v21030: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:23.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:09:23 smithi083 ceph-mon[77242]: pgmap v21030: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:24.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:09:23 smithi153 ceph-mon[70044]: pgmap v21030: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:24.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:09:24 smithi083 ceph-mon[79930]: from='client.68515 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:09:24.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:09:24 smithi083 ceph-mon[79930]: from='client.58590 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:09:24.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:09:24 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1743784810' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T17:09:24.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:09:24 smithi083 ceph-mon[77242]: from='client.68515 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:09:24.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:09:24 smithi083 ceph-mon[77242]: from='client.58590 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:09:24.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:09:24 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1743784810' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T17:09:25.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:09:24 smithi153 ceph-mon[70044]: from='client.68515 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:09:25.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:09:24 smithi153 ceph-mon[70044]: from='client.58590 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:09:25.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:09:24 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1743784810' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T17:09:25.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:09:25 smithi083 ceph-mon[79930]: pgmap v21031: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:25.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:09:25 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T17:09:25.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:09:25 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T17:09:25.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:09:25 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T17:09:25.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:09:25 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:09:25.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:09:25 smithi083 ceph-mon[77242]: pgmap v21031: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:25.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:09:25 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T17:09:25.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:09:25 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T17:09:25.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:09:25 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T17:09:25.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:09:25 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:09:26.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:09:25 smithi153 ceph-mon[70044]: pgmap v21031: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:26.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:09:25 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T17:09:26.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:09:25 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T17:09:26.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:09:25 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T17:09:26.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:09:25 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:09:26.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:09:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:09:26.147Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:09:27.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:09:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:09:26.899Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:09:27.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:09:27 smithi083 ceph-mon[79930]: pgmap v21032: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:27.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:09:27 smithi083 ceph-mon[77242]: pgmap v21032: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:28.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:09:27 smithi153 ceph-mon[70044]: pgmap v21032: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:29.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:09:29 smithi083 ceph-mon[77242]: pgmap v21033: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:29.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:09:29 smithi083 ceph-mon[79930]: pgmap v21033: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:30.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:09:29 smithi153 ceph-mon[70044]: pgmap v21033: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 17:09:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:09:31] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T17:09:31.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:09:31 smithi083 ceph-mon[79930]: pgmap v21034: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:31.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:09:31 smithi083 ceph-mon[77242]: pgmap v21034: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:32.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:09:31 smithi153 ceph-mon[70044]: pgmap v21034: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:09:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:09:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T17:09:33.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:09:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:09:32.807Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:09:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 17:09:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:09:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T17:09:33.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:09:33 smithi083 ceph-mon[79930]: pgmap v21035: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:33.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:09:33 smithi083 ceph-mon[77242]: pgmap v21035: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:34.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:09:33 smithi153 ceph-mon[70044]: pgmap v21035: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:35.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:09:35 smithi153 ceph-mon[70044]: pgmap v21036: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:35.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:09:35 smithi083 ceph-mon[79930]: pgmap v21036: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:35.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:09:35 smithi083 ceph-mon[77242]: pgmap v21036: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:36.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:09:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:09:36.148Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:09:37.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:09:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:09:36.899Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:09:37.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:09:37 smithi083 ceph-mon[79930]: pgmap v21037: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:37.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:09:37 smithi083 ceph-mon[77242]: pgmap v21037: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:38.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:09:37 smithi153 ceph-mon[70044]: pgmap v21037: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:39.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:09:39 smithi083 ceph-mon[77242]: pgmap v21038: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:39.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:09:39 smithi083 ceph-mon[79930]: pgmap v21038: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:40.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:09:39 smithi153 ceph-mon[70044]: pgmap v21038: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:41.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:09:41 smithi083 ceph-mon[79930]: pgmap v21039: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:41.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:09:41 smithi083 ceph-mon[77242]: pgmap v21039: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:41.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 17:09:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:09:41] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T17:09:42.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:09:41 smithi153 ceph-mon[70044]: pgmap v21039: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:42.656 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:09:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:09:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T17:09:42.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:09:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T17:09:42.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:09:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T17:09:42.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:09:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T17:09:42.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:09:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T17:09:42.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:09:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:09:42.808Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:09:42.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:09:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T17:09:42.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:09:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T17:09:42.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:09:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T17:09:42.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:09:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T17:09:43.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:09:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T17:09:43.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:09:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T17:09:43.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:09:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T17:09:43.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:09:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T17:09:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 17:09:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:09:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T17:09:43.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:09:43 smithi083 ceph-mon[79930]: pgmap v21040: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:43.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:09:43 smithi083 ceph-mon[77242]: pgmap v21040: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:44.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:09:43 smithi153 ceph-mon[70044]: pgmap v21040: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:45.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:09:45 smithi153 ceph-mon[70044]: pgmap v21041: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:45.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:09:45 smithi083 ceph-mon[79930]: pgmap v21041: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:45.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:09:45 smithi083 ceph-mon[77242]: pgmap v21041: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:46.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:09:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:09:46.149Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:09:47.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:09:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:09:46.900Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:09:47.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:09:47 smithi083 ceph-mon[79930]: pgmap v21042: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:47.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:09:47 smithi083 ceph-mon[77242]: pgmap v21042: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:48.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:09:47 smithi153 ceph-mon[70044]: pgmap v21042: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:49.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:09:49 smithi083 ceph-mon[77242]: pgmap v21043: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:49.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:09:49 smithi083 ceph-mon[79930]: pgmap v21043: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:50.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:09:49 smithi153 ceph-mon[70044]: pgmap v21043: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 17:09:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:09:51] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T17:09:51.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:09:51 smithi083 ceph-mon[79930]: pgmap v21044: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:51.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:09:51 smithi083 ceph-mon[77242]: pgmap v21044: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:52.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:09:51 smithi153 ceph-mon[70044]: pgmap v21044: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:09:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:09:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T17:09:53.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:09:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:09:52.808Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:09:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 17:09:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:09:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T17:09:53.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:09:53 smithi083 ceph-mon[79930]: pgmap v21045: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:53.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:09:53 smithi083 ceph-mon[77242]: pgmap v21045: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:54.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:09:53 smithi153 ceph-mon[70044]: pgmap v21045: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:54.268 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T17:09:54.554 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T17:09:54.554 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 7m ago 11h 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T17:09:54.554 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 5m ago 11h 64.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T17:09:54.555 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 7m ago 11h 26.1M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T17:09:54.555 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 5m ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T17:09:54.555 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (11h) 5m ago 11h 452M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T17:09:54.555 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11h) 7m ago 11h 1100M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T17:09:54.556 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11h) 7m ago 11h 2503M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T17:09:54.556 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (11h) 5m ago 11h 1123M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T17:09:54.556 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (11h) 7m ago 11h 1513M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T17:09:54.556 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 7m ago 11h 24.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T17:09:54.556 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 5m ago 11h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T17:09:54.557 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11h) 7m ago 11h 1829M 2021M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T17:09:54.557 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11h) 7m ago 11h 1814M 2021M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T17:09:54.557 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11h) 7m ago 11h 1409M 2021M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T17:09:54.557 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (11h) 7m ago 11h 1779M 2021M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T17:09:54.558 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (11h) 5m ago 11h 1558M 2478M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T17:09:54.558 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (11h) 5m ago 11h 1959M 2478M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T17:09:54.558 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (11h) 5m ago 11h 1752M 2478M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T17:09:54.558 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (11h) 5m ago 11h 2085M 2478M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T17:09:54.559 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (11h) 5m ago 11h 190M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T17:09:54.909 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T17:09:54.910 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T17:09:54.910 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T17:09:54.911 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:09:54.911 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T17:09:54.912 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T17:09:54.912 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:09:54.912 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T17:09:54.913 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T17:09:54.913 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:09:54.914 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T17:09:54.914 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T17:09:54.915 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:09:54.915 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T17:09:54.916 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T17:09:54.916 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T17:09:54.916 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T17:09:54.917 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T17:09:55.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:09:55 smithi153 ceph-mon[70044]: from='client.58602 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:09:55.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:09:55 smithi153 ceph-mon[70044]: from='client.68539 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:09:55.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:09:55 smithi153 ceph-mon[70044]: pgmap v21046: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:55.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:09:55 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2589999984' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T17:09:55.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:09:55 smithi083 ceph-mon[77242]: from='client.58602 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:09:55.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:09:55 smithi083 ceph-mon[77242]: from='client.68539 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:09:55.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:09:55 smithi083 ceph-mon[77242]: pgmap v21046: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:55.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:09:55 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2589999984' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T17:09:55.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:09:55 smithi083 ceph-mon[79930]: from='client.58602 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:09:55.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:09:55 smithi083 ceph-mon[79930]: from='client.68539 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:09:55.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:09:55 smithi083 ceph-mon[79930]: pgmap v21046: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:55.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:09:55 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2589999984' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T17:09:56.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:09:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:09:56.149Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:09:57.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:09:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:09:56.901Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:09:57.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:09:57 smithi083 ceph-mon[79930]: pgmap v21047: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:57.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:09:57 smithi083 ceph-mon[77242]: pgmap v21047: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:58.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:09:57 smithi153 ceph-mon[70044]: pgmap v21047: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:59.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:09:59 smithi083 ceph-mon[79930]: pgmap v21048: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:09:59.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:09:59 smithi083 ceph-mon[77242]: pgmap v21048: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:00.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:09:59 smithi153 ceph-mon[70044]: pgmap v21048: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:00.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:10:00 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mon-a[77219]: 2022-09-29T17:09:59.999+0000 7f160f665700 -1 log_channel(cluster) log [ERR] : overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T17:10:00.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:10:00 smithi083 ceph-mon[79930]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T17:10:00.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:10:00 smithi083 ceph-mon[77242]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T17:10:01.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:10:00 smithi153 ceph-mon[70044]: overall HEALTH_ERR all OSDs are running quincy or later but require_osd_release < quincy; Telemetry requires re-opt-in; Upgrade: failed due to an unexpected exception 2022-09-29T17:10:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 17:10:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:10:01] "GET /metrics HTTP/1.1" 200 225248 "" "Prometheus/2.33.4" 2022-09-29T17:10:01.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:10:01 smithi083 ceph-mon[79930]: pgmap v21049: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:01.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:10:01 smithi083 ceph-mon[77242]: pgmap v21049: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:02.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:10:01 smithi153 ceph-mon[70044]: pgmap v21049: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:10:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:10:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T17:10:03.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:10:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:10:02.809Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:10:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 17:10:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:10:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T17:10:03.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:10:03 smithi083 ceph-mon[79930]: pgmap v21050: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:03.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:10:03 smithi083 ceph-mon[77242]: pgmap v21050: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:04.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:10:03 smithi153 ceph-mon[70044]: pgmap v21050: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:05.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:10:05 smithi153 ceph-mon[70044]: pgmap v21051: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:05.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:10:05 smithi083 ceph-mon[79930]: pgmap v21051: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:05.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:10:05 smithi083 ceph-mon[77242]: pgmap v21051: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:06.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:10:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:10:06.150Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:10:07.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:10:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:10:06.901Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:10:07.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:10:07 smithi083 ceph-mon[79930]: pgmap v21052: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:07.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:10:07 smithi083 ceph-mon[77242]: pgmap v21052: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:08.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:10:07 smithi153 ceph-mon[70044]: pgmap v21052: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:09.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:10:09 smithi083 ceph-mon[79930]: pgmap v21053: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:09.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:10:09 smithi083 ceph-mon[77242]: pgmap v21053: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:10.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:10:09 smithi153 ceph-mon[70044]: pgmap v21053: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 17:10:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:10:11] "GET /metrics HTTP/1.1" 200 225246 "" "Prometheus/2.33.4" 2022-09-29T17:10:11.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:10:11 smithi083 ceph-mon[79930]: pgmap v21054: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:11.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:10:11 smithi083 ceph-mon[77242]: pgmap v21054: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:12.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:10:11 smithi153 ceph-mon[70044]: pgmap v21054: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:10:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:10:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T17:10:13.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:10:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:10:12.809Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:10:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 17:10:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:10:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T17:10:13.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:10:13 smithi083 ceph-mon[79930]: pgmap v21055: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:13.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:10:13 smithi083 ceph-mon[77242]: pgmap v21055: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:14.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:10:13 smithi153 ceph-mon[70044]: pgmap v21055: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:15.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:10:15 smithi153 ceph-mon[70044]: pgmap v21056: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:15.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:10:15 smithi083 ceph-mon[79930]: pgmap v21056: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:15.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:10:15 smithi083 ceph-mon[77242]: pgmap v21056: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:16.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:10:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:10:16.150Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:10:17.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:10:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:10:16.902Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:10:17.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:10:17 smithi083 ceph-mon[77242]: pgmap v21057: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:17.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:10:17 smithi083 ceph-mon[79930]: pgmap v21057: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:18.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:10:17 smithi153 ceph-mon[70044]: pgmap v21057: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:19.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:10:19 smithi083 ceph-mon[79930]: pgmap v21058: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:19.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:10:19 smithi083 ceph-mon[77242]: pgmap v21058: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:20.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:10:19 smithi153 ceph-mon[70044]: pgmap v21058: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 17:10:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:10:21] "GET /metrics HTTP/1.1" 200 225246 "" "Prometheus/2.33.4" 2022-09-29T17:10:21.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:10:21 smithi083 ceph-mon[79930]: pgmap v21059: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:21.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:10:21 smithi083 ceph-mon[77242]: pgmap v21059: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:22.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:10:21 smithi153 ceph-mon[70044]: pgmap v21059: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:22.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:10:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:10:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T17:10:23.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:10:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:10:22.810Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:10:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 17:10:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:10:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T17:10:23.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:10:23 smithi083 ceph-mon[79930]: pgmap v21060: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:23.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:10:23 smithi083 ceph-mon[77242]: pgmap v21060: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:24.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:10:23 smithi153 ceph-mon[70044]: pgmap v21060: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:25.236 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T17:10:25.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:10:25 smithi153 ceph-mon[70044]: pgmap v21061: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:25.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:10:25 smithi083 ceph-mon[79930]: pgmap v21061: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:25.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:10:25 smithi083 ceph-mon[77242]: pgmap v21061: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:25.532 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T17:10:25.533 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 8m ago 11h 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T17:10:25.533 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 6m ago 11h 64.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T17:10:25.533 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 8m ago 11h 26.1M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T17:10:25.533 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 6m ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T17:10:25.534 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (11h) 6m ago 11h 452M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T17:10:25.534 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11h) 8m ago 11h 1100M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T17:10:25.534 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11h) 8m ago 11h 2503M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T17:10:25.534 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (11h) 6m ago 11h 1123M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T17:10:25.534 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (11h) 8m ago 11h 1513M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T17:10:25.535 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 8m ago 11h 24.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T17:10:25.535 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 6m ago 11h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T17:10:25.535 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11h) 8m ago 11h 1829M 2021M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T17:10:25.535 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11h) 8m ago 11h 1814M 2021M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T17:10:25.536 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11h) 8m ago 11h 1409M 2021M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T17:10:25.536 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (11h) 8m ago 11h 1779M 2021M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T17:10:25.536 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (11h) 6m ago 11h 1558M 2478M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T17:10:25.536 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (11h) 6m ago 11h 1959M 2478M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T17:10:25.536 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (11h) 6m ago 11h 1752M 2478M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T17:10:25.537 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (11h) 6m ago 11h 2085M 2478M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T17:10:25.537 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (11h) 6m ago 11h 190M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T17:10:25.909 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T17:10:25.910 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T17:10:25.910 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T17:10:25.910 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:10:25.910 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T17:10:25.911 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T17:10:25.911 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:10:25.911 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T17:10:25.911 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T17:10:25.912 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:10:25.912 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T17:10:25.912 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T17:10:25.913 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:10:25.913 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T17:10:25.913 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T17:10:25.913 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T17:10:25.914 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T17:10:25.914 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T17:10:26.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:10:26 smithi153 ceph-mon[70044]: from='client.58620 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:10:26.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:10:26 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T17:10:26.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:10:26 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T17:10:26.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:10:26 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T17:10:26.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:10:26 smithi153 ceph-mon[70044]: from='client.68557 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:10:26.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:10:26 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/1510023812' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T17:10:26.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:10:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:10:26.150Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:10:26.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:10:26 smithi083 ceph-mon[79930]: from='client.58620 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:10:26.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:10:26 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T17:10:26.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:10:26 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T17:10:26.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:10:26 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T17:10:26.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:10:26 smithi083 ceph-mon[79930]: from='client.68557 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:10:26.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:10:26 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/1510023812' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T17:10:26.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:10:26 smithi083 ceph-mon[77242]: from='client.58620 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:10:26.465 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:10:26 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T17:10:26.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:10:26 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T17:10:26.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:10:26 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T17:10:26.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:10:26 smithi083 ceph-mon[77242]: from='client.68557 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:10:26.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:10:26 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/1510023812' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T17:10:27.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:10:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:10:26.903Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:10:27.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:10:27 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:10:27.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:10:27 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:10:27.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:10:27 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:10:27.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:10:27 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:10:27.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:10:27 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:10:27.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:10:27 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:10:27.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:10:27 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:10:27.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:10:27 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:10:27.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:10:27 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:10:27.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:10:27 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:10:27.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:10:27 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:10:27.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:10:27 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:10:27.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:10:27 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:10:27.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:10:27 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:10:27.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:10:27 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:10:27.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:10:27 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:10:27.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:10:27 smithi083 ceph-mon[77242]: Adjusting osd_memory_target on smithi083 to 2007M 2022-09-29T17:10:27.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:10:27 smithi083 ceph-mon[77242]: Adjusting osd_memory_target on smithi153 to 2475M 2022-09-29T17:10:27.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:10:27 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:10:27.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:10:27 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:10:27.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:10:27 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:10:27.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:10:27 smithi083 ceph-mon[77242]: pgmap v21062: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:27.217 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:10:27 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:10:27.218 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:10:27 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:10:27.218 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:10:27 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:10:27.218 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:10:27 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:10:27.218 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:10:27 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:10:27.219 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:10:27 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:10:27.219 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:10:27 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:10:27.219 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:10:27 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:10:27.219 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:10:27 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:10:27.220 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:10:27 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:10:27.220 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:10:27 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:10:27.220 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:10:27 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:10:27.220 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:10:27 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:10:27.220 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:10:27 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:10:27.221 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:10:27 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:10:27.221 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:10:27 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:10:27.221 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:10:27 smithi083 ceph-mon[79930]: Adjusting osd_memory_target on smithi083 to 2007M 2022-09-29T17:10:27.221 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:10:27 smithi083 ceph-mon[79930]: Adjusting osd_memory_target on smithi153 to 2475M 2022-09-29T17:10:27.222 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:10:27 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:10:27.222 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:10:27 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:10:27.222 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:10:27 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:10:27.222 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:10:27 smithi083 ceph-mon[79930]: pgmap v21062: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:27.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:10:27 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:10:27.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:10:27 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:10:27.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:10:27 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:10:27.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:10:27 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:10:27.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:10:27 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:10:27.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:10:27 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:10:27.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:10:27 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:10:27.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:10:27 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:10:27.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:10:27 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:10:27.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:10:27 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:10:27.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:10:27 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:10:27.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:10:27 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:10:27.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:10:27 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:10:27.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:10:27 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:10:27.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:10:27 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:10:27.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:10:27 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2022-09-29T17:10:27.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:10:27 smithi153 ceph-mon[70044]: Adjusting osd_memory_target on smithi083 to 2007M 2022-09-29T17:10:27.312 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:10:27 smithi153 ceph-mon[70044]: Adjusting osd_memory_target on smithi153 to 2475M 2022-09-29T17:10:27.312 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:10:27 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:10:27.312 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:10:27 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:10:27.312 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:10:27 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:10:27.313 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:10:27 smithi153 ceph-mon[70044]: pgmap v21062: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:29.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:10:29 smithi083 ceph-mon[79930]: pgmap v21063: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:29.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:10:29 smithi083 ceph-mon[77242]: pgmap v21063: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:30.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:10:29 smithi153 ceph-mon[70044]: pgmap v21063: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 17:10:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:10:31] "GET /metrics HTTP/1.1" 200 225246 "" "Prometheus/2.33.4" 2022-09-29T17:10:31.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:10:31 smithi083 ceph-mon[79930]: pgmap v21064: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:31.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:10:31 smithi083 ceph-mon[77242]: pgmap v21064: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:32.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:10:31 smithi153 ceph-mon[70044]: pgmap v21064: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:10:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:10:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T17:10:33.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:10:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:10:32.811Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:10:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 17:10:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:10:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T17:10:33.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:10:33 smithi083 ceph-mon[79930]: pgmap v21065: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:33.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:10:33 smithi083 ceph-mon[77242]: pgmap v21065: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:34.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:10:33 smithi153 ceph-mon[70044]: pgmap v21065: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:35.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:10:35 smithi153 ceph-mon[70044]: pgmap v21066: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:35.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:10:35 smithi083 ceph-mon[77242]: pgmap v21066: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:35.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:10:35 smithi083 ceph-mon[79930]: pgmap v21066: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:36.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:10:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:10:36.152Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:10:37.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:10:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:10:36.903Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:10:37.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:10:37 smithi083 ceph-mon[79930]: pgmap v21067: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:37.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:10:37 smithi083 ceph-mon[77242]: pgmap v21067: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:38.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:10:37 smithi153 ceph-mon[70044]: pgmap v21067: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:39.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:10:39 smithi083 ceph-mon[79930]: pgmap v21068: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:39.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:10:39 smithi083 ceph-mon[77242]: pgmap v21068: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:10:39 smithi153 ceph-mon[70044]: pgmap v21068: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:41.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:10:41 smithi083 ceph-mon[79930]: pgmap v21069: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:41.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:10:41 smithi083 ceph-mon[77242]: pgmap v21069: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:41.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 17:10:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:10:41] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T17:10:42.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:10:41 smithi153 ceph-mon[70044]: pgmap v21069: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:42.675 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:10:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:10:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T17:10:42.961 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:10:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:10:42.812Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:10:42.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:10:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T17:10:42.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:10:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T17:10:42.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:10:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T17:10:42.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:10:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T17:10:42.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:10:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T17:10:42.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:10:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T17:10:42.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:10:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T17:10:42.964 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:10:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T17:10:43.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:10:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T17:10:43.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:10:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T17:10:43.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:10:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T17:10:43.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:10:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T17:10:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 17:10:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:10:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T17:10:43.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:10:43 smithi083 ceph-mon[79930]: pgmap v21070: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:43.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:10:43 smithi083 ceph-mon[77242]: pgmap v21070: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:44.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:10:43 smithi153 ceph-mon[70044]: pgmap v21070: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:45.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:10:45 smithi153 ceph-mon[70044]: pgmap v21071: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:45.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:10:45 smithi083 ceph-mon[79930]: pgmap v21071: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:45.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:10:45 smithi083 ceph-mon[77242]: pgmap v21071: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:46.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:10:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:10:46.153Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:10:47.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:10:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:10:46.904Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:10:47.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:10:47 smithi083 ceph-mon[79930]: pgmap v21072: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:47.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:10:47 smithi083 ceph-mon[77242]: pgmap v21072: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:48.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:10:47 smithi153 ceph-mon[70044]: pgmap v21072: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:49.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:10:49 smithi083 ceph-mon[79930]: pgmap v21073: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:49.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:10:49 smithi083 ceph-mon[77242]: pgmap v21073: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:50.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:10:49 smithi153 ceph-mon[70044]: pgmap v21073: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:51.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:10:51 smithi083 ceph-mon[79930]: pgmap v21074: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:51.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 17:10:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:10:51] "GET /metrics HTTP/1.1" 200 225244 "" "Prometheus/2.33.4" 2022-09-29T17:10:51.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:10:51 smithi083 ceph-mon[77242]: pgmap v21074: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:52.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:10:51 smithi153 ceph-mon[70044]: pgmap v21074: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:52.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:10:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:10:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T17:10:53.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:10:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:10:52.812Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:10:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 17:10:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:10:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T17:10:53.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:10:53 smithi083 ceph-mon[79930]: pgmap v21075: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:53.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:10:53 smithi083 ceph-mon[77242]: pgmap v21075: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:54.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:10:53 smithi153 ceph-mon[70044]: pgmap v21075: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:55.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:10:55 smithi153 ceph-mon[70044]: pgmap v21076: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:55.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:10:55 smithi083 ceph-mon[79930]: pgmap v21076: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:55.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:10:55 smithi083 ceph-mon[77242]: pgmap v21076: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:56.238 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T17:10:56.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:10:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:10:56.153Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:10:56.526 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T17:10:56.527 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 8m ago 11h 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T17:10:56.527 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 6m ago 11h 64.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T17:10:56.527 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 8m ago 11h 26.1M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T17:10:56.527 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 6m ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T17:10:56.528 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (11h) 6m ago 11h 452M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T17:10:56.528 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11h) 8m ago 11h 1100M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T17:10:56.528 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11h) 8m ago 11h 2503M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T17:10:56.528 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (11h) 6m ago 11h 1123M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T17:10:56.529 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (11h) 8m ago 11h 1513M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T17:10:56.529 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 8m ago 11h 24.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T17:10:56.529 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 6m ago 11h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T17:10:56.529 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11h) 8m ago 11h 1829M 2007M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T17:10:56.530 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11h) 8m ago 11h 1814M 2007M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T17:10:56.530 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11h) 8m ago 11h 1409M 2007M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T17:10:56.530 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (11h) 8m ago 11h 1779M 2007M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T17:10:56.531 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (11h) 6m ago 11h 1558M 2475M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T17:10:56.531 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (11h) 6m ago 11h 1959M 2475M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T17:10:56.531 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (11h) 6m ago 11h 1752M 2475M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T17:10:56.531 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (11h) 6m ago 11h 2085M 2475M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T17:10:56.531 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (11h) 6m ago 11h 190M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T17:10:56.885 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T17:10:56.886 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T17:10:56.886 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T17:10:56.886 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:10:56.887 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T17:10:56.887 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T17:10:56.887 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:10:56.887 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T17:10:56.888 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T17:10:56.888 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:10:56.888 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T17:10:56.889 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T17:10:56.889 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:10:56.890 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T17:10:56.890 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T17:10:56.891 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T17:10:56.891 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T17:10:56.892 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T17:10:57.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:10:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:10:56.904Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:10:57.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:10:57 smithi083 ceph-mon[79930]: from='client.58638 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:10:57.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:10:57 smithi083 ceph-mon[79930]: from='client.68575 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:10:57.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:10:57 smithi083 ceph-mon[79930]: pgmap v21077: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:57.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:10:57 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3103016727' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T17:10:57.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:10:57 smithi083 ceph-mon[77242]: from='client.58638 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:10:57.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:10:57 smithi083 ceph-mon[77242]: from='client.68575 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:10:57.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:10:57 smithi083 ceph-mon[77242]: pgmap v21077: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:57.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:10:57 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3103016727' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T17:10:58.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:10:57 smithi153 ceph-mon[70044]: from='client.58638 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:10:58.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:10:57 smithi153 ceph-mon[70044]: from='client.68575 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:10:58.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:10:57 smithi153 ceph-mon[70044]: pgmap v21077: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:58.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:10:57 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3103016727' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T17:10:59.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:10:59 smithi083 ceph-mon[79930]: pgmap v21078: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:10:59.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:10:59 smithi083 ceph-mon[77242]: pgmap v21078: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:00.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:10:59 smithi153 ceph-mon[70044]: pgmap v21078: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:01.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:11:01 smithi083 ceph-mon[77242]: pgmap v21079: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:01.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:11:01 smithi083 ceph-mon[79930]: pgmap v21079: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:01.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 17:11:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:11:01] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T17:11:02.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:11:01 smithi153 ceph-mon[70044]: pgmap v21079: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:02.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:11:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:11:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T17:11:03.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:11:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:11:02.813Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:11:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 17:11:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:11:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T17:11:03.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:11:03 smithi083 ceph-mon[79930]: pgmap v21080: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:03.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:11:03 smithi083 ceph-mon[77242]: pgmap v21080: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:04.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:11:03 smithi153 ceph-mon[70044]: pgmap v21080: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:05.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:11:05 smithi153 ceph-mon[70044]: pgmap v21081: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:05.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:11:05 smithi083 ceph-mon[79930]: pgmap v21081: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:05.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:11:05 smithi083 ceph-mon[77242]: pgmap v21081: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:06.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:11:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:11:06.154Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:11:07.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:11:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:11:06.906Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:11:07.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:11:07 smithi083 ceph-mon[79930]: pgmap v21082: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:07.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:11:07 smithi083 ceph-mon[77242]: pgmap v21082: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:08.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:11:07 smithi153 ceph-mon[70044]: pgmap v21082: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:09.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:11:09 smithi083 ceph-mon[79930]: pgmap v21083: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:09.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:11:09 smithi083 ceph-mon[77242]: pgmap v21083: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:10.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:11:09 smithi153 ceph-mon[70044]: pgmap v21083: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 17:11:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:11:11] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T17:11:11.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:11:11 smithi083 ceph-mon[79930]: pgmap v21084: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:11.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:11:11 smithi083 ceph-mon[77242]: pgmap v21084: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:12.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:11:11 smithi153 ceph-mon[70044]: pgmap v21084: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:12.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:11:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:11:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T17:11:13.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:11:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:11:12.813Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:11:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 17:11:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:11:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T17:11:13.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:11:13 smithi083 ceph-mon[79930]: pgmap v21085: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:13.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:11:13 smithi083 ceph-mon[77242]: pgmap v21085: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:14.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:11:13 smithi153 ceph-mon[70044]: pgmap v21085: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:15.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:11:15 smithi153 ceph-mon[70044]: pgmap v21086: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:15.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:11:15 smithi083 ceph-mon[79930]: pgmap v21086: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:15.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:11:15 smithi083 ceph-mon[77242]: pgmap v21086: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:16.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:11:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:11:16.155Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:11:17.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:11:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:11:16.906Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:11:17.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:11:17 smithi083 ceph-mon[79930]: pgmap v21087: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:17.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:11:17 smithi083 ceph-mon[77242]: pgmap v21087: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:18.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:11:17 smithi153 ceph-mon[70044]: pgmap v21087: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:19.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:11:19 smithi083 ceph-mon[79930]: pgmap v21088: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:19.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:11:19 smithi083 ceph-mon[77242]: pgmap v21088: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:20.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:11:19 smithi153 ceph-mon[70044]: pgmap v21088: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:21.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:11:21 smithi083 ceph-mon[77242]: pgmap v21089: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 17:11:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:11:21] "GET /metrics HTTP/1.1" 200 225241 "" "Prometheus/2.33.4" 2022-09-29T17:11:21.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:11:21 smithi083 ceph-mon[79930]: pgmap v21089: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:22.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:11:21 smithi153 ceph-mon[70044]: pgmap v21089: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:22.734 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:11:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:11:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T17:11:23.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:11:22 smithi153 ceph-mon[70044]: pgmap v21090: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:23.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:11:22 smithi083 ceph-mon[77242]: pgmap v21090: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:23.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:11:22 smithi083 ceph-mon[79930]: pgmap v21090: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:23.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:11:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:11:22.813Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:11:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 17:11:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:11:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T17:11:25.313 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:11:25 smithi153 ceph-mon[70044]: pgmap v21091: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:25.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:11:25 smithi083 ceph-mon[79930]: pgmap v21091: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:25.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:11:25 smithi083 ceph-mon[77242]: pgmap v21091: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:26.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:11:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:11:26.157Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:11:26.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:11:26 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T17:11:26.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:11:26 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T17:11:26.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:11:26 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T17:11:26.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:11:26 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T17:11:26.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:11:26 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T17:11:26.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:11:26 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T17:11:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:11:26 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T17:11:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:11:26 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T17:11:26.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:11:26 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T17:11:27.200 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T17:11:27.206 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:11:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:11:26.908Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:11:27.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:11:27 smithi083 ceph-mon[77242]: pgmap v21092: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:27.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:11:27 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:11:27.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:11:27 smithi083 ceph-mon[79930]: pgmap v21092: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:27.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:11:27 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:11:27.491 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T17:11:27.492 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 9m ago 11h 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T17:11:27.492 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 7m ago 11h 64.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T17:11:27.492 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 9m ago 11h 26.1M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T17:11:27.493 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 7m ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T17:11:27.493 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (11h) 7m ago 11h 452M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T17:11:27.493 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11h) 9m ago 11h 1100M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T17:11:27.493 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11h) 9m ago 11h 2503M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T17:11:27.494 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (11h) 7m ago 11h 1123M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T17:11:27.494 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (11h) 9m ago 11h 1513M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T17:11:27.494 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 9m ago 11h 24.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T17:11:27.494 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 7m ago 11h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T17:11:27.495 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11h) 9m ago 11h 1829M 2007M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T17:11:27.495 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11h) 9m ago 11h 1814M 2007M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T17:11:27.495 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11h) 9m ago 11h 1409M 2007M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T17:11:27.495 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (11h) 9m ago 11h 1779M 2007M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T17:11:27.496 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (11h) 7m ago 11h 1558M 2475M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T17:11:27.496 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (11h) 7m ago 11h 1959M 2475M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T17:11:27.496 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (11h) 7m ago 11h 1752M 2475M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T17:11:27.496 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (11h) 7m ago 11h 2085M 2475M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T17:11:27.497 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (11h) 7m ago 11h 190M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T17:11:27.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:11:27 smithi153 ceph-mon[70044]: pgmap v21092: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:27.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:11:27 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:11:27.849 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T17:11:27.849 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T17:11:27.850 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T17:11:27.850 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:11:27.850 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T17:11:27.850 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T17:11:27.850 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:11:27.851 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T17:11:27.851 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T17:11:27.851 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:11:27.851 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T17:11:27.852 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T17:11:27.852 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:11:27.852 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T17:11:27.852 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T17:11:27.852 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T17:11:27.853 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T17:11:27.853 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T17:11:28.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:11:28 smithi083 ceph-mon[79930]: from='client.58656 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:11:28.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:11:28 smithi083 ceph-mon[79930]: from='client.58662 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:11:28.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:11:28 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4252201760' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T17:11:28.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:11:28 smithi083 ceph-mon[77242]: from='client.58656 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:11:28.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:11:28 smithi083 ceph-mon[77242]: from='client.58662 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:11:28.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:11:28 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4252201760' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T17:11:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:11:28 smithi153 ceph-mon[70044]: from='client.58656 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:11:28.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:11:28 smithi153 ceph-mon[70044]: from='client.58662 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:11:28.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:11:28 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4252201760' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T17:11:29.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:11:29 smithi153 ceph-mon[70044]: pgmap v21093: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:29.711 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:11:29 smithi083 ceph-mon[79930]: pgmap v21093: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:29.711 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:11:29 smithi083 ceph-mon[77242]: pgmap v21093: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 17:11:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:11:31] "GET /metrics HTTP/1.1" 200 225246 "" "Prometheus/2.33.4" 2022-09-29T17:11:31.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:11:31 smithi083 ceph-mon[79930]: pgmap v21094: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:31.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:11:31 smithi083 ceph-mon[77242]: pgmap v21094: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:32.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:11:31 smithi153 ceph-mon[70044]: pgmap v21094: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:11:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:11:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T17:11:33.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:11:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:11:32.814Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:11:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 17:11:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:11:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T17:11:33.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:11:33 smithi083 ceph-mon[79930]: pgmap v21095: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:33.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:11:33 smithi083 ceph-mon[77242]: pgmap v21095: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:34.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:11:33 smithi153 ceph-mon[70044]: pgmap v21095: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:35.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:11:35 smithi153 ceph-mon[70044]: pgmap v21096: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:35.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:11:35 smithi083 ceph-mon[79930]: pgmap v21096: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:35.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:11:35 smithi083 ceph-mon[77242]: pgmap v21096: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:36.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:11:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:11:36.156Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:11:37.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:11:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:11:36.907Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:11:37.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:11:37 smithi083 ceph-mon[79930]: pgmap v21097: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:37.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:11:37 smithi083 ceph-mon[77242]: pgmap v21097: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:38.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:11:37 smithi153 ceph-mon[70044]: pgmap v21097: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:39.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:11:39 smithi083 ceph-mon[79930]: pgmap v21098: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:39.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:11:39 smithi083 ceph-mon[77242]: pgmap v21098: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:40.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:11:39 smithi153 ceph-mon[70044]: pgmap v21098: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 17:11:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:11:41] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T17:11:41.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:11:41 smithi083 ceph-mon[79930]: pgmap v21099: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:41.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:11:41 smithi083 ceph-mon[77242]: pgmap v21099: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:42.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:11:41 smithi153 ceph-mon[70044]: pgmap v21099: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:42.692 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:11:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:11:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T17:11:42.962 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:11:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:11:42.815Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:11:42.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:11:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T17:11:42.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:11:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T17:11:42.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:11:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T17:11:42.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:11:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T17:11:42.965 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:11:42 smithi083 ceph-mon[77242]: pgmap v21100: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:42.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:11:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T17:11:42.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:11:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T17:11:42.966 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:11:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T17:11:42.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:11:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T17:11:42.967 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:11:42 smithi083 ceph-mon[79930]: pgmap v21100: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:43.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:11:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T17:11:43.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:11:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T17:11:43.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:11:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T17:11:43.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:11:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T17:11:43.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:11:42 smithi153 ceph-mon[70044]: pgmap v21100: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:43.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 17:11:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:11:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T17:11:45.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:11:45 smithi153 ceph-mon[70044]: pgmap v21101: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:45.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:11:45 smithi083 ceph-mon[77242]: pgmap v21101: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:45.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:11:45 smithi083 ceph-mon[79930]: pgmap v21101: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:46.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:11:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:11:46.157Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:11:47.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:11:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:11:46.908Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:11:47.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:11:47 smithi083 ceph-mon[77242]: pgmap v21102: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:47.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:11:47 smithi083 ceph-mon[79930]: pgmap v21102: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:48.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:11:47 smithi153 ceph-mon[70044]: pgmap v21102: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:49.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:11:48 smithi153 ceph-mon[70044]: pgmap v21103: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:49.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:11:48 smithi083 ceph-mon[79930]: pgmap v21103: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:49.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:11:48 smithi083 ceph-mon[77242]: pgmap v21103: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:51.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:11:51 smithi083 ceph-mon[79930]: pgmap v21104: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:51.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:11:51 smithi083 ceph-mon[77242]: pgmap v21104: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:51.963 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 17:11:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:11:51] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T17:11:52.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:11:51 smithi153 ceph-mon[70044]: pgmap v21104: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:52.739 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:11:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:11:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T17:11:53.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:11:52 smithi153 ceph-mon[70044]: pgmap v21105: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:53.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:11:52 smithi083 ceph-mon[79930]: pgmap v21105: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:53.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:11:52 smithi083 ceph-mon[77242]: pgmap v21105: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:53.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:11:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:11:52.816Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:11:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 17:11:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:11:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T17:11:55.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:11:55 smithi153 ceph-mon[70044]: pgmap v21106: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:55.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:11:55 smithi083 ceph-mon[79930]: pgmap v21106: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:55.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:11:55 smithi083 ceph-mon[77242]: pgmap v21106: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:56.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:11:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:11:56.157Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:11:57.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:11:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:11:56.908Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:11:57.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:11:57 smithi083 ceph-mon[79930]: pgmap v21107: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:57.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:11:57 smithi083 ceph-mon[77242]: pgmap v21107: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:58.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:11:57 smithi153 ceph-mon[70044]: pgmap v21107: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:58.174 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T17:11:58.461 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T17:11:58.461 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 9m ago 11h 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T17:11:58.461 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 7m ago 11h 64.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T17:11:58.462 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 9m ago 11h 26.1M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T17:11:58.462 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 7m ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T17:11:58.462 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (11h) 7m ago 11h 452M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T17:11:58.462 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11h) 9m ago 11h 1100M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T17:11:58.463 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11h) 9m ago 11h 2503M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T17:11:58.463 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (11h) 7m ago 11h 1123M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T17:11:58.463 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (11h) 9m ago 11h 1513M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T17:11:58.464 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 9m ago 11h 24.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T17:11:58.464 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 7m ago 11h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T17:11:58.465 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11h) 9m ago 11h 1829M 2007M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T17:11:58.465 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11h) 9m ago 11h 1814M 2007M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T17:11:58.465 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11h) 9m ago 11h 1409M 2007M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T17:11:58.465 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (11h) 9m ago 11h 1779M 2007M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T17:11:58.466 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (11h) 7m ago 11h 1558M 2475M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T17:11:58.466 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (11h) 7m ago 11h 1959M 2475M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T17:11:58.466 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (11h) 7m ago 11h 1752M 2475M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T17:11:58.466 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (11h) 7m ago 11h 2085M 2475M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T17:11:58.466 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (11h) 7m ago 11h 190M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T17:11:58.826 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T17:11:58.826 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T17:11:58.827 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T17:11:58.827 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:11:58.827 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T17:11:58.828 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T17:11:58.828 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:11:58.829 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T17:11:58.829 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T17:11:58.830 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:11:58.830 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T17:11:58.830 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T17:11:58.831 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:11:58.831 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T17:11:58.831 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T17:11:58.831 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T17:11:58.832 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T17:11:58.832 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T17:11:58.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:11:58 smithi083 ceph-mon[79930]: from='client.58674 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:11:58.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:11:58 smithi083 ceph-mon[79930]: from='client.68611 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:11:58.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:11:58 smithi083 ceph-mon[79930]: pgmap v21108: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:58.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:11:58 smithi083 ceph-mon[77242]: from='client.58674 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:11:58.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:11:58 smithi083 ceph-mon[77242]: from='client.68611 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:11:58.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:11:58 smithi083 ceph-mon[77242]: pgmap v21108: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:59.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:11:58 smithi153 ceph-mon[70044]: from='client.58674 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:11:59.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:11:58 smithi153 ceph-mon[70044]: from='client.68611 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:11:59.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:11:58 smithi153 ceph-mon[70044]: pgmap v21108: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:11:59.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:11:59 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/4171140718' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T17:11:59.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:11:59 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/4171140718' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T17:12:00.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:11:59 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/4171140718' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T17:12:00.961 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:12:00 smithi083 ceph-mon[77242]: pgmap v21109: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:12:00.961 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:12:00 smithi083 ceph-mon[79930]: pgmap v21109: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:12:01.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:12:00 smithi153 ceph-mon[70044]: pgmap v21109: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:12:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 17:12:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:12:01] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T17:12:02.740 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:12:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:12:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T17:12:03.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:12:02 smithi153 ceph-mon[70044]: pgmap v21110: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:12:03.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:12:02 smithi083 ceph-mon[77242]: pgmap v21110: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:12:03.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:12:02 smithi083 ceph-mon[79930]: pgmap v21110: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:12:03.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:12:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:12:02.817Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:12:03.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 17:12:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:12:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T17:12:05.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:12:05 smithi153 ceph-mon[70044]: pgmap v21111: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:12:05.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:12:05 smithi083 ceph-mon[79930]: pgmap v21111: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:12:05.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:12:05 smithi083 ceph-mon[77242]: pgmap v21111: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:12:06.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:12:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:12:06.157Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:12:07.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:12:06 smithi153 ceph-mon[70044]: pgmap v21112: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:12:07.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:12:06 smithi083 ceph-mon[79930]: pgmap v21112: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:12:07.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:12:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:12:06.910Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:12:07.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:12:06 smithi083 ceph-mon[77242]: pgmap v21112: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:12:09.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:12:08 smithi153 ceph-mon[70044]: pgmap v21113: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:12:09.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:12:08 smithi083 ceph-mon[77242]: pgmap v21113: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:12:09.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:12:08 smithi083 ceph-mon[79930]: pgmap v21113: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:12:11.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:12:10 smithi153 ceph-mon[70044]: pgmap v21114: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:12:11.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:12:10 smithi083 ceph-mon[79930]: pgmap v21114: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:12:11.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:12:10 smithi083 ceph-mon[77242]: pgmap v21114: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:12:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 17:12:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:12:11] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T17:12:12.744 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:12:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:12:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T17:12:13.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:12:12 smithi153 ceph-mon[70044]: pgmap v21115: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:12:13.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:12:12 smithi083 ceph-mon[79930]: pgmap v21115: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:12:13.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:12:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:12:12.818Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:12:13.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:12:12 smithi083 ceph-mon[77242]: pgmap v21115: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:12:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 17:12:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:12:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T17:12:15.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:12:15 smithi153 ceph-mon[70044]: pgmap v21116: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:12:15.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:12:15 smithi083 ceph-mon[79930]: pgmap v21116: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:12:15.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:12:15 smithi083 ceph-mon[77242]: pgmap v21116: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:12:16.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:12:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:12:16.158Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:12:17.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:12:16 smithi153 ceph-mon[70044]: pgmap v21117: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:12:17.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:12:16 smithi083 ceph-mon[77242]: pgmap v21117: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:12:17.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:12:16 smithi083 ceph-mon[79930]: pgmap v21117: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:12:17.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:12:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:12:16.910Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:12:19.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:12:18 smithi153 ceph-mon[70044]: pgmap v21118: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:12:19.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:12:18 smithi083 ceph-mon[79930]: pgmap v21118: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:12:19.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:12:18 smithi083 ceph-mon[77242]: pgmap v21118: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:12:21.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:12:20 smithi153 ceph-mon[70044]: pgmap v21119: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:12:21.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:12:20 smithi083 ceph-mon[79930]: pgmap v21119: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:12:21.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:12:20 smithi083 ceph-mon[77242]: pgmap v21119: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:12:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 17:12:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:12:21] "GET /metrics HTTP/1.1" 200 225240 "" "Prometheus/2.33.4" 2022-09-29T17:12:22.745 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:12:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:12:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T17:12:23.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:12:22 smithi153 ceph-mon[70044]: pgmap v21120: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:12:23.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:12:22 smithi083 ceph-mon[79930]: pgmap v21120: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:12:23.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:12:22 smithi083 ceph-mon[77242]: pgmap v21120: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:12:23.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:12:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:12:22.818Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:12:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 17:12:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:12:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T17:12:25.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:12:25 smithi153 ceph-mon[70044]: pgmap v21121: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:12:25.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:12:25 smithi083 ceph-mon[79930]: pgmap v21121: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:12:25.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:12:25 smithi083 ceph-mon[77242]: pgmap v21121: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:12:26.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:12:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:12:26.158Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:12:27.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:12:26 smithi153 ceph-mon[70044]: pgmap v21122: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:12:27.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:12:26 smithi083 ceph-mon[79930]: pgmap v21122: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:12:27.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:12:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:12:26.910Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:12:27.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:12:26 smithi083 ceph-mon[77242]: pgmap v21122: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:12:28.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:12:27 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T17:12:28.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:12:27 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T17:12:28.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:12:27 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T17:12:28.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:12:27 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:12:28.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:12:27 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:12:28.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:12:27 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:12:28.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:12:27 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:12:28.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:12:27 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T17:12:28.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:12:27 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T17:12:28.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:12:27 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T17:12:28.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:12:27 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:12:28.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:12:27 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:12:28.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:12:27 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:12:28.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:12:27 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:12:28.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:12:27 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T17:12:28.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:12:27 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T17:12:28.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:12:27 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T17:12:28.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:12:27 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:12:28.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:12:27 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:12:28.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:12:27 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:12:28.218 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:12:27 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:12:29.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:12:28 smithi153 ceph-mon[70044]: pgmap v21123: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:12:29.176 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T17:12:29.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:12:28 smithi083 ceph-mon[79930]: pgmap v21123: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:12:29.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:12:28 smithi083 ceph-mon[77242]: pgmap v21123: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:12:29.490 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T17:12:29.490 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 10m ago 11h 26.9M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T17:12:29.491 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 8m ago 11h 64.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T17:12:29.491 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 10m ago 11h 26.1M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T17:12:29.491 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 8m ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T17:12:29.491 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (11h) 8m ago 11h 452M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T17:12:29.491 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11h) 10m ago 11h 1100M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T17:12:29.492 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11h) 10m ago 11h 2503M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T17:12:29.492 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (11h) 8m ago 11h 1123M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T17:12:29.492 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (11h) 10m ago 11h 1513M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T17:12:29.492 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 10m ago 11h 24.5M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T17:12:29.493 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 8m ago 11h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T17:12:29.493 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11h) 10m ago 11h 1829M 2007M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T17:12:29.493 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11h) 10m ago 11h 1814M 2007M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T17:12:29.493 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11h) 10m ago 11h 1409M 2007M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T17:12:29.493 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (11h) 10m ago 11h 1779M 2007M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T17:12:29.494 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (11h) 8m ago 11h 1558M 2475M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T17:12:29.494 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (11h) 8m ago 11h 1959M 2475M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T17:12:29.494 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (11h) 8m ago 11h 1752M 2475M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T17:12:29.494 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (11h) 8m ago 11h 2085M 2475M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T17:12:29.495 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (11h) 8m ago 11h 190M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T17:12:29.859 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T17:12:29.859 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T17:12:29.860 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T17:12:29.860 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:12:29.860 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T17:12:29.860 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T17:12:29.861 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:12:29.861 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T17:12:29.861 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T17:12:29.861 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:12:29.861 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T17:12:29.862 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T17:12:29.862 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:12:29.862 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T17:12:29.862 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T17:12:29.863 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T17:12:29.863 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T17:12:29.864 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T17:12:30.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:12:30 smithi153 ceph-mon[70044]: from='client.58692 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:12:30.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:12:30 smithi153 ceph-mon[70044]: from='client.68629 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:12:30.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:12:30 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/100502187' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T17:12:30.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:12:30 smithi083 ceph-mon[79930]: from='client.58692 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:12:30.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:12:30 smithi083 ceph-mon[79930]: from='client.68629 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:12:30.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:12:30 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/100502187' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T17:12:30.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:12:30 smithi083 ceph-mon[77242]: from='client.58692 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:12:30.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:12:30 smithi083 ceph-mon[77242]: from='client.68629 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:12:30.464 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:12:30 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/100502187' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T17:12:31.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:12:31 smithi153 ceph-mon[70044]: pgmap v21124: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:12:31.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:12:31 smithi083 ceph-mon[79930]: pgmap v21124: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:12:31.530 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:12:31 smithi083 ceph-mon[77242]: pgmap v21124: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:12:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 17:12:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:12:31] "GET /metrics HTTP/1.1" 200 225246 "" "Prometheus/2.33.4" 2022-09-29T17:12:32.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:12:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:12:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T17:12:32.822 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:12:32 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:12:32.822 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:12:32 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:12:32.822 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:12:32 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:12:32.823 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:12:32 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:12:32.823 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:12:32 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:12:32.823 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:12:32 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:12:32.824 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:12:32 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:12:32.824 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:12:32 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:12:32.824 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:12:32 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:12:33.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:12:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:12:32.818Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:12:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 17:12:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:12:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T17:12:33.807 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:12:33 smithi153 ceph-mon[70044]: pgmap v21125: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:12:33.964 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:12:33 smithi083 ceph-mon[77242]: pgmap v21125: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:12:33.965 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:12:33 smithi083 ceph-mon[79930]: pgmap v21125: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:12:35.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:12:35 smithi153 ceph-mon[70044]: pgmap v21126: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:12:35.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:12:35 smithi083 ceph-mon[77242]: pgmap v21126: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:12:35.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:12:35 smithi083 ceph-mon[79930]: pgmap v21126: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:12:36.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:12:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:12:36.159Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:12:37.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:12:36 smithi153 ceph-mon[70044]: pgmap v21127: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:12:37.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:12:36 smithi083 ceph-mon[79930]: pgmap v21127: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:12:37.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:12:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:12:36.911Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:12:37.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:12:36 smithi083 ceph-mon[77242]: pgmap v21127: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:12:39.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:12:38 smithi153 ceph-mon[70044]: pgmap v21128: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:12:39.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:12:38 smithi083 ceph-mon[79930]: pgmap v21128: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:12:39.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:12:38 smithi083 ceph-mon[77242]: pgmap v21128: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:12:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:12:40 smithi153 ceph-mon[70044]: pgmap v21129: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:12:41.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:12:40 smithi083 ceph-mon[79930]: pgmap v21129: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:12:41.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:12:40 smithi083 ceph-mon[77242]: pgmap v21129: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:12:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 17:12:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:12:41] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T17:12:42.318 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:12:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T17:12:42.319 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:12:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T17:12:42.319 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:12:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T17:12:42.319 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:12:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T17:12:42.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:12:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T17:12:42.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:12:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T17:12:42.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:12:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T17:12:42.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:12:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T17:12:42.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:12:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T17:12:42.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:12:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T17:12:42.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:12:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T17:12:42.463 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:12:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T17:12:42.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:12:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:12:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T17:12:43.161 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:12:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:12:42.819Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:12:43.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:12:43 smithi153 ceph-mon[70044]: pgmap v21130: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:12:43.312 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 17:12:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:12:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T17:12:43.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:12:43 smithi083 ceph-mon[79930]: pgmap v21130: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:12:43.496 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:12:43 smithi083 ceph-mon[77242]: pgmap v21130: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:12:45.308 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:12:45 smithi153 ceph-mon[70044]: pgmap v21131: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:12:45.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:12:45 smithi083 ceph-mon[79930]: pgmap v21131: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:12:45.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:12:45 smithi083 ceph-mon[77242]: pgmap v21131: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:12:46.462 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:12:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:12:46.160Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:12:47.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:12:46 smithi153 ceph-mon[70044]: pgmap v21132: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:12:47.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:12:46 smithi083 ceph-mon[77242]: pgmap v21132: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:12:47.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:12:46 smithi083 ceph-mon[79930]: pgmap v21132: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:12:47.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:12:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:12:46.913Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:12:49.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:12:48 smithi153 ceph-mon[70044]: pgmap v21133: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:12:49.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:12:48 smithi083 ceph-mon[77242]: pgmap v21133: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:12:49.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:12:48 smithi083 ceph-mon[79930]: pgmap v21133: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:12:51.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:12:50 smithi153 ceph-mon[70044]: pgmap v21134: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:12:51.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:12:50 smithi083 ceph-mon[77242]: pgmap v21134: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:12:51.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:12:50 smithi083 ceph-mon[79930]: pgmap v21134: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:12:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 17:12:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:12:51] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T17:12:52.751 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:12:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:12:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T17:12:53.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:12:52 smithi153 ceph-mon[70044]: pgmap v21135: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:12:53.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:12:52 smithi083 ceph-mon[77242]: pgmap v21135: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:12:53.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:12:52 smithi083 ceph-mon[79930]: pgmap v21135: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:12:53.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:12:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:12:52.820Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:12:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 17:12:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:12:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T17:12:55.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:12:55 smithi153 ceph-mon[70044]: pgmap v21136: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:12:55.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:12:55 smithi083 ceph-mon[79930]: pgmap v21136: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:12:55.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:12:55 smithi083 ceph-mon[77242]: pgmap v21136: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:12:56.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:12:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:12:56.161Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:12:57.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:12:56 smithi153 ceph-mon[70044]: pgmap v21137: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:12:57.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:12:56 smithi083 ceph-mon[79930]: pgmap v21137: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:12:57.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:12:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:12:56.913Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:12:57.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:12:56 smithi083 ceph-mon[77242]: pgmap v21137: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:12:59.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:12:58 smithi153 ceph-mon[70044]: pgmap v21138: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:12:59.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:12:58 smithi083 ceph-mon[79930]: pgmap v21138: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:12:59.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:12:58 smithi083 ceph-mon[77242]: pgmap v21138: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:00.186 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T17:13:00.477 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T17:13:00.478 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 28s ago 11h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T17:13:00.478 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 8m ago 11h 64.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T17:13:00.478 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 28s ago 11h 26.1M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T17:13:00.479 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 8m ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T17:13:00.480 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (11h) 8m ago 11h 452M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T17:13:00.480 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11h) 28s ago 11h 1108M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T17:13:00.480 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11h) 28s ago 11h 2533M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T17:13:00.481 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (11h) 8m ago 11h 1123M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T17:13:00.481 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (11h) 28s ago 11h 1532M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T17:13:00.481 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 28s ago 11h 24.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T17:13:00.481 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 8m ago 11h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T17:13:00.481 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11h) 28s ago 11h 1855M 2007M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T17:13:00.482 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11h) 28s ago 11h 1840M 2007M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T17:13:00.482 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11h) 28s ago 11h 1429M 2007M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T17:13:00.482 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (11h) 28s ago 11h 1805M 2007M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T17:13:00.482 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (11h) 8m ago 11h 1558M 2475M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T17:13:00.483 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (11h) 8m ago 11h 1959M 2475M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T17:13:00.483 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (11h) 8m ago 11h 1752M 2475M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T17:13:00.483 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (11h) 8m ago 11h 2085M 2475M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T17:13:00.483 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (11h) 8m ago 11h 190M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T17:13:00.835 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T17:13:00.836 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T17:13:00.836 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T17:13:00.836 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:13:00.836 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T17:13:00.837 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T17:13:00.837 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:13:00.837 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T17:13:00.837 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T17:13:00.838 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:13:00.838 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T17:13:00.838 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T17:13:00.838 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:13:00.839 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T17:13:00.839 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T17:13:00.839 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T17:13:00.840 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T17:13:00.840 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T17:13:01.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:13:00 smithi153 ceph-mon[70044]: from='client.58710 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:13:01.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:13:00 smithi153 ceph-mon[70044]: from='client.58716 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:13:01.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:13:00 smithi153 ceph-mon[70044]: pgmap v21139: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:01.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:13:00 smithi083 ceph-mon[79930]: from='client.58710 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:13:01.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:13:00 smithi083 ceph-mon[79930]: from='client.58716 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:13:01.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:13:00 smithi083 ceph-mon[79930]: pgmap v21139: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:01.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:13:00 smithi083 ceph-mon[77242]: from='client.58710 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:13:01.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:13:00 smithi083 ceph-mon[77242]: from='client.58716 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:13:01.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:13:00 smithi083 ceph-mon[77242]: pgmap v21139: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 17:13:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:13:01] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T17:13:01.962 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:13:01 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/2655137' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T17:13:01.963 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:13:01 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/2655137' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T17:13:02.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:13:01 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/2655137' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T17:13:02.756 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:13:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:13:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T17:13:03.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:13:02 smithi153 ceph-mon[70044]: pgmap v21140: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:03.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:13:02 smithi083 ceph-mon[79930]: pgmap v21140: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:03.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:13:02 smithi083 ceph-mon[77242]: pgmap v21140: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:03.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:13:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:13:02.821Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:13:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 17:13:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:13:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T17:13:05.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:13:05 smithi083 ceph-mon[79930]: pgmap v21141: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:05.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:13:05 smithi083 ceph-mon[77242]: pgmap v21141: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:05.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:13:05 smithi153 ceph-mon[70044]: pgmap v21141: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:06.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:13:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:13:06.161Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:13:07.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:13:06 smithi153 ceph-mon[70044]: pgmap v21142: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:07.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:13:06 smithi083 ceph-mon[79930]: pgmap v21142: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:07.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:13:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:13:06.914Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:13:07.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:13:06 smithi083 ceph-mon[77242]: pgmap v21142: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:09.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:13:08 smithi153 ceph-mon[70044]: pgmap v21143: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:09.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:13:08 smithi083 ceph-mon[79930]: pgmap v21143: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:09.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:13:08 smithi083 ceph-mon[77242]: pgmap v21143: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:11.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:13:10 smithi153 ceph-mon[70044]: pgmap v21144: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:11.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:13:10 smithi083 ceph-mon[79930]: pgmap v21144: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:11.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:13:10 smithi083 ceph-mon[77242]: pgmap v21144: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 17:13:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:13:11] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T17:13:12.752 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:13:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:13:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T17:13:13.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:13:12 smithi153 ceph-mon[70044]: pgmap v21145: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:13.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:13:12 smithi083 ceph-mon[79930]: pgmap v21145: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:13.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:13:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:13:12.823Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:13:13.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:13:12 smithi083 ceph-mon[77242]: pgmap v21145: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 17:13:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:13:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T17:13:15.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:13:15 smithi083 ceph-mon[79930]: pgmap v21146: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:15.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:13:15 smithi083 ceph-mon[77242]: pgmap v21146: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:15.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:13:15 smithi153 ceph-mon[70044]: pgmap v21146: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:16.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:13:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:13:16.161Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:13:17.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:13:16 smithi153 ceph-mon[70044]: pgmap v21147: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:17.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:13:16 smithi083 ceph-mon[79930]: pgmap v21147: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:17.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:13:16 smithi083 ceph-mon[77242]: pgmap v21147: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:17.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:13:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:13:16.915Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:13:19.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:13:18 smithi153 ceph-mon[70044]: pgmap v21148: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:19.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:13:18 smithi083 ceph-mon[77242]: pgmap v21148: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:19.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:13:18 smithi083 ceph-mon[79930]: pgmap v21148: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:21.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:13:20 smithi153 ceph-mon[70044]: pgmap v21149: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:21.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:13:20 smithi083 ceph-mon[79930]: pgmap v21149: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:21.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:13:20 smithi083 ceph-mon[77242]: pgmap v21149: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:21.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 17:13:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:13:21] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T17:13:22.753 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:13:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:13:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T17:13:23.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:13:22 smithi153 ceph-mon[70044]: pgmap v21150: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:23.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:13:22 smithi083 ceph-mon[79930]: pgmap v21150: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:23.214 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:13:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:13:22.823Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:13:23.215 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:13:22 smithi083 ceph-mon[77242]: pgmap v21150: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 17:13:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:13:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T17:13:25.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:13:25 smithi083 ceph-mon[77242]: pgmap v21151: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:25.462 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:13:25 smithi083 ceph-mon[79930]: pgmap v21151: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:25.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:13:25 smithi153 ceph-mon[70044]: pgmap v21151: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:26.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:13:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:13:26.162Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:13:27.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:13:26 smithi153 ceph-mon[70044]: pgmap v21152: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:27.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:13:26 smithi083 ceph-mon[79930]: pgmap v21152: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:27.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:13:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:13:26.915Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:13:27.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:13:26 smithi083 ceph-mon[77242]: pgmap v21152: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:29.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:13:28 smithi153 ceph-mon[70044]: pgmap v21153: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:29.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:13:28 smithi083 ceph-mon[77242]: pgmap v21153: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:29.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:13:28 smithi083 ceph-mon[79930]: pgmap v21153: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:31.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:13:30 smithi153 ceph-mon[70044]: pgmap v21154: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:31.163 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T17:13:31.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:13:30 smithi083 ceph-mon[79930]: pgmap v21154: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:31.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:13:30 smithi083 ceph-mon[77242]: pgmap v21154: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:31.450 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T17:13:31.451 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 59s ago 11h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T17:13:31.451 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 9m ago 11h 64.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T17:13:31.451 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 59s ago 11h 26.1M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T17:13:31.451 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 9m ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T17:13:31.452 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (11h) 9m ago 11h 452M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T17:13:31.452 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11h) 59s ago 11h 1108M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T17:13:31.452 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11h) 59s ago 11h 2533M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T17:13:31.453 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (11h) 9m ago 11h 1123M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T17:13:31.453 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (11h) 59s ago 11h 1532M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T17:13:31.453 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 59s ago 11h 24.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T17:13:31.454 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 9m ago 11h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T17:13:31.454 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11h) 59s ago 11h 1855M 2007M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T17:13:31.455 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11h) 59s ago 11h 1840M 2007M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T17:13:31.455 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11h) 59s ago 11h 1429M 2007M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T17:13:31.456 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (11h) 59s ago 11h 1805M 2007M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T17:13:31.456 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (11h) 9m ago 11h 1558M 2475M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T17:13:31.457 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (11h) 9m ago 11h 1959M 2475M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T17:13:31.457 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (11h) 9m ago 11h 1752M 2475M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T17:13:31.457 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (11h) 9m ago 11h 2085M 2475M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T17:13:31.458 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (11h) 9m ago 11h 190M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T17:13:31.807 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T17:13:31.808 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T17:13:31.808 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T17:13:31.808 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:13:31.808 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T17:13:31.809 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T17:13:31.809 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:13:31.809 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T17:13:31.809 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T17:13:31.810 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:13:31.810 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T17:13:31.810 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T17:13:31.810 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:13:31.810 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T17:13:31.811 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T17:13:31.811 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T17:13:31.811 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T17:13:31.811 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T17:13:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 17:13:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:13:31] "GET /metrics HTTP/1.1" 200 225247 "" "Prometheus/2.33.4" 2022-09-29T17:13:31.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:13:31 smithi083 ceph-mon[77242]: from='client.58728 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:13:31.962 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:13:31 smithi083 ceph-mon[77242]: from='client.58734 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:13:31.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:13:31 smithi083 ceph-mon[79930]: from='client.58728 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:13:31.963 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:13:31 smithi083 ceph-mon[79930]: from='client.58734 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:13:32.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:13:31 smithi153 ceph-mon[70044]: from='client.58728 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:13:32.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:13:31 smithi153 ceph-mon[70044]: from='client.58734 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:13:32.760 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:13:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:13:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T17:13:33.026 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:13:32 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3783579534' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T17:13:33.026 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:13:32 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T17:13:33.026 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:13:32 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T17:13:33.026 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:13:32 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T17:13:33.027 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:13:32 smithi153 ceph-mon[70044]: pgmap v21155: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:33.027 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:13:32 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:13:33.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:13:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:13:32.822Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:13:33.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:13:32 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3783579534' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T17:13:33.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:13:32 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T17:13:33.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:13:32 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T17:13:33.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:13:32 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T17:13:33.215 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:13:32 smithi083 ceph-mon[79930]: pgmap v21155: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:33.216 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:13:32 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:13:33.216 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:13:32 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3783579534' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T17:13:33.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:13:32 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T17:13:33.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:13:32 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T17:13:33.217 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:13:32 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T17:13:33.218 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:13:32 smithi083 ceph-mon[77242]: pgmap v21155: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:33.218 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:13:32 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' 2022-09-29T17:13:33.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 17:13:33 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:13:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T17:13:35.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:13:35 smithi083 ceph-mon[79930]: pgmap v21156: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:35.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:13:35 smithi083 ceph-mon[77242]: pgmap v21156: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:35.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:13:35 smithi153 ceph-mon[70044]: pgmap v21156: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:36.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:13:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:13:36.162Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:13:37.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:13:36 smithi153 ceph-mon[70044]: pgmap v21157: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:37.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:13:36 smithi083 ceph-mon[77242]: pgmap v21157: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:37.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:13:36 smithi083 ceph-mon[79930]: pgmap v21157: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:37.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:13:36 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:13:36.916Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:13:39.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:13:38 smithi153 ceph-mon[70044]: pgmap v21158: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:39.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:13:38 smithi083 ceph-mon[79930]: pgmap v21158: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:39.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:13:38 smithi083 ceph-mon[77242]: pgmap v21158: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:41.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:13:40 smithi153 ceph-mon[70044]: pgmap v21159: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:41.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:13:40 smithi083 ceph-mon[79930]: pgmap v21159: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:41.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:13:40 smithi083 ceph-mon[77242]: pgmap v21159: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 17:13:41 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:13:41] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T17:13:42.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:13:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T17:13:42.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:13:42 smithi083 ceph-mon[77242]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T17:13:42.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:13:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T17:13:42.463 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:13:42 smithi083 ceph-mon[77242]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T17:13:42.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:13:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T17:13:42.464 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:13:42 smithi083 ceph-mon[79930]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T17:13:42.465 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:13:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T17:13:42.466 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:13:42 smithi083 ceph-mon[79930]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T17:13:42.557 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:13:42 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:13:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T17:13:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:13:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T17:13:42.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:13:42 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T17:13:42.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:13:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-09-29T17:13:42.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:13:42 smithi153 ceph-mon[70044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-09-29T17:13:43.166 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:13:42 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:13:42.823Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:13:43.307 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:13:43 smithi153 ceph-mon[70044]: pgmap v21160: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:43.308 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 17:13:43 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:13:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T17:13:43.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:13:43 smithi083 ceph-mon[79930]: pgmap v21160: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:43.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:13:43 smithi083 ceph-mon[77242]: pgmap v21160: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:45.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:13:45 smithi083 ceph-mon[79930]: pgmap v21161: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:45.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:13:45 smithi083 ceph-mon[77242]: pgmap v21161: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:45.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:13:45 smithi153 ceph-mon[70044]: pgmap v21161: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:46.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:13:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:13:46.163Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:13:47.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:13:46 smithi153 ceph-mon[70044]: pgmap v21162: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:47.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:13:46 smithi083 ceph-mon[79930]: pgmap v21162: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:47.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:13:46 smithi083 ceph-mon[77242]: pgmap v21162: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:47.214 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:13:46 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:13:46.917Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:13:49.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:13:48 smithi153 ceph-mon[70044]: pgmap v21163: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:49.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:13:48 smithi083 ceph-mon[77242]: pgmap v21163: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:49.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:13:48 smithi083 ceph-mon[79930]: pgmap v21163: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:51.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:13:50 smithi153 ceph-mon[70044]: pgmap v21164: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:51.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:13:50 smithi083 ceph-mon[79930]: pgmap v21164: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:51.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:13:50 smithi083 ceph-mon[77242]: pgmap v21164: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:51.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 17:13:51 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:13:51] "GET /metrics HTTP/1.1" 200 225245 "" "Prometheus/2.33.4" 2022-09-29T17:13:52.759 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:13:52 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:13:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T17:13:53.028 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:13:52 smithi153 ceph-mon[70044]: pgmap v21165: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:53.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:13:52 smithi083 ceph-mon[79930]: pgmap v21165: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:53.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:13:52 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:13:52.824Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:13:53.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:13:52 smithi083 ceph-mon[77242]: pgmap v21165: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:53.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 17:13:53 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:13:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T17:13:55.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:13:55 smithi083 ceph-mon[79930]: pgmap v21166: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:55.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:13:55 smithi083 ceph-mon[77242]: pgmap v21166: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:55.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:13:55 smithi153 ceph-mon[70044]: pgmap v21166: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:56.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:13:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:13:56.163Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:13:57.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:13:56 smithi153 ceph-mon[70044]: pgmap v21167: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:57.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:13:56 smithi083 ceph-mon[79930]: pgmap v21167: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:57.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:13:56 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:13:56.917Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:13:57.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:13:56 smithi083 ceph-mon[77242]: pgmap v21167: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:59.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:13:58 smithi153 ceph-mon[70044]: pgmap v21168: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:59.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:13:58 smithi083 ceph-mon[79930]: pgmap v21168: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:13:59.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:13:58 smithi083 ceph-mon[77242]: pgmap v21168: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:14:01.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:14:00 smithi153 ceph-mon[70044]: pgmap v21169: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:14:01.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:14:00 smithi083 ceph-mon[79930]: pgmap v21169: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:14:01.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:14:00 smithi083 ceph-mon[77242]: pgmap v21169: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:14:01.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 17:14:01 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:14:01] "GET /metrics HTTP/1.1" 200 225246 "" "Prometheus/2.33.4" 2022-09-29T17:14:02.136 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T17:14:02.422 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-09-29T17:14:02.422 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (11h) 90s ago 11h 27.0M - 0.23.0 ba2b418f427c 4af0d0fa9b28 2022-09-29T17:14:02.422 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi153 *:3000 running (11h) 9m ago 11h 64.4M - 8.3.5 dad864ee21e9 faa72bb81473 2022-09-29T17:14:02.423 INFO:teuthology.orchestra.run.smithi083.stdout:mds.a smithi083 running (11h) 90s ago 11h 26.1M - 16.2.10 c56e13ff90f2 7741d74b494d 2022-09-29T17:14:02.423 INFO:teuthology.orchestra.run.smithi083.stdout:mds.b smithi153 running (11h) 9m ago 11h 28.3M - 16.2.10 c56e13ff90f2 365a49055b33 2022-09-29T17:14:02.423 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi153 *:8443,9283 running (11h) 9m ago 11h 452M - 18.0.0-128-g652bf754 6d6412b0c71e d539b7e85b69 2022-09-29T17:14:02.424 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:8443,9283 running (11h) 90s ago 11h 1108M - 18.0.0-128-g652bf754 6d6412b0c71e 4ff0215b55e8 2022-09-29T17:14:02.424 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11h) 90s ago 11h 2533M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 5252f1d43407 2022-09-29T17:14:02.424 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi153 running (11h) 9m ago 11h 1123M 2048M 18.0.0-128-g652bf754 6d6412b0c71e f8e8b54cae07 2022-09-29T17:14:02.425 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (11h) 90s ago 11h 1532M 2048M 18.0.0-128-g652bf754 6d6412b0c71e 54044e5b4f35 2022-09-29T17:14:02.425 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (11h) 90s ago 11h 24.1M - 1.3.1 1dbe0e931976 98012608edcd 2022-09-29T17:14:02.425 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi153 *:9100 running (11h) 9m ago 11h 26.5M - 1.3.1 1dbe0e931976 a95f214c24de 2022-09-29T17:14:02.425 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11h) 90s ago 11h 1855M 2007M 18.0.0-128-g652bf754 6d6412b0c71e bf55222fed6d 2022-09-29T17:14:02.426 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11h) 90s ago 11h 1840M 2007M 18.0.0-128-g652bf754 6d6412b0c71e 6e1a78274241 2022-09-29T17:14:02.426 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11h) 90s ago 11h 1429M 2007M 18.0.0-128-g652bf754 6d6412b0c71e 4eebe9e46fcb 2022-09-29T17:14:02.426 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (11h) 90s ago 11h 1805M 2007M 18.0.0-128-g652bf754 6d6412b0c71e f045a5c0be19 2022-09-29T17:14:02.426 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi153 running (11h) 9m ago 11h 1558M 2475M 18.0.0-128-g652bf754 6d6412b0c71e 4642d6bd58ae 2022-09-29T17:14:02.427 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi153 running (11h) 9m ago 11h 1959M 2475M 18.0.0-128-g652bf754 6d6412b0c71e a041d8e19416 2022-09-29T17:14:02.427 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi153 running (11h) 9m ago 11h 1752M 2475M 18.0.0-128-g652bf754 6d6412b0c71e 760baeb754de 2022-09-29T17:14:02.427 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi153 running (11h) 9m ago 11h 2085M 2475M 18.0.0-128-g652bf754 6d6412b0c71e 6b12b9081600 2022-09-29T17:14:02.427 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi153 *:9095 running (11h) 9m ago 11h 190M - 2.33.4 514e6a882f6e 40a54c5a196a 2022-09-29T17:14:02.760 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:14:02 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:14:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T17:14:02.780 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2022-09-29T17:14:02.780 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2022-09-29T17:14:02.780 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 3 2022-09-29T17:14:02.781 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:14:02.781 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2022-09-29T17:14:02.781 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 2 2022-09-29T17:14:02.781 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:14:02.782 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2022-09-29T17:14:02.782 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 8 2022-09-29T17:14:02.782 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:14:02.783 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2022-09-29T17:14:02.783 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2 2022-09-29T17:14:02.783 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2022-09-29T17:14:02.783 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2022-09-29T17:14:02.784 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)": 2, 2022-09-29T17:14:02.784 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 18.0.0-128-g652bf754 (652bf75409c45c34ae939b241b00adba20f12869) quincy (dev)": 13 2022-09-29T17:14:02.784 INFO:teuthology.orchestra.run.smithi083.stdout: } 2022-09-29T17:14:02.784 INFO:teuthology.orchestra.run.smithi083.stdout:} 2022-09-29T17:14:03.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:14:02 smithi153 ceph-mon[70044]: from='client.58746 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:14:03.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:14:02 smithi153 ceph-mon[70044]: from='client.68683 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:14:03.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:14:02 smithi153 ceph-mon[70044]: pgmap v21170: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:14:03.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:14:02 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:14:02.823Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:14:03.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:14:02 smithi083 ceph-mon[77242]: from='client.58746 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:14:03.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:14:02 smithi083 ceph-mon[77242]: from='client.68683 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:14:03.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:14:02 smithi083 ceph-mon[77242]: pgmap v21170: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:14:03.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:14:02 smithi083 ceph-mon[79930]: from='client.58746 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:14:03.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:14:02 smithi083 ceph-mon[79930]: from='client.68683 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-09-29T17:14:03.213 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:14:02 smithi083 ceph-mon[79930]: pgmap v21170: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:14:03.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 17:14:03 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:14:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T17:14:04.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:14:03 smithi153 ceph-mon[70044]: from='client.? 172.21.15.83:0/3964060304' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T17:14:04.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:14:03 smithi083 ceph-mon[79930]: from='client.? 172.21.15.83:0/3964060304' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T17:14:04.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:14:03 smithi083 ceph-mon[77242]: from='client.? 172.21.15.83:0/3964060304' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-09-29T17:14:05.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:14:05 smithi083 ceph-mon[79930]: pgmap v21171: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:14:05.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:14:05 smithi083 ceph-mon[77242]: pgmap v21171: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:14:05.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:14:05 smithi153 ceph-mon[70044]: pgmap v21171: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:14:06.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:14:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:14:06.163Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:14:07.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:14:06 smithi153 ceph-mon[70044]: pgmap v21172: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:14:07.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:14:06 smithi083 ceph-mon[77242]: pgmap v21172: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:14:07.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:14:06 smithi083 ceph-mon[79930]: pgmap v21172: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:14:07.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:14:06 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:14:06.918Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:14:09.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:14:08 smithi153 ceph-mon[70044]: pgmap v21173: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:14:09.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:14:08 smithi083 ceph-mon[79930]: pgmap v21173: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:14:09.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:14:08 smithi083 ceph-mon[77242]: pgmap v21173: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:14:11.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:14:10 smithi153 ceph-mon[70044]: pgmap v21174: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:14:11.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:14:10 smithi083 ceph-mon[79930]: pgmap v21174: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:14:11.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:14:10 smithi083 ceph-mon[77242]: pgmap v21174: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:14:11.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 17:14:11 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:14:11] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T17:14:12.762 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:14:12 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:14:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T17:14:13.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:14:12 smithi153 ceph-mon[70044]: pgmap v21175: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:14:13.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:14:12 smithi083 ceph-mon[77242]: pgmap v21175: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:14:13.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:14:12 smithi083 ceph-mon[79930]: pgmap v21175: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:14:13.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:14:12 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:14:12.824Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:14:13.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 17:14:13 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:14:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T17:14:15.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:14:15 smithi083 ceph-mon[79930]: pgmap v21176: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:14:15.461 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:14:15 smithi083 ceph-mon[77242]: pgmap v21176: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:14:15.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:14:15 smithi153 ceph-mon[70044]: pgmap v21176: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:14:16.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:14:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:14:16.164Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:14:17.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:14:16 smithi153 ceph-mon[70044]: pgmap v21177: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:14:17.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:14:16 smithi083 ceph-mon[79930]: pgmap v21177: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:14:17.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:14:16 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:14:16.918Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:14:17.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:14:16 smithi083 ceph-mon[77242]: pgmap v21177: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:14:19.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:14:18 smithi153 ceph-mon[70044]: pgmap v21178: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:14:19.211 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:14:18 smithi083 ceph-mon[77242]: pgmap v21178: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:14:19.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:14:18 smithi083 ceph-mon[79930]: pgmap v21178: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:14:21.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:14:20 smithi153 ceph-mon[70044]: pgmap v21179: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:14:21.212 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:14:20 smithi083 ceph-mon[79930]: pgmap v21179: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:14:21.213 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:14:20 smithi083 ceph-mon[77242]: pgmap v21179: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:14:21.962 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 17:14:21 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:14:21] "GET /metrics HTTP/1.1" 200 225243 "" "Prometheus/2.33.4" 2022-09-29T17:14:22.766 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:14:22 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:14:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T17:14:23.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:14:22 smithi153 ceph-mon[70044]: pgmap v21180: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:14:23.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:14:22 smithi083 ceph-mon[79930]: pgmap v21180: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:14:23.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:14:22 smithi083 ceph-mon[77242]: pgmap v21180: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:14:23.213 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:14:22 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:14:22.825Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:14:23.307 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Sep 29 17:14:23 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-x[65933]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:14:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2022-09-29T17:14:25.461 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:14:25 smithi083 ceph-mon[79930]: pgmap v21181: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:14:25.462 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:14:25 smithi083 ceph-mon[77242]: pgmap v21181: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:14:25.557 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:14:25 smithi153 ceph-mon[70044]: pgmap v21181: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:14:26.461 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:14:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:14:26.165Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:14:27.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:14:26 smithi153 ceph-mon[70044]: pgmap v21182: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:14:27.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:14:26 smithi083 ceph-mon[79930]: pgmap v21182: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:14:27.212 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:14:26 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:14:26.919Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:14:27.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:14:26 smithi083 ceph-mon[77242]: pgmap v21182: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:14:29.057 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:14:28 smithi153 ceph-mon[70044]: pgmap v21183: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:14:29.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:14:28 smithi083 ceph-mon[79930]: pgmap v21183: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:14:29.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:14:28 smithi083 ceph-mon[77242]: pgmap v21183: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:14:31.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:14:30 smithi153 ceph-mon[70044]: pgmap v21184: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:14:31.211 INFO:journalctl@ceph.mon.c.smithi083.stdout:Sep 29 17:14:30 smithi083 ceph-mon[79930]: pgmap v21184: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:14:31.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:14:30 smithi083 ceph-mon[77242]: pgmap v21184: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:14:31.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Sep 29 17:14:31 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-mgr-y[73067]: ::ffff:172.21.15.153 - - [29/Sep/2022:17:14:31] "GET /metrics HTTP/1.1" 200 225242 "" "Prometheus/2.33.4" 2022-09-29T17:14:32.767 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Sep 29 17:14:32 smithi153 ceph-69111522-3fb6-11ed-8431-001a4aab830c-prometheus-a[65374]: ts=2022-09-29T17:14:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi083:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.83:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2022-09-29T17:14:33.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:14:32 smithi153 ceph-mon[70044]: pgmap v21185: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:14:33.024 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:14:32 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-09-29T17:14:33.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:14:32 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-09-29T17:14:33.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Sep 29 17:14:32 smithi153 ceph-mon[70044]: from='mgr.34107 172.21.15.83:0/1341710697' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-09-29T17:14:33.114 INFO:teuthology.orchestra.run.smithi083.stdout:true 2022-09-29T17:14:33.211 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Sep 29 17:14:32 smithi083 ceph-69111522-3fb6-11ed-8431-001a4aab830c-alertmanager-a[67247]: level=error ts=2022-09-29T17:14:32.826Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.83:8443//api/prometheus_receiver" 2022-09-29T17:14:33.212 INFO:journalctl@ceph.mon.a.smithi083.stdout:Sep 29 17:14:32 smithi083 ceph-mon[77242]: pgmap v21185: 105 pgs: 105 active+clean; 583 KiB data, 70 MiB used, 715 GiB / 715 GiB avail 2022-09-29T17:14:33.230 DEBUG:teuthology.exit:Got signal 15; running 1 handler... 2022-09-29T17:14:33.257 DEBUG:teuthology.task.console_log:Killing console logger for smithi083 2022-09-29T17:14:33.385 DEBUG:teuthology.task.console_log:Killing console logger for smithi153 2022-09-29T17:14:33.387 DEBUG:teuthology.exit:Finished running handlers